1//===-- examples/ParallelJIT/ParallelJIT.cpp - Exercise threaded-safe JIT -===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Parallel JIT
11//
12// This test program creates two LLVM functions then calls them from three
13// separate threads.  It requires the pthreads library.
14// The three threads are created and then block waiting on a condition variable.
15// Once all threads are blocked on the conditional variable, the main thread
16// wakes them up. This complicated work is performed so that all three threads
17// call into the JIT at the same time (or the best possible approximation of the
18// same time). This test had assertion errors until I got the locking right.
19//
20//===----------------------------------------------------------------------===//
21
22#include "llvm/ADT/APInt.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ExecutionEngine/ExecutionEngine.h"
25#include "llvm/ExecutionEngine/GenericValue.h"
26#include "llvm/IR/Argument.h"
27#include "llvm/IR/BasicBlock.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/InstrTypes.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/LLVMContext.h"
35#include "llvm/IR/Module.h"
36#include "llvm/IR/Type.h"
37#include "llvm/Support/Casting.h"
38#include "llvm/Support/TargetSelect.h"
39#include <algorithm>
40#include <cassert>
41#include <cstddef>
42#include <cstdint>
43#include <iostream>
44#include <memory>
45#include <vector>
46#include <pthread.h>
47
48using namespace llvm;
49
50static Function* createAdd1(Module *M) {
51  // Create the add1 function entry and insert this entry into module M.  The
52  // function will have a return type of "int" and take an argument of "int".
53  // The '0' terminates the list of argument types.
54  Function *Add1F =
55    cast<Function>(M->getOrInsertFunction("add1",
56                                          Type::getInt32Ty(M->getContext()),
57                                          Type::getInt32Ty(M->getContext()),
58                                          nullptr));
59
60  // Add a basic block to the function. As before, it automatically inserts
61  // because of the last argument.
62  BasicBlock *BB = BasicBlock::Create(M->getContext(), "EntryBlock", Add1F);
63
64  // Get pointers to the constant `1'.
65  Value *One = ConstantInt::get(Type::getInt32Ty(M->getContext()), 1);
66
67  // Get pointers to the integer argument of the add1 function...
68  assert(Add1F->arg_begin() != Add1F->arg_end()); // Make sure there's an arg
69  Argument *ArgX = &*Add1F->arg_begin();          // Get the arg
70  ArgX->setName("AnArg");            // Give it a nice symbolic name for fun.
71
72  // Create the add instruction, inserting it into the end of BB.
73  Instruction *Add = BinaryOperator::CreateAdd(One, ArgX, "addresult", BB);
74
75  // Create the return instruction and add it to the basic block
76  ReturnInst::Create(M->getContext(), Add, BB);
77
78  // Now, function add1 is ready.
79  return Add1F;
80}
81
82static Function *CreateFibFunction(Module *M) {
83  // Create the fib function and insert it into module M.  This function is said
84  // to return an int and take an int parameter.
85  Function *FibF =
86    cast<Function>(M->getOrInsertFunction("fib",
87                                          Type::getInt32Ty(M->getContext()),
88                                          Type::getInt32Ty(M->getContext()),
89                                          nullptr));
90
91  // Add a basic block to the function.
92  BasicBlock *BB = BasicBlock::Create(M->getContext(), "EntryBlock", FibF);
93
94  // Get pointers to the constants.
95  Value *One = ConstantInt::get(Type::getInt32Ty(M->getContext()), 1);
96  Value *Two = ConstantInt::get(Type::getInt32Ty(M->getContext()), 2);
97
98  // Get pointer to the integer argument of the add1 function...
99  Argument *ArgX = &*FibF->arg_begin(); // Get the arg.
100  ArgX->setName("AnArg");            // Give it a nice symbolic name for fun.
101
102  // Create the true_block.
103  BasicBlock *RetBB = BasicBlock::Create(M->getContext(), "return", FibF);
104  // Create an exit block.
105  BasicBlock* RecurseBB = BasicBlock::Create(M->getContext(), "recurse", FibF);
106
107  // Create the "if (arg < 2) goto exitbb"
108  Value *CondInst = new ICmpInst(*BB, ICmpInst::ICMP_SLE, ArgX, Two, "cond");
109  BranchInst::Create(RetBB, RecurseBB, CondInst, BB);
110
111  // Create: ret int 1
112  ReturnInst::Create(M->getContext(), One, RetBB);
113
114  // create fib(x-1)
115  Value *Sub = BinaryOperator::CreateSub(ArgX, One, "arg", RecurseBB);
116  Value *CallFibX1 = CallInst::Create(FibF, Sub, "fibx1", RecurseBB);
117
118  // create fib(x-2)
119  Sub = BinaryOperator::CreateSub(ArgX, Two, "arg", RecurseBB);
120  Value *CallFibX2 = CallInst::Create(FibF, Sub, "fibx2", RecurseBB);
121
122  // fib(x-1)+fib(x-2)
123  Value *Sum =
124    BinaryOperator::CreateAdd(CallFibX1, CallFibX2, "addresult", RecurseBB);
125
126  // Create the return instruction and add it to the basic block
127  ReturnInst::Create(M->getContext(), Sum, RecurseBB);
128
129  return FibF;
130}
131
132struct threadParams {
133  ExecutionEngine* EE;
134  Function* F;
135  int value;
136};
137
138// We block the subthreads just before they begin to execute:
139// we want all of them to call into the JIT at the same time,
140// to verify that the locking is working correctly.
141class WaitForThreads
142{
143public:
144  WaitForThreads()
145  {
146    n = 0;
147    waitFor = 0;
148
149    int result = pthread_cond_init( &condition, nullptr );
150    assert( result == 0 );
151
152    result = pthread_mutex_init( &mutex, nullptr );
153    assert( result == 0 );
154  }
155
156  ~WaitForThreads()
157  {
158    int result = pthread_cond_destroy( &condition );
159    (void)result;
160    assert( result == 0 );
161
162    result = pthread_mutex_destroy( &mutex );
163    assert( result == 0 );
164  }
165
166  // All threads will stop here until another thread calls releaseThreads
167  void block()
168  {
169    int result = pthread_mutex_lock( &mutex );
170    (void)result;
171    assert( result == 0 );
172    n ++;
173    //~ std::cout << "block() n " << n << " waitFor " << waitFor << std::endl;
174
175    assert( waitFor == 0 || n <= waitFor );
176    if ( waitFor > 0 && n == waitFor )
177    {
178      // There are enough threads blocked that we can release all of them
179      std::cout << "Unblocking threads from block()" << std::endl;
180      unblockThreads();
181    }
182    else
183    {
184      // We just need to wait until someone unblocks us
185      result = pthread_cond_wait( &condition, &mutex );
186      assert( result == 0 );
187    }
188
189    // unlock the mutex before returning
190    result = pthread_mutex_unlock( &mutex );
191    assert( result == 0 );
192  }
193
194  // If there are num or more threads blocked, it will signal them all
195  // Otherwise, this thread blocks until there are enough OTHER threads
196  // blocked
197  void releaseThreads( size_t num )
198  {
199    int result = pthread_mutex_lock( &mutex );
200    (void)result;
201    assert( result == 0 );
202
203    if ( n >= num ) {
204      std::cout << "Unblocking threads from releaseThreads()" << std::endl;
205      unblockThreads();
206    }
207    else
208    {
209      waitFor = num;
210      pthread_cond_wait( &condition, &mutex );
211    }
212
213    // unlock the mutex before returning
214    result = pthread_mutex_unlock( &mutex );
215    assert( result == 0 );
216  }
217
218private:
219  void unblockThreads()
220  {
221    // Reset the counters to zero: this way, if any new threads
222    // enter while threads are exiting, they will block instead
223    // of triggering a new release of threads
224    n = 0;
225
226    // Reset waitFor to zero: this way, if waitFor threads enter
227    // while threads are exiting, they will block instead of
228    // triggering a new release of threads
229    waitFor = 0;
230
231    int result = pthread_cond_broadcast( &condition );
232    (void)result;
233    assert(result == 0);
234  }
235
236  size_t n;
237  size_t waitFor;
238  pthread_cond_t condition;
239  pthread_mutex_t mutex;
240};
241
242static WaitForThreads synchronize;
243
244void* callFunc( void* param )
245{
246  struct threadParams* p = (struct threadParams*) param;
247
248  // Call the `foo' function with no arguments:
249  std::vector<GenericValue> Args(1);
250  Args[0].IntVal = APInt(32, p->value);
251
252  synchronize.block(); // wait until other threads are at this point
253  GenericValue gv = p->EE->runFunction(p->F, Args);
254
255  return (void*)(intptr_t)gv.IntVal.getZExtValue();
256}
257
258int main() {
259  InitializeNativeTarget();
260  LLVMContext Context;
261
262  // Create some module to put our function into it.
263  std::unique_ptr<Module> Owner = make_unique<Module>("test", Context);
264  Module *M = Owner.get();
265
266  Function* add1F = createAdd1( M );
267  Function* fibF = CreateFibFunction( M );
268
269  // Now we create the JIT.
270  ExecutionEngine* EE = EngineBuilder(std::move(Owner)).create();
271
272  //~ std::cout << "We just constructed this LLVM module:\n\n" << *M;
273  //~ std::cout << "\n\nRunning foo: " << std::flush;
274
275  // Create one thread for add1 and two threads for fib
276  struct threadParams add1 = { EE, add1F, 1000 };
277  struct threadParams fib1 = { EE, fibF, 39 };
278  struct threadParams fib2 = { EE, fibF, 42 };
279
280  pthread_t add1Thread;
281  int result = pthread_create( &add1Thread, nullptr, callFunc, &add1 );
282  if ( result != 0 ) {
283          std::cerr << "Could not create thread" << std::endl;
284          return 1;
285  }
286
287  pthread_t fibThread1;
288  result = pthread_create( &fibThread1, nullptr, callFunc, &fib1 );
289  if ( result != 0 ) {
290          std::cerr << "Could not create thread" << std::endl;
291          return 1;
292  }
293
294  pthread_t fibThread2;
295  result = pthread_create( &fibThread2, nullptr, callFunc, &fib2 );
296  if ( result != 0 ) {
297          std::cerr << "Could not create thread" << std::endl;
298          return 1;
299  }
300
301  synchronize.releaseThreads(3); // wait until other threads are at this point
302
303  void* returnValue;
304  result = pthread_join( add1Thread, &returnValue );
305  if ( result != 0 ) {
306          std::cerr << "Could not join thread" << std::endl;
307          return 1;
308  }
309  std::cout << "Add1 returned " << intptr_t(returnValue) << std::endl;
310
311  result = pthread_join( fibThread1, &returnValue );
312  if ( result != 0 ) {
313          std::cerr << "Could not join thread" << std::endl;
314          return 1;
315  }
316  std::cout << "Fib1 returned " << intptr_t(returnValue) << std::endl;
317
318  result = pthread_join( fibThread2, &returnValue );
319  if ( result != 0 ) {
320          std::cerr << "Could not join thread" << std::endl;
321          return 1;
322  }
323  std::cout << "Fib2 returned " << intptr_t(returnValue) << std::endl;
324
325  return 0;
326}
327