1//===-- examples/ParallelJIT/ParallelJIT.cpp - Exercise threaded-safe JIT -===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Parallel JIT
11//
12// This test program creates two LLVM functions then calls them from three
13// separate threads.  It requires the pthreads library.
14// The three threads are created and then block waiting on a condition variable.
15// Once all threads are blocked on the conditional variable, the main thread
16// wakes them up. This complicated work is performed so that all three threads
17// call into the JIT at the same time (or the best possible approximation of the
18// same time). This test had assertion errors until I got the locking right.
19
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ExecutionEngine/GenericValue.h"
22#include "llvm/ExecutionEngine/Interpreter.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/Instructions.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/IR/Module.h"
28#include "llvm/Support/TargetSelect.h"
29#include <iostream>
30#include <pthread.h>
31
32using namespace llvm;
33
34static Function* createAdd1(Module *M) {
35  // Create the add1 function entry and insert this entry into module M.  The
36  // function will have a return type of "int" and take an argument of "int".
37  // The '0' terminates the list of argument types.
38  Function *Add1F =
39    cast<Function>(M->getOrInsertFunction("add1",
40                                          Type::getInt32Ty(M->getContext()),
41                                          Type::getInt32Ty(M->getContext()),
42                                          nullptr));
43
44  // Add a basic block to the function. As before, it automatically inserts
45  // because of the last argument.
46  BasicBlock *BB = BasicBlock::Create(M->getContext(), "EntryBlock", Add1F);
47
48  // Get pointers to the constant `1'.
49  Value *One = ConstantInt::get(Type::getInt32Ty(M->getContext()), 1);
50
51  // Get pointers to the integer argument of the add1 function...
52  assert(Add1F->arg_begin() != Add1F->arg_end()); // Make sure there's an arg
53  Argument *ArgX = &*Add1F->arg_begin();          // Get the arg
54  ArgX->setName("AnArg");            // Give it a nice symbolic name for fun.
55
56  // Create the add instruction, inserting it into the end of BB.
57  Instruction *Add = BinaryOperator::CreateAdd(One, ArgX, "addresult", BB);
58
59  // Create the return instruction and add it to the basic block
60  ReturnInst::Create(M->getContext(), Add, BB);
61
62  // Now, function add1 is ready.
63  return Add1F;
64}
65
66static Function *CreateFibFunction(Module *M) {
67  // Create the fib function and insert it into module M.  This function is said
68  // to return an int and take an int parameter.
69  Function *FibF =
70    cast<Function>(M->getOrInsertFunction("fib",
71                                          Type::getInt32Ty(M->getContext()),
72                                          Type::getInt32Ty(M->getContext()),
73                                          nullptr));
74
75  // Add a basic block to the function.
76  BasicBlock *BB = BasicBlock::Create(M->getContext(), "EntryBlock", FibF);
77
78  // Get pointers to the constants.
79  Value *One = ConstantInt::get(Type::getInt32Ty(M->getContext()), 1);
80  Value *Two = ConstantInt::get(Type::getInt32Ty(M->getContext()), 2);
81
82  // Get pointer to the integer argument of the add1 function...
83  Argument *ArgX = &*FibF->arg_begin(); // Get the arg.
84  ArgX->setName("AnArg");            // Give it a nice symbolic name for fun.
85
86  // Create the true_block.
87  BasicBlock *RetBB = BasicBlock::Create(M->getContext(), "return", FibF);
88  // Create an exit block.
89  BasicBlock* RecurseBB = BasicBlock::Create(M->getContext(), "recurse", FibF);
90
91  // Create the "if (arg < 2) goto exitbb"
92  Value *CondInst = new ICmpInst(*BB, ICmpInst::ICMP_SLE, ArgX, Two, "cond");
93  BranchInst::Create(RetBB, RecurseBB, CondInst, BB);
94
95  // Create: ret int 1
96  ReturnInst::Create(M->getContext(), One, RetBB);
97
98  // create fib(x-1)
99  Value *Sub = BinaryOperator::CreateSub(ArgX, One, "arg", RecurseBB);
100  Value *CallFibX1 = CallInst::Create(FibF, Sub, "fibx1", RecurseBB);
101
102  // create fib(x-2)
103  Sub = BinaryOperator::CreateSub(ArgX, Two, "arg", RecurseBB);
104  Value *CallFibX2 = CallInst::Create(FibF, Sub, "fibx2", RecurseBB);
105
106  // fib(x-1)+fib(x-2)
107  Value *Sum =
108    BinaryOperator::CreateAdd(CallFibX1, CallFibX2, "addresult", RecurseBB);
109
110  // Create the return instruction and add it to the basic block
111  ReturnInst::Create(M->getContext(), Sum, RecurseBB);
112
113  return FibF;
114}
115
116struct threadParams {
117  ExecutionEngine* EE;
118  Function* F;
119  int value;
120};
121
122// We block the subthreads just before they begin to execute:
123// we want all of them to call into the JIT at the same time,
124// to verify that the locking is working correctly.
125class WaitForThreads
126{
127public:
128  WaitForThreads()
129  {
130    n = 0;
131    waitFor = 0;
132
133    int result = pthread_cond_init( &condition, nullptr );
134    assert( result == 0 );
135
136    result = pthread_mutex_init( &mutex, nullptr );
137    assert( result == 0 );
138  }
139
140  ~WaitForThreads()
141  {
142    int result = pthread_cond_destroy( &condition );
143    (void)result;
144    assert( result == 0 );
145
146    result = pthread_mutex_destroy( &mutex );
147    assert( result == 0 );
148  }
149
150  // All threads will stop here until another thread calls releaseThreads
151  void block()
152  {
153    int result = pthread_mutex_lock( &mutex );
154    (void)result;
155    assert( result == 0 );
156    n ++;
157    //~ std::cout << "block() n " << n << " waitFor " << waitFor << std::endl;
158
159    assert( waitFor == 0 || n <= waitFor );
160    if ( waitFor > 0 && n == waitFor )
161    {
162      // There are enough threads blocked that we can release all of them
163      std::cout << "Unblocking threads from block()" << std::endl;
164      unblockThreads();
165    }
166    else
167    {
168      // We just need to wait until someone unblocks us
169      result = pthread_cond_wait( &condition, &mutex );
170      assert( result == 0 );
171    }
172
173    // unlock the mutex before returning
174    result = pthread_mutex_unlock( &mutex );
175    assert( result == 0 );
176  }
177
178  // If there are num or more threads blocked, it will signal them all
179  // Otherwise, this thread blocks until there are enough OTHER threads
180  // blocked
181  void releaseThreads( size_t num )
182  {
183    int result = pthread_mutex_lock( &mutex );
184    (void)result;
185    assert( result == 0 );
186
187    if ( n >= num ) {
188      std::cout << "Unblocking threads from releaseThreads()" << std::endl;
189      unblockThreads();
190    }
191    else
192    {
193      waitFor = num;
194      pthread_cond_wait( &condition, &mutex );
195    }
196
197    // unlock the mutex before returning
198    result = pthread_mutex_unlock( &mutex );
199    assert( result == 0 );
200  }
201
202private:
203  void unblockThreads()
204  {
205    // Reset the counters to zero: this way, if any new threads
206    // enter while threads are exiting, they will block instead
207    // of triggering a new release of threads
208    n = 0;
209
210    // Reset waitFor to zero: this way, if waitFor threads enter
211    // while threads are exiting, they will block instead of
212    // triggering a new release of threads
213    waitFor = 0;
214
215    int result = pthread_cond_broadcast( &condition );
216    (void)result;
217    assert(result == 0);
218  }
219
220  size_t n;
221  size_t waitFor;
222  pthread_cond_t condition;
223  pthread_mutex_t mutex;
224};
225
226static WaitForThreads synchronize;
227
228void* callFunc( void* param )
229{
230  struct threadParams* p = (struct threadParams*) param;
231
232  // Call the `foo' function with no arguments:
233  std::vector<GenericValue> Args(1);
234  Args[0].IntVal = APInt(32, p->value);
235
236  synchronize.block(); // wait until other threads are at this point
237  GenericValue gv = p->EE->runFunction(p->F, Args);
238
239  return (void*)(intptr_t)gv.IntVal.getZExtValue();
240}
241
242int main() {
243  InitializeNativeTarget();
244  LLVMContext Context;
245
246  // Create some module to put our function into it.
247  std::unique_ptr<Module> Owner = make_unique<Module>("test", Context);
248  Module *M = Owner.get();
249
250  Function* add1F = createAdd1( M );
251  Function* fibF = CreateFibFunction( M );
252
253  // Now we create the JIT.
254  ExecutionEngine* EE = EngineBuilder(std::move(Owner)).create();
255
256  //~ std::cout << "We just constructed this LLVM module:\n\n" << *M;
257  //~ std::cout << "\n\nRunning foo: " << std::flush;
258
259  // Create one thread for add1 and two threads for fib
260  struct threadParams add1 = { EE, add1F, 1000 };
261  struct threadParams fib1 = { EE, fibF, 39 };
262  struct threadParams fib2 = { EE, fibF, 42 };
263
264  pthread_t add1Thread;
265  int result = pthread_create( &add1Thread, nullptr, callFunc, &add1 );
266  if ( result != 0 ) {
267          std::cerr << "Could not create thread" << std::endl;
268          return 1;
269  }
270
271  pthread_t fibThread1;
272  result = pthread_create( &fibThread1, nullptr, callFunc, &fib1 );
273  if ( result != 0 ) {
274          std::cerr << "Could not create thread" << std::endl;
275          return 1;
276  }
277
278  pthread_t fibThread2;
279  result = pthread_create( &fibThread2, nullptr, callFunc, &fib2 );
280  if ( result != 0 ) {
281          std::cerr << "Could not create thread" << std::endl;
282          return 1;
283  }
284
285  synchronize.releaseThreads(3); // wait until other threads are at this point
286
287  void* returnValue;
288  result = pthread_join( add1Thread, &returnValue );
289  if ( result != 0 ) {
290          std::cerr << "Could not join thread" << std::endl;
291          return 1;
292  }
293  std::cout << "Add1 returned " << intptr_t(returnValue) << std::endl;
294
295  result = pthread_join( fibThread1, &returnValue );
296  if ( result != 0 ) {
297          std::cerr << "Could not join thread" << std::endl;
298          return 1;
299  }
300  std::cout << "Fib1 returned " << intptr_t(returnValue) << std::endl;
301
302  result = pthread_join( fibThread2, &returnValue );
303  if ( result != 0 ) {
304          std::cerr << "Could not join thread" << std::endl;
305          return 1;
306  }
307  std::cout << "Fib2 returned " << intptr_t(returnValue) << std::endl;
308
309  return 0;
310}
311