draw_llvm.c revision 1570760a9756234978ff56a9db4c781a630fe6ce 
1#include "draw_llvm.h"
2
3#include "draw_context.h"
4#include "draw_vs.h"
5
6#include "gallivm/lp_bld_arit.h"
7#include "gallivm/lp_bld_interp.h"
8#include "gallivm/lp_bld_struct.h"
9#include "gallivm/lp_bld_type.h"
10#include "gallivm/lp_bld_flow.h"
11#include "gallivm/lp_bld_debug.h"
12#include "gallivm/lp_bld_tgsi.h"
13#include "gallivm/lp_bld_printf.h"
14
15#include "util/u_cpu_detect.h"
16#include "tgsi/tgsi_dump.h"
17
18#include <llvm-c/Transforms/Scalar.h>
19
20#define DEBUG_STORE 0
21
22
23/* generates the draw jit function */
24static void
25draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
26
27static void
28init_globals(struct draw_llvm *llvm)
29{
30   LLVMTypeRef texture_type;
31
32   /* struct draw_jit_texture */
33   {
34      LLVMTypeRef elem_types[4];
35
36      elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
37      elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
38      elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
39      elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
40
41      texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
42
43      LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
44                             llvm->target, texture_type,
45                             DRAW_JIT_TEXTURE_WIDTH);
46      LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
47                             llvm->target, texture_type,
48                             DRAW_JIT_TEXTURE_HEIGHT);
49      LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
50                             llvm->target, texture_type,
51                             DRAW_JIT_TEXTURE_STRIDE);
52      LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
53                             llvm->target, texture_type,
54                             DRAW_JIT_TEXTURE_DATA);
55      LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
56                           llvm->target, texture_type);
57
58      LLVMAddTypeName(llvm->module, "texture", texture_type);
59   }
60
61
62   /* struct draw_jit_context */
63   {
64      LLVMTypeRef elem_types[3];
65      LLVMTypeRef context_type;
66
67      elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
68      elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
69      elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
70
71      context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
72
73      LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
74                             llvm->target, context_type, 0);
75      LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
76                             llvm->target, context_type, 1);
77      LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
78                             llvm->target, context_type,
79                             DRAW_JIT_CONTEXT_TEXTURES_INDEX);
80      LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
81                           llvm->target, context_type);
82
83      LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
84
85      llvm->context_ptr_type = LLVMPointerType(context_type, 0);
86   }
87   {
88      LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
89      llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
90   }
91}
92
93static LLVMTypeRef
94create_vertex_header(struct draw_llvm *llvm, int data_elems)
95{
96   /* struct vertex_header */
97   LLVMTypeRef elem_types[3];
98   LLVMTypeRef vertex_header;
99   char struct_name[24];
100
101   snprintf(struct_name, 23, "vertex_header%d", data_elems);
102
103   elem_types[0]  = LLVMIntType(32);
104   elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
105   elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
106
107   vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
108
109   /* these are bit-fields and we can't take address of them
110      LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
111      llvm->target, vertex_header,
112      DRAW_JIT_VERTEX_CLIPMASK);
113      LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
114      llvm->target, vertex_header,
115      DRAW_JIT_VERTEX_EDGEFLAG);
116      LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
117      llvm->target, vertex_header,
118      DRAW_JIT_VERTEX_PAD);
119      LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
120      llvm->target, vertex_header,
121      DRAW_JIT_VERTEX_VERTEX_ID);
122   */
123   LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
124                          llvm->target, vertex_header,
125                          DRAW_JIT_VERTEX_CLIP);
126   LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
127                          llvm->target, vertex_header,
128                          DRAW_JIT_VERTEX_DATA);
129
130   LLVMAddTypeName(llvm->module, struct_name, vertex_header);
131
132   return LLVMPointerType(vertex_header, 0);
133}
134
135struct draw_llvm *
136draw_llvm_create(struct draw_context *draw)
137{
138   struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm );
139
140   util_cpu_detect();
141
142   llvm->draw = draw;
143   llvm->engine = draw->engine;
144
145   debug_assert(llvm->engine);
146
147   llvm->module = LLVMModuleCreateWithName("draw_llvm");
148   llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
149
150   LLVMAddModuleProvider(llvm->engine, llvm->provider);
151
152   llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
153
154   llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
155   LLVMAddTargetData(llvm->target, llvm->pass);
156   /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
157    * but there are more on SVN. */
158   /* TODO: Add more passes */
159   LLVMAddConstantPropagationPass(llvm->pass);
160   if(util_cpu_caps.has_sse4_1) {
161      /* FIXME: There is a bug in this pass, whereby the combination of fptosi
162       * and sitofp (necessary for trunc/floor/ceil/round implementation)
163       * somehow becomes invalid code.
164       */
165      LLVMAddInstructionCombiningPass(llvm->pass);
166   }
167   LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
168   LLVMAddGVNPass(llvm->pass);
169   LLVMAddCFGSimplificationPass(llvm->pass);
170
171   init_globals(llvm);
172
173
174#if 1
175   LLVMDumpModule(llvm->module);
176#endif
177
178   return llvm;
179}
180
181void
182draw_llvm_destroy(struct draw_llvm *llvm)
183{
184   free(llvm);
185}
186
187struct draw_llvm_variant *
188draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
189{
190   struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
191
192   draw_llvm_make_variant_key(llvm, &variant->key);
193
194   llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
195
196   draw_llvm_generate(llvm, variant);
197
198   return variant;
199}
200
201
202struct draw_context *draw_create_with_llvm(LLVMExecutionEngineRef engine)
203{
204   struct draw_context *draw = CALLOC_STRUCT( draw_context );
205   if (draw == NULL)
206      goto fail;
207   draw->engine = engine;
208
209   if (!draw_init(draw))
210      goto fail;
211
212   return draw;
213
214fail:
215   draw_destroy( draw );
216   return NULL;
217}
218
219static void
220generate_vs(struct draw_llvm *llvm,
221            LLVMBuilderRef builder,
222            LLVMValueRef (*outputs)[NUM_CHANNELS],
223            const LLVMValueRef (*inputs)[NUM_CHANNELS],
224            LLVMValueRef context_ptr)
225{
226   const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
227   struct lp_type vs_type;
228   LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
229
230   memset(&vs_type, 0, sizeof vs_type);
231   vs_type.floating = TRUE; /* floating point values */
232   vs_type.sign = TRUE;     /* values are signed */
233   vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
234   vs_type.width = 32;      /* 32-bit float */
235   vs_type.length = 4;      /* 4 elements per vector */
236#if 0
237   num_vs = 4;              /* number of vertices per block */
238#endif
239
240   tgsi_dump(tokens, 0);
241   lp_build_tgsi_soa(builder,
242                     tokens,
243                     vs_type,
244                     NULL /*struct lp_build_mask_context *mask*/,
245                     consts_ptr,
246                     NULL /*pos*/,
247                     inputs,
248                     outputs,
249                     NULL/*sampler*/);
250}
251
252#if DEBUG_STORE
253static void print_vectorf(LLVMBuilderRef builder,
254                         LLVMValueRef vec)
255{
256   LLVMValueRef val[4];
257   val[0] = LLVMBuildExtractElement(builder, vec,
258                                    LLVMConstInt(LLVMInt32Type(), 0, 0), "");
259   val[1] = LLVMBuildExtractElement(builder, vec,
260                                    LLVMConstInt(LLVMInt32Type(), 1, 0), "");
261   val[2] = LLVMBuildExtractElement(builder, vec,
262                                    LLVMConstInt(LLVMInt32Type(), 2, 0), "");
263   val[3] = LLVMBuildExtractElement(builder, vec,
264                                    LLVMConstInt(LLVMInt32Type(), 3, 0), "");
265   lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
266                   val[0], val[1], val[2], val[3]);
267}
268#endif
269
270static void
271generate_fetch(LLVMBuilderRef builder,
272               LLVMValueRef vbuffers_ptr,
273               LLVMValueRef *res,
274               struct pipe_vertex_element *velem,
275               struct pipe_vertex_buffer *vbuf,
276               LLVMValueRef index)
277{
278   LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
279   LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
280                                           &indices, 1, "");
281   LLVMValueRef stride = LLVMBuildMul(builder,
282                                      LLVMConstInt(LLVMInt32Type(), vbuf->stride, 0),
283                                      index, "");
284
285   vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
286
287   stride = LLVMBuildAdd(builder, stride,
288                         LLVMConstInt(LLVMInt32Type(), vbuf->buffer_offset, 0),
289                         "");
290   stride = LLVMBuildAdd(builder, stride,
291                         LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
292                         "");
293
294   /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
295   vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
296
297   *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
298}
299
300static LLVMValueRef
301aos_to_soa(LLVMBuilderRef builder,
302           LLVMValueRef val0,
303           LLVMValueRef val1,
304           LLVMValueRef val2,
305           LLVMValueRef val3,
306           LLVMValueRef channel)
307{
308   LLVMValueRef ex, res;
309
310   ex = LLVMBuildExtractElement(builder, val0,
311                                channel, "");
312   res = LLVMBuildInsertElement(builder,
313                                LLVMConstNull(LLVMTypeOf(val0)),
314                                ex,
315                                LLVMConstInt(LLVMInt32Type(), 0, 0),
316                                "");
317
318   ex = LLVMBuildExtractElement(builder, val1,
319                                channel, "");
320   res = LLVMBuildInsertElement(builder,
321                                res, ex,
322                                LLVMConstInt(LLVMInt32Type(), 1, 0),
323                                "");
324
325   ex = LLVMBuildExtractElement(builder, val2,
326                                channel, "");
327   res = LLVMBuildInsertElement(builder,
328                                res, ex,
329                                LLVMConstInt(LLVMInt32Type(), 2, 0),
330                                "");
331
332   ex = LLVMBuildExtractElement(builder, val3,
333                                channel, "");
334   res = LLVMBuildInsertElement(builder,
335                                res, ex,
336                                LLVMConstInt(LLVMInt32Type(), 3, 0),
337                                "");
338
339   return res;
340}
341
342static void
343soa_to_aos(LLVMBuilderRef builder,
344           LLVMValueRef soa[NUM_CHANNELS],
345           LLVMValueRef aos[NUM_CHANNELS])
346{
347   LLVMValueRef comp;
348   int i = 0;
349
350   debug_assert(NUM_CHANNELS == 4);
351
352   aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
353   aos[1] = aos[2] = aos[3] = aos[0];
354
355   for (i = 0; i < NUM_CHANNELS; ++i) {
356      LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
357
358      comp = LLVMBuildExtractElement(builder, soa[i],
359                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
360      aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
361
362      comp = LLVMBuildExtractElement(builder, soa[i],
363                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
364      aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
365
366      comp = LLVMBuildExtractElement(builder, soa[i],
367                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
368      aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
369
370      comp = LLVMBuildExtractElement(builder, soa[i],
371                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
372      aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
373
374   }
375}
376
377static void
378convert_to_soa(LLVMBuilderRef builder,
379               LLVMValueRef (*aos)[NUM_CHANNELS],
380               LLVMValueRef (*soa)[NUM_CHANNELS],
381               int num_attribs)
382{
383   int i;
384
385   debug_assert(NUM_CHANNELS == 4);
386
387   for (i = 0; i < num_attribs; ++i) {
388      LLVMValueRef val0 = aos[i][0];
389      LLVMValueRef val1 = aos[i][1];
390      LLVMValueRef val2 = aos[i][2];
391      LLVMValueRef val3 = aos[i][3];
392
393      soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
394                             LLVMConstInt(LLVMInt32Type(), 0, 0));
395      soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
396                             LLVMConstInt(LLVMInt32Type(), 1, 0));
397      soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
398                             LLVMConstInt(LLVMInt32Type(), 2, 0));
399      soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
400                             LLVMConstInt(LLVMInt32Type(), 3, 0));
401   }
402}
403
404static void
405store_aos(LLVMBuilderRef builder,
406          LLVMValueRef io_ptr,
407          LLVMValueRef index,
408          LLVMValueRef value)
409{
410   LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
411   LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
412   LLVMValueRef indices[3];
413
414   indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
415   indices[1] = index;
416   indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
417
418   /* undefined vertex */
419   LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
420                                        0xffff, 0), id_ptr);
421
422#if DEBUG_STORE
423   lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
424#endif
425#if 0
426   /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
427     print_vectorf(builder, value);*/
428   data_ptr = LLVMBuildBitCast(builder, data_ptr,
429                               LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
430                               "datavec");
431   data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
432
433   LLVMBuildStore(builder, value, data_ptr);
434#else
435   {
436      LLVMValueRef x, y, z, w;
437      LLVMValueRef idx0, idx1, idx2, idx3;
438      LLVMValueRef gep0, gep1, gep2, gep3;
439      data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
440
441      idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
442      idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
443      idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
444      idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
445
446      x = LLVMBuildExtractElement(builder, value,
447                                  idx0, "");
448      y = LLVMBuildExtractElement(builder, value,
449                                  idx1, "");
450      z = LLVMBuildExtractElement(builder, value,
451                                  idx2, "");
452      w = LLVMBuildExtractElement(builder, value,
453                                  idx3, "");
454
455      gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
456      gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
457      gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
458      gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
459
460      /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
461        x, gep0, y, gep1, z, gep2, w, gep3);*/
462      LLVMBuildStore(builder, x, gep0);
463      LLVMBuildStore(builder, y, gep1);
464      LLVMBuildStore(builder, z, gep2);
465      LLVMBuildStore(builder, w, gep3);
466   }
467#endif
468}
469
470static void
471store_aos_array(LLVMBuilderRef builder,
472                LLVMValueRef io_ptr,
473                LLVMValueRef aos[NUM_CHANNELS],
474                int attrib,
475                int num_outputs)
476{
477   LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
478   LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
479   LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
480   LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
481   LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
482   LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
483
484   debug_assert(NUM_CHANNELS == 4);
485
486   io0_ptr = LLVMBuildGEP(builder, io_ptr,
487                          &ind0, 1, "");
488   io1_ptr = LLVMBuildGEP(builder, io_ptr,
489                          &ind1, 1, "");
490   io2_ptr = LLVMBuildGEP(builder, io_ptr,
491                          &ind2, 1, "");
492   io3_ptr = LLVMBuildGEP(builder, io_ptr,
493                          &ind3, 1, "");
494
495#if DEBUG_STORE
496   lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
497                   io_ptr, ind0, ind1, ind2, ind3);
498#endif
499
500   store_aos(builder, io0_ptr, attr_index, aos[0]);
501   store_aos(builder, io1_ptr, attr_index, aos[1]);
502   store_aos(builder, io2_ptr, attr_index, aos[2]);
503   store_aos(builder, io3_ptr, attr_index, aos[3]);
504}
505
506static void
507convert_to_aos(LLVMBuilderRef builder,
508               LLVMValueRef io,
509               LLVMValueRef (*outputs)[NUM_CHANNELS],
510               int num_outputs,
511               int max_vertices)
512{
513   unsigned chan, attrib;
514
515#if DEBUG_STORE
516   lp_build_printf(builder, "   # storing begin\n");
517#endif
518   for (attrib = 0; attrib < num_outputs; ++attrib) {
519      LLVMValueRef soa[4];
520      LLVMValueRef aos[4];
521      for(chan = 0; chan < NUM_CHANNELS; ++chan) {
522         if(outputs[attrib][chan]) {
523            LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
524            lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
525            /*lp_build_printf(builder, "output %d : %d ",
526                            LLVMConstInt(LLVMInt32Type(), attrib, 0),
527                            LLVMConstInt(LLVMInt32Type(), chan, 0));
528              print_vectorf(builder, out);*/
529            soa[chan] = out;
530         } else
531            soa[chan] = 0;
532      }
533      soa_to_aos(builder, soa, aos);
534      store_aos_array(builder,
535                      io,
536                      aos,
537                      attrib,
538                      num_outputs);
539   }
540#if DEBUG_STORE
541   lp_build_printf(builder, "   # storing end\n");
542#endif
543}
544
545static void
546draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
547{
548   LLVMTypeRef arg_types[6];
549   LLVMTypeRef func_type;
550   LLVMValueRef context_ptr;
551   LLVMBasicBlockRef block;
552   LLVMBuilderRef builder;
553   LLVMValueRef start, end, count, stride, step, io_itr;
554   LLVMValueRef io_ptr, vbuffers_ptr;
555   struct draw_context *draw = llvm->draw;
556   unsigned i, j;
557   struct lp_build_context bld;
558   struct lp_build_loop_state lp_loop;
559   struct lp_type vs_type = lp_type_float_vec(32);
560   const int max_vertices = 4;
561   LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
562
563   arg_types[0] = llvm->context_ptr_type;           /* context */
564   arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
565   arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
566   arg_types[3] = LLVMInt32Type();                  /* start */
567   arg_types[4] = LLVMInt32Type();                  /* count */
568   arg_types[5] = LLVMInt32Type();                  /* stride */
569
570   func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
571
572   variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
573   LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
574   for(i = 0; i < Elements(arg_types); ++i)
575      if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
576         LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
577
578   context_ptr  = LLVMGetParam(variant->function, 0);
579   io_ptr       = LLVMGetParam(variant->function, 1);
580   vbuffers_ptr = LLVMGetParam(variant->function, 2);
581   start        = LLVMGetParam(variant->function, 3);
582   count        = LLVMGetParam(variant->function, 4);
583   stride       = LLVMGetParam(variant->function, 5);
584
585   lp_build_name(context_ptr, "context");
586   lp_build_name(io_ptr, "io");
587   lp_build_name(vbuffers_ptr, "vbuffers");
588   lp_build_name(start, "start");
589   lp_build_name(count, "count");
590   lp_build_name(stride, "stride");
591
592   /*
593    * Function body
594    */
595
596   block = LLVMAppendBasicBlock(variant->function, "entry");
597   builder = LLVMCreateBuilder();
598   LLVMPositionBuilderAtEnd(builder, block);
599
600   lp_build_context_init(&bld, builder, vs_type);
601
602   end = lp_build_add(&bld, start, count);
603
604   step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
605
606#if DEBUG_STORE
607   lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
608                   start, end, step);
609#endif
610   lp_build_loop_begin(builder, start, &lp_loop);
611   {
612      LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
613      LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
614      LLVMValueRef io;
615      const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
616
617      io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
618      io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
619#if DEBUG_STORE
620      lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
621                      io_itr, io, lp_loop.counter);
622#endif
623      for (i = 0; i < NUM_CHANNELS; ++i) {
624         LLVMValueRef true_index = LLVMBuildAdd(
625            builder,
626            lp_loop.counter,
627            LLVMConstInt(LLVMInt32Type(), i, 0), "");
628         for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
629            struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
630            struct pipe_vertex_buffer *vbuf = &draw->pt.vertex_buffer[
631               velem->vertex_buffer_index];
632            generate_fetch(builder, vbuffers_ptr,
633                           &aos_attribs[j][i], velem, vbuf, true_index);
634         }
635      }
636      convert_to_soa(builder, aos_attribs, inputs,
637                     draw->pt.nr_vertex_elements);
638
639      ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
640      generate_vs(llvm,
641                  builder,
642                  outputs,
643                  ptr_aos,
644                  context_ptr);
645
646      convert_to_aos(builder, io, outputs,
647                     draw->vs.vertex_shader->info.num_outputs,
648                     max_vertices);
649   }
650   lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
651
652   LLVMBuildRetVoid(builder);
653
654   LLVMDisposeBuilder(builder);
655
656   /*
657    * Translate the LLVM IR into machine code.
658    */
659
660#ifdef DEBUG
661   if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
662      LLVMDumpValue(variant->function);
663      assert(0);
664   }
665#endif
666
667   LLVMRunFunctionPassManager(llvm->pass, variant->function);
668
669   if (0) {
670      LLVMDumpValue(variant->function);
671      debug_printf("\n");
672   }
673   variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
674
675   if (0)
676      lp_disassemble(variant->jit_func);
677}
678
679void
680draw_llvm_make_variant_key(struct draw_llvm *llvm,
681                           struct draw_llvm_variant_key *key)
682{
683   key->nr_vertex_buffers = llvm->draw->pt.nr_vertex_buffers;
684   key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
685
686   memcpy(key->vertex_buffer,
687          llvm->draw->pt.vertex_buffer,
688          sizeof(struct pipe_vertex_buffer) * PIPE_MAX_ATTRIBS);
689
690   memcpy(key->vertex_element,
691          llvm->draw->pt.vertex_element,
692          sizeof(struct pipe_vertex_element) * PIPE_MAX_ATTRIBS);
693
694   memcpy(&key->vs,
695          &llvm->draw->vs.vertex_shader->state,
696          sizeof(struct pipe_shader_state));
697}
698