draw_llvm.c revision c09f3705196f1472871782d23d588e9fd14a50c7 
1#include "draw_llvm.h"
2
3#include "draw_context.h"
4#include "draw_vs.h"
5
6#include "gallivm/lp_bld_arit.h"
7#include "gallivm/lp_bld_struct.h"
8#include "gallivm/lp_bld_type.h"
9#include "gallivm/lp_bld_flow.h"
10#include "gallivm/lp_bld_debug.h"
11#include "gallivm/lp_bld_tgsi.h"
12#include "gallivm/lp_bld_printf.h"
13
14#include "tgsi/tgsi_exec.h"
15
16#include "util/u_cpu_detect.h"
17#include "util/u_string.h"
18
19#include <llvm-c/Transforms/Scalar.h>
20
21#define DEBUG_STORE 0
22
23
24/* generates the draw jit function */
25static void
26draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
27
28static void
29init_globals(struct draw_llvm *llvm)
30{
31   LLVMTypeRef texture_type;
32
33   /* struct draw_jit_texture */
34   {
35      LLVMTypeRef elem_types[4];
36
37      elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
38      elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
39      elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
40      elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
41
42      texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
43
44      LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
45                             llvm->target, texture_type,
46                             DRAW_JIT_TEXTURE_WIDTH);
47      LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
48                             llvm->target, texture_type,
49                             DRAW_JIT_TEXTURE_HEIGHT);
50      LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
51                             llvm->target, texture_type,
52                             DRAW_JIT_TEXTURE_STRIDE);
53      LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
54                             llvm->target, texture_type,
55                             DRAW_JIT_TEXTURE_DATA);
56      LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
57                           llvm->target, texture_type);
58
59      LLVMAddTypeName(llvm->module, "texture", texture_type);
60   }
61
62
63   /* struct draw_jit_context */
64   {
65      LLVMTypeRef elem_types[3];
66      LLVMTypeRef context_type;
67
68      elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
69      elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
70      elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
71
72      context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
73
74      LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
75                             llvm->target, context_type, 0);
76      LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
77                             llvm->target, context_type, 1);
78      LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
79                             llvm->target, context_type,
80                             DRAW_JIT_CONTEXT_TEXTURES_INDEX);
81      LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
82                           llvm->target, context_type);
83
84      LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
85
86      llvm->context_ptr_type = LLVMPointerType(context_type, 0);
87   }
88   {
89      LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
90      llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
91   }
92   /* struct pipe_vertex_buffer */
93   {
94      LLVMTypeRef elem_types[4];
95      LLVMTypeRef vb_type;
96
97      elem_types[0] = LLVMInt32Type();
98      elem_types[1] = LLVMInt32Type();
99      elem_types[2] = LLVMInt32Type();
100      elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
101
102      vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
103
104      LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
105                             llvm->target, vb_type, 0);
106      LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
107                             llvm->target, vb_type, 2);
108      LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
109                           llvm->target, vb_type);
110
111      LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
112
113      llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
114   }
115}
116
117static LLVMTypeRef
118create_vertex_header(struct draw_llvm *llvm, int data_elems)
119{
120   /* struct vertex_header */
121   LLVMTypeRef elem_types[3];
122   LLVMTypeRef vertex_header;
123   char struct_name[24];
124
125   util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
126
127   elem_types[0]  = LLVMIntType(32);
128   elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
129   elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
130
131   vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
132
133   /* these are bit-fields and we can't take address of them
134      LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
135      llvm->target, vertex_header,
136      DRAW_JIT_VERTEX_CLIPMASK);
137      LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
138      llvm->target, vertex_header,
139      DRAW_JIT_VERTEX_EDGEFLAG);
140      LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
141      llvm->target, vertex_header,
142      DRAW_JIT_VERTEX_PAD);
143      LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
144      llvm->target, vertex_header,
145      DRAW_JIT_VERTEX_VERTEX_ID);
146   */
147   LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
148                          llvm->target, vertex_header,
149                          DRAW_JIT_VERTEX_CLIP);
150   LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
151                          llvm->target, vertex_header,
152                          DRAW_JIT_VERTEX_DATA);
153
154   LLVMAddTypeName(llvm->module, struct_name, vertex_header);
155
156   return LLVMPointerType(vertex_header, 0);
157}
158
159struct draw_llvm *
160draw_llvm_create(struct draw_context *draw)
161{
162   struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm );
163
164   util_cpu_detect();
165
166   llvm->draw = draw;
167   llvm->engine = draw->engine;
168
169   debug_assert(llvm->engine);
170
171   llvm->module = LLVMModuleCreateWithName("draw_llvm");
172   llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
173
174   LLVMAddModuleProvider(llvm->engine, llvm->provider);
175
176   llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
177
178   llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
179   LLVMAddTargetData(llvm->target, llvm->pass);
180   /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
181    * but there are more on SVN. */
182   /* TODO: Add more passes */
183   LLVMAddConstantPropagationPass(llvm->pass);
184   if(util_cpu_caps.has_sse4_1) {
185      /* FIXME: There is a bug in this pass, whereby the combination of fptosi
186       * and sitofp (necessary for trunc/floor/ceil/round implementation)
187       * somehow becomes invalid code.
188       */
189      LLVMAddInstructionCombiningPass(llvm->pass);
190   }
191   LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
192   LLVMAddGVNPass(llvm->pass);
193   LLVMAddCFGSimplificationPass(llvm->pass);
194
195   init_globals(llvm);
196
197
198#if 0
199   LLVMDumpModule(llvm->module);
200#endif
201
202   return llvm;
203}
204
205void
206draw_llvm_destroy(struct draw_llvm *llvm)
207{
208   FREE(llvm);
209}
210
211struct draw_llvm_variant *
212draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
213{
214   struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
215
216   draw_llvm_make_variant_key(llvm, &variant->key);
217
218   llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
219
220   draw_llvm_generate(llvm, variant);
221
222   return variant;
223}
224
225static void
226generate_vs(struct draw_llvm *llvm,
227            LLVMBuilderRef builder,
228            LLVMValueRef (*outputs)[NUM_CHANNELS],
229            const LLVMValueRef (*inputs)[NUM_CHANNELS],
230            LLVMValueRef context_ptr)
231{
232   const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
233   struct lp_type vs_type;
234   LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
235
236   memset(&vs_type, 0, sizeof vs_type);
237   vs_type.floating = TRUE; /* floating point values */
238   vs_type.sign = TRUE;     /* values are signed */
239   vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
240   vs_type.width = 32;      /* 32-bit float */
241   vs_type.length = 4;      /* 4 elements per vector */
242#if 0
243   num_vs = 4;              /* number of vertices per block */
244#endif
245
246   /*tgsi_dump(tokens, 0);*/
247   lp_build_tgsi_soa(builder,
248                     tokens,
249                     vs_type,
250                     NULL /*struct lp_build_mask_context *mask*/,
251                     consts_ptr,
252                     NULL /*pos*/,
253                     inputs,
254                     outputs,
255                     NULL/*sampler*/,
256                     &llvm->draw->vs.vertex_shader->info);
257}
258
259#if DEBUG_STORE
260static void print_vectorf(LLVMBuilderRef builder,
261                         LLVMValueRef vec)
262{
263   LLVMValueRef val[4];
264   val[0] = LLVMBuildExtractElement(builder, vec,
265                                    LLVMConstInt(LLVMInt32Type(), 0, 0), "");
266   val[1] = LLVMBuildExtractElement(builder, vec,
267                                    LLVMConstInt(LLVMInt32Type(), 1, 0), "");
268   val[2] = LLVMBuildExtractElement(builder, vec,
269                                    LLVMConstInt(LLVMInt32Type(), 2, 0), "");
270   val[3] = LLVMBuildExtractElement(builder, vec,
271                                    LLVMConstInt(LLVMInt32Type(), 3, 0), "");
272   lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
273                   val[0], val[1], val[2], val[3]);
274}
275#endif
276
277static void
278generate_fetch(LLVMBuilderRef builder,
279               LLVMValueRef vbuffers_ptr,
280               LLVMValueRef *res,
281               struct pipe_vertex_element *velem,
282               LLVMValueRef vbuf,
283               LLVMValueRef index)
284{
285   LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
286   LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
287                                           &indices, 1, "");
288   LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
289   LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
290   LLVMValueRef stride = LLVMBuildMul(builder,
291                                      vb_stride,
292                                      index, "");
293
294   vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
295
296   stride = LLVMBuildAdd(builder, stride,
297                         vb_buffer_offset,
298                         "");
299   stride = LLVMBuildAdd(builder, stride,
300                         LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
301                         "");
302
303   /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
304   vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
305
306   *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
307}
308
309static LLVMValueRef
310aos_to_soa(LLVMBuilderRef builder,
311           LLVMValueRef val0,
312           LLVMValueRef val1,
313           LLVMValueRef val2,
314           LLVMValueRef val3,
315           LLVMValueRef channel)
316{
317   LLVMValueRef ex, res;
318
319   ex = LLVMBuildExtractElement(builder, val0,
320                                channel, "");
321   res = LLVMBuildInsertElement(builder,
322                                LLVMConstNull(LLVMTypeOf(val0)),
323                                ex,
324                                LLVMConstInt(LLVMInt32Type(), 0, 0),
325                                "");
326
327   ex = LLVMBuildExtractElement(builder, val1,
328                                channel, "");
329   res = LLVMBuildInsertElement(builder,
330                                res, ex,
331                                LLVMConstInt(LLVMInt32Type(), 1, 0),
332                                "");
333
334   ex = LLVMBuildExtractElement(builder, val2,
335                                channel, "");
336   res = LLVMBuildInsertElement(builder,
337                                res, ex,
338                                LLVMConstInt(LLVMInt32Type(), 2, 0),
339                                "");
340
341   ex = LLVMBuildExtractElement(builder, val3,
342                                channel, "");
343   res = LLVMBuildInsertElement(builder,
344                                res, ex,
345                                LLVMConstInt(LLVMInt32Type(), 3, 0),
346                                "");
347
348   return res;
349}
350
351static void
352soa_to_aos(LLVMBuilderRef builder,
353           LLVMValueRef soa[NUM_CHANNELS],
354           LLVMValueRef aos[NUM_CHANNELS])
355{
356   LLVMValueRef comp;
357   int i = 0;
358
359   debug_assert(NUM_CHANNELS == 4);
360
361   aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
362   aos[1] = aos[2] = aos[3] = aos[0];
363
364   for (i = 0; i < NUM_CHANNELS; ++i) {
365      LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
366
367      comp = LLVMBuildExtractElement(builder, soa[i],
368                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
369      aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
370
371      comp = LLVMBuildExtractElement(builder, soa[i],
372                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
373      aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
374
375      comp = LLVMBuildExtractElement(builder, soa[i],
376                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
377      aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
378
379      comp = LLVMBuildExtractElement(builder, soa[i],
380                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
381      aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
382
383   }
384}
385
386static void
387convert_to_soa(LLVMBuilderRef builder,
388               LLVMValueRef (*aos)[NUM_CHANNELS],
389               LLVMValueRef (*soa)[NUM_CHANNELS],
390               int num_attribs)
391{
392   int i;
393
394   debug_assert(NUM_CHANNELS == 4);
395
396   for (i = 0; i < num_attribs; ++i) {
397      LLVMValueRef val0 = aos[i][0];
398      LLVMValueRef val1 = aos[i][1];
399      LLVMValueRef val2 = aos[i][2];
400      LLVMValueRef val3 = aos[i][3];
401
402      soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
403                             LLVMConstInt(LLVMInt32Type(), 0, 0));
404      soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
405                             LLVMConstInt(LLVMInt32Type(), 1, 0));
406      soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
407                             LLVMConstInt(LLVMInt32Type(), 2, 0));
408      soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
409                             LLVMConstInt(LLVMInt32Type(), 3, 0));
410   }
411}
412
413static void
414store_aos(LLVMBuilderRef builder,
415          LLVMValueRef io_ptr,
416          LLVMValueRef index,
417          LLVMValueRef value)
418{
419   LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
420   LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
421   LLVMValueRef indices[3];
422
423   indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
424   indices[1] = index;
425   indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
426
427   /* undefined vertex */
428   LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
429                                        0xffff, 0), id_ptr);
430
431#if DEBUG_STORE
432   lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
433#endif
434#if 0
435   /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
436     print_vectorf(builder, value);*/
437   data_ptr = LLVMBuildBitCast(builder, data_ptr,
438                               LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
439                               "datavec");
440   data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
441
442   LLVMBuildStore(builder, value, data_ptr);
443#else
444   {
445      LLVMValueRef x, y, z, w;
446      LLVMValueRef idx0, idx1, idx2, idx3;
447      LLVMValueRef gep0, gep1, gep2, gep3;
448      data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
449
450      idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
451      idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
452      idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
453      idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
454
455      x = LLVMBuildExtractElement(builder, value,
456                                  idx0, "");
457      y = LLVMBuildExtractElement(builder, value,
458                                  idx1, "");
459      z = LLVMBuildExtractElement(builder, value,
460                                  idx2, "");
461      w = LLVMBuildExtractElement(builder, value,
462                                  idx3, "");
463
464      gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
465      gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
466      gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
467      gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
468
469      /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
470        x, gep0, y, gep1, z, gep2, w, gep3);*/
471      LLVMBuildStore(builder, x, gep0);
472      LLVMBuildStore(builder, y, gep1);
473      LLVMBuildStore(builder, z, gep2);
474      LLVMBuildStore(builder, w, gep3);
475   }
476#endif
477}
478
479static void
480store_aos_array(LLVMBuilderRef builder,
481                LLVMValueRef io_ptr,
482                LLVMValueRef aos[NUM_CHANNELS],
483                int attrib,
484                int num_outputs)
485{
486   LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
487   LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
488   LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
489   LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
490   LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
491   LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
492
493   debug_assert(NUM_CHANNELS == 4);
494
495   io0_ptr = LLVMBuildGEP(builder, io_ptr,
496                          &ind0, 1, "");
497   io1_ptr = LLVMBuildGEP(builder, io_ptr,
498                          &ind1, 1, "");
499   io2_ptr = LLVMBuildGEP(builder, io_ptr,
500                          &ind2, 1, "");
501   io3_ptr = LLVMBuildGEP(builder, io_ptr,
502                          &ind3, 1, "");
503
504#if DEBUG_STORE
505   lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
506                   io_ptr, ind0, ind1, ind2, ind3);
507#endif
508
509   store_aos(builder, io0_ptr, attr_index, aos[0]);
510   store_aos(builder, io1_ptr, attr_index, aos[1]);
511   store_aos(builder, io2_ptr, attr_index, aos[2]);
512   store_aos(builder, io3_ptr, attr_index, aos[3]);
513}
514
515static void
516convert_to_aos(LLVMBuilderRef builder,
517               LLVMValueRef io,
518               LLVMValueRef (*outputs)[NUM_CHANNELS],
519               int num_outputs,
520               int max_vertices)
521{
522   unsigned chan, attrib;
523
524#if DEBUG_STORE
525   lp_build_printf(builder, "   # storing begin\n");
526#endif
527   for (attrib = 0; attrib < num_outputs; ++attrib) {
528      LLVMValueRef soa[4];
529      LLVMValueRef aos[4];
530      for(chan = 0; chan < NUM_CHANNELS; ++chan) {
531         if(outputs[attrib][chan]) {
532            LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
533            lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
534            /*lp_build_printf(builder, "output %d : %d ",
535                            LLVMConstInt(LLVMInt32Type(), attrib, 0),
536                            LLVMConstInt(LLVMInt32Type(), chan, 0));
537              print_vectorf(builder, out);*/
538            soa[chan] = out;
539         } else
540            soa[chan] = 0;
541      }
542      soa_to_aos(builder, soa, aos);
543      store_aos_array(builder,
544                      io,
545                      aos,
546                      attrib,
547                      num_outputs);
548   }
549#if DEBUG_STORE
550   lp_build_printf(builder, "   # storing end\n");
551#endif
552}
553
554static void
555draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
556{
557   LLVMTypeRef arg_types[7];
558   LLVMTypeRef func_type;
559   LLVMValueRef context_ptr;
560   LLVMBasicBlockRef block;
561   LLVMBuilderRef builder;
562   LLVMValueRef start, end, count, stride, step, io_itr;
563   LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
564   struct draw_context *draw = llvm->draw;
565   unsigned i, j;
566   struct lp_build_context bld;
567   struct lp_build_loop_state lp_loop;
568   struct lp_type vs_type = lp_type_float_vec(32);
569   const int max_vertices = 4;
570   LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
571
572   arg_types[0] = llvm->context_ptr_type;           /* context */
573   arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
574   arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
575   arg_types[3] = LLVMInt32Type();                  /* start */
576   arg_types[4] = LLVMInt32Type();                  /* count */
577   arg_types[5] = LLVMInt32Type();                  /* stride */
578   arg_types[6] = llvm->vb_ptr_type;                /* pipe_vertex_buffer's */
579
580   func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
581
582   variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
583   LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
584   for(i = 0; i < Elements(arg_types); ++i)
585      if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
586         LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
587
588   context_ptr  = LLVMGetParam(variant->function, 0);
589   io_ptr       = LLVMGetParam(variant->function, 1);
590   vbuffers_ptr = LLVMGetParam(variant->function, 2);
591   start        = LLVMGetParam(variant->function, 3);
592   count        = LLVMGetParam(variant->function, 4);
593   stride       = LLVMGetParam(variant->function, 5);
594   vb_ptr       = LLVMGetParam(variant->function, 6);
595
596   lp_build_name(context_ptr, "context");
597   lp_build_name(io_ptr, "io");
598   lp_build_name(vbuffers_ptr, "vbuffers");
599   lp_build_name(start, "start");
600   lp_build_name(count, "count");
601   lp_build_name(stride, "stride");
602   lp_build_name(vb_ptr, "vb");
603
604   /*
605    * Function body
606    */
607
608   block = LLVMAppendBasicBlock(variant->function, "entry");
609   builder = LLVMCreateBuilder();
610   LLVMPositionBuilderAtEnd(builder, block);
611
612   lp_build_context_init(&bld, builder, vs_type);
613
614   end = lp_build_add(&bld, start, count);
615
616   step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
617
618#if DEBUG_STORE
619   lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
620                   start, end, step);
621#endif
622   lp_build_loop_begin(builder, start, &lp_loop);
623   {
624      LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
625      LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
626      LLVMValueRef io;
627      const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
628
629      io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
630      io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
631#if DEBUG_STORE
632      lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
633                      io_itr, io, lp_loop.counter);
634#endif
635      for (i = 0; i < NUM_CHANNELS; ++i) {
636         LLVMValueRef true_index = LLVMBuildAdd(
637            builder,
638            lp_loop.counter,
639            LLVMConstInt(LLVMInt32Type(), i, 0), "");
640         for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
641            struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
642            LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
643                                                 velem->vertex_buffer_index,
644                                                 0);
645            LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
646                                           &vb_index, 1, "");
647            generate_fetch(builder, vbuffers_ptr,
648                           &aos_attribs[j][i], velem, vb, true_index);
649         }
650      }
651      convert_to_soa(builder, aos_attribs, inputs,
652                     draw->pt.nr_vertex_elements);
653
654      ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
655      generate_vs(llvm,
656                  builder,
657                  outputs,
658                  ptr_aos,
659                  context_ptr);
660
661      convert_to_aos(builder, io, outputs,
662                     draw->vs.vertex_shader->info.num_outputs,
663                     max_vertices);
664   }
665   lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
666
667   LLVMBuildRetVoid(builder);
668
669   LLVMDisposeBuilder(builder);
670
671   /*
672    * Translate the LLVM IR into machine code.
673    */
674#ifdef DEBUG
675   if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
676      LLVMDumpValue(variant->function);
677      assert(0);
678   }
679#endif
680
681   LLVMRunFunctionPassManager(llvm->pass, variant->function);
682
683   if (0) {
684      LLVMDumpValue(variant->function);
685      debug_printf("\n");
686   }
687   variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
688
689   if (0)
690      lp_disassemble(variant->jit_func);
691}
692
693void
694draw_llvm_make_variant_key(struct draw_llvm *llvm,
695                           struct draw_llvm_variant_key *key)
696{
697   memset(key, 0, sizeof(struct draw_llvm_variant_key));
698
699   key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
700
701   memcpy(key->vertex_element,
702          llvm->draw->pt.vertex_element,
703          sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
704
705   memcpy(&key->vs,
706          &llvm->draw->vs.vertex_shader->state,
707          sizeof(struct pipe_shader_state));
708}
709