draw_llvm.c revision 4d52a25326506f8a3e876ded029ff4db7749f8a9
1#include "draw_llvm.h" 2 3#include "draw_context.h" 4#include "draw_vs.h" 5 6#include "gallivm/lp_bld_arit.h" 7#include "gallivm/lp_bld_struct.h" 8#include "gallivm/lp_bld_type.h" 9#include "gallivm/lp_bld_flow.h" 10#include "gallivm/lp_bld_debug.h" 11#include "gallivm/lp_bld_tgsi.h" 12#include "gallivm/lp_bld_printf.h" 13 14#include "tgsi/tgsi_exec.h" 15 16#include "util/u_cpu_detect.h" 17#include "util/u_string.h" 18 19#include <llvm-c/Transforms/Scalar.h> 20 21#define DEBUG_STORE 0 22 23 24/* generates the draw jit function */ 25static void 26draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var); 27static void 28draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var); 29 30static void 31init_globals(struct draw_llvm *llvm) 32{ 33 LLVMTypeRef texture_type; 34 35 /* struct draw_jit_texture */ 36 { 37 LLVMTypeRef elem_types[4]; 38 39 elem_types[DRAW_JIT_TEXTURE_WIDTH] = LLVMInt32Type(); 40 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type(); 41 elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type(); 42 elem_types[DRAW_JIT_TEXTURE_DATA] = LLVMPointerType(LLVMInt8Type(), 0); 43 44 texture_type = LLVMStructType(elem_types, Elements(elem_types), 0); 45 46 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width, 47 llvm->target, texture_type, 48 DRAW_JIT_TEXTURE_WIDTH); 49 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height, 50 llvm->target, texture_type, 51 DRAW_JIT_TEXTURE_HEIGHT); 52 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride, 53 llvm->target, texture_type, 54 DRAW_JIT_TEXTURE_STRIDE); 55 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data, 56 llvm->target, texture_type, 57 DRAW_JIT_TEXTURE_DATA); 58 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, 59 llvm->target, texture_type); 60 61 LLVMAddTypeName(llvm->module, "texture", texture_type); 62 } 63 64 65 /* struct draw_jit_context */ 66 { 67 LLVMTypeRef elem_types[3]; 68 LLVMTypeRef context_type; 69 70 elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */ 71 elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */ 72 elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */ 73 74 context_type = LLVMStructType(elem_types, Elements(elem_types), 0); 75 76 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants, 77 llvm->target, context_type, 0); 78 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants, 79 llvm->target, context_type, 1); 80 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures, 81 llvm->target, context_type, 82 DRAW_JIT_CONTEXT_TEXTURES_INDEX); 83 LP_CHECK_STRUCT_SIZE(struct draw_jit_context, 84 llvm->target, context_type); 85 86 LLVMAddTypeName(llvm->module, "draw_jit_context", context_type); 87 88 llvm->context_ptr_type = LLVMPointerType(context_type, 0); 89 } 90 { 91 LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0); 92 llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0); 93 } 94 /* struct pipe_vertex_buffer */ 95 { 96 LLVMTypeRef elem_types[4]; 97 LLVMTypeRef vb_type; 98 99 elem_types[0] = LLVMInt32Type(); 100 elem_types[1] = LLVMInt32Type(); 101 elem_types[2] = LLVMInt32Type(); 102 elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */ 103 104 vb_type = LLVMStructType(elem_types, Elements(elem_types), 0); 105 106 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride, 107 llvm->target, vb_type, 0); 108 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset, 109 llvm->target, vb_type, 2); 110 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, 111 llvm->target, vb_type); 112 113 LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type); 114 115 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0); 116 } 117} 118 119static LLVMTypeRef 120create_vertex_header(struct draw_llvm *llvm, int data_elems) 121{ 122 /* struct vertex_header */ 123 LLVMTypeRef elem_types[3]; 124 LLVMTypeRef vertex_header; 125 char struct_name[24]; 126 127 util_snprintf(struct_name, 23, "vertex_header%d", data_elems); 128 129 elem_types[0] = LLVMIntType(32); 130 elem_types[1] = LLVMArrayType(LLVMFloatType(), 4); 131 elem_types[2] = LLVMArrayType(elem_types[1], data_elems); 132 133 vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0); 134 135 /* these are bit-fields and we can't take address of them 136 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask, 137 llvm->target, vertex_header, 138 DRAW_JIT_VERTEX_CLIPMASK); 139 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag, 140 llvm->target, vertex_header, 141 DRAW_JIT_VERTEX_EDGEFLAG); 142 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad, 143 llvm->target, vertex_header, 144 DRAW_JIT_VERTEX_PAD); 145 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id, 146 llvm->target, vertex_header, 147 DRAW_JIT_VERTEX_VERTEX_ID); 148 */ 149 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip, 150 llvm->target, vertex_header, 151 DRAW_JIT_VERTEX_CLIP); 152 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data, 153 llvm->target, vertex_header, 154 DRAW_JIT_VERTEX_DATA); 155 156 LLVMAddTypeName(llvm->module, struct_name, vertex_header); 157 158 return LLVMPointerType(vertex_header, 0); 159} 160 161struct draw_llvm * 162draw_llvm_create(struct draw_context *draw) 163{ 164 struct draw_llvm *llvm; 165 166#ifdef PIPE_ARCH_X86 167 util_cpu_detect(); 168 /* require SSE2 due to LLVM PR6960. */ 169 if (!util_cpu_caps.has_sse2) 170 return NULL; 171#endif 172 173 llvm = CALLOC_STRUCT( draw_llvm ); 174 175 llvm->draw = draw; 176 llvm->engine = draw->engine; 177 178 debug_assert(llvm->engine); 179 180 llvm->module = LLVMModuleCreateWithName("draw_llvm"); 181 llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module); 182 183 LLVMAddModuleProvider(llvm->engine, llvm->provider); 184 185 llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine); 186 187 llvm->pass = LLVMCreateFunctionPassManager(llvm->provider); 188 LLVMAddTargetData(llvm->target, llvm->pass); 189 /* These are the passes currently listed in llvm-c/Transforms/Scalar.h, 190 * but there are more on SVN. */ 191 /* TODO: Add more passes */ 192 LLVMAddCFGSimplificationPass(llvm->pass); 193 LLVMAddPromoteMemoryToRegisterPass(llvm->pass); 194 LLVMAddConstantPropagationPass(llvm->pass); 195 if(util_cpu_caps.has_sse4_1) { 196 /* FIXME: There is a bug in this pass, whereby the combination of fptosi 197 * and sitofp (necessary for trunc/floor/ceil/round implementation) 198 * somehow becomes invalid code. 199 */ 200 LLVMAddInstructionCombiningPass(llvm->pass); 201 } 202 LLVMAddGVNPass(llvm->pass); 203 204 init_globals(llvm); 205 206 207#if 0 208 LLVMDumpModule(llvm->module); 209#endif 210 211 return llvm; 212} 213 214void 215draw_llvm_destroy(struct draw_llvm *llvm) 216{ 217 LLVMDisposePassManager(llvm->pass); 218 219 FREE(llvm); 220} 221 222struct draw_llvm_variant * 223draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs) 224{ 225 struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant)); 226 227 draw_llvm_make_variant_key(llvm, &variant->key); 228 229 llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs); 230 231 draw_llvm_generate(llvm, variant); 232 draw_llvm_generate_elts(llvm, variant); 233 234 return variant; 235} 236 237static void 238generate_vs(struct draw_llvm *llvm, 239 LLVMBuilderRef builder, 240 LLVMValueRef (*outputs)[NUM_CHANNELS], 241 const LLVMValueRef (*inputs)[NUM_CHANNELS], 242 LLVMValueRef context_ptr) 243{ 244 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens; 245 struct lp_type vs_type; 246 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr); 247 248 memset(&vs_type, 0, sizeof vs_type); 249 vs_type.floating = TRUE; /* floating point values */ 250 vs_type.sign = TRUE; /* values are signed */ 251 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 252 vs_type.width = 32; /* 32-bit float */ 253 vs_type.length = 4; /* 4 elements per vector */ 254#if 0 255 num_vs = 4; /* number of vertices per block */ 256#endif 257 258 /*tgsi_dump(tokens, 0);*/ 259 lp_build_tgsi_soa(builder, 260 tokens, 261 vs_type, 262 NULL /*struct lp_build_mask_context *mask*/, 263 consts_ptr, 264 NULL /*pos*/, 265 inputs, 266 outputs, 267 NULL/*sampler*/, 268 &llvm->draw->vs.vertex_shader->info); 269} 270 271#if DEBUG_STORE 272static void print_vectorf(LLVMBuilderRef builder, 273 LLVMValueRef vec) 274{ 275 LLVMValueRef val[4]; 276 val[0] = LLVMBuildExtractElement(builder, vec, 277 LLVMConstInt(LLVMInt32Type(), 0, 0), ""); 278 val[1] = LLVMBuildExtractElement(builder, vec, 279 LLVMConstInt(LLVMInt32Type(), 1, 0), ""); 280 val[2] = LLVMBuildExtractElement(builder, vec, 281 LLVMConstInt(LLVMInt32Type(), 2, 0), ""); 282 val[3] = LLVMBuildExtractElement(builder, vec, 283 LLVMConstInt(LLVMInt32Type(), 3, 0), ""); 284 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n", 285 val[0], val[1], val[2], val[3]); 286} 287#endif 288 289static void 290generate_fetch(LLVMBuilderRef builder, 291 LLVMValueRef vbuffers_ptr, 292 LLVMValueRef *res, 293 struct pipe_vertex_element *velem, 294 LLVMValueRef vbuf, 295 LLVMValueRef index) 296{ 297 LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0); 298 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr, 299 &indices, 1, ""); 300 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf); 301 LLVMValueRef vb_max_index = draw_jit_vbuffer_max_index(builder, vbuf); 302 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf); 303 LLVMValueRef cond; 304 LLVMValueRef stride; 305 306 cond = LLVMBuildICmp(builder, LLVMIntULE, index, vb_max_index, ""); 307 308 index = LLVMBuildSelect(builder, cond, index, vb_max_index, ""); 309 310 stride = LLVMBuildMul(builder, vb_stride, index, ""); 311 312 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer"); 313 314 stride = LLVMBuildAdd(builder, stride, 315 vb_buffer_offset, 316 ""); 317 stride = LLVMBuildAdd(builder, stride, 318 LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0), 319 ""); 320 321 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/ 322 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, ""); 323 324 *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format); 325} 326 327static LLVMValueRef 328aos_to_soa(LLVMBuilderRef builder, 329 LLVMValueRef val0, 330 LLVMValueRef val1, 331 LLVMValueRef val2, 332 LLVMValueRef val3, 333 LLVMValueRef channel) 334{ 335 LLVMValueRef ex, res; 336 337 ex = LLVMBuildExtractElement(builder, val0, 338 channel, ""); 339 res = LLVMBuildInsertElement(builder, 340 LLVMConstNull(LLVMTypeOf(val0)), 341 ex, 342 LLVMConstInt(LLVMInt32Type(), 0, 0), 343 ""); 344 345 ex = LLVMBuildExtractElement(builder, val1, 346 channel, ""); 347 res = LLVMBuildInsertElement(builder, 348 res, ex, 349 LLVMConstInt(LLVMInt32Type(), 1, 0), 350 ""); 351 352 ex = LLVMBuildExtractElement(builder, val2, 353 channel, ""); 354 res = LLVMBuildInsertElement(builder, 355 res, ex, 356 LLVMConstInt(LLVMInt32Type(), 2, 0), 357 ""); 358 359 ex = LLVMBuildExtractElement(builder, val3, 360 channel, ""); 361 res = LLVMBuildInsertElement(builder, 362 res, ex, 363 LLVMConstInt(LLVMInt32Type(), 3, 0), 364 ""); 365 366 return res; 367} 368 369static void 370soa_to_aos(LLVMBuilderRef builder, 371 LLVMValueRef soa[NUM_CHANNELS], 372 LLVMValueRef aos[NUM_CHANNELS]) 373{ 374 LLVMValueRef comp; 375 int i = 0; 376 377 debug_assert(NUM_CHANNELS == 4); 378 379 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0])); 380 aos[1] = aos[2] = aos[3] = aos[0]; 381 382 for (i = 0; i < NUM_CHANNELS; ++i) { 383 LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0); 384 385 comp = LLVMBuildExtractElement(builder, soa[i], 386 LLVMConstInt(LLVMInt32Type(), 0, 0), ""); 387 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, ""); 388 389 comp = LLVMBuildExtractElement(builder, soa[i], 390 LLVMConstInt(LLVMInt32Type(), 1, 0), ""); 391 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, ""); 392 393 comp = LLVMBuildExtractElement(builder, soa[i], 394 LLVMConstInt(LLVMInt32Type(), 2, 0), ""); 395 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, ""); 396 397 comp = LLVMBuildExtractElement(builder, soa[i], 398 LLVMConstInt(LLVMInt32Type(), 3, 0), ""); 399 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, ""); 400 401 } 402} 403 404static void 405convert_to_soa(LLVMBuilderRef builder, 406 LLVMValueRef (*aos)[NUM_CHANNELS], 407 LLVMValueRef (*soa)[NUM_CHANNELS], 408 int num_attribs) 409{ 410 int i; 411 412 debug_assert(NUM_CHANNELS == 4); 413 414 for (i = 0; i < num_attribs; ++i) { 415 LLVMValueRef val0 = aos[i][0]; 416 LLVMValueRef val1 = aos[i][1]; 417 LLVMValueRef val2 = aos[i][2]; 418 LLVMValueRef val3 = aos[i][3]; 419 420 soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3, 421 LLVMConstInt(LLVMInt32Type(), 0, 0)); 422 soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3, 423 LLVMConstInt(LLVMInt32Type(), 1, 0)); 424 soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3, 425 LLVMConstInt(LLVMInt32Type(), 2, 0)); 426 soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3, 427 LLVMConstInt(LLVMInt32Type(), 3, 0)); 428 } 429} 430 431static void 432store_aos(LLVMBuilderRef builder, 433 LLVMValueRef io_ptr, 434 LLVMValueRef index, 435 LLVMValueRef value) 436{ 437 LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr); 438 LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr); 439 LLVMValueRef indices[3]; 440 441 indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0); 442 indices[1] = index; 443 indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0); 444 445 /* undefined vertex */ 446 LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(), 447 0xffff, 0), id_ptr); 448 449#if DEBUG_STORE 450 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr); 451#endif 452#if 0 453 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr); 454 print_vectorf(builder, value);*/ 455 data_ptr = LLVMBuildBitCast(builder, data_ptr, 456 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0), 457 "datavec"); 458 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, ""); 459 460 LLVMBuildStore(builder, value, data_ptr); 461#else 462 { 463 LLVMValueRef x, y, z, w; 464 LLVMValueRef idx0, idx1, idx2, idx3; 465 LLVMValueRef gep0, gep1, gep2, gep3; 466 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, ""); 467 468 idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0); 469 idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0); 470 idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0); 471 idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0); 472 473 x = LLVMBuildExtractElement(builder, value, 474 idx0, ""); 475 y = LLVMBuildExtractElement(builder, value, 476 idx1, ""); 477 z = LLVMBuildExtractElement(builder, value, 478 idx2, ""); 479 w = LLVMBuildExtractElement(builder, value, 480 idx3, ""); 481 482 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, ""); 483 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, ""); 484 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, ""); 485 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, ""); 486 487 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n", 488 x, gep0, y, gep1, z, gep2, w, gep3);*/ 489 LLVMBuildStore(builder, x, gep0); 490 LLVMBuildStore(builder, y, gep1); 491 LLVMBuildStore(builder, z, gep2); 492 LLVMBuildStore(builder, w, gep3); 493 } 494#endif 495} 496 497static void 498store_aos_array(LLVMBuilderRef builder, 499 LLVMValueRef io_ptr, 500 LLVMValueRef aos[NUM_CHANNELS], 501 int attrib, 502 int num_outputs) 503{ 504 LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0); 505 LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0); 506 LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0); 507 LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0); 508 LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0); 509 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 510 511 debug_assert(NUM_CHANNELS == 4); 512 513 io0_ptr = LLVMBuildGEP(builder, io_ptr, 514 &ind0, 1, ""); 515 io1_ptr = LLVMBuildGEP(builder, io_ptr, 516 &ind1, 1, ""); 517 io2_ptr = LLVMBuildGEP(builder, io_ptr, 518 &ind2, 1, ""); 519 io3_ptr = LLVMBuildGEP(builder, io_ptr, 520 &ind3, 1, ""); 521 522#if DEBUG_STORE 523 lp_build_printf(builder, " io = %p, indexes[%d, %d, %d, %d]\n", 524 io_ptr, ind0, ind1, ind2, ind3); 525#endif 526 527 store_aos(builder, io0_ptr, attr_index, aos[0]); 528 store_aos(builder, io1_ptr, attr_index, aos[1]); 529 store_aos(builder, io2_ptr, attr_index, aos[2]); 530 store_aos(builder, io3_ptr, attr_index, aos[3]); 531} 532 533static void 534convert_to_aos(LLVMBuilderRef builder, 535 LLVMValueRef io, 536 LLVMValueRef (*outputs)[NUM_CHANNELS], 537 int num_outputs, 538 int max_vertices) 539{ 540 unsigned chan, attrib; 541 542#if DEBUG_STORE 543 lp_build_printf(builder, " # storing begin\n"); 544#endif 545 for (attrib = 0; attrib < num_outputs; ++attrib) { 546 LLVMValueRef soa[4]; 547 LLVMValueRef aos[4]; 548 for(chan = 0; chan < NUM_CHANNELS; ++chan) { 549 if(outputs[attrib][chan]) { 550 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 551 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]); 552 /*lp_build_printf(builder, "output %d : %d ", 553 LLVMConstInt(LLVMInt32Type(), attrib, 0), 554 LLVMConstInt(LLVMInt32Type(), chan, 0)); 555 print_vectorf(builder, out);*/ 556 soa[chan] = out; 557 } else 558 soa[chan] = 0; 559 } 560 soa_to_aos(builder, soa, aos); 561 store_aos_array(builder, 562 io, 563 aos, 564 attrib, 565 num_outputs); 566 } 567#if DEBUG_STORE 568 lp_build_printf(builder, " # storing end\n"); 569#endif 570} 571 572static void 573draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant) 574{ 575 LLVMTypeRef arg_types[7]; 576 LLVMTypeRef func_type; 577 LLVMValueRef context_ptr; 578 LLVMBasicBlockRef block; 579 LLVMBuilderRef builder; 580 LLVMValueRef start, end, count, stride, step, io_itr; 581 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 582 struct draw_context *draw = llvm->draw; 583 unsigned i, j; 584 struct lp_build_context bld; 585 struct lp_build_loop_state lp_loop; 586 struct lp_type vs_type = lp_type_float_vec(32); 587 const int max_vertices = 4; 588 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS]; 589 590 arg_types[0] = llvm->context_ptr_type; /* context */ 591 arg_types[1] = llvm->vertex_header_ptr_type; /* vertex_header */ 592 arg_types[2] = llvm->buffer_ptr_type; /* vbuffers */ 593 arg_types[3] = LLVMInt32Type(); /* start */ 594 arg_types[4] = LLVMInt32Type(); /* count */ 595 arg_types[5] = LLVMInt32Type(); /* stride */ 596 arg_types[6] = llvm->vb_ptr_type; /* pipe_vertex_buffer's */ 597 598 func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0); 599 600 variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type); 601 LLVMSetFunctionCallConv(variant->function, LLVMCCallConv); 602 for(i = 0; i < Elements(arg_types); ++i) 603 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 604 LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute); 605 606 context_ptr = LLVMGetParam(variant->function, 0); 607 io_ptr = LLVMGetParam(variant->function, 1); 608 vbuffers_ptr = LLVMGetParam(variant->function, 2); 609 start = LLVMGetParam(variant->function, 3); 610 count = LLVMGetParam(variant->function, 4); 611 stride = LLVMGetParam(variant->function, 5); 612 vb_ptr = LLVMGetParam(variant->function, 6); 613 614 lp_build_name(context_ptr, "context"); 615 lp_build_name(io_ptr, "io"); 616 lp_build_name(vbuffers_ptr, "vbuffers"); 617 lp_build_name(start, "start"); 618 lp_build_name(count, "count"); 619 lp_build_name(stride, "stride"); 620 lp_build_name(vb_ptr, "vb"); 621 622 /* 623 * Function body 624 */ 625 626 block = LLVMAppendBasicBlock(variant->function, "entry"); 627 builder = LLVMCreateBuilder(); 628 LLVMPositionBuilderAtEnd(builder, block); 629 630 lp_build_context_init(&bld, builder, vs_type); 631 632 end = lp_build_add(&bld, start, count); 633 634 step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0); 635 636#if DEBUG_STORE 637 lp_build_printf(builder, "start = %d, end = %d, step = %d\n", 638 start, end, step); 639#endif 640 lp_build_loop_begin(builder, start, &lp_loop); 641 { 642 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS]; 643 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } }; 644 LLVMValueRef io; 645 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS]; 646 647 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, ""); 648 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 649#if DEBUG_STORE 650 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n", 651 io_itr, io, lp_loop.counter); 652#endif 653 for (i = 0; i < NUM_CHANNELS; ++i) { 654 LLVMValueRef true_index = LLVMBuildAdd( 655 builder, 656 lp_loop.counter, 657 LLVMConstInt(LLVMInt32Type(), i, 0), ""); 658 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 659 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 660 LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(), 661 velem->vertex_buffer_index, 662 0); 663 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, 664 &vb_index, 1, ""); 665 generate_fetch(builder, vbuffers_ptr, 666 &aos_attribs[j][i], velem, vb, true_index); 667 } 668 } 669 convert_to_soa(builder, aos_attribs, inputs, 670 draw->pt.nr_vertex_elements); 671 672 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs; 673 generate_vs(llvm, 674 builder, 675 outputs, 676 ptr_aos, 677 context_ptr); 678 679 convert_to_aos(builder, io, outputs, 680 draw->vs.vertex_shader->info.num_outputs, 681 max_vertices); 682 } 683 lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop); 684 685 LLVMBuildRetVoid(builder); 686 687 LLVMDisposeBuilder(builder); 688 689 /* 690 * Translate the LLVM IR into machine code. 691 */ 692#ifdef DEBUG 693 if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) { 694 LLVMDumpValue(variant->function); 695 assert(0); 696 } 697#endif 698 699 LLVMRunFunctionPassManager(llvm->pass, variant->function); 700 701 if (0) { 702 LLVMDumpValue(variant->function); 703 debug_printf("\n"); 704 } 705 variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function); 706 707 if (0) 708 lp_disassemble(variant->jit_func); 709} 710 711 712static void 713draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant) 714{ 715 LLVMTypeRef arg_types[7]; 716 LLVMTypeRef func_type; 717 LLVMValueRef context_ptr; 718 LLVMBasicBlockRef block; 719 LLVMBuilderRef builder; 720 LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr; 721 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 722 struct draw_context *draw = llvm->draw; 723 unsigned i, j; 724 struct lp_build_context bld; 725 struct lp_build_context bld_int; 726 struct lp_build_loop_state lp_loop; 727 struct lp_type vs_type = lp_type_float_vec(32); 728 const int max_vertices = 4; 729 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS]; 730 LLVMValueRef fetch_max; 731 732 arg_types[0] = llvm->context_ptr_type; /* context */ 733 arg_types[1] = llvm->vertex_header_ptr_type; /* vertex_header */ 734 arg_types[2] = llvm->buffer_ptr_type; /* vbuffers */ 735 arg_types[3] = LLVMPointerType(LLVMInt32Type(), 0); /* fetch_elts * */ 736 arg_types[4] = LLVMInt32Type(); /* fetch_count */ 737 arg_types[5] = LLVMInt32Type(); /* stride */ 738 arg_types[6] = llvm->vb_ptr_type; /* pipe_vertex_buffer's */ 739 740 func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0); 741 742 variant->function_elts = LLVMAddFunction(llvm->module, "draw_llvm_shader_elts", func_type); 743 LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv); 744 for(i = 0; i < Elements(arg_types); ++i) 745 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 746 LLVMAddAttribute(LLVMGetParam(variant->function_elts, i), LLVMNoAliasAttribute); 747 748 context_ptr = LLVMGetParam(variant->function_elts, 0); 749 io_ptr = LLVMGetParam(variant->function_elts, 1); 750 vbuffers_ptr = LLVMGetParam(variant->function_elts, 2); 751 fetch_elts = LLVMGetParam(variant->function_elts, 3); 752 fetch_count = LLVMGetParam(variant->function_elts, 4); 753 stride = LLVMGetParam(variant->function_elts, 5); 754 vb_ptr = LLVMGetParam(variant->function_elts, 6); 755 756 lp_build_name(context_ptr, "context"); 757 lp_build_name(io_ptr, "io"); 758 lp_build_name(vbuffers_ptr, "vbuffers"); 759 lp_build_name(fetch_elts, "fetch_elts"); 760 lp_build_name(fetch_count, "fetch_count"); 761 lp_build_name(stride, "stride"); 762 lp_build_name(vb_ptr, "vb"); 763 764 /* 765 * Function body 766 */ 767 768 block = LLVMAppendBasicBlock(variant->function_elts, "entry"); 769 builder = LLVMCreateBuilder(); 770 LLVMPositionBuilderAtEnd(builder, block); 771 772 lp_build_context_init(&bld, builder, vs_type); 773 lp_build_context_init(&bld_int, builder, lp_type_int(32)); 774 775 step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0); 776 777 fetch_max = LLVMBuildSub(builder, fetch_count, 778 LLVMConstInt(LLVMInt32Type(), 1, 0), 779 "fetch_max"); 780 781 lp_build_loop_begin(builder, LLVMConstInt(LLVMInt32Type(), 0, 0), &lp_loop); 782 { 783 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS]; 784 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } }; 785 LLVMValueRef io; 786 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS]; 787 788 io_itr = lp_loop.counter; 789 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 790#if DEBUG_STORE 791 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n", 792 io_itr, io, lp_loop.counter); 793#endif 794 for (i = 0; i < NUM_CHANNELS; ++i) { 795 LLVMValueRef true_index = LLVMBuildAdd( 796 builder, 797 lp_loop.counter, 798 LLVMConstInt(LLVMInt32Type(), i, 0), ""); 799 LLVMValueRef fetch_ptr; 800 801 /* make sure we're not out of bounds which can happen 802 * if fetch_count % 4 != 0, because on the last iteration 803 * a few of the 4 vertex fetches will be out of bounds */ 804 true_index = lp_build_min(&bld_int, true_index, fetch_max); 805 806 fetch_ptr = LLVMBuildGEP(builder, fetch_elts, 807 &true_index, 1, ""); 808 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt"); 809 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 810 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 811 LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(), 812 velem->vertex_buffer_index, 813 0); 814 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, 815 &vb_index, 1, ""); 816 generate_fetch(builder, vbuffers_ptr, 817 &aos_attribs[j][i], velem, vb, true_index); 818 } 819 } 820 convert_to_soa(builder, aos_attribs, inputs, 821 draw->pt.nr_vertex_elements); 822 823 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs; 824 generate_vs(llvm, 825 builder, 826 outputs, 827 ptr_aos, 828 context_ptr); 829 830 convert_to_aos(builder, io, outputs, 831 draw->vs.vertex_shader->info.num_outputs, 832 max_vertices); 833 } 834 lp_build_loop_end_cond(builder, fetch_count, step, LLVMIntUGE, &lp_loop); 835 836 LLVMBuildRetVoid(builder); 837 838 LLVMDisposeBuilder(builder); 839 840 /* 841 * Translate the LLVM IR into machine code. 842 */ 843#ifdef DEBUG 844 if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) { 845 LLVMDumpValue(variant->function_elts); 846 assert(0); 847 } 848#endif 849 850 LLVMRunFunctionPassManager(llvm->pass, variant->function_elts); 851 852 if (0) { 853 LLVMDumpValue(variant->function_elts); 854 debug_printf("\n"); 855 } 856 variant->jit_func_elts = (draw_jit_vert_func_elts)LLVMGetPointerToGlobal( 857 llvm->draw->engine, variant->function_elts); 858 859 if (0) 860 lp_disassemble(variant->jit_func_elts); 861} 862 863void 864draw_llvm_make_variant_key(struct draw_llvm *llvm, 865 struct draw_llvm_variant_key *key) 866{ 867 memset(key, 0, sizeof(struct draw_llvm_variant_key)); 868 869 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements; 870 871 memcpy(key->vertex_element, 872 llvm->draw->pt.vertex_element, 873 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements); 874 875 memcpy(&key->vs, 876 &llvm->draw->vs.vertex_shader->state, 877 sizeof(struct pipe_shader_state)); 878} 879