draw_llvm.c revision e9bf09a98a624e594bdea2503326bb693b8cf9b8
1#include "draw_llvm.h" 2 3#include "draw_context.h" 4#include "draw_vs.h" 5 6#include "gallivm/lp_bld_arit.h" 7#include "gallivm/lp_bld_struct.h" 8#include "gallivm/lp_bld_type.h" 9#include "gallivm/lp_bld_flow.h" 10#include "gallivm/lp_bld_debug.h" 11#include "gallivm/lp_bld_tgsi.h" 12#include "gallivm/lp_bld_printf.h" 13 14#include "tgsi/tgsi_exec.h" 15 16#include "util/u_cpu_detect.h" 17#include "util/u_string.h" 18 19#include <llvm-c/Transforms/Scalar.h> 20 21#define DEBUG_STORE 0 22 23 24/* generates the draw jit function */ 25static void 26draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var); 27static void 28draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var); 29 30static void 31init_globals(struct draw_llvm *llvm) 32{ 33 LLVMTypeRef texture_type; 34 35 /* struct draw_jit_texture */ 36 { 37 LLVMTypeRef elem_types[4]; 38 39 elem_types[DRAW_JIT_TEXTURE_WIDTH] = LLVMInt32Type(); 40 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type(); 41 elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type(); 42 elem_types[DRAW_JIT_TEXTURE_DATA] = LLVMPointerType(LLVMInt8Type(), 0); 43 44 texture_type = LLVMStructType(elem_types, Elements(elem_types), 0); 45 46 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width, 47 llvm->target, texture_type, 48 DRAW_JIT_TEXTURE_WIDTH); 49 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height, 50 llvm->target, texture_type, 51 DRAW_JIT_TEXTURE_HEIGHT); 52 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride, 53 llvm->target, texture_type, 54 DRAW_JIT_TEXTURE_STRIDE); 55 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data, 56 llvm->target, texture_type, 57 DRAW_JIT_TEXTURE_DATA); 58 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, 59 llvm->target, texture_type); 60 61 LLVMAddTypeName(llvm->module, "texture", texture_type); 62 } 63 64 65 /* struct draw_jit_context */ 66 { 67 LLVMTypeRef elem_types[3]; 68 LLVMTypeRef context_type; 69 70 elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */ 71 elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */ 72 elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */ 73 74 context_type = LLVMStructType(elem_types, Elements(elem_types), 0); 75 76 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants, 77 llvm->target, context_type, 0); 78 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants, 79 llvm->target, context_type, 1); 80 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures, 81 llvm->target, context_type, 82 DRAW_JIT_CONTEXT_TEXTURES_INDEX); 83 LP_CHECK_STRUCT_SIZE(struct draw_jit_context, 84 llvm->target, context_type); 85 86 LLVMAddTypeName(llvm->module, "draw_jit_context", context_type); 87 88 llvm->context_ptr_type = LLVMPointerType(context_type, 0); 89 } 90 { 91 LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0); 92 llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0); 93 } 94 /* struct pipe_vertex_buffer */ 95 { 96 LLVMTypeRef elem_types[4]; 97 LLVMTypeRef vb_type; 98 99 elem_types[0] = LLVMInt32Type(); 100 elem_types[1] = LLVMInt32Type(); 101 elem_types[2] = LLVMInt32Type(); 102 elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */ 103 104 vb_type = LLVMStructType(elem_types, Elements(elem_types), 0); 105 106 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride, 107 llvm->target, vb_type, 0); 108 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset, 109 llvm->target, vb_type, 2); 110 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, 111 llvm->target, vb_type); 112 113 LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type); 114 115 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0); 116 } 117} 118 119static LLVMTypeRef 120create_vertex_header(struct draw_llvm *llvm, int data_elems) 121{ 122 /* struct vertex_header */ 123 LLVMTypeRef elem_types[3]; 124 LLVMTypeRef vertex_header; 125 char struct_name[24]; 126 127 util_snprintf(struct_name, 23, "vertex_header%d", data_elems); 128 129 elem_types[0] = LLVMIntType(32); 130 elem_types[1] = LLVMArrayType(LLVMFloatType(), 4); 131 elem_types[2] = LLVMArrayType(elem_types[1], data_elems); 132 133 vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0); 134 135 /* these are bit-fields and we can't take address of them 136 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask, 137 llvm->target, vertex_header, 138 DRAW_JIT_VERTEX_CLIPMASK); 139 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag, 140 llvm->target, vertex_header, 141 DRAW_JIT_VERTEX_EDGEFLAG); 142 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad, 143 llvm->target, vertex_header, 144 DRAW_JIT_VERTEX_PAD); 145 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id, 146 llvm->target, vertex_header, 147 DRAW_JIT_VERTEX_VERTEX_ID); 148 */ 149 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip, 150 llvm->target, vertex_header, 151 DRAW_JIT_VERTEX_CLIP); 152 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data, 153 llvm->target, vertex_header, 154 DRAW_JIT_VERTEX_DATA); 155 156 LLVMAddTypeName(llvm->module, struct_name, vertex_header); 157 158 return LLVMPointerType(vertex_header, 0); 159} 160 161struct draw_llvm * 162draw_llvm_create(struct draw_context *draw) 163{ 164 struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm ); 165 166 util_cpu_detect(); 167 168 llvm->draw = draw; 169 llvm->engine = draw->engine; 170 171 debug_assert(llvm->engine); 172 173 llvm->module = LLVMModuleCreateWithName("draw_llvm"); 174 llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module); 175 176 LLVMAddModuleProvider(llvm->engine, llvm->provider); 177 178 llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine); 179 180 llvm->pass = LLVMCreateFunctionPassManager(llvm->provider); 181 LLVMAddTargetData(llvm->target, llvm->pass); 182 /* These are the passes currently listed in llvm-c/Transforms/Scalar.h, 183 * but there are more on SVN. */ 184 /* TODO: Add more passes */ 185 LLVMAddConstantPropagationPass(llvm->pass); 186 if(util_cpu_caps.has_sse4_1) { 187 /* FIXME: There is a bug in this pass, whereby the combination of fptosi 188 * and sitofp (necessary for trunc/floor/ceil/round implementation) 189 * somehow becomes invalid code. 190 */ 191 LLVMAddInstructionCombiningPass(llvm->pass); 192 } 193 LLVMAddPromoteMemoryToRegisterPass(llvm->pass); 194 LLVMAddGVNPass(llvm->pass); 195 LLVMAddCFGSimplificationPass(llvm->pass); 196 197 init_globals(llvm); 198 199 200#if 0 201 LLVMDumpModule(llvm->module); 202#endif 203 204 return llvm; 205} 206 207void 208draw_llvm_destroy(struct draw_llvm *llvm) 209{ 210 LLVMDisposePassManager(llvm->pass); 211 212 FREE(llvm); 213} 214 215struct draw_llvm_variant * 216draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs) 217{ 218 struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant)); 219 220 draw_llvm_make_variant_key(llvm, &variant->key); 221 222 llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs); 223 224 draw_llvm_generate(llvm, variant); 225 draw_llvm_generate_elts(llvm, variant); 226 227 return variant; 228} 229 230static void 231generate_vs(struct draw_llvm *llvm, 232 LLVMBuilderRef builder, 233 LLVMValueRef (*outputs)[NUM_CHANNELS], 234 const LLVMValueRef (*inputs)[NUM_CHANNELS], 235 LLVMValueRef context_ptr) 236{ 237 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens; 238 struct lp_type vs_type; 239 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr); 240 241 memset(&vs_type, 0, sizeof vs_type); 242 vs_type.floating = TRUE; /* floating point values */ 243 vs_type.sign = TRUE; /* values are signed */ 244 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 245 vs_type.width = 32; /* 32-bit float */ 246 vs_type.length = 4; /* 4 elements per vector */ 247#if 0 248 num_vs = 4; /* number of vertices per block */ 249#endif 250 251 /*tgsi_dump(tokens, 0);*/ 252 lp_build_tgsi_soa(builder, 253 tokens, 254 vs_type, 255 NULL /*struct lp_build_mask_context *mask*/, 256 consts_ptr, 257 NULL /*pos*/, 258 inputs, 259 outputs, 260 NULL/*sampler*/, 261 &llvm->draw->vs.vertex_shader->info); 262} 263 264#if DEBUG_STORE 265static void print_vectorf(LLVMBuilderRef builder, 266 LLVMValueRef vec) 267{ 268 LLVMValueRef val[4]; 269 val[0] = LLVMBuildExtractElement(builder, vec, 270 LLVMConstInt(LLVMInt32Type(), 0, 0), ""); 271 val[1] = LLVMBuildExtractElement(builder, vec, 272 LLVMConstInt(LLVMInt32Type(), 1, 0), ""); 273 val[2] = LLVMBuildExtractElement(builder, vec, 274 LLVMConstInt(LLVMInt32Type(), 2, 0), ""); 275 val[3] = LLVMBuildExtractElement(builder, vec, 276 LLVMConstInt(LLVMInt32Type(), 3, 0), ""); 277 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n", 278 val[0], val[1], val[2], val[3]); 279} 280#endif 281 282static void 283generate_fetch(LLVMBuilderRef builder, 284 LLVMValueRef vbuffers_ptr, 285 LLVMValueRef *res, 286 struct pipe_vertex_element *velem, 287 LLVMValueRef vbuf, 288 LLVMValueRef index) 289{ 290 LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0); 291 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr, 292 &indices, 1, ""); 293 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf); 294 LLVMValueRef vb_max_index = draw_jit_vbuffer_max_index(builder, vbuf); 295 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf); 296 LLVMValueRef cond; 297 LLVMValueRef stride; 298 299 cond = LLVMBuildICmp(builder, LLVMIntULE, index, vb_max_index, ""); 300 301 index = LLVMBuildSelect(builder, cond, index, vb_max_index, ""); 302 303 stride = LLVMBuildMul(builder, vb_stride, index, ""); 304 305 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer"); 306 307 stride = LLVMBuildAdd(builder, stride, 308 vb_buffer_offset, 309 ""); 310 stride = LLVMBuildAdd(builder, stride, 311 LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0), 312 ""); 313 314 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/ 315 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, ""); 316 317 *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format); 318} 319 320static LLVMValueRef 321aos_to_soa(LLVMBuilderRef builder, 322 LLVMValueRef val0, 323 LLVMValueRef val1, 324 LLVMValueRef val2, 325 LLVMValueRef val3, 326 LLVMValueRef channel) 327{ 328 LLVMValueRef ex, res; 329 330 ex = LLVMBuildExtractElement(builder, val0, 331 channel, ""); 332 res = LLVMBuildInsertElement(builder, 333 LLVMConstNull(LLVMTypeOf(val0)), 334 ex, 335 LLVMConstInt(LLVMInt32Type(), 0, 0), 336 ""); 337 338 ex = LLVMBuildExtractElement(builder, val1, 339 channel, ""); 340 res = LLVMBuildInsertElement(builder, 341 res, ex, 342 LLVMConstInt(LLVMInt32Type(), 1, 0), 343 ""); 344 345 ex = LLVMBuildExtractElement(builder, val2, 346 channel, ""); 347 res = LLVMBuildInsertElement(builder, 348 res, ex, 349 LLVMConstInt(LLVMInt32Type(), 2, 0), 350 ""); 351 352 ex = LLVMBuildExtractElement(builder, val3, 353 channel, ""); 354 res = LLVMBuildInsertElement(builder, 355 res, ex, 356 LLVMConstInt(LLVMInt32Type(), 3, 0), 357 ""); 358 359 return res; 360} 361 362static void 363soa_to_aos(LLVMBuilderRef builder, 364 LLVMValueRef soa[NUM_CHANNELS], 365 LLVMValueRef aos[NUM_CHANNELS]) 366{ 367 LLVMValueRef comp; 368 int i = 0; 369 370 debug_assert(NUM_CHANNELS == 4); 371 372 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0])); 373 aos[1] = aos[2] = aos[3] = aos[0]; 374 375 for (i = 0; i < NUM_CHANNELS; ++i) { 376 LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0); 377 378 comp = LLVMBuildExtractElement(builder, soa[i], 379 LLVMConstInt(LLVMInt32Type(), 0, 0), ""); 380 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, ""); 381 382 comp = LLVMBuildExtractElement(builder, soa[i], 383 LLVMConstInt(LLVMInt32Type(), 1, 0), ""); 384 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, ""); 385 386 comp = LLVMBuildExtractElement(builder, soa[i], 387 LLVMConstInt(LLVMInt32Type(), 2, 0), ""); 388 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, ""); 389 390 comp = LLVMBuildExtractElement(builder, soa[i], 391 LLVMConstInt(LLVMInt32Type(), 3, 0), ""); 392 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, ""); 393 394 } 395} 396 397static void 398convert_to_soa(LLVMBuilderRef builder, 399 LLVMValueRef (*aos)[NUM_CHANNELS], 400 LLVMValueRef (*soa)[NUM_CHANNELS], 401 int num_attribs) 402{ 403 int i; 404 405 debug_assert(NUM_CHANNELS == 4); 406 407 for (i = 0; i < num_attribs; ++i) { 408 LLVMValueRef val0 = aos[i][0]; 409 LLVMValueRef val1 = aos[i][1]; 410 LLVMValueRef val2 = aos[i][2]; 411 LLVMValueRef val3 = aos[i][3]; 412 413 soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3, 414 LLVMConstInt(LLVMInt32Type(), 0, 0)); 415 soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3, 416 LLVMConstInt(LLVMInt32Type(), 1, 0)); 417 soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3, 418 LLVMConstInt(LLVMInt32Type(), 2, 0)); 419 soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3, 420 LLVMConstInt(LLVMInt32Type(), 3, 0)); 421 } 422} 423 424static void 425store_aos(LLVMBuilderRef builder, 426 LLVMValueRef io_ptr, 427 LLVMValueRef index, 428 LLVMValueRef value) 429{ 430 LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr); 431 LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr); 432 LLVMValueRef indices[3]; 433 434 indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0); 435 indices[1] = index; 436 indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0); 437 438 /* undefined vertex */ 439 LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(), 440 0xffff, 0), id_ptr); 441 442#if DEBUG_STORE 443 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr); 444#endif 445#if 0 446 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr); 447 print_vectorf(builder, value);*/ 448 data_ptr = LLVMBuildBitCast(builder, data_ptr, 449 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0), 450 "datavec"); 451 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, ""); 452 453 LLVMBuildStore(builder, value, data_ptr); 454#else 455 { 456 LLVMValueRef x, y, z, w; 457 LLVMValueRef idx0, idx1, idx2, idx3; 458 LLVMValueRef gep0, gep1, gep2, gep3; 459 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, ""); 460 461 idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0); 462 idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0); 463 idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0); 464 idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0); 465 466 x = LLVMBuildExtractElement(builder, value, 467 idx0, ""); 468 y = LLVMBuildExtractElement(builder, value, 469 idx1, ""); 470 z = LLVMBuildExtractElement(builder, value, 471 idx2, ""); 472 w = LLVMBuildExtractElement(builder, value, 473 idx3, ""); 474 475 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, ""); 476 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, ""); 477 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, ""); 478 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, ""); 479 480 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n", 481 x, gep0, y, gep1, z, gep2, w, gep3);*/ 482 LLVMBuildStore(builder, x, gep0); 483 LLVMBuildStore(builder, y, gep1); 484 LLVMBuildStore(builder, z, gep2); 485 LLVMBuildStore(builder, w, gep3); 486 } 487#endif 488} 489 490static void 491store_aos_array(LLVMBuilderRef builder, 492 LLVMValueRef io_ptr, 493 LLVMValueRef aos[NUM_CHANNELS], 494 int attrib, 495 int num_outputs) 496{ 497 LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0); 498 LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0); 499 LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0); 500 LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0); 501 LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0); 502 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 503 504 debug_assert(NUM_CHANNELS == 4); 505 506 io0_ptr = LLVMBuildGEP(builder, io_ptr, 507 &ind0, 1, ""); 508 io1_ptr = LLVMBuildGEP(builder, io_ptr, 509 &ind1, 1, ""); 510 io2_ptr = LLVMBuildGEP(builder, io_ptr, 511 &ind2, 1, ""); 512 io3_ptr = LLVMBuildGEP(builder, io_ptr, 513 &ind3, 1, ""); 514 515#if DEBUG_STORE 516 lp_build_printf(builder, " io = %p, indexes[%d, %d, %d, %d]\n", 517 io_ptr, ind0, ind1, ind2, ind3); 518#endif 519 520 store_aos(builder, io0_ptr, attr_index, aos[0]); 521 store_aos(builder, io1_ptr, attr_index, aos[1]); 522 store_aos(builder, io2_ptr, attr_index, aos[2]); 523 store_aos(builder, io3_ptr, attr_index, aos[3]); 524} 525 526static void 527convert_to_aos(LLVMBuilderRef builder, 528 LLVMValueRef io, 529 LLVMValueRef (*outputs)[NUM_CHANNELS], 530 int num_outputs, 531 int max_vertices) 532{ 533 unsigned chan, attrib; 534 535#if DEBUG_STORE 536 lp_build_printf(builder, " # storing begin\n"); 537#endif 538 for (attrib = 0; attrib < num_outputs; ++attrib) { 539 LLVMValueRef soa[4]; 540 LLVMValueRef aos[4]; 541 for(chan = 0; chan < NUM_CHANNELS; ++chan) { 542 if(outputs[attrib][chan]) { 543 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 544 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]); 545 /*lp_build_printf(builder, "output %d : %d ", 546 LLVMConstInt(LLVMInt32Type(), attrib, 0), 547 LLVMConstInt(LLVMInt32Type(), chan, 0)); 548 print_vectorf(builder, out);*/ 549 soa[chan] = out; 550 } else 551 soa[chan] = 0; 552 } 553 soa_to_aos(builder, soa, aos); 554 store_aos_array(builder, 555 io, 556 aos, 557 attrib, 558 num_outputs); 559 } 560#if DEBUG_STORE 561 lp_build_printf(builder, " # storing end\n"); 562#endif 563} 564 565static void 566draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant) 567{ 568 LLVMTypeRef arg_types[7]; 569 LLVMTypeRef func_type; 570 LLVMValueRef context_ptr; 571 LLVMBasicBlockRef block; 572 LLVMBuilderRef builder; 573 LLVMValueRef start, end, count, stride, step, io_itr; 574 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 575 struct draw_context *draw = llvm->draw; 576 unsigned i, j; 577 struct lp_build_context bld; 578 struct lp_build_loop_state lp_loop; 579 struct lp_type vs_type = lp_type_float_vec(32); 580 const int max_vertices = 4; 581 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS]; 582 583 arg_types[0] = llvm->context_ptr_type; /* context */ 584 arg_types[1] = llvm->vertex_header_ptr_type; /* vertex_header */ 585 arg_types[2] = llvm->buffer_ptr_type; /* vbuffers */ 586 arg_types[3] = LLVMInt32Type(); /* start */ 587 arg_types[4] = LLVMInt32Type(); /* count */ 588 arg_types[5] = LLVMInt32Type(); /* stride */ 589 arg_types[6] = llvm->vb_ptr_type; /* pipe_vertex_buffer's */ 590 591 func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0); 592 593 variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type); 594 LLVMSetFunctionCallConv(variant->function, LLVMCCallConv); 595 for(i = 0; i < Elements(arg_types); ++i) 596 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 597 LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute); 598 599 context_ptr = LLVMGetParam(variant->function, 0); 600 io_ptr = LLVMGetParam(variant->function, 1); 601 vbuffers_ptr = LLVMGetParam(variant->function, 2); 602 start = LLVMGetParam(variant->function, 3); 603 count = LLVMGetParam(variant->function, 4); 604 stride = LLVMGetParam(variant->function, 5); 605 vb_ptr = LLVMGetParam(variant->function, 6); 606 607 lp_build_name(context_ptr, "context"); 608 lp_build_name(io_ptr, "io"); 609 lp_build_name(vbuffers_ptr, "vbuffers"); 610 lp_build_name(start, "start"); 611 lp_build_name(count, "count"); 612 lp_build_name(stride, "stride"); 613 lp_build_name(vb_ptr, "vb"); 614 615 /* 616 * Function body 617 */ 618 619 block = LLVMAppendBasicBlock(variant->function, "entry"); 620 builder = LLVMCreateBuilder(); 621 LLVMPositionBuilderAtEnd(builder, block); 622 623 lp_build_context_init(&bld, builder, vs_type); 624 625 end = lp_build_add(&bld, start, count); 626 627 step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0); 628 629#if DEBUG_STORE 630 lp_build_printf(builder, "start = %d, end = %d, step = %d\n", 631 start, end, step); 632#endif 633 lp_build_loop_begin(builder, start, &lp_loop); 634 { 635 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS]; 636 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } }; 637 LLVMValueRef io; 638 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS]; 639 640 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, ""); 641 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 642#if DEBUG_STORE 643 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n", 644 io_itr, io, lp_loop.counter); 645#endif 646 for (i = 0; i < NUM_CHANNELS; ++i) { 647 LLVMValueRef true_index = LLVMBuildAdd( 648 builder, 649 lp_loop.counter, 650 LLVMConstInt(LLVMInt32Type(), i, 0), ""); 651 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 652 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 653 LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(), 654 velem->vertex_buffer_index, 655 0); 656 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, 657 &vb_index, 1, ""); 658 generate_fetch(builder, vbuffers_ptr, 659 &aos_attribs[j][i], velem, vb, true_index); 660 } 661 } 662 convert_to_soa(builder, aos_attribs, inputs, 663 draw->pt.nr_vertex_elements); 664 665 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs; 666 generate_vs(llvm, 667 builder, 668 outputs, 669 ptr_aos, 670 context_ptr); 671 672 convert_to_aos(builder, io, outputs, 673 draw->vs.vertex_shader->info.num_outputs, 674 max_vertices); 675 } 676 lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop); 677 678 LLVMBuildRetVoid(builder); 679 680 LLVMDisposeBuilder(builder); 681 682 /* 683 * Translate the LLVM IR into machine code. 684 */ 685#ifdef DEBUG 686 if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) { 687 LLVMDumpValue(variant->function); 688 assert(0); 689 } 690#endif 691 692 LLVMRunFunctionPassManager(llvm->pass, variant->function); 693 694 if (0) { 695 LLVMDumpValue(variant->function); 696 debug_printf("\n"); 697 } 698 variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function); 699 700 if (0) 701 lp_disassemble(variant->jit_func); 702} 703 704 705static void 706draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant) 707{ 708 LLVMTypeRef arg_types[7]; 709 LLVMTypeRef func_type; 710 LLVMValueRef context_ptr; 711 LLVMBasicBlockRef block; 712 LLVMBuilderRef builder; 713 LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr; 714 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 715 struct draw_context *draw = llvm->draw; 716 unsigned i, j; 717 struct lp_build_context bld; 718 struct lp_build_context bld_int; 719 struct lp_build_loop_state lp_loop; 720 struct lp_type vs_type = lp_type_float_vec(32); 721 const int max_vertices = 4; 722 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS]; 723 LLVMValueRef fetch_max; 724 725 arg_types[0] = llvm->context_ptr_type; /* context */ 726 arg_types[1] = llvm->vertex_header_ptr_type; /* vertex_header */ 727 arg_types[2] = llvm->buffer_ptr_type; /* vbuffers */ 728 arg_types[3] = LLVMPointerType(LLVMInt32Type(), 0); /* fetch_elts * */ 729 arg_types[4] = LLVMInt32Type(); /* fetch_count */ 730 arg_types[5] = LLVMInt32Type(); /* stride */ 731 arg_types[6] = llvm->vb_ptr_type; /* pipe_vertex_buffer's */ 732 733 func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0); 734 735 variant->function_elts = LLVMAddFunction(llvm->module, "draw_llvm_shader_elts", func_type); 736 LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv); 737 for(i = 0; i < Elements(arg_types); ++i) 738 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 739 LLVMAddAttribute(LLVMGetParam(variant->function_elts, i), LLVMNoAliasAttribute); 740 741 context_ptr = LLVMGetParam(variant->function_elts, 0); 742 io_ptr = LLVMGetParam(variant->function_elts, 1); 743 vbuffers_ptr = LLVMGetParam(variant->function_elts, 2); 744 fetch_elts = LLVMGetParam(variant->function_elts, 3); 745 fetch_count = LLVMGetParam(variant->function_elts, 4); 746 stride = LLVMGetParam(variant->function_elts, 5); 747 vb_ptr = LLVMGetParam(variant->function_elts, 6); 748 749 lp_build_name(context_ptr, "context"); 750 lp_build_name(io_ptr, "io"); 751 lp_build_name(vbuffers_ptr, "vbuffers"); 752 lp_build_name(fetch_elts, "fetch_elts"); 753 lp_build_name(fetch_count, "fetch_count"); 754 lp_build_name(stride, "stride"); 755 lp_build_name(vb_ptr, "vb"); 756 757 /* 758 * Function body 759 */ 760 761 block = LLVMAppendBasicBlock(variant->function_elts, "entry"); 762 builder = LLVMCreateBuilder(); 763 LLVMPositionBuilderAtEnd(builder, block); 764 765 lp_build_context_init(&bld, builder, vs_type); 766 lp_build_context_init(&bld_int, builder, lp_type_int(32)); 767 768 step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0); 769 770 fetch_max = LLVMBuildSub(builder, fetch_count, 771 LLVMConstInt(LLVMInt32Type(), 1, 0), 772 "fetch_max"); 773 774 lp_build_loop_begin(builder, LLVMConstInt(LLVMInt32Type(), 0, 0), &lp_loop); 775 { 776 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS]; 777 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } }; 778 LLVMValueRef io; 779 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS]; 780 781 io_itr = lp_loop.counter; 782 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 783#if DEBUG_STORE 784 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n", 785 io_itr, io, lp_loop.counter); 786#endif 787 for (i = 0; i < NUM_CHANNELS; ++i) { 788 LLVMValueRef true_index = LLVMBuildAdd( 789 builder, 790 lp_loop.counter, 791 LLVMConstInt(LLVMInt32Type(), i, 0), ""); 792 LLVMValueRef fetch_ptr; 793 794 /* make sure we're not out of bounds which can happen 795 * if fetch_count % 4 != 0, because on the last iteration 796 * a few of the 4 vertex fetches will be out of bounds */ 797 true_index = lp_build_min(&bld_int, true_index, fetch_max); 798 799 fetch_ptr = LLVMBuildGEP(builder, fetch_elts, 800 &true_index, 1, ""); 801 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt"); 802 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 803 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 804 LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(), 805 velem->vertex_buffer_index, 806 0); 807 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, 808 &vb_index, 1, ""); 809 generate_fetch(builder, vbuffers_ptr, 810 &aos_attribs[j][i], velem, vb, true_index); 811 } 812 } 813 convert_to_soa(builder, aos_attribs, inputs, 814 draw->pt.nr_vertex_elements); 815 816 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs; 817 generate_vs(llvm, 818 builder, 819 outputs, 820 ptr_aos, 821 context_ptr); 822 823 convert_to_aos(builder, io, outputs, 824 draw->vs.vertex_shader->info.num_outputs, 825 max_vertices); 826 } 827 lp_build_loop_end_cond(builder, fetch_count, step, LLVMIntUGE, &lp_loop); 828 829 LLVMBuildRetVoid(builder); 830 831 LLVMDisposeBuilder(builder); 832 833 /* 834 * Translate the LLVM IR into machine code. 835 */ 836#ifdef DEBUG 837 if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) { 838 LLVMDumpValue(variant->function_elts); 839 assert(0); 840 } 841#endif 842 843 LLVMRunFunctionPassManager(llvm->pass, variant->function_elts); 844 845 if (0) { 846 LLVMDumpValue(variant->function_elts); 847 debug_printf("\n"); 848 } 849 variant->jit_func_elts = (draw_jit_vert_func_elts)LLVMGetPointerToGlobal( 850 llvm->draw->engine, variant->function_elts); 851 852 if (0) 853 lp_disassemble(variant->jit_func_elts); 854} 855 856void 857draw_llvm_make_variant_key(struct draw_llvm *llvm, 858 struct draw_llvm_variant_key *key) 859{ 860 memset(key, 0, sizeof(struct draw_llvm_variant_key)); 861 862 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements; 863 864 memcpy(key->vertex_element, 865 llvm->draw->pt.vertex_element, 866 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements); 867 868 memcpy(&key->vs, 869 &llvm->draw->vs.vertex_shader->state, 870 sizeof(struct pipe_shader_state)); 871} 872