draw_llvm.c revision 5f996e2b1d09dad64c088ccabb1a4a53ebfb8102
1/************************************************************************** 2 * 3 * Copyright 2010 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28#include "draw_llvm.h" 29 30#include "draw_context.h" 31#include "draw_vs.h" 32 33#include "gallivm/lp_bld_arit.h" 34#include "gallivm/lp_bld_logic.h" 35#include "gallivm/lp_bld_const.h" 36#include "gallivm/lp_bld_swizzle.h" 37#include "gallivm/lp_bld_struct.h" 38#include "gallivm/lp_bld_type.h" 39#include "gallivm/lp_bld_flow.h" 40#include "gallivm/lp_bld_debug.h" 41#include "gallivm/lp_bld_tgsi.h" 42#include "gallivm/lp_bld_printf.h" 43#include "gallivm/lp_bld_intr.h" 44#include "gallivm/lp_bld_init.h" 45#include "gallivm/lp_bld_type.h" 46 47#include "tgsi/tgsi_exec.h" 48#include "tgsi/tgsi_dump.h" 49 50#include "util/u_math.h" 51#include "util/u_pointer.h" 52#include "util/u_string.h" 53#include "util/u_simple_list.h" 54 55 56#define DEBUG_STORE 0 57 58 59/** 60 * This function is called by the gallivm "garbage collector" when 61 * the LLVM global data structures are freed. We must free all LLVM-related 62 * data. Specifically, all JIT'd shader variants. 63 */ 64static void 65draw_llvm_garbage_collect_callback(void *cb_data) 66{ 67 struct draw_llvm *llvm = (struct draw_llvm *) cb_data; 68 struct draw_llvm_variant_list_item *li; 69 70 /* free all shader variants */ 71 li = first_elem(&llvm->vs_variants_list); 72 while (!at_end(&llvm->vs_variants_list, li)) { 73 struct draw_llvm_variant_list_item *next = next_elem(li); 74 draw_llvm_destroy_variant(li->base); 75 li = next; 76 } 77 78 /* Null-out these pointers so they get remade next time they're needed. 79 * See the accessor functions below. 80 */ 81 llvm->context_ptr_type = NULL; 82 llvm->buffer_ptr_type = NULL; 83 llvm->vb_ptr_type = NULL; 84 llvm->vertex_header_ptr_type = NULL; 85} 86 87 88static void 89draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var); 90 91static void 92draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var); 93 94 95/** 96 * Create LLVM type for struct draw_jit_texture 97 */ 98static LLVMTypeRef 99create_jit_texture_type(struct gallivm_state *gallivm) 100{ 101 LLVMTargetDataRef target = gallivm->target; 102 LLVMTypeRef texture_type; 103 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS]; 104 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 105 106 elem_types[DRAW_JIT_TEXTURE_WIDTH] = 107 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = 108 elem_types[DRAW_JIT_TEXTURE_DEPTH] = 109 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type; 110 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] = 111 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] = 112 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS); 113 elem_types[DRAW_JIT_TEXTURE_DATA] = 114 LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0), 115 PIPE_MAX_TEXTURE_LEVELS); 116 elem_types[DRAW_JIT_TEXTURE_MIN_LOD] = 117 elem_types[DRAW_JIT_TEXTURE_MAX_LOD] = 118 elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context); 119 elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] = 120 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 121 122 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types, 123 Elements(elem_types), 0); 124 125 /* Make sure the target's struct layout cache doesn't return 126 * stale/invalid data. 127 */ 128 LLVMInvalidateStructLayout(gallivm->target, texture_type); 129 130 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width, 131 target, texture_type, 132 DRAW_JIT_TEXTURE_WIDTH); 133 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height, 134 target, texture_type, 135 DRAW_JIT_TEXTURE_HEIGHT); 136 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth, 137 target, texture_type, 138 DRAW_JIT_TEXTURE_DEPTH); 139 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level, 140 target, texture_type, 141 DRAW_JIT_TEXTURE_LAST_LEVEL); 142 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride, 143 target, texture_type, 144 DRAW_JIT_TEXTURE_ROW_STRIDE); 145 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride, 146 target, texture_type, 147 DRAW_JIT_TEXTURE_IMG_STRIDE); 148 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data, 149 target, texture_type, 150 DRAW_JIT_TEXTURE_DATA); 151 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod, 152 target, texture_type, 153 DRAW_JIT_TEXTURE_MIN_LOD); 154 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod, 155 target, texture_type, 156 DRAW_JIT_TEXTURE_MAX_LOD); 157 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias, 158 target, texture_type, 159 DRAW_JIT_TEXTURE_LOD_BIAS); 160 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color, 161 target, texture_type, 162 DRAW_JIT_TEXTURE_BORDER_COLOR); 163 164 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type); 165 166 return texture_type; 167} 168 169 170/** 171 * Create LLVM type for struct draw_jit_texture 172 */ 173static LLVMTypeRef 174create_jit_context_type(struct gallivm_state *gallivm, 175 LLVMTypeRef texture_type) 176{ 177 LLVMTargetDataRef target = gallivm->target; 178 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 179 LLVMTypeRef elem_types[5]; 180 LLVMTypeRef context_type; 181 182 elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */ 183 elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */ 184 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), 12), 0); /* planes */ 185 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */ 186 elem_types[4] = LLVMArrayType(texture_type, 187 PIPE_MAX_VERTEX_SAMPLERS); /* textures */ 188 189 context_type = LLVMStructTypeInContext(gallivm->context, elem_types, 190 Elements(elem_types), 0); 191 192 LLVMInvalidateStructLayout(gallivm->target, context_type); 193 194 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants, 195 target, context_type, 0); 196 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants, 197 target, context_type, 1); 198 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes, 199 target, context_type, 2); 200 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures, 201 target, context_type, 202 DRAW_JIT_CTX_TEXTURES); 203 LP_CHECK_STRUCT_SIZE(struct draw_jit_context, 204 target, context_type); 205 206 return context_type; 207} 208 209 210/** 211 * Create LLVM type for struct pipe_vertex_buffer 212 */ 213static LLVMTypeRef 214create_jit_vertex_buffer_type(struct gallivm_state *gallivm) 215{ 216 LLVMTargetDataRef target = gallivm->target; 217 LLVMTypeRef elem_types[3]; 218 LLVMTypeRef vb_type; 219 220 elem_types[0] = 221 elem_types[1] = LLVMInt32TypeInContext(gallivm->context); 222 elem_types[2] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */ 223 224 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types, 225 Elements(elem_types), 0); 226 227 LLVMInvalidateStructLayout(gallivm->target, vb_type); 228 229 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride, 230 target, vb_type, 0); 231 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset, 232 target, vb_type, 1); 233 234 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type); 235 236 return vb_type; 237} 238 239 240/** 241 * Create LLVM type for struct vertex_header; 242 */ 243static LLVMTypeRef 244create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems) 245{ 246 LLVMTargetDataRef target = gallivm->target; 247 LLVMTypeRef elem_types[3]; 248 LLVMTypeRef vertex_header; 249 char struct_name[24]; 250 251 util_snprintf(struct_name, 23, "vertex_header%d", data_elems); 252 253 elem_types[0] = LLVMIntTypeInContext(gallivm->context, 32); 254 elem_types[1] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 255 elem_types[2] = LLVMArrayType(elem_types[1], data_elems); 256 257 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types, 258 Elements(elem_types), 0); 259 260 LLVMInvalidateStructLayout(gallivm->target, vertex_header); 261 262 /* these are bit-fields and we can't take address of them 263 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask, 264 target, vertex_header, 265 DRAW_JIT_VERTEX_CLIPMASK); 266 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag, 267 target, vertex_header, 268 DRAW_JIT_VERTEX_EDGEFLAG); 269 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad, 270 target, vertex_header, 271 DRAW_JIT_VERTEX_PAD); 272 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id, 273 target, vertex_header, 274 DRAW_JIT_VERTEX_VERTEX_ID); 275 */ 276 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip, 277 target, vertex_header, 278 DRAW_JIT_VERTEX_CLIP); 279 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data, 280 target, vertex_header, 281 DRAW_JIT_VERTEX_DATA); 282 283 LLVMAddTypeName(gallivm->module, struct_name, vertex_header); 284 285 return vertex_header; 286} 287 288 289/** 290 * Create LLVM types for various structures. 291 */ 292static void 293create_jit_types(struct draw_llvm *llvm) 294{ 295 struct gallivm_state *gallivm = llvm->gallivm; 296 LLVMTypeRef texture_type, context_type, buffer_type, vb_type; 297 298 texture_type = create_jit_texture_type(gallivm); 299 LLVMAddTypeName(gallivm->module, "texture", texture_type); 300 301 context_type = create_jit_context_type(gallivm, texture_type); 302 LLVMAddTypeName(gallivm->module, "draw_jit_context", context_type); 303 llvm->context_ptr_type = LLVMPointerType(context_type, 0); 304 305 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0); 306 LLVMAddTypeName(gallivm->module, "buffer", buffer_type); 307 llvm->buffer_ptr_type = LLVMPointerType(buffer_type, 0); 308 309 vb_type = create_jit_vertex_buffer_type(gallivm); 310 LLVMAddTypeName(gallivm->module, "pipe_vertex_buffer", vb_type); 311 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0); 312} 313 314 315static LLVMTypeRef 316get_context_ptr_type(struct draw_llvm *llvm) 317{ 318 if (!llvm->context_ptr_type) 319 create_jit_types(llvm); 320 return llvm->context_ptr_type; 321} 322 323 324static LLVMTypeRef 325get_buffer_ptr_type(struct draw_llvm *llvm) 326{ 327 if (!llvm->buffer_ptr_type) 328 create_jit_types(llvm); 329 return llvm->buffer_ptr_type; 330} 331 332 333static LLVMTypeRef 334get_vb_ptr_type(struct draw_llvm *llvm) 335{ 336 if (!llvm->vb_ptr_type) 337 create_jit_types(llvm); 338 return llvm->vb_ptr_type; 339} 340 341static LLVMTypeRef 342get_vertex_header_ptr_type(struct draw_llvm *llvm) 343{ 344 if (!llvm->vertex_header_ptr_type) 345 create_jit_types(llvm); 346 return llvm->vertex_header_ptr_type; 347} 348 349 350/** 351 * Create per-context LLVM info. 352 */ 353struct draw_llvm * 354draw_llvm_create(struct draw_context *draw, struct gallivm_state *gallivm) 355{ 356 struct draw_llvm *llvm; 357 358 llvm = CALLOC_STRUCT( draw_llvm ); 359 if (!llvm) 360 return NULL; 361 362 lp_build_init(); 363 364 llvm->draw = draw; 365 llvm->gallivm = gallivm; 366 367 if (gallivm_debug & GALLIVM_DEBUG_IR) { 368 LLVMDumpModule(llvm->gallivm->module); 369 } 370 371 llvm->nr_variants = 0; 372 make_empty_list(&llvm->vs_variants_list); 373 374 gallivm_register_garbage_collector_callback( 375 draw_llvm_garbage_collect_callback, llvm); 376 377 return llvm; 378} 379 380 381/** 382 * Free per-context LLVM info. 383 */ 384void 385draw_llvm_destroy(struct draw_llvm *llvm) 386{ 387 gallivm_remove_garbage_collector_callback( 388 draw_llvm_garbage_collect_callback, llvm); 389 390 /* XXX free other draw_llvm data? */ 391 FREE(llvm); 392} 393 394 395/** 396 * Create LLVM-generated code for a vertex shader. 397 */ 398struct draw_llvm_variant * 399draw_llvm_create_variant(struct draw_llvm *llvm, 400 unsigned num_inputs, 401 const struct draw_llvm_variant_key *key) 402{ 403 struct draw_llvm_variant *variant; 404 struct llvm_vertex_shader *shader = 405 llvm_vertex_shader(llvm->draw->vs.vertex_shader); 406 LLVMTypeRef vertex_header; 407 408 variant = MALLOC(sizeof *variant + 409 shader->variant_key_size - 410 sizeof variant->key); 411 if (variant == NULL) 412 return NULL; 413 414 variant->llvm = llvm; 415 416 memcpy(&variant->key, key, shader->variant_key_size); 417 418 vertex_header = create_jit_vertex_header(llvm->gallivm, num_inputs); 419 420 llvm->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0); 421 422 draw_llvm_generate(llvm, variant); 423 draw_llvm_generate_elts(llvm, variant); 424 425 variant->shader = shader; 426 variant->list_item_global.base = variant; 427 variant->list_item_local.base = variant; 428 /*variant->no = */shader->variants_created++; 429 variant->list_item_global.base = variant; 430 431 return variant; 432} 433 434static void 435generate_vs(struct draw_llvm *llvm, 436 LLVMBuilderRef builder, 437 LLVMValueRef (*outputs)[NUM_CHANNELS], 438 const LLVMValueRef (*inputs)[NUM_CHANNELS], 439 LLVMValueRef system_values_array, 440 LLVMValueRef context_ptr, 441 struct lp_build_sampler_soa *draw_sampler, 442 boolean clamp_vertex_color) 443{ 444 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens; 445 struct lp_type vs_type; 446 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(llvm->gallivm, context_ptr); 447 struct lp_build_sampler_soa *sampler = 0; 448 449 memset(&vs_type, 0, sizeof vs_type); 450 vs_type.floating = TRUE; /* floating point values */ 451 vs_type.sign = TRUE; /* values are signed */ 452 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 453 vs_type.width = 32; /* 32-bit float */ 454 vs_type.length = 4; /* 4 elements per vector */ 455#if 0 456 num_vs = 4; /* number of vertices per block */ 457#endif 458 459 if (gallivm_debug & GALLIVM_DEBUG_IR) { 460 tgsi_dump(tokens, 0); 461 } 462 463 if (llvm->draw->num_sampler_views && 464 llvm->draw->num_samplers) 465 sampler = draw_sampler; 466 467 lp_build_tgsi_soa(llvm->gallivm, 468 tokens, 469 vs_type, 470 NULL /*struct lp_build_mask_context *mask*/, 471 consts_ptr, 472 system_values_array, 473 NULL /*pos*/, 474 inputs, 475 outputs, 476 sampler, 477 &llvm->draw->vs.vertex_shader->info); 478 479 if(clamp_vertex_color) 480 { 481 LLVMValueRef out; 482 unsigned chan, attrib; 483 struct lp_build_context bld; 484 struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info; 485 lp_build_context_init(&bld, llvm->gallivm, vs_type); 486 487 for (attrib = 0; attrib < info->num_outputs; ++attrib) { 488 for(chan = 0; chan < NUM_CHANNELS; ++chan) { 489 if(outputs[attrib][chan]) { 490 switch (info->output_semantic_name[attrib]) { 491 case TGSI_SEMANTIC_COLOR: 492 case TGSI_SEMANTIC_BCOLOR: 493 out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 494 out = lp_build_clamp(&bld, out, bld.zero, bld.one); 495 LLVMBuildStore(builder, out, outputs[attrib][chan]); 496 break; 497 } 498 } 499 } 500 } 501 } 502} 503 504#if DEBUG_STORE 505static void print_vectorf(LLVMBuilderRef builder, 506 LLVMValueRef vec) 507{ 508 LLVMValueRef val[4]; 509 val[0] = LLVMBuildExtractElement(builder, vec, 510 lp_build_const_int32(gallivm, 0), ""); 511 val[1] = LLVMBuildExtractElement(builder, vec, 512 lp_build_const_int32(gallivm, 1), ""); 513 val[2] = LLVMBuildExtractElement(builder, vec, 514 lp_build_const_int32(gallivm, 2), ""); 515 val[3] = LLVMBuildExtractElement(builder, vec, 516 lp_build_const_int32(gallivm, 3), ""); 517 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n", 518 val[0], val[1], val[2], val[3]); 519} 520#endif 521 522static void 523generate_fetch(struct gallivm_state *gallivm, 524 LLVMValueRef vbuffers_ptr, 525 LLVMValueRef *res, 526 struct pipe_vertex_element *velem, 527 LLVMValueRef vbuf, 528 LLVMValueRef index, 529 LLVMValueRef instance_id) 530{ 531 LLVMBuilderRef builder = gallivm->builder; 532 LLVMValueRef indices = 533 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 534 velem->vertex_buffer_index, 0); 535 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr, 536 &indices, 1, ""); 537 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf); 538 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf); 539 LLVMValueRef stride; 540 541 if (velem->instance_divisor) { 542 /* array index = instance_id / instance_divisor */ 543 index = LLVMBuildUDiv(builder, instance_id, 544 lp_build_const_int32(gallivm, velem->instance_divisor), 545 "instance_divisor"); 546 } 547 548 stride = LLVMBuildMul(builder, vb_stride, index, ""); 549 550 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer"); 551 552 stride = LLVMBuildAdd(builder, stride, 553 vb_buffer_offset, 554 ""); 555 stride = LLVMBuildAdd(builder, stride, 556 lp_build_const_int32(gallivm, velem->src_offset), 557 ""); 558 559 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/ 560 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, ""); 561 562 *res = draw_llvm_translate_from(gallivm, vbuffer_ptr, velem->src_format); 563} 564 565static LLVMValueRef 566aos_to_soa(struct gallivm_state *gallivm, 567 LLVMValueRef val0, 568 LLVMValueRef val1, 569 LLVMValueRef val2, 570 LLVMValueRef val3, 571 LLVMValueRef channel) 572{ 573 LLVMBuilderRef builder = gallivm->builder; 574 LLVMValueRef ex, res; 575 576 ex = LLVMBuildExtractElement(builder, val0, 577 channel, ""); 578 res = LLVMBuildInsertElement(builder, 579 LLVMConstNull(LLVMTypeOf(val0)), 580 ex, 581 lp_build_const_int32(gallivm, 0), 582 ""); 583 584 ex = LLVMBuildExtractElement(builder, val1, 585 channel, ""); 586 res = LLVMBuildInsertElement(builder, 587 res, ex, 588 lp_build_const_int32(gallivm, 1), 589 ""); 590 591 ex = LLVMBuildExtractElement(builder, val2, 592 channel, ""); 593 res = LLVMBuildInsertElement(builder, 594 res, ex, 595 lp_build_const_int32(gallivm, 2), 596 ""); 597 598 ex = LLVMBuildExtractElement(builder, val3, 599 channel, ""); 600 res = LLVMBuildInsertElement(builder, 601 res, ex, 602 lp_build_const_int32(gallivm, 3), 603 ""); 604 605 return res; 606} 607 608static void 609soa_to_aos(struct gallivm_state *gallivm, 610 LLVMValueRef soa[NUM_CHANNELS], 611 LLVMValueRef aos[NUM_CHANNELS]) 612{ 613 LLVMBuilderRef builder = gallivm->builder; 614 LLVMValueRef comp; 615 int i = 0; 616 617 debug_assert(NUM_CHANNELS == 4); 618 619 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0])); 620 aos[1] = aos[2] = aos[3] = aos[0]; 621 622 for (i = 0; i < NUM_CHANNELS; ++i) { 623 LLVMValueRef channel = lp_build_const_int32(gallivm, i); 624 625 comp = LLVMBuildExtractElement(builder, soa[i], 626 lp_build_const_int32(gallivm, 0), ""); 627 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, ""); 628 629 comp = LLVMBuildExtractElement(builder, soa[i], 630 lp_build_const_int32(gallivm, 1), ""); 631 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, ""); 632 633 comp = LLVMBuildExtractElement(builder, soa[i], 634 lp_build_const_int32(gallivm, 2), ""); 635 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, ""); 636 637 comp = LLVMBuildExtractElement(builder, soa[i], 638 lp_build_const_int32(gallivm, 3), ""); 639 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, ""); 640 641 } 642} 643 644static void 645convert_to_soa(struct gallivm_state *gallivm, 646 LLVMValueRef (*aos)[NUM_CHANNELS], 647 LLVMValueRef (*soa)[NUM_CHANNELS], 648 int num_attribs) 649{ 650 int i; 651 652 debug_assert(NUM_CHANNELS == 4); 653 654 for (i = 0; i < num_attribs; ++i) { 655 LLVMValueRef val0 = aos[i][0]; 656 LLVMValueRef val1 = aos[i][1]; 657 LLVMValueRef val2 = aos[i][2]; 658 LLVMValueRef val3 = aos[i][3]; 659 660 soa[i][0] = aos_to_soa(gallivm, val0, val1, val2, val3, 661 lp_build_const_int32(gallivm, 0)); 662 soa[i][1] = aos_to_soa(gallivm, val0, val1, val2, val3, 663 lp_build_const_int32(gallivm, 1)); 664 soa[i][2] = aos_to_soa(gallivm, val0, val1, val2, val3, 665 lp_build_const_int32(gallivm, 2)); 666 soa[i][3] = aos_to_soa(gallivm, val0, val1, val2, val3, 667 lp_build_const_int32(gallivm, 3)); 668 } 669} 670 671static void 672store_aos(struct gallivm_state *gallivm, 673 LLVMValueRef io_ptr, 674 LLVMValueRef index, 675 LLVMValueRef value, 676 LLVMValueRef clipmask) 677{ 678 LLVMBuilderRef builder = gallivm->builder; 679 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptr); 680 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr); 681 LLVMValueRef indices[3]; 682 LLVMValueRef val, shift; 683 684 indices[0] = lp_build_const_int32(gallivm, 0); 685 indices[1] = index; 686 indices[2] = lp_build_const_int32(gallivm, 0); 687 688 /* initialize vertex id:16 = 0xffff, pad:3 = 0, edgeflag:1 = 1 */ 689 val = lp_build_const_int32(gallivm, 0xffff1); 690 shift = lp_build_const_int32(gallivm, 12); 691 val = LLVMBuildShl(builder, val, shift, ""); 692 /* add clipmask:12 */ 693 val = LLVMBuildOr(builder, val, clipmask, ""); 694 695 /* store vertex header */ 696 LLVMBuildStore(builder, val, id_ptr); 697 698 699#if DEBUG_STORE 700 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr); 701#endif 702#if 0 703 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr); 704 print_vectorf(builder, value);*/ 705 data_ptr = LLVMBuildBitCast(builder, data_ptr, 706 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0), 0), 707 "datavec"); 708 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, ""); 709 710 LLVMBuildStore(builder, value, data_ptr); 711#else 712 { 713 LLVMValueRef x, y, z, w; 714 LLVMValueRef idx0, idx1, idx2, idx3; 715 LLVMValueRef gep0, gep1, gep2, gep3; 716 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, ""); 717 718 idx0 = lp_build_const_int32(gallivm, 0); 719 idx1 = lp_build_const_int32(gallivm, 1); 720 idx2 = lp_build_const_int32(gallivm, 2); 721 idx3 = lp_build_const_int32(gallivm, 3); 722 723 x = LLVMBuildExtractElement(builder, value, 724 idx0, ""); 725 y = LLVMBuildExtractElement(builder, value, 726 idx1, ""); 727 z = LLVMBuildExtractElement(builder, value, 728 idx2, ""); 729 w = LLVMBuildExtractElement(builder, value, 730 idx3, ""); 731 732 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, ""); 733 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, ""); 734 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, ""); 735 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, ""); 736 737 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n", 738 x, gep0, y, gep1, z, gep2, w, gep3);*/ 739 LLVMBuildStore(builder, x, gep0); 740 LLVMBuildStore(builder, y, gep1); 741 LLVMBuildStore(builder, z, gep2); 742 LLVMBuildStore(builder, w, gep3); 743 } 744#endif 745} 746 747static void 748store_aos_array(struct gallivm_state *gallivm, 749 LLVMValueRef io_ptr, 750 LLVMValueRef aos[NUM_CHANNELS], 751 int attrib, 752 int num_outputs, 753 LLVMValueRef clipmask) 754{ 755 LLVMBuilderRef builder = gallivm->builder; 756 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib); 757 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0); 758 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1); 759 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2); 760 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3); 761 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 762 LLVMValueRef clipmask0, clipmask1, clipmask2, clipmask3; 763 764 debug_assert(NUM_CHANNELS == 4); 765 766 io0_ptr = LLVMBuildGEP(builder, io_ptr, 767 &ind0, 1, ""); 768 io1_ptr = LLVMBuildGEP(builder, io_ptr, 769 &ind1, 1, ""); 770 io2_ptr = LLVMBuildGEP(builder, io_ptr, 771 &ind2, 1, ""); 772 io3_ptr = LLVMBuildGEP(builder, io_ptr, 773 &ind3, 1, ""); 774 775 clipmask0 = LLVMBuildExtractElement(builder, clipmask, 776 ind0, ""); 777 clipmask1 = LLVMBuildExtractElement(builder, clipmask, 778 ind1, ""); 779 clipmask2 = LLVMBuildExtractElement(builder, clipmask, 780 ind2, ""); 781 clipmask3 = LLVMBuildExtractElement(builder, clipmask, 782 ind3, ""); 783 784#if DEBUG_STORE 785 lp_build_printf(builder, "io = %p, indexes[%d, %d, %d, %d]\n, clipmask0 = %x, clipmask1 = %x, clipmask2 = %x, clipmask3 = %x\n", 786 io_ptr, ind0, ind1, ind2, ind3, clipmask0, clipmask1, clipmask2, clipmask3); 787#endif 788 /* store for each of the 4 vertices */ 789 store_aos(gallivm, io0_ptr, attr_index, aos[0], clipmask0); 790 store_aos(gallivm, io1_ptr, attr_index, aos[1], clipmask1); 791 store_aos(gallivm, io2_ptr, attr_index, aos[2], clipmask2); 792 store_aos(gallivm, io3_ptr, attr_index, aos[3], clipmask3); 793} 794 795static void 796convert_to_aos(struct gallivm_state *gallivm, 797 LLVMValueRef io, 798 LLVMValueRef (*outputs)[NUM_CHANNELS], 799 LLVMValueRef clipmask, 800 int num_outputs, 801 int max_vertices) 802{ 803 LLVMBuilderRef builder = gallivm->builder; 804 unsigned chan, attrib; 805 806#if DEBUG_STORE 807 lp_build_printf(builder, " # storing begin\n"); 808#endif 809 for (attrib = 0; attrib < num_outputs; ++attrib) { 810 LLVMValueRef soa[4]; 811 LLVMValueRef aos[4]; 812 for(chan = 0; chan < NUM_CHANNELS; ++chan) { 813 if(outputs[attrib][chan]) { 814 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 815 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]); 816 /*lp_build_printf(builder, "output %d : %d ", 817 LLVMConstInt(LLVMInt32Type(), attrib, 0), 818 LLVMConstInt(LLVMInt32Type(), chan, 0)); 819 print_vectorf(builder, out);*/ 820 soa[chan] = out; 821 } else 822 soa[chan] = 0; 823 } 824 soa_to_aos(gallivm, soa, aos); 825 store_aos_array(gallivm, 826 io, 827 aos, 828 attrib, 829 num_outputs, 830 clipmask); 831 } 832#if DEBUG_STORE 833 lp_build_printf(builder, " # storing end\n"); 834#endif 835} 836 837/* 838 * Stores original vertex positions in clip coordinates 839 * There is probably a more efficient way to do this, 4 floats at once 840 * rather than extracting each element one by one. 841 */ 842static void 843store_clip(struct gallivm_state *gallivm, 844 LLVMValueRef io_ptr, 845 LLVMValueRef (*outputs)[NUM_CHANNELS]) 846{ 847 LLVMBuilderRef builder = gallivm->builder; 848 LLVMValueRef out[4]; 849 LLVMValueRef indices[2]; 850 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 851 LLVMValueRef clip_ptr0, clip_ptr1, clip_ptr2, clip_ptr3; 852 LLVMValueRef clip0_ptr, clip1_ptr, clip2_ptr, clip3_ptr; 853 LLVMValueRef out0elem, out1elem, out2elem, out3elem; 854 int i; 855 856 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0); 857 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1); 858 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2); 859 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3); 860 861 indices[0] = 862 indices[1] = lp_build_const_int32(gallivm, 0); 863 864 out[0] = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/ 865 out[1] = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/ 866 out[2] = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/ 867 out[3] = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 868 869 io0_ptr = LLVMBuildGEP(builder, io_ptr, &ind0, 1, ""); 870 io1_ptr = LLVMBuildGEP(builder, io_ptr, &ind1, 1, ""); 871 io2_ptr = LLVMBuildGEP(builder, io_ptr, &ind2, 1, ""); 872 io3_ptr = LLVMBuildGEP(builder, io_ptr, &ind3, 1, ""); 873 874 clip_ptr0 = draw_jit_header_clip(gallivm, io0_ptr); 875 clip_ptr1 = draw_jit_header_clip(gallivm, io1_ptr); 876 clip_ptr2 = draw_jit_header_clip(gallivm, io2_ptr); 877 clip_ptr3 = draw_jit_header_clip(gallivm, io3_ptr); 878 879 for (i = 0; i<4; i++){ 880 clip0_ptr = LLVMBuildGEP(builder, clip_ptr0, indices, 2, ""); /* x0 */ 881 clip1_ptr = LLVMBuildGEP(builder, clip_ptr1, indices, 2, ""); /* x1 */ 882 clip2_ptr = LLVMBuildGEP(builder, clip_ptr2, indices, 2, ""); /* x2 */ 883 clip3_ptr = LLVMBuildGEP(builder, clip_ptr3, indices, 2, ""); /* x3 */ 884 885 out0elem = LLVMBuildExtractElement(builder, out[i], ind0, ""); /* x0 */ 886 out1elem = LLVMBuildExtractElement(builder, out[i], ind1, ""); /* x1 */ 887 out2elem = LLVMBuildExtractElement(builder, out[i], ind2, ""); /* x2 */ 888 out3elem = LLVMBuildExtractElement(builder, out[i], ind3, ""); /* x3 */ 889 890 LLVMBuildStore(builder, out0elem, clip0_ptr); 891 LLVMBuildStore(builder, out1elem, clip1_ptr); 892 LLVMBuildStore(builder, out2elem, clip2_ptr); 893 LLVMBuildStore(builder, out3elem, clip3_ptr); 894 895 indices[1]= LLVMBuildAdd(builder, indices[1], ind1, ""); 896 } 897 898} 899 900/* Equivalent of _mm_set1_ps(a) 901 */ 902static LLVMValueRef 903vec4f_from_scalar(struct gallivm_state *gallivm, 904 LLVMValueRef a, 905 const char *name) 906{ 907 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 908 LLVMValueRef res = LLVMGetUndef(LLVMVectorType(float_type, 4)); 909 int i; 910 911 for(i = 0; i < 4; ++i) { 912 LLVMValueRef index = lp_build_const_int32(gallivm, i); 913 res = LLVMBuildInsertElement(gallivm->builder, res, a, 914 index, i == 3 ? name : ""); 915 } 916 917 return res; 918} 919 920/* 921 * Transforms the outputs for viewport mapping 922 */ 923static void 924generate_viewport(struct draw_llvm *llvm, 925 LLVMBuilderRef builder, 926 LLVMValueRef (*outputs)[NUM_CHANNELS], 927 LLVMValueRef context_ptr) 928{ 929 int i; 930 struct gallivm_state *gallivm = llvm->gallivm; 931 struct lp_type f32_type = lp_type_float_vec(32); 932 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 933 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/ 934 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr); 935 936 /* for 1/w convention*/ 937 out3 = LLVMBuildFDiv(builder, const1, out3, ""); 938 LLVMBuildStore(builder, out3, outputs[0][3]); 939 940 /* Viewport Mapping */ 941 for (i=0; i<3; i++){ 942 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 x2 x3*/ 943 LLVMValueRef scale; 944 LLVMValueRef trans; 945 LLVMValueRef scale_i; 946 LLVMValueRef trans_i; 947 LLVMValueRef index; 948 949 index = lp_build_const_int32(gallivm, i); 950 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 951 952 index = lp_build_const_int32(gallivm, i+4); 953 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 954 955 scale = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, scale_i, ""), "scale"); 956 trans = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, trans_i, ""), "trans"); 957 958 /* divide by w */ 959 out = LLVMBuildFMul(builder, out, out3, ""); 960 /* mult by scale */ 961 out = LLVMBuildFMul(builder, out, scale, ""); 962 /* add translation */ 963 out = LLVMBuildFAdd(builder, out, trans, ""); 964 965 /* store transformed outputs */ 966 LLVMBuildStore(builder, out, outputs[0][i]); 967 } 968 969} 970 971 972/* 973 * Returns clipmask as 4xi32 bitmask for the 4 vertices 974 */ 975static LLVMValueRef 976generate_clipmask(struct gallivm_state *gallivm, 977 LLVMValueRef (*outputs)[NUM_CHANNELS], 978 boolean clip_xy, 979 boolean clip_z, 980 boolean clip_user, 981 boolean clip_halfz, 982 unsigned nr, 983 LLVMValueRef context_ptr) 984{ 985 LLVMBuilderRef builder = gallivm->builder; 986 LLVMValueRef mask; /* stores the <4xi32> clipmasks */ 987 LLVMValueRef test, temp; 988 LLVMValueRef zero, shift; 989 LLVMValueRef pos_x, pos_y, pos_z, pos_w; 990 LLVMValueRef plane1, planes, plane_ptr, sum; 991 992 unsigned i; 993 994 struct lp_type f32_type = lp_type_float_vec(32); 995 996 mask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 997 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 998 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */ 999 shift = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1); /* 1 1 1 1 */ 1000 1001 /* Assuming position stored at output[0] */ 1002 pos_x = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/ 1003 pos_y = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/ 1004 pos_z = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/ 1005 pos_w = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 1006 1007 /* Cliptest, for hardwired planes */ 1008 if (clip_xy){ 1009 /* plane 1 */ 1010 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w); 1011 temp = shift; 1012 test = LLVMBuildAnd(builder, test, temp, ""); 1013 mask = test; 1014 1015 /* plane 2 */ 1016 test = LLVMBuildFAdd(builder, pos_x, pos_w, ""); 1017 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1018 temp = LLVMBuildShl(builder, temp, shift, ""); 1019 test = LLVMBuildAnd(builder, test, temp, ""); 1020 mask = LLVMBuildOr(builder, mask, test, ""); 1021 1022 /* plane 3 */ 1023 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w); 1024 temp = LLVMBuildShl(builder, temp, shift, ""); 1025 test = LLVMBuildAnd(builder, test, temp, ""); 1026 mask = LLVMBuildOr(builder, mask, test, ""); 1027 1028 /* plane 4 */ 1029 test = LLVMBuildFAdd(builder, pos_y, pos_w, ""); 1030 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1031 temp = LLVMBuildShl(builder, temp, shift, ""); 1032 test = LLVMBuildAnd(builder, test, temp, ""); 1033 mask = LLVMBuildOr(builder, mask, test, ""); 1034 } 1035 1036 if (clip_z){ 1037 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 16); 1038 if (clip_halfz){ 1039 /* plane 5 */ 1040 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z); 1041 test = LLVMBuildAnd(builder, test, temp, ""); 1042 mask = LLVMBuildOr(builder, mask, test, ""); 1043 } 1044 else{ 1045 /* plane 5 */ 1046 test = LLVMBuildFAdd(builder, pos_z, pos_w, ""); 1047 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1048 test = LLVMBuildAnd(builder, test, temp, ""); 1049 mask = LLVMBuildOr(builder, mask, test, ""); 1050 } 1051 /* plane 6 */ 1052 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w); 1053 temp = LLVMBuildShl(builder, temp, shift, ""); 1054 test = LLVMBuildAnd(builder, test, temp, ""); 1055 mask = LLVMBuildOr(builder, mask, test, ""); 1056 } 1057 1058 if (clip_user){ 1059 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr); 1060 LLVMValueRef indices[3]; 1061 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 32); 1062 1063 /* userclip planes */ 1064 for (i = 6; i < nr; i++) { 1065 indices[0] = lp_build_const_int32(gallivm, 0); 1066 indices[1] = lp_build_const_int32(gallivm, i); 1067 1068 indices[2] = lp_build_const_int32(gallivm, 0); 1069 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1070 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x"); 1071 planes = vec4f_from_scalar(gallivm, plane1, "plane4_x"); 1072 sum = LLVMBuildFMul(builder, planes, pos_x, ""); 1073 1074 indices[2] = lp_build_const_int32(gallivm, 1); 1075 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1076 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y"); 1077 planes = vec4f_from_scalar(gallivm, plane1, "plane4_y"); 1078 test = LLVMBuildFMul(builder, planes, pos_y, ""); 1079 sum = LLVMBuildFAdd(builder, sum, test, ""); 1080 1081 indices[2] = lp_build_const_int32(gallivm, 2); 1082 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1083 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z"); 1084 planes = vec4f_from_scalar(gallivm, plane1, "plane4_z"); 1085 test = LLVMBuildFMul(builder, planes, pos_z, ""); 1086 sum = LLVMBuildFAdd(builder, sum, test, ""); 1087 1088 indices[2] = lp_build_const_int32(gallivm, 3); 1089 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1090 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w"); 1091 planes = vec4f_from_scalar(gallivm, plane1, "plane4_w"); 1092 test = LLVMBuildFMul(builder, planes, pos_w, ""); 1093 sum = LLVMBuildFAdd(builder, sum, test, ""); 1094 1095 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum); 1096 temp = LLVMBuildShl(builder, temp, shift, ""); 1097 test = LLVMBuildAnd(builder, test, temp, ""); 1098 mask = LLVMBuildOr(builder, mask, test, ""); 1099 } 1100 } 1101 return mask; 1102} 1103 1104/* 1105 * Returns boolean if any clipping has occurred 1106 * Used zero/non-zero i32 value to represent boolean 1107 */ 1108static void 1109clipmask_bool(struct gallivm_state *gallivm, 1110 LLVMValueRef clipmask, 1111 LLVMValueRef ret_ptr) 1112{ 1113 LLVMBuilderRef builder = gallivm->builder; 1114 LLVMValueRef ret = LLVMBuildLoad(builder, ret_ptr, ""); 1115 LLVMValueRef temp; 1116 int i; 1117 1118 for (i=0; i<4; i++){ 1119 temp = LLVMBuildExtractElement(builder, clipmask, 1120 lp_build_const_int32(gallivm, i) , ""); 1121 ret = LLVMBuildOr(builder, ret, temp, ""); 1122 } 1123 1124 LLVMBuildStore(builder, ret, ret_ptr); 1125} 1126 1127static void 1128draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant) 1129{ 1130 struct gallivm_state *gallivm = llvm->gallivm; 1131 LLVMContextRef context = gallivm->context; 1132 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 1133 LLVMTypeRef arg_types[8]; 1134 LLVMTypeRef func_type; 1135 LLVMValueRef context_ptr; 1136 LLVMBasicBlockRef block; 1137 LLVMBuilderRef builder; 1138 LLVMValueRef start, end, count, stride, step, io_itr; 1139 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 1140 LLVMValueRef instance_id; 1141 LLVMValueRef system_values_array; 1142 struct draw_context *draw = llvm->draw; 1143 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info; 1144 unsigned i, j; 1145 struct lp_build_context bld; 1146 struct lp_build_loop_state lp_loop; 1147 const int max_vertices = 4; 1148 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS]; 1149 void *code; 1150 struct lp_build_sampler_soa *sampler = 0; 1151 LLVMValueRef ret, ret_ptr; 1152 boolean bypass_viewport = variant->key.bypass_viewport; 1153 boolean enable_cliptest = variant->key.clip_xy || 1154 variant->key.clip_z || 1155 variant->key.clip_user; 1156 1157 arg_types[0] = get_context_ptr_type(llvm); /* context */ 1158 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */ 1159 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */ 1160 arg_types[3] = int32_type; /* start */ 1161 arg_types[4] = int32_type; /* count */ 1162 arg_types[5] = int32_type; /* stride */ 1163 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */ 1164 arg_types[7] = int32_type; /* instance_id */ 1165 1166 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0); 1167 1168 variant->function = LLVMAddFunction(gallivm->module, "draw_llvm_shader", 1169 func_type); 1170 LLVMSetFunctionCallConv(variant->function, LLVMCCallConv); 1171 for(i = 0; i < Elements(arg_types); ++i) 1172 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 1173 LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute); 1174 1175 context_ptr = LLVMGetParam(variant->function, 0); 1176 io_ptr = LLVMGetParam(variant->function, 1); 1177 vbuffers_ptr = LLVMGetParam(variant->function, 2); 1178 start = LLVMGetParam(variant->function, 3); 1179 count = LLVMGetParam(variant->function, 4); 1180 stride = LLVMGetParam(variant->function, 5); 1181 vb_ptr = LLVMGetParam(variant->function, 6); 1182 instance_id = LLVMGetParam(variant->function, 7); 1183 1184 lp_build_name(context_ptr, "context"); 1185 lp_build_name(io_ptr, "io"); 1186 lp_build_name(vbuffers_ptr, "vbuffers"); 1187 lp_build_name(start, "start"); 1188 lp_build_name(count, "count"); 1189 lp_build_name(stride, "stride"); 1190 lp_build_name(vb_ptr, "vb"); 1191 lp_build_name(instance_id, "instance_id"); 1192 1193 /* 1194 * Function body 1195 */ 1196 1197 block = LLVMAppendBasicBlockInContext(gallivm->context, variant->function, "entry"); 1198 builder = gallivm->builder; 1199 assert(builder); 1200 LLVMPositionBuilderAtEnd(builder, block); 1201 1202 lp_build_context_init(&bld, llvm->gallivm, lp_type_int(32)); 1203 1204 system_values_array = lp_build_system_values_array(gallivm, vs_info, 1205 instance_id, NULL); 1206 1207 end = lp_build_add(&bld, start, count); 1208 1209 step = lp_build_const_int32(gallivm, max_vertices); 1210 1211 /* function will return non-zero i32 value if any clipped vertices */ 1212 ret_ptr = lp_build_alloca(gallivm, int32_type, ""); 1213 LLVMBuildStore(builder, lp_build_const_int32(gallivm, 0), ret_ptr); 1214 1215 /* code generated texture sampling */ 1216 sampler = draw_llvm_sampler_soa_create( 1217 draw_llvm_variant_key_samplers(&variant->key), 1218 context_ptr); 1219 1220#if DEBUG_STORE 1221 lp_build_printf(builder, "start = %d, end = %d, step = %d\n", 1222 start, end, step); 1223#endif 1224 lp_build_loop_begin(&lp_loop, llvm->gallivm, start); 1225 { 1226 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS]; 1227 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } }; 1228 LLVMValueRef io; 1229 LLVMValueRef clipmask; /* holds the clipmask value */ 1230 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS]; 1231 1232 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, ""); 1233 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 1234#if DEBUG_STORE 1235 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n", 1236 io_itr, io, lp_loop.counter); 1237#endif 1238 for (i = 0; i < NUM_CHANNELS; ++i) { 1239 LLVMValueRef true_index = LLVMBuildAdd( 1240 builder, 1241 lp_loop.counter, 1242 lp_build_const_int32(gallivm, i), ""); 1243 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 1244 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 1245 LLVMValueRef vb_index = lp_build_const_int32(gallivm, velem->vertex_buffer_index); 1246 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, 1247 &vb_index, 1, ""); 1248 generate_fetch(llvm->gallivm, vbuffers_ptr, 1249 &aos_attribs[j][i], velem, vb, true_index, 1250 instance_id); 1251 } 1252 } 1253 convert_to_soa(gallivm, aos_attribs, inputs, 1254 draw->pt.nr_vertex_elements); 1255 1256 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs; 1257 generate_vs(llvm, 1258 builder, 1259 outputs, 1260 ptr_aos, 1261 system_values_array, 1262 context_ptr, 1263 sampler, 1264 variant->key.clamp_vertex_color); 1265 1266 /* store original positions in clip before further manipulation */ 1267 store_clip(gallivm, io, outputs); 1268 1269 /* do cliptest */ 1270 if (enable_cliptest){ 1271 /* allocate clipmask, assign it integer type */ 1272 clipmask = generate_clipmask(gallivm, outputs, 1273 variant->key.clip_xy, 1274 variant->key.clip_z, 1275 variant->key.clip_user, 1276 variant->key.clip_halfz, 1277 variant->key.nr_planes, 1278 context_ptr); 1279 /* return clipping boolean value for function */ 1280 clipmask_bool(gallivm, clipmask, ret_ptr); 1281 } 1282 else{ 1283 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1284 } 1285 1286 /* do viewport mapping */ 1287 if (!bypass_viewport){ 1288 generate_viewport(llvm, builder, outputs, context_ptr); 1289 } 1290 1291 /* store clipmask in vertex header and positions in data */ 1292 convert_to_aos(gallivm, io, outputs, clipmask, 1293 vs_info->num_outputs, max_vertices); 1294 } 1295 1296 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE); 1297 1298 sampler->destroy(sampler); 1299 1300 ret = LLVMBuildLoad(builder, ret_ptr,""); 1301 LLVMBuildRet(builder, ret); 1302 1303 /* 1304 * Translate the LLVM IR into machine code. 1305 */ 1306#ifdef DEBUG 1307 if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) { 1308 lp_debug_dump_value(variant->function); 1309 assert(0); 1310 } 1311#endif 1312 1313 LLVMRunFunctionPassManager(gallivm->passmgr, variant->function); 1314 1315 if (gallivm_debug & GALLIVM_DEBUG_IR) { 1316 lp_debug_dump_value(variant->function); 1317 debug_printf("\n"); 1318 } 1319 1320 code = LLVMGetPointerToGlobal(gallivm->engine, variant->function); 1321 variant->jit_func = (draw_jit_vert_func)pointer_to_func(code); 1322 1323 if (gallivm_debug & GALLIVM_DEBUG_ASM) { 1324 lp_disassemble(code); 1325 } 1326 lp_func_delete_body(variant->function); 1327} 1328 1329 1330static void 1331draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant) 1332{ 1333 struct gallivm_state *gallivm = llvm->gallivm; 1334 LLVMContextRef context = gallivm->context; 1335 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 1336 LLVMTypeRef arg_types[8]; 1337 LLVMTypeRef func_type; 1338 LLVMValueRef context_ptr; 1339 LLVMBasicBlockRef block; 1340 LLVMBuilderRef builder; 1341 LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr; 1342 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 1343 LLVMValueRef instance_id; 1344 LLVMValueRef system_values_array; 1345 struct draw_context *draw = llvm->draw; 1346 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info; 1347 unsigned i, j; 1348 struct lp_build_context bld; 1349 struct lp_build_loop_state lp_loop; 1350 const int max_vertices = 4; 1351 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS]; 1352 LLVMValueRef fetch_max; 1353 void *code; 1354 struct lp_build_sampler_soa *sampler = 0; 1355 LLVMValueRef ret, ret_ptr; 1356 boolean bypass_viewport = variant->key.bypass_viewport; 1357 boolean enable_cliptest = variant->key.clip_xy || 1358 variant->key.clip_z || 1359 variant->key.clip_user; 1360 1361 arg_types[0] = get_context_ptr_type(llvm); /* context */ 1362 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */ 1363 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */ 1364 arg_types[3] = LLVMPointerType(int32_type, 0); /* fetch_elts * */ 1365 arg_types[4] = int32_type; /* fetch_count */ 1366 arg_types[5] = int32_type; /* stride */ 1367 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */ 1368 arg_types[7] = int32_type; /* instance_id */ 1369 1370 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0); 1371 1372 variant->function_elts = LLVMAddFunction(gallivm->module, "draw_llvm_shader_elts", func_type); 1373 LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv); 1374 for(i = 0; i < Elements(arg_types); ++i) 1375 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 1376 LLVMAddAttribute(LLVMGetParam(variant->function_elts, i), 1377 LLVMNoAliasAttribute); 1378 1379 context_ptr = LLVMGetParam(variant->function_elts, 0); 1380 io_ptr = LLVMGetParam(variant->function_elts, 1); 1381 vbuffers_ptr = LLVMGetParam(variant->function_elts, 2); 1382 fetch_elts = LLVMGetParam(variant->function_elts, 3); 1383 fetch_count = LLVMGetParam(variant->function_elts, 4); 1384 stride = LLVMGetParam(variant->function_elts, 5); 1385 vb_ptr = LLVMGetParam(variant->function_elts, 6); 1386 instance_id = LLVMGetParam(variant->function_elts, 7); 1387 1388 lp_build_name(context_ptr, "context"); 1389 lp_build_name(io_ptr, "io"); 1390 lp_build_name(vbuffers_ptr, "vbuffers"); 1391 lp_build_name(fetch_elts, "fetch_elts"); 1392 lp_build_name(fetch_count, "fetch_count"); 1393 lp_build_name(stride, "stride"); 1394 lp_build_name(vb_ptr, "vb"); 1395 lp_build_name(instance_id, "instance_id"); 1396 1397 /* 1398 * Function body 1399 */ 1400 1401 block = LLVMAppendBasicBlockInContext(gallivm->context, variant->function_elts, "entry"); 1402 builder = gallivm->builder; 1403 LLVMPositionBuilderAtEnd(builder, block); 1404 1405 lp_build_context_init(&bld, gallivm, lp_type_int(32)); 1406 1407 system_values_array = lp_build_system_values_array(gallivm, vs_info, 1408 instance_id, NULL); 1409 1410 1411 step = lp_build_const_int32(gallivm, max_vertices); 1412 1413 /* code generated texture sampling */ 1414 sampler = draw_llvm_sampler_soa_create( 1415 draw_llvm_variant_key_samplers(&variant->key), 1416 context_ptr); 1417 1418 fetch_max = LLVMBuildSub(builder, fetch_count, 1419 lp_build_const_int32(gallivm, 1), 1420 "fetch_max"); 1421 1422 /* function returns non-zero i32 value if any clipped vertices */ 1423 ret_ptr = lp_build_alloca(gallivm, int32_type, ""); 1424 LLVMBuildStore(builder, lp_build_const_int32(gallivm, 0), ret_ptr); 1425 1426 lp_build_loop_begin(&lp_loop, gallivm, lp_build_const_int32(gallivm, 0)); 1427 { 1428 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS]; 1429 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } }; 1430 LLVMValueRef io; 1431 LLVMValueRef clipmask; /* holds the clipmask value */ 1432 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS]; 1433 1434 io_itr = lp_loop.counter; 1435 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 1436#if DEBUG_STORE 1437 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n", 1438 io_itr, io, lp_loop.counter); 1439#endif 1440 for (i = 0; i < NUM_CHANNELS; ++i) { 1441 LLVMValueRef true_index = LLVMBuildAdd( 1442 builder, 1443 lp_loop.counter, 1444 lp_build_const_int32(gallivm, i), ""); 1445 LLVMValueRef fetch_ptr; 1446 1447 /* make sure we're not out of bounds which can happen 1448 * if fetch_count % 4 != 0, because on the last iteration 1449 * a few of the 4 vertex fetches will be out of bounds */ 1450 true_index = lp_build_min(&bld, true_index, fetch_max); 1451 1452 fetch_ptr = LLVMBuildGEP(builder, fetch_elts, 1453 &true_index, 1, ""); 1454 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt"); 1455 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 1456 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 1457 LLVMValueRef vb_index = lp_build_const_int32(gallivm, velem->vertex_buffer_index); 1458 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, 1459 &vb_index, 1, ""); 1460 generate_fetch(gallivm, vbuffers_ptr, 1461 &aos_attribs[j][i], velem, vb, true_index, 1462 instance_id); 1463 } 1464 } 1465 convert_to_soa(gallivm, aos_attribs, inputs, 1466 draw->pt.nr_vertex_elements); 1467 1468 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs; 1469 generate_vs(llvm, 1470 builder, 1471 outputs, 1472 ptr_aos, 1473 system_values_array, 1474 context_ptr, 1475 sampler, 1476 variant->key.clamp_vertex_color); 1477 1478 /* store original positions in clip before further manipulation */ 1479 store_clip(gallivm, io, outputs); 1480 1481 /* do cliptest */ 1482 if (enable_cliptest){ 1483 /* allocate clipmask, assign it integer type */ 1484 clipmask = generate_clipmask(gallivm, outputs, 1485 variant->key.clip_xy, 1486 variant->key.clip_z, 1487 variant->key.clip_user, 1488 variant->key.clip_halfz, 1489 variant->key.nr_planes, 1490 context_ptr); 1491 /* return clipping boolean value for function */ 1492 clipmask_bool(gallivm, clipmask, ret_ptr); 1493 } 1494 else{ 1495 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1496 } 1497 1498 /* do viewport mapping */ 1499 if (!bypass_viewport){ 1500 generate_viewport(llvm, builder, outputs, context_ptr); 1501 } 1502 1503 /* store clipmask in vertex header, 1504 * original positions in clip 1505 * and transformed positions in data 1506 */ 1507 convert_to_aos(gallivm, io, outputs, clipmask, 1508 vs_info->num_outputs, max_vertices); 1509 } 1510 1511 lp_build_loop_end_cond(&lp_loop, fetch_count, step, LLVMIntUGE); 1512 1513 sampler->destroy(sampler); 1514 1515 ret = LLVMBuildLoad(builder, ret_ptr,""); 1516 LLVMBuildRet(builder, ret); 1517 1518 /* 1519 * Translate the LLVM IR into machine code. 1520 */ 1521#ifdef DEBUG 1522 if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) { 1523 lp_debug_dump_value(variant->function_elts); 1524 assert(0); 1525 } 1526#endif 1527 1528 LLVMRunFunctionPassManager(gallivm->passmgr, variant->function_elts); 1529 1530 if (gallivm_debug & GALLIVM_DEBUG_IR) { 1531 lp_debug_dump_value(variant->function_elts); 1532 debug_printf("\n"); 1533 } 1534 1535 code = LLVMGetPointerToGlobal(gallivm->engine, variant->function_elts); 1536 variant->jit_func_elts = (draw_jit_vert_func_elts)pointer_to_func(code); 1537 1538 if (gallivm_debug & GALLIVM_DEBUG_ASM) { 1539 lp_disassemble(code); 1540 } 1541 lp_func_delete_body(variant->function_elts); 1542} 1543 1544 1545struct draw_llvm_variant_key * 1546draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store) 1547{ 1548 unsigned i; 1549 struct draw_llvm_variant_key *key; 1550 struct lp_sampler_static_state *sampler; 1551 1552 key = (struct draw_llvm_variant_key *)store; 1553 1554 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/ 1555 1556 /* Presumably all variants of the shader should have the same 1557 * number of vertex elements - ie the number of shader inputs. 1558 */ 1559 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements; 1560 1561 /* will have to rig this up properly later */ 1562 key->clip_xy = llvm->draw->clip_xy; 1563 key->clip_z = llvm->draw->clip_z; 1564 key->clip_user = llvm->draw->clip_user; 1565 key->bypass_viewport = llvm->draw->identity_viewport; 1566 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules; 1567 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE); 1568 key->nr_planes = llvm->draw->nr_planes; 1569 key->pad = 0; 1570 1571 /* All variants of this shader will have the same value for 1572 * nr_samplers. Not yet trying to compact away holes in the 1573 * sampler array. 1574 */ 1575 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1; 1576 1577 sampler = draw_llvm_variant_key_samplers(key); 1578 1579 memcpy(key->vertex_element, 1580 llvm->draw->pt.vertex_element, 1581 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements); 1582 1583 memset(sampler, 0, key->nr_samplers * sizeof *sampler); 1584 1585 for (i = 0 ; i < key->nr_samplers; i++) { 1586 lp_sampler_static_state(&sampler[i], 1587 llvm->draw->sampler_views[i], 1588 llvm->draw->samplers[i]); 1589 } 1590 1591 return key; 1592} 1593 1594void 1595draw_llvm_set_mapped_texture(struct draw_context *draw, 1596 unsigned sampler_idx, 1597 uint32_t width, uint32_t height, uint32_t depth, 1598 uint32_t last_level, 1599 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS], 1600 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS], 1601 const void *data[PIPE_MAX_TEXTURE_LEVELS]) 1602{ 1603 unsigned j; 1604 struct draw_jit_texture *jit_tex; 1605 1606 assert(sampler_idx < PIPE_MAX_VERTEX_SAMPLERS); 1607 1608 1609 jit_tex = &draw->llvm->jit_context.textures[sampler_idx]; 1610 1611 jit_tex->width = width; 1612 jit_tex->height = height; 1613 jit_tex->depth = depth; 1614 jit_tex->last_level = last_level; 1615 1616 for (j = 0; j <= last_level; j++) { 1617 jit_tex->data[j] = data[j]; 1618 jit_tex->row_stride[j] = row_stride[j]; 1619 jit_tex->img_stride[j] = img_stride[j]; 1620 } 1621} 1622 1623 1624void 1625draw_llvm_set_sampler_state(struct draw_context *draw) 1626{ 1627 unsigned i; 1628 1629 for (i = 0; i < draw->num_samplers; i++) { 1630 struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i]; 1631 1632 if (draw->samplers[i]) { 1633 jit_tex->min_lod = draw->samplers[i]->min_lod; 1634 jit_tex->max_lod = draw->samplers[i]->max_lod; 1635 jit_tex->lod_bias = draw->samplers[i]->lod_bias; 1636 COPY_4V(jit_tex->border_color, draw->samplers[i]->border_color); 1637 } 1638 } 1639} 1640 1641 1642void 1643draw_llvm_destroy_variant(struct draw_llvm_variant *variant) 1644{ 1645 struct draw_llvm *llvm = variant->llvm; 1646 1647 if (variant->function_elts) { 1648 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine, 1649 variant->function_elts); 1650 LLVMDeleteFunction(variant->function_elts); 1651 } 1652 1653 if (variant->function) { 1654 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine, 1655 variant->function); 1656 LLVMDeleteFunction(variant->function); 1657 } 1658 1659 remove_from_list(&variant->list_item_local); 1660 variant->shader->variants_cached--; 1661 remove_from_list(&variant->list_item_global); 1662 llvm->nr_variants--; 1663 FREE(variant); 1664} 1665