draw_llvm.c revision dc4c821f0817a3db716f965692fb701079f66340
1/************************************************************************** 2 * 3 * Copyright 2010 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28#include "draw_llvm.h" 29 30#include "draw_context.h" 31#include "draw_vs.h" 32 33#include "gallivm/lp_bld_arit.h" 34#include "gallivm/lp_bld_logic.h" 35#include "gallivm/lp_bld_const.h" 36#include "gallivm/lp_bld_swizzle.h" 37#include "gallivm/lp_bld_struct.h" 38#include "gallivm/lp_bld_type.h" 39#include "gallivm/lp_bld_flow.h" 40#include "gallivm/lp_bld_debug.h" 41#include "gallivm/lp_bld_tgsi.h" 42#include "gallivm/lp_bld_printf.h" 43#include "gallivm/lp_bld_intr.h" 44#include "gallivm/lp_bld_init.h" 45#include "gallivm/lp_bld_type.h" 46 47#include "tgsi/tgsi_exec.h" 48#include "tgsi/tgsi_dump.h" 49 50#include "util/u_math.h" 51#include "util/u_pointer.h" 52#include "util/u_string.h" 53#include "util/u_simple_list.h" 54 55 56#define DEBUG_STORE 0 57 58 59/** 60 * This function is called by the gallivm "garbage collector" when 61 * the LLVM global data structures are freed. We must free all LLVM-related 62 * data. Specifically, all JIT'd shader variants. 63 */ 64static void 65draw_llvm_garbage_collect_callback(void *cb_data) 66{ 67 struct draw_llvm *llvm = (struct draw_llvm *) cb_data; 68 struct draw_llvm_variant_list_item *li; 69 70 /* free all shader variants */ 71 li = first_elem(&llvm->vs_variants_list); 72 while (!at_end(&llvm->vs_variants_list, li)) { 73 struct draw_llvm_variant_list_item *next = next_elem(li); 74 draw_llvm_destroy_variant(li->base); 75 li = next; 76 } 77 78 /* Null-out these pointers so they get remade next time they're needed. 79 * See the accessor functions below. 80 */ 81 llvm->context_ptr_type = NULL; 82 llvm->buffer_ptr_type = NULL; 83 llvm->vb_ptr_type = NULL; 84 llvm->vertex_header_ptr_type = NULL; 85} 86 87 88static void 89draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var, 90 boolean elts); 91 92 93/** 94 * Create LLVM type for struct draw_jit_texture 95 */ 96static LLVMTypeRef 97create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name) 98{ 99 LLVMTargetDataRef target = gallivm->target; 100 LLVMTypeRef texture_type; 101 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS]; 102 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 103 104 elem_types[DRAW_JIT_TEXTURE_WIDTH] = 105 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = 106 elem_types[DRAW_JIT_TEXTURE_DEPTH] = 107 elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] = 108 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type; 109 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] = 110 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] = 111 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS); 112 elem_types[DRAW_JIT_TEXTURE_DATA] = 113 LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0), 114 PIPE_MAX_TEXTURE_LEVELS); 115 elem_types[DRAW_JIT_TEXTURE_MIN_LOD] = 116 elem_types[DRAW_JIT_TEXTURE_MAX_LOD] = 117 elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context); 118 elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] = 119 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 120 121#if HAVE_LLVM >= 0x0300 122 texture_type = LLVMStructCreateNamed(gallivm->context, struct_name); 123 LLVMStructSetBody(texture_type, elem_types, 124 Elements(elem_types), 0); 125#else 126 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types, 127 Elements(elem_types), 0); 128 129 LLVMAddTypeName(gallivm->module, struct_name, texture_type); 130 131 /* Make sure the target's struct layout cache doesn't return 132 * stale/invalid data. 133 */ 134 LLVMInvalidateStructLayout(gallivm->target, texture_type); 135#endif 136 137 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width, 138 target, texture_type, 139 DRAW_JIT_TEXTURE_WIDTH); 140 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height, 141 target, texture_type, 142 DRAW_JIT_TEXTURE_HEIGHT); 143 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth, 144 target, texture_type, 145 DRAW_JIT_TEXTURE_DEPTH); 146 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level, 147 target, texture_type, 148 DRAW_JIT_TEXTURE_FIRST_LEVEL); 149 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level, 150 target, texture_type, 151 DRAW_JIT_TEXTURE_LAST_LEVEL); 152 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride, 153 target, texture_type, 154 DRAW_JIT_TEXTURE_ROW_STRIDE); 155 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride, 156 target, texture_type, 157 DRAW_JIT_TEXTURE_IMG_STRIDE); 158 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data, 159 target, texture_type, 160 DRAW_JIT_TEXTURE_DATA); 161 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod, 162 target, texture_type, 163 DRAW_JIT_TEXTURE_MIN_LOD); 164 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod, 165 target, texture_type, 166 DRAW_JIT_TEXTURE_MAX_LOD); 167 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias, 168 target, texture_type, 169 DRAW_JIT_TEXTURE_LOD_BIAS); 170 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color, 171 target, texture_type, 172 DRAW_JIT_TEXTURE_BORDER_COLOR); 173 174 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type); 175 176 return texture_type; 177} 178 179 180/** 181 * Create LLVM type for struct draw_jit_texture 182 */ 183static LLVMTypeRef 184create_jit_context_type(struct gallivm_state *gallivm, 185 LLVMTypeRef texture_type, const char *struct_name) 186{ 187 LLVMTargetDataRef target = gallivm->target; 188 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 189 LLVMTypeRef elem_types[5]; 190 LLVMTypeRef context_type; 191 192 elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */ 193 elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */ 194 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), 195 DRAW_TOTAL_CLIP_PLANES), 0); 196 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */ 197 elem_types[4] = LLVMArrayType(texture_type, 198 PIPE_MAX_VERTEX_SAMPLERS); /* textures */ 199#if HAVE_LLVM >= 0x0300 200 context_type = LLVMStructCreateNamed(gallivm->context, struct_name); 201 LLVMStructSetBody(context_type, elem_types, 202 Elements(elem_types), 0); 203#else 204 context_type = LLVMStructTypeInContext(gallivm->context, elem_types, 205 Elements(elem_types), 0); 206 LLVMAddTypeName(gallivm->module, struct_name, context_type); 207 208 LLVMInvalidateStructLayout(gallivm->target, context_type); 209#endif 210 211 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants, 212 target, context_type, 0); 213 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants, 214 target, context_type, 1); 215 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes, 216 target, context_type, 2); 217 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures, 218 target, context_type, 219 DRAW_JIT_CTX_TEXTURES); 220 LP_CHECK_STRUCT_SIZE(struct draw_jit_context, 221 target, context_type); 222 223 return context_type; 224} 225 226 227/** 228 * Create LLVM type for struct pipe_vertex_buffer 229 */ 230static LLVMTypeRef 231create_jit_vertex_buffer_type(struct gallivm_state *gallivm, const char *struct_name) 232{ 233 LLVMTargetDataRef target = gallivm->target; 234 LLVMTypeRef elem_types[3]; 235 LLVMTypeRef vb_type; 236 237 elem_types[0] = 238 elem_types[1] = LLVMInt32TypeInContext(gallivm->context); 239 elem_types[2] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */ 240 241#if HAVE_LLVM >= 0x0300 242 vb_type = LLVMStructCreateNamed(gallivm->context, struct_name); 243 LLVMStructSetBody(vb_type, elem_types, 244 Elements(elem_types), 0); 245#else 246 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types, 247 Elements(elem_types), 0); 248 LLVMAddTypeName(gallivm->module, struct_name, vb_type); 249 250 LLVMInvalidateStructLayout(gallivm->target, vb_type); 251#endif 252 253 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride, 254 target, vb_type, 0); 255 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset, 256 target, vb_type, 1); 257 258 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type); 259 260 return vb_type; 261} 262 263 264/** 265 * Create LLVM type for struct vertex_header; 266 */ 267static LLVMTypeRef 268create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems) 269{ 270 LLVMTargetDataRef target = gallivm->target; 271 LLVMTypeRef elem_types[3]; 272 LLVMTypeRef vertex_header; 273 char struct_name[24]; 274 275 util_snprintf(struct_name, 23, "vertex_header%d", data_elems); 276 277 elem_types[0] = LLVMIntTypeInContext(gallivm->context, 32); 278 elem_types[1] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 279 elem_types[2] = LLVMArrayType(elem_types[1], data_elems); 280 281#if HAVE_LLVM >= 0x0300 282 vertex_header = LLVMStructCreateNamed(gallivm->context, struct_name); 283 LLVMStructSetBody(vertex_header, elem_types, 284 Elements(elem_types), 0); 285#else 286 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types, 287 Elements(elem_types), 0); 288 LLVMAddTypeName(gallivm->module, struct_name, vertex_header); 289 290 LLVMInvalidateStructLayout(gallivm->target, vertex_header); 291#endif 292 293 /* these are bit-fields and we can't take address of them 294 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask, 295 target, vertex_header, 296 DRAW_JIT_VERTEX_CLIPMASK); 297 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag, 298 target, vertex_header, 299 DRAW_JIT_VERTEX_EDGEFLAG); 300 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad, 301 target, vertex_header, 302 DRAW_JIT_VERTEX_PAD); 303 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id, 304 target, vertex_header, 305 DRAW_JIT_VERTEX_VERTEX_ID); 306 */ 307 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip, 308 target, vertex_header, 309 DRAW_JIT_VERTEX_CLIP); 310 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data, 311 target, vertex_header, 312 DRAW_JIT_VERTEX_DATA); 313 314 return vertex_header; 315} 316 317 318/** 319 * Create LLVM types for various structures. 320 */ 321static void 322create_jit_types(struct draw_llvm *llvm) 323{ 324 struct gallivm_state *gallivm = llvm->gallivm; 325 LLVMTypeRef texture_type, context_type, buffer_type, vb_type; 326 327 texture_type = create_jit_texture_type(gallivm, "texture"); 328 329 context_type = create_jit_context_type(gallivm, texture_type, "draw_jit_context"); 330 llvm->context_ptr_type = LLVMPointerType(context_type, 0); 331 332 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0); 333 llvm->buffer_ptr_type = LLVMPointerType(buffer_type, 0); 334 335 vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer"); 336 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0); 337} 338 339 340static LLVMTypeRef 341get_context_ptr_type(struct draw_llvm *llvm) 342{ 343 if (!llvm->context_ptr_type) 344 create_jit_types(llvm); 345 return llvm->context_ptr_type; 346} 347 348 349static LLVMTypeRef 350get_buffer_ptr_type(struct draw_llvm *llvm) 351{ 352 if (!llvm->buffer_ptr_type) 353 create_jit_types(llvm); 354 return llvm->buffer_ptr_type; 355} 356 357 358static LLVMTypeRef 359get_vb_ptr_type(struct draw_llvm *llvm) 360{ 361 if (!llvm->vb_ptr_type) 362 create_jit_types(llvm); 363 return llvm->vb_ptr_type; 364} 365 366static LLVMTypeRef 367get_vertex_header_ptr_type(struct draw_llvm *llvm) 368{ 369 if (!llvm->vertex_header_ptr_type) 370 create_jit_types(llvm); 371 return llvm->vertex_header_ptr_type; 372} 373 374 375/** 376 * Create per-context LLVM info. 377 */ 378struct draw_llvm * 379draw_llvm_create(struct draw_context *draw, struct gallivm_state *gallivm) 380{ 381 struct draw_llvm *llvm; 382 383 llvm = CALLOC_STRUCT( draw_llvm ); 384 if (!llvm) 385 return NULL; 386 387 lp_build_init(); 388 389 llvm->draw = draw; 390 llvm->gallivm = gallivm; 391 392 if (gallivm_debug & GALLIVM_DEBUG_IR) { 393 LLVMDumpModule(llvm->gallivm->module); 394 } 395 396 llvm->nr_variants = 0; 397 make_empty_list(&llvm->vs_variants_list); 398 399 gallivm_register_garbage_collector_callback( 400 draw_llvm_garbage_collect_callback, llvm); 401 402 return llvm; 403} 404 405 406/** 407 * Free per-context LLVM info. 408 */ 409void 410draw_llvm_destroy(struct draw_llvm *llvm) 411{ 412 gallivm_remove_garbage_collector_callback( 413 draw_llvm_garbage_collect_callback, llvm); 414 415 /* XXX free other draw_llvm data? */ 416 FREE(llvm); 417} 418 419 420/** 421 * Create LLVM-generated code for a vertex shader. 422 */ 423struct draw_llvm_variant * 424draw_llvm_create_variant(struct draw_llvm *llvm, 425 unsigned num_inputs, 426 const struct draw_llvm_variant_key *key) 427{ 428 struct draw_llvm_variant *variant; 429 struct llvm_vertex_shader *shader = 430 llvm_vertex_shader(llvm->draw->vs.vertex_shader); 431 LLVMTypeRef vertex_header; 432 433 variant = MALLOC(sizeof *variant + 434 shader->variant_key_size - 435 sizeof variant->key); 436 if (variant == NULL) 437 return NULL; 438 439 variant->llvm = llvm; 440 441 memcpy(&variant->key, key, shader->variant_key_size); 442 443 vertex_header = create_jit_vertex_header(llvm->gallivm, num_inputs); 444 445 llvm->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0); 446 447 draw_llvm_generate(llvm, variant, FALSE); /* linear */ 448 draw_llvm_generate(llvm, variant, TRUE); /* elts */ 449 450 variant->shader = shader; 451 variant->list_item_global.base = variant; 452 variant->list_item_local.base = variant; 453 /*variant->no = */shader->variants_created++; 454 variant->list_item_global.base = variant; 455 456 return variant; 457} 458 459 460static void 461generate_vs(struct draw_llvm *llvm, 462 LLVMBuilderRef builder, 463 LLVMValueRef (*outputs)[NUM_CHANNELS], 464 const LLVMValueRef (*inputs)[NUM_CHANNELS], 465 LLVMValueRef system_values_array, 466 LLVMValueRef context_ptr, 467 struct lp_build_sampler_soa *draw_sampler, 468 boolean clamp_vertex_color) 469{ 470 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens; 471 struct lp_type vs_type; 472 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(llvm->gallivm, context_ptr); 473 struct lp_build_sampler_soa *sampler = 0; 474 475 memset(&vs_type, 0, sizeof vs_type); 476 vs_type.floating = TRUE; /* floating point values */ 477 vs_type.sign = TRUE; /* values are signed */ 478 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 479 vs_type.width = 32; /* 32-bit float */ 480 vs_type.length = 4; /* 4 elements per vector */ 481#if 0 482 num_vs = 4; /* number of vertices per block */ 483#endif 484 485 if (gallivm_debug & GALLIVM_DEBUG_IR) { 486 tgsi_dump(tokens, 0); 487 } 488 489 if (llvm->draw->num_sampler_views && llvm->draw->num_samplers) 490 sampler = draw_sampler; 491 492 lp_build_tgsi_soa(llvm->gallivm, 493 tokens, 494 vs_type, 495 NULL /*struct lp_build_mask_context *mask*/, 496 consts_ptr, 497 system_values_array, 498 NULL /*pos*/, 499 inputs, 500 outputs, 501 sampler, 502 &llvm->draw->vs.vertex_shader->info); 503 504 if (clamp_vertex_color) { 505 LLVMValueRef out; 506 unsigned chan, attrib; 507 struct lp_build_context bld; 508 struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info; 509 lp_build_context_init(&bld, llvm->gallivm, vs_type); 510 511 for (attrib = 0; attrib < info->num_outputs; ++attrib) { 512 for (chan = 0; chan < NUM_CHANNELS; ++chan) { 513 if (outputs[attrib][chan]) { 514 switch (info->output_semantic_name[attrib]) { 515 case TGSI_SEMANTIC_COLOR: 516 case TGSI_SEMANTIC_BCOLOR: 517 out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 518 out = lp_build_clamp(&bld, out, bld.zero, bld.one); 519 LLVMBuildStore(builder, out, outputs[attrib][chan]); 520 break; 521 } 522 } 523 } 524 } 525 } 526} 527 528 529#if DEBUG_STORE 530static void print_vectorf(LLVMBuilderRef builder, 531 LLVMValueRef vec) 532{ 533 LLVMValueRef val[4]; 534 val[0] = LLVMBuildExtractElement(builder, vec, 535 lp_build_const_int32(gallivm, 0), ""); 536 val[1] = LLVMBuildExtractElement(builder, vec, 537 lp_build_const_int32(gallivm, 1), ""); 538 val[2] = LLVMBuildExtractElement(builder, vec, 539 lp_build_const_int32(gallivm, 2), ""); 540 val[3] = LLVMBuildExtractElement(builder, vec, 541 lp_build_const_int32(gallivm, 3), ""); 542 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n", 543 val[0], val[1], val[2], val[3]); 544} 545#endif 546 547 548static void 549generate_fetch(struct gallivm_state *gallivm, 550 LLVMValueRef vbuffers_ptr, 551 LLVMValueRef *res, 552 struct pipe_vertex_element *velem, 553 LLVMValueRef vbuf, 554 LLVMValueRef index, 555 LLVMValueRef instance_id) 556{ 557 LLVMBuilderRef builder = gallivm->builder; 558 LLVMValueRef indices = 559 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 560 velem->vertex_buffer_index, 0); 561 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr, 562 &indices, 1, ""); 563 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf); 564 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf); 565 LLVMValueRef stride; 566 567 if (velem->instance_divisor) { 568 /* array index = instance_id / instance_divisor */ 569 index = LLVMBuildUDiv(builder, instance_id, 570 lp_build_const_int32(gallivm, velem->instance_divisor), 571 "instance_divisor"); 572 } 573 574 stride = LLVMBuildMul(builder, vb_stride, index, ""); 575 576 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer"); 577 578 stride = LLVMBuildAdd(builder, stride, 579 vb_buffer_offset, 580 ""); 581 stride = LLVMBuildAdd(builder, stride, 582 lp_build_const_int32(gallivm, velem->src_offset), 583 ""); 584 585 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/ 586 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, ""); 587 588 *res = draw_llvm_translate_from(gallivm, vbuffer_ptr, velem->src_format); 589} 590 591 592static LLVMValueRef 593aos_to_soa(struct gallivm_state *gallivm, 594 LLVMValueRef val0, 595 LLVMValueRef val1, 596 LLVMValueRef val2, 597 LLVMValueRef val3, 598 LLVMValueRef channel) 599{ 600 LLVMBuilderRef builder = gallivm->builder; 601 LLVMValueRef ex, res; 602 603 ex = LLVMBuildExtractElement(builder, val0, 604 channel, ""); 605 res = LLVMBuildInsertElement(builder, 606 LLVMConstNull(LLVMTypeOf(val0)), 607 ex, 608 lp_build_const_int32(gallivm, 0), 609 ""); 610 611 ex = LLVMBuildExtractElement(builder, val1, 612 channel, ""); 613 res = LLVMBuildInsertElement(builder, 614 res, ex, 615 lp_build_const_int32(gallivm, 1), 616 ""); 617 618 ex = LLVMBuildExtractElement(builder, val2, 619 channel, ""); 620 res = LLVMBuildInsertElement(builder, 621 res, ex, 622 lp_build_const_int32(gallivm, 2), 623 ""); 624 625 ex = LLVMBuildExtractElement(builder, val3, 626 channel, ""); 627 res = LLVMBuildInsertElement(builder, 628 res, ex, 629 lp_build_const_int32(gallivm, 3), 630 ""); 631 632 return res; 633} 634 635 636static void 637soa_to_aos(struct gallivm_state *gallivm, 638 LLVMValueRef soa[NUM_CHANNELS], 639 LLVMValueRef aos[NUM_CHANNELS]) 640{ 641 LLVMBuilderRef builder = gallivm->builder; 642 LLVMValueRef comp; 643 int i = 0; 644 645 debug_assert(NUM_CHANNELS == 4); 646 647 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0])); 648 aos[1] = aos[2] = aos[3] = aos[0]; 649 650 for (i = 0; i < NUM_CHANNELS; ++i) { 651 LLVMValueRef channel = lp_build_const_int32(gallivm, i); 652 653 comp = LLVMBuildExtractElement(builder, soa[i], 654 lp_build_const_int32(gallivm, 0), ""); 655 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, ""); 656 657 comp = LLVMBuildExtractElement(builder, soa[i], 658 lp_build_const_int32(gallivm, 1), ""); 659 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, ""); 660 661 comp = LLVMBuildExtractElement(builder, soa[i], 662 lp_build_const_int32(gallivm, 2), ""); 663 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, ""); 664 665 comp = LLVMBuildExtractElement(builder, soa[i], 666 lp_build_const_int32(gallivm, 3), ""); 667 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, ""); 668 669 } 670} 671 672 673static void 674convert_to_soa(struct gallivm_state *gallivm, 675 LLVMValueRef (*aos)[NUM_CHANNELS], 676 LLVMValueRef (*soa)[NUM_CHANNELS], 677 int num_attribs) 678{ 679 int i; 680 681 debug_assert(NUM_CHANNELS == 4); 682 683 for (i = 0; i < num_attribs; ++i) { 684 LLVMValueRef val0 = aos[i][0]; 685 LLVMValueRef val1 = aos[i][1]; 686 LLVMValueRef val2 = aos[i][2]; 687 LLVMValueRef val3 = aos[i][3]; 688 689 soa[i][0] = aos_to_soa(gallivm, val0, val1, val2, val3, 690 lp_build_const_int32(gallivm, 0)); 691 soa[i][1] = aos_to_soa(gallivm, val0, val1, val2, val3, 692 lp_build_const_int32(gallivm, 1)); 693 soa[i][2] = aos_to_soa(gallivm, val0, val1, val2, val3, 694 lp_build_const_int32(gallivm, 2)); 695 soa[i][3] = aos_to_soa(gallivm, val0, val1, val2, val3, 696 lp_build_const_int32(gallivm, 3)); 697 } 698} 699 700 701static void 702store_aos(struct gallivm_state *gallivm, 703 LLVMValueRef io_ptr, 704 LLVMValueRef index, 705 LLVMValueRef value, 706 LLVMValueRef clipmask) 707{ 708 LLVMBuilderRef builder = gallivm->builder; 709 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptr); 710 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr); 711 LLVMValueRef indices[3]; 712 LLVMValueRef val; 713 int vertex_id_pad_edgeflag; 714 715 indices[0] = lp_build_const_int32(gallivm, 0); 716 indices[1] = index; 717 indices[2] = lp_build_const_int32(gallivm, 0); 718 719 /* If this assertion fails, it means we need to update the bit twidding 720 * code here. See struct vertex_header in draw_private.h. 721 */ 722 assert(DRAW_TOTAL_CLIP_PLANES==14); 723 /* initialize vertex id:16 = 0xffff, pad:1 = 0, edgeflag:1 = 1 */ 724 vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES); 725 val = lp_build_const_int32(gallivm, vertex_id_pad_edgeflag); 726 /* OR with the clipmask */ 727 val = LLVMBuildOr(builder, val, clipmask, ""); 728 729 /* store vertex header */ 730 LLVMBuildStore(builder, val, id_ptr); 731 732 733#if DEBUG_STORE 734 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr); 735#endif 736#if 0 737 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr); 738 print_vectorf(builder, value);*/ 739 data_ptr = LLVMBuildBitCast(builder, data_ptr, 740 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0), 0), 741 "datavec"); 742 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, ""); 743 744 LLVMBuildStore(builder, value, data_ptr); 745#else 746 { 747 LLVMValueRef x, y, z, w; 748 LLVMValueRef idx0, idx1, idx2, idx3; 749 LLVMValueRef gep0, gep1, gep2, gep3; 750 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, ""); 751 752 idx0 = lp_build_const_int32(gallivm, 0); 753 idx1 = lp_build_const_int32(gallivm, 1); 754 idx2 = lp_build_const_int32(gallivm, 2); 755 idx3 = lp_build_const_int32(gallivm, 3); 756 757 x = LLVMBuildExtractElement(builder, value, 758 idx0, ""); 759 y = LLVMBuildExtractElement(builder, value, 760 idx1, ""); 761 z = LLVMBuildExtractElement(builder, value, 762 idx2, ""); 763 w = LLVMBuildExtractElement(builder, value, 764 idx3, ""); 765 766 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, ""); 767 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, ""); 768 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, ""); 769 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, ""); 770 771 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n", 772 x, gep0, y, gep1, z, gep2, w, gep3);*/ 773 LLVMBuildStore(builder, x, gep0); 774 LLVMBuildStore(builder, y, gep1); 775 LLVMBuildStore(builder, z, gep2); 776 LLVMBuildStore(builder, w, gep3); 777 } 778#endif 779} 780 781 782static void 783store_aos_array(struct gallivm_state *gallivm, 784 LLVMValueRef io_ptr, 785 LLVMValueRef aos[NUM_CHANNELS], 786 int attrib, 787 int num_outputs, 788 LLVMValueRef clipmask) 789{ 790 LLVMBuilderRef builder = gallivm->builder; 791 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib); 792 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0); 793 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1); 794 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2); 795 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3); 796 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 797 LLVMValueRef clipmask0, clipmask1, clipmask2, clipmask3; 798 799 debug_assert(NUM_CHANNELS == 4); 800 801 io0_ptr = LLVMBuildGEP(builder, io_ptr, 802 &ind0, 1, ""); 803 io1_ptr = LLVMBuildGEP(builder, io_ptr, 804 &ind1, 1, ""); 805 io2_ptr = LLVMBuildGEP(builder, io_ptr, 806 &ind2, 1, ""); 807 io3_ptr = LLVMBuildGEP(builder, io_ptr, 808 &ind3, 1, ""); 809 810 clipmask0 = LLVMBuildExtractElement(builder, clipmask, 811 ind0, ""); 812 clipmask1 = LLVMBuildExtractElement(builder, clipmask, 813 ind1, ""); 814 clipmask2 = LLVMBuildExtractElement(builder, clipmask, 815 ind2, ""); 816 clipmask3 = LLVMBuildExtractElement(builder, clipmask, 817 ind3, ""); 818 819#if DEBUG_STORE 820 lp_build_printf(builder, "io = %p, indexes[%d, %d, %d, %d]\n, clipmask0 = %x, clipmask1 = %x, clipmask2 = %x, clipmask3 = %x\n", 821 io_ptr, ind0, ind1, ind2, ind3, clipmask0, clipmask1, clipmask2, clipmask3); 822#endif 823 /* store for each of the 4 vertices */ 824 store_aos(gallivm, io0_ptr, attr_index, aos[0], clipmask0); 825 store_aos(gallivm, io1_ptr, attr_index, aos[1], clipmask1); 826 store_aos(gallivm, io2_ptr, attr_index, aos[2], clipmask2); 827 store_aos(gallivm, io3_ptr, attr_index, aos[3], clipmask3); 828} 829 830 831static void 832convert_to_aos(struct gallivm_state *gallivm, 833 LLVMValueRef io, 834 LLVMValueRef (*outputs)[NUM_CHANNELS], 835 LLVMValueRef clipmask, 836 int num_outputs, 837 int max_vertices) 838{ 839 LLVMBuilderRef builder = gallivm->builder; 840 unsigned chan, attrib; 841 842#if DEBUG_STORE 843 lp_build_printf(builder, " # storing begin\n"); 844#endif 845 for (attrib = 0; attrib < num_outputs; ++attrib) { 846 LLVMValueRef soa[4]; 847 LLVMValueRef aos[4]; 848 for (chan = 0; chan < NUM_CHANNELS; ++chan) { 849 if (outputs[attrib][chan]) { 850 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 851 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]); 852 /*lp_build_printf(builder, "output %d : %d ", 853 LLVMConstInt(LLVMInt32Type(), attrib, 0), 854 LLVMConstInt(LLVMInt32Type(), chan, 0)); 855 print_vectorf(builder, out);*/ 856 soa[chan] = out; 857 } 858 else { 859 soa[chan] = 0; 860 } 861 } 862 soa_to_aos(gallivm, soa, aos); 863 store_aos_array(gallivm, 864 io, 865 aos, 866 attrib, 867 num_outputs, 868 clipmask); 869 } 870#if DEBUG_STORE 871 lp_build_printf(builder, " # storing end\n"); 872#endif 873} 874 875 876/** 877 * Stores original vertex positions in clip coordinates 878 * There is probably a more efficient way to do this, 4 floats at once 879 * rather than extracting each element one by one. 880 */ 881static void 882store_clip(struct gallivm_state *gallivm, 883 LLVMValueRef io_ptr, 884 LLVMValueRef (*outputs)[NUM_CHANNELS]) 885{ 886 LLVMBuilderRef builder = gallivm->builder; 887 LLVMValueRef out[4]; 888 LLVMValueRef indices[2]; 889 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 890 LLVMValueRef clip_ptr0, clip_ptr1, clip_ptr2, clip_ptr3; 891 LLVMValueRef clip0_ptr, clip1_ptr, clip2_ptr, clip3_ptr; 892 LLVMValueRef out0elem, out1elem, out2elem, out3elem; 893 int i; 894 895 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0); 896 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1); 897 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2); 898 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3); 899 900 indices[0] = 901 indices[1] = lp_build_const_int32(gallivm, 0); 902 903 out[0] = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/ 904 out[1] = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/ 905 out[2] = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/ 906 out[3] = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 907 908 io0_ptr = LLVMBuildGEP(builder, io_ptr, &ind0, 1, ""); 909 io1_ptr = LLVMBuildGEP(builder, io_ptr, &ind1, 1, ""); 910 io2_ptr = LLVMBuildGEP(builder, io_ptr, &ind2, 1, ""); 911 io3_ptr = LLVMBuildGEP(builder, io_ptr, &ind3, 1, ""); 912 913 clip_ptr0 = draw_jit_header_clip(gallivm, io0_ptr); 914 clip_ptr1 = draw_jit_header_clip(gallivm, io1_ptr); 915 clip_ptr2 = draw_jit_header_clip(gallivm, io2_ptr); 916 clip_ptr3 = draw_jit_header_clip(gallivm, io3_ptr); 917 918 for (i = 0; i<4; i++) { 919 clip0_ptr = LLVMBuildGEP(builder, clip_ptr0, indices, 2, ""); /* x0 */ 920 clip1_ptr = LLVMBuildGEP(builder, clip_ptr1, indices, 2, ""); /* x1 */ 921 clip2_ptr = LLVMBuildGEP(builder, clip_ptr2, indices, 2, ""); /* x2 */ 922 clip3_ptr = LLVMBuildGEP(builder, clip_ptr3, indices, 2, ""); /* x3 */ 923 924 out0elem = LLVMBuildExtractElement(builder, out[i], ind0, ""); /* x0 */ 925 out1elem = LLVMBuildExtractElement(builder, out[i], ind1, ""); /* x1 */ 926 out2elem = LLVMBuildExtractElement(builder, out[i], ind2, ""); /* x2 */ 927 out3elem = LLVMBuildExtractElement(builder, out[i], ind3, ""); /* x3 */ 928 929 LLVMBuildStore(builder, out0elem, clip0_ptr); 930 LLVMBuildStore(builder, out1elem, clip1_ptr); 931 LLVMBuildStore(builder, out2elem, clip2_ptr); 932 LLVMBuildStore(builder, out3elem, clip3_ptr); 933 934 indices[1]= LLVMBuildAdd(builder, indices[1], ind1, ""); 935 } 936 937} 938 939 940/** 941 * Equivalent of _mm_set1_ps(a) 942 */ 943static LLVMValueRef 944vec4f_from_scalar(struct gallivm_state *gallivm, 945 LLVMValueRef a, 946 const char *name) 947{ 948 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 949 LLVMValueRef res = LLVMGetUndef(LLVMVectorType(float_type, 4)); 950 int i; 951 952 for (i = 0; i < 4; ++i) { 953 LLVMValueRef index = lp_build_const_int32(gallivm, i); 954 res = LLVMBuildInsertElement(gallivm->builder, res, a, 955 index, i == 3 ? name : ""); 956 } 957 958 return res; 959} 960 961 962/** 963 * Transforms the outputs for viewport mapping 964 */ 965static void 966generate_viewport(struct draw_llvm *llvm, 967 LLVMBuilderRef builder, 968 LLVMValueRef (*outputs)[NUM_CHANNELS], 969 LLVMValueRef context_ptr) 970{ 971 int i; 972 struct gallivm_state *gallivm = llvm->gallivm; 973 struct lp_type f32_type = lp_type_float_vec(32); 974 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 975 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/ 976 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr); 977 978 /* for 1/w convention*/ 979 out3 = LLVMBuildFDiv(builder, const1, out3, ""); 980 LLVMBuildStore(builder, out3, outputs[0][3]); 981 982 /* Viewport Mapping */ 983 for (i=0; i<3; i++) { 984 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 x2 x3*/ 985 LLVMValueRef scale; 986 LLVMValueRef trans; 987 LLVMValueRef scale_i; 988 LLVMValueRef trans_i; 989 LLVMValueRef index; 990 991 index = lp_build_const_int32(gallivm, i); 992 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 993 994 index = lp_build_const_int32(gallivm, i+4); 995 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 996 997 scale = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, scale_i, ""), "scale"); 998 trans = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, trans_i, ""), "trans"); 999 1000 /* divide by w */ 1001 out = LLVMBuildFMul(builder, out, out3, ""); 1002 /* mult by scale */ 1003 out = LLVMBuildFMul(builder, out, scale, ""); 1004 /* add translation */ 1005 out = LLVMBuildFAdd(builder, out, trans, ""); 1006 1007 /* store transformed outputs */ 1008 LLVMBuildStore(builder, out, outputs[0][i]); 1009 } 1010 1011} 1012 1013 1014/** 1015 * Returns clipmask as 4xi32 bitmask for the 4 vertices 1016 */ 1017static LLVMValueRef 1018generate_clipmask(struct gallivm_state *gallivm, 1019 LLVMValueRef (*outputs)[NUM_CHANNELS], 1020 boolean clip_xy, 1021 boolean clip_z, 1022 boolean clip_user, 1023 boolean clip_halfz, 1024 unsigned ucp_enable, 1025 LLVMValueRef context_ptr) 1026{ 1027 LLVMBuilderRef builder = gallivm->builder; 1028 LLVMValueRef mask; /* stores the <4xi32> clipmasks */ 1029 LLVMValueRef test, temp; 1030 LLVMValueRef zero, shift; 1031 LLVMValueRef pos_x, pos_y, pos_z, pos_w; 1032 LLVMValueRef plane1, planes, plane_ptr, sum; 1033 struct lp_type f32_type = lp_type_float_vec(32); 1034 1035 mask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1036 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1037 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */ 1038 shift = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1); /* 1 1 1 1 */ 1039 1040 /* Assuming position stored at output[0] */ 1041 pos_x = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/ 1042 pos_y = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/ 1043 pos_z = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/ 1044 pos_w = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 1045 1046 /* Cliptest, for hardwired planes */ 1047 if (clip_xy) { 1048 /* plane 1 */ 1049 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w); 1050 temp = shift; 1051 test = LLVMBuildAnd(builder, test, temp, ""); 1052 mask = test; 1053 1054 /* plane 2 */ 1055 test = LLVMBuildFAdd(builder, pos_x, pos_w, ""); 1056 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1057 temp = LLVMBuildShl(builder, temp, shift, ""); 1058 test = LLVMBuildAnd(builder, test, temp, ""); 1059 mask = LLVMBuildOr(builder, mask, test, ""); 1060 1061 /* plane 3 */ 1062 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w); 1063 temp = LLVMBuildShl(builder, temp, shift, ""); 1064 test = LLVMBuildAnd(builder, test, temp, ""); 1065 mask = LLVMBuildOr(builder, mask, test, ""); 1066 1067 /* plane 4 */ 1068 test = LLVMBuildFAdd(builder, pos_y, pos_w, ""); 1069 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1070 temp = LLVMBuildShl(builder, temp, shift, ""); 1071 test = LLVMBuildAnd(builder, test, temp, ""); 1072 mask = LLVMBuildOr(builder, mask, test, ""); 1073 } 1074 1075 if (clip_z) { 1076 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 16); 1077 if (clip_halfz) { 1078 /* plane 5 */ 1079 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z); 1080 test = LLVMBuildAnd(builder, test, temp, ""); 1081 mask = LLVMBuildOr(builder, mask, test, ""); 1082 } 1083 else { 1084 /* plane 5 */ 1085 test = LLVMBuildFAdd(builder, pos_z, pos_w, ""); 1086 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1087 test = LLVMBuildAnd(builder, test, temp, ""); 1088 mask = LLVMBuildOr(builder, mask, test, ""); 1089 } 1090 /* plane 6 */ 1091 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w); 1092 temp = LLVMBuildShl(builder, temp, shift, ""); 1093 test = LLVMBuildAnd(builder, test, temp, ""); 1094 mask = LLVMBuildOr(builder, mask, test, ""); 1095 } 1096 1097 if (clip_user) { 1098 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr); 1099 LLVMValueRef indices[3]; 1100 1101 /* userclip planes */ 1102 while (ucp_enable) { 1103 unsigned plane_idx = ffs(ucp_enable)-1; 1104 ucp_enable &= ~(1 << plane_idx); 1105 plane_idx += 6; 1106 1107 indices[0] = lp_build_const_int32(gallivm, 0); 1108 indices[1] = lp_build_const_int32(gallivm, plane_idx); 1109 1110 indices[2] = lp_build_const_int32(gallivm, 0); 1111 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1112 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x"); 1113 planes = vec4f_from_scalar(gallivm, plane1, "plane4_x"); 1114 sum = LLVMBuildFMul(builder, planes, pos_x, ""); 1115 1116 indices[2] = lp_build_const_int32(gallivm, 1); 1117 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1118 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y"); 1119 planes = vec4f_from_scalar(gallivm, plane1, "plane4_y"); 1120 test = LLVMBuildFMul(builder, planes, pos_y, ""); 1121 sum = LLVMBuildFAdd(builder, sum, test, ""); 1122 1123 indices[2] = lp_build_const_int32(gallivm, 2); 1124 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1125 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z"); 1126 planes = vec4f_from_scalar(gallivm, plane1, "plane4_z"); 1127 test = LLVMBuildFMul(builder, planes, pos_z, ""); 1128 sum = LLVMBuildFAdd(builder, sum, test, ""); 1129 1130 indices[2] = lp_build_const_int32(gallivm, 3); 1131 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1132 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w"); 1133 planes = vec4f_from_scalar(gallivm, plane1, "plane4_w"); 1134 test = LLVMBuildFMul(builder, planes, pos_w, ""); 1135 sum = LLVMBuildFAdd(builder, sum, test, ""); 1136 1137 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum); 1138 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1 << plane_idx); 1139 test = LLVMBuildAnd(builder, test, temp, ""); 1140 mask = LLVMBuildOr(builder, mask, test, ""); 1141 } 1142 } 1143 return mask; 1144} 1145 1146 1147/** 1148 * Returns boolean if any clipping has occurred 1149 * Used zero/non-zero i32 value to represent boolean 1150 */ 1151static void 1152clipmask_bool(struct gallivm_state *gallivm, 1153 LLVMValueRef clipmask, 1154 LLVMValueRef ret_ptr) 1155{ 1156 LLVMBuilderRef builder = gallivm->builder; 1157 LLVMValueRef ret = LLVMBuildLoad(builder, ret_ptr, ""); 1158 LLVMValueRef temp; 1159 int i; 1160 1161 for (i=0; i<4; i++) { 1162 temp = LLVMBuildExtractElement(builder, clipmask, 1163 lp_build_const_int32(gallivm, i) , ""); 1164 ret = LLVMBuildOr(builder, ret, temp, ""); 1165 } 1166 1167 LLVMBuildStore(builder, ret, ret_ptr); 1168} 1169 1170 1171static void 1172draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant, 1173 boolean elts) 1174{ 1175 struct gallivm_state *gallivm = llvm->gallivm; 1176 LLVMContextRef context = gallivm->context; 1177 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 1178 LLVMTypeRef arg_types[8]; 1179 LLVMTypeRef func_type; 1180 LLVMValueRef context_ptr; 1181 LLVMBasicBlockRef block; 1182 LLVMBuilderRef builder; 1183 LLVMValueRef end, start; 1184 LLVMValueRef count, fetch_elts, fetch_count; 1185 LLVMValueRef stride, step, io_itr; 1186 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 1187 LLVMValueRef instance_id; 1188 LLVMValueRef system_values_array; 1189 LLVMValueRef zero = lp_build_const_int32(gallivm, 0); 1190 LLVMValueRef one = lp_build_const_int32(gallivm, 1); 1191 struct draw_context *draw = llvm->draw; 1192 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info; 1193 unsigned i, j; 1194 struct lp_build_context bld; 1195 struct lp_build_loop_state lp_loop; 1196 const int max_vertices = 4; 1197 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS]; 1198 LLVMValueRef fetch_max; 1199 void *code; 1200 struct lp_build_sampler_soa *sampler = 0; 1201 LLVMValueRef ret, ret_ptr; 1202 const boolean bypass_viewport = variant->key.bypass_viewport; 1203 const boolean enable_cliptest = variant->key.clip_xy || 1204 variant->key.clip_z || 1205 variant->key.clip_user; 1206 LLVMValueRef variant_func; 1207 1208 arg_types[0] = get_context_ptr_type(llvm); /* context */ 1209 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */ 1210 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */ 1211 if (elts) 1212 arg_types[3] = LLVMPointerType(int32_type, 0);/* fetch_elts * */ 1213 else 1214 arg_types[3] = int32_type; /* start */ 1215 arg_types[4] = int32_type; /* fetch_count / count */ 1216 arg_types[5] = int32_type; /* stride */ 1217 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */ 1218 arg_types[7] = int32_type; /* instance_id */ 1219 1220 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0); 1221 1222 variant_func = LLVMAddFunction(gallivm->module, 1223 elts ? "draw_llvm_shader_elts" : "draw_llvm_shader", 1224 func_type); 1225 1226 if (elts) 1227 variant->function_elts = variant_func; 1228 else 1229 variant->function = variant_func; 1230 1231 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); 1232 for (i = 0; i < Elements(arg_types); ++i) 1233 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 1234 LLVMAddAttribute(LLVMGetParam(variant_func, i), 1235 LLVMNoAliasAttribute); 1236 1237 context_ptr = LLVMGetParam(variant_func, 0); 1238 io_ptr = LLVMGetParam(variant_func, 1); 1239 vbuffers_ptr = LLVMGetParam(variant_func, 2); 1240 stride = LLVMGetParam(variant_func, 5); 1241 vb_ptr = LLVMGetParam(variant_func, 6); 1242 instance_id = LLVMGetParam(variant_func, 7); 1243 1244 lp_build_name(context_ptr, "context"); 1245 lp_build_name(io_ptr, "io"); 1246 lp_build_name(vbuffers_ptr, "vbuffers"); 1247 lp_build_name(stride, "stride"); 1248 lp_build_name(vb_ptr, "vb"); 1249 lp_build_name(instance_id, "instance_id"); 1250 1251 if (elts) { 1252 fetch_elts = LLVMGetParam(variant_func, 3); 1253 fetch_count = LLVMGetParam(variant_func, 4); 1254 lp_build_name(fetch_elts, "fetch_elts"); 1255 lp_build_name(fetch_count, "fetch_count"); 1256 start = count = NULL; 1257 } 1258 else { 1259 start = LLVMGetParam(variant_func, 3); 1260 count = LLVMGetParam(variant_func, 4); 1261 lp_build_name(start, "start"); 1262 lp_build_name(count, "count"); 1263 fetch_elts = fetch_count = NULL; 1264 } 1265 1266 /* 1267 * Function body 1268 */ 1269 1270 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry"); 1271 builder = gallivm->builder; 1272 LLVMPositionBuilderAtEnd(builder, block); 1273 1274 lp_build_context_init(&bld, gallivm, lp_type_int(32)); 1275 1276 system_values_array = lp_build_system_values_array(gallivm, vs_info, 1277 instance_id, NULL); 1278 1279 /* function will return non-zero i32 value if any clipped vertices */ 1280 ret_ptr = lp_build_alloca(gallivm, int32_type, ""); 1281 LLVMBuildStore(builder, zero, ret_ptr); 1282 1283 /* code generated texture sampling */ 1284 sampler = draw_llvm_sampler_soa_create( 1285 draw_llvm_variant_key_samplers(&variant->key), 1286 context_ptr); 1287 1288 if (elts) { 1289 start = zero; 1290 end = fetch_count; 1291 } 1292 else { 1293 end = lp_build_add(&bld, start, count); 1294 } 1295 1296 step = lp_build_const_int32(gallivm, max_vertices); 1297 1298 fetch_max = LLVMBuildSub(builder, end, one, "fetch_max"); 1299 1300 lp_build_loop_begin(&lp_loop, gallivm, start); 1301 { 1302 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS]; 1303 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } }; 1304 LLVMValueRef io; 1305 LLVMValueRef clipmask; /* holds the clipmask value */ 1306 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS]; 1307 1308 if (elts) 1309 io_itr = lp_loop.counter; 1310 else 1311 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, ""); 1312 1313 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 1314#if DEBUG_STORE 1315 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n", 1316 io_itr, io, lp_loop.counter); 1317#endif 1318 for (i = 0; i < NUM_CHANNELS; ++i) { 1319 LLVMValueRef true_index = 1320 LLVMBuildAdd(builder, 1321 lp_loop.counter, 1322 lp_build_const_int32(gallivm, i), ""); 1323 1324 /* make sure we're not out of bounds which can happen 1325 * if fetch_count % 4 != 0, because on the last iteration 1326 * a few of the 4 vertex fetches will be out of bounds */ 1327 true_index = lp_build_min(&bld, true_index, fetch_max); 1328 1329 if (elts) { 1330 LLVMValueRef fetch_ptr; 1331 fetch_ptr = LLVMBuildGEP(builder, fetch_elts, 1332 &true_index, 1, ""); 1333 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt"); 1334 } 1335 1336 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 1337 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 1338 LLVMValueRef vb_index = 1339 lp_build_const_int32(gallivm, velem->vertex_buffer_index); 1340 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, ""); 1341 generate_fetch(gallivm, vbuffers_ptr, 1342 &aos_attribs[j][i], velem, vb, true_index, 1343 instance_id); 1344 } 1345 } 1346 convert_to_soa(gallivm, aos_attribs, inputs, 1347 draw->pt.nr_vertex_elements); 1348 1349 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs; 1350 generate_vs(llvm, 1351 builder, 1352 outputs, 1353 ptr_aos, 1354 system_values_array, 1355 context_ptr, 1356 sampler, 1357 variant->key.clamp_vertex_color); 1358 1359 /* store original positions in clip before further manipulation */ 1360 store_clip(gallivm, io, outputs); 1361 1362 /* do cliptest */ 1363 if (enable_cliptest) { 1364 /* allocate clipmask, assign it integer type */ 1365 clipmask = generate_clipmask(gallivm, outputs, 1366 variant->key.clip_xy, 1367 variant->key.clip_z, 1368 variant->key.clip_user, 1369 variant->key.clip_halfz, 1370 variant->key.ucp_enable, 1371 context_ptr); 1372 /* return clipping boolean value for function */ 1373 clipmask_bool(gallivm, clipmask, ret_ptr); 1374 } 1375 else { 1376 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1377 } 1378 1379 /* do viewport mapping */ 1380 if (!bypass_viewport) { 1381 generate_viewport(llvm, builder, outputs, context_ptr); 1382 } 1383 1384 /* store clipmask in vertex header, 1385 * original positions in clip 1386 * and transformed positions in data 1387 */ 1388 convert_to_aos(gallivm, io, outputs, clipmask, 1389 vs_info->num_outputs, max_vertices); 1390 } 1391 1392 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE); 1393 1394 sampler->destroy(sampler); 1395 1396 ret = LLVMBuildLoad(builder, ret_ptr, ""); 1397 LLVMBuildRet(builder, ret); 1398 1399 /* 1400 * Translate the LLVM IR into machine code. 1401 */ 1402#ifdef DEBUG 1403 if (LLVMVerifyFunction(variant_func, LLVMPrintMessageAction)) { 1404 lp_debug_dump_value(variant_func); 1405 assert(0); 1406 } 1407#endif 1408 1409 LLVMRunFunctionPassManager(gallivm->passmgr, variant_func); 1410 1411 if (gallivm_debug & GALLIVM_DEBUG_IR) { 1412 lp_debug_dump_value(variant_func); 1413 debug_printf("\n"); 1414 } 1415 1416 code = LLVMGetPointerToGlobal(gallivm->engine, variant_func); 1417 if (elts) 1418 variant->jit_func_elts = (draw_jit_vert_func_elts) pointer_to_func(code); 1419 else 1420 variant->jit_func = (draw_jit_vert_func) pointer_to_func(code); 1421 1422 if (gallivm_debug & GALLIVM_DEBUG_ASM) { 1423 lp_disassemble(code); 1424 } 1425 lp_func_delete_body(variant_func); 1426} 1427 1428 1429struct draw_llvm_variant_key * 1430draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store) 1431{ 1432 unsigned i; 1433 struct draw_llvm_variant_key *key; 1434 struct lp_sampler_static_state *sampler; 1435 1436 key = (struct draw_llvm_variant_key *)store; 1437 1438 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/ 1439 1440 /* Presumably all variants of the shader should have the same 1441 * number of vertex elements - ie the number of shader inputs. 1442 */ 1443 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements; 1444 1445 /* will have to rig this up properly later */ 1446 key->clip_xy = llvm->draw->clip_xy; 1447 key->clip_z = llvm->draw->clip_z; 1448 key->clip_user = llvm->draw->clip_user; 1449 key->bypass_viewport = llvm->draw->identity_viewport; 1450 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules; 1451 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE); 1452 key->ucp_enable = llvm->draw->rasterizer->clip_plane_enable; 1453 key->pad = 0; 1454 1455 /* All variants of this shader will have the same value for 1456 * nr_samplers. Not yet trying to compact away holes in the 1457 * sampler array. 1458 */ 1459 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1; 1460 1461 sampler = draw_llvm_variant_key_samplers(key); 1462 1463 memcpy(key->vertex_element, 1464 llvm->draw->pt.vertex_element, 1465 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements); 1466 1467 memset(sampler, 0, key->nr_samplers * sizeof *sampler); 1468 1469 for (i = 0 ; i < key->nr_samplers; i++) { 1470 lp_sampler_static_state(&sampler[i], 1471 llvm->draw->sampler_views[i], 1472 llvm->draw->samplers[i]); 1473 } 1474 1475 return key; 1476} 1477 1478 1479void 1480draw_llvm_set_mapped_texture(struct draw_context *draw, 1481 unsigned sampler_idx, 1482 uint32_t width, uint32_t height, uint32_t depth, 1483 uint32_t first_level, uint32_t last_level, 1484 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS], 1485 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS], 1486 const void *data[PIPE_MAX_TEXTURE_LEVELS]) 1487{ 1488 unsigned j; 1489 struct draw_jit_texture *jit_tex; 1490 1491 assert(sampler_idx < PIPE_MAX_VERTEX_SAMPLERS); 1492 1493 jit_tex = &draw->llvm->jit_context.textures[sampler_idx]; 1494 1495 jit_tex->width = width; 1496 jit_tex->height = height; 1497 jit_tex->depth = depth; 1498 jit_tex->first_level = first_level; 1499 jit_tex->last_level = last_level; 1500 1501 for (j = first_level; j <= last_level; j++) { 1502 jit_tex->data[j] = data[j]; 1503 jit_tex->row_stride[j] = row_stride[j]; 1504 jit_tex->img_stride[j] = img_stride[j]; 1505 } 1506} 1507 1508 1509void 1510draw_llvm_set_sampler_state(struct draw_context *draw) 1511{ 1512 unsigned i; 1513 1514 for (i = 0; i < draw->num_samplers; i++) { 1515 struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i]; 1516 1517 if (draw->samplers[i]) { 1518 jit_tex->min_lod = draw->samplers[i]->min_lod; 1519 jit_tex->max_lod = draw->samplers[i]->max_lod; 1520 jit_tex->lod_bias = draw->samplers[i]->lod_bias; 1521 COPY_4V(jit_tex->border_color, draw->samplers[i]->border_color.f); 1522 } 1523 } 1524} 1525 1526 1527void 1528draw_llvm_destroy_variant(struct draw_llvm_variant *variant) 1529{ 1530 struct draw_llvm *llvm = variant->llvm; 1531 1532 if (variant->function_elts) { 1533 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine, 1534 variant->function_elts); 1535 LLVMDeleteFunction(variant->function_elts); 1536 } 1537 1538 if (variant->function) { 1539 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine, 1540 variant->function); 1541 LLVMDeleteFunction(variant->function); 1542 } 1543 1544 remove_from_list(&variant->list_item_local); 1545 variant->shader->variants_cached--; 1546 remove_from_list(&variant->list_item_global); 1547 llvm->nr_variants--; 1548 FREE(variant); 1549} 1550