draw_llvm.c revision 4552fd50d959ab99546cfa994f8ba5bdf5d66bc7
1/************************************************************************** 2 * 3 * Copyright 2010 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28#include "draw_llvm.h" 29 30#include "draw_context.h" 31#include "draw_vs.h" 32 33#include "gallivm/lp_bld_arit.h" 34#include "gallivm/lp_bld_logic.h" 35#include "gallivm/lp_bld_const.h" 36#include "gallivm/lp_bld_swizzle.h" 37#include "gallivm/lp_bld_struct.h" 38#include "gallivm/lp_bld_type.h" 39#include "gallivm/lp_bld_flow.h" 40#include "gallivm/lp_bld_debug.h" 41#include "gallivm/lp_bld_tgsi.h" 42#include "gallivm/lp_bld_printf.h" 43#include "gallivm/lp_bld_intr.h" 44#include "gallivm/lp_bld_init.h" 45#include "gallivm/lp_bld_type.h" 46 47#include "tgsi/tgsi_exec.h" 48#include "tgsi/tgsi_dump.h" 49 50#include "util/u_math.h" 51#include "util/u_pointer.h" 52#include "util/u_string.h" 53#include "util/u_simple_list.h" 54 55 56#define DEBUG_STORE 0 57 58 59/** 60 * This function is called by the gallivm "garbage collector" when 61 * the LLVM global data structures are freed. We must free all LLVM-related 62 * data. Specifically, all JIT'd shader variants. 63 */ 64static void 65draw_llvm_garbage_collect_callback(void *cb_data) 66{ 67 struct draw_llvm *llvm = (struct draw_llvm *) cb_data; 68 struct draw_context *draw = llvm->draw; 69 struct draw_llvm_variant_list_item *li; 70 71 /* Ensure prepare will be run and shaders recompiled */ 72 assert(!draw->suspend_flushing); 73 draw_do_flush(draw, DRAW_FLUSH_STATE_CHANGE); 74 75 /* free all shader variants */ 76 li = first_elem(&llvm->vs_variants_list); 77 while (!at_end(&llvm->vs_variants_list, li)) { 78 struct draw_llvm_variant_list_item *next = next_elem(li); 79 draw_llvm_destroy_variant(li->base); 80 li = next; 81 } 82 83 /* Null-out these pointers so they get remade next time they're needed. 84 * See the accessor functions below. 85 */ 86 llvm->context_ptr_type = NULL; 87 llvm->buffer_ptr_type = NULL; 88 llvm->vb_ptr_type = NULL; 89 llvm->vertex_header_ptr_type = NULL; 90} 91 92 93static void 94draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var, 95 boolean elts); 96 97 98/** 99 * Create LLVM type for struct draw_jit_texture 100 */ 101static LLVMTypeRef 102create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name) 103{ 104 LLVMTargetDataRef target = gallivm->target; 105 LLVMTypeRef texture_type; 106 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS]; 107 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 108 109 elem_types[DRAW_JIT_TEXTURE_WIDTH] = 110 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = 111 elem_types[DRAW_JIT_TEXTURE_DEPTH] = 112 elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] = 113 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type; 114 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] = 115 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] = 116 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS); 117 elem_types[DRAW_JIT_TEXTURE_DATA] = 118 LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0), 119 PIPE_MAX_TEXTURE_LEVELS); 120 elem_types[DRAW_JIT_TEXTURE_MIN_LOD] = 121 elem_types[DRAW_JIT_TEXTURE_MAX_LOD] = 122 elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context); 123 elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] = 124 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 125 126#if HAVE_LLVM >= 0x0300 127 texture_type = LLVMStructCreateNamed(gallivm->context, struct_name); 128 LLVMStructSetBody(texture_type, elem_types, 129 Elements(elem_types), 0); 130#else 131 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types, 132 Elements(elem_types), 0); 133 134 LLVMAddTypeName(gallivm->module, struct_name, texture_type); 135 136 /* Make sure the target's struct layout cache doesn't return 137 * stale/invalid data. 138 */ 139 LLVMInvalidateStructLayout(gallivm->target, texture_type); 140#endif 141 142 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width, 143 target, texture_type, 144 DRAW_JIT_TEXTURE_WIDTH); 145 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height, 146 target, texture_type, 147 DRAW_JIT_TEXTURE_HEIGHT); 148 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth, 149 target, texture_type, 150 DRAW_JIT_TEXTURE_DEPTH); 151 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level, 152 target, texture_type, 153 DRAW_JIT_TEXTURE_FIRST_LEVEL); 154 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level, 155 target, texture_type, 156 DRAW_JIT_TEXTURE_LAST_LEVEL); 157 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride, 158 target, texture_type, 159 DRAW_JIT_TEXTURE_ROW_STRIDE); 160 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride, 161 target, texture_type, 162 DRAW_JIT_TEXTURE_IMG_STRIDE); 163 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data, 164 target, texture_type, 165 DRAW_JIT_TEXTURE_DATA); 166 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod, 167 target, texture_type, 168 DRAW_JIT_TEXTURE_MIN_LOD); 169 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod, 170 target, texture_type, 171 DRAW_JIT_TEXTURE_MAX_LOD); 172 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias, 173 target, texture_type, 174 DRAW_JIT_TEXTURE_LOD_BIAS); 175 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color, 176 target, texture_type, 177 DRAW_JIT_TEXTURE_BORDER_COLOR); 178 179 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type); 180 181 return texture_type; 182} 183 184 185/** 186 * Create LLVM type for struct draw_jit_texture 187 */ 188static LLVMTypeRef 189create_jit_context_type(struct gallivm_state *gallivm, 190 LLVMTypeRef texture_type, const char *struct_name) 191{ 192 LLVMTargetDataRef target = gallivm->target; 193 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 194 LLVMTypeRef elem_types[5]; 195 LLVMTypeRef context_type; 196 197 elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */ 198 elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */ 199 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), 200 DRAW_TOTAL_CLIP_PLANES), 0); 201 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */ 202 elem_types[4] = LLVMArrayType(texture_type, 203 PIPE_MAX_VERTEX_SAMPLERS); /* textures */ 204#if HAVE_LLVM >= 0x0300 205 context_type = LLVMStructCreateNamed(gallivm->context, struct_name); 206 LLVMStructSetBody(context_type, elem_types, 207 Elements(elem_types), 0); 208#else 209 context_type = LLVMStructTypeInContext(gallivm->context, elem_types, 210 Elements(elem_types), 0); 211 LLVMAddTypeName(gallivm->module, struct_name, context_type); 212 213 LLVMInvalidateStructLayout(gallivm->target, context_type); 214#endif 215 216 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants, 217 target, context_type, 0); 218 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants, 219 target, context_type, 1); 220 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes, 221 target, context_type, 2); 222 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures, 223 target, context_type, 224 DRAW_JIT_CTX_TEXTURES); 225 LP_CHECK_STRUCT_SIZE(struct draw_jit_context, 226 target, context_type); 227 228 return context_type; 229} 230 231 232/** 233 * Create LLVM type for struct pipe_vertex_buffer 234 */ 235static LLVMTypeRef 236create_jit_vertex_buffer_type(struct gallivm_state *gallivm, const char *struct_name) 237{ 238 LLVMTargetDataRef target = gallivm->target; 239 LLVMTypeRef elem_types[4]; 240 LLVMTypeRef vb_type; 241 242 elem_types[0] = 243 elem_types[1] = LLVMInt32TypeInContext(gallivm->context); 244 elem_types[2] = 245 elem_types[3] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */ 246 247#if HAVE_LLVM >= 0x0300 248 vb_type = LLVMStructCreateNamed(gallivm->context, struct_name); 249 LLVMStructSetBody(vb_type, elem_types, 250 Elements(elem_types), 0); 251#else 252 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types, 253 Elements(elem_types), 0); 254 LLVMAddTypeName(gallivm->module, struct_name, vb_type); 255 256 LLVMInvalidateStructLayout(gallivm->target, vb_type); 257#endif 258 259 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride, 260 target, vb_type, 0); 261 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset, 262 target, vb_type, 1); 263 264 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type); 265 266 return vb_type; 267} 268 269 270/** 271 * Create LLVM type for struct vertex_header; 272 */ 273static LLVMTypeRef 274create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems) 275{ 276 LLVMTargetDataRef target = gallivm->target; 277 LLVMTypeRef elem_types[4]; 278 LLVMTypeRef vertex_header; 279 char struct_name[24]; 280 281 util_snprintf(struct_name, 23, "vertex_header%d", data_elems); 282 283 elem_types[DRAW_JIT_VERTEX_VERTEX_ID] = LLVMIntTypeInContext(gallivm->context, 32); 284 elem_types[DRAW_JIT_VERTEX_CLIP] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 285 elem_types[DRAW_JIT_VERTEX_PRE_CLIP_POS] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 286 elem_types[DRAW_JIT_VERTEX_DATA] = LLVMArrayType(elem_types[1], data_elems); 287 288#if HAVE_LLVM >= 0x0300 289 vertex_header = LLVMStructCreateNamed(gallivm->context, struct_name); 290 LLVMStructSetBody(vertex_header, elem_types, 291 Elements(elem_types), 0); 292#else 293 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types, 294 Elements(elem_types), 0); 295 LLVMAddTypeName(gallivm->module, struct_name, vertex_header); 296 297 LLVMInvalidateStructLayout(gallivm->target, vertex_header); 298#endif 299 300 /* these are bit-fields and we can't take address of them 301 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask, 302 target, vertex_header, 303 DRAW_JIT_VERTEX_CLIPMASK); 304 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag, 305 target, vertex_header, 306 DRAW_JIT_VERTEX_EDGEFLAG); 307 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad, 308 target, vertex_header, 309 DRAW_JIT_VERTEX_PAD); 310 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id, 311 target, vertex_header, 312 DRAW_JIT_VERTEX_VERTEX_ID); 313 */ 314 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip, 315 target, vertex_header, 316 DRAW_JIT_VERTEX_CLIP); 317 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pre_clip_pos, 318 target, vertex_header, 319 DRAW_JIT_VERTEX_PRE_CLIP_POS); 320 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data, 321 target, vertex_header, 322 DRAW_JIT_VERTEX_DATA); 323 324 return vertex_header; 325} 326 327 328/** 329 * Create LLVM types for various structures. 330 */ 331static void 332create_jit_types(struct draw_llvm *llvm) 333{ 334 struct gallivm_state *gallivm = llvm->gallivm; 335 LLVMTypeRef texture_type, context_type, buffer_type, vb_type; 336 337 texture_type = create_jit_texture_type(gallivm, "texture"); 338 339 context_type = create_jit_context_type(gallivm, texture_type, "draw_jit_context"); 340 llvm->context_ptr_type = LLVMPointerType(context_type, 0); 341 342 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0); 343 llvm->buffer_ptr_type = LLVMPointerType(buffer_type, 0); 344 345 vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer"); 346 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0); 347} 348 349 350static LLVMTypeRef 351get_context_ptr_type(struct draw_llvm *llvm) 352{ 353 if (!llvm->context_ptr_type) 354 create_jit_types(llvm); 355 return llvm->context_ptr_type; 356} 357 358 359static LLVMTypeRef 360get_buffer_ptr_type(struct draw_llvm *llvm) 361{ 362 if (!llvm->buffer_ptr_type) 363 create_jit_types(llvm); 364 return llvm->buffer_ptr_type; 365} 366 367 368static LLVMTypeRef 369get_vb_ptr_type(struct draw_llvm *llvm) 370{ 371 if (!llvm->vb_ptr_type) 372 create_jit_types(llvm); 373 return llvm->vb_ptr_type; 374} 375 376static LLVMTypeRef 377get_vertex_header_ptr_type(struct draw_llvm *llvm) 378{ 379 if (!llvm->vertex_header_ptr_type) 380 create_jit_types(llvm); 381 return llvm->vertex_header_ptr_type; 382} 383 384 385/** 386 * Create per-context LLVM info. 387 */ 388struct draw_llvm * 389draw_llvm_create(struct draw_context *draw, struct gallivm_state *gallivm) 390{ 391 struct draw_llvm *llvm; 392 393 llvm = CALLOC_STRUCT( draw_llvm ); 394 if (!llvm) 395 return NULL; 396 397 lp_build_init(); 398 399 llvm->draw = draw; 400 llvm->gallivm = gallivm; 401 402 if (gallivm_debug & GALLIVM_DEBUG_IR) { 403 LLVMDumpModule(llvm->gallivm->module); 404 } 405 406 llvm->nr_variants = 0; 407 make_empty_list(&llvm->vs_variants_list); 408 409 gallivm_register_garbage_collector_callback( 410 draw_llvm_garbage_collect_callback, llvm); 411 412 return llvm; 413} 414 415 416/** 417 * Free per-context LLVM info. 418 */ 419void 420draw_llvm_destroy(struct draw_llvm *llvm) 421{ 422 gallivm_remove_garbage_collector_callback( 423 draw_llvm_garbage_collect_callback, llvm); 424 425 /* XXX free other draw_llvm data? */ 426 FREE(llvm); 427} 428 429 430/** 431 * Create LLVM-generated code for a vertex shader. 432 */ 433struct draw_llvm_variant * 434draw_llvm_create_variant(struct draw_llvm *llvm, 435 unsigned num_inputs, 436 const struct draw_llvm_variant_key *key) 437{ 438 struct draw_llvm_variant *variant; 439 struct llvm_vertex_shader *shader = 440 llvm_vertex_shader(llvm->draw->vs.vertex_shader); 441 LLVMTypeRef vertex_header; 442 443 variant = MALLOC(sizeof *variant + 444 shader->variant_key_size - 445 sizeof variant->key); 446 if (variant == NULL) 447 return NULL; 448 449 variant->llvm = llvm; 450 451 memcpy(&variant->key, key, shader->variant_key_size); 452 453 vertex_header = create_jit_vertex_header(llvm->gallivm, num_inputs); 454 455 llvm->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0); 456 457 draw_llvm_generate(llvm, variant, FALSE); /* linear */ 458 draw_llvm_generate(llvm, variant, TRUE); /* elts */ 459 460 variant->shader = shader; 461 variant->list_item_global.base = variant; 462 variant->list_item_local.base = variant; 463 /*variant->no = */shader->variants_created++; 464 variant->list_item_global.base = variant; 465 466 return variant; 467} 468 469 470static void 471generate_vs(struct draw_llvm *llvm, 472 LLVMBuilderRef builder, 473 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 474 const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS], 475 LLVMValueRef system_values_array, 476 LLVMValueRef context_ptr, 477 struct lp_build_sampler_soa *draw_sampler, 478 boolean clamp_vertex_color) 479{ 480 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens; 481 struct lp_type vs_type; 482 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(llvm->gallivm, context_ptr); 483 struct lp_build_sampler_soa *sampler = 0; 484 485 memset(&vs_type, 0, sizeof vs_type); 486 vs_type.floating = TRUE; /* floating point values */ 487 vs_type.sign = TRUE; /* values are signed */ 488 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 489 vs_type.width = 32; /* 32-bit float */ 490 vs_type.length = 4; /* 4 elements per vector */ 491#if 0 492 num_vs = 4; /* number of vertices per block */ 493#endif 494 495 if (gallivm_debug & GALLIVM_DEBUG_IR) { 496 tgsi_dump(tokens, 0); 497 } 498 499 if (llvm->draw->num_sampler_views && llvm->draw->num_samplers) 500 sampler = draw_sampler; 501 502 lp_build_tgsi_soa(llvm->gallivm, 503 tokens, 504 vs_type, 505 NULL /*struct lp_build_mask_context *mask*/, 506 consts_ptr, 507 system_values_array, 508 NULL /*pos*/, 509 inputs, 510 outputs, 511 sampler, 512 &llvm->draw->vs.vertex_shader->info); 513 514 { 515 LLVMValueRef out; 516 unsigned chan, attrib; 517 struct lp_build_context bld; 518 struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info; 519 lp_build_context_init(&bld, llvm->gallivm, vs_type); 520 521 for (attrib = 0; attrib < info->num_outputs; ++attrib) { 522 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 523 if (outputs[attrib][chan]) { 524 switch (info->output_semantic_name[attrib]) { 525 case TGSI_SEMANTIC_COLOR: 526 case TGSI_SEMANTIC_BCOLOR: 527 if (clamp_vertex_color) { 528 out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 529 out = lp_build_clamp(&bld, out, bld.zero, bld.one); 530 LLVMBuildStore(builder, out, outputs[attrib][chan]); 531 } 532 break; 533 case TGSI_SEMANTIC_FOG: 534 if (chan == 1 || chan == 2) 535 LLVMBuildStore(builder, bld.zero, outputs[attrib][chan]); 536 else if (chan == 3) 537 LLVMBuildStore(builder, bld.one, outputs[attrib][chan]); 538 break; 539 } 540 } 541 } 542 } 543 } 544} 545 546 547#if DEBUG_STORE 548static void print_vectorf(LLVMBuilderRef builder, 549 LLVMValueRef vec) 550{ 551 LLVMValueRef val[4]; 552 val[0] = LLVMBuildExtractElement(builder, vec, 553 lp_build_const_int32(gallivm, 0), ""); 554 val[1] = LLVMBuildExtractElement(builder, vec, 555 lp_build_const_int32(gallivm, 1), ""); 556 val[2] = LLVMBuildExtractElement(builder, vec, 557 lp_build_const_int32(gallivm, 2), ""); 558 val[3] = LLVMBuildExtractElement(builder, vec, 559 lp_build_const_int32(gallivm, 3), ""); 560 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n", 561 val[0], val[1], val[2], val[3]); 562} 563#endif 564 565 566static void 567generate_fetch(struct gallivm_state *gallivm, 568 LLVMValueRef vbuffers_ptr, 569 LLVMValueRef *res, 570 struct pipe_vertex_element *velem, 571 LLVMValueRef vbuf, 572 LLVMValueRef index, 573 LLVMValueRef instance_id) 574{ 575 LLVMBuilderRef builder = gallivm->builder; 576 LLVMValueRef indices = 577 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 578 velem->vertex_buffer_index, 0); 579 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr, 580 &indices, 1, ""); 581 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf); 582 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf); 583 LLVMValueRef stride; 584 585 if (velem->instance_divisor) { 586 /* array index = instance_id / instance_divisor */ 587 index = LLVMBuildUDiv(builder, instance_id, 588 lp_build_const_int32(gallivm, velem->instance_divisor), 589 "instance_divisor"); 590 } 591 592 stride = LLVMBuildMul(builder, vb_stride, index, ""); 593 594 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer"); 595 596 stride = LLVMBuildAdd(builder, stride, 597 vb_buffer_offset, 598 ""); 599 stride = LLVMBuildAdd(builder, stride, 600 lp_build_const_int32(gallivm, velem->src_offset), 601 ""); 602 603 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/ 604 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, ""); 605 606 *res = draw_llvm_translate_from(gallivm, vbuffer_ptr, velem->src_format); 607} 608 609 610static LLVMValueRef 611aos_to_soa(struct gallivm_state *gallivm, 612 LLVMValueRef val0, 613 LLVMValueRef val1, 614 LLVMValueRef val2, 615 LLVMValueRef val3, 616 LLVMValueRef channel) 617{ 618 LLVMBuilderRef builder = gallivm->builder; 619 LLVMValueRef ex, res; 620 621 ex = LLVMBuildExtractElement(builder, val0, 622 channel, ""); 623 res = LLVMBuildInsertElement(builder, 624 LLVMConstNull(LLVMTypeOf(val0)), 625 ex, 626 lp_build_const_int32(gallivm, 0), 627 ""); 628 629 ex = LLVMBuildExtractElement(builder, val1, 630 channel, ""); 631 res = LLVMBuildInsertElement(builder, 632 res, ex, 633 lp_build_const_int32(gallivm, 1), 634 ""); 635 636 ex = LLVMBuildExtractElement(builder, val2, 637 channel, ""); 638 res = LLVMBuildInsertElement(builder, 639 res, ex, 640 lp_build_const_int32(gallivm, 2), 641 ""); 642 643 ex = LLVMBuildExtractElement(builder, val3, 644 channel, ""); 645 res = LLVMBuildInsertElement(builder, 646 res, ex, 647 lp_build_const_int32(gallivm, 3), 648 ""); 649 650 return res; 651} 652 653 654static void 655soa_to_aos(struct gallivm_state *gallivm, 656 LLVMValueRef soa[TGSI_NUM_CHANNELS], 657 LLVMValueRef aos[TGSI_NUM_CHANNELS]) 658{ 659 LLVMBuilderRef builder = gallivm->builder; 660 LLVMValueRef comp; 661 int i = 0; 662 663 debug_assert(TGSI_NUM_CHANNELS == 4); 664 665 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0])); 666 aos[1] = aos[2] = aos[3] = aos[0]; 667 668 for (i = 0; i < TGSI_NUM_CHANNELS; ++i) { 669 LLVMValueRef channel = lp_build_const_int32(gallivm, i); 670 671 comp = LLVMBuildExtractElement(builder, soa[i], 672 lp_build_const_int32(gallivm, 0), ""); 673 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, ""); 674 675 comp = LLVMBuildExtractElement(builder, soa[i], 676 lp_build_const_int32(gallivm, 1), ""); 677 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, ""); 678 679 comp = LLVMBuildExtractElement(builder, soa[i], 680 lp_build_const_int32(gallivm, 2), ""); 681 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, ""); 682 683 comp = LLVMBuildExtractElement(builder, soa[i], 684 lp_build_const_int32(gallivm, 3), ""); 685 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, ""); 686 687 } 688} 689 690 691static void 692convert_to_soa(struct gallivm_state *gallivm, 693 LLVMValueRef (*aos)[TGSI_NUM_CHANNELS], 694 LLVMValueRef (*soa)[TGSI_NUM_CHANNELS], 695 int num_attribs) 696{ 697 int i; 698 699 debug_assert(TGSI_NUM_CHANNELS == 4); 700 701 for (i = 0; i < num_attribs; ++i) { 702 LLVMValueRef val0 = aos[i][0]; 703 LLVMValueRef val1 = aos[i][1]; 704 LLVMValueRef val2 = aos[i][2]; 705 LLVMValueRef val3 = aos[i][3]; 706 707 soa[i][0] = aos_to_soa(gallivm, val0, val1, val2, val3, 708 lp_build_const_int32(gallivm, 0)); 709 soa[i][1] = aos_to_soa(gallivm, val0, val1, val2, val3, 710 lp_build_const_int32(gallivm, 1)); 711 soa[i][2] = aos_to_soa(gallivm, val0, val1, val2, val3, 712 lp_build_const_int32(gallivm, 2)); 713 soa[i][3] = aos_to_soa(gallivm, val0, val1, val2, val3, 714 lp_build_const_int32(gallivm, 3)); 715 } 716} 717 718 719static void 720store_aos(struct gallivm_state *gallivm, 721 LLVMValueRef io_ptr, 722 LLVMValueRef index, 723 LLVMValueRef value, 724 LLVMValueRef clipmask, boolean have_clipdist) 725{ 726 LLVMBuilderRef builder = gallivm->builder; 727 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptr); 728 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr); 729 LLVMValueRef indices[3]; 730 LLVMValueRef val; 731 int vertex_id_pad_edgeflag; 732 733 indices[0] = lp_build_const_int32(gallivm, 0); 734 indices[1] = index; 735 indices[2] = lp_build_const_int32(gallivm, 0); 736 737 /* If this assertion fails, it means we need to update the bit twidding 738 * code here. See struct vertex_header in draw_private.h. 739 */ 740 assert(DRAW_TOTAL_CLIP_PLANES==14); 741 /* initialize vertex id:16 = 0xffff, have_clipdist:1 = 0, edgeflag:1 = 1 */ 742 vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES); 743 if (have_clipdist) 744 vertex_id_pad_edgeflag |= 1 << (DRAW_TOTAL_CLIP_PLANES+1); 745 val = lp_build_const_int32(gallivm, vertex_id_pad_edgeflag); 746 /* OR with the clipmask */ 747 val = LLVMBuildOr(builder, val, clipmask, ""); 748 749 /* store vertex header */ 750 LLVMBuildStore(builder, val, id_ptr); 751 752 753#if DEBUG_STORE 754 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr); 755#endif 756#if 0 757 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr); 758 print_vectorf(builder, value);*/ 759 data_ptr = LLVMBuildBitCast(builder, data_ptr, 760 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0), 0), 761 "datavec"); 762 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, ""); 763 764 LLVMBuildStore(builder, value, data_ptr); 765#else 766 { 767 LLVMValueRef x, y, z, w; 768 LLVMValueRef idx0, idx1, idx2, idx3; 769 LLVMValueRef gep0, gep1, gep2, gep3; 770 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, ""); 771 772 idx0 = lp_build_const_int32(gallivm, 0); 773 idx1 = lp_build_const_int32(gallivm, 1); 774 idx2 = lp_build_const_int32(gallivm, 2); 775 idx3 = lp_build_const_int32(gallivm, 3); 776 777 x = LLVMBuildExtractElement(builder, value, 778 idx0, ""); 779 y = LLVMBuildExtractElement(builder, value, 780 idx1, ""); 781 z = LLVMBuildExtractElement(builder, value, 782 idx2, ""); 783 w = LLVMBuildExtractElement(builder, value, 784 idx3, ""); 785 786 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, ""); 787 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, ""); 788 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, ""); 789 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, ""); 790 791 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n", 792 x, gep0, y, gep1, z, gep2, w, gep3);*/ 793 LLVMBuildStore(builder, x, gep0); 794 LLVMBuildStore(builder, y, gep1); 795 LLVMBuildStore(builder, z, gep2); 796 LLVMBuildStore(builder, w, gep3); 797 } 798#endif 799} 800 801 802static void 803store_aos_array(struct gallivm_state *gallivm, 804 LLVMValueRef io_ptr, 805 LLVMValueRef aos[TGSI_NUM_CHANNELS], 806 int attrib, 807 int num_outputs, 808 LLVMValueRef clipmask, 809 boolean have_clipdist) 810{ 811 LLVMBuilderRef builder = gallivm->builder; 812 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib); 813 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0); 814 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1); 815 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2); 816 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3); 817 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 818 LLVMValueRef clipmask0, clipmask1, clipmask2, clipmask3; 819 820 debug_assert(TGSI_NUM_CHANNELS == 4); 821 822 io0_ptr = LLVMBuildGEP(builder, io_ptr, 823 &ind0, 1, ""); 824 io1_ptr = LLVMBuildGEP(builder, io_ptr, 825 &ind1, 1, ""); 826 io2_ptr = LLVMBuildGEP(builder, io_ptr, 827 &ind2, 1, ""); 828 io3_ptr = LLVMBuildGEP(builder, io_ptr, 829 &ind3, 1, ""); 830 831 clipmask0 = LLVMBuildExtractElement(builder, clipmask, 832 ind0, ""); 833 clipmask1 = LLVMBuildExtractElement(builder, clipmask, 834 ind1, ""); 835 clipmask2 = LLVMBuildExtractElement(builder, clipmask, 836 ind2, ""); 837 clipmask3 = LLVMBuildExtractElement(builder, clipmask, 838 ind3, ""); 839 840#if DEBUG_STORE 841 lp_build_printf(builder, "io = %p, indexes[%d, %d, %d, %d]\n, clipmask0 = %x, clipmask1 = %x, clipmask2 = %x, clipmask3 = %x\n", 842 io_ptr, ind0, ind1, ind2, ind3, clipmask0, clipmask1, clipmask2, clipmask3); 843#endif 844 /* store for each of the 4 vertices */ 845 store_aos(gallivm, io0_ptr, attr_index, aos[0], clipmask0, have_clipdist); 846 store_aos(gallivm, io1_ptr, attr_index, aos[1], clipmask1, have_clipdist); 847 store_aos(gallivm, io2_ptr, attr_index, aos[2], clipmask2, have_clipdist); 848 store_aos(gallivm, io3_ptr, attr_index, aos[3], clipmask3, have_clipdist); 849} 850 851 852static void 853convert_to_aos(struct gallivm_state *gallivm, 854 LLVMValueRef io, 855 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 856 LLVMValueRef clipmask, 857 int num_outputs, 858 int max_vertices, boolean have_clipdist) 859{ 860 LLVMBuilderRef builder = gallivm->builder; 861 unsigned chan, attrib; 862 863#if DEBUG_STORE 864 lp_build_printf(builder, " # storing begin\n"); 865#endif 866 for (attrib = 0; attrib < num_outputs; ++attrib) { 867 LLVMValueRef soa[4]; 868 LLVMValueRef aos[4]; 869 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 870 if (outputs[attrib][chan]) { 871 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 872 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]); 873 /*lp_build_printf(builder, "output %d : %d ", 874 LLVMConstInt(LLVMInt32Type(), attrib, 0), 875 LLVMConstInt(LLVMInt32Type(), chan, 0)); 876 print_vectorf(builder, out);*/ 877 soa[chan] = out; 878 } 879 else { 880 soa[chan] = 0; 881 } 882 } 883 soa_to_aos(gallivm, soa, aos); 884 store_aos_array(gallivm, 885 io, 886 aos, 887 attrib, 888 num_outputs, 889 clipmask, have_clipdist); 890 } 891#if DEBUG_STORE 892 lp_build_printf(builder, " # storing end\n"); 893#endif 894} 895 896 897/** 898 * Stores original vertex positions in clip coordinates 899 * There is probably a more efficient way to do this, 4 floats at once 900 * rather than extracting each element one by one. 901 * idx is the output to store things too, if pre_clip_pos is set 902 * we store the pos to the idx, if not we store the clipvertex to it. 903 */ 904static void 905store_clip(struct gallivm_state *gallivm, 906 LLVMValueRef io_ptr, 907 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 908 boolean pre_clip_pos, int idx) 909{ 910 LLVMBuilderRef builder = gallivm->builder; 911 LLVMValueRef out[4]; 912 LLVMValueRef indices[2]; 913 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 914 LLVMValueRef clip_ptr0, clip_ptr1, clip_ptr2, clip_ptr3; 915 LLVMValueRef clip0_ptr, clip1_ptr, clip2_ptr, clip3_ptr; 916 LLVMValueRef out0elem, out1elem, out2elem, out3elem; 917 int i; 918 919 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0); 920 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1); 921 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2); 922 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3); 923 924 indices[0] = 925 indices[1] = lp_build_const_int32(gallivm, 0); 926 927 out[0] = LLVMBuildLoad(builder, outputs[idx][0], ""); /*x0 x1 x2 x3*/ 928 out[1] = LLVMBuildLoad(builder, outputs[idx][1], ""); /*y0 y1 y2 y3*/ 929 out[2] = LLVMBuildLoad(builder, outputs[idx][2], ""); /*z0 z1 z2 z3*/ 930 out[3] = LLVMBuildLoad(builder, outputs[idx][3], ""); /*w0 w1 w2 w3*/ 931 932 io0_ptr = LLVMBuildGEP(builder, io_ptr, &ind0, 1, ""); 933 io1_ptr = LLVMBuildGEP(builder, io_ptr, &ind1, 1, ""); 934 io2_ptr = LLVMBuildGEP(builder, io_ptr, &ind2, 1, ""); 935 io3_ptr = LLVMBuildGEP(builder, io_ptr, &ind3, 1, ""); 936 937 if (!pre_clip_pos) { 938 clip_ptr0 = draw_jit_header_clip(gallivm, io0_ptr); 939 clip_ptr1 = draw_jit_header_clip(gallivm, io1_ptr); 940 clip_ptr2 = draw_jit_header_clip(gallivm, io2_ptr); 941 clip_ptr3 = draw_jit_header_clip(gallivm, io3_ptr); 942 } else { 943 clip_ptr0 = draw_jit_header_pre_clip_pos(gallivm, io0_ptr); 944 clip_ptr1 = draw_jit_header_pre_clip_pos(gallivm, io1_ptr); 945 clip_ptr2 = draw_jit_header_pre_clip_pos(gallivm, io2_ptr); 946 clip_ptr3 = draw_jit_header_pre_clip_pos(gallivm, io3_ptr); 947 } 948 949 for (i = 0; i<4; i++) { 950 clip0_ptr = LLVMBuildGEP(builder, clip_ptr0, indices, 2, ""); /* x0 */ 951 clip1_ptr = LLVMBuildGEP(builder, clip_ptr1, indices, 2, ""); /* x1 */ 952 clip2_ptr = LLVMBuildGEP(builder, clip_ptr2, indices, 2, ""); /* x2 */ 953 clip3_ptr = LLVMBuildGEP(builder, clip_ptr3, indices, 2, ""); /* x3 */ 954 955 out0elem = LLVMBuildExtractElement(builder, out[i], ind0, ""); /* x0 */ 956 out1elem = LLVMBuildExtractElement(builder, out[i], ind1, ""); /* x1 */ 957 out2elem = LLVMBuildExtractElement(builder, out[i], ind2, ""); /* x2 */ 958 out3elem = LLVMBuildExtractElement(builder, out[i], ind3, ""); /* x3 */ 959 960 LLVMBuildStore(builder, out0elem, clip0_ptr); 961 LLVMBuildStore(builder, out1elem, clip1_ptr); 962 LLVMBuildStore(builder, out2elem, clip2_ptr); 963 LLVMBuildStore(builder, out3elem, clip3_ptr); 964 965 indices[1]= LLVMBuildAdd(builder, indices[1], ind1, ""); 966 } 967 968} 969 970 971/** 972 * Equivalent of _mm_set1_ps(a) 973 */ 974static LLVMValueRef 975vec4f_from_scalar(struct gallivm_state *gallivm, 976 LLVMValueRef a, 977 const char *name) 978{ 979 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 980 LLVMValueRef res = LLVMGetUndef(LLVMVectorType(float_type, 4)); 981 int i; 982 983 for (i = 0; i < 4; ++i) { 984 LLVMValueRef index = lp_build_const_int32(gallivm, i); 985 res = LLVMBuildInsertElement(gallivm->builder, res, a, 986 index, i == 3 ? name : ""); 987 } 988 989 return res; 990} 991 992 993/** 994 * Transforms the outputs for viewport mapping 995 */ 996static void 997generate_viewport(struct draw_llvm *llvm, 998 LLVMBuilderRef builder, 999 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 1000 LLVMValueRef context_ptr) 1001{ 1002 int i; 1003 struct gallivm_state *gallivm = llvm->gallivm; 1004 struct lp_type f32_type = lp_type_float_vec(32); 1005 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 1006 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/ 1007 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr); 1008 1009 /* for 1/w convention*/ 1010 out3 = LLVMBuildFDiv(builder, const1, out3, ""); 1011 LLVMBuildStore(builder, out3, outputs[0][3]); 1012 1013 /* Viewport Mapping */ 1014 for (i=0; i<3; i++) { 1015 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 x2 x3*/ 1016 LLVMValueRef scale; 1017 LLVMValueRef trans; 1018 LLVMValueRef scale_i; 1019 LLVMValueRef trans_i; 1020 LLVMValueRef index; 1021 1022 index = lp_build_const_int32(gallivm, i); 1023 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 1024 1025 index = lp_build_const_int32(gallivm, i+4); 1026 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 1027 1028 scale = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, scale_i, ""), "scale"); 1029 trans = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, trans_i, ""), "trans"); 1030 1031 /* divide by w */ 1032 out = LLVMBuildFMul(builder, out, out3, ""); 1033 /* mult by scale */ 1034 out = LLVMBuildFMul(builder, out, scale, ""); 1035 /* add translation */ 1036 out = LLVMBuildFAdd(builder, out, trans, ""); 1037 1038 /* store transformed outputs */ 1039 LLVMBuildStore(builder, out, outputs[0][i]); 1040 } 1041 1042} 1043 1044 1045/** 1046 * Returns clipmask as 4xi32 bitmask for the 4 vertices 1047 */ 1048static LLVMValueRef 1049generate_clipmask(struct draw_llvm *llvm, 1050 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 1051 boolean clip_xy, 1052 boolean clip_z, 1053 boolean clip_user, 1054 boolean clip_halfz, 1055 unsigned ucp_enable, 1056 LLVMValueRef context_ptr, 1057 boolean *have_clipdist) 1058{ 1059 struct gallivm_state *gallivm = llvm->gallivm; 1060 LLVMBuilderRef builder = gallivm->builder; 1061 LLVMValueRef mask; /* stores the <4xi32> clipmasks */ 1062 LLVMValueRef test, temp; 1063 LLVMValueRef zero, shift; 1064 LLVMValueRef pos_x, pos_y, pos_z, pos_w; 1065 LLVMValueRef cv_x, cv_y, cv_z, cv_w; 1066 LLVMValueRef plane1, planes, plane_ptr, sum; 1067 struct lp_type f32_type = lp_type_float_vec(32); 1068 const unsigned pos = draw_current_shader_position_output(llvm->draw); 1069 const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw); 1070 int num_written_clipdistance = llvm->draw->vs.vertex_shader->info.num_written_clipdistance; 1071 bool have_cd = false; 1072 unsigned cd[2]; 1073 1074 cd[0] = draw_current_shader_clipdistance_output(llvm->draw, 0); 1075 cd[1] = draw_current_shader_clipdistance_output(llvm->draw, 1); 1076 1077 if (cd[0] != pos || cd[1] != pos) 1078 have_cd = true; 1079 1080 mask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1081 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1082 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */ 1083 shift = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1); /* 1 1 1 1 */ 1084 1085 /* 1086 * load clipvertex and position from correct locations. 1087 * if they are the same just load them once. 1088 */ 1089 pos_x = LLVMBuildLoad(builder, outputs[pos][0], ""); /*x0 x1 x2 x3*/ 1090 pos_y = LLVMBuildLoad(builder, outputs[pos][1], ""); /*y0 y1 y2 y3*/ 1091 pos_z = LLVMBuildLoad(builder, outputs[pos][2], ""); /*z0 z1 z2 z3*/ 1092 pos_w = LLVMBuildLoad(builder, outputs[pos][3], ""); /*w0 w1 w2 w3*/ 1093 1094 if (clip_user && cv != pos) { 1095 cv_x = LLVMBuildLoad(builder, outputs[cv][0], ""); /*x0 x1 x2 x3*/ 1096 cv_y = LLVMBuildLoad(builder, outputs[cv][1], ""); /*y0 y1 y2 y3*/ 1097 cv_z = LLVMBuildLoad(builder, outputs[cv][2], ""); /*z0 z1 z2 z3*/ 1098 cv_w = LLVMBuildLoad(builder, outputs[cv][3], ""); /*w0 w1 w2 w3*/ 1099 } else { 1100 cv_x = pos_x; 1101 cv_y = pos_y; 1102 cv_z = pos_z; 1103 cv_w = pos_w; 1104 } 1105 1106 /* Cliptest, for hardwired planes */ 1107 if (clip_xy) { 1108 /* plane 1 */ 1109 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w); 1110 temp = shift; 1111 test = LLVMBuildAnd(builder, test, temp, ""); 1112 mask = test; 1113 1114 /* plane 2 */ 1115 test = LLVMBuildFAdd(builder, pos_x, pos_w, ""); 1116 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1117 temp = LLVMBuildShl(builder, temp, shift, ""); 1118 test = LLVMBuildAnd(builder, test, temp, ""); 1119 mask = LLVMBuildOr(builder, mask, test, ""); 1120 1121 /* plane 3 */ 1122 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w); 1123 temp = LLVMBuildShl(builder, temp, shift, ""); 1124 test = LLVMBuildAnd(builder, test, temp, ""); 1125 mask = LLVMBuildOr(builder, mask, test, ""); 1126 1127 /* plane 4 */ 1128 test = LLVMBuildFAdd(builder, pos_y, pos_w, ""); 1129 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1130 temp = LLVMBuildShl(builder, temp, shift, ""); 1131 test = LLVMBuildAnd(builder, test, temp, ""); 1132 mask = LLVMBuildOr(builder, mask, test, ""); 1133 } 1134 1135 if (clip_z) { 1136 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 16); 1137 if (clip_halfz) { 1138 /* plane 5 */ 1139 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z); 1140 test = LLVMBuildAnd(builder, test, temp, ""); 1141 mask = LLVMBuildOr(builder, mask, test, ""); 1142 } 1143 else { 1144 /* plane 5 */ 1145 test = LLVMBuildFAdd(builder, pos_z, pos_w, ""); 1146 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1147 test = LLVMBuildAnd(builder, test, temp, ""); 1148 mask = LLVMBuildOr(builder, mask, test, ""); 1149 } 1150 /* plane 6 */ 1151 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w); 1152 temp = LLVMBuildShl(builder, temp, shift, ""); 1153 test = LLVMBuildAnd(builder, test, temp, ""); 1154 mask = LLVMBuildOr(builder, mask, test, ""); 1155 } 1156 1157 if (clip_user) { 1158 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr); 1159 LLVMValueRef indices[3]; 1160 1161 /* userclip planes */ 1162 while (ucp_enable) { 1163 unsigned plane_idx = ffs(ucp_enable)-1; 1164 ucp_enable &= ~(1 << plane_idx); 1165 plane_idx += 6; 1166 1167 if (have_cd && num_written_clipdistance) { 1168 LLVMValueRef clipdist; 1169 int i; 1170 i = plane_idx - 6; 1171 1172 *have_clipdist = TRUE; 1173 if (i < 4) { 1174 clipdist = LLVMBuildLoad(builder, outputs[cd[0]][i], ""); 1175 } else { 1176 clipdist = LLVMBuildLoad(builder, outputs[cd[1]][i-4], ""); 1177 } 1178 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, clipdist); 1179 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1 << plane_idx); 1180 test = LLVMBuildAnd(builder, test, temp, ""); 1181 mask = LLVMBuildOr(builder, mask, test, ""); 1182 } else { 1183 indices[0] = lp_build_const_int32(gallivm, 0); 1184 indices[1] = lp_build_const_int32(gallivm, plane_idx); 1185 1186 indices[2] = lp_build_const_int32(gallivm, 0); 1187 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1188 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x"); 1189 planes = vec4f_from_scalar(gallivm, plane1, "plane4_x"); 1190 sum = LLVMBuildFMul(builder, planes, cv_x, ""); 1191 1192 indices[2] = lp_build_const_int32(gallivm, 1); 1193 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1194 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y"); 1195 planes = vec4f_from_scalar(gallivm, plane1, "plane4_y"); 1196 test = LLVMBuildFMul(builder, planes, cv_y, ""); 1197 sum = LLVMBuildFAdd(builder, sum, test, ""); 1198 1199 indices[2] = lp_build_const_int32(gallivm, 2); 1200 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1201 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z"); 1202 planes = vec4f_from_scalar(gallivm, plane1, "plane4_z"); 1203 test = LLVMBuildFMul(builder, planes, cv_z, ""); 1204 sum = LLVMBuildFAdd(builder, sum, test, ""); 1205 1206 indices[2] = lp_build_const_int32(gallivm, 3); 1207 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1208 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w"); 1209 planes = vec4f_from_scalar(gallivm, plane1, "plane4_w"); 1210 test = LLVMBuildFMul(builder, planes, cv_w, ""); 1211 sum = LLVMBuildFAdd(builder, sum, test, ""); 1212 1213 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum); 1214 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1 << plane_idx); 1215 test = LLVMBuildAnd(builder, test, temp, ""); 1216 mask = LLVMBuildOr(builder, mask, test, ""); 1217 } 1218 } 1219 } 1220 return mask; 1221} 1222 1223 1224/** 1225 * Returns boolean if any clipping has occurred 1226 * Used zero/non-zero i32 value to represent boolean 1227 */ 1228static void 1229clipmask_bool(struct gallivm_state *gallivm, 1230 LLVMValueRef clipmask, 1231 LLVMValueRef ret_ptr) 1232{ 1233 LLVMBuilderRef builder = gallivm->builder; 1234 LLVMValueRef ret = LLVMBuildLoad(builder, ret_ptr, ""); 1235 LLVMValueRef temp; 1236 int i; 1237 1238 for (i=0; i<4; i++) { 1239 temp = LLVMBuildExtractElement(builder, clipmask, 1240 lp_build_const_int32(gallivm, i) , ""); 1241 ret = LLVMBuildOr(builder, ret, temp, ""); 1242 } 1243 1244 LLVMBuildStore(builder, ret, ret_ptr); 1245} 1246 1247 1248static void 1249draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant, 1250 boolean elts) 1251{ 1252 struct gallivm_state *gallivm = llvm->gallivm; 1253 LLVMContextRef context = gallivm->context; 1254 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 1255 LLVMTypeRef arg_types[8]; 1256 LLVMTypeRef func_type; 1257 LLVMValueRef context_ptr; 1258 LLVMBasicBlockRef block; 1259 LLVMBuilderRef builder; 1260 LLVMValueRef end, start; 1261 LLVMValueRef count, fetch_elts, fetch_count; 1262 LLVMValueRef stride, step, io_itr; 1263 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 1264 LLVMValueRef instance_id; 1265 LLVMValueRef system_values_array; 1266 LLVMValueRef zero = lp_build_const_int32(gallivm, 0); 1267 LLVMValueRef one = lp_build_const_int32(gallivm, 1); 1268 struct draw_context *draw = llvm->draw; 1269 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info; 1270 unsigned i, j; 1271 struct lp_build_context bld; 1272 struct lp_build_loop_state lp_loop; 1273 const int max_vertices = 4; 1274 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS]; 1275 LLVMValueRef fetch_max; 1276 void *code; 1277 struct lp_build_sampler_soa *sampler = 0; 1278 LLVMValueRef ret, ret_ptr; 1279 const boolean bypass_viewport = variant->key.bypass_viewport; 1280 const boolean enable_cliptest = variant->key.clip_xy || 1281 variant->key.clip_z || 1282 variant->key.clip_user; 1283 LLVMValueRef variant_func; 1284 const unsigned pos = draw_current_shader_position_output(llvm->draw); 1285 const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw); 1286 boolean have_clipdist = FALSE; 1287 1288 arg_types[0] = get_context_ptr_type(llvm); /* context */ 1289 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */ 1290 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */ 1291 if (elts) 1292 arg_types[3] = LLVMPointerType(int32_type, 0);/* fetch_elts * */ 1293 else 1294 arg_types[3] = int32_type; /* start */ 1295 arg_types[4] = int32_type; /* fetch_count / count */ 1296 arg_types[5] = int32_type; /* stride */ 1297 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */ 1298 arg_types[7] = int32_type; /* instance_id */ 1299 1300 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0); 1301 1302 variant_func = LLVMAddFunction(gallivm->module, 1303 elts ? "draw_llvm_shader_elts" : "draw_llvm_shader", 1304 func_type); 1305 1306 if (elts) 1307 variant->function_elts = variant_func; 1308 else 1309 variant->function = variant_func; 1310 1311 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); 1312 for (i = 0; i < Elements(arg_types); ++i) 1313 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 1314 LLVMAddAttribute(LLVMGetParam(variant_func, i), 1315 LLVMNoAliasAttribute); 1316 1317 context_ptr = LLVMGetParam(variant_func, 0); 1318 io_ptr = LLVMGetParam(variant_func, 1); 1319 vbuffers_ptr = LLVMGetParam(variant_func, 2); 1320 stride = LLVMGetParam(variant_func, 5); 1321 vb_ptr = LLVMGetParam(variant_func, 6); 1322 instance_id = LLVMGetParam(variant_func, 7); 1323 1324 lp_build_name(context_ptr, "context"); 1325 lp_build_name(io_ptr, "io"); 1326 lp_build_name(vbuffers_ptr, "vbuffers"); 1327 lp_build_name(stride, "stride"); 1328 lp_build_name(vb_ptr, "vb"); 1329 lp_build_name(instance_id, "instance_id"); 1330 1331 if (elts) { 1332 fetch_elts = LLVMGetParam(variant_func, 3); 1333 fetch_count = LLVMGetParam(variant_func, 4); 1334 lp_build_name(fetch_elts, "fetch_elts"); 1335 lp_build_name(fetch_count, "fetch_count"); 1336 start = count = NULL; 1337 } 1338 else { 1339 start = LLVMGetParam(variant_func, 3); 1340 count = LLVMGetParam(variant_func, 4); 1341 lp_build_name(start, "start"); 1342 lp_build_name(count, "count"); 1343 fetch_elts = fetch_count = NULL; 1344 } 1345 1346 /* 1347 * Function body 1348 */ 1349 1350 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry"); 1351 builder = gallivm->builder; 1352 LLVMPositionBuilderAtEnd(builder, block); 1353 1354 lp_build_context_init(&bld, gallivm, lp_type_int(32)); 1355 1356 system_values_array = lp_build_system_values_array(gallivm, vs_info, 1357 instance_id, NULL); 1358 1359 /* function will return non-zero i32 value if any clipped vertices */ 1360 ret_ptr = lp_build_alloca(gallivm, int32_type, ""); 1361 LLVMBuildStore(builder, zero, ret_ptr); 1362 1363 /* code generated texture sampling */ 1364 sampler = draw_llvm_sampler_soa_create( 1365 draw_llvm_variant_key_samplers(&variant->key), 1366 context_ptr); 1367 1368 if (elts) { 1369 start = zero; 1370 end = fetch_count; 1371 } 1372 else { 1373 end = lp_build_add(&bld, start, count); 1374 } 1375 1376 step = lp_build_const_int32(gallivm, max_vertices); 1377 1378 fetch_max = LLVMBuildSub(builder, end, one, "fetch_max"); 1379 1380 lp_build_loop_begin(&lp_loop, gallivm, start); 1381 { 1382 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS]; 1383 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS] = { { 0 } }; 1384 LLVMValueRef io; 1385 LLVMValueRef clipmask; /* holds the clipmask value */ 1386 const LLVMValueRef (*ptr_aos)[TGSI_NUM_CHANNELS]; 1387 1388 if (elts) 1389 io_itr = lp_loop.counter; 1390 else 1391 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, ""); 1392 1393 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 1394#if DEBUG_STORE 1395 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n", 1396 io_itr, io, lp_loop.counter); 1397#endif 1398 for (i = 0; i < TGSI_NUM_CHANNELS; ++i) { 1399 LLVMValueRef true_index = 1400 LLVMBuildAdd(builder, 1401 lp_loop.counter, 1402 lp_build_const_int32(gallivm, i), ""); 1403 1404 /* make sure we're not out of bounds which can happen 1405 * if fetch_count % 4 != 0, because on the last iteration 1406 * a few of the 4 vertex fetches will be out of bounds */ 1407 true_index = lp_build_min(&bld, true_index, fetch_max); 1408 1409 if (elts) { 1410 LLVMValueRef fetch_ptr; 1411 fetch_ptr = LLVMBuildGEP(builder, fetch_elts, 1412 &true_index, 1, ""); 1413 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt"); 1414 } 1415 1416 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 1417 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 1418 LLVMValueRef vb_index = 1419 lp_build_const_int32(gallivm, velem->vertex_buffer_index); 1420 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, ""); 1421 generate_fetch(gallivm, vbuffers_ptr, 1422 &aos_attribs[j][i], velem, vb, true_index, 1423 instance_id); 1424 } 1425 } 1426 convert_to_soa(gallivm, aos_attribs, inputs, 1427 draw->pt.nr_vertex_elements); 1428 1429 ptr_aos = (const LLVMValueRef (*)[TGSI_NUM_CHANNELS]) inputs; 1430 generate_vs(llvm, 1431 builder, 1432 outputs, 1433 ptr_aos, 1434 system_values_array, 1435 context_ptr, 1436 sampler, 1437 variant->key.clamp_vertex_color); 1438 1439 /* store original positions in clip before further manipulation */ 1440 store_clip(gallivm, io, outputs, 0, cv); 1441 store_clip(gallivm, io, outputs, 1, pos); 1442 1443 /* do cliptest */ 1444 if (enable_cliptest) { 1445 /* allocate clipmask, assign it integer type */ 1446 clipmask = generate_clipmask(llvm, outputs, 1447 variant->key.clip_xy, 1448 variant->key.clip_z, 1449 variant->key.clip_user, 1450 variant->key.clip_halfz, 1451 variant->key.ucp_enable, 1452 context_ptr, &have_clipdist); 1453 /* return clipping boolean value for function */ 1454 clipmask_bool(gallivm, clipmask, ret_ptr); 1455 } 1456 else { 1457 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1458 } 1459 1460 /* do viewport mapping */ 1461 if (!bypass_viewport) { 1462 generate_viewport(llvm, builder, outputs, context_ptr); 1463 } 1464 1465 /* store clipmask in vertex header, 1466 * original positions in clip 1467 * and transformed positions in data 1468 */ 1469 convert_to_aos(gallivm, io, outputs, clipmask, 1470 vs_info->num_outputs, max_vertices, have_clipdist); 1471 } 1472 1473 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE); 1474 1475 sampler->destroy(sampler); 1476 1477 ret = LLVMBuildLoad(builder, ret_ptr, ""); 1478 LLVMBuildRet(builder, ret); 1479 1480 /* 1481 * Translate the LLVM IR into machine code. 1482 */ 1483#ifdef DEBUG 1484 if (LLVMVerifyFunction(variant_func, LLVMPrintMessageAction)) { 1485 lp_debug_dump_value(variant_func); 1486 assert(0); 1487 } 1488#endif 1489 1490 LLVMRunFunctionPassManager(gallivm->passmgr, variant_func); 1491 1492 if (gallivm_debug & GALLIVM_DEBUG_IR) { 1493 lp_debug_dump_value(variant_func); 1494 debug_printf("\n"); 1495 } 1496 1497 code = LLVMGetPointerToGlobal(gallivm->engine, variant_func); 1498 if (elts) 1499 variant->jit_func_elts = (draw_jit_vert_func_elts) pointer_to_func(code); 1500 else 1501 variant->jit_func = (draw_jit_vert_func) pointer_to_func(code); 1502 1503 if (gallivm_debug & GALLIVM_DEBUG_ASM) { 1504 lp_disassemble(code); 1505 } 1506 lp_func_delete_body(variant_func); 1507} 1508 1509 1510struct draw_llvm_variant_key * 1511draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store) 1512{ 1513 unsigned i; 1514 struct draw_llvm_variant_key *key; 1515 struct lp_sampler_static_state *sampler; 1516 1517 key = (struct draw_llvm_variant_key *)store; 1518 1519 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/ 1520 1521 /* Presumably all variants of the shader should have the same 1522 * number of vertex elements - ie the number of shader inputs. 1523 */ 1524 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements; 1525 1526 /* will have to rig this up properly later */ 1527 key->clip_xy = llvm->draw->clip_xy; 1528 key->clip_z = llvm->draw->clip_z; 1529 key->clip_user = llvm->draw->clip_user; 1530 key->bypass_viewport = llvm->draw->identity_viewport; 1531 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules; 1532 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE); 1533 key->ucp_enable = llvm->draw->rasterizer->clip_plane_enable; 1534 key->pad = 0; 1535 1536 /* All variants of this shader will have the same value for 1537 * nr_samplers. Not yet trying to compact away holes in the 1538 * sampler array. 1539 */ 1540 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1; 1541 1542 sampler = draw_llvm_variant_key_samplers(key); 1543 1544 memcpy(key->vertex_element, 1545 llvm->draw->pt.vertex_element, 1546 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements); 1547 1548 memset(sampler, 0, key->nr_samplers * sizeof *sampler); 1549 1550 for (i = 0 ; i < key->nr_samplers; i++) { 1551 lp_sampler_static_state(&sampler[i], 1552 llvm->draw->sampler_views[i], 1553 llvm->draw->samplers[i]); 1554 } 1555 1556 return key; 1557} 1558 1559 1560void 1561draw_llvm_set_mapped_texture(struct draw_context *draw, 1562 unsigned sampler_idx, 1563 uint32_t width, uint32_t height, uint32_t depth, 1564 uint32_t first_level, uint32_t last_level, 1565 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS], 1566 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS], 1567 const void *data[PIPE_MAX_TEXTURE_LEVELS]) 1568{ 1569 unsigned j; 1570 struct draw_jit_texture *jit_tex; 1571 1572 assert(sampler_idx < PIPE_MAX_VERTEX_SAMPLERS); 1573 1574 jit_tex = &draw->llvm->jit_context.textures[sampler_idx]; 1575 1576 jit_tex->width = width; 1577 jit_tex->height = height; 1578 jit_tex->depth = depth; 1579 jit_tex->first_level = first_level; 1580 jit_tex->last_level = last_level; 1581 1582 for (j = first_level; j <= last_level; j++) { 1583 jit_tex->data[j] = data[j]; 1584 jit_tex->row_stride[j] = row_stride[j]; 1585 jit_tex->img_stride[j] = img_stride[j]; 1586 } 1587} 1588 1589 1590void 1591draw_llvm_set_sampler_state(struct draw_context *draw) 1592{ 1593 unsigned i; 1594 1595 for (i = 0; i < draw->num_samplers; i++) { 1596 struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i]; 1597 1598 if (draw->samplers[i]) { 1599 jit_tex->min_lod = draw->samplers[i]->min_lod; 1600 jit_tex->max_lod = draw->samplers[i]->max_lod; 1601 jit_tex->lod_bias = draw->samplers[i]->lod_bias; 1602 COPY_4V(jit_tex->border_color, draw->samplers[i]->border_color.f); 1603 } 1604 } 1605} 1606 1607 1608void 1609draw_llvm_destroy_variant(struct draw_llvm_variant *variant) 1610{ 1611 struct draw_llvm *llvm = variant->llvm; 1612 1613 if (variant->function_elts) { 1614 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine, 1615 variant->function_elts); 1616 LLVMDeleteFunction(variant->function_elts); 1617 } 1618 1619 if (variant->function) { 1620 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine, 1621 variant->function); 1622 LLVMDeleteFunction(variant->function); 1623 } 1624 1625 remove_from_list(&variant->list_item_local); 1626 variant->shader->variants_cached--; 1627 remove_from_list(&variant->list_item_global); 1628 llvm->nr_variants--; 1629 FREE(variant); 1630} 1631