draw_llvm.c revision e6c237cfd6f53ff569f68255d5d6da15148cd0f5
1/************************************************************************** 2 * 3 * Copyright 2010 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28#include "draw_llvm.h" 29 30#include "draw_context.h" 31#include "draw_vs.h" 32 33#include "gallivm/lp_bld_arit.h" 34#include "gallivm/lp_bld_logic.h" 35#include "gallivm/lp_bld_const.h" 36#include "gallivm/lp_bld_swizzle.h" 37#include "gallivm/lp_bld_struct.h" 38#include "gallivm/lp_bld_type.h" 39#include "gallivm/lp_bld_flow.h" 40#include "gallivm/lp_bld_debug.h" 41#include "gallivm/lp_bld_tgsi.h" 42#include "gallivm/lp_bld_printf.h" 43#include "gallivm/lp_bld_intr.h" 44#include "gallivm/lp_bld_init.h" 45#include "gallivm/lp_bld_type.h" 46 47#include "tgsi/tgsi_exec.h" 48#include "tgsi/tgsi_dump.h" 49 50#include "util/u_math.h" 51#include "util/u_pointer.h" 52#include "util/u_string.h" 53#include "util/u_simple_list.h" 54 55 56#define DEBUG_STORE 0 57 58 59/** 60 * This function is called by the gallivm "garbage collector" when 61 * the LLVM global data structures are freed. We must free all LLVM-related 62 * data. Specifically, all JIT'd shader variants. 63 */ 64static void 65draw_llvm_garbage_collect_callback(void *cb_data) 66{ 67 struct draw_llvm *llvm = (struct draw_llvm *) cb_data; 68 struct draw_llvm_variant_list_item *li; 69 70 /* free all shader variants */ 71 li = first_elem(&llvm->vs_variants_list); 72 while (!at_end(&llvm->vs_variants_list, li)) { 73 struct draw_llvm_variant_list_item *next = next_elem(li); 74 draw_llvm_destroy_variant(li->base); 75 li = next; 76 } 77 78 /* Null-out these pointers so they get remade next time they're needed. 79 * See the accessor functions below. 80 */ 81 llvm->context_ptr_type = NULL; 82 llvm->buffer_ptr_type = NULL; 83 llvm->vb_ptr_type = NULL; 84 llvm->vertex_header_ptr_type = NULL; 85} 86 87 88static void 89draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var, 90 boolean elts); 91 92 93/** 94 * Create LLVM type for struct draw_jit_texture 95 */ 96static LLVMTypeRef 97create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name) 98{ 99 LLVMTargetDataRef target = gallivm->target; 100 LLVMTypeRef texture_type; 101 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS]; 102 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 103 104 elem_types[DRAW_JIT_TEXTURE_WIDTH] = 105 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = 106 elem_types[DRAW_JIT_TEXTURE_DEPTH] = 107 elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] = 108 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type; 109 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] = 110 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] = 111 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS); 112 elem_types[DRAW_JIT_TEXTURE_DATA] = 113 LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0), 114 PIPE_MAX_TEXTURE_LEVELS); 115 elem_types[DRAW_JIT_TEXTURE_MIN_LOD] = 116 elem_types[DRAW_JIT_TEXTURE_MAX_LOD] = 117 elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context); 118 elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] = 119 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 120 121#if HAVE_LLVM >= 0x0300 122 texture_type = LLVMStructCreateNamed(gallivm->context, struct_name); 123 LLVMStructSetBody(texture_type, elem_types, 124 Elements(elem_types), 0); 125#else 126 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types, 127 Elements(elem_types), 0); 128 129 LLVMAddTypeName(gallivm->module, struct_name, texture_type); 130 131 /* Make sure the target's struct layout cache doesn't return 132 * stale/invalid data. 133 */ 134 LLVMInvalidateStructLayout(gallivm->target, texture_type); 135#endif 136 137 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width, 138 target, texture_type, 139 DRAW_JIT_TEXTURE_WIDTH); 140 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height, 141 target, texture_type, 142 DRAW_JIT_TEXTURE_HEIGHT); 143 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth, 144 target, texture_type, 145 DRAW_JIT_TEXTURE_DEPTH); 146 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level, 147 target, texture_type, 148 DRAW_JIT_TEXTURE_FIRST_LEVEL); 149 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level, 150 target, texture_type, 151 DRAW_JIT_TEXTURE_LAST_LEVEL); 152 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride, 153 target, texture_type, 154 DRAW_JIT_TEXTURE_ROW_STRIDE); 155 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride, 156 target, texture_type, 157 DRAW_JIT_TEXTURE_IMG_STRIDE); 158 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data, 159 target, texture_type, 160 DRAW_JIT_TEXTURE_DATA); 161 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod, 162 target, texture_type, 163 DRAW_JIT_TEXTURE_MIN_LOD); 164 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod, 165 target, texture_type, 166 DRAW_JIT_TEXTURE_MAX_LOD); 167 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias, 168 target, texture_type, 169 DRAW_JIT_TEXTURE_LOD_BIAS); 170 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color, 171 target, texture_type, 172 DRAW_JIT_TEXTURE_BORDER_COLOR); 173 174 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type); 175 176 return texture_type; 177} 178 179 180/** 181 * Create LLVM type for struct draw_jit_texture 182 */ 183static LLVMTypeRef 184create_jit_context_type(struct gallivm_state *gallivm, 185 LLVMTypeRef texture_type, const char *struct_name) 186{ 187 LLVMTargetDataRef target = gallivm->target; 188 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 189 LLVMTypeRef elem_types[5]; 190 LLVMTypeRef context_type; 191 192 elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */ 193 elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */ 194 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), 195 DRAW_TOTAL_CLIP_PLANES), 0); 196 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */ 197 elem_types[4] = LLVMArrayType(texture_type, 198 PIPE_MAX_VERTEX_SAMPLERS); /* textures */ 199#if HAVE_LLVM >= 0x0300 200 context_type = LLVMStructCreateNamed(gallivm->context, struct_name); 201 LLVMStructSetBody(context_type, elem_types, 202 Elements(elem_types), 0); 203#else 204 context_type = LLVMStructTypeInContext(gallivm->context, elem_types, 205 Elements(elem_types), 0); 206 LLVMAddTypeName(gallivm->module, struct_name, context_type); 207 208 LLVMInvalidateStructLayout(gallivm->target, context_type); 209#endif 210 211 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants, 212 target, context_type, 0); 213 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants, 214 target, context_type, 1); 215 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes, 216 target, context_type, 2); 217 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures, 218 target, context_type, 219 DRAW_JIT_CTX_TEXTURES); 220 LP_CHECK_STRUCT_SIZE(struct draw_jit_context, 221 target, context_type); 222 223 return context_type; 224} 225 226 227/** 228 * Create LLVM type for struct pipe_vertex_buffer 229 */ 230static LLVMTypeRef 231create_jit_vertex_buffer_type(struct gallivm_state *gallivm, const char *struct_name) 232{ 233 LLVMTargetDataRef target = gallivm->target; 234 LLVMTypeRef elem_types[3]; 235 LLVMTypeRef vb_type; 236 237 elem_types[0] = 238 elem_types[1] = LLVMInt32TypeInContext(gallivm->context); 239 elem_types[2] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */ 240 241#if HAVE_LLVM >= 0x0300 242 vb_type = LLVMStructCreateNamed(gallivm->context, struct_name); 243 LLVMStructSetBody(vb_type, elem_types, 244 Elements(elem_types), 0); 245#else 246 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types, 247 Elements(elem_types), 0); 248 LLVMAddTypeName(gallivm->module, struct_name, vb_type); 249 250 LLVMInvalidateStructLayout(gallivm->target, vb_type); 251#endif 252 253 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride, 254 target, vb_type, 0); 255 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset, 256 target, vb_type, 1); 257 258 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type); 259 260 return vb_type; 261} 262 263 264/** 265 * Create LLVM type for struct vertex_header; 266 */ 267static LLVMTypeRef 268create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems) 269{ 270 LLVMTargetDataRef target = gallivm->target; 271 LLVMTypeRef elem_types[3]; 272 LLVMTypeRef vertex_header; 273 char struct_name[24]; 274 275 util_snprintf(struct_name, 23, "vertex_header%d", data_elems); 276 277 elem_types[0] = LLVMIntTypeInContext(gallivm->context, 32); 278 elem_types[1] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 279 elem_types[2] = LLVMArrayType(elem_types[1], data_elems); 280 281#if HAVE_LLVM >= 0x0300 282 vertex_header = LLVMStructCreateNamed(gallivm->context, struct_name); 283 LLVMStructSetBody(vertex_header, elem_types, 284 Elements(elem_types), 0); 285#else 286 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types, 287 Elements(elem_types), 0); 288 LLVMAddTypeName(gallivm->module, struct_name, vertex_header); 289 290 LLVMInvalidateStructLayout(gallivm->target, vertex_header); 291#endif 292 293 /* these are bit-fields and we can't take address of them 294 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask, 295 target, vertex_header, 296 DRAW_JIT_VERTEX_CLIPMASK); 297 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag, 298 target, vertex_header, 299 DRAW_JIT_VERTEX_EDGEFLAG); 300 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad, 301 target, vertex_header, 302 DRAW_JIT_VERTEX_PAD); 303 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id, 304 target, vertex_header, 305 DRAW_JIT_VERTEX_VERTEX_ID); 306 */ 307 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip, 308 target, vertex_header, 309 DRAW_JIT_VERTEX_CLIP); 310 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data, 311 target, vertex_header, 312 DRAW_JIT_VERTEX_DATA); 313 314 return vertex_header; 315} 316 317 318/** 319 * Create LLVM types for various structures. 320 */ 321static void 322create_jit_types(struct draw_llvm *llvm) 323{ 324 struct gallivm_state *gallivm = llvm->gallivm; 325 LLVMTypeRef texture_type, context_type, buffer_type, vb_type; 326 327 texture_type = create_jit_texture_type(gallivm, "texture"); 328 329 context_type = create_jit_context_type(gallivm, texture_type, "draw_jit_context"); 330 llvm->context_ptr_type = LLVMPointerType(context_type, 0); 331 332 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0); 333 llvm->buffer_ptr_type = LLVMPointerType(buffer_type, 0); 334 335 vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer"); 336 llvm->vb_ptr_type = LLVMPointerType(vb_type, 0); 337} 338 339 340static LLVMTypeRef 341get_context_ptr_type(struct draw_llvm *llvm) 342{ 343 if (!llvm->context_ptr_type) 344 create_jit_types(llvm); 345 return llvm->context_ptr_type; 346} 347 348 349static LLVMTypeRef 350get_buffer_ptr_type(struct draw_llvm *llvm) 351{ 352 if (!llvm->buffer_ptr_type) 353 create_jit_types(llvm); 354 return llvm->buffer_ptr_type; 355} 356 357 358static LLVMTypeRef 359get_vb_ptr_type(struct draw_llvm *llvm) 360{ 361 if (!llvm->vb_ptr_type) 362 create_jit_types(llvm); 363 return llvm->vb_ptr_type; 364} 365 366static LLVMTypeRef 367get_vertex_header_ptr_type(struct draw_llvm *llvm) 368{ 369 if (!llvm->vertex_header_ptr_type) 370 create_jit_types(llvm); 371 return llvm->vertex_header_ptr_type; 372} 373 374 375/** 376 * Create per-context LLVM info. 377 */ 378struct draw_llvm * 379draw_llvm_create(struct draw_context *draw, struct gallivm_state *gallivm) 380{ 381 struct draw_llvm *llvm; 382 383 llvm = CALLOC_STRUCT( draw_llvm ); 384 if (!llvm) 385 return NULL; 386 387 lp_build_init(); 388 389 llvm->draw = draw; 390 llvm->gallivm = gallivm; 391 392 if (gallivm_debug & GALLIVM_DEBUG_IR) { 393 LLVMDumpModule(llvm->gallivm->module); 394 } 395 396 llvm->nr_variants = 0; 397 make_empty_list(&llvm->vs_variants_list); 398 399 gallivm_register_garbage_collector_callback( 400 draw_llvm_garbage_collect_callback, llvm); 401 402 return llvm; 403} 404 405 406/** 407 * Free per-context LLVM info. 408 */ 409void 410draw_llvm_destroy(struct draw_llvm *llvm) 411{ 412 gallivm_remove_garbage_collector_callback( 413 draw_llvm_garbage_collect_callback, llvm); 414 415 /* XXX free other draw_llvm data? */ 416 FREE(llvm); 417} 418 419 420/** 421 * Create LLVM-generated code for a vertex shader. 422 */ 423struct draw_llvm_variant * 424draw_llvm_create_variant(struct draw_llvm *llvm, 425 unsigned num_inputs, 426 const struct draw_llvm_variant_key *key) 427{ 428 struct draw_llvm_variant *variant; 429 struct llvm_vertex_shader *shader = 430 llvm_vertex_shader(llvm->draw->vs.vertex_shader); 431 LLVMTypeRef vertex_header; 432 433 variant = MALLOC(sizeof *variant + 434 shader->variant_key_size - 435 sizeof variant->key); 436 if (variant == NULL) 437 return NULL; 438 439 variant->llvm = llvm; 440 441 memcpy(&variant->key, key, shader->variant_key_size); 442 443 vertex_header = create_jit_vertex_header(llvm->gallivm, num_inputs); 444 445 llvm->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0); 446 447 draw_llvm_generate(llvm, variant, FALSE); /* linear */ 448 draw_llvm_generate(llvm, variant, TRUE); /* elts */ 449 450 variant->shader = shader; 451 variant->list_item_global.base = variant; 452 variant->list_item_local.base = variant; 453 /*variant->no = */shader->variants_created++; 454 variant->list_item_global.base = variant; 455 456 return variant; 457} 458 459 460static void 461generate_vs(struct draw_llvm *llvm, 462 LLVMBuilderRef builder, 463 LLVMValueRef (*outputs)[NUM_CHANNELS], 464 const LLVMValueRef (*inputs)[NUM_CHANNELS], 465 LLVMValueRef system_values_array, 466 LLVMValueRef context_ptr, 467 struct lp_build_sampler_soa *draw_sampler, 468 boolean clamp_vertex_color) 469{ 470 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens; 471 struct lp_type vs_type; 472 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(llvm->gallivm, context_ptr); 473 struct lp_build_sampler_soa *sampler = 0; 474 475 memset(&vs_type, 0, sizeof vs_type); 476 vs_type.floating = TRUE; /* floating point values */ 477 vs_type.sign = TRUE; /* values are signed */ 478 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 479 vs_type.width = 32; /* 32-bit float */ 480 vs_type.length = 4; /* 4 elements per vector */ 481#if 0 482 num_vs = 4; /* number of vertices per block */ 483#endif 484 485 if (gallivm_debug & GALLIVM_DEBUG_IR) { 486 tgsi_dump(tokens, 0); 487 } 488 489 if (llvm->draw->num_sampler_views && llvm->draw->num_samplers) 490 sampler = draw_sampler; 491 492 lp_build_tgsi_soa(llvm->gallivm, 493 tokens, 494 vs_type, 495 NULL /*struct lp_build_mask_context *mask*/, 496 consts_ptr, 497 system_values_array, 498 NULL /*pos*/, 499 inputs, 500 outputs, 501 sampler, 502 &llvm->draw->vs.vertex_shader->info); 503 504 if (clamp_vertex_color) { 505 LLVMValueRef out; 506 unsigned chan, attrib; 507 struct lp_build_context bld; 508 struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info; 509 lp_build_context_init(&bld, llvm->gallivm, vs_type); 510 511 for (attrib = 0; attrib < info->num_outputs; ++attrib) { 512 for (chan = 0; chan < NUM_CHANNELS; ++chan) { 513 if (outputs[attrib][chan]) { 514 switch (info->output_semantic_name[attrib]) { 515 case TGSI_SEMANTIC_COLOR: 516 case TGSI_SEMANTIC_BCOLOR: 517 out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 518 out = lp_build_clamp(&bld, out, bld.zero, bld.one); 519 LLVMBuildStore(builder, out, outputs[attrib][chan]); 520 break; 521 } 522 } 523 } 524 } 525 } 526} 527 528 529#if DEBUG_STORE 530static void print_vectorf(LLVMBuilderRef builder, 531 LLVMValueRef vec) 532{ 533 LLVMValueRef val[4]; 534 val[0] = LLVMBuildExtractElement(builder, vec, 535 lp_build_const_int32(gallivm, 0), ""); 536 val[1] = LLVMBuildExtractElement(builder, vec, 537 lp_build_const_int32(gallivm, 1), ""); 538 val[2] = LLVMBuildExtractElement(builder, vec, 539 lp_build_const_int32(gallivm, 2), ""); 540 val[3] = LLVMBuildExtractElement(builder, vec, 541 lp_build_const_int32(gallivm, 3), ""); 542 lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n", 543 val[0], val[1], val[2], val[3]); 544} 545#endif 546 547 548static void 549generate_fetch(struct gallivm_state *gallivm, 550 LLVMValueRef vbuffers_ptr, 551 LLVMValueRef *res, 552 struct pipe_vertex_element *velem, 553 LLVMValueRef vbuf, 554 LLVMValueRef index, 555 LLVMValueRef instance_id) 556{ 557 LLVMBuilderRef builder = gallivm->builder; 558 LLVMValueRef indices = 559 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 560 velem->vertex_buffer_index, 0); 561 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr, 562 &indices, 1, ""); 563 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf); 564 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf); 565 LLVMValueRef stride; 566 567 if (velem->instance_divisor) { 568 /* array index = instance_id / instance_divisor */ 569 index = LLVMBuildUDiv(builder, instance_id, 570 lp_build_const_int32(gallivm, velem->instance_divisor), 571 "instance_divisor"); 572 } 573 574 stride = LLVMBuildMul(builder, vb_stride, index, ""); 575 576 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer"); 577 578 stride = LLVMBuildAdd(builder, stride, 579 vb_buffer_offset, 580 ""); 581 stride = LLVMBuildAdd(builder, stride, 582 lp_build_const_int32(gallivm, velem->src_offset), 583 ""); 584 585 /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/ 586 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, ""); 587 588 *res = draw_llvm_translate_from(gallivm, vbuffer_ptr, velem->src_format); 589} 590 591 592static LLVMValueRef 593aos_to_soa(struct gallivm_state *gallivm, 594 LLVMValueRef val0, 595 LLVMValueRef val1, 596 LLVMValueRef val2, 597 LLVMValueRef val3, 598 LLVMValueRef channel) 599{ 600 LLVMBuilderRef builder = gallivm->builder; 601 LLVMValueRef ex, res; 602 603 ex = LLVMBuildExtractElement(builder, val0, 604 channel, ""); 605 res = LLVMBuildInsertElement(builder, 606 LLVMConstNull(LLVMTypeOf(val0)), 607 ex, 608 lp_build_const_int32(gallivm, 0), 609 ""); 610 611 ex = LLVMBuildExtractElement(builder, val1, 612 channel, ""); 613 res = LLVMBuildInsertElement(builder, 614 res, ex, 615 lp_build_const_int32(gallivm, 1), 616 ""); 617 618 ex = LLVMBuildExtractElement(builder, val2, 619 channel, ""); 620 res = LLVMBuildInsertElement(builder, 621 res, ex, 622 lp_build_const_int32(gallivm, 2), 623 ""); 624 625 ex = LLVMBuildExtractElement(builder, val3, 626 channel, ""); 627 res = LLVMBuildInsertElement(builder, 628 res, ex, 629 lp_build_const_int32(gallivm, 3), 630 ""); 631 632 return res; 633} 634 635 636static void 637soa_to_aos(struct gallivm_state *gallivm, 638 LLVMValueRef soa[NUM_CHANNELS], 639 LLVMValueRef aos[NUM_CHANNELS]) 640{ 641 LLVMBuilderRef builder = gallivm->builder; 642 LLVMValueRef comp; 643 int i = 0; 644 645 debug_assert(NUM_CHANNELS == 4); 646 647 aos[0] = LLVMConstNull(LLVMTypeOf(soa[0])); 648 aos[1] = aos[2] = aos[3] = aos[0]; 649 650 for (i = 0; i < NUM_CHANNELS; ++i) { 651 LLVMValueRef channel = lp_build_const_int32(gallivm, i); 652 653 comp = LLVMBuildExtractElement(builder, soa[i], 654 lp_build_const_int32(gallivm, 0), ""); 655 aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, ""); 656 657 comp = LLVMBuildExtractElement(builder, soa[i], 658 lp_build_const_int32(gallivm, 1), ""); 659 aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, ""); 660 661 comp = LLVMBuildExtractElement(builder, soa[i], 662 lp_build_const_int32(gallivm, 2), ""); 663 aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, ""); 664 665 comp = LLVMBuildExtractElement(builder, soa[i], 666 lp_build_const_int32(gallivm, 3), ""); 667 aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, ""); 668 669 } 670} 671 672 673static void 674convert_to_soa(struct gallivm_state *gallivm, 675 LLVMValueRef (*aos)[NUM_CHANNELS], 676 LLVMValueRef (*soa)[NUM_CHANNELS], 677 int num_attribs) 678{ 679 int i; 680 681 debug_assert(NUM_CHANNELS == 4); 682 683 for (i = 0; i < num_attribs; ++i) { 684 LLVMValueRef val0 = aos[i][0]; 685 LLVMValueRef val1 = aos[i][1]; 686 LLVMValueRef val2 = aos[i][2]; 687 LLVMValueRef val3 = aos[i][3]; 688 689 soa[i][0] = aos_to_soa(gallivm, val0, val1, val2, val3, 690 lp_build_const_int32(gallivm, 0)); 691 soa[i][1] = aos_to_soa(gallivm, val0, val1, val2, val3, 692 lp_build_const_int32(gallivm, 1)); 693 soa[i][2] = aos_to_soa(gallivm, val0, val1, val2, val3, 694 lp_build_const_int32(gallivm, 2)); 695 soa[i][3] = aos_to_soa(gallivm, val0, val1, val2, val3, 696 lp_build_const_int32(gallivm, 3)); 697 } 698} 699 700 701static void 702store_aos(struct gallivm_state *gallivm, 703 LLVMValueRef io_ptr, 704 LLVMValueRef index, 705 LLVMValueRef value, 706 LLVMValueRef clipmask) 707{ 708 LLVMBuilderRef builder = gallivm->builder; 709 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptr); 710 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr); 711 LLVMValueRef indices[3]; 712 LLVMValueRef val; 713 int vertex_id_pad_edgeflag; 714 715 indices[0] = lp_build_const_int32(gallivm, 0); 716 indices[1] = index; 717 indices[2] = lp_build_const_int32(gallivm, 0); 718 719 /* If this assertion fails, it means we need to update the bit twidding 720 * code here. See struct vertex_header in draw_private.h. 721 */ 722 assert(DRAW_TOTAL_CLIP_PLANES==14); 723 /* initialize vertex id:16 = 0xffff, pad:1 = 0, edgeflag:1 = 1 */ 724 vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES); 725 val = lp_build_const_int32(gallivm, vertex_id_pad_edgeflag); 726 /* OR with the clipmask */ 727 val = LLVMBuildOr(builder, val, clipmask, ""); 728 729 /* store vertex header */ 730 LLVMBuildStore(builder, val, id_ptr); 731 732 733#if DEBUG_STORE 734 lp_build_printf(builder, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr); 735#endif 736#if 0 737 /*lp_build_printf(builder, " ---- %p storing at %d (%p) ", io_ptr, index, data_ptr); 738 print_vectorf(builder, value);*/ 739 data_ptr = LLVMBuildBitCast(builder, data_ptr, 740 LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0), 0), 741 "datavec"); 742 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, ""); 743 744 LLVMBuildStore(builder, value, data_ptr); 745#else 746 { 747 LLVMValueRef x, y, z, w; 748 LLVMValueRef idx0, idx1, idx2, idx3; 749 LLVMValueRef gep0, gep1, gep2, gep3; 750 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, ""); 751 752 idx0 = lp_build_const_int32(gallivm, 0); 753 idx1 = lp_build_const_int32(gallivm, 1); 754 idx2 = lp_build_const_int32(gallivm, 2); 755 idx3 = lp_build_const_int32(gallivm, 3); 756 757 x = LLVMBuildExtractElement(builder, value, 758 idx0, ""); 759 y = LLVMBuildExtractElement(builder, value, 760 idx1, ""); 761 z = LLVMBuildExtractElement(builder, value, 762 idx2, ""); 763 w = LLVMBuildExtractElement(builder, value, 764 idx3, ""); 765 766 gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, ""); 767 gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, ""); 768 gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, ""); 769 gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, ""); 770 771 /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n", 772 x, gep0, y, gep1, z, gep2, w, gep3);*/ 773 LLVMBuildStore(builder, x, gep0); 774 LLVMBuildStore(builder, y, gep1); 775 LLVMBuildStore(builder, z, gep2); 776 LLVMBuildStore(builder, w, gep3); 777 } 778#endif 779} 780 781 782static void 783store_aos_array(struct gallivm_state *gallivm, 784 LLVMValueRef io_ptr, 785 LLVMValueRef aos[NUM_CHANNELS], 786 int attrib, 787 int num_outputs, 788 LLVMValueRef clipmask) 789{ 790 LLVMBuilderRef builder = gallivm->builder; 791 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib); 792 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0); 793 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1); 794 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2); 795 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3); 796 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 797 LLVMValueRef clipmask0, clipmask1, clipmask2, clipmask3; 798 799 debug_assert(NUM_CHANNELS == 4); 800 801 io0_ptr = LLVMBuildGEP(builder, io_ptr, 802 &ind0, 1, ""); 803 io1_ptr = LLVMBuildGEP(builder, io_ptr, 804 &ind1, 1, ""); 805 io2_ptr = LLVMBuildGEP(builder, io_ptr, 806 &ind2, 1, ""); 807 io3_ptr = LLVMBuildGEP(builder, io_ptr, 808 &ind3, 1, ""); 809 810 clipmask0 = LLVMBuildExtractElement(builder, clipmask, 811 ind0, ""); 812 clipmask1 = LLVMBuildExtractElement(builder, clipmask, 813 ind1, ""); 814 clipmask2 = LLVMBuildExtractElement(builder, clipmask, 815 ind2, ""); 816 clipmask3 = LLVMBuildExtractElement(builder, clipmask, 817 ind3, ""); 818 819#if DEBUG_STORE 820 lp_build_printf(builder, "io = %p, indexes[%d, %d, %d, %d]\n, clipmask0 = %x, clipmask1 = %x, clipmask2 = %x, clipmask3 = %x\n", 821 io_ptr, ind0, ind1, ind2, ind3, clipmask0, clipmask1, clipmask2, clipmask3); 822#endif 823 /* store for each of the 4 vertices */ 824 store_aos(gallivm, io0_ptr, attr_index, aos[0], clipmask0); 825 store_aos(gallivm, io1_ptr, attr_index, aos[1], clipmask1); 826 store_aos(gallivm, io2_ptr, attr_index, aos[2], clipmask2); 827 store_aos(gallivm, io3_ptr, attr_index, aos[3], clipmask3); 828} 829 830 831static void 832convert_to_aos(struct gallivm_state *gallivm, 833 LLVMValueRef io, 834 LLVMValueRef (*outputs)[NUM_CHANNELS], 835 LLVMValueRef clipmask, 836 int num_outputs, 837 int max_vertices) 838{ 839 LLVMBuilderRef builder = gallivm->builder; 840 unsigned chan, attrib; 841 842#if DEBUG_STORE 843 lp_build_printf(builder, " # storing begin\n"); 844#endif 845 for (attrib = 0; attrib < num_outputs; ++attrib) { 846 LLVMValueRef soa[4]; 847 LLVMValueRef aos[4]; 848 for (chan = 0; chan < NUM_CHANNELS; ++chan) { 849 if (outputs[attrib][chan]) { 850 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 851 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]); 852 /*lp_build_printf(builder, "output %d : %d ", 853 LLVMConstInt(LLVMInt32Type(), attrib, 0), 854 LLVMConstInt(LLVMInt32Type(), chan, 0)); 855 print_vectorf(builder, out);*/ 856 soa[chan] = out; 857 } 858 else { 859 soa[chan] = 0; 860 } 861 } 862 soa_to_aos(gallivm, soa, aos); 863 store_aos_array(gallivm, 864 io, 865 aos, 866 attrib, 867 num_outputs, 868 clipmask); 869 } 870#if DEBUG_STORE 871 lp_build_printf(builder, " # storing end\n"); 872#endif 873} 874 875 876/** 877 * Stores original vertex positions in clip coordinates 878 * There is probably a more efficient way to do this, 4 floats at once 879 * rather than extracting each element one by one. 880 */ 881static void 882store_clip(struct gallivm_state *gallivm, 883 LLVMValueRef io_ptr, 884 LLVMValueRef (*outputs)[NUM_CHANNELS]) 885{ 886 LLVMBuilderRef builder = gallivm->builder; 887 LLVMValueRef out[4]; 888 LLVMValueRef indices[2]; 889 LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr; 890 LLVMValueRef clip_ptr0, clip_ptr1, clip_ptr2, clip_ptr3; 891 LLVMValueRef clip0_ptr, clip1_ptr, clip2_ptr, clip3_ptr; 892 LLVMValueRef out0elem, out1elem, out2elem, out3elem; 893 int i; 894 895 LLVMValueRef ind0 = lp_build_const_int32(gallivm, 0); 896 LLVMValueRef ind1 = lp_build_const_int32(gallivm, 1); 897 LLVMValueRef ind2 = lp_build_const_int32(gallivm, 2); 898 LLVMValueRef ind3 = lp_build_const_int32(gallivm, 3); 899 900 indices[0] = 901 indices[1] = lp_build_const_int32(gallivm, 0); 902 903 out[0] = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/ 904 out[1] = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/ 905 out[2] = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/ 906 out[3] = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 907 908 io0_ptr = LLVMBuildGEP(builder, io_ptr, &ind0, 1, ""); 909 io1_ptr = LLVMBuildGEP(builder, io_ptr, &ind1, 1, ""); 910 io2_ptr = LLVMBuildGEP(builder, io_ptr, &ind2, 1, ""); 911 io3_ptr = LLVMBuildGEP(builder, io_ptr, &ind3, 1, ""); 912 913 clip_ptr0 = draw_jit_header_clip(gallivm, io0_ptr); 914 clip_ptr1 = draw_jit_header_clip(gallivm, io1_ptr); 915 clip_ptr2 = draw_jit_header_clip(gallivm, io2_ptr); 916 clip_ptr3 = draw_jit_header_clip(gallivm, io3_ptr); 917 918 for (i = 0; i<4; i++) { 919 clip0_ptr = LLVMBuildGEP(builder, clip_ptr0, indices, 2, ""); /* x0 */ 920 clip1_ptr = LLVMBuildGEP(builder, clip_ptr1, indices, 2, ""); /* x1 */ 921 clip2_ptr = LLVMBuildGEP(builder, clip_ptr2, indices, 2, ""); /* x2 */ 922 clip3_ptr = LLVMBuildGEP(builder, clip_ptr3, indices, 2, ""); /* x3 */ 923 924 out0elem = LLVMBuildExtractElement(builder, out[i], ind0, ""); /* x0 */ 925 out1elem = LLVMBuildExtractElement(builder, out[i], ind1, ""); /* x1 */ 926 out2elem = LLVMBuildExtractElement(builder, out[i], ind2, ""); /* x2 */ 927 out3elem = LLVMBuildExtractElement(builder, out[i], ind3, ""); /* x3 */ 928 929 LLVMBuildStore(builder, out0elem, clip0_ptr); 930 LLVMBuildStore(builder, out1elem, clip1_ptr); 931 LLVMBuildStore(builder, out2elem, clip2_ptr); 932 LLVMBuildStore(builder, out3elem, clip3_ptr); 933 934 indices[1]= LLVMBuildAdd(builder, indices[1], ind1, ""); 935 } 936 937} 938 939 940/** 941 * Equivalent of _mm_set1_ps(a) 942 */ 943static LLVMValueRef 944vec4f_from_scalar(struct gallivm_state *gallivm, 945 LLVMValueRef a, 946 const char *name) 947{ 948 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 949 LLVMValueRef res = LLVMGetUndef(LLVMVectorType(float_type, 4)); 950 int i; 951 952 for (i = 0; i < 4; ++i) { 953 LLVMValueRef index = lp_build_const_int32(gallivm, i); 954 res = LLVMBuildInsertElement(gallivm->builder, res, a, 955 index, i == 3 ? name : ""); 956 } 957 958 return res; 959} 960 961 962/** 963 * Transforms the outputs for viewport mapping 964 */ 965static void 966generate_viewport(struct draw_llvm *llvm, 967 LLVMBuilderRef builder, 968 LLVMValueRef (*outputs)[NUM_CHANNELS], 969 LLVMValueRef context_ptr) 970{ 971 int i; 972 struct gallivm_state *gallivm = llvm->gallivm; 973 struct lp_type f32_type = lp_type_float_vec(32); 974 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 975 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/ 976 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr); 977 978 /* for 1/w convention*/ 979 out3 = LLVMBuildFDiv(builder, const1, out3, ""); 980 LLVMBuildStore(builder, out3, outputs[0][3]); 981 982 /* Viewport Mapping */ 983 for (i=0; i<3; i++) { 984 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 x2 x3*/ 985 LLVMValueRef scale; 986 LLVMValueRef trans; 987 LLVMValueRef scale_i; 988 LLVMValueRef trans_i; 989 LLVMValueRef index; 990 991 index = lp_build_const_int32(gallivm, i); 992 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 993 994 index = lp_build_const_int32(gallivm, i+4); 995 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 996 997 scale = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, scale_i, ""), "scale"); 998 trans = vec4f_from_scalar(gallivm, LLVMBuildLoad(builder, trans_i, ""), "trans"); 999 1000 /* divide by w */ 1001 out = LLVMBuildFMul(builder, out, out3, ""); 1002 /* mult by scale */ 1003 out = LLVMBuildFMul(builder, out, scale, ""); 1004 /* add translation */ 1005 out = LLVMBuildFAdd(builder, out, trans, ""); 1006 1007 /* store transformed outputs */ 1008 LLVMBuildStore(builder, out, outputs[0][i]); 1009 } 1010 1011} 1012 1013 1014/** 1015 * Returns clipmask as 4xi32 bitmask for the 4 vertices 1016 */ 1017static LLVMValueRef 1018generate_clipmask(struct gallivm_state *gallivm, 1019 LLVMValueRef (*outputs)[NUM_CHANNELS], 1020 boolean clip_xy, 1021 boolean clip_z, 1022 boolean clip_user, 1023 boolean clip_halfz, 1024 unsigned nr, 1025 LLVMValueRef context_ptr) 1026{ 1027 LLVMBuilderRef builder = gallivm->builder; 1028 LLVMValueRef mask; /* stores the <4xi32> clipmasks */ 1029 LLVMValueRef test, temp; 1030 LLVMValueRef zero, shift; 1031 LLVMValueRef pos_x, pos_y, pos_z, pos_w; 1032 LLVMValueRef plane1, planes, plane_ptr, sum; 1033 unsigned i; 1034 struct lp_type f32_type = lp_type_float_vec(32); 1035 1036 mask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1037 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1038 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */ 1039 shift = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 1); /* 1 1 1 1 */ 1040 1041 /* Assuming position stored at output[0] */ 1042 pos_x = LLVMBuildLoad(builder, outputs[0][0], ""); /*x0 x1 x2 x3*/ 1043 pos_y = LLVMBuildLoad(builder, outputs[0][1], ""); /*y0 y1 y2 y3*/ 1044 pos_z = LLVMBuildLoad(builder, outputs[0][2], ""); /*z0 z1 z2 z3*/ 1045 pos_w = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 w2 w3*/ 1046 1047 /* Cliptest, for hardwired planes */ 1048 if (clip_xy) { 1049 /* plane 1 */ 1050 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w); 1051 temp = shift; 1052 test = LLVMBuildAnd(builder, test, temp, ""); 1053 mask = test; 1054 1055 /* plane 2 */ 1056 test = LLVMBuildFAdd(builder, pos_x, pos_w, ""); 1057 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1058 temp = LLVMBuildShl(builder, temp, shift, ""); 1059 test = LLVMBuildAnd(builder, test, temp, ""); 1060 mask = LLVMBuildOr(builder, mask, test, ""); 1061 1062 /* plane 3 */ 1063 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w); 1064 temp = LLVMBuildShl(builder, temp, shift, ""); 1065 test = LLVMBuildAnd(builder, test, temp, ""); 1066 mask = LLVMBuildOr(builder, mask, test, ""); 1067 1068 /* plane 4 */ 1069 test = LLVMBuildFAdd(builder, pos_y, pos_w, ""); 1070 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1071 temp = LLVMBuildShl(builder, temp, shift, ""); 1072 test = LLVMBuildAnd(builder, test, temp, ""); 1073 mask = LLVMBuildOr(builder, mask, test, ""); 1074 } 1075 1076 if (clip_z) { 1077 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 16); 1078 if (clip_halfz) { 1079 /* plane 5 */ 1080 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z); 1081 test = LLVMBuildAnd(builder, test, temp, ""); 1082 mask = LLVMBuildOr(builder, mask, test, ""); 1083 } 1084 else { 1085 /* plane 5 */ 1086 test = LLVMBuildFAdd(builder, pos_z, pos_w, ""); 1087 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 1088 test = LLVMBuildAnd(builder, test, temp, ""); 1089 mask = LLVMBuildOr(builder, mask, test, ""); 1090 } 1091 /* plane 6 */ 1092 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w); 1093 temp = LLVMBuildShl(builder, temp, shift, ""); 1094 test = LLVMBuildAnd(builder, test, temp, ""); 1095 mask = LLVMBuildOr(builder, mask, test, ""); 1096 } 1097 1098 if (clip_user) { 1099 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr); 1100 LLVMValueRef indices[3]; 1101 temp = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 32); 1102 1103 /* userclip planes */ 1104 for (i = 6; i < nr; i++) { 1105 indices[0] = lp_build_const_int32(gallivm, 0); 1106 indices[1] = lp_build_const_int32(gallivm, i); 1107 1108 indices[2] = lp_build_const_int32(gallivm, 0); 1109 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1110 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x"); 1111 planes = vec4f_from_scalar(gallivm, plane1, "plane4_x"); 1112 sum = LLVMBuildFMul(builder, planes, pos_x, ""); 1113 1114 indices[2] = lp_build_const_int32(gallivm, 1); 1115 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1116 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y"); 1117 planes = vec4f_from_scalar(gallivm, plane1, "plane4_y"); 1118 test = LLVMBuildFMul(builder, planes, pos_y, ""); 1119 sum = LLVMBuildFAdd(builder, sum, test, ""); 1120 1121 indices[2] = lp_build_const_int32(gallivm, 2); 1122 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1123 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z"); 1124 planes = vec4f_from_scalar(gallivm, plane1, "plane4_z"); 1125 test = LLVMBuildFMul(builder, planes, pos_z, ""); 1126 sum = LLVMBuildFAdd(builder, sum, test, ""); 1127 1128 indices[2] = lp_build_const_int32(gallivm, 3); 1129 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1130 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w"); 1131 planes = vec4f_from_scalar(gallivm, plane1, "plane4_w"); 1132 test = LLVMBuildFMul(builder, planes, pos_w, ""); 1133 sum = LLVMBuildFAdd(builder, sum, test, ""); 1134 1135 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum); 1136 temp = LLVMBuildShl(builder, temp, shift, ""); 1137 test = LLVMBuildAnd(builder, test, temp, ""); 1138 mask = LLVMBuildOr(builder, mask, test, ""); 1139 } 1140 } 1141 return mask; 1142} 1143 1144 1145/** 1146 * Returns boolean if any clipping has occurred 1147 * Used zero/non-zero i32 value to represent boolean 1148 */ 1149static void 1150clipmask_bool(struct gallivm_state *gallivm, 1151 LLVMValueRef clipmask, 1152 LLVMValueRef ret_ptr) 1153{ 1154 LLVMBuilderRef builder = gallivm->builder; 1155 LLVMValueRef ret = LLVMBuildLoad(builder, ret_ptr, ""); 1156 LLVMValueRef temp; 1157 int i; 1158 1159 for (i=0; i<4; i++) { 1160 temp = LLVMBuildExtractElement(builder, clipmask, 1161 lp_build_const_int32(gallivm, i) , ""); 1162 ret = LLVMBuildOr(builder, ret, temp, ""); 1163 } 1164 1165 LLVMBuildStore(builder, ret, ret_ptr); 1166} 1167 1168 1169static void 1170draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant, 1171 boolean elts) 1172{ 1173 struct gallivm_state *gallivm = llvm->gallivm; 1174 LLVMContextRef context = gallivm->context; 1175 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 1176 LLVMTypeRef arg_types[8]; 1177 LLVMTypeRef func_type; 1178 LLVMValueRef context_ptr; 1179 LLVMBasicBlockRef block; 1180 LLVMBuilderRef builder; 1181 LLVMValueRef end, start; 1182 LLVMValueRef count, fetch_elts, fetch_count; 1183 LLVMValueRef stride, step, io_itr; 1184 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 1185 LLVMValueRef instance_id; 1186 LLVMValueRef system_values_array; 1187 LLVMValueRef zero = lp_build_const_int32(gallivm, 0); 1188 LLVMValueRef one = lp_build_const_int32(gallivm, 1); 1189 struct draw_context *draw = llvm->draw; 1190 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info; 1191 unsigned i, j; 1192 struct lp_build_context bld; 1193 struct lp_build_loop_state lp_loop; 1194 const int max_vertices = 4; 1195 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS]; 1196 LLVMValueRef fetch_max; 1197 void *code; 1198 struct lp_build_sampler_soa *sampler = 0; 1199 LLVMValueRef ret, ret_ptr; 1200 const boolean bypass_viewport = variant->key.bypass_viewport; 1201 const boolean enable_cliptest = variant->key.clip_xy || 1202 variant->key.clip_z || 1203 variant->key.clip_user; 1204 LLVMValueRef variant_func; 1205 1206 arg_types[0] = get_context_ptr_type(llvm); /* context */ 1207 arg_types[1] = get_vertex_header_ptr_type(llvm); /* vertex_header */ 1208 arg_types[2] = get_buffer_ptr_type(llvm); /* vbuffers */ 1209 if (elts) 1210 arg_types[3] = LLVMPointerType(int32_type, 0);/* fetch_elts * */ 1211 else 1212 arg_types[3] = int32_type; /* start */ 1213 arg_types[4] = int32_type; /* fetch_count / count */ 1214 arg_types[5] = int32_type; /* stride */ 1215 arg_types[6] = get_vb_ptr_type(llvm); /* pipe_vertex_buffer's */ 1216 arg_types[7] = int32_type; /* instance_id */ 1217 1218 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0); 1219 1220 variant_func = LLVMAddFunction(gallivm->module, 1221 elts ? "draw_llvm_shader_elts" : "draw_llvm_shader", 1222 func_type); 1223 1224 if (elts) 1225 variant->function_elts = variant_func; 1226 else 1227 variant->function = variant_func; 1228 1229 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); 1230 for (i = 0; i < Elements(arg_types); ++i) 1231 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 1232 LLVMAddAttribute(LLVMGetParam(variant_func, i), 1233 LLVMNoAliasAttribute); 1234 1235 context_ptr = LLVMGetParam(variant_func, 0); 1236 io_ptr = LLVMGetParam(variant_func, 1); 1237 vbuffers_ptr = LLVMGetParam(variant_func, 2); 1238 stride = LLVMGetParam(variant_func, 5); 1239 vb_ptr = LLVMGetParam(variant_func, 6); 1240 instance_id = LLVMGetParam(variant_func, 7); 1241 1242 lp_build_name(context_ptr, "context"); 1243 lp_build_name(io_ptr, "io"); 1244 lp_build_name(vbuffers_ptr, "vbuffers"); 1245 lp_build_name(stride, "stride"); 1246 lp_build_name(vb_ptr, "vb"); 1247 lp_build_name(instance_id, "instance_id"); 1248 1249 if (elts) { 1250 fetch_elts = LLVMGetParam(variant_func, 3); 1251 fetch_count = LLVMGetParam(variant_func, 4); 1252 lp_build_name(fetch_elts, "fetch_elts"); 1253 lp_build_name(fetch_count, "fetch_count"); 1254 } 1255 else { 1256 start = LLVMGetParam(variant_func, 3); 1257 count = LLVMGetParam(variant_func, 4); 1258 lp_build_name(start, "start"); 1259 lp_build_name(count, "count"); 1260 } 1261 1262 /* 1263 * Function body 1264 */ 1265 1266 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry"); 1267 builder = gallivm->builder; 1268 LLVMPositionBuilderAtEnd(builder, block); 1269 1270 lp_build_context_init(&bld, gallivm, lp_type_int(32)); 1271 1272 system_values_array = lp_build_system_values_array(gallivm, vs_info, 1273 instance_id, NULL); 1274 1275 /* function will return non-zero i32 value if any clipped vertices */ 1276 ret_ptr = lp_build_alloca(gallivm, int32_type, ""); 1277 LLVMBuildStore(builder, zero, ret_ptr); 1278 1279 /* code generated texture sampling */ 1280 sampler = draw_llvm_sampler_soa_create( 1281 draw_llvm_variant_key_samplers(&variant->key), 1282 context_ptr); 1283 1284 if (elts) { 1285 start = zero; 1286 end = fetch_count; 1287 } 1288 else { 1289 end = lp_build_add(&bld, start, count); 1290 } 1291 1292 step = lp_build_const_int32(gallivm, max_vertices); 1293 1294 fetch_max = LLVMBuildSub(builder, end, one, "fetch_max"); 1295 1296 lp_build_loop_begin(&lp_loop, gallivm, start); 1297 { 1298 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS]; 1299 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } }; 1300 LLVMValueRef io; 1301 LLVMValueRef clipmask; /* holds the clipmask value */ 1302 const LLVMValueRef (*ptr_aos)[NUM_CHANNELS]; 1303 1304 if (elts) 1305 io_itr = lp_loop.counter; 1306 else 1307 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, ""); 1308 1309 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 1310#if DEBUG_STORE 1311 lp_build_printf(builder, " --- io %d = %p, loop counter %d\n", 1312 io_itr, io, lp_loop.counter); 1313#endif 1314 for (i = 0; i < NUM_CHANNELS; ++i) { 1315 LLVMValueRef true_index = 1316 LLVMBuildAdd(builder, 1317 lp_loop.counter, 1318 lp_build_const_int32(gallivm, i), ""); 1319 1320 /* make sure we're not out of bounds which can happen 1321 * if fetch_count % 4 != 0, because on the last iteration 1322 * a few of the 4 vertex fetches will be out of bounds */ 1323 true_index = lp_build_min(&bld, true_index, fetch_max); 1324 1325 if (elts) { 1326 LLVMValueRef fetch_ptr; 1327 fetch_ptr = LLVMBuildGEP(builder, fetch_elts, 1328 &true_index, 1, ""); 1329 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt"); 1330 } 1331 1332 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 1333 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 1334 LLVMValueRef vb_index = 1335 lp_build_const_int32(gallivm, velem->vertex_buffer_index); 1336 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, ""); 1337 generate_fetch(gallivm, vbuffers_ptr, 1338 &aos_attribs[j][i], velem, vb, true_index, 1339 instance_id); 1340 } 1341 } 1342 convert_to_soa(gallivm, aos_attribs, inputs, 1343 draw->pt.nr_vertex_elements); 1344 1345 ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs; 1346 generate_vs(llvm, 1347 builder, 1348 outputs, 1349 ptr_aos, 1350 system_values_array, 1351 context_ptr, 1352 sampler, 1353 variant->key.clamp_vertex_color); 1354 1355 /* store original positions in clip before further manipulation */ 1356 store_clip(gallivm, io, outputs); 1357 1358 /* do cliptest */ 1359 if (enable_cliptest) { 1360 /* allocate clipmask, assign it integer type */ 1361 clipmask = generate_clipmask(gallivm, outputs, 1362 variant->key.clip_xy, 1363 variant->key.clip_z, 1364 variant->key.clip_user, 1365 variant->key.clip_halfz, 1366 variant->key.nr_planes, 1367 context_ptr); 1368 /* return clipping boolean value for function */ 1369 clipmask_bool(gallivm, clipmask, ret_ptr); 1370 } 1371 else { 1372 clipmask = lp_build_const_int_vec(gallivm, lp_type_int_vec(32), 0); 1373 } 1374 1375 /* do viewport mapping */ 1376 if (!bypass_viewport) { 1377 generate_viewport(llvm, builder, outputs, context_ptr); 1378 } 1379 1380 /* store clipmask in vertex header, 1381 * original positions in clip 1382 * and transformed positions in data 1383 */ 1384 convert_to_aos(gallivm, io, outputs, clipmask, 1385 vs_info->num_outputs, max_vertices); 1386 } 1387 1388 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE); 1389 1390 sampler->destroy(sampler); 1391 1392 ret = LLVMBuildLoad(builder, ret_ptr, ""); 1393 LLVMBuildRet(builder, ret); 1394 1395 /* 1396 * Translate the LLVM IR into machine code. 1397 */ 1398#ifdef DEBUG 1399 if (LLVMVerifyFunction(variant_func, LLVMPrintMessageAction)) { 1400 lp_debug_dump_value(variant_func); 1401 assert(0); 1402 } 1403#endif 1404 1405 LLVMRunFunctionPassManager(gallivm->passmgr, variant_func); 1406 1407 if (gallivm_debug & GALLIVM_DEBUG_IR) { 1408 lp_debug_dump_value(variant_func); 1409 debug_printf("\n"); 1410 } 1411 1412 code = LLVMGetPointerToGlobal(gallivm->engine, variant_func); 1413 if (elts) 1414 variant->jit_func_elts = (draw_jit_vert_func_elts) pointer_to_func(code); 1415 else 1416 variant->jit_func = (draw_jit_vert_func) pointer_to_func(code); 1417 1418 if (gallivm_debug & GALLIVM_DEBUG_ASM) { 1419 lp_disassemble(code); 1420 } 1421 lp_func_delete_body(variant_func); 1422} 1423 1424 1425struct draw_llvm_variant_key * 1426draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store) 1427{ 1428 unsigned i; 1429 struct draw_llvm_variant_key *key; 1430 struct lp_sampler_static_state *sampler; 1431 1432 key = (struct draw_llvm_variant_key *)store; 1433 1434 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/ 1435 1436 /* Presumably all variants of the shader should have the same 1437 * number of vertex elements - ie the number of shader inputs. 1438 */ 1439 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements; 1440 1441 /* will have to rig this up properly later */ 1442 key->clip_xy = llvm->draw->clip_xy; 1443 key->clip_z = llvm->draw->clip_z; 1444 key->clip_user = llvm->draw->clip_user; 1445 key->bypass_viewport = llvm->draw->identity_viewport; 1446 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules; 1447 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE); 1448 key->nr_planes = llvm->draw->nr_planes; 1449 key->pad = 0; 1450 1451 /* All variants of this shader will have the same value for 1452 * nr_samplers. Not yet trying to compact away holes in the 1453 * sampler array. 1454 */ 1455 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1; 1456 1457 sampler = draw_llvm_variant_key_samplers(key); 1458 1459 memcpy(key->vertex_element, 1460 llvm->draw->pt.vertex_element, 1461 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements); 1462 1463 memset(sampler, 0, key->nr_samplers * sizeof *sampler); 1464 1465 for (i = 0 ; i < key->nr_samplers; i++) { 1466 lp_sampler_static_state(&sampler[i], 1467 llvm->draw->sampler_views[i], 1468 llvm->draw->samplers[i]); 1469 } 1470 1471 return key; 1472} 1473 1474 1475void 1476draw_llvm_set_mapped_texture(struct draw_context *draw, 1477 unsigned sampler_idx, 1478 uint32_t width, uint32_t height, uint32_t depth, 1479 uint32_t first_level, uint32_t last_level, 1480 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS], 1481 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS], 1482 const void *data[PIPE_MAX_TEXTURE_LEVELS]) 1483{ 1484 unsigned j; 1485 struct draw_jit_texture *jit_tex; 1486 1487 assert(sampler_idx < PIPE_MAX_VERTEX_SAMPLERS); 1488 1489 jit_tex = &draw->llvm->jit_context.textures[sampler_idx]; 1490 1491 jit_tex->width = width; 1492 jit_tex->height = height; 1493 jit_tex->depth = depth; 1494 jit_tex->first_level = first_level; 1495 jit_tex->last_level = last_level; 1496 1497 for (j = first_level; j <= last_level; j++) { 1498 jit_tex->data[j] = data[j]; 1499 jit_tex->row_stride[j] = row_stride[j]; 1500 jit_tex->img_stride[j] = img_stride[j]; 1501 } 1502} 1503 1504 1505void 1506draw_llvm_set_sampler_state(struct draw_context *draw) 1507{ 1508 unsigned i; 1509 1510 for (i = 0; i < draw->num_samplers; i++) { 1511 struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i]; 1512 1513 if (draw->samplers[i]) { 1514 jit_tex->min_lod = draw->samplers[i]->min_lod; 1515 jit_tex->max_lod = draw->samplers[i]->max_lod; 1516 jit_tex->lod_bias = draw->samplers[i]->lod_bias; 1517 COPY_4V(jit_tex->border_color, draw->samplers[i]->border_color.f); 1518 } 1519 } 1520} 1521 1522 1523void 1524draw_llvm_destroy_variant(struct draw_llvm_variant *variant) 1525{ 1526 struct draw_llvm *llvm = variant->llvm; 1527 1528 if (variant->function_elts) { 1529 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine, 1530 variant->function_elts); 1531 LLVMDeleteFunction(variant->function_elts); 1532 } 1533 1534 if (variant->function) { 1535 LLVMFreeMachineCodeForFunction(llvm->gallivm->engine, 1536 variant->function); 1537 LLVMDeleteFunction(variant->function); 1538 } 1539 1540 remove_from_list(&variant->list_item_local); 1541 variant->shader->variants_cached--; 1542 remove_from_list(&variant->list_item_global); 1543 llvm->nr_variants--; 1544 FREE(variant); 1545} 1546