lp_bld_swizzle.c revision 638779e44555b3c3789638a95693357cf59f5595 
1/**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28/**
29 * @file
30 * Helper functions for swizzling/shuffling.
31 *
32 * @author Jose Fonseca <jfonseca@vmware.com>
33 */
34
35
36#include "util/u_debug.h"
37
38#include "lp_bld_type.h"
39#include "lp_bld_const.h"
40#include "lp_bld_init.h"
41#include "lp_bld_logic.h"
42#include "lp_bld_swizzle.h"
43
44
45LLVMValueRef
46lp_build_broadcast(struct gallivm_state *gallivm,
47                   LLVMTypeRef vec_type,
48                   LLVMValueRef scalar)
49{
50   LLVMValueRef res;
51
52   if (LLVMGetTypeKind(vec_type) != LLVMVectorTypeKind) {
53      /* scalar */
54      assert(vec_type == LLVMTypeOf(scalar));
55      res = scalar;
56   } else {
57      LLVMBuilderRef builder = gallivm->builder;
58      const unsigned length = LLVMGetVectorSize(vec_type);
59      LLVMValueRef undef = LLVMGetUndef(vec_type);
60      LLVMTypeRef i32_type = LLVMInt32TypeInContext(gallivm->context);
61
62      assert(LLVMGetElementType(vec_type) == LLVMTypeOf(scalar));
63
64      if (HAVE_LLVM >= 0x207) {
65         /* The shuffle vector is always made of int32 elements */
66         LLVMTypeRef i32_vec_type = LLVMVectorType(i32_type, length);
67         res = LLVMBuildInsertElement(builder, undef, scalar, LLVMConstNull(i32_type), "");
68         res = LLVMBuildShuffleVector(builder, res, undef, LLVMConstNull(i32_vec_type), "");
69      } else {
70         /* XXX: The above path provokes a bug in LLVM 2.6 */
71         unsigned i;
72         res = undef;
73         for(i = 0; i < length; ++i) {
74            LLVMValueRef index = lp_build_const_int32(gallivm, i);
75            res = LLVMBuildInsertElement(builder, res, scalar, index, "");
76         }
77      }
78   }
79
80   return res;
81}
82
83
84/**
85 * Broadcast
86 */
87LLVMValueRef
88lp_build_broadcast_scalar(struct lp_build_context *bld,
89                          LLVMValueRef scalar)
90{
91   assert(lp_check_elem_type(bld->type, LLVMTypeOf(scalar)));
92
93   return lp_build_broadcast(bld->gallivm, bld->vec_type, scalar);
94}
95
96
97/**
98 * Combined extract and broadcast (or a mere shuffle when the two types match)
99 */
100LLVMValueRef
101lp_build_extract_broadcast(struct gallivm_state *gallivm,
102                           struct lp_type src_type,
103                           struct lp_type dst_type,
104                           LLVMValueRef vector,
105                           LLVMValueRef index)
106{
107   LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
108   LLVMValueRef res;
109
110   assert(src_type.floating == dst_type.floating);
111   assert(src_type.width    == dst_type.width);
112
113   assert(lp_check_value(src_type, vector));
114   assert(LLVMTypeOf(index) == i32t);
115
116   if (src_type.length == 1) {
117      if (dst_type.length == 1) {
118         /*
119          * Trivial scalar -> scalar.
120          */
121
122         res = vector;
123      }
124      else {
125         /*
126          * Broadcast scalar -> vector.
127          */
128
129         res = lp_build_broadcast(gallivm,
130                                  lp_build_vec_type(gallivm, dst_type),
131                                  vector);
132      }
133   }
134   else {
135      if (dst_type.length == src_type.length) {
136         /*
137          * Special shuffle of the same size.
138          */
139
140         LLVMValueRef shuffle;
141         shuffle = lp_build_broadcast(gallivm,
142                                      LLVMVectorType(i32t, dst_type.length),
143                                      index);
144         res = LLVMBuildShuffleVector(gallivm->builder, vector,
145                                      LLVMGetUndef(lp_build_vec_type(gallivm, dst_type)),
146                                      shuffle, "");
147      }
148      else {
149         LLVMValueRef scalar;
150         scalar = LLVMBuildExtractElement(gallivm->builder, vector, index, "");
151         if (dst_type.length == 1) {
152            /*
153             * Trivial extract scalar from vector.
154             */
155
156            res = scalar;
157         }
158         else {
159            /*
160             * General case of different sized vectors.
161             */
162
163            res = lp_build_broadcast(gallivm,
164                                     lp_build_vec_type(gallivm, dst_type),
165                                     vector);
166         }
167      }
168   }
169
170   return res;
171}
172
173
174/**
175 * Swizzle one channel into all other three channels.
176 */
177LLVMValueRef
178lp_build_swizzle_scalar_aos(struct lp_build_context *bld,
179                            LLVMValueRef a,
180                            unsigned channel)
181{
182   LLVMBuilderRef builder = bld->gallivm->builder;
183   const struct lp_type type = bld->type;
184   const unsigned n = type.length;
185   unsigned i, j;
186
187   if(a == bld->undef || a == bld->zero || a == bld->one)
188      return a;
189
190   /* XXX: SSE3 has PSHUFB which should be better than bitmasks, but forcing
191    * using shuffles here actually causes worst results. More investigation is
192    * needed. */
193   if (type.width >= 16) {
194      /*
195       * Shuffle.
196       */
197      LLVMTypeRef elem_type = LLVMInt32TypeInContext(bld->gallivm->context);
198      LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
199
200      for(j = 0; j < n; j += 4)
201         for(i = 0; i < 4; ++i)
202            shuffles[j + i] = LLVMConstInt(elem_type, j + channel, 0);
203
204      return LLVMBuildShuffleVector(builder, a, bld->undef, LLVMConstVector(shuffles, n), "");
205   }
206   else {
207      /*
208       * Bit mask and recursive shifts
209       *
210       *   XYZW XYZW .... XYZW  <= input
211       *   0Y00 0Y00 .... 0Y00
212       *   YY00 YY00 .... YY00
213       *   YYYY YYYY .... YYYY  <= output
214       */
215      struct lp_type type4;
216      const char shifts[4][2] = {
217         { 1,  2},
218         {-1,  2},
219         { 1, -2},
220         {-1, -2}
221      };
222      unsigned i;
223
224      a = LLVMBuildAnd(builder, a,
225                       lp_build_const_mask_aos(bld->gallivm,
226                                               type, 1 << channel), "");
227
228      /*
229       * Build a type where each element is an integer that cover the four
230       * channels.
231       */
232
233      type4 = type;
234      type4.floating = FALSE;
235      type4.width *= 4;
236      type4.length /= 4;
237
238      a = LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type4), "");
239
240      for(i = 0; i < 2; ++i) {
241         LLVMValueRef tmp = NULL;
242         int shift = shifts[channel][i];
243
244#ifdef PIPE_ARCH_LITTLE_ENDIAN
245         shift = -shift;
246#endif
247
248         if(shift > 0)
249            tmp = LLVMBuildLShr(builder, a, lp_build_const_int_vec(bld->gallivm, type4, shift*type.width), "");
250         if(shift < 0)
251            tmp = LLVMBuildShl(builder, a, lp_build_const_int_vec(bld->gallivm, type4, -shift*type.width), "");
252
253         assert(tmp);
254         if(tmp)
255            a = LLVMBuildOr(builder, a, tmp, "");
256      }
257
258      return LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type), "");
259   }
260}
261
262
263LLVMValueRef
264lp_build_swizzle_aos(struct lp_build_context *bld,
265                     LLVMValueRef a,
266                     const unsigned char swizzles[4])
267{
268   LLVMBuilderRef builder = bld->gallivm->builder;
269   const struct lp_type type = bld->type;
270   const unsigned n = type.length;
271   unsigned i, j;
272
273   if (swizzles[0] == PIPE_SWIZZLE_RED &&
274       swizzles[1] == PIPE_SWIZZLE_GREEN &&
275       swizzles[2] == PIPE_SWIZZLE_BLUE &&
276       swizzles[3] == PIPE_SWIZZLE_ALPHA) {
277      return a;
278   }
279
280   if (swizzles[0] == swizzles[1] &&
281       swizzles[1] == swizzles[2] &&
282       swizzles[2] == swizzles[3]) {
283      switch (swizzles[0]) {
284      case PIPE_SWIZZLE_RED:
285      case PIPE_SWIZZLE_GREEN:
286      case PIPE_SWIZZLE_BLUE:
287      case PIPE_SWIZZLE_ALPHA:
288         return lp_build_swizzle_scalar_aos(bld, a, swizzles[0]);
289      case PIPE_SWIZZLE_ZERO:
290         return bld->zero;
291      case PIPE_SWIZZLE_ONE:
292         return bld->one;
293      default:
294         assert(0);
295         return bld->undef;
296      }
297   }
298
299   if (type.width >= 16) {
300      /*
301       * Shuffle.
302       */
303      LLVMValueRef undef = LLVMGetUndef(lp_build_elem_type(bld->gallivm, type));
304      LLVMTypeRef i32t = LLVMInt32TypeInContext(bld->gallivm->context);
305      LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
306      LLVMValueRef aux[LP_MAX_VECTOR_LENGTH];
307
308      memset(aux, 0, sizeof aux);
309
310      for(j = 0; j < n; j += 4) {
311         for(i = 0; i < 4; ++i) {
312            unsigned shuffle;
313            switch (swizzles[i]) {
314            default:
315               assert(0);
316               /* fall through */
317            case PIPE_SWIZZLE_RED:
318            case PIPE_SWIZZLE_GREEN:
319            case PIPE_SWIZZLE_BLUE:
320            case PIPE_SWIZZLE_ALPHA:
321               shuffle = j + swizzles[i];
322               break;
323            case PIPE_SWIZZLE_ZERO:
324               shuffle = type.length + 0;
325               if (!aux[0]) {
326                  aux[0] = lp_build_const_elem(bld->gallivm, type, 0.0);
327               }
328               break;
329            case PIPE_SWIZZLE_ONE:
330               shuffle = type.length + 1;
331               if (!aux[1]) {
332                  aux[1] = lp_build_const_elem(bld->gallivm, type, 1.0);
333               }
334               break;
335            }
336            shuffles[j + i] = LLVMConstInt(i32t, shuffle, 0);
337         }
338      }
339
340      for (i = 0; i < n; ++i) {
341         if (!aux[i]) {
342            aux[i] = undef;
343         }
344      }
345
346      return LLVMBuildShuffleVector(builder, a,
347                                    LLVMConstVector(aux, n),
348                                    LLVMConstVector(shuffles, n), "");
349   } else {
350      /*
351       * Bit mask and shifts.
352       *
353       * For example, this will convert BGRA to RGBA by doing
354       *
355       *   rgba = (bgra & 0x00ff0000) >> 16
356       *        | (bgra & 0xff00ff00)
357       *        | (bgra & 0x000000ff) << 16
358       *
359       * This is necessary not only for faster cause, but because X86 backend
360       * will refuse shuffles of <4 x i8> vectors
361       */
362      LLVMValueRef res;
363      struct lp_type type4;
364      unsigned cond = 0;
365      unsigned chan;
366      int shift;
367
368      /*
369       * Start with a mixture of 1 and 0.
370       */
371      for (chan = 0; chan < 4; ++chan) {
372         if (swizzles[chan] == PIPE_SWIZZLE_ONE) {
373            cond |= 1 << chan;
374         }
375      }
376      res = lp_build_select_aos(bld, cond, bld->one, bld->zero);
377
378      /*
379       * Build a type where each element is an integer that cover the four
380       * channels.
381       */
382      type4 = type;
383      type4.floating = FALSE;
384      type4.width *= 4;
385      type4.length /= 4;
386
387      a = LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type4), "");
388      res = LLVMBuildBitCast(builder, res, lp_build_vec_type(bld->gallivm, type4), "");
389
390      /*
391       * Mask and shift the channels, trying to group as many channels in the
392       * same shift as possible
393       */
394      for (shift = -3; shift <= 3; ++shift) {
395         unsigned long long mask = 0;
396
397         assert(type4.width <= sizeof(mask)*8);
398
399         for (chan = 0; chan < 4; ++chan) {
400            /* FIXME: big endian */
401            if (swizzles[chan] < 4 &&
402                chan - swizzles[chan] == shift) {
403               mask |= ((1ULL << type.width) - 1) << (swizzles[chan] * type.width);
404            }
405         }
406
407         if (mask) {
408            LLVMValueRef masked;
409            LLVMValueRef shifted;
410
411            if (0)
412               debug_printf("shift = %i, mask = 0x%08llx\n", shift, mask);
413
414            masked = LLVMBuildAnd(builder, a,
415                                  lp_build_const_int_vec(bld->gallivm, type4, mask), "");
416            if (shift > 0) {
417               shifted = LLVMBuildShl(builder, masked,
418                                      lp_build_const_int_vec(bld->gallivm, type4, shift*type.width), "");
419            } else if (shift < 0) {
420               shifted = LLVMBuildLShr(builder, masked,
421                                       lp_build_const_int_vec(bld->gallivm, type4, -shift*type.width), "");
422            } else {
423               shifted = masked;
424            }
425
426            res = LLVMBuildOr(builder, res, shifted, "");
427         }
428      }
429
430      return LLVMBuildBitCast(builder, res,
431                              lp_build_vec_type(bld->gallivm, type), "");
432   }
433}
434
435
436/**
437 * Extended swizzle of a single channel of a SoA vector.
438 *
439 * @param bld         building context
440 * @param unswizzled  array with the 4 unswizzled values
441 * @param swizzle     one of the PIPE_SWIZZLE_*
442 *
443 * @return  the swizzled value.
444 */
445LLVMValueRef
446lp_build_swizzle_soa_channel(struct lp_build_context *bld,
447                             const LLVMValueRef *unswizzled,
448                             unsigned swizzle)
449{
450   switch (swizzle) {
451   case PIPE_SWIZZLE_RED:
452   case PIPE_SWIZZLE_GREEN:
453   case PIPE_SWIZZLE_BLUE:
454   case PIPE_SWIZZLE_ALPHA:
455      return unswizzled[swizzle];
456   case PIPE_SWIZZLE_ZERO:
457      return bld->zero;
458   case PIPE_SWIZZLE_ONE:
459      return bld->one;
460   default:
461      assert(0);
462      return bld->undef;
463   }
464}
465
466
467/**
468 * Extended swizzle of a SoA vector.
469 *
470 * @param bld         building context
471 * @param unswizzled  array with the 4 unswizzled values
472 * @param swizzles    array of PIPE_SWIZZLE_*
473 * @param swizzled    output swizzled values
474 */
475void
476lp_build_swizzle_soa(struct lp_build_context *bld,
477                     const LLVMValueRef *unswizzled,
478                     const unsigned char swizzles[4],
479                     LLVMValueRef *swizzled)
480{
481   unsigned chan;
482
483   for (chan = 0; chan < 4; ++chan) {
484      swizzled[chan] = lp_build_swizzle_soa_channel(bld, unswizzled,
485                                                    swizzles[chan]);
486   }
487}
488
489
490/**
491 * Do an extended swizzle of a SoA vector inplace.
492 *
493 * @param bld         building context
494 * @param values      intput/output array with the 4 values
495 * @param swizzles    array of PIPE_SWIZZLE_*
496 */
497void
498lp_build_swizzle_soa_inplace(struct lp_build_context *bld,
499                             LLVMValueRef *values,
500                             const unsigned char swizzles[4])
501{
502   LLVMValueRef unswizzled[4];
503   unsigned chan;
504
505   for (chan = 0; chan < 4; ++chan) {
506      unswizzled[chan] = values[chan];
507   }
508
509   lp_build_swizzle_soa(bld, unswizzled, swizzles, values);
510}
511