1/* 2 * Copyright 2014 The Android Open Source Project 3 * 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the LICENSE file. 6 */ 7 8#ifndef SkColor_opts_SSE2_DEFINED 9#define SkColor_opts_SSE2_DEFINED 10 11#include <emmintrin.h> 12 13#define ASSERT_EQ(a,b) SkASSERT(0xffff == _mm_movemask_epi8(_mm_cmpeq_epi8((a), (b)))) 14 15// Because no _mm_mul_epi32() in SSE2, we emulate it here. 16// Multiplies 4 32-bit integers from a by 4 32-bit intergers from b. 17// The 4 multiplication results should be represented within 32-bit 18// integers, otherwise they would be overflow. 19static inline __m128i Multiply32_SSE2(const __m128i& a, const __m128i& b) { 20 // Calculate results of a0 * b0 and a2 * b2. 21 __m128i r1 = _mm_mul_epu32(a, b); 22 // Calculate results of a1 * b1 and a3 * b3. 23 __m128i r2 = _mm_mul_epu32(_mm_srli_si128(a, 4), _mm_srli_si128(b, 4)); 24 // Shuffle results to [63..0] and interleave the results. 25 __m128i r = _mm_unpacklo_epi32(_mm_shuffle_epi32(r1, _MM_SHUFFLE(0,0,2,0)), 26 _mm_shuffle_epi32(r2, _MM_SHUFFLE(0,0,2,0))); 27 return r; 28} 29 30static inline __m128i SkAlpha255To256_SSE2(const __m128i& alpha) { 31 return _mm_add_epi32(alpha, _mm_set1_epi32(1)); 32} 33 34// See #define SkAlphaMulAlpha(a, b) SkMulDiv255Round(a, b) in SkXfermode.cpp. 35static inline __m128i SkAlphaMulAlpha_SSE2(const __m128i& a, 36 const __m128i& b) { 37 __m128i prod = _mm_mullo_epi16(a, b); 38 prod = _mm_add_epi32(prod, _mm_set1_epi32(128)); 39 prod = _mm_add_epi32(prod, _mm_srli_epi32(prod, 8)); 40 prod = _mm_srli_epi32(prod, 8); 41 42 return prod; 43} 44 45// Portable version SkAlphaMulQ is in SkColorData.h. 46static inline __m128i SkAlphaMulQ_SSE2(const __m128i& c, const __m128i& scale) { 47 const __m128i mask = _mm_set1_epi32(0xFF00FF); 48 __m128i s = _mm_or_si128(_mm_slli_epi32(scale, 16), scale); 49 50 // uint32_t rb = ((c & mask) * scale) >> 8 51 __m128i rb = _mm_and_si128(mask, c); 52 rb = _mm_mullo_epi16(rb, s); 53 rb = _mm_srli_epi16(rb, 8); 54 55 // uint32_t ag = ((c >> 8) & mask) * scale 56 __m128i ag = _mm_srli_epi16(c, 8); 57 ASSERT_EQ(ag, _mm_and_si128(mask, ag)); // ag = _mm_srli_epi16(c, 8) did this for us. 58 ag = _mm_mullo_epi16(ag, s); 59 60 // (rb & mask) | (ag & ~mask) 61 ASSERT_EQ(rb, _mm_and_si128(mask, rb)); // rb = _mm_srli_epi16(rb, 8) did this for us. 62 ag = _mm_andnot_si128(mask, ag); 63 return _mm_or_si128(rb, ag); 64} 65 66// Fast path for SkAlphaMulQ_SSE2 with a constant scale factor. 67static inline __m128i SkAlphaMulQ_SSE2(const __m128i& c, const unsigned scale) { 68 const __m128i mask = _mm_set1_epi32(0xFF00FF); 69 __m128i s = _mm_set1_epi16(scale << 8); // Move scale factor to upper byte of word. 70 71 // With mulhi, red and blue values are already in the right place and 72 // don't need to be divided by 256. 73 __m128i rb = _mm_and_si128(mask, c); 74 rb = _mm_mulhi_epu16(rb, s); 75 76 __m128i ag = _mm_andnot_si128(mask, c); 77 ag = _mm_mulhi_epu16(ag, s); // Alpha and green values are in the higher byte of each word. 78 ag = _mm_andnot_si128(mask, ag); 79 80 return _mm_or_si128(rb, ag); 81} 82 83// Portable version SkFastFourByteInterp256 is in SkColorData.h. 84static inline __m128i SkFastFourByteInterp256_SSE2(const __m128i& src, const __m128i& dst, const unsigned src_scale) { 85 // Computes dst + (((src - dst)*src_scale)>>8) 86 const __m128i mask = _mm_set1_epi32(0x00FF00FF); 87 88 // Unpack the 16x8-bit source into 2 8x16-bit splayed halves. 89 __m128i src_rb = _mm_and_si128(mask, src); 90 __m128i src_ag = _mm_srli_epi16(src, 8); 91 __m128i dst_rb = _mm_and_si128(mask, dst); 92 __m128i dst_ag = _mm_srli_epi16(dst, 8); 93 94 // Compute scaled differences. 95 __m128i diff_rb = _mm_sub_epi16(src_rb, dst_rb); 96 __m128i diff_ag = _mm_sub_epi16(src_ag, dst_ag); 97 __m128i s = _mm_set1_epi16(src_scale); 98 diff_rb = _mm_mullo_epi16(diff_rb, s); 99 diff_ag = _mm_mullo_epi16(diff_ag, s); 100 101 // Pack the differences back together. 102 diff_rb = _mm_srli_epi16(diff_rb, 8); 103 diff_ag = _mm_andnot_si128(mask, diff_ag); 104 __m128i diff = _mm_or_si128(diff_rb, diff_ag); 105 106 // Add difference to destination. 107 return _mm_add_epi8(dst, diff); 108} 109 110// Portable version SkPMLerp is in SkColorData.h 111static inline __m128i SkPMLerp_SSE2(const __m128i& src, const __m128i& dst, const unsigned scale) { 112 return SkFastFourByteInterp256_SSE2(src, dst, scale); 113} 114 115static inline __m128i SkGetPackedA32_SSE2(const __m128i& src) { 116#if SK_A32_SHIFT == 24 // It's very common (universal?) that alpha is the top byte. 117 return _mm_srli_epi32(src, 24); // You'd hope the compiler would remove the left shift then, 118#else // but I've seen Clang just do a dumb left shift of zero. :( 119 __m128i a = _mm_slli_epi32(src, (24 - SK_A32_SHIFT)); 120 return _mm_srli_epi32(a, 24); 121#endif 122} 123 124static inline __m128i SkGetPackedR32_SSE2(const __m128i& src) { 125 __m128i r = _mm_slli_epi32(src, (24 - SK_R32_SHIFT)); 126 return _mm_srli_epi32(r, 24); 127} 128 129static inline __m128i SkGetPackedG32_SSE2(const __m128i& src) { 130 __m128i g = _mm_slli_epi32(src, (24 - SK_G32_SHIFT)); 131 return _mm_srli_epi32(g, 24); 132} 133 134static inline __m128i SkGetPackedB32_SSE2(const __m128i& src) { 135 __m128i b = _mm_slli_epi32(src, (24 - SK_B32_SHIFT)); 136 return _mm_srli_epi32(b, 24); 137} 138 139static inline __m128i SkMul16ShiftRound_SSE2(const __m128i& a, 140 const __m128i& b, int shift) { 141 __m128i prod = _mm_mullo_epi16(a, b); 142 prod = _mm_add_epi16(prod, _mm_set1_epi16(1 << (shift - 1))); 143 prod = _mm_add_epi16(prod, _mm_srli_epi16(prod, shift)); 144 prod = _mm_srli_epi16(prod, shift); 145 146 return prod; 147} 148 149static inline __m128i SkPackRGB16_SSE2(const __m128i& r, 150 const __m128i& g, const __m128i& b) { 151 __m128i dr = _mm_slli_epi16(r, SK_R16_SHIFT); 152 __m128i dg = _mm_slli_epi16(g, SK_G16_SHIFT); 153 __m128i db = _mm_slli_epi16(b, SK_B16_SHIFT); 154 155 __m128i c = _mm_or_si128(dr, dg); 156 return _mm_or_si128(c, db); 157} 158 159static inline __m128i SkPackARGB32_SSE2(const __m128i& a, const __m128i& r, 160 const __m128i& g, const __m128i& b) { 161 __m128i da = _mm_slli_epi32(a, SK_A32_SHIFT); 162 __m128i dr = _mm_slli_epi32(r, SK_R32_SHIFT); 163 __m128i dg = _mm_slli_epi32(g, SK_G32_SHIFT); 164 __m128i db = _mm_slli_epi32(b, SK_B32_SHIFT); 165 166 __m128i c = _mm_or_si128(da, dr); 167 c = _mm_or_si128(c, dg); 168 return _mm_or_si128(c, db); 169} 170 171static inline __m128i SkPacked16ToR32_SSE2(const __m128i& src) { 172 __m128i r = _mm_srli_epi32(src, SK_R16_SHIFT); 173 r = _mm_and_si128(r, _mm_set1_epi32(SK_R16_MASK)); 174 r = _mm_or_si128(_mm_slli_epi32(r, (8 - SK_R16_BITS)), 175 _mm_srli_epi32(r, (2 * SK_R16_BITS - 8))); 176 177 return r; 178} 179 180static inline __m128i SkPacked16ToG32_SSE2(const __m128i& src) { 181 __m128i g = _mm_srli_epi32(src, SK_G16_SHIFT); 182 g = _mm_and_si128(g, _mm_set1_epi32(SK_G16_MASK)); 183 g = _mm_or_si128(_mm_slli_epi32(g, (8 - SK_G16_BITS)), 184 _mm_srli_epi32(g, (2 * SK_G16_BITS - 8))); 185 186 return g; 187} 188 189static inline __m128i SkPacked16ToB32_SSE2(const __m128i& src) { 190 __m128i b = _mm_srli_epi32(src, SK_B16_SHIFT); 191 b = _mm_and_si128(b, _mm_set1_epi32(SK_B16_MASK)); 192 b = _mm_or_si128(_mm_slli_epi32(b, (8 - SK_B16_BITS)), 193 _mm_srli_epi32(b, (2 * SK_B16_BITS - 8))); 194 195 return b; 196} 197 198static inline __m128i SkPixel16ToPixel32_SSE2(const __m128i& src) { 199 __m128i r = SkPacked16ToR32_SSE2(src); 200 __m128i g = SkPacked16ToG32_SSE2(src); 201 __m128i b = SkPacked16ToB32_SSE2(src); 202 203 return SkPackARGB32_SSE2(_mm_set1_epi32(0xFF), r, g, b); 204} 205 206static inline __m128i SkPixel32ToPixel16_ToU16_SSE2(const __m128i& src_pixel1, 207 const __m128i& src_pixel2) { 208 // Calculate result r. 209 __m128i r1 = _mm_srli_epi32(src_pixel1, 210 SK_R32_SHIFT + (8 - SK_R16_BITS)); 211 r1 = _mm_and_si128(r1, _mm_set1_epi32(SK_R16_MASK)); 212 __m128i r2 = _mm_srli_epi32(src_pixel2, 213 SK_R32_SHIFT + (8 - SK_R16_BITS)); 214 r2 = _mm_and_si128(r2, _mm_set1_epi32(SK_R16_MASK)); 215 __m128i r = _mm_packs_epi32(r1, r2); 216 217 // Calculate result g. 218 __m128i g1 = _mm_srli_epi32(src_pixel1, 219 SK_G32_SHIFT + (8 - SK_G16_BITS)); 220 g1 = _mm_and_si128(g1, _mm_set1_epi32(SK_G16_MASK)); 221 __m128i g2 = _mm_srli_epi32(src_pixel2, 222 SK_G32_SHIFT + (8 - SK_G16_BITS)); 223 g2 = _mm_and_si128(g2, _mm_set1_epi32(SK_G16_MASK)); 224 __m128i g = _mm_packs_epi32(g1, g2); 225 226 // Calculate result b. 227 __m128i b1 = _mm_srli_epi32(src_pixel1, 228 SK_B32_SHIFT + (8 - SK_B16_BITS)); 229 b1 = _mm_and_si128(b1, _mm_set1_epi32(SK_B16_MASK)); 230 __m128i b2 = _mm_srli_epi32(src_pixel2, 231 SK_B32_SHIFT + (8 - SK_B16_BITS)); 232 b2 = _mm_and_si128(b2, _mm_set1_epi32(SK_B16_MASK)); 233 __m128i b = _mm_packs_epi32(b1, b2); 234 235 // Store 8 16-bit colors in dst. 236 __m128i d_pixel = SkPackRGB16_SSE2(r, g, b); 237 238 return d_pixel; 239} 240 241// Portable version is SkPMSrcOver in SkColorData.h. 242static inline __m128i SkPMSrcOver_SSE2(const __m128i& src, const __m128i& dst) { 243 return _mm_add_epi32(src, 244 SkAlphaMulQ_SSE2(dst, _mm_sub_epi32(_mm_set1_epi32(256), 245 SkGetPackedA32_SSE2(src)))); 246} 247 248// Fast path for SkBlendARGB32_SSE2 with a constant alpha factor. 249static inline __m128i SkBlendARGB32_SSE2(const __m128i& src, const __m128i& dst, 250 const unsigned aa) { 251 unsigned alpha = SkAlpha255To256(aa); 252 __m128i src_scale = _mm_set1_epi16(alpha); 253 // SkAlphaMulInv256(SkGetPackedA32(src), src_scale) 254 __m128i dst_scale = SkGetPackedA32_SSE2(src); 255 // High words in dst_scale are 0, so it's safe to multiply with 16-bit src_scale. 256 dst_scale = _mm_mullo_epi16(dst_scale, src_scale); 257 dst_scale = _mm_sub_epi32(_mm_set1_epi32(0xFFFF), dst_scale); 258 dst_scale = _mm_add_epi32(dst_scale, _mm_srli_epi32(dst_scale, 8)); 259 dst_scale = _mm_srli_epi32(dst_scale, 8); 260 // Duplicate scales into 2x16-bit pattern per pixel. 261 dst_scale = _mm_shufflelo_epi16(dst_scale, _MM_SHUFFLE(2, 2, 0, 0)); 262 dst_scale = _mm_shufflehi_epi16(dst_scale, _MM_SHUFFLE(2, 2, 0, 0)); 263 264 const __m128i mask = _mm_set1_epi32(0x00FF00FF); 265 266 // Unpack the 16x8-bit source/destination into 2 8x16-bit splayed halves. 267 __m128i src_rb = _mm_and_si128(mask, src); 268 __m128i src_ag = _mm_srli_epi16(src, 8); 269 __m128i dst_rb = _mm_and_si128(mask, dst); 270 __m128i dst_ag = _mm_srli_epi16(dst, 8); 271 272 // Scale them. 273 src_rb = _mm_mullo_epi16(src_rb, src_scale); 274 src_ag = _mm_mullo_epi16(src_ag, src_scale); 275 dst_rb = _mm_mullo_epi16(dst_rb, dst_scale); 276 dst_ag = _mm_mullo_epi16(dst_ag, dst_scale); 277 278 // Add the scaled source and destination. 279 dst_rb = _mm_add_epi16(src_rb, dst_rb); 280 dst_ag = _mm_add_epi16(src_ag, dst_ag); 281 282 // Unsplay the halves back together. 283 dst_rb = _mm_srli_epi16(dst_rb, 8); 284 dst_ag = _mm_andnot_si128(mask, dst_ag); 285 return _mm_or_si128(dst_rb, dst_ag); 286} 287 288#undef ASSERT_EQ 289#endif // SkColor_opts_SSE2_DEFINED 290