1// Copyright 2015 Google Inc. All Rights Reserved. 2// 3// Use of this source code is governed by a BSD-style license 4// that can be found in the COPYING file in the root of the source 5// tree. An additional intellectual property rights grant can be found 6// in the file PATENTS. All contributing project authors may 7// be found in the AUTHORS file in the root of the source tree. 8// ----------------------------------------------------------------------------- 9// 10// SSE2 variant of methods for lossless encoder 11// 12// Author: Skal (pascal.massimino@gmail.com) 13 14#include "./dsp.h" 15 16#if defined(WEBP_USE_SSE2) 17#include <assert.h> 18#include <emmintrin.h> 19#include "./lossless.h" 20 21// For sign-extended multiplying constants, pre-shifted by 5: 22#define CST_5b(X) (((int16_t)((uint16_t)X << 8)) >> 5) 23 24//------------------------------------------------------------------------------ 25// Subtract-Green Transform 26 27static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) { 28 int i; 29 for (i = 0; i + 4 <= num_pixels; i += 4) { 30 const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb 31 const __m128i A = _mm_srli_epi16(in, 8); // 0 a 0 g 32 const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); 33 const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // 0g0g 34 const __m128i out = _mm_sub_epi8(in, C); 35 _mm_storeu_si128((__m128i*)&argb_data[i], out); 36 } 37 // fallthrough and finish off with plain-C 38 VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i); 39} 40 41//------------------------------------------------------------------------------ 42// Color Transform 43 44static void TransformColor(const VP8LMultipliers* const m, 45 uint32_t* argb_data, int num_pixels) { 46 const __m128i mults_rb = _mm_set_epi16( 47 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_), 48 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_), 49 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_), 50 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_)); 51 const __m128i mults_b2 = _mm_set_epi16( 52 CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0, 53 CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0); 54 const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks 55 const __m128i mask_rb = _mm_set1_epi32(0x00ff00ff); // red-blue masks 56 int i; 57 for (i = 0; i + 4 <= num_pixels; i += 4) { 58 const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb 59 const __m128i A = _mm_and_si128(in, mask_ag); // a 0 g 0 60 const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); 61 const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // g0g0 62 const __m128i D = _mm_mulhi_epi16(C, mults_rb); // x dr x db1 63 const __m128i E = _mm_slli_epi16(in, 8); // r 0 b 0 64 const __m128i F = _mm_mulhi_epi16(E, mults_b2); // x db2 0 0 65 const __m128i G = _mm_srli_epi32(F, 16); // 0 0 x db2 66 const __m128i H = _mm_add_epi8(G, D); // x dr x db 67 const __m128i I = _mm_and_si128(H, mask_rb); // 0 dr 0 db 68 const __m128i out = _mm_sub_epi8(in, I); 69 _mm_storeu_si128((__m128i*)&argb_data[i], out); 70 } 71 // fallthrough and finish off with plain-C 72 VP8LTransformColor_C(m, argb_data + i, num_pixels - i); 73} 74 75//------------------------------------------------------------------------------ 76#define SPAN 8 77static void CollectColorBlueTransforms(const uint32_t* argb, int stride, 78 int tile_width, int tile_height, 79 int green_to_blue, int red_to_blue, 80 int histo[]) { 81 const __m128i mults_r = _mm_set_epi16( 82 CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0, 83 CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0); 84 const __m128i mults_g = _mm_set_epi16( 85 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue), 86 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue)); 87 const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask 88 const __m128i mask_b = _mm_set1_epi32(0x0000ff); // blue mask 89 int y; 90 for (y = 0; y < tile_height; ++y) { 91 const uint32_t* const src = argb + y * stride; 92 int i, x; 93 for (x = 0; x + SPAN <= tile_width; x += SPAN) { 94 uint16_t values[SPAN]; 95 const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); 96 const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); 97 const __m128i A0 = _mm_slli_epi16(in0, 8); // r 0 | b 0 98 const __m128i A1 = _mm_slli_epi16(in1, 8); 99 const __m128i B0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0 100 const __m128i B1 = _mm_and_si128(in1, mask_g); 101 const __m128i C0 = _mm_mulhi_epi16(A0, mults_r); // x db | 0 0 102 const __m128i C1 = _mm_mulhi_epi16(A1, mults_r); 103 const __m128i D0 = _mm_mulhi_epi16(B0, mults_g); // 0 0 | x db 104 const __m128i D1 = _mm_mulhi_epi16(B1, mults_g); 105 const __m128i E0 = _mm_sub_epi8(in0, D0); // x x | x b' 106 const __m128i E1 = _mm_sub_epi8(in1, D1); 107 const __m128i F0 = _mm_srli_epi32(C0, 16); // 0 0 | x db 108 const __m128i F1 = _mm_srli_epi32(C1, 16); 109 const __m128i G0 = _mm_sub_epi8(E0, F0); // 0 0 | x b' 110 const __m128i G1 = _mm_sub_epi8(E1, F1); 111 const __m128i H0 = _mm_and_si128(G0, mask_b); // 0 0 | 0 b 112 const __m128i H1 = _mm_and_si128(G1, mask_b); 113 const __m128i I = _mm_packs_epi32(H0, H1); // 0 b' | 0 b' 114 _mm_storeu_si128((__m128i*)values, I); 115 for (i = 0; i < SPAN; ++i) ++histo[values[i]]; 116 } 117 } 118 { 119 const int left_over = tile_width & (SPAN - 1); 120 if (left_over > 0) { 121 VP8LCollectColorBlueTransforms_C(argb + tile_width - left_over, stride, 122 left_over, tile_height, 123 green_to_blue, red_to_blue, histo); 124 } 125 } 126} 127 128static void CollectColorRedTransforms(const uint32_t* argb, int stride, 129 int tile_width, int tile_height, 130 int green_to_red, int histo[]) { 131 const __m128i mults_g = _mm_set_epi16( 132 0, CST_5b(green_to_red), 0, CST_5b(green_to_red), 133 0, CST_5b(green_to_red), 0, CST_5b(green_to_red)); 134 const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask 135 const __m128i mask = _mm_set1_epi32(0xff); 136 137 int y; 138 for (y = 0; y < tile_height; ++y) { 139 const uint32_t* const src = argb + y * stride; 140 int i, x; 141 for (x = 0; x + SPAN <= tile_width; x += SPAN) { 142 uint16_t values[SPAN]; 143 const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); 144 const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); 145 const __m128i A0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0 146 const __m128i A1 = _mm_and_si128(in1, mask_g); 147 const __m128i B0 = _mm_srli_epi32(in0, 16); // 0 0 | x r 148 const __m128i B1 = _mm_srli_epi32(in1, 16); 149 const __m128i C0 = _mm_mulhi_epi16(A0, mults_g); // 0 0 | x dr 150 const __m128i C1 = _mm_mulhi_epi16(A1, mults_g); 151 const __m128i E0 = _mm_sub_epi8(B0, C0); // x x | x r' 152 const __m128i E1 = _mm_sub_epi8(B1, C1); 153 const __m128i F0 = _mm_and_si128(E0, mask); // 0 0 | 0 r' 154 const __m128i F1 = _mm_and_si128(E1, mask); 155 const __m128i I = _mm_packs_epi32(F0, F1); 156 _mm_storeu_si128((__m128i*)values, I); 157 for (i = 0; i < SPAN; ++i) ++histo[values[i]]; 158 } 159 } 160 { 161 const int left_over = tile_width & (SPAN - 1); 162 if (left_over > 0) { 163 VP8LCollectColorRedTransforms_C(argb + tile_width - left_over, stride, 164 left_over, tile_height, 165 green_to_red, histo); 166 } 167 } 168} 169#undef SPAN 170 171//------------------------------------------------------------------------------ 172 173#define LINE_SIZE 16 // 8 or 16 174static void AddVector(const uint32_t* a, const uint32_t* b, uint32_t* out, 175 int size) { 176 int i; 177 assert(size % LINE_SIZE == 0); 178 for (i = 0; i < size; i += LINE_SIZE) { 179 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]); 180 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); 181#if (LINE_SIZE == 16) 182 const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]); 183 const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]); 184#endif 185 const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[i + 0]); 186 const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[i + 4]); 187#if (LINE_SIZE == 16) 188 const __m128i b2 = _mm_loadu_si128((const __m128i*)&b[i + 8]); 189 const __m128i b3 = _mm_loadu_si128((const __m128i*)&b[i + 12]); 190#endif 191 _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0)); 192 _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1)); 193#if (LINE_SIZE == 16) 194 _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2)); 195 _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3)); 196#endif 197 } 198} 199 200static void AddVectorEq(const uint32_t* a, uint32_t* out, int size) { 201 int i; 202 assert(size % LINE_SIZE == 0); 203 for (i = 0; i < size; i += LINE_SIZE) { 204 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]); 205 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); 206#if (LINE_SIZE == 16) 207 const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]); 208 const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]); 209#endif 210 const __m128i b0 = _mm_loadu_si128((const __m128i*)&out[i + 0]); 211 const __m128i b1 = _mm_loadu_si128((const __m128i*)&out[i + 4]); 212#if (LINE_SIZE == 16) 213 const __m128i b2 = _mm_loadu_si128((const __m128i*)&out[i + 8]); 214 const __m128i b3 = _mm_loadu_si128((const __m128i*)&out[i + 12]); 215#endif 216 _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0)); 217 _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1)); 218#if (LINE_SIZE == 16) 219 _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2)); 220 _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3)); 221#endif 222 } 223} 224#undef LINE_SIZE 225 226// Note we are adding uint32_t's as *signed* int32's (using _mm_add_epi32). But 227// that's ok since the histogram values are less than 1<<28 (max picture size). 228static void HistogramAdd(const VP8LHistogram* const a, 229 const VP8LHistogram* const b, 230 VP8LHistogram* const out) { 231 int i; 232 const int literal_size = VP8LHistogramNumCodes(a->palette_code_bits_); 233 assert(a->palette_code_bits_ == b->palette_code_bits_); 234 if (b != out) { 235 AddVector(a->literal_, b->literal_, out->literal_, NUM_LITERAL_CODES); 236 AddVector(a->red_, b->red_, out->red_, NUM_LITERAL_CODES); 237 AddVector(a->blue_, b->blue_, out->blue_, NUM_LITERAL_CODES); 238 AddVector(a->alpha_, b->alpha_, out->alpha_, NUM_LITERAL_CODES); 239 } else { 240 AddVectorEq(a->literal_, out->literal_, NUM_LITERAL_CODES); 241 AddVectorEq(a->red_, out->red_, NUM_LITERAL_CODES); 242 AddVectorEq(a->blue_, out->blue_, NUM_LITERAL_CODES); 243 AddVectorEq(a->alpha_, out->alpha_, NUM_LITERAL_CODES); 244 } 245 for (i = NUM_LITERAL_CODES; i < literal_size; ++i) { 246 out->literal_[i] = a->literal_[i] + b->literal_[i]; 247 } 248 for (i = 0; i < NUM_DISTANCE_CODES; ++i) { 249 out->distance_[i] = a->distance_[i] + b->distance_[i]; 250 } 251} 252 253//------------------------------------------------------------------------------ 254// Entropy 255 256// Checks whether the X or Y contribution is worth computing and adding. 257// Used in loop unrolling. 258#define ANALYZE_X_OR_Y(x_or_y, j) \ 259 do { \ 260 if (x_or_y[i + j] != 0) retval -= VP8LFastSLog2(x_or_y[i + j]); \ 261 } while (0) 262 263// Checks whether the X + Y contribution is worth computing and adding. 264// Used in loop unrolling. 265#define ANALYZE_XY(j) \ 266 do { \ 267 if (tmp[j] != 0) { \ 268 retval -= VP8LFastSLog2(tmp[j]); \ 269 ANALYZE_X_OR_Y(X, j); \ 270 } \ 271 } while (0) 272 273static float CombinedShannonEntropy(const int X[256], const int Y[256]) { 274 int i; 275 double retval = 0.; 276 int sumX, sumXY; 277 int32_t tmp[4]; 278 __m128i zero = _mm_setzero_si128(); 279 // Sums up X + Y, 4 ints at a time (and will merge it at the end for sumXY). 280 __m128i sumXY_128 = zero; 281 __m128i sumX_128 = zero; 282 283 for (i = 0; i < 256; i += 4) { 284 const __m128i x = _mm_loadu_si128((const __m128i*)(X + i)); 285 const __m128i y = _mm_loadu_si128((const __m128i*)(Y + i)); 286 287 // Check if any X is non-zero: this actually provides a speedup as X is 288 // usually sparse. 289 if (_mm_movemask_epi8(_mm_cmpeq_epi32(x, zero)) != 0xFFFF) { 290 const __m128i xy_128 = _mm_add_epi32(x, y); 291 sumXY_128 = _mm_add_epi32(sumXY_128, xy_128); 292 293 sumX_128 = _mm_add_epi32(sumX_128, x); 294 295 // Analyze the different X + Y. 296 _mm_storeu_si128((__m128i*)tmp, xy_128); 297 298 ANALYZE_XY(0); 299 ANALYZE_XY(1); 300 ANALYZE_XY(2); 301 ANALYZE_XY(3); 302 } else { 303 // X is fully 0, so only deal with Y. 304 sumXY_128 = _mm_add_epi32(sumXY_128, y); 305 306 ANALYZE_X_OR_Y(Y, 0); 307 ANALYZE_X_OR_Y(Y, 1); 308 ANALYZE_X_OR_Y(Y, 2); 309 ANALYZE_X_OR_Y(Y, 3); 310 } 311 } 312 313 // Sum up sumX_128 to get sumX. 314 _mm_storeu_si128((__m128i*)tmp, sumX_128); 315 sumX = tmp[3] + tmp[2] + tmp[1] + tmp[0]; 316 317 // Sum up sumXY_128 to get sumXY. 318 _mm_storeu_si128((__m128i*)tmp, sumXY_128); 319 sumXY = tmp[3] + tmp[2] + tmp[1] + tmp[0]; 320 321 retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY); 322 return (float)retval; 323} 324#undef ANALYZE_X_OR_Y 325#undef ANALYZE_XY 326 327//------------------------------------------------------------------------------ 328// Entry point 329 330extern void VP8LEncDspInitSSE2(void); 331 332WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) { 333 VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed; 334 VP8LTransformColor = TransformColor; 335 VP8LCollectColorBlueTransforms = CollectColorBlueTransforms; 336 VP8LCollectColorRedTransforms = CollectColorRedTransforms; 337 VP8LHistogramAdd = HistogramAdd; 338 VP8LCombinedShannonEntropy = CombinedShannonEntropy; 339} 340 341#else // !WEBP_USE_SSE2 342 343WEBP_DSP_INIT_STUB(VP8LEncDspInitSSE2) 344 345#endif // WEBP_USE_SSE2 346