10912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// Copyright 2016 Google Inc. All Rights Reserved. 20912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// 30912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// Use of this source code is governed by a BSD-style license 40912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// that can be found in the COPYING file in the root of the source 50912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// tree. An additional intellectual property rights grant can be found 60912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// in the file PATENTS. All contributing project authors may 70912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// be found in the AUTHORS file in the root of the source tree. 80912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// ----------------------------------------------------------------------------- 90912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// 100912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// SSE2 code common to several files. 110912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// 120912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// Author: Vincent Rabaud (vrabaud@google.com) 130912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 140912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#ifndef WEBP_DSP_COMMON_SSE2_H_ 150912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#define WEBP_DSP_COMMON_SSE2_H_ 160912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 170912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#ifdef __cplusplus 180912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zernextern "C" { 190912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#endif 200912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 210912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#if defined(WEBP_USE_SSE2) 220912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 230912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#include <emmintrin.h> 240912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 250912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern//------------------------------------------------------------------------------ 260912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// Quite useful macro for debugging. Left here for convenience. 270912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 280912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#if 0 290912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#include <stdio.h> 300912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zernstatic WEBP_INLINE void PrintReg(const __m128i r, const char* const name, 310912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern int size) { 320912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern int n; 330912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern union { 340912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern __m128i r; 350912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern uint8_t i8[16]; 360912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern uint16_t i16[8]; 370912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern uint32_t i32[4]; 380912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern uint64_t i64[2]; 390912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern } tmp; 400912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern tmp.r = r; 410912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern fprintf(stderr, "%s\t: ", name); 420912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern if (size == 8) { 430912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern for (n = 0; n < 16; ++n) fprintf(stderr, "%.2x ", tmp.i8[n]); 440912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern } else if (size == 16) { 450912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern for (n = 0; n < 8; ++n) fprintf(stderr, "%.4x ", tmp.i16[n]); 460912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern } else if (size == 32) { 470912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern for (n = 0; n < 4; ++n) fprintf(stderr, "%.8x ", tmp.i32[n]); 480912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern } else { 490912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern for (n = 0; n < 2; ++n) fprintf(stderr, "%.16lx ", tmp.i64[n]); 500912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern } 510912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern fprintf(stderr, "\n"); 520912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern} 530912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#endif 540912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 550912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern//------------------------------------------------------------------------------ 560912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// Math functions. 570912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 580912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// Return the sum of all the 8b in the register. 590912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zernstatic WEBP_INLINE int VP8HorizontalAdd8b(const __m128i* const a) { 600912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i zero = _mm_setzero_si128(); 610912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i sad8x2 = _mm_sad_epu8(*a, zero); 620912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // sum the two sads: sad8x2[0:1] + sad8x2[8:9] 630912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i sum = _mm_add_epi32(sad8x2, _mm_shuffle_epi32(sad8x2, 2)); 640912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern return _mm_cvtsi128_si32(sum); 650912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern} 660912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 670912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// Transpose two 4x4 16b matrices horizontally stored in registers. 680912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zernstatic WEBP_INLINE void VP8Transpose_2_4x4_16b( 690912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i* const in0, const __m128i* const in1, 700912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i* const in2, const __m128i* const in3, __m128i* const out0, 710912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern __m128i* const out1, __m128i* const out2, __m128i* const out3) { 720912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // Transpose the two 4x4. 730912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a00 a01 a02 a03 b00 b01 b02 b03 740912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a10 a11 a12 a13 b10 b11 b12 b13 750912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a20 a21 a22 a23 b20 b21 b22 b23 760912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a30 a31 a32 a33 b30 b31 b32 b33 770912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i transpose0_0 = _mm_unpacklo_epi16(*in0, *in1); 780912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i transpose0_1 = _mm_unpacklo_epi16(*in2, *in3); 790912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i transpose0_2 = _mm_unpackhi_epi16(*in0, *in1); 800912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i transpose0_3 = _mm_unpackhi_epi16(*in2, *in3); 810912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a00 a10 a01 a11 a02 a12 a03 a13 820912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a20 a30 a21 a31 a22 a32 a23 a33 830912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // b00 b10 b01 b11 b02 b12 b03 b13 840912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // b20 b30 b21 b31 b22 b32 b23 b33 850912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1); 860912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3); 870912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); 880912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3); 890912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a00 a10 a20 a30 a01 a11 a21 a31 900912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // b00 b10 b20 b30 b01 b11 b21 b31 910912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a02 a12 a22 a32 a03 a13 a23 a33 920912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // b02 b12 a22 b32 b03 b13 b23 b33 930912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern *out0 = _mm_unpacklo_epi64(transpose1_0, transpose1_1); 940912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern *out1 = _mm_unpackhi_epi64(transpose1_0, transpose1_1); 950912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern *out2 = _mm_unpacklo_epi64(transpose1_2, transpose1_3); 960912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern *out3 = _mm_unpackhi_epi64(transpose1_2, transpose1_3); 970912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a00 a10 a20 a30 b00 b10 b20 b30 980912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a01 a11 a21 a31 b01 b11 b21 b31 990912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a02 a12 a22 a32 b02 b12 b22 b32 1000912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a03 a13 a23 a33 b03 b13 b23 b33 1010912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern} 1020912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 103fa39824bb690c5806358871f46940d0450973d8aJames Zern//------------------------------------------------------------------------------ 104fa39824bb690c5806358871f46940d0450973d8aJames Zern// Channel mixing. 105fa39824bb690c5806358871f46940d0450973d8aJames Zern 106fa39824bb690c5806358871f46940d0450973d8aJames Zern// Function used several times in VP8PlanarTo24b. 107fa39824bb690c5806358871f46940d0450973d8aJames Zern// It samples the in buffer as follows: one every two unsigned char is stored 108fa39824bb690c5806358871f46940d0450973d8aJames Zern// at the beginning of the buffer, while the other half is stored at the end. 109fa39824bb690c5806358871f46940d0450973d8aJames Zern#define VP8PlanarTo24bHelper(IN, OUT) \ 110fa39824bb690c5806358871f46940d0450973d8aJames Zern do { \ 111fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i v_mask = _mm_set1_epi16(0x00ff); \ 112fa39824bb690c5806358871f46940d0450973d8aJames Zern /* Take one every two upper 8b values.*/ \ 113fa39824bb690c5806358871f46940d0450973d8aJames Zern (OUT##0) = _mm_packus_epi16(_mm_and_si128((IN##0), v_mask), \ 114fa39824bb690c5806358871f46940d0450973d8aJames Zern _mm_and_si128((IN##1), v_mask)); \ 115fa39824bb690c5806358871f46940d0450973d8aJames Zern (OUT##1) = _mm_packus_epi16(_mm_and_si128((IN##2), v_mask), \ 116fa39824bb690c5806358871f46940d0450973d8aJames Zern _mm_and_si128((IN##3), v_mask)); \ 117fa39824bb690c5806358871f46940d0450973d8aJames Zern (OUT##2) = _mm_packus_epi16(_mm_and_si128((IN##4), v_mask), \ 118fa39824bb690c5806358871f46940d0450973d8aJames Zern _mm_and_si128((IN##5), v_mask)); \ 119fa39824bb690c5806358871f46940d0450973d8aJames Zern /* Take one every two lower 8b values.*/ \ 120fa39824bb690c5806358871f46940d0450973d8aJames Zern (OUT##3) = _mm_packus_epi16(_mm_srli_epi16((IN##0), 8), \ 121fa39824bb690c5806358871f46940d0450973d8aJames Zern _mm_srli_epi16((IN##1), 8)); \ 122fa39824bb690c5806358871f46940d0450973d8aJames Zern (OUT##4) = _mm_packus_epi16(_mm_srli_epi16((IN##2), 8), \ 123fa39824bb690c5806358871f46940d0450973d8aJames Zern _mm_srli_epi16((IN##3), 8)); \ 124fa39824bb690c5806358871f46940d0450973d8aJames Zern (OUT##5) = _mm_packus_epi16(_mm_srli_epi16((IN##4), 8), \ 125fa39824bb690c5806358871f46940d0450973d8aJames Zern _mm_srli_epi16((IN##5), 8)); \ 126fa39824bb690c5806358871f46940d0450973d8aJames Zern } while (0) 127fa39824bb690c5806358871f46940d0450973d8aJames Zern 128fa39824bb690c5806358871f46940d0450973d8aJames Zern// Pack the planar buffers 129fa39824bb690c5806358871f46940d0450973d8aJames Zern// rrrr... rrrr... gggg... gggg... bbbb... bbbb.... 130fa39824bb690c5806358871f46940d0450973d8aJames Zern// triplet by triplet in the output buffer rgb as rgbrgbrgbrgb ... 131fa39824bb690c5806358871f46940d0450973d8aJames Zernstatic WEBP_INLINE void VP8PlanarTo24b(__m128i* const in0, __m128i* const in1, 132fa39824bb690c5806358871f46940d0450973d8aJames Zern __m128i* const in2, __m128i* const in3, 133fa39824bb690c5806358871f46940d0450973d8aJames Zern __m128i* const in4, __m128i* const in5) { 134fa39824bb690c5806358871f46940d0450973d8aJames Zern // The input is 6 registers of sixteen 8b but for the sake of explanation, 135fa39824bb690c5806358871f46940d0450973d8aJames Zern // let's take 6 registers of four 8b values. 136fa39824bb690c5806358871f46940d0450973d8aJames Zern // To pack, we will keep taking one every two 8b integer and move it 137fa39824bb690c5806358871f46940d0450973d8aJames Zern // around as follows: 138fa39824bb690c5806358871f46940d0450973d8aJames Zern // Input: 139fa39824bb690c5806358871f46940d0450973d8aJames Zern // r0r1r2r3 | r4r5r6r7 | g0g1g2g3 | g4g5g6g7 | b0b1b2b3 | b4b5b6b7 140fa39824bb690c5806358871f46940d0450973d8aJames Zern // Split the 6 registers in two sets of 3 registers: the first set as the even 141fa39824bb690c5806358871f46940d0450973d8aJames Zern // 8b bytes, the second the odd ones: 142fa39824bb690c5806358871f46940d0450973d8aJames Zern // r0r2r4r6 | g0g2g4g6 | b0b2b4b6 | r1r3r5r7 | g1g3g5g7 | b1b3b5b7 143fa39824bb690c5806358871f46940d0450973d8aJames Zern // Repeat the same permutations twice more: 144fa39824bb690c5806358871f46940d0450973d8aJames Zern // r0r4g0g4 | b0b4r1r5 | g1g5b1b5 | r2r6g2g6 | b2b6r3r7 | g3g7b3b7 145fa39824bb690c5806358871f46940d0450973d8aJames Zern // r0g0b0r1 | g1b1r2g2 | b2r3g3b3 | r4g4b4r5 | g5b5r6g6 | b6r7g7b7 146fa39824bb690c5806358871f46940d0450973d8aJames Zern __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; 147fa39824bb690c5806358871f46940d0450973d8aJames Zern VP8PlanarTo24bHelper(*in, tmp); 148fa39824bb690c5806358871f46940d0450973d8aJames Zern VP8PlanarTo24bHelper(tmp, *in); 149fa39824bb690c5806358871f46940d0450973d8aJames Zern VP8PlanarTo24bHelper(*in, tmp); 150fa39824bb690c5806358871f46940d0450973d8aJames Zern // We need to do it two more times than the example as we have sixteen bytes. 151fa39824bb690c5806358871f46940d0450973d8aJames Zern { 152fa39824bb690c5806358871f46940d0450973d8aJames Zern __m128i out0, out1, out2, out3, out4, out5; 153fa39824bb690c5806358871f46940d0450973d8aJames Zern VP8PlanarTo24bHelper(tmp, out); 154fa39824bb690c5806358871f46940d0450973d8aJames Zern VP8PlanarTo24bHelper(out, *in); 155fa39824bb690c5806358871f46940d0450973d8aJames Zern } 156fa39824bb690c5806358871f46940d0450973d8aJames Zern} 157fa39824bb690c5806358871f46940d0450973d8aJames Zern 158fa39824bb690c5806358871f46940d0450973d8aJames Zern#undef VP8PlanarTo24bHelper 159fa39824bb690c5806358871f46940d0450973d8aJames Zern 160fa39824bb690c5806358871f46940d0450973d8aJames Zern// Convert four packed four-channel buffers like argbargbargbargb... into the 161fa39824bb690c5806358871f46940d0450973d8aJames Zern// split channels aaaaa ... rrrr ... gggg .... bbbbb ...... 162fa39824bb690c5806358871f46940d0450973d8aJames Zernstatic WEBP_INLINE void VP8L32bToPlanar(__m128i* const in0, 163fa39824bb690c5806358871f46940d0450973d8aJames Zern __m128i* const in1, 164fa39824bb690c5806358871f46940d0450973d8aJames Zern __m128i* const in2, 165fa39824bb690c5806358871f46940d0450973d8aJames Zern __m128i* const in3) { 166fa39824bb690c5806358871f46940d0450973d8aJames Zern // Column-wise transpose. 167fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i A0 = _mm_unpacklo_epi8(*in0, *in1); 168fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i A1 = _mm_unpackhi_epi8(*in0, *in1); 169fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i A2 = _mm_unpacklo_epi8(*in2, *in3); 170fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i A3 = _mm_unpackhi_epi8(*in2, *in3); 171fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i B0 = _mm_unpacklo_epi8(A0, A1); 172fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i B1 = _mm_unpackhi_epi8(A0, A1); 173fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i B2 = _mm_unpacklo_epi8(A2, A3); 174fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i B3 = _mm_unpackhi_epi8(A2, A3); 175fa39824bb690c5806358871f46940d0450973d8aJames Zern // C0 = g7 g6 ... g1 g0 | b7 b6 ... b1 b0 176fa39824bb690c5806358871f46940d0450973d8aJames Zern // C1 = a7 a6 ... a1 a0 | r7 r6 ... r1 r0 177fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i C0 = _mm_unpacklo_epi8(B0, B1); 178fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i C1 = _mm_unpackhi_epi8(B0, B1); 179fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i C2 = _mm_unpacklo_epi8(B2, B3); 180fa39824bb690c5806358871f46940d0450973d8aJames Zern const __m128i C3 = _mm_unpackhi_epi8(B2, B3); 181fa39824bb690c5806358871f46940d0450973d8aJames Zern // Gather the channels. 182fa39824bb690c5806358871f46940d0450973d8aJames Zern *in0 = _mm_unpackhi_epi64(C1, C3); 183fa39824bb690c5806358871f46940d0450973d8aJames Zern *in1 = _mm_unpacklo_epi64(C1, C3); 184fa39824bb690c5806358871f46940d0450973d8aJames Zern *in2 = _mm_unpackhi_epi64(C0, C2); 185fa39824bb690c5806358871f46940d0450973d8aJames Zern *in3 = _mm_unpacklo_epi64(C0, C2); 186fa39824bb690c5806358871f46940d0450973d8aJames Zern} 187fa39824bb690c5806358871f46940d0450973d8aJames Zern 1880912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#endif // WEBP_USE_SSE2 1890912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 1900912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#ifdef __cplusplus 1910912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern} // extern "C" 1920912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#endif 1930912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern 1940912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#endif // WEBP_DSP_COMMON_SSE2_H_ 195