1ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang/* 2ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang * Copyright (c) 2012 The WebM project authors. All Rights Reserved. 3ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang * 4ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang * Use of this source code is governed by a BSD-style license 5ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang * that can be found in the LICENSE file in the root of the source 6ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang * tree. An additional intellectual property rights grant can be found 7ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang * in the file PATENTS. All contributing project authors may 8ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang * be found in the AUTHORS file in the root of the source tree. 9ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang */ 10ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 11ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#include <emmintrin.h> // SSE2 12ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#include "vp9/common/vp9_idct.h" // for cospi constants 1391037db265ecdd914a26e056cf69207b4f50924ehkuang#include "vpx_ports/mem.h" 14ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 155ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuangvoid vp9_fdct4x4_sse2(const int16_t *input, int16_t *output, int stride) { 16b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // This 2D transform implements 4 vertical 1D transforms followed 17b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // by 4 horizontal 1D transforms. The multiplies and adds are as given 18b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // by Chen, Smith and Fralick ('77). The commands for moving the data 19b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // around have been minimized by hand. 20b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // For the purposes of the comments, the 16 inputs are referred to at i0 21b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // through iF (in raster order), intermediate variables are a0, b0, c0 22b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // through f, and correspond to the in-place computations mapped to input 23b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // locations. The outputs, o0 through oF are labeled according to the 24b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // output locations. 25b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 26ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Constants 27b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // These are the coefficients used for the multiplies. 28b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // In the comments, pN means cos(N pi /64) and mN is -cos(N pi /64), 29b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // where cospi_N_64 = cos(N pi /64) 30b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i k__cospi_A = _mm_setr_epi16(cospi_16_64, cospi_16_64, 31b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, cospi_16_64, 32b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, -cospi_16_64, 33b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, -cospi_16_64); 34b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i k__cospi_B = _mm_setr_epi16(cospi_16_64, -cospi_16_64, 35b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, -cospi_16_64, 36b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, cospi_16_64, 37b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, cospi_16_64); 38b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i k__cospi_C = _mm_setr_epi16(cospi_8_64, cospi_24_64, 39b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_8_64, cospi_24_64, 40b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_24_64, -cospi_8_64, 41b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_24_64, -cospi_8_64); 42b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i k__cospi_D = _mm_setr_epi16(cospi_24_64, -cospi_8_64, 43b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_24_64, -cospi_8_64, 44b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_8_64, cospi_24_64, 45b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_8_64, cospi_24_64); 46b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i k__cospi_E = _mm_setr_epi16(cospi_16_64, cospi_16_64, 47b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, cospi_16_64, 48b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, cospi_16_64, 49b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, cospi_16_64); 50b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i k__cospi_F = _mm_setr_epi16(cospi_16_64, -cospi_16_64, 51b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, -cospi_16_64, 52b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, -cospi_16_64, 53b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_16_64, -cospi_16_64); 54b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i k__cospi_G = _mm_setr_epi16(cospi_8_64, cospi_24_64, 55b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_8_64, cospi_24_64, 56b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian -cospi_8_64, -cospi_24_64, 57b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian -cospi_8_64, -cospi_24_64); 58b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i k__cospi_H = _mm_setr_epi16(cospi_24_64, -cospi_8_64, 59b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian cospi_24_64, -cospi_8_64, 60b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian -cospi_24_64, cospi_8_64, 61b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian -cospi_24_64, cospi_8_64); 62b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 63ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 64b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // This second rounding constant saves doing some extra adds at the end 65b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i k__DCT_CONST_ROUNDING2 = _mm_set1_epi32(DCT_CONST_ROUNDING 66b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian +(DCT_CONST_ROUNDING << 1)); 67b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const int DCT_CONST_BITS2 = DCT_CONST_BITS+2; 68ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1); 69ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0); 70b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian __m128i in0, in1; 71b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 72ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Load inputs. 73ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 74ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in0 = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); 75ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in1 = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); 76b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian in1 = _mm_unpacklo_epi64(in1, _mm_loadl_epi64((const __m128i *) 77b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian (input + 2 * stride))); 78b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian in0 = _mm_unpacklo_epi64(in0, _mm_loadl_epi64((const __m128i *) 79b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian (input + 3 * stride))); 80b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // in0 = [i0 i1 i2 i3 iC iD iE iF] 81b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // in1 = [i4 i5 i6 i7 i8 i9 iA iB] 82b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 83b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 84b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // multiply by 16 to give some extra precision 85ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in0 = _mm_slli_epi16(in0, 4); 86ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in1 = _mm_slli_epi16(in1, 4); 87ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // if (i == 0 && input[0]) input[0] += 1; 88b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // add 1 to the upper left pixel if it is non-zero, which helps reduce 89b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // the round-trip error 90ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 91b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // The mask will only contain whether the first value is zero, all 92ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // other comparison will fail as something shifted by 4 (above << 4) 93ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // can never be equal to one. To increment in the non-zero case, we 94ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // add the mask and one for the first element: 95ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // - if zero, mask = -1, v = v - 1 + 1 = v 96ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // - if non-zero, mask = 0, v = v + 0 + 1 = v + 1 97ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i mask = _mm_cmpeq_epi16(in0, k__nonzero_bias_a); 98ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in0 = _mm_add_epi16(in0, mask); 99ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in0 = _mm_add_epi16(in0, k__nonzero_bias_b); 100ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 101ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 102b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // There are 4 total stages, alternating between an add/subtract stage 103b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // followed by an multiply-and-add stage. 104b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian { 105b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Stage 1: Add/subtract 106b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 107b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // in0 = [i0 i1 i2 i3 iC iD iE iF] 108b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // in1 = [i4 i5 i6 i7 i8 i9 iA iB] 109b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i r0 = _mm_unpacklo_epi16(in0, in1); 110b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i r1 = _mm_unpackhi_epi16(in0, in1); 111b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // r0 = [i0 i4 i1 i5 i2 i6 i3 i7] 112b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // r1 = [iC i8 iD i9 iE iA iF iB] 113b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i r2 = _mm_shuffle_epi32(r0, 0xB4); 114b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i r3 = _mm_shuffle_epi32(r1, 0xB4); 115b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // r2 = [i0 i4 i1 i5 i3 i7 i2 i6] 116b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // r3 = [iC i8 iD i9 iF iB iE iA] 117b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 118b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i t0 = _mm_add_epi16(r2, r3); 119b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i t1 = _mm_sub_epi16(r2, r3); 120b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // t0 = [a0 a4 a1 a5 a3 a7 a2 a6] 121b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // t1 = [aC a8 aD a9 aF aB aE aA] 122b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 123b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Stage 2: multiply by constants (which gets us into 32 bits). 124b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // The constants needed here are: 125b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // k__cospi_A = [p16 p16 p16 p16 p16 m16 p16 m16] 126b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // k__cospi_B = [p16 m16 p16 m16 p16 p16 p16 p16] 127b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // k__cospi_C = [p08 p24 p08 p24 p24 m08 p24 m08] 128b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // k__cospi_D = [p24 m08 p24 m08 p08 p24 p08 p24] 129b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i u0 = _mm_madd_epi16(t0, k__cospi_A); 130b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i u2 = _mm_madd_epi16(t0, k__cospi_B); 131b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i u1 = _mm_madd_epi16(t1, k__cospi_C); 132b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i u3 = _mm_madd_epi16(t1, k__cospi_D); 133b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Then add and right-shift to get back to 16-bit range 134ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 135b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 136ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 137b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 138ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 139b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 140ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 141b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 142b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // w0 = [b0 b1 b7 b6] 143b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // w1 = [b8 b9 bF bE] 144b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // w2 = [b4 b5 b3 b2] 145b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // w3 = [bC bD bB bA] 146b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i x0 = _mm_packs_epi32(w0, w1); 147b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i x1 = _mm_packs_epi32(w2, w3); 148b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // x0 = [b0 b1 b7 b6 b8 b9 bF bE] 149b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // x1 = [b4 b5 b3 b2 bC bD bB bA] 150b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian in0 = _mm_shuffle_epi32(x0, 0xD8); 151b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian in1 = _mm_shuffle_epi32(x1, 0x8D); 152b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // in0 = [b0 b1 b8 b9 b7 b6 bF bE] 153b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // in1 = [b3 b2 bB bA b4 b5 bC bD] 154b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian } 155b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian { 156b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // vertical DCTs finished. Now we do the horizontal DCTs. 157b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Stage 3: Add/subtract 158b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 159b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i t0 = _mm_add_epi16(in0, in1); 160b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i t1 = _mm_sub_epi16(in0, in1); 161b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // t0 = [c0 c1 c8 c9 c4 c5 cC cD] 162b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // t1 = [c3 c2 cB cA -c7 -c6 -cF -cE] 163b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 164b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Stage 4: multiply by constants (which gets us into 32 bits). 165b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // The constants needed here are: 166b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // k__cospi_E = [p16 p16 p16 p16 p16 p16 p16 p16] 167b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // k__cospi_F = [p16 m16 p16 m16 p16 m16 p16 m16] 168b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // k__cospi_G = [p08 p24 p08 p24 m08 m24 m08 m24] 169b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // k__cospi_H = [p24 m08 p24 m08 m24 p08 m24 p08] 170b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i u0 = _mm_madd_epi16(t0, k__cospi_E); 171b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i u1 = _mm_madd_epi16(t0, k__cospi_F); 172b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i u2 = _mm_madd_epi16(t1, k__cospi_G); 173b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i u3 = _mm_madd_epi16(t1, k__cospi_H); 174b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Then add and right-shift to get back to 16-bit range 175b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // but this combines the final right-shift as well to save operations 176b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // This unusual rounding operations is to maintain bit-accurate 177b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // compatibility with the c version of this function which has two 178b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // rounding steps in a row. 179b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING2); 180b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING2); 181b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING2); 182b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING2); 183b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS2); 184b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS2); 185b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS2); 186b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS2); 187b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // w0 = [o0 o4 o8 oC] 188b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // w1 = [o2 o6 oA oE] 189b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // w2 = [o1 o5 o9 oD] 190b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // w3 = [o3 o7 oB oF] 191b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // remember the o's are numbered according to the correct output location 192b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i x0 = _mm_packs_epi32(w0, w1); 193b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i x1 = _mm_packs_epi32(w2, w3); 194b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // x0 = [o0 o4 o8 oC o2 o6 oA oE] 195b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // x1 = [o1 o5 o9 oD o3 o7 oB oF] 196b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i y0 = _mm_unpacklo_epi16(x0, x1); 197b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian const __m128i y1 = _mm_unpackhi_epi16(x0, x1); 198b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // y0 = [o0 o1 o4 o5 o8 o9 oC oD] 199b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // y1 = [o2 o3 o6 o7 oA oB oE oF] 200b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian in0 = _mm_unpacklo_epi32(y0, y1); 201b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // in0 = [o0 o1 o2 o3 o4 o5 o6 o7] 202b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian in1 = _mm_unpackhi_epi32(y0, y1); 203b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // in1 = [o8 o9 oA oB oC oD oE oF] 204b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian } 205b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Post-condition (v + 1) >> 2 is now incorporated into previous 206b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // add and right-shift commands. Only 2 store instructions needed 207b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // because we are using the fact that 1/3 are stored just after 0/2. 208b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian { 209b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian _mm_storeu_si128((__m128i *)(output + 0 * 4), in0); 210b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian _mm_storeu_si128((__m128i *)(output + 2 * 4), in1); 211ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 212ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 213ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 214b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 2155ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuangstatic INLINE void load_buffer_4x4(const int16_t *input, __m128i *in, 2165ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang int stride) { 21791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1); 21891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0); 21991037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i mask; 22091037db265ecdd914a26e056cf69207b4f50924ehkuang 22191037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_loadl_epi64((const __m128i *)(input + 0 * stride)); 22291037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_loadl_epi64((const __m128i *)(input + 1 * stride)); 22391037db265ecdd914a26e056cf69207b4f50924ehkuang in[2] = _mm_loadl_epi64((const __m128i *)(input + 2 * stride)); 22491037db265ecdd914a26e056cf69207b4f50924ehkuang in[3] = _mm_loadl_epi64((const __m128i *)(input + 3 * stride)); 22591037db265ecdd914a26e056cf69207b4f50924ehkuang 22691037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_slli_epi16(in[0], 4); 22791037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_slli_epi16(in[1], 4); 22891037db265ecdd914a26e056cf69207b4f50924ehkuang in[2] = _mm_slli_epi16(in[2], 4); 22991037db265ecdd914a26e056cf69207b4f50924ehkuang in[3] = _mm_slli_epi16(in[3], 4); 23091037db265ecdd914a26e056cf69207b4f50924ehkuang 23191037db265ecdd914a26e056cf69207b4f50924ehkuang mask = _mm_cmpeq_epi16(in[0], k__nonzero_bias_a); 23291037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_add_epi16(in[0], mask); 23391037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_add_epi16(in[0], k__nonzero_bias_b); 23491037db265ecdd914a26e056cf69207b4f50924ehkuang} 23591037db265ecdd914a26e056cf69207b4f50924ehkuang 23691037db265ecdd914a26e056cf69207b4f50924ehkuangstatic INLINE void write_buffer_4x4(int16_t *output, __m128i *res) { 23791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i kOne = _mm_set1_epi16(1); 23891037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in01 = _mm_unpacklo_epi64(res[0], res[1]); 23991037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in23 = _mm_unpacklo_epi64(res[2], res[3]); 24091037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i out01 = _mm_add_epi16(in01, kOne); 24191037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i out23 = _mm_add_epi16(in23, kOne); 24291037db265ecdd914a26e056cf69207b4f50924ehkuang out01 = _mm_srai_epi16(out01, 2); 24391037db265ecdd914a26e056cf69207b4f50924ehkuang out23 = _mm_srai_epi16(out23, 2); 24491037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 0 * 8), out01); 24591037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 1 * 8), out23); 24691037db265ecdd914a26e056cf69207b4f50924ehkuang} 24791037db265ecdd914a26e056cf69207b4f50924ehkuang 24891037db265ecdd914a26e056cf69207b4f50924ehkuangstatic INLINE void transpose_4x4(__m128i *res) { 24991037db265ecdd914a26e056cf69207b4f50924ehkuang // Combine and transpose 25091037db265ecdd914a26e056cf69207b4f50924ehkuang // 00 01 02 03 20 21 22 23 25191037db265ecdd914a26e056cf69207b4f50924ehkuang // 10 11 12 13 30 31 32 33 25291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_0 = _mm_unpacklo_epi16(res[0], res[1]); 25391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_1 = _mm_unpackhi_epi16(res[0], res[1]); 25491037db265ecdd914a26e056cf69207b4f50924ehkuang 25591037db265ecdd914a26e056cf69207b4f50924ehkuang // 00 10 01 11 02 12 03 13 25691037db265ecdd914a26e056cf69207b4f50924ehkuang // 20 30 21 31 22 32 23 33 25791037db265ecdd914a26e056cf69207b4f50924ehkuang res[0] = _mm_unpacklo_epi32(tr0_0, tr0_1); 25891037db265ecdd914a26e056cf69207b4f50924ehkuang res[2] = _mm_unpackhi_epi32(tr0_0, tr0_1); 25991037db265ecdd914a26e056cf69207b4f50924ehkuang 26091037db265ecdd914a26e056cf69207b4f50924ehkuang // 00 10 20 30 01 11 21 31 26191037db265ecdd914a26e056cf69207b4f50924ehkuang // 02 12 22 32 03 13 23 33 26291037db265ecdd914a26e056cf69207b4f50924ehkuang // only use the first 4 16-bit integers 26391037db265ecdd914a26e056cf69207b4f50924ehkuang res[1] = _mm_unpackhi_epi64(res[0], res[0]); 26491037db265ecdd914a26e056cf69207b4f50924ehkuang res[3] = _mm_unpackhi_epi64(res[2], res[2]); 26591037db265ecdd914a26e056cf69207b4f50924ehkuang} 26691037db265ecdd914a26e056cf69207b4f50924ehkuang 267b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid fdct4_sse2(__m128i *in) { 26891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64); 26991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); 2705ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); 2715ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); 27291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 27391037db265ecdd914a26e056cf69207b4f50924ehkuang 27491037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i u[4], v[4]; 2755ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang u[0]=_mm_unpacklo_epi16(in[0], in[1]); 2765ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang u[1]=_mm_unpacklo_epi16(in[3], in[2]); 2775ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang 2785ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang v[0] = _mm_add_epi16(u[0], u[1]); 2795ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang v[1] = _mm_sub_epi16(u[0], u[1]); 28091037db265ecdd914a26e056cf69207b4f50924ehkuang 28191037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_madd_epi16(v[0], k__cospi_p16_p16); // 0 28291037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_madd_epi16(v[0], k__cospi_p16_m16); // 2 2835ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang u[2] = _mm_madd_epi16(v[1], k__cospi_p08_p24); // 1 2845ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang u[3] = _mm_madd_epi16(v[1], k__cospi_p24_m08); // 3 28591037db265ecdd914a26e056cf69207b4f50924ehkuang 28691037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); 28791037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); 28891037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); 28991037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); 29091037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); 29191037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); 29291037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); 29391037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); 29491037db265ecdd914a26e056cf69207b4f50924ehkuang 29591037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_packs_epi32(u[0], u[1]); 29691037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_packs_epi32(u[2], u[3]); 29791037db265ecdd914a26e056cf69207b4f50924ehkuang transpose_4x4(in); 29891037db265ecdd914a26e056cf69207b4f50924ehkuang} 29991037db265ecdd914a26e056cf69207b4f50924ehkuang 300b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid fadst4_sse2(__m128i *in) { 30191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__sinpi_p01_p02 = pair_set_epi16(sinpi_1_9, sinpi_2_9); 30291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__sinpi_p04_m01 = pair_set_epi16(sinpi_4_9, -sinpi_1_9); 30391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__sinpi_p03_p04 = pair_set_epi16(sinpi_3_9, sinpi_4_9); 30491037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__sinpi_m03_p02 = pair_set_epi16(-sinpi_3_9, sinpi_2_9); 30591037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__sinpi_p03_p03 = _mm_set1_epi16(sinpi_3_9); 30691037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i kZero = _mm_set1_epi16(0); 30791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 30891037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i u[8], v[8]; 30991037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in7 = _mm_add_epi16(in[0], in[1]); 31091037db265ecdd914a26e056cf69207b4f50924ehkuang 31191037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(in[0], in[1]); 31291037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpacklo_epi16(in[2], in[3]); 31391037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_unpacklo_epi16(in7, kZero); 31491037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_unpacklo_epi16(in[2], kZero); 3159b35249446b07f40ac5fcc3205f2c048616efacchkuang u[4] = _mm_unpacklo_epi16(in[3], kZero); 31691037db265ecdd914a26e056cf69207b4f50924ehkuang 31791037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__sinpi_p01_p02); // s0 + s2 31891037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__sinpi_p03_p04); // s4 + s5 31991037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[2], k__sinpi_p03_p03); // x1 32091037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[0], k__sinpi_p04_m01); // s1 - s3 32191037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_madd_epi16(u[1], k__sinpi_m03_p02); // -s4 + s6 32291037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_madd_epi16(u[3], k__sinpi_p03_p03); // s4 3239b35249446b07f40ac5fcc3205f2c048616efacchkuang v[6] = _mm_madd_epi16(u[4], k__sinpi_p03_p03); 32491037db265ecdd914a26e056cf69207b4f50924ehkuang 32591037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], v[1]); 3269b35249446b07f40ac5fcc3205f2c048616efacchkuang u[1] = _mm_sub_epi32(v[2], v[6]); 32791037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[3], v[4]); 32891037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_sub_epi32(u[2], u[0]); 32991037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_slli_epi32(v[5], 2); 33091037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_sub_epi32(u[4], v[5]); 33191037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_add_epi32(u[3], u[5]); 33291037db265ecdd914a26e056cf69207b4f50924ehkuang 33391037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); 33491037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); 33591037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); 33691037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); 33791037db265ecdd914a26e056cf69207b4f50924ehkuang 33891037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); 33991037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); 34091037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); 34191037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); 34291037db265ecdd914a26e056cf69207b4f50924ehkuang 34391037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_packs_epi32(u[0], u[2]); 34491037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_packs_epi32(u[1], u[3]); 34591037db265ecdd914a26e056cf69207b4f50924ehkuang transpose_4x4(in); 34691037db265ecdd914a26e056cf69207b4f50924ehkuang} 34791037db265ecdd914a26e056cf69207b4f50924ehkuang 348b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid vp9_fht4x4_sse2(const int16_t *input, int16_t *output, 349b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian int stride, int tx_type) { 35091037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in[4]; 351b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 35291037db265ecdd914a26e056cf69207b4f50924ehkuang switch (tx_type) { 353b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case DCT_DCT: 354b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian vp9_fdct4x4_sse2(input, output, stride); 35591037db265ecdd914a26e056cf69207b4f50924ehkuang break; 356b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case ADST_DCT: 357b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian load_buffer_4x4(input, in, stride); 358b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst4_sse2(in); 359b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fdct4_sse2(in); 360b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian write_buffer_4x4(output, in); 36191037db265ecdd914a26e056cf69207b4f50924ehkuang break; 362b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case DCT_ADST: 363b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian load_buffer_4x4(input, in, stride); 364b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fdct4_sse2(in); 365b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst4_sse2(in); 366b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian write_buffer_4x4(output, in); 36791037db265ecdd914a26e056cf69207b4f50924ehkuang break; 368b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case ADST_ADST: 369b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian load_buffer_4x4(input, in, stride); 370b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst4_sse2(in); 371b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst4_sse2(in); 372b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian write_buffer_4x4(output, in); 37391037db265ecdd914a26e056cf69207b4f50924ehkuang break; 374b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian default: 375b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian assert(0); 376b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian break; 37791037db265ecdd914a26e056cf69207b4f50924ehkuang } 37891037db265ecdd914a26e056cf69207b4f50924ehkuang} 37991037db265ecdd914a26e056cf69207b4f50924ehkuang 3805ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuangvoid vp9_fdct8x8_sse2(const int16_t *input, int16_t *output, int stride) { 381ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang int pass; 382ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Constants 383ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // When we use them, in one case, they are all the same. In all others 384ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // it's a pair of them that we need to repeat four times. This is done 385ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // by constructing the 32 bit constant corresponding to that pair. 386ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64); 387ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); 388ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); 389ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); 390ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); 391ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); 392ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); 393ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); 394ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 395ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Load input 39691037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride)); 39791037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride)); 39891037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride)); 39991037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride)); 40091037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride)); 40191037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride)); 40291037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride)); 40391037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride)); 404ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Pre-condition input (shift by two) 405ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in0 = _mm_slli_epi16(in0, 2); 406ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in1 = _mm_slli_epi16(in1, 2); 407ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in2 = _mm_slli_epi16(in2, 2); 408ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in3 = _mm_slli_epi16(in3, 2); 409ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in4 = _mm_slli_epi16(in4, 2); 410ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in5 = _mm_slli_epi16(in5, 2); 411ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in6 = _mm_slli_epi16(in6, 2); 412ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in7 = _mm_slli_epi16(in7, 2); 413ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 414ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // We do two passes, first the columns, then the rows. The results of the 415ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // first pass are transposed so that the same column code can be reused. The 416ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // results of the second pass are also transposed so that the rows (processed 417ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // as columns) are put back in row positions. 418ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang for (pass = 0; pass < 2; pass++) { 419ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // To store results of each pass before the transpose. 420ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i res0, res1, res2, res3, res4, res5, res6, res7; 421b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Add/subtract 422ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q0 = _mm_add_epi16(in0, in7); 423ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q1 = _mm_add_epi16(in1, in6); 424ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q2 = _mm_add_epi16(in2, in5); 425ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q3 = _mm_add_epi16(in3, in4); 426ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q4 = _mm_sub_epi16(in3, in4); 427ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q5 = _mm_sub_epi16(in2, in5); 428ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q6 = _mm_sub_epi16(in1, in6); 429ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q7 = _mm_sub_epi16(in0, in7); 430ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Work on first four results 431ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 432b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Add/subtract 433ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r0 = _mm_add_epi16(q0, q3); 434ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r1 = _mm_add_epi16(q1, q2); 435ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r2 = _mm_sub_epi16(q1, q2); 436ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r3 = _mm_sub_epi16(q0, q3); 437ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Interleave to do the multiply by constants which gets us into 32bits 438ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(r0, r1); 439ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(r0, r1); 440ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(r2, r3); 441ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(r2, r3); 442ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16); 443ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16); 444ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16); 445ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16); 446ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08); 447ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08); 448ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24); 449ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24); 450ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 451ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 452ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 453ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 454ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 455ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); 456ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); 457ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); 458ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); 459ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 460ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 461ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 462ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 463ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); 464ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); 465ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); 466ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); 467ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 468ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res0 = _mm_packs_epi32(w0, w1); 469ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res4 = _mm_packs_epi32(w2, w3); 470ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res2 = _mm_packs_epi32(w4, w5); 471ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res6 = _mm_packs_epi32(w6, w7); 472ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 473ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Work on next four results 474ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 475ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Interleave to do the multiply by constants which gets us into 32bits 476ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i d0 = _mm_unpacklo_epi16(q6, q5); 477ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i d1 = _mm_unpackhi_epi16(q6, q5); 478ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16); 479ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16); 480ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16); 481ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16); 482ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 483ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING); 484ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING); 485ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING); 486ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING); 487ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS); 488ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS); 489ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS); 490ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS); 491ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 492ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r0 = _mm_packs_epi32(s0, s1); 493ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r1 = _mm_packs_epi32(s2, s3); 494b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Add/subtract 495ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i x0 = _mm_add_epi16(q4, r0); 496ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i x1 = _mm_sub_epi16(q4, r0); 497ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i x2 = _mm_sub_epi16(q7, r1); 498ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i x3 = _mm_add_epi16(q7, r1); 499ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Interleave to do the multiply by constants which gets us into 32bits 500ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(x0, x3); 501ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(x0, x3); 502ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(x1, x2); 503ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(x1, x2); 504ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04); 505ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04); 506ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28); 507ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28); 508ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20); 509ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20); 510ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12); 511ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12); 512ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 513ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 514ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 515ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 516ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 517ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); 518ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); 519ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); 520ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); 521ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 522ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 523ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 524ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 525ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); 526ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); 527ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); 528ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); 529ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 530ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res1 = _mm_packs_epi32(w0, w1); 531ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res7 = _mm_packs_epi32(w2, w3); 532ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res5 = _mm_packs_epi32(w4, w5); 533ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res3 = _mm_packs_epi32(w6, w7); 534ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 535ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Transpose the 8x8. 536ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 537ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 01 02 03 04 05 06 07 538ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 10 11 12 13 14 15 16 17 539ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 20 21 22 23 24 25 26 27 540ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 30 31 32 33 34 35 36 37 541ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 40 41 42 43 44 45 46 47 542ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 50 51 52 53 54 55 56 57 543ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 60 61 62 63 64 65 66 67 544ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 70 71 72 73 74 75 76 77 545ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1); 546ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3); 547ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1); 548ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3); 549ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5); 550ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7); 551ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5); 552ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7); 553ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 10 01 11 02 12 03 13 554ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 20 30 21 31 22 32 23 33 555ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 04 14 05 15 06 16 07 17 556ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 24 34 25 35 26 36 27 37 557ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 40 50 41 51 42 52 43 53 558ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 60 70 61 71 62 72 63 73 559ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 54 54 55 55 56 56 57 57 560ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 64 74 65 75 66 76 67 77 561ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); 562ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); 563ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); 564ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); 565ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); 566ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); 567ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); 568ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); 569ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 10 20 30 01 11 21 31 570ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 40 50 60 70 41 51 61 71 571ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 02 12 22 32 03 13 23 33 572ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 42 52 62 72 43 53 63 73 573ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 04 14 24 34 05 15 21 36 574ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 44 54 64 74 45 55 61 76 575ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 06 16 26 36 07 17 27 37 576ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 46 56 66 76 47 57 67 77 577ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in0 = _mm_unpacklo_epi64(tr1_0, tr1_4); 578ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in1 = _mm_unpackhi_epi64(tr1_0, tr1_4); 579ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in2 = _mm_unpacklo_epi64(tr1_2, tr1_6); 580ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in3 = _mm_unpackhi_epi64(tr1_2, tr1_6); 581ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in4 = _mm_unpacklo_epi64(tr1_1, tr1_5); 582ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in5 = _mm_unpackhi_epi64(tr1_1, tr1_5); 583ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in6 = _mm_unpacklo_epi64(tr1_3, tr1_7); 584ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in7 = _mm_unpackhi_epi64(tr1_3, tr1_7); 585ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 10 20 30 40 50 60 70 586ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 01 11 21 31 41 51 61 71 587ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 02 12 22 32 42 52 62 72 588ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 03 13 23 33 43 53 63 73 589ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 04 14 24 34 44 54 64 74 590ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 05 15 25 35 45 55 65 75 591ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 06 16 26 36 46 56 66 76 592ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 07 17 27 37 47 57 67 77 593ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 594ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 595ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Post-condition output and store it 596ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 597ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Post-condition (division by two) 598ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // division of two 16 bits signed numbers using shifts 599ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // n / 2 = (n - (n >> 15)) >> 1 600ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i sign_in0 = _mm_srai_epi16(in0, 15); 601ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i sign_in1 = _mm_srai_epi16(in1, 15); 602ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i sign_in2 = _mm_srai_epi16(in2, 15); 603ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i sign_in3 = _mm_srai_epi16(in3, 15); 604ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i sign_in4 = _mm_srai_epi16(in4, 15); 605ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i sign_in5 = _mm_srai_epi16(in5, 15); 606ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i sign_in6 = _mm_srai_epi16(in6, 15); 607ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i sign_in7 = _mm_srai_epi16(in7, 15); 608ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in0 = _mm_sub_epi16(in0, sign_in0); 609ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in1 = _mm_sub_epi16(in1, sign_in1); 610ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in2 = _mm_sub_epi16(in2, sign_in2); 611ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in3 = _mm_sub_epi16(in3, sign_in3); 612ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in4 = _mm_sub_epi16(in4, sign_in4); 613ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in5 = _mm_sub_epi16(in5, sign_in5); 614ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in6 = _mm_sub_epi16(in6, sign_in6); 615ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in7 = _mm_sub_epi16(in7, sign_in7); 616ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in0 = _mm_srai_epi16(in0, 1); 617ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in1 = _mm_srai_epi16(in1, 1); 618ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in2 = _mm_srai_epi16(in2, 1); 619ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in3 = _mm_srai_epi16(in3, 1); 620ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in4 = _mm_srai_epi16(in4, 1); 621ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in5 = _mm_srai_epi16(in5, 1); 622ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in6 = _mm_srai_epi16(in6, 1); 623ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in7 = _mm_srai_epi16(in7, 1); 624ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // store results 62591037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 0 * 8), in0); 62691037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 1 * 8), in1); 62791037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 2 * 8), in2); 62891037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 3 * 8), in3); 62991037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 4 * 8), in4); 63091037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 5 * 8), in5); 63191037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 6 * 8), in6); 63291037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 7 * 8), in7); 63391037db265ecdd914a26e056cf69207b4f50924ehkuang } 63491037db265ecdd914a26e056cf69207b4f50924ehkuang} 63591037db265ecdd914a26e056cf69207b4f50924ehkuang 63691037db265ecdd914a26e056cf69207b4f50924ehkuang// load 8x8 array 6375ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuangstatic INLINE void load_buffer_8x8(const int16_t *input, __m128i *in, 6385ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang int stride) { 6395ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang in[0] = _mm_load_si128((const __m128i *)(input + 0 * stride)); 6405ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang in[1] = _mm_load_si128((const __m128i *)(input + 1 * stride)); 6415ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang in[2] = _mm_load_si128((const __m128i *)(input + 2 * stride)); 6425ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang in[3] = _mm_load_si128((const __m128i *)(input + 3 * stride)); 6435ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang in[4] = _mm_load_si128((const __m128i *)(input + 4 * stride)); 6445ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang in[5] = _mm_load_si128((const __m128i *)(input + 5 * stride)); 6455ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang in[6] = _mm_load_si128((const __m128i *)(input + 6 * stride)); 6465ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang in[7] = _mm_load_si128((const __m128i *)(input + 7 * stride)); 64791037db265ecdd914a26e056cf69207b4f50924ehkuang 64891037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_slli_epi16(in[0], 2); 64991037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_slli_epi16(in[1], 2); 65091037db265ecdd914a26e056cf69207b4f50924ehkuang in[2] = _mm_slli_epi16(in[2], 2); 65191037db265ecdd914a26e056cf69207b4f50924ehkuang in[3] = _mm_slli_epi16(in[3], 2); 65291037db265ecdd914a26e056cf69207b4f50924ehkuang in[4] = _mm_slli_epi16(in[4], 2); 65391037db265ecdd914a26e056cf69207b4f50924ehkuang in[5] = _mm_slli_epi16(in[5], 2); 65491037db265ecdd914a26e056cf69207b4f50924ehkuang in[6] = _mm_slli_epi16(in[6], 2); 65591037db265ecdd914a26e056cf69207b4f50924ehkuang in[7] = _mm_slli_epi16(in[7], 2); 65691037db265ecdd914a26e056cf69207b4f50924ehkuang} 65791037db265ecdd914a26e056cf69207b4f50924ehkuang 65891037db265ecdd914a26e056cf69207b4f50924ehkuang// right shift and rounding 65991037db265ecdd914a26e056cf69207b4f50924ehkuangstatic INLINE void right_shift_8x8(__m128i *res, int const bit) { 66091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i kOne = _mm_set1_epi16(1); 66191037db265ecdd914a26e056cf69207b4f50924ehkuang const int bit_m02 = bit - 2; 66291037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i sign0 = _mm_srai_epi16(res[0], 15); 66391037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i sign1 = _mm_srai_epi16(res[1], 15); 66491037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i sign2 = _mm_srai_epi16(res[2], 15); 66591037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i sign3 = _mm_srai_epi16(res[3], 15); 66691037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i sign4 = _mm_srai_epi16(res[4], 15); 66791037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i sign5 = _mm_srai_epi16(res[5], 15); 66891037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i sign6 = _mm_srai_epi16(res[6], 15); 66991037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i sign7 = _mm_srai_epi16(res[7], 15); 67091037db265ecdd914a26e056cf69207b4f50924ehkuang 67191037db265ecdd914a26e056cf69207b4f50924ehkuang if (bit_m02 >= 0) { 67291037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i k_const_rounding = _mm_slli_epi16(kOne, bit_m02); 67391037db265ecdd914a26e056cf69207b4f50924ehkuang res[0] = _mm_add_epi16(res[0], k_const_rounding); 67491037db265ecdd914a26e056cf69207b4f50924ehkuang res[1] = _mm_add_epi16(res[1], k_const_rounding); 67591037db265ecdd914a26e056cf69207b4f50924ehkuang res[2] = _mm_add_epi16(res[2], k_const_rounding); 67691037db265ecdd914a26e056cf69207b4f50924ehkuang res[3] = _mm_add_epi16(res[3], k_const_rounding); 67791037db265ecdd914a26e056cf69207b4f50924ehkuang res[4] = _mm_add_epi16(res[4], k_const_rounding); 67891037db265ecdd914a26e056cf69207b4f50924ehkuang res[5] = _mm_add_epi16(res[5], k_const_rounding); 67991037db265ecdd914a26e056cf69207b4f50924ehkuang res[6] = _mm_add_epi16(res[6], k_const_rounding); 68091037db265ecdd914a26e056cf69207b4f50924ehkuang res[7] = _mm_add_epi16(res[7], k_const_rounding); 68191037db265ecdd914a26e056cf69207b4f50924ehkuang } 68291037db265ecdd914a26e056cf69207b4f50924ehkuang 68391037db265ecdd914a26e056cf69207b4f50924ehkuang res[0] = _mm_sub_epi16(res[0], sign0); 68491037db265ecdd914a26e056cf69207b4f50924ehkuang res[1] = _mm_sub_epi16(res[1], sign1); 68591037db265ecdd914a26e056cf69207b4f50924ehkuang res[2] = _mm_sub_epi16(res[2], sign2); 68691037db265ecdd914a26e056cf69207b4f50924ehkuang res[3] = _mm_sub_epi16(res[3], sign3); 68791037db265ecdd914a26e056cf69207b4f50924ehkuang res[4] = _mm_sub_epi16(res[4], sign4); 68891037db265ecdd914a26e056cf69207b4f50924ehkuang res[5] = _mm_sub_epi16(res[5], sign5); 68991037db265ecdd914a26e056cf69207b4f50924ehkuang res[6] = _mm_sub_epi16(res[6], sign6); 69091037db265ecdd914a26e056cf69207b4f50924ehkuang res[7] = _mm_sub_epi16(res[7], sign7); 69191037db265ecdd914a26e056cf69207b4f50924ehkuang 69291037db265ecdd914a26e056cf69207b4f50924ehkuang res[0] = _mm_srai_epi16(res[0], bit); 69391037db265ecdd914a26e056cf69207b4f50924ehkuang res[1] = _mm_srai_epi16(res[1], bit); 69491037db265ecdd914a26e056cf69207b4f50924ehkuang res[2] = _mm_srai_epi16(res[2], bit); 69591037db265ecdd914a26e056cf69207b4f50924ehkuang res[3] = _mm_srai_epi16(res[3], bit); 69691037db265ecdd914a26e056cf69207b4f50924ehkuang res[4] = _mm_srai_epi16(res[4], bit); 69791037db265ecdd914a26e056cf69207b4f50924ehkuang res[5] = _mm_srai_epi16(res[5], bit); 69891037db265ecdd914a26e056cf69207b4f50924ehkuang res[6] = _mm_srai_epi16(res[6], bit); 69991037db265ecdd914a26e056cf69207b4f50924ehkuang res[7] = _mm_srai_epi16(res[7], bit); 70091037db265ecdd914a26e056cf69207b4f50924ehkuang} 70191037db265ecdd914a26e056cf69207b4f50924ehkuang 70291037db265ecdd914a26e056cf69207b4f50924ehkuang// write 8x8 array 70391037db265ecdd914a26e056cf69207b4f50924ehkuangstatic INLINE void write_buffer_8x8(int16_t *output, __m128i *res, int stride) { 70491037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 0 * stride), res[0]); 70591037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 1 * stride), res[1]); 70691037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 2 * stride), res[2]); 70791037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 3 * stride), res[3]); 70891037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 4 * stride), res[4]); 70991037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 5 * stride), res[5]); 71091037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 6 * stride), res[6]); 71191037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(output + 7 * stride), res[7]); 71291037db265ecdd914a26e056cf69207b4f50924ehkuang} 71391037db265ecdd914a26e056cf69207b4f50924ehkuang 71491037db265ecdd914a26e056cf69207b4f50924ehkuang// perform in-place transpose 71591037db265ecdd914a26e056cf69207b4f50924ehkuangstatic INLINE void array_transpose_8x8(__m128i *in, __m128i *res) { 71691037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_0 = _mm_unpacklo_epi16(in[0], in[1]); 71791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_1 = _mm_unpacklo_epi16(in[2], in[3]); 71891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_2 = _mm_unpackhi_epi16(in[0], in[1]); 71991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_3 = _mm_unpackhi_epi16(in[2], in[3]); 72091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_4 = _mm_unpacklo_epi16(in[4], in[5]); 72191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_5 = _mm_unpacklo_epi16(in[6], in[7]); 72291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_6 = _mm_unpackhi_epi16(in[4], in[5]); 72391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr0_7 = _mm_unpackhi_epi16(in[6], in[7]); 72491037db265ecdd914a26e056cf69207b4f50924ehkuang // 00 10 01 11 02 12 03 13 72591037db265ecdd914a26e056cf69207b4f50924ehkuang // 20 30 21 31 22 32 23 33 72691037db265ecdd914a26e056cf69207b4f50924ehkuang // 04 14 05 15 06 16 07 17 72791037db265ecdd914a26e056cf69207b4f50924ehkuang // 24 34 25 35 26 36 27 37 72891037db265ecdd914a26e056cf69207b4f50924ehkuang // 40 50 41 51 42 52 43 53 72991037db265ecdd914a26e056cf69207b4f50924ehkuang // 60 70 61 71 62 72 63 73 73091037db265ecdd914a26e056cf69207b4f50924ehkuang // 44 54 45 55 46 56 47 57 73191037db265ecdd914a26e056cf69207b4f50924ehkuang // 64 74 65 75 66 76 67 77 73291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); 73391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_4, tr0_5); 73491037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); 73591037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_4, tr0_5); 73691037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_2, tr0_3); 73791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); 73891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_2, tr0_3); 73991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); 74091037db265ecdd914a26e056cf69207b4f50924ehkuang // 00 10 20 30 01 11 21 31 74191037db265ecdd914a26e056cf69207b4f50924ehkuang // 40 50 60 70 41 51 61 71 74291037db265ecdd914a26e056cf69207b4f50924ehkuang // 02 12 22 32 03 13 23 33 74391037db265ecdd914a26e056cf69207b4f50924ehkuang // 42 52 62 72 43 53 63 73 74491037db265ecdd914a26e056cf69207b4f50924ehkuang // 04 14 24 34 05 15 25 35 74591037db265ecdd914a26e056cf69207b4f50924ehkuang // 44 54 64 74 45 55 65 75 74691037db265ecdd914a26e056cf69207b4f50924ehkuang // 06 16 26 36 07 17 27 37 74791037db265ecdd914a26e056cf69207b4f50924ehkuang // 46 56 66 76 47 57 67 77 74891037db265ecdd914a26e056cf69207b4f50924ehkuang res[0] = _mm_unpacklo_epi64(tr1_0, tr1_1); 74991037db265ecdd914a26e056cf69207b4f50924ehkuang res[1] = _mm_unpackhi_epi64(tr1_0, tr1_1); 75091037db265ecdd914a26e056cf69207b4f50924ehkuang res[2] = _mm_unpacklo_epi64(tr1_2, tr1_3); 75191037db265ecdd914a26e056cf69207b4f50924ehkuang res[3] = _mm_unpackhi_epi64(tr1_2, tr1_3); 75291037db265ecdd914a26e056cf69207b4f50924ehkuang res[4] = _mm_unpacklo_epi64(tr1_4, tr1_5); 75391037db265ecdd914a26e056cf69207b4f50924ehkuang res[5] = _mm_unpackhi_epi64(tr1_4, tr1_5); 75491037db265ecdd914a26e056cf69207b4f50924ehkuang res[6] = _mm_unpacklo_epi64(tr1_6, tr1_7); 75591037db265ecdd914a26e056cf69207b4f50924ehkuang res[7] = _mm_unpackhi_epi64(tr1_6, tr1_7); 75691037db265ecdd914a26e056cf69207b4f50924ehkuang // 00 10 20 30 40 50 60 70 75791037db265ecdd914a26e056cf69207b4f50924ehkuang // 01 11 21 31 41 51 61 71 75891037db265ecdd914a26e056cf69207b4f50924ehkuang // 02 12 22 32 42 52 62 72 75991037db265ecdd914a26e056cf69207b4f50924ehkuang // 03 13 23 33 43 53 63 73 76091037db265ecdd914a26e056cf69207b4f50924ehkuang // 04 14 24 34 44 54 64 74 76191037db265ecdd914a26e056cf69207b4f50924ehkuang // 05 15 25 35 45 55 65 75 76291037db265ecdd914a26e056cf69207b4f50924ehkuang // 06 16 26 36 46 56 66 76 76391037db265ecdd914a26e056cf69207b4f50924ehkuang // 07 17 27 37 47 57 67 77 76491037db265ecdd914a26e056cf69207b4f50924ehkuang} 76591037db265ecdd914a26e056cf69207b4f50924ehkuang 766b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid fdct8_sse2(__m128i *in) { 76791037db265ecdd914a26e056cf69207b4f50924ehkuang // constants 76891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64); 76991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); 77091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); 77191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); 77291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); 77391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); 77491037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); 77591037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); 77691037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 77791037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i u0, u1, u2, u3, u4, u5, u6, u7; 77891037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i v0, v1, v2, v3, v4, v5, v6, v7; 77991037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i s0, s1, s2, s3, s4, s5, s6, s7; 78091037db265ecdd914a26e056cf69207b4f50924ehkuang 78191037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 1 78291037db265ecdd914a26e056cf69207b4f50924ehkuang s0 = _mm_add_epi16(in[0], in[7]); 78391037db265ecdd914a26e056cf69207b4f50924ehkuang s1 = _mm_add_epi16(in[1], in[6]); 78491037db265ecdd914a26e056cf69207b4f50924ehkuang s2 = _mm_add_epi16(in[2], in[5]); 78591037db265ecdd914a26e056cf69207b4f50924ehkuang s3 = _mm_add_epi16(in[3], in[4]); 78691037db265ecdd914a26e056cf69207b4f50924ehkuang s4 = _mm_sub_epi16(in[3], in[4]); 78791037db265ecdd914a26e056cf69207b4f50924ehkuang s5 = _mm_sub_epi16(in[2], in[5]); 78891037db265ecdd914a26e056cf69207b4f50924ehkuang s6 = _mm_sub_epi16(in[1], in[6]); 78991037db265ecdd914a26e056cf69207b4f50924ehkuang s7 = _mm_sub_epi16(in[0], in[7]); 79091037db265ecdd914a26e056cf69207b4f50924ehkuang 79191037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_add_epi16(s0, s3); 79291037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_add_epi16(s1, s2); 79391037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_sub_epi16(s1, s2); 79491037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_sub_epi16(s0, s3); 79591037db265ecdd914a26e056cf69207b4f50924ehkuang // interleave and perform butterfly multiplication/addition 79691037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_unpacklo_epi16(u0, u1); 79791037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_unpackhi_epi16(u0, u1); 79891037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_unpacklo_epi16(u2, u3); 79991037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_unpackhi_epi16(u2, u3); 80091037db265ecdd914a26e056cf69207b4f50924ehkuang 80191037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_madd_epi16(v0, k__cospi_p16_p16); 80291037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_madd_epi16(v1, k__cospi_p16_p16); 80391037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_madd_epi16(v0, k__cospi_p16_m16); 80491037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_madd_epi16(v1, k__cospi_p16_m16); 80591037db265ecdd914a26e056cf69207b4f50924ehkuang u4 = _mm_madd_epi16(v2, k__cospi_p24_p08); 80691037db265ecdd914a26e056cf69207b4f50924ehkuang u5 = _mm_madd_epi16(v3, k__cospi_p24_p08); 80791037db265ecdd914a26e056cf69207b4f50924ehkuang u6 = _mm_madd_epi16(v2, k__cospi_m08_p24); 80891037db265ecdd914a26e056cf69207b4f50924ehkuang u7 = _mm_madd_epi16(v3, k__cospi_m08_p24); 80991037db265ecdd914a26e056cf69207b4f50924ehkuang 81091037db265ecdd914a26e056cf69207b4f50924ehkuang // shift and rounding 81191037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 81291037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 81391037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 81491037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 81591037db265ecdd914a26e056cf69207b4f50924ehkuang v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); 81691037db265ecdd914a26e056cf69207b4f50924ehkuang v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); 81791037db265ecdd914a26e056cf69207b4f50924ehkuang v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); 81891037db265ecdd914a26e056cf69207b4f50924ehkuang v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); 81991037db265ecdd914a26e056cf69207b4f50924ehkuang 82091037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 82191037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 82291037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 82391037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 82491037db265ecdd914a26e056cf69207b4f50924ehkuang u4 = _mm_srai_epi32(v4, DCT_CONST_BITS); 82591037db265ecdd914a26e056cf69207b4f50924ehkuang u5 = _mm_srai_epi32(v5, DCT_CONST_BITS); 82691037db265ecdd914a26e056cf69207b4f50924ehkuang u6 = _mm_srai_epi32(v6, DCT_CONST_BITS); 82791037db265ecdd914a26e056cf69207b4f50924ehkuang u7 = _mm_srai_epi32(v7, DCT_CONST_BITS); 82891037db265ecdd914a26e056cf69207b4f50924ehkuang 82991037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_packs_epi32(u0, u1); 83091037db265ecdd914a26e056cf69207b4f50924ehkuang in[2] = _mm_packs_epi32(u4, u5); 83191037db265ecdd914a26e056cf69207b4f50924ehkuang in[4] = _mm_packs_epi32(u2, u3); 83291037db265ecdd914a26e056cf69207b4f50924ehkuang in[6] = _mm_packs_epi32(u6, u7); 83391037db265ecdd914a26e056cf69207b4f50924ehkuang 83491037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 2 83591037db265ecdd914a26e056cf69207b4f50924ehkuang // interleave and perform butterfly multiplication/addition 83691037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_unpacklo_epi16(s6, s5); 83791037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_unpackhi_epi16(s6, s5); 83891037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_madd_epi16(u0, k__cospi_p16_m16); 83991037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_madd_epi16(u1, k__cospi_p16_m16); 84091037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_madd_epi16(u0, k__cospi_p16_p16); 84191037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_madd_epi16(u1, k__cospi_p16_p16); 84291037db265ecdd914a26e056cf69207b4f50924ehkuang 84391037db265ecdd914a26e056cf69207b4f50924ehkuang // shift and rounding 84491037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING); 84591037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING); 84691037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING); 84791037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING); 84891037db265ecdd914a26e056cf69207b4f50924ehkuang 84991037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_srai_epi32(u0, DCT_CONST_BITS); 85091037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_srai_epi32(u1, DCT_CONST_BITS); 85191037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_srai_epi32(u2, DCT_CONST_BITS); 85291037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_srai_epi32(u3, DCT_CONST_BITS); 85391037db265ecdd914a26e056cf69207b4f50924ehkuang 85491037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_packs_epi32(v0, v1); 85591037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_packs_epi32(v2, v3); 85691037db265ecdd914a26e056cf69207b4f50924ehkuang 85791037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 3 85891037db265ecdd914a26e056cf69207b4f50924ehkuang s0 = _mm_add_epi16(s4, u0); 85991037db265ecdd914a26e056cf69207b4f50924ehkuang s1 = _mm_sub_epi16(s4, u0); 86091037db265ecdd914a26e056cf69207b4f50924ehkuang s2 = _mm_sub_epi16(s7, u1); 86191037db265ecdd914a26e056cf69207b4f50924ehkuang s3 = _mm_add_epi16(s7, u1); 86291037db265ecdd914a26e056cf69207b4f50924ehkuang 86391037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 4 86491037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_unpacklo_epi16(s0, s3); 86591037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_unpackhi_epi16(s0, s3); 86691037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_unpacklo_epi16(s1, s2); 86791037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_unpackhi_epi16(s1, s2); 86891037db265ecdd914a26e056cf69207b4f50924ehkuang 86991037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_madd_epi16(u0, k__cospi_p28_p04); 87091037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_madd_epi16(u1, k__cospi_p28_p04); 87191037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_madd_epi16(u2, k__cospi_p12_p20); 87291037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_madd_epi16(u3, k__cospi_p12_p20); 87391037db265ecdd914a26e056cf69207b4f50924ehkuang v4 = _mm_madd_epi16(u2, k__cospi_m20_p12); 87491037db265ecdd914a26e056cf69207b4f50924ehkuang v5 = _mm_madd_epi16(u3, k__cospi_m20_p12); 87591037db265ecdd914a26e056cf69207b4f50924ehkuang v6 = _mm_madd_epi16(u0, k__cospi_m04_p28); 87691037db265ecdd914a26e056cf69207b4f50924ehkuang v7 = _mm_madd_epi16(u1, k__cospi_m04_p28); 87791037db265ecdd914a26e056cf69207b4f50924ehkuang 87891037db265ecdd914a26e056cf69207b4f50924ehkuang // shift and rounding 87991037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING); 88091037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING); 88191037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING); 88291037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING); 88391037db265ecdd914a26e056cf69207b4f50924ehkuang u4 = _mm_add_epi32(v4, k__DCT_CONST_ROUNDING); 88491037db265ecdd914a26e056cf69207b4f50924ehkuang u5 = _mm_add_epi32(v5, k__DCT_CONST_ROUNDING); 88591037db265ecdd914a26e056cf69207b4f50924ehkuang u6 = _mm_add_epi32(v6, k__DCT_CONST_ROUNDING); 88691037db265ecdd914a26e056cf69207b4f50924ehkuang u7 = _mm_add_epi32(v7, k__DCT_CONST_ROUNDING); 88791037db265ecdd914a26e056cf69207b4f50924ehkuang 88891037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_srai_epi32(u0, DCT_CONST_BITS); 88991037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_srai_epi32(u1, DCT_CONST_BITS); 89091037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_srai_epi32(u2, DCT_CONST_BITS); 89191037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_srai_epi32(u3, DCT_CONST_BITS); 89291037db265ecdd914a26e056cf69207b4f50924ehkuang v4 = _mm_srai_epi32(u4, DCT_CONST_BITS); 89391037db265ecdd914a26e056cf69207b4f50924ehkuang v5 = _mm_srai_epi32(u5, DCT_CONST_BITS); 89491037db265ecdd914a26e056cf69207b4f50924ehkuang v6 = _mm_srai_epi32(u6, DCT_CONST_BITS); 89591037db265ecdd914a26e056cf69207b4f50924ehkuang v7 = _mm_srai_epi32(u7, DCT_CONST_BITS); 89691037db265ecdd914a26e056cf69207b4f50924ehkuang 89791037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_packs_epi32(v0, v1); 89891037db265ecdd914a26e056cf69207b4f50924ehkuang in[3] = _mm_packs_epi32(v4, v5); 89991037db265ecdd914a26e056cf69207b4f50924ehkuang in[5] = _mm_packs_epi32(v2, v3); 90091037db265ecdd914a26e056cf69207b4f50924ehkuang in[7] = _mm_packs_epi32(v6, v7); 90191037db265ecdd914a26e056cf69207b4f50924ehkuang 90291037db265ecdd914a26e056cf69207b4f50924ehkuang // transpose 90391037db265ecdd914a26e056cf69207b4f50924ehkuang array_transpose_8x8(in, in); 90491037db265ecdd914a26e056cf69207b4f50924ehkuang} 90591037db265ecdd914a26e056cf69207b4f50924ehkuang 906b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid fadst8_sse2(__m128i *in) { 90791037db265ecdd914a26e056cf69207b4f50924ehkuang // Constants 90891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64); 90991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64); 91091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p10_p22 = pair_set_epi16(cospi_10_64, cospi_22_64); 91191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p22_m10 = pair_set_epi16(cospi_22_64, -cospi_10_64); 91291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p18_p14 = pair_set_epi16(cospi_18_64, cospi_14_64); 91391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p14_m18 = pair_set_epi16(cospi_14_64, -cospi_18_64); 91491037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64); 91591037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64); 91691037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); 91791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); 91891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m24_p08 = pair_set_epi16(-cospi_24_64, cospi_8_64); 91991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); 92091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64); 92191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__const_0 = _mm_set1_epi16(0); 92291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 92391037db265ecdd914a26e056cf69207b4f50924ehkuang 92491037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i u0, u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, u14, u15; 92591037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15; 92691037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; 92791037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i s0, s1, s2, s3, s4, s5, s6, s7; 92891037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in0, in1, in2, in3, in4, in5, in6, in7; 92991037db265ecdd914a26e056cf69207b4f50924ehkuang 93091037db265ecdd914a26e056cf69207b4f50924ehkuang // properly aligned for butterfly input 93191037db265ecdd914a26e056cf69207b4f50924ehkuang in0 = in[7]; 93291037db265ecdd914a26e056cf69207b4f50924ehkuang in1 = in[0]; 93391037db265ecdd914a26e056cf69207b4f50924ehkuang in2 = in[5]; 93491037db265ecdd914a26e056cf69207b4f50924ehkuang in3 = in[2]; 93591037db265ecdd914a26e056cf69207b4f50924ehkuang in4 = in[3]; 93691037db265ecdd914a26e056cf69207b4f50924ehkuang in5 = in[4]; 93791037db265ecdd914a26e056cf69207b4f50924ehkuang in6 = in[1]; 93891037db265ecdd914a26e056cf69207b4f50924ehkuang in7 = in[6]; 93991037db265ecdd914a26e056cf69207b4f50924ehkuang 94091037db265ecdd914a26e056cf69207b4f50924ehkuang // column transformation 94191037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 1 94291037db265ecdd914a26e056cf69207b4f50924ehkuang // interleave and multiply/add into 32-bit integer 94391037db265ecdd914a26e056cf69207b4f50924ehkuang s0 = _mm_unpacklo_epi16(in0, in1); 94491037db265ecdd914a26e056cf69207b4f50924ehkuang s1 = _mm_unpackhi_epi16(in0, in1); 94591037db265ecdd914a26e056cf69207b4f50924ehkuang s2 = _mm_unpacklo_epi16(in2, in3); 94691037db265ecdd914a26e056cf69207b4f50924ehkuang s3 = _mm_unpackhi_epi16(in2, in3); 94791037db265ecdd914a26e056cf69207b4f50924ehkuang s4 = _mm_unpacklo_epi16(in4, in5); 94891037db265ecdd914a26e056cf69207b4f50924ehkuang s5 = _mm_unpackhi_epi16(in4, in5); 94991037db265ecdd914a26e056cf69207b4f50924ehkuang s6 = _mm_unpacklo_epi16(in6, in7); 95091037db265ecdd914a26e056cf69207b4f50924ehkuang s7 = _mm_unpackhi_epi16(in6, in7); 95191037db265ecdd914a26e056cf69207b4f50924ehkuang 95291037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_madd_epi16(s0, k__cospi_p02_p30); 95391037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_madd_epi16(s1, k__cospi_p02_p30); 95491037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_madd_epi16(s0, k__cospi_p30_m02); 95591037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_madd_epi16(s1, k__cospi_p30_m02); 95691037db265ecdd914a26e056cf69207b4f50924ehkuang u4 = _mm_madd_epi16(s2, k__cospi_p10_p22); 95791037db265ecdd914a26e056cf69207b4f50924ehkuang u5 = _mm_madd_epi16(s3, k__cospi_p10_p22); 95891037db265ecdd914a26e056cf69207b4f50924ehkuang u6 = _mm_madd_epi16(s2, k__cospi_p22_m10); 95991037db265ecdd914a26e056cf69207b4f50924ehkuang u7 = _mm_madd_epi16(s3, k__cospi_p22_m10); 96091037db265ecdd914a26e056cf69207b4f50924ehkuang u8 = _mm_madd_epi16(s4, k__cospi_p18_p14); 96191037db265ecdd914a26e056cf69207b4f50924ehkuang u9 = _mm_madd_epi16(s5, k__cospi_p18_p14); 96291037db265ecdd914a26e056cf69207b4f50924ehkuang u10 = _mm_madd_epi16(s4, k__cospi_p14_m18); 96391037db265ecdd914a26e056cf69207b4f50924ehkuang u11 = _mm_madd_epi16(s5, k__cospi_p14_m18); 96491037db265ecdd914a26e056cf69207b4f50924ehkuang u12 = _mm_madd_epi16(s6, k__cospi_p26_p06); 96591037db265ecdd914a26e056cf69207b4f50924ehkuang u13 = _mm_madd_epi16(s7, k__cospi_p26_p06); 96691037db265ecdd914a26e056cf69207b4f50924ehkuang u14 = _mm_madd_epi16(s6, k__cospi_p06_m26); 96791037db265ecdd914a26e056cf69207b4f50924ehkuang u15 = _mm_madd_epi16(s7, k__cospi_p06_m26); 96891037db265ecdd914a26e056cf69207b4f50924ehkuang 96991037db265ecdd914a26e056cf69207b4f50924ehkuang // addition 97091037db265ecdd914a26e056cf69207b4f50924ehkuang w0 = _mm_add_epi32(u0, u8); 97191037db265ecdd914a26e056cf69207b4f50924ehkuang w1 = _mm_add_epi32(u1, u9); 97291037db265ecdd914a26e056cf69207b4f50924ehkuang w2 = _mm_add_epi32(u2, u10); 97391037db265ecdd914a26e056cf69207b4f50924ehkuang w3 = _mm_add_epi32(u3, u11); 97491037db265ecdd914a26e056cf69207b4f50924ehkuang w4 = _mm_add_epi32(u4, u12); 97591037db265ecdd914a26e056cf69207b4f50924ehkuang w5 = _mm_add_epi32(u5, u13); 97691037db265ecdd914a26e056cf69207b4f50924ehkuang w6 = _mm_add_epi32(u6, u14); 97791037db265ecdd914a26e056cf69207b4f50924ehkuang w7 = _mm_add_epi32(u7, u15); 97891037db265ecdd914a26e056cf69207b4f50924ehkuang w8 = _mm_sub_epi32(u0, u8); 97991037db265ecdd914a26e056cf69207b4f50924ehkuang w9 = _mm_sub_epi32(u1, u9); 98091037db265ecdd914a26e056cf69207b4f50924ehkuang w10 = _mm_sub_epi32(u2, u10); 98191037db265ecdd914a26e056cf69207b4f50924ehkuang w11 = _mm_sub_epi32(u3, u11); 98291037db265ecdd914a26e056cf69207b4f50924ehkuang w12 = _mm_sub_epi32(u4, u12); 98391037db265ecdd914a26e056cf69207b4f50924ehkuang w13 = _mm_sub_epi32(u5, u13); 98491037db265ecdd914a26e056cf69207b4f50924ehkuang w14 = _mm_sub_epi32(u6, u14); 98591037db265ecdd914a26e056cf69207b4f50924ehkuang w15 = _mm_sub_epi32(u7, u15); 98691037db265ecdd914a26e056cf69207b4f50924ehkuang 98791037db265ecdd914a26e056cf69207b4f50924ehkuang // shift and rounding 98891037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_add_epi32(w0, k__DCT_CONST_ROUNDING); 98991037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_add_epi32(w1, k__DCT_CONST_ROUNDING); 99091037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_add_epi32(w2, k__DCT_CONST_ROUNDING); 99191037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_add_epi32(w3, k__DCT_CONST_ROUNDING); 99291037db265ecdd914a26e056cf69207b4f50924ehkuang v4 = _mm_add_epi32(w4, k__DCT_CONST_ROUNDING); 99391037db265ecdd914a26e056cf69207b4f50924ehkuang v5 = _mm_add_epi32(w5, k__DCT_CONST_ROUNDING); 99491037db265ecdd914a26e056cf69207b4f50924ehkuang v6 = _mm_add_epi32(w6, k__DCT_CONST_ROUNDING); 99591037db265ecdd914a26e056cf69207b4f50924ehkuang v7 = _mm_add_epi32(w7, k__DCT_CONST_ROUNDING); 99691037db265ecdd914a26e056cf69207b4f50924ehkuang v8 = _mm_add_epi32(w8, k__DCT_CONST_ROUNDING); 99791037db265ecdd914a26e056cf69207b4f50924ehkuang v9 = _mm_add_epi32(w9, k__DCT_CONST_ROUNDING); 99891037db265ecdd914a26e056cf69207b4f50924ehkuang v10 = _mm_add_epi32(w10, k__DCT_CONST_ROUNDING); 99991037db265ecdd914a26e056cf69207b4f50924ehkuang v11 = _mm_add_epi32(w11, k__DCT_CONST_ROUNDING); 100091037db265ecdd914a26e056cf69207b4f50924ehkuang v12 = _mm_add_epi32(w12, k__DCT_CONST_ROUNDING); 100191037db265ecdd914a26e056cf69207b4f50924ehkuang v13 = _mm_add_epi32(w13, k__DCT_CONST_ROUNDING); 100291037db265ecdd914a26e056cf69207b4f50924ehkuang v14 = _mm_add_epi32(w14, k__DCT_CONST_ROUNDING); 100391037db265ecdd914a26e056cf69207b4f50924ehkuang v15 = _mm_add_epi32(w15, k__DCT_CONST_ROUNDING); 100491037db265ecdd914a26e056cf69207b4f50924ehkuang 100591037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 100691037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 100791037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 100891037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 100991037db265ecdd914a26e056cf69207b4f50924ehkuang u4 = _mm_srai_epi32(v4, DCT_CONST_BITS); 101091037db265ecdd914a26e056cf69207b4f50924ehkuang u5 = _mm_srai_epi32(v5, DCT_CONST_BITS); 101191037db265ecdd914a26e056cf69207b4f50924ehkuang u6 = _mm_srai_epi32(v6, DCT_CONST_BITS); 101291037db265ecdd914a26e056cf69207b4f50924ehkuang u7 = _mm_srai_epi32(v7, DCT_CONST_BITS); 101391037db265ecdd914a26e056cf69207b4f50924ehkuang u8 = _mm_srai_epi32(v8, DCT_CONST_BITS); 101491037db265ecdd914a26e056cf69207b4f50924ehkuang u9 = _mm_srai_epi32(v9, DCT_CONST_BITS); 101591037db265ecdd914a26e056cf69207b4f50924ehkuang u10 = _mm_srai_epi32(v10, DCT_CONST_BITS); 101691037db265ecdd914a26e056cf69207b4f50924ehkuang u11 = _mm_srai_epi32(v11, DCT_CONST_BITS); 101791037db265ecdd914a26e056cf69207b4f50924ehkuang u12 = _mm_srai_epi32(v12, DCT_CONST_BITS); 101891037db265ecdd914a26e056cf69207b4f50924ehkuang u13 = _mm_srai_epi32(v13, DCT_CONST_BITS); 101991037db265ecdd914a26e056cf69207b4f50924ehkuang u14 = _mm_srai_epi32(v14, DCT_CONST_BITS); 102091037db265ecdd914a26e056cf69207b4f50924ehkuang u15 = _mm_srai_epi32(v15, DCT_CONST_BITS); 102191037db265ecdd914a26e056cf69207b4f50924ehkuang 102291037db265ecdd914a26e056cf69207b4f50924ehkuang // back to 16-bit and pack 8 integers into __m128i 102391037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_packs_epi32(u0, u1); 102491037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_packs_epi32(u2, u3); 102591037db265ecdd914a26e056cf69207b4f50924ehkuang in[2] = _mm_packs_epi32(u4, u5); 102691037db265ecdd914a26e056cf69207b4f50924ehkuang in[3] = _mm_packs_epi32(u6, u7); 102791037db265ecdd914a26e056cf69207b4f50924ehkuang in[4] = _mm_packs_epi32(u8, u9); 102891037db265ecdd914a26e056cf69207b4f50924ehkuang in[5] = _mm_packs_epi32(u10, u11); 102991037db265ecdd914a26e056cf69207b4f50924ehkuang in[6] = _mm_packs_epi32(u12, u13); 103091037db265ecdd914a26e056cf69207b4f50924ehkuang in[7] = _mm_packs_epi32(u14, u15); 103191037db265ecdd914a26e056cf69207b4f50924ehkuang 103291037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 2 103391037db265ecdd914a26e056cf69207b4f50924ehkuang s0 = _mm_add_epi16(in[0], in[2]); 103491037db265ecdd914a26e056cf69207b4f50924ehkuang s1 = _mm_add_epi16(in[1], in[3]); 103591037db265ecdd914a26e056cf69207b4f50924ehkuang s2 = _mm_sub_epi16(in[0], in[2]); 103691037db265ecdd914a26e056cf69207b4f50924ehkuang s3 = _mm_sub_epi16(in[1], in[3]); 103791037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_unpacklo_epi16(in[4], in[5]); 103891037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_unpackhi_epi16(in[4], in[5]); 103991037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_unpacklo_epi16(in[6], in[7]); 104091037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_unpackhi_epi16(in[6], in[7]); 104191037db265ecdd914a26e056cf69207b4f50924ehkuang 104291037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_madd_epi16(u0, k__cospi_p08_p24); 104391037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_madd_epi16(u1, k__cospi_p08_p24); 104491037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_madd_epi16(u0, k__cospi_p24_m08); 104591037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_madd_epi16(u1, k__cospi_p24_m08); 104691037db265ecdd914a26e056cf69207b4f50924ehkuang v4 = _mm_madd_epi16(u2, k__cospi_m24_p08); 104791037db265ecdd914a26e056cf69207b4f50924ehkuang v5 = _mm_madd_epi16(u3, k__cospi_m24_p08); 104891037db265ecdd914a26e056cf69207b4f50924ehkuang v6 = _mm_madd_epi16(u2, k__cospi_p08_p24); 104991037db265ecdd914a26e056cf69207b4f50924ehkuang v7 = _mm_madd_epi16(u3, k__cospi_p08_p24); 105091037db265ecdd914a26e056cf69207b4f50924ehkuang 105191037db265ecdd914a26e056cf69207b4f50924ehkuang w0 = _mm_add_epi32(v0, v4); 105291037db265ecdd914a26e056cf69207b4f50924ehkuang w1 = _mm_add_epi32(v1, v5); 105391037db265ecdd914a26e056cf69207b4f50924ehkuang w2 = _mm_add_epi32(v2, v6); 105491037db265ecdd914a26e056cf69207b4f50924ehkuang w3 = _mm_add_epi32(v3, v7); 105591037db265ecdd914a26e056cf69207b4f50924ehkuang w4 = _mm_sub_epi32(v0, v4); 105691037db265ecdd914a26e056cf69207b4f50924ehkuang w5 = _mm_sub_epi32(v1, v5); 105791037db265ecdd914a26e056cf69207b4f50924ehkuang w6 = _mm_sub_epi32(v2, v6); 105891037db265ecdd914a26e056cf69207b4f50924ehkuang w7 = _mm_sub_epi32(v3, v7); 105991037db265ecdd914a26e056cf69207b4f50924ehkuang 106091037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_add_epi32(w0, k__DCT_CONST_ROUNDING); 106191037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_add_epi32(w1, k__DCT_CONST_ROUNDING); 106291037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_add_epi32(w2, k__DCT_CONST_ROUNDING); 106391037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_add_epi32(w3, k__DCT_CONST_ROUNDING); 106491037db265ecdd914a26e056cf69207b4f50924ehkuang v4 = _mm_add_epi32(w4, k__DCT_CONST_ROUNDING); 106591037db265ecdd914a26e056cf69207b4f50924ehkuang v5 = _mm_add_epi32(w5, k__DCT_CONST_ROUNDING); 106691037db265ecdd914a26e056cf69207b4f50924ehkuang v6 = _mm_add_epi32(w6, k__DCT_CONST_ROUNDING); 106791037db265ecdd914a26e056cf69207b4f50924ehkuang v7 = _mm_add_epi32(w7, k__DCT_CONST_ROUNDING); 106891037db265ecdd914a26e056cf69207b4f50924ehkuang 106991037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 107091037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 107191037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 107291037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 107391037db265ecdd914a26e056cf69207b4f50924ehkuang u4 = _mm_srai_epi32(v4, DCT_CONST_BITS); 107491037db265ecdd914a26e056cf69207b4f50924ehkuang u5 = _mm_srai_epi32(v5, DCT_CONST_BITS); 107591037db265ecdd914a26e056cf69207b4f50924ehkuang u6 = _mm_srai_epi32(v6, DCT_CONST_BITS); 107691037db265ecdd914a26e056cf69207b4f50924ehkuang u7 = _mm_srai_epi32(v7, DCT_CONST_BITS); 107791037db265ecdd914a26e056cf69207b4f50924ehkuang 107891037db265ecdd914a26e056cf69207b4f50924ehkuang // back to 16-bit intergers 107991037db265ecdd914a26e056cf69207b4f50924ehkuang s4 = _mm_packs_epi32(u0, u1); 108091037db265ecdd914a26e056cf69207b4f50924ehkuang s5 = _mm_packs_epi32(u2, u3); 108191037db265ecdd914a26e056cf69207b4f50924ehkuang s6 = _mm_packs_epi32(u4, u5); 108291037db265ecdd914a26e056cf69207b4f50924ehkuang s7 = _mm_packs_epi32(u6, u7); 108391037db265ecdd914a26e056cf69207b4f50924ehkuang 108491037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 3 108591037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_unpacklo_epi16(s2, s3); 108691037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_unpackhi_epi16(s2, s3); 108791037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_unpacklo_epi16(s6, s7); 108891037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_unpackhi_epi16(s6, s7); 108991037db265ecdd914a26e056cf69207b4f50924ehkuang 109091037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_madd_epi16(u0, k__cospi_p16_p16); 109191037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_madd_epi16(u1, k__cospi_p16_p16); 109291037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_madd_epi16(u0, k__cospi_p16_m16); 109391037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_madd_epi16(u1, k__cospi_p16_m16); 109491037db265ecdd914a26e056cf69207b4f50924ehkuang v4 = _mm_madd_epi16(u2, k__cospi_p16_p16); 109591037db265ecdd914a26e056cf69207b4f50924ehkuang v5 = _mm_madd_epi16(u3, k__cospi_p16_p16); 109691037db265ecdd914a26e056cf69207b4f50924ehkuang v6 = _mm_madd_epi16(u2, k__cospi_p16_m16); 109791037db265ecdd914a26e056cf69207b4f50924ehkuang v7 = _mm_madd_epi16(u3, k__cospi_p16_m16); 109891037db265ecdd914a26e056cf69207b4f50924ehkuang 109991037db265ecdd914a26e056cf69207b4f50924ehkuang u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING); 110091037db265ecdd914a26e056cf69207b4f50924ehkuang u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING); 110191037db265ecdd914a26e056cf69207b4f50924ehkuang u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING); 110291037db265ecdd914a26e056cf69207b4f50924ehkuang u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING); 110391037db265ecdd914a26e056cf69207b4f50924ehkuang u4 = _mm_add_epi32(v4, k__DCT_CONST_ROUNDING); 110491037db265ecdd914a26e056cf69207b4f50924ehkuang u5 = _mm_add_epi32(v5, k__DCT_CONST_ROUNDING); 110591037db265ecdd914a26e056cf69207b4f50924ehkuang u6 = _mm_add_epi32(v6, k__DCT_CONST_ROUNDING); 110691037db265ecdd914a26e056cf69207b4f50924ehkuang u7 = _mm_add_epi32(v7, k__DCT_CONST_ROUNDING); 110791037db265ecdd914a26e056cf69207b4f50924ehkuang 110891037db265ecdd914a26e056cf69207b4f50924ehkuang v0 = _mm_srai_epi32(u0, DCT_CONST_BITS); 110991037db265ecdd914a26e056cf69207b4f50924ehkuang v1 = _mm_srai_epi32(u1, DCT_CONST_BITS); 111091037db265ecdd914a26e056cf69207b4f50924ehkuang v2 = _mm_srai_epi32(u2, DCT_CONST_BITS); 111191037db265ecdd914a26e056cf69207b4f50924ehkuang v3 = _mm_srai_epi32(u3, DCT_CONST_BITS); 111291037db265ecdd914a26e056cf69207b4f50924ehkuang v4 = _mm_srai_epi32(u4, DCT_CONST_BITS); 111391037db265ecdd914a26e056cf69207b4f50924ehkuang v5 = _mm_srai_epi32(u5, DCT_CONST_BITS); 111491037db265ecdd914a26e056cf69207b4f50924ehkuang v6 = _mm_srai_epi32(u6, DCT_CONST_BITS); 111591037db265ecdd914a26e056cf69207b4f50924ehkuang v7 = _mm_srai_epi32(u7, DCT_CONST_BITS); 111691037db265ecdd914a26e056cf69207b4f50924ehkuang 111791037db265ecdd914a26e056cf69207b4f50924ehkuang s2 = _mm_packs_epi32(v0, v1); 111891037db265ecdd914a26e056cf69207b4f50924ehkuang s3 = _mm_packs_epi32(v2, v3); 111991037db265ecdd914a26e056cf69207b4f50924ehkuang s6 = _mm_packs_epi32(v4, v5); 112091037db265ecdd914a26e056cf69207b4f50924ehkuang s7 = _mm_packs_epi32(v6, v7); 112191037db265ecdd914a26e056cf69207b4f50924ehkuang 112291037db265ecdd914a26e056cf69207b4f50924ehkuang // FIXME(jingning): do subtract using bit inversion? 112391037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = s0; 112491037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_sub_epi16(k__const_0, s4); 112591037db265ecdd914a26e056cf69207b4f50924ehkuang in[2] = s6; 112691037db265ecdd914a26e056cf69207b4f50924ehkuang in[3] = _mm_sub_epi16(k__const_0, s2); 112791037db265ecdd914a26e056cf69207b4f50924ehkuang in[4] = s3; 112891037db265ecdd914a26e056cf69207b4f50924ehkuang in[5] = _mm_sub_epi16(k__const_0, s7); 112991037db265ecdd914a26e056cf69207b4f50924ehkuang in[6] = s5; 113091037db265ecdd914a26e056cf69207b4f50924ehkuang in[7] = _mm_sub_epi16(k__const_0, s1); 113191037db265ecdd914a26e056cf69207b4f50924ehkuang 113291037db265ecdd914a26e056cf69207b4f50924ehkuang // transpose 113391037db265ecdd914a26e056cf69207b4f50924ehkuang array_transpose_8x8(in, in); 113491037db265ecdd914a26e056cf69207b4f50924ehkuang} 113591037db265ecdd914a26e056cf69207b4f50924ehkuang 1136b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid vp9_fht8x8_sse2(const int16_t *input, int16_t *output, 1137b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian int stride, int tx_type) { 113891037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in[8]; 1139b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 114091037db265ecdd914a26e056cf69207b4f50924ehkuang switch (tx_type) { 1141b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case DCT_DCT: 1142b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian vp9_fdct8x8_sse2(input, output, stride); 114391037db265ecdd914a26e056cf69207b4f50924ehkuang break; 1144b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case ADST_DCT: 1145b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian load_buffer_8x8(input, in, stride); 1146b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst8_sse2(in); 1147b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fdct8_sse2(in); 1148b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian right_shift_8x8(in, 1); 1149b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian write_buffer_8x8(output, in, 8); 115091037db265ecdd914a26e056cf69207b4f50924ehkuang break; 1151b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case DCT_ADST: 1152b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian load_buffer_8x8(input, in, stride); 1153b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fdct8_sse2(in); 1154b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst8_sse2(in); 1155b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian right_shift_8x8(in, 1); 1156b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian write_buffer_8x8(output, in, 8); 115791037db265ecdd914a26e056cf69207b4f50924ehkuang break; 1158b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case ADST_ADST: 1159b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian load_buffer_8x8(input, in, stride); 1160b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst8_sse2(in); 1161b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst8_sse2(in); 1162b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian right_shift_8x8(in, 1); 1163b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian write_buffer_8x8(output, in, 8); 116491037db265ecdd914a26e056cf69207b4f50924ehkuang break; 116591037db265ecdd914a26e056cf69207b4f50924ehkuang default: 116691037db265ecdd914a26e056cf69207b4f50924ehkuang assert(0); 116791037db265ecdd914a26e056cf69207b4f50924ehkuang break; 1168ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1169ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 1170ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang 11715ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuangvoid vp9_fdct16x16_sse2(const int16_t *input, int16_t *output, int stride) { 1172ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // The 2D transform is done with two passes which are actually pretty 1173ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // similar. In the first one, we transform the columns and transpose 1174ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // the results. In the second one, we transform the rows. To achieve that, 1175b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // as the first pass results are transposed, we transpose the columns (that 1176ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // is the transposed rows) and transpose the results (so that it goes back 1177ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // in normal/row positions). 1178ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang int pass; 1179ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // We need an intermediate buffer between passes. 118091037db265ecdd914a26e056cf69207b4f50924ehkuang DECLARE_ALIGNED_ARRAY(16, int16_t, intermediate, 256); 11815ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang const int16_t *in = input; 1182ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang int16_t *out = intermediate; 1183ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Constants 1184ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // When we use them, in one case, they are all the same. In all others 1185ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // it's a pair of them that we need to repeat four times. This is done 1186ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // by constructing the 32 bit constant corresponding to that pair. 1187ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64); 1188ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); 1189ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); 1190ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64); 1191ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); 1192ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); 1193ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); 1194ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); 1195ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); 1196ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p30_p02 = pair_set_epi16(cospi_30_64, cospi_2_64); 1197ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p14_p18 = pair_set_epi16(cospi_14_64, cospi_18_64); 1198ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64); 1199ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64); 1200ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p22_p10 = pair_set_epi16(cospi_22_64, cospi_10_64); 1201ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_p06_p26 = pair_set_epi16(cospi_6_64, cospi_26_64); 1202ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m10_p22 = pair_set_epi16(-cospi_10_64, cospi_22_64); 1203ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__cospi_m26_p06 = pair_set_epi16(-cospi_26_64, cospi_6_64); 1204ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 1205ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i kOne = _mm_set1_epi16(1); 1206ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Do the two transform/transpose passes 1207ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang for (pass = 0; pass < 2; ++pass) { 1208ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // We process eight columns (transposed rows in second pass) at a time. 1209ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang int column_start; 1210ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang for (column_start = 0; column_start < 16; column_start += 8) { 1211ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i in00, in01, in02, in03, in04, in05, in06, in07; 1212ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i in08, in09, in10, in11, in12, in13, in14, in15; 1213ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i input0, input1, input2, input3, input4, input5, input6, input7; 1214ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i step1_0, step1_1, step1_2, step1_3; 1215ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i step1_4, step1_5, step1_6, step1_7; 1216ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i step2_1, step2_2, step2_3, step2_4, step2_5, step2_6; 1217ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i step3_0, step3_1, step3_2, step3_3; 1218ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i step3_4, step3_5, step3_6, step3_7; 1219ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i res00, res01, res02, res03, res04, res05, res06, res07; 1220ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang __m128i res08, res09, res10, res11, res12, res13, res14, res15; 1221ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Load and pre-condition input. 1222ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang if (0 == pass) { 122391037db265ecdd914a26e056cf69207b4f50924ehkuang in00 = _mm_load_si128((const __m128i *)(in + 0 * stride)); 122491037db265ecdd914a26e056cf69207b4f50924ehkuang in01 = _mm_load_si128((const __m128i *)(in + 1 * stride)); 122591037db265ecdd914a26e056cf69207b4f50924ehkuang in02 = _mm_load_si128((const __m128i *)(in + 2 * stride)); 122691037db265ecdd914a26e056cf69207b4f50924ehkuang in03 = _mm_load_si128((const __m128i *)(in + 3 * stride)); 122791037db265ecdd914a26e056cf69207b4f50924ehkuang in04 = _mm_load_si128((const __m128i *)(in + 4 * stride)); 122891037db265ecdd914a26e056cf69207b4f50924ehkuang in05 = _mm_load_si128((const __m128i *)(in + 5 * stride)); 122991037db265ecdd914a26e056cf69207b4f50924ehkuang in06 = _mm_load_si128((const __m128i *)(in + 6 * stride)); 123091037db265ecdd914a26e056cf69207b4f50924ehkuang in07 = _mm_load_si128((const __m128i *)(in + 7 * stride)); 123191037db265ecdd914a26e056cf69207b4f50924ehkuang in08 = _mm_load_si128((const __m128i *)(in + 8 * stride)); 123291037db265ecdd914a26e056cf69207b4f50924ehkuang in09 = _mm_load_si128((const __m128i *)(in + 9 * stride)); 123391037db265ecdd914a26e056cf69207b4f50924ehkuang in10 = _mm_load_si128((const __m128i *)(in + 10 * stride)); 123491037db265ecdd914a26e056cf69207b4f50924ehkuang in11 = _mm_load_si128((const __m128i *)(in + 11 * stride)); 123591037db265ecdd914a26e056cf69207b4f50924ehkuang in12 = _mm_load_si128((const __m128i *)(in + 12 * stride)); 123691037db265ecdd914a26e056cf69207b4f50924ehkuang in13 = _mm_load_si128((const __m128i *)(in + 13 * stride)); 123791037db265ecdd914a26e056cf69207b4f50924ehkuang in14 = _mm_load_si128((const __m128i *)(in + 14 * stride)); 123891037db265ecdd914a26e056cf69207b4f50924ehkuang in15 = _mm_load_si128((const __m128i *)(in + 15 * stride)); 1239ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // x = x << 2 1240ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in00 = _mm_slli_epi16(in00, 2); 1241ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in01 = _mm_slli_epi16(in01, 2); 1242ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in02 = _mm_slli_epi16(in02, 2); 1243ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in03 = _mm_slli_epi16(in03, 2); 1244ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in04 = _mm_slli_epi16(in04, 2); 1245ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in05 = _mm_slli_epi16(in05, 2); 1246ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in06 = _mm_slli_epi16(in06, 2); 1247ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in07 = _mm_slli_epi16(in07, 2); 1248ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in08 = _mm_slli_epi16(in08, 2); 1249ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in09 = _mm_slli_epi16(in09, 2); 1250ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in10 = _mm_slli_epi16(in10, 2); 1251ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in11 = _mm_slli_epi16(in11, 2); 1252ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in12 = _mm_slli_epi16(in12, 2); 1253ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in13 = _mm_slli_epi16(in13, 2); 1254ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in14 = _mm_slli_epi16(in14, 2); 1255ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in15 = _mm_slli_epi16(in15, 2); 1256ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } else { 125791037db265ecdd914a26e056cf69207b4f50924ehkuang in00 = _mm_load_si128((const __m128i *)(in + 0 * 16)); 125891037db265ecdd914a26e056cf69207b4f50924ehkuang in01 = _mm_load_si128((const __m128i *)(in + 1 * 16)); 125991037db265ecdd914a26e056cf69207b4f50924ehkuang in02 = _mm_load_si128((const __m128i *)(in + 2 * 16)); 126091037db265ecdd914a26e056cf69207b4f50924ehkuang in03 = _mm_load_si128((const __m128i *)(in + 3 * 16)); 126191037db265ecdd914a26e056cf69207b4f50924ehkuang in04 = _mm_load_si128((const __m128i *)(in + 4 * 16)); 126291037db265ecdd914a26e056cf69207b4f50924ehkuang in05 = _mm_load_si128((const __m128i *)(in + 5 * 16)); 126391037db265ecdd914a26e056cf69207b4f50924ehkuang in06 = _mm_load_si128((const __m128i *)(in + 6 * 16)); 126491037db265ecdd914a26e056cf69207b4f50924ehkuang in07 = _mm_load_si128((const __m128i *)(in + 7 * 16)); 126591037db265ecdd914a26e056cf69207b4f50924ehkuang in08 = _mm_load_si128((const __m128i *)(in + 8 * 16)); 126691037db265ecdd914a26e056cf69207b4f50924ehkuang in09 = _mm_load_si128((const __m128i *)(in + 9 * 16)); 126791037db265ecdd914a26e056cf69207b4f50924ehkuang in10 = _mm_load_si128((const __m128i *)(in + 10 * 16)); 126891037db265ecdd914a26e056cf69207b4f50924ehkuang in11 = _mm_load_si128((const __m128i *)(in + 11 * 16)); 126991037db265ecdd914a26e056cf69207b4f50924ehkuang in12 = _mm_load_si128((const __m128i *)(in + 12 * 16)); 127091037db265ecdd914a26e056cf69207b4f50924ehkuang in13 = _mm_load_si128((const __m128i *)(in + 13 * 16)); 127191037db265ecdd914a26e056cf69207b4f50924ehkuang in14 = _mm_load_si128((const __m128i *)(in + 14 * 16)); 127291037db265ecdd914a26e056cf69207b4f50924ehkuang in15 = _mm_load_si128((const __m128i *)(in + 15 * 16)); 1273ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // x = (x + 1) >> 2 1274ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in00 = _mm_add_epi16(in00, kOne); 1275ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in01 = _mm_add_epi16(in01, kOne); 1276ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in02 = _mm_add_epi16(in02, kOne); 1277ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in03 = _mm_add_epi16(in03, kOne); 1278ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in04 = _mm_add_epi16(in04, kOne); 1279ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in05 = _mm_add_epi16(in05, kOne); 1280ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in06 = _mm_add_epi16(in06, kOne); 1281ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in07 = _mm_add_epi16(in07, kOne); 1282ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in08 = _mm_add_epi16(in08, kOne); 1283ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in09 = _mm_add_epi16(in09, kOne); 1284ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in10 = _mm_add_epi16(in10, kOne); 1285ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in11 = _mm_add_epi16(in11, kOne); 1286ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in12 = _mm_add_epi16(in12, kOne); 1287ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in13 = _mm_add_epi16(in13, kOne); 1288ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in14 = _mm_add_epi16(in14, kOne); 1289ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in15 = _mm_add_epi16(in15, kOne); 1290ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in00 = _mm_srai_epi16(in00, 2); 1291ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in01 = _mm_srai_epi16(in01, 2); 1292ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in02 = _mm_srai_epi16(in02, 2); 1293ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in03 = _mm_srai_epi16(in03, 2); 1294ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in04 = _mm_srai_epi16(in04, 2); 1295ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in05 = _mm_srai_epi16(in05, 2); 1296ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in06 = _mm_srai_epi16(in06, 2); 1297ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in07 = _mm_srai_epi16(in07, 2); 1298ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in08 = _mm_srai_epi16(in08, 2); 1299ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in09 = _mm_srai_epi16(in09, 2); 1300ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in10 = _mm_srai_epi16(in10, 2); 1301ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in11 = _mm_srai_epi16(in11, 2); 1302ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in12 = _mm_srai_epi16(in12, 2); 1303ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in13 = _mm_srai_epi16(in13, 2); 1304ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in14 = _mm_srai_epi16(in14, 2); 1305ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in15 = _mm_srai_epi16(in15, 2); 1306ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1307ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in += 8; 1308ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Calculate input for the first 8 results. 1309ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1310ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang input0 = _mm_add_epi16(in00, in15); 1311ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang input1 = _mm_add_epi16(in01, in14); 1312ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang input2 = _mm_add_epi16(in02, in13); 1313ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang input3 = _mm_add_epi16(in03, in12); 1314ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang input4 = _mm_add_epi16(in04, in11); 1315ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang input5 = _mm_add_epi16(in05, in10); 1316ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang input6 = _mm_add_epi16(in06, in09); 1317ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang input7 = _mm_add_epi16(in07, in08); 1318ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1319ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Calculate input for the next 8 results. 1320ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1321ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_0 = _mm_sub_epi16(in07, in08); 1322ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_1 = _mm_sub_epi16(in06, in09); 1323ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_2 = _mm_sub_epi16(in05, in10); 1324ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_3 = _mm_sub_epi16(in04, in11); 1325ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_4 = _mm_sub_epi16(in03, in12); 1326ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_5 = _mm_sub_epi16(in02, in13); 1327ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_6 = _mm_sub_epi16(in01, in14); 1328ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_7 = _mm_sub_epi16(in00, in15); 1329ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1330b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Work on the first eight values; fdct8(input, even_results); 1331ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1332b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Add/subtract 1333ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q0 = _mm_add_epi16(input0, input7); 1334ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q1 = _mm_add_epi16(input1, input6); 1335ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q2 = _mm_add_epi16(input2, input5); 1336ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q3 = _mm_add_epi16(input3, input4); 1337ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q4 = _mm_sub_epi16(input3, input4); 1338ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q5 = _mm_sub_epi16(input2, input5); 1339ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q6 = _mm_sub_epi16(input1, input6); 1340ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i q7 = _mm_sub_epi16(input0, input7); 1341ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Work on first four results 1342ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1343b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Add/subtract 1344ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r0 = _mm_add_epi16(q0, q3); 1345ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r1 = _mm_add_epi16(q1, q2); 1346ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r2 = _mm_sub_epi16(q1, q2); 1347ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r3 = _mm_sub_epi16(q0, q3); 1348ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Interleave to do the multiply by constants which gets us 1349ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // into 32 bits. 1350ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(r0, r1); 1351ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(r0, r1); 1352ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(r2, r3); 1353ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(r2, r3); 1354ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16); 1355ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16); 1356ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16); 1357ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16); 1358ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08); 1359ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08); 1360ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24); 1361ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24); 1362ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1363ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1364ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1365ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1366ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1367ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); 1368ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); 1369ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); 1370ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); 1371ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1372ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1373ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1374ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1375ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); 1376ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); 1377ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); 1378ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); 1379ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1380ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res00 = _mm_packs_epi32(w0, w1); 1381ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res08 = _mm_packs_epi32(w2, w3); 1382ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res04 = _mm_packs_epi32(w4, w5); 1383ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res12 = _mm_packs_epi32(w6, w7); 1384ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1385ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Work on next four results 1386ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1387ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Interleave to do the multiply by constants which gets us 1388ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // into 32 bits. 1389ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i d0 = _mm_unpacklo_epi16(q6, q5); 1390ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i d1 = _mm_unpackhi_epi16(q6, q5); 1391ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16); 1392ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16); 1393ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16); 1394ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16); 1395ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1396ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING); 1397ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING); 1398ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING); 1399ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING); 1400ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS); 1401ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS); 1402ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS); 1403ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS); 1404ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1405ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r0 = _mm_packs_epi32(s0, s1); 1406ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i r1 = _mm_packs_epi32(s2, s3); 1407b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian // Add/subtract 1408ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i x0 = _mm_add_epi16(q4, r0); 1409ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i x1 = _mm_sub_epi16(q4, r0); 1410ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i x2 = _mm_sub_epi16(q7, r1); 1411ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i x3 = _mm_add_epi16(q7, r1); 1412ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Interleave to do the multiply by constants which gets us 1413ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // into 32 bits. 1414ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(x0, x3); 1415ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(x0, x3); 1416ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(x1, x2); 1417ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(x1, x2); 1418ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04); 1419ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04); 1420ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28); 1421ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28); 1422ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20); 1423ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20); 1424ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12); 1425ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12); 1426ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1427ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1428ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1429ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1430ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1431ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING); 1432ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING); 1433ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING); 1434ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING); 1435ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1436ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1437ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1438ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1439ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS); 1440ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS); 1441ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS); 1442ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS); 1443ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1444ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res02 = _mm_packs_epi32(w0, w1); 1445ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res14 = _mm_packs_epi32(w2, w3); 1446ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res10 = _mm_packs_epi32(w4, w5); 1447ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res06 = _mm_packs_epi32(w6, w7); 1448ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1449ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1450ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Work on the next eight values; step1 -> odd_results 1451ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1452ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // step 2 1453ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1454ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(step1_5, step1_2); 1455ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(step1_5, step1_2); 1456ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(step1_4, step1_3); 1457ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(step1_4, step1_3); 1458ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_m16); 1459ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_m16); 1460ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p16_m16); 1461ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p16_m16); 1462ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1463ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1464ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1465ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1466ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1467ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1468ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1469ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1470ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1471ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1472ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step2_2 = _mm_packs_epi32(w0, w1); 1473ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step2_3 = _mm_packs_epi32(w2, w3); 1474ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1475ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1476ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(step1_5, step1_2); 1477ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(step1_5, step1_2); 1478ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(step1_4, step1_3); 1479ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(step1_4, step1_3); 1480ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16); 1481ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16); 1482ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p16_p16); 1483ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p16_p16); 1484ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1485ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1486ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1487ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1488ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1489ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1490ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1491ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1492ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1493ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1494ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step2_5 = _mm_packs_epi32(w0, w1); 1495ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step2_4 = _mm_packs_epi32(w2, w3); 1496ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1497ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // step 3 1498ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1499ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step3_0 = _mm_add_epi16(step1_0, step2_3); 1500ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step3_1 = _mm_add_epi16(step1_1, step2_2); 1501ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step3_2 = _mm_sub_epi16(step1_1, step2_2); 1502ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step3_3 = _mm_sub_epi16(step1_0, step2_3); 1503ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step3_4 = _mm_sub_epi16(step1_7, step2_4); 1504ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step3_5 = _mm_sub_epi16(step1_6, step2_5); 1505ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step3_6 = _mm_add_epi16(step1_6, step2_5); 1506ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step3_7 = _mm_add_epi16(step1_7, step2_4); 1507ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1508ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // step 4 1509ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1510ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(step3_1, step3_6); 1511ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(step3_1, step3_6); 1512ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(step3_2, step3_5); 1513ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(step3_2, step3_5); 1514ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_m08_p24); 1515ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_m08_p24); 1516ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m24_m08); 1517ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m24_m08); 1518ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1519ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1520ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1521ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1522ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1523ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1524ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1525ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1526ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1527ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1528ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step2_1 = _mm_packs_epi32(w0, w1); 1529ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step2_2 = _mm_packs_epi32(w2, w3); 1530ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1531ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1532ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(step3_1, step3_6); 1533ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(step3_1, step3_6); 1534ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(step3_2, step3_5); 1535ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(step3_2, step3_5); 1536ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p24_p08); 1537ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p24_p08); 1538ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m08_p24); 1539ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m08_p24); 1540ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1541ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1542ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1543ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1544ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1545ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1546ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1547ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1548ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1549ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1550ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step2_6 = _mm_packs_epi32(w0, w1); 1551ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step2_5 = _mm_packs_epi32(w2, w3); 1552ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1553ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // step 5 1554ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1555ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_0 = _mm_add_epi16(step3_0, step2_1); 1556ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_1 = _mm_sub_epi16(step3_0, step2_1); 1557ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_2 = _mm_sub_epi16(step3_3, step2_2); 1558ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_3 = _mm_add_epi16(step3_3, step2_2); 1559ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_4 = _mm_add_epi16(step3_4, step2_5); 1560ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_5 = _mm_sub_epi16(step3_4, step2_5); 1561ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_6 = _mm_sub_epi16(step3_7, step2_6); 1562ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang step1_7 = _mm_add_epi16(step3_7, step2_6); 1563ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1564ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // step 6 1565ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1566ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(step1_0, step1_7); 1567ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(step1_0, step1_7); 1568ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(step1_1, step1_6); 1569ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(step1_1, step1_6); 1570ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p30_p02); 1571ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p30_p02); 1572ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p14_p18); 1573ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p14_p18); 1574ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1575ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1576ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1577ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1578ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1579ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1580ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1581ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1582ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1583ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1584ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res01 = _mm_packs_epi32(w0, w1); 1585ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res09 = _mm_packs_epi32(w2, w3); 1586ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1587ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1588ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(step1_2, step1_5); 1589ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(step1_2, step1_5); 1590ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(step1_3, step1_4); 1591ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(step1_3, step1_4); 1592ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p22_p10); 1593ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p22_p10); 1594ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p06_p26); 1595ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p06_p26); 1596ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1597ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1598ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1599ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1600ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1601ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1602ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1603ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1604ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1605ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1606ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res05 = _mm_packs_epi32(w0, w1); 1607ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res13 = _mm_packs_epi32(w2, w3); 1608ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1609ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1610ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(step1_2, step1_5); 1611ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(step1_2, step1_5); 1612ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(step1_3, step1_4); 1613ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(step1_3, step1_4); 1614ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_m10_p22); 1615ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_m10_p22); 1616ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m26_p06); 1617ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m26_p06); 1618ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1619ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1620ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1621ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1622ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1623ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1624ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1625ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1626ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1627ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1628ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res11 = _mm_packs_epi32(w0, w1); 1629ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res03 = _mm_packs_epi32(w2, w3); 1630ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1631ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1632ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t0 = _mm_unpacklo_epi16(step1_0, step1_7); 1633ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t1 = _mm_unpackhi_epi16(step1_0, step1_7); 1634ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t2 = _mm_unpacklo_epi16(step1_1, step1_6); 1635ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i t3 = _mm_unpackhi_epi16(step1_1, step1_6); 1636ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u0 = _mm_madd_epi16(t0, k__cospi_m02_p30); 1637ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u1 = _mm_madd_epi16(t1, k__cospi_m02_p30); 1638ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m18_p14); 1639ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m18_p14); 1640ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // dct_const_round_shift 1641ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING); 1642ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING); 1643ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING); 1644ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING); 1645ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS); 1646ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS); 1647ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS); 1648ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS); 1649ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Combine 1650ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res15 = _mm_packs_epi32(w0, w1); 1651ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang res07 = _mm_packs_epi32(w2, w3); 1652ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1653ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1654ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Transpose the results, do it as two 8x8 transposes. 1655ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1656ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 01 02 03 04 05 06 07 1657ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 10 11 12 13 14 15 16 17 1658ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 20 21 22 23 24 25 26 27 1659ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 30 31 32 33 34 35 36 37 1660ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 40 41 42 43 44 45 46 47 1661ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 50 51 52 53 54 55 56 57 1662ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 60 61 62 63 64 65 66 67 1663ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 70 71 72 73 74 75 76 77 1664ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_0 = _mm_unpacklo_epi16(res00, res01); 1665ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_1 = _mm_unpacklo_epi16(res02, res03); 1666ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_2 = _mm_unpackhi_epi16(res00, res01); 1667ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_3 = _mm_unpackhi_epi16(res02, res03); 1668ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_4 = _mm_unpacklo_epi16(res04, res05); 1669ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_5 = _mm_unpacklo_epi16(res06, res07); 1670ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_6 = _mm_unpackhi_epi16(res04, res05); 1671ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_7 = _mm_unpackhi_epi16(res06, res07); 1672ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 10 01 11 02 12 03 13 1673ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 20 30 21 31 22 32 23 33 1674ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 04 14 05 15 06 16 07 17 1675ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 24 34 25 35 26 36 27 37 1676ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 40 50 41 51 42 52 43 53 1677ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 60 70 61 71 62 72 63 73 1678ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 54 54 55 55 56 56 57 57 1679ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 64 74 65 75 66 76 67 77 1680ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); 1681ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); 1682ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); 1683ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); 1684ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); 1685ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); 1686ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); 1687ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); 1688ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 10 20 30 01 11 21 31 1689ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 40 50 60 70 41 51 61 71 1690ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 02 12 22 32 03 13 23 33 1691ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 42 52 62 72 43 53 63 73 1692ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 04 14 24 34 05 15 21 36 1693ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 44 54 64 74 45 55 61 76 1694ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 06 16 26 36 07 17 27 37 1695ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 46 56 66 76 47 57 67 77 1696ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_0 = _mm_unpacklo_epi64(tr1_0, tr1_4); 1697ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_1 = _mm_unpackhi_epi64(tr1_0, tr1_4); 1698ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_2 = _mm_unpacklo_epi64(tr1_2, tr1_6); 1699ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_3 = _mm_unpackhi_epi64(tr1_2, tr1_6); 1700ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_4 = _mm_unpacklo_epi64(tr1_1, tr1_5); 1701ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_5 = _mm_unpackhi_epi64(tr1_1, tr1_5); 1702ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_6 = _mm_unpacklo_epi64(tr1_3, tr1_7); 1703ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_7 = _mm_unpackhi_epi64(tr1_3, tr1_7); 1704ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 10 20 30 40 50 60 70 1705ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 01 11 21 31 41 51 61 71 1706ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 02 12 22 32 42 52 62 72 1707ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 03 13 23 33 43 53 63 73 1708ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 04 14 24 34 44 54 64 74 1709ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 05 15 25 35 45 55 65 75 1710ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 06 16 26 36 46 56 66 76 1711ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 07 17 27 37 47 57 67 77 1712ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang _mm_storeu_si128((__m128i *)(out + 0 * 16), tr2_0); 1713ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang _mm_storeu_si128((__m128i *)(out + 1 * 16), tr2_1); 1714ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang _mm_storeu_si128((__m128i *)(out + 2 * 16), tr2_2); 1715ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang _mm_storeu_si128((__m128i *)(out + 3 * 16), tr2_3); 1716ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang _mm_storeu_si128((__m128i *)(out + 4 * 16), tr2_4); 1717ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang _mm_storeu_si128((__m128i *)(out + 5 * 16), tr2_5); 1718ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang _mm_storeu_si128((__m128i *)(out + 6 * 16), tr2_6); 1719ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang _mm_storeu_si128((__m128i *)(out + 7 * 16), tr2_7); 1720ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1721ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang { 1722ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 01 02 03 04 05 06 07 1723ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 10 11 12 13 14 15 16 17 1724ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 20 21 22 23 24 25 26 27 1725ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 30 31 32 33 34 35 36 37 1726ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 40 41 42 43 44 45 46 47 1727ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 50 51 52 53 54 55 56 57 1728ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 60 61 62 63 64 65 66 67 1729ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 70 71 72 73 74 75 76 77 1730ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_0 = _mm_unpacklo_epi16(res08, res09); 1731ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_1 = _mm_unpacklo_epi16(res10, res11); 1732ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_2 = _mm_unpackhi_epi16(res08, res09); 1733ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_3 = _mm_unpackhi_epi16(res10, res11); 1734ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_4 = _mm_unpacklo_epi16(res12, res13); 1735ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_5 = _mm_unpacklo_epi16(res14, res15); 1736ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_6 = _mm_unpackhi_epi16(res12, res13); 1737ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr0_7 = _mm_unpackhi_epi16(res14, res15); 1738ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 10 01 11 02 12 03 13 1739ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 20 30 21 31 22 32 23 33 1740ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 04 14 05 15 06 16 07 17 1741ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 24 34 25 35 26 36 27 37 1742ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 40 50 41 51 42 52 43 53 1743ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 60 70 61 71 62 72 63 73 1744ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 54 54 55 55 56 56 57 57 1745ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 64 74 65 75 66 76 67 77 1746ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1); 1747ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3); 1748ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1); 1749ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3); 1750ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5); 1751ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7); 1752ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5); 1753ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7); 1754ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 10 20 30 01 11 21 31 1755ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 40 50 60 70 41 51 61 71 1756ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 02 12 22 32 03 13 23 33 1757ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 42 52 62 72 43 53 63 73 1758ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 04 14 24 34 05 15 21 36 1759ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 44 54 64 74 45 55 61 76 1760ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 06 16 26 36 07 17 27 37 1761ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 46 56 66 76 47 57 67 77 1762ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_0 = _mm_unpacklo_epi64(tr1_0, tr1_4); 1763ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_1 = _mm_unpackhi_epi64(tr1_0, tr1_4); 1764ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_2 = _mm_unpacklo_epi64(tr1_2, tr1_6); 1765ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_3 = _mm_unpackhi_epi64(tr1_2, tr1_6); 1766ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_4 = _mm_unpacklo_epi64(tr1_1, tr1_5); 1767ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_5 = _mm_unpackhi_epi64(tr1_1, tr1_5); 1768ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_6 = _mm_unpacklo_epi64(tr1_3, tr1_7); 1769ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang const __m128i tr2_7 = _mm_unpackhi_epi64(tr1_3, tr1_7); 1770ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 00 10 20 30 40 50 60 70 1771ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 01 11 21 31 41 51 61 71 1772ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 02 12 22 32 42 52 62 72 1773ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 03 13 23 33 43 53 63 73 1774ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 04 14 24 34 44 54 64 74 1775ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 05 15 25 35 45 55 65 75 1776ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 06 16 26 36 46 56 66 76 1777ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // 07 17 27 37 47 57 67 77 1778ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Store results 177991037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(out + 8 + 0 * 16), tr2_0); 178091037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(out + 8 + 1 * 16), tr2_1); 178191037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(out + 8 + 2 * 16), tr2_2); 178291037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(out + 8 + 3 * 16), tr2_3); 178391037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(out + 8 + 4 * 16), tr2_4); 178491037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(out + 8 + 5 * 16), tr2_5); 178591037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(out + 8 + 6 * 16), tr2_6); 178691037db265ecdd914a26e056cf69207b4f50924ehkuang _mm_store_si128((__m128i *)(out + 8 + 7 * 16), tr2_7); 1787ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1788ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang out += 8*16; 1789ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1790ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang // Setup in/out for next pass. 1791ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang in = intermediate; 1792ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang out = output; 1793ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang } 1794ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang} 179591037db265ecdd914a26e056cf69207b4f50924ehkuang 17965ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuangstatic INLINE void load_buffer_16x16(const int16_t* input, __m128i *in0, 179791037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i *in1, int stride) { 179891037db265ecdd914a26e056cf69207b4f50924ehkuang // load first 8 columns 179991037db265ecdd914a26e056cf69207b4f50924ehkuang load_buffer_8x8(input, in0, stride); 180091037db265ecdd914a26e056cf69207b4f50924ehkuang load_buffer_8x8(input + 8 * stride, in0 + 8, stride); 180191037db265ecdd914a26e056cf69207b4f50924ehkuang 180291037db265ecdd914a26e056cf69207b4f50924ehkuang input += 8; 180391037db265ecdd914a26e056cf69207b4f50924ehkuang // load second 8 columns 180491037db265ecdd914a26e056cf69207b4f50924ehkuang load_buffer_8x8(input, in1, stride); 180591037db265ecdd914a26e056cf69207b4f50924ehkuang load_buffer_8x8(input + 8 * stride, in1 + 8, stride); 180691037db265ecdd914a26e056cf69207b4f50924ehkuang} 180791037db265ecdd914a26e056cf69207b4f50924ehkuang 180891037db265ecdd914a26e056cf69207b4f50924ehkuangstatic INLINE void write_buffer_16x16(int16_t *output, __m128i *in0, 180991037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i *in1, int stride) { 181091037db265ecdd914a26e056cf69207b4f50924ehkuang // write first 8 columns 181191037db265ecdd914a26e056cf69207b4f50924ehkuang write_buffer_8x8(output, in0, stride); 181291037db265ecdd914a26e056cf69207b4f50924ehkuang write_buffer_8x8(output + 8 * stride, in0 + 8, stride); 181391037db265ecdd914a26e056cf69207b4f50924ehkuang // write second 8 columns 181491037db265ecdd914a26e056cf69207b4f50924ehkuang output += 8; 181591037db265ecdd914a26e056cf69207b4f50924ehkuang write_buffer_8x8(output, in1, stride); 181691037db265ecdd914a26e056cf69207b4f50924ehkuang write_buffer_8x8(output + 8 * stride, in1 + 8, stride); 181791037db265ecdd914a26e056cf69207b4f50924ehkuang} 181891037db265ecdd914a26e056cf69207b4f50924ehkuang 181991037db265ecdd914a26e056cf69207b4f50924ehkuangstatic INLINE void array_transpose_16x16(__m128i *res0, __m128i *res1) { 182091037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i tbuf[8]; 182191037db265ecdd914a26e056cf69207b4f50924ehkuang array_transpose_8x8(res0, res0); 182291037db265ecdd914a26e056cf69207b4f50924ehkuang array_transpose_8x8(res1, tbuf); 182391037db265ecdd914a26e056cf69207b4f50924ehkuang array_transpose_8x8(res0 + 8, res1); 182491037db265ecdd914a26e056cf69207b4f50924ehkuang array_transpose_8x8(res1 + 8, res1 + 8); 182591037db265ecdd914a26e056cf69207b4f50924ehkuang 182691037db265ecdd914a26e056cf69207b4f50924ehkuang res0[8] = tbuf[0]; 182791037db265ecdd914a26e056cf69207b4f50924ehkuang res0[9] = tbuf[1]; 182891037db265ecdd914a26e056cf69207b4f50924ehkuang res0[10] = tbuf[2]; 182991037db265ecdd914a26e056cf69207b4f50924ehkuang res0[11] = tbuf[3]; 183091037db265ecdd914a26e056cf69207b4f50924ehkuang res0[12] = tbuf[4]; 183191037db265ecdd914a26e056cf69207b4f50924ehkuang res0[13] = tbuf[5]; 183291037db265ecdd914a26e056cf69207b4f50924ehkuang res0[14] = tbuf[6]; 183391037db265ecdd914a26e056cf69207b4f50924ehkuang res0[15] = tbuf[7]; 183491037db265ecdd914a26e056cf69207b4f50924ehkuang} 183591037db265ecdd914a26e056cf69207b4f50924ehkuang 183691037db265ecdd914a26e056cf69207b4f50924ehkuangstatic INLINE void right_shift_16x16(__m128i *res0, __m128i *res1) { 183791037db265ecdd914a26e056cf69207b4f50924ehkuang // perform rounding operations 183891037db265ecdd914a26e056cf69207b4f50924ehkuang right_shift_8x8(res0, 2); 183991037db265ecdd914a26e056cf69207b4f50924ehkuang right_shift_8x8(res0 + 8, 2); 184091037db265ecdd914a26e056cf69207b4f50924ehkuang right_shift_8x8(res1, 2); 184191037db265ecdd914a26e056cf69207b4f50924ehkuang right_shift_8x8(res1 + 8, 2); 184291037db265ecdd914a26e056cf69207b4f50924ehkuang} 184391037db265ecdd914a26e056cf69207b4f50924ehkuang 1844b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid fdct16_8col(__m128i *in) { 184591037db265ecdd914a26e056cf69207b4f50924ehkuang // perform 16x16 1-D DCT for 8 columns 184691037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i i[8], s[8], p[8], t[8], u[16], v[16]; 184791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64); 184891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); 184991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64); 185091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64); 185191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64); 185291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64); 185391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64); 185491037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64); 185591037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64); 185691037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64); 185791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p30_p02 = pair_set_epi16(cospi_30_64, cospi_2_64); 185891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p14_p18 = pair_set_epi16(cospi_14_64, cospi_18_64); 185991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64); 186091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64); 186191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p22_p10 = pair_set_epi16(cospi_22_64, cospi_10_64); 186291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p06_p26 = pair_set_epi16(cospi_6_64, cospi_26_64); 186391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m10_p22 = pair_set_epi16(-cospi_10_64, cospi_22_64); 186491037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m26_p06 = pair_set_epi16(-cospi_26_64, cospi_6_64); 186591037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 186691037db265ecdd914a26e056cf69207b4f50924ehkuang 186791037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 1 186891037db265ecdd914a26e056cf69207b4f50924ehkuang i[0] = _mm_add_epi16(in[0], in[15]); 186991037db265ecdd914a26e056cf69207b4f50924ehkuang i[1] = _mm_add_epi16(in[1], in[14]); 187091037db265ecdd914a26e056cf69207b4f50924ehkuang i[2] = _mm_add_epi16(in[2], in[13]); 187191037db265ecdd914a26e056cf69207b4f50924ehkuang i[3] = _mm_add_epi16(in[3], in[12]); 187291037db265ecdd914a26e056cf69207b4f50924ehkuang i[4] = _mm_add_epi16(in[4], in[11]); 187391037db265ecdd914a26e056cf69207b4f50924ehkuang i[5] = _mm_add_epi16(in[5], in[10]); 187491037db265ecdd914a26e056cf69207b4f50924ehkuang i[6] = _mm_add_epi16(in[6], in[9]); 187591037db265ecdd914a26e056cf69207b4f50924ehkuang i[7] = _mm_add_epi16(in[7], in[8]); 187691037db265ecdd914a26e056cf69207b4f50924ehkuang 187791037db265ecdd914a26e056cf69207b4f50924ehkuang s[0] = _mm_sub_epi16(in[7], in[8]); 187891037db265ecdd914a26e056cf69207b4f50924ehkuang s[1] = _mm_sub_epi16(in[6], in[9]); 187991037db265ecdd914a26e056cf69207b4f50924ehkuang s[2] = _mm_sub_epi16(in[5], in[10]); 188091037db265ecdd914a26e056cf69207b4f50924ehkuang s[3] = _mm_sub_epi16(in[4], in[11]); 188191037db265ecdd914a26e056cf69207b4f50924ehkuang s[4] = _mm_sub_epi16(in[3], in[12]); 188291037db265ecdd914a26e056cf69207b4f50924ehkuang s[5] = _mm_sub_epi16(in[2], in[13]); 188391037db265ecdd914a26e056cf69207b4f50924ehkuang s[6] = _mm_sub_epi16(in[1], in[14]); 188491037db265ecdd914a26e056cf69207b4f50924ehkuang s[7] = _mm_sub_epi16(in[0], in[15]); 188591037db265ecdd914a26e056cf69207b4f50924ehkuang 188691037db265ecdd914a26e056cf69207b4f50924ehkuang p[0] = _mm_add_epi16(i[0], i[7]); 188791037db265ecdd914a26e056cf69207b4f50924ehkuang p[1] = _mm_add_epi16(i[1], i[6]); 188891037db265ecdd914a26e056cf69207b4f50924ehkuang p[2] = _mm_add_epi16(i[2], i[5]); 188991037db265ecdd914a26e056cf69207b4f50924ehkuang p[3] = _mm_add_epi16(i[3], i[4]); 189091037db265ecdd914a26e056cf69207b4f50924ehkuang p[4] = _mm_sub_epi16(i[3], i[4]); 189191037db265ecdd914a26e056cf69207b4f50924ehkuang p[5] = _mm_sub_epi16(i[2], i[5]); 189291037db265ecdd914a26e056cf69207b4f50924ehkuang p[6] = _mm_sub_epi16(i[1], i[6]); 189391037db265ecdd914a26e056cf69207b4f50924ehkuang p[7] = _mm_sub_epi16(i[0], i[7]); 189491037db265ecdd914a26e056cf69207b4f50924ehkuang 189591037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi16(p[0], p[3]); 189691037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi16(p[1], p[2]); 189791037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_sub_epi16(p[1], p[2]); 189891037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_sub_epi16(p[0], p[3]); 189991037db265ecdd914a26e056cf69207b4f50924ehkuang 190091037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_unpacklo_epi16(u[0], u[1]); 190191037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_unpackhi_epi16(u[0], u[1]); 190291037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_unpacklo_epi16(u[2], u[3]); 190391037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_unpackhi_epi16(u[2], u[3]); 190491037db265ecdd914a26e056cf69207b4f50924ehkuang 190591037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_madd_epi16(v[0], k__cospi_p16_p16); 190691037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_madd_epi16(v[1], k__cospi_p16_p16); 190791037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_madd_epi16(v[0], k__cospi_p16_m16); 190891037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_madd_epi16(v[1], k__cospi_p16_m16); 190991037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_madd_epi16(v[2], k__cospi_p24_p08); 191091037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_madd_epi16(v[3], k__cospi_p24_p08); 191191037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_madd_epi16(v[2], k__cospi_m08_p24); 191291037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_madd_epi16(v[3], k__cospi_m08_p24); 191391037db265ecdd914a26e056cf69207b4f50924ehkuang 191491037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); 191591037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); 191691037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); 191791037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); 191891037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); 191991037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); 192091037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); 192191037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); 192291037db265ecdd914a26e056cf69207b4f50924ehkuang 192391037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); 192491037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); 192591037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); 192691037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); 192791037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); 192891037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); 192991037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); 193091037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); 193191037db265ecdd914a26e056cf69207b4f50924ehkuang 193291037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = _mm_packs_epi32(u[0], u[1]); 193391037db265ecdd914a26e056cf69207b4f50924ehkuang in[4] = _mm_packs_epi32(u[4], u[5]); 193491037db265ecdd914a26e056cf69207b4f50924ehkuang in[8] = _mm_packs_epi32(u[2], u[3]); 193591037db265ecdd914a26e056cf69207b4f50924ehkuang in[12] = _mm_packs_epi32(u[6], u[7]); 193691037db265ecdd914a26e056cf69207b4f50924ehkuang 193791037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(p[5], p[6]); 193891037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpackhi_epi16(p[5], p[6]); 193991037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__cospi_m16_p16); 194091037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__cospi_m16_p16); 194191037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[0], k__cospi_p16_p16); 194291037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[1], k__cospi_p16_p16); 194391037db265ecdd914a26e056cf69207b4f50924ehkuang 194491037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); 194591037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); 194691037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); 194791037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); 194891037db265ecdd914a26e056cf69207b4f50924ehkuang 194991037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); 195091037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); 195191037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); 195291037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); 195391037db265ecdd914a26e056cf69207b4f50924ehkuang 195491037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_packs_epi32(v[0], v[1]); 195591037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_packs_epi32(v[2], v[3]); 195691037db265ecdd914a26e056cf69207b4f50924ehkuang 195791037db265ecdd914a26e056cf69207b4f50924ehkuang t[0] = _mm_add_epi16(p[4], u[0]); 195891037db265ecdd914a26e056cf69207b4f50924ehkuang t[1] = _mm_sub_epi16(p[4], u[0]); 195991037db265ecdd914a26e056cf69207b4f50924ehkuang t[2] = _mm_sub_epi16(p[7], u[1]); 196091037db265ecdd914a26e056cf69207b4f50924ehkuang t[3] = _mm_add_epi16(p[7], u[1]); 196191037db265ecdd914a26e056cf69207b4f50924ehkuang 196291037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(t[0], t[3]); 196391037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpackhi_epi16(t[0], t[3]); 196491037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_unpacklo_epi16(t[1], t[2]); 196591037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_unpackhi_epi16(t[1], t[2]); 196691037db265ecdd914a26e056cf69207b4f50924ehkuang 196791037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__cospi_p28_p04); 196891037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__cospi_p28_p04); 196991037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[2], k__cospi_p12_p20); 197091037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[3], k__cospi_p12_p20); 197191037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_madd_epi16(u[2], k__cospi_m20_p12); 197291037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_madd_epi16(u[3], k__cospi_m20_p12); 197391037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_madd_epi16(u[0], k__cospi_m04_p28); 197491037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_madd_epi16(u[1], k__cospi_m04_p28); 197591037db265ecdd914a26e056cf69207b4f50924ehkuang 197691037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); 197791037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); 197891037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); 197991037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); 198091037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); 198191037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); 198291037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); 198391037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); 198491037db265ecdd914a26e056cf69207b4f50924ehkuang 198591037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); 198691037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); 198791037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); 198891037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); 198991037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); 199091037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); 199191037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); 199291037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); 199391037db265ecdd914a26e056cf69207b4f50924ehkuang 199491037db265ecdd914a26e056cf69207b4f50924ehkuang in[2] = _mm_packs_epi32(v[0], v[1]); 199591037db265ecdd914a26e056cf69207b4f50924ehkuang in[6] = _mm_packs_epi32(v[4], v[5]); 199691037db265ecdd914a26e056cf69207b4f50924ehkuang in[10] = _mm_packs_epi32(v[2], v[3]); 199791037db265ecdd914a26e056cf69207b4f50924ehkuang in[14] = _mm_packs_epi32(v[6], v[7]); 199891037db265ecdd914a26e056cf69207b4f50924ehkuang 199991037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 2 200091037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(s[2], s[5]); 200191037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpackhi_epi16(s[2], s[5]); 200291037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_unpacklo_epi16(s[3], s[4]); 200391037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_unpackhi_epi16(s[3], s[4]); 200491037db265ecdd914a26e056cf69207b4f50924ehkuang 200591037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__cospi_m16_p16); 200691037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__cospi_m16_p16); 200791037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[2], k__cospi_m16_p16); 200891037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[3], k__cospi_m16_p16); 200991037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_madd_epi16(u[2], k__cospi_p16_p16); 201091037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_madd_epi16(u[3], k__cospi_p16_p16); 201191037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_madd_epi16(u[0], k__cospi_p16_p16); 201291037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_madd_epi16(u[1], k__cospi_p16_p16); 201391037db265ecdd914a26e056cf69207b4f50924ehkuang 201491037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); 201591037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); 201691037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); 201791037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); 201891037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); 201991037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); 202091037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); 202191037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); 202291037db265ecdd914a26e056cf69207b4f50924ehkuang 202391037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); 202491037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); 202591037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); 202691037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); 202791037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); 202891037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); 202991037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); 203091037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); 203191037db265ecdd914a26e056cf69207b4f50924ehkuang 203291037db265ecdd914a26e056cf69207b4f50924ehkuang t[2] = _mm_packs_epi32(v[0], v[1]); 203391037db265ecdd914a26e056cf69207b4f50924ehkuang t[3] = _mm_packs_epi32(v[2], v[3]); 203491037db265ecdd914a26e056cf69207b4f50924ehkuang t[4] = _mm_packs_epi32(v[4], v[5]); 203591037db265ecdd914a26e056cf69207b4f50924ehkuang t[5] = _mm_packs_epi32(v[6], v[7]); 203691037db265ecdd914a26e056cf69207b4f50924ehkuang 203791037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 3 203891037db265ecdd914a26e056cf69207b4f50924ehkuang p[0] = _mm_add_epi16(s[0], t[3]); 203991037db265ecdd914a26e056cf69207b4f50924ehkuang p[1] = _mm_add_epi16(s[1], t[2]); 204091037db265ecdd914a26e056cf69207b4f50924ehkuang p[2] = _mm_sub_epi16(s[1], t[2]); 204191037db265ecdd914a26e056cf69207b4f50924ehkuang p[3] = _mm_sub_epi16(s[0], t[3]); 204291037db265ecdd914a26e056cf69207b4f50924ehkuang p[4] = _mm_sub_epi16(s[7], t[4]); 204391037db265ecdd914a26e056cf69207b4f50924ehkuang p[5] = _mm_sub_epi16(s[6], t[5]); 204491037db265ecdd914a26e056cf69207b4f50924ehkuang p[6] = _mm_add_epi16(s[6], t[5]); 204591037db265ecdd914a26e056cf69207b4f50924ehkuang p[7] = _mm_add_epi16(s[7], t[4]); 204691037db265ecdd914a26e056cf69207b4f50924ehkuang 204791037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 4 204891037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(p[1], p[6]); 204991037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpackhi_epi16(p[1], p[6]); 205091037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_unpacklo_epi16(p[2], p[5]); 205191037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_unpackhi_epi16(p[2], p[5]); 205291037db265ecdd914a26e056cf69207b4f50924ehkuang 205391037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__cospi_m08_p24); 205491037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__cospi_m08_p24); 205591037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[2], k__cospi_m24_m08); 205691037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[3], k__cospi_m24_m08); 205791037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_madd_epi16(u[2], k__cospi_m08_p24); 205891037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_madd_epi16(u[3], k__cospi_m08_p24); 205991037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_madd_epi16(u[0], k__cospi_p24_p08); 206091037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_madd_epi16(u[1], k__cospi_p24_p08); 206191037db265ecdd914a26e056cf69207b4f50924ehkuang 206291037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); 206391037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); 206491037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); 206591037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); 206691037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); 206791037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); 206891037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); 206991037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); 207091037db265ecdd914a26e056cf69207b4f50924ehkuang 207191037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); 207291037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); 207391037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); 207491037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); 207591037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); 207691037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); 207791037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); 207891037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); 207991037db265ecdd914a26e056cf69207b4f50924ehkuang 208091037db265ecdd914a26e056cf69207b4f50924ehkuang t[1] = _mm_packs_epi32(v[0], v[1]); 208191037db265ecdd914a26e056cf69207b4f50924ehkuang t[2] = _mm_packs_epi32(v[2], v[3]); 208291037db265ecdd914a26e056cf69207b4f50924ehkuang t[5] = _mm_packs_epi32(v[4], v[5]); 208391037db265ecdd914a26e056cf69207b4f50924ehkuang t[6] = _mm_packs_epi32(v[6], v[7]); 208491037db265ecdd914a26e056cf69207b4f50924ehkuang 208591037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 5 208691037db265ecdd914a26e056cf69207b4f50924ehkuang s[0] = _mm_add_epi16(p[0], t[1]); 208791037db265ecdd914a26e056cf69207b4f50924ehkuang s[1] = _mm_sub_epi16(p[0], t[1]); 208891037db265ecdd914a26e056cf69207b4f50924ehkuang s[2] = _mm_sub_epi16(p[3], t[2]); 208991037db265ecdd914a26e056cf69207b4f50924ehkuang s[3] = _mm_add_epi16(p[3], t[2]); 209091037db265ecdd914a26e056cf69207b4f50924ehkuang s[4] = _mm_add_epi16(p[4], t[5]); 209191037db265ecdd914a26e056cf69207b4f50924ehkuang s[5] = _mm_sub_epi16(p[4], t[5]); 209291037db265ecdd914a26e056cf69207b4f50924ehkuang s[6] = _mm_sub_epi16(p[7], t[6]); 209391037db265ecdd914a26e056cf69207b4f50924ehkuang s[7] = _mm_add_epi16(p[7], t[6]); 209491037db265ecdd914a26e056cf69207b4f50924ehkuang 209591037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 6 209691037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(s[0], s[7]); 209791037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpackhi_epi16(s[0], s[7]); 209891037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_unpacklo_epi16(s[1], s[6]); 209991037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_unpackhi_epi16(s[1], s[6]); 210091037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_unpacklo_epi16(s[2], s[5]); 210191037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_unpackhi_epi16(s[2], s[5]); 210291037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_unpacklo_epi16(s[3], s[4]); 210391037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_unpackhi_epi16(s[3], s[4]); 210491037db265ecdd914a26e056cf69207b4f50924ehkuang 210591037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__cospi_p30_p02); 210691037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__cospi_p30_p02); 210791037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[2], k__cospi_p14_p18); 210891037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[3], k__cospi_p14_p18); 210991037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_madd_epi16(u[4], k__cospi_p22_p10); 211091037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_madd_epi16(u[5], k__cospi_p22_p10); 211191037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_madd_epi16(u[6], k__cospi_p06_p26); 211291037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_madd_epi16(u[7], k__cospi_p06_p26); 211391037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_madd_epi16(u[6], k__cospi_m26_p06); 211491037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_madd_epi16(u[7], k__cospi_m26_p06); 211591037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_madd_epi16(u[4], k__cospi_m10_p22); 211691037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_madd_epi16(u[5], k__cospi_m10_p22); 211791037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_madd_epi16(u[2], k__cospi_m18_p14); 211891037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_madd_epi16(u[3], k__cospi_m18_p14); 211991037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_madd_epi16(u[0], k__cospi_m02_p30); 212091037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_madd_epi16(u[1], k__cospi_m02_p30); 212191037db265ecdd914a26e056cf69207b4f50924ehkuang 212291037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); 212391037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); 212491037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); 212591037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); 212691037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); 212791037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); 212891037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); 212991037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); 213091037db265ecdd914a26e056cf69207b4f50924ehkuang u[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING); 213191037db265ecdd914a26e056cf69207b4f50924ehkuang u[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING); 213291037db265ecdd914a26e056cf69207b4f50924ehkuang u[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING); 213391037db265ecdd914a26e056cf69207b4f50924ehkuang u[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING); 213491037db265ecdd914a26e056cf69207b4f50924ehkuang u[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING); 213591037db265ecdd914a26e056cf69207b4f50924ehkuang u[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING); 213691037db265ecdd914a26e056cf69207b4f50924ehkuang u[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING); 213791037db265ecdd914a26e056cf69207b4f50924ehkuang u[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING); 213891037db265ecdd914a26e056cf69207b4f50924ehkuang 213991037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); 214091037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); 214191037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); 214291037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); 214391037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); 214491037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); 214591037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); 214691037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); 214791037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); 214891037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); 214991037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); 215091037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); 215191037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); 215291037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); 215391037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); 215491037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); 215591037db265ecdd914a26e056cf69207b4f50924ehkuang 215691037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_packs_epi32(v[0], v[1]); 215791037db265ecdd914a26e056cf69207b4f50924ehkuang in[9] = _mm_packs_epi32(v[2], v[3]); 215891037db265ecdd914a26e056cf69207b4f50924ehkuang in[5] = _mm_packs_epi32(v[4], v[5]); 215991037db265ecdd914a26e056cf69207b4f50924ehkuang in[13] = _mm_packs_epi32(v[6], v[7]); 216091037db265ecdd914a26e056cf69207b4f50924ehkuang in[3] = _mm_packs_epi32(v[8], v[9]); 216191037db265ecdd914a26e056cf69207b4f50924ehkuang in[11] = _mm_packs_epi32(v[10], v[11]); 216291037db265ecdd914a26e056cf69207b4f50924ehkuang in[7] = _mm_packs_epi32(v[12], v[13]); 216391037db265ecdd914a26e056cf69207b4f50924ehkuang in[15] = _mm_packs_epi32(v[14], v[15]); 216491037db265ecdd914a26e056cf69207b4f50924ehkuang} 216591037db265ecdd914a26e056cf69207b4f50924ehkuang 2166b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid fadst16_8col(__m128i *in) { 216791037db265ecdd914a26e056cf69207b4f50924ehkuang // perform 16x16 1-D ADST for 8 columns 216891037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i s[16], x[16], u[32], v[32]; 216991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p01_p31 = pair_set_epi16(cospi_1_64, cospi_31_64); 217091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p31_m01 = pair_set_epi16(cospi_31_64, -cospi_1_64); 217191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p05_p27 = pair_set_epi16(cospi_5_64, cospi_27_64); 217291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p27_m05 = pair_set_epi16(cospi_27_64, -cospi_5_64); 217391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p09_p23 = pair_set_epi16(cospi_9_64, cospi_23_64); 217491037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p23_m09 = pair_set_epi16(cospi_23_64, -cospi_9_64); 217591037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p13_p19 = pair_set_epi16(cospi_13_64, cospi_19_64); 217691037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p19_m13 = pair_set_epi16(cospi_19_64, -cospi_13_64); 217791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p17_p15 = pair_set_epi16(cospi_17_64, cospi_15_64); 217891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p15_m17 = pair_set_epi16(cospi_15_64, -cospi_17_64); 217991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p21_p11 = pair_set_epi16(cospi_21_64, cospi_11_64); 218091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p11_m21 = pair_set_epi16(cospi_11_64, -cospi_21_64); 218191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p25_p07 = pair_set_epi16(cospi_25_64, cospi_7_64); 218291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p07_m25 = pair_set_epi16(cospi_7_64, -cospi_25_64); 218391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p29_p03 = pair_set_epi16(cospi_29_64, cospi_3_64); 218491037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p03_m29 = pair_set_epi16(cospi_3_64, -cospi_29_64); 218591037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64); 218691037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64); 218791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p20_p12 = pair_set_epi16(cospi_20_64, cospi_12_64); 218891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p12_m20 = pair_set_epi16(cospi_12_64, -cospi_20_64); 218991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m28_p04 = pair_set_epi16(-cospi_28_64, cospi_4_64); 219091037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m12_p20 = pair_set_epi16(-cospi_12_64, cospi_20_64); 219191037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64); 219291037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64); 219391037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m24_p08 = pair_set_epi16(-cospi_24_64, cospi_8_64); 219491037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m16_m16 = _mm_set1_epi16(-cospi_16_64); 219591037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64); 219691037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64); 219791037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64); 219891037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING); 219991037db265ecdd914a26e056cf69207b4f50924ehkuang const __m128i kZero = _mm_set1_epi16(0); 220091037db265ecdd914a26e056cf69207b4f50924ehkuang 220191037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(in[15], in[0]); 220291037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpackhi_epi16(in[15], in[0]); 220391037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_unpacklo_epi16(in[13], in[2]); 220491037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_unpackhi_epi16(in[13], in[2]); 220591037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_unpacklo_epi16(in[11], in[4]); 220691037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_unpackhi_epi16(in[11], in[4]); 220791037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_unpacklo_epi16(in[9], in[6]); 220891037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_unpackhi_epi16(in[9], in[6]); 220991037db265ecdd914a26e056cf69207b4f50924ehkuang u[8] = _mm_unpacklo_epi16(in[7], in[8]); 221091037db265ecdd914a26e056cf69207b4f50924ehkuang u[9] = _mm_unpackhi_epi16(in[7], in[8]); 221191037db265ecdd914a26e056cf69207b4f50924ehkuang u[10] = _mm_unpacklo_epi16(in[5], in[10]); 221291037db265ecdd914a26e056cf69207b4f50924ehkuang u[11] = _mm_unpackhi_epi16(in[5], in[10]); 221391037db265ecdd914a26e056cf69207b4f50924ehkuang u[12] = _mm_unpacklo_epi16(in[3], in[12]); 221491037db265ecdd914a26e056cf69207b4f50924ehkuang u[13] = _mm_unpackhi_epi16(in[3], in[12]); 221591037db265ecdd914a26e056cf69207b4f50924ehkuang u[14] = _mm_unpacklo_epi16(in[1], in[14]); 221691037db265ecdd914a26e056cf69207b4f50924ehkuang u[15] = _mm_unpackhi_epi16(in[1], in[14]); 221791037db265ecdd914a26e056cf69207b4f50924ehkuang 221891037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__cospi_p01_p31); 221991037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__cospi_p01_p31); 222091037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[0], k__cospi_p31_m01); 222191037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[1], k__cospi_p31_m01); 222291037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_madd_epi16(u[2], k__cospi_p05_p27); 222391037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_madd_epi16(u[3], k__cospi_p05_p27); 222491037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_madd_epi16(u[2], k__cospi_p27_m05); 222591037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_madd_epi16(u[3], k__cospi_p27_m05); 222691037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_madd_epi16(u[4], k__cospi_p09_p23); 222791037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_madd_epi16(u[5], k__cospi_p09_p23); 222891037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_madd_epi16(u[4], k__cospi_p23_m09); 222991037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_madd_epi16(u[5], k__cospi_p23_m09); 223091037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_madd_epi16(u[6], k__cospi_p13_p19); 223191037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_madd_epi16(u[7], k__cospi_p13_p19); 223291037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_madd_epi16(u[6], k__cospi_p19_m13); 223391037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_madd_epi16(u[7], k__cospi_p19_m13); 223491037db265ecdd914a26e056cf69207b4f50924ehkuang v[16] = _mm_madd_epi16(u[8], k__cospi_p17_p15); 223591037db265ecdd914a26e056cf69207b4f50924ehkuang v[17] = _mm_madd_epi16(u[9], k__cospi_p17_p15); 223691037db265ecdd914a26e056cf69207b4f50924ehkuang v[18] = _mm_madd_epi16(u[8], k__cospi_p15_m17); 223791037db265ecdd914a26e056cf69207b4f50924ehkuang v[19] = _mm_madd_epi16(u[9], k__cospi_p15_m17); 223891037db265ecdd914a26e056cf69207b4f50924ehkuang v[20] = _mm_madd_epi16(u[10], k__cospi_p21_p11); 223991037db265ecdd914a26e056cf69207b4f50924ehkuang v[21] = _mm_madd_epi16(u[11], k__cospi_p21_p11); 224091037db265ecdd914a26e056cf69207b4f50924ehkuang v[22] = _mm_madd_epi16(u[10], k__cospi_p11_m21); 224191037db265ecdd914a26e056cf69207b4f50924ehkuang v[23] = _mm_madd_epi16(u[11], k__cospi_p11_m21); 224291037db265ecdd914a26e056cf69207b4f50924ehkuang v[24] = _mm_madd_epi16(u[12], k__cospi_p25_p07); 224391037db265ecdd914a26e056cf69207b4f50924ehkuang v[25] = _mm_madd_epi16(u[13], k__cospi_p25_p07); 224491037db265ecdd914a26e056cf69207b4f50924ehkuang v[26] = _mm_madd_epi16(u[12], k__cospi_p07_m25); 224591037db265ecdd914a26e056cf69207b4f50924ehkuang v[27] = _mm_madd_epi16(u[13], k__cospi_p07_m25); 224691037db265ecdd914a26e056cf69207b4f50924ehkuang v[28] = _mm_madd_epi16(u[14], k__cospi_p29_p03); 224791037db265ecdd914a26e056cf69207b4f50924ehkuang v[29] = _mm_madd_epi16(u[15], k__cospi_p29_p03); 224891037db265ecdd914a26e056cf69207b4f50924ehkuang v[30] = _mm_madd_epi16(u[14], k__cospi_p03_m29); 224991037db265ecdd914a26e056cf69207b4f50924ehkuang v[31] = _mm_madd_epi16(u[15], k__cospi_p03_m29); 225091037db265ecdd914a26e056cf69207b4f50924ehkuang 225191037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], v[16]); 225291037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(v[1], v[17]); 225391037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[2], v[18]); 225491037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(v[3], v[19]); 225591037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_add_epi32(v[4], v[20]); 225691037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_add_epi32(v[5], v[21]); 225791037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_add_epi32(v[6], v[22]); 225891037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_add_epi32(v[7], v[23]); 225991037db265ecdd914a26e056cf69207b4f50924ehkuang u[8] = _mm_add_epi32(v[8], v[24]); 226091037db265ecdd914a26e056cf69207b4f50924ehkuang u[9] = _mm_add_epi32(v[9], v[25]); 226191037db265ecdd914a26e056cf69207b4f50924ehkuang u[10] = _mm_add_epi32(v[10], v[26]); 226291037db265ecdd914a26e056cf69207b4f50924ehkuang u[11] = _mm_add_epi32(v[11], v[27]); 226391037db265ecdd914a26e056cf69207b4f50924ehkuang u[12] = _mm_add_epi32(v[12], v[28]); 226491037db265ecdd914a26e056cf69207b4f50924ehkuang u[13] = _mm_add_epi32(v[13], v[29]); 226591037db265ecdd914a26e056cf69207b4f50924ehkuang u[14] = _mm_add_epi32(v[14], v[30]); 226691037db265ecdd914a26e056cf69207b4f50924ehkuang u[15] = _mm_add_epi32(v[15], v[31]); 226791037db265ecdd914a26e056cf69207b4f50924ehkuang u[16] = _mm_sub_epi32(v[0], v[16]); 226891037db265ecdd914a26e056cf69207b4f50924ehkuang u[17] = _mm_sub_epi32(v[1], v[17]); 226991037db265ecdd914a26e056cf69207b4f50924ehkuang u[18] = _mm_sub_epi32(v[2], v[18]); 227091037db265ecdd914a26e056cf69207b4f50924ehkuang u[19] = _mm_sub_epi32(v[3], v[19]); 227191037db265ecdd914a26e056cf69207b4f50924ehkuang u[20] = _mm_sub_epi32(v[4], v[20]); 227291037db265ecdd914a26e056cf69207b4f50924ehkuang u[21] = _mm_sub_epi32(v[5], v[21]); 227391037db265ecdd914a26e056cf69207b4f50924ehkuang u[22] = _mm_sub_epi32(v[6], v[22]); 227491037db265ecdd914a26e056cf69207b4f50924ehkuang u[23] = _mm_sub_epi32(v[7], v[23]); 227591037db265ecdd914a26e056cf69207b4f50924ehkuang u[24] = _mm_sub_epi32(v[8], v[24]); 227691037db265ecdd914a26e056cf69207b4f50924ehkuang u[25] = _mm_sub_epi32(v[9], v[25]); 227791037db265ecdd914a26e056cf69207b4f50924ehkuang u[26] = _mm_sub_epi32(v[10], v[26]); 227891037db265ecdd914a26e056cf69207b4f50924ehkuang u[27] = _mm_sub_epi32(v[11], v[27]); 227991037db265ecdd914a26e056cf69207b4f50924ehkuang u[28] = _mm_sub_epi32(v[12], v[28]); 228091037db265ecdd914a26e056cf69207b4f50924ehkuang u[29] = _mm_sub_epi32(v[13], v[29]); 228191037db265ecdd914a26e056cf69207b4f50924ehkuang u[30] = _mm_sub_epi32(v[14], v[30]); 228291037db265ecdd914a26e056cf69207b4f50924ehkuang u[31] = _mm_sub_epi32(v[15], v[31]); 228391037db265ecdd914a26e056cf69207b4f50924ehkuang 228491037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); 228591037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); 228691037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); 228791037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); 228891037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); 228991037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); 229091037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); 229191037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); 229291037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); 229391037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); 229491037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); 229591037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); 229691037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); 229791037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); 229891037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); 229991037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); 230091037db265ecdd914a26e056cf69207b4f50924ehkuang v[16] = _mm_add_epi32(u[16], k__DCT_CONST_ROUNDING); 230191037db265ecdd914a26e056cf69207b4f50924ehkuang v[17] = _mm_add_epi32(u[17], k__DCT_CONST_ROUNDING); 230291037db265ecdd914a26e056cf69207b4f50924ehkuang v[18] = _mm_add_epi32(u[18], k__DCT_CONST_ROUNDING); 230391037db265ecdd914a26e056cf69207b4f50924ehkuang v[19] = _mm_add_epi32(u[19], k__DCT_CONST_ROUNDING); 230491037db265ecdd914a26e056cf69207b4f50924ehkuang v[20] = _mm_add_epi32(u[20], k__DCT_CONST_ROUNDING); 230591037db265ecdd914a26e056cf69207b4f50924ehkuang v[21] = _mm_add_epi32(u[21], k__DCT_CONST_ROUNDING); 230691037db265ecdd914a26e056cf69207b4f50924ehkuang v[22] = _mm_add_epi32(u[22], k__DCT_CONST_ROUNDING); 230791037db265ecdd914a26e056cf69207b4f50924ehkuang v[23] = _mm_add_epi32(u[23], k__DCT_CONST_ROUNDING); 230891037db265ecdd914a26e056cf69207b4f50924ehkuang v[24] = _mm_add_epi32(u[24], k__DCT_CONST_ROUNDING); 230991037db265ecdd914a26e056cf69207b4f50924ehkuang v[25] = _mm_add_epi32(u[25], k__DCT_CONST_ROUNDING); 231091037db265ecdd914a26e056cf69207b4f50924ehkuang v[26] = _mm_add_epi32(u[26], k__DCT_CONST_ROUNDING); 231191037db265ecdd914a26e056cf69207b4f50924ehkuang v[27] = _mm_add_epi32(u[27], k__DCT_CONST_ROUNDING); 231291037db265ecdd914a26e056cf69207b4f50924ehkuang v[28] = _mm_add_epi32(u[28], k__DCT_CONST_ROUNDING); 231391037db265ecdd914a26e056cf69207b4f50924ehkuang v[29] = _mm_add_epi32(u[29], k__DCT_CONST_ROUNDING); 231491037db265ecdd914a26e056cf69207b4f50924ehkuang v[30] = _mm_add_epi32(u[30], k__DCT_CONST_ROUNDING); 231591037db265ecdd914a26e056cf69207b4f50924ehkuang v[31] = _mm_add_epi32(u[31], k__DCT_CONST_ROUNDING); 231691037db265ecdd914a26e056cf69207b4f50924ehkuang 231791037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); 231891037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); 231991037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); 232091037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); 232191037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); 232291037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); 232391037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); 232491037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); 232591037db265ecdd914a26e056cf69207b4f50924ehkuang u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); 232691037db265ecdd914a26e056cf69207b4f50924ehkuang u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); 232791037db265ecdd914a26e056cf69207b4f50924ehkuang u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); 232891037db265ecdd914a26e056cf69207b4f50924ehkuang u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); 232991037db265ecdd914a26e056cf69207b4f50924ehkuang u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); 233091037db265ecdd914a26e056cf69207b4f50924ehkuang u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); 233191037db265ecdd914a26e056cf69207b4f50924ehkuang u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); 233291037db265ecdd914a26e056cf69207b4f50924ehkuang u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); 233391037db265ecdd914a26e056cf69207b4f50924ehkuang u[16] = _mm_srai_epi32(v[16], DCT_CONST_BITS); 233491037db265ecdd914a26e056cf69207b4f50924ehkuang u[17] = _mm_srai_epi32(v[17], DCT_CONST_BITS); 233591037db265ecdd914a26e056cf69207b4f50924ehkuang u[18] = _mm_srai_epi32(v[18], DCT_CONST_BITS); 233691037db265ecdd914a26e056cf69207b4f50924ehkuang u[19] = _mm_srai_epi32(v[19], DCT_CONST_BITS); 233791037db265ecdd914a26e056cf69207b4f50924ehkuang u[20] = _mm_srai_epi32(v[20], DCT_CONST_BITS); 233891037db265ecdd914a26e056cf69207b4f50924ehkuang u[21] = _mm_srai_epi32(v[21], DCT_CONST_BITS); 233991037db265ecdd914a26e056cf69207b4f50924ehkuang u[22] = _mm_srai_epi32(v[22], DCT_CONST_BITS); 234091037db265ecdd914a26e056cf69207b4f50924ehkuang u[23] = _mm_srai_epi32(v[23], DCT_CONST_BITS); 234191037db265ecdd914a26e056cf69207b4f50924ehkuang u[24] = _mm_srai_epi32(v[24], DCT_CONST_BITS); 234291037db265ecdd914a26e056cf69207b4f50924ehkuang u[25] = _mm_srai_epi32(v[25], DCT_CONST_BITS); 234391037db265ecdd914a26e056cf69207b4f50924ehkuang u[26] = _mm_srai_epi32(v[26], DCT_CONST_BITS); 234491037db265ecdd914a26e056cf69207b4f50924ehkuang u[27] = _mm_srai_epi32(v[27], DCT_CONST_BITS); 234591037db265ecdd914a26e056cf69207b4f50924ehkuang u[28] = _mm_srai_epi32(v[28], DCT_CONST_BITS); 234691037db265ecdd914a26e056cf69207b4f50924ehkuang u[29] = _mm_srai_epi32(v[29], DCT_CONST_BITS); 234791037db265ecdd914a26e056cf69207b4f50924ehkuang u[30] = _mm_srai_epi32(v[30], DCT_CONST_BITS); 234891037db265ecdd914a26e056cf69207b4f50924ehkuang u[31] = _mm_srai_epi32(v[31], DCT_CONST_BITS); 234991037db265ecdd914a26e056cf69207b4f50924ehkuang 235091037db265ecdd914a26e056cf69207b4f50924ehkuang s[0] = _mm_packs_epi32(u[0], u[1]); 235191037db265ecdd914a26e056cf69207b4f50924ehkuang s[1] = _mm_packs_epi32(u[2], u[3]); 235291037db265ecdd914a26e056cf69207b4f50924ehkuang s[2] = _mm_packs_epi32(u[4], u[5]); 235391037db265ecdd914a26e056cf69207b4f50924ehkuang s[3] = _mm_packs_epi32(u[6], u[7]); 235491037db265ecdd914a26e056cf69207b4f50924ehkuang s[4] = _mm_packs_epi32(u[8], u[9]); 235591037db265ecdd914a26e056cf69207b4f50924ehkuang s[5] = _mm_packs_epi32(u[10], u[11]); 235691037db265ecdd914a26e056cf69207b4f50924ehkuang s[6] = _mm_packs_epi32(u[12], u[13]); 235791037db265ecdd914a26e056cf69207b4f50924ehkuang s[7] = _mm_packs_epi32(u[14], u[15]); 235891037db265ecdd914a26e056cf69207b4f50924ehkuang s[8] = _mm_packs_epi32(u[16], u[17]); 235991037db265ecdd914a26e056cf69207b4f50924ehkuang s[9] = _mm_packs_epi32(u[18], u[19]); 236091037db265ecdd914a26e056cf69207b4f50924ehkuang s[10] = _mm_packs_epi32(u[20], u[21]); 236191037db265ecdd914a26e056cf69207b4f50924ehkuang s[11] = _mm_packs_epi32(u[22], u[23]); 236291037db265ecdd914a26e056cf69207b4f50924ehkuang s[12] = _mm_packs_epi32(u[24], u[25]); 236391037db265ecdd914a26e056cf69207b4f50924ehkuang s[13] = _mm_packs_epi32(u[26], u[27]); 236491037db265ecdd914a26e056cf69207b4f50924ehkuang s[14] = _mm_packs_epi32(u[28], u[29]); 236591037db265ecdd914a26e056cf69207b4f50924ehkuang s[15] = _mm_packs_epi32(u[30], u[31]); 236691037db265ecdd914a26e056cf69207b4f50924ehkuang 236791037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 2 236891037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(s[8], s[9]); 236991037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpackhi_epi16(s[8], s[9]); 237091037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_unpacklo_epi16(s[10], s[11]); 237191037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_unpackhi_epi16(s[10], s[11]); 237291037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_unpacklo_epi16(s[12], s[13]); 237391037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_unpackhi_epi16(s[12], s[13]); 237491037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_unpacklo_epi16(s[14], s[15]); 237591037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_unpackhi_epi16(s[14], s[15]); 237691037db265ecdd914a26e056cf69207b4f50924ehkuang 237791037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__cospi_p04_p28); 237891037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__cospi_p04_p28); 237991037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[0], k__cospi_p28_m04); 238091037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[1], k__cospi_p28_m04); 238191037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_madd_epi16(u[2], k__cospi_p20_p12); 238291037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_madd_epi16(u[3], k__cospi_p20_p12); 238391037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_madd_epi16(u[2], k__cospi_p12_m20); 238491037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_madd_epi16(u[3], k__cospi_p12_m20); 238591037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_madd_epi16(u[4], k__cospi_m28_p04); 238691037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_madd_epi16(u[5], k__cospi_m28_p04); 238791037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_madd_epi16(u[4], k__cospi_p04_p28); 238891037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_madd_epi16(u[5], k__cospi_p04_p28); 238991037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_madd_epi16(u[6], k__cospi_m12_p20); 239091037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_madd_epi16(u[7], k__cospi_m12_p20); 239191037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_madd_epi16(u[6], k__cospi_p20_p12); 239291037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_madd_epi16(u[7], k__cospi_p20_p12); 239391037db265ecdd914a26e056cf69207b4f50924ehkuang 239491037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], v[8]); 239591037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(v[1], v[9]); 239691037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[2], v[10]); 239791037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(v[3], v[11]); 239891037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_add_epi32(v[4], v[12]); 239991037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_add_epi32(v[5], v[13]); 240091037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_add_epi32(v[6], v[14]); 240191037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_add_epi32(v[7], v[15]); 240291037db265ecdd914a26e056cf69207b4f50924ehkuang u[8] = _mm_sub_epi32(v[0], v[8]); 240391037db265ecdd914a26e056cf69207b4f50924ehkuang u[9] = _mm_sub_epi32(v[1], v[9]); 240491037db265ecdd914a26e056cf69207b4f50924ehkuang u[10] = _mm_sub_epi32(v[2], v[10]); 240591037db265ecdd914a26e056cf69207b4f50924ehkuang u[11] = _mm_sub_epi32(v[3], v[11]); 240691037db265ecdd914a26e056cf69207b4f50924ehkuang u[12] = _mm_sub_epi32(v[4], v[12]); 240791037db265ecdd914a26e056cf69207b4f50924ehkuang u[13] = _mm_sub_epi32(v[5], v[13]); 240891037db265ecdd914a26e056cf69207b4f50924ehkuang u[14] = _mm_sub_epi32(v[6], v[14]); 240991037db265ecdd914a26e056cf69207b4f50924ehkuang u[15] = _mm_sub_epi32(v[7], v[15]); 241091037db265ecdd914a26e056cf69207b4f50924ehkuang 241191037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); 241291037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); 241391037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); 241491037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); 241591037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); 241691037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); 241791037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); 241891037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); 241991037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); 242091037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); 242191037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); 242291037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); 242391037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); 242491037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); 242591037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); 242691037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); 242791037db265ecdd914a26e056cf69207b4f50924ehkuang 242891037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS); 242991037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS); 243091037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS); 243191037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS); 243291037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS); 243391037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS); 243491037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS); 243591037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS); 243691037db265ecdd914a26e056cf69207b4f50924ehkuang u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS); 243791037db265ecdd914a26e056cf69207b4f50924ehkuang u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS); 243891037db265ecdd914a26e056cf69207b4f50924ehkuang u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS); 243991037db265ecdd914a26e056cf69207b4f50924ehkuang u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS); 244091037db265ecdd914a26e056cf69207b4f50924ehkuang u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS); 244191037db265ecdd914a26e056cf69207b4f50924ehkuang u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS); 244291037db265ecdd914a26e056cf69207b4f50924ehkuang u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS); 244391037db265ecdd914a26e056cf69207b4f50924ehkuang u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS); 244491037db265ecdd914a26e056cf69207b4f50924ehkuang 244591037db265ecdd914a26e056cf69207b4f50924ehkuang x[0] = _mm_add_epi16(s[0], s[4]); 244691037db265ecdd914a26e056cf69207b4f50924ehkuang x[1] = _mm_add_epi16(s[1], s[5]); 244791037db265ecdd914a26e056cf69207b4f50924ehkuang x[2] = _mm_add_epi16(s[2], s[6]); 244891037db265ecdd914a26e056cf69207b4f50924ehkuang x[3] = _mm_add_epi16(s[3], s[7]); 244991037db265ecdd914a26e056cf69207b4f50924ehkuang x[4] = _mm_sub_epi16(s[0], s[4]); 245091037db265ecdd914a26e056cf69207b4f50924ehkuang x[5] = _mm_sub_epi16(s[1], s[5]); 245191037db265ecdd914a26e056cf69207b4f50924ehkuang x[6] = _mm_sub_epi16(s[2], s[6]); 245291037db265ecdd914a26e056cf69207b4f50924ehkuang x[7] = _mm_sub_epi16(s[3], s[7]); 245391037db265ecdd914a26e056cf69207b4f50924ehkuang x[8] = _mm_packs_epi32(u[0], u[1]); 245491037db265ecdd914a26e056cf69207b4f50924ehkuang x[9] = _mm_packs_epi32(u[2], u[3]); 245591037db265ecdd914a26e056cf69207b4f50924ehkuang x[10] = _mm_packs_epi32(u[4], u[5]); 245691037db265ecdd914a26e056cf69207b4f50924ehkuang x[11] = _mm_packs_epi32(u[6], u[7]); 245791037db265ecdd914a26e056cf69207b4f50924ehkuang x[12] = _mm_packs_epi32(u[8], u[9]); 245891037db265ecdd914a26e056cf69207b4f50924ehkuang x[13] = _mm_packs_epi32(u[10], u[11]); 245991037db265ecdd914a26e056cf69207b4f50924ehkuang x[14] = _mm_packs_epi32(u[12], u[13]); 246091037db265ecdd914a26e056cf69207b4f50924ehkuang x[15] = _mm_packs_epi32(u[14], u[15]); 246191037db265ecdd914a26e056cf69207b4f50924ehkuang 246291037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 3 246391037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(x[4], x[5]); 246491037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpackhi_epi16(x[4], x[5]); 246591037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_unpacklo_epi16(x[6], x[7]); 246691037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_unpackhi_epi16(x[6], x[7]); 246791037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_unpacklo_epi16(x[12], x[13]); 246891037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_unpackhi_epi16(x[12], x[13]); 246991037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_unpacklo_epi16(x[14], x[15]); 247091037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_unpackhi_epi16(x[14], x[15]); 247191037db265ecdd914a26e056cf69207b4f50924ehkuang 247291037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__cospi_p08_p24); 247391037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__cospi_p08_p24); 247491037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[0], k__cospi_p24_m08); 247591037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[1], k__cospi_p24_m08); 247691037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_madd_epi16(u[2], k__cospi_m24_p08); 247791037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_madd_epi16(u[3], k__cospi_m24_p08); 247891037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_madd_epi16(u[2], k__cospi_p08_p24); 247991037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_madd_epi16(u[3], k__cospi_p08_p24); 248091037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_madd_epi16(u[4], k__cospi_p08_p24); 248191037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_madd_epi16(u[5], k__cospi_p08_p24); 248291037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_madd_epi16(u[4], k__cospi_p24_m08); 248391037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_madd_epi16(u[5], k__cospi_p24_m08); 248491037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_madd_epi16(u[6], k__cospi_m24_p08); 248591037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_madd_epi16(u[7], k__cospi_m24_p08); 248691037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_madd_epi16(u[6], k__cospi_p08_p24); 248791037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_madd_epi16(u[7], k__cospi_p08_p24); 248891037db265ecdd914a26e056cf69207b4f50924ehkuang 248991037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], v[4]); 249091037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(v[1], v[5]); 249191037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[2], v[6]); 249291037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(v[3], v[7]); 249391037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_sub_epi32(v[0], v[4]); 249491037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_sub_epi32(v[1], v[5]); 249591037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_sub_epi32(v[2], v[6]); 249691037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_sub_epi32(v[3], v[7]); 249791037db265ecdd914a26e056cf69207b4f50924ehkuang u[8] = _mm_add_epi32(v[8], v[12]); 249891037db265ecdd914a26e056cf69207b4f50924ehkuang u[9] = _mm_add_epi32(v[9], v[13]); 249991037db265ecdd914a26e056cf69207b4f50924ehkuang u[10] = _mm_add_epi32(v[10], v[14]); 250091037db265ecdd914a26e056cf69207b4f50924ehkuang u[11] = _mm_add_epi32(v[11], v[15]); 250191037db265ecdd914a26e056cf69207b4f50924ehkuang u[12] = _mm_sub_epi32(v[8], v[12]); 250291037db265ecdd914a26e056cf69207b4f50924ehkuang u[13] = _mm_sub_epi32(v[9], v[13]); 250391037db265ecdd914a26e056cf69207b4f50924ehkuang u[14] = _mm_sub_epi32(v[10], v[14]); 250491037db265ecdd914a26e056cf69207b4f50924ehkuang u[15] = _mm_sub_epi32(v[11], v[15]); 250591037db265ecdd914a26e056cf69207b4f50924ehkuang 250691037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING); 250791037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING); 250891037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING); 250991037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING); 251091037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING); 251191037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING); 251291037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING); 251391037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING); 251491037db265ecdd914a26e056cf69207b4f50924ehkuang u[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING); 251591037db265ecdd914a26e056cf69207b4f50924ehkuang u[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING); 251691037db265ecdd914a26e056cf69207b4f50924ehkuang u[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING); 251791037db265ecdd914a26e056cf69207b4f50924ehkuang u[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING); 251891037db265ecdd914a26e056cf69207b4f50924ehkuang u[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING); 251991037db265ecdd914a26e056cf69207b4f50924ehkuang u[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING); 252091037db265ecdd914a26e056cf69207b4f50924ehkuang u[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING); 252191037db265ecdd914a26e056cf69207b4f50924ehkuang u[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING); 252291037db265ecdd914a26e056cf69207b4f50924ehkuang 252391037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); 252491037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); 252591037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); 252691037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); 252791037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); 252891037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); 252991037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); 253091037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); 253191037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); 253291037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); 253391037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); 253491037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); 253591037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); 253691037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); 253791037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); 253891037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); 253991037db265ecdd914a26e056cf69207b4f50924ehkuang 254091037db265ecdd914a26e056cf69207b4f50924ehkuang s[0] = _mm_add_epi16(x[0], x[2]); 254191037db265ecdd914a26e056cf69207b4f50924ehkuang s[1] = _mm_add_epi16(x[1], x[3]); 254291037db265ecdd914a26e056cf69207b4f50924ehkuang s[2] = _mm_sub_epi16(x[0], x[2]); 254391037db265ecdd914a26e056cf69207b4f50924ehkuang s[3] = _mm_sub_epi16(x[1], x[3]); 254491037db265ecdd914a26e056cf69207b4f50924ehkuang s[4] = _mm_packs_epi32(v[0], v[1]); 254591037db265ecdd914a26e056cf69207b4f50924ehkuang s[5] = _mm_packs_epi32(v[2], v[3]); 254691037db265ecdd914a26e056cf69207b4f50924ehkuang s[6] = _mm_packs_epi32(v[4], v[5]); 254791037db265ecdd914a26e056cf69207b4f50924ehkuang s[7] = _mm_packs_epi32(v[6], v[7]); 254891037db265ecdd914a26e056cf69207b4f50924ehkuang s[8] = _mm_add_epi16(x[8], x[10]); 254991037db265ecdd914a26e056cf69207b4f50924ehkuang s[9] = _mm_add_epi16(x[9], x[11]); 255091037db265ecdd914a26e056cf69207b4f50924ehkuang s[10] = _mm_sub_epi16(x[8], x[10]); 255191037db265ecdd914a26e056cf69207b4f50924ehkuang s[11] = _mm_sub_epi16(x[9], x[11]); 255291037db265ecdd914a26e056cf69207b4f50924ehkuang s[12] = _mm_packs_epi32(v[8], v[9]); 255391037db265ecdd914a26e056cf69207b4f50924ehkuang s[13] = _mm_packs_epi32(v[10], v[11]); 255491037db265ecdd914a26e056cf69207b4f50924ehkuang s[14] = _mm_packs_epi32(v[12], v[13]); 255591037db265ecdd914a26e056cf69207b4f50924ehkuang s[15] = _mm_packs_epi32(v[14], v[15]); 255691037db265ecdd914a26e056cf69207b4f50924ehkuang 255791037db265ecdd914a26e056cf69207b4f50924ehkuang // stage 4 255891037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_unpacklo_epi16(s[2], s[3]); 255991037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_unpackhi_epi16(s[2], s[3]); 256091037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_unpacklo_epi16(s[6], s[7]); 256191037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_unpackhi_epi16(s[6], s[7]); 256291037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_unpacklo_epi16(s[10], s[11]); 256391037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_unpackhi_epi16(s[10], s[11]); 256491037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_unpacklo_epi16(s[14], s[15]); 256591037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_unpackhi_epi16(s[14], s[15]); 256691037db265ecdd914a26e056cf69207b4f50924ehkuang 256791037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_madd_epi16(u[0], k__cospi_m16_m16); 256891037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_madd_epi16(u[1], k__cospi_m16_m16); 256991037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_madd_epi16(u[0], k__cospi_p16_m16); 257091037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_madd_epi16(u[1], k__cospi_p16_m16); 257191037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_madd_epi16(u[2], k__cospi_p16_p16); 257291037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_madd_epi16(u[3], k__cospi_p16_p16); 257391037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_madd_epi16(u[2], k__cospi_m16_p16); 257491037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_madd_epi16(u[3], k__cospi_m16_p16); 257591037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_madd_epi16(u[4], k__cospi_p16_p16); 257691037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_madd_epi16(u[5], k__cospi_p16_p16); 257791037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_madd_epi16(u[4], k__cospi_m16_p16); 257891037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_madd_epi16(u[5], k__cospi_m16_p16); 257991037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_madd_epi16(u[6], k__cospi_m16_m16); 258091037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_madd_epi16(u[7], k__cospi_m16_m16); 258191037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_madd_epi16(u[6], k__cospi_p16_m16); 258291037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_madd_epi16(u[7], k__cospi_p16_m16); 258391037db265ecdd914a26e056cf69207b4f50924ehkuang 258491037db265ecdd914a26e056cf69207b4f50924ehkuang u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING); 258591037db265ecdd914a26e056cf69207b4f50924ehkuang u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING); 258691037db265ecdd914a26e056cf69207b4f50924ehkuang u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING); 258791037db265ecdd914a26e056cf69207b4f50924ehkuang u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING); 258891037db265ecdd914a26e056cf69207b4f50924ehkuang u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING); 258991037db265ecdd914a26e056cf69207b4f50924ehkuang u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING); 259091037db265ecdd914a26e056cf69207b4f50924ehkuang u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING); 259191037db265ecdd914a26e056cf69207b4f50924ehkuang u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING); 259291037db265ecdd914a26e056cf69207b4f50924ehkuang u[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING); 259391037db265ecdd914a26e056cf69207b4f50924ehkuang u[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING); 259491037db265ecdd914a26e056cf69207b4f50924ehkuang u[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING); 259591037db265ecdd914a26e056cf69207b4f50924ehkuang u[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING); 259691037db265ecdd914a26e056cf69207b4f50924ehkuang u[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING); 259791037db265ecdd914a26e056cf69207b4f50924ehkuang u[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING); 259891037db265ecdd914a26e056cf69207b4f50924ehkuang u[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING); 259991037db265ecdd914a26e056cf69207b4f50924ehkuang u[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING); 260091037db265ecdd914a26e056cf69207b4f50924ehkuang 260191037db265ecdd914a26e056cf69207b4f50924ehkuang v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS); 260291037db265ecdd914a26e056cf69207b4f50924ehkuang v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS); 260391037db265ecdd914a26e056cf69207b4f50924ehkuang v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS); 260491037db265ecdd914a26e056cf69207b4f50924ehkuang v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS); 260591037db265ecdd914a26e056cf69207b4f50924ehkuang v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS); 260691037db265ecdd914a26e056cf69207b4f50924ehkuang v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS); 260791037db265ecdd914a26e056cf69207b4f50924ehkuang v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS); 260891037db265ecdd914a26e056cf69207b4f50924ehkuang v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS); 260991037db265ecdd914a26e056cf69207b4f50924ehkuang v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS); 261091037db265ecdd914a26e056cf69207b4f50924ehkuang v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS); 261191037db265ecdd914a26e056cf69207b4f50924ehkuang v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS); 261291037db265ecdd914a26e056cf69207b4f50924ehkuang v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS); 261391037db265ecdd914a26e056cf69207b4f50924ehkuang v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS); 261491037db265ecdd914a26e056cf69207b4f50924ehkuang v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS); 261591037db265ecdd914a26e056cf69207b4f50924ehkuang v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS); 261691037db265ecdd914a26e056cf69207b4f50924ehkuang v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS); 261791037db265ecdd914a26e056cf69207b4f50924ehkuang 261891037db265ecdd914a26e056cf69207b4f50924ehkuang in[0] = s[0]; 261991037db265ecdd914a26e056cf69207b4f50924ehkuang in[1] = _mm_sub_epi16(kZero, s[8]); 262091037db265ecdd914a26e056cf69207b4f50924ehkuang in[2] = s[12]; 262191037db265ecdd914a26e056cf69207b4f50924ehkuang in[3] = _mm_sub_epi16(kZero, s[4]); 262291037db265ecdd914a26e056cf69207b4f50924ehkuang in[4] = _mm_packs_epi32(v[4], v[5]); 262391037db265ecdd914a26e056cf69207b4f50924ehkuang in[5] = _mm_packs_epi32(v[12], v[13]); 262491037db265ecdd914a26e056cf69207b4f50924ehkuang in[6] = _mm_packs_epi32(v[8], v[9]); 262591037db265ecdd914a26e056cf69207b4f50924ehkuang in[7] = _mm_packs_epi32(v[0], v[1]); 262691037db265ecdd914a26e056cf69207b4f50924ehkuang in[8] = _mm_packs_epi32(v[2], v[3]); 262791037db265ecdd914a26e056cf69207b4f50924ehkuang in[9] = _mm_packs_epi32(v[10], v[11]); 262891037db265ecdd914a26e056cf69207b4f50924ehkuang in[10] = _mm_packs_epi32(v[14], v[15]); 262991037db265ecdd914a26e056cf69207b4f50924ehkuang in[11] = _mm_packs_epi32(v[6], v[7]); 263091037db265ecdd914a26e056cf69207b4f50924ehkuang in[12] = s[5]; 263191037db265ecdd914a26e056cf69207b4f50924ehkuang in[13] = _mm_sub_epi16(kZero, s[13]); 263291037db265ecdd914a26e056cf69207b4f50924ehkuang in[14] = s[9]; 263391037db265ecdd914a26e056cf69207b4f50924ehkuang in[15] = _mm_sub_epi16(kZero, s[1]); 263491037db265ecdd914a26e056cf69207b4f50924ehkuang} 263591037db265ecdd914a26e056cf69207b4f50924ehkuang 2636b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid fdct16_sse2(__m128i *in0, __m128i *in1) { 2637b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fdct16_8col(in0); 2638b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fdct16_8col(in1); 263991037db265ecdd914a26e056cf69207b4f50924ehkuang array_transpose_16x16(in0, in1); 264091037db265ecdd914a26e056cf69207b4f50924ehkuang} 264191037db265ecdd914a26e056cf69207b4f50924ehkuang 2642b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid fadst16_sse2(__m128i *in0, __m128i *in1) { 2643b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst16_8col(in0); 2644b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst16_8col(in1); 264591037db265ecdd914a26e056cf69207b4f50924ehkuang array_transpose_16x16(in0, in1); 264691037db265ecdd914a26e056cf69207b4f50924ehkuang} 264791037db265ecdd914a26e056cf69207b4f50924ehkuang 2648b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanianvoid vp9_fht16x16_sse2(const int16_t *input, int16_t *output, 2649b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian int stride, int tx_type) { 265091037db265ecdd914a26e056cf69207b4f50924ehkuang __m128i in0[16], in1[16]; 2651b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian 265291037db265ecdd914a26e056cf69207b4f50924ehkuang switch (tx_type) { 2653b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case DCT_DCT: 2654b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian vp9_fdct16x16_sse2(input, output, stride); 265591037db265ecdd914a26e056cf69207b4f50924ehkuang break; 2656b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case ADST_DCT: 2657b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian load_buffer_16x16(input, in0, in1, stride); 2658b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst16_sse2(in0, in1); 265991037db265ecdd914a26e056cf69207b4f50924ehkuang right_shift_16x16(in0, in1); 2660b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fdct16_sse2(in0, in1); 2661b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian write_buffer_16x16(output, in0, in1, 16); 266291037db265ecdd914a26e056cf69207b4f50924ehkuang break; 2663b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case DCT_ADST: 2664b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian load_buffer_16x16(input, in0, in1, stride); 2665b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fdct16_sse2(in0, in1); 266691037db265ecdd914a26e056cf69207b4f50924ehkuang right_shift_16x16(in0, in1); 2667b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst16_sse2(in0, in1); 2668b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian write_buffer_16x16(output, in0, in1, 16); 266991037db265ecdd914a26e056cf69207b4f50924ehkuang break; 2670b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian case ADST_ADST: 2671b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian load_buffer_16x16(input, in0, in1, stride); 2672b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst16_sse2(in0, in1); 267391037db265ecdd914a26e056cf69207b4f50924ehkuang right_shift_16x16(in0, in1); 2674b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian fadst16_sse2(in0, in1); 2675b08e2e23eec181e9951df33cd704ac294c5407b6Vignesh Venkatasubramanian write_buffer_16x16(output, in0, in1, 16); 267691037db265ecdd914a26e056cf69207b4f50924ehkuang break; 267791037db265ecdd914a26e056cf69207b4f50924ehkuang default: 267891037db265ecdd914a26e056cf69207b4f50924ehkuang assert(0); 267991037db265ecdd914a26e056cf69207b4f50924ehkuang break; 268091037db265ecdd914a26e056cf69207b4f50924ehkuang } 268191037db265ecdd914a26e056cf69207b4f50924ehkuang} 268291037db265ecdd914a26e056cf69207b4f50924ehkuang 26835ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang#define FDCT32x32_2D vp9_fdct32x32_rd_sse2 26841184aebb761cbeac9124c37189a80a1a58f04b6bhkuang#define FDCT32x32_HIGH_PRECISION 0 26851184aebb761cbeac9124c37189a80a1a58f04b6bhkuang#include "vp9/encoder/x86/vp9_dct32x32_sse2.c" 26861184aebb761cbeac9124c37189a80a1a58f04b6bhkuang#undef FDCT32x32_2D 26871184aebb761cbeac9124c37189a80a1a58f04b6bhkuang#undef FDCT32x32_HIGH_PRECISION 26881184aebb761cbeac9124c37189a80a1a58f04b6bhkuang 26895ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang#define FDCT32x32_2D vp9_fdct32x32_sse2 26901184aebb761cbeac9124c37189a80a1a58f04b6bhkuang#define FDCT32x32_HIGH_PRECISION 1 26911184aebb761cbeac9124c37189a80a1a58f04b6bhkuang#include "vp9/encoder/x86/vp9_dct32x32_sse2.c" // NOLINT 26921184aebb761cbeac9124c37189a80a1a58f04b6bhkuang#undef FDCT32x32_2D 26931184aebb761cbeac9124c37189a80a1a58f04b6bhkuang#undef FDCT32x32_HIGH_PRECISION 2694