1df37111358d02836cb29bbcb9c6e4c95dff90a16Johann/* 2df37111358d02836cb29bbcb9c6e4c95dff90a16Johann * Copyright (c) 2017 The WebM project authors. All Rights Reserved. 3df37111358d02836cb29bbcb9c6e4c95dff90a16Johann * 4df37111358d02836cb29bbcb9c6e4c95dff90a16Johann * Use of this source code is governed by a BSD-style license 5df37111358d02836cb29bbcb9c6e4c95dff90a16Johann * that can be found in the LICENSE file in the root of the source 6df37111358d02836cb29bbcb9c6e4c95dff90a16Johann * tree. An additional intellectual property rights grant can be found 7df37111358d02836cb29bbcb9c6e4c95dff90a16Johann * in the file PATENTS. All contributing project authors may 8df37111358d02836cb29bbcb9c6e4c95dff90a16Johann * be found in the AUTHORS file in the root of the source tree. 9df37111358d02836cb29bbcb9c6e4c95dff90a16Johann */ 10df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 11df37111358d02836cb29bbcb9c6e4c95dff90a16Johann#include <arm_neon.h> 12df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 13df37111358d02836cb29bbcb9c6e4c95dff90a16Johann#include "./vpx_config.h" 14df37111358d02836cb29bbcb9c6e4c95dff90a16Johann#include "./vpx_dsp_rtcd.h" 15df37111358d02836cb29bbcb9c6e4c95dff90a16Johann#include "vpx_dsp/txfm_common.h" 16df37111358d02836cb29bbcb9c6e4c95dff90a16Johann#include "vpx_dsp/arm/mem_neon.h" 17df37111358d02836cb29bbcb9c6e4c95dff90a16Johann#include "vpx_dsp/arm/transpose_neon.h" 18df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 19df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// Some builds of gcc 4.9.2 and .3 have trouble with some of the inline 20df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// functions. 21df37111358d02836cb29bbcb9c6e4c95dff90a16Johann#if !defined(__clang__) && !defined(__ANDROID__) && defined(__GNUC__) && \ 22df37111358d02836cb29bbcb9c6e4c95dff90a16Johann __GNUC__ == 4 && __GNUC_MINOR__ == 9 && __GNUC_PATCHLEVEL__ < 4 23df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 24df37111358d02836cb29bbcb9c6e4c95dff90a16Johannvoid vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) { 25df37111358d02836cb29bbcb9c6e4c95dff90a16Johann vpx_fdct16x16_c(input, output, stride); 26df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 27df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 28df37111358d02836cb29bbcb9c6e4c95dff90a16Johann#else 29df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 30df37111358d02836cb29bbcb9c6e4c95dff90a16Johannstatic INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) { 31df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[0] = vld1q_s16(a); 32df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 33df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[1] = vld1q_s16(a); 34df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 35df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[2] = vld1q_s16(a); 36df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 37df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[3] = vld1q_s16(a); 38df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 39df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[4] = vld1q_s16(a); 40df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 41df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[5] = vld1q_s16(a); 42df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 43df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[6] = vld1q_s16(a); 44df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 45df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[7] = vld1q_s16(a); 46df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 47df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[8] = vld1q_s16(a); 48df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 49df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[9] = vld1q_s16(a); 50df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 51df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[10] = vld1q_s16(a); 52df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 53df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[11] = vld1q_s16(a); 54df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 55df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[12] = vld1q_s16(a); 56df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 57df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[13] = vld1q_s16(a); 58df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 59df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[14] = vld1q_s16(a); 60df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += stride; 61df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[15] = vld1q_s16(a); 62df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 63df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 64df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// Store 8 16x8 values, assuming stride == 16. 65df37111358d02836cb29bbcb9c6e4c95dff90a16Johannstatic INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) { 66df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store_s16q_to_tran_low(a, b[0]); 67df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += 16; 68df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store_s16q_to_tran_low(a, b[1]); 69df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += 16; 70df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store_s16q_to_tran_low(a, b[2]); 71df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += 16; 72df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store_s16q_to_tran_low(a, b[3]); 73df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += 16; 74df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store_s16q_to_tran_low(a, b[4]); 75df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += 16; 76df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store_s16q_to_tran_low(a, b[5]); 77df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += 16; 78df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store_s16q_to_tran_low(a, b[6]); 79df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a += 16; 80df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store_s16q_to_tran_low(a, b[7]); 81df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 82df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 83df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// Load step of each pass. Add and subtract clear across the input, requiring 84df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// all 16 values to be loaded. For the first pass it also multiplies by 4. 85df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 86df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// To maybe reduce register usage this could be combined with the load() step to 87df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// get the first 4 and last 4 values, cross those, then load the middle 8 values 88df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// and cross them. 89df37111358d02836cb29bbcb9c6e4c95dff90a16Johannstatic INLINE void cross_input(const int16x8_t *a /*[16]*/, 90df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t *b /*[16]*/, const int pass) { 91df37111358d02836cb29bbcb9c6e4c95dff90a16Johann if (pass == 0) { 92df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2); 93df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2); 94df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2); 95df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2); 96df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2); 97df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2); 98df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2); 99df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2); 100df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 101df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2); 102df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2); 103df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2); 104df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2); 105df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2); 106df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2); 107df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2); 108df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2); 109df37111358d02836cb29bbcb9c6e4c95dff90a16Johann } else { 110df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[0] = vaddq_s16(a[0], a[15]); 111df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[1] = vaddq_s16(a[1], a[14]); 112df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[2] = vaddq_s16(a[2], a[13]); 113df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[3] = vaddq_s16(a[3], a[12]); 114df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[4] = vaddq_s16(a[4], a[11]); 115df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[5] = vaddq_s16(a[5], a[10]); 116df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[6] = vaddq_s16(a[6], a[9]); 117df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[7] = vaddq_s16(a[7], a[8]); 118df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 119df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[8] = vsubq_s16(a[7], a[8]); 120df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[9] = vsubq_s16(a[6], a[9]); 121df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[10] = vsubq_s16(a[5], a[10]); 122df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[11] = vsubq_s16(a[4], a[11]); 123df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[12] = vsubq_s16(a[3], a[12]); 124df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[13] = vsubq_s16(a[2], a[13]); 125df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[14] = vsubq_s16(a[1], a[14]); 126df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[15] = vsubq_s16(a[0], a[15]); 127df37111358d02836cb29bbcb9c6e4c95dff90a16Johann } 128df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 129df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 130df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// Quarter round at the beginning of the second pass. Can't use vrshr (rounding) 131df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// because this only adds 1, not 1 << 2. 132df37111358d02836cb29bbcb9c6e4c95dff90a16Johannstatic INLINE void partial_round_shift(int16x8_t *a /*[16]*/) { 133df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x8_t one = vdupq_n_s16(1); 134df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2); 135df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2); 136df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2); 137df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2); 138df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2); 139df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2); 140df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2); 141df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2); 142df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2); 143df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2); 144df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2); 145df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2); 146df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2); 147df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2); 148df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2); 149df37111358d02836cb29bbcb9c6e4c95dff90a16Johann a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2); 150df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 151df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 152df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// fdct_round_shift((a +/- b) * c) 153df37111358d02836cb29bbcb9c6e4c95dff90a16Johannstatic INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b, 154df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const tran_high_t c, int16x8_t *add, 155df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t *sub) { 156df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c); 157df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c); 158df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c); 159df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c); 160df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c); 161df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c); 162df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); 163df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); 164df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); 165df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); 166df37111358d02836cb29bbcb9c6e4c95dff90a16Johann *add = vcombine_s16(rounded0, rounded1); 167df37111358d02836cb29bbcb9c6e4c95dff90a16Johann *sub = vcombine_s16(rounded2, rounded3); 168df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 169df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 170df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// fdct_round_shift(a * c0 +/- b * c1) 171df37111358d02836cb29bbcb9c6e4c95dff90a16Johannstatic INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b, 172df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const tran_coef_t c0, 173df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const tran_coef_t c1, int16x8_t *add, 174df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t *sub) { 175df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0); 176df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0); 177df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1); 178df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1); 179df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0); 180df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0); 181df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1); 182df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1); 183df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); 184df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); 185df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); 186df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); 187df37111358d02836cb29bbcb9c6e4c95dff90a16Johann *add = vcombine_s16(rounded0, rounded1); 188df37111358d02836cb29bbcb9c6e4c95dff90a16Johann *sub = vcombine_s16(rounded2, rounded3); 189df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 190df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 191df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// Transpose 8x8 to a new location. Don't use transpose_neon.h because those 192df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// are all in-place. 193df37111358d02836cb29bbcb9c6e4c95dff90a16Johannstatic INLINE void transpose_8x8(const int16x8_t *a /*[8]*/, 194df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t *b /*[8]*/) { 195df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Swap 16 bit elements. 196df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]); 197df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]); 198df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]); 199df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]); 200df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 201df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Swap 32 bit elements. 202df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]), 203df37111358d02836cb29bbcb9c6e4c95dff90a16Johann vreinterpretq_s32_s16(c1.val[0])); 204df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]), 205df37111358d02836cb29bbcb9c6e4c95dff90a16Johann vreinterpretq_s32_s16(c1.val[1])); 206df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]), 207df37111358d02836cb29bbcb9c6e4c95dff90a16Johann vreinterpretq_s32_s16(c3.val[0])); 208df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]), 209df37111358d02836cb29bbcb9c6e4c95dff90a16Johann vreinterpretq_s32_s16(c3.val[1])); 210df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 211df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Swap 64 bit elements 212df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]); 213df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]); 214df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]); 215df37111358d02836cb29bbcb9c6e4c95dff90a16Johann const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]); 216df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 217df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[0] = e0.val[0]; 218df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[1] = e1.val[0]; 219df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[2] = e2.val[0]; 220df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[3] = e3.val[0]; 221df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[4] = e0.val[1]; 222df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[5] = e1.val[1]; 223df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[6] = e2.val[1]; 224df37111358d02836cb29bbcb9c6e4c95dff90a16Johann b[7] = e3.val[1]; 225df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 226df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 227df37111358d02836cb29bbcb9c6e4c95dff90a16Johann// Main body of fdct16x16. 228df37111358d02836cb29bbcb9c6e4c95dff90a16Johannstatic void dct_body(const int16x8_t *in /*[16]*/, int16x8_t *out /*[16]*/) { 229df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t s[8]; 230df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t x[4]; 231df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t step[8]; 232df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 233df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // stage 1 234df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // From fwd_txfm.c: Work on the first eight values; fdct8(input, 235df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // even_results);" 236df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[0] = vaddq_s16(in[0], in[7]); 237df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[1] = vaddq_s16(in[1], in[6]); 238df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[2] = vaddq_s16(in[2], in[5]); 239df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[3] = vaddq_s16(in[3], in[4]); 240df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[4] = vsubq_s16(in[3], in[4]); 241df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[5] = vsubq_s16(in[2], in[5]); 242df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[6] = vsubq_s16(in[1], in[6]); 243df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[7] = vsubq_s16(in[0], in[7]); 244df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 245df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // fdct4(step, step); 246df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[0] = vaddq_s16(s[0], s[3]); 247df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[1] = vaddq_s16(s[1], s[2]); 248df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[2] = vsubq_s16(s[1], s[2]); 249df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[3] = vsubq_s16(s[0], s[3]); 250df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 251df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[0] = fdct_round_shift((x0 + x1) * cospi_16_64) 252df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[8] = fdct_round_shift((x0 - x1) * cospi_16_64) 253df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]); 254df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64); 255df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64); 256df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]); 257df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 258df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Stage 2 259df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Re-using source s5/s6 260df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // s5 = fdct_round_shift((s6 - s5) * cospi_16_64) 261df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // s6 = fdct_round_shift((s6 + s5) * cospi_16_64) 262df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]); 263df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 264df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Stage 3 265df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[0] = vaddq_s16(s[4], s[5]); 266df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[1] = vsubq_s16(s[4], s[5]); 267df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[2] = vsubq_s16(s[7], s[6]); 268df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[3] = vaddq_s16(s[7], s[6]); 269df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 270df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Stage 4 271df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64) 272df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64) 273df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]); 274df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64) 275df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64) 276df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]); 277df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 278df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step 2 279df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // From fwd_txfm.c: Work on the next eight values; step1 -> odd_results" 280df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // That file distinguished between "in_high" and "step1" but the only 281df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // difference is that "in_high" is the first 8 values and "step 1" is the 282df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // second. Here, since they are all in one array, "step1" values are += 8. 283df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 284df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64) 285df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64) 286df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64) 287df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64) 288df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]); 289df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]); 290df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 291df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step 3 292df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[0] = vaddq_s16(in[8], s[3]); 293df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[1] = vaddq_s16(in[9], s[2]); 294df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[0] = vsubq_s16(in[9], s[2]); 295df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[1] = vsubq_s16(in[8], s[3]); 296df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[2] = vsubq_s16(in[15], s[4]); 297df37111358d02836cb29bbcb9c6e4c95dff90a16Johann x[3] = vsubq_s16(in[14], s[5]); 298df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[6] = vaddq_s16(in[14], s[5]); 299df37111358d02836cb29bbcb9c6e4c95dff90a16Johann s[7] = vaddq_s16(in[15], s[4]); 300df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 301df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step 4 302df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64) 303df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64) 304df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]); 305df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 306df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64) 307df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64) 308df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]); 309df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 310df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step 5 311df37111358d02836cb29bbcb9c6e4c95dff90a16Johann step[0] = vaddq_s16(s[0], s[1]); 312df37111358d02836cb29bbcb9c6e4c95dff90a16Johann step[1] = vsubq_s16(s[0], s[1]); 313df37111358d02836cb29bbcb9c6e4c95dff90a16Johann step[2] = vaddq_s16(x[1], s[2]); 314df37111358d02836cb29bbcb9c6e4c95dff90a16Johann step[3] = vsubq_s16(x[1], s[2]); 315df37111358d02836cb29bbcb9c6e4c95dff90a16Johann step[4] = vsubq_s16(x[2], s[5]); 316df37111358d02836cb29bbcb9c6e4c95dff90a16Johann step[5] = vaddq_s16(x[2], s[5]); 317df37111358d02836cb29bbcb9c6e4c95dff90a16Johann step[6] = vsubq_s16(s[7], s[6]); 318df37111358d02836cb29bbcb9c6e4c95dff90a16Johann step[7] = vaddq_s16(s[7], s[6]); 319df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 320df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // step 6 321df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64) 322df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64) 323df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64) 324df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64) 325df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64) 326df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] * 327df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // cospi_22_64) 328df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64) 329df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64) 330df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9], 331df37111358d02836cb29bbcb9c6e4c95dff90a16Johann &out[7]); 332df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1], 333df37111358d02836cb29bbcb9c6e4c95dff90a16Johann &out[15]); 334df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13], 335df37111358d02836cb29bbcb9c6e4c95dff90a16Johann &out[3]); 336df37111358d02836cb29bbcb9c6e4c95dff90a16Johann butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5], 337df37111358d02836cb29bbcb9c6e4c95dff90a16Johann &out[11]); 338df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 339df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 340df37111358d02836cb29bbcb9c6e4c95dff90a16Johannvoid vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) { 341df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t temp0[16]; 342df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t temp1[16]; 343df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t temp2[16]; 344df37111358d02836cb29bbcb9c6e4c95dff90a16Johann int16x8_t temp3[16]; 345df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 346df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Left half. 347df37111358d02836cb29bbcb9c6e4c95dff90a16Johann load(input, stride, temp0); 348df37111358d02836cb29bbcb9c6e4c95dff90a16Johann cross_input(temp0, temp1, 0); 349df37111358d02836cb29bbcb9c6e4c95dff90a16Johann dct_body(temp1, temp0); 350df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 351df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Right half. 352df37111358d02836cb29bbcb9c6e4c95dff90a16Johann load(input + 8, stride, temp1); 353df37111358d02836cb29bbcb9c6e4c95dff90a16Johann cross_input(temp1, temp2, 0); 354df37111358d02836cb29bbcb9c6e4c95dff90a16Johann dct_body(temp2, temp1); 355df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 356df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Transpose top left and top right quarters into one contiguous location to 357df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // process to the top half. 358df37111358d02836cb29bbcb9c6e4c95dff90a16Johann transpose_8x8(&temp0[0], &temp2[0]); 359df37111358d02836cb29bbcb9c6e4c95dff90a16Johann transpose_8x8(&temp1[0], &temp2[8]); 360df37111358d02836cb29bbcb9c6e4c95dff90a16Johann partial_round_shift(temp2); 361df37111358d02836cb29bbcb9c6e4c95dff90a16Johann cross_input(temp2, temp3, 1); 362df37111358d02836cb29bbcb9c6e4c95dff90a16Johann dct_body(temp3, temp2); 363df37111358d02836cb29bbcb9c6e4c95dff90a16Johann transpose_s16_8x8(&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4], 364df37111358d02836cb29bbcb9c6e4c95dff90a16Johann &temp2[5], &temp2[6], &temp2[7]); 365df37111358d02836cb29bbcb9c6e4c95dff90a16Johann transpose_s16_8x8(&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12], 366df37111358d02836cb29bbcb9c6e4c95dff90a16Johann &temp2[13], &temp2[14], &temp2[15]); 367df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store(output, temp2); 368df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store(output + 8, temp2 + 8); 369df37111358d02836cb29bbcb9c6e4c95dff90a16Johann output += 8 * 16; 370df37111358d02836cb29bbcb9c6e4c95dff90a16Johann 371df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // Transpose bottom left and bottom right quarters into one contiguous 372df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // location to process to the bottom half. 373df37111358d02836cb29bbcb9c6e4c95dff90a16Johann transpose_8x8(&temp0[8], &temp1[0]); 374df37111358d02836cb29bbcb9c6e4c95dff90a16Johann transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12], 375df37111358d02836cb29bbcb9c6e4c95dff90a16Johann &temp1[13], &temp1[14], &temp1[15]); 376df37111358d02836cb29bbcb9c6e4c95dff90a16Johann partial_round_shift(temp1); 377df37111358d02836cb29bbcb9c6e4c95dff90a16Johann cross_input(temp1, temp0, 1); 378df37111358d02836cb29bbcb9c6e4c95dff90a16Johann dct_body(temp0, temp1); 379df37111358d02836cb29bbcb9c6e4c95dff90a16Johann transpose_s16_8x8(&temp1[0], &temp1[1], &temp1[2], &temp1[3], &temp1[4], 380df37111358d02836cb29bbcb9c6e4c95dff90a16Johann &temp1[5], &temp1[6], &temp1[7]); 381df37111358d02836cb29bbcb9c6e4c95dff90a16Johann transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12], 382df37111358d02836cb29bbcb9c6e4c95dff90a16Johann &temp1[13], &temp1[14], &temp1[15]); 383df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store(output, temp1); 384df37111358d02836cb29bbcb9c6e4c95dff90a16Johann store(output + 8, temp1 + 8); 385df37111358d02836cb29bbcb9c6e4c95dff90a16Johann} 386df37111358d02836cb29bbcb9c6e4c95dff90a16Johann#endif // !defined(__clang__) && !defined(__ANDROID__) && defined(__GNUC__) && 387df37111358d02836cb29bbcb9c6e4c95dff90a16Johann // __GNUC__ == 4 && __GNUC_MINOR__ == 9 && __GNUC_PATCHLEVEL__ < 4 388