idct4x4_add_neon.c revision 7ce0a1d1337c01056ba24006efab21f00e179e04 
1/*
2 *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include <arm_neon.h>
12
13void vpx_idct4x4_16_add_neon(
14        int16_t *input,
15        uint8_t *dest,
16        int dest_stride) {
17    uint8x8_t d26u8, d27u8;
18    uint32x2_t d26u32, d27u32;
19    uint16x8_t q8u16, q9u16;
20    int16x4_t d16s16, d17s16, d18s16, d19s16, d20s16, d21s16;
21    int16x4_t d22s16, d23s16, d24s16, d26s16, d27s16, d28s16, d29s16;
22    int16x8_t q8s16, q9s16, q13s16, q14s16;
23    int32x4_t q1s32, q13s32, q14s32, q15s32;
24    int16x4x2_t d0x2s16, d1x2s16;
25    int32x4x2_t q0x2s32;
26    uint8_t *d;
27    int16_t cospi_8_64 = 15137;
28    int16_t cospi_16_64 = 11585;
29    int16_t cospi_24_64 = 6270;
30
31    d26u32 = d27u32 = vdup_n_u32(0);
32
33    q8s16 = vld1q_s16(input);
34    q9s16 = vld1q_s16(input + 8);
35
36    d16s16 = vget_low_s16(q8s16);
37    d17s16 = vget_high_s16(q8s16);
38    d18s16 = vget_low_s16(q9s16);
39    d19s16 = vget_high_s16(q9s16);
40
41    d0x2s16 = vtrn_s16(d16s16, d17s16);
42    d1x2s16 = vtrn_s16(d18s16, d19s16);
43    q8s16 = vcombine_s16(d0x2s16.val[0], d0x2s16.val[1]);
44    q9s16 = vcombine_s16(d1x2s16.val[0], d1x2s16.val[1]);
45
46    d20s16 = vdup_n_s16(cospi_8_64);
47    d21s16 = vdup_n_s16(cospi_16_64);
48
49    q0x2s32 = vtrnq_s32(vreinterpretq_s32_s16(q8s16),
50                        vreinterpretq_s32_s16(q9s16));
51    d16s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[0]));
52    d17s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[0]));
53    d18s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[1]));
54    d19s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[1]));
55
56    d22s16 = vdup_n_s16(cospi_24_64);
57
58    // stage 1
59    d23s16 = vadd_s16(d16s16, d18s16);
60    d24s16 = vsub_s16(d16s16, d18s16);
61
62    q15s32 = vmull_s16(d17s16, d22s16);
63    q1s32  = vmull_s16(d17s16, d20s16);
64    q13s32 = vmull_s16(d23s16, d21s16);
65    q14s32 = vmull_s16(d24s16, d21s16);
66
67    q15s32 = vmlsl_s16(q15s32, d19s16, d20s16);
68    q1s32  = vmlal_s16(q1s32,  d19s16, d22s16);
69
70    d26s16 = vqrshrn_n_s32(q13s32, 14);
71    d27s16 = vqrshrn_n_s32(q14s32, 14);
72    d29s16 = vqrshrn_n_s32(q15s32, 14);
73    d28s16 = vqrshrn_n_s32(q1s32,  14);
74    q13s16 = vcombine_s16(d26s16, d27s16);
75    q14s16 = vcombine_s16(d28s16, d29s16);
76
77    // stage 2
78    q8s16 = vaddq_s16(q13s16, q14s16);
79    q9s16 = vsubq_s16(q13s16, q14s16);
80
81    d16s16 = vget_low_s16(q8s16);
82    d17s16 = vget_high_s16(q8s16);
83    d18s16 = vget_high_s16(q9s16);  // vswp d18 d19
84    d19s16 = vget_low_s16(q9s16);
85
86    d0x2s16 = vtrn_s16(d16s16, d17s16);
87    d1x2s16 = vtrn_s16(d18s16, d19s16);
88    q8s16 = vcombine_s16(d0x2s16.val[0], d0x2s16.val[1]);
89    q9s16 = vcombine_s16(d1x2s16.val[0], d1x2s16.val[1]);
90
91    q0x2s32 = vtrnq_s32(vreinterpretq_s32_s16(q8s16),
92                        vreinterpretq_s32_s16(q9s16));
93    d16s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[0]));
94    d17s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[0]));
95    d18s16 = vget_low_s16(vreinterpretq_s16_s32(q0x2s32.val[1]));
96    d19s16 = vget_high_s16(vreinterpretq_s16_s32(q0x2s32.val[1]));
97
98    // do the transform on columns
99    // stage 1
100    d23s16 = vadd_s16(d16s16, d18s16);
101    d24s16 = vsub_s16(d16s16, d18s16);
102
103    q15s32 = vmull_s16(d17s16, d22s16);
104    q1s32  = vmull_s16(d17s16, d20s16);
105    q13s32 = vmull_s16(d23s16, d21s16);
106    q14s32 = vmull_s16(d24s16, d21s16);
107
108    q15s32 = vmlsl_s16(q15s32, d19s16, d20s16);
109    q1s32  = vmlal_s16(q1s32,  d19s16, d22s16);
110
111    d26s16 = vqrshrn_n_s32(q13s32, 14);
112    d27s16 = vqrshrn_n_s32(q14s32, 14);
113    d29s16 = vqrshrn_n_s32(q15s32, 14);
114    d28s16 = vqrshrn_n_s32(q1s32,  14);
115    q13s16 = vcombine_s16(d26s16, d27s16);
116    q14s16 = vcombine_s16(d28s16, d29s16);
117
118    // stage 2
119    q8s16 = vaddq_s16(q13s16, q14s16);
120    q9s16 = vsubq_s16(q13s16, q14s16);
121
122    q8s16 = vrshrq_n_s16(q8s16, 4);
123    q9s16 = vrshrq_n_s16(q9s16, 4);
124
125    d = dest;
126    d26u32 = vld1_lane_u32((const uint32_t *)d, d26u32, 0);
127    d += dest_stride;
128    d26u32 = vld1_lane_u32((const uint32_t *)d, d26u32, 1);
129    d += dest_stride;
130    d27u32 = vld1_lane_u32((const uint32_t *)d, d27u32, 1);
131    d += dest_stride;
132    d27u32 = vld1_lane_u32((const uint32_t *)d, d27u32, 0);
133
134    q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16),
135                     vreinterpret_u8_u32(d26u32));
136    q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16),
137                     vreinterpret_u8_u32(d27u32));
138
139    d26u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16));
140    d27u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16));
141
142    d = dest;
143    vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d26u8), 0);
144    d += dest_stride;
145    vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d26u8), 1);
146    d += dest_stride;
147    vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d27u8), 1);
148    d += dest_stride;
149    vst1_lane_u32((uint32_t *)d, vreinterpret_u32_u8(d27u8), 0);
150    return;
151}
152