1/*
2 *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include <arm_neon.h>
12#include "./vp9_rtcd.h"
13
14#include "vpx_ports/mem.h"
15#include "vpx/vpx_integer.h"
16
17#include "vp9/common/vp9_common.h"
18#include "vp9/common/vp9_filter.h"
19
20#include "vp9/encoder/vp9_variance.h"
21
22enum { kWidth8 = 8 };
23enum { kHeight8 = 8 };
24enum { kHeight8PlusOne = 9 };
25enum { kWidth16 = 16 };
26enum { kHeight16 = 16 };
27enum { kHeight16PlusOne = 17 };
28enum { kWidth32 = 32 };
29enum { kHeight32 = 32 };
30enum { kHeight32PlusOne = 33 };
31enum { kPixelStepOne = 1 };
32enum { kAlign16 = 16 };
33
34static INLINE int horizontal_add_s16x8(const int16x8_t v_16x8) {
35  const int32x4_t a = vpaddlq_s16(v_16x8);
36  const int64x2_t b = vpaddlq_s32(a);
37  const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
38                               vreinterpret_s32_s64(vget_high_s64(b)));
39  return vget_lane_s32(c, 0);
40}
41
42static INLINE int horizontal_add_s32x4(const int32x4_t v_32x4) {
43  const int64x2_t b = vpaddlq_s32(v_32x4);
44  const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
45                               vreinterpret_s32_s64(vget_high_s64(b)));
46  return vget_lane_s32(c, 0);
47}
48
49static void variance_neon_w8(const uint8_t *a, int a_stride,
50                             const uint8_t *b, int b_stride,
51                             int w, int h, unsigned int *sse, int *sum) {
52  int i, j;
53  int16x8_t v_sum = vdupq_n_s16(0);
54  int32x4_t v_sse_lo = vdupq_n_s32(0);
55  int32x4_t v_sse_hi = vdupq_n_s32(0);
56
57  for (i = 0; i < h; ++i) {
58    for (j = 0; j < w; j += 8) {
59      const uint8x8_t v_a = vld1_u8(&a[j]);
60      const uint8x8_t v_b = vld1_u8(&b[j]);
61      const uint16x8_t v_diff = vsubl_u8(v_a, v_b);
62      const int16x8_t sv_diff = vreinterpretq_s16_u16(v_diff);
63      v_sum = vaddq_s16(v_sum, sv_diff);
64      v_sse_lo = vmlal_s16(v_sse_lo,
65                           vget_low_s16(sv_diff),
66                           vget_low_s16(sv_diff));
67      v_sse_hi = vmlal_s16(v_sse_hi,
68                           vget_high_s16(sv_diff),
69                           vget_high_s16(sv_diff));
70    }
71    a += a_stride;
72    b += b_stride;
73  }
74
75  *sum = horizontal_add_s16x8(v_sum);
76  *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi));
77}
78
79void vp9_get8x8var_neon(const uint8_t *src_ptr, int source_stride,
80                        const uint8_t *ref_ptr, int ref_stride,
81                        unsigned int *sse, int *sum) {
82  variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth8,
83                   kHeight8, sse, sum);
84}
85
86unsigned int vp9_variance8x8_neon(const uint8_t *a, int a_stride,
87                                  const uint8_t *b, int b_stride,
88                                  unsigned int *sse) {
89  int sum;
90  variance_neon_w8(a, a_stride, b, b_stride, kWidth8, kHeight8, sse, &sum);
91  return *sse - (((int64_t)sum * sum) / (kWidth8 * kHeight8));
92}
93
94void vp9_get16x16var_neon(const uint8_t *src_ptr, int source_stride,
95                          const uint8_t *ref_ptr, int ref_stride,
96                          unsigned int *sse, int *sum) {
97  variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth16,
98                   kHeight16, sse, sum);
99}
100
101unsigned int vp9_variance16x16_neon(const uint8_t *a, int a_stride,
102                                    const uint8_t *b, int b_stride,
103                                    unsigned int *sse) {
104  int sum;
105  variance_neon_w8(a, a_stride, b, b_stride, kWidth16, kHeight16, sse, &sum);
106  return *sse - (((int64_t)sum * sum) / (kWidth16 * kHeight16));
107}
108
109static void var_filter_block2d_bil_w8(const uint8_t *src_ptr,
110                                      uint8_t *output_ptr,
111                                      unsigned int src_pixels_per_line,
112                                      int pixel_step,
113                                      unsigned int output_height,
114                                      unsigned int output_width,
115                                      const int16_t *vp9_filter) {
116  const uint8x8_t f0 = vmov_n_u8((uint8_t)vp9_filter[0]);
117  const uint8x8_t f1 = vmov_n_u8((uint8_t)vp9_filter[1]);
118  unsigned int i;
119  for (i = 0; i < output_height; ++i) {
120    const uint8x8_t src_0 = vld1_u8(&src_ptr[0]);
121    const uint8x8_t src_1 = vld1_u8(&src_ptr[pixel_step]);
122    const uint16x8_t a = vmull_u8(src_0, f0);
123    const uint16x8_t b = vmlal_u8(a, src_1, f1);
124    const uint8x8_t out = vrshrn_n_u16(b, FILTER_BITS);
125    vst1_u8(&output_ptr[0], out);
126    // Next row...
127    src_ptr += src_pixels_per_line;
128    output_ptr += output_width;
129  }
130}
131
132static void var_filter_block2d_bil_w16(const uint8_t *src_ptr,
133                                       uint8_t *output_ptr,
134                                       unsigned int src_pixels_per_line,
135                                       int pixel_step,
136                                       unsigned int output_height,
137                                       unsigned int output_width,
138                                       const int16_t *vp9_filter) {
139  const uint8x8_t f0 = vmov_n_u8((uint8_t)vp9_filter[0]);
140  const uint8x8_t f1 = vmov_n_u8((uint8_t)vp9_filter[1]);
141  unsigned int i, j;
142  for (i = 0; i < output_height; ++i) {
143    for (j = 0; j < output_width; j += 16) {
144      const uint8x16_t src_0 = vld1q_u8(&src_ptr[j]);
145      const uint8x16_t src_1 = vld1q_u8(&src_ptr[j + pixel_step]);
146      const uint16x8_t a = vmull_u8(vget_low_u8(src_0), f0);
147      const uint16x8_t b = vmlal_u8(a, vget_low_u8(src_1), f1);
148      const uint8x8_t out_lo = vrshrn_n_u16(b, FILTER_BITS);
149      const uint16x8_t c = vmull_u8(vget_high_u8(src_0), f0);
150      const uint16x8_t d = vmlal_u8(c, vget_high_u8(src_1), f1);
151      const uint8x8_t out_hi = vrshrn_n_u16(d, FILTER_BITS);
152      vst1q_u8(&output_ptr[j], vcombine_u8(out_lo, out_hi));
153    }
154    // Next row...
155    src_ptr += src_pixels_per_line;
156    output_ptr += output_width;
157  }
158}
159
160unsigned int vp9_sub_pixel_variance8x8_neon(const uint8_t *src,
161                                            int src_stride,
162                                            int xoffset,
163                                            int yoffset,
164                                            const uint8_t *dst,
165                                            int dst_stride,
166                                            unsigned int *sse) {
167  DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight8 * kWidth8);
168  DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight8PlusOne * kWidth8);
169
170  var_filter_block2d_bil_w8(src, fdata3, src_stride, kPixelStepOne,
171                            kHeight8PlusOne, kWidth8,
172                            BILINEAR_FILTERS_2TAP(xoffset));
173  var_filter_block2d_bil_w8(fdata3, temp2, kWidth8, kWidth8, kHeight8,
174                            kWidth8, BILINEAR_FILTERS_2TAP(yoffset));
175  return vp9_variance8x8_neon(temp2, kWidth8, dst, dst_stride, sse);
176}
177
178unsigned int vp9_sub_pixel_variance16x16_neon(const uint8_t *src,
179                                              int src_stride,
180                                              int xoffset,
181                                              int yoffset,
182                                              const uint8_t *dst,
183                                              int dst_stride,
184                                              unsigned int *sse) {
185  DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight16 * kWidth16);
186  DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight16PlusOne * kWidth16);
187
188  var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne,
189                             kHeight16PlusOne, kWidth16,
190                             BILINEAR_FILTERS_2TAP(xoffset));
191  var_filter_block2d_bil_w16(fdata3, temp2, kWidth16, kWidth16, kHeight16,
192                             kWidth16, BILINEAR_FILTERS_2TAP(yoffset));
193  return vp9_variance16x16_neon(temp2, kWidth16, dst, dst_stride, sse);
194}
195
196void vp9_get32x32var_neon(const uint8_t *src_ptr, int source_stride,
197                          const uint8_t *ref_ptr, int ref_stride,
198                          unsigned int *sse, int *sum) {
199  variance_neon_w8(src_ptr, source_stride, ref_ptr, ref_stride, kWidth32,
200                   kHeight32, sse, sum);
201}
202
203unsigned int vp9_variance32x32_neon(const uint8_t *a, int a_stride,
204                                    const uint8_t *b, int b_stride,
205                                    unsigned int *sse) {
206  int sum;
207  variance_neon_w8(a, a_stride, b, b_stride, kWidth32, kHeight32, sse, &sum);
208  return *sse - (((int64_t)sum * sum) / (kWidth32 * kHeight32));
209}
210
211unsigned int vp9_sub_pixel_variance32x32_neon(const uint8_t *src,
212                                              int src_stride,
213                                              int xoffset,
214                                              int yoffset,
215                                              const uint8_t *dst,
216                                              int dst_stride,
217                                              unsigned int *sse) {
218  DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, temp2, kHeight32 * kWidth32);
219  DECLARE_ALIGNED_ARRAY(kAlign16, uint8_t, fdata3, kHeight32PlusOne * kWidth32);
220
221  var_filter_block2d_bil_w16(src, fdata3, src_stride, kPixelStepOne,
222                             kHeight32PlusOne, kWidth32,
223                             BILINEAR_FILTERS_2TAP(xoffset));
224  var_filter_block2d_bil_w16(fdata3, temp2, kWidth32, kWidth32, kHeight32,
225                             kWidth32, BILINEAR_FILTERS_2TAP(yoffset));
226  return vp9_variance32x32_neon(temp2, kWidth32, dst, dst_stride, sse);
227}
228