1/*
2 *  Copyright (c) 2015 The WebM project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include <arm_neon.h>
12#include <assert.h>
13
14#include "./vp9_rtcd.h"
15#include "./vpx_config.h"
16
17#include "vpx/vpx_integer.h"
18
19static INLINE unsigned int horizontal_add_u16x8(const uint16x8_t v_16x8) {
20  const uint32x4_t a = vpaddlq_u16(v_16x8);
21  const uint64x2_t b = vpaddlq_u32(a);
22  const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)),
23                                vreinterpret_u32_u64(vget_high_u64(b)));
24  return vget_lane_u32(c, 0);
25}
26
27unsigned int vp9_avg_8x8_neon(const uint8_t *s, int p) {
28  uint8x8_t v_s0 = vld1_u8(s);
29  const uint8x8_t v_s1 = vld1_u8(s + p);
30  uint16x8_t v_sum = vaddl_u8(v_s0, v_s1);
31
32  v_s0 = vld1_u8(s + 2 * p);
33  v_sum = vaddw_u8(v_sum, v_s0);
34
35  v_s0 = vld1_u8(s + 3 * p);
36  v_sum = vaddw_u8(v_sum, v_s0);
37
38  v_s0 = vld1_u8(s + 4 * p);
39  v_sum = vaddw_u8(v_sum, v_s0);
40
41  v_s0 = vld1_u8(s + 5 * p);
42  v_sum = vaddw_u8(v_sum, v_s0);
43
44  v_s0 = vld1_u8(s + 6 * p);
45  v_sum = vaddw_u8(v_sum, v_s0);
46
47  v_s0 = vld1_u8(s + 7 * p);
48  v_sum = vaddw_u8(v_sum, v_s0);
49
50  return (horizontal_add_u16x8(v_sum) + 32) >> 6;
51}
52
53void vp9_int_pro_row_neon(int16_t hbuf[16], uint8_t const *ref,
54                          const int ref_stride, const int height) {
55  int i;
56  uint16x8_t vec_sum_lo = vdupq_n_u16(0);
57  uint16x8_t vec_sum_hi = vdupq_n_u16(0);
58  const int shift_factor = ((height >> 5) + 3) * -1;
59  const int16x8_t vec_shift = vdupq_n_s16(shift_factor);
60
61  for (i = 0; i < height; i += 8) {
62    const uint8x16_t vec_row1 = vld1q_u8(ref);
63    const uint8x16_t vec_row2 = vld1q_u8(ref + ref_stride);
64    const uint8x16_t vec_row3 = vld1q_u8(ref + ref_stride * 2);
65    const uint8x16_t vec_row4 = vld1q_u8(ref + ref_stride * 3);
66    const uint8x16_t vec_row5 = vld1q_u8(ref + ref_stride * 4);
67    const uint8x16_t vec_row6 = vld1q_u8(ref + ref_stride * 5);
68    const uint8x16_t vec_row7 = vld1q_u8(ref + ref_stride * 6);
69    const uint8x16_t vec_row8 = vld1q_u8(ref + ref_stride * 7);
70
71    vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row1));
72    vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row1));
73
74    vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row2));
75    vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row2));
76
77    vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row3));
78    vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row3));
79
80    vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row4));
81    vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row4));
82
83    vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row5));
84    vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row5));
85
86    vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row6));
87    vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row6));
88
89    vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row7));
90    vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row7));
91
92    vec_sum_lo = vaddw_u8(vec_sum_lo, vget_low_u8(vec_row8));
93    vec_sum_hi = vaddw_u8(vec_sum_hi, vget_high_u8(vec_row8));
94
95    ref += ref_stride * 8;
96  }
97
98  vec_sum_lo = vshlq_u16(vec_sum_lo, vec_shift);
99  vec_sum_hi = vshlq_u16(vec_sum_hi, vec_shift);
100
101  vst1q_s16(hbuf, vreinterpretq_s16_u16(vec_sum_lo));
102  hbuf += 8;
103  vst1q_s16(hbuf, vreinterpretq_s16_u16(vec_sum_hi));
104}
105
106int16_t vp9_int_pro_col_neon(uint8_t const *ref, const int width) {
107  int i;
108  uint16x8_t vec_sum = vdupq_n_u16(0);
109
110  for (i = 0; i < width; i += 16) {
111    const uint8x16_t vec_row = vld1q_u8(ref);
112    vec_sum = vaddw_u8(vec_sum, vget_low_u8(vec_row));
113    vec_sum = vaddw_u8(vec_sum, vget_high_u8(vec_row));
114    ref += 16;
115  }
116
117  return horizontal_add_u16x8(vec_sum);
118}
119
120// ref, src = [0, 510] - max diff = 16-bits
121// bwl = {2, 3, 4}, width = {16, 32, 64}
122int vp9_vector_var_neon(int16_t const *ref, int16_t const *src, const int bwl) {
123  int width = 4 << bwl;
124  int32x4_t sse = vdupq_n_s32(0);
125  int16x8_t total = vdupq_n_s16(0);
126
127  assert(width >= 8);
128  assert((width % 8) == 0);
129
130  do {
131    const int16x8_t r = vld1q_s16(ref);
132    const int16x8_t s = vld1q_s16(src);
133    const int16x8_t diff = vsubq_s16(r, s);  // [-510, 510], 10 bits.
134    const int16x4_t diff_lo = vget_low_s16(diff);
135    const int16x4_t diff_hi = vget_high_s16(diff);
136    sse = vmlal_s16(sse, diff_lo, diff_lo);  // dynamic range 26 bits.
137    sse = vmlal_s16(sse, diff_hi, diff_hi);
138    total = vaddq_s16(total, diff);  // dynamic range 16 bits.
139
140    ref += 8;
141    src += 8;
142    width -= 8;
143  } while (width != 0);
144
145  {
146    // Note: 'total''s pairwise addition could be implemented similarly to
147    // horizontal_add_u16x8(), but one less vpaddl with 'total' when paired
148    // with the summation of 'sse' performed better on a Cortex-A15.
149    const int32x4_t t0 = vpaddlq_s16(total);  // cascading summation of 'total'
150    const int32x2_t t1 = vadd_s32(vget_low_s32(t0), vget_high_s32(t0));
151    const int32x2_t t2 = vpadd_s32(t1, t1);
152    const int t = vget_lane_s32(t2, 0);
153    const int64x2_t s0 = vpaddlq_s32(sse);  // cascading summation of 'sse'.
154    const int32x2_t s1 = vadd_s32(vreinterpret_s32_s64(vget_low_s64(s0)),
155                                  vreinterpret_s32_s64(vget_high_s64(s0)));
156    const int s = vget_lane_s32(s1, 0);
157    const int shift_factor = bwl + 2;
158    return s - ((t * t) >> shift_factor);
159  }
160}
161