1// Copyright 2017 The Gemmlowp Authors. All Rights Reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// simd_wrappers_neon.h: NEON specialization of simd_wrappers.h
16
17#ifndef GEMMLOWP_INTERNAL_SIMD_WRAPPERS_NEON_H_
18#define GEMMLOWP_INTERNAL_SIMD_WRAPPERS_NEON_H_
19
20#include <arm_neon.h>
21
22namespace gemmlowp {
23
24using Int32x4 = int32x4_t;
25using Uint8x8 = uint8x8_t;
26
27template <int ScalarCount>
28struct RegisterType<std::int32_t, ScalarCount> {
29  using Type =
30      typename std::conditional<ScalarCount >= 4, Int32x4, std::int32_t>::type;
31};
32
33template <int ScalarCount>
34struct RegisterType<std::uint8_t, ScalarCount> {
35  using Type = typename std::conditional<
36      ScalarCount >= 8, Uint8x8,
37      typename std::conditional<ScalarCount >= 4, std::uint32_t,
38                                std::uint8_t>::type>::type;
39};
40
41inline Int32x4 LoadInt32x4(const std::int32_t* src) { return vld1q_s32(src); }
42
43inline void StoreInt32x4(std::int32_t* dst, Int32x4 value) {
44  vst1q_s32(dst, value);
45}
46
47template <int Lane>
48std::int32_t GetLane(Int32x4 value) {
49  return vgetq_lane_s32(value, Lane);
50}
51
52template <int Lane>
53Int32x4 DupLane(Int32x4 value) {
54  switch (Lane) {
55    case 0:
56      return vdupq_lane_s32(vget_low_s32(value), 0);
57    case 1:
58      return vdupq_lane_s32(vget_low_s32(value), 1);
59    case 2:
60      return vdupq_lane_s32(vget_high_s32(value), 0);
61    case 3:
62      return vdupq_lane_s32(vget_high_s32(value), 1);
63    default:
64      static_assert(Lane >= 0 && Lane <= 3, "");
65      return vdupq_n_s32(0);
66  }
67}
68
69inline Int32x4 Mul(Int32x4 a, std::int32_t b) { return vmulq_n_s32(a, b); }
70
71inline Int32x4 Min(Int32x4 a, Int32x4 b) { return vminq_s32(a, b); }
72
73inline Int32x4 Max(Int32x4 a, Int32x4 b) { return vmaxq_s32(a, b); }
74
75inline Int32x4 SaturatingRoundingDoublingHighMul(Int32x4 a, std::int32_t b) {
76  return vqrdmulhq_n_s32(a, b);
77}
78
79template <int Lane>
80Int32x4 MulByRhsLane(Int32x4 a, Int32x4 b) {
81  switch (Lane) {
82    case 0:
83      return vmulq_lane_s32(a, vget_low_s32(b), 0);
84    case 1:
85      return vmulq_lane_s32(a, vget_low_s32(b), 1);
86    case 2:
87      return vmulq_lane_s32(a, vget_high_s32(b), 0);
88    case 3:
89      return vmulq_lane_s32(a, vget_high_s32(b), 1);
90    default:
91      static_assert(Lane >= 0 && Lane <= 3, "");
92      return vdupq_n_s32(0);
93  }
94}
95
96inline void MulAdd(Int32x4 lhs, Int32x4 rhs, Int32x4* acc) {
97  *acc = vmlaq_s32(*acc, lhs, rhs);
98}
99
100inline void MulAdd(Int32x4 lhs, std::int32_t rhs, Int32x4* acc) {
101  *acc = vmlaq_n_s32(*acc, lhs, rhs);
102}
103
104template <int Lane>
105inline void MulAddByRhsLane(Int32x4 lhs, Int32x4 rhs, Int32x4* acc) {
106  switch (Lane) {
107    case 0:
108      *acc = vmlaq_lane_s32(*acc, lhs, vget_low_s32(rhs), 0);
109      break;
110    case 1:
111      *acc = vmlaq_lane_s32(*acc, lhs, vget_low_s32(rhs), 1);
112      break;
113    case 2:
114      *acc = vmlaq_lane_s32(*acc, lhs, vget_high_s32(rhs), 0);
115      break;
116    case 3:
117      *acc = vmlaq_lane_s32(*acc, lhs, vget_high_s32(rhs), 1);
118      break;
119    default:
120      static_assert(Lane >= 0 && Lane <= 3, "");
121  }
122}
123
124template <>
125struct LoadContiguousImpl<RegBlockUint8<8, 8>> {
126  static RegBlockUint8<8, 8> Run(const std::uint8_t* src) {
127    RegBlockUint8<8, 8> result;
128    for (int i = 0; i < 8; i++) {
129      result.buf.reg[i] = vld1_u8(src + 8 * i);
130    }
131    return result;
132  }
133};
134
135template <>
136struct LoadContiguousImpl<RegBlockInt32<8, 8>> {
137  static RegBlockInt32<8, 8> Run(const std::int32_t* src) {
138    RegBlockInt32<8, 8> result;
139    for (int i = 0; i < 16; i++) {
140      result.buf.reg[i] = vld1q_s32(src + 4 * i);
141    }
142    return result;
143  }
144};
145
146}  // end namespace gemmlowp
147
148#include "simd_wrappers_common_neon_sse.h"
149
150#endif  // GEMMLOWP_INTERNAL_SIMD_WRAPPERS_NEON_H_
151