11e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora// Copyright 2011 Google Inc. All Rights Reserved.
21e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora//
30406ce1417f76f2034833414dcecc9f56253640cVikas Arora// Use of this source code is governed by a BSD-style license
40406ce1417f76f2034833414dcecc9f56253640cVikas Arora// that can be found in the COPYING file in the root of the source
50406ce1417f76f2034833414dcecc9f56253640cVikas Arora// tree. An additional intellectual property rights grant can be found
60406ce1417f76f2034833414dcecc9f56253640cVikas Arora// in the file PATENTS. All contributing project authors may
70406ce1417f76f2034833414dcecc9f56253640cVikas Arora// be found in the AUTHORS file in the root of the source tree.
81e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora// -----------------------------------------------------------------------------
91e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora//
101e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora// NEON version of YUV to RGB upsampling functions.
111e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora//
121e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora// Author: mans@mansr.com (Mans Rullgard)
131e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora// Based on SSE code by: somnath@google.com (Somnath Banerjee)
141e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
151e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#include "./dsp.h"
161e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
171e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#if defined(WEBP_USE_NEON)
181e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
191e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#include <assert.h>
201e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#include <arm_neon.h>
211e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#include <string.h>
22af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora#include "./neon.h"
231e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#include "./yuv.h"
241e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
251e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#ifdef FANCY_UPSAMPLING
261e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
278b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora//-----------------------------------------------------------------------------
288b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora// U/V upsampling
298b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora
301e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora// Loads 9 pixels each from rows r1 and r2 and generates 16 pixels.
311e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#define UPSAMPLE_16PIXELS(r1, r2, out) {                                \
321e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint8x8_t a = vld1_u8(r1);                                            \
331e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint8x8_t b = vld1_u8(r1 + 1);                                        \
341e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint8x8_t c = vld1_u8(r2);                                            \
351e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint8x8_t d = vld1_u8(r2 + 1);                                        \
361e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
371e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint16x8_t al = vshll_n_u8(a, 1);                                     \
381e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint16x8_t bl = vshll_n_u8(b, 1);                                     \
391e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint16x8_t cl = vshll_n_u8(c, 1);                                     \
401e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint16x8_t dl = vshll_n_u8(d, 1);                                     \
411e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
421e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint8x8_t diag1, diag2;                                               \
431e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint16x8_t sl;                                                        \
441e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
451e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  /* a + b + c + d */                                                   \
461e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  sl = vaddl_u8(a,  b);                                                 \
471e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  sl = vaddw_u8(sl, c);                                                 \
481e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  sl = vaddw_u8(sl, d);                                                 \
491e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
501e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  al = vaddq_u16(sl, al); /* 3a +  b +  c +  d */                       \
511e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  bl = vaddq_u16(sl, bl); /*  a + 3b +  c +  d */                       \
521e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
531e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  al = vaddq_u16(al, dl); /* 3a +  b +  c + 3d */                       \
541e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  bl = vaddq_u16(bl, cl); /*  a + 3b + 3c +  d */                       \
551e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
561e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  diag2 = vshrn_n_u16(al, 3);                                           \
571e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  diag1 = vshrn_n_u16(bl, 3);                                           \
581e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
591e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  a = vrhadd_u8(a, diag1);                                              \
601e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  b = vrhadd_u8(b, diag2);                                              \
611e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  c = vrhadd_u8(c, diag2);                                              \
621e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  d = vrhadd_u8(d, diag1);                                              \
631e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
641e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  {                                                                     \
65af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora    uint8x8x2_t a_b, c_d;                                               \
66af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora    INIT_VECTOR2(a_b, a, b);                                            \
67af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora    INIT_VECTOR2(c_d, c, d);                                            \
681e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    vst2_u8(out,      a_b);                                             \
691e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    vst2_u8(out + 32, c_d);                                             \
701e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  }                                                                     \
711e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora}
721e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
731e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora// Turn the macro into a function for reducing code-size when non-critical
741e7bf8805bd030c19924a5306837ecd72c295751Vikas Arorastatic void Upsample16Pixels(const uint8_t *r1, const uint8_t *r2,
751e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                             uint8_t *out) {
761e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  UPSAMPLE_16PIXELS(r1, r2, out);
771e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora}
781e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
791e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#define UPSAMPLE_LAST_BLOCK(tb, bb, num_pixels, out) {                  \
801e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint8_t r1[9], r2[9];                                                 \
811e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  memcpy(r1, (tb), (num_pixels));                                       \
821e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  memcpy(r2, (bb), (num_pixels));                                       \
831e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  /* replicate last byte */                                             \
841e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  memset(r1 + (num_pixels), r1[(num_pixels) - 1], 9 - (num_pixels));    \
851e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  memset(r2 + (num_pixels), r2[(num_pixels) - 1], 9 - (num_pixels));    \
861e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  Upsample16Pixels(r1, r2, out);                                        \
871e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora}
881e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
898b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora//-----------------------------------------------------------------------------
908b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora// YUV->RGB conversion
911e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
928b720228d581a84fd173b6dcb2fa295b59db489aVikas Arorastatic const int16_t kCoeffs[4] = { kYScale, kVToR, kUToG, kVToG };
931e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
94af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora#define v255 vdup_n_u8(255)
951e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
968b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#define STORE_Rgb(out, r, g, b) do {                                    \
97af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  uint8x8x3_t r_g_b;                                                    \
98af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  INIT_VECTOR3(r_g_b, r, g, b);                                         \
991e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  vst3_u8(out, r_g_b);                                                  \
1001e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora} while (0)
1011e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1028b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#define STORE_Bgr(out, r, g, b) do {                                    \
103af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  uint8x8x3_t b_g_r;                                                    \
104af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  INIT_VECTOR3(b_g_r, b, g, r);                                         \
1051e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  vst3_u8(out, b_g_r);                                                  \
1061e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora} while (0)
1071e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1088b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#define STORE_Rgba(out, r, g, b) do {                                   \
109af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  uint8x8x4_t r_g_b_v255;                                               \
110af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  INIT_VECTOR4(r_g_b_v255, r, g, b, v255);                              \
1111e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  vst4_u8(out, r_g_b_v255);                                             \
1121e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora} while (0)
1131e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1148b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#define STORE_Bgra(out, r, g, b) do {                                   \
115af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  uint8x8x4_t b_g_r_v255;                                               \
116af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  INIT_VECTOR4(b_g_r_v255, b, g, r, v255);                              \
1171e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  vst4_u8(out, b_g_r_v255);                                             \
1181e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora} while (0)
1191e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1208b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#define CONVERT8(FMT, XSTEP, N, src_y, src_uv, out, cur_x) {            \
1218b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  int i;                                                                \
1228b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  for (i = 0; i < N; i += 8) {                                          \
1238b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int off = ((cur_x) + i) * XSTEP;                              \
1248b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    uint8x8_t y  = vld1_u8((src_y) + (cur_x)  + i);                     \
1258b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    uint8x8_t u  = vld1_u8((src_uv) + i);                               \
1268b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    uint8x8_t v  = vld1_u8((src_uv) + i + 16);                          \
1278b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int16x8_t yy = vreinterpretq_s16_u16(vsubl_u8(y, u16));       \
1288b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int16x8_t uu = vreinterpretq_s16_u16(vsubl_u8(u, u128));      \
1298b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int16x8_t vv = vreinterpretq_s16_u16(vsubl_u8(v, u128));      \
1308b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    int32x4_t yl = vmull_lane_s16(vget_low_s16(yy),  cf16, 0);          \
1318b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    int32x4_t yh = vmull_lane_s16(vget_high_s16(yy), cf16, 0);          \
1328b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int32x4_t rl = vmlal_lane_s16(yl, vget_low_s16(vv),  cf16, 1);\
1338b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int32x4_t rh = vmlal_lane_s16(yh, vget_high_s16(vv), cf16, 1);\
1348b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    int32x4_t gl = vmlsl_lane_s16(yl, vget_low_s16(uu),  cf16, 2);      \
1358b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    int32x4_t gh = vmlsl_lane_s16(yh, vget_high_s16(uu), cf16, 2);      \
1368b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int32x4_t bl = vmovl_s16(vget_low_s16(uu));                   \
1378b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int32x4_t bh = vmovl_s16(vget_high_s16(uu));                  \
1388b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    gl = vmlsl_lane_s16(gl, vget_low_s16(vv),  cf16, 3);                \
1398b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    gh = vmlsl_lane_s16(gh, vget_high_s16(vv), cf16, 3);                \
1408b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    yl = vmlaq_lane_s32(yl, bl, cf32, 0);                               \
1418b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    yh = vmlaq_lane_s32(yh, bh, cf32, 0);                               \
1428b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    /* vrshrn_n_s32() already incorporates the rounding constant */     \
1438b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    y = vqmovun_s16(vcombine_s16(vrshrn_n_s32(rl, YUV_FIX2),            \
1448b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora                                 vrshrn_n_s32(rh, YUV_FIX2)));          \
1458b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    u = vqmovun_s16(vcombine_s16(vrshrn_n_s32(gl, YUV_FIX2),            \
1468b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora                                 vrshrn_n_s32(gh, YUV_FIX2)));          \
1478b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    v = vqmovun_s16(vcombine_s16(vrshrn_n_s32(yl, YUV_FIX2),            \
1488b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora                                 vrshrn_n_s32(yh, YUV_FIX2)));          \
1498b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    STORE_ ## FMT(out + off, y, u, v);                                  \
1508b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  }                                                                     \
1518b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora}
1528b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora
1538b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#define CONVERT1(FUNC, XSTEP, N, src_y, src_uv, rgb, cur_x) {           \
1541e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  int i;                                                                \
1551e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  for (i = 0; i < N; i++) {                                             \
1568b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int off = ((cur_x) + i) * XSTEP;                              \
1578b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int y = src_y[(cur_x) + i];                                   \
1588b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int u = (src_uv)[i];                                          \
1598b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    const int v = (src_uv)[i + 16];                                     \
1608b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    FUNC(y, u, v, rgb + off);                                           \
1611e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  }                                                                     \
1621e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora}
1631e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1641e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#define CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, uv,                  \
1651e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                      top_dst, bottom_dst, cur_x, len) {                \
1668b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  CONVERT8(FMT, XSTEP, len, top_y, uv, top_dst, cur_x)                  \
1678b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  if (bottom_y != NULL) {                                               \
1681e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    CONVERT8(FMT, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x)   \
1691e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  }                                                                     \
1701e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora}
1711e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1728b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#define CONVERT2RGB_1(FUNC, XSTEP, top_y, bottom_y, uv,                 \
1731e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                      top_dst, bottom_dst, cur_x, len) {                \
1748b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  CONVERT1(FUNC, XSTEP, len, top_y, uv, top_dst, cur_x);                \
1758b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  if (bottom_y != NULL) {                                               \
1768b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora    CONVERT1(FUNC, XSTEP, len, bottom_y, (uv) + 32, bottom_dst, cur_x); \
1771e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  }                                                                     \
1781e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora}
1791e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
1801e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#define NEON_UPSAMPLE_FUNC(FUNC_NAME, FMT, XSTEP)                       \
1811e7bf8805bd030c19924a5306837ecd72c295751Vikas Arorastatic void FUNC_NAME(const uint8_t *top_y, const uint8_t *bottom_y,    \
1821e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                      const uint8_t *top_u, const uint8_t *top_v,       \
1831e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                      const uint8_t *cur_u, const uint8_t *cur_v,       \
1841e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                      uint8_t *top_dst, uint8_t *bottom_dst, int len) { \
1851e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  int block;                                                            \
1861e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  /* 16 byte aligned array to cache reconstructed u and v */            \
1871e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint8_t uv_buf[2 * 32 + 15];                                          \
1881e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  uint8_t *const r_uv = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15);     \
1891e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  const int uv_len = (len + 1) >> 1;                                    \
1901e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  /* 9 pixels must be read-able for each block */                       \
1911e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  const int num_blocks = (uv_len - 1) >> 3;                             \
1921e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  const int leftover = uv_len - num_blocks * 8;                         \
1931e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  const int last_pos = 1 + 16 * num_blocks;                             \
1941e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
1951e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  const int u_diag = ((top_u[0] + cur_u[0]) >> 1) + 1;                  \
1961e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  const int v_diag = ((top_v[0] + cur_v[0]) >> 1) + 1;                  \
1971e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
1988b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  const int16x4_t cf16 = vld1_s16(kCoeffs);                             \
199af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  const int32x2_t cf32 = vdup_n_s32(kUToB);                             \
200af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  const uint8x8_t u16  = vdup_n_u8(16);                                 \
201af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  const uint8x8_t u128 = vdup_n_u8(128);                                \
2021e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
2031e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  /* Treat the first pixel in regular way */                            \
2048b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  assert(top_y != NULL);                                                \
2058b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  {                                                                     \
2061e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    const int u0 = (top_u[0] + u_diag) >> 1;                            \
2071e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    const int v0 = (top_v[0] + v_diag) >> 1;                            \
2081e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    VP8YuvTo ## FMT(top_y[0], u0, v0, top_dst);                         \
2091e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  }                                                                     \
2108b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  if (bottom_y != NULL) {                                               \
2111e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    const int u0 = (cur_u[0] + u_diag) >> 1;                            \
2121e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    const int v0 = (cur_v[0] + v_diag) >> 1;                            \
2131e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    VP8YuvTo ## FMT(bottom_y[0], u0, v0, bottom_dst);                   \
2141e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  }                                                                     \
2151e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
2161e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  for (block = 0; block < num_blocks; ++block) {                        \
2171e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    UPSAMPLE_16PIXELS(top_u, cur_u, r_uv);                              \
2181e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    UPSAMPLE_16PIXELS(top_v, cur_v, r_uv + 16);                         \
2191e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    CONVERT2RGB_8(FMT, XSTEP, top_y, bottom_y, r_uv,                    \
2201e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                  top_dst, bottom_dst, 16 * block + 1, 16);             \
2211e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    top_u += 8;                                                         \
2221e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    cur_u += 8;                                                         \
2231e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    top_v += 8;                                                         \
2241e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora    cur_v += 8;                                                         \
2251e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  }                                                                     \
2261e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                                                                        \
2271e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  UPSAMPLE_LAST_BLOCK(top_u, cur_u, leftover, r_uv);                    \
2281e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora  UPSAMPLE_LAST_BLOCK(top_v, cur_v, leftover, r_uv + 16);               \
2298b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora  CONVERT2RGB_1(VP8YuvTo ## FMT, XSTEP, top_y, bottom_y, r_uv,          \
2301e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora                top_dst, bottom_dst, last_pos, len - last_pos);         \
2311e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora}
2321e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
2331e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora// NEON variants of the fancy upsampler.
234af51b94a435132e9014c324e25fb686b3d07a8c8Vikas AroraNEON_UPSAMPLE_FUNC(UpsampleRgbLinePair,  Rgb,  3)
235af51b94a435132e9014c324e25fb686b3d07a8c8Vikas AroraNEON_UPSAMPLE_FUNC(UpsampleBgrLinePair,  Bgr,  3)
236af51b94a435132e9014c324e25fb686b3d07a8c8Vikas AroraNEON_UPSAMPLE_FUNC(UpsampleRgbaLinePair, Rgba, 4)
237af51b94a435132e9014c324e25fb686b3d07a8c8Vikas AroraNEON_UPSAMPLE_FUNC(UpsampleBgraLinePair, Bgra, 4)
2381e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
2391e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#endif  // FANCY_UPSAMPLING
2401e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
2411e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#endif   // WEBP_USE_NEON
2421e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
2431e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora//------------------------------------------------------------------------------
2441e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
245af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Aroraextern void WebPInitUpsamplersNEON(void);
246af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora
2478b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#ifdef FANCY_UPSAMPLING
2488b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora
2491e7bf8805bd030c19924a5306837ecd72c295751Vikas Aroraextern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */];
2501e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
2511e7bf8805bd030c19924a5306837ecd72c295751Vikas Aroravoid WebPInitUpsamplersNEON(void) {
2521e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#if defined(WEBP_USE_NEON)
253af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  WebPUpsamplers[MODE_RGB]  = UpsampleRgbLinePair;
254af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair;
255af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  WebPUpsamplers[MODE_BGR]  = UpsampleBgrLinePair;
256af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair;
257af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  WebPUpsamplers[MODE_rgbA] = UpsampleRgbaLinePair;
258af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Arora  WebPUpsamplers[MODE_bgrA] = UpsampleBgraLinePair;
2591e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora#endif   // WEBP_USE_NEON
2601e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora}
2611e7bf8805bd030c19924a5306837ecd72c295751Vikas Arora
2628b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#else
2638b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora
2648b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora// this empty function is to avoid an empty .o
265af51b94a435132e9014c324e25fb686b3d07a8c8Vikas Aroravoid WebPInitUpsamplersNEON(void) {}
2668b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora
2678b720228d581a84fd173b6dcb2fa295b59db489aVikas Arora#endif  // FANCY_UPSAMPLING
268