17c8da7ce66017295a65ec028084b90800be377f8James Zern// Copyright 2015 Google Inc. All Rights Reserved. 27c8da7ce66017295a65ec028084b90800be377f8James Zern// 37c8da7ce66017295a65ec028084b90800be377f8James Zern// Use of this source code is governed by a BSD-style license 47c8da7ce66017295a65ec028084b90800be377f8James Zern// that can be found in the COPYING file in the root of the source 57c8da7ce66017295a65ec028084b90800be377f8James Zern// tree. An additional intellectual property rights grant can be found 67c8da7ce66017295a65ec028084b90800be377f8James Zern// in the file PATENTS. All contributing project authors may 77c8da7ce66017295a65ec028084b90800be377f8James Zern// be found in the AUTHORS file in the root of the source tree. 87c8da7ce66017295a65ec028084b90800be377f8James Zern// ----------------------------------------------------------------------------- 97c8da7ce66017295a65ec028084b90800be377f8James Zern// 107c8da7ce66017295a65ec028084b90800be377f8James Zern// NEON version of rescaling functions 117c8da7ce66017295a65ec028084b90800be377f8James Zern// 127c8da7ce66017295a65ec028084b90800be377f8James Zern// Author: Skal (pascal.massimino@gmail.com) 137c8da7ce66017295a65ec028084b90800be377f8James Zern 147c8da7ce66017295a65ec028084b90800be377f8James Zern#include "./dsp.h" 157c8da7ce66017295a65ec028084b90800be377f8James Zern 167c8da7ce66017295a65ec028084b90800be377f8James Zern#if defined(WEBP_USE_NEON) 177c8da7ce66017295a65ec028084b90800be377f8James Zern 187c8da7ce66017295a65ec028084b90800be377f8James Zern#include <arm_neon.h> 197c8da7ce66017295a65ec028084b90800be377f8James Zern#include <assert.h> 207c8da7ce66017295a65ec028084b90800be377f8James Zern#include "./neon.h" 217c8da7ce66017295a65ec028084b90800be377f8James Zern#include "../utils/rescaler.h" 227c8da7ce66017295a65ec028084b90800be377f8James Zern 237c8da7ce66017295a65ec028084b90800be377f8James Zern#define ROUNDER (WEBP_RESCALER_ONE >> 1) 247c8da7ce66017295a65ec028084b90800be377f8James Zern#define MULT_FIX_C(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) 257c8da7ce66017295a65ec028084b90800be377f8James Zern 267c8da7ce66017295a65ec028084b90800be377f8James Zern#define LOAD_32x4(SRC, DST) const uint32x4_t DST = vld1q_u32((SRC)) 277c8da7ce66017295a65ec028084b90800be377f8James Zern#define LOAD_32x8(SRC, DST0, DST1) \ 287c8da7ce66017295a65ec028084b90800be377f8James Zern LOAD_32x4(SRC + 0, DST0); \ 297c8da7ce66017295a65ec028084b90800be377f8James Zern LOAD_32x4(SRC + 4, DST1) 307c8da7ce66017295a65ec028084b90800be377f8James Zern 317c8da7ce66017295a65ec028084b90800be377f8James Zern#define STORE_32x8(SRC0, SRC1, DST) do { \ 327c8da7ce66017295a65ec028084b90800be377f8James Zern vst1q_u32((DST) + 0, SRC0); \ 337c8da7ce66017295a65ec028084b90800be377f8James Zern vst1q_u32((DST) + 4, SRC1); \ 347c8da7ce66017295a65ec028084b90800be377f8James Zern} while (0); 357c8da7ce66017295a65ec028084b90800be377f8James Zern 367c8da7ce66017295a65ec028084b90800be377f8James Zern#if (WEBP_RESCALER_RFIX == 32) 377c8da7ce66017295a65ec028084b90800be377f8James Zern#define MAKE_HALF_CST(C) vdupq_n_s32((int32_t)((C) >> 1)) 387c8da7ce66017295a65ec028084b90800be377f8James Zern#define MULT_FIX(A, B) /* note: B is actualy scale>>1. See MAKE_HALF_CST */ \ 397c8da7ce66017295a65ec028084b90800be377f8James Zern vreinterpretq_u32_s32(vqrdmulhq_s32(vreinterpretq_s32_u32((A)), (B))) 407c8da7ce66017295a65ec028084b90800be377f8James Zern#else 417c8da7ce66017295a65ec028084b90800be377f8James Zern#error "MULT_FIX/WEBP_RESCALER_RFIX need some more work" 427c8da7ce66017295a65ec028084b90800be377f8James Zern#endif 437c8da7ce66017295a65ec028084b90800be377f8James Zern 447c8da7ce66017295a65ec028084b90800be377f8James Zernstatic uint32x4_t Interpolate(const rescaler_t* const frow, 457c8da7ce66017295a65ec028084b90800be377f8James Zern const rescaler_t* const irow, 467c8da7ce66017295a65ec028084b90800be377f8James Zern uint32_t A, uint32_t B) { 477c8da7ce66017295a65ec028084b90800be377f8James Zern LOAD_32x4(frow, A0); 487c8da7ce66017295a65ec028084b90800be377f8James Zern LOAD_32x4(irow, B0); 497c8da7ce66017295a65ec028084b90800be377f8James Zern const uint64x2_t C0 = vmull_n_u32(vget_low_u32(A0), A); 507c8da7ce66017295a65ec028084b90800be377f8James Zern const uint64x2_t C1 = vmull_n_u32(vget_high_u32(A0), A); 517c8da7ce66017295a65ec028084b90800be377f8James Zern const uint64x2_t D0 = vmlal_n_u32(C0, vget_low_u32(B0), B); 527c8da7ce66017295a65ec028084b90800be377f8James Zern const uint64x2_t D1 = vmlal_n_u32(C1, vget_high_u32(B0), B); 537c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t E = vcombine_u32( 547c8da7ce66017295a65ec028084b90800be377f8James Zern vrshrn_n_u64(D0, WEBP_RESCALER_RFIX), 557c8da7ce66017295a65ec028084b90800be377f8James Zern vrshrn_n_u64(D1, WEBP_RESCALER_RFIX)); 567c8da7ce66017295a65ec028084b90800be377f8James Zern return E; 577c8da7ce66017295a65ec028084b90800be377f8James Zern} 587c8da7ce66017295a65ec028084b90800be377f8James Zern 597c8da7ce66017295a65ec028084b90800be377f8James Zernstatic void RescalerExportRowExpand(WebPRescaler* const wrk) { 607c8da7ce66017295a65ec028084b90800be377f8James Zern int x_out; 617c8da7ce66017295a65ec028084b90800be377f8James Zern uint8_t* const dst = wrk->dst; 627c8da7ce66017295a65ec028084b90800be377f8James Zern rescaler_t* const irow = wrk->irow; 637c8da7ce66017295a65ec028084b90800be377f8James Zern const int x_out_max = wrk->dst_width * wrk->num_channels; 647c8da7ce66017295a65ec028084b90800be377f8James Zern const int max_span = x_out_max & ~7; 657c8da7ce66017295a65ec028084b90800be377f8James Zern const rescaler_t* const frow = wrk->frow; 667c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t fy_scale = wrk->fy_scale; 677c8da7ce66017295a65ec028084b90800be377f8James Zern const int32x4_t fy_scale_half = MAKE_HALF_CST(fy_scale); 687c8da7ce66017295a65ec028084b90800be377f8James Zern assert(!WebPRescalerOutputDone(wrk)); 697c8da7ce66017295a65ec028084b90800be377f8James Zern assert(wrk->y_accum <= 0); 707c8da7ce66017295a65ec028084b90800be377f8James Zern assert(wrk->y_expand); 717c8da7ce66017295a65ec028084b90800be377f8James Zern assert(wrk->y_sub != 0); 727c8da7ce66017295a65ec028084b90800be377f8James Zern if (wrk->y_accum == 0) { 737c8da7ce66017295a65ec028084b90800be377f8James Zern for (x_out = 0; x_out < max_span; x_out += 8) { 747c8da7ce66017295a65ec028084b90800be377f8James Zern LOAD_32x4(frow + x_out + 0, A0); 757c8da7ce66017295a65ec028084b90800be377f8James Zern LOAD_32x4(frow + x_out + 4, A1); 767c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t B0 = MULT_FIX(A0, fy_scale_half); 777c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t B1 = MULT_FIX(A1, fy_scale_half); 787c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16x4_t C0 = vmovn_u32(B0); 797c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16x4_t C1 = vmovn_u32(B1); 807c8da7ce66017295a65ec028084b90800be377f8James Zern const uint8x8_t D = vmovn_u16(vcombine_u16(C0, C1)); 817c8da7ce66017295a65ec028084b90800be377f8James Zern vst1_u8(dst + x_out, D); 827c8da7ce66017295a65ec028084b90800be377f8James Zern } 837c8da7ce66017295a65ec028084b90800be377f8James Zern for (; x_out < x_out_max; ++x_out) { 847c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t J = frow[x_out]; 857c8da7ce66017295a65ec028084b90800be377f8James Zern const int v = (int)MULT_FIX_C(J, fy_scale); 867c8da7ce66017295a65ec028084b90800be377f8James Zern assert(v >= 0 && v <= 255); 877c8da7ce66017295a65ec028084b90800be377f8James Zern dst[x_out] = v; 887c8da7ce66017295a65ec028084b90800be377f8James Zern } 897c8da7ce66017295a65ec028084b90800be377f8James Zern } else { 907c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); 917c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); 927c8da7ce66017295a65ec028084b90800be377f8James Zern for (x_out = 0; x_out < max_span; x_out += 8) { 937c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t C0 = 947c8da7ce66017295a65ec028084b90800be377f8James Zern Interpolate(frow + x_out + 0, irow + x_out + 0, A, B); 957c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t C1 = 967c8da7ce66017295a65ec028084b90800be377f8James Zern Interpolate(frow + x_out + 4, irow + x_out + 4, A, B); 977c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t D0 = MULT_FIX(C0, fy_scale_half); 987c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t D1 = MULT_FIX(C1, fy_scale_half); 997c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16x4_t E0 = vmovn_u32(D0); 1007c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16x4_t E1 = vmovn_u32(D1); 1017c8da7ce66017295a65ec028084b90800be377f8James Zern const uint8x8_t F = vmovn_u16(vcombine_u16(E0, E1)); 1027c8da7ce66017295a65ec028084b90800be377f8James Zern vst1_u8(dst + x_out, F); 1037c8da7ce66017295a65ec028084b90800be377f8James Zern } 1047c8da7ce66017295a65ec028084b90800be377f8James Zern for (; x_out < x_out_max; ++x_out) { 1057c8da7ce66017295a65ec028084b90800be377f8James Zern const uint64_t I = (uint64_t)A * frow[x_out] 1067c8da7ce66017295a65ec028084b90800be377f8James Zern + (uint64_t)B * irow[x_out]; 1077c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); 1087c8da7ce66017295a65ec028084b90800be377f8James Zern const int v = (int)MULT_FIX_C(J, fy_scale); 1097c8da7ce66017295a65ec028084b90800be377f8James Zern assert(v >= 0 && v <= 255); 1107c8da7ce66017295a65ec028084b90800be377f8James Zern dst[x_out] = v; 1117c8da7ce66017295a65ec028084b90800be377f8James Zern } 1127c8da7ce66017295a65ec028084b90800be377f8James Zern } 1137c8da7ce66017295a65ec028084b90800be377f8James Zern} 1147c8da7ce66017295a65ec028084b90800be377f8James Zern 1157c8da7ce66017295a65ec028084b90800be377f8James Zernstatic void RescalerExportRowShrink(WebPRescaler* const wrk) { 1167c8da7ce66017295a65ec028084b90800be377f8James Zern int x_out; 1177c8da7ce66017295a65ec028084b90800be377f8James Zern uint8_t* const dst = wrk->dst; 1187c8da7ce66017295a65ec028084b90800be377f8James Zern rescaler_t* const irow = wrk->irow; 1197c8da7ce66017295a65ec028084b90800be377f8James Zern const int x_out_max = wrk->dst_width * wrk->num_channels; 1207c8da7ce66017295a65ec028084b90800be377f8James Zern const int max_span = x_out_max & ~7; 1217c8da7ce66017295a65ec028084b90800be377f8James Zern const rescaler_t* const frow = wrk->frow; 1227c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); 1237c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t fxy_scale = wrk->fxy_scale; 1247c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t zero = vdupq_n_u32(0); 1257c8da7ce66017295a65ec028084b90800be377f8James Zern const int32x4_t yscale_half = MAKE_HALF_CST(yscale); 1267c8da7ce66017295a65ec028084b90800be377f8James Zern const int32x4_t fxy_scale_half = MAKE_HALF_CST(fxy_scale); 1277c8da7ce66017295a65ec028084b90800be377f8James Zern assert(!WebPRescalerOutputDone(wrk)); 1287c8da7ce66017295a65ec028084b90800be377f8James Zern assert(wrk->y_accum <= 0); 1297c8da7ce66017295a65ec028084b90800be377f8James Zern assert(!wrk->y_expand); 1307c8da7ce66017295a65ec028084b90800be377f8James Zern if (yscale) { 1317c8da7ce66017295a65ec028084b90800be377f8James Zern for (x_out = 0; x_out < max_span; x_out += 8) { 1327c8da7ce66017295a65ec028084b90800be377f8James Zern LOAD_32x8(frow + x_out, in0, in1); 1337c8da7ce66017295a65ec028084b90800be377f8James Zern LOAD_32x8(irow + x_out, in2, in3); 1347c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t A0 = MULT_FIX(in0, yscale_half); 1357c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t A1 = MULT_FIX(in1, yscale_half); 1367c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t B0 = vqsubq_u32(in2, A0); 1377c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t B1 = vqsubq_u32(in3, A1); 1387c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t C0 = MULT_FIX(B0, fxy_scale_half); 1397c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t C1 = MULT_FIX(B1, fxy_scale_half); 1407c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16x4_t D0 = vmovn_u32(C0); 1417c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16x4_t D1 = vmovn_u32(C1); 1427c8da7ce66017295a65ec028084b90800be377f8James Zern const uint8x8_t E = vmovn_u16(vcombine_u16(D0, D1)); 1437c8da7ce66017295a65ec028084b90800be377f8James Zern vst1_u8(dst + x_out, E); 1447c8da7ce66017295a65ec028084b90800be377f8James Zern STORE_32x8(A0, A1, irow + x_out); 1457c8da7ce66017295a65ec028084b90800be377f8James Zern } 1467c8da7ce66017295a65ec028084b90800be377f8James Zern for (; x_out < x_out_max; ++x_out) { 1477c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t frac = (uint32_t)MULT_FIX_C(frow[x_out], yscale); 1487c8da7ce66017295a65ec028084b90800be377f8James Zern const int v = (int)MULT_FIX_C(irow[x_out] - frac, wrk->fxy_scale); 1497c8da7ce66017295a65ec028084b90800be377f8James Zern assert(v >= 0 && v <= 255); 1507c8da7ce66017295a65ec028084b90800be377f8James Zern dst[x_out] = v; 1517c8da7ce66017295a65ec028084b90800be377f8James Zern irow[x_out] = frac; // new fractional start 1527c8da7ce66017295a65ec028084b90800be377f8James Zern } 1537c8da7ce66017295a65ec028084b90800be377f8James Zern } else { 1547c8da7ce66017295a65ec028084b90800be377f8James Zern for (x_out = 0; x_out < max_span; x_out += 8) { 1557c8da7ce66017295a65ec028084b90800be377f8James Zern LOAD_32x8(irow + x_out, in0, in1); 1567c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t A0 = MULT_FIX(in0, fxy_scale_half); 1577c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32x4_t A1 = MULT_FIX(in1, fxy_scale_half); 1587c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16x4_t B0 = vmovn_u32(A0); 1597c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16x4_t B1 = vmovn_u32(A1); 1607c8da7ce66017295a65ec028084b90800be377f8James Zern const uint8x8_t C = vmovn_u16(vcombine_u16(B0, B1)); 1617c8da7ce66017295a65ec028084b90800be377f8James Zern vst1_u8(dst + x_out, C); 1627c8da7ce66017295a65ec028084b90800be377f8James Zern STORE_32x8(zero, zero, irow + x_out); 1637c8da7ce66017295a65ec028084b90800be377f8James Zern } 1647c8da7ce66017295a65ec028084b90800be377f8James Zern for (; x_out < x_out_max; ++x_out) { 1657c8da7ce66017295a65ec028084b90800be377f8James Zern const int v = (int)MULT_FIX_C(irow[x_out], fxy_scale); 1667c8da7ce66017295a65ec028084b90800be377f8James Zern assert(v >= 0 && v <= 255); 1677c8da7ce66017295a65ec028084b90800be377f8James Zern dst[x_out] = v; 1687c8da7ce66017295a65ec028084b90800be377f8James Zern irow[x_out] = 0; 1697c8da7ce66017295a65ec028084b90800be377f8James Zern } 1707c8da7ce66017295a65ec028084b90800be377f8James Zern } 1717c8da7ce66017295a65ec028084b90800be377f8James Zern} 1727c8da7ce66017295a65ec028084b90800be377f8James Zern 1737c8da7ce66017295a65ec028084b90800be377f8James Zern//------------------------------------------------------------------------------ 1747c8da7ce66017295a65ec028084b90800be377f8James Zern 1757c8da7ce66017295a65ec028084b90800be377f8James Zernextern void WebPRescalerDspInitNEON(void); 1767c8da7ce66017295a65ec028084b90800be377f8James Zern 1777c8da7ce66017295a65ec028084b90800be377f8James ZernWEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitNEON(void) { 1787c8da7ce66017295a65ec028084b90800be377f8James Zern WebPRescalerExportRowExpand = RescalerExportRowExpand; 1797c8da7ce66017295a65ec028084b90800be377f8James Zern WebPRescalerExportRowShrink = RescalerExportRowShrink; 1807c8da7ce66017295a65ec028084b90800be377f8James Zern} 1817c8da7ce66017295a65ec028084b90800be377f8James Zern 1827c8da7ce66017295a65ec028084b90800be377f8James Zern#else // !WEBP_USE_NEON 1837c8da7ce66017295a65ec028084b90800be377f8James Zern 1847c8da7ce66017295a65ec028084b90800be377f8James ZernWEBP_DSP_INIT_STUB(WebPRescalerDspInitNEON) 1857c8da7ce66017295a65ec028084b90800be377f8James Zern 1867c8da7ce66017295a65ec028084b90800be377f8James Zern#endif // WEBP_USE_NEON 187