17c8da7ce66017295a65ec028084b90800be377f8James Zern// Copyright 2015 Google Inc. All Rights Reserved. 27c8da7ce66017295a65ec028084b90800be377f8James Zern// 37c8da7ce66017295a65ec028084b90800be377f8James Zern// Use of this source code is governed by a BSD-style license 47c8da7ce66017295a65ec028084b90800be377f8James Zern// that can be found in the COPYING file in the root of the source 57c8da7ce66017295a65ec028084b90800be377f8James Zern// tree. An additional intellectual property rights grant can be found 67c8da7ce66017295a65ec028084b90800be377f8James Zern// in the file PATENTS. All contributing project authors may 77c8da7ce66017295a65ec028084b90800be377f8James Zern// be found in the AUTHORS file in the root of the source tree. 87c8da7ce66017295a65ec028084b90800be377f8James Zern// ----------------------------------------------------------------------------- 97c8da7ce66017295a65ec028084b90800be377f8James Zern// 107c8da7ce66017295a65ec028084b90800be377f8James Zern// SSE2 Rescaling functions 117c8da7ce66017295a65ec028084b90800be377f8James Zern// 127c8da7ce66017295a65ec028084b90800be377f8James Zern// Author: Skal (pascal.massimino@gmail.com) 137c8da7ce66017295a65ec028084b90800be377f8James Zern 147c8da7ce66017295a65ec028084b90800be377f8James Zern#include "./dsp.h" 157c8da7ce66017295a65ec028084b90800be377f8James Zern 167c8da7ce66017295a65ec028084b90800be377f8James Zern#if defined(WEBP_USE_SSE2) 177c8da7ce66017295a65ec028084b90800be377f8James Zern#include <emmintrin.h> 187c8da7ce66017295a65ec028084b90800be377f8James Zern 197c8da7ce66017295a65ec028084b90800be377f8James Zern#include <assert.h> 20fa39824bb690c5806358871f46940d0450973d8aJames Zern#include "../utils/rescaler_utils.h" 210912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#include "../utils/utils.h" 227c8da7ce66017295a65ec028084b90800be377f8James Zern 237c8da7ce66017295a65ec028084b90800be377f8James Zern//------------------------------------------------------------------------------ 247c8da7ce66017295a65ec028084b90800be377f8James Zern// Implementations of critical functions ImportRow / ExportRow 257c8da7ce66017295a65ec028084b90800be377f8James Zern 267c8da7ce66017295a65ec028084b90800be377f8James Zern#define ROUNDER (WEBP_RESCALER_ONE >> 1) 277c8da7ce66017295a65ec028084b90800be377f8James Zern#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) 287c8da7ce66017295a65ec028084b90800be377f8James Zern 297c8da7ce66017295a65ec028084b90800be377f8James Zern// input: 8 bytes ABCDEFGH -> output: A0E0B0F0C0G0D0H0 307c8da7ce66017295a65ec028084b90800be377f8James Zernstatic void LoadTwoPixels(const uint8_t* const src, __m128i* out) { 317c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i zero = _mm_setzero_si128(); 327c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i A = _mm_loadl_epi64((const __m128i*)(src)); // ABCDEFGH 337c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i B = _mm_unpacklo_epi8(A, zero); // A0B0C0D0E0F0G0H0 347c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C = _mm_srli_si128(B, 8); // E0F0G0H0 357c8da7ce66017295a65ec028084b90800be377f8James Zern *out = _mm_unpacklo_epi16(B, C); 367c8da7ce66017295a65ec028084b90800be377f8James Zern} 377c8da7ce66017295a65ec028084b90800be377f8James Zern 387c8da7ce66017295a65ec028084b90800be377f8James Zern// input: 8 bytes ABCDEFGH -> output: A0B0C0D0E0F0G0H0 397c8da7ce66017295a65ec028084b90800be377f8James Zernstatic void LoadHeightPixels(const uint8_t* const src, __m128i* out) { 407c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i zero = _mm_setzero_si128(); 417c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i A = _mm_loadl_epi64((const __m128i*)(src)); // ABCDEFGH 427c8da7ce66017295a65ec028084b90800be377f8James Zern *out = _mm_unpacklo_epi8(A, zero); 437c8da7ce66017295a65ec028084b90800be377f8James Zern} 447c8da7ce66017295a65ec028084b90800be377f8James Zern 457c8da7ce66017295a65ec028084b90800be377f8James Zernstatic void RescalerImportRowExpandSSE2(WebPRescaler* const wrk, 467c8da7ce66017295a65ec028084b90800be377f8James Zern const uint8_t* src) { 477c8da7ce66017295a65ec028084b90800be377f8James Zern rescaler_t* frow = wrk->frow; 487c8da7ce66017295a65ec028084b90800be377f8James Zern const rescaler_t* const frow_end = frow + wrk->dst_width * wrk->num_channels; 497c8da7ce66017295a65ec028084b90800be377f8James Zern const int x_add = wrk->x_add; 507c8da7ce66017295a65ec028084b90800be377f8James Zern int accum = x_add; 517c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i cur_pixels; 527c8da7ce66017295a65ec028084b90800be377f8James Zern 537c8da7ce66017295a65ec028084b90800be377f8James Zern assert(!WebPRescalerInputDone(wrk)); 547c8da7ce66017295a65ec028084b90800be377f8James Zern assert(wrk->x_expand); 557c8da7ce66017295a65ec028084b90800be377f8James Zern if (wrk->num_channels == 4) { 567c8da7ce66017295a65ec028084b90800be377f8James Zern if (wrk->src_width < 2) { 577c8da7ce66017295a65ec028084b90800be377f8James Zern WebPRescalerImportRowExpandC(wrk, src); 587c8da7ce66017295a65ec028084b90800be377f8James Zern return; 597c8da7ce66017295a65ec028084b90800be377f8James Zern } 607c8da7ce66017295a65ec028084b90800be377f8James Zern LoadTwoPixels(src, &cur_pixels); 617c8da7ce66017295a65ec028084b90800be377f8James Zern src += 4; 627c8da7ce66017295a65ec028084b90800be377f8James Zern while (1) { 637c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mult = _mm_set1_epi32(((x_add - accum) << 16) | accum); 647c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i out = _mm_madd_epi16(cur_pixels, mult); 657c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)frow, out); 667c8da7ce66017295a65ec028084b90800be377f8James Zern frow += 4; 677c8da7ce66017295a65ec028084b90800be377f8James Zern if (frow >= frow_end) break; 687c8da7ce66017295a65ec028084b90800be377f8James Zern accum -= wrk->x_sub; 697c8da7ce66017295a65ec028084b90800be377f8James Zern if (accum < 0) { 707c8da7ce66017295a65ec028084b90800be377f8James Zern LoadTwoPixels(src, &cur_pixels); 717c8da7ce66017295a65ec028084b90800be377f8James Zern src += 4; 727c8da7ce66017295a65ec028084b90800be377f8James Zern accum += x_add; 737c8da7ce66017295a65ec028084b90800be377f8James Zern } 747c8da7ce66017295a65ec028084b90800be377f8James Zern } 757c8da7ce66017295a65ec028084b90800be377f8James Zern } else { 767c8da7ce66017295a65ec028084b90800be377f8James Zern int left; 777c8da7ce66017295a65ec028084b90800be377f8James Zern const uint8_t* const src_limit = src + wrk->src_width - 8; 787c8da7ce66017295a65ec028084b90800be377f8James Zern if (wrk->src_width < 8) { 797c8da7ce66017295a65ec028084b90800be377f8James Zern WebPRescalerImportRowExpandC(wrk, src); 807c8da7ce66017295a65ec028084b90800be377f8James Zern return; 817c8da7ce66017295a65ec028084b90800be377f8James Zern } 827c8da7ce66017295a65ec028084b90800be377f8James Zern LoadHeightPixels(src, &cur_pixels); 837c8da7ce66017295a65ec028084b90800be377f8James Zern src += 7; 847c8da7ce66017295a65ec028084b90800be377f8James Zern left = 7; 857c8da7ce66017295a65ec028084b90800be377f8James Zern while (1) { 867c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mult = _mm_cvtsi32_si128(((x_add - accum) << 16) | accum); 877c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i out = _mm_madd_epi16(cur_pixels, mult); 887c8da7ce66017295a65ec028084b90800be377f8James Zern assert(sizeof(*frow) == sizeof(uint32_t)); 897c8da7ce66017295a65ec028084b90800be377f8James Zern WebPUint32ToMem((uint8_t*)frow, _mm_cvtsi128_si32(out)); 907c8da7ce66017295a65ec028084b90800be377f8James Zern frow += 1; 917c8da7ce66017295a65ec028084b90800be377f8James Zern if (frow >= frow_end) break; 927c8da7ce66017295a65ec028084b90800be377f8James Zern accum -= wrk->x_sub; 937c8da7ce66017295a65ec028084b90800be377f8James Zern if (accum < 0) { 947c8da7ce66017295a65ec028084b90800be377f8James Zern if (--left) { 957c8da7ce66017295a65ec028084b90800be377f8James Zern cur_pixels = _mm_srli_si128(cur_pixels, 2); 967c8da7ce66017295a65ec028084b90800be377f8James Zern } else if (src <= src_limit) { 977c8da7ce66017295a65ec028084b90800be377f8James Zern LoadHeightPixels(src, &cur_pixels); 987c8da7ce66017295a65ec028084b90800be377f8James Zern src += 7; 997c8da7ce66017295a65ec028084b90800be377f8James Zern left = 7; 1007c8da7ce66017295a65ec028084b90800be377f8James Zern } else { // tail 1017c8da7ce66017295a65ec028084b90800be377f8James Zern cur_pixels = _mm_srli_si128(cur_pixels, 2); 1027c8da7ce66017295a65ec028084b90800be377f8James Zern cur_pixels = _mm_insert_epi16(cur_pixels, src[1], 1); 1037c8da7ce66017295a65ec028084b90800be377f8James Zern src += 1; 1047c8da7ce66017295a65ec028084b90800be377f8James Zern left = 1; 1057c8da7ce66017295a65ec028084b90800be377f8James Zern } 1067c8da7ce66017295a65ec028084b90800be377f8James Zern accum += x_add; 1077c8da7ce66017295a65ec028084b90800be377f8James Zern } 1087c8da7ce66017295a65ec028084b90800be377f8James Zern } 1097c8da7ce66017295a65ec028084b90800be377f8James Zern } 1107c8da7ce66017295a65ec028084b90800be377f8James Zern assert(accum == 0); 1117c8da7ce66017295a65ec028084b90800be377f8James Zern} 1127c8da7ce66017295a65ec028084b90800be377f8James Zern 1137c8da7ce66017295a65ec028084b90800be377f8James Zernstatic void RescalerImportRowShrinkSSE2(WebPRescaler* const wrk, 1147c8da7ce66017295a65ec028084b90800be377f8James Zern const uint8_t* src) { 1157c8da7ce66017295a65ec028084b90800be377f8James Zern const int x_sub = wrk->x_sub; 1167c8da7ce66017295a65ec028084b90800be377f8James Zern int accum = 0; 1177c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i zero = _mm_setzero_si128(); 1187c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mult0 = _mm_set1_epi16(x_sub); 1197c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mult1 = _mm_set1_epi32(wrk->fx_scale); 1207c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); 1217c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i sum = zero; 1227c8da7ce66017295a65ec028084b90800be377f8James Zern rescaler_t* frow = wrk->frow; 1237c8da7ce66017295a65ec028084b90800be377f8James Zern const rescaler_t* const frow_end = wrk->frow + 4 * wrk->dst_width; 1247c8da7ce66017295a65ec028084b90800be377f8James Zern 1257c8da7ce66017295a65ec028084b90800be377f8James Zern if (wrk->num_channels != 4 || wrk->x_add > (x_sub << 7)) { 1267c8da7ce66017295a65ec028084b90800be377f8James Zern WebPRescalerImportRowShrinkC(wrk, src); 1277c8da7ce66017295a65ec028084b90800be377f8James Zern return; 1287c8da7ce66017295a65ec028084b90800be377f8James Zern } 1297c8da7ce66017295a65ec028084b90800be377f8James Zern assert(!WebPRescalerInputDone(wrk)); 1307c8da7ce66017295a65ec028084b90800be377f8James Zern assert(!wrk->x_expand); 1317c8da7ce66017295a65ec028084b90800be377f8James Zern 1327c8da7ce66017295a65ec028084b90800be377f8James Zern for (; frow < frow_end; frow += 4) { 1337c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i base = zero; 1347c8da7ce66017295a65ec028084b90800be377f8James Zern accum += wrk->x_add; 1357c8da7ce66017295a65ec028084b90800be377f8James Zern while (accum > 0) { 1367c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i A = _mm_cvtsi32_si128(WebPMemToUint32(src)); 1377c8da7ce66017295a65ec028084b90800be377f8James Zern src += 4; 1387c8da7ce66017295a65ec028084b90800be377f8James Zern base = _mm_unpacklo_epi8(A, zero); 1397c8da7ce66017295a65ec028084b90800be377f8James Zern // To avoid overflow, we need: base * x_add / x_sub < 32768 1407c8da7ce66017295a65ec028084b90800be377f8James Zern // => x_add < x_sub << 7. That's a 1/128 reduction ratio limit. 1417c8da7ce66017295a65ec028084b90800be377f8James Zern sum = _mm_add_epi16(sum, base); 1427c8da7ce66017295a65ec028084b90800be377f8James Zern accum -= x_sub; 1437c8da7ce66017295a65ec028084b90800be377f8James Zern } 1447c8da7ce66017295a65ec028084b90800be377f8James Zern { // Emit next horizontal pixel. 1457c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mult = _mm_set1_epi16(-accum); 1467c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i frac0 = _mm_mullo_epi16(base, mult); // 16b x 16b -> 32b 1477c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i frac1 = _mm_mulhi_epu16(base, mult); 1487c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i frac = _mm_unpacklo_epi16(frac0, frac1); // frac is 32b 1497c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i A0 = _mm_mullo_epi16(sum, mult0); 1507c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i A1 = _mm_mulhi_epu16(sum, mult0); 1517c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i B0 = _mm_unpacklo_epi16(A0, A1); // sum * x_sub 1527c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i frow_out = _mm_sub_epi32(B0, frac); // sum * x_sub - frac 1537c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D0 = _mm_srli_epi64(frac, 32); 1547c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D1 = _mm_mul_epu32(frac, mult1); // 32b x 16b -> 64b 1557c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D2 = _mm_mul_epu32(D0, mult1); 1567c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E1 = _mm_add_epi64(D1, rounder); 1577c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E2 = _mm_add_epi64(D2, rounder); 1587c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i F1 = _mm_shuffle_epi32(E1, 1 | (3 << 2)); 1597c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i F2 = _mm_shuffle_epi32(E2, 1 | (3 << 2)); 1607c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i G = _mm_unpacklo_epi32(F1, F2); 1617c8da7ce66017295a65ec028084b90800be377f8James Zern sum = _mm_packs_epi32(G, zero); 1627c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)frow, frow_out); 1637c8da7ce66017295a65ec028084b90800be377f8James Zern } 1647c8da7ce66017295a65ec028084b90800be377f8James Zern } 1657c8da7ce66017295a65ec028084b90800be377f8James Zern assert(accum == 0); 1667c8da7ce66017295a65ec028084b90800be377f8James Zern} 1677c8da7ce66017295a65ec028084b90800be377f8James Zern 1687c8da7ce66017295a65ec028084b90800be377f8James Zern//------------------------------------------------------------------------------ 1697c8da7ce66017295a65ec028084b90800be377f8James Zern// Row export 1707c8da7ce66017295a65ec028084b90800be377f8James Zern 1717c8da7ce66017295a65ec028084b90800be377f8James Zern// load *src as epi64, multiply by mult and store result in [out0 ... out3] 1727c8da7ce66017295a65ec028084b90800be377f8James Zernstatic WEBP_INLINE void LoadDispatchAndMult(const rescaler_t* const src, 1737c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i* const mult, 1747c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i* const out0, 1757c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i* const out1, 1767c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i* const out2, 1777c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i* const out3) { 1787c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + 0)); 1797c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + 4)); 1807c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i A2 = _mm_srli_epi64(A0, 32); 1817c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i A3 = _mm_srli_epi64(A1, 32); 1827c8da7ce66017295a65ec028084b90800be377f8James Zern if (mult != NULL) { 1837c8da7ce66017295a65ec028084b90800be377f8James Zern *out0 = _mm_mul_epu32(A0, *mult); 1847c8da7ce66017295a65ec028084b90800be377f8James Zern *out1 = _mm_mul_epu32(A1, *mult); 1857c8da7ce66017295a65ec028084b90800be377f8James Zern *out2 = _mm_mul_epu32(A2, *mult); 1867c8da7ce66017295a65ec028084b90800be377f8James Zern *out3 = _mm_mul_epu32(A3, *mult); 1877c8da7ce66017295a65ec028084b90800be377f8James Zern } else { 1887c8da7ce66017295a65ec028084b90800be377f8James Zern *out0 = A0; 1897c8da7ce66017295a65ec028084b90800be377f8James Zern *out1 = A1; 1907c8da7ce66017295a65ec028084b90800be377f8James Zern *out2 = A2; 1917c8da7ce66017295a65ec028084b90800be377f8James Zern *out3 = A3; 1927c8da7ce66017295a65ec028084b90800be377f8James Zern } 1937c8da7ce66017295a65ec028084b90800be377f8James Zern} 1947c8da7ce66017295a65ec028084b90800be377f8James Zern 1957c8da7ce66017295a65ec028084b90800be377f8James Zernstatic WEBP_INLINE void ProcessRow(const __m128i* const A0, 1967c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i* const A1, 1977c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i* const A2, 1987c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i* const A3, 1997c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i* const mult, 2007c8da7ce66017295a65ec028084b90800be377f8James Zern uint8_t* const dst) { 2017c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); 2027c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mask = _mm_set_epi32(0xffffffffu, 0, 0xffffffffu, 0); 2037c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i B0 = _mm_mul_epu32(*A0, *mult); 2047c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i B1 = _mm_mul_epu32(*A1, *mult); 2057c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i B2 = _mm_mul_epu32(*A2, *mult); 2067c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i B3 = _mm_mul_epu32(*A3, *mult); 2077c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C0 = _mm_add_epi64(B0, rounder); 2087c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C1 = _mm_add_epi64(B1, rounder); 2097c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C2 = _mm_add_epi64(B2, rounder); 2107c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C3 = _mm_add_epi64(B3, rounder); 2117c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX); 2127c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX); 2137c8da7ce66017295a65ec028084b90800be377f8James Zern#if (WEBP_RESCALER_FIX < 32) 2147c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D2 = 2157c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_and_si128(_mm_slli_epi64(C2, 32 - WEBP_RESCALER_RFIX), mask); 2167c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D3 = 2177c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_and_si128(_mm_slli_epi64(C3, 32 - WEBP_RESCALER_RFIX), mask); 2187c8da7ce66017295a65ec028084b90800be377f8James Zern#else 2197c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D2 = _mm_and_si128(C2, mask); 2207c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D3 = _mm_and_si128(C3, mask); 2217c8da7ce66017295a65ec028084b90800be377f8James Zern#endif 2227c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E0 = _mm_or_si128(D0, D2); 2237c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E1 = _mm_or_si128(D1, D3); 2247c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i F = _mm_packs_epi32(E0, E1); 2257c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i G = _mm_packus_epi16(F, F); 2267c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storel_epi64((__m128i*)dst, G); 2277c8da7ce66017295a65ec028084b90800be377f8James Zern} 2287c8da7ce66017295a65ec028084b90800be377f8James Zern 2297c8da7ce66017295a65ec028084b90800be377f8James Zernstatic void RescalerExportRowExpandSSE2(WebPRescaler* const wrk) { 2307c8da7ce66017295a65ec028084b90800be377f8James Zern int x_out; 2317c8da7ce66017295a65ec028084b90800be377f8James Zern uint8_t* const dst = wrk->dst; 2327c8da7ce66017295a65ec028084b90800be377f8James Zern rescaler_t* const irow = wrk->irow; 2337c8da7ce66017295a65ec028084b90800be377f8James Zern const int x_out_max = wrk->dst_width * wrk->num_channels; 2347c8da7ce66017295a65ec028084b90800be377f8James Zern const rescaler_t* const frow = wrk->frow; 2357c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mult = _mm_set_epi32(0, wrk->fy_scale, 0, wrk->fy_scale); 2367c8da7ce66017295a65ec028084b90800be377f8James Zern 2377c8da7ce66017295a65ec028084b90800be377f8James Zern assert(!WebPRescalerOutputDone(wrk)); 2387c8da7ce66017295a65ec028084b90800be377f8James Zern assert(wrk->y_accum <= 0 && wrk->y_sub + wrk->y_accum >= 0); 2397c8da7ce66017295a65ec028084b90800be377f8James Zern assert(wrk->y_expand); 2407c8da7ce66017295a65ec028084b90800be377f8James Zern if (wrk->y_accum == 0) { 2417c8da7ce66017295a65ec028084b90800be377f8James Zern for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { 2427c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i A0, A1, A2, A3; 2437c8da7ce66017295a65ec028084b90800be377f8James Zern LoadDispatchAndMult(frow + x_out, NULL, &A0, &A1, &A2, &A3); 2447c8da7ce66017295a65ec028084b90800be377f8James Zern ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out); 2457c8da7ce66017295a65ec028084b90800be377f8James Zern } 2467c8da7ce66017295a65ec028084b90800be377f8James Zern for (; x_out < x_out_max; ++x_out) { 2477c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t J = frow[x_out]; 2487c8da7ce66017295a65ec028084b90800be377f8James Zern const int v = (int)MULT_FIX(J, wrk->fy_scale); 2497c8da7ce66017295a65ec028084b90800be377f8James Zern assert(v >= 0 && v <= 255); 2507c8da7ce66017295a65ec028084b90800be377f8James Zern dst[x_out] = v; 2517c8da7ce66017295a65ec028084b90800be377f8James Zern } 2527c8da7ce66017295a65ec028084b90800be377f8James Zern } else { 2537c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); 2547c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); 2557c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mA = _mm_set_epi32(0, A, 0, A); 2567c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mB = _mm_set_epi32(0, B, 0, B); 2577c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); 2587c8da7ce66017295a65ec028084b90800be377f8James Zern for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { 2597c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i A0, A1, A2, A3, B0, B1, B2, B3; 2607c8da7ce66017295a65ec028084b90800be377f8James Zern LoadDispatchAndMult(frow + x_out, &mA, &A0, &A1, &A2, &A3); 2617c8da7ce66017295a65ec028084b90800be377f8James Zern LoadDispatchAndMult(irow + x_out, &mB, &B0, &B1, &B2, &B3); 2627c8da7ce66017295a65ec028084b90800be377f8James Zern { 2637c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C0 = _mm_add_epi64(A0, B0); 2647c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C1 = _mm_add_epi64(A1, B1); 2657c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C2 = _mm_add_epi64(A2, B2); 2667c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C3 = _mm_add_epi64(A3, B3); 2677c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D0 = _mm_add_epi64(C0, rounder); 2687c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D1 = _mm_add_epi64(C1, rounder); 2697c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D2 = _mm_add_epi64(C2, rounder); 2707c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D3 = _mm_add_epi64(C3, rounder); 2717c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E0 = _mm_srli_epi64(D0, WEBP_RESCALER_RFIX); 2727c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E1 = _mm_srli_epi64(D1, WEBP_RESCALER_RFIX); 2737c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E2 = _mm_srli_epi64(D2, WEBP_RESCALER_RFIX); 2747c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E3 = _mm_srli_epi64(D3, WEBP_RESCALER_RFIX); 2757c8da7ce66017295a65ec028084b90800be377f8James Zern ProcessRow(&E0, &E1, &E2, &E3, &mult, dst + x_out); 2767c8da7ce66017295a65ec028084b90800be377f8James Zern } 2777c8da7ce66017295a65ec028084b90800be377f8James Zern } 2787c8da7ce66017295a65ec028084b90800be377f8James Zern for (; x_out < x_out_max; ++x_out) { 2797c8da7ce66017295a65ec028084b90800be377f8James Zern const uint64_t I = (uint64_t)A * frow[x_out] 2807c8da7ce66017295a65ec028084b90800be377f8James Zern + (uint64_t)B * irow[x_out]; 2817c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); 2827c8da7ce66017295a65ec028084b90800be377f8James Zern const int v = (int)MULT_FIX(J, wrk->fy_scale); 2837c8da7ce66017295a65ec028084b90800be377f8James Zern assert(v >= 0 && v <= 255); 2847c8da7ce66017295a65ec028084b90800be377f8James Zern dst[x_out] = v; 2857c8da7ce66017295a65ec028084b90800be377f8James Zern } 2867c8da7ce66017295a65ec028084b90800be377f8James Zern } 2877c8da7ce66017295a65ec028084b90800be377f8James Zern} 2887c8da7ce66017295a65ec028084b90800be377f8James Zern 2897c8da7ce66017295a65ec028084b90800be377f8James Zernstatic void RescalerExportRowShrinkSSE2(WebPRescaler* const wrk) { 2907c8da7ce66017295a65ec028084b90800be377f8James Zern int x_out; 2917c8da7ce66017295a65ec028084b90800be377f8James Zern uint8_t* const dst = wrk->dst; 2927c8da7ce66017295a65ec028084b90800be377f8James Zern rescaler_t* const irow = wrk->irow; 2937c8da7ce66017295a65ec028084b90800be377f8James Zern const int x_out_max = wrk->dst_width * wrk->num_channels; 2947c8da7ce66017295a65ec028084b90800be377f8James Zern const rescaler_t* const frow = wrk->frow; 2957c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); 2967c8da7ce66017295a65ec028084b90800be377f8James Zern assert(!WebPRescalerOutputDone(wrk)); 2977c8da7ce66017295a65ec028084b90800be377f8James Zern assert(wrk->y_accum <= 0); 2987c8da7ce66017295a65ec028084b90800be377f8James Zern assert(!wrk->y_expand); 2997c8da7ce66017295a65ec028084b90800be377f8James Zern if (yscale) { 3007c8da7ce66017295a65ec028084b90800be377f8James Zern const int scale_xy = wrk->fxy_scale; 3017c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mult_xy = _mm_set_epi32(0, scale_xy, 0, scale_xy); 3027c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mult_y = _mm_set_epi32(0, yscale, 0, yscale); 3037c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); 3047c8da7ce66017295a65ec028084b90800be377f8James Zern for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { 3057c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i A0, A1, A2, A3, B0, B1, B2, B3; 3067c8da7ce66017295a65ec028084b90800be377f8James Zern LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3); 3077c8da7ce66017295a65ec028084b90800be377f8James Zern LoadDispatchAndMult(frow + x_out, &mult_y, &B0, &B1, &B2, &B3); 3087c8da7ce66017295a65ec028084b90800be377f8James Zern { 3097c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C0 = _mm_add_epi64(B0, rounder); 3107c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C1 = _mm_add_epi64(B1, rounder); 3117c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C2 = _mm_add_epi64(B2, rounder); 3127c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i C3 = _mm_add_epi64(B3, rounder); 3137c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX); // = frac 3147c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX); 3157c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D2 = _mm_srli_epi64(C2, WEBP_RESCALER_RFIX); 3167c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i D3 = _mm_srli_epi64(C3, WEBP_RESCALER_RFIX); 3177c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E0 = _mm_sub_epi64(A0, D0); // irow[x] - frac 3187c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E1 = _mm_sub_epi64(A1, D1); 3197c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E2 = _mm_sub_epi64(A2, D2); 3207c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i E3 = _mm_sub_epi64(A3, D3); 3217c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i F2 = _mm_slli_epi64(D2, 32); 3227c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i F3 = _mm_slli_epi64(D3, 32); 3237c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i G0 = _mm_or_si128(D0, F2); 3247c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i G1 = _mm_or_si128(D1, F3); 3257c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)(irow + x_out + 0), G0); 3267c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)(irow + x_out + 4), G1); 3277c8da7ce66017295a65ec028084b90800be377f8James Zern ProcessRow(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out); 3287c8da7ce66017295a65ec028084b90800be377f8James Zern } 3297c8da7ce66017295a65ec028084b90800be377f8James Zern } 3307c8da7ce66017295a65ec028084b90800be377f8James Zern for (; x_out < x_out_max; ++x_out) { 3317c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t frac = (int)MULT_FIX(frow[x_out], yscale); 3327c8da7ce66017295a65ec028084b90800be377f8James Zern const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); 3337c8da7ce66017295a65ec028084b90800be377f8James Zern assert(v >= 0 && v <= 255); 3347c8da7ce66017295a65ec028084b90800be377f8James Zern dst[x_out] = v; 3357c8da7ce66017295a65ec028084b90800be377f8James Zern irow[x_out] = frac; // new fractional start 3367c8da7ce66017295a65ec028084b90800be377f8James Zern } 3377c8da7ce66017295a65ec028084b90800be377f8James Zern } else { 3387c8da7ce66017295a65ec028084b90800be377f8James Zern const uint32_t scale = wrk->fxy_scale; 3397c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i mult = _mm_set_epi32(0, scale, 0, scale); 3407c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i zero = _mm_setzero_si128(); 3417c8da7ce66017295a65ec028084b90800be377f8James Zern for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { 3427c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i A0, A1, A2, A3; 3437c8da7ce66017295a65ec028084b90800be377f8James Zern LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3); 3447c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)(irow + x_out + 0), zero); 3457c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)(irow + x_out + 4), zero); 3467c8da7ce66017295a65ec028084b90800be377f8James Zern ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out); 3477c8da7ce66017295a65ec028084b90800be377f8James Zern } 3487c8da7ce66017295a65ec028084b90800be377f8James Zern for (; x_out < x_out_max; ++x_out) { 3497c8da7ce66017295a65ec028084b90800be377f8James Zern const int v = (int)MULT_FIX(irow[x_out], scale); 3507c8da7ce66017295a65ec028084b90800be377f8James Zern assert(v >= 0 && v <= 255); 3517c8da7ce66017295a65ec028084b90800be377f8James Zern dst[x_out] = v; 3527c8da7ce66017295a65ec028084b90800be377f8James Zern irow[x_out] = 0; 3537c8da7ce66017295a65ec028084b90800be377f8James Zern } 3547c8da7ce66017295a65ec028084b90800be377f8James Zern } 3557c8da7ce66017295a65ec028084b90800be377f8James Zern} 3567c8da7ce66017295a65ec028084b90800be377f8James Zern 3577c8da7ce66017295a65ec028084b90800be377f8James Zern#undef MULT_FIX 3587c8da7ce66017295a65ec028084b90800be377f8James Zern#undef ROUNDER 3597c8da7ce66017295a65ec028084b90800be377f8James Zern 3607c8da7ce66017295a65ec028084b90800be377f8James Zern//------------------------------------------------------------------------------ 3617c8da7ce66017295a65ec028084b90800be377f8James Zern 3627c8da7ce66017295a65ec028084b90800be377f8James Zernextern void WebPRescalerDspInitSSE2(void); 3637c8da7ce66017295a65ec028084b90800be377f8James Zern 3647c8da7ce66017295a65ec028084b90800be377f8James ZernWEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitSSE2(void) { 3657c8da7ce66017295a65ec028084b90800be377f8James Zern WebPRescalerImportRowExpand = RescalerImportRowExpandSSE2; 3667c8da7ce66017295a65ec028084b90800be377f8James Zern WebPRescalerImportRowShrink = RescalerImportRowShrinkSSE2; 3677c8da7ce66017295a65ec028084b90800be377f8James Zern WebPRescalerExportRowExpand = RescalerExportRowExpandSSE2; 3687c8da7ce66017295a65ec028084b90800be377f8James Zern WebPRescalerExportRowShrink = RescalerExportRowShrinkSSE2; 3697c8da7ce66017295a65ec028084b90800be377f8James Zern} 3707c8da7ce66017295a65ec028084b90800be377f8James Zern 3717c8da7ce66017295a65ec028084b90800be377f8James Zern#else // !WEBP_USE_SSE2 3727c8da7ce66017295a65ec028084b90800be377f8James Zern 3737c8da7ce66017295a65ec028084b90800be377f8James ZernWEBP_DSP_INIT_STUB(WebPRescalerDspInitSSE2) 3747c8da7ce66017295a65ec028084b90800be377f8James Zern 3757c8da7ce66017295a65ec028084b90800be377f8James Zern#endif // WEBP_USE_SSE2 376