17c8da7ce66017295a65ec028084b90800be377f8James Zern// Copyright 2015 Google Inc. All Rights Reserved. 27c8da7ce66017295a65ec028084b90800be377f8James Zern// 37c8da7ce66017295a65ec028084b90800be377f8James Zern// Use of this source code is governed by a BSD-style license 47c8da7ce66017295a65ec028084b90800be377f8James Zern// that can be found in the COPYING file in the root of the source 57c8da7ce66017295a65ec028084b90800be377f8James Zern// tree. An additional intellectual property rights grant can be found 67c8da7ce66017295a65ec028084b90800be377f8James Zern// in the file PATENTS. All contributing project authors may 77c8da7ce66017295a65ec028084b90800be377f8James Zern// be found in the AUTHORS file in the root of the source tree. 87c8da7ce66017295a65ec028084b90800be377f8James Zern// ----------------------------------------------------------------------------- 97c8da7ce66017295a65ec028084b90800be377f8James Zern// 107c8da7ce66017295a65ec028084b90800be377f8James Zern// SSE4 version of some encoding functions. 117c8da7ce66017295a65ec028084b90800be377f8James Zern// 127c8da7ce66017295a65ec028084b90800be377f8James Zern// Author: Skal (pascal.massimino@gmail.com) 137c8da7ce66017295a65ec028084b90800be377f8James Zern 147c8da7ce66017295a65ec028084b90800be377f8James Zern#include "./dsp.h" 157c8da7ce66017295a65ec028084b90800be377f8James Zern 167c8da7ce66017295a65ec028084b90800be377f8James Zern#if defined(WEBP_USE_SSE41) 177c8da7ce66017295a65ec028084b90800be377f8James Zern#include <smmintrin.h> 187c8da7ce66017295a65ec028084b90800be377f8James Zern#include <stdlib.h> // for abs() 197c8da7ce66017295a65ec028084b90800be377f8James Zern 200912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern#include "./common_sse2.h" 21fa39824bb690c5806358871f46940d0450973d8aJames Zern#include "../enc/vp8i_enc.h" 227c8da7ce66017295a65ec028084b90800be377f8James Zern 237c8da7ce66017295a65ec028084b90800be377f8James Zern//------------------------------------------------------------------------------ 247c8da7ce66017295a65ec028084b90800be377f8James Zern// Compute susceptibility based on DCT-coeff histograms. 257c8da7ce66017295a65ec028084b90800be377f8James Zern 267c8da7ce66017295a65ec028084b90800be377f8James Zernstatic void CollectHistogram(const uint8_t* ref, const uint8_t* pred, 277c8da7ce66017295a65ec028084b90800be377f8James Zern int start_block, int end_block, 287c8da7ce66017295a65ec028084b90800be377f8James Zern VP8Histogram* const histo) { 297c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i max_coeff_thresh = _mm_set1_epi16(MAX_COEFF_THRESH); 307c8da7ce66017295a65ec028084b90800be377f8James Zern int j; 317c8da7ce66017295a65ec028084b90800be377f8James Zern int distribution[MAX_COEFF_THRESH + 1] = { 0 }; 327c8da7ce66017295a65ec028084b90800be377f8James Zern for (j = start_block; j < end_block; ++j) { 337c8da7ce66017295a65ec028084b90800be377f8James Zern int16_t out[16]; 347c8da7ce66017295a65ec028084b90800be377f8James Zern int k; 357c8da7ce66017295a65ec028084b90800be377f8James Zern 367c8da7ce66017295a65ec028084b90800be377f8James Zern VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); 377c8da7ce66017295a65ec028084b90800be377f8James Zern 387c8da7ce66017295a65ec028084b90800be377f8James Zern // Convert coefficients to bin (within out[]). 397c8da7ce66017295a65ec028084b90800be377f8James Zern { 407c8da7ce66017295a65ec028084b90800be377f8James Zern // Load. 417c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i out0 = _mm_loadu_si128((__m128i*)&out[0]); 427c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i out1 = _mm_loadu_si128((__m128i*)&out[8]); 437c8da7ce66017295a65ec028084b90800be377f8James Zern // v = abs(out) >> 3 447c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i abs0 = _mm_abs_epi16(out0); 457c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i abs1 = _mm_abs_epi16(out1); 467c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i v0 = _mm_srai_epi16(abs0, 3); 477c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i v1 = _mm_srai_epi16(abs1, 3); 487c8da7ce66017295a65ec028084b90800be377f8James Zern // bin = min(v, MAX_COEFF_THRESH) 497c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh); 507c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh); 517c8da7ce66017295a65ec028084b90800be377f8James Zern // Store. 527c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)&out[0], bin0); 537c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)&out[8], bin1); 547c8da7ce66017295a65ec028084b90800be377f8James Zern } 557c8da7ce66017295a65ec028084b90800be377f8James Zern 567c8da7ce66017295a65ec028084b90800be377f8James Zern // Convert coefficients to bin. 577c8da7ce66017295a65ec028084b90800be377f8James Zern for (k = 0; k < 16; ++k) { 587c8da7ce66017295a65ec028084b90800be377f8James Zern ++distribution[out[k]]; 597c8da7ce66017295a65ec028084b90800be377f8James Zern } 607c8da7ce66017295a65ec028084b90800be377f8James Zern } 617c8da7ce66017295a65ec028084b90800be377f8James Zern VP8SetHistogramData(distribution, histo); 627c8da7ce66017295a65ec028084b90800be377f8James Zern} 637c8da7ce66017295a65ec028084b90800be377f8James Zern 647c8da7ce66017295a65ec028084b90800be377f8James Zern//------------------------------------------------------------------------------ 657c8da7ce66017295a65ec028084b90800be377f8James Zern// Texture distortion 667c8da7ce66017295a65ec028084b90800be377f8James Zern// 677c8da7ce66017295a65ec028084b90800be377f8James Zern// We try to match the spectral content (weighted) between source and 687c8da7ce66017295a65ec028084b90800be377f8James Zern// reconstructed samples. 697c8da7ce66017295a65ec028084b90800be377f8James Zern 707c8da7ce66017295a65ec028084b90800be377f8James Zern// Hadamard transform 710912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// Returns the weighted sum of the absolute value of transformed coefficients. 720912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern// w[] contains a row-major 4 by 4 symmetric matrix. 737c8da7ce66017295a65ec028084b90800be377f8James Zernstatic int TTransform(const uint8_t* inA, const uint8_t* inB, 747c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16_t* const w) { 750912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern int32_t sum[4]; 767c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i tmp_0, tmp_1, tmp_2, tmp_3; 777c8da7ce66017295a65ec028084b90800be377f8James Zern 780912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // Load and combine inputs. 797c8da7ce66017295a65ec028084b90800be377f8James Zern { 800912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inA_0 = _mm_loadu_si128((const __m128i*)&inA[BPS * 0]); 810912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inA_1 = _mm_loadu_si128((const __m128i*)&inA[BPS * 1]); 820912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inA_2 = _mm_loadu_si128((const __m128i*)&inA[BPS * 2]); 830912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // In SSE4.1, with gcc 4.8 at least (maybe other versions), 840912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // _mm_loadu_si128 is faster than _mm_loadl_epi64. But for the last lump 850912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // of inA and inB, _mm_loadl_epi64 is still used not to have an out of 860912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // bound read. 877c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i inA_3 = _mm_loadl_epi64((const __m128i*)&inA[BPS * 3]); 880912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inB_0 = _mm_loadu_si128((const __m128i*)&inB[BPS * 0]); 890912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inB_1 = _mm_loadu_si128((const __m128i*)&inB[BPS * 1]); 900912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inB_2 = _mm_loadu_si128((const __m128i*)&inB[BPS * 2]); 917c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i inB_3 = _mm_loadl_epi64((const __m128i*)&inB[BPS * 3]); 927c8da7ce66017295a65ec028084b90800be377f8James Zern 937c8da7ce66017295a65ec028084b90800be377f8James Zern // Combine inA and inB (we'll do two transforms in parallel). 940912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inAB_0 = _mm_unpacklo_epi32(inA_0, inB_0); 950912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inAB_1 = _mm_unpacklo_epi32(inA_1, inB_1); 960912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inAB_2 = _mm_unpacklo_epi32(inA_2, inB_2); 970912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern const __m128i inAB_3 = _mm_unpacklo_epi32(inA_3, inB_3); 980912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern tmp_0 = _mm_cvtepu8_epi16(inAB_0); 990912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern tmp_1 = _mm_cvtepu8_epi16(inAB_1); 1000912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern tmp_2 = _mm_cvtepu8_epi16(inAB_2); 1010912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern tmp_3 = _mm_cvtepu8_epi16(inAB_3); 1020912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a00 a01 a02 a03 b00 b01 b02 b03 1030912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a10 a11 a12 a13 b10 b11 b12 b13 1040912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a20 a21 a22 a23 b20 b21 b22 b23 1050912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // a30 a31 a32 a33 b30 b31 b32 b33 1067c8da7ce66017295a65ec028084b90800be377f8James Zern } 1077c8da7ce66017295a65ec028084b90800be377f8James Zern 1080912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // Vertical pass first to avoid a transpose (vertical and horizontal passes 1090912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // are commutative because w/kWeightY is symmetric) and subsequent transpose. 1107c8da7ce66017295a65ec028084b90800be377f8James Zern { 1117c8da7ce66017295a65ec028084b90800be377f8James Zern // Calculate a and b (two 4x4 at once). 1127c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); 1137c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); 1147c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); 1157c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); 1167c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i b0 = _mm_add_epi16(a0, a1); 1177c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i b1 = _mm_add_epi16(a3, a2); 1187c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i b2 = _mm_sub_epi16(a3, a2); 1197c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i b3 = _mm_sub_epi16(a0, a1); 1207c8da7ce66017295a65ec028084b90800be377f8James Zern // a00 a01 a02 a03 b00 b01 b02 b03 1217c8da7ce66017295a65ec028084b90800be377f8James Zern // a10 a11 a12 a13 b10 b11 b12 b13 1227c8da7ce66017295a65ec028084b90800be377f8James Zern // a20 a21 a22 a23 b20 b21 b22 b23 1237c8da7ce66017295a65ec028084b90800be377f8James Zern // a30 a31 a32 a33 b30 b31 b32 b33 1247c8da7ce66017295a65ec028084b90800be377f8James Zern 1257c8da7ce66017295a65ec028084b90800be377f8James Zern // Transpose the two 4x4. 1260912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern VP8Transpose_2_4x4_16b(&b0, &b1, &b2, &b3, &tmp_0, &tmp_1, &tmp_2, &tmp_3); 1277c8da7ce66017295a65ec028084b90800be377f8James Zern } 1287c8da7ce66017295a65ec028084b90800be377f8James Zern 1290912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern // Horizontal pass and difference of weighted sums. 1307c8da7ce66017295a65ec028084b90800be377f8James Zern { 1317c8da7ce66017295a65ec028084b90800be377f8James Zern // Load all inputs. 1327c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i w_0 = _mm_loadu_si128((const __m128i*)&w[0]); 1337c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i w_8 = _mm_loadu_si128((const __m128i*)&w[8]); 1347c8da7ce66017295a65ec028084b90800be377f8James Zern 1357c8da7ce66017295a65ec028084b90800be377f8James Zern // Calculate a and b (two 4x4 at once). 1367c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); 1377c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); 1387c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); 1397c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); 1407c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i b0 = _mm_add_epi16(a0, a1); 1417c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i b1 = _mm_add_epi16(a3, a2); 1427c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i b2 = _mm_sub_epi16(a3, a2); 1437c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i b3 = _mm_sub_epi16(a0, a1); 1447c8da7ce66017295a65ec028084b90800be377f8James Zern 1457c8da7ce66017295a65ec028084b90800be377f8James Zern // Separate the transforms of inA and inB. 1467c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i A_b0 = _mm_unpacklo_epi64(b0, b1); 1477c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i A_b2 = _mm_unpacklo_epi64(b2, b3); 1487c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i B_b0 = _mm_unpackhi_epi64(b0, b1); 1497c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i B_b2 = _mm_unpackhi_epi64(b2, b3); 1507c8da7ce66017295a65ec028084b90800be377f8James Zern 1517c8da7ce66017295a65ec028084b90800be377f8James Zern A_b0 = _mm_abs_epi16(A_b0); 1527c8da7ce66017295a65ec028084b90800be377f8James Zern A_b2 = _mm_abs_epi16(A_b2); 1537c8da7ce66017295a65ec028084b90800be377f8James Zern B_b0 = _mm_abs_epi16(B_b0); 1547c8da7ce66017295a65ec028084b90800be377f8James Zern B_b2 = _mm_abs_epi16(B_b2); 1557c8da7ce66017295a65ec028084b90800be377f8James Zern 1567c8da7ce66017295a65ec028084b90800be377f8James Zern // weighted sums 1577c8da7ce66017295a65ec028084b90800be377f8James Zern A_b0 = _mm_madd_epi16(A_b0, w_0); 1587c8da7ce66017295a65ec028084b90800be377f8James Zern A_b2 = _mm_madd_epi16(A_b2, w_8); 1597c8da7ce66017295a65ec028084b90800be377f8James Zern B_b0 = _mm_madd_epi16(B_b0, w_0); 1607c8da7ce66017295a65ec028084b90800be377f8James Zern B_b2 = _mm_madd_epi16(B_b2, w_8); 1617c8da7ce66017295a65ec028084b90800be377f8James Zern A_b0 = _mm_add_epi32(A_b0, A_b2); 1627c8da7ce66017295a65ec028084b90800be377f8James Zern B_b0 = _mm_add_epi32(B_b0, B_b2); 1637c8da7ce66017295a65ec028084b90800be377f8James Zern 1647c8da7ce66017295a65ec028084b90800be377f8James Zern // difference of weighted sums 1657c8da7ce66017295a65ec028084b90800be377f8James Zern A_b2 = _mm_sub_epi32(A_b0, B_b0); 1660912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern _mm_storeu_si128((__m128i*)&sum[0], A_b2); 1677c8da7ce66017295a65ec028084b90800be377f8James Zern } 1680912efc2528d03c59d45dd9bdc9ff9ec800a3fc1James Zern return sum[0] + sum[1] + sum[2] + sum[3]; 1697c8da7ce66017295a65ec028084b90800be377f8James Zern} 1707c8da7ce66017295a65ec028084b90800be377f8James Zern 1717c8da7ce66017295a65ec028084b90800be377f8James Zernstatic int Disto4x4(const uint8_t* const a, const uint8_t* const b, 1727c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16_t* const w) { 1737c8da7ce66017295a65ec028084b90800be377f8James Zern const int diff_sum = TTransform(a, b, w); 1747c8da7ce66017295a65ec028084b90800be377f8James Zern return abs(diff_sum) >> 5; 1757c8da7ce66017295a65ec028084b90800be377f8James Zern} 1767c8da7ce66017295a65ec028084b90800be377f8James Zern 1777c8da7ce66017295a65ec028084b90800be377f8James Zernstatic int Disto16x16(const uint8_t* const a, const uint8_t* const b, 1787c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16_t* const w) { 1797c8da7ce66017295a65ec028084b90800be377f8James Zern int D = 0; 1807c8da7ce66017295a65ec028084b90800be377f8James Zern int x, y; 1817c8da7ce66017295a65ec028084b90800be377f8James Zern for (y = 0; y < 16 * BPS; y += 4 * BPS) { 1827c8da7ce66017295a65ec028084b90800be377f8James Zern for (x = 0; x < 16; x += 4) { 1837c8da7ce66017295a65ec028084b90800be377f8James Zern D += Disto4x4(a + x + y, b + x + y, w); 1847c8da7ce66017295a65ec028084b90800be377f8James Zern } 1857c8da7ce66017295a65ec028084b90800be377f8James Zern } 1867c8da7ce66017295a65ec028084b90800be377f8James Zern return D; 1877c8da7ce66017295a65ec028084b90800be377f8James Zern} 1887c8da7ce66017295a65ec028084b90800be377f8James Zern 1897c8da7ce66017295a65ec028084b90800be377f8James Zern//------------------------------------------------------------------------------ 1907c8da7ce66017295a65ec028084b90800be377f8James Zern// Quantization 1917c8da7ce66017295a65ec028084b90800be377f8James Zern// 1927c8da7ce66017295a65ec028084b90800be377f8James Zern 1937c8da7ce66017295a65ec028084b90800be377f8James Zern// Generates a pshufb constant for shuffling 16b words. 1947c8da7ce66017295a65ec028084b90800be377f8James Zern#define PSHUFB_CST(A,B,C,D,E,F,G,H) \ 1957c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_set_epi8(2 * (H) + 1, 2 * (H) + 0, 2 * (G) + 1, 2 * (G) + 0, \ 1967c8da7ce66017295a65ec028084b90800be377f8James Zern 2 * (F) + 1, 2 * (F) + 0, 2 * (E) + 1, 2 * (E) + 0, \ 1977c8da7ce66017295a65ec028084b90800be377f8James Zern 2 * (D) + 1, 2 * (D) + 0, 2 * (C) + 1, 2 * (C) + 0, \ 1987c8da7ce66017295a65ec028084b90800be377f8James Zern 2 * (B) + 1, 2 * (B) + 0, 2 * (A) + 1, 2 * (A) + 0) 1997c8da7ce66017295a65ec028084b90800be377f8James Zern 2007c8da7ce66017295a65ec028084b90800be377f8James Zernstatic WEBP_INLINE int DoQuantizeBlock(int16_t in[16], int16_t out[16], 2017c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16_t* const sharpen, 2027c8da7ce66017295a65ec028084b90800be377f8James Zern const VP8Matrix* const mtx) { 2037c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i max_coeff_2047 = _mm_set1_epi16(MAX_LEVEL); 2047c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i zero = _mm_setzero_si128(); 2057c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i out0, out8; 2067c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i packed_out; 2077c8da7ce66017295a65ec028084b90800be377f8James Zern 2087c8da7ce66017295a65ec028084b90800be377f8James Zern // Load all inputs. 2097c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i in0 = _mm_loadu_si128((__m128i*)&in[0]); 2107c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i in8 = _mm_loadu_si128((__m128i*)&in[8]); 2117c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i iq0 = _mm_loadu_si128((const __m128i*)&mtx->iq_[0]); 2127c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i iq8 = _mm_loadu_si128((const __m128i*)&mtx->iq_[8]); 2137c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i q0 = _mm_loadu_si128((const __m128i*)&mtx->q_[0]); 2147c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i q8 = _mm_loadu_si128((const __m128i*)&mtx->q_[8]); 2157c8da7ce66017295a65ec028084b90800be377f8James Zern 2167c8da7ce66017295a65ec028084b90800be377f8James Zern // coeff = abs(in) 2177c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i coeff0 = _mm_abs_epi16(in0); 2187c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i coeff8 = _mm_abs_epi16(in8); 2197c8da7ce66017295a65ec028084b90800be377f8James Zern 2207c8da7ce66017295a65ec028084b90800be377f8James Zern // coeff = abs(in) + sharpen 2217c8da7ce66017295a65ec028084b90800be377f8James Zern if (sharpen != NULL) { 2227c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i sharpen0 = _mm_loadu_si128((const __m128i*)&sharpen[0]); 2237c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i sharpen8 = _mm_loadu_si128((const __m128i*)&sharpen[8]); 2247c8da7ce66017295a65ec028084b90800be377f8James Zern coeff0 = _mm_add_epi16(coeff0, sharpen0); 2257c8da7ce66017295a65ec028084b90800be377f8James Zern coeff8 = _mm_add_epi16(coeff8, sharpen8); 2267c8da7ce66017295a65ec028084b90800be377f8James Zern } 2277c8da7ce66017295a65ec028084b90800be377f8James Zern 2287c8da7ce66017295a65ec028084b90800be377f8James Zern // out = (coeff * iQ + B) >> QFIX 2297c8da7ce66017295a65ec028084b90800be377f8James Zern { 2307c8da7ce66017295a65ec028084b90800be377f8James Zern // doing calculations with 32b precision (QFIX=17) 2317c8da7ce66017295a65ec028084b90800be377f8James Zern // out = (coeff * iQ) 2327c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i coeff_iQ0H = _mm_mulhi_epu16(coeff0, iq0); 2337c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i coeff_iQ0L = _mm_mullo_epi16(coeff0, iq0); 2347c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i coeff_iQ8H = _mm_mulhi_epu16(coeff8, iq8); 2357c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i coeff_iQ8L = _mm_mullo_epi16(coeff8, iq8); 2367c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i out_00 = _mm_unpacklo_epi16(coeff_iQ0L, coeff_iQ0H); 2377c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i out_04 = _mm_unpackhi_epi16(coeff_iQ0L, coeff_iQ0H); 2387c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i out_08 = _mm_unpacklo_epi16(coeff_iQ8L, coeff_iQ8H); 2397c8da7ce66017295a65ec028084b90800be377f8James Zern __m128i out_12 = _mm_unpackhi_epi16(coeff_iQ8L, coeff_iQ8H); 2407c8da7ce66017295a65ec028084b90800be377f8James Zern // out = (coeff * iQ + B) 2417c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i bias_00 = _mm_loadu_si128((const __m128i*)&mtx->bias_[0]); 2427c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i bias_04 = _mm_loadu_si128((const __m128i*)&mtx->bias_[4]); 2437c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i bias_08 = _mm_loadu_si128((const __m128i*)&mtx->bias_[8]); 2447c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i bias_12 = _mm_loadu_si128((const __m128i*)&mtx->bias_[12]); 2457c8da7ce66017295a65ec028084b90800be377f8James Zern out_00 = _mm_add_epi32(out_00, bias_00); 2467c8da7ce66017295a65ec028084b90800be377f8James Zern out_04 = _mm_add_epi32(out_04, bias_04); 2477c8da7ce66017295a65ec028084b90800be377f8James Zern out_08 = _mm_add_epi32(out_08, bias_08); 2487c8da7ce66017295a65ec028084b90800be377f8James Zern out_12 = _mm_add_epi32(out_12, bias_12); 2497c8da7ce66017295a65ec028084b90800be377f8James Zern // out = QUANTDIV(coeff, iQ, B, QFIX) 2507c8da7ce66017295a65ec028084b90800be377f8James Zern out_00 = _mm_srai_epi32(out_00, QFIX); 2517c8da7ce66017295a65ec028084b90800be377f8James Zern out_04 = _mm_srai_epi32(out_04, QFIX); 2527c8da7ce66017295a65ec028084b90800be377f8James Zern out_08 = _mm_srai_epi32(out_08, QFIX); 2537c8da7ce66017295a65ec028084b90800be377f8James Zern out_12 = _mm_srai_epi32(out_12, QFIX); 2547c8da7ce66017295a65ec028084b90800be377f8James Zern 2557c8da7ce66017295a65ec028084b90800be377f8James Zern // pack result as 16b 2567c8da7ce66017295a65ec028084b90800be377f8James Zern out0 = _mm_packs_epi32(out_00, out_04); 2577c8da7ce66017295a65ec028084b90800be377f8James Zern out8 = _mm_packs_epi32(out_08, out_12); 2587c8da7ce66017295a65ec028084b90800be377f8James Zern 2597c8da7ce66017295a65ec028084b90800be377f8James Zern // if (coeff > 2047) coeff = 2047 2607c8da7ce66017295a65ec028084b90800be377f8James Zern out0 = _mm_min_epi16(out0, max_coeff_2047); 2617c8da7ce66017295a65ec028084b90800be377f8James Zern out8 = _mm_min_epi16(out8, max_coeff_2047); 2627c8da7ce66017295a65ec028084b90800be377f8James Zern } 2637c8da7ce66017295a65ec028084b90800be377f8James Zern 2647c8da7ce66017295a65ec028084b90800be377f8James Zern // put sign back 2657c8da7ce66017295a65ec028084b90800be377f8James Zern out0 = _mm_sign_epi16(out0, in0); 2667c8da7ce66017295a65ec028084b90800be377f8James Zern out8 = _mm_sign_epi16(out8, in8); 2677c8da7ce66017295a65ec028084b90800be377f8James Zern 2687c8da7ce66017295a65ec028084b90800be377f8James Zern // in = out * Q 2697c8da7ce66017295a65ec028084b90800be377f8James Zern in0 = _mm_mullo_epi16(out0, q0); 2707c8da7ce66017295a65ec028084b90800be377f8James Zern in8 = _mm_mullo_epi16(out8, q8); 2717c8da7ce66017295a65ec028084b90800be377f8James Zern 2727c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)&in[0], in0); 2737c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)&in[8], in8); 2747c8da7ce66017295a65ec028084b90800be377f8James Zern 2757c8da7ce66017295a65ec028084b90800be377f8James Zern // zigzag the output before storing it. The re-ordering is: 2767c8da7ce66017295a65ec028084b90800be377f8James Zern // 0 1 2 3 4 5 6 7 | 8 9 10 11 12 13 14 15 2777c8da7ce66017295a65ec028084b90800be377f8James Zern // -> 0 1 4[8]5 2 3 6 | 9 12 13 10 [7]11 14 15 2787c8da7ce66017295a65ec028084b90800be377f8James Zern // There's only two misplaced entries ([8] and [7]) that are crossing the 2797c8da7ce66017295a65ec028084b90800be377f8James Zern // reg's boundaries. 2807c8da7ce66017295a65ec028084b90800be377f8James Zern // We use pshufb instead of pshuflo/pshufhi. 2817c8da7ce66017295a65ec028084b90800be377f8James Zern { 2827c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i kCst_lo = PSHUFB_CST(0, 1, 4, -1, 5, 2, 3, 6); 2837c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i kCst_7 = PSHUFB_CST(-1, -1, -1, -1, 7, -1, -1, -1); 2847c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i tmp_lo = _mm_shuffle_epi8(out0, kCst_lo); 2857c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i tmp_7 = _mm_shuffle_epi8(out0, kCst_7); // extract #7 2867c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i kCst_hi = PSHUFB_CST(1, 4, 5, 2, -1, 3, 6, 7); 2877c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i kCst_8 = PSHUFB_CST(-1, -1, -1, 0, -1, -1, -1, -1); 2887c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i tmp_hi = _mm_shuffle_epi8(out8, kCst_hi); 2897c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i tmp_8 = _mm_shuffle_epi8(out8, kCst_8); // extract #8 2907c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i out_z0 = _mm_or_si128(tmp_lo, tmp_8); 2917c8da7ce66017295a65ec028084b90800be377f8James Zern const __m128i out_z8 = _mm_or_si128(tmp_hi, tmp_7); 2927c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)&out[0], out_z0); 2937c8da7ce66017295a65ec028084b90800be377f8James Zern _mm_storeu_si128((__m128i*)&out[8], out_z8); 2947c8da7ce66017295a65ec028084b90800be377f8James Zern packed_out = _mm_packs_epi16(out_z0, out_z8); 2957c8da7ce66017295a65ec028084b90800be377f8James Zern } 2967c8da7ce66017295a65ec028084b90800be377f8James Zern 2977c8da7ce66017295a65ec028084b90800be377f8James Zern // detect if all 'out' values are zeroes or not 2987c8da7ce66017295a65ec028084b90800be377f8James Zern return (_mm_movemask_epi8(_mm_cmpeq_epi8(packed_out, zero)) != 0xffff); 2997c8da7ce66017295a65ec028084b90800be377f8James Zern} 3007c8da7ce66017295a65ec028084b90800be377f8James Zern 3017c8da7ce66017295a65ec028084b90800be377f8James Zern#undef PSHUFB_CST 3027c8da7ce66017295a65ec028084b90800be377f8James Zern 3037c8da7ce66017295a65ec028084b90800be377f8James Zernstatic int QuantizeBlock(int16_t in[16], int16_t out[16], 3047c8da7ce66017295a65ec028084b90800be377f8James Zern const VP8Matrix* const mtx) { 3057c8da7ce66017295a65ec028084b90800be377f8James Zern return DoQuantizeBlock(in, out, &mtx->sharpen_[0], mtx); 3067c8da7ce66017295a65ec028084b90800be377f8James Zern} 3077c8da7ce66017295a65ec028084b90800be377f8James Zern 3087c8da7ce66017295a65ec028084b90800be377f8James Zernstatic int QuantizeBlockWHT(int16_t in[16], int16_t out[16], 3097c8da7ce66017295a65ec028084b90800be377f8James Zern const VP8Matrix* const mtx) { 3107c8da7ce66017295a65ec028084b90800be377f8James Zern return DoQuantizeBlock(in, out, NULL, mtx); 3117c8da7ce66017295a65ec028084b90800be377f8James Zern} 3127c8da7ce66017295a65ec028084b90800be377f8James Zern 3137c8da7ce66017295a65ec028084b90800be377f8James Zernstatic int Quantize2Blocks(int16_t in[32], int16_t out[32], 3147c8da7ce66017295a65ec028084b90800be377f8James Zern const VP8Matrix* const mtx) { 3157c8da7ce66017295a65ec028084b90800be377f8James Zern int nz; 3167c8da7ce66017295a65ec028084b90800be377f8James Zern const uint16_t* const sharpen = &mtx->sharpen_[0]; 3177c8da7ce66017295a65ec028084b90800be377f8James Zern nz = DoQuantizeBlock(in + 0 * 16, out + 0 * 16, sharpen, mtx) << 0; 3187c8da7ce66017295a65ec028084b90800be377f8James Zern nz |= DoQuantizeBlock(in + 1 * 16, out + 1 * 16, sharpen, mtx) << 1; 3197c8da7ce66017295a65ec028084b90800be377f8James Zern return nz; 3207c8da7ce66017295a65ec028084b90800be377f8James Zern} 3217c8da7ce66017295a65ec028084b90800be377f8James Zern 3227c8da7ce66017295a65ec028084b90800be377f8James Zern//------------------------------------------------------------------------------ 3237c8da7ce66017295a65ec028084b90800be377f8James Zern// Entry point 3247c8da7ce66017295a65ec028084b90800be377f8James Zern 3257c8da7ce66017295a65ec028084b90800be377f8James Zernextern void VP8EncDspInitSSE41(void); 3267c8da7ce66017295a65ec028084b90800be377f8James ZernWEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitSSE41(void) { 3277c8da7ce66017295a65ec028084b90800be377f8James Zern VP8CollectHistogram = CollectHistogram; 3287c8da7ce66017295a65ec028084b90800be377f8James Zern VP8EncQuantizeBlock = QuantizeBlock; 3297c8da7ce66017295a65ec028084b90800be377f8James Zern VP8EncQuantize2Blocks = Quantize2Blocks; 3307c8da7ce66017295a65ec028084b90800be377f8James Zern VP8EncQuantizeBlockWHT = QuantizeBlockWHT; 3317c8da7ce66017295a65ec028084b90800be377f8James Zern VP8TDisto4x4 = Disto4x4; 3327c8da7ce66017295a65ec028084b90800be377f8James Zern VP8TDisto16x16 = Disto16x16; 3337c8da7ce66017295a65ec028084b90800be377f8James Zern} 3347c8da7ce66017295a65ec028084b90800be377f8James Zern 3357c8da7ce66017295a65ec028084b90800be377f8James Zern#else // !WEBP_USE_SSE41 3367c8da7ce66017295a65ec028084b90800be377f8James Zern 3377c8da7ce66017295a65ec028084b90800be377f8James ZernWEBP_DSP_INIT_STUB(VP8EncDspInitSSE41) 3387c8da7ce66017295a65ec028084b90800be377f8James Zern 3397c8da7ce66017295a65ec028084b90800be377f8James Zern#endif // WEBP_USE_SSE41 340