1// Copyright 2011 Google Inc. All Rights Reserved.
2//
3// Use of this source code is governed by a BSD-style license
4// that can be found in the COPYING file in the root of the source
5// tree. An additional intellectual property rights grant can be found
6// in the file PATENTS. All contributing project authors may
7// be found in the AUTHORS file in the root of the source tree.
8// -----------------------------------------------------------------------------
9//
10//   Speed-critical functions.
11//
12// Author: Skal (pascal.massimino@gmail.com)
13
14#ifndef WEBP_DSP_DSP_H_
15#define WEBP_DSP_DSP_H_
16
17#include "../webp/types.h"
18#include "../utils/utils.h"
19
20#ifdef __cplusplus
21extern "C" {
22#endif
23
24#define BPS 32   // this is the common stride for enc/dec
25
26//------------------------------------------------------------------------------
27// CPU detection
28
29#if defined(__GNUC__)
30# define LOCAL_GCC_VERSION ((__GNUC__ << 8) | __GNUC_MINOR__)
31# define LOCAL_GCC_PREREQ(maj, min) \
32    (LOCAL_GCC_VERSION >= (((maj) << 8) | (min)))
33#else
34# define LOCAL_GCC_VERSION 0
35# define LOCAL_GCC_PREREQ(maj, min) 0
36#endif
37
38#ifndef __has_builtin
39# define __has_builtin(x) 0
40#endif
41
42#if defined(_MSC_VER) && _MSC_VER > 1310 && \
43    (defined(_M_X64) || defined(_M_IX86))
44#define WEBP_MSC_SSE2  // Visual C++ SSE2 targets
45#endif
46
47#if defined(_MSC_VER) && _MSC_VER >= 1500 && \
48    (defined(_M_X64) || defined(_M_IX86))
49#define WEBP_MSC_SSE41  // Visual C++ SSE4.1 targets
50#endif
51
52// WEBP_HAVE_* are used to indicate the presence of the instruction set in dsp
53// files without intrinsics, allowing the corresponding Init() to be called.
54// Files containing intrinsics will need to be built targeting the instruction
55// set so should succeed on one of the earlier tests.
56#if defined(__SSE2__) || defined(WEBP_MSC_SSE2) || defined(WEBP_HAVE_SSE2)
57#define WEBP_USE_SSE2
58#endif
59
60#if defined(__SSE4_1__) || defined(WEBP_MSC_SSE41) || defined(WEBP_HAVE_SSE41)
61#define WEBP_USE_SSE41
62#endif
63
64#if defined(__AVX2__) || defined(WEBP_HAVE_AVX2)
65#define WEBP_USE_AVX2
66#endif
67
68#if defined(__ANDROID__) && defined(__ARM_ARCH_7A__)
69#define WEBP_ANDROID_NEON  // Android targets that might support NEON
70#endif
71
72// The intrinsics currently cause compiler errors with arm-nacl-gcc and the
73// inline assembly would need to be modified for use with Native Client.
74#if (defined(__ARM_NEON__) || defined(__aarch64__)) \
75    && !defined(__native_client__)
76#define WEBP_USE_NEON
77#endif
78
79#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM)
80#define WEBP_USE_NEON
81#define WEBP_USE_INTRINSICS
82#endif
83
84#if defined(__mips__) && !defined(__mips64) && \
85    defined(__mips_isa_rev) && (__mips_isa_rev >= 1) && (__mips_isa_rev < 6)
86#define WEBP_USE_MIPS32
87#if (__mips_isa_rev >= 2)
88#define WEBP_USE_MIPS32_R2
89#if defined(__mips_dspr2) || (__mips_dsp_rev >= 2)
90#define WEBP_USE_MIPS_DSP_R2
91#endif
92#endif
93#endif
94
95// This macro prevents thread_sanitizer from reporting known concurrent writes.
96#define WEBP_TSAN_IGNORE_FUNCTION
97#if defined(__has_feature)
98#if __has_feature(thread_sanitizer)
99#undef WEBP_TSAN_IGNORE_FUNCTION
100#define WEBP_TSAN_IGNORE_FUNCTION __attribute__((no_sanitize_thread))
101#endif
102#endif
103
104typedef enum {
105  kSSE2,
106  kSSE3,
107  kSSE4_1,
108  kAVX,
109  kAVX2,
110  kNEON,
111  kMIPS32,
112  kMIPSdspR2
113} CPUFeature;
114// returns true if the CPU supports the feature.
115typedef int (*VP8CPUInfo)(CPUFeature feature);
116WEBP_EXTERN(VP8CPUInfo) VP8GetCPUInfo;
117
118//------------------------------------------------------------------------------
119// Init stub generator
120
121// Defines an init function stub to ensure each module exposes a symbol,
122// avoiding a compiler warning.
123#define WEBP_DSP_INIT_STUB(func) \
124  extern void func(void); \
125  WEBP_TSAN_IGNORE_FUNCTION void func(void) {}
126
127//------------------------------------------------------------------------------
128// Encoding
129
130// Transforms
131// VP8Idct: Does one of two inverse transforms. If do_two is set, the transforms
132//          will be done for (ref, in, dst) and (ref + 4, in + 16, dst + 4).
133typedef void (*VP8Idct)(const uint8_t* ref, const int16_t* in, uint8_t* dst,
134                        int do_two);
135typedef void (*VP8Fdct)(const uint8_t* src, const uint8_t* ref, int16_t* out);
136typedef void (*VP8WHT)(const int16_t* in, int16_t* out);
137extern VP8Idct VP8ITransform;
138extern VP8Fdct VP8FTransform;
139extern VP8Fdct VP8FTransform2;   // performs two transforms at a time
140extern VP8WHT VP8FTransformWHT;
141// Predictions
142// *dst is the destination block. *top and *left can be NULL.
143typedef void (*VP8IntraPreds)(uint8_t *dst, const uint8_t* left,
144                              const uint8_t* top);
145typedef void (*VP8Intra4Preds)(uint8_t *dst, const uint8_t* top);
146extern VP8Intra4Preds VP8EncPredLuma4;
147extern VP8IntraPreds VP8EncPredLuma16;
148extern VP8IntraPreds VP8EncPredChroma8;
149
150typedef int (*VP8Metric)(const uint8_t* pix, const uint8_t* ref);
151extern VP8Metric VP8SSE16x16, VP8SSE16x8, VP8SSE8x8, VP8SSE4x4;
152typedef int (*VP8WMetric)(const uint8_t* pix, const uint8_t* ref,
153                          const uint16_t* const weights);
154extern VP8WMetric VP8TDisto4x4, VP8TDisto16x16;
155
156typedef void (*VP8BlockCopy)(const uint8_t* src, uint8_t* dst);
157extern VP8BlockCopy VP8Copy4x4;
158extern VP8BlockCopy VP8Copy16x8;
159// Quantization
160struct VP8Matrix;   // forward declaration
161typedef int (*VP8QuantizeBlock)(int16_t in[16], int16_t out[16],
162                                const struct VP8Matrix* const mtx);
163// Same as VP8QuantizeBlock, but quantizes two consecutive blocks.
164typedef int (*VP8Quantize2Blocks)(int16_t in[32], int16_t out[32],
165                                  const struct VP8Matrix* const mtx);
166
167extern VP8QuantizeBlock VP8EncQuantizeBlock;
168extern VP8Quantize2Blocks VP8EncQuantize2Blocks;
169
170// specific to 2nd transform:
171typedef int (*VP8QuantizeBlockWHT)(int16_t in[16], int16_t out[16],
172                                   const struct VP8Matrix* const mtx);
173extern VP8QuantizeBlockWHT VP8EncQuantizeBlockWHT;
174
175extern const int VP8DspScan[16 + 4 + 4];
176
177// Collect histogram for susceptibility calculation.
178#define MAX_COEFF_THRESH   31   // size of histogram used by CollectHistogram.
179typedef struct {
180  // We only need to store max_value and last_non_zero, not the distribution.
181  int max_value;
182  int last_non_zero;
183} VP8Histogram;
184typedef void (*VP8CHisto)(const uint8_t* ref, const uint8_t* pred,
185                          int start_block, int end_block,
186                          VP8Histogram* const histo);
187extern VP8CHisto VP8CollectHistogram;
188// General-purpose util function to help VP8CollectHistogram().
189void VP8SetHistogramData(const int distribution[MAX_COEFF_THRESH + 1],
190                         VP8Histogram* const histo);
191
192// must be called before using any of the above
193void VP8EncDspInit(void);
194
195//------------------------------------------------------------------------------
196// cost functions (encoding)
197
198extern const uint16_t VP8EntropyCost[256];        // 8bit fixed-point log(p)
199// approximate cost per level:
200extern const uint16_t VP8LevelFixedCosts[2047 /*MAX_LEVEL*/ + 1];
201extern const uint8_t VP8EncBands[16 + 1];
202
203struct VP8Residual;
204typedef void (*VP8SetResidualCoeffsFunc)(const int16_t* const coeffs,
205                                         struct VP8Residual* const res);
206extern VP8SetResidualCoeffsFunc VP8SetResidualCoeffs;
207
208// Cost calculation function.
209typedef int (*VP8GetResidualCostFunc)(int ctx0,
210                                      const struct VP8Residual* const res);
211extern VP8GetResidualCostFunc VP8GetResidualCost;
212
213// must be called before anything using the above
214void VP8EncDspCostInit(void);
215
216//------------------------------------------------------------------------------
217// Decoding
218
219typedef void (*VP8DecIdct)(const int16_t* coeffs, uint8_t* dst);
220// when doing two transforms, coeffs is actually int16_t[2][16].
221typedef void (*VP8DecIdct2)(const int16_t* coeffs, uint8_t* dst, int do_two);
222extern VP8DecIdct2 VP8Transform;
223extern VP8DecIdct VP8TransformAC3;
224extern VP8DecIdct VP8TransformUV;
225extern VP8DecIdct VP8TransformDC;
226extern VP8DecIdct VP8TransformDCUV;
227extern VP8WHT VP8TransformWHT;
228
229// *dst is the destination block, with stride BPS. Boundary samples are
230// assumed accessible when needed.
231typedef void (*VP8PredFunc)(uint8_t* dst);
232extern VP8PredFunc VP8PredLuma16[/* NUM_B_DC_MODES */];
233extern VP8PredFunc VP8PredChroma8[/* NUM_B_DC_MODES */];
234extern VP8PredFunc VP8PredLuma4[/* NUM_BMODES */];
235
236// clipping tables (for filtering)
237extern const int8_t* const VP8ksclip1;  // clips [-1020, 1020] to [-128, 127]
238extern const int8_t* const VP8ksclip2;  // clips [-112, 112] to [-16, 15]
239extern const uint8_t* const VP8kclip1;  // clips [-255,511] to [0,255]
240extern const uint8_t* const VP8kabs0;   // abs(x) for x in [-255,255]
241// must be called first
242void VP8InitClipTables(void);
243
244// simple filter (only for luma)
245typedef void (*VP8SimpleFilterFunc)(uint8_t* p, int stride, int thresh);
246extern VP8SimpleFilterFunc VP8SimpleVFilter16;
247extern VP8SimpleFilterFunc VP8SimpleHFilter16;
248extern VP8SimpleFilterFunc VP8SimpleVFilter16i;  // filter 3 inner edges
249extern VP8SimpleFilterFunc VP8SimpleHFilter16i;
250
251// regular filter (on both macroblock edges and inner edges)
252typedef void (*VP8LumaFilterFunc)(uint8_t* luma, int stride,
253                                  int thresh, int ithresh, int hev_t);
254typedef void (*VP8ChromaFilterFunc)(uint8_t* u, uint8_t* v, int stride,
255                                    int thresh, int ithresh, int hev_t);
256// on outer edge
257extern VP8LumaFilterFunc VP8VFilter16;
258extern VP8LumaFilterFunc VP8HFilter16;
259extern VP8ChromaFilterFunc VP8VFilter8;
260extern VP8ChromaFilterFunc VP8HFilter8;
261
262// on inner edge
263extern VP8LumaFilterFunc VP8VFilter16i;   // filtering 3 inner edges altogether
264extern VP8LumaFilterFunc VP8HFilter16i;
265extern VP8ChromaFilterFunc VP8VFilter8i;  // filtering u and v altogether
266extern VP8ChromaFilterFunc VP8HFilter8i;
267
268// must be called before anything using the above
269void VP8DspInit(void);
270
271//------------------------------------------------------------------------------
272// WebP I/O
273
274#define FANCY_UPSAMPLING   // undefined to remove fancy upsampling support
275
276// Convert a pair of y/u/v lines together to the output rgb/a colorspace.
277// bottom_y can be NULL if only one line of output is needed (at top/bottom).
278typedef void (*WebPUpsampleLinePairFunc)(
279    const uint8_t* top_y, const uint8_t* bottom_y,
280    const uint8_t* top_u, const uint8_t* top_v,
281    const uint8_t* cur_u, const uint8_t* cur_v,
282    uint8_t* top_dst, uint8_t* bottom_dst, int len);
283
284#ifdef FANCY_UPSAMPLING
285
286// Fancy upsampling functions to convert YUV to RGB(A) modes
287extern WebPUpsampleLinePairFunc WebPUpsamplers[/* MODE_LAST */];
288
289#endif    // FANCY_UPSAMPLING
290
291// Per-row point-sampling methods.
292typedef void (*WebPSamplerRowFunc)(const uint8_t* y,
293                                   const uint8_t* u, const uint8_t* v,
294                                   uint8_t* dst, int len);
295// Generic function to apply 'WebPSamplerRowFunc' to the whole plane:
296void WebPSamplerProcessPlane(const uint8_t* y, int y_stride,
297                             const uint8_t* u, const uint8_t* v, int uv_stride,
298                             uint8_t* dst, int dst_stride,
299                             int width, int height, WebPSamplerRowFunc func);
300
301// Sampling functions to convert rows of YUV to RGB(A)
302extern WebPSamplerRowFunc WebPSamplers[/* MODE_LAST */];
303
304// General function for converting two lines of ARGB or RGBA.
305// 'alpha_is_last' should be true if 0xff000000 is stored in memory as
306// as 0x00, 0x00, 0x00, 0xff (little endian).
307WebPUpsampleLinePairFunc WebPGetLinePairConverter(int alpha_is_last);
308
309// YUV444->RGB converters
310typedef void (*WebPYUV444Converter)(const uint8_t* y,
311                                    const uint8_t* u, const uint8_t* v,
312                                    uint8_t* dst, int len);
313
314extern WebPYUV444Converter WebPYUV444Converters[/* MODE_LAST */];
315
316// Must be called before using the WebPUpsamplers[] (and for premultiplied
317// colorspaces like rgbA, rgbA4444, etc)
318void WebPInitUpsamplers(void);
319// Must be called before using WebPSamplers[]
320void WebPInitSamplers(void);
321// Must be called before using WebPYUV444Converters[]
322void WebPInitYUV444Converters(void);
323
324//------------------------------------------------------------------------------
325// ARGB -> YUV converters
326
327// Convert ARGB samples to luma Y.
328extern void (*WebPConvertARGBToY)(const uint32_t* argb, uint8_t* y, int width);
329// Convert ARGB samples to U/V with downsampling. do_store should be '1' for
330// even lines and '0' for odd ones. 'src_width' is the original width, not
331// the U/V one.
332extern void (*WebPConvertARGBToUV)(const uint32_t* argb, uint8_t* u, uint8_t* v,
333                                   int src_width, int do_store);
334
335// Convert a row of accumulated (four-values) of rgba32 toward U/V
336extern void (*WebPConvertRGBA32ToUV)(const uint16_t* rgb,
337                                     uint8_t* u, uint8_t* v, int width);
338
339// Convert RGB or BGR to Y
340extern void (*WebPConvertRGB24ToY)(const uint8_t* rgb, uint8_t* y, int width);
341extern void (*WebPConvertBGR24ToY)(const uint8_t* bgr, uint8_t* y, int width);
342
343// used for plain-C fallback.
344extern void WebPConvertARGBToUV_C(const uint32_t* argb, uint8_t* u, uint8_t* v,
345                                  int src_width, int do_store);
346extern void WebPConvertRGBA32ToUV_C(const uint16_t* rgb,
347                                    uint8_t* u, uint8_t* v, int width);
348
349// Must be called before using the above.
350void WebPInitConvertARGBToYUV(void);
351
352//------------------------------------------------------------------------------
353// Rescaler
354
355struct WebPRescaler;
356
357// Import a row of data and save its contribution in the rescaler.
358// 'channel' denotes the channel number to be imported. 'Expand' corresponds to
359// the wrk->x_expand case. Otherwise, 'Shrink' is to be used.
360typedef void (*WebPRescalerImportRowFunc)(struct WebPRescaler* const wrk,
361                                          const uint8_t* src);
362
363extern WebPRescalerImportRowFunc WebPRescalerImportRowExpand;
364extern WebPRescalerImportRowFunc WebPRescalerImportRowShrink;
365
366// Export one row (starting at x_out position) from rescaler.
367// 'Expand' corresponds to the wrk->y_expand case.
368// Otherwise 'Shrink' is to be used
369typedef void (*WebPRescalerExportRowFunc)(struct WebPRescaler* const wrk);
370extern WebPRescalerExportRowFunc WebPRescalerExportRowExpand;
371extern WebPRescalerExportRowFunc WebPRescalerExportRowShrink;
372
373// Plain-C implementation, as fall-back.
374extern void WebPRescalerImportRowExpandC(struct WebPRescaler* const wrk,
375                                         const uint8_t* src);
376extern void WebPRescalerImportRowShrinkC(struct WebPRescaler* const wrk,
377                                         const uint8_t* src);
378extern void WebPRescalerExportRowExpandC(struct WebPRescaler* const wrk);
379extern void WebPRescalerExportRowShrinkC(struct WebPRescaler* const wrk);
380
381// Main entry calls:
382extern void WebPRescalerImportRow(struct WebPRescaler* const wrk,
383                                  const uint8_t* src);
384// Export one row (starting at x_out position) from rescaler.
385extern void WebPRescalerExportRow(struct WebPRescaler* const wrk);
386
387// Must be called first before using the above.
388void WebPRescalerDspInit(void);
389
390//------------------------------------------------------------------------------
391// Utilities for processing transparent channel.
392
393// Apply alpha pre-multiply on an rgba, bgra or argb plane of size w * h.
394// alpha_first should be 0 for argb, 1 for rgba or bgra (where alpha is last).
395extern void (*WebPApplyAlphaMultiply)(
396    uint8_t* rgba, int alpha_first, int w, int h, int stride);
397
398// Same, buf specifically for RGBA4444 format
399extern void (*WebPApplyAlphaMultiply4444)(
400    uint8_t* rgba4444, int w, int h, int stride);
401
402// Dispatch the values from alpha[] plane to the ARGB destination 'dst'.
403// Returns true if alpha[] plane has non-trivial values different from 0xff.
404extern int (*WebPDispatchAlpha)(const uint8_t* alpha, int alpha_stride,
405                                int width, int height,
406                                uint8_t* dst, int dst_stride);
407
408// Transfer packed 8b alpha[] values to green channel in dst[], zero'ing the
409// A/R/B values. 'dst_stride' is the stride for dst[] in uint32_t units.
410extern void (*WebPDispatchAlphaToGreen)(const uint8_t* alpha, int alpha_stride,
411                                        int width, int height,
412                                        uint32_t* dst, int dst_stride);
413
414// Extract the alpha values from 32b values in argb[] and pack them into alpha[]
415// (this is the opposite of WebPDispatchAlpha).
416// Returns true if there's only trivial 0xff alpha values.
417extern int (*WebPExtractAlpha)(const uint8_t* argb, int argb_stride,
418                               int width, int height,
419                               uint8_t* alpha, int alpha_stride);
420
421// Pre-Multiply operation transforms x into x * A / 255  (where x=Y,R,G or B).
422// Un-Multiply operation transforms x into x * 255 / A.
423
424// Pre-Multiply or Un-Multiply (if 'inverse' is true) argb values in a row.
425extern void (*WebPMultARGBRow)(uint32_t* const ptr, int width, int inverse);
426
427// Same a WebPMultARGBRow(), but for several rows.
428void WebPMultARGBRows(uint8_t* ptr, int stride, int width, int num_rows,
429                      int inverse);
430
431// Same for a row of single values, with side alpha values.
432extern void (*WebPMultRow)(uint8_t* const ptr, const uint8_t* const alpha,
433                           int width, int inverse);
434
435// Same a WebPMultRow(), but for several 'num_rows' rows.
436void WebPMultRows(uint8_t* ptr, int stride,
437                  const uint8_t* alpha, int alpha_stride,
438                  int width, int num_rows, int inverse);
439
440// Plain-C versions, used as fallback by some implementations.
441void WebPMultRowC(uint8_t* const ptr, const uint8_t* const alpha,
442                  int width, int inverse);
443void WebPMultARGBRowC(uint32_t* const ptr, int width, int inverse);
444
445// To be called first before using the above.
446void WebPInitAlphaProcessing(void);
447
448// ARGB packing function: a/r/g/b input is rgba or bgra order.
449extern void (*VP8PackARGB)(const uint8_t* a, const uint8_t* r,
450                           const uint8_t* g, const uint8_t* b, int len,
451                           uint32_t* out);
452
453// RGB packing function. 'step' can be 3 or 4. r/g/b input is rgb or bgr order.
454extern void (*VP8PackRGB)(const uint8_t* r, const uint8_t* g, const uint8_t* b,
455                          int len, int step, uint32_t* out);
456
457// To be called first before using the above.
458void VP8EncDspARGBInit(void);
459
460//------------------------------------------------------------------------------
461// Filter functions
462
463typedef enum {     // Filter types.
464  WEBP_FILTER_NONE = 0,
465  WEBP_FILTER_HORIZONTAL,
466  WEBP_FILTER_VERTICAL,
467  WEBP_FILTER_GRADIENT,
468  WEBP_FILTER_LAST = WEBP_FILTER_GRADIENT + 1,  // end marker
469  WEBP_FILTER_BEST,    // meta-types
470  WEBP_FILTER_FAST
471} WEBP_FILTER_TYPE;
472
473typedef void (*WebPFilterFunc)(const uint8_t* in, int width, int height,
474                               int stride, uint8_t* out);
475typedef void (*WebPUnfilterFunc)(int width, int height, int stride,
476                                 int row, int num_rows, uint8_t* data);
477
478// Filter the given data using the given predictor.
479// 'in' corresponds to a 2-dimensional pixel array of size (stride * height)
480// in raster order.
481// 'stride' is number of bytes per scan line (with possible padding).
482// 'out' should be pre-allocated.
483extern WebPFilterFunc WebPFilters[WEBP_FILTER_LAST];
484
485// In-place reconstruct the original data from the given filtered data.
486// The reconstruction will be done for 'num_rows' rows starting from 'row'
487// (assuming rows upto 'row - 1' are already reconstructed).
488extern WebPUnfilterFunc WebPUnfilters[WEBP_FILTER_LAST];
489
490// To be called first before using the above.
491void VP8FiltersInit(void);
492
493#ifdef __cplusplus
494}    // extern "C"
495#endif
496
497#endif  /* WEBP_DSP_DSP_H_ */
498