1ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#if !defined(_FX_JPEG_TURBO_)
2ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov/*
3ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * jidctint.c
4ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *
5ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * Copyright (C) 1991-1998, Thomas G. Lane.
6ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * This file is part of the Independent JPEG Group's software.
7ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * For conditions of distribution and use, see the accompanying README file.
8ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *
9ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * This file contains a slow-but-accurate integer implementation of the
10ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * inverse DCT (Discrete Cosine Transform).  In the IJG code, this routine
11ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * must also perform dequantization of the input coefficients.
12ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *
13ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * A 2-D IDCT can be done by 1-D IDCT on each column followed by 1-D IDCT
14ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * on each row (or vice versa, but it's more convenient to emit a row at
15ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * a time).  Direct algorithms are also available, but they are much more
16ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * complex and seem not to be any faster when reduced to code.
17ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *
18ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * This implementation is based on an algorithm described in
19ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *   C. Loeffler, A. Ligtenberg and G. Moschytz, "Practical Fast 1-D DCT
20ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *   Algorithms with 11 Multiplications", Proc. Int'l. Conf. on Acoustics,
21ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *   Speech, and Signal Processing 1989 (ICASSP '89), pp. 988-991.
22ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * The primary algorithm described there uses 11 multiplies and 29 adds.
23ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * We use their alternate method with 12 multiplies and 32 adds.
24ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * The advantage of this method is that no data path contains more than one
25ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * multiplication; this allows a very simple and accurate implementation in
26ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * scaled fixed-point arithmetic, with a minimal number of shifts.
27ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov */
28ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
29ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define JPEG_INTERNALS
30ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#include "jinclude.h"
31ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#include "jpeglib.h"
32ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#include "jdct.h"		/* Private declarations for DCT subsystem */
33ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
34ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#ifdef DCT_ISLOW_SUPPORTED
35ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
36ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
37ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov/*
38ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * This module is specialized to the case DCTSIZE = 8.
39ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov */
40ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
41ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#if DCTSIZE != 8
42ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  Sorry, this code only copes with 8x8 DCTs. /* deliberate syntax err */
43ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#endif
44ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
45ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
46ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov/*
47ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * The poop on this scaling stuff is as follows:
48ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *
49ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * Each 1-D IDCT step produces outputs which are a factor of sqrt(N)
50ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * larger than the true IDCT outputs.  The final outputs are therefore
51ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * a factor of N larger than desired; since N=8 this can be cured by
52ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * a simple right shift at the end of the algorithm.  The advantage of
53ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * this arrangement is that we save two multiplications per 1-D IDCT,
54ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * because the y0 and y4 inputs need not be divided by sqrt(N).
55ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *
56ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * We have to do addition and subtraction of the integer inputs, which
57ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * is no problem, and multiplication by fractional constants, which is
58ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * a problem to do in integer arithmetic.  We multiply all the constants
59ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * by CONST_SCALE and convert them to integer constants (thus retaining
60ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * CONST_BITS bits of precision in the constants).  After doing a
61ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * multiplication we have to divide the product by CONST_SCALE, with proper
62ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * rounding, to produce the correct output.  This division can be done
63ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * cheaply as a right shift of CONST_BITS bits.  We postpone shifting
64ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * as long as possible so that partial sums can be added together with
65ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * full fractional precision.
66ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *
67ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * The outputs of the first pass are scaled up by PASS1_BITS bits so that
68ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * they are represented to better-than-integral precision.  These outputs
69ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * require BITS_IN_JSAMPLE + PASS1_BITS + 3 bits; this fits in a 16-bit word
70ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * with the recommended scaling.  (To scale up 12-bit sample data further, an
71ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * intermediate INT32 array would be needed.)
72ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov *
73ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * To avoid overflow of the 32-bit intermediate results in pass 2, we must
74ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * have BITS_IN_JSAMPLE + CONST_BITS + PASS1_BITS <= 26.  Error analysis
75ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * shows that the values given below are the most effective.
76ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov */
77ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
78ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#if BITS_IN_JSAMPLE == 8
79ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define CONST_BITS  13
80ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define PASS1_BITS  2
81ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#else
82ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define CONST_BITS  13
83ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define PASS1_BITS  1		/* lose a little precision to avoid overflow */
84ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#endif
85ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
86ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov/* Some C compilers fail to reduce "FIX(constant)" at compile time, thus
87ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * causing a lot of useless floating-point operations at run time.
88ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * To get around this we use the following pre-calculated constants.
89ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * If you change CONST_BITS you may want to add appropriate values.
90ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * (With a reasonable C compiler, you can just rely on the FIX() macro...)
91ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov */
92ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
93ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#if CONST_BITS == 13
94ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_298631336  ((INT32)  2446)	/* FIX(0.298631336) */
95ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_390180644  ((INT32)  3196)	/* FIX(0.390180644) */
96ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_541196100  ((INT32)  4433)	/* FIX(0.541196100) */
97ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_765366865  ((INT32)  6270)	/* FIX(0.765366865) */
98ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_899976223  ((INT32)  7373)	/* FIX(0.899976223) */
99ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_1_175875602  ((INT32)  9633)	/* FIX(1.175875602) */
100ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_1_501321110  ((INT32)  12299)	/* FIX(1.501321110) */
101ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_1_847759065  ((INT32)  15137)	/* FIX(1.847759065) */
102ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_1_961570560  ((INT32)  16069)	/* FIX(1.961570560) */
103ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_2_053119869  ((INT32)  16819)	/* FIX(2.053119869) */
104ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_2_562915447  ((INT32)  20995)	/* FIX(2.562915447) */
105ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_3_072711026  ((INT32)  25172)	/* FIX(3.072711026) */
106ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#else
107ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_298631336  FIX(0.298631336)
108ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_390180644  FIX(0.390180644)
109ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_541196100  FIX(0.541196100)
110ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_765366865  FIX(0.765366865)
111ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_0_899976223  FIX(0.899976223)
112ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_1_175875602  FIX(1.175875602)
113ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_1_501321110  FIX(1.501321110)
114ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_1_847759065  FIX(1.847759065)
115ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_1_961570560  FIX(1.961570560)
116ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_2_053119869  FIX(2.053119869)
117ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_2_562915447  FIX(2.562915447)
118ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define FIX_3_072711026  FIX(3.072711026)
119ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#endif
120ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
121ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
122ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov/* Multiply an INT32 variable by an INT32 constant to yield an INT32 result.
123ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * For 8-bit samples with the recommended scaling, all the variable
124ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * and constant values involved are no more than 16 bits wide, so a
125ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * 16x16->32 bit multiply can be used instead of a full 32x32 multiply.
126ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * For 12-bit samples, a full 32-bit multiplication will be needed.
127ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov */
128ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
129ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#if BITS_IN_JSAMPLE == 8
130ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define MULTIPLY(var,const)  MULTIPLY16C16(var,const)
131ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#else
132ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define MULTIPLY(var,const)  ((var) * (const))
133ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#endif
134ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
135ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
136ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov/* Dequantize a coefficient by multiplying it by the multiplier-table
137ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * entry; produce an int result.  In this module, both inputs and result
138ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * are 16 bits or less, so either int or short multiply will work.
139ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov */
140ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
141ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#define DEQUANTIZE(coef,quantval)  (((ISLOW_MULT_TYPE) (coef)) * (quantval))
142ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
143ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
144ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov/*
145ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov * Perform dequantization and inverse DCT on one block of coefficients.
146ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov */
147ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
148ee451cb395940862dad63c85adfe8f2fd55e864cSvet GanovGLOBAL(void)
149ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganovjpeg_idct_islow (j_decompress_ptr cinfo, jpeg_component_info * compptr,
150ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov		 JCOEFPTR coef_block,
151ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov		 JSAMPARRAY output_buf, JDIMENSION output_col)
152ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov{
153ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  INT32 tmp0, tmp1, tmp2, tmp3;
154ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  INT32 tmp10, tmp11, tmp12, tmp13;
155ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  INT32 z1, z2, z3, z4, z5;
156ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  JCOEFPTR inptr;
157ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  ISLOW_MULT_TYPE * quantptr;
158ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  int * wsptr;
159ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  JSAMPROW outptr;
160ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  JSAMPLE *range_limit = IDCT_range_limit(cinfo);
161ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  int ctr;
162ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  int workspace[DCTSIZE2];	/* buffers data between passes */
163ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  SHIFT_TEMPS
164ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
165ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  /* Pass 1: process columns from input, store into work array. */
166ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  /* Note results are scaled up by sqrt(8) compared to a true IDCT; */
167ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  /* furthermore, we scale the results by 2**PASS1_BITS. */
168ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
169ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  inptr = coef_block;
170ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  quantptr = (ISLOW_MULT_TYPE *) compptr->dct_table;
171ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  wsptr = workspace;
172ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  for (ctr = DCTSIZE; ctr > 0; ctr--) {
173ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* Due to quantization, we will usually find that many of the input
174ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * coefficients are zero, especially the AC terms.  We can exploit this
175ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * by short-circuiting the IDCT calculation for any column in which all
176ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * the AC terms are zero.  In that case each output is equal to the
177ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * DC coefficient (with scale factor as needed).
178ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * With typical images and quantization tables, half or more of the
179ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * column DCT calculations can be simplified this way.
180ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     */
181ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
182ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    if (inptr[DCTSIZE*1] == 0 && inptr[DCTSIZE*2] == 0 &&
183ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov	inptr[DCTSIZE*3] == 0 && inptr[DCTSIZE*4] == 0 &&
184ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov	inptr[DCTSIZE*5] == 0 && inptr[DCTSIZE*6] == 0 &&
185ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov	inptr[DCTSIZE*7] == 0) {
186ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      /* AC terms all zero */
187ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      int dcval = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]) << PASS1_BITS;
188ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
189ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr[DCTSIZE*0] = dcval;
190ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr[DCTSIZE*1] = dcval;
191ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr[DCTSIZE*2] = dcval;
192ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr[DCTSIZE*3] = dcval;
193ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr[DCTSIZE*4] = dcval;
194ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr[DCTSIZE*5] = dcval;
195ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr[DCTSIZE*6] = dcval;
196ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr[DCTSIZE*7] = dcval;
197ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
198ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      inptr++;			/* advance pointers to next column */
199ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      quantptr++;
200ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr++;
201ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      continue;
202ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    }
203ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
204ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* Even part: reverse the even part of the forward DCT. */
205ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* The rotator is sqrt(2)*c(-6). */
206ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
207ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z2 = DEQUANTIZE(inptr[DCTSIZE*2], quantptr[DCTSIZE*2]);
208ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z3 = DEQUANTIZE(inptr[DCTSIZE*6], quantptr[DCTSIZE*6]);
209ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
210ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z1 = MULTIPLY(z2 + z3, FIX_0_541196100);
211ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp2 = z1 + MULTIPLY(z3, - FIX_1_847759065);
212ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp3 = z1 + MULTIPLY(z2, FIX_0_765366865);
213ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
214ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z2 = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]);
215ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z3 = DEQUANTIZE(inptr[DCTSIZE*4], quantptr[DCTSIZE*4]);
216ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
217ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp0 = (z2 + z3) << CONST_BITS;
218ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp1 = (z2 - z3) << CONST_BITS;
219ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
220ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp10 = tmp0 + tmp3;
221ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp13 = tmp0 - tmp3;
222ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp11 = tmp1 + tmp2;
223ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp12 = tmp1 - tmp2;
224ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
225ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* Odd part per figure 8; the matrix is unitary and hence its
226ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * transpose is its inverse.  i0..i3 are y7,y5,y3,y1 respectively.
227ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     */
228ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
229ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp0 = DEQUANTIZE(inptr[DCTSIZE*7], quantptr[DCTSIZE*7]);
230ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp1 = DEQUANTIZE(inptr[DCTSIZE*5], quantptr[DCTSIZE*5]);
231ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp2 = DEQUANTIZE(inptr[DCTSIZE*3], quantptr[DCTSIZE*3]);
232ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp3 = DEQUANTIZE(inptr[DCTSIZE*1], quantptr[DCTSIZE*1]);
233ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
234ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z1 = tmp0 + tmp3;
235ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z2 = tmp1 + tmp2;
236ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z3 = tmp0 + tmp2;
237ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z4 = tmp1 + tmp3;
238ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z5 = MULTIPLY(z3 + z4, FIX_1_175875602); /* sqrt(2) * c3 */
239ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
240ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp0 = MULTIPLY(tmp0, FIX_0_298631336); /* sqrt(2) * (-c1+c3+c5-c7) */
241ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp1 = MULTIPLY(tmp1, FIX_2_053119869); /* sqrt(2) * ( c1+c3-c5+c7) */
242ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp2 = MULTIPLY(tmp2, FIX_3_072711026); /* sqrt(2) * ( c1+c3+c5-c7) */
243ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp3 = MULTIPLY(tmp3, FIX_1_501321110); /* sqrt(2) * ( c1+c3-c5-c7) */
244ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z1 = MULTIPLY(z1, - FIX_0_899976223); /* sqrt(2) * (c7-c3) */
245ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z2 = MULTIPLY(z2, - FIX_2_562915447); /* sqrt(2) * (-c1-c3) */
246ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z3 = MULTIPLY(z3, - FIX_1_961570560); /* sqrt(2) * (-c3-c5) */
247ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z4 = MULTIPLY(z4, - FIX_0_390180644); /* sqrt(2) * (c5-c3) */
248ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
249ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z3 += z5;
250ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z4 += z5;
251ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
252ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp0 += z1 + z3;
253ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp1 += z2 + z4;
254ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp2 += z2 + z3;
255ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp3 += z1 + z4;
256ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
257ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* Final output stage: inputs are tmp10..tmp13, tmp0..tmp3 */
258ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
259ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr[DCTSIZE*0] = (int) DESCALE(tmp10 + tmp3, CONST_BITS-PASS1_BITS);
260ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr[DCTSIZE*7] = (int) DESCALE(tmp10 - tmp3, CONST_BITS-PASS1_BITS);
261ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr[DCTSIZE*1] = (int) DESCALE(tmp11 + tmp2, CONST_BITS-PASS1_BITS);
262ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr[DCTSIZE*6] = (int) DESCALE(tmp11 - tmp2, CONST_BITS-PASS1_BITS);
263ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr[DCTSIZE*2] = (int) DESCALE(tmp12 + tmp1, CONST_BITS-PASS1_BITS);
264ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr[DCTSIZE*5] = (int) DESCALE(tmp12 - tmp1, CONST_BITS-PASS1_BITS);
265ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr[DCTSIZE*3] = (int) DESCALE(tmp13 + tmp0, CONST_BITS-PASS1_BITS);
266ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr[DCTSIZE*4] = (int) DESCALE(tmp13 - tmp0, CONST_BITS-PASS1_BITS);
267ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
268ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    inptr++;			/* advance pointers to next column */
269ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    quantptr++;
270ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr++;
271ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  }
272ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
273ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  /* Pass 2: process rows from work array, store into output array. */
274ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  /* Note that we must descale the results by a factor of 8 == 2**3, */
275ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  /* and also undo the PASS1_BITS scaling. */
276ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
277ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  wsptr = workspace;
278ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  for (ctr = 0; ctr < DCTSIZE; ctr++) {
279ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    outptr = output_buf[ctr] + output_col;
280ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* Rows of zeroes can be exploited in the same way as we did with columns.
281ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * However, the column calculation has created many nonzero AC terms, so
282ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * the simplification applies less often (typically 5% to 10% of the time).
283ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * On machines with very fast multiplication, it's possible that the
284ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * test takes more time than it's worth.  In that case this section
285ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * may be commented out.
286ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     */
287ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
288ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#ifndef NO_ZERO_ROW_TEST
289ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    if (wsptr[1] == 0 && wsptr[2] == 0 && wsptr[3] == 0 && wsptr[4] == 0 &&
290ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov	wsptr[5] == 0 && wsptr[6] == 0 && wsptr[7] == 0) {
291ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      /* AC terms all zero */
292ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      JSAMPLE dcval = range_limit[(int) DESCALE((INT32) wsptr[0], PASS1_BITS+3)
293ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov				  & RANGE_MASK];
294ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
295ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      outptr[0] = dcval;
296ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      outptr[1] = dcval;
297ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      outptr[2] = dcval;
298ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      outptr[3] = dcval;
299ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      outptr[4] = dcval;
300ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      outptr[5] = dcval;
301ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      outptr[6] = dcval;
302ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      outptr[7] = dcval;
303ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
304ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      wsptr += DCTSIZE;		/* advance pointer to next row */
305ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov      continue;
306ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    }
307ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#endif
308ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
309ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* Even part: reverse the even part of the forward DCT. */
310ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* The rotator is sqrt(2)*c(-6). */
311ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
312ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z2 = (INT32) wsptr[2];
313ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z3 = (INT32) wsptr[6];
314ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
315ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z1 = MULTIPLY(z2 + z3, FIX_0_541196100);
316ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp2 = z1 + MULTIPLY(z3, - FIX_1_847759065);
317ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp3 = z1 + MULTIPLY(z2, FIX_0_765366865);
318ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
319ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp0 = ((INT32) wsptr[0] + (INT32) wsptr[4]) << CONST_BITS;
320ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp1 = ((INT32) wsptr[0] - (INT32) wsptr[4]) << CONST_BITS;
321ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
322ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp10 = tmp0 + tmp3;
323ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp13 = tmp0 - tmp3;
324ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp11 = tmp1 + tmp2;
325ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp12 = tmp1 - tmp2;
326ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
327ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* Odd part per figure 8; the matrix is unitary and hence its
328ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     * transpose is its inverse.  i0..i3 are y7,y5,y3,y1 respectively.
329ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov     */
330ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
331ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp0 = (INT32) wsptr[7];
332ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp1 = (INT32) wsptr[5];
333ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp2 = (INT32) wsptr[3];
334ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp3 = (INT32) wsptr[1];
335ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
336ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z1 = tmp0 + tmp3;
337ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z2 = tmp1 + tmp2;
338ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z3 = tmp0 + tmp2;
339ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z4 = tmp1 + tmp3;
340ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z5 = MULTIPLY(z3 + z4, FIX_1_175875602); /* sqrt(2) * c3 */
341ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
342ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp0 = MULTIPLY(tmp0, FIX_0_298631336); /* sqrt(2) * (-c1+c3+c5-c7) */
343ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp1 = MULTIPLY(tmp1, FIX_2_053119869); /* sqrt(2) * ( c1+c3-c5+c7) */
344ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp2 = MULTIPLY(tmp2, FIX_3_072711026); /* sqrt(2) * ( c1+c3+c5-c7) */
345ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp3 = MULTIPLY(tmp3, FIX_1_501321110); /* sqrt(2) * ( c1+c3-c5-c7) */
346ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z1 = MULTIPLY(z1, - FIX_0_899976223); /* sqrt(2) * (c7-c3) */
347ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z2 = MULTIPLY(z2, - FIX_2_562915447); /* sqrt(2) * (-c1-c3) */
348ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z3 = MULTIPLY(z3, - FIX_1_961570560); /* sqrt(2) * (-c3-c5) */
349ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z4 = MULTIPLY(z4, - FIX_0_390180644); /* sqrt(2) * (c5-c3) */
350ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
351ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z3 += z5;
352ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    z4 += z5;
353ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
354ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp0 += z1 + z3;
355ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp1 += z2 + z4;
356ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp2 += z2 + z3;
357ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    tmp3 += z1 + z4;
358ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
359ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    /* Final output stage: inputs are tmp10..tmp13, tmp0..tmp3 */
360ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
361ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    outptr[0] = range_limit[(int) DESCALE(tmp10 + tmp3,
362ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov					  CONST_BITS+PASS1_BITS+3)
363ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov			    & RANGE_MASK];
364ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    outptr[7] = range_limit[(int) DESCALE(tmp10 - tmp3,
365ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov					  CONST_BITS+PASS1_BITS+3)
366ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov			    & RANGE_MASK];
367ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    outptr[1] = range_limit[(int) DESCALE(tmp11 + tmp2,
368ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov					  CONST_BITS+PASS1_BITS+3)
369ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov			    & RANGE_MASK];
370ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    outptr[6] = range_limit[(int) DESCALE(tmp11 - tmp2,
371ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov					  CONST_BITS+PASS1_BITS+3)
372ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov			    & RANGE_MASK];
373ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    outptr[2] = range_limit[(int) DESCALE(tmp12 + tmp1,
374ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov					  CONST_BITS+PASS1_BITS+3)
375ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov			    & RANGE_MASK];
376ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    outptr[5] = range_limit[(int) DESCALE(tmp12 - tmp1,
377ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov					  CONST_BITS+PASS1_BITS+3)
378ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov			    & RANGE_MASK];
379ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    outptr[3] = range_limit[(int) DESCALE(tmp13 + tmp0,
380ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov					  CONST_BITS+PASS1_BITS+3)
381ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov			    & RANGE_MASK];
382ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    outptr[4] = range_limit[(int) DESCALE(tmp13 - tmp0,
383ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov					  CONST_BITS+PASS1_BITS+3)
384ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov			    & RANGE_MASK];
385ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
386ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov    wsptr += DCTSIZE;		/* advance pointer to next row */
387ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov  }
388ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov}
389ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
390ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#endif /* DCT_ISLOW_SUPPORTED */
391ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov
392ee451cb395940862dad63c85adfe8f2fd55e864cSvet Ganov#endif //_FX_JPEG_TURBO_
393