1;
2; jdclrss2.asm - colorspace conversion (SSE2)
3;
4; Copyright 2009, 2012 Pierre Ossman <ossman@cendio.se> for Cendio AB
5; Copyright 2012 D. R. Commander
6;
7; Based on
8; x86 SIMD extension for IJG JPEG library
9; Copyright (C) 1999-2006, MIYASAKA Masaru.
10; For conditions of distribution and use, see copyright notice in jsimdext.inc
11;
12; This file should be assembled with NASM (Netwide Assembler),
13; can *not* be assembled with Microsoft's MASM or any compatible
14; assembler (including Borland's Turbo Assembler).
15; NASM is available from http://nasm.sourceforge.net/ or
16; http://sourceforge.net/project/showfiles.php?group_id=6208
17;
18; [TAB8]
19
20%include "jcolsamp.inc"
21				
22; --------------------------------------------------------------------------
23;
24; Convert some rows of samples to the output colorspace.
25;
26; GLOBAL(void)
27; jsimd_ycc_rgb_convert_sse2 (JDIMENSION out_width,
28;                             JSAMPIMAGE input_buf, JDIMENSION input_row,
29;                             JSAMPARRAY output_buf, int num_rows)
30;
31
32%define out_width(b)	(b)+8			; JDIMENSION out_width
33%define input_buf(b)	(b)+12		; JSAMPIMAGE input_buf
34%define input_row(b)	(b)+16		; JDIMENSION input_row
35%define output_buf(b)	(b)+20		; JSAMPARRAY output_buf
36%define num_rows(b)	(b)+24		; int num_rows
37
38%define original_ebp	ebp+0
39%define wk(i)		ebp-(WK_NUM-(i))*SIZEOF_XMMWORD	; xmmword wk[WK_NUM]
40%define WK_NUM		2
41%define gotptr		wk(0)-SIZEOF_POINTER	; void * gotptr
42
43	align	16
44	global	EXTN(jsimd_ycc_rgb_convert_sse2) PRIVATE
45
46EXTN(jsimd_ycc_rgb_convert_sse2):
47	push	ebp
48	mov	eax,esp				; eax = original ebp
49	sub	esp, byte 4
50	and	esp, byte (-SIZEOF_XMMWORD)	; align to 128 bits
51	mov	[esp],eax
52	mov	ebp,esp				; ebp = aligned ebp
53	lea	esp, [wk(0)]
54	pushpic	eax		; make a room for GOT address
55	push	ebx
56;	push	ecx		; need not be preserved
57;	push	edx		; need not be preserved
58	push	esi
59	push	edi
60
61	get_GOT	ebx			; get GOT address
62	movpic	POINTER [gotptr], ebx	; save GOT address
63
64	mov	ecx, JDIMENSION [out_width(eax)]	; num_cols
65	test	ecx,ecx
66	jz	near .return
67
68	push	ecx
69
70	mov	edi, JSAMPIMAGE [input_buf(eax)]
71	mov	ecx, JDIMENSION [input_row(eax)]
72	mov	esi, JSAMPARRAY [edi+0*SIZEOF_JSAMPARRAY]
73	mov	ebx, JSAMPARRAY [edi+1*SIZEOF_JSAMPARRAY]
74	mov	edx, JSAMPARRAY [edi+2*SIZEOF_JSAMPARRAY]
75	lea	esi, [esi+ecx*SIZEOF_JSAMPROW]
76	lea	ebx, [ebx+ecx*SIZEOF_JSAMPROW]
77	lea	edx, [edx+ecx*SIZEOF_JSAMPROW]
78
79	pop	ecx
80
81	mov	edi, JSAMPARRAY [output_buf(eax)]
82	mov	eax, INT [num_rows(eax)]
83	test	eax,eax
84	jle	near .return
85	alignx	16,7
86.rowloop:
87	push	eax
88	push	edi
89	push	edx
90	push	ebx
91	push	esi
92	push	ecx			; col
93
94	mov	esi, JSAMPROW [esi]	; inptr0
95	mov	ebx, JSAMPROW [ebx]	; inptr1
96	mov	edx, JSAMPROW [edx]	; inptr2
97	mov	edi, JSAMPROW [edi]	; outptr
98	movpic	eax, POINTER [gotptr]	; load GOT address (eax)
99	alignx	16,7
100.columnloop:
101
102	movdqa	xmm5, XMMWORD [ebx]	; xmm5=Cb(0123456789ABCDEF)
103	movdqa	xmm1, XMMWORD [edx]	; xmm1=Cr(0123456789ABCDEF)
104
105	pcmpeqw	xmm4,xmm4
106	pcmpeqw	xmm7,xmm7
107	psrlw	xmm4,BYTE_BIT
108	psllw	xmm7,7			; xmm7={0xFF80 0xFF80 0xFF80 0xFF80 ..}
109	movdqa	xmm0,xmm4		; xmm0=xmm4={0xFF 0x00 0xFF 0x00 ..}
110
111	pand	xmm4,xmm5		; xmm4=Cb(02468ACE)=CbE
112	psrlw	xmm5,BYTE_BIT		; xmm5=Cb(13579BDF)=CbO
113	pand	xmm0,xmm1		; xmm0=Cr(02468ACE)=CrE
114	psrlw	xmm1,BYTE_BIT		; xmm1=Cr(13579BDF)=CrO
115
116	paddw	xmm4,xmm7
117	paddw	xmm5,xmm7
118	paddw	xmm0,xmm7
119	paddw	xmm1,xmm7
120
121	; (Original)
122	; R = Y                + 1.40200 * Cr
123	; G = Y - 0.34414 * Cb - 0.71414 * Cr
124	; B = Y + 1.77200 * Cb
125	;
126	; (This implementation)
127	; R = Y                + 0.40200 * Cr + Cr
128	; G = Y - 0.34414 * Cb + 0.28586 * Cr - Cr
129	; B = Y - 0.22800 * Cb + Cb + Cb
130
131	movdqa	xmm2,xmm4		; xmm2=CbE
132	movdqa	xmm3,xmm5		; xmm3=CbO
133	paddw	xmm4,xmm4		; xmm4=2*CbE
134	paddw	xmm5,xmm5		; xmm5=2*CbO
135	movdqa	xmm6,xmm0		; xmm6=CrE
136	movdqa	xmm7,xmm1		; xmm7=CrO
137	paddw	xmm0,xmm0		; xmm0=2*CrE
138	paddw	xmm1,xmm1		; xmm1=2*CrO
139
140	pmulhw	xmm4,[GOTOFF(eax,PW_MF0228)]	; xmm4=(2*CbE * -FIX(0.22800))
141	pmulhw	xmm5,[GOTOFF(eax,PW_MF0228)]	; xmm5=(2*CbO * -FIX(0.22800))
142	pmulhw	xmm0,[GOTOFF(eax,PW_F0402)]	; xmm0=(2*CrE * FIX(0.40200))
143	pmulhw	xmm1,[GOTOFF(eax,PW_F0402)]	; xmm1=(2*CrO * FIX(0.40200))
144
145	paddw	xmm4,[GOTOFF(eax,PW_ONE)]
146	paddw	xmm5,[GOTOFF(eax,PW_ONE)]
147	psraw	xmm4,1			; xmm4=(CbE * -FIX(0.22800))
148	psraw	xmm5,1			; xmm5=(CbO * -FIX(0.22800))
149	paddw	xmm0,[GOTOFF(eax,PW_ONE)]
150	paddw	xmm1,[GOTOFF(eax,PW_ONE)]
151	psraw	xmm0,1			; xmm0=(CrE * FIX(0.40200))
152	psraw	xmm1,1			; xmm1=(CrO * FIX(0.40200))
153
154	paddw	xmm4,xmm2
155	paddw	xmm5,xmm3
156	paddw	xmm4,xmm2		; xmm4=(CbE * FIX(1.77200))=(B-Y)E
157	paddw	xmm5,xmm3		; xmm5=(CbO * FIX(1.77200))=(B-Y)O
158	paddw	xmm0,xmm6		; xmm0=(CrE * FIX(1.40200))=(R-Y)E
159	paddw	xmm1,xmm7		; xmm1=(CrO * FIX(1.40200))=(R-Y)O
160
161	movdqa	XMMWORD [wk(0)], xmm4	; wk(0)=(B-Y)E
162	movdqa	XMMWORD [wk(1)], xmm5	; wk(1)=(B-Y)O
163
164	movdqa    xmm4,xmm2
165	movdqa    xmm5,xmm3
166	punpcklwd xmm2,xmm6
167	punpckhwd xmm4,xmm6
168	pmaddwd   xmm2,[GOTOFF(eax,PW_MF0344_F0285)]
169	pmaddwd   xmm4,[GOTOFF(eax,PW_MF0344_F0285)]
170	punpcklwd xmm3,xmm7
171	punpckhwd xmm5,xmm7
172	pmaddwd   xmm3,[GOTOFF(eax,PW_MF0344_F0285)]
173	pmaddwd   xmm5,[GOTOFF(eax,PW_MF0344_F0285)]
174
175	paddd     xmm2,[GOTOFF(eax,PD_ONEHALF)]
176	paddd     xmm4,[GOTOFF(eax,PD_ONEHALF)]
177	psrad     xmm2,SCALEBITS
178	psrad     xmm4,SCALEBITS
179	paddd     xmm3,[GOTOFF(eax,PD_ONEHALF)]
180	paddd     xmm5,[GOTOFF(eax,PD_ONEHALF)]
181	psrad     xmm3,SCALEBITS
182	psrad     xmm5,SCALEBITS
183
184	packssdw  xmm2,xmm4	; xmm2=CbE*-FIX(0.344)+CrE*FIX(0.285)
185	packssdw  xmm3,xmm5	; xmm3=CbO*-FIX(0.344)+CrO*FIX(0.285)
186	psubw     xmm2,xmm6	; xmm2=CbE*-FIX(0.344)+CrE*-FIX(0.714)=(G-Y)E
187	psubw     xmm3,xmm7	; xmm3=CbO*-FIX(0.344)+CrO*-FIX(0.714)=(G-Y)O
188
189	movdqa    xmm5, XMMWORD [esi]	; xmm5=Y(0123456789ABCDEF)
190
191	pcmpeqw   xmm4,xmm4
192	psrlw     xmm4,BYTE_BIT		; xmm4={0xFF 0x00 0xFF 0x00 ..}
193	pand      xmm4,xmm5		; xmm4=Y(02468ACE)=YE
194	psrlw     xmm5,BYTE_BIT		; xmm5=Y(13579BDF)=YO
195
196	paddw     xmm0,xmm4		; xmm0=((R-Y)E+YE)=RE=R(02468ACE)
197	paddw     xmm1,xmm5		; xmm1=((R-Y)O+YO)=RO=R(13579BDF)
198	packuswb  xmm0,xmm0		; xmm0=R(02468ACE********)
199	packuswb  xmm1,xmm1		; xmm1=R(13579BDF********)
200
201	paddw     xmm2,xmm4		; xmm2=((G-Y)E+YE)=GE=G(02468ACE)
202	paddw     xmm3,xmm5		; xmm3=((G-Y)O+YO)=GO=G(13579BDF)
203	packuswb  xmm2,xmm2		; xmm2=G(02468ACE********)
204	packuswb  xmm3,xmm3		; xmm3=G(13579BDF********)
205
206	paddw     xmm4, XMMWORD [wk(0)]	; xmm4=(YE+(B-Y)E)=BE=B(02468ACE)
207	paddw     xmm5, XMMWORD [wk(1)]	; xmm5=(YO+(B-Y)O)=BO=B(13579BDF)
208	packuswb  xmm4,xmm4		; xmm4=B(02468ACE********)
209	packuswb  xmm5,xmm5		; xmm5=B(13579BDF********)
210
211%if RGB_PIXELSIZE == 3 ; ---------------
212
213	; xmmA=(00 02 04 06 08 0A 0C 0E **), xmmB=(01 03 05 07 09 0B 0D 0F **)
214	; xmmC=(10 12 14 16 18 1A 1C 1E **), xmmD=(11 13 15 17 19 1B 1D 1F **)
215	; xmmE=(20 22 24 26 28 2A 2C 2E **), xmmF=(21 23 25 27 29 2B 2D 2F **)
216	; xmmG=(** ** ** ** ** ** ** ** **), xmmH=(** ** ** ** ** ** ** ** **)
217
218	punpcklbw xmmA,xmmC	; xmmA=(00 10 02 12 04 14 06 16 08 18 0A 1A 0C 1C 0E 1E)
219	punpcklbw xmmE,xmmB	; xmmE=(20 01 22 03 24 05 26 07 28 09 2A 0B 2C 0D 2E 0F)
220	punpcklbw xmmD,xmmF	; xmmD=(11 21 13 23 15 25 17 27 19 29 1B 2B 1D 2D 1F 2F)
221
222	movdqa    xmmG,xmmA
223	movdqa    xmmH,xmmA
224	punpcklwd xmmA,xmmE	; xmmA=(00 10 20 01 02 12 22 03 04 14 24 05 06 16 26 07)
225	punpckhwd xmmG,xmmE	; xmmG=(08 18 28 09 0A 1A 2A 0B 0C 1C 2C 0D 0E 1E 2E 0F)
226
227	psrldq    xmmH,2	; xmmH=(02 12 04 14 06 16 08 18 0A 1A 0C 1C 0E 1E -- --)
228	psrldq    xmmE,2	; xmmE=(22 03 24 05 26 07 28 09 2A 0B 2C 0D 2E 0F -- --)
229
230	movdqa    xmmC,xmmD
231	movdqa    xmmB,xmmD
232	punpcklwd xmmD,xmmH	; xmmD=(11 21 02 12 13 23 04 14 15 25 06 16 17 27 08 18)
233	punpckhwd xmmC,xmmH	; xmmC=(19 29 0A 1A 1B 2B 0C 1C 1D 2D 0E 1E 1F 2F -- --)
234
235	psrldq    xmmB,2	; xmmB=(13 23 15 25 17 27 19 29 1B 2B 1D 2D 1F 2F -- --)
236
237	movdqa    xmmF,xmmE
238	punpcklwd xmmE,xmmB	; xmmE=(22 03 13 23 24 05 15 25 26 07 17 27 28 09 19 29)
239	punpckhwd xmmF,xmmB	; xmmF=(2A 0B 1B 2B 2C 0D 1D 2D 2E 0F 1F 2F -- -- -- --)
240
241	pshufd    xmmH,xmmA,0x4E; xmmH=(04 14 24 05 06 16 26 07 00 10 20 01 02 12 22 03)
242	movdqa    xmmB,xmmE
243	punpckldq xmmA,xmmD	; xmmA=(00 10 20 01 11 21 02 12 02 12 22 03 13 23 04 14)
244	punpckldq xmmE,xmmH	; xmmE=(22 03 13 23 04 14 24 05 24 05 15 25 06 16 26 07)
245	punpckhdq xmmD,xmmB	; xmmD=(15 25 06 16 26 07 17 27 17 27 08 18 28 09 19 29)
246
247	pshufd    xmmH,xmmG,0x4E; xmmH=(0C 1C 2C 0D 0E 1E 2E 0F 08 18 28 09 0A 1A 2A 0B)
248	movdqa    xmmB,xmmF
249	punpckldq xmmG,xmmC	; xmmG=(08 18 28 09 19 29 0A 1A 0A 1A 2A 0B 1B 2B 0C 1C)
250	punpckldq xmmF,xmmH	; xmmF=(2A 0B 1B 2B 0C 1C 2C 0D 2C 0D 1D 2D 0E 1E 2E 0F)
251	punpckhdq xmmC,xmmB	; xmmC=(1D 2D 0E 1E 2E 0F 1F 2F 1F 2F -- -- -- -- -- --)
252
253	punpcklqdq xmmA,xmmE	; xmmA=(00 10 20 01 11 21 02 12 22 03 13 23 04 14 24 05)
254	punpcklqdq xmmD,xmmG	; xmmD=(15 25 06 16 26 07 17 27 08 18 28 09 19 29 0A 1A)
255	punpcklqdq xmmF,xmmC	; xmmF=(2A 0B 1B 2B 0C 1C 2C 0D 1D 2D 0E 1E 2E 0F 1F 2F)
256
257	cmp	ecx, byte SIZEOF_XMMWORD
258	jb	short .column_st32
259
260	test	edi, SIZEOF_XMMWORD-1
261	jnz	short .out1
262	; --(aligned)-------------------
263	movntdq	XMMWORD [edi+0*SIZEOF_XMMWORD], xmmA
264	movntdq	XMMWORD [edi+1*SIZEOF_XMMWORD], xmmD
265	movntdq	XMMWORD [edi+2*SIZEOF_XMMWORD], xmmF
266	jmp	short .out0
267.out1:	; --(unaligned)-----------------
268	movdqu	XMMWORD [edi+0*SIZEOF_XMMWORD], xmmA
269	movdqu	XMMWORD [edi+1*SIZEOF_XMMWORD], xmmD
270	movdqu	XMMWORD [edi+2*SIZEOF_XMMWORD], xmmF
271.out0:
272	add	edi, byte RGB_PIXELSIZE*SIZEOF_XMMWORD	; outptr
273	sub	ecx, byte SIZEOF_XMMWORD
274	jz	near .nextrow
275
276	add	esi, byte SIZEOF_XMMWORD	; inptr0
277	add	ebx, byte SIZEOF_XMMWORD	; inptr1
278	add	edx, byte SIZEOF_XMMWORD	; inptr2
279	jmp	near .columnloop
280	alignx	16,7
281
282.column_st32:
283	lea	ecx, [ecx+ecx*2]		; imul ecx, RGB_PIXELSIZE
284	cmp	ecx, byte 2*SIZEOF_XMMWORD
285	jb	short .column_st16
286	movdqu	XMMWORD [edi+0*SIZEOF_XMMWORD], xmmA
287	movdqu	XMMWORD [edi+1*SIZEOF_XMMWORD], xmmD
288	add	edi, byte 2*SIZEOF_XMMWORD	; outptr
289	movdqa	xmmA,xmmF
290	sub	ecx, byte 2*SIZEOF_XMMWORD
291	jmp	short .column_st15
292.column_st16:
293	cmp	ecx, byte SIZEOF_XMMWORD
294	jb	short .column_st15
295	movdqu	XMMWORD [edi+0*SIZEOF_XMMWORD], xmmA
296	add	edi, byte SIZEOF_XMMWORD	; outptr
297	movdqa	xmmA,xmmD
298	sub	ecx, byte SIZEOF_XMMWORD
299.column_st15:
300	; Store the lower 8 bytes of xmmA to the output when it has enough
301	; space.
302	cmp	ecx, byte SIZEOF_MMWORD
303	jb	short .column_st7
304	movq	XMM_MMWORD [edi], xmmA
305	add	edi, byte SIZEOF_MMWORD
306	sub	ecx, byte SIZEOF_MMWORD
307	psrldq	xmmA, SIZEOF_MMWORD
308.column_st7:
309	; Store the lower 4 bytes of xmmA to the output when it has enough
310	; space.
311	cmp	ecx, byte SIZEOF_DWORD
312	jb	short .column_st3
313	movd	XMM_DWORD [edi], xmmA
314	add	edi, byte SIZEOF_DWORD
315	sub	ecx, byte SIZEOF_DWORD
316	psrldq	xmmA, SIZEOF_DWORD
317.column_st3:
318	; Store the lower 2 bytes of eax to the output when it has enough
319	; space.
320	movd	eax, xmmA
321	cmp	ecx, byte SIZEOF_WORD
322	jb	short .column_st1
323	mov	WORD [edi], ax
324	add	edi, byte SIZEOF_WORD
325	sub	ecx, byte SIZEOF_WORD
326	shr	eax, 16
327.column_st1:
328	; Store the lower 1 byte of eax to the output when it has enough
329	; space.
330	test	ecx, ecx
331	jz	short .nextrow
332	mov	BYTE [edi], al
333
334%else ; RGB_PIXELSIZE == 4 ; -----------
335
336%ifdef RGBX_FILLER_0XFF
337	pcmpeqb   xmm6,xmm6		; xmm6=XE=X(02468ACE********)
338	pcmpeqb   xmm7,xmm7		; xmm7=XO=X(13579BDF********)
339%else
340	pxor      xmm6,xmm6		; xmm6=XE=X(02468ACE********)
341	pxor      xmm7,xmm7		; xmm7=XO=X(13579BDF********)
342%endif
343	; xmmA=(00 02 04 06 08 0A 0C 0E **), xmmB=(01 03 05 07 09 0B 0D 0F **)
344	; xmmC=(10 12 14 16 18 1A 1C 1E **), xmmD=(11 13 15 17 19 1B 1D 1F **)
345	; xmmE=(20 22 24 26 28 2A 2C 2E **), xmmF=(21 23 25 27 29 2B 2D 2F **)
346	; xmmG=(30 32 34 36 38 3A 3C 3E **), xmmH=(31 33 35 37 39 3B 3D 3F **)
347
348	punpcklbw xmmA,xmmC	; xmmA=(00 10 02 12 04 14 06 16 08 18 0A 1A 0C 1C 0E 1E)
349	punpcklbw xmmE,xmmG	; xmmE=(20 30 22 32 24 34 26 36 28 38 2A 3A 2C 3C 2E 3E)
350	punpcklbw xmmB,xmmD	; xmmB=(01 11 03 13 05 15 07 17 09 19 0B 1B 0D 1D 0F 1F)
351	punpcklbw xmmF,xmmH	; xmmF=(21 31 23 33 25 35 27 37 29 39 2B 3B 2D 3D 2F 3F)
352
353	movdqa    xmmC,xmmA
354	punpcklwd xmmA,xmmE	; xmmA=(00 10 20 30 02 12 22 32 04 14 24 34 06 16 26 36)
355	punpckhwd xmmC,xmmE	; xmmC=(08 18 28 38 0A 1A 2A 3A 0C 1C 2C 3C 0E 1E 2E 3E)
356	movdqa    xmmG,xmmB
357	punpcklwd xmmB,xmmF	; xmmB=(01 11 21 31 03 13 23 33 05 15 25 35 07 17 27 37)
358	punpckhwd xmmG,xmmF	; xmmG=(09 19 29 39 0B 1B 2B 3B 0D 1D 2D 3D 0F 1F 2F 3F)
359
360	movdqa    xmmD,xmmA
361	punpckldq xmmA,xmmB	; xmmA=(00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33)
362	punpckhdq xmmD,xmmB	; xmmD=(04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37)
363	movdqa    xmmH,xmmC
364	punpckldq xmmC,xmmG	; xmmC=(08 18 28 38 09 19 29 39 0A 1A 2A 3A 0B 1B 2B 3B)
365	punpckhdq xmmH,xmmG	; xmmH=(0C 1C 2C 3C 0D 1D 2D 3D 0E 1E 2E 3E 0F 1F 2F 3F)
366
367	cmp	ecx, byte SIZEOF_XMMWORD
368	jb	short .column_st32
369
370	test	edi, SIZEOF_XMMWORD-1
371	jnz	short .out1
372	; --(aligned)-------------------
373	movntdq	XMMWORD [edi+0*SIZEOF_XMMWORD], xmmA
374	movntdq	XMMWORD [edi+1*SIZEOF_XMMWORD], xmmD
375	movntdq	XMMWORD [edi+2*SIZEOF_XMMWORD], xmmC
376	movntdq	XMMWORD [edi+3*SIZEOF_XMMWORD], xmmH
377	jmp	short .out0
378.out1:	; --(unaligned)-----------------
379	movdqu	XMMWORD [edi+0*SIZEOF_XMMWORD], xmmA
380	movdqu	XMMWORD [edi+1*SIZEOF_XMMWORD], xmmD
381	movdqu	XMMWORD [edi+2*SIZEOF_XMMWORD], xmmC
382	movdqu	XMMWORD [edi+3*SIZEOF_XMMWORD], xmmH
383.out0:
384	add	edi, byte RGB_PIXELSIZE*SIZEOF_XMMWORD	; outptr
385	sub	ecx, byte SIZEOF_XMMWORD
386	jz	near .nextrow
387
388	add	esi, byte SIZEOF_XMMWORD	; inptr0
389	add	ebx, byte SIZEOF_XMMWORD	; inptr1
390	add	edx, byte SIZEOF_XMMWORD	; inptr2
391	jmp	near .columnloop
392	alignx	16,7
393
394.column_st32:
395	cmp	ecx, byte SIZEOF_XMMWORD/2
396	jb	short .column_st16
397	movdqu	XMMWORD [edi+0*SIZEOF_XMMWORD], xmmA
398	movdqu	XMMWORD [edi+1*SIZEOF_XMMWORD], xmmD
399	add	edi, byte 2*SIZEOF_XMMWORD	; outptr
400	movdqa	xmmA,xmmC
401	movdqa	xmmD,xmmH
402	sub	ecx, byte SIZEOF_XMMWORD/2
403.column_st16:
404	cmp	ecx, byte SIZEOF_XMMWORD/4
405	jb	short .column_st15
406	movdqu	XMMWORD [edi+0*SIZEOF_XMMWORD], xmmA
407	add	edi, byte SIZEOF_XMMWORD	; outptr
408	movdqa	xmmA,xmmD
409	sub	ecx, byte SIZEOF_XMMWORD/4
410.column_st15:
411	; Store two pixels (8 bytes) of xmmA to the output when it has enough
412	; space.
413	cmp	ecx, byte SIZEOF_XMMWORD/8
414	jb	short .column_st7
415	movq	XMM_MMWORD [edi], xmmA
416	add	edi, byte SIZEOF_XMMWORD/8*4
417	sub	ecx, byte SIZEOF_XMMWORD/8
418	psrldq	xmmA, SIZEOF_XMMWORD/8*4
419.column_st7:
420	; Store one pixel (4 bytes) of xmmA to the output when it has enough
421	; space.
422	test	ecx, ecx
423	jz	short .nextrow
424	movd	XMM_DWORD [edi], xmmA
425
426%endif ; RGB_PIXELSIZE ; ---------------
427
428	alignx	16,7
429
430.nextrow:
431	pop	ecx
432	pop	esi
433	pop	ebx
434	pop	edx
435	pop	edi
436	pop	eax
437
438	add	esi, byte SIZEOF_JSAMPROW
439	add	ebx, byte SIZEOF_JSAMPROW
440	add	edx, byte SIZEOF_JSAMPROW
441	add	edi, byte SIZEOF_JSAMPROW	; output_buf
442	dec	eax				; num_rows
443	jg	near .rowloop
444
445	sfence		; flush the write buffer
446
447.return:
448	pop	edi
449	pop	esi
450;	pop	edx		; need not be preserved
451;	pop	ecx		; need not be preserved
452	pop	ebx
453	mov	esp,ebp		; esp <- aligned ebp
454	pop	esp		; esp <- original ebp
455	pop	ebp
456	ret
457
458; For some reason, the OS X linker does not honor the request to align the
459; segment unless we do this.
460	align	16
461