190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber;
2f71323e297a928af368937089d3ed71239786f86Andreas Huber;  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber;
4f71323e297a928af368937089d3ed71239786f86Andreas Huber;  Use of this source code is governed by a BSD-style license
5f71323e297a928af368937089d3ed71239786f86Andreas Huber;  that can be found in the LICENSE file in the root of the source
6f71323e297a928af368937089d3ed71239786f86Andreas Huber;  tree. An additional intellectual property rights grant can be found
7f71323e297a928af368937089d3ed71239786f86Andreas Huber;  in the file PATENTS.  All contributing project authors may
8f71323e297a928af368937089d3ed71239786f86Andreas Huber;  be found in the AUTHORS file in the root of the source tree.
990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber;
1090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
1190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
1290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber%include "vpx_ports/x86_abi_support.asm"
1390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
141b362b15af34006e6a11974088a46d42b903418eJohann
151b362b15af34006e6a11974088a46d42b903418eJohann;void vp8_dequantize_b_impl_mmx(short *sq, short *dq, short *q)
161b362b15af34006e6a11974088a46d42b903418eJohannglobal sym(vp8_dequantize_b_impl_mmx) PRIVATE
171b362b15af34006e6a11974088a46d42b903418eJohannsym(vp8_dequantize_b_impl_mmx):
1890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    push        rbp
1990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    mov         rbp, rsp
2090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    SHADOW_ARGS_TO_STACK 3
211b362b15af34006e6a11974088a46d42b903418eJohann    push        rsi
221b362b15af34006e6a11974088a46d42b903418eJohann    push        rdi
231b362b15af34006e6a11974088a46d42b903418eJohann    ; end prolog
241b362b15af34006e6a11974088a46d42b903418eJohann
251b362b15af34006e6a11974088a46d42b903418eJohann        mov       rsi, arg(0) ;sq
261b362b15af34006e6a11974088a46d42b903418eJohann        mov       rdi, arg(1) ;dq
271b362b15af34006e6a11974088a46d42b903418eJohann        mov       rax, arg(2) ;q
281b362b15af34006e6a11974088a46d42b903418eJohann
291b362b15af34006e6a11974088a46d42b903418eJohann        movq      mm1, [rsi]
301b362b15af34006e6a11974088a46d42b903418eJohann        pmullw    mm1, [rax+0]            ; mm4 *= kernel 0 modifiers.
311b362b15af34006e6a11974088a46d42b903418eJohann        movq      [rdi], mm1
321b362b15af34006e6a11974088a46d42b903418eJohann
331b362b15af34006e6a11974088a46d42b903418eJohann        movq      mm1, [rsi+8]
341b362b15af34006e6a11974088a46d42b903418eJohann        pmullw    mm1, [rax+8]            ; mm4 *= kernel 0 modifiers.
351b362b15af34006e6a11974088a46d42b903418eJohann        movq      [rdi+8], mm1
361b362b15af34006e6a11974088a46d42b903418eJohann
371b362b15af34006e6a11974088a46d42b903418eJohann        movq      mm1, [rsi+16]
381b362b15af34006e6a11974088a46d42b903418eJohann        pmullw    mm1, [rax+16]            ; mm4 *= kernel 0 modifiers.
391b362b15af34006e6a11974088a46d42b903418eJohann        movq      [rdi+16], mm1
401b362b15af34006e6a11974088a46d42b903418eJohann
411b362b15af34006e6a11974088a46d42b903418eJohann        movq      mm1, [rsi+24]
421b362b15af34006e6a11974088a46d42b903418eJohann        pmullw    mm1, [rax+24]            ; mm4 *= kernel 0 modifiers.
431b362b15af34006e6a11974088a46d42b903418eJohann        movq      [rdi+24], mm1
441b362b15af34006e6a11974088a46d42b903418eJohann
451b362b15af34006e6a11974088a46d42b903418eJohann    ; begin epilog
461b362b15af34006e6a11974088a46d42b903418eJohann    pop rdi
471b362b15af34006e6a11974088a46d42b903418eJohann    pop rsi
481b362b15af34006e6a11974088a46d42b903418eJohann    UNSHADOW_ARGS
491b362b15af34006e6a11974088a46d42b903418eJohann    pop         rbp
501b362b15af34006e6a11974088a46d42b903418eJohann    ret
511b362b15af34006e6a11974088a46d42b903418eJohann
521b362b15af34006e6a11974088a46d42b903418eJohann
531b362b15af34006e6a11974088a46d42b903418eJohann;void dequant_idct_add_mmx(
541b362b15af34006e6a11974088a46d42b903418eJohann;short *input,            0
551b362b15af34006e6a11974088a46d42b903418eJohann;short *dq,               1
561b362b15af34006e6a11974088a46d42b903418eJohann;unsigned char *dest,     2
571b362b15af34006e6a11974088a46d42b903418eJohann;int stride)              3
581b362b15af34006e6a11974088a46d42b903418eJohannglobal sym(vp8_dequant_idct_add_mmx) PRIVATE
591b362b15af34006e6a11974088a46d42b903418eJohannsym(vp8_dequant_idct_add_mmx):
601b362b15af34006e6a11974088a46d42b903418eJohann    push        rbp
611b362b15af34006e6a11974088a46d42b903418eJohann    mov         rbp, rsp
621b362b15af34006e6a11974088a46d42b903418eJohann    SHADOW_ARGS_TO_STACK 4
6390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    GET_GOT     rbx
641b362b15af34006e6a11974088a46d42b903418eJohann    push        rdi
6590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    ; end prolog
6690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
671b362b15af34006e6a11974088a46d42b903418eJohann        mov         rax,    arg(0) ;input
681b362b15af34006e6a11974088a46d42b903418eJohann        mov         rdx,    arg(1) ;dq
691b362b15af34006e6a11974088a46d42b903418eJohann
701b362b15af34006e6a11974088a46d42b903418eJohann
711b362b15af34006e6a11974088a46d42b903418eJohann        movq        mm0,    [rax   ]
721b362b15af34006e6a11974088a46d42b903418eJohann        pmullw      mm0,    [rdx]
731b362b15af34006e6a11974088a46d42b903418eJohann
741b362b15af34006e6a11974088a46d42b903418eJohann        movq        mm1,    [rax +8]
751b362b15af34006e6a11974088a46d42b903418eJohann        pmullw      mm1,    [rdx +8]
761b362b15af34006e6a11974088a46d42b903418eJohann
771b362b15af34006e6a11974088a46d42b903418eJohann        movq        mm2,    [rax+16]
781b362b15af34006e6a11974088a46d42b903418eJohann        pmullw      mm2,    [rdx+16]
791b362b15af34006e6a11974088a46d42b903418eJohann
801b362b15af34006e6a11974088a46d42b903418eJohann        movq        mm3,    [rax+24]
811b362b15af34006e6a11974088a46d42b903418eJohann        pmullw      mm3,    [rdx+24]
821b362b15af34006e6a11974088a46d42b903418eJohann
831b362b15af34006e6a11974088a46d42b903418eJohann        mov         rdx,    arg(2) ;dest
841b362b15af34006e6a11974088a46d42b903418eJohann
851b362b15af34006e6a11974088a46d42b903418eJohann        pxor        mm7,    mm7
8690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
8790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
881b362b15af34006e6a11974088a46d42b903418eJohann        movq        [rax],   mm7
891b362b15af34006e6a11974088a46d42b903418eJohann        movq        [rax+8], mm7
9090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
911b362b15af34006e6a11974088a46d42b903418eJohann        movq        [rax+16],mm7
921b362b15af34006e6a11974088a46d42b903418eJohann        movq        [rax+24],mm7
931b362b15af34006e6a11974088a46d42b903418eJohann
941b362b15af34006e6a11974088a46d42b903418eJohann
951b362b15af34006e6a11974088a46d42b903418eJohann        movsxd      rdi,            dword ptr arg(3) ;stride
9690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
9790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psubw       mm0,            mm2             ; b1= 0-2
9890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm2,            mm2             ;
9990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
10090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm5,            mm1
10190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm2,            mm0             ; a1 =0+2
10290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
1031b362b15af34006e6a11974088a46d42b903418eJohann        pmulhw      mm5,            [GLOBAL(x_s1sqr2)];
10490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm5,            mm1             ; ip1 * sin(pi/8) * sqrt(2)
10590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
10690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm7,            mm3             ;
1071b362b15af34006e6a11974088a46d42b903418eJohann        pmulhw      mm7,            [GLOBAL(x_c1sqr2less1)];
10890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
10990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm7,            mm3             ; ip3 * cos(pi/8) * sqrt(2)
11090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psubw       mm7,            mm5             ; c1
11190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
11290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm5,            mm1
11390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm4,            mm3
11490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
115538f6170b788de7408b06efc6613dc98579aa6a6Andreas Huber        pmulhw      mm5,            [GLOBAL(x_c1sqr2less1)]
11690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm5,            mm1
11790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
118538f6170b788de7408b06efc6613dc98579aa6a6Andreas Huber        pmulhw      mm3,            [GLOBAL(x_s1sqr2)]
11990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm3,            mm4
12090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
12190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm3,            mm5             ; d1
12290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm6,            mm2             ; a1
12390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
12490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm4,            mm0             ; b1
12590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm2,            mm3             ;0
12690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
12790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm4,            mm7             ;1
12890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psubw       mm0,            mm7             ;2
12990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
13090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psubw       mm6,            mm3             ;3
13190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
13290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm1,            mm2             ; 03 02 01 00
13390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm3,            mm4             ; 23 22 21 20
13490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
13590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpcklwd   mm1,            mm0             ; 11 01 10 00
13690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckhwd   mm2,            mm0             ; 13 03 12 02
13790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
13890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpcklwd   mm3,            mm6             ; 31 21 30 20
13990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckhwd   mm4,            mm6             ; 33 23 32 22
14090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
14190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm0,            mm1             ; 11 01 10 00
14290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm5,            mm2             ; 13 03 12 02
14390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
14490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckldq   mm0,            mm3             ; 30 20 10 00
14590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckhdq   mm1,            mm3             ; 31 21 11 01
14690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
14790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckldq   mm2,            mm4             ; 32 22 12 02
14890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckhdq   mm5,            mm4             ; 33 23 13 03
14990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
15090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm3,            mm5             ; 33 23 13 03
15190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
15290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psubw       mm0,            mm2             ; b1= 0-2
15390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm2,            mm2             ;
15490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
15590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm5,            mm1
15690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm2,            mm0             ; a1 =0+2
15790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
1581b362b15af34006e6a11974088a46d42b903418eJohann        pmulhw      mm5,            [GLOBAL(x_s1sqr2)];
15990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm5,            mm1             ; ip1 * sin(pi/8) * sqrt(2)
16090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
16190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm7,            mm3             ;
1621b362b15af34006e6a11974088a46d42b903418eJohann        pmulhw      mm7,            [GLOBAL(x_c1sqr2less1)];
16390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
16490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm7,            mm3             ; ip3 * cos(pi/8) * sqrt(2)
16590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psubw       mm7,            mm5             ; c1
16690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
16790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm5,            mm1
16890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm4,            mm3
16990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
170538f6170b788de7408b06efc6613dc98579aa6a6Andreas Huber        pmulhw      mm5,            [GLOBAL(x_c1sqr2less1)]
17190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm5,            mm1
17290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
173538f6170b788de7408b06efc6613dc98579aa6a6Andreas Huber        pmulhw      mm3,            [GLOBAL(x_s1sqr2)]
17490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm3,            mm4
17590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
17690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm3,            mm5             ; d1
177538f6170b788de7408b06efc6613dc98579aa6a6Andreas Huber        paddw       mm0,            [GLOBAL(fours)]
17890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
179538f6170b788de7408b06efc6613dc98579aa6a6Andreas Huber        paddw       mm2,            [GLOBAL(fours)]
18090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm6,            mm2             ; a1
18190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
18290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm4,            mm0             ; b1
18390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm2,            mm3             ;0
18490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
18590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        paddw       mm4,            mm7             ;1
18690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psubw       mm0,            mm7             ;2
18790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
18890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psubw       mm6,            mm3             ;3
18990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psraw       mm2,            3
19090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
19190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psraw       mm0,            3
19290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psraw       mm4,            3
19390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
19490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        psraw       mm6,            3
19590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
19690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm1,            mm2             ; 03 02 01 00
19790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm3,            mm4             ; 23 22 21 20
19890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
19990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpcklwd   mm1,            mm0             ; 11 01 10 00
20090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckhwd   mm2,            mm0             ; 13 03 12 02
20190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
20290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpcklwd   mm3,            mm6             ; 31 21 30 20
20390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckhwd   mm4,            mm6             ; 33 23 32 22
20490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
20590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm0,            mm1             ; 11 01 10 00
20690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        movq        mm5,            mm2             ; 13 03 12 02
20790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
20890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckldq   mm0,            mm3             ; 30 20 10 00
20990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckhdq   mm1,            mm3             ; 31 21 11 01
21090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
21190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckldq   mm2,            mm4             ; 32 22 12 02
21290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber        punpckhdq   mm5,            mm4             ; 33 23 13 03
21390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
2141b362b15af34006e6a11974088a46d42b903418eJohann        pxor        mm7,            mm7
21590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
2161b362b15af34006e6a11974088a46d42b903418eJohann        movd        mm4,            [rdx]
2171b362b15af34006e6a11974088a46d42b903418eJohann        punpcklbw   mm4,            mm7
2181b362b15af34006e6a11974088a46d42b903418eJohann        paddsw      mm0,            mm4
2191b362b15af34006e6a11974088a46d42b903418eJohann        packuswb    mm0,            mm7
2201b362b15af34006e6a11974088a46d42b903418eJohann        movd        [rdx],          mm0
22190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
2221b362b15af34006e6a11974088a46d42b903418eJohann        movd        mm4,            [rdx+rdi]
2231b362b15af34006e6a11974088a46d42b903418eJohann        punpcklbw   mm4,            mm7
2241b362b15af34006e6a11974088a46d42b903418eJohann        paddsw      mm1,            mm4
2251b362b15af34006e6a11974088a46d42b903418eJohann        packuswb    mm1,            mm7
2261b362b15af34006e6a11974088a46d42b903418eJohann        movd        [rdx+rdi],      mm1
22790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
2281b362b15af34006e6a11974088a46d42b903418eJohann        movd        mm4,            [rdx+2*rdi]
2291b362b15af34006e6a11974088a46d42b903418eJohann        punpcklbw   mm4,            mm7
2301b362b15af34006e6a11974088a46d42b903418eJohann        paddsw      mm2,            mm4
2311b362b15af34006e6a11974088a46d42b903418eJohann        packuswb    mm2,            mm7
2321b362b15af34006e6a11974088a46d42b903418eJohann        movd        [rdx+rdi*2],    mm2
23390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
2341b362b15af34006e6a11974088a46d42b903418eJohann        add         rdx,            rdi
23590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
2361b362b15af34006e6a11974088a46d42b903418eJohann        movd        mm4,            [rdx+2*rdi]
2371b362b15af34006e6a11974088a46d42b903418eJohann        punpcklbw   mm4,            mm7
2381b362b15af34006e6a11974088a46d42b903418eJohann        paddsw      mm5,            mm4
2391b362b15af34006e6a11974088a46d42b903418eJohann        packuswb    mm5,            mm7
2401b362b15af34006e6a11974088a46d42b903418eJohann        movd        [rdx+rdi*2],    mm5
24190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
24290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    ; begin epilog
243f71323e297a928af368937089d3ed71239786f86Andreas Huber    pop rdi
24490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    RESTORE_GOT
24590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    UNSHADOW_ARGS
24690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    pop         rbp
24790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    ret
24890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber
24990d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas HuberSECTION_RODATA
25090d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huberalign 16
25190d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huberx_s1sqr2:
25290d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    times 4 dw 0x8A8C
25390d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huberalign 16
25490d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huberx_c1sqr2less1:
25590d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    times 4 dw 0x4E7B
25690d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huberalign 16
25790d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huberfours:
25890d3ed91ae9228e1c8bab561b6138d4cb8c1e4fdAndreas Huber    times 4 dw 0x0004
259