191037db265ecdd914a26e056cf69207b4f50924ehkuang; 291037db265ecdd914a26e056cf69207b4f50924ehkuang; Copyright (c) 2010 The WebM project authors. All Rights Reserved. 391037db265ecdd914a26e056cf69207b4f50924ehkuang; 491037db265ecdd914a26e056cf69207b4f50924ehkuang; Use of this source code is governed by a BSD-style license 591037db265ecdd914a26e056cf69207b4f50924ehkuang; that can be found in the LICENSE file in the root of the source 691037db265ecdd914a26e056cf69207b4f50924ehkuang; tree. An additional intellectual property rights grant can be found 791037db265ecdd914a26e056cf69207b4f50924ehkuang; in the file PATENTS. All contributing project authors may 891037db265ecdd914a26e056cf69207b4f50924ehkuang; be found in the AUTHORS file in the root of the source tree. 991037db265ecdd914a26e056cf69207b4f50924ehkuang; 1091037db265ecdd914a26e056cf69207b4f50924ehkuang 1191037db265ecdd914a26e056cf69207b4f50924ehkuang%include "third_party/x86inc/x86inc.asm" 1291037db265ecdd914a26e056cf69207b4f50924ehkuang 1391037db265ecdd914a26e056cf69207b4f50924ehkuangSECTION .text 1491037db265ecdd914a26e056cf69207b4f50924ehkuang 15c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%macro convolve_fn 1-2 1668e1c830ade592be74773e249bf94e2bbfb50de7Johann%ifidn %1, avg 1768e1c830ade592be74773e249bf94e2bbfb50de7Johann%define AUX_XMM_REGS 4 1868e1c830ade592be74773e249bf94e2bbfb50de7Johann%else 1968e1c830ade592be74773e249bf94e2bbfb50de7Johann%define AUX_XMM_REGS 0 2068e1c830ade592be74773e249bf94e2bbfb50de7Johann%endif 21c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%ifidn %2, highbd 22c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%define pavg pavgw 2368e1c830ade592be74773e249bf94e2bbfb50de7Johanncglobal %2_convolve_%1, 4, 7, 4+AUX_XMM_REGS, src, src_stride, \ 2468e1c830ade592be74773e249bf94e2bbfb50de7Johann dst, dst_stride, \ 2568e1c830ade592be74773e249bf94e2bbfb50de7Johann fx, fxs, fy, fys, w, h, bd 26c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%else 27c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%define pavg pavgb 2868e1c830ade592be74773e249bf94e2bbfb50de7Johanncglobal convolve_%1, 4, 7, 4+AUX_XMM_REGS, src, src_stride, \ 2968e1c830ade592be74773e249bf94e2bbfb50de7Johann dst, dst_stride, \ 3068e1c830ade592be74773e249bf94e2bbfb50de7Johann fx, fxs, fy, fys, w, h 31c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%endif 3291037db265ecdd914a26e056cf69207b4f50924ehkuang mov r4d, dword wm 33c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%ifidn %2, highbd 34c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann shl r4d, 1 35c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann shl src_strideq, 1 36c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann shl dst_strideq, 1 37c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%else 3891037db265ecdd914a26e056cf69207b4f50924ehkuang cmp r4d, 4 3991037db265ecdd914a26e056cf69207b4f50924ehkuang je .w4 40c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%endif 4191037db265ecdd914a26e056cf69207b4f50924ehkuang cmp r4d, 8 4291037db265ecdd914a26e056cf69207b4f50924ehkuang je .w8 4391037db265ecdd914a26e056cf69207b4f50924ehkuang cmp r4d, 16 4491037db265ecdd914a26e056cf69207b4f50924ehkuang je .w16 4591037db265ecdd914a26e056cf69207b4f50924ehkuang cmp r4d, 32 4691037db265ecdd914a26e056cf69207b4f50924ehkuang je .w32 47c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%ifidn %2, highbd 48c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann cmp r4d, 64 49c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann je .w64 5091037db265ecdd914a26e056cf69207b4f50924ehkuang 5191037db265ecdd914a26e056cf69207b4f50924ehkuang mov r4d, dword hm 52c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann.loop128: 53c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann movu m0, [srcq] 54c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann movu m1, [srcq+16] 55c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann movu m2, [srcq+32] 56c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann movu m3, [srcq+48] 57c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%ifidn %1, avg 58c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m0, [dstq] 59c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m1, [dstq+16] 60c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m2, [dstq+32] 61c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m3, [dstq+48] 62c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%endif 63c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann mova [dstq ], m0 64c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann mova [dstq+16], m1 65c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann mova [dstq+32], m2 66c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann mova [dstq+48], m3 67c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann movu m0, [srcq+64] 68c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann movu m1, [srcq+80] 69c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann movu m2, [srcq+96] 70c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann movu m3, [srcq+112] 71c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann add srcq, src_strideq 72c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%ifidn %1, avg 73c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m0, [dstq+64] 74c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m1, [dstq+80] 75c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m2, [dstq+96] 76c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m3, [dstq+112] 77c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%endif 78c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann mova [dstq+64], m0 79c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann mova [dstq+80], m1 80c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann mova [dstq+96], m2 81c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann mova [dstq+112], m3 82c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann add dstq, dst_strideq 83c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann dec r4d 84c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann jnz .loop128 85c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann RET 86c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%endif 87c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann 887bc9febe8749e98a3812a0dc4380ceae75c29450Johann.w64: 89c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann mov r4d, dword hm 9091037db265ecdd914a26e056cf69207b4f50924ehkuang.loop64: 9191037db265ecdd914a26e056cf69207b4f50924ehkuang movu m0, [srcq] 9291037db265ecdd914a26e056cf69207b4f50924ehkuang movu m1, [srcq+16] 9391037db265ecdd914a26e056cf69207b4f50924ehkuang movu m2, [srcq+32] 9491037db265ecdd914a26e056cf69207b4f50924ehkuang movu m3, [srcq+48] 9591037db265ecdd914a26e056cf69207b4f50924ehkuang add srcq, src_strideq 9691037db265ecdd914a26e056cf69207b4f50924ehkuang%ifidn %1, avg 97c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m0, [dstq] 98c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m1, [dstq+16] 99c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m2, [dstq+32] 100c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m3, [dstq+48] 10191037db265ecdd914a26e056cf69207b4f50924ehkuang%endif 10291037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq ], m0 10391037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq+16], m1 10491037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq+32], m2 10591037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq+48], m3 10691037db265ecdd914a26e056cf69207b4f50924ehkuang add dstq, dst_strideq 10791037db265ecdd914a26e056cf69207b4f50924ehkuang dec r4d 10891037db265ecdd914a26e056cf69207b4f50924ehkuang jnz .loop64 10991037db265ecdd914a26e056cf69207b4f50924ehkuang RET 11091037db265ecdd914a26e056cf69207b4f50924ehkuang 11191037db265ecdd914a26e056cf69207b4f50924ehkuang.w32: 11291037db265ecdd914a26e056cf69207b4f50924ehkuang mov r4d, dword hm 11391037db265ecdd914a26e056cf69207b4f50924ehkuang.loop32: 11491037db265ecdd914a26e056cf69207b4f50924ehkuang movu m0, [srcq] 11591037db265ecdd914a26e056cf69207b4f50924ehkuang movu m1, [srcq+16] 11691037db265ecdd914a26e056cf69207b4f50924ehkuang movu m2, [srcq+src_strideq] 11791037db265ecdd914a26e056cf69207b4f50924ehkuang movu m3, [srcq+src_strideq+16] 11891037db265ecdd914a26e056cf69207b4f50924ehkuang lea srcq, [srcq+src_strideq*2] 11991037db265ecdd914a26e056cf69207b4f50924ehkuang%ifidn %1, avg 120c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m0, [dstq] 121c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m1, [dstq +16] 122c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m2, [dstq+dst_strideq] 123c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m3, [dstq+dst_strideq+16] 12491037db265ecdd914a26e056cf69207b4f50924ehkuang%endif 12591037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq ], m0 12691037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq +16], m1 12791037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq+dst_strideq ], m2 12891037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq+dst_strideq+16], m3 12991037db265ecdd914a26e056cf69207b4f50924ehkuang lea dstq, [dstq+dst_strideq*2] 13091037db265ecdd914a26e056cf69207b4f50924ehkuang sub r4d, 2 13191037db265ecdd914a26e056cf69207b4f50924ehkuang jnz .loop32 13291037db265ecdd914a26e056cf69207b4f50924ehkuang RET 13391037db265ecdd914a26e056cf69207b4f50924ehkuang 13491037db265ecdd914a26e056cf69207b4f50924ehkuang.w16: 13591037db265ecdd914a26e056cf69207b4f50924ehkuang mov r4d, dword hm 13691037db265ecdd914a26e056cf69207b4f50924ehkuang lea r5q, [src_strideq*3] 13791037db265ecdd914a26e056cf69207b4f50924ehkuang lea r6q, [dst_strideq*3] 13891037db265ecdd914a26e056cf69207b4f50924ehkuang.loop16: 13991037db265ecdd914a26e056cf69207b4f50924ehkuang movu m0, [srcq] 14091037db265ecdd914a26e056cf69207b4f50924ehkuang movu m1, [srcq+src_strideq] 14191037db265ecdd914a26e056cf69207b4f50924ehkuang movu m2, [srcq+src_strideq*2] 14291037db265ecdd914a26e056cf69207b4f50924ehkuang movu m3, [srcq+r5q] 14391037db265ecdd914a26e056cf69207b4f50924ehkuang lea srcq, [srcq+src_strideq*4] 14491037db265ecdd914a26e056cf69207b4f50924ehkuang%ifidn %1, avg 145c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m0, [dstq] 146c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m1, [dstq+dst_strideq] 147c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m2, [dstq+dst_strideq*2] 148c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m3, [dstq+r6q] 14991037db265ecdd914a26e056cf69207b4f50924ehkuang%endif 15091037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq ], m0 15191037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq+dst_strideq ], m1 15291037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq+dst_strideq*2], m2 15391037db265ecdd914a26e056cf69207b4f50924ehkuang mova [dstq+r6q ], m3 15491037db265ecdd914a26e056cf69207b4f50924ehkuang lea dstq, [dstq+dst_strideq*4] 15591037db265ecdd914a26e056cf69207b4f50924ehkuang sub r4d, 4 15691037db265ecdd914a26e056cf69207b4f50924ehkuang jnz .loop16 15791037db265ecdd914a26e056cf69207b4f50924ehkuang RET 15891037db265ecdd914a26e056cf69207b4f50924ehkuang 15991037db265ecdd914a26e056cf69207b4f50924ehkuang.w8: 16091037db265ecdd914a26e056cf69207b4f50924ehkuang mov r4d, dword hm 16191037db265ecdd914a26e056cf69207b4f50924ehkuang lea r5q, [src_strideq*3] 16291037db265ecdd914a26e056cf69207b4f50924ehkuang lea r6q, [dst_strideq*3] 16391037db265ecdd914a26e056cf69207b4f50924ehkuang.loop8: 16468e1c830ade592be74773e249bf94e2bbfb50de7Johann movh m0, [srcq] 16568e1c830ade592be74773e249bf94e2bbfb50de7Johann movh m1, [srcq+src_strideq] 16668e1c830ade592be74773e249bf94e2bbfb50de7Johann movh m2, [srcq+src_strideq*2] 16768e1c830ade592be74773e249bf94e2bbfb50de7Johann movh m3, [srcq+r5q] 16891037db265ecdd914a26e056cf69207b4f50924ehkuang lea srcq, [srcq+src_strideq*4] 16991037db265ecdd914a26e056cf69207b4f50924ehkuang%ifidn %1, avg 17068e1c830ade592be74773e249bf94e2bbfb50de7Johann movh m4, [dstq] 17168e1c830ade592be74773e249bf94e2bbfb50de7Johann movh m5, [dstq+dst_strideq] 17268e1c830ade592be74773e249bf94e2bbfb50de7Johann movh m6, [dstq+dst_strideq*2] 17368e1c830ade592be74773e249bf94e2bbfb50de7Johann movh m7, [dstq+r6q] 17468e1c830ade592be74773e249bf94e2bbfb50de7Johann pavg m0, m4 17568e1c830ade592be74773e249bf94e2bbfb50de7Johann pavg m1, m5 17668e1c830ade592be74773e249bf94e2bbfb50de7Johann pavg m2, m6 17768e1c830ade592be74773e249bf94e2bbfb50de7Johann pavg m3, m7 17891037db265ecdd914a26e056cf69207b4f50924ehkuang%endif 17968e1c830ade592be74773e249bf94e2bbfb50de7Johann movh [dstq ], m0 18068e1c830ade592be74773e249bf94e2bbfb50de7Johann movh [dstq+dst_strideq ], m1 18168e1c830ade592be74773e249bf94e2bbfb50de7Johann movh [dstq+dst_strideq*2], m2 18268e1c830ade592be74773e249bf94e2bbfb50de7Johann movh [dstq+r6q ], m3 18391037db265ecdd914a26e056cf69207b4f50924ehkuang lea dstq, [dstq+dst_strideq*4] 18491037db265ecdd914a26e056cf69207b4f50924ehkuang sub r4d, 4 18591037db265ecdd914a26e056cf69207b4f50924ehkuang jnz .loop8 18691037db265ecdd914a26e056cf69207b4f50924ehkuang RET 18791037db265ecdd914a26e056cf69207b4f50924ehkuang 188c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%ifnidn %2, highbd 18991037db265ecdd914a26e056cf69207b4f50924ehkuang.w4: 19091037db265ecdd914a26e056cf69207b4f50924ehkuang mov r4d, dword hm 19191037db265ecdd914a26e056cf69207b4f50924ehkuang lea r5q, [src_strideq*3] 19291037db265ecdd914a26e056cf69207b4f50924ehkuang lea r6q, [dst_strideq*3] 19391037db265ecdd914a26e056cf69207b4f50924ehkuang.loop4: 19468e1c830ade592be74773e249bf94e2bbfb50de7Johann movd m0, [srcq] 19568e1c830ade592be74773e249bf94e2bbfb50de7Johann movd m1, [srcq+src_strideq] 19668e1c830ade592be74773e249bf94e2bbfb50de7Johann movd m2, [srcq+src_strideq*2] 19768e1c830ade592be74773e249bf94e2bbfb50de7Johann movd m3, [srcq+r5q] 19891037db265ecdd914a26e056cf69207b4f50924ehkuang lea srcq, [srcq+src_strideq*4] 19991037db265ecdd914a26e056cf69207b4f50924ehkuang%ifidn %1, avg 20068e1c830ade592be74773e249bf94e2bbfb50de7Johann movd m4, [dstq] 20168e1c830ade592be74773e249bf94e2bbfb50de7Johann movd m5, [dstq+dst_strideq] 20268e1c830ade592be74773e249bf94e2bbfb50de7Johann movd m6, [dstq+dst_strideq*2] 20368e1c830ade592be74773e249bf94e2bbfb50de7Johann movd m7, [dstq+r6q] 204c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m0, m4 205c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m1, m5 206c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m2, m6 207c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann pavg m3, m7 20891037db265ecdd914a26e056cf69207b4f50924ehkuang%endif 20968e1c830ade592be74773e249bf94e2bbfb50de7Johann movd [dstq ], m0 21068e1c830ade592be74773e249bf94e2bbfb50de7Johann movd [dstq+dst_strideq ], m1 21168e1c830ade592be74773e249bf94e2bbfb50de7Johann movd [dstq+dst_strideq*2], m2 21268e1c830ade592be74773e249bf94e2bbfb50de7Johann movd [dstq+r6q ], m3 21391037db265ecdd914a26e056cf69207b4f50924ehkuang lea dstq, [dstq+dst_strideq*4] 21491037db265ecdd914a26e056cf69207b4f50924ehkuang sub r4d, 4 21591037db265ecdd914a26e056cf69207b4f50924ehkuang jnz .loop4 21691037db265ecdd914a26e056cf69207b4f50924ehkuang RET 217c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%endif 21891037db265ecdd914a26e056cf69207b4f50924ehkuang%endmacro 21991037db265ecdd914a26e056cf69207b4f50924ehkuang 22068e1c830ade592be74773e249bf94e2bbfb50de7JohannINIT_XMM sse2 22191037db265ecdd914a26e056cf69207b4f50924ehkuangconvolve_fn copy 22291037db265ecdd914a26e056cf69207b4f50924ehkuangconvolve_fn avg 223c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%if CONFIG_VP9_HIGHBITDEPTH 224c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohannconvolve_fn copy, highbd 225c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohannconvolve_fn avg, highbd 226c927526be9a7b72fb5edb3f29c4e8ceabe0ec98aJohann%endif 227