1;***************************************************************************** 2;* x86inc.asm: x264asm abstraction layer 3;***************************************************************************** 4;* Copyright (C) 2005-2012 x264 project 5;* 6;* Authors: Loren Merritt <lorenm@u.washington.edu> 7;* Anton Mitrofanov <BugMaster@narod.ru> 8;* Jason Garrett-Glaser <darkshikari@gmail.com> 9;* Henrik Gramner <hengar-6@student.ltu.se> 10;* 11;* Permission to use, copy, modify, and/or distribute this software for any 12;* purpose with or without fee is hereby granted, provided that the above 13;* copyright notice and this permission notice appear in all copies. 14;* 15;* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 16;* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 17;* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 18;* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 19;* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 20;* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 21;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 22;***************************************************************************** 23 24; This is a header file for the x264ASM assembly language, which uses 25; NASM/YASM syntax combined with a large number of macros to provide easy 26; abstraction between different calling conventions (x86_32, win64, linux64). 27; It also has various other useful features to simplify writing the kind of 28; DSP functions that are most often used in x264. 29 30; Unlike the rest of x264, this file is available under an ISC license, as it 31; has significant usefulness outside of x264 and we want it to be available 32; to the largest audience possible. Of course, if you modify it for your own 33; purposes to add a new feature, we strongly encourage contributing a patch 34; as this feature might be useful for others as well. Send patches or ideas 35; to x264-devel@videolan.org . 36 37%include "vpx_config.asm" 38 39%define program_name vp9 40 41 42%define UNIX64 0 43%define WIN64 0 44%if ARCH_X86_64 45 %ifidn __OUTPUT_FORMAT__,win32 46 %define WIN64 1 47 %elifidn __OUTPUT_FORMAT__,win64 48 %define WIN64 1 49 %elifidn __OUTPUT_FORMAT__,x64 50 %define WIN64 1 51 %else 52 %define UNIX64 1 53 %endif 54%endif 55 56%ifidn __OUTPUT_FORMAT__,elf32 57 %define mangle(x) x 58%elifidn __OUTPUT_FORMAT__,elf64 59 %define mangle(x) x 60%elifidn __OUTPUT_FORMAT__,elf 61 %define mangle(x) x 62%elifidn __OUTPUT_FORMAT__,x64 63 %define mangle(x) x 64%elifidn __OUTPUT_FORMAT__,win64 65 %define mangle(x) x 66%else 67 %define mangle(x) _ %+ x 68%endif 69 70; FIXME: All of the 64bit asm functions that take a stride as an argument 71; via register, assume that the high dword of that register is filled with 0. 72; This is true in practice (since we never do any 64bit arithmetic on strides, 73; and x264's strides are all positive), but is not guaranteed by the ABI. 74 75; Name of the .rodata section. 76; Kludge: Something on OS X fails to align .rodata even given an align attribute, 77; so use a different read-only section. 78%macro SECTION_RODATA 0-1 16 79 %ifidn __OUTPUT_FORMAT__,macho64 80 SECTION .text align=%1 81 %elifidn __OUTPUT_FORMAT__,macho 82 SECTION .text align=%1 83 fakegot: 84 %elifidn __OUTPUT_FORMAT__,aout 85 section .text 86 %else 87 SECTION .rodata align=%1 88 %endif 89%endmacro 90 91; aout does not support align= 92%macro SECTION_TEXT 0-1 16 93 %ifidn __OUTPUT_FORMAT__,aout 94 SECTION .text 95 %else 96 SECTION .text align=%1 97 %endif 98%endmacro 99 100; PIC macros are copied from vpx_ports/x86_abi_support.asm. The "define PIC" 101; from original code is added in for 64bit. 102%ifidn __OUTPUT_FORMAT__,elf32 103%define ABI_IS_32BIT 1 104%elifidn __OUTPUT_FORMAT__,macho32 105%define ABI_IS_32BIT 1 106%elifidn __OUTPUT_FORMAT__,win32 107%define ABI_IS_32BIT 1 108%elifidn __OUTPUT_FORMAT__,aout 109%define ABI_IS_32BIT 1 110%else 111%define ABI_IS_32BIT 0 112%endif 113 114%if ABI_IS_32BIT 115 %if CONFIG_PIC=1 116 %ifidn __OUTPUT_FORMAT__,elf32 117 %define GET_GOT_SAVE_ARG 1 118 %define WRT_PLT wrt ..plt 119 %macro GET_GOT 1 120 extern _GLOBAL_OFFSET_TABLE_ 121 push %1 122 call %%get_got 123 %%sub_offset: 124 jmp %%exitGG 125 %%get_got: 126 mov %1, [esp] 127 add %1, _GLOBAL_OFFSET_TABLE_ + $$ - %%sub_offset wrt ..gotpc 128 ret 129 %%exitGG: 130 %undef GLOBAL 131 %define GLOBAL(x) x + %1 wrt ..gotoff 132 %undef RESTORE_GOT 133 %define RESTORE_GOT pop %1 134 %endmacro 135 %elifidn __OUTPUT_FORMAT__,macho32 136 %define GET_GOT_SAVE_ARG 1 137 %macro GET_GOT 1 138 push %1 139 call %%get_got 140 %%get_got: 141 pop %1 142 %undef GLOBAL 143 %define GLOBAL(x) x + %1 - %%get_got 144 %undef RESTORE_GOT 145 %define RESTORE_GOT pop %1 146 %endmacro 147 %endif 148 %endif 149 150 %if ARCH_X86_64 == 0 151 %undef PIC 152 %endif 153 154%else 155 %macro GET_GOT 1 156 %endmacro 157 %define GLOBAL(x) rel x 158 %define WRT_PLT wrt ..plt 159 160 %if WIN64 161 %define PIC 162 %elifidn __OUTPUT_FORMAT__,macho64 163 %define PIC 164 %elif CONFIG_PIC 165 %define PIC 166 %endif 167%endif 168 169%ifnmacro GET_GOT 170 %macro GET_GOT 1 171 %endmacro 172 %define GLOBAL(x) x 173%endif 174%ifndef RESTORE_GOT 175%define RESTORE_GOT 176%endif 177%ifndef WRT_PLT 178%define WRT_PLT 179%endif 180 181%ifdef PIC 182 default rel 183%endif 184; Done with PIC macros 185 186; Always use long nops (reduces 0x90 spam in disassembly on x86_32) 187%ifndef __NASM_VER__ 188CPU amdnop 189%else 190%use smartalign 191ALIGNMODE k7 192%endif 193 194; Macros to eliminate most code duplication between x86_32 and x86_64: 195; Currently this works only for leaf functions which load all their arguments 196; into registers at the start, and make no other use of the stack. Luckily that 197; covers most of x264's asm. 198 199; PROLOGUE: 200; %1 = number of arguments. loads them from stack if needed. 201; %2 = number of registers used. pushes callee-saved regs if needed. 202; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed. 203; %4 = list of names to define to registers 204; PROLOGUE can also be invoked by adding the same options to cglobal 205 206; e.g. 207; cglobal foo, 2,3,0, dst, src, tmp 208; declares a function (foo), taking two args (dst and src) and one local variable (tmp) 209 210; TODO Some functions can use some args directly from the stack. If they're the 211; last args then you can just not declare them, but if they're in the middle 212; we need more flexible macro. 213 214; RET: 215; Pops anything that was pushed by PROLOGUE, and returns. 216 217; REP_RET: 218; Same, but if it doesn't pop anything it becomes a 2-byte ret, for athlons 219; which are slow when a normal ret follows a branch. 220 221; registers: 222; rN and rNq are the native-size register holding function argument N 223; rNd, rNw, rNb are dword, word, and byte size 224; rNm is the original location of arg N (a register or on the stack), dword 225; rNmp is native size 226 227%macro DECLARE_REG 5-6 228 %define r%1q %2 229 %define r%1d %3 230 %define r%1w %4 231 %define r%1b %5 232 %if %0 == 5 233 %define r%1m %3 234 %define r%1mp %2 235 %elif ARCH_X86_64 ; memory 236 %define r%1m [rsp + stack_offset + %6] 237 %define r%1mp qword r %+ %1m 238 %else 239 %define r%1m [esp + stack_offset + %6] 240 %define r%1mp dword r %+ %1m 241 %endif 242 %define r%1 %2 243%endmacro 244 245%macro DECLARE_REG_SIZE 2 246 %define r%1q r%1 247 %define e%1q r%1 248 %define r%1d e%1 249 %define e%1d e%1 250 %define r%1w %1 251 %define e%1w %1 252 %define r%1b %2 253 %define e%1b %2 254%if ARCH_X86_64 == 0 255 %define r%1 e%1 256%endif 257%endmacro 258 259DECLARE_REG_SIZE ax, al 260DECLARE_REG_SIZE bx, bl 261DECLARE_REG_SIZE cx, cl 262DECLARE_REG_SIZE dx, dl 263DECLARE_REG_SIZE si, sil 264DECLARE_REG_SIZE di, dil 265DECLARE_REG_SIZE bp, bpl 266 267; t# defines for when per-arch register allocation is more complex than just function arguments 268 269%macro DECLARE_REG_TMP 1-* 270 %assign %%i 0 271 %rep %0 272 CAT_XDEFINE t, %%i, r%1 273 %assign %%i %%i+1 274 %rotate 1 275 %endrep 276%endmacro 277 278%macro DECLARE_REG_TMP_SIZE 0-* 279 %rep %0 280 %define t%1q t%1 %+ q 281 %define t%1d t%1 %+ d 282 %define t%1w t%1 %+ w 283 %define t%1b t%1 %+ b 284 %rotate 1 285 %endrep 286%endmacro 287 288DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 289 290%if ARCH_X86_64 291 %define gprsize 8 292%else 293 %define gprsize 4 294%endif 295 296%macro PUSH 1 297 push %1 298 %assign stack_offset stack_offset+gprsize 299%endmacro 300 301%macro POP 1 302 pop %1 303 %assign stack_offset stack_offset-gprsize 304%endmacro 305 306%macro PUSH_IF_USED 1-* 307 %rep %0 308 %if %1 < regs_used 309 PUSH r%1 310 %endif 311 %rotate 1 312 %endrep 313%endmacro 314 315%macro POP_IF_USED 1-* 316 %rep %0 317 %if %1 < regs_used 318 pop r%1 319 %endif 320 %rotate 1 321 %endrep 322%endmacro 323 324%macro LOAD_IF_USED 1-* 325 %rep %0 326 %if %1 < num_args 327 mov r%1, r %+ %1 %+ mp 328 %endif 329 %rotate 1 330 %endrep 331%endmacro 332 333%macro SUB 2 334 sub %1, %2 335 %ifidn %1, rsp 336 %assign stack_offset stack_offset+(%2) 337 %endif 338%endmacro 339 340%macro ADD 2 341 add %1, %2 342 %ifidn %1, rsp 343 %assign stack_offset stack_offset-(%2) 344 %endif 345%endmacro 346 347%macro movifnidn 2 348 %ifnidn %1, %2 349 mov %1, %2 350 %endif 351%endmacro 352 353%macro movsxdifnidn 2 354 %ifnidn %1, %2 355 movsxd %1, %2 356 %endif 357%endmacro 358 359%macro ASSERT 1 360 %if (%1) == 0 361 %error assert failed 362 %endif 363%endmacro 364 365%macro DEFINE_ARGS 0-* 366 %ifdef n_arg_names 367 %assign %%i 0 368 %rep n_arg_names 369 CAT_UNDEF arg_name %+ %%i, q 370 CAT_UNDEF arg_name %+ %%i, d 371 CAT_UNDEF arg_name %+ %%i, w 372 CAT_UNDEF arg_name %+ %%i, b 373 CAT_UNDEF arg_name %+ %%i, m 374 CAT_UNDEF arg_name %+ %%i, mp 375 CAT_UNDEF arg_name, %%i 376 %assign %%i %%i+1 377 %endrep 378 %endif 379 380 %xdefine %%stack_offset stack_offset 381 %undef stack_offset ; so that the current value of stack_offset doesn't get baked in by xdefine 382 %assign %%i 0 383 %rep %0 384 %xdefine %1q r %+ %%i %+ q 385 %xdefine %1d r %+ %%i %+ d 386 %xdefine %1w r %+ %%i %+ w 387 %xdefine %1b r %+ %%i %+ b 388 %xdefine %1m r %+ %%i %+ m 389 %xdefine %1mp r %+ %%i %+ mp 390 CAT_XDEFINE arg_name, %%i, %1 391 %assign %%i %%i+1 392 %rotate 1 393 %endrep 394 %xdefine stack_offset %%stack_offset 395 %assign n_arg_names %0 396%endmacro 397 398%if WIN64 ; Windows x64 ;================================================= 399 400DECLARE_REG 0, rcx, ecx, cx, cl 401DECLARE_REG 1, rdx, edx, dx, dl 402DECLARE_REG 2, R8, R8D, R8W, R8B 403DECLARE_REG 3, R9, R9D, R9W, R9B 404DECLARE_REG 4, R10, R10D, R10W, R10B, 40 405DECLARE_REG 5, R11, R11D, R11W, R11B, 48 406DECLARE_REG 6, rax, eax, ax, al, 56 407DECLARE_REG 7, rdi, edi, di, dil, 64 408DECLARE_REG 8, rsi, esi, si, sil, 72 409DECLARE_REG 9, rbx, ebx, bx, bl, 80 410DECLARE_REG 10, rbp, ebp, bp, bpl, 88 411DECLARE_REG 11, R12, R12D, R12W, R12B, 96 412DECLARE_REG 12, R13, R13D, R13W, R13B, 104 413DECLARE_REG 13, R14, R14D, R14W, R14B, 112 414DECLARE_REG 14, R15, R15D, R15W, R15B, 120 415 416%macro PROLOGUE 2-4+ 0 ; #args, #regs, #xmm_regs, arg_names... 417 %assign num_args %1 418 %assign regs_used %2 419 ASSERT regs_used >= num_args 420 ASSERT regs_used <= 15 421 PUSH_IF_USED 7, 8, 9, 10, 11, 12, 13, 14 422 %if mmsize == 8 423 %assign xmm_regs_used 0 424 %else 425 WIN64_SPILL_XMM %3 426 %endif 427 LOAD_IF_USED 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 428 DEFINE_ARGS %4 429%endmacro 430 431%macro WIN64_SPILL_XMM 1 432 %assign xmm_regs_used %1 433 ASSERT xmm_regs_used <= 16 434 %if xmm_regs_used > 6 435 SUB rsp, (xmm_regs_used-6)*16+16 436 %assign %%i xmm_regs_used 437 %rep (xmm_regs_used-6) 438 %assign %%i %%i-1 439 movdqa [rsp + (%%i-6)*16+(~stack_offset&8)], xmm %+ %%i 440 %endrep 441 %endif 442%endmacro 443 444%macro WIN64_RESTORE_XMM_INTERNAL 1 445 %if xmm_regs_used > 6 446 %assign %%i xmm_regs_used 447 %rep (xmm_regs_used-6) 448 %assign %%i %%i-1 449 movdqa xmm %+ %%i, [%1 + (%%i-6)*16+(~stack_offset&8)] 450 %endrep 451 add %1, (xmm_regs_used-6)*16+16 452 %endif 453%endmacro 454 455%macro WIN64_RESTORE_XMM 1 456 WIN64_RESTORE_XMM_INTERNAL %1 457 %assign stack_offset stack_offset-(xmm_regs_used-6)*16+16 458 %assign xmm_regs_used 0 459%endmacro 460 461%macro RET 0 462 WIN64_RESTORE_XMM_INTERNAL rsp 463 POP_IF_USED 14, 13, 12, 11, 10, 9, 8, 7 464 ret 465%endmacro 466 467%macro REP_RET 0 468 %if regs_used > 7 || xmm_regs_used > 6 469 RET 470 %else 471 rep ret 472 %endif 473%endmacro 474 475%elif ARCH_X86_64 ; *nix x64 ;============================================= 476 477DECLARE_REG 0, rdi, edi, di, dil 478DECLARE_REG 1, rsi, esi, si, sil 479DECLARE_REG 2, rdx, edx, dx, dl 480DECLARE_REG 3, rcx, ecx, cx, cl 481DECLARE_REG 4, R8, R8D, R8W, R8B 482DECLARE_REG 5, R9, R9D, R9W, R9B 483DECLARE_REG 6, rax, eax, ax, al, 8 484DECLARE_REG 7, R10, R10D, R10W, R10B, 16 485DECLARE_REG 8, R11, R11D, R11W, R11B, 24 486DECLARE_REG 9, rbx, ebx, bx, bl, 32 487DECLARE_REG 10, rbp, ebp, bp, bpl, 40 488DECLARE_REG 11, R12, R12D, R12W, R12B, 48 489DECLARE_REG 12, R13, R13D, R13W, R13B, 56 490DECLARE_REG 13, R14, R14D, R14W, R14B, 64 491DECLARE_REG 14, R15, R15D, R15W, R15B, 72 492 493%macro PROLOGUE 2-4+ ; #args, #regs, #xmm_regs, arg_names... 494 %assign num_args %1 495 %assign regs_used %2 496 ASSERT regs_used >= num_args 497 ASSERT regs_used <= 15 498 PUSH_IF_USED 9, 10, 11, 12, 13, 14 499 LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 500 DEFINE_ARGS %4 501%endmacro 502 503%macro RET 0 504 POP_IF_USED 14, 13, 12, 11, 10, 9 505 ret 506%endmacro 507 508%macro REP_RET 0 509 %if regs_used > 9 510 RET 511 %else 512 rep ret 513 %endif 514%endmacro 515 516%else ; X86_32 ;============================================================== 517 518DECLARE_REG 0, eax, eax, ax, al, 4 519DECLARE_REG 1, ecx, ecx, cx, cl, 8 520DECLARE_REG 2, edx, edx, dx, dl, 12 521DECLARE_REG 3, ebx, ebx, bx, bl, 16 522DECLARE_REG 4, esi, esi, si, null, 20 523DECLARE_REG 5, edi, edi, di, null, 24 524DECLARE_REG 6, ebp, ebp, bp, null, 28 525%define rsp esp 526 527%macro DECLARE_ARG 1-* 528 %rep %0 529 %define r%1m [esp + stack_offset + 4*%1 + 4] 530 %define r%1mp dword r%1m 531 %rotate 1 532 %endrep 533%endmacro 534 535DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 536 537%macro PROLOGUE 2-4+ ; #args, #regs, #xmm_regs, arg_names... 538 %assign num_args %1 539 %assign regs_used %2 540 %if regs_used > 7 541 %assign regs_used 7 542 %endif 543 ASSERT regs_used >= num_args 544 PUSH_IF_USED 3, 4, 5, 6 545 LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 546 DEFINE_ARGS %4 547%endmacro 548 549%macro RET 0 550 POP_IF_USED 6, 5, 4, 3 551 ret 552%endmacro 553 554%macro REP_RET 0 555 %if regs_used > 3 556 RET 557 %else 558 rep ret 559 %endif 560%endmacro 561 562%endif ;====================================================================== 563 564%if WIN64 == 0 565%macro WIN64_SPILL_XMM 1 566%endmacro 567%macro WIN64_RESTORE_XMM 1 568%endmacro 569%endif 570 571;============================================================================= 572; arch-independent part 573;============================================================================= 574 575%assign function_align 16 576 577; Begin a function. 578; Applies any symbol mangling needed for C linkage, and sets up a define such that 579; subsequent uses of the function name automatically refer to the mangled version. 580; Appends cpuflags to the function name if cpuflags has been specified. 581%macro cglobal 1-2+ ; name, [PROLOGUE args] 582%if %0 == 1 583 cglobal_internal %1 %+ SUFFIX 584%else 585 cglobal_internal %1 %+ SUFFIX, %2 586%endif 587%endmacro 588%macro cglobal_internal 1-2+ 589 %ifndef cglobaled_%1 590 %xdefine %1 mangle(program_name %+ _ %+ %1) 591 %xdefine %1.skip_prologue %1 %+ .skip_prologue 592 CAT_XDEFINE cglobaled_, %1, 1 593 %endif 594 %xdefine current_function %1 595 %ifidn __OUTPUT_FORMAT__,elf 596 global %1:function hidden 597 %elifidn __OUTPUT_FORMAT__,elf32 598 global %1:function hidden 599 %elifidn __OUTPUT_FORMAT__,elf64 600 global %1:function hidden 601 %elifidn __OUTPUT_FORMAT__,macho32 602 global %1:private_extern 603 %elifidn __OUTPUT_FORMAT__,macho64 604 global %1:private_extern 605 %else 606 global %1 607 %endif 608 align function_align 609 %1: 610 RESET_MM_PERMUTATION ; not really needed, but makes disassembly somewhat nicer 611 %assign stack_offset 0 612 %if %0 > 1 613 PROLOGUE %2 614 %endif 615%endmacro 616 617%macro cextern 1 618 %xdefine %1 mangle(program_name %+ _ %+ %1) 619 CAT_XDEFINE cglobaled_, %1, 1 620 extern %1 621%endmacro 622 623; like cextern, but without the prefix 624%macro cextern_naked 1 625 %xdefine %1 mangle(%1) 626 CAT_XDEFINE cglobaled_, %1, 1 627 extern %1 628%endmacro 629 630%macro const 2+ 631 %xdefine %1 mangle(program_name %+ _ %+ %1) 632 global %1 633 %1: %2 634%endmacro 635 636; This is needed for ELF, otherwise the GNU linker assumes the stack is 637; executable by default. 638%ifidn __OUTPUT_FORMAT__,elf 639SECTION .note.GNU-stack noalloc noexec nowrite progbits 640%elifidn __OUTPUT_FORMAT__,elf32 641SECTION .note.GNU-stack noalloc noexec nowrite progbits 642%elifidn __OUTPUT_FORMAT__,elf64 643SECTION .note.GNU-stack noalloc noexec nowrite progbits 644%endif 645 646; cpuflags 647 648%assign cpuflags_mmx (1<<0) 649%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx 650%assign cpuflags_3dnow (1<<2) | cpuflags_mmx 651%assign cpuflags_3dnow2 (1<<3) | cpuflags_3dnow 652%assign cpuflags_sse (1<<4) | cpuflags_mmx2 653%assign cpuflags_sse2 (1<<5) | cpuflags_sse 654%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 655%assign cpuflags_sse3 (1<<7) | cpuflags_sse2 656%assign cpuflags_ssse3 (1<<8) | cpuflags_sse3 657%assign cpuflags_sse4 (1<<9) | cpuflags_ssse3 658%assign cpuflags_sse42 (1<<10)| cpuflags_sse4 659%assign cpuflags_avx (1<<11)| cpuflags_sse42 660%assign cpuflags_xop (1<<12)| cpuflags_avx 661%assign cpuflags_fma4 (1<<13)| cpuflags_avx 662 663%assign cpuflags_cache32 (1<<16) 664%assign cpuflags_cache64 (1<<17) 665%assign cpuflags_slowctz (1<<18) 666%assign cpuflags_lzcnt (1<<19) 667%assign cpuflags_misalign (1<<20) 668%assign cpuflags_aligned (1<<21) ; not a cpu feature, but a function variant 669%assign cpuflags_atom (1<<22) 670 671%define cpuflag(x) ((cpuflags & (cpuflags_ %+ x)) == (cpuflags_ %+ x)) 672%define notcpuflag(x) ((cpuflags & (cpuflags_ %+ x)) != (cpuflags_ %+ x)) 673 674; Takes up to 2 cpuflags from the above list. 675; All subsequent functions (up to the next INIT_CPUFLAGS) is built for the specified cpu. 676; You shouldn't need to invoke this macro directly, it's a subroutine for INIT_MMX &co. 677%macro INIT_CPUFLAGS 0-2 678 %if %0 >= 1 679 %xdefine cpuname %1 680 %assign cpuflags cpuflags_%1 681 %if %0 >= 2 682 %xdefine cpuname %1_%2 683 %assign cpuflags cpuflags | cpuflags_%2 684 %endif 685 %xdefine SUFFIX _ %+ cpuname 686 %if cpuflag(avx) 687 %assign avx_enabled 1 688 %endif 689 %if mmsize == 16 && notcpuflag(sse2) 690 %define mova movaps 691 %define movu movups 692 %define movnta movntps 693 %endif 694 %if cpuflag(aligned) 695 %define movu mova 696 %elifidn %1, sse3 697 %define movu lddqu 698 %endif 699 %else 700 %xdefine SUFFIX 701 %undef cpuname 702 %undef cpuflags 703 %endif 704%endmacro 705 706; merge mmx and sse* 707 708%macro CAT_XDEFINE 3 709 %xdefine %1%2 %3 710%endmacro 711 712%macro CAT_UNDEF 2 713 %undef %1%2 714%endmacro 715 716%macro INIT_MMX 0-1+ 717 %assign avx_enabled 0 718 %define RESET_MM_PERMUTATION INIT_MMX %1 719 %define mmsize 8 720 %define num_mmregs 8 721 %define mova movq 722 %define movu movq 723 %define movh movd 724 %define movnta movntq 725 %assign %%i 0 726 %rep 8 727 CAT_XDEFINE m, %%i, mm %+ %%i 728 CAT_XDEFINE nmm, %%i, %%i 729 %assign %%i %%i+1 730 %endrep 731 %rep 8 732 CAT_UNDEF m, %%i 733 CAT_UNDEF nmm, %%i 734 %assign %%i %%i+1 735 %endrep 736 INIT_CPUFLAGS %1 737%endmacro 738 739%macro INIT_XMM 0-1+ 740 %assign avx_enabled 0 741 %define RESET_MM_PERMUTATION INIT_XMM %1 742 %define mmsize 16 743 %define num_mmregs 8 744 %if ARCH_X86_64 745 %define num_mmregs 16 746 %endif 747 %define mova movdqa 748 %define movu movdqu 749 %define movh movq 750 %define movnta movntdq 751 %assign %%i 0 752 %rep num_mmregs 753 CAT_XDEFINE m, %%i, xmm %+ %%i 754 CAT_XDEFINE nxmm, %%i, %%i 755 %assign %%i %%i+1 756 %endrep 757 INIT_CPUFLAGS %1 758%endmacro 759 760; FIXME: INIT_AVX can be replaced by INIT_XMM avx 761%macro INIT_AVX 0 762 INIT_XMM 763 %assign avx_enabled 1 764 %define PALIGNR PALIGNR_SSSE3 765 %define RESET_MM_PERMUTATION INIT_AVX 766%endmacro 767 768%macro INIT_YMM 0-1+ 769 %assign avx_enabled 1 770 %define RESET_MM_PERMUTATION INIT_YMM %1 771 %define mmsize 32 772 %define num_mmregs 8 773 %if ARCH_X86_64 774 %define num_mmregs 16 775 %endif 776 %define mova vmovaps 777 %define movu vmovups 778 %undef movh 779 %define movnta vmovntps 780 %assign %%i 0 781 %rep num_mmregs 782 CAT_XDEFINE m, %%i, ymm %+ %%i 783 CAT_XDEFINE nymm, %%i, %%i 784 %assign %%i %%i+1 785 %endrep 786 INIT_CPUFLAGS %1 787%endmacro 788 789INIT_XMM 790 791; I often want to use macros that permute their arguments. e.g. there's no 792; efficient way to implement butterfly or transpose or dct without swapping some 793; arguments. 794; 795; I would like to not have to manually keep track of the permutations: 796; If I insert a permutation in the middle of a function, it should automatically 797; change everything that follows. For more complex macros I may also have multiple 798; implementations, e.g. the SSE2 and SSSE3 versions may have different permutations. 799; 800; Hence these macros. Insert a PERMUTE or some SWAPs at the end of a macro that 801; permutes its arguments. It's equivalent to exchanging the contents of the 802; registers, except that this way you exchange the register names instead, so it 803; doesn't cost any cycles. 804 805%macro PERMUTE 2-* ; takes a list of pairs to swap 806%rep %0/2 807 %xdefine tmp%2 m%2 808 %xdefine ntmp%2 nm%2 809 %rotate 2 810%endrep 811%rep %0/2 812 %xdefine m%1 tmp%2 813 %xdefine nm%1 ntmp%2 814 %undef tmp%2 815 %undef ntmp%2 816 %rotate 2 817%endrep 818%endmacro 819 820%macro SWAP 2-* ; swaps a single chain (sometimes more concise than pairs) 821%rep %0-1 822%ifdef m%1 823 %xdefine tmp m%1 824 %xdefine m%1 m%2 825 %xdefine m%2 tmp 826 CAT_XDEFINE n, m%1, %1 827 CAT_XDEFINE n, m%2, %2 828%else 829 ; If we were called as "SWAP m0,m1" rather than "SWAP 0,1" infer the original numbers here. 830 ; Be careful using this mode in nested macros though, as in some cases there may be 831 ; other copies of m# that have already been dereferenced and don't get updated correctly. 832 %xdefine %%n1 n %+ %1 833 %xdefine %%n2 n %+ %2 834 %xdefine tmp m %+ %%n1 835 CAT_XDEFINE m, %%n1, m %+ %%n2 836 CAT_XDEFINE m, %%n2, tmp 837 CAT_XDEFINE n, m %+ %%n1, %%n1 838 CAT_XDEFINE n, m %+ %%n2, %%n2 839%endif 840 %undef tmp 841 %rotate 1 842%endrep 843%endmacro 844 845; If SAVE_MM_PERMUTATION is placed at the end of a function, then any later 846; calls to that function will automatically load the permutation, so values can 847; be returned in mmregs. 848%macro SAVE_MM_PERMUTATION 0-1 849 %if %0 850 %xdefine %%f %1_m 851 %else 852 %xdefine %%f current_function %+ _m 853 %endif 854 %assign %%i 0 855 %rep num_mmregs 856 CAT_XDEFINE %%f, %%i, m %+ %%i 857 %assign %%i %%i+1 858 %endrep 859%endmacro 860 861%macro LOAD_MM_PERMUTATION 1 ; name to load from 862 %ifdef %1_m0 863 %assign %%i 0 864 %rep num_mmregs 865 CAT_XDEFINE m, %%i, %1_m %+ %%i 866 CAT_XDEFINE n, m %+ %%i, %%i 867 %assign %%i %%i+1 868 %endrep 869 %endif 870%endmacro 871 872; Append cpuflags to the callee's name iff the appended name is known and the plain name isn't 873%macro call 1 874 call_internal %1, %1 %+ SUFFIX 875%endmacro 876%macro call_internal 2 877 %xdefine %%i %1 878 %ifndef cglobaled_%1 879 %ifdef cglobaled_%2 880 %xdefine %%i %2 881 %endif 882 %endif 883 call %%i 884 LOAD_MM_PERMUTATION %%i 885%endmacro 886 887; Substitutions that reduce instruction size but are functionally equivalent 888%macro add 2 889 %ifnum %2 890 %if %2==128 891 sub %1, -128 892 %else 893 add %1, %2 894 %endif 895 %else 896 add %1, %2 897 %endif 898%endmacro 899 900%macro sub 2 901 %ifnum %2 902 %if %2==128 903 add %1, -128 904 %else 905 sub %1, %2 906 %endif 907 %else 908 sub %1, %2 909 %endif 910%endmacro 911 912;============================================================================= 913; AVX abstraction layer 914;============================================================================= 915 916%assign i 0 917%rep 16 918 %if i < 8 919 CAT_XDEFINE sizeofmm, i, 8 920 %endif 921 CAT_XDEFINE sizeofxmm, i, 16 922 CAT_XDEFINE sizeofymm, i, 32 923%assign i i+1 924%endrep 925%undef i 926 927;%1 == instruction 928;%2 == 1 if float, 0 if int 929;%3 == 1 if 4-operand (xmm, xmm, xmm, imm), 0 if 2- or 3-operand (xmm, xmm, xmm) 930;%4 == number of operands given 931;%5+: operands 932%macro RUN_AVX_INSTR 6-7+ 933 %ifid %5 934 %define %%size sizeof%5 935 %else 936 %define %%size mmsize 937 %endif 938 %if %%size==32 939 %if %0 >= 7 940 v%1 %5, %6, %7 941 %else 942 v%1 %5, %6 943 %endif 944 %else 945 %if %%size==8 946 %define %%regmov movq 947 %elif %2 948 %define %%regmov movaps 949 %else 950 %define %%regmov movdqa 951 %endif 952 953 %if %4>=3+%3 954 %ifnidn %5, %6 955 %if avx_enabled && sizeof%5==16 956 v%1 %5, %6, %7 957 %else 958 %%regmov %5, %6 959 %1 %5, %7 960 %endif 961 %else 962 %1 %5, %7 963 %endif 964 %elif %3 965 %1 %5, %6, %7 966 %else 967 %1 %5, %6 968 %endif 969 %endif 970%endmacro 971 972; 3arg AVX ops with a memory arg can only have it in src2, 973; whereas SSE emulation of 3arg prefers to have it in src1 (i.e. the mov). 974; So, if the op is symmetric and the wrong one is memory, swap them. 975%macro RUN_AVX_INSTR1 8 976 %assign %%swap 0 977 %if avx_enabled 978 %ifnid %6 979 %assign %%swap 1 980 %endif 981 %elifnidn %5, %6 982 %ifnid %7 983 %assign %%swap 1 984 %endif 985 %endif 986 %if %%swap && %3 == 0 && %8 == 1 987 RUN_AVX_INSTR %1, %2, %3, %4, %5, %7, %6 988 %else 989 RUN_AVX_INSTR %1, %2, %3, %4, %5, %6, %7 990 %endif 991%endmacro 992 993;%1 == instruction 994;%2 == 1 if float, 0 if int 995;%3 == 1 if 4-operand (xmm, xmm, xmm, imm), 0 if 3-operand (xmm, xmm, xmm) 996;%4 == 1 if symmetric (i.e. doesn't matter which src arg is which), 0 if not 997%macro AVX_INSTR 4 998 %macro %1 2-9 fnord, fnord, fnord, %1, %2, %3, %4 999 %ifidn %3, fnord 1000 RUN_AVX_INSTR %6, %7, %8, 2, %1, %2 1001 %elifidn %4, fnord 1002 RUN_AVX_INSTR1 %6, %7, %8, 3, %1, %2, %3, %9 1003 %elifidn %5, fnord 1004 RUN_AVX_INSTR %6, %7, %8, 4, %1, %2, %3, %4 1005 %else 1006 RUN_AVX_INSTR %6, %7, %8, 5, %1, %2, %3, %4, %5 1007 %endif 1008 %endmacro 1009%endmacro 1010 1011AVX_INSTR addpd, 1, 0, 1 1012AVX_INSTR addps, 1, 0, 1 1013AVX_INSTR addsd, 1, 0, 1 1014AVX_INSTR addss, 1, 0, 1 1015AVX_INSTR addsubpd, 1, 0, 0 1016AVX_INSTR addsubps, 1, 0, 0 1017AVX_INSTR andpd, 1, 0, 1 1018AVX_INSTR andps, 1, 0, 1 1019AVX_INSTR andnpd, 1, 0, 0 1020AVX_INSTR andnps, 1, 0, 0 1021AVX_INSTR blendpd, 1, 0, 0 1022AVX_INSTR blendps, 1, 0, 0 1023AVX_INSTR blendvpd, 1, 0, 0 1024AVX_INSTR blendvps, 1, 0, 0 1025AVX_INSTR cmppd, 1, 0, 0 1026AVX_INSTR cmpps, 1, 0, 0 1027AVX_INSTR cmpsd, 1, 0, 0 1028AVX_INSTR cmpss, 1, 0, 0 1029AVX_INSTR cvtdq2ps, 1, 0, 0 1030AVX_INSTR cvtps2dq, 1, 0, 0 1031AVX_INSTR divpd, 1, 0, 0 1032AVX_INSTR divps, 1, 0, 0 1033AVX_INSTR divsd, 1, 0, 0 1034AVX_INSTR divss, 1, 0, 0 1035AVX_INSTR dppd, 1, 1, 0 1036AVX_INSTR dpps, 1, 1, 0 1037AVX_INSTR haddpd, 1, 0, 0 1038AVX_INSTR haddps, 1, 0, 0 1039AVX_INSTR hsubpd, 1, 0, 0 1040AVX_INSTR hsubps, 1, 0, 0 1041AVX_INSTR maxpd, 1, 0, 1 1042AVX_INSTR maxps, 1, 0, 1 1043AVX_INSTR maxsd, 1, 0, 1 1044AVX_INSTR maxss, 1, 0, 1 1045AVX_INSTR minpd, 1, 0, 1 1046AVX_INSTR minps, 1, 0, 1 1047AVX_INSTR minsd, 1, 0, 1 1048AVX_INSTR minss, 1, 0, 1 1049AVX_INSTR movhlps, 1, 0, 0 1050AVX_INSTR movlhps, 1, 0, 0 1051AVX_INSTR movsd, 1, 0, 0 1052AVX_INSTR movss, 1, 0, 0 1053AVX_INSTR mpsadbw, 0, 1, 0 1054AVX_INSTR mulpd, 1, 0, 1 1055AVX_INSTR mulps, 1, 0, 1 1056AVX_INSTR mulsd, 1, 0, 1 1057AVX_INSTR mulss, 1, 0, 1 1058AVX_INSTR orpd, 1, 0, 1 1059AVX_INSTR orps, 1, 0, 1 1060AVX_INSTR packsswb, 0, 0, 0 1061AVX_INSTR packssdw, 0, 0, 0 1062AVX_INSTR packuswb, 0, 0, 0 1063AVX_INSTR packusdw, 0, 0, 0 1064AVX_INSTR paddb, 0, 0, 1 1065AVX_INSTR paddw, 0, 0, 1 1066AVX_INSTR paddd, 0, 0, 1 1067AVX_INSTR paddq, 0, 0, 1 1068AVX_INSTR paddsb, 0, 0, 1 1069AVX_INSTR paddsw, 0, 0, 1 1070AVX_INSTR paddusb, 0, 0, 1 1071AVX_INSTR paddusw, 0, 0, 1 1072AVX_INSTR palignr, 0, 1, 0 1073AVX_INSTR pand, 0, 0, 1 1074AVX_INSTR pandn, 0, 0, 0 1075AVX_INSTR pavgb, 0, 0, 1 1076AVX_INSTR pavgw, 0, 0, 1 1077AVX_INSTR pblendvb, 0, 0, 0 1078AVX_INSTR pblendw, 0, 1, 0 1079AVX_INSTR pcmpestri, 0, 0, 0 1080AVX_INSTR pcmpestrm, 0, 0, 0 1081AVX_INSTR pcmpistri, 0, 0, 0 1082AVX_INSTR pcmpistrm, 0, 0, 0 1083AVX_INSTR pcmpeqb, 0, 0, 1 1084AVX_INSTR pcmpeqw, 0, 0, 1 1085AVX_INSTR pcmpeqd, 0, 0, 1 1086AVX_INSTR pcmpeqq, 0, 0, 1 1087AVX_INSTR pcmpgtb, 0, 0, 0 1088AVX_INSTR pcmpgtw, 0, 0, 0 1089AVX_INSTR pcmpgtd, 0, 0, 0 1090AVX_INSTR pcmpgtq, 0, 0, 0 1091AVX_INSTR phaddw, 0, 0, 0 1092AVX_INSTR phaddd, 0, 0, 0 1093AVX_INSTR phaddsw, 0, 0, 0 1094AVX_INSTR phsubw, 0, 0, 0 1095AVX_INSTR phsubd, 0, 0, 0 1096AVX_INSTR phsubsw, 0, 0, 0 1097AVX_INSTR pmaddwd, 0, 0, 1 1098AVX_INSTR pmaddubsw, 0, 0, 0 1099AVX_INSTR pmaxsb, 0, 0, 1 1100AVX_INSTR pmaxsw, 0, 0, 1 1101AVX_INSTR pmaxsd, 0, 0, 1 1102AVX_INSTR pmaxub, 0, 0, 1 1103AVX_INSTR pmaxuw, 0, 0, 1 1104AVX_INSTR pmaxud, 0, 0, 1 1105AVX_INSTR pminsb, 0, 0, 1 1106AVX_INSTR pminsw, 0, 0, 1 1107AVX_INSTR pminsd, 0, 0, 1 1108AVX_INSTR pminub, 0, 0, 1 1109AVX_INSTR pminuw, 0, 0, 1 1110AVX_INSTR pminud, 0, 0, 1 1111AVX_INSTR pmulhuw, 0, 0, 1 1112AVX_INSTR pmulhrsw, 0, 0, 1 1113AVX_INSTR pmulhw, 0, 0, 1 1114AVX_INSTR pmullw, 0, 0, 1 1115AVX_INSTR pmulld, 0, 0, 1 1116AVX_INSTR pmuludq, 0, 0, 1 1117AVX_INSTR pmuldq, 0, 0, 1 1118AVX_INSTR por, 0, 0, 1 1119AVX_INSTR psadbw, 0, 0, 1 1120AVX_INSTR pshufb, 0, 0, 0 1121AVX_INSTR psignb, 0, 0, 0 1122AVX_INSTR psignw, 0, 0, 0 1123AVX_INSTR psignd, 0, 0, 0 1124AVX_INSTR psllw, 0, 0, 0 1125AVX_INSTR pslld, 0, 0, 0 1126AVX_INSTR psllq, 0, 0, 0 1127AVX_INSTR pslldq, 0, 0, 0 1128AVX_INSTR psraw, 0, 0, 0 1129AVX_INSTR psrad, 0, 0, 0 1130AVX_INSTR psrlw, 0, 0, 0 1131AVX_INSTR psrld, 0, 0, 0 1132AVX_INSTR psrlq, 0, 0, 0 1133AVX_INSTR psrldq, 0, 0, 0 1134AVX_INSTR psubb, 0, 0, 0 1135AVX_INSTR psubw, 0, 0, 0 1136AVX_INSTR psubd, 0, 0, 0 1137AVX_INSTR psubq, 0, 0, 0 1138AVX_INSTR psubsb, 0, 0, 0 1139AVX_INSTR psubsw, 0, 0, 0 1140AVX_INSTR psubusb, 0, 0, 0 1141AVX_INSTR psubusw, 0, 0, 0 1142AVX_INSTR punpckhbw, 0, 0, 0 1143AVX_INSTR punpckhwd, 0, 0, 0 1144AVX_INSTR punpckhdq, 0, 0, 0 1145AVX_INSTR punpckhqdq, 0, 0, 0 1146AVX_INSTR punpcklbw, 0, 0, 0 1147AVX_INSTR punpcklwd, 0, 0, 0 1148AVX_INSTR punpckldq, 0, 0, 0 1149AVX_INSTR punpcklqdq, 0, 0, 0 1150AVX_INSTR pxor, 0, 0, 1 1151AVX_INSTR shufps, 1, 1, 0 1152AVX_INSTR subpd, 1, 0, 0 1153AVX_INSTR subps, 1, 0, 0 1154AVX_INSTR subsd, 1, 0, 0 1155AVX_INSTR subss, 1, 0, 0 1156AVX_INSTR unpckhpd, 1, 0, 0 1157AVX_INSTR unpckhps, 1, 0, 0 1158AVX_INSTR unpcklpd, 1, 0, 0 1159AVX_INSTR unpcklps, 1, 0, 0 1160AVX_INSTR xorpd, 1, 0, 1 1161AVX_INSTR xorps, 1, 0, 1 1162 1163; 3DNow instructions, for sharing code between AVX, SSE and 3DN 1164AVX_INSTR pfadd, 1, 0, 1 1165AVX_INSTR pfsub, 1, 0, 0 1166AVX_INSTR pfmul, 1, 0, 1 1167 1168; base-4 constants for shuffles 1169%assign i 0 1170%rep 256 1171 %assign j ((i>>6)&3)*1000 + ((i>>4)&3)*100 + ((i>>2)&3)*10 + (i&3) 1172 %if j < 10 1173 CAT_XDEFINE q000, j, i 1174 %elif j < 100 1175 CAT_XDEFINE q00, j, i 1176 %elif j < 1000 1177 CAT_XDEFINE q0, j, i 1178 %else 1179 CAT_XDEFINE q, j, i 1180 %endif 1181%assign i i+1 1182%endrep 1183%undef i 1184%undef j 1185 1186%macro FMA_INSTR 3 1187 %macro %1 4-7 %1, %2, %3 1188 %if cpuflag(xop) 1189 v%5 %1, %2, %3, %4 1190 %else 1191 %6 %1, %2, %3 1192 %7 %1, %4 1193 %endif 1194 %endmacro 1195%endmacro 1196 1197FMA_INSTR pmacsdd, pmulld, paddd 1198FMA_INSTR pmacsww, pmullw, paddw 1199FMA_INSTR pmadcswd, pmaddwd, paddd 1200