armv4-mont.S revision e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5
1d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#include "arm_arch.h" 2d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 3d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.text 4d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.code 32 5d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 6e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley#if __ARM_MAX_ARCH__>=7 7d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.align 5 8d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.LOPENSSL_armcap: 9e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley.word OPENSSL_armcap_P-.Lbn_mul_mont 10d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#endif 11d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 12e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley.globl bn_mul_mont 13d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.hidden bn_mul_mont 14d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.type bn_mul_mont,%function 15d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 16d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.align 5 17d9e397b599b13d642138480a28c14db7a136bf0Adam Langleybn_mul_mont: 18e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley.Lbn_mul_mont: 19d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr ip,[sp,#4] @ load num 20d9e397b599b13d642138480a28c14db7a136bf0Adam Langley stmdb sp!,{r0,r2} @ sp points at argument block 21e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley#if __ARM_MAX_ARCH__>=7 22d9e397b599b13d642138480a28c14db7a136bf0Adam Langley tst ip,#7 23d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .Lialu 24d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adr r0,bn_mul_mont 25d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r2,.LOPENSSL_armcap 26d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r0,[r0,r2] 27e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley#ifdef __APPLE__ 28e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldr r0,[r0] 29e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley#endif 30d9e397b599b13d642138480a28c14db7a136bf0Adam Langley tst r0,#1 @ NEON available? 31d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldmia sp, {r0,r2} 32d9e397b599b13d642138480a28c14db7a136bf0Adam Langley beq .Lialu 33d9e397b599b13d642138480a28c14db7a136bf0Adam Langley add sp,sp,#8 34d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b bn_mul8x_mont_neon 35d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.align 4 36d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.Lialu: 37d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#endif 38d9e397b599b13d642138480a28c14db7a136bf0Adam Langley cmp ip,#2 39d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r0,ip @ load num 40d9e397b599b13d642138480a28c14db7a136bf0Adam Langley movlt r0,#0 41d9e397b599b13d642138480a28c14db7a136bf0Adam Langley addlt sp,sp,#2*4 42d9e397b599b13d642138480a28c14db7a136bf0Adam Langley blt .Labrt 43d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 44e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley stmdb sp!,{r4,r5,r6,r7,r8,r9,r10,r11,r12,lr} @ save 10 registers 45d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 46d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r0,r0,lsl#2 @ rescale r0 for byte count 47d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub sp,sp,r0 @ alloca(4*num) 48d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub sp,sp,#4 @ +extra dword 49d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r0,r0,#4 @ "num=num-1" 50d9e397b599b13d642138480a28c14db7a136bf0Adam Langley add r4,r2,r0 @ &bp[num-1] 51d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 52d9e397b599b13d642138480a28c14db7a136bf0Adam Langley add r0,sp,r0 @ r0 to point at &tp[num-1] 53d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r8,[r0,#14*4] @ &n0 54d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r2,[r2] @ bp[0] 55d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r5,[r1],#4 @ ap[0],ap++ 56d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r6,[r3],#4 @ np[0],np++ 57d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r8,[r8] @ *n0 58d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r4,[r0,#15*4] @ save &bp[num] 59d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 60d9e397b599b13d642138480a28c14db7a136bf0Adam Langley umull r10,r11,r5,r2 @ ap[0]*bp[0] 61d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r8,[r0,#14*4] @ save n0 value 62d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul r8,r10,r8 @ "tp[0]"*n0 63d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r12,#0 64d9e397b599b13d642138480a28c14db7a136bf0Adam Langley umlal r10,r12,r6,r8 @ np[0]*n0+"t[0]" 65d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r4,sp 66d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 67d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.L1st: 68d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r5,[r1],#4 @ ap[j],ap++ 69d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r10,r11 70d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r6,[r3],#4 @ np[j],np++ 71d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r11,#0 72d9e397b599b13d642138480a28c14db7a136bf0Adam Langley umlal r10,r11,r5,r2 @ ap[j]*bp[0] 73d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r14,#0 74d9e397b599b13d642138480a28c14db7a136bf0Adam Langley umlal r12,r14,r6,r8 @ np[j]*n0 75d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adds r12,r12,r10 76d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r12,[r4],#4 @ tp[j-1]=,tp++ 77d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adc r12,r14,#0 78d9e397b599b13d642138480a28c14db7a136bf0Adam Langley cmp r4,r0 79d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .L1st 80d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 81d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adds r12,r12,r11 82d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r4,[r0,#13*4] @ restore bp 83d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r14,#0 84d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r8,[r0,#14*4] @ restore n0 85d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adc r14,r14,#0 86d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r12,[r0] @ tp[num-1]= 87d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r14,[r0,#4] @ tp[num]= 88e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley 89d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.Louter: 90d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r7,r0,sp @ "original" r0-1 value 91d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r1,r1,r7 @ "rewind" ap to &ap[1] 92d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r2,[r4,#4]! @ *(++bp) 93d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r3,r3,r7 @ "rewind" np to &np[1] 94d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r5,[r1,#-4] @ ap[0] 95d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r10,[sp] @ tp[0] 96d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r6,[r3,#-4] @ np[0] 97d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r7,[sp,#4] @ tp[1] 98d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 99d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r11,#0 100d9e397b599b13d642138480a28c14db7a136bf0Adam Langley umlal r10,r11,r5,r2 @ ap[0]*bp[i]+tp[0] 101d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r4,[r0,#13*4] @ save bp 102d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mul r8,r10,r8 103d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r12,#0 104d9e397b599b13d642138480a28c14db7a136bf0Adam Langley umlal r10,r12,r6,r8 @ np[0]*n0+"tp[0]" 105d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r4,sp 106d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 107d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.Linner: 108d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r5,[r1],#4 @ ap[j],ap++ 109d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adds r10,r11,r7 @ +=tp[j] 110d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r6,[r3],#4 @ np[j],np++ 111d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r11,#0 112d9e397b599b13d642138480a28c14db7a136bf0Adam Langley umlal r10,r11,r5,r2 @ ap[j]*bp[i] 113d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r14,#0 114d9e397b599b13d642138480a28c14db7a136bf0Adam Langley umlal r12,r14,r6,r8 @ np[j]*n0 115d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adc r11,r11,#0 116d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r7,[r4,#8] @ tp[j+1] 117d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adds r12,r12,r10 118d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r12,[r4],#4 @ tp[j-1]=,tp++ 119d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adc r12,r14,#0 120d9e397b599b13d642138480a28c14db7a136bf0Adam Langley cmp r4,r0 121d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .Linner 122d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 123d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adds r12,r12,r11 124d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r14,#0 125d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r4,[r0,#13*4] @ restore bp 126d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adc r14,r14,#0 127d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r8,[r0,#14*4] @ restore n0 128d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adds r12,r12,r7 129d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r7,[r0,#15*4] @ restore &bp[num] 130d9e397b599b13d642138480a28c14db7a136bf0Adam Langley adc r14,r14,#0 131d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r12,[r0] @ tp[num-1]= 132d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r14,[r0,#4] @ tp[num]= 133d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 134d9e397b599b13d642138480a28c14db7a136bf0Adam Langley cmp r4,r7 135d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .Louter 136e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley 137d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r2,[r0,#12*4] @ pull rp 138d9e397b599b13d642138480a28c14db7a136bf0Adam Langley add r0,r0,#4 @ r0 to point at &tp[num] 139d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r5,r0,sp @ "original" num value 140d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r4,sp @ "rewind" r4 141d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r1,r4 @ "borrow" r1 142d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r3,r3,r5 @ "rewind" r3 to &np[0] 143d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 144d9e397b599b13d642138480a28c14db7a136bf0Adam Langley subs r7,r7,r7 @ "clear" carry flag 145d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.Lsub: ldr r7,[r4],#4 146d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r6,[r3],#4 147d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sbcs r7,r7,r6 @ tp[j]-np[j] 148d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r7,[r2],#4 @ rp[j]= 149d9e397b599b13d642138480a28c14db7a136bf0Adam Langley teq r4,r0 @ preserve carry 150d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .Lsub 151d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sbcs r14,r14,#0 @ upmost carry 152d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r4,sp @ "rewind" r4 153d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r2,r2,r5 @ "rewind" r2 154d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 155d9e397b599b13d642138480a28c14db7a136bf0Adam Langley and r1,r4,r14 156d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bic r3,r2,r14 157d9e397b599b13d642138480a28c14db7a136bf0Adam Langley orr r1,r1,r3 @ ap=borrow?tp:rp 158d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 159d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.Lcopy: ldr r7,[r1],#4 @ copy or in-place refresh 160d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str sp,[r4],#4 @ zap tp 161d9e397b599b13d642138480a28c14db7a136bf0Adam Langley str r7,[r2],#4 162d9e397b599b13d642138480a28c14db7a136bf0Adam Langley cmp r4,r0 163d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .Lcopy 164d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 165d9e397b599b13d642138480a28c14db7a136bf0Adam Langley add sp,r0,#4 @ skip over tp[num+1] 166e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldmia sp!,{r4,r5,r6,r7,r8,r9,r10,r11,r12,lr} @ restore registers 167d9e397b599b13d642138480a28c14db7a136bf0Adam Langley add sp,sp,#2*4 @ skip over {r0,r2} 168d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r0,#1 169e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley.Labrt: 170e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley#if __ARM_ARCH__>=5 171e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley bx lr @ .word 0xe12fff1e 172e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley#else 173e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley tst lr,#1 174d9e397b599b13d642138480a28c14db7a136bf0Adam Langley moveq pc,lr @ be binary compatible with V4, yet 175e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley.word 0xe12fff1e @ interoperable with Thumb ISA:-) 176e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley#endif 177d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.size bn_mul_mont,.-bn_mul_mont 178e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley#if __ARM_MAX_ARCH__>=7 179e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley.arch armv7-a 180d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.fpu neon 181d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 182d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.type bn_mul8x_mont_neon,%function 183d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.align 5 184d9e397b599b13d642138480a28c14db7a136bf0Adam Langleybn_mul8x_mont_neon: 185d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov ip,sp 186e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley stmdb sp!,{r4,r5,r6,r7,r8,r9,r10,r11} 187e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vstmdb sp!,{d8,d9,d10,d11,d12,d13,d14,d15} @ ABI specification says so 188e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldmia ip,{r4,r5} @ load rest of parameter block 189e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley 190e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley sub r7,sp,#16 191e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d28[0]}, [r2,:32]! 192e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley sub r7,r7,r5,lsl#4 193e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d0,d1,d2,d3}, [r1]! @ can't specify :32 :-( 194e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley and r7,r7,#-64 195e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d30[0]}, [r4,:32] 196e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley mov sp,r7 @ alloca 197e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley veor d8,d8,d8 198e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley subs r8,r5,#8 199e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d28,d8 200d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 201d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q6,d28,d0[0] 202d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q7,d28,d0[1] 203d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q8,d28,d1[0] 204d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshl.i64 d10,d13,#16 205d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q9,d28,d1[1] 206d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 207d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d10,d10,d12 208e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley veor d8,d8,d8 209d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmul.u32 d29,d10,d30 210d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 211d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q10,d28,d2[0] 212e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d4,d5,d6,d7}, [r3]! 213d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q11,d28,d2[1] 214d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q12,d28,d3[0] 215e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d29,d8 216d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q13,d28,d3[1] 217d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 218d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .LNEON_1st 219d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 220d9e397b599b13d642138480a28c14db7a136bf0Adam Langley @ special case for num=8, everything is in register bank... 221d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 222d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q6,d29,d4[0] 223e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley sub r9,r5,#1 224d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q7,d29,d4[1] 225d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q8,d29,d5[0] 226d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q9,d29,d5[1] 227d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 228d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q10,d29,d6[0] 229e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q5,q6 230d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q11,d29,d6[1] 231e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q6,q7 232d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q12,d29,d7[0] 233e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q7,q8 234d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q13,d29,d7[1] 235e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q8,q9 236e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q9,q10 237d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d10,#16 238e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q10,q11 239e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q11,q12 240d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d10,d10,d11 241e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q12,q13 242e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley veor q13,q13 243d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d10,#16 244d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 245d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b .LNEON_outer8 246d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 247d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.align 4 248d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.LNEON_outer8: 249e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d28[0]}, [r2,:32]! 250e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley veor d8,d8,d8 251e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d28,d8 252d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d12,d12,d10 253d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 254d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q6,d28,d0[0] 255d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q7,d28,d0[1] 256d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q8,d28,d1[0] 257d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshl.i64 d10,d13,#16 258d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q9,d28,d1[1] 259d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 260d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d10,d10,d12 261e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley veor d8,d8,d8 262e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley subs r9,r9,#1 263d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmul.u32 d29,d10,d30 264d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 265d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q10,d28,d2[0] 266d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q11,d28,d2[1] 267d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q12,d28,d3[0] 268e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d29,d8 269d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q13,d28,d3[1] 270d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 271d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q6,d29,d4[0] 272d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q7,d29,d4[1] 273d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q8,d29,d5[0] 274d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q9,d29,d5[1] 275d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 276d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q10,d29,d6[0] 277e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q5,q6 278d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q11,d29,d6[1] 279e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q6,q7 280d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q12,d29,d7[0] 281e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q7,q8 282d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q13,d29,d7[1] 283e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q8,q9 284e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q9,q10 285d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d10,#16 286e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q10,q11 287e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q11,q12 288d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d10,d10,d11 289e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vmov q12,q13 290e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley veor q13,q13 291d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d10,#16 292d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 293d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .LNEON_outer8 294d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 295d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d12,d12,d10 296e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley mov r7,sp 297d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d12,#16 298e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley mov r8,r5 299d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d13,d13,d10 300e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley add r6,sp,#16 301d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d13,#16 302e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d12,d13 303d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 304d9e397b599b13d642138480a28c14db7a136bf0Adam Langley b .LNEON_tail2 305d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 306d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.align 4 307d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.LNEON_1st: 308d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q6,d29,d4[0] 309e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d0,d1,d2,d3}, [r1]! 310d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q7,d29,d4[1] 311e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley subs r8,r8,#8 312d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q8,d29,d5[0] 313d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q9,d29,d5[1] 314d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 315d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q10,d29,d6[0] 316e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d4,d5}, [r3]! 317d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q11,d29,d6[1] 318e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q6,q7}, [r7,:256]! 319d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q12,d29,d7[0] 320d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q13,d29,d7[1] 321e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q8,q9}, [r7,:256]! 322d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 323d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q6,d28,d0[0] 324e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d6,d7}, [r3]! 325d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q7,d28,d0[1] 326e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q10,q11}, [r7,:256]! 327d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q8,d28,d1[0] 328d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q9,d28,d1[1] 329e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q12,q13}, [r7,:256]! 330d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 331d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q10,d28,d2[0] 332d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q11,d28,d2[1] 333d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q12,d28,d3[0] 334d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmull.u32 q13,d28,d3[1] 335d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 336d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .LNEON_1st 337d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 338d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q6,d29,d4[0] 339e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley add r6,sp,#16 340d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q7,d29,d4[1] 341e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley sub r1,r1,r5,lsl#2 @ rewind r1 342d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q8,d29,d5[0] 343e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q5}, [sp,:128] 344d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q9,d29,d5[1] 345e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley sub r9,r5,#1 346d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 347d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q10,d29,d6[0] 348e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q6,q7}, [r7,:256]! 349d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q11,d29,d6[1] 350d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d10,#16 351e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q6}, [r6, :128]! 352d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q12,d29,d7[0] 353e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q8,q9}, [r7,:256]! 354d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q13,d29,d7[1] 355d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 356e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q10,q11}, [r7,:256]! 357d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d10,d10,d11 358e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley veor q4,q4,q4 359e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q12,q13}, [r7,:256]! 360e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q7,q8}, [r6, :256]! 361e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q4}, [r7,:128] 362d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d10,#16 363d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 364e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley b .LNEON_outer 365d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 366d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.align 4 367d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.LNEON_outer: 368e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d28[0]}, [r2,:32]! 369e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley sub r3,r3,r5,lsl#2 @ rewind r3 370e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d0,d1,d2,d3}, [r1]! 371e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley veor d8,d8,d8 372e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley mov r7,sp 373e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d28,d8 374e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley sub r8,r5,#8 375d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d12,d12,d10 376d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 377d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q6,d28,d0[0] 378e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q9,q10},[r6,:256]! 379d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q7,d28,d0[1] 380d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q8,d28,d1[0] 381e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q11,q12},[r6,:256]! 382d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q9,d28,d1[1] 383d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 384d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshl.i64 d10,d13,#16 385e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley veor d8,d8,d8 386d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d10,d10,d12 387e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q13},[r6,:128]! 388d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmul.u32 d29,d10,d30 389d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 390d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q10,d28,d2[0] 391e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d4,d5,d6,d7}, [r3]! 392d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q11,d28,d2[1] 393d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q12,d28,d3[0] 394e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d29,d8 395d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q13,d28,d3[1] 396d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 397d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.LNEON_inner: 398d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q6,d29,d4[0] 399e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d0,d1,d2,d3}, [r1]! 400d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q7,d29,d4[1] 401e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley subs r8,r8,#8 402d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q8,d29,d5[0] 403d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q9,d29,d5[1] 404e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q6,q7}, [r7,:256]! 405d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 406d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q10,d29,d6[0] 407e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q6}, [r6, :128]! 408d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q11,d29,d6[1] 409e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q8,q9}, [r7,:256]! 410d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q12,d29,d7[0] 411e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q7,q8}, [r6, :256]! 412d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q13,d29,d7[1] 413e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q10,q11}, [r7,:256]! 414d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 415d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q6,d28,d0[0] 416e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q9,q10}, [r6, :256]! 417d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q7,d28,d0[1] 418e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q12,q13}, [r7,:256]! 419d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q8,d28,d1[0] 420e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q11,q12}, [r6, :256]! 421d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q9,d28,d1[1] 422e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.32 {d4,d5,d6,d7}, [r3]! 423d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 424d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q10,d28,d2[0] 425e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q13}, [r6, :128]! 426d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q11,d28,d2[1] 427d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q12,d28,d3[0] 428d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q13,d28,d3[1] 429d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 430d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .LNEON_inner 431d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 432d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q6,d29,d4[0] 433e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley add r6,sp,#16 434d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q7,d29,d4[1] 435e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley sub r1,r1,r5,lsl#2 @ rewind r1 436d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q8,d29,d5[0] 437e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q5}, [sp,:128] 438d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q9,d29,d5[1] 439e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley subs r9,r9,#1 440d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 441d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q10,d29,d6[0] 442e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q6,q7}, [r7,:256]! 443d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q11,d29,d6[1] 444e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q6}, [r6, :128]! 445d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d10,#16 446e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q8,q9}, [r7,:256]! 447d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q12,d29,d7[0] 448e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q7,q8}, [r6, :256]! 449d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vmlal.u32 q13,d29,d7[1] 450d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 451e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q10,q11}, [r7,:256]! 452d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d10,d10,d11 453e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q12,q13}, [r7,:256]! 454d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d10,#16 455d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 456d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .LNEON_outer 457d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 458e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley mov r7,sp 459e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley mov r8,r5 460d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 461d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.LNEON_tail: 462d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d12,d12,d10 463e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q9,q10}, [r6, :256]! 464d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d12,#16 465d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d13,d13,d10 466e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q11,q12}, [r6, :256]! 467d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d13,#16 468e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q13}, [r6, :128]! 469e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d12,d13 470d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 471d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.LNEON_tail2: 472d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d14,d14,d10 473e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.32 {d12[0]}, [r7, :32]! 474d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d14,#16 475d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d15,d15,d10 476d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d15,#16 477e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d14,d15 478d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 479d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d16,d16,d10 480e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.32 {d14[0]}, [r7, :32]! 481d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d16,#16 482d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d17,d17,d10 483d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d17,#16 484e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d16,d17 485d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 486d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d18,d18,d10 487e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.32 {d16[0]}, [r7, :32]! 488d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d18,#16 489d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d19,d19,d10 490d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d19,#16 491e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d18,d19 492d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 493d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d20,d20,d10 494e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.32 {d18[0]}, [r7, :32]! 495d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d20,#16 496d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d21,d21,d10 497d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d21,#16 498e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d20,d21 499d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 500d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d22,d22,d10 501e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.32 {d20[0]}, [r7, :32]! 502d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d22,#16 503d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d23,d23,d10 504d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d23,#16 505e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d22,d23 506d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 507d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d24,d24,d10 508e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.32 {d22[0]}, [r7, :32]! 509d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d24,#16 510d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d25,d25,d10 511e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q6}, [r6, :128]! 512d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d25,#16 513e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d24,d25 514d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 515d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d26,d26,d10 516e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.32 {d24[0]}, [r7, :32]! 517d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d26,#16 518d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vadd.u64 d27,d27,d10 519e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vld1.64 {q7,q8}, [r6, :256]! 520d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vshr.u64 d10,d27,#16 521e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vzip.16 d26,d27 522e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley subs r8,r8,#8 523e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.32 {d26[0]}, [r7, :32]! 524d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 525d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .LNEON_tail 526d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 527d9e397b599b13d642138480a28c14db7a136bf0Adam Langley vst1.32 {d10[0]}, [r7, :32] @ top-most bit 528d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r3,r3,r5,lsl#2 @ rewind r3 529d9e397b599b13d642138480a28c14db7a136bf0Adam Langley subs r1,sp,#0 @ clear carry flag 530d9e397b599b13d642138480a28c14db7a136bf0Adam Langley add r2,sp,r5,lsl#2 531d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 532d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.LNEON_sub: 533e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldmia r1!, {r4,r5,r6,r7} 534e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldmia r3!, {r8,r9,r10,r11} 535d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sbcs r8, r4,r8 536d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sbcs r9, r5,r9 537d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sbcs r10,r6,r10 538d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sbcs r11,r7,r11 539d9e397b599b13d642138480a28c14db7a136bf0Adam Langley teq r1,r2 @ preserves carry 540e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley stmia r0!, {r8,r9,r10,r11} 541d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .LNEON_sub 542d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 543d9e397b599b13d642138480a28c14db7a136bf0Adam Langley ldr r10, [r1] @ load top-most bit 544d9e397b599b13d642138480a28c14db7a136bf0Adam Langley veor q0,q0,q0 545d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r11,r2,sp @ this is num*4 546d9e397b599b13d642138480a28c14db7a136bf0Adam Langley veor q1,q1,q1 547d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r1,sp 548d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r0,r0,r11 @ rewind r0 549d9e397b599b13d642138480a28c14db7a136bf0Adam Langley mov r3,r2 @ second 3/4th of frame 550d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sbcs r10,r10,#0 @ result is carry flag 551d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 552d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.LNEON_copy_n_zap: 553e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldmia r1!, {r4,r5,r6,r7} 554e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldmia r0, {r8,r9,r10,r11} 555d9e397b599b13d642138480a28c14db7a136bf0Adam Langley movcc r8, r4 556e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q0,q1}, [r3,:256]! @ wipe 557d9e397b599b13d642138480a28c14db7a136bf0Adam Langley movcc r9, r5 558d9e397b599b13d642138480a28c14db7a136bf0Adam Langley movcc r10,r6 559e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q0,q1}, [r3,:256]! @ wipe 560d9e397b599b13d642138480a28c14db7a136bf0Adam Langley movcc r11,r7 561e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldmia r1, {r4,r5,r6,r7} 562e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley stmia r0!, {r8,r9,r10,r11} 563d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub r1,r1,#16 564e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldmia r0, {r8,r9,r10,r11} 565d9e397b599b13d642138480a28c14db7a136bf0Adam Langley movcc r8, r4 566e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q0,q1}, [r1,:256]! @ wipe 567d9e397b599b13d642138480a28c14db7a136bf0Adam Langley movcc r9, r5 568d9e397b599b13d642138480a28c14db7a136bf0Adam Langley movcc r10,r6 569e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vst1.64 {q0,q1}, [r3,:256]! @ wipe 570d9e397b599b13d642138480a28c14db7a136bf0Adam Langley movcc r11,r7 571d9e397b599b13d642138480a28c14db7a136bf0Adam Langley teq r1,r2 @ preserves carry 572e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley stmia r0!, {r8,r9,r10,r11} 573d9e397b599b13d642138480a28c14db7a136bf0Adam Langley bne .LNEON_copy_n_zap 574d9e397b599b13d642138480a28c14db7a136bf0Adam Langley 575d9e397b599b13d642138480a28c14db7a136bf0Adam Langley sub sp,ip,#96 576e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley vldmia sp!,{d8,d9,d10,d11,d12,d13,d14,d15} 577e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley ldmia sp!,{r4,r5,r6,r7,r8,r9,r10,r11} 578e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley bx lr @ .word 0xe12fff1e 579d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.size bn_mul8x_mont_neon,.-bn_mul8x_mont_neon 580d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#endif 581e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley.byte 77,111,110,116,103,111,109,101,114,121,32,109,117,108,116,105,112,108,105,99,97,116,105,111,110,32,102,111,114,32,65,82,77,118,52,47,78,69,79,78,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 582e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley.align 2 583d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.align 2 584e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley#if __ARM_MAX_ARCH__>=7 585d9e397b599b13d642138480a28c14db7a136bf0Adam Langley.comm OPENSSL_armcap_P,4,4 586e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley.hidden OPENSSL_armcap_P 587d9e397b599b13d642138480a28c14db7a136bf0Adam Langley#endif 588