1e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleydefault	rel
2e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley%define XMMWORD
3e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley%define YMMWORD
4e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley%define ZMMWORD
5e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleysection	.text code align=64
6d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
7d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
8e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam LangleyEXTERN	OPENSSL_ia32cap_P
9e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley
10e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleyglobal	bn_mul_mont
11d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
12d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
13e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleybn_mul_mont:
14e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[8+rsp],rdi	;WIN64 prologue
15e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[16+rsp],rsi
16d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rax,rsp
17e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$SEH_begin_bn_mul_mont:
18d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rcx
19d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rsi,rdx
20d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdx,r8
21d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rcx,r9
22e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r8,QWORD[40+rsp]
23e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r9,QWORD[48+rsp]
24d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
25d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
26a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
27a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r9d,r9d
28a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rax,rsp
29a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
30d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	test	r9d,3
31e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jnz	NEAR $L$mul_enter
32d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	r9d,8
33e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jb	NEAR $L$mul_enter
34d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	rdx,rsi
35e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jne	NEAR $L$mul4x_enter
36d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	test	r9d,7
37e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jz	NEAR $L$sqr8x_enter
38e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$mul4x_enter
39d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
40d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
41e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$mul_enter:
42d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbx
43a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
44d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbp
45a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
46d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r12
47a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
48d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r13
49a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
50d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r14
51a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
52d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r15
53d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
54a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
55a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	neg	r9
56d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rsp
57a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	r10,[((-16))+r9*8+rsp]
58a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	neg	r9
59a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	r10,-1024
60a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
61a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
62a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
63a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
64a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
65a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
66a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
67a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
68a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
69a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	sub	r11,r10
70a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	r11,-4096
71a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rsp,[r11*1+r10]
72a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r11,QWORD[rsp]
73a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	cmp	rsp,r10
74a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	ja	NEAR $L$mul_page_walk
75a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	jmp	NEAR $L$mul_page_walk_done
76a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
77a94fe0531b3c196ad078174259af2201b2e3a246Robert SloanALIGN	16
78a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan$L$mul_page_walk:
79a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rsp,[((-4096))+rsp]
80a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r11,QWORD[rsp]
81a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	cmp	rsp,r10
82a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	ja	NEAR $L$mul_page_walk
83a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan$L$mul_page_walk_done:
84a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
85a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	QWORD[8+r9*8+rsp],rax
86d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
87e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$mul_body:
88d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r12,rdx
89e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r8,QWORD[r8]
90e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[r12]
91e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rsi]
92d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
93d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r14,r14
94d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r15,r15
95d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
96d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rbp,r8
97d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
98d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rax
99e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rcx]
100d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
101d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	imul	rbp,r10
102d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
103d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
104d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
105d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
106e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+rsi]
107d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
108d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
109d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
110e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r15,[1+r15]
111e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$1st_enter
112d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
113d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
114e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$1st:
115d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
116e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[r15*8+rsi]
117d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
118d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r11
119d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,r10
120d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
121e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-16))+r15*8+rsp],r13
122d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
123d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
124e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$1st_enter:
125d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
126d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
127e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[r15*8+rcx]
128d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
129e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r15,[1+r15]
130d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rdx
131d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
132d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
133d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	r15,r9
134e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jne	NEAR $L$1st
135d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
136d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
137e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rsi]
138d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
139d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r11
140d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
141e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-16))+r15*8+rsp],r13
142d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
143d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,r10
144d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
145d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	rdx,rdx
146d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r11
147d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
148e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-8))+r9*8+rsp],r13
149e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[r9*8+rsp],rdx
150d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
151e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r14,[1+r14]
152e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$outer
153d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
154e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$outer:
155e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[r14*8+r12]
156d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r15,r15
157d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rbp,r8
158e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10,QWORD[rsp]
159d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
160d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
161e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rcx]
162d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
163d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
164d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	imul	rbp,r10
165d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
166d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
167d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
168d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
169e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+rsi]
170d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
171e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10,QWORD[8+rsp]
172d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
173d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
174e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r15,[1+r15]
175e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$inner_enter
176d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
177d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
178e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$inner:
179d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
180e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[r15*8+rsi]
181d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
182d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
183e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10,QWORD[r15*8+rsp]
184d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
185e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-16))+r15*8+rsp],r13
186d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
187d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
188e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$inner_enter:
189d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
190d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
191e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[r15*8+rcx]
192d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
193d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,r11
194d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
195d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	r11,0
196e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r15,[1+r15]
197d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
198d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
199d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	r15,r9
200e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jne	NEAR $L$inner
201d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
202d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
203e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rsi]
204d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
205d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
206e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10,QWORD[r15*8+rsp]
207d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
208e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-16))+r15*8+rsp],r13
209d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
210d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
211d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	rdx,rdx
212d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r11
213d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
214d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
215d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
216e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-8))+r9*8+rsp],r13
217e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[r9*8+rsp],rdx
218d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
219e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r14,[1+r14]
220d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	r14,r9
221e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jb	NEAR $L$outer
222d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
223d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r14,r14
224e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rsp]
225e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	rsi,[rsp]
226d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r15,r9
227e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$sub
228d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
2298ff035535f7cf2903f02bbe94d2fa10b7ab855f1Robert Sloan$L$sub:
2308ff035535f7cf2903f02bbe94d2fa10b7ab855f1Robert Sloan	sbb	rax,QWORD[r14*8+rcx]
231e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[r14*8+rdi],rax
232e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+r14*8+rsi]
233e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r14,[1+r14]
234d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	dec	r15
235e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jnz	NEAR $L$sub
236d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
237d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	sbb	rax,0
238d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r14,r14
239a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	rsi,rax
240a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	not	rax
241a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rcx,rdi
242a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	rcx,rax
243d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r15,r9
244a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	or	rsi,rcx
245d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
246e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$copy:
247a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rax,QWORD[r14*8+rsi]
248e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[r14*8+rsp],r14
249a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	QWORD[r14*8+rdi],rax
250e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r14,[1+r14]
251d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	sub	r15,1
252e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jnz	NEAR $L$copy
253d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
254e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rsi,QWORD[8+r9*8+rsp]
255a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
256d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rax,1
257a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r15,QWORD[((-48))+rsi]
258a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
259a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r14,QWORD[((-40))+rsi]
260a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
261a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r13,QWORD[((-32))+rsi]
262a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
263a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r12,QWORD[((-24))+rsi]
264a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
265a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rbp,QWORD[((-16))+rsi]
266a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
267a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rbx,QWORD[((-8))+rsi]
268a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
269a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rsp,[rsi]
270a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
271e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$mul_epilogue:
272e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
273e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rsi,QWORD[16+rsp]
274d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	DB	0F3h,0C3h		;repret
275a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
276e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$SEH_end_bn_mul_mont:
277d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
278d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
279e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleybn_mul4x_mont:
280e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[8+rsp],rdi	;WIN64 prologue
281e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[16+rsp],rsi
282d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rax,rsp
283e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$SEH_begin_bn_mul4x_mont:
284d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rcx
285d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rsi,rdx
286d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdx,r8
287d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rcx,r9
288e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r8,QWORD[40+rsp]
289e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r9,QWORD[48+rsp]
290d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
291d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
292a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
293a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r9d,r9d
294a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rax,rsp
295a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
296e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$mul4x_enter:
297d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbx
298a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
299d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbp
300a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
301d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r12
302a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
303d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r13
304a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
305d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r14
306a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
307d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r15
308d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
309a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
310a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	neg	r9
311d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rsp
312a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	r10,[((-32))+r9*8+rsp]
313a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	neg	r9
314a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	r10,-1024
315a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
316a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	sub	r11,r10
317a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	r11,-4096
318a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rsp,[r11*1+r10]
319a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r11,QWORD[rsp]
320a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	cmp	rsp,r10
321a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	ja	NEAR $L$mul4x_page_walk
322a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	jmp	NEAR $L$mul4x_page_walk_done
323a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
324a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan$L$mul4x_page_walk:
325a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rsp,[((-4096))+rsp]
326a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r11,QWORD[rsp]
327a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	cmp	rsp,r10
328a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	ja	NEAR $L$mul4x_page_walk
329a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan$L$mul4x_page_walk_done:
330a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
331a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	QWORD[8+r9*8+rsp],rax
332d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
333e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$mul4x_body:
334e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[16+r9*8+rsp],rdi
335d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r12,rdx
336e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r8,QWORD[r8]
337e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[r12]
338e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rsi]
339d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
340d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r14,r14
341d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r15,r15
342d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
343d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rbp,r8
344d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
345d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rax
346e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rcx]
347d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
348d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	imul	rbp,r10
349d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
350d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
351d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
352d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
353e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+rsi]
354d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
355d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rdx
356d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
357d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
358d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
359e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+rcx]
360d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
361d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rdx
362d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
363d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
364d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,rax
365e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[16+rsi]
366d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
367d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,r11
368e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r15,[4+r15]
369d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
370e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[rsp],rdi
371d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
372e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$1st4x
373d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
374e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$1st4x:
375d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
376d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
377e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-16))+r15*8+rcx]
378d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
379d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
380d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
381d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
382d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
383e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-8))+r15*8+rsi]
384d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
385d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
386d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
387e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-24))+r15*8+rsp],r13
388d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rdx
389d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
390d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
391d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
392e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-8))+r15*8+rcx]
393d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
394d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rdx
395d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
396d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
397d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,rax
398e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[r15*8+rsi]
399d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
400d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,r11
401d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
402e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-16))+r15*8+rsp],rdi
403d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
404d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
405d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
406d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
407e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[r15*8+rcx]
408d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
409d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
410d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
411d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
412d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
413e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+r15*8+rsi]
414d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
415d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
416d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
417e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-8))+r15*8+rsp],r13
418d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rdx
419d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
420d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
421d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
422e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+r15*8+rcx]
423d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
424e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r15,[4+r15]
425d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rdx
426d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
427d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
428d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,rax
429e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-16))+r15*8+rsi]
430d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
431d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,r11
432d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
433e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-32))+r15*8+rsp],rdi
434d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
435d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	r15,r9
436e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jb	NEAR $L$1st4x
437d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
438d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
439d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
440e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-16))+r15*8+rcx]
441d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
442d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
443d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
444d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
445d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
446e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-8))+r15*8+rsi]
447d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
448d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
449d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
450e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-24))+r15*8+rsp],r13
451d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rdx
452d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
453d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
454d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
455e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-8))+r15*8+rcx]
456d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
457d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rdx
458d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
459d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
460d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,rax
461e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rsi]
462d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
463d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,r11
464d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
465e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-16))+r15*8+rsp],rdi
466d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
467d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
468d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	rdi,rdi
469d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
470d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdi,0
471e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-8))+r15*8+rsp],r13
472e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[r15*8+rsp],rdi
473d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
474e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r14,[1+r14]
475d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	4
476e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$outer4x:
477e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[r14*8+r12]
478d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r15,r15
479e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10,QWORD[rsp]
480d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rbp,r8
481d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
482d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
483e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rcx]
484d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
485d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
486d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	imul	rbp,r10
487d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
488d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
489d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
490d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
491e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+rsi]
492d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
493d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rdx
494d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
495d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
496d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
497e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+rcx]
498d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
499e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	add	r11,QWORD[8+rsp]
500d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
501d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rdx
502d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
503d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
504d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,rax
505e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[16+rsi]
506d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
507d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,r11
508e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r15,[4+r15]
509d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
510e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[rsp],rdi
511d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
512e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$inner4x
513d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
514e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$inner4x:
515d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
516d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
517e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-16))+r15*8+rcx]
518d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
519e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	add	r10,QWORD[((-16))+r15*8+rsp]
520d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
521d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
522d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
523d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
524d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
525e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-8))+r15*8+rsi]
526d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
527d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
528d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
529e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-24))+r15*8+rsp],r13
530d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rdx
531d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
532d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
533d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
534e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-8))+r15*8+rcx]
535d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
536e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	add	r11,QWORD[((-8))+r15*8+rsp]
537d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
538d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rdx
539d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
540d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
541d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,rax
542e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[r15*8+rsi]
543d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
544d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,r11
545d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
546e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-16))+r15*8+rsp],rdi
547d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
548d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
549d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
550d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
551e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[r15*8+rcx]
552d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
553e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	add	r10,QWORD[r15*8+rsp]
554d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
555d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
556d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
557d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
558d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
559e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+r15*8+rsi]
560d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
561d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
562d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
563e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-8))+r15*8+rsp],r13
564d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rdx
565d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
566d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
567d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
568e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+r15*8+rcx]
569d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
570e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	add	r11,QWORD[8+r15*8+rsp]
571d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
572e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r15,[4+r15]
573d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rdx
574d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
575d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
576d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,rax
577e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-16))+r15*8+rsi]
578d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
579d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,r11
580d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
581e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-32))+r15*8+rsp],rdi
582d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
583d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	r15,r9
584e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jb	NEAR $L$inner4x
585d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
586d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
587d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r10,rax
588e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-16))+r15*8+rcx]
589d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
590e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	add	r10,QWORD[((-16))+r15*8+rsp]
591d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
592d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r11,rdx
593d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
594d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
595d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,rax
596e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-8))+r15*8+rsi]
597d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
598d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
599d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
600e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-24))+r15*8+rsp],r13
601d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rdx
602d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
603d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbx
604d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r11,rax
605e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[((-8))+r15*8+rcx]
606d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
607e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	add	r11,QWORD[((-8))+r15*8+rsp]
608d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
609e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r14,[1+r14]
610d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,rdx
611d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
612d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mul	rbp
613d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,rax
614e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rsi]
615d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
616d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rdi,r11
617d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdx,0
618e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-16))+r15*8+rsp],rdi
619d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r13,rdx
620d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
621d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	rdi,rdi
622d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	r13,r10
623d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdi,0
624e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	add	r13,QWORD[r9*8+rsp]
625d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	adc	rdi,0
626e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[((-8))+r15*8+rsp],r13
627e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[r15*8+rsp],rdi
628d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
629d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	r14,r9
630e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jb	NEAR $L$outer4x
631e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rdi,QWORD[16+r9*8+rsp]
632a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	r15,[((-4))+r9]
633e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[rsp]
634a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	pxor	xmm0,xmm0
635e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rdx,QWORD[8+rsp]
636a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	shr	r15,2
637e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	rsi,[rsp]
638d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r14,r14
639d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
640e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	sub	rax,QWORD[rcx]
641e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[16+rsi]
642e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbp,QWORD[24+rsi]
643e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	sbb	rdx,QWORD[8+rcx]
644e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$sub4x
645d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
646e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$sub4x:
647e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[r14*8+rdi],rax
648e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[8+r14*8+rdi],rdx
649e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	sbb	rbx,QWORD[16+r14*8+rcx]
650e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[32+r14*8+rsi]
651e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rdx,QWORD[40+r14*8+rsi]
652e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	sbb	rbp,QWORD[24+r14*8+rcx]
653e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[16+r14*8+rdi],rbx
654e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[24+r14*8+rdi],rbp
655e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	sbb	rax,QWORD[32+r14*8+rcx]
656e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[48+r14*8+rsi]
657e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbp,QWORD[56+r14*8+rsi]
658e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	sbb	rdx,QWORD[40+r14*8+rcx]
659e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r14,[4+r14]
660d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	dec	r15
661e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jnz	NEAR $L$sub4x
662d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
663e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[r14*8+rdi],rax
664e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[32+r14*8+rsi]
665e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	sbb	rbx,QWORD[16+r14*8+rcx]
666e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[8+r14*8+rdi],rdx
667e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	sbb	rbp,QWORD[24+r14*8+rcx]
668e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[16+r14*8+rdi],rbx
669d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
670d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	sbb	rax,0
671e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[24+r14*8+rdi],rbp
672d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	r14,r14
673a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	rsi,rax
674a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	not	rax
675a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rcx,rdi
676a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	rcx,rax
677a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	r15,[((-4))+r9]
678a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	or	rsi,rcx
679a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	shr	r15,2
680a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
681a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqu	xmm1,XMMWORD[rsi]
682a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqa	XMMWORD[rsp],xmm0
683a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqu	XMMWORD[rdi],xmm1
684e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$copy4x
685d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
686e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$copy4x:
687a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqu	xmm2,XMMWORD[16+r14*1+rsi]
688a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqu	xmm1,XMMWORD[32+r14*1+rsi]
689a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqa	XMMWORD[16+r14*1+rsp],xmm0
690a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqu	XMMWORD[16+r14*1+rdi],xmm2
691a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqa	XMMWORD[32+r14*1+rsp],xmm0
692a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqu	XMMWORD[32+r14*1+rdi],xmm1
693e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r14,[32+r14]
694d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	dec	r15
695e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jnz	NEAR $L$copy4x
696d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
697a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqu	xmm2,XMMWORD[16+r14*1+rsi]
698a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqa	XMMWORD[16+r14*1+rsp],xmm0
699a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	movdqu	XMMWORD[16+r14*1+rdi],xmm2
700e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rsi,QWORD[8+r9*8+rsp]
701a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
702d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rax,1
703a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r15,QWORD[((-48))+rsi]
704a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
705a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r14,QWORD[((-40))+rsi]
706a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
707a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r13,QWORD[((-32))+rsi]
708a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
709a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r12,QWORD[((-24))+rsi]
710a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
711a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rbp,QWORD[((-16))+rsi]
712a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
713a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rbx,QWORD[((-8))+rsi]
714a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
715a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rsp,[rsi]
716a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
717e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$mul4x_epilogue:
718e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
719e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rsi,QWORD[16+rsp]
720d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	DB	0F3h,0C3h		;repret
721a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
722e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$SEH_end_bn_mul4x_mont:
723e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam LangleyEXTERN	bn_sqr8x_internal
724d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
725d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
726d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	32
727e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleybn_sqr8x_mont:
728e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[8+rsp],rdi	;WIN64 prologue
729e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[16+rsp],rsi
730d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rax,rsp
731e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$SEH_begin_bn_sqr8x_mont:
732d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdi,rcx
733d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rsi,rdx
734d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rdx,r8
735d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rcx,r9
736e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r8,QWORD[40+rsp]
737e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r9,QWORD[48+rsp]
738d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
739d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
740a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
741d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rax,rsp
742a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
743a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan$L$sqr8x_enter:
744d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbx
745a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
746d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbp
747a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
748d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r12
749a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
750d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r13
751a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
752d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r14
753a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
754d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r15
755d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
756a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan$L$sqr8x_prologue:
757a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
758d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10d,r9d
759d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	shl	r9d,3
760d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	shl	r10,3+2
761d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	neg	r9
762d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
763d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
764d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
765d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
766d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
767d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
7684969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	r11,[((-64))+r9*2+rsp]
769a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	rbp,rsp
770e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r8,QWORD[r8]
771d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	sub	r11,rsi
772d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	and	r11,4095
773d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	r10,r11
774e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jb	NEAR $L$sqr8x_sp_alt
775a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	sub	rbp,r11
776a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rbp,[((-64))+r9*2+rbp]
777e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jmp	NEAR $L$sqr8x_sp_done
778d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
779d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	32
780e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$sqr8x_sp_alt:
7814969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	r10,[((4096-64))+r9*2]
782a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rbp,[((-64))+r9*2+rbp]
783d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	sub	r11,r10
784d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,0
785d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmovc	r11,r10
786a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	sub	rbp,r11
787e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$sqr8x_sp_done:
788a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	rbp,-64
789a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r11,rsp
790a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	sub	r11,rbp
791a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	and	r11,-4096
792a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rsp,[rbp*1+r11]
793a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r10,QWORD[rsp]
794a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	cmp	rsp,rbp
795a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	ja	NEAR $L$sqr8x_page_walk
796a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	jmp	NEAR $L$sqr8x_page_walk_done
797a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
798a94fe0531b3c196ad078174259af2201b2e3a246Robert SloanALIGN	16
799a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan$L$sqr8x_page_walk:
800a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	rsp,[((-4096))+rsp]
801a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r10,QWORD[rsp]
802a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	cmp	rsp,rbp
803a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	ja	NEAR $L$sqr8x_page_walk
804a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan$L$sqr8x_page_walk_done:
805a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
806d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	r10,r9
807d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	neg	r9
808d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
809e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[32+rsp],r8
810e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[40+rsp],rax
811a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
812e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$sqr8x_body:
813d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
8144969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid BenjaminDB	102,72,15,110,209
815d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pxor	xmm0,xmm0
816d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	102,72,15,110,207
817d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	102,73,15,110,218
818d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	call	bn_sqr8x_internal
819d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
8204969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin
8214969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin
8224969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin
8234969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	rbx,[r9*1+rdi]
8244969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	rcx,r9
8254969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	rdx,r9
8264969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid BenjaminDB	102,72,15,126,207
8274969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	sar	rcx,3+2
8284969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	jmp	NEAR $L$sqr8x_sub
8294969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin
8304969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid BenjaminALIGN	32
8314969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin$L$sqr8x_sub:
8324969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	r12,QWORD[rbx]
8334969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	r13,QWORD[8+rbx]
8344969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	r14,QWORD[16+rbx]
8354969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	r15,QWORD[24+rbx]
8364969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	rbx,[32+rbx]
8374969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	sbb	r12,QWORD[rbp]
8384969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	sbb	r13,QWORD[8+rbp]
8394969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	sbb	r14,QWORD[16+rbp]
8404969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	sbb	r15,QWORD[24+rbp]
8414969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	rbp,[32+rbp]
8424969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	QWORD[rdi],r12
8434969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	QWORD[8+rdi],r13
8444969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	QWORD[16+rdi],r14
8454969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	mov	QWORD[24+rdi],r15
8464969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	rdi,[32+rdi]
8474969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	inc	rcx
8484969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	jnz	NEAR $L$sqr8x_sub
8494969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin
8504969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	sbb	rax,0
8514969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	rbx,[r9*1+rbx]
8524969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	rdi,[r9*1+rdi]
8534969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin
8544969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid BenjaminDB	102,72,15,110,200
855d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pxor	xmm0,xmm0
8564969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	pshufd	xmm1,xmm1,0
857e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rsi,QWORD[40+rsp]
858a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
8594969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	jmp	NEAR $L$sqr8x_cond_copy
860d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
861d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	32
8624969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin$L$sqr8x_cond_copy:
8634969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqa	xmm2,XMMWORD[rbx]
8644969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqa	xmm3,XMMWORD[16+rbx]
8654969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	rbx,[32+rbx]
8664969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqu	xmm4,XMMWORD[rdi]
8674969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqu	xmm5,XMMWORD[16+rdi]
8684969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	lea	rdi,[32+rdi]
8694969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqa	XMMWORD[(-32)+rbx],xmm0
8704969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqa	XMMWORD[(-16)+rbx],xmm0
8714969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqa	XMMWORD[(-32)+rdx*1+rbx],xmm0
8724969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqa	XMMWORD[(-16)+rdx*1+rbx],xmm0
8734969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	pcmpeqd	xmm0,xmm1
8744969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	pand	xmm2,xmm1
8754969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	pand	xmm3,xmm1
8764969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	pand	xmm4,xmm0
8774969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	pand	xmm5,xmm0
8784969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	pxor	xmm0,xmm0
8794969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	por	xmm4,xmm2
8804969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	por	xmm5,xmm3
8814969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqu	XMMWORD[(-32)+rdi],xmm4
8824969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	movdqu	XMMWORD[(-16)+rdi],xmm5
8834969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	add	r9,32
8844969cc9b0ab2905ec478277f50ed3849b37a6c6bDavid Benjamin	jnz	NEAR $L$sqr8x_cond_copy
885d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
886d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rax,1
887e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r15,QWORD[((-48))+rsi]
888a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
889e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r14,QWORD[((-40))+rsi]
890a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
891e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r13,QWORD[((-32))+rsi]
892a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
893e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r12,QWORD[((-24))+rsi]
894a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
895e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbp,QWORD[((-16))+rsi]
896a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
897e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[((-8))+rsi]
898a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
899e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	rsp,[rsi]
900a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
901e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$sqr8x_epilogue:
902e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
903e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rsi,QWORD[16+rsp]
904d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	DB	0F3h,0C3h		;repret
905a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
906e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$SEH_end_bn_sqr8x_mont:
907d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105
908d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	112,108,105,99,97,116,105,111,110,32,102,111,114,32,120,56
909d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83
910d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115
911d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	115,108,46,111,114,103,62,0
912d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
913e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam LangleyEXTERN	__imp_RtlVirtualUnwind
914d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
915d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
916e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleymul_handler:
917d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rsi
918d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rdi
919d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbx
920d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbp
921d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r12
922d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r13
923d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r14
924d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r15
925d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pushfq
926d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	sub	rsp,64
927d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
928e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[120+r8]
929e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[248+r8]
930d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
931e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rsi,QWORD[8+r9]
932e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r11,QWORD[56+r9]
933d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
934e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10d,DWORD[r11]
935e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r10,[r10*1+rsi]
936d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	rbx,r10
937e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jb	NEAR $L$common_seh_tail
938d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
939e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[152+r8]
940d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
941e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10d,DWORD[4+r11]
942e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r10,[r10*1+rsi]
943d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	rbx,r10
944e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jae	NEAR $L$common_seh_tail
945e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley
946e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10,QWORD[192+r8]
947e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[8+r10*8+rax]
948d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
949a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	jmp	NEAR $L$common_pop_regs
950d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
951d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
952d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
953d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	16
954e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleysqr_handler:
955d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rsi
956d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rdi
957d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbx
958d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	rbp
959d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r12
960d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r13
961d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r14
962d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	push	r15
963d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pushfq
964d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	sub	rsp,64
965d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
966e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[120+r8]
967e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[248+r8]
968d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
969e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rsi,QWORD[8+r9]
970e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r11,QWORD[56+r9]
971d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
972e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10d,DWORD[r11]
973e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r10,[r10*1+rsi]
974d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	rbx,r10
975e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jb	NEAR $L$common_seh_tail
976d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
977a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r10d,DWORD[4+r11]
978a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	lea	r10,[r10*1+rsi]
979a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	cmp	rbx,r10
980a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	jb	NEAR $L$common_pop_regs
981a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan
982e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[152+r8]
983d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
984a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	mov	r10d,DWORD[8+r11]
985e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r10,[r10*1+rsi]
986d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	cmp	rbx,r10
987e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	jae	NEAR $L$common_seh_tail
988e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley
989e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rax,QWORD[40+rax]
990e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley
991a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan$L$common_pop_regs:
992e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbx,QWORD[((-8))+rax]
993e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rbp,QWORD[((-16))+rax]
994e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r12,QWORD[((-24))+rax]
995e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r13,QWORD[((-32))+rax]
996e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r14,QWORD[((-40))+rax]
997e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r15,QWORD[((-48))+rax]
998e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[144+r8],rbx
999e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[160+r8],rbp
1000e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[216+r8],r12
1001e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[224+r8],r13
1002e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[232+r8],r14
1003e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[240+r8],r15
1004e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley
1005e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$common_seh_tail:
1006e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rdi,QWORD[8+rax]
1007e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rsi,QWORD[16+rax]
1008e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[152+r8],rax
1009e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[168+r8],rsi
1010e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[176+r8],rdi
1011e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley
1012e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rdi,QWORD[40+r9]
1013d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rsi,r8
1014d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	ecx,154
1015e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	0xa548f3fc
1016d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
1017d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	rsi,r9
1018d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	xor	rcx,rcx
1019e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	rdx,QWORD[8+rsi]
1020e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r8,QWORD[rsi]
1021e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r9,QWORD[16+rsi]
1022e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	r10,QWORD[40+rsi]
1023e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r11,[56+rsi]
1024e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	lea	r12,[24+rsi]
1025e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[32+rsp],r10
1026e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[40+rsp],r11
1027e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[48+rsp],r12
1028e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	mov	QWORD[56+rsp],rcx
1029e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	call	QWORD[__imp_RtlVirtualUnwind]
1030d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
1031d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	mov	eax,1
1032d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	add	rsp,64
1033d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	popfq
1034d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pop	r15
1035d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pop	r14
1036d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pop	r13
1037d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pop	r12
1038d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pop	rbp
1039d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pop	rbx
1040d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pop	rdi
1041d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	pop	rsi
1042d9e397b599b13d642138480a28c14db7a136bf0Adam Langley	DB	0F3h,0C3h		;repret
1043d9e397b599b13d642138480a28c14db7a136bf0Adam Langley
1044e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley
1045e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleysection	.pdata rdata align=4
1046d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	4
1047e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$SEH_begin_bn_mul_mont wrt ..imagebase
1048e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$SEH_end_bn_mul_mont wrt ..imagebase
1049e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$SEH_info_bn_mul_mont wrt ..imagebase
1050e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley
1051e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$SEH_begin_bn_mul4x_mont wrt ..imagebase
1052e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$SEH_end_bn_mul4x_mont wrt ..imagebase
1053e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$SEH_info_bn_mul4x_mont wrt ..imagebase
1054e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley
1055e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$SEH_begin_bn_sqr8x_mont wrt ..imagebase
1056e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$SEH_end_bn_sqr8x_mont wrt ..imagebase
1057e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$SEH_info_bn_sqr8x_mont wrt ..imagebase
1058e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langleysection	.xdata rdata align=8
1059d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyALIGN	8
1060e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$SEH_info_bn_mul_mont:
1061d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	9,0,0,0
1062e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	mul_handler wrt ..imagebase
1063e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase
1064e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$SEH_info_bn_mul4x_mont:
1065d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	9,0,0,0
1066e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	mul_handler wrt ..imagebase
1067e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	$L$mul4x_body wrt ..imagebase,$L$mul4x_epilogue wrt ..imagebase
1068e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley$L$SEH_info_bn_sqr8x_mont:
1069d9e397b599b13d642138480a28c14db7a136bf0Adam LangleyDB	9,0,0,0
1070e9ada863a7b3e81f5d2b1e3bdd2305da902a87f5Adam Langley	DD	sqr_handler wrt ..imagebase
1071a94fe0531b3c196ad078174259af2201b2e3a246Robert Sloan	DD	$L$sqr8x_prologue wrt ..imagebase,$L$sqr8x_body wrt ..imagebase,$L$sqr8x_epilogue wrt ..imagebase
1072a94fe0531b3c196ad078174259af2201b2e3a246Robert SloanALIGN	8
1073