1#!/usr/bin/env perl
2
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9
10# January 2007.
11
12# Montgomery multiplication for ARMv4.
13#
14# Performance improvement naturally varies among CPU implementations
15# and compilers. The code was observed to provide +65-35% improvement
16# [depending on key length, less for longer keys] on ARM920T, and
17# +115-80% on Intel IXP425. This is compared to pre-bn_mul_mont code
18# base and compiler generated code with in-lined umull and even umlal
19# instructions. The latter means that this code didn't really have an
20# "advantage" of utilizing some "secret" instruction.
21#
22# The code is interoperable with Thumb ISA and is rather compact, less
23# than 1/2KB. Windows CE port would be trivial, as it's exclusively
24# about decorations, ABI and instruction syntax are identical.
25
26# November 2013
27#
28# Add NEON code path, which handles lengths divisible by 8. RSA/DSA
29# performance improvement on Cortex-A8 is ~45-100% depending on key
30# length, more for longer keys. On Cortex-A15 the span is ~10-105%.
31# On Snapdragon S4 improvement was measured to vary from ~70% to
32# incredible ~380%, yes, 4.8x faster, for RSA4096 sign. But this is
33# rather because original integer-only code seems to perform
34# suboptimally on S4. Situation on Cortex-A9 is unfortunately
35# different. It's being looked into, but the trouble is that
36# performance for vectors longer than 256 bits is actually couple
37# of percent worse than for integer-only code. The code is chosen
38# for execution on all NEON-capable processors, because gain on
39# others outweighs the marginal loss on Cortex-A9.
40
41$flavour = shift;
42if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
43else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
44
45if ($flavour && $flavour ne "void") {
46    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
47    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
48    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
49    die "can't locate arm-xlate.pl";
50
51    open STDOUT,"| \"$^X\" $xlate $flavour $output";
52} else {
53    open STDOUT,">$output";
54}
55
56$num="r0";	# starts as num argument, but holds &tp[num-1]
57$ap="r1";
58$bp="r2"; $bi="r2"; $rp="r2";
59$np="r3";
60$tp="r4";
61$aj="r5";
62$nj="r6";
63$tj="r7";
64$n0="r8";
65###########	# r9 is reserved by ELF as platform specific, e.g. TLS pointer
66$alo="r10";	# sl, gcc uses it to keep @GOT
67$ahi="r11";	# fp
68$nlo="r12";	# ip
69###########	# r13 is stack pointer
70$nhi="r14";	# lr
71###########	# r15 is program counter
72
73#### argument block layout relative to &tp[num-1], a.k.a. $num
74$_rp="$num,#12*4";
75# ap permanently resides in r1
76$_bp="$num,#13*4";
77# np permanently resides in r3
78$_n0="$num,#14*4";
79$_num="$num,#15*4";	$_bpend=$_num;
80
81$code=<<___;
82#include <openssl/arm_arch.h>
83
84.text
85.code	32
86
87#if __ARM_MAX_ARCH__>=7
88.align	5
89.LOPENSSL_armcap:
90.word	OPENSSL_armcap_P-.Lbn_mul_mont
91#endif
92
93.global	bn_mul_mont
94.type	bn_mul_mont,%function
95
96.align	5
97bn_mul_mont:
98.Lbn_mul_mont:
99	ldr	ip,[sp,#4]		@ load num
100	stmdb	sp!,{r0,r2}		@ sp points at argument block
101#if __ARM_MAX_ARCH__>=7
102	tst	ip,#7
103	bne	.Lialu
104	adr	r0,bn_mul_mont
105	ldr	r2,.LOPENSSL_armcap
106	ldr	r0,[r0,r2]
107#ifdef	__APPLE__
108	ldr	r0,[r0]
109#endif
110	tst	r0,#ARMV7_NEON		@ NEON available?
111	ldmia	sp, {r0,r2}
112	beq	.Lialu
113	add	sp,sp,#8
114	b	bn_mul8x_mont_neon
115.align	4
116.Lialu:
117#endif
118	cmp	ip,#2
119	mov	$num,ip			@ load num
120	movlt	r0,#0
121	addlt	sp,sp,#2*4
122	blt	.Labrt
123
124	stmdb	sp!,{r4-r12,lr}		@ save 10 registers
125
126	mov	$num,$num,lsl#2		@ rescale $num for byte count
127	sub	sp,sp,$num		@ alloca(4*num)
128	sub	sp,sp,#4		@ +extra dword
129	sub	$num,$num,#4		@ "num=num-1"
130	add	$tp,$bp,$num		@ &bp[num-1]
131
132	add	$num,sp,$num		@ $num to point at &tp[num-1]
133	ldr	$n0,[$_n0]		@ &n0
134	ldr	$bi,[$bp]		@ bp[0]
135	ldr	$aj,[$ap],#4		@ ap[0],ap++
136	ldr	$nj,[$np],#4		@ np[0],np++
137	ldr	$n0,[$n0]		@ *n0
138	str	$tp,[$_bpend]		@ save &bp[num]
139
140	umull	$alo,$ahi,$aj,$bi	@ ap[0]*bp[0]
141	str	$n0,[$_n0]		@ save n0 value
142	mul	$n0,$alo,$n0		@ "tp[0]"*n0
143	mov	$nlo,#0
144	umlal	$alo,$nlo,$nj,$n0	@ np[0]*n0+"t[0]"
145	mov	$tp,sp
146
147.L1st:
148	ldr	$aj,[$ap],#4		@ ap[j],ap++
149	mov	$alo,$ahi
150	ldr	$nj,[$np],#4		@ np[j],np++
151	mov	$ahi,#0
152	umlal	$alo,$ahi,$aj,$bi	@ ap[j]*bp[0]
153	mov	$nhi,#0
154	umlal	$nlo,$nhi,$nj,$n0	@ np[j]*n0
155	adds	$nlo,$nlo,$alo
156	str	$nlo,[$tp],#4		@ tp[j-1]=,tp++
157	adc	$nlo,$nhi,#0
158	cmp	$tp,$num
159	bne	.L1st
160
161	adds	$nlo,$nlo,$ahi
162	ldr	$tp,[$_bp]		@ restore bp
163	mov	$nhi,#0
164	ldr	$n0,[$_n0]		@ restore n0
165	adc	$nhi,$nhi,#0
166	str	$nlo,[$num]		@ tp[num-1]=
167	str	$nhi,[$num,#4]		@ tp[num]=
168
169.Louter:
170	sub	$tj,$num,sp		@ "original" $num-1 value
171	sub	$ap,$ap,$tj		@ "rewind" ap to &ap[1]
172	ldr	$bi,[$tp,#4]!		@ *(++bp)
173	sub	$np,$np,$tj		@ "rewind" np to &np[1]
174	ldr	$aj,[$ap,#-4]		@ ap[0]
175	ldr	$alo,[sp]		@ tp[0]
176	ldr	$nj,[$np,#-4]		@ np[0]
177	ldr	$tj,[sp,#4]		@ tp[1]
178
179	mov	$ahi,#0
180	umlal	$alo,$ahi,$aj,$bi	@ ap[0]*bp[i]+tp[0]
181	str	$tp,[$_bp]		@ save bp
182	mul	$n0,$alo,$n0
183	mov	$nlo,#0
184	umlal	$alo,$nlo,$nj,$n0	@ np[0]*n0+"tp[0]"
185	mov	$tp,sp
186
187.Linner:
188	ldr	$aj,[$ap],#4		@ ap[j],ap++
189	adds	$alo,$ahi,$tj		@ +=tp[j]
190	ldr	$nj,[$np],#4		@ np[j],np++
191	mov	$ahi,#0
192	umlal	$alo,$ahi,$aj,$bi	@ ap[j]*bp[i]
193	mov	$nhi,#0
194	umlal	$nlo,$nhi,$nj,$n0	@ np[j]*n0
195	adc	$ahi,$ahi,#0
196	ldr	$tj,[$tp,#8]		@ tp[j+1]
197	adds	$nlo,$nlo,$alo
198	str	$nlo,[$tp],#4		@ tp[j-1]=,tp++
199	adc	$nlo,$nhi,#0
200	cmp	$tp,$num
201	bne	.Linner
202
203	adds	$nlo,$nlo,$ahi
204	mov	$nhi,#0
205	ldr	$tp,[$_bp]		@ restore bp
206	adc	$nhi,$nhi,#0
207	ldr	$n0,[$_n0]		@ restore n0
208	adds	$nlo,$nlo,$tj
209	ldr	$tj,[$_bpend]		@ restore &bp[num]
210	adc	$nhi,$nhi,#0
211	str	$nlo,[$num]		@ tp[num-1]=
212	str	$nhi,[$num,#4]		@ tp[num]=
213
214	cmp	$tp,$tj
215	bne	.Louter
216
217	ldr	$rp,[$_rp]		@ pull rp
218	add	$num,$num,#4		@ $num to point at &tp[num]
219	sub	$aj,$num,sp		@ "original" num value
220	mov	$tp,sp			@ "rewind" $tp
221	mov	$ap,$tp			@ "borrow" $ap
222	sub	$np,$np,$aj		@ "rewind" $np to &np[0]
223
224	subs	$tj,$tj,$tj		@ "clear" carry flag
225.Lsub:	ldr	$tj,[$tp],#4
226	ldr	$nj,[$np],#4
227	sbcs	$tj,$tj,$nj		@ tp[j]-np[j]
228	str	$tj,[$rp],#4		@ rp[j]=
229	teq	$tp,$num		@ preserve carry
230	bne	.Lsub
231	sbcs	$nhi,$nhi,#0		@ upmost carry
232	mov	$tp,sp			@ "rewind" $tp
233	sub	$rp,$rp,$aj		@ "rewind" $rp
234
235	and	$ap,$tp,$nhi
236	bic	$np,$rp,$nhi
237	orr	$ap,$ap,$np		@ ap=borrow?tp:rp
238
239.Lcopy:	ldr	$tj,[$ap],#4		@ copy or in-place refresh
240	str	sp,[$tp],#4		@ zap tp
241	str	$tj,[$rp],#4
242	cmp	$tp,$num
243	bne	.Lcopy
244
245	add	sp,$num,#4		@ skip over tp[num+1]
246	ldmia	sp!,{r4-r12,lr}		@ restore registers
247	add	sp,sp,#2*4		@ skip over {r0,r2}
248	mov	r0,#1
249.Labrt:
250#if __ARM_ARCH__>=5
251	ret				@ bx lr
252#else
253	tst	lr,#1
254	moveq	pc,lr			@ be binary compatible with V4, yet
255	bx	lr			@ interoperable with Thumb ISA:-)
256#endif
257.size	bn_mul_mont,.-bn_mul_mont
258___
259{
260sub Dlo()   { shift=~m|q([1]?[0-9])|?"d".($1*2):"";     }
261sub Dhi()   { shift=~m|q([1]?[0-9])|?"d".($1*2+1):"";   }
262
263my ($A0,$A1,$A2,$A3)=map("d$_",(0..3));
264my ($N0,$N1,$N2,$N3)=map("d$_",(4..7));
265my ($Z,$Temp)=("q4","q5");
266my ($A0xB,$A1xB,$A2xB,$A3xB,$A4xB,$A5xB,$A6xB,$A7xB)=map("q$_",(6..13));
267my ($Bi,$Ni,$M0)=map("d$_",(28..31));
268my $zero=&Dlo($Z);
269my $temp=&Dlo($Temp);
270
271my ($rptr,$aptr,$bptr,$nptr,$n0,$num)=map("r$_",(0..5));
272my ($tinptr,$toutptr,$inner,$outer)=map("r$_",(6..9));
273
274$code.=<<___;
275#if __ARM_MAX_ARCH__>=7
276.arch	armv7-a
277.fpu	neon
278
279.type	bn_mul8x_mont_neon,%function
280.align	5
281bn_mul8x_mont_neon:
282	mov	ip,sp
283	stmdb	sp!,{r4-r11}
284	vstmdb	sp!,{d8-d15}		@ ABI specification says so
285	ldmia	ip,{r4-r5}		@ load rest of parameter block
286
287	sub		$toutptr,sp,#16
288	vld1.32		{${Bi}[0]}, [$bptr,:32]!
289	sub		$toutptr,$toutptr,$num,lsl#4
290	vld1.32		{$A0-$A3},  [$aptr]!		@ can't specify :32 :-(
291	and		$toutptr,$toutptr,#-64
292	vld1.32		{${M0}[0]}, [$n0,:32]
293	mov		sp,$toutptr			@ alloca
294	veor		$zero,$zero,$zero
295	subs		$inner,$num,#8
296	vzip.16		$Bi,$zero
297
298	vmull.u32	$A0xB,$Bi,${A0}[0]
299	vmull.u32	$A1xB,$Bi,${A0}[1]
300	vmull.u32	$A2xB,$Bi,${A1}[0]
301	vshl.i64	$temp,`&Dhi("$A0xB")`,#16
302	vmull.u32	$A3xB,$Bi,${A1}[1]
303
304	vadd.u64	$temp,$temp,`&Dlo("$A0xB")`
305	veor		$zero,$zero,$zero
306	vmul.u32	$Ni,$temp,$M0
307
308	vmull.u32	$A4xB,$Bi,${A2}[0]
309	 vld1.32	{$N0-$N3}, [$nptr]!
310	vmull.u32	$A5xB,$Bi,${A2}[1]
311	vmull.u32	$A6xB,$Bi,${A3}[0]
312	vzip.16		$Ni,$zero
313	vmull.u32	$A7xB,$Bi,${A3}[1]
314
315	bne	.LNEON_1st
316
317	@ special case for num=8, everything is in register bank...
318
319	vmlal.u32	$A0xB,$Ni,${N0}[0]
320	sub		$outer,$num,#1
321	vmlal.u32	$A1xB,$Ni,${N0}[1]
322	vmlal.u32	$A2xB,$Ni,${N1}[0]
323	vmlal.u32	$A3xB,$Ni,${N1}[1]
324
325	vmlal.u32	$A4xB,$Ni,${N2}[0]
326	vmov		$Temp,$A0xB
327	vmlal.u32	$A5xB,$Ni,${N2}[1]
328	vmov		$A0xB,$A1xB
329	vmlal.u32	$A6xB,$Ni,${N3}[0]
330	vmov		$A1xB,$A2xB
331	vmlal.u32	$A7xB,$Ni,${N3}[1]
332	vmov		$A2xB,$A3xB
333	vmov		$A3xB,$A4xB
334	vshr.u64	$temp,$temp,#16
335	vmov		$A4xB,$A5xB
336	vmov		$A5xB,$A6xB
337	vadd.u64	$temp,$temp,`&Dhi("$Temp")`
338	vmov		$A6xB,$A7xB
339	veor		$A7xB,$A7xB
340	vshr.u64	$temp,$temp,#16
341
342	b	.LNEON_outer8
343
344.align	4
345.LNEON_outer8:
346	vld1.32		{${Bi}[0]}, [$bptr,:32]!
347	veor		$zero,$zero,$zero
348	vzip.16		$Bi,$zero
349	vadd.u64	`&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp
350
351	vmlal.u32	$A0xB,$Bi,${A0}[0]
352	vmlal.u32	$A1xB,$Bi,${A0}[1]
353	vmlal.u32	$A2xB,$Bi,${A1}[0]
354	vshl.i64	$temp,`&Dhi("$A0xB")`,#16
355	vmlal.u32	$A3xB,$Bi,${A1}[1]
356
357	vadd.u64	$temp,$temp,`&Dlo("$A0xB")`
358	veor		$zero,$zero,$zero
359	subs		$outer,$outer,#1
360	vmul.u32	$Ni,$temp,$M0
361
362	vmlal.u32	$A4xB,$Bi,${A2}[0]
363	vmlal.u32	$A5xB,$Bi,${A2}[1]
364	vmlal.u32	$A6xB,$Bi,${A3}[0]
365	vzip.16		$Ni,$zero
366	vmlal.u32	$A7xB,$Bi,${A3}[1]
367
368	vmlal.u32	$A0xB,$Ni,${N0}[0]
369	vmlal.u32	$A1xB,$Ni,${N0}[1]
370	vmlal.u32	$A2xB,$Ni,${N1}[0]
371	vmlal.u32	$A3xB,$Ni,${N1}[1]
372
373	vmlal.u32	$A4xB,$Ni,${N2}[0]
374	vmov		$Temp,$A0xB
375	vmlal.u32	$A5xB,$Ni,${N2}[1]
376	vmov		$A0xB,$A1xB
377	vmlal.u32	$A6xB,$Ni,${N3}[0]
378	vmov		$A1xB,$A2xB
379	vmlal.u32	$A7xB,$Ni,${N3}[1]
380	vmov		$A2xB,$A3xB
381	vmov		$A3xB,$A4xB
382	vshr.u64	$temp,$temp,#16
383	vmov		$A4xB,$A5xB
384	vmov		$A5xB,$A6xB
385	vadd.u64	$temp,$temp,`&Dhi("$Temp")`
386	vmov		$A6xB,$A7xB
387	veor		$A7xB,$A7xB
388	vshr.u64	$temp,$temp,#16
389
390	bne	.LNEON_outer8
391
392	vadd.u64	`&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp
393	mov		$toutptr,sp
394	vshr.u64	$temp,`&Dlo("$A0xB")`,#16
395	mov		$inner,$num
396	vadd.u64	`&Dhi("$A0xB")`,`&Dhi("$A0xB")`,$temp
397	add		$tinptr,sp,#16
398	vshr.u64	$temp,`&Dhi("$A0xB")`,#16
399	vzip.16		`&Dlo("$A0xB")`,`&Dhi("$A0xB")`
400
401	b	.LNEON_tail2
402
403.align	4
404.LNEON_1st:
405	vmlal.u32	$A0xB,$Ni,${N0}[0]
406	 vld1.32	{$A0-$A3}, [$aptr]!
407	vmlal.u32	$A1xB,$Ni,${N0}[1]
408	subs		$inner,$inner,#8
409	vmlal.u32	$A2xB,$Ni,${N1}[0]
410	vmlal.u32	$A3xB,$Ni,${N1}[1]
411
412	vmlal.u32	$A4xB,$Ni,${N2}[0]
413	 vld1.32	{$N0-$N1}, [$nptr]!
414	vmlal.u32	$A5xB,$Ni,${N2}[1]
415	 vst1.64	{$A0xB-$A1xB}, [$toutptr,:256]!
416	vmlal.u32	$A6xB,$Ni,${N3}[0]
417	vmlal.u32	$A7xB,$Ni,${N3}[1]
418	 vst1.64	{$A2xB-$A3xB}, [$toutptr,:256]!
419
420	vmull.u32	$A0xB,$Bi,${A0}[0]
421	 vld1.32	{$N2-$N3}, [$nptr]!
422	vmull.u32	$A1xB,$Bi,${A0}[1]
423	 vst1.64	{$A4xB-$A5xB}, [$toutptr,:256]!
424	vmull.u32	$A2xB,$Bi,${A1}[0]
425	vmull.u32	$A3xB,$Bi,${A1}[1]
426	 vst1.64	{$A6xB-$A7xB}, [$toutptr,:256]!
427
428	vmull.u32	$A4xB,$Bi,${A2}[0]
429	vmull.u32	$A5xB,$Bi,${A2}[1]
430	vmull.u32	$A6xB,$Bi,${A3}[0]
431	vmull.u32	$A7xB,$Bi,${A3}[1]
432
433	bne	.LNEON_1st
434
435	vmlal.u32	$A0xB,$Ni,${N0}[0]
436	add		$tinptr,sp,#16
437	vmlal.u32	$A1xB,$Ni,${N0}[1]
438	sub		$aptr,$aptr,$num,lsl#2		@ rewind $aptr
439	vmlal.u32	$A2xB,$Ni,${N1}[0]
440	 vld1.64	{$Temp}, [sp,:128]
441	vmlal.u32	$A3xB,$Ni,${N1}[1]
442	sub		$outer,$num,#1
443
444	vmlal.u32	$A4xB,$Ni,${N2}[0]
445	vst1.64		{$A0xB-$A1xB}, [$toutptr,:256]!
446	vmlal.u32	$A5xB,$Ni,${N2}[1]
447	vshr.u64	$temp,$temp,#16
448	 vld1.64	{$A0xB},       [$tinptr, :128]!
449	vmlal.u32	$A6xB,$Ni,${N3}[0]
450	vst1.64		{$A2xB-$A3xB}, [$toutptr,:256]!
451	vmlal.u32	$A7xB,$Ni,${N3}[1]
452
453	vst1.64		{$A4xB-$A5xB}, [$toutptr,:256]!
454	vadd.u64	$temp,$temp,`&Dhi("$Temp")`
455	veor		$Z,$Z,$Z
456	vst1.64		{$A6xB-$A7xB}, [$toutptr,:256]!
457	 vld1.64	{$A1xB-$A2xB}, [$tinptr, :256]!
458	vst1.64		{$Z},          [$toutptr,:128]
459	vshr.u64	$temp,$temp,#16
460
461	b		.LNEON_outer
462
463.align	4
464.LNEON_outer:
465	vld1.32		{${Bi}[0]}, [$bptr,:32]!
466	sub		$nptr,$nptr,$num,lsl#2		@ rewind $nptr
467	vld1.32		{$A0-$A3},  [$aptr]!
468	veor		$zero,$zero,$zero
469	mov		$toutptr,sp
470	vzip.16		$Bi,$zero
471	sub		$inner,$num,#8
472	vadd.u64	`&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp
473
474	vmlal.u32	$A0xB,$Bi,${A0}[0]
475	 vld1.64	{$A3xB-$A4xB},[$tinptr,:256]!
476	vmlal.u32	$A1xB,$Bi,${A0}[1]
477	vmlal.u32	$A2xB,$Bi,${A1}[0]
478	 vld1.64	{$A5xB-$A6xB},[$tinptr,:256]!
479	vmlal.u32	$A3xB,$Bi,${A1}[1]
480
481	vshl.i64	$temp,`&Dhi("$A0xB")`,#16
482	veor		$zero,$zero,$zero
483	vadd.u64	$temp,$temp,`&Dlo("$A0xB")`
484	 vld1.64	{$A7xB},[$tinptr,:128]!
485	vmul.u32	$Ni,$temp,$M0
486
487	vmlal.u32	$A4xB,$Bi,${A2}[0]
488	 vld1.32	{$N0-$N3}, [$nptr]!
489	vmlal.u32	$A5xB,$Bi,${A2}[1]
490	vmlal.u32	$A6xB,$Bi,${A3}[0]
491	vzip.16		$Ni,$zero
492	vmlal.u32	$A7xB,$Bi,${A3}[1]
493
494.LNEON_inner:
495	vmlal.u32	$A0xB,$Ni,${N0}[0]
496	 vld1.32	{$A0-$A3}, [$aptr]!
497	vmlal.u32	$A1xB,$Ni,${N0}[1]
498	 subs		$inner,$inner,#8
499	vmlal.u32	$A2xB,$Ni,${N1}[0]
500	vmlal.u32	$A3xB,$Ni,${N1}[1]
501	vst1.64		{$A0xB-$A1xB}, [$toutptr,:256]!
502
503	vmlal.u32	$A4xB,$Ni,${N2}[0]
504	 vld1.64	{$A0xB},       [$tinptr, :128]!
505	vmlal.u32	$A5xB,$Ni,${N2}[1]
506	vst1.64		{$A2xB-$A3xB}, [$toutptr,:256]!
507	vmlal.u32	$A6xB,$Ni,${N3}[0]
508	 vld1.64	{$A1xB-$A2xB}, [$tinptr, :256]!
509	vmlal.u32	$A7xB,$Ni,${N3}[1]
510	vst1.64		{$A4xB-$A5xB}, [$toutptr,:256]!
511
512	vmlal.u32	$A0xB,$Bi,${A0}[0]
513	 vld1.64	{$A3xB-$A4xB}, [$tinptr, :256]!
514	vmlal.u32	$A1xB,$Bi,${A0}[1]
515	vst1.64		{$A6xB-$A7xB}, [$toutptr,:256]!
516	vmlal.u32	$A2xB,$Bi,${A1}[0]
517	 vld1.64	{$A5xB-$A6xB}, [$tinptr, :256]!
518	vmlal.u32	$A3xB,$Bi,${A1}[1]
519	 vld1.32	{$N0-$N3}, [$nptr]!
520
521	vmlal.u32	$A4xB,$Bi,${A2}[0]
522	 vld1.64	{$A7xB},       [$tinptr, :128]!
523	vmlal.u32	$A5xB,$Bi,${A2}[1]
524	vmlal.u32	$A6xB,$Bi,${A3}[0]
525	vmlal.u32	$A7xB,$Bi,${A3}[1]
526
527	bne	.LNEON_inner
528
529	vmlal.u32	$A0xB,$Ni,${N0}[0]
530	add		$tinptr,sp,#16
531	vmlal.u32	$A1xB,$Ni,${N0}[1]
532	sub		$aptr,$aptr,$num,lsl#2		@ rewind $aptr
533	vmlal.u32	$A2xB,$Ni,${N1}[0]
534	 vld1.64	{$Temp}, [sp,:128]
535	vmlal.u32	$A3xB,$Ni,${N1}[1]
536	subs		$outer,$outer,#1
537
538	vmlal.u32	$A4xB,$Ni,${N2}[0]
539	vst1.64		{$A0xB-$A1xB}, [$toutptr,:256]!
540	vmlal.u32	$A5xB,$Ni,${N2}[1]
541	 vld1.64	{$A0xB},       [$tinptr, :128]!
542	vshr.u64	$temp,$temp,#16
543	vst1.64		{$A2xB-$A3xB}, [$toutptr,:256]!
544	vmlal.u32	$A6xB,$Ni,${N3}[0]
545	 vld1.64	{$A1xB-$A2xB}, [$tinptr, :256]!
546	vmlal.u32	$A7xB,$Ni,${N3}[1]
547
548	vst1.64		{$A4xB-$A5xB}, [$toutptr,:256]!
549	vadd.u64	$temp,$temp,`&Dhi("$Temp")`
550	vst1.64		{$A6xB-$A7xB}, [$toutptr,:256]!
551	vshr.u64	$temp,$temp,#16
552
553	bne	.LNEON_outer
554
555	mov		$toutptr,sp
556	mov		$inner,$num
557
558.LNEON_tail:
559	vadd.u64	`&Dlo("$A0xB")`,`&Dlo("$A0xB")`,$temp
560	vld1.64		{$A3xB-$A4xB}, [$tinptr, :256]!
561	vshr.u64	$temp,`&Dlo("$A0xB")`,#16
562	vadd.u64	`&Dhi("$A0xB")`,`&Dhi("$A0xB")`,$temp
563	vld1.64		{$A5xB-$A6xB}, [$tinptr, :256]!
564	vshr.u64	$temp,`&Dhi("$A0xB")`,#16
565	vld1.64		{$A7xB},       [$tinptr, :128]!
566	vzip.16		`&Dlo("$A0xB")`,`&Dhi("$A0xB")`
567
568.LNEON_tail2:
569	vadd.u64	`&Dlo("$A1xB")`,`&Dlo("$A1xB")`,$temp
570	vst1.32		{`&Dlo("$A0xB")`[0]}, [$toutptr, :32]!
571	vshr.u64	$temp,`&Dlo("$A1xB")`,#16
572	vadd.u64	`&Dhi("$A1xB")`,`&Dhi("$A1xB")`,$temp
573	vshr.u64	$temp,`&Dhi("$A1xB")`,#16
574	vzip.16		`&Dlo("$A1xB")`,`&Dhi("$A1xB")`
575
576	vadd.u64	`&Dlo("$A2xB")`,`&Dlo("$A2xB")`,$temp
577	vst1.32		{`&Dlo("$A1xB")`[0]}, [$toutptr, :32]!
578	vshr.u64	$temp,`&Dlo("$A2xB")`,#16
579	vadd.u64	`&Dhi("$A2xB")`,`&Dhi("$A2xB")`,$temp
580	vshr.u64	$temp,`&Dhi("$A2xB")`,#16
581	vzip.16		`&Dlo("$A2xB")`,`&Dhi("$A2xB")`
582
583	vadd.u64	`&Dlo("$A3xB")`,`&Dlo("$A3xB")`,$temp
584	vst1.32		{`&Dlo("$A2xB")`[0]}, [$toutptr, :32]!
585	vshr.u64	$temp,`&Dlo("$A3xB")`,#16
586	vadd.u64	`&Dhi("$A3xB")`,`&Dhi("$A3xB")`,$temp
587	vshr.u64	$temp,`&Dhi("$A3xB")`,#16
588	vzip.16		`&Dlo("$A3xB")`,`&Dhi("$A3xB")`
589
590	vadd.u64	`&Dlo("$A4xB")`,`&Dlo("$A4xB")`,$temp
591	vst1.32		{`&Dlo("$A3xB")`[0]}, [$toutptr, :32]!
592	vshr.u64	$temp,`&Dlo("$A4xB")`,#16
593	vadd.u64	`&Dhi("$A4xB")`,`&Dhi("$A4xB")`,$temp
594	vshr.u64	$temp,`&Dhi("$A4xB")`,#16
595	vzip.16		`&Dlo("$A4xB")`,`&Dhi("$A4xB")`
596
597	vadd.u64	`&Dlo("$A5xB")`,`&Dlo("$A5xB")`,$temp
598	vst1.32		{`&Dlo("$A4xB")`[0]}, [$toutptr, :32]!
599	vshr.u64	$temp,`&Dlo("$A5xB")`,#16
600	vadd.u64	`&Dhi("$A5xB")`,`&Dhi("$A5xB")`,$temp
601	vshr.u64	$temp,`&Dhi("$A5xB")`,#16
602	vzip.16		`&Dlo("$A5xB")`,`&Dhi("$A5xB")`
603
604	vadd.u64	`&Dlo("$A6xB")`,`&Dlo("$A6xB")`,$temp
605	vst1.32		{`&Dlo("$A5xB")`[0]}, [$toutptr, :32]!
606	vshr.u64	$temp,`&Dlo("$A6xB")`,#16
607	vadd.u64	`&Dhi("$A6xB")`,`&Dhi("$A6xB")`,$temp
608	vld1.64		{$A0xB}, [$tinptr, :128]!
609	vshr.u64	$temp,`&Dhi("$A6xB")`,#16
610	vzip.16		`&Dlo("$A6xB")`,`&Dhi("$A6xB")`
611
612	vadd.u64	`&Dlo("$A7xB")`,`&Dlo("$A7xB")`,$temp
613	vst1.32		{`&Dlo("$A6xB")`[0]}, [$toutptr, :32]!
614	vshr.u64	$temp,`&Dlo("$A7xB")`,#16
615	vadd.u64	`&Dhi("$A7xB")`,`&Dhi("$A7xB")`,$temp
616	vld1.64		{$A1xB-$A2xB},	[$tinptr, :256]!
617	vshr.u64	$temp,`&Dhi("$A7xB")`,#16
618	vzip.16		`&Dlo("$A7xB")`,`&Dhi("$A7xB")`
619	subs		$inner,$inner,#8
620	vst1.32		{`&Dlo("$A7xB")`[0]}, [$toutptr, :32]!
621
622	bne	.LNEON_tail
623
624	vst1.32	{${temp}[0]}, [$toutptr, :32]		@ top-most bit
625	sub	$nptr,$nptr,$num,lsl#2			@ rewind $nptr
626	subs	$aptr,sp,#0				@ clear carry flag
627	add	$bptr,sp,$num,lsl#2
628
629.LNEON_sub:
630	ldmia	$aptr!, {r4-r7}
631	ldmia	$nptr!, {r8-r11}
632	sbcs	r8, r4,r8
633	sbcs	r9, r5,r9
634	sbcs	r10,r6,r10
635	sbcs	r11,r7,r11
636	teq	$aptr,$bptr				@ preserves carry
637	stmia	$rptr!, {r8-r11}
638	bne	.LNEON_sub
639
640	ldr	r10, [$aptr]				@ load top-most bit
641	veor	q0,q0,q0
642	sub	r11,$bptr,sp				@ this is num*4
643	veor	q1,q1,q1
644	mov	$aptr,sp
645	sub	$rptr,$rptr,r11				@ rewind $rptr
646	mov	$nptr,$bptr				@ second 3/4th of frame
647	sbcs	r10,r10,#0				@ result is carry flag
648
649.LNEON_copy_n_zap:
650	ldmia	$aptr!, {r4-r7}
651	ldmia	$rptr,  {r8-r11}
652	movcc	r8, r4
653	vst1.64	{q0-q1}, [$nptr,:256]!			@ wipe
654	movcc	r9, r5
655	movcc	r10,r6
656	vst1.64	{q0-q1}, [$nptr,:256]!			@ wipe
657	movcc	r11,r7
658	ldmia	$aptr, {r4-r7}
659	stmia	$rptr!, {r8-r11}
660	sub	$aptr,$aptr,#16
661	ldmia	$rptr, {r8-r11}
662	movcc	r8, r4
663	vst1.64	{q0-q1}, [$aptr,:256]!			@ wipe
664	movcc	r9, r5
665	movcc	r10,r6
666	vst1.64	{q0-q1}, [$nptr,:256]!			@ wipe
667	movcc	r11,r7
668	teq	$aptr,$bptr				@ preserves carry
669	stmia	$rptr!, {r8-r11}
670	bne	.LNEON_copy_n_zap
671
672	sub	sp,ip,#96
673        vldmia  sp!,{d8-d15}
674        ldmia   sp!,{r4-r11}
675	ret						@ bx lr
676.size	bn_mul8x_mont_neon,.-bn_mul8x_mont_neon
677#endif
678___
679}
680$code.=<<___;
681.asciz	"Montgomery multiplication for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
682.align	2
683#if __ARM_MAX_ARCH__>=7
684.comm	OPENSSL_armcap_P,4,4
685.hidden	OPENSSL_armcap_P
686#endif
687___
688
689$code =~ s/\`([^\`]*)\`/eval $1/gem;
690$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm;	# make it possible to compile with -march=armv4
691$code =~ s/\bret\b/bx	lr/gm;
692print $code;
693close STDOUT;
694