1/* Copyright (c) 2014, Linaro Limited
2   All rights reserved.
3
4   Redistribution and use in source and binary forms, with or without
5   modification, are permitted provided that the following conditions are met:
6       * Redistributions of source code must retain the above copyright
7         notice, this list of conditions and the following disclaimer.
8       * Redistributions in binary form must reproduce the above copyright
9         notice, this list of conditions and the following disclaimer in the
10         documentation and/or other materials provided with the distribution.
11       * Neither the name of the Linaro nor the
12         names of its contributors may be used to endorse or promote products
13         derived from this software without specific prior written permission.
14
15   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19   HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26*/
27
28/* Assumptions:
29 *
30 * ARMv8-a, AArch64
31 */
32
33#include <private/bionic_asm.h>
34
35#define REP8_01 0x0101010101010101
36#define REP8_7f 0x7f7f7f7f7f7f7f7f
37#define REP8_80 0x8080808080808080
38
39/* Parameters and result.  */
40#define src1		x0
41#define src2		x1
42#define limit		x2
43#define result		x0
44
45/* Internal variables.  */
46#define data1		x3
47#define data1w		w3
48#define data2		x4
49#define data2w		w4
50#define has_nul		x5
51#define diff		x6
52#define syndrome	x7
53#define tmp1		x8
54#define tmp2		x9
55#define tmp3		x10
56#define zeroones	x11
57#define pos		x12
58#define limit_wd	x13
59#define mask		x14
60#define endloop		x15
61
62	.text
63	.p2align 6
64	.rep 7
65	nop	/* Pad so that the loop below fits a cache line.  */
66	.endr
67ENTRY(strncmp)
68	cbz	limit, .Lret0
69	eor	tmp1, src1, src2
70	mov	zeroones, #REP8_01
71	tst	tmp1, #7
72	b.ne	.Lmisaligned8
73	ands	tmp1, src1, #7
74	b.ne	.Lmutual_align
75	/* Calculate the number of full and partial words -1.  */
76	sub	limit_wd, limit, #1	/* limit != 0, so no underflow.  */
77	lsr	limit_wd, limit_wd, #3	/* Convert to Dwords.  */
78
79	/* NUL detection works on the principle that (X - 1) & (~X) & 0x80
80	   (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
81	   can be done in parallel across the entire word.  */
82	/* Start of performance-critical section  -- one 64B cache line.  */
83.Lloop_aligned:
84	ldr	data1, [src1], #8
85	ldr	data2, [src2], #8
86.Lstart_realigned:
87	subs	limit_wd, limit_wd, #1
88	sub	tmp1, data1, zeroones
89	orr	tmp2, data1, #REP8_7f
90	eor	diff, data1, data2	/* Non-zero if differences found.  */
91	csinv	endloop, diff, xzr, pl	/* Last Dword or differences.  */
92	bics	has_nul, tmp1, tmp2	/* Non-zero if NUL terminator.  */
93	ccmp	endloop, #0, #0, eq
94	b.eq	.Lloop_aligned
95	/* End of performance-critical section  -- one 64B cache line.  */
96
97	/* Not reached the limit, must have found the end or a diff.  */
98	tbz	limit_wd, #63, .Lnot_limit
99
100	/* Limit % 8 == 0 => all bytes significant.  */
101	ands	limit, limit, #7
102	b.eq	.Lnot_limit
103
104	lsl	limit, limit, #3	/* Bits -> bytes.  */
105	mov	mask, #~0
106#ifdef __AARCH64EB__
107	lsr	mask, mask, limit
108#else
109	lsl	mask, mask, limit
110#endif
111	bic	data1, data1, mask
112	bic	data2, data2, mask
113
114	/* Make sure that the NUL byte is marked in the syndrome.  */
115	orr	has_nul, has_nul, mask
116
117.Lnot_limit:
118	orr	syndrome, diff, has_nul
119
120#ifndef	__AARCH64EB__
121	rev	syndrome, syndrome
122	rev	data1, data1
123	/* The MS-non-zero bit of the syndrome marks either the first bit
124	   that is different, or the top bit of the first zero byte.
125	   Shifting left now will bring the critical information into the
126	   top bits.  */
127	clz	pos, syndrome
128	rev	data2, data2
129	lsl	data1, data1, pos
130	lsl	data2, data2, pos
131	/* But we need to zero-extend (char is unsigned) the value and then
132	   perform a signed 32-bit subtraction.  */
133	lsr	data1, data1, #56
134	sub	result, data1, data2, lsr #56
135	ret
136#else
137	/* For big-endian we cannot use the trick with the syndrome value
138	   as carry-propagation can corrupt the upper bits if the trailing
139	   bytes in the string contain 0x01.  */
140	/* However, if there is no NUL byte in the dword, we can generate
141	   the result directly.  We can't just subtract the bytes as the
142	   MSB might be significant.  */
143	cbnz	has_nul, 1f
144	cmp	data1, data2
145	cset	result, ne
146	cneg	result, result, lo
147	ret
1481:
149	/* Re-compute the NUL-byte detection, using a byte-reversed value.  */
150	rev	tmp3, data1
151	sub	tmp1, tmp3, zeroones
152	orr	tmp2, tmp3, #REP8_7f
153	bic	has_nul, tmp1, tmp2
154	rev	has_nul, has_nul
155	orr	syndrome, diff, has_nul
156	clz	pos, syndrome
157	/* The MS-non-zero bit of the syndrome marks either the first bit
158	   that is different, or the top bit of the first zero byte.
159	   Shifting left now will bring the critical information into the
160	   top bits.  */
161	lsl	data1, data1, pos
162	lsl	data2, data2, pos
163	/* But we need to zero-extend (char is unsigned) the value and then
164	   perform a signed 32-bit subtraction.  */
165	lsr	data1, data1, #56
166	sub	result, data1, data2, lsr #56
167	ret
168#endif
169
170.Lmutual_align:
171	/* Sources are mutually aligned, but are not currently at an
172	   alignment boundary.  Round down the addresses and then mask off
173	   the bytes that precede the start point.
174	   We also need to adjust the limit calculations, but without
175	   overflowing if the limit is near ULONG_MAX.  */
176	bic	src1, src1, #7
177	bic	src2, src2, #7
178	ldr	data1, [src1], #8
179	neg	tmp3, tmp1, lsl #3	/* 64 - bits(bytes beyond align). */
180	ldr	data2, [src2], #8
181	mov	tmp2, #~0
182	sub	limit_wd, limit, #1	/* limit != 0, so no underflow.  */
183#ifdef __AARCH64EB__
184	/* Big-endian.  Early bytes are at MSB.  */
185	lsl	tmp2, tmp2, tmp3	/* Shift (tmp1 & 63).  */
186#else
187	/* Little-endian.  Early bytes are at LSB.  */
188	lsr	tmp2, tmp2, tmp3	/* Shift (tmp1 & 63).  */
189#endif
190	and	tmp3, limit_wd, #7
191	lsr	limit_wd, limit_wd, #3
192	/* Adjust the limit. Only low 3 bits used, so overflow irrelevant.  */
193	add	limit, limit, tmp1
194	add	tmp3, tmp3, tmp1
195	orr	data1, data1, tmp2
196	orr	data2, data2, tmp2
197	add	limit_wd, limit_wd, tmp3, lsr #3
198	b	.Lstart_realigned
199
200.Lret0:
201	mov	result, #0
202	ret
203
204	.p2align 6
205.Lmisaligned8:
206	sub	limit, limit, #1
2071:
208	/* Perhaps we can do better than this.  */
209	ldrb	data1w, [src1], #1
210	ldrb	data2w, [src2], #1
211	subs	limit, limit, #1
212	ccmp	data1w, #1, #0, cs	/* NZCV = 0b0000.  */
213	ccmp	data1w, data2w, #0, cs	/* NZCV = 0b0000.  */
214	b.eq	1b
215	sub	result, data1, data2
216	ret
217END(strncmp)
218