1/* Copyright 2002 Andi Kleen */
2
3#include <linux/linkage.h>
4
5#include <asm/cpufeature.h>
6#include <asm/dwarf2.h>
7#include <asm/alternative-asm.h>
8
9/*
10 * memcpy - Copy a memory block.
11 *
12 * Input:
13 *  rdi destination
14 *  rsi source
15 *  rdx count
16 *
17 * Output:
18 * rax original destination
19 */
20
21/*
22 * memcpy_c() - fast string ops (REP MOVSQ) based variant.
23 *
24 * This gets patched over the unrolled variant (below) via the
25 * alternative instructions framework:
26 */
27	.section .altinstr_replacement, "ax", @progbits
28.Lmemcpy_c:
29	movq %rdi, %rax
30	movq %rdx, %rcx
31	shrq $3, %rcx
32	andl $7, %edx
33	rep movsq
34	movl %edx, %ecx
35	rep movsb
36	ret
37.Lmemcpy_e:
38	.previous
39
40/*
41 * memcpy_c_e() - enhanced fast string memcpy. This is faster and simpler than
42 * memcpy_c. Use memcpy_c_e when possible.
43 *
44 * This gets patched over the unrolled variant (below) via the
45 * alternative instructions framework:
46 */
47	.section .altinstr_replacement, "ax", @progbits
48.Lmemcpy_c_e:
49	movq %rdi, %rax
50	movq %rdx, %rcx
51	rep movsb
52	ret
53.Lmemcpy_e_e:
54	.previous
55
56ENTRY(__memcpy)
57ENTRY(memcpy)
58	CFI_STARTPROC
59	movq %rdi, %rax
60
61	cmpq $0x20, %rdx
62	jb .Lhandle_tail
63
64	/*
65	 * We check whether memory false dependence could occur,
66	 * then jump to corresponding copy mode.
67	 */
68	cmp  %dil, %sil
69	jl .Lcopy_backward
70	subq $0x20, %rdx
71.Lcopy_forward_loop:
72	subq $0x20,	%rdx
73
74	/*
75	 * Move in blocks of 4x8 bytes:
76	 */
77	movq 0*8(%rsi),	%r8
78	movq 1*8(%rsi),	%r9
79	movq 2*8(%rsi),	%r10
80	movq 3*8(%rsi),	%r11
81	leaq 4*8(%rsi),	%rsi
82
83	movq %r8,	0*8(%rdi)
84	movq %r9,	1*8(%rdi)
85	movq %r10,	2*8(%rdi)
86	movq %r11,	3*8(%rdi)
87	leaq 4*8(%rdi),	%rdi
88	jae  .Lcopy_forward_loop
89	addl $0x20,	%edx
90	jmp  .Lhandle_tail
91
92.Lcopy_backward:
93	/*
94	 * Calculate copy position to tail.
95	 */
96	addq %rdx,	%rsi
97	addq %rdx,	%rdi
98	subq $0x20,	%rdx
99	/*
100	 * At most 3 ALU operations in one cycle,
101	 * so append NOPS in the same 16 bytes trunk.
102	 */
103	.p2align 4
104.Lcopy_backward_loop:
105	subq $0x20,	%rdx
106	movq -1*8(%rsi),	%r8
107	movq -2*8(%rsi),	%r9
108	movq -3*8(%rsi),	%r10
109	movq -4*8(%rsi),	%r11
110	leaq -4*8(%rsi),	%rsi
111	movq %r8,		-1*8(%rdi)
112	movq %r9,		-2*8(%rdi)
113	movq %r10,		-3*8(%rdi)
114	movq %r11,		-4*8(%rdi)
115	leaq -4*8(%rdi),	%rdi
116	jae  .Lcopy_backward_loop
117
118	/*
119	 * Calculate copy position to head.
120	 */
121	addl $0x20,	%edx
122	subq %rdx,	%rsi
123	subq %rdx,	%rdi
124.Lhandle_tail:
125	cmpl $16,	%edx
126	jb   .Lless_16bytes
127
128	/*
129	 * Move data from 16 bytes to 31 bytes.
130	 */
131	movq 0*8(%rsi), %r8
132	movq 1*8(%rsi),	%r9
133	movq -2*8(%rsi, %rdx),	%r10
134	movq -1*8(%rsi, %rdx),	%r11
135	movq %r8,	0*8(%rdi)
136	movq %r9,	1*8(%rdi)
137	movq %r10,	-2*8(%rdi, %rdx)
138	movq %r11,	-1*8(%rdi, %rdx)
139	retq
140	.p2align 4
141.Lless_16bytes:
142	cmpl $8,	%edx
143	jb   .Lless_8bytes
144	/*
145	 * Move data from 8 bytes to 15 bytes.
146	 */
147	movq 0*8(%rsi),	%r8
148	movq -1*8(%rsi, %rdx),	%r9
149	movq %r8,	0*8(%rdi)
150	movq %r9,	-1*8(%rdi, %rdx)
151	retq
152	.p2align 4
153.Lless_8bytes:
154	cmpl $4,	%edx
155	jb   .Lless_3bytes
156
157	/*
158	 * Move data from 4 bytes to 7 bytes.
159	 */
160	movl (%rsi), %ecx
161	movl -4(%rsi, %rdx), %r8d
162	movl %ecx, (%rdi)
163	movl %r8d, -4(%rdi, %rdx)
164	retq
165	.p2align 4
166.Lless_3bytes:
167	subl $1, %edx
168	jb .Lend
169	/*
170	 * Move data from 1 bytes to 3 bytes.
171	 */
172	movzbl (%rsi), %ecx
173	jz .Lstore_1byte
174	movzbq 1(%rsi), %r8
175	movzbq (%rsi, %rdx), %r9
176	movb %r8b, 1(%rdi)
177	movb %r9b, (%rdi, %rdx)
178.Lstore_1byte:
179	movb %cl, (%rdi)
180
181.Lend:
182	retq
183	CFI_ENDPROC
184ENDPROC(memcpy)
185ENDPROC(__memcpy)
186
187	/*
188	 * Some CPUs are adding enhanced REP MOVSB/STOSB feature
189	 * If the feature is supported, memcpy_c_e() is the first choice.
190	 * If enhanced rep movsb copy is not available, use fast string copy
191	 * memcpy_c() when possible. This is faster and code is simpler than
192	 * original memcpy().
193	 * Otherwise, original memcpy() is used.
194	 * In .altinstructions section, ERMS feature is placed after REG_GOOD
195         * feature to implement the right patch order.
196	 *
197	 * Replace only beginning, memcpy is used to apply alternatives,
198	 * so it is silly to overwrite itself with nops - reboot is the
199	 * only outcome...
200	 */
201	.section .altinstructions, "a"
202	altinstruction_entry memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\
203			     .Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c
204	altinstruction_entry memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \
205			     .Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e
206	.previous
207