1/*
2 * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7#include <arch.h>
8#include <asm_macros.S>
9
10	.globl	flush_dcache_range
11	.globl	clean_dcache_range
12	.globl	inv_dcache_range
13	.globl	dcsw_op_louis
14	.globl	dcsw_op_all
15	.globl	dcsw_op_level1
16	.globl	dcsw_op_level2
17	.globl	dcsw_op_level3
18
19/*
20 * This macro can be used for implementing various data cache operations `op`
21 */
22.macro do_dcache_maintenance_by_mva op
23	/* Exit early if size is zero */
24	cbz	x1, exit_loop_\op
25	dcache_line_size x2, x3
26	add	x1, x0, x1
27	sub	x3, x2, #1
28	bic	x0, x0, x3
29loop_\op:
30	dc	\op, x0
31	add	x0, x0, x2
32	cmp	x0, x1
33	b.lo    loop_\op
34	dsb	sy
35exit_loop_\op:
36	ret
37.endm
38	/* ------------------------------------------
39	 * Clean+Invalidate from base address till
40	 * size. 'x0' = addr, 'x1' = size
41	 * ------------------------------------------
42	 */
43func flush_dcache_range
44	do_dcache_maintenance_by_mva civac
45endfunc flush_dcache_range
46
47	/* ------------------------------------------
48	 * Clean from base address till size.
49	 * 'x0' = addr, 'x1' = size
50	 * ------------------------------------------
51	 */
52func clean_dcache_range
53	do_dcache_maintenance_by_mva cvac
54endfunc clean_dcache_range
55
56	/* ------------------------------------------
57	 * Invalidate from base address till
58	 * size. 'x0' = addr, 'x1' = size
59	 * ------------------------------------------
60	 */
61func inv_dcache_range
62	do_dcache_maintenance_by_mva ivac
63endfunc inv_dcache_range
64
65
66	/* ---------------------------------------------------------------
67	 * Data cache operations by set/way to the level specified
68	 *
69	 * The main function, do_dcsw_op requires:
70	 * x0: The operation type (0-2), as defined in arch.h
71	 * x3: The last cache level to operate on
72	 * x9: clidr_el1
73	 * x10: The cache level to begin operation from
74	 * and will carry out the operation on each data cache from level 0
75	 * to the level in x3 in sequence
76	 *
77	 * The dcsw_op macro sets up the x3 and x9 parameters based on
78	 * clidr_el1 cache information before invoking the main function
79	 * ---------------------------------------------------------------
80	 */
81
82	.macro	dcsw_op shift, fw, ls
83	mrs	x9, clidr_el1
84	ubfx	x3, x9, \shift, \fw
85	lsl	x3, x3, \ls
86	mov	x10, xzr
87	b	do_dcsw_op
88	.endm
89
90func do_dcsw_op
91	cbz	x3, exit
92	adr	x14, dcsw_loop_table	// compute inner loop address
93	add	x14, x14, x0, lsl #5	// inner loop is 8x32-bit instructions
94	mov	x0, x9
95	mov	w8, #1
96loop1:
97	add	x2, x10, x10, lsr #1	// work out 3x current cache level
98	lsr	x1, x0, x2		// extract cache type bits from clidr
99	and	x1, x1, #7		// mask the bits for current cache only
100	cmp	x1, #2			// see what cache we have at this level
101	b.lo	level_done		// nothing to do if no cache or icache
102
103	msr	csselr_el1, x10		// select current cache level in csselr
104	isb				// isb to sych the new cssr&csidr
105	mrs	x1, ccsidr_el1		// read the new ccsidr
106	and	x2, x1, #7		// extract the length of the cache lines
107	add	x2, x2, #4		// add 4 (line length offset)
108	ubfx	x4, x1, #3, #10		// maximum way number
109	clz	w5, w4			// bit position of way size increment
110	lsl	w9, w4, w5		// w9 = aligned max way number
111	lsl	w16, w8, w5		// w16 = way number loop decrement
112	orr	w9, w10, w9		// w9 = combine way and cache number
113	ubfx	w6, w1, #13, #15	// w6 = max set number
114	lsl	w17, w8, w2		// w17 = set number loop decrement
115	dsb	sy			// barrier before we start this level
116	br	x14			// jump to DC operation specific loop
117
118	.macro	dcsw_loop _op
119loop2_\_op:
120	lsl	w7, w6, w2		// w7 = aligned max set number
121
122loop3_\_op:
123	orr	w11, w9, w7		// combine cache, way and set number
124	dc	\_op, x11
125	subs	w7, w7, w17		// decrement set number
126	b.hs	loop3_\_op
127
128	subs	x9, x9, x16		// decrement way number
129	b.hs	loop2_\_op
130
131	b	level_done
132	.endm
133
134level_done:
135	add	x10, x10, #2		// increment cache number
136	cmp	x3, x10
137	b.hi    loop1
138	msr	csselr_el1, xzr		// select cache level 0 in csselr
139	dsb	sy			// barrier to complete final cache operation
140	isb
141exit:
142	ret
143endfunc do_dcsw_op
144
145dcsw_loop_table:
146	dcsw_loop isw
147	dcsw_loop cisw
148	dcsw_loop csw
149
150
151func dcsw_op_louis
152	dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
153endfunc dcsw_op_louis
154
155
156func dcsw_op_all
157	dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
158endfunc dcsw_op_all
159
160	/* ---------------------------------------------------------------
161	 *  Helper macro for data cache operations by set/way for the
162	 *  level specified
163	 * ---------------------------------------------------------------
164	 */
165	.macro dcsw_op_level level
166	mrs	x9, clidr_el1
167	mov	x3, \level
168	sub	x10, x3, #2
169	b	do_dcsw_op
170	.endm
171
172	/* ---------------------------------------------------------------
173	 * Data cache operations by set/way for level 1 cache
174	 *
175	 * The main function, do_dcsw_op requires:
176	 * x0: The operation type (0-2), as defined in arch.h
177	 * ---------------------------------------------------------------
178	 */
179func dcsw_op_level1
180	dcsw_op_level #(1 << LEVEL_SHIFT)
181endfunc dcsw_op_level1
182
183	/* ---------------------------------------------------------------
184	 * Data cache operations by set/way for level 2 cache
185	 *
186	 * The main function, do_dcsw_op requires:
187	 * x0: The operation type (0-2), as defined in arch.h
188	 * ---------------------------------------------------------------
189	 */
190func dcsw_op_level2
191	dcsw_op_level #(2 << LEVEL_SHIFT)
192endfunc dcsw_op_level2
193
194	/* ---------------------------------------------------------------
195	 * Data cache operations by set/way for level 3 cache
196	 *
197	 * The main function, do_dcsw_op requires:
198	 * x0: The operation type (0-2), as defined in arch.h
199	 * ---------------------------------------------------------------
200	 */
201func dcsw_op_level3
202	dcsw_op_level #(3 << LEVEL_SHIFT)
203endfunc dcsw_op_level3
204