1/*
2 * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are met:
6 *
7 * Redistributions of source code must retain the above copyright notice, this
8 * list of conditions and the following disclaimer.
9 *
10 * Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * Neither the name of ARM nor the names of its contributors may be used
15 * to endorse or promote products derived from this software without specific
16 * prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#include <arch.h>
32#include <asm_macros.S>
33
34	.globl	flush_dcache_range
35	.globl	inv_dcache_range
36	.globl	dcsw_op_louis
37	.globl	dcsw_op_all
38	.globl	dcsw_op_level1
39	.globl	dcsw_op_level2
40	.globl	dcsw_op_level3
41
42	/* ------------------------------------------
43	 * Clean+Invalidate from base address till
44	 * size. 'x0' = addr, 'x1' = size
45	 * ------------------------------------------
46	 */
47func flush_dcache_range
48	dcache_line_size x2, x3
49	add	x1, x0, x1
50	sub	x3, x2, #1
51	bic	x0, x0, x3
52flush_loop:
53	dc	civac, x0
54	add	x0, x0, x2
55	cmp	x0, x1
56	b.lo    flush_loop
57	dsb	sy
58	ret
59
60
61	/* ------------------------------------------
62	 * Invalidate from base address till
63	 * size. 'x0' = addr, 'x1' = size
64	 * ------------------------------------------
65	 */
66func inv_dcache_range
67	dcache_line_size x2, x3
68	add	x1, x0, x1
69	sub	x3, x2, #1
70	bic	x0, x0, x3
71inv_loop:
72	dc	ivac, x0
73	add	x0, x0, x2
74	cmp	x0, x1
75	b.lo    inv_loop
76	dsb	sy
77	ret
78
79
80	/* ---------------------------------------------------------------
81	 * Data cache operations by set/way to the level specified
82	 *
83	 * The main function, do_dcsw_op requires:
84	 * x0: The operation type (0-2), as defined in arch.h
85	 * x3: The last cache level to operate on
86	 * x9: clidr_el1
87	 * x10: The cache level to begin operation from
88	 * and will carry out the operation on each data cache from level 0
89	 * to the level in x3 in sequence
90	 *
91	 * The dcsw_op macro sets up the x3 and x9 parameters based on
92	 * clidr_el1 cache information before invoking the main function
93	 * ---------------------------------------------------------------
94	 */
95
96	.macro	dcsw_op shift, fw, ls
97	mrs	x9, clidr_el1
98	ubfx	x3, x9, \shift, \fw
99	lsl	x3, x3, \ls
100	mov	x10, xzr
101	b	do_dcsw_op
102	.endm
103
104func do_dcsw_op
105	cbz	x3, exit
106	adr	x14, dcsw_loop_table	// compute inner loop address
107	add	x14, x14, x0, lsl #5	// inner loop is 8x32-bit instructions
108	mov	x0, x9
109	mov	w8, #1
110loop1:
111	add	x2, x10, x10, lsr #1	// work out 3x current cache level
112	lsr	x1, x0, x2		// extract cache type bits from clidr
113	and	x1, x1, #7		// mask the bits for current cache only
114	cmp	x1, #2			// see what cache we have at this level
115	b.lt	level_done		// nothing to do if no cache or icache
116
117	msr	csselr_el1, x10		// select current cache level in csselr
118	isb				// isb to sych the new cssr&csidr
119	mrs	x1, ccsidr_el1		// read the new ccsidr
120	and	x2, x1, #7		// extract the length of the cache lines
121	add	x2, x2, #4		// add 4 (line length offset)
122	ubfx	x4, x1, #3, #10		// maximum way number
123	clz	w5, w4			// bit position of way size increment
124	lsl	w9, w4, w5		// w9 = aligned max way number
125	lsl	w16, w8, w5		// w16 = way number loop decrement
126	orr	w9, w10, w9		// w9 = combine way and cache number
127	ubfx	w6, w1, #13, #15	// w6 = max set number
128	lsl	w17, w8, w2		// w17 = set number loop decrement
129	dsb	sy			// barrier before we start this level
130	br	x14			// jump to DC operation specific loop
131
132	.macro	dcsw_loop _op
133loop2_\_op:
134	lsl	w7, w6, w2		// w7 = aligned max set number
135
136loop3_\_op:
137	orr	w11, w9, w7		// combine cache, way and set number
138	dc	\_op, x11
139	subs	w7, w7, w17		// decrement set number
140	b.ge	loop3_\_op
141
142	subs	x9, x9, x16		// decrement way number
143	b.ge	loop2_\_op
144
145	b	level_done
146	.endm
147
148level_done:
149	add	x10, x10, #2		// increment cache number
150	cmp	x3, x10
151	b.gt    loop1
152	msr	csselr_el1, xzr		// select cache level 0 in csselr
153	dsb	sy			// barrier to complete final cache operation
154	isb
155exit:
156	ret
157
158dcsw_loop_table:
159	dcsw_loop isw
160	dcsw_loop cisw
161	dcsw_loop csw
162
163
164func dcsw_op_louis
165	dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
166
167
168func dcsw_op_all
169	dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
170
171	/* ---------------------------------------------------------------
172	 *  Helper macro for data cache operations by set/way for the
173	 *  level specified
174	 * ---------------------------------------------------------------
175	 */
176	.macro dcsw_op_level level
177	mrs	x9, clidr_el1
178	mov	x3, \level
179	sub	x10, x3, #2
180	b	do_dcsw_op
181	.endm
182
183	/* ---------------------------------------------------------------
184	 * Data cache operations by set/way for level 1 cache
185	 *
186	 * The main function, do_dcsw_op requires:
187	 * x0: The operation type (0-2), as defined in arch.h
188	 * ---------------------------------------------------------------
189	 */
190func dcsw_op_level1
191	dcsw_op_level #(1 << LEVEL_SHIFT)
192
193	/* ---------------------------------------------------------------
194	 * Data cache operations by set/way for level 2 cache
195	 *
196	 * The main function, do_dcsw_op requires:
197	 * x0: The operation type (0-2), as defined in arch.h
198	 * ---------------------------------------------------------------
199	 */
200func dcsw_op_level2
201	dcsw_op_level #(2 << LEVEL_SHIFT)
202
203	/* ---------------------------------------------------------------
204	 * Data cache operations by set/way for level 3 cache
205	 *
206	 * The main function, do_dcsw_op requires:
207	 * x0: The operation type (0-2), as defined in arch.h
208	 * ---------------------------------------------------------------
209	 */
210func dcsw_op_level3
211	dcsw_op_level #(3 << LEVEL_SHIFT)
212