1/*
2 * Copyright (c) 2017-2019, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6#include <arch.h>
7#include <asm_macros.S>
8#include <assert_macros.S>
9#include <common/debug.h>
10#include <cortex_a57.h>
11#include <cpu_macros.S>
12
13	/* ---------------------------------------------
14	 * Disable intra-cluster coherency
15	 * Clobbers: r0-r1
16	 * ---------------------------------------------
17	 */
18func cortex_a57_disable_smp
19	ldcopr16	r0, r1, CORTEX_A57_ECTLR
20	bic64_imm	r0, r1, CORTEX_A57_ECTLR_SMP_BIT
21	stcopr16	r0, r1, CORTEX_A57_ECTLR
22	bx	lr
23endfunc cortex_a57_disable_smp
24
25	/* ---------------------------------------------
26	 * Disable all types of L2 prefetches.
27	 * Clobbers: r0-r2
28	 * ---------------------------------------------
29	 */
30func cortex_a57_disable_l2_prefetch
31	ldcopr16	r0, r1, CORTEX_A57_ECTLR
32	orr64_imm	r0, r1, CORTEX_A57_ECTLR_DIS_TWD_ACC_PFTCH_BIT
33	bic64_imm	r0, r1, (CORTEX_A57_ECTLR_L2_IPFTCH_DIST_MASK | \
34				 CORTEX_A57_ECTLR_L2_DPFTCH_DIST_MASK)
35	stcopr16	r0, r1, CORTEX_A57_ECTLR
36	isb
37	dsb	ish
38	bx	lr
39endfunc cortex_a57_disable_l2_prefetch
40
41	/* ---------------------------------------------
42	 * Disable debug interfaces
43	 * ---------------------------------------------
44	 */
45func cortex_a57_disable_ext_debug
46	mov	r0, #1
47	stcopr	r0, DBGOSDLR
48	isb
49#if ERRATA_A57_817169
50	/*
51	 * Invalidate any TLB address
52	 */
53	mov	r0, #0
54	stcopr	r0, TLBIMVA
55#endif
56	dsb	sy
57	bx	lr
58endfunc cortex_a57_disable_ext_debug
59
60	/* --------------------------------------------------
61	 * Errata Workaround for Cortex A57 Errata #806969.
62	 * This applies only to revision r0p0 of Cortex A57.
63	 * Inputs:
64	 * r0: variant[4:7] and revision[0:3] of current cpu.
65	 * Shall clobber: r0-r3
66	 * --------------------------------------------------
67	 */
68func errata_a57_806969_wa
69	/*
70	 * Compare r0 against revision r0p0
71	 */
72	mov		r2, lr
73	bl		check_errata_806969
74	mov		lr, r2
75	cmp		r0, #ERRATA_NOT_APPLIES
76	beq		1f
77	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
78	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_NO_ALLOC_WBWA
79	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
801:
81	bx	lr
82endfunc errata_a57_806969_wa
83
84func check_errata_806969
85	mov	r1, #0x00
86	b	cpu_rev_var_ls
87endfunc check_errata_806969
88
89	/* ---------------------------------------------------
90	 * Errata Workaround for Cortex A57 Errata #813419.
91	 * This applies only to revision r0p0 of Cortex A57.
92	 * ---------------------------------------------------
93	 */
94func check_errata_813419
95	/*
96	 * Even though this is only needed for revision r0p0, it
97	 * is always applied due to limitations of the current
98	 * errata framework.
99	 */
100	mov	r0, #ERRATA_APPLIES
101	bx	lr
102endfunc check_errata_813419
103
104	/* ---------------------------------------------------
105	 * Errata Workaround for Cortex A57 Errata #813420.
106	 * This applies only to revision r0p0 of Cortex A57.
107	 * Inputs:
108	 * r0: variant[4:7] and revision[0:3] of current cpu.
109	 * Shall clobber: r0-r3
110	 * ---------------------------------------------------
111	 */
112func errata_a57_813420_wa
113	/*
114	 * Compare r0 against revision r0p0
115	 */
116	mov		r2, lr
117	bl		check_errata_813420
118	mov		lr, r2
119	cmp		r0, #ERRATA_NOT_APPLIES
120	beq		1f
121	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
122	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DCC_AS_DCCI
123	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
1241:
125	bx		lr
126endfunc errata_a57_813420_wa
127
128func check_errata_813420
129	mov	r1, #0x00
130	b	cpu_rev_var_ls
131endfunc check_errata_813420
132
133	/* ---------------------------------------------------
134	 * Errata Workaround for Cortex A57 Errata #814670.
135	 * This applies only to revision r0p0 of Cortex A57.
136	 * Inputs:
137	 * r0: variant[4:7] and revision[0:3] of current cpu.
138	 * Shall clobber: r0-r3
139	 * ---------------------------------------------------
140	 */
141func errata_a57_814670_wa
142	/*
143	 * Compare r0 against revision r0p0
144	 */
145	mov		r2, lr
146	bl		check_errata_814670
147	cmp		r0, #ERRATA_NOT_APPLIES
148	beq		1f
149	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
150	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_DMB_NULLIFICATION
151	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
152	isb
1531:
154	bx		r2
155endfunc errata_a57_814670_wa
156
157func check_errata_814670
158	mov	r1, #0x00
159	b	cpu_rev_var_ls
160endfunc check_errata_814670
161
162	/* ----------------------------------------------------
163	 * Errata Workaround for Cortex A57 Errata #817169.
164	 * This applies only to revision <= r0p1 of Cortex A57.
165	 * ----------------------------------------------------
166	 */
167func check_errata_817169
168	/*
169	 * Even though this is only needed for revision <= r0p1, it
170	 * is always applied because of the low cost of the workaround.
171	 */
172	mov	r0, #ERRATA_APPLIES
173	bx	lr
174endfunc check_errata_817169
175
176	/* --------------------------------------------------------------------
177	 * Disable the over-read from the LDNP instruction.
178	 *
179	 * This applies to all revisions <= r1p2. The performance degradation
180	 * observed with LDNP/STNP has been fixed on r1p3 and onwards.
181	 *
182	 * Inputs:
183	 * r0: variant[4:7] and revision[0:3] of current cpu.
184	 * Shall clobber: r0-r3
185	 * ---------------------------------------------------------------------
186	 */
187func a57_disable_ldnp_overread
188	/*
189	 * Compare r0 against revision r1p2
190	 */
191	mov		r2, lr
192	bl		check_errata_disable_ldnp_overread
193	mov		lr, r2
194	cmp		r0, #ERRATA_NOT_APPLIES
195	beq		1f
196	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
197	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_OVERREAD
198	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
1991:
200	bx		lr
201endfunc a57_disable_ldnp_overread
202
203func check_errata_disable_ldnp_overread
204	mov	r1, #0x12
205	b	cpu_rev_var_ls
206endfunc check_errata_disable_ldnp_overread
207
208	/* ---------------------------------------------------
209	 * Errata Workaround for Cortex A57 Errata #826974.
210	 * This applies only to revision <= r1p1 of Cortex A57.
211	 * Inputs:
212	 * r0: variant[4:7] and revision[0:3] of current cpu.
213	 * Shall clobber: r0-r3
214	 * ---------------------------------------------------
215	 */
216func errata_a57_826974_wa
217	/*
218	 * Compare r0 against revision r1p1
219	 */
220	mov		r2, lr
221	bl		check_errata_826974
222	mov		lr, r2
223	cmp		r0, #ERRATA_NOT_APPLIES
224	beq		1f
225	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
226	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_LOAD_PASS_DMB
227	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
2281:
229	bx		lr
230endfunc errata_a57_826974_wa
231
232func check_errata_826974
233	mov	r1, #0x11
234	b	cpu_rev_var_ls
235endfunc check_errata_826974
236
237	/* ---------------------------------------------------
238	 * Errata Workaround for Cortex A57 Errata #826977.
239	 * This applies only to revision <= r1p1 of Cortex A57.
240	 * Inputs:
241	 * r0: variant[4:7] and revision[0:3] of current cpu.
242	 * Shall clobber: r0-r3
243	 * ---------------------------------------------------
244	 */
245func errata_a57_826977_wa
246	/*
247	 * Compare r0 against revision r1p1
248	 */
249	mov		r2, lr
250	bl		check_errata_826977
251	mov		lr, r2
252	cmp		r0, #ERRATA_NOT_APPLIES
253	beq		1f
254	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
255	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_GRE_NGRE_AS_NGNRE
256	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
2571:
258	bx		lr
259endfunc errata_a57_826977_wa
260
261func check_errata_826977
262	mov	r1, #0x11
263	b	cpu_rev_var_ls
264endfunc check_errata_826977
265
266	/* ---------------------------------------------------
267	 * Errata Workaround for Cortex A57 Errata #828024.
268	 * This applies only to revision <= r1p1 of Cortex A57.
269	 * Inputs:
270	 * r0: variant[4:7] and revision[0:3] of current cpu.
271	 * Shall clobber: r0-r3
272	 * ---------------------------------------------------
273	 */
274func errata_a57_828024_wa
275	/*
276	 * Compare r0 against revision r1p1
277	 */
278	mov		r2, lr
279	bl		check_errata_828024
280	mov		lr, r2
281	cmp		r0, #ERRATA_NOT_APPLIES
282	beq		1f
283	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
284	/*
285	 * Setting the relevant bits in CORTEX_A57_CPUACTLR has to be done in 2
286	 * instructions here because the resulting bitmask doesn't fit in a
287	 * 16-bit value so it cannot be encoded in a single instruction.
288	 */
289	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_NO_ALLOC_WBWA
290	orr64_imm	r0, r1, (CORTEX_A57_CPUACTLR_DIS_L1_STREAMING | CORTEX_A57_CPUACTLR_DIS_STREAMING)
291	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
2921:
293	bx		lr
294endfunc errata_a57_828024_wa
295
296func check_errata_828024
297	mov	r1, #0x11
298	b	cpu_rev_var_ls
299endfunc check_errata_828024
300
301	/* ---------------------------------------------------
302	 * Errata Workaround for Cortex A57 Errata #829520.
303	 * This applies only to revision <= r1p2 of Cortex A57.
304	 * Inputs:
305	 * r0: variant[4:7] and revision[0:3] of current cpu.
306	 * Shall clobber: r0-r3
307	 * ---------------------------------------------------
308	 */
309func errata_a57_829520_wa
310	/*
311	 * Compare r0 against revision r1p2
312	 */
313	mov		r2, lr
314	bl		check_errata_829520
315	mov		lr, r2
316	cmp		r0, #ERRATA_NOT_APPLIES
317	beq		1f
318	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
319	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_INDIRECT_PREDICTOR
320	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
3211:
322	bx		lr
323endfunc errata_a57_829520_wa
324
325func check_errata_829520
326	mov	r1, #0x12
327	b	cpu_rev_var_ls
328endfunc check_errata_829520
329
330	/* ---------------------------------------------------
331	 * Errata Workaround for Cortex A57 Errata #833471.
332	 * This applies only to revision <= r1p2 of Cortex A57.
333	 * Inputs:
334	 * r0: variant[4:7] and revision[0:3] of current cpu.
335	 * Shall clobber: r0-r3
336	 * ---------------------------------------------------
337	 */
338func errata_a57_833471_wa
339	/*
340	 * Compare r0 against revision r1p2
341	 */
342	mov		r2, lr
343	bl		check_errata_833471
344	mov		lr, r2
345	cmp		r0, #ERRATA_NOT_APPLIES
346	beq		1f
347	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
348	orr64_imm	r1, r1, CORTEX_A57_CPUACTLR_FORCE_FPSCR_FLUSH
349	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
3501:
351	bx		lr
352endfunc errata_a57_833471_wa
353
354func check_errata_833471
355	mov	r1, #0x12
356	b	cpu_rev_var_ls
357endfunc check_errata_833471
358
359	/* ---------------------------------------------------
360	 * Errata Workaround for Cortex A57 Errata #859972.
361	 * This applies only to revision <= r1p3 of Cortex A57.
362	 * Inputs:
363	 * r0: variant[4:7] and revision[0:3] of current cpu.
364	 * Shall clobber: r0-r3
365	 * ---------------------------------------------------
366	 */
367func errata_a57_859972_wa
368	mov		r2, lr
369	bl		check_errata_859972
370	mov		lr, r2
371	cmp		r0, #ERRATA_NOT_APPLIES
372	beq		1f
373	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
374	orr64_imm	r1, r1, CORTEX_A57_CPUACTLR_DIS_INSTR_PREFETCH
375	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
3761:
377	bx		lr
378endfunc errata_a57_859972_wa
379
380func check_errata_859972
381	mov	r1, #0x13
382	b	cpu_rev_var_ls
383endfunc check_errata_859972
384
385func check_errata_cve_2017_5715
386	mov	r0, #ERRATA_MISSING
387	bx	lr
388endfunc check_errata_cve_2017_5715
389
390func check_errata_cve_2018_3639
391#if WORKAROUND_CVE_2018_3639
392	mov	r0, #ERRATA_APPLIES
393#else
394	mov	r0, #ERRATA_MISSING
395#endif
396	bx	lr
397endfunc check_errata_cve_2018_3639
398
399	/* -------------------------------------------------
400	 * The CPU Ops reset function for Cortex-A57.
401	 * Shall clobber: r0-r6
402	 * -------------------------------------------------
403	 */
404func cortex_a57_reset_func
405	mov	r5, lr
406	bl	cpu_get_rev_var
407	mov	r4, r0
408
409#if ERRATA_A57_806969
410	mov	r0, r4
411	bl	errata_a57_806969_wa
412#endif
413
414#if ERRATA_A57_813420
415	mov	r0, r4
416	bl	errata_a57_813420_wa
417#endif
418
419#if ERRATA_A57_814670
420	mov	r0, r4
421	bl	errata_a57_814670_wa
422#endif
423
424#if A57_DISABLE_NON_TEMPORAL_HINT
425	mov	r0, r4
426	bl	a57_disable_ldnp_overread
427#endif
428
429#if ERRATA_A57_826974
430	mov	r0, r4
431	bl	errata_a57_826974_wa
432#endif
433
434#if ERRATA_A57_826977
435	mov	r0, r4
436	bl	errata_a57_826977_wa
437#endif
438
439#if ERRATA_A57_828024
440	mov	r0, r4
441	bl	errata_a57_828024_wa
442#endif
443
444#if ERRATA_A57_829520
445	mov	r0, r4
446	bl	errata_a57_829520_wa
447#endif
448
449#if ERRATA_A57_833471
450	mov	r0, r4
451	bl	errata_a57_833471_wa
452#endif
453
454#if ERRATA_A57_859972
455	mov	r0, r4
456	bl	errata_a57_859972_wa
457#endif
458
459#if WORKAROUND_CVE_2018_3639
460	ldcopr16	r0, r1, CORTEX_A57_CPUACTLR
461	orr64_imm	r0, r1, CORTEX_A57_CPUACTLR_DIS_LOAD_PASS_STORE
462	stcopr16	r0, r1, CORTEX_A57_CPUACTLR
463	isb
464	dsb	sy
465#endif
466
467	/* ---------------------------------------------
468	 * Enable the SMP bit.
469	 * ---------------------------------------------
470	 */
471	ldcopr16	r0, r1, CORTEX_A57_ECTLR
472	orr64_imm	r0, r1, CORTEX_A57_ECTLR_SMP_BIT
473	stcopr16	r0, r1,	CORTEX_A57_ECTLR
474	isb
475	bx	r5
476endfunc cortex_a57_reset_func
477
478	/* ----------------------------------------------------
479	 * The CPU Ops core power down function for Cortex-A57.
480	 * ----------------------------------------------------
481	 */
482func cortex_a57_core_pwr_dwn
483	push	{r12, lr}
484
485	/* Assert if cache is enabled */
486#if ENABLE_ASSERTIONS
487	ldcopr	r0, SCTLR
488	tst	r0, #SCTLR_C_BIT
489	ASM_ASSERT(eq)
490#endif
491
492	/* ---------------------------------------------
493	 * Disable the L2 prefetches.
494	 * ---------------------------------------------
495	 */
496	bl	cortex_a57_disable_l2_prefetch
497
498	/* ---------------------------------------------
499	 * Flush L1 caches.
500	 * ---------------------------------------------
501	 */
502	mov	r0, #DC_OP_CISW
503	bl	dcsw_op_level1
504
505	/* ---------------------------------------------
506	 * Come out of intra cluster coherency
507	 * ---------------------------------------------
508	 */
509	bl	cortex_a57_disable_smp
510
511	/* ---------------------------------------------
512	 * Force the debug interfaces to be quiescent
513	 * ---------------------------------------------
514	 */
515	pop	{r12, lr}
516	b	cortex_a57_disable_ext_debug
517endfunc cortex_a57_core_pwr_dwn
518
519	/* -------------------------------------------------------
520	 * The CPU Ops cluster power down function for Cortex-A57.
521	 * Clobbers: r0-r3
522	 * -------------------------------------------------------
523	 */
524func cortex_a57_cluster_pwr_dwn
525	push	{r12, lr}
526
527	/* Assert if cache is enabled */
528#if ENABLE_ASSERTIONS
529	ldcopr	r0, SCTLR
530	tst	r0, #SCTLR_C_BIT
531	ASM_ASSERT(eq)
532#endif
533
534	/* ---------------------------------------------
535	 * Disable the L2 prefetches.
536	 * ---------------------------------------------
537	 */
538	bl	cortex_a57_disable_l2_prefetch
539
540	/* ---------------------------------------------
541	 * Flush L1 caches.
542	 * ---------------------------------------------
543	 */
544	mov	r0, #DC_OP_CISW
545	bl	dcsw_op_level1
546
547	/* ---------------------------------------------
548	 * Disable the optional ACP.
549	 * ---------------------------------------------
550	 */
551	bl	plat_disable_acp
552
553	/* ---------------------------------------------
554	 * Flush L2 caches.
555	 * ---------------------------------------------
556	 */
557	mov	r0, #DC_OP_CISW
558	bl	dcsw_op_level2
559
560	/* ---------------------------------------------
561	 * Come out of intra cluster coherency
562	 * ---------------------------------------------
563	 */
564	bl	cortex_a57_disable_smp
565
566	/* ---------------------------------------------
567	 * Force the debug interfaces to be quiescent
568	 * ---------------------------------------------
569	 */
570	pop	{r12, lr}
571	b	cortex_a57_disable_ext_debug
572endfunc cortex_a57_cluster_pwr_dwn
573
574#if REPORT_ERRATA
575/*
576 * Errata printing function for Cortex A57. Must follow AAPCS.
577 */
578func cortex_a57_errata_report
579	push	{r12, lr}
580
581	bl	cpu_get_rev_var
582	mov	r4, r0
583
584	/*
585	 * Report all errata. The revision-variant information is passed to
586	 * checking functions of each errata.
587	 */
588	report_errata ERRATA_A57_806969, cortex_a57, 806969
589	report_errata ERRATA_A57_813419, cortex_a57, 813419
590	report_errata ERRATA_A57_813420, cortex_a57, 813420
591	report_errata ERRATA_A57_814670, cortex_a57, 814670
592	report_errata ERRATA_A57_817169, cortex_a57, 817169
593	report_errata A57_DISABLE_NON_TEMPORAL_HINT, cortex_a57, \
594		disable_ldnp_overread
595	report_errata ERRATA_A57_826974, cortex_a57, 826974
596	report_errata ERRATA_A57_826977, cortex_a57, 826977
597	report_errata ERRATA_A57_828024, cortex_a57, 828024
598	report_errata ERRATA_A57_829520, cortex_a57, 829520
599	report_errata ERRATA_A57_833471, cortex_a57, 833471
600	report_errata ERRATA_A57_859972, cortex_a57, 859972
601	report_errata WORKAROUND_CVE_2017_5715, cortex_a57, cve_2017_5715
602	report_errata WORKAROUND_CVE_2018_3639, cortex_a57, cve_2018_3639
603
604	pop	{r12, lr}
605	bx	lr
606endfunc cortex_a57_errata_report
607#endif
608
609declare_cpu_ops cortex_a57, CORTEX_A57_MIDR, \
610	cortex_a57_reset_func, \
611	cortex_a57_core_pwr_dwn, \
612	cortex_a57_cluster_pwr_dwn
613