1 /*
2 * Copyright (c) 2015-2020, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8
9 #include <arch.h>
10 #include <arch_helpers.h>
11 #include <common/debug.h>
12 #include <common/interrupt_props.h>
13 #include <drivers/arm/gicv3.h>
14 #include <lib/spinlock.h>
15
16 #include "gicv3_private.h"
17
18 const gicv3_driver_data_t *gicv3_driver_data;
19
20 /*
21 * Spinlock to guard registers needing read-modify-write. APIs protected by this
22 * spinlock are used either at boot time (when only a single CPU is active), or
23 * when the system is fully coherent.
24 */
25 static spinlock_t gic_lock;
26
27 /*
28 * Redistributor power operations are weakly bound so that they can be
29 * overridden
30 */
31 #pragma weak gicv3_rdistif_off
32 #pragma weak gicv3_rdistif_on
33
34 /* Check interrupt ID for SGI/(E)PPI and (E)SPIs */
35 static bool is_sgi_ppi(unsigned int id);
36
37 /*
38 * Helper macros to save and restore GICR and GICD registers
39 * corresponding to their numbers to and from the context
40 */
41 #define RESTORE_GICR_REG(base, ctx, name, i) \
42 gicr_write_##name((base), (i), (ctx)->gicr_##name[(i)])
43
44 #define SAVE_GICR_REG(base, ctx, name, i) \
45 (ctx)->gicr_##name[(i)] = gicr_read_##name((base), (i))
46
47 /* Helper macros to save and restore GICD registers to and from the context */
48 #define RESTORE_GICD_REGS(base, ctx, intr_num, reg, REG) \
49 do { \
50 for (unsigned int int_id = MIN_SPI_ID; int_id < (intr_num);\
51 int_id += (1U << REG##R_SHIFT)) { \
52 gicd_write_##reg((base), int_id, \
53 (ctx)->gicd_##reg[(int_id - MIN_SPI_ID) >> \
54 REG##R_SHIFT]); \
55 } \
56 } while (false)
57
58 #define SAVE_GICD_REGS(base, ctx, intr_num, reg, REG) \
59 do { \
60 for (unsigned int int_id = MIN_SPI_ID; int_id < (intr_num);\
61 int_id += (1U << REG##R_SHIFT)) { \
62 (ctx)->gicd_##reg[(int_id - MIN_SPI_ID) >> \
63 REG##R_SHIFT] = gicd_read_##reg((base), int_id); \
64 } \
65 } while (false)
66
67 #if GIC_EXT_INTID
68 #define RESTORE_GICD_EREGS(base, ctx, intr_num, reg, REG) \
69 do { \
70 for (unsigned int int_id = MIN_ESPI_ID; int_id < (intr_num);\
71 int_id += (1U << REG##R_SHIFT)) { \
72 gicd_write_##reg((base), int_id, \
73 (ctx)->gicd_##reg[(int_id - (MIN_ESPI_ID - MIN_SPI_ID))\
74 >> REG##R_SHIFT]); \
75 } \
76 } while (false)
77
78 #define SAVE_GICD_EREGS(base, ctx, intr_num, reg, REG) \
79 do { \
80 for (unsigned int int_id = MIN_ESPI_ID; int_id < (intr_num);\
81 int_id += (1U << REG##R_SHIFT)) { \
82 (ctx)->gicd_##reg[(int_id - (MIN_ESPI_ID - MIN_SPI_ID))\
83 >> REG##R_SHIFT] = gicd_read_##reg((base), int_id);\
84 } \
85 } while (false)
86 #else
87 #define SAVE_GICD_EREGS(base, ctx, intr_num, reg, REG)
88 #define RESTORE_GICD_EREGS(base, ctx, intr_num, reg, REG)
89 #endif /* GIC_EXT_INTID */
90
91 /*******************************************************************************
92 * This function initialises the ARM GICv3 driver in EL3 with provided platform
93 * inputs.
94 ******************************************************************************/
gicv3_driver_init(const gicv3_driver_data_t * plat_driver_data)95 void __init gicv3_driver_init(const gicv3_driver_data_t *plat_driver_data)
96 {
97 unsigned int gic_version;
98 unsigned int gicv2_compat;
99
100 assert(plat_driver_data != NULL);
101 assert(plat_driver_data->gicd_base != 0U);
102 assert(plat_driver_data->rdistif_num != 0U);
103 assert(plat_driver_data->rdistif_base_addrs != NULL);
104
105 assert(IS_IN_EL3());
106
107 assert((plat_driver_data->interrupt_props_num != 0U) ?
108 (plat_driver_data->interrupt_props != NULL) : 1);
109
110 /* Check for system register support */
111 #ifndef __aarch64__
112 assert((read_id_pfr1() &
113 (ID_PFR1_GIC_MASK << ID_PFR1_GIC_SHIFT)) != 0U);
114 #else
115 assert((read_id_aa64pfr0_el1() &
116 (ID_AA64PFR0_GIC_MASK << ID_AA64PFR0_GIC_SHIFT)) != 0U);
117 #endif /* !__aarch64__ */
118
119 gic_version = gicd_read_pidr2(plat_driver_data->gicd_base);
120 gic_version >>= PIDR2_ARCH_REV_SHIFT;
121 gic_version &= PIDR2_ARCH_REV_MASK;
122
123 /* Check GIC version */
124 #if GIC_ENABLE_V4_EXTN
125 assert(gic_version == ARCH_REV_GICV4);
126
127 /* GICv4 supports Direct Virtual LPI injection */
128 assert((gicd_read_typer(plat_driver_data->gicd_base)
129 & TYPER_DVIS) != 0);
130 #else
131 assert(gic_version == ARCH_REV_GICV3);
132 #endif
133 /*
134 * Find out whether the GIC supports the GICv2 compatibility mode.
135 * The ARE_S bit resets to 0 if supported
136 */
137 gicv2_compat = gicd_read_ctlr(plat_driver_data->gicd_base);
138 gicv2_compat >>= CTLR_ARE_S_SHIFT;
139 gicv2_compat = gicv2_compat & CTLR_ARE_S_MASK;
140
141 if (plat_driver_data->gicr_base != 0U) {
142 /*
143 * Find the base address of each implemented Redistributor interface.
144 * The number of interfaces should be equal to the number of CPUs in the
145 * system. The memory for saving these addresses has to be allocated by
146 * the platform port
147 */
148 gicv3_rdistif_base_addrs_probe(plat_driver_data->rdistif_base_addrs,
149 plat_driver_data->rdistif_num,
150 plat_driver_data->gicr_base,
151 plat_driver_data->mpidr_to_core_pos);
152 #if !HW_ASSISTED_COHERENCY
153 /*
154 * Flush the rdistif_base_addrs[] contents linked to the GICv3 driver.
155 */
156 flush_dcache_range((uintptr_t)(plat_driver_data->rdistif_base_addrs),
157 plat_driver_data->rdistif_num *
158 sizeof(*(plat_driver_data->rdistif_base_addrs)));
159 #endif
160 }
161 gicv3_driver_data = plat_driver_data;
162
163 /*
164 * The GIC driver data is initialized by the primary CPU with caches
165 * enabled. When the secondary CPU boots up, it initializes the
166 * GICC/GICR interface with the caches disabled. Hence flush the
167 * driver data to ensure coherency. This is not required if the
168 * platform has HW_ASSISTED_COHERENCY enabled.
169 */
170 #if !HW_ASSISTED_COHERENCY
171 flush_dcache_range((uintptr_t)&gicv3_driver_data,
172 sizeof(gicv3_driver_data));
173 flush_dcache_range((uintptr_t)gicv3_driver_data,
174 sizeof(*gicv3_driver_data));
175 #endif
176 INFO("GICv%u with%s legacy support detected.\n", gic_version,
177 (gicv2_compat == 0U) ? "" : "out");
178 INFO("ARM GICv%u driver initialized in EL3\n", gic_version);
179 }
180
181 /*******************************************************************************
182 * This function initialises the GIC distributor interface based upon the data
183 * provided by the platform while initialising the driver.
184 ******************************************************************************/
gicv3_distif_init(void)185 void __init gicv3_distif_init(void)
186 {
187 unsigned int bitmap;
188
189 assert(gicv3_driver_data != NULL);
190 assert(gicv3_driver_data->gicd_base != 0U);
191
192 assert(IS_IN_EL3());
193
194 /*
195 * Clear the "enable" bits for G0/G1S/G1NS interrupts before configuring
196 * the ARE_S bit. The Distributor might generate a system error
197 * otherwise.
198 */
199 gicd_clr_ctlr(gicv3_driver_data->gicd_base,
200 CTLR_ENABLE_G0_BIT |
201 CTLR_ENABLE_G1S_BIT |
202 CTLR_ENABLE_G1NS_BIT,
203 RWP_TRUE);
204
205 /* Set the ARE_S and ARE_NS bit now that interrupts have been disabled */
206 gicd_set_ctlr(gicv3_driver_data->gicd_base,
207 CTLR_ARE_S_BIT | CTLR_ARE_NS_BIT, RWP_TRUE);
208
209 /* Set the default attribute of all (E)SPIs */
210 gicv3_spis_config_defaults(gicv3_driver_data->gicd_base);
211
212 bitmap = gicv3_secure_spis_config_props(
213 gicv3_driver_data->gicd_base,
214 gicv3_driver_data->interrupt_props,
215 gicv3_driver_data->interrupt_props_num);
216
217 /* Enable the secure (E)SPIs now that they have been configured */
218 gicd_set_ctlr(gicv3_driver_data->gicd_base, bitmap, RWP_TRUE);
219 }
220
221 /*******************************************************************************
222 * This function initialises the GIC Redistributor interface of the calling CPU
223 * (identified by the 'proc_num' parameter) based upon the data provided by the
224 * platform while initialising the driver.
225 ******************************************************************************/
gicv3_rdistif_init(unsigned int proc_num)226 void gicv3_rdistif_init(unsigned int proc_num)
227 {
228 uintptr_t gicr_base;
229 unsigned int bitmap;
230 uint32_t ctlr;
231
232 assert(gicv3_driver_data != NULL);
233 assert(proc_num < gicv3_driver_data->rdistif_num);
234 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
235 assert(gicv3_driver_data->gicd_base != 0U);
236
237 ctlr = gicd_read_ctlr(gicv3_driver_data->gicd_base);
238 assert((ctlr & CTLR_ARE_S_BIT) != 0U);
239
240 assert(IS_IN_EL3());
241
242 /* Power on redistributor */
243 gicv3_rdistif_on(proc_num);
244
245 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
246 assert(gicr_base != 0U);
247
248 /* Set the default attribute of all SGIs and (E)PPIs */
249 gicv3_ppi_sgi_config_defaults(gicr_base);
250
251 bitmap = gicv3_secure_ppi_sgi_config_props(gicr_base,
252 gicv3_driver_data->interrupt_props,
253 gicv3_driver_data->interrupt_props_num);
254
255 /* Enable interrupt groups as required, if not already */
256 if ((ctlr & bitmap) != bitmap) {
257 gicd_set_ctlr(gicv3_driver_data->gicd_base, bitmap, RWP_TRUE);
258 }
259 }
260
261 /*******************************************************************************
262 * Functions to perform power operations on GIC Redistributor
263 ******************************************************************************/
gicv3_rdistif_off(unsigned int proc_num)264 void gicv3_rdistif_off(unsigned int proc_num)
265 {
266 }
267
gicv3_rdistif_on(unsigned int proc_num)268 void gicv3_rdistif_on(unsigned int proc_num)
269 {
270 }
271
272 /*******************************************************************************
273 * This function enables the GIC CPU interface of the calling CPU using only
274 * system register accesses.
275 ******************************************************************************/
gicv3_cpuif_enable(unsigned int proc_num)276 void gicv3_cpuif_enable(unsigned int proc_num)
277 {
278 uintptr_t gicr_base;
279 u_register_t scr_el3;
280 unsigned int icc_sre_el3;
281
282 assert(gicv3_driver_data != NULL);
283 assert(proc_num < gicv3_driver_data->rdistif_num);
284 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
285 assert(IS_IN_EL3());
286
287 /* Mark the connected core as awake */
288 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
289 gicv3_rdistif_mark_core_awake(gicr_base);
290
291 /* Disable the legacy interrupt bypass */
292 icc_sre_el3 = ICC_SRE_DIB_BIT | ICC_SRE_DFB_BIT;
293
294 /*
295 * Enable system register access for EL3 and allow lower exception
296 * levels to configure the same for themselves. If the legacy mode is
297 * not supported, the SRE bit is RAO/WI
298 */
299 icc_sre_el3 |= (ICC_SRE_EN_BIT | ICC_SRE_SRE_BIT);
300 write_icc_sre_el3(read_icc_sre_el3() | icc_sre_el3);
301
302 scr_el3 = read_scr_el3();
303
304 /*
305 * Switch to NS state to write Non secure ICC_SRE_EL1 and
306 * ICC_SRE_EL2 registers.
307 */
308 write_scr_el3(scr_el3 | SCR_NS_BIT);
309 isb();
310
311 write_icc_sre_el2(read_icc_sre_el2() | icc_sre_el3);
312 write_icc_sre_el1(ICC_SRE_SRE_BIT);
313 isb();
314
315 /* Switch to secure state. */
316 write_scr_el3(scr_el3 & (~SCR_NS_BIT));
317 isb();
318
319 /* Write the secure ICC_SRE_EL1 register */
320 write_icc_sre_el1(ICC_SRE_SRE_BIT);
321 isb();
322
323 /* Program the idle priority in the PMR */
324 write_icc_pmr_el1(GIC_PRI_MASK);
325
326 /* Enable Group0 interrupts */
327 write_icc_igrpen0_el1(IGRPEN1_EL1_ENABLE_G0_BIT);
328
329 /* Enable Group1 Secure interrupts */
330 write_icc_igrpen1_el3(read_icc_igrpen1_el3() |
331 IGRPEN1_EL3_ENABLE_G1S_BIT);
332 isb();
333 }
334
335 /*******************************************************************************
336 * This function disables the GIC CPU interface of the calling CPU using
337 * only system register accesses.
338 ******************************************************************************/
gicv3_cpuif_disable(unsigned int proc_num)339 void gicv3_cpuif_disable(unsigned int proc_num)
340 {
341 uintptr_t gicr_base;
342
343 assert(gicv3_driver_data != NULL);
344 assert(proc_num < gicv3_driver_data->rdistif_num);
345 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
346
347 assert(IS_IN_EL3());
348
349 /* Disable legacy interrupt bypass */
350 write_icc_sre_el3(read_icc_sre_el3() |
351 (ICC_SRE_DIB_BIT | ICC_SRE_DFB_BIT));
352
353 /* Disable Group0 interrupts */
354 write_icc_igrpen0_el1(read_icc_igrpen0_el1() &
355 ~IGRPEN1_EL1_ENABLE_G0_BIT);
356
357 /* Disable Group1 Secure and Non-Secure interrupts */
358 write_icc_igrpen1_el3(read_icc_igrpen1_el3() &
359 ~(IGRPEN1_EL3_ENABLE_G1NS_BIT |
360 IGRPEN1_EL3_ENABLE_G1S_BIT));
361
362 /* Synchronise accesses to group enable registers */
363 isb();
364
365 /* Mark the connected core as asleep */
366 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
367 assert(gicr_base != 0U);
368 gicv3_rdistif_mark_core_asleep(gicr_base);
369 }
370
371 /*******************************************************************************
372 * This function returns the id of the highest priority pending interrupt at
373 * the GIC cpu interface.
374 ******************************************************************************/
gicv3_get_pending_interrupt_id(void)375 unsigned int gicv3_get_pending_interrupt_id(void)
376 {
377 unsigned int id;
378
379 assert(IS_IN_EL3());
380 id = (uint32_t)read_icc_hppir0_el1() & HPPIR0_EL1_INTID_MASK;
381
382 /*
383 * If the ID is special identifier corresponding to G1S or G1NS
384 * interrupt, then read the highest pending group 1 interrupt.
385 */
386 if ((id == PENDING_G1S_INTID) || (id == PENDING_G1NS_INTID)) {
387 return (uint32_t)read_icc_hppir1_el1() & HPPIR1_EL1_INTID_MASK;
388 }
389
390 return id;
391 }
392
393 /*******************************************************************************
394 * This function returns the type of the highest priority pending interrupt at
395 * the GIC cpu interface. The return values can be one of the following :
396 * PENDING_G1S_INTID : The interrupt type is secure Group 1.
397 * PENDING_G1NS_INTID : The interrupt type is non secure Group 1.
398 * 0 - 1019 : The interrupt type is secure Group 0.
399 * GIC_SPURIOUS_INTERRUPT : there is no pending interrupt with
400 * sufficient priority to be signaled
401 ******************************************************************************/
gicv3_get_pending_interrupt_type(void)402 unsigned int gicv3_get_pending_interrupt_type(void)
403 {
404 assert(IS_IN_EL3());
405 return (uint32_t)read_icc_hppir0_el1() & HPPIR0_EL1_INTID_MASK;
406 }
407
408 /*******************************************************************************
409 * This function returns the type of the interrupt id depending upon the group
410 * this interrupt has been configured under by the interrupt controller i.e.
411 * group0 or group1 Secure / Non Secure. The return value can be one of the
412 * following :
413 * INTR_GROUP0 : The interrupt type is a Secure Group 0 interrupt
414 * INTR_GROUP1S : The interrupt type is a Secure Group 1 secure interrupt
415 * INTR_GROUP1NS: The interrupt type is a Secure Group 1 non secure
416 * interrupt.
417 ******************************************************************************/
gicv3_get_interrupt_type(unsigned int id,unsigned int proc_num)418 unsigned int gicv3_get_interrupt_type(unsigned int id, unsigned int proc_num)
419 {
420 unsigned int igroup, grpmodr;
421 uintptr_t gicr_base;
422
423 assert(IS_IN_EL3());
424 assert(gicv3_driver_data != NULL);
425
426 /* Ensure the parameters are valid */
427 assert((id < PENDING_G1S_INTID) || (id >= MIN_LPI_ID));
428 assert(proc_num < gicv3_driver_data->rdistif_num);
429
430 /* All LPI interrupts are Group 1 non secure */
431 if (id >= MIN_LPI_ID) {
432 return INTR_GROUP1NS;
433 }
434
435 /* Check interrupt ID */
436 if (is_sgi_ppi(id)) {
437 /* SGIs: 0-15, PPIs: 16-31, EPPIs: 1056-1119 */
438 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
439 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
440 igroup = gicr_get_igroupr(gicr_base, id);
441 grpmodr = gicr_get_igrpmodr(gicr_base, id);
442 } else {
443 /* SPIs: 32-1019, ESPIs: 4096-5119 */
444 assert(gicv3_driver_data->gicd_base != 0U);
445 igroup = gicd_get_igroupr(gicv3_driver_data->gicd_base, id);
446 grpmodr = gicd_get_igrpmodr(gicv3_driver_data->gicd_base, id);
447 }
448
449 /*
450 * If the IGROUP bit is set, then it is a Group 1 Non secure
451 * interrupt
452 */
453 if (igroup != 0U) {
454 return INTR_GROUP1NS;
455 }
456
457 /* If the GRPMOD bit is set, then it is a Group 1 Secure interrupt */
458 if (grpmodr != 0U) {
459 return INTR_GROUP1S;
460 }
461
462 /* Else it is a Group 0 Secure interrupt */
463 return INTR_GROUP0;
464 }
465
466 /*****************************************************************************
467 * Function to save and disable the GIC ITS register context. The power
468 * management of GIC ITS is implementation-defined and this function doesn't
469 * save any memory structures required to support ITS. As the sequence to save
470 * this state is implementation defined, it should be executed in platform
471 * specific code. Calling this function alone and then powering down the GIC and
472 * ITS without implementing the aforementioned platform specific code will
473 * corrupt the ITS state.
474 *
475 * This function must be invoked after the GIC CPU interface is disabled.
476 *****************************************************************************/
gicv3_its_save_disable(uintptr_t gits_base,gicv3_its_ctx_t * const its_ctx)477 void gicv3_its_save_disable(uintptr_t gits_base,
478 gicv3_its_ctx_t * const its_ctx)
479 {
480 unsigned int i;
481
482 assert(gicv3_driver_data != NULL);
483 assert(IS_IN_EL3());
484 assert(its_ctx != NULL);
485 assert(gits_base != 0U);
486
487 its_ctx->gits_ctlr = gits_read_ctlr(gits_base);
488
489 /* Disable the ITS */
490 gits_write_ctlr(gits_base, its_ctx->gits_ctlr & ~GITS_CTLR_ENABLED_BIT);
491
492 /* Wait for quiescent state */
493 gits_wait_for_quiescent_bit(gits_base);
494
495 its_ctx->gits_cbaser = gits_read_cbaser(gits_base);
496 its_ctx->gits_cwriter = gits_read_cwriter(gits_base);
497
498 for (i = 0U; i < ARRAY_SIZE(its_ctx->gits_baser); i++) {
499 its_ctx->gits_baser[i] = gits_read_baser(gits_base, i);
500 }
501 }
502
503 /*****************************************************************************
504 * Function to restore the GIC ITS register context. The power
505 * management of GIC ITS is implementation defined and this function doesn't
506 * restore any memory structures required to support ITS. The assumption is
507 * that these structures are in memory and are retained during system suspend.
508 *
509 * This must be invoked before the GIC CPU interface is enabled.
510 *****************************************************************************/
gicv3_its_restore(uintptr_t gits_base,const gicv3_its_ctx_t * const its_ctx)511 void gicv3_its_restore(uintptr_t gits_base,
512 const gicv3_its_ctx_t * const its_ctx)
513 {
514 unsigned int i;
515
516 assert(gicv3_driver_data != NULL);
517 assert(IS_IN_EL3());
518 assert(its_ctx != NULL);
519 assert(gits_base != 0U);
520
521 /* Assert that the GITS is disabled and quiescent */
522 assert((gits_read_ctlr(gits_base) & GITS_CTLR_ENABLED_BIT) == 0U);
523 assert((gits_read_ctlr(gits_base) & GITS_CTLR_QUIESCENT_BIT) != 0U);
524
525 gits_write_cbaser(gits_base, its_ctx->gits_cbaser);
526 gits_write_cwriter(gits_base, its_ctx->gits_cwriter);
527
528 for (i = 0U; i < ARRAY_SIZE(its_ctx->gits_baser); i++) {
529 gits_write_baser(gits_base, i, its_ctx->gits_baser[i]);
530 }
531
532 /* Restore the ITS CTLR but leave the ITS disabled */
533 gits_write_ctlr(gits_base, its_ctx->gits_ctlr & ~GITS_CTLR_ENABLED_BIT);
534 }
535
536 /*****************************************************************************
537 * Function to save the GIC Redistributor register context. This function
538 * must be invoked after CPU interface disable and prior to Distributor save.
539 *****************************************************************************/
gicv3_rdistif_save(unsigned int proc_num,gicv3_redist_ctx_t * const rdist_ctx)540 void gicv3_rdistif_save(unsigned int proc_num,
541 gicv3_redist_ctx_t * const rdist_ctx)
542 {
543 uintptr_t gicr_base;
544 unsigned int i, ppi_regs_num, regs_num;
545
546 assert(gicv3_driver_data != NULL);
547 assert(proc_num < gicv3_driver_data->rdistif_num);
548 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
549 assert(IS_IN_EL3());
550 assert(rdist_ctx != NULL);
551
552 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
553
554 #if GIC_EXT_INTID
555 /* Calculate number of PPI registers */
556 ppi_regs_num = (unsigned int)((gicr_read_typer(gicr_base) >>
557 TYPER_PPI_NUM_SHIFT) & TYPER_PPI_NUM_MASK) + 1;
558 /* All other values except PPInum [0-2] are reserved */
559 if (ppi_regs_num > 3U) {
560 ppi_regs_num = 1U;
561 }
562 #else
563 ppi_regs_num = 1U;
564 #endif
565 /*
566 * Wait for any write to GICR_CTLR to complete before trying to save any
567 * state.
568 */
569 gicr_wait_for_pending_write(gicr_base);
570
571 rdist_ctx->gicr_ctlr = gicr_read_ctlr(gicr_base);
572
573 rdist_ctx->gicr_propbaser = gicr_read_propbaser(gicr_base);
574 rdist_ctx->gicr_pendbaser = gicr_read_pendbaser(gicr_base);
575
576 /* 32 interrupt IDs per register */
577 for (i = 0U; i < ppi_regs_num; ++i) {
578 SAVE_GICR_REG(gicr_base, rdist_ctx, igroupr, i);
579 SAVE_GICR_REG(gicr_base, rdist_ctx, isenabler, i);
580 SAVE_GICR_REG(gicr_base, rdist_ctx, ispendr, i);
581 SAVE_GICR_REG(gicr_base, rdist_ctx, isactiver, i);
582 SAVE_GICR_REG(gicr_base, rdist_ctx, igrpmodr, i);
583 }
584
585 /* 16 interrupt IDs per GICR_ICFGR register */
586 regs_num = ppi_regs_num << 1;
587 for (i = 0U; i < regs_num; ++i) {
588 SAVE_GICR_REG(gicr_base, rdist_ctx, icfgr, i);
589 }
590
591 rdist_ctx->gicr_nsacr = gicr_read_nsacr(gicr_base);
592
593 /* 4 interrupt IDs per GICR_IPRIORITYR register */
594 regs_num = ppi_regs_num << 3;
595 for (i = 0U; i < regs_num; ++i) {
596 rdist_ctx->gicr_ipriorityr[i] =
597 gicr_ipriorityr_read(gicr_base, i);
598 }
599
600 /*
601 * Call the pre-save hook that implements the IMP DEF sequence that may
602 * be required on some GIC implementations. As this may need to access
603 * the Redistributor registers, we pass it proc_num.
604 */
605 gicv3_distif_pre_save(proc_num);
606 }
607
608 /*****************************************************************************
609 * Function to restore the GIC Redistributor register context. We disable
610 * LPI and per-cpu interrupts before we start restore of the Redistributor.
611 * This function must be invoked after Distributor restore but prior to
612 * CPU interface enable. The pending and active interrupts are restored
613 * after the interrupts are fully configured and enabled.
614 *****************************************************************************/
gicv3_rdistif_init_restore(unsigned int proc_num,const gicv3_redist_ctx_t * const rdist_ctx)615 void gicv3_rdistif_init_restore(unsigned int proc_num,
616 const gicv3_redist_ctx_t * const rdist_ctx)
617 {
618 uintptr_t gicr_base;
619 unsigned int i, ppi_regs_num, regs_num;
620
621 assert(gicv3_driver_data != NULL);
622 assert(proc_num < gicv3_driver_data->rdistif_num);
623 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
624 assert(IS_IN_EL3());
625 assert(rdist_ctx != NULL);
626
627 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
628
629 #if GIC_EXT_INTID
630 /* Calculate number of PPI registers */
631 ppi_regs_num = (unsigned int)((gicr_read_typer(gicr_base) >>
632 TYPER_PPI_NUM_SHIFT) & TYPER_PPI_NUM_MASK) + 1;
633 /* All other values except PPInum [0-2] are reserved */
634 if (ppi_regs_num > 3U) {
635 ppi_regs_num = 1U;
636 }
637 #else
638 ppi_regs_num = 1U;
639 #endif
640 /* Power on redistributor */
641 gicv3_rdistif_on(proc_num);
642
643 /*
644 * Call the post-restore hook that implements the IMP DEF sequence that
645 * may be required on some GIC implementations. As this may need to
646 * access the Redistributor registers, we pass it proc_num.
647 */
648 gicv3_distif_post_restore(proc_num);
649
650 /*
651 * Disable all SGIs (imp. def.)/(E)PPIs before configuring them.
652 * This is a more scalable approach as it avoids clearing the enable
653 * bits in the GICD_CTLR.
654 */
655 for (i = 0U; i < ppi_regs_num; ++i) {
656 gicr_write_icenabler(gicr_base, i, ~0U);
657 }
658
659 /* Wait for pending writes to GICR_ICENABLER */
660 gicr_wait_for_pending_write(gicr_base);
661
662 /*
663 * Disable the LPIs to avoid unpredictable behavior when writing to
664 * GICR_PROPBASER and GICR_PENDBASER.
665 */
666 gicr_write_ctlr(gicr_base,
667 rdist_ctx->gicr_ctlr & ~(GICR_CTLR_EN_LPIS_BIT));
668
669 /* Restore registers' content */
670 gicr_write_propbaser(gicr_base, rdist_ctx->gicr_propbaser);
671 gicr_write_pendbaser(gicr_base, rdist_ctx->gicr_pendbaser);
672
673 /* 32 interrupt IDs per register */
674 for (i = 0U; i < ppi_regs_num; ++i) {
675 RESTORE_GICR_REG(gicr_base, rdist_ctx, igroupr, i);
676 RESTORE_GICR_REG(gicr_base, rdist_ctx, igrpmodr, i);
677 }
678
679 /* 4 interrupt IDs per GICR_IPRIORITYR register */
680 regs_num = ppi_regs_num << 3;
681 for (i = 0U; i < regs_num; ++i) {
682 gicr_ipriorityr_write(gicr_base, i,
683 rdist_ctx->gicr_ipriorityr[i]);
684 }
685
686 /* 16 interrupt IDs per GICR_ICFGR register */
687 regs_num = ppi_regs_num << 1;
688 for (i = 0U; i < regs_num; ++i) {
689 RESTORE_GICR_REG(gicr_base, rdist_ctx, icfgr, i);
690 }
691
692 gicr_write_nsacr(gicr_base, rdist_ctx->gicr_nsacr);
693
694 /* Restore after group and priorities are set.
695 * 32 interrupt IDs per register
696 */
697 for (i = 0U; i < ppi_regs_num; ++i) {
698 RESTORE_GICR_REG(gicr_base, rdist_ctx, ispendr, i);
699 RESTORE_GICR_REG(gicr_base, rdist_ctx, isactiver, i);
700 }
701
702 /*
703 * Wait for all writes to the Distributor to complete before enabling
704 * the SGI and (E)PPIs.
705 */
706 gicr_wait_for_upstream_pending_write(gicr_base);
707
708 /* 32 interrupt IDs per GICR_ISENABLER register */
709 for (i = 0U; i < ppi_regs_num; ++i) {
710 RESTORE_GICR_REG(gicr_base, rdist_ctx, isenabler, i);
711 }
712
713 /*
714 * Restore GICR_CTLR.Enable_LPIs bit and wait for pending writes in case
715 * the first write to GICR_CTLR was still in flight (this write only
716 * restores GICR_CTLR.Enable_LPIs and no waiting is required for this
717 * bit).
718 */
719 gicr_write_ctlr(gicr_base, rdist_ctx->gicr_ctlr);
720 gicr_wait_for_pending_write(gicr_base);
721 }
722
723 /*****************************************************************************
724 * Function to save the GIC Distributor register context. This function
725 * must be invoked after CPU interface disable and Redistributor save.
726 *****************************************************************************/
gicv3_distif_save(gicv3_dist_ctx_t * const dist_ctx)727 void gicv3_distif_save(gicv3_dist_ctx_t * const dist_ctx)
728 {
729 unsigned int typer_reg, num_ints;
730 #if GIC_EXT_INTID
731 unsigned int num_eints;
732 #endif
733
734 assert(gicv3_driver_data != NULL);
735 assert(gicv3_driver_data->gicd_base != 0U);
736 assert(IS_IN_EL3());
737 assert(dist_ctx != NULL);
738
739 uintptr_t gicd_base = gicv3_driver_data->gicd_base;
740
741 typer_reg = gicd_read_typer(gicd_base);
742
743 /* Maximum SPI INTID is 32 * (GICD_TYPER.ITLinesNumber + 1) - 1 */
744 num_ints = ((typer_reg & TYPER_IT_LINES_NO_MASK) + 1U) << 5;
745
746 /* Filter out special INTIDs 1020-1023 */
747 if (num_ints > (MAX_SPI_ID + 1U)) {
748 num_ints = MAX_SPI_ID + 1U;
749 }
750
751 #if GIC_EXT_INTID
752 /* Check if extended SPI range is implemented */
753 if ((typer_reg & TYPER_ESPI) != 0U) {
754 /*
755 * Maximum ESPI INTID is 32 * (GICD_TYPER.ESPI_range + 1) + 4095
756 */
757 num_eints = ((((typer_reg >> TYPER_ESPI_RANGE_SHIFT) &
758 TYPER_ESPI_RANGE_MASK) + 1U) << 5) + MIN_ESPI_ID - 1;
759 } else {
760 num_eints = 0U;
761 }
762 #endif
763 /* Wait for pending write to complete */
764 gicd_wait_for_pending_write(gicd_base);
765
766 /* Save the GICD_CTLR */
767 dist_ctx->gicd_ctlr = gicd_read_ctlr(gicd_base);
768
769 /* Save GICD_IGROUPR for INTIDs 32 - 1019 */
770 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, igroupr, IGROUP);
771
772 /* Save GICD_IGROUPRE for INTIDs 4096 - 5119 */
773 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igroupr, IGROUP);
774
775 /* Save GICD_ISENABLER for INT_IDs 32 - 1019 */
776 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, isenabler, ISENABLE);
777
778 /* Save GICD_ISENABLERE for INT_IDs 4096 - 5119 */
779 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isenabler, ISENABLE);
780
781 /* Save GICD_ISPENDR for INTIDs 32 - 1019 */
782 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, ispendr, ISPEND);
783
784 /* Save GICD_ISPENDRE for INTIDs 4096 - 5119 */
785 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ispendr, ISPEND);
786
787 /* Save GICD_ISACTIVER for INTIDs 32 - 1019 */
788 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, isactiver, ISACTIVE);
789
790 /* Save GICD_ISACTIVERE for INTIDs 4096 - 5119 */
791 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isactiver, ISACTIVE);
792
793 /* Save GICD_IPRIORITYR for INTIDs 32 - 1019 */
794 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, ipriorityr, IPRIORITY);
795
796 /* Save GICD_IPRIORITYRE for INTIDs 4096 - 5119 */
797 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ipriorityr, IPRIORITY);
798
799 /* Save GICD_ICFGR for INTIDs 32 - 1019 */
800 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, icfgr, ICFG);
801
802 /* Save GICD_ICFGRE for INTIDs 4096 - 5119 */
803 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, icfgr, ICFG);
804
805 /* Save GICD_IGRPMODR for INTIDs 32 - 1019 */
806 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, igrpmodr, IGRPMOD);
807
808 /* Save GICD_IGRPMODRE for INTIDs 4096 - 5119 */
809 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igrpmodr, IGRPMOD);
810
811 /* Save GICD_NSACR for INTIDs 32 - 1019 */
812 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, nsacr, NSAC);
813
814 /* Save GICD_NSACRE for INTIDs 4096 - 5119 */
815 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, nsacr, NSAC);
816
817 /* Save GICD_IROUTER for INTIDs 32 - 1019 */
818 SAVE_GICD_REGS(gicd_base, dist_ctx, num_ints, irouter, IROUTE);
819
820 /* Save GICD_IROUTERE for INTIDs 4096 - 5119 */
821 SAVE_GICD_EREGS(gicd_base, dist_ctx, num_eints, irouter, IROUTE);
822
823 /*
824 * GICD_ITARGETSR<n> and GICD_SPENDSGIR<n> are RAZ/WI when
825 * GICD_CTLR.ARE_(S|NS) bits are set which is the case for our GICv3
826 * driver.
827 */
828 }
829
830 /*****************************************************************************
831 * Function to restore the GIC Distributor register context. We disable G0, G1S
832 * and G1NS interrupt groups before we start restore of the Distributor. This
833 * function must be invoked prior to Redistributor restore and CPU interface
834 * enable. The pending and active interrupts are restored after the interrupts
835 * are fully configured and enabled.
836 *****************************************************************************/
gicv3_distif_init_restore(const gicv3_dist_ctx_t * const dist_ctx)837 void gicv3_distif_init_restore(const gicv3_dist_ctx_t * const dist_ctx)
838 {
839 unsigned int typer_reg, num_ints;
840 #if GIC_EXT_INTID
841 unsigned int num_eints;
842 #endif
843
844 assert(gicv3_driver_data != NULL);
845 assert(gicv3_driver_data->gicd_base != 0U);
846 assert(IS_IN_EL3());
847 assert(dist_ctx != NULL);
848
849 uintptr_t gicd_base = gicv3_driver_data->gicd_base;
850
851 /*
852 * Clear the "enable" bits for G0/G1S/G1NS interrupts before configuring
853 * the ARE_S bit. The Distributor might generate a system error
854 * otherwise.
855 */
856 gicd_clr_ctlr(gicd_base,
857 CTLR_ENABLE_G0_BIT |
858 CTLR_ENABLE_G1S_BIT |
859 CTLR_ENABLE_G1NS_BIT,
860 RWP_TRUE);
861
862 /* Set the ARE_S and ARE_NS bit now that interrupts have been disabled */
863 gicd_set_ctlr(gicd_base, CTLR_ARE_S_BIT | CTLR_ARE_NS_BIT, RWP_TRUE);
864
865 typer_reg = gicd_read_typer(gicd_base);
866
867 /* Maximum SPI INTID is 32 * (GICD_TYPER.ITLinesNumber + 1) - 1 */
868 num_ints = ((typer_reg & TYPER_IT_LINES_NO_MASK) + 1U) << 5;
869
870 /* Filter out special INTIDs 1020-1023 */
871 if (num_ints > (MAX_SPI_ID + 1U)) {
872 num_ints = MAX_SPI_ID + 1U;
873 }
874
875 #if GIC_EXT_INTID
876 /* Check if extended SPI range is implemented */
877 if ((typer_reg & TYPER_ESPI) != 0U) {
878 /*
879 * Maximum ESPI INTID is 32 * (GICD_TYPER.ESPI_range + 1) + 4095
880 */
881 num_eints = ((((typer_reg >> TYPER_ESPI_RANGE_SHIFT) &
882 TYPER_ESPI_RANGE_MASK) + 1U) << 5) + MIN_ESPI_ID - 1;
883 } else {
884 num_eints = 0U;
885 }
886 #endif
887 /* Restore GICD_IGROUPR for INTIDs 32 - 1019 */
888 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, igroupr, IGROUP);
889
890 /* Restore GICD_IGROUPRE for INTIDs 4096 - 5119 */
891 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igroupr, IGROUP);
892
893 /* Restore GICD_IPRIORITYR for INTIDs 32 - 1019 */
894 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, ipriorityr, IPRIORITY);
895
896 /* Restore GICD_IPRIORITYRE for INTIDs 4096 - 5119 */
897 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ipriorityr, IPRIORITY);
898
899 /* Restore GICD_ICFGR for INTIDs 32 - 1019 */
900 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, icfgr, ICFG);
901
902 /* Restore GICD_ICFGRE for INTIDs 4096 - 5119 */
903 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, icfgr, ICFG);
904
905 /* Restore GICD_IGRPMODR for INTIDs 32 - 1019 */
906 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, igrpmodr, IGRPMOD);
907
908 /* Restore GICD_IGRPMODRE for INTIDs 4096 - 5119 */
909 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, igrpmodr, IGRPMOD);
910
911 /* Restore GICD_NSACR for INTIDs 32 - 1019 */
912 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, nsacr, NSAC);
913
914 /* Restore GICD_NSACRE for INTIDs 4096 - 5119 */
915 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, nsacr, NSAC);
916
917 /* Restore GICD_IROUTER for INTIDs 32 - 1019 */
918 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, irouter, IROUTE);
919
920 /* Restore GICD_IROUTERE for INTIDs 4096 - 5119 */
921 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, irouter, IROUTE);
922
923 /*
924 * Restore ISENABLER(E), ISPENDR(E) and ISACTIVER(E) after
925 * the interrupts are configured.
926 */
927
928 /* Restore GICD_ISENABLER for INT_IDs 32 - 1019 */
929 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, isenabler, ISENABLE);
930
931 /* Restore GICD_ISENABLERE for INT_IDs 4096 - 5119 */
932 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isenabler, ISENABLE);
933
934 /* Restore GICD_ISPENDR for INTIDs 32 - 1019 */
935 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, ispendr, ISPEND);
936
937 /* Restore GICD_ISPENDRE for INTIDs 4096 - 5119 */
938 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, ispendr, ISPEND);
939
940 /* Restore GICD_ISACTIVER for INTIDs 32 - 1019 */
941 RESTORE_GICD_REGS(gicd_base, dist_ctx, num_ints, isactiver, ISACTIVE);
942
943 /* Restore GICD_ISACTIVERE for INTIDs 4096 - 5119 */
944 RESTORE_GICD_EREGS(gicd_base, dist_ctx, num_eints, isactiver, ISACTIVE);
945
946 /* Restore the GICD_CTLR */
947 gicd_write_ctlr(gicd_base, dist_ctx->gicd_ctlr);
948 gicd_wait_for_pending_write(gicd_base);
949 }
950
951 /*******************************************************************************
952 * This function gets the priority of the interrupt the processor is currently
953 * servicing.
954 ******************************************************************************/
gicv3_get_running_priority(void)955 unsigned int gicv3_get_running_priority(void)
956 {
957 return (unsigned int)read_icc_rpr_el1();
958 }
959
960 /*******************************************************************************
961 * This function checks if the interrupt identified by id is active (whether the
962 * state is either active, or active and pending). The proc_num is used if the
963 * interrupt is SGI or (E)PPI and programs the corresponding Redistributor
964 * interface.
965 ******************************************************************************/
gicv3_get_interrupt_active(unsigned int id,unsigned int proc_num)966 unsigned int gicv3_get_interrupt_active(unsigned int id, unsigned int proc_num)
967 {
968 assert(gicv3_driver_data != NULL);
969 assert(gicv3_driver_data->gicd_base != 0U);
970 assert(proc_num < gicv3_driver_data->rdistif_num);
971 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
972
973 /* Check interrupt ID */
974 if (is_sgi_ppi(id)) {
975 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
976 return gicr_get_isactiver(
977 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
978 }
979
980 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
981 return gicd_get_isactiver(gicv3_driver_data->gicd_base, id);
982 }
983
984 /*******************************************************************************
985 * This function enables the interrupt identified by id. The proc_num
986 * is used if the interrupt is SGI or PPI, and programs the corresponding
987 * Redistributor interface.
988 ******************************************************************************/
gicv3_enable_interrupt(unsigned int id,unsigned int proc_num)989 void gicv3_enable_interrupt(unsigned int id, unsigned int proc_num)
990 {
991 assert(gicv3_driver_data != NULL);
992 assert(gicv3_driver_data->gicd_base != 0U);
993 assert(proc_num < gicv3_driver_data->rdistif_num);
994 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
995
996 /*
997 * Ensure that any shared variable updates depending on out of band
998 * interrupt trigger are observed before enabling interrupt.
999 */
1000 dsbishst();
1001
1002 /* Check interrupt ID */
1003 if (is_sgi_ppi(id)) {
1004 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1005 gicr_set_isenabler(
1006 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
1007 } else {
1008 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1009 gicd_set_isenabler(gicv3_driver_data->gicd_base, id);
1010 }
1011 }
1012
1013 /*******************************************************************************
1014 * This function disables the interrupt identified by id. The proc_num
1015 * is used if the interrupt is SGI or PPI, and programs the corresponding
1016 * Redistributor interface.
1017 ******************************************************************************/
gicv3_disable_interrupt(unsigned int id,unsigned int proc_num)1018 void gicv3_disable_interrupt(unsigned int id, unsigned int proc_num)
1019 {
1020 assert(gicv3_driver_data != NULL);
1021 assert(gicv3_driver_data->gicd_base != 0U);
1022 assert(proc_num < gicv3_driver_data->rdistif_num);
1023 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1024
1025 /*
1026 * Disable interrupt, and ensure that any shared variable updates
1027 * depending on out of band interrupt trigger are observed afterwards.
1028 */
1029
1030 /* Check interrupt ID */
1031 if (is_sgi_ppi(id)) {
1032 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1033 gicr_set_icenabler(
1034 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
1035
1036 /* Write to clear enable requires waiting for pending writes */
1037 gicr_wait_for_pending_write(
1038 gicv3_driver_data->rdistif_base_addrs[proc_num]);
1039 } else {
1040 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1041 gicd_set_icenabler(gicv3_driver_data->gicd_base, id);
1042
1043 /* Write to clear enable requires waiting for pending writes */
1044 gicd_wait_for_pending_write(gicv3_driver_data->gicd_base);
1045 }
1046
1047 dsbishst();
1048 }
1049
1050 /*******************************************************************************
1051 * This function sets the interrupt priority as supplied for the given interrupt
1052 * id.
1053 ******************************************************************************/
gicv3_set_interrupt_priority(unsigned int id,unsigned int proc_num,unsigned int priority)1054 void gicv3_set_interrupt_priority(unsigned int id, unsigned int proc_num,
1055 unsigned int priority)
1056 {
1057 uintptr_t gicr_base;
1058
1059 assert(gicv3_driver_data != NULL);
1060 assert(gicv3_driver_data->gicd_base != 0U);
1061 assert(proc_num < gicv3_driver_data->rdistif_num);
1062 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1063
1064 /* Check interrupt ID */
1065 if (is_sgi_ppi(id)) {
1066 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1067 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
1068 gicr_set_ipriorityr(gicr_base, id, priority);
1069 } else {
1070 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1071 gicd_set_ipriorityr(gicv3_driver_data->gicd_base, id, priority);
1072 }
1073 }
1074
1075 /*******************************************************************************
1076 * This function assigns group for the interrupt identified by id. The proc_num
1077 * is used if the interrupt is SGI or (E)PPI, and programs the corresponding
1078 * Redistributor interface. The group can be any of GICV3_INTR_GROUP*
1079 ******************************************************************************/
gicv3_set_interrupt_type(unsigned int id,unsigned int proc_num,unsigned int type)1080 void gicv3_set_interrupt_type(unsigned int id, unsigned int proc_num,
1081 unsigned int type)
1082 {
1083 bool igroup = false, grpmod = false;
1084 uintptr_t gicr_base;
1085
1086 assert(gicv3_driver_data != NULL);
1087 assert(gicv3_driver_data->gicd_base != 0U);
1088 assert(proc_num < gicv3_driver_data->rdistif_num);
1089 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1090
1091 switch (type) {
1092 case INTR_GROUP1S:
1093 igroup = false;
1094 grpmod = true;
1095 break;
1096 case INTR_GROUP0:
1097 igroup = false;
1098 grpmod = false;
1099 break;
1100 case INTR_GROUP1NS:
1101 igroup = true;
1102 grpmod = false;
1103 break;
1104 default:
1105 assert(false);
1106 break;
1107 }
1108
1109 /* Check interrupt ID */
1110 if (is_sgi_ppi(id)) {
1111 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1112 gicr_base = gicv3_driver_data->rdistif_base_addrs[proc_num];
1113
1114 igroup ? gicr_set_igroupr(gicr_base, id) :
1115 gicr_clr_igroupr(gicr_base, id);
1116 grpmod ? gicr_set_igrpmodr(gicr_base, id) :
1117 gicr_clr_igrpmodr(gicr_base, id);
1118 } else {
1119 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1120
1121 /* Serialize read-modify-write to Distributor registers */
1122 spin_lock(&gic_lock);
1123
1124 igroup ? gicd_set_igroupr(gicv3_driver_data->gicd_base, id) :
1125 gicd_clr_igroupr(gicv3_driver_data->gicd_base, id);
1126 grpmod ? gicd_set_igrpmodr(gicv3_driver_data->gicd_base, id) :
1127 gicd_clr_igrpmodr(gicv3_driver_data->gicd_base, id);
1128
1129 spin_unlock(&gic_lock);
1130 }
1131 }
1132
1133 /*******************************************************************************
1134 * This function raises the specified Secure Group 0 SGI.
1135 *
1136 * The target parameter must be a valid MPIDR in the system.
1137 ******************************************************************************/
gicv3_raise_secure_g0_sgi(unsigned int sgi_num,u_register_t target)1138 void gicv3_raise_secure_g0_sgi(unsigned int sgi_num, u_register_t target)
1139 {
1140 unsigned int tgt, aff3, aff2, aff1, aff0;
1141 uint64_t sgi_val;
1142
1143 /* Verify interrupt number is in the SGI range */
1144 assert((sgi_num >= MIN_SGI_ID) && (sgi_num < MIN_PPI_ID));
1145
1146 /* Extract affinity fields from target */
1147 aff0 = MPIDR_AFFLVL0_VAL(target);
1148 aff1 = MPIDR_AFFLVL1_VAL(target);
1149 aff2 = MPIDR_AFFLVL2_VAL(target);
1150 aff3 = MPIDR_AFFLVL3_VAL(target);
1151
1152 /*
1153 * Make target list from affinity 0, and ensure GICv3 SGI can target
1154 * this PE.
1155 */
1156 assert(aff0 < GICV3_MAX_SGI_TARGETS);
1157 tgt = BIT_32(aff0);
1158
1159 /* Raise SGI to PE specified by its affinity */
1160 sgi_val = GICV3_SGIR_VALUE(aff3, aff2, aff1, sgi_num, SGIR_IRM_TO_AFF,
1161 tgt);
1162
1163 /*
1164 * Ensure that any shared variable updates depending on out of band
1165 * interrupt trigger are observed before raising SGI.
1166 */
1167 dsbishst();
1168 write_icc_sgi0r_el1(sgi_val);
1169 isb();
1170 }
1171
1172 /*******************************************************************************
1173 * This function sets the interrupt routing for the given (E)SPI interrupt id.
1174 * The interrupt routing is specified in routing mode and mpidr.
1175 *
1176 * The routing mode can be either of:
1177 * - GICV3_IRM_ANY
1178 * - GICV3_IRM_PE
1179 *
1180 * The mpidr is the affinity of the PE to which the interrupt will be routed,
1181 * and is ignored for routing mode GICV3_IRM_ANY.
1182 ******************************************************************************/
gicv3_set_spi_routing(unsigned int id,unsigned int irm,u_register_t mpidr)1183 void gicv3_set_spi_routing(unsigned int id, unsigned int irm, u_register_t mpidr)
1184 {
1185 unsigned long long aff;
1186 uint64_t router;
1187
1188 assert(gicv3_driver_data != NULL);
1189 assert(gicv3_driver_data->gicd_base != 0U);
1190
1191 assert((irm == GICV3_IRM_ANY) || (irm == GICV3_IRM_PE));
1192
1193 assert(IS_SPI(id));
1194
1195 aff = gicd_irouter_val_from_mpidr(mpidr, irm);
1196 gicd_write_irouter(gicv3_driver_data->gicd_base, id, aff);
1197
1198 /*
1199 * In implementations that do not require 1 of N distribution of SPIs,
1200 * IRM might be RAZ/WI. Read back and verify IRM bit.
1201 */
1202 if (irm == GICV3_IRM_ANY) {
1203 router = gicd_read_irouter(gicv3_driver_data->gicd_base, id);
1204 if (((router >> IROUTER_IRM_SHIFT) & IROUTER_IRM_MASK) == 0U) {
1205 ERROR("GICv3 implementation doesn't support routing ANY\n");
1206 panic();
1207 }
1208 }
1209 }
1210
1211 /*******************************************************************************
1212 * This function clears the pending status of an interrupt identified by id.
1213 * The proc_num is used if the interrupt is SGI or (E)PPI, and programs the
1214 * corresponding Redistributor interface.
1215 ******************************************************************************/
gicv3_clear_interrupt_pending(unsigned int id,unsigned int proc_num)1216 void gicv3_clear_interrupt_pending(unsigned int id, unsigned int proc_num)
1217 {
1218 assert(gicv3_driver_data != NULL);
1219 assert(gicv3_driver_data->gicd_base != 0U);
1220 assert(proc_num < gicv3_driver_data->rdistif_num);
1221 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1222
1223 /*
1224 * Clear pending interrupt, and ensure that any shared variable updates
1225 * depending on out of band interrupt trigger are observed afterwards.
1226 */
1227
1228 /* Check interrupt ID */
1229 if (is_sgi_ppi(id)) {
1230 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1231 gicr_set_icpendr(
1232 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
1233 } else {
1234 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1235 gicd_set_icpendr(gicv3_driver_data->gicd_base, id);
1236 }
1237
1238 dsbishst();
1239 }
1240
1241 /*******************************************************************************
1242 * This function sets the pending status of an interrupt identified by id.
1243 * The proc_num is used if the interrupt is SGI or PPI and programs the
1244 * corresponding Redistributor interface.
1245 ******************************************************************************/
gicv3_set_interrupt_pending(unsigned int id,unsigned int proc_num)1246 void gicv3_set_interrupt_pending(unsigned int id, unsigned int proc_num)
1247 {
1248 assert(gicv3_driver_data != NULL);
1249 assert(gicv3_driver_data->gicd_base != 0U);
1250 assert(proc_num < gicv3_driver_data->rdistif_num);
1251 assert(gicv3_driver_data->rdistif_base_addrs != NULL);
1252
1253 /*
1254 * Ensure that any shared variable updates depending on out of band
1255 * interrupt trigger are observed before setting interrupt pending.
1256 */
1257 dsbishst();
1258
1259 /* Check interrupt ID */
1260 if (is_sgi_ppi(id)) {
1261 /* For SGIs: 0-15, PPIs: 16-31 and EPPIs: 1056-1119 */
1262 gicr_set_ispendr(
1263 gicv3_driver_data->rdistif_base_addrs[proc_num], id);
1264 } else {
1265 /* For SPIs: 32-1019 and ESPIs: 4096-5119 */
1266 gicd_set_ispendr(gicv3_driver_data->gicd_base, id);
1267 }
1268 }
1269
1270 /*******************************************************************************
1271 * This function sets the PMR register with the supplied value. Returns the
1272 * original PMR.
1273 ******************************************************************************/
gicv3_set_pmr(unsigned int mask)1274 unsigned int gicv3_set_pmr(unsigned int mask)
1275 {
1276 unsigned int old_mask;
1277
1278 old_mask = (unsigned int)read_icc_pmr_el1();
1279
1280 /*
1281 * Order memory updates w.r.t. PMR write, and ensure they're visible
1282 * before potential out of band interrupt trigger because of PMR update.
1283 * PMR system register writes are self-synchronizing, so no ISB required
1284 * thereafter.
1285 */
1286 dsbishst();
1287 write_icc_pmr_el1(mask);
1288
1289 return old_mask;
1290 }
1291
1292 /*******************************************************************************
1293 * This function delegates the responsibility of discovering the corresponding
1294 * Redistributor frames to each CPU itself. It is a modified version of
1295 * gicv3_rdistif_base_addrs_probe() and is executed by each CPU in the platform
1296 * unlike the previous way in which only the Primary CPU did the discovery of
1297 * all the Redistributor frames for every CPU. It also handles the scenario in
1298 * which the frames of various CPUs are not contiguous in physical memory.
1299 ******************************************************************************/
gicv3_rdistif_probe(const uintptr_t gicr_frame)1300 int gicv3_rdistif_probe(const uintptr_t gicr_frame)
1301 {
1302 u_register_t mpidr;
1303 unsigned int proc_num, proc_self;
1304 uint64_t typer_val;
1305 uintptr_t rdistif_base;
1306 bool gicr_frame_found = false;
1307
1308 assert(gicv3_driver_data->gicr_base == 0U);
1309
1310 /* Ensure this function is called with Data Cache enabled */
1311 #ifndef __aarch64__
1312 assert((read_sctlr() & SCTLR_C_BIT) != 0U);
1313 #else
1314 assert((read_sctlr_el3() & SCTLR_C_BIT) != 0U);
1315 #endif /* !__aarch64__ */
1316
1317 proc_self = gicv3_driver_data->mpidr_to_core_pos(read_mpidr_el1());
1318 rdistif_base = gicr_frame;
1319 do {
1320 typer_val = gicr_read_typer(rdistif_base);
1321 if (gicv3_driver_data->mpidr_to_core_pos != NULL) {
1322 mpidr = mpidr_from_gicr_typer(typer_val);
1323 proc_num = gicv3_driver_data->mpidr_to_core_pos(mpidr);
1324 } else {
1325 proc_num = (unsigned int)(typer_val >>
1326 TYPER_PROC_NUM_SHIFT) & TYPER_PROC_NUM_MASK;
1327 }
1328 if (proc_num == proc_self) {
1329 /* The base address doesn't need to be initialized on
1330 * every warm boot.
1331 */
1332 if (gicv3_driver_data->rdistif_base_addrs[proc_num]
1333 != 0U) {
1334 return 0;
1335 }
1336 gicv3_driver_data->rdistif_base_addrs[proc_num] =
1337 rdistif_base;
1338 gicr_frame_found = true;
1339 break;
1340 }
1341 rdistif_base += (uintptr_t)(ULL(1) << GICR_PCPUBASE_SHIFT);
1342 } while ((typer_val & TYPER_LAST_BIT) == 0U);
1343
1344 if (!gicr_frame_found) {
1345 return -1;
1346 }
1347
1348 /*
1349 * Flush the driver data to ensure coherency. This is
1350 * not required if platform has HW_ASSISTED_COHERENCY
1351 * enabled.
1352 */
1353 #if !HW_ASSISTED_COHERENCY
1354 /*
1355 * Flush the rdistif_base_addrs[] contents linked to the GICv3 driver.
1356 */
1357 flush_dcache_range((uintptr_t)&(gicv3_driver_data->rdistif_base_addrs[proc_num]),
1358 sizeof(*(gicv3_driver_data->rdistif_base_addrs)));
1359 #endif
1360 return 0; /* Found matching GICR frame */
1361 }
1362
1363 /******************************************************************************
1364 * This function checks the interrupt ID and returns true for SGIs and (E)PPIs
1365 * and false for (E)SPIs IDs.
1366 *****************************************************************************/
is_sgi_ppi(unsigned int id)1367 static bool is_sgi_ppi(unsigned int id)
1368 {
1369 /* SGIs: 0-15, PPIs: 16-31, EPPIs: 1056-1119 */
1370 if (IS_SGI_PPI(id)) {
1371 return true;
1372 }
1373
1374 /* SPIs: 32-1019, ESPIs: 4096-5119 */
1375 if (IS_SPI(id)) {
1376 return false;
1377 }
1378
1379 assert(false);
1380 panic();
1381 }
1382