1 /*
2 * Copyright (c) 2016-2019, ARM Limited and Contributors. All rights reserved.
3 * Copyright (c) 2020, NVIDIA Corporation. All rights reserved.
4 *
5 * SPDX-License-Identifier: BSD-3-Clause
6 */
7
8 #include <assert.h>
9 #include <lib/xlat_tables/xlat_tables_v2.h>
10 #include <stdbool.h>
11 #include <string.h>
12
13 #include <arch_helpers.h>
14 #include <bl31/bl31.h>
15 #include <bl31/interrupt_mgmt.h>
16 #include <common/bl_common.h>
17 #include <common/debug.h>
18 #include <common/runtime_svc.h>
19 #include <lib/el3_runtime/context_mgmt.h>
20 #include <lib/smccc.h>
21 #include <plat/common/platform.h>
22 #include <tools_share/uuid.h>
23
24 #include "sm_err.h"
25 #include "smcall.h"
26
27 /* Trusty UID: RFC-4122 compliant UUID version 4 */
28 DEFINE_SVC_UUID2(trusty_uuid,
29 0x40ee25f0, 0xa2bc, 0x304c, 0x8c, 0x4c,
30 0xa1, 0x73, 0xc5, 0x7d, 0x8a, 0xf1);
31
32 /* macro to check if Hypervisor is enabled in the HCR_EL2 register */
33 #define HYP_ENABLE_FLAG 0x286001U
34
35 /* length of Trusty's input parameters (in bytes) */
36 #define TRUSTY_PARAMS_LEN_BYTES (4096U * 2)
37
38 struct trusty_stack {
39 uint8_t space[PLATFORM_STACK_SIZE] __aligned(16);
40 uint32_t end;
41 };
42
43 struct trusty_cpu_ctx {
44 cpu_context_t cpu_ctx;
45 void *saved_sp;
46 uint32_t saved_security_state;
47 int32_t fiq_handler_active;
48 uint64_t fiq_handler_pc;
49 uint64_t fiq_handler_cpsr;
50 uint64_t fiq_handler_sp;
51 uint64_t fiq_pc;
52 uint64_t fiq_cpsr;
53 uint64_t fiq_sp_el1;
54 gp_regs_t fiq_gpregs;
55 struct trusty_stack secure_stack;
56 };
57
58 struct smc_args {
59 uint64_t r0;
60 uint64_t r1;
61 uint64_t r2;
62 uint64_t r3;
63 uint64_t r4;
64 uint64_t r5;
65 uint64_t r6;
66 uint64_t r7;
67 };
68
69 static struct trusty_cpu_ctx trusty_cpu_ctx[PLATFORM_CORE_COUNT];
70
71 struct smc_args trusty_init_context_stack(void **sp, void *new_stack);
72 struct smc_args trusty_context_switch_helper(void **sp, void *smc_params);
73
74 static uint32_t current_vmid;
75
get_trusty_ctx(void)76 static struct trusty_cpu_ctx *get_trusty_ctx(void)
77 {
78 return &trusty_cpu_ctx[plat_my_core_pos()];
79 }
80
is_hypervisor_mode(void)81 static bool is_hypervisor_mode(void)
82 {
83 uint64_t hcr = read_hcr();
84
85 return ((hcr & HYP_ENABLE_FLAG) != 0U) ? true : false;
86 }
87
trusty_context_switch(uint32_t security_state,uint64_t r0,uint64_t r1,uint64_t r2,uint64_t r3)88 static struct smc_args trusty_context_switch(uint32_t security_state, uint64_t r0,
89 uint64_t r1, uint64_t r2, uint64_t r3)
90 {
91 struct smc_args args, ret_args;
92 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
93 struct trusty_cpu_ctx *ctx_smc;
94
95 assert(ctx->saved_security_state != security_state);
96
97 args.r7 = 0;
98 if (is_hypervisor_mode()) {
99 /* According to the ARM DEN0028A spec, VMID is stored in x7 */
100 ctx_smc = cm_get_context(NON_SECURE);
101 assert(ctx_smc != NULL);
102 args.r7 = SMC_GET_GP(ctx_smc, CTX_GPREG_X7);
103 }
104 /* r4, r5, r6 reserved for future use. */
105 args.r6 = 0;
106 args.r5 = 0;
107 args.r4 = 0;
108 args.r3 = r3;
109 args.r2 = r2;
110 args.r1 = r1;
111 args.r0 = r0;
112
113 /*
114 * To avoid the additional overhead in PSCI flow, skip FP context
115 * saving/restoring in case of CPU suspend and resume, assuming that
116 * when it's needed the PSCI caller has preserved FP context before
117 * going here.
118 */
119 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME)
120 fpregs_context_save(get_fpregs_ctx(cm_get_context(security_state)));
121 cm_el1_sysregs_context_save(security_state);
122
123 ctx->saved_security_state = security_state;
124 ret_args = trusty_context_switch_helper(&ctx->saved_sp, &args);
125
126 assert(ctx->saved_security_state == ((security_state == 0U) ? 1U : 0U));
127
128 cm_el1_sysregs_context_restore(security_state);
129 if (r0 != SMC_FC_CPU_SUSPEND && r0 != SMC_FC_CPU_RESUME)
130 fpregs_context_restore(get_fpregs_ctx(cm_get_context(security_state)));
131
132 cm_set_next_eret_context(security_state);
133
134 return ret_args;
135 }
136
trusty_fiq_handler(uint32_t id,uint32_t flags,void * handle,void * cookie)137 static uint64_t trusty_fiq_handler(uint32_t id,
138 uint32_t flags,
139 void *handle,
140 void *cookie)
141 {
142 struct smc_args ret;
143 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
144
145 assert(!is_caller_secure(flags));
146
147 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_ENTER, 0, 0, 0);
148 if (ret.r0 != 0U) {
149 SMC_RET0(handle);
150 }
151
152 if (ctx->fiq_handler_active != 0) {
153 INFO("%s: fiq handler already active\n", __func__);
154 SMC_RET0(handle);
155 }
156
157 ctx->fiq_handler_active = 1;
158 (void)memcpy(&ctx->fiq_gpregs, get_gpregs_ctx(handle), sizeof(ctx->fiq_gpregs));
159 ctx->fiq_pc = SMC_GET_EL3(handle, CTX_ELR_EL3);
160 ctx->fiq_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3);
161 ctx->fiq_sp_el1 = read_ctx_reg(get_el1_sysregs_ctx(handle), CTX_SP_EL1);
162
163 write_ctx_reg(get_el1_sysregs_ctx(handle), CTX_SP_EL1, ctx->fiq_handler_sp);
164 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_handler_pc, (uint32_t)ctx->fiq_handler_cpsr);
165
166 SMC_RET0(handle);
167 }
168
trusty_set_fiq_handler(void * handle,uint64_t cpu,uint64_t handler,uint64_t stack)169 static uint64_t trusty_set_fiq_handler(void *handle, uint64_t cpu,
170 uint64_t handler, uint64_t stack)
171 {
172 struct trusty_cpu_ctx *ctx;
173
174 if (cpu >= (uint64_t)PLATFORM_CORE_COUNT) {
175 ERROR("%s: cpu %lld >= %d\n", __func__, cpu, PLATFORM_CORE_COUNT);
176 return (uint64_t)SM_ERR_INVALID_PARAMETERS;
177 }
178
179 ctx = &trusty_cpu_ctx[cpu];
180 ctx->fiq_handler_pc = handler;
181 ctx->fiq_handler_cpsr = SMC_GET_EL3(handle, CTX_SPSR_EL3);
182 ctx->fiq_handler_sp = stack;
183
184 SMC_RET1(handle, 0);
185 }
186
trusty_get_fiq_regs(void * handle)187 static uint64_t trusty_get_fiq_regs(void *handle)
188 {
189 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
190 uint64_t sp_el0 = read_ctx_reg(&ctx->fiq_gpregs, CTX_GPREG_SP_EL0);
191
192 SMC_RET4(handle, ctx->fiq_pc, ctx->fiq_cpsr, sp_el0, ctx->fiq_sp_el1);
193 }
194
trusty_fiq_exit(void * handle,uint64_t x1,uint64_t x2,uint64_t x3)195 static uint64_t trusty_fiq_exit(void *handle, uint64_t x1, uint64_t x2, uint64_t x3)
196 {
197 struct smc_args ret;
198 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
199
200 if (ctx->fiq_handler_active == 0) {
201 NOTICE("%s: fiq handler not active\n", __func__);
202 SMC_RET1(handle, (uint64_t)SM_ERR_INVALID_PARAMETERS);
203 }
204
205 ret = trusty_context_switch(NON_SECURE, SMC_FC_FIQ_EXIT, 0, 0, 0);
206 if (ret.r0 != 1U) {
207 INFO("%s(%p) SMC_FC_FIQ_EXIT returned unexpected value, %lld\n",
208 __func__, handle, ret.r0);
209 }
210
211 /*
212 * Restore register state to state recorded on fiq entry.
213 *
214 * x0, sp_el1, pc and cpsr need to be restored because el1 cannot
215 * restore them.
216 *
217 * x1-x4 and x8-x17 need to be restored here because smc_handler64
218 * corrupts them (el1 code also restored them).
219 */
220 (void)memcpy(get_gpregs_ctx(handle), &ctx->fiq_gpregs, sizeof(ctx->fiq_gpregs));
221 ctx->fiq_handler_active = 0;
222 write_ctx_reg(get_el1_sysregs_ctx(handle), CTX_SP_EL1, ctx->fiq_sp_el1);
223 cm_set_elr_spsr_el3(NON_SECURE, ctx->fiq_pc, (uint32_t)ctx->fiq_cpsr);
224
225 SMC_RET0(handle);
226 }
227
trusty_smc_handler(uint32_t smc_fid,u_register_t x1,u_register_t x2,u_register_t x3,u_register_t x4,void * cookie,void * handle,u_register_t flags)228 static uintptr_t trusty_smc_handler(uint32_t smc_fid,
229 u_register_t x1,
230 u_register_t x2,
231 u_register_t x3,
232 u_register_t x4,
233 void *cookie,
234 void *handle,
235 u_register_t flags)
236 {
237 struct smc_args ret;
238 uint32_t vmid = 0U;
239 entry_point_info_t *ep_info = bl31_plat_get_next_image_ep_info(SECURE);
240
241 /*
242 * Return success for SET_ROT_PARAMS if Trusty is not present, as
243 * Verified Boot is not even supported and returning success here
244 * would not compromise the boot process.
245 */
246 if ((ep_info == NULL) && (smc_fid == SMC_YC_SET_ROT_PARAMS)) {
247 SMC_RET1(handle, 0);
248 } else if (ep_info == NULL) {
249 SMC_RET1(handle, SMC_UNK);
250 } else {
251 ; /* do nothing */
252 }
253
254 if (is_caller_secure(flags)) {
255 if (smc_fid == SMC_YC_NS_RETURN) {
256 ret = trusty_context_switch(SECURE, x1, 0, 0, 0);
257 SMC_RET8(handle, ret.r0, ret.r1, ret.r2, ret.r3,
258 ret.r4, ret.r5, ret.r6, ret.r7);
259 }
260 INFO("%s (0x%x, 0x%lx, 0x%lx, 0x%lx, 0x%lx, %p, %p, 0x%lx) \
261 cpu %d, unknown smc\n",
262 __func__, smc_fid, x1, x2, x3, x4, cookie, handle, flags,
263 plat_my_core_pos());
264 SMC_RET1(handle, SMC_UNK);
265 } else {
266 switch (smc_fid) {
267 case SMC_FC64_GET_UUID:
268 case SMC_FC_GET_UUID:
269 /* provide the UUID for the service to the client */
270 SMC_UUID_RET(handle, trusty_uuid);
271 break;
272 case SMC_FC64_SET_FIQ_HANDLER:
273 return trusty_set_fiq_handler(handle, x1, x2, x3);
274 case SMC_FC64_GET_FIQ_REGS:
275 return trusty_get_fiq_regs(handle);
276 case SMC_FC_FIQ_EXIT:
277 return trusty_fiq_exit(handle, x1, x2, x3);
278 default:
279 /* Not all OENs greater than SMC_ENTITY_SECURE_MONITOR are supported */
280 if (SMC_ENTITY(smc_fid) > SMC_ENTITY_SECURE_MONITOR) {
281 VERBOSE("%s: unsupported SMC FID (0x%x)\n", __func__, smc_fid);
282 SMC_RET1(handle, SMC_UNK);
283 }
284
285 if (is_hypervisor_mode())
286 vmid = SMC_GET_GP(handle, CTX_GPREG_X7);
287
288 if ((current_vmid != 0) && (current_vmid != vmid)) {
289 /* This message will cause SMC mechanism
290 * abnormal in multi-guest environment.
291 * Change it to WARN in case you need it.
292 */
293 VERBOSE("Previous SMC not finished.\n");
294 SMC_RET1(handle, SM_ERR_BUSY);
295 }
296 current_vmid = vmid;
297 ret = trusty_context_switch(NON_SECURE, smc_fid, x1,
298 x2, x3);
299 current_vmid = 0;
300 SMC_RET1(handle, ret.r0);
301 }
302 }
303 }
304
trusty_init(void)305 static int32_t trusty_init(void)
306 {
307 entry_point_info_t *ep_info;
308 struct smc_args zero_args = {0};
309 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
310 uint32_t cpu = plat_my_core_pos();
311 uint64_t reg_width = GET_RW(read_ctx_reg(get_el3state_ctx(&ctx->cpu_ctx),
312 CTX_SPSR_EL3));
313
314 /*
315 * Get information about the Trusty image. Its absence is a critical
316 * failure.
317 */
318 ep_info = bl31_plat_get_next_image_ep_info(SECURE);
319 assert(ep_info != NULL);
320
321 fpregs_context_save(get_fpregs_ctx(cm_get_context(NON_SECURE)));
322 cm_el1_sysregs_context_save(NON_SECURE);
323
324 cm_set_context(&ctx->cpu_ctx, SECURE);
325 cm_init_my_context(ep_info);
326
327 /*
328 * Adjust secondary cpu entry point for 32 bit images to the
329 * end of exception vectors
330 */
331 if ((cpu != 0U) && (reg_width == MODE_RW_32)) {
332 INFO("trusty: cpu %d, adjust entry point to 0x%lx\n",
333 cpu, ep_info->pc + (1U << 5));
334 cm_set_elr_el3(SECURE, ep_info->pc + (1U << 5));
335 }
336
337 cm_el1_sysregs_context_restore(SECURE);
338 fpregs_context_restore(get_fpregs_ctx(cm_get_context(SECURE)));
339 cm_set_next_eret_context(SECURE);
340
341 ctx->saved_security_state = ~0U; /* initial saved state is invalid */
342 (void)trusty_init_context_stack(&ctx->saved_sp, &ctx->secure_stack.end);
343
344 (void)trusty_context_switch_helper(&ctx->saved_sp, &zero_args);
345
346 cm_el1_sysregs_context_restore(NON_SECURE);
347 fpregs_context_restore(get_fpregs_ctx(cm_get_context(NON_SECURE)));
348 cm_set_next_eret_context(NON_SECURE);
349
350 return 1;
351 }
352
trusty_cpu_suspend(uint32_t off)353 static void trusty_cpu_suspend(uint32_t off)
354 {
355 struct smc_args ret;
356
357 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_SUSPEND, off, 0, 0);
358 if (ret.r0 != 0U) {
359 INFO("%s: cpu %d, SMC_FC_CPU_SUSPEND returned unexpected value, %lld\n",
360 __func__, plat_my_core_pos(), ret.r0);
361 }
362 }
363
trusty_cpu_resume(uint32_t on)364 static void trusty_cpu_resume(uint32_t on)
365 {
366 struct smc_args ret;
367
368 ret = trusty_context_switch(NON_SECURE, SMC_FC_CPU_RESUME, on, 0, 0);
369 if (ret.r0 != 0U) {
370 INFO("%s: cpu %d, SMC_FC_CPU_RESUME returned unexpected value, %lld\n",
371 __func__, plat_my_core_pos(), ret.r0);
372 }
373 }
374
trusty_cpu_off_handler(u_register_t max_off_lvl)375 static int32_t trusty_cpu_off_handler(u_register_t max_off_lvl)
376 {
377 trusty_cpu_suspend(max_off_lvl);
378
379 return 0;
380 }
381
trusty_cpu_on_finish_handler(u_register_t max_off_lvl)382 static void trusty_cpu_on_finish_handler(u_register_t max_off_lvl)
383 {
384 struct trusty_cpu_ctx *ctx = get_trusty_ctx();
385
386 if (ctx->saved_sp == NULL) {
387 (void)trusty_init();
388 } else {
389 trusty_cpu_resume(max_off_lvl);
390 }
391 }
392
trusty_cpu_suspend_handler(u_register_t max_off_lvl)393 static void trusty_cpu_suspend_handler(u_register_t max_off_lvl)
394 {
395 trusty_cpu_suspend(max_off_lvl);
396 }
397
trusty_cpu_suspend_finish_handler(u_register_t max_off_lvl)398 static void trusty_cpu_suspend_finish_handler(u_register_t max_off_lvl)
399 {
400 trusty_cpu_resume(max_off_lvl);
401 }
402
403 static const spd_pm_ops_t trusty_pm = {
404 .svc_off = trusty_cpu_off_handler,
405 .svc_suspend = trusty_cpu_suspend_handler,
406 .svc_on_finish = trusty_cpu_on_finish_handler,
407 .svc_suspend_finish = trusty_cpu_suspend_finish_handler,
408 };
409
410 void plat_trusty_set_boot_args(aapcs64_params_t *args);
411
412 #if !defined(TSP_SEC_MEM_SIZE) && defined(BL32_MEM_SIZE)
413 #define TSP_SEC_MEM_SIZE BL32_MEM_SIZE
414 #endif
415
416 #ifdef TSP_SEC_MEM_SIZE
417 #pragma weak plat_trusty_set_boot_args
plat_trusty_set_boot_args(aapcs64_params_t * args)418 void plat_trusty_set_boot_args(aapcs64_params_t *args)
419 {
420 args->arg0 = TSP_SEC_MEM_SIZE;
421 }
422 #endif
423
trusty_setup(void)424 static int32_t trusty_setup(void)
425 {
426 entry_point_info_t *ep_info;
427 uint32_t instr;
428 uint32_t flags;
429 int32_t ret;
430 bool aarch32 = false;
431
432 /* Get trusty's entry point info */
433 ep_info = bl31_plat_get_next_image_ep_info(SECURE);
434 if (ep_info == NULL) {
435 VERBOSE("Trusty image missing.\n");
436 return -1;
437 }
438
439 /* memmap first page of trusty's code memory before peeking */
440 ret = mmap_add_dynamic_region(ep_info->pc, /* PA */
441 ep_info->pc, /* VA */
442 PAGE_SIZE, /* size */
443 MT_SECURE | MT_RW_DATA); /* attrs */
444 assert(ret == 0);
445
446 /* peek into trusty's code to see if we have a 32-bit or 64-bit image */
447 instr = *(uint32_t *)ep_info->pc;
448
449 if (instr >> 24 == 0xeaU) {
450 INFO("trusty: Found 32 bit image\n");
451 aarch32 = true;
452 } else if (instr >> 8 == 0xd53810U || instr >> 16 == 0x9400U) {
453 INFO("trusty: Found 64 bit image\n");
454 } else {
455 ERROR("trusty: Found unknown image, 0x%x\n", instr);
456 return -1;
457 }
458
459 /* unmap trusty's memory page */
460 (void)mmap_remove_dynamic_region(ep_info->pc, PAGE_SIZE);
461
462 SET_PARAM_HEAD(ep_info, PARAM_EP, VERSION_1, SECURE | EP_ST_ENABLE);
463 if (!aarch32)
464 ep_info->spsr = SPSR_64(MODE_EL1, MODE_SP_ELX,
465 DISABLE_ALL_EXCEPTIONS);
466 else
467 ep_info->spsr = SPSR_MODE32(MODE32_svc, SPSR_T_ARM,
468 SPSR_E_LITTLE,
469 DAIF_FIQ_BIT |
470 DAIF_IRQ_BIT |
471 DAIF_ABT_BIT);
472 (void)memset(&ep_info->args, 0, sizeof(ep_info->args));
473 plat_trusty_set_boot_args(&ep_info->args);
474
475 /* register init handler */
476 bl31_register_bl32_init(trusty_init);
477
478 /* register power management hooks */
479 psci_register_spd_pm_hook(&trusty_pm);
480
481 /* register interrupt handler */
482 flags = 0;
483 set_interrupt_rm_flag(flags, NON_SECURE);
484 ret = register_interrupt_type_handler(INTR_TYPE_S_EL1,
485 trusty_fiq_handler,
486 flags);
487 if (ret != 0) {
488 VERBOSE("trusty: failed to register fiq handler, ret = %d\n", ret);
489 }
490
491 if (aarch32) {
492 entry_point_info_t *ns_ep_info;
493 uint32_t spsr;
494
495 ns_ep_info = bl31_plat_get_next_image_ep_info(NON_SECURE);
496 if (ns_ep_info == NULL) {
497 NOTICE("Trusty: non-secure image missing.\n");
498 return -1;
499 }
500 spsr = ns_ep_info->spsr;
501 if (GET_RW(spsr) == MODE_RW_64 && GET_EL(spsr) == MODE_EL2) {
502 spsr &= ~(MODE_EL_MASK << MODE_EL_SHIFT);
503 spsr |= MODE_EL1 << MODE_EL_SHIFT;
504 }
505 if (GET_RW(spsr) == MODE_RW_32 && GET_M32(spsr) == MODE32_hyp) {
506 spsr &= ~(MODE32_MASK << MODE32_SHIFT);
507 spsr |= MODE32_svc << MODE32_SHIFT;
508 }
509 if (spsr != ns_ep_info->spsr) {
510 NOTICE("Trusty: Switch bl33 from EL2 to EL1 (spsr 0x%x -> 0x%x)\n",
511 ns_ep_info->spsr, spsr);
512 ns_ep_info->spsr = spsr;
513 }
514 }
515
516 return 0;
517 }
518
519 /* Define a SPD runtime service descriptor for fast SMC calls */
520 DECLARE_RT_SVC(
521 trusty_fast,
522
523 OEN_TOS_START,
524 OEN_TOS_END,
525 SMC_TYPE_FAST,
526 trusty_setup,
527 trusty_smc_handler
528 );
529
530 /* Define a SPD runtime service descriptor for yielding SMC calls */
531 DECLARE_RT_SVC(
532 trusty_std,
533
534 OEN_TAP_START,
535 SMC_ENTITY_SECURE_MONITOR,
536 SMC_TYPE_YIELD,
537 NULL,
538 trusty_smc_handler
539 );
540