1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 *
  4 * Copyright (C) 2015 ARM Limited
  5 */
  6
  7#define pr_fmt(fmt) "psci: " fmt
  8
  9#include <linux/acpi.h>
 10#include <linux/arm-smccc.h>
 11#include <linux/cpuidle.h>
 12#include <linux/errno.h>
 13#include <linux/linkage.h>
 14#include <linux/of.h>
 15#include <linux/pm.h>
 16#include <linux/printk.h>
 17#include <linux/psci.h>
 18#include <linux/reboot.h>
 19#include <linux/slab.h>
 20#include <linux/suspend.h>
 21
 22#include <uapi/linux/psci.h>
 23
 24#include <asm/cpuidle.h>
 25#include <asm/cputype.h>
 26#include <asm/hypervisor.h>
 27#include <asm/system_misc.h>
 28#include <asm/smp_plat.h>
 29#include <asm/suspend.h>
 30
 31/*
 32 * While a 64-bit OS can make calls with SMC32 calling conventions, for some
 33 * calls it is necessary to use SMC64 to pass or return 64-bit values.
 34 * For such calls PSCI_FN_NATIVE(version, name) will choose the appropriate
 35 * (native-width) function ID.
 36 */
 37#ifdef CONFIG_64BIT
 38#define PSCI_FN_NATIVE(version, name)	PSCI_##version##_FN64_##name
 39#else
 40#define PSCI_FN_NATIVE(version, name)	PSCI_##version##_FN_##name
 41#endif
 42
 43/*
 44 * The CPU any Trusted OS is resident on. The trusted OS may reject CPU_OFF
 45 * calls to its resident CPU, so we must avoid issuing those. We never migrate
 46 * a Trusted OS even if it claims to be capable of migration -- doing so will
 47 * require cooperation with a Trusted OS driver.
 48 */
 49static int resident_cpu = -1;
 50struct psci_operations psci_ops;
 51static enum arm_smccc_conduit psci_conduit = SMCCC_CONDUIT_NONE;
 52
 53bool psci_tos_resident_on(int cpu)
 54{
 55	return cpu == resident_cpu;
 56}
 57
 58typedef unsigned long (psci_fn)(unsigned long, unsigned long,
 59				unsigned long, unsigned long);
 60static psci_fn *invoke_psci_fn;
 61
 62static struct psci_0_1_function_ids psci_0_1_function_ids;
 63
 64struct psci_0_1_function_ids get_psci_0_1_function_ids(void)
 65{
 66	return psci_0_1_function_ids;
 67}
 68
 69#define PSCI_0_2_POWER_STATE_MASK		\
 70				(PSCI_0_2_POWER_STATE_ID_MASK | \
 71				PSCI_0_2_POWER_STATE_TYPE_MASK | \
 72				PSCI_0_2_POWER_STATE_AFFL_MASK)
 73
 74#define PSCI_1_0_EXT_POWER_STATE_MASK		\
 75				(PSCI_1_0_EXT_POWER_STATE_ID_MASK | \
 76				PSCI_1_0_EXT_POWER_STATE_TYPE_MASK)
 77
 78static u32 psci_cpu_suspend_feature;
 79static bool psci_system_reset2_supported;
 80
 81static inline bool psci_has_ext_power_state(void)
 82{
 83	return psci_cpu_suspend_feature &
 84				PSCI_1_0_FEATURES_CPU_SUSPEND_PF_MASK;
 85}
 86
 87bool psci_has_osi_support(void)
 88{
 89	return psci_cpu_suspend_feature & PSCI_1_0_OS_INITIATED;
 90}
 91
 92static inline bool psci_power_state_loses_context(u32 state)
 93{
 94	const u32 mask = psci_has_ext_power_state() ?
 95					PSCI_1_0_EXT_POWER_STATE_TYPE_MASK :
 96					PSCI_0_2_POWER_STATE_TYPE_MASK;
 97
 98	return state & mask;
 99}
100
101bool psci_power_state_is_valid(u32 state)
102{
103	const u32 valid_mask = psci_has_ext_power_state() ?
104			       PSCI_1_0_EXT_POWER_STATE_MASK :
105			       PSCI_0_2_POWER_STATE_MASK;
106
107	return !(state & ~valid_mask);
108}
109
110static unsigned long __invoke_psci_fn_hvc(unsigned long function_id,
111			unsigned long arg0, unsigned long arg1,
112			unsigned long arg2)
113{
114	struct arm_smccc_res res;
115
116	arm_smccc_hvc(function_id, arg0, arg1, arg2, 0, 0, 0, 0, &res);
117	return res.a0;
118}
119
120static unsigned long __invoke_psci_fn_smc(unsigned long function_id,
121			unsigned long arg0, unsigned long arg1,
122			unsigned long arg2)
123{
124	struct arm_smccc_res res;
125
126	arm_smccc_smc(function_id, arg0, arg1, arg2, 0, 0, 0, 0, &res);
127	return res.a0;
128}
129
130static int psci_to_linux_errno(int errno)
131{
132	switch (errno) {
133	case PSCI_RET_SUCCESS:
134		return 0;
135	case PSCI_RET_NOT_SUPPORTED:
136		return -EOPNOTSUPP;
137	case PSCI_RET_INVALID_PARAMS:
138	case PSCI_RET_INVALID_ADDRESS:
139		return -EINVAL;
140	case PSCI_RET_DENIED:
141		return -EPERM;
142	}
143
144	return -EINVAL;
145}
146
147static u32 psci_0_1_get_version(void)
148{
149	return PSCI_VERSION(0, 1);
150}
151
152static u32 psci_0_2_get_version(void)
153{
154	return invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0);
155}
156
157int psci_set_osi_mode(bool enable)
158{
159	unsigned long suspend_mode;
160	int err;
161
162	suspend_mode = enable ? PSCI_1_0_SUSPEND_MODE_OSI :
163			PSCI_1_0_SUSPEND_MODE_PC;
164
165	err = invoke_psci_fn(PSCI_1_0_FN_SET_SUSPEND_MODE, suspend_mode, 0, 0);
166	return psci_to_linux_errno(err);
167}
168
169static int __psci_cpu_suspend(u32 fn, u32 state, unsigned long entry_point)
170{
171	int err;
172
173	err = invoke_psci_fn(fn, state, entry_point, 0);
174	return psci_to_linux_errno(err);
175}
176
177static int psci_0_1_cpu_suspend(u32 state, unsigned long entry_point)
178{
179	return __psci_cpu_suspend(psci_0_1_function_ids.cpu_suspend,
180				  state, entry_point);
181}
182
183static int psci_0_2_cpu_suspend(u32 state, unsigned long entry_point)
184{
185	return __psci_cpu_suspend(PSCI_FN_NATIVE(0_2, CPU_SUSPEND),
186				  state, entry_point);
187}
188
189static int __psci_cpu_off(u32 fn, u32 state)
190{
191	int err;
192
193	err = invoke_psci_fn(fn, state, 0, 0);
194	return psci_to_linux_errno(err);
195}
196
197static int psci_0_1_cpu_off(u32 state)
198{
199	return __psci_cpu_off(psci_0_1_function_ids.cpu_off, state);
200}
201
202static int psci_0_2_cpu_off(u32 state)
203{
204	return __psci_cpu_off(PSCI_0_2_FN_CPU_OFF, state);
205}
206
207static int __psci_cpu_on(u32 fn, unsigned long cpuid, unsigned long entry_point)
208{
209	int err;
210
211	err = invoke_psci_fn(fn, cpuid, entry_point, 0);
212	return psci_to_linux_errno(err);
213}
214
215static int psci_0_1_cpu_on(unsigned long cpuid, unsigned long entry_point)
216{
217	return __psci_cpu_on(psci_0_1_function_ids.cpu_on, cpuid, entry_point);
218}
219
220static int psci_0_2_cpu_on(unsigned long cpuid, unsigned long entry_point)
221{
222	return __psci_cpu_on(PSCI_FN_NATIVE(0_2, CPU_ON), cpuid, entry_point);
223}
224
225static int __psci_migrate(u32 fn, unsigned long cpuid)
226{
227	int err;
228
229	err = invoke_psci_fn(fn, cpuid, 0, 0);
230	return psci_to_linux_errno(err);
231}
232
233static int psci_0_1_migrate(unsigned long cpuid)
234{
235	return __psci_migrate(psci_0_1_function_ids.migrate, cpuid);
236}
237
238static int psci_0_2_migrate(unsigned long cpuid)
239{
240	return __psci_migrate(PSCI_FN_NATIVE(0_2, MIGRATE), cpuid);
241}
242
243static int psci_affinity_info(unsigned long target_affinity,
244		unsigned long lowest_affinity_level)
245{
246	return invoke_psci_fn(PSCI_FN_NATIVE(0_2, AFFINITY_INFO),
247			      target_affinity, lowest_affinity_level, 0);
248}
249
250static int psci_migrate_info_type(void)
251{
252	return invoke_psci_fn(PSCI_0_2_FN_MIGRATE_INFO_TYPE, 0, 0, 0);
253}
254
255static unsigned long psci_migrate_info_up_cpu(void)
256{
257	return invoke_psci_fn(PSCI_FN_NATIVE(0_2, MIGRATE_INFO_UP_CPU),
258			      0, 0, 0);
259}
260
261static void set_conduit(enum arm_smccc_conduit conduit)
262{
263	switch (conduit) {
264	case SMCCC_CONDUIT_HVC:
265		invoke_psci_fn = __invoke_psci_fn_hvc;
266		break;
267	case SMCCC_CONDUIT_SMC:
268		invoke_psci_fn = __invoke_psci_fn_smc;
269		break;
270	default:
271		WARN(1, "Unexpected PSCI conduit %d\n", conduit);
272	}
273
274	psci_conduit = conduit;
275}
276
277static int get_set_conduit_method(struct device_node *np)
278{
279	const char *method;
280
281	pr_info("probing for conduit method from DT.\n");
282
283	if (of_property_read_string(np, "method", &method)) {
284		pr_warn("missing \"method\" property\n");
285		return -ENXIO;
286	}
287
288	if (!strcmp("hvc", method)) {
289		set_conduit(SMCCC_CONDUIT_HVC);
290	} else if (!strcmp("smc", method)) {
291		set_conduit(SMCCC_CONDUIT_SMC);
292	} else {
293		pr_warn("invalid \"method\" property: %s\n", method);
294		return -EINVAL;
295	}
296	return 0;
297}
298
299static int psci_sys_reset(struct notifier_block *nb, unsigned long action,
300			  void *data)
301{
302	if ((reboot_mode == REBOOT_WARM || reboot_mode == REBOOT_SOFT) &&
303	    psci_system_reset2_supported) {
304		/*
305		 * reset_type[31] = 0 (architectural)
306		 * reset_type[30:0] = 0 (SYSTEM_WARM_RESET)
307		 * cookie = 0 (ignored by the implementation)
308		 */
309		invoke_psci_fn(PSCI_FN_NATIVE(1_1, SYSTEM_RESET2), 0, 0, 0);
310	} else {
311		invoke_psci_fn(PSCI_0_2_FN_SYSTEM_RESET, 0, 0, 0);
312	}
313
314	return NOTIFY_DONE;
315}
316
317static struct notifier_block psci_sys_reset_nb = {
318	.notifier_call = psci_sys_reset,
319	.priority = 129,
320};
321
322static void psci_sys_poweroff(void)
323{
324	invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0);
325}
326
327static int __init psci_features(u32 psci_func_id)
328{
329	return invoke_psci_fn(PSCI_1_0_FN_PSCI_FEATURES,
330			      psci_func_id, 0, 0);
331}
332
333#ifdef CONFIG_CPU_IDLE
334static int psci_suspend_finisher(unsigned long state)
335{
336	u32 power_state = state;
337	phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume));
338
339	return psci_ops.cpu_suspend(power_state, pa_cpu_resume);
340}
341
342int psci_cpu_suspend_enter(u32 state)
343{
344	int ret;
345
346	if (!psci_power_state_loses_context(state)) {
347		struct arm_cpuidle_irq_context context;
348
349		arm_cpuidle_save_irq_context(&context);
350		ret = psci_ops.cpu_suspend(state, 0);
351		arm_cpuidle_restore_irq_context(&context);
352	} else {
353		ret = cpu_suspend(state, psci_suspend_finisher);
354	}
355
356	return ret;
357}
358#endif
359
360static int psci_system_suspend(unsigned long unused)
361{
362	phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume));
363
364	return invoke_psci_fn(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND),
365			      pa_cpu_resume, 0, 0);
366}
367
368static int psci_system_suspend_enter(suspend_state_t state)
369{
370	return cpu_suspend(0, psci_system_suspend);
371}
372
373static const struct platform_suspend_ops psci_suspend_ops = {
374	.valid          = suspend_valid_only_mem,
375	.enter          = psci_system_suspend_enter,
376};
377
378static void __init psci_init_system_reset2(void)
379{
380	int ret;
381
382	ret = psci_features(PSCI_FN_NATIVE(1_1, SYSTEM_RESET2));
383
384	if (ret != PSCI_RET_NOT_SUPPORTED)
385		psci_system_reset2_supported = true;
386}
387
388static void __init psci_init_system_suspend(void)
389{
390	int ret;
391
392	if (!IS_ENABLED(CONFIG_SUSPEND))
393		return;
394
395	ret = psci_features(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND));
396
397	if (ret != PSCI_RET_NOT_SUPPORTED)
398		suspend_set_ops(&psci_suspend_ops);
399}
400
401static void __init psci_init_cpu_suspend(void)
402{
403	int feature = psci_features(PSCI_FN_NATIVE(0_2, CPU_SUSPEND));
404
405	if (feature != PSCI_RET_NOT_SUPPORTED)
406		psci_cpu_suspend_feature = feature;
407}
408
409/*
410 * Detect the presence of a resident Trusted OS which may cause CPU_OFF to
411 * return DENIED (which would be fatal).
412 */
413static void __init psci_init_migrate(void)
414{
415	unsigned long cpuid;
416	int type, cpu = -1;
417
418	type = psci_ops.migrate_info_type();
419
420	if (type == PSCI_0_2_TOS_MP) {
421		pr_info("Trusted OS migration not required\n");
422		return;
423	}
424
425	if (type == PSCI_RET_NOT_SUPPORTED) {
426		pr_info("MIGRATE_INFO_TYPE not supported.\n");
427		return;
428	}
429
430	if (type != PSCI_0_2_TOS_UP_MIGRATE &&
431	    type != PSCI_0_2_TOS_UP_NO_MIGRATE) {
432		pr_err("MIGRATE_INFO_TYPE returned unknown type (%d)\n", type);
433		return;
434	}
435
436	cpuid = psci_migrate_info_up_cpu();
437	if (cpuid & ~MPIDR_HWID_BITMASK) {
438		pr_warn("MIGRATE_INFO_UP_CPU reported invalid physical ID (0x%lx)\n",
439			cpuid);
440		return;
441	}
442
443	cpu = get_logical_index(cpuid);
444	resident_cpu = cpu >= 0 ? cpu : -1;
445
446	pr_info("Trusted OS resident on physical CPU 0x%lx\n", cpuid);
447}
448
449static void __init psci_init_smccc(void)
450{
451	u32 ver = ARM_SMCCC_VERSION_1_0;
452	int feature;
453
454	feature = psci_features(ARM_SMCCC_VERSION_FUNC_ID);
455
456	if (feature != PSCI_RET_NOT_SUPPORTED) {
457		u32 ret;
458		ret = invoke_psci_fn(ARM_SMCCC_VERSION_FUNC_ID, 0, 0, 0);
459		if (ret >= ARM_SMCCC_VERSION_1_1) {
460			arm_smccc_version_init(ret, psci_conduit);
461			ver = ret;
462		}
463	}
464
465	/*
466	 * Conveniently, the SMCCC and PSCI versions are encoded the
467	 * same way. No, this isn't accidental.
468	 */
469	pr_info("SMC Calling Convention v%d.%d\n",
470		PSCI_VERSION_MAJOR(ver), PSCI_VERSION_MINOR(ver));
471
472}
473
474static void __init psci_0_2_set_functions(void)
475{
476	pr_info("Using standard PSCI v0.2 function IDs\n");
477
478	psci_ops = (struct psci_operations){
479		.get_version = psci_0_2_get_version,
480		.cpu_suspend = psci_0_2_cpu_suspend,
481		.cpu_off = psci_0_2_cpu_off,
482		.cpu_on = psci_0_2_cpu_on,
483		.migrate = psci_0_2_migrate,
484		.affinity_info = psci_affinity_info,
485		.migrate_info_type = psci_migrate_info_type,
486	};
487
488	register_restart_handler(&psci_sys_reset_nb);
489
490	pm_power_off = psci_sys_poweroff;
491}
492
493/*
494 * Probe function for PSCI firmware versions >= 0.2
495 */
496static int __init psci_probe(void)
497{
498	u32 ver = psci_0_2_get_version();
499
500	pr_info("PSCIv%d.%d detected in firmware.\n",
501			PSCI_VERSION_MAJOR(ver),
502			PSCI_VERSION_MINOR(ver));
503
504	if (PSCI_VERSION_MAJOR(ver) == 0 && PSCI_VERSION_MINOR(ver) < 2) {
505		pr_err("Conflicting PSCI version detected.\n");
506		return -EINVAL;
507	}
508
509	psci_0_2_set_functions();
510
511	psci_init_migrate();
512
513	if (PSCI_VERSION_MAJOR(ver) >= 1) {
514		psci_init_smccc();
515		psci_init_cpu_suspend();
516		psci_init_system_suspend();
517		psci_init_system_reset2();
518		kvm_init_hyp_services();
519	}
520
521	return 0;
522}
523
524typedef int (*psci_initcall_t)(const struct device_node *);
525
526/*
527 * PSCI init function for PSCI versions >=0.2
528 *
529 * Probe based on PSCI PSCI_VERSION function
530 */
531static int __init psci_0_2_init(struct device_node *np)
532{
533	int err;
534
535	err = get_set_conduit_method(np);
536	if (err)
537		return err;
538
539	/*
540	 * Starting with v0.2, the PSCI specification introduced a call
541	 * (PSCI_VERSION) that allows probing the firmware version, so
542	 * that PSCI function IDs and version specific initialization
543	 * can be carried out according to the specific version reported
544	 * by firmware
545	 */
546	return psci_probe();
547}
548
549/*
550 * PSCI < v0.2 get PSCI Function IDs via DT.
551 */
552static int __init psci_0_1_init(struct device_node *np)
553{
554	u32 id;
555	int err;
556
557	err = get_set_conduit_method(np);
558	if (err)
559		return err;
560
561	pr_info("Using PSCI v0.1 Function IDs from DT\n");
562
563	psci_ops.get_version = psci_0_1_get_version;
564
565	if (!of_property_read_u32(np, "cpu_suspend", &id)) {
566		psci_0_1_function_ids.cpu_suspend = id;
567		psci_ops.cpu_suspend = psci_0_1_cpu_suspend;
568	}
569
570	if (!of_property_read_u32(np, "cpu_off", &id)) {
571		psci_0_1_function_ids.cpu_off = id;
572		psci_ops.cpu_off = psci_0_1_cpu_off;
573	}
574
575	if (!of_property_read_u32(np, "cpu_on", &id)) {
576		psci_0_1_function_ids.cpu_on = id;
577		psci_ops.cpu_on = psci_0_1_cpu_on;
578	}
579
580	if (!of_property_read_u32(np, "migrate", &id)) {
581		psci_0_1_function_ids.migrate = id;
582		psci_ops.migrate = psci_0_1_migrate;
583	}
584
585	return 0;
586}
587
588static int __init psci_1_0_init(struct device_node *np)
589{
590	int err;
591
592	err = psci_0_2_init(np);
593	if (err)
594		return err;
595
596	if (psci_has_osi_support()) {
597		pr_info("OSI mode supported.\n");
598
599		/* Default to PC mode. */
600		psci_set_osi_mode(false);
601	}
602
603	return 0;
604}
605
606static const struct of_device_id psci_of_match[] __initconst = {
607	{ .compatible = "arm,psci",	.data = psci_0_1_init},
608	{ .compatible = "arm,psci-0.2",	.data = psci_0_2_init},
609	{ .compatible = "arm,psci-1.0",	.data = psci_1_0_init},
610	{},
611};
612
613int __init psci_dt_init(void)
614{
615	struct device_node *np;
616	const struct of_device_id *matched_np;
617	psci_initcall_t init_fn;
618	int ret;
619
620	np = of_find_matching_node_and_match(NULL, psci_of_match, &matched_np);
621
622	if (!np || !of_device_is_available(np))
623		return -ENODEV;
624
625	init_fn = (psci_initcall_t)matched_np->data;
626	ret = init_fn(np);
627
628	of_node_put(np);
629	return ret;
630}
631
632#ifdef CONFIG_ACPI
633/*
634 * We use PSCI 0.2+ when ACPI is deployed on ARM64 and it's
635 * explicitly clarified in SBBR
636 */
637int __init psci_acpi_init(void)
638{
639	if (!acpi_psci_present()) {
640		pr_info("is not implemented in ACPI.\n");
641		return -EOPNOTSUPP;
642	}
643
644	pr_info("probing for conduit method from ACPI.\n");
645
646	if (acpi_psci_use_hvc())
647		set_conduit(SMCCC_CONDUIT_HVC);
648	else
649		set_conduit(SMCCC_CONDUIT_SMC);
650
651	return psci_probe();
652}
653#endif