Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <linux/arm-smccc.h>
8#include <linux/preempt.h>
9#include <linux/kvm_host.h>
10#include <linux/uaccess.h>
11#include <linux/wait.h>
12
13#include <asm/cputype.h>
14#include <asm/kvm_emulate.h>
15
16#include <kvm/arm_psci.h>
17#include <kvm/arm_hypercalls.h>
18
19/*
20 * This is an implementation of the Power State Coordination Interface
21 * as described in ARM document number ARM DEN 0022A.
22 */
23
24#define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1)
25
26static unsigned long psci_affinity_mask(unsigned long affinity_level)
27{
28 if (affinity_level <= 3)
29 return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level);
30
31 return 0;
32}
33
34static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu)
35{
36 /*
37 * NOTE: For simplicity, we make VCPU suspend emulation to be
38 * same-as WFI (Wait-for-interrupt) emulation.
39 *
40 * This means for KVM the wakeup events are interrupts and
41 * this is consistent with intended use of StateID as described
42 * in section 5.4.1 of PSCI v0.2 specification (ARM DEN 0022A).
43 *
44 * Further, we also treat power-down request to be same as
45 * stand-by request as-per section 5.4.2 clause 3 of PSCI v0.2
46 * specification (ARM DEN 0022A). This means all suspend states
47 * for KVM will preserve the register state.
48 */
49 kvm_vcpu_wfi(vcpu);
50
51 return PSCI_RET_SUCCESS;
52}
53
54static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu,
55 unsigned long affinity)
56{
57 return !(affinity & ~MPIDR_HWID_BITMASK);
58}
59
60static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
61{
62 struct vcpu_reset_state *reset_state;
63 struct kvm *kvm = source_vcpu->kvm;
64 struct kvm_vcpu *vcpu = NULL;
65 int ret = PSCI_RET_SUCCESS;
66 unsigned long cpu_id;
67
68 cpu_id = smccc_get_arg1(source_vcpu);
69 if (!kvm_psci_valid_affinity(source_vcpu, cpu_id))
70 return PSCI_RET_INVALID_PARAMS;
71
72 vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id);
73
74 /*
75 * Make sure the caller requested a valid CPU and that the CPU is
76 * turned off.
77 */
78 if (!vcpu)
79 return PSCI_RET_INVALID_PARAMS;
80
81 spin_lock(&vcpu->arch.mp_state_lock);
82 if (!kvm_arm_vcpu_stopped(vcpu)) {
83 if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1)
84 ret = PSCI_RET_ALREADY_ON;
85 else
86 ret = PSCI_RET_INVALID_PARAMS;
87
88 goto out_unlock;
89 }
90
91 reset_state = &vcpu->arch.reset_state;
92
93 reset_state->pc = smccc_get_arg2(source_vcpu);
94
95 /* Propagate caller endianness */
96 reset_state->be = kvm_vcpu_is_be(source_vcpu);
97
98 /*
99 * NOTE: We always update r0 (or x0) because for PSCI v0.1
100 * the general purpose registers are undefined upon CPU_ON.
101 */
102 reset_state->r0 = smccc_get_arg3(source_vcpu);
103
104 reset_state->reset = true;
105 kvm_make_request(KVM_REQ_VCPU_RESET, vcpu);
106
107 /*
108 * Make sure the reset request is observed if the RUNNABLE mp_state is
109 * observed.
110 */
111 smp_wmb();
112
113 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
114 kvm_vcpu_wake_up(vcpu);
115
116out_unlock:
117 spin_unlock(&vcpu->arch.mp_state_lock);
118 return ret;
119}
120
121static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu)
122{
123 int matching_cpus = 0;
124 unsigned long i, mpidr;
125 unsigned long target_affinity;
126 unsigned long target_affinity_mask;
127 unsigned long lowest_affinity_level;
128 struct kvm *kvm = vcpu->kvm;
129 struct kvm_vcpu *tmp;
130
131 target_affinity = smccc_get_arg1(vcpu);
132 lowest_affinity_level = smccc_get_arg2(vcpu);
133
134 if (!kvm_psci_valid_affinity(vcpu, target_affinity))
135 return PSCI_RET_INVALID_PARAMS;
136
137 /* Determine target affinity mask */
138 target_affinity_mask = psci_affinity_mask(lowest_affinity_level);
139 if (!target_affinity_mask)
140 return PSCI_RET_INVALID_PARAMS;
141
142 /* Ignore other bits of target affinity */
143 target_affinity &= target_affinity_mask;
144
145 /*
146 * If one or more VCPU matching target affinity are running
147 * then ON else OFF
148 */
149 kvm_for_each_vcpu(i, tmp, kvm) {
150 mpidr = kvm_vcpu_get_mpidr_aff(tmp);
151 if ((mpidr & target_affinity_mask) == target_affinity) {
152 matching_cpus++;
153 if (!kvm_arm_vcpu_stopped(tmp))
154 return PSCI_0_2_AFFINITY_LEVEL_ON;
155 }
156 }
157
158 if (!matching_cpus)
159 return PSCI_RET_INVALID_PARAMS;
160
161 return PSCI_0_2_AFFINITY_LEVEL_OFF;
162}
163
164static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags)
165{
166 unsigned long i;
167 struct kvm_vcpu *tmp;
168
169 /*
170 * The KVM ABI specifies that a system event exit may call KVM_RUN
171 * again and may perform shutdown/reboot at a later time that when the
172 * actual request is made. Since we are implementing PSCI and a
173 * caller of PSCI reboot and shutdown expects that the system shuts
174 * down or reboots immediately, let's make sure that VCPUs are not run
175 * after this call is handled and before the VCPUs have been
176 * re-initialized.
177 */
178 kvm_for_each_vcpu(i, tmp, vcpu->kvm) {
179 spin_lock(&tmp->arch.mp_state_lock);
180 WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
181 spin_unlock(&tmp->arch.mp_state_lock);
182 }
183 kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP);
184
185 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
186 vcpu->run->system_event.type = type;
187 vcpu->run->system_event.ndata = 1;
188 vcpu->run->system_event.data[0] = flags;
189 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
190}
191
192static void kvm_psci_system_off(struct kvm_vcpu *vcpu)
193{
194 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_SHUTDOWN, 0);
195}
196
197static void kvm_psci_system_off2(struct kvm_vcpu *vcpu)
198{
199 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_SHUTDOWN,
200 KVM_SYSTEM_EVENT_SHUTDOWN_FLAG_PSCI_OFF2);
201}
202
203static void kvm_psci_system_reset(struct kvm_vcpu *vcpu)
204{
205 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, 0);
206}
207
208static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu)
209{
210 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET,
211 KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2);
212}
213
214static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu)
215{
216 struct kvm_run *run = vcpu->run;
217
218 memset(&run->system_event, 0, sizeof(vcpu->run->system_event));
219 run->system_event.type = KVM_SYSTEM_EVENT_SUSPEND;
220 run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
221}
222
223static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu)
224{
225 int i;
226
227 /*
228 * Zero the input registers' upper 32 bits. They will be fully
229 * zeroed on exit, so we're fine changing them in place.
230 */
231 for (i = 1; i < 4; i++)
232 vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i)));
233}
234
235static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn)
236{
237 /*
238 * Prevent 32 bit guests from calling 64 bit PSCI functions.
239 */
240 if ((fn & PSCI_0_2_64BIT) && vcpu_mode_is_32bit(vcpu))
241 return PSCI_RET_NOT_SUPPORTED;
242
243 return 0;
244}
245
246static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
247{
248 u32 psci_fn = smccc_get_function(vcpu);
249 unsigned long val;
250 int ret = 1;
251
252 switch (psci_fn) {
253 case PSCI_0_2_FN_PSCI_VERSION:
254 /*
255 * Bits[31:16] = Major Version = 0
256 * Bits[15:0] = Minor Version = 2
257 */
258 val = KVM_ARM_PSCI_0_2;
259 break;
260 case PSCI_0_2_FN_CPU_SUSPEND:
261 case PSCI_0_2_FN64_CPU_SUSPEND:
262 val = kvm_psci_vcpu_suspend(vcpu);
263 break;
264 case PSCI_0_2_FN_CPU_OFF:
265 kvm_arm_vcpu_power_off(vcpu);
266 val = PSCI_RET_SUCCESS;
267 break;
268 case PSCI_0_2_FN_CPU_ON:
269 kvm_psci_narrow_to_32bit(vcpu);
270 fallthrough;
271 case PSCI_0_2_FN64_CPU_ON:
272 val = kvm_psci_vcpu_on(vcpu);
273 break;
274 case PSCI_0_2_FN_AFFINITY_INFO:
275 kvm_psci_narrow_to_32bit(vcpu);
276 fallthrough;
277 case PSCI_0_2_FN64_AFFINITY_INFO:
278 val = kvm_psci_vcpu_affinity_info(vcpu);
279 break;
280 case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
281 /*
282 * Trusted OS is MP hence does not require migration
283 * or
284 * Trusted OS is not present
285 */
286 val = PSCI_0_2_TOS_MP;
287 break;
288 case PSCI_0_2_FN_SYSTEM_OFF:
289 kvm_psci_system_off(vcpu);
290 /*
291 * We shouldn't be going back to guest VCPU after
292 * receiving SYSTEM_OFF request.
293 *
294 * If user space accidentally/deliberately resumes
295 * guest VCPU after SYSTEM_OFF request then guest
296 * VCPU should see internal failure from PSCI return
297 * value. To achieve this, we preload r0 (or x0) with
298 * PSCI return value INTERNAL_FAILURE.
299 */
300 val = PSCI_RET_INTERNAL_FAILURE;
301 ret = 0;
302 break;
303 case PSCI_0_2_FN_SYSTEM_RESET:
304 kvm_psci_system_reset(vcpu);
305 /*
306 * Same reason as SYSTEM_OFF for preloading r0 (or x0)
307 * with PSCI return value INTERNAL_FAILURE.
308 */
309 val = PSCI_RET_INTERNAL_FAILURE;
310 ret = 0;
311 break;
312 default:
313 val = PSCI_RET_NOT_SUPPORTED;
314 break;
315 }
316
317 smccc_set_retval(vcpu, val, 0, 0, 0);
318 return ret;
319}
320
321static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor)
322{
323 unsigned long val = PSCI_RET_NOT_SUPPORTED;
324 u32 psci_fn = smccc_get_function(vcpu);
325 struct kvm *kvm = vcpu->kvm;
326 u32 arg;
327 int ret = 1;
328
329 switch(psci_fn) {
330 case PSCI_0_2_FN_PSCI_VERSION:
331 val = PSCI_VERSION(1, minor);
332 break;
333 case PSCI_1_0_FN_PSCI_FEATURES:
334 arg = smccc_get_arg1(vcpu);
335 val = kvm_psci_check_allowed_function(vcpu, arg);
336 if (val)
337 break;
338
339 val = PSCI_RET_NOT_SUPPORTED;
340
341 switch(arg) {
342 case PSCI_0_2_FN_PSCI_VERSION:
343 case PSCI_0_2_FN_CPU_SUSPEND:
344 case PSCI_0_2_FN64_CPU_SUSPEND:
345 case PSCI_0_2_FN_CPU_OFF:
346 case PSCI_0_2_FN_CPU_ON:
347 case PSCI_0_2_FN64_CPU_ON:
348 case PSCI_0_2_FN_AFFINITY_INFO:
349 case PSCI_0_2_FN64_AFFINITY_INFO:
350 case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
351 case PSCI_0_2_FN_SYSTEM_OFF:
352 case PSCI_0_2_FN_SYSTEM_RESET:
353 case PSCI_1_0_FN_PSCI_FEATURES:
354 case ARM_SMCCC_VERSION_FUNC_ID:
355 val = 0;
356 break;
357 case PSCI_1_0_FN_SYSTEM_SUSPEND:
358 case PSCI_1_0_FN64_SYSTEM_SUSPEND:
359 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags))
360 val = 0;
361 break;
362 case PSCI_1_1_FN_SYSTEM_RESET2:
363 case PSCI_1_1_FN64_SYSTEM_RESET2:
364 if (minor >= 1)
365 val = 0;
366 break;
367 case PSCI_1_3_FN_SYSTEM_OFF2:
368 case PSCI_1_3_FN64_SYSTEM_OFF2:
369 if (minor >= 3)
370 val = PSCI_1_3_OFF_TYPE_HIBERNATE_OFF;
371 break;
372 }
373 break;
374 case PSCI_1_0_FN_SYSTEM_SUSPEND:
375 kvm_psci_narrow_to_32bit(vcpu);
376 fallthrough;
377 case PSCI_1_0_FN64_SYSTEM_SUSPEND:
378 /*
379 * Return directly to userspace without changing the vCPU's
380 * registers. Userspace depends on reading the SMCCC parameters
381 * to implement SYSTEM_SUSPEND.
382 */
383 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) {
384 kvm_psci_system_suspend(vcpu);
385 return 0;
386 }
387 break;
388 case PSCI_1_1_FN_SYSTEM_RESET2:
389 kvm_psci_narrow_to_32bit(vcpu);
390 fallthrough;
391 case PSCI_1_1_FN64_SYSTEM_RESET2:
392 if (minor >= 1) {
393 arg = smccc_get_arg1(vcpu);
394
395 if (arg <= PSCI_1_1_RESET_TYPE_SYSTEM_WARM_RESET ||
396 arg >= PSCI_1_1_RESET_TYPE_VENDOR_START) {
397 kvm_psci_system_reset2(vcpu);
398 vcpu_set_reg(vcpu, 0, PSCI_RET_INTERNAL_FAILURE);
399 return 0;
400 }
401
402 val = PSCI_RET_INVALID_PARAMS;
403 break;
404 }
405 break;
406 case PSCI_1_3_FN_SYSTEM_OFF2:
407 kvm_psci_narrow_to_32bit(vcpu);
408 fallthrough;
409 case PSCI_1_3_FN64_SYSTEM_OFF2:
410 if (minor < 3)
411 break;
412
413 arg = smccc_get_arg1(vcpu);
414 /*
415 * SYSTEM_OFF2 defaults to HIBERNATE_OFF if arg1 is zero. arg2
416 * must be zero.
417 */
418 if ((arg && arg != PSCI_1_3_OFF_TYPE_HIBERNATE_OFF) ||
419 smccc_get_arg2(vcpu) != 0) {
420 val = PSCI_RET_INVALID_PARAMS;
421 break;
422 }
423 kvm_psci_system_off2(vcpu);
424 /*
425 * We shouldn't be going back to the guest after receiving a
426 * SYSTEM_OFF2 request. Preload a return value of
427 * INTERNAL_FAILURE should userspace ignore the exit and resume
428 * the vCPU.
429 */
430 val = PSCI_RET_INTERNAL_FAILURE;
431 ret = 0;
432 break;
433 default:
434 return kvm_psci_0_2_call(vcpu);
435 }
436
437 smccc_set_retval(vcpu, val, 0, 0, 0);
438 return ret;
439}
440
441static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
442{
443 u32 psci_fn = smccc_get_function(vcpu);
444 unsigned long val;
445
446 switch (psci_fn) {
447 case KVM_PSCI_FN_CPU_OFF:
448 kvm_arm_vcpu_power_off(vcpu);
449 val = PSCI_RET_SUCCESS;
450 break;
451 case KVM_PSCI_FN_CPU_ON:
452 val = kvm_psci_vcpu_on(vcpu);
453 break;
454 default:
455 val = PSCI_RET_NOT_SUPPORTED;
456 break;
457 }
458
459 smccc_set_retval(vcpu, val, 0, 0, 0);
460 return 1;
461}
462
463/**
464 * kvm_psci_call - handle PSCI call if r0 value is in range
465 * @vcpu: Pointer to the VCPU struct
466 *
467 * Handle PSCI calls from guests through traps from HVC instructions.
468 * The calling convention is similar to SMC calls to the secure world
469 * where the function number is placed in r0.
470 *
471 * This function returns: > 0 (success), 0 (success but exit to user
472 * space), and < 0 (errors)
473 *
474 * Errors:
475 * -EINVAL: Unrecognized PSCI function
476 */
477int kvm_psci_call(struct kvm_vcpu *vcpu)
478{
479 u32 psci_fn = smccc_get_function(vcpu);
480 int version = kvm_psci_version(vcpu);
481 unsigned long val;
482
483 val = kvm_psci_check_allowed_function(vcpu, psci_fn);
484 if (val) {
485 smccc_set_retval(vcpu, val, 0, 0, 0);
486 return 1;
487 }
488
489 switch (version) {
490 case KVM_ARM_PSCI_1_3:
491 return kvm_psci_1_x_call(vcpu, 3);
492 case KVM_ARM_PSCI_1_2:
493 return kvm_psci_1_x_call(vcpu, 2);
494 case KVM_ARM_PSCI_1_1:
495 return kvm_psci_1_x_call(vcpu, 1);
496 case KVM_ARM_PSCI_1_0:
497 return kvm_psci_1_x_call(vcpu, 0);
498 case KVM_ARM_PSCI_0_2:
499 return kvm_psci_0_2_call(vcpu);
500 case KVM_ARM_PSCI_0_1:
501 return kvm_psci_0_1_call(vcpu);
502 default:
503 WARN_ONCE(1, "Unknown PSCI version %d", version);
504 smccc_set_retval(vcpu, SMCCC_RET_NOT_SUPPORTED, 0, 0, 0);
505 return 1;
506 }
507}