Loading...
Note: File does not exist in v3.5.6.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2015 Linaro Ltd.
4 * Author: Shannon Zhao <shannon.zhao@linaro.org>
5 */
6
7#include <linux/cpu.h>
8#include <linux/kvm.h>
9#include <linux/kvm_host.h>
10#include <linux/perf_event.h>
11#include <linux/perf/arm_pmu.h>
12#include <linux/uaccess.h>
13#include <asm/kvm_emulate.h>
14#include <kvm/arm_pmu.h>
15#include <kvm/arm_vgic.h>
16
17static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx);
18
19#define PERF_ATTR_CFG1_KVM_PMU_CHAINED 0x1
20
21/**
22 * kvm_pmu_idx_is_64bit - determine if select_idx is a 64bit counter
23 * @vcpu: The vcpu pointer
24 * @select_idx: The counter index
25 */
26static bool kvm_pmu_idx_is_64bit(struct kvm_vcpu *vcpu, u64 select_idx)
27{
28 return (select_idx == ARMV8_PMU_CYCLE_IDX &&
29 __vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_LC);
30}
31
32static struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc)
33{
34 struct kvm_pmu *pmu;
35 struct kvm_vcpu_arch *vcpu_arch;
36
37 pmc -= pmc->idx;
38 pmu = container_of(pmc, struct kvm_pmu, pmc[0]);
39 vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
40 return container_of(vcpu_arch, struct kvm_vcpu, arch);
41}
42
43/**
44 * kvm_pmu_pmc_is_chained - determine if the pmc is chained
45 * @pmc: The PMU counter pointer
46 */
47static bool kvm_pmu_pmc_is_chained(struct kvm_pmc *pmc)
48{
49 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
50
51 return test_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
52}
53
54/**
55 * kvm_pmu_idx_is_high_counter - determine if select_idx is a high/low counter
56 * @select_idx: The counter index
57 */
58static bool kvm_pmu_idx_is_high_counter(u64 select_idx)
59{
60 return select_idx & 0x1;
61}
62
63/**
64 * kvm_pmu_get_canonical_pmc - obtain the canonical pmc
65 * @pmc: The PMU counter pointer
66 *
67 * When a pair of PMCs are chained together we use the low counter (canonical)
68 * to hold the underlying perf event.
69 */
70static struct kvm_pmc *kvm_pmu_get_canonical_pmc(struct kvm_pmc *pmc)
71{
72 if (kvm_pmu_pmc_is_chained(pmc) &&
73 kvm_pmu_idx_is_high_counter(pmc->idx))
74 return pmc - 1;
75
76 return pmc;
77}
78
79/**
80 * kvm_pmu_idx_has_chain_evtype - determine if the event type is chain
81 * @vcpu: The vcpu pointer
82 * @select_idx: The counter index
83 */
84static bool kvm_pmu_idx_has_chain_evtype(struct kvm_vcpu *vcpu, u64 select_idx)
85{
86 u64 eventsel, reg;
87
88 select_idx |= 0x1;
89
90 if (select_idx == ARMV8_PMU_CYCLE_IDX)
91 return false;
92
93 reg = PMEVTYPER0_EL0 + select_idx;
94 eventsel = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_EVENT;
95
96 return eventsel == ARMV8_PMUV3_PERFCTR_CHAIN;
97}
98
99/**
100 * kvm_pmu_get_pair_counter_value - get PMU counter value
101 * @vcpu: The vcpu pointer
102 * @pmc: The PMU counter pointer
103 */
104static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu,
105 struct kvm_pmc *pmc)
106{
107 u64 counter, counter_high, reg, enabled, running;
108
109 if (kvm_pmu_pmc_is_chained(pmc)) {
110 pmc = kvm_pmu_get_canonical_pmc(pmc);
111 reg = PMEVCNTR0_EL0 + pmc->idx;
112
113 counter = __vcpu_sys_reg(vcpu, reg);
114 counter_high = __vcpu_sys_reg(vcpu, reg + 1);
115
116 counter = lower_32_bits(counter) | (counter_high << 32);
117 } else {
118 reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
119 ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
120 counter = __vcpu_sys_reg(vcpu, reg);
121 }
122
123 /*
124 * The real counter value is equal to the value of counter register plus
125 * the value perf event counts.
126 */
127 if (pmc->perf_event)
128 counter += perf_event_read_value(pmc->perf_event, &enabled,
129 &running);
130
131 return counter;
132}
133
134/**
135 * kvm_pmu_get_counter_value - get PMU counter value
136 * @vcpu: The vcpu pointer
137 * @select_idx: The counter index
138 */
139u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
140{
141 u64 counter;
142 struct kvm_pmu *pmu = &vcpu->arch.pmu;
143 struct kvm_pmc *pmc = &pmu->pmc[select_idx];
144
145 counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
146
147 if (kvm_pmu_pmc_is_chained(pmc) &&
148 kvm_pmu_idx_is_high_counter(select_idx))
149 counter = upper_32_bits(counter);
150 else if (select_idx != ARMV8_PMU_CYCLE_IDX)
151 counter = lower_32_bits(counter);
152
153 return counter;
154}
155
156/**
157 * kvm_pmu_set_counter_value - set PMU counter value
158 * @vcpu: The vcpu pointer
159 * @select_idx: The counter index
160 * @val: The counter value
161 */
162void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
163{
164 u64 reg;
165
166 reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
167 ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
168 __vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
169
170 /* Recreate the perf event to reflect the updated sample_period */
171 kvm_pmu_create_perf_event(vcpu, select_idx);
172}
173
174/**
175 * kvm_pmu_release_perf_event - remove the perf event
176 * @pmc: The PMU counter pointer
177 */
178static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)
179{
180 pmc = kvm_pmu_get_canonical_pmc(pmc);
181 if (pmc->perf_event) {
182 perf_event_disable(pmc->perf_event);
183 perf_event_release_kernel(pmc->perf_event);
184 pmc->perf_event = NULL;
185 }
186}
187
188/**
189 * kvm_pmu_stop_counter - stop PMU counter
190 * @pmc: The PMU counter pointer
191 *
192 * If this counter has been configured to monitor some event, release it here.
193 */
194static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)
195{
196 u64 counter, reg, val;
197
198 pmc = kvm_pmu_get_canonical_pmc(pmc);
199 if (!pmc->perf_event)
200 return;
201
202 counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
203
204 if (pmc->idx == ARMV8_PMU_CYCLE_IDX) {
205 reg = PMCCNTR_EL0;
206 val = counter;
207 } else {
208 reg = PMEVCNTR0_EL0 + pmc->idx;
209 val = lower_32_bits(counter);
210 }
211
212 __vcpu_sys_reg(vcpu, reg) = val;
213
214 if (kvm_pmu_pmc_is_chained(pmc))
215 __vcpu_sys_reg(vcpu, reg + 1) = upper_32_bits(counter);
216
217 kvm_pmu_release_perf_event(pmc);
218}
219
220/**
221 * kvm_pmu_vcpu_init - assign pmu counter idx for cpu
222 * @vcpu: The vcpu pointer
223 *
224 */
225void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu)
226{
227 int i;
228 struct kvm_pmu *pmu = &vcpu->arch.pmu;
229
230 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
231 pmu->pmc[i].idx = i;
232}
233
234/**
235 * kvm_pmu_vcpu_reset - reset pmu state for cpu
236 * @vcpu: The vcpu pointer
237 *
238 */
239void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
240{
241 int i;
242 struct kvm_pmu *pmu = &vcpu->arch.pmu;
243
244 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
245 kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);
246
247 bitmap_zero(vcpu->arch.pmu.chained, ARMV8_PMU_MAX_COUNTER_PAIRS);
248}
249
250/**
251 * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu
252 * @vcpu: The vcpu pointer
253 *
254 */
255void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)
256{
257 int i;
258 struct kvm_pmu *pmu = &vcpu->arch.pmu;
259
260 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
261 kvm_pmu_release_perf_event(&pmu->pmc[i]);
262}
263
264u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
265{
266 u64 val = __vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;
267
268 val &= ARMV8_PMU_PMCR_N_MASK;
269 if (val == 0)
270 return BIT(ARMV8_PMU_CYCLE_IDX);
271 else
272 return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX);
273}
274
275/**
276 * kvm_pmu_enable_counter_mask - enable selected PMU counters
277 * @vcpu: The vcpu pointer
278 * @val: the value guest writes to PMCNTENSET register
279 *
280 * Call perf_event_enable to start counting the perf event
281 */
282void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
283{
284 int i;
285 struct kvm_pmu *pmu = &vcpu->arch.pmu;
286 struct kvm_pmc *pmc;
287
288 if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val)
289 return;
290
291 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
292 if (!(val & BIT(i)))
293 continue;
294
295 pmc = &pmu->pmc[i];
296
297 /*
298 * For high counters of chained events we must recreate the
299 * perf event with the long (64bit) attribute set.
300 */
301 if (kvm_pmu_pmc_is_chained(pmc) &&
302 kvm_pmu_idx_is_high_counter(i)) {
303 kvm_pmu_create_perf_event(vcpu, i);
304 continue;
305 }
306
307 /* At this point, pmc must be the canonical */
308 if (pmc->perf_event) {
309 perf_event_enable(pmc->perf_event);
310 if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
311 kvm_debug("fail to enable perf event\n");
312 }
313 }
314}
315
316/**
317 * kvm_pmu_disable_counter_mask - disable selected PMU counters
318 * @vcpu: The vcpu pointer
319 * @val: the value guest writes to PMCNTENCLR register
320 *
321 * Call perf_event_disable to stop counting the perf event
322 */
323void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
324{
325 int i;
326 struct kvm_pmu *pmu = &vcpu->arch.pmu;
327 struct kvm_pmc *pmc;
328
329 if (!val)
330 return;
331
332 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
333 if (!(val & BIT(i)))
334 continue;
335
336 pmc = &pmu->pmc[i];
337
338 /*
339 * For high counters of chained events we must recreate the
340 * perf event with the long (64bit) attribute unset.
341 */
342 if (kvm_pmu_pmc_is_chained(pmc) &&
343 kvm_pmu_idx_is_high_counter(i)) {
344 kvm_pmu_create_perf_event(vcpu, i);
345 continue;
346 }
347
348 /* At this point, pmc must be the canonical */
349 if (pmc->perf_event)
350 perf_event_disable(pmc->perf_event);
351 }
352}
353
354static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
355{
356 u64 reg = 0;
357
358 if ((__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) {
359 reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0);
360 reg &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
361 reg &= __vcpu_sys_reg(vcpu, PMINTENSET_EL1);
362 reg &= kvm_pmu_valid_counter_mask(vcpu);
363 }
364
365 return reg;
366}
367
368static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
369{
370 struct kvm_pmu *pmu = &vcpu->arch.pmu;
371 bool overflow;
372
373 if (!kvm_arm_pmu_v3_ready(vcpu))
374 return;
375
376 overflow = !!kvm_pmu_overflow_status(vcpu);
377 if (pmu->irq_level == overflow)
378 return;
379
380 pmu->irq_level = overflow;
381
382 if (likely(irqchip_in_kernel(vcpu->kvm))) {
383 int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
384 pmu->irq_num, overflow, pmu);
385 WARN_ON(ret);
386 }
387}
388
389bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
390{
391 struct kvm_pmu *pmu = &vcpu->arch.pmu;
392 struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
393 bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU;
394
395 if (likely(irqchip_in_kernel(vcpu->kvm)))
396 return false;
397
398 return pmu->irq_level != run_level;
399}
400
401/*
402 * Reflect the PMU overflow interrupt output level into the kvm_run structure
403 */
404void kvm_pmu_update_run(struct kvm_vcpu *vcpu)
405{
406 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
407
408 /* Populate the timer bitmap for user space */
409 regs->device_irq_level &= ~KVM_ARM_DEV_PMU;
410 if (vcpu->arch.pmu.irq_level)
411 regs->device_irq_level |= KVM_ARM_DEV_PMU;
412}
413
414/**
415 * kvm_pmu_flush_hwstate - flush pmu state to cpu
416 * @vcpu: The vcpu pointer
417 *
418 * Check if the PMU has overflowed while we were running in the host, and inject
419 * an interrupt if that was the case.
420 */
421void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)
422{
423 kvm_pmu_update_state(vcpu);
424}
425
426/**
427 * kvm_pmu_sync_hwstate - sync pmu state from cpu
428 * @vcpu: The vcpu pointer
429 *
430 * Check if the PMU has overflowed while we were running in the guest, and
431 * inject an interrupt if that was the case.
432 */
433void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
434{
435 kvm_pmu_update_state(vcpu);
436}
437
438/**
439 * When the perf event overflows, set the overflow status and inform the vcpu.
440 */
441static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
442 struct perf_sample_data *data,
443 struct pt_regs *regs)
444{
445 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
446 struct arm_pmu *cpu_pmu = to_arm_pmu(perf_event->pmu);
447 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
448 int idx = pmc->idx;
449 u64 period;
450
451 cpu_pmu->pmu.stop(perf_event, PERF_EF_UPDATE);
452
453 /*
454 * Reset the sample period to the architectural limit,
455 * i.e. the point where the counter overflows.
456 */
457 period = -(local64_read(&perf_event->count));
458
459 if (!kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
460 period &= GENMASK(31, 0);
461
462 local64_set(&perf_event->hw.period_left, 0);
463 perf_event->attr.sample_period = period;
464 perf_event->hw.sample_period = period;
465
466 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx);
467
468 if (kvm_pmu_overflow_status(vcpu)) {
469 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
470 kvm_vcpu_kick(vcpu);
471 }
472
473 cpu_pmu->pmu.start(perf_event, PERF_EF_RELOAD);
474}
475
476/**
477 * kvm_pmu_software_increment - do software increment
478 * @vcpu: The vcpu pointer
479 * @val: the value guest writes to PMSWINC register
480 */
481void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
482{
483 int i;
484 u64 type, enable, reg;
485
486 if (val == 0)
487 return;
488
489 enable = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
490 for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
491 if (!(val & BIT(i)))
492 continue;
493 type = __vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i)
494 & ARMV8_PMU_EVTYPE_EVENT;
495 if ((type == ARMV8_PMUV3_PERFCTR_SW_INCR)
496 && (enable & BIT(i))) {
497 reg = __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
498 reg = lower_32_bits(reg);
499 __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
500 if (!reg)
501 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i);
502 }
503 }
504}
505
506/**
507 * kvm_pmu_handle_pmcr - handle PMCR register
508 * @vcpu: The vcpu pointer
509 * @val: the value guest writes to PMCR register
510 */
511void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)
512{
513 u64 mask;
514 int i;
515
516 mask = kvm_pmu_valid_counter_mask(vcpu);
517 if (val & ARMV8_PMU_PMCR_E) {
518 kvm_pmu_enable_counter_mask(vcpu,
519 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask);
520 } else {
521 kvm_pmu_disable_counter_mask(vcpu, mask);
522 }
523
524 if (val & ARMV8_PMU_PMCR_C)
525 kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);
526
527 if (val & ARMV8_PMU_PMCR_P) {
528 for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++)
529 kvm_pmu_set_counter_value(vcpu, i, 0);
530 }
531}
532
533static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)
534{
535 return (__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
536 (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));
537}
538
539/**
540 * kvm_pmu_create_perf_event - create a perf event for a counter
541 * @vcpu: The vcpu pointer
542 * @select_idx: The number of selected counter
543 */
544static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx)
545{
546 struct kvm_pmu *pmu = &vcpu->arch.pmu;
547 struct kvm_pmc *pmc;
548 struct perf_event *event;
549 struct perf_event_attr attr;
550 u64 eventsel, counter, reg, data;
551
552 /*
553 * For chained counters the event type and filtering attributes are
554 * obtained from the low/even counter. We also use this counter to
555 * determine if the event is enabled/disabled.
556 */
557 pmc = kvm_pmu_get_canonical_pmc(&pmu->pmc[select_idx]);
558
559 reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
560 ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + pmc->idx;
561 data = __vcpu_sys_reg(vcpu, reg);
562
563 kvm_pmu_stop_counter(vcpu, pmc);
564 eventsel = data & ARMV8_PMU_EVTYPE_EVENT;
565
566 /* Software increment event does't need to be backed by a perf event */
567 if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR &&
568 pmc->idx != ARMV8_PMU_CYCLE_IDX)
569 return;
570
571 memset(&attr, 0, sizeof(struct perf_event_attr));
572 attr.type = PERF_TYPE_RAW;
573 attr.size = sizeof(attr);
574 attr.pinned = 1;
575 attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, pmc->idx);
576 attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
577 attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
578 attr.exclude_hv = 1; /* Don't count EL2 events */
579 attr.exclude_host = 1; /* Don't count host events */
580 attr.config = (pmc->idx == ARMV8_PMU_CYCLE_IDX) ?
581 ARMV8_PMUV3_PERFCTR_CPU_CYCLES : eventsel;
582
583 counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
584
585 if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {
586 /**
587 * The initial sample period (overflow count) of an event. For
588 * chained counters we only support overflow interrupts on the
589 * high counter.
590 */
591 attr.sample_period = (-counter) & GENMASK(63, 0);
592 if (kvm_pmu_counter_is_enabled(vcpu, pmc->idx + 1))
593 attr.config1 |= PERF_ATTR_CFG1_KVM_PMU_CHAINED;
594
595 event = perf_event_create_kernel_counter(&attr, -1, current,
596 kvm_pmu_perf_overflow,
597 pmc + 1);
598 } else {
599 /* The initial sample period (overflow count) of an event. */
600 if (kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
601 attr.sample_period = (-counter) & GENMASK(63, 0);
602 else
603 attr.sample_period = (-counter) & GENMASK(31, 0);
604
605 event = perf_event_create_kernel_counter(&attr, -1, current,
606 kvm_pmu_perf_overflow, pmc);
607 }
608
609 if (IS_ERR(event)) {
610 pr_err_once("kvm: pmu event creation failed %ld\n",
611 PTR_ERR(event));
612 return;
613 }
614
615 pmc->perf_event = event;
616}
617
618/**
619 * kvm_pmu_update_pmc_chained - update chained bitmap
620 * @vcpu: The vcpu pointer
621 * @select_idx: The number of selected counter
622 *
623 * Update the chained bitmap based on the event type written in the
624 * typer register.
625 */
626static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx)
627{
628 struct kvm_pmu *pmu = &vcpu->arch.pmu;
629 struct kvm_pmc *pmc = &pmu->pmc[select_idx];
630
631 if (kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx)) {
632 /*
633 * During promotion from !chained to chained we must ensure
634 * the adjacent counter is stopped and its event destroyed
635 */
636 if (!kvm_pmu_pmc_is_chained(pmc))
637 kvm_pmu_stop_counter(vcpu, pmc);
638
639 set_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
640 } else {
641 clear_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
642 }
643}
644
645/**
646 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
647 * @vcpu: The vcpu pointer
648 * @data: The data guest writes to PMXEVTYPER_EL0
649 * @select_idx: The number of selected counter
650 *
651 * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
652 * event with given hardware event number. Here we call perf_event API to
653 * emulate this action and create a kernel perf event for it.
654 */
655void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
656 u64 select_idx)
657{
658 u64 reg, event_type = data & ARMV8_PMU_EVTYPE_MASK;
659
660 reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
661 ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + select_idx;
662
663 __vcpu_sys_reg(vcpu, reg) = event_type;
664
665 kvm_pmu_update_pmc_chained(vcpu, select_idx);
666 kvm_pmu_create_perf_event(vcpu, select_idx);
667}
668
669bool kvm_arm_support_pmu_v3(void)
670{
671 /*
672 * Check if HW_PERF_EVENTS are supported by checking the number of
673 * hardware performance counters. This could ensure the presence of
674 * a physical PMU and CONFIG_PERF_EVENT is selected.
675 */
676 return (perf_num_counters() > 0);
677}
678
679int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu)
680{
681 if (!vcpu->arch.pmu.created)
682 return 0;
683
684 /*
685 * A valid interrupt configuration for the PMU is either to have a
686 * properly configured interrupt number and using an in-kernel
687 * irqchip, or to not have an in-kernel GIC and not set an IRQ.
688 */
689 if (irqchip_in_kernel(vcpu->kvm)) {
690 int irq = vcpu->arch.pmu.irq_num;
691 if (!kvm_arm_pmu_irq_initialized(vcpu))
692 return -EINVAL;
693
694 /*
695 * If we are using an in-kernel vgic, at this point we know
696 * the vgic will be initialized, so we can check the PMU irq
697 * number against the dimensions of the vgic and make sure
698 * it's valid.
699 */
700 if (!irq_is_ppi(irq) && !vgic_valid_spi(vcpu->kvm, irq))
701 return -EINVAL;
702 } else if (kvm_arm_pmu_irq_initialized(vcpu)) {
703 return -EINVAL;
704 }
705
706 kvm_pmu_vcpu_reset(vcpu);
707 vcpu->arch.pmu.ready = true;
708
709 return 0;
710}
711
712static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu)
713{
714 if (!kvm_arm_support_pmu_v3())
715 return -ENODEV;
716
717 if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
718 return -ENXIO;
719
720 if (vcpu->arch.pmu.created)
721 return -EBUSY;
722
723 if (irqchip_in_kernel(vcpu->kvm)) {
724 int ret;
725
726 /*
727 * If using the PMU with an in-kernel virtual GIC
728 * implementation, we require the GIC to be already
729 * initialized when initializing the PMU.
730 */
731 if (!vgic_initialized(vcpu->kvm))
732 return -ENODEV;
733
734 if (!kvm_arm_pmu_irq_initialized(vcpu))
735 return -ENXIO;
736
737 ret = kvm_vgic_set_owner(vcpu, vcpu->arch.pmu.irq_num,
738 &vcpu->arch.pmu);
739 if (ret)
740 return ret;
741 }
742
743 vcpu->arch.pmu.created = true;
744 return 0;
745}
746
747/*
748 * For one VM the interrupt type must be same for each vcpu.
749 * As a PPI, the interrupt number is the same for all vcpus,
750 * while as an SPI it must be a separate number per vcpu.
751 */
752static bool pmu_irq_is_valid(struct kvm *kvm, int irq)
753{
754 int i;
755 struct kvm_vcpu *vcpu;
756
757 kvm_for_each_vcpu(i, vcpu, kvm) {
758 if (!kvm_arm_pmu_irq_initialized(vcpu))
759 continue;
760
761 if (irq_is_ppi(irq)) {
762 if (vcpu->arch.pmu.irq_num != irq)
763 return false;
764 } else {
765 if (vcpu->arch.pmu.irq_num == irq)
766 return false;
767 }
768 }
769
770 return true;
771}
772
773int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
774{
775 switch (attr->attr) {
776 case KVM_ARM_VCPU_PMU_V3_IRQ: {
777 int __user *uaddr = (int __user *)(long)attr->addr;
778 int irq;
779
780 if (!irqchip_in_kernel(vcpu->kvm))
781 return -EINVAL;
782
783 if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
784 return -ENODEV;
785
786 if (get_user(irq, uaddr))
787 return -EFAULT;
788
789 /* The PMU overflow interrupt can be a PPI or a valid SPI. */
790 if (!(irq_is_ppi(irq) || irq_is_spi(irq)))
791 return -EINVAL;
792
793 if (!pmu_irq_is_valid(vcpu->kvm, irq))
794 return -EINVAL;
795
796 if (kvm_arm_pmu_irq_initialized(vcpu))
797 return -EBUSY;
798
799 kvm_debug("Set kvm ARM PMU irq: %d\n", irq);
800 vcpu->arch.pmu.irq_num = irq;
801 return 0;
802 }
803 case KVM_ARM_VCPU_PMU_V3_INIT:
804 return kvm_arm_pmu_v3_init(vcpu);
805 }
806
807 return -ENXIO;
808}
809
810int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
811{
812 switch (attr->attr) {
813 case KVM_ARM_VCPU_PMU_V3_IRQ: {
814 int __user *uaddr = (int __user *)(long)attr->addr;
815 int irq;
816
817 if (!irqchip_in_kernel(vcpu->kvm))
818 return -EINVAL;
819
820 if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
821 return -ENODEV;
822
823 if (!kvm_arm_pmu_irq_initialized(vcpu))
824 return -ENXIO;
825
826 irq = vcpu->arch.pmu.irq_num;
827 return put_user(irq, uaddr);
828 }
829 }
830
831 return -ENXIO;
832}
833
834int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
835{
836 switch (attr->attr) {
837 case KVM_ARM_VCPU_PMU_V3_IRQ:
838 case KVM_ARM_VCPU_PMU_V3_INIT:
839 if (kvm_arm_support_pmu_v3() &&
840 test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
841 return 0;
842 }
843
844 return -ENXIO;
845}