Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Kernel-based Virtual Machine -- Performance Monitoring Unit support
4 *
5 * Copyright 2015 Red Hat, Inc. and/or its affiliates.
6 *
7 * Authors:
8 * Avi Kivity <avi@redhat.com>
9 * Gleb Natapov <gleb@redhat.com>
10 * Wei Huang <wei@redhat.com>
11 */
12#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13
14#include <linux/types.h>
15#include <linux/kvm_host.h>
16#include <linux/perf_event.h>
17#include <linux/bsearch.h>
18#include <linux/sort.h>
19#include <asm/perf_event.h>
20#include <asm/cpu_device_id.h>
21#include "x86.h"
22#include "cpuid.h"
23#include "lapic.h"
24#include "pmu.h"
25
26/* This is enough to filter the vast majority of currently defined events. */
27#define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300
28
29struct x86_pmu_capability __read_mostly kvm_pmu_cap;
30EXPORT_SYMBOL_GPL(kvm_pmu_cap);
31
32struct kvm_pmu_emulated_event_selectors __read_mostly kvm_pmu_eventsel;
33EXPORT_SYMBOL_GPL(kvm_pmu_eventsel);
34
35/* Precise Distribution of Instructions Retired (PDIR) */
36static const struct x86_cpu_id vmx_pebs_pdir_cpu[] = {
37 X86_MATCH_VFM(INTEL_ICELAKE_D, NULL),
38 X86_MATCH_VFM(INTEL_ICELAKE_X, NULL),
39 /* Instruction-Accurate PDIR (PDIR++) */
40 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL),
41 {}
42};
43
44/* Precise Distribution (PDist) */
45static const struct x86_cpu_id vmx_pebs_pdist_cpu[] = {
46 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL),
47 {}
48};
49
50/* NOTE:
51 * - Each perf counter is defined as "struct kvm_pmc";
52 * - There are two types of perf counters: general purpose (gp) and fixed.
53 * gp counters are stored in gp_counters[] and fixed counters are stored
54 * in fixed_counters[] respectively. Both of them are part of "struct
55 * kvm_pmu";
56 * - pmu.c understands the difference between gp counters and fixed counters.
57 * However AMD doesn't support fixed-counters;
58 * - There are three types of index to access perf counters (PMC):
59 * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD
60 * has MSR_K7_PERFCTRn and, for families 15H and later,
61 * MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are
62 * aliased to MSR_K7_PERFCTRn.
63 * 2. MSR Index (named idx): This normally is used by RDPMC instruction.
64 * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access
65 * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except
66 * that it also supports fixed counters. idx can be used to as index to
67 * gp and fixed counters.
68 * 3. Global PMC Index (named pmc): pmc is an index specific to PMU
69 * code. Each pmc, stored in kvm_pmc.idx field, is unique across
70 * all perf counters (both gp and fixed). The mapping relationship
71 * between pmc and perf counters is as the following:
72 * * Intel: [0 .. KVM_MAX_NR_INTEL_GP_COUNTERS-1] <=> gp counters
73 * [KVM_FIXED_PMC_BASE_IDX .. KVM_FIXED_PMC_BASE_IDX + 2] <=> fixed
74 * * AMD: [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H
75 * and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters
76 */
77
78static struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
79
80#define KVM_X86_PMU_OP(func) \
81 DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \
82 *(((struct kvm_pmu_ops *)0)->func));
83#define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP
84#include <asm/kvm-x86-pmu-ops.h>
85
86void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops)
87{
88 memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops));
89
90#define __KVM_X86_PMU_OP(func) \
91 static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func);
92#define KVM_X86_PMU_OP(func) \
93 WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func)
94#define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP
95#include <asm/kvm-x86-pmu-ops.h>
96#undef __KVM_X86_PMU_OP
97}
98
99static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi)
100{
101 struct kvm_pmu *pmu = pmc_to_pmu(pmc);
102 bool skip_pmi = false;
103
104 if (pmc->perf_event && pmc->perf_event->attr.precise_ip) {
105 if (!in_pmi) {
106 /*
107 * TODO: KVM is currently _choosing_ to not generate records
108 * for emulated instructions, avoiding BUFFER_OVF PMI when
109 * there are no records. Strictly speaking, it should be done
110 * as well in the right context to improve sampling accuracy.
111 */
112 skip_pmi = true;
113 } else {
114 /* Indicate PEBS overflow PMI to guest. */
115 skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT,
116 (unsigned long *)&pmu->global_status);
117 }
118 } else {
119 __set_bit(pmc->idx, (unsigned long *)&pmu->global_status);
120 }
121
122 if (pmc->intr && !skip_pmi)
123 kvm_make_request(KVM_REQ_PMI, pmc->vcpu);
124}
125
126static void kvm_perf_overflow(struct perf_event *perf_event,
127 struct perf_sample_data *data,
128 struct pt_regs *regs)
129{
130 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
131
132 /*
133 * Ignore asynchronous overflow events for counters that are scheduled
134 * to be reprogrammed, e.g. if a PMI for the previous event races with
135 * KVM's handling of a related guest WRMSR.
136 */
137 if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi))
138 return;
139
140 __kvm_perf_overflow(pmc, true);
141
142 kvm_make_request(KVM_REQ_PMU, pmc->vcpu);
143}
144
145static u64 pmc_get_pebs_precise_level(struct kvm_pmc *pmc)
146{
147 /*
148 * For some model specific pebs counters with special capabilities
149 * (PDIR, PDIR++, PDIST), KVM needs to raise the event precise
150 * level to the maximum value (currently 3, backwards compatible)
151 * so that the perf subsystem would assign specific hardware counter
152 * with that capability for vPMC.
153 */
154 if ((pmc->idx == 0 && x86_match_cpu(vmx_pebs_pdist_cpu)) ||
155 (pmc->idx == 32 && x86_match_cpu(vmx_pebs_pdir_cpu)))
156 return 3;
157
158 /*
159 * The non-zero precision level of guest event makes the ordinary
160 * guest event becomes a guest PEBS event and triggers the host
161 * PEBS PMI handler to determine whether the PEBS overflow PMI
162 * comes from the host counters or the guest.
163 */
164 return 1;
165}
166
167static u64 get_sample_period(struct kvm_pmc *pmc, u64 counter_value)
168{
169 u64 sample_period = (-counter_value) & pmc_bitmask(pmc);
170
171 if (!sample_period)
172 sample_period = pmc_bitmask(pmc) + 1;
173 return sample_period;
174}
175
176static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config,
177 bool exclude_user, bool exclude_kernel,
178 bool intr)
179{
180 struct kvm_pmu *pmu = pmc_to_pmu(pmc);
181 struct perf_event *event;
182 struct perf_event_attr attr = {
183 .type = type,
184 .size = sizeof(attr),
185 .pinned = true,
186 .exclude_idle = true,
187 .exclude_host = 1,
188 .exclude_user = exclude_user,
189 .exclude_kernel = exclude_kernel,
190 .config = config,
191 };
192 bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable);
193
194 attr.sample_period = get_sample_period(pmc, pmc->counter);
195
196 if ((attr.config & HSW_IN_TX_CHECKPOINTED) &&
197 (boot_cpu_has(X86_FEATURE_RTM) || boot_cpu_has(X86_FEATURE_HLE))) {
198 /*
199 * HSW_IN_TX_CHECKPOINTED is not supported with nonzero
200 * period. Just clear the sample period so at least
201 * allocating the counter doesn't fail.
202 */
203 attr.sample_period = 0;
204 }
205 if (pebs) {
206 /*
207 * For most PEBS hardware events, the difference in the software
208 * precision levels of guest and host PEBS events will not affect
209 * the accuracy of the PEBS profiling result, because the "event IP"
210 * in the PEBS record is calibrated on the guest side.
211 */
212 attr.precise_ip = pmc_get_pebs_precise_level(pmc);
213 }
214
215 event = perf_event_create_kernel_counter(&attr, -1, current,
216 kvm_perf_overflow, pmc);
217 if (IS_ERR(event)) {
218 pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n",
219 PTR_ERR(event), pmc->idx);
220 return PTR_ERR(event);
221 }
222
223 pmc->perf_event = event;
224 pmc_to_pmu(pmc)->event_count++;
225 pmc->is_paused = false;
226 pmc->intr = intr || pebs;
227 return 0;
228}
229
230static bool pmc_pause_counter(struct kvm_pmc *pmc)
231{
232 u64 counter = pmc->counter;
233 u64 prev_counter;
234
235 /* update counter, reset event value to avoid redundant accumulation */
236 if (pmc->perf_event && !pmc->is_paused)
237 counter += perf_event_pause(pmc->perf_event, true);
238
239 /*
240 * Snapshot the previous counter *after* accumulating state from perf.
241 * If overflow already happened, hardware (via perf) is responsible for
242 * generating a PMI. KVM just needs to detect overflow on emulated
243 * counter events that haven't yet been processed.
244 */
245 prev_counter = counter & pmc_bitmask(pmc);
246
247 counter += pmc->emulated_counter;
248 pmc->counter = counter & pmc_bitmask(pmc);
249
250 pmc->emulated_counter = 0;
251 pmc->is_paused = true;
252
253 return pmc->counter < prev_counter;
254}
255
256static bool pmc_resume_counter(struct kvm_pmc *pmc)
257{
258 if (!pmc->perf_event)
259 return false;
260
261 /* recalibrate sample period and check if it's accepted by perf core */
262 if (is_sampling_event(pmc->perf_event) &&
263 perf_event_period(pmc->perf_event,
264 get_sample_period(pmc, pmc->counter)))
265 return false;
266
267 if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) !=
268 (!!pmc->perf_event->attr.precise_ip))
269 return false;
270
271 /* reuse perf_event to serve as pmc_reprogram_counter() does*/
272 perf_event_enable(pmc->perf_event);
273 pmc->is_paused = false;
274
275 return true;
276}
277
278static void pmc_release_perf_event(struct kvm_pmc *pmc)
279{
280 if (pmc->perf_event) {
281 perf_event_release_kernel(pmc->perf_event);
282 pmc->perf_event = NULL;
283 pmc->current_config = 0;
284 pmc_to_pmu(pmc)->event_count--;
285 }
286}
287
288static void pmc_stop_counter(struct kvm_pmc *pmc)
289{
290 if (pmc->perf_event) {
291 pmc->counter = pmc_read_counter(pmc);
292 pmc_release_perf_event(pmc);
293 }
294}
295
296static void pmc_update_sample_period(struct kvm_pmc *pmc)
297{
298 if (!pmc->perf_event || pmc->is_paused ||
299 !is_sampling_event(pmc->perf_event))
300 return;
301
302 perf_event_period(pmc->perf_event,
303 get_sample_period(pmc, pmc->counter));
304}
305
306void pmc_write_counter(struct kvm_pmc *pmc, u64 val)
307{
308 /*
309 * Drop any unconsumed accumulated counts, the WRMSR is a write, not a
310 * read-modify-write. Adjust the counter value so that its value is
311 * relative to the current count, as reading the current count from
312 * perf is faster than pausing and repgrogramming the event in order to
313 * reset it to '0'. Note, this very sneakily offsets the accumulated
314 * emulated count too, by using pmc_read_counter()!
315 */
316 pmc->emulated_counter = 0;
317 pmc->counter += val - pmc_read_counter(pmc);
318 pmc->counter &= pmc_bitmask(pmc);
319 pmc_update_sample_period(pmc);
320}
321EXPORT_SYMBOL_GPL(pmc_write_counter);
322
323static int filter_cmp(const void *pa, const void *pb, u64 mask)
324{
325 u64 a = *(u64 *)pa & mask;
326 u64 b = *(u64 *)pb & mask;
327
328 return (a > b) - (a < b);
329}
330
331
332static int filter_sort_cmp(const void *pa, const void *pb)
333{
334 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT |
335 KVM_PMU_MASKED_ENTRY_EXCLUDE));
336}
337
338/*
339 * For the event filter, searching is done on the 'includes' list and
340 * 'excludes' list separately rather than on the 'events' list (which
341 * has both). As a result the exclude bit can be ignored.
342 */
343static int filter_event_cmp(const void *pa, const void *pb)
344{
345 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT));
346}
347
348static int find_filter_index(u64 *events, u64 nevents, u64 key)
349{
350 u64 *fe = bsearch(&key, events, nevents, sizeof(events[0]),
351 filter_event_cmp);
352
353 if (!fe)
354 return -1;
355
356 return fe - events;
357}
358
359static bool is_filter_entry_match(u64 filter_event, u64 umask)
360{
361 u64 mask = filter_event >> (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8);
362 u64 match = filter_event & KVM_PMU_MASKED_ENTRY_UMASK_MATCH;
363
364 BUILD_BUG_ON((KVM_PMU_ENCODE_MASKED_ENTRY(0, 0xff, 0, false) >>
365 (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8)) !=
366 ARCH_PERFMON_EVENTSEL_UMASK);
367
368 return (umask & mask) == match;
369}
370
371static bool filter_contains_match(u64 *events, u64 nevents, u64 eventsel)
372{
373 u64 event_select = eventsel & kvm_pmu_ops.EVENTSEL_EVENT;
374 u64 umask = eventsel & ARCH_PERFMON_EVENTSEL_UMASK;
375 int i, index;
376
377 index = find_filter_index(events, nevents, event_select);
378 if (index < 0)
379 return false;
380
381 /*
382 * Entries are sorted by the event select. Walk the list in both
383 * directions to process all entries with the targeted event select.
384 */
385 for (i = index; i < nevents; i++) {
386 if (filter_event_cmp(&events[i], &event_select))
387 break;
388
389 if (is_filter_entry_match(events[i], umask))
390 return true;
391 }
392
393 for (i = index - 1; i >= 0; i--) {
394 if (filter_event_cmp(&events[i], &event_select))
395 break;
396
397 if (is_filter_entry_match(events[i], umask))
398 return true;
399 }
400
401 return false;
402}
403
404static bool is_gp_event_allowed(struct kvm_x86_pmu_event_filter *f,
405 u64 eventsel)
406{
407 if (filter_contains_match(f->includes, f->nr_includes, eventsel) &&
408 !filter_contains_match(f->excludes, f->nr_excludes, eventsel))
409 return f->action == KVM_PMU_EVENT_ALLOW;
410
411 return f->action == KVM_PMU_EVENT_DENY;
412}
413
414static bool is_fixed_event_allowed(struct kvm_x86_pmu_event_filter *filter,
415 int idx)
416{
417 int fixed_idx = idx - KVM_FIXED_PMC_BASE_IDX;
418
419 if (filter->action == KVM_PMU_EVENT_DENY &&
420 test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap))
421 return false;
422 if (filter->action == KVM_PMU_EVENT_ALLOW &&
423 !test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap))
424 return false;
425
426 return true;
427}
428
429static bool check_pmu_event_filter(struct kvm_pmc *pmc)
430{
431 struct kvm_x86_pmu_event_filter *filter;
432 struct kvm *kvm = pmc->vcpu->kvm;
433
434 filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu);
435 if (!filter)
436 return true;
437
438 if (pmc_is_gp(pmc))
439 return is_gp_event_allowed(filter, pmc->eventsel);
440
441 return is_fixed_event_allowed(filter, pmc->idx);
442}
443
444static bool pmc_event_is_allowed(struct kvm_pmc *pmc)
445{
446 return pmc_is_globally_enabled(pmc) && pmc_speculative_in_use(pmc) &&
447 check_pmu_event_filter(pmc);
448}
449
450static int reprogram_counter(struct kvm_pmc *pmc)
451{
452 struct kvm_pmu *pmu = pmc_to_pmu(pmc);
453 u64 eventsel = pmc->eventsel;
454 u64 new_config = eventsel;
455 bool emulate_overflow;
456 u8 fixed_ctr_ctrl;
457
458 emulate_overflow = pmc_pause_counter(pmc);
459
460 if (!pmc_event_is_allowed(pmc))
461 return 0;
462
463 if (emulate_overflow)
464 __kvm_perf_overflow(pmc, false);
465
466 if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL)
467 printk_once("kvm pmu: pin control bit is ignored\n");
468
469 if (pmc_is_fixed(pmc)) {
470 fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl,
471 pmc->idx - KVM_FIXED_PMC_BASE_IDX);
472 if (fixed_ctr_ctrl & INTEL_FIXED_0_KERNEL)
473 eventsel |= ARCH_PERFMON_EVENTSEL_OS;
474 if (fixed_ctr_ctrl & INTEL_FIXED_0_USER)
475 eventsel |= ARCH_PERFMON_EVENTSEL_USR;
476 if (fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI)
477 eventsel |= ARCH_PERFMON_EVENTSEL_INT;
478 new_config = (u64)fixed_ctr_ctrl;
479 }
480
481 if (pmc->current_config == new_config && pmc_resume_counter(pmc))
482 return 0;
483
484 pmc_release_perf_event(pmc);
485
486 pmc->current_config = new_config;
487
488 return pmc_reprogram_counter(pmc, PERF_TYPE_RAW,
489 (eventsel & pmu->raw_event_mask),
490 !(eventsel & ARCH_PERFMON_EVENTSEL_USR),
491 !(eventsel & ARCH_PERFMON_EVENTSEL_OS),
492 eventsel & ARCH_PERFMON_EVENTSEL_INT);
493}
494
495void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
496{
497 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX);
498 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
499 struct kvm_pmc *pmc;
500 int bit;
501
502 bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX);
503
504 /*
505 * The reprogramming bitmap can be written asynchronously by something
506 * other than the task that holds vcpu->mutex, take care to clear only
507 * the bits that will actually processed.
508 */
509 BUILD_BUG_ON(sizeof(bitmap) != sizeof(atomic64_t));
510 atomic64_andnot(*(s64 *)bitmap, &pmu->__reprogram_pmi);
511
512 kvm_for_each_pmc(pmu, pmc, bit, bitmap) {
513 /*
514 * If reprogramming fails, e.g. due to contention, re-set the
515 * regprogram bit set, i.e. opportunistically try again on the
516 * next PMU refresh. Don't make a new request as doing so can
517 * stall the guest if reprogramming repeatedly fails.
518 */
519 if (reprogram_counter(pmc))
520 set_bit(pmc->idx, pmu->reprogram_pmi);
521 }
522
523 /*
524 * Release unused perf_events if the corresponding guest MSRs weren't
525 * accessed during the last vCPU time slice (need_cleanup is set when
526 * the vCPU is scheduled back in).
527 */
528 if (unlikely(pmu->need_cleanup))
529 kvm_pmu_cleanup(vcpu);
530}
531
532int kvm_pmu_check_rdpmc_early(struct kvm_vcpu *vcpu, unsigned int idx)
533{
534 /*
535 * On Intel, VMX interception has priority over RDPMC exceptions that
536 * aren't already handled by the emulator, i.e. there are no additional
537 * check needed for Intel PMUs.
538 *
539 * On AMD, _all_ exceptions on RDPMC have priority over SVM intercepts,
540 * i.e. an invalid PMC results in a #GP, not #VMEXIT.
541 */
542 if (!kvm_pmu_ops.check_rdpmc_early)
543 return 0;
544
545 return kvm_pmu_call(check_rdpmc_early)(vcpu, idx);
546}
547
548bool is_vmware_backdoor_pmc(u32 pmc_idx)
549{
550 switch (pmc_idx) {
551 case VMWARE_BACKDOOR_PMC_HOST_TSC:
552 case VMWARE_BACKDOOR_PMC_REAL_TIME:
553 case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
554 return true;
555 }
556 return false;
557}
558
559static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
560{
561 u64 ctr_val;
562
563 switch (idx) {
564 case VMWARE_BACKDOOR_PMC_HOST_TSC:
565 ctr_val = rdtsc();
566 break;
567 case VMWARE_BACKDOOR_PMC_REAL_TIME:
568 ctr_val = ktime_get_boottime_ns();
569 break;
570 case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
571 ctr_val = ktime_get_boottime_ns() +
572 vcpu->kvm->arch.kvmclock_offset;
573 break;
574 default:
575 return 1;
576 }
577
578 *data = ctr_val;
579 return 0;
580}
581
582int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
583{
584 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
585 struct kvm_pmc *pmc;
586 u64 mask = ~0ull;
587
588 if (!pmu->version)
589 return 1;
590
591 if (is_vmware_backdoor_pmc(idx))
592 return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
593
594 pmc = kvm_pmu_call(rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
595 if (!pmc)
596 return 1;
597
598 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCE) &&
599 (kvm_x86_call(get_cpl)(vcpu) != 0) &&
600 kvm_is_cr0_bit_set(vcpu, X86_CR0_PE))
601 return 1;
602
603 *data = pmc_read_counter(pmc) & mask;
604 return 0;
605}
606
607void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
608{
609 if (lapic_in_kernel(vcpu)) {
610 kvm_pmu_call(deliver_pmi)(vcpu);
611 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
612 }
613}
614
615bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
616{
617 switch (msr) {
618 case MSR_CORE_PERF_GLOBAL_STATUS:
619 case MSR_CORE_PERF_GLOBAL_CTRL:
620 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
621 return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu));
622 default:
623 break;
624 }
625 return kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr) ||
626 kvm_pmu_call(is_valid_msr)(vcpu, msr);
627}
628
629static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
630{
631 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
632 struct kvm_pmc *pmc = kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr);
633
634 if (pmc)
635 __set_bit(pmc->idx, pmu->pmc_in_use);
636}
637
638int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
639{
640 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
641 u32 msr = msr_info->index;
642
643 switch (msr) {
644 case MSR_CORE_PERF_GLOBAL_STATUS:
645 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
646 msr_info->data = pmu->global_status;
647 break;
648 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
649 case MSR_CORE_PERF_GLOBAL_CTRL:
650 msr_info->data = pmu->global_ctrl;
651 break;
652 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
653 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
654 msr_info->data = 0;
655 break;
656 default:
657 return kvm_pmu_call(get_msr)(vcpu, msr_info);
658 }
659
660 return 0;
661}
662
663int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
664{
665 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
666 u32 msr = msr_info->index;
667 u64 data = msr_info->data;
668 u64 diff;
669
670 /*
671 * Note, AMD ignores writes to reserved bits and read-only PMU MSRs,
672 * whereas Intel generates #GP on attempts to write reserved/RO MSRs.
673 */
674 switch (msr) {
675 case MSR_CORE_PERF_GLOBAL_STATUS:
676 if (!msr_info->host_initiated)
677 return 1; /* RO MSR */
678 fallthrough;
679 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
680 /* Per PPR, Read-only MSR. Writes are ignored. */
681 if (!msr_info->host_initiated)
682 break;
683
684 if (data & pmu->global_status_rsvd)
685 return 1;
686
687 pmu->global_status = data;
688 break;
689 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
690 data &= ~pmu->global_ctrl_rsvd;
691 fallthrough;
692 case MSR_CORE_PERF_GLOBAL_CTRL:
693 if (!kvm_valid_perf_global_ctrl(pmu, data))
694 return 1;
695
696 if (pmu->global_ctrl != data) {
697 diff = pmu->global_ctrl ^ data;
698 pmu->global_ctrl = data;
699 reprogram_counters(pmu, diff);
700 }
701 break;
702 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
703 /*
704 * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in
705 * GLOBAL_STATUS, and so the set of reserved bits is the same.
706 */
707 if (data & pmu->global_status_rsvd)
708 return 1;
709 fallthrough;
710 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
711 if (!msr_info->host_initiated)
712 pmu->global_status &= ~data;
713 break;
714 default:
715 kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
716 return kvm_pmu_call(set_msr)(vcpu, msr_info);
717 }
718
719 return 0;
720}
721
722static void kvm_pmu_reset(struct kvm_vcpu *vcpu)
723{
724 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
725 struct kvm_pmc *pmc;
726 int i;
727
728 pmu->need_cleanup = false;
729
730 bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX);
731
732 kvm_for_each_pmc(pmu, pmc, i, pmu->all_valid_pmc_idx) {
733 pmc_stop_counter(pmc);
734 pmc->counter = 0;
735 pmc->emulated_counter = 0;
736
737 if (pmc_is_gp(pmc))
738 pmc->eventsel = 0;
739 }
740
741 pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0;
742
743 kvm_pmu_call(reset)(vcpu);
744}
745
746
747/*
748 * Refresh the PMU configuration for the vCPU, e.g. if userspace changes CPUID
749 * and/or PERF_CAPABILITIES.
750 */
751void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
752{
753 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
754
755 if (KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm))
756 return;
757
758 /*
759 * Stop/release all existing counters/events before realizing the new
760 * vPMU model.
761 */
762 kvm_pmu_reset(vcpu);
763
764 pmu->version = 0;
765 pmu->nr_arch_gp_counters = 0;
766 pmu->nr_arch_fixed_counters = 0;
767 pmu->counter_bitmask[KVM_PMC_GP] = 0;
768 pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
769 pmu->reserved_bits = 0xffffffff00200000ull;
770 pmu->raw_event_mask = X86_RAW_EVENT_MASK;
771 pmu->global_ctrl_rsvd = ~0ull;
772 pmu->global_status_rsvd = ~0ull;
773 pmu->fixed_ctr_ctrl_rsvd = ~0ull;
774 pmu->pebs_enable_rsvd = ~0ull;
775 pmu->pebs_data_cfg_rsvd = ~0ull;
776 bitmap_zero(pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX);
777
778 if (!vcpu->kvm->arch.enable_pmu)
779 return;
780
781 kvm_pmu_call(refresh)(vcpu);
782
783 /*
784 * At RESET, both Intel and AMD CPUs set all enable bits for general
785 * purpose counters in IA32_PERF_GLOBAL_CTRL (so that software that
786 * was written for v1 PMUs don't unknowingly leave GP counters disabled
787 * in the global controls). Emulate that behavior when refreshing the
788 * PMU so that userspace doesn't need to manually set PERF_GLOBAL_CTRL.
789 */
790 if (kvm_pmu_has_perf_global_ctrl(pmu) && pmu->nr_arch_gp_counters)
791 pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0);
792}
793
794void kvm_pmu_init(struct kvm_vcpu *vcpu)
795{
796 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
797
798 memset(pmu, 0, sizeof(*pmu));
799 kvm_pmu_call(init)(vcpu);
800 kvm_pmu_refresh(vcpu);
801}
802
803/* Release perf_events for vPMCs that have been unused for a full time slice. */
804void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
805{
806 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
807 struct kvm_pmc *pmc = NULL;
808 DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX);
809 int i;
810
811 pmu->need_cleanup = false;
812
813 bitmap_andnot(bitmask, pmu->all_valid_pmc_idx,
814 pmu->pmc_in_use, X86_PMC_IDX_MAX);
815
816 kvm_for_each_pmc(pmu, pmc, i, bitmask) {
817 if (pmc->perf_event && !pmc_speculative_in_use(pmc))
818 pmc_stop_counter(pmc);
819 }
820
821 kvm_pmu_call(cleanup)(vcpu);
822
823 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
824}
825
826void kvm_pmu_destroy(struct kvm_vcpu *vcpu)
827{
828 kvm_pmu_reset(vcpu);
829}
830
831static void kvm_pmu_incr_counter(struct kvm_pmc *pmc)
832{
833 pmc->emulated_counter++;
834 kvm_pmu_request_counter_reprogram(pmc);
835}
836
837static inline bool cpl_is_matched(struct kvm_pmc *pmc)
838{
839 bool select_os, select_user;
840 u64 config;
841
842 if (pmc_is_gp(pmc)) {
843 config = pmc->eventsel;
844 select_os = config & ARCH_PERFMON_EVENTSEL_OS;
845 select_user = config & ARCH_PERFMON_EVENTSEL_USR;
846 } else {
847 config = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl,
848 pmc->idx - KVM_FIXED_PMC_BASE_IDX);
849 select_os = config & INTEL_FIXED_0_KERNEL;
850 select_user = config & INTEL_FIXED_0_USER;
851 }
852
853 /*
854 * Skip the CPL lookup, which isn't free on Intel, if the result will
855 * be the same regardless of the CPL.
856 */
857 if (select_os == select_user)
858 return select_os;
859
860 return (kvm_x86_call(get_cpl)(pmc->vcpu) == 0) ? select_os :
861 select_user;
862}
863
864void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 eventsel)
865{
866 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX);
867 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
868 struct kvm_pmc *pmc;
869 int i;
870
871 BUILD_BUG_ON(sizeof(pmu->global_ctrl) * BITS_PER_BYTE != X86_PMC_IDX_MAX);
872
873 if (!kvm_pmu_has_perf_global_ctrl(pmu))
874 bitmap_copy(bitmap, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX);
875 else if (!bitmap_and(bitmap, pmu->all_valid_pmc_idx,
876 (unsigned long *)&pmu->global_ctrl, X86_PMC_IDX_MAX))
877 return;
878
879 kvm_for_each_pmc(pmu, pmc, i, bitmap) {
880 /*
881 * Ignore checks for edge detect (all events currently emulated
882 * but KVM are always rising edges), pin control (unsupported
883 * by modern CPUs), and counter mask and its invert flag (KVM
884 * doesn't emulate multiple events in a single clock cycle).
885 *
886 * Note, the uppermost nibble of AMD's mask overlaps Intel's
887 * IN_TX (bit 32) and IN_TXCP (bit 33), as well as two reserved
888 * bits (bits 35:34). Checking the "in HLE/RTM transaction"
889 * flags is correct as the vCPU can't be in a transaction if
890 * KVM is emulating an instruction. Checking the reserved bits
891 * might be wrong if they are defined in the future, but so
892 * could ignoring them, so do the simple thing for now.
893 */
894 if (((pmc->eventsel ^ eventsel) & AMD64_RAW_EVENT_MASK_NB) ||
895 !pmc_event_is_allowed(pmc) || !cpl_is_matched(pmc))
896 continue;
897
898 kvm_pmu_incr_counter(pmc);
899 }
900}
901EXPORT_SYMBOL_GPL(kvm_pmu_trigger_event);
902
903static bool is_masked_filter_valid(const struct kvm_x86_pmu_event_filter *filter)
904{
905 u64 mask = kvm_pmu_ops.EVENTSEL_EVENT |
906 KVM_PMU_MASKED_ENTRY_UMASK_MASK |
907 KVM_PMU_MASKED_ENTRY_UMASK_MATCH |
908 KVM_PMU_MASKED_ENTRY_EXCLUDE;
909 int i;
910
911 for (i = 0; i < filter->nevents; i++) {
912 if (filter->events[i] & ~mask)
913 return false;
914 }
915
916 return true;
917}
918
919static void convert_to_masked_filter(struct kvm_x86_pmu_event_filter *filter)
920{
921 int i, j;
922
923 for (i = 0, j = 0; i < filter->nevents; i++) {
924 /*
925 * Skip events that are impossible to match against a guest
926 * event. When filtering, only the event select + unit mask
927 * of the guest event is used. To maintain backwards
928 * compatibility, impossible filters can't be rejected :-(
929 */
930 if (filter->events[i] & ~(kvm_pmu_ops.EVENTSEL_EVENT |
931 ARCH_PERFMON_EVENTSEL_UMASK))
932 continue;
933 /*
934 * Convert userspace events to a common in-kernel event so
935 * only one code path is needed to support both events. For
936 * the in-kernel events use masked events because they are
937 * flexible enough to handle both cases. To convert to masked
938 * events all that's needed is to add an "all ones" umask_mask,
939 * (unmasked filter events don't support EXCLUDE).
940 */
941 filter->events[j++] = filter->events[i] |
942 (0xFFULL << KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT);
943 }
944
945 filter->nevents = j;
946}
947
948static int prepare_filter_lists(struct kvm_x86_pmu_event_filter *filter)
949{
950 int i;
951
952 if (!(filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS))
953 convert_to_masked_filter(filter);
954 else if (!is_masked_filter_valid(filter))
955 return -EINVAL;
956
957 /*
958 * Sort entries by event select and includes vs. excludes so that all
959 * entries for a given event select can be processed efficiently during
960 * filtering. The EXCLUDE flag uses a more significant bit than the
961 * event select, and so the sorted list is also effectively split into
962 * includes and excludes sub-lists.
963 */
964 sort(&filter->events, filter->nevents, sizeof(filter->events[0]),
965 filter_sort_cmp, NULL);
966
967 i = filter->nevents;
968 /* Find the first EXCLUDE event (only supported for masked events). */
969 if (filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS) {
970 for (i = 0; i < filter->nevents; i++) {
971 if (filter->events[i] & KVM_PMU_MASKED_ENTRY_EXCLUDE)
972 break;
973 }
974 }
975
976 filter->nr_includes = i;
977 filter->nr_excludes = filter->nevents - filter->nr_includes;
978 filter->includes = filter->events;
979 filter->excludes = filter->events + filter->nr_includes;
980
981 return 0;
982}
983
984int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp)
985{
986 struct kvm_pmu_event_filter __user *user_filter = argp;
987 struct kvm_x86_pmu_event_filter *filter;
988 struct kvm_pmu_event_filter tmp;
989 struct kvm_vcpu *vcpu;
990 unsigned long i;
991 size_t size;
992 int r;
993
994 if (copy_from_user(&tmp, user_filter, sizeof(tmp)))
995 return -EFAULT;
996
997 if (tmp.action != KVM_PMU_EVENT_ALLOW &&
998 tmp.action != KVM_PMU_EVENT_DENY)
999 return -EINVAL;
1000
1001 if (tmp.flags & ~KVM_PMU_EVENT_FLAGS_VALID_MASK)
1002 return -EINVAL;
1003
1004 if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS)
1005 return -E2BIG;
1006
1007 size = struct_size(filter, events, tmp.nevents);
1008 filter = kzalloc(size, GFP_KERNEL_ACCOUNT);
1009 if (!filter)
1010 return -ENOMEM;
1011
1012 filter->action = tmp.action;
1013 filter->nevents = tmp.nevents;
1014 filter->fixed_counter_bitmap = tmp.fixed_counter_bitmap;
1015 filter->flags = tmp.flags;
1016
1017 r = -EFAULT;
1018 if (copy_from_user(filter->events, user_filter->events,
1019 sizeof(filter->events[0]) * filter->nevents))
1020 goto cleanup;
1021
1022 r = prepare_filter_lists(filter);
1023 if (r)
1024 goto cleanup;
1025
1026 mutex_lock(&kvm->lock);
1027 filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter,
1028 mutex_is_locked(&kvm->lock));
1029 mutex_unlock(&kvm->lock);
1030 synchronize_srcu_expedited(&kvm->srcu);
1031
1032 BUILD_BUG_ON(sizeof(((struct kvm_pmu *)0)->reprogram_pmi) >
1033 sizeof(((struct kvm_pmu *)0)->__reprogram_pmi));
1034
1035 kvm_for_each_vcpu(i, vcpu, kvm)
1036 atomic64_set(&vcpu_to_pmu(vcpu)->__reprogram_pmi, -1ull);
1037
1038 kvm_make_all_cpus_request(kvm, KVM_REQ_PMU);
1039
1040 r = 0;
1041cleanup:
1042 kfree(filter);
1043 return r;
1044}
1/*
2 * Kernel-based Virtual Machine -- Performance Monitoring Unit support
3 *
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates.
5 *
6 * Authors:
7 * Avi Kivity <avi@redhat.com>
8 * Gleb Natapov <gleb@redhat.com>
9 *
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
12 *
13 */
14
15#include <linux/types.h>
16#include <linux/kvm_host.h>
17#include <linux/perf_event.h>
18#include "x86.h"
19#include "cpuid.h"
20#include "lapic.h"
21
22static struct kvm_arch_event_perf_mapping {
23 u8 eventsel;
24 u8 unit_mask;
25 unsigned event_type;
26 bool inexact;
27} arch_events[] = {
28 /* Index must match CPUID 0x0A.EBX bit vector */
29 [0] = { 0x3c, 0x00, PERF_COUNT_HW_CPU_CYCLES },
30 [1] = { 0xc0, 0x00, PERF_COUNT_HW_INSTRUCTIONS },
31 [2] = { 0x3c, 0x01, PERF_COUNT_HW_BUS_CYCLES },
32 [3] = { 0x2e, 0x4f, PERF_COUNT_HW_CACHE_REFERENCES },
33 [4] = { 0x2e, 0x41, PERF_COUNT_HW_CACHE_MISSES },
34 [5] = { 0xc4, 0x00, PERF_COUNT_HW_BRANCH_INSTRUCTIONS },
35 [6] = { 0xc5, 0x00, PERF_COUNT_HW_BRANCH_MISSES },
36 [7] = { 0x00, 0x30, PERF_COUNT_HW_REF_CPU_CYCLES },
37};
38
39/* mapping between fixed pmc index and arch_events array */
40int fixed_pmc_events[] = {1, 0, 7};
41
42static bool pmc_is_gp(struct kvm_pmc *pmc)
43{
44 return pmc->type == KVM_PMC_GP;
45}
46
47static inline u64 pmc_bitmask(struct kvm_pmc *pmc)
48{
49 struct kvm_pmu *pmu = &pmc->vcpu->arch.pmu;
50
51 return pmu->counter_bitmask[pmc->type];
52}
53
54static inline bool pmc_enabled(struct kvm_pmc *pmc)
55{
56 struct kvm_pmu *pmu = &pmc->vcpu->arch.pmu;
57 return test_bit(pmc->idx, (unsigned long *)&pmu->global_ctrl);
58}
59
60static inline struct kvm_pmc *get_gp_pmc(struct kvm_pmu *pmu, u32 msr,
61 u32 base)
62{
63 if (msr >= base && msr < base + pmu->nr_arch_gp_counters)
64 return &pmu->gp_counters[msr - base];
65 return NULL;
66}
67
68static inline struct kvm_pmc *get_fixed_pmc(struct kvm_pmu *pmu, u32 msr)
69{
70 int base = MSR_CORE_PERF_FIXED_CTR0;
71 if (msr >= base && msr < base + pmu->nr_arch_fixed_counters)
72 return &pmu->fixed_counters[msr - base];
73 return NULL;
74}
75
76static inline struct kvm_pmc *get_fixed_pmc_idx(struct kvm_pmu *pmu, int idx)
77{
78 return get_fixed_pmc(pmu, MSR_CORE_PERF_FIXED_CTR0 + idx);
79}
80
81static struct kvm_pmc *global_idx_to_pmc(struct kvm_pmu *pmu, int idx)
82{
83 if (idx < INTEL_PMC_IDX_FIXED)
84 return get_gp_pmc(pmu, MSR_P6_EVNTSEL0 + idx, MSR_P6_EVNTSEL0);
85 else
86 return get_fixed_pmc_idx(pmu, idx - INTEL_PMC_IDX_FIXED);
87}
88
89void kvm_deliver_pmi(struct kvm_vcpu *vcpu)
90{
91 if (vcpu->arch.apic)
92 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
93}
94
95static void trigger_pmi(struct irq_work *irq_work)
96{
97 struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu,
98 irq_work);
99 struct kvm_vcpu *vcpu = container_of(pmu, struct kvm_vcpu,
100 arch.pmu);
101
102 kvm_deliver_pmi(vcpu);
103}
104
105static void kvm_perf_overflow(struct perf_event *perf_event,
106 struct perf_sample_data *data,
107 struct pt_regs *regs)
108{
109 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
110 struct kvm_pmu *pmu = &pmc->vcpu->arch.pmu;
111 __set_bit(pmc->idx, (unsigned long *)&pmu->global_status);
112}
113
114static void kvm_perf_overflow_intr(struct perf_event *perf_event,
115 struct perf_sample_data *data, struct pt_regs *regs)
116{
117 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
118 struct kvm_pmu *pmu = &pmc->vcpu->arch.pmu;
119 if (!test_and_set_bit(pmc->idx, (unsigned long *)&pmu->reprogram_pmi)) {
120 kvm_perf_overflow(perf_event, data, regs);
121 kvm_make_request(KVM_REQ_PMU, pmc->vcpu);
122 /*
123 * Inject PMI. If vcpu was in a guest mode during NMI PMI
124 * can be ejected on a guest mode re-entry. Otherwise we can't
125 * be sure that vcpu wasn't executing hlt instruction at the
126 * time of vmexit and is not going to re-enter guest mode until,
127 * woken up. So we should wake it, but this is impossible from
128 * NMI context. Do it from irq work instead.
129 */
130 if (!kvm_is_in_guest())
131 irq_work_queue(&pmc->vcpu->arch.pmu.irq_work);
132 else
133 kvm_make_request(KVM_REQ_PMI, pmc->vcpu);
134 }
135}
136
137static u64 read_pmc(struct kvm_pmc *pmc)
138{
139 u64 counter, enabled, running;
140
141 counter = pmc->counter;
142
143 if (pmc->perf_event)
144 counter += perf_event_read_value(pmc->perf_event,
145 &enabled, &running);
146
147 /* FIXME: Scaling needed? */
148
149 return counter & pmc_bitmask(pmc);
150}
151
152static void stop_counter(struct kvm_pmc *pmc)
153{
154 if (pmc->perf_event) {
155 pmc->counter = read_pmc(pmc);
156 perf_event_release_kernel(pmc->perf_event);
157 pmc->perf_event = NULL;
158 }
159}
160
161static void reprogram_counter(struct kvm_pmc *pmc, u32 type,
162 unsigned config, bool exclude_user, bool exclude_kernel,
163 bool intr, bool in_tx, bool in_tx_cp)
164{
165 struct perf_event *event;
166 struct perf_event_attr attr = {
167 .type = type,
168 .size = sizeof(attr),
169 .pinned = true,
170 .exclude_idle = true,
171 .exclude_host = 1,
172 .exclude_user = exclude_user,
173 .exclude_kernel = exclude_kernel,
174 .config = config,
175 };
176 if (in_tx)
177 attr.config |= HSW_IN_TX;
178 if (in_tx_cp)
179 attr.config |= HSW_IN_TX_CHECKPOINTED;
180
181 attr.sample_period = (-pmc->counter) & pmc_bitmask(pmc);
182
183 event = perf_event_create_kernel_counter(&attr, -1, current,
184 intr ? kvm_perf_overflow_intr :
185 kvm_perf_overflow, pmc);
186 if (IS_ERR(event)) {
187 printk_once("kvm: pmu event creation failed %ld\n",
188 PTR_ERR(event));
189 return;
190 }
191
192 pmc->perf_event = event;
193 clear_bit(pmc->idx, (unsigned long*)&pmc->vcpu->arch.pmu.reprogram_pmi);
194}
195
196static unsigned find_arch_event(struct kvm_pmu *pmu, u8 event_select,
197 u8 unit_mask)
198{
199 int i;
200
201 for (i = 0; i < ARRAY_SIZE(arch_events); i++)
202 if (arch_events[i].eventsel == event_select
203 && arch_events[i].unit_mask == unit_mask
204 && (pmu->available_event_types & (1 << i)))
205 break;
206
207 if (i == ARRAY_SIZE(arch_events))
208 return PERF_COUNT_HW_MAX;
209
210 return arch_events[i].event_type;
211}
212
213static void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
214{
215 unsigned config, type = PERF_TYPE_RAW;
216 u8 event_select, unit_mask;
217
218 if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL)
219 printk_once("kvm pmu: pin control bit is ignored\n");
220
221 pmc->eventsel = eventsel;
222
223 stop_counter(pmc);
224
225 if (!(eventsel & ARCH_PERFMON_EVENTSEL_ENABLE) || !pmc_enabled(pmc))
226 return;
227
228 event_select = eventsel & ARCH_PERFMON_EVENTSEL_EVENT;
229 unit_mask = (eventsel & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
230
231 if (!(eventsel & (ARCH_PERFMON_EVENTSEL_EDGE |
232 ARCH_PERFMON_EVENTSEL_INV |
233 ARCH_PERFMON_EVENTSEL_CMASK |
234 HSW_IN_TX |
235 HSW_IN_TX_CHECKPOINTED))) {
236 config = find_arch_event(&pmc->vcpu->arch.pmu, event_select,
237 unit_mask);
238 if (config != PERF_COUNT_HW_MAX)
239 type = PERF_TYPE_HARDWARE;
240 }
241
242 if (type == PERF_TYPE_RAW)
243 config = eventsel & X86_RAW_EVENT_MASK;
244
245 reprogram_counter(pmc, type, config,
246 !(eventsel & ARCH_PERFMON_EVENTSEL_USR),
247 !(eventsel & ARCH_PERFMON_EVENTSEL_OS),
248 eventsel & ARCH_PERFMON_EVENTSEL_INT,
249 (eventsel & HSW_IN_TX),
250 (eventsel & HSW_IN_TX_CHECKPOINTED));
251}
252
253static void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 en_pmi, int idx)
254{
255 unsigned en = en_pmi & 0x3;
256 bool pmi = en_pmi & 0x8;
257
258 stop_counter(pmc);
259
260 if (!en || !pmc_enabled(pmc))
261 return;
262
263 reprogram_counter(pmc, PERF_TYPE_HARDWARE,
264 arch_events[fixed_pmc_events[idx]].event_type,
265 !(en & 0x2), /* exclude user */
266 !(en & 0x1), /* exclude kernel */
267 pmi, false, false);
268}
269
270static inline u8 fixed_en_pmi(u64 ctrl, int idx)
271{
272 return (ctrl >> (idx * 4)) & 0xf;
273}
274
275static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data)
276{
277 int i;
278
279 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) {
280 u8 en_pmi = fixed_en_pmi(data, i);
281 struct kvm_pmc *pmc = get_fixed_pmc_idx(pmu, i);
282
283 if (fixed_en_pmi(pmu->fixed_ctr_ctrl, i) == en_pmi)
284 continue;
285
286 reprogram_fixed_counter(pmc, en_pmi, i);
287 }
288
289 pmu->fixed_ctr_ctrl = data;
290}
291
292static void reprogram_idx(struct kvm_pmu *pmu, int idx)
293{
294 struct kvm_pmc *pmc = global_idx_to_pmc(pmu, idx);
295
296 if (!pmc)
297 return;
298
299 if (pmc_is_gp(pmc))
300 reprogram_gp_counter(pmc, pmc->eventsel);
301 else {
302 int fidx = idx - INTEL_PMC_IDX_FIXED;
303 reprogram_fixed_counter(pmc,
304 fixed_en_pmi(pmu->fixed_ctr_ctrl, fidx), fidx);
305 }
306}
307
308static void global_ctrl_changed(struct kvm_pmu *pmu, u64 data)
309{
310 int bit;
311 u64 diff = pmu->global_ctrl ^ data;
312
313 pmu->global_ctrl = data;
314
315 for_each_set_bit(bit, (unsigned long *)&diff, X86_PMC_IDX_MAX)
316 reprogram_idx(pmu, bit);
317}
318
319bool kvm_pmu_msr(struct kvm_vcpu *vcpu, u32 msr)
320{
321 struct kvm_pmu *pmu = &vcpu->arch.pmu;
322 int ret;
323
324 switch (msr) {
325 case MSR_CORE_PERF_FIXED_CTR_CTRL:
326 case MSR_CORE_PERF_GLOBAL_STATUS:
327 case MSR_CORE_PERF_GLOBAL_CTRL:
328 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
329 ret = pmu->version > 1;
330 break;
331 default:
332 ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)
333 || get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0)
334 || get_fixed_pmc(pmu, msr);
335 break;
336 }
337 return ret;
338}
339
340int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
341{
342 struct kvm_pmu *pmu = &vcpu->arch.pmu;
343 struct kvm_pmc *pmc;
344
345 switch (index) {
346 case MSR_CORE_PERF_FIXED_CTR_CTRL:
347 *data = pmu->fixed_ctr_ctrl;
348 return 0;
349 case MSR_CORE_PERF_GLOBAL_STATUS:
350 *data = pmu->global_status;
351 return 0;
352 case MSR_CORE_PERF_GLOBAL_CTRL:
353 *data = pmu->global_ctrl;
354 return 0;
355 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
356 *data = pmu->global_ovf_ctrl;
357 return 0;
358 default:
359 if ((pmc = get_gp_pmc(pmu, index, MSR_IA32_PERFCTR0)) ||
360 (pmc = get_fixed_pmc(pmu, index))) {
361 *data = read_pmc(pmc);
362 return 0;
363 } else if ((pmc = get_gp_pmc(pmu, index, MSR_P6_EVNTSEL0))) {
364 *data = pmc->eventsel;
365 return 0;
366 }
367 }
368 return 1;
369}
370
371int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
372{
373 struct kvm_pmu *pmu = &vcpu->arch.pmu;
374 struct kvm_pmc *pmc;
375 u32 index = msr_info->index;
376 u64 data = msr_info->data;
377
378 switch (index) {
379 case MSR_CORE_PERF_FIXED_CTR_CTRL:
380 if (pmu->fixed_ctr_ctrl == data)
381 return 0;
382 if (!(data & 0xfffffffffffff444ull)) {
383 reprogram_fixed_counters(pmu, data);
384 return 0;
385 }
386 break;
387 case MSR_CORE_PERF_GLOBAL_STATUS:
388 if (msr_info->host_initiated) {
389 pmu->global_status = data;
390 return 0;
391 }
392 break; /* RO MSR */
393 case MSR_CORE_PERF_GLOBAL_CTRL:
394 if (pmu->global_ctrl == data)
395 return 0;
396 if (!(data & pmu->global_ctrl_mask)) {
397 global_ctrl_changed(pmu, data);
398 return 0;
399 }
400 break;
401 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
402 if (!(data & (pmu->global_ctrl_mask & ~(3ull<<62)))) {
403 if (!msr_info->host_initiated)
404 pmu->global_status &= ~data;
405 pmu->global_ovf_ctrl = data;
406 return 0;
407 }
408 break;
409 default:
410 if ((pmc = get_gp_pmc(pmu, index, MSR_IA32_PERFCTR0)) ||
411 (pmc = get_fixed_pmc(pmu, index))) {
412 if (!msr_info->host_initiated)
413 data = (s64)(s32)data;
414 pmc->counter += data - read_pmc(pmc);
415 return 0;
416 } else if ((pmc = get_gp_pmc(pmu, index, MSR_P6_EVNTSEL0))) {
417 if (data == pmc->eventsel)
418 return 0;
419 if (!(data & pmu->reserved_bits)) {
420 reprogram_gp_counter(pmc, data);
421 return 0;
422 }
423 }
424 }
425 return 1;
426}
427
428int kvm_pmu_read_pmc(struct kvm_vcpu *vcpu, unsigned pmc, u64 *data)
429{
430 struct kvm_pmu *pmu = &vcpu->arch.pmu;
431 bool fast_mode = pmc & (1u << 31);
432 bool fixed = pmc & (1u << 30);
433 struct kvm_pmc *counters;
434 u64 ctr;
435
436 pmc &= ~(3u << 30);
437 if (!fixed && pmc >= pmu->nr_arch_gp_counters)
438 return 1;
439 if (fixed && pmc >= pmu->nr_arch_fixed_counters)
440 return 1;
441 counters = fixed ? pmu->fixed_counters : pmu->gp_counters;
442 ctr = read_pmc(&counters[pmc]);
443 if (fast_mode)
444 ctr = (u32)ctr;
445 *data = ctr;
446
447 return 0;
448}
449
450void kvm_pmu_cpuid_update(struct kvm_vcpu *vcpu)
451{
452 struct kvm_pmu *pmu = &vcpu->arch.pmu;
453 struct kvm_cpuid_entry2 *entry;
454 unsigned bitmap_len;
455
456 pmu->nr_arch_gp_counters = 0;
457 pmu->nr_arch_fixed_counters = 0;
458 pmu->counter_bitmask[KVM_PMC_GP] = 0;
459 pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
460 pmu->version = 0;
461 pmu->reserved_bits = 0xffffffff00200000ull;
462
463 entry = kvm_find_cpuid_entry(vcpu, 0xa, 0);
464 if (!entry)
465 return;
466
467 pmu->version = entry->eax & 0xff;
468 if (!pmu->version)
469 return;
470
471 pmu->nr_arch_gp_counters = min((int)(entry->eax >> 8) & 0xff,
472 INTEL_PMC_MAX_GENERIC);
473 pmu->counter_bitmask[KVM_PMC_GP] =
474 ((u64)1 << ((entry->eax >> 16) & 0xff)) - 1;
475 bitmap_len = (entry->eax >> 24) & 0xff;
476 pmu->available_event_types = ~entry->ebx & ((1ull << bitmap_len) - 1);
477
478 if (pmu->version == 1) {
479 pmu->nr_arch_fixed_counters = 0;
480 } else {
481 pmu->nr_arch_fixed_counters = min((int)(entry->edx & 0x1f),
482 INTEL_PMC_MAX_FIXED);
483 pmu->counter_bitmask[KVM_PMC_FIXED] =
484 ((u64)1 << ((entry->edx >> 5) & 0xff)) - 1;
485 }
486
487 pmu->global_ctrl = ((1 << pmu->nr_arch_gp_counters) - 1) |
488 (((1ull << pmu->nr_arch_fixed_counters) - 1) << INTEL_PMC_IDX_FIXED);
489 pmu->global_ctrl_mask = ~pmu->global_ctrl;
490
491 entry = kvm_find_cpuid_entry(vcpu, 7, 0);
492 if (entry &&
493 (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) &&
494 (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM)))
495 pmu->reserved_bits ^= HSW_IN_TX|HSW_IN_TX_CHECKPOINTED;
496}
497
498void kvm_pmu_init(struct kvm_vcpu *vcpu)
499{
500 int i;
501 struct kvm_pmu *pmu = &vcpu->arch.pmu;
502
503 memset(pmu, 0, sizeof(*pmu));
504 for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) {
505 pmu->gp_counters[i].type = KVM_PMC_GP;
506 pmu->gp_counters[i].vcpu = vcpu;
507 pmu->gp_counters[i].idx = i;
508 }
509 for (i = 0; i < INTEL_PMC_MAX_FIXED; i++) {
510 pmu->fixed_counters[i].type = KVM_PMC_FIXED;
511 pmu->fixed_counters[i].vcpu = vcpu;
512 pmu->fixed_counters[i].idx = i + INTEL_PMC_IDX_FIXED;
513 }
514 init_irq_work(&pmu->irq_work, trigger_pmi);
515 kvm_pmu_cpuid_update(vcpu);
516}
517
518void kvm_pmu_reset(struct kvm_vcpu *vcpu)
519{
520 struct kvm_pmu *pmu = &vcpu->arch.pmu;
521 int i;
522
523 irq_work_sync(&pmu->irq_work);
524 for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) {
525 struct kvm_pmc *pmc = &pmu->gp_counters[i];
526 stop_counter(pmc);
527 pmc->counter = pmc->eventsel = 0;
528 }
529
530 for (i = 0; i < INTEL_PMC_MAX_FIXED; i++)
531 stop_counter(&pmu->fixed_counters[i]);
532
533 pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status =
534 pmu->global_ovf_ctrl = 0;
535}
536
537void kvm_pmu_destroy(struct kvm_vcpu *vcpu)
538{
539 kvm_pmu_reset(vcpu);
540}
541
542void kvm_handle_pmu_event(struct kvm_vcpu *vcpu)
543{
544 struct kvm_pmu *pmu = &vcpu->arch.pmu;
545 u64 bitmask;
546 int bit;
547
548 bitmask = pmu->reprogram_pmi;
549
550 for_each_set_bit(bit, (unsigned long *)&bitmask, X86_PMC_IDX_MAX) {
551 struct kvm_pmc *pmc = global_idx_to_pmc(pmu, bit);
552
553 if (unlikely(!pmc || !pmc->perf_event)) {
554 clear_bit(bit, (unsigned long *)&pmu->reprogram_pmi);
555 continue;
556 }
557
558 reprogram_idx(pmu, bit);
559 }
560}