Loading...
1/*
2 * Performance events x86 architecture code
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10 * Copyright (C) 2009 Google, Inc., Stephane Eranian
11 *
12 * For licencing details see kernel-base/COPYING
13 */
14
15#include <linux/perf_event.h>
16#include <linux/capability.h>
17#include <linux/notifier.h>
18#include <linux/hardirq.h>
19#include <linux/kprobes.h>
20#include <linux/export.h>
21#include <linux/init.h>
22#include <linux/kdebug.h>
23#include <linux/sched/mm.h>
24#include <linux/sched/clock.h>
25#include <linux/uaccess.h>
26#include <linux/slab.h>
27#include <linux/cpu.h>
28#include <linux/bitops.h>
29#include <linux/device.h>
30#include <linux/nospec.h>
31
32#include <asm/apic.h>
33#include <asm/stacktrace.h>
34#include <asm/nmi.h>
35#include <asm/smp.h>
36#include <asm/alternative.h>
37#include <asm/mmu_context.h>
38#include <asm/tlbflush.h>
39#include <asm/timer.h>
40#include <asm/desc.h>
41#include <asm/ldt.h>
42#include <asm/unwind.h>
43
44#include "perf_event.h"
45
46struct x86_pmu x86_pmu __read_mostly;
47
48DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
49 .enabled = 1,
50};
51
52DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
53
54u64 __read_mostly hw_cache_event_ids
55 [PERF_COUNT_HW_CACHE_MAX]
56 [PERF_COUNT_HW_CACHE_OP_MAX]
57 [PERF_COUNT_HW_CACHE_RESULT_MAX];
58u64 __read_mostly hw_cache_extra_regs
59 [PERF_COUNT_HW_CACHE_MAX]
60 [PERF_COUNT_HW_CACHE_OP_MAX]
61 [PERF_COUNT_HW_CACHE_RESULT_MAX];
62
63/*
64 * Propagate event elapsed time into the generic event.
65 * Can only be executed on the CPU where the event is active.
66 * Returns the delta events processed.
67 */
68u64 x86_perf_event_update(struct perf_event *event)
69{
70 struct hw_perf_event *hwc = &event->hw;
71 int shift = 64 - x86_pmu.cntval_bits;
72 u64 prev_raw_count, new_raw_count;
73 int idx = hwc->idx;
74 u64 delta;
75
76 if (idx == INTEL_PMC_IDX_FIXED_BTS)
77 return 0;
78
79 /*
80 * Careful: an NMI might modify the previous event value.
81 *
82 * Our tactic to handle this is to first atomically read and
83 * exchange a new raw count - then add that new-prev delta
84 * count to the generic event atomically:
85 */
86again:
87 prev_raw_count = local64_read(&hwc->prev_count);
88 rdpmcl(hwc->event_base_rdpmc, new_raw_count);
89
90 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
91 new_raw_count) != prev_raw_count)
92 goto again;
93
94 /*
95 * Now we have the new raw value and have updated the prev
96 * timestamp already. We can now calculate the elapsed delta
97 * (event-)time and add that to the generic event.
98 *
99 * Careful, not all hw sign-extends above the physical width
100 * of the count.
101 */
102 delta = (new_raw_count << shift) - (prev_raw_count << shift);
103 delta >>= shift;
104
105 local64_add(delta, &event->count);
106 local64_sub(delta, &hwc->period_left);
107
108 return new_raw_count;
109}
110
111/*
112 * Find and validate any extra registers to set up.
113 */
114static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
115{
116 struct hw_perf_event_extra *reg;
117 struct extra_reg *er;
118
119 reg = &event->hw.extra_reg;
120
121 if (!x86_pmu.extra_regs)
122 return 0;
123
124 for (er = x86_pmu.extra_regs; er->msr; er++) {
125 if (er->event != (config & er->config_mask))
126 continue;
127 if (event->attr.config1 & ~er->valid_mask)
128 return -EINVAL;
129 /* Check if the extra msrs can be safely accessed*/
130 if (!er->extra_msr_access)
131 return -ENXIO;
132
133 reg->idx = er->idx;
134 reg->config = event->attr.config1;
135 reg->reg = er->msr;
136 break;
137 }
138 return 0;
139}
140
141static atomic_t active_events;
142static atomic_t pmc_refcount;
143static DEFINE_MUTEX(pmc_reserve_mutex);
144
145#ifdef CONFIG_X86_LOCAL_APIC
146
147static bool reserve_pmc_hardware(void)
148{
149 int i;
150
151 for (i = 0; i < x86_pmu.num_counters; i++) {
152 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
153 goto perfctr_fail;
154 }
155
156 for (i = 0; i < x86_pmu.num_counters; i++) {
157 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
158 goto eventsel_fail;
159 }
160
161 return true;
162
163eventsel_fail:
164 for (i--; i >= 0; i--)
165 release_evntsel_nmi(x86_pmu_config_addr(i));
166
167 i = x86_pmu.num_counters;
168
169perfctr_fail:
170 for (i--; i >= 0; i--)
171 release_perfctr_nmi(x86_pmu_event_addr(i));
172
173 return false;
174}
175
176static void release_pmc_hardware(void)
177{
178 int i;
179
180 for (i = 0; i < x86_pmu.num_counters; i++) {
181 release_perfctr_nmi(x86_pmu_event_addr(i));
182 release_evntsel_nmi(x86_pmu_config_addr(i));
183 }
184}
185
186#else
187
188static bool reserve_pmc_hardware(void) { return true; }
189static void release_pmc_hardware(void) {}
190
191#endif
192
193static bool check_hw_exists(void)
194{
195 u64 val, val_fail = -1, val_new= ~0;
196 int i, reg, reg_fail = -1, ret = 0;
197 int bios_fail = 0;
198 int reg_safe = -1;
199
200 /*
201 * Check to see if the BIOS enabled any of the counters, if so
202 * complain and bail.
203 */
204 for (i = 0; i < x86_pmu.num_counters; i++) {
205 reg = x86_pmu_config_addr(i);
206 ret = rdmsrl_safe(reg, &val);
207 if (ret)
208 goto msr_fail;
209 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
210 bios_fail = 1;
211 val_fail = val;
212 reg_fail = reg;
213 } else {
214 reg_safe = i;
215 }
216 }
217
218 if (x86_pmu.num_counters_fixed) {
219 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
220 ret = rdmsrl_safe(reg, &val);
221 if (ret)
222 goto msr_fail;
223 for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
224 if (val & (0x03 << i*4)) {
225 bios_fail = 1;
226 val_fail = val;
227 reg_fail = reg;
228 }
229 }
230 }
231
232 /*
233 * If all the counters are enabled, the below test will always
234 * fail. The tools will also become useless in this scenario.
235 * Just fail and disable the hardware counters.
236 */
237
238 if (reg_safe == -1) {
239 reg = reg_safe;
240 goto msr_fail;
241 }
242
243 /*
244 * Read the current value, change it and read it back to see if it
245 * matches, this is needed to detect certain hardware emulators
246 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
247 */
248 reg = x86_pmu_event_addr(reg_safe);
249 if (rdmsrl_safe(reg, &val))
250 goto msr_fail;
251 val ^= 0xffffUL;
252 ret = wrmsrl_safe(reg, val);
253 ret |= rdmsrl_safe(reg, &val_new);
254 if (ret || val != val_new)
255 goto msr_fail;
256
257 /*
258 * We still allow the PMU driver to operate:
259 */
260 if (bios_fail) {
261 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
262 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
263 reg_fail, val_fail);
264 }
265
266 return true;
267
268msr_fail:
269 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
270 pr_cont("PMU not available due to virtualization, using software events only.\n");
271 } else {
272 pr_cont("Broken PMU hardware detected, using software events only.\n");
273 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n",
274 reg, val_new);
275 }
276
277 return false;
278}
279
280static void hw_perf_event_destroy(struct perf_event *event)
281{
282 x86_release_hardware();
283 atomic_dec(&active_events);
284}
285
286void hw_perf_lbr_event_destroy(struct perf_event *event)
287{
288 hw_perf_event_destroy(event);
289
290 /* undo the lbr/bts event accounting */
291 x86_del_exclusive(x86_lbr_exclusive_lbr);
292}
293
294static inline int x86_pmu_initialized(void)
295{
296 return x86_pmu.handle_irq != NULL;
297}
298
299static inline int
300set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
301{
302 struct perf_event_attr *attr = &event->attr;
303 unsigned int cache_type, cache_op, cache_result;
304 u64 config, val;
305
306 config = attr->config;
307
308 cache_type = (config >> 0) & 0xff;
309 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
310 return -EINVAL;
311 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX);
312
313 cache_op = (config >> 8) & 0xff;
314 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
315 return -EINVAL;
316 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX);
317
318 cache_result = (config >> 16) & 0xff;
319 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
320 return -EINVAL;
321 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX);
322
323 val = hw_cache_event_ids[cache_type][cache_op][cache_result];
324
325 if (val == 0)
326 return -ENOENT;
327
328 if (val == -1)
329 return -EINVAL;
330
331 hwc->config |= val;
332 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
333 return x86_pmu_extra_regs(val, event);
334}
335
336int x86_reserve_hardware(void)
337{
338 int err = 0;
339
340 if (!atomic_inc_not_zero(&pmc_refcount)) {
341 mutex_lock(&pmc_reserve_mutex);
342 if (atomic_read(&pmc_refcount) == 0) {
343 if (!reserve_pmc_hardware())
344 err = -EBUSY;
345 else
346 reserve_ds_buffers();
347 }
348 if (!err)
349 atomic_inc(&pmc_refcount);
350 mutex_unlock(&pmc_reserve_mutex);
351 }
352
353 return err;
354}
355
356void x86_release_hardware(void)
357{
358 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
359 release_pmc_hardware();
360 release_ds_buffers();
361 mutex_unlock(&pmc_reserve_mutex);
362 }
363}
364
365/*
366 * Check if we can create event of a certain type (that no conflicting events
367 * are present).
368 */
369int x86_add_exclusive(unsigned int what)
370{
371 int i;
372
373 /*
374 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS.
375 * LBR and BTS are still mutually exclusive.
376 */
377 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
378 return 0;
379
380 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
381 mutex_lock(&pmc_reserve_mutex);
382 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
383 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
384 goto fail_unlock;
385 }
386 atomic_inc(&x86_pmu.lbr_exclusive[what]);
387 mutex_unlock(&pmc_reserve_mutex);
388 }
389
390 atomic_inc(&active_events);
391 return 0;
392
393fail_unlock:
394 mutex_unlock(&pmc_reserve_mutex);
395 return -EBUSY;
396}
397
398void x86_del_exclusive(unsigned int what)
399{
400 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
401 return;
402
403 atomic_dec(&x86_pmu.lbr_exclusive[what]);
404 atomic_dec(&active_events);
405}
406
407int x86_setup_perfctr(struct perf_event *event)
408{
409 struct perf_event_attr *attr = &event->attr;
410 struct hw_perf_event *hwc = &event->hw;
411 u64 config;
412
413 if (!is_sampling_event(event)) {
414 hwc->sample_period = x86_pmu.max_period;
415 hwc->last_period = hwc->sample_period;
416 local64_set(&hwc->period_left, hwc->sample_period);
417 }
418
419 if (attr->type == PERF_TYPE_RAW)
420 return x86_pmu_extra_regs(event->attr.config, event);
421
422 if (attr->type == PERF_TYPE_HW_CACHE)
423 return set_ext_hw_attr(hwc, event);
424
425 if (attr->config >= x86_pmu.max_events)
426 return -EINVAL;
427
428 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events);
429
430 /*
431 * The generic map:
432 */
433 config = x86_pmu.event_map(attr->config);
434
435 if (config == 0)
436 return -ENOENT;
437
438 if (config == -1LL)
439 return -EINVAL;
440
441 hwc->config |= config;
442
443 return 0;
444}
445
446/*
447 * check that branch_sample_type is compatible with
448 * settings needed for precise_ip > 1 which implies
449 * using the LBR to capture ALL taken branches at the
450 * priv levels of the measurement
451 */
452static inline int precise_br_compat(struct perf_event *event)
453{
454 u64 m = event->attr.branch_sample_type;
455 u64 b = 0;
456
457 /* must capture all branches */
458 if (!(m & PERF_SAMPLE_BRANCH_ANY))
459 return 0;
460
461 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
462
463 if (!event->attr.exclude_user)
464 b |= PERF_SAMPLE_BRANCH_USER;
465
466 if (!event->attr.exclude_kernel)
467 b |= PERF_SAMPLE_BRANCH_KERNEL;
468
469 /*
470 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
471 */
472
473 return m == b;
474}
475
476int x86_pmu_max_precise(void)
477{
478 int precise = 0;
479
480 /* Support for constant skid */
481 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
482 precise++;
483
484 /* Support for IP fixup */
485 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
486 precise++;
487
488 if (x86_pmu.pebs_prec_dist)
489 precise++;
490 }
491 return precise;
492}
493
494int x86_pmu_hw_config(struct perf_event *event)
495{
496 if (event->attr.precise_ip) {
497 int precise = x86_pmu_max_precise();
498
499 if (event->attr.precise_ip > precise)
500 return -EOPNOTSUPP;
501
502 /* There's no sense in having PEBS for non sampling events: */
503 if (!is_sampling_event(event))
504 return -EINVAL;
505 }
506 /*
507 * check that PEBS LBR correction does not conflict with
508 * whatever the user is asking with attr->branch_sample_type
509 */
510 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
511 u64 *br_type = &event->attr.branch_sample_type;
512
513 if (has_branch_stack(event)) {
514 if (!precise_br_compat(event))
515 return -EOPNOTSUPP;
516
517 /* branch_sample_type is compatible */
518
519 } else {
520 /*
521 * user did not specify branch_sample_type
522 *
523 * For PEBS fixups, we capture all
524 * the branches at the priv level of the
525 * event.
526 */
527 *br_type = PERF_SAMPLE_BRANCH_ANY;
528
529 if (!event->attr.exclude_user)
530 *br_type |= PERF_SAMPLE_BRANCH_USER;
531
532 if (!event->attr.exclude_kernel)
533 *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
534 }
535 }
536
537 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
538 event->attach_state |= PERF_ATTACH_TASK_DATA;
539
540 /*
541 * Generate PMC IRQs:
542 * (keep 'enabled' bit clear for now)
543 */
544 event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
545
546 /*
547 * Count user and OS events unless requested not to
548 */
549 if (!event->attr.exclude_user)
550 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
551 if (!event->attr.exclude_kernel)
552 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
553
554 if (event->attr.type == PERF_TYPE_RAW)
555 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
556
557 if (event->attr.sample_period && x86_pmu.limit_period) {
558 if (x86_pmu.limit_period(event, event->attr.sample_period) >
559 event->attr.sample_period)
560 return -EINVAL;
561 }
562
563 /* sample_regs_user never support XMM registers */
564 if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK))
565 return -EINVAL;
566 /*
567 * Besides the general purpose registers, XMM registers may
568 * be collected in PEBS on some platforms, e.g. Icelake
569 */
570 if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) {
571 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS))
572 return -EINVAL;
573
574 if (!event->attr.precise_ip)
575 return -EINVAL;
576 }
577
578 return x86_setup_perfctr(event);
579}
580
581/*
582 * Setup the hardware configuration for a given attr_type
583 */
584static int __x86_pmu_event_init(struct perf_event *event)
585{
586 int err;
587
588 if (!x86_pmu_initialized())
589 return -ENODEV;
590
591 err = x86_reserve_hardware();
592 if (err)
593 return err;
594
595 atomic_inc(&active_events);
596 event->destroy = hw_perf_event_destroy;
597
598 event->hw.idx = -1;
599 event->hw.last_cpu = -1;
600 event->hw.last_tag = ~0ULL;
601
602 /* mark unused */
603 event->hw.extra_reg.idx = EXTRA_REG_NONE;
604 event->hw.branch_reg.idx = EXTRA_REG_NONE;
605
606 return x86_pmu.hw_config(event);
607}
608
609void x86_pmu_disable_all(void)
610{
611 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
612 int idx;
613
614 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
615 u64 val;
616
617 if (!test_bit(idx, cpuc->active_mask))
618 continue;
619 rdmsrl(x86_pmu_config_addr(idx), val);
620 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
621 continue;
622 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
623 wrmsrl(x86_pmu_config_addr(idx), val);
624 }
625}
626
627/*
628 * There may be PMI landing after enabled=0. The PMI hitting could be before or
629 * after disable_all.
630 *
631 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler.
632 * It will not be re-enabled in the NMI handler again, because enabled=0. After
633 * handling the NMI, disable_all will be called, which will not change the
634 * state either. If PMI hits after disable_all, the PMU is already disabled
635 * before entering NMI handler. The NMI handler will not change the state
636 * either.
637 *
638 * So either situation is harmless.
639 */
640static void x86_pmu_disable(struct pmu *pmu)
641{
642 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
643
644 if (!x86_pmu_initialized())
645 return;
646
647 if (!cpuc->enabled)
648 return;
649
650 cpuc->n_added = 0;
651 cpuc->enabled = 0;
652 barrier();
653
654 x86_pmu.disable_all();
655}
656
657void x86_pmu_enable_all(int added)
658{
659 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
660 int idx;
661
662 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
663 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
664
665 if (!test_bit(idx, cpuc->active_mask))
666 continue;
667
668 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
669 }
670}
671
672static struct pmu pmu;
673
674static inline int is_x86_event(struct perf_event *event)
675{
676 return event->pmu == &pmu;
677}
678
679struct pmu *x86_get_pmu(void)
680{
681 return &pmu;
682}
683/*
684 * Event scheduler state:
685 *
686 * Assign events iterating over all events and counters, beginning
687 * with events with least weights first. Keep the current iterator
688 * state in struct sched_state.
689 */
690struct sched_state {
691 int weight;
692 int event; /* event index */
693 int counter; /* counter index */
694 int unassigned; /* number of events to be assigned left */
695 int nr_gp; /* number of GP counters used */
696 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
697};
698
699/* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
700#define SCHED_STATES_MAX 2
701
702struct perf_sched {
703 int max_weight;
704 int max_events;
705 int max_gp;
706 int saved_states;
707 struct event_constraint **constraints;
708 struct sched_state state;
709 struct sched_state saved[SCHED_STATES_MAX];
710};
711
712/*
713 * Initialize interator that runs through all events and counters.
714 */
715static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
716 int num, int wmin, int wmax, int gpmax)
717{
718 int idx;
719
720 memset(sched, 0, sizeof(*sched));
721 sched->max_events = num;
722 sched->max_weight = wmax;
723 sched->max_gp = gpmax;
724 sched->constraints = constraints;
725
726 for (idx = 0; idx < num; idx++) {
727 if (constraints[idx]->weight == wmin)
728 break;
729 }
730
731 sched->state.event = idx; /* start with min weight */
732 sched->state.weight = wmin;
733 sched->state.unassigned = num;
734}
735
736static void perf_sched_save_state(struct perf_sched *sched)
737{
738 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
739 return;
740
741 sched->saved[sched->saved_states] = sched->state;
742 sched->saved_states++;
743}
744
745static bool perf_sched_restore_state(struct perf_sched *sched)
746{
747 if (!sched->saved_states)
748 return false;
749
750 sched->saved_states--;
751 sched->state = sched->saved[sched->saved_states];
752
753 /* continue with next counter: */
754 clear_bit(sched->state.counter++, sched->state.used);
755
756 return true;
757}
758
759/*
760 * Select a counter for the current event to schedule. Return true on
761 * success.
762 */
763static bool __perf_sched_find_counter(struct perf_sched *sched)
764{
765 struct event_constraint *c;
766 int idx;
767
768 if (!sched->state.unassigned)
769 return false;
770
771 if (sched->state.event >= sched->max_events)
772 return false;
773
774 c = sched->constraints[sched->state.event];
775 /* Prefer fixed purpose counters */
776 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
777 idx = INTEL_PMC_IDX_FIXED;
778 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
779 if (!__test_and_set_bit(idx, sched->state.used))
780 goto done;
781 }
782 }
783
784 /* Grab the first unused counter starting with idx */
785 idx = sched->state.counter;
786 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
787 if (!__test_and_set_bit(idx, sched->state.used)) {
788 if (sched->state.nr_gp++ >= sched->max_gp)
789 return false;
790
791 goto done;
792 }
793 }
794
795 return false;
796
797done:
798 sched->state.counter = idx;
799
800 if (c->overlap)
801 perf_sched_save_state(sched);
802
803 return true;
804}
805
806static bool perf_sched_find_counter(struct perf_sched *sched)
807{
808 while (!__perf_sched_find_counter(sched)) {
809 if (!perf_sched_restore_state(sched))
810 return false;
811 }
812
813 return true;
814}
815
816/*
817 * Go through all unassigned events and find the next one to schedule.
818 * Take events with the least weight first. Return true on success.
819 */
820static bool perf_sched_next_event(struct perf_sched *sched)
821{
822 struct event_constraint *c;
823
824 if (!sched->state.unassigned || !--sched->state.unassigned)
825 return false;
826
827 do {
828 /* next event */
829 sched->state.event++;
830 if (sched->state.event >= sched->max_events) {
831 /* next weight */
832 sched->state.event = 0;
833 sched->state.weight++;
834 if (sched->state.weight > sched->max_weight)
835 return false;
836 }
837 c = sched->constraints[sched->state.event];
838 } while (c->weight != sched->state.weight);
839
840 sched->state.counter = 0; /* start with first counter */
841
842 return true;
843}
844
845/*
846 * Assign a counter for each event.
847 */
848int perf_assign_events(struct event_constraint **constraints, int n,
849 int wmin, int wmax, int gpmax, int *assign)
850{
851 struct perf_sched sched;
852
853 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
854
855 do {
856 if (!perf_sched_find_counter(&sched))
857 break; /* failed */
858 if (assign)
859 assign[sched.state.event] = sched.state.counter;
860 } while (perf_sched_next_event(&sched));
861
862 return sched.state.unassigned;
863}
864EXPORT_SYMBOL_GPL(perf_assign_events);
865
866int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
867{
868 struct event_constraint *c;
869 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
870 struct perf_event *e;
871 int n0, i, wmin, wmax, unsched = 0;
872 struct hw_perf_event *hwc;
873
874 bitmap_zero(used_mask, X86_PMC_IDX_MAX);
875
876 /*
877 * Compute the number of events already present; see x86_pmu_add(),
878 * validate_group() and x86_pmu_commit_txn(). For the former two
879 * cpuc->n_events hasn't been updated yet, while for the latter
880 * cpuc->n_txn contains the number of events added in the current
881 * transaction.
882 */
883 n0 = cpuc->n_events;
884 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
885 n0 -= cpuc->n_txn;
886
887 if (x86_pmu.start_scheduling)
888 x86_pmu.start_scheduling(cpuc);
889
890 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
891 c = cpuc->event_constraint[i];
892
893 /*
894 * Previously scheduled events should have a cached constraint,
895 * while new events should not have one.
896 */
897 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0));
898
899 /*
900 * Request constraints for new events; or for those events that
901 * have a dynamic constraint -- for those the constraint can
902 * change due to external factors (sibling state, allow_tfa).
903 */
904 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) {
905 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
906 cpuc->event_constraint[i] = c;
907 }
908
909 wmin = min(wmin, c->weight);
910 wmax = max(wmax, c->weight);
911 }
912
913 /*
914 * fastpath, try to reuse previous register
915 */
916 for (i = 0; i < n; i++) {
917 hwc = &cpuc->event_list[i]->hw;
918 c = cpuc->event_constraint[i];
919
920 /* never assigned */
921 if (hwc->idx == -1)
922 break;
923
924 /* constraint still honored */
925 if (!test_bit(hwc->idx, c->idxmsk))
926 break;
927
928 /* not already used */
929 if (test_bit(hwc->idx, used_mask))
930 break;
931
932 __set_bit(hwc->idx, used_mask);
933 if (assign)
934 assign[i] = hwc->idx;
935 }
936
937 /* slow path */
938 if (i != n) {
939 int gpmax = x86_pmu.num_counters;
940
941 /*
942 * Do not allow scheduling of more than half the available
943 * generic counters.
944 *
945 * This helps avoid counter starvation of sibling thread by
946 * ensuring at most half the counters cannot be in exclusive
947 * mode. There is no designated counters for the limits. Any
948 * N/2 counters can be used. This helps with events with
949 * specific counter constraints.
950 */
951 if (is_ht_workaround_enabled() && !cpuc->is_fake &&
952 READ_ONCE(cpuc->excl_cntrs->exclusive_present))
953 gpmax /= 2;
954
955 unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
956 wmax, gpmax, assign);
957 }
958
959 /*
960 * In case of success (unsched = 0), mark events as committed,
961 * so we do not put_constraint() in case new events are added
962 * and fail to be scheduled
963 *
964 * We invoke the lower level commit callback to lock the resource
965 *
966 * We do not need to do all of this in case we are called to
967 * validate an event group (assign == NULL)
968 */
969 if (!unsched && assign) {
970 for (i = 0; i < n; i++) {
971 e = cpuc->event_list[i];
972 if (x86_pmu.commit_scheduling)
973 x86_pmu.commit_scheduling(cpuc, i, assign[i]);
974 }
975 } else {
976 for (i = n0; i < n; i++) {
977 e = cpuc->event_list[i];
978
979 /*
980 * release events that failed scheduling
981 */
982 if (x86_pmu.put_event_constraints)
983 x86_pmu.put_event_constraints(cpuc, e);
984
985 cpuc->event_constraint[i] = NULL;
986 }
987 }
988
989 if (x86_pmu.stop_scheduling)
990 x86_pmu.stop_scheduling(cpuc);
991
992 return unsched ? -EINVAL : 0;
993}
994
995/*
996 * dogrp: true if must collect siblings events (group)
997 * returns total number of events and error code
998 */
999static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
1000{
1001 struct perf_event *event;
1002 int n, max_count;
1003
1004 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
1005
1006 /* current number of events already accepted */
1007 n = cpuc->n_events;
1008 if (!cpuc->n_events)
1009 cpuc->pebs_output = 0;
1010
1011 if (!cpuc->is_fake && leader->attr.precise_ip) {
1012 /*
1013 * For PEBS->PT, if !aux_event, the group leader (PT) went
1014 * away, the group was broken down and this singleton event
1015 * can't schedule any more.
1016 */
1017 if (is_pebs_pt(leader) && !leader->aux_event)
1018 return -EINVAL;
1019
1020 /*
1021 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS
1022 */
1023 if (cpuc->pebs_output &&
1024 cpuc->pebs_output != is_pebs_pt(leader) + 1)
1025 return -EINVAL;
1026
1027 cpuc->pebs_output = is_pebs_pt(leader) + 1;
1028 }
1029
1030 if (is_x86_event(leader)) {
1031 if (n >= max_count)
1032 return -EINVAL;
1033 cpuc->event_list[n] = leader;
1034 n++;
1035 }
1036 if (!dogrp)
1037 return n;
1038
1039 for_each_sibling_event(event, leader) {
1040 if (!is_x86_event(event) ||
1041 event->state <= PERF_EVENT_STATE_OFF)
1042 continue;
1043
1044 if (n >= max_count)
1045 return -EINVAL;
1046
1047 cpuc->event_list[n] = event;
1048 n++;
1049 }
1050 return n;
1051}
1052
1053static inline void x86_assign_hw_event(struct perf_event *event,
1054 struct cpu_hw_events *cpuc, int i)
1055{
1056 struct hw_perf_event *hwc = &event->hw;
1057
1058 hwc->idx = cpuc->assign[i];
1059 hwc->last_cpu = smp_processor_id();
1060 hwc->last_tag = ++cpuc->tags[i];
1061
1062 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
1063 hwc->config_base = 0;
1064 hwc->event_base = 0;
1065 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
1066 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1067 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
1068 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
1069 } else {
1070 hwc->config_base = x86_pmu_config_addr(hwc->idx);
1071 hwc->event_base = x86_pmu_event_addr(hwc->idx);
1072 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
1073 }
1074}
1075
1076/**
1077 * x86_perf_rdpmc_index - Return PMC counter used for event
1078 * @event: the perf_event to which the PMC counter was assigned
1079 *
1080 * The counter assigned to this performance event may change if interrupts
1081 * are enabled. This counter should thus never be used while interrupts are
1082 * enabled. Before this function is used to obtain the assigned counter the
1083 * event should be checked for validity using, for example,
1084 * perf_event_read_local(), within the same interrupt disabled section in
1085 * which this counter is planned to be used.
1086 *
1087 * Return: The index of the performance monitoring counter assigned to
1088 * @perf_event.
1089 */
1090int x86_perf_rdpmc_index(struct perf_event *event)
1091{
1092 lockdep_assert_irqs_disabled();
1093
1094 return event->hw.event_base_rdpmc;
1095}
1096
1097static inline int match_prev_assignment(struct hw_perf_event *hwc,
1098 struct cpu_hw_events *cpuc,
1099 int i)
1100{
1101 return hwc->idx == cpuc->assign[i] &&
1102 hwc->last_cpu == smp_processor_id() &&
1103 hwc->last_tag == cpuc->tags[i];
1104}
1105
1106static void x86_pmu_start(struct perf_event *event, int flags);
1107
1108static void x86_pmu_enable(struct pmu *pmu)
1109{
1110 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1111 struct perf_event *event;
1112 struct hw_perf_event *hwc;
1113 int i, added = cpuc->n_added;
1114
1115 if (!x86_pmu_initialized())
1116 return;
1117
1118 if (cpuc->enabled)
1119 return;
1120
1121 if (cpuc->n_added) {
1122 int n_running = cpuc->n_events - cpuc->n_added;
1123 /*
1124 * apply assignment obtained either from
1125 * hw_perf_group_sched_in() or x86_pmu_enable()
1126 *
1127 * step1: save events moving to new counters
1128 */
1129 for (i = 0; i < n_running; i++) {
1130 event = cpuc->event_list[i];
1131 hwc = &event->hw;
1132
1133 /*
1134 * we can avoid reprogramming counter if:
1135 * - assigned same counter as last time
1136 * - running on same CPU as last time
1137 * - no other event has used the counter since
1138 */
1139 if (hwc->idx == -1 ||
1140 match_prev_assignment(hwc, cpuc, i))
1141 continue;
1142
1143 /*
1144 * Ensure we don't accidentally enable a stopped
1145 * counter simply because we rescheduled.
1146 */
1147 if (hwc->state & PERF_HES_STOPPED)
1148 hwc->state |= PERF_HES_ARCH;
1149
1150 x86_pmu_stop(event, PERF_EF_UPDATE);
1151 }
1152
1153 /*
1154 * step2: reprogram moved events into new counters
1155 */
1156 for (i = 0; i < cpuc->n_events; i++) {
1157 event = cpuc->event_list[i];
1158 hwc = &event->hw;
1159
1160 if (!match_prev_assignment(hwc, cpuc, i))
1161 x86_assign_hw_event(event, cpuc, i);
1162 else if (i < n_running)
1163 continue;
1164
1165 if (hwc->state & PERF_HES_ARCH)
1166 continue;
1167
1168 x86_pmu_start(event, PERF_EF_RELOAD);
1169 }
1170 cpuc->n_added = 0;
1171 perf_events_lapic_init();
1172 }
1173
1174 cpuc->enabled = 1;
1175 barrier();
1176
1177 x86_pmu.enable_all(added);
1178}
1179
1180static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1181
1182/*
1183 * Set the next IRQ period, based on the hwc->period_left value.
1184 * To be called with the event disabled in hw:
1185 */
1186int x86_perf_event_set_period(struct perf_event *event)
1187{
1188 struct hw_perf_event *hwc = &event->hw;
1189 s64 left = local64_read(&hwc->period_left);
1190 s64 period = hwc->sample_period;
1191 int ret = 0, idx = hwc->idx;
1192
1193 if (idx == INTEL_PMC_IDX_FIXED_BTS)
1194 return 0;
1195
1196 /*
1197 * If we are way outside a reasonable range then just skip forward:
1198 */
1199 if (unlikely(left <= -period)) {
1200 left = period;
1201 local64_set(&hwc->period_left, left);
1202 hwc->last_period = period;
1203 ret = 1;
1204 }
1205
1206 if (unlikely(left <= 0)) {
1207 left += period;
1208 local64_set(&hwc->period_left, left);
1209 hwc->last_period = period;
1210 ret = 1;
1211 }
1212 /*
1213 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1214 */
1215 if (unlikely(left < 2))
1216 left = 2;
1217
1218 if (left > x86_pmu.max_period)
1219 left = x86_pmu.max_period;
1220
1221 if (x86_pmu.limit_period)
1222 left = x86_pmu.limit_period(event, left);
1223
1224 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1225
1226 /*
1227 * The hw event starts counting from this event offset,
1228 * mark it to be able to extra future deltas:
1229 */
1230 local64_set(&hwc->prev_count, (u64)-left);
1231
1232 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
1233
1234 /*
1235 * Due to erratum on certan cpu we need
1236 * a second write to be sure the register
1237 * is updated properly
1238 */
1239 if (x86_pmu.perfctr_second_write) {
1240 wrmsrl(hwc->event_base,
1241 (u64)(-left) & x86_pmu.cntval_mask);
1242 }
1243
1244 perf_event_update_userpage(event);
1245
1246 return ret;
1247}
1248
1249void x86_pmu_enable_event(struct perf_event *event)
1250{
1251 if (__this_cpu_read(cpu_hw_events.enabled))
1252 __x86_pmu_enable_event(&event->hw,
1253 ARCH_PERFMON_EVENTSEL_ENABLE);
1254}
1255
1256/*
1257 * Add a single event to the PMU.
1258 *
1259 * The event is added to the group of enabled events
1260 * but only if it can be scheduled with existing events.
1261 */
1262static int x86_pmu_add(struct perf_event *event, int flags)
1263{
1264 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1265 struct hw_perf_event *hwc;
1266 int assign[X86_PMC_IDX_MAX];
1267 int n, n0, ret;
1268
1269 hwc = &event->hw;
1270
1271 n0 = cpuc->n_events;
1272 ret = n = collect_events(cpuc, event, false);
1273 if (ret < 0)
1274 goto out;
1275
1276 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1277 if (!(flags & PERF_EF_START))
1278 hwc->state |= PERF_HES_ARCH;
1279
1280 /*
1281 * If group events scheduling transaction was started,
1282 * skip the schedulability test here, it will be performed
1283 * at commit time (->commit_txn) as a whole.
1284 *
1285 * If commit fails, we'll call ->del() on all events
1286 * for which ->add() was called.
1287 */
1288 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1289 goto done_collect;
1290
1291 ret = x86_pmu.schedule_events(cpuc, n, assign);
1292 if (ret)
1293 goto out;
1294 /*
1295 * copy new assignment, now we know it is possible
1296 * will be used by hw_perf_enable()
1297 */
1298 memcpy(cpuc->assign, assign, n*sizeof(int));
1299
1300done_collect:
1301 /*
1302 * Commit the collect_events() state. See x86_pmu_del() and
1303 * x86_pmu_*_txn().
1304 */
1305 cpuc->n_events = n;
1306 cpuc->n_added += n - n0;
1307 cpuc->n_txn += n - n0;
1308
1309 if (x86_pmu.add) {
1310 /*
1311 * This is before x86_pmu_enable() will call x86_pmu_start(),
1312 * so we enable LBRs before an event needs them etc..
1313 */
1314 x86_pmu.add(event);
1315 }
1316
1317 ret = 0;
1318out:
1319 return ret;
1320}
1321
1322static void x86_pmu_start(struct perf_event *event, int flags)
1323{
1324 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1325 int idx = event->hw.idx;
1326
1327 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1328 return;
1329
1330 if (WARN_ON_ONCE(idx == -1))
1331 return;
1332
1333 if (flags & PERF_EF_RELOAD) {
1334 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1335 x86_perf_event_set_period(event);
1336 }
1337
1338 event->hw.state = 0;
1339
1340 cpuc->events[idx] = event;
1341 __set_bit(idx, cpuc->active_mask);
1342 __set_bit(idx, cpuc->running);
1343 x86_pmu.enable(event);
1344 perf_event_update_userpage(event);
1345}
1346
1347void perf_event_print_debug(void)
1348{
1349 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1350 u64 pebs, debugctl;
1351 struct cpu_hw_events *cpuc;
1352 unsigned long flags;
1353 int cpu, idx;
1354
1355 if (!x86_pmu.num_counters)
1356 return;
1357
1358 local_irq_save(flags);
1359
1360 cpu = smp_processor_id();
1361 cpuc = &per_cpu(cpu_hw_events, cpu);
1362
1363 if (x86_pmu.version >= 2) {
1364 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1365 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1366 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1367 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1368
1369 pr_info("\n");
1370 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
1371 pr_info("CPU#%d: status: %016llx\n", cpu, status);
1372 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
1373 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
1374 if (x86_pmu.pebs_constraints) {
1375 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1376 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs);
1377 }
1378 if (x86_pmu.lbr_nr) {
1379 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1380 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl);
1381 }
1382 }
1383 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
1384
1385 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1386 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
1387 rdmsrl(x86_pmu_event_addr(idx), pmc_count);
1388
1389 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1390
1391 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
1392 cpu, idx, pmc_ctrl);
1393 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
1394 cpu, idx, pmc_count);
1395 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
1396 cpu, idx, prev_left);
1397 }
1398 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1399 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1400
1401 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1402 cpu, idx, pmc_count);
1403 }
1404 local_irq_restore(flags);
1405}
1406
1407void x86_pmu_stop(struct perf_event *event, int flags)
1408{
1409 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1410 struct hw_perf_event *hwc = &event->hw;
1411
1412 if (test_bit(hwc->idx, cpuc->active_mask)) {
1413 x86_pmu.disable(event);
1414 __clear_bit(hwc->idx, cpuc->active_mask);
1415 cpuc->events[hwc->idx] = NULL;
1416 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1417 hwc->state |= PERF_HES_STOPPED;
1418 }
1419
1420 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1421 /*
1422 * Drain the remaining delta count out of a event
1423 * that we are disabling:
1424 */
1425 x86_perf_event_update(event);
1426 hwc->state |= PERF_HES_UPTODATE;
1427 }
1428}
1429
1430static void x86_pmu_del(struct perf_event *event, int flags)
1431{
1432 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1433 int i;
1434
1435 /*
1436 * If we're called during a txn, we only need to undo x86_pmu.add.
1437 * The events never got scheduled and ->cancel_txn will truncate
1438 * the event_list.
1439 *
1440 * XXX assumes any ->del() called during a TXN will only be on
1441 * an event added during that same TXN.
1442 */
1443 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1444 goto do_del;
1445
1446 /*
1447 * Not a TXN, therefore cleanup properly.
1448 */
1449 x86_pmu_stop(event, PERF_EF_UPDATE);
1450
1451 for (i = 0; i < cpuc->n_events; i++) {
1452 if (event == cpuc->event_list[i])
1453 break;
1454 }
1455
1456 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
1457 return;
1458
1459 /* If we have a newly added event; make sure to decrease n_added. */
1460 if (i >= cpuc->n_events - cpuc->n_added)
1461 --cpuc->n_added;
1462
1463 if (x86_pmu.put_event_constraints)
1464 x86_pmu.put_event_constraints(cpuc, event);
1465
1466 /* Delete the array entry. */
1467 while (++i < cpuc->n_events) {
1468 cpuc->event_list[i-1] = cpuc->event_list[i];
1469 cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
1470 }
1471 cpuc->event_constraint[i-1] = NULL;
1472 --cpuc->n_events;
1473
1474 perf_event_update_userpage(event);
1475
1476do_del:
1477 if (x86_pmu.del) {
1478 /*
1479 * This is after x86_pmu_stop(); so we disable LBRs after any
1480 * event can need them etc..
1481 */
1482 x86_pmu.del(event);
1483 }
1484}
1485
1486int x86_pmu_handle_irq(struct pt_regs *regs)
1487{
1488 struct perf_sample_data data;
1489 struct cpu_hw_events *cpuc;
1490 struct perf_event *event;
1491 int idx, handled = 0;
1492 u64 val;
1493
1494 cpuc = this_cpu_ptr(&cpu_hw_events);
1495
1496 /*
1497 * Some chipsets need to unmask the LVTPC in a particular spot
1498 * inside the nmi handler. As a result, the unmasking was pushed
1499 * into all the nmi handlers.
1500 *
1501 * This generic handler doesn't seem to have any issues where the
1502 * unmasking occurs so it was left at the top.
1503 */
1504 apic_write(APIC_LVTPC, APIC_DM_NMI);
1505
1506 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1507 if (!test_bit(idx, cpuc->active_mask))
1508 continue;
1509
1510 event = cpuc->events[idx];
1511
1512 val = x86_perf_event_update(event);
1513 if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1514 continue;
1515
1516 /*
1517 * event overflow
1518 */
1519 handled++;
1520 perf_sample_data_init(&data, 0, event->hw.last_period);
1521
1522 if (!x86_perf_event_set_period(event))
1523 continue;
1524
1525 if (perf_event_overflow(event, &data, regs))
1526 x86_pmu_stop(event, 0);
1527 }
1528
1529 if (handled)
1530 inc_irq_stat(apic_perf_irqs);
1531
1532 return handled;
1533}
1534
1535void perf_events_lapic_init(void)
1536{
1537 if (!x86_pmu.apic || !x86_pmu_initialized())
1538 return;
1539
1540 /*
1541 * Always use NMI for PMU
1542 */
1543 apic_write(APIC_LVTPC, APIC_DM_NMI);
1544}
1545
1546static int
1547perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1548{
1549 u64 start_clock;
1550 u64 finish_clock;
1551 int ret;
1552
1553 /*
1554 * All PMUs/events that share this PMI handler should make sure to
1555 * increment active_events for their events.
1556 */
1557 if (!atomic_read(&active_events))
1558 return NMI_DONE;
1559
1560 start_clock = sched_clock();
1561 ret = x86_pmu.handle_irq(regs);
1562 finish_clock = sched_clock();
1563
1564 perf_sample_event_took(finish_clock - start_clock);
1565
1566 return ret;
1567}
1568NOKPROBE_SYMBOL(perf_event_nmi_handler);
1569
1570struct event_constraint emptyconstraint;
1571struct event_constraint unconstrained;
1572
1573static int x86_pmu_prepare_cpu(unsigned int cpu)
1574{
1575 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1576 int i;
1577
1578 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
1579 cpuc->kfree_on_online[i] = NULL;
1580 if (x86_pmu.cpu_prepare)
1581 return x86_pmu.cpu_prepare(cpu);
1582 return 0;
1583}
1584
1585static int x86_pmu_dead_cpu(unsigned int cpu)
1586{
1587 if (x86_pmu.cpu_dead)
1588 x86_pmu.cpu_dead(cpu);
1589 return 0;
1590}
1591
1592static int x86_pmu_online_cpu(unsigned int cpu)
1593{
1594 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1595 int i;
1596
1597 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
1598 kfree(cpuc->kfree_on_online[i]);
1599 cpuc->kfree_on_online[i] = NULL;
1600 }
1601 return 0;
1602}
1603
1604static int x86_pmu_starting_cpu(unsigned int cpu)
1605{
1606 if (x86_pmu.cpu_starting)
1607 x86_pmu.cpu_starting(cpu);
1608 return 0;
1609}
1610
1611static int x86_pmu_dying_cpu(unsigned int cpu)
1612{
1613 if (x86_pmu.cpu_dying)
1614 x86_pmu.cpu_dying(cpu);
1615 return 0;
1616}
1617
1618static void __init pmu_check_apic(void)
1619{
1620 if (boot_cpu_has(X86_FEATURE_APIC))
1621 return;
1622
1623 x86_pmu.apic = 0;
1624 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1625 pr_info("no hardware sampling interrupt available.\n");
1626
1627 /*
1628 * If we have a PMU initialized but no APIC
1629 * interrupts, we cannot sample hardware
1630 * events (user-space has to fall back and
1631 * sample via a hrtimer based software event):
1632 */
1633 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1634
1635}
1636
1637static struct attribute_group x86_pmu_format_group __ro_after_init = {
1638 .name = "format",
1639 .attrs = NULL,
1640};
1641
1642ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
1643{
1644 struct perf_pmu_events_attr *pmu_attr = \
1645 container_of(attr, struct perf_pmu_events_attr, attr);
1646 u64 config = x86_pmu.event_map(pmu_attr->id);
1647
1648 /* string trumps id */
1649 if (pmu_attr->event_str)
1650 return sprintf(page, "%s", pmu_attr->event_str);
1651
1652 return x86_pmu.events_sysfs_show(page, config);
1653}
1654EXPORT_SYMBOL_GPL(events_sysfs_show);
1655
1656ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1657 char *page)
1658{
1659 struct perf_pmu_events_ht_attr *pmu_attr =
1660 container_of(attr, struct perf_pmu_events_ht_attr, attr);
1661
1662 /*
1663 * Report conditional events depending on Hyper-Threading.
1664 *
1665 * This is overly conservative as usually the HT special
1666 * handling is not needed if the other CPU thread is idle.
1667 *
1668 * Note this does not (and cannot) handle the case when thread
1669 * siblings are invisible, for example with virtualization
1670 * if they are owned by some other guest. The user tool
1671 * has to re-read when a thread sibling gets onlined later.
1672 */
1673 return sprintf(page, "%s",
1674 topology_max_smt_threads() > 1 ?
1675 pmu_attr->event_str_ht :
1676 pmu_attr->event_str_noht);
1677}
1678
1679EVENT_ATTR(cpu-cycles, CPU_CYCLES );
1680EVENT_ATTR(instructions, INSTRUCTIONS );
1681EVENT_ATTR(cache-references, CACHE_REFERENCES );
1682EVENT_ATTR(cache-misses, CACHE_MISSES );
1683EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS );
1684EVENT_ATTR(branch-misses, BRANCH_MISSES );
1685EVENT_ATTR(bus-cycles, BUS_CYCLES );
1686EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND );
1687EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND );
1688EVENT_ATTR(ref-cycles, REF_CPU_CYCLES );
1689
1690static struct attribute *empty_attrs;
1691
1692static struct attribute *events_attr[] = {
1693 EVENT_PTR(CPU_CYCLES),
1694 EVENT_PTR(INSTRUCTIONS),
1695 EVENT_PTR(CACHE_REFERENCES),
1696 EVENT_PTR(CACHE_MISSES),
1697 EVENT_PTR(BRANCH_INSTRUCTIONS),
1698 EVENT_PTR(BRANCH_MISSES),
1699 EVENT_PTR(BUS_CYCLES),
1700 EVENT_PTR(STALLED_CYCLES_FRONTEND),
1701 EVENT_PTR(STALLED_CYCLES_BACKEND),
1702 EVENT_PTR(REF_CPU_CYCLES),
1703 NULL,
1704};
1705
1706/*
1707 * Remove all undefined events (x86_pmu.event_map(id) == 0)
1708 * out of events_attr attributes.
1709 */
1710static umode_t
1711is_visible(struct kobject *kobj, struct attribute *attr, int idx)
1712{
1713 struct perf_pmu_events_attr *pmu_attr;
1714
1715 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
1716 /* str trumps id */
1717 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0;
1718}
1719
1720static struct attribute_group x86_pmu_events_group __ro_after_init = {
1721 .name = "events",
1722 .attrs = events_attr,
1723 .is_visible = is_visible,
1724};
1725
1726ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
1727{
1728 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
1729 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
1730 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE);
1731 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
1732 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY);
1733 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV);
1734 ssize_t ret;
1735
1736 /*
1737 * We have whole page size to spend and just little data
1738 * to write, so we can safely use sprintf.
1739 */
1740 ret = sprintf(page, "event=0x%02llx", event);
1741
1742 if (umask)
1743 ret += sprintf(page + ret, ",umask=0x%02llx", umask);
1744
1745 if (edge)
1746 ret += sprintf(page + ret, ",edge");
1747
1748 if (pc)
1749 ret += sprintf(page + ret, ",pc");
1750
1751 if (any)
1752 ret += sprintf(page + ret, ",any");
1753
1754 if (inv)
1755 ret += sprintf(page + ret, ",inv");
1756
1757 if (cmask)
1758 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
1759
1760 ret += sprintf(page + ret, "\n");
1761
1762 return ret;
1763}
1764
1765static struct attribute_group x86_pmu_attr_group;
1766static struct attribute_group x86_pmu_caps_group;
1767
1768static int __init init_hw_perf_events(void)
1769{
1770 struct x86_pmu_quirk *quirk;
1771 int err;
1772
1773 pr_info("Performance Events: ");
1774
1775 switch (boot_cpu_data.x86_vendor) {
1776 case X86_VENDOR_INTEL:
1777 err = intel_pmu_init();
1778 break;
1779 case X86_VENDOR_AMD:
1780 err = amd_pmu_init();
1781 break;
1782 case X86_VENDOR_HYGON:
1783 err = amd_pmu_init();
1784 x86_pmu.name = "HYGON";
1785 break;
1786 default:
1787 err = -ENOTSUPP;
1788 }
1789 if (err != 0) {
1790 pr_cont("no PMU driver, software events only.\n");
1791 return 0;
1792 }
1793
1794 pmu_check_apic();
1795
1796 /* sanity check that the hardware exists or is emulated */
1797 if (!check_hw_exists())
1798 return 0;
1799
1800 pr_cont("%s PMU driver.\n", x86_pmu.name);
1801
1802 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1803
1804 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
1805 quirk->func();
1806
1807 if (!x86_pmu.intel_ctrl)
1808 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
1809
1810 perf_events_lapic_init();
1811 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
1812
1813 unconstrained = (struct event_constraint)
1814 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1815 0, x86_pmu.num_counters, 0, 0);
1816
1817 x86_pmu_format_group.attrs = x86_pmu.format_attrs;
1818
1819 if (!x86_pmu.events_sysfs_show)
1820 x86_pmu_events_group.attrs = &empty_attrs;
1821
1822 pmu.attr_update = x86_pmu.attr_update;
1823
1824 pr_info("... version: %d\n", x86_pmu.version);
1825 pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
1826 pr_info("... generic registers: %d\n", x86_pmu.num_counters);
1827 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);
1828 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
1829 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed);
1830 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl);
1831
1832 /*
1833 * Install callbacks. Core will call them for each online
1834 * cpu.
1835 */
1836 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare",
1837 x86_pmu_prepare_cpu, x86_pmu_dead_cpu);
1838 if (err)
1839 return err;
1840
1841 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING,
1842 "perf/x86:starting", x86_pmu_starting_cpu,
1843 x86_pmu_dying_cpu);
1844 if (err)
1845 goto out;
1846
1847 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online",
1848 x86_pmu_online_cpu, NULL);
1849 if (err)
1850 goto out1;
1851
1852 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1853 if (err)
1854 goto out2;
1855
1856 return 0;
1857
1858out2:
1859 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE);
1860out1:
1861 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING);
1862out:
1863 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE);
1864 return err;
1865}
1866early_initcall(init_hw_perf_events);
1867
1868static inline void x86_pmu_read(struct perf_event *event)
1869{
1870 if (x86_pmu.read)
1871 return x86_pmu.read(event);
1872 x86_perf_event_update(event);
1873}
1874
1875/*
1876 * Start group events scheduling transaction
1877 * Set the flag to make pmu::enable() not perform the
1878 * schedulability test, it will be performed at commit time
1879 *
1880 * We only support PERF_PMU_TXN_ADD transactions. Save the
1881 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
1882 * transactions.
1883 */
1884static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
1885{
1886 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1887
1888 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */
1889
1890 cpuc->txn_flags = txn_flags;
1891 if (txn_flags & ~PERF_PMU_TXN_ADD)
1892 return;
1893
1894 perf_pmu_disable(pmu);
1895 __this_cpu_write(cpu_hw_events.n_txn, 0);
1896}
1897
1898/*
1899 * Stop group events scheduling transaction
1900 * Clear the flag and pmu::enable() will perform the
1901 * schedulability test.
1902 */
1903static void x86_pmu_cancel_txn(struct pmu *pmu)
1904{
1905 unsigned int txn_flags;
1906 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1907
1908 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1909
1910 txn_flags = cpuc->txn_flags;
1911 cpuc->txn_flags = 0;
1912 if (txn_flags & ~PERF_PMU_TXN_ADD)
1913 return;
1914
1915 /*
1916 * Truncate collected array by the number of events added in this
1917 * transaction. See x86_pmu_add() and x86_pmu_*_txn().
1918 */
1919 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
1920 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
1921 perf_pmu_enable(pmu);
1922}
1923
1924/*
1925 * Commit group events scheduling transaction
1926 * Perform the group schedulability test as a whole
1927 * Return 0 if success
1928 *
1929 * Does not cancel the transaction on failure; expects the caller to do this.
1930 */
1931static int x86_pmu_commit_txn(struct pmu *pmu)
1932{
1933 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1934 int assign[X86_PMC_IDX_MAX];
1935 int n, ret;
1936
1937 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1938
1939 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
1940 cpuc->txn_flags = 0;
1941 return 0;
1942 }
1943
1944 n = cpuc->n_events;
1945
1946 if (!x86_pmu_initialized())
1947 return -EAGAIN;
1948
1949 ret = x86_pmu.schedule_events(cpuc, n, assign);
1950 if (ret)
1951 return ret;
1952
1953 /*
1954 * copy new assignment, now we know it is possible
1955 * will be used by hw_perf_enable()
1956 */
1957 memcpy(cpuc->assign, assign, n*sizeof(int));
1958
1959 cpuc->txn_flags = 0;
1960 perf_pmu_enable(pmu);
1961 return 0;
1962}
1963/*
1964 * a fake_cpuc is used to validate event groups. Due to
1965 * the extra reg logic, we need to also allocate a fake
1966 * per_core and per_cpu structure. Otherwise, group events
1967 * using extra reg may conflict without the kernel being
1968 * able to catch this when the last event gets added to
1969 * the group.
1970 */
1971static void free_fake_cpuc(struct cpu_hw_events *cpuc)
1972{
1973 intel_cpuc_finish(cpuc);
1974 kfree(cpuc);
1975}
1976
1977static struct cpu_hw_events *allocate_fake_cpuc(void)
1978{
1979 struct cpu_hw_events *cpuc;
1980 int cpu = raw_smp_processor_id();
1981
1982 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
1983 if (!cpuc)
1984 return ERR_PTR(-ENOMEM);
1985 cpuc->is_fake = 1;
1986
1987 if (intel_cpuc_prepare(cpuc, cpu))
1988 goto error;
1989
1990 return cpuc;
1991error:
1992 free_fake_cpuc(cpuc);
1993 return ERR_PTR(-ENOMEM);
1994}
1995
1996/*
1997 * validate that we can schedule this event
1998 */
1999static int validate_event(struct perf_event *event)
2000{
2001 struct cpu_hw_events *fake_cpuc;
2002 struct event_constraint *c;
2003 int ret = 0;
2004
2005 fake_cpuc = allocate_fake_cpuc();
2006 if (IS_ERR(fake_cpuc))
2007 return PTR_ERR(fake_cpuc);
2008
2009 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event);
2010
2011 if (!c || !c->weight)
2012 ret = -EINVAL;
2013
2014 if (x86_pmu.put_event_constraints)
2015 x86_pmu.put_event_constraints(fake_cpuc, event);
2016
2017 free_fake_cpuc(fake_cpuc);
2018
2019 return ret;
2020}
2021
2022/*
2023 * validate a single event group
2024 *
2025 * validation include:
2026 * - check events are compatible which each other
2027 * - events do not compete for the same counter
2028 * - number of events <= number of counters
2029 *
2030 * validation ensures the group can be loaded onto the
2031 * PMU if it was the only group available.
2032 */
2033static int validate_group(struct perf_event *event)
2034{
2035 struct perf_event *leader = event->group_leader;
2036 struct cpu_hw_events *fake_cpuc;
2037 int ret = -EINVAL, n;
2038
2039 fake_cpuc = allocate_fake_cpuc();
2040 if (IS_ERR(fake_cpuc))
2041 return PTR_ERR(fake_cpuc);
2042 /*
2043 * the event is not yet connected with its
2044 * siblings therefore we must first collect
2045 * existing siblings, then add the new event
2046 * before we can simulate the scheduling
2047 */
2048 n = collect_events(fake_cpuc, leader, true);
2049 if (n < 0)
2050 goto out;
2051
2052 fake_cpuc->n_events = n;
2053 n = collect_events(fake_cpuc, event, false);
2054 if (n < 0)
2055 goto out;
2056
2057 fake_cpuc->n_events = 0;
2058 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
2059
2060out:
2061 free_fake_cpuc(fake_cpuc);
2062 return ret;
2063}
2064
2065static int x86_pmu_event_init(struct perf_event *event)
2066{
2067 struct pmu *tmp;
2068 int err;
2069
2070 switch (event->attr.type) {
2071 case PERF_TYPE_RAW:
2072 case PERF_TYPE_HARDWARE:
2073 case PERF_TYPE_HW_CACHE:
2074 break;
2075
2076 default:
2077 return -ENOENT;
2078 }
2079
2080 err = __x86_pmu_event_init(event);
2081 if (!err) {
2082 /*
2083 * we temporarily connect event to its pmu
2084 * such that validate_group() can classify
2085 * it as an x86 event using is_x86_event()
2086 */
2087 tmp = event->pmu;
2088 event->pmu = &pmu;
2089
2090 if (event->group_leader != event)
2091 err = validate_group(event);
2092 else
2093 err = validate_event(event);
2094
2095 event->pmu = tmp;
2096 }
2097 if (err) {
2098 if (event->destroy)
2099 event->destroy(event);
2100 }
2101
2102 if (READ_ONCE(x86_pmu.attr_rdpmc) &&
2103 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
2104 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
2105
2106 return err;
2107}
2108
2109static void refresh_pce(void *ignored)
2110{
2111 load_mm_cr4_irqsoff(this_cpu_read(cpu_tlbstate.loaded_mm));
2112}
2113
2114static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
2115{
2116 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2117 return;
2118
2119 /*
2120 * This function relies on not being called concurrently in two
2121 * tasks in the same mm. Otherwise one task could observe
2122 * perf_rdpmc_allowed > 1 and return all the way back to
2123 * userspace with CR4.PCE clear while another task is still
2124 * doing on_each_cpu_mask() to propagate CR4.PCE.
2125 *
2126 * For now, this can't happen because all callers hold mmap_sem
2127 * for write. If this changes, we'll need a different solution.
2128 */
2129 lockdep_assert_held_write(&mm->mmap_sem);
2130
2131 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
2132 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2133}
2134
2135static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
2136{
2137
2138 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2139 return;
2140
2141 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
2142 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2143}
2144
2145static int x86_pmu_event_idx(struct perf_event *event)
2146{
2147 int idx = event->hw.idx;
2148
2149 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2150 return 0;
2151
2152 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
2153 idx -= INTEL_PMC_IDX_FIXED;
2154 idx |= 1 << 30;
2155 }
2156
2157 return idx + 1;
2158}
2159
2160static ssize_t get_attr_rdpmc(struct device *cdev,
2161 struct device_attribute *attr,
2162 char *buf)
2163{
2164 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
2165}
2166
2167static ssize_t set_attr_rdpmc(struct device *cdev,
2168 struct device_attribute *attr,
2169 const char *buf, size_t count)
2170{
2171 unsigned long val;
2172 ssize_t ret;
2173
2174 ret = kstrtoul(buf, 0, &val);
2175 if (ret)
2176 return ret;
2177
2178 if (val > 2)
2179 return -EINVAL;
2180
2181 if (x86_pmu.attr_rdpmc_broken)
2182 return -ENOTSUPP;
2183
2184 if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) {
2185 /*
2186 * Changing into or out of always available, aka
2187 * perf-event-bypassing mode. This path is extremely slow,
2188 * but only root can trigger it, so it's okay.
2189 */
2190 if (val == 2)
2191 static_branch_inc(&rdpmc_always_available_key);
2192 else
2193 static_branch_dec(&rdpmc_always_available_key);
2194 on_each_cpu(refresh_pce, NULL, 1);
2195 }
2196
2197 x86_pmu.attr_rdpmc = val;
2198
2199 return count;
2200}
2201
2202static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
2203
2204static struct attribute *x86_pmu_attrs[] = {
2205 &dev_attr_rdpmc.attr,
2206 NULL,
2207};
2208
2209static struct attribute_group x86_pmu_attr_group __ro_after_init = {
2210 .attrs = x86_pmu_attrs,
2211};
2212
2213static ssize_t max_precise_show(struct device *cdev,
2214 struct device_attribute *attr,
2215 char *buf)
2216{
2217 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
2218}
2219
2220static DEVICE_ATTR_RO(max_precise);
2221
2222static struct attribute *x86_pmu_caps_attrs[] = {
2223 &dev_attr_max_precise.attr,
2224 NULL
2225};
2226
2227static struct attribute_group x86_pmu_caps_group __ro_after_init = {
2228 .name = "caps",
2229 .attrs = x86_pmu_caps_attrs,
2230};
2231
2232static const struct attribute_group *x86_pmu_attr_groups[] = {
2233 &x86_pmu_attr_group,
2234 &x86_pmu_format_group,
2235 &x86_pmu_events_group,
2236 &x86_pmu_caps_group,
2237 NULL,
2238};
2239
2240static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
2241{
2242 if (x86_pmu.sched_task)
2243 x86_pmu.sched_task(ctx, sched_in);
2244}
2245
2246void perf_check_microcode(void)
2247{
2248 if (x86_pmu.check_microcode)
2249 x86_pmu.check_microcode();
2250}
2251
2252static int x86_pmu_check_period(struct perf_event *event, u64 value)
2253{
2254 if (x86_pmu.check_period && x86_pmu.check_period(event, value))
2255 return -EINVAL;
2256
2257 if (value && x86_pmu.limit_period) {
2258 if (x86_pmu.limit_period(event, value) > value)
2259 return -EINVAL;
2260 }
2261
2262 return 0;
2263}
2264
2265static int x86_pmu_aux_output_match(struct perf_event *event)
2266{
2267 if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT))
2268 return 0;
2269
2270 if (x86_pmu.aux_output_match)
2271 return x86_pmu.aux_output_match(event);
2272
2273 return 0;
2274}
2275
2276static struct pmu pmu = {
2277 .pmu_enable = x86_pmu_enable,
2278 .pmu_disable = x86_pmu_disable,
2279
2280 .attr_groups = x86_pmu_attr_groups,
2281
2282 .event_init = x86_pmu_event_init,
2283
2284 .event_mapped = x86_pmu_event_mapped,
2285 .event_unmapped = x86_pmu_event_unmapped,
2286
2287 .add = x86_pmu_add,
2288 .del = x86_pmu_del,
2289 .start = x86_pmu_start,
2290 .stop = x86_pmu_stop,
2291 .read = x86_pmu_read,
2292
2293 .start_txn = x86_pmu_start_txn,
2294 .cancel_txn = x86_pmu_cancel_txn,
2295 .commit_txn = x86_pmu_commit_txn,
2296
2297 .event_idx = x86_pmu_event_idx,
2298 .sched_task = x86_pmu_sched_task,
2299 .task_ctx_size = sizeof(struct x86_perf_task_context),
2300 .check_period = x86_pmu_check_period,
2301
2302 .aux_output_match = x86_pmu_aux_output_match,
2303};
2304
2305void arch_perf_update_userpage(struct perf_event *event,
2306 struct perf_event_mmap_page *userpg, u64 now)
2307{
2308 struct cyc2ns_data data;
2309 u64 offset;
2310
2311 userpg->cap_user_time = 0;
2312 userpg->cap_user_time_zero = 0;
2313 userpg->cap_user_rdpmc =
2314 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
2315 userpg->pmc_width = x86_pmu.cntval_bits;
2316
2317 if (!using_native_sched_clock() || !sched_clock_stable())
2318 return;
2319
2320 cyc2ns_read_begin(&data);
2321
2322 offset = data.cyc2ns_offset + __sched_clock_offset;
2323
2324 /*
2325 * Internal timekeeping for enabled/running/stopped times
2326 * is always in the local_clock domain.
2327 */
2328 userpg->cap_user_time = 1;
2329 userpg->time_mult = data.cyc2ns_mul;
2330 userpg->time_shift = data.cyc2ns_shift;
2331 userpg->time_offset = offset - now;
2332
2333 /*
2334 * cap_user_time_zero doesn't make sense when we're using a different
2335 * time base for the records.
2336 */
2337 if (!event->attr.use_clockid) {
2338 userpg->cap_user_time_zero = 1;
2339 userpg->time_zero = offset;
2340 }
2341
2342 cyc2ns_read_end();
2343}
2344
2345/*
2346 * Determine whether the regs were taken from an irq/exception handler rather
2347 * than from perf_arch_fetch_caller_regs().
2348 */
2349static bool perf_hw_regs(struct pt_regs *regs)
2350{
2351 return regs->flags & X86_EFLAGS_FIXED;
2352}
2353
2354void
2355perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2356{
2357 struct unwind_state state;
2358 unsigned long addr;
2359
2360 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2361 /* TODO: We don't support guest os callchain now */
2362 return;
2363 }
2364
2365 if (perf_callchain_store(entry, regs->ip))
2366 return;
2367
2368 if (perf_hw_regs(regs))
2369 unwind_start(&state, current, regs, NULL);
2370 else
2371 unwind_start(&state, current, NULL, (void *)regs->sp);
2372
2373 for (; !unwind_done(&state); unwind_next_frame(&state)) {
2374 addr = unwind_get_return_address(&state);
2375 if (!addr || perf_callchain_store(entry, addr))
2376 return;
2377 }
2378}
2379
2380static inline int
2381valid_user_frame(const void __user *fp, unsigned long size)
2382{
2383 return (__range_not_ok(fp, size, TASK_SIZE) == 0);
2384}
2385
2386static unsigned long get_segment_base(unsigned int segment)
2387{
2388 struct desc_struct *desc;
2389 unsigned int idx = segment >> 3;
2390
2391 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2392#ifdef CONFIG_MODIFY_LDT_SYSCALL
2393 struct ldt_struct *ldt;
2394
2395 /* IRQs are off, so this synchronizes with smp_store_release */
2396 ldt = READ_ONCE(current->active_mm->context.ldt);
2397 if (!ldt || idx >= ldt->nr_entries)
2398 return 0;
2399
2400 desc = &ldt->entries[idx];
2401#else
2402 return 0;
2403#endif
2404 } else {
2405 if (idx >= GDT_ENTRIES)
2406 return 0;
2407
2408 desc = raw_cpu_ptr(gdt_page.gdt) + idx;
2409 }
2410
2411 return get_desc_base(desc);
2412}
2413
2414#ifdef CONFIG_IA32_EMULATION
2415
2416#include <linux/compat.h>
2417
2418static inline int
2419perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2420{
2421 /* 32-bit process in 64-bit kernel. */
2422 unsigned long ss_base, cs_base;
2423 struct stack_frame_ia32 frame;
2424 const void __user *fp;
2425
2426 if (!test_thread_flag(TIF_IA32))
2427 return 0;
2428
2429 cs_base = get_segment_base(regs->cs);
2430 ss_base = get_segment_base(regs->ss);
2431
2432 fp = compat_ptr(ss_base + regs->bp);
2433 pagefault_disable();
2434 while (entry->nr < entry->max_stack) {
2435 unsigned long bytes;
2436 frame.next_frame = 0;
2437 frame.return_address = 0;
2438
2439 if (!valid_user_frame(fp, sizeof(frame)))
2440 break;
2441
2442 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4);
2443 if (bytes != 0)
2444 break;
2445 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4);
2446 if (bytes != 0)
2447 break;
2448
2449 perf_callchain_store(entry, cs_base + frame.return_address);
2450 fp = compat_ptr(ss_base + frame.next_frame);
2451 }
2452 pagefault_enable();
2453 return 1;
2454}
2455#else
2456static inline int
2457perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2458{
2459 return 0;
2460}
2461#endif
2462
2463void
2464perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2465{
2466 struct stack_frame frame;
2467 const unsigned long __user *fp;
2468
2469 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2470 /* TODO: We don't support guest os callchain now */
2471 return;
2472 }
2473
2474 /*
2475 * We don't know what to do with VM86 stacks.. ignore them for now.
2476 */
2477 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
2478 return;
2479
2480 fp = (unsigned long __user *)regs->bp;
2481
2482 perf_callchain_store(entry, regs->ip);
2483
2484 if (!nmi_uaccess_okay())
2485 return;
2486
2487 if (perf_callchain_user32(regs, entry))
2488 return;
2489
2490 pagefault_disable();
2491 while (entry->nr < entry->max_stack) {
2492 unsigned long bytes;
2493
2494 frame.next_frame = NULL;
2495 frame.return_address = 0;
2496
2497 if (!valid_user_frame(fp, sizeof(frame)))
2498 break;
2499
2500 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp));
2501 if (bytes != 0)
2502 break;
2503 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp));
2504 if (bytes != 0)
2505 break;
2506
2507 perf_callchain_store(entry, frame.return_address);
2508 fp = (void __user *)frame.next_frame;
2509 }
2510 pagefault_enable();
2511}
2512
2513/*
2514 * Deal with code segment offsets for the various execution modes:
2515 *
2516 * VM86 - the good olde 16 bit days, where the linear address is
2517 * 20 bits and we use regs->ip + 0x10 * regs->cs.
2518 *
2519 * IA32 - Where we need to look at GDT/LDT segment descriptor tables
2520 * to figure out what the 32bit base address is.
2521 *
2522 * X32 - has TIF_X32 set, but is running in x86_64
2523 *
2524 * X86_64 - CS,DS,SS,ES are all zero based.
2525 */
2526static unsigned long code_segment_base(struct pt_regs *regs)
2527{
2528 /*
2529 * For IA32 we look at the GDT/LDT segment base to convert the
2530 * effective IP to a linear address.
2531 */
2532
2533#ifdef CONFIG_X86_32
2534 /*
2535 * If we are in VM86 mode, add the segment offset to convert to a
2536 * linear address.
2537 */
2538 if (regs->flags & X86_VM_MASK)
2539 return 0x10 * regs->cs;
2540
2541 if (user_mode(regs) && regs->cs != __USER_CS)
2542 return get_segment_base(regs->cs);
2543#else
2544 if (user_mode(regs) && !user_64bit_mode(regs) &&
2545 regs->cs != __USER32_CS)
2546 return get_segment_base(regs->cs);
2547#endif
2548 return 0;
2549}
2550
2551unsigned long perf_instruction_pointer(struct pt_regs *regs)
2552{
2553 if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
2554 return perf_guest_cbs->get_guest_ip();
2555
2556 return regs->ip + code_segment_base(regs);
2557}
2558
2559unsigned long perf_misc_flags(struct pt_regs *regs)
2560{
2561 int misc = 0;
2562
2563 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2564 if (perf_guest_cbs->is_user_mode())
2565 misc |= PERF_RECORD_MISC_GUEST_USER;
2566 else
2567 misc |= PERF_RECORD_MISC_GUEST_KERNEL;
2568 } else {
2569 if (user_mode(regs))
2570 misc |= PERF_RECORD_MISC_USER;
2571 else
2572 misc |= PERF_RECORD_MISC_KERNEL;
2573 }
2574
2575 if (regs->flags & PERF_EFLAGS_EXACT)
2576 misc |= PERF_RECORD_MISC_EXACT_IP;
2577
2578 return misc;
2579}
2580
2581void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
2582{
2583 cap->version = x86_pmu.version;
2584 cap->num_counters_gp = x86_pmu.num_counters;
2585 cap->num_counters_fixed = x86_pmu.num_counters_fixed;
2586 cap->bit_width_gp = x86_pmu.cntval_bits;
2587 cap->bit_width_fixed = x86_pmu.cntval_bits;
2588 cap->events_mask = (unsigned int)x86_pmu.events_maskl;
2589 cap->events_mask_len = x86_pmu.events_mask_len;
2590}
2591EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);
1/*
2 * Performance events x86 architecture code
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10 * Copyright (C) 2009 Google, Inc., Stephane Eranian
11 *
12 * For licencing details see kernel-base/COPYING
13 */
14
15#include <linux/perf_event.h>
16#include <linux/capability.h>
17#include <linux/notifier.h>
18#include <linux/hardirq.h>
19#include <linux/kprobes.h>
20#include <linux/export.h>
21#include <linux/init.h>
22#include <linux/kdebug.h>
23#include <linux/sched/mm.h>
24#include <linux/sched/clock.h>
25#include <linux/uaccess.h>
26#include <linux/slab.h>
27#include <linux/cpu.h>
28#include <linux/bitops.h>
29#include <linux/device.h>
30#include <linux/nospec.h>
31#include <linux/static_call.h>
32
33#include <asm/apic.h>
34#include <asm/stacktrace.h>
35#include <asm/nmi.h>
36#include <asm/smp.h>
37#include <asm/alternative.h>
38#include <asm/mmu_context.h>
39#include <asm/tlbflush.h>
40#include <asm/timer.h>
41#include <asm/desc.h>
42#include <asm/ldt.h>
43#include <asm/unwind.h>
44#include <asm/uprobes.h>
45#include <asm/ibt.h>
46
47#include "perf_event.h"
48
49struct x86_pmu x86_pmu __read_mostly;
50static struct pmu pmu;
51
52DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
53 .enabled = 1,
54 .pmu = &pmu,
55};
56
57DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key);
58DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
59DEFINE_STATIC_KEY_FALSE(perf_is_hybrid);
60
61/*
62 * This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined
63 * from just a typename, as opposed to an actual function.
64 */
65DEFINE_STATIC_CALL_NULL(x86_pmu_handle_irq, *x86_pmu.handle_irq);
66DEFINE_STATIC_CALL_NULL(x86_pmu_disable_all, *x86_pmu.disable_all);
67DEFINE_STATIC_CALL_NULL(x86_pmu_enable_all, *x86_pmu.enable_all);
68DEFINE_STATIC_CALL_NULL(x86_pmu_enable, *x86_pmu.enable);
69DEFINE_STATIC_CALL_NULL(x86_pmu_disable, *x86_pmu.disable);
70
71DEFINE_STATIC_CALL_NULL(x86_pmu_assign, *x86_pmu.assign);
72
73DEFINE_STATIC_CALL_NULL(x86_pmu_add, *x86_pmu.add);
74DEFINE_STATIC_CALL_NULL(x86_pmu_del, *x86_pmu.del);
75DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read);
76
77DEFINE_STATIC_CALL_NULL(x86_pmu_set_period, *x86_pmu.set_period);
78DEFINE_STATIC_CALL_NULL(x86_pmu_update, *x86_pmu.update);
79DEFINE_STATIC_CALL_NULL(x86_pmu_limit_period, *x86_pmu.limit_period);
80
81DEFINE_STATIC_CALL_NULL(x86_pmu_schedule_events, *x86_pmu.schedule_events);
82DEFINE_STATIC_CALL_NULL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints);
83DEFINE_STATIC_CALL_NULL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints);
84
85DEFINE_STATIC_CALL_NULL(x86_pmu_start_scheduling, *x86_pmu.start_scheduling);
86DEFINE_STATIC_CALL_NULL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling);
87DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling);
88
89DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task, *x86_pmu.sched_task);
90DEFINE_STATIC_CALL_NULL(x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx);
91
92DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs);
93DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);
94
95DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter);
96
97/*
98 * This one is magic, it will get called even when PMU init fails (because
99 * there is no PMU), in which case it should simply return NULL.
100 */
101DEFINE_STATIC_CALL_RET0(x86_pmu_guest_get_msrs, *x86_pmu.guest_get_msrs);
102
103u64 __read_mostly hw_cache_event_ids
104 [PERF_COUNT_HW_CACHE_MAX]
105 [PERF_COUNT_HW_CACHE_OP_MAX]
106 [PERF_COUNT_HW_CACHE_RESULT_MAX];
107u64 __read_mostly hw_cache_extra_regs
108 [PERF_COUNT_HW_CACHE_MAX]
109 [PERF_COUNT_HW_CACHE_OP_MAX]
110 [PERF_COUNT_HW_CACHE_RESULT_MAX];
111
112/*
113 * Propagate event elapsed time into the generic event.
114 * Can only be executed on the CPU where the event is active.
115 * Returns the delta events processed.
116 */
117u64 x86_perf_event_update(struct perf_event *event)
118{
119 struct hw_perf_event *hwc = &event->hw;
120 int shift = 64 - x86_pmu.cntval_bits;
121 u64 prev_raw_count, new_raw_count;
122 u64 delta;
123
124 if (unlikely(!hwc->event_base))
125 return 0;
126
127 /*
128 * Careful: an NMI might modify the previous event value.
129 *
130 * Our tactic to handle this is to first atomically read and
131 * exchange a new raw count - then add that new-prev delta
132 * count to the generic event atomically:
133 */
134 prev_raw_count = local64_read(&hwc->prev_count);
135 do {
136 rdpmcl(hwc->event_base_rdpmc, new_raw_count);
137 } while (!local64_try_cmpxchg(&hwc->prev_count,
138 &prev_raw_count, new_raw_count));
139
140 /*
141 * Now we have the new raw value and have updated the prev
142 * timestamp already. We can now calculate the elapsed delta
143 * (event-)time and add that to the generic event.
144 *
145 * Careful, not all hw sign-extends above the physical width
146 * of the count.
147 */
148 delta = (new_raw_count << shift) - (prev_raw_count << shift);
149 delta >>= shift;
150
151 local64_add(delta, &event->count);
152 local64_sub(delta, &hwc->period_left);
153
154 return new_raw_count;
155}
156
157/*
158 * Find and validate any extra registers to set up.
159 */
160static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
161{
162 struct extra_reg *extra_regs = hybrid(event->pmu, extra_regs);
163 struct hw_perf_event_extra *reg;
164 struct extra_reg *er;
165
166 reg = &event->hw.extra_reg;
167
168 if (!extra_regs)
169 return 0;
170
171 for (er = extra_regs; er->msr; er++) {
172 if (er->event != (config & er->config_mask))
173 continue;
174 if (event->attr.config1 & ~er->valid_mask)
175 return -EINVAL;
176 /* Check if the extra msrs can be safely accessed*/
177 if (!er->extra_msr_access)
178 return -ENXIO;
179
180 reg->idx = er->idx;
181 reg->config = event->attr.config1;
182 reg->reg = er->msr;
183 break;
184 }
185 return 0;
186}
187
188static atomic_t active_events;
189static atomic_t pmc_refcount;
190static DEFINE_MUTEX(pmc_reserve_mutex);
191
192#ifdef CONFIG_X86_LOCAL_APIC
193
194static inline u64 get_possible_counter_mask(void)
195{
196 u64 cntr_mask = x86_pmu.cntr_mask64;
197 int i;
198
199 if (!is_hybrid())
200 return cntr_mask;
201
202 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++)
203 cntr_mask |= x86_pmu.hybrid_pmu[i].cntr_mask64;
204
205 return cntr_mask;
206}
207
208static bool reserve_pmc_hardware(void)
209{
210 u64 cntr_mask = get_possible_counter_mask();
211 int i, end;
212
213 for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) {
214 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
215 goto perfctr_fail;
216 }
217
218 for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) {
219 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
220 goto eventsel_fail;
221 }
222
223 return true;
224
225eventsel_fail:
226 end = i;
227 for_each_set_bit(i, (unsigned long *)&cntr_mask, end)
228 release_evntsel_nmi(x86_pmu_config_addr(i));
229 i = X86_PMC_IDX_MAX;
230
231perfctr_fail:
232 end = i;
233 for_each_set_bit(i, (unsigned long *)&cntr_mask, end)
234 release_perfctr_nmi(x86_pmu_event_addr(i));
235
236 return false;
237}
238
239static void release_pmc_hardware(void)
240{
241 u64 cntr_mask = get_possible_counter_mask();
242 int i;
243
244 for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) {
245 release_perfctr_nmi(x86_pmu_event_addr(i));
246 release_evntsel_nmi(x86_pmu_config_addr(i));
247 }
248}
249
250#else
251
252static bool reserve_pmc_hardware(void) { return true; }
253static void release_pmc_hardware(void) {}
254
255#endif
256
257bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask,
258 unsigned long *fixed_cntr_mask)
259{
260 u64 val, val_fail = -1, val_new= ~0;
261 int i, reg, reg_fail = -1, ret = 0;
262 int bios_fail = 0;
263 int reg_safe = -1;
264
265 /*
266 * Check to see if the BIOS enabled any of the counters, if so
267 * complain and bail.
268 */
269 for_each_set_bit(i, cntr_mask, X86_PMC_IDX_MAX) {
270 reg = x86_pmu_config_addr(i);
271 ret = rdmsrl_safe(reg, &val);
272 if (ret)
273 goto msr_fail;
274 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
275 bios_fail = 1;
276 val_fail = val;
277 reg_fail = reg;
278 } else {
279 reg_safe = i;
280 }
281 }
282
283 if (*(u64 *)fixed_cntr_mask) {
284 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
285 ret = rdmsrl_safe(reg, &val);
286 if (ret)
287 goto msr_fail;
288 for_each_set_bit(i, fixed_cntr_mask, X86_PMC_IDX_MAX) {
289 if (fixed_counter_disabled(i, pmu))
290 continue;
291 if (val & (0x03ULL << i*4)) {
292 bios_fail = 1;
293 val_fail = val;
294 reg_fail = reg;
295 }
296 }
297 }
298
299 /*
300 * If all the counters are enabled, the below test will always
301 * fail. The tools will also become useless in this scenario.
302 * Just fail and disable the hardware counters.
303 */
304
305 if (reg_safe == -1) {
306 reg = reg_safe;
307 goto msr_fail;
308 }
309
310 /*
311 * Read the current value, change it and read it back to see if it
312 * matches, this is needed to detect certain hardware emulators
313 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
314 */
315 reg = x86_pmu_event_addr(reg_safe);
316 if (rdmsrl_safe(reg, &val))
317 goto msr_fail;
318 val ^= 0xffffUL;
319 ret = wrmsrl_safe(reg, val);
320 ret |= rdmsrl_safe(reg, &val_new);
321 if (ret || val != val_new)
322 goto msr_fail;
323
324 /*
325 * We still allow the PMU driver to operate:
326 */
327 if (bios_fail) {
328 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
329 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
330 reg_fail, val_fail);
331 }
332
333 return true;
334
335msr_fail:
336 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
337 pr_cont("PMU not available due to virtualization, using software events only.\n");
338 } else {
339 pr_cont("Broken PMU hardware detected, using software events only.\n");
340 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n",
341 reg, val_new);
342 }
343
344 return false;
345}
346
347static void hw_perf_event_destroy(struct perf_event *event)
348{
349 x86_release_hardware();
350 atomic_dec(&active_events);
351}
352
353void hw_perf_lbr_event_destroy(struct perf_event *event)
354{
355 hw_perf_event_destroy(event);
356
357 /* undo the lbr/bts event accounting */
358 x86_del_exclusive(x86_lbr_exclusive_lbr);
359}
360
361static inline int x86_pmu_initialized(void)
362{
363 return x86_pmu.handle_irq != NULL;
364}
365
366static inline int
367set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
368{
369 struct perf_event_attr *attr = &event->attr;
370 unsigned int cache_type, cache_op, cache_result;
371 u64 config, val;
372
373 config = attr->config;
374
375 cache_type = (config >> 0) & 0xff;
376 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
377 return -EINVAL;
378 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX);
379
380 cache_op = (config >> 8) & 0xff;
381 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
382 return -EINVAL;
383 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX);
384
385 cache_result = (config >> 16) & 0xff;
386 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
387 return -EINVAL;
388 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX);
389
390 val = hybrid_var(event->pmu, hw_cache_event_ids)[cache_type][cache_op][cache_result];
391 if (val == 0)
392 return -ENOENT;
393
394 if (val == -1)
395 return -EINVAL;
396
397 hwc->config |= val;
398 attr->config1 = hybrid_var(event->pmu, hw_cache_extra_regs)[cache_type][cache_op][cache_result];
399 return x86_pmu_extra_regs(val, event);
400}
401
402int x86_reserve_hardware(void)
403{
404 int err = 0;
405
406 if (!atomic_inc_not_zero(&pmc_refcount)) {
407 mutex_lock(&pmc_reserve_mutex);
408 if (atomic_read(&pmc_refcount) == 0) {
409 if (!reserve_pmc_hardware()) {
410 err = -EBUSY;
411 } else {
412 reserve_ds_buffers();
413 reserve_lbr_buffers();
414 }
415 }
416 if (!err)
417 atomic_inc(&pmc_refcount);
418 mutex_unlock(&pmc_reserve_mutex);
419 }
420
421 return err;
422}
423
424void x86_release_hardware(void)
425{
426 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
427 release_pmc_hardware();
428 release_ds_buffers();
429 release_lbr_buffers();
430 mutex_unlock(&pmc_reserve_mutex);
431 }
432}
433
434/*
435 * Check if we can create event of a certain type (that no conflicting events
436 * are present).
437 */
438int x86_add_exclusive(unsigned int what)
439{
440 int i;
441
442 /*
443 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS.
444 * LBR and BTS are still mutually exclusive.
445 */
446 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
447 goto out;
448
449 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
450 mutex_lock(&pmc_reserve_mutex);
451 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
452 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
453 goto fail_unlock;
454 }
455 atomic_inc(&x86_pmu.lbr_exclusive[what]);
456 mutex_unlock(&pmc_reserve_mutex);
457 }
458
459out:
460 atomic_inc(&active_events);
461 return 0;
462
463fail_unlock:
464 mutex_unlock(&pmc_reserve_mutex);
465 return -EBUSY;
466}
467
468void x86_del_exclusive(unsigned int what)
469{
470 atomic_dec(&active_events);
471
472 /*
473 * See the comment in x86_add_exclusive().
474 */
475 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
476 return;
477
478 atomic_dec(&x86_pmu.lbr_exclusive[what]);
479}
480
481int x86_setup_perfctr(struct perf_event *event)
482{
483 struct perf_event_attr *attr = &event->attr;
484 struct hw_perf_event *hwc = &event->hw;
485 u64 config;
486
487 if (!is_sampling_event(event)) {
488 hwc->sample_period = x86_pmu.max_period;
489 hwc->last_period = hwc->sample_period;
490 local64_set(&hwc->period_left, hwc->sample_period);
491 }
492
493 if (attr->type == event->pmu->type)
494 return x86_pmu_extra_regs(event->attr.config, event);
495
496 if (attr->type == PERF_TYPE_HW_CACHE)
497 return set_ext_hw_attr(hwc, event);
498
499 if (attr->config >= x86_pmu.max_events)
500 return -EINVAL;
501
502 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events);
503
504 /*
505 * The generic map:
506 */
507 config = x86_pmu.event_map(attr->config);
508
509 if (config == 0)
510 return -ENOENT;
511
512 if (config == -1LL)
513 return -EINVAL;
514
515 hwc->config |= config;
516
517 return 0;
518}
519
520/*
521 * check that branch_sample_type is compatible with
522 * settings needed for precise_ip > 1 which implies
523 * using the LBR to capture ALL taken branches at the
524 * priv levels of the measurement
525 */
526static inline int precise_br_compat(struct perf_event *event)
527{
528 u64 m = event->attr.branch_sample_type;
529 u64 b = 0;
530
531 /* must capture all branches */
532 if (!(m & PERF_SAMPLE_BRANCH_ANY))
533 return 0;
534
535 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
536
537 if (!event->attr.exclude_user)
538 b |= PERF_SAMPLE_BRANCH_USER;
539
540 if (!event->attr.exclude_kernel)
541 b |= PERF_SAMPLE_BRANCH_KERNEL;
542
543 /*
544 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
545 */
546
547 return m == b;
548}
549
550int x86_pmu_max_precise(void)
551{
552 int precise = 0;
553
554 /* Support for constant skid */
555 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
556 precise++;
557
558 /* Support for IP fixup */
559 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
560 precise++;
561
562 if (x86_pmu.pebs_prec_dist)
563 precise++;
564 }
565 return precise;
566}
567
568int x86_pmu_hw_config(struct perf_event *event)
569{
570 if (event->attr.precise_ip) {
571 int precise = x86_pmu_max_precise();
572
573 if (event->attr.precise_ip > precise)
574 return -EOPNOTSUPP;
575
576 /* There's no sense in having PEBS for non sampling events: */
577 if (!is_sampling_event(event))
578 return -EINVAL;
579 }
580 /*
581 * check that PEBS LBR correction does not conflict with
582 * whatever the user is asking with attr->branch_sample_type
583 */
584 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
585 u64 *br_type = &event->attr.branch_sample_type;
586
587 if (has_branch_stack(event)) {
588 if (!precise_br_compat(event))
589 return -EOPNOTSUPP;
590
591 /* branch_sample_type is compatible */
592
593 } else {
594 /*
595 * user did not specify branch_sample_type
596 *
597 * For PEBS fixups, we capture all
598 * the branches at the priv level of the
599 * event.
600 */
601 *br_type = PERF_SAMPLE_BRANCH_ANY;
602
603 if (!event->attr.exclude_user)
604 *br_type |= PERF_SAMPLE_BRANCH_USER;
605
606 if (!event->attr.exclude_kernel)
607 *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
608 }
609 }
610
611 if (branch_sample_call_stack(event))
612 event->attach_state |= PERF_ATTACH_TASK_DATA;
613
614 /*
615 * Generate PMC IRQs:
616 * (keep 'enabled' bit clear for now)
617 */
618 event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
619
620 /*
621 * Count user and OS events unless requested not to
622 */
623 if (!event->attr.exclude_user)
624 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
625 if (!event->attr.exclude_kernel)
626 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
627
628 if (event->attr.type == event->pmu->type)
629 event->hw.config |= x86_pmu_get_event_config(event);
630
631 if (!event->attr.freq && x86_pmu.limit_period) {
632 s64 left = event->attr.sample_period;
633 x86_pmu.limit_period(event, &left);
634 if (left > event->attr.sample_period)
635 return -EINVAL;
636 }
637
638 /* sample_regs_user never support XMM registers */
639 if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK))
640 return -EINVAL;
641 /*
642 * Besides the general purpose registers, XMM registers may
643 * be collected in PEBS on some platforms, e.g. Icelake
644 */
645 if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) {
646 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS))
647 return -EINVAL;
648
649 if (!event->attr.precise_ip)
650 return -EINVAL;
651 }
652
653 return x86_setup_perfctr(event);
654}
655
656/*
657 * Setup the hardware configuration for a given attr_type
658 */
659static int __x86_pmu_event_init(struct perf_event *event)
660{
661 int err;
662
663 if (!x86_pmu_initialized())
664 return -ENODEV;
665
666 err = x86_reserve_hardware();
667 if (err)
668 return err;
669
670 atomic_inc(&active_events);
671 event->destroy = hw_perf_event_destroy;
672
673 event->hw.idx = -1;
674 event->hw.last_cpu = -1;
675 event->hw.last_tag = ~0ULL;
676
677 /* mark unused */
678 event->hw.extra_reg.idx = EXTRA_REG_NONE;
679 event->hw.branch_reg.idx = EXTRA_REG_NONE;
680
681 return x86_pmu.hw_config(event);
682}
683
684void x86_pmu_disable_all(void)
685{
686 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
687 int idx;
688
689 for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {
690 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
691 u64 val;
692
693 if (!test_bit(idx, cpuc->active_mask))
694 continue;
695 rdmsrl(x86_pmu_config_addr(idx), val);
696 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
697 continue;
698 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
699 wrmsrl(x86_pmu_config_addr(idx), val);
700 if (is_counter_pair(hwc))
701 wrmsrl(x86_pmu_config_addr(idx + 1), 0);
702 }
703}
704
705struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data)
706{
707 return static_call(x86_pmu_guest_get_msrs)(nr, data);
708}
709EXPORT_SYMBOL_GPL(perf_guest_get_msrs);
710
711/*
712 * There may be PMI landing after enabled=0. The PMI hitting could be before or
713 * after disable_all.
714 *
715 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler.
716 * It will not be re-enabled in the NMI handler again, because enabled=0. After
717 * handling the NMI, disable_all will be called, which will not change the
718 * state either. If PMI hits after disable_all, the PMU is already disabled
719 * before entering NMI handler. The NMI handler will not change the state
720 * either.
721 *
722 * So either situation is harmless.
723 */
724static void x86_pmu_disable(struct pmu *pmu)
725{
726 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
727
728 if (!x86_pmu_initialized())
729 return;
730
731 if (!cpuc->enabled)
732 return;
733
734 cpuc->n_added = 0;
735 cpuc->enabled = 0;
736 barrier();
737
738 static_call(x86_pmu_disable_all)();
739}
740
741void x86_pmu_enable_all(int added)
742{
743 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
744 int idx;
745
746 for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {
747 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
748
749 if (!test_bit(idx, cpuc->active_mask))
750 continue;
751
752 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
753 }
754}
755
756static inline int is_x86_event(struct perf_event *event)
757{
758 int i;
759
760 if (!is_hybrid())
761 return event->pmu == &pmu;
762
763 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
764 if (event->pmu == &x86_pmu.hybrid_pmu[i].pmu)
765 return true;
766 }
767
768 return false;
769}
770
771struct pmu *x86_get_pmu(unsigned int cpu)
772{
773 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
774
775 /*
776 * All CPUs of the hybrid type have been offline.
777 * The x86_get_pmu() should not be invoked.
778 */
779 if (WARN_ON_ONCE(!cpuc->pmu))
780 return &pmu;
781
782 return cpuc->pmu;
783}
784/*
785 * Event scheduler state:
786 *
787 * Assign events iterating over all events and counters, beginning
788 * with events with least weights first. Keep the current iterator
789 * state in struct sched_state.
790 */
791struct sched_state {
792 int weight;
793 int event; /* event index */
794 int counter; /* counter index */
795 int unassigned; /* number of events to be assigned left */
796 int nr_gp; /* number of GP counters used */
797 u64 used;
798};
799
800/* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
801#define SCHED_STATES_MAX 2
802
803struct perf_sched {
804 int max_weight;
805 int max_events;
806 int max_gp;
807 int saved_states;
808 struct event_constraint **constraints;
809 struct sched_state state;
810 struct sched_state saved[SCHED_STATES_MAX];
811};
812
813/*
814 * Initialize iterator that runs through all events and counters.
815 */
816static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
817 int num, int wmin, int wmax, int gpmax)
818{
819 int idx;
820
821 memset(sched, 0, sizeof(*sched));
822 sched->max_events = num;
823 sched->max_weight = wmax;
824 sched->max_gp = gpmax;
825 sched->constraints = constraints;
826
827 for (idx = 0; idx < num; idx++) {
828 if (constraints[idx]->weight == wmin)
829 break;
830 }
831
832 sched->state.event = idx; /* start with min weight */
833 sched->state.weight = wmin;
834 sched->state.unassigned = num;
835}
836
837static void perf_sched_save_state(struct perf_sched *sched)
838{
839 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
840 return;
841
842 sched->saved[sched->saved_states] = sched->state;
843 sched->saved_states++;
844}
845
846static bool perf_sched_restore_state(struct perf_sched *sched)
847{
848 if (!sched->saved_states)
849 return false;
850
851 sched->saved_states--;
852 sched->state = sched->saved[sched->saved_states];
853
854 /* this assignment didn't work out */
855 /* XXX broken vs EVENT_PAIR */
856 sched->state.used &= ~BIT_ULL(sched->state.counter);
857
858 /* try the next one */
859 sched->state.counter++;
860
861 return true;
862}
863
864/*
865 * Select a counter for the current event to schedule. Return true on
866 * success.
867 */
868static bool __perf_sched_find_counter(struct perf_sched *sched)
869{
870 struct event_constraint *c;
871 int idx;
872
873 if (!sched->state.unassigned)
874 return false;
875
876 if (sched->state.event >= sched->max_events)
877 return false;
878
879 c = sched->constraints[sched->state.event];
880 /* Prefer fixed purpose counters */
881 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
882 idx = INTEL_PMC_IDX_FIXED;
883 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
884 u64 mask = BIT_ULL(idx);
885
886 if (sched->state.used & mask)
887 continue;
888
889 sched->state.used |= mask;
890 goto done;
891 }
892 }
893
894 /* Grab the first unused counter starting with idx */
895 idx = sched->state.counter;
896 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
897 u64 mask = BIT_ULL(idx);
898
899 if (c->flags & PERF_X86_EVENT_PAIR)
900 mask |= mask << 1;
901
902 if (sched->state.used & mask)
903 continue;
904
905 if (sched->state.nr_gp++ >= sched->max_gp)
906 return false;
907
908 sched->state.used |= mask;
909 goto done;
910 }
911
912 return false;
913
914done:
915 sched->state.counter = idx;
916
917 if (c->overlap)
918 perf_sched_save_state(sched);
919
920 return true;
921}
922
923static bool perf_sched_find_counter(struct perf_sched *sched)
924{
925 while (!__perf_sched_find_counter(sched)) {
926 if (!perf_sched_restore_state(sched))
927 return false;
928 }
929
930 return true;
931}
932
933/*
934 * Go through all unassigned events and find the next one to schedule.
935 * Take events with the least weight first. Return true on success.
936 */
937static bool perf_sched_next_event(struct perf_sched *sched)
938{
939 struct event_constraint *c;
940
941 if (!sched->state.unassigned || !--sched->state.unassigned)
942 return false;
943
944 do {
945 /* next event */
946 sched->state.event++;
947 if (sched->state.event >= sched->max_events) {
948 /* next weight */
949 sched->state.event = 0;
950 sched->state.weight++;
951 if (sched->state.weight > sched->max_weight)
952 return false;
953 }
954 c = sched->constraints[sched->state.event];
955 } while (c->weight != sched->state.weight);
956
957 sched->state.counter = 0; /* start with first counter */
958
959 return true;
960}
961
962/*
963 * Assign a counter for each event.
964 */
965int perf_assign_events(struct event_constraint **constraints, int n,
966 int wmin, int wmax, int gpmax, int *assign)
967{
968 struct perf_sched sched;
969
970 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
971
972 do {
973 if (!perf_sched_find_counter(&sched))
974 break; /* failed */
975 if (assign)
976 assign[sched.state.event] = sched.state.counter;
977 } while (perf_sched_next_event(&sched));
978
979 return sched.state.unassigned;
980}
981EXPORT_SYMBOL_GPL(perf_assign_events);
982
983int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
984{
985 struct event_constraint *c;
986 struct perf_event *e;
987 int n0, i, wmin, wmax, unsched = 0;
988 struct hw_perf_event *hwc;
989 u64 used_mask = 0;
990
991 /*
992 * Compute the number of events already present; see x86_pmu_add(),
993 * validate_group() and x86_pmu_commit_txn(). For the former two
994 * cpuc->n_events hasn't been updated yet, while for the latter
995 * cpuc->n_txn contains the number of events added in the current
996 * transaction.
997 */
998 n0 = cpuc->n_events;
999 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1000 n0 -= cpuc->n_txn;
1001
1002 static_call_cond(x86_pmu_start_scheduling)(cpuc);
1003
1004 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
1005 c = cpuc->event_constraint[i];
1006
1007 /*
1008 * Previously scheduled events should have a cached constraint,
1009 * while new events should not have one.
1010 */
1011 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0));
1012
1013 /*
1014 * Request constraints for new events; or for those events that
1015 * have a dynamic constraint -- for those the constraint can
1016 * change due to external factors (sibling state, allow_tfa).
1017 */
1018 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) {
1019 c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]);
1020 cpuc->event_constraint[i] = c;
1021 }
1022
1023 wmin = min(wmin, c->weight);
1024 wmax = max(wmax, c->weight);
1025 }
1026
1027 /*
1028 * fastpath, try to reuse previous register
1029 */
1030 for (i = 0; i < n; i++) {
1031 u64 mask;
1032
1033 hwc = &cpuc->event_list[i]->hw;
1034 c = cpuc->event_constraint[i];
1035
1036 /* never assigned */
1037 if (hwc->idx == -1)
1038 break;
1039
1040 /* constraint still honored */
1041 if (!test_bit(hwc->idx, c->idxmsk))
1042 break;
1043
1044 mask = BIT_ULL(hwc->idx);
1045 if (is_counter_pair(hwc))
1046 mask |= mask << 1;
1047
1048 /* not already used */
1049 if (used_mask & mask)
1050 break;
1051
1052 used_mask |= mask;
1053
1054 if (assign)
1055 assign[i] = hwc->idx;
1056 }
1057
1058 /* slow path */
1059 if (i != n) {
1060 int gpmax = x86_pmu_max_num_counters(cpuc->pmu);
1061
1062 /*
1063 * Do not allow scheduling of more than half the available
1064 * generic counters.
1065 *
1066 * This helps avoid counter starvation of sibling thread by
1067 * ensuring at most half the counters cannot be in exclusive
1068 * mode. There is no designated counters for the limits. Any
1069 * N/2 counters can be used. This helps with events with
1070 * specific counter constraints.
1071 */
1072 if (is_ht_workaround_enabled() && !cpuc->is_fake &&
1073 READ_ONCE(cpuc->excl_cntrs->exclusive_present))
1074 gpmax /= 2;
1075
1076 /*
1077 * Reduce the amount of available counters to allow fitting
1078 * the extra Merge events needed by large increment events.
1079 */
1080 if (x86_pmu.flags & PMU_FL_PAIR) {
1081 gpmax -= cpuc->n_pair;
1082 WARN_ON(gpmax <= 0);
1083 }
1084
1085 unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
1086 wmax, gpmax, assign);
1087 }
1088
1089 /*
1090 * In case of success (unsched = 0), mark events as committed,
1091 * so we do not put_constraint() in case new events are added
1092 * and fail to be scheduled
1093 *
1094 * We invoke the lower level commit callback to lock the resource
1095 *
1096 * We do not need to do all of this in case we are called to
1097 * validate an event group (assign == NULL)
1098 */
1099 if (!unsched && assign) {
1100 for (i = 0; i < n; i++)
1101 static_call_cond(x86_pmu_commit_scheduling)(cpuc, i, assign[i]);
1102 } else {
1103 for (i = n0; i < n; i++) {
1104 e = cpuc->event_list[i];
1105
1106 /*
1107 * release events that failed scheduling
1108 */
1109 static_call_cond(x86_pmu_put_event_constraints)(cpuc, e);
1110
1111 cpuc->event_constraint[i] = NULL;
1112 }
1113 }
1114
1115 static_call_cond(x86_pmu_stop_scheduling)(cpuc);
1116
1117 return unsched ? -EINVAL : 0;
1118}
1119
1120static int add_nr_metric_event(struct cpu_hw_events *cpuc,
1121 struct perf_event *event)
1122{
1123 if (is_metric_event(event)) {
1124 if (cpuc->n_metric == INTEL_TD_METRIC_NUM)
1125 return -EINVAL;
1126 cpuc->n_metric++;
1127 cpuc->n_txn_metric++;
1128 }
1129
1130 return 0;
1131}
1132
1133static void del_nr_metric_event(struct cpu_hw_events *cpuc,
1134 struct perf_event *event)
1135{
1136 if (is_metric_event(event))
1137 cpuc->n_metric--;
1138}
1139
1140static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event,
1141 int max_count, int n)
1142{
1143 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap);
1144
1145 if (intel_cap.perf_metrics && add_nr_metric_event(cpuc, event))
1146 return -EINVAL;
1147
1148 if (n >= max_count + cpuc->n_metric)
1149 return -EINVAL;
1150
1151 cpuc->event_list[n] = event;
1152 if (is_counter_pair(&event->hw)) {
1153 cpuc->n_pair++;
1154 cpuc->n_txn_pair++;
1155 }
1156
1157 return 0;
1158}
1159
1160/*
1161 * dogrp: true if must collect siblings events (group)
1162 * returns total number of events and error code
1163 */
1164static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
1165{
1166 struct perf_event *event;
1167 int n, max_count;
1168
1169 max_count = x86_pmu_num_counters(cpuc->pmu) + x86_pmu_num_counters_fixed(cpuc->pmu);
1170
1171 /* current number of events already accepted */
1172 n = cpuc->n_events;
1173 if (!cpuc->n_events)
1174 cpuc->pebs_output = 0;
1175
1176 if (!cpuc->is_fake && leader->attr.precise_ip) {
1177 /*
1178 * For PEBS->PT, if !aux_event, the group leader (PT) went
1179 * away, the group was broken down and this singleton event
1180 * can't schedule any more.
1181 */
1182 if (is_pebs_pt(leader) && !leader->aux_event)
1183 return -EINVAL;
1184
1185 /*
1186 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS
1187 */
1188 if (cpuc->pebs_output &&
1189 cpuc->pebs_output != is_pebs_pt(leader) + 1)
1190 return -EINVAL;
1191
1192 cpuc->pebs_output = is_pebs_pt(leader) + 1;
1193 }
1194
1195 if (is_x86_event(leader)) {
1196 if (collect_event(cpuc, leader, max_count, n))
1197 return -EINVAL;
1198 n++;
1199 }
1200
1201 if (!dogrp)
1202 return n;
1203
1204 for_each_sibling_event(event, leader) {
1205 if (!is_x86_event(event) || event->state <= PERF_EVENT_STATE_OFF)
1206 continue;
1207
1208 if (collect_event(cpuc, event, max_count, n))
1209 return -EINVAL;
1210
1211 n++;
1212 }
1213 return n;
1214}
1215
1216static inline void x86_assign_hw_event(struct perf_event *event,
1217 struct cpu_hw_events *cpuc, int i)
1218{
1219 struct hw_perf_event *hwc = &event->hw;
1220 int idx;
1221
1222 idx = hwc->idx = cpuc->assign[i];
1223 hwc->last_cpu = smp_processor_id();
1224 hwc->last_tag = ++cpuc->tags[i];
1225
1226 static_call_cond(x86_pmu_assign)(event, idx);
1227
1228 switch (hwc->idx) {
1229 case INTEL_PMC_IDX_FIXED_BTS:
1230 case INTEL_PMC_IDX_FIXED_VLBR:
1231 hwc->config_base = 0;
1232 hwc->event_base = 0;
1233 break;
1234
1235 case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
1236 /* All the metric events are mapped onto the fixed counter 3. */
1237 idx = INTEL_PMC_IDX_FIXED_SLOTS;
1238 fallthrough;
1239 case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1:
1240 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1241 hwc->event_base = x86_pmu_fixed_ctr_addr(idx - INTEL_PMC_IDX_FIXED);
1242 hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) |
1243 INTEL_PMC_FIXED_RDPMC_BASE;
1244 break;
1245
1246 default:
1247 hwc->config_base = x86_pmu_config_addr(hwc->idx);
1248 hwc->event_base = x86_pmu_event_addr(hwc->idx);
1249 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
1250 break;
1251 }
1252}
1253
1254/**
1255 * x86_perf_rdpmc_index - Return PMC counter used for event
1256 * @event: the perf_event to which the PMC counter was assigned
1257 *
1258 * The counter assigned to this performance event may change if interrupts
1259 * are enabled. This counter should thus never be used while interrupts are
1260 * enabled. Before this function is used to obtain the assigned counter the
1261 * event should be checked for validity using, for example,
1262 * perf_event_read_local(), within the same interrupt disabled section in
1263 * which this counter is planned to be used.
1264 *
1265 * Return: The index of the performance monitoring counter assigned to
1266 * @perf_event.
1267 */
1268int x86_perf_rdpmc_index(struct perf_event *event)
1269{
1270 lockdep_assert_irqs_disabled();
1271
1272 return event->hw.event_base_rdpmc;
1273}
1274
1275static inline int match_prev_assignment(struct hw_perf_event *hwc,
1276 struct cpu_hw_events *cpuc,
1277 int i)
1278{
1279 return hwc->idx == cpuc->assign[i] &&
1280 hwc->last_cpu == smp_processor_id() &&
1281 hwc->last_tag == cpuc->tags[i];
1282}
1283
1284static void x86_pmu_start(struct perf_event *event, int flags);
1285
1286static void x86_pmu_enable(struct pmu *pmu)
1287{
1288 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1289 struct perf_event *event;
1290 struct hw_perf_event *hwc;
1291 int i, added = cpuc->n_added;
1292
1293 if (!x86_pmu_initialized())
1294 return;
1295
1296 if (cpuc->enabled)
1297 return;
1298
1299 if (cpuc->n_added) {
1300 int n_running = cpuc->n_events - cpuc->n_added;
1301 /*
1302 * apply assignment obtained either from
1303 * hw_perf_group_sched_in() or x86_pmu_enable()
1304 *
1305 * step1: save events moving to new counters
1306 */
1307 for (i = 0; i < n_running; i++) {
1308 event = cpuc->event_list[i];
1309 hwc = &event->hw;
1310
1311 /*
1312 * we can avoid reprogramming counter if:
1313 * - assigned same counter as last time
1314 * - running on same CPU as last time
1315 * - no other event has used the counter since
1316 */
1317 if (hwc->idx == -1 ||
1318 match_prev_assignment(hwc, cpuc, i))
1319 continue;
1320
1321 /*
1322 * Ensure we don't accidentally enable a stopped
1323 * counter simply because we rescheduled.
1324 */
1325 if (hwc->state & PERF_HES_STOPPED)
1326 hwc->state |= PERF_HES_ARCH;
1327
1328 x86_pmu_stop(event, PERF_EF_UPDATE);
1329 }
1330
1331 /*
1332 * step2: reprogram moved events into new counters
1333 */
1334 for (i = 0; i < cpuc->n_events; i++) {
1335 event = cpuc->event_list[i];
1336 hwc = &event->hw;
1337
1338 if (!match_prev_assignment(hwc, cpuc, i))
1339 x86_assign_hw_event(event, cpuc, i);
1340 else if (i < n_running)
1341 continue;
1342
1343 if (hwc->state & PERF_HES_ARCH)
1344 continue;
1345
1346 /*
1347 * if cpuc->enabled = 0, then no wrmsr as
1348 * per x86_pmu_enable_event()
1349 */
1350 x86_pmu_start(event, PERF_EF_RELOAD);
1351 }
1352 cpuc->n_added = 0;
1353 perf_events_lapic_init();
1354 }
1355
1356 cpuc->enabled = 1;
1357 barrier();
1358
1359 static_call(x86_pmu_enable_all)(added);
1360}
1361
1362DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1363
1364/*
1365 * Set the next IRQ period, based on the hwc->period_left value.
1366 * To be called with the event disabled in hw:
1367 */
1368int x86_perf_event_set_period(struct perf_event *event)
1369{
1370 struct hw_perf_event *hwc = &event->hw;
1371 s64 left = local64_read(&hwc->period_left);
1372 s64 period = hwc->sample_period;
1373 int ret = 0, idx = hwc->idx;
1374
1375 if (unlikely(!hwc->event_base))
1376 return 0;
1377
1378 /*
1379 * If we are way outside a reasonable range then just skip forward:
1380 */
1381 if (unlikely(left <= -period)) {
1382 left = period;
1383 local64_set(&hwc->period_left, left);
1384 hwc->last_period = period;
1385 ret = 1;
1386 }
1387
1388 if (unlikely(left <= 0)) {
1389 left += period;
1390 local64_set(&hwc->period_left, left);
1391 hwc->last_period = period;
1392 ret = 1;
1393 }
1394 /*
1395 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1396 */
1397 if (unlikely(left < 2))
1398 left = 2;
1399
1400 if (left > x86_pmu.max_period)
1401 left = x86_pmu.max_period;
1402
1403 static_call_cond(x86_pmu_limit_period)(event, &left);
1404
1405 this_cpu_write(pmc_prev_left[idx], left);
1406
1407 /*
1408 * The hw event starts counting from this event offset,
1409 * mark it to be able to extra future deltas:
1410 */
1411 local64_set(&hwc->prev_count, (u64)-left);
1412
1413 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
1414
1415 /*
1416 * Sign extend the Merge event counter's upper 16 bits since
1417 * we currently declare a 48-bit counter width
1418 */
1419 if (is_counter_pair(hwc))
1420 wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff);
1421
1422 perf_event_update_userpage(event);
1423
1424 return ret;
1425}
1426
1427void x86_pmu_enable_event(struct perf_event *event)
1428{
1429 if (__this_cpu_read(cpu_hw_events.enabled))
1430 __x86_pmu_enable_event(&event->hw,
1431 ARCH_PERFMON_EVENTSEL_ENABLE);
1432}
1433
1434/*
1435 * Add a single event to the PMU.
1436 *
1437 * The event is added to the group of enabled events
1438 * but only if it can be scheduled with existing events.
1439 */
1440static int x86_pmu_add(struct perf_event *event, int flags)
1441{
1442 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1443 struct hw_perf_event *hwc;
1444 int assign[X86_PMC_IDX_MAX];
1445 int n, n0, ret;
1446
1447 hwc = &event->hw;
1448
1449 n0 = cpuc->n_events;
1450 ret = n = collect_events(cpuc, event, false);
1451 if (ret < 0)
1452 goto out;
1453
1454 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1455 if (!(flags & PERF_EF_START))
1456 hwc->state |= PERF_HES_ARCH;
1457
1458 /*
1459 * If group events scheduling transaction was started,
1460 * skip the schedulability test here, it will be performed
1461 * at commit time (->commit_txn) as a whole.
1462 *
1463 * If commit fails, we'll call ->del() on all events
1464 * for which ->add() was called.
1465 */
1466 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1467 goto done_collect;
1468
1469 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
1470 if (ret)
1471 goto out;
1472 /*
1473 * copy new assignment, now we know it is possible
1474 * will be used by hw_perf_enable()
1475 */
1476 memcpy(cpuc->assign, assign, n*sizeof(int));
1477
1478done_collect:
1479 /*
1480 * Commit the collect_events() state. See x86_pmu_del() and
1481 * x86_pmu_*_txn().
1482 */
1483 cpuc->n_events = n;
1484 cpuc->n_added += n - n0;
1485 cpuc->n_txn += n - n0;
1486
1487 /*
1488 * This is before x86_pmu_enable() will call x86_pmu_start(),
1489 * so we enable LBRs before an event needs them etc..
1490 */
1491 static_call_cond(x86_pmu_add)(event);
1492
1493 ret = 0;
1494out:
1495 return ret;
1496}
1497
1498static void x86_pmu_start(struct perf_event *event, int flags)
1499{
1500 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1501 int idx = event->hw.idx;
1502
1503 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1504 return;
1505
1506 if (WARN_ON_ONCE(idx == -1))
1507 return;
1508
1509 if (flags & PERF_EF_RELOAD) {
1510 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1511 static_call(x86_pmu_set_period)(event);
1512 }
1513
1514 event->hw.state = 0;
1515
1516 cpuc->events[idx] = event;
1517 __set_bit(idx, cpuc->active_mask);
1518 static_call(x86_pmu_enable)(event);
1519 perf_event_update_userpage(event);
1520}
1521
1522void perf_event_print_debug(void)
1523{
1524 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1525 unsigned long *cntr_mask, *fixed_cntr_mask;
1526 struct event_constraint *pebs_constraints;
1527 struct cpu_hw_events *cpuc;
1528 u64 pebs, debugctl;
1529 int cpu, idx;
1530
1531 guard(irqsave)();
1532
1533 cpu = smp_processor_id();
1534 cpuc = &per_cpu(cpu_hw_events, cpu);
1535 cntr_mask = hybrid(cpuc->pmu, cntr_mask);
1536 fixed_cntr_mask = hybrid(cpuc->pmu, fixed_cntr_mask);
1537 pebs_constraints = hybrid(cpuc->pmu, pebs_constraints);
1538
1539 if (!*(u64 *)cntr_mask)
1540 return;
1541
1542 if (x86_pmu.version >= 2) {
1543 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1544 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1545 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1546 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1547
1548 pr_info("\n");
1549 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
1550 pr_info("CPU#%d: status: %016llx\n", cpu, status);
1551 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
1552 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
1553 if (pebs_constraints) {
1554 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1555 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs);
1556 }
1557 if (x86_pmu.lbr_nr) {
1558 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1559 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl);
1560 }
1561 }
1562 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
1563
1564 for_each_set_bit(idx, cntr_mask, X86_PMC_IDX_MAX) {
1565 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
1566 rdmsrl(x86_pmu_event_addr(idx), pmc_count);
1567
1568 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1569
1570 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
1571 cpu, idx, pmc_ctrl);
1572 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
1573 cpu, idx, pmc_count);
1574 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
1575 cpu, idx, prev_left);
1576 }
1577 for_each_set_bit(idx, fixed_cntr_mask, X86_PMC_IDX_MAX) {
1578 if (fixed_counter_disabled(idx, cpuc->pmu))
1579 continue;
1580 rdmsrl(x86_pmu_fixed_ctr_addr(idx), pmc_count);
1581
1582 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1583 cpu, idx, pmc_count);
1584 }
1585}
1586
1587void x86_pmu_stop(struct perf_event *event, int flags)
1588{
1589 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1590 struct hw_perf_event *hwc = &event->hw;
1591
1592 if (test_bit(hwc->idx, cpuc->active_mask)) {
1593 static_call(x86_pmu_disable)(event);
1594 __clear_bit(hwc->idx, cpuc->active_mask);
1595 cpuc->events[hwc->idx] = NULL;
1596 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1597 hwc->state |= PERF_HES_STOPPED;
1598 }
1599
1600 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1601 /*
1602 * Drain the remaining delta count out of a event
1603 * that we are disabling:
1604 */
1605 static_call(x86_pmu_update)(event);
1606 hwc->state |= PERF_HES_UPTODATE;
1607 }
1608}
1609
1610static void x86_pmu_del(struct perf_event *event, int flags)
1611{
1612 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1613 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap);
1614 int i;
1615
1616 /*
1617 * If we're called during a txn, we only need to undo x86_pmu.add.
1618 * The events never got scheduled and ->cancel_txn will truncate
1619 * the event_list.
1620 *
1621 * XXX assumes any ->del() called during a TXN will only be on
1622 * an event added during that same TXN.
1623 */
1624 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1625 goto do_del;
1626
1627 __set_bit(event->hw.idx, cpuc->dirty);
1628
1629 /*
1630 * Not a TXN, therefore cleanup properly.
1631 */
1632 x86_pmu_stop(event, PERF_EF_UPDATE);
1633
1634 for (i = 0; i < cpuc->n_events; i++) {
1635 if (event == cpuc->event_list[i])
1636 break;
1637 }
1638
1639 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
1640 return;
1641
1642 /* If we have a newly added event; make sure to decrease n_added. */
1643 if (i >= cpuc->n_events - cpuc->n_added)
1644 --cpuc->n_added;
1645
1646 static_call_cond(x86_pmu_put_event_constraints)(cpuc, event);
1647
1648 /* Delete the array entry. */
1649 while (++i < cpuc->n_events) {
1650 cpuc->event_list[i-1] = cpuc->event_list[i];
1651 cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
1652 cpuc->assign[i-1] = cpuc->assign[i];
1653 }
1654 cpuc->event_constraint[i-1] = NULL;
1655 --cpuc->n_events;
1656 if (intel_cap.perf_metrics)
1657 del_nr_metric_event(cpuc, event);
1658
1659 perf_event_update_userpage(event);
1660
1661do_del:
1662
1663 /*
1664 * This is after x86_pmu_stop(); so we disable LBRs after any
1665 * event can need them etc..
1666 */
1667 static_call_cond(x86_pmu_del)(event);
1668}
1669
1670int x86_pmu_handle_irq(struct pt_regs *regs)
1671{
1672 struct perf_sample_data data;
1673 struct cpu_hw_events *cpuc;
1674 struct perf_event *event;
1675 int idx, handled = 0;
1676 u64 val;
1677
1678 cpuc = this_cpu_ptr(&cpu_hw_events);
1679
1680 /*
1681 * Some chipsets need to unmask the LVTPC in a particular spot
1682 * inside the nmi handler. As a result, the unmasking was pushed
1683 * into all the nmi handlers.
1684 *
1685 * This generic handler doesn't seem to have any issues where the
1686 * unmasking occurs so it was left at the top.
1687 */
1688 apic_write(APIC_LVTPC, APIC_DM_NMI);
1689
1690 for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) {
1691 if (!test_bit(idx, cpuc->active_mask))
1692 continue;
1693
1694 event = cpuc->events[idx];
1695
1696 val = static_call(x86_pmu_update)(event);
1697 if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1698 continue;
1699
1700 /*
1701 * event overflow
1702 */
1703 handled++;
1704
1705 if (!static_call(x86_pmu_set_period)(event))
1706 continue;
1707
1708 perf_sample_data_init(&data, 0, event->hw.last_period);
1709
1710 if (has_branch_stack(event))
1711 perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
1712
1713 if (perf_event_overflow(event, &data, regs))
1714 x86_pmu_stop(event, 0);
1715 }
1716
1717 if (handled)
1718 inc_irq_stat(apic_perf_irqs);
1719
1720 return handled;
1721}
1722
1723void perf_events_lapic_init(void)
1724{
1725 if (!x86_pmu.apic || !x86_pmu_initialized())
1726 return;
1727
1728 /*
1729 * Always use NMI for PMU
1730 */
1731 apic_write(APIC_LVTPC, APIC_DM_NMI);
1732}
1733
1734static int
1735perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1736{
1737 u64 start_clock;
1738 u64 finish_clock;
1739 int ret;
1740
1741 /*
1742 * All PMUs/events that share this PMI handler should make sure to
1743 * increment active_events for their events.
1744 */
1745 if (!atomic_read(&active_events))
1746 return NMI_DONE;
1747
1748 start_clock = sched_clock();
1749 ret = static_call(x86_pmu_handle_irq)(regs);
1750 finish_clock = sched_clock();
1751
1752 perf_sample_event_took(finish_clock - start_clock);
1753
1754 return ret;
1755}
1756NOKPROBE_SYMBOL(perf_event_nmi_handler);
1757
1758struct event_constraint emptyconstraint;
1759struct event_constraint unconstrained;
1760
1761static int x86_pmu_prepare_cpu(unsigned int cpu)
1762{
1763 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1764 int i;
1765
1766 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
1767 cpuc->kfree_on_online[i] = NULL;
1768 if (x86_pmu.cpu_prepare)
1769 return x86_pmu.cpu_prepare(cpu);
1770 return 0;
1771}
1772
1773static int x86_pmu_dead_cpu(unsigned int cpu)
1774{
1775 if (x86_pmu.cpu_dead)
1776 x86_pmu.cpu_dead(cpu);
1777 return 0;
1778}
1779
1780static int x86_pmu_online_cpu(unsigned int cpu)
1781{
1782 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1783 int i;
1784
1785 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
1786 kfree(cpuc->kfree_on_online[i]);
1787 cpuc->kfree_on_online[i] = NULL;
1788 }
1789 return 0;
1790}
1791
1792static int x86_pmu_starting_cpu(unsigned int cpu)
1793{
1794 if (x86_pmu.cpu_starting)
1795 x86_pmu.cpu_starting(cpu);
1796 return 0;
1797}
1798
1799static int x86_pmu_dying_cpu(unsigned int cpu)
1800{
1801 if (x86_pmu.cpu_dying)
1802 x86_pmu.cpu_dying(cpu);
1803 return 0;
1804}
1805
1806static void __init pmu_check_apic(void)
1807{
1808 if (boot_cpu_has(X86_FEATURE_APIC))
1809 return;
1810
1811 x86_pmu.apic = 0;
1812 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1813 pr_info("no hardware sampling interrupt available.\n");
1814
1815 /*
1816 * If we have a PMU initialized but no APIC
1817 * interrupts, we cannot sample hardware
1818 * events (user-space has to fall back and
1819 * sample via a hrtimer based software event):
1820 */
1821 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1822
1823}
1824
1825static struct attribute_group x86_pmu_format_group __ro_after_init = {
1826 .name = "format",
1827 .attrs = NULL,
1828};
1829
1830ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
1831{
1832 struct perf_pmu_events_attr *pmu_attr =
1833 container_of(attr, struct perf_pmu_events_attr, attr);
1834 u64 config = 0;
1835
1836 if (pmu_attr->id < x86_pmu.max_events)
1837 config = x86_pmu.event_map(pmu_attr->id);
1838
1839 /* string trumps id */
1840 if (pmu_attr->event_str)
1841 return sprintf(page, "%s\n", pmu_attr->event_str);
1842
1843 return x86_pmu.events_sysfs_show(page, config);
1844}
1845EXPORT_SYMBOL_GPL(events_sysfs_show);
1846
1847ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1848 char *page)
1849{
1850 struct perf_pmu_events_ht_attr *pmu_attr =
1851 container_of(attr, struct perf_pmu_events_ht_attr, attr);
1852
1853 /*
1854 * Report conditional events depending on Hyper-Threading.
1855 *
1856 * This is overly conservative as usually the HT special
1857 * handling is not needed if the other CPU thread is idle.
1858 *
1859 * Note this does not (and cannot) handle the case when thread
1860 * siblings are invisible, for example with virtualization
1861 * if they are owned by some other guest. The user tool
1862 * has to re-read when a thread sibling gets onlined later.
1863 */
1864 return sprintf(page, "%s",
1865 topology_max_smt_threads() > 1 ?
1866 pmu_attr->event_str_ht :
1867 pmu_attr->event_str_noht);
1868}
1869
1870ssize_t events_hybrid_sysfs_show(struct device *dev,
1871 struct device_attribute *attr,
1872 char *page)
1873{
1874 struct perf_pmu_events_hybrid_attr *pmu_attr =
1875 container_of(attr, struct perf_pmu_events_hybrid_attr, attr);
1876 struct x86_hybrid_pmu *pmu;
1877 const char *str, *next_str;
1878 int i;
1879
1880 if (hweight64(pmu_attr->pmu_type) == 1)
1881 return sprintf(page, "%s", pmu_attr->event_str);
1882
1883 /*
1884 * Hybrid PMUs may support the same event name, but with different
1885 * event encoding, e.g., the mem-loads event on an Atom PMU has
1886 * different event encoding from a Core PMU.
1887 *
1888 * The event_str includes all event encodings. Each event encoding
1889 * is divided by ";". The order of the event encodings must follow
1890 * the order of the hybrid PMU index.
1891 */
1892 pmu = container_of(dev_get_drvdata(dev), struct x86_hybrid_pmu, pmu);
1893
1894 str = pmu_attr->event_str;
1895 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
1896 if (!(x86_pmu.hybrid_pmu[i].pmu_type & pmu_attr->pmu_type))
1897 continue;
1898 if (x86_pmu.hybrid_pmu[i].pmu_type & pmu->pmu_type) {
1899 next_str = strchr(str, ';');
1900 if (next_str)
1901 return snprintf(page, next_str - str + 1, "%s", str);
1902 else
1903 return sprintf(page, "%s", str);
1904 }
1905 str = strchr(str, ';');
1906 str++;
1907 }
1908
1909 return 0;
1910}
1911EXPORT_SYMBOL_GPL(events_hybrid_sysfs_show);
1912
1913EVENT_ATTR(cpu-cycles, CPU_CYCLES );
1914EVENT_ATTR(instructions, INSTRUCTIONS );
1915EVENT_ATTR(cache-references, CACHE_REFERENCES );
1916EVENT_ATTR(cache-misses, CACHE_MISSES );
1917EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS );
1918EVENT_ATTR(branch-misses, BRANCH_MISSES );
1919EVENT_ATTR(bus-cycles, BUS_CYCLES );
1920EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND );
1921EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND );
1922EVENT_ATTR(ref-cycles, REF_CPU_CYCLES );
1923
1924static struct attribute *empty_attrs;
1925
1926static struct attribute *events_attr[] = {
1927 EVENT_PTR(CPU_CYCLES),
1928 EVENT_PTR(INSTRUCTIONS),
1929 EVENT_PTR(CACHE_REFERENCES),
1930 EVENT_PTR(CACHE_MISSES),
1931 EVENT_PTR(BRANCH_INSTRUCTIONS),
1932 EVENT_PTR(BRANCH_MISSES),
1933 EVENT_PTR(BUS_CYCLES),
1934 EVENT_PTR(STALLED_CYCLES_FRONTEND),
1935 EVENT_PTR(STALLED_CYCLES_BACKEND),
1936 EVENT_PTR(REF_CPU_CYCLES),
1937 NULL,
1938};
1939
1940/*
1941 * Remove all undefined events (x86_pmu.event_map(id) == 0)
1942 * out of events_attr attributes.
1943 */
1944static umode_t
1945is_visible(struct kobject *kobj, struct attribute *attr, int idx)
1946{
1947 struct perf_pmu_events_attr *pmu_attr;
1948
1949 if (idx >= x86_pmu.max_events)
1950 return 0;
1951
1952 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
1953 /* str trumps id */
1954 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0;
1955}
1956
1957static struct attribute_group x86_pmu_events_group __ro_after_init = {
1958 .name = "events",
1959 .attrs = events_attr,
1960 .is_visible = is_visible,
1961};
1962
1963ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
1964{
1965 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
1966 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
1967 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE);
1968 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
1969 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY);
1970 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV);
1971 ssize_t ret;
1972
1973 /*
1974 * We have whole page size to spend and just little data
1975 * to write, so we can safely use sprintf.
1976 */
1977 ret = sprintf(page, "event=0x%02llx", event);
1978
1979 if (umask)
1980 ret += sprintf(page + ret, ",umask=0x%02llx", umask);
1981
1982 if (edge)
1983 ret += sprintf(page + ret, ",edge");
1984
1985 if (pc)
1986 ret += sprintf(page + ret, ",pc");
1987
1988 if (any)
1989 ret += sprintf(page + ret, ",any");
1990
1991 if (inv)
1992 ret += sprintf(page + ret, ",inv");
1993
1994 if (cmask)
1995 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
1996
1997 ret += sprintf(page + ret, "\n");
1998
1999 return ret;
2000}
2001
2002static struct attribute_group x86_pmu_attr_group;
2003static struct attribute_group x86_pmu_caps_group;
2004
2005static void x86_pmu_static_call_update(void)
2006{
2007 static_call_update(x86_pmu_handle_irq, x86_pmu.handle_irq);
2008 static_call_update(x86_pmu_disable_all, x86_pmu.disable_all);
2009 static_call_update(x86_pmu_enable_all, x86_pmu.enable_all);
2010 static_call_update(x86_pmu_enable, x86_pmu.enable);
2011 static_call_update(x86_pmu_disable, x86_pmu.disable);
2012
2013 static_call_update(x86_pmu_assign, x86_pmu.assign);
2014
2015 static_call_update(x86_pmu_add, x86_pmu.add);
2016 static_call_update(x86_pmu_del, x86_pmu.del);
2017 static_call_update(x86_pmu_read, x86_pmu.read);
2018
2019 static_call_update(x86_pmu_set_period, x86_pmu.set_period);
2020 static_call_update(x86_pmu_update, x86_pmu.update);
2021 static_call_update(x86_pmu_limit_period, x86_pmu.limit_period);
2022
2023 static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events);
2024 static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints);
2025 static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints);
2026
2027 static_call_update(x86_pmu_start_scheduling, x86_pmu.start_scheduling);
2028 static_call_update(x86_pmu_commit_scheduling, x86_pmu.commit_scheduling);
2029 static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling);
2030
2031 static_call_update(x86_pmu_sched_task, x86_pmu.sched_task);
2032 static_call_update(x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx);
2033
2034 static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs);
2035 static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases);
2036
2037 static_call_update(x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs);
2038 static_call_update(x86_pmu_filter, x86_pmu.filter);
2039}
2040
2041static void _x86_pmu_read(struct perf_event *event)
2042{
2043 static_call(x86_pmu_update)(event);
2044}
2045
2046void x86_pmu_show_pmu_cap(struct pmu *pmu)
2047{
2048 pr_info("... version: %d\n", x86_pmu.version);
2049 pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
2050 pr_info("... generic registers: %d\n", x86_pmu_num_counters(pmu));
2051 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);
2052 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
2053 pr_info("... fixed-purpose events: %d\n", x86_pmu_num_counters_fixed(pmu));
2054 pr_info("... event mask: %016Lx\n", hybrid(pmu, intel_ctrl));
2055}
2056
2057static int __init init_hw_perf_events(void)
2058{
2059 struct x86_pmu_quirk *quirk;
2060 int err;
2061
2062 pr_info("Performance Events: ");
2063
2064 switch (boot_cpu_data.x86_vendor) {
2065 case X86_VENDOR_INTEL:
2066 err = intel_pmu_init();
2067 break;
2068 case X86_VENDOR_AMD:
2069 err = amd_pmu_init();
2070 break;
2071 case X86_VENDOR_HYGON:
2072 err = amd_pmu_init();
2073 x86_pmu.name = "HYGON";
2074 break;
2075 case X86_VENDOR_ZHAOXIN:
2076 case X86_VENDOR_CENTAUR:
2077 err = zhaoxin_pmu_init();
2078 break;
2079 default:
2080 err = -ENOTSUPP;
2081 }
2082 if (err != 0) {
2083 pr_cont("no PMU driver, software events only.\n");
2084 err = 0;
2085 goto out_bad_pmu;
2086 }
2087
2088 pmu_check_apic();
2089
2090 /* sanity check that the hardware exists or is emulated */
2091 if (!check_hw_exists(&pmu, x86_pmu.cntr_mask, x86_pmu.fixed_cntr_mask))
2092 goto out_bad_pmu;
2093
2094 pr_cont("%s PMU driver.\n", x86_pmu.name);
2095
2096 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
2097
2098 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
2099 quirk->func();
2100
2101 if (!x86_pmu.intel_ctrl)
2102 x86_pmu.intel_ctrl = x86_pmu.cntr_mask64;
2103
2104 if (!x86_pmu.config_mask)
2105 x86_pmu.config_mask = X86_RAW_EVENT_MASK;
2106
2107 perf_events_lapic_init();
2108 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
2109
2110 unconstrained = (struct event_constraint)
2111 __EVENT_CONSTRAINT(0, x86_pmu.cntr_mask64,
2112 0, x86_pmu_num_counters(NULL), 0, 0);
2113
2114 x86_pmu_format_group.attrs = x86_pmu.format_attrs;
2115
2116 if (!x86_pmu.events_sysfs_show)
2117 x86_pmu_events_group.attrs = &empty_attrs;
2118
2119 pmu.attr_update = x86_pmu.attr_update;
2120
2121 if (!is_hybrid())
2122 x86_pmu_show_pmu_cap(NULL);
2123
2124 if (!x86_pmu.read)
2125 x86_pmu.read = _x86_pmu_read;
2126
2127 if (!x86_pmu.guest_get_msrs)
2128 x86_pmu.guest_get_msrs = (void *)&__static_call_return0;
2129
2130 if (!x86_pmu.set_period)
2131 x86_pmu.set_period = x86_perf_event_set_period;
2132
2133 if (!x86_pmu.update)
2134 x86_pmu.update = x86_perf_event_update;
2135
2136 x86_pmu_static_call_update();
2137
2138 /*
2139 * Install callbacks. Core will call them for each online
2140 * cpu.
2141 */
2142 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare",
2143 x86_pmu_prepare_cpu, x86_pmu_dead_cpu);
2144 if (err)
2145 return err;
2146
2147 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING,
2148 "perf/x86:starting", x86_pmu_starting_cpu,
2149 x86_pmu_dying_cpu);
2150 if (err)
2151 goto out;
2152
2153 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online",
2154 x86_pmu_online_cpu, NULL);
2155 if (err)
2156 goto out1;
2157
2158 if (!is_hybrid()) {
2159 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
2160 if (err)
2161 goto out2;
2162 } else {
2163 struct x86_hybrid_pmu *hybrid_pmu;
2164 int i, j;
2165
2166 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
2167 hybrid_pmu = &x86_pmu.hybrid_pmu[i];
2168
2169 hybrid_pmu->pmu = pmu;
2170 hybrid_pmu->pmu.type = -1;
2171 hybrid_pmu->pmu.attr_update = x86_pmu.attr_update;
2172 hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE;
2173
2174 err = perf_pmu_register(&hybrid_pmu->pmu, hybrid_pmu->name,
2175 (hybrid_pmu->pmu_type == hybrid_big) ? PERF_TYPE_RAW : -1);
2176 if (err)
2177 break;
2178 }
2179
2180 if (i < x86_pmu.num_hybrid_pmus) {
2181 for (j = 0; j < i; j++)
2182 perf_pmu_unregister(&x86_pmu.hybrid_pmu[j].pmu);
2183 pr_warn("Failed to register hybrid PMUs\n");
2184 kfree(x86_pmu.hybrid_pmu);
2185 x86_pmu.hybrid_pmu = NULL;
2186 x86_pmu.num_hybrid_pmus = 0;
2187 goto out2;
2188 }
2189 }
2190
2191 return 0;
2192
2193out2:
2194 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE);
2195out1:
2196 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING);
2197out:
2198 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE);
2199out_bad_pmu:
2200 memset(&x86_pmu, 0, sizeof(x86_pmu));
2201 return err;
2202}
2203early_initcall(init_hw_perf_events);
2204
2205static void x86_pmu_read(struct perf_event *event)
2206{
2207 static_call(x86_pmu_read)(event);
2208}
2209
2210/*
2211 * Start group events scheduling transaction
2212 * Set the flag to make pmu::enable() not perform the
2213 * schedulability test, it will be performed at commit time
2214 *
2215 * We only support PERF_PMU_TXN_ADD transactions. Save the
2216 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
2217 * transactions.
2218 */
2219static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
2220{
2221 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2222
2223 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */
2224
2225 cpuc->txn_flags = txn_flags;
2226 if (txn_flags & ~PERF_PMU_TXN_ADD)
2227 return;
2228
2229 perf_pmu_disable(pmu);
2230 __this_cpu_write(cpu_hw_events.n_txn, 0);
2231 __this_cpu_write(cpu_hw_events.n_txn_pair, 0);
2232 __this_cpu_write(cpu_hw_events.n_txn_metric, 0);
2233}
2234
2235/*
2236 * Stop group events scheduling transaction
2237 * Clear the flag and pmu::enable() will perform the
2238 * schedulability test.
2239 */
2240static void x86_pmu_cancel_txn(struct pmu *pmu)
2241{
2242 unsigned int txn_flags;
2243 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2244
2245 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
2246
2247 txn_flags = cpuc->txn_flags;
2248 cpuc->txn_flags = 0;
2249 if (txn_flags & ~PERF_PMU_TXN_ADD)
2250 return;
2251
2252 /*
2253 * Truncate collected array by the number of events added in this
2254 * transaction. See x86_pmu_add() and x86_pmu_*_txn().
2255 */
2256 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
2257 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
2258 __this_cpu_sub(cpu_hw_events.n_pair, __this_cpu_read(cpu_hw_events.n_txn_pair));
2259 __this_cpu_sub(cpu_hw_events.n_metric, __this_cpu_read(cpu_hw_events.n_txn_metric));
2260 perf_pmu_enable(pmu);
2261}
2262
2263/*
2264 * Commit group events scheduling transaction
2265 * Perform the group schedulability test as a whole
2266 * Return 0 if success
2267 *
2268 * Does not cancel the transaction on failure; expects the caller to do this.
2269 */
2270static int x86_pmu_commit_txn(struct pmu *pmu)
2271{
2272 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2273 int assign[X86_PMC_IDX_MAX];
2274 int n, ret;
2275
2276 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
2277
2278 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
2279 cpuc->txn_flags = 0;
2280 return 0;
2281 }
2282
2283 n = cpuc->n_events;
2284
2285 if (!x86_pmu_initialized())
2286 return -EAGAIN;
2287
2288 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
2289 if (ret)
2290 return ret;
2291
2292 /*
2293 * copy new assignment, now we know it is possible
2294 * will be used by hw_perf_enable()
2295 */
2296 memcpy(cpuc->assign, assign, n*sizeof(int));
2297
2298 cpuc->txn_flags = 0;
2299 perf_pmu_enable(pmu);
2300 return 0;
2301}
2302/*
2303 * a fake_cpuc is used to validate event groups. Due to
2304 * the extra reg logic, we need to also allocate a fake
2305 * per_core and per_cpu structure. Otherwise, group events
2306 * using extra reg may conflict without the kernel being
2307 * able to catch this when the last event gets added to
2308 * the group.
2309 */
2310static void free_fake_cpuc(struct cpu_hw_events *cpuc)
2311{
2312 intel_cpuc_finish(cpuc);
2313 kfree(cpuc);
2314}
2315
2316static struct cpu_hw_events *allocate_fake_cpuc(struct pmu *event_pmu)
2317{
2318 struct cpu_hw_events *cpuc;
2319 int cpu;
2320
2321 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
2322 if (!cpuc)
2323 return ERR_PTR(-ENOMEM);
2324 cpuc->is_fake = 1;
2325
2326 if (is_hybrid()) {
2327 struct x86_hybrid_pmu *h_pmu;
2328
2329 h_pmu = hybrid_pmu(event_pmu);
2330 if (cpumask_empty(&h_pmu->supported_cpus))
2331 goto error;
2332 cpu = cpumask_first(&h_pmu->supported_cpus);
2333 } else
2334 cpu = raw_smp_processor_id();
2335 cpuc->pmu = event_pmu;
2336
2337 if (intel_cpuc_prepare(cpuc, cpu))
2338 goto error;
2339
2340 return cpuc;
2341error:
2342 free_fake_cpuc(cpuc);
2343 return ERR_PTR(-ENOMEM);
2344}
2345
2346/*
2347 * validate that we can schedule this event
2348 */
2349static int validate_event(struct perf_event *event)
2350{
2351 struct cpu_hw_events *fake_cpuc;
2352 struct event_constraint *c;
2353 int ret = 0;
2354
2355 fake_cpuc = allocate_fake_cpuc(event->pmu);
2356 if (IS_ERR(fake_cpuc))
2357 return PTR_ERR(fake_cpuc);
2358
2359 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event);
2360
2361 if (!c || !c->weight)
2362 ret = -EINVAL;
2363
2364 if (x86_pmu.put_event_constraints)
2365 x86_pmu.put_event_constraints(fake_cpuc, event);
2366
2367 free_fake_cpuc(fake_cpuc);
2368
2369 return ret;
2370}
2371
2372/*
2373 * validate a single event group
2374 *
2375 * validation include:
2376 * - check events are compatible which each other
2377 * - events do not compete for the same counter
2378 * - number of events <= number of counters
2379 *
2380 * validation ensures the group can be loaded onto the
2381 * PMU if it was the only group available.
2382 */
2383static int validate_group(struct perf_event *event)
2384{
2385 struct perf_event *leader = event->group_leader;
2386 struct cpu_hw_events *fake_cpuc;
2387 int ret = -EINVAL, n;
2388
2389 /*
2390 * Reject events from different hybrid PMUs.
2391 */
2392 if (is_hybrid()) {
2393 struct perf_event *sibling;
2394 struct pmu *pmu = NULL;
2395
2396 if (is_x86_event(leader))
2397 pmu = leader->pmu;
2398
2399 for_each_sibling_event(sibling, leader) {
2400 if (!is_x86_event(sibling))
2401 continue;
2402 if (!pmu)
2403 pmu = sibling->pmu;
2404 else if (pmu != sibling->pmu)
2405 return ret;
2406 }
2407 }
2408
2409 fake_cpuc = allocate_fake_cpuc(event->pmu);
2410 if (IS_ERR(fake_cpuc))
2411 return PTR_ERR(fake_cpuc);
2412 /*
2413 * the event is not yet connected with its
2414 * siblings therefore we must first collect
2415 * existing siblings, then add the new event
2416 * before we can simulate the scheduling
2417 */
2418 n = collect_events(fake_cpuc, leader, true);
2419 if (n < 0)
2420 goto out;
2421
2422 fake_cpuc->n_events = n;
2423 n = collect_events(fake_cpuc, event, false);
2424 if (n < 0)
2425 goto out;
2426
2427 fake_cpuc->n_events = 0;
2428 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
2429
2430out:
2431 free_fake_cpuc(fake_cpuc);
2432 return ret;
2433}
2434
2435static int x86_pmu_event_init(struct perf_event *event)
2436{
2437 struct x86_hybrid_pmu *pmu = NULL;
2438 int err;
2439
2440 if ((event->attr.type != event->pmu->type) &&
2441 (event->attr.type != PERF_TYPE_HARDWARE) &&
2442 (event->attr.type != PERF_TYPE_HW_CACHE))
2443 return -ENOENT;
2444
2445 if (is_hybrid() && (event->cpu != -1)) {
2446 pmu = hybrid_pmu(event->pmu);
2447 if (!cpumask_test_cpu(event->cpu, &pmu->supported_cpus))
2448 return -ENOENT;
2449 }
2450
2451 err = __x86_pmu_event_init(event);
2452 if (!err) {
2453 if (event->group_leader != event)
2454 err = validate_group(event);
2455 else
2456 err = validate_event(event);
2457 }
2458 if (err) {
2459 if (event->destroy)
2460 event->destroy(event);
2461 event->destroy = NULL;
2462 }
2463
2464 if (READ_ONCE(x86_pmu.attr_rdpmc) &&
2465 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
2466 event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT;
2467
2468 return err;
2469}
2470
2471void perf_clear_dirty_counters(void)
2472{
2473 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2474 int i;
2475
2476 /* Don't need to clear the assigned counter. */
2477 for (i = 0; i < cpuc->n_events; i++)
2478 __clear_bit(cpuc->assign[i], cpuc->dirty);
2479
2480 if (bitmap_empty(cpuc->dirty, X86_PMC_IDX_MAX))
2481 return;
2482
2483 for_each_set_bit(i, cpuc->dirty, X86_PMC_IDX_MAX) {
2484 if (i >= INTEL_PMC_IDX_FIXED) {
2485 /* Metrics and fake events don't have corresponding HW counters. */
2486 if (!test_bit(i - INTEL_PMC_IDX_FIXED, hybrid(cpuc->pmu, fixed_cntr_mask)))
2487 continue;
2488
2489 wrmsrl(x86_pmu_fixed_ctr_addr(i - INTEL_PMC_IDX_FIXED), 0);
2490 } else {
2491 wrmsrl(x86_pmu_event_addr(i), 0);
2492 }
2493 }
2494
2495 bitmap_zero(cpuc->dirty, X86_PMC_IDX_MAX);
2496}
2497
2498static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
2499{
2500 if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT))
2501 return;
2502
2503 /*
2504 * This function relies on not being called concurrently in two
2505 * tasks in the same mm. Otherwise one task could observe
2506 * perf_rdpmc_allowed > 1 and return all the way back to
2507 * userspace with CR4.PCE clear while another task is still
2508 * doing on_each_cpu_mask() to propagate CR4.PCE.
2509 *
2510 * For now, this can't happen because all callers hold mmap_lock
2511 * for write. If this changes, we'll need a different solution.
2512 */
2513 mmap_assert_write_locked(mm);
2514
2515 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
2516 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
2517}
2518
2519static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
2520{
2521 if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT))
2522 return;
2523
2524 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
2525 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
2526}
2527
2528static int x86_pmu_event_idx(struct perf_event *event)
2529{
2530 struct hw_perf_event *hwc = &event->hw;
2531
2532 if (!(hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT))
2533 return 0;
2534
2535 if (is_metric_idx(hwc->idx))
2536 return INTEL_PMC_FIXED_RDPMC_METRICS + 1;
2537 else
2538 return hwc->event_base_rdpmc + 1;
2539}
2540
2541static ssize_t get_attr_rdpmc(struct device *cdev,
2542 struct device_attribute *attr,
2543 char *buf)
2544{
2545 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
2546}
2547
2548static ssize_t set_attr_rdpmc(struct device *cdev,
2549 struct device_attribute *attr,
2550 const char *buf, size_t count)
2551{
2552 static DEFINE_MUTEX(rdpmc_mutex);
2553 unsigned long val;
2554 ssize_t ret;
2555
2556 ret = kstrtoul(buf, 0, &val);
2557 if (ret)
2558 return ret;
2559
2560 if (val > 2)
2561 return -EINVAL;
2562
2563 if (x86_pmu.attr_rdpmc_broken)
2564 return -ENOTSUPP;
2565
2566 guard(mutex)(&rdpmc_mutex);
2567
2568 if (val != x86_pmu.attr_rdpmc) {
2569 /*
2570 * Changing into or out of never available or always available,
2571 * aka perf-event-bypassing mode. This path is extremely slow,
2572 * but only root can trigger it, so it's okay.
2573 */
2574 if (val == 0)
2575 static_branch_inc(&rdpmc_never_available_key);
2576 else if (x86_pmu.attr_rdpmc == 0)
2577 static_branch_dec(&rdpmc_never_available_key);
2578
2579 if (val == 2)
2580 static_branch_inc(&rdpmc_always_available_key);
2581 else if (x86_pmu.attr_rdpmc == 2)
2582 static_branch_dec(&rdpmc_always_available_key);
2583
2584 on_each_cpu(cr4_update_pce, NULL, 1);
2585 x86_pmu.attr_rdpmc = val;
2586 }
2587
2588 return count;
2589}
2590
2591static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
2592
2593static struct attribute *x86_pmu_attrs[] = {
2594 &dev_attr_rdpmc.attr,
2595 NULL,
2596};
2597
2598static struct attribute_group x86_pmu_attr_group __ro_after_init = {
2599 .attrs = x86_pmu_attrs,
2600};
2601
2602static ssize_t max_precise_show(struct device *cdev,
2603 struct device_attribute *attr,
2604 char *buf)
2605{
2606 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
2607}
2608
2609static DEVICE_ATTR_RO(max_precise);
2610
2611static struct attribute *x86_pmu_caps_attrs[] = {
2612 &dev_attr_max_precise.attr,
2613 NULL
2614};
2615
2616static struct attribute_group x86_pmu_caps_group __ro_after_init = {
2617 .name = "caps",
2618 .attrs = x86_pmu_caps_attrs,
2619};
2620
2621static const struct attribute_group *x86_pmu_attr_groups[] = {
2622 &x86_pmu_attr_group,
2623 &x86_pmu_format_group,
2624 &x86_pmu_events_group,
2625 &x86_pmu_caps_group,
2626 NULL,
2627};
2628
2629static void x86_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
2630{
2631 static_call_cond(x86_pmu_sched_task)(pmu_ctx, sched_in);
2632}
2633
2634static void x86_pmu_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
2635 struct perf_event_pmu_context *next_epc)
2636{
2637 static_call_cond(x86_pmu_swap_task_ctx)(prev_epc, next_epc);
2638}
2639
2640void perf_check_microcode(void)
2641{
2642 if (x86_pmu.check_microcode)
2643 x86_pmu.check_microcode();
2644}
2645
2646static int x86_pmu_check_period(struct perf_event *event, u64 value)
2647{
2648 if (x86_pmu.check_period && x86_pmu.check_period(event, value))
2649 return -EINVAL;
2650
2651 if (value && x86_pmu.limit_period) {
2652 s64 left = value;
2653 x86_pmu.limit_period(event, &left);
2654 if (left > value)
2655 return -EINVAL;
2656 }
2657
2658 return 0;
2659}
2660
2661static int x86_pmu_aux_output_match(struct perf_event *event)
2662{
2663 if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT))
2664 return 0;
2665
2666 if (x86_pmu.aux_output_match)
2667 return x86_pmu.aux_output_match(event);
2668
2669 return 0;
2670}
2671
2672static bool x86_pmu_filter(struct pmu *pmu, int cpu)
2673{
2674 bool ret = false;
2675
2676 static_call_cond(x86_pmu_filter)(pmu, cpu, &ret);
2677
2678 return ret;
2679}
2680
2681static struct pmu pmu = {
2682 .pmu_enable = x86_pmu_enable,
2683 .pmu_disable = x86_pmu_disable,
2684
2685 .attr_groups = x86_pmu_attr_groups,
2686
2687 .event_init = x86_pmu_event_init,
2688
2689 .event_mapped = x86_pmu_event_mapped,
2690 .event_unmapped = x86_pmu_event_unmapped,
2691
2692 .add = x86_pmu_add,
2693 .del = x86_pmu_del,
2694 .start = x86_pmu_start,
2695 .stop = x86_pmu_stop,
2696 .read = x86_pmu_read,
2697
2698 .start_txn = x86_pmu_start_txn,
2699 .cancel_txn = x86_pmu_cancel_txn,
2700 .commit_txn = x86_pmu_commit_txn,
2701
2702 .event_idx = x86_pmu_event_idx,
2703 .sched_task = x86_pmu_sched_task,
2704 .swap_task_ctx = x86_pmu_swap_task_ctx,
2705 .check_period = x86_pmu_check_period,
2706
2707 .aux_output_match = x86_pmu_aux_output_match,
2708
2709 .filter = x86_pmu_filter,
2710};
2711
2712void arch_perf_update_userpage(struct perf_event *event,
2713 struct perf_event_mmap_page *userpg, u64 now)
2714{
2715 struct cyc2ns_data data;
2716 u64 offset;
2717
2718 userpg->cap_user_time = 0;
2719 userpg->cap_user_time_zero = 0;
2720 userpg->cap_user_rdpmc =
2721 !!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT);
2722 userpg->pmc_width = x86_pmu.cntval_bits;
2723
2724 if (!using_native_sched_clock() || !sched_clock_stable())
2725 return;
2726
2727 cyc2ns_read_begin(&data);
2728
2729 offset = data.cyc2ns_offset + __sched_clock_offset;
2730
2731 /*
2732 * Internal timekeeping for enabled/running/stopped times
2733 * is always in the local_clock domain.
2734 */
2735 userpg->cap_user_time = 1;
2736 userpg->time_mult = data.cyc2ns_mul;
2737 userpg->time_shift = data.cyc2ns_shift;
2738 userpg->time_offset = offset - now;
2739
2740 /*
2741 * cap_user_time_zero doesn't make sense when we're using a different
2742 * time base for the records.
2743 */
2744 if (!event->attr.use_clockid) {
2745 userpg->cap_user_time_zero = 1;
2746 userpg->time_zero = offset;
2747 }
2748
2749 cyc2ns_read_end();
2750}
2751
2752/*
2753 * Determine whether the regs were taken from an irq/exception handler rather
2754 * than from perf_arch_fetch_caller_regs().
2755 */
2756static bool perf_hw_regs(struct pt_regs *regs)
2757{
2758 return regs->flags & X86_EFLAGS_FIXED;
2759}
2760
2761void
2762perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2763{
2764 struct unwind_state state;
2765 unsigned long addr;
2766
2767 if (perf_guest_state()) {
2768 /* TODO: We don't support guest os callchain now */
2769 return;
2770 }
2771
2772 if (perf_callchain_store(entry, regs->ip))
2773 return;
2774
2775 if (perf_hw_regs(regs))
2776 unwind_start(&state, current, regs, NULL);
2777 else
2778 unwind_start(&state, current, NULL, (void *)regs->sp);
2779
2780 for (; !unwind_done(&state); unwind_next_frame(&state)) {
2781 addr = unwind_get_return_address(&state);
2782 if (!addr || perf_callchain_store(entry, addr))
2783 return;
2784 }
2785}
2786
2787static inline int
2788valid_user_frame(const void __user *fp, unsigned long size)
2789{
2790 return __access_ok(fp, size);
2791}
2792
2793static unsigned long get_segment_base(unsigned int segment)
2794{
2795 struct desc_struct *desc;
2796 unsigned int idx = segment >> 3;
2797
2798 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2799#ifdef CONFIG_MODIFY_LDT_SYSCALL
2800 struct ldt_struct *ldt;
2801
2802 /* IRQs are off, so this synchronizes with smp_store_release */
2803 ldt = READ_ONCE(current->active_mm->context.ldt);
2804 if (!ldt || idx >= ldt->nr_entries)
2805 return 0;
2806
2807 desc = &ldt->entries[idx];
2808#else
2809 return 0;
2810#endif
2811 } else {
2812 if (idx >= GDT_ENTRIES)
2813 return 0;
2814
2815 desc = raw_cpu_ptr(gdt_page.gdt) + idx;
2816 }
2817
2818 return get_desc_base(desc);
2819}
2820
2821#ifdef CONFIG_UPROBES
2822/*
2823 * Heuristic-based check if uprobe is installed at the function entry.
2824 *
2825 * Under assumption of user code being compiled with frame pointers,
2826 * `push %rbp/%ebp` is a good indicator that we indeed are.
2827 *
2828 * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern.
2829 * If we get this wrong, captured stack trace might have one extra bogus
2830 * entry, but the rest of stack trace will still be meaningful.
2831 */
2832static bool is_uprobe_at_func_entry(struct pt_regs *regs)
2833{
2834 struct arch_uprobe *auprobe;
2835
2836 if (!current->utask)
2837 return false;
2838
2839 auprobe = current->utask->auprobe;
2840 if (!auprobe)
2841 return false;
2842
2843 /* push %rbp/%ebp */
2844 if (auprobe->insn[0] == 0x55)
2845 return true;
2846
2847 /* endbr64 (64-bit only) */
2848 if (user_64bit_mode(regs) && is_endbr(*(u32 *)auprobe->insn))
2849 return true;
2850
2851 return false;
2852}
2853
2854#else
2855static bool is_uprobe_at_func_entry(struct pt_regs *regs)
2856{
2857 return false;
2858}
2859#endif /* CONFIG_UPROBES */
2860
2861#ifdef CONFIG_IA32_EMULATION
2862
2863#include <linux/compat.h>
2864
2865static inline int
2866perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2867{
2868 /* 32-bit process in 64-bit kernel. */
2869 unsigned long ss_base, cs_base;
2870 struct stack_frame_ia32 frame;
2871 const struct stack_frame_ia32 __user *fp;
2872 u32 ret_addr;
2873
2874 if (user_64bit_mode(regs))
2875 return 0;
2876
2877 cs_base = get_segment_base(regs->cs);
2878 ss_base = get_segment_base(regs->ss);
2879
2880 fp = compat_ptr(ss_base + regs->bp);
2881 pagefault_disable();
2882
2883 /* see perf_callchain_user() below for why we do this */
2884 if (is_uprobe_at_func_entry(regs) &&
2885 !get_user(ret_addr, (const u32 __user *)regs->sp))
2886 perf_callchain_store(entry, ret_addr);
2887
2888 while (entry->nr < entry->max_stack) {
2889 if (!valid_user_frame(fp, sizeof(frame)))
2890 break;
2891
2892 if (__get_user(frame.next_frame, &fp->next_frame))
2893 break;
2894 if (__get_user(frame.return_address, &fp->return_address))
2895 break;
2896
2897 perf_callchain_store(entry, cs_base + frame.return_address);
2898 fp = compat_ptr(ss_base + frame.next_frame);
2899 }
2900 pagefault_enable();
2901 return 1;
2902}
2903#else
2904static inline int
2905perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2906{
2907 return 0;
2908}
2909#endif
2910
2911void
2912perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2913{
2914 struct stack_frame frame;
2915 const struct stack_frame __user *fp;
2916 unsigned long ret_addr;
2917
2918 if (perf_guest_state()) {
2919 /* TODO: We don't support guest os callchain now */
2920 return;
2921 }
2922
2923 /*
2924 * We don't know what to do with VM86 stacks.. ignore them for now.
2925 */
2926 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
2927 return;
2928
2929 fp = (void __user *)regs->bp;
2930
2931 perf_callchain_store(entry, regs->ip);
2932
2933 if (!nmi_uaccess_okay())
2934 return;
2935
2936 if (perf_callchain_user32(regs, entry))
2937 return;
2938
2939 pagefault_disable();
2940
2941 /*
2942 * If we are called from uprobe handler, and we are indeed at the very
2943 * entry to user function (which is normally a `push %rbp` instruction,
2944 * under assumption of application being compiled with frame pointers),
2945 * we should read return address from *regs->sp before proceeding
2946 * to follow frame pointers, otherwise we'll skip immediate caller
2947 * as %rbp is not yet setup.
2948 */
2949 if (is_uprobe_at_func_entry(regs) &&
2950 !get_user(ret_addr, (const unsigned long __user *)regs->sp))
2951 perf_callchain_store(entry, ret_addr);
2952
2953 while (entry->nr < entry->max_stack) {
2954 if (!valid_user_frame(fp, sizeof(frame)))
2955 break;
2956
2957 if (__get_user(frame.next_frame, &fp->next_frame))
2958 break;
2959 if (__get_user(frame.return_address, &fp->return_address))
2960 break;
2961
2962 perf_callchain_store(entry, frame.return_address);
2963 fp = (void __user *)frame.next_frame;
2964 }
2965 pagefault_enable();
2966}
2967
2968/*
2969 * Deal with code segment offsets for the various execution modes:
2970 *
2971 * VM86 - the good olde 16 bit days, where the linear address is
2972 * 20 bits and we use regs->ip + 0x10 * regs->cs.
2973 *
2974 * IA32 - Where we need to look at GDT/LDT segment descriptor tables
2975 * to figure out what the 32bit base address is.
2976 *
2977 * X32 - has TIF_X32 set, but is running in x86_64
2978 *
2979 * X86_64 - CS,DS,SS,ES are all zero based.
2980 */
2981static unsigned long code_segment_base(struct pt_regs *regs)
2982{
2983 /*
2984 * For IA32 we look at the GDT/LDT segment base to convert the
2985 * effective IP to a linear address.
2986 */
2987
2988#ifdef CONFIG_X86_32
2989 /*
2990 * If we are in VM86 mode, add the segment offset to convert to a
2991 * linear address.
2992 */
2993 if (regs->flags & X86_VM_MASK)
2994 return 0x10 * regs->cs;
2995
2996 if (user_mode(regs) && regs->cs != __USER_CS)
2997 return get_segment_base(regs->cs);
2998#else
2999 if (user_mode(regs) && !user_64bit_mode(regs) &&
3000 regs->cs != __USER32_CS)
3001 return get_segment_base(regs->cs);
3002#endif
3003 return 0;
3004}
3005
3006unsigned long perf_arch_instruction_pointer(struct pt_regs *regs)
3007{
3008 return regs->ip + code_segment_base(regs);
3009}
3010
3011static unsigned long common_misc_flags(struct pt_regs *regs)
3012{
3013 if (regs->flags & PERF_EFLAGS_EXACT)
3014 return PERF_RECORD_MISC_EXACT_IP;
3015
3016 return 0;
3017}
3018
3019static unsigned long guest_misc_flags(struct pt_regs *regs)
3020{
3021 unsigned long guest_state = perf_guest_state();
3022
3023 if (!(guest_state & PERF_GUEST_ACTIVE))
3024 return 0;
3025
3026 if (guest_state & PERF_GUEST_USER)
3027 return PERF_RECORD_MISC_GUEST_USER;
3028 else
3029 return PERF_RECORD_MISC_GUEST_KERNEL;
3030
3031}
3032
3033static unsigned long host_misc_flags(struct pt_regs *regs)
3034{
3035 if (user_mode(regs))
3036 return PERF_RECORD_MISC_USER;
3037 else
3038 return PERF_RECORD_MISC_KERNEL;
3039}
3040
3041unsigned long perf_arch_guest_misc_flags(struct pt_regs *regs)
3042{
3043 unsigned long flags = common_misc_flags(regs);
3044
3045 flags |= guest_misc_flags(regs);
3046
3047 return flags;
3048}
3049
3050unsigned long perf_arch_misc_flags(struct pt_regs *regs)
3051{
3052 unsigned long flags = common_misc_flags(regs);
3053
3054 flags |= host_misc_flags(regs);
3055
3056 return flags;
3057}
3058
3059void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
3060{
3061 /* This API doesn't currently support enumerating hybrid PMUs. */
3062 if (WARN_ON_ONCE(cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) ||
3063 !x86_pmu_initialized()) {
3064 memset(cap, 0, sizeof(*cap));
3065 return;
3066 }
3067
3068 /*
3069 * Note, hybrid CPU models get tracked as having hybrid PMUs even when
3070 * all E-cores are disabled via BIOS. When E-cores are disabled, the
3071 * base PMU holds the correct number of counters for P-cores.
3072 */
3073 cap->version = x86_pmu.version;
3074 cap->num_counters_gp = x86_pmu_num_counters(NULL);
3075 cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL);
3076 cap->bit_width_gp = x86_pmu.cntval_bits;
3077 cap->bit_width_fixed = x86_pmu.cntval_bits;
3078 cap->events_mask = (unsigned int)x86_pmu.events_maskl;
3079 cap->events_mask_len = x86_pmu.events_mask_len;
3080 cap->pebs_ept = x86_pmu.pebs_ept;
3081}
3082EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);
3083
3084u64 perf_get_hw_event_config(int hw_event)
3085{
3086 int max = x86_pmu.max_events;
3087
3088 if (hw_event < max)
3089 return x86_pmu.event_map(array_index_nospec(hw_event, max));
3090
3091 return 0;
3092}
3093EXPORT_SYMBOL_GPL(perf_get_hw_event_config);