Loading...
Note: File does not exist in v5.4.
1#include <linux/perf_event.h>
2#include <linux/export.h>
3#include <linux/types.h>
4#include <linux/init.h>
5#include <linux/slab.h>
6#include <asm/apicdef.h>
7
8#include "perf_event.h"
9
10static __initconst const u64 amd_hw_cache_event_ids
11 [PERF_COUNT_HW_CACHE_MAX]
12 [PERF_COUNT_HW_CACHE_OP_MAX]
13 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
14{
15 [ C(L1D) ] = {
16 [ C(OP_READ) ] = {
17 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
18 [ C(RESULT_MISS) ] = 0x0141, /* Data Cache Misses */
19 },
20 [ C(OP_WRITE) ] = {
21 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
22 [ C(RESULT_MISS) ] = 0,
23 },
24 [ C(OP_PREFETCH) ] = {
25 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts */
26 [ C(RESULT_MISS) ] = 0x0167, /* Data Prefetcher :cancelled */
27 },
28 },
29 [ C(L1I ) ] = {
30 [ C(OP_READ) ] = {
31 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches */
32 [ C(RESULT_MISS) ] = 0x0081, /* Instruction cache misses */
33 },
34 [ C(OP_WRITE) ] = {
35 [ C(RESULT_ACCESS) ] = -1,
36 [ C(RESULT_MISS) ] = -1,
37 },
38 [ C(OP_PREFETCH) ] = {
39 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
40 [ C(RESULT_MISS) ] = 0,
41 },
42 },
43 [ C(LL ) ] = {
44 [ C(OP_READ) ] = {
45 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
46 [ C(RESULT_MISS) ] = 0x037E, /* L2 Cache Misses : IC+DC */
47 },
48 [ C(OP_WRITE) ] = {
49 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback */
50 [ C(RESULT_MISS) ] = 0,
51 },
52 [ C(OP_PREFETCH) ] = {
53 [ C(RESULT_ACCESS) ] = 0,
54 [ C(RESULT_MISS) ] = 0,
55 },
56 },
57 [ C(DTLB) ] = {
58 [ C(OP_READ) ] = {
59 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
60 [ C(RESULT_MISS) ] = 0x0746, /* L1_DTLB_AND_L2_DLTB_MISS.ALL */
61 },
62 [ C(OP_WRITE) ] = {
63 [ C(RESULT_ACCESS) ] = 0,
64 [ C(RESULT_MISS) ] = 0,
65 },
66 [ C(OP_PREFETCH) ] = {
67 [ C(RESULT_ACCESS) ] = 0,
68 [ C(RESULT_MISS) ] = 0,
69 },
70 },
71 [ C(ITLB) ] = {
72 [ C(OP_READ) ] = {
73 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes */
74 [ C(RESULT_MISS) ] = 0x0385, /* L1_ITLB_AND_L2_ITLB_MISS.ALL */
75 },
76 [ C(OP_WRITE) ] = {
77 [ C(RESULT_ACCESS) ] = -1,
78 [ C(RESULT_MISS) ] = -1,
79 },
80 [ C(OP_PREFETCH) ] = {
81 [ C(RESULT_ACCESS) ] = -1,
82 [ C(RESULT_MISS) ] = -1,
83 },
84 },
85 [ C(BPU ) ] = {
86 [ C(OP_READ) ] = {
87 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr. */
88 [ C(RESULT_MISS) ] = 0x00c3, /* Retired Mispredicted BI */
89 },
90 [ C(OP_WRITE) ] = {
91 [ C(RESULT_ACCESS) ] = -1,
92 [ C(RESULT_MISS) ] = -1,
93 },
94 [ C(OP_PREFETCH) ] = {
95 [ C(RESULT_ACCESS) ] = -1,
96 [ C(RESULT_MISS) ] = -1,
97 },
98 },
99 [ C(NODE) ] = {
100 [ C(OP_READ) ] = {
101 [ C(RESULT_ACCESS) ] = 0xb8e9, /* CPU Request to Memory, l+r */
102 [ C(RESULT_MISS) ] = 0x98e9, /* CPU Request to Memory, r */
103 },
104 [ C(OP_WRITE) ] = {
105 [ C(RESULT_ACCESS) ] = -1,
106 [ C(RESULT_MISS) ] = -1,
107 },
108 [ C(OP_PREFETCH) ] = {
109 [ C(RESULT_ACCESS) ] = -1,
110 [ C(RESULT_MISS) ] = -1,
111 },
112 },
113};
114
115/*
116 * AMD Performance Monitor K7 and later.
117 */
118static const u64 amd_perfmon_event_map[] =
119{
120 [PERF_COUNT_HW_CPU_CYCLES] = 0x0076,
121 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
122 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080,
123 [PERF_COUNT_HW_CACHE_MISSES] = 0x0081,
124 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2,
125 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3,
126 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */
127 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x00d1, /* "Dispatch stalls" event */
128};
129
130static u64 amd_pmu_event_map(int hw_event)
131{
132 return amd_perfmon_event_map[hw_event];
133}
134
135/*
136 * Previously calculated offsets
137 */
138static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly;
139static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly;
140
141/*
142 * Legacy CPUs:
143 * 4 counters starting at 0xc0010000 each offset by 1
144 *
145 * CPUs with core performance counter extensions:
146 * 6 counters starting at 0xc0010200 each offset by 2
147 */
148static inline int amd_pmu_addr_offset(int index, bool eventsel)
149{
150 int offset;
151
152 if (!index)
153 return index;
154
155 if (eventsel)
156 offset = event_offsets[index];
157 else
158 offset = count_offsets[index];
159
160 if (offset)
161 return offset;
162
163 if (!cpu_has_perfctr_core)
164 offset = index;
165 else
166 offset = index << 1;
167
168 if (eventsel)
169 event_offsets[index] = offset;
170 else
171 count_offsets[index] = offset;
172
173 return offset;
174}
175
176static int amd_core_hw_config(struct perf_event *event)
177{
178 if (event->attr.exclude_host && event->attr.exclude_guest)
179 /*
180 * When HO == GO == 1 the hardware treats that as GO == HO == 0
181 * and will count in both modes. We don't want to count in that
182 * case so we emulate no-counting by setting US = OS = 0.
183 */
184 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
185 ARCH_PERFMON_EVENTSEL_OS);
186 else if (event->attr.exclude_host)
187 event->hw.config |= AMD64_EVENTSEL_GUESTONLY;
188 else if (event->attr.exclude_guest)
189 event->hw.config |= AMD64_EVENTSEL_HOSTONLY;
190
191 return 0;
192}
193
194/*
195 * AMD64 events are detected based on their event codes.
196 */
197static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc)
198{
199 return ((hwc->config >> 24) & 0x0f00) | (hwc->config & 0x00ff);
200}
201
202static inline int amd_is_nb_event(struct hw_perf_event *hwc)
203{
204 return (hwc->config & 0xe0) == 0xe0;
205}
206
207static inline int amd_has_nb(struct cpu_hw_events *cpuc)
208{
209 struct amd_nb *nb = cpuc->amd_nb;
210
211 return nb && nb->nb_id != -1;
212}
213
214static int amd_pmu_hw_config(struct perf_event *event)
215{
216 int ret;
217
218 /* pass precise event sampling to ibs: */
219 if (event->attr.precise_ip && get_ibs_caps())
220 return -ENOENT;
221
222 if (has_branch_stack(event))
223 return -EOPNOTSUPP;
224
225 ret = x86_pmu_hw_config(event);
226 if (ret)
227 return ret;
228
229 if (event->attr.type == PERF_TYPE_RAW)
230 event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
231
232 return amd_core_hw_config(event);
233}
234
235static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc,
236 struct perf_event *event)
237{
238 struct amd_nb *nb = cpuc->amd_nb;
239 int i;
240
241 /*
242 * need to scan whole list because event may not have
243 * been assigned during scheduling
244 *
245 * no race condition possible because event can only
246 * be removed on one CPU at a time AND PMU is disabled
247 * when we come here
248 */
249 for (i = 0; i < x86_pmu.num_counters; i++) {
250 if (cmpxchg(nb->owners + i, event, NULL) == event)
251 break;
252 }
253}
254
255 /*
256 * AMD64 NorthBridge events need special treatment because
257 * counter access needs to be synchronized across all cores
258 * of a package. Refer to BKDG section 3.12
259 *
260 * NB events are events measuring L3 cache, Hypertransport
261 * traffic. They are identified by an event code >= 0xe00.
262 * They measure events on the NorthBride which is shared
263 * by all cores on a package. NB events are counted on a
264 * shared set of counters. When a NB event is programmed
265 * in a counter, the data actually comes from a shared
266 * counter. Thus, access to those counters needs to be
267 * synchronized.
268 *
269 * We implement the synchronization such that no two cores
270 * can be measuring NB events using the same counters. Thus,
271 * we maintain a per-NB allocation table. The available slot
272 * is propagated using the event_constraint structure.
273 *
274 * We provide only one choice for each NB event based on
275 * the fact that only NB events have restrictions. Consequently,
276 * if a counter is available, there is a guarantee the NB event
277 * will be assigned to it. If no slot is available, an empty
278 * constraint is returned and scheduling will eventually fail
279 * for this event.
280 *
281 * Note that all cores attached the same NB compete for the same
282 * counters to host NB events, this is why we use atomic ops. Some
283 * multi-chip CPUs may have more than one NB.
284 *
285 * Given that resources are allocated (cmpxchg), they must be
286 * eventually freed for others to use. This is accomplished by
287 * calling __amd_put_nb_event_constraints()
288 *
289 * Non NB events are not impacted by this restriction.
290 */
291static struct event_constraint *
292__amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
293 struct event_constraint *c)
294{
295 struct hw_perf_event *hwc = &event->hw;
296 struct amd_nb *nb = cpuc->amd_nb;
297 struct perf_event *old;
298 int idx, new = -1;
299
300 if (!c)
301 c = &unconstrained;
302
303 if (cpuc->is_fake)
304 return c;
305
306 /*
307 * detect if already present, if so reuse
308 *
309 * cannot merge with actual allocation
310 * because of possible holes
311 *
312 * event can already be present yet not assigned (in hwc->idx)
313 * because of successive calls to x86_schedule_events() from
314 * hw_perf_group_sched_in() without hw_perf_enable()
315 */
316 for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) {
317 if (new == -1 || hwc->idx == idx)
318 /* assign free slot, prefer hwc->idx */
319 old = cmpxchg(nb->owners + idx, NULL, event);
320 else if (nb->owners[idx] == event)
321 /* event already present */
322 old = event;
323 else
324 continue;
325
326 if (old && old != event)
327 continue;
328
329 /* reassign to this slot */
330 if (new != -1)
331 cmpxchg(nb->owners + new, event, NULL);
332 new = idx;
333
334 /* already present, reuse */
335 if (old == event)
336 break;
337 }
338
339 if (new == -1)
340 return &emptyconstraint;
341
342 return &nb->event_constraints[new];
343}
344
345static struct amd_nb *amd_alloc_nb(int cpu)
346{
347 struct amd_nb *nb;
348 int i;
349
350 nb = kzalloc_node(sizeof(struct amd_nb), GFP_KERNEL, cpu_to_node(cpu));
351 if (!nb)
352 return NULL;
353
354 nb->nb_id = -1;
355
356 /*
357 * initialize all possible NB constraints
358 */
359 for (i = 0; i < x86_pmu.num_counters; i++) {
360 __set_bit(i, nb->event_constraints[i].idxmsk);
361 nb->event_constraints[i].weight = 1;
362 }
363 return nb;
364}
365
366static int amd_pmu_cpu_prepare(int cpu)
367{
368 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
369
370 WARN_ON_ONCE(cpuc->amd_nb);
371
372 if (boot_cpu_data.x86_max_cores < 2)
373 return NOTIFY_OK;
374
375 cpuc->amd_nb = amd_alloc_nb(cpu);
376 if (!cpuc->amd_nb)
377 return NOTIFY_BAD;
378
379 return NOTIFY_OK;
380}
381
382static void amd_pmu_cpu_starting(int cpu)
383{
384 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
385 struct amd_nb *nb;
386 int i, nb_id;
387
388 cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
389
390 if (boot_cpu_data.x86_max_cores < 2)
391 return;
392
393 nb_id = amd_get_nb_id(cpu);
394 WARN_ON_ONCE(nb_id == BAD_APICID);
395
396 for_each_online_cpu(i) {
397 nb = per_cpu(cpu_hw_events, i).amd_nb;
398 if (WARN_ON_ONCE(!nb))
399 continue;
400
401 if (nb->nb_id == nb_id) {
402 cpuc->kfree_on_online = cpuc->amd_nb;
403 cpuc->amd_nb = nb;
404 break;
405 }
406 }
407
408 cpuc->amd_nb->nb_id = nb_id;
409 cpuc->amd_nb->refcnt++;
410}
411
412static void amd_pmu_cpu_dead(int cpu)
413{
414 struct cpu_hw_events *cpuhw;
415
416 if (boot_cpu_data.x86_max_cores < 2)
417 return;
418
419 cpuhw = &per_cpu(cpu_hw_events, cpu);
420
421 if (cpuhw->amd_nb) {
422 struct amd_nb *nb = cpuhw->amd_nb;
423
424 if (nb->nb_id == -1 || --nb->refcnt == 0)
425 kfree(nb);
426
427 cpuhw->amd_nb = NULL;
428 }
429}
430
431static struct event_constraint *
432amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
433{
434 /*
435 * if not NB event or no NB, then no constraints
436 */
437 if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)))
438 return &unconstrained;
439
440 return __amd_get_nb_event_constraints(cpuc, event, NULL);
441}
442
443static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
444 struct perf_event *event)
445{
446 if (amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))
447 __amd_put_nb_event_constraints(cpuc, event);
448}
449
450PMU_FORMAT_ATTR(event, "config:0-7,32-35");
451PMU_FORMAT_ATTR(umask, "config:8-15" );
452PMU_FORMAT_ATTR(edge, "config:18" );
453PMU_FORMAT_ATTR(inv, "config:23" );
454PMU_FORMAT_ATTR(cmask, "config:24-31" );
455
456static struct attribute *amd_format_attr[] = {
457 &format_attr_event.attr,
458 &format_attr_umask.attr,
459 &format_attr_edge.attr,
460 &format_attr_inv.attr,
461 &format_attr_cmask.attr,
462 NULL,
463};
464
465/* AMD Family 15h */
466
467#define AMD_EVENT_TYPE_MASK 0x000000F0ULL
468
469#define AMD_EVENT_FP 0x00000000ULL ... 0x00000010ULL
470#define AMD_EVENT_LS 0x00000020ULL ... 0x00000030ULL
471#define AMD_EVENT_DC 0x00000040ULL ... 0x00000050ULL
472#define AMD_EVENT_CU 0x00000060ULL ... 0x00000070ULL
473#define AMD_EVENT_IC_DE 0x00000080ULL ... 0x00000090ULL
474#define AMD_EVENT_EX_LS 0x000000C0ULL
475#define AMD_EVENT_DE 0x000000D0ULL
476#define AMD_EVENT_NB 0x000000E0ULL ... 0x000000F0ULL
477
478/*
479 * AMD family 15h event code/PMC mappings:
480 *
481 * type = event_code & 0x0F0:
482 *
483 * 0x000 FP PERF_CTL[5:3]
484 * 0x010 FP PERF_CTL[5:3]
485 * 0x020 LS PERF_CTL[5:0]
486 * 0x030 LS PERF_CTL[5:0]
487 * 0x040 DC PERF_CTL[5:0]
488 * 0x050 DC PERF_CTL[5:0]
489 * 0x060 CU PERF_CTL[2:0]
490 * 0x070 CU PERF_CTL[2:0]
491 * 0x080 IC/DE PERF_CTL[2:0]
492 * 0x090 IC/DE PERF_CTL[2:0]
493 * 0x0A0 ---
494 * 0x0B0 ---
495 * 0x0C0 EX/LS PERF_CTL[5:0]
496 * 0x0D0 DE PERF_CTL[2:0]
497 * 0x0E0 NB NB_PERF_CTL[3:0]
498 * 0x0F0 NB NB_PERF_CTL[3:0]
499 *
500 * Exceptions:
501 *
502 * 0x000 FP PERF_CTL[3], PERF_CTL[5:3] (*)
503 * 0x003 FP PERF_CTL[3]
504 * 0x004 FP PERF_CTL[3], PERF_CTL[5:3] (*)
505 * 0x00B FP PERF_CTL[3]
506 * 0x00D FP PERF_CTL[3]
507 * 0x023 DE PERF_CTL[2:0]
508 * 0x02D LS PERF_CTL[3]
509 * 0x02E LS PERF_CTL[3,0]
510 * 0x031 LS PERF_CTL[2:0] (**)
511 * 0x043 CU PERF_CTL[2:0]
512 * 0x045 CU PERF_CTL[2:0]
513 * 0x046 CU PERF_CTL[2:0]
514 * 0x054 CU PERF_CTL[2:0]
515 * 0x055 CU PERF_CTL[2:0]
516 * 0x08F IC PERF_CTL[0]
517 * 0x187 DE PERF_CTL[0]
518 * 0x188 DE PERF_CTL[0]
519 * 0x0DB EX PERF_CTL[5:0]
520 * 0x0DC LS PERF_CTL[5:0]
521 * 0x0DD LS PERF_CTL[5:0]
522 * 0x0DE LS PERF_CTL[5:0]
523 * 0x0DF LS PERF_CTL[5:0]
524 * 0x1C0 EX PERF_CTL[5:3]
525 * 0x1D6 EX PERF_CTL[5:0]
526 * 0x1D8 EX PERF_CTL[5:0]
527 *
528 * (*) depending on the umask all FPU counters may be used
529 * (**) only one unitmask enabled at a time
530 */
531
532static struct event_constraint amd_f15_PMC0 = EVENT_CONSTRAINT(0, 0x01, 0);
533static struct event_constraint amd_f15_PMC20 = EVENT_CONSTRAINT(0, 0x07, 0);
534static struct event_constraint amd_f15_PMC3 = EVENT_CONSTRAINT(0, 0x08, 0);
535static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
536static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0);
537static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
538
539static struct event_constraint *
540amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *event)
541{
542 struct hw_perf_event *hwc = &event->hw;
543 unsigned int event_code = amd_get_event_code(hwc);
544
545 switch (event_code & AMD_EVENT_TYPE_MASK) {
546 case AMD_EVENT_FP:
547 switch (event_code) {
548 case 0x000:
549 if (!(hwc->config & 0x0000F000ULL))
550 break;
551 if (!(hwc->config & 0x00000F00ULL))
552 break;
553 return &amd_f15_PMC3;
554 case 0x004:
555 if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
556 break;
557 return &amd_f15_PMC3;
558 case 0x003:
559 case 0x00B:
560 case 0x00D:
561 return &amd_f15_PMC3;
562 }
563 return &amd_f15_PMC53;
564 case AMD_EVENT_LS:
565 case AMD_EVENT_DC:
566 case AMD_EVENT_EX_LS:
567 switch (event_code) {
568 case 0x023:
569 case 0x043:
570 case 0x045:
571 case 0x046:
572 case 0x054:
573 case 0x055:
574 return &amd_f15_PMC20;
575 case 0x02D:
576 return &amd_f15_PMC3;
577 case 0x02E:
578 return &amd_f15_PMC30;
579 case 0x031:
580 if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
581 return &amd_f15_PMC20;
582 return &emptyconstraint;
583 case 0x1C0:
584 return &amd_f15_PMC53;
585 default:
586 return &amd_f15_PMC50;
587 }
588 case AMD_EVENT_CU:
589 case AMD_EVENT_IC_DE:
590 case AMD_EVENT_DE:
591 switch (event_code) {
592 case 0x08F:
593 case 0x187:
594 case 0x188:
595 return &amd_f15_PMC0;
596 case 0x0DB ... 0x0DF:
597 case 0x1D6:
598 case 0x1D8:
599 return &amd_f15_PMC50;
600 default:
601 return &amd_f15_PMC20;
602 }
603 case AMD_EVENT_NB:
604 /* moved to perf_event_amd_uncore.c */
605 return &emptyconstraint;
606 default:
607 return &emptyconstraint;
608 }
609}
610
611static ssize_t amd_event_sysfs_show(char *page, u64 config)
612{
613 u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT) |
614 (config & AMD64_EVENTSEL_EVENT) >> 24;
615
616 return x86_event_sysfs_show(page, config, event);
617}
618
619static __initconst const struct x86_pmu amd_pmu = {
620 .name = "AMD",
621 .handle_irq = x86_pmu_handle_irq,
622 .disable_all = x86_pmu_disable_all,
623 .enable_all = x86_pmu_enable_all,
624 .enable = x86_pmu_enable_event,
625 .disable = x86_pmu_disable_event,
626 .hw_config = amd_pmu_hw_config,
627 .schedule_events = x86_schedule_events,
628 .eventsel = MSR_K7_EVNTSEL0,
629 .perfctr = MSR_K7_PERFCTR0,
630 .addr_offset = amd_pmu_addr_offset,
631 .event_map = amd_pmu_event_map,
632 .max_events = ARRAY_SIZE(amd_perfmon_event_map),
633 .num_counters = AMD64_NUM_COUNTERS,
634 .cntval_bits = 48,
635 .cntval_mask = (1ULL << 48) - 1,
636 .apic = 1,
637 /* use highest bit to detect overflow */
638 .max_period = (1ULL << 47) - 1,
639 .get_event_constraints = amd_get_event_constraints,
640 .put_event_constraints = amd_put_event_constraints,
641
642 .format_attrs = amd_format_attr,
643 .events_sysfs_show = amd_event_sysfs_show,
644
645 .cpu_prepare = amd_pmu_cpu_prepare,
646 .cpu_starting = amd_pmu_cpu_starting,
647 .cpu_dead = amd_pmu_cpu_dead,
648};
649
650static int __init amd_core_pmu_init(void)
651{
652 if (!cpu_has_perfctr_core)
653 return 0;
654
655 switch (boot_cpu_data.x86) {
656 case 0x15:
657 pr_cont("Fam15h ");
658 x86_pmu.get_event_constraints = amd_get_event_constraints_f15h;
659 break;
660
661 default:
662 pr_err("core perfctr but no constraints; unknown hardware!\n");
663 return -ENODEV;
664 }
665
666 /*
667 * If core performance counter extensions exists, we must use
668 * MSR_F15H_PERF_CTL/MSR_F15H_PERF_CTR msrs. See also
669 * amd_pmu_addr_offset().
670 */
671 x86_pmu.eventsel = MSR_F15H_PERF_CTL;
672 x86_pmu.perfctr = MSR_F15H_PERF_CTR;
673 x86_pmu.num_counters = AMD64_NUM_COUNTERS_CORE;
674
675 pr_cont("core perfctr, ");
676 return 0;
677}
678
679__init int amd_pmu_init(void)
680{
681 int ret;
682
683 /* Performance-monitoring supported from K7 and later: */
684 if (boot_cpu_data.x86 < 6)
685 return -ENODEV;
686
687 x86_pmu = amd_pmu;
688
689 ret = amd_core_pmu_init();
690 if (ret)
691 return ret;
692
693 /* Events are common for all AMDs */
694 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
695 sizeof(hw_cache_event_ids));
696
697 return 0;
698}
699
700void amd_pmu_enable_virt(void)
701{
702 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
703
704 cpuc->perf_ctr_virt_mask = 0;
705
706 /* Reload all events */
707 x86_pmu_disable_all();
708 x86_pmu_enable_all(0);
709}
710EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
711
712void amd_pmu_disable_virt(void)
713{
714 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
715
716 /*
717 * We only mask out the Host-only bit so that host-only counting works
718 * when SVM is disabled. If someone sets up a guest-only counter when
719 * SVM is disabled the Guest-only bits still gets set and the counter
720 * will not count anything.
721 */
722 cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
723
724 /* Reload all events */
725 x86_pmu_disable_all();
726 x86_pmu_enable_all(0);
727}
728EXPORT_SYMBOL_GPL(amd_pmu_disable_virt);