Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Performance event support for the System z CPU-measurement Sampling Facility
4 *
5 * Copyright IBM Corp. 2013, 2018
6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
7 */
8#define KMSG_COMPONENT "cpum_sf"
9#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/kernel_stat.h>
13#include <linux/perf_event.h>
14#include <linux/percpu.h>
15#include <linux/pid.h>
16#include <linux/notifier.h>
17#include <linux/export.h>
18#include <linux/slab.h>
19#include <linux/mm.h>
20#include <linux/moduleparam.h>
21#include <asm/cpu_mf.h>
22#include <asm/irq.h>
23#include <asm/debug.h>
24#include <asm/timex.h>
25#include <linux/io.h>
26
27/* Perf PMU definitions for the sampling facility */
28#define PERF_CPUM_SF_MAX_CTR 2
29#define PERF_EVENT_CPUM_SF 0xB0000UL /* Event: Basic-sampling */
30#define PERF_EVENT_CPUM_SF_DIAG 0xBD000UL /* Event: Combined-sampling */
31#define PERF_CPUM_SF_BASIC_MODE 0x0001 /* Basic-sampling flag */
32#define PERF_CPUM_SF_DIAG_MODE 0x0002 /* Diagnostic-sampling flag */
33#define PERF_CPUM_SF_FREQ_MODE 0x0008 /* Sampling with frequency */
34
35#define OVERFLOW_REG(hwc) ((hwc)->extra_reg.config)
36#define SFB_ALLOC_REG(hwc) ((hwc)->extra_reg.alloc)
37#define TEAR_REG(hwc) ((hwc)->last_tag)
38#define SAMPL_RATE(hwc) ((hwc)->event_base)
39#define SAMPL_FLAGS(hwc) ((hwc)->config_base)
40#define SAMPL_DIAG_MODE(hwc) (SAMPL_FLAGS(hwc) & PERF_CPUM_SF_DIAG_MODE)
41#define SAMPL_FREQ_MODE(hwc) (SAMPL_FLAGS(hwc) & PERF_CPUM_SF_FREQ_MODE)
42
43/* Minimum number of sample-data-block-tables:
44 * At least one table is required for the sampling buffer structure.
45 * A single table contains up to 511 pointers to sample-data-blocks.
46 */
47#define CPUM_SF_MIN_SDBT 1
48
49/* Number of sample-data-blocks per sample-data-block-table (SDBT):
50 * A table contains SDB pointers (8 bytes) and one table-link entry
51 * that points to the origin of the next SDBT.
52 */
53#define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8)
54
55/* Maximum page offset for an SDBT table-link entry:
56 * If this page offset is reached, a table-link entry to the next SDBT
57 * must be added.
58 */
59#define CPUM_SF_SDBT_TL_OFFSET (CPUM_SF_SDB_PER_TABLE * 8)
60static inline int require_table_link(const void *sdbt)
61{
62 return ((unsigned long)sdbt & ~PAGE_MASK) == CPUM_SF_SDBT_TL_OFFSET;
63}
64
65/* Minimum and maximum sampling buffer sizes:
66 *
67 * This number represents the maximum size of the sampling buffer taking
68 * the number of sample-data-block-tables into account. Note that these
69 * numbers apply to the basic-sampling function only.
70 * The maximum number of SDBs is increased by CPUM_SF_SDB_DIAG_FACTOR if
71 * the diagnostic-sampling function is active.
72 *
73 * Sampling buffer size Buffer characteristics
74 * ---------------------------------------------------
75 * 64KB == 16 pages (4KB per page)
76 * 1 page for SDB-tables
77 * 15 pages for SDBs
78 *
79 * 32MB == 8192 pages (4KB per page)
80 * 16 pages for SDB-tables
81 * 8176 pages for SDBs
82 */
83static unsigned long __read_mostly CPUM_SF_MIN_SDB = 15;
84static unsigned long __read_mostly CPUM_SF_MAX_SDB = 8176;
85static unsigned long __read_mostly CPUM_SF_SDB_DIAG_FACTOR = 1;
86
87struct sf_buffer {
88 unsigned long *sdbt; /* Sample-data-block-table origin */
89 /* buffer characteristics (required for buffer increments) */
90 unsigned long num_sdb; /* Number of sample-data-blocks */
91 unsigned long num_sdbt; /* Number of sample-data-block-tables */
92 unsigned long *tail; /* last sample-data-block-table */
93};
94
95struct aux_buffer {
96 struct sf_buffer sfb;
97 unsigned long head; /* index of SDB of buffer head */
98 unsigned long alert_mark; /* index of SDB of alert request position */
99 unsigned long empty_mark; /* mark of SDB not marked full */
100 unsigned long *sdb_index; /* SDB address for fast lookup */
101 unsigned long *sdbt_index; /* SDBT address for fast lookup */
102};
103
104struct cpu_hw_sf {
105 /* CPU-measurement sampling information block */
106 struct hws_qsi_info_block qsi;
107 /* CPU-measurement sampling control block */
108 struct hws_lsctl_request_block lsctl;
109 struct sf_buffer sfb; /* Sampling buffer */
110 unsigned int flags; /* Status flags */
111 struct perf_event *event; /* Scheduled perf event */
112 struct perf_output_handle handle; /* AUX buffer output handle */
113};
114static DEFINE_PER_CPU(struct cpu_hw_sf, cpu_hw_sf);
115
116/* Debug feature */
117static debug_info_t *sfdbg;
118
119/* Sampling control helper functions */
120static inline unsigned long freq_to_sample_rate(struct hws_qsi_info_block *qsi,
121 unsigned long freq)
122{
123 return (USEC_PER_SEC / freq) * qsi->cpu_speed;
124}
125
126static inline unsigned long sample_rate_to_freq(struct hws_qsi_info_block *qsi,
127 unsigned long rate)
128{
129 return USEC_PER_SEC * qsi->cpu_speed / rate;
130}
131
132/* Return pointer to trailer entry of an sample data block */
133static inline struct hws_trailer_entry *trailer_entry_ptr(unsigned long v)
134{
135 void *ret;
136
137 ret = (void *)v;
138 ret += PAGE_SIZE;
139 ret -= sizeof(struct hws_trailer_entry);
140
141 return ret;
142}
143
144/*
145 * Return true if the entry in the sample data block table (sdbt)
146 * is a link to the next sdbt
147 */
148static inline int is_link_entry(unsigned long *s)
149{
150 return *s & 0x1UL ? 1 : 0;
151}
152
153/* Return pointer to the linked sdbt */
154static inline unsigned long *get_next_sdbt(unsigned long *s)
155{
156 return phys_to_virt(*s & ~0x1UL);
157}
158
159/*
160 * sf_disable() - Switch off sampling facility
161 */
162static void sf_disable(void)
163{
164 struct hws_lsctl_request_block sreq;
165
166 memset(&sreq, 0, sizeof(sreq));
167 lsctl(&sreq);
168}
169
170/*
171 * sf_buffer_available() - Check for an allocated sampling buffer
172 */
173static int sf_buffer_available(struct cpu_hw_sf *cpuhw)
174{
175 return !!cpuhw->sfb.sdbt;
176}
177
178/*
179 * deallocate sampling facility buffer
180 */
181static void free_sampling_buffer(struct sf_buffer *sfb)
182{
183 unsigned long *sdbt, *curr, *head;
184
185 sdbt = sfb->sdbt;
186 if (!sdbt)
187 return;
188 sfb->sdbt = NULL;
189 /* Free the SDBT after all SDBs are processed... */
190 head = sdbt;
191 curr = sdbt;
192 do {
193 if (is_link_entry(curr)) {
194 /* Process table-link entries */
195 curr = get_next_sdbt(curr);
196 free_page((unsigned long)sdbt);
197 sdbt = curr;
198 } else {
199 /* Process SDB pointer */
200 free_page((unsigned long)phys_to_virt(*curr));
201 curr++;
202 }
203 } while (curr != head);
204 memset(sfb, 0, sizeof(*sfb));
205}
206
207static int alloc_sample_data_block(unsigned long *sdbt, gfp_t gfp_flags)
208{
209 struct hws_trailer_entry *te;
210 unsigned long sdb;
211
212 /* Allocate and initialize sample-data-block */
213 sdb = get_zeroed_page(gfp_flags);
214 if (!sdb)
215 return -ENOMEM;
216 te = trailer_entry_ptr(sdb);
217 te->header.a = 1;
218
219 /* Link SDB into the sample-data-block-table */
220 *sdbt = virt_to_phys((void *)sdb);
221
222 return 0;
223}
224
225/*
226 * realloc_sampling_buffer() - extend sampler memory
227 *
228 * Allocates new sample-data-blocks and adds them to the specified sampling
229 * buffer memory.
230 *
231 * Important: This modifies the sampling buffer and must be called when the
232 * sampling facility is disabled.
233 *
234 * Returns zero on success, non-zero otherwise.
235 */
236static int realloc_sampling_buffer(struct sf_buffer *sfb,
237 unsigned long num_sdb, gfp_t gfp_flags)
238{
239 int i, rc;
240 unsigned long *new, *tail, *tail_prev = NULL;
241
242 if (!sfb->sdbt || !sfb->tail)
243 return -EINVAL;
244
245 if (!is_link_entry(sfb->tail))
246 return -EINVAL;
247
248 /* Append to the existing sampling buffer, overwriting the table-link
249 * register.
250 * The tail variables always points to the "tail" (last and table-link)
251 * entry in an SDB-table.
252 */
253 tail = sfb->tail;
254
255 /* Do a sanity check whether the table-link entry points to
256 * the sampling buffer origin.
257 */
258 if (sfb->sdbt != get_next_sdbt(tail)) {
259 debug_sprintf_event(sfdbg, 3, "%s buffer not linked origin %#lx tail %#lx\n",
260 __func__, (unsigned long)sfb->sdbt,
261 (unsigned long)tail);
262 return -EINVAL;
263 }
264
265 /* Allocate remaining SDBs */
266 rc = 0;
267 for (i = 0; i < num_sdb; i++) {
268 /* Allocate a new SDB-table if it is full. */
269 if (require_table_link(tail)) {
270 new = (unsigned long *)get_zeroed_page(gfp_flags);
271 if (!new) {
272 rc = -ENOMEM;
273 break;
274 }
275 sfb->num_sdbt++;
276 /* Link current page to tail of chain */
277 *tail = virt_to_phys((void *)new) + 1;
278 tail_prev = tail;
279 tail = new;
280 }
281
282 /* Allocate a new sample-data-block.
283 * If there is not enough memory, stop the realloc process
284 * and simply use what was allocated. If this is a temporary
285 * issue, a new realloc call (if required) might succeed.
286 */
287 rc = alloc_sample_data_block(tail, gfp_flags);
288 if (rc) {
289 /* Undo last SDBT. An SDBT with no SDB at its first
290 * entry but with an SDBT entry instead can not be
291 * handled by the interrupt handler code.
292 * Avoid this situation.
293 */
294 if (tail_prev) {
295 sfb->num_sdbt--;
296 free_page((unsigned long)new);
297 tail = tail_prev;
298 }
299 break;
300 }
301 sfb->num_sdb++;
302 tail++;
303 tail_prev = new = NULL; /* Allocated at least one SBD */
304 }
305
306 /* Link sampling buffer to its origin */
307 *tail = virt_to_phys(sfb->sdbt) + 1;
308 sfb->tail = tail;
309
310 return rc;
311}
312
313/*
314 * allocate_sampling_buffer() - allocate sampler memory
315 *
316 * Allocates and initializes a sampling buffer structure using the
317 * specified number of sample-data-blocks (SDB). For each allocation,
318 * a 4K page is used. The number of sample-data-block-tables (SDBT)
319 * are calculated from SDBs.
320 * Also set the ALERT_REQ mask in each SDBs trailer.
321 *
322 * Returns zero on success, non-zero otherwise.
323 */
324static int alloc_sampling_buffer(struct sf_buffer *sfb, unsigned long num_sdb)
325{
326 int rc;
327
328 if (sfb->sdbt)
329 return -EINVAL;
330
331 /* Allocate the sample-data-block-table origin */
332 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
333 if (!sfb->sdbt)
334 return -ENOMEM;
335 sfb->num_sdb = 0;
336 sfb->num_sdbt = 1;
337
338 /* Link the table origin to point to itself to prepare for
339 * realloc_sampling_buffer() invocation.
340 */
341 sfb->tail = sfb->sdbt;
342 *sfb->tail = virt_to_phys((void *)sfb->sdbt) + 1;
343
344 /* Allocate requested number of sample-data-blocks */
345 rc = realloc_sampling_buffer(sfb, num_sdb, GFP_KERNEL);
346 if (rc)
347 free_sampling_buffer(sfb);
348 return rc;
349}
350
351static void sfb_set_limits(unsigned long min, unsigned long max)
352{
353 struct hws_qsi_info_block si;
354
355 CPUM_SF_MIN_SDB = min;
356 CPUM_SF_MAX_SDB = max;
357
358 memset(&si, 0, sizeof(si));
359 qsi(&si);
360 CPUM_SF_SDB_DIAG_FACTOR = DIV_ROUND_UP(si.dsdes, si.bsdes);
361}
362
363static unsigned long sfb_max_limit(struct hw_perf_event *hwc)
364{
365 return SAMPL_DIAG_MODE(hwc) ? CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR
366 : CPUM_SF_MAX_SDB;
367}
368
369static unsigned long sfb_pending_allocs(struct sf_buffer *sfb,
370 struct hw_perf_event *hwc)
371{
372 if (!sfb->sdbt)
373 return SFB_ALLOC_REG(hwc);
374 if (SFB_ALLOC_REG(hwc) > sfb->num_sdb)
375 return SFB_ALLOC_REG(hwc) - sfb->num_sdb;
376 return 0;
377}
378
379static void sfb_account_allocs(unsigned long num, struct hw_perf_event *hwc)
380{
381 /* Limit the number of SDBs to not exceed the maximum */
382 num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc));
383 if (num)
384 SFB_ALLOC_REG(hwc) += num;
385}
386
387static void sfb_init_allocs(unsigned long num, struct hw_perf_event *hwc)
388{
389 SFB_ALLOC_REG(hwc) = 0;
390 sfb_account_allocs(num, hwc);
391}
392
393static void deallocate_buffers(struct cpu_hw_sf *cpuhw)
394{
395 if (sf_buffer_available(cpuhw))
396 free_sampling_buffer(&cpuhw->sfb);
397}
398
399static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc)
400{
401 unsigned long n_sdb, freq;
402
403 /* Calculate sampling buffers using 4K pages
404 *
405 * 1. The sampling size is 32 bytes for basic sampling. This size
406 * is the same for all machine types. Diagnostic
407 * sampling uses auxlilary data buffer setup which provides the
408 * memory for SDBs using linux common code auxiliary trace
409 * setup.
410 *
411 * 2. Function alloc_sampling_buffer() sets the Alert Request
412 * Control indicator to trigger a measurement-alert to harvest
413 * sample-data-blocks (SDB). This is done per SDB. This
414 * measurement alert interrupt fires quick enough to handle
415 * one SDB, on very high frequency and work loads there might
416 * be 2 to 3 SBDs available for sample processing.
417 * Currently there is no need for setup alert request on every
418 * n-th page. This is counterproductive as one IRQ triggers
419 * a very high number of samples to be processed at one IRQ.
420 *
421 * 3. Use the sampling frequency as input.
422 * Compute the number of SDBs and ensure a minimum
423 * of CPUM_SF_MIN_SDB. Depending on frequency add some more
424 * SDBs to handle a higher sampling rate.
425 * Use a minimum of CPUM_SF_MIN_SDB and allow for 100 samples
426 * (one SDB) for every 10000 HZ frequency increment.
427 *
428 * 4. Compute the number of sample-data-block-tables (SDBT) and
429 * ensure a minimum of CPUM_SF_MIN_SDBT (one table can manage up
430 * to 511 SDBs).
431 */
432 freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc));
433 n_sdb = CPUM_SF_MIN_SDB + DIV_ROUND_UP(freq, 10000);
434
435 /* If there is already a sampling buffer allocated, it is very likely
436 * that the sampling facility is enabled too. If the event to be
437 * initialized requires a greater sampling buffer, the allocation must
438 * be postponed. Changing the sampling buffer requires the sampling
439 * facility to be in the disabled state. So, account the number of
440 * required SDBs and let cpumsf_pmu_enable() resize the buffer just
441 * before the event is started.
442 */
443 sfb_init_allocs(n_sdb, hwc);
444 if (sf_buffer_available(cpuhw))
445 return 0;
446
447 return alloc_sampling_buffer(&cpuhw->sfb,
448 sfb_pending_allocs(&cpuhw->sfb, hwc));
449}
450
451static unsigned long min_percent(unsigned int percent, unsigned long base,
452 unsigned long min)
453{
454 return min_t(unsigned long, min, DIV_ROUND_UP(percent * base, 100));
455}
456
457static unsigned long compute_sfb_extent(unsigned long ratio, unsigned long base)
458{
459 /* Use a percentage-based approach to extend the sampling facility
460 * buffer. Accept up to 5% sample data loss.
461 * Vary the extents between 1% to 5% of the current number of
462 * sample-data-blocks.
463 */
464 if (ratio <= 5)
465 return 0;
466 if (ratio <= 25)
467 return min_percent(1, base, 1);
468 if (ratio <= 50)
469 return min_percent(1, base, 1);
470 if (ratio <= 75)
471 return min_percent(2, base, 2);
472 if (ratio <= 100)
473 return min_percent(3, base, 3);
474 if (ratio <= 250)
475 return min_percent(4, base, 4);
476
477 return min_percent(5, base, 8);
478}
479
480static void sfb_account_overflows(struct cpu_hw_sf *cpuhw,
481 struct hw_perf_event *hwc)
482{
483 unsigned long ratio, num;
484
485 if (!OVERFLOW_REG(hwc))
486 return;
487
488 /* The sample_overflow contains the average number of sample data
489 * that has been lost because sample-data-blocks were full.
490 *
491 * Calculate the total number of sample data entries that has been
492 * discarded. Then calculate the ratio of lost samples to total samples
493 * per second in percent.
494 */
495 ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb,
496 sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)));
497
498 /* Compute number of sample-data-blocks */
499 num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb);
500 if (num)
501 sfb_account_allocs(num, hwc);
502
503 OVERFLOW_REG(hwc) = 0;
504}
505
506/* extend_sampling_buffer() - Extend sampling buffer
507 * @sfb: Sampling buffer structure (for local CPU)
508 * @hwc: Perf event hardware structure
509 *
510 * Use this function to extend the sampling buffer based on the overflow counter
511 * and postponed allocation extents stored in the specified Perf event hardware.
512 *
513 * Important: This function disables the sampling facility in order to safely
514 * change the sampling buffer structure. Do not call this function
515 * when the PMU is active.
516 */
517static void extend_sampling_buffer(struct sf_buffer *sfb,
518 struct hw_perf_event *hwc)
519{
520 unsigned long num;
521
522 num = sfb_pending_allocs(sfb, hwc);
523 if (!num)
524 return;
525
526 /* Disable the sampling facility to reset any states and also
527 * clear pending measurement alerts.
528 */
529 sf_disable();
530
531 /* Extend the sampling buffer.
532 * This memory allocation typically happens in an atomic context when
533 * called by perf. Because this is a reallocation, it is fine if the
534 * new SDB-request cannot be satisfied immediately.
535 */
536 realloc_sampling_buffer(sfb, num, GFP_ATOMIC);
537}
538
539/* Number of perf events counting hardware events */
540static refcount_t num_events;
541/* Used to avoid races in calling reserve/release_cpumf_hardware */
542static DEFINE_MUTEX(pmc_reserve_mutex);
543
544#define PMC_INIT 0
545#define PMC_RELEASE 1
546static void setup_pmc_cpu(void *flags)
547{
548 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
549
550 sf_disable();
551 switch (*((int *)flags)) {
552 case PMC_INIT:
553 memset(cpuhw, 0, sizeof(*cpuhw));
554 qsi(&cpuhw->qsi);
555 cpuhw->flags |= PMU_F_RESERVED;
556 break;
557 case PMC_RELEASE:
558 cpuhw->flags &= ~PMU_F_RESERVED;
559 deallocate_buffers(cpuhw);
560 break;
561 }
562}
563
564static void release_pmc_hardware(void)
565{
566 int flags = PMC_RELEASE;
567
568 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
569 on_each_cpu(setup_pmc_cpu, &flags, 1);
570}
571
572static void reserve_pmc_hardware(void)
573{
574 int flags = PMC_INIT;
575
576 on_each_cpu(setup_pmc_cpu, &flags, 1);
577 irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
578}
579
580static void hw_perf_event_destroy(struct perf_event *event)
581{
582 /* Release PMC if this is the last perf event */
583 if (refcount_dec_and_mutex_lock(&num_events, &pmc_reserve_mutex)) {
584 release_pmc_hardware();
585 mutex_unlock(&pmc_reserve_mutex);
586 }
587}
588
589static void hw_init_period(struct hw_perf_event *hwc, u64 period)
590{
591 hwc->sample_period = period;
592 hwc->last_period = hwc->sample_period;
593 local64_set(&hwc->period_left, hwc->sample_period);
594}
595
596static unsigned long hw_limit_rate(const struct hws_qsi_info_block *si,
597 unsigned long rate)
598{
599 return clamp_t(unsigned long, rate,
600 si->min_sampl_rate, si->max_sampl_rate);
601}
602
603static u32 cpumsf_pid_type(struct perf_event *event,
604 u32 pid, enum pid_type type)
605{
606 struct task_struct *tsk;
607
608 /* Idle process */
609 if (!pid)
610 goto out;
611
612 tsk = find_task_by_pid_ns(pid, &init_pid_ns);
613 pid = -1;
614 if (tsk) {
615 /*
616 * Only top level events contain the pid namespace in which
617 * they are created.
618 */
619 if (event->parent)
620 event = event->parent;
621 pid = __task_pid_nr_ns(tsk, type, event->ns);
622 /*
623 * See also 1d953111b648
624 * "perf/core: Don't report zero PIDs for exiting tasks".
625 */
626 if (!pid && !pid_alive(tsk))
627 pid = -1;
628 }
629out:
630 return pid;
631}
632
633static void cpumsf_output_event_pid(struct perf_event *event,
634 struct perf_sample_data *data,
635 struct pt_regs *regs)
636{
637 u32 pid;
638 struct perf_event_header header;
639 struct perf_output_handle handle;
640
641 /*
642 * Obtain the PID from the basic-sampling data entry and
643 * correct the data->tid_entry.pid value.
644 */
645 pid = data->tid_entry.pid;
646
647 /* Protect callchain buffers, tasks */
648 rcu_read_lock();
649
650 perf_prepare_sample(data, event, regs);
651 perf_prepare_header(&header, data, event, regs);
652 if (perf_output_begin(&handle, data, event, header.size))
653 goto out;
654
655 /* Update the process ID (see also kernel/events/core.c) */
656 data->tid_entry.pid = cpumsf_pid_type(event, pid, PIDTYPE_TGID);
657 data->tid_entry.tid = cpumsf_pid_type(event, pid, PIDTYPE_PID);
658
659 perf_output_sample(&handle, &header, data, event);
660 perf_output_end(&handle);
661out:
662 rcu_read_unlock();
663}
664
665static unsigned long getrate(bool freq, unsigned long sample,
666 struct hws_qsi_info_block *si)
667{
668 unsigned long rate;
669
670 if (freq) {
671 rate = freq_to_sample_rate(si, sample);
672 rate = hw_limit_rate(si, rate);
673 } else {
674 /* The min/max sampling rates specifies the valid range
675 * of sample periods. If the specified sample period is
676 * out of range, limit the period to the range boundary.
677 */
678 rate = hw_limit_rate(si, sample);
679
680 /* The perf core maintains a maximum sample rate that is
681 * configurable through the sysctl interface. Ensure the
682 * sampling rate does not exceed this value. This also helps
683 * to avoid throttling when pushing samples with
684 * perf_event_overflow().
685 */
686 if (sample_rate_to_freq(si, rate) >
687 sysctl_perf_event_sample_rate) {
688 rate = 0;
689 }
690 }
691 return rate;
692}
693
694/* The sampling information (si) contains information about the
695 * min/max sampling intervals and the CPU speed. So calculate the
696 * correct sampling interval and avoid the whole period adjust
697 * feedback loop.
698 *
699 * Since the CPU Measurement sampling facility can not handle frequency
700 * calculate the sampling interval when frequency is specified using
701 * this formula:
702 * interval := cpu_speed * 1000000 / sample_freq
703 *
704 * Returns errno on bad input and zero on success with parameter interval
705 * set to the correct sampling rate.
706 *
707 * Note: This function turns off freq bit to avoid calling function
708 * perf_adjust_period(). This causes frequency adjustment in the common
709 * code part which causes tremendous variations in the counter values.
710 */
711static int __hw_perf_event_init_rate(struct perf_event *event,
712 struct hws_qsi_info_block *si)
713{
714 struct perf_event_attr *attr = &event->attr;
715 struct hw_perf_event *hwc = &event->hw;
716 unsigned long rate;
717
718 if (attr->freq) {
719 if (!attr->sample_freq)
720 return -EINVAL;
721 rate = getrate(attr->freq, attr->sample_freq, si);
722 attr->freq = 0; /* Don't call perf_adjust_period() */
723 SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_FREQ_MODE;
724 } else {
725 rate = getrate(attr->freq, attr->sample_period, si);
726 if (!rate)
727 return -EINVAL;
728 }
729 attr->sample_period = rate;
730 SAMPL_RATE(hwc) = rate;
731 hw_init_period(hwc, SAMPL_RATE(hwc));
732 return 0;
733}
734
735static int __hw_perf_event_init(struct perf_event *event)
736{
737 struct cpu_hw_sf *cpuhw;
738 struct hws_qsi_info_block si;
739 struct perf_event_attr *attr = &event->attr;
740 struct hw_perf_event *hwc = &event->hw;
741 int cpu, err = 0;
742
743 /* Reserve CPU-measurement sampling facility */
744 mutex_lock(&pmc_reserve_mutex);
745 if (!refcount_inc_not_zero(&num_events)) {
746 reserve_pmc_hardware();
747 refcount_set(&num_events, 1);
748 }
749 event->destroy = hw_perf_event_destroy;
750
751 /* Access per-CPU sampling information (query sampling info) */
752 /*
753 * The event->cpu value can be -1 to count on every CPU, for example,
754 * when attaching to a task. If this is specified, use the query
755 * sampling info from the current CPU, otherwise use event->cpu to
756 * retrieve the per-CPU information.
757 * Later, cpuhw indicates whether to allocate sampling buffers for a
758 * particular CPU (cpuhw!=NULL) or each online CPU (cpuw==NULL).
759 */
760 memset(&si, 0, sizeof(si));
761 cpuhw = NULL;
762 if (event->cpu == -1) {
763 qsi(&si);
764 } else {
765 /* Event is pinned to a particular CPU, retrieve the per-CPU
766 * sampling structure for accessing the CPU-specific QSI.
767 */
768 cpuhw = &per_cpu(cpu_hw_sf, event->cpu);
769 si = cpuhw->qsi;
770 }
771
772 /* Check sampling facility authorization and, if not authorized,
773 * fall back to other PMUs. It is safe to check any CPU because
774 * the authorization is identical for all configured CPUs.
775 */
776 if (!si.as) {
777 err = -ENOENT;
778 goto out;
779 }
780
781 if (si.ribm & CPU_MF_SF_RIBM_NOTAV) {
782 pr_warn("CPU Measurement Facility sampling is temporarily not available\n");
783 err = -EBUSY;
784 goto out;
785 }
786
787 /* Always enable basic sampling */
788 SAMPL_FLAGS(hwc) = PERF_CPUM_SF_BASIC_MODE;
789
790 /* Check if diagnostic sampling is requested. Deny if the required
791 * sampling authorization is missing.
792 */
793 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) {
794 if (!si.ad) {
795 err = -EPERM;
796 goto out;
797 }
798 SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_DIAG_MODE;
799 }
800
801 err = __hw_perf_event_init_rate(event, &si);
802 if (err)
803 goto out;
804
805 /* Use AUX buffer. No need to allocate it by ourself */
806 if (attr->config == PERF_EVENT_CPUM_SF_DIAG)
807 goto out;
808
809 /* Allocate the per-CPU sampling buffer using the CPU information
810 * from the event. If the event is not pinned to a particular
811 * CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling
812 * buffers for each online CPU.
813 */
814 if (cpuhw)
815 /* Event is pinned to a particular CPU */
816 err = allocate_buffers(cpuhw, hwc);
817 else {
818 /* Event is not pinned, allocate sampling buffer on
819 * each online CPU
820 */
821 for_each_online_cpu(cpu) {
822 cpuhw = &per_cpu(cpu_hw_sf, cpu);
823 err = allocate_buffers(cpuhw, hwc);
824 if (err)
825 break;
826 }
827 }
828
829 /* If PID/TID sampling is active, replace the default overflow
830 * handler to extract and resolve the PIDs from the basic-sampling
831 * data entries.
832 */
833 if (event->attr.sample_type & PERF_SAMPLE_TID)
834 if (is_default_overflow_handler(event))
835 event->overflow_handler = cpumsf_output_event_pid;
836out:
837 mutex_unlock(&pmc_reserve_mutex);
838 return err;
839}
840
841static bool is_callchain_event(struct perf_event *event)
842{
843 u64 sample_type = event->attr.sample_type;
844
845 return sample_type & (PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_REGS_USER |
846 PERF_SAMPLE_STACK_USER);
847}
848
849static int cpumsf_pmu_event_init(struct perf_event *event)
850{
851 int err;
852
853 /* No support for taken branch sampling */
854 /* No support for callchain, stacks and registers */
855 if (has_branch_stack(event) || is_callchain_event(event))
856 return -EOPNOTSUPP;
857
858 switch (event->attr.type) {
859 case PERF_TYPE_RAW:
860 if ((event->attr.config != PERF_EVENT_CPUM_SF) &&
861 (event->attr.config != PERF_EVENT_CPUM_SF_DIAG))
862 return -ENOENT;
863 break;
864 case PERF_TYPE_HARDWARE:
865 /* Support sampling of CPU cycles in addition to the
866 * counter facility. However, the counter facility
867 * is more precise and, hence, restrict this PMU to
868 * sampling events only.
869 */
870 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES)
871 return -ENOENT;
872 if (!is_sampling_event(event))
873 return -ENOENT;
874 break;
875 default:
876 return -ENOENT;
877 }
878
879 /* Force reset of idle/hv excludes regardless of what the
880 * user requested.
881 */
882 if (event->attr.exclude_hv)
883 event->attr.exclude_hv = 0;
884 if (event->attr.exclude_idle)
885 event->attr.exclude_idle = 0;
886
887 err = __hw_perf_event_init(event);
888 if (unlikely(err))
889 if (event->destroy)
890 event->destroy(event);
891 return err;
892}
893
894static void cpumsf_pmu_enable(struct pmu *pmu)
895{
896 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
897 struct hw_perf_event *hwc;
898 int err;
899
900 /*
901 * Event must be
902 * - added/started on this CPU (PMU_F_IN_USE set)
903 * - and CPU must be available (PMU_F_RESERVED set)
904 * - and not already enabled (PMU_F_ENABLED not set)
905 * - and not in error condition (PMU_F_ERR_MASK not set)
906 */
907 if (cpuhw->flags != (PMU_F_IN_USE | PMU_F_RESERVED))
908 return;
909
910 /* Check whether to extent the sampling buffer.
911 *
912 * Two conditions trigger an increase of the sampling buffer for a
913 * perf event:
914 * 1. Postponed buffer allocations from the event initialization.
915 * 2. Sampling overflows that contribute to pending allocations.
916 *
917 * Note that the extend_sampling_buffer() function disables the sampling
918 * facility, but it can be fully re-enabled using sampling controls that
919 * have been saved in cpumsf_pmu_disable().
920 */
921 hwc = &cpuhw->event->hw;
922 if (!(SAMPL_DIAG_MODE(hwc))) {
923 /*
924 * Account number of overflow-designated buffer extents
925 */
926 sfb_account_overflows(cpuhw, hwc);
927 extend_sampling_buffer(&cpuhw->sfb, hwc);
928 }
929 /* Rate may be adjusted with ioctl() */
930 cpuhw->lsctl.interval = SAMPL_RATE(hwc);
931
932 /* (Re)enable the PMU and sampling facility */
933 err = lsctl(&cpuhw->lsctl);
934 if (err) {
935 pr_err("Loading sampling controls failed: op 1 err %i\n", err);
936 return;
937 }
938
939 /* Load current program parameter */
940 lpp(&get_lowcore()->lpp);
941 cpuhw->flags |= PMU_F_ENABLED;
942}
943
944static void cpumsf_pmu_disable(struct pmu *pmu)
945{
946 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
947 struct hws_lsctl_request_block inactive;
948 struct hws_qsi_info_block si;
949 int err;
950
951 if (!(cpuhw->flags & PMU_F_ENABLED))
952 return;
953
954 if (cpuhw->flags & PMU_F_ERR_MASK)
955 return;
956
957 /* Switch off sampling activation control */
958 inactive = cpuhw->lsctl;
959 inactive.cs = 0;
960 inactive.cd = 0;
961
962 err = lsctl(&inactive);
963 if (err) {
964 pr_err("Loading sampling controls failed: op 2 err %i\n", err);
965 return;
966 }
967
968 /*
969 * Save state of TEAR and DEAR register contents.
970 * TEAR/DEAR values are valid only if the sampling facility is
971 * enabled. Note that cpumsf_pmu_disable() might be called even
972 * for a disabled sampling facility because cpumsf_pmu_enable()
973 * controls the enable/disable state.
974 */
975 qsi(&si);
976 if (si.es) {
977 cpuhw->lsctl.tear = si.tear;
978 cpuhw->lsctl.dear = si.dear;
979 }
980
981 cpuhw->flags &= ~PMU_F_ENABLED;
982}
983
984/* perf_exclude_event() - Filter event
985 * @event: The perf event
986 * @regs: pt_regs structure
987 * @sde_regs: Sample-data-entry (sde) regs structure
988 *
989 * Filter perf events according to their exclude specification.
990 *
991 * Return non-zero if the event shall be excluded.
992 */
993static int perf_exclude_event(struct perf_event *event, struct pt_regs *regs,
994 struct perf_sf_sde_regs *sde_regs)
995{
996 if (event->attr.exclude_user && user_mode(regs))
997 return 1;
998 if (event->attr.exclude_kernel && !user_mode(regs))
999 return 1;
1000 if (event->attr.exclude_guest && sde_regs->in_guest)
1001 return 1;
1002 if (event->attr.exclude_host && !sde_regs->in_guest)
1003 return 1;
1004 return 0;
1005}
1006
1007/* perf_push_sample() - Push samples to perf
1008 * @event: The perf event
1009 * @sample: Hardware sample data
1010 *
1011 * Use the hardware sample data to create perf event sample. The sample
1012 * is the pushed to the event subsystem and the function checks for
1013 * possible event overflows. If an event overflow occurs, the PMU is
1014 * stopped.
1015 *
1016 * Return non-zero if an event overflow occurred.
1017 */
1018static int perf_push_sample(struct perf_event *event,
1019 struct hws_basic_entry *basic)
1020{
1021 int overflow;
1022 struct pt_regs regs;
1023 struct perf_sf_sde_regs *sde_regs;
1024 struct perf_sample_data data;
1025
1026 /* Setup perf sample */
1027 perf_sample_data_init(&data, 0, event->hw.last_period);
1028
1029 /* Setup pt_regs to look like an CPU-measurement external interrupt
1030 * using the Program Request Alert code. The regs.int_parm_long
1031 * field which is unused contains additional sample-data-entry related
1032 * indicators.
1033 */
1034 memset(®s, 0, sizeof(regs));
1035 regs.int_code = 0x1407;
1036 regs.int_parm = CPU_MF_INT_SF_PRA;
1037 sde_regs = (struct perf_sf_sde_regs *) ®s.int_parm_long;
1038
1039 psw_bits(regs.psw).ia = basic->ia;
1040 psw_bits(regs.psw).dat = basic->T;
1041 psw_bits(regs.psw).wait = basic->W;
1042 psw_bits(regs.psw).pstate = basic->P;
1043 psw_bits(regs.psw).as = basic->AS;
1044
1045 /*
1046 * Use the hardware provided configuration level to decide if the
1047 * sample belongs to a guest or host. If that is not available,
1048 * fall back to the following heuristics:
1049 * A non-zero guest program parameter always indicates a guest
1050 * sample. Some early samples or samples from guests without
1051 * lpp usage would be misaccounted to the host. We use the asn
1052 * value as an addon heuristic to detect most of these guest samples.
1053 * If the value differs from 0xffff (the host value), we assume to
1054 * be a KVM guest.
1055 */
1056 switch (basic->CL) {
1057 case 1: /* logical partition */
1058 sde_regs->in_guest = 0;
1059 break;
1060 case 2: /* virtual machine */
1061 sde_regs->in_guest = 1;
1062 break;
1063 default: /* old machine, use heuristics */
1064 if (basic->gpp || basic->prim_asn != 0xffff)
1065 sde_regs->in_guest = 1;
1066 break;
1067 }
1068
1069 /*
1070 * Store the PID value from the sample-data-entry to be
1071 * processed and resolved by cpumsf_output_event_pid().
1072 */
1073 data.tid_entry.pid = basic->hpp & LPP_PID_MASK;
1074
1075 overflow = 0;
1076 if (perf_exclude_event(event, ®s, sde_regs))
1077 goto out;
1078 if (perf_event_overflow(event, &data, ®s)) {
1079 overflow = 1;
1080 event->pmu->stop(event, 0);
1081 }
1082 perf_event_update_userpage(event);
1083out:
1084 return overflow;
1085}
1086
1087static void perf_event_count_update(struct perf_event *event, u64 count)
1088{
1089 local64_add(count, &event->count);
1090}
1091
1092/* hw_collect_samples() - Walk through a sample-data-block and collect samples
1093 * @event: The perf event
1094 * @sdbt: Sample-data-block table
1095 * @overflow: Event overflow counter
1096 *
1097 * Walks through a sample-data-block and collects sampling data entries that are
1098 * then pushed to the perf event subsystem. Depending on the sampling function,
1099 * there can be either basic-sampling or combined-sampling data entries. A
1100 * combined-sampling data entry consists of a basic- and a diagnostic-sampling
1101 * data entry. The sampling function is determined by the flags in the perf
1102 * event hardware structure. The function always works with a combined-sampling
1103 * data entry but ignores the the diagnostic portion if it is not available.
1104 *
1105 * Note that the implementation focuses on basic-sampling data entries and, if
1106 * such an entry is not valid, the entire combined-sampling data entry is
1107 * ignored.
1108 *
1109 * The overflow variables counts the number of samples that has been discarded
1110 * due to a perf event overflow.
1111 */
1112static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt,
1113 unsigned long long *overflow)
1114{
1115 struct hws_trailer_entry *te;
1116 struct hws_basic_entry *sample;
1117
1118 te = trailer_entry_ptr((unsigned long)sdbt);
1119 sample = (struct hws_basic_entry *)sdbt;
1120 while ((unsigned long *)sample < (unsigned long *)te) {
1121 /* Check for an empty sample */
1122 if (!sample->def || sample->LS)
1123 break;
1124
1125 /* Update perf event period */
1126 perf_event_count_update(event, SAMPL_RATE(&event->hw));
1127
1128 /* Check whether sample is valid */
1129 if (sample->def == 0x0001) {
1130 /* If an event overflow occurred, the PMU is stopped to
1131 * throttle event delivery. Remaining sample data is
1132 * discarded.
1133 */
1134 if (!*overflow) {
1135 /* Check whether sample is consistent */
1136 if (sample->I == 0 && sample->W == 0) {
1137 /* Deliver sample data to perf */
1138 *overflow = perf_push_sample(event,
1139 sample);
1140 }
1141 } else
1142 /* Count discarded samples */
1143 *overflow += 1;
1144 } else {
1145 /* Sample slot is not yet written or other record.
1146 *
1147 * This condition can occur if the buffer was reused
1148 * from a combined basic- and diagnostic-sampling.
1149 * If only basic-sampling is then active, entries are
1150 * written into the larger diagnostic entries.
1151 * This is typically the case for sample-data-blocks
1152 * that are not full. Stop processing if the first
1153 * invalid format was detected.
1154 */
1155 if (!te->header.f)
1156 break;
1157 }
1158
1159 /* Reset sample slot and advance to next sample */
1160 sample->def = 0;
1161 sample++;
1162 }
1163}
1164
1165/* hw_perf_event_update() - Process sampling buffer
1166 * @event: The perf event
1167 * @flush_all: Flag to also flush partially filled sample-data-blocks
1168 *
1169 * Processes the sampling buffer and create perf event samples.
1170 * The sampling buffer position are retrieved and saved in the TEAR_REG
1171 * register of the specified perf event.
1172 *
1173 * Only full sample-data-blocks are processed. Specify the flush_all flag
1174 * to also walk through partially filled sample-data-blocks.
1175 */
1176static void hw_perf_event_update(struct perf_event *event, int flush_all)
1177{
1178 unsigned long long event_overflow, sampl_overflow, num_sdb;
1179 struct hw_perf_event *hwc = &event->hw;
1180 union hws_trailer_header prev, new;
1181 struct hws_trailer_entry *te;
1182 unsigned long *sdbt, sdb;
1183 int done;
1184
1185 /*
1186 * AUX buffer is used when in diagnostic sampling mode.
1187 * No perf events/samples are created.
1188 */
1189 if (SAMPL_DIAG_MODE(hwc))
1190 return;
1191
1192 sdbt = (unsigned long *)TEAR_REG(hwc);
1193 done = event_overflow = sampl_overflow = num_sdb = 0;
1194 while (!done) {
1195 /* Get the trailer entry of the sample-data-block */
1196 sdb = (unsigned long)phys_to_virt(*sdbt);
1197 te = trailer_entry_ptr(sdb);
1198
1199 /* Leave loop if no more work to do (block full indicator) */
1200 if (!te->header.f) {
1201 done = 1;
1202 if (!flush_all)
1203 break;
1204 }
1205
1206 /* Check the sample overflow count */
1207 if (te->header.overflow)
1208 /* Account sample overflows and, if a particular limit
1209 * is reached, extend the sampling buffer.
1210 * For details, see sfb_account_overflows().
1211 */
1212 sampl_overflow += te->header.overflow;
1213
1214 /* Collect all samples from a single sample-data-block and
1215 * flag if an (perf) event overflow happened. If so, the PMU
1216 * is stopped and remaining samples will be discarded.
1217 */
1218 hw_collect_samples(event, (unsigned long *)sdb, &event_overflow);
1219 num_sdb++;
1220
1221 /* Reset trailer (using compare-double-and-swap) */
1222 prev.val = READ_ONCE_ALIGNED_128(te->header.val);
1223 do {
1224 new.val = prev.val;
1225 new.f = 0;
1226 new.a = 1;
1227 new.overflow = 0;
1228 } while (!try_cmpxchg128(&te->header.val, &prev.val, new.val));
1229
1230 /* Advance to next sample-data-block */
1231 sdbt++;
1232 if (is_link_entry(sdbt))
1233 sdbt = get_next_sdbt(sdbt);
1234
1235 /* Update event hardware registers */
1236 TEAR_REG(hwc) = (unsigned long)sdbt;
1237
1238 /* Stop processing sample-data if all samples of the current
1239 * sample-data-block were flushed even if it was not full.
1240 */
1241 if (flush_all && done)
1242 break;
1243 }
1244
1245 /* Account sample overflows in the event hardware structure */
1246 if (sampl_overflow)
1247 OVERFLOW_REG(hwc) = DIV_ROUND_UP(OVERFLOW_REG(hwc) +
1248 sampl_overflow, 1 + num_sdb);
1249
1250 /* Perf_event_overflow() and perf_event_account_interrupt() limit
1251 * the interrupt rate to an upper limit. Roughly 1000 samples per
1252 * task tick.
1253 * Hitting this limit results in a large number
1254 * of throttled REF_REPORT_THROTTLE entries and the samples
1255 * are dropped.
1256 * Slightly increase the interval to avoid hitting this limit.
1257 */
1258 if (event_overflow)
1259 SAMPL_RATE(hwc) += DIV_ROUND_UP(SAMPL_RATE(hwc), 10);
1260}
1261
1262static inline unsigned long aux_sdb_index(struct aux_buffer *aux,
1263 unsigned long i)
1264{
1265 return i % aux->sfb.num_sdb;
1266}
1267
1268static inline unsigned long aux_sdb_num(unsigned long start, unsigned long end)
1269{
1270 return end >= start ? end - start + 1 : 0;
1271}
1272
1273static inline unsigned long aux_sdb_num_alert(struct aux_buffer *aux)
1274{
1275 return aux_sdb_num(aux->head, aux->alert_mark);
1276}
1277
1278static inline unsigned long aux_sdb_num_empty(struct aux_buffer *aux)
1279{
1280 return aux_sdb_num(aux->head, aux->empty_mark);
1281}
1282
1283/*
1284 * Get trailer entry by index of SDB.
1285 */
1286static struct hws_trailer_entry *aux_sdb_trailer(struct aux_buffer *aux,
1287 unsigned long index)
1288{
1289 unsigned long sdb;
1290
1291 index = aux_sdb_index(aux, index);
1292 sdb = aux->sdb_index[index];
1293 return trailer_entry_ptr(sdb);
1294}
1295
1296/*
1297 * Finish sampling on the cpu. Called by cpumsf_pmu_del() with pmu
1298 * disabled. Collect the full SDBs in AUX buffer which have not reached
1299 * the point of alert indicator. And ignore the SDBs which are not
1300 * full.
1301 *
1302 * 1. Scan SDBs to see how much data is there and consume them.
1303 * 2. Remove alert indicator in the buffer.
1304 */
1305static void aux_output_end(struct perf_output_handle *handle)
1306{
1307 unsigned long i, range_scan, idx;
1308 struct aux_buffer *aux;
1309 struct hws_trailer_entry *te;
1310
1311 aux = perf_get_aux(handle);
1312 if (!aux)
1313 return;
1314
1315 range_scan = aux_sdb_num_alert(aux);
1316 for (i = 0, idx = aux->head; i < range_scan; i++, idx++) {
1317 te = aux_sdb_trailer(aux, idx);
1318 if (!te->header.f)
1319 break;
1320 }
1321 /* i is num of SDBs which are full */
1322 perf_aux_output_end(handle, i << PAGE_SHIFT);
1323
1324 /* Remove alert indicators in the buffer */
1325 te = aux_sdb_trailer(aux, aux->alert_mark);
1326 te->header.a = 0;
1327}
1328
1329/*
1330 * Start sampling on the CPU. Called by cpumsf_pmu_add() when an event
1331 * is first added to the CPU or rescheduled again to the CPU. It is called
1332 * with pmu disabled.
1333 *
1334 * 1. Reset the trailer of SDBs to get ready for new data.
1335 * 2. Tell the hardware where to put the data by reset the SDBs buffer
1336 * head(tear/dear).
1337 */
1338static int aux_output_begin(struct perf_output_handle *handle,
1339 struct aux_buffer *aux,
1340 struct cpu_hw_sf *cpuhw)
1341{
1342 unsigned long range, i, range_scan, idx, head, base, offset;
1343 struct hws_trailer_entry *te;
1344
1345 if (handle->head & ~PAGE_MASK)
1346 return -EINVAL;
1347
1348 aux->head = handle->head >> PAGE_SHIFT;
1349 range = (handle->size + 1) >> PAGE_SHIFT;
1350 if (range <= 1)
1351 return -ENOMEM;
1352
1353 /*
1354 * SDBs between aux->head and aux->empty_mark are already ready
1355 * for new data. range_scan is num of SDBs not within them.
1356 */
1357 if (range > aux_sdb_num_empty(aux)) {
1358 range_scan = range - aux_sdb_num_empty(aux);
1359 idx = aux->empty_mark + 1;
1360 for (i = 0; i < range_scan; i++, idx++) {
1361 te = aux_sdb_trailer(aux, idx);
1362 te->header.f = 0;
1363 te->header.a = 0;
1364 te->header.overflow = 0;
1365 }
1366 /* Save the position of empty SDBs */
1367 aux->empty_mark = aux->head + range - 1;
1368 }
1369
1370 /* Set alert indicator */
1371 aux->alert_mark = aux->head + range/2 - 1;
1372 te = aux_sdb_trailer(aux, aux->alert_mark);
1373 te->header.a = 1;
1374
1375 /* Reset hardware buffer head */
1376 head = aux_sdb_index(aux, aux->head);
1377 base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE];
1378 offset = head % CPUM_SF_SDB_PER_TABLE;
1379 cpuhw->lsctl.tear = virt_to_phys((void *)base) + offset * sizeof(unsigned long);
1380 cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]);
1381
1382 return 0;
1383}
1384
1385/*
1386 * Set alert indicator on SDB at index @alert_index while sampler is running.
1387 *
1388 * Return true if successfully.
1389 * Return false if full indicator is already set by hardware sampler.
1390 */
1391static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index,
1392 unsigned long long *overflow)
1393{
1394 union hws_trailer_header prev, new;
1395 struct hws_trailer_entry *te;
1396
1397 te = aux_sdb_trailer(aux, alert_index);
1398 prev.val = READ_ONCE_ALIGNED_128(te->header.val);
1399 do {
1400 new.val = prev.val;
1401 *overflow = prev.overflow;
1402 if (prev.f) {
1403 /*
1404 * SDB is already set by hardware.
1405 * Abort and try to set somewhere
1406 * behind.
1407 */
1408 return false;
1409 }
1410 new.a = 1;
1411 new.overflow = 0;
1412 } while (!try_cmpxchg128(&te->header.val, &prev.val, new.val));
1413 return true;
1414}
1415
1416/*
1417 * aux_reset_buffer() - Scan and setup SDBs for new samples
1418 * @aux: The AUX buffer to set
1419 * @range: The range of SDBs to scan started from aux->head
1420 * @overflow: Set to overflow count
1421 *
1422 * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1423 * marked as empty, check if it is already set full by the hardware sampler.
1424 * If yes, that means new data is already there before we can set an alert
1425 * indicator. Caller should try to set alert indicator to some position behind.
1426 *
1427 * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1428 * previously and have already been consumed by user space. Reset these SDBs
1429 * (clear full indicator and alert indicator) for new data.
1430 * If aux->alert_mark fall in this area, just set it. Overflow count is
1431 * recorded while scanning.
1432 *
1433 * SDBs between aux->head and aux->empty_mark are already reset at last time.
1434 * and ready for new samples. So scanning on this area could be skipped.
1435 *
1436 * Return true if alert indicator is set successfully and false if not.
1437 */
1438static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range,
1439 unsigned long long *overflow)
1440{
1441 union hws_trailer_header prev, new;
1442 unsigned long i, range_scan, idx;
1443 unsigned long long orig_overflow;
1444 struct hws_trailer_entry *te;
1445
1446 if (range <= aux_sdb_num_empty(aux))
1447 /*
1448 * No need to scan. All SDBs in range are marked as empty.
1449 * Just set alert indicator. Should check race with hardware
1450 * sampler.
1451 */
1452 return aux_set_alert(aux, aux->alert_mark, overflow);
1453
1454 if (aux->alert_mark <= aux->empty_mark)
1455 /*
1456 * Set alert indicator on empty SDB. Should check race
1457 * with hardware sampler.
1458 */
1459 if (!aux_set_alert(aux, aux->alert_mark, overflow))
1460 return false;
1461
1462 /*
1463 * Scan the SDBs to clear full and alert indicator used previously.
1464 * Start scanning from one SDB behind empty_mark. If the new alert
1465 * indicator fall into this range, set it.
1466 */
1467 range_scan = range - aux_sdb_num_empty(aux);
1468 idx = aux->empty_mark + 1;
1469 for (i = 0; i < range_scan; i++, idx++) {
1470 te = aux_sdb_trailer(aux, idx);
1471 prev.val = READ_ONCE_ALIGNED_128(te->header.val);
1472 do {
1473 new.val = prev.val;
1474 orig_overflow = prev.overflow;
1475 new.f = 0;
1476 new.overflow = 0;
1477 if (idx == aux->alert_mark)
1478 new.a = 1;
1479 else
1480 new.a = 0;
1481 } while (!try_cmpxchg128(&te->header.val, &prev.val, new.val));
1482 *overflow += orig_overflow;
1483 }
1484
1485 /* Update empty_mark to new position */
1486 aux->empty_mark = aux->head + range - 1;
1487
1488 return true;
1489}
1490
1491/*
1492 * Measurement alert handler for diagnostic mode sampling.
1493 */
1494static void hw_collect_aux(struct cpu_hw_sf *cpuhw)
1495{
1496 struct aux_buffer *aux;
1497 int done = 0;
1498 unsigned long range = 0, size;
1499 unsigned long long overflow = 0;
1500 struct perf_output_handle *handle = &cpuhw->handle;
1501 unsigned long num_sdb;
1502
1503 aux = perf_get_aux(handle);
1504 if (!aux)
1505 return;
1506
1507 /* Inform user space new data arrived */
1508 size = aux_sdb_num_alert(aux) << PAGE_SHIFT;
1509 debug_sprintf_event(sfdbg, 6, "%s #alert %ld\n", __func__,
1510 size >> PAGE_SHIFT);
1511 perf_aux_output_end(handle, size);
1512
1513 num_sdb = aux->sfb.num_sdb;
1514 while (!done) {
1515 /* Get an output handle */
1516 aux = perf_aux_output_begin(handle, cpuhw->event);
1517 if (handle->size == 0) {
1518 pr_err("The AUX buffer with %lu pages for the "
1519 "diagnostic-sampling mode is full\n",
1520 num_sdb);
1521 break;
1522 }
1523 if (!aux)
1524 return;
1525
1526 /* Update head and alert_mark to new position */
1527 aux->head = handle->head >> PAGE_SHIFT;
1528 range = (handle->size + 1) >> PAGE_SHIFT;
1529 if (range == 1)
1530 aux->alert_mark = aux->head;
1531 else
1532 aux->alert_mark = aux->head + range/2 - 1;
1533
1534 if (aux_reset_buffer(aux, range, &overflow)) {
1535 if (!overflow) {
1536 done = 1;
1537 break;
1538 }
1539 size = range << PAGE_SHIFT;
1540 perf_aux_output_end(&cpuhw->handle, size);
1541 pr_err("Sample data caused the AUX buffer with %lu "
1542 "pages to overflow\n", aux->sfb.num_sdb);
1543 } else {
1544 size = aux_sdb_num_alert(aux) << PAGE_SHIFT;
1545 perf_aux_output_end(&cpuhw->handle, size);
1546 }
1547 }
1548}
1549
1550/*
1551 * Callback when freeing AUX buffers.
1552 */
1553static void aux_buffer_free(void *data)
1554{
1555 struct aux_buffer *aux = data;
1556 unsigned long i, num_sdbt;
1557
1558 if (!aux)
1559 return;
1560
1561 /* Free SDBT. SDB is freed by the caller */
1562 num_sdbt = aux->sfb.num_sdbt;
1563 for (i = 0; i < num_sdbt; i++)
1564 free_page(aux->sdbt_index[i]);
1565
1566 kfree(aux->sdbt_index);
1567 kfree(aux->sdb_index);
1568 kfree(aux);
1569}
1570
1571static void aux_sdb_init(unsigned long sdb)
1572{
1573 struct hws_trailer_entry *te;
1574
1575 te = trailer_entry_ptr(sdb);
1576
1577 /* Save clock base */
1578 te->clock_base = 1;
1579 te->progusage2 = tod_clock_base.tod;
1580}
1581
1582/*
1583 * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1584 * @event: Event the buffer is setup for, event->cpu == -1 means current
1585 * @pages: Array of pointers to buffer pages passed from perf core
1586 * @nr_pages: Total pages
1587 * @snapshot: Flag for snapshot mode
1588 *
1589 * This is the callback when setup an event using AUX buffer. Perf tool can
1590 * trigger this by an additional mmap() call on the event. Unlike the buffer
1591 * for basic samples, AUX buffer belongs to the event. It is scheduled with
1592 * the task among online cpus when it is a per-thread event.
1593 *
1594 * Return the private AUX buffer structure if success or NULL if fails.
1595 */
1596static void *aux_buffer_setup(struct perf_event *event, void **pages,
1597 int nr_pages, bool snapshot)
1598{
1599 struct sf_buffer *sfb;
1600 struct aux_buffer *aux;
1601 unsigned long *new, *tail;
1602 int i, n_sdbt;
1603
1604 if (!nr_pages || !pages)
1605 return NULL;
1606
1607 if (nr_pages > CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR) {
1608 pr_err("AUX buffer size (%i pages) is larger than the "
1609 "maximum sampling buffer limit\n",
1610 nr_pages);
1611 return NULL;
1612 } else if (nr_pages < CPUM_SF_MIN_SDB * CPUM_SF_SDB_DIAG_FACTOR) {
1613 pr_err("AUX buffer size (%i pages) is less than the "
1614 "minimum sampling buffer limit\n",
1615 nr_pages);
1616 return NULL;
1617 }
1618
1619 /* Allocate aux_buffer struct for the event */
1620 aux = kzalloc(sizeof(struct aux_buffer), GFP_KERNEL);
1621 if (!aux)
1622 goto no_aux;
1623 sfb = &aux->sfb;
1624
1625 /* Allocate sdbt_index for fast reference */
1626 n_sdbt = DIV_ROUND_UP(nr_pages, CPUM_SF_SDB_PER_TABLE);
1627 aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL);
1628 if (!aux->sdbt_index)
1629 goto no_sdbt_index;
1630
1631 /* Allocate sdb_index for fast reference */
1632 aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL);
1633 if (!aux->sdb_index)
1634 goto no_sdb_index;
1635
1636 /* Allocate the first SDBT */
1637 sfb->num_sdbt = 0;
1638 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
1639 if (!sfb->sdbt)
1640 goto no_sdbt;
1641 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;
1642 tail = sfb->tail = sfb->sdbt;
1643
1644 /*
1645 * Link the provided pages of AUX buffer to SDBT.
1646 * Allocate SDBT if needed.
1647 */
1648 for (i = 0; i < nr_pages; i++, tail++) {
1649 if (require_table_link(tail)) {
1650 new = (unsigned long *)get_zeroed_page(GFP_KERNEL);
1651 if (!new)
1652 goto no_sdbt;
1653 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;
1654 /* Link current page to tail of chain */
1655 *tail = virt_to_phys(new) + 1;
1656 tail = new;
1657 }
1658 /* Tail is the entry in a SDBT */
1659 *tail = virt_to_phys(pages[i]);
1660 aux->sdb_index[i] = (unsigned long)pages[i];
1661 aux_sdb_init((unsigned long)pages[i]);
1662 }
1663 sfb->num_sdb = nr_pages;
1664
1665 /* Link the last entry in the SDBT to the first SDBT */
1666 *tail = virt_to_phys(sfb->sdbt) + 1;
1667 sfb->tail = tail;
1668
1669 /*
1670 * Initial all SDBs are zeroed. Mark it as empty.
1671 * So there is no need to clear the full indicator
1672 * when this event is first added.
1673 */
1674 aux->empty_mark = sfb->num_sdb - 1;
1675
1676 return aux;
1677
1678no_sdbt:
1679 /* SDBs (AUX buffer pages) are freed by caller */
1680 for (i = 0; i < sfb->num_sdbt; i++)
1681 free_page(aux->sdbt_index[i]);
1682 kfree(aux->sdb_index);
1683no_sdb_index:
1684 kfree(aux->sdbt_index);
1685no_sdbt_index:
1686 kfree(aux);
1687no_aux:
1688 return NULL;
1689}
1690
1691static void cpumsf_pmu_read(struct perf_event *event)
1692{
1693 /* Nothing to do ... updates are interrupt-driven */
1694}
1695
1696/* Check if the new sampling period/frequency is appropriate.
1697 *
1698 * Return non-zero on error and zero on passed checks.
1699 */
1700static int cpumsf_pmu_check_period(struct perf_event *event, u64 value)
1701{
1702 struct hws_qsi_info_block si;
1703 unsigned long rate;
1704 bool do_freq;
1705
1706 memset(&si, 0, sizeof(si));
1707 if (event->cpu == -1) {
1708 qsi(&si);
1709 } else {
1710 /* Event is pinned to a particular CPU, retrieve the per-CPU
1711 * sampling structure for accessing the CPU-specific QSI.
1712 */
1713 struct cpu_hw_sf *cpuhw = &per_cpu(cpu_hw_sf, event->cpu);
1714
1715 si = cpuhw->qsi;
1716 }
1717
1718 do_freq = !!SAMPL_FREQ_MODE(&event->hw);
1719 rate = getrate(do_freq, value, &si);
1720 if (!rate)
1721 return -EINVAL;
1722
1723 event->attr.sample_period = rate;
1724 SAMPL_RATE(&event->hw) = rate;
1725 hw_init_period(&event->hw, SAMPL_RATE(&event->hw));
1726 return 0;
1727}
1728
1729/* Activate sampling control.
1730 * Next call of pmu_enable() starts sampling.
1731 */
1732static void cpumsf_pmu_start(struct perf_event *event, int flags)
1733{
1734 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1735
1736 if (!(event->hw.state & PERF_HES_STOPPED))
1737 return;
1738 perf_pmu_disable(event->pmu);
1739 event->hw.state = 0;
1740 cpuhw->lsctl.cs = 1;
1741 if (SAMPL_DIAG_MODE(&event->hw))
1742 cpuhw->lsctl.cd = 1;
1743 perf_pmu_enable(event->pmu);
1744}
1745
1746/* Deactivate sampling control.
1747 * Next call of pmu_enable() stops sampling.
1748 */
1749static void cpumsf_pmu_stop(struct perf_event *event, int flags)
1750{
1751 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1752
1753 if (event->hw.state & PERF_HES_STOPPED)
1754 return;
1755
1756 perf_pmu_disable(event->pmu);
1757 cpuhw->lsctl.cs = 0;
1758 cpuhw->lsctl.cd = 0;
1759 event->hw.state |= PERF_HES_STOPPED;
1760
1761 if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) {
1762 /* CPU hotplug off removes SDBs. No samples to extract. */
1763 if (cpuhw->flags & PMU_F_RESERVED)
1764 hw_perf_event_update(event, 1);
1765 event->hw.state |= PERF_HES_UPTODATE;
1766 }
1767 perf_pmu_enable(event->pmu);
1768}
1769
1770static int cpumsf_pmu_add(struct perf_event *event, int flags)
1771{
1772 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1773 struct aux_buffer *aux;
1774 int err = 0;
1775
1776 if (cpuhw->flags & PMU_F_IN_USE)
1777 return -EAGAIN;
1778
1779 if (!SAMPL_DIAG_MODE(&event->hw) && !sf_buffer_available(cpuhw))
1780 return -EINVAL;
1781
1782 perf_pmu_disable(event->pmu);
1783
1784 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1785
1786 /* Set up sampling controls. Always program the sampling register
1787 * using the SDB-table start. Reset TEAR_REG event hardware register
1788 * that is used by hw_perf_event_update() to store the sampling buffer
1789 * position after samples have been flushed.
1790 */
1791 cpuhw->lsctl.s = 0;
1792 cpuhw->lsctl.h = 1;
1793 cpuhw->lsctl.interval = SAMPL_RATE(&event->hw);
1794 if (!SAMPL_DIAG_MODE(&event->hw)) {
1795 cpuhw->lsctl.tear = virt_to_phys(cpuhw->sfb.sdbt);
1796 cpuhw->lsctl.dear = *(unsigned long *)cpuhw->sfb.sdbt;
1797 TEAR_REG(&event->hw) = (unsigned long)cpuhw->sfb.sdbt;
1798 }
1799
1800 /* Ensure sampling functions are in the disabled state. If disabled,
1801 * switch on sampling enable control. */
1802 if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) {
1803 err = -EAGAIN;
1804 goto out;
1805 }
1806 if (SAMPL_DIAG_MODE(&event->hw)) {
1807 aux = perf_aux_output_begin(&cpuhw->handle, event);
1808 if (!aux) {
1809 err = -EINVAL;
1810 goto out;
1811 }
1812 err = aux_output_begin(&cpuhw->handle, aux, cpuhw);
1813 if (err)
1814 goto out;
1815 cpuhw->lsctl.ed = 1;
1816 }
1817 cpuhw->lsctl.es = 1;
1818
1819 /* Set in_use flag and store event */
1820 cpuhw->event = event;
1821 cpuhw->flags |= PMU_F_IN_USE;
1822
1823 if (flags & PERF_EF_START)
1824 cpumsf_pmu_start(event, PERF_EF_RELOAD);
1825out:
1826 perf_event_update_userpage(event);
1827 perf_pmu_enable(event->pmu);
1828 return err;
1829}
1830
1831static void cpumsf_pmu_del(struct perf_event *event, int flags)
1832{
1833 struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1834
1835 perf_pmu_disable(event->pmu);
1836 cpumsf_pmu_stop(event, PERF_EF_UPDATE);
1837
1838 cpuhw->lsctl.es = 0;
1839 cpuhw->lsctl.ed = 0;
1840 cpuhw->flags &= ~PMU_F_IN_USE;
1841 cpuhw->event = NULL;
1842
1843 if (SAMPL_DIAG_MODE(&event->hw))
1844 aux_output_end(&cpuhw->handle);
1845 perf_event_update_userpage(event);
1846 perf_pmu_enable(event->pmu);
1847}
1848
1849CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC, PERF_EVENT_CPUM_SF);
1850CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC_DIAG, PERF_EVENT_CPUM_SF_DIAG);
1851
1852/* Attribute list for CPU_SF.
1853 *
1854 * The availablitiy depends on the CPU_MF sampling facility authorization
1855 * for basic + diagnositic samples. This is determined at initialization
1856 * time by the sampling facility device driver.
1857 * If the authorization for basic samples is turned off, it should be
1858 * also turned off for diagnostic sampling.
1859 *
1860 * During initialization of the device driver, check the authorization
1861 * level for diagnostic sampling and installs the attribute
1862 * file for diagnostic sampling if necessary.
1863 *
1864 * For now install a placeholder to reference all possible attributes:
1865 * SF_CYCLES_BASIC and SF_CYCLES_BASIC_DIAG.
1866 * Add another entry for the final NULL pointer.
1867 */
1868enum {
1869 SF_CYCLES_BASIC_ATTR_IDX = 0,
1870 SF_CYCLES_BASIC_DIAG_ATTR_IDX,
1871 SF_CYCLES_ATTR_MAX
1872};
1873
1874static struct attribute *cpumsf_pmu_events_attr[SF_CYCLES_ATTR_MAX + 1] = {
1875 [SF_CYCLES_BASIC_ATTR_IDX] = CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC)
1876};
1877
1878PMU_FORMAT_ATTR(event, "config:0-63");
1879
1880static struct attribute *cpumsf_pmu_format_attr[] = {
1881 &format_attr_event.attr,
1882 NULL,
1883};
1884
1885static struct attribute_group cpumsf_pmu_events_group = {
1886 .name = "events",
1887 .attrs = cpumsf_pmu_events_attr,
1888};
1889
1890static struct attribute_group cpumsf_pmu_format_group = {
1891 .name = "format",
1892 .attrs = cpumsf_pmu_format_attr,
1893};
1894
1895static const struct attribute_group *cpumsf_pmu_attr_groups[] = {
1896 &cpumsf_pmu_events_group,
1897 &cpumsf_pmu_format_group,
1898 NULL,
1899};
1900
1901static struct pmu cpumf_sampling = {
1902 .pmu_enable = cpumsf_pmu_enable,
1903 .pmu_disable = cpumsf_pmu_disable,
1904
1905 .event_init = cpumsf_pmu_event_init,
1906 .add = cpumsf_pmu_add,
1907 .del = cpumsf_pmu_del,
1908
1909 .start = cpumsf_pmu_start,
1910 .stop = cpumsf_pmu_stop,
1911 .read = cpumsf_pmu_read,
1912
1913 .attr_groups = cpumsf_pmu_attr_groups,
1914
1915 .setup_aux = aux_buffer_setup,
1916 .free_aux = aux_buffer_free,
1917
1918 .check_period = cpumsf_pmu_check_period,
1919};
1920
1921static void cpumf_measurement_alert(struct ext_code ext_code,
1922 unsigned int alert, unsigned long unused)
1923{
1924 struct cpu_hw_sf *cpuhw;
1925
1926 if (!(alert & CPU_MF_INT_SF_MASK))
1927 return;
1928 inc_irq_stat(IRQEXT_CMS);
1929 cpuhw = this_cpu_ptr(&cpu_hw_sf);
1930
1931 /* Measurement alerts are shared and might happen when the PMU
1932 * is not reserved. Ignore these alerts in this case. */
1933 if (!(cpuhw->flags & PMU_F_RESERVED))
1934 return;
1935
1936 /* The processing below must take care of multiple alert events that
1937 * might be indicated concurrently. */
1938
1939 /* Program alert request */
1940 if (alert & CPU_MF_INT_SF_PRA) {
1941 if (cpuhw->flags & PMU_F_IN_USE) {
1942 if (SAMPL_DIAG_MODE(&cpuhw->event->hw))
1943 hw_collect_aux(cpuhw);
1944 else
1945 hw_perf_event_update(cpuhw->event, 0);
1946 }
1947 }
1948
1949 /* Report measurement alerts only for non-PRA codes */
1950 if (alert != CPU_MF_INT_SF_PRA)
1951 debug_sprintf_event(sfdbg, 6, "%s alert %#x\n", __func__,
1952 alert);
1953
1954 /* Sampling authorization change request */
1955 if (alert & CPU_MF_INT_SF_SACA)
1956 qsi(&cpuhw->qsi);
1957
1958 /* Loss of sample data due to high-priority machine activities */
1959 if (alert & CPU_MF_INT_SF_LSDA) {
1960 pr_err("Sample data was lost\n");
1961 cpuhw->flags |= PMU_F_ERR_LSDA;
1962 sf_disable();
1963 }
1964
1965 /* Invalid sampling buffer entry */
1966 if (alert & (CPU_MF_INT_SF_IAE|CPU_MF_INT_SF_ISE)) {
1967 pr_err("A sampling buffer entry is incorrect (alert=%#x)\n",
1968 alert);
1969 cpuhw->flags |= PMU_F_ERR_IBE;
1970 sf_disable();
1971 }
1972}
1973
1974static int cpusf_pmu_setup(unsigned int cpu, int flags)
1975{
1976 /* Ignore the notification if no events are scheduled on the PMU.
1977 * This might be racy...
1978 */
1979 if (!refcount_read(&num_events))
1980 return 0;
1981
1982 local_irq_disable();
1983 setup_pmc_cpu(&flags);
1984 local_irq_enable();
1985 return 0;
1986}
1987
1988static int s390_pmu_sf_online_cpu(unsigned int cpu)
1989{
1990 return cpusf_pmu_setup(cpu, PMC_INIT);
1991}
1992
1993static int s390_pmu_sf_offline_cpu(unsigned int cpu)
1994{
1995 return cpusf_pmu_setup(cpu, PMC_RELEASE);
1996}
1997
1998static int param_get_sfb_size(char *buffer, const struct kernel_param *kp)
1999{
2000 if (!cpum_sf_avail())
2001 return -ENODEV;
2002 return sprintf(buffer, "%lu,%lu", CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB);
2003}
2004
2005static int param_set_sfb_size(const char *val, const struct kernel_param *kp)
2006{
2007 int rc;
2008 unsigned long min, max;
2009
2010 if (!cpum_sf_avail())
2011 return -ENODEV;
2012 if (!val || !strlen(val))
2013 return -EINVAL;
2014
2015 /* Valid parameter values: "min,max" or "max" */
2016 min = CPUM_SF_MIN_SDB;
2017 max = CPUM_SF_MAX_SDB;
2018 if (strchr(val, ','))
2019 rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL;
2020 else
2021 rc = kstrtoul(val, 10, &max);
2022
2023 if (min < 2 || min >= max || max > get_num_physpages())
2024 rc = -EINVAL;
2025 if (rc)
2026 return rc;
2027
2028 sfb_set_limits(min, max);
2029 pr_info("The sampling buffer limits have changed to: "
2030 "min %lu max %lu (diag %lu)\n",
2031 CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB, CPUM_SF_SDB_DIAG_FACTOR);
2032 return 0;
2033}
2034
2035#define param_check_sfb_size(name, p) __param_check(name, p, void)
2036static const struct kernel_param_ops param_ops_sfb_size = {
2037 .set = param_set_sfb_size,
2038 .get = param_get_sfb_size,
2039};
2040
2041enum {
2042 RS_INIT_FAILURE_BSDES = 2, /* Bad basic sampling size */
2043 RS_INIT_FAILURE_ALRT = 3, /* IRQ registration failure */
2044 RS_INIT_FAILURE_PERF = 4 /* PMU registration failure */
2045};
2046
2047static void __init pr_cpumsf_err(unsigned int reason)
2048{
2049 pr_err("Sampling facility support for perf is not available: "
2050 "reason %#x\n", reason);
2051}
2052
2053static int __init init_cpum_sampling_pmu(void)
2054{
2055 struct hws_qsi_info_block si;
2056 int err;
2057
2058 if (!cpum_sf_avail())
2059 return -ENODEV;
2060
2061 memset(&si, 0, sizeof(si));
2062 qsi(&si);
2063 if (!si.as && !si.ad)
2064 return -ENODEV;
2065
2066 if (si.bsdes != sizeof(struct hws_basic_entry)) {
2067 pr_cpumsf_err(RS_INIT_FAILURE_BSDES);
2068 return -EINVAL;
2069 }
2070
2071 if (si.ad) {
2072 sfb_set_limits(CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB);
2073 /* Sampling of diagnostic data authorized,
2074 * install event into attribute list of PMU device.
2075 */
2076 cpumsf_pmu_events_attr[SF_CYCLES_BASIC_DIAG_ATTR_IDX] =
2077 CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC_DIAG);
2078 }
2079
2080 sfdbg = debug_register(KMSG_COMPONENT, 2, 1, 80);
2081 if (!sfdbg) {
2082 pr_err("Registering for s390dbf failed\n");
2083 return -ENOMEM;
2084 }
2085 debug_register_view(sfdbg, &debug_sprintf_view);
2086
2087 err = register_external_irq(EXT_IRQ_MEASURE_ALERT,
2088 cpumf_measurement_alert);
2089 if (err) {
2090 pr_cpumsf_err(RS_INIT_FAILURE_ALRT);
2091 debug_unregister(sfdbg);
2092 goto out;
2093 }
2094
2095 err = perf_pmu_register(&cpumf_sampling, "cpum_sf", PERF_TYPE_RAW);
2096 if (err) {
2097 pr_cpumsf_err(RS_INIT_FAILURE_PERF);
2098 unregister_external_irq(EXT_IRQ_MEASURE_ALERT,
2099 cpumf_measurement_alert);
2100 debug_unregister(sfdbg);
2101 goto out;
2102 }
2103
2104 cpuhp_setup_state(CPUHP_AP_PERF_S390_SF_ONLINE, "perf/s390/sf:online",
2105 s390_pmu_sf_online_cpu, s390_pmu_sf_offline_cpu);
2106out:
2107 return err;
2108}
2109
2110arch_initcall(init_cpum_sampling_pmu);
2111core_param(cpum_sfb_size, CPUM_SF_MAX_SDB, sfb_size, 0644);