1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Performance event support for the System z CPU-measurement Sampling Facility
   4 *
   5 * Copyright IBM Corp. 2013, 2018
   6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
   7 */
   8#define KMSG_COMPONENT	"cpum_sf"
   9#define pr_fmt(fmt)	KMSG_COMPONENT ": " fmt
  10
  11#include <linux/kernel.h>
  12#include <linux/kernel_stat.h>
  13#include <linux/perf_event.h>
  14#include <linux/percpu.h>
  15#include <linux/pid.h>
  16#include <linux/notifier.h>
  17#include <linux/export.h>
  18#include <linux/slab.h>
  19#include <linux/mm.h>
  20#include <linux/moduleparam.h>
  21#include <asm/cpu_mf.h>
  22#include <asm/irq.h>
  23#include <asm/debug.h>
  24#include <asm/timex.h>
  25#include <linux/io.h>
  26
  27/* Minimum number of sample-data-block-tables:
  28 * At least one table is required for the sampling buffer structure.
  29 * A single table contains up to 511 pointers to sample-data-blocks.
  30 */
  31#define CPUM_SF_MIN_SDBT	1
  32
  33/* Number of sample-data-blocks per sample-data-block-table (SDBT):
  34 * A table contains SDB pointers (8 bytes) and one table-link entry
  35 * that points to the origin of the next SDBT.
  36 */
  37#define CPUM_SF_SDB_PER_TABLE	((PAGE_SIZE - 8) / 8)
  38
  39/* Maximum page offset for an SDBT table-link entry:
  40 * If this page offset is reached, a table-link entry to the next SDBT
  41 * must be added.
  42 */
  43#define CPUM_SF_SDBT_TL_OFFSET	(CPUM_SF_SDB_PER_TABLE * 8)
  44static inline int require_table_link(const void *sdbt)
  45{
  46	return ((unsigned long)sdbt & ~PAGE_MASK) == CPUM_SF_SDBT_TL_OFFSET;
  47}
  48
  49/* Minimum and maximum sampling buffer sizes:
  50 *
  51 * This number represents the maximum size of the sampling buffer taking
  52 * the number of sample-data-block-tables into account.  Note that these
  53 * numbers apply to the basic-sampling function only.
  54 * The maximum number of SDBs is increased by CPUM_SF_SDB_DIAG_FACTOR if
  55 * the diagnostic-sampling function is active.
  56 *
  57 * Sampling buffer size		Buffer characteristics
  58 * ---------------------------------------------------
  59 *	 64KB		    ==	  16 pages (4KB per page)
  60 *				   1 page  for SDB-tables
  61 *				  15 pages for SDBs
  62 *
  63 *  32MB		    ==	8192 pages (4KB per page)
  64 *				  16 pages for SDB-tables
  65 *				8176 pages for SDBs
  66 */
  67static unsigned long __read_mostly CPUM_SF_MIN_SDB = 15;
  68static unsigned long __read_mostly CPUM_SF_MAX_SDB = 8176;
  69static unsigned long __read_mostly CPUM_SF_SDB_DIAG_FACTOR = 1;
  70
  71struct sf_buffer {
  72	unsigned long	 *sdbt;	    /* Sample-data-block-table origin */
  73	/* buffer characteristics (required for buffer increments) */
  74	unsigned long  num_sdb;	    /* Number of sample-data-blocks */
  75	unsigned long num_sdbt;	    /* Number of sample-data-block-tables */
  76	unsigned long	 *tail;	    /* last sample-data-block-table */
  77};
  78
  79struct aux_buffer {
  80	struct sf_buffer sfb;
  81	unsigned long head;	   /* index of SDB of buffer head */
  82	unsigned long alert_mark;  /* index of SDB of alert request position */
  83	unsigned long empty_mark;  /* mark of SDB not marked full */
  84	unsigned long *sdb_index;  /* SDB address for fast lookup */
  85	unsigned long *sdbt_index; /* SDBT address for fast lookup */
  86};
  87
  88struct cpu_hw_sf {
  89	/* CPU-measurement sampling information block */
  90	struct hws_qsi_info_block qsi;
  91	/* CPU-measurement sampling control block */
  92	struct hws_lsctl_request_block lsctl;
  93	struct sf_buffer sfb;	    /* Sampling buffer */
  94	unsigned int flags;	    /* Status flags */
  95	struct perf_event *event;   /* Scheduled perf event */
  96	struct perf_output_handle handle; /* AUX buffer output handle */
  97};
  98static DEFINE_PER_CPU(struct cpu_hw_sf, cpu_hw_sf);
  99
 100/* Debug feature */
 101static debug_info_t *sfdbg;
 102
 103/* Sampling control helper functions */
 104static inline unsigned long freq_to_sample_rate(struct hws_qsi_info_block *qsi,
 105						unsigned long freq)
 106{
 107	return (USEC_PER_SEC / freq) * qsi->cpu_speed;
 108}
 109
 110static inline unsigned long sample_rate_to_freq(struct hws_qsi_info_block *qsi,
 111						unsigned long rate)
 112{
 113	return USEC_PER_SEC * qsi->cpu_speed / rate;
 114}
 115
 116/* Return TOD timestamp contained in an trailer entry */
 117static inline unsigned long long trailer_timestamp(struct hws_trailer_entry *te)
 118{
 119	/* TOD in STCKE format */
 120	if (te->header.t)
 121		return *((unsigned long long *)&te->timestamp[1]);
 122
 123	/* TOD in STCK format */
 124	return *((unsigned long long *)&te->timestamp[0]);
 125}
 126
 127/* Return pointer to trailer entry of an sample data block */
 128static inline struct hws_trailer_entry *trailer_entry_ptr(unsigned long v)
 129{
 130	void *ret;
 131
 132	ret = (void *)v;
 133	ret += PAGE_SIZE;
 134	ret -= sizeof(struct hws_trailer_entry);
 135
 136	return ret;
 137}
 138
 139/*
 140 * Return true if the entry in the sample data block table (sdbt)
 141 * is a link to the next sdbt
 142 */
 143static inline int is_link_entry(unsigned long *s)
 144{
 145	return *s & 0x1UL ? 1 : 0;
 146}
 147
 148/* Return pointer to the linked sdbt */
 149static inline unsigned long *get_next_sdbt(unsigned long *s)
 150{
 151	return phys_to_virt(*s & ~0x1UL);
 152}
 153
 154/*
 155 * sf_disable() - Switch off sampling facility
 156 */
 157static int sf_disable(void)
 158{
 159	struct hws_lsctl_request_block sreq;
 160
 161	memset(&sreq, 0, sizeof(sreq));
 162	return lsctl(&sreq);
 163}
 164
 165/*
 166 * sf_buffer_available() - Check for an allocated sampling buffer
 167 */
 168static int sf_buffer_available(struct cpu_hw_sf *cpuhw)
 169{
 170	return !!cpuhw->sfb.sdbt;
 171}
 172
 173/*
 174 * deallocate sampling facility buffer
 175 */
 176static void free_sampling_buffer(struct sf_buffer *sfb)
 177{
 178	unsigned long *sdbt, *curr;
 179
 180	if (!sfb->sdbt)
 181		return;
 182
 183	sdbt = sfb->sdbt;
 184	curr = sdbt;
 185
 186	/* Free the SDBT after all SDBs are processed... */
 187	while (1) {
 188		if (!*curr || !sdbt)
 189			break;
 190
 191		/* Process table-link entries */
 192		if (is_link_entry(curr)) {
 193			curr = get_next_sdbt(curr);
 194			if (sdbt)
 195				free_page((unsigned long)sdbt);
 196
 197			/* If the origin is reached, sampling buffer is freed */
 198			if (curr == sfb->sdbt)
 199				break;
 200			else
 201				sdbt = curr;
 202		} else {
 203			/* Process SDB pointer */
 204			if (*curr) {
 205				free_page((unsigned long)phys_to_virt(*curr));
 206				curr++;
 207			}
 208		}
 209	}
 210
 211	debug_sprintf_event(sfdbg, 5, "%s: freed sdbt %#lx\n", __func__,
 212			    (unsigned long)sfb->sdbt);
 213	memset(sfb, 0, sizeof(*sfb));
 214}
 215
 216static int alloc_sample_data_block(unsigned long *sdbt, gfp_t gfp_flags)
 217{
 218	struct hws_trailer_entry *te;
 219	unsigned long sdb;
 220
 221	/* Allocate and initialize sample-data-block */
 222	sdb = get_zeroed_page(gfp_flags);
 223	if (!sdb)
 224		return -ENOMEM;
 225	te = trailer_entry_ptr(sdb);
 226	te->header.a = 1;
 227
 228	/* Link SDB into the sample-data-block-table */
 229	*sdbt = virt_to_phys((void *)sdb);
 230
 231	return 0;
 232}
 233
 234/*
 235 * realloc_sampling_buffer() - extend sampler memory
 236 *
 237 * Allocates new sample-data-blocks and adds them to the specified sampling
 238 * buffer memory.
 239 *
 240 * Important: This modifies the sampling buffer and must be called when the
 241 *	      sampling facility is disabled.
 242 *
 243 * Returns zero on success, non-zero otherwise.
 244 */
 245static int realloc_sampling_buffer(struct sf_buffer *sfb,
 246				   unsigned long num_sdb, gfp_t gfp_flags)
 247{
 248	int i, rc;
 249	unsigned long *new, *tail, *tail_prev = NULL;
 250
 251	if (!sfb->sdbt || !sfb->tail)
 252		return -EINVAL;
 253
 254	if (!is_link_entry(sfb->tail))
 255		return -EINVAL;
 256
 257	/* Append to the existing sampling buffer, overwriting the table-link
 258	 * register.
 259	 * The tail variables always points to the "tail" (last and table-link)
 260	 * entry in an SDB-table.
 261	 */
 262	tail = sfb->tail;
 263
 264	/* Do a sanity check whether the table-link entry points to
 265	 * the sampling buffer origin.
 266	 */
 267	if (sfb->sdbt != get_next_sdbt(tail)) {
 268		debug_sprintf_event(sfdbg, 3, "%s: "
 269				    "sampling buffer is not linked: origin %#lx"
 270				    " tail %#lx\n", __func__,
 271				    (unsigned long)sfb->sdbt,
 272				    (unsigned long)tail);
 273		return -EINVAL;
 274	}
 275
 276	/* Allocate remaining SDBs */
 277	rc = 0;
 278	for (i = 0; i < num_sdb; i++) {
 279		/* Allocate a new SDB-table if it is full. */
 280		if (require_table_link(tail)) {
 281			new = (unsigned long *)get_zeroed_page(gfp_flags);
 282			if (!new) {
 283				rc = -ENOMEM;
 284				break;
 285			}
 286			sfb->num_sdbt++;
 287			/* Link current page to tail of chain */
 288			*tail = virt_to_phys((void *)new) + 1;
 289			tail_prev = tail;
 290			tail = new;
 291		}
 292
 293		/* Allocate a new sample-data-block.
 294		 * If there is not enough memory, stop the realloc process
 295		 * and simply use what was allocated.  If this is a temporary
 296		 * issue, a new realloc call (if required) might succeed.
 297		 */
 298		rc = alloc_sample_data_block(tail, gfp_flags);
 299		if (rc) {
 300			/* Undo last SDBT. An SDBT with no SDB at its first
 301			 * entry but with an SDBT entry instead can not be
 302			 * handled by the interrupt handler code.
 303			 * Avoid this situation.
 304			 */
 305			if (tail_prev) {
 306				sfb->num_sdbt--;
 307				free_page((unsigned long)new);
 308				tail = tail_prev;
 309			}
 310			break;
 311		}
 312		sfb->num_sdb++;
 313		tail++;
 314		tail_prev = new = NULL;	/* Allocated at least one SBD */
 315	}
 316
 317	/* Link sampling buffer to its origin */
 318	*tail = virt_to_phys(sfb->sdbt) + 1;
 319	sfb->tail = tail;
 320
 321	debug_sprintf_event(sfdbg, 4, "%s: new buffer"
 322			    " settings: sdbt %lu sdb %lu\n", __func__,
 323			    sfb->num_sdbt, sfb->num_sdb);
 324	return rc;
 325}
 326
 327/*
 328 * allocate_sampling_buffer() - allocate sampler memory
 329 *
 330 * Allocates and initializes a sampling buffer structure using the
 331 * specified number of sample-data-blocks (SDB).  For each allocation,
 332 * a 4K page is used.  The number of sample-data-block-tables (SDBT)
 333 * are calculated from SDBs.
 334 * Also set the ALERT_REQ mask in each SDBs trailer.
 335 *
 336 * Returns zero on success, non-zero otherwise.
 337 */
 338static int alloc_sampling_buffer(struct sf_buffer *sfb, unsigned long num_sdb)
 339{
 340	int rc;
 341
 342	if (sfb->sdbt)
 343		return -EINVAL;
 344
 345	/* Allocate the sample-data-block-table origin */
 346	sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 347	if (!sfb->sdbt)
 348		return -ENOMEM;
 349	sfb->num_sdb = 0;
 350	sfb->num_sdbt = 1;
 351
 352	/* Link the table origin to point to itself to prepare for
 353	 * realloc_sampling_buffer() invocation.
 354	 */
 355	sfb->tail = sfb->sdbt;
 356	*sfb->tail = virt_to_phys((void *)sfb->sdbt) + 1;
 357
 358	/* Allocate requested number of sample-data-blocks */
 359	rc = realloc_sampling_buffer(sfb, num_sdb, GFP_KERNEL);
 360	if (rc) {
 361		free_sampling_buffer(sfb);
 362		debug_sprintf_event(sfdbg, 4, "%s: "
 363			"realloc_sampling_buffer failed with rc %i\n",
 364			__func__, rc);
 365	} else
 366		debug_sprintf_event(sfdbg, 4,
 367			"%s: tear %#lx dear %#lx\n", __func__,
 368			(unsigned long)sfb->sdbt, (unsigned long)*sfb->sdbt);
 369	return rc;
 370}
 371
 372static void sfb_set_limits(unsigned long min, unsigned long max)
 373{
 374	struct hws_qsi_info_block si;
 375
 376	CPUM_SF_MIN_SDB = min;
 377	CPUM_SF_MAX_SDB = max;
 378
 379	memset(&si, 0, sizeof(si));
 380	if (!qsi(&si))
 381		CPUM_SF_SDB_DIAG_FACTOR = DIV_ROUND_UP(si.dsdes, si.bsdes);
 382}
 383
 384static unsigned long sfb_max_limit(struct hw_perf_event *hwc)
 385{
 386	return SAMPL_DIAG_MODE(hwc) ? CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR
 387				    : CPUM_SF_MAX_SDB;
 388}
 389
 390static unsigned long sfb_pending_allocs(struct sf_buffer *sfb,
 391					struct hw_perf_event *hwc)
 392{
 393	if (!sfb->sdbt)
 394		return SFB_ALLOC_REG(hwc);
 395	if (SFB_ALLOC_REG(hwc) > sfb->num_sdb)
 396		return SFB_ALLOC_REG(hwc) - sfb->num_sdb;
 397	return 0;
 398}
 399
 400static int sfb_has_pending_allocs(struct sf_buffer *sfb,
 401				   struct hw_perf_event *hwc)
 402{
 403	return sfb_pending_allocs(sfb, hwc) > 0;
 404}
 405
 406static void sfb_account_allocs(unsigned long num, struct hw_perf_event *hwc)
 407{
 408	/* Limit the number of SDBs to not exceed the maximum */
 409	num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc));
 410	if (num)
 411		SFB_ALLOC_REG(hwc) += num;
 412}
 413
 414static void sfb_init_allocs(unsigned long num, struct hw_perf_event *hwc)
 415{
 416	SFB_ALLOC_REG(hwc) = 0;
 417	sfb_account_allocs(num, hwc);
 418}
 419
 420static void deallocate_buffers(struct cpu_hw_sf *cpuhw)
 421{
 422	if (cpuhw->sfb.sdbt)
 423		free_sampling_buffer(&cpuhw->sfb);
 424}
 425
 426static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc)
 427{
 428	unsigned long n_sdb, freq;
 429	size_t sample_size;
 430
 431	/* Calculate sampling buffers using 4K pages
 432	 *
 433	 *    1. The sampling size is 32 bytes for basic sampling. This size
 434	 *	 is the same for all machine types. Diagnostic
 435	 *	 sampling uses auxlilary data buffer setup which provides the
 436	 *	 memory for SDBs using linux common code auxiliary trace
 437	 *	 setup.
 438	 *
 439	 *    2. Function alloc_sampling_buffer() sets the Alert Request
 440	 *	 Control indicator to trigger a measurement-alert to harvest
 441	 *	 sample-data-blocks (SDB). This is done per SDB. This
 442	 *	 measurement alert interrupt fires quick enough to handle
 443	 *	 one SDB, on very high frequency and work loads there might
 444	 *	 be 2 to 3 SBDs available for sample processing.
 445	 *	 Currently there is no need for setup alert request on every
 446	 *	 n-th page. This is counterproductive as one IRQ triggers
 447	 *	 a very high number of samples to be processed at one IRQ.
 448	 *
 449	 *    3. Use the sampling frequency as input.
 450	 *	 Compute the number of SDBs and ensure a minimum
 451	 *	 of CPUM_SF_MIN_SDB.  Depending on frequency add some more
 452	 *	 SDBs to handle a higher sampling rate.
 453	 *	 Use a minimum of CPUM_SF_MIN_SDB and allow for 100 samples
 454	 *	 (one SDB) for every 10000 HZ frequency increment.
 455	 *
 456	 *    4. Compute the number of sample-data-block-tables (SDBT) and
 457	 *	 ensure a minimum of CPUM_SF_MIN_SDBT (one table can manage up
 458	 *	 to 511 SDBs).
 459	 */
 460	sample_size = sizeof(struct hws_basic_entry);
 461	freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc));
 462	n_sdb = CPUM_SF_MIN_SDB + DIV_ROUND_UP(freq, 10000);
 463
 464	/* If there is already a sampling buffer allocated, it is very likely
 465	 * that the sampling facility is enabled too.  If the event to be
 466	 * initialized requires a greater sampling buffer, the allocation must
 467	 * be postponed.  Changing the sampling buffer requires the sampling
 468	 * facility to be in the disabled state.  So, account the number of
 469	 * required SDBs and let cpumsf_pmu_enable() resize the buffer just
 470	 * before the event is started.
 471	 */
 472	sfb_init_allocs(n_sdb, hwc);
 473	if (sf_buffer_available(cpuhw))
 474		return 0;
 475
 476	debug_sprintf_event(sfdbg, 3,
 477			    "%s: rate %lu f %lu sdb %lu/%lu"
 478			    " sample_size %lu cpuhw %p\n", __func__,
 479			    SAMPL_RATE(hwc), freq, n_sdb, sfb_max_limit(hwc),
 480			    sample_size, cpuhw);
 481
 482	return alloc_sampling_buffer(&cpuhw->sfb,
 483				     sfb_pending_allocs(&cpuhw->sfb, hwc));
 484}
 485
 486static unsigned long min_percent(unsigned int percent, unsigned long base,
 487				 unsigned long min)
 488{
 489	return min_t(unsigned long, min, DIV_ROUND_UP(percent * base, 100));
 490}
 491
 492static unsigned long compute_sfb_extent(unsigned long ratio, unsigned long base)
 493{
 494	/* Use a percentage-based approach to extend the sampling facility
 495	 * buffer.  Accept up to 5% sample data loss.
 496	 * Vary the extents between 1% to 5% of the current number of
 497	 * sample-data-blocks.
 498	 */
 499	if (ratio <= 5)
 500		return 0;
 501	if (ratio <= 25)
 502		return min_percent(1, base, 1);
 503	if (ratio <= 50)
 504		return min_percent(1, base, 1);
 505	if (ratio <= 75)
 506		return min_percent(2, base, 2);
 507	if (ratio <= 100)
 508		return min_percent(3, base, 3);
 509	if (ratio <= 250)
 510		return min_percent(4, base, 4);
 511
 512	return min_percent(5, base, 8);
 513}
 514
 515static void sfb_account_overflows(struct cpu_hw_sf *cpuhw,
 516				  struct hw_perf_event *hwc)
 517{
 518	unsigned long ratio, num;
 519
 520	if (!OVERFLOW_REG(hwc))
 521		return;
 522
 523	/* The sample_overflow contains the average number of sample data
 524	 * that has been lost because sample-data-blocks were full.
 525	 *
 526	 * Calculate the total number of sample data entries that has been
 527	 * discarded.  Then calculate the ratio of lost samples to total samples
 528	 * per second in percent.
 529	 */
 530	ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb,
 531			     sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)));
 532
 533	/* Compute number of sample-data-blocks */
 534	num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb);
 535	if (num)
 536		sfb_account_allocs(num, hwc);
 537
 538	debug_sprintf_event(sfdbg, 5, "%s: overflow %llu ratio %lu num %lu\n",
 539			    __func__, OVERFLOW_REG(hwc), ratio, num);
 540	OVERFLOW_REG(hwc) = 0;
 541}
 542
 543/* extend_sampling_buffer() - Extend sampling buffer
 544 * @sfb:	Sampling buffer structure (for local CPU)
 545 * @hwc:	Perf event hardware structure
 546 *
 547 * Use this function to extend the sampling buffer based on the overflow counter
 548 * and postponed allocation extents stored in the specified Perf event hardware.
 549 *
 550 * Important: This function disables the sampling facility in order to safely
 551 *	      change the sampling buffer structure.  Do not call this function
 552 *	      when the PMU is active.
 553 */
 554static void extend_sampling_buffer(struct sf_buffer *sfb,
 555				   struct hw_perf_event *hwc)
 556{
 557	unsigned long num, num_old;
 558	int rc;
 559
 560	num = sfb_pending_allocs(sfb, hwc);
 561	if (!num)
 562		return;
 563	num_old = sfb->num_sdb;
 564
 565	/* Disable the sampling facility to reset any states and also
 566	 * clear pending measurement alerts.
 567	 */
 568	sf_disable();
 569
 570	/* Extend the sampling buffer.
 571	 * This memory allocation typically happens in an atomic context when
 572	 * called by perf.  Because this is a reallocation, it is fine if the
 573	 * new SDB-request cannot be satisfied immediately.
 574	 */
 575	rc = realloc_sampling_buffer(sfb, num, GFP_ATOMIC);
 576	if (rc)
 577		debug_sprintf_event(sfdbg, 5, "%s: realloc failed with rc %i\n",
 578				    __func__, rc);
 579
 580	if (sfb_has_pending_allocs(sfb, hwc))
 581		debug_sprintf_event(sfdbg, 5, "%s: "
 582				    "req %lu alloc %lu remaining %lu\n",
 583				    __func__, num, sfb->num_sdb - num_old,
 584				    sfb_pending_allocs(sfb, hwc));
 585}
 586
 587/* Number of perf events counting hardware events */
 588static atomic_t num_events;
 589/* Used to avoid races in calling reserve/release_cpumf_hardware */
 590static DEFINE_MUTEX(pmc_reserve_mutex);
 591
 592#define PMC_INIT      0
 593#define PMC_RELEASE   1
 594#define PMC_FAILURE   2
 595static void setup_pmc_cpu(void *flags)
 596{
 
 597	struct cpu_hw_sf *cpusf = this_cpu_ptr(&cpu_hw_sf);
 598	int err = 0;
 599
 600	switch (*((int *)flags)) {
 
 601	case PMC_INIT:
 602		memset(cpusf, 0, sizeof(*cpusf));
 603		err = qsi(&cpusf->qsi);
 604		if (err)
 605			break;
 606		cpusf->flags |= PMU_F_RESERVED;
 607		err = sf_disable();
 
 
 
 
 
 
 608		break;
 609	case PMC_RELEASE:
 610		cpusf->flags &= ~PMU_F_RESERVED;
 611		err = sf_disable();
 612		if (!err)
 
 
 
 613			deallocate_buffers(cpusf);
 
 
 
 614		break;
 615	}
 616	if (err) {
 617		*((int *)flags) |= PMC_FAILURE;
 618		pr_err("Switching off the sampling facility failed with rc %i\n", err);
 619	}
 620}
 621
 622static void release_pmc_hardware(void)
 623{
 624	int flags = PMC_RELEASE;
 625
 626	irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
 627	on_each_cpu(setup_pmc_cpu, &flags, 1);
 628}
 629
 630static int reserve_pmc_hardware(void)
 631{
 632	int flags = PMC_INIT;
 633
 634	on_each_cpu(setup_pmc_cpu, &flags, 1);
 635	if (flags & PMC_FAILURE) {
 636		release_pmc_hardware();
 637		return -ENODEV;
 638	}
 639	irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
 640
 641	return 0;
 642}
 643
 644static void hw_perf_event_destroy(struct perf_event *event)
 645{
 646	/* Release PMC if this is the last perf event */
 647	if (!atomic_add_unless(&num_events, -1, 1)) {
 648		mutex_lock(&pmc_reserve_mutex);
 649		if (atomic_dec_return(&num_events) == 0)
 650			release_pmc_hardware();
 651		mutex_unlock(&pmc_reserve_mutex);
 652	}
 653}
 654
 655static void hw_init_period(struct hw_perf_event *hwc, u64 period)
 656{
 657	hwc->sample_period = period;
 658	hwc->last_period = hwc->sample_period;
 659	local64_set(&hwc->period_left, hwc->sample_period);
 660}
 661
 662static unsigned long hw_limit_rate(const struct hws_qsi_info_block *si,
 663				   unsigned long rate)
 664{
 665	return clamp_t(unsigned long, rate,
 666		       si->min_sampl_rate, si->max_sampl_rate);
 667}
 668
 669static u32 cpumsf_pid_type(struct perf_event *event,
 670			   u32 pid, enum pid_type type)
 671{
 672	struct task_struct *tsk;
 673
 674	/* Idle process */
 675	if (!pid)
 676		goto out;
 677
 678	tsk = find_task_by_pid_ns(pid, &init_pid_ns);
 679	pid = -1;
 680	if (tsk) {
 681		/*
 682		 * Only top level events contain the pid namespace in which
 683		 * they are created.
 684		 */
 685		if (event->parent)
 686			event = event->parent;
 687		pid = __task_pid_nr_ns(tsk, type, event->ns);
 688		/*
 689		 * See also 1d953111b648
 690		 * "perf/core: Don't report zero PIDs for exiting tasks".
 691		 */
 692		if (!pid && !pid_alive(tsk))
 693			pid = -1;
 694	}
 695out:
 696	return pid;
 697}
 698
 699static void cpumsf_output_event_pid(struct perf_event *event,
 700				    struct perf_sample_data *data,
 701				    struct pt_regs *regs)
 702{
 703	u32 pid;
 704	struct perf_event_header header;
 705	struct perf_output_handle handle;
 706
 707	/*
 708	 * Obtain the PID from the basic-sampling data entry and
 709	 * correct the data->tid_entry.pid value.
 710	 */
 711	pid = data->tid_entry.pid;
 712
 713	/* Protect callchain buffers, tasks */
 714	rcu_read_lock();
 715
 716	perf_prepare_sample(data, event, regs);
 717	perf_prepare_header(&header, data, event, regs);
 718	if (perf_output_begin(&handle, data, event, header.size))
 719		goto out;
 720
 721	/* Update the process ID (see also kernel/events/core.c) */
 722	data->tid_entry.pid = cpumsf_pid_type(event, pid, PIDTYPE_TGID);
 723	data->tid_entry.tid = cpumsf_pid_type(event, pid, PIDTYPE_PID);
 724
 725	perf_output_sample(&handle, &header, data, event);
 726	perf_output_end(&handle);
 727out:
 728	rcu_read_unlock();
 729}
 730
 731static unsigned long getrate(bool freq, unsigned long sample,
 732			     struct hws_qsi_info_block *si)
 733{
 734	unsigned long rate;
 735
 736	if (freq) {
 737		rate = freq_to_sample_rate(si, sample);
 738		rate = hw_limit_rate(si, rate);
 739	} else {
 740		/* The min/max sampling rates specifies the valid range
 741		 * of sample periods.  If the specified sample period is
 742		 * out of range, limit the period to the range boundary.
 743		 */
 744		rate = hw_limit_rate(si, sample);
 745
 746		/* The perf core maintains a maximum sample rate that is
 747		 * configurable through the sysctl interface.  Ensure the
 748		 * sampling rate does not exceed this value.  This also helps
 749		 * to avoid throttling when pushing samples with
 750		 * perf_event_overflow().
 751		 */
 752		if (sample_rate_to_freq(si, rate) >
 753		    sysctl_perf_event_sample_rate) {
 754			debug_sprintf_event(sfdbg, 1, "%s: "
 755					    "Sampling rate exceeds maximum "
 756					    "perf sample rate\n", __func__);
 757			rate = 0;
 758		}
 759	}
 760	return rate;
 761}
 762
 763/* The sampling information (si) contains information about the
 764 * min/max sampling intervals and the CPU speed.  So calculate the
 765 * correct sampling interval and avoid the whole period adjust
 766 * feedback loop.
 767 *
 768 * Since the CPU Measurement sampling facility can not handle frequency
 769 * calculate the sampling interval when frequency is specified using
 770 * this formula:
 771 *	interval := cpu_speed * 1000000 / sample_freq
 772 *
 773 * Returns errno on bad input and zero on success with parameter interval
 774 * set to the correct sampling rate.
 775 *
 776 * Note: This function turns off freq bit to avoid calling function
 777 * perf_adjust_period(). This causes frequency adjustment in the common
 778 * code part which causes tremendous variations in the counter values.
 779 */
 780static int __hw_perf_event_init_rate(struct perf_event *event,
 781				     struct hws_qsi_info_block *si)
 782{
 783	struct perf_event_attr *attr = &event->attr;
 784	struct hw_perf_event *hwc = &event->hw;
 785	unsigned long rate;
 786
 787	if (attr->freq) {
 788		if (!attr->sample_freq)
 789			return -EINVAL;
 790		rate = getrate(attr->freq, attr->sample_freq, si);
 791		attr->freq = 0;		/* Don't call  perf_adjust_period() */
 792		SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_FREQ_MODE;
 793	} else {
 794		rate = getrate(attr->freq, attr->sample_period, si);
 795		if (!rate)
 796			return -EINVAL;
 797	}
 798	attr->sample_period = rate;
 799	SAMPL_RATE(hwc) = rate;
 800	hw_init_period(hwc, SAMPL_RATE(hwc));
 801	debug_sprintf_event(sfdbg, 4, "%s: cpu %d period %#llx freq %d,%#lx\n",
 802			    __func__, event->cpu, event->attr.sample_period,
 803			    event->attr.freq, SAMPLE_FREQ_MODE(hwc));
 804	return 0;
 805}
 806
 807static int __hw_perf_event_init(struct perf_event *event)
 808{
 809	struct cpu_hw_sf *cpuhw;
 810	struct hws_qsi_info_block si;
 811	struct perf_event_attr *attr = &event->attr;
 812	struct hw_perf_event *hwc = &event->hw;
 813	int cpu, err;
 814
 815	/* Reserve CPU-measurement sampling facility */
 816	err = 0;
 817	if (!atomic_inc_not_zero(&num_events)) {
 818		mutex_lock(&pmc_reserve_mutex);
 819		if (atomic_read(&num_events) == 0 && reserve_pmc_hardware())
 820			err = -EBUSY;
 821		else
 822			atomic_inc(&num_events);
 823		mutex_unlock(&pmc_reserve_mutex);
 824	}
 825	event->destroy = hw_perf_event_destroy;
 826
 827	if (err)
 828		goto out;
 829
 830	/* Access per-CPU sampling information (query sampling info) */
 831	/*
 832	 * The event->cpu value can be -1 to count on every CPU, for example,
 833	 * when attaching to a task.  If this is specified, use the query
 834	 * sampling info from the current CPU, otherwise use event->cpu to
 835	 * retrieve the per-CPU information.
 836	 * Later, cpuhw indicates whether to allocate sampling buffers for a
 837	 * particular CPU (cpuhw!=NULL) or each online CPU (cpuw==NULL).
 838	 */
 839	memset(&si, 0, sizeof(si));
 840	cpuhw = NULL;
 841	if (event->cpu == -1)
 842		qsi(&si);
 843	else {
 844		/* Event is pinned to a particular CPU, retrieve the per-CPU
 845		 * sampling structure for accessing the CPU-specific QSI.
 846		 */
 847		cpuhw = &per_cpu(cpu_hw_sf, event->cpu);
 848		si = cpuhw->qsi;
 849	}
 850
 851	/* Check sampling facility authorization and, if not authorized,
 852	 * fall back to other PMUs.  It is safe to check any CPU because
 853	 * the authorization is identical for all configured CPUs.
 854	 */
 855	if (!si.as) {
 856		err = -ENOENT;
 857		goto out;
 858	}
 859
 860	if (si.ribm & CPU_MF_SF_RIBM_NOTAV) {
 861		pr_warn("CPU Measurement Facility sampling is temporarily not available\n");
 862		err = -EBUSY;
 863		goto out;
 864	}
 865
 866	/* Always enable basic sampling */
 867	SAMPL_FLAGS(hwc) = PERF_CPUM_SF_BASIC_MODE;
 868
 869	/* Check if diagnostic sampling is requested.  Deny if the required
 870	 * sampling authorization is missing.
 871	 */
 872	if (attr->config == PERF_EVENT_CPUM_SF_DIAG) {
 873		if (!si.ad) {
 874			err = -EPERM;
 875			goto out;
 876		}
 877		SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_DIAG_MODE;
 878	}
 879
 
 
 
 
 880	err =  __hw_perf_event_init_rate(event, &si);
 881	if (err)
 882		goto out;
 883
 884	/* Initialize sample data overflow accounting */
 885	hwc->extra_reg.reg = REG_OVERFLOW;
 886	OVERFLOW_REG(hwc) = 0;
 887
 888	/* Use AUX buffer. No need to allocate it by ourself */
 889	if (attr->config == PERF_EVENT_CPUM_SF_DIAG)
 890		return 0;
 891
 892	/* Allocate the per-CPU sampling buffer using the CPU information
 893	 * from the event.  If the event is not pinned to a particular
 894	 * CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling
 895	 * buffers for each online CPU.
 896	 */
 897	if (cpuhw)
 898		/* Event is pinned to a particular CPU */
 899		err = allocate_buffers(cpuhw, hwc);
 900	else {
 901		/* Event is not pinned, allocate sampling buffer on
 902		 * each online CPU
 903		 */
 904		for_each_online_cpu(cpu) {
 905			cpuhw = &per_cpu(cpu_hw_sf, cpu);
 906			err = allocate_buffers(cpuhw, hwc);
 907			if (err)
 908				break;
 909		}
 910	}
 911
 912	/* If PID/TID sampling is active, replace the default overflow
 913	 * handler to extract and resolve the PIDs from the basic-sampling
 914	 * data entries.
 915	 */
 916	if (event->attr.sample_type & PERF_SAMPLE_TID)
 917		if (is_default_overflow_handler(event))
 918			event->overflow_handler = cpumsf_output_event_pid;
 919out:
 920	return err;
 921}
 922
 923static bool is_callchain_event(struct perf_event *event)
 924{
 925	u64 sample_type = event->attr.sample_type;
 926
 927	return sample_type & (PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_REGS_USER |
 928			      PERF_SAMPLE_STACK_USER);
 929}
 930
 931static int cpumsf_pmu_event_init(struct perf_event *event)
 932{
 933	int err;
 934
 935	/* No support for taken branch sampling */
 936	/* No support for callchain, stacks and registers */
 937	if (has_branch_stack(event) || is_callchain_event(event))
 938		return -EOPNOTSUPP;
 939
 940	switch (event->attr.type) {
 941	case PERF_TYPE_RAW:
 942		if ((event->attr.config != PERF_EVENT_CPUM_SF) &&
 943		    (event->attr.config != PERF_EVENT_CPUM_SF_DIAG))
 944			return -ENOENT;
 945		break;
 946	case PERF_TYPE_HARDWARE:
 947		/* Support sampling of CPU cycles in addition to the
 948		 * counter facility.  However, the counter facility
 949		 * is more precise and, hence, restrict this PMU to
 950		 * sampling events only.
 951		 */
 952		if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES)
 953			return -ENOENT;
 954		if (!is_sampling_event(event))
 955			return -ENOENT;
 956		break;
 957	default:
 958		return -ENOENT;
 959	}
 960
 
 
 
 
 961	/* Force reset of idle/hv excludes regardless of what the
 962	 * user requested.
 963	 */
 964	if (event->attr.exclude_hv)
 965		event->attr.exclude_hv = 0;
 966	if (event->attr.exclude_idle)
 967		event->attr.exclude_idle = 0;
 968
 969	err = __hw_perf_event_init(event);
 970	if (unlikely(err))
 971		if (event->destroy)
 972			event->destroy(event);
 973	return err;
 974}
 975
 976static void cpumsf_pmu_enable(struct pmu *pmu)
 977{
 978	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
 979	struct hw_perf_event *hwc;
 980	int err;
 981
 982	if (cpuhw->flags & PMU_F_ENABLED)
 983		return;
 984
 985	if (cpuhw->flags & PMU_F_ERR_MASK)
 986		return;
 987
 988	/* Check whether to extent the sampling buffer.
 989	 *
 990	 * Two conditions trigger an increase of the sampling buffer for a
 991	 * perf event:
 992	 *    1. Postponed buffer allocations from the event initialization.
 993	 *    2. Sampling overflows that contribute to pending allocations.
 994	 *
 995	 * Note that the extend_sampling_buffer() function disables the sampling
 996	 * facility, but it can be fully re-enabled using sampling controls that
 997	 * have been saved in cpumsf_pmu_disable().
 998	 */
 999	if (cpuhw->event) {
1000		hwc = &cpuhw->event->hw;
1001		if (!(SAMPL_DIAG_MODE(hwc))) {
1002			/*
1003			 * Account number of overflow-designated
1004			 * buffer extents
1005			 */
1006			sfb_account_overflows(cpuhw, hwc);
1007			extend_sampling_buffer(&cpuhw->sfb, hwc);
1008		}
1009		/* Rate may be adjusted with ioctl() */
1010		cpuhw->lsctl.interval = SAMPL_RATE(&cpuhw->event->hw);
1011	}
1012
1013	/* (Re)enable the PMU and sampling facility */
1014	cpuhw->flags |= PMU_F_ENABLED;
1015	barrier();
1016
1017	err = lsctl(&cpuhw->lsctl);
1018	if (err) {
1019		cpuhw->flags &= ~PMU_F_ENABLED;
1020		pr_err("Loading sampling controls failed: op 1 err %i\n", err);
 
1021		return;
1022	}
1023
1024	/* Load current program parameter */
1025	lpp(&S390_lowcore.lpp);
1026
1027	debug_sprintf_event(sfdbg, 6, "%s: es %i cs %i ed %i cd %i "
1028			    "interval %#lx tear %#lx dear %#lx\n", __func__,
1029			    cpuhw->lsctl.es, cpuhw->lsctl.cs, cpuhw->lsctl.ed,
1030			    cpuhw->lsctl.cd, cpuhw->lsctl.interval,
1031			    cpuhw->lsctl.tear, cpuhw->lsctl.dear);
1032}
1033
1034static void cpumsf_pmu_disable(struct pmu *pmu)
1035{
1036	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1037	struct hws_lsctl_request_block inactive;
1038	struct hws_qsi_info_block si;
1039	int err;
1040
1041	if (!(cpuhw->flags & PMU_F_ENABLED))
1042		return;
1043
1044	if (cpuhw->flags & PMU_F_ERR_MASK)
1045		return;
1046
1047	/* Switch off sampling activation control */
1048	inactive = cpuhw->lsctl;
1049	inactive.cs = 0;
1050	inactive.cd = 0;
1051
1052	err = lsctl(&inactive);
1053	if (err) {
1054		pr_err("Loading sampling controls failed: op 2 err %i\n", err);
 
1055		return;
1056	}
1057
1058	/* Save state of TEAR and DEAR register contents */
1059	err = qsi(&si);
1060	if (!err) {
1061		/* TEAR/DEAR values are valid only if the sampling facility is
1062		 * enabled.  Note that cpumsf_pmu_disable() might be called even
1063		 * for a disabled sampling facility because cpumsf_pmu_enable()
1064		 * controls the enable/disable state.
1065		 */
1066		if (si.es) {
1067			cpuhw->lsctl.tear = si.tear;
1068			cpuhw->lsctl.dear = si.dear;
1069		}
1070	} else
1071		debug_sprintf_event(sfdbg, 3, "%s: qsi() failed with err %i\n",
1072				    __func__, err);
1073
1074	cpuhw->flags &= ~PMU_F_ENABLED;
1075}
1076
1077/* perf_exclude_event() - Filter event
1078 * @event:	The perf event
1079 * @regs:	pt_regs structure
1080 * @sde_regs:	Sample-data-entry (sde) regs structure
1081 *
1082 * Filter perf events according to their exclude specification.
1083 *
1084 * Return non-zero if the event shall be excluded.
1085 */
1086static int perf_exclude_event(struct perf_event *event, struct pt_regs *regs,
1087			      struct perf_sf_sde_regs *sde_regs)
1088{
1089	if (event->attr.exclude_user && user_mode(regs))
1090		return 1;
1091	if (event->attr.exclude_kernel && !user_mode(regs))
1092		return 1;
1093	if (event->attr.exclude_guest && sde_regs->in_guest)
1094		return 1;
1095	if (event->attr.exclude_host && !sde_regs->in_guest)
1096		return 1;
1097	return 0;
1098}
1099
1100/* perf_push_sample() - Push samples to perf
1101 * @event:	The perf event
1102 * @sample:	Hardware sample data
1103 *
1104 * Use the hardware sample data to create perf event sample.  The sample
1105 * is the pushed to the event subsystem and the function checks for
1106 * possible event overflows.  If an event overflow occurs, the PMU is
1107 * stopped.
1108 *
1109 * Return non-zero if an event overflow occurred.
1110 */
1111static int perf_push_sample(struct perf_event *event,
1112			    struct hws_basic_entry *basic)
1113{
1114	int overflow;
1115	struct pt_regs regs;
1116	struct perf_sf_sde_regs *sde_regs;
1117	struct perf_sample_data data;
1118
1119	/* Setup perf sample */
1120	perf_sample_data_init(&data, 0, event->hw.last_period);
1121
1122	/* Setup pt_regs to look like an CPU-measurement external interrupt
1123	 * using the Program Request Alert code.  The regs.int_parm_long
1124	 * field which is unused contains additional sample-data-entry related
1125	 * indicators.
1126	 */
1127	memset(&regs, 0, sizeof(regs));
1128	regs.int_code = 0x1407;
1129	regs.int_parm = CPU_MF_INT_SF_PRA;
1130	sde_regs = (struct perf_sf_sde_regs *) &regs.int_parm_long;
1131
1132	psw_bits(regs.psw).ia	= basic->ia;
1133	psw_bits(regs.psw).dat	= basic->T;
1134	psw_bits(regs.psw).wait = basic->W;
1135	psw_bits(regs.psw).pstate = basic->P;
1136	psw_bits(regs.psw).as	= basic->AS;
1137
1138	/*
1139	 * Use the hardware provided configuration level to decide if the
1140	 * sample belongs to a guest or host. If that is not available,
1141	 * fall back to the following heuristics:
1142	 * A non-zero guest program parameter always indicates a guest
1143	 * sample. Some early samples or samples from guests without
1144	 * lpp usage would be misaccounted to the host. We use the asn
1145	 * value as an addon heuristic to detect most of these guest samples.
1146	 * If the value differs from 0xffff (the host value), we assume to
1147	 * be a KVM guest.
1148	 */
1149	switch (basic->CL) {
1150	case 1: /* logical partition */
1151		sde_regs->in_guest = 0;
1152		break;
1153	case 2: /* virtual machine */
1154		sde_regs->in_guest = 1;
1155		break;
1156	default: /* old machine, use heuristics */
1157		if (basic->gpp || basic->prim_asn != 0xffff)
1158			sde_regs->in_guest = 1;
1159		break;
1160	}
1161
1162	/*
1163	 * Store the PID value from the sample-data-entry to be
1164	 * processed and resolved by cpumsf_output_event_pid().
1165	 */
1166	data.tid_entry.pid = basic->hpp & LPP_PID_MASK;
1167
1168	overflow = 0;
1169	if (perf_exclude_event(event, &regs, sde_regs))
1170		goto out;
1171	if (perf_event_overflow(event, &data, &regs)) {
1172		overflow = 1;
1173		event->pmu->stop(event, 0);
1174	}
1175	perf_event_update_userpage(event);
1176out:
1177	return overflow;
1178}
1179
1180static void perf_event_count_update(struct perf_event *event, u64 count)
1181{
1182	local64_add(count, &event->count);
1183}
1184
1185/* hw_collect_samples() - Walk through a sample-data-block and collect samples
1186 * @event:	The perf event
1187 * @sdbt:	Sample-data-block table
1188 * @overflow:	Event overflow counter
1189 *
1190 * Walks through a sample-data-block and collects sampling data entries that are
1191 * then pushed to the perf event subsystem.  Depending on the sampling function,
1192 * there can be either basic-sampling or combined-sampling data entries.  A
1193 * combined-sampling data entry consists of a basic- and a diagnostic-sampling
1194 * data entry.	The sampling function is determined by the flags in the perf
1195 * event hardware structure.  The function always works with a combined-sampling
1196 * data entry but ignores the the diagnostic portion if it is not available.
1197 *
1198 * Note that the implementation focuses on basic-sampling data entries and, if
1199 * such an entry is not valid, the entire combined-sampling data entry is
1200 * ignored.
1201 *
1202 * The overflow variables counts the number of samples that has been discarded
1203 * due to a perf event overflow.
1204 */
1205static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt,
1206			       unsigned long long *overflow)
1207{
1208	struct hws_trailer_entry *te;
1209	struct hws_basic_entry *sample;
1210
1211	te = trailer_entry_ptr((unsigned long)sdbt);
1212	sample = (struct hws_basic_entry *)sdbt;
1213	while ((unsigned long *)sample < (unsigned long *)te) {
1214		/* Check for an empty sample */
1215		if (!sample->def || sample->LS)
1216			break;
1217
1218		/* Update perf event period */
1219		perf_event_count_update(event, SAMPL_RATE(&event->hw));
1220
1221		/* Check whether sample is valid */
1222		if (sample->def == 0x0001) {
1223			/* If an event overflow occurred, the PMU is stopped to
1224			 * throttle event delivery.  Remaining sample data is
1225			 * discarded.
1226			 */
1227			if (!*overflow) {
1228				/* Check whether sample is consistent */
1229				if (sample->I == 0 && sample->W == 0) {
1230					/* Deliver sample data to perf */
1231					*overflow = perf_push_sample(event,
1232								     sample);
1233				}
1234			} else
1235				/* Count discarded samples */
1236				*overflow += 1;
1237		} else {
1238			debug_sprintf_event(sfdbg, 4,
1239					    "%s: Found unknown"
1240					    " sampling data entry: te->f %i"
1241					    " basic.def %#4x (%p)\n", __func__,
1242					    te->header.f, sample->def, sample);
1243			/* Sample slot is not yet written or other record.
1244			 *
1245			 * This condition can occur if the buffer was reused
1246			 * from a combined basic- and diagnostic-sampling.
1247			 * If only basic-sampling is then active, entries are
1248			 * written into the larger diagnostic entries.
1249			 * This is typically the case for sample-data-blocks
1250			 * that are not full.  Stop processing if the first
1251			 * invalid format was detected.
1252			 */
1253			if (!te->header.f)
1254				break;
1255		}
1256
1257		/* Reset sample slot and advance to next sample */
1258		sample->def = 0;
1259		sample++;
1260	}
1261}
1262
1263/* hw_perf_event_update() - Process sampling buffer
1264 * @event:	The perf event
1265 * @flush_all:	Flag to also flush partially filled sample-data-blocks
1266 *
1267 * Processes the sampling buffer and create perf event samples.
1268 * The sampling buffer position are retrieved and saved in the TEAR_REG
1269 * register of the specified perf event.
1270 *
1271 * Only full sample-data-blocks are processed.	Specify the flush_all flag
1272 * to also walk through partially filled sample-data-blocks.
 
 
 
1273 */
1274static void hw_perf_event_update(struct perf_event *event, int flush_all)
1275{
1276	unsigned long long event_overflow, sampl_overflow, num_sdb;
1277	union hws_trailer_header old, prev, new;
1278	struct hw_perf_event *hwc = &event->hw;
1279	struct hws_trailer_entry *te;
1280	unsigned long *sdbt, sdb;
 
1281	int done;
1282
1283	/*
1284	 * AUX buffer is used when in diagnostic sampling mode.
1285	 * No perf events/samples are created.
1286	 */
1287	if (SAMPL_DIAG_MODE(&event->hw))
1288		return;
1289
1290	sdbt = (unsigned long *)TEAR_REG(hwc);
 
 
 
1291	done = event_overflow = sampl_overflow = num_sdb = 0;
1292	while (!done) {
1293		/* Get the trailer entry of the sample-data-block */
1294		sdb = (unsigned long)phys_to_virt(*sdbt);
1295		te = trailer_entry_ptr(sdb);
1296
1297		/* Leave loop if no more work to do (block full indicator) */
1298		if (!te->header.f) {
1299			done = 1;
1300			if (!flush_all)
1301				break;
1302		}
1303
1304		/* Check the sample overflow count */
1305		if (te->header.overflow)
1306			/* Account sample overflows and, if a particular limit
1307			 * is reached, extend the sampling buffer.
1308			 * For details, see sfb_account_overflows().
1309			 */
1310			sampl_overflow += te->header.overflow;
1311
1312		/* Timestamps are valid for full sample-data-blocks only */
1313		debug_sprintf_event(sfdbg, 6, "%s: sdbt %#lx/%#lx "
1314				    "overflow %llu timestamp %#llx\n",
1315				    __func__, sdb, (unsigned long)sdbt,
1316				    te->header.overflow,
1317				    (te->header.f) ? trailer_timestamp(te) : 0ULL);
1318
1319		/* Collect all samples from a single sample-data-block and
1320		 * flag if an (perf) event overflow happened.  If so, the PMU
1321		 * is stopped and remaining samples will be discarded.
1322		 */
1323		hw_collect_samples(event, (unsigned long *)sdb, &event_overflow);
1324		num_sdb++;
1325
1326		/* Reset trailer (using compare-double-and-swap) */
1327		prev.val = READ_ONCE_ALIGNED_128(te->header.val);
1328		do {
1329			old.val = prev.val;
1330			new.val = prev.val;
1331			new.f = 0;
1332			new.a = 1;
1333			new.overflow = 0;
1334			prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1335		} while (prev.val != old.val);
1336
1337		/* Advance to next sample-data-block */
1338		sdbt++;
1339		if (is_link_entry(sdbt))
1340			sdbt = get_next_sdbt(sdbt);
1341
1342		/* Update event hardware registers */
1343		TEAR_REG(hwc) = (unsigned long) sdbt;
1344
1345		/* Stop processing sample-data if all samples of the current
1346		 * sample-data-block were flushed even if it was not full.
1347		 */
1348		if (flush_all && done)
1349			break;
1350	}
1351
1352	/* Account sample overflows in the event hardware structure */
1353	if (sampl_overflow)
1354		OVERFLOW_REG(hwc) = DIV_ROUND_UP(OVERFLOW_REG(hwc) +
1355						 sampl_overflow, 1 + num_sdb);
1356
1357	/* Perf_event_overflow() and perf_event_account_interrupt() limit
1358	 * the interrupt rate to an upper limit. Roughly 1000 samples per
1359	 * task tick.
1360	 * Hitting this limit results in a large number
1361	 * of throttled REF_REPORT_THROTTLE entries and the samples
1362	 * are dropped.
1363	 * Slightly increase the interval to avoid hitting this limit.
1364	 */
1365	if (event_overflow) {
1366		SAMPL_RATE(hwc) += DIV_ROUND_UP(SAMPL_RATE(hwc), 10);
1367		debug_sprintf_event(sfdbg, 1, "%s: rate adjustment %ld\n",
1368				    __func__,
1369				    DIV_ROUND_UP(SAMPL_RATE(hwc), 10));
1370	}
1371
1372	if (sampl_overflow || event_overflow)
1373		debug_sprintf_event(sfdbg, 4, "%s: "
1374				    "overflows: sample %llu event %llu"
1375				    " total %llu num_sdb %llu\n",
1376				    __func__, sampl_overflow, event_overflow,
1377				    OVERFLOW_REG(hwc), num_sdb);
1378}
1379
1380static inline unsigned long aux_sdb_index(struct aux_buffer *aux,
1381					  unsigned long i)
1382{
1383	return i % aux->sfb.num_sdb;
1384}
1385
1386static inline unsigned long aux_sdb_num(unsigned long start, unsigned long end)
1387{
1388	return end >= start ? end - start + 1 : 0;
1389}
1390
1391static inline unsigned long aux_sdb_num_alert(struct aux_buffer *aux)
1392{
1393	return aux_sdb_num(aux->head, aux->alert_mark);
1394}
1395
1396static inline unsigned long aux_sdb_num_empty(struct aux_buffer *aux)
1397{
1398	return aux_sdb_num(aux->head, aux->empty_mark);
1399}
1400
1401/*
1402 * Get trailer entry by index of SDB.
1403 */
1404static struct hws_trailer_entry *aux_sdb_trailer(struct aux_buffer *aux,
1405						 unsigned long index)
1406{
1407	unsigned long sdb;
1408
1409	index = aux_sdb_index(aux, index);
1410	sdb = aux->sdb_index[index];
1411	return trailer_entry_ptr(sdb);
1412}
1413
1414/*
1415 * Finish sampling on the cpu. Called by cpumsf_pmu_del() with pmu
1416 * disabled. Collect the full SDBs in AUX buffer which have not reached
1417 * the point of alert indicator. And ignore the SDBs which are not
1418 * full.
1419 *
1420 * 1. Scan SDBs to see how much data is there and consume them.
1421 * 2. Remove alert indicator in the buffer.
1422 */
1423static void aux_output_end(struct perf_output_handle *handle)
1424{
1425	unsigned long i, range_scan, idx;
1426	struct aux_buffer *aux;
1427	struct hws_trailer_entry *te;
1428
1429	aux = perf_get_aux(handle);
1430	if (!aux)
1431		return;
1432
1433	range_scan = aux_sdb_num_alert(aux);
1434	for (i = 0, idx = aux->head; i < range_scan; i++, idx++) {
1435		te = aux_sdb_trailer(aux, idx);
1436		if (!te->header.f)
1437			break;
1438	}
1439	/* i is num of SDBs which are full */
1440	perf_aux_output_end(handle, i << PAGE_SHIFT);
1441
1442	/* Remove alert indicators in the buffer */
1443	te = aux_sdb_trailer(aux, aux->alert_mark);
1444	te->header.a = 0;
1445
1446	debug_sprintf_event(sfdbg, 6, "%s: SDBs %ld range %ld head %ld\n",
1447			    __func__, i, range_scan, aux->head);
1448}
1449
1450/*
1451 * Start sampling on the CPU. Called by cpumsf_pmu_add() when an event
1452 * is first added to the CPU or rescheduled again to the CPU. It is called
1453 * with pmu disabled.
1454 *
1455 * 1. Reset the trailer of SDBs to get ready for new data.
1456 * 2. Tell the hardware where to put the data by reset the SDBs buffer
1457 *    head(tear/dear).
1458 */
1459static int aux_output_begin(struct perf_output_handle *handle,
1460			    struct aux_buffer *aux,
1461			    struct cpu_hw_sf *cpuhw)
1462{
1463	unsigned long range, i, range_scan, idx, head, base, offset;
 
 
1464	struct hws_trailer_entry *te;
1465
1466	if (WARN_ON_ONCE(handle->head & ~PAGE_MASK))
1467		return -EINVAL;
1468
1469	aux->head = handle->head >> PAGE_SHIFT;
1470	range = (handle->size + 1) >> PAGE_SHIFT;
1471	if (range <= 1)
1472		return -ENOMEM;
1473
1474	/*
1475	 * SDBs between aux->head and aux->empty_mark are already ready
1476	 * for new data. range_scan is num of SDBs not within them.
1477	 */
1478	debug_sprintf_event(sfdbg, 6,
1479			    "%s: range %ld head %ld alert %ld empty %ld\n",
1480			    __func__, range, aux->head, aux->alert_mark,
1481			    aux->empty_mark);
1482	if (range > aux_sdb_num_empty(aux)) {
1483		range_scan = range - aux_sdb_num_empty(aux);
1484		idx = aux->empty_mark + 1;
1485		for (i = 0; i < range_scan; i++, idx++) {
1486			te = aux_sdb_trailer(aux, idx);
1487			te->header.f = 0;
1488			te->header.a = 0;
1489			te->header.overflow = 0;
1490		}
1491		/* Save the position of empty SDBs */
1492		aux->empty_mark = aux->head + range - 1;
1493	}
1494
1495	/* Set alert indicator */
1496	aux->alert_mark = aux->head + range/2 - 1;
1497	te = aux_sdb_trailer(aux, aux->alert_mark);
1498	te->header.a = 1;
1499
1500	/* Reset hardware buffer head */
1501	head = aux_sdb_index(aux, aux->head);
1502	base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE];
1503	offset = head % CPUM_SF_SDB_PER_TABLE;
1504	cpuhw->lsctl.tear = virt_to_phys((void *)base) + offset * sizeof(unsigned long);
1505	cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]);
1506
1507	debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld empty %ld "
1508			    "index %ld tear %#lx dear %#lx\n", __func__,
1509			    aux->head, aux->alert_mark, aux->empty_mark,
1510			    head / CPUM_SF_SDB_PER_TABLE,
1511			    cpuhw->lsctl.tear, cpuhw->lsctl.dear);
1512
1513	return 0;
1514}
1515
1516/*
1517 * Set alert indicator on SDB at index @alert_index while sampler is running.
1518 *
1519 * Return true if successfully.
1520 * Return false if full indicator is already set by hardware sampler.
1521 */
1522static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index,
1523			  unsigned long long *overflow)
1524{
1525	union hws_trailer_header old, prev, new;
1526	struct hws_trailer_entry *te;
1527
1528	te = aux_sdb_trailer(aux, alert_index);
1529	prev.val = READ_ONCE_ALIGNED_128(te->header.val);
1530	do {
1531		old.val = prev.val;
1532		new.val = prev.val;
1533		*overflow = old.overflow;
1534		if (old.f) {
1535			/*
1536			 * SDB is already set by hardware.
1537			 * Abort and try to set somewhere
1538			 * behind.
1539			 */
1540			return false;
1541		}
1542		new.a = 1;
1543		new.overflow = 0;
1544		prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1545	} while (prev.val != old.val);
1546	return true;
1547}
1548
1549/*
1550 * aux_reset_buffer() - Scan and setup SDBs for new samples
1551 * @aux:	The AUX buffer to set
1552 * @range:	The range of SDBs to scan started from aux->head
1553 * @overflow:	Set to overflow count
1554 *
1555 * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1556 * marked as empty, check if it is already set full by the hardware sampler.
1557 * If yes, that means new data is already there before we can set an alert
1558 * indicator. Caller should try to set alert indicator to some position behind.
1559 *
1560 * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1561 * previously and have already been consumed by user space. Reset these SDBs
1562 * (clear full indicator and alert indicator) for new data.
1563 * If aux->alert_mark fall in this area, just set it. Overflow count is
1564 * recorded while scanning.
1565 *
1566 * SDBs between aux->head and aux->empty_mark are already reset at last time.
1567 * and ready for new samples. So scanning on this area could be skipped.
1568 *
1569 * Return true if alert indicator is set successfully and false if not.
1570 */
1571static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range,
1572			     unsigned long long *overflow)
1573{
 
1574	unsigned long i, range_scan, idx, idx_old;
1575	union hws_trailer_header old, prev, new;
1576	unsigned long long orig_overflow;
1577	struct hws_trailer_entry *te;
1578
1579	debug_sprintf_event(sfdbg, 6, "%s: range %ld head %ld alert %ld "
1580			    "empty %ld\n", __func__, range, aux->head,
1581			    aux->alert_mark, aux->empty_mark);
1582	if (range <= aux_sdb_num_empty(aux))
1583		/*
1584		 * No need to scan. All SDBs in range are marked as empty.
1585		 * Just set alert indicator. Should check race with hardware
1586		 * sampler.
1587		 */
1588		return aux_set_alert(aux, aux->alert_mark, overflow);
1589
1590	if (aux->alert_mark <= aux->empty_mark)
1591		/*
1592		 * Set alert indicator on empty SDB. Should check race
1593		 * with hardware sampler.
1594		 */
1595		if (!aux_set_alert(aux, aux->alert_mark, overflow))
1596			return false;
1597
1598	/*
1599	 * Scan the SDBs to clear full and alert indicator used previously.
1600	 * Start scanning from one SDB behind empty_mark. If the new alert
1601	 * indicator fall into this range, set it.
1602	 */
1603	range_scan = range - aux_sdb_num_empty(aux);
1604	idx_old = idx = aux->empty_mark + 1;
1605	for (i = 0; i < range_scan; i++, idx++) {
1606		te = aux_sdb_trailer(aux, idx);
1607		prev.val = READ_ONCE_ALIGNED_128(te->header.val);
1608		do {
1609			old.val = prev.val;
1610			new.val = prev.val;
1611			orig_overflow = old.overflow;
1612			new.f = 0;
1613			new.overflow = 0;
1614			if (idx == aux->alert_mark)
1615				new.a = 1;
1616			else
1617				new.a = 0;
1618			prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1619		} while (prev.val != old.val);
 
1620		*overflow += orig_overflow;
1621	}
1622
1623	/* Update empty_mark to new position */
1624	aux->empty_mark = aux->head + range - 1;
1625
1626	debug_sprintf_event(sfdbg, 6, "%s: range_scan %ld idx %ld..%ld "
1627			    "empty %ld\n", __func__, range_scan, idx_old,
1628			    idx - 1, aux->empty_mark);
1629	return true;
1630}
1631
1632/*
1633 * Measurement alert handler for diagnostic mode sampling.
1634 */
1635static void hw_collect_aux(struct cpu_hw_sf *cpuhw)
1636{
1637	struct aux_buffer *aux;
1638	int done = 0;
1639	unsigned long range = 0, size;
1640	unsigned long long overflow = 0;
1641	struct perf_output_handle *handle = &cpuhw->handle;
1642	unsigned long num_sdb;
1643
1644	aux = perf_get_aux(handle);
1645	if (WARN_ON_ONCE(!aux))
1646		return;
1647
1648	/* Inform user space new data arrived */
1649	size = aux_sdb_num_alert(aux) << PAGE_SHIFT;
1650	debug_sprintf_event(sfdbg, 6, "%s: #alert %ld\n", __func__,
1651			    size >> PAGE_SHIFT);
1652	perf_aux_output_end(handle, size);
1653
1654	num_sdb = aux->sfb.num_sdb;
1655	while (!done) {
1656		/* Get an output handle */
1657		aux = perf_aux_output_begin(handle, cpuhw->event);
1658		if (handle->size == 0) {
1659			pr_err("The AUX buffer with %lu pages for the "
1660			       "diagnostic-sampling mode is full\n",
1661				num_sdb);
 
 
 
1662			break;
1663		}
1664		if (WARN_ON_ONCE(!aux))
1665			return;
1666
1667		/* Update head and alert_mark to new position */
1668		aux->head = handle->head >> PAGE_SHIFT;
1669		range = (handle->size + 1) >> PAGE_SHIFT;
1670		if (range == 1)
1671			aux->alert_mark = aux->head;
1672		else
1673			aux->alert_mark = aux->head + range/2 - 1;
1674
1675		if (aux_reset_buffer(aux, range, &overflow)) {
1676			if (!overflow) {
1677				done = 1;
1678				break;
1679			}
1680			size = range << PAGE_SHIFT;
1681			perf_aux_output_end(&cpuhw->handle, size);
1682			pr_err("Sample data caused the AUX buffer with %lu "
1683			       "pages to overflow\n", aux->sfb.num_sdb);
1684			debug_sprintf_event(sfdbg, 1, "%s: head %ld range %ld "
1685					    "overflow %lld\n", __func__,
1686					    aux->head, range, overflow);
1687		} else {
1688			size = aux_sdb_num_alert(aux) << PAGE_SHIFT;
1689			perf_aux_output_end(&cpuhw->handle, size);
1690			debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld "
1691					    "already full, try another\n",
1692					    __func__,
1693					    aux->head, aux->alert_mark);
1694		}
1695	}
1696
1697	if (done)
1698		debug_sprintf_event(sfdbg, 6, "%s: head %ld alert %ld "
1699				    "empty %ld\n", __func__, aux->head,
1700				    aux->alert_mark, aux->empty_mark);
1701}
1702
1703/*
1704 * Callback when freeing AUX buffers.
1705 */
1706static void aux_buffer_free(void *data)
1707{
1708	struct aux_buffer *aux = data;
1709	unsigned long i, num_sdbt;
1710
1711	if (!aux)
1712		return;
1713
1714	/* Free SDBT. SDB is freed by the caller */
1715	num_sdbt = aux->sfb.num_sdbt;
1716	for (i = 0; i < num_sdbt; i++)
1717		free_page(aux->sdbt_index[i]);
1718
1719	kfree(aux->sdbt_index);
1720	kfree(aux->sdb_index);
1721	kfree(aux);
1722
1723	debug_sprintf_event(sfdbg, 4, "%s: SDBTs %lu\n", __func__, num_sdbt);
1724}
1725
1726static void aux_sdb_init(unsigned long sdb)
1727{
1728	struct hws_trailer_entry *te;
1729
1730	te = trailer_entry_ptr(sdb);
1731
1732	/* Save clock base */
1733	te->clock_base = 1;
1734	te->progusage2 = tod_clock_base.tod;
1735}
1736
1737/*
1738 * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1739 * @event:	Event the buffer is setup for, event->cpu == -1 means current
1740 * @pages:	Array of pointers to buffer pages passed from perf core
1741 * @nr_pages:	Total pages
1742 * @snapshot:	Flag for snapshot mode
1743 *
1744 * This is the callback when setup an event using AUX buffer. Perf tool can
1745 * trigger this by an additional mmap() call on the event. Unlike the buffer
1746 * for basic samples, AUX buffer belongs to the event. It is scheduled with
1747 * the task among online cpus when it is a per-thread event.
1748 *
1749 * Return the private AUX buffer structure if success or NULL if fails.
1750 */
1751static void *aux_buffer_setup(struct perf_event *event, void **pages,
1752			      int nr_pages, bool snapshot)
1753{
1754	struct sf_buffer *sfb;
1755	struct aux_buffer *aux;
1756	unsigned long *new, *tail;
1757	int i, n_sdbt;
1758
1759	if (!nr_pages || !pages)
1760		return NULL;
1761
1762	if (nr_pages > CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR) {
1763		pr_err("AUX buffer size (%i pages) is larger than the "
1764		       "maximum sampling buffer limit\n",
1765		       nr_pages);
1766		return NULL;
1767	} else if (nr_pages < CPUM_SF_MIN_SDB * CPUM_SF_SDB_DIAG_FACTOR) {
1768		pr_err("AUX buffer size (%i pages) is less than the "
1769		       "minimum sampling buffer limit\n",
1770		       nr_pages);
1771		return NULL;
1772	}
1773
1774	/* Allocate aux_buffer struct for the event */
1775	aux = kzalloc(sizeof(struct aux_buffer), GFP_KERNEL);
1776	if (!aux)
1777		goto no_aux;
1778	sfb = &aux->sfb;
1779
1780	/* Allocate sdbt_index for fast reference */
1781	n_sdbt = DIV_ROUND_UP(nr_pages, CPUM_SF_SDB_PER_TABLE);
1782	aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL);
1783	if (!aux->sdbt_index)
1784		goto no_sdbt_index;
1785
1786	/* Allocate sdb_index for fast reference */
1787	aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL);
1788	if (!aux->sdb_index)
1789		goto no_sdb_index;
1790
1791	/* Allocate the first SDBT */
1792	sfb->num_sdbt = 0;
1793	sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
1794	if (!sfb->sdbt)
1795		goto no_sdbt;
1796	aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;
1797	tail = sfb->tail = sfb->sdbt;
1798
1799	/*
1800	 * Link the provided pages of AUX buffer to SDBT.
1801	 * Allocate SDBT if needed.
1802	 */
1803	for (i = 0; i < nr_pages; i++, tail++) {
1804		if (require_table_link(tail)) {
1805			new = (unsigned long *)get_zeroed_page(GFP_KERNEL);
1806			if (!new)
1807				goto no_sdbt;
1808			aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;
1809			/* Link current page to tail of chain */
1810			*tail = virt_to_phys(new) + 1;
1811			tail = new;
1812		}
1813		/* Tail is the entry in a SDBT */
1814		*tail = virt_to_phys(pages[i]);
1815		aux->sdb_index[i] = (unsigned long)pages[i];
1816		aux_sdb_init((unsigned long)pages[i]);
1817	}
1818	sfb->num_sdb = nr_pages;
1819
1820	/* Link the last entry in the SDBT to the first SDBT */
1821	*tail = virt_to_phys(sfb->sdbt) + 1;
1822	sfb->tail = tail;
1823
1824	/*
1825	 * Initial all SDBs are zeroed. Mark it as empty.
1826	 * So there is no need to clear the full indicator
1827	 * when this event is first added.
1828	 */
1829	aux->empty_mark = sfb->num_sdb - 1;
1830
1831	debug_sprintf_event(sfdbg, 4, "%s: SDBTs %lu SDBs %lu\n", __func__,
1832			    sfb->num_sdbt, sfb->num_sdb);
1833
1834	return aux;
1835
1836no_sdbt:
1837	/* SDBs (AUX buffer pages) are freed by caller */
1838	for (i = 0; i < sfb->num_sdbt; i++)
1839		free_page(aux->sdbt_index[i]);
1840	kfree(aux->sdb_index);
1841no_sdb_index:
1842	kfree(aux->sdbt_index);
1843no_sdbt_index:
1844	kfree(aux);
1845no_aux:
1846	return NULL;
1847}
1848
1849static void cpumsf_pmu_read(struct perf_event *event)
1850{
1851	/* Nothing to do ... updates are interrupt-driven */
1852}
1853
1854/* Check if the new sampling period/frequency is appropriate.
1855 *
1856 * Return non-zero on error and zero on passed checks.
1857 */
1858static int cpumsf_pmu_check_period(struct perf_event *event, u64 value)
1859{
1860	struct hws_qsi_info_block si;
1861	unsigned long rate;
1862	bool do_freq;
1863
1864	memset(&si, 0, sizeof(si));
1865	if (event->cpu == -1) {
1866		if (qsi(&si))
1867			return -ENODEV;
1868	} else {
1869		/* Event is pinned to a particular CPU, retrieve the per-CPU
1870		 * sampling structure for accessing the CPU-specific QSI.
1871		 */
1872		struct cpu_hw_sf *cpuhw = &per_cpu(cpu_hw_sf, event->cpu);
1873
1874		si = cpuhw->qsi;
1875	}
1876
1877	do_freq = !!SAMPLE_FREQ_MODE(&event->hw);
1878	rate = getrate(do_freq, value, &si);
1879	if (!rate)
1880		return -EINVAL;
1881
1882	event->attr.sample_period = rate;
1883	SAMPL_RATE(&event->hw) = rate;
1884	hw_init_period(&event->hw, SAMPL_RATE(&event->hw));
1885	debug_sprintf_event(sfdbg, 4, "%s:"
1886			    " cpu %d value %#llx period %#llx freq %d\n",
1887			    __func__, event->cpu, value,
1888			    event->attr.sample_period, do_freq);
1889	return 0;
1890}
1891
1892/* Activate sampling control.
1893 * Next call of pmu_enable() starts sampling.
1894 */
1895static void cpumsf_pmu_start(struct perf_event *event, int flags)
1896{
1897	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1898
1899	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1900		return;
1901
1902	if (flags & PERF_EF_RELOAD)
1903		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1904
1905	perf_pmu_disable(event->pmu);
1906	event->hw.state = 0;
1907	cpuhw->lsctl.cs = 1;
1908	if (SAMPL_DIAG_MODE(&event->hw))
1909		cpuhw->lsctl.cd = 1;
1910	perf_pmu_enable(event->pmu);
1911}
1912
1913/* Deactivate sampling control.
1914 * Next call of pmu_enable() stops sampling.
1915 */
1916static void cpumsf_pmu_stop(struct perf_event *event, int flags)
1917{
1918	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1919
1920	if (event->hw.state & PERF_HES_STOPPED)
1921		return;
1922
1923	perf_pmu_disable(event->pmu);
1924	cpuhw->lsctl.cs = 0;
1925	cpuhw->lsctl.cd = 0;
1926	event->hw.state |= PERF_HES_STOPPED;
1927
1928	if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) {
1929		hw_perf_event_update(event, 1);
1930		event->hw.state |= PERF_HES_UPTODATE;
1931	}
1932	perf_pmu_enable(event->pmu);
1933}
1934
1935static int cpumsf_pmu_add(struct perf_event *event, int flags)
1936{
1937	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
1938	struct aux_buffer *aux;
1939	int err;
1940
1941	if (cpuhw->flags & PMU_F_IN_USE)
1942		return -EAGAIN;
1943
1944	if (!SAMPL_DIAG_MODE(&event->hw) && !cpuhw->sfb.sdbt)
1945		return -EINVAL;
1946
1947	err = 0;
1948	perf_pmu_disable(event->pmu);
1949
1950	event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1951
1952	/* Set up sampling controls.  Always program the sampling register
1953	 * using the SDB-table start.  Reset TEAR_REG event hardware register
1954	 * that is used by hw_perf_event_update() to store the sampling buffer
1955	 * position after samples have been flushed.
1956	 */
1957	cpuhw->lsctl.s = 0;
1958	cpuhw->lsctl.h = 1;
1959	cpuhw->lsctl.interval = SAMPL_RATE(&event->hw);
1960	if (!SAMPL_DIAG_MODE(&event->hw)) {
1961		cpuhw->lsctl.tear = virt_to_phys(cpuhw->sfb.sdbt);
1962		cpuhw->lsctl.dear = *(unsigned long *)cpuhw->sfb.sdbt;
1963		TEAR_REG(&event->hw) = (unsigned long)cpuhw->sfb.sdbt;
1964	}
1965
1966	/* Ensure sampling functions are in the disabled state.  If disabled,
1967	 * switch on sampling enable control. */
1968	if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) {
1969		err = -EAGAIN;
1970		goto out;
1971	}
1972	if (SAMPL_DIAG_MODE(&event->hw)) {
1973		aux = perf_aux_output_begin(&cpuhw->handle, event);
1974		if (!aux) {
1975			err = -EINVAL;
1976			goto out;
1977		}
1978		err = aux_output_begin(&cpuhw->handle, aux, cpuhw);
1979		if (err)
1980			goto out;
1981		cpuhw->lsctl.ed = 1;
1982	}
1983	cpuhw->lsctl.es = 1;
1984
1985	/* Set in_use flag and store event */
1986	cpuhw->event = event;
1987	cpuhw->flags |= PMU_F_IN_USE;
1988
1989	if (flags & PERF_EF_START)
1990		cpumsf_pmu_start(event, PERF_EF_RELOAD);
1991out:
1992	perf_event_update_userpage(event);
1993	perf_pmu_enable(event->pmu);
1994	return err;
1995}
1996
1997static void cpumsf_pmu_del(struct perf_event *event, int flags)
1998{
1999	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
2000
2001	perf_pmu_disable(event->pmu);
2002	cpumsf_pmu_stop(event, PERF_EF_UPDATE);
2003
2004	cpuhw->lsctl.es = 0;
2005	cpuhw->lsctl.ed = 0;
2006	cpuhw->flags &= ~PMU_F_IN_USE;
2007	cpuhw->event = NULL;
2008
2009	if (SAMPL_DIAG_MODE(&event->hw))
2010		aux_output_end(&cpuhw->handle);
2011	perf_event_update_userpage(event);
2012	perf_pmu_enable(event->pmu);
2013}
2014
2015CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC, PERF_EVENT_CPUM_SF);
2016CPUMF_EVENT_ATTR(SF, SF_CYCLES_BASIC_DIAG, PERF_EVENT_CPUM_SF_DIAG);
2017
2018/* Attribute list for CPU_SF.
2019 *
2020 * The availablitiy depends on the CPU_MF sampling facility authorization
2021 * for basic + diagnositic samples. This is determined at initialization
2022 * time by the sampling facility device driver.
2023 * If the authorization for basic samples is turned off, it should be
2024 * also turned off for diagnostic sampling.
2025 *
2026 * During initialization of the device driver, check the authorization
2027 * level for diagnostic sampling and installs the attribute
2028 * file for diagnostic sampling if necessary.
2029 *
2030 * For now install a placeholder to reference all possible attributes:
2031 * SF_CYCLES_BASIC and SF_CYCLES_BASIC_DIAG.
2032 * Add another entry for the final NULL pointer.
2033 */
2034enum {
2035	SF_CYCLES_BASIC_ATTR_IDX = 0,
2036	SF_CYCLES_BASIC_DIAG_ATTR_IDX,
2037	SF_CYCLES_ATTR_MAX
2038};
2039
2040static struct attribute *cpumsf_pmu_events_attr[SF_CYCLES_ATTR_MAX + 1] = {
2041	[SF_CYCLES_BASIC_ATTR_IDX] = CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC)
2042};
2043
2044PMU_FORMAT_ATTR(event, "config:0-63");
2045
2046static struct attribute *cpumsf_pmu_format_attr[] = {
2047	&format_attr_event.attr,
2048	NULL,
2049};
2050
2051static struct attribute_group cpumsf_pmu_events_group = {
2052	.name = "events",
2053	.attrs = cpumsf_pmu_events_attr,
2054};
2055
2056static struct attribute_group cpumsf_pmu_format_group = {
2057	.name = "format",
2058	.attrs = cpumsf_pmu_format_attr,
2059};
2060
2061static const struct attribute_group *cpumsf_pmu_attr_groups[] = {
2062	&cpumsf_pmu_events_group,
2063	&cpumsf_pmu_format_group,
2064	NULL,
2065};
2066
2067static struct pmu cpumf_sampling = {
2068	.pmu_enable   = cpumsf_pmu_enable,
2069	.pmu_disable  = cpumsf_pmu_disable,
2070
2071	.event_init   = cpumsf_pmu_event_init,
2072	.add	      = cpumsf_pmu_add,
2073	.del	      = cpumsf_pmu_del,
2074
2075	.start	      = cpumsf_pmu_start,
2076	.stop	      = cpumsf_pmu_stop,
2077	.read	      = cpumsf_pmu_read,
2078
2079	.attr_groups  = cpumsf_pmu_attr_groups,
2080
2081	.setup_aux    = aux_buffer_setup,
2082	.free_aux     = aux_buffer_free,
2083
2084	.check_period = cpumsf_pmu_check_period,
2085};
2086
2087static void cpumf_measurement_alert(struct ext_code ext_code,
2088				    unsigned int alert, unsigned long unused)
2089{
2090	struct cpu_hw_sf *cpuhw;
2091
2092	if (!(alert & CPU_MF_INT_SF_MASK))
2093		return;
2094	inc_irq_stat(IRQEXT_CMS);
2095	cpuhw = this_cpu_ptr(&cpu_hw_sf);
2096
2097	/* Measurement alerts are shared and might happen when the PMU
2098	 * is not reserved.  Ignore these alerts in this case. */
2099	if (!(cpuhw->flags & PMU_F_RESERVED))
2100		return;
2101
2102	/* The processing below must take care of multiple alert events that
2103	 * might be indicated concurrently. */
2104
2105	/* Program alert request */
2106	if (alert & CPU_MF_INT_SF_PRA) {
2107		if (cpuhw->flags & PMU_F_IN_USE)
2108			if (SAMPL_DIAG_MODE(&cpuhw->event->hw))
2109				hw_collect_aux(cpuhw);
2110			else
2111				hw_perf_event_update(cpuhw->event, 0);
2112		else
2113			WARN_ON_ONCE(!(cpuhw->flags & PMU_F_IN_USE));
2114	}
2115
2116	/* Report measurement alerts only for non-PRA codes */
2117	if (alert != CPU_MF_INT_SF_PRA)
2118		debug_sprintf_event(sfdbg, 6, "%s: alert %#x\n", __func__,
2119				    alert);
2120
2121	/* Sampling authorization change request */
2122	if (alert & CPU_MF_INT_SF_SACA)
2123		qsi(&cpuhw->qsi);
2124
2125	/* Loss of sample data due to high-priority machine activities */
2126	if (alert & CPU_MF_INT_SF_LSDA) {
2127		pr_err("Sample data was lost\n");
2128		cpuhw->flags |= PMU_F_ERR_LSDA;
2129		sf_disable();
2130	}
2131
2132	/* Invalid sampling buffer entry */
2133	if (alert & (CPU_MF_INT_SF_IAE|CPU_MF_INT_SF_ISE)) {
2134		pr_err("A sampling buffer entry is incorrect (alert=0x%x)\n",
2135		       alert);
2136		cpuhw->flags |= PMU_F_ERR_IBE;
2137		sf_disable();
2138	}
2139}
2140
2141static int cpusf_pmu_setup(unsigned int cpu, int flags)
2142{
2143	/* Ignore the notification if no events are scheduled on the PMU.
2144	 * This might be racy...
2145	 */
2146	if (!atomic_read(&num_events))
2147		return 0;
2148
2149	local_irq_disable();
2150	setup_pmc_cpu(&flags);
2151	local_irq_enable();
2152	return 0;
2153}
2154
2155static int s390_pmu_sf_online_cpu(unsigned int cpu)
2156{
2157	return cpusf_pmu_setup(cpu, PMC_INIT);
2158}
2159
2160static int s390_pmu_sf_offline_cpu(unsigned int cpu)
2161{
2162	return cpusf_pmu_setup(cpu, PMC_RELEASE);
2163}
2164
2165static int param_get_sfb_size(char *buffer, const struct kernel_param *kp)
2166{
2167	if (!cpum_sf_avail())
2168		return -ENODEV;
2169	return sprintf(buffer, "%lu,%lu", CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB);
2170}
2171
2172static int param_set_sfb_size(const char *val, const struct kernel_param *kp)
2173{
2174	int rc;
2175	unsigned long min, max;
2176
2177	if (!cpum_sf_avail())
2178		return -ENODEV;
2179	if (!val || !strlen(val))
2180		return -EINVAL;
2181
2182	/* Valid parameter values: "min,max" or "max" */
2183	min = CPUM_SF_MIN_SDB;
2184	max = CPUM_SF_MAX_SDB;
2185	if (strchr(val, ','))
2186		rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL;
2187	else
2188		rc = kstrtoul(val, 10, &max);
2189
2190	if (min < 2 || min >= max || max > get_num_physpages())
2191		rc = -EINVAL;
2192	if (rc)
2193		return rc;
2194
2195	sfb_set_limits(min, max);
2196	pr_info("The sampling buffer limits have changed to: "
2197		"min %lu max %lu (diag %lu)\n",
2198		CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB, CPUM_SF_SDB_DIAG_FACTOR);
2199	return 0;
2200}
2201
2202#define param_check_sfb_size(name, p) __param_check(name, p, void)
2203static const struct kernel_param_ops param_ops_sfb_size = {
2204	.set = param_set_sfb_size,
2205	.get = param_get_sfb_size,
2206};
2207
2208#define RS_INIT_FAILURE_QSI	  0x0001
2209#define RS_INIT_FAILURE_BSDES	  0x0002
2210#define RS_INIT_FAILURE_ALRT	  0x0003
2211#define RS_INIT_FAILURE_PERF	  0x0004
2212static void __init pr_cpumsf_err(unsigned int reason)
2213{
2214	pr_err("Sampling facility support for perf is not available: "
2215	       "reason %#x\n", reason);
2216}
2217
2218static int __init init_cpum_sampling_pmu(void)
2219{
2220	struct hws_qsi_info_block si;
2221	int err;
2222
2223	if (!cpum_sf_avail())
2224		return -ENODEV;
2225
2226	memset(&si, 0, sizeof(si));
2227	if (qsi(&si)) {
2228		pr_cpumsf_err(RS_INIT_FAILURE_QSI);
2229		return -ENODEV;
2230	}
2231
2232	if (!si.as && !si.ad)
2233		return -ENODEV;
2234
2235	if (si.bsdes != sizeof(struct hws_basic_entry)) {
2236		pr_cpumsf_err(RS_INIT_FAILURE_BSDES);
2237		return -EINVAL;
2238	}
2239
2240	if (si.ad) {
2241		sfb_set_limits(CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB);
2242		/* Sampling of diagnostic data authorized,
2243		 * install event into attribute list of PMU device.
2244		 */
2245		cpumsf_pmu_events_attr[SF_CYCLES_BASIC_DIAG_ATTR_IDX] =
2246			CPUMF_EVENT_PTR(SF, SF_CYCLES_BASIC_DIAG);
2247	}
2248
2249	sfdbg = debug_register(KMSG_COMPONENT, 2, 1, 80);
2250	if (!sfdbg) {
2251		pr_err("Registering for s390dbf failed\n");
2252		return -ENOMEM;
2253	}
2254	debug_register_view(sfdbg, &debug_sprintf_view);
2255
2256	err = register_external_irq(EXT_IRQ_MEASURE_ALERT,
2257				    cpumf_measurement_alert);
2258	if (err) {
2259		pr_cpumsf_err(RS_INIT_FAILURE_ALRT);
2260		debug_unregister(sfdbg);
2261		goto out;
2262	}
2263
2264	err = perf_pmu_register(&cpumf_sampling, "cpum_sf", PERF_TYPE_RAW);
2265	if (err) {
2266		pr_cpumsf_err(RS_INIT_FAILURE_PERF);
2267		unregister_external_irq(EXT_IRQ_MEASURE_ALERT,
2268					cpumf_measurement_alert);
2269		debug_unregister(sfdbg);
2270		goto out;
2271	}
2272
2273	cpuhp_setup_state(CPUHP_AP_PERF_S390_SF_ONLINE, "perf/s390/sf:online",
2274			  s390_pmu_sf_online_cpu, s390_pmu_sf_offline_cpu);
2275out:
2276	return err;
2277}
2278
2279arch_initcall(init_cpum_sampling_pmu);
2280core_param(cpum_sfb_size, CPUM_SF_MAX_SDB, sfb_size, 0644);