1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Performance event support for s390x - CPU-measurement Counter Facility
   4 *
   5 *  Copyright IBM Corp. 2012, 2023
   6 *  Author(s): Hendrik Brueckner <brueckner@linux.ibm.com>
   7 *	       Thomas Richter <tmricht@linux.ibm.com>
 
 
 
   8 */
   9#define KMSG_COMPONENT	"cpum_cf"
  10#define pr_fmt(fmt)	KMSG_COMPONENT ": " fmt
  11
  12#include <linux/kernel.h>
  13#include <linux/kernel_stat.h>
 
  14#include <linux/percpu.h>
  15#include <linux/notifier.h>
  16#include <linux/init.h>
  17#include <linux/export.h>
  18#include <linux/miscdevice.h>
  19#include <linux/perf_event.h>
  20
  21#include <asm/cpu_mf.h>
  22#include <asm/hwctrset.h>
  23#include <asm/debug.h>
  24
  25/* Perf PMU definitions for the counter facility */
  26#define PERF_CPUM_CF_MAX_CTR		0xffffUL  /* Max ctr for ECCTR */
  27#define PERF_EVENT_CPUM_CF_DIAG		0xBC000UL /* Event: Counter sets */
  28
 
 
 
 
 
 
 
  29enum cpumf_ctr_set {
  30	CPUMF_CTR_SET_BASIC   = 0,    /* Basic Counter Set */
  31	CPUMF_CTR_SET_USER    = 1,    /* Problem-State Counter Set */
  32	CPUMF_CTR_SET_CRYPTO  = 2,    /* Crypto-Activity Counter Set */
  33	CPUMF_CTR_SET_EXT     = 3,    /* Extended Counter Set */
  34	CPUMF_CTR_SET_MT_DIAG = 4,    /* MT-diagnostic Counter Set */
  35
  36	/* Maximum number of counter sets */
  37	CPUMF_CTR_SET_MAX,
  38};
  39
  40#define CPUMF_LCCTL_ENABLE_SHIFT    16
  41#define CPUMF_LCCTL_ACTCTL_SHIFT     0
 
 
 
 
 
 
  42
  43static inline void ctr_set_enable(u64 *state, u64 ctrsets)
  44{
  45	*state |= ctrsets << CPUMF_LCCTL_ENABLE_SHIFT;
  46}
  47
  48static inline void ctr_set_disable(u64 *state, u64 ctrsets)
  49{
  50	*state &= ~(ctrsets << CPUMF_LCCTL_ENABLE_SHIFT);
  51}
  52
  53static inline void ctr_set_start(u64 *state, u64 ctrsets)
  54{
  55	*state |= ctrsets << CPUMF_LCCTL_ACTCTL_SHIFT;
  56}
  57
  58static inline void ctr_set_stop(u64 *state, u64 ctrsets)
  59{
  60	*state &= ~(ctrsets << CPUMF_LCCTL_ACTCTL_SHIFT);
  61}
  62
  63static inline int ctr_stcctm(enum cpumf_ctr_set set, u64 range, u64 *dest)
  64{
  65	switch (set) {
  66	case CPUMF_CTR_SET_BASIC:
  67		return stcctm(BASIC, range, dest);
  68	case CPUMF_CTR_SET_USER:
  69		return stcctm(PROBLEM_STATE, range, dest);
  70	case CPUMF_CTR_SET_CRYPTO:
  71		return stcctm(CRYPTO_ACTIVITY, range, dest);
  72	case CPUMF_CTR_SET_EXT:
  73		return stcctm(EXTENDED, range, dest);
  74	case CPUMF_CTR_SET_MT_DIAG:
  75		return stcctm(MT_DIAG_CLEARING, range, dest);
  76	case CPUMF_CTR_SET_MAX:
  77		return 3;
  78	}
  79	return 3;
  80}
  81
  82struct cpu_cf_events {
  83	refcount_t refcnt;		/* Reference count */
  84	atomic_t		ctr_set[CPUMF_CTR_SET_MAX];
  85	u64			state;		/* For perf_event_open SVC */
  86	u64			dev_state;	/* For /dev/hwctr */
  87	unsigned int		flags;
  88	size_t used;			/* Bytes used in data */
  89	size_t usedss;			/* Bytes used in start/stop */
  90	unsigned char start[PAGE_SIZE];	/* Counter set at event add */
  91	unsigned char stop[PAGE_SIZE];	/* Counter set at event delete */
  92	unsigned char data[PAGE_SIZE];	/* Counter set at /dev/hwctr */
  93	unsigned int sets;		/* # Counter set saved in memory */
  94};
  95
  96static unsigned int cfdiag_cpu_speed;	/* CPU speed for CF_DIAG trailer */
  97static debug_info_t *cf_dbg;
  98
  99/*
 100 * The CPU Measurement query counter information instruction contains
 101 * information which varies per machine generation, but is constant and
 102 * does not change when running on a particular machine, such as counter
 103 * first and second version number. This is needed to determine the size
 104 * of counter sets. Extract this information at device driver initialization.
 105 */
 106static struct cpumf_ctr_info	cpumf_ctr_info;
 107
 108struct cpu_cf_ptr {
 109	struct cpu_cf_events *cpucf;
 110};
 111
 112static struct cpu_cf_root {		/* Anchor to per CPU data */
 113	refcount_t refcnt;		/* Overall active events */
 114	struct cpu_cf_ptr __percpu *cfptr;
 115} cpu_cf_root;
 116
 117/*
 118 * Serialize event initialization and event removal. Both are called from
 119 * user space in task context with perf_event_open() and close()
 120 * system calls.
 121 *
 122 * This mutex serializes functions cpum_cf_alloc_cpu() called at event
 123 * initialization via cpumf_pmu_event_init() and function cpum_cf_free_cpu()
 124 * called at event removal via call back function hw_perf_event_destroy()
 125 * when the event is deleted. They are serialized to enforce correct
 126 * bookkeeping of pointer and reference counts anchored by
 127 * struct cpu_cf_root and the access to cpu_cf_root::refcnt and the
 128 * per CPU pointers stored in cpu_cf_root::cfptr.
 129 */
 130static DEFINE_MUTEX(pmc_reserve_mutex);
 131
 132/*
 133 * Get pointer to per-cpu structure.
 134 *
 135 * Function get_cpu_cfhw() is called from
 136 * - cfset_copy_all(): This function is protected by cpus_read_lock(), so
 137 *   CPU hot plug remove can not happen. Event removal requires a close()
 138 *   first.
 139 *
 140 * Function this_cpu_cfhw() is called from perf common code functions:
 141 * - pmu_{en|dis}able(), pmu_{add|del}()and pmu_{start|stop}():
 142 *   All functions execute with interrupts disabled on that particular CPU.
 143 * - cfset_ioctl_{on|off}, cfset_cpu_read(): see comment cfset_copy_all().
 144 *
 145 * Therefore it is safe to access the CPU specific pointer to the event.
 146 */
 147static struct cpu_cf_events *get_cpu_cfhw(int cpu)
 148{
 149	struct cpu_cf_ptr __percpu *p = cpu_cf_root.cfptr;
 150
 151	if (p) {
 152		struct cpu_cf_ptr *q = per_cpu_ptr(p, cpu);
 153
 154		return q->cpucf;
 155	}
 156	return NULL;
 157}
 
 
 
 
 158
 159static struct cpu_cf_events *this_cpu_cfhw(void)
 160{
 161	return get_cpu_cfhw(smp_processor_id());
 162}
 163
 164/* Disable counter sets on dedicated CPU */
 165static void cpum_cf_reset_cpu(void *flags)
 166{
 167	lcctl(0);
 168}
 
 
 
 
 
 
 
 
 
 
 
 
 169
 170/* Free per CPU data when the last event is removed. */
 171static void cpum_cf_free_root(void)
 172{
 173	if (!refcount_dec_and_test(&cpu_cf_root.refcnt))
 174		return;
 175	free_percpu(cpu_cf_root.cfptr);
 176	cpu_cf_root.cfptr = NULL;
 177	irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
 178	on_each_cpu(cpum_cf_reset_cpu, NULL, 1);
 179	debug_sprintf_event(cf_dbg, 4, "%s root.refcnt %u cfptr %d\n",
 180			    __func__, refcount_read(&cpu_cf_root.refcnt),
 181			    !cpu_cf_root.cfptr);
 182}
 183
 184/*
 185 * On initialization of first event also allocate per CPU data dynamically.
 186 * Start with an array of pointers, the array size is the maximum number of
 187 * CPUs possible, which might be larger than the number of CPUs currently
 188 * online.
 189 */
 190static int cpum_cf_alloc_root(void)
 191{
 192	int rc = 0;
 
 193
 194	if (refcount_inc_not_zero(&cpu_cf_root.refcnt))
 195		return rc;
 196
 197	/* The memory is already zeroed. */
 198	cpu_cf_root.cfptr = alloc_percpu(struct cpu_cf_ptr);
 199	if (cpu_cf_root.cfptr) {
 200		refcount_set(&cpu_cf_root.refcnt, 1);
 201		on_each_cpu(cpum_cf_reset_cpu, NULL, 1);
 202		irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
 203	} else {
 204		rc = -ENOMEM;
 
 
 
 
 
 
 
 
 205	}
 206
 207	return rc;
 
 208}
 209
 210/* Free CPU counter data structure for a PMU */
 211static void cpum_cf_free_cpu(int cpu)
 212{
 213	struct cpu_cf_events *cpuhw;
 214	struct cpu_cf_ptr *p;
 215
 216	mutex_lock(&pmc_reserve_mutex);
 217	/*
 218	 * When invoked via CPU hotplug handler, there might be no events
 219	 * installed or that particular CPU might not have an
 220	 * event installed. This anchor pointer can be NULL!
 221	 */
 222	if (!cpu_cf_root.cfptr)
 223		goto out;
 224	p = per_cpu_ptr(cpu_cf_root.cfptr, cpu);
 225	cpuhw = p->cpucf;
 226	/*
 227	 * Might be zero when called from CPU hotplug handler and no event
 228	 * installed on that CPU, but on different CPUs.
 229	 */
 230	if (!cpuhw)
 231		goto out;
 232
 233	if (refcount_dec_and_test(&cpuhw->refcnt)) {
 234		kfree(cpuhw);
 235		p->cpucf = NULL;
 236	}
 237	cpum_cf_free_root();
 238out:
 239	mutex_unlock(&pmc_reserve_mutex);
 240}
 241
 242/* Allocate CPU counter data structure for a PMU. Called under mutex lock. */
 243static int cpum_cf_alloc_cpu(int cpu)
 244{
 245	struct cpu_cf_events *cpuhw;
 246	struct cpu_cf_ptr *p;
 247	int rc;
 
 
 248
 249	mutex_lock(&pmc_reserve_mutex);
 250	rc = cpum_cf_alloc_root();
 251	if (rc)
 252		goto unlock;
 253	p = per_cpu_ptr(cpu_cf_root.cfptr, cpu);
 254	cpuhw = p->cpucf;
 255
 256	if (!cpuhw) {
 257		cpuhw = kzalloc(sizeof(*cpuhw), GFP_KERNEL);
 258		if (cpuhw) {
 259			p->cpucf = cpuhw;
 260			refcount_set(&cpuhw->refcnt, 1);
 261		} else {
 262			rc = -ENOMEM;
 263		}
 264	} else {
 265		refcount_inc(&cpuhw->refcnt);
 266	}
 267	if (rc) {
 268		/*
 269		 * Error in allocation of event, decrement anchor. Since
 270		 * cpu_cf_event in not created, its destroy() function is not
 271		 * invoked. Adjust the reference counter for the anchor.
 272		 */
 273		cpum_cf_free_root();
 274	}
 275unlock:
 276	mutex_unlock(&pmc_reserve_mutex);
 277	return rc;
 278}
 279
 280/*
 281 * Create/delete per CPU data structures for /dev/hwctr interface and events
 282 * created by perf_event_open().
 283 * If cpu is -1, track task on all available CPUs. This requires
 284 * allocation of hardware data structures for all CPUs. This setup handles
 285 * perf_event_open() with task context and /dev/hwctr interface.
 286 * If cpu is non-zero install event on this CPU only. This setup handles
 287 * perf_event_open() with CPU context.
 288 */
 289static int cpum_cf_alloc(int cpu)
 290{
 291	cpumask_var_t mask;
 292	int rc;
 293
 294	if (cpu == -1) {
 295		if (!zalloc_cpumask_var(&mask, GFP_KERNEL))
 296			return -ENOMEM;
 297		for_each_online_cpu(cpu) {
 298			rc = cpum_cf_alloc_cpu(cpu);
 299			if (rc) {
 300				for_each_cpu(cpu, mask)
 301					cpum_cf_free_cpu(cpu);
 302				break;
 303			}
 304			cpumask_set_cpu(cpu, mask);
 305		}
 306		free_cpumask_var(mask);
 307	} else {
 308		rc = cpum_cf_alloc_cpu(cpu);
 309	}
 310	return rc;
 311}
 312
 313static void cpum_cf_free(int cpu)
 314{
 315	if (cpu == -1) {
 316		for_each_online_cpu(cpu)
 317			cpum_cf_free_cpu(cpu);
 318	} else {
 319		cpum_cf_free_cpu(cpu);
 320	}
 321}
 322
 323#define	CF_DIAG_CTRSET_DEF		0xfeef	/* Counter set header mark */
 324						/* interval in seconds */
 325
 326/* Counter sets are stored as data stream in a page sized memory buffer and
 327 * exported to user space via raw data attached to the event sample data.
 328 * Each counter set starts with an eight byte header consisting of:
 329 * - a two byte eye catcher (0xfeef)
 330 * - a one byte counter set number
 331 * - a two byte counter set size (indicates the number of counters in this set)
 332 * - a three byte reserved value (must be zero) to make the header the same
 333 *   size as a counter value.
 334 * All counter values are eight byte in size.
 335 *
 336 * All counter sets are followed by a 64 byte trailer.
 337 * The trailer consists of a:
 338 * - flag field indicating valid fields when corresponding bit set
 339 * - the counter facility first and second version number
 340 * - the CPU speed if nonzero
 341 * - the time stamp the counter sets have been collected
 342 * - the time of day (TOD) base value
 343 * - the machine type.
 344 *
 345 * The counter sets are saved when the process is prepared to be executed on a
 346 * CPU and saved again when the process is going to be removed from a CPU.
 347 * The difference of both counter sets are calculated and stored in the event
 348 * sample data area.
 349 */
 350struct cf_ctrset_entry {	/* CPU-M CF counter set entry (8 byte) */
 351	unsigned int def:16;	/* 0-15  Data Entry Format */
 352	unsigned int set:16;	/* 16-31 Counter set identifier */
 353	unsigned int ctr:16;	/* 32-47 Number of stored counters */
 354	unsigned int res1:16;	/* 48-63 Reserved */
 355};
 356
 357struct cf_trailer_entry {	/* CPU-M CF_DIAG trailer (64 byte) */
 358	/* 0 - 7 */
 359	union {
 360		struct {
 361			unsigned int clock_base:1;	/* TOD clock base set */
 362			unsigned int speed:1;		/* CPU speed set */
 363			/* Measurement alerts */
 364			unsigned int mtda:1;	/* Loss of MT ctr. data alert */
 365			unsigned int caca:1;	/* Counter auth. change alert */
 366			unsigned int lcda:1;	/* Loss of counter data alert */
 367		};
 368		unsigned long flags;	/* 0-63    All indicators */
 369	};
 370	/* 8 - 15 */
 371	unsigned int cfvn:16;			/* 64-79   Ctr First Version */
 372	unsigned int csvn:16;			/* 80-95   Ctr Second Version */
 373	unsigned int cpu_speed:32;		/* 96-127  CPU speed */
 374	/* 16 - 23 */
 375	unsigned long timestamp;		/* 128-191 Timestamp (TOD) */
 376	/* 24 - 55 */
 377	union {
 378		struct {
 379			unsigned long progusage1;
 380			unsigned long progusage2;
 381			unsigned long progusage3;
 382			unsigned long tod_base;
 383		};
 384		unsigned long progusage[4];
 385	};
 386	/* 56 - 63 */
 387	unsigned int mach_type:16;		/* Machine type */
 388	unsigned int res1:16;			/* Reserved */
 389	unsigned int res2:32;			/* Reserved */
 390};
 391
 392/* Create the trailer data at the end of a page. */
 393static void cfdiag_trailer(struct cf_trailer_entry *te)
 394{
 395	struct cpuid cpuid;
 396
 397	te->cfvn = cpumf_ctr_info.cfvn;		/* Counter version numbers */
 398	te->csvn = cpumf_ctr_info.csvn;
 399
 400	get_cpu_id(&cpuid);			/* Machine type */
 401	te->mach_type = cpuid.machine;
 402	te->cpu_speed = cfdiag_cpu_speed;
 403	if (te->cpu_speed)
 404		te->speed = 1;
 405	te->clock_base = 1;			/* Save clock base */
 406	te->tod_base = tod_clock_base.tod;
 407	te->timestamp = get_tod_clock_fast();
 408}
 409
 410/*
 411 * The number of counters per counter set varies between machine generations,
 412 * but is constant when running on a particular machine generation.
 413 * Determine each counter set size at device driver initialization and
 414 * retrieve it later.
 415 */
 416static size_t cpumf_ctr_setsizes[CPUMF_CTR_SET_MAX];
 417static void cpum_cf_make_setsize(enum cpumf_ctr_set ctrset)
 418{
 419	size_t ctrset_size = 0;
 420
 421	switch (ctrset) {
 422	case CPUMF_CTR_SET_BASIC:
 423		if (cpumf_ctr_info.cfvn >= 1)
 424			ctrset_size = 6;
 425		break;
 426	case CPUMF_CTR_SET_USER:
 427		if (cpumf_ctr_info.cfvn == 1)
 428			ctrset_size = 6;
 429		else if (cpumf_ctr_info.cfvn >= 3)
 430			ctrset_size = 2;
 431		break;
 432	case CPUMF_CTR_SET_CRYPTO:
 433		if (cpumf_ctr_info.csvn >= 1 && cpumf_ctr_info.csvn <= 5)
 434			ctrset_size = 16;
 435		else if (cpumf_ctr_info.csvn >= 6)
 436			ctrset_size = 20;
 437		break;
 438	case CPUMF_CTR_SET_EXT:
 439		if (cpumf_ctr_info.csvn == 1)
 440			ctrset_size = 32;
 441		else if (cpumf_ctr_info.csvn == 2)
 442			ctrset_size = 48;
 443		else if (cpumf_ctr_info.csvn >= 3 && cpumf_ctr_info.csvn <= 5)
 444			ctrset_size = 128;
 445		else if (cpumf_ctr_info.csvn == 6 || cpumf_ctr_info.csvn == 7)
 446			ctrset_size = 160;
 447		break;
 448	case CPUMF_CTR_SET_MT_DIAG:
 449		if (cpumf_ctr_info.csvn > 3)
 450			ctrset_size = 48;
 451		break;
 452	case CPUMF_CTR_SET_MAX:
 453		break;
 454	}
 455	cpumf_ctr_setsizes[ctrset] = ctrset_size;
 456}
 457
 458/*
 459 * Return the maximum possible counter set size (in number of 8 byte counters)
 460 * depending on type and model number.
 
 461 */
 462static size_t cpum_cf_read_setsize(enum cpumf_ctr_set ctrset)
 463{
 464	return cpumf_ctr_setsizes[ctrset];
 465}
 
 466
 467/* Read a counter set. The counter set number determines the counter set and
 468 * the CPUM-CF first and second version number determine the number of
 469 * available counters in each counter set.
 470 * Each counter set starts with header containing the counter set number and
 471 * the number of eight byte counters.
 472 *
 473 * The functions returns the number of bytes occupied by this counter set
 474 * including the header.
 475 * If there is no counter in the counter set, this counter set is useless and
 476 * zero is returned on this case.
 477 *
 478 * Note that the counter sets may not be enabled or active and the stcctm
 479 * instruction might return error 3. Depending on error_ok value this is ok,
 480 * for example when called from cpumf_pmu_start() call back function.
 481 */
 482static size_t cfdiag_getctrset(struct cf_ctrset_entry *ctrdata, int ctrset,
 483			       size_t room, bool error_ok)
 484{
 485	size_t ctrset_size, need = 0;
 486	int rc = 3;				/* Assume write failure */
 487
 488	ctrdata->def = CF_DIAG_CTRSET_DEF;
 489	ctrdata->set = ctrset;
 490	ctrdata->res1 = 0;
 491	ctrset_size = cpum_cf_read_setsize(ctrset);
 492
 493	if (ctrset_size) {			/* Save data */
 494		need = ctrset_size * sizeof(u64) + sizeof(*ctrdata);
 495		if (need <= room) {
 496			rc = ctr_stcctm(ctrset, ctrset_size,
 497					(u64 *)(ctrdata + 1));
 498		}
 499		if (rc != 3 || error_ok)
 500			ctrdata->ctr = ctrset_size;
 501		else
 502			need = 0;
 503	}
 504
 505	return need;
 506}
 507
 508static const u64 cpumf_ctr_ctl[CPUMF_CTR_SET_MAX] = {
 509	[CPUMF_CTR_SET_BASIC]	= 0x02,
 510	[CPUMF_CTR_SET_USER]	= 0x04,
 511	[CPUMF_CTR_SET_CRYPTO]	= 0x08,
 512	[CPUMF_CTR_SET_EXT]	= 0x01,
 513	[CPUMF_CTR_SET_MT_DIAG] = 0x20,
 514};
 515
 516/* Read out all counter sets and save them in the provided data buffer.
 517 * The last 64 byte host an artificial trailer entry.
 518 */
 519static size_t cfdiag_getctr(void *data, size_t sz, unsigned long auth,
 520			    bool error_ok)
 521{
 522	struct cf_trailer_entry *trailer;
 523	size_t offset = 0, done;
 524	int i;
 525
 526	memset(data, 0, sz);
 527	sz -= sizeof(*trailer);		/* Always room for trailer */
 528	for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
 529		struct cf_ctrset_entry *ctrdata = data + offset;
 530
 531		if (!(auth & cpumf_ctr_ctl[i]))
 532			continue;	/* Counter set not authorized */
 533
 534		done = cfdiag_getctrset(ctrdata, i, sz - offset, error_ok);
 535		offset += done;
 536	}
 537	trailer = data + offset;
 538	cfdiag_trailer(trailer);
 539	return offset + sizeof(*trailer);
 540}
 541
 542/* Calculate the difference for each counter in a counter set. */
 543static void cfdiag_diffctrset(u64 *pstart, u64 *pstop, int counters)
 
 544{
 545	for (; --counters >= 0; ++pstart, ++pstop)
 546		if (*pstop >= *pstart)
 547			*pstop -= *pstart;
 548		else
 549			*pstop = *pstart - *pstop + 1;
 550}
 551
 552/* Scan the counter sets and calculate the difference of each counter
 553 * in each set. The result is the increment of each counter during the
 554 * period the counter set has been activated.
 555 *
 556 * Return true on success.
 557 */
 558static int cfdiag_diffctr(struct cpu_cf_events *cpuhw, unsigned long auth)
 559{
 560	struct cf_trailer_entry *trailer_start, *trailer_stop;
 561	struct cf_ctrset_entry *ctrstart, *ctrstop;
 562	size_t offset = 0;
 563	int i;
 564
 565	for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
 566		ctrstart = (struct cf_ctrset_entry *)(cpuhw->start + offset);
 567		ctrstop = (struct cf_ctrset_entry *)(cpuhw->stop + offset);
 568
 569		/* Counter set not authorized */
 570		if (!(auth & cpumf_ctr_ctl[i]))
 571			continue;
 572		/* Counter set size zero was not saved */
 573		if (!cpum_cf_read_setsize(i))
 574			continue;
 575
 576		if (memcmp(ctrstop, ctrstart, sizeof(*ctrstop))) {
 577			pr_err_once("cpum_cf_diag counter set compare error "
 578				    "in set %i\n", ctrstart->set);
 579			return 0;
 580		}
 581		if (ctrstart->def == CF_DIAG_CTRSET_DEF) {
 582			cfdiag_diffctrset((u64 *)(ctrstart + 1),
 583					  (u64 *)(ctrstop + 1), ctrstart->ctr);
 584			offset += ctrstart->ctr * sizeof(u64) +
 585							sizeof(*ctrstart);
 586		}
 587	}
 588
 589	/* Save time_stamp from start of event in stop's trailer */
 590	trailer_start = (struct cf_trailer_entry *)(cpuhw->start + offset);
 591	trailer_stop = (struct cf_trailer_entry *)(cpuhw->stop + offset);
 592	trailer_stop->progusage[0] = trailer_start->timestamp;
 593
 594	return 1;
 595}
 596
 597static enum cpumf_ctr_set get_counter_set(u64 event)
 598{
 599	int set = CPUMF_CTR_SET_MAX;
 
 600
 601	if (event < 32)
 602		set = CPUMF_CTR_SET_BASIC;
 603	else if (event < 64)
 604		set = CPUMF_CTR_SET_USER;
 605	else if (event < 128)
 606		set = CPUMF_CTR_SET_CRYPTO;
 607	else if (event < 288)
 608		set = CPUMF_CTR_SET_EXT;
 609	else if (event >= 448 && event < 496)
 610		set = CPUMF_CTR_SET_MT_DIAG;
 611
 612	return set;
 
 
 613}
 614
 615static int validate_ctr_version(const u64 config, enum cpumf_ctr_set set)
 
 
 616{
 617	u16 mtdiag_ctl;
 618	int err = 0;
 619
 620	/* check required version for counter sets */
 621	switch (set) {
 622	case CPUMF_CTR_SET_BASIC:
 623	case CPUMF_CTR_SET_USER:
 624		if (cpumf_ctr_info.cfvn < 1)
 625			err = -EOPNOTSUPP;
 626		break;
 627	case CPUMF_CTR_SET_CRYPTO:
 628		if ((cpumf_ctr_info.csvn >= 1 && cpumf_ctr_info.csvn <= 5 &&
 629		     config > 79) || (cpumf_ctr_info.csvn >= 6 && config > 83))
 630			err = -EOPNOTSUPP;
 631		break;
 632	case CPUMF_CTR_SET_EXT:
 633		if (cpumf_ctr_info.csvn < 1)
 634			err = -EOPNOTSUPP;
 635		if ((cpumf_ctr_info.csvn == 1 && config > 159) ||
 636		    (cpumf_ctr_info.csvn == 2 && config > 175) ||
 637		    (cpumf_ctr_info.csvn >= 3 && cpumf_ctr_info.csvn <= 5 &&
 638		     config > 255) ||
 639		    (cpumf_ctr_info.csvn >= 6 && config > 287))
 640			err = -EOPNOTSUPP;
 641		break;
 642	case CPUMF_CTR_SET_MT_DIAG:
 643		if (cpumf_ctr_info.csvn <= 3)
 644			err = -EOPNOTSUPP;
 645		/*
 646		 * MT-diagnostic counters are read-only.  The counter set
 647		 * is automatically enabled and activated on all CPUs with
 648		 * multithreading (SMT).  Deactivation of multithreading
 649		 * also disables the counter set.  State changes are ignored
 650		 * by lcctl().	Because Linux controls SMT enablement through
 651		 * a kernel parameter only, the counter set is either disabled
 652		 * or enabled and active.
 653		 *
 654		 * Thus, the counters can only be used if SMT is on and the
 655		 * counter set is enabled and active.
 656		 */
 657		mtdiag_ctl = cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG];
 658		if (!((cpumf_ctr_info.auth_ctl & mtdiag_ctl) &&
 659		      (cpumf_ctr_info.enable_ctl & mtdiag_ctl) &&
 660		      (cpumf_ctr_info.act_ctl & mtdiag_ctl)))
 661			err = -EOPNOTSUPP;
 662		break;
 663	case CPUMF_CTR_SET_MAX:
 664		err = -EOPNOTSUPP;
 665	}
 666
 667	return err;
 
 668}
 669
 670/*
 671 * Change the CPUMF state to active.
 672 * Enable and activate the CPU-counter sets according
 673 * to the per-cpu control state.
 674 */
 675static void cpumf_pmu_enable(struct pmu *pmu)
 676{
 677	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
 678	int err;
 679
 680	if (!cpuhw || (cpuhw->flags & PMU_F_ENABLED))
 681		return;
 682
 683	err = lcctl(cpuhw->state | cpuhw->dev_state);
 684	if (err)
 685		pr_err("Enabling the performance measuring unit failed with rc=%x\n", err);
 686	else
 687		cpuhw->flags |= PMU_F_ENABLED;
 688}
 689
 690/*
 691 * Change the CPUMF state to inactive.
 692 * Disable and enable (inactive) the CPU-counter sets according
 693 * to the per-cpu control state.
 694 */
 695static void cpumf_pmu_disable(struct pmu *pmu)
 696{
 697	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
 698	u64 inactive;
 699	int err;
 700
 701	if (!cpuhw || !(cpuhw->flags & PMU_F_ENABLED))
 702		return;
 703
 704	inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);
 705	inactive |= cpuhw->dev_state;
 706	err = lcctl(inactive);
 707	if (err)
 708		pr_err("Disabling the performance measuring unit failed with rc=%x\n", err);
 709	else
 710		cpuhw->flags &= ~PMU_F_ENABLED;
 711}
 712
 713/* Release the PMU if event is the last perf event */
 714static void hw_perf_event_destroy(struct perf_event *event)
 715{
 716	cpum_cf_free(event->cpu);
 
 
 
 
 
 717}
 718
 719/* CPUMF <-> perf event mappings for kernel+userspace (basic set) */
 720static const int cpumf_generic_events_basic[] = {
 721	[PERF_COUNT_HW_CPU_CYCLES]	    = 0,
 722	[PERF_COUNT_HW_INSTRUCTIONS]	    = 1,
 723	[PERF_COUNT_HW_CACHE_REFERENCES]    = -1,
 724	[PERF_COUNT_HW_CACHE_MISSES]	    = -1,
 725	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1,
 726	[PERF_COUNT_HW_BRANCH_MISSES]	    = -1,
 727	[PERF_COUNT_HW_BUS_CYCLES]	    = -1,
 728};
 729/* CPUMF <-> perf event mappings for userspace (problem-state set) */
 730static const int cpumf_generic_events_user[] = {
 731	[PERF_COUNT_HW_CPU_CYCLES]	    = 32,
 732	[PERF_COUNT_HW_INSTRUCTIONS]	    = 33,
 733	[PERF_COUNT_HW_CACHE_REFERENCES]    = -1,
 734	[PERF_COUNT_HW_CACHE_MISSES]	    = -1,
 735	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1,
 736	[PERF_COUNT_HW_BRANCH_MISSES]	    = -1,
 737	[PERF_COUNT_HW_BUS_CYCLES]	    = -1,
 738};
 739
 740static int is_userspace_event(u64 ev)
 741{
 742	return cpumf_generic_events_user[PERF_COUNT_HW_CPU_CYCLES] == ev ||
 743	       cpumf_generic_events_user[PERF_COUNT_HW_INSTRUCTIONS] == ev;
 744}
 745
 746static int __hw_perf_event_init(struct perf_event *event, unsigned int type)
 747{
 748	struct perf_event_attr *attr = &event->attr;
 749	struct hw_perf_event *hwc = &event->hw;
 750	enum cpumf_ctr_set set;
 751	u64 ev;
 752
 753	switch (type) {
 754	case PERF_TYPE_RAW:
 755		/* Raw events are used to access counters directly,
 756		 * hence do not permit excludes */
 757		if (attr->exclude_kernel || attr->exclude_user ||
 758		    attr->exclude_hv)
 759			return -EOPNOTSUPP;
 760		ev = attr->config;
 761		break;
 762
 763	case PERF_TYPE_HARDWARE:
 764		if (is_sampling_event(event))	/* No sampling support */
 765			return -ENOENT;
 766		ev = attr->config;
 
 767		if (!attr->exclude_user && attr->exclude_kernel) {
 768			/*
 769			 * Count user space (problem-state) only
 770			 * Handle events 32 and 33 as 0:u and 1:u
 771			 */
 772			if (!is_userspace_event(ev)) {
 773				if (ev >= ARRAY_SIZE(cpumf_generic_events_user))
 774					return -EOPNOTSUPP;
 775				ev = cpumf_generic_events_user[ev];
 776			}
 777		} else if (!attr->exclude_kernel && attr->exclude_user) {
 778			/* No support for kernel space counters only */
 779			return -EOPNOTSUPP;
 
 
 780		} else {
 781			/* Count user and kernel space, incl. events 32 + 33 */
 782			if (!is_userspace_event(ev)) {
 783				if (ev >= ARRAY_SIZE(cpumf_generic_events_basic))
 784					return -EOPNOTSUPP;
 785				ev = cpumf_generic_events_basic[ev];
 786			}
 787		}
 788		break;
 789
 790	default:
 791		return -ENOENT;
 792	}
 793
 794	if (ev == -1)
 795		return -ENOENT;
 796
 797	if (ev > PERF_CPUM_CF_MAX_CTR)
 798		return -ENOENT;
 799
 800	/* Obtain the counter set to which the specified counter belongs */
 801	set = get_counter_set(ev);
 802	switch (set) {
 803	case CPUMF_CTR_SET_BASIC:
 804	case CPUMF_CTR_SET_USER:
 805	case CPUMF_CTR_SET_CRYPTO:
 806	case CPUMF_CTR_SET_EXT:
 807	case CPUMF_CTR_SET_MT_DIAG:
 808		/*
 809		 * Use the hardware perf event structure to store the
 810		 * counter number in the 'config' member and the counter
 811		 * set number in the 'config_base' as bit mask.
 812		 * It is later used to enable/disable the counter(s).
 813		 */
 814		hwc->config = ev;
 815		hwc->config_base = cpumf_ctr_ctl[set];
 816		break;
 817	case CPUMF_CTR_SET_MAX:
 818		/* The counter could not be associated to a counter set */
 819		return -EINVAL;
 820	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 821
 822	/* Initialize for using the CPU-measurement counter facility */
 823	if (cpum_cf_alloc(event->cpu))
 824		return -ENOMEM;
 
 
 
 
 
 
 825	event->destroy = hw_perf_event_destroy;
 826
 827	/*
 828	 * Finally, validate version and authorization of the counter set.
 829	 * If the particular CPU counter set is not authorized,
 830	 * return with -ENOENT in order to fall back to other
 831	 * PMUs that might suffice the event request.
 832	 */
 833	if (!(hwc->config_base & cpumf_ctr_info.auth_ctl))
 834		return -ENOENT;
 835	return validate_ctr_version(hwc->config, set);
 836}
 837
 838/* Events CPU_CYCLES and INSTRUCTIONS can be submitted with two different
 839 * attribute::type values:
 840 * - PERF_TYPE_HARDWARE:
 841 * - pmu->type:
 842 * Handle both type of invocations identical. They address the same hardware.
 843 * The result is different when event modifiers exclude_kernel and/or
 844 * exclude_user are also set.
 845 */
 846static int cpumf_pmu_event_type(struct perf_event *event)
 847{
 848	u64 ev = event->attr.config;
 849
 850	if (cpumf_generic_events_basic[PERF_COUNT_HW_CPU_CYCLES] == ev ||
 851	    cpumf_generic_events_basic[PERF_COUNT_HW_INSTRUCTIONS] == ev ||
 852	    cpumf_generic_events_user[PERF_COUNT_HW_CPU_CYCLES] == ev ||
 853	    cpumf_generic_events_user[PERF_COUNT_HW_INSTRUCTIONS] == ev)
 854		return PERF_TYPE_HARDWARE;
 855	return PERF_TYPE_RAW;
 856}
 857
 858static int cpumf_pmu_event_init(struct perf_event *event)
 859{
 860	unsigned int type = event->attr.type;
 861	int err;
 862
 863	if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_RAW)
 864		err = __hw_perf_event_init(event, type);
 865	else if (event->pmu->type == type)
 866		/* Registered as unknown PMU */
 867		err = __hw_perf_event_init(event, cpumf_pmu_event_type(event));
 868	else
 
 869		return -ENOENT;
 
 870
 871	if (unlikely(err) && event->destroy)
 872		event->destroy(event);
 873
 874	return err;
 875}
 876
 877static int hw_perf_event_reset(struct perf_event *event)
 878{
 879	u64 prev, new;
 880	int err;
 881
 882	prev = local64_read(&event->hw.prev_count);
 883	do {
 
 884		err = ecctr(event->hw.config, &new);
 885		if (err) {
 886			if (err != 3)
 887				break;
 888			/* The counter is not (yet) available. This
 889			 * might happen if the counter set to which
 890			 * this counter belongs is in the disabled
 891			 * state.
 892			 */
 893			new = 0;
 894		}
 895	} while (!local64_try_cmpxchg(&event->hw.prev_count, &prev, new));
 896
 897	return err;
 898}
 899
 900static void hw_perf_event_update(struct perf_event *event)
 901{
 902	u64 prev, new, delta;
 903	int err;
 904
 905	prev = local64_read(&event->hw.prev_count);
 906	do {
 
 907		err = ecctr(event->hw.config, &new);
 908		if (err)
 909			return;
 910	} while (!local64_try_cmpxchg(&event->hw.prev_count, &prev, new));
 911
 912	delta = (prev <= new) ? new - prev
 913			      : (-1ULL - prev) + new + 1;	 /* overflow */
 914	local64_add(delta, &event->count);
 
 
 915}
 916
 917static void cpumf_pmu_read(struct perf_event *event)
 918{
 919	if (event->hw.state & PERF_HES_STOPPED)
 920		return;
 921
 922	hw_perf_event_update(event);
 923}
 924
 925static void cpumf_pmu_start(struct perf_event *event, int flags)
 926{
 927	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
 928	struct hw_perf_event *hwc = &event->hw;
 929	int i;
 930
 931	if (!(hwc->state & PERF_HES_STOPPED))
 932		return;
 933
 
 
 
 
 
 
 934	hwc->state = 0;
 935
 936	/* (Re-)enable and activate the counter set */
 937	ctr_set_enable(&cpuhw->state, hwc->config_base);
 938	ctr_set_start(&cpuhw->state, hwc->config_base);
 939
 940	/* The counter set to which this counter belongs can be already active.
 941	 * Because all counters in a set are active, the event->hw.prev_count
 942	 * needs to be synchronized.  At this point, the counter set can be in
 943	 * the inactive or disabled state.
 944	 */
 945	if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) {
 946		cpuhw->usedss = cfdiag_getctr(cpuhw->start,
 947					      sizeof(cpuhw->start),
 948					      hwc->config_base, true);
 949	} else {
 950		hw_perf_event_reset(event);
 951	}
 952
 953	/* Increment refcount for counter sets */
 954	for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i)
 955		if ((hwc->config_base & cpumf_ctr_ctl[i]))
 956			atomic_inc(&cpuhw->ctr_set[i]);
 957}
 958
 959/* Create perf event sample with the counter sets as raw data.	The sample
 960 * is then pushed to the event subsystem and the function checks for
 961 * possible event overflows. If an event overflow occurs, the PMU is
 962 * stopped.
 963 *
 964 * Return non-zero if an event overflow occurred.
 965 */
 966static int cfdiag_push_sample(struct perf_event *event,
 967			      struct cpu_cf_events *cpuhw)
 968{
 969	struct perf_sample_data data;
 970	struct perf_raw_record raw;
 971	struct pt_regs regs;
 972	int overflow;
 973
 974	/* Setup perf sample */
 975	perf_sample_data_init(&data, 0, event->hw.last_period);
 976	memset(&regs, 0, sizeof(regs));
 977	memset(&raw, 0, sizeof(raw));
 978
 979	if (event->attr.sample_type & PERF_SAMPLE_CPU)
 980		data.cpu_entry.cpu = event->cpu;
 981	if (event->attr.sample_type & PERF_SAMPLE_RAW) {
 982		raw.frag.size = cpuhw->usedss;
 983		raw.frag.data = cpuhw->stop;
 984		perf_sample_save_raw_data(&data, event, &raw);
 985	}
 986
 987	overflow = perf_event_overflow(event, &data, &regs);
 988	if (overflow)
 989		event->pmu->stop(event, 0);
 990
 991	perf_event_update_userpage(event);
 992	return overflow;
 993}
 994
 995static void cpumf_pmu_stop(struct perf_event *event, int flags)
 996{
 997	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
 998	struct hw_perf_event *hwc = &event->hw;
 999	int i;
1000
1001	if (!(hwc->state & PERF_HES_STOPPED)) {
1002		/* Decrement reference count for this counter set and if this
1003		 * is the last used counter in the set, clear activation
1004		 * control and set the counter set state to inactive.
1005		 */
1006		for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
1007			if (!(hwc->config_base & cpumf_ctr_ctl[i]))
1008				continue;
1009			if (!atomic_dec_return(&cpuhw->ctr_set[i]))
1010				ctr_set_stop(&cpuhw->state, cpumf_ctr_ctl[i]);
1011		}
1012		hwc->state |= PERF_HES_STOPPED;
1013	}
1014
1015	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1016		if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) {
1017			local64_inc(&event->count);
1018			cpuhw->usedss = cfdiag_getctr(cpuhw->stop,
1019						      sizeof(cpuhw->stop),
1020						      event->hw.config_base,
1021						      false);
1022			if (cfdiag_diffctr(cpuhw, event->hw.config_base))
1023				cfdiag_push_sample(event, cpuhw);
1024		} else {
1025			hw_perf_event_update(event);
1026		}
1027		hwc->state |= PERF_HES_UPTODATE;
1028	}
1029}
1030
1031static int cpumf_pmu_add(struct perf_event *event, int flags)
1032{
1033	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
 
 
 
 
 
 
 
 
 
1034
1035	ctr_set_enable(&cpuhw->state, event->hw.config_base);
1036	event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1037
1038	if (flags & PERF_EF_START)
1039		cpumf_pmu_start(event, PERF_EF_RELOAD);
1040
 
 
1041	return 0;
1042}
1043
1044static void cpumf_pmu_del(struct perf_event *event, int flags)
1045{
1046	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
1047	int i;
1048
1049	cpumf_pmu_stop(event, PERF_EF_UPDATE);
1050
1051	/* Check if any counter in the counter set is still used.  If not used,
1052	 * change the counter set to the disabled state.  This also clears the
1053	 * content of all counters in the set.
1054	 *
1055	 * When a new perf event has been added but not yet started, this can
1056	 * clear enable control and resets all counters in a set.  Therefore,
1057	 * cpumf_pmu_start() always has to re-enable a counter set.
1058	 */
1059	for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i)
1060		if (!atomic_read(&cpuhw->ctr_set[i]))
1061			ctr_set_disable(&cpuhw->state, cpumf_ctr_ctl[i]);
1062}
1063
1064/* Performance monitoring unit for s390x */
1065static struct pmu cpumf_pmu = {
1066	.task_ctx_nr  = perf_sw_context,
1067	.capabilities = PERF_PMU_CAP_NO_INTERRUPT,
1068	.pmu_enable   = cpumf_pmu_enable,
1069	.pmu_disable  = cpumf_pmu_disable,
1070	.event_init   = cpumf_pmu_event_init,
1071	.add	      = cpumf_pmu_add,
1072	.del	      = cpumf_pmu_del,
1073	.start	      = cpumf_pmu_start,
1074	.stop	      = cpumf_pmu_stop,
1075	.read	      = cpumf_pmu_read,
1076};
1077
1078static struct cfset_session {		/* CPUs and counter set bit mask */
1079	struct list_head head;		/* Head of list of active processes */
1080} cfset_session = {
1081	.head = LIST_HEAD_INIT(cfset_session.head)
1082};
1083
1084static refcount_t cfset_opencnt = REFCOUNT_INIT(0);	/* Access count */
1085/*
1086 * Synchronize access to device /dev/hwc. This mutex protects against
1087 * concurrent access to functions cfset_open() and cfset_release().
1088 * Same for CPU hotplug add and remove events triggering
1089 * cpum_cf_online_cpu() and cpum_cf_offline_cpu().
1090 * It also serializes concurrent device ioctl access from multiple
1091 * processes accessing /dev/hwc.
1092 *
1093 * The mutex protects concurrent access to the /dev/hwctr session management
1094 * struct cfset_session and reference counting variable cfset_opencnt.
1095 */
1096static DEFINE_MUTEX(cfset_ctrset_mutex);
1097
1098/*
1099 * CPU hotplug handles only /dev/hwctr device.
1100 * For perf_event_open() the CPU hotplug handling is done on kernel common
1101 * code:
1102 * - CPU add: Nothing is done since a file descriptor can not be created
1103 *   and returned to the user.
1104 * - CPU delete: Handled by common code via pmu_disable(), pmu_stop() and
1105 *   pmu_delete(). The event itself is removed when the file descriptor is
1106 *   closed.
1107 */
1108static int cfset_online_cpu(unsigned int cpu);
1109
1110static int cpum_cf_online_cpu(unsigned int cpu)
1111{
1112	int rc = 0;
1113
1114	/*
1115	 * Ignore notification for perf_event_open().
1116	 * Handle only /dev/hwctr device sessions.
1117	 */
1118	mutex_lock(&cfset_ctrset_mutex);
1119	if (refcount_read(&cfset_opencnt)) {
1120		rc = cpum_cf_alloc_cpu(cpu);
1121		if (!rc)
1122			cfset_online_cpu(cpu);
1123	}
1124	mutex_unlock(&cfset_ctrset_mutex);
1125	return rc;
1126}
1127
1128static int cfset_offline_cpu(unsigned int cpu);
1129
1130static int cpum_cf_offline_cpu(unsigned int cpu)
1131{
1132	/*
1133	 * During task exit processing of grouped perf events triggered by CPU
1134	 * hotplug processing, pmu_disable() is called as part of perf context
1135	 * removal process. Therefore do not trigger event removal now for
1136	 * perf_event_open() created events. Perf common code triggers event
1137	 * destruction when the event file descriptor is closed.
1138	 *
1139	 * Handle only /dev/hwctr device sessions.
1140	 */
1141	mutex_lock(&cfset_ctrset_mutex);
1142	if (refcount_read(&cfset_opencnt)) {
1143		cfset_offline_cpu(cpu);
1144		cpum_cf_free_cpu(cpu);
1145	}
1146	mutex_unlock(&cfset_ctrset_mutex);
1147	return 0;
1148}
1149
1150/* Return true if store counter set multiple instruction is available */
1151static inline int stccm_avail(void)
1152{
1153	return test_facility(142);
1154}
1155
1156/* CPU-measurement alerts for the counter facility */
1157static void cpumf_measurement_alert(struct ext_code ext_code,
1158				    unsigned int alert, unsigned long unused)
1159{
1160	struct cpu_cf_events *cpuhw;
1161
1162	if (!(alert & CPU_MF_INT_CF_MASK))
1163		return;
1164
1165	inc_irq_stat(IRQEXT_CMC);
1166
1167	/*
1168	 * Measurement alerts are shared and might happen when the PMU
1169	 * is not reserved.  Ignore these alerts in this case.
1170	 */
1171	cpuhw = this_cpu_cfhw();
1172	if (!cpuhw)
1173		return;
1174
1175	/* counter authorization change alert */
1176	if (alert & CPU_MF_INT_CF_CACA)
1177		qctri(&cpumf_ctr_info);
1178
1179	/* loss of counter data alert */
1180	if (alert & CPU_MF_INT_CF_LCDA)
1181		pr_err("CPU[%i] Counter data was lost\n", smp_processor_id());
1182
1183	/* loss of MT counter data alert */
1184	if (alert & CPU_MF_INT_CF_MTDA)
1185		pr_warn("CPU[%i] MT counter data was lost\n",
1186			smp_processor_id());
1187}
1188
1189static int cfset_init(void);
1190static int __init cpumf_pmu_init(void)
1191{
1192	int rc;
1193
1194	/* Extract counter measurement facility information */
1195	if (!cpum_cf_avail() || qctri(&cpumf_ctr_info))
1196		return -ENODEV;
1197
1198	/* Determine and store counter set sizes for later reference */
1199	for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc)
1200		cpum_cf_make_setsize(rc);
1201
1202	/*
1203	 * Clear bit 15 of cr0 to unauthorize problem-state to
1204	 * extract measurement counters
1205	 */
1206	system_ctl_clear_bit(0, CR0_CPUMF_EXTRACTION_AUTH_BIT);
1207
1208	/* register handler for measurement-alert interruptions */
1209	rc = register_external_irq(EXT_IRQ_MEASURE_ALERT,
1210				   cpumf_measurement_alert);
1211	if (rc) {
1212		pr_err("Registering for CPU-measurement alerts failed with rc=%i\n", rc);
1213		return rc;
1214	}
1215
1216	/* Setup s390dbf facility */
1217	cf_dbg = debug_register(KMSG_COMPONENT, 2, 1, 128);
1218	if (!cf_dbg) {
1219		pr_err("Registration of s390dbf(cpum_cf) failed\n");
1220		rc = -ENOMEM;
1221		goto out1;
1222	}
1223	debug_register_view(cf_dbg, &debug_sprintf_view);
1224
1225	cpumf_pmu.attr_groups = cpumf_cf_event_group();
1226	rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", -1);
1227	if (rc) {
1228		pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc);
1229		goto out2;
1230	} else if (stccm_avail()) {	/* Setup counter set device */
1231		cfset_init();
1232	}
1233
1234	rc = cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE,
1235			       "perf/s390/cf:online",
1236			       cpum_cf_online_cpu, cpum_cf_offline_cpu);
1237	return rc;
1238
1239out2:
1240	debug_unregister_view(cf_dbg, &debug_sprintf_view);
1241	debug_unregister(cf_dbg);
1242out1:
1243	unregister_external_irq(EXT_IRQ_MEASURE_ALERT, cpumf_measurement_alert);
1244	return rc;
1245}
1246
1247/* Support for the CPU Measurement Facility counter set extraction using
1248 * device /dev/hwctr. This allows user space programs to extract complete
1249 * counter set via normal file operations.
1250 */
1251
1252struct cfset_call_on_cpu_parm {		/* Parm struct for smp_call_on_cpu */
1253	unsigned int sets;		/* Counter set bit mask */
1254	atomic_t cpus_ack;		/* # CPUs successfully executed func */
1255};
1256
1257struct cfset_request {			/* CPUs and counter set bit mask */
1258	unsigned long ctrset;		/* Bit mask of counter set to read */
1259	cpumask_t mask;			/* CPU mask to read from */
1260	struct list_head node;		/* Chain to cfset_session.head */
1261};
1262
1263static void cfset_session_init(void)
1264{
1265	INIT_LIST_HEAD(&cfset_session.head);
1266}
1267
1268/* Remove current request from global bookkeeping. Maintain a counter set bit
1269 * mask on a per CPU basis.
1270 * Done in process context under mutex protection.
1271 */
1272static void cfset_session_del(struct cfset_request *p)
1273{
1274	list_del(&p->node);
1275}
1276
1277/* Add current request to global bookkeeping. Maintain a counter set bit mask
1278 * on a per CPU basis.
1279 * Done in process context under mutex protection.
1280 */
1281static void cfset_session_add(struct cfset_request *p)
1282{
1283	list_add(&p->node, &cfset_session.head);
1284}
1285
1286/* The /dev/hwctr device access uses PMU_F_IN_USE to mark the device access
1287 * path is currently used.
1288 * The cpu_cf_events::dev_state is used to denote counter sets in use by this
1289 * interface. It is always or'ed in. If this interface is not active, its
1290 * value is zero and no additional counter sets will be included.
1291 *
1292 * The cpu_cf_events::state is used by the perf_event_open SVC and remains
1293 * unchanged.
1294 *
1295 * perf_pmu_enable() and perf_pmu_enable() and its call backs
1296 * cpumf_pmu_enable() and  cpumf_pmu_disable() are called by the
1297 * performance measurement subsystem to enable per process
1298 * CPU Measurement counter facility.
1299 * The XXX_enable() and XXX_disable functions are used to turn off
1300 * x86 performance monitoring interrupt (PMI) during scheduling.
1301 * s390 uses these calls to temporarily stop and resume the active CPU
1302 * counters sets during scheduling.
1303 *
1304 * We do allow concurrent access of perf_event_open() SVC and /dev/hwctr
1305 * device access.  The perf_event_open() SVC interface makes a lot of effort
1306 * to only run the counters while the calling process is actively scheduled
1307 * to run.
1308 * When /dev/hwctr interface is also used at the same time, the counter sets
1309 * will keep running, even when the process is scheduled off a CPU.
1310 * However this is not a problem and does not lead to wrong counter values
1311 * for the perf_event_open() SVC. The current counter value will be recorded
1312 * during schedule-in. At schedule-out time the current counter value is
1313 * extracted again and the delta is calculated and added to the event.
1314 */
1315/* Stop all counter sets via ioctl interface */
1316static void cfset_ioctl_off(void *parm)
1317{
1318	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
1319	struct cfset_call_on_cpu_parm *p = parm;
1320	int rc;
1321
1322	/* Check if any counter set used by /dev/hwctr */
1323	for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc)
1324		if ((p->sets & cpumf_ctr_ctl[rc])) {
1325			if (!atomic_dec_return(&cpuhw->ctr_set[rc])) {
1326				ctr_set_disable(&cpuhw->dev_state,
1327						cpumf_ctr_ctl[rc]);
1328				ctr_set_stop(&cpuhw->dev_state,
1329					     cpumf_ctr_ctl[rc]);
1330			}
1331		}
1332	/* Keep perf_event_open counter sets */
1333	rc = lcctl(cpuhw->dev_state | cpuhw->state);
1334	if (rc)
1335		pr_err("Counter set stop %#llx of /dev/%s failed rc=%i\n",
1336		       cpuhw->state, S390_HWCTR_DEVICE, rc);
1337	if (!cpuhw->dev_state)
1338		cpuhw->flags &= ~PMU_F_IN_USE;
1339}
1340
1341/* Start counter sets on particular CPU */
1342static void cfset_ioctl_on(void *parm)
1343{
1344	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
1345	struct cfset_call_on_cpu_parm *p = parm;
1346	int rc;
1347
1348	cpuhw->flags |= PMU_F_IN_USE;
1349	ctr_set_enable(&cpuhw->dev_state, p->sets);
1350	ctr_set_start(&cpuhw->dev_state, p->sets);
1351	for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc)
1352		if ((p->sets & cpumf_ctr_ctl[rc]))
1353			atomic_inc(&cpuhw->ctr_set[rc]);
1354	rc = lcctl(cpuhw->dev_state | cpuhw->state);	/* Start counter sets */
1355	if (!rc)
1356		atomic_inc(&p->cpus_ack);
1357	else
1358		pr_err("Counter set start %#llx of /dev/%s failed rc=%i\n",
1359		       cpuhw->dev_state | cpuhw->state, S390_HWCTR_DEVICE, rc);
1360}
1361
1362static void cfset_release_cpu(void *p)
1363{
1364	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
1365	int rc;
1366
1367	cpuhw->dev_state = 0;
1368	rc = lcctl(cpuhw->state);	/* Keep perf_event_open counter sets */
1369	if (rc)
1370		pr_err("Counter set release %#llx of /dev/%s failed rc=%i\n",
1371		       cpuhw->state, S390_HWCTR_DEVICE, rc);
1372}
1373
1374/* This modifies the process CPU mask to adopt it to the currently online
1375 * CPUs. Offline CPUs can not be addresses. This call terminates the access
1376 * and is usually followed by close() or a new iotcl(..., START, ...) which
1377 * creates a new request structure.
1378 */
1379static void cfset_all_stop(struct cfset_request *req)
1380{
1381	struct cfset_call_on_cpu_parm p = {
1382		.sets = req->ctrset,
1383	};
1384
1385	cpumask_and(&req->mask, &req->mask, cpu_online_mask);
1386	on_each_cpu_mask(&req->mask, cfset_ioctl_off, &p, 1);
1387}
1388
1389/* Release function is also called when application gets terminated without
1390 * doing a proper ioctl(..., S390_HWCTR_STOP, ...) command.
1391 */
1392static int cfset_release(struct inode *inode, struct file *file)
1393{
1394	mutex_lock(&cfset_ctrset_mutex);
1395	/* Open followed by close/exit has no private_data */
1396	if (file->private_data) {
1397		cfset_all_stop(file->private_data);
1398		cfset_session_del(file->private_data);
1399		kfree(file->private_data);
1400		file->private_data = NULL;
1401	}
1402	if (refcount_dec_and_test(&cfset_opencnt)) {	/* Last close */
1403		on_each_cpu(cfset_release_cpu, NULL, 1);
1404		cpum_cf_free(-1);
1405	}
1406	mutex_unlock(&cfset_ctrset_mutex);
1407	return 0;
1408}
1409
1410/*
1411 * Open via /dev/hwctr device. Allocate all per CPU resources on the first
1412 * open of the device. The last close releases all per CPU resources.
1413 * Parallel perf_event_open system calls also use per CPU resources.
1414 * These invocations are handled via reference counting on the per CPU data
1415 * structures.
1416 */
1417static int cfset_open(struct inode *inode, struct file *file)
1418{
1419	int rc = 0;
1420
1421	if (!perfmon_capable())
1422		return -EPERM;
1423	file->private_data = NULL;
1424
1425	mutex_lock(&cfset_ctrset_mutex);
1426	if (!refcount_inc_not_zero(&cfset_opencnt)) {	/* First open */
1427		rc = cpum_cf_alloc(-1);
1428		if (!rc) {
1429			cfset_session_init();
1430			refcount_set(&cfset_opencnt, 1);
1431		}
1432	}
1433	mutex_unlock(&cfset_ctrset_mutex);
1434
1435	/* nonseekable_open() never fails */
1436	return rc ?: nonseekable_open(inode, file);
1437}
1438
1439static int cfset_all_start(struct cfset_request *req)
1440{
1441	struct cfset_call_on_cpu_parm p = {
1442		.sets = req->ctrset,
1443		.cpus_ack = ATOMIC_INIT(0),
1444	};
1445	cpumask_var_t mask;
1446	int rc = 0;
1447
1448	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
1449		return -ENOMEM;
1450	cpumask_and(mask, &req->mask, cpu_online_mask);
1451	on_each_cpu_mask(mask, cfset_ioctl_on, &p, 1);
1452	if (atomic_read(&p.cpus_ack) != cpumask_weight(mask)) {
1453		on_each_cpu_mask(mask, cfset_ioctl_off, &p, 1);
1454		rc = -EIO;
1455	}
1456	free_cpumask_var(mask);
1457	return rc;
1458}
1459
1460/* Return the maximum required space for all possible CPUs in case one
1461 * CPU will be onlined during the START, READ, STOP cycles.
1462 * To find out the size of the counter sets, any one CPU will do. They
1463 * all have the same counter sets.
1464 */
1465static size_t cfset_needspace(unsigned int sets)
1466{
1467	size_t bytes = 0;
1468	int i;
1469
1470	for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
1471		if (!(sets & cpumf_ctr_ctl[i]))
1472			continue;
1473		bytes += cpum_cf_read_setsize(i) * sizeof(u64) +
1474			 sizeof(((struct s390_ctrset_setdata *)0)->set) +
1475			 sizeof(((struct s390_ctrset_setdata *)0)->no_cnts);
1476	}
1477	bytes = sizeof(((struct s390_ctrset_read *)0)->no_cpus) + nr_cpu_ids *
1478		(bytes + sizeof(((struct s390_ctrset_cpudata *)0)->cpu_nr) +
1479		     sizeof(((struct s390_ctrset_cpudata *)0)->no_sets));
1480	return bytes;
1481}
1482
1483static int cfset_all_copy(unsigned long arg, cpumask_t *mask)
1484{
1485	struct s390_ctrset_read __user *ctrset_read;
1486	unsigned int cpu, cpus, rc = 0;
1487	void __user *uptr;
1488
1489	ctrset_read = (struct s390_ctrset_read __user *)arg;
1490	uptr = ctrset_read->data;
1491	for_each_cpu(cpu, mask) {
1492		struct cpu_cf_events *cpuhw = get_cpu_cfhw(cpu);
1493		struct s390_ctrset_cpudata __user *ctrset_cpudata;
1494
1495		ctrset_cpudata = uptr;
1496		rc  = put_user(cpu, &ctrset_cpudata->cpu_nr);
1497		rc |= put_user(cpuhw->sets, &ctrset_cpudata->no_sets);
1498		rc |= copy_to_user(ctrset_cpudata->data, cpuhw->data,
1499				   cpuhw->used);
1500		if (rc) {
1501			rc = -EFAULT;
1502			goto out;
1503		}
1504		uptr += sizeof(struct s390_ctrset_cpudata) + cpuhw->used;
1505		cond_resched();
1506	}
1507	cpus = cpumask_weight(mask);
1508	if (put_user(cpus, &ctrset_read->no_cpus))
1509		rc = -EFAULT;
1510out:
1511	return rc;
1512}
1513
1514static size_t cfset_cpuset_read(struct s390_ctrset_setdata *p, int ctrset,
1515				int ctrset_size, size_t room)
1516{
1517	size_t need = 0;
1518	int rc = -1;
1519
1520	need = sizeof(*p) + sizeof(u64) * ctrset_size;
1521	if (need <= room) {
1522		p->set = cpumf_ctr_ctl[ctrset];
1523		p->no_cnts = ctrset_size;
1524		rc = ctr_stcctm(ctrset, ctrset_size, (u64 *)p->cv);
1525		if (rc == 3)		/* Nothing stored */
1526			need = 0;
1527	}
1528	return need;
1529}
1530
1531/* Read all counter sets. */
1532static void cfset_cpu_read(void *parm)
1533{
1534	struct cpu_cf_events *cpuhw = this_cpu_cfhw();
1535	struct cfset_call_on_cpu_parm *p = parm;
1536	int set, set_size;
1537	size_t space;
1538
1539	/* No data saved yet */
1540	cpuhw->used = 0;
1541	cpuhw->sets = 0;
1542	memset(cpuhw->data, 0, sizeof(cpuhw->data));
1543
1544	/* Scan the counter sets */
1545	for (set = CPUMF_CTR_SET_BASIC; set < CPUMF_CTR_SET_MAX; ++set) {
1546		struct s390_ctrset_setdata *sp = (void *)cpuhw->data +
1547						 cpuhw->used;
1548
1549		if (!(p->sets & cpumf_ctr_ctl[set]))
1550			continue;	/* Counter set not in list */
1551		set_size = cpum_cf_read_setsize(set);
1552		space = sizeof(cpuhw->data) - cpuhw->used;
1553		space = cfset_cpuset_read(sp, set, set_size, space);
1554		if (space) {
1555			cpuhw->used += space;
1556			cpuhw->sets += 1;
1557		}
1558	}
1559}
1560
1561static int cfset_all_read(unsigned long arg, struct cfset_request *req)
1562{
1563	struct cfset_call_on_cpu_parm p;
1564	cpumask_var_t mask;
1565	int rc;
1566
1567	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
1568		return -ENOMEM;
1569
1570	p.sets = req->ctrset;
1571	cpumask_and(mask, &req->mask, cpu_online_mask);
1572	on_each_cpu_mask(mask, cfset_cpu_read, &p, 1);
1573	rc = cfset_all_copy(arg, mask);
1574	free_cpumask_var(mask);
1575	return rc;
1576}
1577
1578static long cfset_ioctl_read(unsigned long arg, struct cfset_request *req)
1579{
1580	int ret = -ENODATA;
1581
1582	if (req && req->ctrset)
1583		ret = cfset_all_read(arg, req);
1584	return ret;
1585}
1586
1587static long cfset_ioctl_stop(struct file *file)
1588{
1589	struct cfset_request *req = file->private_data;
1590	int ret = -ENXIO;
1591
1592	if (req) {
1593		cfset_all_stop(req);
1594		cfset_session_del(req);
1595		kfree(req);
1596		file->private_data = NULL;
1597		ret = 0;
1598	}
1599	return ret;
1600}
1601
1602static long cfset_ioctl_start(unsigned long arg, struct file *file)
1603{
1604	struct s390_ctrset_start __user *ustart;
1605	struct s390_ctrset_start start;
1606	struct cfset_request *preq;
1607	void __user *umask;
1608	unsigned int len;
1609	int ret = 0;
1610	size_t need;
1611
1612	if (file->private_data)
1613		return -EBUSY;
1614	ustart = (struct s390_ctrset_start __user *)arg;
1615	if (copy_from_user(&start, ustart, sizeof(start)))
1616		return -EFAULT;
1617	if (start.version != S390_HWCTR_START_VERSION)
1618		return -EINVAL;
1619	if (start.counter_sets & ~(cpumf_ctr_ctl[CPUMF_CTR_SET_BASIC] |
1620				   cpumf_ctr_ctl[CPUMF_CTR_SET_USER] |
1621				   cpumf_ctr_ctl[CPUMF_CTR_SET_CRYPTO] |
1622				   cpumf_ctr_ctl[CPUMF_CTR_SET_EXT] |
1623				   cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG]))
1624		return -EINVAL;		/* Invalid counter set */
1625	if (!start.counter_sets)
1626		return -EINVAL;		/* No counter set at all? */
1627
1628	preq = kzalloc(sizeof(*preq), GFP_KERNEL);
1629	if (!preq)
1630		return -ENOMEM;
1631	cpumask_clear(&preq->mask);
1632	len = min_t(u64, start.cpumask_len, cpumask_size());
1633	umask = (void __user *)start.cpumask;
1634	if (copy_from_user(&preq->mask, umask, len)) {
1635		kfree(preq);
1636		return -EFAULT;
1637	}
1638	if (cpumask_empty(&preq->mask)) {
1639		kfree(preq);
1640		return -EINVAL;
1641	}
1642	need = cfset_needspace(start.counter_sets);
1643	if (put_user(need, &ustart->data_bytes)) {
1644		kfree(preq);
1645		return -EFAULT;
1646	}
1647	preq->ctrset = start.counter_sets;
1648	ret = cfset_all_start(preq);
1649	if (!ret) {
1650		cfset_session_add(preq);
1651		file->private_data = preq;
1652	} else {
1653		kfree(preq);
1654	}
1655	return ret;
1656}
1657
1658/* Entry point to the /dev/hwctr device interface.
1659 * The ioctl system call supports three subcommands:
1660 * S390_HWCTR_START: Start the specified counter sets on a CPU list. The
1661 *    counter set keeps running until explicitly stopped. Returns the number
1662 *    of bytes needed to store the counter values. If another S390_HWCTR_START
1663 *    ioctl subcommand is called without a previous S390_HWCTR_STOP stop
1664 *    command on the same file descriptor, -EBUSY is returned.
1665 * S390_HWCTR_READ: Read the counter set values from specified CPU list given
1666 *    with the S390_HWCTR_START command.
1667 * S390_HWCTR_STOP: Stops the counter sets on the CPU list given with the
1668 *    previous S390_HWCTR_START subcommand.
1669 */
1670static long cfset_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1671{
1672	int ret;
1673
1674	cpus_read_lock();
1675	mutex_lock(&cfset_ctrset_mutex);
1676	switch (cmd) {
1677	case S390_HWCTR_START:
1678		ret = cfset_ioctl_start(arg, file);
1679		break;
1680	case S390_HWCTR_STOP:
1681		ret = cfset_ioctl_stop(file);
1682		break;
1683	case S390_HWCTR_READ:
1684		ret = cfset_ioctl_read(arg, file->private_data);
1685		break;
1686	default:
1687		ret = -ENOTTY;
1688		break;
1689	}
1690	mutex_unlock(&cfset_ctrset_mutex);
1691	cpus_read_unlock();
1692	return ret;
1693}
1694
1695static const struct file_operations cfset_fops = {
1696	.owner = THIS_MODULE,
1697	.open = cfset_open,
1698	.release = cfset_release,
1699	.unlocked_ioctl	= cfset_ioctl,
1700	.compat_ioctl = cfset_ioctl,
1701};
1702
1703static struct miscdevice cfset_dev = {
1704	.name	= S390_HWCTR_DEVICE,
1705	.minor	= MISC_DYNAMIC_MINOR,
1706	.fops	= &cfset_fops,
1707	.mode	= 0666,
1708};
1709
1710/* Hotplug add of a CPU. Scan through all active processes and add
1711 * that CPU to the list of CPUs supplied with ioctl(..., START, ...).
1712 */
1713static int cfset_online_cpu(unsigned int cpu)
1714{
1715	struct cfset_call_on_cpu_parm p;
1716	struct cfset_request *rp;
1717
1718	if (!list_empty(&cfset_session.head)) {
1719		list_for_each_entry(rp, &cfset_session.head, node) {
1720			p.sets = rp->ctrset;
1721			cfset_ioctl_on(&p);
1722			cpumask_set_cpu(cpu, &rp->mask);
1723		}
1724	}
1725	return 0;
1726}
1727
1728/* Hotplug remove of a CPU. Scan through all active processes and clear
1729 * that CPU from the list of CPUs supplied with ioctl(..., START, ...).
1730 * Adjust reference counts.
1731 */
1732static int cfset_offline_cpu(unsigned int cpu)
1733{
1734	struct cfset_call_on_cpu_parm p;
1735	struct cfset_request *rp;
1736
1737	if (!list_empty(&cfset_session.head)) {
1738		list_for_each_entry(rp, &cfset_session.head, node) {
1739			p.sets = rp->ctrset;
1740			cfset_ioctl_off(&p);
1741			cpumask_clear_cpu(cpu, &rp->mask);
1742		}
1743	}
1744	return 0;
1745}
1746
1747static void cfdiag_read(struct perf_event *event)
1748{
1749}
1750
1751static int get_authctrsets(void)
1752{
1753	unsigned long auth = 0;
1754	enum cpumf_ctr_set i;
1755
1756	for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
1757		if (cpumf_ctr_info.auth_ctl & cpumf_ctr_ctl[i])
1758			auth |= cpumf_ctr_ctl[i];
1759	}
1760	return auth;
1761}
1762
1763/* Setup the event. Test for authorized counter sets and only include counter
1764 * sets which are authorized at the time of the setup. Including unauthorized
1765 * counter sets result in specification exception (and panic).
1766 */
1767static int cfdiag_event_init2(struct perf_event *event)
1768{
1769	struct perf_event_attr *attr = &event->attr;
1770	int err = 0;
1771
1772	/* Set sample_period to indicate sampling */
1773	event->hw.config = attr->config;
1774	event->hw.sample_period = attr->sample_period;
1775	local64_set(&event->hw.period_left, event->hw.sample_period);
1776	local64_set(&event->count, 0);
1777	event->hw.last_period = event->hw.sample_period;
1778
1779	/* Add all authorized counter sets to config_base. The
1780	 * the hardware init function is either called per-cpu or just once
1781	 * for all CPUS (event->cpu == -1).  This depends on the whether
1782	 * counting is started for all CPUs or on a per workload base where
1783	 * the perf event moves from one CPU to another CPU.
1784	 * Checking the authorization on any CPU is fine as the hardware
1785	 * applies the same authorization settings to all CPUs.
1786	 */
1787	event->hw.config_base = get_authctrsets();
1788
1789	/* No authorized counter sets, nothing to count/sample */
1790	if (!event->hw.config_base)
1791		err = -EINVAL;
1792
1793	return err;
1794}
1795
1796static int cfdiag_event_init(struct perf_event *event)
1797{
1798	struct perf_event_attr *attr = &event->attr;
1799	int err = -ENOENT;
1800
1801	if (event->attr.config != PERF_EVENT_CPUM_CF_DIAG ||
1802	    event->attr.type != event->pmu->type)
1803		goto out;
1804
1805	/* Raw events are used to access counters directly,
1806	 * hence do not permit excludes.
1807	 * This event is useless without PERF_SAMPLE_RAW to return counter set
1808	 * values as raw data.
1809	 */
1810	if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv ||
1811	    !(attr->sample_type & (PERF_SAMPLE_CPU | PERF_SAMPLE_RAW))) {
1812		err = -EOPNOTSUPP;
1813		goto out;
1814	}
1815
1816	/* Initialize for using the CPU-measurement counter facility */
1817	if (cpum_cf_alloc(event->cpu))
1818		return -ENOMEM;
1819	event->destroy = hw_perf_event_destroy;
1820
1821	err = cfdiag_event_init2(event);
1822	if (unlikely(err))
1823		event->destroy(event);
1824out:
1825	return err;
1826}
1827
1828/* Create cf_diag/events/CF_DIAG event sysfs file. This counter is used
1829 * to collect the complete counter sets for a scheduled process. Target
1830 * are complete counter sets attached as raw data to the artificial event.
1831 * This results in complete counter sets available when a process is
1832 * scheduled. Contains the delta of every counter while the process was
1833 * running.
1834 */
1835CPUMF_EVENT_ATTR(CF_DIAG, CF_DIAG, PERF_EVENT_CPUM_CF_DIAG);
1836
1837static struct attribute *cfdiag_events_attr[] = {
1838	CPUMF_EVENT_PTR(CF_DIAG, CF_DIAG),
1839	NULL,
1840};
1841
1842PMU_FORMAT_ATTR(event, "config:0-63");
1843
1844static struct attribute *cfdiag_format_attr[] = {
1845	&format_attr_event.attr,
1846	NULL,
1847};
1848
1849static struct attribute_group cfdiag_events_group = {
1850	.name = "events",
1851	.attrs = cfdiag_events_attr,
1852};
1853static struct attribute_group cfdiag_format_group = {
1854	.name = "format",
1855	.attrs = cfdiag_format_attr,
1856};
1857static const struct attribute_group *cfdiag_attr_groups[] = {
1858	&cfdiag_events_group,
1859	&cfdiag_format_group,
1860	NULL,
1861};
1862
1863/* Performance monitoring unit for event CF_DIAG. Since this event
1864 * is also started and stopped via the perf_event_open() system call, use
1865 * the same event enable/disable call back functions. They do not
1866 * have a pointer to the perf_event structure as first parameter.
1867 *
1868 * The functions XXX_add, XXX_del, XXX_start and XXX_stop are also common.
1869 * Reuse them and distinguish the event (always first parameter) via
1870 * 'config' member.
1871 */
1872static struct pmu cf_diag = {
1873	.task_ctx_nr  = perf_sw_context,
1874	.event_init   = cfdiag_event_init,
1875	.pmu_enable   = cpumf_pmu_enable,
1876	.pmu_disable  = cpumf_pmu_disable,
 
1877	.add	      = cpumf_pmu_add,
1878	.del	      = cpumf_pmu_del,
1879	.start	      = cpumf_pmu_start,
1880	.stop	      = cpumf_pmu_stop,
1881	.read	      = cfdiag_read,
1882
1883	.attr_groups  = cfdiag_attr_groups
 
1884};
1885
1886/* Calculate memory needed to store all counter sets together with header and
1887 * trailer data. This is independent of the counter set authorization which
1888 * can vary depending on the configuration.
1889 */
1890static size_t cfdiag_maxsize(struct cpumf_ctr_info *info)
1891{
1892	size_t max_size = sizeof(struct cf_trailer_entry);
1893	enum cpumf_ctr_set i;
1894
1895	for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) {
1896		size_t size = cpum_cf_read_setsize(i);
1897
1898		if (size)
1899			max_size += size * sizeof(u64) +
1900				    sizeof(struct cf_ctrset_entry);
1901	}
1902	return max_size;
1903}
1904
1905/* Get the CPU speed, try sampling facility first and CPU attributes second. */
1906static void cfdiag_get_cpu_speed(void)
1907{
1908	unsigned long mhz;
1909
1910	if (cpum_sf_avail()) {			/* Sampling facility first */
1911		struct hws_qsi_info_block si;
1912
1913		memset(&si, 0, sizeof(si));
1914		if (!qsi(&si)) {
1915			cfdiag_cpu_speed = si.cpu_speed;
1916			return;
1917		}
1918	}
1919
1920	/* Fallback: CPU speed extract static part. Used in case
1921	 * CPU Measurement Sampling Facility is turned off.
1922	 */
1923	mhz = __ecag(ECAG_CPU_ATTRIBUTE, 0);
1924	if (mhz != -1UL)
1925		cfdiag_cpu_speed = mhz & 0xffffffff;
1926}
1927
1928static int cfset_init(void)
1929{
1930	size_t need;
1931	int rc;
1932
1933	cfdiag_get_cpu_speed();
1934	/* Make sure the counter set data fits into predefined buffer. */
1935	need = cfdiag_maxsize(&cpumf_ctr_info);
1936	if (need > sizeof(((struct cpu_cf_events *)0)->start)) {
1937		pr_err("Insufficient memory for PMU(cpum_cf_diag) need=%zu\n",
1938		       need);
1939		return -ENOMEM;
1940	}
1941
1942	rc = misc_register(&cfset_dev);
 
 
1943	if (rc) {
1944		pr_err("Registration of /dev/%s failed rc=%i\n",
1945		       cfset_dev.name, rc);
1946		goto out;
1947	}
1948
1949	rc = perf_pmu_register(&cf_diag, "cpum_cf_diag", -1);
 
1950	if (rc) {
1951		misc_deregister(&cfset_dev);
1952		pr_err("Registration of PMU(cpum_cf_diag) failed with rc=%i\n",
1953		       rc);
 
1954	}
1955out:
1956	return rc;
 
1957}
1958
1959device_initcall(cpumf_pmu_init);