1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright 2019 Arm Limited
  4 * Author: Andrew Murray <Andrew.Murray@arm.com>
  5 */
  6#include <linux/kvm_host.h>
  7#include <linux/perf_event.h>
  8#include <linux/perf/arm_pmu.h>
  9#include <linux/perf/arm_pmuv3.h>
 10
 11static DEFINE_PER_CPU(struct kvm_pmu_events, kvm_pmu_events);
 12
 13/*
 14 * Given the perf event attributes and system type, determine
 15 * if we are going to need to switch counters at guest entry/exit.
 16 */
 17static bool kvm_pmu_switch_needed(struct perf_event_attr *attr)
 18{
 19	/**
 20	 * With VHE the guest kernel runs at EL1 and the host at EL2,
 21	 * where user (EL0) is excluded then we have no reason to switch
 22	 * counters.
 23	 */
 24	if (has_vhe() && attr->exclude_user)
 25		return false;
 26
 27	/* Only switch if attributes are different */
 28	return (attr->exclude_host != attr->exclude_guest);
 29}
 30
 31struct kvm_pmu_events *kvm_get_pmu_events(void)
 32{
 33	return this_cpu_ptr(&kvm_pmu_events);
 34}
 35
 36/*
 37 * Add events to track that we may want to switch at guest entry/exit
 38 * time.
 39 */
 40void kvm_set_pmu_events(u64 set, struct perf_event_attr *attr)
 41{
 42	struct kvm_pmu_events *pmu = kvm_get_pmu_events();
 43
 44	if (!kvm_arm_support_pmu_v3() || !kvm_pmu_switch_needed(attr))
 45		return;
 46
 47	if (!attr->exclude_host)
 48		pmu->events_host |= set;
 49	if (!attr->exclude_guest)
 50		pmu->events_guest |= set;
 51}
 52
 53/*
 54 * Stop tracking events
 55 */
 56void kvm_clr_pmu_events(u64 clr)
 57{
 58	struct kvm_pmu_events *pmu = kvm_get_pmu_events();
 59
 60	if (!kvm_arm_support_pmu_v3())
 61		return;
 62
 63	pmu->events_host &= ~clr;
 64	pmu->events_guest &= ~clr;
 65}
 66
 67/*
 68 * Read a value direct from PMEVTYPER<idx> where idx is 0-30
 69 * or PMxCFILTR_EL0 where idx is 31-32.
 70 */
 71static u64 kvm_vcpu_pmu_read_evtype_direct(int idx)
 72{
 73	if (idx == ARMV8_PMU_CYCLE_IDX)
 74		return read_pmccfiltr();
 75	else if (idx == ARMV8_PMU_INSTR_IDX)
 76		return read_pmicfiltr();
 77
 78	return read_pmevtypern(idx);
 79}
 80
 81/*
 82 * Write a value direct to PMEVTYPER<idx> where idx is 0-30
 83 * or PMxCFILTR_EL0 where idx is 31-32.
 84 */
 85static void kvm_vcpu_pmu_write_evtype_direct(int idx, u32 val)
 86{
 87	if (idx == ARMV8_PMU_CYCLE_IDX)
 88		write_pmccfiltr(val);
 89	else if (idx == ARMV8_PMU_INSTR_IDX)
 90		write_pmicfiltr(val);
 91	else
 92		write_pmevtypern(idx, val);
 93}
 94
 95/*
 96 * Modify ARMv8 PMU events to include EL0 counting
 97 */
 98static void kvm_vcpu_pmu_enable_el0(unsigned long events)
 99{
100	u64 typer;
101	u32 counter;
102
103	for_each_set_bit(counter, &events, ARMPMU_MAX_HWEVENTS) {
104		typer = kvm_vcpu_pmu_read_evtype_direct(counter);
105		typer &= ~ARMV8_PMU_EXCLUDE_EL0;
106		kvm_vcpu_pmu_write_evtype_direct(counter, typer);
107	}
108}
109
110/*
111 * Modify ARMv8 PMU events to exclude EL0 counting
112 */
113static void kvm_vcpu_pmu_disable_el0(unsigned long events)
114{
115	u64 typer;
116	u32 counter;
117
118	for_each_set_bit(counter, &events, ARMPMU_MAX_HWEVENTS) {
119		typer = kvm_vcpu_pmu_read_evtype_direct(counter);
120		typer |= ARMV8_PMU_EXCLUDE_EL0;
121		kvm_vcpu_pmu_write_evtype_direct(counter, typer);
122	}
123}
124
125/*
126 * On VHE ensure that only guest events have EL0 counting enabled.
127 * This is called from both vcpu_{load,put} and the sysreg handling.
128 * Since the latter is preemptible, special care must be taken to
129 * disable preemption.
130 */
131void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu)
132{
133	struct kvm_pmu_events *pmu;
134	u64 events_guest, events_host;
135
136	if (!kvm_arm_support_pmu_v3() || !has_vhe())
137		return;
138
139	preempt_disable();
140	pmu = kvm_get_pmu_events();
141	events_guest = pmu->events_guest;
142	events_host = pmu->events_host;
143
144	kvm_vcpu_pmu_enable_el0(events_guest);
145	kvm_vcpu_pmu_disable_el0(events_host);
146	preempt_enable();
147}
148
149/*
150 * On VHE ensure that only host events have EL0 counting enabled
151 */
152void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu)
153{
154	struct kvm_pmu_events *pmu;
155	u64 events_guest, events_host;
156
157	if (!kvm_arm_support_pmu_v3() || !has_vhe())
158		return;
159
160	pmu = kvm_get_pmu_events();
161	events_guest = pmu->events_guest;
162	events_host = pmu->events_host;
163
164	kvm_vcpu_pmu_enable_el0(events_host);
165	kvm_vcpu_pmu_disable_el0(events_guest);
166}
167
168/*
169 * With VHE, keep track of the PMUSERENR_EL0 value for the host EL0 on the pCPU
170 * where PMUSERENR_EL0 for the guest is loaded, since PMUSERENR_EL0 is switched
171 * to the value for the guest on vcpu_load().  The value for the host EL0
172 * will be restored on vcpu_put(), before returning to userspace.
173 * This isn't necessary for nVHE, as the register is context switched for
174 * every guest enter/exit.
175 *
176 * Return true if KVM takes care of the register. Otherwise return false.
177 */
178bool kvm_set_pmuserenr(u64 val)
179{
180	struct kvm_cpu_context *hctxt;
181	struct kvm_vcpu *vcpu;
182
183	if (!kvm_arm_support_pmu_v3() || !has_vhe())
184		return false;
185
186	vcpu = kvm_get_running_vcpu();
187	if (!vcpu || !vcpu_get_flag(vcpu, PMUSERENR_ON_CPU))
188		return false;
189
190	hctxt = host_data_ptr(host_ctxt);
191	ctxt_sys_reg(hctxt, PMUSERENR_EL0) = val;
192	return true;
193}
194
195/*
196 * If we interrupted the guest to update the host PMU context, make
197 * sure we re-apply the guest EL0 state.
198 */
199void kvm_vcpu_pmu_resync_el0(void)
200{
201	struct kvm_vcpu *vcpu;
202
203	if (!has_vhe() || !in_interrupt())
204		return;
205
206	vcpu = kvm_get_running_vcpu();
207	if (!vcpu)
208		return;
209
210	kvm_make_request(KVM_REQ_RESYNC_PMU_EL0, vcpu);
211}