1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
  4 * Author: Marc Zyngier <marc.zyngier@arm.com>
  5 */
  6
  7#include <linux/interrupt.h>
  8#include <linux/irq.h>
  9#include <linux/irqdomain.h>
 10#include <linux/msi.h>
 11#include <linux/sched.h>
 12
 13#include <linux/irqchip/arm-gic-v4.h>
 14
 15/*
 16 * WARNING: The blurb below assumes that you understand the
 17 * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
 18 * translated into GICv4 commands. So it effectively targets at most
 19 * two individuals. You know who you are.
 20 *
 21 * The core GICv4 code is designed to *avoid* exposing too much of the
 22 * core GIC code (that would in turn leak into the hypervisor code),
 23 * and instead provide a hypervisor agnostic interface to the HW (of
 24 * course, the astute reader will quickly realize that hypervisor
 25 * agnostic actually means KVM-specific - what were you thinking?).
 26 *
 27 * In order to achieve a modicum of isolation, we try to hide most of
 28 * the GICv4 "stuff" behind normal irqchip operations:
 29 *
 30 * - Any guest-visible VLPI is backed by a Linux interrupt (and a
 31 *   physical LPI which gets unmapped when the guest maps the
 32 *   VLPI). This allows the same DevID/EventID pair to be either
 33 *   mapped to the LPI (host) or the VLPI (guest). Note that this is
 34 *   exclusive, and you cannot have both.
 35 *
 36 * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
 37 *
 38 * - Guest INT/CLEAR commands are implemented through
 39 *   irq_set_irqchip_state().
 40 *
 41 * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
 42 *   issuing an INV after changing a priority) gets shoved into the
 43 *   irq_set_vcpu_affinity() method. While this is quite horrible
 44 *   (let's face it, this is the irqchip version of an ioctl), it
 45 *   confines the crap to a single location. And map/unmap really is
 46 *   about setting the affinity of a VLPI to a vcpu, so only INV is
 47 *   majorly out of place. So there.
 48 *
 49 * A number of commands are simply not provided by this interface, as
 50 * they do not make direct sense. For example, MAPD is purely local to
 51 * the virtual ITS (because it references a virtual device, and the
 52 * physical ITS is still very much in charge of the physical
 53 * device). Same goes for things like MAPC (the physical ITS deals
 54 * with the actual vPE affinity, and not the braindead concept of
 55 * collection). SYNC is not provided either, as each and every command
 56 * is followed by a VSYNC. This could be relaxed in the future, should
 57 * this be seen as a bottleneck (yes, this means *never*).
 58 *
 59 * But handling VLPIs is only one side of the job of the GICv4
 60 * code. The other (darker) side is to take care of the doorbell
 61 * interrupts which are delivered when a VLPI targeting a non-running
 62 * vcpu is being made pending.
 63 *
 64 * The choice made here is that each vcpu (VPE in old northern GICv4
 65 * dialect) gets a single doorbell LPI, no matter how many interrupts
 66 * are targeting it. This has a nice property, which is that the
 67 * interrupt becomes a handle for the VPE, and that the hypervisor
 68 * code can manipulate it through the normal interrupt API:
 69 *
 70 * - VMs (or rather the VM abstraction that matters to the GIC)
 71 *   contain an irq domain where each interrupt maps to a VPE. In
 72 *   turn, this domain sits on top of the normal LPI allocator, and a
 73 *   specially crafted irq_chip implementation.
 74 *
 75 * - mask/unmask do what is expected on the doorbell interrupt.
 76 *
 77 * - irq_set_affinity is used to move a VPE from one redistributor to
 78 *   another.
 79 *
 80 * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
 81 *   creating a new sub-API, namely scheduling/descheduling a VPE
 82 *   (which involves programming GICR_V{PROP,PEND}BASER) and
 83 *   performing INVALL operations.
 84 */
 85
 86static struct irq_domain *gic_domain;
 87static const struct irq_domain_ops *vpe_domain_ops;
 88static const struct irq_domain_ops *sgi_domain_ops;
 89
 90#ifdef CONFIG_ARM64
 91#include <asm/cpufeature.h>
 92
 93bool gic_cpuif_has_vsgi(void)
 94{
 95	unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
 96
 97	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT);
 98
 99	return fld >= 0x3;
100}
101#else
102bool gic_cpuif_has_vsgi(void)
103{
104	return false;
105}
106#endif
107
108static bool has_v4_1(void)
109{
110	return !!sgi_domain_ops;
111}
112
113static bool has_v4_1_sgi(void)
114{
115	return has_v4_1() && gic_cpuif_has_vsgi();
116}
117
118static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
119{
120	char *name;
121	int sgi_base;
122
123	if (!has_v4_1_sgi())
124		return 0;
125
126	name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current));
127	if (!name)
128		goto err;
129
130	vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx);
131	if (!vpe->fwnode)
132		goto err;
133
134	kfree(name);
135	name = NULL;
136
137	vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16,
138						   sgi_domain_ops, vpe);
139	if (!vpe->sgi_domain)
140		goto err;
141
142	sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16,
143					       NUMA_NO_NODE, vpe,
144					       false, NULL);
145	if (sgi_base <= 0)
146		goto err;
147
148	return 0;
149
150err:
151	if (vpe->sgi_domain)
152		irq_domain_remove(vpe->sgi_domain);
153	if (vpe->fwnode)
154		irq_domain_free_fwnode(vpe->fwnode);
155	kfree(name);
156	return -ENOMEM;
157}
158
159int its_alloc_vcpu_irqs(struct its_vm *vm)
160{
161	int vpe_base_irq, i;
162
163	vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
164						      task_pid_nr(current));
165	if (!vm->fwnode)
166		goto err;
167
168	vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
169						 vm->fwnode, vpe_domain_ops,
170						 vm);
171	if (!vm->domain)
172		goto err;
173
174	for (i = 0; i < vm->nr_vpes; i++) {
175		vm->vpes[i]->its_vm = vm;
176		vm->vpes[i]->idai = true;
177	}
178
179	vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes,
180					       NUMA_NO_NODE, vm,
181					       false, NULL);
182	if (vpe_base_irq <= 0)
183		goto err;
184
185	for (i = 0; i < vm->nr_vpes; i++) {
186		int ret;
187		vm->vpes[i]->irq = vpe_base_irq + i;
188		ret = its_alloc_vcpu_sgis(vm->vpes[i], i);
189		if (ret)
190			goto err;
191	}
192
193	return 0;
194
195err:
196	if (vm->domain)
197		irq_domain_remove(vm->domain);
198	if (vm->fwnode)
199		irq_domain_free_fwnode(vm->fwnode);
200
201	return -ENOMEM;
202}
203
204static void its_free_sgi_irqs(struct its_vm *vm)
205{
206	int i;
207
208	if (!has_v4_1_sgi())
209		return;
210
211	for (i = 0; i < vm->nr_vpes; i++) {
212		unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
213
214		if (WARN_ON(!irq))
215			continue;
216
217		irq_domain_free_irqs(irq, 16);
218		irq_domain_remove(vm->vpes[i]->sgi_domain);
219		irq_domain_free_fwnode(vm->vpes[i]->fwnode);
220	}
221}
222
223void its_free_vcpu_irqs(struct its_vm *vm)
224{
225	its_free_sgi_irqs(vm);
226	irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
227	irq_domain_remove(vm->domain);
228	irq_domain_free_fwnode(vm->fwnode);
229}
230
231static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
232{
233	return irq_set_vcpu_affinity(vpe->irq, info);
234}
235
236int its_make_vpe_non_resident(struct its_vpe *vpe, bool db)
237{
238	struct irq_desc *desc = irq_to_desc(vpe->irq);
239	struct its_cmd_info info = { };
240	int ret;
241
242	WARN_ON(preemptible());
243
244	info.cmd_type = DESCHEDULE_VPE;
245	if (has_v4_1()) {
246		/* GICv4.1 can directly deal with doorbells */
247		info.req_db = db;
248	} else {
249		/* Undo the nested disable_irq() calls... */
250		while (db && irqd_irq_disabled(&desc->irq_data))
251			enable_irq(vpe->irq);
252	}
253
254	ret = its_send_vpe_cmd(vpe, &info);
255	if (!ret)
256		vpe->resident = false;
257
258	vpe->ready = false;
259
260	return ret;
261}
262
263int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en)
264{
265	struct its_cmd_info info = { };
266	int ret;
267
268	WARN_ON(preemptible());
269
270	info.cmd_type = SCHEDULE_VPE;
271	if (has_v4_1()) {
272		info.g0en = g0en;
273		info.g1en = g1en;
274	} else {
275		/* Disabled the doorbell, as we're about to enter the guest */
276		disable_irq_nosync(vpe->irq);
277	}
278
279	ret = its_send_vpe_cmd(vpe, &info);
280	if (!ret)
281		vpe->resident = true;
282
283	return ret;
284}
285
286int its_commit_vpe(struct its_vpe *vpe)
287{
288	struct its_cmd_info info = {
289		.cmd_type = COMMIT_VPE,
290	};
291	int ret;
292
293	WARN_ON(preemptible());
294
295	ret = its_send_vpe_cmd(vpe, &info);
296	if (!ret)
297		vpe->ready = true;
298
299	return ret;
300}
301
302
303int its_invall_vpe(struct its_vpe *vpe)
304{
305	struct its_cmd_info info = {
306		.cmd_type = INVALL_VPE,
307	};
308
309	return its_send_vpe_cmd(vpe, &info);
310}
311
312int its_map_vlpi(int irq, struct its_vlpi_map *map)
313{
314	struct its_cmd_info info = {
315		.cmd_type = MAP_VLPI,
316		{
317			.map      = map,
318		},
319	};
320	int ret;
321
322	/*
323	 * The host will never see that interrupt firing again, so it
324	 * is vital that we don't do any lazy masking.
325	 */
326	irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
327
328	ret = irq_set_vcpu_affinity(irq, &info);
329	if (ret)
330		irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
331
332	return ret;
333}
334
335int its_get_vlpi(int irq, struct its_vlpi_map *map)
336{
337	struct its_cmd_info info = {
338		.cmd_type = GET_VLPI,
339		{
340			.map      = map,
341		},
342	};
343
344	return irq_set_vcpu_affinity(irq, &info);
345}
346
347int its_unmap_vlpi(int irq)
348{
349	irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
350	return irq_set_vcpu_affinity(irq, NULL);
351}
352
353int its_prop_update_vlpi(int irq, u8 config, bool inv)
354{
355	struct its_cmd_info info = {
356		.cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
357		{
358			.config   = config,
359		},
360	};
361
362	return irq_set_vcpu_affinity(irq, &info);
363}
364
365int its_prop_update_vsgi(int irq, u8 priority, bool group)
366{
367	struct its_cmd_info info = {
368		.cmd_type = PROP_UPDATE_VSGI,
369		{
370			.priority	= priority,
371			.group		= group,
372		},
373	};
374
375	return irq_set_vcpu_affinity(irq, &info);
376}
377
378int its_init_v4(struct irq_domain *domain,
379		const struct irq_domain_ops *vpe_ops,
380		const struct irq_domain_ops *sgi_ops)
381{
382	if (domain) {
383		pr_info("ITS: Enabling GICv4 support\n");
384		gic_domain = domain;
385		vpe_domain_ops = vpe_ops;
386		sgi_domain_ops = sgi_ops;
387		return 0;
388	}
389
390	pr_err("ITS: No GICv4 VPE domain allocated\n");
391	return -ENODEV;
392}