Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <linux/interrupt.h>
8#include <linux/irq.h>
9#include <linux/irqdomain.h>
10#include <linux/msi.h>
11#include <linux/sched.h>
12
13#include <linux/irqchip/arm-gic-v4.h>
14
15/*
16 * WARNING: The blurb below assumes that you understand the
17 * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
18 * translated into GICv4 commands. So it effectively targets at most
19 * two individuals. You know who you are.
20 *
21 * The core GICv4 code is designed to *avoid* exposing too much of the
22 * core GIC code (that would in turn leak into the hypervisor code),
23 * and instead provide a hypervisor agnostic interface to the HW (of
24 * course, the astute reader will quickly realize that hypervisor
25 * agnostic actually means KVM-specific - what were you thinking?).
26 *
27 * In order to achieve a modicum of isolation, we try to hide most of
28 * the GICv4 "stuff" behind normal irqchip operations:
29 *
30 * - Any guest-visible VLPI is backed by a Linux interrupt (and a
31 * physical LPI which gets unmapped when the guest maps the
32 * VLPI). This allows the same DevID/EventID pair to be either
33 * mapped to the LPI (host) or the VLPI (guest). Note that this is
34 * exclusive, and you cannot have both.
35 *
36 * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
37 *
38 * - Guest INT/CLEAR commands are implemented through
39 * irq_set_irqchip_state().
40 *
41 * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
42 * issuing an INV after changing a priority) gets shoved into the
43 * irq_set_vcpu_affinity() method. While this is quite horrible
44 * (let's face it, this is the irqchip version of an ioctl), it
45 * confines the crap to a single location. And map/unmap really is
46 * about setting the affinity of a VLPI to a vcpu, so only INV is
47 * majorly out of place. So there.
48 *
49 * A number of commands are simply not provided by this interface, as
50 * they do not make direct sense. For example, MAPD is purely local to
51 * the virtual ITS (because it references a virtual device, and the
52 * physical ITS is still very much in charge of the physical
53 * device). Same goes for things like MAPC (the physical ITS deals
54 * with the actual vPE affinity, and not the braindead concept of
55 * collection). SYNC is not provided either, as each and every command
56 * is followed by a VSYNC. This could be relaxed in the future, should
57 * this be seen as a bottleneck (yes, this means *never*).
58 *
59 * But handling VLPIs is only one side of the job of the GICv4
60 * code. The other (darker) side is to take care of the doorbell
61 * interrupts which are delivered when a VLPI targeting a non-running
62 * vcpu is being made pending.
63 *
64 * The choice made here is that each vcpu (VPE in old northern GICv4
65 * dialect) gets a single doorbell LPI, no matter how many interrupts
66 * are targeting it. This has a nice property, which is that the
67 * interrupt becomes a handle for the VPE, and that the hypervisor
68 * code can manipulate it through the normal interrupt API:
69 *
70 * - VMs (or rather the VM abstraction that matters to the GIC)
71 * contain an irq domain where each interrupt maps to a VPE. In
72 * turn, this domain sits on top of the normal LPI allocator, and a
73 * specially crafted irq_chip implementation.
74 *
75 * - mask/unmask do what is expected on the doorbell interrupt.
76 *
77 * - irq_set_affinity is used to move a VPE from one redistributor to
78 * another.
79 *
80 * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
81 * creating a new sub-API, namely scheduling/descheduling a VPE
82 * (which involves programming GICR_V{PROP,PEND}BASER) and
83 * performing INVALL operations.
84 */
85
86static struct irq_domain *gic_domain;
87static const struct irq_domain_ops *vpe_domain_ops;
88static const struct irq_domain_ops *sgi_domain_ops;
89
90static bool has_v4_1(void)
91{
92 return !!sgi_domain_ops;
93}
94
95static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
96{
97 char *name;
98 int sgi_base;
99
100 if (!has_v4_1())
101 return 0;
102
103 name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current));
104 if (!name)
105 goto err;
106
107 vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx);
108 if (!vpe->fwnode)
109 goto err;
110
111 kfree(name);
112 name = NULL;
113
114 vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16,
115 sgi_domain_ops, vpe);
116 if (!vpe->sgi_domain)
117 goto err;
118
119 sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16,
120 NUMA_NO_NODE, vpe,
121 false, NULL);
122 if (sgi_base <= 0)
123 goto err;
124
125 return 0;
126
127err:
128 if (vpe->sgi_domain)
129 irq_domain_remove(vpe->sgi_domain);
130 if (vpe->fwnode)
131 irq_domain_free_fwnode(vpe->fwnode);
132 kfree(name);
133 return -ENOMEM;
134}
135
136int its_alloc_vcpu_irqs(struct its_vm *vm)
137{
138 int vpe_base_irq, i;
139
140 vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
141 task_pid_nr(current));
142 if (!vm->fwnode)
143 goto err;
144
145 vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
146 vm->fwnode, vpe_domain_ops,
147 vm);
148 if (!vm->domain)
149 goto err;
150
151 for (i = 0; i < vm->nr_vpes; i++) {
152 vm->vpes[i]->its_vm = vm;
153 vm->vpes[i]->idai = true;
154 }
155
156 vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes,
157 NUMA_NO_NODE, vm,
158 false, NULL);
159 if (vpe_base_irq <= 0)
160 goto err;
161
162 for (i = 0; i < vm->nr_vpes; i++) {
163 int ret;
164 vm->vpes[i]->irq = vpe_base_irq + i;
165 ret = its_alloc_vcpu_sgis(vm->vpes[i], i);
166 if (ret)
167 goto err;
168 }
169
170 return 0;
171
172err:
173 if (vm->domain)
174 irq_domain_remove(vm->domain);
175 if (vm->fwnode)
176 irq_domain_free_fwnode(vm->fwnode);
177
178 return -ENOMEM;
179}
180
181static void its_free_sgi_irqs(struct its_vm *vm)
182{
183 int i;
184
185 if (!has_v4_1())
186 return;
187
188 for (i = 0; i < vm->nr_vpes; i++) {
189 unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
190
191 if (WARN_ON(!irq))
192 continue;
193
194 irq_domain_free_irqs(irq, 16);
195 irq_domain_remove(vm->vpes[i]->sgi_domain);
196 irq_domain_free_fwnode(vm->vpes[i]->fwnode);
197 }
198}
199
200void its_free_vcpu_irqs(struct its_vm *vm)
201{
202 its_free_sgi_irqs(vm);
203 irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
204 irq_domain_remove(vm->domain);
205 irq_domain_free_fwnode(vm->fwnode);
206}
207
208static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
209{
210 return irq_set_vcpu_affinity(vpe->irq, info);
211}
212
213int its_make_vpe_non_resident(struct its_vpe *vpe, bool db)
214{
215 struct irq_desc *desc = irq_to_desc(vpe->irq);
216 struct its_cmd_info info = { };
217 int ret;
218
219 WARN_ON(preemptible());
220
221 info.cmd_type = DESCHEDULE_VPE;
222 if (has_v4_1()) {
223 /* GICv4.1 can directly deal with doorbells */
224 info.req_db = db;
225 } else {
226 /* Undo the nested disable_irq() calls... */
227 while (db && irqd_irq_disabled(&desc->irq_data))
228 enable_irq(vpe->irq);
229 }
230
231 ret = its_send_vpe_cmd(vpe, &info);
232 if (!ret)
233 vpe->resident = false;
234
235 return ret;
236}
237
238int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en)
239{
240 struct its_cmd_info info = { };
241 int ret;
242
243 WARN_ON(preemptible());
244
245 info.cmd_type = SCHEDULE_VPE;
246 if (has_v4_1()) {
247 info.g0en = g0en;
248 info.g1en = g1en;
249 } else {
250 /* Disabled the doorbell, as we're about to enter the guest */
251 disable_irq_nosync(vpe->irq);
252 }
253
254 ret = its_send_vpe_cmd(vpe, &info);
255 if (!ret)
256 vpe->resident = true;
257
258 return ret;
259}
260
261int its_invall_vpe(struct its_vpe *vpe)
262{
263 struct its_cmd_info info = {
264 .cmd_type = INVALL_VPE,
265 };
266
267 return its_send_vpe_cmd(vpe, &info);
268}
269
270int its_map_vlpi(int irq, struct its_vlpi_map *map)
271{
272 struct its_cmd_info info = {
273 .cmd_type = MAP_VLPI,
274 {
275 .map = map,
276 },
277 };
278 int ret;
279
280 /*
281 * The host will never see that interrupt firing again, so it
282 * is vital that we don't do any lazy masking.
283 */
284 irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
285
286 ret = irq_set_vcpu_affinity(irq, &info);
287 if (ret)
288 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
289
290 return ret;
291}
292
293int its_get_vlpi(int irq, struct its_vlpi_map *map)
294{
295 struct its_cmd_info info = {
296 .cmd_type = GET_VLPI,
297 {
298 .map = map,
299 },
300 };
301
302 return irq_set_vcpu_affinity(irq, &info);
303}
304
305int its_unmap_vlpi(int irq)
306{
307 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
308 return irq_set_vcpu_affinity(irq, NULL);
309}
310
311int its_prop_update_vlpi(int irq, u8 config, bool inv)
312{
313 struct its_cmd_info info = {
314 .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
315 {
316 .config = config,
317 },
318 };
319
320 return irq_set_vcpu_affinity(irq, &info);
321}
322
323int its_prop_update_vsgi(int irq, u8 priority, bool group)
324{
325 struct its_cmd_info info = {
326 .cmd_type = PROP_UPDATE_VSGI,
327 {
328 .priority = priority,
329 .group = group,
330 },
331 };
332
333 return irq_set_vcpu_affinity(irq, &info);
334}
335
336int its_init_v4(struct irq_domain *domain,
337 const struct irq_domain_ops *vpe_ops,
338 const struct irq_domain_ops *sgi_ops)
339{
340 if (domain) {
341 pr_info("ITS: Enabling GICv4 support\n");
342 gic_domain = domain;
343 vpe_domain_ops = vpe_ops;
344 sgi_domain_ops = sgi_ops;
345 return 0;
346 }
347
348 pr_err("ITS: No GICv4 VPE domain allocated\n");
349 return -ENODEV;
350}
1/*
2 * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/interrupt.h>
19#include <linux/irq.h>
20#include <linux/irqdomain.h>
21#include <linux/msi.h>
22#include <linux/sched.h>
23
24#include <linux/irqchip/arm-gic-v4.h>
25
26/*
27 * WARNING: The blurb below assumes that you understand the
28 * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
29 * translated into GICv4 commands. So it effectively targets at most
30 * two individuals. You know who you are.
31 *
32 * The core GICv4 code is designed to *avoid* exposing too much of the
33 * core GIC code (that would in turn leak into the hypervisor code),
34 * and instead provide a hypervisor agnostic interface to the HW (of
35 * course, the astute reader will quickly realize that hypervisor
36 * agnostic actually means KVM-specific - what were you thinking?).
37 *
38 * In order to achieve a modicum of isolation, we try to hide most of
39 * the GICv4 "stuff" behind normal irqchip operations:
40 *
41 * - Any guest-visible VLPI is backed by a Linux interrupt (and a
42 * physical LPI which gets unmapped when the guest maps the
43 * VLPI). This allows the same DevID/EventID pair to be either
44 * mapped to the LPI (host) or the VLPI (guest). Note that this is
45 * exclusive, and you cannot have both.
46 *
47 * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
48 *
49 * - Guest INT/CLEAR commands are implemented through
50 * irq_set_irqchip_state().
51 *
52 * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
53 * issuing an INV after changing a priority) gets shoved into the
54 * irq_set_vcpu_affinity() method. While this is quite horrible
55 * (let's face it, this is the irqchip version of an ioctl), it
56 * confines the crap to a single location. And map/unmap really is
57 * about setting the affinity of a VLPI to a vcpu, so only INV is
58 * majorly out of place. So there.
59 *
60 * A number of commands are simply not provided by this interface, as
61 * they do not make direct sense. For example, MAPD is purely local to
62 * the virtual ITS (because it references a virtual device, and the
63 * physical ITS is still very much in charge of the physical
64 * device). Same goes for things like MAPC (the physical ITS deals
65 * with the actual vPE affinity, and not the braindead concept of
66 * collection). SYNC is not provided either, as each and every command
67 * is followed by a VSYNC. This could be relaxed in the future, should
68 * this be seen as a bottleneck (yes, this means *never*).
69 *
70 * But handling VLPIs is only one side of the job of the GICv4
71 * code. The other (darker) side is to take care of the doorbell
72 * interrupts which are delivered when a VLPI targeting a non-running
73 * vcpu is being made pending.
74 *
75 * The choice made here is that each vcpu (VPE in old northern GICv4
76 * dialect) gets a single doorbell LPI, no matter how many interrupts
77 * are targeting it. This has a nice property, which is that the
78 * interrupt becomes a handle for the VPE, and that the hypervisor
79 * code can manipulate it through the normal interrupt API:
80 *
81 * - VMs (or rather the VM abstraction that matters to the GIC)
82 * contain an irq domain where each interrupt maps to a VPE. In
83 * turn, this domain sits on top of the normal LPI allocator, and a
84 * specially crafted irq_chip implementation.
85 *
86 * - mask/unmask do what is expected on the doorbell interrupt.
87 *
88 * - irq_set_affinity is used to move a VPE from one redistributor to
89 * another.
90 *
91 * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
92 * creating a new sub-API, namely scheduling/descheduling a VPE
93 * (which involves programming GICR_V{PROP,PEND}BASER) and
94 * performing INVALL operations.
95 */
96
97static struct irq_domain *gic_domain;
98static const struct irq_domain_ops *vpe_domain_ops;
99
100int its_alloc_vcpu_irqs(struct its_vm *vm)
101{
102 int vpe_base_irq, i;
103
104 vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
105 task_pid_nr(current));
106 if (!vm->fwnode)
107 goto err;
108
109 vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
110 vm->fwnode, vpe_domain_ops,
111 vm);
112 if (!vm->domain)
113 goto err;
114
115 for (i = 0; i < vm->nr_vpes; i++) {
116 vm->vpes[i]->its_vm = vm;
117 vm->vpes[i]->idai = true;
118 }
119
120 vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes,
121 NUMA_NO_NODE, vm,
122 false, NULL);
123 if (vpe_base_irq <= 0)
124 goto err;
125
126 for (i = 0; i < vm->nr_vpes; i++)
127 vm->vpes[i]->irq = vpe_base_irq + i;
128
129 return 0;
130
131err:
132 if (vm->domain)
133 irq_domain_remove(vm->domain);
134 if (vm->fwnode)
135 irq_domain_free_fwnode(vm->fwnode);
136
137 return -ENOMEM;
138}
139
140void its_free_vcpu_irqs(struct its_vm *vm)
141{
142 irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
143 irq_domain_remove(vm->domain);
144 irq_domain_free_fwnode(vm->fwnode);
145}
146
147static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
148{
149 return irq_set_vcpu_affinity(vpe->irq, info);
150}
151
152int its_schedule_vpe(struct its_vpe *vpe, bool on)
153{
154 struct its_cmd_info info;
155
156 WARN_ON(preemptible());
157
158 info.cmd_type = on ? SCHEDULE_VPE : DESCHEDULE_VPE;
159
160 return its_send_vpe_cmd(vpe, &info);
161}
162
163int its_invall_vpe(struct its_vpe *vpe)
164{
165 struct its_cmd_info info = {
166 .cmd_type = INVALL_VPE,
167 };
168
169 return its_send_vpe_cmd(vpe, &info);
170}
171
172int its_map_vlpi(int irq, struct its_vlpi_map *map)
173{
174 struct its_cmd_info info = {
175 .cmd_type = MAP_VLPI,
176 {
177 .map = map,
178 },
179 };
180 int ret;
181
182 /*
183 * The host will never see that interrupt firing again, so it
184 * is vital that we don't do any lazy masking.
185 */
186 irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
187
188 ret = irq_set_vcpu_affinity(irq, &info);
189 if (ret)
190 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
191
192 return ret;
193}
194
195int its_get_vlpi(int irq, struct its_vlpi_map *map)
196{
197 struct its_cmd_info info = {
198 .cmd_type = GET_VLPI,
199 {
200 .map = map,
201 },
202 };
203
204 return irq_set_vcpu_affinity(irq, &info);
205}
206
207int its_unmap_vlpi(int irq)
208{
209 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
210 return irq_set_vcpu_affinity(irq, NULL);
211}
212
213int its_prop_update_vlpi(int irq, u8 config, bool inv)
214{
215 struct its_cmd_info info = {
216 .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
217 {
218 .config = config,
219 },
220 };
221
222 return irq_set_vcpu_affinity(irq, &info);
223}
224
225int its_init_v4(struct irq_domain *domain, const struct irq_domain_ops *ops)
226{
227 if (domain) {
228 pr_info("ITS: Enabling GICv4 support\n");
229 gic_domain = domain;
230 vpe_domain_ops = ops;
231 return 0;
232 }
233
234 pr_err("ITS: No GICv4 VPE domain allocated\n");
235 return -ENODEV;
236}