Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2015 Imagination Technologies Ltd
4 * Author: Qais Yousef <qais.yousef@imgtec.com>
5 *
6 * This file contains driver APIs to the IPI subsystem.
7 */
8
9#define pr_fmt(fmt) "genirq/ipi: " fmt
10
11#include <linux/irqdomain.h>
12#include <linux/irq.h>
13
14/**
15 * irq_reserve_ipi() - Setup an IPI to destination cpumask
16 * @domain: IPI domain
17 * @dest: cpumask of cpus which can receive the IPI
18 *
19 * Allocate a virq that can be used to send IPI to any CPU in dest mask.
20 *
21 * On success it'll return linux irq number and error code on failure
22 */
23int irq_reserve_ipi(struct irq_domain *domain,
24 const struct cpumask *dest)
25{
26 unsigned int nr_irqs, offset;
27 struct irq_data *data;
28 int virq, i;
29
30 if (!domain ||!irq_domain_is_ipi(domain)) {
31 pr_warn("Reservation on a non IPI domain\n");
32 return -EINVAL;
33 }
34
35 if (!cpumask_subset(dest, cpu_possible_mask)) {
36 pr_warn("Reservation is not in possible_cpu_mask\n");
37 return -EINVAL;
38 }
39
40 nr_irqs = cpumask_weight(dest);
41 if (!nr_irqs) {
42 pr_warn("Reservation for empty destination mask\n");
43 return -EINVAL;
44 }
45
46 if (irq_domain_is_ipi_single(domain)) {
47 /*
48 * If the underlying implementation uses a single HW irq on
49 * all cpus then we only need a single Linux irq number for
50 * it. We have no restrictions vs. the destination mask. The
51 * underlying implementation can deal with holes nicely.
52 */
53 nr_irqs = 1;
54 offset = 0;
55 } else {
56 unsigned int next;
57
58 /*
59 * The IPI requires a separate HW irq on each CPU. We require
60 * that the destination mask is consecutive. If an
61 * implementation needs to support holes, it can reserve
62 * several IPI ranges.
63 */
64 offset = cpumask_first(dest);
65 /*
66 * Find a hole and if found look for another set bit after the
67 * hole. For now we don't support this scenario.
68 */
69 next = cpumask_next_zero(offset, dest);
70 if (next < nr_cpu_ids)
71 next = cpumask_next(next, dest);
72 if (next < nr_cpu_ids) {
73 pr_warn("Destination mask has holes\n");
74 return -EINVAL;
75 }
76 }
77
78 virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
79 if (virq <= 0) {
80 pr_warn("Can't reserve IPI, failed to alloc descs\n");
81 return -ENOMEM;
82 }
83
84 virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
85 (void *) dest, true, NULL);
86
87 if (virq <= 0) {
88 pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
89 goto free_descs;
90 }
91
92 for (i = 0; i < nr_irqs; i++) {
93 data = irq_get_irq_data(virq + i);
94 cpumask_copy(data->common->affinity, dest);
95 data->common->ipi_offset = offset;
96 irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
97 }
98 return virq;
99
100free_descs:
101 irq_free_descs(virq, nr_irqs);
102 return -EBUSY;
103}
104
105/**
106 * irq_destroy_ipi() - unreserve an IPI that was previously allocated
107 * @irq: linux irq number to be destroyed
108 * @dest: cpumask of cpus which should have the IPI removed
109 *
110 * The IPIs allocated with irq_reserve_ipi() are retuerned to the system
111 * destroying all virqs associated with them.
112 *
113 * Return 0 on success or error code on failure.
114 */
115int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
116{
117 struct irq_data *data = irq_get_irq_data(irq);
118 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
119 struct irq_domain *domain;
120 unsigned int nr_irqs;
121
122 if (!irq || !data || !ipimask)
123 return -EINVAL;
124
125 domain = data->domain;
126 if (WARN_ON(domain == NULL))
127 return -EINVAL;
128
129 if (!irq_domain_is_ipi(domain)) {
130 pr_warn("Trying to destroy a non IPI domain!\n");
131 return -EINVAL;
132 }
133
134 if (WARN_ON(!cpumask_subset(dest, ipimask)))
135 /*
136 * Must be destroying a subset of CPUs to which this IPI
137 * was set up to target
138 */
139 return -EINVAL;
140
141 if (irq_domain_is_ipi_per_cpu(domain)) {
142 irq = irq + cpumask_first(dest) - data->common->ipi_offset;
143 nr_irqs = cpumask_weight(dest);
144 } else {
145 nr_irqs = 1;
146 }
147
148 irq_domain_free_irqs(irq, nr_irqs);
149 return 0;
150}
151
152/**
153 * ipi_get_hwirq - Get the hwirq associated with an IPI to a cpu
154 * @irq: linux irq number
155 * @cpu: the target cpu
156 *
157 * When dealing with coprocessors IPI, we need to inform the coprocessor of
158 * the hwirq it needs to use to receive and send IPIs.
159 *
160 * Returns hwirq value on success and INVALID_HWIRQ on failure.
161 */
162irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
163{
164 struct irq_data *data = irq_get_irq_data(irq);
165 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
166
167 if (!data || !ipimask || cpu >= nr_cpu_ids)
168 return INVALID_HWIRQ;
169
170 if (!cpumask_test_cpu(cpu, ipimask))
171 return INVALID_HWIRQ;
172
173 /*
174 * Get the real hardware irq number if the underlying implementation
175 * uses a separate irq per cpu. If the underlying implementation uses
176 * a single hardware irq for all cpus then the IPI send mechanism
177 * needs to take care of the cpu destinations.
178 */
179 if (irq_domain_is_ipi_per_cpu(data->domain))
180 data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
181
182 return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
183}
184EXPORT_SYMBOL_GPL(ipi_get_hwirq);
185
186static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
187 const struct cpumask *dest, unsigned int cpu)
188{
189 struct cpumask *ipimask = irq_data_get_affinity_mask(data);
190
191 if (!chip || !ipimask)
192 return -EINVAL;
193
194 if (!chip->ipi_send_single && !chip->ipi_send_mask)
195 return -EINVAL;
196
197 if (cpu >= nr_cpu_ids)
198 return -EINVAL;
199
200 if (dest) {
201 if (!cpumask_subset(dest, ipimask))
202 return -EINVAL;
203 } else {
204 if (!cpumask_test_cpu(cpu, ipimask))
205 return -EINVAL;
206 }
207 return 0;
208}
209
210/**
211 * __ipi_send_single - send an IPI to a target Linux SMP CPU
212 * @desc: pointer to irq_desc of the IRQ
213 * @cpu: destination CPU, must in the destination mask passed to
214 * irq_reserve_ipi()
215 *
216 * This function is for architecture or core code to speed up IPI sending. Not
217 * usable from driver code.
218 *
219 * Returns zero on success and negative error number on failure.
220 */
221int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
222{
223 struct irq_data *data = irq_desc_get_irq_data(desc);
224 struct irq_chip *chip = irq_data_get_irq_chip(data);
225
226#ifdef DEBUG
227 /*
228 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
229 * Since the callers should be arch or core code which is generally
230 * trusted, only check for errors when debugging.
231 */
232 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
233 return -EINVAL;
234#endif
235 if (!chip->ipi_send_single) {
236 chip->ipi_send_mask(data, cpumask_of(cpu));
237 return 0;
238 }
239
240 /* FIXME: Store this information in irqdata flags */
241 if (irq_domain_is_ipi_per_cpu(data->domain) &&
242 cpu != data->common->ipi_offset) {
243 /* use the correct data for that cpu */
244 unsigned irq = data->irq + cpu - data->common->ipi_offset;
245
246 data = irq_get_irq_data(irq);
247 }
248 chip->ipi_send_single(data, cpu);
249 return 0;
250}
251
252/**
253 * ipi_send_mask - send an IPI to target Linux SMP CPU(s)
254 * @desc: pointer to irq_desc of the IRQ
255 * @dest: dest CPU(s), must be a subset of the mask passed to
256 * irq_reserve_ipi()
257 *
258 * This function is for architecture or core code to speed up IPI sending. Not
259 * usable from driver code.
260 *
261 * Returns zero on success and negative error number on failure.
262 */
263int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
264{
265 struct irq_data *data = irq_desc_get_irq_data(desc);
266 struct irq_chip *chip = irq_data_get_irq_chip(data);
267 unsigned int cpu;
268
269#ifdef DEBUG
270 /*
271 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
272 * Since the callers should be arch or core code which is generally
273 * trusted, only check for errors when debugging.
274 */
275 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
276 return -EINVAL;
277#endif
278 if (chip->ipi_send_mask) {
279 chip->ipi_send_mask(data, dest);
280 return 0;
281 }
282
283 if (irq_domain_is_ipi_per_cpu(data->domain)) {
284 unsigned int base = data->irq;
285
286 for_each_cpu(cpu, dest) {
287 unsigned irq = base + cpu - data->common->ipi_offset;
288
289 data = irq_get_irq_data(irq);
290 chip->ipi_send_single(data, cpu);
291 }
292 } else {
293 for_each_cpu(cpu, dest)
294 chip->ipi_send_single(data, cpu);
295 }
296 return 0;
297}
298
299/**
300 * ipi_send_single - Send an IPI to a single CPU
301 * @virq: linux irq number from irq_reserve_ipi()
302 * @cpu: destination CPU, must in the destination mask passed to
303 * irq_reserve_ipi()
304 *
305 * Returns zero on success and negative error number on failure.
306 */
307int ipi_send_single(unsigned int virq, unsigned int cpu)
308{
309 struct irq_desc *desc = irq_to_desc(virq);
310 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
311 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
312
313 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
314 return -EINVAL;
315
316 return __ipi_send_single(desc, cpu);
317}
318EXPORT_SYMBOL_GPL(ipi_send_single);
319
320/**
321 * ipi_send_mask - Send an IPI to target CPU(s)
322 * @virq: linux irq number from irq_reserve_ipi()
323 * @dest: dest CPU(s), must be a subset of the mask passed to
324 * irq_reserve_ipi()
325 *
326 * Returns zero on success and negative error number on failure.
327 */
328int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
329{
330 struct irq_desc *desc = irq_to_desc(virq);
331 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
332 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
333
334 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
335 return -EINVAL;
336
337 return __ipi_send_mask(desc, dest);
338}
339EXPORT_SYMBOL_GPL(ipi_send_mask);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2015 Imagination Technologies Ltd
4 * Author: Qais Yousef <qais.yousef@imgtec.com>
5 *
6 * This file contains driver APIs to the IPI subsystem.
7 */
8
9#define pr_fmt(fmt) "genirq/ipi: " fmt
10
11#include <linux/irqdomain.h>
12#include <linux/irq.h>
13
14/**
15 * irq_reserve_ipi() - Setup an IPI to destination cpumask
16 * @domain: IPI domain
17 * @dest: cpumask of cpus which can receive the IPI
18 *
19 * Allocate a virq that can be used to send IPI to any CPU in dest mask.
20 *
21 * On success it'll return linux irq number and error code on failure
22 */
23int irq_reserve_ipi(struct irq_domain *domain,
24 const struct cpumask *dest)
25{
26 unsigned int nr_irqs, offset;
27 struct irq_data *data;
28 int virq, i;
29
30 if (!domain ||!irq_domain_is_ipi(domain)) {
31 pr_warn("Reservation on a non IPI domain\n");
32 return -EINVAL;
33 }
34
35 if (!cpumask_subset(dest, cpu_possible_mask)) {
36 pr_warn("Reservation is not in possible_cpu_mask\n");
37 return -EINVAL;
38 }
39
40 nr_irqs = cpumask_weight(dest);
41 if (!nr_irqs) {
42 pr_warn("Reservation for empty destination mask\n");
43 return -EINVAL;
44 }
45
46 if (irq_domain_is_ipi_single(domain)) {
47 /*
48 * If the underlying implementation uses a single HW irq on
49 * all cpus then we only need a single Linux irq number for
50 * it. We have no restrictions vs. the destination mask. The
51 * underlying implementation can deal with holes nicely.
52 */
53 nr_irqs = 1;
54 offset = 0;
55 } else {
56 unsigned int next;
57
58 /*
59 * The IPI requires a separate HW irq on each CPU. We require
60 * that the destination mask is consecutive. If an
61 * implementation needs to support holes, it can reserve
62 * several IPI ranges.
63 */
64 offset = cpumask_first(dest);
65 /*
66 * Find a hole and if found look for another set bit after the
67 * hole. For now we don't support this scenario.
68 */
69 next = cpumask_next_zero(offset, dest);
70 if (next < nr_cpu_ids)
71 next = cpumask_next(next, dest);
72 if (next < nr_cpu_ids) {
73 pr_warn("Destination mask has holes\n");
74 return -EINVAL;
75 }
76 }
77
78 virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
79 if (virq <= 0) {
80 pr_warn("Can't reserve IPI, failed to alloc descs\n");
81 return -ENOMEM;
82 }
83
84 virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
85 (void *) dest, true, NULL);
86
87 if (virq <= 0) {
88 pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
89 goto free_descs;
90 }
91
92 for (i = 0; i < nr_irqs; i++) {
93 data = irq_get_irq_data(virq + i);
94 cpumask_copy(data->common->affinity, dest);
95 data->common->ipi_offset = offset;
96 irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
97 }
98 return virq;
99
100free_descs:
101 irq_free_descs(virq, nr_irqs);
102 return -EBUSY;
103}
104
105/**
106 * irq_destroy_ipi() - unreserve an IPI that was previously allocated
107 * @irq: linux irq number to be destroyed
108 * @dest: cpumask of cpus which should have the IPI removed
109 *
110 * The IPIs allocated with irq_reserve_ipi() are returned to the system
111 * destroying all virqs associated with them.
112 *
113 * Return 0 on success or error code on failure.
114 */
115int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
116{
117 struct irq_data *data = irq_get_irq_data(irq);
118 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
119 struct irq_domain *domain;
120 unsigned int nr_irqs;
121
122 if (!irq || !data || !ipimask)
123 return -EINVAL;
124
125 domain = data->domain;
126 if (WARN_ON(domain == NULL))
127 return -EINVAL;
128
129 if (!irq_domain_is_ipi(domain)) {
130 pr_warn("Trying to destroy a non IPI domain!\n");
131 return -EINVAL;
132 }
133
134 if (WARN_ON(!cpumask_subset(dest, ipimask)))
135 /*
136 * Must be destroying a subset of CPUs to which this IPI
137 * was set up to target
138 */
139 return -EINVAL;
140
141 if (irq_domain_is_ipi_per_cpu(domain)) {
142 irq = irq + cpumask_first(dest) - data->common->ipi_offset;
143 nr_irqs = cpumask_weight(dest);
144 } else {
145 nr_irqs = 1;
146 }
147
148 irq_domain_free_irqs(irq, nr_irqs);
149 return 0;
150}
151
152/**
153 * ipi_get_hwirq - Get the hwirq associated with an IPI to a cpu
154 * @irq: linux irq number
155 * @cpu: the target cpu
156 *
157 * When dealing with coprocessors IPI, we need to inform the coprocessor of
158 * the hwirq it needs to use to receive and send IPIs.
159 *
160 * Returns hwirq value on success and INVALID_HWIRQ on failure.
161 */
162irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
163{
164 struct irq_data *data = irq_get_irq_data(irq);
165 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
166
167 if (!data || !ipimask || cpu >= nr_cpu_ids)
168 return INVALID_HWIRQ;
169
170 if (!cpumask_test_cpu(cpu, ipimask))
171 return INVALID_HWIRQ;
172
173 /*
174 * Get the real hardware irq number if the underlying implementation
175 * uses a separate irq per cpu. If the underlying implementation uses
176 * a single hardware irq for all cpus then the IPI send mechanism
177 * needs to take care of the cpu destinations.
178 */
179 if (irq_domain_is_ipi_per_cpu(data->domain))
180 data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
181
182 return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
183}
184EXPORT_SYMBOL_GPL(ipi_get_hwirq);
185
186static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
187 const struct cpumask *dest, unsigned int cpu)
188{
189 struct cpumask *ipimask = irq_data_get_affinity_mask(data);
190
191 if (!chip || !ipimask)
192 return -EINVAL;
193
194 if (!chip->ipi_send_single && !chip->ipi_send_mask)
195 return -EINVAL;
196
197 if (cpu >= nr_cpu_ids)
198 return -EINVAL;
199
200 if (dest) {
201 if (!cpumask_subset(dest, ipimask))
202 return -EINVAL;
203 } else {
204 if (!cpumask_test_cpu(cpu, ipimask))
205 return -EINVAL;
206 }
207 return 0;
208}
209
210/**
211 * __ipi_send_single - send an IPI to a target Linux SMP CPU
212 * @desc: pointer to irq_desc of the IRQ
213 * @cpu: destination CPU, must in the destination mask passed to
214 * irq_reserve_ipi()
215 *
216 * This function is for architecture or core code to speed up IPI sending. Not
217 * usable from driver code.
218 *
219 * Returns zero on success and negative error number on failure.
220 */
221int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
222{
223 struct irq_data *data = irq_desc_get_irq_data(desc);
224 struct irq_chip *chip = irq_data_get_irq_chip(data);
225
226#ifdef DEBUG
227 /*
228 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
229 * Since the callers should be arch or core code which is generally
230 * trusted, only check for errors when debugging.
231 */
232 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
233 return -EINVAL;
234#endif
235 if (!chip->ipi_send_single) {
236 chip->ipi_send_mask(data, cpumask_of(cpu));
237 return 0;
238 }
239
240 /* FIXME: Store this information in irqdata flags */
241 if (irq_domain_is_ipi_per_cpu(data->domain) &&
242 cpu != data->common->ipi_offset) {
243 /* use the correct data for that cpu */
244 unsigned irq = data->irq + cpu - data->common->ipi_offset;
245
246 data = irq_get_irq_data(irq);
247 }
248 chip->ipi_send_single(data, cpu);
249 return 0;
250}
251
252/**
253 * ipi_send_mask - send an IPI to target Linux SMP CPU(s)
254 * @desc: pointer to irq_desc of the IRQ
255 * @dest: dest CPU(s), must be a subset of the mask passed to
256 * irq_reserve_ipi()
257 *
258 * This function is for architecture or core code to speed up IPI sending. Not
259 * usable from driver code.
260 *
261 * Returns zero on success and negative error number on failure.
262 */
263int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
264{
265 struct irq_data *data = irq_desc_get_irq_data(desc);
266 struct irq_chip *chip = irq_data_get_irq_chip(data);
267 unsigned int cpu;
268
269#ifdef DEBUG
270 /*
271 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
272 * Since the callers should be arch or core code which is generally
273 * trusted, only check for errors when debugging.
274 */
275 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
276 return -EINVAL;
277#endif
278 if (chip->ipi_send_mask) {
279 chip->ipi_send_mask(data, dest);
280 return 0;
281 }
282
283 if (irq_domain_is_ipi_per_cpu(data->domain)) {
284 unsigned int base = data->irq;
285
286 for_each_cpu(cpu, dest) {
287 unsigned irq = base + cpu - data->common->ipi_offset;
288
289 data = irq_get_irq_data(irq);
290 chip->ipi_send_single(data, cpu);
291 }
292 } else {
293 for_each_cpu(cpu, dest)
294 chip->ipi_send_single(data, cpu);
295 }
296 return 0;
297}
298
299/**
300 * ipi_send_single - Send an IPI to a single CPU
301 * @virq: linux irq number from irq_reserve_ipi()
302 * @cpu: destination CPU, must in the destination mask passed to
303 * irq_reserve_ipi()
304 *
305 * Returns zero on success and negative error number on failure.
306 */
307int ipi_send_single(unsigned int virq, unsigned int cpu)
308{
309 struct irq_desc *desc = irq_to_desc(virq);
310 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
311 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
312
313 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
314 return -EINVAL;
315
316 return __ipi_send_single(desc, cpu);
317}
318EXPORT_SYMBOL_GPL(ipi_send_single);
319
320/**
321 * ipi_send_mask - Send an IPI to target CPU(s)
322 * @virq: linux irq number from irq_reserve_ipi()
323 * @dest: dest CPU(s), must be a subset of the mask passed to
324 * irq_reserve_ipi()
325 *
326 * Returns zero on success and negative error number on failure.
327 */
328int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
329{
330 struct irq_desc *desc = irq_to_desc(virq);
331 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
332 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
333
334 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
335 return -EINVAL;
336
337 return __ipi_send_mask(desc, dest);
338}
339EXPORT_SYMBOL_GPL(ipi_send_mask);