1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2015 Imagination Technologies Ltd
  4 * Author: Qais Yousef <qais.yousef@imgtec.com>
  5 *
  6 * This file contains driver APIs to the IPI subsystem.
  7 */
  8
  9#define pr_fmt(fmt) "genirq/ipi: " fmt
 10
 11#include <linux/irqdomain.h>
 12#include <linux/irq.h>
 13
 14/**
 15 * irq_reserve_ipi() - Setup an IPI to destination cpumask
 16 * @domain:	IPI domain
 17 * @dest:	cpumask of CPUs which can receive the IPI
 18 *
 19 * Allocate a virq that can be used to send IPI to any CPU in dest mask.
 20 *
 21 * Return: Linux IRQ number on success or error code on failure
 22 */
 23int irq_reserve_ipi(struct irq_domain *domain,
 24			     const struct cpumask *dest)
 25{
 26	unsigned int nr_irqs, offset;
 27	struct irq_data *data;
 28	int virq, i;
 29
 30	if (!domain ||!irq_domain_is_ipi(domain)) {
 31		pr_warn("Reservation on a non IPI domain\n");
 32		return -EINVAL;
 33	}
 34
 35	if (!cpumask_subset(dest, cpu_possible_mask)) {
 36		pr_warn("Reservation is not in possible_cpu_mask\n");
 37		return -EINVAL;
 38	}
 39
 40	nr_irqs = cpumask_weight(dest);
 41	if (!nr_irqs) {
 42		pr_warn("Reservation for empty destination mask\n");
 43		return -EINVAL;
 44	}
 45
 46	if (irq_domain_is_ipi_single(domain)) {
 47		/*
 48		 * If the underlying implementation uses a single HW irq on
 49		 * all cpus then we only need a single Linux irq number for
 50		 * it. We have no restrictions vs. the destination mask. The
 51		 * underlying implementation can deal with holes nicely.
 52		 */
 53		nr_irqs = 1;
 54		offset = 0;
 55	} else {
 56		unsigned int next;
 57
 58		/*
 59		 * The IPI requires a separate HW irq on each CPU. We require
 60		 * that the destination mask is consecutive. If an
 61		 * implementation needs to support holes, it can reserve
 62		 * several IPI ranges.
 63		 */
 64		offset = cpumask_first(dest);
 65		/*
 66		 * Find a hole and if found look for another set bit after the
 67		 * hole. For now we don't support this scenario.
 68		 */
 69		next = cpumask_next_zero(offset, dest);
 70		if (next < nr_cpu_ids)
 71			next = cpumask_next(next, dest);
 72		if (next < nr_cpu_ids) {
 73			pr_warn("Destination mask has holes\n");
 74			return -EINVAL;
 75		}
 76	}
 77
 78	virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
 79	if (virq <= 0) {
 80		pr_warn("Can't reserve IPI, failed to alloc descs\n");
 81		return -ENOMEM;
 82	}
 83
 84	virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
 85				       (void *) dest, true, NULL);
 86
 87	if (virq <= 0) {
 88		pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
 89		goto free_descs;
 90	}
 91
 92	for (i = 0; i < nr_irqs; i++) {
 93		data = irq_get_irq_data(virq + i);
 94		cpumask_copy(data->common->affinity, dest);
 95		data->common->ipi_offset = offset;
 96		irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
 97	}
 98	return virq;
 99
100free_descs:
101	irq_free_descs(virq, nr_irqs);
102	return -EBUSY;
103}
104
105/**
106 * irq_destroy_ipi() - unreserve an IPI that was previously allocated
107 * @irq:	Linux IRQ number to be destroyed
108 * @dest:	cpumask of CPUs which should have the IPI removed
109 *
110 * The IPIs allocated with irq_reserve_ipi() are returned to the system
111 * destroying all virqs associated with them.
112 *
113 * Return: %0 on success or error code on failure.
114 */
115int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
116{
117	struct irq_data *data = irq_get_irq_data(irq);
118	const struct cpumask *ipimask;
119	struct irq_domain *domain;
120	unsigned int nr_irqs;
121
122	if (!irq || !data)
123		return -EINVAL;
124
125	domain = data->domain;
126	if (WARN_ON(domain == NULL))
127		return -EINVAL;
128
129	if (!irq_domain_is_ipi(domain)) {
130		pr_warn("Trying to destroy a non IPI domain!\n");
131		return -EINVAL;
132	}
133
134	ipimask = irq_data_get_affinity_mask(data);
135	if (!ipimask || WARN_ON(!cpumask_subset(dest, ipimask)))
136		/*
137		 * Must be destroying a subset of CPUs to which this IPI
138		 * was set up to target
139		 */
140		return -EINVAL;
141
142	if (irq_domain_is_ipi_per_cpu(domain)) {
143		irq = irq + cpumask_first(dest) - data->common->ipi_offset;
144		nr_irqs = cpumask_weight(dest);
145	} else {
146		nr_irqs = 1;
147	}
148
149	irq_domain_free_irqs(irq, nr_irqs);
150	return 0;
151}
152
153/**
154 * ipi_get_hwirq - Get the hwirq associated with an IPI to a CPU
155 * @irq:	Linux IRQ number
156 * @cpu:	the target CPU
157 *
158 * When dealing with coprocessors IPI, we need to inform the coprocessor of
159 * the hwirq it needs to use to receive and send IPIs.
160 *
161 * Return: hwirq value on success or INVALID_HWIRQ on failure.
162 */
163irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
164{
165	struct irq_data *data = irq_get_irq_data(irq);
166	const struct cpumask *ipimask;
167
168	if (!data || cpu >= nr_cpu_ids)
169		return INVALID_HWIRQ;
170
171	ipimask = irq_data_get_affinity_mask(data);
172	if (!ipimask || !cpumask_test_cpu(cpu, ipimask))
173		return INVALID_HWIRQ;
174
175	/*
176	 * Get the real hardware irq number if the underlying implementation
177	 * uses a separate irq per cpu. If the underlying implementation uses
178	 * a single hardware irq for all cpus then the IPI send mechanism
179	 * needs to take care of the cpu destinations.
180	 */
181	if (irq_domain_is_ipi_per_cpu(data->domain))
182		data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
183
184	return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
185}
186EXPORT_SYMBOL_GPL(ipi_get_hwirq);
187
188static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
189			   const struct cpumask *dest, unsigned int cpu)
190{
191	const struct cpumask *ipimask = irq_data_get_affinity_mask(data);
192
193	if (!chip || !ipimask)
194		return -EINVAL;
195
196	if (!chip->ipi_send_single && !chip->ipi_send_mask)
197		return -EINVAL;
198
199	if (cpu >= nr_cpu_ids)
200		return -EINVAL;
201
202	if (dest) {
203		if (!cpumask_subset(dest, ipimask))
204			return -EINVAL;
205	} else {
206		if (!cpumask_test_cpu(cpu, ipimask))
207			return -EINVAL;
208	}
209	return 0;
210}
211
212/**
213 * __ipi_send_single - send an IPI to a target Linux SMP CPU
214 * @desc:	pointer to irq_desc of the IRQ
215 * @cpu:	destination CPU, must in the destination mask passed to
216 *		irq_reserve_ipi()
217 *
218 * This function is for architecture or core code to speed up IPI sending. Not
219 * usable from driver code.
220 *
221 * Return: %0 on success or negative error number on failure.
222 */
223int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
224{
225	struct irq_data *data = irq_desc_get_irq_data(desc);
226	struct irq_chip *chip = irq_data_get_irq_chip(data);
227
228#ifdef DEBUG
229	/*
230	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
231	 * Since the callers should be arch or core code which is generally
232	 * trusted, only check for errors when debugging.
233	 */
234	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
235		return -EINVAL;
236#endif
237	if (!chip->ipi_send_single) {
238		chip->ipi_send_mask(data, cpumask_of(cpu));
239		return 0;
240	}
241
242	/* FIXME: Store this information in irqdata flags */
243	if (irq_domain_is_ipi_per_cpu(data->domain) &&
244	    cpu != data->common->ipi_offset) {
245		/* use the correct data for that cpu */
246		unsigned irq = data->irq + cpu - data->common->ipi_offset;
247
248		data = irq_get_irq_data(irq);
249	}
250	chip->ipi_send_single(data, cpu);
251	return 0;
252}
253
254/**
255 * __ipi_send_mask - send an IPI to target Linux SMP CPU(s)
256 * @desc:	pointer to irq_desc of the IRQ
257 * @dest:	dest CPU(s), must be a subset of the mask passed to
258 *		irq_reserve_ipi()
259 *
260 * This function is for architecture or core code to speed up IPI sending. Not
261 * usable from driver code.
262 *
263 * Return: %0 on success or negative error number on failure.
264 */
265int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
266{
267	struct irq_data *data = irq_desc_get_irq_data(desc);
268	struct irq_chip *chip = irq_data_get_irq_chip(data);
269	unsigned int cpu;
270
271#ifdef DEBUG
272	/*
273	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
274	 * Since the callers should be arch or core code which is generally
275	 * trusted, only check for errors when debugging.
276	 */
277	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
278		return -EINVAL;
279#endif
280	if (chip->ipi_send_mask) {
281		chip->ipi_send_mask(data, dest);
282		return 0;
283	}
284
285	if (irq_domain_is_ipi_per_cpu(data->domain)) {
286		unsigned int base = data->irq;
287
288		for_each_cpu(cpu, dest) {
289			unsigned irq = base + cpu - data->common->ipi_offset;
290
291			data = irq_get_irq_data(irq);
292			chip->ipi_send_single(data, cpu);
293		}
294	} else {
295		for_each_cpu(cpu, dest)
296			chip->ipi_send_single(data, cpu);
297	}
298	return 0;
299}
300
301/**
302 * ipi_send_single - Send an IPI to a single CPU
303 * @virq:	Linux IRQ number from irq_reserve_ipi()
304 * @cpu:	destination CPU, must in the destination mask passed to
305 *		irq_reserve_ipi()
306 *
307 * Return: %0 on success or negative error number on failure.
308 */
309int ipi_send_single(unsigned int virq, unsigned int cpu)
310{
311	struct irq_desc *desc = irq_to_desc(virq);
312	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
313	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
314
315	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
316		return -EINVAL;
317
318	return __ipi_send_single(desc, cpu);
319}
320EXPORT_SYMBOL_GPL(ipi_send_single);
321
322/**
323 * ipi_send_mask - Send an IPI to target CPU(s)
324 * @virq:	Linux IRQ number from irq_reserve_ipi()
325 * @dest:	dest CPU(s), must be a subset of the mask passed to
326 *		irq_reserve_ipi()
327 *
328 * Return: %0 on success or negative error number on failure.
329 */
330int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
331{
332	struct irq_desc *desc = irq_to_desc(virq);
333	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
334	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
335
336	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
337		return -EINVAL;
338
339	return __ipi_send_mask(desc, dest);
340}
341EXPORT_SYMBOL_GPL(ipi_send_mask);