1/*
  2 * linux/kernel/irq/ipi.c
  3 *
  4 * Copyright (C) 2015 Imagination Technologies Ltd
  5 * Author: Qais Yousef <qais.yousef@imgtec.com>
  6 *
  7 * This file contains driver APIs to the IPI subsystem.
  8 */
  9
 10#define pr_fmt(fmt) "genirq/ipi: " fmt
 11
 12#include <linux/irqdomain.h>
 13#include <linux/irq.h>
 14
 15/**
 16 * irq_reserve_ipi() - Setup an IPI to destination cpumask
 17 * @domain:	IPI domain
 18 * @dest:	cpumask of cpus which can receive the IPI
 19 *
 20 * Allocate a virq that can be used to send IPI to any CPU in dest mask.
 21 *
 22 * On success it'll return linux irq number and error code on failure
 23 */
 24int irq_reserve_ipi(struct irq_domain *domain,
 25			     const struct cpumask *dest)
 26{
 27	unsigned int nr_irqs, offset;
 28	struct irq_data *data;
 29	int virq, i;
 30
 31	if (!domain ||!irq_domain_is_ipi(domain)) {
 32		pr_warn("Reservation on a non IPI domain\n");
 33		return -EINVAL;
 34	}
 35
 36	if (!cpumask_subset(dest, cpu_possible_mask)) {
 37		pr_warn("Reservation is not in possible_cpu_mask\n");
 38		return -EINVAL;
 39	}
 40
 41	nr_irqs = cpumask_weight(dest);
 42	if (!nr_irqs) {
 43		pr_warn("Reservation for empty destination mask\n");
 44		return -EINVAL;
 45	}
 46
 47	if (irq_domain_is_ipi_single(domain)) {
 48		/*
 49		 * If the underlying implementation uses a single HW irq on
 50		 * all cpus then we only need a single Linux irq number for
 51		 * it. We have no restrictions vs. the destination mask. The
 52		 * underlying implementation can deal with holes nicely.
 53		 */
 54		nr_irqs = 1;
 55		offset = 0;
 56	} else {
 57		unsigned int next;
 58
 59		/*
 60		 * The IPI requires a seperate HW irq on each CPU. We require
 61		 * that the destination mask is consecutive. If an
 62		 * implementation needs to support holes, it can reserve
 63		 * several IPI ranges.
 64		 */
 65		offset = cpumask_first(dest);
 66		/*
 67		 * Find a hole and if found look for another set bit after the
 68		 * hole. For now we don't support this scenario.
 69		 */
 70		next = cpumask_next_zero(offset, dest);
 71		if (next < nr_cpu_ids)
 72			next = cpumask_next(next, dest);
 73		if (next < nr_cpu_ids) {
 74			pr_warn("Destination mask has holes\n");
 75			return -EINVAL;
 76		}
 77	}
 78
 79	virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
 80	if (virq <= 0) {
 81		pr_warn("Can't reserve IPI, failed to alloc descs\n");
 82		return -ENOMEM;
 83	}
 84
 85	virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
 86				       (void *) dest, true, NULL);
 87
 88	if (virq <= 0) {
 89		pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
 90		goto free_descs;
 91	}
 92
 93	for (i = 0; i < nr_irqs; i++) {
 94		data = irq_get_irq_data(virq + i);
 95		cpumask_copy(data->common->affinity, dest);
 96		data->common->ipi_offset = offset;
 97		irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
 98	}
 99	return virq;
100
101free_descs:
102	irq_free_descs(virq, nr_irqs);
103	return -EBUSY;
104}
105
106/**
107 * irq_destroy_ipi() - unreserve an IPI that was previously allocated
108 * @irq:	linux irq number to be destroyed
109 * @dest:	cpumask of cpus which should have the IPI removed
110 *
111 * The IPIs allocated with irq_reserve_ipi() are retuerned to the system
112 * destroying all virqs associated with them.
113 *
114 * Return 0 on success or error code on failure.
115 */
116int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
117{
118	struct irq_data *data = irq_get_irq_data(irq);
119	struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
120	struct irq_domain *domain;
121	unsigned int nr_irqs;
122
123	if (!irq || !data || !ipimask)
124		return -EINVAL;
125
126	domain = data->domain;
127	if (WARN_ON(domain == NULL))
128		return -EINVAL;
129
130	if (!irq_domain_is_ipi(domain)) {
131		pr_warn("Trying to destroy a non IPI domain!\n");
132		return -EINVAL;
133	}
134
135	if (WARN_ON(!cpumask_subset(dest, ipimask)))
136		/*
137		 * Must be destroying a subset of CPUs to which this IPI
138		 * was set up to target
139		 */
140		return -EINVAL;
141
142	if (irq_domain_is_ipi_per_cpu(domain)) {
143		irq = irq + cpumask_first(dest) - data->common->ipi_offset;
144		nr_irqs = cpumask_weight(dest);
145	} else {
146		nr_irqs = 1;
147	}
148
149	irq_domain_free_irqs(irq, nr_irqs);
150	return 0;
151}
152
153/**
154 * ipi_get_hwirq - Get the hwirq associated with an IPI to a cpu
155 * @irq:	linux irq number
156 * @cpu:	the target cpu
157 *
158 * When dealing with coprocessors IPI, we need to inform the coprocessor of
159 * the hwirq it needs to use to receive and send IPIs.
160 *
161 * Returns hwirq value on success and INVALID_HWIRQ on failure.
162 */
163irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
164{
165	struct irq_data *data = irq_get_irq_data(irq);
166	struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
167
168	if (!data || !ipimask || cpu > nr_cpu_ids)
169		return INVALID_HWIRQ;
170
171	if (!cpumask_test_cpu(cpu, ipimask))
172		return INVALID_HWIRQ;
173
174	/*
175	 * Get the real hardware irq number if the underlying implementation
176	 * uses a seperate irq per cpu. If the underlying implementation uses
177	 * a single hardware irq for all cpus then the IPI send mechanism
178	 * needs to take care of the cpu destinations.
179	 */
180	if (irq_domain_is_ipi_per_cpu(data->domain))
181		data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
182
183	return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
184}
185EXPORT_SYMBOL_GPL(ipi_get_hwirq);
186
187static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
188			   const struct cpumask *dest, unsigned int cpu)
189{
190	struct cpumask *ipimask = irq_data_get_affinity_mask(data);
191
192	if (!chip || !ipimask)
193		return -EINVAL;
194
195	if (!chip->ipi_send_single && !chip->ipi_send_mask)
196		return -EINVAL;
197
198	if (cpu > nr_cpu_ids)
199		return -EINVAL;
200
201	if (dest) {
202		if (!cpumask_subset(dest, ipimask))
203			return -EINVAL;
204	} else {
205		if (!cpumask_test_cpu(cpu, ipimask))
206			return -EINVAL;
207	}
208	return 0;
209}
210
211/**
212 * __ipi_send_single - send an IPI to a target Linux SMP CPU
213 * @desc:	pointer to irq_desc of the IRQ
214 * @cpu:	destination CPU, must in the destination mask passed to
215 *		irq_reserve_ipi()
216 *
217 * This function is for architecture or core code to speed up IPI sending. Not
218 * usable from driver code.
219 *
220 * Returns zero on success and negative error number on failure.
221 */
222int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
223{
224	struct irq_data *data = irq_desc_get_irq_data(desc);
225	struct irq_chip *chip = irq_data_get_irq_chip(data);
226
227#ifdef DEBUG
228	/*
229	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
230	 * Since the callers should be arch or core code which is generally
231	 * trusted, only check for errors when debugging.
232	 */
233	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
234		return -EINVAL;
235#endif
236	if (!chip->ipi_send_single) {
237		chip->ipi_send_mask(data, cpumask_of(cpu));
238		return 0;
239	}
240
241	/* FIXME: Store this information in irqdata flags */
242	if (irq_domain_is_ipi_per_cpu(data->domain) &&
243	    cpu != data->common->ipi_offset) {
244		/* use the correct data for that cpu */
245		unsigned irq = data->irq + cpu - data->common->ipi_offset;
246
247		data = irq_get_irq_data(irq);
248	}
249	chip->ipi_send_single(data, cpu);
250	return 0;
251}
252
253/**
254 * ipi_send_mask - send an IPI to target Linux SMP CPU(s)
255 * @desc:	pointer to irq_desc of the IRQ
256 * @dest:	dest CPU(s), must be a subset of the mask passed to
257 *		irq_reserve_ipi()
258 *
259 * This function is for architecture or core code to speed up IPI sending. Not
260 * usable from driver code.
261 *
262 * Returns zero on success and negative error number on failure.
263 */
264int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
265{
266	struct irq_data *data = irq_desc_get_irq_data(desc);
267	struct irq_chip *chip = irq_data_get_irq_chip(data);
268	unsigned int cpu;
269
270#ifdef DEBUG
271	/*
272	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
273	 * Since the callers should be arch or core code which is generally
274	 * trusted, only check for errors when debugging.
275	 */
276	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
277		return -EINVAL;
278#endif
279	if (chip->ipi_send_mask) {
280		chip->ipi_send_mask(data, dest);
281		return 0;
282	}
283
284	if (irq_domain_is_ipi_per_cpu(data->domain)) {
285		unsigned int base = data->irq;
286
287		for_each_cpu(cpu, dest) {
288			unsigned irq = base + cpu - data->common->ipi_offset;
289
290			data = irq_get_irq_data(irq);
291			chip->ipi_send_single(data, cpu);
292		}
293	} else {
294		for_each_cpu(cpu, dest)
295			chip->ipi_send_single(data, cpu);
296	}
297	return 0;
298}
299
300/**
301 * ipi_send_single - Send an IPI to a single CPU
302 * @virq:	linux irq number from irq_reserve_ipi()
303 * @cpu:	destination CPU, must in the destination mask passed to
304 *		irq_reserve_ipi()
305 *
306 * Returns zero on success and negative error number on failure.
307 */
308int ipi_send_single(unsigned int virq, unsigned int cpu)
309{
310	struct irq_desc *desc = irq_to_desc(virq);
311	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
312	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
313
314	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
315		return -EINVAL;
316
317	return __ipi_send_single(desc, cpu);
318}
319EXPORT_SYMBOL_GPL(ipi_send_single);
320
321/**
322 * ipi_send_mask - Send an IPI to target CPU(s)
323 * @virq:	linux irq number from irq_reserve_ipi()
324 * @dest:	dest CPU(s), must be a subset of the mask passed to
325 *		irq_reserve_ipi()
326 *
327 * Returns zero on success and negative error number on failure.
328 */
329int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
330{
331	struct irq_desc *desc = irq_to_desc(virq);
332	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
333	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
334
335	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
336		return -EINVAL;
337
338	return __ipi_send_mask(desc, dest);
339}
340EXPORT_SYMBOL_GPL(ipi_send_mask);