1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
  4 * Copyright (C) 2022 Ventana Micro Systems Inc.
  5 */
  6
  7#define pr_fmt(fmt) "riscv-imsic: " fmt
  8#include <linux/acpi.h>
  9#include <linux/bitmap.h>
 10#include <linux/cpu.h>
 11#include <linux/interrupt.h>
 12#include <linux/io.h>
 13#include <linux/irq.h>
 14#include <linux/irqchip.h>
 15#include <linux/irqdomain.h>
 16#include <linux/module.h>
 17#include <linux/msi.h>
 18#include <linux/pci.h>
 19#include <linux/platform_device.h>
 20#include <linux/spinlock.h>
 21#include <linux/smp.h>
 22
 23#include "irq-riscv-imsic-state.h"
 24
 25static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index,
 26				phys_addr_t *out_msi_pa)
 27{
 28	struct imsic_global_config *global;
 29	struct imsic_local_config *local;
 30
 31	global = &imsic->global;
 32	local = per_cpu_ptr(global->local, cpu);
 33
 34	if (BIT(global->guest_index_bits) <= guest_index)
 35		return false;
 36
 37	if (out_msi_pa)
 38		*out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ);
 39
 40	return true;
 41}
 42
 43static void imsic_irq_mask(struct irq_data *d)
 44{
 45	imsic_vector_mask(irq_data_get_irq_chip_data(d));
 46}
 47
 48static void imsic_irq_unmask(struct irq_data *d)
 49{
 50	imsic_vector_unmask(irq_data_get_irq_chip_data(d));
 51}
 52
 53static int imsic_irq_retrigger(struct irq_data *d)
 54{
 55	struct imsic_vector *vec = irq_data_get_irq_chip_data(d);
 56	struct imsic_local_config *local;
 57
 58	if (WARN_ON(!vec))
 59		return -ENOENT;
 60
 61	local = per_cpu_ptr(imsic->global.local, vec->cpu);
 62	writel_relaxed(vec->local_id, local->msi_va);
 63	return 0;
 64}
 65
 66static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg)
 67{
 68	phys_addr_t msi_addr;
 69
 70	if (WARN_ON(!vec))
 71		return;
 72
 73	if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr)))
 74		return;
 75
 76	msg->address_hi = upper_32_bits(msi_addr);
 77	msg->address_lo = lower_32_bits(msi_addr);
 78	msg->data = vec->local_id;
 79}
 80
 81static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg)
 82{
 83	imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg);
 84}
 85
 86#ifdef CONFIG_SMP
 87static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec)
 88{
 89	struct msi_msg msg = { };
 90
 91	imsic_irq_compose_vector_msg(vec, &msg);
 92	irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg);
 93}
 94
 95static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 96				  bool force)
 97{
 98	struct imsic_vector *old_vec, *new_vec;
 99	struct irq_data *pd = d->parent_data;
100
101	old_vec = irq_data_get_irq_chip_data(pd);
102	if (WARN_ON(!old_vec))
103		return -ENOENT;
104
105	/* If old vector cpu belongs to the target cpumask then do nothing */
106	if (cpumask_test_cpu(old_vec->cpu, mask_val))
107		return IRQ_SET_MASK_OK_DONE;
108
109	/* If move is already in-flight then return failure */
110	if (imsic_vector_get_move(old_vec))
111		return -EBUSY;
112
113	/* Get a new vector on the desired set of CPUs */
114	new_vec = imsic_vector_alloc(old_vec->hwirq, mask_val);
115	if (!new_vec)
116		return -ENOSPC;
117
118	/* Point device to the new vector */
119	imsic_msi_update_msg(d, new_vec);
120
121	/* Update irq descriptors with the new vector */
122	pd->chip_data = new_vec;
123
124	/* Update effective affinity of parent irq data */
125	irq_data_update_effective_affinity(pd, cpumask_of(new_vec->cpu));
126
127	/* Move state of the old vector to the new vector */
128	imsic_vector_move(old_vec, new_vec);
129
130	return IRQ_SET_MASK_OK_DONE;
131}
132#endif
133
134static struct irq_chip imsic_irq_base_chip = {
135	.name			= "IMSIC",
136	.irq_mask		= imsic_irq_mask,
137	.irq_unmask		= imsic_irq_unmask,
138	.irq_retrigger		= imsic_irq_retrigger,
139	.irq_compose_msi_msg	= imsic_irq_compose_msg,
140	.flags			= IRQCHIP_SKIP_SET_WAKE |
141				  IRQCHIP_MASK_ON_SUSPEND,
142};
143
144static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
145				  unsigned int nr_irqs, void *args)
146{
147	struct imsic_vector *vec;
148
149	/* Multi-MSI is not supported yet. */
150	if (nr_irqs > 1)
151		return -EOPNOTSUPP;
152
153	vec = imsic_vector_alloc(virq, cpu_online_mask);
154	if (!vec)
155		return -ENOSPC;
156
157	irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec,
158			    handle_simple_irq, NULL, NULL);
159	irq_set_noprobe(virq);
160	irq_set_affinity(virq, cpu_online_mask);
161	irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu));
162
163	return 0;
164}
165
166static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
167				  unsigned int nr_irqs)
168{
169	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
170
171	imsic_vector_free(irq_data_get_irq_chip_data(d));
172	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
173}
174
175static int imsic_irq_domain_select(struct irq_domain *domain, struct irq_fwspec *fwspec,
176				   enum irq_domain_bus_token bus_token)
177{
178	const struct msi_parent_ops *ops = domain->msi_parent_ops;
179	u32 busmask = BIT(bus_token);
180
181	if (fwspec->fwnode != domain->fwnode || fwspec->param_count != 0)
182		return 0;
183
184	/* Handle pure domain searches */
185	if (bus_token == ops->bus_select_token)
186		return 1;
187
188	return !!(ops->bus_select_mask & busmask);
189}
190
191#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
192static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d,
193				 struct irq_data *irqd, int ind)
194{
195	if (!irqd) {
196		imsic_vector_debug_show_summary(m, ind);
197		return;
198	}
199
200	imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind);
201}
202#endif
203
204static const struct irq_domain_ops imsic_base_domain_ops = {
205	.alloc		= imsic_irq_domain_alloc,
206	.free		= imsic_irq_domain_free,
207	.select		= imsic_irq_domain_select,
208#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
209	.debug_show	= imsic_irq_debug_show,
210#endif
211};
212
213#ifdef CONFIG_RISCV_IMSIC_PCI
214
215static void imsic_pci_mask_irq(struct irq_data *d)
216{
217	pci_msi_mask_irq(d);
218	irq_chip_mask_parent(d);
219}
220
221static void imsic_pci_unmask_irq(struct irq_data *d)
222{
223	irq_chip_unmask_parent(d);
224	pci_msi_unmask_irq(d);
225}
226
227#define MATCH_PCI_MSI		BIT(DOMAIN_BUS_PCI_MSI)
228
229#else
230
231#define MATCH_PCI_MSI		0
232
233#endif
234
235static bool imsic_init_dev_msi_info(struct device *dev,
236				    struct irq_domain *domain,
237				    struct irq_domain *real_parent,
238				    struct msi_domain_info *info)
239{
240	const struct msi_parent_ops *pops = real_parent->msi_parent_ops;
241
242	/* MSI parent domain specific settings */
243	switch (real_parent->bus_token) {
244	case DOMAIN_BUS_NEXUS:
245		if (WARN_ON_ONCE(domain != real_parent))
246			return false;
247#ifdef CONFIG_SMP
248		info->chip->irq_set_affinity = imsic_irq_set_affinity;
249#endif
250		break;
251	default:
252		WARN_ON_ONCE(1);
253		return false;
254	}
255
256	/* Is the target supported? */
257	switch (info->bus_token) {
258#ifdef CONFIG_RISCV_IMSIC_PCI
259	case DOMAIN_BUS_PCI_DEVICE_MSI:
260	case DOMAIN_BUS_PCI_DEVICE_MSIX:
261		info->chip->irq_mask = imsic_pci_mask_irq;
262		info->chip->irq_unmask = imsic_pci_unmask_irq;
263		break;
264#endif
265	case DOMAIN_BUS_DEVICE_MSI:
266		/*
267		 * Per-device MSI should never have any MSI feature bits
268		 * set. It's sole purpose is to create a dumb interrupt
269		 * chip which has a device specific irq_write_msi_msg()
270		 * callback.
271		 */
272		if (WARN_ON_ONCE(info->flags))
273			return false;
274
275		/* Core managed MSI descriptors */
276		info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS |
277			       MSI_FLAG_FREE_MSI_DESCS;
278		break;
279	case DOMAIN_BUS_WIRED_TO_MSI:
280		break;
281	default:
282		WARN_ON_ONCE(1);
283		return false;
284	}
285
286	/* Use hierarchial chip operations re-trigger */
287	info->chip->irq_retrigger = irq_chip_retrigger_hierarchy;
288
289	/*
290	 * Mask out the domain specific MSI feature flags which are not
291	 * supported by the real parent.
292	 */
293	info->flags &= pops->supported_flags;
294
295	/* Enforce the required flags */
296	info->flags |= pops->required_flags;
297
298	return true;
299}
300
301#define MATCH_PLATFORM_MSI		BIT(DOMAIN_BUS_PLATFORM_MSI)
302
303static const struct msi_parent_ops imsic_msi_parent_ops = {
304	.supported_flags	= MSI_GENERIC_FLAGS_MASK |
305				  MSI_FLAG_PCI_MSIX,
306	.required_flags		= MSI_FLAG_USE_DEF_DOM_OPS |
307				  MSI_FLAG_USE_DEF_CHIP_OPS,
308	.bus_select_token	= DOMAIN_BUS_NEXUS,
309	.bus_select_mask	= MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
310	.init_dev_msi_info	= imsic_init_dev_msi_info,
311};
312
313int imsic_irqdomain_init(void)
314{
315	struct imsic_global_config *global;
316
317	if (!imsic || !imsic->fwnode) {
318		pr_err("early driver not probed\n");
319		return -ENODEV;
320	}
321
322	if (imsic->base_domain) {
323		pr_err("%pfwP: irq domain already created\n", imsic->fwnode);
324		return -ENODEV;
325	}
326
327	/* Create Base IRQ domain */
328	imsic->base_domain = irq_domain_create_tree(imsic->fwnode,
329						    &imsic_base_domain_ops, imsic);
330	if (!imsic->base_domain) {
331		pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode);
332		return -ENOMEM;
333	}
334	imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT;
335	imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops;
336
337	irq_domain_update_bus_token(imsic->base_domain, DOMAIN_BUS_NEXUS);
338
339	global = &imsic->global;
340	pr_info("%pfwP:  hart-index-bits: %d,  guest-index-bits: %d\n",
341		imsic->fwnode, global->hart_index_bits, global->guest_index_bits);
342	pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n",
343		imsic->fwnode, global->group_index_bits, global->group_index_shift);
344	pr_info("%pfwP: per-CPU IDs %d at base address %pa\n",
345		imsic->fwnode, global->nr_ids, &global->base_addr);
346	pr_info("%pfwP: total %d interrupts available\n",
347		imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1));
348
349	return 0;
350}
351
352static int imsic_platform_probe_common(struct fwnode_handle *fwnode)
353{
354	if (imsic && imsic->fwnode != fwnode) {
355		pr_err("%pfwP: fwnode mismatch\n", fwnode);
356		return -ENODEV;
357	}
358
359	return imsic_irqdomain_init();
360}
361
362static int imsic_platform_dt_probe(struct platform_device *pdev)
363{
364	return imsic_platform_probe_common(pdev->dev.fwnode);
365}
366
367#ifdef CONFIG_ACPI
368
369/*
370 *  On ACPI based systems, PCI enumeration happens early during boot in
371 *  acpi_scan_init(). PCI enumeration expects MSI domain setup before
372 *  it calls pci_set_msi_domain(). Hence, unlike in DT where
373 *  imsic-platform drive probe happens late during boot, ACPI based
374 *  systems need to setup the MSI domain early.
375 */
376int imsic_platform_acpi_probe(struct fwnode_handle *fwnode)
377{
378	return imsic_platform_probe_common(fwnode);
379}
380
381#endif
382
383static const struct of_device_id imsic_platform_match[] = {
384	{ .compatible = "riscv,imsics" },
385	{}
386};
387
388static struct platform_driver imsic_platform_driver = {
389	.driver = {
390		.name		= "riscv-imsic",
391		.of_match_table	= imsic_platform_match,
392	},
393	.probe = imsic_platform_dt_probe,
394};
395builtin_platform_driver(imsic_platform_driver);