1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
  4 * Copyright (C) 2022 Ventana Micro Systems Inc.
  5 */
  6
  7#define pr_fmt(fmt) "riscv-imsic: " fmt
  8#include <linux/acpi.h>
  9#include <linux/cpu.h>
 10#include <linux/interrupt.h>
 11#include <linux/io.h>
 12#include <linux/irq.h>
 13#include <linux/irqchip.h>
 14#include <linux/irqchip/chained_irq.h>
 15#include <linux/irqchip/riscv-imsic.h>
 16#include <linux/module.h>
 17#include <linux/pci.h>
 18#include <linux/spinlock.h>
 19#include <linux/smp.h>
 20
 21#include "irq-riscv-imsic-state.h"
 22
 23static int imsic_parent_irq;
 24
 25#ifdef CONFIG_SMP
 26static void imsic_ipi_send(unsigned int cpu)
 27{
 28	struct imsic_local_config *local = per_cpu_ptr(imsic->global.local, cpu);
 29
 30	writel_relaxed(IMSIC_IPI_ID, local->msi_va);
 31}
 32
 33static void imsic_ipi_starting_cpu(void)
 34{
 35	/* Enable IPIs for current CPU. */
 36	__imsic_id_set_enable(IMSIC_IPI_ID);
 37}
 38
 39static void imsic_ipi_dying_cpu(void)
 40{
 41	/* Disable IPIs for current CPU. */
 42	__imsic_id_clear_enable(IMSIC_IPI_ID);
 43}
 44
 45static int __init imsic_ipi_domain_init(void)
 46{
 47	int virq;
 48
 49	/* Create IMSIC IPI multiplexing */
 50	virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send);
 51	if (virq <= 0)
 52		return virq < 0 ? virq : -ENOMEM;
 53
 54	/* Set vIRQ range */
 55	riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI);
 56
 57	/* Announce that IMSIC is providing IPIs */
 58	pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID);
 59
 60	return 0;
 61}
 62#else
 63static void imsic_ipi_starting_cpu(void) { }
 64static void imsic_ipi_dying_cpu(void) { }
 65static int __init imsic_ipi_domain_init(void) { return 0; }
 66#endif
 67
 68/*
 69 * To handle an interrupt, we read the TOPEI CSR and write zero in one
 70 * instruction. If TOPEI CSR is non-zero then we translate TOPEI.ID to
 71 * Linux interrupt number and let Linux IRQ subsystem handle it.
 72 */
 73static void imsic_handle_irq(struct irq_desc *desc)
 74{
 75	struct irq_chip *chip = irq_desc_get_chip(desc);
 76	int err, cpu = smp_processor_id();
 77	struct imsic_vector *vec;
 78	unsigned long local_id;
 79
 80	chained_irq_enter(chip, desc);
 81
 82	while ((local_id = csr_swap(CSR_TOPEI, 0))) {
 83		local_id >>= TOPEI_ID_SHIFT;
 84
 85		if (local_id == IMSIC_IPI_ID) {
 86			if (IS_ENABLED(CONFIG_SMP))
 87				ipi_mux_process();
 88			continue;
 89		}
 90
 91		if (unlikely(!imsic->base_domain))
 92			continue;
 93
 94		vec = imsic_vector_from_local_id(cpu, local_id);
 95		if (!vec) {
 96			pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id);
 97			continue;
 98		}
 99
100		err = generic_handle_domain_irq(imsic->base_domain, vec->hwirq);
101		if (unlikely(err))
102			pr_warn_ratelimited("hwirq 0x%x mapping not found\n", vec->hwirq);
103	}
104
105	chained_irq_exit(chip, desc);
106}
107
108static int imsic_starting_cpu(unsigned int cpu)
109{
110	/* Mark per-CPU IMSIC state as online */
111	imsic_state_online();
112
113	/* Enable per-CPU parent interrupt */
114	enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq));
115
116	/* Setup IPIs */
117	imsic_ipi_starting_cpu();
118
119	/*
120	 * Interrupts identities might have been enabled/disabled while
121	 * this CPU was not running so sync-up local enable/disable state.
122	 */
123	imsic_local_sync_all();
124
125	/* Enable local interrupt delivery */
126	imsic_local_delivery(true);
127
128	return 0;
129}
130
131static int imsic_dying_cpu(unsigned int cpu)
132{
133	/* Cleanup IPIs */
134	imsic_ipi_dying_cpu();
135
136	/* Mark per-CPU IMSIC state as offline */
137	imsic_state_offline();
138
139	return 0;
140}
141
142static int __init imsic_early_probe(struct fwnode_handle *fwnode)
143{
144	struct irq_domain *domain;
145	int rc;
146
147	/* Find parent domain and register chained handler */
148	domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY);
149	if (!domain) {
150		pr_err("%pfwP: Failed to find INTC domain\n", fwnode);
151		return -ENOENT;
152	}
153	imsic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT);
154	if (!imsic_parent_irq) {
155		pr_err("%pfwP: Failed to create INTC mapping\n", fwnode);
156		return -ENOENT;
157	}
158
159	/* Initialize IPI domain */
160	rc = imsic_ipi_domain_init();
161	if (rc) {
162		pr_err("%pfwP: Failed to initialize IPI domain\n", fwnode);
163		return rc;
164	}
165
166	/* Setup chained handler to the parent domain interrupt */
167	irq_set_chained_handler(imsic_parent_irq, imsic_handle_irq);
168
169	/*
170	 * Setup cpuhp state (must be done after setting imsic_parent_irq)
171	 *
172	 * Don't disable per-CPU IMSIC file when CPU goes offline
173	 * because this affects IPI and the masking/unmasking of
174	 * virtual IPIs is done via generic IPI-Mux
175	 */
176	cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting",
177			  imsic_starting_cpu, imsic_dying_cpu);
178
179	return 0;
180}
181
182static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent)
183{
184	struct fwnode_handle *fwnode = &node->fwnode;
185	int rc;
186
187	/* Setup IMSIC state */
188	rc = imsic_setup_state(fwnode, NULL);
189	if (rc) {
190		pr_err("%pfwP: failed to setup state (error %d)\n", fwnode, rc);
191		return rc;
192	}
193
194	/* Do early setup of IPIs */
195	rc = imsic_early_probe(fwnode);
196	if (rc)
197		return rc;
198
199	/* Ensure that OF platform device gets probed */
200	of_node_clear_flag(node, OF_POPULATED);
201	return 0;
202}
203
204IRQCHIP_DECLARE(riscv_imsic, "riscv,imsics", imsic_early_dt_init);
205
206#ifdef CONFIG_ACPI
207
208static struct fwnode_handle *imsic_acpi_fwnode;
209
210struct fwnode_handle *imsic_acpi_get_fwnode(struct device *dev)
211{
212	return imsic_acpi_fwnode;
213}
214
215static int __init imsic_early_acpi_init(union acpi_subtable_headers *header,
216					const unsigned long end)
217{
218	struct acpi_madt_imsic *imsic = (struct acpi_madt_imsic *)header;
219	int rc;
220
221	imsic_acpi_fwnode = irq_domain_alloc_named_fwnode("imsic");
222	if (!imsic_acpi_fwnode) {
223		pr_err("unable to allocate IMSIC FW node\n");
224		return -ENOMEM;
225	}
226
227	/* Setup IMSIC state */
228	rc = imsic_setup_state(imsic_acpi_fwnode, imsic);
229	if (rc) {
230		pr_err("%pfwP: failed to setup state (error %d)\n", imsic_acpi_fwnode, rc);
231		return rc;
232	}
233
234	/* Do early setup of IMSIC state and IPIs */
235	rc = imsic_early_probe(imsic_acpi_fwnode);
236	if (rc) {
237		irq_domain_free_fwnode(imsic_acpi_fwnode);
238		imsic_acpi_fwnode = NULL;
239		return rc;
240	}
241
242	rc = imsic_platform_acpi_probe(imsic_acpi_fwnode);
243
244#ifdef CONFIG_PCI
245	if (!rc)
246		pci_msi_register_fwnode_provider(&imsic_acpi_get_fwnode);
247#endif
248
249	if (rc)
250		pr_err("%pfwP: failed to register IMSIC for MSI functionality (error %d)\n",
251		       imsic_acpi_fwnode, rc);
252
253	/*
254	 * Even if imsic_platform_acpi_probe() fails, the IPI part of IMSIC can
255	 * continue to work. So, no need to return failure. This is similar to
256	 * DT where IPI works but MSI probe fails for some reason.
257	 */
258	return 0;
259}
260
261IRQCHIP_ACPI_DECLARE(riscv_imsic, ACPI_MADT_TYPE_IMSIC, NULL,
262		     1, imsic_early_acpi_init);
263#endif