1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2020-2024 Loongson Technologies, Inc.
  4 */
  5
  6#include <linux/cpuhotplug.h>
  7#include <linux/init.h>
  8#include <linux/interrupt.h>
  9#include <linux/irq.h>
 10#include <linux/irqchip.h>
 11#include <linux/irqchip/chained_irq.h>
 12#include <linux/irqdomain.h>
 13#include <linux/kernel.h>
 14#include <linux/msi.h>
 15#include <linux/radix-tree.h>
 16#include <linux/spinlock.h>
 17
 18#include <asm/loongarch.h>
 19#include <asm/setup.h>
 20
 21#include "irq-msi-lib.h"
 22#include "irq-loongson.h"
 23
 24#define VECTORS_PER_REG		64
 25#define IRR_VECTOR_MASK		0xffUL
 26#define IRR_INVALID_MASK	0x80000000UL
 27#define AVEC_MSG_OFFSET		0x100000
 28
 29#ifdef CONFIG_SMP
 30struct pending_list {
 31	struct list_head	head;
 32};
 33
 34static struct cpumask intersect_mask;
 35static DEFINE_PER_CPU(struct pending_list, pending_list);
 36#endif
 37
 38static DEFINE_PER_CPU(struct irq_desc * [NR_VECTORS], irq_map);
 39
 40struct avecintc_chip {
 41	raw_spinlock_t		lock;
 42	struct fwnode_handle	*fwnode;
 43	struct irq_domain	*domain;
 44	struct irq_matrix	*vector_matrix;
 45	phys_addr_t		msi_base_addr;
 46};
 47
 48static struct avecintc_chip loongarch_avec;
 49
 50struct avecintc_data {
 51	struct list_head	entry;
 52	unsigned int		cpu;
 53	unsigned int		vec;
 54	unsigned int		prev_cpu;
 55	unsigned int		prev_vec;
 56	unsigned int		moving;
 57};
 58
 59static inline void avecintc_ack_irq(struct irq_data *d)
 60{
 61}
 62
 63static inline void avecintc_mask_irq(struct irq_data *d)
 64{
 65}
 66
 67static inline void avecintc_unmask_irq(struct irq_data *d)
 68{
 69}
 70
 71#ifdef CONFIG_SMP
 72static inline void pending_list_init(int cpu)
 73{
 74	struct pending_list *plist = per_cpu_ptr(&pending_list, cpu);
 75
 76	INIT_LIST_HEAD(&plist->head);
 77}
 78
 79static void avecintc_sync(struct avecintc_data *adata)
 80{
 81	struct pending_list *plist;
 82
 83	if (cpu_online(adata->prev_cpu)) {
 84		plist = per_cpu_ptr(&pending_list, adata->prev_cpu);
 85		list_add_tail(&adata->entry, &plist->head);
 86		adata->moving = 1;
 87		mp_ops.send_ipi_single(adata->prev_cpu, ACTION_CLEAR_VECTOR);
 88	}
 89}
 90
 91static int avecintc_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force)
 92{
 93	int cpu, ret, vector;
 94	struct avecintc_data *adata;
 95
 96	scoped_guard(raw_spinlock, &loongarch_avec.lock) {
 97		adata = irq_data_get_irq_chip_data(data);
 98
 99		if (adata->moving)
100			return -EBUSY;
101
102		if (cpu_online(adata->cpu) && cpumask_test_cpu(adata->cpu, dest))
103			return 0;
104
105		cpumask_and(&intersect_mask, dest, cpu_online_mask);
106
107		ret = irq_matrix_alloc(loongarch_avec.vector_matrix, &intersect_mask, false, &cpu);
108		if (ret < 0)
109			return ret;
110
111		vector = ret;
112		adata->cpu = cpu;
113		adata->vec = vector;
114		per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(data);
115		avecintc_sync(adata);
116	}
117
118	irq_data_update_effective_affinity(data, cpumask_of(cpu));
119
120	return IRQ_SET_MASK_OK;
121}
122
123static int avecintc_cpu_online(unsigned int cpu)
124{
125	if (!loongarch_avec.vector_matrix)
126		return 0;
127
128	guard(raw_spinlock)(&loongarch_avec.lock);
129
130	irq_matrix_online(loongarch_avec.vector_matrix);
131
132	pending_list_init(cpu);
133
134	return 0;
135}
136
137static int avecintc_cpu_offline(unsigned int cpu)
138{
139	struct pending_list *plist = per_cpu_ptr(&pending_list, cpu);
140
141	if (!loongarch_avec.vector_matrix)
142		return 0;
143
144	guard(raw_spinlock)(&loongarch_avec.lock);
145
146	if (!list_empty(&plist->head))
147		pr_warn("CPU#%d vector is busy\n", cpu);
148
149	irq_matrix_offline(loongarch_avec.vector_matrix);
150
151	return 0;
152}
153
154void complete_irq_moving(void)
155{
156	struct pending_list *plist = this_cpu_ptr(&pending_list);
157	struct avecintc_data *adata, *tdata;
158	int cpu, vector, bias;
159	uint64_t isr;
160
161	guard(raw_spinlock)(&loongarch_avec.lock);
162
163	list_for_each_entry_safe(adata, tdata, &plist->head, entry) {
164		cpu = adata->prev_cpu;
165		vector = adata->prev_vec;
166		bias = vector / VECTORS_PER_REG;
167		switch (bias) {
168		case 0:
169			isr = csr_read64(LOONGARCH_CSR_ISR0);
170			break;
171		case 1:
172			isr = csr_read64(LOONGARCH_CSR_ISR1);
173			break;
174		case 2:
175			isr = csr_read64(LOONGARCH_CSR_ISR2);
176			break;
177		case 3:
178			isr = csr_read64(LOONGARCH_CSR_ISR3);
179			break;
180		}
181
182		if (isr & (1UL << (vector % VECTORS_PER_REG))) {
183			mp_ops.send_ipi_single(cpu, ACTION_CLEAR_VECTOR);
184			continue;
185		}
186		list_del(&adata->entry);
187		irq_matrix_free(loongarch_avec.vector_matrix, cpu, vector, false);
188		this_cpu_write(irq_map[vector], NULL);
189		adata->moving = 0;
190		adata->prev_cpu = adata->cpu;
191		adata->prev_vec = adata->vec;
192	}
193}
194#endif
195
196static void avecintc_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
197{
198	struct avecintc_data *adata = irq_data_get_irq_chip_data(d);
199
200	msg->address_hi = 0x0;
201	msg->address_lo = (loongarch_avec.msi_base_addr | (adata->vec & 0xff) << 4)
202			  | ((cpu_logical_map(adata->cpu & 0xffff)) << 12);
203	msg->data = 0x0;
204}
205
206static struct irq_chip avec_irq_controller = {
207	.name			= "AVECINTC",
208	.irq_ack		= avecintc_ack_irq,
209	.irq_mask		= avecintc_mask_irq,
210	.irq_unmask		= avecintc_unmask_irq,
211#ifdef CONFIG_SMP
212	.irq_set_affinity	= avecintc_set_affinity,
213#endif
214	.irq_compose_msi_msg	= avecintc_compose_msi_msg,
215};
216
217static void avecintc_irq_dispatch(struct irq_desc *desc)
218{
219	struct irq_chip *chip = irq_desc_get_chip(desc);
220	struct irq_desc *d;
221
222	chained_irq_enter(chip, desc);
223
224	while (true) {
225		unsigned long vector = csr_read64(LOONGARCH_CSR_IRR);
226		if (vector & IRR_INVALID_MASK)
227			break;
228
229		vector &= IRR_VECTOR_MASK;
230
231		d = this_cpu_read(irq_map[vector]);
232		if (d) {
233			generic_handle_irq_desc(d);
234		} else {
235			spurious_interrupt();
236			pr_warn("Unexpected IRQ occurs on CPU#%d [vector %ld]\n", smp_processor_id(), vector);
237		}
238	}
239
240	chained_irq_exit(chip, desc);
241}
242
243static int avecintc_alloc_vector(struct irq_data *irqd, struct avecintc_data *adata)
244{
245	int cpu, ret;
246
247	guard(raw_spinlock_irqsave)(&loongarch_avec.lock);
248
249	ret = irq_matrix_alloc(loongarch_avec.vector_matrix, cpu_online_mask, false, &cpu);
250	if (ret < 0)
251		return ret;
252
253	adata->prev_cpu = adata->cpu = cpu;
254	adata->prev_vec = adata->vec = ret;
255	per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(irqd);
256
257	return 0;
258}
259
260static int avecintc_domain_alloc(struct irq_domain *domain, unsigned int virq,
261				 unsigned int nr_irqs, void *arg)
262{
263	for (unsigned int i = 0; i < nr_irqs; i++) {
264		struct irq_data *irqd = irq_domain_get_irq_data(domain, virq + i);
265		struct avecintc_data *adata = kzalloc(sizeof(*adata), GFP_KERNEL);
266		int ret;
267
268		if (!adata)
269			return -ENOMEM;
270
271		ret = avecintc_alloc_vector(irqd, adata);
272		if (ret < 0) {
273			kfree(adata);
274			return ret;
275		}
276
277		irq_domain_set_info(domain, virq + i, virq + i, &avec_irq_controller,
278				    adata, handle_edge_irq, NULL, NULL);
279		irqd_set_single_target(irqd);
280		irqd_set_affinity_on_activate(irqd);
281	}
282
283	return 0;
284}
285
286static void avecintc_free_vector(struct irq_data *irqd, struct avecintc_data *adata)
287{
288	guard(raw_spinlock_irqsave)(&loongarch_avec.lock);
289
290	per_cpu(irq_map, adata->cpu)[adata->vec] = NULL;
291	irq_matrix_free(loongarch_avec.vector_matrix, adata->cpu, adata->vec, false);
292
293#ifdef CONFIG_SMP
294	if (!adata->moving)
295		return;
296
297	per_cpu(irq_map, adata->prev_cpu)[adata->prev_vec] = NULL;
298	irq_matrix_free(loongarch_avec.vector_matrix, adata->prev_cpu, adata->prev_vec, false);
299	list_del_init(&adata->entry);
300#endif
301}
302
303static void avecintc_domain_free(struct irq_domain *domain, unsigned int virq,
304				 unsigned int nr_irqs)
305{
306	for (unsigned int i = 0; i < nr_irqs; i++) {
307		struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
308
309		if (d) {
310			struct avecintc_data *adata = irq_data_get_irq_chip_data(d);
311
312			avecintc_free_vector(d, adata);
313			irq_domain_reset_irq_data(d);
314			kfree(adata);
315		}
316	}
317}
318
319static const struct irq_domain_ops avecintc_domain_ops = {
320	.alloc		= avecintc_domain_alloc,
321	.free		= avecintc_domain_free,
322	.select		= msi_lib_irq_domain_select,
323};
324
325static int __init irq_matrix_init(void)
326{
327	loongarch_avec.vector_matrix = irq_alloc_matrix(NR_VECTORS, 0, NR_VECTORS);
328	if (!loongarch_avec.vector_matrix)
329		return -ENOMEM;
330
331	for (int i = 0; i < NR_LEGACY_VECTORS; i++)
332		irq_matrix_assign_system(loongarch_avec.vector_matrix, i, false);
333
334	irq_matrix_online(loongarch_avec.vector_matrix);
335
336	return 0;
337}
338
339static int __init avecintc_init(struct irq_domain *parent)
340{
341	int ret, parent_irq;
342	unsigned long value;
343
344	raw_spin_lock_init(&loongarch_avec.lock);
345
346	loongarch_avec.fwnode = irq_domain_alloc_named_fwnode("AVECINTC");
347	if (!loongarch_avec.fwnode) {
348		pr_err("Unable to allocate domain handle\n");
349		ret = -ENOMEM;
350		goto out;
351	}
352
353	loongarch_avec.domain = irq_domain_create_tree(loongarch_avec.fwnode,
354						       &avecintc_domain_ops, NULL);
355	if (!loongarch_avec.domain) {
356		pr_err("Unable to create IRQ domain\n");
357		ret = -ENOMEM;
358		goto out_free_handle;
359	}
360
361	parent_irq = irq_create_mapping(parent, INT_AVEC);
362	if (!parent_irq) {
363		pr_err("Failed to mapping hwirq\n");
364		ret = -EINVAL;
365		goto out_remove_domain;
366	}
367
368	ret = irq_matrix_init();
369	if (ret < 0) {
370		pr_err("Failed to init irq matrix\n");
371		goto out_remove_domain;
372	}
373	irq_set_chained_handler_and_data(parent_irq, avecintc_irq_dispatch, NULL);
374
375#ifdef CONFIG_SMP
376	pending_list_init(0);
377	cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_AVECINTC_STARTING,
378				  "irqchip/loongarch/avecintc:starting",
379				  avecintc_cpu_online, avecintc_cpu_offline);
380#endif
381	value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
382	value |= IOCSR_MISC_FUNC_AVEC_EN;
383	iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
384
385	return ret;
386
387out_remove_domain:
388	irq_domain_remove(loongarch_avec.domain);
389out_free_handle:
390	irq_domain_free_fwnode(loongarch_avec.fwnode);
391out:
392	return ret;
393}
394
395static int __init pch_msi_parse_madt(union acpi_subtable_headers *header,
396				     const unsigned long end)
397{
398	struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header;
399
400	loongarch_avec.msi_base_addr = pchmsi_entry->msg_address - AVEC_MSG_OFFSET;
401
402	return pch_msi_acpi_init_avec(loongarch_avec.domain);
403}
404
405static inline int __init acpi_cascade_irqdomain_init(void)
406{
407	return acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1);
408}
409
410int __init avecintc_acpi_init(struct irq_domain *parent)
411{
412	int ret = avecintc_init(parent);
413	if (ret < 0) {
414		pr_err("Failed to init IRQ domain\n");
415		return ret;
416	}
417
418	ret = acpi_cascade_irqdomain_init();
419	if (ret < 0) {
420		pr_err("Failed to init cascade IRQ domain\n");
421		return ret;
422	}
423
424	return ret;
425}