Loading...
1/*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 */
14
15#include <linux/module.h>
16#include <linux/seq_file.h>
17#include <linux/interrupt.h>
18#include <linux/irq.h>
19#include <linux/kernel_stat.h>
20#include <linux/uaccess.h>
21#include <hv/drv_pcie_rc_intf.h>
22#include <arch/spr_def.h>
23#include <asm/traps.h>
24#include <linux/perf_event.h>
25
26/* Bit-flag stored in irq_desc->chip_data to indicate HW-cleared irqs. */
27#define IS_HW_CLEARED 1
28
29/*
30 * The set of interrupts we enable for arch_local_irq_enable().
31 * This is initialized to have just a single interrupt that the kernel
32 * doesn't actually use as a sentinel. During kernel init,
33 * interrupts are added as the kernel gets prepared to support them.
34 * NOTE: we could probably initialize them all statically up front.
35 */
36DEFINE_PER_CPU(unsigned long long, interrupts_enabled_mask) =
37 INITIAL_INTERRUPTS_ENABLED;
38EXPORT_PER_CPU_SYMBOL(interrupts_enabled_mask);
39
40/* Define per-tile device interrupt statistics state. */
41DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp;
42EXPORT_PER_CPU_SYMBOL(irq_stat);
43
44/*
45 * Define per-tile irq disable mask; the hardware/HV only has a single
46 * mask that we use to implement both masking and disabling.
47 */
48static DEFINE_PER_CPU(unsigned long, irq_disable_mask)
49 ____cacheline_internodealigned_in_smp;
50
51/*
52 * Per-tile IRQ nesting depth. Used to make sure we enable newly
53 * enabled IRQs before exiting the outermost interrupt.
54 */
55static DEFINE_PER_CPU(int, irq_depth);
56
57#if CHIP_HAS_IPI()
58/* Use SPRs to manipulate device interrupts. */
59#define mask_irqs(irq_mask) __insn_mtspr(SPR_IPI_MASK_SET_K, irq_mask)
60#define unmask_irqs(irq_mask) __insn_mtspr(SPR_IPI_MASK_RESET_K, irq_mask)
61#define clear_irqs(irq_mask) __insn_mtspr(SPR_IPI_EVENT_RESET_K, irq_mask)
62#else
63/* Use HV to manipulate device interrupts. */
64#define mask_irqs(irq_mask) hv_disable_intr(irq_mask)
65#define unmask_irqs(irq_mask) hv_enable_intr(irq_mask)
66#define clear_irqs(irq_mask) hv_clear_intr(irq_mask)
67#endif
68
69/*
70 * The interrupt handling path, implemented in terms of HV interrupt
71 * emulation on TILEPro, and IPI hardware on TILE-Gx.
72 * Entered with interrupts disabled.
73 */
74void tile_dev_intr(struct pt_regs *regs, int intnum)
75{
76 int depth = __this_cpu_inc_return(irq_depth);
77 unsigned long original_irqs;
78 unsigned long remaining_irqs;
79 struct pt_regs *old_regs;
80
81#if CHIP_HAS_IPI()
82 /*
83 * Pending interrupts are listed in an SPR. We might be
84 * nested, so be sure to only handle irqs that weren't already
85 * masked by a previous interrupt. Then, mask out the ones
86 * we're going to handle.
87 */
88 unsigned long masked = __insn_mfspr(SPR_IPI_MASK_K);
89 original_irqs = __insn_mfspr(SPR_IPI_EVENT_K) & ~masked;
90 __insn_mtspr(SPR_IPI_MASK_SET_K, original_irqs);
91#else
92 /*
93 * Hypervisor performs the equivalent of the Gx code above and
94 * then puts the pending interrupt mask into a system save reg
95 * for us to find.
96 */
97 original_irqs = __insn_mfspr(SPR_SYSTEM_SAVE_K_3);
98#endif
99 remaining_irqs = original_irqs;
100
101 /* Track time spent here in an interrupt context. */
102 old_regs = set_irq_regs(regs);
103 irq_enter();
104
105#ifdef CONFIG_DEBUG_STACKOVERFLOW
106 /* Debugging check for stack overflow: less than 1/8th stack free? */
107 {
108 long sp = stack_pointer - (long) current_thread_info();
109 if (unlikely(sp < (sizeof(struct thread_info) + STACK_WARN))) {
110 pr_emerg("%s: stack overflow: %ld\n",
111 __func__, sp - sizeof(struct thread_info));
112 dump_stack();
113 }
114 }
115#endif
116 while (remaining_irqs) {
117 unsigned long irq = __ffs(remaining_irqs);
118 remaining_irqs &= ~(1UL << irq);
119
120 /* Count device irqs; Linux IPIs are counted elsewhere. */
121 if (irq != IRQ_RESCHEDULE)
122 __this_cpu_inc(irq_stat.irq_dev_intr_count);
123
124 generic_handle_irq(irq);
125 }
126
127 /*
128 * If we weren't nested, turn on all enabled interrupts,
129 * including any that were reenabled during interrupt
130 * handling.
131 */
132 if (depth == 1)
133 unmask_irqs(~__this_cpu_read(irq_disable_mask));
134
135 __this_cpu_dec(irq_depth);
136
137 /*
138 * Track time spent against the current process again and
139 * process any softirqs if they are waiting.
140 */
141 irq_exit();
142 set_irq_regs(old_regs);
143}
144
145
146/*
147 * Remove an irq from the disabled mask. If we're in an interrupt
148 * context, defer enabling the HW interrupt until we leave.
149 */
150static void tile_irq_chip_enable(struct irq_data *d)
151{
152 get_cpu_var(irq_disable_mask) &= ~(1UL << d->irq);
153 if (__this_cpu_read(irq_depth) == 0)
154 unmask_irqs(1UL << d->irq);
155 put_cpu_var(irq_disable_mask);
156}
157
158/*
159 * Add an irq to the disabled mask. We disable the HW interrupt
160 * immediately so that there's no possibility of it firing. If we're
161 * in an interrupt context, the return path is careful to avoid
162 * unmasking a newly disabled interrupt.
163 */
164static void tile_irq_chip_disable(struct irq_data *d)
165{
166 get_cpu_var(irq_disable_mask) |= (1UL << d->irq);
167 mask_irqs(1UL << d->irq);
168 put_cpu_var(irq_disable_mask);
169}
170
171/* Mask an interrupt. */
172static void tile_irq_chip_mask(struct irq_data *d)
173{
174 mask_irqs(1UL << d->irq);
175}
176
177/* Unmask an interrupt. */
178static void tile_irq_chip_unmask(struct irq_data *d)
179{
180 unmask_irqs(1UL << d->irq);
181}
182
183/*
184 * Clear an interrupt before processing it so that any new assertions
185 * will trigger another irq.
186 */
187static void tile_irq_chip_ack(struct irq_data *d)
188{
189 if ((unsigned long)irq_data_get_irq_chip_data(d) != IS_HW_CLEARED)
190 clear_irqs(1UL << d->irq);
191}
192
193/*
194 * For per-cpu interrupts, we need to avoid unmasking any interrupts
195 * that we disabled via disable_percpu_irq().
196 */
197static void tile_irq_chip_eoi(struct irq_data *d)
198{
199 if (!(__this_cpu_read(irq_disable_mask) & (1UL << d->irq)))
200 unmask_irqs(1UL << d->irq);
201}
202
203static struct irq_chip tile_irq_chip = {
204 .name = "tile_irq_chip",
205 .irq_enable = tile_irq_chip_enable,
206 .irq_disable = tile_irq_chip_disable,
207 .irq_ack = tile_irq_chip_ack,
208 .irq_eoi = tile_irq_chip_eoi,
209 .irq_mask = tile_irq_chip_mask,
210 .irq_unmask = tile_irq_chip_unmask,
211};
212
213void __init init_IRQ(void)
214{
215 ipi_init();
216}
217
218void setup_irq_regs(void)
219{
220 /* Enable interrupt delivery. */
221 unmask_irqs(~0UL);
222#if CHIP_HAS_IPI()
223 arch_local_irq_unmask(INT_IPI_K);
224#endif
225}
226
227void tile_irq_activate(unsigned int irq, int tile_irq_type)
228{
229 /*
230 * We use handle_level_irq() by default because the pending
231 * interrupt vector (whether modeled by the HV on
232 * TILEPro or implemented in hardware on TILE-Gx) has
233 * level-style semantics for each bit. An interrupt fires
234 * whenever a bit is high, not just at edges.
235 */
236 irq_flow_handler_t handle = handle_level_irq;
237 if (tile_irq_type == TILE_IRQ_PERCPU)
238 handle = handle_percpu_irq;
239 irq_set_chip_and_handler(irq, &tile_irq_chip, handle);
240
241 /*
242 * Flag interrupts that are hardware-cleared so that ack()
243 * won't clear them.
244 */
245 if (tile_irq_type == TILE_IRQ_HW_CLEAR)
246 irq_set_chip_data(irq, (void *)IS_HW_CLEARED);
247}
248EXPORT_SYMBOL(tile_irq_activate);
249
250
251void ack_bad_irq(unsigned int irq)
252{
253 pr_err("unexpected IRQ trap at vector %02x\n", irq);
254}
255
256/*
257 * /proc/interrupts printing:
258 */
259int arch_show_interrupts(struct seq_file *p, int prec)
260{
261#ifdef CONFIG_PERF_EVENTS
262 int i;
263
264 seq_printf(p, "%*s: ", prec, "PMI");
265
266 for_each_online_cpu(i)
267 seq_printf(p, "%10llu ", per_cpu(perf_irqs, i));
268 seq_puts(p, " perf_events\n");
269#endif
270 return 0;
271}
272
273#if CHIP_HAS_IPI()
274int arch_setup_hwirq(unsigned int irq, int node)
275{
276 return irq >= NR_IRQS ? -EINVAL : 0;
277}
278
279void arch_teardown_hwirq(unsigned int irq) { }
280#endif
1/*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 */
14
15#include <linux/module.h>
16#include <linux/seq_file.h>
17#include <linux/interrupt.h>
18#include <linux/irq.h>
19#include <linux/kernel_stat.h>
20#include <linux/uaccess.h>
21#include <hv/drv_pcie_rc_intf.h>
22#include <arch/spr_def.h>
23#include <asm/traps.h>
24#include <linux/perf_event.h>
25
26/* Bit-flag stored in irq_desc->chip_data to indicate HW-cleared irqs. */
27#define IS_HW_CLEARED 1
28
29/*
30 * The set of interrupts we enable for arch_local_irq_enable().
31 * This is initialized to have just a single interrupt that the kernel
32 * doesn't actually use as a sentinel. During kernel init,
33 * interrupts are added as the kernel gets prepared to support them.
34 * NOTE: we could probably initialize them all statically up front.
35 */
36DEFINE_PER_CPU(unsigned long long, interrupts_enabled_mask) =
37 INITIAL_INTERRUPTS_ENABLED;
38EXPORT_PER_CPU_SYMBOL(interrupts_enabled_mask);
39
40/* Define per-tile device interrupt statistics state. */
41DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp;
42EXPORT_PER_CPU_SYMBOL(irq_stat);
43
44/*
45 * Define per-tile irq disable mask; the hardware/HV only has a single
46 * mask that we use to implement both masking and disabling.
47 */
48static DEFINE_PER_CPU(unsigned long, irq_disable_mask)
49 ____cacheline_internodealigned_in_smp;
50
51/*
52 * Per-tile IRQ nesting depth. Used to make sure we enable newly
53 * enabled IRQs before exiting the outermost interrupt.
54 */
55static DEFINE_PER_CPU(int, irq_depth);
56
57/* State for allocating IRQs on Gx. */
58#if CHIP_HAS_IPI()
59static unsigned long available_irqs = ((1UL << NR_IRQS) - 1) &
60 (~(1UL << IRQ_RESCHEDULE));
61static DEFINE_SPINLOCK(available_irqs_lock);
62#endif
63
64#if CHIP_HAS_IPI()
65/* Use SPRs to manipulate device interrupts. */
66#define mask_irqs(irq_mask) __insn_mtspr(SPR_IPI_MASK_SET_K, irq_mask)
67#define unmask_irqs(irq_mask) __insn_mtspr(SPR_IPI_MASK_RESET_K, irq_mask)
68#define clear_irqs(irq_mask) __insn_mtspr(SPR_IPI_EVENT_RESET_K, irq_mask)
69#else
70/* Use HV to manipulate device interrupts. */
71#define mask_irqs(irq_mask) hv_disable_intr(irq_mask)
72#define unmask_irqs(irq_mask) hv_enable_intr(irq_mask)
73#define clear_irqs(irq_mask) hv_clear_intr(irq_mask)
74#endif
75
76/*
77 * The interrupt handling path, implemented in terms of HV interrupt
78 * emulation on TILEPro, and IPI hardware on TILE-Gx.
79 * Entered with interrupts disabled.
80 */
81void tile_dev_intr(struct pt_regs *regs, int intnum)
82{
83 int depth = __get_cpu_var(irq_depth)++;
84 unsigned long original_irqs;
85 unsigned long remaining_irqs;
86 struct pt_regs *old_regs;
87
88#if CHIP_HAS_IPI()
89 /*
90 * Pending interrupts are listed in an SPR. We might be
91 * nested, so be sure to only handle irqs that weren't already
92 * masked by a previous interrupt. Then, mask out the ones
93 * we're going to handle.
94 */
95 unsigned long masked = __insn_mfspr(SPR_IPI_MASK_K);
96 original_irqs = __insn_mfspr(SPR_IPI_EVENT_K) & ~masked;
97 __insn_mtspr(SPR_IPI_MASK_SET_K, original_irqs);
98#else
99 /*
100 * Hypervisor performs the equivalent of the Gx code above and
101 * then puts the pending interrupt mask into a system save reg
102 * for us to find.
103 */
104 original_irqs = __insn_mfspr(SPR_SYSTEM_SAVE_K_3);
105#endif
106 remaining_irqs = original_irqs;
107
108 /* Track time spent here in an interrupt context. */
109 old_regs = set_irq_regs(regs);
110 irq_enter();
111
112#ifdef CONFIG_DEBUG_STACKOVERFLOW
113 /* Debugging check for stack overflow: less than 1/8th stack free? */
114 {
115 long sp = stack_pointer - (long) current_thread_info();
116 if (unlikely(sp < (sizeof(struct thread_info) + STACK_WARN))) {
117 pr_emerg("tile_dev_intr: "
118 "stack overflow: %ld\n",
119 sp - sizeof(struct thread_info));
120 dump_stack();
121 }
122 }
123#endif
124 while (remaining_irqs) {
125 unsigned long irq = __ffs(remaining_irqs);
126 remaining_irqs &= ~(1UL << irq);
127
128 /* Count device irqs; Linux IPIs are counted elsewhere. */
129 if (irq != IRQ_RESCHEDULE)
130 __get_cpu_var(irq_stat).irq_dev_intr_count++;
131
132 generic_handle_irq(irq);
133 }
134
135 /*
136 * If we weren't nested, turn on all enabled interrupts,
137 * including any that were reenabled during interrupt
138 * handling.
139 */
140 if (depth == 0)
141 unmask_irqs(~__get_cpu_var(irq_disable_mask));
142
143 __get_cpu_var(irq_depth)--;
144
145 /*
146 * Track time spent against the current process again and
147 * process any softirqs if they are waiting.
148 */
149 irq_exit();
150 set_irq_regs(old_regs);
151}
152
153
154/*
155 * Remove an irq from the disabled mask. If we're in an interrupt
156 * context, defer enabling the HW interrupt until we leave.
157 */
158static void tile_irq_chip_enable(struct irq_data *d)
159{
160 get_cpu_var(irq_disable_mask) &= ~(1UL << d->irq);
161 if (__get_cpu_var(irq_depth) == 0)
162 unmask_irqs(1UL << d->irq);
163 put_cpu_var(irq_disable_mask);
164}
165
166/*
167 * Add an irq to the disabled mask. We disable the HW interrupt
168 * immediately so that there's no possibility of it firing. If we're
169 * in an interrupt context, the return path is careful to avoid
170 * unmasking a newly disabled interrupt.
171 */
172static void tile_irq_chip_disable(struct irq_data *d)
173{
174 get_cpu_var(irq_disable_mask) |= (1UL << d->irq);
175 mask_irqs(1UL << d->irq);
176 put_cpu_var(irq_disable_mask);
177}
178
179/* Mask an interrupt. */
180static void tile_irq_chip_mask(struct irq_data *d)
181{
182 mask_irqs(1UL << d->irq);
183}
184
185/* Unmask an interrupt. */
186static void tile_irq_chip_unmask(struct irq_data *d)
187{
188 unmask_irqs(1UL << d->irq);
189}
190
191/*
192 * Clear an interrupt before processing it so that any new assertions
193 * will trigger another irq.
194 */
195static void tile_irq_chip_ack(struct irq_data *d)
196{
197 if ((unsigned long)irq_data_get_irq_chip_data(d) != IS_HW_CLEARED)
198 clear_irqs(1UL << d->irq);
199}
200
201/*
202 * For per-cpu interrupts, we need to avoid unmasking any interrupts
203 * that we disabled via disable_percpu_irq().
204 */
205static void tile_irq_chip_eoi(struct irq_data *d)
206{
207 if (!(__get_cpu_var(irq_disable_mask) & (1UL << d->irq)))
208 unmask_irqs(1UL << d->irq);
209}
210
211static struct irq_chip tile_irq_chip = {
212 .name = "tile_irq_chip",
213 .irq_enable = tile_irq_chip_enable,
214 .irq_disable = tile_irq_chip_disable,
215 .irq_ack = tile_irq_chip_ack,
216 .irq_eoi = tile_irq_chip_eoi,
217 .irq_mask = tile_irq_chip_mask,
218 .irq_unmask = tile_irq_chip_unmask,
219};
220
221void __init init_IRQ(void)
222{
223 ipi_init();
224}
225
226void setup_irq_regs(void)
227{
228 /* Enable interrupt delivery. */
229 unmask_irqs(~0UL);
230#if CHIP_HAS_IPI()
231 arch_local_irq_unmask(INT_IPI_K);
232#endif
233}
234
235void tile_irq_activate(unsigned int irq, int tile_irq_type)
236{
237 /*
238 * We use handle_level_irq() by default because the pending
239 * interrupt vector (whether modeled by the HV on
240 * TILEPro or implemented in hardware on TILE-Gx) has
241 * level-style semantics for each bit. An interrupt fires
242 * whenever a bit is high, not just at edges.
243 */
244 irq_flow_handler_t handle = handle_level_irq;
245 if (tile_irq_type == TILE_IRQ_PERCPU)
246 handle = handle_percpu_irq;
247 irq_set_chip_and_handler(irq, &tile_irq_chip, handle);
248
249 /*
250 * Flag interrupts that are hardware-cleared so that ack()
251 * won't clear them.
252 */
253 if (tile_irq_type == TILE_IRQ_HW_CLEAR)
254 irq_set_chip_data(irq, (void *)IS_HW_CLEARED);
255}
256EXPORT_SYMBOL(tile_irq_activate);
257
258
259void ack_bad_irq(unsigned int irq)
260{
261 pr_err("unexpected IRQ trap at vector %02x\n", irq);
262}
263
264/*
265 * /proc/interrupts printing:
266 */
267int arch_show_interrupts(struct seq_file *p, int prec)
268{
269#ifdef CONFIG_PERF_EVENTS
270 int i;
271
272 seq_printf(p, "%*s: ", prec, "PMI");
273
274 for_each_online_cpu(i)
275 seq_printf(p, "%10llu ", per_cpu(perf_irqs, i));
276 seq_puts(p, " perf_events\n");
277#endif
278 return 0;
279}
280
281/*
282 * Generic, controller-independent functions:
283 */
284
285#if CHIP_HAS_IPI()
286int create_irq(void)
287{
288 unsigned long flags;
289 int result;
290
291 spin_lock_irqsave(&available_irqs_lock, flags);
292 if (available_irqs == 0)
293 result = -ENOMEM;
294 else {
295 result = __ffs(available_irqs);
296 available_irqs &= ~(1UL << result);
297 dynamic_irq_init(result);
298 }
299 spin_unlock_irqrestore(&available_irqs_lock, flags);
300
301 return result;
302}
303EXPORT_SYMBOL(create_irq);
304
305void destroy_irq(unsigned int irq)
306{
307 unsigned long flags;
308
309 spin_lock_irqsave(&available_irqs_lock, flags);
310 available_irqs |= (1UL << irq);
311 dynamic_irq_cleanup(irq);
312 spin_unlock_irqrestore(&available_irqs_lock, flags);
313}
314EXPORT_SYMBOL(destroy_irq);
315#endif