Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
4 */
5#ifndef _ASM_POWERPC_HW_IRQ_H
6#define _ASM_POWERPC_HW_IRQ_H
7
8#ifdef __KERNEL__
9
10#include <linux/errno.h>
11#include <linux/compiler.h>
12#include <asm/ptrace.h>
13#include <asm/processor.h>
14
15#ifdef CONFIG_PPC64
16
17/*
18 * PACA flags in paca->irq_happened.
19 *
20 * This bits are set when interrupts occur while soft-disabled
21 * and allow a proper replay.
22 *
23 * The PACA_IRQ_HARD_DIS is set whenever we hard disable. It is almost
24 * always in synch with the MSR[EE] state, except:
25 * - A window in interrupt entry, where hardware disables MSR[EE] and that
26 * must be "reconciled" with the soft mask state.
27 * - NMI interrupts that hit in awkward places, until they fix the state.
28 * - When local irqs are being enabled and state is being fixed up.
29 * - When returning from an interrupt there are some windows where this
30 * can become out of synch, but gets fixed before the RFI or before
31 * executing the next user instruction (see arch/powerpc/kernel/interrupt.c).
32 */
33#define PACA_IRQ_HARD_DIS 0x01
34#define PACA_IRQ_DBELL 0x02
35#define PACA_IRQ_EE 0x04
36#define PACA_IRQ_DEC 0x08 /* Or FIT */
37#define PACA_IRQ_HMI 0x10
38#define PACA_IRQ_PMI 0x20
39#define PACA_IRQ_REPLAYING 0x40
40
41/*
42 * Some soft-masked interrupts must be hard masked until they are replayed
43 * (e.g., because the soft-masked handler does not clear the exception).
44 * Interrupt replay itself must remain hard masked too.
45 */
46#ifdef CONFIG_PPC_BOOK3S
47#define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE|PACA_IRQ_PMI|PACA_IRQ_REPLAYING)
48#else
49#define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE|PACA_IRQ_REPLAYING)
50#endif
51
52#endif /* CONFIG_PPC64 */
53
54/*
55 * flags for paca->irq_soft_mask
56 */
57#define IRQS_ENABLED 0
58#define IRQS_DISABLED 1 /* local_irq_disable() interrupts */
59#define IRQS_PMI_DISABLED 2
60#define IRQS_ALL_DISABLED (IRQS_DISABLED | IRQS_PMI_DISABLED)
61
62#ifndef __ASSEMBLY__
63
64static inline void __hard_irq_enable(void)
65{
66 if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
67 wrtee(MSR_EE);
68 else if (IS_ENABLED(CONFIG_PPC_8xx))
69 wrtspr(SPRN_EIE);
70 else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
71 __mtmsrd(MSR_EE | MSR_RI, 1);
72 else
73 mtmsr(mfmsr() | MSR_EE);
74}
75
76static inline void __hard_irq_disable(void)
77{
78 if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
79 wrtee(0);
80 else if (IS_ENABLED(CONFIG_PPC_8xx))
81 wrtspr(SPRN_EID);
82 else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
83 __mtmsrd(MSR_RI, 1);
84 else
85 mtmsr(mfmsr() & ~MSR_EE);
86}
87
88static inline void __hard_EE_RI_disable(void)
89{
90 if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
91 wrtee(0);
92 else if (IS_ENABLED(CONFIG_PPC_8xx))
93 wrtspr(SPRN_NRI);
94 else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
95 __mtmsrd(0, 1);
96 else
97 mtmsr(mfmsr() & ~(MSR_EE | MSR_RI));
98}
99
100static inline void __hard_RI_enable(void)
101{
102 if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
103 return;
104
105 if (IS_ENABLED(CONFIG_PPC_8xx))
106 wrtspr(SPRN_EID);
107 else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
108 __mtmsrd(MSR_RI, 1);
109 else
110 mtmsr(mfmsr() | MSR_RI);
111}
112
113#ifdef CONFIG_PPC64
114#include <asm/paca.h>
115
116static inline notrace unsigned long irq_soft_mask_return(void)
117{
118 unsigned long flags;
119
120 asm volatile(
121 "lbz %0,%1(13)"
122 : "=r" (flags)
123 : "i" (offsetof(struct paca_struct, irq_soft_mask)));
124
125 return flags;
126}
127
128/*
129 * The "memory" clobber acts as both a compiler barrier
130 * for the critical section and as a clobber because
131 * we changed paca->irq_soft_mask
132 */
133static inline notrace void irq_soft_mask_set(unsigned long mask)
134{
135 /*
136 * The irq mask must always include the STD bit if any are set.
137 *
138 * and interrupts don't get replayed until the standard
139 * interrupt (local_irq_disable()) is unmasked.
140 *
141 * Other masks must only provide additional masking beyond
142 * the standard, and they are also not replayed until the
143 * standard interrupt becomes unmasked.
144 *
145 * This could be changed, but it will require partial
146 * unmasks to be replayed, among other things. For now, take
147 * the simple approach.
148 */
149 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
150 WARN_ON(mask && !(mask & IRQS_DISABLED));
151
152 asm volatile(
153 "stb %0,%1(13)"
154 :
155 : "r" (mask),
156 "i" (offsetof(struct paca_struct, irq_soft_mask))
157 : "memory");
158}
159
160static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask)
161{
162 unsigned long flags = irq_soft_mask_return();
163
164 irq_soft_mask_set(mask);
165
166 return flags;
167}
168
169static inline notrace unsigned long irq_soft_mask_or_return(unsigned long mask)
170{
171 unsigned long flags = irq_soft_mask_return();
172
173 irq_soft_mask_set(flags | mask);
174
175 return flags;
176}
177
178static inline notrace unsigned long irq_soft_mask_andc_return(unsigned long mask)
179{
180 unsigned long flags = irq_soft_mask_return();
181
182 irq_soft_mask_set(flags & ~mask);
183
184 return flags;
185}
186
187static inline unsigned long arch_local_save_flags(void)
188{
189 return irq_soft_mask_return();
190}
191
192static inline void arch_local_irq_disable(void)
193{
194 irq_soft_mask_set(IRQS_DISABLED);
195}
196
197extern void arch_local_irq_restore(unsigned long);
198
199static inline void arch_local_irq_enable(void)
200{
201 arch_local_irq_restore(IRQS_ENABLED);
202}
203
204static inline unsigned long arch_local_irq_save(void)
205{
206 return irq_soft_mask_or_return(IRQS_DISABLED);
207}
208
209static inline bool arch_irqs_disabled_flags(unsigned long flags)
210{
211 return flags & IRQS_DISABLED;
212}
213
214static inline bool arch_irqs_disabled(void)
215{
216 return arch_irqs_disabled_flags(arch_local_save_flags());
217}
218
219static inline void set_pmi_irq_pending(void)
220{
221 /*
222 * Invoked from PMU callback functions to set PMI bit in the paca.
223 * This has to be called with irq's disabled (via hard_irq_disable()).
224 */
225 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
226 WARN_ON_ONCE(mfmsr() & MSR_EE);
227
228 get_paca()->irq_happened |= PACA_IRQ_PMI;
229}
230
231static inline void clear_pmi_irq_pending(void)
232{
233 /*
234 * Invoked from PMU callback functions to clear the pending PMI bit
235 * in the paca.
236 */
237 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
238 WARN_ON_ONCE(mfmsr() & MSR_EE);
239
240 get_paca()->irq_happened &= ~PACA_IRQ_PMI;
241}
242
243static inline bool pmi_irq_pending(void)
244{
245 /*
246 * Invoked from PMU callback functions to check if there is a pending
247 * PMI bit in the paca.
248 */
249 if (get_paca()->irq_happened & PACA_IRQ_PMI)
250 return true;
251
252 return false;
253}
254
255#ifdef CONFIG_PPC_BOOK3S
256/*
257 * To support disabling and enabling of irq with PMI, set of
258 * new powerpc_local_irq_pmu_save() and powerpc_local_irq_restore()
259 * functions are added. These macros are implemented using generic
260 * linux local_irq_* code from include/linux/irqflags.h.
261 */
262#define raw_local_irq_pmu_save(flags) \
263 do { \
264 typecheck(unsigned long, flags); \
265 flags = irq_soft_mask_or_return(IRQS_DISABLED | \
266 IRQS_PMI_DISABLED); \
267 } while(0)
268
269#define raw_local_irq_pmu_restore(flags) \
270 do { \
271 typecheck(unsigned long, flags); \
272 arch_local_irq_restore(flags); \
273 } while(0)
274
275#ifdef CONFIG_TRACE_IRQFLAGS
276#define powerpc_local_irq_pmu_save(flags) \
277 do { \
278 raw_local_irq_pmu_save(flags); \
279 if (!raw_irqs_disabled_flags(flags)) \
280 trace_hardirqs_off(); \
281 } while(0)
282#define powerpc_local_irq_pmu_restore(flags) \
283 do { \
284 if (!raw_irqs_disabled_flags(flags)) \
285 trace_hardirqs_on(); \
286 raw_local_irq_pmu_restore(flags); \
287 } while(0)
288#else
289#define powerpc_local_irq_pmu_save(flags) \
290 do { \
291 raw_local_irq_pmu_save(flags); \
292 } while(0)
293#define powerpc_local_irq_pmu_restore(flags) \
294 do { \
295 raw_local_irq_pmu_restore(flags); \
296 } while (0)
297#endif /* CONFIG_TRACE_IRQFLAGS */
298
299#endif /* CONFIG_PPC_BOOK3S */
300
301#define hard_irq_disable() do { \
302 unsigned long flags; \
303 __hard_irq_disable(); \
304 flags = irq_soft_mask_set_return(IRQS_ALL_DISABLED); \
305 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \
306 if (!arch_irqs_disabled_flags(flags)) { \
307 asm volatile("std%X0 %1,%0" : "=m" (local_paca->saved_r1) \
308 : "r" (current_stack_pointer)); \
309 trace_hardirqs_off(); \
310 } \
311} while(0)
312
313static inline bool __lazy_irq_pending(u8 irq_happened)
314{
315 return !!(irq_happened & ~PACA_IRQ_HARD_DIS);
316}
317
318/*
319 * Check if a lazy IRQ is pending. Should be called with IRQs hard disabled.
320 */
321static inline bool lazy_irq_pending(void)
322{
323 return __lazy_irq_pending(get_paca()->irq_happened);
324}
325
326/*
327 * Check if a lazy IRQ is pending, with no debugging checks.
328 * Should be called with IRQs hard disabled.
329 * For use in RI disabled code or other constrained situations.
330 */
331static inline bool lazy_irq_pending_nocheck(void)
332{
333 return __lazy_irq_pending(local_paca->irq_happened);
334}
335
336bool power_pmu_wants_prompt_pmi(void);
337
338/*
339 * This is called by asynchronous interrupts to check whether to
340 * conditionally re-enable hard interrupts after having cleared
341 * the source of the interrupt. They are kept disabled if there
342 * is a different soft-masked interrupt pending that requires hard
343 * masking.
344 */
345static inline bool should_hard_irq_enable(struct pt_regs *regs)
346{
347 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
348 WARN_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
349 WARN_ON(!(get_paca()->irq_happened & PACA_IRQ_HARD_DIS));
350 WARN_ON(mfmsr() & MSR_EE);
351 }
352
353 if (!IS_ENABLED(CONFIG_PERF_EVENTS))
354 return false;
355 /*
356 * If the PMU is not running, there is not much reason to enable
357 * MSR[EE] in irq handlers because any interrupts would just be
358 * soft-masked.
359 *
360 * TODO: Add test for 64e
361 */
362 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
363 if (!power_pmu_wants_prompt_pmi())
364 return false;
365 /*
366 * If PMIs are disabled then IRQs should be disabled as well,
367 * so we shouldn't see this condition, check for it just in
368 * case because we are about to enable PMIs.
369 */
370 if (WARN_ON_ONCE(regs->softe & IRQS_PMI_DISABLED))
371 return false;
372 }
373
374 if (get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK)
375 return false;
376
377 return true;
378}
379
380/*
381 * Do the hard enabling, only call this if should_hard_irq_enable is true.
382 * This allows PMI interrupts to profile irq handlers.
383 */
384static inline void do_hard_irq_enable(void)
385{
386 /*
387 * Asynch interrupts come in with IRQS_ALL_DISABLED,
388 * PACA_IRQ_HARD_DIS, and MSR[EE]=0.
389 */
390 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
391 irq_soft_mask_andc_return(IRQS_PMI_DISABLED);
392 get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
393 __hard_irq_enable();
394}
395
396static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
397{
398 return (regs->softe & IRQS_DISABLED);
399}
400
401extern bool prep_irq_for_idle(void);
402extern bool prep_irq_for_idle_irqsoff(void);
403extern void irq_set_pending_from_srr1(unsigned long srr1);
404
405#define fini_irq_for_idle_irqsoff() trace_hardirqs_off();
406
407extern void force_external_irq_replay(void);
408
409static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
410{
411 regs->softe = val;
412}
413#else /* CONFIG_PPC64 */
414
415static inline notrace unsigned long irq_soft_mask_return(void)
416{
417 return 0;
418}
419
420static inline unsigned long arch_local_save_flags(void)
421{
422 return mfmsr();
423}
424
425static inline void arch_local_irq_restore(unsigned long flags)
426{
427 if (IS_ENABLED(CONFIG_BOOKE))
428 wrtee(flags);
429 else
430 mtmsr(flags);
431}
432
433static inline unsigned long arch_local_irq_save(void)
434{
435 unsigned long flags = arch_local_save_flags();
436
437 if (IS_ENABLED(CONFIG_BOOKE))
438 wrtee(0);
439 else if (IS_ENABLED(CONFIG_PPC_8xx))
440 wrtspr(SPRN_EID);
441 else
442 mtmsr(flags & ~MSR_EE);
443
444 return flags;
445}
446
447static inline void arch_local_irq_disable(void)
448{
449 __hard_irq_disable();
450}
451
452static inline void arch_local_irq_enable(void)
453{
454 __hard_irq_enable();
455}
456
457static inline bool arch_irqs_disabled_flags(unsigned long flags)
458{
459 return (flags & MSR_EE) == 0;
460}
461
462static inline bool arch_irqs_disabled(void)
463{
464 return arch_irqs_disabled_flags(arch_local_save_flags());
465}
466
467#define hard_irq_disable() arch_local_irq_disable()
468
469static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
470{
471 return !(regs->msr & MSR_EE);
472}
473
474static __always_inline bool should_hard_irq_enable(struct pt_regs *regs)
475{
476 return false;
477}
478
479static inline void do_hard_irq_enable(void)
480{
481 BUILD_BUG();
482}
483
484static inline void clear_pmi_irq_pending(void) { }
485static inline void set_pmi_irq_pending(void) { }
486static inline bool pmi_irq_pending(void) { return false; }
487
488static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
489{
490}
491#endif /* CONFIG_PPC64 */
492
493static inline unsigned long mtmsr_isync_irqsafe(unsigned long msr)
494{
495#ifdef CONFIG_PPC64
496 if (arch_irqs_disabled()) {
497 /*
498 * With soft-masking, MSR[EE] can change from 1 to 0
499 * asynchronously when irqs are disabled, and we don't want to
500 * set MSR[EE] back to 1 here if that has happened. A race-free
501 * way to do this is ensure EE is already 0. Another way it
502 * could be done is with a RESTART_TABLE handler, but that's
503 * probably overkill here.
504 */
505 msr &= ~MSR_EE;
506 mtmsr_isync(msr);
507 irq_soft_mask_set(IRQS_ALL_DISABLED);
508 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
509 } else
510#endif
511 mtmsr_isync(msr);
512
513 return msr;
514}
515
516
517#define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST
518
519#endif /* __ASSEMBLY__ */
520#endif /* __KERNEL__ */
521#endif /* _ASM_POWERPC_HW_IRQ_H */
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
4 */
5#ifndef _ASM_POWERPC_HW_IRQ_H
6#define _ASM_POWERPC_HW_IRQ_H
7
8#ifdef __KERNEL__
9
10#include <linux/errno.h>
11#include <linux/compiler.h>
12#include <asm/ptrace.h>
13#include <asm/processor.h>
14
15#ifdef CONFIG_PPC64
16
17/*
18 * PACA flags in paca->irq_happened.
19 *
20 * This bits are set when interrupts occur while soft-disabled
21 * and allow a proper replay.
22 *
23 * The PACA_IRQ_HARD_DIS is set whenever we hard disable. It is almost
24 * always in synch with the MSR[EE] state, except:
25 * - A window in interrupt entry, where hardware disables MSR[EE] and that
26 * must be "reconciled" with the soft mask state.
27 * - NMI interrupts that hit in awkward places, until they fix the state.
28 * - When local irqs are being enabled and state is being fixed up.
29 * - When returning from an interrupt there are some windows where this
30 * can become out of synch, but gets fixed before the RFI or before
31 * executing the next user instruction (see arch/powerpc/kernel/interrupt.c).
32 */
33#define PACA_IRQ_HARD_DIS 0x01
34#define PACA_IRQ_DBELL 0x02
35#define PACA_IRQ_EE 0x04
36#define PACA_IRQ_DEC 0x08 /* Or FIT */
37#define PACA_IRQ_HMI 0x10
38#define PACA_IRQ_PMI 0x20
39
40/*
41 * Some soft-masked interrupts must be hard masked until they are replayed
42 * (e.g., because the soft-masked handler does not clear the exception).
43 */
44#ifdef CONFIG_PPC_BOOK3S
45#define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE|PACA_IRQ_PMI)
46#else
47#define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE)
48#endif
49
50#endif /* CONFIG_PPC64 */
51
52/*
53 * flags for paca->irq_soft_mask
54 */
55#define IRQS_ENABLED 0
56#define IRQS_DISABLED 1 /* local_irq_disable() interrupts */
57#define IRQS_PMI_DISABLED 2
58#define IRQS_ALL_DISABLED (IRQS_DISABLED | IRQS_PMI_DISABLED)
59
60#ifndef __ASSEMBLY__
61
62static inline void __hard_irq_enable(void)
63{
64 if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
65 wrtee(MSR_EE);
66 else if (IS_ENABLED(CONFIG_PPC_8xx))
67 wrtspr(SPRN_EIE);
68 else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
69 __mtmsrd(MSR_EE | MSR_RI, 1);
70 else
71 mtmsr(mfmsr() | MSR_EE);
72}
73
74static inline void __hard_irq_disable(void)
75{
76 if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
77 wrtee(0);
78 else if (IS_ENABLED(CONFIG_PPC_8xx))
79 wrtspr(SPRN_EID);
80 else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
81 __mtmsrd(MSR_RI, 1);
82 else
83 mtmsr(mfmsr() & ~MSR_EE);
84}
85
86static inline void __hard_EE_RI_disable(void)
87{
88 if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
89 wrtee(0);
90 else if (IS_ENABLED(CONFIG_PPC_8xx))
91 wrtspr(SPRN_NRI);
92 else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
93 __mtmsrd(0, 1);
94 else
95 mtmsr(mfmsr() & ~(MSR_EE | MSR_RI));
96}
97
98static inline void __hard_RI_enable(void)
99{
100 if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
101 return;
102
103 if (IS_ENABLED(CONFIG_PPC_8xx))
104 wrtspr(SPRN_EID);
105 else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
106 __mtmsrd(MSR_RI, 1);
107 else
108 mtmsr(mfmsr() | MSR_RI);
109}
110
111#ifdef CONFIG_PPC64
112#include <asm/paca.h>
113
114static inline notrace unsigned long irq_soft_mask_return(void)
115{
116 unsigned long flags;
117
118 asm volatile(
119 "lbz %0,%1(13)"
120 : "=r" (flags)
121 : "i" (offsetof(struct paca_struct, irq_soft_mask)));
122
123 return flags;
124}
125
126/*
127 * The "memory" clobber acts as both a compiler barrier
128 * for the critical section and as a clobber because
129 * we changed paca->irq_soft_mask
130 */
131static inline notrace void irq_soft_mask_set(unsigned long mask)
132{
133 /*
134 * The irq mask must always include the STD bit if any are set.
135 *
136 * and interrupts don't get replayed until the standard
137 * interrupt (local_irq_disable()) is unmasked.
138 *
139 * Other masks must only provide additional masking beyond
140 * the standard, and they are also not replayed until the
141 * standard interrupt becomes unmasked.
142 *
143 * This could be changed, but it will require partial
144 * unmasks to be replayed, among other things. For now, take
145 * the simple approach.
146 */
147 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
148 WARN_ON(mask && !(mask & IRQS_DISABLED));
149
150 asm volatile(
151 "stb %0,%1(13)"
152 :
153 : "r" (mask),
154 "i" (offsetof(struct paca_struct, irq_soft_mask))
155 : "memory");
156}
157
158static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask)
159{
160 unsigned long flags = irq_soft_mask_return();
161
162 irq_soft_mask_set(mask);
163
164 return flags;
165}
166
167static inline notrace unsigned long irq_soft_mask_or_return(unsigned long mask)
168{
169 unsigned long flags = irq_soft_mask_return();
170
171 irq_soft_mask_set(flags | mask);
172
173 return flags;
174}
175
176static inline notrace unsigned long irq_soft_mask_andc_return(unsigned long mask)
177{
178 unsigned long flags = irq_soft_mask_return();
179
180 irq_soft_mask_set(flags & ~mask);
181
182 return flags;
183}
184
185static inline unsigned long arch_local_save_flags(void)
186{
187 return irq_soft_mask_return();
188}
189
190static inline void arch_local_irq_disable(void)
191{
192 irq_soft_mask_set(IRQS_DISABLED);
193}
194
195extern void arch_local_irq_restore(unsigned long);
196
197static inline void arch_local_irq_enable(void)
198{
199 arch_local_irq_restore(IRQS_ENABLED);
200}
201
202static inline unsigned long arch_local_irq_save(void)
203{
204 return irq_soft_mask_or_return(IRQS_DISABLED);
205}
206
207static inline bool arch_irqs_disabled_flags(unsigned long flags)
208{
209 return flags & IRQS_DISABLED;
210}
211
212static inline bool arch_irqs_disabled(void)
213{
214 return arch_irqs_disabled_flags(arch_local_save_flags());
215}
216
217static inline void set_pmi_irq_pending(void)
218{
219 /*
220 * Invoked from PMU callback functions to set PMI bit in the paca.
221 * This has to be called with irq's disabled (via hard_irq_disable()).
222 */
223 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
224 WARN_ON_ONCE(mfmsr() & MSR_EE);
225
226 get_paca()->irq_happened |= PACA_IRQ_PMI;
227}
228
229static inline void clear_pmi_irq_pending(void)
230{
231 /*
232 * Invoked from PMU callback functions to clear the pending PMI bit
233 * in the paca.
234 */
235 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
236 WARN_ON_ONCE(mfmsr() & MSR_EE);
237
238 get_paca()->irq_happened &= ~PACA_IRQ_PMI;
239}
240
241static inline bool pmi_irq_pending(void)
242{
243 /*
244 * Invoked from PMU callback functions to check if there is a pending
245 * PMI bit in the paca.
246 */
247 if (get_paca()->irq_happened & PACA_IRQ_PMI)
248 return true;
249
250 return false;
251}
252
253#ifdef CONFIG_PPC_BOOK3S
254/*
255 * To support disabling and enabling of irq with PMI, set of
256 * new powerpc_local_irq_pmu_save() and powerpc_local_irq_restore()
257 * functions are added. These macros are implemented using generic
258 * linux local_irq_* code from include/linux/irqflags.h.
259 */
260#define raw_local_irq_pmu_save(flags) \
261 do { \
262 typecheck(unsigned long, flags); \
263 flags = irq_soft_mask_or_return(IRQS_DISABLED | \
264 IRQS_PMI_DISABLED); \
265 } while(0)
266
267#define raw_local_irq_pmu_restore(flags) \
268 do { \
269 typecheck(unsigned long, flags); \
270 arch_local_irq_restore(flags); \
271 } while(0)
272
273#ifdef CONFIG_TRACE_IRQFLAGS
274#define powerpc_local_irq_pmu_save(flags) \
275 do { \
276 raw_local_irq_pmu_save(flags); \
277 if (!raw_irqs_disabled_flags(flags)) \
278 trace_hardirqs_off(); \
279 } while(0)
280#define powerpc_local_irq_pmu_restore(flags) \
281 do { \
282 if (!raw_irqs_disabled_flags(flags)) \
283 trace_hardirqs_on(); \
284 raw_local_irq_pmu_restore(flags); \
285 } while(0)
286#else
287#define powerpc_local_irq_pmu_save(flags) \
288 do { \
289 raw_local_irq_pmu_save(flags); \
290 } while(0)
291#define powerpc_local_irq_pmu_restore(flags) \
292 do { \
293 raw_local_irq_pmu_restore(flags); \
294 } while (0)
295#endif /* CONFIG_TRACE_IRQFLAGS */
296
297#endif /* CONFIG_PPC_BOOK3S */
298
299#define hard_irq_disable() do { \
300 unsigned long flags; \
301 __hard_irq_disable(); \
302 flags = irq_soft_mask_set_return(IRQS_ALL_DISABLED); \
303 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \
304 if (!arch_irqs_disabled_flags(flags)) { \
305 asm volatile("std%X0 %1,%0" : "=m" (local_paca->saved_r1) \
306 : "r" (current_stack_pointer)); \
307 trace_hardirqs_off(); \
308 } \
309} while(0)
310
311static inline bool __lazy_irq_pending(u8 irq_happened)
312{
313 return !!(irq_happened & ~PACA_IRQ_HARD_DIS);
314}
315
316/*
317 * Check if a lazy IRQ is pending. Should be called with IRQs hard disabled.
318 */
319static inline bool lazy_irq_pending(void)
320{
321 return __lazy_irq_pending(get_paca()->irq_happened);
322}
323
324/*
325 * Check if a lazy IRQ is pending, with no debugging checks.
326 * Should be called with IRQs hard disabled.
327 * For use in RI disabled code or other constrained situations.
328 */
329static inline bool lazy_irq_pending_nocheck(void)
330{
331 return __lazy_irq_pending(local_paca->irq_happened);
332}
333
334bool power_pmu_wants_prompt_pmi(void);
335
336/*
337 * This is called by asynchronous interrupts to check whether to
338 * conditionally re-enable hard interrupts after having cleared
339 * the source of the interrupt. They are kept disabled if there
340 * is a different soft-masked interrupt pending that requires hard
341 * masking.
342 */
343static inline bool should_hard_irq_enable(struct pt_regs *regs)
344{
345 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
346 WARN_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
347 WARN_ON(!(get_paca()->irq_happened & PACA_IRQ_HARD_DIS));
348 WARN_ON(mfmsr() & MSR_EE);
349 }
350
351 if (!IS_ENABLED(CONFIG_PERF_EVENTS))
352 return false;
353 /*
354 * If the PMU is not running, there is not much reason to enable
355 * MSR[EE] in irq handlers because any interrupts would just be
356 * soft-masked.
357 *
358 * TODO: Add test for 64e
359 */
360 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
361 if (!power_pmu_wants_prompt_pmi())
362 return false;
363 /*
364 * If PMIs are disabled then IRQs should be disabled as well,
365 * so we shouldn't see this condition, check for it just in
366 * case because we are about to enable PMIs.
367 */
368 if (WARN_ON_ONCE(regs->softe & IRQS_PMI_DISABLED))
369 return false;
370 }
371
372 if (get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK)
373 return false;
374
375 return true;
376}
377
378/*
379 * Do the hard enabling, only call this if should_hard_irq_enable is true.
380 * This allows PMI interrupts to profile irq handlers.
381 */
382static inline void do_hard_irq_enable(void)
383{
384 /*
385 * Asynch interrupts come in with IRQS_ALL_DISABLED,
386 * PACA_IRQ_HARD_DIS, and MSR[EE]=0.
387 */
388 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
389 irq_soft_mask_andc_return(IRQS_PMI_DISABLED);
390 get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
391 __hard_irq_enable();
392}
393
394static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
395{
396 return (regs->softe & IRQS_DISABLED);
397}
398
399extern bool prep_irq_for_idle(void);
400extern bool prep_irq_for_idle_irqsoff(void);
401extern void irq_set_pending_from_srr1(unsigned long srr1);
402
403#define fini_irq_for_idle_irqsoff() trace_hardirqs_off();
404
405extern void force_external_irq_replay(void);
406
407static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
408{
409 regs->softe = val;
410}
411#else /* CONFIG_PPC64 */
412
413static inline notrace unsigned long irq_soft_mask_return(void)
414{
415 return 0;
416}
417
418static inline unsigned long arch_local_save_flags(void)
419{
420 return mfmsr();
421}
422
423static inline void arch_local_irq_restore(unsigned long flags)
424{
425 if (IS_ENABLED(CONFIG_BOOKE))
426 wrtee(flags);
427 else
428 mtmsr(flags);
429}
430
431static inline unsigned long arch_local_irq_save(void)
432{
433 unsigned long flags = arch_local_save_flags();
434
435 if (IS_ENABLED(CONFIG_BOOKE))
436 wrtee(0);
437 else if (IS_ENABLED(CONFIG_PPC_8xx))
438 wrtspr(SPRN_EID);
439 else
440 mtmsr(flags & ~MSR_EE);
441
442 return flags;
443}
444
445static inline void arch_local_irq_disable(void)
446{
447 __hard_irq_disable();
448}
449
450static inline void arch_local_irq_enable(void)
451{
452 __hard_irq_enable();
453}
454
455static inline bool arch_irqs_disabled_flags(unsigned long flags)
456{
457 return (flags & MSR_EE) == 0;
458}
459
460static inline bool arch_irqs_disabled(void)
461{
462 return arch_irqs_disabled_flags(arch_local_save_flags());
463}
464
465#define hard_irq_disable() arch_local_irq_disable()
466
467static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
468{
469 return !(regs->msr & MSR_EE);
470}
471
472static __always_inline bool should_hard_irq_enable(struct pt_regs *regs)
473{
474 return false;
475}
476
477static inline void do_hard_irq_enable(void)
478{
479 BUILD_BUG();
480}
481
482static inline void clear_pmi_irq_pending(void) { }
483static inline void set_pmi_irq_pending(void) { }
484static inline bool pmi_irq_pending(void) { return false; }
485
486static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
487{
488}
489#endif /* CONFIG_PPC64 */
490
491static inline unsigned long mtmsr_isync_irqsafe(unsigned long msr)
492{
493#ifdef CONFIG_PPC64
494 if (arch_irqs_disabled()) {
495 /*
496 * With soft-masking, MSR[EE] can change from 1 to 0
497 * asynchronously when irqs are disabled, and we don't want to
498 * set MSR[EE] back to 1 here if that has happened. A race-free
499 * way to do this is ensure EE is already 0. Another way it
500 * could be done is with a RESTART_TABLE handler, but that's
501 * probably overkill here.
502 */
503 msr &= ~MSR_EE;
504 mtmsr_isync(msr);
505 irq_soft_mask_set(IRQS_ALL_DISABLED);
506 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
507 } else
508#endif
509 mtmsr_isync(msr);
510
511 return msr;
512}
513
514
515#define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST
516
517#endif /* __ASSEMBLY__ */
518#endif /* __KERNEL__ */
519#endif /* _ASM_POWERPC_HW_IRQ_H */