1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _ASM_X86_IRQ_STACK_H
  3#define _ASM_X86_IRQ_STACK_H
  4
  5#include <linux/ptrace.h>
  6
  7#include <asm/processor.h>
  8
  9#ifdef CONFIG_X86_64
 10static __always_inline bool irqstack_active(void)
 11{
 12	return __this_cpu_read(irq_count) != -1;
 13}
 14
 15void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
 16void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
 17			      struct pt_regs *regs);
 18void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
 19			   struct irq_desc *desc);
 20
 21static __always_inline void __run_on_irqstack(void (*func)(void))
 22{
 23	void *tos = __this_cpu_read(hardirq_stack_ptr);
 24
 25	__this_cpu_add(irq_count, 1);
 26	asm_call_on_stack(tos - 8, func, NULL);
 27	__this_cpu_sub(irq_count, 1);
 28}
 29
 30static __always_inline void
 31__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
 32			 struct pt_regs *regs)
 33{
 34	void *tos = __this_cpu_read(hardirq_stack_ptr);
 35
 36	__this_cpu_add(irq_count, 1);
 37	asm_call_sysvec_on_stack(tos - 8, func, regs);
 38	__this_cpu_sub(irq_count, 1);
 39}
 40
 41static __always_inline void
 42__run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
 43		      struct irq_desc *desc)
 44{
 45	void *tos = __this_cpu_read(hardirq_stack_ptr);
 46
 47	__this_cpu_add(irq_count, 1);
 48	asm_call_irq_on_stack(tos - 8, func, desc);
 49	__this_cpu_sub(irq_count, 1);
 50}
 51
 52#else /* CONFIG_X86_64 */
 53static inline bool irqstack_active(void) { return false; }
 54static inline void __run_on_irqstack(void (*func)(void)) { }
 55static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
 56					    struct pt_regs *regs) { }
 57static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
 58					 struct irq_desc *desc) { }
 59#endif /* !CONFIG_X86_64 */
 60
 61static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
 62{
 63	if (IS_ENABLED(CONFIG_X86_32))
 64		return false;
 65	if (!regs)
 66		return !irqstack_active();
 67	return !user_mode(regs) && !irqstack_active();
 68}
 69
 70
 71static __always_inline void run_on_irqstack_cond(void (*func)(void),
 72						 struct pt_regs *regs)
 73{
 74	lockdep_assert_irqs_disabled();
 75
 76	if (irq_needs_irq_stack(regs))
 77		__run_on_irqstack(func);
 78	else
 79		func();
 80}
 81
 82static __always_inline void
 83run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs),
 84			    struct pt_regs *regs)
 85{
 86	lockdep_assert_irqs_disabled();
 87
 88	if (irq_needs_irq_stack(regs))
 89		__run_sysvec_on_irqstack(func, regs);
 90	else
 91		func(regs);
 92}
 93
 94static __always_inline void
 95run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc,
 96			 struct pt_regs *regs)
 97{
 98	lockdep_assert_irqs_disabled();
 99
100	if (irq_needs_irq_stack(regs))
101		__run_irq_on_irqstack(func, desc);
102	else
103		func(desc);
104}
105
106#endif