1/* SPDX-License-Identifier: GPL-2.0 */
  2
  3#ifndef _ASM_X86_CPU_ENTRY_AREA_H
  4#define _ASM_X86_CPU_ENTRY_AREA_H
  5
  6#include <linux/percpu-defs.h>
  7#include <asm/processor.h>
  8#include <asm/intel_ds.h>
  9#include <asm/pgtable_areas.h>
 10
 11#ifdef CONFIG_X86_64
 12
 13#ifdef CONFIG_AMD_MEM_ENCRYPT
 14#define VC_EXCEPTION_STKSZ	EXCEPTION_STKSZ
 15#else
 16#define VC_EXCEPTION_STKSZ	0
 17#endif
 18
 19/* Macro to enforce the same ordering and stack sizes */
 20#define ESTACKS_MEMBERS(guardsize, optional_stack_size)		\
 21	char	DF_stack_guard[guardsize];			\
 22	char	DF_stack[EXCEPTION_STKSZ];			\
 23	char	NMI_stack_guard[guardsize];			\
 24	char	NMI_stack[EXCEPTION_STKSZ];			\
 25	char	DB_stack_guard[guardsize];			\
 26	char	DB_stack[EXCEPTION_STKSZ];			\
 27	char	MCE_stack_guard[guardsize];			\
 28	char	MCE_stack[EXCEPTION_STKSZ];			\
 29	char	VC_stack_guard[guardsize];			\
 30	char	VC_stack[optional_stack_size];			\
 31	char	VC2_stack_guard[guardsize];			\
 32	char	VC2_stack[optional_stack_size];			\
 33	char	IST_top_guard[guardsize];			\
 34
 35/* The exception stacks' physical storage. No guard pages required */
 36struct exception_stacks {
 37	ESTACKS_MEMBERS(0, VC_EXCEPTION_STKSZ)
 38};
 39
 40/* The effective cpu entry area mapping with guard pages. */
 41struct cea_exception_stacks {
 42	ESTACKS_MEMBERS(PAGE_SIZE, EXCEPTION_STKSZ)
 43};
 44
 45/*
 46 * The exception stack ordering in [cea_]exception_stacks
 47 */
 48enum exception_stack_ordering {
 49	ESTACK_DF,
 50	ESTACK_NMI,
 51	ESTACK_DB,
 52	ESTACK_MCE,
 53	ESTACK_VC,
 54	ESTACK_VC2,
 55	N_EXCEPTION_STACKS
 56};
 57
 58#define CEA_ESTACK_SIZE(st)					\
 59	sizeof(((struct cea_exception_stacks *)0)->st## _stack)
 60
 61#define CEA_ESTACK_BOT(ceastp, st)				\
 62	((unsigned long)&(ceastp)->st## _stack)
 63
 64#define CEA_ESTACK_TOP(ceastp, st)				\
 65	(CEA_ESTACK_BOT(ceastp, st) + CEA_ESTACK_SIZE(st))
 66
 67#define CEA_ESTACK_OFFS(st)					\
 68	offsetof(struct cea_exception_stacks, st## _stack)
 69
 70#define CEA_ESTACK_PAGES					\
 71	(sizeof(struct cea_exception_stacks) / PAGE_SIZE)
 72
 73#endif
 74
 75#ifdef CONFIG_X86_32
 76struct doublefault_stack {
 77	unsigned long stack[(PAGE_SIZE - sizeof(struct x86_hw_tss)) / sizeof(unsigned long)];
 78	struct x86_hw_tss tss;
 79} __aligned(PAGE_SIZE);
 80#endif
 81
 82/*
 83 * cpu_entry_area is a percpu region that contains things needed by the CPU
 84 * and early entry/exit code.  Real types aren't used for all fields here
 85 * to avoid circular header dependencies.
 86 *
 87 * Every field is a virtual alias of some other allocated backing store.
 88 * There is no direct allocation of a struct cpu_entry_area.
 89 */
 90struct cpu_entry_area {
 91	char gdt[PAGE_SIZE];
 92
 93	/*
 94	 * The GDT is just below entry_stack and thus serves (on x86_64) as
 95	 * a read-only guard page. On 32-bit the GDT must be writeable, so
 96	 * it needs an extra guard page.
 97	 */
 98#ifdef CONFIG_X86_32
 99	char guard_entry_stack[PAGE_SIZE];
100#endif
101	struct entry_stack_page entry_stack_page;
102
103#ifdef CONFIG_X86_32
104	char guard_doublefault_stack[PAGE_SIZE];
105	struct doublefault_stack doublefault_stack;
106#endif
107
108	/*
109	 * On x86_64, the TSS is mapped RO.  On x86_32, it's mapped RW because
110	 * we need task switches to work, and task switches write to the TSS.
111	 */
112	struct tss_struct tss;
113
114#ifdef CONFIG_X86_64
115	/*
116	 * Exception stacks used for IST entries with guard pages.
117	 */
118	struct cea_exception_stacks estacks;
119#endif
120	/*
121	 * Per CPU debug store for Intel performance monitoring. Wastes a
122	 * full page at the moment.
123	 */
124	struct debug_store cpu_debug_store;
125	/*
126	 * The actual PEBS/BTS buffers must be mapped to user space
127	 * Reserve enough fixmap PTEs.
128	 */
129	struct debug_store_buffers cpu_debug_buffers;
130};
131
132#define CPU_ENTRY_AREA_SIZE		(sizeof(struct cpu_entry_area))
133
134DECLARE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
135DECLARE_PER_CPU(struct cea_exception_stacks *, cea_exception_stacks);
136
137extern void setup_cpu_entry_areas(void);
138extern void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags);
139
140extern struct cpu_entry_area *get_cpu_entry_area(int cpu);
141
142static __always_inline struct entry_stack *cpu_entry_stack(int cpu)
143{
144	return &get_cpu_entry_area(cpu)->entry_stack_page.stack;
145}
146
147#define __this_cpu_ist_top_va(name)					\
148	CEA_ESTACK_TOP(__this_cpu_read(cea_exception_stacks), name)
149
150#define __this_cpu_ist_bottom_va(name)					\
151	CEA_ESTACK_BOT(__this_cpu_read(cea_exception_stacks), name)
152
153#endif