Loading...
1/* ----------------------------------------------------------------------- *
2 *
3 * Copyright 2014 Intel Corporation; author: H. Peter Anvin
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * ----------------------------------------------------------------------- */
15
16/*
17 * The IRET instruction, when returning to a 16-bit segment, only
18 * restores the bottom 16 bits of the user space stack pointer. This
19 * causes some 16-bit software to break, but it also leaks kernel state
20 * to user space.
21 *
22 * This works around this by creating percpu "ministacks", each of which
23 * is mapped 2^16 times 64K apart. When we detect that the return SS is
24 * on the LDT, we copy the IRET frame to the ministack and use the
25 * relevant alias to return to userspace. The ministacks are mapped
26 * readonly, so if the IRET fault we promote #GP to #DF which is an IST
27 * vector and thus has its own stack; we then do the fixup in the #DF
28 * handler.
29 *
30 * This file sets up the ministacks and the related page tables. The
31 * actual ministack invocation is in entry_64.S.
32 */
33
34#include <linux/init.h>
35#include <linux/init_task.h>
36#include <linux/kernel.h>
37#include <linux/percpu.h>
38#include <linux/gfp.h>
39#include <linux/random.h>
40#include <asm/pgtable.h>
41#include <asm/pgalloc.h>
42#include <asm/setup.h>
43#include <asm/espfix.h>
44
45/*
46 * Note: we only need 6*8 = 48 bytes for the espfix stack, but round
47 * it up to a cache line to avoid unnecessary sharing.
48 */
49#define ESPFIX_STACK_SIZE (8*8UL)
50#define ESPFIX_STACKS_PER_PAGE (PAGE_SIZE/ESPFIX_STACK_SIZE)
51
52/* There is address space for how many espfix pages? */
53#define ESPFIX_PAGE_SPACE (1UL << (P4D_SHIFT-PAGE_SHIFT-16))
54
55#define ESPFIX_MAX_CPUS (ESPFIX_STACKS_PER_PAGE * ESPFIX_PAGE_SPACE)
56#if CONFIG_NR_CPUS > ESPFIX_MAX_CPUS
57# error "Need more virtual address space for the ESPFIX hack"
58#endif
59
60#define PGALLOC_GFP (GFP_KERNEL | __GFP_ZERO)
61
62/* This contains the *bottom* address of the espfix stack */
63DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack);
64DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_waddr);
65
66/* Initialization mutex - should this be a spinlock? */
67static DEFINE_MUTEX(espfix_init_mutex);
68
69/* Page allocation bitmap - each page serves ESPFIX_STACKS_PER_PAGE CPUs */
70#define ESPFIX_MAX_PAGES DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE)
71static void *espfix_pages[ESPFIX_MAX_PAGES];
72
73static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD]
74 __aligned(PAGE_SIZE);
75
76static unsigned int page_random, slot_random;
77
78/*
79 * This returns the bottom address of the espfix stack for a specific CPU.
80 * The math allows for a non-power-of-two ESPFIX_STACK_SIZE, in which case
81 * we have to account for some amount of padding at the end of each page.
82 */
83static inline unsigned long espfix_base_addr(unsigned int cpu)
84{
85 unsigned long page, slot;
86 unsigned long addr;
87
88 page = (cpu / ESPFIX_STACKS_PER_PAGE) ^ page_random;
89 slot = (cpu + slot_random) % ESPFIX_STACKS_PER_PAGE;
90 addr = (page << PAGE_SHIFT) + (slot * ESPFIX_STACK_SIZE);
91 addr = (addr & 0xffffUL) | ((addr & ~0xffffUL) << 16);
92 addr += ESPFIX_BASE_ADDR;
93 return addr;
94}
95
96#define PTE_STRIDE (65536/PAGE_SIZE)
97#define ESPFIX_PTE_CLONES (PTRS_PER_PTE/PTE_STRIDE)
98#define ESPFIX_PMD_CLONES PTRS_PER_PMD
99#define ESPFIX_PUD_CLONES (65536/(ESPFIX_PTE_CLONES*ESPFIX_PMD_CLONES))
100
101#define PGTABLE_PROT ((_KERNPG_TABLE & ~_PAGE_RW) | _PAGE_NX)
102
103static void init_espfix_random(void)
104{
105 unsigned long rand;
106
107 /*
108 * This is run before the entropy pools are initialized,
109 * but this is hopefully better than nothing.
110 */
111 if (!arch_get_random_long(&rand)) {
112 /* The constant is an arbitrary large prime */
113 rand = rdtsc();
114 rand *= 0xc345c6b72fd16123UL;
115 }
116
117 slot_random = rand % ESPFIX_STACKS_PER_PAGE;
118 page_random = (rand / ESPFIX_STACKS_PER_PAGE)
119 & (ESPFIX_PAGE_SPACE - 1);
120}
121
122void __init init_espfix_bsp(void)
123{
124 pgd_t *pgd;
125 p4d_t *p4d;
126
127 /* Install the espfix pud into the kernel page directory */
128 pgd = &init_top_pgt[pgd_index(ESPFIX_BASE_ADDR)];
129 p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR);
130 p4d_populate(&init_mm, p4d, espfix_pud_page);
131
132 /* Randomize the locations */
133 init_espfix_random();
134
135 /* The rest is the same as for any other processor */
136 init_espfix_ap(0);
137}
138
139void init_espfix_ap(int cpu)
140{
141 unsigned int page;
142 unsigned long addr;
143 pud_t pud, *pud_p;
144 pmd_t pmd, *pmd_p;
145 pte_t pte, *pte_p;
146 int n, node;
147 void *stack_page;
148 pteval_t ptemask;
149
150 /* We only have to do this once... */
151 if (likely(per_cpu(espfix_stack, cpu)))
152 return; /* Already initialized */
153
154 addr = espfix_base_addr(cpu);
155 page = cpu/ESPFIX_STACKS_PER_PAGE;
156
157 /* Did another CPU already set this up? */
158 stack_page = READ_ONCE(espfix_pages[page]);
159 if (likely(stack_page))
160 goto done;
161
162 mutex_lock(&espfix_init_mutex);
163
164 /* Did we race on the lock? */
165 stack_page = READ_ONCE(espfix_pages[page]);
166 if (stack_page)
167 goto unlock_done;
168
169 node = cpu_to_node(cpu);
170 ptemask = __supported_pte_mask;
171
172 pud_p = &espfix_pud_page[pud_index(addr)];
173 pud = *pud_p;
174 if (!pud_present(pud)) {
175 struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
176
177 pmd_p = (pmd_t *)page_address(page);
178 pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask));
179 paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT);
180 for (n = 0; n < ESPFIX_PUD_CLONES; n++)
181 set_pud(&pud_p[n], pud);
182 }
183
184 pmd_p = pmd_offset(&pud, addr);
185 pmd = *pmd_p;
186 if (!pmd_present(pmd)) {
187 struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
188
189 pte_p = (pte_t *)page_address(page);
190 pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask));
191 paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT);
192 for (n = 0; n < ESPFIX_PMD_CLONES; n++)
193 set_pmd(&pmd_p[n], pmd);
194 }
195
196 pte_p = pte_offset_kernel(&pmd, addr);
197 stack_page = page_address(alloc_pages_node(node, GFP_KERNEL, 0));
198 /*
199 * __PAGE_KERNEL_* includes _PAGE_GLOBAL, which we want since
200 * this is mapped to userspace.
201 */
202 pte = __pte(__pa(stack_page) | ((__PAGE_KERNEL_RO | _PAGE_ENC) & ptemask));
203 for (n = 0; n < ESPFIX_PTE_CLONES; n++)
204 set_pte(&pte_p[n*PTE_STRIDE], pte);
205
206 /* Job is done for this CPU and any CPU which shares this page */
207 WRITE_ONCE(espfix_pages[page], stack_page);
208
209unlock_done:
210 mutex_unlock(&espfix_init_mutex);
211done:
212 per_cpu(espfix_stack, cpu) = addr;
213 per_cpu(espfix_waddr, cpu) = (unsigned long)stack_page
214 + (addr & ~PAGE_MASK);
215}
1// SPDX-License-Identifier: GPL-2.0-only
2/* ----------------------------------------------------------------------- *
3 *
4 * Copyright 2014 Intel Corporation; author: H. Peter Anvin
5 *
6 * ----------------------------------------------------------------------- */
7
8/*
9 * The IRET instruction, when returning to a 16-bit segment, only
10 * restores the bottom 16 bits of the user space stack pointer. This
11 * causes some 16-bit software to break, but it also leaks kernel state
12 * to user space.
13 *
14 * This works around this by creating percpu "ministacks", each of which
15 * is mapped 2^16 times 64K apart. When we detect that the return SS is
16 * on the LDT, we copy the IRET frame to the ministack and use the
17 * relevant alias to return to userspace. The ministacks are mapped
18 * readonly, so if the IRET fault we promote #GP to #DF which is an IST
19 * vector and thus has its own stack; we then do the fixup in the #DF
20 * handler.
21 *
22 * This file sets up the ministacks and the related page tables. The
23 * actual ministack invocation is in entry_64.S.
24 */
25
26#include <linux/init.h>
27#include <linux/init_task.h>
28#include <linux/kernel.h>
29#include <linux/percpu.h>
30#include <linux/gfp.h>
31#include <linux/random.h>
32#include <linux/pgtable.h>
33#include <asm/pgalloc.h>
34#include <asm/setup.h>
35#include <asm/espfix.h>
36
37/*
38 * Note: we only need 6*8 = 48 bytes for the espfix stack, but round
39 * it up to a cache line to avoid unnecessary sharing.
40 */
41#define ESPFIX_STACK_SIZE (8*8UL)
42#define ESPFIX_STACKS_PER_PAGE (PAGE_SIZE/ESPFIX_STACK_SIZE)
43
44/* There is address space for how many espfix pages? */
45#define ESPFIX_PAGE_SPACE (1UL << (P4D_SHIFT-PAGE_SHIFT-16))
46
47#define ESPFIX_MAX_CPUS (ESPFIX_STACKS_PER_PAGE * ESPFIX_PAGE_SPACE)
48#if CONFIG_NR_CPUS > ESPFIX_MAX_CPUS
49# error "Need more virtual address space for the ESPFIX hack"
50#endif
51
52#define PGALLOC_GFP (GFP_KERNEL | __GFP_ZERO)
53
54/* This contains the *bottom* address of the espfix stack */
55DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack);
56DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_waddr);
57
58/* Initialization mutex - should this be a spinlock? */
59static DEFINE_MUTEX(espfix_init_mutex);
60
61/* Page allocation bitmap - each page serves ESPFIX_STACKS_PER_PAGE CPUs */
62#define ESPFIX_MAX_PAGES DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE)
63static void *espfix_pages[ESPFIX_MAX_PAGES];
64
65static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD]
66 __aligned(PAGE_SIZE);
67
68static unsigned int page_random, slot_random;
69
70/*
71 * This returns the bottom address of the espfix stack for a specific CPU.
72 * The math allows for a non-power-of-two ESPFIX_STACK_SIZE, in which case
73 * we have to account for some amount of padding at the end of each page.
74 */
75static inline unsigned long espfix_base_addr(unsigned int cpu)
76{
77 unsigned long page, slot;
78 unsigned long addr;
79
80 page = (cpu / ESPFIX_STACKS_PER_PAGE) ^ page_random;
81 slot = (cpu + slot_random) % ESPFIX_STACKS_PER_PAGE;
82 addr = (page << PAGE_SHIFT) + (slot * ESPFIX_STACK_SIZE);
83 addr = (addr & 0xffffUL) | ((addr & ~0xffffUL) << 16);
84 addr += ESPFIX_BASE_ADDR;
85 return addr;
86}
87
88#define PTE_STRIDE (65536/PAGE_SIZE)
89#define ESPFIX_PTE_CLONES (PTRS_PER_PTE/PTE_STRIDE)
90#define ESPFIX_PMD_CLONES PTRS_PER_PMD
91#define ESPFIX_PUD_CLONES (65536/(ESPFIX_PTE_CLONES*ESPFIX_PMD_CLONES))
92
93#define PGTABLE_PROT ((_KERNPG_TABLE & ~_PAGE_RW) | _PAGE_NX)
94
95static void init_espfix_random(void)
96{
97 unsigned long rand = get_random_long();
98
99 slot_random = rand % ESPFIX_STACKS_PER_PAGE;
100 page_random = (rand / ESPFIX_STACKS_PER_PAGE)
101 & (ESPFIX_PAGE_SPACE - 1);
102}
103
104void __init init_espfix_bsp(void)
105{
106 pgd_t *pgd;
107 p4d_t *p4d;
108
109 /* Install the espfix pud into the kernel page directory */
110 pgd = &init_top_pgt[pgd_index(ESPFIX_BASE_ADDR)];
111 p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR);
112 p4d_populate(&init_mm, p4d, espfix_pud_page);
113
114 /* Randomize the locations */
115 init_espfix_random();
116
117 /* The rest is the same as for any other processor */
118 init_espfix_ap(0);
119}
120
121void init_espfix_ap(int cpu)
122{
123 unsigned int page;
124 unsigned long addr;
125 pud_t pud, *pud_p;
126 pmd_t pmd, *pmd_p;
127 pte_t pte, *pte_p;
128 int n, node;
129 void *stack_page;
130 pteval_t ptemask;
131
132 /* We only have to do this once... */
133 if (likely(per_cpu(espfix_stack, cpu)))
134 return; /* Already initialized */
135
136 addr = espfix_base_addr(cpu);
137 page = cpu/ESPFIX_STACKS_PER_PAGE;
138
139 /* Did another CPU already set this up? */
140 stack_page = READ_ONCE(espfix_pages[page]);
141 if (likely(stack_page))
142 goto done;
143
144 mutex_lock(&espfix_init_mutex);
145
146 /* Did we race on the lock? */
147 stack_page = READ_ONCE(espfix_pages[page]);
148 if (stack_page)
149 goto unlock_done;
150
151 node = cpu_to_node(cpu);
152 ptemask = __supported_pte_mask;
153
154 pud_p = &espfix_pud_page[pud_index(addr)];
155 pud = *pud_p;
156 if (!pud_present(pud)) {
157 struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
158
159 pmd_p = (pmd_t *)page_address(page);
160 pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask));
161 paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT);
162 for (n = 0; n < ESPFIX_PUD_CLONES; n++)
163 set_pud(&pud_p[n], pud);
164 }
165
166 pmd_p = pmd_offset(&pud, addr);
167 pmd = *pmd_p;
168 if (!pmd_present(pmd)) {
169 struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
170
171 pte_p = (pte_t *)page_address(page);
172 pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask));
173 paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT);
174 for (n = 0; n < ESPFIX_PMD_CLONES; n++)
175 set_pmd(&pmd_p[n], pmd);
176 }
177
178 pte_p = pte_offset_kernel(&pmd, addr);
179 stack_page = page_address(alloc_pages_node(node, GFP_KERNEL, 0));
180 /*
181 * __PAGE_KERNEL_* includes _PAGE_GLOBAL, which we want since
182 * this is mapped to userspace.
183 */
184 pte = __pte(__pa(stack_page) | ((__PAGE_KERNEL_RO | _PAGE_ENC) & ptemask));
185 for (n = 0; n < ESPFIX_PTE_CLONES; n++)
186 set_pte(&pte_p[n*PTE_STRIDE], pte);
187
188 /* Job is done for this CPU and any CPU which shares this page */
189 WRITE_ONCE(espfix_pages[page], stack_page);
190
191unlock_done:
192 mutex_unlock(&espfix_init_mutex);
193done:
194 per_cpu(espfix_stack, cpu) = addr;
195 per_cpu(espfix_waddr, cpu) = (unsigned long)stack_page
196 + (addr & ~PAGE_MASK);
197}