1/* ----------------------------------------------------------------------- *
  2 *
  3 *   Copyright 2014 Intel Corporation; author: H. Peter Anvin
  4 *
  5 *   This program is free software; you can redistribute it and/or modify it
  6 *   under the terms and conditions of the GNU General Public License,
  7 *   version 2, as published by the Free Software Foundation.
  8 *
  9 *   This program is distributed in the hope it will be useful, but WITHOUT
 10 *   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11 *   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 12 *   more details.
 13 *
 14 * ----------------------------------------------------------------------- */
 15
 16/*
 17 * The IRET instruction, when returning to a 16-bit segment, only
 18 * restores the bottom 16 bits of the user space stack pointer.  This
 19 * causes some 16-bit software to break, but it also leaks kernel state
 20 * to user space.
 21 *
 22 * This works around this by creating percpu "ministacks", each of which
 23 * is mapped 2^16 times 64K apart.  When we detect that the return SS is
 24 * on the LDT, we copy the IRET frame to the ministack and use the
 25 * relevant alias to return to userspace.  The ministacks are mapped
 26 * readonly, so if the IRET fault we promote #GP to #DF which is an IST
 27 * vector and thus has its own stack; we then do the fixup in the #DF
 28 * handler.
 29 *
 30 * This file sets up the ministacks and the related page tables.  The
 31 * actual ministack invocation is in entry_64.S.
 32 */
 33
 34#include <linux/init.h>
 35#include <linux/init_task.h>
 36#include <linux/kernel.h>
 37#include <linux/percpu.h>
 38#include <linux/gfp.h>
 39#include <linux/random.h>
 40#include <asm/pgtable.h>
 41#include <asm/pgalloc.h>
 42#include <asm/setup.h>
 43#include <asm/espfix.h>
 44
 45/*
 46 * Note: we only need 6*8 = 48 bytes for the espfix stack, but round
 47 * it up to a cache line to avoid unnecessary sharing.
 48 */
 49#define ESPFIX_STACK_SIZE	(8*8UL)
 50#define ESPFIX_STACKS_PER_PAGE	(PAGE_SIZE/ESPFIX_STACK_SIZE)
 51
 52/* There is address space for how many espfix pages? */
 53#define ESPFIX_PAGE_SPACE	(1UL << (P4D_SHIFT-PAGE_SHIFT-16))
 54
 55#define ESPFIX_MAX_CPUS		(ESPFIX_STACKS_PER_PAGE * ESPFIX_PAGE_SPACE)
 56#if CONFIG_NR_CPUS > ESPFIX_MAX_CPUS
 57# error "Need more virtual address space for the ESPFIX hack"
 58#endif
 59
 60#define PGALLOC_GFP (GFP_KERNEL | __GFP_ZERO)
 61
 62/* This contains the *bottom* address of the espfix stack */
 63DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack);
 64DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_waddr);
 65
 66/* Initialization mutex - should this be a spinlock? */
 67static DEFINE_MUTEX(espfix_init_mutex);
 68
 69/* Page allocation bitmap - each page serves ESPFIX_STACKS_PER_PAGE CPUs */
 70#define ESPFIX_MAX_PAGES  DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE)
 71static void *espfix_pages[ESPFIX_MAX_PAGES];
 72
 73static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD]
 74	__aligned(PAGE_SIZE);
 75
 76static unsigned int page_random, slot_random;
 77
 78/*
 79 * This returns the bottom address of the espfix stack for a specific CPU.
 80 * The math allows for a non-power-of-two ESPFIX_STACK_SIZE, in which case
 81 * we have to account for some amount of padding at the end of each page.
 82 */
 83static inline unsigned long espfix_base_addr(unsigned int cpu)
 84{
 85	unsigned long page, slot;
 86	unsigned long addr;
 87
 88	page = (cpu / ESPFIX_STACKS_PER_PAGE) ^ page_random;
 89	slot = (cpu + slot_random) % ESPFIX_STACKS_PER_PAGE;
 90	addr = (page << PAGE_SHIFT) + (slot * ESPFIX_STACK_SIZE);
 91	addr = (addr & 0xffffUL) | ((addr & ~0xffffUL) << 16);
 92	addr += ESPFIX_BASE_ADDR;
 93	return addr;
 94}
 95
 96#define PTE_STRIDE        (65536/PAGE_SIZE)
 97#define ESPFIX_PTE_CLONES (PTRS_PER_PTE/PTE_STRIDE)
 98#define ESPFIX_PMD_CLONES PTRS_PER_PMD
 99#define ESPFIX_PUD_CLONES (65536/(ESPFIX_PTE_CLONES*ESPFIX_PMD_CLONES))
100
101#define PGTABLE_PROT	  ((_KERNPG_TABLE & ~_PAGE_RW) | _PAGE_NX)
102
103static void init_espfix_random(void)
104{
105	unsigned long rand;
106
107	/*
108	 * This is run before the entropy pools are initialized,
109	 * but this is hopefully better than nothing.
110	 */
111	if (!arch_get_random_long(&rand)) {
112		/* The constant is an arbitrary large prime */
113		rand = rdtsc();
114		rand *= 0xc345c6b72fd16123UL;
115	}
116
117	slot_random = rand % ESPFIX_STACKS_PER_PAGE;
118	page_random = (rand / ESPFIX_STACKS_PER_PAGE)
119		& (ESPFIX_PAGE_SPACE - 1);
120}
121
122void __init init_espfix_bsp(void)
123{
124	pgd_t *pgd;
125	p4d_t *p4d;
126
127	/* Install the espfix pud into the kernel page directory */
128	pgd = &init_top_pgt[pgd_index(ESPFIX_BASE_ADDR)];
129	p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR);
130	p4d_populate(&init_mm, p4d, espfix_pud_page);
131
132	/* Randomize the locations */
133	init_espfix_random();
134
135	/* The rest is the same as for any other processor */
136	init_espfix_ap(0);
137}
138
139void init_espfix_ap(int cpu)
140{
141	unsigned int page;
142	unsigned long addr;
143	pud_t pud, *pud_p;
144	pmd_t pmd, *pmd_p;
145	pte_t pte, *pte_p;
146	int n, node;
147	void *stack_page;
148	pteval_t ptemask;
149
150	/* We only have to do this once... */
151	if (likely(per_cpu(espfix_stack, cpu)))
152		return;		/* Already initialized */
153
154	addr = espfix_base_addr(cpu);
155	page = cpu/ESPFIX_STACKS_PER_PAGE;
156
157	/* Did another CPU already set this up? */
158	stack_page = READ_ONCE(espfix_pages[page]);
159	if (likely(stack_page))
160		goto done;
161
162	mutex_lock(&espfix_init_mutex);
163
164	/* Did we race on the lock? */
165	stack_page = READ_ONCE(espfix_pages[page]);
166	if (stack_page)
167		goto unlock_done;
168
169	node = cpu_to_node(cpu);
170	ptemask = __supported_pte_mask;
171
172	pud_p = &espfix_pud_page[pud_index(addr)];
173	pud = *pud_p;
174	if (!pud_present(pud)) {
175		struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
176
177		pmd_p = (pmd_t *)page_address(page);
178		pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask));
179		paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT);
180		for (n = 0; n < ESPFIX_PUD_CLONES; n++)
181			set_pud(&pud_p[n], pud);
182	}
183
184	pmd_p = pmd_offset(&pud, addr);
185	pmd = *pmd_p;
186	if (!pmd_present(pmd)) {
187		struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0);
188
189		pte_p = (pte_t *)page_address(page);
190		pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask));
191		paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT);
192		for (n = 0; n < ESPFIX_PMD_CLONES; n++)
193			set_pmd(&pmd_p[n], pmd);
194	}
195
196	pte_p = pte_offset_kernel(&pmd, addr);
197	stack_page = page_address(alloc_pages_node(node, GFP_KERNEL, 0));
198	/*
199	 * __PAGE_KERNEL_* includes _PAGE_GLOBAL, which we want since
200	 * this is mapped to userspace.
201	 */
202	pte = __pte(__pa(stack_page) | ((__PAGE_KERNEL_RO | _PAGE_ENC) & ptemask));
203	for (n = 0; n < ESPFIX_PTE_CLONES; n++)
204		set_pte(&pte_p[n*PTE_STRIDE], pte);
205
206	/* Job is done for this CPU and any CPU which shares this page */
207	WRITE_ONCE(espfix_pages[page], stack_page);
208
209unlock_done:
210	mutex_unlock(&espfix_init_mutex);
211done:
212	per_cpu(espfix_stack, cpu) = addr;
213	per_cpu(espfix_waddr, cpu) = (unsigned long)stack_page
214				      + (addr & ~PAGE_MASK);
215}