1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/io.h>
  3#include <linux/slab.h>
  4#include <linux/memblock.h>
  5#include <linux/mem_encrypt.h>
  6
  7#include <asm/set_memory.h>
  8#include <asm/pgtable.h>
  9#include <asm/realmode.h>
 10#include <asm/tlbflush.h>
 11
 12struct real_mode_header *real_mode_header;
 13u32 *trampoline_cr4_features;
 14
 15/* Hold the pgd entry used on booting additional CPUs */
 16pgd_t trampoline_pgd_entry;
 17
 18void __init reserve_real_mode(void)
 19{
 20	phys_addr_t mem;
 21	size_t size = real_mode_size_needed();
 22
 23	if (!size)
 24		return;
 25
 26	WARN_ON(slab_is_available());
 27
 28	/* Has to be under 1M so we can execute real-mode AP code. */
 29	mem = memblock_find_in_range(0, 1<<20, size, PAGE_SIZE);
 30	if (!mem) {
 31		pr_info("No sub-1M memory is available for the trampoline\n");
 32		return;
 33	}
 34
 35	memblock_reserve(mem, size);
 36	set_real_mode_mem(mem);
 37}
 38
 39static void __init setup_real_mode(void)
 40{
 41	u16 real_mode_seg;
 42	const u32 *rel;
 43	u32 count;
 44	unsigned char *base;
 45	unsigned long phys_base;
 46	struct trampoline_header *trampoline_header;
 47	size_t size = PAGE_ALIGN(real_mode_blob_end - real_mode_blob);
 48#ifdef CONFIG_X86_64
 49	u64 *trampoline_pgd;
 50	u64 efer;
 51#endif
 52
 53	base = (unsigned char *)real_mode_header;
 54
 55	/*
 56	 * If SME is active, the trampoline area will need to be in
 57	 * decrypted memory in order to bring up other processors
 58	 * successfully. This is not needed for SEV.
 59	 */
 60	if (sme_active())
 61		set_memory_decrypted((unsigned long)base, size >> PAGE_SHIFT);
 62
 63	memcpy(base, real_mode_blob, size);
 64
 65	phys_base = __pa(base);
 66	real_mode_seg = phys_base >> 4;
 67
 68	rel = (u32 *) real_mode_relocs;
 69
 70	/* 16-bit segment relocations. */
 71	count = *rel++;
 72	while (count--) {
 73		u16 *seg = (u16 *) (base + *rel++);
 74		*seg = real_mode_seg;
 75	}
 76
 77	/* 32-bit linear relocations. */
 78	count = *rel++;
 79	while (count--) {
 80		u32 *ptr = (u32 *) (base + *rel++);
 81		*ptr += phys_base;
 82	}
 83
 84	/* Must be perfomed *after* relocation. */
 85	trampoline_header = (struct trampoline_header *)
 86		__va(real_mode_header->trampoline_header);
 87
 88#ifdef CONFIG_X86_32
 89	trampoline_header->start = __pa_symbol(startup_32_smp);
 90	trampoline_header->gdt_limit = __BOOT_DS + 7;
 91	trampoline_header->gdt_base = __pa_symbol(boot_gdt);
 92#else
 93	/*
 94	 * Some AMD processors will #GP(0) if EFER.LMA is set in WRMSR
 95	 * so we need to mask it out.
 96	 */
 97	rdmsrl(MSR_EFER, efer);
 98	trampoline_header->efer = efer & ~EFER_LMA;
 99
100	trampoline_header->start = (u64) secondary_startup_64;
101	trampoline_cr4_features = &trampoline_header->cr4;
102	*trampoline_cr4_features = mmu_cr4_features;
103
104	trampoline_header->flags = 0;
105	if (sme_active())
106		trampoline_header->flags |= TH_FLAGS_SME_ACTIVE;
107
108	trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);
109	trampoline_pgd[0] = trampoline_pgd_entry.pgd;
110	trampoline_pgd[511] = init_top_pgt[511].pgd;
111#endif
112}
113
114/*
115 * reserve_real_mode() gets called very early, to guarantee the
116 * availability of low memory. This is before the proper kernel page
117 * tables are set up, so we cannot set page permissions in that
118 * function. Also trampoline code will be executed by APs so we
119 * need to mark it executable at do_pre_smp_initcalls() at least,
120 * thus run it as a early_initcall().
121 */
122static void __init set_real_mode_permissions(void)
123{
124	unsigned char *base = (unsigned char *) real_mode_header;
125	size_t size = PAGE_ALIGN(real_mode_blob_end - real_mode_blob);
126
127	size_t ro_size =
128		PAGE_ALIGN(real_mode_header->ro_end) -
129		__pa(base);
130
131	size_t text_size =
132		PAGE_ALIGN(real_mode_header->ro_end) -
133		real_mode_header->text_start;
134
135	unsigned long text_start =
136		(unsigned long) __va(real_mode_header->text_start);
137
138	set_memory_nx((unsigned long) base, size >> PAGE_SHIFT);
139	set_memory_ro((unsigned long) base, ro_size >> PAGE_SHIFT);
140	set_memory_x((unsigned long) text_start, text_size >> PAGE_SHIFT);
141}
142
143static int __init init_real_mode(void)
144{
145	if (!real_mode_header)
146		panic("Real mode trampoline was not allocated");
147
148	setup_real_mode();
149	set_real_mode_permissions();
150
151	return 0;
152}
153early_initcall(init_real_mode);