Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * AMD Memory Encryption Support
4 *
5 * Copyright (C) 2016 Advanced Micro Devices, Inc.
6 *
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 */
9
10#define DISABLE_BRANCH_PROFILING
11
12#include <linux/linkage.h>
13#include <linux/init.h>
14#include <linux/mm.h>
15#include <linux/dma-direct.h>
16#include <linux/swiotlb.h>
17#include <linux/mem_encrypt.h>
18#include <linux/device.h>
19#include <linux/kernel.h>
20#include <linux/bitops.h>
21#include <linux/dma-mapping.h>
22#include <linux/virtio_config.h>
23#include <linux/virtio_anchor.h>
24#include <linux/cc_platform.h>
25
26#include <asm/tlbflush.h>
27#include <asm/fixmap.h>
28#include <asm/setup.h>
29#include <asm/mem_encrypt.h>
30#include <asm/bootparam.h>
31#include <asm/set_memory.h>
32#include <asm/cacheflush.h>
33#include <asm/processor-flags.h>
34#include <asm/msr.h>
35#include <asm/cmdline.h>
36#include <asm/sev.h>
37
38#include "mm_internal.h"
39
40/*
41 * Since SME related variables are set early in the boot process they must
42 * reside in the .data section so as not to be zeroed out when the .bss
43 * section is later cleared.
44 */
45u64 sme_me_mask __section(".data") = 0;
46u64 sev_status __section(".data") = 0;
47u64 sev_check_data __section(".data") = 0;
48EXPORT_SYMBOL(sme_me_mask);
49
50/* Buffer used for early in-place encryption by BSP, no locking needed */
51static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);
52
53/*
54 * SNP-specific routine which needs to additionally change the page state from
55 * private to shared before copying the data from the source to destination and
56 * restore after the copy.
57 */
58static inline void __init snp_memcpy(void *dst, void *src, size_t sz,
59 unsigned long paddr, bool decrypt)
60{
61 unsigned long npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
62
63 if (decrypt) {
64 /*
65 * @paddr needs to be accessed decrypted, mark the page shared in
66 * the RMP table before copying it.
67 */
68 early_snp_set_memory_shared((unsigned long)__va(paddr), paddr, npages);
69
70 memcpy(dst, src, sz);
71
72 /* Restore the page state after the memcpy. */
73 early_snp_set_memory_private((unsigned long)__va(paddr), paddr, npages);
74 } else {
75 /*
76 * @paddr need to be accessed encrypted, no need for the page state
77 * change.
78 */
79 memcpy(dst, src, sz);
80 }
81}
82
83/*
84 * This routine does not change the underlying encryption setting of the
85 * page(s) that map this memory. It assumes that eventually the memory is
86 * meant to be accessed as either encrypted or decrypted but the contents
87 * are currently not in the desired state.
88 *
89 * This routine follows the steps outlined in the AMD64 Architecture
90 * Programmer's Manual Volume 2, Section 7.10.8 Encrypt-in-Place.
91 */
92static void __init __sme_early_enc_dec(resource_size_t paddr,
93 unsigned long size, bool enc)
94{
95 void *src, *dst;
96 size_t len;
97
98 if (!sme_me_mask)
99 return;
100
101 wbinvd();
102
103 /*
104 * There are limited number of early mapping slots, so map (at most)
105 * one page at time.
106 */
107 while (size) {
108 len = min_t(size_t, sizeof(sme_early_buffer), size);
109
110 /*
111 * Create mappings for the current and desired format of
112 * the memory. Use a write-protected mapping for the source.
113 */
114 src = enc ? early_memremap_decrypted_wp(paddr, len) :
115 early_memremap_encrypted_wp(paddr, len);
116
117 dst = enc ? early_memremap_encrypted(paddr, len) :
118 early_memremap_decrypted(paddr, len);
119
120 /*
121 * If a mapping can't be obtained to perform the operation,
122 * then eventual access of that area in the desired mode
123 * will cause a crash.
124 */
125 BUG_ON(!src || !dst);
126
127 /*
128 * Use a temporary buffer, of cache-line multiple size, to
129 * avoid data corruption as documented in the APM.
130 */
131 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
132 snp_memcpy(sme_early_buffer, src, len, paddr, enc);
133 snp_memcpy(dst, sme_early_buffer, len, paddr, !enc);
134 } else {
135 memcpy(sme_early_buffer, src, len);
136 memcpy(dst, sme_early_buffer, len);
137 }
138
139 early_memunmap(dst, len);
140 early_memunmap(src, len);
141
142 paddr += len;
143 size -= len;
144 }
145}
146
147void __init sme_early_encrypt(resource_size_t paddr, unsigned long size)
148{
149 __sme_early_enc_dec(paddr, size, true);
150}
151
152void __init sme_early_decrypt(resource_size_t paddr, unsigned long size)
153{
154 __sme_early_enc_dec(paddr, size, false);
155}
156
157static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size,
158 bool map)
159{
160 unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET;
161 pmdval_t pmd_flags, pmd;
162
163 /* Use early_pmd_flags but remove the encryption mask */
164 pmd_flags = __sme_clr(early_pmd_flags);
165
166 do {
167 pmd = map ? (paddr & PMD_MASK) + pmd_flags : 0;
168 __early_make_pgtable((unsigned long)vaddr, pmd);
169
170 vaddr += PMD_SIZE;
171 paddr += PMD_SIZE;
172 size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE;
173 } while (size);
174
175 flush_tlb_local();
176}
177
178void __init sme_unmap_bootdata(char *real_mode_data)
179{
180 struct boot_params *boot_data;
181 unsigned long cmdline_paddr;
182
183 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
184 return;
185
186 /* Get the command line address before unmapping the real_mode_data */
187 boot_data = (struct boot_params *)real_mode_data;
188 cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
189
190 __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), false);
191
192 if (!cmdline_paddr)
193 return;
194
195 __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, false);
196}
197
198void __init sme_map_bootdata(char *real_mode_data)
199{
200 struct boot_params *boot_data;
201 unsigned long cmdline_paddr;
202
203 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
204 return;
205
206 __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), true);
207
208 /* Get the command line address after mapping the real_mode_data */
209 boot_data = (struct boot_params *)real_mode_data;
210 cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
211
212 if (!cmdline_paddr)
213 return;
214
215 __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true);
216}
217
218void __init sev_setup_arch(void)
219{
220 phys_addr_t total_mem = memblock_phys_mem_size();
221 unsigned long size;
222
223 if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
224 return;
225
226 /*
227 * For SEV, all DMA has to occur via shared/unencrypted pages.
228 * SEV uses SWIOTLB to make this happen without changing device
229 * drivers. However, depending on the workload being run, the
230 * default 64MB of SWIOTLB may not be enough and SWIOTLB may
231 * run out of buffers for DMA, resulting in I/O errors and/or
232 * performance degradation especially with high I/O workloads.
233 *
234 * Adjust the default size of SWIOTLB for SEV guests using
235 * a percentage of guest memory for SWIOTLB buffers.
236 * Also, as the SWIOTLB bounce buffer memory is allocated
237 * from low memory, ensure that the adjusted size is within
238 * the limits of low available memory.
239 *
240 * The percentage of guest memory used here for SWIOTLB buffers
241 * is more of an approximation of the static adjustment which
242 * 64MB for <1G, and ~128M to 256M for 1G-to-4G, i.e., the 6%
243 */
244 size = total_mem * 6 / 100;
245 size = clamp_val(size, IO_TLB_DEFAULT_SIZE, SZ_1G);
246 swiotlb_adjust_size(size);
247
248 /* Set restricted memory access for virtio. */
249 virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc);
250}
251
252static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot)
253{
254 unsigned long pfn = 0;
255 pgprot_t prot;
256
257 switch (level) {
258 case PG_LEVEL_4K:
259 pfn = pte_pfn(*kpte);
260 prot = pte_pgprot(*kpte);
261 break;
262 case PG_LEVEL_2M:
263 pfn = pmd_pfn(*(pmd_t *)kpte);
264 prot = pmd_pgprot(*(pmd_t *)kpte);
265 break;
266 case PG_LEVEL_1G:
267 pfn = pud_pfn(*(pud_t *)kpte);
268 prot = pud_pgprot(*(pud_t *)kpte);
269 break;
270 default:
271 WARN_ONCE(1, "Invalid level for kpte\n");
272 return 0;
273 }
274
275 if (ret_prot)
276 *ret_prot = prot;
277
278 return pfn;
279}
280
281static bool amd_enc_tlb_flush_required(bool enc)
282{
283 return true;
284}
285
286static bool amd_enc_cache_flush_required(void)
287{
288 return !cpu_feature_enabled(X86_FEATURE_SME_COHERENT);
289}
290
291static void enc_dec_hypercall(unsigned long vaddr, int npages, bool enc)
292{
293#ifdef CONFIG_PARAVIRT
294 unsigned long sz = npages << PAGE_SHIFT;
295 unsigned long vaddr_end = vaddr + sz;
296
297 while (vaddr < vaddr_end) {
298 int psize, pmask, level;
299 unsigned long pfn;
300 pte_t *kpte;
301
302 kpte = lookup_address(vaddr, &level);
303 if (!kpte || pte_none(*kpte)) {
304 WARN_ONCE(1, "kpte lookup for vaddr\n");
305 return;
306 }
307
308 pfn = pg_level_to_pfn(level, kpte, NULL);
309 if (!pfn)
310 continue;
311
312 psize = page_level_size(level);
313 pmask = page_level_mask(level);
314
315 notify_page_enc_status_changed(pfn, psize >> PAGE_SHIFT, enc);
316
317 vaddr = (vaddr & pmask) + psize;
318 }
319#endif
320}
321
322static void amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool enc)
323{
324 /*
325 * To maintain the security guarantees of SEV-SNP guests, make sure
326 * to invalidate the memory before encryption attribute is cleared.
327 */
328 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && !enc)
329 snp_set_memory_shared(vaddr, npages);
330}
331
332/* Return true unconditionally: return value doesn't matter for the SEV side */
333static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool enc)
334{
335 /*
336 * After memory is mapped encrypted in the page table, validate it
337 * so that it is consistent with the page table updates.
338 */
339 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && enc)
340 snp_set_memory_private(vaddr, npages);
341
342 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
343 enc_dec_hypercall(vaddr, npages, enc);
344
345 return true;
346}
347
348static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
349{
350 pgprot_t old_prot, new_prot;
351 unsigned long pfn, pa, size;
352 pte_t new_pte;
353
354 pfn = pg_level_to_pfn(level, kpte, &old_prot);
355 if (!pfn)
356 return;
357
358 new_prot = old_prot;
359 if (enc)
360 pgprot_val(new_prot) |= _PAGE_ENC;
361 else
362 pgprot_val(new_prot) &= ~_PAGE_ENC;
363
364 /* If prot is same then do nothing. */
365 if (pgprot_val(old_prot) == pgprot_val(new_prot))
366 return;
367
368 pa = pfn << PAGE_SHIFT;
369 size = page_level_size(level);
370
371 /*
372 * We are going to perform in-place en-/decryption and change the
373 * physical page attribute from C=1 to C=0 or vice versa. Flush the
374 * caches to ensure that data gets accessed with the correct C-bit.
375 */
376 clflush_cache_range(__va(pa), size);
377
378 /* Encrypt/decrypt the contents in-place */
379 if (enc) {
380 sme_early_encrypt(pa, size);
381 } else {
382 sme_early_decrypt(pa, size);
383
384 /*
385 * ON SNP, the page state in the RMP table must happen
386 * before the page table updates.
387 */
388 early_snp_set_memory_shared((unsigned long)__va(pa), pa, 1);
389 }
390
391 /* Change the page encryption mask. */
392 new_pte = pfn_pte(pfn, new_prot);
393 set_pte_atomic(kpte, new_pte);
394
395 /*
396 * If page is set encrypted in the page table, then update the RMP table to
397 * add this page as private.
398 */
399 if (enc)
400 early_snp_set_memory_private((unsigned long)__va(pa), pa, 1);
401}
402
403static int __init early_set_memory_enc_dec(unsigned long vaddr,
404 unsigned long size, bool enc)
405{
406 unsigned long vaddr_end, vaddr_next, start;
407 unsigned long psize, pmask;
408 int split_page_size_mask;
409 int level, ret;
410 pte_t *kpte;
411
412 start = vaddr;
413 vaddr_next = vaddr;
414 vaddr_end = vaddr + size;
415
416 for (; vaddr < vaddr_end; vaddr = vaddr_next) {
417 kpte = lookup_address(vaddr, &level);
418 if (!kpte || pte_none(*kpte)) {
419 ret = 1;
420 goto out;
421 }
422
423 if (level == PG_LEVEL_4K) {
424 __set_clr_pte_enc(kpte, level, enc);
425 vaddr_next = (vaddr & PAGE_MASK) + PAGE_SIZE;
426 continue;
427 }
428
429 psize = page_level_size(level);
430 pmask = page_level_mask(level);
431
432 /*
433 * Check whether we can change the large page in one go.
434 * We request a split when the address is not aligned and
435 * the number of pages to set/clear encryption bit is smaller
436 * than the number of pages in the large page.
437 */
438 if (vaddr == (vaddr & pmask) &&
439 ((vaddr_end - vaddr) >= psize)) {
440 __set_clr_pte_enc(kpte, level, enc);
441 vaddr_next = (vaddr & pmask) + psize;
442 continue;
443 }
444
445 /*
446 * The virtual address is part of a larger page, create the next
447 * level page table mapping (4K or 2M). If it is part of a 2M
448 * page then we request a split of the large page into 4K
449 * chunks. A 1GB large page is split into 2M pages, resp.
450 */
451 if (level == PG_LEVEL_2M)
452 split_page_size_mask = 0;
453 else
454 split_page_size_mask = 1 << PG_LEVEL_2M;
455
456 /*
457 * kernel_physical_mapping_change() does not flush the TLBs, so
458 * a TLB flush is required after we exit from the for loop.
459 */
460 kernel_physical_mapping_change(__pa(vaddr & pmask),
461 __pa((vaddr_end & pmask) + psize),
462 split_page_size_mask);
463 }
464
465 ret = 0;
466
467 early_set_mem_enc_dec_hypercall(start, PAGE_ALIGN(size) >> PAGE_SHIFT, enc);
468out:
469 __flush_tlb_all();
470 return ret;
471}
472
473int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size)
474{
475 return early_set_memory_enc_dec(vaddr, size, false);
476}
477
478int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)
479{
480 return early_set_memory_enc_dec(vaddr, size, true);
481}
482
483void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, int npages, bool enc)
484{
485 enc_dec_hypercall(vaddr, npages, enc);
486}
487
488void __init sme_early_init(void)
489{
490 if (!sme_me_mask)
491 return;
492
493 early_pmd_flags = __sme_set(early_pmd_flags);
494
495 __supported_pte_mask = __sme_set(__supported_pte_mask);
496
497 /* Update the protection map with memory encryption mask */
498 add_encrypt_protection_map();
499
500 x86_platform.guest.enc_status_change_prepare = amd_enc_status_change_prepare;
501 x86_platform.guest.enc_status_change_finish = amd_enc_status_change_finish;
502 x86_platform.guest.enc_tlb_flush_required = amd_enc_tlb_flush_required;
503 x86_platform.guest.enc_cache_flush_required = amd_enc_cache_flush_required;
504}
505
506void __init mem_encrypt_free_decrypted_mem(void)
507{
508 unsigned long vaddr, vaddr_end, npages;
509 int r;
510
511 vaddr = (unsigned long)__start_bss_decrypted_unused;
512 vaddr_end = (unsigned long)__end_bss_decrypted;
513 npages = (vaddr_end - vaddr) >> PAGE_SHIFT;
514
515 /*
516 * The unused memory range was mapped decrypted, change the encryption
517 * attribute from decrypted to encrypted before freeing it.
518 */
519 if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
520 r = set_memory_encrypted(vaddr, npages);
521 if (r) {
522 pr_warn("failed to free unused decrypted pages\n");
523 return;
524 }
525 }
526
527 free_init_pages("unused decrypted", vaddr, vaddr_end);
528}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * AMD Memory Encryption Support
4 *
5 * Copyright (C) 2016 Advanced Micro Devices, Inc.
6 *
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 */
9
10#define DISABLE_BRANCH_PROFILING
11
12#include <linux/linkage.h>
13#include <linux/init.h>
14#include <linux/mm.h>
15#include <linux/dma-direct.h>
16#include <linux/swiotlb.h>
17#include <linux/mem_encrypt.h>
18#include <linux/device.h>
19#include <linux/kernel.h>
20#include <linux/bitops.h>
21#include <linux/dma-mapping.h>
22#include <linux/cc_platform.h>
23
24#include <asm/tlbflush.h>
25#include <asm/fixmap.h>
26#include <asm/setup.h>
27#include <asm/mem_encrypt.h>
28#include <asm/bootparam.h>
29#include <asm/set_memory.h>
30#include <asm/cacheflush.h>
31#include <asm/processor-flags.h>
32#include <asm/msr.h>
33#include <asm/cmdline.h>
34#include <asm/sev.h>
35#include <asm/ia32.h>
36
37#include "mm_internal.h"
38
39/*
40 * Since SME related variables are set early in the boot process they must
41 * reside in the .data section so as not to be zeroed out when the .bss
42 * section is later cleared.
43 */
44u64 sme_me_mask __section(".data") = 0;
45u64 sev_status __section(".data") = 0;
46u64 sev_check_data __section(".data") = 0;
47EXPORT_SYMBOL(sme_me_mask);
48
49/* Buffer used for early in-place encryption by BSP, no locking needed */
50static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);
51
52/*
53 * SNP-specific routine which needs to additionally change the page state from
54 * private to shared before copying the data from the source to destination and
55 * restore after the copy.
56 */
57static inline void __init snp_memcpy(void *dst, void *src, size_t sz,
58 unsigned long paddr, bool decrypt)
59{
60 unsigned long npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
61
62 if (decrypt) {
63 /*
64 * @paddr needs to be accessed decrypted, mark the page shared in
65 * the RMP table before copying it.
66 */
67 early_snp_set_memory_shared((unsigned long)__va(paddr), paddr, npages);
68
69 memcpy(dst, src, sz);
70
71 /* Restore the page state after the memcpy. */
72 early_snp_set_memory_private((unsigned long)__va(paddr), paddr, npages);
73 } else {
74 /*
75 * @paddr need to be accessed encrypted, no need for the page state
76 * change.
77 */
78 memcpy(dst, src, sz);
79 }
80}
81
82/*
83 * This routine does not change the underlying encryption setting of the
84 * page(s) that map this memory. It assumes that eventually the memory is
85 * meant to be accessed as either encrypted or decrypted but the contents
86 * are currently not in the desired state.
87 *
88 * This routine follows the steps outlined in the AMD64 Architecture
89 * Programmer's Manual Volume 2, Section 7.10.8 Encrypt-in-Place.
90 */
91static void __init __sme_early_enc_dec(resource_size_t paddr,
92 unsigned long size, bool enc)
93{
94 void *src, *dst;
95 size_t len;
96
97 if (!sme_me_mask)
98 return;
99
100 wbinvd();
101
102 /*
103 * There are limited number of early mapping slots, so map (at most)
104 * one page at time.
105 */
106 while (size) {
107 len = min_t(size_t, sizeof(sme_early_buffer), size);
108
109 /*
110 * Create mappings for the current and desired format of
111 * the memory. Use a write-protected mapping for the source.
112 */
113 src = enc ? early_memremap_decrypted_wp(paddr, len) :
114 early_memremap_encrypted_wp(paddr, len);
115
116 dst = enc ? early_memremap_encrypted(paddr, len) :
117 early_memremap_decrypted(paddr, len);
118
119 /*
120 * If a mapping can't be obtained to perform the operation,
121 * then eventual access of that area in the desired mode
122 * will cause a crash.
123 */
124 BUG_ON(!src || !dst);
125
126 /*
127 * Use a temporary buffer, of cache-line multiple size, to
128 * avoid data corruption as documented in the APM.
129 */
130 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
131 snp_memcpy(sme_early_buffer, src, len, paddr, enc);
132 snp_memcpy(dst, sme_early_buffer, len, paddr, !enc);
133 } else {
134 memcpy(sme_early_buffer, src, len);
135 memcpy(dst, sme_early_buffer, len);
136 }
137
138 early_memunmap(dst, len);
139 early_memunmap(src, len);
140
141 paddr += len;
142 size -= len;
143 }
144}
145
146void __init sme_early_encrypt(resource_size_t paddr, unsigned long size)
147{
148 __sme_early_enc_dec(paddr, size, true);
149}
150
151void __init sme_early_decrypt(resource_size_t paddr, unsigned long size)
152{
153 __sme_early_enc_dec(paddr, size, false);
154}
155
156static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size,
157 bool map)
158{
159 unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET;
160 pmdval_t pmd_flags, pmd;
161
162 /* Use early_pmd_flags but remove the encryption mask */
163 pmd_flags = __sme_clr(early_pmd_flags);
164
165 do {
166 pmd = map ? (paddr & PMD_MASK) + pmd_flags : 0;
167 __early_make_pgtable((unsigned long)vaddr, pmd);
168
169 vaddr += PMD_SIZE;
170 paddr += PMD_SIZE;
171 size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE;
172 } while (size);
173
174 flush_tlb_local();
175}
176
177void __init sme_unmap_bootdata(char *real_mode_data)
178{
179 struct boot_params *boot_data;
180 unsigned long cmdline_paddr;
181
182 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
183 return;
184
185 /* Get the command line address before unmapping the real_mode_data */
186 boot_data = (struct boot_params *)real_mode_data;
187 cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
188
189 __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), false);
190
191 if (!cmdline_paddr)
192 return;
193
194 __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, false);
195}
196
197void __init sme_map_bootdata(char *real_mode_data)
198{
199 struct boot_params *boot_data;
200 unsigned long cmdline_paddr;
201
202 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
203 return;
204
205 __sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), true);
206
207 /* Get the command line address after mapping the real_mode_data */
208 boot_data = (struct boot_params *)real_mode_data;
209 cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
210
211 if (!cmdline_paddr)
212 return;
213
214 __sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true);
215}
216
217static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot)
218{
219 unsigned long pfn = 0;
220 pgprot_t prot;
221
222 switch (level) {
223 case PG_LEVEL_4K:
224 pfn = pte_pfn(*kpte);
225 prot = pte_pgprot(*kpte);
226 break;
227 case PG_LEVEL_2M:
228 pfn = pmd_pfn(*(pmd_t *)kpte);
229 prot = pmd_pgprot(*(pmd_t *)kpte);
230 break;
231 case PG_LEVEL_1G:
232 pfn = pud_pfn(*(pud_t *)kpte);
233 prot = pud_pgprot(*(pud_t *)kpte);
234 break;
235 default:
236 WARN_ONCE(1, "Invalid level for kpte\n");
237 return 0;
238 }
239
240 if (ret_prot)
241 *ret_prot = prot;
242
243 return pfn;
244}
245
246static bool amd_enc_tlb_flush_required(bool enc)
247{
248 return true;
249}
250
251static bool amd_enc_cache_flush_required(void)
252{
253 return !cpu_feature_enabled(X86_FEATURE_SME_COHERENT);
254}
255
256static void enc_dec_hypercall(unsigned long vaddr, unsigned long size, bool enc)
257{
258#ifdef CONFIG_PARAVIRT
259 unsigned long vaddr_end = vaddr + size;
260
261 while (vaddr < vaddr_end) {
262 int psize, pmask, level;
263 unsigned long pfn;
264 pte_t *kpte;
265
266 kpte = lookup_address(vaddr, &level);
267 if (!kpte || pte_none(*kpte)) {
268 WARN_ONCE(1, "kpte lookup for vaddr\n");
269 return;
270 }
271
272 pfn = pg_level_to_pfn(level, kpte, NULL);
273 if (!pfn)
274 continue;
275
276 psize = page_level_size(level);
277 pmask = page_level_mask(level);
278
279 notify_page_enc_status_changed(pfn, psize >> PAGE_SHIFT, enc);
280
281 vaddr = (vaddr & pmask) + psize;
282 }
283#endif
284}
285
286static bool amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool enc)
287{
288 /*
289 * To maintain the security guarantees of SEV-SNP guests, make sure
290 * to invalidate the memory before encryption attribute is cleared.
291 */
292 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && !enc)
293 snp_set_memory_shared(vaddr, npages);
294
295 return true;
296}
297
298/* Return true unconditionally: return value doesn't matter for the SEV side */
299static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool enc)
300{
301 /*
302 * After memory is mapped encrypted in the page table, validate it
303 * so that it is consistent with the page table updates.
304 */
305 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && enc)
306 snp_set_memory_private(vaddr, npages);
307
308 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
309 enc_dec_hypercall(vaddr, npages << PAGE_SHIFT, enc);
310
311 return true;
312}
313
314static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
315{
316 pgprot_t old_prot, new_prot;
317 unsigned long pfn, pa, size;
318 pte_t new_pte;
319
320 pfn = pg_level_to_pfn(level, kpte, &old_prot);
321 if (!pfn)
322 return;
323
324 new_prot = old_prot;
325 if (enc)
326 pgprot_val(new_prot) |= _PAGE_ENC;
327 else
328 pgprot_val(new_prot) &= ~_PAGE_ENC;
329
330 /* If prot is same then do nothing. */
331 if (pgprot_val(old_prot) == pgprot_val(new_prot))
332 return;
333
334 pa = pfn << PAGE_SHIFT;
335 size = page_level_size(level);
336
337 /*
338 * We are going to perform in-place en-/decryption and change the
339 * physical page attribute from C=1 to C=0 or vice versa. Flush the
340 * caches to ensure that data gets accessed with the correct C-bit.
341 */
342 clflush_cache_range(__va(pa), size);
343
344 /* Encrypt/decrypt the contents in-place */
345 if (enc) {
346 sme_early_encrypt(pa, size);
347 } else {
348 sme_early_decrypt(pa, size);
349
350 /*
351 * ON SNP, the page state in the RMP table must happen
352 * before the page table updates.
353 */
354 early_snp_set_memory_shared((unsigned long)__va(pa), pa, 1);
355 }
356
357 /* Change the page encryption mask. */
358 new_pte = pfn_pte(pfn, new_prot);
359 set_pte_atomic(kpte, new_pte);
360
361 /*
362 * If page is set encrypted in the page table, then update the RMP table to
363 * add this page as private.
364 */
365 if (enc)
366 early_snp_set_memory_private((unsigned long)__va(pa), pa, 1);
367}
368
369static int __init early_set_memory_enc_dec(unsigned long vaddr,
370 unsigned long size, bool enc)
371{
372 unsigned long vaddr_end, vaddr_next, start;
373 unsigned long psize, pmask;
374 int split_page_size_mask;
375 int level, ret;
376 pte_t *kpte;
377
378 start = vaddr;
379 vaddr_next = vaddr;
380 vaddr_end = vaddr + size;
381
382 for (; vaddr < vaddr_end; vaddr = vaddr_next) {
383 kpte = lookup_address(vaddr, &level);
384 if (!kpte || pte_none(*kpte)) {
385 ret = 1;
386 goto out;
387 }
388
389 if (level == PG_LEVEL_4K) {
390 __set_clr_pte_enc(kpte, level, enc);
391 vaddr_next = (vaddr & PAGE_MASK) + PAGE_SIZE;
392 continue;
393 }
394
395 psize = page_level_size(level);
396 pmask = page_level_mask(level);
397
398 /*
399 * Check whether we can change the large page in one go.
400 * We request a split when the address is not aligned and
401 * the number of pages to set/clear encryption bit is smaller
402 * than the number of pages in the large page.
403 */
404 if (vaddr == (vaddr & pmask) &&
405 ((vaddr_end - vaddr) >= psize)) {
406 __set_clr_pte_enc(kpte, level, enc);
407 vaddr_next = (vaddr & pmask) + psize;
408 continue;
409 }
410
411 /*
412 * The virtual address is part of a larger page, create the next
413 * level page table mapping (4K or 2M). If it is part of a 2M
414 * page then we request a split of the large page into 4K
415 * chunks. A 1GB large page is split into 2M pages, resp.
416 */
417 if (level == PG_LEVEL_2M)
418 split_page_size_mask = 0;
419 else
420 split_page_size_mask = 1 << PG_LEVEL_2M;
421
422 /*
423 * kernel_physical_mapping_change() does not flush the TLBs, so
424 * a TLB flush is required after we exit from the for loop.
425 */
426 kernel_physical_mapping_change(__pa(vaddr & pmask),
427 __pa((vaddr_end & pmask) + psize),
428 split_page_size_mask);
429 }
430
431 ret = 0;
432
433 early_set_mem_enc_dec_hypercall(start, size, enc);
434out:
435 __flush_tlb_all();
436 return ret;
437}
438
439int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size)
440{
441 return early_set_memory_enc_dec(vaddr, size, false);
442}
443
444int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)
445{
446 return early_set_memory_enc_dec(vaddr, size, true);
447}
448
449void __init early_set_mem_enc_dec_hypercall(unsigned long vaddr, unsigned long size, bool enc)
450{
451 enc_dec_hypercall(vaddr, size, enc);
452}
453
454void __init sme_early_init(void)
455{
456 if (!sme_me_mask)
457 return;
458
459 early_pmd_flags = __sme_set(early_pmd_flags);
460
461 __supported_pte_mask = __sme_set(__supported_pte_mask);
462
463 /* Update the protection map with memory encryption mask */
464 add_encrypt_protection_map();
465
466 x86_platform.guest.enc_status_change_prepare = amd_enc_status_change_prepare;
467 x86_platform.guest.enc_status_change_finish = amd_enc_status_change_finish;
468 x86_platform.guest.enc_tlb_flush_required = amd_enc_tlb_flush_required;
469 x86_platform.guest.enc_cache_flush_required = amd_enc_cache_flush_required;
470
471 /*
472 * AMD-SEV-ES intercepts the RDMSR to read the X2APIC ID in the
473 * parallel bringup low level code. That raises #VC which cannot be
474 * handled there.
475 * It does not provide a RDMSR GHCB protocol so the early startup
476 * code cannot directly communicate with the secure firmware. The
477 * alternative solution to retrieve the APIC ID via CPUID(0xb),
478 * which is covered by the GHCB protocol, is not viable either
479 * because there is no enforcement of the CPUID(0xb) provided
480 * "initial" APIC ID to be the same as the real APIC ID.
481 * Disable parallel bootup.
482 */
483 if (sev_status & MSR_AMD64_SEV_ES_ENABLED)
484 x86_cpuinit.parallel_bringup = false;
485
486 /*
487 * The VMM is capable of injecting interrupt 0x80 and triggering the
488 * compatibility syscall path.
489 *
490 * By default, the 32-bit emulation is disabled in order to ensure
491 * the safety of the VM.
492 */
493 if (sev_status & MSR_AMD64_SEV_ENABLED)
494 ia32_disable();
495}
496
497void __init mem_encrypt_free_decrypted_mem(void)
498{
499 unsigned long vaddr, vaddr_end, npages;
500 int r;
501
502 vaddr = (unsigned long)__start_bss_decrypted_unused;
503 vaddr_end = (unsigned long)__end_bss_decrypted;
504 npages = (vaddr_end - vaddr) >> PAGE_SHIFT;
505
506 /*
507 * If the unused memory range was mapped decrypted, change the encryption
508 * attribute from decrypted to encrypted before freeing it. Base the
509 * re-encryption on the same condition used for the decryption in
510 * sme_postprocess_startup(). Higher level abstractions, such as
511 * CC_ATTR_MEM_ENCRYPT, aren't necessarily equivalent in a Hyper-V VM
512 * using vTOM, where sme_me_mask is always zero.
513 */
514 if (sme_me_mask) {
515 r = set_memory_encrypted(vaddr, npages);
516 if (r) {
517 pr_warn("failed to free unused decrypted pages\n");
518 return;
519 }
520 }
521
522 free_init_pages("unused decrypted", vaddr, vaddr_end);
523}