Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2
3#ifndef KVM_X86_MMU_SPTE_H
4#define KVM_X86_MMU_SPTE_H
5
6#include <asm/vmx.h>
7
8#include "mmu.h"
9#include "mmu_internal.h"
10
11/*
12 * A MMU present SPTE is backed by actual memory and may or may not be present
13 * in hardware. E.g. MMIO SPTEs are not considered present. Use bit 11, as it
14 * is ignored by all flavors of SPTEs and checking a low bit often generates
15 * better code than for a high bit, e.g. 56+. MMU present checks are pervasive
16 * enough that the improved code generation is noticeable in KVM's footprint.
17 */
18#define SPTE_MMU_PRESENT_MASK BIT_ULL(11)
19
20/*
21 * TDP SPTES (more specifically, EPT SPTEs) may not have A/D bits, and may also
22 * be restricted to using write-protection (for L2 when CPU dirty logging, i.e.
23 * PML, is enabled). Use bits 52 and 53 to hold the type of A/D tracking that
24 * is must be employed for a given TDP SPTE.
25 *
26 * Note, the "enabled" mask must be '0', as bits 62:52 are _reserved_ for PAE
27 * paging, including NPT PAE. This scheme works because legacy shadow paging
28 * is guaranteed to have A/D bits and write-protection is forced only for
29 * TDP with CPU dirty logging (PML). If NPT ever gains PML-like support, it
30 * must be restricted to 64-bit KVM.
31 */
32#define SPTE_TDP_AD_SHIFT 52
33#define SPTE_TDP_AD_MASK (3ULL << SPTE_TDP_AD_SHIFT)
34#define SPTE_TDP_AD_ENABLED (0ULL << SPTE_TDP_AD_SHIFT)
35#define SPTE_TDP_AD_DISABLED (1ULL << SPTE_TDP_AD_SHIFT)
36#define SPTE_TDP_AD_WRPROT_ONLY (2ULL << SPTE_TDP_AD_SHIFT)
37static_assert(SPTE_TDP_AD_ENABLED == 0);
38
39#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
40#define SPTE_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1))
41#else
42#define SPTE_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
43#endif
44
45#define SPTE_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
46 | shadow_x_mask | shadow_nx_mask | shadow_me_mask)
47
48#define ACC_EXEC_MASK 1
49#define ACC_WRITE_MASK PT_WRITABLE_MASK
50#define ACC_USER_MASK PT_USER_MASK
51#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
52
53/* The mask for the R/X bits in EPT PTEs */
54#define SPTE_EPT_READABLE_MASK 0x1ull
55#define SPTE_EPT_EXECUTABLE_MASK 0x4ull
56
57#define SPTE_LEVEL_BITS 9
58#define SPTE_LEVEL_SHIFT(level) __PT_LEVEL_SHIFT(level, SPTE_LEVEL_BITS)
59#define SPTE_INDEX(address, level) __PT_INDEX(address, level, SPTE_LEVEL_BITS)
60#define SPTE_ENT_PER_PAGE __PT_ENT_PER_PAGE(SPTE_LEVEL_BITS)
61
62/*
63 * The mask/shift to use for saving the original R/X bits when marking the PTE
64 * as not-present for access tracking purposes. We do not save the W bit as the
65 * PTEs being access tracked also need to be dirty tracked, so the W bit will be
66 * restored only when a write is attempted to the page. This mask obviously
67 * must not overlap the A/D type mask.
68 */
69#define SHADOW_ACC_TRACK_SAVED_BITS_MASK (SPTE_EPT_READABLE_MASK | \
70 SPTE_EPT_EXECUTABLE_MASK)
71#define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT 54
72#define SHADOW_ACC_TRACK_SAVED_MASK (SHADOW_ACC_TRACK_SAVED_BITS_MASK << \
73 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
74static_assert(!(SPTE_TDP_AD_MASK & SHADOW_ACC_TRACK_SAVED_MASK));
75
76/*
77 * {DEFAULT,EPT}_SPTE_{HOST,MMU}_WRITABLE are used to keep track of why a given
78 * SPTE is write-protected. See is_writable_pte() for details.
79 */
80
81/* Bits 9 and 10 are ignored by all non-EPT PTEs. */
82#define DEFAULT_SPTE_HOST_WRITABLE BIT_ULL(9)
83#define DEFAULT_SPTE_MMU_WRITABLE BIT_ULL(10)
84
85/*
86 * Low ignored bits are at a premium for EPT, use high ignored bits, taking care
87 * to not overlap the A/D type mask or the saved access bits of access-tracked
88 * SPTEs when A/D bits are disabled.
89 */
90#define EPT_SPTE_HOST_WRITABLE BIT_ULL(57)
91#define EPT_SPTE_MMU_WRITABLE BIT_ULL(58)
92
93static_assert(!(EPT_SPTE_HOST_WRITABLE & SPTE_TDP_AD_MASK));
94static_assert(!(EPT_SPTE_MMU_WRITABLE & SPTE_TDP_AD_MASK));
95static_assert(!(EPT_SPTE_HOST_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
96static_assert(!(EPT_SPTE_MMU_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
97
98/* Defined only to keep the above static asserts readable. */
99#undef SHADOW_ACC_TRACK_SAVED_MASK
100
101/*
102 * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of
103 * the memslots generation and is derived as follows:
104 *
105 * Bits 0-7 of the MMIO generation are propagated to spte bits 3-10
106 * Bits 8-18 of the MMIO generation are propagated to spte bits 52-62
107 *
108 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in
109 * the MMIO generation number, as doing so would require stealing a bit from
110 * the "real" generation number and thus effectively halve the maximum number
111 * of MMIO generations that can be handled before encountering a wrap (which
112 * requires a full MMU zap). The flag is instead explicitly queried when
113 * checking for MMIO spte cache hits.
114 */
115
116#define MMIO_SPTE_GEN_LOW_START 3
117#define MMIO_SPTE_GEN_LOW_END 10
118
119#define MMIO_SPTE_GEN_HIGH_START 52
120#define MMIO_SPTE_GEN_HIGH_END 62
121
122#define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \
123 MMIO_SPTE_GEN_LOW_START)
124#define MMIO_SPTE_GEN_HIGH_MASK GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \
125 MMIO_SPTE_GEN_HIGH_START)
126static_assert(!(SPTE_MMU_PRESENT_MASK &
127 (MMIO_SPTE_GEN_LOW_MASK | MMIO_SPTE_GEN_HIGH_MASK)));
128
129/*
130 * The SPTE MMIO mask must NOT overlap the MMIO generation bits or the
131 * MMU-present bit. The generation obviously co-exists with the magic MMIO
132 * mask/value, and MMIO SPTEs are considered !MMU-present.
133 *
134 * The SPTE MMIO mask is allowed to use hardware "present" bits (i.e. all EPT
135 * RWX bits), all physical address bits (legal PA bits are used for "fast" MMIO
136 * and so they're off-limits for generation; additional checks ensure the mask
137 * doesn't overlap legal PA bits), and bit 63 (carved out for future usage).
138 */
139#define SPTE_MMIO_ALLOWED_MASK (BIT_ULL(63) | GENMASK_ULL(51, 12) | GENMASK_ULL(2, 0))
140static_assert(!(SPTE_MMIO_ALLOWED_MASK &
141 (SPTE_MMU_PRESENT_MASK | MMIO_SPTE_GEN_LOW_MASK | MMIO_SPTE_GEN_HIGH_MASK)));
142
143#define MMIO_SPTE_GEN_LOW_BITS (MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1)
144#define MMIO_SPTE_GEN_HIGH_BITS (MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1)
145
146/* remember to adjust the comment above as well if you change these */
147static_assert(MMIO_SPTE_GEN_LOW_BITS == 8 && MMIO_SPTE_GEN_HIGH_BITS == 11);
148
149#define MMIO_SPTE_GEN_LOW_SHIFT (MMIO_SPTE_GEN_LOW_START - 0)
150#define MMIO_SPTE_GEN_HIGH_SHIFT (MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS)
151
152#define MMIO_SPTE_GEN_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0)
153
154/*
155 * Non-present SPTE value needs to set bit 63 for TDX, in order to suppress
156 * #VE and get EPT violations on non-present PTEs. We can use the
157 * same value also without TDX for both VMX and SVM:
158 *
159 * For SVM NPT, for non-present spte (bit 0 = 0), other bits are ignored.
160 * For VMX EPT, bit 63 is ignored if #VE is disabled. (EPT_VIOLATION_VE=0)
161 * bit 63 is #VE suppress if #VE is enabled. (EPT_VIOLATION_VE=1)
162 */
163#ifdef CONFIG_X86_64
164#define SHADOW_NONPRESENT_VALUE BIT_ULL(63)
165static_assert(!(SHADOW_NONPRESENT_VALUE & SPTE_MMU_PRESENT_MASK));
166#else
167#define SHADOW_NONPRESENT_VALUE 0ULL
168#endif
169
170
171/*
172 * True if A/D bits are supported in hardware and are enabled by KVM. When
173 * enabled, KVM uses A/D bits for all non-nested MMUs. Because L1 can disable
174 * A/D bits in EPTP12, SP and SPTE variants are needed to handle the scenario
175 * where KVM is using A/D bits for L1, but not L2.
176 */
177extern bool __read_mostly kvm_ad_enabled;
178
179extern u64 __read_mostly shadow_host_writable_mask;
180extern u64 __read_mostly shadow_mmu_writable_mask;
181extern u64 __read_mostly shadow_nx_mask;
182extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
183extern u64 __read_mostly shadow_user_mask;
184extern u64 __read_mostly shadow_accessed_mask;
185extern u64 __read_mostly shadow_dirty_mask;
186extern u64 __read_mostly shadow_mmio_value;
187extern u64 __read_mostly shadow_mmio_mask;
188extern u64 __read_mostly shadow_mmio_access_mask;
189extern u64 __read_mostly shadow_present_mask;
190extern u64 __read_mostly shadow_memtype_mask;
191extern u64 __read_mostly shadow_me_value;
192extern u64 __read_mostly shadow_me_mask;
193
194/*
195 * SPTEs in MMUs without A/D bits are marked with SPTE_TDP_AD_DISABLED;
196 * shadow_acc_track_mask is the set of bits to be cleared in non-accessed
197 * pages.
198 */
199extern u64 __read_mostly shadow_acc_track_mask;
200
201/*
202 * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order
203 * to guard against L1TF attacks.
204 */
205extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask;
206
207/*
208 * The number of high-order 1 bits to use in the mask above.
209 */
210#define SHADOW_NONPRESENT_OR_RSVD_MASK_LEN 5
211
212/*
213 * If a thread running without exclusive control of the MMU lock must perform a
214 * multi-part operation on an SPTE, it can set the SPTE to FROZEN_SPTE as a
215 * non-present intermediate value. Other threads which encounter this value
216 * should not modify the SPTE.
217 *
218 * Use a semi-arbitrary value that doesn't set RWX bits, i.e. is not-present on
219 * both AMD and Intel CPUs, and doesn't set PFN bits, i.e. doesn't create a L1TF
220 * vulnerability.
221 *
222 * Only used by the TDP MMU.
223 */
224#define FROZEN_SPTE (SHADOW_NONPRESENT_VALUE | 0x5a0ULL)
225
226/* Frozen SPTEs must not be misconstrued as shadow present PTEs. */
227static_assert(!(FROZEN_SPTE & SPTE_MMU_PRESENT_MASK));
228
229static inline bool is_frozen_spte(u64 spte)
230{
231 return spte == FROZEN_SPTE;
232}
233
234/* Get an SPTE's index into its parent's page table (and the spt array). */
235static inline int spte_index(u64 *sptep)
236{
237 return ((unsigned long)sptep / sizeof(*sptep)) & (SPTE_ENT_PER_PAGE - 1);
238}
239
240/*
241 * In some cases, we need to preserve the GFN of a non-present or reserved
242 * SPTE when we usurp the upper five bits of the physical address space to
243 * defend against L1TF, e.g. for MMIO SPTEs. To preserve the GFN, we'll
244 * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask
245 * left into the reserved bits, i.e. the GFN in the SPTE will be split into
246 * high and low parts. This mask covers the lower bits of the GFN.
247 */
248extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;
249
250static inline struct kvm_mmu_page *to_shadow_page(hpa_t shadow_page)
251{
252 struct page *page = pfn_to_page((shadow_page) >> PAGE_SHIFT);
253
254 return (struct kvm_mmu_page *)page_private(page);
255}
256
257static inline struct kvm_mmu_page *spte_to_child_sp(u64 spte)
258{
259 return to_shadow_page(spte & SPTE_BASE_ADDR_MASK);
260}
261
262static inline struct kvm_mmu_page *sptep_to_sp(u64 *sptep)
263{
264 return to_shadow_page(__pa(sptep));
265}
266
267static inline struct kvm_mmu_page *root_to_sp(hpa_t root)
268{
269 if (kvm_mmu_is_dummy_root(root))
270 return NULL;
271
272 /*
273 * The "root" may be a special root, e.g. a PAE entry, treat it as a
274 * SPTE to ensure any non-PA bits are dropped.
275 */
276 return spte_to_child_sp(root);
277}
278
279static inline bool is_mmio_spte(struct kvm *kvm, u64 spte)
280{
281 return (spte & shadow_mmio_mask) == kvm->arch.shadow_mmio_value &&
282 likely(enable_mmio_caching);
283}
284
285static inline bool is_shadow_present_pte(u64 pte)
286{
287 return !!(pte & SPTE_MMU_PRESENT_MASK);
288}
289
290static inline bool is_ept_ve_possible(u64 spte)
291{
292 return (shadow_present_mask & VMX_EPT_SUPPRESS_VE_BIT) &&
293 !(spte & VMX_EPT_SUPPRESS_VE_BIT) &&
294 (spte & VMX_EPT_RWX_MASK) != VMX_EPT_MISCONFIG_WX_VALUE;
295}
296
297static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
298{
299 return sp->role.ad_disabled;
300}
301
302static inline bool spte_ad_enabled(u64 spte)
303{
304 KVM_MMU_WARN_ON(!is_shadow_present_pte(spte));
305 return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_DISABLED;
306}
307
308static inline bool spte_ad_need_write_protect(u64 spte)
309{
310 KVM_MMU_WARN_ON(!is_shadow_present_pte(spte));
311 /*
312 * This is benign for non-TDP SPTEs as SPTE_TDP_AD_ENABLED is '0',
313 * and non-TDP SPTEs will never set these bits. Optimize for 64-bit
314 * TDP and do the A/D type check unconditionally.
315 */
316 return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_ENABLED;
317}
318
319static inline bool is_access_track_spte(u64 spte)
320{
321 return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
322}
323
324static inline bool is_large_pte(u64 pte)
325{
326 return pte & PT_PAGE_SIZE_MASK;
327}
328
329static inline bool is_last_spte(u64 pte, int level)
330{
331 return (level == PG_LEVEL_4K) || is_large_pte(pte);
332}
333
334static inline bool is_executable_pte(u64 spte)
335{
336 return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
337}
338
339static inline kvm_pfn_t spte_to_pfn(u64 pte)
340{
341 return (pte & SPTE_BASE_ADDR_MASK) >> PAGE_SHIFT;
342}
343
344static inline bool is_accessed_spte(u64 spte)
345{
346 return spte & shadow_accessed_mask;
347}
348
349static inline u64 get_rsvd_bits(struct rsvd_bits_validate *rsvd_check, u64 pte,
350 int level)
351{
352 int bit7 = (pte >> 7) & 1;
353
354 return rsvd_check->rsvd_bits_mask[bit7][level-1];
355}
356
357static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check,
358 u64 pte, int level)
359{
360 return pte & get_rsvd_bits(rsvd_check, pte, level);
361}
362
363static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check,
364 u64 pte)
365{
366 return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f);
367}
368
369static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check,
370 u64 spte, int level)
371{
372 return __is_bad_mt_xwr(rsvd_check, spte) ||
373 __is_rsvd_bits_set(rsvd_check, spte, level);
374}
375
376/*
377 * A shadow-present leaf SPTE may be non-writable for 4 possible reasons:
378 *
379 * 1. To intercept writes for dirty logging. KVM write-protects huge pages
380 * so that they can be split down into the dirty logging
381 * granularity (4KiB) whenever the guest writes to them. KVM also
382 * write-protects 4KiB pages so that writes can be recorded in the dirty log
383 * (e.g. if not using PML). SPTEs are write-protected for dirty logging
384 * during the VM-iotcls that enable dirty logging.
385 *
386 * 2. To intercept writes to guest page tables that KVM is shadowing. When a
387 * guest writes to its page table the corresponding shadow page table will
388 * be marked "unsync". That way KVM knows which shadow page tables need to
389 * be updated on the next TLB flush, INVLPG, etc. and which do not.
390 *
391 * 3. To prevent guest writes to read-only memory, such as for memory in a
392 * read-only memslot or guest memory backed by a read-only VMA. Writes to
393 * such pages are disallowed entirely.
394 *
395 * 4. To emulate the Accessed bit for SPTEs without A/D bits. Note, in this
396 * case, the SPTE is access-protected, not just write-protected!
397 *
398 * For cases #1 and #4, KVM can safely make such SPTEs writable without taking
399 * mmu_lock as capturing the Accessed/Dirty state doesn't require taking it.
400 * To differentiate #1 and #4 from #2 and #3, KVM uses two software-only bits
401 * in the SPTE:
402 *
403 * shadow_mmu_writable_mask, aka MMU-writable -
404 * Cleared on SPTEs that KVM is currently write-protecting for shadow paging
405 * purposes (case 2 above).
406 *
407 * shadow_host_writable_mask, aka Host-writable -
408 * Cleared on SPTEs that are not host-writable (case 3 above)
409 *
410 * Note, not all possible combinations of PT_WRITABLE_MASK,
411 * shadow_mmu_writable_mask, and shadow_host_writable_mask are valid. A given
412 * SPTE can be in only one of the following states, which map to the
413 * aforementioned 3 cases:
414 *
415 * shadow_host_writable_mask | shadow_mmu_writable_mask | PT_WRITABLE_MASK
416 * ------------------------- | ------------------------ | ----------------
417 * 1 | 1 | 1 (writable)
418 * 1 | 1 | 0 (case 1)
419 * 1 | 0 | 0 (case 2)
420 * 0 | 0 | 0 (case 3)
421 *
422 * The valid combinations of these bits are checked by
423 * check_spte_writable_invariants() whenever an SPTE is modified.
424 *
425 * Clearing the MMU-writable bit is always done under the MMU lock and always
426 * accompanied by a TLB flush before dropping the lock to avoid corrupting the
427 * shadow page tables between vCPUs. Write-protecting an SPTE for dirty logging
428 * (which does not clear the MMU-writable bit), does not flush TLBs before
429 * dropping the lock, as it only needs to synchronize guest writes with the
430 * dirty bitmap. Similarly, making the SPTE inaccessible (and non-writable) for
431 * access-tracking via the clear_young() MMU notifier also does not flush TLBs.
432 *
433 * So, there is the problem: clearing the MMU-writable bit can encounter a
434 * write-protected SPTE while CPUs still have writable mappings for that SPTE
435 * cached in their TLB. To address this, KVM always flushes TLBs when
436 * write-protecting SPTEs if the MMU-writable bit is set on the old SPTE.
437 *
438 * The Host-writable bit is not modified on present SPTEs, it is only set or
439 * cleared when an SPTE is first faulted in from non-present and then remains
440 * immutable.
441 */
442static inline bool is_writable_pte(unsigned long pte)
443{
444 return pte & PT_WRITABLE_MASK;
445}
446
447/* Note: spte must be a shadow-present leaf SPTE. */
448static inline void check_spte_writable_invariants(u64 spte)
449{
450 if (spte & shadow_mmu_writable_mask)
451 WARN_ONCE(!(spte & shadow_host_writable_mask),
452 KBUILD_MODNAME ": MMU-writable SPTE is not Host-writable: %llx",
453 spte);
454 else
455 WARN_ONCE(is_writable_pte(spte),
456 KBUILD_MODNAME ": Writable SPTE is not MMU-writable: %llx", spte);
457}
458
459static inline bool is_mmu_writable_spte(u64 spte)
460{
461 return spte & shadow_mmu_writable_mask;
462}
463
464/*
465 * Returns true if the access indicated by @fault is allowed by the existing
466 * SPTE protections. Note, the caller is responsible for checking that the
467 * SPTE is a shadow-present, leaf SPTE (either before or after).
468 */
469static inline bool is_access_allowed(struct kvm_page_fault *fault, u64 spte)
470{
471 if (fault->exec)
472 return is_executable_pte(spte);
473
474 if (fault->write)
475 return is_writable_pte(spte);
476
477 /* Fault was on Read access */
478 return spte & PT_PRESENT_MASK;
479}
480
481/*
482 * If the MMU-writable flag is cleared, i.e. the SPTE is write-protected for
483 * write-tracking, remote TLBs must be flushed, even if the SPTE was read-only,
484 * as KVM allows stale Writable TLB entries to exist. When dirty logging, KVM
485 * flushes TLBs based on whether or not dirty bitmap/ring entries were reaped,
486 * not whether or not SPTEs were modified, i.e. only the write-tracking case
487 * needs to flush at the time the SPTEs is modified, before dropping mmu_lock.
488 *
489 * Don't flush if the Accessed bit is cleared, as access tracking tolerates
490 * false negatives, e.g. KVM x86 omits TLB flushes even when aging SPTEs for a
491 * mmu_notifier.clear_flush_young() event.
492 *
493 * Lastly, don't flush if the Dirty bit is cleared, as KVM unconditionally
494 * flushes when enabling dirty logging (see kvm_mmu_slot_apply_flags()), and
495 * when clearing dirty logs, KVM flushes based on whether or not dirty entries
496 * were reaped from the bitmap/ring, not whether or not dirty SPTEs were found.
497 *
498 * Note, this logic only applies to shadow-present leaf SPTEs. The caller is
499 * responsible for checking that the old SPTE is shadow-present, and is also
500 * responsible for determining whether or not a TLB flush is required when
501 * modifying a shadow-present non-leaf SPTE.
502 */
503static inline bool leaf_spte_change_needs_tlb_flush(u64 old_spte, u64 new_spte)
504{
505 return is_mmu_writable_spte(old_spte) && !is_mmu_writable_spte(new_spte);
506}
507
508static inline u64 get_mmio_spte_generation(u64 spte)
509{
510 u64 gen;
511
512 gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT;
513 gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT;
514 return gen;
515}
516
517bool spte_has_volatile_bits(u64 spte);
518
519bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
520 const struct kvm_memory_slot *slot,
521 unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn,
522 u64 old_spte, bool prefetch, bool synchronizing,
523 bool host_writable, u64 *new_spte);
524u64 make_small_spte(struct kvm *kvm, u64 huge_spte,
525 union kvm_mmu_page_role role, int index);
526u64 make_huge_spte(struct kvm *kvm, u64 small_spte, int level);
527u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled);
528u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access);
529u64 mark_spte_for_access_track(u64 spte);
530
531/* Restore an acc-track PTE back to a regular PTE */
532static inline u64 restore_acc_track_spte(u64 spte)
533{
534 u64 saved_bits = (spte >> SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
535 & SHADOW_ACC_TRACK_SAVED_BITS_MASK;
536
537 spte &= ~shadow_acc_track_mask;
538 spte &= ~(SHADOW_ACC_TRACK_SAVED_BITS_MASK <<
539 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT);
540 spte |= saved_bits;
541
542 return spte;
543}
544
545void __init kvm_mmu_spte_module_init(void);
546void kvm_mmu_reset_all_pte_masks(void);
547
548#endif
1// SPDX-License-Identifier: GPL-2.0-only
2
3#ifndef KVM_X86_MMU_SPTE_H
4#define KVM_X86_MMU_SPTE_H
5
6#include "mmu_internal.h"
7
8/*
9 * A MMU present SPTE is backed by actual memory and may or may not be present
10 * in hardware. E.g. MMIO SPTEs are not considered present. Use bit 11, as it
11 * is ignored by all flavors of SPTEs and checking a low bit often generates
12 * better code than for a high bit, e.g. 56+. MMU present checks are pervasive
13 * enough that the improved code generation is noticeable in KVM's footprint.
14 */
15#define SPTE_MMU_PRESENT_MASK BIT_ULL(11)
16
17/*
18 * TDP SPTES (more specifically, EPT SPTEs) may not have A/D bits, and may also
19 * be restricted to using write-protection (for L2 when CPU dirty logging, i.e.
20 * PML, is enabled). Use bits 52 and 53 to hold the type of A/D tracking that
21 * is must be employed for a given TDP SPTE.
22 *
23 * Note, the "enabled" mask must be '0', as bits 62:52 are _reserved_ for PAE
24 * paging, including NPT PAE. This scheme works because legacy shadow paging
25 * is guaranteed to have A/D bits and write-protection is forced only for
26 * TDP with CPU dirty logging (PML). If NPT ever gains PML-like support, it
27 * must be restricted to 64-bit KVM.
28 */
29#define SPTE_TDP_AD_SHIFT 52
30#define SPTE_TDP_AD_MASK (3ULL << SPTE_TDP_AD_SHIFT)
31#define SPTE_TDP_AD_ENABLED_MASK (0ULL << SPTE_TDP_AD_SHIFT)
32#define SPTE_TDP_AD_DISABLED_MASK (1ULL << SPTE_TDP_AD_SHIFT)
33#define SPTE_TDP_AD_WRPROT_ONLY_MASK (2ULL << SPTE_TDP_AD_SHIFT)
34static_assert(SPTE_TDP_AD_ENABLED_MASK == 0);
35
36#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
37#define SPTE_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1))
38#else
39#define SPTE_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
40#endif
41
42#define SPTE_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
43 | shadow_x_mask | shadow_nx_mask | shadow_me_mask)
44
45#define ACC_EXEC_MASK 1
46#define ACC_WRITE_MASK PT_WRITABLE_MASK
47#define ACC_USER_MASK PT_USER_MASK
48#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
49
50/* The mask for the R/X bits in EPT PTEs */
51#define SPTE_EPT_READABLE_MASK 0x1ull
52#define SPTE_EPT_EXECUTABLE_MASK 0x4ull
53
54#define SPTE_LEVEL_BITS 9
55#define SPTE_LEVEL_SHIFT(level) __PT_LEVEL_SHIFT(level, SPTE_LEVEL_BITS)
56#define SPTE_INDEX(address, level) __PT_INDEX(address, level, SPTE_LEVEL_BITS)
57#define SPTE_ENT_PER_PAGE __PT_ENT_PER_PAGE(SPTE_LEVEL_BITS)
58
59/*
60 * The mask/shift to use for saving the original R/X bits when marking the PTE
61 * as not-present for access tracking purposes. We do not save the W bit as the
62 * PTEs being access tracked also need to be dirty tracked, so the W bit will be
63 * restored only when a write is attempted to the page. This mask obviously
64 * must not overlap the A/D type mask.
65 */
66#define SHADOW_ACC_TRACK_SAVED_BITS_MASK (SPTE_EPT_READABLE_MASK | \
67 SPTE_EPT_EXECUTABLE_MASK)
68#define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT 54
69#define SHADOW_ACC_TRACK_SAVED_MASK (SHADOW_ACC_TRACK_SAVED_BITS_MASK << \
70 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
71static_assert(!(SPTE_TDP_AD_MASK & SHADOW_ACC_TRACK_SAVED_MASK));
72
73/*
74 * {DEFAULT,EPT}_SPTE_{HOST,MMU}_WRITABLE are used to keep track of why a given
75 * SPTE is write-protected. See is_writable_pte() for details.
76 */
77
78/* Bits 9 and 10 are ignored by all non-EPT PTEs. */
79#define DEFAULT_SPTE_HOST_WRITABLE BIT_ULL(9)
80#define DEFAULT_SPTE_MMU_WRITABLE BIT_ULL(10)
81
82/*
83 * Low ignored bits are at a premium for EPT, use high ignored bits, taking care
84 * to not overlap the A/D type mask or the saved access bits of access-tracked
85 * SPTEs when A/D bits are disabled.
86 */
87#define EPT_SPTE_HOST_WRITABLE BIT_ULL(57)
88#define EPT_SPTE_MMU_WRITABLE BIT_ULL(58)
89
90static_assert(!(EPT_SPTE_HOST_WRITABLE & SPTE_TDP_AD_MASK));
91static_assert(!(EPT_SPTE_MMU_WRITABLE & SPTE_TDP_AD_MASK));
92static_assert(!(EPT_SPTE_HOST_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
93static_assert(!(EPT_SPTE_MMU_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
94
95/* Defined only to keep the above static asserts readable. */
96#undef SHADOW_ACC_TRACK_SAVED_MASK
97
98/*
99 * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of
100 * the memslots generation and is derived as follows:
101 *
102 * Bits 0-7 of the MMIO generation are propagated to spte bits 3-10
103 * Bits 8-18 of the MMIO generation are propagated to spte bits 52-62
104 *
105 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in
106 * the MMIO generation number, as doing so would require stealing a bit from
107 * the "real" generation number and thus effectively halve the maximum number
108 * of MMIO generations that can be handled before encountering a wrap (which
109 * requires a full MMU zap). The flag is instead explicitly queried when
110 * checking for MMIO spte cache hits.
111 */
112
113#define MMIO_SPTE_GEN_LOW_START 3
114#define MMIO_SPTE_GEN_LOW_END 10
115
116#define MMIO_SPTE_GEN_HIGH_START 52
117#define MMIO_SPTE_GEN_HIGH_END 62
118
119#define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \
120 MMIO_SPTE_GEN_LOW_START)
121#define MMIO_SPTE_GEN_HIGH_MASK GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \
122 MMIO_SPTE_GEN_HIGH_START)
123static_assert(!(SPTE_MMU_PRESENT_MASK &
124 (MMIO_SPTE_GEN_LOW_MASK | MMIO_SPTE_GEN_HIGH_MASK)));
125
126/*
127 * The SPTE MMIO mask must NOT overlap the MMIO generation bits or the
128 * MMU-present bit. The generation obviously co-exists with the magic MMIO
129 * mask/value, and MMIO SPTEs are considered !MMU-present.
130 *
131 * The SPTE MMIO mask is allowed to use hardware "present" bits (i.e. all EPT
132 * RWX bits), all physical address bits (legal PA bits are used for "fast" MMIO
133 * and so they're off-limits for generation; additional checks ensure the mask
134 * doesn't overlap legal PA bits), and bit 63 (carved out for future usage).
135 */
136#define SPTE_MMIO_ALLOWED_MASK (BIT_ULL(63) | GENMASK_ULL(51, 12) | GENMASK_ULL(2, 0))
137static_assert(!(SPTE_MMIO_ALLOWED_MASK &
138 (SPTE_MMU_PRESENT_MASK | MMIO_SPTE_GEN_LOW_MASK | MMIO_SPTE_GEN_HIGH_MASK)));
139
140#define MMIO_SPTE_GEN_LOW_BITS (MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1)
141#define MMIO_SPTE_GEN_HIGH_BITS (MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1)
142
143/* remember to adjust the comment above as well if you change these */
144static_assert(MMIO_SPTE_GEN_LOW_BITS == 8 && MMIO_SPTE_GEN_HIGH_BITS == 11);
145
146#define MMIO_SPTE_GEN_LOW_SHIFT (MMIO_SPTE_GEN_LOW_START - 0)
147#define MMIO_SPTE_GEN_HIGH_SHIFT (MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS)
148
149#define MMIO_SPTE_GEN_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0)
150
151extern u64 __read_mostly shadow_host_writable_mask;
152extern u64 __read_mostly shadow_mmu_writable_mask;
153extern u64 __read_mostly shadow_nx_mask;
154extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
155extern u64 __read_mostly shadow_user_mask;
156extern u64 __read_mostly shadow_accessed_mask;
157extern u64 __read_mostly shadow_dirty_mask;
158extern u64 __read_mostly shadow_mmio_value;
159extern u64 __read_mostly shadow_mmio_mask;
160extern u64 __read_mostly shadow_mmio_access_mask;
161extern u64 __read_mostly shadow_present_mask;
162extern u64 __read_mostly shadow_memtype_mask;
163extern u64 __read_mostly shadow_me_value;
164extern u64 __read_mostly shadow_me_mask;
165
166/*
167 * SPTEs in MMUs without A/D bits are marked with SPTE_TDP_AD_DISABLED_MASK;
168 * shadow_acc_track_mask is the set of bits to be cleared in non-accessed
169 * pages.
170 */
171extern u64 __read_mostly shadow_acc_track_mask;
172
173/*
174 * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order
175 * to guard against L1TF attacks.
176 */
177extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask;
178
179/*
180 * The number of high-order 1 bits to use in the mask above.
181 */
182#define SHADOW_NONPRESENT_OR_RSVD_MASK_LEN 5
183
184/*
185 * If a thread running without exclusive control of the MMU lock must perform a
186 * multi-part operation on an SPTE, it can set the SPTE to REMOVED_SPTE as a
187 * non-present intermediate value. Other threads which encounter this value
188 * should not modify the SPTE.
189 *
190 * Use a semi-arbitrary value that doesn't set RWX bits, i.e. is not-present on
191 * both AMD and Intel CPUs, and doesn't set PFN bits, i.e. doesn't create a L1TF
192 * vulnerability. Use only low bits to avoid 64-bit immediates.
193 *
194 * Only used by the TDP MMU.
195 */
196#define REMOVED_SPTE 0x5a0ULL
197
198/* Removed SPTEs must not be misconstrued as shadow present PTEs. */
199static_assert(!(REMOVED_SPTE & SPTE_MMU_PRESENT_MASK));
200
201static inline bool is_removed_spte(u64 spte)
202{
203 return spte == REMOVED_SPTE;
204}
205
206/* Get an SPTE's index into its parent's page table (and the spt array). */
207static inline int spte_index(u64 *sptep)
208{
209 return ((unsigned long)sptep / sizeof(*sptep)) & (SPTE_ENT_PER_PAGE - 1);
210}
211
212/*
213 * In some cases, we need to preserve the GFN of a non-present or reserved
214 * SPTE when we usurp the upper five bits of the physical address space to
215 * defend against L1TF, e.g. for MMIO SPTEs. To preserve the GFN, we'll
216 * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask
217 * left into the reserved bits, i.e. the GFN in the SPTE will be split into
218 * high and low parts. This mask covers the lower bits of the GFN.
219 */
220extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;
221
222static inline struct kvm_mmu_page *to_shadow_page(hpa_t shadow_page)
223{
224 struct page *page = pfn_to_page((shadow_page) >> PAGE_SHIFT);
225
226 return (struct kvm_mmu_page *)page_private(page);
227}
228
229static inline struct kvm_mmu_page *spte_to_child_sp(u64 spte)
230{
231 return to_shadow_page(spte & SPTE_BASE_ADDR_MASK);
232}
233
234static inline struct kvm_mmu_page *sptep_to_sp(u64 *sptep)
235{
236 return to_shadow_page(__pa(sptep));
237}
238
239static inline bool is_mmio_spte(u64 spte)
240{
241 return (spte & shadow_mmio_mask) == shadow_mmio_value &&
242 likely(enable_mmio_caching);
243}
244
245static inline bool is_shadow_present_pte(u64 pte)
246{
247 return !!(pte & SPTE_MMU_PRESENT_MASK);
248}
249
250/*
251 * Returns true if A/D bits are supported in hardware and are enabled by KVM.
252 * When enabled, KVM uses A/D bits for all non-nested MMUs. Because L1 can
253 * disable A/D bits in EPTP12, SP and SPTE variants are needed to handle the
254 * scenario where KVM is using A/D bits for L1, but not L2.
255 */
256static inline bool kvm_ad_enabled(void)
257{
258 return !!shadow_accessed_mask;
259}
260
261static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
262{
263 return sp->role.ad_disabled;
264}
265
266static inline bool spte_ad_enabled(u64 spte)
267{
268 MMU_WARN_ON(!is_shadow_present_pte(spte));
269 return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_DISABLED_MASK;
270}
271
272static inline bool spte_ad_need_write_protect(u64 spte)
273{
274 MMU_WARN_ON(!is_shadow_present_pte(spte));
275 /*
276 * This is benign for non-TDP SPTEs as SPTE_TDP_AD_ENABLED_MASK is '0',
277 * and non-TDP SPTEs will never set these bits. Optimize for 64-bit
278 * TDP and do the A/D type check unconditionally.
279 */
280 return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_ENABLED_MASK;
281}
282
283static inline u64 spte_shadow_accessed_mask(u64 spte)
284{
285 MMU_WARN_ON(!is_shadow_present_pte(spte));
286 return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
287}
288
289static inline u64 spte_shadow_dirty_mask(u64 spte)
290{
291 MMU_WARN_ON(!is_shadow_present_pte(spte));
292 return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
293}
294
295static inline bool is_access_track_spte(u64 spte)
296{
297 return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
298}
299
300static inline bool is_large_pte(u64 pte)
301{
302 return pte & PT_PAGE_SIZE_MASK;
303}
304
305static inline bool is_last_spte(u64 pte, int level)
306{
307 return (level == PG_LEVEL_4K) || is_large_pte(pte);
308}
309
310static inline bool is_executable_pte(u64 spte)
311{
312 return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
313}
314
315static inline kvm_pfn_t spte_to_pfn(u64 pte)
316{
317 return (pte & SPTE_BASE_ADDR_MASK) >> PAGE_SHIFT;
318}
319
320static inline bool is_accessed_spte(u64 spte)
321{
322 u64 accessed_mask = spte_shadow_accessed_mask(spte);
323
324 return accessed_mask ? spte & accessed_mask
325 : !is_access_track_spte(spte);
326}
327
328static inline bool is_dirty_spte(u64 spte)
329{
330 u64 dirty_mask = spte_shadow_dirty_mask(spte);
331
332 return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK;
333}
334
335static inline u64 get_rsvd_bits(struct rsvd_bits_validate *rsvd_check, u64 pte,
336 int level)
337{
338 int bit7 = (pte >> 7) & 1;
339
340 return rsvd_check->rsvd_bits_mask[bit7][level-1];
341}
342
343static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check,
344 u64 pte, int level)
345{
346 return pte & get_rsvd_bits(rsvd_check, pte, level);
347}
348
349static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check,
350 u64 pte)
351{
352 return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f);
353}
354
355static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check,
356 u64 spte, int level)
357{
358 return __is_bad_mt_xwr(rsvd_check, spte) ||
359 __is_rsvd_bits_set(rsvd_check, spte, level);
360}
361
362/*
363 * A shadow-present leaf SPTE may be non-writable for 4 possible reasons:
364 *
365 * 1. To intercept writes for dirty logging. KVM write-protects huge pages
366 * so that they can be split down into the dirty logging
367 * granularity (4KiB) whenever the guest writes to them. KVM also
368 * write-protects 4KiB pages so that writes can be recorded in the dirty log
369 * (e.g. if not using PML). SPTEs are write-protected for dirty logging
370 * during the VM-iotcls that enable dirty logging.
371 *
372 * 2. To intercept writes to guest page tables that KVM is shadowing. When a
373 * guest writes to its page table the corresponding shadow page table will
374 * be marked "unsync". That way KVM knows which shadow page tables need to
375 * be updated on the next TLB flush, INVLPG, etc. and which do not.
376 *
377 * 3. To prevent guest writes to read-only memory, such as for memory in a
378 * read-only memslot or guest memory backed by a read-only VMA. Writes to
379 * such pages are disallowed entirely.
380 *
381 * 4. To emulate the Accessed bit for SPTEs without A/D bits. Note, in this
382 * case, the SPTE is access-protected, not just write-protected!
383 *
384 * For cases #1 and #4, KVM can safely make such SPTEs writable without taking
385 * mmu_lock as capturing the Accessed/Dirty state doesn't require taking it.
386 * To differentiate #1 and #4 from #2 and #3, KVM uses two software-only bits
387 * in the SPTE:
388 *
389 * shadow_mmu_writable_mask, aka MMU-writable -
390 * Cleared on SPTEs that KVM is currently write-protecting for shadow paging
391 * purposes (case 2 above).
392 *
393 * shadow_host_writable_mask, aka Host-writable -
394 * Cleared on SPTEs that are not host-writable (case 3 above)
395 *
396 * Note, not all possible combinations of PT_WRITABLE_MASK,
397 * shadow_mmu_writable_mask, and shadow_host_writable_mask are valid. A given
398 * SPTE can be in only one of the following states, which map to the
399 * aforementioned 3 cases:
400 *
401 * shadow_host_writable_mask | shadow_mmu_writable_mask | PT_WRITABLE_MASK
402 * ------------------------- | ------------------------ | ----------------
403 * 1 | 1 | 1 (writable)
404 * 1 | 1 | 0 (case 1)
405 * 1 | 0 | 0 (case 2)
406 * 0 | 0 | 0 (case 3)
407 *
408 * The valid combinations of these bits are checked by
409 * check_spte_writable_invariants() whenever an SPTE is modified.
410 *
411 * Clearing the MMU-writable bit is always done under the MMU lock and always
412 * accompanied by a TLB flush before dropping the lock to avoid corrupting the
413 * shadow page tables between vCPUs. Write-protecting an SPTE for dirty logging
414 * (which does not clear the MMU-writable bit), does not flush TLBs before
415 * dropping the lock, as it only needs to synchronize guest writes with the
416 * dirty bitmap. Similarly, making the SPTE inaccessible (and non-writable) for
417 * access-tracking via the clear_young() MMU notifier also does not flush TLBs.
418 *
419 * So, there is the problem: clearing the MMU-writable bit can encounter a
420 * write-protected SPTE while CPUs still have writable mappings for that SPTE
421 * cached in their TLB. To address this, KVM always flushes TLBs when
422 * write-protecting SPTEs if the MMU-writable bit is set on the old SPTE.
423 *
424 * The Host-writable bit is not modified on present SPTEs, it is only set or
425 * cleared when an SPTE is first faulted in from non-present and then remains
426 * immutable.
427 */
428static inline bool is_writable_pte(unsigned long pte)
429{
430 return pte & PT_WRITABLE_MASK;
431}
432
433/* Note: spte must be a shadow-present leaf SPTE. */
434static inline void check_spte_writable_invariants(u64 spte)
435{
436 if (spte & shadow_mmu_writable_mask)
437 WARN_ONCE(!(spte & shadow_host_writable_mask),
438 "kvm: MMU-writable SPTE is not Host-writable: %llx",
439 spte);
440 else
441 WARN_ONCE(is_writable_pte(spte),
442 "kvm: Writable SPTE is not MMU-writable: %llx", spte);
443}
444
445static inline bool is_mmu_writable_spte(u64 spte)
446{
447 return spte & shadow_mmu_writable_mask;
448}
449
450static inline u64 get_mmio_spte_generation(u64 spte)
451{
452 u64 gen;
453
454 gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT;
455 gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT;
456 return gen;
457}
458
459bool spte_has_volatile_bits(u64 spte);
460
461bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
462 const struct kvm_memory_slot *slot,
463 unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn,
464 u64 old_spte, bool prefetch, bool can_unsync,
465 bool host_writable, u64 *new_spte);
466u64 make_huge_page_split_spte(struct kvm *kvm, u64 huge_spte,
467 union kvm_mmu_page_role role, int index);
468u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled);
469u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access);
470u64 mark_spte_for_access_track(u64 spte);
471
472/* Restore an acc-track PTE back to a regular PTE */
473static inline u64 restore_acc_track_spte(u64 spte)
474{
475 u64 saved_bits = (spte >> SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
476 & SHADOW_ACC_TRACK_SAVED_BITS_MASK;
477
478 spte &= ~shadow_acc_track_mask;
479 spte &= ~(SHADOW_ACC_TRACK_SAVED_BITS_MASK <<
480 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT);
481 spte |= saved_bits;
482
483 return spte;
484}
485
486u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn);
487
488void __init kvm_mmu_spte_module_init(void);
489void kvm_mmu_reset_all_pte_masks(void);
490
491#endif