Loading...
Note: File does not exist in v3.5.6.
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * KVM/MIPS MMU handling in the KVM module.
7 *
8 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved.
9 * Authors: Sanjay Lal <sanjayl@kymasys.com>
10 */
11
12#include <linux/highmem.h>
13#include <linux/kvm_host.h>
14#include <linux/uaccess.h>
15#include <asm/mmu_context.h>
16#include <asm/pgalloc.h>
17
18/*
19 * KVM_MMU_CACHE_MIN_PAGES is the number of GPA page table translation levels
20 * for which pages need to be cached.
21 */
22#if defined(__PAGETABLE_PMD_FOLDED)
23#define KVM_MMU_CACHE_MIN_PAGES 1
24#else
25#define KVM_MMU_CACHE_MIN_PAGES 2
26#endif
27
28static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
29 int min, int max)
30{
31 void *page;
32
33 BUG_ON(max > KVM_NR_MEM_OBJS);
34 if (cache->nobjs >= min)
35 return 0;
36 while (cache->nobjs < max) {
37 page = (void *)__get_free_page(GFP_KERNEL);
38 if (!page)
39 return -ENOMEM;
40 cache->objects[cache->nobjs++] = page;
41 }
42 return 0;
43}
44
45static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
46{
47 while (mc->nobjs)
48 free_page((unsigned long)mc->objects[--mc->nobjs]);
49}
50
51static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
52{
53 void *p;
54
55 BUG_ON(!mc || !mc->nobjs);
56 p = mc->objects[--mc->nobjs];
57 return p;
58}
59
60void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
61{
62 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
63}
64
65/**
66 * kvm_pgd_init() - Initialise KVM GPA page directory.
67 * @page: Pointer to page directory (PGD) for KVM GPA.
68 *
69 * Initialise a KVM GPA page directory with pointers to the invalid table, i.e.
70 * representing no mappings. This is similar to pgd_init(), however it
71 * initialises all the page directory pointers, not just the ones corresponding
72 * to the userland address space (since it is for the guest physical address
73 * space rather than a virtual address space).
74 */
75static void kvm_pgd_init(void *page)
76{
77 unsigned long *p, *end;
78 unsigned long entry;
79
80#ifdef __PAGETABLE_PMD_FOLDED
81 entry = (unsigned long)invalid_pte_table;
82#else
83 entry = (unsigned long)invalid_pmd_table;
84#endif
85
86 p = (unsigned long *)page;
87 end = p + PTRS_PER_PGD;
88
89 do {
90 p[0] = entry;
91 p[1] = entry;
92 p[2] = entry;
93 p[3] = entry;
94 p[4] = entry;
95 p += 8;
96 p[-3] = entry;
97 p[-2] = entry;
98 p[-1] = entry;
99 } while (p != end);
100}
101
102/**
103 * kvm_pgd_alloc() - Allocate and initialise a KVM GPA page directory.
104 *
105 * Allocate a blank KVM GPA page directory (PGD) for representing guest physical
106 * to host physical page mappings.
107 *
108 * Returns: Pointer to new KVM GPA page directory.
109 * NULL on allocation failure.
110 */
111pgd_t *kvm_pgd_alloc(void)
112{
113 pgd_t *ret;
114
115 ret = (pgd_t *)__get_free_pages(GFP_KERNEL, PGD_ORDER);
116 if (ret)
117 kvm_pgd_init(ret);
118
119 return ret;
120}
121
122/**
123 * kvm_mips_walk_pgd() - Walk page table with optional allocation.
124 * @pgd: Page directory pointer.
125 * @addr: Address to index page table using.
126 * @cache: MMU page cache to allocate new page tables from, or NULL.
127 *
128 * Walk the page tables pointed to by @pgd to find the PTE corresponding to the
129 * address @addr. If page tables don't exist for @addr, they will be created
130 * from the MMU cache if @cache is not NULL.
131 *
132 * Returns: Pointer to pte_t corresponding to @addr.
133 * NULL if a page table doesn't exist for @addr and !@cache.
134 * NULL if a page table allocation failed.
135 */
136static pte_t *kvm_mips_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache,
137 unsigned long addr)
138{
139 pud_t *pud;
140 pmd_t *pmd;
141
142 pgd += pgd_index(addr);
143 if (pgd_none(*pgd)) {
144 /* Not used on MIPS yet */
145 BUG();
146 return NULL;
147 }
148 pud = pud_offset(pgd, addr);
149 if (pud_none(*pud)) {
150 pmd_t *new_pmd;
151
152 if (!cache)
153 return NULL;
154 new_pmd = mmu_memory_cache_alloc(cache);
155 pmd_init((unsigned long)new_pmd,
156 (unsigned long)invalid_pte_table);
157 pud_populate(NULL, pud, new_pmd);
158 }
159 pmd = pmd_offset(pud, addr);
160 if (pmd_none(*pmd)) {
161 pte_t *new_pte;
162
163 if (!cache)
164 return NULL;
165 new_pte = mmu_memory_cache_alloc(cache);
166 clear_page(new_pte);
167 pmd_populate_kernel(NULL, pmd, new_pte);
168 }
169 return pte_offset(pmd, addr);
170}
171
172/* Caller must hold kvm->mm_lock */
173static pte_t *kvm_mips_pte_for_gpa(struct kvm *kvm,
174 struct kvm_mmu_memory_cache *cache,
175 unsigned long addr)
176{
177 return kvm_mips_walk_pgd(kvm->arch.gpa_mm.pgd, cache, addr);
178}
179
180/*
181 * kvm_mips_flush_gpa_{pte,pmd,pud,pgd,pt}.
182 * Flush a range of guest physical address space from the VM's GPA page tables.
183 */
184
185static bool kvm_mips_flush_gpa_pte(pte_t *pte, unsigned long start_gpa,
186 unsigned long end_gpa)
187{
188 int i_min = __pte_offset(start_gpa);
189 int i_max = __pte_offset(end_gpa);
190 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1);
191 int i;
192
193 for (i = i_min; i <= i_max; ++i) {
194 if (!pte_present(pte[i]))
195 continue;
196
197 set_pte(pte + i, __pte(0));
198 }
199 return safe_to_remove;
200}
201
202static bool kvm_mips_flush_gpa_pmd(pmd_t *pmd, unsigned long start_gpa,
203 unsigned long end_gpa)
204{
205 pte_t *pte;
206 unsigned long end = ~0ul;
207 int i_min = __pmd_offset(start_gpa);
208 int i_max = __pmd_offset(end_gpa);
209 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1);
210 int i;
211
212 for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
213 if (!pmd_present(pmd[i]))
214 continue;
215
216 pte = pte_offset(pmd + i, 0);
217 if (i == i_max)
218 end = end_gpa;
219
220 if (kvm_mips_flush_gpa_pte(pte, start_gpa, end)) {
221 pmd_clear(pmd + i);
222 pte_free_kernel(NULL, pte);
223 } else {
224 safe_to_remove = false;
225 }
226 }
227 return safe_to_remove;
228}
229
230static bool kvm_mips_flush_gpa_pud(pud_t *pud, unsigned long start_gpa,
231 unsigned long end_gpa)
232{
233 pmd_t *pmd;
234 unsigned long end = ~0ul;
235 int i_min = __pud_offset(start_gpa);
236 int i_max = __pud_offset(end_gpa);
237 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1);
238 int i;
239
240 for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
241 if (!pud_present(pud[i]))
242 continue;
243
244 pmd = pmd_offset(pud + i, 0);
245 if (i == i_max)
246 end = end_gpa;
247
248 if (kvm_mips_flush_gpa_pmd(pmd, start_gpa, end)) {
249 pud_clear(pud + i);
250 pmd_free(NULL, pmd);
251 } else {
252 safe_to_remove = false;
253 }
254 }
255 return safe_to_remove;
256}
257
258static bool kvm_mips_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa,
259 unsigned long end_gpa)
260{
261 pud_t *pud;
262 unsigned long end = ~0ul;
263 int i_min = pgd_index(start_gpa);
264 int i_max = pgd_index(end_gpa);
265 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1);
266 int i;
267
268 for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
269 if (!pgd_present(pgd[i]))
270 continue;
271
272 pud = pud_offset(pgd + i, 0);
273 if (i == i_max)
274 end = end_gpa;
275
276 if (kvm_mips_flush_gpa_pud(pud, start_gpa, end)) {
277 pgd_clear(pgd + i);
278 pud_free(NULL, pud);
279 } else {
280 safe_to_remove = false;
281 }
282 }
283 return safe_to_remove;
284}
285
286/**
287 * kvm_mips_flush_gpa_pt() - Flush a range of guest physical addresses.
288 * @kvm: KVM pointer.
289 * @start_gfn: Guest frame number of first page in GPA range to flush.
290 * @end_gfn: Guest frame number of last page in GPA range to flush.
291 *
292 * Flushes a range of GPA mappings from the GPA page tables.
293 *
294 * The caller must hold the @kvm->mmu_lock spinlock.
295 *
296 * Returns: Whether its safe to remove the top level page directory because
297 * all lower levels have been removed.
298 */
299bool kvm_mips_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn)
300{
301 return kvm_mips_flush_gpa_pgd(kvm->arch.gpa_mm.pgd,
302 start_gfn << PAGE_SHIFT,
303 end_gfn << PAGE_SHIFT);
304}
305
306#define BUILD_PTE_RANGE_OP(name, op) \
307static int kvm_mips_##name##_pte(pte_t *pte, unsigned long start, \
308 unsigned long end) \
309{ \
310 int ret = 0; \
311 int i_min = __pte_offset(start); \
312 int i_max = __pte_offset(end); \
313 int i; \
314 pte_t old, new; \
315 \
316 for (i = i_min; i <= i_max; ++i) { \
317 if (!pte_present(pte[i])) \
318 continue; \
319 \
320 old = pte[i]; \
321 new = op(old); \
322 if (pte_val(new) == pte_val(old)) \
323 continue; \
324 set_pte(pte + i, new); \
325 ret = 1; \
326 } \
327 return ret; \
328} \
329 \
330/* returns true if anything was done */ \
331static int kvm_mips_##name##_pmd(pmd_t *pmd, unsigned long start, \
332 unsigned long end) \
333{ \
334 int ret = 0; \
335 pte_t *pte; \
336 unsigned long cur_end = ~0ul; \
337 int i_min = __pmd_offset(start); \
338 int i_max = __pmd_offset(end); \
339 int i; \
340 \
341 for (i = i_min; i <= i_max; ++i, start = 0) { \
342 if (!pmd_present(pmd[i])) \
343 continue; \
344 \
345 pte = pte_offset(pmd + i, 0); \
346 if (i == i_max) \
347 cur_end = end; \
348 \
349 ret |= kvm_mips_##name##_pte(pte, start, cur_end); \
350 } \
351 return ret; \
352} \
353 \
354static int kvm_mips_##name##_pud(pud_t *pud, unsigned long start, \
355 unsigned long end) \
356{ \
357 int ret = 0; \
358 pmd_t *pmd; \
359 unsigned long cur_end = ~0ul; \
360 int i_min = __pud_offset(start); \
361 int i_max = __pud_offset(end); \
362 int i; \
363 \
364 for (i = i_min; i <= i_max; ++i, start = 0) { \
365 if (!pud_present(pud[i])) \
366 continue; \
367 \
368 pmd = pmd_offset(pud + i, 0); \
369 if (i == i_max) \
370 cur_end = end; \
371 \
372 ret |= kvm_mips_##name##_pmd(pmd, start, cur_end); \
373 } \
374 return ret; \
375} \
376 \
377static int kvm_mips_##name##_pgd(pgd_t *pgd, unsigned long start, \
378 unsigned long end) \
379{ \
380 int ret = 0; \
381 pud_t *pud; \
382 unsigned long cur_end = ~0ul; \
383 int i_min = pgd_index(start); \
384 int i_max = pgd_index(end); \
385 int i; \
386 \
387 for (i = i_min; i <= i_max; ++i, start = 0) { \
388 if (!pgd_present(pgd[i])) \
389 continue; \
390 \
391 pud = pud_offset(pgd + i, 0); \
392 if (i == i_max) \
393 cur_end = end; \
394 \
395 ret |= kvm_mips_##name##_pud(pud, start, cur_end); \
396 } \
397 return ret; \
398}
399
400/*
401 * kvm_mips_mkclean_gpa_pt.
402 * Mark a range of guest physical address space clean (writes fault) in the VM's
403 * GPA page table to allow dirty page tracking.
404 */
405
406BUILD_PTE_RANGE_OP(mkclean, pte_mkclean)
407
408/**
409 * kvm_mips_mkclean_gpa_pt() - Make a range of guest physical addresses clean.
410 * @kvm: KVM pointer.
411 * @start_gfn: Guest frame number of first page in GPA range to flush.
412 * @end_gfn: Guest frame number of last page in GPA range to flush.
413 *
414 * Make a range of GPA mappings clean so that guest writes will fault and
415 * trigger dirty page logging.
416 *
417 * The caller must hold the @kvm->mmu_lock spinlock.
418 *
419 * Returns: Whether any GPA mappings were modified, which would require
420 * derived mappings (GVA page tables & TLB enties) to be
421 * invalidated.
422 */
423int kvm_mips_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn)
424{
425 return kvm_mips_mkclean_pgd(kvm->arch.gpa_mm.pgd,
426 start_gfn << PAGE_SHIFT,
427 end_gfn << PAGE_SHIFT);
428}
429
430/**
431 * kvm_arch_mmu_enable_log_dirty_pt_masked() - write protect dirty pages
432 * @kvm: The KVM pointer
433 * @slot: The memory slot associated with mask
434 * @gfn_offset: The gfn offset in memory slot
435 * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory
436 * slot to be write protected
437 *
438 * Walks bits set in mask write protects the associated pte's. Caller must
439 * acquire @kvm->mmu_lock.
440 */
441void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
442 struct kvm_memory_slot *slot,
443 gfn_t gfn_offset, unsigned long mask)
444{
445 gfn_t base_gfn = slot->base_gfn + gfn_offset;
446 gfn_t start = base_gfn + __ffs(mask);
447 gfn_t end = base_gfn + __fls(mask);
448
449 kvm_mips_mkclean_gpa_pt(kvm, start, end);
450}
451
452/*
453 * kvm_mips_mkold_gpa_pt.
454 * Mark a range of guest physical address space old (all accesses fault) in the
455 * VM's GPA page table to allow detection of commonly used pages.
456 */
457
458BUILD_PTE_RANGE_OP(mkold, pte_mkold)
459
460static int kvm_mips_mkold_gpa_pt(struct kvm *kvm, gfn_t start_gfn,
461 gfn_t end_gfn)
462{
463 return kvm_mips_mkold_pgd(kvm->arch.gpa_mm.pgd,
464 start_gfn << PAGE_SHIFT,
465 end_gfn << PAGE_SHIFT);
466}
467
468static int handle_hva_to_gpa(struct kvm *kvm,
469 unsigned long start,
470 unsigned long end,
471 int (*handler)(struct kvm *kvm, gfn_t gfn,
472 gpa_t gfn_end,
473 struct kvm_memory_slot *memslot,
474 void *data),
475 void *data)
476{
477 struct kvm_memslots *slots;
478 struct kvm_memory_slot *memslot;
479 int ret = 0;
480
481 slots = kvm_memslots(kvm);
482
483 /* we only care about the pages that the guest sees */
484 kvm_for_each_memslot(memslot, slots) {
485 unsigned long hva_start, hva_end;
486 gfn_t gfn, gfn_end;
487
488 hva_start = max(start, memslot->userspace_addr);
489 hva_end = min(end, memslot->userspace_addr +
490 (memslot->npages << PAGE_SHIFT));
491 if (hva_start >= hva_end)
492 continue;
493
494 /*
495 * {gfn(page) | page intersects with [hva_start, hva_end)} =
496 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
497 */
498 gfn = hva_to_gfn_memslot(hva_start, memslot);
499 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
500
501 ret |= handler(kvm, gfn, gfn_end, memslot, data);
502 }
503
504 return ret;
505}
506
507
508static int kvm_unmap_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
509 struct kvm_memory_slot *memslot, void *data)
510{
511 kvm_mips_flush_gpa_pt(kvm, gfn, gfn_end);
512 return 1;
513}
514
515int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
516{
517 unsigned long end = hva + PAGE_SIZE;
518
519 handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL);
520
521 kvm_mips_callbacks->flush_shadow_all(kvm);
522 return 0;
523}
524
525int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
526{
527 handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);
528
529 kvm_mips_callbacks->flush_shadow_all(kvm);
530 return 0;
531}
532
533static int kvm_set_spte_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
534 struct kvm_memory_slot *memslot, void *data)
535{
536 gpa_t gpa = gfn << PAGE_SHIFT;
537 pte_t hva_pte = *(pte_t *)data;
538 pte_t *gpa_pte = kvm_mips_pte_for_gpa(kvm, NULL, gpa);
539 pte_t old_pte;
540
541 if (!gpa_pte)
542 return 0;
543
544 /* Mapping may need adjusting depending on memslot flags */
545 old_pte = *gpa_pte;
546 if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES && !pte_dirty(old_pte))
547 hva_pte = pte_mkclean(hva_pte);
548 else if (memslot->flags & KVM_MEM_READONLY)
549 hva_pte = pte_wrprotect(hva_pte);
550
551 set_pte(gpa_pte, hva_pte);
552
553 /* Replacing an absent or old page doesn't need flushes */
554 if (!pte_present(old_pte) || !pte_young(old_pte))
555 return 0;
556
557 /* Pages swapped, aged, moved, or cleaned require flushes */
558 return !pte_present(hva_pte) ||
559 !pte_young(hva_pte) ||
560 pte_pfn(old_pte) != pte_pfn(hva_pte) ||
561 (pte_dirty(old_pte) && !pte_dirty(hva_pte));
562}
563
564void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
565{
566 unsigned long end = hva + PAGE_SIZE;
567 int ret;
568
569 ret = handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &pte);
570 if (ret)
571 kvm_mips_callbacks->flush_shadow_all(kvm);
572}
573
574static int kvm_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
575 struct kvm_memory_slot *memslot, void *data)
576{
577 return kvm_mips_mkold_gpa_pt(kvm, gfn, gfn_end);
578}
579
580static int kvm_test_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
581 struct kvm_memory_slot *memslot, void *data)
582{
583 gpa_t gpa = gfn << PAGE_SHIFT;
584 pte_t *gpa_pte = kvm_mips_pte_for_gpa(kvm, NULL, gpa);
585
586 if (!gpa_pte)
587 return 0;
588 return pte_young(*gpa_pte);
589}
590
591int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
592{
593 return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
594}
595
596int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
597{
598 return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);
599}
600
601/**
602 * _kvm_mips_map_page_fast() - Fast path GPA fault handler.
603 * @vcpu: VCPU pointer.
604 * @gpa: Guest physical address of fault.
605 * @write_fault: Whether the fault was due to a write.
606 * @out_entry: New PTE for @gpa (written on success unless NULL).
607 * @out_buddy: New PTE for @gpa's buddy (written on success unless
608 * NULL).
609 *
610 * Perform fast path GPA fault handling, doing all that can be done without
611 * calling into KVM. This handles marking old pages young (for idle page
612 * tracking), and dirtying of clean pages (for dirty page logging).
613 *
614 * Returns: 0 on success, in which case we can update derived mappings and
615 * resume guest execution.
616 * -EFAULT on failure due to absent GPA mapping or write to
617 * read-only page, in which case KVM must be consulted.
618 */
619static int _kvm_mips_map_page_fast(struct kvm_vcpu *vcpu, unsigned long gpa,
620 bool write_fault,
621 pte_t *out_entry, pte_t *out_buddy)
622{
623 struct kvm *kvm = vcpu->kvm;
624 gfn_t gfn = gpa >> PAGE_SHIFT;
625 pte_t *ptep;
626 kvm_pfn_t pfn = 0; /* silence bogus GCC warning */
627 bool pfn_valid = false;
628 int ret = 0;
629
630 spin_lock(&kvm->mmu_lock);
631
632 /* Fast path - just check GPA page table for an existing entry */
633 ptep = kvm_mips_pte_for_gpa(kvm, NULL, gpa);
634 if (!ptep || !pte_present(*ptep)) {
635 ret = -EFAULT;
636 goto out;
637 }
638
639 /* Track access to pages marked old */
640 if (!pte_young(*ptep)) {
641 set_pte(ptep, pte_mkyoung(*ptep));
642 pfn = pte_pfn(*ptep);
643 pfn_valid = true;
644 /* call kvm_set_pfn_accessed() after unlock */
645 }
646 if (write_fault && !pte_dirty(*ptep)) {
647 if (!pte_write(*ptep)) {
648 ret = -EFAULT;
649 goto out;
650 }
651
652 /* Track dirtying of writeable pages */
653 set_pte(ptep, pte_mkdirty(*ptep));
654 pfn = pte_pfn(*ptep);
655 mark_page_dirty(kvm, gfn);
656 kvm_set_pfn_dirty(pfn);
657 }
658
659 if (out_entry)
660 *out_entry = *ptep;
661 if (out_buddy)
662 *out_buddy = *ptep_buddy(ptep);
663
664out:
665 spin_unlock(&kvm->mmu_lock);
666 if (pfn_valid)
667 kvm_set_pfn_accessed(pfn);
668 return ret;
669}
670
671/**
672 * kvm_mips_map_page() - Map a guest physical page.
673 * @vcpu: VCPU pointer.
674 * @gpa: Guest physical address of fault.
675 * @write_fault: Whether the fault was due to a write.
676 * @out_entry: New PTE for @gpa (written on success unless NULL).
677 * @out_buddy: New PTE for @gpa's buddy (written on success unless
678 * NULL).
679 *
680 * Handle GPA faults by creating a new GPA mapping (or updating an existing
681 * one).
682 *
683 * This takes care of marking pages young or dirty (idle/dirty page tracking),
684 * asking KVM for the corresponding PFN, and creating a mapping in the GPA page
685 * tables. Derived mappings (GVA page tables and TLBs) must be handled by the
686 * caller.
687 *
688 * Returns: 0 on success, in which case the caller may use the @out_entry
689 * and @out_buddy PTEs to update derived mappings and resume guest
690 * execution.
691 * -EFAULT if there is no memory region at @gpa or a write was
692 * attempted to a read-only memory region. This is usually handled
693 * as an MMIO access.
694 */
695static int kvm_mips_map_page(struct kvm_vcpu *vcpu, unsigned long gpa,
696 bool write_fault,
697 pte_t *out_entry, pte_t *out_buddy)
698{
699 struct kvm *kvm = vcpu->kvm;
700 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
701 gfn_t gfn = gpa >> PAGE_SHIFT;
702 int srcu_idx, err;
703 kvm_pfn_t pfn;
704 pte_t *ptep, entry, old_pte;
705 bool writeable;
706 unsigned long prot_bits;
707 unsigned long mmu_seq;
708
709 /* Try the fast path to handle old / clean pages */
710 srcu_idx = srcu_read_lock(&kvm->srcu);
711 err = _kvm_mips_map_page_fast(vcpu, gpa, write_fault, out_entry,
712 out_buddy);
713 if (!err)
714 goto out;
715
716 /* We need a minimum of cached pages ready for page table creation */
717 err = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES,
718 KVM_NR_MEM_OBJS);
719 if (err)
720 goto out;
721
722retry:
723 /*
724 * Used to check for invalidations in progress, of the pfn that is
725 * returned by pfn_to_pfn_prot below.
726 */
727 mmu_seq = kvm->mmu_notifier_seq;
728 /*
729 * Ensure the read of mmu_notifier_seq isn't reordered with PTE reads in
730 * gfn_to_pfn_prot() (which calls get_user_pages()), so that we don't
731 * risk the page we get a reference to getting unmapped before we have a
732 * chance to grab the mmu_lock without mmu_notifier_retry() noticing.
733 *
734 * This smp_rmb() pairs with the effective smp_wmb() of the combination
735 * of the pte_unmap_unlock() after the PTE is zapped, and the
736 * spin_lock() in kvm_mmu_notifier_invalidate_<page|range_end>() before
737 * mmu_notifier_seq is incremented.
738 */
739 smp_rmb();
740
741 /* Slow path - ask KVM core whether we can access this GPA */
742 pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writeable);
743 if (is_error_noslot_pfn(pfn)) {
744 err = -EFAULT;
745 goto out;
746 }
747
748 spin_lock(&kvm->mmu_lock);
749 /* Check if an invalidation has taken place since we got pfn */
750 if (mmu_notifier_retry(kvm, mmu_seq)) {
751 /*
752 * This can happen when mappings are changed asynchronously, but
753 * also synchronously if a COW is triggered by
754 * gfn_to_pfn_prot().
755 */
756 spin_unlock(&kvm->mmu_lock);
757 kvm_release_pfn_clean(pfn);
758 goto retry;
759 }
760
761 /* Ensure page tables are allocated */
762 ptep = kvm_mips_pte_for_gpa(kvm, memcache, gpa);
763
764 /* Set up the PTE */
765 prot_bits = _PAGE_PRESENT | __READABLE | _page_cachable_default;
766 if (writeable) {
767 prot_bits |= _PAGE_WRITE;
768 if (write_fault) {
769 prot_bits |= __WRITEABLE;
770 mark_page_dirty(kvm, gfn);
771 kvm_set_pfn_dirty(pfn);
772 }
773 }
774 entry = pfn_pte(pfn, __pgprot(prot_bits));
775
776 /* Write the PTE */
777 old_pte = *ptep;
778 set_pte(ptep, entry);
779
780 err = 0;
781 if (out_entry)
782 *out_entry = *ptep;
783 if (out_buddy)
784 *out_buddy = *ptep_buddy(ptep);
785
786 spin_unlock(&kvm->mmu_lock);
787 kvm_release_pfn_clean(pfn);
788 kvm_set_pfn_accessed(pfn);
789out:
790 srcu_read_unlock(&kvm->srcu, srcu_idx);
791 return err;
792}
793
794static pte_t *kvm_trap_emul_pte_for_gva(struct kvm_vcpu *vcpu,
795 unsigned long addr)
796{
797 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
798 pgd_t *pgdp;
799 int ret;
800
801 /* We need a minimum of cached pages ready for page table creation */
802 ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES,
803 KVM_NR_MEM_OBJS);
804 if (ret)
805 return NULL;
806
807 if (KVM_GUEST_KERNEL_MODE(vcpu))
808 pgdp = vcpu->arch.guest_kernel_mm.pgd;
809 else
810 pgdp = vcpu->arch.guest_user_mm.pgd;
811
812 return kvm_mips_walk_pgd(pgdp, memcache, addr);
813}
814
815void kvm_trap_emul_invalidate_gva(struct kvm_vcpu *vcpu, unsigned long addr,
816 bool user)
817{
818 pgd_t *pgdp;
819 pte_t *ptep;
820
821 addr &= PAGE_MASK << 1;
822
823 pgdp = vcpu->arch.guest_kernel_mm.pgd;
824 ptep = kvm_mips_walk_pgd(pgdp, NULL, addr);
825 if (ptep) {
826 ptep[0] = pfn_pte(0, __pgprot(0));
827 ptep[1] = pfn_pte(0, __pgprot(0));
828 }
829
830 if (user) {
831 pgdp = vcpu->arch.guest_user_mm.pgd;
832 ptep = kvm_mips_walk_pgd(pgdp, NULL, addr);
833 if (ptep) {
834 ptep[0] = pfn_pte(0, __pgprot(0));
835 ptep[1] = pfn_pte(0, __pgprot(0));
836 }
837 }
838}
839
840/*
841 * kvm_mips_flush_gva_{pte,pmd,pud,pgd,pt}.
842 * Flush a range of guest physical address space from the VM's GPA page tables.
843 */
844
845static bool kvm_mips_flush_gva_pte(pte_t *pte, unsigned long start_gva,
846 unsigned long end_gva)
847{
848 int i_min = __pte_offset(start_gva);
849 int i_max = __pte_offset(end_gva);
850 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1);
851 int i;
852
853 /*
854 * There's no freeing to do, so there's no point clearing individual
855 * entries unless only part of the last level page table needs flushing.
856 */
857 if (safe_to_remove)
858 return true;
859
860 for (i = i_min; i <= i_max; ++i) {
861 if (!pte_present(pte[i]))
862 continue;
863
864 set_pte(pte + i, __pte(0));
865 }
866 return false;
867}
868
869static bool kvm_mips_flush_gva_pmd(pmd_t *pmd, unsigned long start_gva,
870 unsigned long end_gva)
871{
872 pte_t *pte;
873 unsigned long end = ~0ul;
874 int i_min = __pmd_offset(start_gva);
875 int i_max = __pmd_offset(end_gva);
876 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1);
877 int i;
878
879 for (i = i_min; i <= i_max; ++i, start_gva = 0) {
880 if (!pmd_present(pmd[i]))
881 continue;
882
883 pte = pte_offset(pmd + i, 0);
884 if (i == i_max)
885 end = end_gva;
886
887 if (kvm_mips_flush_gva_pte(pte, start_gva, end)) {
888 pmd_clear(pmd + i);
889 pte_free_kernel(NULL, pte);
890 } else {
891 safe_to_remove = false;
892 }
893 }
894 return safe_to_remove;
895}
896
897static bool kvm_mips_flush_gva_pud(pud_t *pud, unsigned long start_gva,
898 unsigned long end_gva)
899{
900 pmd_t *pmd;
901 unsigned long end = ~0ul;
902 int i_min = __pud_offset(start_gva);
903 int i_max = __pud_offset(end_gva);
904 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1);
905 int i;
906
907 for (i = i_min; i <= i_max; ++i, start_gva = 0) {
908 if (!pud_present(pud[i]))
909 continue;
910
911 pmd = pmd_offset(pud + i, 0);
912 if (i == i_max)
913 end = end_gva;
914
915 if (kvm_mips_flush_gva_pmd(pmd, start_gva, end)) {
916 pud_clear(pud + i);
917 pmd_free(NULL, pmd);
918 } else {
919 safe_to_remove = false;
920 }
921 }
922 return safe_to_remove;
923}
924
925static bool kvm_mips_flush_gva_pgd(pgd_t *pgd, unsigned long start_gva,
926 unsigned long end_gva)
927{
928 pud_t *pud;
929 unsigned long end = ~0ul;
930 int i_min = pgd_index(start_gva);
931 int i_max = pgd_index(end_gva);
932 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1);
933 int i;
934
935 for (i = i_min; i <= i_max; ++i, start_gva = 0) {
936 if (!pgd_present(pgd[i]))
937 continue;
938
939 pud = pud_offset(pgd + i, 0);
940 if (i == i_max)
941 end = end_gva;
942
943 if (kvm_mips_flush_gva_pud(pud, start_gva, end)) {
944 pgd_clear(pgd + i);
945 pud_free(NULL, pud);
946 } else {
947 safe_to_remove = false;
948 }
949 }
950 return safe_to_remove;
951}
952
953void kvm_mips_flush_gva_pt(pgd_t *pgd, enum kvm_mips_flush flags)
954{
955 if (flags & KMF_GPA) {
956 /* all of guest virtual address space could be affected */
957 if (flags & KMF_KERN)
958 /* useg, kseg0, seg2/3 */
959 kvm_mips_flush_gva_pgd(pgd, 0, 0x7fffffff);
960 else
961 /* useg */
962 kvm_mips_flush_gva_pgd(pgd, 0, 0x3fffffff);
963 } else {
964 /* useg */
965 kvm_mips_flush_gva_pgd(pgd, 0, 0x3fffffff);
966
967 /* kseg2/3 */
968 if (flags & KMF_KERN)
969 kvm_mips_flush_gva_pgd(pgd, 0x60000000, 0x7fffffff);
970 }
971}
972
973static pte_t kvm_mips_gpa_pte_to_gva_unmapped(pte_t pte)
974{
975 /*
976 * Don't leak writeable but clean entries from GPA page tables. We don't
977 * want the normal Linux tlbmod handler to handle dirtying when KVM
978 * accesses guest memory.
979 */
980 if (!pte_dirty(pte))
981 pte = pte_wrprotect(pte);
982
983 return pte;
984}
985
986static pte_t kvm_mips_gpa_pte_to_gva_mapped(pte_t pte, long entrylo)
987{
988 /* Guest EntryLo overrides host EntryLo */
989 if (!(entrylo & ENTRYLO_D))
990 pte = pte_mkclean(pte);
991
992 return kvm_mips_gpa_pte_to_gva_unmapped(pte);
993}
994
995#ifdef CONFIG_KVM_MIPS_VZ
996int kvm_mips_handle_vz_root_tlb_fault(unsigned long badvaddr,
997 struct kvm_vcpu *vcpu,
998 bool write_fault)
999{
1000 int ret;
1001
1002 ret = kvm_mips_map_page(vcpu, badvaddr, write_fault, NULL, NULL);
1003 if (ret)
1004 return ret;
1005
1006 /* Invalidate this entry in the TLB */
1007 return kvm_vz_host_tlb_inv(vcpu, badvaddr);
1008}
1009#endif
1010
1011/* XXXKYMA: Must be called with interrupts disabled */
1012int kvm_mips_handle_kseg0_tlb_fault(unsigned long badvaddr,
1013 struct kvm_vcpu *vcpu,
1014 bool write_fault)
1015{
1016 unsigned long gpa;
1017 pte_t pte_gpa[2], *ptep_gva;
1018 int idx;
1019
1020 if (KVM_GUEST_KSEGX(badvaddr) != KVM_GUEST_KSEG0) {
1021 kvm_err("%s: Invalid BadVaddr: %#lx\n", __func__, badvaddr);
1022 kvm_mips_dump_host_tlbs();
1023 return -1;
1024 }
1025
1026 /* Get the GPA page table entry */
1027 gpa = KVM_GUEST_CPHYSADDR(badvaddr);
1028 idx = (badvaddr >> PAGE_SHIFT) & 1;
1029 if (kvm_mips_map_page(vcpu, gpa, write_fault, &pte_gpa[idx],
1030 &pte_gpa[!idx]) < 0)
1031 return -1;
1032
1033 /* Get the GVA page table entry */
1034 ptep_gva = kvm_trap_emul_pte_for_gva(vcpu, badvaddr & ~PAGE_SIZE);
1035 if (!ptep_gva) {
1036 kvm_err("No ptep for gva %lx\n", badvaddr);
1037 return -1;
1038 }
1039
1040 /* Copy a pair of entries from GPA page table to GVA page table */
1041 ptep_gva[0] = kvm_mips_gpa_pte_to_gva_unmapped(pte_gpa[0]);
1042 ptep_gva[1] = kvm_mips_gpa_pte_to_gva_unmapped(pte_gpa[1]);
1043
1044 /* Invalidate this entry in the TLB, guest kernel ASID only */
1045 kvm_mips_host_tlb_inv(vcpu, badvaddr, false, true);
1046 return 0;
1047}
1048
1049int kvm_mips_handle_mapped_seg_tlb_fault(struct kvm_vcpu *vcpu,
1050 struct kvm_mips_tlb *tlb,
1051 unsigned long gva,
1052 bool write_fault)
1053{
1054 struct kvm *kvm = vcpu->kvm;
1055 long tlb_lo[2];
1056 pte_t pte_gpa[2], *ptep_buddy, *ptep_gva;
1057 unsigned int idx = TLB_LO_IDX(*tlb, gva);
1058 bool kernel = KVM_GUEST_KERNEL_MODE(vcpu);
1059
1060 tlb_lo[0] = tlb->tlb_lo[0];
1061 tlb_lo[1] = tlb->tlb_lo[1];
1062
1063 /*
1064 * The commpage address must not be mapped to anything else if the guest
1065 * TLB contains entries nearby, or commpage accesses will break.
1066 */
1067 if (!((gva ^ KVM_GUEST_COMMPAGE_ADDR) & VPN2_MASK & (PAGE_MASK << 1)))
1068 tlb_lo[TLB_LO_IDX(*tlb, KVM_GUEST_COMMPAGE_ADDR)] = 0;
1069
1070 /* Get the GPA page table entry */
1071 if (kvm_mips_map_page(vcpu, mips3_tlbpfn_to_paddr(tlb_lo[idx]),
1072 write_fault, &pte_gpa[idx], NULL) < 0)
1073 return -1;
1074
1075 /* And its GVA buddy's GPA page table entry if it also exists */
1076 pte_gpa[!idx] = pfn_pte(0, __pgprot(0));
1077 if (tlb_lo[!idx] & ENTRYLO_V) {
1078 spin_lock(&kvm->mmu_lock);
1079 ptep_buddy = kvm_mips_pte_for_gpa(kvm, NULL,
1080 mips3_tlbpfn_to_paddr(tlb_lo[!idx]));
1081 if (ptep_buddy)
1082 pte_gpa[!idx] = *ptep_buddy;
1083 spin_unlock(&kvm->mmu_lock);
1084 }
1085
1086 /* Get the GVA page table entry pair */
1087 ptep_gva = kvm_trap_emul_pte_for_gva(vcpu, gva & ~PAGE_SIZE);
1088 if (!ptep_gva) {
1089 kvm_err("No ptep for gva %lx\n", gva);
1090 return -1;
1091 }
1092
1093 /* Copy a pair of entries from GPA page table to GVA page table */
1094 ptep_gva[0] = kvm_mips_gpa_pte_to_gva_mapped(pte_gpa[0], tlb_lo[0]);
1095 ptep_gva[1] = kvm_mips_gpa_pte_to_gva_mapped(pte_gpa[1], tlb_lo[1]);
1096
1097 /* Invalidate this entry in the TLB, current guest mode ASID only */
1098 kvm_mips_host_tlb_inv(vcpu, gva, !kernel, kernel);
1099
1100 kvm_debug("@ %#lx tlb_lo0: 0x%08lx tlb_lo1: 0x%08lx\n", vcpu->arch.pc,
1101 tlb->tlb_lo[0], tlb->tlb_lo[1]);
1102
1103 return 0;
1104}
1105
1106int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr,
1107 struct kvm_vcpu *vcpu)
1108{
1109 kvm_pfn_t pfn;
1110 pte_t *ptep;
1111
1112 ptep = kvm_trap_emul_pte_for_gva(vcpu, badvaddr);
1113 if (!ptep) {
1114 kvm_err("No ptep for commpage %lx\n", badvaddr);
1115 return -1;
1116 }
1117
1118 pfn = PFN_DOWN(virt_to_phys(vcpu->arch.kseg0_commpage));
1119 /* Also set valid and dirty, so refill handler doesn't have to */
1120 *ptep = pte_mkyoung(pte_mkdirty(pfn_pte(pfn, PAGE_SHARED)));
1121
1122 /* Invalidate this entry in the TLB, guest kernel ASID only */
1123 kvm_mips_host_tlb_inv(vcpu, badvaddr, false, true);
1124 return 0;
1125}
1126
1127/**
1128 * kvm_mips_migrate_count() - Migrate timer.
1129 * @vcpu: Virtual CPU.
1130 *
1131 * Migrate CP0_Count hrtimer to the current CPU by cancelling and restarting it
1132 * if it was running prior to being cancelled.
1133 *
1134 * Must be called when the VCPU is migrated to a different CPU to ensure that
1135 * timer expiry during guest execution interrupts the guest and causes the
1136 * interrupt to be delivered in a timely manner.
1137 */
1138static void kvm_mips_migrate_count(struct kvm_vcpu *vcpu)
1139{
1140 if (hrtimer_cancel(&vcpu->arch.comparecount_timer))
1141 hrtimer_restart(&vcpu->arch.comparecount_timer);
1142}
1143
1144/* Restore ASID once we are scheduled back after preemption */
1145void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1146{
1147 unsigned long flags;
1148
1149 kvm_debug("%s: vcpu %p, cpu: %d\n", __func__, vcpu, cpu);
1150
1151 local_irq_save(flags);
1152
1153 vcpu->cpu = cpu;
1154 if (vcpu->arch.last_sched_cpu != cpu) {
1155 kvm_debug("[%d->%d]KVM VCPU[%d] switch\n",
1156 vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id);
1157 /*
1158 * Migrate the timer interrupt to the current CPU so that it
1159 * always interrupts the guest and synchronously triggers a
1160 * guest timer interrupt.
1161 */
1162 kvm_mips_migrate_count(vcpu);
1163 }
1164
1165 /* restore guest state to registers */
1166 kvm_mips_callbacks->vcpu_load(vcpu, cpu);
1167
1168 local_irq_restore(flags);
1169}
1170
1171/* ASID can change if another task is scheduled during preemption */
1172void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1173{
1174 unsigned long flags;
1175 int cpu;
1176
1177 local_irq_save(flags);
1178
1179 cpu = smp_processor_id();
1180 vcpu->arch.last_sched_cpu = cpu;
1181 vcpu->cpu = -1;
1182
1183 /* save guest state in registers */
1184 kvm_mips_callbacks->vcpu_put(vcpu, cpu);
1185
1186 local_irq_restore(flags);
1187}
1188
1189/**
1190 * kvm_trap_emul_gva_fault() - Safely attempt to handle a GVA access fault.
1191 * @vcpu: Virtual CPU.
1192 * @gva: Guest virtual address to be accessed.
1193 * @write: True if write attempted (must be dirtied and made writable).
1194 *
1195 * Safely attempt to handle a GVA fault, mapping GVA pages if necessary, and
1196 * dirtying the page if @write so that guest instructions can be modified.
1197 *
1198 * Returns: KVM_MIPS_MAPPED on success.
1199 * KVM_MIPS_GVA if bad guest virtual address.
1200 * KVM_MIPS_GPA if bad guest physical address.
1201 * KVM_MIPS_TLB if guest TLB not present.
1202 * KVM_MIPS_TLBINV if guest TLB present but not valid.
1203 * KVM_MIPS_TLBMOD if guest TLB read only.
1204 */
1205enum kvm_mips_fault_result kvm_trap_emul_gva_fault(struct kvm_vcpu *vcpu,
1206 unsigned long gva,
1207 bool write)
1208{
1209 struct mips_coproc *cop0 = vcpu->arch.cop0;
1210 struct kvm_mips_tlb *tlb;
1211 int index;
1212
1213 if (KVM_GUEST_KSEGX(gva) == KVM_GUEST_KSEG0) {
1214 if (kvm_mips_handle_kseg0_tlb_fault(gva, vcpu, write) < 0)
1215 return KVM_MIPS_GPA;
1216 } else if ((KVM_GUEST_KSEGX(gva) < KVM_GUEST_KSEG0) ||
1217 KVM_GUEST_KSEGX(gva) == KVM_GUEST_KSEG23) {
1218 /* Address should be in the guest TLB */
1219 index = kvm_mips_guest_tlb_lookup(vcpu, (gva & VPN2_MASK) |
1220 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID));
1221 if (index < 0)
1222 return KVM_MIPS_TLB;
1223 tlb = &vcpu->arch.guest_tlb[index];
1224
1225 /* Entry should be valid, and dirty for writes */
1226 if (!TLB_IS_VALID(*tlb, gva))
1227 return KVM_MIPS_TLBINV;
1228 if (write && !TLB_IS_DIRTY(*tlb, gva))
1229 return KVM_MIPS_TLBMOD;
1230
1231 if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, gva, write))
1232 return KVM_MIPS_GPA;
1233 } else {
1234 return KVM_MIPS_GVA;
1235 }
1236
1237 return KVM_MIPS_MAPPED;
1238}
1239
1240int kvm_get_inst(u32 *opc, struct kvm_vcpu *vcpu, u32 *out)
1241{
1242 int err;
1243
1244 if (WARN(IS_ENABLED(CONFIG_KVM_MIPS_VZ),
1245 "Expect BadInstr/BadInstrP registers to be used with VZ\n"))
1246 return -EINVAL;
1247
1248retry:
1249 kvm_trap_emul_gva_lockless_begin(vcpu);
1250 err = get_user(*out, opc);
1251 kvm_trap_emul_gva_lockless_end(vcpu);
1252
1253 if (unlikely(err)) {
1254 /*
1255 * Try to handle the fault, maybe we just raced with a GVA
1256 * invalidation.
1257 */
1258 err = kvm_trap_emul_gva_fault(vcpu, (unsigned long)opc,
1259 false);
1260 if (unlikely(err)) {
1261 kvm_err("%s: illegal address: %p\n",
1262 __func__, opc);
1263 return -EFAULT;
1264 }
1265
1266 /* Hopefully it'll work now */
1267 goto retry;
1268 }
1269 return 0;
1270}