Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * KVM dirty ring implementation
4 *
5 * Copyright 2019 Red Hat, Inc.
6 */
7#include <linux/kvm_host.h>
8#include <linux/kvm.h>
9#include <linux/vmalloc.h>
10#include <linux/kvm_dirty_ring.h>
11#include <trace/events/kvm.h>
12#include "kvm_mm.h"
13
14int __weak kvm_cpu_dirty_log_size(void)
15{
16 return 0;
17}
18
19u32 kvm_dirty_ring_get_rsvd_entries(void)
20{
21 return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size();
22}
23
24bool kvm_use_dirty_bitmap(struct kvm *kvm)
25{
26 lockdep_assert_held(&kvm->slots_lock);
27
28 return !kvm->dirty_ring_size || kvm->dirty_ring_with_bitmap;
29}
30
31#ifndef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP
32bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm)
33{
34 return false;
35}
36#endif
37
38static u32 kvm_dirty_ring_used(struct kvm_dirty_ring *ring)
39{
40 return READ_ONCE(ring->dirty_index) - READ_ONCE(ring->reset_index);
41}
42
43static bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)
44{
45 return kvm_dirty_ring_used(ring) >= ring->soft_limit;
46}
47
48static bool kvm_dirty_ring_full(struct kvm_dirty_ring *ring)
49{
50 return kvm_dirty_ring_used(ring) >= ring->size;
51}
52
53static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask)
54{
55 struct kvm_memory_slot *memslot;
56 int as_id, id;
57
58 if (!mask)
59 return;
60
61 as_id = slot >> 16;
62 id = (u16)slot;
63
64 if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
65 return;
66
67 memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
68
69 if (!memslot || (offset + __fls(mask)) >= memslot->npages)
70 return;
71
72 KVM_MMU_LOCK(kvm);
73 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask);
74 KVM_MMU_UNLOCK(kvm);
75}
76
77int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size)
78{
79 ring->dirty_gfns = vzalloc(size);
80 if (!ring->dirty_gfns)
81 return -ENOMEM;
82
83 ring->size = size / sizeof(struct kvm_dirty_gfn);
84 ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries();
85 ring->dirty_index = 0;
86 ring->reset_index = 0;
87 ring->index = index;
88
89 return 0;
90}
91
92static inline void kvm_dirty_gfn_set_invalid(struct kvm_dirty_gfn *gfn)
93{
94 smp_store_release(&gfn->flags, 0);
95}
96
97static inline void kvm_dirty_gfn_set_dirtied(struct kvm_dirty_gfn *gfn)
98{
99 gfn->flags = KVM_DIRTY_GFN_F_DIRTY;
100}
101
102static inline bool kvm_dirty_gfn_harvested(struct kvm_dirty_gfn *gfn)
103{
104 return smp_load_acquire(&gfn->flags) & KVM_DIRTY_GFN_F_RESET;
105}
106
107int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring)
108{
109 u32 cur_slot, next_slot;
110 u64 cur_offset, next_offset;
111 unsigned long mask;
112 int count = 0;
113 struct kvm_dirty_gfn *entry;
114 bool first_round = true;
115
116 /* This is only needed to make compilers happy */
117 cur_slot = cur_offset = mask = 0;
118
119 while (true) {
120 entry = &ring->dirty_gfns[ring->reset_index & (ring->size - 1)];
121
122 if (!kvm_dirty_gfn_harvested(entry))
123 break;
124
125 next_slot = READ_ONCE(entry->slot);
126 next_offset = READ_ONCE(entry->offset);
127
128 /* Update the flags to reflect that this GFN is reset */
129 kvm_dirty_gfn_set_invalid(entry);
130
131 ring->reset_index++;
132 count++;
133 /*
134 * Try to coalesce the reset operations when the guest is
135 * scanning pages in the same slot.
136 */
137 if (!first_round && next_slot == cur_slot) {
138 s64 delta = next_offset - cur_offset;
139
140 if (delta >= 0 && delta < BITS_PER_LONG) {
141 mask |= 1ull << delta;
142 continue;
143 }
144
145 /* Backwards visit, careful about overflows! */
146 if (delta > -BITS_PER_LONG && delta < 0 &&
147 (mask << -delta >> -delta) == mask) {
148 cur_offset = next_offset;
149 mask = (mask << -delta) | 1;
150 continue;
151 }
152 }
153 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
154 cur_slot = next_slot;
155 cur_offset = next_offset;
156 mask = 1;
157 first_round = false;
158 }
159
160 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
161
162 /*
163 * The request KVM_REQ_DIRTY_RING_SOFT_FULL will be cleared
164 * by the VCPU thread next time when it enters the guest.
165 */
166
167 trace_kvm_dirty_ring_reset(ring);
168
169 return count;
170}
171
172void kvm_dirty_ring_push(struct kvm_vcpu *vcpu, u32 slot, u64 offset)
173{
174 struct kvm_dirty_ring *ring = &vcpu->dirty_ring;
175 struct kvm_dirty_gfn *entry;
176
177 /* It should never get full */
178 WARN_ON_ONCE(kvm_dirty_ring_full(ring));
179
180 entry = &ring->dirty_gfns[ring->dirty_index & (ring->size - 1)];
181
182 entry->slot = slot;
183 entry->offset = offset;
184 /*
185 * Make sure the data is filled in before we publish this to
186 * the userspace program. There's no paired kernel-side reader.
187 */
188 smp_wmb();
189 kvm_dirty_gfn_set_dirtied(entry);
190 ring->dirty_index++;
191 trace_kvm_dirty_ring_push(ring, slot, offset);
192
193 if (kvm_dirty_ring_soft_full(ring))
194 kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu);
195}
196
197bool kvm_dirty_ring_check_request(struct kvm_vcpu *vcpu)
198{
199 /*
200 * The VCPU isn't runnable when the dirty ring becomes soft full.
201 * The KVM_REQ_DIRTY_RING_SOFT_FULL event is always set to prevent
202 * the VCPU from running until the dirty pages are harvested and
203 * the dirty ring is reset by userspace.
204 */
205 if (kvm_check_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu) &&
206 kvm_dirty_ring_soft_full(&vcpu->dirty_ring)) {
207 kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu);
208 vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL;
209 trace_kvm_dirty_ring_exit(vcpu);
210 return true;
211 }
212
213 return false;
214}
215
216struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset)
217{
218 return vmalloc_to_page((void *)ring->dirty_gfns + offset * PAGE_SIZE);
219}
220
221void kvm_dirty_ring_free(struct kvm_dirty_ring *ring)
222{
223 vfree(ring->dirty_gfns);
224 ring->dirty_gfns = NULL;
225}
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * KVM dirty ring implementation
4 *
5 * Copyright 2019 Red Hat, Inc.
6 */
7#include <linux/kvm_host.h>
8#include <linux/kvm.h>
9#include <linux/vmalloc.h>
10#include <linux/kvm_dirty_ring.h>
11#include <trace/events/kvm.h>
12#include "mmu_lock.h"
13
14int __weak kvm_cpu_dirty_log_size(void)
15{
16 return 0;
17}
18
19u32 kvm_dirty_ring_get_rsvd_entries(void)
20{
21 return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size();
22}
23
24static u32 kvm_dirty_ring_used(struct kvm_dirty_ring *ring)
25{
26 return READ_ONCE(ring->dirty_index) - READ_ONCE(ring->reset_index);
27}
28
29bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)
30{
31 return kvm_dirty_ring_used(ring) >= ring->soft_limit;
32}
33
34static bool kvm_dirty_ring_full(struct kvm_dirty_ring *ring)
35{
36 return kvm_dirty_ring_used(ring) >= ring->size;
37}
38
39struct kvm_dirty_ring *kvm_dirty_ring_get(struct kvm *kvm)
40{
41 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
42
43 WARN_ON_ONCE(vcpu->kvm != kvm);
44
45 return &vcpu->dirty_ring;
46}
47
48static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask)
49{
50 struct kvm_memory_slot *memslot;
51 int as_id, id;
52
53 as_id = slot >> 16;
54 id = (u16)slot;
55
56 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
57 return;
58
59 memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
60
61 if (!memslot || (offset + __fls(mask)) >= memslot->npages)
62 return;
63
64 KVM_MMU_LOCK(kvm);
65 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask);
66 KVM_MMU_UNLOCK(kvm);
67}
68
69int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size)
70{
71 ring->dirty_gfns = vzalloc(size);
72 if (!ring->dirty_gfns)
73 return -ENOMEM;
74
75 ring->size = size / sizeof(struct kvm_dirty_gfn);
76 ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries();
77 ring->dirty_index = 0;
78 ring->reset_index = 0;
79 ring->index = index;
80
81 return 0;
82}
83
84static inline void kvm_dirty_gfn_set_invalid(struct kvm_dirty_gfn *gfn)
85{
86 gfn->flags = 0;
87}
88
89static inline void kvm_dirty_gfn_set_dirtied(struct kvm_dirty_gfn *gfn)
90{
91 gfn->flags = KVM_DIRTY_GFN_F_DIRTY;
92}
93
94static inline bool kvm_dirty_gfn_invalid(struct kvm_dirty_gfn *gfn)
95{
96 return gfn->flags == 0;
97}
98
99static inline bool kvm_dirty_gfn_harvested(struct kvm_dirty_gfn *gfn)
100{
101 return gfn->flags & KVM_DIRTY_GFN_F_RESET;
102}
103
104int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring)
105{
106 u32 cur_slot, next_slot;
107 u64 cur_offset, next_offset;
108 unsigned long mask;
109 int count = 0;
110 struct kvm_dirty_gfn *entry;
111 bool first_round = true;
112
113 /* This is only needed to make compilers happy */
114 cur_slot = cur_offset = mask = 0;
115
116 while (true) {
117 entry = &ring->dirty_gfns[ring->reset_index & (ring->size - 1)];
118
119 if (!kvm_dirty_gfn_harvested(entry))
120 break;
121
122 next_slot = READ_ONCE(entry->slot);
123 next_offset = READ_ONCE(entry->offset);
124
125 /* Update the flags to reflect that this GFN is reset */
126 kvm_dirty_gfn_set_invalid(entry);
127
128 ring->reset_index++;
129 count++;
130 /*
131 * Try to coalesce the reset operations when the guest is
132 * scanning pages in the same slot.
133 */
134 if (!first_round && next_slot == cur_slot) {
135 s64 delta = next_offset - cur_offset;
136
137 if (delta >= 0 && delta < BITS_PER_LONG) {
138 mask |= 1ull << delta;
139 continue;
140 }
141
142 /* Backwards visit, careful about overflows! */
143 if (delta > -BITS_PER_LONG && delta < 0 &&
144 (mask << -delta >> -delta) == mask) {
145 cur_offset = next_offset;
146 mask = (mask << -delta) | 1;
147 continue;
148 }
149 }
150 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
151 cur_slot = next_slot;
152 cur_offset = next_offset;
153 mask = 1;
154 first_round = false;
155 }
156
157 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
158
159 trace_kvm_dirty_ring_reset(ring);
160
161 return count;
162}
163
164void kvm_dirty_ring_push(struct kvm_dirty_ring *ring, u32 slot, u64 offset)
165{
166 struct kvm_dirty_gfn *entry;
167
168 /* It should never get full */
169 WARN_ON_ONCE(kvm_dirty_ring_full(ring));
170
171 entry = &ring->dirty_gfns[ring->dirty_index & (ring->size - 1)];
172
173 entry->slot = slot;
174 entry->offset = offset;
175 /*
176 * Make sure the data is filled in before we publish this to
177 * the userspace program. There's no paired kernel-side reader.
178 */
179 smp_wmb();
180 kvm_dirty_gfn_set_dirtied(entry);
181 ring->dirty_index++;
182 trace_kvm_dirty_ring_push(ring, slot, offset);
183}
184
185struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset)
186{
187 return vmalloc_to_page((void *)ring->dirty_gfns + offset * PAGE_SIZE);
188}
189
190void kvm_dirty_ring_free(struct kvm_dirty_ring *ring)
191{
192 vfree(ring->dirty_gfns);
193 ring->dirty_gfns = NULL;
194}