1/*
  2 * Support KVM gust page tracking
  3 *
  4 * This feature allows us to track page access in guest. Currently, only
  5 * write access is tracked.
  6 *
  7 * Copyright(C) 2015 Intel Corporation.
  8 *
  9 * Author:
 10 *   Xiao Guangrong <guangrong.xiao@linux.intel.com>
 11 *
 12 * This work is licensed under the terms of the GNU GPL, version 2.  See
 13 * the COPYING file in the top-level directory.
 14 */
 15
 16#include <linux/kvm_host.h>
 17#include <linux/rculist.h>
 18
 19#include <asm/kvm_host.h>
 20#include <asm/kvm_page_track.h>
 21
 22#include "mmu.h"
 23
 24void kvm_page_track_free_memslot(struct kvm_memory_slot *free,
 25				 struct kvm_memory_slot *dont)
 26{
 27	int i;
 28
 29	for (i = 0; i < KVM_PAGE_TRACK_MAX; i++)
 30		if (!dont || free->arch.gfn_track[i] !=
 31		      dont->arch.gfn_track[i]) {
 32			kvfree(free->arch.gfn_track[i]);
 33			free->arch.gfn_track[i] = NULL;
 34		}
 35}
 36
 37int kvm_page_track_create_memslot(struct kvm_memory_slot *slot,
 38				  unsigned long npages)
 39{
 40	int  i;
 41
 42	for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) {
 43		slot->arch.gfn_track[i] = kvzalloc(npages *
 44					    sizeof(*slot->arch.gfn_track[i]), GFP_KERNEL);
 45		if (!slot->arch.gfn_track[i])
 46			goto track_free;
 47	}
 48
 49	return 0;
 50
 51track_free:
 52	kvm_page_track_free_memslot(slot, NULL);
 53	return -ENOMEM;
 54}
 55
 56static inline bool page_track_mode_is_valid(enum kvm_page_track_mode mode)
 57{
 58	if (mode < 0 || mode >= KVM_PAGE_TRACK_MAX)
 59		return false;
 60
 61	return true;
 62}
 63
 64static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn,
 65			     enum kvm_page_track_mode mode, short count)
 66{
 67	int index, val;
 68
 69	index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL);
 70
 71	val = slot->arch.gfn_track[mode][index];
 72
 73	if (WARN_ON(val + count < 0 || val + count > USHRT_MAX))
 74		return;
 75
 76	slot->arch.gfn_track[mode][index] += count;
 77}
 78
 79/*
 80 * add guest page to the tracking pool so that corresponding access on that
 81 * page will be intercepted.
 82 *
 83 * It should be called under the protection both of mmu-lock and kvm->srcu
 84 * or kvm->slots_lock.
 85 *
 86 * @kvm: the guest instance we are interested in.
 87 * @slot: the @gfn belongs to.
 88 * @gfn: the guest page.
 89 * @mode: tracking mode, currently only write track is supported.
 90 */
 91void kvm_slot_page_track_add_page(struct kvm *kvm,
 92				  struct kvm_memory_slot *slot, gfn_t gfn,
 93				  enum kvm_page_track_mode mode)
 94{
 95
 96	if (WARN_ON(!page_track_mode_is_valid(mode)))
 97		return;
 98
 99	update_gfn_track(slot, gfn, mode, 1);
100
101	/*
102	 * new track stops large page mapping for the
103	 * tracked page.
104	 */
105	kvm_mmu_gfn_disallow_lpage(slot, gfn);
106
107	if (mode == KVM_PAGE_TRACK_WRITE)
108		if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn))
109			kvm_flush_remote_tlbs(kvm);
110}
111EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page);
112
113/*
114 * remove the guest page from the tracking pool which stops the interception
115 * of corresponding access on that page. It is the opposed operation of
116 * kvm_slot_page_track_add_page().
117 *
118 * It should be called under the protection both of mmu-lock and kvm->srcu
119 * or kvm->slots_lock.
120 *
121 * @kvm: the guest instance we are interested in.
122 * @slot: the @gfn belongs to.
123 * @gfn: the guest page.
124 * @mode: tracking mode, currently only write track is supported.
125 */
126void kvm_slot_page_track_remove_page(struct kvm *kvm,
127				     struct kvm_memory_slot *slot, gfn_t gfn,
128				     enum kvm_page_track_mode mode)
129{
130	if (WARN_ON(!page_track_mode_is_valid(mode)))
131		return;
132
133	update_gfn_track(slot, gfn, mode, -1);
134
135	/*
136	 * allow large page mapping for the tracked page
137	 * after the tracker is gone.
138	 */
139	kvm_mmu_gfn_allow_lpage(slot, gfn);
140}
141EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page);
142
143/*
144 * check if the corresponding access on the specified guest page is tracked.
145 */
146bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn,
147			      enum kvm_page_track_mode mode)
148{
149	struct kvm_memory_slot *slot;
150	int index;
151
152	if (WARN_ON(!page_track_mode_is_valid(mode)))
153		return false;
154
155	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
156	if (!slot)
157		return false;
158
159	index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL);
160	return !!READ_ONCE(slot->arch.gfn_track[mode][index]);
161}
162
163void kvm_page_track_cleanup(struct kvm *kvm)
164{
165	struct kvm_page_track_notifier_head *head;
166
167	head = &kvm->arch.track_notifier_head;
168	cleanup_srcu_struct(&head->track_srcu);
169}
170
171void kvm_page_track_init(struct kvm *kvm)
172{
173	struct kvm_page_track_notifier_head *head;
174
175	head = &kvm->arch.track_notifier_head;
176	init_srcu_struct(&head->track_srcu);
177	INIT_HLIST_HEAD(&head->track_notifier_list);
178}
179
180/*
181 * register the notifier so that event interception for the tracked guest
182 * pages can be received.
183 */
184void
185kvm_page_track_register_notifier(struct kvm *kvm,
186				 struct kvm_page_track_notifier_node *n)
187{
188	struct kvm_page_track_notifier_head *head;
189
190	head = &kvm->arch.track_notifier_head;
191
192	spin_lock(&kvm->mmu_lock);
193	hlist_add_head_rcu(&n->node, &head->track_notifier_list);
194	spin_unlock(&kvm->mmu_lock);
195}
196EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier);
197
198/*
199 * stop receiving the event interception. It is the opposed operation of
200 * kvm_page_track_register_notifier().
201 */
202void
203kvm_page_track_unregister_notifier(struct kvm *kvm,
204				   struct kvm_page_track_notifier_node *n)
205{
206	struct kvm_page_track_notifier_head *head;
207
208	head = &kvm->arch.track_notifier_head;
209
210	spin_lock(&kvm->mmu_lock);
211	hlist_del_rcu(&n->node);
212	spin_unlock(&kvm->mmu_lock);
213	synchronize_srcu(&head->track_srcu);
214}
215EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier);
216
217/*
218 * Notify the node that write access is intercepted and write emulation is
219 * finished at this time.
220 *
221 * The node should figure out if the written page is the one that node is
222 * interested in by itself.
223 */
224void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
225			  int bytes)
226{
227	struct kvm_page_track_notifier_head *head;
228	struct kvm_page_track_notifier_node *n;
229	int idx;
230
231	head = &vcpu->kvm->arch.track_notifier_head;
232
233	if (hlist_empty(&head->track_notifier_list))
234		return;
235
236	idx = srcu_read_lock(&head->track_srcu);
237	hlist_for_each_entry_rcu(n, &head->track_notifier_list, node)
238		if (n->track_write)
239			n->track_write(vcpu, gpa, new, bytes, n);
240	srcu_read_unlock(&head->track_srcu, idx);
241}
242
243/*
244 * Notify the node that memory slot is being removed or moved so that it can
245 * drop write-protection for the pages in the memory slot.
246 *
247 * The node should figure out it has any write-protected pages in this slot
248 * by itself.
249 */
250void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot)
251{
252	struct kvm_page_track_notifier_head *head;
253	struct kvm_page_track_notifier_node *n;
254	int idx;
255
256	head = &kvm->arch.track_notifier_head;
257
258	if (hlist_empty(&head->track_notifier_list))
259		return;
260
261	idx = srcu_read_lock(&head->track_srcu);
262	hlist_for_each_entry_rcu(n, &head->track_notifier_list, node)
263		if (n->track_flush_slot)
264			n->track_flush_slot(kvm, slot, n);
265	srcu_read_unlock(&head->track_srcu, idx);
266}