1/*
   2 * Kernel-based Virtual Machine driver for Linux
   3 *
   4 * This module enables machines with Intel VT-x extensions to run virtual
   5 * machines without emulation or binary translation.
   6 *
   7 * Copyright (C) 2006 Qumranet, Inc.
   8 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9 *
  10 * Authors:
  11 *   Avi Kivity   <avi@qumranet.com>
  12 *   Yaniv Kamay  <yaniv@qumranet.com>
  13 *
  14 * This work is licensed under the terms of the GNU GPL, version 2.  See
  15 * the COPYING file in the top-level directory.
  16 *
  17 */
  18
  19#include "iodev.h"
  20
  21#include <linux/kvm_host.h>
  22#include <linux/kvm.h>
  23#include <linux/module.h>
  24#include <linux/errno.h>
  25#include <linux/percpu.h>
  26#include <linux/mm.h>
  27#include <linux/miscdevice.h>
  28#include <linux/vmalloc.h>
  29#include <linux/reboot.h>
  30#include <linux/debugfs.h>
  31#include <linux/highmem.h>
  32#include <linux/file.h>
  33#include <linux/syscore_ops.h>
  34#include <linux/cpu.h>
  35#include <linux/sched.h>
  36#include <linux/cpumask.h>
  37#include <linux/smp.h>
  38#include <linux/anon_inodes.h>
  39#include <linux/profile.h>
  40#include <linux/kvm_para.h>
  41#include <linux/pagemap.h>
  42#include <linux/mman.h>
  43#include <linux/swap.h>
  44#include <linux/bitops.h>
  45#include <linux/spinlock.h>
  46#include <linux/compat.h>
  47#include <linux/srcu.h>
  48#include <linux/hugetlb.h>
  49#include <linux/slab.h>
 
 
  50
  51#include <asm/processor.h>
  52#include <asm/io.h>
  53#include <asm/uaccess.h>
  54#include <asm/pgtable.h>
  55
  56#include "coalesced_mmio.h"
  57#include "async_pf.h"
  58
  59#define CREATE_TRACE_POINTS
  60#include <trace/events/kvm.h>
  61
  62MODULE_AUTHOR("Qumranet");
  63MODULE_LICENSE("GPL");
  64
  65/*
  66 * Ordering of locks:
  67 *
  68 * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  69 */
  70
  71DEFINE_RAW_SPINLOCK(kvm_lock);
 
  72LIST_HEAD(vm_list);
  73
  74static cpumask_var_t cpus_hardware_enabled;
  75static int kvm_usage_count = 0;
  76static atomic_t hardware_enable_failed;
  77
  78struct kmem_cache *kvm_vcpu_cache;
  79EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  80
  81static __read_mostly struct preempt_ops kvm_preempt_ops;
  82
  83struct dentry *kvm_debugfs_dir;
  84
  85static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
  86			   unsigned long arg);
  87#ifdef CONFIG_COMPAT
  88static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
  89				  unsigned long arg);
  90#endif
  91static int hardware_enable_all(void);
  92static void hardware_disable_all(void);
  93
  94static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
 
 
  95
  96bool kvm_rebooting;
 
 
 
 
  97EXPORT_SYMBOL_GPL(kvm_rebooting);
  98
  99static bool largepages_enabled = true;
 100
 101static struct page *hwpoison_page;
 102static pfn_t hwpoison_pfn;
 103
 104struct page *fault_page;
 105pfn_t fault_pfn;
 106
 107inline int kvm_is_mmio_pfn(pfn_t pfn)
 108{
 109	if (pfn_valid(pfn)) {
 110		int reserved;
 111		struct page *tail = pfn_to_page(pfn);
 112		struct page *head = compound_trans_head(tail);
 113		reserved = PageReserved(head);
 114		if (head != tail) {
 115			/*
 116			 * "head" is not a dangling pointer
 117			 * (compound_trans_head takes care of that)
 118			 * but the hugepage may have been splitted
 119			 * from under us (and we may not hold a
 120			 * reference count on the head page so it can
 121			 * be reused before we run PageReferenced), so
 122			 * we've to check PageTail before returning
 123			 * what we just read.
 124			 */
 125			smp_rmb();
 126			if (PageTail(tail))
 127				return reserved;
 128		}
 129		return PageReserved(tail);
 130	}
 131
 132	return true;
 133}
 134
 135/*
 136 * Switches to specified vcpu, until a matching vcpu_put()
 137 */
 138void vcpu_load(struct kvm_vcpu *vcpu)
 139{
 140	int cpu;
 141
 142	mutex_lock(&vcpu->mutex);
 
 143	if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
 144		/* The thread running this VCPU changed. */
 145		struct pid *oldpid = vcpu->pid;
 146		struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
 147		rcu_assign_pointer(vcpu->pid, newpid);
 148		synchronize_rcu();
 149		put_pid(oldpid);
 150	}
 151	cpu = get_cpu();
 152	preempt_notifier_register(&vcpu->preempt_notifier);
 153	kvm_arch_vcpu_load(vcpu, cpu);
 154	put_cpu();
 
 155}
 156
 157void vcpu_put(struct kvm_vcpu *vcpu)
 158{
 159	preempt_disable();
 160	kvm_arch_vcpu_put(vcpu);
 161	preempt_notifier_unregister(&vcpu->preempt_notifier);
 162	preempt_enable();
 163	mutex_unlock(&vcpu->mutex);
 164}
 165
 166static void ack_flush(void *_completed)
 167{
 168}
 169
 170static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
 171{
 172	int i, cpu, me;
 173	cpumask_var_t cpus;
 174	bool called = true;
 175	struct kvm_vcpu *vcpu;
 176
 177	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
 178
 179	me = get_cpu();
 180	kvm_for_each_vcpu(i, vcpu, kvm) {
 181		kvm_make_request(req, vcpu);
 182		cpu = vcpu->cpu;
 183
 184		/* Set ->requests bit before we read ->mode */
 185		smp_mb();
 186
 187		if (cpus != NULL && cpu != -1 && cpu != me &&
 188		      kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
 189			cpumask_set_cpu(cpu, cpus);
 190	}
 191	if (unlikely(cpus == NULL))
 192		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
 193	else if (!cpumask_empty(cpus))
 194		smp_call_function_many(cpus, ack_flush, NULL, 1);
 195	else
 196		called = false;
 197	put_cpu();
 198	free_cpumask_var(cpus);
 199	return called;
 200}
 201
 202void kvm_flush_remote_tlbs(struct kvm *kvm)
 203{
 204	int dirty_count = kvm->tlbs_dirty;
 205
 206	smp_mb();
 207	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
 208		++kvm->stat.remote_tlb_flush;
 209	cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
 210}
 
 211
 212void kvm_reload_remote_mmus(struct kvm *kvm)
 213{
 214	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
 215}
 216
 
 
 
 
 
 
 
 
 
 
 217int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 218{
 219	struct page *page;
 220	int r;
 221
 222	mutex_init(&vcpu->mutex);
 223	vcpu->cpu = -1;
 224	vcpu->kvm = kvm;
 225	vcpu->vcpu_id = id;
 226	vcpu->pid = NULL;
 227	init_waitqueue_head(&vcpu->wq);
 228	kvm_async_pf_vcpu_init(vcpu);
 229
 230	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 231	if (!page) {
 232		r = -ENOMEM;
 233		goto fail;
 234	}
 235	vcpu->run = page_address(page);
 236
 
 
 
 
 237	r = kvm_arch_vcpu_init(vcpu);
 238	if (r < 0)
 239		goto fail_free_run;
 240	return 0;
 241
 242fail_free_run:
 243	free_page((unsigned long)vcpu->run);
 244fail:
 245	return r;
 246}
 247EXPORT_SYMBOL_GPL(kvm_vcpu_init);
 248
 249void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 250{
 251	put_pid(vcpu->pid);
 252	kvm_arch_vcpu_uninit(vcpu);
 253	free_page((unsigned long)vcpu->run);
 254}
 255EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
 256
 257#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 258static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 259{
 260	return container_of(mn, struct kvm, mmu_notifier);
 261}
 262
 263static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 264					     struct mm_struct *mm,
 265					     unsigned long address)
 266{
 267	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 268	int need_tlb_flush, idx;
 269
 270	/*
 271	 * When ->invalidate_page runs, the linux pte has been zapped
 272	 * already but the page is still allocated until
 273	 * ->invalidate_page returns. So if we increase the sequence
 274	 * here the kvm page fault will notice if the spte can't be
 275	 * established because the page is going to be freed. If
 276	 * instead the kvm page fault establishes the spte before
 277	 * ->invalidate_page runs, kvm_unmap_hva will release it
 278	 * before returning.
 279	 *
 280	 * The sequence increase only need to be seen at spin_unlock
 281	 * time, and not at spin_lock time.
 282	 *
 283	 * Increasing the sequence after the spin_unlock would be
 284	 * unsafe because the kvm page fault could then establish the
 285	 * pte after kvm_unmap_hva returned, without noticing the page
 286	 * is going to be freed.
 287	 */
 288	idx = srcu_read_lock(&kvm->srcu);
 289	spin_lock(&kvm->mmu_lock);
 
 290	kvm->mmu_notifier_seq++;
 291	need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
 292	spin_unlock(&kvm->mmu_lock);
 293	srcu_read_unlock(&kvm->srcu, idx);
 294
 295	/* we've to flush the tlb before the pages can be freed */
 296	if (need_tlb_flush)
 297		kvm_flush_remote_tlbs(kvm);
 298
 
 
 299}
 300
 301static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
 302					struct mm_struct *mm,
 303					unsigned long address,
 304					pte_t pte)
 305{
 306	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 307	int idx;
 308
 309	idx = srcu_read_lock(&kvm->srcu);
 310	spin_lock(&kvm->mmu_lock);
 311	kvm->mmu_notifier_seq++;
 312	kvm_set_spte_hva(kvm, address, pte);
 313	spin_unlock(&kvm->mmu_lock);
 314	srcu_read_unlock(&kvm->srcu, idx);
 315}
 316
 317static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 318						    struct mm_struct *mm,
 319						    unsigned long start,
 320						    unsigned long end)
 321{
 322	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 323	int need_tlb_flush = 0, idx;
 324
 325	idx = srcu_read_lock(&kvm->srcu);
 326	spin_lock(&kvm->mmu_lock);
 327	/*
 328	 * The count increase must become visible at unlock time as no
 329	 * spte can be established without taking the mmu_lock and
 330	 * count is also read inside the mmu_lock critical section.
 331	 */
 332	kvm->mmu_notifier_count++;
 333	for (; start < end; start += PAGE_SIZE)
 334		need_tlb_flush |= kvm_unmap_hva(kvm, start);
 335	need_tlb_flush |= kvm->tlbs_dirty;
 336	spin_unlock(&kvm->mmu_lock);
 337	srcu_read_unlock(&kvm->srcu, idx);
 338
 339	/* we've to flush the tlb before the pages can be freed */
 340	if (need_tlb_flush)
 341		kvm_flush_remote_tlbs(kvm);
 
 
 
 342}
 343
 344static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 345						  struct mm_struct *mm,
 346						  unsigned long start,
 347						  unsigned long end)
 348{
 349	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 350
 351	spin_lock(&kvm->mmu_lock);
 352	/*
 353	 * This sequence increase will notify the kvm page fault that
 354	 * the page that is going to be mapped in the spte could have
 355	 * been freed.
 356	 */
 357	kvm->mmu_notifier_seq++;
 
 358	/*
 359	 * The above sequence increase must be visible before the
 360	 * below count decrease but both values are read by the kvm
 361	 * page fault under mmu_lock spinlock so we don't need to add
 362	 * a smb_wmb() here in between the two.
 363	 */
 364	kvm->mmu_notifier_count--;
 365	spin_unlock(&kvm->mmu_lock);
 366
 367	BUG_ON(kvm->mmu_notifier_count < 0);
 368}
 369
 370static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
 371					      struct mm_struct *mm,
 372					      unsigned long address)
 373{
 374	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 375	int young, idx;
 376
 377	idx = srcu_read_lock(&kvm->srcu);
 378	spin_lock(&kvm->mmu_lock);
 379	young = kvm_age_hva(kvm, address);
 380	spin_unlock(&kvm->mmu_lock);
 381	srcu_read_unlock(&kvm->srcu, idx);
 382
 
 383	if (young)
 384		kvm_flush_remote_tlbs(kvm);
 385
 
 
 
 386	return young;
 387}
 388
 389static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
 390				       struct mm_struct *mm,
 391				       unsigned long address)
 392{
 393	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 394	int young, idx;
 395
 396	idx = srcu_read_lock(&kvm->srcu);
 397	spin_lock(&kvm->mmu_lock);
 398	young = kvm_test_age_hva(kvm, address);
 399	spin_unlock(&kvm->mmu_lock);
 400	srcu_read_unlock(&kvm->srcu, idx);
 401
 402	return young;
 403}
 404
 405static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
 406				     struct mm_struct *mm)
 407{
 408	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 409	int idx;
 410
 411	idx = srcu_read_lock(&kvm->srcu);
 412	kvm_arch_flush_shadow(kvm);
 413	srcu_read_unlock(&kvm->srcu, idx);
 414}
 415
 416static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
 417	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
 418	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
 419	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
 420	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
 421	.test_young		= kvm_mmu_notifier_test_young,
 422	.change_pte		= kvm_mmu_notifier_change_pte,
 423	.release		= kvm_mmu_notifier_release,
 424};
 425
 426static int kvm_init_mmu_notifier(struct kvm *kvm)
 427{
 428	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
 429	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
 430}
 431
 432#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
 433
 434static int kvm_init_mmu_notifier(struct kvm *kvm)
 435{
 436	return 0;
 437}
 438
 439#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
 440
 441static struct kvm *kvm_create_vm(void)
 
 
 
 
 
 
 
 
 
 442{
 443	int r, i;
 444	struct kvm *kvm = kvm_arch_alloc_vm();
 445
 446	if (!kvm)
 447		return ERR_PTR(-ENOMEM);
 448
 449	r = kvm_arch_init_vm(kvm);
 450	if (r)
 451		goto out_err_nodisable;
 452
 453	r = hardware_enable_all();
 454	if (r)
 455		goto out_err_nodisable;
 456
 457#ifdef CONFIG_HAVE_KVM_IRQCHIP
 458	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
 459	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
 460#endif
 461
 
 
 462	r = -ENOMEM;
 463	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 464	if (!kvm->memslots)
 465		goto out_err_nosrcu;
 
 466	if (init_srcu_struct(&kvm->srcu))
 467		goto out_err_nosrcu;
 468	for (i = 0; i < KVM_NR_BUSES; i++) {
 469		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
 470					GFP_KERNEL);
 471		if (!kvm->buses[i])
 472			goto out_err;
 473	}
 474
 475	spin_lock_init(&kvm->mmu_lock);
 476	kvm->mm = current->mm;
 477	atomic_inc(&kvm->mm->mm_count);
 478	kvm_eventfd_init(kvm);
 479	mutex_init(&kvm->lock);
 480	mutex_init(&kvm->irq_lock);
 481	mutex_init(&kvm->slots_lock);
 482	atomic_set(&kvm->users_count, 1);
 
 483
 484	r = kvm_init_mmu_notifier(kvm);
 485	if (r)
 486		goto out_err;
 487
 488	raw_spin_lock(&kvm_lock);
 489	list_add(&kvm->vm_list, &vm_list);
 490	raw_spin_unlock(&kvm_lock);
 491
 492	return kvm;
 493
 494out_err:
 495	cleanup_srcu_struct(&kvm->srcu);
 496out_err_nosrcu:
 497	hardware_disable_all();
 498out_err_nodisable:
 499	for (i = 0; i < KVM_NR_BUSES; i++)
 500		kfree(kvm->buses[i]);
 501	kfree(kvm->memslots);
 502	kvm_arch_free_vm(kvm);
 503	return ERR_PTR(r);
 504}
 505
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 506static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
 507{
 508	if (!memslot->dirty_bitmap)
 509		return;
 510
 511	if (2 * kvm_dirty_bitmap_bytes(memslot) > PAGE_SIZE)
 512		vfree(memslot->dirty_bitmap_head);
 513	else
 514		kfree(memslot->dirty_bitmap_head);
 515
 516	memslot->dirty_bitmap = NULL;
 517	memslot->dirty_bitmap_head = NULL;
 518}
 519
 520/*
 521 * Free any memory in @free but not in @dont.
 522 */
 523static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 524				  struct kvm_memory_slot *dont)
 525{
 526	int i;
 527
 528	if (!dont || free->rmap != dont->rmap)
 529		vfree(free->rmap);
 530
 531	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 532		kvm_destroy_dirty_bitmap(free);
 533
 534
 535	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
 536		if (!dont || free->lpage_info[i] != dont->lpage_info[i]) {
 537			vfree(free->lpage_info[i]);
 538			free->lpage_info[i] = NULL;
 539		}
 540	}
 541
 542	free->npages = 0;
 543	free->rmap = NULL;
 544}
 545
 546void kvm_free_physmem(struct kvm *kvm)
 547{
 548	int i;
 549	struct kvm_memslots *slots = kvm->memslots;
 
 550
 551	for (i = 0; i < slots->nmemslots; ++i)
 552		kvm_free_physmem_slot(&slots->memslots[i], NULL);
 553
 554	kfree(kvm->memslots);
 555}
 556
 
 
 
 
 
 
 
 
 
 
 
 
 
 557static void kvm_destroy_vm(struct kvm *kvm)
 558{
 559	int i;
 560	struct mm_struct *mm = kvm->mm;
 561
 562	kvm_arch_sync_events(kvm);
 563	raw_spin_lock(&kvm_lock);
 564	list_del(&kvm->vm_list);
 565	raw_spin_unlock(&kvm_lock);
 566	kvm_free_irq_routing(kvm);
 567	for (i = 0; i < KVM_NR_BUSES; i++)
 568		kvm_io_bus_destroy(kvm->buses[i]);
 569	kvm_coalesced_mmio_free(kvm);
 570#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 571	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
 572#else
 573	kvm_arch_flush_shadow(kvm);
 574#endif
 575	kvm_arch_destroy_vm(kvm);
 
 576	kvm_free_physmem(kvm);
 577	cleanup_srcu_struct(&kvm->srcu);
 578	kvm_arch_free_vm(kvm);
 579	hardware_disable_all();
 580	mmdrop(mm);
 581}
 582
 583void kvm_get_kvm(struct kvm *kvm)
 584{
 585	atomic_inc(&kvm->users_count);
 586}
 587EXPORT_SYMBOL_GPL(kvm_get_kvm);
 588
 589void kvm_put_kvm(struct kvm *kvm)
 590{
 591	if (atomic_dec_and_test(&kvm->users_count))
 592		kvm_destroy_vm(kvm);
 593}
 594EXPORT_SYMBOL_GPL(kvm_put_kvm);
 595
 596
 597static int kvm_vm_release(struct inode *inode, struct file *filp)
 598{
 599	struct kvm *kvm = filp->private_data;
 600
 601	kvm_irqfd_release(kvm);
 602
 603	kvm_put_kvm(kvm);
 604	return 0;
 605}
 606
 607#ifndef CONFIG_S390
 608/*
 609 * Allocation size is twice as large as the actual dirty bitmap size.
 610 * This makes it possible to do double buffering: see x86's
 611 * kvm_vm_ioctl_get_dirty_log().
 612 */
 613static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
 614{
 
 615	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
 616
 617	if (dirty_bytes > PAGE_SIZE)
 618		memslot->dirty_bitmap = vzalloc(dirty_bytes);
 619	else
 620		memslot->dirty_bitmap = kzalloc(dirty_bytes, GFP_KERNEL);
 621
 622	if (!memslot->dirty_bitmap)
 623		return -ENOMEM;
 624
 625	memslot->dirty_bitmap_head = memslot->dirty_bitmap;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 626	return 0;
 627}
 628#endif /* !CONFIG_S390 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 629
 630/*
 631 * Allocate some memory and give it an address in the guest physical address
 632 * space.
 633 *
 634 * Discontiguous memory is allowed, mostly for framebuffers.
 635 *
 636 * Must be called holding mmap_sem for write.
 637 */
 638int __kvm_set_memory_region(struct kvm *kvm,
 639			    struct kvm_userspace_memory_region *mem,
 640			    int user_alloc)
 641{
 642	int r;
 643	gfn_t base_gfn;
 644	unsigned long npages;
 645	unsigned long i;
 646	struct kvm_memory_slot *memslot;
 647	struct kvm_memory_slot old, new;
 648	struct kvm_memslots *slots, *old_memslots;
 
 
 
 
 
 649
 650	r = -EINVAL;
 651	/* General sanity checks */
 652	if (mem->memory_size & (PAGE_SIZE - 1))
 653		goto out;
 654	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 655		goto out;
 656	/* We can read the guest memory with __xxx_user() later on. */
 657	if (user_alloc &&
 658	    ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
 659	     !access_ok(VERIFY_WRITE,
 660			(void __user *)(unsigned long)mem->userspace_addr,
 661			mem->memory_size)))
 662		goto out;
 663	if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
 664		goto out;
 665	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 666		goto out;
 667
 668	memslot = &kvm->memslots->memslots[mem->slot];
 669	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 670	npages = mem->memory_size >> PAGE_SHIFT;
 671
 672	r = -EINVAL;
 673	if (npages > KVM_MEM_MAX_NR_PAGES)
 674		goto out;
 675
 676	if (!npages)
 677		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 678
 679	new = old = *memslot;
 680
 681	new.id = mem->slot;
 682	new.base_gfn = base_gfn;
 683	new.npages = npages;
 684	new.flags = mem->flags;
 685
 686	/* Disallow changing a memory slot's size. */
 687	r = -EINVAL;
 688	if (npages && old.npages && npages != old.npages)
 689		goto out_free;
 
 
 
 
 
 
 690
 691	/* Check for overlaps */
 692	r = -EEXIST;
 693	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 694		struct kvm_memory_slot *s = &kvm->memslots->memslots[i];
 695
 696		if (s == memslot || !s->npages)
 697			continue;
 698		if (!((base_gfn + npages <= s->base_gfn) ||
 699		      (base_gfn >= s->base_gfn + s->npages)))
 700			goto out_free;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 701	}
 702
 703	/* Free page dirty bitmap if unneeded */
 704	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 705		new.dirty_bitmap = NULL;
 706
 707	r = -ENOMEM;
 708
 709	/* Allocate if a slot is being created */
 710#ifndef CONFIG_S390
 711	if (npages && !new.rmap) {
 712		new.rmap = vzalloc(npages * sizeof(*new.rmap));
 713
 714		if (!new.rmap)
 715			goto out_free;
 716
 717		new.user_alloc = user_alloc;
 718		new.userspace_addr = mem->userspace_addr;
 719	}
 720	if (!npages)
 721		goto skip_lpage;
 722
 723	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
 724		unsigned long ugfn;
 725		unsigned long j;
 726		int lpages;
 727		int level = i + 2;
 728
 729		/* Avoid unused variable warning if no large pages */
 730		(void)level;
 731
 732		if (new.lpage_info[i])
 733			continue;
 734
 735		lpages = 1 + ((base_gfn + npages - 1)
 736			     >> KVM_HPAGE_GFN_SHIFT(level));
 737		lpages -= base_gfn >> KVM_HPAGE_GFN_SHIFT(level);
 738
 739		new.lpage_info[i] = vzalloc(lpages * sizeof(*new.lpage_info[i]));
 740
 741		if (!new.lpage_info[i])
 742			goto out_free;
 743
 744		if (base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
 745			new.lpage_info[i][0].write_count = 1;
 746		if ((base_gfn+npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
 747			new.lpage_info[i][lpages - 1].write_count = 1;
 748		ugfn = new.userspace_addr >> PAGE_SHIFT;
 749		/*
 750		 * If the gfn and userspace address are not aligned wrt each
 751		 * other, or if explicitly asked to, disable large page
 752		 * support for this slot
 753		 */
 754		if ((base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
 755		    !largepages_enabled)
 756			for (j = 0; j < lpages; ++j)
 757				new.lpage_info[i][j].write_count = 1;
 758	}
 759
 760skip_lpage:
 761
 762	/* Allocate page dirty bitmap if needed */
 763	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 764		if (kvm_create_dirty_bitmap(&new) < 0)
 765			goto out_free;
 766		/* destroy any largepage mappings for dirty tracking */
 767	}
 768#else  /* not defined CONFIG_S390 */
 769	new.user_alloc = user_alloc;
 770	if (user_alloc)
 771		new.userspace_addr = mem->userspace_addr;
 772#endif /* not defined CONFIG_S390 */
 773
 774	if (!npages) {
 775		r = -ENOMEM;
 776		slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 
 777		if (!slots)
 778			goto out_free;
 779		memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
 780		if (mem->slot >= slots->nmemslots)
 781			slots->nmemslots = mem->slot + 1;
 782		slots->generation++;
 783		slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID;
 784
 785		old_memslots = kvm->memslots;
 786		rcu_assign_pointer(kvm->memslots, slots);
 787		synchronize_srcu_expedited(&kvm->srcu);
 788		/* From this point no new shadow pages pointing to a deleted
 789		 * memslot will be created.
 790		 *
 791		 * validation of sp->gfn happens in:
 792		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
 793		 * 	- kvm_is_visible_gfn (mmu_check_roots)
 794		 */
 795		kvm_arch_flush_shadow(kvm);
 796		kfree(old_memslots);
 797	}
 798
 799	r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
 800	if (r)
 801		goto out_free;
 802
 803	/* map the pages in iommu page table */
 804	if (npages) {
 805		r = kvm_iommu_map_pages(kvm, &new);
 806		if (r)
 
 
 
 
 
 
 807			goto out_free;
 808	}
 809
 810	r = -ENOMEM;
 811	slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 812	if (!slots)
 813		goto out_free;
 814	memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
 815	if (mem->slot >= slots->nmemslots)
 816		slots->nmemslots = mem->slot + 1;
 817	slots->generation++;
 818
 819	/* actual memory is freed via old in kvm_free_physmem_slot below */
 820	if (!npages) {
 821		new.rmap = NULL;
 822		new.dirty_bitmap = NULL;
 823		for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i)
 824			new.lpage_info[i] = NULL;
 825	}
 826
 827	slots->memslots[mem->slot] = new;
 828	old_memslots = kvm->memslots;
 829	rcu_assign_pointer(kvm->memslots, slots);
 830	synchronize_srcu_expedited(&kvm->srcu);
 831
 832	kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
 
 833
 834	/*
 835	 * If the new memory slot is created, we need to clear all
 836	 * mmio sptes.
 
 
 
 
 
 837	 */
 838	if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT)
 839		kvm_arch_flush_shadow(kvm);
 840
 841	kvm_free_physmem_slot(&old, &new);
 842	kfree(old_memslots);
 843
 844	return 0;
 845
 
 
 846out_free:
 847	kvm_free_physmem_slot(&new, &old);
 848out:
 849	return r;
 850
 851}
 852EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
 853
 854int kvm_set_memory_region(struct kvm *kvm,
 855			  struct kvm_userspace_memory_region *mem,
 856			  int user_alloc)
 857{
 858	int r;
 859
 860	mutex_lock(&kvm->slots_lock);
 861	r = __kvm_set_memory_region(kvm, mem, user_alloc);
 862	mutex_unlock(&kvm->slots_lock);
 863	return r;
 864}
 865EXPORT_SYMBOL_GPL(kvm_set_memory_region);
 866
 867int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 868				   struct
 869				   kvm_userspace_memory_region *mem,
 870				   int user_alloc)
 871{
 872	if (mem->slot >= KVM_MEMORY_SLOTS)
 873		return -EINVAL;
 874	return kvm_set_memory_region(kvm, mem, user_alloc);
 875}
 876
 877int kvm_get_dirty_log(struct kvm *kvm,
 878			struct kvm_dirty_log *log, int *is_dirty)
 879{
 880	struct kvm_memory_slot *memslot;
 881	int r, i;
 882	unsigned long n;
 883	unsigned long any = 0;
 884
 885	r = -EINVAL;
 886	if (log->slot >= KVM_MEMORY_SLOTS)
 887		goto out;
 888
 889	memslot = &kvm->memslots->memslots[log->slot];
 890	r = -ENOENT;
 891	if (!memslot->dirty_bitmap)
 892		goto out;
 893
 894	n = kvm_dirty_bitmap_bytes(memslot);
 895
 896	for (i = 0; !any && i < n/sizeof(long); ++i)
 897		any = memslot->dirty_bitmap[i];
 898
 899	r = -EFAULT;
 900	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 901		goto out;
 902
 903	if (any)
 904		*is_dirty = 1;
 905
 906	r = 0;
 907out:
 908	return r;
 909}
 
 910
 911void kvm_disable_largepages(void)
 912{
 913	largepages_enabled = false;
 914}
 915EXPORT_SYMBOL_GPL(kvm_disable_largepages);
 916
 917int is_error_page(struct page *page)
 918{
 919	return page == bad_page || page == hwpoison_page || page == fault_page;
 920}
 921EXPORT_SYMBOL_GPL(is_error_page);
 922
 923int is_error_pfn(pfn_t pfn)
 924{
 925	return pfn == bad_pfn || pfn == hwpoison_pfn || pfn == fault_pfn;
 926}
 927EXPORT_SYMBOL_GPL(is_error_pfn);
 928
 929int is_hwpoison_pfn(pfn_t pfn)
 930{
 931	return pfn == hwpoison_pfn;
 932}
 933EXPORT_SYMBOL_GPL(is_hwpoison_pfn);
 934
 935int is_fault_pfn(pfn_t pfn)
 936{
 937	return pfn == fault_pfn;
 938}
 939EXPORT_SYMBOL_GPL(is_fault_pfn);
 940
 941int is_noslot_pfn(pfn_t pfn)
 942{
 943	return pfn == bad_pfn;
 944}
 945EXPORT_SYMBOL_GPL(is_noslot_pfn);
 946
 947int is_invalid_pfn(pfn_t pfn)
 948{
 949	return pfn == hwpoison_pfn || pfn == fault_pfn;
 950}
 951EXPORT_SYMBOL_GPL(is_invalid_pfn);
 952
 953static inline unsigned long bad_hva(void)
 954{
 955	return PAGE_OFFSET;
 956}
 957
 958int kvm_is_error_hva(unsigned long addr)
 959{
 960	return addr == bad_hva();
 961}
 962EXPORT_SYMBOL_GPL(kvm_is_error_hva);
 963
 964static struct kvm_memory_slot *__gfn_to_memslot(struct kvm_memslots *slots,
 965						gfn_t gfn)
 966{
 967	int i;
 968
 969	for (i = 0; i < slots->nmemslots; ++i) {
 970		struct kvm_memory_slot *memslot = &slots->memslots[i];
 971
 972		if (gfn >= memslot->base_gfn
 973		    && gfn < memslot->base_gfn + memslot->npages)
 974			return memslot;
 975	}
 976	return NULL;
 977}
 
 978
 979struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 980{
 981	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
 982}
 983EXPORT_SYMBOL_GPL(gfn_to_memslot);
 984
 985int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
 986{
 987	int i;
 988	struct kvm_memslots *slots = kvm_memslots(kvm);
 989
 990	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 991		struct kvm_memory_slot *memslot = &slots->memslots[i];
 992
 993		if (memslot->flags & KVM_MEMSLOT_INVALID)
 994			continue;
 995
 996		if (gfn >= memslot->base_gfn
 997		    && gfn < memslot->base_gfn + memslot->npages)
 998			return 1;
 999	}
1000	return 0;
1001}
1002EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1003
1004unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1005{
1006	struct vm_area_struct *vma;
1007	unsigned long addr, size;
1008
1009	size = PAGE_SIZE;
1010
1011	addr = gfn_to_hva(kvm, gfn);
1012	if (kvm_is_error_hva(addr))
1013		return PAGE_SIZE;
1014
1015	down_read(&current->mm->mmap_sem);
1016	vma = find_vma(current->mm, addr);
1017	if (!vma)
1018		goto out;
1019
1020	size = vma_kernel_pagesize(vma);
1021
1022out:
1023	up_read(&current->mm->mmap_sem);
1024
1025	return size;
1026}
1027
1028static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1029				     gfn_t *nr_pages)
 
 
 
 
 
1030{
1031	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1032		return bad_hva();
 
 
 
1033
1034	if (nr_pages)
1035		*nr_pages = slot->npages - (gfn - slot->base_gfn);
1036
1037	return gfn_to_hva_memslot(slot, gfn);
1038}
1039
 
 
 
 
 
 
 
 
 
 
 
 
 
1040unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1041{
1042	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1043}
1044EXPORT_SYMBOL_GPL(gfn_to_hva);
1045
1046static pfn_t get_fault_pfn(void)
 
 
 
 
1047{
1048	get_page(fault_page);
1049	return fault_pfn;
 
 
 
 
 
1050}
1051
1052int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
 
 
 
 
 
 
 
 
 
 
1053	unsigned long start, int write, struct page **page)
1054{
1055	int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1056
1057	if (write)
1058		flags |= FOLL_WRITE;
1059
1060	return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
1061}
1062
1063static inline int check_user_page_hwpoison(unsigned long addr)
1064{
1065	int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1066
1067	rc = __get_user_pages(current, current->mm, addr, 1,
1068			      flags, NULL, NULL, NULL);
1069	return rc == -EHWPOISON;
1070}
1071
1072static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr, bool atomic,
1073			bool *async, bool write_fault, bool *writable)
 
 
 
 
1074{
1075	struct page *page[1];
1076	int npages = 0;
1077	pfn_t pfn;
1078
1079	/* we can do it either atomically or asynchronously, not both */
1080	BUG_ON(atomic && async);
1081
1082	BUG_ON(!write_fault && !writable);
 
 
 
 
 
 
1083
1084	if (writable)
1085		*writable = true;
 
1086
1087	if (atomic || async)
1088		npages = __get_user_pages_fast(addr, 1, 1, page);
 
 
1089
1090	if (unlikely(npages != 1) && !atomic) {
1091		might_sleep();
1092
1093		if (writable)
1094			*writable = write_fault;
 
 
 
 
 
 
 
1095
1096		if (async) {
1097			down_read(&current->mm->mmap_sem);
1098			npages = get_user_page_nowait(current, current->mm,
1099						     addr, write_fault, page);
1100			up_read(&current->mm->mmap_sem);
1101		} else
1102			npages = get_user_pages_fast(addr, 1, write_fault,
1103						     page);
1104
1105		/* map read fault as writable if possible */
1106		if (unlikely(!write_fault) && npages == 1) {
1107			struct page *wpage[1];
1108
1109			npages = __get_user_pages_fast(addr, 1, 1, wpage);
1110			if (npages == 1) {
1111				*writable = true;
1112				put_page(page[0]);
1113				page[0] = wpage[0];
1114			}
1115			npages = 1;
 
 
 
 
 
 
 
 
 
 
 
 
1116		}
 
 
1117	}
 
 
 
1118
1119	if (unlikely(npages != 1)) {
1120		struct vm_area_struct *vma;
 
 
1121
1122		if (atomic)
1123			return get_fault_pfn();
1124
1125		down_read(&current->mm->mmap_sem);
1126		if (npages == -EHWPOISON ||
1127			(!async && check_user_page_hwpoison(addr))) {
1128			up_read(&current->mm->mmap_sem);
1129			get_page(hwpoison_page);
1130			return page_to_pfn(hwpoison_page);
1131		}
1132
1133		vma = find_vma_intersection(current->mm, addr, addr+1);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1134
1135		if (vma == NULL)
1136			pfn = get_fault_pfn();
1137		else if ((vma->vm_flags & VM_PFNMAP)) {
1138			pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1139				vma->vm_pgoff;
1140			BUG_ON(!kvm_is_mmio_pfn(pfn));
1141		} else {
1142			if (async && (vma->vm_flags & VM_WRITE))
1143				*async = true;
1144			pfn = get_fault_pfn();
1145		}
1146		up_read(&current->mm->mmap_sem);
1147	} else
1148		pfn = page_to_pfn(page[0]);
1149
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1150	return pfn;
1151}
1152
1153pfn_t hva_to_pfn_atomic(struct kvm *kvm, unsigned long addr)
 
 
1154{
1155	return hva_to_pfn(kvm, addr, true, NULL, true, NULL);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1156}
1157EXPORT_SYMBOL_GPL(hva_to_pfn_atomic);
1158
1159static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
1160			  bool write_fault, bool *writable)
1161{
1162	unsigned long addr;
1163
1164	if (async)
1165		*async = false;
1166
1167	addr = gfn_to_hva(kvm, gfn);
1168	if (kvm_is_error_hva(addr)) {
1169		get_page(bad_page);
1170		return page_to_pfn(bad_page);
1171	}
1172
1173	return hva_to_pfn(kvm, addr, atomic, async, write_fault, writable);
 
1174}
1175
1176pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1177{
1178	return __gfn_to_pfn(kvm, gfn, true, NULL, true, NULL);
1179}
1180EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1181
1182pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
1183		       bool write_fault, bool *writable)
1184{
1185	return __gfn_to_pfn(kvm, gfn, false, async, write_fault, writable);
1186}
1187EXPORT_SYMBOL_GPL(gfn_to_pfn_async);
1188
1189pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1190{
1191	return __gfn_to_pfn(kvm, gfn, false, NULL, true, NULL);
1192}
1193EXPORT_SYMBOL_GPL(gfn_to_pfn);
1194
1195pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1196		      bool *writable)
1197{
1198	return __gfn_to_pfn(kvm, gfn, false, NULL, write_fault, writable);
1199}
1200EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1201
1202pfn_t gfn_to_pfn_memslot(struct kvm *kvm,
1203			 struct kvm_memory_slot *slot, gfn_t gfn)
 
 
 
 
1204{
1205	unsigned long addr = gfn_to_hva_memslot(slot, gfn);
1206	return hva_to_pfn(kvm, addr, false, NULL, true, NULL);
1207}
 
1208
1209int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
1210								  int nr_pages)
1211{
1212	unsigned long addr;
1213	gfn_t entry;
1214
1215	addr = gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, &entry);
1216	if (kvm_is_error_hva(addr))
1217		return -1;
1218
1219	if (entry < nr_pages)
1220		return 0;
1221
1222	return __get_user_pages_fast(addr, nr_pages, 1, pages);
1223}
1224EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1225
 
 
 
 
 
 
 
 
 
 
 
 
 
1226struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1227{
1228	pfn_t pfn;
1229
1230	pfn = gfn_to_pfn(kvm, gfn);
1231	if (!kvm_is_mmio_pfn(pfn))
1232		return pfn_to_page(pfn);
1233
1234	WARN_ON(kvm_is_mmio_pfn(pfn));
1235
1236	get_page(bad_page);
1237	return bad_page;
1238}
1239
1240EXPORT_SYMBOL_GPL(gfn_to_page);
1241
1242void kvm_release_page_clean(struct page *page)
1243{
 
 
1244	kvm_release_pfn_clean(page_to_pfn(page));
1245}
1246EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1247
1248void kvm_release_pfn_clean(pfn_t pfn)
1249{
1250	if (!kvm_is_mmio_pfn(pfn))
1251		put_page(pfn_to_page(pfn));
1252}
1253EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1254
1255void kvm_release_page_dirty(struct page *page)
1256{
 
 
1257	kvm_release_pfn_dirty(page_to_pfn(page));
1258}
1259EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1260
1261void kvm_release_pfn_dirty(pfn_t pfn)
1262{
1263	kvm_set_pfn_dirty(pfn);
1264	kvm_release_pfn_clean(pfn);
1265}
1266EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1267
1268void kvm_set_page_dirty(struct page *page)
1269{
1270	kvm_set_pfn_dirty(page_to_pfn(page));
1271}
1272EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
1273
1274void kvm_set_pfn_dirty(pfn_t pfn)
1275{
1276	if (!kvm_is_mmio_pfn(pfn)) {
1277		struct page *page = pfn_to_page(pfn);
1278		if (!PageReserved(page))
1279			SetPageDirty(page);
1280	}
1281}
1282EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1283
1284void kvm_set_pfn_accessed(pfn_t pfn)
1285{
1286	if (!kvm_is_mmio_pfn(pfn))
1287		mark_page_accessed(pfn_to_page(pfn));
1288}
1289EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1290
1291void kvm_get_pfn(pfn_t pfn)
1292{
1293	if (!kvm_is_mmio_pfn(pfn))
1294		get_page(pfn_to_page(pfn));
1295}
1296EXPORT_SYMBOL_GPL(kvm_get_pfn);
1297
1298static int next_segment(unsigned long len, int offset)
1299{
1300	if (len > PAGE_SIZE - offset)
1301		return PAGE_SIZE - offset;
1302	else
1303		return len;
1304}
1305
1306int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1307			int len)
1308{
1309	int r;
1310	unsigned long addr;
1311
1312	addr = gfn_to_hva(kvm, gfn);
1313	if (kvm_is_error_hva(addr))
1314		return -EFAULT;
1315	r = __copy_from_user(data, (void __user *)addr + offset, len);
1316	if (r)
1317		return -EFAULT;
1318	return 0;
1319}
1320EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1321
1322int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1323{
1324	gfn_t gfn = gpa >> PAGE_SHIFT;
1325	int seg;
1326	int offset = offset_in_page(gpa);
1327	int ret;
1328
1329	while ((seg = next_segment(len, offset)) != 0) {
1330		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1331		if (ret < 0)
1332			return ret;
1333		offset = 0;
1334		len -= seg;
1335		data += seg;
1336		++gfn;
1337	}
1338	return 0;
1339}
1340EXPORT_SYMBOL_GPL(kvm_read_guest);
1341
1342int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1343			  unsigned long len)
1344{
1345	int r;
1346	unsigned long addr;
1347	gfn_t gfn = gpa >> PAGE_SHIFT;
1348	int offset = offset_in_page(gpa);
1349
1350	addr = gfn_to_hva(kvm, gfn);
1351	if (kvm_is_error_hva(addr))
1352		return -EFAULT;
1353	pagefault_disable();
1354	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1355	pagefault_enable();
1356	if (r)
1357		return -EFAULT;
1358	return 0;
1359}
1360EXPORT_SYMBOL(kvm_read_guest_atomic);
1361
1362int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1363			 int offset, int len)
1364{
1365	int r;
1366	unsigned long addr;
1367
1368	addr = gfn_to_hva(kvm, gfn);
1369	if (kvm_is_error_hva(addr))
1370		return -EFAULT;
1371	r = __copy_to_user((void __user *)addr + offset, data, len);
1372	if (r)
1373		return -EFAULT;
1374	mark_page_dirty(kvm, gfn);
1375	return 0;
1376}
1377EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1378
1379int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1380		    unsigned long len)
1381{
1382	gfn_t gfn = gpa >> PAGE_SHIFT;
1383	int seg;
1384	int offset = offset_in_page(gpa);
1385	int ret;
1386
1387	while ((seg = next_segment(len, offset)) != 0) {
1388		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1389		if (ret < 0)
1390			return ret;
1391		offset = 0;
1392		len -= seg;
1393		data += seg;
1394		++gfn;
1395	}
1396	return 0;
1397}
1398
1399int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1400			      gpa_t gpa)
1401{
1402	struct kvm_memslots *slots = kvm_memslots(kvm);
1403	int offset = offset_in_page(gpa);
1404	gfn_t gfn = gpa >> PAGE_SHIFT;
 
 
 
1405
1406	ghc->gpa = gpa;
1407	ghc->generation = slots->generation;
1408	ghc->memslot = __gfn_to_memslot(slots, gfn);
1409	ghc->hva = gfn_to_hva_many(ghc->memslot, gfn, NULL);
1410	if (!kvm_is_error_hva(ghc->hva))
 
1411		ghc->hva += offset;
1412	else
1413		return -EFAULT;
1414
 
 
 
 
 
 
 
 
 
 
 
 
 
1415	return 0;
1416}
1417EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1418
1419int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1420			   void *data, unsigned long len)
1421{
1422	struct kvm_memslots *slots = kvm_memslots(kvm);
1423	int r;
1424
 
 
1425	if (slots->generation != ghc->generation)
1426		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
 
 
 
1427
1428	if (kvm_is_error_hva(ghc->hva))
1429		return -EFAULT;
1430
1431	r = __copy_to_user((void __user *)ghc->hva, data, len);
1432	if (r)
1433		return -EFAULT;
1434	mark_page_dirty_in_slot(kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT);
1435
1436	return 0;
1437}
1438EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1439
1440int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1441			   void *data, unsigned long len)
1442{
1443	struct kvm_memslots *slots = kvm_memslots(kvm);
1444	int r;
1445
 
 
1446	if (slots->generation != ghc->generation)
1447		kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa);
 
 
 
1448
1449	if (kvm_is_error_hva(ghc->hva))
1450		return -EFAULT;
1451
1452	r = __copy_from_user(data, (void __user *)ghc->hva, len);
1453	if (r)
1454		return -EFAULT;
1455
1456	return 0;
1457}
1458EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
1459
1460int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1461{
1462	return kvm_write_guest_page(kvm, gfn, (const void *) empty_zero_page,
1463				    offset, len);
 
1464}
1465EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1466
1467int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1468{
1469	gfn_t gfn = gpa >> PAGE_SHIFT;
1470	int seg;
1471	int offset = offset_in_page(gpa);
1472	int ret;
1473
1474        while ((seg = next_segment(len, offset)) != 0) {
1475		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1476		if (ret < 0)
1477			return ret;
1478		offset = 0;
1479		len -= seg;
1480		++gfn;
1481	}
1482	return 0;
1483}
1484EXPORT_SYMBOL_GPL(kvm_clear_guest);
1485
1486void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
1487			     gfn_t gfn)
 
1488{
1489	if (memslot && memslot->dirty_bitmap) {
1490		unsigned long rel_gfn = gfn - memslot->base_gfn;
1491
1492		__set_bit_le(rel_gfn, memslot->dirty_bitmap);
1493	}
1494}
1495
1496void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1497{
1498	struct kvm_memory_slot *memslot;
1499
1500	memslot = gfn_to_memslot(kvm, gfn);
1501	mark_page_dirty_in_slot(kvm, memslot, gfn);
1502}
 
1503
1504/*
1505 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1506 */
1507void kvm_vcpu_block(struct kvm_vcpu *vcpu)
1508{
1509	DEFINE_WAIT(wait);
1510
1511	for (;;) {
1512		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
1513
1514		if (kvm_arch_vcpu_runnable(vcpu)) {
1515			kvm_make_request(KVM_REQ_UNHALT, vcpu);
1516			break;
1517		}
1518		if (kvm_cpu_has_pending_timer(vcpu))
1519			break;
1520		if (signal_pending(current))
1521			break;
1522
1523		schedule();
1524	}
1525
1526	finish_wait(&vcpu->wq, &wait);
1527}
 
1528
1529void kvm_resched(struct kvm_vcpu *vcpu)
 
 
 
 
1530{
1531	if (!need_resched())
1532		return;
1533	cond_resched();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1534}
1535EXPORT_SYMBOL_GPL(kvm_resched);
1536
1537void kvm_vcpu_on_spin(struct kvm_vcpu *me)
1538{
1539	struct kvm *kvm = me->kvm;
1540	struct kvm_vcpu *vcpu;
1541	int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
1542	int yielded = 0;
 
1543	int pass;
1544	int i;
1545
 
1546	/*
1547	 * We boost the priority of a VCPU that is runnable but not
1548	 * currently running, because it got preempted by something
1549	 * else and called schedule in __vcpu_run.  Hopefully that
1550	 * VCPU is holding the lock that we need and will release it.
1551	 * We approximate round-robin by starting at the last boosted VCPU.
1552	 */
1553	for (pass = 0; pass < 2 && !yielded; pass++) {
1554		kvm_for_each_vcpu(i, vcpu, kvm) {
1555			struct task_struct *task = NULL;
1556			struct pid *pid;
1557			if (!pass && i < last_boosted_vcpu) {
1558				i = last_boosted_vcpu;
1559				continue;
1560			} else if (pass && i > last_boosted_vcpu)
1561				break;
1562			if (vcpu == me)
1563				continue;
1564			if (waitqueue_active(&vcpu->wq))
1565				continue;
1566			rcu_read_lock();
1567			pid = rcu_dereference(vcpu->pid);
1568			if (pid)
1569				task = get_pid_task(vcpu->pid, PIDTYPE_PID);
1570			rcu_read_unlock();
1571			if (!task)
1572				continue;
1573			if (task->flags & PF_VCPU) {
1574				put_task_struct(task);
1575				continue;
1576			}
1577			if (yield_to(task, 1)) {
1578				put_task_struct(task);
1579				kvm->last_boosted_vcpu = i;
1580				yielded = 1;
1581				break;
 
 
 
 
1582			}
1583			put_task_struct(task);
1584		}
1585	}
 
 
 
 
1586}
1587EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
1588
1589static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1590{
1591	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
1592	struct page *page;
1593
1594	if (vmf->pgoff == 0)
1595		page = virt_to_page(vcpu->run);
1596#ifdef CONFIG_X86
1597	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
1598		page = virt_to_page(vcpu->arch.pio_data);
1599#endif
1600#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1601	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
1602		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
1603#endif
1604	else
1605		return VM_FAULT_SIGBUS;
1606	get_page(page);
1607	vmf->page = page;
1608	return 0;
1609}
1610
1611static const struct vm_operations_struct kvm_vcpu_vm_ops = {
1612	.fault = kvm_vcpu_fault,
1613};
1614
1615static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
1616{
1617	vma->vm_ops = &kvm_vcpu_vm_ops;
1618	return 0;
1619}
1620
1621static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1622{
1623	struct kvm_vcpu *vcpu = filp->private_data;
1624
1625	kvm_put_kvm(vcpu->kvm);
1626	return 0;
1627}
1628
1629static struct file_operations kvm_vcpu_fops = {
1630	.release        = kvm_vcpu_release,
1631	.unlocked_ioctl = kvm_vcpu_ioctl,
1632#ifdef CONFIG_COMPAT
1633	.compat_ioctl   = kvm_vcpu_compat_ioctl,
1634#endif
1635	.mmap           = kvm_vcpu_mmap,
1636	.llseek		= noop_llseek,
1637};
1638
1639/*
1640 * Allocates an inode for the vcpu.
1641 */
1642static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1643{
1644	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
1645}
1646
1647/*
1648 * Creates some virtual cpus.  Good luck creating more than one.
1649 */
1650static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
1651{
1652	int r;
1653	struct kvm_vcpu *vcpu, *v;
1654
 
 
 
1655	vcpu = kvm_arch_vcpu_create(kvm, id);
1656	if (IS_ERR(vcpu))
1657		return PTR_ERR(vcpu);
1658
1659	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
1660
1661	r = kvm_arch_vcpu_setup(vcpu);
1662	if (r)
1663		goto vcpu_destroy;
1664
1665	mutex_lock(&kvm->lock);
 
 
 
 
1666	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
1667		r = -EINVAL;
1668		goto unlock_vcpu_destroy;
1669	}
1670
1671	kvm_for_each_vcpu(r, v, kvm)
1672		if (v->vcpu_id == id) {
1673			r = -EEXIST;
1674			goto unlock_vcpu_destroy;
1675		}
1676
1677	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
1678
1679	/* Now it's all set up, let userspace reach it */
1680	kvm_get_kvm(kvm);
1681	r = create_vcpu_fd(vcpu);
1682	if (r < 0) {
1683		kvm_put_kvm(kvm);
1684		goto unlock_vcpu_destroy;
1685	}
1686
1687	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
1688	smp_wmb();
1689	atomic_inc(&kvm->online_vcpus);
1690
1691#ifdef CONFIG_KVM_APIC_ARCHITECTURE
1692	if (kvm->bsp_vcpu_id == id)
1693		kvm->bsp_vcpu = vcpu;
1694#endif
1695	mutex_unlock(&kvm->lock);
 
1696	return r;
1697
1698unlock_vcpu_destroy:
1699	mutex_unlock(&kvm->lock);
1700vcpu_destroy:
1701	kvm_arch_vcpu_destroy(vcpu);
1702	return r;
1703}
1704
1705static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
1706{
1707	if (sigset) {
1708		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
1709		vcpu->sigset_active = 1;
1710		vcpu->sigset = *sigset;
1711	} else
1712		vcpu->sigset_active = 0;
1713	return 0;
1714}
1715
1716static long kvm_vcpu_ioctl(struct file *filp,
1717			   unsigned int ioctl, unsigned long arg)
1718{
1719	struct kvm_vcpu *vcpu = filp->private_data;
1720	void __user *argp = (void __user *)arg;
1721	int r;
1722	struct kvm_fpu *fpu = NULL;
1723	struct kvm_sregs *kvm_sregs = NULL;
1724
1725	if (vcpu->kvm->mm != current->mm)
1726		return -EIO;
1727
1728#if defined(CONFIG_S390) || defined(CONFIG_PPC)
1729	/*
1730	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
1731	 * so vcpu_load() would break it.
1732	 */
1733	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
1734		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1735#endif
1736
1737
1738	vcpu_load(vcpu);
 
 
1739	switch (ioctl) {
1740	case KVM_RUN:
1741		r = -EINVAL;
1742		if (arg)
1743			goto out;
1744		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
1745		trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
1746		break;
1747	case KVM_GET_REGS: {
1748		struct kvm_regs *kvm_regs;
1749
1750		r = -ENOMEM;
1751		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1752		if (!kvm_regs)
1753			goto out;
1754		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
1755		if (r)
1756			goto out_free1;
1757		r = -EFAULT;
1758		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
1759			goto out_free1;
1760		r = 0;
1761out_free1:
1762		kfree(kvm_regs);
1763		break;
1764	}
1765	case KVM_SET_REGS: {
1766		struct kvm_regs *kvm_regs;
1767
1768		r = -ENOMEM;
1769		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1770		if (!kvm_regs)
 
1771			goto out;
1772		r = -EFAULT;
1773		if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
1774			goto out_free2;
1775		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
1776		if (r)
1777			goto out_free2;
1778		r = 0;
1779out_free2:
1780		kfree(kvm_regs);
1781		break;
1782	}
1783	case KVM_GET_SREGS: {
1784		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1785		r = -ENOMEM;
1786		if (!kvm_sregs)
1787			goto out;
1788		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
1789		if (r)
1790			goto out;
1791		r = -EFAULT;
1792		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
1793			goto out;
1794		r = 0;
1795		break;
1796	}
1797	case KVM_SET_SREGS: {
1798		kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1799		r = -ENOMEM;
1800		if (!kvm_sregs)
1801			goto out;
1802		r = -EFAULT;
1803		if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
1804			goto out;
 
1805		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
1806		if (r)
1807			goto out;
1808		r = 0;
1809		break;
1810	}
1811	case KVM_GET_MP_STATE: {
1812		struct kvm_mp_state mp_state;
1813
1814		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
1815		if (r)
1816			goto out;
1817		r = -EFAULT;
1818		if (copy_to_user(argp, &mp_state, sizeof mp_state))
1819			goto out;
1820		r = 0;
1821		break;
1822	}
1823	case KVM_SET_MP_STATE: {
1824		struct kvm_mp_state mp_state;
1825
1826		r = -EFAULT;
1827		if (copy_from_user(&mp_state, argp, sizeof mp_state))
1828			goto out;
1829		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
1830		if (r)
1831			goto out;
1832		r = 0;
1833		break;
1834	}
1835	case KVM_TRANSLATE: {
1836		struct kvm_translation tr;
1837
1838		r = -EFAULT;
1839		if (copy_from_user(&tr, argp, sizeof tr))
1840			goto out;
1841		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
1842		if (r)
1843			goto out;
1844		r = -EFAULT;
1845		if (copy_to_user(argp, &tr, sizeof tr))
1846			goto out;
1847		r = 0;
1848		break;
1849	}
1850	case KVM_SET_GUEST_DEBUG: {
1851		struct kvm_guest_debug dbg;
1852
1853		r = -EFAULT;
1854		if (copy_from_user(&dbg, argp, sizeof dbg))
1855			goto out;
1856		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
1857		if (r)
1858			goto out;
1859		r = 0;
1860		break;
1861	}
1862	case KVM_SET_SIGNAL_MASK: {
1863		struct kvm_signal_mask __user *sigmask_arg = argp;
1864		struct kvm_signal_mask kvm_sigmask;
1865		sigset_t sigset, *p;
1866
1867		p = NULL;
1868		if (argp) {
1869			r = -EFAULT;
1870			if (copy_from_user(&kvm_sigmask, argp,
1871					   sizeof kvm_sigmask))
1872				goto out;
1873			r = -EINVAL;
1874			if (kvm_sigmask.len != sizeof sigset)
1875				goto out;
1876			r = -EFAULT;
1877			if (copy_from_user(&sigset, sigmask_arg->sigset,
1878					   sizeof sigset))
1879				goto out;
1880			p = &sigset;
1881		}
1882		r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
1883		break;
1884	}
1885	case KVM_GET_FPU: {
1886		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1887		r = -ENOMEM;
1888		if (!fpu)
1889			goto out;
1890		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
1891		if (r)
1892			goto out;
1893		r = -EFAULT;
1894		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
1895			goto out;
1896		r = 0;
1897		break;
1898	}
1899	case KVM_SET_FPU: {
1900		fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1901		r = -ENOMEM;
1902		if (!fpu)
1903			goto out;
1904		r = -EFAULT;
1905		if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
1906			goto out;
 
1907		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
1908		if (r)
1909			goto out;
1910		r = 0;
1911		break;
1912	}
1913	default:
1914		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1915	}
1916out:
1917	vcpu_put(vcpu);
1918	kfree(fpu);
1919	kfree(kvm_sregs);
1920	return r;
1921}
1922
1923#ifdef CONFIG_COMPAT
1924static long kvm_vcpu_compat_ioctl(struct file *filp,
1925				  unsigned int ioctl, unsigned long arg)
1926{
1927	struct kvm_vcpu *vcpu = filp->private_data;
1928	void __user *argp = compat_ptr(arg);
1929	int r;
1930
1931	if (vcpu->kvm->mm != current->mm)
1932		return -EIO;
1933
1934	switch (ioctl) {
1935	case KVM_SET_SIGNAL_MASK: {
1936		struct kvm_signal_mask __user *sigmask_arg = argp;
1937		struct kvm_signal_mask kvm_sigmask;
1938		compat_sigset_t csigset;
1939		sigset_t sigset;
1940
1941		if (argp) {
1942			r = -EFAULT;
1943			if (copy_from_user(&kvm_sigmask, argp,
1944					   sizeof kvm_sigmask))
1945				goto out;
1946			r = -EINVAL;
1947			if (kvm_sigmask.len != sizeof csigset)
1948				goto out;
1949			r = -EFAULT;
1950			if (copy_from_user(&csigset, sigmask_arg->sigset,
1951					   sizeof csigset))
1952				goto out;
1953		}
1954		sigset_from_compat(&sigset, &csigset);
1955		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
 
1956		break;
1957	}
1958	default:
1959		r = kvm_vcpu_ioctl(filp, ioctl, arg);
1960	}
1961
1962out:
1963	return r;
1964}
1965#endif
1966
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1967static long kvm_vm_ioctl(struct file *filp,
1968			   unsigned int ioctl, unsigned long arg)
1969{
1970	struct kvm *kvm = filp->private_data;
1971	void __user *argp = (void __user *)arg;
1972	int r;
1973
1974	if (kvm->mm != current->mm)
1975		return -EIO;
1976	switch (ioctl) {
1977	case KVM_CREATE_VCPU:
1978		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
1979		if (r < 0)
1980			goto out;
1981		break;
1982	case KVM_SET_USER_MEMORY_REGION: {
1983		struct kvm_userspace_memory_region kvm_userspace_mem;
1984
1985		r = -EFAULT;
1986		if (copy_from_user(&kvm_userspace_mem, argp,
1987						sizeof kvm_userspace_mem))
1988			goto out;
1989
1990		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
1991		if (r)
1992			goto out;
1993		break;
1994	}
1995	case KVM_GET_DIRTY_LOG: {
1996		struct kvm_dirty_log log;
1997
1998		r = -EFAULT;
1999		if (copy_from_user(&log, argp, sizeof log))
2000			goto out;
2001		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2002		if (r)
2003			goto out;
2004		break;
2005	}
2006#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2007	case KVM_REGISTER_COALESCED_MMIO: {
2008		struct kvm_coalesced_mmio_zone zone;
2009		r = -EFAULT;
2010		if (copy_from_user(&zone, argp, sizeof zone))
2011			goto out;
2012		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2013		if (r)
2014			goto out;
2015		r = 0;
2016		break;
2017	}
2018	case KVM_UNREGISTER_COALESCED_MMIO: {
2019		struct kvm_coalesced_mmio_zone zone;
2020		r = -EFAULT;
2021		if (copy_from_user(&zone, argp, sizeof zone))
2022			goto out;
2023		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
2024		if (r)
2025			goto out;
2026		r = 0;
2027		break;
2028	}
2029#endif
2030	case KVM_IRQFD: {
2031		struct kvm_irqfd data;
2032
2033		r = -EFAULT;
2034		if (copy_from_user(&data, argp, sizeof data))
2035			goto out;
2036		r = kvm_irqfd(kvm, data.fd, data.gsi, data.flags);
2037		break;
2038	}
2039	case KVM_IOEVENTFD: {
2040		struct kvm_ioeventfd data;
2041
2042		r = -EFAULT;
2043		if (copy_from_user(&data, argp, sizeof data))
2044			goto out;
2045		r = kvm_ioeventfd(kvm, &data);
2046		break;
2047	}
2048#ifdef CONFIG_KVM_APIC_ARCHITECTURE
2049	case KVM_SET_BOOT_CPU_ID:
2050		r = 0;
2051		mutex_lock(&kvm->lock);
2052		if (atomic_read(&kvm->online_vcpus) != 0)
2053			r = -EBUSY;
2054		else
2055			kvm->bsp_vcpu_id = arg;
2056		mutex_unlock(&kvm->lock);
2057		break;
2058#endif
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2059	default:
2060		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2061		if (r == -ENOTTY)
2062			r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
2063	}
2064out:
2065	return r;
2066}
2067
2068#ifdef CONFIG_COMPAT
2069struct compat_kvm_dirty_log {
2070	__u32 slot;
2071	__u32 padding1;
2072	union {
2073		compat_uptr_t dirty_bitmap; /* one bit per page */
2074		__u64 padding2;
2075	};
2076};
2077
2078static long kvm_vm_compat_ioctl(struct file *filp,
2079			   unsigned int ioctl, unsigned long arg)
2080{
2081	struct kvm *kvm = filp->private_data;
2082	int r;
2083
2084	if (kvm->mm != current->mm)
2085		return -EIO;
2086	switch (ioctl) {
2087	case KVM_GET_DIRTY_LOG: {
2088		struct compat_kvm_dirty_log compat_log;
2089		struct kvm_dirty_log log;
2090
2091		r = -EFAULT;
2092		if (copy_from_user(&compat_log, (void __user *)arg,
2093				   sizeof(compat_log)))
2094			goto out;
2095		log.slot	 = compat_log.slot;
2096		log.padding1	 = compat_log.padding1;
2097		log.padding2	 = compat_log.padding2;
2098		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
2099
2100		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2101		if (r)
2102			goto out;
2103		break;
2104	}
2105	default:
2106		r = kvm_vm_ioctl(filp, ioctl, arg);
2107	}
2108
2109out:
2110	return r;
2111}
2112#endif
2113
2114static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2115{
2116	struct page *page[1];
2117	unsigned long addr;
2118	int npages;
2119	gfn_t gfn = vmf->pgoff;
2120	struct kvm *kvm = vma->vm_file->private_data;
2121
2122	addr = gfn_to_hva(kvm, gfn);
2123	if (kvm_is_error_hva(addr))
2124		return VM_FAULT_SIGBUS;
2125
2126	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
2127				NULL);
2128	if (unlikely(npages != 1))
2129		return VM_FAULT_SIGBUS;
2130
2131	vmf->page = page[0];
2132	return 0;
2133}
2134
2135static const struct vm_operations_struct kvm_vm_vm_ops = {
2136	.fault = kvm_vm_fault,
2137};
2138
2139static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2140{
2141	vma->vm_ops = &kvm_vm_vm_ops;
2142	return 0;
2143}
2144
2145static struct file_operations kvm_vm_fops = {
2146	.release        = kvm_vm_release,
2147	.unlocked_ioctl = kvm_vm_ioctl,
2148#ifdef CONFIG_COMPAT
2149	.compat_ioctl   = kvm_vm_compat_ioctl,
2150#endif
2151	.mmap           = kvm_vm_mmap,
2152	.llseek		= noop_llseek,
2153};
2154
2155static int kvm_dev_ioctl_create_vm(void)
2156{
2157	int r;
2158	struct kvm *kvm;
2159
2160	kvm = kvm_create_vm();
2161	if (IS_ERR(kvm))
2162		return PTR_ERR(kvm);
2163#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2164	r = kvm_coalesced_mmio_init(kvm);
2165	if (r < 0) {
2166		kvm_put_kvm(kvm);
2167		return r;
2168	}
2169#endif
2170	r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
2171	if (r < 0)
2172		kvm_put_kvm(kvm);
2173
2174	return r;
2175}
2176
2177static long kvm_dev_ioctl_check_extension_generic(long arg)
2178{
2179	switch (arg) {
2180	case KVM_CAP_USER_MEMORY:
2181	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2182	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2183#ifdef CONFIG_KVM_APIC_ARCHITECTURE
2184	case KVM_CAP_SET_BOOT_CPU_ID:
2185#endif
2186	case KVM_CAP_INTERNAL_ERROR_DATA:
 
 
 
 
 
 
2187		return 1;
2188#ifdef CONFIG_HAVE_KVM_IRQCHIP
2189	case KVM_CAP_IRQ_ROUTING:
2190		return KVM_MAX_IRQ_ROUTES;
2191#endif
2192	default:
2193		break;
2194	}
2195	return kvm_dev_ioctl_check_extension(arg);
2196}
2197
2198static long kvm_dev_ioctl(struct file *filp,
2199			  unsigned int ioctl, unsigned long arg)
2200{
2201	long r = -EINVAL;
2202
2203	switch (ioctl) {
2204	case KVM_GET_API_VERSION:
2205		r = -EINVAL;
2206		if (arg)
2207			goto out;
2208		r = KVM_API_VERSION;
2209		break;
2210	case KVM_CREATE_VM:
2211		r = -EINVAL;
2212		if (arg)
2213			goto out;
2214		r = kvm_dev_ioctl_create_vm();
2215		break;
2216	case KVM_CHECK_EXTENSION:
2217		r = kvm_dev_ioctl_check_extension_generic(arg);
2218		break;
2219	case KVM_GET_VCPU_MMAP_SIZE:
2220		r = -EINVAL;
2221		if (arg)
2222			goto out;
2223		r = PAGE_SIZE;     /* struct kvm_run */
2224#ifdef CONFIG_X86
2225		r += PAGE_SIZE;    /* pio data page */
2226#endif
2227#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2228		r += PAGE_SIZE;    /* coalesced mmio ring page */
2229#endif
2230		break;
2231	case KVM_TRACE_ENABLE:
2232	case KVM_TRACE_PAUSE:
2233	case KVM_TRACE_DISABLE:
2234		r = -EOPNOTSUPP;
2235		break;
2236	default:
2237		return kvm_arch_dev_ioctl(filp, ioctl, arg);
2238	}
2239out:
2240	return r;
2241}
2242
2243static struct file_operations kvm_chardev_ops = {
2244	.unlocked_ioctl = kvm_dev_ioctl,
2245	.compat_ioctl   = kvm_dev_ioctl,
2246	.llseek		= noop_llseek,
2247};
2248
2249static struct miscdevice kvm_dev = {
2250	KVM_MINOR,
2251	"kvm",
2252	&kvm_chardev_ops,
2253};
2254
2255static void hardware_enable_nolock(void *junk)
2256{
2257	int cpu = raw_smp_processor_id();
2258	int r;
2259
2260	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
2261		return;
2262
2263	cpumask_set_cpu(cpu, cpus_hardware_enabled);
2264
2265	r = kvm_arch_hardware_enable(NULL);
2266
2267	if (r) {
2268		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2269		atomic_inc(&hardware_enable_failed);
2270		printk(KERN_INFO "kvm: enabling virtualization on "
2271				 "CPU%d failed\n", cpu);
2272	}
2273}
2274
2275static void hardware_enable(void *junk)
2276{
2277	raw_spin_lock(&kvm_lock);
2278	hardware_enable_nolock(junk);
2279	raw_spin_unlock(&kvm_lock);
 
2280}
2281
2282static void hardware_disable_nolock(void *junk)
2283{
2284	int cpu = raw_smp_processor_id();
2285
2286	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
2287		return;
2288	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
2289	kvm_arch_hardware_disable(NULL);
2290}
2291
2292static void hardware_disable(void *junk)
2293{
2294	raw_spin_lock(&kvm_lock);
2295	hardware_disable_nolock(junk);
2296	raw_spin_unlock(&kvm_lock);
 
2297}
2298
2299static void hardware_disable_all_nolock(void)
2300{
2301	BUG_ON(!kvm_usage_count);
2302
2303	kvm_usage_count--;
2304	if (!kvm_usage_count)
2305		on_each_cpu(hardware_disable_nolock, NULL, 1);
2306}
2307
2308static void hardware_disable_all(void)
2309{
2310	raw_spin_lock(&kvm_lock);
2311	hardware_disable_all_nolock();
2312	raw_spin_unlock(&kvm_lock);
2313}
2314
2315static int hardware_enable_all(void)
2316{
2317	int r = 0;
2318
2319	raw_spin_lock(&kvm_lock);
2320
2321	kvm_usage_count++;
2322	if (kvm_usage_count == 1) {
2323		atomic_set(&hardware_enable_failed, 0);
2324		on_each_cpu(hardware_enable_nolock, NULL, 1);
2325
2326		if (atomic_read(&hardware_enable_failed)) {
2327			hardware_disable_all_nolock();
2328			r = -EBUSY;
2329		}
2330	}
2331
2332	raw_spin_unlock(&kvm_lock);
2333
2334	return r;
2335}
2336
2337static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2338			   void *v)
2339{
2340	int cpu = (long)v;
2341
2342	if (!kvm_usage_count)
2343		return NOTIFY_OK;
2344
2345	val &= ~CPU_TASKS_FROZEN;
2346	switch (val) {
2347	case CPU_DYING:
2348		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2349		       cpu);
2350		hardware_disable(NULL);
2351		break;
2352	case CPU_STARTING:
2353		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2354		       cpu);
2355		hardware_enable(NULL);
2356		break;
2357	}
2358	return NOTIFY_OK;
2359}
2360
2361
2362asmlinkage void kvm_spurious_fault(void)
2363{
2364	/* Fault while not rebooting.  We want the trace. */
2365	BUG();
2366}
2367EXPORT_SYMBOL_GPL(kvm_spurious_fault);
2368
2369static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2370		      void *v)
2371{
2372	/*
2373	 * Some (well, at least mine) BIOSes hang on reboot if
2374	 * in vmx root mode.
2375	 *
2376	 * And Intel TXT required VMX off for all cpu when system shutdown.
2377	 */
2378	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2379	kvm_rebooting = true;
2380	on_each_cpu(hardware_disable_nolock, NULL, 1);
2381	return NOTIFY_OK;
2382}
2383
2384static struct notifier_block kvm_reboot_notifier = {
2385	.notifier_call = kvm_reboot,
2386	.priority = 0,
2387};
2388
2389static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2390{
2391	int i;
2392
2393	for (i = 0; i < bus->dev_count; i++) {
2394		struct kvm_io_device *pos = bus->devs[i];
2395
2396		kvm_iodevice_destructor(pos);
2397	}
2398	kfree(bus);
2399}
2400
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2401/* kvm_io_bus_write - called under kvm->slots_lock */
2402int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2403		     int len, const void *val)
2404{
2405	int i;
2406	struct kvm_io_bus *bus;
 
 
 
 
 
 
 
2407
2408	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2409	for (i = 0; i < bus->dev_count; i++)
2410		if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
2411			return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2412	return -EOPNOTSUPP;
2413}
2414
2415/* kvm_io_bus_read - called under kvm->slots_lock */
2416int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
2417		    int len, void *val)
2418{
2419	int i;
2420	struct kvm_io_bus *bus;
 
 
 
 
 
 
 
2421
2422	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
2423	for (i = 0; i < bus->dev_count; i++)
2424		if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
2425			return 0;
2426	return -EOPNOTSUPP;
2427}
2428
 
2429/* Caller must hold slots_lock. */
2430int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
2431			    struct kvm_io_device *dev)
2432{
2433	struct kvm_io_bus *new_bus, *bus;
2434
2435	bus = kvm->buses[bus_idx];
2436	if (bus->dev_count > NR_IOBUS_DEVS-1)
 
2437		return -ENOSPC;
2438
2439	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
 
2440	if (!new_bus)
2441		return -ENOMEM;
2442	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
2443	new_bus->devs[new_bus->dev_count++] = dev;
 
2444	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2445	synchronize_srcu_expedited(&kvm->srcu);
2446	kfree(bus);
2447
2448	return 0;
2449}
2450
2451/* Caller must hold slots_lock. */
2452int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
2453			      struct kvm_io_device *dev)
2454{
2455	int i, r;
2456	struct kvm_io_bus *new_bus, *bus;
2457
2458	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
2459	if (!new_bus)
2460		return -ENOMEM;
2461
2462	bus = kvm->buses[bus_idx];
2463	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
2464
2465	r = -ENOENT;
2466	for (i = 0; i < new_bus->dev_count; i++)
2467		if (new_bus->devs[i] == dev) {
2468			r = 0;
2469			new_bus->devs[i] = new_bus->devs[--new_bus->dev_count];
2470			break;
2471		}
2472
2473	if (r) {
2474		kfree(new_bus);
2475		return r;
2476	}
 
 
 
 
 
 
 
 
 
2477
2478	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
2479	synchronize_srcu_expedited(&kvm->srcu);
2480	kfree(bus);
2481	return r;
2482}
2483
2484static struct notifier_block kvm_cpu_notifier = {
2485	.notifier_call = kvm_cpu_hotplug,
2486};
2487
2488static int vm_stat_get(void *_offset, u64 *val)
2489{
2490	unsigned offset = (long)_offset;
2491	struct kvm *kvm;
2492
2493	*val = 0;
2494	raw_spin_lock(&kvm_lock);
2495	list_for_each_entry(kvm, &vm_list, vm_list)
2496		*val += *(u32 *)((void *)kvm + offset);
2497	raw_spin_unlock(&kvm_lock);
2498	return 0;
2499}
2500
2501DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
2502
2503static int vcpu_stat_get(void *_offset, u64 *val)
2504{
2505	unsigned offset = (long)_offset;
2506	struct kvm *kvm;
2507	struct kvm_vcpu *vcpu;
2508	int i;
2509
2510	*val = 0;
2511	raw_spin_lock(&kvm_lock);
2512	list_for_each_entry(kvm, &vm_list, vm_list)
2513		kvm_for_each_vcpu(i, vcpu, kvm)
2514			*val += *(u32 *)((void *)vcpu + offset);
2515
2516	raw_spin_unlock(&kvm_lock);
2517	return 0;
2518}
2519
2520DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
2521
2522static const struct file_operations *stat_fops[] = {
2523	[KVM_STAT_VCPU] = &vcpu_stat_fops,
2524	[KVM_STAT_VM]   = &vm_stat_fops,
2525};
2526
2527static void kvm_init_debug(void)
2528{
 
2529	struct kvm_stats_debugfs_item *p;
2530
2531	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
2532	for (p = debugfs_entries; p->name; ++p)
 
 
 
2533		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
2534						(void *)(long)p->offset,
2535						stat_fops[p->kind]);
 
 
 
 
 
 
 
 
 
 
2536}
2537
2538static void kvm_exit_debug(void)
2539{
2540	struct kvm_stats_debugfs_item *p;
2541
2542	for (p = debugfs_entries; p->name; ++p)
2543		debugfs_remove(p->dentry);
2544	debugfs_remove(kvm_debugfs_dir);
2545}
2546
2547static int kvm_suspend(void)
2548{
2549	if (kvm_usage_count)
2550		hardware_disable_nolock(NULL);
2551	return 0;
2552}
2553
2554static void kvm_resume(void)
2555{
2556	if (kvm_usage_count) {
2557		WARN_ON(raw_spin_is_locked(&kvm_lock));
2558		hardware_enable_nolock(NULL);
2559	}
2560}
2561
2562static struct syscore_ops kvm_syscore_ops = {
2563	.suspend = kvm_suspend,
2564	.resume = kvm_resume,
2565};
2566
2567struct page *bad_page;
2568pfn_t bad_pfn;
2569
2570static inline
2571struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
2572{
2573	return container_of(pn, struct kvm_vcpu, preempt_notifier);
2574}
2575
2576static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
2577{
2578	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
 
 
2579
2580	kvm_arch_vcpu_load(vcpu, cpu);
2581}
2582
2583static void kvm_sched_out(struct preempt_notifier *pn,
2584			  struct task_struct *next)
2585{
2586	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
2587
 
 
2588	kvm_arch_vcpu_put(vcpu);
2589}
2590
2591int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
2592		  struct module *module)
2593{
2594	int r;
2595	int cpu;
2596
2597	r = kvm_arch_init(opaque);
2598	if (r)
2599		goto out_fail;
2600
2601	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2602
2603	if (bad_page == NULL) {
2604		r = -ENOMEM;
2605		goto out;
2606	}
2607
2608	bad_pfn = page_to_pfn(bad_page);
2609
2610	hwpoison_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2611
2612	if (hwpoison_page == NULL) {
2613		r = -ENOMEM;
2614		goto out_free_0;
2615	}
2616
2617	hwpoison_pfn = page_to_pfn(hwpoison_page);
2618
2619	fault_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2620
2621	if (fault_page == NULL) {
2622		r = -ENOMEM;
2623		goto out_free_0;
2624	}
2625
2626	fault_pfn = page_to_pfn(fault_page);
2627
2628	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
2629		r = -ENOMEM;
2630		goto out_free_0;
2631	}
2632
2633	r = kvm_arch_hardware_setup();
2634	if (r < 0)
2635		goto out_free_0a;
2636
2637	for_each_online_cpu(cpu) {
2638		smp_call_function_single(cpu,
2639				kvm_arch_check_processor_compat,
2640				&r, 1);
2641		if (r < 0)
2642			goto out_free_1;
2643	}
2644
2645	r = register_cpu_notifier(&kvm_cpu_notifier);
2646	if (r)
2647		goto out_free_2;
2648	register_reboot_notifier(&kvm_reboot_notifier);
2649
2650	/* A kmem cache lets us meet the alignment requirements of fx_save. */
2651	if (!vcpu_align)
2652		vcpu_align = __alignof__(struct kvm_vcpu);
2653	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
2654					   0, NULL);
2655	if (!kvm_vcpu_cache) {
2656		r = -ENOMEM;
2657		goto out_free_3;
2658	}
2659
2660	r = kvm_async_pf_init();
2661	if (r)
2662		goto out_free;
2663
2664	kvm_chardev_ops.owner = module;
2665	kvm_vm_fops.owner = module;
2666	kvm_vcpu_fops.owner = module;
2667
2668	r = misc_register(&kvm_dev);
2669	if (r) {
2670		printk(KERN_ERR "kvm: misc device register failed\n");
2671		goto out_unreg;
2672	}
2673
2674	register_syscore_ops(&kvm_syscore_ops);
2675
2676	kvm_preempt_ops.sched_in = kvm_sched_in;
2677	kvm_preempt_ops.sched_out = kvm_sched_out;
2678
2679	kvm_init_debug();
 
 
 
 
2680
2681	return 0;
2682
 
 
 
2683out_unreg:
2684	kvm_async_pf_deinit();
2685out_free:
2686	kmem_cache_destroy(kvm_vcpu_cache);
2687out_free_3:
2688	unregister_reboot_notifier(&kvm_reboot_notifier);
2689	unregister_cpu_notifier(&kvm_cpu_notifier);
2690out_free_2:
2691out_free_1:
2692	kvm_arch_hardware_unsetup();
2693out_free_0a:
2694	free_cpumask_var(cpus_hardware_enabled);
2695out_free_0:
2696	if (fault_page)
2697		__free_page(fault_page);
2698	if (hwpoison_page)
2699		__free_page(hwpoison_page);
2700	__free_page(bad_page);
2701out:
2702	kvm_arch_exit();
2703out_fail:
2704	return r;
2705}
2706EXPORT_SYMBOL_GPL(kvm_init);
2707
2708void kvm_exit(void)
2709{
2710	kvm_exit_debug();
2711	misc_deregister(&kvm_dev);
2712	kmem_cache_destroy(kvm_vcpu_cache);
2713	kvm_async_pf_deinit();
2714	unregister_syscore_ops(&kvm_syscore_ops);
2715	unregister_reboot_notifier(&kvm_reboot_notifier);
2716	unregister_cpu_notifier(&kvm_cpu_notifier);
2717	on_each_cpu(hardware_disable_nolock, NULL, 1);
2718	kvm_arch_hardware_unsetup();
2719	kvm_arch_exit();
 
2720	free_cpumask_var(cpus_hardware_enabled);
2721	__free_page(hwpoison_page);
2722	__free_page(bad_page);
2723}
2724EXPORT_SYMBOL_GPL(kvm_exit);