1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *
   4 * Copyright IBM Corp. 2007
   5 * Copyright 2010-2011 Freescale Semiconductor, Inc.
   6 *
   7 * Authors: Hollis Blanchard <hollisb@us.ibm.com>
   8 *          Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
   9 *          Scott Wood <scottwood@freescale.com>
  10 *          Varun Sethi <varun.sethi@freescale.com>
  11 */
  12
  13#include <linux/errno.h>
  14#include <linux/err.h>
  15#include <linux/kvm_host.h>
  16#include <linux/gfp.h>
  17#include <linux/module.h>
  18#include <linux/vmalloc.h>
  19#include <linux/fs.h>
  20
  21#include <asm/cputable.h>
  22#include <linux/uaccess.h>
  23#include <asm/interrupt.h>
  24#include <asm/kvm_ppc.h>
  25#include <asm/cacheflush.h>
  26#include <asm/dbell.h>
  27#include <asm/hw_irq.h>
  28#include <asm/irq.h>
  29#include <asm/time.h>
  30
  31#include "timing.h"
  32#include "booke.h"
  33
  34#define CREATE_TRACE_POINTS
  35#include "trace_booke.h"
  36
  37unsigned long kvmppc_booke_handlers;
  38
  39const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
  40	KVM_GENERIC_VM_STATS(),
  41	STATS_DESC_ICOUNTER(VM, num_2M_pages),
  42	STATS_DESC_ICOUNTER(VM, num_1G_pages)
  43};
  44
  45const struct kvm_stats_header kvm_vm_stats_header = {
  46	.name_size = KVM_STATS_NAME_SIZE,
  47	.num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
  48	.id_offset = sizeof(struct kvm_stats_header),
  49	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  50	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  51		       sizeof(kvm_vm_stats_desc),
  52};
  53
  54const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
  55	KVM_GENERIC_VCPU_STATS(),
  56	STATS_DESC_COUNTER(VCPU, sum_exits),
  57	STATS_DESC_COUNTER(VCPU, mmio_exits),
  58	STATS_DESC_COUNTER(VCPU, signal_exits),
  59	STATS_DESC_COUNTER(VCPU, light_exits),
  60	STATS_DESC_COUNTER(VCPU, itlb_real_miss_exits),
  61	STATS_DESC_COUNTER(VCPU, itlb_virt_miss_exits),
  62	STATS_DESC_COUNTER(VCPU, dtlb_real_miss_exits),
  63	STATS_DESC_COUNTER(VCPU, dtlb_virt_miss_exits),
  64	STATS_DESC_COUNTER(VCPU, syscall_exits),
  65	STATS_DESC_COUNTER(VCPU, isi_exits),
  66	STATS_DESC_COUNTER(VCPU, dsi_exits),
  67	STATS_DESC_COUNTER(VCPU, emulated_inst_exits),
  68	STATS_DESC_COUNTER(VCPU, dec_exits),
  69	STATS_DESC_COUNTER(VCPU, ext_intr_exits),
  70	STATS_DESC_COUNTER(VCPU, halt_successful_wait),
  71	STATS_DESC_COUNTER(VCPU, dbell_exits),
  72	STATS_DESC_COUNTER(VCPU, gdbell_exits),
  73	STATS_DESC_COUNTER(VCPU, ld),
  74	STATS_DESC_COUNTER(VCPU, st),
  75	STATS_DESC_COUNTER(VCPU, pthru_all),
  76	STATS_DESC_COUNTER(VCPU, pthru_host),
  77	STATS_DESC_COUNTER(VCPU, pthru_bad_aff)
  78};
  79
  80const struct kvm_stats_header kvm_vcpu_stats_header = {
  81	.name_size = KVM_STATS_NAME_SIZE,
  82	.num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
  83	.id_offset = sizeof(struct kvm_stats_header),
  84	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  85	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  86		       sizeof(kvm_vcpu_stats_desc),
  87};
  88
  89/* TODO: use vcpu_printf() */
  90void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu)
  91{
  92	int i;
  93
  94	printk("pc:   %08lx msr:  %08llx\n", vcpu->arch.regs.nip,
  95			vcpu->arch.shared->msr);
  96	printk("lr:   %08lx ctr:  %08lx\n", vcpu->arch.regs.link,
  97			vcpu->arch.regs.ctr);
  98	printk("srr0: %08llx srr1: %08llx\n", vcpu->arch.shared->srr0,
  99					    vcpu->arch.shared->srr1);
 100
 101	printk("exceptions: %08lx\n", vcpu->arch.pending_exceptions);
 102
 103	for (i = 0; i < 32; i += 4) {
 104		printk("gpr%02d: %08lx %08lx %08lx %08lx\n", i,
 105		       kvmppc_get_gpr(vcpu, i),
 106		       kvmppc_get_gpr(vcpu, i+1),
 107		       kvmppc_get_gpr(vcpu, i+2),
 108		       kvmppc_get_gpr(vcpu, i+3));
 109	}
 110}
 111
 112#ifdef CONFIG_SPE
 113void kvmppc_vcpu_disable_spe(struct kvm_vcpu *vcpu)
 114{
 115	preempt_disable();
 116	enable_kernel_spe();
 117	kvmppc_save_guest_spe(vcpu);
 118	disable_kernel_spe();
 119	vcpu->arch.shadow_msr &= ~MSR_SPE;
 120	preempt_enable();
 121}
 122
 123static void kvmppc_vcpu_enable_spe(struct kvm_vcpu *vcpu)
 124{
 125	preempt_disable();
 126	enable_kernel_spe();
 127	kvmppc_load_guest_spe(vcpu);
 128	disable_kernel_spe();
 129	vcpu->arch.shadow_msr |= MSR_SPE;
 130	preempt_enable();
 131}
 132
 133static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
 134{
 135	if (vcpu->arch.shared->msr & MSR_SPE) {
 136		if (!(vcpu->arch.shadow_msr & MSR_SPE))
 137			kvmppc_vcpu_enable_spe(vcpu);
 138	} else if (vcpu->arch.shadow_msr & MSR_SPE) {
 139		kvmppc_vcpu_disable_spe(vcpu);
 140	}
 141}
 142#else
 143static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
 144{
 145}
 146#endif
 147
 148/*
 149 * Load up guest vcpu FP state if it's needed.
 150 * It also set the MSR_FP in thread so that host know
 151 * we're holding FPU, and then host can help to save
 152 * guest vcpu FP state if other threads require to use FPU.
 153 * This simulates an FP unavailable fault.
 154 *
 155 * It requires to be called with preemption disabled.
 156 */
 157static inline void kvmppc_load_guest_fp(struct kvm_vcpu *vcpu)
 158{
 159#ifdef CONFIG_PPC_FPU
 160	if (!(current->thread.regs->msr & MSR_FP)) {
 161		enable_kernel_fp();
 162		load_fp_state(&vcpu->arch.fp);
 163		disable_kernel_fp();
 164		current->thread.fp_save_area = &vcpu->arch.fp;
 165		current->thread.regs->msr |= MSR_FP;
 166	}
 167#endif
 168}
 169
 170/*
 171 * Save guest vcpu FP state into thread.
 172 * It requires to be called with preemption disabled.
 173 */
 174static inline void kvmppc_save_guest_fp(struct kvm_vcpu *vcpu)
 175{
 176#ifdef CONFIG_PPC_FPU
 177	if (current->thread.regs->msr & MSR_FP)
 178		giveup_fpu(current);
 179	current->thread.fp_save_area = NULL;
 180#endif
 181}
 182
 183static void kvmppc_vcpu_sync_fpu(struct kvm_vcpu *vcpu)
 184{
 185#if defined(CONFIG_PPC_FPU) && !defined(CONFIG_KVM_BOOKE_HV)
 186	/* We always treat the FP bit as enabled from the host
 187	   perspective, so only need to adjust the shadow MSR */
 188	vcpu->arch.shadow_msr &= ~MSR_FP;
 189	vcpu->arch.shadow_msr |= vcpu->arch.shared->msr & MSR_FP;
 190#endif
 191}
 192
 193/*
 194 * Simulate AltiVec unavailable fault to load guest state
 195 * from thread to AltiVec unit.
 196 * It requires to be called with preemption disabled.
 197 */
 198static inline void kvmppc_load_guest_altivec(struct kvm_vcpu *vcpu)
 199{
 200#ifdef CONFIG_ALTIVEC
 201	if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
 202		if (!(current->thread.regs->msr & MSR_VEC)) {
 203			enable_kernel_altivec();
 204			load_vr_state(&vcpu->arch.vr);
 205			disable_kernel_altivec();
 206			current->thread.vr_save_area = &vcpu->arch.vr;
 207			current->thread.regs->msr |= MSR_VEC;
 208		}
 209	}
 210#endif
 211}
 212
 213/*
 214 * Save guest vcpu AltiVec state into thread.
 215 * It requires to be called with preemption disabled.
 216 */
 217static inline void kvmppc_save_guest_altivec(struct kvm_vcpu *vcpu)
 218{
 219#ifdef CONFIG_ALTIVEC
 220	if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
 221		if (current->thread.regs->msr & MSR_VEC)
 222			giveup_altivec(current);
 223		current->thread.vr_save_area = NULL;
 224	}
 225#endif
 226}
 227
 228static void kvmppc_vcpu_sync_debug(struct kvm_vcpu *vcpu)
 229{
 230	/* Synchronize guest's desire to get debug interrupts into shadow MSR */
 231#ifndef CONFIG_KVM_BOOKE_HV
 232	vcpu->arch.shadow_msr &= ~MSR_DE;
 233	vcpu->arch.shadow_msr |= vcpu->arch.shared->msr & MSR_DE;
 234#endif
 235
 236	/* Force enable debug interrupts when user space wants to debug */
 237	if (vcpu->guest_debug) {
 238#ifdef CONFIG_KVM_BOOKE_HV
 239		/*
 240		 * Since there is no shadow MSR, sync MSR_DE into the guest
 241		 * visible MSR.
 242		 */
 243		vcpu->arch.shared->msr |= MSR_DE;
 244#else
 245		vcpu->arch.shadow_msr |= MSR_DE;
 246		vcpu->arch.shared->msr &= ~MSR_DE;
 247#endif
 248	}
 249}
 250
 251/*
 252 * Helper function for "full" MSR writes.  No need to call this if only
 253 * EE/CE/ME/DE/RI are changing.
 254 */
 255void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr)
 256{
 257	u32 old_msr = vcpu->arch.shared->msr;
 258
 259#ifdef CONFIG_KVM_BOOKE_HV
 260	new_msr |= MSR_GS;
 261#endif
 262
 263	vcpu->arch.shared->msr = new_msr;
 264
 265	kvmppc_mmu_msr_notify(vcpu, old_msr);
 266	kvmppc_vcpu_sync_spe(vcpu);
 267	kvmppc_vcpu_sync_fpu(vcpu);
 268	kvmppc_vcpu_sync_debug(vcpu);
 269}
 270
 271static void kvmppc_booke_queue_irqprio(struct kvm_vcpu *vcpu,
 272                                       unsigned int priority)
 273{
 274	trace_kvm_booke_queue_irqprio(vcpu, priority);
 275	set_bit(priority, &vcpu->arch.pending_exceptions);
 276}
 277
 278void kvmppc_core_queue_dtlb_miss(struct kvm_vcpu *vcpu,
 279				 ulong dear_flags, ulong esr_flags)
 280{
 281	vcpu->arch.queued_dear = dear_flags;
 282	vcpu->arch.queued_esr = esr_flags;
 283	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DTLB_MISS);
 284}
 285
 286void kvmppc_core_queue_data_storage(struct kvm_vcpu *vcpu, ulong srr1_flags,
 287				    ulong dear_flags, ulong esr_flags)
 288{
 289	WARN_ON_ONCE(srr1_flags);
 290	vcpu->arch.queued_dear = dear_flags;
 291	vcpu->arch.queued_esr = esr_flags;
 292	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DATA_STORAGE);
 293}
 294
 295void kvmppc_core_queue_itlb_miss(struct kvm_vcpu *vcpu)
 296{
 297	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS);
 298}
 299
 300void kvmppc_core_queue_inst_storage(struct kvm_vcpu *vcpu, ulong esr_flags)
 301{
 302	vcpu->arch.queued_esr = esr_flags;
 303	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_INST_STORAGE);
 304}
 305
 306static void kvmppc_core_queue_alignment(struct kvm_vcpu *vcpu, ulong dear_flags,
 307					ulong esr_flags)
 308{
 309	vcpu->arch.queued_dear = dear_flags;
 310	vcpu->arch.queued_esr = esr_flags;
 311	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALIGNMENT);
 312}
 313
 314void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong esr_flags)
 315{
 316	vcpu->arch.queued_esr = esr_flags;
 317	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_PROGRAM);
 318}
 319
 320void kvmppc_core_queue_fpunavail(struct kvm_vcpu *vcpu, ulong srr1_flags)
 321{
 322	WARN_ON_ONCE(srr1_flags);
 323	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_FP_UNAVAIL);
 324}
 325
 326#ifdef CONFIG_ALTIVEC
 327void kvmppc_core_queue_vec_unavail(struct kvm_vcpu *vcpu, ulong srr1_flags)
 328{
 329	WARN_ON_ONCE(srr1_flags);
 330	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALTIVEC_UNAVAIL);
 331}
 332#endif
 333
 334void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu)
 335{
 336	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DECREMENTER);
 337}
 338
 339int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu)
 340{
 341	return test_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
 342}
 343
 344void kvmppc_core_dequeue_dec(struct kvm_vcpu *vcpu)
 345{
 346	clear_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
 347}
 348
 349void kvmppc_core_queue_external(struct kvm_vcpu *vcpu,
 350                                struct kvm_interrupt *irq)
 351{
 352	unsigned int prio = BOOKE_IRQPRIO_EXTERNAL;
 353
 354	if (irq->irq == KVM_INTERRUPT_SET_LEVEL)
 355		prio = BOOKE_IRQPRIO_EXTERNAL_LEVEL;
 356
 357	kvmppc_booke_queue_irqprio(vcpu, prio);
 358}
 359
 360void kvmppc_core_dequeue_external(struct kvm_vcpu *vcpu)
 361{
 362	clear_bit(BOOKE_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
 363	clear_bit(BOOKE_IRQPRIO_EXTERNAL_LEVEL, &vcpu->arch.pending_exceptions);
 364}
 365
 366static void kvmppc_core_queue_watchdog(struct kvm_vcpu *vcpu)
 367{
 368	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_WATCHDOG);
 369}
 370
 371static void kvmppc_core_dequeue_watchdog(struct kvm_vcpu *vcpu)
 372{
 373	clear_bit(BOOKE_IRQPRIO_WATCHDOG, &vcpu->arch.pending_exceptions);
 374}
 375
 376void kvmppc_core_queue_debug(struct kvm_vcpu *vcpu)
 377{
 378	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DEBUG);
 379}
 380
 381void kvmppc_core_dequeue_debug(struct kvm_vcpu *vcpu)
 382{
 383	clear_bit(BOOKE_IRQPRIO_DEBUG, &vcpu->arch.pending_exceptions);
 384}
 385
 386static void set_guest_srr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
 387{
 388	kvmppc_set_srr0(vcpu, srr0);
 389	kvmppc_set_srr1(vcpu, srr1);
 390}
 391
 392static void set_guest_csrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
 393{
 394	vcpu->arch.csrr0 = srr0;
 395	vcpu->arch.csrr1 = srr1;
 396}
 397
 398static void set_guest_dsrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
 399{
 400	if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC)) {
 401		vcpu->arch.dsrr0 = srr0;
 402		vcpu->arch.dsrr1 = srr1;
 403	} else {
 404		set_guest_csrr(vcpu, srr0, srr1);
 405	}
 406}
 407
 408static void set_guest_mcsrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
 409{
 410	vcpu->arch.mcsrr0 = srr0;
 411	vcpu->arch.mcsrr1 = srr1;
 412}
 413
 414/* Deliver the interrupt of the corresponding priority, if possible. */
 415static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu,
 416                                        unsigned int priority)
 417{
 418	int allowed = 0;
 419	ulong msr_mask = 0;
 420	bool update_esr = false, update_dear = false, update_epr = false;
 421	ulong crit_raw = vcpu->arch.shared->critical;
 422	ulong crit_r1 = kvmppc_get_gpr(vcpu, 1);
 423	bool crit;
 424	bool keep_irq = false;
 425	enum int_class int_class;
 426	ulong new_msr = vcpu->arch.shared->msr;
 427
 428	/* Truncate crit indicators in 32 bit mode */
 429	if (!(vcpu->arch.shared->msr & MSR_SF)) {
 430		crit_raw &= 0xffffffff;
 431		crit_r1 &= 0xffffffff;
 432	}
 433
 434	/* Critical section when crit == r1 */
 435	crit = (crit_raw == crit_r1);
 436	/* ... and we're in supervisor mode */
 437	crit = crit && !(vcpu->arch.shared->msr & MSR_PR);
 438
 439	if (priority == BOOKE_IRQPRIO_EXTERNAL_LEVEL) {
 440		priority = BOOKE_IRQPRIO_EXTERNAL;
 441		keep_irq = true;
 442	}
 443
 444	if ((priority == BOOKE_IRQPRIO_EXTERNAL) && vcpu->arch.epr_flags)
 445		update_epr = true;
 446
 447	switch (priority) {
 448	case BOOKE_IRQPRIO_DTLB_MISS:
 449	case BOOKE_IRQPRIO_DATA_STORAGE:
 450	case BOOKE_IRQPRIO_ALIGNMENT:
 451		update_dear = true;
 452		fallthrough;
 453	case BOOKE_IRQPRIO_INST_STORAGE:
 454	case BOOKE_IRQPRIO_PROGRAM:
 455		update_esr = true;
 456		fallthrough;
 457	case BOOKE_IRQPRIO_ITLB_MISS:
 458	case BOOKE_IRQPRIO_SYSCALL:
 459	case BOOKE_IRQPRIO_FP_UNAVAIL:
 460#ifdef CONFIG_SPE_POSSIBLE
 461	case BOOKE_IRQPRIO_SPE_UNAVAIL:
 462	case BOOKE_IRQPRIO_SPE_FP_DATA:
 463	case BOOKE_IRQPRIO_SPE_FP_ROUND:
 464#endif
 465#ifdef CONFIG_ALTIVEC
 466	case BOOKE_IRQPRIO_ALTIVEC_UNAVAIL:
 467	case BOOKE_IRQPRIO_ALTIVEC_ASSIST:
 468#endif
 469	case BOOKE_IRQPRIO_AP_UNAVAIL:
 470		allowed = 1;
 471		msr_mask = MSR_CE | MSR_ME | MSR_DE;
 472		int_class = INT_CLASS_NONCRIT;
 473		break;
 474	case BOOKE_IRQPRIO_WATCHDOG:
 475	case BOOKE_IRQPRIO_CRITICAL:
 476	case BOOKE_IRQPRIO_DBELL_CRIT:
 477		allowed = vcpu->arch.shared->msr & MSR_CE;
 478		allowed = allowed && !crit;
 479		msr_mask = MSR_ME;
 480		int_class = INT_CLASS_CRIT;
 481		break;
 482	case BOOKE_IRQPRIO_MACHINE_CHECK:
 483		allowed = vcpu->arch.shared->msr & MSR_ME;
 484		allowed = allowed && !crit;
 485		int_class = INT_CLASS_MC;
 486		break;
 487	case BOOKE_IRQPRIO_DECREMENTER:
 488	case BOOKE_IRQPRIO_FIT:
 489		keep_irq = true;
 490		fallthrough;
 491	case BOOKE_IRQPRIO_EXTERNAL:
 492	case BOOKE_IRQPRIO_DBELL:
 493		allowed = vcpu->arch.shared->msr & MSR_EE;
 494		allowed = allowed && !crit;
 495		msr_mask = MSR_CE | MSR_ME | MSR_DE;
 496		int_class = INT_CLASS_NONCRIT;
 497		break;
 498	case BOOKE_IRQPRIO_DEBUG:
 499		allowed = vcpu->arch.shared->msr & MSR_DE;
 500		allowed = allowed && !crit;
 501		msr_mask = MSR_ME;
 502		if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC))
 503			int_class = INT_CLASS_DBG;
 504		else
 505			int_class = INT_CLASS_CRIT;
 506
 507		break;
 508	}
 509
 510	if (allowed) {
 511		switch (int_class) {
 512		case INT_CLASS_NONCRIT:
 513			set_guest_srr(vcpu, vcpu->arch.regs.nip,
 514				      vcpu->arch.shared->msr);
 515			break;
 516		case INT_CLASS_CRIT:
 517			set_guest_csrr(vcpu, vcpu->arch.regs.nip,
 518				       vcpu->arch.shared->msr);
 519			break;
 520		case INT_CLASS_DBG:
 521			set_guest_dsrr(vcpu, vcpu->arch.regs.nip,
 522				       vcpu->arch.shared->msr);
 523			break;
 524		case INT_CLASS_MC:
 525			set_guest_mcsrr(vcpu, vcpu->arch.regs.nip,
 526					vcpu->arch.shared->msr);
 527			break;
 528		}
 529
 530		vcpu->arch.regs.nip = vcpu->arch.ivpr |
 531					vcpu->arch.ivor[priority];
 532		if (update_esr)
 533			kvmppc_set_esr(vcpu, vcpu->arch.queued_esr);
 534		if (update_dear)
 535			kvmppc_set_dar(vcpu, vcpu->arch.queued_dear);
 536		if (update_epr) {
 537			if (vcpu->arch.epr_flags & KVMPPC_EPR_USER)
 538				kvm_make_request(KVM_REQ_EPR_EXIT, vcpu);
 539			else if (vcpu->arch.epr_flags & KVMPPC_EPR_KERNEL) {
 540				BUG_ON(vcpu->arch.irq_type != KVMPPC_IRQ_MPIC);
 541				kvmppc_mpic_set_epr(vcpu);
 542			}
 543		}
 544
 545		new_msr &= msr_mask;
 546#if defined(CONFIG_64BIT)
 547		if (vcpu->arch.epcr & SPRN_EPCR_ICM)
 548			new_msr |= MSR_CM;
 549#endif
 550		kvmppc_set_msr(vcpu, new_msr);
 551
 552		if (!keep_irq)
 553			clear_bit(priority, &vcpu->arch.pending_exceptions);
 554	}
 555
 556#ifdef CONFIG_KVM_BOOKE_HV
 557	/*
 558	 * If an interrupt is pending but masked, raise a guest doorbell
 559	 * so that we are notified when the guest enables the relevant
 560	 * MSR bit.
 561	 */
 562	if (vcpu->arch.pending_exceptions & BOOKE_IRQMASK_EE)
 563		kvmppc_set_pending_interrupt(vcpu, INT_CLASS_NONCRIT);
 564	if (vcpu->arch.pending_exceptions & BOOKE_IRQMASK_CE)
 565		kvmppc_set_pending_interrupt(vcpu, INT_CLASS_CRIT);
 566	if (vcpu->arch.pending_exceptions & BOOKE_IRQPRIO_MACHINE_CHECK)
 567		kvmppc_set_pending_interrupt(vcpu, INT_CLASS_MC);
 568#endif
 569
 570	return allowed;
 571}
 572
 573/*
 574 * Return the number of jiffies until the next timeout.  If the timeout is
 575 * longer than the NEXT_TIMER_MAX_DELTA, then return NEXT_TIMER_MAX_DELTA
 576 * because the larger value can break the timer APIs.
 577 */
 578static unsigned long watchdog_next_timeout(struct kvm_vcpu *vcpu)
 579{
 580	u64 tb, wdt_tb, wdt_ticks = 0;
 581	u64 nr_jiffies = 0;
 582	u32 period = TCR_GET_WP(vcpu->arch.tcr);
 583
 584	wdt_tb = 1ULL << (63 - period);
 585	tb = get_tb();
 586	/*
 587	 * The watchdog timeout will hapeen when TB bit corresponding
 588	 * to watchdog will toggle from 0 to 1.
 589	 */
 590	if (tb & wdt_tb)
 591		wdt_ticks = wdt_tb;
 592
 593	wdt_ticks += wdt_tb - (tb & (wdt_tb - 1));
 594
 595	/* Convert timebase ticks to jiffies */
 596	nr_jiffies = wdt_ticks;
 597
 598	if (do_div(nr_jiffies, tb_ticks_per_jiffy))
 599		nr_jiffies++;
 600
 601	return min_t(unsigned long long, nr_jiffies, NEXT_TIMER_MAX_DELTA);
 602}
 603
 604static void arm_next_watchdog(struct kvm_vcpu *vcpu)
 605{
 606	unsigned long nr_jiffies;
 607	unsigned long flags;
 608
 609	/*
 610	 * If TSR_ENW and TSR_WIS are not set then no need to exit to
 611	 * userspace, so clear the KVM_REQ_WATCHDOG request.
 612	 */
 613	if ((vcpu->arch.tsr & (TSR_ENW | TSR_WIS)) != (TSR_ENW | TSR_WIS))
 614		kvm_clear_request(KVM_REQ_WATCHDOG, vcpu);
 615
 616	spin_lock_irqsave(&vcpu->arch.wdt_lock, flags);
 617	nr_jiffies = watchdog_next_timeout(vcpu);
 618	/*
 619	 * If the number of jiffies of watchdog timer >= NEXT_TIMER_MAX_DELTA
 620	 * then do not run the watchdog timer as this can break timer APIs.
 621	 */
 622	if (nr_jiffies < NEXT_TIMER_MAX_DELTA)
 623		mod_timer(&vcpu->arch.wdt_timer, jiffies + nr_jiffies);
 624	else
 625		del_timer(&vcpu->arch.wdt_timer);
 626	spin_unlock_irqrestore(&vcpu->arch.wdt_lock, flags);
 627}
 628
 629static void kvmppc_watchdog_func(struct timer_list *t)
 630{
 631	struct kvm_vcpu *vcpu = from_timer(vcpu, t, arch.wdt_timer);
 632	u32 tsr, new_tsr;
 633	int final;
 634
 635	do {
 636		new_tsr = tsr = vcpu->arch.tsr;
 637		final = 0;
 638
 639		/* Time out event */
 640		if (tsr & TSR_ENW) {
 641			if (tsr & TSR_WIS)
 642				final = 1;
 643			else
 644				new_tsr = tsr | TSR_WIS;
 645		} else {
 646			new_tsr = tsr | TSR_ENW;
 647		}
 648	} while (cmpxchg(&vcpu->arch.tsr, tsr, new_tsr) != tsr);
 649
 650	if (new_tsr & TSR_WIS) {
 651		smp_wmb();
 652		kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
 653		kvm_vcpu_kick(vcpu);
 654	}
 655
 656	/*
 657	 * If this is final watchdog expiry and some action is required
 658	 * then exit to userspace.
 659	 */
 660	if (final && (vcpu->arch.tcr & TCR_WRC_MASK) &&
 661	    vcpu->arch.watchdog_enabled) {
 662		smp_wmb();
 663		kvm_make_request(KVM_REQ_WATCHDOG, vcpu);
 664		kvm_vcpu_kick(vcpu);
 665	}
 666
 667	/*
 668	 * Stop running the watchdog timer after final expiration to
 669	 * prevent the host from being flooded with timers if the
 670	 * guest sets a short period.
 671	 * Timers will resume when TSR/TCR is updated next time.
 672	 */
 673	if (!final)
 674		arm_next_watchdog(vcpu);
 675}
 676
 677static void update_timer_ints(struct kvm_vcpu *vcpu)
 678{
 679	if ((vcpu->arch.tcr & TCR_DIE) && (vcpu->arch.tsr & TSR_DIS))
 680		kvmppc_core_queue_dec(vcpu);
 681	else
 682		kvmppc_core_dequeue_dec(vcpu);
 683
 684	if ((vcpu->arch.tcr & TCR_WIE) && (vcpu->arch.tsr & TSR_WIS))
 685		kvmppc_core_queue_watchdog(vcpu);
 686	else
 687		kvmppc_core_dequeue_watchdog(vcpu);
 688}
 689
 690static void kvmppc_core_check_exceptions(struct kvm_vcpu *vcpu)
 691{
 692	unsigned long *pending = &vcpu->arch.pending_exceptions;
 693	unsigned int priority;
 694
 695	priority = __ffs(*pending);
 696	while (priority < BOOKE_IRQPRIO_MAX) {
 697		if (kvmppc_booke_irqprio_deliver(vcpu, priority))
 698			break;
 699
 700		priority = find_next_bit(pending,
 701		                         BITS_PER_BYTE * sizeof(*pending),
 702		                         priority + 1);
 703	}
 704
 705	/* Tell the guest about our interrupt status */
 706	vcpu->arch.shared->int_pending = !!*pending;
 707}
 708
 709/* Check pending exceptions and deliver one, if possible. */
 710int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu)
 711{
 712	int r = 0;
 713	WARN_ON_ONCE(!irqs_disabled());
 714
 715	kvmppc_core_check_exceptions(vcpu);
 716
 717	if (kvm_request_pending(vcpu)) {
 718		/* Exception delivery raised request; start over */
 719		return 1;
 720	}
 721
 722	if (vcpu->arch.shared->msr & MSR_WE) {
 723		local_irq_enable();
 724		kvm_vcpu_halt(vcpu);
 725		hard_irq_disable();
 726
 727		kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);
 728		r = 1;
 729	}
 730
 731	return r;
 732}
 733
 734int kvmppc_core_check_requests(struct kvm_vcpu *vcpu)
 735{
 736	int r = 1; /* Indicate we want to get back into the guest */
 737
 738	if (kvm_check_request(KVM_REQ_PENDING_TIMER, vcpu))
 739		update_timer_ints(vcpu);
 740#if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
 741	if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
 742		kvmppc_core_flush_tlb(vcpu);
 743#endif
 744
 745	if (kvm_check_request(KVM_REQ_WATCHDOG, vcpu)) {
 746		vcpu->run->exit_reason = KVM_EXIT_WATCHDOG;
 747		r = 0;
 748	}
 749
 750	if (kvm_check_request(KVM_REQ_EPR_EXIT, vcpu)) {
 751		vcpu->run->epr.epr = 0;
 752		vcpu->arch.epr_needed = true;
 753		vcpu->run->exit_reason = KVM_EXIT_EPR;
 754		r = 0;
 755	}
 756
 757	return r;
 758}
 759
 760int kvmppc_vcpu_run(struct kvm_vcpu *vcpu)
 761{
 762	int ret, s;
 763	struct debug_reg debug;
 764
 765	if (!vcpu->arch.sane) {
 766		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 767		return -EINVAL;
 768	}
 769
 770	s = kvmppc_prepare_to_enter(vcpu);
 771	if (s <= 0) {
 772		ret = s;
 773		goto out;
 774	}
 775	/* interrupts now hard-disabled */
 776
 777#ifdef CONFIG_PPC_FPU
 778	/* Save userspace FPU state in stack */
 779	enable_kernel_fp();
 780
 781	/*
 782	 * Since we can't trap on MSR_FP in GS-mode, we consider the guest
 783	 * as always using the FPU.
 784	 */
 785	kvmppc_load_guest_fp(vcpu);
 786#endif
 787
 788#ifdef CONFIG_ALTIVEC
 789	/* Save userspace AltiVec state in stack */
 790	if (cpu_has_feature(CPU_FTR_ALTIVEC))
 791		enable_kernel_altivec();
 792	/*
 793	 * Since we can't trap on MSR_VEC in GS-mode, we consider the guest
 794	 * as always using the AltiVec.
 795	 */
 796	kvmppc_load_guest_altivec(vcpu);
 797#endif
 798
 799	/* Switch to guest debug context */
 800	debug = vcpu->arch.dbg_reg;
 801	switch_booke_debug_regs(&debug);
 802	debug = current->thread.debug;
 803	current->thread.debug = vcpu->arch.dbg_reg;
 804
 805	vcpu->arch.pgdir = vcpu->kvm->mm->pgd;
 806	kvmppc_fix_ee_before_entry();
 807
 808	ret = __kvmppc_vcpu_run(vcpu);
 809
 810	/* No need for guest_exit. It's done in handle_exit.
 811	   We also get here with interrupts enabled. */
 812
 813	/* Switch back to user space debug context */
 814	switch_booke_debug_regs(&debug);
 815	current->thread.debug = debug;
 816
 817#ifdef CONFIG_PPC_FPU
 818	kvmppc_save_guest_fp(vcpu);
 819#endif
 820
 821#ifdef CONFIG_ALTIVEC
 822	kvmppc_save_guest_altivec(vcpu);
 823#endif
 824
 825out:
 826	vcpu->mode = OUTSIDE_GUEST_MODE;
 827	return ret;
 828}
 829
 830static int emulation_exit(struct kvm_vcpu *vcpu)
 831{
 832	enum emulation_result er;
 833
 834	er = kvmppc_emulate_instruction(vcpu);
 835	switch (er) {
 836	case EMULATE_DONE:
 837		/* don't overwrite subtypes, just account kvm_stats */
 838		kvmppc_account_exit_stat(vcpu, EMULATED_INST_EXITS);
 839		/* Future optimization: only reload non-volatiles if
 840		 * they were actually modified by emulation. */
 841		return RESUME_GUEST_NV;
 842
 843	case EMULATE_AGAIN:
 844		return RESUME_GUEST;
 845
 846	case EMULATE_FAIL:
 847		printk(KERN_CRIT "%s: emulation at %lx failed (%08lx)\n",
 848		       __func__, vcpu->arch.regs.nip, vcpu->arch.last_inst);
 849		/* For debugging, encode the failing instruction and
 850		 * report it to userspace. */
 851		vcpu->run->hw.hardware_exit_reason = ~0ULL << 32;
 852		vcpu->run->hw.hardware_exit_reason |= vcpu->arch.last_inst;
 853		kvmppc_core_queue_program(vcpu, ESR_PIL);
 854		return RESUME_HOST;
 855
 856	case EMULATE_EXIT_USER:
 857		return RESUME_HOST;
 858
 859	default:
 860		BUG();
 861	}
 862}
 863
 864static int kvmppc_handle_debug(struct kvm_vcpu *vcpu)
 865{
 866	struct kvm_run *run = vcpu->run;
 867	struct debug_reg *dbg_reg = &(vcpu->arch.dbg_reg);
 868	u32 dbsr = vcpu->arch.dbsr;
 869
 870	if (vcpu->guest_debug == 0) {
 871		/*
 872		 * Debug resources belong to Guest.
 873		 * Imprecise debug event is not injected
 874		 */
 875		if (dbsr & DBSR_IDE) {
 876			dbsr &= ~DBSR_IDE;
 877			if (!dbsr)
 878				return RESUME_GUEST;
 879		}
 880
 881		if (dbsr && (vcpu->arch.shared->msr & MSR_DE) &&
 882			    (vcpu->arch.dbg_reg.dbcr0 & DBCR0_IDM))
 883			kvmppc_core_queue_debug(vcpu);
 884
 885		/* Inject a program interrupt if trap debug is not allowed */
 886		if ((dbsr & DBSR_TIE) && !(vcpu->arch.shared->msr & MSR_DE))
 887			kvmppc_core_queue_program(vcpu, ESR_PTR);
 888
 889		return RESUME_GUEST;
 890	}
 891
 892	/*
 893	 * Debug resource owned by userspace.
 894	 * Clear guest dbsr (vcpu->arch.dbsr)
 895	 */
 896	vcpu->arch.dbsr = 0;
 897	run->debug.arch.status = 0;
 898	run->debug.arch.address = vcpu->arch.regs.nip;
 899
 900	if (dbsr & (DBSR_IAC1 | DBSR_IAC2 | DBSR_IAC3 | DBSR_IAC4)) {
 901		run->debug.arch.status |= KVMPPC_DEBUG_BREAKPOINT;
 902	} else {
 903		if (dbsr & (DBSR_DAC1W | DBSR_DAC2W))
 904			run->debug.arch.status |= KVMPPC_DEBUG_WATCH_WRITE;
 905		else if (dbsr & (DBSR_DAC1R | DBSR_DAC2R))
 906			run->debug.arch.status |= KVMPPC_DEBUG_WATCH_READ;
 907		if (dbsr & (DBSR_DAC1R | DBSR_DAC1W))
 908			run->debug.arch.address = dbg_reg->dac1;
 909		else if (dbsr & (DBSR_DAC2R | DBSR_DAC2W))
 910			run->debug.arch.address = dbg_reg->dac2;
 911	}
 912
 913	return RESUME_HOST;
 914}
 915
 916static void kvmppc_fill_pt_regs(struct pt_regs *regs)
 917{
 918	ulong r1, msr, lr;
 919
 920	asm("mr %0, 1" : "=r"(r1));
 921	asm("mflr %0" : "=r"(lr));
 922	asm("mfmsr %0" : "=r"(msr));
 923
 924	memset(regs, 0, sizeof(*regs));
 925	regs->gpr[1] = r1;
 926	regs->nip = _THIS_IP_;
 927	regs->msr = msr;
 928	regs->link = lr;
 929}
 930
 931/*
 932 * For interrupts needed to be handled by host interrupt handlers,
 933 * corresponding host handler are called from here in similar way
 934 * (but not exact) as they are called from low level handler
 935 * (such as from arch/powerpc/kernel/head_fsl_booke.S).
 936 */
 937static void kvmppc_restart_interrupt(struct kvm_vcpu *vcpu,
 938				     unsigned int exit_nr)
 939{
 940	struct pt_regs regs;
 941
 942	switch (exit_nr) {
 943	case BOOKE_INTERRUPT_EXTERNAL:
 944		kvmppc_fill_pt_regs(&regs);
 945		do_IRQ(&regs);
 946		break;
 947	case BOOKE_INTERRUPT_DECREMENTER:
 948		kvmppc_fill_pt_regs(&regs);
 949		timer_interrupt(&regs);
 950		break;
 951#if defined(CONFIG_PPC_DOORBELL)
 952	case BOOKE_INTERRUPT_DOORBELL:
 953		kvmppc_fill_pt_regs(&regs);
 954		doorbell_exception(&regs);
 955		break;
 956#endif
 957	case BOOKE_INTERRUPT_MACHINE_CHECK:
 958		/* FIXME */
 959		break;
 960	case BOOKE_INTERRUPT_PERFORMANCE_MONITOR:
 961		kvmppc_fill_pt_regs(&regs);
 962		performance_monitor_exception(&regs);
 963		break;
 964	case BOOKE_INTERRUPT_WATCHDOG:
 965		kvmppc_fill_pt_regs(&regs);
 966#ifdef CONFIG_BOOKE_WDT
 967		WatchdogException(&regs);
 968#else
 969		unknown_exception(&regs);
 970#endif
 971		break;
 972	case BOOKE_INTERRUPT_CRITICAL:
 973		kvmppc_fill_pt_regs(&regs);
 974		unknown_exception(&regs);
 975		break;
 976	case BOOKE_INTERRUPT_DEBUG:
 977		/* Save DBSR before preemption is enabled */
 978		vcpu->arch.dbsr = mfspr(SPRN_DBSR);
 979		kvmppc_clear_dbsr();
 980		break;
 981	}
 982}
 983
 984static int kvmppc_resume_inst_load(struct kvm_vcpu *vcpu,
 985				  enum emulation_result emulated, u32 last_inst)
 986{
 987	switch (emulated) {
 988	case EMULATE_AGAIN:
 989		return RESUME_GUEST;
 990
 991	case EMULATE_FAIL:
 992		pr_debug("%s: load instruction from guest address %lx failed\n",
 993		       __func__, vcpu->arch.regs.nip);
 994		/* For debugging, encode the failing instruction and
 995		 * report it to userspace. */
 996		vcpu->run->hw.hardware_exit_reason = ~0ULL << 32;
 997		vcpu->run->hw.hardware_exit_reason |= last_inst;
 998		kvmppc_core_queue_program(vcpu, ESR_PIL);
 999		return RESUME_HOST;
1000
1001	default:
1002		BUG();
1003	}
1004}
1005
1006/*
1007 * kvmppc_handle_exit
1008 *
1009 * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV)
1010 */
1011int kvmppc_handle_exit(struct kvm_vcpu *vcpu, unsigned int exit_nr)
1012{
1013	struct kvm_run *run = vcpu->run;
1014	int r = RESUME_HOST;
1015	int s;
1016	int idx;
1017	u32 last_inst = KVM_INST_FETCH_FAILED;
1018	ppc_inst_t pinst;
1019	enum emulation_result emulated = EMULATE_DONE;
1020
1021	/* Fix irq state (pairs with kvmppc_fix_ee_before_entry()) */
1022	kvmppc_fix_ee_after_exit();
1023
1024	/* update before a new last_exit_type is rewritten */
1025	kvmppc_update_timing_stats(vcpu);
1026
1027	/* restart interrupts if they were meant for the host */
1028	kvmppc_restart_interrupt(vcpu, exit_nr);
1029
1030	/*
1031	 * get last instruction before being preempted
1032	 * TODO: for e6500 check also BOOKE_INTERRUPT_LRAT_ERROR & ESR_DATA
1033	 */
1034	switch (exit_nr) {
1035	case BOOKE_INTERRUPT_DATA_STORAGE:
1036	case BOOKE_INTERRUPT_DTLB_MISS:
1037	case BOOKE_INTERRUPT_HV_PRIV:
1038		emulated = kvmppc_get_last_inst(vcpu, INST_GENERIC, &pinst);
1039		last_inst = ppc_inst_val(pinst);
1040		break;
1041	case BOOKE_INTERRUPT_PROGRAM:
1042		/* SW breakpoints arrive as illegal instructions on HV */
1043		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
1044			emulated = kvmppc_get_last_inst(vcpu, INST_GENERIC, &pinst);
1045			last_inst = ppc_inst_val(pinst);
1046		}
1047		break;
1048	default:
1049		break;
1050	}
1051
1052	trace_kvm_exit(exit_nr, vcpu);
1053
1054	context_tracking_guest_exit();
1055	if (!vtime_accounting_enabled_this_cpu()) {
1056		local_irq_enable();
1057		/*
1058		 * Service IRQs here before vtime_account_guest_exit() so any
1059		 * ticks that occurred while running the guest are accounted to
1060		 * the guest. If vtime accounting is enabled, accounting uses
1061		 * TB rather than ticks, so it can be done without enabling
1062		 * interrupts here, which has the problem that it accounts
1063		 * interrupt processing overhead to the host.
1064		 */
1065		local_irq_disable();
1066	}
1067	vtime_account_guest_exit();
1068
1069	local_irq_enable();
1070
1071	run->exit_reason = KVM_EXIT_UNKNOWN;
1072	run->ready_for_interrupt_injection = 1;
1073
1074	if (emulated != EMULATE_DONE) {
1075		r = kvmppc_resume_inst_load(vcpu, emulated, last_inst);
1076		goto out;
1077	}
1078
1079	switch (exit_nr) {
1080	case BOOKE_INTERRUPT_MACHINE_CHECK:
1081		printk("MACHINE CHECK: %lx\n", mfspr(SPRN_MCSR));
1082		kvmppc_dump_vcpu(vcpu);
1083		/* For debugging, send invalid exit reason to user space */
1084		run->hw.hardware_exit_reason = ~1ULL << 32;
1085		run->hw.hardware_exit_reason |= mfspr(SPRN_MCSR);
1086		r = RESUME_HOST;
1087		break;
1088
1089	case BOOKE_INTERRUPT_EXTERNAL:
1090		kvmppc_account_exit(vcpu, EXT_INTR_EXITS);
1091		r = RESUME_GUEST;
1092		break;
1093
1094	case BOOKE_INTERRUPT_DECREMENTER:
1095		kvmppc_account_exit(vcpu, DEC_EXITS);
1096		r = RESUME_GUEST;
1097		break;
1098
1099	case BOOKE_INTERRUPT_WATCHDOG:
1100		r = RESUME_GUEST;
1101		break;
1102
1103	case BOOKE_INTERRUPT_DOORBELL:
1104		kvmppc_account_exit(vcpu, DBELL_EXITS);
1105		r = RESUME_GUEST;
1106		break;
1107
1108	case BOOKE_INTERRUPT_GUEST_DBELL_CRIT:
1109		kvmppc_account_exit(vcpu, GDBELL_EXITS);
1110
1111		/*
1112		 * We are here because there is a pending guest interrupt
1113		 * which could not be delivered as MSR_CE or MSR_ME was not
1114		 * set.  Once we break from here we will retry delivery.
1115		 */
1116		r = RESUME_GUEST;
1117		break;
1118
1119	case BOOKE_INTERRUPT_GUEST_DBELL:
1120		kvmppc_account_exit(vcpu, GDBELL_EXITS);
1121
1122		/*
1123		 * We are here because there is a pending guest interrupt
1124		 * which could not be delivered as MSR_EE was not set.  Once
1125		 * we break from here we will retry delivery.
1126		 */
1127		r = RESUME_GUEST;
1128		break;
1129
1130	case BOOKE_INTERRUPT_PERFORMANCE_MONITOR:
1131		r = RESUME_GUEST;
1132		break;
1133
1134	case BOOKE_INTERRUPT_HV_PRIV:
1135		r = emulation_exit(vcpu);
1136		break;
1137
1138	case BOOKE_INTERRUPT_PROGRAM:
1139		if ((vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) &&
1140			(last_inst == KVMPPC_INST_SW_BREAKPOINT)) {
1141			/*
1142			 * We are here because of an SW breakpoint instr,
1143			 * so lets return to host to handle.
1144			 */
1145			r = kvmppc_handle_debug(vcpu);
1146			run->exit_reason = KVM_EXIT_DEBUG;
1147			kvmppc_account_exit(vcpu, DEBUG_EXITS);
1148			break;
1149		}
1150
1151		if (vcpu->arch.shared->msr & (MSR_PR | MSR_GS)) {
1152			/*
1153			 * Program traps generated by user-level software must
1154			 * be handled by the guest kernel.
1155			 *
1156			 * In GS mode, hypervisor privileged instructions trap
1157			 * on BOOKE_INTERRUPT_HV_PRIV, not here, so these are
1158			 * actual program interrupts, handled by the guest.
1159			 */
1160			kvmppc_core_queue_program(vcpu, vcpu->arch.fault_esr);
1161			r = RESUME_GUEST;
1162			kvmppc_account_exit(vcpu, USR_PR_INST);
1163			break;
1164		}
1165
1166		r = emulation_exit(vcpu);
1167		break;
1168
1169	case BOOKE_INTERRUPT_FP_UNAVAIL:
1170		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_FP_UNAVAIL);
1171		kvmppc_account_exit(vcpu, FP_UNAVAIL);
1172		r = RESUME_GUEST;
1173		break;
1174
1175#ifdef CONFIG_SPE
1176	case BOOKE_INTERRUPT_SPE_UNAVAIL: {
1177		if (vcpu->arch.shared->msr & MSR_SPE)
1178			kvmppc_vcpu_enable_spe(vcpu);
1179		else
1180			kvmppc_booke_queue_irqprio(vcpu,
1181						   BOOKE_IRQPRIO_SPE_UNAVAIL);
1182		r = RESUME_GUEST;
1183		break;
1184	}
1185
1186	case BOOKE_INTERRUPT_SPE_FP_DATA:
1187		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_DATA);
1188		r = RESUME_GUEST;
1189		break;
1190
1191	case BOOKE_INTERRUPT_SPE_FP_ROUND:
1192		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_ROUND);
1193		r = RESUME_GUEST;
1194		break;
1195#elif defined(CONFIG_SPE_POSSIBLE)
1196	case BOOKE_INTERRUPT_SPE_UNAVAIL:
1197		/*
1198		 * Guest wants SPE, but host kernel doesn't support it.  Send
1199		 * an "unimplemented operation" program check to the guest.
1200		 */
1201		kvmppc_core_queue_program(vcpu, ESR_PUO | ESR_SPV);
1202		r = RESUME_GUEST;
1203		break;
1204
1205	/*
1206	 * These really should never happen without CONFIG_SPE,
1207	 * as we should never enable the real MSR[SPE] in the guest.
1208	 */
1209	case BOOKE_INTERRUPT_SPE_FP_DATA:
1210	case BOOKE_INTERRUPT_SPE_FP_ROUND:
1211		printk(KERN_CRIT "%s: unexpected SPE interrupt %u at %08lx\n",
1212		       __func__, exit_nr, vcpu->arch.regs.nip);
1213		run->hw.hardware_exit_reason = exit_nr;
1214		r = RESUME_HOST;
1215		break;
1216#endif /* CONFIG_SPE_POSSIBLE */
1217
1218/*
1219 * On cores with Vector category, KVM is loaded only if CONFIG_ALTIVEC,
1220 * see kvmppc_e500mc_check_processor_compat().
1221 */
1222#ifdef CONFIG_ALTIVEC
1223	case BOOKE_INTERRUPT_ALTIVEC_UNAVAIL:
1224		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALTIVEC_UNAVAIL);
1225		r = RESUME_GUEST;
1226		break;
1227
1228	case BOOKE_INTERRUPT_ALTIVEC_ASSIST:
1229		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALTIVEC_ASSIST);
1230		r = RESUME_GUEST;
1231		break;
1232#endif
1233
1234	case BOOKE_INTERRUPT_DATA_STORAGE:
1235		kvmppc_core_queue_data_storage(vcpu, 0, vcpu->arch.fault_dear,
1236		                               vcpu->arch.fault_esr);
1237		kvmppc_account_exit(vcpu, DSI_EXITS);
1238		r = RESUME_GUEST;
1239		break;
1240
1241	case BOOKE_INTERRUPT_INST_STORAGE:
1242		kvmppc_core_queue_inst_storage(vcpu, vcpu->arch.fault_esr);
1243		kvmppc_account_exit(vcpu, ISI_EXITS);
1244		r = RESUME_GUEST;
1245		break;
1246
1247	case BOOKE_INTERRUPT_ALIGNMENT:
1248		kvmppc_core_queue_alignment(vcpu, vcpu->arch.fault_dear,
1249		                            vcpu->arch.fault_esr);
1250		r = RESUME_GUEST;
1251		break;
1252
1253#ifdef CONFIG_KVM_BOOKE_HV
1254	case BOOKE_INTERRUPT_HV_SYSCALL:
1255		if (!(vcpu->arch.shared->msr & MSR_PR)) {
1256			kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
1257		} else {
1258			/*
1259			 * hcall from guest userspace -- send privileged
1260			 * instruction program check.
1261			 */
1262			kvmppc_core_queue_program(vcpu, ESR_PPR);
1263		}
1264
1265		r = RESUME_GUEST;
1266		break;
1267#else
1268	case BOOKE_INTERRUPT_SYSCALL:
1269		if (!(vcpu->arch.shared->msr & MSR_PR) &&
1270		    (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
1271			/* KVM PV hypercalls */
1272			kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
1273			r = RESUME_GUEST;
1274		} else {
1275			/* Guest syscalls */
1276			kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SYSCALL);
1277		}
1278		kvmppc_account_exit(vcpu, SYSCALL_EXITS);
1279		r = RESUME_GUEST;
1280		break;
1281#endif
1282
1283	case BOOKE_INTERRUPT_DTLB_MISS: {
1284		unsigned long eaddr = vcpu->arch.fault_dear;
1285		int gtlb_index;
1286		gpa_t gpaddr;
1287		gfn_t gfn;
1288
1289#ifdef CONFIG_KVM_E500V2
1290		if (!(vcpu->arch.shared->msr & MSR_PR) &&
1291		    (eaddr & PAGE_MASK) == vcpu->arch.magic_page_ea) {
1292			kvmppc_map_magic(vcpu);
1293			kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
1294			r = RESUME_GUEST;
1295
1296			break;
1297		}
1298#endif
1299
1300		/* Check the guest TLB. */
1301		gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
1302		if (gtlb_index < 0) {
1303			/* The guest didn't have a mapping for it. */
1304			kvmppc_core_queue_dtlb_miss(vcpu,
1305			                            vcpu->arch.fault_dear,
1306			                            vcpu->arch.fault_esr);
1307			kvmppc_mmu_dtlb_miss(vcpu);
1308			kvmppc_account_exit(vcpu, DTLB_REAL_MISS_EXITS);
1309			r = RESUME_GUEST;
1310			break;
1311		}
1312
1313		idx = srcu_read_lock(&vcpu->kvm->srcu);
1314
1315		gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
1316		gfn = gpaddr >> PAGE_SHIFT;
1317
1318		if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
1319			/* The guest TLB had a mapping, but the shadow TLB
1320			 * didn't, and it is RAM. This could be because:
1321			 * a) the entry is mapping the host kernel, or
1322			 * b) the guest used a large mapping which we're faking
1323			 * Either way, we need to satisfy the fault without
1324			 * invoking the guest. */
1325			kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
1326			kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
1327			r = RESUME_GUEST;
1328		} else {
1329			/* Guest has mapped and accessed a page which is not
1330			 * actually RAM. */
1331			vcpu->arch.paddr_accessed = gpaddr;
1332			vcpu->arch.vaddr_accessed = eaddr;
1333			r = kvmppc_emulate_mmio(vcpu);
1334			kvmppc_account_exit(vcpu, MMIO_EXITS);
1335		}
1336
1337		srcu_read_unlock(&vcpu->kvm->srcu, idx);
1338		break;
1339	}
1340
1341	case BOOKE_INTERRUPT_ITLB_MISS: {
1342		unsigned long eaddr = vcpu->arch.regs.nip;
1343		gpa_t gpaddr;
1344		gfn_t gfn;
1345		int gtlb_index;
1346
1347		r = RESUME_GUEST;
1348
1349		/* Check the guest TLB. */
1350		gtlb_index = kvmppc_mmu_itlb_index(vcpu, eaddr);
1351		if (gtlb_index < 0) {
1352			/* The guest didn't have a mapping for it. */
1353			kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS);
1354			kvmppc_mmu_itlb_miss(vcpu);
1355			kvmppc_account_exit(vcpu, ITLB_REAL_MISS_EXITS);
1356			break;
1357		}
1358
1359		kvmppc_account_exit(vcpu, ITLB_VIRT_MISS_EXITS);
1360
1361		idx = srcu_read_lock(&vcpu->kvm->srcu);
1362
1363		gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
1364		gfn = gpaddr >> PAGE_SHIFT;
1365
1366		if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
1367			/* The guest TLB had a mapping, but the shadow TLB
1368			 * didn't. This could be because:
1369			 * a) the entry is mapping the host kernel, or
1370			 * b) the guest used a large mapping which we're faking
1371			 * Either way, we need to satisfy the fault without
1372			 * invoking the guest. */
1373			kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
1374		} else {
1375			/* Guest mapped and leaped at non-RAM! */
1376			kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_MACHINE_CHECK);
1377		}
1378
1379		srcu_read_unlock(&vcpu->kvm->srcu, idx);
1380		break;
1381	}
1382
1383	case BOOKE_INTERRUPT_DEBUG: {
1384		r = kvmppc_handle_debug(vcpu);
1385		if (r == RESUME_HOST)
1386			run->exit_reason = KVM_EXIT_DEBUG;
1387		kvmppc_account_exit(vcpu, DEBUG_EXITS);
1388		break;
1389	}
1390
1391	default:
1392		printk(KERN_EMERG "exit_nr %d\n", exit_nr);
1393		BUG();
1394	}
1395
1396out:
1397	/*
1398	 * To avoid clobbering exit_reason, only check for signals if we
1399	 * aren't already exiting to userspace for some other reason.
1400	 */
1401	if (!(r & RESUME_HOST)) {
1402		s = kvmppc_prepare_to_enter(vcpu);
1403		if (s <= 0)
1404			r = (s << 2) | RESUME_HOST | (r & RESUME_FLAG_NV);
1405		else {
1406			/* interrupts now hard-disabled */
1407			kvmppc_fix_ee_before_entry();
1408			kvmppc_load_guest_fp(vcpu);
1409			kvmppc_load_guest_altivec(vcpu);
1410		}
1411	}
1412
1413	return r;
1414}
1415
1416static void kvmppc_set_tsr(struct kvm_vcpu *vcpu, u32 new_tsr)
1417{
1418	u32 old_tsr = vcpu->arch.tsr;
1419
1420	vcpu->arch.tsr = new_tsr;
1421
1422	if ((old_tsr ^ vcpu->arch.tsr) & (TSR_ENW | TSR_WIS))
1423		arm_next_watchdog(vcpu);
1424
1425	update_timer_ints(vcpu);
1426}
1427
1428int kvmppc_subarch_vcpu_init(struct kvm_vcpu *vcpu)
1429{
1430	/* setup watchdog timer once */
1431	spin_lock_init(&vcpu->arch.wdt_lock);
1432	timer_setup(&vcpu->arch.wdt_timer, kvmppc_watchdog_func, 0);
1433
1434	/*
1435	 * Clear DBSR.MRR to avoid guest debug interrupt as
1436	 * this is of host interest
1437	 */
1438	mtspr(SPRN_DBSR, DBSR_MRR);
1439	return 0;
1440}
1441
1442void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu *vcpu)
1443{
1444	del_timer_sync(&vcpu->arch.wdt_timer);
1445}
1446
1447int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1448{
1449	int i;
1450
1451	vcpu_load(vcpu);
1452
1453	regs->pc = vcpu->arch.regs.nip;
1454	regs->cr = kvmppc_get_cr(vcpu);
1455	regs->ctr = vcpu->arch.regs.ctr;
1456	regs->lr = vcpu->arch.regs.link;
1457	regs->xer = kvmppc_get_xer(vcpu);
1458	regs->msr = vcpu->arch.shared->msr;
1459	regs->srr0 = kvmppc_get_srr0(vcpu);
1460	regs->srr1 = kvmppc_get_srr1(vcpu);
1461	regs->pid = vcpu->arch.pid;
1462	regs->sprg0 = kvmppc_get_sprg0(vcpu);
1463	regs->sprg1 = kvmppc_get_sprg1(vcpu);
1464	regs->sprg2 = kvmppc_get_sprg2(vcpu);
1465	regs->sprg3 = kvmppc_get_sprg3(vcpu);
1466	regs->sprg4 = kvmppc_get_sprg4(vcpu);
1467	regs->sprg5 = kvmppc_get_sprg5(vcpu);
1468	regs->sprg6 = kvmppc_get_sprg6(vcpu);
1469	regs->sprg7 = kvmppc_get_sprg7(vcpu);
1470
1471	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
1472		regs->gpr[i] = kvmppc_get_gpr(vcpu, i);
1473
1474	vcpu_put(vcpu);
1475	return 0;
1476}
1477
1478int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1479{
1480	int i;
1481
1482	vcpu_load(vcpu);
1483
1484	vcpu->arch.regs.nip = regs->pc;
1485	kvmppc_set_cr(vcpu, regs->cr);
1486	vcpu->arch.regs.ctr = regs->ctr;
1487	vcpu->arch.regs.link = regs->lr;
1488	kvmppc_set_xer(vcpu, regs->xer);
1489	kvmppc_set_msr(vcpu, regs->msr);
1490	kvmppc_set_srr0(vcpu, regs->srr0);
1491	kvmppc_set_srr1(vcpu, regs->srr1);
1492	kvmppc_set_pid(vcpu, regs->pid);
1493	kvmppc_set_sprg0(vcpu, regs->sprg0);
1494	kvmppc_set_sprg1(vcpu, regs->sprg1);
1495	kvmppc_set_sprg2(vcpu, regs->sprg2);
1496	kvmppc_set_sprg3(vcpu, regs->sprg3);
1497	kvmppc_set_sprg4(vcpu, regs->sprg4);
1498	kvmppc_set_sprg5(vcpu, regs->sprg5);
1499	kvmppc_set_sprg6(vcpu, regs->sprg6);
1500	kvmppc_set_sprg7(vcpu, regs->sprg7);
1501
1502	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
1503		kvmppc_set_gpr(vcpu, i, regs->gpr[i]);
1504
1505	vcpu_put(vcpu);
1506	return 0;
1507}
1508
1509static void get_sregs_base(struct kvm_vcpu *vcpu,
1510                           struct kvm_sregs *sregs)
1511{
1512	u64 tb = get_tb();
1513
1514	sregs->u.e.features |= KVM_SREGS_E_BASE;
1515
1516	sregs->u.e.csrr0 = vcpu->arch.csrr0;
1517	sregs->u.e.csrr1 = vcpu->arch.csrr1;
1518	sregs->u.e.mcsr = vcpu->arch.mcsr;
1519	sregs->u.e.esr = kvmppc_get_esr(vcpu);
1520	sregs->u.e.dear = kvmppc_get_dar(vcpu);
1521	sregs->u.e.tsr = vcpu->arch.tsr;
1522	sregs->u.e.tcr = vcpu->arch.tcr;
1523	sregs->u.e.dec = kvmppc_get_dec(vcpu, tb);
1524	sregs->u.e.tb = tb;
1525	sregs->u.e.vrsave = vcpu->arch.vrsave;
1526}
1527
1528static int set_sregs_base(struct kvm_vcpu *vcpu,
1529                          struct kvm_sregs *sregs)
1530{
1531	if (!(sregs->u.e.features & KVM_SREGS_E_BASE))
1532		return 0;
1533
1534	vcpu->arch.csrr0 = sregs->u.e.csrr0;
1535	vcpu->arch.csrr1 = sregs->u.e.csrr1;
1536	vcpu->arch.mcsr = sregs->u.e.mcsr;
1537	kvmppc_set_esr(vcpu, sregs->u.e.esr);
1538	kvmppc_set_dar(vcpu, sregs->u.e.dear);
1539	vcpu->arch.vrsave = sregs->u.e.vrsave;
1540	kvmppc_set_tcr(vcpu, sregs->u.e.tcr);
1541
1542	if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_DEC) {
1543		vcpu->arch.dec = sregs->u.e.dec;
1544		kvmppc_emulate_dec(vcpu);
1545	}
1546
1547	if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_TSR)
1548		kvmppc_set_tsr(vcpu, sregs->u.e.tsr);
1549
1550	return 0;
1551}
1552
1553static void get_sregs_arch206(struct kvm_vcpu *vcpu,
1554                              struct kvm_sregs *sregs)
1555{
1556	sregs->u.e.features |= KVM_SREGS_E_ARCH206;
1557
1558	sregs->u.e.pir = vcpu->vcpu_id;
1559	sregs->u.e.mcsrr0 = vcpu->arch.mcsrr0;
1560	sregs->u.e.mcsrr1 = vcpu->arch.mcsrr1;
1561	sregs->u.e.decar = vcpu->arch.decar;
1562	sregs->u.e.ivpr = vcpu->arch.ivpr;
1563}
1564
1565static int set_sregs_arch206(struct kvm_vcpu *vcpu,
1566                             struct kvm_sregs *sregs)
1567{
1568	if (!(sregs->u.e.features & KVM_SREGS_E_ARCH206))
1569		return 0;
1570
1571	if (sregs->u.e.pir != vcpu->vcpu_id)
1572		return -EINVAL;
1573
1574	vcpu->arch.mcsrr0 = sregs->u.e.mcsrr0;
1575	vcpu->arch.mcsrr1 = sregs->u.e.mcsrr1;
1576	vcpu->arch.decar = sregs->u.e.decar;
1577	vcpu->arch.ivpr = sregs->u.e.ivpr;
1578
1579	return 0;
1580}
1581
1582int kvmppc_get_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
1583{
1584	sregs->u.e.features |= KVM_SREGS_E_IVOR;
1585
1586	sregs->u.e.ivor_low[0] = vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL];
1587	sregs->u.e.ivor_low[1] = vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK];
1588	sregs->u.e.ivor_low[2] = vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE];
1589	sregs->u.e.ivor_low[3] = vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE];
1590	sregs->u.e.ivor_low[4] = vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL];
1591	sregs->u.e.ivor_low[5] = vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT];
1592	sregs->u.e.ivor_low[6] = vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM];
1593	sregs->u.e.ivor_low[7] = vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL];
1594	sregs->u.e.ivor_low[8] = vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL];
1595	sregs->u.e.ivor_low[9] = vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL];
1596	sregs->u.e.ivor_low[10] = vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER];
1597	sregs->u.e.ivor_low[11] = vcpu->arch.ivor[BOOKE_IRQPRIO_FIT];
1598	sregs->u.e.ivor_low[12] = vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG];
1599	sregs->u.e.ivor_low[13] = vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS];
1600	sregs->u.e.ivor_low[14] = vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS];
1601	sregs->u.e.ivor_low[15] = vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG];
1602	return 0;
1603}
1604
1605int kvmppc_set_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
1606{
1607	if (!(sregs->u.e.features & KVM_SREGS_E_IVOR))
1608		return 0;
1609
1610	vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL] = sregs->u.e.ivor_low[0];
1611	vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK] = sregs->u.e.ivor_low[1];
1612	vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE] = sregs->u.e.ivor_low[2];
1613	vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE] = sregs->u.e.ivor_low[3];
1614	vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL] = sregs->u.e.ivor_low[4];
1615	vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT] = sregs->u.e.ivor_low[5];
1616	vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM] = sregs->u.e.ivor_low[6];
1617	vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL] = sregs->u.e.ivor_low[7];
1618	vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL] = sregs->u.e.ivor_low[8];
1619	vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL] = sregs->u.e.ivor_low[9];
1620	vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER] = sregs->u.e.ivor_low[10];
1621	vcpu->arch.ivor[BOOKE_IRQPRIO_FIT] = sregs->u.e.ivor_low[11];
1622	vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG] = sregs->u.e.ivor_low[12];
1623	vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS] = sregs->u.e.ivor_low[13];
1624	vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS] = sregs->u.e.ivor_low[14];
1625	vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG] = sregs->u.e.ivor_low[15];
1626
1627	return 0;
1628}
1629
1630int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1631                                  struct kvm_sregs *sregs)
1632{
1633	int ret;
1634
1635	vcpu_load(vcpu);
1636
1637	sregs->pvr = vcpu->arch.pvr;
1638
1639	get_sregs_base(vcpu, sregs);
1640	get_sregs_arch206(vcpu, sregs);
1641	ret = vcpu->kvm->arch.kvm_ops->get_sregs(vcpu, sregs);
1642
1643	vcpu_put(vcpu);
1644	return ret;
1645}
1646
1647int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1648                                  struct kvm_sregs *sregs)
1649{
1650	int ret = -EINVAL;
1651
1652	vcpu_load(vcpu);
1653	if (vcpu->arch.pvr != sregs->pvr)
1654		goto out;
1655
1656	ret = set_sregs_base(vcpu, sregs);
1657	if (ret < 0)
1658		goto out;
1659
1660	ret = set_sregs_arch206(vcpu, sregs);
1661	if (ret < 0)
1662		goto out;
1663
1664	ret = vcpu->kvm->arch.kvm_ops->set_sregs(vcpu, sregs);
1665
1666out:
1667	vcpu_put(vcpu);
1668	return ret;
1669}
1670
1671int kvmppc_get_one_reg(struct kvm_vcpu *vcpu, u64 id,
1672			union kvmppc_one_reg *val)
1673{
1674	int r = 0;
1675
1676	switch (id) {
1677	case KVM_REG_PPC_IAC1:
1678		*val = get_reg_val(id, vcpu->arch.dbg_reg.iac1);
1679		break;
1680	case KVM_REG_PPC_IAC2:
1681		*val = get_reg_val(id, vcpu->arch.dbg_reg.iac2);
1682		break;
1683#if CONFIG_PPC_ADV_DEBUG_IACS > 2
1684	case KVM_REG_PPC_IAC3:
1685		*val = get_reg_val(id, vcpu->arch.dbg_reg.iac3);
1686		break;
1687	case KVM_REG_PPC_IAC4:
1688		*val = get_reg_val(id, vcpu->arch.dbg_reg.iac4);
1689		break;
1690#endif
1691	case KVM_REG_PPC_DAC1:
1692		*val = get_reg_val(id, vcpu->arch.dbg_reg.dac1);
1693		break;
1694	case KVM_REG_PPC_DAC2:
1695		*val = get_reg_val(id, vcpu->arch.dbg_reg.dac2);
1696		break;
1697	case KVM_REG_PPC_EPR: {
1698		u32 epr = kvmppc_get_epr(vcpu);
1699		*val = get_reg_val(id, epr);
1700		break;
1701	}
1702#if defined(CONFIG_64BIT)
1703	case KVM_REG_PPC_EPCR:
1704		*val = get_reg_val(id, vcpu->arch.epcr);
1705		break;
1706#endif
1707	case KVM_REG_PPC_TCR:
1708		*val = get_reg_val(id, vcpu->arch.tcr);
1709		break;
1710	case KVM_REG_PPC_TSR:
1711		*val = get_reg_val(id, vcpu->arch.tsr);
1712		break;
1713	case KVM_REG_PPC_DEBUG_INST:
1714		*val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
1715		break;
1716	case KVM_REG_PPC_VRSAVE:
1717		*val = get_reg_val(id, vcpu->arch.vrsave);
1718		break;
1719	default:
1720		r = vcpu->kvm->arch.kvm_ops->get_one_reg(vcpu, id, val);
1721		break;
1722	}
1723
1724	return r;
1725}
1726
1727int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id,
1728			union kvmppc_one_reg *val)
1729{
1730	int r = 0;
1731
1732	switch (id) {
1733	case KVM_REG_PPC_IAC1:
1734		vcpu->arch.dbg_reg.iac1 = set_reg_val(id, *val);
1735		break;
1736	case KVM_REG_PPC_IAC2:
1737		vcpu->arch.dbg_reg.iac2 = set_reg_val(id, *val);
1738		break;
1739#if CONFIG_PPC_ADV_DEBUG_IACS > 2
1740	case KVM_REG_PPC_IAC3:
1741		vcpu->arch.dbg_reg.iac3 = set_reg_val(id, *val);
1742		break;
1743	case KVM_REG_PPC_IAC4:
1744		vcpu->arch.dbg_reg.iac4 = set_reg_val(id, *val);
1745		break;
1746#endif
1747	case KVM_REG_PPC_DAC1:
1748		vcpu->arch.dbg_reg.dac1 = set_reg_val(id, *val);
1749		break;
1750	case KVM_REG_PPC_DAC2:
1751		vcpu->arch.dbg_reg.dac2 = set_reg_val(id, *val);
1752		break;
1753	case KVM_REG_PPC_EPR: {
1754		u32 new_epr = set_reg_val(id, *val);
1755		kvmppc_set_epr(vcpu, new_epr);
1756		break;
1757	}
1758#if defined(CONFIG_64BIT)
1759	case KVM_REG_PPC_EPCR: {
1760		u32 new_epcr = set_reg_val(id, *val);
1761		kvmppc_set_epcr(vcpu, new_epcr);
1762		break;
1763	}
1764#endif
1765	case KVM_REG_PPC_OR_TSR: {
1766		u32 tsr_bits = set_reg_val(id, *val);
1767		kvmppc_set_tsr_bits(vcpu, tsr_bits);
1768		break;
1769	}
1770	case KVM_REG_PPC_CLEAR_TSR: {
1771		u32 tsr_bits = set_reg_val(id, *val);
1772		kvmppc_clr_tsr_bits(vcpu, tsr_bits);
1773		break;
1774	}
1775	case KVM_REG_PPC_TSR: {
1776		u32 tsr = set_reg_val(id, *val);
1777		kvmppc_set_tsr(vcpu, tsr);
1778		break;
1779	}
1780	case KVM_REG_PPC_TCR: {
1781		u32 tcr = set_reg_val(id, *val);
1782		kvmppc_set_tcr(vcpu, tcr);
1783		break;
1784	}
1785	case KVM_REG_PPC_VRSAVE:
1786		vcpu->arch.vrsave = set_reg_val(id, *val);
1787		break;
1788	default:
1789		r = vcpu->kvm->arch.kvm_ops->set_one_reg(vcpu, id, val);
1790		break;
1791	}
1792
1793	return r;
1794}
1795
1796int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1797{
1798	return -EOPNOTSUPP;
1799}
1800
1801int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1802{
1803	return -EOPNOTSUPP;
1804}
1805
1806int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1807                                  struct kvm_translation *tr)
1808{
1809	int r;
1810
1811	vcpu_load(vcpu);
1812	r = kvmppc_core_vcpu_translate(vcpu, tr);
1813	vcpu_put(vcpu);
1814	return r;
1815}
1816
1817void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1818{
1819
1820}
1821
1822int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
1823{
1824	return -EOPNOTSUPP;
1825}
1826
1827void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
1828{
1829}
1830
1831int kvmppc_core_prepare_memory_region(struct kvm *kvm,
1832				      const struct kvm_memory_slot *old,
1833				      struct kvm_memory_slot *new,
1834				      enum kvm_mr_change change)
1835{
1836	return 0;
1837}
1838
1839void kvmppc_core_commit_memory_region(struct kvm *kvm,
1840				struct kvm_memory_slot *old,
1841				const struct kvm_memory_slot *new,
1842				enum kvm_mr_change change)
1843{
1844}
1845
1846void kvmppc_core_flush_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot)
1847{
1848}
1849
1850void kvmppc_set_epcr(struct kvm_vcpu *vcpu, u32 new_epcr)
1851{
1852#if defined(CONFIG_64BIT)
1853	vcpu->arch.epcr = new_epcr;
1854#ifdef CONFIG_KVM_BOOKE_HV
1855	vcpu->arch.shadow_epcr &= ~SPRN_EPCR_GICM;
1856	if (vcpu->arch.epcr  & SPRN_EPCR_ICM)
1857		vcpu->arch.shadow_epcr |= SPRN_EPCR_GICM;
1858#endif
1859#endif
1860}
1861
1862void kvmppc_set_tcr(struct kvm_vcpu *vcpu, u32 new_tcr)
1863{
1864	vcpu->arch.tcr = new_tcr;
1865	arm_next_watchdog(vcpu);
1866	update_timer_ints(vcpu);
1867}
1868
1869void kvmppc_set_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits)
1870{
1871	set_bits(tsr_bits, &vcpu->arch.tsr);
1872	smp_wmb();
1873	kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
1874	kvm_vcpu_kick(vcpu);
1875}
1876
1877void kvmppc_clr_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits)
1878{
1879	clear_bits(tsr_bits, &vcpu->arch.tsr);
1880
1881	/*
1882	 * We may have stopped the watchdog due to
1883	 * being stuck on final expiration.
1884	 */
1885	if (tsr_bits & (TSR_ENW | TSR_WIS))
1886		arm_next_watchdog(vcpu);
1887
1888	update_timer_ints(vcpu);
1889}
1890
1891void kvmppc_decrementer_func(struct kvm_vcpu *vcpu)
1892{
1893	if (vcpu->arch.tcr & TCR_ARE) {
1894		vcpu->arch.dec = vcpu->arch.decar;
1895		kvmppc_emulate_dec(vcpu);
1896	}
1897
1898	kvmppc_set_tsr_bits(vcpu, TSR_DIS);
1899}
1900
1901static int kvmppc_booke_add_breakpoint(struct debug_reg *dbg_reg,
1902				       uint64_t addr, int index)
1903{
1904	switch (index) {
1905	case 0:
1906		dbg_reg->dbcr0 |= DBCR0_IAC1;
1907		dbg_reg->iac1 = addr;
1908		break;
1909	case 1:
1910		dbg_reg->dbcr0 |= DBCR0_IAC2;
1911		dbg_reg->iac2 = addr;
1912		break;
1913#if CONFIG_PPC_ADV_DEBUG_IACS > 2
1914	case 2:
1915		dbg_reg->dbcr0 |= DBCR0_IAC3;
1916		dbg_reg->iac3 = addr;
1917		break;
1918	case 3:
1919		dbg_reg->dbcr0 |= DBCR0_IAC4;
1920		dbg_reg->iac4 = addr;
1921		break;
1922#endif
1923	default:
1924		return -EINVAL;
1925	}
1926
1927	dbg_reg->dbcr0 |= DBCR0_IDM;
1928	return 0;
1929}
1930
1931static int kvmppc_booke_add_watchpoint(struct debug_reg *dbg_reg, uint64_t addr,
1932				       int type, int index)
1933{
1934	switch (index) {
1935	case 0:
1936		if (type & KVMPPC_DEBUG_WATCH_READ)
1937			dbg_reg->dbcr0 |= DBCR0_DAC1R;
1938		if (type & KVMPPC_DEBUG_WATCH_WRITE)
1939			dbg_reg->dbcr0 |= DBCR0_DAC1W;
1940		dbg_reg->dac1 = addr;
1941		break;
1942	case 1:
1943		if (type & KVMPPC_DEBUG_WATCH_READ)
1944			dbg_reg->dbcr0 |= DBCR0_DAC2R;
1945		if (type & KVMPPC_DEBUG_WATCH_WRITE)
1946			dbg_reg->dbcr0 |= DBCR0_DAC2W;
1947		dbg_reg->dac2 = addr;
1948		break;
1949	default:
1950		return -EINVAL;
1951	}
1952
1953	dbg_reg->dbcr0 |= DBCR0_IDM;
1954	return 0;
1955}
1956static void kvm_guest_protect_msr(struct kvm_vcpu *vcpu, ulong prot_bitmap,
1957				  bool set)
1958{
1959	/* XXX: Add similar MSR protection for BookE-PR */
1960#ifdef CONFIG_KVM_BOOKE_HV
1961	BUG_ON(prot_bitmap & ~(MSRP_UCLEP | MSRP_DEP | MSRP_PMMP));
1962	if (set) {
1963		if (prot_bitmap & MSR_UCLE)
1964			vcpu->arch.shadow_msrp |= MSRP_UCLEP;
1965		if (prot_bitmap & MSR_DE)
1966			vcpu->arch.shadow_msrp |= MSRP_DEP;
1967		if (prot_bitmap & MSR_PMM)
1968			vcpu->arch.shadow_msrp |= MSRP_PMMP;
1969	} else {
1970		if (prot_bitmap & MSR_UCLE)
1971			vcpu->arch.shadow_msrp &= ~MSRP_UCLEP;
1972		if (prot_bitmap & MSR_DE)
1973			vcpu->arch.shadow_msrp &= ~MSRP_DEP;
1974		if (prot_bitmap & MSR_PMM)
1975			vcpu->arch.shadow_msrp &= ~MSRP_PMMP;
1976	}
1977#endif
1978}
1979
1980int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, enum xlate_instdata xlid,
1981		 enum xlate_readwrite xlrw, struct kvmppc_pte *pte)
1982{
1983	int gtlb_index;
1984	gpa_t gpaddr;
1985
1986#ifdef CONFIG_KVM_E500V2
1987	if (!(vcpu->arch.shared->msr & MSR_PR) &&
1988	    (eaddr & PAGE_MASK) == vcpu->arch.magic_page_ea) {
1989		pte->eaddr = eaddr;
1990		pte->raddr = (vcpu->arch.magic_page_pa & PAGE_MASK) |
1991			     (eaddr & ~PAGE_MASK);
1992		pte->vpage = eaddr >> PAGE_SHIFT;
1993		pte->may_read = true;
1994		pte->may_write = true;
1995		pte->may_execute = true;
1996
1997		return 0;
1998	}
1999#endif
2000
2001	/* Check the guest TLB. */
2002	switch (xlid) {
2003	case XLATE_INST:
2004		gtlb_index = kvmppc_mmu_itlb_index(vcpu, eaddr);
2005		break;
2006	case XLATE_DATA:
2007		gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
2008		break;
2009	default:
2010		BUG();
2011	}
2012
2013	/* Do we have a TLB entry at all? */
2014	if (gtlb_index < 0)
2015		return -ENOENT;
2016
2017	gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
2018
2019	pte->eaddr = eaddr;
2020	pte->raddr = (gpaddr & PAGE_MASK) | (eaddr & ~PAGE_MASK);
2021	pte->vpage = eaddr >> PAGE_SHIFT;
2022
2023	/* XXX read permissions from the guest TLB */
2024	pte->may_read = true;
2025	pte->may_write = true;
2026	pte->may_execute = true;
2027
2028	return 0;
2029}
2030
2031int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2032					 struct kvm_guest_debug *dbg)
2033{
2034	struct debug_reg *dbg_reg;
2035	int n, b = 0, w = 0;
2036	int ret = 0;
2037
2038	vcpu_load(vcpu);
2039
2040	if (!(dbg->control & KVM_GUESTDBG_ENABLE)) {
2041		vcpu->arch.dbg_reg.dbcr0 = 0;
2042		vcpu->guest_debug = 0;
2043		kvm_guest_protect_msr(vcpu, MSR_DE, false);
2044		goto out;
2045	}
2046
2047	kvm_guest_protect_msr(vcpu, MSR_DE, true);
2048	vcpu->guest_debug = dbg->control;
2049	vcpu->arch.dbg_reg.dbcr0 = 0;
2050
2051	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
2052		vcpu->arch.dbg_reg.dbcr0 |= DBCR0_IDM | DBCR0_IC;
2053
2054	/* Code below handles only HW breakpoints */
2055	dbg_reg = &(vcpu->arch.dbg_reg);
2056
2057#ifdef CONFIG_KVM_BOOKE_HV
2058	/*
2059	 * On BookE-HV (e500mc) the guest is always executed with MSR.GS=1
2060	 * DBCR1 and DBCR2 are set to trigger debug events when MSR.PR is 0
2061	 */
2062	dbg_reg->dbcr1 = 0;
2063	dbg_reg->dbcr2 = 0;
2064#else
2065	/*
2066	 * On BookE-PR (e500v2) the guest is always executed with MSR.PR=1
2067	 * We set DBCR1 and DBCR2 to only trigger debug events when MSR.PR
2068	 * is set.
2069	 */
2070	dbg_reg->dbcr1 = DBCR1_IAC1US | DBCR1_IAC2US | DBCR1_IAC3US |
2071			  DBCR1_IAC4US;
2072	dbg_reg->dbcr2 = DBCR2_DAC1US | DBCR2_DAC2US;
2073#endif
2074
2075	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
2076		goto out;
2077
2078	ret = -EINVAL;
2079	for (n = 0; n < (KVMPPC_BOOKE_IAC_NUM + KVMPPC_BOOKE_DAC_NUM); n++) {
2080		uint64_t addr = dbg->arch.bp[n].addr;
2081		uint32_t type = dbg->arch.bp[n].type;
2082
2083		if (type == KVMPPC_DEBUG_NONE)
2084			continue;
2085
2086		if (type & ~(KVMPPC_DEBUG_WATCH_READ |
2087			     KVMPPC_DEBUG_WATCH_WRITE |
2088			     KVMPPC_DEBUG_BREAKPOINT))
2089			goto out;
2090
2091		if (type & KVMPPC_DEBUG_BREAKPOINT) {
2092			/* Setting H/W breakpoint */
2093			if (kvmppc_booke_add_breakpoint(dbg_reg, addr, b++))
2094				goto out;
2095		} else {
2096			/* Setting H/W watchpoint */
2097			if (kvmppc_booke_add_watchpoint(dbg_reg, addr,
2098							type, w++))
2099				goto out;
2100		}
2101	}
2102
2103	ret = 0;
2104out:
2105	vcpu_put(vcpu);
2106	return ret;
2107}
2108
2109void kvmppc_booke_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2110{
2111	vcpu->cpu = smp_processor_id();
2112	current->thread.kvm_vcpu = vcpu;
2113}
2114
2115void kvmppc_booke_vcpu_put(struct kvm_vcpu *vcpu)
2116{
2117	current->thread.kvm_vcpu = NULL;
2118	vcpu->cpu = -1;
2119
2120	/* Clear pending debug event in DBSR */
2121	kvmppc_clear_dbsr();
2122}
2123
2124int kvmppc_core_init_vm(struct kvm *kvm)
2125{
2126	return kvm->arch.kvm_ops->init_vm(kvm);
2127}
2128
2129int kvmppc_core_vcpu_create(struct kvm_vcpu *vcpu)
2130{
2131	int i;
2132	int r;
2133
2134	r = vcpu->kvm->arch.kvm_ops->vcpu_create(vcpu);
2135	if (r)
2136		return r;
2137
2138	/* Initial guest state: 16MB mapping 0 -> 0, PC = 0, MSR = 0, R1 = 16MB */
2139	vcpu->arch.regs.nip = 0;
2140	vcpu->arch.shared->pir = vcpu->vcpu_id;
2141	kvmppc_set_gpr(vcpu, 1, (16<<20) - 8); /* -8 for the callee-save LR slot */
2142	kvmppc_set_msr(vcpu, 0);
2143
2144#ifndef CONFIG_KVM_BOOKE_HV
2145	vcpu->arch.shadow_msr = MSR_USER | MSR_IS | MSR_DS;
2146	vcpu->arch.shadow_pid = 1;
2147	vcpu->arch.shared->msr = 0;
2148#endif
2149
2150	/* Eye-catching numbers so we know if the guest takes an interrupt
2151	 * before it's programmed its own IVPR/IVORs. */
2152	vcpu->arch.ivpr = 0x55550000;
2153	for (i = 0; i < BOOKE_IRQPRIO_MAX; i++)
2154		vcpu->arch.ivor[i] = 0x7700 | i * 4;
2155
2156	kvmppc_init_timing_stats(vcpu);
2157
2158	r = kvmppc_core_vcpu_setup(vcpu);
2159	if (r)
2160		vcpu->kvm->arch.kvm_ops->vcpu_free(vcpu);
2161	kvmppc_sanity_check(vcpu);
2162	return r;
2163}
2164
2165void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
2166{
2167	vcpu->kvm->arch.kvm_ops->vcpu_free(vcpu);
2168}
2169
2170void kvmppc_core_destroy_vm(struct kvm *kvm)
2171{
2172	kvm->arch.kvm_ops->destroy_vm(kvm);
2173}
2174
2175void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2176{
2177	vcpu->kvm->arch.kvm_ops->vcpu_load(vcpu, cpu);
2178}
2179
2180void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
2181{
2182	vcpu->kvm->arch.kvm_ops->vcpu_put(vcpu);
2183}
2184
2185int __init kvmppc_booke_init(void)
2186{
2187#ifndef CONFIG_KVM_BOOKE_HV
2188	unsigned long ivor[16];
2189	unsigned long *handler = kvmppc_booke_handler_addr;
2190	unsigned long max_ivor = 0;
2191	unsigned long handler_len;
2192	int i;
2193
2194	/* We install our own exception handlers by hijacking IVPR. IVPR must
2195	 * be 16-bit aligned, so we need a 64KB allocation. */
2196	kvmppc_booke_handlers = __get_free_pages(GFP_KERNEL | __GFP_ZERO,
2197	                                         VCPU_SIZE_ORDER);
2198	if (!kvmppc_booke_handlers)
2199		return -ENOMEM;
2200
2201	/* XXX make sure our handlers are smaller than Linux's */
2202
2203	/* Copy our interrupt handlers to match host IVORs. That way we don't
2204	 * have to swap the IVORs on every guest/host transition. */
2205	ivor[0] = mfspr(SPRN_IVOR0);
2206	ivor[1] = mfspr(SPRN_IVOR1);
2207	ivor[2] = mfspr(SPRN_IVOR2);
2208	ivor[3] = mfspr(SPRN_IVOR3);
2209	ivor[4] = mfspr(SPRN_IVOR4);
2210	ivor[5] = mfspr(SPRN_IVOR5);
2211	ivor[6] = mfspr(SPRN_IVOR6);
2212	ivor[7] = mfspr(SPRN_IVOR7);
2213	ivor[8] = mfspr(SPRN_IVOR8);
2214	ivor[9] = mfspr(SPRN_IVOR9);
2215	ivor[10] = mfspr(SPRN_IVOR10);
2216	ivor[11] = mfspr(SPRN_IVOR11);
2217	ivor[12] = mfspr(SPRN_IVOR12);
2218	ivor[13] = mfspr(SPRN_IVOR13);
2219	ivor[14] = mfspr(SPRN_IVOR14);
2220	ivor[15] = mfspr(SPRN_IVOR15);
2221
2222	for (i = 0; i < 16; i++) {
2223		if (ivor[i] > max_ivor)
2224			max_ivor = i;
2225
2226		handler_len = handler[i + 1] - handler[i];
2227		memcpy((void *)kvmppc_booke_handlers + ivor[i],
2228		       (void *)handler[i], handler_len);
2229	}
2230
2231	handler_len = handler[max_ivor + 1] - handler[max_ivor];
2232	flush_icache_range(kvmppc_booke_handlers, kvmppc_booke_handlers +
2233			   ivor[max_ivor] + handler_len);
2234#endif /* !BOOKE_HV */
2235	return 0;
2236}
2237
2238void __exit kvmppc_booke_exit(void)
2239{
2240	free_pages(kvmppc_booke_handlers, VCPU_SIZE_ORDER);
2241	kvm_exit();
2242}