Loading...
Note: File does not exist in v3.1.
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Generation of main entry point for the guest, exception handling.
7 *
8 * Copyright (C) 2012 MIPS Technologies, Inc.
9 * Authors: Sanjay Lal <sanjayl@kymasys.com>
10 *
11 * Copyright (C) 2016 Imagination Technologies Ltd.
12 */
13
14#include <linux/kvm_host.h>
15#include <linux/log2.h>
16#include <asm/mmu_context.h>
17#include <asm/msa.h>
18#include <asm/setup.h>
19#include <asm/tlbex.h>
20#include <asm/uasm.h>
21
22/* Register names */
23#define ZERO 0
24#define AT 1
25#define V0 2
26#define V1 3
27#define A0 4
28#define A1 5
29
30#if _MIPS_SIM == _MIPS_SIM_ABI32
31#define T0 8
32#define T1 9
33#define T2 10
34#define T3 11
35#endif /* _MIPS_SIM == _MIPS_SIM_ABI32 */
36
37#if _MIPS_SIM == _MIPS_SIM_ABI64 || _MIPS_SIM == _MIPS_SIM_NABI32
38#define T0 12
39#define T1 13
40#define T2 14
41#define T3 15
42#endif /* _MIPS_SIM == _MIPS_SIM_ABI64 || _MIPS_SIM == _MIPS_SIM_NABI32 */
43
44#define S0 16
45#define S1 17
46#define T9 25
47#define K0 26
48#define K1 27
49#define GP 28
50#define SP 29
51#define RA 31
52
53/* Some CP0 registers */
54#define C0_PWBASE 5, 5
55#define C0_HWRENA 7, 0
56#define C0_BADVADDR 8, 0
57#define C0_BADINSTR 8, 1
58#define C0_BADINSTRP 8, 2
59#define C0_PGD 9, 7
60#define C0_ENTRYHI 10, 0
61#define C0_GUESTCTL1 10, 4
62#define C0_STATUS 12, 0
63#define C0_GUESTCTL0 12, 6
64#define C0_CAUSE 13, 0
65#define C0_EPC 14, 0
66#define C0_EBASE 15, 1
67#define C0_CONFIG5 16, 5
68#define C0_DDATA_LO 28, 3
69#define C0_ERROREPC 30, 0
70
71#define CALLFRAME_SIZ 32
72
73#ifdef CONFIG_64BIT
74#define ST0_KX_IF_64 ST0_KX
75#else
76#define ST0_KX_IF_64 0
77#endif
78
79static unsigned int scratch_vcpu[2] = { C0_DDATA_LO };
80static unsigned int scratch_tmp[2] = { C0_ERROREPC };
81
82enum label_id {
83 label_fpu_1 = 1,
84 label_msa_1,
85 label_return_to_host,
86 label_kernel_asid,
87 label_exit_common,
88};
89
90UASM_L_LA(_fpu_1)
91UASM_L_LA(_msa_1)
92UASM_L_LA(_return_to_host)
93UASM_L_LA(_kernel_asid)
94UASM_L_LA(_exit_common)
95
96static void *kvm_mips_build_enter_guest(void *addr);
97static void *kvm_mips_build_ret_from_exit(void *addr);
98static void *kvm_mips_build_ret_to_guest(void *addr);
99static void *kvm_mips_build_ret_to_host(void *addr);
100
101/*
102 * The version of this function in tlbex.c uses current_cpu_type(), but for KVM
103 * we assume symmetry.
104 */
105static int c0_kscratch(void)
106{
107 switch (boot_cpu_type()) {
108 case CPU_XLP:
109 case CPU_XLR:
110 return 22;
111 default:
112 return 31;
113 }
114}
115
116/**
117 * kvm_mips_entry_setup() - Perform global setup for entry code.
118 *
119 * Perform global setup for entry code, such as choosing a scratch register.
120 *
121 * Returns: 0 on success.
122 * -errno on failure.
123 */
124int kvm_mips_entry_setup(void)
125{
126 /*
127 * We prefer to use KScratchN registers if they are available over the
128 * defaults above, which may not work on all cores.
129 */
130 unsigned int kscratch_mask = cpu_data[0].kscratch_mask;
131
132 if (pgd_reg != -1)
133 kscratch_mask &= ~BIT(pgd_reg);
134
135 /* Pick a scratch register for storing VCPU */
136 if (kscratch_mask) {
137 scratch_vcpu[0] = c0_kscratch();
138 scratch_vcpu[1] = ffs(kscratch_mask) - 1;
139 kscratch_mask &= ~BIT(scratch_vcpu[1]);
140 }
141
142 /* Pick a scratch register to use as a temp for saving state */
143 if (kscratch_mask) {
144 scratch_tmp[0] = c0_kscratch();
145 scratch_tmp[1] = ffs(kscratch_mask) - 1;
146 kscratch_mask &= ~BIT(scratch_tmp[1]);
147 }
148
149 return 0;
150}
151
152static void kvm_mips_build_save_scratch(u32 **p, unsigned int tmp,
153 unsigned int frame)
154{
155 /* Save the VCPU scratch register value in cp0_epc of the stack frame */
156 UASM_i_MFC0(p, tmp, scratch_vcpu[0], scratch_vcpu[1]);
157 UASM_i_SW(p, tmp, offsetof(struct pt_regs, cp0_epc), frame);
158
159 /* Save the temp scratch register value in cp0_cause of stack frame */
160 if (scratch_tmp[0] == c0_kscratch()) {
161 UASM_i_MFC0(p, tmp, scratch_tmp[0], scratch_tmp[1]);
162 UASM_i_SW(p, tmp, offsetof(struct pt_regs, cp0_cause), frame);
163 }
164}
165
166static void kvm_mips_build_restore_scratch(u32 **p, unsigned int tmp,
167 unsigned int frame)
168{
169 /*
170 * Restore host scratch register values saved by
171 * kvm_mips_build_save_scratch().
172 */
173 UASM_i_LW(p, tmp, offsetof(struct pt_regs, cp0_epc), frame);
174 UASM_i_MTC0(p, tmp, scratch_vcpu[0], scratch_vcpu[1]);
175
176 if (scratch_tmp[0] == c0_kscratch()) {
177 UASM_i_LW(p, tmp, offsetof(struct pt_regs, cp0_cause), frame);
178 UASM_i_MTC0(p, tmp, scratch_tmp[0], scratch_tmp[1]);
179 }
180}
181
182/**
183 * build_set_exc_base() - Assemble code to write exception base address.
184 * @p: Code buffer pointer.
185 * @reg: Source register (generated code may set WG bit in @reg).
186 *
187 * Assemble code to modify the exception base address in the EBase register,
188 * using the appropriately sized access and setting the WG bit if necessary.
189 */
190static inline void build_set_exc_base(u32 **p, unsigned int reg)
191{
192 if (cpu_has_ebase_wg) {
193 /* Set WG so that all the bits get written */
194 uasm_i_ori(p, reg, reg, MIPS_EBASE_WG);
195 UASM_i_MTC0(p, reg, C0_EBASE);
196 } else {
197 uasm_i_mtc0(p, reg, C0_EBASE);
198 }
199}
200
201/**
202 * kvm_mips_build_vcpu_run() - Assemble function to start running a guest VCPU.
203 * @addr: Address to start writing code.
204 *
205 * Assemble the start of the vcpu_run function to run a guest VCPU. The function
206 * conforms to the following prototype:
207 *
208 * int vcpu_run(struct kvm_vcpu *vcpu);
209 *
210 * The exit from the guest and return to the caller is handled by the code
211 * generated by kvm_mips_build_ret_to_host().
212 *
213 * Returns: Next address after end of written function.
214 */
215void *kvm_mips_build_vcpu_run(void *addr)
216{
217 u32 *p = addr;
218 unsigned int i;
219
220 /*
221 * A0: vcpu
222 */
223
224 /* k0/k1 not being used in host kernel context */
225 UASM_i_ADDIU(&p, K1, SP, -(int)sizeof(struct pt_regs));
226 for (i = 16; i < 32; ++i) {
227 if (i == 24)
228 i = 28;
229 UASM_i_SW(&p, i, offsetof(struct pt_regs, regs[i]), K1);
230 }
231
232 /* Save host status */
233 uasm_i_mfc0(&p, V0, C0_STATUS);
234 UASM_i_SW(&p, V0, offsetof(struct pt_regs, cp0_status), K1);
235
236 /* Save scratch registers, will be used to store pointer to vcpu etc */
237 kvm_mips_build_save_scratch(&p, V1, K1);
238
239 /* VCPU scratch register has pointer to vcpu */
240 UASM_i_MTC0(&p, A0, scratch_vcpu[0], scratch_vcpu[1]);
241
242 /* Offset into vcpu->arch */
243 UASM_i_ADDIU(&p, K1, A0, offsetof(struct kvm_vcpu, arch));
244
245 /*
246 * Save the host stack to VCPU, used for exception processing
247 * when we exit from the Guest
248 */
249 UASM_i_SW(&p, SP, offsetof(struct kvm_vcpu_arch, host_stack), K1);
250
251 /* Save the kernel gp as well */
252 UASM_i_SW(&p, GP, offsetof(struct kvm_vcpu_arch, host_gp), K1);
253
254 /*
255 * Setup status register for running the guest in UM, interrupts
256 * are disabled
257 */
258 UASM_i_LA(&p, K0, ST0_EXL | KSU_USER | ST0_BEV | ST0_KX_IF_64);
259 uasm_i_mtc0(&p, K0, C0_STATUS);
260 uasm_i_ehb(&p);
261
262 /* load up the new EBASE */
263 UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, guest_ebase), K1);
264 build_set_exc_base(&p, K0);
265
266 /*
267 * Now that the new EBASE has been loaded, unset BEV, set
268 * interrupt mask as it was but make sure that timer interrupts
269 * are enabled
270 */
271 uasm_i_addiu(&p, K0, ZERO, ST0_EXL | KSU_USER | ST0_IE | ST0_KX_IF_64);
272 uasm_i_andi(&p, V0, V0, ST0_IM);
273 uasm_i_or(&p, K0, K0, V0);
274 uasm_i_mtc0(&p, K0, C0_STATUS);
275 uasm_i_ehb(&p);
276
277 p = kvm_mips_build_enter_guest(p);
278
279 return p;
280}
281
282/**
283 * kvm_mips_build_enter_guest() - Assemble code to resume guest execution.
284 * @addr: Address to start writing code.
285 *
286 * Assemble the code to resume guest execution. This code is common between the
287 * initial entry into the guest from the host, and returning from the exit
288 * handler back to the guest.
289 *
290 * Returns: Next address after end of written function.
291 */
292static void *kvm_mips_build_enter_guest(void *addr)
293{
294 u32 *p = addr;
295 unsigned int i;
296 struct uasm_label labels[2];
297 struct uasm_reloc relocs[2];
298 struct uasm_label __maybe_unused *l = labels;
299 struct uasm_reloc __maybe_unused *r = relocs;
300
301 memset(labels, 0, sizeof(labels));
302 memset(relocs, 0, sizeof(relocs));
303
304 /* Set Guest EPC */
305 UASM_i_LW(&p, T0, offsetof(struct kvm_vcpu_arch, pc), K1);
306 UASM_i_MTC0(&p, T0, C0_EPC);
307
308 /* Save normal linux process pgd (VZ guarantees pgd_reg is set) */
309 if (cpu_has_ldpte)
310 UASM_i_MFC0(&p, K0, C0_PWBASE);
311 else
312 UASM_i_MFC0(&p, K0, c0_kscratch(), pgd_reg);
313 UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, host_pgd), K1);
314
315 /*
316 * Set up KVM GPA pgd.
317 * This does roughly the same as TLBMISS_HANDLER_SETUP_PGD():
318 * - call tlbmiss_handler_setup_pgd(mm->pgd)
319 * - write mm->pgd into CP0_PWBase
320 *
321 * We keep S0 pointing at struct kvm so we can load the ASID below.
322 */
323 UASM_i_LW(&p, S0, (int)offsetof(struct kvm_vcpu, kvm) -
324 (int)offsetof(struct kvm_vcpu, arch), K1);
325 UASM_i_LW(&p, A0, offsetof(struct kvm, arch.gpa_mm.pgd), S0);
326 UASM_i_LA(&p, T9, (unsigned long)tlbmiss_handler_setup_pgd);
327 uasm_i_jalr(&p, RA, T9);
328 /* delay slot */
329 if (cpu_has_htw)
330 UASM_i_MTC0(&p, A0, C0_PWBASE);
331 else
332 uasm_i_nop(&p);
333
334 /* Set GM bit to setup eret to VZ guest context */
335 uasm_i_addiu(&p, V1, ZERO, 1);
336 uasm_i_mfc0(&p, K0, C0_GUESTCTL0);
337 uasm_i_ins(&p, K0, V1, MIPS_GCTL0_GM_SHIFT, 1);
338 uasm_i_mtc0(&p, K0, C0_GUESTCTL0);
339
340 if (cpu_has_guestid) {
341 /*
342 * Set root mode GuestID, so that root TLB refill handler can
343 * use the correct GuestID in the root TLB.
344 */
345
346 /* Get current GuestID */
347 uasm_i_mfc0(&p, T0, C0_GUESTCTL1);
348 /* Set GuestCtl1.RID = GuestCtl1.ID */
349 uasm_i_ext(&p, T1, T0, MIPS_GCTL1_ID_SHIFT,
350 MIPS_GCTL1_ID_WIDTH);
351 uasm_i_ins(&p, T0, T1, MIPS_GCTL1_RID_SHIFT,
352 MIPS_GCTL1_RID_WIDTH);
353 uasm_i_mtc0(&p, T0, C0_GUESTCTL1);
354
355 /* GuestID handles dealiasing so we don't need to touch ASID */
356 goto skip_asid_restore;
357 }
358
359 /* Root ASID Dealias (RAD) */
360
361 /* Save host ASID */
362 UASM_i_MFC0(&p, K0, C0_ENTRYHI);
363 UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, host_entryhi),
364 K1);
365
366 /* Set the root ASID for the Guest */
367 UASM_i_ADDIU(&p, T1, S0,
368 offsetof(struct kvm, arch.gpa_mm.context.asid));
369
370 /* t1: contains the base of the ASID array, need to get the cpu id */
371 /* smp_processor_id */
372 uasm_i_lw(&p, T2, offsetof(struct thread_info, cpu), GP);
373 /* index the ASID array */
374 uasm_i_sll(&p, T2, T2, ilog2(sizeof(long)));
375 UASM_i_ADDU(&p, T3, T1, T2);
376 UASM_i_LW(&p, K0, 0, T3);
377#ifdef CONFIG_MIPS_ASID_BITS_VARIABLE
378 /*
379 * reuse ASID array offset
380 * cpuinfo_mips is a multiple of sizeof(long)
381 */
382 uasm_i_addiu(&p, T3, ZERO, sizeof(struct cpuinfo_mips)/sizeof(long));
383 uasm_i_mul(&p, T2, T2, T3);
384
385 UASM_i_LA_mostly(&p, AT, (long)&cpu_data[0].asid_mask);
386 UASM_i_ADDU(&p, AT, AT, T2);
387 UASM_i_LW(&p, T2, uasm_rel_lo((long)&cpu_data[0].asid_mask), AT);
388 uasm_i_and(&p, K0, K0, T2);
389#else
390 uasm_i_andi(&p, K0, K0, MIPS_ENTRYHI_ASID);
391#endif
392
393 /* Set up KVM VZ root ASID (!guestid) */
394 uasm_i_mtc0(&p, K0, C0_ENTRYHI);
395skip_asid_restore:
396 uasm_i_ehb(&p);
397
398 /* Disable RDHWR access */
399 uasm_i_mtc0(&p, ZERO, C0_HWRENA);
400
401 /* load the guest context from VCPU and return */
402 for (i = 1; i < 32; ++i) {
403 /* Guest k0/k1 loaded later */
404 if (i == K0 || i == K1)
405 continue;
406 UASM_i_LW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1);
407 }
408
409#ifndef CONFIG_CPU_MIPSR6
410 /* Restore hi/lo */
411 UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, hi), K1);
412 uasm_i_mthi(&p, K0);
413
414 UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, lo), K1);
415 uasm_i_mtlo(&p, K0);
416#endif
417
418 /* Restore the guest's k0/k1 registers */
419 UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, gprs[K0]), K1);
420 UASM_i_LW(&p, K1, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1);
421
422 /* Jump to guest */
423 uasm_i_eret(&p);
424
425 uasm_resolve_relocs(relocs, labels);
426
427 return p;
428}
429
430/**
431 * kvm_mips_build_tlb_refill_exception() - Assemble TLB refill handler.
432 * @addr: Address to start writing code.
433 * @handler: Address of common handler (within range of @addr).
434 *
435 * Assemble TLB refill exception fast path handler for guest execution.
436 *
437 * Returns: Next address after end of written function.
438 */
439void *kvm_mips_build_tlb_refill_exception(void *addr, void *handler)
440{
441 u32 *p = addr;
442 struct uasm_label labels[2];
443 struct uasm_reloc relocs[2];
444#ifndef CONFIG_CPU_LOONGSON64
445 struct uasm_label *l = labels;
446 struct uasm_reloc *r = relocs;
447#endif
448
449 memset(labels, 0, sizeof(labels));
450 memset(relocs, 0, sizeof(relocs));
451
452 /* Save guest k1 into scratch register */
453 UASM_i_MTC0(&p, K1, scratch_tmp[0], scratch_tmp[1]);
454
455 /* Get the VCPU pointer from the VCPU scratch register */
456 UASM_i_MFC0(&p, K1, scratch_vcpu[0], scratch_vcpu[1]);
457
458 /* Save guest k0 into VCPU structure */
459 UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu, arch.gprs[K0]), K1);
460
461 /*
462 * Some of the common tlbex code uses current_cpu_type(). For KVM we
463 * assume symmetry and just disable preemption to silence the warning.
464 */
465 preempt_disable();
466
467#ifdef CONFIG_CPU_LOONGSON64
468 UASM_i_MFC0(&p, K1, C0_PGD);
469 uasm_i_lddir(&p, K0, K1, 3); /* global page dir */
470#ifndef __PAGETABLE_PMD_FOLDED
471 uasm_i_lddir(&p, K1, K0, 1); /* middle page dir */
472#endif
473 uasm_i_ldpte(&p, K1, 0); /* even */
474 uasm_i_ldpte(&p, K1, 1); /* odd */
475 uasm_i_tlbwr(&p);
476#else
477 /*
478 * Now for the actual refill bit. A lot of this can be common with the
479 * Linux TLB refill handler, however we don't need to handle so many
480 * cases. We only need to handle user mode refills, and user mode runs
481 * with 32-bit addressing.
482 *
483 * Therefore the branch to label_vmalloc generated by build_get_pmde64()
484 * that isn't resolved should never actually get taken and is harmless
485 * to leave in place for now.
486 */
487
488#ifdef CONFIG_64BIT
489 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
490#else
491 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
492#endif
493
494 /* we don't support huge pages yet */
495
496 build_get_ptep(&p, K0, K1);
497 build_update_entries(&p, K0, K1);
498 build_tlb_write_entry(&p, &l, &r, tlb_random);
499#endif
500
501 preempt_enable();
502
503 /* Get the VCPU pointer from the VCPU scratch register again */
504 UASM_i_MFC0(&p, K1, scratch_vcpu[0], scratch_vcpu[1]);
505
506 /* Restore the guest's k0/k1 registers */
507 UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu, arch.gprs[K0]), K1);
508 uasm_i_ehb(&p);
509 UASM_i_MFC0(&p, K1, scratch_tmp[0], scratch_tmp[1]);
510
511 /* Jump to guest */
512 uasm_i_eret(&p);
513
514 return p;
515}
516
517/**
518 * kvm_mips_build_exception() - Assemble first level guest exception handler.
519 * @addr: Address to start writing code.
520 * @handler: Address of common handler (within range of @addr).
521 *
522 * Assemble exception vector code for guest execution. The generated vector will
523 * branch to the common exception handler generated by kvm_mips_build_exit().
524 *
525 * Returns: Next address after end of written function.
526 */
527void *kvm_mips_build_exception(void *addr, void *handler)
528{
529 u32 *p = addr;
530 struct uasm_label labels[2];
531 struct uasm_reloc relocs[2];
532 struct uasm_label *l = labels;
533 struct uasm_reloc *r = relocs;
534
535 memset(labels, 0, sizeof(labels));
536 memset(relocs, 0, sizeof(relocs));
537
538 /* Save guest k1 into scratch register */
539 UASM_i_MTC0(&p, K1, scratch_tmp[0], scratch_tmp[1]);
540
541 /* Get the VCPU pointer from the VCPU scratch register */
542 UASM_i_MFC0(&p, K1, scratch_vcpu[0], scratch_vcpu[1]);
543 UASM_i_ADDIU(&p, K1, K1, offsetof(struct kvm_vcpu, arch));
544
545 /* Save guest k0 into VCPU structure */
546 UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, gprs[K0]), K1);
547
548 /* Branch to the common handler */
549 uasm_il_b(&p, &r, label_exit_common);
550 uasm_i_nop(&p);
551
552 uasm_l_exit_common(&l, handler);
553 uasm_resolve_relocs(relocs, labels);
554
555 return p;
556}
557
558/**
559 * kvm_mips_build_exit() - Assemble common guest exit handler.
560 * @addr: Address to start writing code.
561 *
562 * Assemble the generic guest exit handling code. This is called by the
563 * exception vectors (generated by kvm_mips_build_exception()), and calls
564 * kvm_mips_handle_exit(), then either resumes the guest or returns to the host
565 * depending on the return value.
566 *
567 * Returns: Next address after end of written function.
568 */
569void *kvm_mips_build_exit(void *addr)
570{
571 u32 *p = addr;
572 unsigned int i;
573 struct uasm_label labels[3];
574 struct uasm_reloc relocs[3];
575 struct uasm_label *l = labels;
576 struct uasm_reloc *r = relocs;
577
578 memset(labels, 0, sizeof(labels));
579 memset(relocs, 0, sizeof(relocs));
580
581 /*
582 * Generic Guest exception handler. We end up here when the guest
583 * does something that causes a trap to kernel mode.
584 *
585 * Both k0/k1 registers will have already been saved (k0 into the vcpu
586 * structure, and k1 into the scratch_tmp register).
587 *
588 * The k1 register will already contain the kvm_vcpu_arch pointer.
589 */
590
591 /* Start saving Guest context to VCPU */
592 for (i = 0; i < 32; ++i) {
593 /* Guest k0/k1 saved later */
594 if (i == K0 || i == K1)
595 continue;
596 UASM_i_SW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1);
597 }
598
599#ifndef CONFIG_CPU_MIPSR6
600 /* We need to save hi/lo and restore them on the way out */
601 uasm_i_mfhi(&p, T0);
602 UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, hi), K1);
603
604 uasm_i_mflo(&p, T0);
605 UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, lo), K1);
606#endif
607
608 /* Finally save guest k1 to VCPU */
609 uasm_i_ehb(&p);
610 UASM_i_MFC0(&p, T0, scratch_tmp[0], scratch_tmp[1]);
611 UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1);
612
613 /* Now that context has been saved, we can use other registers */
614
615 /* Restore vcpu */
616 UASM_i_MFC0(&p, S0, scratch_vcpu[0], scratch_vcpu[1]);
617
618 /*
619 * Save Host level EPC, BadVaddr and Cause to VCPU, useful to process
620 * the exception
621 */
622 UASM_i_MFC0(&p, K0, C0_EPC);
623 UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, pc), K1);
624
625 UASM_i_MFC0(&p, K0, C0_BADVADDR);
626 UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_badvaddr),
627 K1);
628
629 uasm_i_mfc0(&p, K0, C0_CAUSE);
630 uasm_i_sw(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_cause), K1);
631
632 if (cpu_has_badinstr) {
633 uasm_i_mfc0(&p, K0, C0_BADINSTR);
634 uasm_i_sw(&p, K0, offsetof(struct kvm_vcpu_arch,
635 host_cp0_badinstr), K1);
636 }
637
638 if (cpu_has_badinstrp) {
639 uasm_i_mfc0(&p, K0, C0_BADINSTRP);
640 uasm_i_sw(&p, K0, offsetof(struct kvm_vcpu_arch,
641 host_cp0_badinstrp), K1);
642 }
643
644 /* Now restore the host state just enough to run the handlers */
645
646 /* Switch EBASE to the one used by Linux */
647 /* load up the host EBASE */
648 uasm_i_mfc0(&p, V0, C0_STATUS);
649
650 uasm_i_lui(&p, AT, ST0_BEV >> 16);
651 uasm_i_or(&p, K0, V0, AT);
652
653 uasm_i_mtc0(&p, K0, C0_STATUS);
654 uasm_i_ehb(&p);
655
656 UASM_i_LA_mostly(&p, K0, (long)&ebase);
657 UASM_i_LW(&p, K0, uasm_rel_lo((long)&ebase), K0);
658 build_set_exc_base(&p, K0);
659
660 if (raw_cpu_has_fpu) {
661 /*
662 * If FPU is enabled, save FCR31 and clear it so that later
663 * ctc1's don't trigger FPE for pending exceptions.
664 */
665 uasm_i_lui(&p, AT, ST0_CU1 >> 16);
666 uasm_i_and(&p, V1, V0, AT);
667 uasm_il_beqz(&p, &r, V1, label_fpu_1);
668 uasm_i_nop(&p);
669 uasm_i_cfc1(&p, T0, 31);
670 uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.fcr31),
671 K1);
672 uasm_i_ctc1(&p, ZERO, 31);
673 uasm_l_fpu_1(&l, p);
674 }
675
676 if (cpu_has_msa) {
677 /*
678 * If MSA is enabled, save MSACSR and clear it so that later
679 * instructions don't trigger MSAFPE for pending exceptions.
680 */
681 uasm_i_mfc0(&p, T0, C0_CONFIG5);
682 uasm_i_ext(&p, T0, T0, 27, 1); /* MIPS_CONF5_MSAEN */
683 uasm_il_beqz(&p, &r, T0, label_msa_1);
684 uasm_i_nop(&p);
685 uasm_i_cfcmsa(&p, T0, MSA_CSR);
686 uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.msacsr),
687 K1);
688 uasm_i_ctcmsa(&p, MSA_CSR, ZERO);
689 uasm_l_msa_1(&l, p);
690 }
691
692 /* Restore host ASID */
693 if (!cpu_has_guestid) {
694 UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, host_entryhi),
695 K1);
696 UASM_i_MTC0(&p, K0, C0_ENTRYHI);
697 }
698
699 /*
700 * Set up normal Linux process pgd.
701 * This does roughly the same as TLBMISS_HANDLER_SETUP_PGD():
702 * - call tlbmiss_handler_setup_pgd(mm->pgd)
703 * - write mm->pgd into CP0_PWBase
704 */
705 UASM_i_LW(&p, A0,
706 offsetof(struct kvm_vcpu_arch, host_pgd), K1);
707 UASM_i_LA(&p, T9, (unsigned long)tlbmiss_handler_setup_pgd);
708 uasm_i_jalr(&p, RA, T9);
709 /* delay slot */
710 if (cpu_has_htw)
711 UASM_i_MTC0(&p, A0, C0_PWBASE);
712 else
713 uasm_i_nop(&p);
714
715 /* Clear GM bit so we don't enter guest mode when EXL is cleared */
716 uasm_i_mfc0(&p, K0, C0_GUESTCTL0);
717 uasm_i_ins(&p, K0, ZERO, MIPS_GCTL0_GM_SHIFT, 1);
718 uasm_i_mtc0(&p, K0, C0_GUESTCTL0);
719
720 /* Save GuestCtl0 so we can access GExcCode after CPU migration */
721 uasm_i_sw(&p, K0,
722 offsetof(struct kvm_vcpu_arch, host_cp0_guestctl0), K1);
723
724 if (cpu_has_guestid) {
725 /*
726 * Clear root mode GuestID, so that root TLB operations use the
727 * root GuestID in the root TLB.
728 */
729 uasm_i_mfc0(&p, T0, C0_GUESTCTL1);
730 /* Set GuestCtl1.RID = MIPS_GCTL1_ROOT_GUESTID (i.e. 0) */
731 uasm_i_ins(&p, T0, ZERO, MIPS_GCTL1_RID_SHIFT,
732 MIPS_GCTL1_RID_WIDTH);
733 uasm_i_mtc0(&p, T0, C0_GUESTCTL1);
734 }
735
736 /* Now that the new EBASE has been loaded, unset BEV and KSU_USER */
737 uasm_i_addiu(&p, AT, ZERO, ~(ST0_EXL | KSU_USER | ST0_IE));
738 uasm_i_and(&p, V0, V0, AT);
739 uasm_i_lui(&p, AT, ST0_CU0 >> 16);
740 uasm_i_or(&p, V0, V0, AT);
741#ifdef CONFIG_64BIT
742 uasm_i_ori(&p, V0, V0, ST0_SX | ST0_UX);
743#endif
744 uasm_i_mtc0(&p, V0, C0_STATUS);
745 uasm_i_ehb(&p);
746
747 /* Load up host GP */
748 UASM_i_LW(&p, GP, offsetof(struct kvm_vcpu_arch, host_gp), K1);
749
750 /* Need a stack before we can jump to "C" */
751 UASM_i_LW(&p, SP, offsetof(struct kvm_vcpu_arch, host_stack), K1);
752
753 /* Saved host state */
754 UASM_i_ADDIU(&p, SP, SP, -(int)sizeof(struct pt_regs));
755
756 /*
757 * XXXKYMA do we need to load the host ASID, maybe not because the
758 * kernel entries are marked GLOBAL, need to verify
759 */
760
761 /* Restore host scratch registers, as we'll have clobbered them */
762 kvm_mips_build_restore_scratch(&p, K0, SP);
763
764 /* Restore RDHWR access */
765 UASM_i_LA_mostly(&p, K0, (long)&hwrena);
766 uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0);
767 uasm_i_mtc0(&p, K0, C0_HWRENA);
768
769 /* Jump to handler */
770 /*
771 * XXXKYMA: not sure if this is safe, how large is the stack??
772 * Now jump to the kvm_mips_handle_exit() to see if we can deal
773 * with this in the kernel
774 */
775 uasm_i_move(&p, A0, S0);
776 UASM_i_LA(&p, T9, (unsigned long)kvm_mips_handle_exit);
777 uasm_i_jalr(&p, RA, T9);
778 UASM_i_ADDIU(&p, SP, SP, -CALLFRAME_SIZ);
779
780 uasm_resolve_relocs(relocs, labels);
781
782 p = kvm_mips_build_ret_from_exit(p);
783
784 return p;
785}
786
787/**
788 * kvm_mips_build_ret_from_exit() - Assemble guest exit return handler.
789 * @addr: Address to start writing code.
790 *
791 * Assemble the code to handle the return from kvm_mips_handle_exit(), either
792 * resuming the guest or returning to the host depending on the return value.
793 *
794 * Returns: Next address after end of written function.
795 */
796static void *kvm_mips_build_ret_from_exit(void *addr)
797{
798 u32 *p = addr;
799 struct uasm_label labels[2];
800 struct uasm_reloc relocs[2];
801 struct uasm_label *l = labels;
802 struct uasm_reloc *r = relocs;
803
804 memset(labels, 0, sizeof(labels));
805 memset(relocs, 0, sizeof(relocs));
806
807 /* Return from handler Make sure interrupts are disabled */
808 uasm_i_di(&p, ZERO);
809 uasm_i_ehb(&p);
810
811 /*
812 * XXXKYMA: k0/k1 could have been blown away if we processed
813 * an exception while we were handling the exception from the
814 * guest, reload k1
815 */
816
817 uasm_i_move(&p, K1, S0);
818 UASM_i_ADDIU(&p, K1, K1, offsetof(struct kvm_vcpu, arch));
819
820 /*
821 * Check return value, should tell us if we are returning to the
822 * host (handle I/O etc)or resuming the guest
823 */
824 uasm_i_andi(&p, T0, V0, RESUME_HOST);
825 uasm_il_bnez(&p, &r, T0, label_return_to_host);
826 uasm_i_nop(&p);
827
828 p = kvm_mips_build_ret_to_guest(p);
829
830 uasm_l_return_to_host(&l, p);
831 p = kvm_mips_build_ret_to_host(p);
832
833 uasm_resolve_relocs(relocs, labels);
834
835 return p;
836}
837
838/**
839 * kvm_mips_build_ret_to_guest() - Assemble code to return to the guest.
840 * @addr: Address to start writing code.
841 *
842 * Assemble the code to handle return from the guest exit handler
843 * (kvm_mips_handle_exit()) back to the guest.
844 *
845 * Returns: Next address after end of written function.
846 */
847static void *kvm_mips_build_ret_to_guest(void *addr)
848{
849 u32 *p = addr;
850
851 /* Put the saved pointer to vcpu (s0) back into the scratch register */
852 UASM_i_MTC0(&p, S0, scratch_vcpu[0], scratch_vcpu[1]);
853
854 /* Load up the Guest EBASE to minimize the window where BEV is set */
855 UASM_i_LW(&p, T0, offsetof(struct kvm_vcpu_arch, guest_ebase), K1);
856
857 /* Switch EBASE back to the one used by KVM */
858 uasm_i_mfc0(&p, V1, C0_STATUS);
859 uasm_i_lui(&p, AT, ST0_BEV >> 16);
860 uasm_i_or(&p, K0, V1, AT);
861 uasm_i_mtc0(&p, K0, C0_STATUS);
862 uasm_i_ehb(&p);
863 build_set_exc_base(&p, T0);
864
865 /* Setup status register for running guest in UM */
866 uasm_i_ori(&p, V1, V1, ST0_EXL | KSU_USER | ST0_IE);
867 UASM_i_LA(&p, AT, ~(ST0_CU0 | ST0_MX | ST0_SX | ST0_UX));
868 uasm_i_and(&p, V1, V1, AT);
869 uasm_i_mtc0(&p, V1, C0_STATUS);
870 uasm_i_ehb(&p);
871
872 p = kvm_mips_build_enter_guest(p);
873
874 return p;
875}
876
877/**
878 * kvm_mips_build_ret_to_host() - Assemble code to return to the host.
879 * @addr: Address to start writing code.
880 *
881 * Assemble the code to handle return from the guest exit handler
882 * (kvm_mips_handle_exit()) back to the host, i.e. to the caller of the vcpu_run
883 * function generated by kvm_mips_build_vcpu_run().
884 *
885 * Returns: Next address after end of written function.
886 */
887static void *kvm_mips_build_ret_to_host(void *addr)
888{
889 u32 *p = addr;
890 unsigned int i;
891
892 /* EBASE is already pointing to Linux */
893 UASM_i_LW(&p, K1, offsetof(struct kvm_vcpu_arch, host_stack), K1);
894 UASM_i_ADDIU(&p, K1, K1, -(int)sizeof(struct pt_regs));
895
896 /*
897 * r2/v0 is the return code, shift it down by 2 (arithmetic)
898 * to recover the err code
899 */
900 uasm_i_sra(&p, K0, V0, 2);
901 uasm_i_move(&p, V0, K0);
902
903 /* Load context saved on the host stack */
904 for (i = 16; i < 31; ++i) {
905 if (i == 24)
906 i = 28;
907 UASM_i_LW(&p, i, offsetof(struct pt_regs, regs[i]), K1);
908 }
909
910 /* Restore RDHWR access */
911 UASM_i_LA_mostly(&p, K0, (long)&hwrena);
912 uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0);
913 uasm_i_mtc0(&p, K0, C0_HWRENA);
914
915 /* Restore RA, which is the address we will return to */
916 UASM_i_LW(&p, RA, offsetof(struct pt_regs, regs[RA]), K1);
917 uasm_i_jr(&p, RA);
918 uasm_i_nop(&p);
919
920 return p;
921}
922