1/*
   2 * This file is subject to the terms and conditions of the GNU General Public
   3 * License.  See the file "COPYING" in the main directory of this archive
   4 * for more details.
   5 *
   6 * Synthesize TLB refill handlers at runtime.
   7 *
   8 * Copyright (C) 2004, 2005, 2006, 2008	 Thiemo Seufer
   9 * Copyright (C) 2005, 2007, 2008, 2009	 Maciej W. Rozycki
  10 * Copyright (C) 2006  Ralf Baechle (ralf@linux-mips.org)
  11 * Copyright (C) 2008, 2009 Cavium Networks, Inc.
  12 * Copyright (C) 2011  MIPS Technologies, Inc.
  13 *
  14 * ... and the days got worse and worse and now you see
  15 * I've gone completely out of my mind.
  16 *
  17 * They're coming to take me a away haha
  18 * they're coming to take me a away hoho hihi haha
  19 * to the funny farm where code is beautiful all the time ...
  20 *
  21 * (Condolences to Napoleon XIV)
  22 */
  23
  24#include <linux/bug.h>
  25#include <linux/kernel.h>
  26#include <linux/types.h>
  27#include <linux/smp.h>
  28#include <linux/string.h>
 
  29#include <linux/cache.h>
  30
  31#include <asm/cacheflush.h>
  32#include <asm/cpu-type.h>
  33#include <asm/pgtable.h>
  34#include <asm/war.h>
  35#include <asm/uasm.h>
  36#include <asm/setup.h>
  37
  38static int mips_xpa_disabled;
  39
  40static int __init xpa_disable(char *s)
  41{
  42	mips_xpa_disabled = 1;
  43
  44	return 1;
  45}
  46
  47__setup("noxpa", xpa_disable);
  48
  49/*
  50 * TLB load/store/modify handlers.
  51 *
  52 * Only the fastpath gets synthesized at runtime, the slowpath for
  53 * do_page_fault remains normal asm.
  54 */
  55extern void tlb_do_page_fault_0(void);
  56extern void tlb_do_page_fault_1(void);
  57
  58struct work_registers {
  59	int r1;
  60	int r2;
  61	int r3;
  62};
  63
  64struct tlb_reg_save {
  65	unsigned long a;
  66	unsigned long b;
  67} ____cacheline_aligned_in_smp;
  68
  69static struct tlb_reg_save handler_reg_save[NR_CPUS];
  70
  71static inline int r45k_bvahwbug(void)
  72{
  73	/* XXX: We should probe for the presence of this bug, but we don't. */
  74	return 0;
  75}
  76
  77static inline int r4k_250MHZhwbug(void)
  78{
  79	/* XXX: We should probe for the presence of this bug, but we don't. */
  80	return 0;
  81}
  82
  83static inline int __maybe_unused bcm1250_m3_war(void)
  84{
  85	return BCM1250_M3_WAR;
  86}
  87
  88static inline int __maybe_unused r10000_llsc_war(void)
  89{
  90	return R10000_LLSC_WAR;
  91}
  92
  93static int use_bbit_insns(void)
  94{
  95	switch (current_cpu_type()) {
  96	case CPU_CAVIUM_OCTEON:
  97	case CPU_CAVIUM_OCTEON_PLUS:
  98	case CPU_CAVIUM_OCTEON2:
  99	case CPU_CAVIUM_OCTEON3:
 100		return 1;
 101	default:
 102		return 0;
 103	}
 104}
 105
 106static int use_lwx_insns(void)
 107{
 108	switch (current_cpu_type()) {
 109	case CPU_CAVIUM_OCTEON2:
 110	case CPU_CAVIUM_OCTEON3:
 111		return 1;
 112	default:
 113		return 0;
 114	}
 115}
 116#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \
 117    CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
 118static bool scratchpad_available(void)
 119{
 120	return true;
 121}
 122static int scratchpad_offset(int i)
 123{
 124	/*
 125	 * CVMSEG starts at address -32768 and extends for
 126	 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines.
 127	 */
 128	i += 1; /* Kernel use starts at the top and works down. */
 129	return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;
 130}
 131#else
 132static bool scratchpad_available(void)
 133{
 134	return false;
 135}
 136static int scratchpad_offset(int i)
 137{
 138	BUG();
 139	/* Really unreachable, but evidently some GCC want this. */
 140	return 0;
 141}
 142#endif
 143/*
 144 * Found by experiment: At least some revisions of the 4kc throw under
 145 * some circumstances a machine check exception, triggered by invalid
 146 * values in the index register.  Delaying the tlbp instruction until
 147 * after the next branch,  plus adding an additional nop in front of
 148 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
 149 * why; it's not an issue caused by the core RTL.
 150 *
 151 */
 152static int m4kc_tlbp_war(void)
 153{
 154	return (current_cpu_data.processor_id & 0xffff00) ==
 155	       (PRID_COMP_MIPS | PRID_IMP_4KC);
 156}
 157
 158/* Handle labels (which must be positive integers). */
 159enum label_id {
 160	label_second_part = 1,
 161	label_leave,
 162	label_vmalloc,
 163	label_vmalloc_done,
 164	label_tlbw_hazard_0,
 165	label_split = label_tlbw_hazard_0 + 8,
 166	label_tlbl_goaround1,
 167	label_tlbl_goaround2,
 168	label_nopage_tlbl,
 169	label_nopage_tlbs,
 170	label_nopage_tlbm,
 171	label_smp_pgtable_change,
 172	label_r3000_write_probe_fail,
 173	label_large_segbits_fault,
 174#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
 175	label_tlb_huge_update,
 176#endif
 177};
 178
 179UASM_L_LA(_second_part)
 180UASM_L_LA(_leave)
 181UASM_L_LA(_vmalloc)
 182UASM_L_LA(_vmalloc_done)
 183/* _tlbw_hazard_x is handled differently.  */
 184UASM_L_LA(_split)
 185UASM_L_LA(_tlbl_goaround1)
 186UASM_L_LA(_tlbl_goaround2)
 187UASM_L_LA(_nopage_tlbl)
 188UASM_L_LA(_nopage_tlbs)
 189UASM_L_LA(_nopage_tlbm)
 190UASM_L_LA(_smp_pgtable_change)
 191UASM_L_LA(_r3000_write_probe_fail)
 192UASM_L_LA(_large_segbits_fault)
 193#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
 194UASM_L_LA(_tlb_huge_update)
 195#endif
 196
 197static int hazard_instance;
 198
 199static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance)
 200{
 201	switch (instance) {
 202	case 0 ... 7:
 203		uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance);
 204		return;
 205	default:
 206		BUG();
 207	}
 208}
 209
 210static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance)
 211{
 212	switch (instance) {
 213	case 0 ... 7:
 214		uasm_build_label(l, *p, label_tlbw_hazard_0 + instance);
 215		break;
 216	default:
 217		BUG();
 218	}
 219}
 220
 221/*
 222 * pgtable bits are assigned dynamically depending on processor feature
 223 * and statically based on kernel configuration.  This spits out the actual
 224 * values the kernel is using.	Required to make sense from disassembled
 225 * TLB exception handlers.
 226 */
 227static void output_pgtable_bits_defines(void)
 228{
 229#define pr_define(fmt, ...)					\
 230	pr_debug("#define " fmt, ##__VA_ARGS__)
 231
 232	pr_debug("#include <asm/asm.h>\n");
 233	pr_debug("#include <asm/regdef.h>\n");
 234	pr_debug("\n");
 235
 236	pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT);
 237	pr_define("_PAGE_READ_SHIFT %d\n", _PAGE_READ_SHIFT);
 238	pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
 239	pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT);
 240	pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT);
 241#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
 242	pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
 243#endif
 244#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
 245	if (cpu_has_rixi) {
 246#ifdef _PAGE_NO_EXEC_SHIFT
 247		pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT);
 248		pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT);
 249#endif
 250	}
 251#endif
 252	pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
 253	pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
 254	pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
 255	pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
 256	pr_debug("\n");
 257}
 258
 259static inline void dump_handler(const char *symbol, const u32 *handler, int count)
 260{
 261	int i;
 262
 263	pr_debug("LEAF(%s)\n", symbol);
 264
 265	pr_debug("\t.set push\n");
 266	pr_debug("\t.set noreorder\n");
 267
 268	for (i = 0; i < count; i++)
 269		pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]);
 270
 271	pr_debug("\t.set\tpop\n");
 272
 273	pr_debug("\tEND(%s)\n", symbol);
 274}
 275
 276/* The only general purpose registers allowed in TLB handlers. */
 277#define K0		26
 278#define K1		27
 279
 280/* Some CP0 registers */
 281#define C0_INDEX	0, 0
 282#define C0_ENTRYLO0	2, 0
 283#define C0_TCBIND	2, 2
 284#define C0_ENTRYLO1	3, 0
 285#define C0_CONTEXT	4, 0
 286#define C0_PAGEMASK	5, 0
 287#define C0_BADVADDR	8, 0
 288#define C0_ENTRYHI	10, 0
 289#define C0_EPC		14, 0
 290#define C0_XCONTEXT	20, 0
 291
 292#ifdef CONFIG_64BIT
 293# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
 294#else
 295# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
 296#endif
 297
 298/* The worst case length of the handler is around 18 instructions for
 299 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
 300 * Maximum space available is 32 instructions for R3000 and 64
 301 * instructions for R4000.
 302 *
 303 * We deliberately chose a buffer size of 128, so we won't scribble
 304 * over anything important on overflow before we panic.
 305 */
 306static u32 tlb_handler[128];
 307
 308/* simply assume worst case size for labels and relocs */
 309static struct uasm_label labels[128];
 310static struct uasm_reloc relocs[128];
 311
 312static int check_for_high_segbits;
 313static bool fill_includes_sw_bits;
 
 314
 315static unsigned int kscratch_used_mask;
 316
 317static inline int __maybe_unused c0_kscratch(void)
 318{
 319	switch (current_cpu_type()) {
 320	case CPU_XLP:
 321	case CPU_XLR:
 322		return 22;
 323	default:
 324		return 31;
 325	}
 326}
 327
 328static int allocate_kscratch(void)
 329{
 330	int r;
 331	unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
 332
 333	r = ffs(a);
 334
 335	if (r == 0)
 336		return -1;
 337
 338	r--; /* make it zero based */
 339
 340	kscratch_used_mask |= (1 << r);
 341
 342	return r;
 343}
 344
 345static int scratch_reg;
 346static int pgd_reg;
 347enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
 348
 349static struct work_registers build_get_work_registers(u32 **p)
 350{
 351	struct work_registers r;
 352
 353	if (scratch_reg >= 0) {
 
 
 
 
 354		/* Save in CPU local C0_KScratch? */
 355		UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg);
 356		r.r1 = K0;
 357		r.r2 = K1;
 358		r.r3 = 1;
 359		return r;
 360	}
 361
 362	if (num_possible_cpus() > 1) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 363		/* Get smp_processor_id */
 364		UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG);
 365		UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT);
 366
 367		/* handler_reg_save index in K0 */
 368		UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save)));
 369
 370		UASM_i_LA(p, K1, (long)&handler_reg_save);
 371		UASM_i_ADDU(p, K0, K0, K1);
 372	} else {
 373		UASM_i_LA(p, K0, (long)&handler_reg_save);
 374	}
 375	/* K0 now points to save area, save $1 and $2  */
 376	UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0);
 377	UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0);
 378
 379	r.r1 = K1;
 380	r.r2 = 1;
 381	r.r3 = 2;
 382	return r;
 383}
 384
 385static void build_restore_work_registers(u32 **p)
 386{
 387	if (scratch_reg >= 0) {
 388		UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
 389		return;
 390	}
 391	/* K0 already points to save area, restore $1 and $2  */
 392	UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0);
 393	UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0);
 394}
 395
 396#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 397
 398/*
 399 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
 400 * we cannot do r3000 under these circumstances.
 401 *
 402 * Declare pgd_current here instead of including mmu_context.h to avoid type
 403 * conflicts for tlbmiss_handler_setup_pgd
 404 */
 405extern unsigned long pgd_current[];
 406
 407/*
 408 * The R3000 TLB handler is simple.
 409 */
 410static void build_r3000_tlb_refill_handler(void)
 411{
 412	long pgdc = (long)pgd_current;
 413	u32 *p;
 414
 415	memset(tlb_handler, 0, sizeof(tlb_handler));
 416	p = tlb_handler;
 417
 418	uasm_i_mfc0(&p, K0, C0_BADVADDR);
 419	uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
 420	uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
 421	uasm_i_srl(&p, K0, K0, 22); /* load delay */
 422	uasm_i_sll(&p, K0, K0, 2);
 423	uasm_i_addu(&p, K1, K1, K0);
 424	uasm_i_mfc0(&p, K0, C0_CONTEXT);
 425	uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
 426	uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
 427	uasm_i_addu(&p, K1, K1, K0);
 428	uasm_i_lw(&p, K0, 0, K1);
 429	uasm_i_nop(&p); /* load delay */
 430	uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
 431	uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
 432	uasm_i_tlbwr(&p); /* cp0 delay */
 433	uasm_i_jr(&p, K1);
 434	uasm_i_rfe(&p); /* branch delay */
 435
 436	if (p > tlb_handler + 32)
 437		panic("TLB refill handler space exceeded");
 438
 439	pr_debug("Wrote TLB refill handler (%u instructions).\n",
 440		 (unsigned int)(p - tlb_handler));
 441
 442	memcpy((void *)ebase, tlb_handler, 0x80);
 443	local_flush_icache_range(ebase, ebase + 0x80);
 444
 445	dump_handler("r3000_tlb_refill", (u32 *)ebase, 32);
 446}
 447#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 448
 449/*
 450 * The R4000 TLB handler is much more complicated. We have two
 451 * consecutive handler areas with 32 instructions space each.
 452 * Since they aren't used at the same time, we can overflow in the
 453 * other one.To keep things simple, we first assume linear space,
 454 * then we relocate it to the final handler layout as needed.
 455 */
 456static u32 final_handler[64];
 457
 458/*
 459 * Hazards
 460 *
 461 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
 462 * 2. A timing hazard exists for the TLBP instruction.
 463 *
 464 *	stalling_instruction
 465 *	TLBP
 466 *
 467 * The JTLB is being read for the TLBP throughout the stall generated by the
 468 * previous instruction. This is not really correct as the stalling instruction
 469 * can modify the address used to access the JTLB.  The failure symptom is that
 470 * the TLBP instruction will use an address created for the stalling instruction
 471 * and not the address held in C0_ENHI and thus report the wrong results.
 472 *
 473 * The software work-around is to not allow the instruction preceding the TLBP
 474 * to stall - make it an NOP or some other instruction guaranteed not to stall.
 475 *
 476 * Errata 2 will not be fixed.	This errata is also on the R5000.
 477 *
 478 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
 479 */
 480static void __maybe_unused build_tlb_probe_entry(u32 **p)
 481{
 482	switch (current_cpu_type()) {
 483	/* Found by experiment: R4600 v2.0/R4700 needs this, too.  */
 484	case CPU_R4600:
 485	case CPU_R4700:
 486	case CPU_R5000:
 
 487	case CPU_NEVADA:
 488		uasm_i_nop(p);
 489		uasm_i_tlbp(p);
 490		break;
 491
 492	default:
 493		uasm_i_tlbp(p);
 494		break;
 495	}
 496}
 497
 498/*
 499 * Write random or indexed TLB entry, and care about the hazards from
 500 * the preceding mtc0 and for the following eret.
 501 */
 502enum tlb_write_entry { tlb_random, tlb_indexed };
 503
 504static void build_tlb_write_entry(u32 **p, struct uasm_label **l,
 505				  struct uasm_reloc **r,
 506				  enum tlb_write_entry wmode)
 507{
 508	void(*tlbw)(u32 **) = NULL;
 509
 510	switch (wmode) {
 511	case tlb_random: tlbw = uasm_i_tlbwr; break;
 512	case tlb_indexed: tlbw = uasm_i_tlbwi; break;
 513	}
 514
 515	if (cpu_has_mips_r2_r6) {
 516		if (cpu_has_mips_r2_exec_hazard)
 517			uasm_i_ehb(p);
 518		tlbw(p);
 519		return;
 520	}
 521
 522	switch (current_cpu_type()) {
 523	case CPU_R4000PC:
 524	case CPU_R4000SC:
 525	case CPU_R4000MC:
 526	case CPU_R4400PC:
 527	case CPU_R4400SC:
 528	case CPU_R4400MC:
 529		/*
 530		 * This branch uses up a mtc0 hazard nop slot and saves
 531		 * two nops after the tlbw instruction.
 532		 */
 533		uasm_bgezl_hazard(p, r, hazard_instance);
 534		tlbw(p);
 535		uasm_bgezl_label(l, p, hazard_instance);
 536		hazard_instance++;
 537		uasm_i_nop(p);
 538		break;
 539
 540	case CPU_R4600:
 541	case CPU_R4700:
 
 
 542		uasm_i_nop(p);
 543		tlbw(p);
 544		uasm_i_nop(p);
 545		break;
 546
 547	case CPU_R5000:
 548	case CPU_NEVADA:
 549		uasm_i_nop(p); /* QED specifies 2 nops hazard */
 550		uasm_i_nop(p); /* QED specifies 2 nops hazard */
 551		tlbw(p);
 552		break;
 553
 554	case CPU_R4300:
 555	case CPU_5KC:
 556	case CPU_TX49XX:
 557	case CPU_PR4450:
 558	case CPU_XLR:
 559		uasm_i_nop(p);
 560		tlbw(p);
 561		break;
 562
 563	case CPU_R10000:
 564	case CPU_R12000:
 565	case CPU_R14000:
 566	case CPU_R16000:
 567	case CPU_4KC:
 568	case CPU_4KEC:
 569	case CPU_M14KC:
 570	case CPU_M14KEC:
 571	case CPU_SB1:
 572	case CPU_SB1A:
 573	case CPU_4KSC:
 574	case CPU_20KC:
 575	case CPU_25KF:
 576	case CPU_BMIPS32:
 577	case CPU_BMIPS3300:
 578	case CPU_BMIPS4350:
 579	case CPU_BMIPS4380:
 580	case CPU_BMIPS5000:
 581	case CPU_LOONGSON2:
 582	case CPU_LOONGSON3:
 583	case CPU_R5500:
 584		if (m4kc_tlbp_war())
 585			uasm_i_nop(p);
 586	case CPU_ALCHEMY:
 587		tlbw(p);
 588		break;
 589
 
 
 
 
 
 
 
 
 
 
 
 590	case CPU_RM7000:
 591		uasm_i_nop(p);
 592		uasm_i_nop(p);
 593		uasm_i_nop(p);
 594		uasm_i_nop(p);
 595		tlbw(p);
 596		break;
 597
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 598	case CPU_VR4111:
 599	case CPU_VR4121:
 600	case CPU_VR4122:
 601	case CPU_VR4181:
 602	case CPU_VR4181A:
 603		uasm_i_nop(p);
 604		uasm_i_nop(p);
 605		tlbw(p);
 606		uasm_i_nop(p);
 607		uasm_i_nop(p);
 608		break;
 609
 610	case CPU_VR4131:
 611	case CPU_VR4133:
 612	case CPU_R5432:
 613		uasm_i_nop(p);
 614		uasm_i_nop(p);
 615		tlbw(p);
 616		break;
 617
 618	case CPU_JZRISC:
 619		tlbw(p);
 620		uasm_i_nop(p);
 621		break;
 622
 623	default:
 624		panic("No TLB refill handler yet (CPU type: %d)",
 625		      current_cpu_type());
 626		break;
 627	}
 628}
 629
 630static __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
 631							unsigned int reg)
 632{
 633	if (cpu_has_rixi && _PAGE_NO_EXEC) {
 634		if (fill_includes_sw_bits) {
 635			UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL));
 636		} else {
 637			UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC));
 638			UASM_i_ROTR(p, reg, reg,
 639				    ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
 640		}
 641	} else {
 642#ifdef CONFIG_PHYS_ADDR_T_64BIT
 643		uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
 644#else
 645		UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL));
 646#endif
 647	}
 648}
 649
 650#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
 651
 652static void build_restore_pagemask(u32 **p, struct uasm_reloc **r,
 653				   unsigned int tmp, enum label_id lid,
 654				   int restore_scratch)
 
 
 655{
 656	if (restore_scratch) {
 657		/* Reset default page size */
 658		if (PM_DEFAULT_MASK >> 16) {
 659			uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
 660			uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
 661			uasm_i_mtc0(p, tmp, C0_PAGEMASK);
 662			uasm_il_b(p, r, lid);
 663		} else if (PM_DEFAULT_MASK) {
 664			uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
 665			uasm_i_mtc0(p, tmp, C0_PAGEMASK);
 666			uasm_il_b(p, r, lid);
 667		} else {
 668			uasm_i_mtc0(p, 0, C0_PAGEMASK);
 669			uasm_il_b(p, r, lid);
 670		}
 671		if (scratch_reg >= 0)
 672			UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
 673		else
 674			UASM_i_LW(p, 1, scratchpad_offset(0), 0);
 675	} else {
 676		/* Reset default page size */
 677		if (PM_DEFAULT_MASK >> 16) {
 678			uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
 679			uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
 680			uasm_il_b(p, r, lid);
 681			uasm_i_mtc0(p, tmp, C0_PAGEMASK);
 682		} else if (PM_DEFAULT_MASK) {
 683			uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
 684			uasm_il_b(p, r, lid);
 685			uasm_i_mtc0(p, tmp, C0_PAGEMASK);
 686		} else {
 687			uasm_il_b(p, r, lid);
 688			uasm_i_mtc0(p, 0, C0_PAGEMASK);
 689		}
 690	}
 691}
 692
 693static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l,
 694				       struct uasm_reloc **r,
 695				       unsigned int tmp,
 696				       enum tlb_write_entry wmode,
 697				       int restore_scratch)
 
 698{
 699	/* Set huge page tlb entry size */
 700	uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
 701	uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
 702	uasm_i_mtc0(p, tmp, C0_PAGEMASK);
 703
 704	build_tlb_write_entry(p, l, r, wmode);
 705
 706	build_restore_pagemask(p, r, tmp, label_leave, restore_scratch);
 707}
 708
 709/*
 710 * Check if Huge PTE is present, if so then jump to LABEL.
 711 */
 712static void
 713build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
 714		  unsigned int pmd, int lid)
 715{
 716	UASM_i_LW(p, tmp, 0, pmd);
 717	if (use_bbit_insns()) {
 718		uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
 719	} else {
 720		uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
 721		uasm_il_bnez(p, r, tmp, lid);
 722	}
 723}
 724
 725static void build_huge_update_entries(u32 **p, unsigned int pte,
 726				      unsigned int tmp)
 
 727{
 728	int small_sequence;
 729
 730	/*
 731	 * A huge PTE describes an area the size of the
 732	 * configured huge page size. This is twice the
 733	 * of the large TLB entry size we intend to use.
 734	 * A TLB entry half the size of the configured
 735	 * huge page size is configured into entrylo0
 736	 * and entrylo1 to cover the contiguous huge PTE
 737	 * address space.
 738	 */
 739	small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
 740
 741	/* We can clobber tmp.	It isn't used after this.*/
 742	if (!small_sequence)
 743		uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
 744
 745	build_convert_pte_to_entrylo(p, pte);
 746	UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */
 747	/* convert to entrylo1 */
 748	if (small_sequence)
 749		UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
 750	else
 751		UASM_i_ADDU(p, pte, pte, tmp);
 752
 753	UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
 754}
 755
 756static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r,
 757				    struct uasm_label **l,
 758				    unsigned int pte,
 759				    unsigned int ptr)
 
 760{
 761#ifdef CONFIG_SMP
 762	UASM_i_SC(p, pte, 0, ptr);
 763	uasm_il_beqz(p, r, pte, label_tlb_huge_update);
 764	UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */
 765#else
 766	UASM_i_SW(p, pte, 0, ptr);
 767#endif
 768	build_huge_update_entries(p, pte, ptr);
 769	build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0);
 770}
 771#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
 772
 773#ifdef CONFIG_64BIT
 774/*
 775 * TMP and PTR are scratch.
 776 * TMP will be clobbered, PTR will hold the pmd entry.
 777 */
 778static void
 779build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 780		 unsigned int tmp, unsigned int ptr)
 781{
 782#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 783	long pgdc = (long)pgd_current;
 784#endif
 785	/*
 786	 * The vmalloc handling is not in the hotpath.
 787	 */
 788	uasm_i_dmfc0(p, tmp, C0_BADVADDR);
 789
 790	if (check_for_high_segbits) {
 791		/*
 792		 * The kernel currently implicitely assumes that the
 793		 * MIPS SEGBITS parameter for the processor is
 794		 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
 795		 * allocate virtual addresses outside the maximum
 796		 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
 797		 * that doesn't prevent user code from accessing the
 798		 * higher xuseg addresses.  Here, we make sure that
 799		 * everything but the lower xuseg addresses goes down
 800		 * the module_alloc/vmalloc path.
 801		 */
 802		uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
 803		uasm_il_bnez(p, r, ptr, label_vmalloc);
 804	} else {
 805		uasm_il_bltz(p, r, tmp, label_vmalloc);
 806	}
 807	/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
 808
 
 809	if (pgd_reg != -1) {
 810		/* pgd is in pgd_reg */
 811		UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
 812	} else {
 813#if defined(CONFIG_MIPS_PGD_C0_CONTEXT)
 814		/*
 815		 * &pgd << 11 stored in CONTEXT [23..63].
 816		 */
 817		UASM_i_MFC0(p, ptr, C0_CONTEXT);
 818
 819		/* Clear lower 23 bits of context. */
 820		uasm_i_dins(p, ptr, 0, 0, 23);
 821
 822		/* 1 0	1 0 1  << 6  xkphys cached */
 823		uasm_i_ori(p, ptr, ptr, 0x540);
 824		uasm_i_drotr(p, ptr, ptr, 11);
 
 825#elif defined(CONFIG_SMP)
 826		UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG);
 827		uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
 828		UASM_i_LA_mostly(p, tmp, pgdc);
 829		uasm_i_daddu(p, ptr, ptr, tmp);
 830		uasm_i_dmfc0(p, tmp, C0_BADVADDR);
 831		uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
 
 
 
 
 
 
 
 
 
 
 
 
 832#else
 833		UASM_i_LA_mostly(p, ptr, pgdc);
 834		uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
 835#endif
 836	}
 837
 838	uasm_l_vmalloc_done(l, *p);
 839
 840	/* get pgd offset in bytes */
 841	uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);
 842
 843	uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
 844	uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
 845#ifndef __PAGETABLE_PMD_FOLDED
 846	uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
 847	uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
 848	uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
 849	uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
 850	uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
 851#endif
 852}
 853
 854/*
 855 * BVADDR is the faulting address, PTR is scratch.
 856 * PTR will hold the pgd for vmalloc.
 857 */
 858static void
 859build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 860			unsigned int bvaddr, unsigned int ptr,
 861			enum vmalloc64_mode mode)
 862{
 863	long swpd = (long)swapper_pg_dir;
 864	int single_insn_swpd;
 865	int did_vmalloc_branch = 0;
 866
 867	single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
 868
 869	uasm_l_vmalloc(l, *p);
 870
 871	if (mode != not_refill && check_for_high_segbits) {
 872		if (single_insn_swpd) {
 873			uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
 874			uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
 875			did_vmalloc_branch = 1;
 876			/* fall through */
 877		} else {
 878			uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
 879		}
 880	}
 881	if (!did_vmalloc_branch) {
 882		if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
 883			uasm_il_b(p, r, label_vmalloc_done);
 884			uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
 885		} else {
 886			UASM_i_LA_mostly(p, ptr, swpd);
 887			uasm_il_b(p, r, label_vmalloc_done);
 888			if (uasm_in_compat_space_p(swpd))
 889				uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
 890			else
 891				uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
 892		}
 893	}
 894	if (mode != not_refill && check_for_high_segbits) {
 895		uasm_l_large_segbits_fault(l, *p);
 896		/*
 897		 * We get here if we are an xsseg address, or if we are
 898		 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
 899		 *
 900		 * Ignoring xsseg (assume disabled so would generate
 901		 * (address errors?), the only remaining possibility
 902		 * is the upper xuseg addresses.  On processors with
 903		 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
 904		 * addresses would have taken an address error. We try
 905		 * to mimic that here by taking a load/istream page
 906		 * fault.
 907		 */
 908		UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
 909		uasm_i_jr(p, ptr);
 910
 911		if (mode == refill_scratch) {
 912			if (scratch_reg >= 0)
 913				UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
 914			else
 915				UASM_i_LW(p, 1, scratchpad_offset(0), 0);
 916		} else {
 917			uasm_i_nop(p);
 918		}
 919	}
 920}
 921
 922#else /* !CONFIG_64BIT */
 923
 924/*
 925 * TMP and PTR are scratch.
 926 * TMP will be clobbered, PTR will hold the pgd entry.
 927 */
 928static void __maybe_unused
 929build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
 930{
 931	if (pgd_reg != -1) {
 932		/* pgd is in pgd_reg */
 933		uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg);
 934		uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
 935	} else {
 936		long pgdc = (long)pgd_current;
 937
 938		/* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
 939#ifdef CONFIG_SMP
 940		uasm_i_mfc0(p, ptr, SMP_CPUID_REG);
 941		UASM_i_LA_mostly(p, tmp, pgdc);
 942		uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
 943		uasm_i_addu(p, ptr, tmp, ptr);
 
 
 
 944#else
 945		UASM_i_LA_mostly(p, ptr, pgdc);
 
 
 
 
 
 946#endif
 947		uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
 948		uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
 949	}
 
 
 
 950	uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
 951	uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
 952	uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
 953}
 954
 955#endif /* !CONFIG_64BIT */
 956
 957static void build_adjust_context(u32 **p, unsigned int ctx)
 958{
 959	unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
 960	unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
 961
 962	switch (current_cpu_type()) {
 963	case CPU_VR41XX:
 964	case CPU_VR4111:
 965	case CPU_VR4121:
 966	case CPU_VR4122:
 967	case CPU_VR4131:
 968	case CPU_VR4181:
 969	case CPU_VR4181A:
 970	case CPU_VR4133:
 971		shift += 2;
 972		break;
 973
 974	default:
 975		break;
 976	}
 977
 978	if (shift)
 979		UASM_i_SRL(p, ctx, ctx, shift);
 980	uasm_i_andi(p, ctx, ctx, mask);
 981}
 982
 983static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
 984{
 985	/*
 986	 * Bug workaround for the Nevada. It seems as if under certain
 987	 * circumstances the move from cp0_context might produce a
 988	 * bogus result when the mfc0 instruction and its consumer are
 989	 * in a different cacheline or a load instruction, probably any
 990	 * memory reference, is between them.
 991	 */
 992	switch (current_cpu_type()) {
 993	case CPU_NEVADA:
 994		UASM_i_LW(p, ptr, 0, ptr);
 995		GET_CONTEXT(p, tmp); /* get context reg */
 996		break;
 997
 998	default:
 999		GET_CONTEXT(p, tmp); /* get context reg */
1000		UASM_i_LW(p, ptr, 0, ptr);
1001		break;
1002	}
1003
1004	build_adjust_context(p, tmp);
1005	UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
1006}
1007
1008static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep)
 
1009{
1010	/*
1011	 * 64bit address support (36bit on a 32bit CPU) in a 32bit
1012	 * Kernel is a special case. Only a few CPUs use it.
1013	 */
1014	if (config_enabled(CONFIG_PHYS_ADDR_T_64BIT) && !cpu_has_64bits) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1015		int pte_off_even = sizeof(pte_t) / 2;
1016		int pte_off_odd = pte_off_even + sizeof(pte_t);
1017#ifdef CONFIG_XPA
1018		const int scratch = 1; /* Our extra working register */
1019
1020		uasm_i_addu(p, scratch, 0, ptep);
1021#endif
1022		uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */
1023		uasm_i_lw(p, ptep, pte_off_odd, ptep); /* odd pte */
1024		UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
1025		UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL));
1026		UASM_i_MTC0(p, tmp, C0_ENTRYLO0);
1027		UASM_i_MTC0(p, ptep, C0_ENTRYLO1);
1028#ifdef CONFIG_XPA
1029		uasm_i_lw(p, tmp, 0, scratch);
1030		uasm_i_lw(p, ptep, sizeof(pte_t), scratch);
1031		uasm_i_lui(p, scratch, 0xff);
1032		uasm_i_ori(p, scratch, scratch, 0xffff);
1033		uasm_i_and(p, tmp, scratch, tmp);
1034		uasm_i_and(p, ptep, scratch, ptep);
1035		uasm_i_mthc0(p, tmp, C0_ENTRYLO0);
1036		uasm_i_mthc0(p, ptep, C0_ENTRYLO1);
1037#endif
1038		return;
1039	}
1040
1041	UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
1042	UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
1043	if (r45k_bvahwbug())
1044		build_tlb_probe_entry(p);
1045	build_convert_pte_to_entrylo(p, tmp);
1046	if (r4k_250MHZhwbug())
1047		UASM_i_MTC0(p, 0, C0_ENTRYLO0);
1048	UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
1049	build_convert_pte_to_entrylo(p, ptep);
1050	if (r45k_bvahwbug())
1051		uasm_i_mfc0(p, tmp, C0_INDEX);
 
 
 
 
 
 
 
 
 
 
1052	if (r4k_250MHZhwbug())
1053		UASM_i_MTC0(p, 0, C0_ENTRYLO1);
1054	UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
 
1055}
1056
1057struct mips_huge_tlb_info {
1058	int huge_pte;
1059	int restore_scratch;
1060	bool need_reload_pte;
1061};
1062
1063static struct mips_huge_tlb_info
1064build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
1065			       struct uasm_reloc **r, unsigned int tmp,
1066			       unsigned int ptr, int c0_scratch_reg)
1067{
1068	struct mips_huge_tlb_info rv;
1069	unsigned int even, odd;
1070	int vmalloc_branch_delay_filled = 0;
1071	const int scratch = 1; /* Our extra working register */
1072
1073	rv.huge_pte = scratch;
1074	rv.restore_scratch = 0;
1075	rv.need_reload_pte = false;
1076
1077	if (check_for_high_segbits) {
1078		UASM_i_MFC0(p, tmp, C0_BADVADDR);
1079
1080		if (pgd_reg != -1)
1081			UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
1082		else
1083			UASM_i_MFC0(p, ptr, C0_CONTEXT);
1084
1085		if (c0_scratch_reg >= 0)
1086			UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
1087		else
1088			UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1089
1090		uasm_i_dsrl_safe(p, scratch, tmp,
1091				 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
1092		uasm_il_bnez(p, r, scratch, label_vmalloc);
1093
1094		if (pgd_reg == -1) {
1095			vmalloc_branch_delay_filled = 1;
1096			/* Clear lower 23 bits of context. */
1097			uasm_i_dins(p, ptr, 0, 0, 23);
1098		}
1099	} else {
1100		if (pgd_reg != -1)
1101			UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
1102		else
1103			UASM_i_MFC0(p, ptr, C0_CONTEXT);
1104
1105		UASM_i_MFC0(p, tmp, C0_BADVADDR);
1106
1107		if (c0_scratch_reg >= 0)
1108			UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
1109		else
1110			UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1111
1112		if (pgd_reg == -1)
1113			/* Clear lower 23 bits of context. */
1114			uasm_i_dins(p, ptr, 0, 0, 23);
1115
1116		uasm_il_bltz(p, r, tmp, label_vmalloc);
1117	}
1118
1119	if (pgd_reg == -1) {
1120		vmalloc_branch_delay_filled = 1;
1121		/* 1 0	1 0 1  << 6  xkphys cached */
1122		uasm_i_ori(p, ptr, ptr, 0x540);
1123		uasm_i_drotr(p, ptr, ptr, 11);
1124	}
1125
1126#ifdef __PAGETABLE_PMD_FOLDED
1127#define LOC_PTEP scratch
1128#else
1129#define LOC_PTEP ptr
1130#endif
1131
1132	if (!vmalloc_branch_delay_filled)
1133		/* get pgd offset in bytes */
1134		uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1135
1136	uasm_l_vmalloc_done(l, *p);
1137
1138	/*
1139	 *			   tmp		ptr
1140	 * fall-through case =	 badvaddr  *pgd_current
1141	 * vmalloc case	     =	 badvaddr  swapper_pg_dir
1142	 */
1143
1144	if (vmalloc_branch_delay_filled)
1145		/* get pgd offset in bytes */
1146		uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1147
1148#ifdef __PAGETABLE_PMD_FOLDED
1149	GET_CONTEXT(p, tmp); /* get context reg */
1150#endif
1151	uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3);
1152
1153	if (use_lwx_insns()) {
1154		UASM_i_LWX(p, LOC_PTEP, scratch, ptr);
1155	} else {
1156		uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */
1157		uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */
1158	}
1159
1160#ifndef __PAGETABLE_PMD_FOLDED
1161	/* get pmd offset in bytes */
1162	uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3);
1163	uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3);
1164	GET_CONTEXT(p, tmp); /* get context reg */
1165
1166	if (use_lwx_insns()) {
1167		UASM_i_LWX(p, scratch, scratch, ptr);
1168	} else {
1169		uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */
1170		UASM_i_LW(p, scratch, 0, ptr);
1171	}
1172#endif
1173	/* Adjust the context during the load latency. */
1174	build_adjust_context(p, tmp);
1175
1176#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1177	uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update);
1178	/*
1179	 * The in the LWX case we don't want to do the load in the
1180	 * delay slot.	It cannot issue in the same cycle and may be
1181	 * speculative and unneeded.
1182	 */
1183	if (use_lwx_insns())
1184		uasm_i_nop(p);
1185#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
1186
1187
1188	/* build_update_entries */
1189	if (use_lwx_insns()) {
1190		even = ptr;
1191		odd = tmp;
1192		UASM_i_LWX(p, even, scratch, tmp);
1193		UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t));
1194		UASM_i_LWX(p, odd, scratch, tmp);
1195	} else {
1196		UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */
1197		even = tmp;
1198		odd = ptr;
1199		UASM_i_LW(p, even, 0, ptr); /* get even pte */
1200		UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */
1201	}
1202	if (cpu_has_rixi) {
1203		uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL));
 
 
 
1204		UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
1205		uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL));
 
1206	} else {
1207		uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL));
1208		UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
1209		uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL));
1210	}
1211	UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
1212
1213	if (c0_scratch_reg >= 0) {
1214		UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg);
1215		build_tlb_write_entry(p, l, r, tlb_random);
1216		uasm_l_leave(l, *p);
1217		rv.restore_scratch = 1;
1218	} else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13)  {
1219		build_tlb_write_entry(p, l, r, tlb_random);
1220		uasm_l_leave(l, *p);
1221		UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1222	} else {
1223		UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1224		build_tlb_write_entry(p, l, r, tlb_random);
1225		uasm_l_leave(l, *p);
1226		rv.restore_scratch = 1;
1227	}
1228
1229	uasm_i_eret(p); /* return from trap */
1230
1231	return rv;
1232}
1233
1234/*
1235 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
1236 * because EXL == 0.  If we wrap, we can also use the 32 instruction
1237 * slots before the XTLB refill exception handler which belong to the
1238 * unused TLB refill exception.
1239 */
1240#define MIPS64_REFILL_INSNS 32
1241
1242static void build_r4000_tlb_refill_handler(void)
1243{
1244	u32 *p = tlb_handler;
1245	struct uasm_label *l = labels;
1246	struct uasm_reloc *r = relocs;
1247	u32 *f;
1248	unsigned int final_len;
1249	struct mips_huge_tlb_info htlb_info __maybe_unused;
1250	enum vmalloc64_mode vmalloc_mode __maybe_unused;
1251
1252	memset(tlb_handler, 0, sizeof(tlb_handler));
1253	memset(labels, 0, sizeof(labels));
1254	memset(relocs, 0, sizeof(relocs));
1255	memset(final_handler, 0, sizeof(final_handler));
1256
1257	if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
1258		htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
1259							  scratch_reg);
1260		vmalloc_mode = refill_scratch;
1261	} else {
1262		htlb_info.huge_pte = K0;
1263		htlb_info.restore_scratch = 0;
1264		htlb_info.need_reload_pte = true;
1265		vmalloc_mode = refill_noscratch;
1266		/*
1267		 * create the plain linear handler
1268		 */
1269		if (bcm1250_m3_war()) {
1270			unsigned int segbits = 44;
1271
1272			uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1273			uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
1274			uasm_i_xor(&p, K0, K0, K1);
1275			uasm_i_dsrl_safe(&p, K1, K0, 62);
1276			uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
1277			uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
1278			uasm_i_or(&p, K0, K0, K1);
1279			uasm_il_bnez(&p, &r, K0, label_leave);
1280			/* No need for uasm_i_nop */
1281		}
1282
1283#ifdef CONFIG_64BIT
1284		build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
1285#else
1286		build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
1287#endif
1288
1289#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1290		build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
1291#endif
1292
1293		build_get_ptep(&p, K0, K1);
1294		build_update_entries(&p, K0, K1);
1295		build_tlb_write_entry(&p, &l, &r, tlb_random);
1296		uasm_l_leave(&l, p);
1297		uasm_i_eret(&p); /* return from trap */
1298	}
1299#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1300	uasm_l_tlb_huge_update(&l, p);
1301	if (htlb_info.need_reload_pte)
1302		UASM_i_LW(&p, htlb_info.huge_pte, 0, K1);
1303	build_huge_update_entries(&p, htlb_info.huge_pte, K1);
1304	build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
1305				   htlb_info.restore_scratch);
1306#endif
1307
1308#ifdef CONFIG_64BIT
1309	build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode);
1310#endif
1311
1312	/*
1313	 * Overflow check: For the 64bit handler, we need at least one
1314	 * free instruction slot for the wrap-around branch. In worst
1315	 * case, if the intended insertion point is a delay slot, we
1316	 * need three, with the second nop'ed and the third being
1317	 * unused.
1318	 */
1319	switch (boot_cpu_type()) {
1320	default:
1321		if (sizeof(long) == 4) {
1322	case CPU_LOONGSON2:
1323		/* Loongson2 ebase is different than r4k, we have more space */
1324			if ((p - tlb_handler) > 64)
1325				panic("TLB refill handler space exceeded");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1326			/*
1327			 * Now fold the handler in the TLB refill handler space.
 
 
1328			 */
1329			f = final_handler;
1330			/* Simplest case, just copy the handler. */
1331			uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1332			final_len = p - tlb_handler;
1333			break;
1334		} else {
1335			if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
1336			    || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
1337				&& uasm_insn_has_bdelay(relocs,
1338							tlb_handler + MIPS64_REFILL_INSNS - 3)))
1339				panic("TLB refill handler space exceeded");
1340			/*
1341			 * Now fold the handler in the TLB refill handler space.
 
 
1342			 */
1343			f = final_handler + MIPS64_REFILL_INSNS;
1344			if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
1345				/* Just copy the handler. */
1346				uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1347				final_len = p - tlb_handler;
1348			} else {
1349#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1350				const enum label_id ls = label_tlb_huge_update;
1351#else
1352				const enum label_id ls = label_vmalloc;
1353#endif
1354				u32 *split;
1355				int ov = 0;
1356				int i;
1357
1358				for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
1359					;
1360				BUG_ON(i == ARRAY_SIZE(labels));
1361				split = labels[i].addr;
1362
1363				/*
1364				 * See if we have overflown one way or the other.
1365				 */
1366				if (split > tlb_handler + MIPS64_REFILL_INSNS ||
1367				    split < p - MIPS64_REFILL_INSNS)
1368					ov = 1;
1369
1370				if (ov) {
1371					/*
1372					 * Split two instructions before the end.  One
1373					 * for the branch and one for the instruction
1374					 * in the delay slot.
1375					 */
1376					split = tlb_handler + MIPS64_REFILL_INSNS - 2;
1377
1378					/*
1379					 * If the branch would fall in a delay slot,
1380					 * we must back up an additional instruction
1381					 * so that it is no longer in a delay slot.
1382					 */
1383					if (uasm_insn_has_bdelay(relocs, split - 1))
1384						split--;
1385				}
1386				/* Copy first part of the handler. */
1387				uasm_copy_handler(relocs, labels, tlb_handler, split, f);
1388				f += split - tlb_handler;
1389
1390				if (ov) {
1391					/* Insert branch. */
1392					uasm_l_split(&l, final_handler);
1393					uasm_il_b(&f, &r, label_split);
1394					if (uasm_insn_has_bdelay(relocs, split))
1395						uasm_i_nop(&f);
1396					else {
1397						uasm_copy_handler(relocs, labels,
1398								  split, split + 1, f);
1399						uasm_move_labels(labels, f, f + 1, -1);
1400						f++;
1401						split++;
1402					}
1403				}
1404
1405				/* Copy the rest of the handler. */
1406				uasm_copy_handler(relocs, labels, split, p, final_handler);
1407				final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
1408					    (p - split);
1409			}
1410		}
1411		break;
 
 
 
 
1412	}
 
1413
1414	uasm_resolve_relocs(relocs, labels);
1415	pr_debug("Wrote TLB refill handler (%u instructions).\n",
1416		 final_len);
1417
1418	memcpy((void *)ebase, final_handler, 0x100);
1419	local_flush_icache_range(ebase, ebase + 0x100);
1420
1421	dump_handler("r4000_tlb_refill", (u32 *)ebase, 64);
1422}
1423
1424extern u32 handle_tlbl[], handle_tlbl_end[];
1425extern u32 handle_tlbs[], handle_tlbs_end[];
1426extern u32 handle_tlbm[], handle_tlbm_end[];
1427extern u32 tlbmiss_handler_setup_pgd_start[], tlbmiss_handler_setup_pgd[];
1428extern u32 tlbmiss_handler_setup_pgd_end[];
 
 
 
 
 
 
1429
1430static void build_setup_pgd(void)
1431{
1432	const int a0 = 4;
1433	const int __maybe_unused a1 = 5;
1434	const int __maybe_unused a2 = 6;
1435	u32 *p = tlbmiss_handler_setup_pgd_start;
1436	const int tlbmiss_handler_setup_pgd_size =
1437		tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd_start;
1438#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
1439	long pgdc = (long)pgd_current;
1440#endif
1441
1442	memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size *
1443					sizeof(tlbmiss_handler_setup_pgd[0]));
1444	memset(labels, 0, sizeof(labels));
1445	memset(relocs, 0, sizeof(relocs));
 
1446	pgd_reg = allocate_kscratch();
1447#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
1448	if (pgd_reg == -1) {
1449		struct uasm_label *l = labels;
1450		struct uasm_reloc *r = relocs;
1451
 
1452		/* PGD << 11 in c0_Context */
1453		/*
1454		 * If it is a ckseg0 address, convert to a physical
1455		 * address.  Shifting right by 29 and adding 4 will
1456		 * result in zero for these addresses.
1457		 *
1458		 */
1459		UASM_i_SRA(&p, a1, a0, 29);
1460		UASM_i_ADDIU(&p, a1, a1, 4);
1461		uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
1462		uasm_i_nop(&p);
1463		uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
1464		uasm_l_tlbl_goaround1(&l, p);
1465		UASM_i_SLL(&p, a0, a0, 11);
1466		uasm_i_jr(&p, 31);
1467		UASM_i_MTC0(&p, a0, C0_CONTEXT);
1468	} else {
1469		/* PGD in c0_KScratch */
1470		uasm_i_jr(&p, 31);
1471		UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
1472	}
1473#else
1474#ifdef CONFIG_SMP
1475	/* Save PGD to pgd_current[smp_processor_id()] */
1476	UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG);
1477	UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT);
1478	UASM_i_LA_mostly(&p, a2, pgdc);
1479	UASM_i_ADDU(&p, a2, a2, a1);
1480	UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
1481#else
1482	UASM_i_LA_mostly(&p, a2, pgdc);
1483	UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
1484#endif /* SMP */
1485	uasm_i_jr(&p, 31);
1486
1487	/* if pgd_reg is allocated, save PGD also to scratch register */
1488	if (pgd_reg != -1)
1489		UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
1490	else
1491		uasm_i_nop(&p);
1492#endif
1493	if (p >= tlbmiss_handler_setup_pgd_end)
1494		panic("tlbmiss_handler_setup_pgd space exceeded");
1495
1496	uasm_resolve_relocs(relocs, labels);
1497	pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
1498		 (unsigned int)(p - tlbmiss_handler_setup_pgd));
1499
1500	dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd,
1501					tlbmiss_handler_setup_pgd_size);
1502}
 
1503
1504static void
1505iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
1506{
1507#ifdef CONFIG_SMP
1508# ifdef CONFIG_PHYS_ADDR_T_64BIT
1509	if (cpu_has_64bits)
1510		uasm_i_lld(p, pte, 0, ptr);
1511	else
1512# endif
1513		UASM_i_LL(p, pte, 0, ptr);
1514#else
1515# ifdef CONFIG_PHYS_ADDR_T_64BIT
1516	if (cpu_has_64bits)
1517		uasm_i_ld(p, pte, 0, ptr);
1518	else
1519# endif
1520		UASM_i_LW(p, pte, 0, ptr);
1521#endif
1522}
1523
1524static void
1525iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
1526	unsigned int mode)
1527{
1528#ifdef CONFIG_PHYS_ADDR_T_64BIT
1529	unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
1530
1531	if (!cpu_has_64bits) {
1532		const int scratch = 1; /* Our extra working register */
1533
1534		uasm_i_lui(p, scratch, (mode >> 16));
1535		uasm_i_or(p, pte, pte, scratch);
1536	} else
1537#endif
 
1538	uasm_i_ori(p, pte, pte, mode);
1539#ifdef CONFIG_SMP
1540# ifdef CONFIG_PHYS_ADDR_T_64BIT
1541	if (cpu_has_64bits)
1542		uasm_i_scd(p, pte, 0, ptr);
1543	else
1544# endif
1545		UASM_i_SC(p, pte, 0, ptr);
1546
1547	if (r10000_llsc_war())
1548		uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
1549	else
1550		uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1551
1552# ifdef CONFIG_PHYS_ADDR_T_64BIT
1553	if (!cpu_has_64bits) {
1554		/* no uasm_i_nop needed */
1555		uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
1556		uasm_i_ori(p, pte, pte, hwmode);
1557		uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
1558		uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1559		/* no uasm_i_nop needed */
1560		uasm_i_lw(p, pte, 0, ptr);
1561	} else
1562		uasm_i_nop(p);
1563# else
1564	uasm_i_nop(p);
1565# endif
1566#else
1567# ifdef CONFIG_PHYS_ADDR_T_64BIT
1568	if (cpu_has_64bits)
1569		uasm_i_sd(p, pte, 0, ptr);
1570	else
1571# endif
1572		UASM_i_SW(p, pte, 0, ptr);
1573
1574# ifdef CONFIG_PHYS_ADDR_T_64BIT
1575	if (!cpu_has_64bits) {
1576		uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
1577		uasm_i_ori(p, pte, pte, hwmode);
1578		uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
1579		uasm_i_lw(p, pte, 0, ptr);
1580	}
1581# endif
1582#endif
1583}
1584
1585/*
1586 * Check if PTE is present, if not then jump to LABEL. PTR points to
1587 * the page table where this PTE is located, PTE will be re-loaded
1588 * with it's original value.
1589 */
1590static void
1591build_pte_present(u32 **p, struct uasm_reloc **r,
1592		  int pte, int ptr, int scratch, enum label_id lid)
1593{
1594	int t = scratch >= 0 ? scratch : pte;
1595	int cur = pte;
1596
1597	if (cpu_has_rixi) {
1598		if (use_bbit_insns()) {
1599			uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
1600			uasm_i_nop(p);
1601		} else {
1602			if (_PAGE_PRESENT_SHIFT) {
1603				uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
1604				cur = t;
1605			}
1606			uasm_i_andi(p, t, cur, 1);
1607			uasm_il_beqz(p, r, t, lid);
1608			if (pte == t)
1609				/* You lose the SMP race :-(*/
1610				iPTE_LW(p, pte, ptr);
1611		}
1612	} else {
1613		if (_PAGE_PRESENT_SHIFT) {
1614			uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
1615			cur = t;
1616		}
1617		uasm_i_andi(p, t, cur,
1618			(_PAGE_PRESENT | _PAGE_READ) >> _PAGE_PRESENT_SHIFT);
1619		uasm_i_xori(p, t, t,
1620			(_PAGE_PRESENT | _PAGE_READ) >> _PAGE_PRESENT_SHIFT);
1621		uasm_il_bnez(p, r, t, lid);
1622		if (pte == t)
1623			/* You lose the SMP race :-(*/
1624			iPTE_LW(p, pte, ptr);
1625	}
1626}
1627
1628/* Make PTE valid, store result in PTR. */
1629static void
1630build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
1631		 unsigned int ptr)
1632{
1633	unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
1634
1635	iPTE_SW(p, r, pte, ptr, mode);
1636}
1637
1638/*
1639 * Check if PTE can be written to, if not branch to LABEL. Regardless
1640 * restore PTE with value from PTR when done.
1641 */
1642static void
1643build_pte_writable(u32 **p, struct uasm_reloc **r,
1644		   unsigned int pte, unsigned int ptr, int scratch,
1645		   enum label_id lid)
1646{
1647	int t = scratch >= 0 ? scratch : pte;
1648	int cur = pte;
1649
1650	if (_PAGE_PRESENT_SHIFT) {
1651		uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
1652		cur = t;
1653	}
1654	uasm_i_andi(p, t, cur,
1655		    (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT);
1656	uasm_i_xori(p, t, t,
1657		    (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT);
1658	uasm_il_bnez(p, r, t, lid);
1659	if (pte == t)
1660		/* You lose the SMP race :-(*/
1661		iPTE_LW(p, pte, ptr);
1662	else
1663		uasm_i_nop(p);
1664}
1665
1666/* Make PTE writable, update software status bits as well, then store
1667 * at PTR.
1668 */
1669static void
1670build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
1671		 unsigned int ptr)
1672{
1673	unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
1674			     | _PAGE_DIRTY);
1675
1676	iPTE_SW(p, r, pte, ptr, mode);
1677}
1678
1679/*
1680 * Check if PTE can be modified, if not branch to LABEL. Regardless
1681 * restore PTE with value from PTR when done.
1682 */
1683static void
1684build_pte_modifiable(u32 **p, struct uasm_reloc **r,
1685		     unsigned int pte, unsigned int ptr, int scratch,
1686		     enum label_id lid)
1687{
1688	if (use_bbit_insns()) {
1689		uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
1690		uasm_i_nop(p);
1691	} else {
1692		int t = scratch >= 0 ? scratch : pte;
1693		uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT);
1694		uasm_i_andi(p, t, t, 1);
1695		uasm_il_beqz(p, r, t, lid);
1696		if (pte == t)
1697			/* You lose the SMP race :-(*/
1698			iPTE_LW(p, pte, ptr);
1699	}
1700}
1701
1702#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
1703
1704
1705/*
1706 * R3000 style TLB load/store/modify handlers.
1707 */
1708
1709/*
1710 * This places the pte into ENTRYLO0 and writes it with tlbwi.
1711 * Then it returns.
1712 */
1713static void
1714build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
1715{
1716	uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1717	uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
1718	uasm_i_tlbwi(p);
1719	uasm_i_jr(p, tmp);
1720	uasm_i_rfe(p); /* branch delay */
1721}
1722
1723/*
1724 * This places the pte into ENTRYLO0 and writes it with tlbwi
1725 * or tlbwr as appropriate.  This is because the index register
1726 * may have the probe fail bit set as a result of a trap on a
1727 * kseg2 access, i.e. without refill.  Then it returns.
1728 */
1729static void
1730build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
1731			     struct uasm_reloc **r, unsigned int pte,
1732			     unsigned int tmp)
1733{
1734	uasm_i_mfc0(p, tmp, C0_INDEX);
1735	uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1736	uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
1737	uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
1738	uasm_i_tlbwi(p); /* cp0 delay */
1739	uasm_i_jr(p, tmp);
1740	uasm_i_rfe(p); /* branch delay */
1741	uasm_l_r3000_write_probe_fail(l, *p);
1742	uasm_i_tlbwr(p); /* cp0 delay */
1743	uasm_i_jr(p, tmp);
1744	uasm_i_rfe(p); /* branch delay */
1745}
1746
1747static void
1748build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1749				   unsigned int ptr)
1750{
1751	long pgdc = (long)pgd_current;
1752
1753	uasm_i_mfc0(p, pte, C0_BADVADDR);
1754	uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
1755	uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
1756	uasm_i_srl(p, pte, pte, 22); /* load delay */
1757	uasm_i_sll(p, pte, pte, 2);
1758	uasm_i_addu(p, ptr, ptr, pte);
1759	uasm_i_mfc0(p, pte, C0_CONTEXT);
1760	uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
1761	uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
1762	uasm_i_addu(p, ptr, ptr, pte);
1763	uasm_i_lw(p, pte, 0, ptr);
1764	uasm_i_tlbp(p); /* load delay */
1765}
1766
1767static void build_r3000_tlb_load_handler(void)
1768{
1769	u32 *p = handle_tlbl;
1770	const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
1771	struct uasm_label *l = labels;
1772	struct uasm_reloc *r = relocs;
1773
1774	memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
1775	memset(labels, 0, sizeof(labels));
1776	memset(relocs, 0, sizeof(relocs));
1777
1778	build_r3000_tlbchange_handler_head(&p, K0, K1);
1779	build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl);
1780	uasm_i_nop(&p); /* load delay */
1781	build_make_valid(&p, &r, K0, K1);
1782	build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1783
1784	uasm_l_nopage_tlbl(&l, p);
1785	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1786	uasm_i_nop(&p);
1787
1788	if (p >= handle_tlbl_end)
1789		panic("TLB load handler fastpath space exceeded");
1790
1791	uasm_resolve_relocs(relocs, labels);
1792	pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1793		 (unsigned int)(p - handle_tlbl));
1794
1795	dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size);
1796}
1797
1798static void build_r3000_tlb_store_handler(void)
1799{
1800	u32 *p = handle_tlbs;
1801	const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
1802	struct uasm_label *l = labels;
1803	struct uasm_reloc *r = relocs;
1804
1805	memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
1806	memset(labels, 0, sizeof(labels));
1807	memset(relocs, 0, sizeof(relocs));
1808
1809	build_r3000_tlbchange_handler_head(&p, K0, K1);
1810	build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs);
1811	uasm_i_nop(&p); /* load delay */
1812	build_make_write(&p, &r, K0, K1);
1813	build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1814
1815	uasm_l_nopage_tlbs(&l, p);
1816	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1817	uasm_i_nop(&p);
1818
1819	if (p >= handle_tlbs_end)
1820		panic("TLB store handler fastpath space exceeded");
1821
1822	uasm_resolve_relocs(relocs, labels);
1823	pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1824		 (unsigned int)(p - handle_tlbs));
1825
1826	dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size);
1827}
1828
1829static void build_r3000_tlb_modify_handler(void)
1830{
1831	u32 *p = handle_tlbm;
1832	const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
1833	struct uasm_label *l = labels;
1834	struct uasm_reloc *r = relocs;
1835
1836	memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
1837	memset(labels, 0, sizeof(labels));
1838	memset(relocs, 0, sizeof(relocs));
1839
1840	build_r3000_tlbchange_handler_head(&p, K0, K1);
1841	build_pte_modifiable(&p, &r, K0, K1,  -1, label_nopage_tlbm);
1842	uasm_i_nop(&p); /* load delay */
1843	build_make_write(&p, &r, K0, K1);
1844	build_r3000_pte_reload_tlbwi(&p, K0, K1);
1845
1846	uasm_l_nopage_tlbm(&l, p);
1847	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1848	uasm_i_nop(&p);
1849
1850	if (p >= handle_tlbm_end)
1851		panic("TLB modify handler fastpath space exceeded");
1852
1853	uasm_resolve_relocs(relocs, labels);
1854	pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1855		 (unsigned int)(p - handle_tlbm));
1856
1857	dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_size);
1858}
1859#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
1860
1861/*
1862 * R4000 style TLB load/store/modify handlers.
1863 */
1864static struct work_registers
1865build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
1866				   struct uasm_reloc **r)
1867{
1868	struct work_registers wr = build_get_work_registers(p);
1869
1870#ifdef CONFIG_64BIT
1871	build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */
1872#else
1873	build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */
1874#endif
1875
1876#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1877	/*
1878	 * For huge tlb entries, pmd doesn't contain an address but
1879	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
1880	 * see if we need to jump to huge tlb processing.
1881	 */
1882	build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update);
1883#endif
1884
1885	UASM_i_MFC0(p, wr.r1, C0_BADVADDR);
1886	UASM_i_LW(p, wr.r2, 0, wr.r2);
1887	UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
1888	uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
1889	UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1);
1890
1891#ifdef CONFIG_SMP
1892	uasm_l_smp_pgtable_change(l, *p);
1893#endif
1894	iPTE_LW(p, wr.r1, wr.r2); /* get even pte */
1895	if (!m4kc_tlbp_war()) {
1896		build_tlb_probe_entry(p);
1897		if (cpu_has_htw) {
1898			/* race condition happens, leaving */
1899			uasm_i_ehb(p);
1900			uasm_i_mfc0(p, wr.r3, C0_INDEX);
1901			uasm_il_bltz(p, r, wr.r3, label_leave);
1902			uasm_i_nop(p);
1903		}
1904	}
1905	return wr;
1906}
1907
1908static void
1909build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
1910				   struct uasm_reloc **r, unsigned int tmp,
1911				   unsigned int ptr)
1912{
1913	uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
1914	uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
1915	build_update_entries(p, tmp, ptr);
1916	build_tlb_write_entry(p, l, r, tlb_indexed);
1917	uasm_l_leave(l, *p);
1918	build_restore_work_registers(p);
1919	uasm_i_eret(p); /* return from trap */
1920
1921#ifdef CONFIG_64BIT
1922	build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
1923#endif
1924}
1925
1926static void build_r4000_tlb_load_handler(void)
1927{
1928	u32 *p = handle_tlbl;
1929	const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
1930	struct uasm_label *l = labels;
1931	struct uasm_reloc *r = relocs;
1932	struct work_registers wr;
1933
1934	memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
1935	memset(labels, 0, sizeof(labels));
1936	memset(relocs, 0, sizeof(relocs));
1937
1938	if (bcm1250_m3_war()) {
1939		unsigned int segbits = 44;
1940
1941		uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1942		uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
1943		uasm_i_xor(&p, K0, K0, K1);
1944		uasm_i_dsrl_safe(&p, K1, K0, 62);
1945		uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
1946		uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
1947		uasm_i_or(&p, K0, K0, K1);
1948		uasm_il_bnez(&p, &r, K0, label_leave);
1949		/* No need for uasm_i_nop */
1950	}
1951
1952	wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
1953	build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
1954	if (m4kc_tlbp_war())
1955		build_tlb_probe_entry(&p);
1956
1957	if (cpu_has_rixi && !cpu_has_rixiex) {
1958		/*
1959		 * If the page is not _PAGE_VALID, RI or XI could not
1960		 * have triggered it.  Skip the expensive test..
1961		 */
1962		if (use_bbit_insns()) {
1963			uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
1964				      label_tlbl_goaround1);
1965		} else {
1966			uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
1967			uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1);
1968		}
1969		uasm_i_nop(&p);
1970
1971		uasm_i_tlbr(&p);
1972
1973		switch (current_cpu_type()) {
1974		default:
1975			if (cpu_has_mips_r2_exec_hazard) {
1976				uasm_i_ehb(&p);
1977
1978		case CPU_CAVIUM_OCTEON:
1979		case CPU_CAVIUM_OCTEON_PLUS:
1980		case CPU_CAVIUM_OCTEON2:
1981				break;
1982			}
1983		}
1984
1985		/* Examine  entrylo 0 or 1 based on ptr. */
1986		if (use_bbit_insns()) {
1987			uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
1988		} else {
1989			uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
1990			uasm_i_beqz(&p, wr.r3, 8);
1991		}
1992		/* load it in the delay slot*/
1993		UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
1994		/* load it if ptr is odd */
1995		UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
1996		/*
1997		 * If the entryLo (now in wr.r3) is valid (bit 1), RI or
1998		 * XI must have triggered it.
1999		 */
2000		if (use_bbit_insns()) {
2001			uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl);
2002			uasm_i_nop(&p);
2003			uasm_l_tlbl_goaround1(&l, p);
2004		} else {
2005			uasm_i_andi(&p, wr.r3, wr.r3, 2);
2006			uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl);
2007			uasm_i_nop(&p);
2008		}
2009		uasm_l_tlbl_goaround1(&l, p);
2010	}
2011	build_make_valid(&p, &r, wr.r1, wr.r2);
2012	build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
2013
2014#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
2015	/*
2016	 * This is the entry point when build_r4000_tlbchange_handler_head
2017	 * spots a huge page.
2018	 */
2019	uasm_l_tlb_huge_update(&l, p);
2020	iPTE_LW(&p, wr.r1, wr.r2);
2021	build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
2022	build_tlb_probe_entry(&p);
2023
2024	if (cpu_has_rixi && !cpu_has_rixiex) {
2025		/*
2026		 * If the page is not _PAGE_VALID, RI or XI could not
2027		 * have triggered it.  Skip the expensive test..
2028		 */
2029		if (use_bbit_insns()) {
2030			uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
2031				      label_tlbl_goaround2);
2032		} else {
2033			uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
2034			uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
2035		}
2036		uasm_i_nop(&p);
2037
2038		uasm_i_tlbr(&p);
2039
2040		switch (current_cpu_type()) {
2041		default:
2042			if (cpu_has_mips_r2_exec_hazard) {
2043				uasm_i_ehb(&p);
2044
2045		case CPU_CAVIUM_OCTEON:
2046		case CPU_CAVIUM_OCTEON_PLUS:
2047		case CPU_CAVIUM_OCTEON2:
2048				break;
2049			}
2050		}
2051
2052		/* Examine  entrylo 0 or 1 based on ptr. */
2053		if (use_bbit_insns()) {
2054			uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
2055		} else {
2056			uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
2057			uasm_i_beqz(&p, wr.r3, 8);
2058		}
2059		/* load it in the delay slot*/
2060		UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
2061		/* load it if ptr is odd */
2062		UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
2063		/*
2064		 * If the entryLo (now in wr.r3) is valid (bit 1), RI or
2065		 * XI must have triggered it.
2066		 */
2067		if (use_bbit_insns()) {
2068			uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2);
2069		} else {
2070			uasm_i_andi(&p, wr.r3, wr.r3, 2);
2071			uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
2072		}
2073		if (PM_DEFAULT_MASK == 0)
2074			uasm_i_nop(&p);
2075		/*
2076		 * We clobbered C0_PAGEMASK, restore it.  On the other branch
2077		 * it is restored in build_huge_tlb_write_entry.
2078		 */
2079		build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0);
2080
2081		uasm_l_tlbl_goaround2(&l, p);
2082	}
2083	uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID));
2084	build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
2085#endif
2086
2087	uasm_l_nopage_tlbl(&l, p);
2088	build_restore_work_registers(&p);
2089#ifdef CONFIG_CPU_MICROMIPS
2090	if ((unsigned long)tlb_do_page_fault_0 & 1) {
2091		uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0));
2092		uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0));
2093		uasm_i_jr(&p, K0);
2094	} else
2095#endif
2096	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
2097	uasm_i_nop(&p);
2098
2099	if (p >= handle_tlbl_end)
2100		panic("TLB load handler fastpath space exceeded");
2101
2102	uasm_resolve_relocs(relocs, labels);
2103	pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
2104		 (unsigned int)(p - handle_tlbl));
2105
2106	dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size);
2107}
2108
2109static void build_r4000_tlb_store_handler(void)
2110{
2111	u32 *p = handle_tlbs;
2112	const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
2113	struct uasm_label *l = labels;
2114	struct uasm_reloc *r = relocs;
2115	struct work_registers wr;
2116
2117	memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
2118	memset(labels, 0, sizeof(labels));
2119	memset(relocs, 0, sizeof(relocs));
2120
2121	wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2122	build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
2123	if (m4kc_tlbp_war())
2124		build_tlb_probe_entry(&p);
2125	build_make_write(&p, &r, wr.r1, wr.r2);
2126	build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
2127
2128#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
2129	/*
2130	 * This is the entry point when
2131	 * build_r4000_tlbchange_handler_head spots a huge page.
2132	 */
2133	uasm_l_tlb_huge_update(&l, p);
2134	iPTE_LW(&p, wr.r1, wr.r2);
2135	build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
2136	build_tlb_probe_entry(&p);
2137	uasm_i_ori(&p, wr.r1, wr.r1,
2138		   _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
2139	build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
2140#endif
2141
2142	uasm_l_nopage_tlbs(&l, p);
2143	build_restore_work_registers(&p);
2144#ifdef CONFIG_CPU_MICROMIPS
2145	if ((unsigned long)tlb_do_page_fault_1 & 1) {
2146		uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
2147		uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
2148		uasm_i_jr(&p, K0);
2149	} else
2150#endif
2151	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2152	uasm_i_nop(&p);
2153
2154	if (p >= handle_tlbs_end)
2155		panic("TLB store handler fastpath space exceeded");
2156
2157	uasm_resolve_relocs(relocs, labels);
2158	pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
2159		 (unsigned int)(p - handle_tlbs));
2160
2161	dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size);
2162}
2163
2164static void build_r4000_tlb_modify_handler(void)
2165{
2166	u32 *p = handle_tlbm;
2167	const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
2168	struct uasm_label *l = labels;
2169	struct uasm_reloc *r = relocs;
2170	struct work_registers wr;
2171
2172	memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
2173	memset(labels, 0, sizeof(labels));
2174	memset(relocs, 0, sizeof(relocs));
2175
2176	wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2177	build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
2178	if (m4kc_tlbp_war())
2179		build_tlb_probe_entry(&p);
2180	/* Present and writable bits set, set accessed and dirty bits. */
2181	build_make_write(&p, &r, wr.r1, wr.r2);
2182	build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
2183
2184#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
2185	/*
2186	 * This is the entry point when
2187	 * build_r4000_tlbchange_handler_head spots a huge page.
2188	 */
2189	uasm_l_tlb_huge_update(&l, p);
2190	iPTE_LW(&p, wr.r1, wr.r2);
2191	build_pte_modifiable(&p, &r, wr.r1, wr.r2,  wr.r3, label_nopage_tlbm);
2192	build_tlb_probe_entry(&p);
2193	uasm_i_ori(&p, wr.r1, wr.r1,
2194		   _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
2195	build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
2196#endif
2197
2198	uasm_l_nopage_tlbm(&l, p);
2199	build_restore_work_registers(&p);
2200#ifdef CONFIG_CPU_MICROMIPS
2201	if ((unsigned long)tlb_do_page_fault_1 & 1) {
2202		uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
2203		uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
2204		uasm_i_jr(&p, K0);
2205	} else
2206#endif
2207	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2208	uasm_i_nop(&p);
2209
2210	if (p >= handle_tlbm_end)
2211		panic("TLB modify handler fastpath space exceeded");
2212
2213	uasm_resolve_relocs(relocs, labels);
2214	pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
2215		 (unsigned int)(p - handle_tlbm));
2216
2217	dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size);
2218}
2219
2220static void flush_tlb_handlers(void)
2221{
2222	local_flush_icache_range((unsigned long)handle_tlbl,
2223			   (unsigned long)handle_tlbl_end);
2224	local_flush_icache_range((unsigned long)handle_tlbs,
2225			   (unsigned long)handle_tlbs_end);
2226	local_flush_icache_range((unsigned long)handle_tlbm,
2227			   (unsigned long)handle_tlbm_end);
2228	local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
2229			   (unsigned long)tlbmiss_handler_setup_pgd_end);
2230}
2231
2232static void print_htw_config(void)
2233{
2234	unsigned long config;
2235	unsigned int pwctl;
2236	const int field = 2 * sizeof(unsigned long);
2237
2238	config = read_c0_pwfield();
2239	pr_debug("PWField (0x%0*lx): GDI: 0x%02lx  UDI: 0x%02lx  MDI: 0x%02lx  PTI: 0x%02lx  PTEI: 0x%02lx\n",
2240		field, config,
2241		(config & MIPS_PWFIELD_GDI_MASK) >> MIPS_PWFIELD_GDI_SHIFT,
2242		(config & MIPS_PWFIELD_UDI_MASK) >> MIPS_PWFIELD_UDI_SHIFT,
2243		(config & MIPS_PWFIELD_MDI_MASK) >> MIPS_PWFIELD_MDI_SHIFT,
2244		(config & MIPS_PWFIELD_PTI_MASK) >> MIPS_PWFIELD_PTI_SHIFT,
2245		(config & MIPS_PWFIELD_PTEI_MASK) >> MIPS_PWFIELD_PTEI_SHIFT);
2246
2247	config = read_c0_pwsize();
2248	pr_debug("PWSize  (0x%0*lx): GDW: 0x%02lx  UDW: 0x%02lx  MDW: 0x%02lx  PTW: 0x%02lx  PTEW: 0x%02lx\n",
2249		field, config,
2250		(config & MIPS_PWSIZE_GDW_MASK) >> MIPS_PWSIZE_GDW_SHIFT,
2251		(config & MIPS_PWSIZE_UDW_MASK) >> MIPS_PWSIZE_UDW_SHIFT,
2252		(config & MIPS_PWSIZE_MDW_MASK) >> MIPS_PWSIZE_MDW_SHIFT,
2253		(config & MIPS_PWSIZE_PTW_MASK) >> MIPS_PWSIZE_PTW_SHIFT,
2254		(config & MIPS_PWSIZE_PTEW_MASK) >> MIPS_PWSIZE_PTEW_SHIFT);
2255
2256	pwctl = read_c0_pwctl();
2257	pr_debug("PWCtl   (0x%x): PWEn: 0x%x  DPH: 0x%x  HugePg: 0x%x  Psn: 0x%x\n",
2258		pwctl,
2259		(pwctl & MIPS_PWCTL_PWEN_MASK) >> MIPS_PWCTL_PWEN_SHIFT,
2260		(pwctl & MIPS_PWCTL_DPH_MASK) >> MIPS_PWCTL_DPH_SHIFT,
2261		(pwctl & MIPS_PWCTL_HUGEPG_MASK) >> MIPS_PWCTL_HUGEPG_SHIFT,
2262		(pwctl & MIPS_PWCTL_PSN_MASK) >> MIPS_PWCTL_PSN_SHIFT);
2263}
2264
2265static void config_htw_params(void)
2266{
2267	unsigned long pwfield, pwsize, ptei;
2268	unsigned int config;
2269
2270	/*
2271	 * We are using 2-level page tables, so we only need to
2272	 * setup GDW and PTW appropriately. UDW and MDW will remain 0.
2273	 * The default value of GDI/UDI/MDI/PTI is 0xc. It is illegal to
2274	 * write values less than 0xc in these fields because the entire
2275	 * write will be dropped. As a result of which, we must preserve
2276	 * the original reset values and overwrite only what we really want.
2277	 */
2278
2279	pwfield = read_c0_pwfield();
2280	/* re-initialize the GDI field */
2281	pwfield &= ~MIPS_PWFIELD_GDI_MASK;
2282	pwfield |= PGDIR_SHIFT << MIPS_PWFIELD_GDI_SHIFT;
2283	/* re-initialize the PTI field including the even/odd bit */
2284	pwfield &= ~MIPS_PWFIELD_PTI_MASK;
2285	pwfield |= PAGE_SHIFT << MIPS_PWFIELD_PTI_SHIFT;
2286	if (CONFIG_PGTABLE_LEVELS >= 3) {
2287		pwfield &= ~MIPS_PWFIELD_MDI_MASK;
2288		pwfield |= PMD_SHIFT << MIPS_PWFIELD_MDI_SHIFT;
2289	}
2290	/* Set the PTEI right shift */
2291	ptei = _PAGE_GLOBAL_SHIFT << MIPS_PWFIELD_PTEI_SHIFT;
2292	pwfield |= ptei;
2293	write_c0_pwfield(pwfield);
2294	/* Check whether the PTEI value is supported */
2295	back_to_back_c0_hazard();
2296	pwfield = read_c0_pwfield();
2297	if (((pwfield & MIPS_PWFIELD_PTEI_MASK) << MIPS_PWFIELD_PTEI_SHIFT)
2298		!= ptei) {
2299		pr_warn("Unsupported PTEI field value: 0x%lx. HTW will not be enabled",
2300			ptei);
2301		/*
2302		 * Drop option to avoid HTW being enabled via another path
2303		 * (eg htw_reset())
2304		 */
2305		current_cpu_data.options &= ~MIPS_CPU_HTW;
2306		return;
2307	}
2308
2309	pwsize = ilog2(PTRS_PER_PGD) << MIPS_PWSIZE_GDW_SHIFT;
2310	pwsize |= ilog2(PTRS_PER_PTE) << MIPS_PWSIZE_PTW_SHIFT;
2311	if (CONFIG_PGTABLE_LEVELS >= 3)
2312		pwsize |= ilog2(PTRS_PER_PMD) << MIPS_PWSIZE_MDW_SHIFT;
2313
2314	/* If XPA has been enabled, PTEs are 64-bit in size. */
2315	if (config_enabled(CONFIG_64BITS) || (read_c0_pagegrain() & PG_ELPA))
2316		pwsize |= 1;
2317
2318	write_c0_pwsize(pwsize);
2319
2320	/* Make sure everything is set before we enable the HTW */
2321	back_to_back_c0_hazard();
2322
2323	/* Enable HTW and disable the rest of the pwctl fields */
2324	config = 1 << MIPS_PWCTL_PWEN_SHIFT;
2325	write_c0_pwctl(config);
2326	pr_info("Hardware Page Table Walker enabled\n");
2327
2328	print_htw_config();
2329}
2330
2331static void config_xpa_params(void)
2332{
2333#ifdef CONFIG_XPA
2334	unsigned int pagegrain;
2335
2336	if (mips_xpa_disabled) {
2337		pr_info("Extended Physical Addressing (XPA) disabled\n");
2338		return;
2339	}
2340
2341	pagegrain = read_c0_pagegrain();
2342	write_c0_pagegrain(pagegrain | PG_ELPA);
2343	back_to_back_c0_hazard();
2344	pagegrain = read_c0_pagegrain();
2345
2346	if (pagegrain & PG_ELPA)
2347		pr_info("Extended Physical Addressing (XPA) enabled\n");
2348	else
2349		panic("Extended Physical Addressing (XPA) disabled");
2350#endif
2351}
2352
2353static void check_pabits(void)
2354{
2355	unsigned long entry;
2356	unsigned pabits, fillbits;
2357
2358	if (!cpu_has_rixi || !_PAGE_NO_EXEC) {
2359		/*
2360		 * We'll only be making use of the fact that we can rotate bits
2361		 * into the fill if the CPU supports RIXI, so don't bother
2362		 * probing this for CPUs which don't.
2363		 */
2364		return;
2365	}
2366
2367	write_c0_entrylo0(~0ul);
2368	back_to_back_c0_hazard();
2369	entry = read_c0_entrylo0();
2370
2371	/* clear all non-PFN bits */
2372	entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1);
2373	entry &= ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI);
2374
2375	/* find a lower bound on PABITS, and upper bound on fill bits */
2376	pabits = fls_long(entry) + 6;
2377	fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0);
2378
2379	/* minus the RI & XI bits */
2380	fillbits -= min_t(unsigned, fillbits, 2);
2381
2382	if (fillbits >= ilog2(_PAGE_NO_EXEC))
2383		fill_includes_sw_bits = true;
2384
2385	pr_debug("Entry* registers contain %u fill bits\n", fillbits);
2386}
2387
2388void build_tlb_refill_handler(void)
2389{
2390	/*
2391	 * The refill handler is generated per-CPU, multi-node systems
2392	 * may have local storage for it. The other handlers are only
2393	 * needed once.
2394	 */
2395	static int run_once = 0;
2396
2397	output_pgtable_bits_defines();
2398	check_pabits();
2399
2400#ifdef CONFIG_64BIT
2401	check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
2402#endif
2403
2404	switch (current_cpu_type()) {
2405	case CPU_R2000:
2406	case CPU_R3000:
2407	case CPU_R3000A:
2408	case CPU_R3081E:
2409	case CPU_TX3912:
2410	case CPU_TX3922:
2411	case CPU_TX3927:
2412#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
2413		if (cpu_has_local_ebase)
2414			build_r3000_tlb_refill_handler();
2415		if (!run_once) {
2416			if (!cpu_has_local_ebase)
2417				build_r3000_tlb_refill_handler();
2418			build_setup_pgd();
2419			build_r3000_tlb_load_handler();
2420			build_r3000_tlb_store_handler();
2421			build_r3000_tlb_modify_handler();
2422			flush_tlb_handlers();
2423			run_once++;
2424		}
2425#else
2426		panic("No R3000 TLB refill handler");
2427#endif
2428		break;
2429
2430	case CPU_R6000:
2431	case CPU_R6000A:
2432		panic("No R6000 TLB refill handler yet");
2433		break;
2434
2435	case CPU_R8000:
2436		panic("No R8000 TLB refill handler yet");
2437		break;
2438
2439	default:
2440		if (!run_once) {
2441			scratch_reg = allocate_kscratch();
2442			build_setup_pgd();
 
 
2443			build_r4000_tlb_load_handler();
2444			build_r4000_tlb_store_handler();
2445			build_r4000_tlb_modify_handler();
2446			if (!cpu_has_local_ebase)
2447				build_r4000_tlb_refill_handler();
2448			flush_tlb_handlers();
2449			run_once++;
2450		}
2451		if (cpu_has_local_ebase)
2452			build_r4000_tlb_refill_handler();
2453		if (cpu_has_xpa)
2454			config_xpa_params();
2455		if (cpu_has_htw)
2456			config_htw_params();
2457	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2458}