1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 * ultra.S: Don't expand these all over the place...
   4 *
   5 * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net)
   6 */
   7
   8#include <linux/pgtable.h>
   9#include <asm/asi.h>
  10#include <asm/page.h>
  11#include <asm/spitfire.h>
  12#include <asm/mmu_context.h>
  13#include <asm/mmu.h>
  14#include <asm/pil.h>
  15#include <asm/head.h>
  16#include <asm/thread_info.h>
  17#include <asm/cacheflush.h>
  18#include <asm/hypervisor.h>
  19#include <asm/cpudata.h>
  20
  21	/* Basically, most of the Spitfire vs. Cheetah madness
  22	 * has to do with the fact that Cheetah does not support
  23	 * IMMU flushes out of the secondary context.  Someone needs
  24	 * to throw a south lake birthday party for the folks
  25	 * in Microelectronics who refused to fix this shit.
  26	 */
  27
  28	/* This file is meant to be read efficiently by the CPU, not humans.
  29	 * Staraj sie tego nikomu nie pierdolnac...
  30	 */
  31	.text
  32	.align		32
  33	.globl		__flush_tlb_mm
  34__flush_tlb_mm:		/* 19 insns */
  35	/* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
  36	ldxa		[%o1] ASI_DMMU, %g2
  37	cmp		%g2, %o0
  38	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
  39	 mov		0x50, %g3
  40	stxa		%g0, [%g3] ASI_DMMU_DEMAP
  41	stxa		%g0, [%g3] ASI_IMMU_DEMAP
  42	sethi		%hi(KERNBASE), %g3
  43	flush		%g3
  44	retl
  45	 nop
  46	nop
  47	nop
  48	nop
  49	nop
  50	nop
  51	nop
  52	nop
  53	nop
  54	nop
  55
  56	.align		32
  57	.globl		__flush_tlb_page
  58__flush_tlb_page:	/* 22 insns */
  59	/* %o0 = context, %o1 = vaddr */
  60	rdpr		%pstate, %g7
  61	andn		%g7, PSTATE_IE, %g2
  62	wrpr		%g2, %pstate
  63	mov		SECONDARY_CONTEXT, %o4
  64	ldxa		[%o4] ASI_DMMU, %g2
  65	stxa		%o0, [%o4] ASI_DMMU
  66	andcc		%o1, 1, %g0
  67	andn		%o1, 1, %o3
  68	be,pn		%icc, 1f
  69	 or		%o3, 0x10, %o3
  70	stxa		%g0, [%o3] ASI_IMMU_DEMAP
  711:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
  72	membar		#Sync
  73	stxa		%g2, [%o4] ASI_DMMU
  74	sethi		%hi(KERNBASE), %o4
  75	flush		%o4
  76	retl
  77	 wrpr		%g7, 0x0, %pstate
  78	nop
  79	nop
  80	nop
  81	nop
  82
  83	.align		32
  84	.globl		__flush_tlb_pending
  85__flush_tlb_pending:	/* 27 insns */
  86	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
  87	rdpr		%pstate, %g7
  88	sllx		%o1, 3, %o1
  89	andn		%g7, PSTATE_IE, %g2
  90	wrpr		%g2, %pstate
  91	mov		SECONDARY_CONTEXT, %o4
  92	ldxa		[%o4] ASI_DMMU, %g2
  93	stxa		%o0, [%o4] ASI_DMMU
  941:	sub		%o1, (1 << 3), %o1
  95	ldx		[%o2 + %o1], %o3
  96	andcc		%o3, 1, %g0
  97	andn		%o3, 1, %o3
  98	be,pn		%icc, 2f
  99	 or		%o3, 0x10, %o3
 100	stxa		%g0, [%o3] ASI_IMMU_DEMAP
 1012:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
 102	membar		#Sync
 103	brnz,pt		%o1, 1b
 104	 nop
 105	stxa		%g2, [%o4] ASI_DMMU
 106	sethi		%hi(KERNBASE), %o4
 107	flush		%o4
 108	retl
 109	 wrpr		%g7, 0x0, %pstate
 110	nop
 111	nop
 112	nop
 113	nop
 114
 115	.align		32
 116	.globl		__flush_tlb_kernel_range
 117__flush_tlb_kernel_range:	/* 31 insns */
 118	/* %o0=start, %o1=end */
 119	cmp		%o0, %o1
 120	be,pn		%xcc, 2f
 121	 sub		%o1, %o0, %o3
 122	srlx		%o3, 18, %o4
 123	brnz,pn		%o4, __spitfire_flush_tlb_kernel_range_slow
 124	 sethi		%hi(PAGE_SIZE), %o4
 125	sub		%o3, %o4, %o3
 126	or		%o0, 0x20, %o0		! Nucleus
 1271:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
 128	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
 129	membar		#Sync
 130	brnz,pt		%o3, 1b
 131	 sub		%o3, %o4, %o3
 1322:	sethi		%hi(KERNBASE), %o3
 133	flush		%o3
 134	retl
 135	 nop
 136	nop
 137	nop
 138	nop
 139	nop
 140	nop
 141	nop
 142	nop
 143	nop
 144	nop
 145	nop
 146	nop
 147	nop
 148	nop
 149	nop
 150
 151__spitfire_flush_tlb_kernel_range_slow:
 152	mov		63 * 8, %o4
 1531:	ldxa		[%o4] ASI_ITLB_DATA_ACCESS, %o3
 154	andcc		%o3, 0x40, %g0			/* _PAGE_L_4U */
 155	bne,pn		%xcc, 2f
 156	 mov		TLB_TAG_ACCESS, %o3
 157	stxa		%g0, [%o3] ASI_IMMU
 158	stxa		%g0, [%o4] ASI_ITLB_DATA_ACCESS
 159	membar		#Sync
 1602:	ldxa		[%o4] ASI_DTLB_DATA_ACCESS, %o3
 161	andcc		%o3, 0x40, %g0
 162	bne,pn		%xcc, 2f
 163	 mov		TLB_TAG_ACCESS, %o3
 164	stxa		%g0, [%o3] ASI_DMMU
 165	stxa		%g0, [%o4] ASI_DTLB_DATA_ACCESS
 166	membar		#Sync
 1672:	sub		%o4, 8, %o4
 168	brgez,pt	%o4, 1b
 169	 nop
 170	retl
 171	 nop
 172
 173__spitfire_flush_tlb_mm_slow:
 174	rdpr		%pstate, %g1
 175	wrpr		%g1, PSTATE_IE, %pstate
 176	stxa		%o0, [%o1] ASI_DMMU
 177	stxa		%g0, [%g3] ASI_DMMU_DEMAP
 178	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 179	flush		%g6
 180	stxa		%g2, [%o1] ASI_DMMU
 181	sethi		%hi(KERNBASE), %o1
 182	flush		%o1
 183	retl
 184	 wrpr		%g1, 0, %pstate
 185
 186/*
 187 * The following code flushes one page_size worth.
 188 */
 189	.section .kprobes.text, "ax"
 190	.align		32
 191	.globl		__flush_icache_page
 192__flush_icache_page:	/* %o0 = phys_page */
 193	srlx		%o0, PAGE_SHIFT, %o0
 194	sethi		%hi(PAGE_OFFSET), %g1
 195	sllx		%o0, PAGE_SHIFT, %o0
 196	sethi		%hi(PAGE_SIZE), %g2
 197	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
 198	add		%o0, %g1, %o0
 1991:	subcc		%g2, 32, %g2
 200	bne,pt		%icc, 1b
 201	 flush		%o0 + %g2
 202	retl
 203	 nop
 204
 205#ifdef DCACHE_ALIASING_POSSIBLE
 206
 207#if (PAGE_SHIFT != 13)
 208#error only page shift of 13 is supported by dcache flush
 209#endif
 210
 211#define DTAG_MASK 0x3
 212
 213	/* This routine is Spitfire specific so the hardcoded
 214	 * D-cache size and line-size are OK.
 215	 */
 216	.align		64
 217	.globl		__flush_dcache_page
 218__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
 219	sethi		%hi(PAGE_OFFSET), %g1
 220	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
 221	sub		%o0, %g1, %o0			! physical address
 222	srlx		%o0, 11, %o0			! make D-cache TAG
 223	sethi		%hi(1 << 14), %o2		! D-cache size
 224	sub		%o2, (1 << 5), %o2		! D-cache line size
 2251:	ldxa		[%o2] ASI_DCACHE_TAG, %o3	! load D-cache TAG
 226	andcc		%o3, DTAG_MASK, %g0		! Valid?
 227	be,pn		%xcc, 2f			! Nope, branch
 228	 andn		%o3, DTAG_MASK, %o3		! Clear valid bits
 229	cmp		%o3, %o0			! TAG match?
 230	bne,pt		%xcc, 2f			! Nope, branch
 231	 nop
 232	stxa		%g0, [%o2] ASI_DCACHE_TAG	! Invalidate TAG
 233	membar		#Sync
 2342:	brnz,pt		%o2, 1b
 235	 sub		%o2, (1 << 5), %o2		! D-cache line size
 236
 237	/* The I-cache does not snoop local stores so we
 238	 * better flush that too when necessary.
 239	 */
 240	brnz,pt		%o1, __flush_icache_page
 241	 sllx		%o0, 11, %o0
 242	retl
 243	 nop
 244
 245#endif /* DCACHE_ALIASING_POSSIBLE */
 246
 247	.previous
 248
 249	/* Cheetah specific versions, patched at boot time. */
 250__cheetah_flush_tlb_mm: /* 19 insns */
 251	rdpr		%pstate, %g7
 252	andn		%g7, PSTATE_IE, %g2
 253	wrpr		%g2, 0x0, %pstate
 254	wrpr		%g0, 1, %tl
 255	mov		PRIMARY_CONTEXT, %o2
 256	mov		0x40, %g3
 257	ldxa		[%o2] ASI_DMMU, %g2
 258	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o1
 259	sllx		%o1, CTX_PGSZ1_NUC_SHIFT, %o1
 260	or		%o0, %o1, %o0	/* Preserve nucleus page size fields */
 261	stxa		%o0, [%o2] ASI_DMMU
 262	stxa		%g0, [%g3] ASI_DMMU_DEMAP
 263	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 264	stxa		%g2, [%o2] ASI_DMMU
 265	sethi		%hi(KERNBASE), %o2
 266	flush		%o2
 267	wrpr		%g0, 0, %tl
 268	retl
 269	 wrpr		%g7, 0x0, %pstate
 270
 271__cheetah_flush_tlb_page:	/* 22 insns */
 272	/* %o0 = context, %o1 = vaddr */
 273	rdpr		%pstate, %g7
 274	andn		%g7, PSTATE_IE, %g2
 275	wrpr		%g2, 0x0, %pstate
 276	wrpr		%g0, 1, %tl
 277	mov		PRIMARY_CONTEXT, %o4
 278	ldxa		[%o4] ASI_DMMU, %g2
 279	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
 280	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
 281	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
 282	stxa		%o0, [%o4] ASI_DMMU
 283	andcc		%o1, 1, %g0
 284	be,pn		%icc, 1f
 285	 andn		%o1, 1, %o3
 286	stxa		%g0, [%o3] ASI_IMMU_DEMAP
 2871:	stxa		%g0, [%o3] ASI_DMMU_DEMAP	
 288	membar		#Sync
 289	stxa		%g2, [%o4] ASI_DMMU
 290	sethi		%hi(KERNBASE), %o4
 291	flush		%o4
 292	wrpr		%g0, 0, %tl
 293	retl
 294	 wrpr		%g7, 0x0, %pstate
 295
 296__cheetah_flush_tlb_pending:	/* 27 insns */
 297	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
 298	rdpr		%pstate, %g7
 299	sllx		%o1, 3, %o1
 300	andn		%g7, PSTATE_IE, %g2
 301	wrpr		%g2, 0x0, %pstate
 302	wrpr		%g0, 1, %tl
 303	mov		PRIMARY_CONTEXT, %o4
 304	ldxa		[%o4] ASI_DMMU, %g2
 305	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
 306	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
 307	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
 308	stxa		%o0, [%o4] ASI_DMMU
 3091:	sub		%o1, (1 << 3), %o1
 310	ldx		[%o2 + %o1], %o3
 311	andcc		%o3, 1, %g0
 312	be,pn		%icc, 2f
 313	 andn		%o3, 1, %o3
 314	stxa		%g0, [%o3] ASI_IMMU_DEMAP
 3152:	stxa		%g0, [%o3] ASI_DMMU_DEMAP	
 316	membar		#Sync
 317	brnz,pt		%o1, 1b
 318	 nop
 319	stxa		%g2, [%o4] ASI_DMMU
 320	sethi		%hi(KERNBASE), %o4
 321	flush		%o4
 322	wrpr		%g0, 0, %tl
 323	retl
 324	 wrpr		%g7, 0x0, %pstate
 325
 326__cheetah_flush_tlb_kernel_range:	/* 31 insns */
 327	/* %o0=start, %o1=end */
 328	cmp		%o0, %o1
 329	be,pn		%xcc, 2f
 330	 sub		%o1, %o0, %o3
 331	srlx		%o3, 18, %o4
 332	brnz,pn		%o4, 3f
 333	 sethi		%hi(PAGE_SIZE), %o4
 334	sub		%o3, %o4, %o3
 335	or		%o0, 0x20, %o0		! Nucleus
 3361:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
 337	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
 338	membar		#Sync
 339	brnz,pt		%o3, 1b
 340	 sub		%o3, %o4, %o3
 3412:	sethi		%hi(KERNBASE), %o3
 342	flush		%o3
 343	retl
 344	 nop
 3453:	mov		0x80, %o4
 346	stxa		%g0, [%o4] ASI_DMMU_DEMAP
 347	membar		#Sync
 348	stxa		%g0, [%o4] ASI_IMMU_DEMAP
 349	membar		#Sync
 350	retl
 351	 nop
 352	nop
 353	nop
 354	nop
 355	nop
 356	nop
 357	nop
 358	nop
 359
 360#ifdef DCACHE_ALIASING_POSSIBLE
 361__cheetah_flush_dcache_page: /* 11 insns */
 362	sethi		%hi(PAGE_OFFSET), %g1
 363	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
 364	sub		%o0, %g1, %o0
 365	sethi		%hi(PAGE_SIZE), %o4
 3661:	subcc		%o4, (1 << 5), %o4
 367	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
 368	membar		#Sync
 369	bne,pt		%icc, 1b
 370	 nop
 371	retl		/* I-cache flush never needed on Cheetah, see callers. */
 372	 nop
 373#endif /* DCACHE_ALIASING_POSSIBLE */
 374
 375	/* Hypervisor specific versions, patched at boot time.  */
 376__hypervisor_tlb_tl0_error:
 377	save		%sp, -192, %sp
 378	mov		%i0, %o0
 379	call		hypervisor_tlbop_error
 380	 mov		%i1, %o1
 381	ret
 382	 restore
 383
 384__hypervisor_flush_tlb_mm: /* 19 insns */
 385	mov		%o0, %o2	/* ARG2: mmu context */
 386	mov		0, %o0		/* ARG0: CPU lists unimplemented */
 387	mov		0, %o1		/* ARG1: CPU lists unimplemented */
 388	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
 389	mov		HV_FAST_MMU_DEMAP_CTX, %o5
 390	ta		HV_FAST_TRAP
 391	brnz,pn		%o0, 1f
 392	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
 393	retl
 394	 nop
 3951:	sethi		%hi(__hypervisor_tlb_tl0_error), %o5
 396	jmpl		%o5 + %lo(__hypervisor_tlb_tl0_error), %g0
 397	 nop
 398	nop
 399	nop
 400	nop
 401	nop
 402	nop
 403	nop
 404
 405__hypervisor_flush_tlb_page: /* 22 insns */
 406	/* %o0 = context, %o1 = vaddr */
 407	mov		%o0, %g2
 408	mov		%o1, %o0              /* ARG0: vaddr + IMMU-bit */
 409	mov		%g2, %o1	      /* ARG1: mmu context */
 410	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
 411	srlx		%o0, PAGE_SHIFT, %o0
 412	sllx		%o0, PAGE_SHIFT, %o0
 413	ta		HV_MMU_UNMAP_ADDR_TRAP
 414	brnz,pn		%o0, 1f
 415	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
 416	retl
 417	 nop
 4181:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
 419	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
 420	 nop
 421	nop
 422	nop
 423	nop
 424	nop
 425	nop
 426	nop
 427	nop
 428	nop
 429
 430__hypervisor_flush_tlb_pending: /* 27 insns */
 431	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
 432	sllx		%o1, 3, %g1
 433	mov		%o2, %g2
 434	mov		%o0, %g3
 4351:	sub		%g1, (1 << 3), %g1
 436	ldx		[%g2 + %g1], %o0      /* ARG0: vaddr + IMMU-bit */
 437	mov		%g3, %o1	      /* ARG1: mmu context */
 438	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
 439	srlx		%o0, PAGE_SHIFT, %o0
 440	sllx		%o0, PAGE_SHIFT, %o0
 441	ta		HV_MMU_UNMAP_ADDR_TRAP
 442	brnz,pn		%o0, 1f
 443	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
 444	brnz,pt		%g1, 1b
 445	 nop
 446	retl
 447	 nop
 4481:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
 449	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
 450	 nop
 451	nop
 452	nop
 453	nop
 454	nop
 455	nop
 456	nop
 457	nop
 458	nop
 459
 460__hypervisor_flush_tlb_kernel_range: /* 31 insns */
 461	/* %o0=start, %o1=end */
 462	cmp		%o0, %o1
 463	be,pn		%xcc, 2f
 464	 sub		%o1, %o0, %g2
 465	srlx		%g2, 18, %g3
 466	brnz,pn		%g3, 4f
 467	 mov		%o0, %g1
 468	sethi		%hi(PAGE_SIZE), %g3
 469	sub		%g2, %g3, %g2
 4701:	add		%g1, %g2, %o0	/* ARG0: virtual address */
 471	mov		0, %o1		/* ARG1: mmu context */
 472	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
 473	ta		HV_MMU_UNMAP_ADDR_TRAP
 474	brnz,pn		%o0, 3f
 475	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
 476	brnz,pt		%g2, 1b
 477	 sub		%g2, %g3, %g2
 4782:	retl
 479	 nop
 4803:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
 481	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
 482	 nop
 4834:	mov		0, %o0		/* ARG0: CPU lists unimplemented */
 484	mov		0, %o1		/* ARG1: CPU lists unimplemented */
 485	mov		0, %o2		/* ARG2: mmu context == nucleus */
 486	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
 487	mov		HV_FAST_MMU_DEMAP_CTX, %o5
 488	ta		HV_FAST_TRAP
 489	brnz,pn		%o0, 3b
 490	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
 491	retl
 492	 nop
 493
 494#ifdef DCACHE_ALIASING_POSSIBLE
 495	/* XXX Niagara and friends have an 8K cache, so no aliasing is
 496	 * XXX possible, but nothing explicit in the Hypervisor API
 497	 * XXX guarantees this.
 498	 */
 499__hypervisor_flush_dcache_page:	/* 2 insns */
 500	retl
 501	 nop
 502#endif
 503
 504tlb_patch_one:
 5051:	lduw		[%o1], %g1
 506	stw		%g1, [%o0]
 507	flush		%o0
 508	subcc		%o2, 1, %o2
 509	add		%o1, 4, %o1
 510	bne,pt		%icc, 1b
 511	 add		%o0, 4, %o0
 512	retl
 513	 nop
 514
 515#ifdef CONFIG_SMP
 516	/* These are all called by the slaves of a cross call, at
 517	 * trap level 1, with interrupts fully disabled.
 518	 *
 519	 * Register usage:
 520	 *   %g5	mm->context	(all tlb flushes)
 521	 *   %g1	address arg 1	(tlb page and range flushes)
 522	 *   %g7	address arg 2	(tlb range flush only)
 523	 *
 524	 *   %g6	scratch 1
 525	 *   %g2	scratch 2
 526	 *   %g3	scratch 3
 527	 *   %g4	scratch 4
 528	 */
 529	.align		32
 530	.globl		xcall_flush_tlb_mm
 531xcall_flush_tlb_mm:	/* 24 insns */
 532	mov		PRIMARY_CONTEXT, %g2
 533	ldxa		[%g2] ASI_DMMU, %g3
 534	srlx		%g3, CTX_PGSZ1_NUC_SHIFT, %g4
 535	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
 536	or		%g5, %g4, %g5	/* Preserve nucleus page size fields */
 537	stxa		%g5, [%g2] ASI_DMMU
 538	mov		0x40, %g4
 539	stxa		%g0, [%g4] ASI_DMMU_DEMAP
 540	stxa		%g0, [%g4] ASI_IMMU_DEMAP
 541	stxa		%g3, [%g2] ASI_DMMU
 542	retry
 543	nop
 544	nop
 545	nop
 546	nop
 547	nop
 548	nop
 549	nop
 550	nop
 551	nop
 552	nop
 553	nop
 554	nop
 555	nop
 556
 557	.globl		xcall_flush_tlb_page
 558xcall_flush_tlb_page:	/* 20 insns */
 559	/* %g5=context, %g1=vaddr */
 560	mov		PRIMARY_CONTEXT, %g4
 561	ldxa		[%g4] ASI_DMMU, %g2
 562	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %g4
 563	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
 564	or		%g5, %g4, %g5
 565	mov		PRIMARY_CONTEXT, %g4
 566	stxa		%g5, [%g4] ASI_DMMU
 567	andcc		%g1, 0x1, %g0
 568	be,pn		%icc, 2f
 569	 andn		%g1, 0x1, %g5
 570	stxa		%g0, [%g5] ASI_IMMU_DEMAP
 5712:	stxa		%g0, [%g5] ASI_DMMU_DEMAP
 572	membar		#Sync
 573	stxa		%g2, [%g4] ASI_DMMU
 574	retry
 575	nop
 576	nop
 577	nop
 578	nop
 579	nop
 580
 581	.globl		xcall_flush_tlb_kernel_range
 582xcall_flush_tlb_kernel_range:	/* 44 insns */
 583	sethi		%hi(PAGE_SIZE - 1), %g2
 584	or		%g2, %lo(PAGE_SIZE - 1), %g2
 585	andn		%g1, %g2, %g1
 586	andn		%g7, %g2, %g7
 587	sub		%g7, %g1, %g3
 588	srlx		%g3, 18, %g2
 589	brnz,pn		%g2, 2f
 590	 sethi		%hi(PAGE_SIZE), %g2
 591	sub		%g3, %g2, %g3
 592	or		%g1, 0x20, %g1		! Nucleus
 5931:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
 594	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
 595	membar		#Sync
 596	brnz,pt		%g3, 1b
 597	 sub		%g3, %g2, %g3
 598	retry
 5992:	mov		63 * 8, %g1
 6001:	ldxa		[%g1] ASI_ITLB_DATA_ACCESS, %g2
 601	andcc		%g2, 0x40, %g0			/* _PAGE_L_4U */
 602	bne,pn		%xcc, 2f
 603	 mov		TLB_TAG_ACCESS, %g2
 604	stxa		%g0, [%g2] ASI_IMMU
 605	stxa		%g0, [%g1] ASI_ITLB_DATA_ACCESS
 606	membar		#Sync
 6072:	ldxa		[%g1] ASI_DTLB_DATA_ACCESS, %g2
 608	andcc		%g2, 0x40, %g0
 609	bne,pn		%xcc, 2f
 610	 mov		TLB_TAG_ACCESS, %g2
 611	stxa		%g0, [%g2] ASI_DMMU
 612	stxa		%g0, [%g1] ASI_DTLB_DATA_ACCESS
 613	membar		#Sync
 6142:	sub		%g1, 8, %g1
 615	brgez,pt	%g1, 1b
 616	 nop
 617	retry
 618	nop
 619	nop
 620	nop
 621	nop
 622	nop
 623	nop
 624	nop
 625	nop
 626	nop
 627
 628	/* This runs in a very controlled environment, so we do
 629	 * not need to worry about BH races etc.
 630	 */
 631	.globl		xcall_sync_tick
 632xcall_sync_tick:
 633
 634661:	rdpr		%pstate, %g2
 635	wrpr		%g2, PSTATE_IG | PSTATE_AG, %pstate
 636	.section	.sun4v_2insn_patch, "ax"
 637	.word		661b
 638	nop
 639	nop
 640	.previous
 641
 642	rdpr		%pil, %g2
 643	wrpr		%g0, PIL_NORMAL_MAX, %pil
 644	sethi		%hi(109f), %g7
 645	b,pt		%xcc, etrap_irq
 646109:	 or		%g7, %lo(109b), %g7
 647#ifdef CONFIG_TRACE_IRQFLAGS
 648	call		trace_hardirqs_off
 649	 nop
 650#endif
 651	call		smp_synchronize_tick_client
 652	 nop
 653	b		rtrap_xcall
 654	 ldx		[%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
 655
 656	.globl		xcall_fetch_glob_regs
 657xcall_fetch_glob_regs:
 658	sethi		%hi(global_cpu_snapshot), %g1
 659	or		%g1, %lo(global_cpu_snapshot), %g1
 660	__GET_CPUID(%g2)
 661	sllx		%g2, 6, %g3
 662	add		%g1, %g3, %g1
 663	rdpr		%tstate, %g7
 664	stx		%g7, [%g1 + GR_SNAP_TSTATE]
 665	rdpr		%tpc, %g7
 666	stx		%g7, [%g1 + GR_SNAP_TPC]
 667	rdpr		%tnpc, %g7
 668	stx		%g7, [%g1 + GR_SNAP_TNPC]
 669	stx		%o7, [%g1 + GR_SNAP_O7]
 670	stx		%i7, [%g1 + GR_SNAP_I7]
 671	/* Don't try this at home kids... */
 672	rdpr		%cwp, %g3
 673	sub		%g3, 1, %g7
 674	wrpr		%g7, %cwp
 675	mov		%i7, %g7
 676	wrpr		%g3, %cwp
 677	stx		%g7, [%g1 + GR_SNAP_RPC]
 678	sethi		%hi(trap_block), %g7
 679	or		%g7, %lo(trap_block), %g7
 680	sllx		%g2, TRAP_BLOCK_SZ_SHIFT, %g2
 681	add		%g7, %g2, %g7
 682	ldx		[%g7 + TRAP_PER_CPU_THREAD], %g3
 683	stx		%g3, [%g1 + GR_SNAP_THREAD]
 684	retry
 685
 686	.globl		xcall_fetch_glob_pmu
 687xcall_fetch_glob_pmu:
 688	sethi		%hi(global_cpu_snapshot), %g1
 689	or		%g1, %lo(global_cpu_snapshot), %g1
 690	__GET_CPUID(%g2)
 691	sllx		%g2, 6, %g3
 692	add		%g1, %g3, %g1
 693	rd		%pic, %g7
 694	stx		%g7, [%g1 + (4 * 8)]
 695	rd		%pcr, %g7
 696	stx		%g7, [%g1 + (0 * 8)]
 697	retry
 698
 699	.globl		xcall_fetch_glob_pmu_n4
 700xcall_fetch_glob_pmu_n4:
 701	sethi		%hi(global_cpu_snapshot), %g1
 702	or		%g1, %lo(global_cpu_snapshot), %g1
 703	__GET_CPUID(%g2)
 704	sllx		%g2, 6, %g3
 705	add		%g1, %g3, %g1
 706
 707	ldxa		[%g0] ASI_PIC, %g7
 708	stx		%g7, [%g1 + (4 * 8)]
 709	mov		0x08, %g3
 710	ldxa		[%g3] ASI_PIC, %g7
 711	stx		%g7, [%g1 + (5 * 8)]
 712	mov		0x10, %g3
 713	ldxa		[%g3] ASI_PIC, %g7
 714	stx		%g7, [%g1 + (6 * 8)]
 715	mov		0x18, %g3
 716	ldxa		[%g3] ASI_PIC, %g7
 717	stx		%g7, [%g1 + (7 * 8)]
 718
 719	mov		%o0, %g2
 720	mov		%o1, %g3
 721	mov		%o5, %g7
 722
 723	mov		HV_FAST_VT_GET_PERFREG, %o5
 724	mov		3, %o0
 725	ta		HV_FAST_TRAP
 726	stx		%o1, [%g1 + (3 * 8)]
 727	mov		HV_FAST_VT_GET_PERFREG, %o5
 728	mov		2, %o0
 729	ta		HV_FAST_TRAP
 730	stx		%o1, [%g1 + (2 * 8)]
 731	mov		HV_FAST_VT_GET_PERFREG, %o5
 732	mov		1, %o0
 733	ta		HV_FAST_TRAP
 734	stx		%o1, [%g1 + (1 * 8)]
 735	mov		HV_FAST_VT_GET_PERFREG, %o5
 736	mov		0, %o0
 737	ta		HV_FAST_TRAP
 738	stx		%o1, [%g1 + (0 * 8)]
 739
 740	mov		%g2, %o0
 741	mov		%g3, %o1
 742	mov		%g7, %o5
 743
 744	retry
 745
 746__cheetah_xcall_flush_tlb_kernel_range:	/* 44 insns */
 747	sethi		%hi(PAGE_SIZE - 1), %g2
 748	or		%g2, %lo(PAGE_SIZE - 1), %g2
 749	andn		%g1, %g2, %g1
 750	andn		%g7, %g2, %g7
 751	sub		%g7, %g1, %g3
 752	srlx		%g3, 18, %g2
 753	brnz,pn		%g2, 2f
 754	 sethi		%hi(PAGE_SIZE), %g2
 755	sub		%g3, %g2, %g3
 756	or		%g1, 0x20, %g1		! Nucleus
 7571:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
 758	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
 759	membar		#Sync
 760	brnz,pt		%g3, 1b
 761	 sub		%g3, %g2, %g3
 762	retry
 7632:	mov		0x80, %g2
 764	stxa		%g0, [%g2] ASI_DMMU_DEMAP
 765	membar		#Sync
 766	stxa		%g0, [%g2] ASI_IMMU_DEMAP
 767	membar		#Sync
 768	retry
 769	nop
 770	nop
 771	nop
 772	nop
 773	nop
 774	nop
 775	nop
 776	nop
 777	nop
 778	nop
 779	nop
 780	nop
 781	nop
 782	nop
 783	nop
 784	nop
 785	nop
 786	nop
 787	nop
 788	nop
 789	nop
 790	nop
 791
 792#ifdef DCACHE_ALIASING_POSSIBLE
 793	.align		32
 794	.globl		xcall_flush_dcache_page_cheetah
 795xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
 796	sethi		%hi(PAGE_SIZE), %g3
 7971:	subcc		%g3, (1 << 5), %g3
 798	stxa		%g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
 799	membar		#Sync
 800	bne,pt		%icc, 1b
 801	 nop
 802	retry
 803	nop
 804#endif /* DCACHE_ALIASING_POSSIBLE */
 805
 806	.globl		xcall_flush_dcache_page_spitfire
 807xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
 808				     %g7 == kernel page virtual address
 809				     %g5 == (page->mapping != NULL)  */
 810#ifdef DCACHE_ALIASING_POSSIBLE
 811	srlx		%g1, (13 - 2), %g1	! Form tag comparitor
 812	sethi		%hi(L1DCACHE_SIZE), %g3	! D$ size == 16K
 813	sub		%g3, (1 << 5), %g3	! D$ linesize == 32
 8141:	ldxa		[%g3] ASI_DCACHE_TAG, %g2
 815	andcc		%g2, 0x3, %g0
 816	be,pn		%xcc, 2f
 817	 andn		%g2, 0x3, %g2
 818	cmp		%g2, %g1
 819
 820	bne,pt		%xcc, 2f
 821	 nop
 822	stxa		%g0, [%g3] ASI_DCACHE_TAG
 823	membar		#Sync
 8242:	cmp		%g3, 0
 825	bne,pt		%xcc, 1b
 826	 sub		%g3, (1 << 5), %g3
 827
 828	brz,pn		%g5, 2f
 829#endif /* DCACHE_ALIASING_POSSIBLE */
 830	 sethi		%hi(PAGE_SIZE), %g3
 831
 8321:	flush		%g7
 833	subcc		%g3, (1 << 5), %g3
 834	bne,pt		%icc, 1b
 835	 add		%g7, (1 << 5), %g7
 836
 8372:	retry
 838	nop
 839	nop
 840
 841	/* %g5:	error
 842	 * %g6:	tlb op
 843	 */
 844__hypervisor_tlb_xcall_error:
 845	mov	%g5, %g4
 846	mov	%g6, %g5
 847	ba,pt	%xcc, etrap
 848	 rd	%pc, %g7
 849	mov	%l4, %o0
 850	call	hypervisor_tlbop_error_xcall
 851	 mov	%l5, %o1
 852	ba,a,pt	%xcc, rtrap
 853
 854	.globl		__hypervisor_xcall_flush_tlb_mm
 855__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
 856	/* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
 857	mov		%o0, %g2
 858	mov		%o1, %g3
 859	mov		%o2, %g4
 860	mov		%o3, %g1
 861	mov		%o5, %g7
 862	clr		%o0		/* ARG0: CPU lists unimplemented */
 863	clr		%o1		/* ARG1: CPU lists unimplemented */
 864	mov		%g5, %o2	/* ARG2: mmu context */
 865	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
 866	mov		HV_FAST_MMU_DEMAP_CTX, %o5
 867	ta		HV_FAST_TRAP
 868	mov		HV_FAST_MMU_DEMAP_CTX, %g6
 869	brnz,pn		%o0, 1f
 870	 mov		%o0, %g5
 871	mov		%g2, %o0
 872	mov		%g3, %o1
 873	mov		%g4, %o2
 874	mov		%g1, %o3
 875	mov		%g7, %o5
 876	membar		#Sync
 877	retry
 8781:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
 879	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
 880	 nop
 881
 882	.globl		__hypervisor_xcall_flush_tlb_page
 883__hypervisor_xcall_flush_tlb_page: /* 20 insns */
 884	/* %g5=ctx, %g1=vaddr */
 885	mov		%o0, %g2
 886	mov		%o1, %g3
 887	mov		%o2, %g4
 888	mov		%g1, %o0	        /* ARG0: virtual address */
 889	mov		%g5, %o1		/* ARG1: mmu context */
 890	mov		HV_MMU_ALL, %o2		/* ARG2: flags */
 891	srlx		%o0, PAGE_SHIFT, %o0
 892	sllx		%o0, PAGE_SHIFT, %o0
 893	ta		HV_MMU_UNMAP_ADDR_TRAP
 894	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
 895	brnz,a,pn	%o0, 1f
 896	 mov		%o0, %g5
 897	mov		%g2, %o0
 898	mov		%g3, %o1
 899	mov		%g4, %o2
 900	membar		#Sync
 901	retry
 9021:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
 903	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
 904	 nop
 905
 906	.globl		__hypervisor_xcall_flush_tlb_kernel_range
 907__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
 908	/* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
 909	sethi		%hi(PAGE_SIZE - 1), %g2
 910	or		%g2, %lo(PAGE_SIZE - 1), %g2
 911	andn		%g1, %g2, %g1
 912	andn		%g7, %g2, %g7
 913	sub		%g7, %g1, %g3
 914	srlx		%g3, 18, %g7
 915	add		%g2, 1, %g2
 916	sub		%g3, %g2, %g3
 917	mov		%o0, %g2
 918	mov		%o1, %g4
 919	brnz,pn		%g7, 2f
 920	 mov		%o2, %g7
 9211:	add		%g1, %g3, %o0	/* ARG0: virtual address */
 922	mov		0, %o1		/* ARG1: mmu context */
 923	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
 924	ta		HV_MMU_UNMAP_ADDR_TRAP
 925	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
 926	brnz,pn		%o0, 1f
 927	 mov		%o0, %g5
 928	sethi		%hi(PAGE_SIZE), %o2
 929	brnz,pt		%g3, 1b
 930	 sub		%g3, %o2, %g3
 9315:	mov		%g2, %o0
 932	mov		%g4, %o1
 933	mov		%g7, %o2
 934	membar		#Sync
 935	retry
 9361:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
 937	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
 938	 nop
 9392:	mov		%o3, %g1
 940	mov		%o5, %g3
 941	mov		0, %o0		/* ARG0: CPU lists unimplemented */
 942	mov		0, %o1		/* ARG1: CPU lists unimplemented */
 943	mov		0, %o2		/* ARG2: mmu context == nucleus */
 944	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
 945	mov		HV_FAST_MMU_DEMAP_CTX, %o5
 946	ta		HV_FAST_TRAP
 947	mov		%g1, %o3
 948	brz,pt		%o0, 5b
 949	 mov		%g3, %o5
 950	mov		HV_FAST_MMU_DEMAP_CTX, %g6
 951	ba,pt		%xcc, 1b
 952	 clr		%g5
 953
 954	/* These just get rescheduled to PIL vectors. */
 955	.globl		xcall_call_function
 956xcall_call_function:
 957	wr		%g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
 958	retry
 959
 960	.globl		xcall_call_function_single
 961xcall_call_function_single:
 962	wr		%g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
 963	retry
 964
 965	.globl		xcall_receive_signal
 966xcall_receive_signal:
 967	wr		%g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
 968	retry
 969
 970	.globl		xcall_capture
 971xcall_capture:
 972	wr		%g0, (1 << PIL_SMP_CAPTURE), %set_softint
 973	retry
 974
 975#ifdef CONFIG_KGDB
 976	.globl		xcall_kgdb_capture
 977xcall_kgdb_capture:
 978	wr		%g0, (1 << PIL_KGDB_CAPTURE), %set_softint
 979	retry
 980#endif
 981
 982#endif /* CONFIG_SMP */
 983
 984	.globl		cheetah_patch_cachetlbops
 985cheetah_patch_cachetlbops:
 986	save		%sp, -128, %sp
 987
 988	sethi		%hi(__flush_tlb_mm), %o0
 989	or		%o0, %lo(__flush_tlb_mm), %o0
 990	sethi		%hi(__cheetah_flush_tlb_mm), %o1
 991	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
 992	call		tlb_patch_one
 993	 mov		19, %o2
 994
 995	sethi		%hi(__flush_tlb_page), %o0
 996	or		%o0, %lo(__flush_tlb_page), %o0
 997	sethi		%hi(__cheetah_flush_tlb_page), %o1
 998	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
 999	call		tlb_patch_one
1000	 mov		22, %o2
1001
1002	sethi		%hi(__flush_tlb_pending), %o0
1003	or		%o0, %lo(__flush_tlb_pending), %o0
1004	sethi		%hi(__cheetah_flush_tlb_pending), %o1
1005	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
1006	call		tlb_patch_one
1007	 mov		27, %o2
1008
1009	sethi		%hi(__flush_tlb_kernel_range), %o0
1010	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1011	sethi		%hi(__cheetah_flush_tlb_kernel_range), %o1
1012	or		%o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
1013	call		tlb_patch_one
1014	 mov		31, %o2
1015
1016#ifdef DCACHE_ALIASING_POSSIBLE
1017	sethi		%hi(__flush_dcache_page), %o0
1018	or		%o0, %lo(__flush_dcache_page), %o0
1019	sethi		%hi(__cheetah_flush_dcache_page), %o1
1020	or		%o1, %lo(__cheetah_flush_dcache_page), %o1
1021	call		tlb_patch_one
1022	 mov		11, %o2
1023#endif /* DCACHE_ALIASING_POSSIBLE */
1024
1025#ifdef CONFIG_SMP
1026	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1027	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1028	sethi		%hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
1029	or		%o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
1030	call		tlb_patch_one
1031	 mov		44, %o2
1032#endif /* CONFIG_SMP */
1033
1034	ret
1035	 restore
1036
1037	.globl		hypervisor_patch_cachetlbops
1038hypervisor_patch_cachetlbops:
1039	save		%sp, -128, %sp
1040
1041	sethi		%hi(__flush_tlb_mm), %o0
1042	or		%o0, %lo(__flush_tlb_mm), %o0
1043	sethi		%hi(__hypervisor_flush_tlb_mm), %o1
1044	or		%o1, %lo(__hypervisor_flush_tlb_mm), %o1
1045	call		tlb_patch_one
1046	 mov		19, %o2
1047
1048	sethi		%hi(__flush_tlb_page), %o0
1049	or		%o0, %lo(__flush_tlb_page), %o0
1050	sethi		%hi(__hypervisor_flush_tlb_page), %o1
1051	or		%o1, %lo(__hypervisor_flush_tlb_page), %o1
1052	call		tlb_patch_one
1053	 mov		22, %o2
1054
1055	sethi		%hi(__flush_tlb_pending), %o0
1056	or		%o0, %lo(__flush_tlb_pending), %o0
1057	sethi		%hi(__hypervisor_flush_tlb_pending), %o1
1058	or		%o1, %lo(__hypervisor_flush_tlb_pending), %o1
1059	call		tlb_patch_one
1060	 mov		27, %o2
1061
1062	sethi		%hi(__flush_tlb_kernel_range), %o0
1063	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1064	sethi		%hi(__hypervisor_flush_tlb_kernel_range), %o1
1065	or		%o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
1066	call		tlb_patch_one
1067	 mov		31, %o2
1068
1069#ifdef DCACHE_ALIASING_POSSIBLE
1070	sethi		%hi(__flush_dcache_page), %o0
1071	or		%o0, %lo(__flush_dcache_page), %o0
1072	sethi		%hi(__hypervisor_flush_dcache_page), %o1
1073	or		%o1, %lo(__hypervisor_flush_dcache_page), %o1
1074	call		tlb_patch_one
1075	 mov		2, %o2
1076#endif /* DCACHE_ALIASING_POSSIBLE */
1077
1078#ifdef CONFIG_SMP
1079	sethi		%hi(xcall_flush_tlb_mm), %o0
1080	or		%o0, %lo(xcall_flush_tlb_mm), %o0
1081	sethi		%hi(__hypervisor_xcall_flush_tlb_mm), %o1
1082	or		%o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
1083	call		tlb_patch_one
1084	 mov		24, %o2
1085
1086	sethi		%hi(xcall_flush_tlb_page), %o0
1087	or		%o0, %lo(xcall_flush_tlb_page), %o0
1088	sethi		%hi(__hypervisor_xcall_flush_tlb_page), %o1
1089	or		%o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
1090	call		tlb_patch_one
1091	 mov		20, %o2
1092
1093	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1094	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1095	sethi		%hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1096	or		%o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1097	call		tlb_patch_one
1098	 mov		44, %o2
1099#endif /* CONFIG_SMP */
1100
1101	ret
1102	 restore