Linux Audio

Check our new training course

Loading...
v4.17
  1/* SPDX-License-Identifier: GPL-2.0 */
  2/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
  3 *
  4 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
  5 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
  6 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
  7 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
  8 */
  9
 
 10#include <asm/head.h>
 11#include <asm/asi.h>
 12#include <asm/page.h>
 13#include <asm/pgtable.h>
 14#include <asm/tsb.h>
 15
 16	.text
 17	.align		32
 18
 19kvmap_itlb:
 20	/* g6: TAG TARGET */
 21	mov		TLB_TAG_ACCESS, %g4
 22	ldxa		[%g4] ASI_IMMU, %g4
 23
 24	/* The kernel executes in context zero, therefore we do not
 25	 * need to clear the context ID bits out of %g4 here.
 26	 */
 27
 28	/* sun4v_itlb_miss branches here with the missing virtual
 29	 * address already loaded into %g4
 30	 */
 31kvmap_itlb_4v:
 32
 33	/* Catch kernel NULL pointer calls.  */
 34	sethi		%hi(PAGE_SIZE), %g5
 35	cmp		%g4, %g5
 36	blu,pn		%xcc, kvmap_itlb_longpath
 37	 nop
 38
 39	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
 40
 41kvmap_itlb_tsb_miss:
 42	sethi		%hi(LOW_OBP_ADDRESS), %g5
 43	cmp		%g4, %g5
 44	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
 45	 mov		0x1, %g5
 46	sllx		%g5, 32, %g5
 47	cmp		%g4, %g5
 48	blu,pn		%xcc, kvmap_itlb_obp
 49	 nop
 50
 51kvmap_itlb_vmalloc_addr:
 52	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
 53
 54	TSB_LOCK_TAG(%g1, %g2, %g7)
 55	TSB_WRITE(%g1, %g5, %g6)
 56
 57	/* fallthrough to TLB load */
 58
 59kvmap_itlb_load:
 60
 61661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
 62	retry
 63	.section	.sun4v_2insn_patch, "ax"
 64	.word		661b
 65	nop
 66	nop
 67	.previous
 68
 69	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
 70	 * instruction get nop'd out and we get here to branch
 71	 * to the sun4v tlb load code.  The registers are setup
 72	 * as follows:
 73	 *
 74	 * %g4: vaddr
 75	 * %g5: PTE
 76	 * %g6:	TAG
 77	 *
 78	 * The sun4v TLB load wants the PTE in %g3 so we fix that
 79	 * up here.
 80	 */
 81	ba,pt		%xcc, sun4v_itlb_load
 82	 mov		%g5, %g3
 83
 84kvmap_itlb_longpath:
 85
 86661:	rdpr	%pstate, %g5
 87	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
 88	.section .sun4v_2insn_patch, "ax"
 89	.word	661b
 90	SET_GL(1)
 91	nop
 92	.previous
 93
 94	rdpr	%tpc, %g5
 95	ba,pt	%xcc, sparc64_realfault_common
 96	 mov	FAULT_CODE_ITLB, %g4
 97
 98kvmap_itlb_obp:
 99	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
100
101	TSB_LOCK_TAG(%g1, %g2, %g7)
102
103	TSB_WRITE(%g1, %g5, %g6)
104
105	ba,pt		%xcc, kvmap_itlb_load
106	 nop
107
108kvmap_dtlb_obp:
109	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
110
111	TSB_LOCK_TAG(%g1, %g2, %g7)
112
113	TSB_WRITE(%g1, %g5, %g6)
114
115	ba,pt		%xcc, kvmap_dtlb_load
116	 nop
117
118kvmap_linear_early:
119	sethi		%hi(kern_linear_pte_xor), %g7
120	ldx		[%g7 + %lo(kern_linear_pte_xor)], %g2
121	ba,pt		%xcc, kvmap_dtlb_tsb4m_load
122	 xor		%g2, %g4, %g5
123
124	.align		32
125kvmap_dtlb_tsb4m_load:
126	TSB_LOCK_TAG(%g1, %g2, %g7)
127	TSB_WRITE(%g1, %g5, %g6)
128	ba,pt		%xcc, kvmap_dtlb_load
129	 nop
130
131kvmap_dtlb:
132	/* %g6: TAG TARGET */
133	mov		TLB_TAG_ACCESS, %g4
134	ldxa		[%g4] ASI_DMMU, %g4
135
136	/* The kernel executes in context zero, therefore we do not
137	 * need to clear the context ID bits out of %g4 here.
138	 */
139
140	/* sun4v_dtlb_miss branches here with the missing virtual
141	 * address already loaded into %g4
142	 */
143kvmap_dtlb_4v:
144	brgez,pn	%g4, kvmap_dtlb_nonlinear
145	 nop
146
147#ifdef CONFIG_DEBUG_PAGEALLOC
148	/* Index through the base page size TSB even for linear
149	 * mappings when using page allocation debugging.
150	 */
151	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
152#else
153	/* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
154	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
155#endif
156	/* Linear mapping TSB lookup failed.  Fallthrough to kernel
157	 * page table based lookup.
158	 */
159	.globl		kvmap_linear_patch
160kvmap_linear_patch:
161	ba,a,pt		%xcc, kvmap_linear_early
162
163kvmap_dtlb_vmalloc_addr:
164	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
165
166	TSB_LOCK_TAG(%g1, %g2, %g7)
167	TSB_WRITE(%g1, %g5, %g6)
168
169	/* fallthrough to TLB load */
170
171kvmap_dtlb_load:
172
173661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
174	retry
175	.section	.sun4v_2insn_patch, "ax"
176	.word		661b
177	nop
178	nop
179	.previous
180
181	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
182	 * instruction get nop'd out and we get here to branch
183	 * to the sun4v tlb load code.  The registers are setup
184	 * as follows:
185	 *
186	 * %g4: vaddr
187	 * %g5: PTE
188	 * %g6:	TAG
189	 *
190	 * The sun4v TLB load wants the PTE in %g3 so we fix that
191	 * up here.
192	 */
193	ba,pt		%xcc, sun4v_dtlb_load
194	 mov		%g5, %g3
195
196#ifdef CONFIG_SPARSEMEM_VMEMMAP
197kvmap_vmemmap:
198	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
199	ba,a,pt		%xcc, kvmap_dtlb_load
200#endif
201
202kvmap_dtlb_nonlinear:
203	/* Catch kernel NULL pointer derefs.  */
204	sethi		%hi(PAGE_SIZE), %g5
205	cmp		%g4, %g5
206	bleu,pn		%xcc, kvmap_dtlb_longpath
207	 nop
208
209#ifdef CONFIG_SPARSEMEM_VMEMMAP
210	/* Do not use the TSB for vmemmap.  */
211	sethi		%hi(VMEMMAP_BASE), %g5
212	ldx		[%g5 + %lo(VMEMMAP_BASE)], %g5
213	cmp		%g4,%g5
214	bgeu,pn		%xcc, kvmap_vmemmap
215	 nop
216#endif
217
218	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
219
220kvmap_dtlb_tsbmiss:
221	sethi		%hi(MODULES_VADDR), %g5
222	cmp		%g4, %g5
223	blu,pn		%xcc, kvmap_dtlb_longpath
224	 sethi		%hi(VMALLOC_END), %g5
225	ldx		[%g5 + %lo(VMALLOC_END)], %g5
226	cmp		%g4, %g5
227	bgeu,pn		%xcc, kvmap_dtlb_longpath
228	 nop
229
230kvmap_check_obp:
231	sethi		%hi(LOW_OBP_ADDRESS), %g5
232	cmp		%g4, %g5
233	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
234	 mov		0x1, %g5
235	sllx		%g5, 32, %g5
236	cmp		%g4, %g5
237	blu,pn		%xcc, kvmap_dtlb_obp
238	 nop
239	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
240	 nop
241
242kvmap_dtlb_longpath:
243
244661:	rdpr	%pstate, %g5
245	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
246	.section .sun4v_2insn_patch, "ax"
247	.word	661b
248	SET_GL(1)
249	ldxa		[%g0] ASI_SCRATCHPAD, %g5
250	.previous
251
252	rdpr	%tl, %g3
253	cmp	%g3, 1
254
255661:	mov	TLB_TAG_ACCESS, %g4
256	ldxa	[%g4] ASI_DMMU, %g5
257	.section .sun4v_2insn_patch, "ax"
258	.word	661b
259	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
260	nop
261	.previous
262
263	/* The kernel executes in context zero, therefore we do not
264	 * need to clear the context ID bits out of %g5 here.
265	 */
266
267	be,pt	%xcc, sparc64_realfault_common
268	 mov	FAULT_CODE_DTLB, %g4
269	ba,pt	%xcc, winfix_trampoline
270	 nop
v6.8
  1/* SPDX-License-Identifier: GPL-2.0 */
  2/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
  3 *
  4 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
  5 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
  6 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
  7 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
  8 */
  9
 10#include <linux/pgtable.h>
 11#include <asm/head.h>
 12#include <asm/asi.h>
 13#include <asm/page.h>
 
 14#include <asm/tsb.h>
 15
 16	.text
 17	.align		32
 18
 19kvmap_itlb:
 20	/* g6: TAG TARGET */
 21	mov		TLB_TAG_ACCESS, %g4
 22	ldxa		[%g4] ASI_IMMU, %g4
 23
 24	/* The kernel executes in context zero, therefore we do not
 25	 * need to clear the context ID bits out of %g4 here.
 26	 */
 27
 28	/* sun4v_itlb_miss branches here with the missing virtual
 29	 * address already loaded into %g4
 30	 */
 31kvmap_itlb_4v:
 32
 33	/* Catch kernel NULL pointer calls.  */
 34	sethi		%hi(PAGE_SIZE), %g5
 35	cmp		%g4, %g5
 36	blu,pn		%xcc, kvmap_itlb_longpath
 37	 nop
 38
 39	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
 40
 41kvmap_itlb_tsb_miss:
 42	sethi		%hi(LOW_OBP_ADDRESS), %g5
 43	cmp		%g4, %g5
 44	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
 45	 mov		0x1, %g5
 46	sllx		%g5, 32, %g5
 47	cmp		%g4, %g5
 48	blu,pn		%xcc, kvmap_itlb_obp
 49	 nop
 50
 51kvmap_itlb_vmalloc_addr:
 52	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
 53
 54	TSB_LOCK_TAG(%g1, %g2, %g7)
 55	TSB_WRITE(%g1, %g5, %g6)
 56
 57	/* fallthrough to TLB load */
 58
 59kvmap_itlb_load:
 60
 61661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
 62	retry
 63	.section	.sun4v_2insn_patch, "ax"
 64	.word		661b
 65	nop
 66	nop
 67	.previous
 68
 69	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
 70	 * instruction get nop'd out and we get here to branch
 71	 * to the sun4v tlb load code.  The registers are setup
 72	 * as follows:
 73	 *
 74	 * %g4: vaddr
 75	 * %g5: PTE
 76	 * %g6:	TAG
 77	 *
 78	 * The sun4v TLB load wants the PTE in %g3 so we fix that
 79	 * up here.
 80	 */
 81	ba,pt		%xcc, sun4v_itlb_load
 82	 mov		%g5, %g3
 83
 84kvmap_itlb_longpath:
 85
 86661:	rdpr	%pstate, %g5
 87	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
 88	.section .sun4v_2insn_patch, "ax"
 89	.word	661b
 90	SET_GL(1)
 91	nop
 92	.previous
 93
 94	rdpr	%tpc, %g5
 95	ba,pt	%xcc, sparc64_realfault_common
 96	 mov	FAULT_CODE_ITLB, %g4
 97
 98kvmap_itlb_obp:
 99	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
100
101	TSB_LOCK_TAG(%g1, %g2, %g7)
102
103	TSB_WRITE(%g1, %g5, %g6)
104
105	ba,pt		%xcc, kvmap_itlb_load
106	 nop
107
108kvmap_dtlb_obp:
109	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
110
111	TSB_LOCK_TAG(%g1, %g2, %g7)
112
113	TSB_WRITE(%g1, %g5, %g6)
114
115	ba,pt		%xcc, kvmap_dtlb_load
116	 nop
117
118kvmap_linear_early:
119	sethi		%hi(kern_linear_pte_xor), %g7
120	ldx		[%g7 + %lo(kern_linear_pte_xor)], %g2
121	ba,pt		%xcc, kvmap_dtlb_tsb4m_load
122	 xor		%g2, %g4, %g5
123
124	.align		32
125kvmap_dtlb_tsb4m_load:
126	TSB_LOCK_TAG(%g1, %g2, %g7)
127	TSB_WRITE(%g1, %g5, %g6)
128	ba,pt		%xcc, kvmap_dtlb_load
129	 nop
130
131kvmap_dtlb:
132	/* %g6: TAG TARGET */
133	mov		TLB_TAG_ACCESS, %g4
134	ldxa		[%g4] ASI_DMMU, %g4
135
136	/* The kernel executes in context zero, therefore we do not
137	 * need to clear the context ID bits out of %g4 here.
138	 */
139
140	/* sun4v_dtlb_miss branches here with the missing virtual
141	 * address already loaded into %g4
142	 */
143kvmap_dtlb_4v:
144	brgez,pn	%g4, kvmap_dtlb_nonlinear
145	 nop
146
147#ifdef CONFIG_DEBUG_PAGEALLOC
148	/* Index through the base page size TSB even for linear
149	 * mappings when using page allocation debugging.
150	 */
151	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
152#else
153	/* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
154	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
155#endif
156	/* Linear mapping TSB lookup failed.  Fallthrough to kernel
157	 * page table based lookup.
158	 */
159	.globl		kvmap_linear_patch
160kvmap_linear_patch:
161	ba,a,pt		%xcc, kvmap_linear_early
162
163kvmap_dtlb_vmalloc_addr:
164	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
165
166	TSB_LOCK_TAG(%g1, %g2, %g7)
167	TSB_WRITE(%g1, %g5, %g6)
168
169	/* fallthrough to TLB load */
170
171kvmap_dtlb_load:
172
173661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
174	retry
175	.section	.sun4v_2insn_patch, "ax"
176	.word		661b
177	nop
178	nop
179	.previous
180
181	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
182	 * instruction get nop'd out and we get here to branch
183	 * to the sun4v tlb load code.  The registers are setup
184	 * as follows:
185	 *
186	 * %g4: vaddr
187	 * %g5: PTE
188	 * %g6:	TAG
189	 *
190	 * The sun4v TLB load wants the PTE in %g3 so we fix that
191	 * up here.
192	 */
193	ba,pt		%xcc, sun4v_dtlb_load
194	 mov		%g5, %g3
195
196#ifdef CONFIG_SPARSEMEM_VMEMMAP
197kvmap_vmemmap:
198	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
199	ba,a,pt		%xcc, kvmap_dtlb_load
200#endif
201
202kvmap_dtlb_nonlinear:
203	/* Catch kernel NULL pointer derefs.  */
204	sethi		%hi(PAGE_SIZE), %g5
205	cmp		%g4, %g5
206	bleu,pn		%xcc, kvmap_dtlb_longpath
207	 nop
208
209#ifdef CONFIG_SPARSEMEM_VMEMMAP
210	/* Do not use the TSB for vmemmap.  */
211	sethi		%hi(VMEMMAP_BASE), %g5
212	ldx		[%g5 + %lo(VMEMMAP_BASE)], %g5
213	cmp		%g4,%g5
214	bgeu,pn		%xcc, kvmap_vmemmap
215	 nop
216#endif
217
218	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
219
220kvmap_dtlb_tsbmiss:
221	sethi		%hi(MODULES_VADDR), %g5
222	cmp		%g4, %g5
223	blu,pn		%xcc, kvmap_dtlb_longpath
224	 sethi		%hi(VMALLOC_END), %g5
225	ldx		[%g5 + %lo(VMALLOC_END)], %g5
226	cmp		%g4, %g5
227	bgeu,pn		%xcc, kvmap_dtlb_longpath
228	 nop
229
230kvmap_check_obp:
231	sethi		%hi(LOW_OBP_ADDRESS), %g5
232	cmp		%g4, %g5
233	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
234	 mov		0x1, %g5
235	sllx		%g5, 32, %g5
236	cmp		%g4, %g5
237	blu,pn		%xcc, kvmap_dtlb_obp
238	 nop
239	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
240	 nop
241
242kvmap_dtlb_longpath:
243
244661:	rdpr	%pstate, %g5
245	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
246	.section .sun4v_2insn_patch, "ax"
247	.word	661b
248	SET_GL(1)
249	ldxa		[%g0] ASI_SCRATCHPAD, %g5
250	.previous
251
252	rdpr	%tl, %g3
253	cmp	%g3, 1
254
255661:	mov	TLB_TAG_ACCESS, %g4
256	ldxa	[%g4] ASI_DMMU, %g5
257	.section .sun4v_2insn_patch, "ax"
258	.word	661b
259	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
260	nop
261	.previous
262
263	/* The kernel executes in context zero, therefore we do not
264	 * need to clear the context ID bits out of %g5 here.
265	 */
266
267	be,pt	%xcc, sparc64_realfault_common
268	 mov	FAULT_CODE_DTLB, %g4
269	ba,pt	%xcc, winfix_trampoline
270	 nop