Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * r2300.c: R2000 and R3000 specific mmu/cache code.
4 *
5 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
6 *
7 * with a lot of changes to make this thing work for R3000s
8 * Tx39XX R4k style caches added. HK
9 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
10 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
11 * Copyright (C) 2002 Ralf Baechle
12 * Copyright (C) 2002 Maciej W. Rozycki
13 */
14#include <linux/kernel.h>
15#include <linux/sched.h>
16#include <linux/smp.h>
17#include <linux/mm.h>
18
19#include <asm/page.h>
20#include <asm/pgtable.h>
21#include <asm/mmu_context.h>
22#include <asm/tlbmisc.h>
23#include <asm/isadep.h>
24#include <asm/io.h>
25#include <asm/bootinfo.h>
26#include <asm/cpu.h>
27
28#undef DEBUG_TLB
29
30extern void build_tlb_refill_handler(void);
31
32/* CP0 hazard avoidance. */
33#define BARRIER \
34 __asm__ __volatile__( \
35 ".set push\n\t" \
36 ".set noreorder\n\t" \
37 "nop\n\t" \
38 ".set pop\n\t")
39
40int r3k_have_wired_reg; /* Should be in cpu_data? */
41
42/* TLB operations. */
43static void local_flush_tlb_from(int entry)
44{
45 unsigned long old_ctx;
46
47 old_ctx = read_c0_entryhi() & cpu_asid_mask(¤t_cpu_data);
48 write_c0_entrylo0(0);
49 while (entry < current_cpu_data.tlbsize) {
50 write_c0_index(entry << 8);
51 write_c0_entryhi((entry | 0x80000) << 12);
52 entry++; /* BARRIER */
53 tlb_write_indexed();
54 }
55 write_c0_entryhi(old_ctx);
56}
57
58void local_flush_tlb_all(void)
59{
60 unsigned long flags;
61
62#ifdef DEBUG_TLB
63 printk("[tlball]");
64#endif
65 local_irq_save(flags);
66 local_flush_tlb_from(r3k_have_wired_reg ? read_c0_wired() : 8);
67 local_irq_restore(flags);
68}
69
70void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
71 unsigned long end)
72{
73 unsigned long asid_mask = cpu_asid_mask(¤t_cpu_data);
74 struct mm_struct *mm = vma->vm_mm;
75 int cpu = smp_processor_id();
76
77 if (cpu_context(cpu, mm) != 0) {
78 unsigned long size, flags;
79
80#ifdef DEBUG_TLB
81 printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
82 cpu_context(cpu, mm) & asid_mask, start, end);
83#endif
84 local_irq_save(flags);
85 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
86 if (size <= current_cpu_data.tlbsize) {
87 int oldpid = read_c0_entryhi() & asid_mask;
88 int newpid = cpu_context(cpu, mm) & asid_mask;
89
90 start &= PAGE_MASK;
91 end += PAGE_SIZE - 1;
92 end &= PAGE_MASK;
93 while (start < end) {
94 int idx;
95
96 write_c0_entryhi(start | newpid);
97 start += PAGE_SIZE; /* BARRIER */
98 tlb_probe();
99 idx = read_c0_index();
100 write_c0_entrylo0(0);
101 write_c0_entryhi(KSEG0);
102 if (idx < 0) /* BARRIER */
103 continue;
104 tlb_write_indexed();
105 }
106 write_c0_entryhi(oldpid);
107 } else {
108 drop_mmu_context(mm);
109 }
110 local_irq_restore(flags);
111 }
112}
113
114void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
115{
116 unsigned long size, flags;
117
118#ifdef DEBUG_TLB
119 printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
120#endif
121 local_irq_save(flags);
122 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
123 if (size <= current_cpu_data.tlbsize) {
124 int pid = read_c0_entryhi();
125
126 start &= PAGE_MASK;
127 end += PAGE_SIZE - 1;
128 end &= PAGE_MASK;
129
130 while (start < end) {
131 int idx;
132
133 write_c0_entryhi(start);
134 start += PAGE_SIZE; /* BARRIER */
135 tlb_probe();
136 idx = read_c0_index();
137 write_c0_entrylo0(0);
138 write_c0_entryhi(KSEG0);
139 if (idx < 0) /* BARRIER */
140 continue;
141 tlb_write_indexed();
142 }
143 write_c0_entryhi(pid);
144 } else {
145 local_flush_tlb_all();
146 }
147 local_irq_restore(flags);
148}
149
150void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
151{
152 unsigned long asid_mask = cpu_asid_mask(¤t_cpu_data);
153 int cpu = smp_processor_id();
154
155 if (cpu_context(cpu, vma->vm_mm) != 0) {
156 unsigned long flags;
157 int oldpid, newpid, idx;
158
159#ifdef DEBUG_TLB
160 printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
161#endif
162 newpid = cpu_context(cpu, vma->vm_mm) & asid_mask;
163 page &= PAGE_MASK;
164 local_irq_save(flags);
165 oldpid = read_c0_entryhi() & asid_mask;
166 write_c0_entryhi(page | newpid);
167 BARRIER;
168 tlb_probe();
169 idx = read_c0_index();
170 write_c0_entrylo0(0);
171 write_c0_entryhi(KSEG0);
172 if (idx < 0) /* BARRIER */
173 goto finish;
174 tlb_write_indexed();
175
176finish:
177 write_c0_entryhi(oldpid);
178 local_irq_restore(flags);
179 }
180}
181
182void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
183{
184 unsigned long asid_mask = cpu_asid_mask(¤t_cpu_data);
185 unsigned long flags;
186 int idx, pid;
187
188 /*
189 * Handle debugger faulting in for debugee.
190 */
191 if (current->active_mm != vma->vm_mm)
192 return;
193
194 pid = read_c0_entryhi() & asid_mask;
195
196#ifdef DEBUG_TLB
197 if ((pid != (cpu_context(cpu, vma->vm_mm) & asid_mask)) || (cpu_context(cpu, vma->vm_mm) == 0)) {
198 printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
199 (cpu_context(cpu, vma->vm_mm)), pid);
200 }
201#endif
202
203 local_irq_save(flags);
204 address &= PAGE_MASK;
205 write_c0_entryhi(address | pid);
206 BARRIER;
207 tlb_probe();
208 idx = read_c0_index();
209 write_c0_entrylo0(pte_val(pte));
210 write_c0_entryhi(address | pid);
211 if (idx < 0) { /* BARRIER */
212 tlb_write_random();
213 } else {
214 tlb_write_indexed();
215 }
216 write_c0_entryhi(pid);
217 local_irq_restore(flags);
218}
219
220void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
221 unsigned long entryhi, unsigned long pagemask)
222{
223 unsigned long asid_mask = cpu_asid_mask(¤t_cpu_data);
224 unsigned long flags;
225 unsigned long old_ctx;
226 static unsigned long wired = 0;
227
228 if (r3k_have_wired_reg) { /* TX39XX */
229 unsigned long old_pagemask;
230 unsigned long w;
231
232#ifdef DEBUG_TLB
233 printk("[tlbwired<entry lo0 %8x, hi %8x\n, pagemask %8x>]\n",
234 entrylo0, entryhi, pagemask);
235#endif
236
237 local_irq_save(flags);
238 /* Save old context and create impossible VPN2 value */
239 old_ctx = read_c0_entryhi() & asid_mask;
240 old_pagemask = read_c0_pagemask();
241 w = read_c0_wired();
242 write_c0_wired(w + 1);
243 write_c0_index(w << 8);
244 write_c0_pagemask(pagemask);
245 write_c0_entryhi(entryhi);
246 write_c0_entrylo0(entrylo0);
247 BARRIER;
248 tlb_write_indexed();
249
250 write_c0_entryhi(old_ctx);
251 write_c0_pagemask(old_pagemask);
252 local_flush_tlb_all();
253 local_irq_restore(flags);
254
255 } else if (wired < 8) {
256#ifdef DEBUG_TLB
257 printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
258 entrylo0, entryhi);
259#endif
260
261 local_irq_save(flags);
262 old_ctx = read_c0_entryhi() & asid_mask;
263 write_c0_entrylo0(entrylo0);
264 write_c0_entryhi(entryhi);
265 write_c0_index(wired);
266 wired++; /* BARRIER */
267 tlb_write_indexed();
268 write_c0_entryhi(old_ctx);
269 local_flush_tlb_all();
270 local_irq_restore(flags);
271 }
272}
273
274void tlb_init(void)
275{
276 switch (current_cpu_type()) {
277 case CPU_TX3922:
278 case CPU_TX3927:
279 r3k_have_wired_reg = 1;
280 write_c0_wired(0); /* Set to 8 on reset... */
281 break;
282 }
283 local_flush_tlb_from(0);
284 build_tlb_refill_handler();
285}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * r2300.c: R2000 and R3000 specific mmu/cache code.
4 *
5 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
6 *
7 * with a lot of changes to make this thing work for R3000s
8 * Tx39XX R4k style caches added. HK
9 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
10 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
11 * Copyright (C) 2002 Ralf Baechle
12 * Copyright (C) 2002 Maciej W. Rozycki
13 */
14#include <linux/kernel.h>
15#include <linux/sched.h>
16#include <linux/smp.h>
17#include <linux/mm.h>
18
19#include <asm/page.h>
20#include <asm/pgtable.h>
21#include <asm/mmu_context.h>
22#include <asm/tlbmisc.h>
23#include <asm/isadep.h>
24#include <asm/io.h>
25#include <asm/bootinfo.h>
26#include <asm/cpu.h>
27
28#undef DEBUG_TLB
29
30extern void build_tlb_refill_handler(void);
31
32/* CP0 hazard avoidance. */
33#define BARRIER \
34 __asm__ __volatile__( \
35 ".set push\n\t" \
36 ".set noreorder\n\t" \
37 "nop\n\t" \
38 ".set pop\n\t")
39
40int r3k_have_wired_reg; /* Should be in cpu_data? */
41
42/* TLB operations. */
43static void local_flush_tlb_from(int entry)
44{
45 unsigned long old_ctx;
46
47 old_ctx = read_c0_entryhi() & cpu_asid_mask(¤t_cpu_data);
48 write_c0_entrylo0(0);
49 while (entry < current_cpu_data.tlbsize) {
50 write_c0_index(entry << 8);
51 write_c0_entryhi((entry | 0x80000) << 12);
52 entry++; /* BARRIER */
53 tlb_write_indexed();
54 }
55 write_c0_entryhi(old_ctx);
56}
57
58void local_flush_tlb_all(void)
59{
60 unsigned long flags;
61
62#ifdef DEBUG_TLB
63 printk("[tlball]");
64#endif
65 local_irq_save(flags);
66 local_flush_tlb_from(r3k_have_wired_reg ? read_c0_wired() : 8);
67 local_irq_restore(flags);
68}
69
70void local_flush_tlb_mm(struct mm_struct *mm)
71{
72 int cpu = smp_processor_id();
73
74 if (cpu_context(cpu, mm) != 0) {
75#ifdef DEBUG_TLB
76 printk("[tlbmm<%lu>]", (unsigned long)cpu_context(cpu, mm));
77#endif
78 drop_mmu_context(mm, cpu);
79 }
80}
81
82void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
83 unsigned long end)
84{
85 unsigned long asid_mask = cpu_asid_mask(¤t_cpu_data);
86 struct mm_struct *mm = vma->vm_mm;
87 int cpu = smp_processor_id();
88
89 if (cpu_context(cpu, mm) != 0) {
90 unsigned long size, flags;
91
92#ifdef DEBUG_TLB
93 printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
94 cpu_context(cpu, mm) & asid_mask, start, end);
95#endif
96 local_irq_save(flags);
97 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
98 if (size <= current_cpu_data.tlbsize) {
99 int oldpid = read_c0_entryhi() & asid_mask;
100 int newpid = cpu_context(cpu, mm) & asid_mask;
101
102 start &= PAGE_MASK;
103 end += PAGE_SIZE - 1;
104 end &= PAGE_MASK;
105 while (start < end) {
106 int idx;
107
108 write_c0_entryhi(start | newpid);
109 start += PAGE_SIZE; /* BARRIER */
110 tlb_probe();
111 idx = read_c0_index();
112 write_c0_entrylo0(0);
113 write_c0_entryhi(KSEG0);
114 if (idx < 0) /* BARRIER */
115 continue;
116 tlb_write_indexed();
117 }
118 write_c0_entryhi(oldpid);
119 } else {
120 drop_mmu_context(mm, cpu);
121 }
122 local_irq_restore(flags);
123 }
124}
125
126void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
127{
128 unsigned long size, flags;
129
130#ifdef DEBUG_TLB
131 printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
132#endif
133 local_irq_save(flags);
134 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
135 if (size <= current_cpu_data.tlbsize) {
136 int pid = read_c0_entryhi();
137
138 start &= PAGE_MASK;
139 end += PAGE_SIZE - 1;
140 end &= PAGE_MASK;
141
142 while (start < end) {
143 int idx;
144
145 write_c0_entryhi(start);
146 start += PAGE_SIZE; /* BARRIER */
147 tlb_probe();
148 idx = read_c0_index();
149 write_c0_entrylo0(0);
150 write_c0_entryhi(KSEG0);
151 if (idx < 0) /* BARRIER */
152 continue;
153 tlb_write_indexed();
154 }
155 write_c0_entryhi(pid);
156 } else {
157 local_flush_tlb_all();
158 }
159 local_irq_restore(flags);
160}
161
162void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
163{
164 unsigned long asid_mask = cpu_asid_mask(¤t_cpu_data);
165 int cpu = smp_processor_id();
166
167 if (cpu_context(cpu, vma->vm_mm) != 0) {
168 unsigned long flags;
169 int oldpid, newpid, idx;
170
171#ifdef DEBUG_TLB
172 printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
173#endif
174 newpid = cpu_context(cpu, vma->vm_mm) & asid_mask;
175 page &= PAGE_MASK;
176 local_irq_save(flags);
177 oldpid = read_c0_entryhi() & asid_mask;
178 write_c0_entryhi(page | newpid);
179 BARRIER;
180 tlb_probe();
181 idx = read_c0_index();
182 write_c0_entrylo0(0);
183 write_c0_entryhi(KSEG0);
184 if (idx < 0) /* BARRIER */
185 goto finish;
186 tlb_write_indexed();
187
188finish:
189 write_c0_entryhi(oldpid);
190 local_irq_restore(flags);
191 }
192}
193
194void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
195{
196 unsigned long asid_mask = cpu_asid_mask(¤t_cpu_data);
197 unsigned long flags;
198 int idx, pid;
199
200 /*
201 * Handle debugger faulting in for debugee.
202 */
203 if (current->active_mm != vma->vm_mm)
204 return;
205
206 pid = read_c0_entryhi() & asid_mask;
207
208#ifdef DEBUG_TLB
209 if ((pid != (cpu_context(cpu, vma->vm_mm) & asid_mask)) || (cpu_context(cpu, vma->vm_mm) == 0)) {
210 printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
211 (cpu_context(cpu, vma->vm_mm)), pid);
212 }
213#endif
214
215 local_irq_save(flags);
216 address &= PAGE_MASK;
217 write_c0_entryhi(address | pid);
218 BARRIER;
219 tlb_probe();
220 idx = read_c0_index();
221 write_c0_entrylo0(pte_val(pte));
222 write_c0_entryhi(address | pid);
223 if (idx < 0) { /* BARRIER */
224 tlb_write_random();
225 } else {
226 tlb_write_indexed();
227 }
228 write_c0_entryhi(pid);
229 local_irq_restore(flags);
230}
231
232void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
233 unsigned long entryhi, unsigned long pagemask)
234{
235 unsigned long asid_mask = cpu_asid_mask(¤t_cpu_data);
236 unsigned long flags;
237 unsigned long old_ctx;
238 static unsigned long wired = 0;
239
240 if (r3k_have_wired_reg) { /* TX39XX */
241 unsigned long old_pagemask;
242 unsigned long w;
243
244#ifdef DEBUG_TLB
245 printk("[tlbwired<entry lo0 %8x, hi %8x\n, pagemask %8x>]\n",
246 entrylo0, entryhi, pagemask);
247#endif
248
249 local_irq_save(flags);
250 /* Save old context and create impossible VPN2 value */
251 old_ctx = read_c0_entryhi() & asid_mask;
252 old_pagemask = read_c0_pagemask();
253 w = read_c0_wired();
254 write_c0_wired(w + 1);
255 write_c0_index(w << 8);
256 write_c0_pagemask(pagemask);
257 write_c0_entryhi(entryhi);
258 write_c0_entrylo0(entrylo0);
259 BARRIER;
260 tlb_write_indexed();
261
262 write_c0_entryhi(old_ctx);
263 write_c0_pagemask(old_pagemask);
264 local_flush_tlb_all();
265 local_irq_restore(flags);
266
267 } else if (wired < 8) {
268#ifdef DEBUG_TLB
269 printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
270 entrylo0, entryhi);
271#endif
272
273 local_irq_save(flags);
274 old_ctx = read_c0_entryhi() & asid_mask;
275 write_c0_entrylo0(entrylo0);
276 write_c0_entryhi(entryhi);
277 write_c0_index(wired);
278 wired++; /* BARRIER */
279 tlb_write_indexed();
280 write_c0_entryhi(old_ctx);
281 local_flush_tlb_all();
282 local_irq_restore(flags);
283 }
284}
285
286void tlb_init(void)
287{
288 switch (current_cpu_type()) {
289 case CPU_TX3922:
290 case CPU_TX3927:
291 r3k_have_wired_reg = 1;
292 write_c0_wired(0); /* Set to 8 on reset... */
293 break;
294 }
295 local_flush_tlb_from(0);
296 build_tlb_refill_handler();
297}