Loading...
1/*
2 * arch/sh/mm/tlb-flush_64.c
3 *
4 * Copyright (C) 2000, 2001 Paolo Alberelli
5 * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes)
6 * Copyright (C) 2003 - 2012 Paul Mundt
7 *
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
10 * for more details.
11 */
12#include <linux/signal.h>
13#include <linux/rwsem.h>
14#include <linux/sched.h>
15#include <linux/kernel.h>
16#include <linux/errno.h>
17#include <linux/string.h>
18#include <linux/types.h>
19#include <linux/ptrace.h>
20#include <linux/mman.h>
21#include <linux/mm.h>
22#include <linux/smp.h>
23#include <linux/perf_event.h>
24#include <linux/interrupt.h>
25#include <asm/io.h>
26#include <asm/tlb.h>
27#include <asm/uaccess.h>
28#include <asm/pgalloc.h>
29#include <asm/mmu_context.h>
30
31void local_flush_tlb_one(unsigned long asid, unsigned long page)
32{
33 unsigned long long match, pteh=0, lpage;
34 unsigned long tlb;
35
36 /*
37 * Sign-extend based on neff.
38 */
39 lpage = neff_sign_extend(page);
40 match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
41 match |= lpage;
42
43 for_each_itlb_entry(tlb) {
44 asm volatile ("getcfg %1, 0, %0"
45 : "=r" (pteh)
46 : "r" (tlb) );
47
48 if (pteh == match) {
49 __flush_tlb_slot(tlb);
50 break;
51 }
52 }
53
54 for_each_dtlb_entry(tlb) {
55 asm volatile ("getcfg %1, 0, %0"
56 : "=r" (pteh)
57 : "r" (tlb) );
58
59 if (pteh == match) {
60 __flush_tlb_slot(tlb);
61 break;
62 }
63
64 }
65}
66
67void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
68{
69 unsigned long flags;
70
71 if (vma->vm_mm) {
72 page &= PAGE_MASK;
73 local_irq_save(flags);
74 local_flush_tlb_one(get_asid(), page);
75 local_irq_restore(flags);
76 }
77}
78
79void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
80 unsigned long end)
81{
82 unsigned long flags;
83 unsigned long long match, pteh=0, pteh_epn, pteh_low;
84 unsigned long tlb;
85 unsigned int cpu = smp_processor_id();
86 struct mm_struct *mm;
87
88 mm = vma->vm_mm;
89 if (cpu_context(cpu, mm) == NO_CONTEXT)
90 return;
91
92 local_irq_save(flags);
93
94 start &= PAGE_MASK;
95 end &= PAGE_MASK;
96
97 match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;
98
99 /* Flush ITLB */
100 for_each_itlb_entry(tlb) {
101 asm volatile ("getcfg %1, 0, %0"
102 : "=r" (pteh)
103 : "r" (tlb) );
104
105 pteh_epn = pteh & PAGE_MASK;
106 pteh_low = pteh & ~PAGE_MASK;
107
108 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
109 __flush_tlb_slot(tlb);
110 }
111
112 /* Flush DTLB */
113 for_each_dtlb_entry(tlb) {
114 asm volatile ("getcfg %1, 0, %0"
115 : "=r" (pteh)
116 : "r" (tlb) );
117
118 pteh_epn = pteh & PAGE_MASK;
119 pteh_low = pteh & ~PAGE_MASK;
120
121 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
122 __flush_tlb_slot(tlb);
123 }
124
125 local_irq_restore(flags);
126}
127
128void local_flush_tlb_mm(struct mm_struct *mm)
129{
130 unsigned long flags;
131 unsigned int cpu = smp_processor_id();
132
133 if (cpu_context(cpu, mm) == NO_CONTEXT)
134 return;
135
136 local_irq_save(flags);
137
138 cpu_context(cpu, mm) = NO_CONTEXT;
139 if (mm == current->mm)
140 activate_context(mm, cpu);
141
142 local_irq_restore(flags);
143}
144
145void local_flush_tlb_all(void)
146{
147 /* Invalidate all, including shared pages, excluding fixed TLBs */
148 unsigned long flags, tlb;
149
150 local_irq_save(flags);
151
152 /* Flush each ITLB entry */
153 for_each_itlb_entry(tlb)
154 __flush_tlb_slot(tlb);
155
156 /* Flush each DTLB entry */
157 for_each_dtlb_entry(tlb)
158 __flush_tlb_slot(tlb);
159
160 local_irq_restore(flags);
161}
162
163void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
164{
165 /* FIXME: Optimize this later.. */
166 flush_tlb_all();
167}
168
169void __flush_tlb_global(void)
170{
171 flush_tlb_all();
172}
1/*
2 * arch/sh/mm/tlb-flush_64.c
3 *
4 * Copyright (C) 2000, 2001 Paolo Alberelli
5 * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes)
6 * Copyright (C) 2003 - 2009 Paul Mundt
7 *
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
10 * for more details.
11 */
12#include <linux/signal.h>
13#include <linux/rwsem.h>
14#include <linux/sched.h>
15#include <linux/kernel.h>
16#include <linux/errno.h>
17#include <linux/string.h>
18#include <linux/types.h>
19#include <linux/ptrace.h>
20#include <linux/mman.h>
21#include <linux/mm.h>
22#include <linux/smp.h>
23#include <linux/perf_event.h>
24#include <linux/interrupt.h>
25#include <asm/system.h>
26#include <asm/io.h>
27#include <asm/tlb.h>
28#include <asm/uaccess.h>
29#include <asm/pgalloc.h>
30#include <asm/mmu_context.h>
31
32extern void die(const char *,struct pt_regs *,long);
33
34#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" )
35#define PPROT(flag) PFLAG(pgprot_val(prot),flag)
36
37static inline void print_prots(pgprot_t prot)
38{
39 printk("prot is 0x%016llx\n",pgprot_val(prot));
40
41 printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
42 PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
43}
44
45static inline void print_vma(struct vm_area_struct *vma)
46{
47 printk("vma start 0x%08lx\n", vma->vm_start);
48 printk("vma end 0x%08lx\n", vma->vm_end);
49
50 print_prots(vma->vm_page_prot);
51 printk("vm_flags 0x%08lx\n", vma->vm_flags);
52}
53
54static inline void print_task(struct task_struct *tsk)
55{
56 printk("Task pid %d\n", task_pid_nr(tsk));
57}
58
59static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
60{
61 pgd_t *dir;
62 pud_t *pud;
63 pmd_t *pmd;
64 pte_t *pte;
65 pte_t entry;
66
67 dir = pgd_offset(mm, address);
68 if (pgd_none(*dir))
69 return NULL;
70
71 pud = pud_offset(dir, address);
72 if (pud_none(*pud))
73 return NULL;
74
75 pmd = pmd_offset(pud, address);
76 if (pmd_none(*pmd))
77 return NULL;
78
79 pte = pte_offset_kernel(pmd, address);
80 entry = *pte;
81 if (pte_none(entry) || !pte_present(entry))
82 return NULL;
83
84 return pte;
85}
86
87/*
88 * This routine handles page faults. It determines the address,
89 * and the problem, and then passes it off to one of the appropriate
90 * routines.
91 */
92asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
93 unsigned long textaccess, unsigned long address)
94{
95 struct task_struct *tsk;
96 struct mm_struct *mm;
97 struct vm_area_struct * vma;
98 const struct exception_table_entry *fixup;
99 pte_t *pte;
100 int fault;
101
102 /* SIM
103 * Note this is now called with interrupts still disabled
104 * This is to cope with being called for a missing IO port
105 * address with interrupts disabled. This should be fixed as
106 * soon as we have a better 'fast path' miss handler.
107 *
108 * Plus take care how you try and debug this stuff.
109 * For example, writing debug data to a port which you
110 * have just faulted on is not going to work.
111 */
112
113 tsk = current;
114 mm = tsk->mm;
115
116 /* Not an IO address, so reenable interrupts */
117 local_irq_enable();
118
119 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
120
121 /*
122 * If we're in an interrupt or have no user
123 * context, we must not take the fault..
124 */
125 if (in_atomic() || !mm)
126 goto no_context;
127
128 /* TLB misses upon some cache flushes get done under cli() */
129 down_read(&mm->mmap_sem);
130
131 vma = find_vma(mm, address);
132
133 if (!vma) {
134#ifdef DEBUG_FAULT
135 print_task(tsk);
136 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
137 __func__, __LINE__,
138 address,regs->pc,textaccess,writeaccess);
139 show_regs(regs);
140#endif
141 goto bad_area;
142 }
143 if (vma->vm_start <= address) {
144 goto good_area;
145 }
146
147 if (!(vma->vm_flags & VM_GROWSDOWN)) {
148#ifdef DEBUG_FAULT
149 print_task(tsk);
150 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
151 __func__, __LINE__,
152 address,regs->pc,textaccess,writeaccess);
153 show_regs(regs);
154
155 print_vma(vma);
156#endif
157 goto bad_area;
158 }
159 if (expand_stack(vma, address)) {
160#ifdef DEBUG_FAULT
161 print_task(tsk);
162 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
163 __func__, __LINE__,
164 address,regs->pc,textaccess,writeaccess);
165 show_regs(regs);
166#endif
167 goto bad_area;
168 }
169/*
170 * Ok, we have a good vm_area for this memory access, so
171 * we can handle it..
172 */
173good_area:
174 if (textaccess) {
175 if (!(vma->vm_flags & VM_EXEC))
176 goto bad_area;
177 } else {
178 if (writeaccess) {
179 if (!(vma->vm_flags & VM_WRITE))
180 goto bad_area;
181 } else {
182 if (!(vma->vm_flags & VM_READ))
183 goto bad_area;
184 }
185 }
186
187 /*
188 * If for any reason at all we couldn't handle the fault,
189 * make sure we exit gracefully rather than endlessly redo
190 * the fault.
191 */
192 fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0);
193 if (unlikely(fault & VM_FAULT_ERROR)) {
194 if (fault & VM_FAULT_OOM)
195 goto out_of_memory;
196 else if (fault & VM_FAULT_SIGBUS)
197 goto do_sigbus;
198 BUG();
199 }
200
201 if (fault & VM_FAULT_MAJOR) {
202 tsk->maj_flt++;
203 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
204 regs, address);
205 } else {
206 tsk->min_flt++;
207 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
208 regs, address);
209 }
210
211 /* If we get here, the page fault has been handled. Do the TLB refill
212 now from the newly-setup PTE, to avoid having to fault again right
213 away on the same instruction. */
214 pte = lookup_pte (mm, address);
215 if (!pte) {
216 /* From empirical evidence, we can get here, due to
217 !pte_present(pte). (e.g. if a swap-in occurs, and the page
218 is swapped back out again before the process that wanted it
219 gets rescheduled?) */
220 goto no_pte;
221 }
222
223 __do_tlb_refill(address, textaccess, pte);
224
225no_pte:
226
227 up_read(&mm->mmap_sem);
228 return;
229
230/*
231 * Something tried to access memory that isn't in our memory map..
232 * Fix it, but check if it's kernel or user first..
233 */
234bad_area:
235#ifdef DEBUG_FAULT
236 printk("fault:bad area\n");
237#endif
238 up_read(&mm->mmap_sem);
239
240 if (user_mode(regs)) {
241 static int count=0;
242 siginfo_t info;
243 if (count < 4) {
244 /* This is really to help debug faults when starting
245 * usermode, so only need a few */
246 count++;
247 printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
248 address, task_pid_nr(current), current->comm,
249 (unsigned long) regs->pc);
250#if 0
251 show_regs(regs);
252#endif
253 }
254 if (is_global_init(tsk)) {
255 panic("INIT had user mode bad_area\n");
256 }
257 tsk->thread.address = address;
258 tsk->thread.error_code = writeaccess;
259 info.si_signo = SIGSEGV;
260 info.si_errno = 0;
261 info.si_addr = (void *) address;
262 force_sig_info(SIGSEGV, &info, tsk);
263 return;
264 }
265
266no_context:
267#ifdef DEBUG_FAULT
268 printk("fault:No context\n");
269#endif
270 /* Are we prepared to handle this kernel fault? */
271 fixup = search_exception_tables(regs->pc);
272 if (fixup) {
273 regs->pc = fixup->fixup;
274 return;
275 }
276
277/*
278 * Oops. The kernel tried to access some bad page. We'll have to
279 * terminate things with extreme prejudice.
280 *
281 */
282 if (address < PAGE_SIZE)
283 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
284 else
285 printk(KERN_ALERT "Unable to handle kernel paging request");
286 printk(" at virtual address %08lx\n", address);
287 printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
288 die("Oops", regs, writeaccess);
289 do_exit(SIGKILL);
290
291/*
292 * We ran out of memory, or some other thing happened to us that made
293 * us unable to handle the page fault gracefully.
294 */
295out_of_memory:
296 up_read(&mm->mmap_sem);
297 if (!user_mode(regs))
298 goto no_context;
299 pagefault_out_of_memory();
300 return;
301
302do_sigbus:
303 printk("fault:Do sigbus\n");
304 up_read(&mm->mmap_sem);
305
306 /*
307 * Send a sigbus, regardless of whether we were in kernel
308 * or user mode.
309 */
310 tsk->thread.address = address;
311 tsk->thread.error_code = writeaccess;
312 tsk->thread.trap_no = 14;
313 force_sig(SIGBUS, tsk);
314
315 /* Kernel mode? Handle exceptions or die */
316 if (!user_mode(regs))
317 goto no_context;
318}
319
320void local_flush_tlb_one(unsigned long asid, unsigned long page)
321{
322 unsigned long long match, pteh=0, lpage;
323 unsigned long tlb;
324
325 /*
326 * Sign-extend based on neff.
327 */
328 lpage = neff_sign_extend(page);
329 match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
330 match |= lpage;
331
332 for_each_itlb_entry(tlb) {
333 asm volatile ("getcfg %1, 0, %0"
334 : "=r" (pteh)
335 : "r" (tlb) );
336
337 if (pteh == match) {
338 __flush_tlb_slot(tlb);
339 break;
340 }
341 }
342
343 for_each_dtlb_entry(tlb) {
344 asm volatile ("getcfg %1, 0, %0"
345 : "=r" (pteh)
346 : "r" (tlb) );
347
348 if (pteh == match) {
349 __flush_tlb_slot(tlb);
350 break;
351 }
352
353 }
354}
355
356void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
357{
358 unsigned long flags;
359
360 if (vma->vm_mm) {
361 page &= PAGE_MASK;
362 local_irq_save(flags);
363 local_flush_tlb_one(get_asid(), page);
364 local_irq_restore(flags);
365 }
366}
367
368void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
369 unsigned long end)
370{
371 unsigned long flags;
372 unsigned long long match, pteh=0, pteh_epn, pteh_low;
373 unsigned long tlb;
374 unsigned int cpu = smp_processor_id();
375 struct mm_struct *mm;
376
377 mm = vma->vm_mm;
378 if (cpu_context(cpu, mm) == NO_CONTEXT)
379 return;
380
381 local_irq_save(flags);
382
383 start &= PAGE_MASK;
384 end &= PAGE_MASK;
385
386 match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;
387
388 /* Flush ITLB */
389 for_each_itlb_entry(tlb) {
390 asm volatile ("getcfg %1, 0, %0"
391 : "=r" (pteh)
392 : "r" (tlb) );
393
394 pteh_epn = pteh & PAGE_MASK;
395 pteh_low = pteh & ~PAGE_MASK;
396
397 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
398 __flush_tlb_slot(tlb);
399 }
400
401 /* Flush DTLB */
402 for_each_dtlb_entry(tlb) {
403 asm volatile ("getcfg %1, 0, %0"
404 : "=r" (pteh)
405 : "r" (tlb) );
406
407 pteh_epn = pteh & PAGE_MASK;
408 pteh_low = pteh & ~PAGE_MASK;
409
410 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
411 __flush_tlb_slot(tlb);
412 }
413
414 local_irq_restore(flags);
415}
416
417void local_flush_tlb_mm(struct mm_struct *mm)
418{
419 unsigned long flags;
420 unsigned int cpu = smp_processor_id();
421
422 if (cpu_context(cpu, mm) == NO_CONTEXT)
423 return;
424
425 local_irq_save(flags);
426
427 cpu_context(cpu, mm) = NO_CONTEXT;
428 if (mm == current->mm)
429 activate_context(mm, cpu);
430
431 local_irq_restore(flags);
432}
433
434void local_flush_tlb_all(void)
435{
436 /* Invalidate all, including shared pages, excluding fixed TLBs */
437 unsigned long flags, tlb;
438
439 local_irq_save(flags);
440
441 /* Flush each ITLB entry */
442 for_each_itlb_entry(tlb)
443 __flush_tlb_slot(tlb);
444
445 /* Flush each DTLB entry */
446 for_each_dtlb_entry(tlb)
447 __flush_tlb_slot(tlb);
448
449 local_irq_restore(flags);
450}
451
452void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
453{
454 /* FIXME: Optimize this later.. */
455 flush_tlb_all();
456}
457
458void __flush_tlb_global(void)
459{
460 flush_tlb_all();
461}
462
463void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
464{
465}