Loading...
1/*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 *
14 * From i386 code copyright (C) 1995 Linus Torvalds
15 */
16
17#include <linux/signal.h>
18#include <linux/sched.h>
19#include <linux/kernel.h>
20#include <linux/errno.h>
21#include <linux/string.h>
22#include <linux/types.h>
23#include <linux/ptrace.h>
24#include <linux/mman.h>
25#include <linux/mm.h>
26#include <linux/smp.h>
27#include <linux/interrupt.h>
28#include <linux/init.h>
29#include <linux/tty.h>
30#include <linux/vt_kern.h> /* For unblank_screen() */
31#include <linux/highmem.h>
32#include <linux/module.h>
33#include <linux/kprobes.h>
34#include <linux/hugetlb.h>
35#include <linux/syscalls.h>
36#include <linux/uaccess.h>
37
38#include <asm/pgalloc.h>
39#include <asm/sections.h>
40#include <asm/traps.h>
41#include <asm/syscalls.h>
42
43#include <arch/interrupts.h>
44
45static noinline void force_sig_info_fault(const char *type, int si_signo,
46 int si_code, unsigned long address,
47 int fault_num,
48 struct task_struct *tsk,
49 struct pt_regs *regs)
50{
51 siginfo_t info;
52
53 if (unlikely(tsk->pid < 2)) {
54 panic("Signal %d (code %d) at %#lx sent to %s!",
55 si_signo, si_code & 0xffff, address,
56 is_idle_task(tsk) ? "the idle task" : "init");
57 }
58
59 info.si_signo = si_signo;
60 info.si_errno = 0;
61 info.si_code = si_code;
62 info.si_addr = (void __user *)address;
63 info.si_trapno = fault_num;
64 trace_unhandled_signal(type, regs, address, si_signo);
65 force_sig_info(si_signo, &info, tsk);
66}
67
68#ifndef __tilegx__
69/*
70 * Synthesize the fault a PL0 process would get by doing a word-load of
71 * an unaligned address or a high kernel address.
72 */
73SYSCALL_DEFINE2(cmpxchg_badaddr, unsigned long, address,
74 struct pt_regs *, regs)
75{
76 if (address >= PAGE_OFFSET)
77 force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR,
78 address, INT_DTLB_MISS, current, regs);
79 else
80 force_sig_info_fault("atomic alignment fault", SIGBUS,
81 BUS_ADRALN, address,
82 INT_UNALIGN_DATA, current, regs);
83
84 /*
85 * Adjust pc to point at the actual instruction, which is unusual
86 * for syscalls normally, but is appropriate when we are claiming
87 * that a syscall swint1 caused a page fault or bus error.
88 */
89 regs->pc -= 8;
90
91 /*
92 * Mark this as a caller-save interrupt, like a normal page fault,
93 * so that when we go through the signal handler path we will
94 * properly restore r0, r1, and r2 for the signal handler arguments.
95 */
96 regs->flags |= PT_FLAGS_CALLER_SAVES;
97
98 return 0;
99}
100#endif
101
102static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
103{
104 unsigned index = pgd_index(address);
105 pgd_t *pgd_k;
106 pud_t *pud, *pud_k;
107 pmd_t *pmd, *pmd_k;
108
109 pgd += index;
110 pgd_k = init_mm.pgd + index;
111
112 if (!pgd_present(*pgd_k))
113 return NULL;
114
115 pud = pud_offset(pgd, address);
116 pud_k = pud_offset(pgd_k, address);
117 if (!pud_present(*pud_k))
118 return NULL;
119
120 pmd = pmd_offset(pud, address);
121 pmd_k = pmd_offset(pud_k, address);
122 if (!pmd_present(*pmd_k))
123 return NULL;
124 if (!pmd_present(*pmd)) {
125 set_pmd(pmd, *pmd_k);
126 arch_flush_lazy_mmu_mode();
127 } else
128 BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k));
129 return pmd_k;
130}
131
132/*
133 * Handle a fault on the vmalloc area.
134 */
135static inline int vmalloc_fault(pgd_t *pgd, unsigned long address)
136{
137 pmd_t *pmd_k;
138 pte_t *pte_k;
139
140 /* Make sure we are in vmalloc area */
141 if (!(address >= VMALLOC_START && address < VMALLOC_END))
142 return -1;
143
144 /*
145 * Synchronize this task's top level page-table
146 * with the 'reference' page table.
147 */
148 pmd_k = vmalloc_sync_one(pgd, address);
149 if (!pmd_k)
150 return -1;
151 if (pmd_huge(*pmd_k))
152 return 0; /* support TILE huge_vmap() API */
153 pte_k = pte_offset_kernel(pmd_k, address);
154 if (!pte_present(*pte_k))
155 return -1;
156 return 0;
157}
158
159/* Wait until this PTE has completed migration. */
160static void wait_for_migration(pte_t *pte)
161{
162 if (pte_migrating(*pte)) {
163 /*
164 * Wait until the migrater fixes up this pte.
165 * We scale the loop count by the clock rate so we'll wait for
166 * a few seconds here.
167 */
168 int retries = 0;
169 int bound = get_clock_rate();
170 while (pte_migrating(*pte)) {
171 barrier();
172 if (++retries > bound)
173 panic("Hit migrating PTE (%#llx) and"
174 " page PFN %#lx still migrating",
175 pte->val, pte_pfn(*pte));
176 }
177 }
178}
179
180/*
181 * It's not generally safe to use "current" to get the page table pointer,
182 * since we might be running an oprofile interrupt in the middle of a
183 * task switch.
184 */
185static pgd_t *get_current_pgd(void)
186{
187 HV_Context ctx = hv_inquire_context();
188 unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT;
189 struct page *pgd_page = pfn_to_page(pgd_pfn);
190 BUG_ON(PageHighMem(pgd_page));
191 return (pgd_t *) __va(ctx.page_table);
192}
193
194/*
195 * We can receive a page fault from a migrating PTE at any time.
196 * Handle it by just waiting until the fault resolves.
197 *
198 * It's also possible to get a migrating kernel PTE that resolves
199 * itself during the downcall from hypervisor to Linux. We just check
200 * here to see if the PTE seems valid, and if so we retry it.
201 *
202 * NOTE! We MUST NOT take any locks for this case. We may be in an
203 * interrupt or a critical region, and must do as little as possible.
204 * Similarly, we can't use atomic ops here, since we may be handling a
205 * fault caused by an atomic op access.
206 *
207 * If we find a migrating PTE while we're in an NMI context, and we're
208 * at a PC that has a registered exception handler, we don't wait,
209 * since this thread may (e.g.) have been interrupted while migrating
210 * its own stack, which would then cause us to self-deadlock.
211 */
212static int handle_migrating_pte(pgd_t *pgd, int fault_num,
213 unsigned long address, unsigned long pc,
214 int is_kernel_mode, int write)
215{
216 pud_t *pud;
217 pmd_t *pmd;
218 pte_t *pte;
219 pte_t pteval;
220
221 if (pgd_addr_invalid(address))
222 return 0;
223
224 pgd += pgd_index(address);
225 pud = pud_offset(pgd, address);
226 if (!pud || !pud_present(*pud))
227 return 0;
228 pmd = pmd_offset(pud, address);
229 if (!pmd || !pmd_present(*pmd))
230 return 0;
231 pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) :
232 pte_offset_kernel(pmd, address);
233 pteval = *pte;
234 if (pte_migrating(pteval)) {
235 if (in_nmi() && search_exception_tables(pc))
236 return 0;
237 wait_for_migration(pte);
238 return 1;
239 }
240
241 if (!is_kernel_mode || !pte_present(pteval))
242 return 0;
243 if (fault_num == INT_ITLB_MISS) {
244 if (pte_exec(pteval))
245 return 1;
246 } else if (write) {
247 if (pte_write(pteval))
248 return 1;
249 } else {
250 if (pte_read(pteval))
251 return 1;
252 }
253
254 return 0;
255}
256
257/*
258 * This routine is responsible for faulting in user pages.
259 * It passes the work off to one of the appropriate routines.
260 * It returns true if the fault was successfully handled.
261 */
262static int handle_page_fault(struct pt_regs *regs,
263 int fault_num,
264 int is_page_fault,
265 unsigned long address,
266 int write)
267{
268 struct task_struct *tsk;
269 struct mm_struct *mm;
270 struct vm_area_struct *vma;
271 unsigned long stack_offset;
272 int fault;
273 int si_code;
274 int is_kernel_mode;
275 pgd_t *pgd;
276 unsigned int flags;
277
278 /* on TILE, protection faults are always writes */
279 if (!is_page_fault)
280 write = 1;
281
282 flags = (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
283 (write ? FAULT_FLAG_WRITE : 0));
284
285 is_kernel_mode = (EX1_PL(regs->ex1) != USER_PL);
286
287 tsk = validate_current();
288
289 /*
290 * Check to see if we might be overwriting the stack, and bail
291 * out if so. The page fault code is a relatively likely
292 * place to get trapped in an infinite regress, and once we
293 * overwrite the whole stack, it becomes very hard to recover.
294 */
295 stack_offset = stack_pointer & (THREAD_SIZE-1);
296 if (stack_offset < THREAD_SIZE / 8) {
297 pr_alert("Potential stack overrun: sp %#lx\n",
298 stack_pointer);
299 show_regs(regs);
300 pr_alert("Killing current process %d/%s\n",
301 tsk->pid, tsk->comm);
302 do_group_exit(SIGKILL);
303 }
304
305 /*
306 * Early on, we need to check for migrating PTE entries;
307 * see homecache.c. If we find a migrating PTE, we wait until
308 * the backing page claims to be done migrating, then we proceed.
309 * For kernel PTEs, we rewrite the PTE and return and retry.
310 * Otherwise, we treat the fault like a normal "no PTE" fault,
311 * rather than trying to patch up the existing PTE.
312 */
313 pgd = get_current_pgd();
314 if (handle_migrating_pte(pgd, fault_num, address, regs->pc,
315 is_kernel_mode, write))
316 return 1;
317
318 si_code = SEGV_MAPERR;
319
320 /*
321 * We fault-in kernel-space virtual memory on-demand. The
322 * 'reference' page table is init_mm.pgd.
323 *
324 * NOTE! We MUST NOT take any locks for this case. We may
325 * be in an interrupt or a critical region, and should
326 * only copy the information from the master page table,
327 * nothing more.
328 *
329 * This verifies that the fault happens in kernel space
330 * and that the fault was not a protection fault.
331 */
332 if (unlikely(address >= TASK_SIZE &&
333 !is_arch_mappable_range(address, 0))) {
334 if (is_kernel_mode && is_page_fault &&
335 vmalloc_fault(pgd, address) >= 0)
336 return 1;
337 /*
338 * Don't take the mm semaphore here. If we fixup a prefetch
339 * fault we could otherwise deadlock.
340 */
341 mm = NULL; /* happy compiler */
342 vma = NULL;
343 goto bad_area_nosemaphore;
344 }
345
346 /*
347 * If we're trying to touch user-space addresses, we must
348 * be either at PL0, or else with interrupts enabled in the
349 * kernel, so either way we can re-enable interrupts here
350 * unless we are doing atomic access to user space with
351 * interrupts disabled.
352 */
353 if (!(regs->flags & PT_FLAGS_DISABLE_IRQ))
354 local_irq_enable();
355
356 mm = tsk->mm;
357
358 /*
359 * If we're in an interrupt, have no user context or are running in an
360 * atomic region then we must not take the fault.
361 */
362 if (in_atomic() || !mm) {
363 vma = NULL; /* happy compiler */
364 goto bad_area_nosemaphore;
365 }
366
367 /*
368 * When running in the kernel we expect faults to occur only to
369 * addresses in user space. All other faults represent errors in the
370 * kernel and should generate an OOPS. Unfortunately, in the case of an
371 * erroneous fault occurring in a code path which already holds mmap_sem
372 * we will deadlock attempting to validate the fault against the
373 * address space. Luckily the kernel only validly references user
374 * space from well defined areas of code, which are listed in the
375 * exceptions table.
376 *
377 * As the vast majority of faults will be valid we will only perform
378 * the source reference check when there is a possibility of a deadlock.
379 * Attempt to lock the address space, if we cannot we then validate the
380 * source. If this is invalid we can skip the address space check,
381 * thus avoiding the deadlock.
382 */
383 if (!down_read_trylock(&mm->mmap_sem)) {
384 if (is_kernel_mode &&
385 !search_exception_tables(regs->pc)) {
386 vma = NULL; /* happy compiler */
387 goto bad_area_nosemaphore;
388 }
389
390retry:
391 down_read(&mm->mmap_sem);
392 }
393
394 vma = find_vma(mm, address);
395 if (!vma)
396 goto bad_area;
397 if (vma->vm_start <= address)
398 goto good_area;
399 if (!(vma->vm_flags & VM_GROWSDOWN))
400 goto bad_area;
401 if (regs->sp < PAGE_OFFSET) {
402 /*
403 * accessing the stack below sp is always a bug.
404 */
405 if (address < regs->sp)
406 goto bad_area;
407 }
408 if (expand_stack(vma, address))
409 goto bad_area;
410
411/*
412 * Ok, we have a good vm_area for this memory access, so
413 * we can handle it..
414 */
415good_area:
416 si_code = SEGV_ACCERR;
417 if (fault_num == INT_ITLB_MISS) {
418 if (!(vma->vm_flags & VM_EXEC))
419 goto bad_area;
420 } else if (write) {
421#ifdef TEST_VERIFY_AREA
422 if (!is_page_fault && regs->cs == KERNEL_CS)
423 pr_err("WP fault at "REGFMT"\n", regs->eip);
424#endif
425 if (!(vma->vm_flags & VM_WRITE))
426 goto bad_area;
427 } else {
428 if (!is_page_fault || !(vma->vm_flags & VM_READ))
429 goto bad_area;
430 }
431
432 survive:
433 /*
434 * If for any reason at all we couldn't handle the fault,
435 * make sure we exit gracefully rather than endlessly redo
436 * the fault.
437 */
438 fault = handle_mm_fault(mm, vma, address, flags);
439
440 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
441 return 0;
442
443 if (unlikely(fault & VM_FAULT_ERROR)) {
444 if (fault & VM_FAULT_OOM)
445 goto out_of_memory;
446 else if (fault & VM_FAULT_SIGBUS)
447 goto do_sigbus;
448 BUG();
449 }
450 if (flags & FAULT_FLAG_ALLOW_RETRY) {
451 if (fault & VM_FAULT_MAJOR)
452 tsk->maj_flt++;
453 else
454 tsk->min_flt++;
455 if (fault & VM_FAULT_RETRY) {
456 flags &= ~FAULT_FLAG_ALLOW_RETRY;
457
458 /*
459 * No need to up_read(&mm->mmap_sem) as we would
460 * have already released it in __lock_page_or_retry
461 * in mm/filemap.c.
462 */
463 goto retry;
464 }
465 }
466
467#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
468 /*
469 * If this was an asynchronous fault,
470 * restart the appropriate engine.
471 */
472 switch (fault_num) {
473#if CHIP_HAS_TILE_DMA()
474 case INT_DMATLB_MISS:
475 case INT_DMATLB_MISS_DWNCL:
476 case INT_DMATLB_ACCESS:
477 case INT_DMATLB_ACCESS_DWNCL:
478 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
479 break;
480#endif
481#if CHIP_HAS_SN_PROC()
482 case INT_SNITLB_MISS:
483 case INT_SNITLB_MISS_DWNCL:
484 __insn_mtspr(SPR_SNCTL,
485 __insn_mfspr(SPR_SNCTL) &
486 ~SPR_SNCTL__FRZPROC_MASK);
487 break;
488#endif
489 }
490#endif
491
492 up_read(&mm->mmap_sem);
493 return 1;
494
495/*
496 * Something tried to access memory that isn't in our memory map..
497 * Fix it, but check if it's kernel or user first..
498 */
499bad_area:
500 up_read(&mm->mmap_sem);
501
502bad_area_nosemaphore:
503 /* User mode accesses just cause a SIGSEGV */
504 if (!is_kernel_mode) {
505 /*
506 * It's possible to have interrupts off here.
507 */
508 local_irq_enable();
509
510 force_sig_info_fault("segfault", SIGSEGV, si_code, address,
511 fault_num, tsk, regs);
512 return 0;
513 }
514
515no_context:
516 /* Are we prepared to handle this kernel fault? */
517 if (fixup_exception(regs))
518 return 0;
519
520/*
521 * Oops. The kernel tried to access some bad page. We'll have to
522 * terminate things with extreme prejudice.
523 */
524
525 bust_spinlocks(1);
526
527 /* FIXME: no lookup_address() yet */
528#ifdef SUPPORT_LOOKUP_ADDRESS
529 if (fault_num == INT_ITLB_MISS) {
530 pte_t *pte = lookup_address(address);
531
532 if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
533 pr_crit("kernel tried to execute"
534 " non-executable page - exploit attempt?"
535 " (uid: %d)\n", current->uid);
536 }
537#endif
538 if (address < PAGE_SIZE)
539 pr_alert("Unable to handle kernel NULL pointer dereference\n");
540 else
541 pr_alert("Unable to handle kernel paging request\n");
542 pr_alert(" at virtual address "REGFMT", pc "REGFMT"\n",
543 address, regs->pc);
544
545 show_regs(regs);
546
547 if (unlikely(tsk->pid < 2)) {
548 panic("Kernel page fault running %s!",
549 is_idle_task(tsk) ? "the idle task" : "init");
550 }
551
552 /*
553 * More FIXME: we should probably copy the i386 here and
554 * implement a generic die() routine. Not today.
555 */
556#ifdef SUPPORT_DIE
557 die("Oops", regs);
558#endif
559 bust_spinlocks(1);
560
561 do_group_exit(SIGKILL);
562
563/*
564 * We ran out of memory, or some other thing happened to us that made
565 * us unable to handle the page fault gracefully.
566 */
567out_of_memory:
568 up_read(&mm->mmap_sem);
569 if (is_global_init(tsk)) {
570 yield();
571 down_read(&mm->mmap_sem);
572 goto survive;
573 }
574 pr_alert("VM: killing process %s\n", tsk->comm);
575 if (!is_kernel_mode)
576 do_group_exit(SIGKILL);
577 goto no_context;
578
579do_sigbus:
580 up_read(&mm->mmap_sem);
581
582 /* Kernel mode? Handle exceptions or die */
583 if (is_kernel_mode)
584 goto no_context;
585
586 force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address,
587 fault_num, tsk, regs);
588 return 0;
589}
590
591#ifndef __tilegx__
592
593/* We must release ICS before panicking or we won't get anywhere. */
594#define ics_panic(fmt, ...) do { \
595 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \
596 panic(fmt, __VA_ARGS__); \
597} while (0)
598
599/*
600 * When we take an ITLB or DTLB fault or access violation in the
601 * supervisor while the critical section bit is set, the hypervisor is
602 * reluctant to write new values into the EX_CONTEXT_K_x registers,
603 * since that might indicate we have not yet squirreled the SPR
604 * contents away and can thus safely take a recursive interrupt.
605 * Accordingly, the hypervisor passes us the PC via SYSTEM_SAVE_K_2.
606 *
607 * Note that this routine is called before homecache_tlb_defer_enter(),
608 * which means that we can properly unlock any atomics that might
609 * be used there (good), but also means we must be very sensitive
610 * to not touch any data structures that might be located in memory
611 * that could migrate, as we could be entering the kernel on a dataplane
612 * cpu that has been deferring kernel TLB updates. This means, for
613 * example, that we can't migrate init_mm or its pgd.
614 */
615struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num,
616 unsigned long address,
617 unsigned long info)
618{
619 unsigned long pc = info & ~1;
620 int write = info & 1;
621 pgd_t *pgd = get_current_pgd();
622
623 /* Retval is 1 at first since we will handle the fault fully. */
624 struct intvec_state state = {
625 do_page_fault, fault_num, address, write, 1
626 };
627
628 /* Validate that we are plausibly in the right routine. */
629 if ((pc & 0x7) != 0 || pc < PAGE_OFFSET ||
630 (fault_num != INT_DTLB_MISS &&
631 fault_num != INT_DTLB_ACCESS)) {
632 unsigned long old_pc = regs->pc;
633 regs->pc = pc;
634 ics_panic("Bad ICS page fault args:"
635 " old PC %#lx, fault %d/%d at %#lx\n",
636 old_pc, fault_num, write, address);
637 }
638
639 /* We might be faulting on a vmalloc page, so check that first. */
640 if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0)
641 return state;
642
643 /*
644 * If we faulted with ICS set in sys_cmpxchg, we are providing
645 * a user syscall service that should generate a signal on
646 * fault. We didn't set up a kernel stack on initial entry to
647 * sys_cmpxchg, but instead had one set up by the fault, which
648 * (because sys_cmpxchg never releases ICS) came to us via the
649 * SYSTEM_SAVE_K_2 mechanism, and thus EX_CONTEXT_K_[01] are
650 * still referencing the original user code. We release the
651 * atomic lock and rewrite pt_regs so that it appears that we
652 * came from user-space directly, and after we finish the
653 * fault we'll go back to user space and re-issue the swint.
654 * This way the backtrace information is correct if we need to
655 * emit a stack dump at any point while handling this.
656 *
657 * Must match register use in sys_cmpxchg().
658 */
659 if (pc >= (unsigned long) sys_cmpxchg &&
660 pc < (unsigned long) __sys_cmpxchg_end) {
661#ifdef CONFIG_SMP
662 /* Don't unlock before we could have locked. */
663 if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) {
664 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
665 __atomic_fault_unlock(lock_ptr);
666 }
667#endif
668 regs->sp = regs->regs[27];
669 }
670
671 /*
672 * We can also fault in the atomic assembly, in which
673 * case we use the exception table to do the first-level fixup.
674 * We may re-fixup again in the real fault handler if it
675 * turns out the faulting address is just bad, and not,
676 * for example, migrating.
677 */
678 else if (pc >= (unsigned long) __start_atomic_asm_code &&
679 pc < (unsigned long) __end_atomic_asm_code) {
680 const struct exception_table_entry *fixup;
681#ifdef CONFIG_SMP
682 /* Unlock the atomic lock. */
683 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
684 __atomic_fault_unlock(lock_ptr);
685#endif
686 fixup = search_exception_tables(pc);
687 if (!fixup)
688 ics_panic("ICS atomic fault not in table:"
689 " PC %#lx, fault %d", pc, fault_num);
690 regs->pc = fixup->fixup;
691 regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0);
692 }
693
694 /*
695 * Now that we have released the atomic lock (if necessary),
696 * it's safe to spin if the PTE that caused the fault was migrating.
697 */
698 if (fault_num == INT_DTLB_ACCESS)
699 write = 1;
700 if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write))
701 return state;
702
703 /* Return zero so that we continue on with normal fault handling. */
704 state.retval = 0;
705 return state;
706}
707
708#endif /* !__tilegx__ */
709
710/*
711 * This routine handles page faults. It determines the address, and the
712 * problem, and then passes it handle_page_fault() for normal DTLB and
713 * ITLB issues, and for DMA or SN processor faults when we are in user
714 * space. For the latter, if we're in kernel mode, we just save the
715 * interrupt away appropriately and return immediately. We can't do
716 * page faults for user code while in kernel mode.
717 */
718void do_page_fault(struct pt_regs *regs, int fault_num,
719 unsigned long address, unsigned long write)
720{
721 int is_page_fault;
722
723 /* This case should have been handled by do_page_fault_ics(). */
724 BUG_ON(write & ~1);
725
726#if CHIP_HAS_TILE_DMA()
727 /*
728 * If it's a DMA fault, suspend the transfer while we're
729 * handling the miss; we'll restart after it's handled. If we
730 * don't suspend, it's possible that this process could swap
731 * out and back in, and restart the engine since the DMA is
732 * still 'running'.
733 */
734 if (fault_num == INT_DMATLB_MISS ||
735 fault_num == INT_DMATLB_ACCESS ||
736 fault_num == INT_DMATLB_MISS_DWNCL ||
737 fault_num == INT_DMATLB_ACCESS_DWNCL) {
738 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
739 while (__insn_mfspr(SPR_DMA_USER_STATUS) &
740 SPR_DMA_STATUS__BUSY_MASK)
741 ;
742 }
743#endif
744
745 /* Validate fault num and decide if this is a first-time page fault. */
746 switch (fault_num) {
747 case INT_ITLB_MISS:
748 case INT_DTLB_MISS:
749#if CHIP_HAS_TILE_DMA()
750 case INT_DMATLB_MISS:
751 case INT_DMATLB_MISS_DWNCL:
752#endif
753#if CHIP_HAS_SN_PROC()
754 case INT_SNITLB_MISS:
755 case INT_SNITLB_MISS_DWNCL:
756#endif
757 is_page_fault = 1;
758 break;
759
760 case INT_DTLB_ACCESS:
761#if CHIP_HAS_TILE_DMA()
762 case INT_DMATLB_ACCESS:
763 case INT_DMATLB_ACCESS_DWNCL:
764#endif
765 is_page_fault = 0;
766 break;
767
768 default:
769 panic("Bad fault number %d in do_page_fault", fault_num);
770 }
771
772#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
773 if (EX1_PL(regs->ex1) != USER_PL) {
774 struct async_tlb *async;
775 switch (fault_num) {
776#if CHIP_HAS_TILE_DMA()
777 case INT_DMATLB_MISS:
778 case INT_DMATLB_ACCESS:
779 case INT_DMATLB_MISS_DWNCL:
780 case INT_DMATLB_ACCESS_DWNCL:
781 async = ¤t->thread.dma_async_tlb;
782 break;
783#endif
784#if CHIP_HAS_SN_PROC()
785 case INT_SNITLB_MISS:
786 case INT_SNITLB_MISS_DWNCL:
787 async = ¤t->thread.sn_async_tlb;
788 break;
789#endif
790 default:
791 async = NULL;
792 }
793 if (async) {
794
795 /*
796 * No vmalloc check required, so we can allow
797 * interrupts immediately at this point.
798 */
799 local_irq_enable();
800
801 set_thread_flag(TIF_ASYNC_TLB);
802 if (async->fault_num != 0) {
803 panic("Second async fault %d;"
804 " old fault was %d (%#lx/%ld)",
805 fault_num, async->fault_num,
806 address, write);
807 }
808 BUG_ON(fault_num == 0);
809 async->fault_num = fault_num;
810 async->is_fault = is_page_fault;
811 async->is_write = write;
812 async->address = address;
813 return;
814 }
815 }
816#endif
817
818 handle_page_fault(regs, fault_num, is_page_fault, address, write);
819}
820
821
822#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
823/*
824 * Check an async_tlb structure to see if a deferred fault is waiting,
825 * and if so pass it to the page-fault code.
826 */
827static void handle_async_page_fault(struct pt_regs *regs,
828 struct async_tlb *async)
829{
830 if (async->fault_num) {
831 /*
832 * Clear async->fault_num before calling the page-fault
833 * handler so that if we re-interrupt before returning
834 * from the function we have somewhere to put the
835 * information from the new interrupt.
836 */
837 int fault_num = async->fault_num;
838 async->fault_num = 0;
839 handle_page_fault(regs, fault_num, async->is_fault,
840 async->address, async->is_write);
841 }
842}
843
844/*
845 * This routine effectively re-issues asynchronous page faults
846 * when we are returning to user space.
847 */
848void do_async_page_fault(struct pt_regs *regs)
849{
850 /*
851 * Clear thread flag early. If we re-interrupt while processing
852 * code here, we will reset it and recall this routine before
853 * returning to user space.
854 */
855 clear_thread_flag(TIF_ASYNC_TLB);
856
857#if CHIP_HAS_TILE_DMA()
858 handle_async_page_fault(regs, ¤t->thread.dma_async_tlb);
859#endif
860#if CHIP_HAS_SN_PROC()
861 handle_async_page_fault(regs, ¤t->thread.sn_async_tlb);
862#endif
863}
864#endif /* CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC() */
865
866
867void vmalloc_sync_all(void)
868{
869#ifdef __tilegx__
870 /* Currently all L1 kernel pmd's are static and shared. */
871 BUG_ON(pgd_index(VMALLOC_END) != pgd_index(VMALLOC_START));
872#else
873 /*
874 * Note that races in the updates of insync and start aren't
875 * problematic: insync can only get set bits added, and updates to
876 * start are only improving performance (without affecting correctness
877 * if undone).
878 */
879 static DECLARE_BITMAP(insync, PTRS_PER_PGD);
880 static unsigned long start = PAGE_OFFSET;
881 unsigned long address;
882
883 BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK);
884 for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) {
885 if (!test_bit(pgd_index(address), insync)) {
886 unsigned long flags;
887 struct list_head *pos;
888
889 spin_lock_irqsave(&pgd_lock, flags);
890 list_for_each(pos, &pgd_list)
891 if (!vmalloc_sync_one(list_to_pgd(pos),
892 address)) {
893 /* Must be at first entry in list. */
894 BUG_ON(pos != pgd_list.next);
895 break;
896 }
897 spin_unlock_irqrestore(&pgd_lock, flags);
898 if (pos != pgd_list.next)
899 set_bit(pgd_index(address), insync);
900 }
901 if (address == start && test_bit(pgd_index(address), insync))
902 start = address + PGDIR_SIZE;
903 }
904#endif
905}
1/*
2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 *
14 * From i386 code copyright (C) 1995 Linus Torvalds
15 */
16
17#include <linux/signal.h>
18#include <linux/sched.h>
19#include <linux/kernel.h>
20#include <linux/errno.h>
21#include <linux/string.h>
22#include <linux/types.h>
23#include <linux/ptrace.h>
24#include <linux/mman.h>
25#include <linux/mm.h>
26#include <linux/smp.h>
27#include <linux/interrupt.h>
28#include <linux/init.h>
29#include <linux/tty.h>
30#include <linux/vt_kern.h> /* For unblank_screen() */
31#include <linux/highmem.h>
32#include <linux/module.h>
33#include <linux/kprobes.h>
34#include <linux/hugetlb.h>
35#include <linux/syscalls.h>
36#include <linux/uaccess.h>
37#include <linux/kdebug.h>
38
39#include <asm/pgalloc.h>
40#include <asm/sections.h>
41#include <asm/traps.h>
42#include <asm/syscalls.h>
43
44#include <arch/interrupts.h>
45
46static noinline void force_sig_info_fault(const char *type, int si_signo,
47 int si_code, unsigned long address,
48 int fault_num,
49 struct task_struct *tsk,
50 struct pt_regs *regs)
51{
52 siginfo_t info;
53
54 if (unlikely(tsk->pid < 2)) {
55 panic("Signal %d (code %d) at %#lx sent to %s!",
56 si_signo, si_code & 0xffff, address,
57 is_idle_task(tsk) ? "the idle task" : "init");
58 }
59
60 info.si_signo = si_signo;
61 info.si_errno = 0;
62 info.si_code = si_code;
63 info.si_addr = (void __user *)address;
64 info.si_trapno = fault_num;
65 trace_unhandled_signal(type, regs, address, si_signo);
66 force_sig_info(si_signo, &info, tsk);
67}
68
69#ifndef __tilegx__
70/*
71 * Synthesize the fault a PL0 process would get by doing a word-load of
72 * an unaligned address or a high kernel address.
73 */
74SYSCALL_DEFINE1(cmpxchg_badaddr, unsigned long, address)
75{
76 struct pt_regs *regs = current_pt_regs();
77
78 if (address >= PAGE_OFFSET)
79 force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR,
80 address, INT_DTLB_MISS, current, regs);
81 else
82 force_sig_info_fault("atomic alignment fault", SIGBUS,
83 BUS_ADRALN, address,
84 INT_UNALIGN_DATA, current, regs);
85
86 /*
87 * Adjust pc to point at the actual instruction, which is unusual
88 * for syscalls normally, but is appropriate when we are claiming
89 * that a syscall swint1 caused a page fault or bus error.
90 */
91 regs->pc -= 8;
92
93 /*
94 * Mark this as a caller-save interrupt, like a normal page fault,
95 * so that when we go through the signal handler path we will
96 * properly restore r0, r1, and r2 for the signal handler arguments.
97 */
98 regs->flags |= PT_FLAGS_CALLER_SAVES;
99
100 return 0;
101}
102#endif
103
104static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
105{
106 unsigned index = pgd_index(address);
107 pgd_t *pgd_k;
108 pud_t *pud, *pud_k;
109 pmd_t *pmd, *pmd_k;
110
111 pgd += index;
112 pgd_k = init_mm.pgd + index;
113
114 if (!pgd_present(*pgd_k))
115 return NULL;
116
117 pud = pud_offset(pgd, address);
118 pud_k = pud_offset(pgd_k, address);
119 if (!pud_present(*pud_k))
120 return NULL;
121
122 pmd = pmd_offset(pud, address);
123 pmd_k = pmd_offset(pud_k, address);
124 if (!pmd_present(*pmd_k))
125 return NULL;
126 if (!pmd_present(*pmd))
127 set_pmd(pmd, *pmd_k);
128 else
129 BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k));
130 return pmd_k;
131}
132
133/*
134 * Handle a fault on the vmalloc area.
135 */
136static inline int vmalloc_fault(pgd_t *pgd, unsigned long address)
137{
138 pmd_t *pmd_k;
139 pte_t *pte_k;
140
141 /* Make sure we are in vmalloc area */
142 if (!(address >= VMALLOC_START && address < VMALLOC_END))
143 return -1;
144
145 /*
146 * Synchronize this task's top level page-table
147 * with the 'reference' page table.
148 */
149 pmd_k = vmalloc_sync_one(pgd, address);
150 if (!pmd_k)
151 return -1;
152 pte_k = pte_offset_kernel(pmd_k, address);
153 if (!pte_present(*pte_k))
154 return -1;
155 return 0;
156}
157
158/* Wait until this PTE has completed migration. */
159static void wait_for_migration(pte_t *pte)
160{
161 if (pte_migrating(*pte)) {
162 /*
163 * Wait until the migrater fixes up this pte.
164 * We scale the loop count by the clock rate so we'll wait for
165 * a few seconds here.
166 */
167 int retries = 0;
168 int bound = get_clock_rate();
169 while (pte_migrating(*pte)) {
170 barrier();
171 if (++retries > bound)
172 panic("Hit migrating PTE (%#llx) and"
173 " page PFN %#lx still migrating",
174 pte->val, pte_pfn(*pte));
175 }
176 }
177}
178
179/*
180 * It's not generally safe to use "current" to get the page table pointer,
181 * since we might be running an oprofile interrupt in the middle of a
182 * task switch.
183 */
184static pgd_t *get_current_pgd(void)
185{
186 HV_Context ctx = hv_inquire_context();
187 unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT;
188 struct page *pgd_page = pfn_to_page(pgd_pfn);
189 BUG_ON(PageHighMem(pgd_page));
190 return (pgd_t *) __va(ctx.page_table);
191}
192
193/*
194 * We can receive a page fault from a migrating PTE at any time.
195 * Handle it by just waiting until the fault resolves.
196 *
197 * It's also possible to get a migrating kernel PTE that resolves
198 * itself during the downcall from hypervisor to Linux. We just check
199 * here to see if the PTE seems valid, and if so we retry it.
200 *
201 * NOTE! We MUST NOT take any locks for this case. We may be in an
202 * interrupt or a critical region, and must do as little as possible.
203 * Similarly, we can't use atomic ops here, since we may be handling a
204 * fault caused by an atomic op access.
205 *
206 * If we find a migrating PTE while we're in an NMI context, and we're
207 * at a PC that has a registered exception handler, we don't wait,
208 * since this thread may (e.g.) have been interrupted while migrating
209 * its own stack, which would then cause us to self-deadlock.
210 */
211static int handle_migrating_pte(pgd_t *pgd, int fault_num,
212 unsigned long address, unsigned long pc,
213 int is_kernel_mode, int write)
214{
215 pud_t *pud;
216 pmd_t *pmd;
217 pte_t *pte;
218 pte_t pteval;
219
220 if (pgd_addr_invalid(address))
221 return 0;
222
223 pgd += pgd_index(address);
224 pud = pud_offset(pgd, address);
225 if (!pud || !pud_present(*pud))
226 return 0;
227 pmd = pmd_offset(pud, address);
228 if (!pmd || !pmd_present(*pmd))
229 return 0;
230 pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) :
231 pte_offset_kernel(pmd, address);
232 pteval = *pte;
233 if (pte_migrating(pteval)) {
234 if (in_nmi() && search_exception_tables(pc))
235 return 0;
236 wait_for_migration(pte);
237 return 1;
238 }
239
240 if (!is_kernel_mode || !pte_present(pteval))
241 return 0;
242 if (fault_num == INT_ITLB_MISS) {
243 if (pte_exec(pteval))
244 return 1;
245 } else if (write) {
246 if (pte_write(pteval))
247 return 1;
248 } else {
249 if (pte_read(pteval))
250 return 1;
251 }
252
253 return 0;
254}
255
256/*
257 * This routine is responsible for faulting in user pages.
258 * It passes the work off to one of the appropriate routines.
259 * It returns true if the fault was successfully handled.
260 */
261static int handle_page_fault(struct pt_regs *regs,
262 int fault_num,
263 int is_page_fault,
264 unsigned long address,
265 int write)
266{
267 struct task_struct *tsk;
268 struct mm_struct *mm;
269 struct vm_area_struct *vma;
270 unsigned long stack_offset;
271 int fault;
272 int si_code;
273 int is_kernel_mode;
274 pgd_t *pgd;
275 unsigned int flags;
276
277 /* on TILE, protection faults are always writes */
278 if (!is_page_fault)
279 write = 1;
280
281 flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
282
283 is_kernel_mode = !user_mode(regs);
284
285 tsk = validate_current();
286
287 /*
288 * Check to see if we might be overwriting the stack, and bail
289 * out if so. The page fault code is a relatively likely
290 * place to get trapped in an infinite regress, and once we
291 * overwrite the whole stack, it becomes very hard to recover.
292 */
293 stack_offset = stack_pointer & (THREAD_SIZE-1);
294 if (stack_offset < THREAD_SIZE / 8) {
295 pr_alert("Potential stack overrun: sp %#lx\n",
296 stack_pointer);
297 show_regs(regs);
298 pr_alert("Killing current process %d/%s\n",
299 tsk->pid, tsk->comm);
300 do_group_exit(SIGKILL);
301 }
302
303 /*
304 * Early on, we need to check for migrating PTE entries;
305 * see homecache.c. If we find a migrating PTE, we wait until
306 * the backing page claims to be done migrating, then we proceed.
307 * For kernel PTEs, we rewrite the PTE and return and retry.
308 * Otherwise, we treat the fault like a normal "no PTE" fault,
309 * rather than trying to patch up the existing PTE.
310 */
311 pgd = get_current_pgd();
312 if (handle_migrating_pte(pgd, fault_num, address, regs->pc,
313 is_kernel_mode, write))
314 return 1;
315
316 si_code = SEGV_MAPERR;
317
318 /*
319 * We fault-in kernel-space virtual memory on-demand. The
320 * 'reference' page table is init_mm.pgd.
321 *
322 * NOTE! We MUST NOT take any locks for this case. We may
323 * be in an interrupt or a critical region, and should
324 * only copy the information from the master page table,
325 * nothing more.
326 *
327 * This verifies that the fault happens in kernel space
328 * and that the fault was not a protection fault.
329 */
330 if (unlikely(address >= TASK_SIZE &&
331 !is_arch_mappable_range(address, 0))) {
332 if (is_kernel_mode && is_page_fault &&
333 vmalloc_fault(pgd, address) >= 0)
334 return 1;
335 /*
336 * Don't take the mm semaphore here. If we fixup a prefetch
337 * fault we could otherwise deadlock.
338 */
339 mm = NULL; /* happy compiler */
340 vma = NULL;
341 goto bad_area_nosemaphore;
342 }
343
344 /*
345 * If we're trying to touch user-space addresses, we must
346 * be either at PL0, or else with interrupts enabled in the
347 * kernel, so either way we can re-enable interrupts here
348 * unless we are doing atomic access to user space with
349 * interrupts disabled.
350 */
351 if (!(regs->flags & PT_FLAGS_DISABLE_IRQ))
352 local_irq_enable();
353
354 mm = tsk->mm;
355
356 /*
357 * If we're in an interrupt, have no user context or are running in an
358 * atomic region then we must not take the fault.
359 */
360 if (in_atomic() || !mm) {
361 vma = NULL; /* happy compiler */
362 goto bad_area_nosemaphore;
363 }
364
365 if (!is_kernel_mode)
366 flags |= FAULT_FLAG_USER;
367
368 /*
369 * When running in the kernel we expect faults to occur only to
370 * addresses in user space. All other faults represent errors in the
371 * kernel and should generate an OOPS. Unfortunately, in the case of an
372 * erroneous fault occurring in a code path which already holds mmap_sem
373 * we will deadlock attempting to validate the fault against the
374 * address space. Luckily the kernel only validly references user
375 * space from well defined areas of code, which are listed in the
376 * exceptions table.
377 *
378 * As the vast majority of faults will be valid we will only perform
379 * the source reference check when there is a possibility of a deadlock.
380 * Attempt to lock the address space, if we cannot we then validate the
381 * source. If this is invalid we can skip the address space check,
382 * thus avoiding the deadlock.
383 */
384 if (!down_read_trylock(&mm->mmap_sem)) {
385 if (is_kernel_mode &&
386 !search_exception_tables(regs->pc)) {
387 vma = NULL; /* happy compiler */
388 goto bad_area_nosemaphore;
389 }
390
391retry:
392 down_read(&mm->mmap_sem);
393 }
394
395 vma = find_vma(mm, address);
396 if (!vma)
397 goto bad_area;
398 if (vma->vm_start <= address)
399 goto good_area;
400 if (!(vma->vm_flags & VM_GROWSDOWN))
401 goto bad_area;
402 if (regs->sp < PAGE_OFFSET) {
403 /*
404 * accessing the stack below sp is always a bug.
405 */
406 if (address < regs->sp)
407 goto bad_area;
408 }
409 if (expand_stack(vma, address))
410 goto bad_area;
411
412/*
413 * Ok, we have a good vm_area for this memory access, so
414 * we can handle it..
415 */
416good_area:
417 si_code = SEGV_ACCERR;
418 if (fault_num == INT_ITLB_MISS) {
419 if (!(vma->vm_flags & VM_EXEC))
420 goto bad_area;
421 } else if (write) {
422#ifdef TEST_VERIFY_AREA
423 if (!is_page_fault && regs->cs == KERNEL_CS)
424 pr_err("WP fault at "REGFMT"\n", regs->eip);
425#endif
426 if (!(vma->vm_flags & VM_WRITE))
427 goto bad_area;
428 flags |= FAULT_FLAG_WRITE;
429 } else {
430 if (!is_page_fault || !(vma->vm_flags & VM_READ))
431 goto bad_area;
432 }
433
434 /*
435 * If for any reason at all we couldn't handle the fault,
436 * make sure we exit gracefully rather than endlessly redo
437 * the fault.
438 */
439 fault = handle_mm_fault(mm, vma, address, flags);
440
441 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
442 return 0;
443
444 if (unlikely(fault & VM_FAULT_ERROR)) {
445 if (fault & VM_FAULT_OOM)
446 goto out_of_memory;
447 else if (fault & VM_FAULT_SIGBUS)
448 goto do_sigbus;
449 BUG();
450 }
451 if (flags & FAULT_FLAG_ALLOW_RETRY) {
452 if (fault & VM_FAULT_MAJOR)
453 tsk->maj_flt++;
454 else
455 tsk->min_flt++;
456 if (fault & VM_FAULT_RETRY) {
457 flags &= ~FAULT_FLAG_ALLOW_RETRY;
458 flags |= FAULT_FLAG_TRIED;
459
460 /*
461 * No need to up_read(&mm->mmap_sem) as we would
462 * have already released it in __lock_page_or_retry
463 * in mm/filemap.c.
464 */
465 goto retry;
466 }
467 }
468
469#if CHIP_HAS_TILE_DMA()
470 /* If this was a DMA TLB fault, restart the DMA engine. */
471 switch (fault_num) {
472 case INT_DMATLB_MISS:
473 case INT_DMATLB_MISS_DWNCL:
474 case INT_DMATLB_ACCESS:
475 case INT_DMATLB_ACCESS_DWNCL:
476 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
477 break;
478 }
479#endif
480
481 up_read(&mm->mmap_sem);
482 return 1;
483
484/*
485 * Something tried to access memory that isn't in our memory map..
486 * Fix it, but check if it's kernel or user first..
487 */
488bad_area:
489 up_read(&mm->mmap_sem);
490
491bad_area_nosemaphore:
492 /* User mode accesses just cause a SIGSEGV */
493 if (!is_kernel_mode) {
494 /*
495 * It's possible to have interrupts off here.
496 */
497 local_irq_enable();
498
499 force_sig_info_fault("segfault", SIGSEGV, si_code, address,
500 fault_num, tsk, regs);
501 return 0;
502 }
503
504no_context:
505 /* Are we prepared to handle this kernel fault? */
506 if (fixup_exception(regs))
507 return 0;
508
509/*
510 * Oops. The kernel tried to access some bad page. We'll have to
511 * terminate things with extreme prejudice.
512 */
513
514 bust_spinlocks(1);
515
516 /* FIXME: no lookup_address() yet */
517#ifdef SUPPORT_LOOKUP_ADDRESS
518 if (fault_num == INT_ITLB_MISS) {
519 pte_t *pte = lookup_address(address);
520
521 if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
522 pr_crit("kernel tried to execute"
523 " non-executable page - exploit attempt?"
524 " (uid: %d)\n", current->uid);
525 }
526#endif
527 if (address < PAGE_SIZE)
528 pr_alert("Unable to handle kernel NULL pointer dereference\n");
529 else
530 pr_alert("Unable to handle kernel paging request\n");
531 pr_alert(" at virtual address "REGFMT", pc "REGFMT"\n",
532 address, regs->pc);
533
534 show_regs(regs);
535
536 if (unlikely(tsk->pid < 2)) {
537 panic("Kernel page fault running %s!",
538 is_idle_task(tsk) ? "the idle task" : "init");
539 }
540
541 /*
542 * More FIXME: we should probably copy the i386 here and
543 * implement a generic die() routine. Not today.
544 */
545#ifdef SUPPORT_DIE
546 die("Oops", regs);
547#endif
548 bust_spinlocks(1);
549
550 do_group_exit(SIGKILL);
551
552/*
553 * We ran out of memory, or some other thing happened to us that made
554 * us unable to handle the page fault gracefully.
555 */
556out_of_memory:
557 up_read(&mm->mmap_sem);
558 if (is_kernel_mode)
559 goto no_context;
560 pagefault_out_of_memory();
561 return 0;
562
563do_sigbus:
564 up_read(&mm->mmap_sem);
565
566 /* Kernel mode? Handle exceptions or die */
567 if (is_kernel_mode)
568 goto no_context;
569
570 force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address,
571 fault_num, tsk, regs);
572 return 0;
573}
574
575#ifndef __tilegx__
576
577/* We must release ICS before panicking or we won't get anywhere. */
578#define ics_panic(fmt, ...) do { \
579 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \
580 panic(fmt, __VA_ARGS__); \
581} while (0)
582
583/*
584 * When we take an ITLB or DTLB fault or access violation in the
585 * supervisor while the critical section bit is set, the hypervisor is
586 * reluctant to write new values into the EX_CONTEXT_K_x registers,
587 * since that might indicate we have not yet squirreled the SPR
588 * contents away and can thus safely take a recursive interrupt.
589 * Accordingly, the hypervisor passes us the PC via SYSTEM_SAVE_K_2.
590 *
591 * Note that this routine is called before homecache_tlb_defer_enter(),
592 * which means that we can properly unlock any atomics that might
593 * be used there (good), but also means we must be very sensitive
594 * to not touch any data structures that might be located in memory
595 * that could migrate, as we could be entering the kernel on a dataplane
596 * cpu that has been deferring kernel TLB updates. This means, for
597 * example, that we can't migrate init_mm or its pgd.
598 */
599struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num,
600 unsigned long address,
601 unsigned long info)
602{
603 unsigned long pc = info & ~1;
604 int write = info & 1;
605 pgd_t *pgd = get_current_pgd();
606
607 /* Retval is 1 at first since we will handle the fault fully. */
608 struct intvec_state state = {
609 do_page_fault, fault_num, address, write, 1
610 };
611
612 /* Validate that we are plausibly in the right routine. */
613 if ((pc & 0x7) != 0 || pc < PAGE_OFFSET ||
614 (fault_num != INT_DTLB_MISS &&
615 fault_num != INT_DTLB_ACCESS)) {
616 unsigned long old_pc = regs->pc;
617 regs->pc = pc;
618 ics_panic("Bad ICS page fault args:"
619 " old PC %#lx, fault %d/%d at %#lx\n",
620 old_pc, fault_num, write, address);
621 }
622
623 /* We might be faulting on a vmalloc page, so check that first. */
624 if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0)
625 return state;
626
627 /*
628 * If we faulted with ICS set in sys_cmpxchg, we are providing
629 * a user syscall service that should generate a signal on
630 * fault. We didn't set up a kernel stack on initial entry to
631 * sys_cmpxchg, but instead had one set up by the fault, which
632 * (because sys_cmpxchg never releases ICS) came to us via the
633 * SYSTEM_SAVE_K_2 mechanism, and thus EX_CONTEXT_K_[01] are
634 * still referencing the original user code. We release the
635 * atomic lock and rewrite pt_regs so that it appears that we
636 * came from user-space directly, and after we finish the
637 * fault we'll go back to user space and re-issue the swint.
638 * This way the backtrace information is correct if we need to
639 * emit a stack dump at any point while handling this.
640 *
641 * Must match register use in sys_cmpxchg().
642 */
643 if (pc >= (unsigned long) sys_cmpxchg &&
644 pc < (unsigned long) __sys_cmpxchg_end) {
645#ifdef CONFIG_SMP
646 /* Don't unlock before we could have locked. */
647 if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) {
648 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
649 __atomic_fault_unlock(lock_ptr);
650 }
651#endif
652 regs->sp = regs->regs[27];
653 }
654
655 /*
656 * We can also fault in the atomic assembly, in which
657 * case we use the exception table to do the first-level fixup.
658 * We may re-fixup again in the real fault handler if it
659 * turns out the faulting address is just bad, and not,
660 * for example, migrating.
661 */
662 else if (pc >= (unsigned long) __start_atomic_asm_code &&
663 pc < (unsigned long) __end_atomic_asm_code) {
664 const struct exception_table_entry *fixup;
665#ifdef CONFIG_SMP
666 /* Unlock the atomic lock. */
667 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
668 __atomic_fault_unlock(lock_ptr);
669#endif
670 fixup = search_exception_tables(pc);
671 if (!fixup)
672 ics_panic("ICS atomic fault not in table:"
673 " PC %#lx, fault %d", pc, fault_num);
674 regs->pc = fixup->fixup;
675 regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0);
676 }
677
678 /*
679 * Now that we have released the atomic lock (if necessary),
680 * it's safe to spin if the PTE that caused the fault was migrating.
681 */
682 if (fault_num == INT_DTLB_ACCESS)
683 write = 1;
684 if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write))
685 return state;
686
687 /* Return zero so that we continue on with normal fault handling. */
688 state.retval = 0;
689 return state;
690}
691
692#endif /* !__tilegx__ */
693
694/*
695 * This routine handles page faults. It determines the address, and the
696 * problem, and then passes it handle_page_fault() for normal DTLB and
697 * ITLB issues, and for DMA or SN processor faults when we are in user
698 * space. For the latter, if we're in kernel mode, we just save the
699 * interrupt away appropriately and return immediately. We can't do
700 * page faults for user code while in kernel mode.
701 */
702void do_page_fault(struct pt_regs *regs, int fault_num,
703 unsigned long address, unsigned long write)
704{
705 int is_page_fault;
706
707#ifdef CONFIG_KPROBES
708 /*
709 * This is to notify the fault handler of the kprobes. The
710 * exception code is redundant as it is also carried in REGS,
711 * but we pass it anyhow.
712 */
713 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1,
714 regs->faultnum, SIGSEGV) == NOTIFY_STOP)
715 return;
716#endif
717
718#ifdef __tilegx__
719 /*
720 * We don't need early do_page_fault_ics() support, since unlike
721 * Pro we don't need to worry about unlocking the atomic locks.
722 * There is only one current case in GX where we touch any memory
723 * under ICS other than our own kernel stack, and we handle that
724 * here. (If we crash due to trying to touch our own stack,
725 * we're in too much trouble for C code to help out anyway.)
726 */
727 if (write & ~1) {
728 unsigned long pc = write & ~1;
729 if (pc >= (unsigned long) __start_unalign_asm_code &&
730 pc < (unsigned long) __end_unalign_asm_code) {
731 struct thread_info *ti = current_thread_info();
732 /*
733 * Our EX_CONTEXT is still what it was from the
734 * initial unalign exception, but now we've faulted
735 * on the JIT page. We would like to complete the
736 * page fault however is appropriate, and then retry
737 * the instruction that caused the unalign exception.
738 * Our state has been "corrupted" by setting the low
739 * bit in "sp", and stashing r0..r3 in the
740 * thread_info area, so we revert all of that, then
741 * continue as if this were a normal page fault.
742 */
743 regs->sp &= ~1UL;
744 regs->regs[0] = ti->unalign_jit_tmp[0];
745 regs->regs[1] = ti->unalign_jit_tmp[1];
746 regs->regs[2] = ti->unalign_jit_tmp[2];
747 regs->regs[3] = ti->unalign_jit_tmp[3];
748 write &= 1;
749 } else {
750 pr_alert("%s/%d: ICS set at page fault at %#lx: %#lx\n",
751 current->comm, current->pid, pc, address);
752 show_regs(regs);
753 do_group_exit(SIGKILL);
754 return;
755 }
756 }
757#else
758 /* This case should have been handled by do_page_fault_ics(). */
759 BUG_ON(write & ~1);
760#endif
761
762#if CHIP_HAS_TILE_DMA()
763 /*
764 * If it's a DMA fault, suspend the transfer while we're
765 * handling the miss; we'll restart after it's handled. If we
766 * don't suspend, it's possible that this process could swap
767 * out and back in, and restart the engine since the DMA is
768 * still 'running'.
769 */
770 if (fault_num == INT_DMATLB_MISS ||
771 fault_num == INT_DMATLB_ACCESS ||
772 fault_num == INT_DMATLB_MISS_DWNCL ||
773 fault_num == INT_DMATLB_ACCESS_DWNCL) {
774 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
775 while (__insn_mfspr(SPR_DMA_USER_STATUS) &
776 SPR_DMA_STATUS__BUSY_MASK)
777 ;
778 }
779#endif
780
781 /* Validate fault num and decide if this is a first-time page fault. */
782 switch (fault_num) {
783 case INT_ITLB_MISS:
784 case INT_DTLB_MISS:
785#if CHIP_HAS_TILE_DMA()
786 case INT_DMATLB_MISS:
787 case INT_DMATLB_MISS_DWNCL:
788#endif
789 is_page_fault = 1;
790 break;
791
792 case INT_DTLB_ACCESS:
793#if CHIP_HAS_TILE_DMA()
794 case INT_DMATLB_ACCESS:
795 case INT_DMATLB_ACCESS_DWNCL:
796#endif
797 is_page_fault = 0;
798 break;
799
800 default:
801 panic("Bad fault number %d in do_page_fault", fault_num);
802 }
803
804#if CHIP_HAS_TILE_DMA()
805 if (!user_mode(regs)) {
806 struct async_tlb *async;
807 switch (fault_num) {
808#if CHIP_HAS_TILE_DMA()
809 case INT_DMATLB_MISS:
810 case INT_DMATLB_ACCESS:
811 case INT_DMATLB_MISS_DWNCL:
812 case INT_DMATLB_ACCESS_DWNCL:
813 async = ¤t->thread.dma_async_tlb;
814 break;
815#endif
816 default:
817 async = NULL;
818 }
819 if (async) {
820
821 /*
822 * No vmalloc check required, so we can allow
823 * interrupts immediately at this point.
824 */
825 local_irq_enable();
826
827 set_thread_flag(TIF_ASYNC_TLB);
828 if (async->fault_num != 0) {
829 panic("Second async fault %d;"
830 " old fault was %d (%#lx/%ld)",
831 fault_num, async->fault_num,
832 address, write);
833 }
834 BUG_ON(fault_num == 0);
835 async->fault_num = fault_num;
836 async->is_fault = is_page_fault;
837 async->is_write = write;
838 async->address = address;
839 return;
840 }
841 }
842#endif
843
844 handle_page_fault(regs, fault_num, is_page_fault, address, write);
845}
846
847
848#if CHIP_HAS_TILE_DMA()
849/*
850 * This routine effectively re-issues asynchronous page faults
851 * when we are returning to user space.
852 */
853void do_async_page_fault(struct pt_regs *regs)
854{
855 struct async_tlb *async = ¤t->thread.dma_async_tlb;
856
857 /*
858 * Clear thread flag early. If we re-interrupt while processing
859 * code here, we will reset it and recall this routine before
860 * returning to user space.
861 */
862 clear_thread_flag(TIF_ASYNC_TLB);
863
864 if (async->fault_num) {
865 /*
866 * Clear async->fault_num before calling the page-fault
867 * handler so that if we re-interrupt before returning
868 * from the function we have somewhere to put the
869 * information from the new interrupt.
870 */
871 int fault_num = async->fault_num;
872 async->fault_num = 0;
873 handle_page_fault(regs, fault_num, async->is_fault,
874 async->address, async->is_write);
875 }
876}
877#endif /* CHIP_HAS_TILE_DMA() */
878
879
880void vmalloc_sync_all(void)
881{
882#ifdef __tilegx__
883 /* Currently all L1 kernel pmd's are static and shared. */
884 BUILD_BUG_ON(pgd_index(VMALLOC_END - PAGE_SIZE) !=
885 pgd_index(VMALLOC_START));
886#else
887 /*
888 * Note that races in the updates of insync and start aren't
889 * problematic: insync can only get set bits added, and updates to
890 * start are only improving performance (without affecting correctness
891 * if undone).
892 */
893 static DECLARE_BITMAP(insync, PTRS_PER_PGD);
894 static unsigned long start = PAGE_OFFSET;
895 unsigned long address;
896
897 BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK);
898 for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) {
899 if (!test_bit(pgd_index(address), insync)) {
900 unsigned long flags;
901 struct list_head *pos;
902
903 spin_lock_irqsave(&pgd_lock, flags);
904 list_for_each(pos, &pgd_list)
905 if (!vmalloc_sync_one(list_to_pgd(pos),
906 address)) {
907 /* Must be at first entry in list. */
908 BUG_ON(pos != pgd_list.next);
909 break;
910 }
911 spin_unlock_irqrestore(&pgd_lock, flags);
912 if (pos != pgd_list.next)
913 set_bit(pgd_index(address), insync);
914 }
915 if (address == start && test_bit(pgd_index(address), insync))
916 start = address + PGDIR_SIZE;
917 }
918#endif
919}