Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Derived from "arch/i386/kernel/process.c"
4 * Copyright (C) 1995 Linus Torvalds
5 *
6 * Updated and modified by Cort Dougan (cort@cs.nmt.edu) and
7 * Paul Mackerras (paulus@cs.anu.edu.au)
8 *
9 * PowerPC version
10 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
11 */
12
13#include <linux/errno.h>
14#include <linux/sched.h>
15#include <linux/sched/debug.h>
16#include <linux/sched/task.h>
17#include <linux/sched/task_stack.h>
18#include <linux/kernel.h>
19#include <linux/mm.h>
20#include <linux/smp.h>
21#include <linux/stddef.h>
22#include <linux/unistd.h>
23#include <linux/ptrace.h>
24#include <linux/slab.h>
25#include <linux/user.h>
26#include <linux/elf.h>
27#include <linux/prctl.h>
28#include <linux/init_task.h>
29#include <linux/export.h>
30#include <linux/kallsyms.h>
31#include <linux/mqueue.h>
32#include <linux/hardirq.h>
33#include <linux/utsname.h>
34#include <linux/ftrace.h>
35#include <linux/kernel_stat.h>
36#include <linux/personality.h>
37#include <linux/random.h>
38#include <linux/hw_breakpoint.h>
39#include <linux/uaccess.h>
40#include <linux/elf-randomize.h>
41#include <linux/pkeys.h>
42#include <linux/seq_buf.h>
43
44#include <asm/io.h>
45#include <asm/processor.h>
46#include <asm/mmu.h>
47#include <asm/prom.h>
48#include <asm/machdep.h>
49#include <asm/time.h>
50#include <asm/runlatch.h>
51#include <asm/syscalls.h>
52#include <asm/switch_to.h>
53#include <asm/tm.h>
54#include <asm/debug.h>
55#ifdef CONFIG_PPC64
56#include <asm/firmware.h>
57#include <asm/hw_irq.h>
58#endif
59#include <asm/code-patching.h>
60#include <asm/exec.h>
61#include <asm/livepatch.h>
62#include <asm/cpu_has_feature.h>
63#include <asm/asm-prototypes.h>
64#include <asm/stacktrace.h>
65#include <asm/hw_breakpoint.h>
66
67#include <linux/kprobes.h>
68#include <linux/kdebug.h>
69
70/* Transactional Memory debug */
71#ifdef TM_DEBUG_SW
72#define TM_DEBUG(x...) printk(KERN_INFO x)
73#else
74#define TM_DEBUG(x...) do { } while(0)
75#endif
76
77extern unsigned long _get_SP(void);
78
79#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
80/*
81 * Are we running in "Suspend disabled" mode? If so we have to block any
82 * sigreturn that would get us into suspended state, and we also warn in some
83 * other paths that we should never reach with suspend disabled.
84 */
85bool tm_suspend_disabled __ro_after_init = false;
86
87static void check_if_tm_restore_required(struct task_struct *tsk)
88{
89 /*
90 * If we are saving the current thread's registers, and the
91 * thread is in a transactional state, set the TIF_RESTORE_TM
92 * bit so that we know to restore the registers before
93 * returning to userspace.
94 */
95 if (tsk == current && tsk->thread.regs &&
96 MSR_TM_ACTIVE(tsk->thread.regs->msr) &&
97 !test_thread_flag(TIF_RESTORE_TM)) {
98 tsk->thread.ckpt_regs.msr = tsk->thread.regs->msr;
99 set_thread_flag(TIF_RESTORE_TM);
100 }
101}
102
103#else
104static inline void check_if_tm_restore_required(struct task_struct *tsk) { }
105#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
106
107bool strict_msr_control;
108EXPORT_SYMBOL(strict_msr_control);
109
110static int __init enable_strict_msr_control(char *str)
111{
112 strict_msr_control = true;
113 pr_info("Enabling strict facility control\n");
114
115 return 0;
116}
117early_param("ppc_strict_facility_enable", enable_strict_msr_control);
118
119/* notrace because it's called by restore_math */
120unsigned long notrace msr_check_and_set(unsigned long bits)
121{
122 unsigned long oldmsr = mfmsr();
123 unsigned long newmsr;
124
125 newmsr = oldmsr | bits;
126
127#ifdef CONFIG_VSX
128 if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
129 newmsr |= MSR_VSX;
130#endif
131
132 if (oldmsr != newmsr)
133 mtmsr_isync(newmsr);
134
135 return newmsr;
136}
137EXPORT_SYMBOL_GPL(msr_check_and_set);
138
139/* notrace because it's called by restore_math */
140void notrace __msr_check_and_clear(unsigned long bits)
141{
142 unsigned long oldmsr = mfmsr();
143 unsigned long newmsr;
144
145 newmsr = oldmsr & ~bits;
146
147#ifdef CONFIG_VSX
148 if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
149 newmsr &= ~MSR_VSX;
150#endif
151
152 if (oldmsr != newmsr)
153 mtmsr_isync(newmsr);
154}
155EXPORT_SYMBOL(__msr_check_and_clear);
156
157#ifdef CONFIG_PPC_FPU
158static void __giveup_fpu(struct task_struct *tsk)
159{
160 unsigned long msr;
161
162 save_fpu(tsk);
163 msr = tsk->thread.regs->msr;
164 msr &= ~(MSR_FP|MSR_FE0|MSR_FE1);
165#ifdef CONFIG_VSX
166 if (cpu_has_feature(CPU_FTR_VSX))
167 msr &= ~MSR_VSX;
168#endif
169 tsk->thread.regs->msr = msr;
170}
171
172void giveup_fpu(struct task_struct *tsk)
173{
174 check_if_tm_restore_required(tsk);
175
176 msr_check_and_set(MSR_FP);
177 __giveup_fpu(tsk);
178 msr_check_and_clear(MSR_FP);
179}
180EXPORT_SYMBOL(giveup_fpu);
181
182/*
183 * Make sure the floating-point register state in the
184 * the thread_struct is up to date for task tsk.
185 */
186void flush_fp_to_thread(struct task_struct *tsk)
187{
188 if (tsk->thread.regs) {
189 /*
190 * We need to disable preemption here because if we didn't,
191 * another process could get scheduled after the regs->msr
192 * test but before we have finished saving the FP registers
193 * to the thread_struct. That process could take over the
194 * FPU, and then when we get scheduled again we would store
195 * bogus values for the remaining FP registers.
196 */
197 preempt_disable();
198 if (tsk->thread.regs->msr & MSR_FP) {
199 /*
200 * This should only ever be called for current or
201 * for a stopped child process. Since we save away
202 * the FP register state on context switch,
203 * there is something wrong if a stopped child appears
204 * to still have its FP state in the CPU registers.
205 */
206 BUG_ON(tsk != current);
207 giveup_fpu(tsk);
208 }
209 preempt_enable();
210 }
211}
212EXPORT_SYMBOL_GPL(flush_fp_to_thread);
213
214void enable_kernel_fp(void)
215{
216 unsigned long cpumsr;
217
218 WARN_ON(preemptible());
219
220 cpumsr = msr_check_and_set(MSR_FP);
221
222 if (current->thread.regs && (current->thread.regs->msr & MSR_FP)) {
223 check_if_tm_restore_required(current);
224 /*
225 * If a thread has already been reclaimed then the
226 * checkpointed registers are on the CPU but have definitely
227 * been saved by the reclaim code. Don't need to and *cannot*
228 * giveup as this would save to the 'live' structure not the
229 * checkpointed structure.
230 */
231 if (!MSR_TM_ACTIVE(cpumsr) &&
232 MSR_TM_ACTIVE(current->thread.regs->msr))
233 return;
234 __giveup_fpu(current);
235 }
236}
237EXPORT_SYMBOL(enable_kernel_fp);
238#endif /* CONFIG_PPC_FPU */
239
240#ifdef CONFIG_ALTIVEC
241static void __giveup_altivec(struct task_struct *tsk)
242{
243 unsigned long msr;
244
245 save_altivec(tsk);
246 msr = tsk->thread.regs->msr;
247 msr &= ~MSR_VEC;
248#ifdef CONFIG_VSX
249 if (cpu_has_feature(CPU_FTR_VSX))
250 msr &= ~MSR_VSX;
251#endif
252 tsk->thread.regs->msr = msr;
253}
254
255void giveup_altivec(struct task_struct *tsk)
256{
257 check_if_tm_restore_required(tsk);
258
259 msr_check_and_set(MSR_VEC);
260 __giveup_altivec(tsk);
261 msr_check_and_clear(MSR_VEC);
262}
263EXPORT_SYMBOL(giveup_altivec);
264
265void enable_kernel_altivec(void)
266{
267 unsigned long cpumsr;
268
269 WARN_ON(preemptible());
270
271 cpumsr = msr_check_and_set(MSR_VEC);
272
273 if (current->thread.regs && (current->thread.regs->msr & MSR_VEC)) {
274 check_if_tm_restore_required(current);
275 /*
276 * If a thread has already been reclaimed then the
277 * checkpointed registers are on the CPU but have definitely
278 * been saved by the reclaim code. Don't need to and *cannot*
279 * giveup as this would save to the 'live' structure not the
280 * checkpointed structure.
281 */
282 if (!MSR_TM_ACTIVE(cpumsr) &&
283 MSR_TM_ACTIVE(current->thread.regs->msr))
284 return;
285 __giveup_altivec(current);
286 }
287}
288EXPORT_SYMBOL(enable_kernel_altivec);
289
290/*
291 * Make sure the VMX/Altivec register state in the
292 * the thread_struct is up to date for task tsk.
293 */
294void flush_altivec_to_thread(struct task_struct *tsk)
295{
296 if (tsk->thread.regs) {
297 preempt_disable();
298 if (tsk->thread.regs->msr & MSR_VEC) {
299 BUG_ON(tsk != current);
300 giveup_altivec(tsk);
301 }
302 preempt_enable();
303 }
304}
305EXPORT_SYMBOL_GPL(flush_altivec_to_thread);
306#endif /* CONFIG_ALTIVEC */
307
308#ifdef CONFIG_VSX
309static void __giveup_vsx(struct task_struct *tsk)
310{
311 unsigned long msr = tsk->thread.regs->msr;
312
313 /*
314 * We should never be ssetting MSR_VSX without also setting
315 * MSR_FP and MSR_VEC
316 */
317 WARN_ON((msr & MSR_VSX) && !((msr & MSR_FP) && (msr & MSR_VEC)));
318
319 /* __giveup_fpu will clear MSR_VSX */
320 if (msr & MSR_FP)
321 __giveup_fpu(tsk);
322 if (msr & MSR_VEC)
323 __giveup_altivec(tsk);
324}
325
326static void giveup_vsx(struct task_struct *tsk)
327{
328 check_if_tm_restore_required(tsk);
329
330 msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
331 __giveup_vsx(tsk);
332 msr_check_and_clear(MSR_FP|MSR_VEC|MSR_VSX);
333}
334
335void enable_kernel_vsx(void)
336{
337 unsigned long cpumsr;
338
339 WARN_ON(preemptible());
340
341 cpumsr = msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
342
343 if (current->thread.regs &&
344 (current->thread.regs->msr & (MSR_VSX|MSR_VEC|MSR_FP))) {
345 check_if_tm_restore_required(current);
346 /*
347 * If a thread has already been reclaimed then the
348 * checkpointed registers are on the CPU but have definitely
349 * been saved by the reclaim code. Don't need to and *cannot*
350 * giveup as this would save to the 'live' structure not the
351 * checkpointed structure.
352 */
353 if (!MSR_TM_ACTIVE(cpumsr) &&
354 MSR_TM_ACTIVE(current->thread.regs->msr))
355 return;
356 __giveup_vsx(current);
357 }
358}
359EXPORT_SYMBOL(enable_kernel_vsx);
360
361void flush_vsx_to_thread(struct task_struct *tsk)
362{
363 if (tsk->thread.regs) {
364 preempt_disable();
365 if (tsk->thread.regs->msr & (MSR_VSX|MSR_VEC|MSR_FP)) {
366 BUG_ON(tsk != current);
367 giveup_vsx(tsk);
368 }
369 preempt_enable();
370 }
371}
372EXPORT_SYMBOL_GPL(flush_vsx_to_thread);
373#endif /* CONFIG_VSX */
374
375#ifdef CONFIG_SPE
376void giveup_spe(struct task_struct *tsk)
377{
378 check_if_tm_restore_required(tsk);
379
380 msr_check_and_set(MSR_SPE);
381 __giveup_spe(tsk);
382 msr_check_and_clear(MSR_SPE);
383}
384EXPORT_SYMBOL(giveup_spe);
385
386void enable_kernel_spe(void)
387{
388 WARN_ON(preemptible());
389
390 msr_check_and_set(MSR_SPE);
391
392 if (current->thread.regs && (current->thread.regs->msr & MSR_SPE)) {
393 check_if_tm_restore_required(current);
394 __giveup_spe(current);
395 }
396}
397EXPORT_SYMBOL(enable_kernel_spe);
398
399void flush_spe_to_thread(struct task_struct *tsk)
400{
401 if (tsk->thread.regs) {
402 preempt_disable();
403 if (tsk->thread.regs->msr & MSR_SPE) {
404 BUG_ON(tsk != current);
405 tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
406 giveup_spe(tsk);
407 }
408 preempt_enable();
409 }
410}
411#endif /* CONFIG_SPE */
412
413static unsigned long msr_all_available;
414
415static int __init init_msr_all_available(void)
416{
417#ifdef CONFIG_PPC_FPU
418 msr_all_available |= MSR_FP;
419#endif
420#ifdef CONFIG_ALTIVEC
421 if (cpu_has_feature(CPU_FTR_ALTIVEC))
422 msr_all_available |= MSR_VEC;
423#endif
424#ifdef CONFIG_VSX
425 if (cpu_has_feature(CPU_FTR_VSX))
426 msr_all_available |= MSR_VSX;
427#endif
428#ifdef CONFIG_SPE
429 if (cpu_has_feature(CPU_FTR_SPE))
430 msr_all_available |= MSR_SPE;
431#endif
432
433 return 0;
434}
435early_initcall(init_msr_all_available);
436
437void giveup_all(struct task_struct *tsk)
438{
439 unsigned long usermsr;
440
441 if (!tsk->thread.regs)
442 return;
443
444 check_if_tm_restore_required(tsk);
445
446 usermsr = tsk->thread.regs->msr;
447
448 if ((usermsr & msr_all_available) == 0)
449 return;
450
451 msr_check_and_set(msr_all_available);
452
453 WARN_ON((usermsr & MSR_VSX) && !((usermsr & MSR_FP) && (usermsr & MSR_VEC)));
454
455#ifdef CONFIG_PPC_FPU
456 if (usermsr & MSR_FP)
457 __giveup_fpu(tsk);
458#endif
459#ifdef CONFIG_ALTIVEC
460 if (usermsr & MSR_VEC)
461 __giveup_altivec(tsk);
462#endif
463#ifdef CONFIG_SPE
464 if (usermsr & MSR_SPE)
465 __giveup_spe(tsk);
466#endif
467
468 msr_check_and_clear(msr_all_available);
469}
470EXPORT_SYMBOL(giveup_all);
471
472#ifdef CONFIG_PPC_BOOK3S_64
473#ifdef CONFIG_PPC_FPU
474static bool should_restore_fp(void)
475{
476 if (current->thread.load_fp) {
477 current->thread.load_fp++;
478 return true;
479 }
480 return false;
481}
482
483static void do_restore_fp(void)
484{
485 load_fp_state(¤t->thread.fp_state);
486}
487#else
488static bool should_restore_fp(void) { return false; }
489static void do_restore_fp(void) { }
490#endif /* CONFIG_PPC_FPU */
491
492#ifdef CONFIG_ALTIVEC
493static bool should_restore_altivec(void)
494{
495 if (cpu_has_feature(CPU_FTR_ALTIVEC) && (current->thread.load_vec)) {
496 current->thread.load_vec++;
497 return true;
498 }
499 return false;
500}
501
502static void do_restore_altivec(void)
503{
504 load_vr_state(¤t->thread.vr_state);
505 current->thread.used_vr = 1;
506}
507#else
508static bool should_restore_altivec(void) { return false; }
509static void do_restore_altivec(void) { }
510#endif /* CONFIG_ALTIVEC */
511
512#ifdef CONFIG_VSX
513static bool should_restore_vsx(void)
514{
515 if (cpu_has_feature(CPU_FTR_VSX))
516 return true;
517 return false;
518}
519static void do_restore_vsx(void)
520{
521 current->thread.used_vsr = 1;
522}
523#else
524static bool should_restore_vsx(void) { return false; }
525static void do_restore_vsx(void) { }
526#endif /* CONFIG_VSX */
527
528/*
529 * The exception exit path calls restore_math() with interrupts hard disabled
530 * but the soft irq state not "reconciled". ftrace code that calls
531 * local_irq_save/restore causes warnings.
532 *
533 * Rather than complicate the exit path, just don't trace restore_math. This
534 * could be done by having ftrace entry code check for this un-reconciled
535 * condition where MSR[EE]=0 and PACA_IRQ_HARD_DIS is not set, and
536 * temporarily fix it up for the duration of the ftrace call.
537 */
538void notrace restore_math(struct pt_regs *regs)
539{
540 unsigned long msr;
541 unsigned long new_msr = 0;
542
543 msr = regs->msr;
544
545 /*
546 * new_msr tracks the facilities that are to be restored. Only reload
547 * if the bit is not set in the user MSR (if it is set, the registers
548 * are live for the user thread).
549 */
550 if ((!(msr & MSR_FP)) && should_restore_fp())
551 new_msr |= MSR_FP;
552
553 if ((!(msr & MSR_VEC)) && should_restore_altivec())
554 new_msr |= MSR_VEC;
555
556 if ((!(msr & MSR_VSX)) && should_restore_vsx()) {
557 if (((msr | new_msr) & (MSR_FP | MSR_VEC)) == (MSR_FP | MSR_VEC))
558 new_msr |= MSR_VSX;
559 }
560
561 if (new_msr) {
562 unsigned long fpexc_mode = 0;
563
564 msr_check_and_set(new_msr);
565
566 if (new_msr & MSR_FP) {
567 do_restore_fp();
568
569 // This also covers VSX, because VSX implies FP
570 fpexc_mode = current->thread.fpexc_mode;
571 }
572
573 if (new_msr & MSR_VEC)
574 do_restore_altivec();
575
576 if (new_msr & MSR_VSX)
577 do_restore_vsx();
578
579 msr_check_and_clear(new_msr);
580
581 regs->msr |= new_msr | fpexc_mode;
582 }
583}
584#endif
585
586static void save_all(struct task_struct *tsk)
587{
588 unsigned long usermsr;
589
590 if (!tsk->thread.regs)
591 return;
592
593 usermsr = tsk->thread.regs->msr;
594
595 if ((usermsr & msr_all_available) == 0)
596 return;
597
598 msr_check_and_set(msr_all_available);
599
600 WARN_ON((usermsr & MSR_VSX) && !((usermsr & MSR_FP) && (usermsr & MSR_VEC)));
601
602 if (usermsr & MSR_FP)
603 save_fpu(tsk);
604
605 if (usermsr & MSR_VEC)
606 save_altivec(tsk);
607
608 if (usermsr & MSR_SPE)
609 __giveup_spe(tsk);
610
611 msr_check_and_clear(msr_all_available);
612 thread_pkey_regs_save(&tsk->thread);
613}
614
615void flush_all_to_thread(struct task_struct *tsk)
616{
617 if (tsk->thread.regs) {
618 preempt_disable();
619 BUG_ON(tsk != current);
620#ifdef CONFIG_SPE
621 if (tsk->thread.regs->msr & MSR_SPE)
622 tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
623#endif
624 save_all(tsk);
625
626 preempt_enable();
627 }
628}
629EXPORT_SYMBOL(flush_all_to_thread);
630
631#ifdef CONFIG_PPC_ADV_DEBUG_REGS
632void do_send_trap(struct pt_regs *regs, unsigned long address,
633 unsigned long error_code, int breakpt)
634{
635 current->thread.trap_nr = TRAP_HWBKPT;
636 if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
637 11, SIGSEGV) == NOTIFY_STOP)
638 return;
639
640 /* Deliver the signal to userspace */
641 force_sig_ptrace_errno_trap(breakpt, /* breakpoint or watchpoint id */
642 (void __user *)address);
643}
644#else /* !CONFIG_PPC_ADV_DEBUG_REGS */
645void do_break (struct pt_regs *regs, unsigned long address,
646 unsigned long error_code)
647{
648 current->thread.trap_nr = TRAP_HWBKPT;
649 if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
650 11, SIGSEGV) == NOTIFY_STOP)
651 return;
652
653 if (debugger_break_match(regs))
654 return;
655
656 /* Deliver the signal to userspace */
657 force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)address);
658}
659#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
660
661static DEFINE_PER_CPU(struct arch_hw_breakpoint, current_brk[HBP_NUM_MAX]);
662
663#ifdef CONFIG_PPC_ADV_DEBUG_REGS
664/*
665 * Set the debug registers back to their default "safe" values.
666 */
667static void set_debug_reg_defaults(struct thread_struct *thread)
668{
669 thread->debug.iac1 = thread->debug.iac2 = 0;
670#if CONFIG_PPC_ADV_DEBUG_IACS > 2
671 thread->debug.iac3 = thread->debug.iac4 = 0;
672#endif
673 thread->debug.dac1 = thread->debug.dac2 = 0;
674#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
675 thread->debug.dvc1 = thread->debug.dvc2 = 0;
676#endif
677 thread->debug.dbcr0 = 0;
678#ifdef CONFIG_BOOKE
679 /*
680 * Force User/Supervisor bits to b11 (user-only MSR[PR]=1)
681 */
682 thread->debug.dbcr1 = DBCR1_IAC1US | DBCR1_IAC2US |
683 DBCR1_IAC3US | DBCR1_IAC4US;
684 /*
685 * Force Data Address Compare User/Supervisor bits to be User-only
686 * (0b11 MSR[PR]=1) and set all other bits in DBCR2 register to be 0.
687 */
688 thread->debug.dbcr2 = DBCR2_DAC1US | DBCR2_DAC2US;
689#else
690 thread->debug.dbcr1 = 0;
691#endif
692}
693
694static void prime_debug_regs(struct debug_reg *debug)
695{
696 /*
697 * We could have inherited MSR_DE from userspace, since
698 * it doesn't get cleared on exception entry. Make sure
699 * MSR_DE is clear before we enable any debug events.
700 */
701 mtmsr(mfmsr() & ~MSR_DE);
702
703 mtspr(SPRN_IAC1, debug->iac1);
704 mtspr(SPRN_IAC2, debug->iac2);
705#if CONFIG_PPC_ADV_DEBUG_IACS > 2
706 mtspr(SPRN_IAC3, debug->iac3);
707 mtspr(SPRN_IAC4, debug->iac4);
708#endif
709 mtspr(SPRN_DAC1, debug->dac1);
710 mtspr(SPRN_DAC2, debug->dac2);
711#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
712 mtspr(SPRN_DVC1, debug->dvc1);
713 mtspr(SPRN_DVC2, debug->dvc2);
714#endif
715 mtspr(SPRN_DBCR0, debug->dbcr0);
716 mtspr(SPRN_DBCR1, debug->dbcr1);
717#ifdef CONFIG_BOOKE
718 mtspr(SPRN_DBCR2, debug->dbcr2);
719#endif
720}
721/*
722 * Unless neither the old or new thread are making use of the
723 * debug registers, set the debug registers from the values
724 * stored in the new thread.
725 */
726void switch_booke_debug_regs(struct debug_reg *new_debug)
727{
728 if ((current->thread.debug.dbcr0 & DBCR0_IDM)
729 || (new_debug->dbcr0 & DBCR0_IDM))
730 prime_debug_regs(new_debug);
731}
732EXPORT_SYMBOL_GPL(switch_booke_debug_regs);
733#else /* !CONFIG_PPC_ADV_DEBUG_REGS */
734#ifndef CONFIG_HAVE_HW_BREAKPOINT
735static void set_breakpoint(int i, struct arch_hw_breakpoint *brk)
736{
737 preempt_disable();
738 __set_breakpoint(i, brk);
739 preempt_enable();
740}
741
742static void set_debug_reg_defaults(struct thread_struct *thread)
743{
744 int i;
745 struct arch_hw_breakpoint null_brk = {0};
746
747 for (i = 0; i < nr_wp_slots(); i++) {
748 thread->hw_brk[i] = null_brk;
749 if (ppc_breakpoint_available())
750 set_breakpoint(i, &thread->hw_brk[i]);
751 }
752}
753
754static inline bool hw_brk_match(struct arch_hw_breakpoint *a,
755 struct arch_hw_breakpoint *b)
756{
757 if (a->address != b->address)
758 return false;
759 if (a->type != b->type)
760 return false;
761 if (a->len != b->len)
762 return false;
763 /* no need to check hw_len. it's calculated from address and len */
764 return true;
765}
766
767static void switch_hw_breakpoint(struct task_struct *new)
768{
769 int i;
770
771 for (i = 0; i < nr_wp_slots(); i++) {
772 if (likely(hw_brk_match(this_cpu_ptr(¤t_brk[i]),
773 &new->thread.hw_brk[i])))
774 continue;
775
776 __set_breakpoint(i, &new->thread.hw_brk[i]);
777 }
778}
779#endif /* !CONFIG_HAVE_HW_BREAKPOINT */
780#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
781
782#ifdef CONFIG_PPC_ADV_DEBUG_REGS
783static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
784{
785 mtspr(SPRN_DAC1, dabr);
786#ifdef CONFIG_PPC_47x
787 isync();
788#endif
789 return 0;
790}
791#elif defined(CONFIG_PPC_BOOK3S)
792static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
793{
794 mtspr(SPRN_DABR, dabr);
795 if (cpu_has_feature(CPU_FTR_DABRX))
796 mtspr(SPRN_DABRX, dabrx);
797 return 0;
798}
799#else
800static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
801{
802 return -EINVAL;
803}
804#endif
805
806static inline int set_dabr(struct arch_hw_breakpoint *brk)
807{
808 unsigned long dabr, dabrx;
809
810 dabr = brk->address | (brk->type & HW_BRK_TYPE_DABR);
811 dabrx = ((brk->type >> 3) & 0x7);
812
813 if (ppc_md.set_dabr)
814 return ppc_md.set_dabr(dabr, dabrx);
815
816 return __set_dabr(dabr, dabrx);
817}
818
819static inline int set_breakpoint_8xx(struct arch_hw_breakpoint *brk)
820{
821 unsigned long lctrl1 = LCTRL1_CTE_GT | LCTRL1_CTF_LT | LCTRL1_CRWE_RW |
822 LCTRL1_CRWF_RW;
823 unsigned long lctrl2 = LCTRL2_LW0EN | LCTRL2_LW0LADC | LCTRL2_SLW0EN;
824 unsigned long start_addr = ALIGN_DOWN(brk->address, HW_BREAKPOINT_SIZE);
825 unsigned long end_addr = ALIGN(brk->address + brk->len, HW_BREAKPOINT_SIZE);
826
827 if (start_addr == 0)
828 lctrl2 |= LCTRL2_LW0LA_F;
829 else if (end_addr == 0)
830 lctrl2 |= LCTRL2_LW0LA_E;
831 else
832 lctrl2 |= LCTRL2_LW0LA_EandF;
833
834 mtspr(SPRN_LCTRL2, 0);
835
836 if ((brk->type & HW_BRK_TYPE_RDWR) == 0)
837 return 0;
838
839 if ((brk->type & HW_BRK_TYPE_RDWR) == HW_BRK_TYPE_READ)
840 lctrl1 |= LCTRL1_CRWE_RO | LCTRL1_CRWF_RO;
841 if ((brk->type & HW_BRK_TYPE_RDWR) == HW_BRK_TYPE_WRITE)
842 lctrl1 |= LCTRL1_CRWE_WO | LCTRL1_CRWF_WO;
843
844 mtspr(SPRN_CMPE, start_addr - 1);
845 mtspr(SPRN_CMPF, end_addr);
846 mtspr(SPRN_LCTRL1, lctrl1);
847 mtspr(SPRN_LCTRL2, lctrl2);
848
849 return 0;
850}
851
852void __set_breakpoint(int nr, struct arch_hw_breakpoint *brk)
853{
854 memcpy(this_cpu_ptr(¤t_brk[nr]), brk, sizeof(*brk));
855
856 if (dawr_enabled())
857 // Power8 or later
858 set_dawr(nr, brk);
859 else if (IS_ENABLED(CONFIG_PPC_8xx))
860 set_breakpoint_8xx(brk);
861 else if (!cpu_has_feature(CPU_FTR_ARCH_207S))
862 // Power7 or earlier
863 set_dabr(brk);
864 else
865 // Shouldn't happen due to higher level checks
866 WARN_ON_ONCE(1);
867}
868
869/* Check if we have DAWR or DABR hardware */
870bool ppc_breakpoint_available(void)
871{
872 if (dawr_enabled())
873 return true; /* POWER8 DAWR or POWER9 forced DAWR */
874 if (cpu_has_feature(CPU_FTR_ARCH_207S))
875 return false; /* POWER9 with DAWR disabled */
876 /* DABR: Everything but POWER8 and POWER9 */
877 return true;
878}
879EXPORT_SYMBOL_GPL(ppc_breakpoint_available);
880
881#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
882
883static inline bool tm_enabled(struct task_struct *tsk)
884{
885 return tsk && tsk->thread.regs && (tsk->thread.regs->msr & MSR_TM);
886}
887
888static void tm_reclaim_thread(struct thread_struct *thr, uint8_t cause)
889{
890 /*
891 * Use the current MSR TM suspended bit to track if we have
892 * checkpointed state outstanding.
893 * On signal delivery, we'd normally reclaim the checkpointed
894 * state to obtain stack pointer (see:get_tm_stackpointer()).
895 * This will then directly return to userspace without going
896 * through __switch_to(). However, if the stack frame is bad,
897 * we need to exit this thread which calls __switch_to() which
898 * will again attempt to reclaim the already saved tm state.
899 * Hence we need to check that we've not already reclaimed
900 * this state.
901 * We do this using the current MSR, rather tracking it in
902 * some specific thread_struct bit, as it has the additional
903 * benefit of checking for a potential TM bad thing exception.
904 */
905 if (!MSR_TM_SUSPENDED(mfmsr()))
906 return;
907
908 giveup_all(container_of(thr, struct task_struct, thread));
909
910 tm_reclaim(thr, cause);
911
912 /*
913 * If we are in a transaction and FP is off then we can't have
914 * used FP inside that transaction. Hence the checkpointed
915 * state is the same as the live state. We need to copy the
916 * live state to the checkpointed state so that when the
917 * transaction is restored, the checkpointed state is correct
918 * and the aborted transaction sees the correct state. We use
919 * ckpt_regs.msr here as that's what tm_reclaim will use to
920 * determine if it's going to write the checkpointed state or
921 * not. So either this will write the checkpointed registers,
922 * or reclaim will. Similarly for VMX.
923 */
924 if ((thr->ckpt_regs.msr & MSR_FP) == 0)
925 memcpy(&thr->ckfp_state, &thr->fp_state,
926 sizeof(struct thread_fp_state));
927 if ((thr->ckpt_regs.msr & MSR_VEC) == 0)
928 memcpy(&thr->ckvr_state, &thr->vr_state,
929 sizeof(struct thread_vr_state));
930}
931
932void tm_reclaim_current(uint8_t cause)
933{
934 tm_enable();
935 tm_reclaim_thread(¤t->thread, cause);
936}
937
938static inline void tm_reclaim_task(struct task_struct *tsk)
939{
940 /* We have to work out if we're switching from/to a task that's in the
941 * middle of a transaction.
942 *
943 * In switching we need to maintain a 2nd register state as
944 * oldtask->thread.ckpt_regs. We tm_reclaim(oldproc); this saves the
945 * checkpointed (tbegin) state in ckpt_regs, ckfp_state and
946 * ckvr_state
947 *
948 * We also context switch (save) TFHAR/TEXASR/TFIAR in here.
949 */
950 struct thread_struct *thr = &tsk->thread;
951
952 if (!thr->regs)
953 return;
954
955 if (!MSR_TM_ACTIVE(thr->regs->msr))
956 goto out_and_saveregs;
957
958 WARN_ON(tm_suspend_disabled);
959
960 TM_DEBUG("--- tm_reclaim on pid %d (NIP=%lx, "
961 "ccr=%lx, msr=%lx, trap=%lx)\n",
962 tsk->pid, thr->regs->nip,
963 thr->regs->ccr, thr->regs->msr,
964 thr->regs->trap);
965
966 tm_reclaim_thread(thr, TM_CAUSE_RESCHED);
967
968 TM_DEBUG("--- tm_reclaim on pid %d complete\n",
969 tsk->pid);
970
971out_and_saveregs:
972 /* Always save the regs here, even if a transaction's not active.
973 * This context-switches a thread's TM info SPRs. We do it here to
974 * be consistent with the restore path (in recheckpoint) which
975 * cannot happen later in _switch().
976 */
977 tm_save_sprs(thr);
978}
979
980extern void __tm_recheckpoint(struct thread_struct *thread);
981
982void tm_recheckpoint(struct thread_struct *thread)
983{
984 unsigned long flags;
985
986 if (!(thread->regs->msr & MSR_TM))
987 return;
988
989 /* We really can't be interrupted here as the TEXASR registers can't
990 * change and later in the trecheckpoint code, we have a userspace R1.
991 * So let's hard disable over this region.
992 */
993 local_irq_save(flags);
994 hard_irq_disable();
995
996 /* The TM SPRs are restored here, so that TEXASR.FS can be set
997 * before the trecheckpoint and no explosion occurs.
998 */
999 tm_restore_sprs(thread);
1000
1001 __tm_recheckpoint(thread);
1002
1003 local_irq_restore(flags);
1004}
1005
1006static inline void tm_recheckpoint_new_task(struct task_struct *new)
1007{
1008 if (!cpu_has_feature(CPU_FTR_TM))
1009 return;
1010
1011 /* Recheckpoint the registers of the thread we're about to switch to.
1012 *
1013 * If the task was using FP, we non-lazily reload both the original and
1014 * the speculative FP register states. This is because the kernel
1015 * doesn't see if/when a TM rollback occurs, so if we take an FP
1016 * unavailable later, we are unable to determine which set of FP regs
1017 * need to be restored.
1018 */
1019 if (!tm_enabled(new))
1020 return;
1021
1022 if (!MSR_TM_ACTIVE(new->thread.regs->msr)){
1023 tm_restore_sprs(&new->thread);
1024 return;
1025 }
1026 /* Recheckpoint to restore original checkpointed register state. */
1027 TM_DEBUG("*** tm_recheckpoint of pid %d (new->msr 0x%lx)\n",
1028 new->pid, new->thread.regs->msr);
1029
1030 tm_recheckpoint(&new->thread);
1031
1032 /*
1033 * The checkpointed state has been restored but the live state has
1034 * not, ensure all the math functionality is turned off to trigger
1035 * restore_math() to reload.
1036 */
1037 new->thread.regs->msr &= ~(MSR_FP | MSR_VEC | MSR_VSX);
1038
1039 TM_DEBUG("*** tm_recheckpoint of pid %d complete "
1040 "(kernel msr 0x%lx)\n",
1041 new->pid, mfmsr());
1042}
1043
1044static inline void __switch_to_tm(struct task_struct *prev,
1045 struct task_struct *new)
1046{
1047 if (cpu_has_feature(CPU_FTR_TM)) {
1048 if (tm_enabled(prev) || tm_enabled(new))
1049 tm_enable();
1050
1051 if (tm_enabled(prev)) {
1052 prev->thread.load_tm++;
1053 tm_reclaim_task(prev);
1054 if (!MSR_TM_ACTIVE(prev->thread.regs->msr) && prev->thread.load_tm == 0)
1055 prev->thread.regs->msr &= ~MSR_TM;
1056 }
1057
1058 tm_recheckpoint_new_task(new);
1059 }
1060}
1061
1062/*
1063 * This is called if we are on the way out to userspace and the
1064 * TIF_RESTORE_TM flag is set. It checks if we need to reload
1065 * FP and/or vector state and does so if necessary.
1066 * If userspace is inside a transaction (whether active or
1067 * suspended) and FP/VMX/VSX instructions have ever been enabled
1068 * inside that transaction, then we have to keep them enabled
1069 * and keep the FP/VMX/VSX state loaded while ever the transaction
1070 * continues. The reason is that if we didn't, and subsequently
1071 * got a FP/VMX/VSX unavailable interrupt inside a transaction,
1072 * we don't know whether it's the same transaction, and thus we
1073 * don't know which of the checkpointed state and the transactional
1074 * state to use.
1075 */
1076void restore_tm_state(struct pt_regs *regs)
1077{
1078 unsigned long msr_diff;
1079
1080 /*
1081 * This is the only moment we should clear TIF_RESTORE_TM as
1082 * it is here that ckpt_regs.msr and pt_regs.msr become the same
1083 * again, anything else could lead to an incorrect ckpt_msr being
1084 * saved and therefore incorrect signal contexts.
1085 */
1086 clear_thread_flag(TIF_RESTORE_TM);
1087 if (!MSR_TM_ACTIVE(regs->msr))
1088 return;
1089
1090 msr_diff = current->thread.ckpt_regs.msr & ~regs->msr;
1091 msr_diff &= MSR_FP | MSR_VEC | MSR_VSX;
1092
1093 /* Ensure that restore_math() will restore */
1094 if (msr_diff & MSR_FP)
1095 current->thread.load_fp = 1;
1096#ifdef CONFIG_ALTIVEC
1097 if (cpu_has_feature(CPU_FTR_ALTIVEC) && msr_diff & MSR_VEC)
1098 current->thread.load_vec = 1;
1099#endif
1100 restore_math(regs);
1101
1102 regs->msr |= msr_diff;
1103}
1104
1105#else
1106#define tm_recheckpoint_new_task(new)
1107#define __switch_to_tm(prev, new)
1108#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
1109
1110static inline void save_sprs(struct thread_struct *t)
1111{
1112#ifdef CONFIG_ALTIVEC
1113 if (cpu_has_feature(CPU_FTR_ALTIVEC))
1114 t->vrsave = mfspr(SPRN_VRSAVE);
1115#endif
1116#ifdef CONFIG_PPC_BOOK3S_64
1117 if (cpu_has_feature(CPU_FTR_DSCR))
1118 t->dscr = mfspr(SPRN_DSCR);
1119
1120 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
1121 t->bescr = mfspr(SPRN_BESCR);
1122 t->ebbhr = mfspr(SPRN_EBBHR);
1123 t->ebbrr = mfspr(SPRN_EBBRR);
1124
1125 t->fscr = mfspr(SPRN_FSCR);
1126
1127 /*
1128 * Note that the TAR is not available for use in the kernel.
1129 * (To provide this, the TAR should be backed up/restored on
1130 * exception entry/exit instead, and be in pt_regs. FIXME,
1131 * this should be in pt_regs anyway (for debug).)
1132 */
1133 t->tar = mfspr(SPRN_TAR);
1134 }
1135#endif
1136
1137 thread_pkey_regs_save(t);
1138}
1139
1140static inline void restore_sprs(struct thread_struct *old_thread,
1141 struct thread_struct *new_thread)
1142{
1143#ifdef CONFIG_ALTIVEC
1144 if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
1145 old_thread->vrsave != new_thread->vrsave)
1146 mtspr(SPRN_VRSAVE, new_thread->vrsave);
1147#endif
1148#ifdef CONFIG_PPC_BOOK3S_64
1149 if (cpu_has_feature(CPU_FTR_DSCR)) {
1150 u64 dscr = get_paca()->dscr_default;
1151 if (new_thread->dscr_inherit)
1152 dscr = new_thread->dscr;
1153
1154 if (old_thread->dscr != dscr)
1155 mtspr(SPRN_DSCR, dscr);
1156 }
1157
1158 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
1159 if (old_thread->bescr != new_thread->bescr)
1160 mtspr(SPRN_BESCR, new_thread->bescr);
1161 if (old_thread->ebbhr != new_thread->ebbhr)
1162 mtspr(SPRN_EBBHR, new_thread->ebbhr);
1163 if (old_thread->ebbrr != new_thread->ebbrr)
1164 mtspr(SPRN_EBBRR, new_thread->ebbrr);
1165
1166 if (old_thread->fscr != new_thread->fscr)
1167 mtspr(SPRN_FSCR, new_thread->fscr);
1168
1169 if (old_thread->tar != new_thread->tar)
1170 mtspr(SPRN_TAR, new_thread->tar);
1171 }
1172
1173 if (cpu_has_feature(CPU_FTR_P9_TIDR) &&
1174 old_thread->tidr != new_thread->tidr)
1175 mtspr(SPRN_TIDR, new_thread->tidr);
1176#endif
1177
1178 thread_pkey_regs_restore(new_thread, old_thread);
1179}
1180
1181struct task_struct *__switch_to(struct task_struct *prev,
1182 struct task_struct *new)
1183{
1184 struct thread_struct *new_thread, *old_thread;
1185 struct task_struct *last;
1186#ifdef CONFIG_PPC_BOOK3S_64
1187 struct ppc64_tlb_batch *batch;
1188#endif
1189
1190 new_thread = &new->thread;
1191 old_thread = ¤t->thread;
1192
1193 WARN_ON(!irqs_disabled());
1194
1195#ifdef CONFIG_PPC_BOOK3S_64
1196 batch = this_cpu_ptr(&ppc64_tlb_batch);
1197 if (batch->active) {
1198 current_thread_info()->local_flags |= _TLF_LAZY_MMU;
1199 if (batch->index)
1200 __flush_tlb_pending(batch);
1201 batch->active = 0;
1202 }
1203#endif /* CONFIG_PPC_BOOK3S_64 */
1204
1205#ifdef CONFIG_PPC_ADV_DEBUG_REGS
1206 switch_booke_debug_regs(&new->thread.debug);
1207#else
1208/*
1209 * For PPC_BOOK3S_64, we use the hw-breakpoint interfaces that would
1210 * schedule DABR
1211 */
1212#ifndef CONFIG_HAVE_HW_BREAKPOINT
1213 switch_hw_breakpoint(new);
1214#endif /* CONFIG_HAVE_HW_BREAKPOINT */
1215#endif
1216
1217 /*
1218 * We need to save SPRs before treclaim/trecheckpoint as these will
1219 * change a number of them.
1220 */
1221 save_sprs(&prev->thread);
1222
1223 /* Save FPU, Altivec, VSX and SPE state */
1224 giveup_all(prev);
1225
1226 __switch_to_tm(prev, new);
1227
1228 if (!radix_enabled()) {
1229 /*
1230 * We can't take a PMU exception inside _switch() since there
1231 * is a window where the kernel stack SLB and the kernel stack
1232 * are out of sync. Hard disable here.
1233 */
1234 hard_irq_disable();
1235 }
1236
1237 /*
1238 * Call restore_sprs() before calling _switch(). If we move it after
1239 * _switch() then we miss out on calling it for new tasks. The reason
1240 * for this is we manually create a stack frame for new tasks that
1241 * directly returns through ret_from_fork() or
1242 * ret_from_kernel_thread(). See copy_thread() for details.
1243 */
1244 restore_sprs(old_thread, new_thread);
1245
1246 last = _switch(old_thread, new_thread);
1247
1248#ifdef CONFIG_PPC_BOOK3S_64
1249 if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
1250 current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
1251 batch = this_cpu_ptr(&ppc64_tlb_batch);
1252 batch->active = 1;
1253 }
1254
1255 if (current->thread.regs) {
1256 restore_math(current->thread.regs);
1257
1258 /*
1259 * The copy-paste buffer can only store into foreign real
1260 * addresses, so unprivileged processes can not see the
1261 * data or use it in any way unless they have foreign real
1262 * mappings. If the new process has the foreign real address
1263 * mappings, we must issue a cp_abort to clear any state and
1264 * prevent snooping, corruption or a covert channel.
1265 */
1266 if (current->mm &&
1267 atomic_read(¤t->mm->context.vas_windows))
1268 asm volatile(PPC_CP_ABORT);
1269 }
1270#endif /* CONFIG_PPC_BOOK3S_64 */
1271
1272 return last;
1273}
1274
1275#define NR_INSN_TO_PRINT 16
1276
1277static void show_instructions(struct pt_regs *regs)
1278{
1279 int i;
1280 unsigned long nip = regs->nip;
1281 unsigned long pc = regs->nip - (NR_INSN_TO_PRINT * 3 / 4 * sizeof(int));
1282
1283 printk("Instruction dump:");
1284
1285 /*
1286 * If we were executing with the MMU off for instructions, adjust pc
1287 * rather than printing XXXXXXXX.
1288 */
1289 if (!IS_ENABLED(CONFIG_BOOKE) && !(regs->msr & MSR_IR)) {
1290 pc = (unsigned long)phys_to_virt(pc);
1291 nip = (unsigned long)phys_to_virt(regs->nip);
1292 }
1293
1294 for (i = 0; i < NR_INSN_TO_PRINT; i++) {
1295 int instr;
1296
1297 if (!(i % 8))
1298 pr_cont("\n");
1299
1300 if (!__kernel_text_address(pc) ||
1301 get_kernel_nofault(instr, (const void *)pc)) {
1302 pr_cont("XXXXXXXX ");
1303 } else {
1304 if (nip == pc)
1305 pr_cont("<%08x> ", instr);
1306 else
1307 pr_cont("%08x ", instr);
1308 }
1309
1310 pc += sizeof(int);
1311 }
1312
1313 pr_cont("\n");
1314}
1315
1316void show_user_instructions(struct pt_regs *regs)
1317{
1318 unsigned long pc;
1319 int n = NR_INSN_TO_PRINT;
1320 struct seq_buf s;
1321 char buf[96]; /* enough for 8 times 9 + 2 chars */
1322
1323 pc = regs->nip - (NR_INSN_TO_PRINT * 3 / 4 * sizeof(int));
1324
1325 seq_buf_init(&s, buf, sizeof(buf));
1326
1327 while (n) {
1328 int i;
1329
1330 seq_buf_clear(&s);
1331
1332 for (i = 0; i < 8 && n; i++, n--, pc += sizeof(int)) {
1333 int instr;
1334
1335 if (copy_from_user_nofault(&instr, (void __user *)pc,
1336 sizeof(instr))) {
1337 seq_buf_printf(&s, "XXXXXXXX ");
1338 continue;
1339 }
1340 seq_buf_printf(&s, regs->nip == pc ? "<%08x> " : "%08x ", instr);
1341 }
1342
1343 if (!seq_buf_has_overflowed(&s))
1344 pr_info("%s[%d]: code: %s\n", current->comm,
1345 current->pid, s.buffer);
1346 }
1347}
1348
1349struct regbit {
1350 unsigned long bit;
1351 const char *name;
1352};
1353
1354static struct regbit msr_bits[] = {
1355#if defined(CONFIG_PPC64) && !defined(CONFIG_BOOKE)
1356 {MSR_SF, "SF"},
1357 {MSR_HV, "HV"},
1358#endif
1359 {MSR_VEC, "VEC"},
1360 {MSR_VSX, "VSX"},
1361#ifdef CONFIG_BOOKE
1362 {MSR_CE, "CE"},
1363#endif
1364 {MSR_EE, "EE"},
1365 {MSR_PR, "PR"},
1366 {MSR_FP, "FP"},
1367 {MSR_ME, "ME"},
1368#ifdef CONFIG_BOOKE
1369 {MSR_DE, "DE"},
1370#else
1371 {MSR_SE, "SE"},
1372 {MSR_BE, "BE"},
1373#endif
1374 {MSR_IR, "IR"},
1375 {MSR_DR, "DR"},
1376 {MSR_PMM, "PMM"},
1377#ifndef CONFIG_BOOKE
1378 {MSR_RI, "RI"},
1379 {MSR_LE, "LE"},
1380#endif
1381 {0, NULL}
1382};
1383
1384static void print_bits(unsigned long val, struct regbit *bits, const char *sep)
1385{
1386 const char *s = "";
1387
1388 for (; bits->bit; ++bits)
1389 if (val & bits->bit) {
1390 pr_cont("%s%s", s, bits->name);
1391 s = sep;
1392 }
1393}
1394
1395#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1396static struct regbit msr_tm_bits[] = {
1397 {MSR_TS_T, "T"},
1398 {MSR_TS_S, "S"},
1399 {MSR_TM, "E"},
1400 {0, NULL}
1401};
1402
1403static void print_tm_bits(unsigned long val)
1404{
1405/*
1406 * This only prints something if at least one of the TM bit is set.
1407 * Inside the TM[], the output means:
1408 * E: Enabled (bit 32)
1409 * S: Suspended (bit 33)
1410 * T: Transactional (bit 34)
1411 */
1412 if (val & (MSR_TM | MSR_TS_S | MSR_TS_T)) {
1413 pr_cont(",TM[");
1414 print_bits(val, msr_tm_bits, "");
1415 pr_cont("]");
1416 }
1417}
1418#else
1419static void print_tm_bits(unsigned long val) {}
1420#endif
1421
1422static void print_msr_bits(unsigned long val)
1423{
1424 pr_cont("<");
1425 print_bits(val, msr_bits, ",");
1426 print_tm_bits(val);
1427 pr_cont(">");
1428}
1429
1430#ifdef CONFIG_PPC64
1431#define REG "%016lx"
1432#define REGS_PER_LINE 4
1433#define LAST_VOLATILE 13
1434#else
1435#define REG "%08lx"
1436#define REGS_PER_LINE 8
1437#define LAST_VOLATILE 12
1438#endif
1439
1440void show_regs(struct pt_regs * regs)
1441{
1442 int i, trap;
1443
1444 show_regs_print_info(KERN_DEFAULT);
1445
1446 printk("NIP: "REG" LR: "REG" CTR: "REG"\n",
1447 regs->nip, regs->link, regs->ctr);
1448 printk("REGS: %px TRAP: %04lx %s (%s)\n",
1449 regs, regs->trap, print_tainted(), init_utsname()->release);
1450 printk("MSR: "REG" ", regs->msr);
1451 print_msr_bits(regs->msr);
1452 pr_cont(" CR: %08lx XER: %08lx\n", regs->ccr, regs->xer);
1453 trap = TRAP(regs);
1454 if (!trap_is_syscall(regs) && cpu_has_feature(CPU_FTR_CFAR))
1455 pr_cont("CFAR: "REG" ", regs->orig_gpr3);
1456 if (trap == 0x200 || trap == 0x300 || trap == 0x600)
1457#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
1458 pr_cont("DEAR: "REG" ESR: "REG" ", regs->dar, regs->dsisr);
1459#else
1460 pr_cont("DAR: "REG" DSISR: %08lx ", regs->dar, regs->dsisr);
1461#endif
1462#ifdef CONFIG_PPC64
1463 pr_cont("IRQMASK: %lx ", regs->softe);
1464#endif
1465#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1466 if (MSR_TM_ACTIVE(regs->msr))
1467 pr_cont("\nPACATMSCRATCH: %016llx ", get_paca()->tm_scratch);
1468#endif
1469
1470 for (i = 0; i < 32; i++) {
1471 if ((i % REGS_PER_LINE) == 0)
1472 pr_cont("\nGPR%02d: ", i);
1473 pr_cont(REG " ", regs->gpr[i]);
1474 if (i == LAST_VOLATILE && !FULL_REGS(regs))
1475 break;
1476 }
1477 pr_cont("\n");
1478#ifdef CONFIG_KALLSYMS
1479 /*
1480 * Lookup NIP late so we have the best change of getting the
1481 * above info out without failing
1482 */
1483 printk("NIP ["REG"] %pS\n", regs->nip, (void *)regs->nip);
1484 printk("LR ["REG"] %pS\n", regs->link, (void *)regs->link);
1485#endif
1486 show_stack(current, (unsigned long *) regs->gpr[1], KERN_DEFAULT);
1487 if (!user_mode(regs))
1488 show_instructions(regs);
1489}
1490
1491void flush_thread(void)
1492{
1493#ifdef CONFIG_HAVE_HW_BREAKPOINT
1494 flush_ptrace_hw_breakpoint(current);
1495#else /* CONFIG_HAVE_HW_BREAKPOINT */
1496 set_debug_reg_defaults(¤t->thread);
1497#endif /* CONFIG_HAVE_HW_BREAKPOINT */
1498}
1499
1500#ifdef CONFIG_PPC_BOOK3S_64
1501void arch_setup_new_exec(void)
1502{
1503 if (radix_enabled())
1504 return;
1505 hash__setup_new_exec();
1506}
1507#endif
1508
1509#ifdef CONFIG_PPC64
1510/**
1511 * Assign a TIDR (thread ID) for task @t and set it in the thread
1512 * structure. For now, we only support setting TIDR for 'current' task.
1513 *
1514 * Since the TID value is a truncated form of it PID, it is possible
1515 * (but unlikely) for 2 threads to have the same TID. In the unlikely event
1516 * that 2 threads share the same TID and are waiting, one of the following
1517 * cases will happen:
1518 *
1519 * 1. The correct thread is running, the wrong thread is not
1520 * In this situation, the correct thread is woken and proceeds to pass it's
1521 * condition check.
1522 *
1523 * 2. Neither threads are running
1524 * In this situation, neither thread will be woken. When scheduled, the waiting
1525 * threads will execute either a wait, which will return immediately, followed
1526 * by a condition check, which will pass for the correct thread and fail
1527 * for the wrong thread, or they will execute the condition check immediately.
1528 *
1529 * 3. The wrong thread is running, the correct thread is not
1530 * The wrong thread will be woken, but will fail it's condition check and
1531 * re-execute wait. The correct thread, when scheduled, will execute either
1532 * it's condition check (which will pass), or wait, which returns immediately
1533 * when called the first time after the thread is scheduled, followed by it's
1534 * condition check (which will pass).
1535 *
1536 * 4. Both threads are running
1537 * Both threads will be woken. The wrong thread will fail it's condition check
1538 * and execute another wait, while the correct thread will pass it's condition
1539 * check.
1540 *
1541 * @t: the task to set the thread ID for
1542 */
1543int set_thread_tidr(struct task_struct *t)
1544{
1545 if (!cpu_has_feature(CPU_FTR_P9_TIDR))
1546 return -EINVAL;
1547
1548 if (t != current)
1549 return -EINVAL;
1550
1551 if (t->thread.tidr)
1552 return 0;
1553
1554 t->thread.tidr = (u16)task_pid_nr(t);
1555 mtspr(SPRN_TIDR, t->thread.tidr);
1556
1557 return 0;
1558}
1559EXPORT_SYMBOL_GPL(set_thread_tidr);
1560
1561#endif /* CONFIG_PPC64 */
1562
1563void
1564release_thread(struct task_struct *t)
1565{
1566}
1567
1568/*
1569 * this gets called so that we can store coprocessor state into memory and
1570 * copy the current task into the new thread.
1571 */
1572int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
1573{
1574 flush_all_to_thread(src);
1575 /*
1576 * Flush TM state out so we can copy it. __switch_to_tm() does this
1577 * flush but it removes the checkpointed state from the current CPU and
1578 * transitions the CPU out of TM mode. Hence we need to call
1579 * tm_recheckpoint_new_task() (on the same task) to restore the
1580 * checkpointed state back and the TM mode.
1581 *
1582 * Can't pass dst because it isn't ready. Doesn't matter, passing
1583 * dst is only important for __switch_to()
1584 */
1585 __switch_to_tm(src, src);
1586
1587 *dst = *src;
1588
1589 clear_task_ebb(dst);
1590
1591 return 0;
1592}
1593
1594static void setup_ksp_vsid(struct task_struct *p, unsigned long sp)
1595{
1596#ifdef CONFIG_PPC_BOOK3S_64
1597 unsigned long sp_vsid;
1598 unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp;
1599
1600 if (radix_enabled())
1601 return;
1602
1603 if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
1604 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_1T)
1605 << SLB_VSID_SHIFT_1T;
1606 else
1607 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_256M)
1608 << SLB_VSID_SHIFT;
1609 sp_vsid |= SLB_VSID_KERNEL | llp;
1610 p->thread.ksp_vsid = sp_vsid;
1611#endif
1612}
1613
1614/*
1615 * Copy a thread..
1616 */
1617
1618/*
1619 * Copy architecture-specific thread state
1620 */
1621int copy_thread(unsigned long clone_flags, unsigned long usp,
1622 unsigned long kthread_arg, struct task_struct *p,
1623 unsigned long tls)
1624{
1625 struct pt_regs *childregs, *kregs;
1626 extern void ret_from_fork(void);
1627 extern void ret_from_fork_scv(void);
1628 extern void ret_from_kernel_thread(void);
1629 void (*f)(void);
1630 unsigned long sp = (unsigned long)task_stack_page(p) + THREAD_SIZE;
1631 struct thread_info *ti = task_thread_info(p);
1632#ifdef CONFIG_HAVE_HW_BREAKPOINT
1633 int i;
1634#endif
1635
1636 klp_init_thread_info(p);
1637
1638 /* Copy registers */
1639 sp -= sizeof(struct pt_regs);
1640 childregs = (struct pt_regs *) sp;
1641 if (unlikely(p->flags & PF_KTHREAD)) {
1642 /* kernel thread */
1643 memset(childregs, 0, sizeof(struct pt_regs));
1644 childregs->gpr[1] = sp + sizeof(struct pt_regs);
1645 /* function */
1646 if (usp)
1647 childregs->gpr[14] = ppc_function_entry((void *)usp);
1648#ifdef CONFIG_PPC64
1649 clear_tsk_thread_flag(p, TIF_32BIT);
1650 childregs->softe = IRQS_ENABLED;
1651#endif
1652 childregs->gpr[15] = kthread_arg;
1653 p->thread.regs = NULL; /* no user register state */
1654 ti->flags |= _TIF_RESTOREALL;
1655 f = ret_from_kernel_thread;
1656 } else {
1657 /* user thread */
1658 struct pt_regs *regs = current_pt_regs();
1659 CHECK_FULL_REGS(regs);
1660 *childregs = *regs;
1661 if (usp)
1662 childregs->gpr[1] = usp;
1663 p->thread.regs = childregs;
1664 /* 64s sets this in ret_from_fork */
1665 if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64))
1666 childregs->gpr[3] = 0; /* Result from fork() */
1667 if (clone_flags & CLONE_SETTLS) {
1668 if (!is_32bit_task())
1669 childregs->gpr[13] = tls;
1670 else
1671 childregs->gpr[2] = tls;
1672 }
1673
1674 if (trap_is_scv(regs))
1675 f = ret_from_fork_scv;
1676 else
1677 f = ret_from_fork;
1678 }
1679 childregs->msr &= ~(MSR_FP|MSR_VEC|MSR_VSX);
1680 sp -= STACK_FRAME_OVERHEAD;
1681
1682 /*
1683 * The way this works is that at some point in the future
1684 * some task will call _switch to switch to the new task.
1685 * That will pop off the stack frame created below and start
1686 * the new task running at ret_from_fork. The new task will
1687 * do some house keeping and then return from the fork or clone
1688 * system call, using the stack frame created above.
1689 */
1690 ((unsigned long *)sp)[0] = 0;
1691 sp -= sizeof(struct pt_regs);
1692 kregs = (struct pt_regs *) sp;
1693 sp -= STACK_FRAME_OVERHEAD;
1694 p->thread.ksp = sp;
1695#ifdef CONFIG_PPC32
1696 p->thread.ksp_limit = (unsigned long)end_of_stack(p);
1697#endif
1698#ifdef CONFIG_HAVE_HW_BREAKPOINT
1699 for (i = 0; i < nr_wp_slots(); i++)
1700 p->thread.ptrace_bps[i] = NULL;
1701#endif
1702
1703 p->thread.fp_save_area = NULL;
1704#ifdef CONFIG_ALTIVEC
1705 p->thread.vr_save_area = NULL;
1706#endif
1707
1708 setup_ksp_vsid(p, sp);
1709
1710#ifdef CONFIG_PPC64
1711 if (cpu_has_feature(CPU_FTR_DSCR)) {
1712 p->thread.dscr_inherit = current->thread.dscr_inherit;
1713 p->thread.dscr = mfspr(SPRN_DSCR);
1714 }
1715 if (cpu_has_feature(CPU_FTR_HAS_PPR))
1716 childregs->ppr = DEFAULT_PPR;
1717
1718 p->thread.tidr = 0;
1719#endif
1720 kregs->nip = ppc_function_entry(f);
1721 return 0;
1722}
1723
1724void preload_new_slb_context(unsigned long start, unsigned long sp);
1725
1726/*
1727 * Set up a thread for executing a new program
1728 */
1729void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
1730{
1731#ifdef CONFIG_PPC64
1732 unsigned long load_addr = regs->gpr[2]; /* saved by ELF_PLAT_INIT */
1733
1734#ifdef CONFIG_PPC_BOOK3S_64
1735 if (!radix_enabled())
1736 preload_new_slb_context(start, sp);
1737#endif
1738#endif
1739
1740 /*
1741 * If we exec out of a kernel thread then thread.regs will not be
1742 * set. Do it now.
1743 */
1744 if (!current->thread.regs) {
1745 struct pt_regs *regs = task_stack_page(current) + THREAD_SIZE;
1746 current->thread.regs = regs - 1;
1747 }
1748
1749#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1750 /*
1751 * Clear any transactional state, we're exec()ing. The cause is
1752 * not important as there will never be a recheckpoint so it's not
1753 * user visible.
1754 */
1755 if (MSR_TM_SUSPENDED(mfmsr()))
1756 tm_reclaim_current(0);
1757#endif
1758
1759 memset(regs->gpr, 0, sizeof(regs->gpr));
1760 regs->ctr = 0;
1761 regs->link = 0;
1762 regs->xer = 0;
1763 regs->ccr = 0;
1764 regs->gpr[1] = sp;
1765
1766 /*
1767 * We have just cleared all the nonvolatile GPRs, so make
1768 * FULL_REGS(regs) return true. This is necessary to allow
1769 * ptrace to examine the thread immediately after exec.
1770 */
1771 SET_FULL_REGS(regs);
1772
1773#ifdef CONFIG_PPC32
1774 regs->mq = 0;
1775 regs->nip = start;
1776 regs->msr = MSR_USER;
1777#else
1778 if (!is_32bit_task()) {
1779 unsigned long entry;
1780
1781 if (is_elf2_task()) {
1782 /* Look ma, no function descriptors! */
1783 entry = start;
1784
1785 /*
1786 * Ulrich says:
1787 * The latest iteration of the ABI requires that when
1788 * calling a function (at its global entry point),
1789 * the caller must ensure r12 holds the entry point
1790 * address (so that the function can quickly
1791 * establish addressability).
1792 */
1793 regs->gpr[12] = start;
1794 /* Make sure that's restored on entry to userspace. */
1795 set_thread_flag(TIF_RESTOREALL);
1796 } else {
1797 unsigned long toc;
1798
1799 /* start is a relocated pointer to the function
1800 * descriptor for the elf _start routine. The first
1801 * entry in the function descriptor is the entry
1802 * address of _start and the second entry is the TOC
1803 * value we need to use.
1804 */
1805 __get_user(entry, (unsigned long __user *)start);
1806 __get_user(toc, (unsigned long __user *)start+1);
1807
1808 /* Check whether the e_entry function descriptor entries
1809 * need to be relocated before we can use them.
1810 */
1811 if (load_addr != 0) {
1812 entry += load_addr;
1813 toc += load_addr;
1814 }
1815 regs->gpr[2] = toc;
1816 }
1817 regs->nip = entry;
1818 regs->msr = MSR_USER64;
1819 } else {
1820 regs->nip = start;
1821 regs->gpr[2] = 0;
1822 regs->msr = MSR_USER32;
1823 }
1824#endif
1825#ifdef CONFIG_VSX
1826 current->thread.used_vsr = 0;
1827#endif
1828 current->thread.load_slb = 0;
1829 current->thread.load_fp = 0;
1830 memset(¤t->thread.fp_state, 0, sizeof(current->thread.fp_state));
1831 current->thread.fp_save_area = NULL;
1832#ifdef CONFIG_ALTIVEC
1833 memset(¤t->thread.vr_state, 0, sizeof(current->thread.vr_state));
1834 current->thread.vr_state.vscr.u[3] = 0x00010000; /* Java mode disabled */
1835 current->thread.vr_save_area = NULL;
1836 current->thread.vrsave = 0;
1837 current->thread.used_vr = 0;
1838 current->thread.load_vec = 0;
1839#endif /* CONFIG_ALTIVEC */
1840#ifdef CONFIG_SPE
1841 memset(current->thread.evr, 0, sizeof(current->thread.evr));
1842 current->thread.acc = 0;
1843 current->thread.spefscr = 0;
1844 current->thread.used_spe = 0;
1845#endif /* CONFIG_SPE */
1846#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1847 current->thread.tm_tfhar = 0;
1848 current->thread.tm_texasr = 0;
1849 current->thread.tm_tfiar = 0;
1850 current->thread.load_tm = 0;
1851#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
1852
1853 thread_pkey_regs_init(¤t->thread);
1854}
1855EXPORT_SYMBOL(start_thread);
1856
1857#define PR_FP_ALL_EXCEPT (PR_FP_EXC_DIV | PR_FP_EXC_OVF | PR_FP_EXC_UND \
1858 | PR_FP_EXC_RES | PR_FP_EXC_INV)
1859
1860int set_fpexc_mode(struct task_struct *tsk, unsigned int val)
1861{
1862 struct pt_regs *regs = tsk->thread.regs;
1863
1864 /* This is a bit hairy. If we are an SPE enabled processor
1865 * (have embedded fp) we store the IEEE exception enable flags in
1866 * fpexc_mode. fpexc_mode is also used for setting FP exception
1867 * mode (asyn, precise, disabled) for 'Classic' FP. */
1868 if (val & PR_FP_EXC_SW_ENABLE) {
1869#ifdef CONFIG_SPE
1870 if (cpu_has_feature(CPU_FTR_SPE)) {
1871 /*
1872 * When the sticky exception bits are set
1873 * directly by userspace, it must call prctl
1874 * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
1875 * in the existing prctl settings) or
1876 * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
1877 * the bits being set). <fenv.h> functions
1878 * saving and restoring the whole
1879 * floating-point environment need to do so
1880 * anyway to restore the prctl settings from
1881 * the saved environment.
1882 */
1883 tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
1884 tsk->thread.fpexc_mode = val &
1885 (PR_FP_EXC_SW_ENABLE | PR_FP_ALL_EXCEPT);
1886 return 0;
1887 } else {
1888 return -EINVAL;
1889 }
1890#else
1891 return -EINVAL;
1892#endif
1893 }
1894
1895 /* on a CONFIG_SPE this does not hurt us. The bits that
1896 * __pack_fe01 use do not overlap with bits used for
1897 * PR_FP_EXC_SW_ENABLE. Additionally, the MSR[FE0,FE1] bits
1898 * on CONFIG_SPE implementations are reserved so writing to
1899 * them does not change anything */
1900 if (val > PR_FP_EXC_PRECISE)
1901 return -EINVAL;
1902 tsk->thread.fpexc_mode = __pack_fe01(val);
1903 if (regs != NULL && (regs->msr & MSR_FP) != 0)
1904 regs->msr = (regs->msr & ~(MSR_FE0|MSR_FE1))
1905 | tsk->thread.fpexc_mode;
1906 return 0;
1907}
1908
1909int get_fpexc_mode(struct task_struct *tsk, unsigned long adr)
1910{
1911 unsigned int val;
1912
1913 if (tsk->thread.fpexc_mode & PR_FP_EXC_SW_ENABLE)
1914#ifdef CONFIG_SPE
1915 if (cpu_has_feature(CPU_FTR_SPE)) {
1916 /*
1917 * When the sticky exception bits are set
1918 * directly by userspace, it must call prctl
1919 * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
1920 * in the existing prctl settings) or
1921 * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
1922 * the bits being set). <fenv.h> functions
1923 * saving and restoring the whole
1924 * floating-point environment need to do so
1925 * anyway to restore the prctl settings from
1926 * the saved environment.
1927 */
1928 tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
1929 val = tsk->thread.fpexc_mode;
1930 } else
1931 return -EINVAL;
1932#else
1933 return -EINVAL;
1934#endif
1935 else
1936 val = __unpack_fe01(tsk->thread.fpexc_mode);
1937 return put_user(val, (unsigned int __user *) adr);
1938}
1939
1940int set_endian(struct task_struct *tsk, unsigned int val)
1941{
1942 struct pt_regs *regs = tsk->thread.regs;
1943
1944 if ((val == PR_ENDIAN_LITTLE && !cpu_has_feature(CPU_FTR_REAL_LE)) ||
1945 (val == PR_ENDIAN_PPC_LITTLE && !cpu_has_feature(CPU_FTR_PPC_LE)))
1946 return -EINVAL;
1947
1948 if (regs == NULL)
1949 return -EINVAL;
1950
1951 if (val == PR_ENDIAN_BIG)
1952 regs->msr &= ~MSR_LE;
1953 else if (val == PR_ENDIAN_LITTLE || val == PR_ENDIAN_PPC_LITTLE)
1954 regs->msr |= MSR_LE;
1955 else
1956 return -EINVAL;
1957
1958 return 0;
1959}
1960
1961int get_endian(struct task_struct *tsk, unsigned long adr)
1962{
1963 struct pt_regs *regs = tsk->thread.regs;
1964 unsigned int val;
1965
1966 if (!cpu_has_feature(CPU_FTR_PPC_LE) &&
1967 !cpu_has_feature(CPU_FTR_REAL_LE))
1968 return -EINVAL;
1969
1970 if (regs == NULL)
1971 return -EINVAL;
1972
1973 if (regs->msr & MSR_LE) {
1974 if (cpu_has_feature(CPU_FTR_REAL_LE))
1975 val = PR_ENDIAN_LITTLE;
1976 else
1977 val = PR_ENDIAN_PPC_LITTLE;
1978 } else
1979 val = PR_ENDIAN_BIG;
1980
1981 return put_user(val, (unsigned int __user *)adr);
1982}
1983
1984int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
1985{
1986 tsk->thread.align_ctl = val;
1987 return 0;
1988}
1989
1990int get_unalign_ctl(struct task_struct *tsk, unsigned long adr)
1991{
1992 return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr);
1993}
1994
1995static inline int valid_irq_stack(unsigned long sp, struct task_struct *p,
1996 unsigned long nbytes)
1997{
1998 unsigned long stack_page;
1999 unsigned long cpu = task_cpu(p);
2000
2001 stack_page = (unsigned long)hardirq_ctx[cpu];
2002 if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes)
2003 return 1;
2004
2005 stack_page = (unsigned long)softirq_ctx[cpu];
2006 if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes)
2007 return 1;
2008
2009 return 0;
2010}
2011
2012static inline int valid_emergency_stack(unsigned long sp, struct task_struct *p,
2013 unsigned long nbytes)
2014{
2015#ifdef CONFIG_PPC64
2016 unsigned long stack_page;
2017 unsigned long cpu = task_cpu(p);
2018
2019 stack_page = (unsigned long)paca_ptrs[cpu]->emergency_sp - THREAD_SIZE;
2020 if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes)
2021 return 1;
2022
2023# ifdef CONFIG_PPC_BOOK3S_64
2024 stack_page = (unsigned long)paca_ptrs[cpu]->nmi_emergency_sp - THREAD_SIZE;
2025 if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes)
2026 return 1;
2027
2028 stack_page = (unsigned long)paca_ptrs[cpu]->mc_emergency_sp - THREAD_SIZE;
2029 if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes)
2030 return 1;
2031# endif
2032#endif
2033
2034 return 0;
2035}
2036
2037
2038int validate_sp(unsigned long sp, struct task_struct *p,
2039 unsigned long nbytes)
2040{
2041 unsigned long stack_page = (unsigned long)task_stack_page(p);
2042
2043 if (sp < THREAD_SIZE)
2044 return 0;
2045
2046 if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes)
2047 return 1;
2048
2049 if (valid_irq_stack(sp, p, nbytes))
2050 return 1;
2051
2052 return valid_emergency_stack(sp, p, nbytes);
2053}
2054
2055EXPORT_SYMBOL(validate_sp);
2056
2057static unsigned long __get_wchan(struct task_struct *p)
2058{
2059 unsigned long ip, sp;
2060 int count = 0;
2061
2062 if (!p || p == current || p->state == TASK_RUNNING)
2063 return 0;
2064
2065 sp = p->thread.ksp;
2066 if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD))
2067 return 0;
2068
2069 do {
2070 sp = *(unsigned long *)sp;
2071 if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD) ||
2072 p->state == TASK_RUNNING)
2073 return 0;
2074 if (count > 0) {
2075 ip = ((unsigned long *)sp)[STACK_FRAME_LR_SAVE];
2076 if (!in_sched_functions(ip))
2077 return ip;
2078 }
2079 } while (count++ < 16);
2080 return 0;
2081}
2082
2083unsigned long get_wchan(struct task_struct *p)
2084{
2085 unsigned long ret;
2086
2087 if (!try_get_task_stack(p))
2088 return 0;
2089
2090 ret = __get_wchan(p);
2091
2092 put_task_stack(p);
2093
2094 return ret;
2095}
2096
2097static int kstack_depth_to_print = CONFIG_PRINT_STACK_DEPTH;
2098
2099void show_stack(struct task_struct *tsk, unsigned long *stack,
2100 const char *loglvl)
2101{
2102 unsigned long sp, ip, lr, newsp;
2103 int count = 0;
2104 int firstframe = 1;
2105#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2106 unsigned long ret_addr;
2107 int ftrace_idx = 0;
2108#endif
2109
2110 if (tsk == NULL)
2111 tsk = current;
2112
2113 if (!try_get_task_stack(tsk))
2114 return;
2115
2116 sp = (unsigned long) stack;
2117 if (sp == 0) {
2118 if (tsk == current)
2119 sp = current_stack_frame();
2120 else
2121 sp = tsk->thread.ksp;
2122 }
2123
2124 lr = 0;
2125 printk("%sCall Trace:\n", loglvl);
2126 do {
2127 if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD))
2128 break;
2129
2130 stack = (unsigned long *) sp;
2131 newsp = stack[0];
2132 ip = stack[STACK_FRAME_LR_SAVE];
2133 if (!firstframe || ip != lr) {
2134 printk("%s["REG"] ["REG"] %pS",
2135 loglvl, sp, ip, (void *)ip);
2136#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2137 ret_addr = ftrace_graph_ret_addr(current,
2138 &ftrace_idx, ip, stack);
2139 if (ret_addr != ip)
2140 pr_cont(" (%pS)", (void *)ret_addr);
2141#endif
2142 if (firstframe)
2143 pr_cont(" (unreliable)");
2144 pr_cont("\n");
2145 }
2146 firstframe = 0;
2147
2148 /*
2149 * See if this is an exception frame.
2150 * We look for the "regshere" marker in the current frame.
2151 */
2152 if (validate_sp(sp, tsk, STACK_INT_FRAME_SIZE)
2153 && stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
2154 struct pt_regs *regs = (struct pt_regs *)
2155 (sp + STACK_FRAME_OVERHEAD);
2156 lr = regs->link;
2157 printk("%s--- interrupt: %lx at %pS\n LR = %pS\n",
2158 loglvl, regs->trap,
2159 (void *)regs->nip, (void *)lr);
2160 firstframe = 1;
2161 }
2162
2163 sp = newsp;
2164 } while (count++ < kstack_depth_to_print);
2165
2166 put_task_stack(tsk);
2167}
2168
2169#ifdef CONFIG_PPC64
2170/* Called with hard IRQs off */
2171void notrace __ppc64_runlatch_on(void)
2172{
2173 struct thread_info *ti = current_thread_info();
2174
2175 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
2176 /*
2177 * Least significant bit (RUN) is the only writable bit of
2178 * the CTRL register, so we can avoid mfspr. 2.06 is not the
2179 * earliest ISA where this is the case, but it's convenient.
2180 */
2181 mtspr(SPRN_CTRLT, CTRL_RUNLATCH);
2182 } else {
2183 unsigned long ctrl;
2184
2185 /*
2186 * Some architectures (e.g., Cell) have writable fields other
2187 * than RUN, so do the read-modify-write.
2188 */
2189 ctrl = mfspr(SPRN_CTRLF);
2190 ctrl |= CTRL_RUNLATCH;
2191 mtspr(SPRN_CTRLT, ctrl);
2192 }
2193
2194 ti->local_flags |= _TLF_RUNLATCH;
2195}
2196
2197/* Called with hard IRQs off */
2198void notrace __ppc64_runlatch_off(void)
2199{
2200 struct thread_info *ti = current_thread_info();
2201
2202 ti->local_flags &= ~_TLF_RUNLATCH;
2203
2204 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
2205 mtspr(SPRN_CTRLT, 0);
2206 } else {
2207 unsigned long ctrl;
2208
2209 ctrl = mfspr(SPRN_CTRLF);
2210 ctrl &= ~CTRL_RUNLATCH;
2211 mtspr(SPRN_CTRLT, ctrl);
2212 }
2213}
2214#endif /* CONFIG_PPC64 */
2215
2216unsigned long arch_align_stack(unsigned long sp)
2217{
2218 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
2219 sp -= get_random_int() & ~PAGE_MASK;
2220 return sp & ~0xf;
2221}
2222
2223static inline unsigned long brk_rnd(void)
2224{
2225 unsigned long rnd = 0;
2226
2227 /* 8MB for 32bit, 1GB for 64bit */
2228 if (is_32bit_task())
2229 rnd = (get_random_long() % (1UL<<(23-PAGE_SHIFT)));
2230 else
2231 rnd = (get_random_long() % (1UL<<(30-PAGE_SHIFT)));
2232
2233 return rnd << PAGE_SHIFT;
2234}
2235
2236unsigned long arch_randomize_brk(struct mm_struct *mm)
2237{
2238 unsigned long base = mm->brk;
2239 unsigned long ret;
2240
2241#ifdef CONFIG_PPC_BOOK3S_64
2242 /*
2243 * If we are using 1TB segments and we are allowed to randomise
2244 * the heap, we can put it above 1TB so it is backed by a 1TB
2245 * segment. Otherwise the heap will be in the bottom 1TB
2246 * which always uses 256MB segments and this may result in a
2247 * performance penalty. We don't need to worry about radix. For
2248 * radix, mmu_highuser_ssize remains unchanged from 256MB.
2249 */
2250 if (!is_32bit_task() && (mmu_highuser_ssize == MMU_SEGSIZE_1T))
2251 base = max_t(unsigned long, mm->brk, 1UL << SID_SHIFT_1T);
2252#endif
2253
2254 ret = PAGE_ALIGN(base + brk_rnd());
2255
2256 if (ret < mm->brk)
2257 return mm->brk;
2258
2259 return ret;
2260}
2261
1/*
2 * Derived from "arch/i386/kernel/process.c"
3 * Copyright (C) 1995 Linus Torvalds
4 *
5 * Updated and modified by Cort Dougan (cort@cs.nmt.edu) and
6 * Paul Mackerras (paulus@cs.anu.edu.au)
7 *
8 * PowerPC version
9 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
10 *
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version
14 * 2 of the License, or (at your option) any later version.
15 */
16
17#include <linux/errno.h>
18#include <linux/sched.h>
19#include <linux/kernel.h>
20#include <linux/mm.h>
21#include <linux/smp.h>
22#include <linux/stddef.h>
23#include <linux/unistd.h>
24#include <linux/ptrace.h>
25#include <linux/slab.h>
26#include <linux/user.h>
27#include <linux/elf.h>
28#include <linux/init.h>
29#include <linux/prctl.h>
30#include <linux/init_task.h>
31#include <linux/module.h>
32#include <linux/kallsyms.h>
33#include <linux/mqueue.h>
34#include <linux/hardirq.h>
35#include <linux/utsname.h>
36#include <linux/ftrace.h>
37#include <linux/kernel_stat.h>
38#include <linux/personality.h>
39#include <linux/random.h>
40#include <linux/hw_breakpoint.h>
41
42#include <asm/pgtable.h>
43#include <asm/uaccess.h>
44#include <asm/system.h>
45#include <asm/io.h>
46#include <asm/processor.h>
47#include <asm/mmu.h>
48#include <asm/prom.h>
49#include <asm/machdep.h>
50#include <asm/time.h>
51#include <asm/syscalls.h>
52#ifdef CONFIG_PPC64
53#include <asm/firmware.h>
54#endif
55#include <linux/kprobes.h>
56#include <linux/kdebug.h>
57
58extern unsigned long _get_SP(void);
59
60#ifndef CONFIG_SMP
61struct task_struct *last_task_used_math = NULL;
62struct task_struct *last_task_used_altivec = NULL;
63struct task_struct *last_task_used_vsx = NULL;
64struct task_struct *last_task_used_spe = NULL;
65#endif
66
67/*
68 * Make sure the floating-point register state in the
69 * the thread_struct is up to date for task tsk.
70 */
71void flush_fp_to_thread(struct task_struct *tsk)
72{
73 if (tsk->thread.regs) {
74 /*
75 * We need to disable preemption here because if we didn't,
76 * another process could get scheduled after the regs->msr
77 * test but before we have finished saving the FP registers
78 * to the thread_struct. That process could take over the
79 * FPU, and then when we get scheduled again we would store
80 * bogus values for the remaining FP registers.
81 */
82 preempt_disable();
83 if (tsk->thread.regs->msr & MSR_FP) {
84#ifdef CONFIG_SMP
85 /*
86 * This should only ever be called for current or
87 * for a stopped child process. Since we save away
88 * the FP register state on context switch on SMP,
89 * there is something wrong if a stopped child appears
90 * to still have its FP state in the CPU registers.
91 */
92 BUG_ON(tsk != current);
93#endif
94 giveup_fpu(tsk);
95 }
96 preempt_enable();
97 }
98}
99EXPORT_SYMBOL_GPL(flush_fp_to_thread);
100
101void enable_kernel_fp(void)
102{
103 WARN_ON(preemptible());
104
105#ifdef CONFIG_SMP
106 if (current->thread.regs && (current->thread.regs->msr & MSR_FP))
107 giveup_fpu(current);
108 else
109 giveup_fpu(NULL); /* just enables FP for kernel */
110#else
111 giveup_fpu(last_task_used_math);
112#endif /* CONFIG_SMP */
113}
114EXPORT_SYMBOL(enable_kernel_fp);
115
116#ifdef CONFIG_ALTIVEC
117void enable_kernel_altivec(void)
118{
119 WARN_ON(preemptible());
120
121#ifdef CONFIG_SMP
122 if (current->thread.regs && (current->thread.regs->msr & MSR_VEC))
123 giveup_altivec(current);
124 else
125 giveup_altivec(NULL); /* just enable AltiVec for kernel - force */
126#else
127 giveup_altivec(last_task_used_altivec);
128#endif /* CONFIG_SMP */
129}
130EXPORT_SYMBOL(enable_kernel_altivec);
131
132/*
133 * Make sure the VMX/Altivec register state in the
134 * the thread_struct is up to date for task tsk.
135 */
136void flush_altivec_to_thread(struct task_struct *tsk)
137{
138 if (tsk->thread.regs) {
139 preempt_disable();
140 if (tsk->thread.regs->msr & MSR_VEC) {
141#ifdef CONFIG_SMP
142 BUG_ON(tsk != current);
143#endif
144 giveup_altivec(tsk);
145 }
146 preempt_enable();
147 }
148}
149EXPORT_SYMBOL_GPL(flush_altivec_to_thread);
150#endif /* CONFIG_ALTIVEC */
151
152#ifdef CONFIG_VSX
153#if 0
154/* not currently used, but some crazy RAID module might want to later */
155void enable_kernel_vsx(void)
156{
157 WARN_ON(preemptible());
158
159#ifdef CONFIG_SMP
160 if (current->thread.regs && (current->thread.regs->msr & MSR_VSX))
161 giveup_vsx(current);
162 else
163 giveup_vsx(NULL); /* just enable vsx for kernel - force */
164#else
165 giveup_vsx(last_task_used_vsx);
166#endif /* CONFIG_SMP */
167}
168EXPORT_SYMBOL(enable_kernel_vsx);
169#endif
170
171void giveup_vsx(struct task_struct *tsk)
172{
173 giveup_fpu(tsk);
174 giveup_altivec(tsk);
175 __giveup_vsx(tsk);
176}
177
178void flush_vsx_to_thread(struct task_struct *tsk)
179{
180 if (tsk->thread.regs) {
181 preempt_disable();
182 if (tsk->thread.regs->msr & MSR_VSX) {
183#ifdef CONFIG_SMP
184 BUG_ON(tsk != current);
185#endif
186 giveup_vsx(tsk);
187 }
188 preempt_enable();
189 }
190}
191EXPORT_SYMBOL_GPL(flush_vsx_to_thread);
192#endif /* CONFIG_VSX */
193
194#ifdef CONFIG_SPE
195
196void enable_kernel_spe(void)
197{
198 WARN_ON(preemptible());
199
200#ifdef CONFIG_SMP
201 if (current->thread.regs && (current->thread.regs->msr & MSR_SPE))
202 giveup_spe(current);
203 else
204 giveup_spe(NULL); /* just enable SPE for kernel - force */
205#else
206 giveup_spe(last_task_used_spe);
207#endif /* __SMP __ */
208}
209EXPORT_SYMBOL(enable_kernel_spe);
210
211void flush_spe_to_thread(struct task_struct *tsk)
212{
213 if (tsk->thread.regs) {
214 preempt_disable();
215 if (tsk->thread.regs->msr & MSR_SPE) {
216#ifdef CONFIG_SMP
217 BUG_ON(tsk != current);
218#endif
219 tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
220 giveup_spe(tsk);
221 }
222 preempt_enable();
223 }
224}
225#endif /* CONFIG_SPE */
226
227#ifndef CONFIG_SMP
228/*
229 * If we are doing lazy switching of CPU state (FP, altivec or SPE),
230 * and the current task has some state, discard it.
231 */
232void discard_lazy_cpu_state(void)
233{
234 preempt_disable();
235 if (last_task_used_math == current)
236 last_task_used_math = NULL;
237#ifdef CONFIG_ALTIVEC
238 if (last_task_used_altivec == current)
239 last_task_used_altivec = NULL;
240#endif /* CONFIG_ALTIVEC */
241#ifdef CONFIG_VSX
242 if (last_task_used_vsx == current)
243 last_task_used_vsx = NULL;
244#endif /* CONFIG_VSX */
245#ifdef CONFIG_SPE
246 if (last_task_used_spe == current)
247 last_task_used_spe = NULL;
248#endif
249 preempt_enable();
250}
251#endif /* CONFIG_SMP */
252
253#ifdef CONFIG_PPC_ADV_DEBUG_REGS
254void do_send_trap(struct pt_regs *regs, unsigned long address,
255 unsigned long error_code, int signal_code, int breakpt)
256{
257 siginfo_t info;
258
259 if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
260 11, SIGSEGV) == NOTIFY_STOP)
261 return;
262
263 /* Deliver the signal to userspace */
264 info.si_signo = SIGTRAP;
265 info.si_errno = breakpt; /* breakpoint or watchpoint id */
266 info.si_code = signal_code;
267 info.si_addr = (void __user *)address;
268 force_sig_info(SIGTRAP, &info, current);
269}
270#else /* !CONFIG_PPC_ADV_DEBUG_REGS */
271void do_dabr(struct pt_regs *regs, unsigned long address,
272 unsigned long error_code)
273{
274 siginfo_t info;
275
276 if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
277 11, SIGSEGV) == NOTIFY_STOP)
278 return;
279
280 if (debugger_dabr_match(regs))
281 return;
282
283 /* Clear the DABR */
284 set_dabr(0);
285
286 /* Deliver the signal to userspace */
287 info.si_signo = SIGTRAP;
288 info.si_errno = 0;
289 info.si_code = TRAP_HWBKPT;
290 info.si_addr = (void __user *)address;
291 force_sig_info(SIGTRAP, &info, current);
292}
293#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
294
295static DEFINE_PER_CPU(unsigned long, current_dabr);
296
297#ifdef CONFIG_PPC_ADV_DEBUG_REGS
298/*
299 * Set the debug registers back to their default "safe" values.
300 */
301static void set_debug_reg_defaults(struct thread_struct *thread)
302{
303 thread->iac1 = thread->iac2 = 0;
304#if CONFIG_PPC_ADV_DEBUG_IACS > 2
305 thread->iac3 = thread->iac4 = 0;
306#endif
307 thread->dac1 = thread->dac2 = 0;
308#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
309 thread->dvc1 = thread->dvc2 = 0;
310#endif
311 thread->dbcr0 = 0;
312#ifdef CONFIG_BOOKE
313 /*
314 * Force User/Supervisor bits to b11 (user-only MSR[PR]=1)
315 */
316 thread->dbcr1 = DBCR1_IAC1US | DBCR1_IAC2US | \
317 DBCR1_IAC3US | DBCR1_IAC4US;
318 /*
319 * Force Data Address Compare User/Supervisor bits to be User-only
320 * (0b11 MSR[PR]=1) and set all other bits in DBCR2 register to be 0.
321 */
322 thread->dbcr2 = DBCR2_DAC1US | DBCR2_DAC2US;
323#else
324 thread->dbcr1 = 0;
325#endif
326}
327
328static void prime_debug_regs(struct thread_struct *thread)
329{
330 mtspr(SPRN_IAC1, thread->iac1);
331 mtspr(SPRN_IAC2, thread->iac2);
332#if CONFIG_PPC_ADV_DEBUG_IACS > 2
333 mtspr(SPRN_IAC3, thread->iac3);
334 mtspr(SPRN_IAC4, thread->iac4);
335#endif
336 mtspr(SPRN_DAC1, thread->dac1);
337 mtspr(SPRN_DAC2, thread->dac2);
338#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
339 mtspr(SPRN_DVC1, thread->dvc1);
340 mtspr(SPRN_DVC2, thread->dvc2);
341#endif
342 mtspr(SPRN_DBCR0, thread->dbcr0);
343 mtspr(SPRN_DBCR1, thread->dbcr1);
344#ifdef CONFIG_BOOKE
345 mtspr(SPRN_DBCR2, thread->dbcr2);
346#endif
347}
348/*
349 * Unless neither the old or new thread are making use of the
350 * debug registers, set the debug registers from the values
351 * stored in the new thread.
352 */
353static void switch_booke_debug_regs(struct thread_struct *new_thread)
354{
355 if ((current->thread.dbcr0 & DBCR0_IDM)
356 || (new_thread->dbcr0 & DBCR0_IDM))
357 prime_debug_regs(new_thread);
358}
359#else /* !CONFIG_PPC_ADV_DEBUG_REGS */
360#ifndef CONFIG_HAVE_HW_BREAKPOINT
361static void set_debug_reg_defaults(struct thread_struct *thread)
362{
363 if (thread->dabr) {
364 thread->dabr = 0;
365 set_dabr(0);
366 }
367}
368#endif /* !CONFIG_HAVE_HW_BREAKPOINT */
369#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
370
371int set_dabr(unsigned long dabr)
372{
373 __get_cpu_var(current_dabr) = dabr;
374
375 if (ppc_md.set_dabr)
376 return ppc_md.set_dabr(dabr);
377
378 /* XXX should we have a CPU_FTR_HAS_DABR ? */
379#ifdef CONFIG_PPC_ADV_DEBUG_REGS
380 mtspr(SPRN_DAC1, dabr);
381#ifdef CONFIG_PPC_47x
382 isync();
383#endif
384#elif defined(CONFIG_PPC_BOOK3S)
385 mtspr(SPRN_DABR, dabr);
386#endif
387
388
389 return 0;
390}
391
392#ifdef CONFIG_PPC64
393DEFINE_PER_CPU(struct cpu_usage, cpu_usage_array);
394#endif
395
396struct task_struct *__switch_to(struct task_struct *prev,
397 struct task_struct *new)
398{
399 struct thread_struct *new_thread, *old_thread;
400 unsigned long flags;
401 struct task_struct *last;
402#ifdef CONFIG_PPC_BOOK3S_64
403 struct ppc64_tlb_batch *batch;
404#endif
405
406#ifdef CONFIG_SMP
407 /* avoid complexity of lazy save/restore of fpu
408 * by just saving it every time we switch out if
409 * this task used the fpu during the last quantum.
410 *
411 * If it tries to use the fpu again, it'll trap and
412 * reload its fp regs. So we don't have to do a restore
413 * every switch, just a save.
414 * -- Cort
415 */
416 if (prev->thread.regs && (prev->thread.regs->msr & MSR_FP))
417 giveup_fpu(prev);
418#ifdef CONFIG_ALTIVEC
419 /*
420 * If the previous thread used altivec in the last quantum
421 * (thus changing altivec regs) then save them.
422 * We used to check the VRSAVE register but not all apps
423 * set it, so we don't rely on it now (and in fact we need
424 * to save & restore VSCR even if VRSAVE == 0). -- paulus
425 *
426 * On SMP we always save/restore altivec regs just to avoid the
427 * complexity of changing processors.
428 * -- Cort
429 */
430 if (prev->thread.regs && (prev->thread.regs->msr & MSR_VEC))
431 giveup_altivec(prev);
432#endif /* CONFIG_ALTIVEC */
433#ifdef CONFIG_VSX
434 if (prev->thread.regs && (prev->thread.regs->msr & MSR_VSX))
435 /* VMX and FPU registers are already save here */
436 __giveup_vsx(prev);
437#endif /* CONFIG_VSX */
438#ifdef CONFIG_SPE
439 /*
440 * If the previous thread used spe in the last quantum
441 * (thus changing spe regs) then save them.
442 *
443 * On SMP we always save/restore spe regs just to avoid the
444 * complexity of changing processors.
445 */
446 if ((prev->thread.regs && (prev->thread.regs->msr & MSR_SPE)))
447 giveup_spe(prev);
448#endif /* CONFIG_SPE */
449
450#else /* CONFIG_SMP */
451#ifdef CONFIG_ALTIVEC
452 /* Avoid the trap. On smp this this never happens since
453 * we don't set last_task_used_altivec -- Cort
454 */
455 if (new->thread.regs && last_task_used_altivec == new)
456 new->thread.regs->msr |= MSR_VEC;
457#endif /* CONFIG_ALTIVEC */
458#ifdef CONFIG_VSX
459 if (new->thread.regs && last_task_used_vsx == new)
460 new->thread.regs->msr |= MSR_VSX;
461#endif /* CONFIG_VSX */
462#ifdef CONFIG_SPE
463 /* Avoid the trap. On smp this this never happens since
464 * we don't set last_task_used_spe
465 */
466 if (new->thread.regs && last_task_used_spe == new)
467 new->thread.regs->msr |= MSR_SPE;
468#endif /* CONFIG_SPE */
469
470#endif /* CONFIG_SMP */
471
472#ifdef CONFIG_PPC_ADV_DEBUG_REGS
473 switch_booke_debug_regs(&new->thread);
474#else
475/*
476 * For PPC_BOOK3S_64, we use the hw-breakpoint interfaces that would
477 * schedule DABR
478 */
479#ifndef CONFIG_HAVE_HW_BREAKPOINT
480 if (unlikely(__get_cpu_var(current_dabr) != new->thread.dabr))
481 set_dabr(new->thread.dabr);
482#endif /* CONFIG_HAVE_HW_BREAKPOINT */
483#endif
484
485
486 new_thread = &new->thread;
487 old_thread = ¤t->thread;
488
489#if defined(CONFIG_PPC_BOOK3E_64)
490 /* XXX Current Book3E code doesn't deal with kernel side DBCR0,
491 * we always hold the user values, so we set it now.
492 *
493 * However, we ensure the kernel MSR:DE is appropriately cleared too
494 * to avoid spurrious single step exceptions in the kernel.
495 *
496 * This will have to change to merge with the ppc32 code at some point,
497 * but I don't like much what ppc32 is doing today so there's some
498 * thinking needed there
499 */
500 if ((new_thread->dbcr0 | old_thread->dbcr0) & DBCR0_IDM) {
501 u32 dbcr0;
502
503 mtmsr(mfmsr() & ~MSR_DE);
504 isync();
505 dbcr0 = mfspr(SPRN_DBCR0);
506 dbcr0 = (dbcr0 & DBCR0_EDM) | new_thread->dbcr0;
507 mtspr(SPRN_DBCR0, dbcr0);
508 }
509#endif /* CONFIG_PPC64_BOOK3E */
510
511#ifdef CONFIG_PPC64
512 /*
513 * Collect processor utilization data per process
514 */
515 if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
516 struct cpu_usage *cu = &__get_cpu_var(cpu_usage_array);
517 long unsigned start_tb, current_tb;
518 start_tb = old_thread->start_tb;
519 cu->current_tb = current_tb = mfspr(SPRN_PURR);
520 old_thread->accum_tb += (current_tb - start_tb);
521 new_thread->start_tb = current_tb;
522 }
523#endif /* CONFIG_PPC64 */
524
525#ifdef CONFIG_PPC_BOOK3S_64
526 batch = &__get_cpu_var(ppc64_tlb_batch);
527 if (batch->active) {
528 current_thread_info()->local_flags |= _TLF_LAZY_MMU;
529 if (batch->index)
530 __flush_tlb_pending(batch);
531 batch->active = 0;
532 }
533#endif /* CONFIG_PPC_BOOK3S_64 */
534
535 local_irq_save(flags);
536
537 account_system_vtime(current);
538 account_process_vtime(current);
539
540 /*
541 * We can't take a PMU exception inside _switch() since there is a
542 * window where the kernel stack SLB and the kernel stack are out
543 * of sync. Hard disable here.
544 */
545 hard_irq_disable();
546 last = _switch(old_thread, new_thread);
547
548#ifdef CONFIG_PPC_BOOK3S_64
549 if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
550 current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
551 batch = &__get_cpu_var(ppc64_tlb_batch);
552 batch->active = 1;
553 }
554#endif /* CONFIG_PPC_BOOK3S_64 */
555
556 local_irq_restore(flags);
557
558 return last;
559}
560
561static int instructions_to_print = 16;
562
563static void show_instructions(struct pt_regs *regs)
564{
565 int i;
566 unsigned long pc = regs->nip - (instructions_to_print * 3 / 4 *
567 sizeof(int));
568
569 printk("Instruction dump:");
570
571 for (i = 0; i < instructions_to_print; i++) {
572 int instr;
573
574 if (!(i % 8))
575 printk("\n");
576
577#if !defined(CONFIG_BOOKE)
578 /* If executing with the IMMU off, adjust pc rather
579 * than print XXXXXXXX.
580 */
581 if (!(regs->msr & MSR_IR))
582 pc = (unsigned long)phys_to_virt(pc);
583#endif
584
585 /* We use __get_user here *only* to avoid an OOPS on a
586 * bad address because the pc *should* only be a
587 * kernel address.
588 */
589 if (!__kernel_text_address(pc) ||
590 __get_user(instr, (unsigned int __user *)pc)) {
591 printk("XXXXXXXX ");
592 } else {
593 if (regs->nip == pc)
594 printk("<%08x> ", instr);
595 else
596 printk("%08x ", instr);
597 }
598
599 pc += sizeof(int);
600 }
601
602 printk("\n");
603}
604
605static struct regbit {
606 unsigned long bit;
607 const char *name;
608} msr_bits[] = {
609 {MSR_EE, "EE"},
610 {MSR_PR, "PR"},
611 {MSR_FP, "FP"},
612 {MSR_VEC, "VEC"},
613 {MSR_VSX, "VSX"},
614 {MSR_ME, "ME"},
615 {MSR_CE, "CE"},
616 {MSR_DE, "DE"},
617 {MSR_IR, "IR"},
618 {MSR_DR, "DR"},
619 {0, NULL}
620};
621
622static void printbits(unsigned long val, struct regbit *bits)
623{
624 const char *sep = "";
625
626 printk("<");
627 for (; bits->bit; ++bits)
628 if (val & bits->bit) {
629 printk("%s%s", sep, bits->name);
630 sep = ",";
631 }
632 printk(">");
633}
634
635#ifdef CONFIG_PPC64
636#define REG "%016lx"
637#define REGS_PER_LINE 4
638#define LAST_VOLATILE 13
639#else
640#define REG "%08lx"
641#define REGS_PER_LINE 8
642#define LAST_VOLATILE 12
643#endif
644
645void show_regs(struct pt_regs * regs)
646{
647 int i, trap;
648
649 printk("NIP: "REG" LR: "REG" CTR: "REG"\n",
650 regs->nip, regs->link, regs->ctr);
651 printk("REGS: %p TRAP: %04lx %s (%s)\n",
652 regs, regs->trap, print_tainted(), init_utsname()->release);
653 printk("MSR: "REG" ", regs->msr);
654 printbits(regs->msr, msr_bits);
655 printk(" CR: %08lx XER: %08lx\n", regs->ccr, regs->xer);
656 trap = TRAP(regs);
657 if ((regs->trap != 0xc00) && cpu_has_feature(CPU_FTR_CFAR))
658 printk("CFAR: "REG"\n", regs->orig_gpr3);
659 if (trap == 0x300 || trap == 0x600)
660#ifdef CONFIG_PPC_ADV_DEBUG_REGS
661 printk("DEAR: "REG", ESR: "REG"\n", regs->dar, regs->dsisr);
662#else
663 printk("DAR: "REG", DSISR: %08lx\n", regs->dar, regs->dsisr);
664#endif
665 printk("TASK = %p[%d] '%s' THREAD: %p",
666 current, task_pid_nr(current), current->comm, task_thread_info(current));
667
668#ifdef CONFIG_SMP
669 printk(" CPU: %d", raw_smp_processor_id());
670#endif /* CONFIG_SMP */
671
672 for (i = 0; i < 32; i++) {
673 if ((i % REGS_PER_LINE) == 0)
674 printk("\nGPR%02d: ", i);
675 printk(REG " ", regs->gpr[i]);
676 if (i == LAST_VOLATILE && !FULL_REGS(regs))
677 break;
678 }
679 printk("\n");
680#ifdef CONFIG_KALLSYMS
681 /*
682 * Lookup NIP late so we have the best change of getting the
683 * above info out without failing
684 */
685 printk("NIP ["REG"] %pS\n", regs->nip, (void *)regs->nip);
686 printk("LR ["REG"] %pS\n", regs->link, (void *)regs->link);
687#endif
688 show_stack(current, (unsigned long *) regs->gpr[1]);
689 if (!user_mode(regs))
690 show_instructions(regs);
691}
692
693void exit_thread(void)
694{
695 discard_lazy_cpu_state();
696}
697
698void flush_thread(void)
699{
700 discard_lazy_cpu_state();
701
702#ifdef CONFIG_HAVE_HW_BREAKPOINT
703 flush_ptrace_hw_breakpoint(current);
704#else /* CONFIG_HAVE_HW_BREAKPOINT */
705 set_debug_reg_defaults(¤t->thread);
706#endif /* CONFIG_HAVE_HW_BREAKPOINT */
707}
708
709void
710release_thread(struct task_struct *t)
711{
712}
713
714/*
715 * This gets called before we allocate a new thread and copy
716 * the current task into it.
717 */
718void prepare_to_copy(struct task_struct *tsk)
719{
720 flush_fp_to_thread(current);
721 flush_altivec_to_thread(current);
722 flush_vsx_to_thread(current);
723 flush_spe_to_thread(current);
724#ifdef CONFIG_HAVE_HW_BREAKPOINT
725 flush_ptrace_hw_breakpoint(tsk);
726#endif /* CONFIG_HAVE_HW_BREAKPOINT */
727}
728
729/*
730 * Copy a thread..
731 */
732extern unsigned long dscr_default; /* defined in arch/powerpc/kernel/sysfs.c */
733
734int copy_thread(unsigned long clone_flags, unsigned long usp,
735 unsigned long unused, struct task_struct *p,
736 struct pt_regs *regs)
737{
738 struct pt_regs *childregs, *kregs;
739 extern void ret_from_fork(void);
740 unsigned long sp = (unsigned long)task_stack_page(p) + THREAD_SIZE;
741
742 CHECK_FULL_REGS(regs);
743 /* Copy registers */
744 sp -= sizeof(struct pt_regs);
745 childregs = (struct pt_regs *) sp;
746 *childregs = *regs;
747 if ((childregs->msr & MSR_PR) == 0) {
748 /* for kernel thread, set `current' and stackptr in new task */
749 childregs->gpr[1] = sp + sizeof(struct pt_regs);
750#ifdef CONFIG_PPC32
751 childregs->gpr[2] = (unsigned long) p;
752#else
753 clear_tsk_thread_flag(p, TIF_32BIT);
754#endif
755 p->thread.regs = NULL; /* no user register state */
756 } else {
757 childregs->gpr[1] = usp;
758 p->thread.regs = childregs;
759 if (clone_flags & CLONE_SETTLS) {
760#ifdef CONFIG_PPC64
761 if (!is_32bit_task())
762 childregs->gpr[13] = childregs->gpr[6];
763 else
764#endif
765 childregs->gpr[2] = childregs->gpr[6];
766 }
767 }
768 childregs->gpr[3] = 0; /* Result from fork() */
769 sp -= STACK_FRAME_OVERHEAD;
770
771 /*
772 * The way this works is that at some point in the future
773 * some task will call _switch to switch to the new task.
774 * That will pop off the stack frame created below and start
775 * the new task running at ret_from_fork. The new task will
776 * do some house keeping and then return from the fork or clone
777 * system call, using the stack frame created above.
778 */
779 sp -= sizeof(struct pt_regs);
780 kregs = (struct pt_regs *) sp;
781 sp -= STACK_FRAME_OVERHEAD;
782 p->thread.ksp = sp;
783 p->thread.ksp_limit = (unsigned long)task_stack_page(p) +
784 _ALIGN_UP(sizeof(struct thread_info), 16);
785
786#ifdef CONFIG_PPC_STD_MMU_64
787 if (mmu_has_feature(MMU_FTR_SLB)) {
788 unsigned long sp_vsid;
789 unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp;
790
791 if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
792 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_1T)
793 << SLB_VSID_SHIFT_1T;
794 else
795 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_256M)
796 << SLB_VSID_SHIFT;
797 sp_vsid |= SLB_VSID_KERNEL | llp;
798 p->thread.ksp_vsid = sp_vsid;
799 }
800#endif /* CONFIG_PPC_STD_MMU_64 */
801#ifdef CONFIG_PPC64
802 if (cpu_has_feature(CPU_FTR_DSCR)) {
803 if (current->thread.dscr_inherit) {
804 p->thread.dscr_inherit = 1;
805 p->thread.dscr = current->thread.dscr;
806 } else if (0 != dscr_default) {
807 p->thread.dscr_inherit = 1;
808 p->thread.dscr = dscr_default;
809 } else {
810 p->thread.dscr_inherit = 0;
811 p->thread.dscr = 0;
812 }
813 }
814#endif
815
816 /*
817 * The PPC64 ABI makes use of a TOC to contain function
818 * pointers. The function (ret_from_except) is actually a pointer
819 * to the TOC entry. The first entry is a pointer to the actual
820 * function.
821 */
822#ifdef CONFIG_PPC64
823 kregs->nip = *((unsigned long *)ret_from_fork);
824#else
825 kregs->nip = (unsigned long)ret_from_fork;
826#endif
827
828 return 0;
829}
830
831/*
832 * Set up a thread for executing a new program
833 */
834void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
835{
836#ifdef CONFIG_PPC64
837 unsigned long load_addr = regs->gpr[2]; /* saved by ELF_PLAT_INIT */
838#endif
839
840 /*
841 * If we exec out of a kernel thread then thread.regs will not be
842 * set. Do it now.
843 */
844 if (!current->thread.regs) {
845 struct pt_regs *regs = task_stack_page(current) + THREAD_SIZE;
846 current->thread.regs = regs - 1;
847 }
848
849 memset(regs->gpr, 0, sizeof(regs->gpr));
850 regs->ctr = 0;
851 regs->link = 0;
852 regs->xer = 0;
853 regs->ccr = 0;
854 regs->gpr[1] = sp;
855
856 /*
857 * We have just cleared all the nonvolatile GPRs, so make
858 * FULL_REGS(regs) return true. This is necessary to allow
859 * ptrace to examine the thread immediately after exec.
860 */
861 regs->trap &= ~1UL;
862
863#ifdef CONFIG_PPC32
864 regs->mq = 0;
865 regs->nip = start;
866 regs->msr = MSR_USER;
867#else
868 if (!is_32bit_task()) {
869 unsigned long entry, toc;
870
871 /* start is a relocated pointer to the function descriptor for
872 * the elf _start routine. The first entry in the function
873 * descriptor is the entry address of _start and the second
874 * entry is the TOC value we need to use.
875 */
876 __get_user(entry, (unsigned long __user *)start);
877 __get_user(toc, (unsigned long __user *)start+1);
878
879 /* Check whether the e_entry function descriptor entries
880 * need to be relocated before we can use them.
881 */
882 if (load_addr != 0) {
883 entry += load_addr;
884 toc += load_addr;
885 }
886 regs->nip = entry;
887 regs->gpr[2] = toc;
888 regs->msr = MSR_USER64;
889 } else {
890 regs->nip = start;
891 regs->gpr[2] = 0;
892 regs->msr = MSR_USER32;
893 }
894#endif
895
896 discard_lazy_cpu_state();
897#ifdef CONFIG_VSX
898 current->thread.used_vsr = 0;
899#endif
900 memset(current->thread.fpr, 0, sizeof(current->thread.fpr));
901 current->thread.fpscr.val = 0;
902#ifdef CONFIG_ALTIVEC
903 memset(current->thread.vr, 0, sizeof(current->thread.vr));
904 memset(¤t->thread.vscr, 0, sizeof(current->thread.vscr));
905 current->thread.vscr.u[3] = 0x00010000; /* Java mode disabled */
906 current->thread.vrsave = 0;
907 current->thread.used_vr = 0;
908#endif /* CONFIG_ALTIVEC */
909#ifdef CONFIG_SPE
910 memset(current->thread.evr, 0, sizeof(current->thread.evr));
911 current->thread.acc = 0;
912 current->thread.spefscr = 0;
913 current->thread.used_spe = 0;
914#endif /* CONFIG_SPE */
915}
916
917#define PR_FP_ALL_EXCEPT (PR_FP_EXC_DIV | PR_FP_EXC_OVF | PR_FP_EXC_UND \
918 | PR_FP_EXC_RES | PR_FP_EXC_INV)
919
920int set_fpexc_mode(struct task_struct *tsk, unsigned int val)
921{
922 struct pt_regs *regs = tsk->thread.regs;
923
924 /* This is a bit hairy. If we are an SPE enabled processor
925 * (have embedded fp) we store the IEEE exception enable flags in
926 * fpexc_mode. fpexc_mode is also used for setting FP exception
927 * mode (asyn, precise, disabled) for 'Classic' FP. */
928 if (val & PR_FP_EXC_SW_ENABLE) {
929#ifdef CONFIG_SPE
930 if (cpu_has_feature(CPU_FTR_SPE)) {
931 tsk->thread.fpexc_mode = val &
932 (PR_FP_EXC_SW_ENABLE | PR_FP_ALL_EXCEPT);
933 return 0;
934 } else {
935 return -EINVAL;
936 }
937#else
938 return -EINVAL;
939#endif
940 }
941
942 /* on a CONFIG_SPE this does not hurt us. The bits that
943 * __pack_fe01 use do not overlap with bits used for
944 * PR_FP_EXC_SW_ENABLE. Additionally, the MSR[FE0,FE1] bits
945 * on CONFIG_SPE implementations are reserved so writing to
946 * them does not change anything */
947 if (val > PR_FP_EXC_PRECISE)
948 return -EINVAL;
949 tsk->thread.fpexc_mode = __pack_fe01(val);
950 if (regs != NULL && (regs->msr & MSR_FP) != 0)
951 regs->msr = (regs->msr & ~(MSR_FE0|MSR_FE1))
952 | tsk->thread.fpexc_mode;
953 return 0;
954}
955
956int get_fpexc_mode(struct task_struct *tsk, unsigned long adr)
957{
958 unsigned int val;
959
960 if (tsk->thread.fpexc_mode & PR_FP_EXC_SW_ENABLE)
961#ifdef CONFIG_SPE
962 if (cpu_has_feature(CPU_FTR_SPE))
963 val = tsk->thread.fpexc_mode;
964 else
965 return -EINVAL;
966#else
967 return -EINVAL;
968#endif
969 else
970 val = __unpack_fe01(tsk->thread.fpexc_mode);
971 return put_user(val, (unsigned int __user *) adr);
972}
973
974int set_endian(struct task_struct *tsk, unsigned int val)
975{
976 struct pt_regs *regs = tsk->thread.regs;
977
978 if ((val == PR_ENDIAN_LITTLE && !cpu_has_feature(CPU_FTR_REAL_LE)) ||
979 (val == PR_ENDIAN_PPC_LITTLE && !cpu_has_feature(CPU_FTR_PPC_LE)))
980 return -EINVAL;
981
982 if (regs == NULL)
983 return -EINVAL;
984
985 if (val == PR_ENDIAN_BIG)
986 regs->msr &= ~MSR_LE;
987 else if (val == PR_ENDIAN_LITTLE || val == PR_ENDIAN_PPC_LITTLE)
988 regs->msr |= MSR_LE;
989 else
990 return -EINVAL;
991
992 return 0;
993}
994
995int get_endian(struct task_struct *tsk, unsigned long adr)
996{
997 struct pt_regs *regs = tsk->thread.regs;
998 unsigned int val;
999
1000 if (!cpu_has_feature(CPU_FTR_PPC_LE) &&
1001 !cpu_has_feature(CPU_FTR_REAL_LE))
1002 return -EINVAL;
1003
1004 if (regs == NULL)
1005 return -EINVAL;
1006
1007 if (regs->msr & MSR_LE) {
1008 if (cpu_has_feature(CPU_FTR_REAL_LE))
1009 val = PR_ENDIAN_LITTLE;
1010 else
1011 val = PR_ENDIAN_PPC_LITTLE;
1012 } else
1013 val = PR_ENDIAN_BIG;
1014
1015 return put_user(val, (unsigned int __user *)adr);
1016}
1017
1018int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
1019{
1020 tsk->thread.align_ctl = val;
1021 return 0;
1022}
1023
1024int get_unalign_ctl(struct task_struct *tsk, unsigned long adr)
1025{
1026 return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr);
1027}
1028
1029#define TRUNC_PTR(x) ((typeof(x))(((unsigned long)(x)) & 0xffffffff))
1030
1031int sys_clone(unsigned long clone_flags, unsigned long usp,
1032 int __user *parent_tidp, void __user *child_threadptr,
1033 int __user *child_tidp, int p6,
1034 struct pt_regs *regs)
1035{
1036 CHECK_FULL_REGS(regs);
1037 if (usp == 0)
1038 usp = regs->gpr[1]; /* stack pointer for child */
1039#ifdef CONFIG_PPC64
1040 if (is_32bit_task()) {
1041 parent_tidp = TRUNC_PTR(parent_tidp);
1042 child_tidp = TRUNC_PTR(child_tidp);
1043 }
1044#endif
1045 return do_fork(clone_flags, usp, regs, 0, parent_tidp, child_tidp);
1046}
1047
1048int sys_fork(unsigned long p1, unsigned long p2, unsigned long p3,
1049 unsigned long p4, unsigned long p5, unsigned long p6,
1050 struct pt_regs *regs)
1051{
1052 CHECK_FULL_REGS(regs);
1053 return do_fork(SIGCHLD, regs->gpr[1], regs, 0, NULL, NULL);
1054}
1055
1056int sys_vfork(unsigned long p1, unsigned long p2, unsigned long p3,
1057 unsigned long p4, unsigned long p5, unsigned long p6,
1058 struct pt_regs *regs)
1059{
1060 CHECK_FULL_REGS(regs);
1061 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->gpr[1],
1062 regs, 0, NULL, NULL);
1063}
1064
1065int sys_execve(unsigned long a0, unsigned long a1, unsigned long a2,
1066 unsigned long a3, unsigned long a4, unsigned long a5,
1067 struct pt_regs *regs)
1068{
1069 int error;
1070 char *filename;
1071
1072 filename = getname((const char __user *) a0);
1073 error = PTR_ERR(filename);
1074 if (IS_ERR(filename))
1075 goto out;
1076 flush_fp_to_thread(current);
1077 flush_altivec_to_thread(current);
1078 flush_spe_to_thread(current);
1079 error = do_execve(filename,
1080 (const char __user *const __user *) a1,
1081 (const char __user *const __user *) a2, regs);
1082 putname(filename);
1083out:
1084 return error;
1085}
1086
1087static inline int valid_irq_stack(unsigned long sp, struct task_struct *p,
1088 unsigned long nbytes)
1089{
1090 unsigned long stack_page;
1091 unsigned long cpu = task_cpu(p);
1092
1093 /*
1094 * Avoid crashing if the stack has overflowed and corrupted
1095 * task_cpu(p), which is in the thread_info struct.
1096 */
1097 if (cpu < NR_CPUS && cpu_possible(cpu)) {
1098 stack_page = (unsigned long) hardirq_ctx[cpu];
1099 if (sp >= stack_page + sizeof(struct thread_struct)
1100 && sp <= stack_page + THREAD_SIZE - nbytes)
1101 return 1;
1102
1103 stack_page = (unsigned long) softirq_ctx[cpu];
1104 if (sp >= stack_page + sizeof(struct thread_struct)
1105 && sp <= stack_page + THREAD_SIZE - nbytes)
1106 return 1;
1107 }
1108 return 0;
1109}
1110
1111int validate_sp(unsigned long sp, struct task_struct *p,
1112 unsigned long nbytes)
1113{
1114 unsigned long stack_page = (unsigned long)task_stack_page(p);
1115
1116 if (sp >= stack_page + sizeof(struct thread_struct)
1117 && sp <= stack_page + THREAD_SIZE - nbytes)
1118 return 1;
1119
1120 return valid_irq_stack(sp, p, nbytes);
1121}
1122
1123EXPORT_SYMBOL(validate_sp);
1124
1125unsigned long get_wchan(struct task_struct *p)
1126{
1127 unsigned long ip, sp;
1128 int count = 0;
1129
1130 if (!p || p == current || p->state == TASK_RUNNING)
1131 return 0;
1132
1133 sp = p->thread.ksp;
1134 if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD))
1135 return 0;
1136
1137 do {
1138 sp = *(unsigned long *)sp;
1139 if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD))
1140 return 0;
1141 if (count > 0) {
1142 ip = ((unsigned long *)sp)[STACK_FRAME_LR_SAVE];
1143 if (!in_sched_functions(ip))
1144 return ip;
1145 }
1146 } while (count++ < 16);
1147 return 0;
1148}
1149
1150static int kstack_depth_to_print = CONFIG_PRINT_STACK_DEPTH;
1151
1152void show_stack(struct task_struct *tsk, unsigned long *stack)
1153{
1154 unsigned long sp, ip, lr, newsp;
1155 int count = 0;
1156 int firstframe = 1;
1157#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1158 int curr_frame = current->curr_ret_stack;
1159 extern void return_to_handler(void);
1160 unsigned long rth = (unsigned long)return_to_handler;
1161 unsigned long mrth = -1;
1162#ifdef CONFIG_PPC64
1163 extern void mod_return_to_handler(void);
1164 rth = *(unsigned long *)rth;
1165 mrth = (unsigned long)mod_return_to_handler;
1166 mrth = *(unsigned long *)mrth;
1167#endif
1168#endif
1169
1170 sp = (unsigned long) stack;
1171 if (tsk == NULL)
1172 tsk = current;
1173 if (sp == 0) {
1174 if (tsk == current)
1175 asm("mr %0,1" : "=r" (sp));
1176 else
1177 sp = tsk->thread.ksp;
1178 }
1179
1180 lr = 0;
1181 printk("Call Trace:\n");
1182 do {
1183 if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD))
1184 return;
1185
1186 stack = (unsigned long *) sp;
1187 newsp = stack[0];
1188 ip = stack[STACK_FRAME_LR_SAVE];
1189 if (!firstframe || ip != lr) {
1190 printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip);
1191#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1192 if ((ip == rth || ip == mrth) && curr_frame >= 0) {
1193 printk(" (%pS)",
1194 (void *)current->ret_stack[curr_frame].ret);
1195 curr_frame--;
1196 }
1197#endif
1198 if (firstframe)
1199 printk(" (unreliable)");
1200 printk("\n");
1201 }
1202 firstframe = 0;
1203
1204 /*
1205 * See if this is an exception frame.
1206 * We look for the "regshere" marker in the current frame.
1207 */
1208 if (validate_sp(sp, tsk, STACK_INT_FRAME_SIZE)
1209 && stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
1210 struct pt_regs *regs = (struct pt_regs *)
1211 (sp + STACK_FRAME_OVERHEAD);
1212 lr = regs->link;
1213 printk("--- Exception: %lx at %pS\n LR = %pS\n",
1214 regs->trap, (void *)regs->nip, (void *)lr);
1215 firstframe = 1;
1216 }
1217
1218 sp = newsp;
1219 } while (count++ < kstack_depth_to_print);
1220}
1221
1222void dump_stack(void)
1223{
1224 show_stack(current, NULL);
1225}
1226EXPORT_SYMBOL(dump_stack);
1227
1228#ifdef CONFIG_PPC64
1229void ppc64_runlatch_on(void)
1230{
1231 unsigned long ctrl;
1232
1233 if (cpu_has_feature(CPU_FTR_CTRL) && !test_thread_flag(TIF_RUNLATCH)) {
1234 HMT_medium();
1235
1236 ctrl = mfspr(SPRN_CTRLF);
1237 ctrl |= CTRL_RUNLATCH;
1238 mtspr(SPRN_CTRLT, ctrl);
1239
1240 set_thread_flag(TIF_RUNLATCH);
1241 }
1242}
1243
1244void __ppc64_runlatch_off(void)
1245{
1246 unsigned long ctrl;
1247
1248 HMT_medium();
1249
1250 clear_thread_flag(TIF_RUNLATCH);
1251
1252 ctrl = mfspr(SPRN_CTRLF);
1253 ctrl &= ~CTRL_RUNLATCH;
1254 mtspr(SPRN_CTRLT, ctrl);
1255}
1256#endif
1257
1258#if THREAD_SHIFT < PAGE_SHIFT
1259
1260static struct kmem_cache *thread_info_cache;
1261
1262struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node)
1263{
1264 struct thread_info *ti;
1265
1266 ti = kmem_cache_alloc_node(thread_info_cache, GFP_KERNEL, node);
1267 if (unlikely(ti == NULL))
1268 return NULL;
1269#ifdef CONFIG_DEBUG_STACK_USAGE
1270 memset(ti, 0, THREAD_SIZE);
1271#endif
1272 return ti;
1273}
1274
1275void free_thread_info(struct thread_info *ti)
1276{
1277 kmem_cache_free(thread_info_cache, ti);
1278}
1279
1280void thread_info_cache_init(void)
1281{
1282 thread_info_cache = kmem_cache_create("thread_info", THREAD_SIZE,
1283 THREAD_SIZE, 0, NULL);
1284 BUG_ON(thread_info_cache == NULL);
1285}
1286
1287#endif /* THREAD_SHIFT < PAGE_SHIFT */
1288
1289unsigned long arch_align_stack(unsigned long sp)
1290{
1291 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
1292 sp -= get_random_int() & ~PAGE_MASK;
1293 return sp & ~0xf;
1294}
1295
1296static inline unsigned long brk_rnd(void)
1297{
1298 unsigned long rnd = 0;
1299
1300 /* 8MB for 32bit, 1GB for 64bit */
1301 if (is_32bit_task())
1302 rnd = (long)(get_random_int() % (1<<(23-PAGE_SHIFT)));
1303 else
1304 rnd = (long)(get_random_int() % (1<<(30-PAGE_SHIFT)));
1305
1306 return rnd << PAGE_SHIFT;
1307}
1308
1309unsigned long arch_randomize_brk(struct mm_struct *mm)
1310{
1311 unsigned long base = mm->brk;
1312 unsigned long ret;
1313
1314#ifdef CONFIG_PPC_STD_MMU_64
1315 /*
1316 * If we are using 1TB segments and we are allowed to randomise
1317 * the heap, we can put it above 1TB so it is backed by a 1TB
1318 * segment. Otherwise the heap will be in the bottom 1TB
1319 * which always uses 256MB segments and this may result in a
1320 * performance penalty.
1321 */
1322 if (!is_32bit_task() && (mmu_highuser_ssize == MMU_SEGSIZE_1T))
1323 base = max_t(unsigned long, mm->brk, 1UL << SID_SHIFT_1T);
1324#endif
1325
1326 ret = PAGE_ALIGN(base + brk_rnd());
1327
1328 if (ret < mm->brk)
1329 return mm->brk;
1330
1331 return ret;
1332}
1333
1334unsigned long randomize_et_dyn(unsigned long base)
1335{
1336 unsigned long ret = PAGE_ALIGN(base + brk_rnd());
1337
1338 if (ret < base)
1339 return base;
1340
1341 return ret;
1342}