Loading...
Note: File does not exist in v5.4.
1#include <asm/asm-offsets.h>
2#include <asm/bug.h>
3#ifdef CONFIG_PPC_BOOK3S
4#include <asm/exception-64s.h>
5#else
6#include <asm/exception-64e.h>
7#endif
8#include <asm/feature-fixups.h>
9#include <asm/head-64.h>
10#include <asm/hw_irq.h>
11#include <asm/kup.h>
12#include <asm/mmu.h>
13#include <asm/ppc_asm.h>
14#include <asm/ptrace.h>
15
16 .section ".toc","aw"
17SYS_CALL_TABLE:
18 .tc sys_call_table[TC],sys_call_table
19
20#ifdef CONFIG_COMPAT
21COMPAT_SYS_CALL_TABLE:
22 .tc compat_sys_call_table[TC],compat_sys_call_table
23#endif
24 .previous
25
26 .align 7
27
28.macro DEBUG_SRR_VALID srr
29#ifdef CONFIG_PPC_RFI_SRR_DEBUG
30 .ifc \srr,srr
31 mfspr r11,SPRN_SRR0
32 ld r12,_NIP(r1)
33100: tdne r11,r12
34 EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
35 mfspr r11,SPRN_SRR1
36 ld r12,_MSR(r1)
37100: tdne r11,r12
38 EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
39 .else
40 mfspr r11,SPRN_HSRR0
41 ld r12,_NIP(r1)
42100: tdne r11,r12
43 EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
44 mfspr r11,SPRN_HSRR1
45 ld r12,_MSR(r1)
46100: tdne r11,r12
47 EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
48 .endif
49#endif
50.endm
51
52#ifdef CONFIG_PPC_BOOK3S
53.macro system_call_vectored name trapnr
54 .globl system_call_vectored_\name
55system_call_vectored_\name:
56_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
57 SCV_INTERRUPT_TO_KERNEL
58 mr r10,r1
59 ld r1,PACAKSAVE(r13)
60 std r10,0(r1)
61 std r11,_NIP(r1)
62 std r12,_MSR(r1)
63 std r0,GPR0(r1)
64 std r10,GPR1(r1)
65 std r2,GPR2(r1)
66 ld r2,PACATOC(r13)
67 mfcr r12
68 li r11,0
69 /* Can we avoid saving r3-r8 in common case? */
70 std r3,GPR3(r1)
71 std r4,GPR4(r1)
72 std r5,GPR5(r1)
73 std r6,GPR6(r1)
74 std r7,GPR7(r1)
75 std r8,GPR8(r1)
76 /* Zero r9-r12, this should only be required when restoring all GPRs */
77 std r11,GPR9(r1)
78 std r11,GPR10(r1)
79 std r11,GPR11(r1)
80 std r11,GPR12(r1)
81 std r9,GPR13(r1)
82 SAVE_NVGPRS(r1)
83 std r11,_XER(r1)
84 std r11,_LINK(r1)
85 std r11,_CTR(r1)
86
87 li r11,\trapnr
88 std r11,_TRAP(r1)
89 std r12,_CCR(r1)
90 addi r10,r1,STACK_FRAME_OVERHEAD
91 ld r11,exception_marker@toc(r2)
92 std r11,-16(r10) /* "regshere" marker */
93
94BEGIN_FTR_SECTION
95 HMT_MEDIUM
96END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
97
98 /*
99 * scv enters with MSR[EE]=1 and is immediately considered soft-masked.
100 * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED,
101 * and interrupts may be masked and pending already.
102 * system_call_exception() will call trace_hardirqs_off() which means
103 * interrupts could already have been blocked before trace_hardirqs_off,
104 * but this is the best we can do.
105 */
106
107 /* Calling convention has r9 = orig r0, r10 = regs */
108 mr r9,r0
109 bl system_call_exception
110
111.Lsyscall_vectored_\name\()_exit:
112 addi r4,r1,STACK_FRAME_OVERHEAD
113 li r5,1 /* scv */
114 bl syscall_exit_prepare
115 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
116.Lsyscall_vectored_\name\()_rst_start:
117 lbz r11,PACAIRQHAPPENED(r13)
118 andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
119 bne- syscall_vectored_\name\()_restart
120 li r11,IRQS_ENABLED
121 stb r11,PACAIRQSOFTMASK(r13)
122 li r11,0
123 stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
124
125 ld r2,_CCR(r1)
126 ld r4,_NIP(r1)
127 ld r5,_MSR(r1)
128
129BEGIN_FTR_SECTION
130 stdcx. r0,0,r1 /* to clear the reservation */
131END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
132
133BEGIN_FTR_SECTION
134 HMT_MEDIUM_LOW
135END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
136
137 cmpdi r3,0
138 bne .Lsyscall_vectored_\name\()_restore_regs
139
140 /* rfscv returns with LR->NIA and CTR->MSR */
141 mtlr r4
142 mtctr r5
143
144 /* Could zero these as per ABI, but we may consider a stricter ABI
145 * which preserves these if libc implementations can benefit, so
146 * restore them for now until further measurement is done. */
147 ld r0,GPR0(r1)
148 ld r4,GPR4(r1)
149 ld r5,GPR5(r1)
150 ld r6,GPR6(r1)
151 ld r7,GPR7(r1)
152 ld r8,GPR8(r1)
153 /* Zero volatile regs that may contain sensitive kernel data */
154 li r9,0
155 li r10,0
156 li r11,0
157 li r12,0
158 mtspr SPRN_XER,r0
159
160 /*
161 * We don't need to restore AMR on the way back to userspace for KUAP.
162 * The value of AMR only matters while we're in the kernel.
163 */
164 mtcr r2
165 ld r2,GPR2(r1)
166 ld r3,GPR3(r1)
167 ld r13,GPR13(r1)
168 ld r1,GPR1(r1)
169 RFSCV_TO_USER
170 b . /* prevent speculative execution */
171
172.Lsyscall_vectored_\name\()_restore_regs:
173 mtspr SPRN_SRR0,r4
174 mtspr SPRN_SRR1,r5
175
176 ld r3,_CTR(r1)
177 ld r4,_LINK(r1)
178 ld r5,_XER(r1)
179
180 REST_NVGPRS(r1)
181 ld r0,GPR0(r1)
182 mtcr r2
183 mtctr r3
184 mtlr r4
185 mtspr SPRN_XER,r5
186 REST_10GPRS(2, r1)
187 REST_2GPRS(12, r1)
188 ld r1,GPR1(r1)
189 RFI_TO_USER
190.Lsyscall_vectored_\name\()_rst_end:
191
192syscall_vectored_\name\()_restart:
193_ASM_NOKPROBE_SYMBOL(syscall_vectored_\name\()_restart)
194 GET_PACA(r13)
195 ld r1,PACA_EXIT_SAVE_R1(r13)
196 ld r2,PACATOC(r13)
197 ld r3,RESULT(r1)
198 addi r4,r1,STACK_FRAME_OVERHEAD
199 li r11,IRQS_ALL_DISABLED
200 stb r11,PACAIRQSOFTMASK(r13)
201 bl syscall_exit_restart
202 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
203 b .Lsyscall_vectored_\name\()_rst_start
2041:
205
206SOFT_MASK_TABLE(.Lsyscall_vectored_\name\()_rst_start, 1b)
207RESTART_TABLE(.Lsyscall_vectored_\name\()_rst_start, .Lsyscall_vectored_\name\()_rst_end, syscall_vectored_\name\()_restart)
208
209.endm
210
211system_call_vectored common 0x3000
212
213/*
214 * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
215 * which is tested by system_call_exception when r0 is -1 (as set by vector
216 * entry code).
217 */
218system_call_vectored sigill 0x7ff0
219
220
221/*
222 * Entered via kernel return set up by kernel/sstep.c, must match entry regs
223 */
224 .globl system_call_vectored_emulate
225system_call_vectored_emulate:
226_ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
227 li r10,IRQS_ALL_DISABLED
228 stb r10,PACAIRQSOFTMASK(r13)
229 b system_call_vectored_common
230#endif /* CONFIG_PPC_BOOK3S */
231
232 .balign IFETCH_ALIGN_BYTES
233 .globl system_call_common_real
234system_call_common_real:
235_ASM_NOKPROBE_SYMBOL(system_call_common_real)
236 ld r10,PACAKMSR(r13) /* get MSR value for kernel */
237 mtmsrd r10
238
239 .balign IFETCH_ALIGN_BYTES
240 .globl system_call_common
241system_call_common:
242_ASM_NOKPROBE_SYMBOL(system_call_common)
243 mr r10,r1
244 ld r1,PACAKSAVE(r13)
245 std r10,0(r1)
246 std r11,_NIP(r1)
247 std r12,_MSR(r1)
248 std r0,GPR0(r1)
249 std r10,GPR1(r1)
250 std r2,GPR2(r1)
251#ifdef CONFIG_PPC_FSL_BOOK3E
252START_BTB_FLUSH_SECTION
253 BTB_FLUSH(r10)
254END_BTB_FLUSH_SECTION
255#endif
256 ld r2,PACATOC(r13)
257 mfcr r12
258 li r11,0
259 /* Can we avoid saving r3-r8 in common case? */
260 std r3,GPR3(r1)
261 std r4,GPR4(r1)
262 std r5,GPR5(r1)
263 std r6,GPR6(r1)
264 std r7,GPR7(r1)
265 std r8,GPR8(r1)
266 /* Zero r9-r12, this should only be required when restoring all GPRs */
267 std r11,GPR9(r1)
268 std r11,GPR10(r1)
269 std r11,GPR11(r1)
270 std r11,GPR12(r1)
271 std r9,GPR13(r1)
272 SAVE_NVGPRS(r1)
273 std r11,_XER(r1)
274 std r11,_CTR(r1)
275 mflr r10
276
277 /*
278 * This clears CR0.SO (bit 28), which is the error indication on
279 * return from this system call.
280 */
281 rldimi r12,r11,28,(63-28)
282 li r11,0xc00
283 std r10,_LINK(r1)
284 std r11,_TRAP(r1)
285 std r12,_CCR(r1)
286 addi r10,r1,STACK_FRAME_OVERHEAD
287 ld r11,exception_marker@toc(r2)
288 std r11,-16(r10) /* "regshere" marker */
289
290#ifdef CONFIG_PPC_BOOK3S
291 li r11,1
292 stb r11,PACASRR_VALID(r13)
293#endif
294
295 /*
296 * We always enter kernel from userspace with irq soft-mask enabled and
297 * nothing pending. system_call_exception() will call
298 * trace_hardirqs_off().
299 */
300 li r11,IRQS_ALL_DISABLED
301 stb r11,PACAIRQSOFTMASK(r13)
302#ifdef CONFIG_PPC_BOOK3S
303 li r12,-1 /* Set MSR_EE and MSR_RI */
304 mtmsrd r12,1
305#else
306 wrteei 1
307#endif
308
309 /* Calling convention has r9 = orig r0, r10 = regs */
310 mr r9,r0
311 bl system_call_exception
312
313.Lsyscall_exit:
314 addi r4,r1,STACK_FRAME_OVERHEAD
315 li r5,0 /* !scv */
316 bl syscall_exit_prepare
317 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
318#ifdef CONFIG_PPC_BOOK3S
319.Lsyscall_rst_start:
320 lbz r11,PACAIRQHAPPENED(r13)
321 andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
322 bne- syscall_restart
323#endif
324 li r11,IRQS_ENABLED
325 stb r11,PACAIRQSOFTMASK(r13)
326 li r11,0
327 stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
328
329 ld r2,_CCR(r1)
330 ld r6,_LINK(r1)
331 mtlr r6
332
333#ifdef CONFIG_PPC_BOOK3S
334 lbz r4,PACASRR_VALID(r13)
335 cmpdi r4,0
336 bne 1f
337 li r4,0
338 stb r4,PACASRR_VALID(r13)
339#endif
340 ld r4,_NIP(r1)
341 ld r5,_MSR(r1)
342 mtspr SPRN_SRR0,r4
343 mtspr SPRN_SRR1,r5
3441:
345 DEBUG_SRR_VALID srr
346
347BEGIN_FTR_SECTION
348 stdcx. r0,0,r1 /* to clear the reservation */
349END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
350
351 cmpdi r3,0
352 bne .Lsyscall_restore_regs
353 /* Zero volatile regs that may contain sensitive kernel data */
354 li r0,0
355 li r4,0
356 li r5,0
357 li r6,0
358 li r7,0
359 li r8,0
360 li r9,0
361 li r10,0
362 li r11,0
363 li r12,0
364 mtctr r0
365 mtspr SPRN_XER,r0
366.Lsyscall_restore_regs_cont:
367
368BEGIN_FTR_SECTION
369 HMT_MEDIUM_LOW
370END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
371
372 /*
373 * We don't need to restore AMR on the way back to userspace for KUAP.
374 * The value of AMR only matters while we're in the kernel.
375 */
376 mtcr r2
377 ld r2,GPR2(r1)
378 ld r3,GPR3(r1)
379 ld r13,GPR13(r1)
380 ld r1,GPR1(r1)
381 RFI_TO_USER
382 b . /* prevent speculative execution */
383
384.Lsyscall_restore_regs:
385 ld r3,_CTR(r1)
386 ld r4,_XER(r1)
387 REST_NVGPRS(r1)
388 mtctr r3
389 mtspr SPRN_XER,r4
390 ld r0,GPR0(r1)
391 REST_8GPRS(4, r1)
392 ld r12,GPR12(r1)
393 b .Lsyscall_restore_regs_cont
394.Lsyscall_rst_end:
395
396#ifdef CONFIG_PPC_BOOK3S
397syscall_restart:
398_ASM_NOKPROBE_SYMBOL(syscall_restart)
399 GET_PACA(r13)
400 ld r1,PACA_EXIT_SAVE_R1(r13)
401 ld r2,PACATOC(r13)
402 ld r3,RESULT(r1)
403 addi r4,r1,STACK_FRAME_OVERHEAD
404 li r11,IRQS_ALL_DISABLED
405 stb r11,PACAIRQSOFTMASK(r13)
406 bl syscall_exit_restart
407 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
408 b .Lsyscall_rst_start
4091:
410
411SOFT_MASK_TABLE(.Lsyscall_rst_start, 1b)
412RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart)
413#endif
414
415 /*
416 * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
417 * touched, no exit work created, then this can be used.
418 */
419 .balign IFETCH_ALIGN_BYTES
420 .globl fast_interrupt_return_srr
421fast_interrupt_return_srr:
422_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)
423 kuap_check_amr r3, r4
424 ld r5,_MSR(r1)
425 andi. r0,r5,MSR_PR
426#ifdef CONFIG_PPC_BOOK3S
427 beq 1f
428 kuap_user_restore r3, r4
429 b .Lfast_user_interrupt_return_srr
4301: kuap_kernel_restore r3, r4
431 andi. r0,r5,MSR_RI
432 li r3,0 /* 0 return value, no EMULATE_STACK_STORE */
433 bne+ .Lfast_kernel_interrupt_return_srr
434 addi r3,r1,STACK_FRAME_OVERHEAD
435 bl unrecoverable_exception
436 b . /* should not get here */
437#else
438 bne .Lfast_user_interrupt_return_srr
439 b .Lfast_kernel_interrupt_return_srr
440#endif
441
442.macro interrupt_return_macro srr
443 .balign IFETCH_ALIGN_BYTES
444 .globl interrupt_return_\srr
445interrupt_return_\srr\():
446_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
447 ld r4,_MSR(r1)
448 andi. r0,r4,MSR_PR
449 beq interrupt_return_\srr\()_kernel
450interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */
451_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
452 addi r3,r1,STACK_FRAME_OVERHEAD
453 bl interrupt_exit_user_prepare
454 cmpdi r3,0
455 bne- .Lrestore_nvgprs_\srr
456.Lrestore_nvgprs_\srr\()_cont:
457 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
458#ifdef CONFIG_PPC_BOOK3S
459.Linterrupt_return_\srr\()_user_rst_start:
460 lbz r11,PACAIRQHAPPENED(r13)
461 andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
462 bne- interrupt_return_\srr\()_user_restart
463#endif
464 li r11,IRQS_ENABLED
465 stb r11,PACAIRQSOFTMASK(r13)
466 li r11,0
467 stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
468
469.Lfast_user_interrupt_return_\srr\():
470#ifdef CONFIG_PPC_BOOK3S
471 .ifc \srr,srr
472 lbz r4,PACASRR_VALID(r13)
473 .else
474 lbz r4,PACAHSRR_VALID(r13)
475 .endif
476 cmpdi r4,0
477 li r4,0
478 bne 1f
479#endif
480 ld r11,_NIP(r1)
481 ld r12,_MSR(r1)
482 .ifc \srr,srr
483 mtspr SPRN_SRR0,r11
484 mtspr SPRN_SRR1,r12
4851:
486#ifdef CONFIG_PPC_BOOK3S
487 stb r4,PACASRR_VALID(r13)
488#endif
489 .else
490 mtspr SPRN_HSRR0,r11
491 mtspr SPRN_HSRR1,r12
4921:
493#ifdef CONFIG_PPC_BOOK3S
494 stb r4,PACAHSRR_VALID(r13)
495#endif
496 .endif
497 DEBUG_SRR_VALID \srr
498
499#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
500 lbz r4,PACAIRQSOFTMASK(r13)
501 tdnei r4,IRQS_ENABLED
502#endif
503
504BEGIN_FTR_SECTION
505 ld r10,_PPR(r1)
506 mtspr SPRN_PPR,r10
507END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
508
509BEGIN_FTR_SECTION
510 stdcx. r0,0,r1 /* to clear the reservation */
511FTR_SECTION_ELSE
512 ldarx r0,0,r1
513ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
514
515 ld r3,_CCR(r1)
516 ld r4,_LINK(r1)
517 ld r5,_CTR(r1)
518 ld r6,_XER(r1)
519 li r0,0
520
521 REST_4GPRS(7, r1)
522 REST_2GPRS(11, r1)
523 REST_GPR(13, r1)
524
525 mtcr r3
526 mtlr r4
527 mtctr r5
528 mtspr SPRN_XER,r6
529
530 REST_4GPRS(2, r1)
531 REST_GPR(6, r1)
532 REST_GPR(0, r1)
533 REST_GPR(1, r1)
534 .ifc \srr,srr
535 RFI_TO_USER
536 .else
537 HRFI_TO_USER
538 .endif
539 b . /* prevent speculative execution */
540.Linterrupt_return_\srr\()_user_rst_end:
541
542.Lrestore_nvgprs_\srr\():
543 REST_NVGPRS(r1)
544 b .Lrestore_nvgprs_\srr\()_cont
545
546#ifdef CONFIG_PPC_BOOK3S
547interrupt_return_\srr\()_user_restart:
548_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user_restart)
549 GET_PACA(r13)
550 ld r1,PACA_EXIT_SAVE_R1(r13)
551 ld r2,PACATOC(r13)
552 addi r3,r1,STACK_FRAME_OVERHEAD
553 li r11,IRQS_ALL_DISABLED
554 stb r11,PACAIRQSOFTMASK(r13)
555 bl interrupt_exit_user_restart
556 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
557 b .Linterrupt_return_\srr\()_user_rst_start
5581:
559
560SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_user_rst_start, 1b)
561RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart)
562#endif
563
564 .balign IFETCH_ALIGN_BYTES
565interrupt_return_\srr\()_kernel:
566_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel)
567 addi r3,r1,STACK_FRAME_OVERHEAD
568 bl interrupt_exit_kernel_prepare
569
570 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
571.Linterrupt_return_\srr\()_kernel_rst_start:
572 ld r11,SOFTE(r1)
573 cmpwi r11,IRQS_ENABLED
574 stb r11,PACAIRQSOFTMASK(r13)
575 bne 1f
576#ifdef CONFIG_PPC_BOOK3S
577 lbz r11,PACAIRQHAPPENED(r13)
578 andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
579 bne- interrupt_return_\srr\()_kernel_restart
580#endif
581 li r11,0
582 stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
5831:
584
585.Lfast_kernel_interrupt_return_\srr\():
586 cmpdi cr1,r3,0
587#ifdef CONFIG_PPC_BOOK3S
588 .ifc \srr,srr
589 lbz r4,PACASRR_VALID(r13)
590 .else
591 lbz r4,PACAHSRR_VALID(r13)
592 .endif
593 cmpdi r4,0
594 li r4,0
595 bne 1f
596#endif
597 ld r11,_NIP(r1)
598 ld r12,_MSR(r1)
599 .ifc \srr,srr
600 mtspr SPRN_SRR0,r11
601 mtspr SPRN_SRR1,r12
6021:
603#ifdef CONFIG_PPC_BOOK3S
604 stb r4,PACASRR_VALID(r13)
605#endif
606 .else
607 mtspr SPRN_HSRR0,r11
608 mtspr SPRN_HSRR1,r12
6091:
610#ifdef CONFIG_PPC_BOOK3S
611 stb r4,PACAHSRR_VALID(r13)
612#endif
613 .endif
614 DEBUG_SRR_VALID \srr
615
616BEGIN_FTR_SECTION
617 stdcx. r0,0,r1 /* to clear the reservation */
618FTR_SECTION_ELSE
619 ldarx r0,0,r1
620ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
621
622 ld r3,_LINK(r1)
623 ld r4,_CTR(r1)
624 ld r5,_XER(r1)
625 ld r6,_CCR(r1)
626 li r0,0
627
628 REST_4GPRS(7, r1)
629 REST_2GPRS(11, r1)
630
631 mtlr r3
632 mtctr r4
633 mtspr SPRN_XER,r5
634
635 /*
636 * Leaving a stale exception_marker on the stack can confuse
637 * the reliable stack unwinder later on. Clear it.
638 */
639 std r0,STACK_FRAME_OVERHEAD-16(r1)
640
641 REST_4GPRS(2, r1)
642
643 bne- cr1,1f /* emulate stack store */
644 mtcr r6
645 REST_GPR(6, r1)
646 REST_GPR(0, r1)
647 REST_GPR(1, r1)
648 .ifc \srr,srr
649 RFI_TO_KERNEL
650 .else
651 HRFI_TO_KERNEL
652 .endif
653 b . /* prevent speculative execution */
654
6551: /*
656 * Emulate stack store with update. New r1 value was already calculated
657 * and updated in our interrupt regs by emulate_loadstore, but we can't
658 * store the previous value of r1 to the stack before re-loading our
659 * registers from it, otherwise they could be clobbered. Use
660 * PACA_EXGEN as temporary storage to hold the store data, as
661 * interrupts are disabled here so it won't be clobbered.
662 */
663 mtcr r6
664 std r9,PACA_EXGEN+0(r13)
665 addi r9,r1,INT_FRAME_SIZE /* get original r1 */
666 REST_GPR(6, r1)
667 REST_GPR(0, r1)
668 REST_GPR(1, r1)
669 std r9,0(r1) /* perform store component of stdu */
670 ld r9,PACA_EXGEN+0(r13)
671
672 .ifc \srr,srr
673 RFI_TO_KERNEL
674 .else
675 HRFI_TO_KERNEL
676 .endif
677 b . /* prevent speculative execution */
678.Linterrupt_return_\srr\()_kernel_rst_end:
679
680#ifdef CONFIG_PPC_BOOK3S
681interrupt_return_\srr\()_kernel_restart:
682_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel_restart)
683 GET_PACA(r13)
684 ld r1,PACA_EXIT_SAVE_R1(r13)
685 ld r2,PACATOC(r13)
686 addi r3,r1,STACK_FRAME_OVERHEAD
687 li r11,IRQS_ALL_DISABLED
688 stb r11,PACAIRQSOFTMASK(r13)
689 bl interrupt_exit_kernel_restart
690 std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
691 b .Linterrupt_return_\srr\()_kernel_rst_start
6921:
693
694SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, 1b)
695RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart)
696#endif
697
698.endm
699
700interrupt_return_macro srr
701#ifdef CONFIG_PPC_BOOK3S
702interrupt_return_macro hsrr
703
704 .globl __end_soft_masked
705__end_soft_masked:
706DEFINE_FIXED_SYMBOL(__end_soft_masked)
707#endif /* CONFIG_PPC_BOOK3S */
708
709#ifdef CONFIG_PPC_BOOK3S
710_GLOBAL(ret_from_fork_scv)
711 bl schedule_tail
712 REST_NVGPRS(r1)
713 li r3,0 /* fork() return value */
714 b .Lsyscall_vectored_common_exit
715#endif
716
717_GLOBAL(ret_from_fork)
718 bl schedule_tail
719 REST_NVGPRS(r1)
720 li r3,0 /* fork() return value */
721 b .Lsyscall_exit
722
723_GLOBAL(ret_from_kernel_thread)
724 bl schedule_tail
725 REST_NVGPRS(r1)
726 mtctr r14
727 mr r3,r15
728#ifdef PPC64_ELF_ABI_v2
729 mr r12,r14
730#endif
731 bctrl
732 li r3,0
733 b .Lsyscall_exit