Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * S390 low-level entry points.
4 *
5 * Copyright IBM Corp. 1999, 2012
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 */
10
11#include <linux/init.h>
12#include <linux/linkage.h>
13#include <asm/asm-extable.h>
14#include <asm/alternative-asm.h>
15#include <asm/processor.h>
16#include <asm/cache.h>
17#include <asm/dwarf.h>
18#include <asm/errno.h>
19#include <asm/ptrace.h>
20#include <asm/thread_info.h>
21#include <asm/asm-offsets.h>
22#include <asm/unistd.h>
23#include <asm/page.h>
24#include <asm/sigp.h>
25#include <asm/irq.h>
26#include <asm/vx-insn.h>
27#include <asm/setup.h>
28#include <asm/nmi.h>
29#include <asm/export.h>
30#include <asm/nospec-insn.h>
31
32STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
33STACK_SIZE = 1 << STACK_SHIFT
34STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
35
36_LPP_OFFSET = __LC_LPP
37
38 .macro STBEAR address
39 ALTERNATIVE "nop", ".insn s,0xb2010000,\address", 193
40 .endm
41
42 .macro LBEAR address
43 ALTERNATIVE "nop", ".insn s,0xb2000000,\address", 193
44 .endm
45
46 .macro LPSWEY address,lpswe
47 ALTERNATIVE "b \lpswe; nopr", ".insn siy,0xeb0000000071,\address,0", 193
48 .endm
49
50 .macro MBEAR reg
51 ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK), 193
52 .endm
53
54 .macro CHECK_STACK savearea
55#ifdef CONFIG_CHECK_STACK
56 tml %r15,STACK_SIZE - CONFIG_STACK_GUARD
57 lghi %r14,\savearea
58 jz stack_overflow
59#endif
60 .endm
61
62 .macro CHECK_VMAP_STACK savearea,oklabel
63#ifdef CONFIG_VMAP_STACK
64 lgr %r14,%r15
65 nill %r14,0x10000 - STACK_SIZE
66 oill %r14,STACK_INIT
67 clg %r14,__LC_KERNEL_STACK
68 je \oklabel
69 clg %r14,__LC_ASYNC_STACK
70 je \oklabel
71 clg %r14,__LC_MCCK_STACK
72 je \oklabel
73 clg %r14,__LC_NODAT_STACK
74 je \oklabel
75 clg %r14,__LC_RESTART_STACK
76 je \oklabel
77 lghi %r14,\savearea
78 j stack_overflow
79#else
80 j \oklabel
81#endif
82 .endm
83
84 /*
85 * The TSTMSK macro generates a test-under-mask instruction by
86 * calculating the memory offset for the specified mask value.
87 * Mask value can be any constant. The macro shifts the mask
88 * value to calculate the memory offset for the test-under-mask
89 * instruction.
90 */
91 .macro TSTMSK addr, mask, size=8, bytepos=0
92 .if (\bytepos < \size) && (\mask >> 8)
93 .if (\mask & 0xff)
94 .error "Mask exceeds byte boundary"
95 .endif
96 TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
97 .exitm
98 .endif
99 .ifeq \mask
100 .error "Mask must not be zero"
101 .endif
102 off = \size - \bytepos - 1
103 tm off+\addr, \mask
104 .endm
105
106 .macro BPOFF
107 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", 82
108 .endm
109
110 .macro BPON
111 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", 82
112 .endm
113
114 .macro BPENTER tif_ptr,tif_mask
115 ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \
116 "j .+12; nop; nop", 82
117 .endm
118
119 .macro BPEXIT tif_ptr,tif_mask
120 TSTMSK \tif_ptr,\tif_mask
121 ALTERNATIVE "jz .+8; .insn rrf,0xb2e80000,0,0,12,0", \
122 "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", 82
123 .endm
124
125#if IS_ENABLED(CONFIG_KVM)
126 /*
127 * The OUTSIDE macro jumps to the provided label in case the value
128 * in the provided register is outside of the provided range. The
129 * macro is useful for checking whether a PSW stored in a register
130 * pair points inside or outside of a block of instructions.
131 * @reg: register to check
132 * @start: start of the range
133 * @end: end of the range
134 * @outside_label: jump here if @reg is outside of [@start..@end)
135 */
136 .macro OUTSIDE reg,start,end,outside_label
137 lgr %r14,\reg
138 larl %r13,\start
139 slgr %r14,%r13
140#ifdef CONFIG_AS_IS_LLVM
141 clgfrl %r14,.Lrange_size\@
142#else
143 clgfi %r14,\end - \start
144#endif
145 jhe \outside_label
146#ifdef CONFIG_AS_IS_LLVM
147 .section .rodata, "a"
148 .align 4
149.Lrange_size\@:
150 .long \end - \start
151 .previous
152#endif
153 .endm
154
155 .macro SIEEXIT
156 lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer
157 ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
158 lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce
159 larl %r9,sie_exit # skip forward to sie_exit
160 .endm
161#endif
162
163 GEN_BR_THUNK %r14
164
165 .section .kprobes.text, "ax"
166.Ldummy:
167 /*
168 * This nop exists only in order to avoid that __bpon starts at
169 * the beginning of the kprobes text section. In that case we would
170 * have several symbols at the same address. E.g. objdump would take
171 * an arbitrary symbol name when disassembling this code.
172 * With the added nop in between the __bpon symbol is unique
173 * again.
174 */
175 nop 0
176
177ENTRY(__bpon)
178 .globl __bpon
179 BPON
180 BR_EX %r14
181ENDPROC(__bpon)
182
183/*
184 * Scheduler resume function, called by switch_to
185 * gpr2 = (task_struct *) prev
186 * gpr3 = (task_struct *) next
187 * Returns:
188 * gpr2 = prev
189 */
190ENTRY(__switch_to)
191 stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
192 lghi %r4,__TASK_stack
193 lghi %r1,__TASK_thread
194 llill %r5,STACK_INIT
195 stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev
196 lg %r15,0(%r4,%r3) # start of kernel stack of next
197 agr %r15,%r5 # end of kernel stack of next
198 stg %r3,__LC_CURRENT # store task struct of next
199 stg %r15,__LC_KERNEL_STACK # store end of kernel stack
200 lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next
201 aghi %r3,__TASK_pid
202 mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next
203 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
204 ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
205 BR_EX %r14
206ENDPROC(__switch_to)
207
208#if IS_ENABLED(CONFIG_KVM)
209/*
210 * __sie64a calling convention:
211 * %r2 pointer to sie control block phys
212 * %r3 pointer to sie control block virt
213 * %r4 guest register save area
214 */
215ENTRY(__sie64a)
216 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
217 lg %r12,__LC_CURRENT
218 stg %r2,__SF_SIE_CONTROL_PHYS(%r15) # save sie block physical..
219 stg %r3,__SF_SIE_CONTROL(%r15) # ...and virtual addresses
220 stg %r4,__SF_SIE_SAVEAREA(%r15) # save guest register save area
221 xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
222 mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
223 lmg %r0,%r13,0(%r4) # load guest gprs 0-13
224 lg %r14,__LC_GMAP # get gmap pointer
225 ltgr %r14,%r14
226 jz .Lsie_gmap
227 lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce
228.Lsie_gmap:
229 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
230 oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
231 tm __SIE_PROG20+3(%r14),3 # last exit...
232 jnz .Lsie_skip
233 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
234 jo .Lsie_skip # exit if fp/vx regs changed
235 lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr
236 BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
237.Lsie_entry:
238 sie 0(%r14)
239# Let the next instruction be NOP to avoid triggering a machine check
240# and handling it in a guest as result of the instruction execution.
241 nopr 7
242.Lsie_leave:
243 BPOFF
244 BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
245.Lsie_skip:
246 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
247 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
248 lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce
249.Lsie_done:
250# some program checks are suppressing. C code (e.g. do_protection_exception)
251# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
252# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
253# Other instructions between __sie64a and .Lsie_done should not cause program
254# interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
255.Lrewind_pad6:
256 nopr 7
257.Lrewind_pad4:
258 nopr 7
259.Lrewind_pad2:
260 nopr 7
261 .globl sie_exit
262sie_exit:
263 lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area
264 stmg %r0,%r13,0(%r14) # save guest gprs 0-13
265 xgr %r0,%r0 # clear guest registers to
266 xgr %r1,%r1 # prevent speculative use
267 xgr %r3,%r3
268 xgr %r4,%r4
269 xgr %r5,%r5
270 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
271 lg %r2,__SF_SIE_REASON(%r15) # return exit reason code
272 BR_EX %r14
273.Lsie_fault:
274 lghi %r14,-EFAULT
275 stg %r14,__SF_SIE_REASON(%r15) # set exit reason code
276 j sie_exit
277
278 EX_TABLE(.Lrewind_pad6,.Lsie_fault)
279 EX_TABLE(.Lrewind_pad4,.Lsie_fault)
280 EX_TABLE(.Lrewind_pad2,.Lsie_fault)
281 EX_TABLE(sie_exit,.Lsie_fault)
282ENDPROC(__sie64a)
283EXPORT_SYMBOL(__sie64a)
284EXPORT_SYMBOL(sie_exit)
285#endif
286
287/*
288 * SVC interrupt handler routine. System calls are synchronous events and
289 * are entered with interrupts disabled.
290 */
291
292ENTRY(system_call)
293 stpt __LC_SYS_ENTER_TIMER
294 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
295 BPOFF
296 lghi %r14,0
297.Lsysc_per:
298 STBEAR __LC_LAST_BREAK
299 lctlg %c1,%c1,__LC_KERNEL_ASCE
300 lg %r12,__LC_CURRENT
301 lg %r15,__LC_KERNEL_STACK
302 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
303 stmg %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
304 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
305 # clear user controlled register to prevent speculative use
306 xgr %r0,%r0
307 xgr %r1,%r1
308 xgr %r4,%r4
309 xgr %r5,%r5
310 xgr %r6,%r6
311 xgr %r7,%r7
312 xgr %r8,%r8
313 xgr %r9,%r9
314 xgr %r10,%r10
315 xgr %r11,%r11
316 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
317 mvc __PT_R8(64,%r2),__LC_SAVE_AREA_SYNC
318 MBEAR %r2
319 lgr %r3,%r14
320 brasl %r14,__do_syscall
321 lctlg %c1,%c1,__LC_USER_ASCE
322 mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
323 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
324 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
325 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
326 stpt __LC_EXIT_TIMER
327 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
328ENDPROC(system_call)
329
330#
331# a new process exits the kernel with ret_from_fork
332#
333ENTRY(ret_from_fork)
334 lgr %r3,%r11
335 brasl %r14,__ret_from_fork
336 lctlg %c1,%c1,__LC_USER_ASCE
337 mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
338 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
339 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
340 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
341 stpt __LC_EXIT_TIMER
342 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
343ENDPROC(ret_from_fork)
344
345/*
346 * Program check handler routine
347 */
348
349ENTRY(pgm_check_handler)
350 stpt __LC_SYS_ENTER_TIMER
351 BPOFF
352 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
353 lg %r12,__LC_CURRENT
354 lghi %r10,0
355 lmg %r8,%r9,__LC_PGM_OLD_PSW
356 tmhh %r8,0x0001 # coming from user space?
357 jno .Lpgm_skip_asce
358 lctlg %c1,%c1,__LC_KERNEL_ASCE
359 j 3f # -> fault in user space
360.Lpgm_skip_asce:
361#if IS_ENABLED(CONFIG_KVM)
362 # cleanup critical section for program checks in __sie64a
363 OUTSIDE %r9,.Lsie_gmap,.Lsie_done,1f
364 SIEEXIT
365 lghi %r10,_PIF_GUEST_FAULT
366#endif
3671: tmhh %r8,0x4000 # PER bit set in old PSW ?
368 jnz 2f # -> enabled, can't be a double fault
369 tm __LC_PGM_ILC+3,0x80 # check for per exception
370 jnz .Lpgm_svcper # -> single stepped svc
3712: CHECK_STACK __LC_SAVE_AREA_SYNC
372 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
373 # CHECK_VMAP_STACK branches to stack_overflow or 4f
374 CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f
3753: BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
376 lg %r15,__LC_KERNEL_STACK
3774: la %r11,STACK_FRAME_OVERHEAD(%r15)
378 stg %r10,__PT_FLAGS(%r11)
379 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
380 stmg %r0,%r7,__PT_R0(%r11)
381 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
382 mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK
383 stmg %r8,%r9,__PT_PSW(%r11)
384
385 # clear user controlled registers to prevent speculative use
386 xgr %r0,%r0
387 xgr %r1,%r1
388 xgr %r3,%r3
389 xgr %r4,%r4
390 xgr %r5,%r5
391 xgr %r6,%r6
392 xgr %r7,%r7
393 lgr %r2,%r11
394 brasl %r14,__do_pgm_check
395 tmhh %r8,0x0001 # returning to user space?
396 jno .Lpgm_exit_kernel
397 lctlg %c1,%c1,__LC_USER_ASCE
398 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
399 stpt __LC_EXIT_TIMER
400.Lpgm_exit_kernel:
401 mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
402 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
403 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
404 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
405
406#
407# single stepped system call
408#
409.Lpgm_svcper:
410 mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
411 larl %r14,.Lsysc_per
412 stg %r14,__LC_RETURN_PSW+8
413 lghi %r14,1
414 LBEAR __LC_PGM_LAST_BREAK
415 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per
416ENDPROC(pgm_check_handler)
417
418/*
419 * Interrupt handler macro used for external and IO interrupts.
420 */
421.macro INT_HANDLER name,lc_old_psw,handler
422ENTRY(\name)
423 stckf __LC_INT_CLOCK
424 stpt __LC_SYS_ENTER_TIMER
425 STBEAR __LC_LAST_BREAK
426 BPOFF
427 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
428 lg %r12,__LC_CURRENT
429 lmg %r8,%r9,\lc_old_psw
430 tmhh %r8,0x0001 # interrupting from user ?
431 jnz 1f
432#if IS_ENABLED(CONFIG_KVM)
433 OUTSIDE %r9,.Lsie_gmap,.Lsie_done,0f
434 BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
435 SIEEXIT
436#endif
4370: CHECK_STACK __LC_SAVE_AREA_ASYNC
438 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
439 j 2f
4401: BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
441 lctlg %c1,%c1,__LC_KERNEL_ASCE
442 lg %r15,__LC_KERNEL_STACK
4432: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
444 la %r11,STACK_FRAME_OVERHEAD(%r15)
445 stmg %r0,%r7,__PT_R0(%r11)
446 # clear user controlled registers to prevent speculative use
447 xgr %r0,%r0
448 xgr %r1,%r1
449 xgr %r3,%r3
450 xgr %r4,%r4
451 xgr %r5,%r5
452 xgr %r6,%r6
453 xgr %r7,%r7
454 xgr %r10,%r10
455 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
456 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
457 MBEAR %r11
458 stmg %r8,%r9,__PT_PSW(%r11)
459 lgr %r2,%r11 # pass pointer to pt_regs
460 brasl %r14,\handler
461 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
462 tmhh %r8,0x0001 # returning to user ?
463 jno 2f
464 lctlg %c1,%c1,__LC_USER_ASCE
465 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
466 stpt __LC_EXIT_TIMER
4672: LBEAR __PT_LAST_BREAK(%r11)
468 lmg %r0,%r15,__PT_R0(%r11)
469 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
470ENDPROC(\name)
471.endm
472
473INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
474INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
475
476/*
477 * Load idle PSW.
478 */
479ENTRY(psw_idle)
480 stg %r14,(__SF_GPRS+8*8)(%r15)
481 stg %r3,__SF_EMPTY(%r15)
482 larl %r1,psw_idle_exit
483 stg %r1,__SF_EMPTY+8(%r15)
484 larl %r1,smp_cpu_mtid
485 llgf %r1,0(%r1)
486 ltgr %r1,%r1
487 jz .Lpsw_idle_stcctm
488 .insn rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2)
489.Lpsw_idle_stcctm:
490 oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
491 BPON
492 stckf __CLOCK_IDLE_ENTER(%r2)
493 stpt __TIMER_IDLE_ENTER(%r2)
494 lpswe __SF_EMPTY(%r15)
495.globl psw_idle_exit
496psw_idle_exit:
497 BR_EX %r14
498ENDPROC(psw_idle)
499
500/*
501 * Machine check handler routines
502 */
503ENTRY(mcck_int_handler)
504 stckf __LC_MCCK_CLOCK
505 BPOFF
506 la %r1,4095 # validate r1
507 spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer
508 LBEAR __LC_LAST_BREAK_SAVE_AREA-4095(%r1) # validate bear
509 lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs
510 lg %r12,__LC_CURRENT
511 lmg %r8,%r9,__LC_MCK_OLD_PSW
512 TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
513 jo .Lmcck_panic # yes -> rest of mcck code invalid
514 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID
515 jno .Lmcck_panic # control registers invalid -> panic
516 la %r14,4095
517 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs
518 ptlb
519 lghi %r14,__LC_CPU_TIMER_SAVE_AREA
520 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
521 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
522 jo 3f
523 la %r14,__LC_SYS_ENTER_TIMER
524 clc 0(8,%r14),__LC_EXIT_TIMER
525 jl 1f
526 la %r14,__LC_EXIT_TIMER
5271: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
528 jl 2f
529 la %r14,__LC_LAST_UPDATE_TIMER
5302: spt 0(%r14)
531 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
5323: TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
533 jno .Lmcck_panic
534 tmhh %r8,0x0001 # interrupting from user ?
535 jnz .Lmcck_user
536 TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
537 jno .Lmcck_panic
538#if IS_ENABLED(CONFIG_KVM)
539 OUTSIDE %r9,.Lsie_gmap,.Lsie_done,.Lmcck_stack
540 OUTSIDE %r9,.Lsie_entry,.Lsie_leave,4f
541 oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
5424: BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
543 SIEEXIT
544 j .Lmcck_stack
545#endif
546.Lmcck_user:
547 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
548.Lmcck_stack:
549 lg %r15,__LC_MCCK_STACK
550 la %r11,STACK_FRAME_OVERHEAD(%r15)
551 stctg %c1,%c1,__PT_CR1(%r11)
552 lctlg %c1,%c1,__LC_KERNEL_ASCE
553 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
554 lghi %r14,__LC_GPREGS_SAVE_AREA+64
555 stmg %r0,%r7,__PT_R0(%r11)
556 # clear user controlled registers to prevent speculative use
557 xgr %r0,%r0
558 xgr %r1,%r1
559 xgr %r3,%r3
560 xgr %r4,%r4
561 xgr %r5,%r5
562 xgr %r6,%r6
563 xgr %r7,%r7
564 xgr %r10,%r10
565 mvc __PT_R8(64,%r11),0(%r14)
566 stmg %r8,%r9,__PT_PSW(%r11)
567 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
568 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
569 lgr %r2,%r11 # pass pointer to pt_regs
570 brasl %r14,s390_do_machine_check
571 cghi %r2,0
572 je .Lmcck_return
573 lg %r1,__LC_KERNEL_STACK # switch to kernel stack
574 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
575 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
576 la %r11,STACK_FRAME_OVERHEAD(%r1)
577 lgr %r2,%r11
578 lgr %r15,%r1
579 brasl %r14,s390_handle_mcck
580.Lmcck_return:
581 lctlg %c1,%c1,__PT_CR1(%r11)
582 lmg %r0,%r10,__PT_R0(%r11)
583 mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
584 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
585 jno 0f
586 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
587 stpt __LC_EXIT_TIMER
5880: ALTERNATIVE "nop", __stringify(lghi %r12,__LC_LAST_BREAK_SAVE_AREA),193
589 LBEAR 0(%r12)
590 lmg %r11,%r15,__PT_R11(%r11)
591 LPSWEY __LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE
592
593.Lmcck_panic:
594 /*
595 * Iterate over all possible CPU addresses in the range 0..0xffff
596 * and stop each CPU using signal processor. Use compare and swap
597 * to allow just one CPU-stopper and prevent concurrent CPUs from
598 * stopping each other while leaving the others running.
599 */
600 lhi %r5,0
601 lhi %r6,1
602 larl %r7,.Lstop_lock
603 cs %r5,%r6,0(%r7) # single CPU-stopper only
604 jnz 4f
605 larl %r7,.Lthis_cpu
606 stap 0(%r7) # this CPU address
607 lh %r4,0(%r7)
608 nilh %r4,0
609 lhi %r0,1
610 sll %r0,16 # CPU counter
611 lhi %r3,0 # next CPU address
6120: cr %r3,%r4
613 je 2f
6141: sigp %r1,%r3,SIGP_STOP # stop next CPU
615 brc SIGP_CC_BUSY,1b
6162: ahi %r3,1
617 brct %r0,0b
6183: sigp %r1,%r4,SIGP_STOP # stop this CPU
619 brc SIGP_CC_BUSY,3b
6204: j 4b
621ENDPROC(mcck_int_handler)
622
623ENTRY(restart_int_handler)
624 ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
625 stg %r15,__LC_SAVE_AREA_RESTART
626 TSTMSK __LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4
627 jz 0f
628 la %r15,4095
629 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r15)
6300: larl %r15,.Lstosm_tmp
631 stosm 0(%r15),0x04 # turn dat on, keep irqs off
632 lg %r15,__LC_RESTART_STACK
633 xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
634 stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
635 mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
636 mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
637 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
638 lg %r1,__LC_RESTART_FN # load fn, parm & source cpu
639 lg %r2,__LC_RESTART_DATA
640 lgf %r3,__LC_RESTART_SOURCE
641 ltgr %r3,%r3 # test source cpu address
642 jm 1f # negative -> skip source stop
6430: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
644 brc 10,0b # wait for status stored
6451: basr %r14,%r1 # call function
646 stap __SF_EMPTY(%r15) # store cpu address
647 llgh %r3,__SF_EMPTY(%r15)
6482: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
649 brc 2,2b
6503: j 3b
651ENDPROC(restart_int_handler)
652
653 .section .kprobes.text, "ax"
654
655#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
656/*
657 * The synchronous or the asynchronous stack overflowed. We are dead.
658 * No need to properly save the registers, we are going to panic anyway.
659 * Setup a pt_regs so that show_trace can provide a good call trace.
660 */
661ENTRY(stack_overflow)
662 lg %r15,__LC_NODAT_STACK # change to panic stack
663 la %r11,STACK_FRAME_OVERHEAD(%r15)
664 stmg %r0,%r7,__PT_R0(%r11)
665 stmg %r8,%r9,__PT_PSW(%r11)
666 mvc __PT_R8(64,%r11),0(%r14)
667 stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
668 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
669 lgr %r2,%r11 # pass pointer to pt_regs
670 jg kernel_stack_overflow
671ENDPROC(stack_overflow)
672#endif
673
674 .section .data, "aw"
675 .align 4
676.Lstop_lock: .long 0
677.Lthis_cpu: .short 0
678.Lstosm_tmp: .byte 0
679 .section .rodata, "a"
680#define SYSCALL(esame,emu) .quad __s390x_ ## esame
681 .globl sys_call_table
682sys_call_table:
683#include "asm/syscall_table.h"
684#undef SYSCALL
685
686#ifdef CONFIG_COMPAT
687
688#define SYSCALL(esame,emu) .quad __s390_ ## emu
689 .globl sys_call_table_emu
690sys_call_table_emu:
691#include "asm/syscall_table.h"
692#undef SYSCALL
693#endif
1/*
2 * arch/s390/kernel/entry.S
3 * S390 low-level entry points.
4 *
5 * Copyright (C) IBM Corp. 1999,2006
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 */
11
12#include <linux/init.h>
13#include <linux/linkage.h>
14#include <asm/cache.h>
15#include <asm/errno.h>
16#include <asm/ptrace.h>
17#include <asm/thread_info.h>
18#include <asm/asm-offsets.h>
19#include <asm/unistd.h>
20#include <asm/page.h>
21
22/*
23 * Stack layout for the system_call stack entry.
24 * The first few entries are identical to the user_regs_struct.
25 */
26SP_PTREGS = STACK_FRAME_OVERHEAD
27SP_ARGS = STACK_FRAME_OVERHEAD + __PT_ARGS
28SP_PSW = STACK_FRAME_OVERHEAD + __PT_PSW
29SP_R0 = STACK_FRAME_OVERHEAD + __PT_GPRS
30SP_R1 = STACK_FRAME_OVERHEAD + __PT_GPRS + 4
31SP_R2 = STACK_FRAME_OVERHEAD + __PT_GPRS + 8
32SP_R3 = STACK_FRAME_OVERHEAD + __PT_GPRS + 12
33SP_R4 = STACK_FRAME_OVERHEAD + __PT_GPRS + 16
34SP_R5 = STACK_FRAME_OVERHEAD + __PT_GPRS + 20
35SP_R6 = STACK_FRAME_OVERHEAD + __PT_GPRS + 24
36SP_R7 = STACK_FRAME_OVERHEAD + __PT_GPRS + 28
37SP_R8 = STACK_FRAME_OVERHEAD + __PT_GPRS + 32
38SP_R9 = STACK_FRAME_OVERHEAD + __PT_GPRS + 36
39SP_R10 = STACK_FRAME_OVERHEAD + __PT_GPRS + 40
40SP_R11 = STACK_FRAME_OVERHEAD + __PT_GPRS + 44
41SP_R12 = STACK_FRAME_OVERHEAD + __PT_GPRS + 48
42SP_R13 = STACK_FRAME_OVERHEAD + __PT_GPRS + 52
43SP_R14 = STACK_FRAME_OVERHEAD + __PT_GPRS + 56
44SP_R15 = STACK_FRAME_OVERHEAD + __PT_GPRS + 60
45SP_ORIG_R2 = STACK_FRAME_OVERHEAD + __PT_ORIG_GPR2
46SP_ILC = STACK_FRAME_OVERHEAD + __PT_ILC
47SP_SVCNR = STACK_FRAME_OVERHEAD + __PT_SVCNR
48SP_SIZE = STACK_FRAME_OVERHEAD + __PT_SIZE
49
50_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
51 _TIF_MCCK_PENDING | _TIF_RESTART_SVC | _TIF_PER_TRAP )
52_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
53 _TIF_MCCK_PENDING)
54_TIF_SYSCALL = (_TIF_SYSCALL_TRACE>>8 | _TIF_SYSCALL_AUDIT>>8 | \
55 _TIF_SECCOMP>>8 | _TIF_SYSCALL_TRACEPOINT>>8)
56
57STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
58STACK_SIZE = 1 << STACK_SHIFT
59
60#define BASED(name) name-system_call(%r13)
61
62#ifdef CONFIG_TRACE_IRQFLAGS
63 .macro TRACE_IRQS_ON
64 basr %r2,%r0
65 l %r1,BASED(.Ltrace_irq_on_caller)
66 basr %r14,%r1
67 .endm
68
69 .macro TRACE_IRQS_OFF
70 basr %r2,%r0
71 l %r1,BASED(.Ltrace_irq_off_caller)
72 basr %r14,%r1
73 .endm
74#else
75#define TRACE_IRQS_ON
76#define TRACE_IRQS_OFF
77#endif
78
79#ifdef CONFIG_LOCKDEP
80 .macro LOCKDEP_SYS_EXIT
81 tm SP_PSW+1(%r15),0x01 # returning to user ?
82 jz 0f
83 l %r1,BASED(.Llockdep_sys_exit)
84 basr %r14,%r1
850:
86 .endm
87#else
88#define LOCKDEP_SYS_EXIT
89#endif
90
91/*
92 * Register usage in interrupt handlers:
93 * R9 - pointer to current task structure
94 * R13 - pointer to literal pool
95 * R14 - return register for function calls
96 * R15 - kernel stack pointer
97 */
98
99 .macro UPDATE_VTIME lc_from,lc_to,lc_sum
100 lm %r10,%r11,\lc_from
101 sl %r10,\lc_to
102 sl %r11,\lc_to+4
103 bc 3,BASED(0f)
104 sl %r10,BASED(.Lc_1)
1050: al %r10,\lc_sum
106 al %r11,\lc_sum+4
107 bc 12,BASED(1f)
108 al %r10,BASED(.Lc_1)
1091: stm %r10,%r11,\lc_sum
110 .endm
111
112 .macro SAVE_ALL_SVC psworg,savearea
113 stm %r12,%r15,\savearea
114 l %r13,__LC_SVC_NEW_PSW+4 # load &system_call to %r13
115 l %r15,__LC_KERNEL_STACK # problem state -> load ksp
116 s %r15,BASED(.Lc_spsize) # make room for registers & psw
117 .endm
118
119 .macro SAVE_ALL_BASE savearea
120 stm %r12,%r15,\savearea
121 l %r13,__LC_SVC_NEW_PSW+4 # load &system_call to %r13
122 .endm
123
124 .macro SAVE_ALL_PGM psworg,savearea
125 tm \psworg+1,0x01 # test problem state bit
126#ifdef CONFIG_CHECK_STACK
127 bnz BASED(1f)
128 tml %r15,STACK_SIZE - CONFIG_STACK_GUARD
129 bnz BASED(2f)
130 la %r12,\psworg
131 b BASED(stack_overflow)
132#else
133 bz BASED(2f)
134#endif
1351: l %r15,__LC_KERNEL_STACK # problem state -> load ksp
1362: s %r15,BASED(.Lc_spsize) # make room for registers & psw
137 .endm
138
139 .macro SAVE_ALL_ASYNC psworg,savearea
140 stm %r12,%r15,\savearea
141 l %r13,__LC_SVC_NEW_PSW+4 # load &system_call to %r13
142 la %r12,\psworg
143 tm \psworg+1,0x01 # test problem state bit
144 bnz BASED(1f) # from user -> load async stack
145 clc \psworg+4(4),BASED(.Lcritical_end)
146 bhe BASED(0f)
147 clc \psworg+4(4),BASED(.Lcritical_start)
148 bl BASED(0f)
149 l %r14,BASED(.Lcleanup_critical)
150 basr %r14,%r14
151 tm 1(%r12),0x01 # retest problem state after cleanup
152 bnz BASED(1f)
1530: l %r14,__LC_ASYNC_STACK # are we already on the async stack ?
154 slr %r14,%r15
155 sra %r14,STACK_SHIFT
156#ifdef CONFIG_CHECK_STACK
157 bnz BASED(1f)
158 tml %r15,STACK_SIZE - CONFIG_STACK_GUARD
159 bnz BASED(2f)
160 b BASED(stack_overflow)
161#else
162 bz BASED(2f)
163#endif
1641: l %r15,__LC_ASYNC_STACK
1652: s %r15,BASED(.Lc_spsize) # make room for registers & psw
166 .endm
167
168 .macro CREATE_STACK_FRAME savearea
169 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
170 st %r2,SP_ORIG_R2(%r15) # store original content of gpr 2
171 mvc SP_R12(16,%r15),\savearea # move %r12-%r15 to stack
172 stm %r0,%r11,SP_R0(%r15) # store gprs %r0-%r11 to kernel stack
173 .endm
174
175 .macro RESTORE_ALL psworg,sync
176 mvc \psworg(8),SP_PSW(%r15) # move user PSW to lowcore
177 .if !\sync
178 ni \psworg+1,0xfd # clear wait state bit
179 .endif
180 lm %r0,%r15,SP_R0(%r15) # load gprs 0-15 of user
181 stpt __LC_EXIT_TIMER
182 lpsw \psworg # back to caller
183 .endm
184
185 .macro REENABLE_IRQS
186 mvc __SF_EMPTY(1,%r15),SP_PSW(%r15)
187 ni __SF_EMPTY(%r15),0xbf
188 ssm __SF_EMPTY(%r15)
189 .endm
190
191 .section .kprobes.text, "ax"
192
193/*
194 * Scheduler resume function, called by switch_to
195 * gpr2 = (task_struct *) prev
196 * gpr3 = (task_struct *) next
197 * Returns:
198 * gpr2 = prev
199 */
200ENTRY(__switch_to)
201 basr %r1,0
2020: l %r4,__THREAD_info(%r2) # get thread_info of prev
203 l %r5,__THREAD_info(%r3) # get thread_info of next
204 tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending?
205 bz 1f-0b(%r1)
206 ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev
207 oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next
2081: stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
209 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev
210 l %r15,__THREAD_ksp(%r3) # load kernel stack of next
211 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
212 lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
213 st %r3,__LC_CURRENT # store task struct of next
214 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
215 st %r5,__LC_THREAD_INFO # store thread info of next
216 ahi %r5,STACK_SIZE # end of kernel stack of next
217 st %r5,__LC_KERNEL_STACK # store end of kernel stack
218 br %r14
219
220__critical_start:
221/*
222 * SVC interrupt handler routine. System calls are synchronous events and
223 * are executed with interrupts enabled.
224 */
225
226ENTRY(system_call)
227 stpt __LC_SYNC_ENTER_TIMER
228sysc_saveall:
229 SAVE_ALL_SVC __LC_SVC_OLD_PSW,__LC_SAVE_AREA
230 CREATE_STACK_FRAME __LC_SAVE_AREA
231 mvc SP_PSW(8,%r15),__LC_SVC_OLD_PSW
232 mvc SP_ILC(4,%r15),__LC_SVC_ILC
233 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
234sysc_vtime:
235 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
236sysc_stime:
237 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
238sysc_update:
239 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
240sysc_do_svc:
241 xr %r7,%r7
242 icm %r7,3,SP_SVCNR(%r15) # load svc number and test for svc 0
243 bnz BASED(sysc_nr_ok) # svc number > 0
244 # svc 0: system call number in %r1
245 cl %r1,BASED(.Lnr_syscalls)
246 bnl BASED(sysc_nr_ok)
247 sth %r1,SP_SVCNR(%r15)
248 lr %r7,%r1 # copy svc number to %r7
249sysc_nr_ok:
250 sll %r7,2 # svc number *4
251 l %r10,BASED(.Lsysc_table)
252 tm __TI_flags+2(%r12),_TIF_SYSCALL
253 mvc SP_ARGS(4,%r15),SP_R7(%r15)
254 l %r8,0(%r7,%r10) # get system call addr.
255 bnz BASED(sysc_tracesys)
256 basr %r14,%r8 # call sys_xxxx
257 st %r2,SP_R2(%r15) # store return value (change R2 on stack)
258
259sysc_return:
260 LOCKDEP_SYS_EXIT
261sysc_tif:
262 tm __TI_flags+3(%r12),_TIF_WORK_SVC
263 bnz BASED(sysc_work) # there is work to do (signals etc.)
264sysc_restore:
265 RESTORE_ALL __LC_RETURN_PSW,1
266sysc_done:
267
268#
269# There is work to do, but first we need to check if we return to userspace.
270#
271sysc_work:
272 tm SP_PSW+1(%r15),0x01 # returning to user ?
273 bno BASED(sysc_restore)
274
275#
276# One of the work bits is on. Find out which one.
277#
278sysc_work_tif:
279 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
280 bo BASED(sysc_mcck_pending)
281 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
282 bo BASED(sysc_reschedule)
283 tm __TI_flags+3(%r12),_TIF_SIGPENDING
284 bo BASED(sysc_sigpending)
285 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
286 bo BASED(sysc_notify_resume)
287 tm __TI_flags+3(%r12),_TIF_RESTART_SVC
288 bo BASED(sysc_restart)
289 tm __TI_flags+3(%r12),_TIF_PER_TRAP
290 bo BASED(sysc_singlestep)
291 b BASED(sysc_return) # beware of critical section cleanup
292
293#
294# _TIF_NEED_RESCHED is set, call schedule
295#
296sysc_reschedule:
297 l %r1,BASED(.Lschedule)
298 la %r14,BASED(sysc_return)
299 br %r1 # call scheduler
300
301#
302# _TIF_MCCK_PENDING is set, call handler
303#
304sysc_mcck_pending:
305 l %r1,BASED(.Ls390_handle_mcck)
306 la %r14,BASED(sysc_return)
307 br %r1 # TIF bit will be cleared by handler
308
309#
310# _TIF_SIGPENDING is set, call do_signal
311#
312sysc_sigpending:
313 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP
314 la %r2,SP_PTREGS(%r15) # load pt_regs
315 l %r1,BASED(.Ldo_signal)
316 basr %r14,%r1 # call do_signal
317 tm __TI_flags+3(%r12),_TIF_RESTART_SVC
318 bo BASED(sysc_restart)
319 tm __TI_flags+3(%r12),_TIF_PER_TRAP
320 bo BASED(sysc_singlestep)
321 b BASED(sysc_return)
322
323#
324# _TIF_NOTIFY_RESUME is set, call do_notify_resume
325#
326sysc_notify_resume:
327 la %r2,SP_PTREGS(%r15) # load pt_regs
328 l %r1,BASED(.Ldo_notify_resume)
329 la %r14,BASED(sysc_return)
330 br %r1 # call do_notify_resume
331
332
333#
334# _TIF_RESTART_SVC is set, set up registers and restart svc
335#
336sysc_restart:
337 ni __TI_flags+3(%r12),255-_TIF_RESTART_SVC # clear TIF_RESTART_SVC
338 l %r7,SP_R2(%r15) # load new svc number
339 mvc SP_R2(4,%r15),SP_ORIG_R2(%r15) # restore first argument
340 lm %r2,%r6,SP_R2(%r15) # load svc arguments
341 sth %r7,SP_SVCNR(%r15)
342 b BASED(sysc_nr_ok) # restart svc
343
344#
345# _TIF_PER_TRAP is set, call do_per_trap
346#
347sysc_singlestep:
348 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP
349 xc SP_SVCNR(2,%r15),SP_SVCNR(%r15) # clear svc number
350 la %r2,SP_PTREGS(%r15) # address of register-save area
351 l %r1,BASED(.Lhandle_per) # load adr. of per handler
352 la %r14,BASED(sysc_return) # load adr. of system return
353 br %r1 # branch to do_per_trap
354
355#
356# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
357# and after the system call
358#
359sysc_tracesys:
360 l %r1,BASED(.Ltrace_entry)
361 la %r2,SP_PTREGS(%r15) # load pt_regs
362 la %r3,0
363 xr %r0,%r0
364 icm %r0,3,SP_SVCNR(%r15)
365 st %r0,SP_R2(%r15)
366 basr %r14,%r1
367 cl %r2,BASED(.Lnr_syscalls)
368 bnl BASED(sysc_tracenogo)
369 lr %r7,%r2
370 sll %r7,2 # svc number *4
371 l %r8,0(%r7,%r10)
372sysc_tracego:
373 lm %r3,%r6,SP_R3(%r15)
374 mvc SP_ARGS(4,%r15),SP_R7(%r15)
375 l %r2,SP_ORIG_R2(%r15)
376 basr %r14,%r8 # call sys_xxx
377 st %r2,SP_R2(%r15) # store return value
378sysc_tracenogo:
379 tm __TI_flags+2(%r12),_TIF_SYSCALL
380 bz BASED(sysc_return)
381 l %r1,BASED(.Ltrace_exit)
382 la %r2,SP_PTREGS(%r15) # load pt_regs
383 la %r14,BASED(sysc_return)
384 br %r1
385
386#
387# a new process exits the kernel with ret_from_fork
388#
389ENTRY(ret_from_fork)
390 l %r13,__LC_SVC_NEW_PSW+4
391 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
392 tm SP_PSW+1(%r15),0x01 # forking a kernel thread ?
393 bo BASED(0f)
394 st %r15,SP_R15(%r15) # store stack pointer for new kthread
3950: l %r1,BASED(.Lschedtail)
396 basr %r14,%r1
397 TRACE_IRQS_ON
398 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
399 b BASED(sysc_tracenogo)
400
401#
402# kernel_execve function needs to deal with pt_regs that is not
403# at the usual place
404#
405ENTRY(kernel_execve)
406 stm %r12,%r15,48(%r15)
407 lr %r14,%r15
408 l %r13,__LC_SVC_NEW_PSW+4
409 s %r15,BASED(.Lc_spsize)
410 st %r14,__SF_BACKCHAIN(%r15)
411 la %r12,SP_PTREGS(%r15)
412 xc 0(__PT_SIZE,%r12),0(%r12)
413 l %r1,BASED(.Ldo_execve)
414 lr %r5,%r12
415 basr %r14,%r1
416 ltr %r2,%r2
417 be BASED(0f)
418 a %r15,BASED(.Lc_spsize)
419 lm %r12,%r15,48(%r15)
420 br %r14
421 # execve succeeded.
4220: stnsm __SF_EMPTY(%r15),0xfc # disable interrupts
423 l %r15,__LC_KERNEL_STACK # load ksp
424 s %r15,BASED(.Lc_spsize) # make room for registers & psw
425 mvc SP_PTREGS(__PT_SIZE,%r15),0(%r12) # copy pt_regs
426 l %r12,__LC_THREAD_INFO
427 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
428 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
429 l %r1,BASED(.Lexecve_tail)
430 basr %r14,%r1
431 b BASED(sysc_return)
432
433/*
434 * Program check handler routine
435 */
436
437ENTRY(pgm_check_handler)
438/*
439 * First we need to check for a special case:
440 * Single stepping an instruction that disables the PER event mask will
441 * cause a PER event AFTER the mask has been set. Example: SVC or LPSW.
442 * For a single stepped SVC the program check handler gets control after
443 * the SVC new PSW has been loaded. But we want to execute the SVC first and
444 * then handle the PER event. Therefore we update the SVC old PSW to point
445 * to the pgm_check_handler and branch to the SVC handler after we checked
446 * if we have to load the kernel stack register.
447 * For every other possible cause for PER event without the PER mask set
448 * we just ignore the PER event (FIXME: is there anything we have to do
449 * for LPSW?).
450 */
451 stpt __LC_SYNC_ENTER_TIMER
452 SAVE_ALL_BASE __LC_SAVE_AREA
453 tm __LC_PGM_INT_CODE+1,0x80 # check whether we got a per exception
454 bnz BASED(pgm_per) # got per exception -> special case
455 SAVE_ALL_PGM __LC_PGM_OLD_PSW,__LC_SAVE_AREA
456 CREATE_STACK_FRAME __LC_SAVE_AREA
457 xc SP_ILC(4,%r15),SP_ILC(%r15)
458 mvc SP_PSW(8,%r15),__LC_PGM_OLD_PSW
459 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
460 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
461 bz BASED(pgm_no_vtime)
462 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
463 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
464 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
465pgm_no_vtime:
466 l %r3,__LC_PGM_ILC # load program interruption code
467 l %r4,__LC_TRANS_EXC_CODE
468 REENABLE_IRQS
469 la %r8,0x7f
470 nr %r8,%r3
471 sll %r8,2
472 l %r1,BASED(.Ljump_table)
473 l %r1,0(%r8,%r1) # load address of handler routine
474 la %r2,SP_PTREGS(%r15) # address of register-save area
475 basr %r14,%r1 # branch to interrupt-handler
476pgm_exit:
477 b BASED(sysc_return)
478
479#
480# handle per exception
481#
482pgm_per:
483 tm __LC_PGM_OLD_PSW,0x40 # test if per event recording is on
484 bnz BASED(pgm_per_std) # ok, normal per event from user space
485# ok its one of the special cases, now we need to find out which one
486 clc __LC_PGM_OLD_PSW(8),__LC_SVC_NEW_PSW
487 be BASED(pgm_svcper)
488# no interesting special case, ignore PER event
489 lm %r12,%r15,__LC_SAVE_AREA
490 lpsw 0x28
491
492#
493# Normal per exception
494#
495pgm_per_std:
496 SAVE_ALL_PGM __LC_PGM_OLD_PSW,__LC_SAVE_AREA
497 CREATE_STACK_FRAME __LC_SAVE_AREA
498 mvc SP_PSW(8,%r15),__LC_PGM_OLD_PSW
499 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
500 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
501 bz BASED(pgm_no_vtime2)
502 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
503 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
504 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
505pgm_no_vtime2:
506 l %r1,__TI_task(%r12)
507 tm SP_PSW+1(%r15),0x01 # kernel per event ?
508 bz BASED(kernel_per)
509 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE
510 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS
511 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID
512 oi __TI_flags+3(%r12),_TIF_PER_TRAP # set TIF_PER_TRAP
513 l %r3,__LC_PGM_ILC # load program interruption code
514 l %r4,__LC_TRANS_EXC_CODE
515 REENABLE_IRQS
516 la %r8,0x7f
517 nr %r8,%r3 # clear per-event-bit and ilc
518 be BASED(pgm_exit2) # only per or per+check ?
519 sll %r8,2
520 l %r1,BASED(.Ljump_table)
521 l %r1,0(%r8,%r1) # load address of handler routine
522 la %r2,SP_PTREGS(%r15) # address of register-save area
523 basr %r14,%r1 # branch to interrupt-handler
524pgm_exit2:
525 b BASED(sysc_return)
526
527#
528# it was a single stepped SVC that is causing all the trouble
529#
530pgm_svcper:
531 SAVE_ALL_PGM __LC_SVC_OLD_PSW,__LC_SAVE_AREA
532 CREATE_STACK_FRAME __LC_SAVE_AREA
533 mvc SP_PSW(8,%r15),__LC_SVC_OLD_PSW
534 mvc SP_ILC(4,%r15),__LC_SVC_ILC
535 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
536 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
537 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
538 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
539 l %r8,__TI_task(%r12)
540 mvc __THREAD_per_cause(2,%r8),__LC_PER_CAUSE
541 mvc __THREAD_per_address(4,%r8),__LC_PER_ADDRESS
542 mvc __THREAD_per_paid(1,%r8),__LC_PER_PAID
543 oi __TI_flags+3(%r12),_TIF_PER_TRAP # set TIF_PER_TRAP
544 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
545 lm %r2,%r6,SP_R2(%r15) # load svc arguments
546 b BASED(sysc_do_svc)
547
548#
549# per was called from kernel, must be kprobes
550#
551kernel_per:
552 REENABLE_IRQS
553 xc SP_SVCNR(2,%r15),SP_SVCNR(%r15)
554 la %r2,SP_PTREGS(%r15) # address of register-save area
555 l %r1,BASED(.Lhandle_per) # load adr. of per handler
556 basr %r14,%r1 # branch to do_single_step
557 b BASED(pgm_exit)
558
559/*
560 * IO interrupt handler routine
561 */
562
563ENTRY(io_int_handler)
564 stck __LC_INT_CLOCK
565 stpt __LC_ASYNC_ENTER_TIMER
566 SAVE_ALL_ASYNC __LC_IO_OLD_PSW,__LC_SAVE_AREA+16
567 CREATE_STACK_FRAME __LC_SAVE_AREA+16
568 mvc SP_PSW(8,%r15),0(%r12) # move user PSW to stack
569 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
570 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
571 bz BASED(io_no_vtime)
572 UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
573 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
574 mvc __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
575io_no_vtime:
576 TRACE_IRQS_OFF
577 l %r1,BASED(.Ldo_IRQ) # load address of do_IRQ
578 la %r2,SP_PTREGS(%r15) # address of register-save area
579 basr %r14,%r1 # branch to standard irq handler
580io_return:
581 LOCKDEP_SYS_EXIT
582 TRACE_IRQS_ON
583io_tif:
584 tm __TI_flags+3(%r12),_TIF_WORK_INT
585 bnz BASED(io_work) # there is work to do (signals etc.)
586io_restore:
587 RESTORE_ALL __LC_RETURN_PSW,0
588io_done:
589
590#
591# There is work todo, find out in which context we have been interrupted:
592# 1) if we return to user space we can do all _TIF_WORK_INT work
593# 2) if we return to kernel code and preemptive scheduling is enabled check
594# the preemption counter and if it is zero call preempt_schedule_irq
595# Before any work can be done, a switch to the kernel stack is required.
596#
597io_work:
598 tm SP_PSW+1(%r15),0x01 # returning to user ?
599 bo BASED(io_work_user) # yes -> do resched & signal
600#ifdef CONFIG_PREEMPT
601 # check for preemptive scheduling
602 icm %r0,15,__TI_precount(%r12)
603 bnz BASED(io_restore) # preemption disabled
604 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
605 bno BASED(io_restore)
606 # switch to kernel stack
607 l %r1,SP_R15(%r15)
608 s %r1,BASED(.Lc_spsize)
609 mvc SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
610 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
611 lr %r15,%r1
612 # TRACE_IRQS_ON already done at io_return, call
613 # TRACE_IRQS_OFF to keep things symmetrical
614 TRACE_IRQS_OFF
615 l %r1,BASED(.Lpreempt_schedule_irq)
616 basr %r14,%r1 # call preempt_schedule_irq
617 b BASED(io_return)
618#else
619 b BASED(io_restore)
620#endif
621
622#
623# Need to do work before returning to userspace, switch to kernel stack
624#
625io_work_user:
626 l %r1,__LC_KERNEL_STACK
627 s %r1,BASED(.Lc_spsize)
628 mvc SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
629 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
630 lr %r15,%r1
631
632#
633# One of the work bits is on. Find out which one.
634# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
635# and _TIF_MCCK_PENDING
636#
637io_work_tif:
638 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
639 bo BASED(io_mcck_pending)
640 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
641 bo BASED(io_reschedule)
642 tm __TI_flags+3(%r12),_TIF_SIGPENDING
643 bo BASED(io_sigpending)
644 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
645 bo BASED(io_notify_resume)
646 b BASED(io_return) # beware of critical section cleanup
647
648#
649# _TIF_MCCK_PENDING is set, call handler
650#
651io_mcck_pending:
652 # TRACE_IRQS_ON already done at io_return
653 l %r1,BASED(.Ls390_handle_mcck)
654 basr %r14,%r1 # TIF bit will be cleared by handler
655 TRACE_IRQS_OFF
656 b BASED(io_return)
657
658#
659# _TIF_NEED_RESCHED is set, call schedule
660#
661io_reschedule:
662 # TRACE_IRQS_ON already done at io_return
663 l %r1,BASED(.Lschedule)
664 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
665 basr %r14,%r1 # call scheduler
666 stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts
667 TRACE_IRQS_OFF
668 b BASED(io_return)
669
670#
671# _TIF_SIGPENDING is set, call do_signal
672#
673io_sigpending:
674 # TRACE_IRQS_ON already done at io_return
675 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
676 la %r2,SP_PTREGS(%r15) # load pt_regs
677 l %r1,BASED(.Ldo_signal)
678 basr %r14,%r1 # call do_signal
679 stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts
680 TRACE_IRQS_OFF
681 b BASED(io_return)
682
683#
684# _TIF_SIGPENDING is set, call do_signal
685#
686io_notify_resume:
687 # TRACE_IRQS_ON already done at io_return
688 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
689 la %r2,SP_PTREGS(%r15) # load pt_regs
690 l %r1,BASED(.Ldo_notify_resume)
691 basr %r14,%r1 # call do_signal
692 stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts
693 TRACE_IRQS_OFF
694 b BASED(io_return)
695
696/*
697 * External interrupt handler routine
698 */
699
700ENTRY(ext_int_handler)
701 stck __LC_INT_CLOCK
702 stpt __LC_ASYNC_ENTER_TIMER
703 SAVE_ALL_ASYNC __LC_EXT_OLD_PSW,__LC_SAVE_AREA+16
704 CREATE_STACK_FRAME __LC_SAVE_AREA+16
705 mvc SP_PSW(8,%r15),0(%r12) # move user PSW to stack
706 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
707 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
708 bz BASED(ext_no_vtime)
709 UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
710 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
711 mvc __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
712ext_no_vtime:
713 TRACE_IRQS_OFF
714 la %r2,SP_PTREGS(%r15) # address of register-save area
715 l %r3,__LC_CPU_ADDRESS # get cpu address + interruption code
716 l %r4,__LC_EXT_PARAMS # get external parameters
717 l %r1,BASED(.Ldo_extint)
718 basr %r14,%r1
719 b BASED(io_return)
720
721__critical_end:
722
723/*
724 * Machine check handler routines
725 */
726
727ENTRY(mcck_int_handler)
728 stck __LC_MCCK_CLOCK
729 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer
730 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs
731 SAVE_ALL_BASE __LC_SAVE_AREA+32
732 la %r12,__LC_MCK_OLD_PSW
733 tm __LC_MCCK_CODE,0x80 # system damage?
734 bo BASED(mcck_int_main) # yes -> rest of mcck code invalid
735 mvc __LC_MCCK_ENTER_TIMER(8),__LC_CPU_TIMER_SAVE_AREA
736 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
737 bo BASED(1f)
738 la %r14,__LC_SYNC_ENTER_TIMER
739 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
740 bl BASED(0f)
741 la %r14,__LC_ASYNC_ENTER_TIMER
7420: clc 0(8,%r14),__LC_EXIT_TIMER
743 bl BASED(0f)
744 la %r14,__LC_EXIT_TIMER
7450: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
746 bl BASED(0f)
747 la %r14,__LC_LAST_UPDATE_TIMER
7480: spt 0(%r14)
749 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
7501: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
751 bno BASED(mcck_int_main) # no -> skip cleanup critical
752 tm __LC_MCK_OLD_PSW+1,0x01 # test problem state bit
753 bnz BASED(mcck_int_main) # from user -> load async stack
754 clc __LC_MCK_OLD_PSW+4(4),BASED(.Lcritical_end)
755 bhe BASED(mcck_int_main)
756 clc __LC_MCK_OLD_PSW+4(4),BASED(.Lcritical_start)
757 bl BASED(mcck_int_main)
758 l %r14,BASED(.Lcleanup_critical)
759 basr %r14,%r14
760mcck_int_main:
761 l %r14,__LC_PANIC_STACK # are we already on the panic stack?
762 slr %r14,%r15
763 sra %r14,PAGE_SHIFT
764 be BASED(0f)
765 l %r15,__LC_PANIC_STACK # load panic stack
7660: s %r15,BASED(.Lc_spsize) # make room for registers & psw
767 CREATE_STACK_FRAME __LC_SAVE_AREA+32
768 mvc SP_PSW(8,%r15),0(%r12)
769 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
770 tm __LC_MCCK_CODE+2,0x08 # mwp of old psw valid?
771 bno BASED(mcck_no_vtime) # no -> skip cleanup critical
772 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
773 bz BASED(mcck_no_vtime)
774 UPDATE_VTIME __LC_EXIT_TIMER,__LC_MCCK_ENTER_TIMER,__LC_USER_TIMER
775 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
776 mvc __LC_LAST_UPDATE_TIMER(8),__LC_MCCK_ENTER_TIMER
777mcck_no_vtime:
778 la %r2,SP_PTREGS(%r15) # load pt_regs
779 l %r1,BASED(.Ls390_mcck)
780 basr %r14,%r1 # call machine check handler
781 tm SP_PSW+1(%r15),0x01 # returning to user ?
782 bno BASED(mcck_return)
783 l %r1,__LC_KERNEL_STACK # switch to kernel stack
784 s %r1,BASED(.Lc_spsize)
785 mvc SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
786 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
787 lr %r15,%r1
788 stosm __SF_EMPTY(%r15),0x04 # turn dat on
789 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
790 bno BASED(mcck_return)
791 TRACE_IRQS_OFF
792 l %r1,BASED(.Ls390_handle_mcck)
793 basr %r14,%r1 # call machine check handler
794 TRACE_IRQS_ON
795mcck_return:
796 mvc __LC_RETURN_MCCK_PSW(8),SP_PSW(%r15) # move return PSW
797 ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit
798 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
799 bno BASED(0f)
800 lm %r0,%r15,SP_R0(%r15) # load gprs 0-15
801 stpt __LC_EXIT_TIMER
802 lpsw __LC_RETURN_MCCK_PSW # back to caller
8030: lm %r0,%r15,SP_R0(%r15) # load gprs 0-15
804 lpsw __LC_RETURN_MCCK_PSW # back to caller
805
806 RESTORE_ALL __LC_RETURN_MCCK_PSW,0
807
808/*
809 * Restart interruption handler, kick starter for additional CPUs
810 */
811#ifdef CONFIG_SMP
812 __CPUINIT
813ENTRY(restart_int_handler)
814 basr %r1,0
815restart_base:
816 spt restart_vtime-restart_base(%r1)
817 stck __LC_LAST_UPDATE_CLOCK
818 mvc __LC_LAST_UPDATE_TIMER(8),restart_vtime-restart_base(%r1)
819 mvc __LC_EXIT_TIMER(8),restart_vtime-restart_base(%r1)
820 l %r15,__LC_SAVE_AREA+60 # load ksp
821 lctl %c0,%c15,__LC_CREGS_SAVE_AREA # get new ctl regs
822 lam %a0,%a15,__LC_AREGS_SAVE_AREA
823 lm %r6,%r15,__SF_GPRS(%r15) # load registers from clone
824 l %r1,__LC_THREAD_INFO
825 mvc __LC_USER_TIMER(8),__TI_user_timer(%r1)
826 mvc __LC_SYSTEM_TIMER(8),__TI_system_timer(%r1)
827 xc __LC_STEAL_TIMER(8),__LC_STEAL_TIMER
828 stosm __SF_EMPTY(%r15),0x04 # now we can turn dat on
829 basr %r14,0
830 l %r14,restart_addr-.(%r14)
831 basr %r14,%r14 # branch to start_secondary
832restart_addr:
833 .long start_secondary
834 .align 8
835restart_vtime:
836 .long 0x7fffffff,0xffffffff
837 .previous
838#else
839/*
840 * If we do not run with SMP enabled, let the new CPU crash ...
841 */
842ENTRY(restart_int_handler)
843 basr %r1,0
844restart_base:
845 lpsw restart_crash-restart_base(%r1)
846 .align 8
847restart_crash:
848 .long 0x000a0000,0x00000000
849restart_go:
850#endif
851
852#
853# PSW restart interrupt handler
854#
855ENTRY(psw_restart_int_handler)
856 st %r15,__LC_SAVE_AREA_64(%r0) # save r15
857 basr %r15,0
8580: l %r15,.Lrestart_stack-0b(%r15) # load restart stack
859 l %r15,0(%r15)
860 ahi %r15,-SP_SIZE # make room for pt_regs
861 stm %r0,%r14,SP_R0(%r15) # store gprs %r0-%r14 to stack
862 mvc SP_R15(4,%r15),__LC_SAVE_AREA_64(%r0)# store saved %r15 to stack
863 mvc SP_PSW(8,%r15),__LC_RST_OLD_PSW(%r0) # store restart old psw
864 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) # set backchain to 0
865 basr %r14,0
8661: l %r14,.Ldo_restart-1b(%r14)
867 basr %r14,%r14
868
869 basr %r14,0 # load disabled wait PSW if
8702: lpsw restart_psw_crash-2b(%r14) # do_restart returns
871 .align 4
872.Ldo_restart:
873 .long do_restart
874.Lrestart_stack:
875 .long restart_stack
876 .align 8
877restart_psw_crash:
878 .long 0x000a0000,0x00000000 + restart_psw_crash
879
880 .section .kprobes.text, "ax"
881
882#ifdef CONFIG_CHECK_STACK
883/*
884 * The synchronous or the asynchronous stack overflowed. We are dead.
885 * No need to properly save the registers, we are going to panic anyway.
886 * Setup a pt_regs so that show_trace can provide a good call trace.
887 */
888stack_overflow:
889 l %r15,__LC_PANIC_STACK # change to panic stack
890 sl %r15,BASED(.Lc_spsize)
891 mvc SP_PSW(8,%r15),0(%r12) # move user PSW to stack
892 stm %r0,%r11,SP_R0(%r15) # store gprs %r0-%r11 to kernel stack
893 la %r1,__LC_SAVE_AREA
894 ch %r12,BASED(.L0x020) # old psw addr == __LC_SVC_OLD_PSW ?
895 be BASED(0f)
896 ch %r12,BASED(.L0x028) # old psw addr == __LC_PGM_OLD_PSW ?
897 be BASED(0f)
898 la %r1,__LC_SAVE_AREA+16
8990: mvc SP_R12(16,%r15),0(%r1) # move %r12-%r15 to stack
900 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) # clear back chain
901 l %r1,BASED(1f) # branch to kernel_stack_overflow
902 la %r2,SP_PTREGS(%r15) # load pt_regs
903 br %r1
9041: .long kernel_stack_overflow
905#endif
906
907cleanup_table_system_call:
908 .long system_call + 0x80000000, sysc_do_svc + 0x80000000
909cleanup_table_sysc_tif:
910 .long sysc_tif + 0x80000000, sysc_restore + 0x80000000
911cleanup_table_sysc_restore:
912 .long sysc_restore + 0x80000000, sysc_done + 0x80000000
913cleanup_table_io_tif:
914 .long io_tif + 0x80000000, io_restore + 0x80000000
915cleanup_table_io_restore:
916 .long io_restore + 0x80000000, io_done + 0x80000000
917
918cleanup_critical:
919 clc 4(4,%r12),BASED(cleanup_table_system_call)
920 bl BASED(0f)
921 clc 4(4,%r12),BASED(cleanup_table_system_call+4)
922 bl BASED(cleanup_system_call)
9230:
924 clc 4(4,%r12),BASED(cleanup_table_sysc_tif)
925 bl BASED(0f)
926 clc 4(4,%r12),BASED(cleanup_table_sysc_tif+4)
927 bl BASED(cleanup_sysc_tif)
9280:
929 clc 4(4,%r12),BASED(cleanup_table_sysc_restore)
930 bl BASED(0f)
931 clc 4(4,%r12),BASED(cleanup_table_sysc_restore+4)
932 bl BASED(cleanup_sysc_restore)
9330:
934 clc 4(4,%r12),BASED(cleanup_table_io_tif)
935 bl BASED(0f)
936 clc 4(4,%r12),BASED(cleanup_table_io_tif+4)
937 bl BASED(cleanup_io_tif)
9380:
939 clc 4(4,%r12),BASED(cleanup_table_io_restore)
940 bl BASED(0f)
941 clc 4(4,%r12),BASED(cleanup_table_io_restore+4)
942 bl BASED(cleanup_io_restore)
9430:
944 br %r14
945
946cleanup_system_call:
947 mvc __LC_RETURN_PSW(8),0(%r12)
948 clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+4)
949 bh BASED(0f)
950 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
951 c %r12,BASED(.Lmck_old_psw)
952 be BASED(0f)
953 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
9540: c %r12,BASED(.Lmck_old_psw)
955 la %r12,__LC_SAVE_AREA+32
956 be BASED(0f)
957 la %r12,__LC_SAVE_AREA+16
9580: clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+8)
959 bhe BASED(cleanup_vtime)
960 clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn)
961 bh BASED(0f)
962 mvc __LC_SAVE_AREA(16),0(%r12)
9630: st %r13,4(%r12)
964 l %r15,__LC_KERNEL_STACK # problem state -> load ksp
965 s %r15,BASED(.Lc_spsize) # make room for registers & psw
966 st %r15,12(%r12)
967 CREATE_STACK_FRAME __LC_SAVE_AREA
968 mvc SP_PSW(8,%r15),__LC_SVC_OLD_PSW
969 mvc SP_ILC(4,%r15),__LC_SVC_ILC
970 mvc 0(4,%r12),__LC_THREAD_INFO
971cleanup_vtime:
972 clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+12)
973 bhe BASED(cleanup_stime)
974 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
975cleanup_stime:
976 clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+16)
977 bh BASED(cleanup_update)
978 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
979cleanup_update:
980 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
981 mvc __LC_RETURN_PSW+4(4),BASED(cleanup_table_system_call+4)
982 la %r12,__LC_RETURN_PSW
983 br %r14
984cleanup_system_call_insn:
985 .long sysc_saveall + 0x80000000
986 .long system_call + 0x80000000
987 .long sysc_vtime + 0x80000000
988 .long sysc_stime + 0x80000000
989 .long sysc_update + 0x80000000
990
991cleanup_sysc_tif:
992 mvc __LC_RETURN_PSW(4),0(%r12)
993 mvc __LC_RETURN_PSW+4(4),BASED(cleanup_table_sysc_tif)
994 la %r12,__LC_RETURN_PSW
995 br %r14
996
997cleanup_sysc_restore:
998 clc 4(4,%r12),BASED(cleanup_sysc_restore_insn)
999 be BASED(2f)
1000 mvc __LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
1001 c %r12,BASED(.Lmck_old_psw)
1002 be BASED(0f)
1003 mvc __LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
10040: clc 4(4,%r12),BASED(cleanup_sysc_restore_insn+4)
1005 be BASED(2f)
1006 mvc __LC_RETURN_PSW(8),SP_PSW(%r15)
1007 c %r12,BASED(.Lmck_old_psw)
1008 la %r12,__LC_SAVE_AREA+32
1009 be BASED(1f)
1010 la %r12,__LC_SAVE_AREA+16
10111: mvc 0(16,%r12),SP_R12(%r15)
1012 lm %r0,%r11,SP_R0(%r15)
1013 l %r15,SP_R15(%r15)
10142: la %r12,__LC_RETURN_PSW
1015 br %r14
1016cleanup_sysc_restore_insn:
1017 .long sysc_done - 4 + 0x80000000
1018 .long sysc_done - 8 + 0x80000000
1019
1020cleanup_io_tif:
1021 mvc __LC_RETURN_PSW(4),0(%r12)
1022 mvc __LC_RETURN_PSW+4(4),BASED(cleanup_table_io_tif)
1023 la %r12,__LC_RETURN_PSW
1024 br %r14
1025
1026cleanup_io_restore:
1027 clc 4(4,%r12),BASED(cleanup_io_restore_insn)
1028 be BASED(1f)
1029 mvc __LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
1030 clc 4(4,%r12),BASED(cleanup_io_restore_insn+4)
1031 be BASED(1f)
1032 mvc __LC_RETURN_PSW(8),SP_PSW(%r15)
1033 mvc __LC_SAVE_AREA+32(16),SP_R12(%r15)
1034 lm %r0,%r11,SP_R0(%r15)
1035 l %r15,SP_R15(%r15)
10361: la %r12,__LC_RETURN_PSW
1037 br %r14
1038cleanup_io_restore_insn:
1039 .long io_done - 4 + 0x80000000
1040 .long io_done - 8 + 0x80000000
1041
1042/*
1043 * Integer constants
1044 */
1045 .align 4
1046.Lc_spsize: .long SP_SIZE
1047.Lc_overhead: .long STACK_FRAME_OVERHEAD
1048.Lnr_syscalls: .long NR_syscalls
1049.L0x018: .short 0x018
1050.L0x020: .short 0x020
1051.L0x028: .short 0x028
1052.L0x030: .short 0x030
1053.L0x038: .short 0x038
1054.Lc_1: .long 1
1055
1056/*
1057 * Symbol constants
1058 */
1059.Ls390_mcck: .long s390_do_machine_check
1060.Ls390_handle_mcck:
1061 .long s390_handle_mcck
1062.Lmck_old_psw: .long __LC_MCK_OLD_PSW
1063.Ldo_IRQ: .long do_IRQ
1064.Ldo_extint: .long do_extint
1065.Ldo_signal: .long do_signal
1066.Ldo_notify_resume:
1067 .long do_notify_resume
1068.Lhandle_per: .long do_per_trap
1069.Ldo_execve: .long do_execve
1070.Lexecve_tail: .long execve_tail
1071.Ljump_table: .long pgm_check_table
1072.Lschedule: .long schedule
1073#ifdef CONFIG_PREEMPT
1074.Lpreempt_schedule_irq:
1075 .long preempt_schedule_irq
1076#endif
1077.Ltrace_entry: .long do_syscall_trace_enter
1078.Ltrace_exit: .long do_syscall_trace_exit
1079.Lschedtail: .long schedule_tail
1080.Lsysc_table: .long sys_call_table
1081#ifdef CONFIG_TRACE_IRQFLAGS
1082.Ltrace_irq_on_caller:
1083 .long trace_hardirqs_on_caller
1084.Ltrace_irq_off_caller:
1085 .long trace_hardirqs_off_caller
1086#endif
1087#ifdef CONFIG_LOCKDEP
1088.Llockdep_sys_exit:
1089 .long lockdep_sys_exit
1090#endif
1091.Lcritical_start:
1092 .long __critical_start + 0x80000000
1093.Lcritical_end:
1094 .long __critical_end + 0x80000000
1095.Lcleanup_critical:
1096 .long cleanup_critical
1097
1098 .section .rodata, "a"
1099#define SYSCALL(esa,esame,emu) .long esa
1100 .globl sys_call_table
1101sys_call_table:
1102#include "syscalls.S"
1103#undef SYSCALL