Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * S390 low-level entry points.
4 *
5 * Copyright IBM Corp. 1999, 2012
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 */
10
11#include <linux/export.h>
12#include <linux/init.h>
13#include <linux/linkage.h>
14#include <asm/asm-extable.h>
15#include <asm/alternative.h>
16#include <asm/processor.h>
17#include <asm/cache.h>
18#include <asm/dwarf.h>
19#include <asm/errno.h>
20#include <asm/ptrace.h>
21#include <asm/thread_info.h>
22#include <asm/asm-offsets.h>
23#include <asm/unistd.h>
24#include <asm/page.h>
25#include <asm/sigp.h>
26#include <asm/irq.h>
27#include <asm/fpu-insn.h>
28#include <asm/setup.h>
29#include <asm/nmi.h>
30#include <asm/nospec-insn.h>
31#include <asm/lowcore.h>
32
33_LPP_OFFSET = __LC_LPP
34
35 .macro STBEAR address
36 ALTERNATIVE "nop", ".insn s,0xb2010000,\address", ALT_FACILITY(193)
37 .endm
38
39 .macro LBEAR address
40 ALTERNATIVE "nop", ".insn s,0xb2000000,\address", ALT_FACILITY(193)
41 .endm
42
43 .macro LPSWEY address, lpswe
44 ALTERNATIVE_2 "b \lpswe;nopr", \
45 ".insn siy,0xeb0000000071,\address,0", ALT_FACILITY(193), \
46 __stringify(.insn siy,0xeb0000000071,LOWCORE_ALT_ADDRESS+\address,0), \
47 ALT_LOWCORE
48 .endm
49
50 .macro MBEAR reg, lowcore
51 ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK(\lowcore)),\
52 ALT_FACILITY(193)
53 .endm
54
55 .macro CHECK_STACK savearea, lowcore
56#ifdef CONFIG_CHECK_STACK
57 tml %r15,THREAD_SIZE - CONFIG_STACK_GUARD
58 la %r14,\savearea(\lowcore)
59 jz stack_overflow
60#endif
61 .endm
62
63 .macro CHECK_VMAP_STACK savearea, lowcore, oklabel
64#ifdef CONFIG_VMAP_STACK
65 lgr %r14,%r15
66 nill %r14,0x10000 - THREAD_SIZE
67 oill %r14,STACK_INIT_OFFSET
68 clg %r14,__LC_KERNEL_STACK(\lowcore)
69 je \oklabel
70 clg %r14,__LC_ASYNC_STACK(\lowcore)
71 je \oklabel
72 clg %r14,__LC_MCCK_STACK(\lowcore)
73 je \oklabel
74 clg %r14,__LC_NODAT_STACK(\lowcore)
75 je \oklabel
76 clg %r14,__LC_RESTART_STACK(\lowcore)
77 je \oklabel
78 la %r14,\savearea(\lowcore)
79 j stack_overflow
80#else
81 j \oklabel
82#endif
83 .endm
84
85 /*
86 * The TSTMSK macro generates a test-under-mask instruction by
87 * calculating the memory offset for the specified mask value.
88 * Mask value can be any constant. The macro shifts the mask
89 * value to calculate the memory offset for the test-under-mask
90 * instruction.
91 */
92 .macro TSTMSK addr, mask, size=8, bytepos=0
93 .if (\bytepos < \size) && (\mask >> 8)
94 .if (\mask & 0xff)
95 .error "Mask exceeds byte boundary"
96 .endif
97 TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
98 .exitm
99 .endif
100 .ifeq \mask
101 .error "Mask must not be zero"
102 .endif
103 off = \size - \bytepos - 1
104 tm off+\addr, \mask
105 .endm
106
107 .macro BPOFF
108 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", ALT_SPEC(82)
109 .endm
110
111 .macro BPON
112 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", ALT_SPEC(82)
113 .endm
114
115 .macro BPENTER tif_ptr,tif_mask
116 ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \
117 "j .+12; nop; nop", ALT_SPEC(82)
118 .endm
119
120 .macro BPEXIT tif_ptr,tif_mask
121 TSTMSK \tif_ptr,\tif_mask
122 ALTERNATIVE "jz .+8; .insn rrf,0xb2e80000,0,0,12,0", \
123 "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", ALT_SPEC(82)
124 .endm
125
126#if IS_ENABLED(CONFIG_KVM)
127 .macro SIEEXIT sie_control,lowcore
128 lg %r9,\sie_control # get control block pointer
129 ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
130 lctlg %c1,%c1,__LC_KERNEL_ASCE(\lowcore) # load primary asce
131 lg %r9,__LC_CURRENT(\lowcore)
132 mvi __TI_sie(%r9),0
133 larl %r9,sie_exit # skip forward to sie_exit
134 .endm
135#endif
136
137 .macro STACKLEAK_ERASE
138#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
139 brasl %r14,stackleak_erase_on_task_stack
140#endif
141 .endm
142
143 GEN_BR_THUNK %r14
144
145 .section .kprobes.text, "ax"
146.Ldummy:
147 /*
148 * The following nop exists only in order to avoid that the next
149 * symbol starts at the beginning of the kprobes text section.
150 * In that case there would be several symbols at the same address.
151 * E.g. objdump would take an arbitrary symbol when disassembling
152 * the code.
153 * With the added nop in between this cannot happen.
154 */
155 nop 0
156
157/*
158 * Scheduler resume function, called by __switch_to
159 * gpr2 = (task_struct *)prev
160 * gpr3 = (task_struct *)next
161 * Returns:
162 * gpr2 = prev
163 */
164SYM_FUNC_START(__switch_to_asm)
165 stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
166 lghi %r4,__TASK_stack
167 lghi %r1,__TASK_thread
168 llill %r5,STACK_INIT_OFFSET
169 stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev
170 lg %r15,0(%r4,%r3) # start of kernel stack of next
171 agr %r15,%r5 # end of kernel stack of next
172 GET_LC %r13
173 stg %r3,__LC_CURRENT(%r13) # store task struct of next
174 stg %r15,__LC_KERNEL_STACK(%r13) # store end of kernel stack
175 lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next
176 aghi %r3,__TASK_pid
177 mvc __LC_CURRENT_PID(4,%r13),0(%r3) # store pid of next
178 ALTERNATIVE "nop", "lpp _LPP_OFFSET(%r13)", ALT_FACILITY(40)
179 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
180 BR_EX %r14
181SYM_FUNC_END(__switch_to_asm)
182
183#if IS_ENABLED(CONFIG_KVM)
184/*
185 * __sie64a calling convention:
186 * %r2 pointer to sie control block phys
187 * %r3 pointer to sie control block virt
188 * %r4 guest register save area
189 * %r5 guest asce
190 */
191SYM_FUNC_START(__sie64a)
192 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
193 GET_LC %r13
194 lg %r14,__LC_CURRENT(%r13)
195 stg %r2,__SF_SIE_CONTROL_PHYS(%r15) # save sie block physical..
196 stg %r3,__SF_SIE_CONTROL(%r15) # ...and virtual addresses
197 stg %r4,__SF_SIE_SAVEAREA(%r15) # save guest register save area
198 stg %r5,__SF_SIE_GUEST_ASCE(%r15) # save guest asce
199 xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
200 mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r14) # copy thread flags
201 lmg %r0,%r13,0(%r4) # load guest gprs 0-13
202 mvi __TI_sie(%r14),1
203 lctlg %c1,%c1,__SF_SIE_GUEST_ASCE(%r15) # load primary asce
204 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
205 oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
206 tm __SIE_PROG20+3(%r14),3 # last exit...
207 jnz .Lsie_skip
208 lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr
209 BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
210.Lsie_entry:
211 sie 0(%r14)
212# Let the next instruction be NOP to avoid triggering a machine check
213# and handling it in a guest as result of the instruction execution.
214 nopr 7
215.Lsie_leave:
216 BPOFF
217 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
218.Lsie_skip:
219 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
220 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
221 GET_LC %r14
222 lctlg %c1,%c1,__LC_KERNEL_ASCE(%r14) # load primary asce
223 lg %r14,__LC_CURRENT(%r14)
224 mvi __TI_sie(%r14),0
225SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL)
226 lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area
227 stmg %r0,%r13,0(%r14) # save guest gprs 0-13
228 xgr %r0,%r0 # clear guest registers to
229 xgr %r1,%r1 # prevent speculative use
230 xgr %r3,%r3
231 xgr %r4,%r4
232 xgr %r5,%r5
233 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
234 lg %r2,__SF_SIE_REASON(%r15) # return exit reason code
235 BR_EX %r14
236SYM_FUNC_END(__sie64a)
237EXPORT_SYMBOL(__sie64a)
238EXPORT_SYMBOL(sie_exit)
239#endif
240
241/*
242 * SVC interrupt handler routine. System calls are synchronous events and
243 * are entered with interrupts disabled.
244 */
245
246SYM_CODE_START(system_call)
247 STMG_LC %r8,%r15,__LC_SAVE_AREA
248 GET_LC %r13
249 stpt __LC_SYS_ENTER_TIMER(%r13)
250 BPOFF
251 lghi %r14,0
252.Lsysc_per:
253 STBEAR __LC_LAST_BREAK(%r13)
254 lctlg %c1,%c1,__LC_KERNEL_ASCE(%r13)
255 lg %r15,__LC_KERNEL_STACK(%r13)
256 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
257 stmg %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
258 # clear user controlled register to prevent speculative use
259 xgr %r0,%r0
260 xgr %r1,%r1
261 xgr %r4,%r4
262 xgr %r5,%r5
263 xgr %r6,%r6
264 xgr %r7,%r7
265 xgr %r8,%r8
266 xgr %r9,%r9
267 xgr %r10,%r10
268 xgr %r11,%r11
269 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
270 mvc __PT_R8(64,%r2),__LC_SAVE_AREA(%r13)
271 MBEAR %r2,%r13
272 lgr %r3,%r14
273 brasl %r14,__do_syscall
274 STACKLEAK_ERASE
275 lctlg %c1,%c1,__LC_USER_ASCE(%r13)
276 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
277 BPON
278 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
279 stpt __LC_EXIT_TIMER(%r13)
280 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
281 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
282SYM_CODE_END(system_call)
283
284#
285# a new process exits the kernel with ret_from_fork
286#
287SYM_CODE_START(ret_from_fork)
288 lgr %r3,%r11
289 brasl %r14,__ret_from_fork
290 STACKLEAK_ERASE
291 GET_LC %r13
292 lctlg %c1,%c1,__LC_USER_ASCE(%r13)
293 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
294 BPON
295 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
296 stpt __LC_EXIT_TIMER(%r13)
297 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
298 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
299SYM_CODE_END(ret_from_fork)
300
301/*
302 * Program check handler routine
303 */
304
305SYM_CODE_START(pgm_check_handler)
306 STMG_LC %r8,%r15,__LC_SAVE_AREA
307 GET_LC %r13
308 stpt __LC_SYS_ENTER_TIMER(%r13)
309 BPOFF
310 lmg %r8,%r9,__LC_PGM_OLD_PSW(%r13)
311 xgr %r10,%r10
312 tmhh %r8,0x0001 # coming from user space?
313 jno .Lpgm_skip_asce
314 lctlg %c1,%c1,__LC_KERNEL_ASCE(%r13)
315 j 3f # -> fault in user space
316.Lpgm_skip_asce:
317#if IS_ENABLED(CONFIG_KVM)
318 lg %r11,__LC_CURRENT(%r13)
319 tm __TI_sie(%r11),0xff
320 jz 1f
321 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
322 SIEEXIT __SF_SIE_CONTROL(%r15),%r13
323 lghi %r10,_PIF_GUEST_FAULT
324#endif
3251: tmhh %r8,0x4000 # PER bit set in old PSW ?
326 jnz 2f # -> enabled, can't be a double fault
327 tm __LC_PGM_ILC+3(%r13),0x80 # check for per exception
328 jnz .Lpgm_svcper # -> single stepped svc
3292: CHECK_STACK __LC_SAVE_AREA,%r13
330 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
331 # CHECK_VMAP_STACK branches to stack_overflow or 4f
332 CHECK_VMAP_STACK __LC_SAVE_AREA,%r13,4f
3333: lg %r15,__LC_KERNEL_STACK(%r13)
3344: la %r11,STACK_FRAME_OVERHEAD(%r15)
335 stg %r10,__PT_FLAGS(%r11)
336 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
337 stmg %r0,%r7,__PT_R0(%r11)
338 mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13)
339 mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK(%r13)
340 stmg %r8,%r9,__PT_PSW(%r11)
341 # clear user controlled registers to prevent speculative use
342 xgr %r0,%r0
343 xgr %r1,%r1
344 xgr %r3,%r3
345 xgr %r4,%r4
346 xgr %r5,%r5
347 xgr %r6,%r6
348 xgr %r7,%r7
349 xgr %r12,%r12
350 lgr %r2,%r11
351 brasl %r14,__do_pgm_check
352 tmhh %r8,0x0001 # returning to user space?
353 jno .Lpgm_exit_kernel
354 STACKLEAK_ERASE
355 lctlg %c1,%c1,__LC_USER_ASCE(%r13)
356 BPON
357 stpt __LC_EXIT_TIMER(%r13)
358.Lpgm_exit_kernel:
359 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
360 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
361 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
362 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
363
364#
365# single stepped system call
366#
367.Lpgm_svcper:
368 mvc __LC_RETURN_PSW(8,%r13),__LC_SVC_NEW_PSW(%r13)
369 larl %r14,.Lsysc_per
370 stg %r14,__LC_RETURN_PSW+8(%r13)
371 lghi %r14,1
372 LBEAR __LC_PGM_LAST_BREAK(%r13)
373 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per
374SYM_CODE_END(pgm_check_handler)
375
376/*
377 * Interrupt handler macro used for external and IO interrupts.
378 */
379.macro INT_HANDLER name,lc_old_psw,handler
380SYM_CODE_START(\name)
381 STMG_LC %r8,%r15,__LC_SAVE_AREA
382 GET_LC %r13
383 stckf __LC_INT_CLOCK(%r13)
384 stpt __LC_SYS_ENTER_TIMER(%r13)
385 STBEAR __LC_LAST_BREAK(%r13)
386 BPOFF
387 lmg %r8,%r9,\lc_old_psw(%r13)
388 tmhh %r8,0x0001 # interrupting from user ?
389 jnz 1f
390#if IS_ENABLED(CONFIG_KVM)
391 lg %r10,__LC_CURRENT(%r13)
392 tm __TI_sie(%r10),0xff
393 jz 0f
394 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
395 SIEEXIT __SF_SIE_CONTROL(%r15),%r13
396#endif
3970: CHECK_STACK __LC_SAVE_AREA,%r13
398 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
399 j 2f
4001: lctlg %c1,%c1,__LC_KERNEL_ASCE(%r13)
401 lg %r15,__LC_KERNEL_STACK(%r13)
4022: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
403 la %r11,STACK_FRAME_OVERHEAD(%r15)
404 stmg %r0,%r7,__PT_R0(%r11)
405 # clear user controlled registers to prevent speculative use
406 xgr %r0,%r0
407 xgr %r1,%r1
408 xgr %r3,%r3
409 xgr %r4,%r4
410 xgr %r5,%r5
411 xgr %r6,%r6
412 xgr %r7,%r7
413 xgr %r10,%r10
414 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
415 mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13)
416 MBEAR %r11,%r13
417 stmg %r8,%r9,__PT_PSW(%r11)
418 lgr %r2,%r11 # pass pointer to pt_regs
419 brasl %r14,\handler
420 mvc __LC_RETURN_PSW(16,%r13),__PT_PSW(%r11)
421 tmhh %r8,0x0001 # returning to user ?
422 jno 2f
423 STACKLEAK_ERASE
424 lctlg %c1,%c1,__LC_USER_ASCE(%r13)
425 BPON
426 stpt __LC_EXIT_TIMER(%r13)
4272: LBEAR __PT_LAST_BREAK(%r11)
428 lmg %r0,%r15,__PT_R0(%r11)
429 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
430SYM_CODE_END(\name)
431.endm
432
433 .section .irqentry.text, "ax"
434
435INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
436INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
437
438 .section .kprobes.text, "ax"
439
440/*
441 * Machine check handler routines
442 */
443SYM_CODE_START(mcck_int_handler)
444 BPOFF
445 GET_LC %r13
446 lmg %r8,%r9,__LC_MCK_OLD_PSW(%r13)
447 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_SYSTEM_DAMAGE
448 jo .Lmcck_panic # yes -> rest of mcck code invalid
449 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_CR_VALID
450 jno .Lmcck_panic # control registers invalid -> panic
451 ptlb
452 lay %r14,__LC_CPU_TIMER_SAVE_AREA(%r13)
453 mvc __LC_MCCK_ENTER_TIMER(8,%r13),0(%r14)
454 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_CPU_TIMER_VALID
455 jo 3f
456 la %r14,__LC_SYS_ENTER_TIMER(%r13)
457 clc 0(8,%r14),__LC_EXIT_TIMER(%r13)
458 jl 1f
459 la %r14,__LC_EXIT_TIMER(%r13)
4601: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER(%r13)
461 jl 2f
462 la %r14,__LC_LAST_UPDATE_TIMER(%r13)
4632: spt 0(%r14)
464 mvc __LC_MCCK_ENTER_TIMER(8,%r13),0(%r14)
4653: TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_PSW_MWP_VALID
466 jno .Lmcck_panic
467 tmhh %r8,0x0001 # interrupting from user ?
468 jnz .Lmcck_user
469 TSTMSK __LC_MCCK_CODE(%r13),MCCK_CODE_PSW_IA_VALID
470 jno .Lmcck_panic
471#if IS_ENABLED(CONFIG_KVM)
472 lg %r10,__LC_CURRENT(%r13)
473 tm __TI_sie(%r10),0xff
474 jz .Lmcck_user
475 # Need to compare the address instead of __TI_SIE flag.
476 # Otherwise there would be a race between setting the flag
477 # and entering SIE (or leaving and clearing the flag). This
478 # would cause machine checks targeted at the guest to be
479 # handled by the host.
480 larl %r14,.Lsie_entry
481 clgrjl %r9,%r14, 4f
482 larl %r14,.Lsie_leave
483 clgrjhe %r9,%r14, 4f
484 lg %r10,__LC_PCPU
485 oi __PCPU_FLAGS+7(%r10), _CIF_MCCK_GUEST
4864: BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
487 SIEEXIT __SF_SIE_CONTROL(%r15),%r13
488#endif
489.Lmcck_user:
490 lg %r15,__LC_MCCK_STACK(%r13)
491 la %r11,STACK_FRAME_OVERHEAD(%r15)
492 stctg %c1,%c1,__PT_CR1(%r11)
493 lctlg %c1,%c1,__LC_KERNEL_ASCE(%r13)
494 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
495 lay %r14,__LC_GPREGS_SAVE_AREA(%r13)
496 mvc __PT_R0(128,%r11),0(%r14)
497 # clear user controlled registers to prevent speculative use
498 xgr %r0,%r0
499 xgr %r1,%r1
500 xgr %r3,%r3
501 xgr %r4,%r4
502 xgr %r5,%r5
503 xgr %r6,%r6
504 xgr %r7,%r7
505 xgr %r10,%r10
506 stmg %r8,%r9,__PT_PSW(%r11)
507 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
508 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
509 lgr %r2,%r11 # pass pointer to pt_regs
510 brasl %r14,s390_do_machine_check
511 lctlg %c1,%c1,__PT_CR1(%r11)
512 lmg %r0,%r10,__PT_R0(%r11)
513 mvc __LC_RETURN_MCCK_PSW(16,%r13),__PT_PSW(%r11) # move return PSW
514 tm __LC_RETURN_MCCK_PSW+1(%r13),0x01 # returning to user ?
515 jno 0f
516 BPON
517 stpt __LC_EXIT_TIMER(%r13)
5180: ALTERNATIVE "brcl 0,0", __stringify(lay %r12,__LC_LAST_BREAK_SAVE_AREA(%r13)),\
519 ALT_FACILITY(193)
520 LBEAR 0(%r12)
521 lmg %r11,%r15,__PT_R11(%r11)
522 LPSWEY __LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE
523
524.Lmcck_panic:
525 /*
526 * Iterate over all possible CPU addresses in the range 0..0xffff
527 * and stop each CPU using signal processor. Use compare and swap
528 * to allow just one CPU-stopper and prevent concurrent CPUs from
529 * stopping each other while leaving the others running.
530 */
531 lhi %r5,0
532 lhi %r6,1
533 larl %r7,stop_lock
534 cs %r5,%r6,0(%r7) # single CPU-stopper only
535 jnz 4f
536 larl %r7,this_cpu
537 stap 0(%r7) # this CPU address
538 lh %r4,0(%r7)
539 nilh %r4,0
540 lhi %r0,1
541 sll %r0,16 # CPU counter
542 lhi %r3,0 # next CPU address
5430: cr %r3,%r4
544 je 2f
5451: sigp %r1,%r3,SIGP_STOP # stop next CPU
546 brc SIGP_CC_BUSY,1b
5472: ahi %r3,1
548 brct %r0,0b
5493: sigp %r1,%r4,SIGP_STOP # stop this CPU
550 brc SIGP_CC_BUSY,3b
5514: j 4b
552SYM_CODE_END(mcck_int_handler)
553
554SYM_CODE_START(restart_int_handler)
555 ALTERNATIVE "nop", "lpp _LPP_OFFSET", ALT_FACILITY(40)
556 stg %r15,__LC_SAVE_AREA_RESTART
557 TSTMSK __LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4
558 jz 0f
559 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA
5600: larl %r15,daton_psw
561 lpswe 0(%r15) # turn dat on, keep irqs off
562.Ldaton:
563 GET_LC %r15
564 lg %r15,__LC_RESTART_STACK(%r15)
565 xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
566 stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
567 GET_LC %r13
568 mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART(%r13)
569 mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW(%r13)
570 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
571 lg %r1,__LC_RESTART_FN(%r13) # load fn, parm & source cpu
572 lg %r2,__LC_RESTART_DATA(%r13)
573 lgf %r3,__LC_RESTART_SOURCE(%r13)
574 ltgr %r3,%r3 # test source cpu address
575 jm 1f # negative -> skip source stop
5760: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
577 brc 10,0b # wait for status stored
5781: basr %r14,%r1 # call function
579 stap __SF_EMPTY(%r15) # store cpu address
580 llgh %r3,__SF_EMPTY(%r15)
5812: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
582 brc 2,2b
5833: j 3b
584SYM_CODE_END(restart_int_handler)
585
586 __INIT
587SYM_CODE_START(early_pgm_check_handler)
588 STMG_LC %r8,%r15,__LC_SAVE_AREA
589 GET_LC %r13
590 aghi %r15,-(STACK_FRAME_OVERHEAD+__PT_SIZE)
591 la %r11,STACK_FRAME_OVERHEAD(%r15)
592 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
593 stmg %r0,%r7,__PT_R0(%r11)
594 mvc __PT_PSW(16,%r11),__LC_PGM_OLD_PSW(%r13)
595 mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13)
596 lgr %r2,%r11
597 brasl %r14,__do_early_pgm_check
598 mvc __LC_RETURN_PSW(16,%r13),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
599 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
600 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
601SYM_CODE_END(early_pgm_check_handler)
602 __FINIT
603
604 .section .kprobes.text, "ax"
605
606#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
607/*
608 * The synchronous or the asynchronous stack overflowed. We are dead.
609 * No need to properly save the registers, we are going to panic anyway.
610 * Setup a pt_regs so that show_trace can provide a good call trace.
611 */
612SYM_CODE_START(stack_overflow)
613 GET_LC %r15
614 lg %r15,__LC_NODAT_STACK(%r15) # change to panic stack
615 la %r11,STACK_FRAME_OVERHEAD(%r15)
616 stmg %r0,%r7,__PT_R0(%r11)
617 stmg %r8,%r9,__PT_PSW(%r11)
618 mvc __PT_R8(64,%r11),0(%r14)
619 stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
620 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
621 lgr %r2,%r11 # pass pointer to pt_regs
622 jg kernel_stack_overflow
623SYM_CODE_END(stack_overflow)
624#endif
625
626 .section .data, "aw"
627 .balign 4
628SYM_DATA_LOCAL(stop_lock, .long 0)
629SYM_DATA_LOCAL(this_cpu, .short 0)
630 .balign 8
631SYM_DATA_START_LOCAL(daton_psw)
632 .quad PSW_KERNEL_BITS
633 .quad .Ldaton
634SYM_DATA_END(daton_psw)
635
636 .section .rodata, "a"
637 .balign 8
638#define SYSCALL(esame,emu) .quad __s390x_ ## esame
639SYM_DATA_START(sys_call_table)
640#include "asm/syscall_table.h"
641SYM_DATA_END(sys_call_table)
642#undef SYSCALL
643
644#ifdef CONFIG_COMPAT
645
646#define SYSCALL(esame,emu) .quad __s390_ ## emu
647SYM_DATA_START(sys_call_table_emu)
648#include "asm/syscall_table.h"
649SYM_DATA_END(sys_call_table_emu)
650#undef SYSCALL
651#endif
1/*
2 * arch/s390/kernel/entry.S
3 * S390 low-level entry points.
4 *
5 * Copyright (C) IBM Corp. 1999,2012
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 */
11
12#include <linux/init.h>
13#include <linux/linkage.h>
14#include <asm/cache.h>
15#include <asm/errno.h>
16#include <asm/ptrace.h>
17#include <asm/thread_info.h>
18#include <asm/asm-offsets.h>
19#include <asm/unistd.h>
20#include <asm/page.h>
21
22__PT_R0 = __PT_GPRS
23__PT_R1 = __PT_GPRS + 4
24__PT_R2 = __PT_GPRS + 8
25__PT_R3 = __PT_GPRS + 12
26__PT_R4 = __PT_GPRS + 16
27__PT_R5 = __PT_GPRS + 20
28__PT_R6 = __PT_GPRS + 24
29__PT_R7 = __PT_GPRS + 28
30__PT_R8 = __PT_GPRS + 32
31__PT_R9 = __PT_GPRS + 36
32__PT_R10 = __PT_GPRS + 40
33__PT_R11 = __PT_GPRS + 44
34__PT_R12 = __PT_GPRS + 48
35__PT_R13 = __PT_GPRS + 524
36__PT_R14 = __PT_GPRS + 56
37__PT_R15 = __PT_GPRS + 60
38
39_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
40 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
41_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
42 _TIF_MCCK_PENDING)
43_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
44 _TIF_SYSCALL_TRACEPOINT)
45
46STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
47STACK_SIZE = 1 << STACK_SHIFT
48
49#define BASED(name) name-system_call(%r13)
50
51 .macro TRACE_IRQS_ON
52#ifdef CONFIG_TRACE_IRQFLAGS
53 basr %r2,%r0
54 l %r1,BASED(.Lhardirqs_on)
55 basr %r14,%r1 # call trace_hardirqs_on_caller
56#endif
57 .endm
58
59 .macro TRACE_IRQS_OFF
60#ifdef CONFIG_TRACE_IRQFLAGS
61 basr %r2,%r0
62 l %r1,BASED(.Lhardirqs_off)
63 basr %r14,%r1 # call trace_hardirqs_off_caller
64#endif
65 .endm
66
67 .macro LOCKDEP_SYS_EXIT
68#ifdef CONFIG_LOCKDEP
69 tm __PT_PSW+1(%r11),0x01 # returning to user ?
70 jz .+10
71 l %r1,BASED(.Llockdep_sys_exit)
72 basr %r14,%r1 # call lockdep_sys_exit
73#endif
74 .endm
75
76 .macro CHECK_STACK stacksize,savearea
77#ifdef CONFIG_CHECK_STACK
78 tml %r15,\stacksize - CONFIG_STACK_GUARD
79 la %r14,\savearea
80 jz stack_overflow
81#endif
82 .endm
83
84 .macro SWITCH_ASYNC savearea,stack,shift
85 tmh %r8,0x0001 # interrupting from user ?
86 jnz 1f
87 lr %r14,%r9
88 sl %r14,BASED(.Lcritical_start)
89 cl %r14,BASED(.Lcritical_length)
90 jhe 0f
91 la %r11,\savearea # inside critical section, do cleanup
92 bras %r14,cleanup_critical
93 tmh %r8,0x0001 # retest problem state after cleanup
94 jnz 1f
950: l %r14,\stack # are we already on the target stack?
96 slr %r14,%r15
97 sra %r14,\shift
98 jnz 1f
99 CHECK_STACK 1<<\shift,\savearea
100 j 2f
1011: l %r15,\stack # load target stack
1022: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
103 la %r11,STACK_FRAME_OVERHEAD(%r15)
104 .endm
105
106 .macro ADD64 high,low,timer
107 al \high,\timer
108 al \low,4+\timer
109 brc 12,.+8
110 ahi \high,1
111 .endm
112
113 .macro SUB64 high,low,timer
114 sl \high,\timer
115 sl \low,4+\timer
116 brc 3,.+8
117 ahi \high,-1
118 .endm
119
120 .macro UPDATE_VTIME high,low,enter_timer
121 lm \high,\low,__LC_EXIT_TIMER
122 SUB64 \high,\low,\enter_timer
123 ADD64 \high,\low,__LC_USER_TIMER
124 stm \high,\low,__LC_USER_TIMER
125 lm \high,\low,__LC_LAST_UPDATE_TIMER
126 SUB64 \high,\low,__LC_EXIT_TIMER
127 ADD64 \high,\low,__LC_SYSTEM_TIMER
128 stm \high,\low,__LC_SYSTEM_TIMER
129 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
130 .endm
131
132 .macro REENABLE_IRQS
133 st %r8,__LC_RETURN_PSW
134 ni __LC_RETURN_PSW,0xbf
135 ssm __LC_RETURN_PSW
136 .endm
137
138 .section .kprobes.text, "ax"
139
140/*
141 * Scheduler resume function, called by switch_to
142 * gpr2 = (task_struct *) prev
143 * gpr3 = (task_struct *) next
144 * Returns:
145 * gpr2 = prev
146 */
147ENTRY(__switch_to)
148 stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
149 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev
150 l %r4,__THREAD_info(%r2) # get thread_info of prev
151 l %r5,__THREAD_info(%r3) # get thread_info of next
152 lr %r15,%r5
153 ahi %r15,STACK_SIZE # end of kernel stack of next
154 st %r3,__LC_CURRENT # store task struct of next
155 st %r5,__LC_THREAD_INFO # store thread info of next
156 st %r15,__LC_KERNEL_STACK # store end of kernel stack
157 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
158 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
159 l %r15,__THREAD_ksp(%r3) # load kernel stack of next
160 tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending?
161 jz 0f
162 ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev
163 oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next
1640: lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
165 br %r14
166
167__critical_start:
168/*
169 * SVC interrupt handler routine. System calls are synchronous events and
170 * are executed with interrupts enabled.
171 */
172
173ENTRY(system_call)
174 stpt __LC_SYNC_ENTER_TIMER
175sysc_stm:
176 stm %r8,%r15,__LC_SAVE_AREA_SYNC
177 l %r12,__LC_THREAD_INFO
178 l %r13,__LC_SVC_NEW_PSW+4
179sysc_per:
180 l %r15,__LC_KERNEL_STACK
181 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
182 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
183sysc_vtime:
184 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
185 stm %r0,%r7,__PT_R0(%r11)
186 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
187 mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW
188 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
189sysc_do_svc:
190 oi __TI_flags+3(%r12),_TIF_SYSCALL
191 lh %r8,__PT_INT_CODE+2(%r11)
192 sla %r8,2 # shift and test for svc0
193 jnz sysc_nr_ok
194 # svc 0: system call number in %r1
195 cl %r1,BASED(.Lnr_syscalls)
196 jnl sysc_nr_ok
197 sth %r1,__PT_INT_CODE+2(%r11)
198 lr %r8,%r1
199 sla %r8,2
200sysc_nr_ok:
201 l %r10,BASED(.Lsys_call_table) # 31 bit system call table
202 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
203 st %r2,__PT_ORIG_GPR2(%r11)
204 st %r7,STACK_FRAME_OVERHEAD(%r15)
205 l %r9,0(%r8,%r10) # get system call addr.
206 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
207 jnz sysc_tracesys
208 basr %r14,%r9 # call sys_xxxx
209 st %r2,__PT_R2(%r11) # store return value
210
211sysc_return:
212 LOCKDEP_SYS_EXIT
213sysc_tif:
214 tm __PT_PSW+1(%r11),0x01 # returning to user ?
215 jno sysc_restore
216 tm __TI_flags+3(%r12),_TIF_WORK_SVC
217 jnz sysc_work # check for work
218 ni __TI_flags+3(%r12),255-_TIF_SYSCALL
219sysc_restore:
220 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
221 stpt __LC_EXIT_TIMER
222 lm %r0,%r15,__PT_R0(%r11)
223 lpsw __LC_RETURN_PSW
224sysc_done:
225
226#
227# One of the work bits is on. Find out which one.
228#
229sysc_work:
230 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
231 jo sysc_mcck_pending
232 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
233 jo sysc_reschedule
234 tm __TI_flags+3(%r12),_TIF_SIGPENDING
235 jo sysc_sigpending
236 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
237 jo sysc_notify_resume
238 tm __TI_flags+3(%r12),_TIF_PER_TRAP
239 jo sysc_singlestep
240 j sysc_return # beware of critical section cleanup
241
242#
243# _TIF_NEED_RESCHED is set, call schedule
244#
245sysc_reschedule:
246 l %r1,BASED(.Lschedule)
247 la %r14,BASED(sysc_return)
248 br %r1 # call schedule
249
250#
251# _TIF_MCCK_PENDING is set, call handler
252#
253sysc_mcck_pending:
254 l %r1,BASED(.Lhandle_mcck)
255 la %r14,BASED(sysc_return)
256 br %r1 # TIF bit will be cleared by handler
257
258#
259# _TIF_SIGPENDING is set, call do_signal
260#
261sysc_sigpending:
262 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP
263 lr %r2,%r11 # pass pointer to pt_regs
264 l %r1,BASED(.Ldo_signal)
265 basr %r14,%r1 # call do_signal
266 tm __TI_flags+3(%r12),_TIF_SYSCALL
267 jno sysc_return
268 lm %r2,%r7,__PT_R2(%r11) # load svc arguments
269 xr %r8,%r8 # svc 0 returns -ENOSYS
270 clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2)
271 jnl sysc_nr_ok # invalid svc number -> do svc 0
272 lh %r8,__PT_INT_CODE+2(%r11) # load new svc number
273 sla %r8,2
274 j sysc_nr_ok # restart svc
275
276#
277# _TIF_NOTIFY_RESUME is set, call do_notify_resume
278#
279sysc_notify_resume:
280 lr %r2,%r11 # pass pointer to pt_regs
281 l %r1,BASED(.Ldo_notify_resume)
282 la %r14,BASED(sysc_return)
283 br %r1 # call do_notify_resume
284
285#
286# _TIF_PER_TRAP is set, call do_per_trap
287#
288sysc_singlestep:
289 ni __TI_flags+3(%r12),255-(_TIF_SYSCALL | _TIF_PER_TRAP)
290 lr %r2,%r11 # pass pointer to pt_regs
291 l %r1,BASED(.Ldo_per_trap)
292 la %r14,BASED(sysc_return)
293 br %r1 # call do_per_trap
294
295#
296# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
297# and after the system call
298#
299sysc_tracesys:
300 l %r1,BASED(.Ltrace_enter)
301 lr %r2,%r11 # pass pointer to pt_regs
302 la %r3,0
303 xr %r0,%r0
304 icm %r0,3,__PT_INT_CODE+2(%r11)
305 st %r0,__PT_R2(%r11)
306 basr %r14,%r1 # call do_syscall_trace_enter
307 cl %r2,BASED(.Lnr_syscalls)
308 jnl sysc_tracenogo
309 lr %r8,%r2
310 sll %r8,2
311 l %r9,0(%r8,%r10)
312sysc_tracego:
313 lm %r3,%r7,__PT_R3(%r11)
314 st %r7,STACK_FRAME_OVERHEAD(%r15)
315 l %r2,__PT_ORIG_GPR2(%r11)
316 basr %r14,%r9 # call sys_xxx
317 st %r2,__PT_R2(%r11) # store return value
318sysc_tracenogo:
319 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
320 jz sysc_return
321 l %r1,BASED(.Ltrace_exit)
322 lr %r2,%r11 # pass pointer to pt_regs
323 la %r14,BASED(sysc_return)
324 br %r1 # call do_syscall_trace_exit
325
326#
327# a new process exits the kernel with ret_from_fork
328#
329ENTRY(ret_from_fork)
330 la %r11,STACK_FRAME_OVERHEAD(%r15)
331 l %r12,__LC_THREAD_INFO
332 l %r13,__LC_SVC_NEW_PSW+4
333 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
334 jo 0f
335 st %r15,__PT_R15(%r11) # store stack pointer for new kthread
3360: l %r1,BASED(.Lschedule_tail)
337 basr %r14,%r1 # call schedule_tail
338 TRACE_IRQS_ON
339 ssm __LC_SVC_NEW_PSW # reenable interrupts
340 j sysc_tracenogo
341
342#
343# kernel_execve function needs to deal with pt_regs that is not
344# at the usual place
345#
346ENTRY(kernel_execve)
347 stm %r12,%r15,48(%r15)
348 lr %r14,%r15
349 l %r13,__LC_SVC_NEW_PSW+4
350 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
351 st %r14,__SF_BACKCHAIN(%r15)
352 la %r12,STACK_FRAME_OVERHEAD(%r15)
353 xc 0(__PT_SIZE,%r12),0(%r12)
354 l %r1,BASED(.Ldo_execve)
355 lr %r5,%r12
356 basr %r14,%r1 # call do_execve
357 ltr %r2,%r2
358 je 0f
359 ahi %r15,(STACK_FRAME_OVERHEAD + __PT_SIZE)
360 lm %r12,%r15,48(%r15)
361 br %r14
362 # execve succeeded.
3630: ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
364 l %r15,__LC_KERNEL_STACK # load ksp
365 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
366 la %r11,STACK_FRAME_OVERHEAD(%r15)
367 mvc 0(__PT_SIZE,%r11),0(%r12) # copy pt_regs
368 l %r12,__LC_THREAD_INFO
369 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
370 ssm __LC_SVC_NEW_PSW # reenable interrupts
371 l %r1,BASED(.Lexecve_tail)
372 basr %r14,%r1 # call execve_tail
373 j sysc_return
374
375/*
376 * Program check handler routine
377 */
378
379ENTRY(pgm_check_handler)
380 stpt __LC_SYNC_ENTER_TIMER
381 stm %r8,%r15,__LC_SAVE_AREA_SYNC
382 l %r12,__LC_THREAD_INFO
383 l %r13,__LC_SVC_NEW_PSW+4
384 lm %r8,%r9,__LC_PGM_OLD_PSW
385 tmh %r8,0x0001 # test problem state bit
386 jnz 1f # -> fault in user space
387 tmh %r8,0x4000 # PER bit set in old PSW ?
388 jnz 0f # -> enabled, can't be a double fault
389 tm __LC_PGM_ILC+3,0x80 # check for per exception
390 jnz pgm_svcper # -> single stepped svc
3910: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
392 j 2f
3931: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
394 l %r15,__LC_KERNEL_STACK
3952: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
396 la %r11,STACK_FRAME_OVERHEAD(%r15)
397 stm %r0,%r7,__PT_R0(%r11)
398 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
399 stm %r8,%r9,__PT_PSW(%r11)
400 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
401 mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE
402 tm __LC_PGM_ILC+3,0x80 # check for per exception
403 jz 0f
404 l %r1,__TI_task(%r12)
405 tmh %r8,0x0001 # kernel per event ?
406 jz pgm_kprobe
407 oi __TI_flags+3(%r12),_TIF_PER_TRAP
408 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS
409 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE
410 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID
4110: REENABLE_IRQS
412 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
413 l %r1,BASED(.Ljump_table)
414 la %r10,0x7f
415 n %r10,__PT_INT_CODE(%r11)
416 je sysc_return
417 sll %r10,2
418 l %r1,0(%r10,%r1) # load address of handler routine
419 lr %r2,%r11 # pass pointer to pt_regs
420 basr %r14,%r1 # branch to interrupt-handler
421 j sysc_return
422
423#
424# PER event in supervisor state, must be kprobes
425#
426pgm_kprobe:
427 REENABLE_IRQS
428 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
429 l %r1,BASED(.Ldo_per_trap)
430 lr %r2,%r11 # pass pointer to pt_regs
431 basr %r14,%r1 # call do_per_trap
432 j sysc_return
433
434#
435# single stepped system call
436#
437pgm_svcper:
438 oi __TI_flags+3(%r12),_TIF_PER_TRAP
439 mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW
440 mvc __LC_RETURN_PSW+4(4),BASED(.Lsysc_per)
441 lpsw __LC_RETURN_PSW # branch to sysc_per and enable irqs
442
443/*
444 * IO interrupt handler routine
445 */
446
447ENTRY(io_int_handler)
448 stck __LC_INT_CLOCK
449 stpt __LC_ASYNC_ENTER_TIMER
450 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
451 l %r12,__LC_THREAD_INFO
452 l %r13,__LC_SVC_NEW_PSW+4
453 lm %r8,%r9,__LC_IO_OLD_PSW
454 tmh %r8,0x0001 # interrupting from user ?
455 jz io_skip
456 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
457io_skip:
458 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
459 stm %r0,%r7,__PT_R0(%r11)
460 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
461 stm %r8,%r9,__PT_PSW(%r11)
462 TRACE_IRQS_OFF
463 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
464 l %r1,BASED(.Ldo_IRQ)
465 lr %r2,%r11 # pass pointer to pt_regs
466 basr %r14,%r1 # call do_IRQ
467io_return:
468 LOCKDEP_SYS_EXIT
469 TRACE_IRQS_ON
470io_tif:
471 tm __TI_flags+3(%r12),_TIF_WORK_INT
472 jnz io_work # there is work to do (signals etc.)
473io_restore:
474 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
475 stpt __LC_EXIT_TIMER
476 lm %r0,%r15,__PT_R0(%r11)
477 lpsw __LC_RETURN_PSW
478io_done:
479
480#
481# There is work todo, find out in which context we have been interrupted:
482# 1) if we return to user space we can do all _TIF_WORK_INT work
483# 2) if we return to kernel code and preemptive scheduling is enabled check
484# the preemption counter and if it is zero call preempt_schedule_irq
485# Before any work can be done, a switch to the kernel stack is required.
486#
487io_work:
488 tm __PT_PSW+1(%r11),0x01 # returning to user ?
489 jo io_work_user # yes -> do resched & signal
490#ifdef CONFIG_PREEMPT
491 # check for preemptive scheduling
492 icm %r0,15,__TI_precount(%r12)
493 jnz io_restore # preemption disabled
494 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
495 jno io_restore
496 # switch to kernel stack
497 l %r1,__PT_R15(%r11)
498 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
499 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
500 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
501 la %r11,STACK_FRAME_OVERHEAD(%r1)
502 lr %r15,%r1
503 # TRACE_IRQS_ON already done at io_return, call
504 # TRACE_IRQS_OFF to keep things symmetrical
505 TRACE_IRQS_OFF
506 l %r1,BASED(.Lpreempt_irq)
507 basr %r14,%r1 # call preempt_schedule_irq
508 j io_return
509#else
510 j io_restore
511#endif
512
513#
514# Need to do work before returning to userspace, switch to kernel stack
515#
516io_work_user:
517 l %r1,__LC_KERNEL_STACK
518 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
519 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
520 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
521 la %r11,STACK_FRAME_OVERHEAD(%r1)
522 lr %r15,%r1
523
524#
525# One of the work bits is on. Find out which one.
526# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
527# and _TIF_MCCK_PENDING
528#
529io_work_tif:
530 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
531 jo io_mcck_pending
532 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
533 jo io_reschedule
534 tm __TI_flags+3(%r12),_TIF_SIGPENDING
535 jo io_sigpending
536 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
537 jo io_notify_resume
538 j io_return # beware of critical section cleanup
539
540#
541# _TIF_MCCK_PENDING is set, call handler
542#
543io_mcck_pending:
544 # TRACE_IRQS_ON already done at io_return
545 l %r1,BASED(.Lhandle_mcck)
546 basr %r14,%r1 # TIF bit will be cleared by handler
547 TRACE_IRQS_OFF
548 j io_return
549
550#
551# _TIF_NEED_RESCHED is set, call schedule
552#
553io_reschedule:
554 # TRACE_IRQS_ON already done at io_return
555 l %r1,BASED(.Lschedule)
556 ssm __LC_SVC_NEW_PSW # reenable interrupts
557 basr %r14,%r1 # call scheduler
558 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
559 TRACE_IRQS_OFF
560 j io_return
561
562#
563# _TIF_SIGPENDING is set, call do_signal
564#
565io_sigpending:
566 # TRACE_IRQS_ON already done at io_return
567 l %r1,BASED(.Ldo_signal)
568 ssm __LC_SVC_NEW_PSW # reenable interrupts
569 lr %r2,%r11 # pass pointer to pt_regs
570 basr %r14,%r1 # call do_signal
571 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
572 TRACE_IRQS_OFF
573 j io_return
574
575#
576# _TIF_SIGPENDING is set, call do_signal
577#
578io_notify_resume:
579 # TRACE_IRQS_ON already done at io_return
580 l %r1,BASED(.Ldo_notify_resume)
581 ssm __LC_SVC_NEW_PSW # reenable interrupts
582 lr %r2,%r11 # pass pointer to pt_regs
583 basr %r14,%r1 # call do_notify_resume
584 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
585 TRACE_IRQS_OFF
586 j io_return
587
588/*
589 * External interrupt handler routine
590 */
591
592ENTRY(ext_int_handler)
593 stck __LC_INT_CLOCK
594 stpt __LC_ASYNC_ENTER_TIMER
595 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
596 l %r12,__LC_THREAD_INFO
597 l %r13,__LC_SVC_NEW_PSW+4
598 lm %r8,%r9,__LC_EXT_OLD_PSW
599 tmh %r8,0x0001 # interrupting from user ?
600 jz ext_skip
601 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
602ext_skip:
603 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
604 stm %r0,%r7,__PT_R0(%r11)
605 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
606 stm %r8,%r9,__PT_PSW(%r11)
607 TRACE_IRQS_OFF
608 lr %r2,%r11 # pass pointer to pt_regs
609 l %r3,__LC_EXT_CPU_ADDR # get cpu address + interruption code
610 l %r4,__LC_EXT_PARAMS # get external parameters
611 l %r1,BASED(.Ldo_extint)
612 basr %r14,%r1 # call do_extint
613 j io_return
614
615/*
616 * Load idle PSW. The second "half" of this function is in cleanup_idle.
617 */
618ENTRY(psw_idle)
619 st %r4,__SF_EMPTY(%r15)
620 basr %r1,0
621 la %r1,psw_idle_lpsw+4-.(%r1)
622 st %r1,__SF_EMPTY+4(%r15)
623 oi __SF_EMPTY+4(%r15),0x80
624 la %r1,.Lvtimer_max-psw_idle_lpsw-4(%r1)
625 stck __IDLE_ENTER(%r2)
626 ltr %r5,%r5
627 stpt __VQ_IDLE_ENTER(%r3)
628 jz psw_idle_lpsw
629 spt 0(%r1)
630psw_idle_lpsw:
631 lpsw __SF_EMPTY(%r15)
632 br %r14
633psw_idle_end:
634
635__critical_end:
636
637/*
638 * Machine check handler routines
639 */
640
641ENTRY(mcck_int_handler)
642 stck __LC_MCCK_CLOCK
643 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer
644 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs
645 l %r12,__LC_THREAD_INFO
646 l %r13,__LC_SVC_NEW_PSW+4
647 lm %r8,%r9,__LC_MCK_OLD_PSW
648 tm __LC_MCCK_CODE,0x80 # system damage?
649 jo mcck_panic # yes -> rest of mcck code invalid
650 la %r14,__LC_CPU_TIMER_SAVE_AREA
651 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
652 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
653 jo 3f
654 la %r14,__LC_SYNC_ENTER_TIMER
655 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
656 jl 0f
657 la %r14,__LC_ASYNC_ENTER_TIMER
6580: clc 0(8,%r14),__LC_EXIT_TIMER
659 jl 1f
660 la %r14,__LC_EXIT_TIMER
6611: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
662 jl 2f
663 la %r14,__LC_LAST_UPDATE_TIMER
6642: spt 0(%r14)
665 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
6663: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
667 jno mcck_panic # no -> skip cleanup critical
668 tm %r8,0x0001 # interrupting from user ?
669 jz mcck_skip
670 UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER
671mcck_skip:
672 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT
673 mvc __PT_R0(64,%r11),__LC_GPREGS_SAVE_AREA
674 stm %r8,%r9,__PT_PSW(%r11)
675 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
676 l %r1,BASED(.Ldo_machine_check)
677 lr %r2,%r11 # pass pointer to pt_regs
678 basr %r14,%r1 # call s390_do_machine_check
679 tm __PT_PSW+1(%r11),0x01 # returning to user ?
680 jno mcck_return
681 l %r1,__LC_KERNEL_STACK # switch to kernel stack
682 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
683 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
684 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
685 la %r11,STACK_FRAME_OVERHEAD(%r15)
686 lr %r15,%r1
687 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
688 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
689 jno mcck_return
690 TRACE_IRQS_OFF
691 l %r1,BASED(.Lhandle_mcck)
692 basr %r14,%r1 # call s390_handle_mcck
693 TRACE_IRQS_ON
694mcck_return:
695 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW
696 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
697 jno 0f
698 lm %r0,%r15,__PT_R0(%r11)
699 stpt __LC_EXIT_TIMER
700 lpsw __LC_RETURN_MCCK_PSW
7010: lm %r0,%r15,__PT_R0(%r11)
702 lpsw __LC_RETURN_MCCK_PSW
703
704mcck_panic:
705 l %r14,__LC_PANIC_STACK
706 slr %r14,%r15
707 sra %r14,PAGE_SHIFT
708 jz 0f
709 l %r15,__LC_PANIC_STACK
7100: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
711 j mcck_skip
712
713#
714# PSW restart interrupt handler
715#
716ENTRY(restart_int_handler)
717 st %r15,__LC_SAVE_AREA_RESTART
718 l %r15,__LC_RESTART_STACK
719 ahi %r15,-__PT_SIZE # create pt_regs on stack
720 xc 0(__PT_SIZE,%r15),0(%r15)
721 stm %r0,%r14,__PT_R0(%r15)
722 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART
723 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw
724 ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
725 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
726 lm %r1,%r3,__LC_RESTART_FN # load fn, parm & source cpu
727 ltr %r3,%r3 # test source cpu address
728 jm 1f # negative -> skip source stop
7290: sigp %r4,%r3,1 # sigp sense to source cpu
730 brc 10,0b # wait for status stored
7311: basr %r14,%r1 # call function
732 stap __SF_EMPTY(%r15) # store cpu address
733 lh %r3,__SF_EMPTY(%r15)
7342: sigp %r4,%r3,5 # sigp stop to current cpu
735 brc 2,2b
7363: j 3b
737
738 .section .kprobes.text, "ax"
739
740#ifdef CONFIG_CHECK_STACK
741/*
742 * The synchronous or the asynchronous stack overflowed. We are dead.
743 * No need to properly save the registers, we are going to panic anyway.
744 * Setup a pt_regs so that show_trace can provide a good call trace.
745 */
746stack_overflow:
747 l %r15,__LC_PANIC_STACK # change to panic stack
748 ahi %r15,-__PT_SIZE # create pt_regs
749 stm %r0,%r7,__PT_R0(%r15)
750 stm %r8,%r9,__PT_PSW(%r15)
751 mvc __PT_R8(32,%r11),0(%r14)
752 lr %r15,%r11
753 ahi %r15,-STACK_FRAME_OVERHEAD
754 l %r1,BASED(1f)
755 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
756 lr %r2,%r11 # pass pointer to pt_regs
757 br %r1 # branch to kernel_stack_overflow
7581: .long kernel_stack_overflow
759#endif
760
761cleanup_table:
762 .long system_call + 0x80000000
763 .long sysc_do_svc + 0x80000000
764 .long sysc_tif + 0x80000000
765 .long sysc_restore + 0x80000000
766 .long sysc_done + 0x80000000
767 .long io_tif + 0x80000000
768 .long io_restore + 0x80000000
769 .long io_done + 0x80000000
770 .long psw_idle + 0x80000000
771 .long psw_idle_end + 0x80000000
772
773cleanup_critical:
774 cl %r9,BASED(cleanup_table) # system_call
775 jl 0f
776 cl %r9,BASED(cleanup_table+4) # sysc_do_svc
777 jl cleanup_system_call
778 cl %r9,BASED(cleanup_table+8) # sysc_tif
779 jl 0f
780 cl %r9,BASED(cleanup_table+12) # sysc_restore
781 jl cleanup_sysc_tif
782 cl %r9,BASED(cleanup_table+16) # sysc_done
783 jl cleanup_sysc_restore
784 cl %r9,BASED(cleanup_table+20) # io_tif
785 jl 0f
786 cl %r9,BASED(cleanup_table+24) # io_restore
787 jl cleanup_io_tif
788 cl %r9,BASED(cleanup_table+28) # io_done
789 jl cleanup_io_restore
790 cl %r9,BASED(cleanup_table+32) # psw_idle
791 jl 0f
792 cl %r9,BASED(cleanup_table+36) # psw_idle_end
793 jl cleanup_idle
7940: br %r14
795
796cleanup_system_call:
797 # check if stpt has been executed
798 cl %r9,BASED(cleanup_system_call_insn)
799 jh 0f
800 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
801 chi %r11,__LC_SAVE_AREA_ASYNC
802 je 0f
803 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
8040: # check if stm has been executed
805 cl %r9,BASED(cleanup_system_call_insn+4)
806 jh 0f
807 mvc __LC_SAVE_AREA_SYNC(32),0(%r11)
8080: # set up saved registers r12, and r13
809 st %r12,16(%r11) # r12 thread-info pointer
810 st %r13,20(%r11) # r13 literal-pool pointer
811 # check if the user time calculation has been done
812 cl %r9,BASED(cleanup_system_call_insn+8)
813 jh 0f
814 l %r10,__LC_EXIT_TIMER
815 l %r15,__LC_EXIT_TIMER+4
816 SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER
817 ADD64 %r10,%r15,__LC_USER_TIMER
818 st %r10,__LC_USER_TIMER
819 st %r15,__LC_USER_TIMER+4
8200: # check if the system time calculation has been done
821 cl %r9,BASED(cleanup_system_call_insn+12)
822 jh 0f
823 l %r10,__LC_LAST_UPDATE_TIMER
824 l %r15,__LC_LAST_UPDATE_TIMER+4
825 SUB64 %r10,%r15,__LC_EXIT_TIMER
826 ADD64 %r10,%r15,__LC_SYSTEM_TIMER
827 st %r10,__LC_SYSTEM_TIMER
828 st %r15,__LC_SYSTEM_TIMER+4
8290: # update accounting time stamp
830 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
831 # set up saved register 11
832 l %r15,__LC_KERNEL_STACK
833 ahi %r15,-__PT_SIZE
834 st %r15,12(%r11) # r11 pt_regs pointer
835 # fill pt_regs
836 mvc __PT_R8(32,%r15),__LC_SAVE_AREA_SYNC
837 stm %r0,%r7,__PT_R0(%r15)
838 mvc __PT_PSW(8,%r15),__LC_SVC_OLD_PSW
839 mvc __PT_INT_CODE(4,%r15),__LC_SVC_ILC
840 # setup saved register 15
841 ahi %r15,-STACK_FRAME_OVERHEAD
842 st %r15,28(%r11) # r15 stack pointer
843 # set new psw address and exit
844 l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000
845 br %r14
846cleanup_system_call_insn:
847 .long system_call + 0x80000000
848 .long sysc_stm + 0x80000000
849 .long sysc_vtime + 0x80000000 + 36
850 .long sysc_vtime + 0x80000000 + 76
851
852cleanup_sysc_tif:
853 l %r9,BASED(cleanup_table+8) # sysc_tif + 0x80000000
854 br %r14
855
856cleanup_sysc_restore:
857 cl %r9,BASED(cleanup_sysc_restore_insn)
858 jhe 0f
859 l %r9,12(%r11) # get saved pointer to pt_regs
860 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
861 mvc 0(32,%r11),__PT_R8(%r9)
862 lm %r0,%r7,__PT_R0(%r9)
8630: lm %r8,%r9,__LC_RETURN_PSW
864 br %r14
865cleanup_sysc_restore_insn:
866 .long sysc_done - 4 + 0x80000000
867
868cleanup_io_tif:
869 l %r9,BASED(cleanup_table+20) # io_tif + 0x80000000
870 br %r14
871
872cleanup_io_restore:
873 cl %r9,BASED(cleanup_io_restore_insn)
874 jhe 0f
875 l %r9,12(%r11) # get saved r11 pointer to pt_regs
876 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
877 mvc 0(32,%r11),__PT_R8(%r9)
878 lm %r0,%r7,__PT_R0(%r9)
8790: lm %r8,%r9,__LC_RETURN_PSW
880 br %r14
881cleanup_io_restore_insn:
882 .long io_done - 4 + 0x80000000
883
884cleanup_idle:
885 # copy interrupt clock & cpu timer
886 mvc __IDLE_EXIT(8,%r2),__LC_INT_CLOCK
887 mvc __VQ_IDLE_EXIT(8,%r3),__LC_ASYNC_ENTER_TIMER
888 chi %r11,__LC_SAVE_AREA_ASYNC
889 je 0f
890 mvc __IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
891 mvc __VQ_IDLE_EXIT(8,%r3),__LC_MCCK_ENTER_TIMER
8920: # check if stck has been executed
893 cl %r9,BASED(cleanup_idle_insn)
894 jhe 1f
895 mvc __IDLE_ENTER(8,%r2),__IDLE_EXIT(%r2)
896 mvc __VQ_IDLE_ENTER(8,%r3),__VQ_IDLE_EXIT(%r3)
897 j 2f
8981: # check if the cpu timer has been reprogrammed
899 ltr %r5,%r5
900 jz 2f
901 spt __VQ_IDLE_ENTER(%r3)
9022: # account system time going idle
903 lm %r9,%r10,__LC_STEAL_TIMER
904 ADD64 %r9,%r10,__IDLE_ENTER(%r2)
905 SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK
906 stm %r9,%r10,__LC_STEAL_TIMER
907 mvc __LC_LAST_UPDATE_CLOCK(8),__IDLE_EXIT(%r2)
908 lm %r9,%r10,__LC_SYSTEM_TIMER
909 ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER
910 SUB64 %r9,%r10,__VQ_IDLE_ENTER(%r3)
911 stm %r9,%r10,__LC_SYSTEM_TIMER
912 mvc __LC_LAST_UPDATE_TIMER(8),__VQ_IDLE_EXIT(%r3)
913 # prepare return psw
914 n %r8,BASED(cleanup_idle_wait) # clear wait state bit
915 l %r9,24(%r11) # return from psw_idle
916 br %r14
917cleanup_idle_insn:
918 .long psw_idle_lpsw + 0x80000000
919cleanup_idle_wait:
920 .long 0xfffdffff
921
922/*
923 * Integer constants
924 */
925 .align 4
926.Lnr_syscalls:
927 .long NR_syscalls
928.Lvtimer_max:
929 .quad 0x7fffffffffffffff
930
931/*
932 * Symbol constants
933 */
934.Ldo_machine_check: .long s390_do_machine_check
935.Lhandle_mcck: .long s390_handle_mcck
936.Ldo_IRQ: .long do_IRQ
937.Ldo_extint: .long do_extint
938.Ldo_signal: .long do_signal
939.Ldo_notify_resume: .long do_notify_resume
940.Ldo_per_trap: .long do_per_trap
941.Ldo_execve: .long do_execve
942.Lexecve_tail: .long execve_tail
943.Ljump_table: .long pgm_check_table
944.Lschedule: .long schedule
945#ifdef CONFIG_PREEMPT
946.Lpreempt_irq: .long preempt_schedule_irq
947#endif
948.Ltrace_enter: .long do_syscall_trace_enter
949.Ltrace_exit: .long do_syscall_trace_exit
950.Lschedule_tail: .long schedule_tail
951.Lsys_call_table: .long sys_call_table
952.Lsysc_per: .long sysc_per + 0x80000000
953#ifdef CONFIG_TRACE_IRQFLAGS
954.Lhardirqs_on: .long trace_hardirqs_on_caller
955.Lhardirqs_off: .long trace_hardirqs_off_caller
956#endif
957#ifdef CONFIG_LOCKDEP
958.Llockdep_sys_exit: .long lockdep_sys_exit
959#endif
960.Lcritical_start: .long __critical_start + 0x80000000
961.Lcritical_length: .long __critical_end - __critical_start
962
963 .section .rodata, "a"
964#define SYSCALL(esa,esame,emu) .long esa
965 .globl sys_call_table
966sys_call_table:
967#include "syscalls.S"
968#undef SYSCALL