Loading...
1/*
2 * arch/s390/kernel/entry.S
3 * S390 low-level entry points.
4 *
5 * Copyright (C) IBM Corp. 1999,2006
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 */
11
12#include <linux/init.h>
13#include <linux/linkage.h>
14#include <asm/cache.h>
15#include <asm/errno.h>
16#include <asm/ptrace.h>
17#include <asm/thread_info.h>
18#include <asm/asm-offsets.h>
19#include <asm/unistd.h>
20#include <asm/page.h>
21
22/*
23 * Stack layout for the system_call stack entry.
24 * The first few entries are identical to the user_regs_struct.
25 */
26SP_PTREGS = STACK_FRAME_OVERHEAD
27SP_ARGS = STACK_FRAME_OVERHEAD + __PT_ARGS
28SP_PSW = STACK_FRAME_OVERHEAD + __PT_PSW
29SP_R0 = STACK_FRAME_OVERHEAD + __PT_GPRS
30SP_R1 = STACK_FRAME_OVERHEAD + __PT_GPRS + 4
31SP_R2 = STACK_FRAME_OVERHEAD + __PT_GPRS + 8
32SP_R3 = STACK_FRAME_OVERHEAD + __PT_GPRS + 12
33SP_R4 = STACK_FRAME_OVERHEAD + __PT_GPRS + 16
34SP_R5 = STACK_FRAME_OVERHEAD + __PT_GPRS + 20
35SP_R6 = STACK_FRAME_OVERHEAD + __PT_GPRS + 24
36SP_R7 = STACK_FRAME_OVERHEAD + __PT_GPRS + 28
37SP_R8 = STACK_FRAME_OVERHEAD + __PT_GPRS + 32
38SP_R9 = STACK_FRAME_OVERHEAD + __PT_GPRS + 36
39SP_R10 = STACK_FRAME_OVERHEAD + __PT_GPRS + 40
40SP_R11 = STACK_FRAME_OVERHEAD + __PT_GPRS + 44
41SP_R12 = STACK_FRAME_OVERHEAD + __PT_GPRS + 48
42SP_R13 = STACK_FRAME_OVERHEAD + __PT_GPRS + 52
43SP_R14 = STACK_FRAME_OVERHEAD + __PT_GPRS + 56
44SP_R15 = STACK_FRAME_OVERHEAD + __PT_GPRS + 60
45SP_ORIG_R2 = STACK_FRAME_OVERHEAD + __PT_ORIG_GPR2
46SP_ILC = STACK_FRAME_OVERHEAD + __PT_ILC
47SP_SVCNR = STACK_FRAME_OVERHEAD + __PT_SVCNR
48SP_SIZE = STACK_FRAME_OVERHEAD + __PT_SIZE
49
50_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
51 _TIF_MCCK_PENDING | _TIF_RESTART_SVC | _TIF_PER_TRAP )
52_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
53 _TIF_MCCK_PENDING)
54_TIF_SYSCALL = (_TIF_SYSCALL_TRACE>>8 | _TIF_SYSCALL_AUDIT>>8 | \
55 _TIF_SECCOMP>>8 | _TIF_SYSCALL_TRACEPOINT>>8)
56
57STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
58STACK_SIZE = 1 << STACK_SHIFT
59
60#define BASED(name) name-system_call(%r13)
61
62#ifdef CONFIG_TRACE_IRQFLAGS
63 .macro TRACE_IRQS_ON
64 basr %r2,%r0
65 l %r1,BASED(.Ltrace_irq_on_caller)
66 basr %r14,%r1
67 .endm
68
69 .macro TRACE_IRQS_OFF
70 basr %r2,%r0
71 l %r1,BASED(.Ltrace_irq_off_caller)
72 basr %r14,%r1
73 .endm
74#else
75#define TRACE_IRQS_ON
76#define TRACE_IRQS_OFF
77#endif
78
79#ifdef CONFIG_LOCKDEP
80 .macro LOCKDEP_SYS_EXIT
81 tm SP_PSW+1(%r15),0x01 # returning to user ?
82 jz 0f
83 l %r1,BASED(.Llockdep_sys_exit)
84 basr %r14,%r1
850:
86 .endm
87#else
88#define LOCKDEP_SYS_EXIT
89#endif
90
91/*
92 * Register usage in interrupt handlers:
93 * R9 - pointer to current task structure
94 * R13 - pointer to literal pool
95 * R14 - return register for function calls
96 * R15 - kernel stack pointer
97 */
98
99 .macro UPDATE_VTIME lc_from,lc_to,lc_sum
100 lm %r10,%r11,\lc_from
101 sl %r10,\lc_to
102 sl %r11,\lc_to+4
103 bc 3,BASED(0f)
104 sl %r10,BASED(.Lc_1)
1050: al %r10,\lc_sum
106 al %r11,\lc_sum+4
107 bc 12,BASED(1f)
108 al %r10,BASED(.Lc_1)
1091: stm %r10,%r11,\lc_sum
110 .endm
111
112 .macro SAVE_ALL_SVC psworg,savearea
113 stm %r12,%r15,\savearea
114 l %r13,__LC_SVC_NEW_PSW+4 # load &system_call to %r13
115 l %r15,__LC_KERNEL_STACK # problem state -> load ksp
116 s %r15,BASED(.Lc_spsize) # make room for registers & psw
117 .endm
118
119 .macro SAVE_ALL_BASE savearea
120 stm %r12,%r15,\savearea
121 l %r13,__LC_SVC_NEW_PSW+4 # load &system_call to %r13
122 .endm
123
124 .macro SAVE_ALL_PGM psworg,savearea
125 tm \psworg+1,0x01 # test problem state bit
126#ifdef CONFIG_CHECK_STACK
127 bnz BASED(1f)
128 tml %r15,STACK_SIZE - CONFIG_STACK_GUARD
129 bnz BASED(2f)
130 la %r12,\psworg
131 b BASED(stack_overflow)
132#else
133 bz BASED(2f)
134#endif
1351: l %r15,__LC_KERNEL_STACK # problem state -> load ksp
1362: s %r15,BASED(.Lc_spsize) # make room for registers & psw
137 .endm
138
139 .macro SAVE_ALL_ASYNC psworg,savearea
140 stm %r12,%r15,\savearea
141 l %r13,__LC_SVC_NEW_PSW+4 # load &system_call to %r13
142 la %r12,\psworg
143 tm \psworg+1,0x01 # test problem state bit
144 bnz BASED(1f) # from user -> load async stack
145 clc \psworg+4(4),BASED(.Lcritical_end)
146 bhe BASED(0f)
147 clc \psworg+4(4),BASED(.Lcritical_start)
148 bl BASED(0f)
149 l %r14,BASED(.Lcleanup_critical)
150 basr %r14,%r14
151 tm 1(%r12),0x01 # retest problem state after cleanup
152 bnz BASED(1f)
1530: l %r14,__LC_ASYNC_STACK # are we already on the async stack ?
154 slr %r14,%r15
155 sra %r14,STACK_SHIFT
156#ifdef CONFIG_CHECK_STACK
157 bnz BASED(1f)
158 tml %r15,STACK_SIZE - CONFIG_STACK_GUARD
159 bnz BASED(2f)
160 b BASED(stack_overflow)
161#else
162 bz BASED(2f)
163#endif
1641: l %r15,__LC_ASYNC_STACK
1652: s %r15,BASED(.Lc_spsize) # make room for registers & psw
166 .endm
167
168 .macro CREATE_STACK_FRAME savearea
169 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
170 st %r2,SP_ORIG_R2(%r15) # store original content of gpr 2
171 mvc SP_R12(16,%r15),\savearea # move %r12-%r15 to stack
172 stm %r0,%r11,SP_R0(%r15) # store gprs %r0-%r11 to kernel stack
173 .endm
174
175 .macro RESTORE_ALL psworg,sync
176 mvc \psworg(8),SP_PSW(%r15) # move user PSW to lowcore
177 .if !\sync
178 ni \psworg+1,0xfd # clear wait state bit
179 .endif
180 lm %r0,%r15,SP_R0(%r15) # load gprs 0-15 of user
181 stpt __LC_EXIT_TIMER
182 lpsw \psworg # back to caller
183 .endm
184
185 .macro REENABLE_IRQS
186 mvc __SF_EMPTY(1,%r15),SP_PSW(%r15)
187 ni __SF_EMPTY(%r15),0xbf
188 ssm __SF_EMPTY(%r15)
189 .endm
190
191 .section .kprobes.text, "ax"
192
193/*
194 * Scheduler resume function, called by switch_to
195 * gpr2 = (task_struct *) prev
196 * gpr3 = (task_struct *) next
197 * Returns:
198 * gpr2 = prev
199 */
200ENTRY(__switch_to)
201 basr %r1,0
2020: l %r4,__THREAD_info(%r2) # get thread_info of prev
203 l %r5,__THREAD_info(%r3) # get thread_info of next
204 tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending?
205 bz 1f-0b(%r1)
206 ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev
207 oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next
2081: stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
209 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev
210 l %r15,__THREAD_ksp(%r3) # load kernel stack of next
211 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
212 lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
213 st %r3,__LC_CURRENT # store task struct of next
214 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
215 st %r5,__LC_THREAD_INFO # store thread info of next
216 ahi %r5,STACK_SIZE # end of kernel stack of next
217 st %r5,__LC_KERNEL_STACK # store end of kernel stack
218 br %r14
219
220__critical_start:
221/*
222 * SVC interrupt handler routine. System calls are synchronous events and
223 * are executed with interrupts enabled.
224 */
225
226ENTRY(system_call)
227 stpt __LC_SYNC_ENTER_TIMER
228sysc_saveall:
229 SAVE_ALL_SVC __LC_SVC_OLD_PSW,__LC_SAVE_AREA
230 CREATE_STACK_FRAME __LC_SAVE_AREA
231 mvc SP_PSW(8,%r15),__LC_SVC_OLD_PSW
232 mvc SP_ILC(4,%r15),__LC_SVC_ILC
233 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
234sysc_vtime:
235 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
236sysc_stime:
237 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
238sysc_update:
239 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
240sysc_do_svc:
241 xr %r7,%r7
242 icm %r7,3,SP_SVCNR(%r15) # load svc number and test for svc 0
243 bnz BASED(sysc_nr_ok) # svc number > 0
244 # svc 0: system call number in %r1
245 cl %r1,BASED(.Lnr_syscalls)
246 bnl BASED(sysc_nr_ok)
247 sth %r1,SP_SVCNR(%r15)
248 lr %r7,%r1 # copy svc number to %r7
249sysc_nr_ok:
250 sll %r7,2 # svc number *4
251 l %r10,BASED(.Lsysc_table)
252 tm __TI_flags+2(%r12),_TIF_SYSCALL
253 mvc SP_ARGS(4,%r15),SP_R7(%r15)
254 l %r8,0(%r7,%r10) # get system call addr.
255 bnz BASED(sysc_tracesys)
256 basr %r14,%r8 # call sys_xxxx
257 st %r2,SP_R2(%r15) # store return value (change R2 on stack)
258
259sysc_return:
260 LOCKDEP_SYS_EXIT
261sysc_tif:
262 tm __TI_flags+3(%r12),_TIF_WORK_SVC
263 bnz BASED(sysc_work) # there is work to do (signals etc.)
264sysc_restore:
265 RESTORE_ALL __LC_RETURN_PSW,1
266sysc_done:
267
268#
269# There is work to do, but first we need to check if we return to userspace.
270#
271sysc_work:
272 tm SP_PSW+1(%r15),0x01 # returning to user ?
273 bno BASED(sysc_restore)
274
275#
276# One of the work bits is on. Find out which one.
277#
278sysc_work_tif:
279 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
280 bo BASED(sysc_mcck_pending)
281 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
282 bo BASED(sysc_reschedule)
283 tm __TI_flags+3(%r12),_TIF_SIGPENDING
284 bo BASED(sysc_sigpending)
285 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
286 bo BASED(sysc_notify_resume)
287 tm __TI_flags+3(%r12),_TIF_RESTART_SVC
288 bo BASED(sysc_restart)
289 tm __TI_flags+3(%r12),_TIF_PER_TRAP
290 bo BASED(sysc_singlestep)
291 b BASED(sysc_return) # beware of critical section cleanup
292
293#
294# _TIF_NEED_RESCHED is set, call schedule
295#
296sysc_reschedule:
297 l %r1,BASED(.Lschedule)
298 la %r14,BASED(sysc_return)
299 br %r1 # call scheduler
300
301#
302# _TIF_MCCK_PENDING is set, call handler
303#
304sysc_mcck_pending:
305 l %r1,BASED(.Ls390_handle_mcck)
306 la %r14,BASED(sysc_return)
307 br %r1 # TIF bit will be cleared by handler
308
309#
310# _TIF_SIGPENDING is set, call do_signal
311#
312sysc_sigpending:
313 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP
314 la %r2,SP_PTREGS(%r15) # load pt_regs
315 l %r1,BASED(.Ldo_signal)
316 basr %r14,%r1 # call do_signal
317 tm __TI_flags+3(%r12),_TIF_RESTART_SVC
318 bo BASED(sysc_restart)
319 tm __TI_flags+3(%r12),_TIF_PER_TRAP
320 bo BASED(sysc_singlestep)
321 b BASED(sysc_return)
322
323#
324# _TIF_NOTIFY_RESUME is set, call do_notify_resume
325#
326sysc_notify_resume:
327 la %r2,SP_PTREGS(%r15) # load pt_regs
328 l %r1,BASED(.Ldo_notify_resume)
329 la %r14,BASED(sysc_return)
330 br %r1 # call do_notify_resume
331
332
333#
334# _TIF_RESTART_SVC is set, set up registers and restart svc
335#
336sysc_restart:
337 ni __TI_flags+3(%r12),255-_TIF_RESTART_SVC # clear TIF_RESTART_SVC
338 l %r7,SP_R2(%r15) # load new svc number
339 mvc SP_R2(4,%r15),SP_ORIG_R2(%r15) # restore first argument
340 lm %r2,%r6,SP_R2(%r15) # load svc arguments
341 sth %r7,SP_SVCNR(%r15)
342 b BASED(sysc_nr_ok) # restart svc
343
344#
345# _TIF_PER_TRAP is set, call do_per_trap
346#
347sysc_singlestep:
348 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP
349 xc SP_SVCNR(2,%r15),SP_SVCNR(%r15) # clear svc number
350 la %r2,SP_PTREGS(%r15) # address of register-save area
351 l %r1,BASED(.Lhandle_per) # load adr. of per handler
352 la %r14,BASED(sysc_return) # load adr. of system return
353 br %r1 # branch to do_per_trap
354
355#
356# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
357# and after the system call
358#
359sysc_tracesys:
360 l %r1,BASED(.Ltrace_entry)
361 la %r2,SP_PTREGS(%r15) # load pt_regs
362 la %r3,0
363 xr %r0,%r0
364 icm %r0,3,SP_SVCNR(%r15)
365 st %r0,SP_R2(%r15)
366 basr %r14,%r1
367 cl %r2,BASED(.Lnr_syscalls)
368 bnl BASED(sysc_tracenogo)
369 lr %r7,%r2
370 sll %r7,2 # svc number *4
371 l %r8,0(%r7,%r10)
372sysc_tracego:
373 lm %r3,%r6,SP_R3(%r15)
374 mvc SP_ARGS(4,%r15),SP_R7(%r15)
375 l %r2,SP_ORIG_R2(%r15)
376 basr %r14,%r8 # call sys_xxx
377 st %r2,SP_R2(%r15) # store return value
378sysc_tracenogo:
379 tm __TI_flags+2(%r12),_TIF_SYSCALL
380 bz BASED(sysc_return)
381 l %r1,BASED(.Ltrace_exit)
382 la %r2,SP_PTREGS(%r15) # load pt_regs
383 la %r14,BASED(sysc_return)
384 br %r1
385
386#
387# a new process exits the kernel with ret_from_fork
388#
389ENTRY(ret_from_fork)
390 l %r13,__LC_SVC_NEW_PSW+4
391 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
392 tm SP_PSW+1(%r15),0x01 # forking a kernel thread ?
393 bo BASED(0f)
394 st %r15,SP_R15(%r15) # store stack pointer for new kthread
3950: l %r1,BASED(.Lschedtail)
396 basr %r14,%r1
397 TRACE_IRQS_ON
398 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
399 b BASED(sysc_tracenogo)
400
401#
402# kernel_execve function needs to deal with pt_regs that is not
403# at the usual place
404#
405ENTRY(kernel_execve)
406 stm %r12,%r15,48(%r15)
407 lr %r14,%r15
408 l %r13,__LC_SVC_NEW_PSW+4
409 s %r15,BASED(.Lc_spsize)
410 st %r14,__SF_BACKCHAIN(%r15)
411 la %r12,SP_PTREGS(%r15)
412 xc 0(__PT_SIZE,%r12),0(%r12)
413 l %r1,BASED(.Ldo_execve)
414 lr %r5,%r12
415 basr %r14,%r1
416 ltr %r2,%r2
417 be BASED(0f)
418 a %r15,BASED(.Lc_spsize)
419 lm %r12,%r15,48(%r15)
420 br %r14
421 # execve succeeded.
4220: stnsm __SF_EMPTY(%r15),0xfc # disable interrupts
423 l %r15,__LC_KERNEL_STACK # load ksp
424 s %r15,BASED(.Lc_spsize) # make room for registers & psw
425 mvc SP_PTREGS(__PT_SIZE,%r15),0(%r12) # copy pt_regs
426 l %r12,__LC_THREAD_INFO
427 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
428 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
429 l %r1,BASED(.Lexecve_tail)
430 basr %r14,%r1
431 b BASED(sysc_return)
432
433/*
434 * Program check handler routine
435 */
436
437ENTRY(pgm_check_handler)
438/*
439 * First we need to check for a special case:
440 * Single stepping an instruction that disables the PER event mask will
441 * cause a PER event AFTER the mask has been set. Example: SVC or LPSW.
442 * For a single stepped SVC the program check handler gets control after
443 * the SVC new PSW has been loaded. But we want to execute the SVC first and
444 * then handle the PER event. Therefore we update the SVC old PSW to point
445 * to the pgm_check_handler and branch to the SVC handler after we checked
446 * if we have to load the kernel stack register.
447 * For every other possible cause for PER event without the PER mask set
448 * we just ignore the PER event (FIXME: is there anything we have to do
449 * for LPSW?).
450 */
451 stpt __LC_SYNC_ENTER_TIMER
452 SAVE_ALL_BASE __LC_SAVE_AREA
453 tm __LC_PGM_INT_CODE+1,0x80 # check whether we got a per exception
454 bnz BASED(pgm_per) # got per exception -> special case
455 SAVE_ALL_PGM __LC_PGM_OLD_PSW,__LC_SAVE_AREA
456 CREATE_STACK_FRAME __LC_SAVE_AREA
457 xc SP_ILC(4,%r15),SP_ILC(%r15)
458 mvc SP_PSW(8,%r15),__LC_PGM_OLD_PSW
459 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
460 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
461 bz BASED(pgm_no_vtime)
462 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
463 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
464 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
465pgm_no_vtime:
466 l %r3,__LC_PGM_ILC # load program interruption code
467 l %r4,__LC_TRANS_EXC_CODE
468 REENABLE_IRQS
469 la %r8,0x7f
470 nr %r8,%r3
471 sll %r8,2
472 l %r1,BASED(.Ljump_table)
473 l %r1,0(%r8,%r1) # load address of handler routine
474 la %r2,SP_PTREGS(%r15) # address of register-save area
475 basr %r14,%r1 # branch to interrupt-handler
476pgm_exit:
477 b BASED(sysc_return)
478
479#
480# handle per exception
481#
482pgm_per:
483 tm __LC_PGM_OLD_PSW,0x40 # test if per event recording is on
484 bnz BASED(pgm_per_std) # ok, normal per event from user space
485# ok its one of the special cases, now we need to find out which one
486 clc __LC_PGM_OLD_PSW(8),__LC_SVC_NEW_PSW
487 be BASED(pgm_svcper)
488# no interesting special case, ignore PER event
489 lm %r12,%r15,__LC_SAVE_AREA
490 lpsw 0x28
491
492#
493# Normal per exception
494#
495pgm_per_std:
496 SAVE_ALL_PGM __LC_PGM_OLD_PSW,__LC_SAVE_AREA
497 CREATE_STACK_FRAME __LC_SAVE_AREA
498 mvc SP_PSW(8,%r15),__LC_PGM_OLD_PSW
499 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
500 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
501 bz BASED(pgm_no_vtime2)
502 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
503 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
504 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
505pgm_no_vtime2:
506 l %r1,__TI_task(%r12)
507 tm SP_PSW+1(%r15),0x01 # kernel per event ?
508 bz BASED(kernel_per)
509 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE
510 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS
511 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID
512 oi __TI_flags+3(%r12),_TIF_PER_TRAP # set TIF_PER_TRAP
513 l %r3,__LC_PGM_ILC # load program interruption code
514 l %r4,__LC_TRANS_EXC_CODE
515 REENABLE_IRQS
516 la %r8,0x7f
517 nr %r8,%r3 # clear per-event-bit and ilc
518 be BASED(pgm_exit2) # only per or per+check ?
519 sll %r8,2
520 l %r1,BASED(.Ljump_table)
521 l %r1,0(%r8,%r1) # load address of handler routine
522 la %r2,SP_PTREGS(%r15) # address of register-save area
523 basr %r14,%r1 # branch to interrupt-handler
524pgm_exit2:
525 b BASED(sysc_return)
526
527#
528# it was a single stepped SVC that is causing all the trouble
529#
530pgm_svcper:
531 SAVE_ALL_PGM __LC_SVC_OLD_PSW,__LC_SAVE_AREA
532 CREATE_STACK_FRAME __LC_SAVE_AREA
533 mvc SP_PSW(8,%r15),__LC_SVC_OLD_PSW
534 mvc SP_ILC(4,%r15),__LC_SVC_ILC
535 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
536 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
537 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
538 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
539 l %r8,__TI_task(%r12)
540 mvc __THREAD_per_cause(2,%r8),__LC_PER_CAUSE
541 mvc __THREAD_per_address(4,%r8),__LC_PER_ADDRESS
542 mvc __THREAD_per_paid(1,%r8),__LC_PER_PAID
543 oi __TI_flags+3(%r12),_TIF_PER_TRAP # set TIF_PER_TRAP
544 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
545 lm %r2,%r6,SP_R2(%r15) # load svc arguments
546 b BASED(sysc_do_svc)
547
548#
549# per was called from kernel, must be kprobes
550#
551kernel_per:
552 REENABLE_IRQS
553 xc SP_SVCNR(2,%r15),SP_SVCNR(%r15)
554 la %r2,SP_PTREGS(%r15) # address of register-save area
555 l %r1,BASED(.Lhandle_per) # load adr. of per handler
556 basr %r14,%r1 # branch to do_single_step
557 b BASED(pgm_exit)
558
559/*
560 * IO interrupt handler routine
561 */
562
563ENTRY(io_int_handler)
564 stck __LC_INT_CLOCK
565 stpt __LC_ASYNC_ENTER_TIMER
566 SAVE_ALL_ASYNC __LC_IO_OLD_PSW,__LC_SAVE_AREA+16
567 CREATE_STACK_FRAME __LC_SAVE_AREA+16
568 mvc SP_PSW(8,%r15),0(%r12) # move user PSW to stack
569 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
570 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
571 bz BASED(io_no_vtime)
572 UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
573 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
574 mvc __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
575io_no_vtime:
576 TRACE_IRQS_OFF
577 l %r1,BASED(.Ldo_IRQ) # load address of do_IRQ
578 la %r2,SP_PTREGS(%r15) # address of register-save area
579 basr %r14,%r1 # branch to standard irq handler
580io_return:
581 LOCKDEP_SYS_EXIT
582 TRACE_IRQS_ON
583io_tif:
584 tm __TI_flags+3(%r12),_TIF_WORK_INT
585 bnz BASED(io_work) # there is work to do (signals etc.)
586io_restore:
587 RESTORE_ALL __LC_RETURN_PSW,0
588io_done:
589
590#
591# There is work todo, find out in which context we have been interrupted:
592# 1) if we return to user space we can do all _TIF_WORK_INT work
593# 2) if we return to kernel code and preemptive scheduling is enabled check
594# the preemption counter and if it is zero call preempt_schedule_irq
595# Before any work can be done, a switch to the kernel stack is required.
596#
597io_work:
598 tm SP_PSW+1(%r15),0x01 # returning to user ?
599 bo BASED(io_work_user) # yes -> do resched & signal
600#ifdef CONFIG_PREEMPT
601 # check for preemptive scheduling
602 icm %r0,15,__TI_precount(%r12)
603 bnz BASED(io_restore) # preemption disabled
604 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
605 bno BASED(io_restore)
606 # switch to kernel stack
607 l %r1,SP_R15(%r15)
608 s %r1,BASED(.Lc_spsize)
609 mvc SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
610 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
611 lr %r15,%r1
612 # TRACE_IRQS_ON already done at io_return, call
613 # TRACE_IRQS_OFF to keep things symmetrical
614 TRACE_IRQS_OFF
615 l %r1,BASED(.Lpreempt_schedule_irq)
616 basr %r14,%r1 # call preempt_schedule_irq
617 b BASED(io_return)
618#else
619 b BASED(io_restore)
620#endif
621
622#
623# Need to do work before returning to userspace, switch to kernel stack
624#
625io_work_user:
626 l %r1,__LC_KERNEL_STACK
627 s %r1,BASED(.Lc_spsize)
628 mvc SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
629 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
630 lr %r15,%r1
631
632#
633# One of the work bits is on. Find out which one.
634# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
635# and _TIF_MCCK_PENDING
636#
637io_work_tif:
638 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
639 bo BASED(io_mcck_pending)
640 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
641 bo BASED(io_reschedule)
642 tm __TI_flags+3(%r12),_TIF_SIGPENDING
643 bo BASED(io_sigpending)
644 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
645 bo BASED(io_notify_resume)
646 b BASED(io_return) # beware of critical section cleanup
647
648#
649# _TIF_MCCK_PENDING is set, call handler
650#
651io_mcck_pending:
652 # TRACE_IRQS_ON already done at io_return
653 l %r1,BASED(.Ls390_handle_mcck)
654 basr %r14,%r1 # TIF bit will be cleared by handler
655 TRACE_IRQS_OFF
656 b BASED(io_return)
657
658#
659# _TIF_NEED_RESCHED is set, call schedule
660#
661io_reschedule:
662 # TRACE_IRQS_ON already done at io_return
663 l %r1,BASED(.Lschedule)
664 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
665 basr %r14,%r1 # call scheduler
666 stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts
667 TRACE_IRQS_OFF
668 b BASED(io_return)
669
670#
671# _TIF_SIGPENDING is set, call do_signal
672#
673io_sigpending:
674 # TRACE_IRQS_ON already done at io_return
675 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
676 la %r2,SP_PTREGS(%r15) # load pt_regs
677 l %r1,BASED(.Ldo_signal)
678 basr %r14,%r1 # call do_signal
679 stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts
680 TRACE_IRQS_OFF
681 b BASED(io_return)
682
683#
684# _TIF_SIGPENDING is set, call do_signal
685#
686io_notify_resume:
687 # TRACE_IRQS_ON already done at io_return
688 stosm __SF_EMPTY(%r15),0x03 # reenable interrupts
689 la %r2,SP_PTREGS(%r15) # load pt_regs
690 l %r1,BASED(.Ldo_notify_resume)
691 basr %r14,%r1 # call do_signal
692 stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts
693 TRACE_IRQS_OFF
694 b BASED(io_return)
695
696/*
697 * External interrupt handler routine
698 */
699
700ENTRY(ext_int_handler)
701 stck __LC_INT_CLOCK
702 stpt __LC_ASYNC_ENTER_TIMER
703 SAVE_ALL_ASYNC __LC_EXT_OLD_PSW,__LC_SAVE_AREA+16
704 CREATE_STACK_FRAME __LC_SAVE_AREA+16
705 mvc SP_PSW(8,%r15),0(%r12) # move user PSW to stack
706 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
707 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
708 bz BASED(ext_no_vtime)
709 UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
710 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
711 mvc __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
712ext_no_vtime:
713 TRACE_IRQS_OFF
714 la %r2,SP_PTREGS(%r15) # address of register-save area
715 l %r3,__LC_CPU_ADDRESS # get cpu address + interruption code
716 l %r4,__LC_EXT_PARAMS # get external parameters
717 l %r1,BASED(.Ldo_extint)
718 basr %r14,%r1
719 b BASED(io_return)
720
721__critical_end:
722
723/*
724 * Machine check handler routines
725 */
726
727ENTRY(mcck_int_handler)
728 stck __LC_MCCK_CLOCK
729 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer
730 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs
731 SAVE_ALL_BASE __LC_SAVE_AREA+32
732 la %r12,__LC_MCK_OLD_PSW
733 tm __LC_MCCK_CODE,0x80 # system damage?
734 bo BASED(mcck_int_main) # yes -> rest of mcck code invalid
735 mvc __LC_MCCK_ENTER_TIMER(8),__LC_CPU_TIMER_SAVE_AREA
736 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
737 bo BASED(1f)
738 la %r14,__LC_SYNC_ENTER_TIMER
739 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
740 bl BASED(0f)
741 la %r14,__LC_ASYNC_ENTER_TIMER
7420: clc 0(8,%r14),__LC_EXIT_TIMER
743 bl BASED(0f)
744 la %r14,__LC_EXIT_TIMER
7450: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
746 bl BASED(0f)
747 la %r14,__LC_LAST_UPDATE_TIMER
7480: spt 0(%r14)
749 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
7501: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
751 bno BASED(mcck_int_main) # no -> skip cleanup critical
752 tm __LC_MCK_OLD_PSW+1,0x01 # test problem state bit
753 bnz BASED(mcck_int_main) # from user -> load async stack
754 clc __LC_MCK_OLD_PSW+4(4),BASED(.Lcritical_end)
755 bhe BASED(mcck_int_main)
756 clc __LC_MCK_OLD_PSW+4(4),BASED(.Lcritical_start)
757 bl BASED(mcck_int_main)
758 l %r14,BASED(.Lcleanup_critical)
759 basr %r14,%r14
760mcck_int_main:
761 l %r14,__LC_PANIC_STACK # are we already on the panic stack?
762 slr %r14,%r15
763 sra %r14,PAGE_SHIFT
764 be BASED(0f)
765 l %r15,__LC_PANIC_STACK # load panic stack
7660: s %r15,BASED(.Lc_spsize) # make room for registers & psw
767 CREATE_STACK_FRAME __LC_SAVE_AREA+32
768 mvc SP_PSW(8,%r15),0(%r12)
769 l %r12,__LC_THREAD_INFO # load pointer to thread_info struct
770 tm __LC_MCCK_CODE+2,0x08 # mwp of old psw valid?
771 bno BASED(mcck_no_vtime) # no -> skip cleanup critical
772 tm SP_PSW+1(%r15),0x01 # interrupting from user ?
773 bz BASED(mcck_no_vtime)
774 UPDATE_VTIME __LC_EXIT_TIMER,__LC_MCCK_ENTER_TIMER,__LC_USER_TIMER
775 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
776 mvc __LC_LAST_UPDATE_TIMER(8),__LC_MCCK_ENTER_TIMER
777mcck_no_vtime:
778 la %r2,SP_PTREGS(%r15) # load pt_regs
779 l %r1,BASED(.Ls390_mcck)
780 basr %r14,%r1 # call machine check handler
781 tm SP_PSW+1(%r15),0x01 # returning to user ?
782 bno BASED(mcck_return)
783 l %r1,__LC_KERNEL_STACK # switch to kernel stack
784 s %r1,BASED(.Lc_spsize)
785 mvc SP_PTREGS(__PT_SIZE,%r1),SP_PTREGS(%r15)
786 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) # clear back chain
787 lr %r15,%r1
788 stosm __SF_EMPTY(%r15),0x04 # turn dat on
789 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
790 bno BASED(mcck_return)
791 TRACE_IRQS_OFF
792 l %r1,BASED(.Ls390_handle_mcck)
793 basr %r14,%r1 # call machine check handler
794 TRACE_IRQS_ON
795mcck_return:
796 mvc __LC_RETURN_MCCK_PSW(8),SP_PSW(%r15) # move return PSW
797 ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit
798 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
799 bno BASED(0f)
800 lm %r0,%r15,SP_R0(%r15) # load gprs 0-15
801 stpt __LC_EXIT_TIMER
802 lpsw __LC_RETURN_MCCK_PSW # back to caller
8030: lm %r0,%r15,SP_R0(%r15) # load gprs 0-15
804 lpsw __LC_RETURN_MCCK_PSW # back to caller
805
806 RESTORE_ALL __LC_RETURN_MCCK_PSW,0
807
808/*
809 * Restart interruption handler, kick starter for additional CPUs
810 */
811#ifdef CONFIG_SMP
812 __CPUINIT
813ENTRY(restart_int_handler)
814 basr %r1,0
815restart_base:
816 spt restart_vtime-restart_base(%r1)
817 stck __LC_LAST_UPDATE_CLOCK
818 mvc __LC_LAST_UPDATE_TIMER(8),restart_vtime-restart_base(%r1)
819 mvc __LC_EXIT_TIMER(8),restart_vtime-restart_base(%r1)
820 l %r15,__LC_SAVE_AREA+60 # load ksp
821 lctl %c0,%c15,__LC_CREGS_SAVE_AREA # get new ctl regs
822 lam %a0,%a15,__LC_AREGS_SAVE_AREA
823 lm %r6,%r15,__SF_GPRS(%r15) # load registers from clone
824 l %r1,__LC_THREAD_INFO
825 mvc __LC_USER_TIMER(8),__TI_user_timer(%r1)
826 mvc __LC_SYSTEM_TIMER(8),__TI_system_timer(%r1)
827 xc __LC_STEAL_TIMER(8),__LC_STEAL_TIMER
828 stosm __SF_EMPTY(%r15),0x04 # now we can turn dat on
829 basr %r14,0
830 l %r14,restart_addr-.(%r14)
831 basr %r14,%r14 # branch to start_secondary
832restart_addr:
833 .long start_secondary
834 .align 8
835restart_vtime:
836 .long 0x7fffffff,0xffffffff
837 .previous
838#else
839/*
840 * If we do not run with SMP enabled, let the new CPU crash ...
841 */
842ENTRY(restart_int_handler)
843 basr %r1,0
844restart_base:
845 lpsw restart_crash-restart_base(%r1)
846 .align 8
847restart_crash:
848 .long 0x000a0000,0x00000000
849restart_go:
850#endif
851
852#
853# PSW restart interrupt handler
854#
855ENTRY(psw_restart_int_handler)
856 st %r15,__LC_SAVE_AREA_64(%r0) # save r15
857 basr %r15,0
8580: l %r15,.Lrestart_stack-0b(%r15) # load restart stack
859 l %r15,0(%r15)
860 ahi %r15,-SP_SIZE # make room for pt_regs
861 stm %r0,%r14,SP_R0(%r15) # store gprs %r0-%r14 to stack
862 mvc SP_R15(4,%r15),__LC_SAVE_AREA_64(%r0)# store saved %r15 to stack
863 mvc SP_PSW(8,%r15),__LC_RST_OLD_PSW(%r0) # store restart old psw
864 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) # set backchain to 0
865 basr %r14,0
8661: l %r14,.Ldo_restart-1b(%r14)
867 basr %r14,%r14
868
869 basr %r14,0 # load disabled wait PSW if
8702: lpsw restart_psw_crash-2b(%r14) # do_restart returns
871 .align 4
872.Ldo_restart:
873 .long do_restart
874.Lrestart_stack:
875 .long restart_stack
876 .align 8
877restart_psw_crash:
878 .long 0x000a0000,0x00000000 + restart_psw_crash
879
880 .section .kprobes.text, "ax"
881
882#ifdef CONFIG_CHECK_STACK
883/*
884 * The synchronous or the asynchronous stack overflowed. We are dead.
885 * No need to properly save the registers, we are going to panic anyway.
886 * Setup a pt_regs so that show_trace can provide a good call trace.
887 */
888stack_overflow:
889 l %r15,__LC_PANIC_STACK # change to panic stack
890 sl %r15,BASED(.Lc_spsize)
891 mvc SP_PSW(8,%r15),0(%r12) # move user PSW to stack
892 stm %r0,%r11,SP_R0(%r15) # store gprs %r0-%r11 to kernel stack
893 la %r1,__LC_SAVE_AREA
894 ch %r12,BASED(.L0x020) # old psw addr == __LC_SVC_OLD_PSW ?
895 be BASED(0f)
896 ch %r12,BASED(.L0x028) # old psw addr == __LC_PGM_OLD_PSW ?
897 be BASED(0f)
898 la %r1,__LC_SAVE_AREA+16
8990: mvc SP_R12(16,%r15),0(%r1) # move %r12-%r15 to stack
900 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) # clear back chain
901 l %r1,BASED(1f) # branch to kernel_stack_overflow
902 la %r2,SP_PTREGS(%r15) # load pt_regs
903 br %r1
9041: .long kernel_stack_overflow
905#endif
906
907cleanup_table_system_call:
908 .long system_call + 0x80000000, sysc_do_svc + 0x80000000
909cleanup_table_sysc_tif:
910 .long sysc_tif + 0x80000000, sysc_restore + 0x80000000
911cleanup_table_sysc_restore:
912 .long sysc_restore + 0x80000000, sysc_done + 0x80000000
913cleanup_table_io_tif:
914 .long io_tif + 0x80000000, io_restore + 0x80000000
915cleanup_table_io_restore:
916 .long io_restore + 0x80000000, io_done + 0x80000000
917
918cleanup_critical:
919 clc 4(4,%r12),BASED(cleanup_table_system_call)
920 bl BASED(0f)
921 clc 4(4,%r12),BASED(cleanup_table_system_call+4)
922 bl BASED(cleanup_system_call)
9230:
924 clc 4(4,%r12),BASED(cleanup_table_sysc_tif)
925 bl BASED(0f)
926 clc 4(4,%r12),BASED(cleanup_table_sysc_tif+4)
927 bl BASED(cleanup_sysc_tif)
9280:
929 clc 4(4,%r12),BASED(cleanup_table_sysc_restore)
930 bl BASED(0f)
931 clc 4(4,%r12),BASED(cleanup_table_sysc_restore+4)
932 bl BASED(cleanup_sysc_restore)
9330:
934 clc 4(4,%r12),BASED(cleanup_table_io_tif)
935 bl BASED(0f)
936 clc 4(4,%r12),BASED(cleanup_table_io_tif+4)
937 bl BASED(cleanup_io_tif)
9380:
939 clc 4(4,%r12),BASED(cleanup_table_io_restore)
940 bl BASED(0f)
941 clc 4(4,%r12),BASED(cleanup_table_io_restore+4)
942 bl BASED(cleanup_io_restore)
9430:
944 br %r14
945
946cleanup_system_call:
947 mvc __LC_RETURN_PSW(8),0(%r12)
948 clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+4)
949 bh BASED(0f)
950 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
951 c %r12,BASED(.Lmck_old_psw)
952 be BASED(0f)
953 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
9540: c %r12,BASED(.Lmck_old_psw)
955 la %r12,__LC_SAVE_AREA+32
956 be BASED(0f)
957 la %r12,__LC_SAVE_AREA+16
9580: clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+8)
959 bhe BASED(cleanup_vtime)
960 clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn)
961 bh BASED(0f)
962 mvc __LC_SAVE_AREA(16),0(%r12)
9630: st %r13,4(%r12)
964 l %r15,__LC_KERNEL_STACK # problem state -> load ksp
965 s %r15,BASED(.Lc_spsize) # make room for registers & psw
966 st %r15,12(%r12)
967 CREATE_STACK_FRAME __LC_SAVE_AREA
968 mvc SP_PSW(8,%r15),__LC_SVC_OLD_PSW
969 mvc SP_ILC(4,%r15),__LC_SVC_ILC
970 mvc 0(4,%r12),__LC_THREAD_INFO
971cleanup_vtime:
972 clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+12)
973 bhe BASED(cleanup_stime)
974 UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER
975cleanup_stime:
976 clc __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+16)
977 bh BASED(cleanup_update)
978 UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
979cleanup_update:
980 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
981 mvc __LC_RETURN_PSW+4(4),BASED(cleanup_table_system_call+4)
982 la %r12,__LC_RETURN_PSW
983 br %r14
984cleanup_system_call_insn:
985 .long sysc_saveall + 0x80000000
986 .long system_call + 0x80000000
987 .long sysc_vtime + 0x80000000
988 .long sysc_stime + 0x80000000
989 .long sysc_update + 0x80000000
990
991cleanup_sysc_tif:
992 mvc __LC_RETURN_PSW(4),0(%r12)
993 mvc __LC_RETURN_PSW+4(4),BASED(cleanup_table_sysc_tif)
994 la %r12,__LC_RETURN_PSW
995 br %r14
996
997cleanup_sysc_restore:
998 clc 4(4,%r12),BASED(cleanup_sysc_restore_insn)
999 be BASED(2f)
1000 mvc __LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
1001 c %r12,BASED(.Lmck_old_psw)
1002 be BASED(0f)
1003 mvc __LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
10040: clc 4(4,%r12),BASED(cleanup_sysc_restore_insn+4)
1005 be BASED(2f)
1006 mvc __LC_RETURN_PSW(8),SP_PSW(%r15)
1007 c %r12,BASED(.Lmck_old_psw)
1008 la %r12,__LC_SAVE_AREA+32
1009 be BASED(1f)
1010 la %r12,__LC_SAVE_AREA+16
10111: mvc 0(16,%r12),SP_R12(%r15)
1012 lm %r0,%r11,SP_R0(%r15)
1013 l %r15,SP_R15(%r15)
10142: la %r12,__LC_RETURN_PSW
1015 br %r14
1016cleanup_sysc_restore_insn:
1017 .long sysc_done - 4 + 0x80000000
1018 .long sysc_done - 8 + 0x80000000
1019
1020cleanup_io_tif:
1021 mvc __LC_RETURN_PSW(4),0(%r12)
1022 mvc __LC_RETURN_PSW+4(4),BASED(cleanup_table_io_tif)
1023 la %r12,__LC_RETURN_PSW
1024 br %r14
1025
1026cleanup_io_restore:
1027 clc 4(4,%r12),BASED(cleanup_io_restore_insn)
1028 be BASED(1f)
1029 mvc __LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
1030 clc 4(4,%r12),BASED(cleanup_io_restore_insn+4)
1031 be BASED(1f)
1032 mvc __LC_RETURN_PSW(8),SP_PSW(%r15)
1033 mvc __LC_SAVE_AREA+32(16),SP_R12(%r15)
1034 lm %r0,%r11,SP_R0(%r15)
1035 l %r15,SP_R15(%r15)
10361: la %r12,__LC_RETURN_PSW
1037 br %r14
1038cleanup_io_restore_insn:
1039 .long io_done - 4 + 0x80000000
1040 .long io_done - 8 + 0x80000000
1041
1042/*
1043 * Integer constants
1044 */
1045 .align 4
1046.Lc_spsize: .long SP_SIZE
1047.Lc_overhead: .long STACK_FRAME_OVERHEAD
1048.Lnr_syscalls: .long NR_syscalls
1049.L0x018: .short 0x018
1050.L0x020: .short 0x020
1051.L0x028: .short 0x028
1052.L0x030: .short 0x030
1053.L0x038: .short 0x038
1054.Lc_1: .long 1
1055
1056/*
1057 * Symbol constants
1058 */
1059.Ls390_mcck: .long s390_do_machine_check
1060.Ls390_handle_mcck:
1061 .long s390_handle_mcck
1062.Lmck_old_psw: .long __LC_MCK_OLD_PSW
1063.Ldo_IRQ: .long do_IRQ
1064.Ldo_extint: .long do_extint
1065.Ldo_signal: .long do_signal
1066.Ldo_notify_resume:
1067 .long do_notify_resume
1068.Lhandle_per: .long do_per_trap
1069.Ldo_execve: .long do_execve
1070.Lexecve_tail: .long execve_tail
1071.Ljump_table: .long pgm_check_table
1072.Lschedule: .long schedule
1073#ifdef CONFIG_PREEMPT
1074.Lpreempt_schedule_irq:
1075 .long preempt_schedule_irq
1076#endif
1077.Ltrace_entry: .long do_syscall_trace_enter
1078.Ltrace_exit: .long do_syscall_trace_exit
1079.Lschedtail: .long schedule_tail
1080.Lsysc_table: .long sys_call_table
1081#ifdef CONFIG_TRACE_IRQFLAGS
1082.Ltrace_irq_on_caller:
1083 .long trace_hardirqs_on_caller
1084.Ltrace_irq_off_caller:
1085 .long trace_hardirqs_off_caller
1086#endif
1087#ifdef CONFIG_LOCKDEP
1088.Llockdep_sys_exit:
1089 .long lockdep_sys_exit
1090#endif
1091.Lcritical_start:
1092 .long __critical_start + 0x80000000
1093.Lcritical_end:
1094 .long __critical_end + 0x80000000
1095.Lcleanup_critical:
1096 .long cleanup_critical
1097
1098 .section .rodata, "a"
1099#define SYSCALL(esa,esame,emu) .long esa
1100 .globl sys_call_table
1101sys_call_table:
1102#include "syscalls.S"
1103#undef SYSCALL
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * S390 low-level entry points.
4 *
5 * Copyright IBM Corp. 1999, 2012
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 */
10
11#include <linux/export.h>
12#include <linux/init.h>
13#include <linux/linkage.h>
14#include <asm/asm-extable.h>
15#include <asm/alternative-asm.h>
16#include <asm/processor.h>
17#include <asm/cache.h>
18#include <asm/dwarf.h>
19#include <asm/errno.h>
20#include <asm/ptrace.h>
21#include <asm/thread_info.h>
22#include <asm/asm-offsets.h>
23#include <asm/unistd.h>
24#include <asm/page.h>
25#include <asm/sigp.h>
26#include <asm/irq.h>
27#include <asm/fpu-insn.h>
28#include <asm/setup.h>
29#include <asm/nmi.h>
30#include <asm/nospec-insn.h>
31
32_LPP_OFFSET = __LC_LPP
33
34 .macro STBEAR address
35 ALTERNATIVE "nop", ".insn s,0xb2010000,\address", 193
36 .endm
37
38 .macro LBEAR address
39 ALTERNATIVE "nop", ".insn s,0xb2000000,\address", 193
40 .endm
41
42 .macro LPSWEY address,lpswe
43 ALTERNATIVE "b \lpswe; nopr", ".insn siy,0xeb0000000071,\address,0", 193
44 .endm
45
46 .macro MBEAR reg
47 ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK), 193
48 .endm
49
50 .macro CHECK_STACK savearea
51#ifdef CONFIG_CHECK_STACK
52 tml %r15,THREAD_SIZE - CONFIG_STACK_GUARD
53 lghi %r14,\savearea
54 jz stack_overflow
55#endif
56 .endm
57
58 .macro CHECK_VMAP_STACK savearea,oklabel
59#ifdef CONFIG_VMAP_STACK
60 lgr %r14,%r15
61 nill %r14,0x10000 - THREAD_SIZE
62 oill %r14,STACK_INIT_OFFSET
63 clg %r14,__LC_KERNEL_STACK
64 je \oklabel
65 clg %r14,__LC_ASYNC_STACK
66 je \oklabel
67 clg %r14,__LC_MCCK_STACK
68 je \oklabel
69 clg %r14,__LC_NODAT_STACK
70 je \oklabel
71 clg %r14,__LC_RESTART_STACK
72 je \oklabel
73 lghi %r14,\savearea
74 j stack_overflow
75#else
76 j \oklabel
77#endif
78 .endm
79
80 /*
81 * The TSTMSK macro generates a test-under-mask instruction by
82 * calculating the memory offset for the specified mask value.
83 * Mask value can be any constant. The macro shifts the mask
84 * value to calculate the memory offset for the test-under-mask
85 * instruction.
86 */
87 .macro TSTMSK addr, mask, size=8, bytepos=0
88 .if (\bytepos < \size) && (\mask >> 8)
89 .if (\mask & 0xff)
90 .error "Mask exceeds byte boundary"
91 .endif
92 TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
93 .exitm
94 .endif
95 .ifeq \mask
96 .error "Mask must not be zero"
97 .endif
98 off = \size - \bytepos - 1
99 tm off+\addr, \mask
100 .endm
101
102 .macro BPOFF
103 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", 82
104 .endm
105
106 .macro BPON
107 ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", 82
108 .endm
109
110 .macro BPENTER tif_ptr,tif_mask
111 ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \
112 "j .+12; nop; nop", 82
113 .endm
114
115 .macro BPEXIT tif_ptr,tif_mask
116 TSTMSK \tif_ptr,\tif_mask
117 ALTERNATIVE "jz .+8; .insn rrf,0xb2e80000,0,0,12,0", \
118 "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", 82
119 .endm
120
121#if IS_ENABLED(CONFIG_KVM)
122 .macro SIEEXIT sie_control
123 lg %r9,\sie_control # get control block pointer
124 ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
125 lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce
126 ni __LC_CPU_FLAGS+7,255-_CIF_SIE
127 larl %r9,sie_exit # skip forward to sie_exit
128 .endm
129#endif
130
131 .macro STACKLEAK_ERASE
132#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
133 brasl %r14,stackleak_erase_on_task_stack
134#endif
135 .endm
136
137 GEN_BR_THUNK %r14
138
139 .section .kprobes.text, "ax"
140.Ldummy:
141 /*
142 * The following nop exists only in order to avoid that the next
143 * symbol starts at the beginning of the kprobes text section.
144 * In that case there would be several symbols at the same address.
145 * E.g. objdump would take an arbitrary symbol when disassembling
146 * the code.
147 * With the added nop in between this cannot happen.
148 */
149 nop 0
150
151/*
152 * Scheduler resume function, called by __switch_to
153 * gpr2 = (task_struct *)prev
154 * gpr3 = (task_struct *)next
155 * Returns:
156 * gpr2 = prev
157 */
158SYM_FUNC_START(__switch_to_asm)
159 stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
160 lghi %r4,__TASK_stack
161 lghi %r1,__TASK_thread
162 llill %r5,STACK_INIT_OFFSET
163 stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev
164 lg %r15,0(%r4,%r3) # start of kernel stack of next
165 agr %r15,%r5 # end of kernel stack of next
166 stg %r3,__LC_CURRENT # store task struct of next
167 stg %r15,__LC_KERNEL_STACK # store end of kernel stack
168 lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next
169 aghi %r3,__TASK_pid
170 mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next
171 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
172 ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
173 BR_EX %r14
174SYM_FUNC_END(__switch_to_asm)
175
176#if IS_ENABLED(CONFIG_KVM)
177/*
178 * __sie64a calling convention:
179 * %r2 pointer to sie control block phys
180 * %r3 pointer to sie control block virt
181 * %r4 guest register save area
182 */
183SYM_FUNC_START(__sie64a)
184 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
185 lg %r12,__LC_CURRENT
186 stg %r2,__SF_SIE_CONTROL_PHYS(%r15) # save sie block physical..
187 stg %r3,__SF_SIE_CONTROL(%r15) # ...and virtual addresses
188 stg %r4,__SF_SIE_SAVEAREA(%r15) # save guest register save area
189 xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
190 mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
191 lmg %r0,%r13,0(%r4) # load guest gprs 0-13
192 lg %r14,__LC_GMAP # get gmap pointer
193 ltgr %r14,%r14
194 jz .Lsie_gmap
195 oi __LC_CPU_FLAGS+7,_CIF_SIE
196 lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce
197.Lsie_gmap:
198 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
199 oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
200 tm __SIE_PROG20+3(%r14),3 # last exit...
201 jnz .Lsie_skip
202 lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr
203 BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
204.Lsie_entry:
205 sie 0(%r14)
206# Let the next instruction be NOP to avoid triggering a machine check
207# and handling it in a guest as result of the instruction execution.
208 nopr 7
209.Lsie_leave:
210 BPOFF
211 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
212.Lsie_skip:
213 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
214 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
215 lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce
216 ni __LC_CPU_FLAGS+7,255-_CIF_SIE
217# some program checks are suppressing. C code (e.g. do_protection_exception)
218# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
219# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
220# Other instructions between __sie64a and .Lsie_done should not cause program
221# interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
222.Lrewind_pad6:
223 nopr 7
224.Lrewind_pad4:
225 nopr 7
226.Lrewind_pad2:
227 nopr 7
228SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL)
229 lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area
230 stmg %r0,%r13,0(%r14) # save guest gprs 0-13
231 xgr %r0,%r0 # clear guest registers to
232 xgr %r1,%r1 # prevent speculative use
233 xgr %r3,%r3
234 xgr %r4,%r4
235 xgr %r5,%r5
236 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
237 lg %r2,__SF_SIE_REASON(%r15) # return exit reason code
238 BR_EX %r14
239.Lsie_fault:
240 lghi %r14,-EFAULT
241 stg %r14,__SF_SIE_REASON(%r15) # set exit reason code
242 j sie_exit
243
244 EX_TABLE(.Lrewind_pad6,.Lsie_fault)
245 EX_TABLE(.Lrewind_pad4,.Lsie_fault)
246 EX_TABLE(.Lrewind_pad2,.Lsie_fault)
247 EX_TABLE(sie_exit,.Lsie_fault)
248SYM_FUNC_END(__sie64a)
249EXPORT_SYMBOL(__sie64a)
250EXPORT_SYMBOL(sie_exit)
251#endif
252
253/*
254 * SVC interrupt handler routine. System calls are synchronous events and
255 * are entered with interrupts disabled.
256 */
257
258SYM_CODE_START(system_call)
259 stpt __LC_SYS_ENTER_TIMER
260 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
261 BPOFF
262 lghi %r14,0
263.Lsysc_per:
264 STBEAR __LC_LAST_BREAK
265 lctlg %c1,%c1,__LC_KERNEL_ASCE
266 lg %r15,__LC_KERNEL_STACK
267 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
268 stmg %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
269 # clear user controlled register to prevent speculative use
270 xgr %r0,%r0
271 xgr %r1,%r1
272 xgr %r4,%r4
273 xgr %r5,%r5
274 xgr %r6,%r6
275 xgr %r7,%r7
276 xgr %r8,%r8
277 xgr %r9,%r9
278 xgr %r10,%r10
279 xgr %r11,%r11
280 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
281 mvc __PT_R8(64,%r2),__LC_SAVE_AREA_SYNC
282 MBEAR %r2
283 lgr %r3,%r14
284 brasl %r14,__do_syscall
285 STACKLEAK_ERASE
286 lctlg %c1,%c1,__LC_USER_ASCE
287 mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
288 BPON
289 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
290 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
291 stpt __LC_EXIT_TIMER
292 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
293SYM_CODE_END(system_call)
294
295#
296# a new process exits the kernel with ret_from_fork
297#
298SYM_CODE_START(ret_from_fork)
299 lgr %r3,%r11
300 brasl %r14,__ret_from_fork
301 STACKLEAK_ERASE
302 lctlg %c1,%c1,__LC_USER_ASCE
303 mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
304 BPON
305 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
306 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
307 stpt __LC_EXIT_TIMER
308 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
309SYM_CODE_END(ret_from_fork)
310
311/*
312 * Program check handler routine
313 */
314
315SYM_CODE_START(pgm_check_handler)
316 stpt __LC_SYS_ENTER_TIMER
317 BPOFF
318 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
319 lgr %r10,%r15
320 lmg %r8,%r9,__LC_PGM_OLD_PSW
321 tmhh %r8,0x0001 # coming from user space?
322 jno .Lpgm_skip_asce
323 lctlg %c1,%c1,__LC_KERNEL_ASCE
324 j 3f # -> fault in user space
325.Lpgm_skip_asce:
3261: tmhh %r8,0x4000 # PER bit set in old PSW ?
327 jnz 2f # -> enabled, can't be a double fault
328 tm __LC_PGM_ILC+3,0x80 # check for per exception
329 jnz .Lpgm_svcper # -> single stepped svc
3302: CHECK_STACK __LC_SAVE_AREA_SYNC
331 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
332 # CHECK_VMAP_STACK branches to stack_overflow or 4f
333 CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f
3343: lg %r15,__LC_KERNEL_STACK
3354: la %r11,STACK_FRAME_OVERHEAD(%r15)
336 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
337 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
338 stmg %r0,%r7,__PT_R0(%r11)
339 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
340 mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK
341 stctg %c1,%c1,__PT_CR1(%r11)
342#if IS_ENABLED(CONFIG_KVM)
343 ltg %r12,__LC_GMAP
344 jz 5f
345 clc __GMAP_ASCE(8,%r12), __PT_CR1(%r11)
346 jne 5f
347 BPENTER __SF_SIE_FLAGS(%r10),_TIF_ISOLATE_BP_GUEST
348 SIEEXIT __SF_SIE_CONTROL(%r10)
349#endif
3505: stmg %r8,%r9,__PT_PSW(%r11)
351 # clear user controlled registers to prevent speculative use
352 xgr %r0,%r0
353 xgr %r1,%r1
354 xgr %r3,%r3
355 xgr %r4,%r4
356 xgr %r5,%r5
357 xgr %r6,%r6
358 xgr %r7,%r7
359 lgr %r2,%r11
360 brasl %r14,__do_pgm_check
361 tmhh %r8,0x0001 # returning to user space?
362 jno .Lpgm_exit_kernel
363 STACKLEAK_ERASE
364 lctlg %c1,%c1,__LC_USER_ASCE
365 BPON
366 stpt __LC_EXIT_TIMER
367.Lpgm_exit_kernel:
368 mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
369 LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
370 lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
371 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
372
373#
374# single stepped system call
375#
376.Lpgm_svcper:
377 mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
378 larl %r14,.Lsysc_per
379 stg %r14,__LC_RETURN_PSW+8
380 lghi %r14,1
381 LBEAR __LC_PGM_LAST_BREAK
382 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per
383SYM_CODE_END(pgm_check_handler)
384
385/*
386 * Interrupt handler macro used for external and IO interrupts.
387 */
388.macro INT_HANDLER name,lc_old_psw,handler
389SYM_CODE_START(\name)
390 stckf __LC_INT_CLOCK
391 stpt __LC_SYS_ENTER_TIMER
392 STBEAR __LC_LAST_BREAK
393 BPOFF
394 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
395 lmg %r8,%r9,\lc_old_psw
396 tmhh %r8,0x0001 # interrupting from user ?
397 jnz 1f
398#if IS_ENABLED(CONFIG_KVM)
399 TSTMSK __LC_CPU_FLAGS,_CIF_SIE
400 jz 0f
401 BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
402 SIEEXIT __SF_SIE_CONTROL(%r15)
403#endif
4040: CHECK_STACK __LC_SAVE_AREA_ASYNC
405 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
406 j 2f
4071: lctlg %c1,%c1,__LC_KERNEL_ASCE
408 lg %r15,__LC_KERNEL_STACK
4092: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
410 la %r11,STACK_FRAME_OVERHEAD(%r15)
411 stmg %r0,%r7,__PT_R0(%r11)
412 # clear user controlled registers to prevent speculative use
413 xgr %r0,%r0
414 xgr %r1,%r1
415 xgr %r3,%r3
416 xgr %r4,%r4
417 xgr %r5,%r5
418 xgr %r6,%r6
419 xgr %r7,%r7
420 xgr %r10,%r10
421 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
422 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
423 MBEAR %r11
424 stmg %r8,%r9,__PT_PSW(%r11)
425 lgr %r2,%r11 # pass pointer to pt_regs
426 brasl %r14,\handler
427 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
428 tmhh %r8,0x0001 # returning to user ?
429 jno 2f
430 STACKLEAK_ERASE
431 lctlg %c1,%c1,__LC_USER_ASCE
432 BPON
433 stpt __LC_EXIT_TIMER
4342: LBEAR __PT_LAST_BREAK(%r11)
435 lmg %r0,%r15,__PT_R0(%r11)
436 LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
437SYM_CODE_END(\name)
438.endm
439
440INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
441INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
442
443/*
444 * Load idle PSW.
445 */
446SYM_FUNC_START(psw_idle)
447 stg %r14,(__SF_GPRS+8*8)(%r15)
448 stg %r3,__SF_EMPTY(%r15)
449 larl %r1,psw_idle_exit
450 stg %r1,__SF_EMPTY+8(%r15)
451 larl %r1,smp_cpu_mtid
452 llgf %r1,0(%r1)
453 ltgr %r1,%r1
454 jz .Lpsw_idle_stcctm
455 .insn rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2)
456.Lpsw_idle_stcctm:
457 oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
458 BPON
459 stckf __CLOCK_IDLE_ENTER(%r2)
460 stpt __TIMER_IDLE_ENTER(%r2)
461 lpswe __SF_EMPTY(%r15)
462SYM_INNER_LABEL(psw_idle_exit, SYM_L_GLOBAL)
463 BR_EX %r14
464SYM_FUNC_END(psw_idle)
465
466/*
467 * Machine check handler routines
468 */
469SYM_CODE_START(mcck_int_handler)
470 BPOFF
471 lmg %r8,%r9,__LC_MCK_OLD_PSW
472 TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
473 jo .Lmcck_panic # yes -> rest of mcck code invalid
474 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID
475 jno .Lmcck_panic # control registers invalid -> panic
476 ptlb
477 lghi %r14,__LC_CPU_TIMER_SAVE_AREA
478 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
479 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
480 jo 3f
481 la %r14,__LC_SYS_ENTER_TIMER
482 clc 0(8,%r14),__LC_EXIT_TIMER
483 jl 1f
484 la %r14,__LC_EXIT_TIMER
4851: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
486 jl 2f
487 la %r14,__LC_LAST_UPDATE_TIMER
4882: spt 0(%r14)
489 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
4903: TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
491 jno .Lmcck_panic
492 tmhh %r8,0x0001 # interrupting from user ?
493 jnz .Lmcck_user
494 TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
495 jno .Lmcck_panic
496#if IS_ENABLED(CONFIG_KVM)
497 TSTMSK __LC_CPU_FLAGS,_CIF_SIE
498 jz .Lmcck_user
499 # Need to compare the address instead of a CIF_SIE* flag.
500 # Otherwise there would be a race between setting the flag
501 # and entering SIE (or leaving and clearing the flag). This
502 # would cause machine checks targeted at the guest to be
503 # handled by the host.
504 larl %r14,.Lsie_entry
505 clgrjl %r9,%r14, 4f
506 larl %r14,.Lsie_leave
507 clgrjhe %r9,%r14, 4f
508 oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
5094: BPENTER __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST
510 SIEEXIT __SF_SIE_CONTROL(%r15)
511#endif
512.Lmcck_user:
513 lg %r15,__LC_MCCK_STACK
514 la %r11,STACK_FRAME_OVERHEAD(%r15)
515 stctg %c1,%c1,__PT_CR1(%r11)
516 lctlg %c1,%c1,__LC_KERNEL_ASCE
517 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
518 lghi %r14,__LC_GPREGS_SAVE_AREA+64
519 stmg %r0,%r7,__PT_R0(%r11)
520 # clear user controlled registers to prevent speculative use
521 xgr %r0,%r0
522 xgr %r1,%r1
523 xgr %r3,%r3
524 xgr %r4,%r4
525 xgr %r5,%r5
526 xgr %r6,%r6
527 xgr %r7,%r7
528 xgr %r10,%r10
529 mvc __PT_R8(64,%r11),0(%r14)
530 stmg %r8,%r9,__PT_PSW(%r11)
531 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
532 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
533 lgr %r2,%r11 # pass pointer to pt_regs
534 brasl %r14,s390_do_machine_check
535 lctlg %c1,%c1,__PT_CR1(%r11)
536 lmg %r0,%r10,__PT_R0(%r11)
537 mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
538 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
539 jno 0f
540 BPON
541 stpt __LC_EXIT_TIMER
5420: ALTERNATIVE "nop", __stringify(lghi %r12,__LC_LAST_BREAK_SAVE_AREA),193
543 LBEAR 0(%r12)
544 lmg %r11,%r15,__PT_R11(%r11)
545 LPSWEY __LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE
546
547.Lmcck_panic:
548 /*
549 * Iterate over all possible CPU addresses in the range 0..0xffff
550 * and stop each CPU using signal processor. Use compare and swap
551 * to allow just one CPU-stopper and prevent concurrent CPUs from
552 * stopping each other while leaving the others running.
553 */
554 lhi %r5,0
555 lhi %r6,1
556 larl %r7,stop_lock
557 cs %r5,%r6,0(%r7) # single CPU-stopper only
558 jnz 4f
559 larl %r7,this_cpu
560 stap 0(%r7) # this CPU address
561 lh %r4,0(%r7)
562 nilh %r4,0
563 lhi %r0,1
564 sll %r0,16 # CPU counter
565 lhi %r3,0 # next CPU address
5660: cr %r3,%r4
567 je 2f
5681: sigp %r1,%r3,SIGP_STOP # stop next CPU
569 brc SIGP_CC_BUSY,1b
5702: ahi %r3,1
571 brct %r0,0b
5723: sigp %r1,%r4,SIGP_STOP # stop this CPU
573 brc SIGP_CC_BUSY,3b
5744: j 4b
575SYM_CODE_END(mcck_int_handler)
576
577SYM_CODE_START(restart_int_handler)
578 ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
579 stg %r15,__LC_SAVE_AREA_RESTART
580 TSTMSK __LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4
581 jz 0f
582 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA
5830: larl %r15,daton_psw
584 lpswe 0(%r15) # turn dat on, keep irqs off
585.Ldaton:
586 lg %r15,__LC_RESTART_STACK
587 xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
588 stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
589 mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
590 mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
591 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
592 lg %r1,__LC_RESTART_FN # load fn, parm & source cpu
593 lg %r2,__LC_RESTART_DATA
594 lgf %r3,__LC_RESTART_SOURCE
595 ltgr %r3,%r3 # test source cpu address
596 jm 1f # negative -> skip source stop
5970: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
598 brc 10,0b # wait for status stored
5991: basr %r14,%r1 # call function
600 stap __SF_EMPTY(%r15) # store cpu address
601 llgh %r3,__SF_EMPTY(%r15)
6022: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
603 brc 2,2b
6043: j 3b
605SYM_CODE_END(restart_int_handler)
606
607 .section .kprobes.text, "ax"
608
609#if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
610/*
611 * The synchronous or the asynchronous stack overflowed. We are dead.
612 * No need to properly save the registers, we are going to panic anyway.
613 * Setup a pt_regs so that show_trace can provide a good call trace.
614 */
615SYM_CODE_START(stack_overflow)
616 lg %r15,__LC_NODAT_STACK # change to panic stack
617 la %r11,STACK_FRAME_OVERHEAD(%r15)
618 stmg %r0,%r7,__PT_R0(%r11)
619 stmg %r8,%r9,__PT_PSW(%r11)
620 mvc __PT_R8(64,%r11),0(%r14)
621 stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
622 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
623 lgr %r2,%r11 # pass pointer to pt_regs
624 jg kernel_stack_overflow
625SYM_CODE_END(stack_overflow)
626#endif
627
628 .section .data, "aw"
629 .balign 4
630SYM_DATA_LOCAL(stop_lock, .long 0)
631SYM_DATA_LOCAL(this_cpu, .short 0)
632 .balign 8
633SYM_DATA_START_LOCAL(daton_psw)
634 .quad PSW_KERNEL_BITS
635 .quad .Ldaton
636SYM_DATA_END(daton_psw)
637
638 .section .rodata, "a"
639 .balign 8
640#define SYSCALL(esame,emu) .quad __s390x_ ## esame
641SYM_DATA_START(sys_call_table)
642#include "asm/syscall_table.h"
643SYM_DATA_END(sys_call_table)
644#undef SYSCALL
645
646#ifdef CONFIG_COMPAT
647
648#define SYSCALL(esame,emu) .quad __s390_ ## emu
649SYM_DATA_START(sys_call_table_emu)
650#include "asm/syscall_table.h"
651SYM_DATA_END(sys_call_table_emu)
652#undef SYSCALL
653#endif