Loading...
1/*
2 * S390 low-level entry points.
3 *
4 * Copyright IBM Corp. 1999, 2012
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
6 * Hartmut Penner (hp@de.ibm.com),
7 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
8 * Heiko Carstens <heiko.carstens@de.ibm.com>
9 */
10
11#include <linux/init.h>
12#include <linux/linkage.h>
13#include <asm/processor.h>
14#include <asm/cache.h>
15#include <asm/errno.h>
16#include <asm/ptrace.h>
17#include <asm/thread_info.h>
18#include <asm/asm-offsets.h>
19#include <asm/unistd.h>
20#include <asm/page.h>
21#include <asm/sigp.h>
22#include <asm/irq.h>
23#include <asm/vx-insn.h>
24#include <asm/setup.h>
25#include <asm/nmi.h>
26#include <asm/export.h>
27
28__PT_R0 = __PT_GPRS
29__PT_R1 = __PT_GPRS + 8
30__PT_R2 = __PT_GPRS + 16
31__PT_R3 = __PT_GPRS + 24
32__PT_R4 = __PT_GPRS + 32
33__PT_R5 = __PT_GPRS + 40
34__PT_R6 = __PT_GPRS + 48
35__PT_R7 = __PT_GPRS + 56
36__PT_R8 = __PT_GPRS + 64
37__PT_R9 = __PT_GPRS + 72
38__PT_R10 = __PT_GPRS + 80
39__PT_R11 = __PT_GPRS + 88
40__PT_R12 = __PT_GPRS + 96
41__PT_R13 = __PT_GPRS + 104
42__PT_R14 = __PT_GPRS + 112
43__PT_R15 = __PT_GPRS + 120
44
45STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
46STACK_SIZE = 1 << STACK_SHIFT
47STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
48
49_TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
50 _TIF_UPROBE)
51_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
52 _TIF_SYSCALL_TRACEPOINT)
53_CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE | _CIF_FPU)
54_PIF_WORK = (_PIF_PER_TRAP)
55
56#define BASED(name) name-cleanup_critical(%r13)
57
58 .macro TRACE_IRQS_ON
59#ifdef CONFIG_TRACE_IRQFLAGS
60 basr %r2,%r0
61 brasl %r14,trace_hardirqs_on_caller
62#endif
63 .endm
64
65 .macro TRACE_IRQS_OFF
66#ifdef CONFIG_TRACE_IRQFLAGS
67 basr %r2,%r0
68 brasl %r14,trace_hardirqs_off_caller
69#endif
70 .endm
71
72 .macro LOCKDEP_SYS_EXIT
73#ifdef CONFIG_LOCKDEP
74 tm __PT_PSW+1(%r11),0x01 # returning to user ?
75 jz .+10
76 brasl %r14,lockdep_sys_exit
77#endif
78 .endm
79
80 .macro CHECK_STACK stacksize,savearea
81#ifdef CONFIG_CHECK_STACK
82 tml %r15,\stacksize - CONFIG_STACK_GUARD
83 lghi %r14,\savearea
84 jz stack_overflow
85#endif
86 .endm
87
88 .macro SWITCH_ASYNC savearea,timer
89 tmhh %r8,0x0001 # interrupting from user ?
90 jnz 1f
91 lgr %r14,%r9
92 slg %r14,BASED(.Lcritical_start)
93 clg %r14,BASED(.Lcritical_length)
94 jhe 0f
95 lghi %r11,\savearea # inside critical section, do cleanup
96 brasl %r14,cleanup_critical
97 tmhh %r8,0x0001 # retest problem state after cleanup
98 jnz 1f
990: lg %r14,__LC_ASYNC_STACK # are we already on the async stack?
100 slgr %r14,%r15
101 srag %r14,%r14,STACK_SHIFT
102 jnz 2f
103 CHECK_STACK 1<<STACK_SHIFT,\savearea
104 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
105 j 3f
1061: LAST_BREAK %r14
107 UPDATE_VTIME %r14,%r15,\timer
1082: lg %r15,__LC_ASYNC_STACK # load async stack
1093: la %r11,STACK_FRAME_OVERHEAD(%r15)
110 .endm
111
112 .macro UPDATE_VTIME w1,w2,enter_timer
113 lg \w1,__LC_EXIT_TIMER
114 lg \w2,__LC_LAST_UPDATE_TIMER
115 slg \w1,\enter_timer
116 slg \w2,__LC_EXIT_TIMER
117 alg \w1,__LC_USER_TIMER
118 alg \w2,__LC_SYSTEM_TIMER
119 stg \w1,__LC_USER_TIMER
120 stg \w2,__LC_SYSTEM_TIMER
121 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
122 .endm
123
124 .macro LAST_BREAK scratch
125 srag \scratch,%r10,23
126#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES
127 jz .+10
128 stg %r10,__TASK_thread+__THREAD_last_break(%r12)
129#else
130 jz .+14
131 lghi \scratch,__TASK_thread
132 stg %r10,__THREAD_last_break(\scratch,%r12)
133#endif
134 .endm
135
136 .macro REENABLE_IRQS
137 stg %r8,__LC_RETURN_PSW
138 ni __LC_RETURN_PSW,0xbf
139 ssm __LC_RETURN_PSW
140 .endm
141
142 .macro STCK savearea
143#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
144 .insn s,0xb27c0000,\savearea # store clock fast
145#else
146 .insn s,0xb2050000,\savearea # store clock
147#endif
148 .endm
149
150 /*
151 * The TSTMSK macro generates a test-under-mask instruction by
152 * calculating the memory offset for the specified mask value.
153 * Mask value can be any constant. The macro shifts the mask
154 * value to calculate the memory offset for the test-under-mask
155 * instruction.
156 */
157 .macro TSTMSK addr, mask, size=8, bytepos=0
158 .if (\bytepos < \size) && (\mask >> 8)
159 .if (\mask & 0xff)
160 .error "Mask exceeds byte boundary"
161 .endif
162 TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
163 .exitm
164 .endif
165 .ifeq \mask
166 .error "Mask must not be zero"
167 .endif
168 off = \size - \bytepos - 1
169 tm off+\addr, \mask
170 .endm
171
172 .section .kprobes.text, "ax"
173.Ldummy:
174 /*
175 * This nop exists only in order to avoid that __switch_to starts at
176 * the beginning of the kprobes text section. In that case we would
177 * have several symbols at the same address. E.g. objdump would take
178 * an arbitrary symbol name when disassembling this code.
179 * With the added nop in between the __switch_to symbol is unique
180 * again.
181 */
182 nop 0
183
184/*
185 * Scheduler resume function, called by switch_to
186 * gpr2 = (task_struct *) prev
187 * gpr3 = (task_struct *) next
188 * Returns:
189 * gpr2 = prev
190 */
191ENTRY(__switch_to)
192 stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
193 lgr %r1,%r2
194 aghi %r1,__TASK_thread # thread_struct of prev task
195 lg %r5,__TASK_stack(%r3) # start of kernel stack of next
196 stg %r15,__THREAD_ksp(%r1) # store kernel stack of prev
197 lgr %r1,%r3
198 aghi %r1,__TASK_thread # thread_struct of next task
199 lgr %r15,%r5
200 aghi %r15,STACK_INIT # end of kernel stack of next
201 stg %r3,__LC_CURRENT # store task struct of next
202 stg %r15,__LC_KERNEL_STACK # store end of kernel stack
203 lg %r15,__THREAD_ksp(%r1) # load kernel stack of next
204 /* c4 is used in guest detection: arch/s390/kernel/perf_cpum_sf.c */
205 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
206 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
207 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
208 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_LPP
209 bzr %r14
210 .insn s,0xb2800000,__LC_LPP # set program parameter
211 br %r14
212
213.L__critical_start:
214
215#if IS_ENABLED(CONFIG_KVM)
216/*
217 * sie64a calling convention:
218 * %r2 pointer to sie control block
219 * %r3 guest register save area
220 */
221ENTRY(sie64a)
222 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
223 stg %r2,__SF_EMPTY(%r15) # save control block pointer
224 stg %r3,__SF_EMPTY+8(%r15) # save guest register save area
225 xc __SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # reason code = 0
226 TSTMSK __LC_CPU_FLAGS,_CIF_FPU # load guest fp/vx registers ?
227 jno .Lsie_load_guest_gprs
228 brasl %r14,load_fpu_regs # load guest fp/vx regs
229.Lsie_load_guest_gprs:
230 lmg %r0,%r13,0(%r3) # load guest gprs 0-13
231 lg %r14,__LC_GMAP # get gmap pointer
232 ltgr %r14,%r14
233 jz .Lsie_gmap
234 lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce
235.Lsie_gmap:
236 lg %r14,__SF_EMPTY(%r15) # get control block pointer
237 oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
238 tm __SIE_PROG20+3(%r14),3 # last exit...
239 jnz .Lsie_skip
240 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
241 jo .Lsie_skip # exit if fp/vx regs changed
242 sie 0(%r14)
243.Lsie_skip:
244 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
245 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
246.Lsie_done:
247# some program checks are suppressing. C code (e.g. do_protection_exception)
248# will rewind the PSW by the ILC, which is 4 bytes in case of SIE. Other
249# instructions between sie64a and .Lsie_done should not cause program
250# interrupts. So lets use a nop (47 00 00 00) as a landing pad.
251# See also .Lcleanup_sie
252.Lrewind_pad:
253 nop 0
254 .globl sie_exit
255sie_exit:
256 lg %r14,__SF_EMPTY+8(%r15) # load guest register save area
257 stmg %r0,%r13,0(%r14) # save guest gprs 0-13
258 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
259 lg %r2,__SF_EMPTY+16(%r15) # return exit reason code
260 br %r14
261.Lsie_fault:
262 lghi %r14,-EFAULT
263 stg %r14,__SF_EMPTY+16(%r15) # set exit reason code
264 j sie_exit
265
266 EX_TABLE(.Lrewind_pad,.Lsie_fault)
267 EX_TABLE(sie_exit,.Lsie_fault)
268EXPORT_SYMBOL(sie64a)
269EXPORT_SYMBOL(sie_exit)
270#endif
271
272/*
273 * SVC interrupt handler routine. System calls are synchronous events and
274 * are executed with interrupts enabled.
275 */
276
277ENTRY(system_call)
278 stpt __LC_SYNC_ENTER_TIMER
279.Lsysc_stmg:
280 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
281 lg %r10,__LC_LAST_BREAK
282 lg %r12,__LC_CURRENT
283 lghi %r14,_PIF_SYSCALL
284.Lsysc_per:
285 lg %r15,__LC_KERNEL_STACK
286 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
287 LAST_BREAK %r13
288.Lsysc_vtime:
289 UPDATE_VTIME %r10,%r13,__LC_SYNC_ENTER_TIMER
290 stmg %r0,%r7,__PT_R0(%r11)
291 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
292 mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW
293 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
294 stg %r14,__PT_FLAGS(%r11)
295.Lsysc_do_svc:
296 # load address of system call table
297#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES
298 lg %r10,__TASK_thread+__THREAD_sysc_table(%r12)
299#else
300 lghi %r13,__TASK_thread
301 lg %r10,__THREAD_sysc_table(%r13,%r12)
302#endif
303 llgh %r8,__PT_INT_CODE+2(%r11)
304 slag %r8,%r8,2 # shift and test for svc 0
305 jnz .Lsysc_nr_ok
306 # svc 0: system call number in %r1
307 llgfr %r1,%r1 # clear high word in r1
308 cghi %r1,NR_syscalls
309 jnl .Lsysc_nr_ok
310 sth %r1,__PT_INT_CODE+2(%r11)
311 slag %r8,%r1,2
312.Lsysc_nr_ok:
313 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
314 stg %r2,__PT_ORIG_GPR2(%r11)
315 stg %r7,STACK_FRAME_OVERHEAD(%r15)
316 lgf %r9,0(%r8,%r10) # get system call add.
317 TSTMSK __TI_flags(%r12),_TIF_TRACE
318 jnz .Lsysc_tracesys
319 basr %r14,%r9 # call sys_xxxx
320 stg %r2,__PT_R2(%r11) # store return value
321
322.Lsysc_return:
323 LOCKDEP_SYS_EXIT
324.Lsysc_tif:
325 TSTMSK __PT_FLAGS(%r11),_PIF_WORK
326 jnz .Lsysc_work
327 TSTMSK __TI_flags(%r12),_TIF_WORK
328 jnz .Lsysc_work # check for work
329 TSTMSK __LC_CPU_FLAGS,_CIF_WORK
330 jnz .Lsysc_work
331.Lsysc_restore:
332 lg %r14,__LC_VDSO_PER_CPU
333 lmg %r0,%r10,__PT_R0(%r11)
334 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
335 stpt __LC_EXIT_TIMER
336 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
337 lmg %r11,%r15,__PT_R11(%r11)
338 lpswe __LC_RETURN_PSW
339.Lsysc_done:
340
341#
342# One of the work bits is on. Find out which one.
343#
344.Lsysc_work:
345 TSTMSK __LC_CPU_FLAGS,_CIF_MCCK_PENDING
346 jo .Lsysc_mcck_pending
347 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED
348 jo .Lsysc_reschedule
349#ifdef CONFIG_UPROBES
350 TSTMSK __TI_flags(%r12),_TIF_UPROBE
351 jo .Lsysc_uprobe_notify
352#endif
353 TSTMSK __PT_FLAGS(%r11),_PIF_PER_TRAP
354 jo .Lsysc_singlestep
355 TSTMSK __TI_flags(%r12),_TIF_SIGPENDING
356 jo .Lsysc_sigpending
357 TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME
358 jo .Lsysc_notify_resume
359 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
360 jo .Lsysc_vxrs
361 TSTMSK __LC_CPU_FLAGS,_CIF_ASCE
362 jo .Lsysc_uaccess
363 j .Lsysc_return # beware of critical section cleanup
364
365#
366# _TIF_NEED_RESCHED is set, call schedule
367#
368.Lsysc_reschedule:
369 larl %r14,.Lsysc_return
370 jg schedule
371
372#
373# _CIF_MCCK_PENDING is set, call handler
374#
375.Lsysc_mcck_pending:
376 larl %r14,.Lsysc_return
377 jg s390_handle_mcck # TIF bit will be cleared by handler
378
379#
380# _CIF_ASCE is set, load user space asce
381#
382.Lsysc_uaccess:
383 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE
384 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
385 j .Lsysc_return
386
387#
388# CIF_FPU is set, restore floating-point controls and floating-point registers.
389#
390.Lsysc_vxrs:
391 larl %r14,.Lsysc_return
392 jg load_fpu_regs
393
394#
395# _TIF_SIGPENDING is set, call do_signal
396#
397.Lsysc_sigpending:
398 lgr %r2,%r11 # pass pointer to pt_regs
399 brasl %r14,do_signal
400 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL
401 jno .Lsysc_return
402 lmg %r2,%r7,__PT_R2(%r11) # load svc arguments
403 lghi %r8,0 # svc 0 returns -ENOSYS
404 llgh %r1,__PT_INT_CODE+2(%r11) # load new svc number
405 cghi %r1,NR_syscalls
406 jnl .Lsysc_nr_ok # invalid svc number -> do svc 0
407 slag %r8,%r1,2
408 j .Lsysc_nr_ok # restart svc
409
410#
411# _TIF_NOTIFY_RESUME is set, call do_notify_resume
412#
413.Lsysc_notify_resume:
414 lgr %r2,%r11 # pass pointer to pt_regs
415 larl %r14,.Lsysc_return
416 jg do_notify_resume
417
418#
419# _TIF_UPROBE is set, call uprobe_notify_resume
420#
421#ifdef CONFIG_UPROBES
422.Lsysc_uprobe_notify:
423 lgr %r2,%r11 # pass pointer to pt_regs
424 larl %r14,.Lsysc_return
425 jg uprobe_notify_resume
426#endif
427
428#
429# _PIF_PER_TRAP is set, call do_per_trap
430#
431.Lsysc_singlestep:
432 ni __PT_FLAGS+7(%r11),255-_PIF_PER_TRAP
433 lgr %r2,%r11 # pass pointer to pt_regs
434 larl %r14,.Lsysc_return
435 jg do_per_trap
436
437#
438# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
439# and after the system call
440#
441.Lsysc_tracesys:
442 lgr %r2,%r11 # pass pointer to pt_regs
443 la %r3,0
444 llgh %r0,__PT_INT_CODE+2(%r11)
445 stg %r0,__PT_R2(%r11)
446 brasl %r14,do_syscall_trace_enter
447 lghi %r0,NR_syscalls
448 clgr %r0,%r2
449 jnh .Lsysc_tracenogo
450 sllg %r8,%r2,2
451 lgf %r9,0(%r8,%r10)
452.Lsysc_tracego:
453 lmg %r3,%r7,__PT_R3(%r11)
454 stg %r7,STACK_FRAME_OVERHEAD(%r15)
455 lg %r2,__PT_ORIG_GPR2(%r11)
456 basr %r14,%r9 # call sys_xxx
457 stg %r2,__PT_R2(%r11) # store return value
458.Lsysc_tracenogo:
459 TSTMSK __TI_flags(%r12),_TIF_TRACE
460 jz .Lsysc_return
461 lgr %r2,%r11 # pass pointer to pt_regs
462 larl %r14,.Lsysc_return
463 jg do_syscall_trace_exit
464
465#
466# a new process exits the kernel with ret_from_fork
467#
468ENTRY(ret_from_fork)
469 la %r11,STACK_FRAME_OVERHEAD(%r15)
470 lg %r12,__LC_CURRENT
471 brasl %r14,schedule_tail
472 TRACE_IRQS_ON
473 ssm __LC_SVC_NEW_PSW # reenable interrupts
474 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
475 jne .Lsysc_tracenogo
476 # it's a kernel thread
477 lmg %r9,%r10,__PT_R9(%r11) # load gprs
478ENTRY(kernel_thread_starter)
479 la %r2,0(%r10)
480 basr %r14,%r9
481 j .Lsysc_tracenogo
482
483/*
484 * Program check handler routine
485 */
486
487ENTRY(pgm_check_handler)
488 stpt __LC_SYNC_ENTER_TIMER
489 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
490 lg %r10,__LC_LAST_BREAK
491 lg %r12,__LC_CURRENT
492 larl %r13,cleanup_critical
493 lmg %r8,%r9,__LC_PGM_OLD_PSW
494 tmhh %r8,0x0001 # test problem state bit
495 jnz 2f # -> fault in user space
496#if IS_ENABLED(CONFIG_KVM)
497 # cleanup critical section for sie64a
498 lgr %r14,%r9
499 slg %r14,BASED(.Lsie_critical_start)
500 clg %r14,BASED(.Lsie_critical_length)
501 jhe 0f
502 brasl %r14,.Lcleanup_sie
503#endif
5040: tmhh %r8,0x4000 # PER bit set in old PSW ?
505 jnz 1f # -> enabled, can't be a double fault
506 tm __LC_PGM_ILC+3,0x80 # check for per exception
507 jnz .Lpgm_svcper # -> single stepped svc
5081: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
509 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
510 j 3f
5112: LAST_BREAK %r14
512 UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
513 lg %r15,__LC_KERNEL_STACK
514 lgr %r14,%r12
515 aghi %r14,__TASK_thread # pointer to thread_struct
516 lghi %r13,__LC_PGM_TDB
517 tm __LC_PGM_ILC+2,0x02 # check for transaction abort
518 jz 3f
519 mvc __THREAD_trap_tdb(256,%r14),0(%r13)
5203: la %r11,STACK_FRAME_OVERHEAD(%r15)
521 stmg %r0,%r7,__PT_R0(%r11)
522 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
523 stmg %r8,%r9,__PT_PSW(%r11)
524 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
525 mvc __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE
526 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
527 stg %r10,__PT_ARGS(%r11)
528 tm __LC_PGM_ILC+3,0x80 # check for per exception
529 jz 4f
530 tmhh %r8,0x0001 # kernel per event ?
531 jz .Lpgm_kprobe
532 oi __PT_FLAGS+7(%r11),_PIF_PER_TRAP
533 mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS
534 mvc __THREAD_per_cause(2,%r14),__LC_PER_CODE
535 mvc __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID
5364: REENABLE_IRQS
537 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
538 larl %r1,pgm_check_table
539 llgh %r10,__PT_INT_CODE+2(%r11)
540 nill %r10,0x007f
541 sll %r10,2
542 je .Lpgm_return
543 lgf %r1,0(%r10,%r1) # load address of handler routine
544 lgr %r2,%r11 # pass pointer to pt_regs
545 basr %r14,%r1 # branch to interrupt-handler
546.Lpgm_return:
547 LOCKDEP_SYS_EXIT
548 tm __PT_PSW+1(%r11),0x01 # returning to user ?
549 jno .Lsysc_restore
550 j .Lsysc_tif
551
552#
553# PER event in supervisor state, must be kprobes
554#
555.Lpgm_kprobe:
556 REENABLE_IRQS
557 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
558 lgr %r2,%r11 # pass pointer to pt_regs
559 brasl %r14,do_per_trap
560 j .Lpgm_return
561
562#
563# single stepped system call
564#
565.Lpgm_svcper:
566 mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
567 larl %r14,.Lsysc_per
568 stg %r14,__LC_RETURN_PSW+8
569 lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP
570 lpswe __LC_RETURN_PSW # branch to .Lsysc_per and enable irqs
571
572/*
573 * IO interrupt handler routine
574 */
575ENTRY(io_int_handler)
576 STCK __LC_INT_CLOCK
577 stpt __LC_ASYNC_ENTER_TIMER
578 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
579 lg %r10,__LC_LAST_BREAK
580 lg %r12,__LC_CURRENT
581 larl %r13,cleanup_critical
582 lmg %r8,%r9,__LC_IO_OLD_PSW
583 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
584 stmg %r0,%r7,__PT_R0(%r11)
585 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
586 stmg %r8,%r9,__PT_PSW(%r11)
587 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
588 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
589 TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ
590 jo .Lio_restore
591 TRACE_IRQS_OFF
592 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
593.Lio_loop:
594 lgr %r2,%r11 # pass pointer to pt_regs
595 lghi %r3,IO_INTERRUPT
596 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ?
597 jz .Lio_call
598 lghi %r3,THIN_INTERRUPT
599.Lio_call:
600 brasl %r14,do_IRQ
601 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_LPAR
602 jz .Lio_return
603 tpi 0
604 jz .Lio_return
605 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
606 j .Lio_loop
607.Lio_return:
608 LOCKDEP_SYS_EXIT
609 TRACE_IRQS_ON
610.Lio_tif:
611 TSTMSK __TI_flags(%r12),_TIF_WORK
612 jnz .Lio_work # there is work to do (signals etc.)
613 TSTMSK __LC_CPU_FLAGS,_CIF_WORK
614 jnz .Lio_work
615.Lio_restore:
616 lg %r14,__LC_VDSO_PER_CPU
617 lmg %r0,%r10,__PT_R0(%r11)
618 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
619 stpt __LC_EXIT_TIMER
620 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
621 lmg %r11,%r15,__PT_R11(%r11)
622 lpswe __LC_RETURN_PSW
623.Lio_done:
624
625#
626# There is work todo, find out in which context we have been interrupted:
627# 1) if we return to user space we can do all _TIF_WORK work
628# 2) if we return to kernel code and kvm is enabled check if we need to
629# modify the psw to leave SIE
630# 3) if we return to kernel code and preemptive scheduling is enabled check
631# the preemption counter and if it is zero call preempt_schedule_irq
632# Before any work can be done, a switch to the kernel stack is required.
633#
634.Lio_work:
635 tm __PT_PSW+1(%r11),0x01 # returning to user ?
636 jo .Lio_work_user # yes -> do resched & signal
637#ifdef CONFIG_PREEMPT
638 # check for preemptive scheduling
639 icm %r0,15,__LC_PREEMPT_COUNT
640 jnz .Lio_restore # preemption is disabled
641 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED
642 jno .Lio_restore
643 # switch to kernel stack
644 lg %r1,__PT_R15(%r11)
645 aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
646 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
647 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
648 la %r11,STACK_FRAME_OVERHEAD(%r1)
649 lgr %r15,%r1
650 # TRACE_IRQS_ON already done at .Lio_return, call
651 # TRACE_IRQS_OFF to keep things symmetrical
652 TRACE_IRQS_OFF
653 brasl %r14,preempt_schedule_irq
654 j .Lio_return
655#else
656 j .Lio_restore
657#endif
658
659#
660# Need to do work before returning to userspace, switch to kernel stack
661#
662.Lio_work_user:
663 lg %r1,__LC_KERNEL_STACK
664 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
665 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
666 la %r11,STACK_FRAME_OVERHEAD(%r1)
667 lgr %r15,%r1
668
669#
670# One of the work bits is on. Find out which one.
671#
672.Lio_work_tif:
673 TSTMSK __LC_CPU_FLAGS,_CIF_MCCK_PENDING
674 jo .Lio_mcck_pending
675 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED
676 jo .Lio_reschedule
677 TSTMSK __TI_flags(%r12),_TIF_SIGPENDING
678 jo .Lio_sigpending
679 TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME
680 jo .Lio_notify_resume
681 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
682 jo .Lio_vxrs
683 TSTMSK __LC_CPU_FLAGS,_CIF_ASCE
684 jo .Lio_uaccess
685 j .Lio_return # beware of critical section cleanup
686
687#
688# _CIF_MCCK_PENDING is set, call handler
689#
690.Lio_mcck_pending:
691 # TRACE_IRQS_ON already done at .Lio_return
692 brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler
693 TRACE_IRQS_OFF
694 j .Lio_return
695
696#
697# _CIF_ASCE is set, load user space asce
698#
699.Lio_uaccess:
700 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE
701 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
702 j .Lio_return
703
704#
705# CIF_FPU is set, restore floating-point controls and floating-point registers.
706#
707.Lio_vxrs:
708 larl %r14,.Lio_return
709 jg load_fpu_regs
710
711#
712# _TIF_NEED_RESCHED is set, call schedule
713#
714.Lio_reschedule:
715 # TRACE_IRQS_ON already done at .Lio_return
716 ssm __LC_SVC_NEW_PSW # reenable interrupts
717 brasl %r14,schedule # call scheduler
718 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
719 TRACE_IRQS_OFF
720 j .Lio_return
721
722#
723# _TIF_SIGPENDING or is set, call do_signal
724#
725.Lio_sigpending:
726 # TRACE_IRQS_ON already done at .Lio_return
727 ssm __LC_SVC_NEW_PSW # reenable interrupts
728 lgr %r2,%r11 # pass pointer to pt_regs
729 brasl %r14,do_signal
730 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
731 TRACE_IRQS_OFF
732 j .Lio_return
733
734#
735# _TIF_NOTIFY_RESUME or is set, call do_notify_resume
736#
737.Lio_notify_resume:
738 # TRACE_IRQS_ON already done at .Lio_return
739 ssm __LC_SVC_NEW_PSW # reenable interrupts
740 lgr %r2,%r11 # pass pointer to pt_regs
741 brasl %r14,do_notify_resume
742 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
743 TRACE_IRQS_OFF
744 j .Lio_return
745
746/*
747 * External interrupt handler routine
748 */
749ENTRY(ext_int_handler)
750 STCK __LC_INT_CLOCK
751 stpt __LC_ASYNC_ENTER_TIMER
752 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
753 lg %r10,__LC_LAST_BREAK
754 lg %r12,__LC_CURRENT
755 larl %r13,cleanup_critical
756 lmg %r8,%r9,__LC_EXT_OLD_PSW
757 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
758 stmg %r0,%r7,__PT_R0(%r11)
759 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
760 stmg %r8,%r9,__PT_PSW(%r11)
761 lghi %r1,__LC_EXT_PARAMS2
762 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
763 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
764 mvc __PT_INT_PARM_LONG(8,%r11),0(%r1)
765 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
766 TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ
767 jo .Lio_restore
768 TRACE_IRQS_OFF
769 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
770 lgr %r2,%r11 # pass pointer to pt_regs
771 lghi %r3,EXT_INTERRUPT
772 brasl %r14,do_IRQ
773 j .Lio_return
774
775/*
776 * Load idle PSW. The second "half" of this function is in .Lcleanup_idle.
777 */
778ENTRY(psw_idle)
779 stg %r3,__SF_EMPTY(%r15)
780 larl %r1,.Lpsw_idle_lpsw+4
781 stg %r1,__SF_EMPTY+8(%r15)
782#ifdef CONFIG_SMP
783 larl %r1,smp_cpu_mtid
784 llgf %r1,0(%r1)
785 ltgr %r1,%r1
786 jz .Lpsw_idle_stcctm
787 .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15)
788.Lpsw_idle_stcctm:
789#endif
790 oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
791 STCK __CLOCK_IDLE_ENTER(%r2)
792 stpt __TIMER_IDLE_ENTER(%r2)
793.Lpsw_idle_lpsw:
794 lpswe __SF_EMPTY(%r15)
795 br %r14
796.Lpsw_idle_end:
797
798/*
799 * Store floating-point controls and floating-point or vector register
800 * depending whether the vector facility is available. A critical section
801 * cleanup assures that the registers are stored even if interrupted for
802 * some other work. The CIF_FPU flag is set to trigger a lazy restore
803 * of the register contents at return from io or a system call.
804 */
805ENTRY(save_fpu_regs)
806 lg %r2,__LC_CURRENT
807 aghi %r2,__TASK_thread
808 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
809 bor %r14
810 stfpc __THREAD_FPU_fpc(%r2)
811 lg %r3,__THREAD_FPU_regs(%r2)
812 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
813 jz .Lsave_fpu_regs_fp # no -> store FP regs
814 VSTM %v0,%v15,0,%r3 # vstm 0,15,0(3)
815 VSTM %v16,%v31,256,%r3 # vstm 16,31,256(3)
816 j .Lsave_fpu_regs_done # -> set CIF_FPU flag
817.Lsave_fpu_regs_fp:
818 std 0,0(%r3)
819 std 1,8(%r3)
820 std 2,16(%r3)
821 std 3,24(%r3)
822 std 4,32(%r3)
823 std 5,40(%r3)
824 std 6,48(%r3)
825 std 7,56(%r3)
826 std 8,64(%r3)
827 std 9,72(%r3)
828 std 10,80(%r3)
829 std 11,88(%r3)
830 std 12,96(%r3)
831 std 13,104(%r3)
832 std 14,112(%r3)
833 std 15,120(%r3)
834.Lsave_fpu_regs_done:
835 oi __LC_CPU_FLAGS+7,_CIF_FPU
836 br %r14
837.Lsave_fpu_regs_end:
838#if IS_ENABLED(CONFIG_KVM)
839EXPORT_SYMBOL(save_fpu_regs)
840#endif
841
842/*
843 * Load floating-point controls and floating-point or vector registers.
844 * A critical section cleanup assures that the register contents are
845 * loaded even if interrupted for some other work.
846 *
847 * There are special calling conventions to fit into sysc and io return work:
848 * %r15: <kernel stack>
849 * The function requires:
850 * %r4
851 */
852load_fpu_regs:
853 lg %r4,__LC_CURRENT
854 aghi %r4,__TASK_thread
855 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
856 bnor %r14
857 lfpc __THREAD_FPU_fpc(%r4)
858 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
859 lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area
860 jz .Lload_fpu_regs_fp # -> no VX, load FP regs
861 VLM %v0,%v15,0,%r4
862 VLM %v16,%v31,256,%r4
863 j .Lload_fpu_regs_done
864.Lload_fpu_regs_fp:
865 ld 0,0(%r4)
866 ld 1,8(%r4)
867 ld 2,16(%r4)
868 ld 3,24(%r4)
869 ld 4,32(%r4)
870 ld 5,40(%r4)
871 ld 6,48(%r4)
872 ld 7,56(%r4)
873 ld 8,64(%r4)
874 ld 9,72(%r4)
875 ld 10,80(%r4)
876 ld 11,88(%r4)
877 ld 12,96(%r4)
878 ld 13,104(%r4)
879 ld 14,112(%r4)
880 ld 15,120(%r4)
881.Lload_fpu_regs_done:
882 ni __LC_CPU_FLAGS+7,255-_CIF_FPU
883 br %r14
884.Lload_fpu_regs_end:
885
886.L__critical_end:
887
888/*
889 * Machine check handler routines
890 */
891ENTRY(mcck_int_handler)
892 STCK __LC_MCCK_CLOCK
893 la %r1,4095 # revalidate r1
894 spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer
895 lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
896 lg %r10,__LC_LAST_BREAK
897 lg %r12,__LC_CURRENT
898 larl %r13,cleanup_critical
899 lmg %r8,%r9,__LC_MCK_OLD_PSW
900 TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
901 jo .Lmcck_panic # yes -> rest of mcck code invalid
902 lghi %r14,__LC_CPU_TIMER_SAVE_AREA
903 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
904 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
905 jo 3f
906 la %r14,__LC_SYNC_ENTER_TIMER
907 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
908 jl 0f
909 la %r14,__LC_ASYNC_ENTER_TIMER
9100: clc 0(8,%r14),__LC_EXIT_TIMER
911 jl 1f
912 la %r14,__LC_EXIT_TIMER
9131: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
914 jl 2f
915 la %r14,__LC_LAST_UPDATE_TIMER
9162: spt 0(%r14)
917 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
9183: TSTMSK __LC_MCCK_CODE,(MCCK_CODE_PSW_MWP_VALID|MCCK_CODE_PSW_IA_VALID)
919 jno .Lmcck_panic # no -> skip cleanup critical
920 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER
921.Lmcck_skip:
922 lghi %r14,__LC_GPREGS_SAVE_AREA+64
923 stmg %r0,%r7,__PT_R0(%r11)
924 mvc __PT_R8(64,%r11),0(%r14)
925 stmg %r8,%r9,__PT_PSW(%r11)
926 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
927 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
928 lgr %r2,%r11 # pass pointer to pt_regs
929 brasl %r14,s390_do_machine_check
930 tm __PT_PSW+1(%r11),0x01 # returning to user ?
931 jno .Lmcck_return
932 lg %r1,__LC_KERNEL_STACK # switch to kernel stack
933 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
934 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
935 la %r11,STACK_FRAME_OVERHEAD(%r1)
936 lgr %r15,%r1
937 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
938 TSTMSK __LC_CPU_FLAGS,_CIF_MCCK_PENDING
939 jno .Lmcck_return
940 TRACE_IRQS_OFF
941 brasl %r14,s390_handle_mcck
942 TRACE_IRQS_ON
943.Lmcck_return:
944 lg %r14,__LC_VDSO_PER_CPU
945 lmg %r0,%r10,__PT_R0(%r11)
946 mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
947 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
948 jno 0f
949 stpt __LC_EXIT_TIMER
950 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
9510: lmg %r11,%r15,__PT_R11(%r11)
952 lpswe __LC_RETURN_MCCK_PSW
953
954.Lmcck_panic:
955 lg %r15,__LC_PANIC_STACK
956 la %r11,STACK_FRAME_OVERHEAD(%r15)
957 j .Lmcck_skip
958
959#
960# PSW restart interrupt handler
961#
962ENTRY(restart_int_handler)
963 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_LPP
964 jz 0f
965 .insn s,0xb2800000,__LC_LPP
9660: stg %r15,__LC_SAVE_AREA_RESTART
967 lg %r15,__LC_RESTART_STACK
968 aghi %r15,-__PT_SIZE # create pt_regs on stack
969 xc 0(__PT_SIZE,%r15),0(%r15)
970 stmg %r0,%r14,__PT_R0(%r15)
971 mvc __PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
972 mvc __PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw
973 aghi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
974 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
975 lg %r1,__LC_RESTART_FN # load fn, parm & source cpu
976 lg %r2,__LC_RESTART_DATA
977 lg %r3,__LC_RESTART_SOURCE
978 ltgr %r3,%r3 # test source cpu address
979 jm 1f # negative -> skip source stop
9800: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
981 brc 10,0b # wait for status stored
9821: basr %r14,%r1 # call function
983 stap __SF_EMPTY(%r15) # store cpu address
984 llgh %r3,__SF_EMPTY(%r15)
9852: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
986 brc 2,2b
9873: j 3b
988
989 .section .kprobes.text, "ax"
990
991#ifdef CONFIG_CHECK_STACK
992/*
993 * The synchronous or the asynchronous stack overflowed. We are dead.
994 * No need to properly save the registers, we are going to panic anyway.
995 * Setup a pt_regs so that show_trace can provide a good call trace.
996 */
997stack_overflow:
998 lg %r15,__LC_PANIC_STACK # change to panic stack
999 la %r11,STACK_FRAME_OVERHEAD(%r15)
1000 stmg %r0,%r7,__PT_R0(%r11)
1001 stmg %r8,%r9,__PT_PSW(%r11)
1002 mvc __PT_R8(64,%r11),0(%r14)
1003 stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
1004 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
1005 lgr %r2,%r11 # pass pointer to pt_regs
1006 jg kernel_stack_overflow
1007#endif
1008
1009cleanup_critical:
1010#if IS_ENABLED(CONFIG_KVM)
1011 clg %r9,BASED(.Lcleanup_table_sie) # .Lsie_gmap
1012 jl 0f
1013 clg %r9,BASED(.Lcleanup_table_sie+8)# .Lsie_done
1014 jl .Lcleanup_sie
1015#endif
1016 clg %r9,BASED(.Lcleanup_table) # system_call
1017 jl 0f
1018 clg %r9,BASED(.Lcleanup_table+8) # .Lsysc_do_svc
1019 jl .Lcleanup_system_call
1020 clg %r9,BASED(.Lcleanup_table+16) # .Lsysc_tif
1021 jl 0f
1022 clg %r9,BASED(.Lcleanup_table+24) # .Lsysc_restore
1023 jl .Lcleanup_sysc_tif
1024 clg %r9,BASED(.Lcleanup_table+32) # .Lsysc_done
1025 jl .Lcleanup_sysc_restore
1026 clg %r9,BASED(.Lcleanup_table+40) # .Lio_tif
1027 jl 0f
1028 clg %r9,BASED(.Lcleanup_table+48) # .Lio_restore
1029 jl .Lcleanup_io_tif
1030 clg %r9,BASED(.Lcleanup_table+56) # .Lio_done
1031 jl .Lcleanup_io_restore
1032 clg %r9,BASED(.Lcleanup_table+64) # psw_idle
1033 jl 0f
1034 clg %r9,BASED(.Lcleanup_table+72) # .Lpsw_idle_end
1035 jl .Lcleanup_idle
1036 clg %r9,BASED(.Lcleanup_table+80) # save_fpu_regs
1037 jl 0f
1038 clg %r9,BASED(.Lcleanup_table+88) # .Lsave_fpu_regs_end
1039 jl .Lcleanup_save_fpu_regs
1040 clg %r9,BASED(.Lcleanup_table+96) # load_fpu_regs
1041 jl 0f
1042 clg %r9,BASED(.Lcleanup_table+104) # .Lload_fpu_regs_end
1043 jl .Lcleanup_load_fpu_regs
10440: br %r14
1045
1046 .align 8
1047.Lcleanup_table:
1048 .quad system_call
1049 .quad .Lsysc_do_svc
1050 .quad .Lsysc_tif
1051 .quad .Lsysc_restore
1052 .quad .Lsysc_done
1053 .quad .Lio_tif
1054 .quad .Lio_restore
1055 .quad .Lio_done
1056 .quad psw_idle
1057 .quad .Lpsw_idle_end
1058 .quad save_fpu_regs
1059 .quad .Lsave_fpu_regs_end
1060 .quad load_fpu_regs
1061 .quad .Lload_fpu_regs_end
1062
1063#if IS_ENABLED(CONFIG_KVM)
1064.Lcleanup_table_sie:
1065 .quad .Lsie_gmap
1066 .quad .Lsie_done
1067
1068.Lcleanup_sie:
1069 lg %r9,__SF_EMPTY(%r15) # get control block pointer
1070 ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
1071 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
1072 larl %r9,sie_exit # skip forward to sie_exit
1073 br %r14
1074#endif
1075
1076.Lcleanup_system_call:
1077 # check if stpt has been executed
1078 clg %r9,BASED(.Lcleanup_system_call_insn)
1079 jh 0f
1080 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
1081 cghi %r11,__LC_SAVE_AREA_ASYNC
1082 je 0f
1083 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
10840: # check if stmg has been executed
1085 clg %r9,BASED(.Lcleanup_system_call_insn+8)
1086 jh 0f
1087 mvc __LC_SAVE_AREA_SYNC(64),0(%r11)
10880: # check if base register setup + TIF bit load has been done
1089 clg %r9,BASED(.Lcleanup_system_call_insn+16)
1090 jhe 0f
1091 # set up saved registers r10 and r12
1092 stg %r10,16(%r11) # r10 last break
1093 stg %r12,32(%r11) # r12 task struct pointer
10940: # check if the user time update has been done
1095 clg %r9,BASED(.Lcleanup_system_call_insn+24)
1096 jh 0f
1097 lg %r15,__LC_EXIT_TIMER
1098 slg %r15,__LC_SYNC_ENTER_TIMER
1099 alg %r15,__LC_USER_TIMER
1100 stg %r15,__LC_USER_TIMER
11010: # check if the system time update has been done
1102 clg %r9,BASED(.Lcleanup_system_call_insn+32)
1103 jh 0f
1104 lg %r15,__LC_LAST_UPDATE_TIMER
1105 slg %r15,__LC_EXIT_TIMER
1106 alg %r15,__LC_SYSTEM_TIMER
1107 stg %r15,__LC_SYSTEM_TIMER
11080: # update accounting time stamp
1109 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
1110 # do LAST_BREAK
1111 lg %r9,16(%r11)
1112 srag %r9,%r9,23
1113 jz 0f
1114 lgr %r9,%r12
1115 aghi %r9,__TASK_thread
1116 mvc __THREAD_last_break(8,%r9),16(%r11)
11170: # set up saved register r11
1118 lg %r15,__LC_KERNEL_STACK
1119 la %r9,STACK_FRAME_OVERHEAD(%r15)
1120 stg %r9,24(%r11) # r11 pt_regs pointer
1121 # fill pt_regs
1122 mvc __PT_R8(64,%r9),__LC_SAVE_AREA_SYNC
1123 stmg %r0,%r7,__PT_R0(%r9)
1124 mvc __PT_PSW(16,%r9),__LC_SVC_OLD_PSW
1125 mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
1126 xc __PT_FLAGS(8,%r9),__PT_FLAGS(%r9)
1127 mvi __PT_FLAGS+7(%r9),_PIF_SYSCALL
1128 # setup saved register r15
1129 stg %r15,56(%r11) # r15 stack pointer
1130 # set new psw address and exit
1131 larl %r9,.Lsysc_do_svc
1132 br %r14
1133.Lcleanup_system_call_insn:
1134 .quad system_call
1135 .quad .Lsysc_stmg
1136 .quad .Lsysc_per
1137 .quad .Lsysc_vtime+36
1138 .quad .Lsysc_vtime+42
1139
1140.Lcleanup_sysc_tif:
1141 larl %r9,.Lsysc_tif
1142 br %r14
1143
1144.Lcleanup_sysc_restore:
1145 clg %r9,BASED(.Lcleanup_sysc_restore_insn)
1146 je 0f
1147 lg %r9,24(%r11) # get saved pointer to pt_regs
1148 mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
1149 mvc 0(64,%r11),__PT_R8(%r9)
1150 lmg %r0,%r7,__PT_R0(%r9)
11510: lmg %r8,%r9,__LC_RETURN_PSW
1152 br %r14
1153.Lcleanup_sysc_restore_insn:
1154 .quad .Lsysc_done - 4
1155
1156.Lcleanup_io_tif:
1157 larl %r9,.Lio_tif
1158 br %r14
1159
1160.Lcleanup_io_restore:
1161 clg %r9,BASED(.Lcleanup_io_restore_insn)
1162 je 0f
1163 lg %r9,24(%r11) # get saved r11 pointer to pt_regs
1164 mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
1165 mvc 0(64,%r11),__PT_R8(%r9)
1166 lmg %r0,%r7,__PT_R0(%r9)
11670: lmg %r8,%r9,__LC_RETURN_PSW
1168 br %r14
1169.Lcleanup_io_restore_insn:
1170 .quad .Lio_done - 4
1171
1172.Lcleanup_idle:
1173 ni __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT
1174 # copy interrupt clock & cpu timer
1175 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
1176 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
1177 cghi %r11,__LC_SAVE_AREA_ASYNC
1178 je 0f
1179 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
1180 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
11810: # check if stck & stpt have been executed
1182 clg %r9,BASED(.Lcleanup_idle_insn)
1183 jhe 1f
1184 mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
1185 mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r2)
11861: # calculate idle cycles
1187#ifdef CONFIG_SMP
1188 clg %r9,BASED(.Lcleanup_idle_insn)
1189 jl 3f
1190 larl %r1,smp_cpu_mtid
1191 llgf %r1,0(%r1)
1192 ltgr %r1,%r1
1193 jz 3f
1194 .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15)
1195 larl %r3,mt_cycles
1196 ag %r3,__LC_PERCPU_OFFSET
1197 la %r4,__SF_EMPTY+16(%r15)
11982: lg %r0,0(%r3)
1199 slg %r0,0(%r4)
1200 alg %r0,64(%r4)
1201 stg %r0,0(%r3)
1202 la %r3,8(%r3)
1203 la %r4,8(%r4)
1204 brct %r1,2b
1205#endif
12063: # account system time going idle
1207 lg %r9,__LC_STEAL_TIMER
1208 alg %r9,__CLOCK_IDLE_ENTER(%r2)
1209 slg %r9,__LC_LAST_UPDATE_CLOCK
1210 stg %r9,__LC_STEAL_TIMER
1211 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
1212 lg %r9,__LC_SYSTEM_TIMER
1213 alg %r9,__LC_LAST_UPDATE_TIMER
1214 slg %r9,__TIMER_IDLE_ENTER(%r2)
1215 stg %r9,__LC_SYSTEM_TIMER
1216 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
1217 # prepare return psw
1218 nihh %r8,0xfcfd # clear irq & wait state bits
1219 lg %r9,48(%r11) # return from psw_idle
1220 br %r14
1221.Lcleanup_idle_insn:
1222 .quad .Lpsw_idle_lpsw
1223
1224.Lcleanup_save_fpu_regs:
1225 larl %r9,save_fpu_regs
1226 br %r14
1227
1228.Lcleanup_load_fpu_regs:
1229 larl %r9,load_fpu_regs
1230 br %r14
1231
1232/*
1233 * Integer constants
1234 */
1235 .align 8
1236.Lcritical_start:
1237 .quad .L__critical_start
1238.Lcritical_length:
1239 .quad .L__critical_end - .L__critical_start
1240#if IS_ENABLED(CONFIG_KVM)
1241.Lsie_critical_start:
1242 .quad .Lsie_gmap
1243.Lsie_critical_length:
1244 .quad .Lsie_done - .Lsie_gmap
1245#endif
1246
1247 .section .rodata, "a"
1248#define SYSCALL(esame,emu) .long esame
1249 .globl sys_call_table
1250sys_call_table:
1251#include "syscalls.S"
1252#undef SYSCALL
1253
1254#ifdef CONFIG_COMPAT
1255
1256#define SYSCALL(esame,emu) .long emu
1257 .globl sys_call_table_emu
1258sys_call_table_emu:
1259#include "syscalls.S"
1260#undef SYSCALL
1261#endif
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * S390 low-level entry points.
4 *
5 * Copyright IBM Corp. 1999, 2012
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 */
11
12#include <linux/init.h>
13#include <linux/linkage.h>
14#include <asm/alternative-asm.h>
15#include <asm/processor.h>
16#include <asm/cache.h>
17#include <asm/ctl_reg.h>
18#include <asm/dwarf.h>
19#include <asm/errno.h>
20#include <asm/ptrace.h>
21#include <asm/thread_info.h>
22#include <asm/asm-offsets.h>
23#include <asm/unistd.h>
24#include <asm/page.h>
25#include <asm/sigp.h>
26#include <asm/irq.h>
27#include <asm/vx-insn.h>
28#include <asm/setup.h>
29#include <asm/nmi.h>
30#include <asm/export.h>
31#include <asm/nospec-insn.h>
32
33__PT_R0 = __PT_GPRS
34__PT_R1 = __PT_GPRS + 8
35__PT_R2 = __PT_GPRS + 16
36__PT_R3 = __PT_GPRS + 24
37__PT_R4 = __PT_GPRS + 32
38__PT_R5 = __PT_GPRS + 40
39__PT_R6 = __PT_GPRS + 48
40__PT_R7 = __PT_GPRS + 56
41__PT_R8 = __PT_GPRS + 64
42__PT_R9 = __PT_GPRS + 72
43__PT_R10 = __PT_GPRS + 80
44__PT_R11 = __PT_GPRS + 88
45__PT_R12 = __PT_GPRS + 96
46__PT_R13 = __PT_GPRS + 104
47__PT_R14 = __PT_GPRS + 112
48__PT_R15 = __PT_GPRS + 120
49
50STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
51STACK_SIZE = 1 << STACK_SHIFT
52STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
53
54_TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
55 _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING)
56_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
57 _TIF_SYSCALL_TRACEPOINT)
58_CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE_PRIMARY | \
59 _CIF_ASCE_SECONDARY | _CIF_FPU)
60_PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
61
62_LPP_OFFSET = __LC_LPP
63
64#define BASED(name) name-cleanup_critical(%r13)
65
66 .macro TRACE_IRQS_ON
67#ifdef CONFIG_TRACE_IRQFLAGS
68 basr %r2,%r0
69 brasl %r14,trace_hardirqs_on_caller
70#endif
71 .endm
72
73 .macro TRACE_IRQS_OFF
74#ifdef CONFIG_TRACE_IRQFLAGS
75 basr %r2,%r0
76 brasl %r14,trace_hardirqs_off_caller
77#endif
78 .endm
79
80 .macro LOCKDEP_SYS_EXIT
81#ifdef CONFIG_LOCKDEP
82 tm __PT_PSW+1(%r11),0x01 # returning to user ?
83 jz .+10
84 brasl %r14,lockdep_sys_exit
85#endif
86 .endm
87
88 .macro CHECK_STACK stacksize,savearea
89#ifdef CONFIG_CHECK_STACK
90 tml %r15,\stacksize - CONFIG_STACK_GUARD
91 lghi %r14,\savearea
92 jz stack_overflow
93#endif
94 .endm
95
96 .macro SWITCH_ASYNC savearea,timer
97 tmhh %r8,0x0001 # interrupting from user ?
98 jnz 1f
99 lgr %r14,%r9
100 slg %r14,BASED(.Lcritical_start)
101 clg %r14,BASED(.Lcritical_length)
102 jhe 0f
103 lghi %r11,\savearea # inside critical section, do cleanup
104 brasl %r14,cleanup_critical
105 tmhh %r8,0x0001 # retest problem state after cleanup
106 jnz 1f
1070: lg %r14,__LC_ASYNC_STACK # are we already on the async stack?
108 slgr %r14,%r15
109 srag %r14,%r14,STACK_SHIFT
110 jnz 2f
111 CHECK_STACK 1<<STACK_SHIFT,\savearea
112 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
113 j 3f
1141: UPDATE_VTIME %r14,%r15,\timer
115 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
1162: lg %r15,__LC_ASYNC_STACK # load async stack
1173: la %r11,STACK_FRAME_OVERHEAD(%r15)
118 .endm
119
120 .macro UPDATE_VTIME w1,w2,enter_timer
121 lg \w1,__LC_EXIT_TIMER
122 lg \w2,__LC_LAST_UPDATE_TIMER
123 slg \w1,\enter_timer
124 slg \w2,__LC_EXIT_TIMER
125 alg \w1,__LC_USER_TIMER
126 alg \w2,__LC_SYSTEM_TIMER
127 stg \w1,__LC_USER_TIMER
128 stg \w2,__LC_SYSTEM_TIMER
129 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
130 .endm
131
132 .macro REENABLE_IRQS
133 stg %r8,__LC_RETURN_PSW
134 ni __LC_RETURN_PSW,0xbf
135 ssm __LC_RETURN_PSW
136 .endm
137
138 .macro STCK savearea
139#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
140 .insn s,0xb27c0000,\savearea # store clock fast
141#else
142 .insn s,0xb2050000,\savearea # store clock
143#endif
144 .endm
145
146 /*
147 * The TSTMSK macro generates a test-under-mask instruction by
148 * calculating the memory offset for the specified mask value.
149 * Mask value can be any constant. The macro shifts the mask
150 * value to calculate the memory offset for the test-under-mask
151 * instruction.
152 */
153 .macro TSTMSK addr, mask, size=8, bytepos=0
154 .if (\bytepos < \size) && (\mask >> 8)
155 .if (\mask & 0xff)
156 .error "Mask exceeds byte boundary"
157 .endif
158 TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
159 .exitm
160 .endif
161 .ifeq \mask
162 .error "Mask must not be zero"
163 .endif
164 off = \size - \bytepos - 1
165 tm off+\addr, \mask
166 .endm
167
168 .macro BPOFF
169 ALTERNATIVE "", ".long 0xb2e8c000", 82
170 .endm
171
172 .macro BPON
173 ALTERNATIVE "", ".long 0xb2e8d000", 82
174 .endm
175
176 .macro BPENTER tif_ptr,tif_mask
177 ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \
178 "", 82
179 .endm
180
181 .macro BPEXIT tif_ptr,tif_mask
182 TSTMSK \tif_ptr,\tif_mask
183 ALTERNATIVE "jz .+8; .long 0xb2e8c000", \
184 "jnz .+8; .long 0xb2e8d000", 82
185 .endm
186
187 GEN_BR_THUNK %r9
188 GEN_BR_THUNK %r14
189 GEN_BR_THUNK %r14,%r11
190
191 .section .kprobes.text, "ax"
192.Ldummy:
193 /*
194 * This nop exists only in order to avoid that __switch_to starts at
195 * the beginning of the kprobes text section. In that case we would
196 * have several symbols at the same address. E.g. objdump would take
197 * an arbitrary symbol name when disassembling this code.
198 * With the added nop in between the __switch_to symbol is unique
199 * again.
200 */
201 nop 0
202
203ENTRY(__bpon)
204 .globl __bpon
205 BPON
206 BR_EX %r14
207
208/*
209 * Scheduler resume function, called by switch_to
210 * gpr2 = (task_struct *) prev
211 * gpr3 = (task_struct *) next
212 * Returns:
213 * gpr2 = prev
214 */
215ENTRY(__switch_to)
216 stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
217 lghi %r4,__TASK_stack
218 lghi %r1,__TASK_thread
219 lg %r5,0(%r4,%r3) # start of kernel stack of next
220 stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev
221 lgr %r15,%r5
222 aghi %r15,STACK_INIT # end of kernel stack of next
223 stg %r3,__LC_CURRENT # store task struct of next
224 stg %r15,__LC_KERNEL_STACK # store end of kernel stack
225 lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next
226 aghi %r3,__TASK_pid
227 mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next
228 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
229 ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
230 BR_EX %r14
231
232.L__critical_start:
233
234#if IS_ENABLED(CONFIG_KVM)
235/*
236 * sie64a calling convention:
237 * %r2 pointer to sie control block
238 * %r3 guest register save area
239 */
240ENTRY(sie64a)
241 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
242 lg %r12,__LC_CURRENT
243 stg %r2,__SF_SIE_CONTROL(%r15) # save control block pointer
244 stg %r3,__SF_SIE_SAVEAREA(%r15) # save guest register save area
245 xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
246 mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
247 TSTMSK __LC_CPU_FLAGS,_CIF_FPU # load guest fp/vx registers ?
248 jno .Lsie_load_guest_gprs
249 brasl %r14,load_fpu_regs # load guest fp/vx regs
250.Lsie_load_guest_gprs:
251 lmg %r0,%r13,0(%r3) # load guest gprs 0-13
252 lg %r14,__LC_GMAP # get gmap pointer
253 ltgr %r14,%r14
254 jz .Lsie_gmap
255 lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce
256.Lsie_gmap:
257 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
258 oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
259 tm __SIE_PROG20+3(%r14),3 # last exit...
260 jnz .Lsie_skip
261 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
262 jo .Lsie_skip # exit if fp/vx regs changed
263 BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
264.Lsie_entry:
265 sie 0(%r14)
266.Lsie_exit:
267 BPOFF
268 BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
269.Lsie_skip:
270 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
271 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
272.Lsie_done:
273# some program checks are suppressing. C code (e.g. do_protection_exception)
274# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
275# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
276# Other instructions between sie64a and .Lsie_done should not cause program
277# interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
278# See also .Lcleanup_sie
279.Lrewind_pad6:
280 nopr 7
281.Lrewind_pad4:
282 nopr 7
283.Lrewind_pad2:
284 nopr 7
285 .globl sie_exit
286sie_exit:
287 lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area
288 stmg %r0,%r13,0(%r14) # save guest gprs 0-13
289 xgr %r0,%r0 # clear guest registers to
290 xgr %r1,%r1 # prevent speculative use
291 xgr %r2,%r2
292 xgr %r3,%r3
293 xgr %r4,%r4
294 xgr %r5,%r5
295 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
296 lg %r2,__SF_SIE_REASON(%r15) # return exit reason code
297 BR_EX %r14
298.Lsie_fault:
299 lghi %r14,-EFAULT
300 stg %r14,__SF_SIE_REASON(%r15) # set exit reason code
301 j sie_exit
302
303 EX_TABLE(.Lrewind_pad6,.Lsie_fault)
304 EX_TABLE(.Lrewind_pad4,.Lsie_fault)
305 EX_TABLE(.Lrewind_pad2,.Lsie_fault)
306 EX_TABLE(sie_exit,.Lsie_fault)
307EXPORT_SYMBOL(sie64a)
308EXPORT_SYMBOL(sie_exit)
309#endif
310
311/*
312 * SVC interrupt handler routine. System calls are synchronous events and
313 * are executed with interrupts enabled.
314 */
315
316ENTRY(system_call)
317 stpt __LC_SYNC_ENTER_TIMER
318.Lsysc_stmg:
319 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
320 BPOFF
321 lg %r12,__LC_CURRENT
322 lghi %r13,__TASK_thread
323 lghi %r14,_PIF_SYSCALL
324.Lsysc_per:
325 lg %r15,__LC_KERNEL_STACK
326 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
327.Lsysc_vtime:
328 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
329 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
330 stmg %r0,%r7,__PT_R0(%r11)
331 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
332 mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW
333 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
334 stg %r14,__PT_FLAGS(%r11)
335.Lsysc_do_svc:
336 # clear user controlled register to prevent speculative use
337 xgr %r0,%r0
338 # load address of system call table
339 lg %r10,__THREAD_sysc_table(%r13,%r12)
340 llgh %r8,__PT_INT_CODE+2(%r11)
341 slag %r8,%r8,2 # shift and test for svc 0
342 jnz .Lsysc_nr_ok
343 # svc 0: system call number in %r1
344 llgfr %r1,%r1 # clear high word in r1
345 cghi %r1,NR_syscalls
346 jnl .Lsysc_nr_ok
347 sth %r1,__PT_INT_CODE+2(%r11)
348 slag %r8,%r1,2
349.Lsysc_nr_ok:
350 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
351 stg %r2,__PT_ORIG_GPR2(%r11)
352 stg %r7,STACK_FRAME_OVERHEAD(%r15)
353 lgf %r9,0(%r8,%r10) # get system call add.
354 TSTMSK __TI_flags(%r12),_TIF_TRACE
355 jnz .Lsysc_tracesys
356 BASR_EX %r14,%r9 # call sys_xxxx
357 stg %r2,__PT_R2(%r11) # store return value
358
359.Lsysc_return:
360 LOCKDEP_SYS_EXIT
361.Lsysc_tif:
362 TSTMSK __PT_FLAGS(%r11),_PIF_WORK
363 jnz .Lsysc_work
364 TSTMSK __TI_flags(%r12),_TIF_WORK
365 jnz .Lsysc_work # check for work
366 TSTMSK __LC_CPU_FLAGS,_CIF_WORK
367 jnz .Lsysc_work
368 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
369.Lsysc_restore:
370 lg %r14,__LC_VDSO_PER_CPU
371 lmg %r0,%r10,__PT_R0(%r11)
372 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
373.Lsysc_exit_timer:
374 stpt __LC_EXIT_TIMER
375 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
376 lmg %r11,%r15,__PT_R11(%r11)
377 lpswe __LC_RETURN_PSW
378.Lsysc_done:
379
380#
381# One of the work bits is on. Find out which one.
382#
383.Lsysc_work:
384 TSTMSK __LC_CPU_FLAGS,_CIF_MCCK_PENDING
385 jo .Lsysc_mcck_pending
386 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED
387 jo .Lsysc_reschedule
388 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
389 jo .Lsysc_syscall_restart
390#ifdef CONFIG_UPROBES
391 TSTMSK __TI_flags(%r12),_TIF_UPROBE
392 jo .Lsysc_uprobe_notify
393#endif
394 TSTMSK __TI_flags(%r12),_TIF_GUARDED_STORAGE
395 jo .Lsysc_guarded_storage
396 TSTMSK __PT_FLAGS(%r11),_PIF_PER_TRAP
397 jo .Lsysc_singlestep
398#ifdef CONFIG_LIVEPATCH
399 TSTMSK __TI_flags(%r12),_TIF_PATCH_PENDING
400 jo .Lsysc_patch_pending # handle live patching just before
401 # signals and possible syscall restart
402#endif
403 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
404 jo .Lsysc_syscall_restart
405 TSTMSK __TI_flags(%r12),_TIF_SIGPENDING
406 jo .Lsysc_sigpending
407 TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME
408 jo .Lsysc_notify_resume
409 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
410 jo .Lsysc_vxrs
411 TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY)
412 jnz .Lsysc_asce
413 j .Lsysc_return # beware of critical section cleanup
414
415#
416# _TIF_NEED_RESCHED is set, call schedule
417#
418.Lsysc_reschedule:
419 larl %r14,.Lsysc_return
420 jg schedule
421
422#
423# _CIF_MCCK_PENDING is set, call handler
424#
425.Lsysc_mcck_pending:
426 larl %r14,.Lsysc_return
427 jg s390_handle_mcck # TIF bit will be cleared by handler
428
429#
430# _CIF_ASCE_PRIMARY and/or _CIF_ASCE_SECONDARY set, load user space asce
431#
432.Lsysc_asce:
433 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY
434 lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce
435 TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY
436 jz .Lsysc_return
437#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
438 tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ?
439 jnz .Lsysc_set_fs_fixup
440 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY
441 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
442 j .Lsysc_return
443.Lsysc_set_fs_fixup:
444#endif
445 larl %r14,.Lsysc_return
446 jg set_fs_fixup
447
448#
449# CIF_FPU is set, restore floating-point controls and floating-point registers.
450#
451.Lsysc_vxrs:
452 larl %r14,.Lsysc_return
453 jg load_fpu_regs
454
455#
456# _TIF_SIGPENDING is set, call do_signal
457#
458.Lsysc_sigpending:
459 lgr %r2,%r11 # pass pointer to pt_regs
460 brasl %r14,do_signal
461 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL
462 jno .Lsysc_return
463.Lsysc_do_syscall:
464 lghi %r13,__TASK_thread
465 lmg %r2,%r7,__PT_R2(%r11) # load svc arguments
466 lghi %r1,0 # svc 0 returns -ENOSYS
467 j .Lsysc_do_svc
468
469#
470# _TIF_NOTIFY_RESUME is set, call do_notify_resume
471#
472.Lsysc_notify_resume:
473 lgr %r2,%r11 # pass pointer to pt_regs
474 larl %r14,.Lsysc_return
475 jg do_notify_resume
476
477#
478# _TIF_UPROBE is set, call uprobe_notify_resume
479#
480#ifdef CONFIG_UPROBES
481.Lsysc_uprobe_notify:
482 lgr %r2,%r11 # pass pointer to pt_regs
483 larl %r14,.Lsysc_return
484 jg uprobe_notify_resume
485#endif
486
487#
488# _TIF_GUARDED_STORAGE is set, call guarded_storage_load
489#
490.Lsysc_guarded_storage:
491 lgr %r2,%r11 # pass pointer to pt_regs
492 larl %r14,.Lsysc_return
493 jg gs_load_bc_cb
494#
495# _TIF_PATCH_PENDING is set, call klp_update_patch_state
496#
497#ifdef CONFIG_LIVEPATCH
498.Lsysc_patch_pending:
499 lg %r2,__LC_CURRENT # pass pointer to task struct
500 larl %r14,.Lsysc_return
501 jg klp_update_patch_state
502#endif
503
504#
505# _PIF_PER_TRAP is set, call do_per_trap
506#
507.Lsysc_singlestep:
508 ni __PT_FLAGS+7(%r11),255-_PIF_PER_TRAP
509 lgr %r2,%r11 # pass pointer to pt_regs
510 larl %r14,.Lsysc_return
511 jg do_per_trap
512
513#
514# _PIF_SYSCALL_RESTART is set, repeat the current system call
515#
516.Lsysc_syscall_restart:
517 ni __PT_FLAGS+7(%r11),255-_PIF_SYSCALL_RESTART
518 lmg %r1,%r7,__PT_R1(%r11) # load svc arguments
519 lg %r2,__PT_ORIG_GPR2(%r11)
520 j .Lsysc_do_svc
521
522#
523# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
524# and after the system call
525#
526.Lsysc_tracesys:
527 lgr %r2,%r11 # pass pointer to pt_regs
528 la %r3,0
529 llgh %r0,__PT_INT_CODE+2(%r11)
530 stg %r0,__PT_R2(%r11)
531 brasl %r14,do_syscall_trace_enter
532 lghi %r0,NR_syscalls
533 clgr %r0,%r2
534 jnh .Lsysc_tracenogo
535 sllg %r8,%r2,2
536 lgf %r9,0(%r8,%r10)
537.Lsysc_tracego:
538 lmg %r3,%r7,__PT_R3(%r11)
539 stg %r7,STACK_FRAME_OVERHEAD(%r15)
540 lg %r2,__PT_ORIG_GPR2(%r11)
541 BASR_EX %r14,%r9 # call sys_xxx
542 stg %r2,__PT_R2(%r11) # store return value
543.Lsysc_tracenogo:
544 TSTMSK __TI_flags(%r12),_TIF_TRACE
545 jz .Lsysc_return
546 lgr %r2,%r11 # pass pointer to pt_regs
547 larl %r14,.Lsysc_return
548 jg do_syscall_trace_exit
549
550#
551# a new process exits the kernel with ret_from_fork
552#
553ENTRY(ret_from_fork)
554 la %r11,STACK_FRAME_OVERHEAD(%r15)
555 lg %r12,__LC_CURRENT
556 brasl %r14,schedule_tail
557 TRACE_IRQS_ON
558 ssm __LC_SVC_NEW_PSW # reenable interrupts
559 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
560 jne .Lsysc_tracenogo
561 # it's a kernel thread
562 lmg %r9,%r10,__PT_R9(%r11) # load gprs
563ENTRY(kernel_thread_starter)
564 la %r2,0(%r10)
565 BASR_EX %r14,%r9
566 j .Lsysc_tracenogo
567
568/*
569 * Program check handler routine
570 */
571
572ENTRY(pgm_check_handler)
573 stpt __LC_SYNC_ENTER_TIMER
574 BPOFF
575 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
576 lg %r10,__LC_LAST_BREAK
577 lg %r12,__LC_CURRENT
578 lghi %r11,0
579 larl %r13,cleanup_critical
580 lmg %r8,%r9,__LC_PGM_OLD_PSW
581 tmhh %r8,0x0001 # test problem state bit
582 jnz 2f # -> fault in user space
583#if IS_ENABLED(CONFIG_KVM)
584 # cleanup critical section for program checks in sie64a
585 lgr %r14,%r9
586 slg %r14,BASED(.Lsie_critical_start)
587 clg %r14,BASED(.Lsie_critical_length)
588 jhe 0f
589 lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
590 ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
591 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
592 larl %r9,sie_exit # skip forward to sie_exit
593 lghi %r11,_PIF_GUEST_FAULT
594#endif
5950: tmhh %r8,0x4000 # PER bit set in old PSW ?
596 jnz 1f # -> enabled, can't be a double fault
597 tm __LC_PGM_ILC+3,0x80 # check for per exception
598 jnz .Lpgm_svcper # -> single stepped svc
5991: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
600 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
601 j 4f
6022: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
603 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
604 lg %r15,__LC_KERNEL_STACK
605 lgr %r14,%r12
606 aghi %r14,__TASK_thread # pointer to thread_struct
607 lghi %r13,__LC_PGM_TDB
608 tm __LC_PGM_ILC+2,0x02 # check for transaction abort
609 jz 3f
610 mvc __THREAD_trap_tdb(256,%r14),0(%r13)
6113: stg %r10,__THREAD_last_break(%r14)
6124: lgr %r13,%r11
613 la %r11,STACK_FRAME_OVERHEAD(%r15)
614 stmg %r0,%r7,__PT_R0(%r11)
615 # clear user controlled registers to prevent speculative use
616 xgr %r0,%r0
617 xgr %r1,%r1
618 xgr %r2,%r2
619 xgr %r3,%r3
620 xgr %r4,%r4
621 xgr %r5,%r5
622 xgr %r6,%r6
623 xgr %r7,%r7
624 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
625 stmg %r8,%r9,__PT_PSW(%r11)
626 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
627 mvc __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE
628 stg %r13,__PT_FLAGS(%r11)
629 stg %r10,__PT_ARGS(%r11)
630 tm __LC_PGM_ILC+3,0x80 # check for per exception
631 jz 5f
632 tmhh %r8,0x0001 # kernel per event ?
633 jz .Lpgm_kprobe
634 oi __PT_FLAGS+7(%r11),_PIF_PER_TRAP
635 mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS
636 mvc __THREAD_per_cause(2,%r14),__LC_PER_CODE
637 mvc __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID
6385: REENABLE_IRQS
639 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
640 larl %r1,pgm_check_table
641 llgh %r10,__PT_INT_CODE+2(%r11)
642 nill %r10,0x007f
643 sll %r10,2
644 je .Lpgm_return
645 lgf %r9,0(%r10,%r1) # load address of handler routine
646 lgr %r2,%r11 # pass pointer to pt_regs
647 BASR_EX %r14,%r9 # branch to interrupt-handler
648.Lpgm_return:
649 LOCKDEP_SYS_EXIT
650 tm __PT_PSW+1(%r11),0x01 # returning to user ?
651 jno .Lsysc_restore
652 TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL
653 jo .Lsysc_do_syscall
654 j .Lsysc_tif
655
656#
657# PER event in supervisor state, must be kprobes
658#
659.Lpgm_kprobe:
660 REENABLE_IRQS
661 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
662 lgr %r2,%r11 # pass pointer to pt_regs
663 brasl %r14,do_per_trap
664 j .Lpgm_return
665
666#
667# single stepped system call
668#
669.Lpgm_svcper:
670 mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
671 lghi %r13,__TASK_thread
672 larl %r14,.Lsysc_per
673 stg %r14,__LC_RETURN_PSW+8
674 lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP
675 lpswe __LC_RETURN_PSW # branch to .Lsysc_per and enable irqs
676
677/*
678 * IO interrupt handler routine
679 */
680ENTRY(io_int_handler)
681 STCK __LC_INT_CLOCK
682 stpt __LC_ASYNC_ENTER_TIMER
683 BPOFF
684 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
685 lg %r12,__LC_CURRENT
686 larl %r13,cleanup_critical
687 lmg %r8,%r9,__LC_IO_OLD_PSW
688 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
689 stmg %r0,%r7,__PT_R0(%r11)
690 # clear user controlled registers to prevent speculative use
691 xgr %r0,%r0
692 xgr %r1,%r1
693 xgr %r2,%r2
694 xgr %r3,%r3
695 xgr %r4,%r4
696 xgr %r5,%r5
697 xgr %r6,%r6
698 xgr %r7,%r7
699 xgr %r10,%r10
700 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
701 stmg %r8,%r9,__PT_PSW(%r11)
702 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
703 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
704 TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ
705 jo .Lio_restore
706 TRACE_IRQS_OFF
707 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
708.Lio_loop:
709 lgr %r2,%r11 # pass pointer to pt_regs
710 lghi %r3,IO_INTERRUPT
711 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ?
712 jz .Lio_call
713 lghi %r3,THIN_INTERRUPT
714.Lio_call:
715 brasl %r14,do_IRQ
716 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_LPAR
717 jz .Lio_return
718 tpi 0
719 jz .Lio_return
720 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
721 j .Lio_loop
722.Lio_return:
723 LOCKDEP_SYS_EXIT
724 TRACE_IRQS_ON
725.Lio_tif:
726 TSTMSK __TI_flags(%r12),_TIF_WORK
727 jnz .Lio_work # there is work to do (signals etc.)
728 TSTMSK __LC_CPU_FLAGS,_CIF_WORK
729 jnz .Lio_work
730.Lio_restore:
731 lg %r14,__LC_VDSO_PER_CPU
732 lmg %r0,%r10,__PT_R0(%r11)
733 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
734 tm __PT_PSW+1(%r11),0x01 # returning to user ?
735 jno .Lio_exit_kernel
736 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
737.Lio_exit_timer:
738 stpt __LC_EXIT_TIMER
739 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
740.Lio_exit_kernel:
741 lmg %r11,%r15,__PT_R11(%r11)
742 lpswe __LC_RETURN_PSW
743.Lio_done:
744
745#
746# There is work todo, find out in which context we have been interrupted:
747# 1) if we return to user space we can do all _TIF_WORK work
748# 2) if we return to kernel code and kvm is enabled check if we need to
749# modify the psw to leave SIE
750# 3) if we return to kernel code and preemptive scheduling is enabled check
751# the preemption counter and if it is zero call preempt_schedule_irq
752# Before any work can be done, a switch to the kernel stack is required.
753#
754.Lio_work:
755 tm __PT_PSW+1(%r11),0x01 # returning to user ?
756 jo .Lio_work_user # yes -> do resched & signal
757#ifdef CONFIG_PREEMPT
758 # check for preemptive scheduling
759 icm %r0,15,__LC_PREEMPT_COUNT
760 jnz .Lio_restore # preemption is disabled
761 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED
762 jno .Lio_restore
763 # switch to kernel stack
764 lg %r1,__PT_R15(%r11)
765 aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
766 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
767 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
768 la %r11,STACK_FRAME_OVERHEAD(%r1)
769 lgr %r15,%r1
770 # TRACE_IRQS_ON already done at .Lio_return, call
771 # TRACE_IRQS_OFF to keep things symmetrical
772 TRACE_IRQS_OFF
773 brasl %r14,preempt_schedule_irq
774 j .Lio_return
775#else
776 j .Lio_restore
777#endif
778
779#
780# Need to do work before returning to userspace, switch to kernel stack
781#
782.Lio_work_user:
783 lg %r1,__LC_KERNEL_STACK
784 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
785 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
786 la %r11,STACK_FRAME_OVERHEAD(%r1)
787 lgr %r15,%r1
788
789#
790# One of the work bits is on. Find out which one.
791#
792.Lio_work_tif:
793 TSTMSK __LC_CPU_FLAGS,_CIF_MCCK_PENDING
794 jo .Lio_mcck_pending
795 TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED
796 jo .Lio_reschedule
797#ifdef CONFIG_LIVEPATCH
798 TSTMSK __TI_flags(%r12),_TIF_PATCH_PENDING
799 jo .Lio_patch_pending
800#endif
801 TSTMSK __TI_flags(%r12),_TIF_SIGPENDING
802 jo .Lio_sigpending
803 TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME
804 jo .Lio_notify_resume
805 TSTMSK __TI_flags(%r12),_TIF_GUARDED_STORAGE
806 jo .Lio_guarded_storage
807 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
808 jo .Lio_vxrs
809 TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY)
810 jnz .Lio_asce
811 j .Lio_return # beware of critical section cleanup
812
813#
814# _CIF_MCCK_PENDING is set, call handler
815#
816.Lio_mcck_pending:
817 # TRACE_IRQS_ON already done at .Lio_return
818 brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler
819 TRACE_IRQS_OFF
820 j .Lio_return
821
822#
823# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce
824#
825.Lio_asce:
826 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY
827 lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce
828 TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY
829 jz .Lio_return
830#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
831 tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ?
832 jnz .Lio_set_fs_fixup
833 ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY
834 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
835 j .Lio_return
836.Lio_set_fs_fixup:
837#endif
838 larl %r14,.Lio_return
839 jg set_fs_fixup
840
841#
842# CIF_FPU is set, restore floating-point controls and floating-point registers.
843#
844.Lio_vxrs:
845 larl %r14,.Lio_return
846 jg load_fpu_regs
847
848#
849# _TIF_GUARDED_STORAGE is set, call guarded_storage_load
850#
851.Lio_guarded_storage:
852 # TRACE_IRQS_ON already done at .Lio_return
853 ssm __LC_SVC_NEW_PSW # reenable interrupts
854 lgr %r2,%r11 # pass pointer to pt_regs
855 brasl %r14,gs_load_bc_cb
856 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
857 TRACE_IRQS_OFF
858 j .Lio_return
859
860#
861# _TIF_NEED_RESCHED is set, call schedule
862#
863.Lio_reschedule:
864 # TRACE_IRQS_ON already done at .Lio_return
865 ssm __LC_SVC_NEW_PSW # reenable interrupts
866 brasl %r14,schedule # call scheduler
867 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
868 TRACE_IRQS_OFF
869 j .Lio_return
870
871#
872# _TIF_PATCH_PENDING is set, call klp_update_patch_state
873#
874#ifdef CONFIG_LIVEPATCH
875.Lio_patch_pending:
876 lg %r2,__LC_CURRENT # pass pointer to task struct
877 larl %r14,.Lio_return
878 jg klp_update_patch_state
879#endif
880
881#
882# _TIF_SIGPENDING or is set, call do_signal
883#
884.Lio_sigpending:
885 # TRACE_IRQS_ON already done at .Lio_return
886 ssm __LC_SVC_NEW_PSW # reenable interrupts
887 lgr %r2,%r11 # pass pointer to pt_regs
888 brasl %r14,do_signal
889 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
890 TRACE_IRQS_OFF
891 j .Lio_return
892
893#
894# _TIF_NOTIFY_RESUME or is set, call do_notify_resume
895#
896.Lio_notify_resume:
897 # TRACE_IRQS_ON already done at .Lio_return
898 ssm __LC_SVC_NEW_PSW # reenable interrupts
899 lgr %r2,%r11 # pass pointer to pt_regs
900 brasl %r14,do_notify_resume
901 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
902 TRACE_IRQS_OFF
903 j .Lio_return
904
905/*
906 * External interrupt handler routine
907 */
908ENTRY(ext_int_handler)
909 STCK __LC_INT_CLOCK
910 stpt __LC_ASYNC_ENTER_TIMER
911 BPOFF
912 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
913 lg %r12,__LC_CURRENT
914 larl %r13,cleanup_critical
915 lmg %r8,%r9,__LC_EXT_OLD_PSW
916 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
917 stmg %r0,%r7,__PT_R0(%r11)
918 # clear user controlled registers to prevent speculative use
919 xgr %r0,%r0
920 xgr %r1,%r1
921 xgr %r2,%r2
922 xgr %r3,%r3
923 xgr %r4,%r4
924 xgr %r5,%r5
925 xgr %r6,%r6
926 xgr %r7,%r7
927 xgr %r10,%r10
928 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
929 stmg %r8,%r9,__PT_PSW(%r11)
930 lghi %r1,__LC_EXT_PARAMS2
931 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
932 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
933 mvc __PT_INT_PARM_LONG(8,%r11),0(%r1)
934 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
935 TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ
936 jo .Lio_restore
937 TRACE_IRQS_OFF
938 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
939 lgr %r2,%r11 # pass pointer to pt_regs
940 lghi %r3,EXT_INTERRUPT
941 brasl %r14,do_IRQ
942 j .Lio_return
943
944/*
945 * Load idle PSW. The second "half" of this function is in .Lcleanup_idle.
946 */
947ENTRY(psw_idle)
948 stg %r3,__SF_EMPTY(%r15)
949 larl %r1,.Lpsw_idle_lpsw+4
950 stg %r1,__SF_EMPTY+8(%r15)
951#ifdef CONFIG_SMP
952 larl %r1,smp_cpu_mtid
953 llgf %r1,0(%r1)
954 ltgr %r1,%r1
955 jz .Lpsw_idle_stcctm
956 .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15)
957.Lpsw_idle_stcctm:
958#endif
959 oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
960 BPON
961 STCK __CLOCK_IDLE_ENTER(%r2)
962 stpt __TIMER_IDLE_ENTER(%r2)
963.Lpsw_idle_lpsw:
964 lpswe __SF_EMPTY(%r15)
965 BR_EX %r14
966.Lpsw_idle_end:
967
968/*
969 * Store floating-point controls and floating-point or vector register
970 * depending whether the vector facility is available. A critical section
971 * cleanup assures that the registers are stored even if interrupted for
972 * some other work. The CIF_FPU flag is set to trigger a lazy restore
973 * of the register contents at return from io or a system call.
974 */
975ENTRY(save_fpu_regs)
976 lg %r2,__LC_CURRENT
977 aghi %r2,__TASK_thread
978 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
979 jo .Lsave_fpu_regs_exit
980 stfpc __THREAD_FPU_fpc(%r2)
981 lg %r3,__THREAD_FPU_regs(%r2)
982 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
983 jz .Lsave_fpu_regs_fp # no -> store FP regs
984 VSTM %v0,%v15,0,%r3 # vstm 0,15,0(3)
985 VSTM %v16,%v31,256,%r3 # vstm 16,31,256(3)
986 j .Lsave_fpu_regs_done # -> set CIF_FPU flag
987.Lsave_fpu_regs_fp:
988 std 0,0(%r3)
989 std 1,8(%r3)
990 std 2,16(%r3)
991 std 3,24(%r3)
992 std 4,32(%r3)
993 std 5,40(%r3)
994 std 6,48(%r3)
995 std 7,56(%r3)
996 std 8,64(%r3)
997 std 9,72(%r3)
998 std 10,80(%r3)
999 std 11,88(%r3)
1000 std 12,96(%r3)
1001 std 13,104(%r3)
1002 std 14,112(%r3)
1003 std 15,120(%r3)
1004.Lsave_fpu_regs_done:
1005 oi __LC_CPU_FLAGS+7,_CIF_FPU
1006.Lsave_fpu_regs_exit:
1007 BR_EX %r14
1008.Lsave_fpu_regs_end:
1009EXPORT_SYMBOL(save_fpu_regs)
1010
1011/*
1012 * Load floating-point controls and floating-point or vector registers.
1013 * A critical section cleanup assures that the register contents are
1014 * loaded even if interrupted for some other work.
1015 *
1016 * There are special calling conventions to fit into sysc and io return work:
1017 * %r15: <kernel stack>
1018 * The function requires:
1019 * %r4
1020 */
1021load_fpu_regs:
1022 lg %r4,__LC_CURRENT
1023 aghi %r4,__TASK_thread
1024 TSTMSK __LC_CPU_FLAGS,_CIF_FPU
1025 jno .Lload_fpu_regs_exit
1026 lfpc __THREAD_FPU_fpc(%r4)
1027 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
1028 lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area
1029 jz .Lload_fpu_regs_fp # -> no VX, load FP regs
1030 VLM %v0,%v15,0,%r4
1031 VLM %v16,%v31,256,%r4
1032 j .Lload_fpu_regs_done
1033.Lload_fpu_regs_fp:
1034 ld 0,0(%r4)
1035 ld 1,8(%r4)
1036 ld 2,16(%r4)
1037 ld 3,24(%r4)
1038 ld 4,32(%r4)
1039 ld 5,40(%r4)
1040 ld 6,48(%r4)
1041 ld 7,56(%r4)
1042 ld 8,64(%r4)
1043 ld 9,72(%r4)
1044 ld 10,80(%r4)
1045 ld 11,88(%r4)
1046 ld 12,96(%r4)
1047 ld 13,104(%r4)
1048 ld 14,112(%r4)
1049 ld 15,120(%r4)
1050.Lload_fpu_regs_done:
1051 ni __LC_CPU_FLAGS+7,255-_CIF_FPU
1052.Lload_fpu_regs_exit:
1053 BR_EX %r14
1054.Lload_fpu_regs_end:
1055
1056.L__critical_end:
1057
1058/*
1059 * Machine check handler routines
1060 */
1061ENTRY(mcck_int_handler)
1062 STCK __LC_MCCK_CLOCK
1063 BPOFF
1064 la %r1,4095 # validate r1
1065 spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer
1066 sckc __LC_CLOCK_COMPARATOR # validate comparator
1067 lam %a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs
1068 lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs
1069 lg %r12,__LC_CURRENT
1070 larl %r13,cleanup_critical
1071 lmg %r8,%r9,__LC_MCK_OLD_PSW
1072 TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
1073 jo .Lmcck_panic # yes -> rest of mcck code invalid
1074 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID
1075 jno .Lmcck_panic # control registers invalid -> panic
1076 la %r14,4095
1077 lctlg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs
1078 ptlb
1079 lg %r11,__LC_MCESAD-4095(%r14) # extended machine check save area
1080 nill %r11,0xfc00 # MCESA_ORIGIN_MASK
1081 TSTMSK __LC_CREGS_SAVE_AREA+16-4095(%r14),CR2_GUARDED_STORAGE
1082 jno 0f
1083 TSTMSK __LC_MCCK_CODE,MCCK_CODE_GS_VALID
1084 jno 0f
1085 .insn rxy,0xe3000000004d,0,__MCESA_GS_SAVE_AREA(%r11) # LGSC
10860: l %r14,__LC_FP_CREG_SAVE_AREA-4095(%r14)
1087 TSTMSK __LC_MCCK_CODE,MCCK_CODE_FC_VALID
1088 jo 0f
1089 sr %r14,%r14
10900: sfpc %r14
1091 TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
1092 jo 0f
1093 lghi %r14,__LC_FPREGS_SAVE_AREA
1094 ld %f0,0(%r14)
1095 ld %f1,8(%r14)
1096 ld %f2,16(%r14)
1097 ld %f3,24(%r14)
1098 ld %f4,32(%r14)
1099 ld %f5,40(%r14)
1100 ld %f6,48(%r14)
1101 ld %f7,56(%r14)
1102 ld %f8,64(%r14)
1103 ld %f9,72(%r14)
1104 ld %f10,80(%r14)
1105 ld %f11,88(%r14)
1106 ld %f12,96(%r14)
1107 ld %f13,104(%r14)
1108 ld %f14,112(%r14)
1109 ld %f15,120(%r14)
1110 j 1f
11110: VLM %v0,%v15,0,%r11
1112 VLM %v16,%v31,256,%r11
11131: lghi %r14,__LC_CPU_TIMER_SAVE_AREA
1114 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
1115 TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
1116 jo 3f
1117 la %r14,__LC_SYNC_ENTER_TIMER
1118 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
1119 jl 0f
1120 la %r14,__LC_ASYNC_ENTER_TIMER
11210: clc 0(8,%r14),__LC_EXIT_TIMER
1122 jl 1f
1123 la %r14,__LC_EXIT_TIMER
11241: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
1125 jl 2f
1126 la %r14,__LC_LAST_UPDATE_TIMER
11272: spt 0(%r14)
1128 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
11293: TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
1130 jno .Lmcck_panic
1131 tmhh %r8,0x0001 # interrupting from user ?
1132 jnz 4f
1133 TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
1134 jno .Lmcck_panic
11354: SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER
1136.Lmcck_skip:
1137 lghi %r14,__LC_GPREGS_SAVE_AREA+64
1138 stmg %r0,%r7,__PT_R0(%r11)
1139 # clear user controlled registers to prevent speculative use
1140 xgr %r0,%r0
1141 xgr %r1,%r1
1142 xgr %r2,%r2
1143 xgr %r3,%r3
1144 xgr %r4,%r4
1145 xgr %r5,%r5
1146 xgr %r6,%r6
1147 xgr %r7,%r7
1148 xgr %r10,%r10
1149 mvc __PT_R8(64,%r11),0(%r14)
1150 stmg %r8,%r9,__PT_PSW(%r11)
1151 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
1152 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
1153 lgr %r2,%r11 # pass pointer to pt_regs
1154 brasl %r14,s390_do_machine_check
1155 tm __PT_PSW+1(%r11),0x01 # returning to user ?
1156 jno .Lmcck_return
1157 lg %r1,__LC_KERNEL_STACK # switch to kernel stack
1158 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
1159 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
1160 la %r11,STACK_FRAME_OVERHEAD(%r1)
1161 lgr %r15,%r1
1162 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
1163 TSTMSK __LC_CPU_FLAGS,_CIF_MCCK_PENDING
1164 jno .Lmcck_return
1165 TRACE_IRQS_OFF
1166 brasl %r14,s390_handle_mcck
1167 TRACE_IRQS_ON
1168.Lmcck_return:
1169 lg %r14,__LC_VDSO_PER_CPU
1170 lmg %r0,%r10,__PT_R0(%r11)
1171 mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
1172 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
1173 jno 0f
1174 BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
1175 stpt __LC_EXIT_TIMER
1176 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
11770: lmg %r11,%r15,__PT_R11(%r11)
1178 lpswe __LC_RETURN_MCCK_PSW
1179
1180.Lmcck_panic:
1181 lg %r15,__LC_PANIC_STACK
1182 la %r11,STACK_FRAME_OVERHEAD(%r15)
1183 j .Lmcck_skip
1184
1185#
1186# PSW restart interrupt handler
1187#
1188ENTRY(restart_int_handler)
1189 ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
1190 stg %r15,__LC_SAVE_AREA_RESTART
1191 lg %r15,__LC_RESTART_STACK
1192 aghi %r15,-__PT_SIZE # create pt_regs on stack
1193 xc 0(__PT_SIZE,%r15),0(%r15)
1194 stmg %r0,%r14,__PT_R0(%r15)
1195 mvc __PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
1196 mvc __PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw
1197 aghi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
1198 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
1199 lg %r1,__LC_RESTART_FN # load fn, parm & source cpu
1200 lg %r2,__LC_RESTART_DATA
1201 lg %r3,__LC_RESTART_SOURCE
1202 ltgr %r3,%r3 # test source cpu address
1203 jm 1f # negative -> skip source stop
12040: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
1205 brc 10,0b # wait for status stored
12061: basr %r14,%r1 # call function
1207 stap __SF_EMPTY(%r15) # store cpu address
1208 llgh %r3,__SF_EMPTY(%r15)
12092: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
1210 brc 2,2b
12113: j 3b
1212
1213 .section .kprobes.text, "ax"
1214
1215#ifdef CONFIG_CHECK_STACK
1216/*
1217 * The synchronous or the asynchronous stack overflowed. We are dead.
1218 * No need to properly save the registers, we are going to panic anyway.
1219 * Setup a pt_regs so that show_trace can provide a good call trace.
1220 */
1221stack_overflow:
1222 lg %r15,__LC_PANIC_STACK # change to panic stack
1223 la %r11,STACK_FRAME_OVERHEAD(%r15)
1224 stmg %r0,%r7,__PT_R0(%r11)
1225 stmg %r8,%r9,__PT_PSW(%r11)
1226 mvc __PT_R8(64,%r11),0(%r14)
1227 stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
1228 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
1229 lgr %r2,%r11 # pass pointer to pt_regs
1230 jg kernel_stack_overflow
1231#endif
1232
1233cleanup_critical:
1234#if IS_ENABLED(CONFIG_KVM)
1235 clg %r9,BASED(.Lcleanup_table_sie) # .Lsie_gmap
1236 jl 0f
1237 clg %r9,BASED(.Lcleanup_table_sie+8)# .Lsie_done
1238 jl .Lcleanup_sie
1239#endif
1240 clg %r9,BASED(.Lcleanup_table) # system_call
1241 jl 0f
1242 clg %r9,BASED(.Lcleanup_table+8) # .Lsysc_do_svc
1243 jl .Lcleanup_system_call
1244 clg %r9,BASED(.Lcleanup_table+16) # .Lsysc_tif
1245 jl 0f
1246 clg %r9,BASED(.Lcleanup_table+24) # .Lsysc_restore
1247 jl .Lcleanup_sysc_tif
1248 clg %r9,BASED(.Lcleanup_table+32) # .Lsysc_done
1249 jl .Lcleanup_sysc_restore
1250 clg %r9,BASED(.Lcleanup_table+40) # .Lio_tif
1251 jl 0f
1252 clg %r9,BASED(.Lcleanup_table+48) # .Lio_restore
1253 jl .Lcleanup_io_tif
1254 clg %r9,BASED(.Lcleanup_table+56) # .Lio_done
1255 jl .Lcleanup_io_restore
1256 clg %r9,BASED(.Lcleanup_table+64) # psw_idle
1257 jl 0f
1258 clg %r9,BASED(.Lcleanup_table+72) # .Lpsw_idle_end
1259 jl .Lcleanup_idle
1260 clg %r9,BASED(.Lcleanup_table+80) # save_fpu_regs
1261 jl 0f
1262 clg %r9,BASED(.Lcleanup_table+88) # .Lsave_fpu_regs_end
1263 jl .Lcleanup_save_fpu_regs
1264 clg %r9,BASED(.Lcleanup_table+96) # load_fpu_regs
1265 jl 0f
1266 clg %r9,BASED(.Lcleanup_table+104) # .Lload_fpu_regs_end
1267 jl .Lcleanup_load_fpu_regs
12680: BR_EX %r14
1269
1270 .align 8
1271.Lcleanup_table:
1272 .quad system_call
1273 .quad .Lsysc_do_svc
1274 .quad .Lsysc_tif
1275 .quad .Lsysc_restore
1276 .quad .Lsysc_done
1277 .quad .Lio_tif
1278 .quad .Lio_restore
1279 .quad .Lio_done
1280 .quad psw_idle
1281 .quad .Lpsw_idle_end
1282 .quad save_fpu_regs
1283 .quad .Lsave_fpu_regs_end
1284 .quad load_fpu_regs
1285 .quad .Lload_fpu_regs_end
1286
1287#if IS_ENABLED(CONFIG_KVM)
1288.Lcleanup_table_sie:
1289 .quad .Lsie_gmap
1290 .quad .Lsie_done
1291
1292.Lcleanup_sie:
1293 cghi %r11,__LC_SAVE_AREA_ASYNC #Is this in normal interrupt?
1294 je 1f
1295 slg %r9,BASED(.Lsie_crit_mcck_start)
1296 clg %r9,BASED(.Lsie_crit_mcck_length)
1297 jh 1f
1298 oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
12991: BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
1300 lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer
1301 ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
1302 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
1303 larl %r9,sie_exit # skip forward to sie_exit
1304 BR_EX %r14
1305#endif
1306
1307.Lcleanup_system_call:
1308 # check if stpt has been executed
1309 clg %r9,BASED(.Lcleanup_system_call_insn)
1310 jh 0f
1311 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
1312 cghi %r11,__LC_SAVE_AREA_ASYNC
1313 je 0f
1314 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
13150: # check if stmg has been executed
1316 clg %r9,BASED(.Lcleanup_system_call_insn+8)
1317 jh 0f
1318 mvc __LC_SAVE_AREA_SYNC(64),0(%r11)
13190: # check if base register setup + TIF bit load has been done
1320 clg %r9,BASED(.Lcleanup_system_call_insn+16)
1321 jhe 0f
1322 # set up saved register r12 task struct pointer
1323 stg %r12,32(%r11)
1324 # set up saved register r13 __TASK_thread offset
1325 mvc 40(8,%r11),BASED(.Lcleanup_system_call_const)
13260: # check if the user time update has been done
1327 clg %r9,BASED(.Lcleanup_system_call_insn+24)
1328 jh 0f
1329 lg %r15,__LC_EXIT_TIMER
1330 slg %r15,__LC_SYNC_ENTER_TIMER
1331 alg %r15,__LC_USER_TIMER
1332 stg %r15,__LC_USER_TIMER
13330: # check if the system time update has been done
1334 clg %r9,BASED(.Lcleanup_system_call_insn+32)
1335 jh 0f
1336 lg %r15,__LC_LAST_UPDATE_TIMER
1337 slg %r15,__LC_EXIT_TIMER
1338 alg %r15,__LC_SYSTEM_TIMER
1339 stg %r15,__LC_SYSTEM_TIMER
13400: # update accounting time stamp
1341 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
1342 BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
1343 # set up saved register r11
1344 lg %r15,__LC_KERNEL_STACK
1345 la %r9,STACK_FRAME_OVERHEAD(%r15)
1346 stg %r9,24(%r11) # r11 pt_regs pointer
1347 # fill pt_regs
1348 mvc __PT_R8(64,%r9),__LC_SAVE_AREA_SYNC
1349 stmg %r0,%r7,__PT_R0(%r9)
1350 mvc __PT_PSW(16,%r9),__LC_SVC_OLD_PSW
1351 mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
1352 xc __PT_FLAGS(8,%r9),__PT_FLAGS(%r9)
1353 mvi __PT_FLAGS+7(%r9),_PIF_SYSCALL
1354 # setup saved register r15
1355 stg %r15,56(%r11) # r15 stack pointer
1356 # set new psw address and exit
1357 larl %r9,.Lsysc_do_svc
1358 BR_EX %r14,%r11
1359.Lcleanup_system_call_insn:
1360 .quad system_call
1361 .quad .Lsysc_stmg
1362 .quad .Lsysc_per
1363 .quad .Lsysc_vtime+36
1364 .quad .Lsysc_vtime+42
1365.Lcleanup_system_call_const:
1366 .quad __TASK_thread
1367
1368.Lcleanup_sysc_tif:
1369 larl %r9,.Lsysc_tif
1370 BR_EX %r14,%r11
1371
1372.Lcleanup_sysc_restore:
1373 # check if stpt has been executed
1374 clg %r9,BASED(.Lcleanup_sysc_restore_insn)
1375 jh 0f
1376 mvc __LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
1377 cghi %r11,__LC_SAVE_AREA_ASYNC
1378 je 0f
1379 mvc __LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
13800: clg %r9,BASED(.Lcleanup_sysc_restore_insn+8)
1381 je 1f
1382 lg %r9,24(%r11) # get saved pointer to pt_regs
1383 mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
1384 mvc 0(64,%r11),__PT_R8(%r9)
1385 lmg %r0,%r7,__PT_R0(%r9)
13861: lmg %r8,%r9,__LC_RETURN_PSW
1387 BR_EX %r14,%r11
1388.Lcleanup_sysc_restore_insn:
1389 .quad .Lsysc_exit_timer
1390 .quad .Lsysc_done - 4
1391
1392.Lcleanup_io_tif:
1393 larl %r9,.Lio_tif
1394 BR_EX %r14,%r11
1395
1396.Lcleanup_io_restore:
1397 # check if stpt has been executed
1398 clg %r9,BASED(.Lcleanup_io_restore_insn)
1399 jh 0f
1400 mvc __LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
14010: clg %r9,BASED(.Lcleanup_io_restore_insn+8)
1402 je 1f
1403 lg %r9,24(%r11) # get saved r11 pointer to pt_regs
1404 mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
1405 mvc 0(64,%r11),__PT_R8(%r9)
1406 lmg %r0,%r7,__PT_R0(%r9)
14071: lmg %r8,%r9,__LC_RETURN_PSW
1408 BR_EX %r14,%r11
1409.Lcleanup_io_restore_insn:
1410 .quad .Lio_exit_timer
1411 .quad .Lio_done - 4
1412
1413.Lcleanup_idle:
1414 ni __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT
1415 # copy interrupt clock & cpu timer
1416 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
1417 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
1418 cghi %r11,__LC_SAVE_AREA_ASYNC
1419 je 0f
1420 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
1421 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
14220: # check if stck & stpt have been executed
1423 clg %r9,BASED(.Lcleanup_idle_insn)
1424 jhe 1f
1425 mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
1426 mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r2)
14271: # calculate idle cycles
1428#ifdef CONFIG_SMP
1429 clg %r9,BASED(.Lcleanup_idle_insn)
1430 jl 3f
1431 larl %r1,smp_cpu_mtid
1432 llgf %r1,0(%r1)
1433 ltgr %r1,%r1
1434 jz 3f
1435 .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15)
1436 larl %r3,mt_cycles
1437 ag %r3,__LC_PERCPU_OFFSET
1438 la %r4,__SF_EMPTY+16(%r15)
14392: lg %r0,0(%r3)
1440 slg %r0,0(%r4)
1441 alg %r0,64(%r4)
1442 stg %r0,0(%r3)
1443 la %r3,8(%r3)
1444 la %r4,8(%r4)
1445 brct %r1,2b
1446#endif
14473: # account system time going idle
1448 lg %r9,__LC_STEAL_TIMER
1449 alg %r9,__CLOCK_IDLE_ENTER(%r2)
1450 slg %r9,__LC_LAST_UPDATE_CLOCK
1451 stg %r9,__LC_STEAL_TIMER
1452 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
1453 lg %r9,__LC_SYSTEM_TIMER
1454 alg %r9,__LC_LAST_UPDATE_TIMER
1455 slg %r9,__TIMER_IDLE_ENTER(%r2)
1456 stg %r9,__LC_SYSTEM_TIMER
1457 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
1458 # prepare return psw
1459 nihh %r8,0xfcfd # clear irq & wait state bits
1460 lg %r9,48(%r11) # return from psw_idle
1461 BR_EX %r14,%r11
1462.Lcleanup_idle_insn:
1463 .quad .Lpsw_idle_lpsw
1464
1465.Lcleanup_save_fpu_regs:
1466 larl %r9,save_fpu_regs
1467 BR_EX %r14,%r11
1468
1469.Lcleanup_load_fpu_regs:
1470 larl %r9,load_fpu_regs
1471 BR_EX %r14,%r11
1472
1473/*
1474 * Integer constants
1475 */
1476 .align 8
1477.Lcritical_start:
1478 .quad .L__critical_start
1479.Lcritical_length:
1480 .quad .L__critical_end - .L__critical_start
1481#if IS_ENABLED(CONFIG_KVM)
1482.Lsie_critical_start:
1483 .quad .Lsie_gmap
1484.Lsie_critical_length:
1485 .quad .Lsie_done - .Lsie_gmap
1486.Lsie_crit_mcck_start:
1487 .quad .Lsie_entry
1488.Lsie_crit_mcck_length:
1489 .quad .Lsie_skip - .Lsie_entry
1490#endif
1491 .section .rodata, "a"
1492#define SYSCALL(esame,emu) .long esame
1493 .globl sys_call_table
1494sys_call_table:
1495#include "asm/syscall_table.h"
1496#undef SYSCALL
1497
1498#ifdef CONFIG_COMPAT
1499
1500#define SYSCALL(esame,emu) .long emu
1501 .globl sys_call_table_emu
1502sys_call_table_emu:
1503#include "asm/syscall_table.h"
1504#undef SYSCALL
1505#endif