Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * This file contains the light-weight system call handlers (fsyscall-handlers).
4 *
5 * Copyright (C) 2003 Hewlett-Packard Co
6 * David Mosberger-Tang <davidm@hpl.hp.com>
7 *
8 * 25-Sep-03 davidm Implement fsys_rt_sigprocmask().
9 * 18-Feb-03 louisk Implement fsys_gettimeofday().
10 * 28-Feb-03 davidm Fixed several bugs in fsys_gettimeofday(). Tuned it some more,
11 * probably broke it along the way... ;-)
12 * 13-Jul-04 clameter Implement fsys_clock_gettime and revise fsys_gettimeofday to make
13 * it capable of using memory based clocks without falling back to C code.
14 * 08-Feb-07 Fenghua Yu Implement fsys_getcpu.
15 *
16 */
17
18#include <asm/asmmacro.h>
19#include <asm/errno.h>
20#include <asm/asm-offsets.h>
21#include <asm/percpu.h>
22#include <asm/thread_info.h>
23#include <asm/sal.h>
24#include <asm/signal.h>
25#include <asm/unistd.h>
26
27#include "entry.h"
28#include <asm/native/inst.h>
29
30/*
31 * See Documentation/ia64/fsys.rst for details on fsyscalls.
32 *
33 * On entry to an fsyscall handler:
34 * r10 = 0 (i.e., defaults to "successful syscall return")
35 * r11 = saved ar.pfs (a user-level value)
36 * r15 = system call number
37 * r16 = "current" task pointer (in normal kernel-mode, this is in r13)
38 * r32-r39 = system call arguments
39 * b6 = return address (a user-level value)
40 * ar.pfs = previous frame-state (a user-level value)
41 * PSR.be = cleared to zero (i.e., little-endian byte order is in effect)
42 * all other registers may contain values passed in from user-mode
43 *
44 * On return from an fsyscall handler:
45 * r11 = saved ar.pfs (as passed into the fsyscall handler)
46 * r15 = system call number (as passed into the fsyscall handler)
47 * r32-r39 = system call arguments (as passed into the fsyscall handler)
48 * b6 = return address (as passed into the fsyscall handler)
49 * ar.pfs = previous frame-state (as passed into the fsyscall handler)
50 */
51
52ENTRY(fsys_ni_syscall)
53 .prologue
54 .altrp b6
55 .body
56 mov r8=ENOSYS
57 mov r10=-1
58 FSYS_RETURN
59END(fsys_ni_syscall)
60
61ENTRY(fsys_getpid)
62 .prologue
63 .altrp b6
64 .body
65 add r17=IA64_TASK_SIGNAL_OFFSET,r16
66 ;;
67 ld8 r17=[r17] // r17 = current->signal
68 add r9=TI_FLAGS+IA64_TASK_SIZE,r16
69 ;;
70 ld4 r9=[r9]
71 add r17=IA64_SIGNAL_PIDS_TGID_OFFSET,r17
72 ;;
73 and r9=TIF_ALLWORK_MASK,r9
74 ld8 r17=[r17] // r17 = current->signal->pids[PIDTYPE_TGID]
75 ;;
76 add r8=IA64_PID_LEVEL_OFFSET,r17
77 ;;
78 ld4 r8=[r8] // r8 = pid->level
79 add r17=IA64_PID_UPID_OFFSET,r17 // r17 = &pid->numbers[0]
80 ;;
81 shl r8=r8,IA64_UPID_SHIFT
82 ;;
83 add r17=r17,r8 // r17 = &pid->numbers[pid->level]
84 ;;
85 ld4 r8=[r17] // r8 = pid->numbers[pid->level].nr
86 ;;
87 mov r17=0
88 ;;
89 cmp.ne p8,p0=0,r9
90(p8) br.spnt.many fsys_fallback_syscall
91 FSYS_RETURN
92END(fsys_getpid)
93
94ENTRY(fsys_set_tid_address)
95 .prologue
96 .altrp b6
97 .body
98 add r9=TI_FLAGS+IA64_TASK_SIZE,r16
99 add r17=IA64_TASK_THREAD_PID_OFFSET,r16
100 ;;
101 ld4 r9=[r9]
102 tnat.z p6,p7=r32 // check argument register for being NaT
103 ld8 r17=[r17] // r17 = current->thread_pid
104 ;;
105 and r9=TIF_ALLWORK_MASK,r9
106 add r8=IA64_PID_LEVEL_OFFSET,r17
107 add r18=IA64_TASK_CLEAR_CHILD_TID_OFFSET,r16
108 ;;
109 ld4 r8=[r8] // r8 = pid->level
110 add r17=IA64_PID_UPID_OFFSET,r17 // r17 = &pid->numbers[0]
111 ;;
112 shl r8=r8,IA64_UPID_SHIFT
113 ;;
114 add r17=r17,r8 // r17 = &pid->numbers[pid->level]
115 ;;
116 ld4 r8=[r17] // r8 = pid->numbers[pid->level].nr
117 ;;
118 cmp.ne p8,p0=0,r9
119 mov r17=-1
120 ;;
121(p6) st8 [r18]=r32
122(p7) st8 [r18]=r17
123(p8) br.spnt.many fsys_fallback_syscall
124 ;;
125 mov r17=0 // i must not leak kernel bits...
126 mov r18=0 // i must not leak kernel bits...
127 FSYS_RETURN
128END(fsys_set_tid_address)
129
130#if IA64_GTOD_SEQ_OFFSET !=0
131#error fsys_gettimeofday incompatible with changes to struct fsyscall_gtod_data_t
132#endif
133#if IA64_ITC_JITTER_OFFSET !=0
134#error fsys_gettimeofday incompatible with changes to struct itc_jitter_data_t
135#endif
136#define CLOCK_REALTIME 0
137#define CLOCK_MONOTONIC 1
138#define CLOCK_DIVIDE_BY_1000 0x4000
139#define CLOCK_ADD_MONOTONIC 0x8000
140
141ENTRY(fsys_gettimeofday)
142 .prologue
143 .altrp b6
144 .body
145 mov r31 = r32
146 tnat.nz p6,p0 = r33 // guard against NaT argument
147(p6) br.cond.spnt.few .fail_einval
148 mov r30 = CLOCK_DIVIDE_BY_1000
149 ;;
150.gettime:
151 // Register map
152 // Incoming r31 = pointer to address where to place result
153 // r30 = flags determining how time is processed
154 // r2,r3 = temp r4-r7 preserved
155 // r8 = result nanoseconds
156 // r9 = result seconds
157 // r10 = temporary storage for clock difference
158 // r11 = preserved: saved ar.pfs
159 // r12 = preserved: memory stack
160 // r13 = preserved: thread pointer
161 // r14 = address of mask / mask value
162 // r15 = preserved: system call number
163 // r16 = preserved: current task pointer
164 // r17 = (not used)
165 // r18 = (not used)
166 // r19 = address of itc_lastcycle
167 // r20 = struct fsyscall_gtod_data (= address of gtod_lock.sequence)
168 // r21 = address of mmio_ptr
169 // r22 = address of wall_time or monotonic_time
170 // r23 = address of shift / value
171 // r24 = address mult factor / cycle_last value
172 // r25 = itc_lastcycle value
173 // r26 = address clocksource cycle_last
174 // r27 = (not used)
175 // r28 = sequence number at the beginning of critical section
176 // r29 = address of itc_jitter
177 // r30 = time processing flags / memory address
178 // r31 = pointer to result
179 // Predicates
180 // p6,p7 short term use
181 // p8 = timesource ar.itc
182 // p9 = timesource mmio64
183 // p10 = timesource mmio32 - not used
184 // p11 = timesource not to be handled by asm code
185 // p12 = memory time source ( = p9 | p10) - not used
186 // p13 = do cmpxchg with itc_lastcycle
187 // p14 = Divide by 1000
188 // p15 = Add monotonic
189 //
190 // Note that instructions are optimized for McKinley. McKinley can
191 // process two bundles simultaneously and therefore we continuously
192 // try to feed the CPU two bundles and then a stop.
193
194 add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
195 tnat.nz p6,p0 = r31 // guard against Nat argument
196(p6) br.cond.spnt.few .fail_einval
197 movl r20 = fsyscall_gtod_data // load fsyscall gettimeofday data address
198 ;;
199 ld4 r2 = [r2] // process work pending flags
200 movl r29 = itc_jitter_data // itc_jitter
201 add r22 = IA64_GTOD_WALL_TIME_OFFSET,r20 // wall_time
202 add r21 = IA64_CLKSRC_MMIO_OFFSET,r20
203 mov pr = r30,0xc000 // Set predicates according to function
204 ;;
205 and r2 = TIF_ALLWORK_MASK,r2
206 add r19 = IA64_ITC_LASTCYCLE_OFFSET,r29
207(p15) add r22 = IA64_GTOD_MONO_TIME_OFFSET,r20 // monotonic_time
208 ;;
209 add r26 = IA64_CLKSRC_CYCLE_LAST_OFFSET,r20 // clksrc_cycle_last
210 cmp.ne p6, p0 = 0, r2 // Fallback if work is scheduled
211(p6) br.cond.spnt.many fsys_fallback_syscall
212 ;;
213 // Begin critical section
214.time_redo:
215 ld4.acq r28 = [r20] // gtod_lock.sequence, Must take first
216 ;;
217 and r28 = ~1,r28 // And make sequence even to force retry if odd
218 ;;
219 ld8 r30 = [r21] // clocksource->mmio_ptr
220 add r24 = IA64_CLKSRC_MULT_OFFSET,r20
221 ld4 r2 = [r29] // itc_jitter value
222 add r23 = IA64_CLKSRC_SHIFT_OFFSET,r20
223 add r14 = IA64_CLKSRC_MASK_OFFSET,r20
224 ;;
225 ld4 r3 = [r24] // clocksource mult value
226 ld8 r14 = [r14] // clocksource mask value
227 cmp.eq p8,p9 = 0,r30 // use cpu timer if no mmio_ptr
228 ;;
229 setf.sig f7 = r3 // Setup for mult scaling of counter
230(p8) cmp.ne p13,p0 = r2,r0 // need itc_jitter compensation, set p13
231 ld4 r23 = [r23] // clocksource shift value
232 ld8 r24 = [r26] // get clksrc_cycle_last value
233(p9) cmp.eq p13,p0 = 0,r30 // if mmio_ptr, clear p13 jitter control
234 ;;
235 .pred.rel.mutex p8,p9
236 MOV_FROM_ITC(p8, p6, r2, r10) // CPU_TIMER. 36 clocks latency!!!
237(p9) ld8 r2 = [r30] // MMIO_TIMER. Could also have latency issues..
238(p13) ld8 r25 = [r19] // get itc_lastcycle value
239 ld8 r9 = [r22],IA64_TIME_SN_SPEC_SNSEC_OFFSET // sec
240 ;;
241 ld8 r8 = [r22],-IA64_TIME_SN_SPEC_SNSEC_OFFSET // snsec
242(p13) sub r3 = r25,r2 // Diff needed before comparison (thanks davidm)
243 ;;
244(p13) cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
245 sub r10 = r2,r24 // current_cycle - last_cycle
246 ;;
247(p6) sub r10 = r25,r24 // time we got was less than last_cycle
248(p7) mov ar.ccv = r25 // more than last_cycle. Prep for cmpxchg
249 ;;
250(p7) cmpxchg8.rel r3 = [r19],r2,ar.ccv
251 ;;
252(p7) cmp.ne p7,p0 = r25,r3 // if cmpxchg not successful
253 ;;
254(p7) sub r10 = r3,r24 // then use new last_cycle instead
255 ;;
256 and r10 = r10,r14 // Apply mask
257 ;;
258 setf.sig f8 = r10
259 nop.i 123
260 ;;
261 // fault check takes 5 cycles and we have spare time
262EX(.fail_efault, probe.w.fault r31, 3)
263 xmpy.l f8 = f8,f7 // nsec_per_cyc*(counter-last_counter)
264 ;;
265 getf.sig r2 = f8
266 mf
267 ;;
268 ld4 r10 = [r20] // gtod_lock.sequence
269 add r8 = r8,r2 // Add xtime.nsecs
270 ;;
271 shr.u r8 = r8,r23 // shift by factor
272 cmp4.ne p7,p0 = r28,r10
273(p7) br.cond.dpnt.few .time_redo // sequence number changed, redo
274 // End critical section.
275 // Now r8=tv->tv_nsec and r9=tv->tv_sec
276 mov r10 = r0
277 movl r2 = 1000000000
278 add r23 = IA64_TIMESPEC_TV_NSEC_OFFSET, r31
279(p14) movl r3 = 2361183241434822607 // Prep for / 1000 hack
280 ;;
281.time_normalize:
282 mov r21 = r8
283 cmp.ge p6,p0 = r8,r2
284(p14) shr.u r20 = r8, 3 // We can repeat this if necessary just wasting time
285 ;;
286(p14) setf.sig f8 = r20
287(p6) sub r8 = r8,r2
288(p6) add r9 = 1,r9 // two nops before the branch.
289(p14) setf.sig f7 = r3 // Chances for repeats are 1 in 10000 for gettod
290(p6) br.cond.dpnt.few .time_normalize
291 ;;
292 // Divided by 8 though shift. Now divide by 125
293 // The compiler was able to do that with a multiply
294 // and a shift and we do the same
295EX(.fail_efault, probe.w.fault r23, 3) // This also costs 5 cycles
296(p14) xmpy.hu f8 = f8, f7 // xmpy has 5 cycles latency so use it
297 ;;
298(p14) getf.sig r2 = f8
299 ;;
300 mov r8 = r0
301(p14) shr.u r21 = r2, 4
302 ;;
303EX(.fail_efault, st8 [r31] = r9)
304EX(.fail_efault, st8 [r23] = r21)
305 FSYS_RETURN
306.fail_einval:
307 mov r8 = EINVAL
308 mov r10 = -1
309 FSYS_RETURN
310.fail_efault:
311 mov r8 = EFAULT
312 mov r10 = -1
313 FSYS_RETURN
314END(fsys_gettimeofday)
315
316ENTRY(fsys_clock_gettime)
317 .prologue
318 .altrp b6
319 .body
320 cmp4.ltu p6, p0 = CLOCK_MONOTONIC, r32
321 // Fallback if this is not CLOCK_REALTIME or CLOCK_MONOTONIC
322(p6) br.spnt.few fsys_fallback_syscall
323 mov r31 = r33
324 shl r30 = r32,15
325 br.many .gettime
326END(fsys_clock_gettime)
327
328/*
329 * fsys_getcpu doesn't use the third parameter in this implementation. It reads
330 * current_thread_info()->cpu and corresponding node in cpu_to_node_map.
331 */
332ENTRY(fsys_getcpu)
333 .prologue
334 .altrp b6
335 .body
336 ;;
337 add r2=TI_FLAGS+IA64_TASK_SIZE,r16
338 tnat.nz p6,p0 = r32 // guard against NaT argument
339 add r3=TI_CPU+IA64_TASK_SIZE,r16
340 ;;
341 ld4 r3=[r3] // M r3 = thread_info->cpu
342 ld4 r2=[r2] // M r2 = thread_info->flags
343(p6) br.cond.spnt.few .fail_einval // B
344 ;;
345 tnat.nz p7,p0 = r33 // I guard against NaT argument
346(p7) br.cond.spnt.few .fail_einval // B
347 ;;
348 cmp.ne p6,p0=r32,r0
349 cmp.ne p7,p0=r33,r0
350 ;;
351#ifdef CONFIG_NUMA
352 movl r17=cpu_to_node_map
353 ;;
354EX(.fail_efault, (p6) probe.w.fault r32, 3) // M This takes 5 cycles
355EX(.fail_efault, (p7) probe.w.fault r33, 3) // M This takes 5 cycles
356 shladd r18=r3,1,r17
357 ;;
358 ld2 r20=[r18] // r20 = cpu_to_node_map[cpu]
359 and r2 = TIF_ALLWORK_MASK,r2
360 ;;
361 cmp.ne p8,p0=0,r2
362(p8) br.spnt.many fsys_fallback_syscall
363 ;;
364 ;;
365EX(.fail_efault, (p6) st4 [r32] = r3)
366EX(.fail_efault, (p7) st2 [r33] = r20)
367 mov r8=0
368 ;;
369#else
370EX(.fail_efault, (p6) probe.w.fault r32, 3) // M This takes 5 cycles
371EX(.fail_efault, (p7) probe.w.fault r33, 3) // M This takes 5 cycles
372 and r2 = TIF_ALLWORK_MASK,r2
373 ;;
374 cmp.ne p8,p0=0,r2
375(p8) br.spnt.many fsys_fallback_syscall
376 ;;
377EX(.fail_efault, (p6) st4 [r32] = r3)
378EX(.fail_efault, (p7) st2 [r33] = r0)
379 mov r8=0
380 ;;
381#endif
382 FSYS_RETURN
383END(fsys_getcpu)
384
385ENTRY(fsys_fallback_syscall)
386 .prologue
387 .altrp b6
388 .body
389 /*
390 * We only get here from light-weight syscall handlers. Thus, we already
391 * know that r15 contains a valid syscall number. No need to re-check.
392 */
393 adds r17=-1024,r15
394 movl r14=sys_call_table
395 ;;
396 RSM_PSR_I(p0, r26, r27)
397 shladd r18=r17,3,r14
398 ;;
399 ld8 r18=[r18] // load normal (heavy-weight) syscall entry-point
400 MOV_FROM_PSR(p0, r29, r26) // read psr (12 cyc load latency)
401 mov r27=ar.rsc
402 mov r21=ar.fpsr
403 mov r26=ar.pfs
404END(fsys_fallback_syscall)
405 /* FALL THROUGH */
406GLOBAL_ENTRY(fsys_bubble_down)
407 .prologue
408 .altrp b6
409 .body
410 /*
411 * We get here for syscalls that don't have a lightweight
412 * handler. For those, we need to bubble down into the kernel
413 * and that requires setting up a minimal pt_regs structure,
414 * and initializing the CPU state more or less as if an
415 * interruption had occurred. To make syscall-restarts work,
416 * we setup pt_regs such that cr_iip points to the second
417 * instruction in syscall_via_break. Decrementing the IP
418 * hence will restart the syscall via break and not
419 * decrementing IP will return us to the caller, as usual.
420 * Note that we preserve the value of psr.pp rather than
421 * initializing it from dcr.pp. This makes it possible to
422 * distinguish fsyscall execution from other privileged
423 * execution.
424 *
425 * On entry:
426 * - normal fsyscall handler register usage, except
427 * that we also have:
428 * - r18: address of syscall entry point
429 * - r21: ar.fpsr
430 * - r26: ar.pfs
431 * - r27: ar.rsc
432 * - r29: psr
433 *
434 * We used to clear some PSR bits here but that requires slow
435 * serialization. Fortunately, that isn't really necessary.
436 * The rationale is as follows: we used to clear bits
437 * ~PSR_PRESERVED_BITS in PSR.L. Since
438 * PSR_PRESERVED_BITS==PSR.{UP,MFL,MFH,PK,DT,PP,SP,RT,IC}, we
439 * ended up clearing PSR.{BE,AC,I,DFL,DFH,DI,DB,SI,TB}.
440 * However,
441 *
442 * PSR.BE : already is turned off in __kernel_syscall_via_epc()
443 * PSR.AC : don't care (kernel normally turns PSR.AC on)
444 * PSR.I : already turned off by the time fsys_bubble_down gets
445 * invoked
446 * PSR.DFL: always 0 (kernel never turns it on)
447 * PSR.DFH: don't care --- kernel never touches f32-f127 on its own
448 * initiative
449 * PSR.DI : always 0 (kernel never turns it on)
450 * PSR.SI : always 0 (kernel never turns it on)
451 * PSR.DB : don't care --- kernel never enables kernel-level
452 * breakpoints
453 * PSR.TB : must be 0 already; if it wasn't zero on entry to
454 * __kernel_syscall_via_epc, the branch to fsys_bubble_down
455 * will trigger a taken branch; the taken-trap-handler then
456 * converts the syscall into a break-based system-call.
457 */
458 /*
459 * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.
460 * The rest we have to synthesize.
461 */
462# define PSR_ONE_BITS ((3 << IA64_PSR_CPL0_BIT) \
463 | (0x1 << IA64_PSR_RI_BIT) \
464 | IA64_PSR_BN | IA64_PSR_I)
465
466 invala // M0|1
467 movl r14=ia64_ret_from_syscall // X
468
469 nop.m 0
470 movl r28=__kernel_syscall_via_break // X create cr.iip
471 ;;
472
473 mov r2=r16 // A get task addr to addl-addressable register
474 adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // A
475 mov r31=pr // I0 save pr (2 cyc)
476 ;;
477 st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
478 addl r22=IA64_RBS_OFFSET,r2 // A compute base of RBS
479 add r3=TI_FLAGS+IA64_TASK_SIZE,r2 // A
480 ;;
481 ld4 r3=[r3] // M0|1 r3 = current_thread_info()->flags
482 lfetch.fault.excl.nt1 [r22] // M0|1 prefetch register backing-store
483 nop.i 0
484 ;;
485 mov ar.rsc=0 // M2 set enforced lazy mode, pl 0, LE, loadrs=0
486#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
487 MOV_FROM_ITC(p0, p6, r30, r23) // M get cycle for accounting
488#else
489 nop.m 0
490#endif
491 nop.i 0
492 ;;
493 mov r23=ar.bspstore // M2 (12 cyc) save ar.bspstore
494 mov.m r24=ar.rnat // M2 (5 cyc) read ar.rnat (dual-issues!)
495 nop.i 0
496 ;;
497 mov ar.bspstore=r22 // M2 (6 cyc) switch to kernel RBS
498 movl r8=PSR_ONE_BITS // X
499 ;;
500 mov r25=ar.unat // M2 (5 cyc) save ar.unat
501 mov r19=b6 // I0 save b6 (2 cyc)
502 mov r20=r1 // A save caller's gp in r20
503 ;;
504 or r29=r8,r29 // A construct cr.ipsr value to save
505 mov b6=r18 // I0 copy syscall entry-point to b6 (7 cyc)
506 addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2 // A compute base of memory stack
507
508 mov r18=ar.bsp // M2 save (kernel) ar.bsp (12 cyc)
509 cmp.ne pKStk,pUStk=r0,r0 // A set pKStk <- 0, pUStk <- 1
510 br.call.sptk.many b7=ia64_syscall_setup // B
511 ;;
512#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
513 // mov.m r30=ar.itc is called in advance
514 add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2
515 add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2
516 ;;
517 ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // time at last check in kernel
518 ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // time at leave kernel
519 ;;
520 ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // cumulated stime
521 ld8 r21=[r17] // cumulated utime
522 sub r22=r19,r18 // stime before leave kernel
523 ;;
524 st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // update stamp
525 sub r18=r30,r19 // elapsed time in user mode
526 ;;
527 add r20=r20,r22 // sum stime
528 add r21=r21,r18 // sum utime
529 ;;
530 st8 [r16]=r20 // update stime
531 st8 [r17]=r21 // update utime
532 ;;
533#endif
534 mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
535 mov rp=r14 // I0 set the real return addr
536 and r3=_TIF_SYSCALL_TRACEAUDIT,r3 // A
537 ;;
538 SSM_PSR_I(p0, p6, r22) // M2 we're on kernel stacks now, reenable irqs
539 cmp.eq p8,p0=r3,r0 // A
540(p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
541
542 nop.m 0
543(p8) br.call.sptk.many b6=b6 // B (ignore return address)
544 br.cond.spnt ia64_trace_syscall // B
545END(fsys_bubble_down)
546
547 .rodata
548 .align 8
549 .globl fsyscall_table
550
551 data8 fsys_bubble_down
552fsyscall_table:
553 data8 fsys_ni_syscall
554 data8 0 // exit // 1025
555 data8 0 // read
556 data8 0 // write
557 data8 0 // open
558 data8 0 // close
559 data8 0 // creat // 1030
560 data8 0 // link
561 data8 0 // unlink
562 data8 0 // execve
563 data8 0 // chdir
564 data8 0 // fchdir // 1035
565 data8 0 // utimes
566 data8 0 // mknod
567 data8 0 // chmod
568 data8 0 // chown
569 data8 0 // lseek // 1040
570 data8 fsys_getpid // getpid
571 data8 0 // getppid
572 data8 0 // mount
573 data8 0 // umount
574 data8 0 // setuid // 1045
575 data8 0 // getuid
576 data8 0 // geteuid
577 data8 0 // ptrace
578 data8 0 // access
579 data8 0 // sync // 1050
580 data8 0 // fsync
581 data8 0 // fdatasync
582 data8 0 // kill
583 data8 0 // rename
584 data8 0 // mkdir // 1055
585 data8 0 // rmdir
586 data8 0 // dup
587 data8 0 // pipe
588 data8 0 // times
589 data8 0 // brk // 1060
590 data8 0 // setgid
591 data8 0 // getgid
592 data8 0 // getegid
593 data8 0 // acct
594 data8 0 // ioctl // 1065
595 data8 0 // fcntl
596 data8 0 // umask
597 data8 0 // chroot
598 data8 0 // ustat
599 data8 0 // dup2 // 1070
600 data8 0 // setreuid
601 data8 0 // setregid
602 data8 0 // getresuid
603 data8 0 // setresuid
604 data8 0 // getresgid // 1075
605 data8 0 // setresgid
606 data8 0 // getgroups
607 data8 0 // setgroups
608 data8 0 // getpgid
609 data8 0 // setpgid // 1080
610 data8 0 // setsid
611 data8 0 // getsid
612 data8 0 // sethostname
613 data8 0 // setrlimit
614 data8 0 // getrlimit // 1085
615 data8 0 // getrusage
616 data8 fsys_gettimeofday // gettimeofday
617 data8 0 // settimeofday
618 data8 0 // select
619 data8 0 // poll // 1090
620 data8 0 // symlink
621 data8 0 // readlink
622 data8 0 // uselib
623 data8 0 // swapon
624 data8 0 // swapoff // 1095
625 data8 0 // reboot
626 data8 0 // truncate
627 data8 0 // ftruncate
628 data8 0 // fchmod
629 data8 0 // fchown // 1100
630 data8 0 // getpriority
631 data8 0 // setpriority
632 data8 0 // statfs
633 data8 0 // fstatfs
634 data8 0 // gettid // 1105
635 data8 0 // semget
636 data8 0 // semop
637 data8 0 // semctl
638 data8 0 // msgget
639 data8 0 // msgsnd // 1110
640 data8 0 // msgrcv
641 data8 0 // msgctl
642 data8 0 // shmget
643 data8 0 // shmat
644 data8 0 // shmdt // 1115
645 data8 0 // shmctl
646 data8 0 // syslog
647 data8 0 // setitimer
648 data8 0 // getitimer
649 data8 0 // 1120
650 data8 0
651 data8 0
652 data8 0 // vhangup
653 data8 0 // lchown
654 data8 0 // remap_file_pages // 1125
655 data8 0 // wait4
656 data8 0 // sysinfo
657 data8 0 // clone
658 data8 0 // setdomainname
659 data8 0 // newuname // 1130
660 data8 0 // adjtimex
661 data8 0
662 data8 0 // init_module
663 data8 0 // delete_module
664 data8 0 // 1135
665 data8 0
666 data8 0 // quotactl
667 data8 0 // bdflush
668 data8 0 // sysfs
669 data8 0 // personality // 1140
670 data8 0 // afs_syscall
671 data8 0 // setfsuid
672 data8 0 // setfsgid
673 data8 0 // getdents
674 data8 0 // flock // 1145
675 data8 0 // readv
676 data8 0 // writev
677 data8 0 // pread64
678 data8 0 // pwrite64
679 data8 0 // sysctl // 1150
680 data8 0 // mmap
681 data8 0 // munmap
682 data8 0 // mlock
683 data8 0 // mlockall
684 data8 0 // mprotect // 1155
685 data8 0 // mremap
686 data8 0 // msync
687 data8 0 // munlock
688 data8 0 // munlockall
689 data8 0 // sched_getparam // 1160
690 data8 0 // sched_setparam
691 data8 0 // sched_getscheduler
692 data8 0 // sched_setscheduler
693 data8 0 // sched_yield
694 data8 0 // sched_get_priority_max // 1165
695 data8 0 // sched_get_priority_min
696 data8 0 // sched_rr_get_interval
697 data8 0 // nanosleep
698 data8 0 // nfsservctl
699 data8 0 // prctl // 1170
700 data8 0 // getpagesize
701 data8 0 // mmap2
702 data8 0 // pciconfig_read
703 data8 0 // pciconfig_write
704 data8 0 // perfmonctl // 1175
705 data8 0 // sigaltstack
706 data8 0 // rt_sigaction
707 data8 0 // rt_sigpending
708 data8 0 // rt_sigprocmask
709 data8 0 // rt_sigqueueinfo // 1180
710 data8 0 // rt_sigreturn
711 data8 0 // rt_sigsuspend
712 data8 0 // rt_sigtimedwait
713 data8 0 // getcwd
714 data8 0 // capget // 1185
715 data8 0 // capset
716 data8 0 // sendfile
717 data8 0
718 data8 0
719 data8 0 // socket // 1190
720 data8 0 // bind
721 data8 0 // connect
722 data8 0 // listen
723 data8 0 // accept
724 data8 0 // getsockname // 1195
725 data8 0 // getpeername
726 data8 0 // socketpair
727 data8 0 // send
728 data8 0 // sendto
729 data8 0 // recv // 1200
730 data8 0 // recvfrom
731 data8 0 // shutdown
732 data8 0 // setsockopt
733 data8 0 // getsockopt
734 data8 0 // sendmsg // 1205
735 data8 0 // recvmsg
736 data8 0 // pivot_root
737 data8 0 // mincore
738 data8 0 // madvise
739 data8 0 // newstat // 1210
740 data8 0 // newlstat
741 data8 0 // newfstat
742 data8 0 // clone2
743 data8 0 // getdents64
744 data8 0 // getunwind // 1215
745 data8 0 // readahead
746 data8 0 // setxattr
747 data8 0 // lsetxattr
748 data8 0 // fsetxattr
749 data8 0 // getxattr // 1220
750 data8 0 // lgetxattr
751 data8 0 // fgetxattr
752 data8 0 // listxattr
753 data8 0 // llistxattr
754 data8 0 // flistxattr // 1225
755 data8 0 // removexattr
756 data8 0 // lremovexattr
757 data8 0 // fremovexattr
758 data8 0 // tkill
759 data8 0 // futex // 1230
760 data8 0 // sched_setaffinity
761 data8 0 // sched_getaffinity
762 data8 fsys_set_tid_address // set_tid_address
763 data8 0 // fadvise64_64
764 data8 0 // tgkill // 1235
765 data8 0 // exit_group
766 data8 0 // lookup_dcookie
767 data8 0 // io_setup
768 data8 0 // io_destroy
769 data8 0 // io_getevents // 1240
770 data8 0 // io_submit
771 data8 0 // io_cancel
772 data8 0 // epoll_create
773 data8 0 // epoll_ctl
774 data8 0 // epoll_wait // 1245
775 data8 0 // restart_syscall
776 data8 0 // semtimedop
777 data8 0 // timer_create
778 data8 0 // timer_settime
779 data8 0 // timer_gettime // 1250
780 data8 0 // timer_getoverrun
781 data8 0 // timer_delete
782 data8 0 // clock_settime
783 data8 fsys_clock_gettime // clock_gettime
784 data8 0 // clock_getres // 1255
785 data8 0 // clock_nanosleep
786 data8 0 // fstatfs64
787 data8 0 // statfs64
788 data8 0 // mbind
789 data8 0 // get_mempolicy // 1260
790 data8 0 // set_mempolicy
791 data8 0 // mq_open
792 data8 0 // mq_unlink
793 data8 0 // mq_timedsend
794 data8 0 // mq_timedreceive // 1265
795 data8 0 // mq_notify
796 data8 0 // mq_getsetattr
797 data8 0 // kexec_load
798 data8 0 // vserver
799 data8 0 // waitid // 1270
800 data8 0 // add_key
801 data8 0 // request_key
802 data8 0 // keyctl
803 data8 0 // ioprio_set
804 data8 0 // ioprio_get // 1275
805 data8 0 // move_pages
806 data8 0 // inotify_init
807 data8 0 // inotify_add_watch
808 data8 0 // inotify_rm_watch
809 data8 0 // migrate_pages // 1280
810 data8 0 // openat
811 data8 0 // mkdirat
812 data8 0 // mknodat
813 data8 0 // fchownat
814 data8 0 // futimesat // 1285
815 data8 0 // newfstatat
816 data8 0 // unlinkat
817 data8 0 // renameat
818 data8 0 // linkat
819 data8 0 // symlinkat // 1290
820 data8 0 // readlinkat
821 data8 0 // fchmodat
822 data8 0 // faccessat
823 data8 0
824 data8 0 // 1295
825 data8 0 // unshare
826 data8 0 // splice
827 data8 0 // set_robust_list
828 data8 0 // get_robust_list
829 data8 0 // sync_file_range // 1300
830 data8 0 // tee
831 data8 0 // vmsplice
832 data8 0
833 data8 fsys_getcpu // getcpu // 1304
834
835 // fill in zeros for the remaining entries
836 .zero:
837 .space fsyscall_table + 8*NR_syscalls - .zero, 0
1/*
2 * This file contains the light-weight system call handlers (fsyscall-handlers).
3 *
4 * Copyright (C) 2003 Hewlett-Packard Co
5 * David Mosberger-Tang <davidm@hpl.hp.com>
6 *
7 * 25-Sep-03 davidm Implement fsys_rt_sigprocmask().
8 * 18-Feb-03 louisk Implement fsys_gettimeofday().
9 * 28-Feb-03 davidm Fixed several bugs in fsys_gettimeofday(). Tuned it some more,
10 * probably broke it along the way... ;-)
11 * 13-Jul-04 clameter Implement fsys_clock_gettime and revise fsys_gettimeofday to make
12 * it capable of using memory based clocks without falling back to C code.
13 * 08-Feb-07 Fenghua Yu Implement fsys_getcpu.
14 *
15 */
16
17#include <asm/asmmacro.h>
18#include <asm/errno.h>
19#include <asm/asm-offsets.h>
20#include <asm/percpu.h>
21#include <asm/thread_info.h>
22#include <asm/sal.h>
23#include <asm/signal.h>
24#include <asm/unistd.h>
25
26#include "entry.h"
27#include "paravirt_inst.h"
28
29/*
30 * See Documentation/ia64/fsys.txt for details on fsyscalls.
31 *
32 * On entry to an fsyscall handler:
33 * r10 = 0 (i.e., defaults to "successful syscall return")
34 * r11 = saved ar.pfs (a user-level value)
35 * r15 = system call number
36 * r16 = "current" task pointer (in normal kernel-mode, this is in r13)
37 * r32-r39 = system call arguments
38 * b6 = return address (a user-level value)
39 * ar.pfs = previous frame-state (a user-level value)
40 * PSR.be = cleared to zero (i.e., little-endian byte order is in effect)
41 * all other registers may contain values passed in from user-mode
42 *
43 * On return from an fsyscall handler:
44 * r11 = saved ar.pfs (as passed into the fsyscall handler)
45 * r15 = system call number (as passed into the fsyscall handler)
46 * r32-r39 = system call arguments (as passed into the fsyscall handler)
47 * b6 = return address (as passed into the fsyscall handler)
48 * ar.pfs = previous frame-state (as passed into the fsyscall handler)
49 */
50
51ENTRY(fsys_ni_syscall)
52 .prologue
53 .altrp b6
54 .body
55 mov r8=ENOSYS
56 mov r10=-1
57 FSYS_RETURN
58END(fsys_ni_syscall)
59
60ENTRY(fsys_getpid)
61 .prologue
62 .altrp b6
63 .body
64 add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
65 ;;
66 ld8 r17=[r17] // r17 = current->group_leader
67 add r9=TI_FLAGS+IA64_TASK_SIZE,r16
68 ;;
69 ld4 r9=[r9]
70 add r17=IA64_TASK_TGIDLINK_OFFSET,r17
71 ;;
72 and r9=TIF_ALLWORK_MASK,r9
73 ld8 r17=[r17] // r17 = current->group_leader->pids[PIDTYPE_PID].pid
74 ;;
75 add r8=IA64_PID_LEVEL_OFFSET,r17
76 ;;
77 ld4 r8=[r8] // r8 = pid->level
78 add r17=IA64_PID_UPID_OFFSET,r17 // r17 = &pid->numbers[0]
79 ;;
80 shl r8=r8,IA64_UPID_SHIFT
81 ;;
82 add r17=r17,r8 // r17 = &pid->numbers[pid->level]
83 ;;
84 ld4 r8=[r17] // r8 = pid->numbers[pid->level].nr
85 ;;
86 mov r17=0
87 ;;
88 cmp.ne p8,p0=0,r9
89(p8) br.spnt.many fsys_fallback_syscall
90 FSYS_RETURN
91END(fsys_getpid)
92
93ENTRY(fsys_getppid)
94 .prologue
95 .altrp b6
96 .body
97 add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
98 ;;
99 ld8 r17=[r17] // r17 = current->group_leader
100 add r9=TI_FLAGS+IA64_TASK_SIZE,r16
101 ;;
102
103 ld4 r9=[r9]
104 add r17=IA64_TASK_REAL_PARENT_OFFSET,r17 // r17 = ¤t->group_leader->real_parent
105 ;;
106 and r9=TIF_ALLWORK_MASK,r9
107
1081: ld8 r18=[r17] // r18 = current->group_leader->real_parent
109 ;;
110 cmp.ne p8,p0=0,r9
111 add r8=IA64_TASK_TGID_OFFSET,r18 // r8 = ¤t->group_leader->real_parent->tgid
112 ;;
113
114 /*
115 * The .acq is needed to ensure that the read of tgid has returned its data before
116 * we re-check "real_parent".
117 */
118 ld4.acq r8=[r8] // r8 = current->group_leader->real_parent->tgid
119#ifdef CONFIG_SMP
120 /*
121 * Re-read current->group_leader->real_parent.
122 */
123 ld8 r19=[r17] // r19 = current->group_leader->real_parent
124(p8) br.spnt.many fsys_fallback_syscall
125 ;;
126 cmp.ne p6,p0=r18,r19 // did real_parent change?
127 mov r19=0 // i must not leak kernel bits...
128(p6) br.cond.spnt.few 1b // yes -> redo the read of tgid and the check
129 ;;
130 mov r17=0 // i must not leak kernel bits...
131 mov r18=0 // i must not leak kernel bits...
132#else
133 mov r17=0 // i must not leak kernel bits...
134 mov r18=0 // i must not leak kernel bits...
135 mov r19=0 // i must not leak kernel bits...
136#endif
137 FSYS_RETURN
138END(fsys_getppid)
139
140ENTRY(fsys_set_tid_address)
141 .prologue
142 .altrp b6
143 .body
144 add r9=TI_FLAGS+IA64_TASK_SIZE,r16
145 add r17=IA64_TASK_TGIDLINK_OFFSET,r16
146 ;;
147 ld4 r9=[r9]
148 tnat.z p6,p7=r32 // check argument register for being NaT
149 ld8 r17=[r17] // r17 = current->pids[PIDTYPE_PID].pid
150 ;;
151 and r9=TIF_ALLWORK_MASK,r9
152 add r8=IA64_PID_LEVEL_OFFSET,r17
153 add r18=IA64_TASK_CLEAR_CHILD_TID_OFFSET,r16
154 ;;
155 ld4 r8=[r8] // r8 = pid->level
156 add r17=IA64_PID_UPID_OFFSET,r17 // r17 = &pid->numbers[0]
157 ;;
158 shl r8=r8,IA64_UPID_SHIFT
159 ;;
160 add r17=r17,r8 // r17 = &pid->numbers[pid->level]
161 ;;
162 ld4 r8=[r17] // r8 = pid->numbers[pid->level].nr
163 ;;
164 cmp.ne p8,p0=0,r9
165 mov r17=-1
166 ;;
167(p6) st8 [r18]=r32
168(p7) st8 [r18]=r17
169(p8) br.spnt.many fsys_fallback_syscall
170 ;;
171 mov r17=0 // i must not leak kernel bits...
172 mov r18=0 // i must not leak kernel bits...
173 FSYS_RETURN
174END(fsys_set_tid_address)
175
176#if IA64_GTOD_SEQ_OFFSET !=0
177#error fsys_gettimeofday incompatible with changes to struct fsyscall_gtod_data_t
178#endif
179#if IA64_ITC_JITTER_OFFSET !=0
180#error fsys_gettimeofday incompatible with changes to struct itc_jitter_data_t
181#endif
182#define CLOCK_REALTIME 0
183#define CLOCK_MONOTONIC 1
184#define CLOCK_DIVIDE_BY_1000 0x4000
185#define CLOCK_ADD_MONOTONIC 0x8000
186
187ENTRY(fsys_gettimeofday)
188 .prologue
189 .altrp b6
190 .body
191 mov r31 = r32
192 tnat.nz p6,p0 = r33 // guard against NaT argument
193(p6) br.cond.spnt.few .fail_einval
194 mov r30 = CLOCK_DIVIDE_BY_1000
195 ;;
196.gettime:
197 // Register map
198 // Incoming r31 = pointer to address where to place result
199 // r30 = flags determining how time is processed
200 // r2,r3 = temp r4-r7 preserved
201 // r8 = result nanoseconds
202 // r9 = result seconds
203 // r10 = temporary storage for clock difference
204 // r11 = preserved: saved ar.pfs
205 // r12 = preserved: memory stack
206 // r13 = preserved: thread pointer
207 // r14 = address of mask / mask value
208 // r15 = preserved: system call number
209 // r16 = preserved: current task pointer
210 // r17 = (not used)
211 // r18 = (not used)
212 // r19 = address of itc_lastcycle
213 // r20 = struct fsyscall_gtod_data (= address of gtod_lock.sequence)
214 // r21 = address of mmio_ptr
215 // r22 = address of wall_time or monotonic_time
216 // r23 = address of shift / value
217 // r24 = address mult factor / cycle_last value
218 // r25 = itc_lastcycle value
219 // r26 = address clocksource cycle_last
220 // r27 = (not used)
221 // r28 = sequence number at the beginning of critcal section
222 // r29 = address of itc_jitter
223 // r30 = time processing flags / memory address
224 // r31 = pointer to result
225 // Predicates
226 // p6,p7 short term use
227 // p8 = timesource ar.itc
228 // p9 = timesource mmio64
229 // p10 = timesource mmio32 - not used
230 // p11 = timesource not to be handled by asm code
231 // p12 = memory time source ( = p9 | p10) - not used
232 // p13 = do cmpxchg with itc_lastcycle
233 // p14 = Divide by 1000
234 // p15 = Add monotonic
235 //
236 // Note that instructions are optimized for McKinley. McKinley can
237 // process two bundles simultaneously and therefore we continuously
238 // try to feed the CPU two bundles and then a stop.
239
240 add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
241 tnat.nz p6,p0 = r31 // guard against Nat argument
242(p6) br.cond.spnt.few .fail_einval
243 movl r20 = fsyscall_gtod_data // load fsyscall gettimeofday data address
244 ;;
245 ld4 r2 = [r2] // process work pending flags
246 movl r29 = itc_jitter_data // itc_jitter
247 add r22 = IA64_GTOD_WALL_TIME_OFFSET,r20 // wall_time
248 add r21 = IA64_CLKSRC_MMIO_OFFSET,r20
249 mov pr = r30,0xc000 // Set predicates according to function
250 ;;
251 and r2 = TIF_ALLWORK_MASK,r2
252 add r19 = IA64_ITC_LASTCYCLE_OFFSET,r29
253(p15) add r22 = IA64_GTOD_MONO_TIME_OFFSET,r20 // monotonic_time
254 ;;
255 add r26 = IA64_CLKSRC_CYCLE_LAST_OFFSET,r20 // clksrc_cycle_last
256 cmp.ne p6, p0 = 0, r2 // Fallback if work is scheduled
257(p6) br.cond.spnt.many fsys_fallback_syscall
258 ;;
259 // Begin critical section
260.time_redo:
261 ld4.acq r28 = [r20] // gtod_lock.sequence, Must take first
262 ;;
263 and r28 = ~1,r28 // And make sequence even to force retry if odd
264 ;;
265 ld8 r30 = [r21] // clocksource->mmio_ptr
266 add r24 = IA64_CLKSRC_MULT_OFFSET,r20
267 ld4 r2 = [r29] // itc_jitter value
268 add r23 = IA64_CLKSRC_SHIFT_OFFSET,r20
269 add r14 = IA64_CLKSRC_MASK_OFFSET,r20
270 ;;
271 ld4 r3 = [r24] // clocksource mult value
272 ld8 r14 = [r14] // clocksource mask value
273 cmp.eq p8,p9 = 0,r30 // use cpu timer if no mmio_ptr
274 ;;
275 setf.sig f7 = r3 // Setup for mult scaling of counter
276(p8) cmp.ne p13,p0 = r2,r0 // need itc_jitter compensation, set p13
277 ld4 r23 = [r23] // clocksource shift value
278 ld8 r24 = [r26] // get clksrc_cycle_last value
279(p9) cmp.eq p13,p0 = 0,r30 // if mmio_ptr, clear p13 jitter control
280 ;;
281 .pred.rel.mutex p8,p9
282 MOV_FROM_ITC(p8, p6, r2, r10) // CPU_TIMER. 36 clocks latency!!!
283(p9) ld8 r2 = [r30] // MMIO_TIMER. Could also have latency issues..
284(p13) ld8 r25 = [r19] // get itc_lastcycle value
285 ld8 r9 = [r22],IA64_TIMESPEC_TV_NSEC_OFFSET // tv_sec
286 ;;
287 ld8 r8 = [r22],-IA64_TIMESPEC_TV_NSEC_OFFSET // tv_nsec
288(p13) sub r3 = r25,r2 // Diff needed before comparison (thanks davidm)
289 ;;
290(p13) cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
291 sub r10 = r2,r24 // current_cycle - last_cycle
292 ;;
293(p6) sub r10 = r25,r24 // time we got was less than last_cycle
294(p7) mov ar.ccv = r25 // more than last_cycle. Prep for cmpxchg
295 ;;
296(p7) cmpxchg8.rel r3 = [r19],r2,ar.ccv
297 ;;
298(p7) cmp.ne p7,p0 = r25,r3 // if cmpxchg not successful
299 ;;
300(p7) sub r10 = r3,r24 // then use new last_cycle instead
301 ;;
302 and r10 = r10,r14 // Apply mask
303 ;;
304 setf.sig f8 = r10
305 nop.i 123
306 ;;
307 // fault check takes 5 cycles and we have spare time
308EX(.fail_efault, probe.w.fault r31, 3)
309 xmpy.l f8 = f8,f7 // nsec_per_cyc*(counter-last_counter)
310 ;;
311 getf.sig r2 = f8
312 mf
313 ;;
314 ld4 r10 = [r20] // gtod_lock.sequence
315 shr.u r2 = r2,r23 // shift by factor
316 ;;
317 add r8 = r8,r2 // Add xtime.nsecs
318 cmp4.ne p7,p0 = r28,r10
319(p7) br.cond.dpnt.few .time_redo // sequence number changed, redo
320 // End critical section.
321 // Now r8=tv->tv_nsec and r9=tv->tv_sec
322 mov r10 = r0
323 movl r2 = 1000000000
324 add r23 = IA64_TIMESPEC_TV_NSEC_OFFSET, r31
325(p14) movl r3 = 2361183241434822607 // Prep for / 1000 hack
326 ;;
327.time_normalize:
328 mov r21 = r8
329 cmp.ge p6,p0 = r8,r2
330(p14) shr.u r20 = r8, 3 // We can repeat this if necessary just wasting time
331 ;;
332(p14) setf.sig f8 = r20
333(p6) sub r8 = r8,r2
334(p6) add r9 = 1,r9 // two nops before the branch.
335(p14) setf.sig f7 = r3 // Chances for repeats are 1 in 10000 for gettod
336(p6) br.cond.dpnt.few .time_normalize
337 ;;
338 // Divided by 8 though shift. Now divide by 125
339 // The compiler was able to do that with a multiply
340 // and a shift and we do the same
341EX(.fail_efault, probe.w.fault r23, 3) // This also costs 5 cycles
342(p14) xmpy.hu f8 = f8, f7 // xmpy has 5 cycles latency so use it
343 ;;
344(p14) getf.sig r2 = f8
345 ;;
346 mov r8 = r0
347(p14) shr.u r21 = r2, 4
348 ;;
349EX(.fail_efault, st8 [r31] = r9)
350EX(.fail_efault, st8 [r23] = r21)
351 FSYS_RETURN
352.fail_einval:
353 mov r8 = EINVAL
354 mov r10 = -1
355 FSYS_RETURN
356.fail_efault:
357 mov r8 = EFAULT
358 mov r10 = -1
359 FSYS_RETURN
360END(fsys_gettimeofday)
361
362ENTRY(fsys_clock_gettime)
363 .prologue
364 .altrp b6
365 .body
366 cmp4.ltu p6, p0 = CLOCK_MONOTONIC, r32
367 // Fallback if this is not CLOCK_REALTIME or CLOCK_MONOTONIC
368(p6) br.spnt.few fsys_fallback_syscall
369 mov r31 = r33
370 shl r30 = r32,15
371 br.many .gettime
372END(fsys_clock_gettime)
373
374/*
375 * fsys_getcpu doesn't use the third parameter in this implementation. It reads
376 * current_thread_info()->cpu and corresponding node in cpu_to_node_map.
377 */
378ENTRY(fsys_getcpu)
379 .prologue
380 .altrp b6
381 .body
382 ;;
383 add r2=TI_FLAGS+IA64_TASK_SIZE,r16
384 tnat.nz p6,p0 = r32 // guard against NaT argument
385 add r3=TI_CPU+IA64_TASK_SIZE,r16
386 ;;
387 ld4 r3=[r3] // M r3 = thread_info->cpu
388 ld4 r2=[r2] // M r2 = thread_info->flags
389(p6) br.cond.spnt.few .fail_einval // B
390 ;;
391 tnat.nz p7,p0 = r33 // I guard against NaT argument
392(p7) br.cond.spnt.few .fail_einval // B
393 ;;
394 cmp.ne p6,p0=r32,r0
395 cmp.ne p7,p0=r33,r0
396 ;;
397#ifdef CONFIG_NUMA
398 movl r17=cpu_to_node_map
399 ;;
400EX(.fail_efault, (p6) probe.w.fault r32, 3) // M This takes 5 cycles
401EX(.fail_efault, (p7) probe.w.fault r33, 3) // M This takes 5 cycles
402 shladd r18=r3,1,r17
403 ;;
404 ld2 r20=[r18] // r20 = cpu_to_node_map[cpu]
405 and r2 = TIF_ALLWORK_MASK,r2
406 ;;
407 cmp.ne p8,p0=0,r2
408(p8) br.spnt.many fsys_fallback_syscall
409 ;;
410 ;;
411EX(.fail_efault, (p6) st4 [r32] = r3)
412EX(.fail_efault, (p7) st2 [r33] = r20)
413 mov r8=0
414 ;;
415#else
416EX(.fail_efault, (p6) probe.w.fault r32, 3) // M This takes 5 cycles
417EX(.fail_efault, (p7) probe.w.fault r33, 3) // M This takes 5 cycles
418 and r2 = TIF_ALLWORK_MASK,r2
419 ;;
420 cmp.ne p8,p0=0,r2
421(p8) br.spnt.many fsys_fallback_syscall
422 ;;
423EX(.fail_efault, (p6) st4 [r32] = r3)
424EX(.fail_efault, (p7) st2 [r33] = r0)
425 mov r8=0
426 ;;
427#endif
428 FSYS_RETURN
429END(fsys_getcpu)
430
431ENTRY(fsys_fallback_syscall)
432 .prologue
433 .altrp b6
434 .body
435 /*
436 * We only get here from light-weight syscall handlers. Thus, we already
437 * know that r15 contains a valid syscall number. No need to re-check.
438 */
439 adds r17=-1024,r15
440 movl r14=sys_call_table
441 ;;
442 RSM_PSR_I(p0, r26, r27)
443 shladd r18=r17,3,r14
444 ;;
445 ld8 r18=[r18] // load normal (heavy-weight) syscall entry-point
446 MOV_FROM_PSR(p0, r29, r26) // read psr (12 cyc load latency)
447 mov r27=ar.rsc
448 mov r21=ar.fpsr
449 mov r26=ar.pfs
450END(fsys_fallback_syscall)
451 /* FALL THROUGH */
452GLOBAL_ENTRY(paravirt_fsys_bubble_down)
453 .prologue
454 .altrp b6
455 .body
456 /*
457 * We get here for syscalls that don't have a lightweight
458 * handler. For those, we need to bubble down into the kernel
459 * and that requires setting up a minimal pt_regs structure,
460 * and initializing the CPU state more or less as if an
461 * interruption had occurred. To make syscall-restarts work,
462 * we setup pt_regs such that cr_iip points to the second
463 * instruction in syscall_via_break. Decrementing the IP
464 * hence will restart the syscall via break and not
465 * decrementing IP will return us to the caller, as usual.
466 * Note that we preserve the value of psr.pp rather than
467 * initializing it from dcr.pp. This makes it possible to
468 * distinguish fsyscall execution from other privileged
469 * execution.
470 *
471 * On entry:
472 * - normal fsyscall handler register usage, except
473 * that we also have:
474 * - r18: address of syscall entry point
475 * - r21: ar.fpsr
476 * - r26: ar.pfs
477 * - r27: ar.rsc
478 * - r29: psr
479 *
480 * We used to clear some PSR bits here but that requires slow
481 * serialization. Fortuntely, that isn't really necessary.
482 * The rationale is as follows: we used to clear bits
483 * ~PSR_PRESERVED_BITS in PSR.L. Since
484 * PSR_PRESERVED_BITS==PSR.{UP,MFL,MFH,PK,DT,PP,SP,RT,IC}, we
485 * ended up clearing PSR.{BE,AC,I,DFL,DFH,DI,DB,SI,TB}.
486 * However,
487 *
488 * PSR.BE : already is turned off in __kernel_syscall_via_epc()
489 * PSR.AC : don't care (kernel normally turns PSR.AC on)
490 * PSR.I : already turned off by the time paravirt_fsys_bubble_down gets
491 * invoked
492 * PSR.DFL: always 0 (kernel never turns it on)
493 * PSR.DFH: don't care --- kernel never touches f32-f127 on its own
494 * initiative
495 * PSR.DI : always 0 (kernel never turns it on)
496 * PSR.SI : always 0 (kernel never turns it on)
497 * PSR.DB : don't care --- kernel never enables kernel-level
498 * breakpoints
499 * PSR.TB : must be 0 already; if it wasn't zero on entry to
500 * __kernel_syscall_via_epc, the branch to paravirt_fsys_bubble_down
501 * will trigger a taken branch; the taken-trap-handler then
502 * converts the syscall into a break-based system-call.
503 */
504 /*
505 * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.
506 * The rest we have to synthesize.
507 */
508# define PSR_ONE_BITS ((3 << IA64_PSR_CPL0_BIT) \
509 | (0x1 << IA64_PSR_RI_BIT) \
510 | IA64_PSR_BN | IA64_PSR_I)
511
512 invala // M0|1
513 movl r14=ia64_ret_from_syscall // X
514
515 nop.m 0
516 movl r28=__kernel_syscall_via_break // X create cr.iip
517 ;;
518
519 mov r2=r16 // A get task addr to addl-addressable register
520 adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // A
521 mov r31=pr // I0 save pr (2 cyc)
522 ;;
523 st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
524 addl r22=IA64_RBS_OFFSET,r2 // A compute base of RBS
525 add r3=TI_FLAGS+IA64_TASK_SIZE,r2 // A
526 ;;
527 ld4 r3=[r3] // M0|1 r3 = current_thread_info()->flags
528 lfetch.fault.excl.nt1 [r22] // M0|1 prefetch register backing-store
529 nop.i 0
530 ;;
531 mov ar.rsc=0 // M2 set enforced lazy mode, pl 0, LE, loadrs=0
532#ifdef CONFIG_VIRT_CPU_ACCOUNTING
533 MOV_FROM_ITC(p0, p6, r30, r23) // M get cycle for accounting
534#else
535 nop.m 0
536#endif
537 nop.i 0
538 ;;
539 mov r23=ar.bspstore // M2 (12 cyc) save ar.bspstore
540 mov.m r24=ar.rnat // M2 (5 cyc) read ar.rnat (dual-issues!)
541 nop.i 0
542 ;;
543 mov ar.bspstore=r22 // M2 (6 cyc) switch to kernel RBS
544 movl r8=PSR_ONE_BITS // X
545 ;;
546 mov r25=ar.unat // M2 (5 cyc) save ar.unat
547 mov r19=b6 // I0 save b6 (2 cyc)
548 mov r20=r1 // A save caller's gp in r20
549 ;;
550 or r29=r8,r29 // A construct cr.ipsr value to save
551 mov b6=r18 // I0 copy syscall entry-point to b6 (7 cyc)
552 addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2 // A compute base of memory stack
553
554 mov r18=ar.bsp // M2 save (kernel) ar.bsp (12 cyc)
555 cmp.ne pKStk,pUStk=r0,r0 // A set pKStk <- 0, pUStk <- 1
556 br.call.sptk.many b7=ia64_syscall_setup // B
557 ;;
558#ifdef CONFIG_VIRT_CPU_ACCOUNTING
559 // mov.m r30=ar.itc is called in advance
560 add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2
561 add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2
562 ;;
563 ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // time at last check in kernel
564 ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // time at leave kernel
565 ;;
566 ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // cumulated stime
567 ld8 r21=[r17] // cumulated utime
568 sub r22=r19,r18 // stime before leave kernel
569 ;;
570 st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // update stamp
571 sub r18=r30,r19 // elapsed time in user mode
572 ;;
573 add r20=r20,r22 // sum stime
574 add r21=r21,r18 // sum utime
575 ;;
576 st8 [r16]=r20 // update stime
577 st8 [r17]=r21 // update utime
578 ;;
579#endif
580 mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
581 mov rp=r14 // I0 set the real return addr
582 and r3=_TIF_SYSCALL_TRACEAUDIT,r3 // A
583 ;;
584 SSM_PSR_I(p0, p6, r22) // M2 we're on kernel stacks now, reenable irqs
585 cmp.eq p8,p0=r3,r0 // A
586(p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
587
588 nop.m 0
589(p8) br.call.sptk.many b6=b6 // B (ignore return address)
590 br.cond.spnt ia64_trace_syscall // B
591END(paravirt_fsys_bubble_down)
592
593 .rodata
594 .align 8
595 .globl paravirt_fsyscall_table
596
597 data8 paravirt_fsys_bubble_down
598paravirt_fsyscall_table:
599 data8 fsys_ni_syscall
600 data8 0 // exit // 1025
601 data8 0 // read
602 data8 0 // write
603 data8 0 // open
604 data8 0 // close
605 data8 0 // creat // 1030
606 data8 0 // link
607 data8 0 // unlink
608 data8 0 // execve
609 data8 0 // chdir
610 data8 0 // fchdir // 1035
611 data8 0 // utimes
612 data8 0 // mknod
613 data8 0 // chmod
614 data8 0 // chown
615 data8 0 // lseek // 1040
616 data8 fsys_getpid // getpid
617 data8 fsys_getppid // getppid
618 data8 0 // mount
619 data8 0 // umount
620 data8 0 // setuid // 1045
621 data8 0 // getuid
622 data8 0 // geteuid
623 data8 0 // ptrace
624 data8 0 // access
625 data8 0 // sync // 1050
626 data8 0 // fsync
627 data8 0 // fdatasync
628 data8 0 // kill
629 data8 0 // rename
630 data8 0 // mkdir // 1055
631 data8 0 // rmdir
632 data8 0 // dup
633 data8 0 // pipe
634 data8 0 // times
635 data8 0 // brk // 1060
636 data8 0 // setgid
637 data8 0 // getgid
638 data8 0 // getegid
639 data8 0 // acct
640 data8 0 // ioctl // 1065
641 data8 0 // fcntl
642 data8 0 // umask
643 data8 0 // chroot
644 data8 0 // ustat
645 data8 0 // dup2 // 1070
646 data8 0 // setreuid
647 data8 0 // setregid
648 data8 0 // getresuid
649 data8 0 // setresuid
650 data8 0 // getresgid // 1075
651 data8 0 // setresgid
652 data8 0 // getgroups
653 data8 0 // setgroups
654 data8 0 // getpgid
655 data8 0 // setpgid // 1080
656 data8 0 // setsid
657 data8 0 // getsid
658 data8 0 // sethostname
659 data8 0 // setrlimit
660 data8 0 // getrlimit // 1085
661 data8 0 // getrusage
662 data8 fsys_gettimeofday // gettimeofday
663 data8 0 // settimeofday
664 data8 0 // select
665 data8 0 // poll // 1090
666 data8 0 // symlink
667 data8 0 // readlink
668 data8 0 // uselib
669 data8 0 // swapon
670 data8 0 // swapoff // 1095
671 data8 0 // reboot
672 data8 0 // truncate
673 data8 0 // ftruncate
674 data8 0 // fchmod
675 data8 0 // fchown // 1100
676 data8 0 // getpriority
677 data8 0 // setpriority
678 data8 0 // statfs
679 data8 0 // fstatfs
680 data8 0 // gettid // 1105
681 data8 0 // semget
682 data8 0 // semop
683 data8 0 // semctl
684 data8 0 // msgget
685 data8 0 // msgsnd // 1110
686 data8 0 // msgrcv
687 data8 0 // msgctl
688 data8 0 // shmget
689 data8 0 // shmat
690 data8 0 // shmdt // 1115
691 data8 0 // shmctl
692 data8 0 // syslog
693 data8 0 // setitimer
694 data8 0 // getitimer
695 data8 0 // 1120
696 data8 0
697 data8 0
698 data8 0 // vhangup
699 data8 0 // lchown
700 data8 0 // remap_file_pages // 1125
701 data8 0 // wait4
702 data8 0 // sysinfo
703 data8 0 // clone
704 data8 0 // setdomainname
705 data8 0 // newuname // 1130
706 data8 0 // adjtimex
707 data8 0
708 data8 0 // init_module
709 data8 0 // delete_module
710 data8 0 // 1135
711 data8 0
712 data8 0 // quotactl
713 data8 0 // bdflush
714 data8 0 // sysfs
715 data8 0 // personality // 1140
716 data8 0 // afs_syscall
717 data8 0 // setfsuid
718 data8 0 // setfsgid
719 data8 0 // getdents
720 data8 0 // flock // 1145
721 data8 0 // readv
722 data8 0 // writev
723 data8 0 // pread64
724 data8 0 // pwrite64
725 data8 0 // sysctl // 1150
726 data8 0 // mmap
727 data8 0 // munmap
728 data8 0 // mlock
729 data8 0 // mlockall
730 data8 0 // mprotect // 1155
731 data8 0 // mremap
732 data8 0 // msync
733 data8 0 // munlock
734 data8 0 // munlockall
735 data8 0 // sched_getparam // 1160
736 data8 0 // sched_setparam
737 data8 0 // sched_getscheduler
738 data8 0 // sched_setscheduler
739 data8 0 // sched_yield
740 data8 0 // sched_get_priority_max // 1165
741 data8 0 // sched_get_priority_min
742 data8 0 // sched_rr_get_interval
743 data8 0 // nanosleep
744 data8 0 // nfsservctl
745 data8 0 // prctl // 1170
746 data8 0 // getpagesize
747 data8 0 // mmap2
748 data8 0 // pciconfig_read
749 data8 0 // pciconfig_write
750 data8 0 // perfmonctl // 1175
751 data8 0 // sigaltstack
752 data8 0 // rt_sigaction
753 data8 0 // rt_sigpending
754 data8 0 // rt_sigprocmask
755 data8 0 // rt_sigqueueinfo // 1180
756 data8 0 // rt_sigreturn
757 data8 0 // rt_sigsuspend
758 data8 0 // rt_sigtimedwait
759 data8 0 // getcwd
760 data8 0 // capget // 1185
761 data8 0 // capset
762 data8 0 // sendfile
763 data8 0
764 data8 0
765 data8 0 // socket // 1190
766 data8 0 // bind
767 data8 0 // connect
768 data8 0 // listen
769 data8 0 // accept
770 data8 0 // getsockname // 1195
771 data8 0 // getpeername
772 data8 0 // socketpair
773 data8 0 // send
774 data8 0 // sendto
775 data8 0 // recv // 1200
776 data8 0 // recvfrom
777 data8 0 // shutdown
778 data8 0 // setsockopt
779 data8 0 // getsockopt
780 data8 0 // sendmsg // 1205
781 data8 0 // recvmsg
782 data8 0 // pivot_root
783 data8 0 // mincore
784 data8 0 // madvise
785 data8 0 // newstat // 1210
786 data8 0 // newlstat
787 data8 0 // newfstat
788 data8 0 // clone2
789 data8 0 // getdents64
790 data8 0 // getunwind // 1215
791 data8 0 // readahead
792 data8 0 // setxattr
793 data8 0 // lsetxattr
794 data8 0 // fsetxattr
795 data8 0 // getxattr // 1220
796 data8 0 // lgetxattr
797 data8 0 // fgetxattr
798 data8 0 // listxattr
799 data8 0 // llistxattr
800 data8 0 // flistxattr // 1225
801 data8 0 // removexattr
802 data8 0 // lremovexattr
803 data8 0 // fremovexattr
804 data8 0 // tkill
805 data8 0 // futex // 1230
806 data8 0 // sched_setaffinity
807 data8 0 // sched_getaffinity
808 data8 fsys_set_tid_address // set_tid_address
809 data8 0 // fadvise64_64
810 data8 0 // tgkill // 1235
811 data8 0 // exit_group
812 data8 0 // lookup_dcookie
813 data8 0 // io_setup
814 data8 0 // io_destroy
815 data8 0 // io_getevents // 1240
816 data8 0 // io_submit
817 data8 0 // io_cancel
818 data8 0 // epoll_create
819 data8 0 // epoll_ctl
820 data8 0 // epoll_wait // 1245
821 data8 0 // restart_syscall
822 data8 0 // semtimedop
823 data8 0 // timer_create
824 data8 0 // timer_settime
825 data8 0 // timer_gettime // 1250
826 data8 0 // timer_getoverrun
827 data8 0 // timer_delete
828 data8 0 // clock_settime
829 data8 fsys_clock_gettime // clock_gettime
830 data8 0 // clock_getres // 1255
831 data8 0 // clock_nanosleep
832 data8 0 // fstatfs64
833 data8 0 // statfs64
834 data8 0 // mbind
835 data8 0 // get_mempolicy // 1260
836 data8 0 // set_mempolicy
837 data8 0 // mq_open
838 data8 0 // mq_unlink
839 data8 0 // mq_timedsend
840 data8 0 // mq_timedreceive // 1265
841 data8 0 // mq_notify
842 data8 0 // mq_getsetattr
843 data8 0 // kexec_load
844 data8 0 // vserver
845 data8 0 // waitid // 1270
846 data8 0 // add_key
847 data8 0 // request_key
848 data8 0 // keyctl
849 data8 0 // ioprio_set
850 data8 0 // ioprio_get // 1275
851 data8 0 // move_pages
852 data8 0 // inotify_init
853 data8 0 // inotify_add_watch
854 data8 0 // inotify_rm_watch
855 data8 0 // migrate_pages // 1280
856 data8 0 // openat
857 data8 0 // mkdirat
858 data8 0 // mknodat
859 data8 0 // fchownat
860 data8 0 // futimesat // 1285
861 data8 0 // newfstatat
862 data8 0 // unlinkat
863 data8 0 // renameat
864 data8 0 // linkat
865 data8 0 // symlinkat // 1290
866 data8 0 // readlinkat
867 data8 0 // fchmodat
868 data8 0 // faccessat
869 data8 0
870 data8 0 // 1295
871 data8 0 // unshare
872 data8 0 // splice
873 data8 0 // set_robust_list
874 data8 0 // get_robust_list
875 data8 0 // sync_file_range // 1300
876 data8 0 // tee
877 data8 0 // vmsplice
878 data8 0
879 data8 fsys_getcpu // getcpu // 1304
880
881 // fill in zeros for the remaining entries
882 .zero:
883 .space paravirt_fsyscall_table + 8*NR_syscalls - .zero, 0