Merge branch 'proc-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/adobriyan...
[linux-2.6] / arch / ia64 / kernel / fsys.S
1 /*
2  * This file contains the light-weight system call handlers (fsyscall-handlers).
3  *
4  * Copyright (C) 2003 Hewlett-Packard Co
5  *      David Mosberger-Tang <davidm@hpl.hp.com>
6  *
7  * 25-Sep-03 davidm     Implement fsys_rt_sigprocmask().
8  * 18-Feb-03 louisk     Implement fsys_gettimeofday().
9  * 28-Feb-03 davidm     Fixed several bugs in fsys_gettimeofday().  Tuned it some more,
10  *                      probably broke it along the way... ;-)
11  * 13-Jul-04 clameter   Implement fsys_clock_gettime and revise fsys_gettimeofday to make
12  *                      it capable of using memory based clocks without falling back to C code.
13  * 08-Feb-07 Fenghua Yu Implement fsys_getcpu.
14  *
15  */
16
17 #include <asm/asmmacro.h>
18 #include <asm/errno.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/percpu.h>
21 #include <asm/thread_info.h>
22 #include <asm/sal.h>
23 #include <asm/signal.h>
24 #include <asm/system.h>
25 #include <asm/unistd.h>
26
27 #include "entry.h"
28
29 /*
30  * See Documentation/ia64/fsys.txt for details on fsyscalls.
31  *
32  * On entry to an fsyscall handler:
33  *   r10        = 0 (i.e., defaults to "successful syscall return")
34  *   r11        = saved ar.pfs (a user-level value)
35  *   r15        = system call number
36  *   r16        = "current" task pointer (in normal kernel-mode, this is in r13)
37  *   r32-r39    = system call arguments
38  *   b6         = return address (a user-level value)
39  *   ar.pfs     = previous frame-state (a user-level value)
40  *   PSR.be     = cleared to zero (i.e., little-endian byte order is in effect)
41  *   all other registers may contain values passed in from user-mode
42  *
43  * On return from an fsyscall handler:
44  *   r11        = saved ar.pfs (as passed into the fsyscall handler)
45  *   r15        = system call number (as passed into the fsyscall handler)
46  *   r32-r39    = system call arguments (as passed into the fsyscall handler)
47  *   b6         = return address (as passed into the fsyscall handler)
48  *   ar.pfs     = previous frame-state (as passed into the fsyscall handler)
49  */
50
51 ENTRY(fsys_ni_syscall)
52         .prologue
53         .altrp b6
54         .body
55         mov r8=ENOSYS
56         mov r10=-1
57         FSYS_RETURN
58 END(fsys_ni_syscall)
59
60 ENTRY(fsys_getpid)
61         .prologue
62         .altrp b6
63         .body
64         add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
65         ;;
66         ld8 r17=[r17]                           // r17 = current->group_leader
67         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
68         ;;
69         ld4 r9=[r9]
70         add r17=IA64_TASK_TGIDLINK_OFFSET,r17
71         ;;
72         and r9=TIF_ALLWORK_MASK,r9
73         ld8 r17=[r17]                           // r17 = current->group_leader->pids[PIDTYPE_PID].pid
74         ;;
75         add r8=IA64_PID_LEVEL_OFFSET,r17
76         ;;
77         ld4 r8=[r8]                             // r8 = pid->level
78         add r17=IA64_PID_UPID_OFFSET,r17        // r17 = &pid->numbers[0]
79         ;;
80         shl r8=r8,IA64_UPID_SHIFT
81         ;;
82         add r17=r17,r8                          // r17 = &pid->numbers[pid->level]
83         ;;
84         ld4 r8=[r17]                            // r8 = pid->numbers[pid->level].nr
85         ;;
86         mov r17=0
87         ;;
88         cmp.ne p8,p0=0,r9
89 (p8)    br.spnt.many fsys_fallback_syscall
90         FSYS_RETURN
91 END(fsys_getpid)
92
93 ENTRY(fsys_getppid)
94         .prologue
95         .altrp b6
96         .body
97         add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
98         ;;
99         ld8 r17=[r17]                           // r17 = current->group_leader
100         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
101         ;;
102
103         ld4 r9=[r9]
104         add r17=IA64_TASK_REAL_PARENT_OFFSET,r17 // r17 = &current->group_leader->real_parent
105         ;;
106         and r9=TIF_ALLWORK_MASK,r9
107
108 1:      ld8 r18=[r17]                           // r18 = current->group_leader->real_parent
109         ;;
110         cmp.ne p8,p0=0,r9
111         add r8=IA64_TASK_TGID_OFFSET,r18        // r8 = &current->group_leader->real_parent->tgid
112         ;;
113
114         /*
115          * The .acq is needed to ensure that the read of tgid has returned its data before
116          * we re-check "real_parent".
117          */
118         ld4.acq r8=[r8]                         // r8 = current->group_leader->real_parent->tgid
119 #ifdef CONFIG_SMP
120         /*
121          * Re-read current->group_leader->real_parent.
122          */
123         ld8 r19=[r17]                           // r19 = current->group_leader->real_parent
124 (p8)    br.spnt.many fsys_fallback_syscall
125         ;;
126         cmp.ne p6,p0=r18,r19                    // did real_parent change?
127         mov r19=0                       // i must not leak kernel bits...
128 (p6)    br.cond.spnt.few 1b                     // yes -> redo the read of tgid and the check
129         ;;
130         mov r17=0                       // i must not leak kernel bits...
131         mov r18=0                       // i must not leak kernel bits...
132 #else
133         mov r17=0                       // i must not leak kernel bits...
134         mov r18=0                       // i must not leak kernel bits...
135         mov r19=0                       // i must not leak kernel bits...
136 #endif
137         FSYS_RETURN
138 END(fsys_getppid)
139
140 ENTRY(fsys_set_tid_address)
141         .prologue
142         .altrp b6
143         .body
144         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
145         add r17=IA64_TASK_TGIDLINK_OFFSET,r16
146         ;;
147         ld4 r9=[r9]
148         tnat.z p6,p7=r32                // check argument register for being NaT
149         ld8 r17=[r17]                           // r17 = current->pids[PIDTYPE_PID].pid
150         ;;
151         and r9=TIF_ALLWORK_MASK,r9
152         add r8=IA64_PID_LEVEL_OFFSET,r17
153         add r18=IA64_TASK_CLEAR_CHILD_TID_OFFSET,r16
154         ;;
155         ld4 r8=[r8]                             // r8 = pid->level
156         add r17=IA64_PID_UPID_OFFSET,r17        // r17 = &pid->numbers[0]
157         ;;
158         shl r8=r8,IA64_UPID_SHIFT
159         ;;
160         add r17=r17,r8                          // r17 = &pid->numbers[pid->level]
161         ;;
162         ld4 r8=[r17]                            // r8 = pid->numbers[pid->level].nr
163         ;;
164         cmp.ne p8,p0=0,r9
165         mov r17=-1
166         ;;
167 (p6)    st8 [r18]=r32
168 (p7)    st8 [r18]=r17
169 (p8)    br.spnt.many fsys_fallback_syscall
170         ;;
171         mov r17=0                       // i must not leak kernel bits...
172         mov r18=0                       // i must not leak kernel bits...
173         FSYS_RETURN
174 END(fsys_set_tid_address)
175
176 #if IA64_GTOD_LOCK_OFFSET !=0
177 #error fsys_gettimeofday incompatible with changes to struct fsyscall_gtod_data_t
178 #endif
179 #if IA64_ITC_JITTER_OFFSET !=0
180 #error fsys_gettimeofday incompatible with changes to struct itc_jitter_data_t
181 #endif
182 #define CLOCK_REALTIME 0
183 #define CLOCK_MONOTONIC 1
184 #define CLOCK_DIVIDE_BY_1000 0x4000
185 #define CLOCK_ADD_MONOTONIC 0x8000
186
187 ENTRY(fsys_gettimeofday)
188         .prologue
189         .altrp b6
190         .body
191         mov r31 = r32
192         tnat.nz p6,p0 = r33             // guard against NaT argument
193 (p6)    br.cond.spnt.few .fail_einval
194         mov r30 = CLOCK_DIVIDE_BY_1000
195         ;;
196 .gettime:
197         // Register map
198         // Incoming r31 = pointer to address where to place result
199         //          r30 = flags determining how time is processed
200         // r2,r3 = temp r4-r7 preserved
201         // r8 = result nanoseconds
202         // r9 = result seconds
203         // r10 = temporary storage for clock difference
204         // r11 = preserved: saved ar.pfs
205         // r12 = preserved: memory stack
206         // r13 = preserved: thread pointer
207         // r14 = address of mask / mask value
208         // r15 = preserved: system call number
209         // r16 = preserved: current task pointer
210         // r17 = (not used)
211         // r18 = (not used)
212         // r19 = address of itc_lastcycle
213         // r20 = struct fsyscall_gtod_data (= address of gtod_lock.sequence)
214         // r21 = address of mmio_ptr
215         // r22 = address of wall_time or monotonic_time
216         // r23 = address of shift / value
217         // r24 = address mult factor / cycle_last value
218         // r25 = itc_lastcycle value
219         // r26 = address clocksource cycle_last
220         // r27 = (not used)
221         // r28 = sequence number at the beginning of critcal section
222         // r29 = address of itc_jitter
223         // r30 = time processing flags / memory address
224         // r31 = pointer to result
225         // Predicates
226         // p6,p7 short term use
227         // p8 = timesource ar.itc
228         // p9 = timesource mmio64
229         // p10 = timesource mmio32 - not used
230         // p11 = timesource not to be handled by asm code
231         // p12 = memory time source ( = p9 | p10) - not used
232         // p13 = do cmpxchg with itc_lastcycle
233         // p14 = Divide by 1000
234         // p15 = Add monotonic
235         //
236         // Note that instructions are optimized for McKinley. McKinley can
237         // process two bundles simultaneously and therefore we continuously
238         // try to feed the CPU two bundles and then a stop.
239
240         add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
241         tnat.nz p6,p0 = r31             // guard against Nat argument
242 (p6)    br.cond.spnt.few .fail_einval
243         movl r20 = fsyscall_gtod_data // load fsyscall gettimeofday data address
244         ;;
245         ld4 r2 = [r2]                   // process work pending flags
246         movl r29 = itc_jitter_data      // itc_jitter
247         add r22 = IA64_GTOD_WALL_TIME_OFFSET,r20        // wall_time
248         add r21 = IA64_CLKSRC_MMIO_OFFSET,r20
249         mov pr = r30,0xc000     // Set predicates according to function
250         ;;
251         and r2 = TIF_ALLWORK_MASK,r2
252         add r19 = IA64_ITC_LASTCYCLE_OFFSET,r29
253 (p15)   add r22 = IA64_GTOD_MONO_TIME_OFFSET,r20        // monotonic_time
254         ;;
255         add r26 = IA64_CLKSRC_CYCLE_LAST_OFFSET,r20     // clksrc_cycle_last
256         cmp.ne p6, p0 = 0, r2   // Fallback if work is scheduled
257 (p6)    br.cond.spnt.many fsys_fallback_syscall
258         ;;
259         // Begin critical section
260 .time_redo:
261         ld4.acq r28 = [r20]     // gtod_lock.sequence, Must take first
262         ;;
263         and r28 = ~1,r28        // And make sequence even to force retry if odd
264         ;;
265         ld8 r30 = [r21]         // clocksource->mmio_ptr
266         add r24 = IA64_CLKSRC_MULT_OFFSET,r20
267         ld4 r2 = [r29]          // itc_jitter value
268         add r23 = IA64_CLKSRC_SHIFT_OFFSET,r20
269         add r14 = IA64_CLKSRC_MASK_OFFSET,r20
270         ;;
271         ld4 r3 = [r24]          // clocksource mult value
272         ld8 r14 = [r14]         // clocksource mask value
273         cmp.eq p8,p9 = 0,r30    // use cpu timer if no mmio_ptr
274         ;;
275         setf.sig f7 = r3        // Setup for mult scaling of counter
276 (p8)    cmp.ne p13,p0 = r2,r0   // need itc_jitter compensation, set p13
277         ld4 r23 = [r23]         // clocksource shift value
278         ld8 r24 = [r26]         // get clksrc_cycle_last value
279 (p9)    cmp.eq p13,p0 = 0,r30   // if mmio_ptr, clear p13 jitter control
280         ;;
281         .pred.rel.mutex p8,p9
282 (p8)    mov r2 = ar.itc         // CPU_TIMER. 36 clocks latency!!!
283 (p9)    ld8 r2 = [r30]          // MMIO_TIMER. Could also have latency issues..
284 (p13)   ld8 r25 = [r19]         // get itc_lastcycle value
285         ld8 r9 = [r22],IA64_TIMESPEC_TV_NSEC_OFFSET     // tv_sec
286         ;;
287         ld8 r8 = [r22],-IA64_TIMESPEC_TV_NSEC_OFFSET    // tv_nsec
288 (p13)   sub r3 = r25,r2         // Diff needed before comparison (thanks davidm)
289         ;;
290 (p13)   cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
291         sub r10 = r2,r24        // current_cycle - last_cycle
292         ;;
293 (p6)    sub r10 = r25,r24       // time we got was less than last_cycle
294 (p7)    mov ar.ccv = r25        // more than last_cycle. Prep for cmpxchg
295         ;;
296 (p7)    cmpxchg8.rel r3 = [r19],r2,ar.ccv
297         ;;
298 (p7)    cmp.ne p7,p0 = r25,r3   // if cmpxchg not successful
299         ;;
300 (p7)    sub r10 = r3,r24        // then use new last_cycle instead
301         ;;
302         and r10 = r10,r14       // Apply mask
303         ;;
304         setf.sig f8 = r10
305         nop.i 123
306         ;;
307         // fault check takes 5 cycles and we have spare time
308 EX(.fail_efault, probe.w.fault r31, 3)
309         xmpy.l f8 = f8,f7       // nsec_per_cyc*(counter-last_counter)
310         ;;
311         getf.sig r2 = f8
312         mf
313         ;;
314         ld4 r10 = [r20]         // gtod_lock.sequence
315         shr.u r2 = r2,r23       // shift by factor
316         ;;
317         add r8 = r8,r2          // Add xtime.nsecs
318         cmp4.ne p7,p0 = r28,r10
319 (p7)    br.cond.dpnt.few .time_redo     // sequence number changed, redo
320         // End critical section.
321         // Now r8=tv->tv_nsec and r9=tv->tv_sec
322         mov r10 = r0
323         movl r2 = 1000000000
324         add r23 = IA64_TIMESPEC_TV_NSEC_OFFSET, r31
325 (p14)   movl r3 = 2361183241434822607   // Prep for / 1000 hack
326         ;;
327 .time_normalize:
328         mov r21 = r8
329         cmp.ge p6,p0 = r8,r2
330 (p14)   shr.u r20 = r8, 3 // We can repeat this if necessary just wasting time
331         ;;
332 (p14)   setf.sig f8 = r20
333 (p6)    sub r8 = r8,r2
334 (p6)    add r9 = 1,r9           // two nops before the branch.
335 (p14)   setf.sig f7 = r3        // Chances for repeats are 1 in 10000 for gettod
336 (p6)    br.cond.dpnt.few .time_normalize
337         ;;
338         // Divided by 8 though shift. Now divide by 125
339         // The compiler was able to do that with a multiply
340         // and a shift and we do the same
341 EX(.fail_efault, probe.w.fault r23, 3)  // This also costs 5 cycles
342 (p14)   xmpy.hu f8 = f8, f7             // xmpy has 5 cycles latency so use it
343         ;;
344 (p14)   getf.sig r2 = f8
345         ;;
346         mov r8 = r0
347 (p14)   shr.u r21 = r2, 4
348         ;;
349 EX(.fail_efault, st8 [r31] = r9)
350 EX(.fail_efault, st8 [r23] = r21)
351         FSYS_RETURN
352 .fail_einval:
353         mov r8 = EINVAL
354         mov r10 = -1
355         FSYS_RETURN
356 .fail_efault:
357         mov r8 = EFAULT
358         mov r10 = -1
359         FSYS_RETURN
360 END(fsys_gettimeofday)
361
362 ENTRY(fsys_clock_gettime)
363         .prologue
364         .altrp b6
365         .body
366         cmp4.ltu p6, p0 = CLOCK_MONOTONIC, r32
367         // Fallback if this is not CLOCK_REALTIME or CLOCK_MONOTONIC
368 (p6)    br.spnt.few fsys_fallback_syscall
369         mov r31 = r33
370         shl r30 = r32,15
371         br.many .gettime
372 END(fsys_clock_gettime)
373
374 /*
375  * long fsys_rt_sigprocmask (int how, sigset_t *set, sigset_t *oset, size_t sigsetsize).
376  */
377 #if _NSIG_WORDS != 1
378 # error Sorry, fsys_rt_sigprocmask() needs to be updated for _NSIG_WORDS != 1.
379 #endif
380 ENTRY(fsys_rt_sigprocmask)
381         .prologue
382         .altrp b6
383         .body
384
385         add r2=IA64_TASK_BLOCKED_OFFSET,r16
386         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
387         cmp4.ltu p6,p0=SIG_SETMASK,r32
388
389         cmp.ne p15,p0=r0,r34                    // oset != NULL?
390         tnat.nz p8,p0=r34
391         add r31=IA64_TASK_SIGHAND_OFFSET,r16
392         ;;
393         ld8 r3=[r2]                             // read/prefetch current->blocked
394         ld4 r9=[r9]
395         tnat.nz.or p6,p0=r35
396
397         cmp.ne.or p6,p0=_NSIG_WORDS*8,r35
398         tnat.nz.or p6,p0=r32
399 (p6)    br.spnt.few .fail_einval                // fail with EINVAL
400         ;;
401 #ifdef CONFIG_SMP
402         ld8 r31=[r31]                           // r31 <- current->sighand
403 #endif
404         and r9=TIF_ALLWORK_MASK,r9
405         tnat.nz.or p8,p0=r33
406         ;;
407         cmp.ne p7,p0=0,r9
408         cmp.eq p6,p0=r0,r33                     // set == NULL?
409         add r31=IA64_SIGHAND_SIGLOCK_OFFSET,r31 // r31 <- current->sighand->siglock
410 (p8)    br.spnt.few .fail_efault                // fail with EFAULT
411 (p7)    br.spnt.many fsys_fallback_syscall      // got pending kernel work...
412 (p6)    br.dpnt.many .store_mask                // -> short-circuit to just reading the signal mask
413
414         /* Argh, we actually have to do some work and _update_ the signal mask: */
415
416 EX(.fail_efault, probe.r.fault r33, 3)          // verify user has read-access to *set
417 EX(.fail_efault, ld8 r14=[r33])                 // r14 <- *set
418         mov r17=(1 << (SIGKILL - 1)) | (1 << (SIGSTOP - 1))
419         ;;
420
421         rsm psr.i                               // mask interrupt delivery
422         mov ar.ccv=0
423         andcm r14=r14,r17                       // filter out SIGKILL & SIGSTOP
424
425 #ifdef CONFIG_SMP
426         mov r17=1
427         ;;
428         cmpxchg4.acq r18=[r31],r17,ar.ccv       // try to acquire the lock
429         mov r8=EINVAL                   // default to EINVAL
430         ;;
431         ld8 r3=[r2]                     // re-read current->blocked now that we hold the lock
432         cmp4.ne p6,p0=r18,r0
433 (p6)    br.cond.spnt.many .lock_contention
434         ;;
435 #else
436         ld8 r3=[r2]                     // re-read current->blocked now that we hold the lock
437         mov r8=EINVAL                   // default to EINVAL
438 #endif
439         add r18=IA64_TASK_PENDING_OFFSET+IA64_SIGPENDING_SIGNAL_OFFSET,r16
440         add r19=IA64_TASK_SIGNAL_OFFSET,r16
441         cmp4.eq p6,p0=SIG_BLOCK,r32
442         ;;
443         ld8 r19=[r19]                   // r19 <- current->signal
444         cmp4.eq p7,p0=SIG_UNBLOCK,r32
445         cmp4.eq p8,p0=SIG_SETMASK,r32
446         ;;
447         ld8 r18=[r18]                   // r18 <- current->pending.signal
448         .pred.rel.mutex p6,p7,p8
449 (p6)    or r14=r3,r14                   // SIG_BLOCK
450 (p7)    andcm r14=r3,r14                // SIG_UNBLOCK
451
452 (p8)    mov r14=r14                     // SIG_SETMASK
453 (p6)    mov r8=0                        // clear error code
454         // recalc_sigpending()
455         add r17=IA64_SIGNAL_GROUP_STOP_COUNT_OFFSET,r19
456
457         add r19=IA64_SIGNAL_SHARED_PENDING_OFFSET+IA64_SIGPENDING_SIGNAL_OFFSET,r19
458         ;;
459         ld4 r17=[r17]           // r17 <- current->signal->group_stop_count
460 (p7)    mov r8=0                // clear error code
461
462         ld8 r19=[r19]           // r19 <- current->signal->shared_pending
463         ;;
464         cmp4.gt p6,p7=r17,r0    // p6/p7 <- (current->signal->group_stop_count > 0)?
465 (p8)    mov r8=0                // clear error code
466
467         or r18=r18,r19          // r18 <- current->pending | current->signal->shared_pending
468         ;;
469         // r18 <- (current->pending | current->signal->shared_pending) & ~current->blocked:
470         andcm r18=r18,r14
471         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
472         ;;
473
474 (p7)    cmp.ne.or.andcm p6,p7=r18,r0            // p6/p7 <- signal pending
475         mov r19=0                                       // i must not leak kernel bits...
476 (p6)    br.cond.dpnt.many .sig_pending
477         ;;
478
479 1:      ld4 r17=[r9]                            // r17 <- current->thread_info->flags
480         ;;
481         mov ar.ccv=r17
482         and r18=~_TIF_SIGPENDING,r17            // r18 <- r17 & ~(1 << TIF_SIGPENDING)
483         ;;
484
485         st8 [r2]=r14                            // update current->blocked with new mask
486         cmpxchg4.acq r8=[r9],r18,ar.ccv         // current->thread_info->flags <- r18
487         ;;
488         cmp.ne p6,p0=r17,r8                     // update failed?
489 (p6)    br.cond.spnt.few 1b                     // yes -> retry
490
491 #ifdef CONFIG_SMP
492         st4.rel [r31]=r0                        // release the lock
493 #endif
494         ssm psr.i
495         ;;
496
497         srlz.d                                  // ensure psr.i is set again
498         mov r18=0                                       // i must not leak kernel bits...
499
500 .store_mask:
501 EX(.fail_efault, (p15) probe.w.fault r34, 3)    // verify user has write-access to *oset
502 EX(.fail_efault, (p15) st8 [r34]=r3)
503         mov r2=0                                        // i must not leak kernel bits...
504         mov r3=0                                        // i must not leak kernel bits...
505         mov r8=0                                // return 0
506         mov r9=0                                        // i must not leak kernel bits...
507         mov r14=0                                       // i must not leak kernel bits...
508         mov r17=0                                       // i must not leak kernel bits...
509         mov r31=0                                       // i must not leak kernel bits...
510         FSYS_RETURN
511
512 .sig_pending:
513 #ifdef CONFIG_SMP
514         st4.rel [r31]=r0                        // release the lock
515 #endif
516         ssm psr.i
517         ;;
518         srlz.d
519         br.sptk.many fsys_fallback_syscall      // with signal pending, do the heavy-weight syscall
520
521 #ifdef CONFIG_SMP
522 .lock_contention:
523         /* Rather than spinning here, fall back on doing a heavy-weight syscall.  */
524         ssm psr.i
525         ;;
526         srlz.d
527         br.sptk.many fsys_fallback_syscall
528 #endif
529 END(fsys_rt_sigprocmask)
530
531 /*
532  * fsys_getcpu doesn't use the third parameter in this implementation. It reads
533  * current_thread_info()->cpu and corresponding node in cpu_to_node_map.
534  */
535 ENTRY(fsys_getcpu)
536         .prologue
537         .altrp b6
538         .body
539         ;;
540         add r2=TI_FLAGS+IA64_TASK_SIZE,r16
541         tnat.nz p6,p0 = r32                     // guard against NaT argument
542         add r3=TI_CPU+IA64_TASK_SIZE,r16
543         ;;
544         ld4 r3=[r3]                             // M r3 = thread_info->cpu
545         ld4 r2=[r2]                             // M r2 = thread_info->flags
546 (p6)    br.cond.spnt.few .fail_einval           // B
547         ;;
548         tnat.nz p7,p0 = r33                     // I guard against NaT argument
549 (p7)    br.cond.spnt.few .fail_einval           // B
550 #ifdef CONFIG_NUMA
551         movl r17=cpu_to_node_map
552         ;;
553 EX(.fail_efault, probe.w.fault r32, 3)          // M This takes 5 cycles
554 EX(.fail_efault, probe.w.fault r33, 3)          // M This takes 5 cycles
555         shladd r18=r3,1,r17
556         ;;
557         ld2 r20=[r18]                           // r20 = cpu_to_node_map[cpu]
558         and r2 = TIF_ALLWORK_MASK,r2
559         ;;
560         cmp.ne p8,p0=0,r2
561 (p8)    br.spnt.many fsys_fallback_syscall
562         ;;
563         ;;
564 EX(.fail_efault, st4 [r32] = r3)
565 EX(.fail_efault, st2 [r33] = r20)
566         mov r8=0
567         ;;
568 #else
569 EX(.fail_efault, probe.w.fault r32, 3)          // M This takes 5 cycles
570 EX(.fail_efault, probe.w.fault r33, 3)          // M This takes 5 cycles
571         and r2 = TIF_ALLWORK_MASK,r2
572         ;;
573         cmp.ne p8,p0=0,r2
574 (p8)    br.spnt.many fsys_fallback_syscall
575         ;;
576 EX(.fail_efault, st4 [r32] = r3)
577 EX(.fail_efault, st2 [r33] = r0)
578         mov r8=0
579         ;;
580 #endif
581         FSYS_RETURN
582 END(fsys_getcpu)
583
584 ENTRY(fsys_fallback_syscall)
585         .prologue
586         .altrp b6
587         .body
588         /*
589          * We only get here from light-weight syscall handlers.  Thus, we already
590          * know that r15 contains a valid syscall number.  No need to re-check.
591          */
592         adds r17=-1024,r15
593         movl r14=sys_call_table
594         ;;
595         rsm psr.i
596         shladd r18=r17,3,r14
597         ;;
598         ld8 r18=[r18]                           // load normal (heavy-weight) syscall entry-point
599         mov r29=psr                             // read psr (12 cyc load latency)
600         mov r27=ar.rsc
601         mov r21=ar.fpsr
602         mov r26=ar.pfs
603 END(fsys_fallback_syscall)
604         /* FALL THROUGH */
605 GLOBAL_ENTRY(fsys_bubble_down)
606         .prologue
607         .altrp b6
608         .body
609         /*
610          * We get here for syscalls that don't have a lightweight
611          * handler.  For those, we need to bubble down into the kernel
612          * and that requires setting up a minimal pt_regs structure,
613          * and initializing the CPU state more or less as if an
614          * interruption had occurred.  To make syscall-restarts work,
615          * we setup pt_regs such that cr_iip points to the second
616          * instruction in syscall_via_break.  Decrementing the IP
617          * hence will restart the syscall via break and not
618          * decrementing IP will return us to the caller, as usual.
619          * Note that we preserve the value of psr.pp rather than
620          * initializing it from dcr.pp.  This makes it possible to
621          * distinguish fsyscall execution from other privileged
622          * execution.
623          *
624          * On entry:
625          *      - normal fsyscall handler register usage, except
626          *        that we also have:
627          *      - r18: address of syscall entry point
628          *      - r21: ar.fpsr
629          *      - r26: ar.pfs
630          *      - r27: ar.rsc
631          *      - r29: psr
632          *
633          * We used to clear some PSR bits here but that requires slow
634          * serialization.  Fortuntely, that isn't really necessary.
635          * The rationale is as follows: we used to clear bits
636          * ~PSR_PRESERVED_BITS in PSR.L.  Since
637          * PSR_PRESERVED_BITS==PSR.{UP,MFL,MFH,PK,DT,PP,SP,RT,IC}, we
638          * ended up clearing PSR.{BE,AC,I,DFL,DFH,DI,DB,SI,TB}.
639          * However,
640          *
641          * PSR.BE : already is turned off in __kernel_syscall_via_epc()
642          * PSR.AC : don't care (kernel normally turns PSR.AC on)
643          * PSR.I  : already turned off by the time fsys_bubble_down gets
644          *          invoked
645          * PSR.DFL: always 0 (kernel never turns it on)
646          * PSR.DFH: don't care --- kernel never touches f32-f127 on its own
647          *          initiative
648          * PSR.DI : always 0 (kernel never turns it on)
649          * PSR.SI : always 0 (kernel never turns it on)
650          * PSR.DB : don't care --- kernel never enables kernel-level
651          *          breakpoints
652          * PSR.TB : must be 0 already; if it wasn't zero on entry to
653          *          __kernel_syscall_via_epc, the branch to fsys_bubble_down
654          *          will trigger a taken branch; the taken-trap-handler then
655          *          converts the syscall into a break-based system-call.
656          */
657         /*
658          * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.
659          * The rest we have to synthesize.
660          */
661 #       define PSR_ONE_BITS             ((3 << IA64_PSR_CPL0_BIT)       \
662                                          | (0x1 << IA64_PSR_RI_BIT)     \
663                                          | IA64_PSR_BN | IA64_PSR_I)
664
665         invala                                  // M0|1
666         movl r14=ia64_ret_from_syscall          // X
667
668         nop.m 0
669         movl r28=__kernel_syscall_via_break     // X    create cr.iip
670         ;;
671
672         mov r2=r16                              // A    get task addr to addl-addressable register
673         adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // A
674         mov r31=pr                              // I0   save pr (2 cyc)
675         ;;
676         st1 [r16]=r0                            // M2|3 clear current->thread.on_ustack flag
677         addl r22=IA64_RBS_OFFSET,r2             // A    compute base of RBS
678         add r3=TI_FLAGS+IA64_TASK_SIZE,r2       // A
679         ;;
680         ld4 r3=[r3]                             // M0|1 r3 = current_thread_info()->flags
681         lfetch.fault.excl.nt1 [r22]             // M0|1 prefetch register backing-store
682         nop.i 0
683         ;;
684         mov ar.rsc=0                            // M2   set enforced lazy mode, pl 0, LE, loadrs=0
685 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
686         mov.m r30=ar.itc                        // M    get cycle for accounting
687 #else
688         nop.m 0
689 #endif
690         nop.i 0
691         ;;
692         mov r23=ar.bspstore                     // M2 (12 cyc) save ar.bspstore
693         mov.m r24=ar.rnat                       // M2 (5 cyc) read ar.rnat (dual-issues!)
694         nop.i 0
695         ;;
696         mov ar.bspstore=r22                     // M2 (6 cyc) switch to kernel RBS
697         movl r8=PSR_ONE_BITS                    // X
698         ;;
699         mov r25=ar.unat                         // M2 (5 cyc) save ar.unat
700         mov r19=b6                              // I0   save b6 (2 cyc)
701         mov r20=r1                              // A    save caller's gp in r20
702         ;;
703         or r29=r8,r29                           // A    construct cr.ipsr value to save
704         mov b6=r18                              // I0   copy syscall entry-point to b6 (7 cyc)
705         addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2 // A compute base of memory stack
706
707         mov r18=ar.bsp                          // M2   save (kernel) ar.bsp (12 cyc)
708         cmp.ne pKStk,pUStk=r0,r0                // A    set pKStk <- 0, pUStk <- 1
709         br.call.sptk.many b7=ia64_syscall_setup // B
710         ;;
711 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
712         // mov.m r30=ar.itc is called in advance
713         add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2
714         add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2
715         ;;
716         ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP   // time at last check in kernel
717         ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE   // time at leave kernel
718         ;;
719         ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME   // cumulated stime
720         ld8 r21=[r17]                           // cumulated utime
721         sub r22=r19,r18                         // stime before leave kernel
722         ;;
723         st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP   // update stamp
724         sub r18=r30,r19                         // elapsed time in user mode
725         ;;
726         add r20=r20,r22                         // sum stime
727         add r21=r21,r18                         // sum utime
728         ;;
729         st8 [r16]=r20                           // update stime
730         st8 [r17]=r21                           // update utime
731         ;;
732 #endif
733         mov ar.rsc=0x3                          // M2   set eager mode, pl 0, LE, loadrs=0
734         mov rp=r14                              // I0   set the real return addr
735         and r3=_TIF_SYSCALL_TRACEAUDIT,r3       // A
736         ;;
737         ssm psr.i                               // M2   we're on kernel stacks now, reenable irqs
738         cmp.eq p8,p0=r3,r0                      // A
739 (p10)   br.cond.spnt.many ia64_ret_from_syscall // B    return if bad call-frame or r15 is a NaT
740
741         nop.m 0
742 (p8)    br.call.sptk.many b6=b6                 // B    (ignore return address)
743         br.cond.spnt ia64_trace_syscall         // B
744 END(fsys_bubble_down)
745
746         .rodata
747         .align 8
748         .globl fsyscall_table
749
750         data8 fsys_bubble_down
751 fsyscall_table:
752         data8 fsys_ni_syscall
753         data8 0                         // exit                 // 1025
754         data8 0                         // read
755         data8 0                         // write
756         data8 0                         // open
757         data8 0                         // close
758         data8 0                         // creat                // 1030
759         data8 0                         // link
760         data8 0                         // unlink
761         data8 0                         // execve
762         data8 0                         // chdir
763         data8 0                         // fchdir               // 1035
764         data8 0                         // utimes
765         data8 0                         // mknod
766         data8 0                         // chmod
767         data8 0                         // chown
768         data8 0                         // lseek                // 1040
769         data8 fsys_getpid               // getpid
770         data8 fsys_getppid              // getppid
771         data8 0                         // mount
772         data8 0                         // umount
773         data8 0                         // setuid               // 1045
774         data8 0                         // getuid
775         data8 0                         // geteuid
776         data8 0                         // ptrace
777         data8 0                         // access
778         data8 0                         // sync                 // 1050
779         data8 0                         // fsync
780         data8 0                         // fdatasync
781         data8 0                         // kill
782         data8 0                         // rename
783         data8 0                         // mkdir                // 1055
784         data8 0                         // rmdir
785         data8 0                         // dup
786         data8 0                         // pipe
787         data8 0                         // times
788         data8 0                         // brk                  // 1060
789         data8 0                         // setgid
790         data8 0                         // getgid
791         data8 0                         // getegid
792         data8 0                         // acct
793         data8 0                         // ioctl                // 1065
794         data8 0                         // fcntl
795         data8 0                         // umask
796         data8 0                         // chroot
797         data8 0                         // ustat
798         data8 0                         // dup2                 // 1070
799         data8 0                         // setreuid
800         data8 0                         // setregid
801         data8 0                         // getresuid
802         data8 0                         // setresuid
803         data8 0                         // getresgid            // 1075
804         data8 0                         // setresgid
805         data8 0                         // getgroups
806         data8 0                         // setgroups
807         data8 0                         // getpgid
808         data8 0                         // setpgid              // 1080
809         data8 0                         // setsid
810         data8 0                         // getsid
811         data8 0                         // sethostname
812         data8 0                         // setrlimit
813         data8 0                         // getrlimit            // 1085
814         data8 0                         // getrusage
815         data8 fsys_gettimeofday         // gettimeofday
816         data8 0                         // settimeofday
817         data8 0                         // select
818         data8 0                         // poll                 // 1090
819         data8 0                         // symlink
820         data8 0                         // readlink
821         data8 0                         // uselib
822         data8 0                         // swapon
823         data8 0                         // swapoff              // 1095
824         data8 0                         // reboot
825         data8 0                         // truncate
826         data8 0                         // ftruncate
827         data8 0                         // fchmod
828         data8 0                         // fchown               // 1100
829         data8 0                         // getpriority
830         data8 0                         // setpriority
831         data8 0                         // statfs
832         data8 0                         // fstatfs
833         data8 0                         // gettid               // 1105
834         data8 0                         // semget
835         data8 0                         // semop
836         data8 0                         // semctl
837         data8 0                         // msgget
838         data8 0                         // msgsnd               // 1110
839         data8 0                         // msgrcv
840         data8 0                         // msgctl
841         data8 0                         // shmget
842         data8 0                         // shmat
843         data8 0                         // shmdt                // 1115
844         data8 0                         // shmctl
845         data8 0                         // syslog
846         data8 0                         // setitimer
847         data8 0                         // getitimer
848         data8 0                                                 // 1120
849         data8 0
850         data8 0
851         data8 0                         // vhangup
852         data8 0                         // lchown
853         data8 0                         // remap_file_pages     // 1125
854         data8 0                         // wait4
855         data8 0                         // sysinfo
856         data8 0                         // clone
857         data8 0                         // setdomainname
858         data8 0                         // newuname             // 1130
859         data8 0                         // adjtimex
860         data8 0
861         data8 0                         // init_module
862         data8 0                         // delete_module
863         data8 0                                                 // 1135
864         data8 0
865         data8 0                         // quotactl
866         data8 0                         // bdflush
867         data8 0                         // sysfs
868         data8 0                         // personality          // 1140
869         data8 0                         // afs_syscall
870         data8 0                         // setfsuid
871         data8 0                         // setfsgid
872         data8 0                         // getdents
873         data8 0                         // flock                // 1145
874         data8 0                         // readv
875         data8 0                         // writev
876         data8 0                         // pread64
877         data8 0                         // pwrite64
878         data8 0                         // sysctl               // 1150
879         data8 0                         // mmap
880         data8 0                         // munmap
881         data8 0                         // mlock
882         data8 0                         // mlockall
883         data8 0                         // mprotect             // 1155
884         data8 0                         // mremap
885         data8 0                         // msync
886         data8 0                         // munlock
887         data8 0                         // munlockall
888         data8 0                         // sched_getparam       // 1160
889         data8 0                         // sched_setparam
890         data8 0                         // sched_getscheduler
891         data8 0                         // sched_setscheduler
892         data8 0                         // sched_yield
893         data8 0                         // sched_get_priority_max       // 1165
894         data8 0                         // sched_get_priority_min
895         data8 0                         // sched_rr_get_interval
896         data8 0                         // nanosleep
897         data8 0                         // nfsservctl
898         data8 0                         // prctl                // 1170
899         data8 0                         // getpagesize
900         data8 0                         // mmap2
901         data8 0                         // pciconfig_read
902         data8 0                         // pciconfig_write
903         data8 0                         // perfmonctl           // 1175
904         data8 0                         // sigaltstack
905         data8 0                         // rt_sigaction
906         data8 0                         // rt_sigpending
907         data8 fsys_rt_sigprocmask       // rt_sigprocmask
908         data8 0                         // rt_sigqueueinfo      // 1180
909         data8 0                         // rt_sigreturn
910         data8 0                         // rt_sigsuspend
911         data8 0                         // rt_sigtimedwait
912         data8 0                         // getcwd
913         data8 0                         // capget               // 1185
914         data8 0                         // capset
915         data8 0                         // sendfile
916         data8 0
917         data8 0
918         data8 0                         // socket               // 1190
919         data8 0                         // bind
920         data8 0                         // connect
921         data8 0                         // listen
922         data8 0                         // accept
923         data8 0                         // getsockname          // 1195
924         data8 0                         // getpeername
925         data8 0                         // socketpair
926         data8 0                         // send
927         data8 0                         // sendto
928         data8 0                         // recv                 // 1200
929         data8 0                         // recvfrom
930         data8 0                         // shutdown
931         data8 0                         // setsockopt
932         data8 0                         // getsockopt
933         data8 0                         // sendmsg              // 1205
934         data8 0                         // recvmsg
935         data8 0                         // pivot_root
936         data8 0                         // mincore
937         data8 0                         // madvise
938         data8 0                         // newstat              // 1210
939         data8 0                         // newlstat
940         data8 0                         // newfstat
941         data8 0                         // clone2
942         data8 0                         // getdents64
943         data8 0                         // getunwind            // 1215
944         data8 0                         // readahead
945         data8 0                         // setxattr
946         data8 0                         // lsetxattr
947         data8 0                         // fsetxattr
948         data8 0                         // getxattr             // 1220
949         data8 0                         // lgetxattr
950         data8 0                         // fgetxattr
951         data8 0                         // listxattr
952         data8 0                         // llistxattr
953         data8 0                         // flistxattr           // 1225
954         data8 0                         // removexattr
955         data8 0                         // lremovexattr
956         data8 0                         // fremovexattr
957         data8 0                         // tkill
958         data8 0                         // futex                // 1230
959         data8 0                         // sched_setaffinity
960         data8 0                         // sched_getaffinity
961         data8 fsys_set_tid_address      // set_tid_address
962         data8 0                         // fadvise64_64
963         data8 0                         // tgkill               // 1235
964         data8 0                         // exit_group
965         data8 0                         // lookup_dcookie
966         data8 0                         // io_setup
967         data8 0                         // io_destroy
968         data8 0                         // io_getevents         // 1240
969         data8 0                         // io_submit
970         data8 0                         // io_cancel
971         data8 0                         // epoll_create
972         data8 0                         // epoll_ctl
973         data8 0                         // epoll_wait           // 1245
974         data8 0                         // restart_syscall
975         data8 0                         // semtimedop
976         data8 0                         // timer_create
977         data8 0                         // timer_settime
978         data8 0                         // timer_gettime        // 1250
979         data8 0                         // timer_getoverrun
980         data8 0                         // timer_delete
981         data8 0                         // clock_settime
982         data8 fsys_clock_gettime        // clock_gettime
983         data8 0                         // clock_getres         // 1255
984         data8 0                         // clock_nanosleep
985         data8 0                         // fstatfs64
986         data8 0                         // statfs64
987         data8 0                         // mbind
988         data8 0                         // get_mempolicy        // 1260
989         data8 0                         // set_mempolicy
990         data8 0                         // mq_open
991         data8 0                         // mq_unlink
992         data8 0                         // mq_timedsend
993         data8 0                         // mq_timedreceive      // 1265
994         data8 0                         // mq_notify
995         data8 0                         // mq_getsetattr
996         data8 0                         // kexec_load
997         data8 0                         // vserver
998         data8 0                         // waitid               // 1270
999         data8 0                         // add_key
1000         data8 0                         // request_key
1001         data8 0                         // keyctl
1002         data8 0                         // ioprio_set
1003         data8 0                         // ioprio_get           // 1275
1004         data8 0                         // move_pages
1005         data8 0                         // inotify_init
1006         data8 0                         // inotify_add_watch
1007         data8 0                         // inotify_rm_watch
1008         data8 0                         // migrate_pages        // 1280
1009         data8 0                         // openat
1010         data8 0                         // mkdirat
1011         data8 0                         // mknodat
1012         data8 0                         // fchownat
1013         data8 0                         // futimesat            // 1285
1014         data8 0                         // newfstatat
1015         data8 0                         // unlinkat
1016         data8 0                         // renameat
1017         data8 0                         // linkat
1018         data8 0                         // symlinkat            // 1290
1019         data8 0                         // readlinkat
1020         data8 0                         // fchmodat
1021         data8 0                         // faccessat
1022         data8 0
1023         data8 0                                                 // 1295
1024         data8 0                         // unshare
1025         data8 0                         // splice
1026         data8 0                         // set_robust_list
1027         data8 0                         // get_robust_list
1028         data8 0                         // sync_file_range      // 1300
1029         data8 0                         // tee
1030         data8 0                         // vmsplice
1031         data8 0
1032         data8 fsys_getcpu               // getcpu               // 1304
1033
1034         // fill in zeros for the remaining entries
1035         .zero:
1036         .space fsyscall_table + 8*NR_syscalls - .zero, 0