Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Architecture-specific unaligned trap handling.
4 *
5 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
6 * Stephane Eranian <eranian@hpl.hp.com>
7 * David Mosberger-Tang <davidm@hpl.hp.com>
8 *
9 * 2002/12/09 Fix rotating register handling (off-by-1 error, missing fr-rotation). Fix
10 * get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
11 * stacked register returns an undefined value; it does NOT trigger a
12 * "rsvd register fault").
13 * 2001/10/11 Fix unaligned access to rotating registers in s/w pipelined loops.
14 * 2001/08/13 Correct size of extended floats (float_fsz) from 16 to 10 bytes.
15 * 2001/01/17 Add support emulation of unaligned kernel accesses.
16 */
17#include <linux/jiffies.h>
18#include <linux/kernel.h>
19#include <linux/sched/signal.h>
20#include <linux/tty.h>
21#include <linux/extable.h>
22#include <linux/ratelimit.h>
23#include <linux/uaccess.h>
24
25#include <asm/intrinsics.h>
26#include <asm/processor.h>
27#include <asm/rse.h>
28#include <asm/exception.h>
29#include <asm/unaligned.h>
30
31extern int die_if_kernel(char *str, struct pt_regs *regs, long err);
32
33#undef DEBUG_UNALIGNED_TRAP
34
35#ifdef DEBUG_UNALIGNED_TRAP
36# define DPRINT(a...) do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0)
37# define DDUMP(str,vp,len) dump(str, vp, len)
38
39static void
40dump (const char *str, void *vp, size_t len)
41{
42 unsigned char *cp = vp;
43 int i;
44
45 printk("%s", str);
46 for (i = 0; i < len; ++i)
47 printk (" %02x", *cp++);
48 printk("\n");
49}
50#else
51# define DPRINT(a...)
52# define DDUMP(str,vp,len)
53#endif
54
55#define IA64_FIRST_STACKED_GR 32
56#define IA64_FIRST_ROTATING_FR 32
57#define SIGN_EXT9 0xffffffffffffff00ul
58
59/*
60 * sysctl settable hook which tells the kernel whether to honor the
61 * IA64_THREAD_UAC_NOPRINT prctl. Because this is user settable, we want
62 * to allow the super user to enable/disable this for security reasons
63 * (i.e. don't allow attacker to fill up logs with unaligned accesses).
64 */
65int no_unaligned_warning;
66int unaligned_dump_stack;
67
68/*
69 * For M-unit:
70 *
71 * opcode | m | x6 |
72 * --------|------|---------|
73 * [40-37] | [36] | [35:30] |
74 * --------|------|---------|
75 * 4 | 1 | 6 | = 11 bits
76 * --------------------------
77 * However bits [31:30] are not directly useful to distinguish between
78 * load/store so we can use [35:32] instead, which gives the following
79 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
80 * checking the m-bit until later in the load/store emulation.
81 */
82#define IA64_OPCODE_MASK 0x1ef
83#define IA64_OPCODE_SHIFT 32
84
85/*
86 * Table C-28 Integer Load/Store
87 *
88 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
89 *
90 * ld8.fill, st8.fill MUST be aligned because the RNATs are based on
91 * the address (bits [8:3]), so we must failed.
92 */
93#define LD_OP 0x080
94#define LDS_OP 0x081
95#define LDA_OP 0x082
96#define LDSA_OP 0x083
97#define LDBIAS_OP 0x084
98#define LDACQ_OP 0x085
99/* 0x086, 0x087 are not relevant */
100#define LDCCLR_OP 0x088
101#define LDCNC_OP 0x089
102#define LDCCLRACQ_OP 0x08a
103#define ST_OP 0x08c
104#define STREL_OP 0x08d
105/* 0x08e,0x8f are not relevant */
106
107/*
108 * Table C-29 Integer Load +Reg
109 *
110 * we use the ld->m (bit [36:36]) field to determine whether or not we have
111 * a load/store of this form.
112 */
113
114/*
115 * Table C-30 Integer Load/Store +Imm
116 *
117 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
118 *
119 * ld8.fill, st8.fill must be aligned because the Nat register are based on
120 * the address, so we must fail and the program must be fixed.
121 */
122#define LD_IMM_OP 0x0a0
123#define LDS_IMM_OP 0x0a1
124#define LDA_IMM_OP 0x0a2
125#define LDSA_IMM_OP 0x0a3
126#define LDBIAS_IMM_OP 0x0a4
127#define LDACQ_IMM_OP 0x0a5
128/* 0x0a6, 0xa7 are not relevant */
129#define LDCCLR_IMM_OP 0x0a8
130#define LDCNC_IMM_OP 0x0a9
131#define LDCCLRACQ_IMM_OP 0x0aa
132#define ST_IMM_OP 0x0ac
133#define STREL_IMM_OP 0x0ad
134/* 0x0ae,0xaf are not relevant */
135
136/*
137 * Table C-32 Floating-point Load/Store
138 */
139#define LDF_OP 0x0c0
140#define LDFS_OP 0x0c1
141#define LDFA_OP 0x0c2
142#define LDFSA_OP 0x0c3
143/* 0x0c6 is irrelevant */
144#define LDFCCLR_OP 0x0c8
145#define LDFCNC_OP 0x0c9
146/* 0x0cb is irrelevant */
147#define STF_OP 0x0cc
148
149/*
150 * Table C-33 Floating-point Load +Reg
151 *
152 * we use the ld->m (bit [36:36]) field to determine whether or not we have
153 * a load/store of this form.
154 */
155
156/*
157 * Table C-34 Floating-point Load/Store +Imm
158 */
159#define LDF_IMM_OP 0x0e0
160#define LDFS_IMM_OP 0x0e1
161#define LDFA_IMM_OP 0x0e2
162#define LDFSA_IMM_OP 0x0e3
163/* 0x0e6 is irrelevant */
164#define LDFCCLR_IMM_OP 0x0e8
165#define LDFCNC_IMM_OP 0x0e9
166#define STF_IMM_OP 0x0ec
167
168typedef struct {
169 unsigned long qp:6; /* [0:5] */
170 unsigned long r1:7; /* [6:12] */
171 unsigned long imm:7; /* [13:19] */
172 unsigned long r3:7; /* [20:26] */
173 unsigned long x:1; /* [27:27] */
174 unsigned long hint:2; /* [28:29] */
175 unsigned long x6_sz:2; /* [30:31] */
176 unsigned long x6_op:4; /* [32:35], x6 = x6_sz|x6_op */
177 unsigned long m:1; /* [36:36] */
178 unsigned long op:4; /* [37:40] */
179 unsigned long pad:23; /* [41:63] */
180} load_store_t;
181
182
183typedef enum {
184 UPD_IMMEDIATE, /* ldXZ r1=[r3],imm(9) */
185 UPD_REG /* ldXZ r1=[r3],r2 */
186} update_t;
187
188/*
189 * We use tables to keep track of the offsets of registers in the saved state.
190 * This way we save having big switch/case statements.
191 *
192 * We use bit 0 to indicate switch_stack or pt_regs.
193 * The offset is simply shifted by 1 bit.
194 * A 2-byte value should be enough to hold any kind of offset
195 *
196 * In case the calling convention changes (and thus pt_regs/switch_stack)
197 * simply use RSW instead of RPT or vice-versa.
198 */
199
200#define RPO(x) ((size_t) &((struct pt_regs *)0)->x)
201#define RSO(x) ((size_t) &((struct switch_stack *)0)->x)
202
203#define RPT(x) (RPO(x) << 1)
204#define RSW(x) (1| RSO(x)<<1)
205
206#define GR_OFFS(x) (gr_info[x]>>1)
207#define GR_IN_SW(x) (gr_info[x] & 0x1)
208
209#define FR_OFFS(x) (fr_info[x]>>1)
210#define FR_IN_SW(x) (fr_info[x] & 0x1)
211
212static u16 gr_info[32]={
213 0, /* r0 is read-only : WE SHOULD NEVER GET THIS */
214
215 RPT(r1), RPT(r2), RPT(r3),
216
217 RSW(r4), RSW(r5), RSW(r6), RSW(r7),
218
219 RPT(r8), RPT(r9), RPT(r10), RPT(r11),
220 RPT(r12), RPT(r13), RPT(r14), RPT(r15),
221
222 RPT(r16), RPT(r17), RPT(r18), RPT(r19),
223 RPT(r20), RPT(r21), RPT(r22), RPT(r23),
224 RPT(r24), RPT(r25), RPT(r26), RPT(r27),
225 RPT(r28), RPT(r29), RPT(r30), RPT(r31)
226};
227
228static u16 fr_info[32]={
229 0, /* constant : WE SHOULD NEVER GET THIS */
230 0, /* constant : WE SHOULD NEVER GET THIS */
231
232 RSW(f2), RSW(f3), RSW(f4), RSW(f5),
233
234 RPT(f6), RPT(f7), RPT(f8), RPT(f9),
235 RPT(f10), RPT(f11),
236
237 RSW(f12), RSW(f13), RSW(f14),
238 RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19),
239 RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24),
240 RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29),
241 RSW(f30), RSW(f31)
242};
243
244/* Invalidate ALAT entry for integer register REGNO. */
245static void
246invala_gr (int regno)
247{
248# define F(reg) case reg: ia64_invala_gr(reg); break
249
250 switch (regno) {
251 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7);
252 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
253 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
254 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
255 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
256 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
257 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
258 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
259 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
260 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
261 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
262 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
263 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
264 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
265 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
266 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
267 }
268# undef F
269}
270
271/* Invalidate ALAT entry for floating-point register REGNO. */
272static void
273invala_fr (int regno)
274{
275# define F(reg) case reg: ia64_invala_fr(reg); break
276
277 switch (regno) {
278 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7);
279 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
280 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
281 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
282 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
283 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
284 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
285 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
286 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
287 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
288 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
289 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
290 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
291 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
292 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
293 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
294 }
295# undef F
296}
297
298static inline unsigned long
299rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg)
300{
301 reg += rrb;
302 if (reg >= sor)
303 reg -= sor;
304 return reg;
305}
306
307static void
308set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat)
309{
310 struct switch_stack *sw = (struct switch_stack *) regs - 1;
311 unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end;
312 unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
313 unsigned long rnats, nat_mask;
314 unsigned long on_kbs;
315 long sof = (regs->cr_ifs) & 0x7f;
316 long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
317 long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
318 long ridx = r1 - 32;
319
320 if (ridx >= sof) {
321 /* this should never happen, as the "rsvd register fault" has higher priority */
322 DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof);
323 return;
324 }
325
326 if (ridx < sor)
327 ridx = rotate_reg(sor, rrb_gr, ridx);
328
329 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
330 r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
331
332 on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
333 addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
334 if (addr >= kbs) {
335 /* the register is on the kernel backing store: easy... */
336 rnat_addr = ia64_rse_rnat_addr(addr);
337 if ((unsigned long) rnat_addr >= sw->ar_bspstore)
338 rnat_addr = &sw->ar_rnat;
339 nat_mask = 1UL << ia64_rse_slot_num(addr);
340
341 *addr = val;
342 if (nat)
343 *rnat_addr |= nat_mask;
344 else
345 *rnat_addr &= ~nat_mask;
346 return;
347 }
348
349 if (!user_stack(current, regs)) {
350 DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1);
351 return;
352 }
353
354 bspstore = (unsigned long *)regs->ar_bspstore;
355 ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
356 bsp = ia64_rse_skip_regs(ubs_end, -sof);
357 addr = ia64_rse_skip_regs(bsp, ridx);
358
359 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
360
361 ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
362
363 rnat_addr = ia64_rse_rnat_addr(addr);
364
365 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
366 DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n",
367 (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1);
368
369 nat_mask = 1UL << ia64_rse_slot_num(addr);
370 if (nat)
371 rnats |= nat_mask;
372 else
373 rnats &= ~nat_mask;
374 ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats);
375
376 DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats);
377}
378
379
380static void
381get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat)
382{
383 struct switch_stack *sw = (struct switch_stack *) regs - 1;
384 unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore;
385 unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
386 unsigned long rnats, nat_mask;
387 unsigned long on_kbs;
388 long sof = (regs->cr_ifs) & 0x7f;
389 long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
390 long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
391 long ridx = r1 - 32;
392
393 if (ridx >= sof) {
394 /* read of out-of-frame register returns an undefined value; 0 in our case. */
395 DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof);
396 goto fail;
397 }
398
399 if (ridx < sor)
400 ridx = rotate_reg(sor, rrb_gr, ridx);
401
402 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
403 r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
404
405 on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
406 addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
407 if (addr >= kbs) {
408 /* the register is on the kernel backing store: easy... */
409 *val = *addr;
410 if (nat) {
411 rnat_addr = ia64_rse_rnat_addr(addr);
412 if ((unsigned long) rnat_addr >= sw->ar_bspstore)
413 rnat_addr = &sw->ar_rnat;
414 nat_mask = 1UL << ia64_rse_slot_num(addr);
415 *nat = (*rnat_addr & nat_mask) != 0;
416 }
417 return;
418 }
419
420 if (!user_stack(current, regs)) {
421 DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1);
422 goto fail;
423 }
424
425 bspstore = (unsigned long *)regs->ar_bspstore;
426 ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
427 bsp = ia64_rse_skip_regs(ubs_end, -sof);
428 addr = ia64_rse_skip_regs(bsp, ridx);
429
430 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
431
432 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
433
434 if (nat) {
435 rnat_addr = ia64_rse_rnat_addr(addr);
436 nat_mask = 1UL << ia64_rse_slot_num(addr);
437
438 DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats);
439
440 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
441 *nat = (rnats & nat_mask) != 0;
442 }
443 return;
444
445 fail:
446 *val = 0;
447 if (nat)
448 *nat = 0;
449 return;
450}
451
452
453static void
454setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs)
455{
456 struct switch_stack *sw = (struct switch_stack *) regs - 1;
457 unsigned long addr;
458 unsigned long bitmask;
459 unsigned long *unat;
460
461 /*
462 * First takes care of stacked registers
463 */
464 if (regnum >= IA64_FIRST_STACKED_GR) {
465 set_rse_reg(regs, regnum, val, nat);
466 return;
467 }
468
469 /*
470 * Using r0 as a target raises a General Exception fault which has higher priority
471 * than the Unaligned Reference fault.
472 */
473
474 /*
475 * Now look at registers in [0-31] range and init correct UNAT
476 */
477 if (GR_IN_SW(regnum)) {
478 addr = (unsigned long)sw;
479 unat = &sw->ar_unat;
480 } else {
481 addr = (unsigned long)regs;
482 unat = &sw->caller_unat;
483 }
484 DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n",
485 addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum));
486 /*
487 * add offset from base of struct
488 * and do it !
489 */
490 addr += GR_OFFS(regnum);
491
492 *(unsigned long *)addr = val;
493
494 /*
495 * We need to clear the corresponding UNAT bit to fully emulate the load
496 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
497 */
498 bitmask = 1UL << (addr >> 3 & 0x3f);
499 DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat);
500 if (nat) {
501 *unat |= bitmask;
502 } else {
503 *unat &= ~bitmask;
504 }
505 DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat);
506}
507
508/*
509 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
510 * range from 32-127, result is in the range from 0-95.
511 */
512static inline unsigned long
513fph_index (struct pt_regs *regs, long regnum)
514{
515 unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;
516 return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));
517}
518
519static void
520setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
521{
522 struct switch_stack *sw = (struct switch_stack *)regs - 1;
523 unsigned long addr;
524
525 /*
526 * From EAS-2.5: FPDisableFault has higher priority than Unaligned
527 * Fault. Thus, when we get here, we know the partition is enabled.
528 * To update f32-f127, there are three choices:
529 *
530 * (1) save f32-f127 to thread.fph and update the values there
531 * (2) use a gigantic switch statement to directly access the registers
532 * (3) generate code on the fly to update the desired register
533 *
534 * For now, we are using approach (1).
535 */
536 if (regnum >= IA64_FIRST_ROTATING_FR) {
537 ia64_sync_fph(current);
538 current->thread.fph[fph_index(regs, regnum)] = *fpval;
539 } else {
540 /*
541 * pt_regs or switch_stack ?
542 */
543 if (FR_IN_SW(regnum)) {
544 addr = (unsigned long)sw;
545 } else {
546 addr = (unsigned long)regs;
547 }
548
549 DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum));
550
551 addr += FR_OFFS(regnum);
552 *(struct ia64_fpreg *)addr = *fpval;
553
554 /*
555 * mark the low partition as being used now
556 *
557 * It is highly unlikely that this bit is not already set, but
558 * let's do it for safety.
559 */
560 regs->cr_ipsr |= IA64_PSR_MFL;
561 }
562}
563
564/*
565 * Those 2 inline functions generate the spilled versions of the constant floating point
566 * registers which can be used with stfX
567 */
568static inline void
569float_spill_f0 (struct ia64_fpreg *final)
570{
571 ia64_stf_spill(final, 0);
572}
573
574static inline void
575float_spill_f1 (struct ia64_fpreg *final)
576{
577 ia64_stf_spill(final, 1);
578}
579
580static void
581getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
582{
583 struct switch_stack *sw = (struct switch_stack *) regs - 1;
584 unsigned long addr;
585
586 /*
587 * From EAS-2.5: FPDisableFault has higher priority than
588 * Unaligned Fault. Thus, when we get here, we know the partition is
589 * enabled.
590 *
591 * When regnum > 31, the register is still live and we need to force a save
592 * to current->thread.fph to get access to it. See discussion in setfpreg()
593 * for reasons and other ways of doing this.
594 */
595 if (regnum >= IA64_FIRST_ROTATING_FR) {
596 ia64_flush_fph(current);
597 *fpval = current->thread.fph[fph_index(regs, regnum)];
598 } else {
599 /*
600 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
601 * not saved, we must generate their spilled form on the fly
602 */
603 switch(regnum) {
604 case 0:
605 float_spill_f0(fpval);
606 break;
607 case 1:
608 float_spill_f1(fpval);
609 break;
610 default:
611 /*
612 * pt_regs or switch_stack ?
613 */
614 addr = FR_IN_SW(regnum) ? (unsigned long)sw
615 : (unsigned long)regs;
616
617 DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n",
618 FR_IN_SW(regnum), addr, FR_OFFS(regnum));
619
620 addr += FR_OFFS(regnum);
621 *fpval = *(struct ia64_fpreg *)addr;
622 }
623 }
624}
625
626
627static void
628getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs)
629{
630 struct switch_stack *sw = (struct switch_stack *) regs - 1;
631 unsigned long addr, *unat;
632
633 if (regnum >= IA64_FIRST_STACKED_GR) {
634 get_rse_reg(regs, regnum, val, nat);
635 return;
636 }
637
638 /*
639 * take care of r0 (read-only always evaluate to 0)
640 */
641 if (regnum == 0) {
642 *val = 0;
643 if (nat)
644 *nat = 0;
645 return;
646 }
647
648 /*
649 * Now look at registers in [0-31] range and init correct UNAT
650 */
651 if (GR_IN_SW(regnum)) {
652 addr = (unsigned long)sw;
653 unat = &sw->ar_unat;
654 } else {
655 addr = (unsigned long)regs;
656 unat = &sw->caller_unat;
657 }
658
659 DPRINT("addr_base=%lx offset=0x%x\n", addr, GR_OFFS(regnum));
660
661 addr += GR_OFFS(regnum);
662
663 *val = *(unsigned long *)addr;
664
665 /*
666 * do it only when requested
667 */
668 if (nat)
669 *nat = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL;
670}
671
672static void
673emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa)
674{
675 /*
676 * IMPORTANT:
677 * Given the way we handle unaligned speculative loads, we should
678 * not get to this point in the code but we keep this sanity check,
679 * just in case.
680 */
681 if (ld.x6_op == 1 || ld.x6_op == 3) {
682 printk(KERN_ERR "%s: register update on speculative load, error\n", __func__);
683 if (die_if_kernel("unaligned reference on speculative load with register update\n",
684 regs, 30))
685 return;
686 }
687
688
689 /*
690 * at this point, we know that the base register to update is valid i.e.,
691 * it's not r0
692 */
693 if (type == UPD_IMMEDIATE) {
694 unsigned long imm;
695
696 /*
697 * Load +Imm: ldXZ r1=[r3],imm(9)
698 *
699 *
700 * form imm9: [13:19] contain the first 7 bits
701 */
702 imm = ld.x << 7 | ld.imm;
703
704 /*
705 * sign extend (1+8bits) if m set
706 */
707 if (ld.m) imm |= SIGN_EXT9;
708
709 /*
710 * ifa == r3 and we know that the NaT bit on r3 was clear so
711 * we can directly use ifa.
712 */
713 ifa += imm;
714
715 setreg(ld.r3, ifa, 0, regs);
716
717 DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa);
718
719 } else if (ld.m) {
720 unsigned long r2;
721 int nat_r2;
722
723 /*
724 * Load +Reg Opcode: ldXZ r1=[r3],r2
725 *
726 * Note: that we update r3 even in the case of ldfX.a
727 * (where the load does not happen)
728 *
729 * The way the load algorithm works, we know that r3 does not
730 * have its NaT bit set (would have gotten NaT consumption
731 * before getting the unaligned fault). So we can use ifa
732 * which equals r3 at this point.
733 *
734 * IMPORTANT:
735 * The above statement holds ONLY because we know that we
736 * never reach this code when trying to do a ldX.s.
737 * If we ever make it to here on an ldfX.s then
738 */
739 getreg(ld.imm, &r2, &nat_r2, regs);
740
741 ifa += r2;
742
743 /*
744 * propagate Nat r2 -> r3
745 */
746 setreg(ld.r3, ifa, nat_r2, regs);
747
748 DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2);
749 }
750}
751
752static int emulate_store(unsigned long ifa, void *val, int len, bool kernel_mode)
753{
754 if (kernel_mode)
755 return copy_to_kernel_nofault((void *)ifa, val, len);
756
757 return copy_to_user((void __user *)ifa, val, len);
758}
759
760static int emulate_load(void *val, unsigned long ifa, int len, bool kernel_mode)
761{
762 if (kernel_mode)
763 return copy_from_kernel_nofault(val, (void *)ifa, len);
764
765 return copy_from_user(val, (void __user *)ifa, len);
766}
767
768static int
769emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs,
770 bool kernel_mode)
771{
772 unsigned int len = 1 << ld.x6_sz;
773 unsigned long val = 0;
774
775 /*
776 * r0, as target, doesn't need to be checked because Illegal Instruction
777 * faults have higher priority than unaligned faults.
778 *
779 * r0 cannot be found as the base as it would never generate an
780 * unaligned reference.
781 */
782
783 /*
784 * ldX.a we will emulate load and also invalidate the ALAT entry.
785 * See comment below for explanation on how we handle ldX.a
786 */
787
788 if (len != 2 && len != 4 && len != 8) {
789 DPRINT("unknown size: x6=%d\n", ld.x6_sz);
790 return -1;
791 }
792 /* this assumes little-endian byte-order: */
793 if (emulate_load(&val, ifa, len, kernel_mode))
794 return -1;
795 setreg(ld.r1, val, 0, regs);
796
797 /*
798 * check for updates on any kind of loads
799 */
800 if (ld.op == 0x5 || ld.m)
801 emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
802
803 /*
804 * handling of various loads (based on EAS2.4):
805 *
806 * ldX.acq (ordered load):
807 * - acquire semantics would have been used, so force fence instead.
808 *
809 * ldX.c.clr (check load and clear):
810 * - if we get to this handler, it's because the entry was not in the ALAT.
811 * Therefore the operation reverts to a normal load
812 *
813 * ldX.c.nc (check load no clear):
814 * - same as previous one
815 *
816 * ldX.c.clr.acq (ordered check load and clear):
817 * - same as above for c.clr part. The load needs to have acquire semantics. So
818 * we use the fence semantics which is stronger and thus ensures correctness.
819 *
820 * ldX.a (advanced load):
821 * - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
822 * address doesn't match requested size alignment. This means that we would
823 * possibly need more than one load to get the result.
824 *
825 * The load part can be handled just like a normal load, however the difficult
826 * part is to get the right thing into the ALAT. The critical piece of information
827 * in the base address of the load & size. To do that, a ld.a must be executed,
828 * clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
829 * if we use the same target register, we will be okay for the check.a instruction.
830 * If we look at the store, basically a stX [r3]=r1 checks the ALAT for any entry
831 * which would overlap within [r3,r3+X] (the size of the load was store in the
832 * ALAT). If such an entry is found the entry is invalidated. But this is not good
833 * enough, take the following example:
834 * r3=3
835 * ld4.a r1=[r3]
836 *
837 * Could be emulated by doing:
838 * ld1.a r1=[r3],1
839 * store to temporary;
840 * ld1.a r1=[r3],1
841 * store & shift to temporary;
842 * ld1.a r1=[r3],1
843 * store & shift to temporary;
844 * ld1.a r1=[r3]
845 * store & shift to temporary;
846 * r1=temporary
847 *
848 * So in this case, you would get the right value is r1 but the wrong info in
849 * the ALAT. Notice that you could do it in reverse to finish with address 3
850 * but you would still get the size wrong. To get the size right, one needs to
851 * execute exactly the same kind of load. You could do it from a aligned
852 * temporary location, but you would get the address wrong.
853 *
854 * So no matter what, it is not possible to emulate an advanced load
855 * correctly. But is that really critical ?
856 *
857 * We will always convert ld.a into a normal load with ALAT invalidated. This
858 * will enable compiler to do optimization where certain code path after ld.a
859 * is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
860 *
861 * If there is a store after the advanced load, one must either do a ld.c.* or
862 * chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
863 * entry found in ALAT), and that's perfectly ok because:
864 *
865 * - ld.c.*, if the entry is not present a normal load is executed
866 * - chk.a.*, if the entry is not present, execution jumps to recovery code
867 *
868 * In either case, the load can be potentially retried in another form.
869 *
870 * ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
871 * up a stale entry later). The register base update MUST also be performed.
872 */
873
874 /*
875 * when the load has the .acq completer then
876 * use ordering fence.
877 */
878 if (ld.x6_op == 0x5 || ld.x6_op == 0xa)
879 mb();
880
881 /*
882 * invalidate ALAT entry in case of advanced load
883 */
884 if (ld.x6_op == 0x2)
885 invala_gr(ld.r1);
886
887 return 0;
888}
889
890static int
891emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs,
892 bool kernel_mode)
893{
894 unsigned long r2;
895 unsigned int len = 1 << ld.x6_sz;
896
897 /*
898 * if we get to this handler, Nat bits on both r3 and r2 have already
899 * been checked. so we don't need to do it
900 *
901 * extract the value to be stored
902 */
903 getreg(ld.imm, &r2, NULL, regs);
904
905 /*
906 * we rely on the macros in unaligned.h for now i.e.,
907 * we let the compiler figure out how to read memory gracefully.
908 *
909 * We need this switch/case because the way the inline function
910 * works. The code is optimized by the compiler and looks like
911 * a single switch/case.
912 */
913 DPRINT("st%d [%lx]=%lx\n", len, ifa, r2);
914
915 if (len != 2 && len != 4 && len != 8) {
916 DPRINT("unknown size: x6=%d\n", ld.x6_sz);
917 return -1;
918 }
919
920 /* this assumes little-endian byte-order: */
921 if (emulate_store(ifa, &r2, len, kernel_mode))
922 return -1;
923
924 /*
925 * stX [r3]=r2,imm(9)
926 *
927 * NOTE:
928 * ld.r3 can never be r0, because r0 would not generate an
929 * unaligned access.
930 */
931 if (ld.op == 0x5) {
932 unsigned long imm;
933
934 /*
935 * form imm9: [12:6] contain first 7bits
936 */
937 imm = ld.x << 7 | ld.r1;
938 /*
939 * sign extend (8bits) if m set
940 */
941 if (ld.m) imm |= SIGN_EXT9;
942 /*
943 * ifa == r3 (NaT is necessarily cleared)
944 */
945 ifa += imm;
946
947 DPRINT("imm=%lx r3=%lx\n", imm, ifa);
948
949 setreg(ld.r3, ifa, 0, regs);
950 }
951 /*
952 * we don't have alat_invalidate_multiple() so we need
953 * to do the complete flush :-<<
954 */
955 ia64_invala();
956
957 /*
958 * stX.rel: use fence instead of release
959 */
960 if (ld.x6_op == 0xd)
961 mb();
962
963 return 0;
964}
965
966/*
967 * floating point operations sizes in bytes
968 */
969static const unsigned char float_fsz[4]={
970 10, /* extended precision (e) */
971 8, /* integer (8) */
972 4, /* single precision (s) */
973 8 /* double precision (d) */
974};
975
976static inline void
977mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
978{
979 ia64_ldfe(6, init);
980 ia64_stop();
981 ia64_stf_spill(final, 6);
982}
983
984static inline void
985mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
986{
987 ia64_ldf8(6, init);
988 ia64_stop();
989 ia64_stf_spill(final, 6);
990}
991
992static inline void
993mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
994{
995 ia64_ldfs(6, init);
996 ia64_stop();
997 ia64_stf_spill(final, 6);
998}
999
1000static inline void
1001mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1002{
1003 ia64_ldfd(6, init);
1004 ia64_stop();
1005 ia64_stf_spill(final, 6);
1006}
1007
1008static inline void
1009float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
1010{
1011 ia64_ldf_fill(6, init);
1012 ia64_stop();
1013 ia64_stfe(final, 6);
1014}
1015
1016static inline void
1017float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
1018{
1019 ia64_ldf_fill(6, init);
1020 ia64_stop();
1021 ia64_stf8(final, 6);
1022}
1023
1024static inline void
1025float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
1026{
1027 ia64_ldf_fill(6, init);
1028 ia64_stop();
1029 ia64_stfs(final, 6);
1030}
1031
1032static inline void
1033float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1034{
1035 ia64_ldf_fill(6, init);
1036 ia64_stop();
1037 ia64_stfd(final, 6);
1038}
1039
1040static int
1041emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs, bool kernel_mode)
1042{
1043 struct ia64_fpreg fpr_init[2];
1044 struct ia64_fpreg fpr_final[2];
1045 unsigned long len = float_fsz[ld.x6_sz];
1046
1047 /*
1048 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1049 * higher priority than unaligned faults.
1050 *
1051 * r0 cannot be found as the base as it would never generate an unaligned
1052 * reference.
1053 */
1054
1055 /*
1056 * make sure we get clean buffers
1057 */
1058 memset(&fpr_init, 0, sizeof(fpr_init));
1059 memset(&fpr_final, 0, sizeof(fpr_final));
1060
1061 /*
1062 * ldfpX.a: we don't try to emulate anything but we must
1063 * invalidate the ALAT entry and execute updates, if any.
1064 */
1065 if (ld.x6_op != 0x2) {
1066 /*
1067 * This assumes little-endian byte-order. Note that there is no "ldfpe"
1068 * instruction:
1069 */
1070 if (emulate_load(&fpr_init[0], ifa, len, kernel_mode)
1071 || emulate_load(&fpr_init[1], (ifa + len), len, kernel_mode))
1072 return -1;
1073
1074 DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz);
1075 DDUMP("frp_init =", &fpr_init, 2*len);
1076 /*
1077 * XXX fixme
1078 * Could optimize inlines by using ldfpX & 2 spills
1079 */
1080 switch( ld.x6_sz ) {
1081 case 0:
1082 mem2float_extended(&fpr_init[0], &fpr_final[0]);
1083 mem2float_extended(&fpr_init[1], &fpr_final[1]);
1084 break;
1085 case 1:
1086 mem2float_integer(&fpr_init[0], &fpr_final[0]);
1087 mem2float_integer(&fpr_init[1], &fpr_final[1]);
1088 break;
1089 case 2:
1090 mem2float_single(&fpr_init[0], &fpr_final[0]);
1091 mem2float_single(&fpr_init[1], &fpr_final[1]);
1092 break;
1093 case 3:
1094 mem2float_double(&fpr_init[0], &fpr_final[0]);
1095 mem2float_double(&fpr_init[1], &fpr_final[1]);
1096 break;
1097 }
1098 DDUMP("fpr_final =", &fpr_final, 2*len);
1099 /*
1100 * XXX fixme
1101 *
1102 * A possible optimization would be to drop fpr_final and directly
1103 * use the storage from the saved context i.e., the actual final
1104 * destination (pt_regs, switch_stack or thread structure).
1105 */
1106 setfpreg(ld.r1, &fpr_final[0], regs);
1107 setfpreg(ld.imm, &fpr_final[1], regs);
1108 }
1109
1110 /*
1111 * Check for updates: only immediate updates are available for this
1112 * instruction.
1113 */
1114 if (ld.m) {
1115 /*
1116 * the immediate is implicit given the ldsz of the operation:
1117 * single: 8 (2x4) and for all others it's 16 (2x8)
1118 */
1119 ifa += len<<1;
1120
1121 /*
1122 * IMPORTANT:
1123 * the fact that we force the NaT of r3 to zero is ONLY valid
1124 * as long as we don't come here with a ldfpX.s.
1125 * For this reason we keep this sanity check
1126 */
1127 if (ld.x6_op == 1 || ld.x6_op == 3)
1128 printk(KERN_ERR "%s: register update on speculative load pair, error\n",
1129 __func__);
1130
1131 setreg(ld.r3, ifa, 0, regs);
1132 }
1133
1134 /*
1135 * Invalidate ALAT entries, if any, for both registers.
1136 */
1137 if (ld.x6_op == 0x2) {
1138 invala_fr(ld.r1);
1139 invala_fr(ld.imm);
1140 }
1141 return 0;
1142}
1143
1144
1145static int
1146emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs,
1147 bool kernel_mode)
1148{
1149 struct ia64_fpreg fpr_init;
1150 struct ia64_fpreg fpr_final;
1151 unsigned long len = float_fsz[ld.x6_sz];
1152
1153 /*
1154 * fr0 & fr1 don't need to be checked because Illegal Instruction
1155 * faults have higher priority than unaligned faults.
1156 *
1157 * r0 cannot be found as the base as it would never generate an
1158 * unaligned reference.
1159 */
1160
1161 /*
1162 * make sure we get clean buffers
1163 */
1164 memset(&fpr_init,0, sizeof(fpr_init));
1165 memset(&fpr_final,0, sizeof(fpr_final));
1166
1167 /*
1168 * ldfX.a we don't try to emulate anything but we must
1169 * invalidate the ALAT entry.
1170 * See comments in ldX for descriptions on how the various loads are handled.
1171 */
1172 if (ld.x6_op != 0x2) {
1173 if (emulate_load(&fpr_init, ifa, len, kernel_mode))
1174 return -1;
1175
1176 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1177 DDUMP("fpr_init =", &fpr_init, len);
1178 /*
1179 * we only do something for x6_op={0,8,9}
1180 */
1181 switch( ld.x6_sz ) {
1182 case 0:
1183 mem2float_extended(&fpr_init, &fpr_final);
1184 break;
1185 case 1:
1186 mem2float_integer(&fpr_init, &fpr_final);
1187 break;
1188 case 2:
1189 mem2float_single(&fpr_init, &fpr_final);
1190 break;
1191 case 3:
1192 mem2float_double(&fpr_init, &fpr_final);
1193 break;
1194 }
1195 DDUMP("fpr_final =", &fpr_final, len);
1196 /*
1197 * XXX fixme
1198 *
1199 * A possible optimization would be to drop fpr_final and directly
1200 * use the storage from the saved context i.e., the actual final
1201 * destination (pt_regs, switch_stack or thread structure).
1202 */
1203 setfpreg(ld.r1, &fpr_final, regs);
1204 }
1205
1206 /*
1207 * check for updates on any loads
1208 */
1209 if (ld.op == 0x7 || ld.m)
1210 emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
1211
1212 /*
1213 * invalidate ALAT entry in case of advanced floating point loads
1214 */
1215 if (ld.x6_op == 0x2)
1216 invala_fr(ld.r1);
1217
1218 return 0;
1219}
1220
1221
1222static int
1223emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs,
1224 bool kernel_mode)
1225{
1226 struct ia64_fpreg fpr_init;
1227 struct ia64_fpreg fpr_final;
1228 unsigned long len = float_fsz[ld.x6_sz];
1229
1230 /*
1231 * make sure we get clean buffers
1232 */
1233 memset(&fpr_init,0, sizeof(fpr_init));
1234 memset(&fpr_final,0, sizeof(fpr_final));
1235
1236 /*
1237 * if we get to this handler, Nat bits on both r3 and r2 have already
1238 * been checked. so we don't need to do it
1239 *
1240 * extract the value to be stored
1241 */
1242 getfpreg(ld.imm, &fpr_init, regs);
1243 /*
1244 * during this step, we extract the spilled registers from the saved
1245 * context i.e., we refill. Then we store (no spill) to temporary
1246 * aligned location
1247 */
1248 switch( ld.x6_sz ) {
1249 case 0:
1250 float2mem_extended(&fpr_init, &fpr_final);
1251 break;
1252 case 1:
1253 float2mem_integer(&fpr_init, &fpr_final);
1254 break;
1255 case 2:
1256 float2mem_single(&fpr_init, &fpr_final);
1257 break;
1258 case 3:
1259 float2mem_double(&fpr_init, &fpr_final);
1260 break;
1261 }
1262 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1263 DDUMP("fpr_init =", &fpr_init, len);
1264 DDUMP("fpr_final =", &fpr_final, len);
1265
1266 if (emulate_store(ifa, &fpr_final, len, kernel_mode))
1267 return -1;
1268
1269 /*
1270 * stfX [r3]=r2,imm(9)
1271 *
1272 * NOTE:
1273 * ld.r3 can never be r0, because r0 would not generate an
1274 * unaligned access.
1275 */
1276 if (ld.op == 0x7) {
1277 unsigned long imm;
1278
1279 /*
1280 * form imm9: [12:6] contain first 7bits
1281 */
1282 imm = ld.x << 7 | ld.r1;
1283 /*
1284 * sign extend (8bits) if m set
1285 */
1286 if (ld.m)
1287 imm |= SIGN_EXT9;
1288 /*
1289 * ifa == r3 (NaT is necessarily cleared)
1290 */
1291 ifa += imm;
1292
1293 DPRINT("imm=%lx r3=%lx\n", imm, ifa);
1294
1295 setreg(ld.r3, ifa, 0, regs);
1296 }
1297 /*
1298 * we don't have alat_invalidate_multiple() so we need
1299 * to do the complete flush :-<<
1300 */
1301 ia64_invala();
1302
1303 return 0;
1304}
1305
1306/*
1307 * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1308 * eventually fix the program. However, we don't want to do that for every access so we
1309 * pace it with jiffies.
1310 */
1311static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5);
1312
1313void
1314ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs)
1315{
1316 struct ia64_psr *ipsr = ia64_psr(regs);
1317 unsigned long bundle[2];
1318 unsigned long opcode;
1319 const struct exception_table_entry *eh = NULL;
1320 union {
1321 unsigned long l;
1322 load_store_t insn;
1323 } u;
1324 int ret = -1;
1325 bool kernel_mode = false;
1326
1327 if (ia64_psr(regs)->be) {
1328 /* we don't support big-endian accesses */
1329 if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0))
1330 return;
1331 goto force_sigbus;
1332 }
1333
1334 /*
1335 * Treat kernel accesses for which there is an exception handler entry the same as
1336 * user-level unaligned accesses. Otherwise, a clever program could trick this
1337 * handler into reading an arbitrary kernel addresses...
1338 */
1339 if (!user_mode(regs))
1340 eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri);
1341 if (user_mode(regs) || eh) {
1342 if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0)
1343 goto force_sigbus;
1344
1345 if (!no_unaligned_warning &&
1346 !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) &&
1347 __ratelimit(&logging_rate_limit))
1348 {
1349 char buf[200]; /* comm[] is at most 16 bytes... */
1350 size_t len;
1351
1352 len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, "
1353 "ip=0x%016lx\n\r", current->comm,
1354 task_pid_nr(current),
1355 ifa, regs->cr_iip + ipsr->ri);
1356 /*
1357 * Don't call tty_write_message() if we're in the kernel; we might
1358 * be holding locks...
1359 */
1360 if (user_mode(regs)) {
1361 struct tty_struct *tty = get_current_tty();
1362 tty_write_message(tty, buf);
1363 tty_kref_put(tty);
1364 }
1365 buf[len-1] = '\0'; /* drop '\r' */
1366 /* watch for command names containing %s */
1367 printk(KERN_WARNING "%s", buf);
1368 } else {
1369 if (no_unaligned_warning) {
1370 printk_once(KERN_WARNING "%s(%d) encountered an "
1371 "unaligned exception which required\n"
1372 "kernel assistance, which degrades "
1373 "the performance of the application.\n"
1374 "Unaligned exception warnings have "
1375 "been disabled by the system "
1376 "administrator\n"
1377 "echo 0 > /proc/sys/kernel/ignore-"
1378 "unaligned-usertrap to re-enable\n",
1379 current->comm, task_pid_nr(current));
1380 }
1381 }
1382 } else {
1383 if (__ratelimit(&logging_rate_limit)) {
1384 printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n",
1385 ifa, regs->cr_iip + ipsr->ri);
1386 if (unaligned_dump_stack)
1387 dump_stack();
1388 }
1389 kernel_mode = true;
1390 }
1391
1392 DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n",
1393 regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it);
1394
1395 if (emulate_load(bundle, regs->cr_iip, 16, kernel_mode))
1396 goto failure;
1397
1398 /*
1399 * extract the instruction from the bundle given the slot number
1400 */
1401 switch (ipsr->ri) {
1402 default:
1403 case 0: u.l = (bundle[0] >> 5); break;
1404 case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break;
1405 case 2: u.l = (bundle[1] >> 23); break;
1406 }
1407 opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK;
1408
1409 DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d "
1410 "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm,
1411 u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op);
1412
1413 /*
1414 * IMPORTANT:
1415 * Notice that the switch statement DOES not cover all possible instructions
1416 * that DO generate unaligned references. This is made on purpose because for some
1417 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1418 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1419 * the program will get a signal and die:
1420 *
1421 * load/store:
1422 * - ldX.spill
1423 * - stX.spill
1424 * Reason: RNATs are based on addresses
1425 * - ld16
1426 * - st16
1427 * Reason: ld16 and st16 are supposed to occur in a single
1428 * memory op
1429 *
1430 * synchronization:
1431 * - cmpxchg
1432 * - fetchadd
1433 * - xchg
1434 * Reason: ATOMIC operations cannot be emulated properly using multiple
1435 * instructions.
1436 *
1437 * speculative loads:
1438 * - ldX.sZ
1439 * Reason: side effects, code must be ready to deal with failure so simpler
1440 * to let the load fail.
1441 * ---------------------------------------------------------------------------------
1442 * XXX fixme
1443 *
1444 * I would like to get rid of this switch case and do something
1445 * more elegant.
1446 */
1447 switch (opcode) {
1448 case LDS_OP:
1449 case LDSA_OP:
1450 if (u.insn.x)
1451 /* oops, really a semaphore op (cmpxchg, etc) */
1452 goto failure;
1453 fallthrough;
1454 case LDS_IMM_OP:
1455 case LDSA_IMM_OP:
1456 case LDFS_OP:
1457 case LDFSA_OP:
1458 case LDFS_IMM_OP:
1459 /*
1460 * The instruction will be retried with deferred exceptions turned on, and
1461 * we should get Nat bit installed
1462 *
1463 * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1464 * are actually executed even though the operation failed. So we don't
1465 * need to take care of this.
1466 */
1467 DPRINT("forcing PSR_ED\n");
1468 regs->cr_ipsr |= IA64_PSR_ED;
1469 goto done;
1470
1471 case LD_OP:
1472 case LDA_OP:
1473 case LDBIAS_OP:
1474 case LDACQ_OP:
1475 case LDCCLR_OP:
1476 case LDCNC_OP:
1477 case LDCCLRACQ_OP:
1478 if (u.insn.x)
1479 /* oops, really a semaphore op (cmpxchg, etc) */
1480 goto failure;
1481 fallthrough;
1482 case LD_IMM_OP:
1483 case LDA_IMM_OP:
1484 case LDBIAS_IMM_OP:
1485 case LDACQ_IMM_OP:
1486 case LDCCLR_IMM_OP:
1487 case LDCNC_IMM_OP:
1488 case LDCCLRACQ_IMM_OP:
1489 ret = emulate_load_int(ifa, u.insn, regs, kernel_mode);
1490 break;
1491
1492 case ST_OP:
1493 case STREL_OP:
1494 if (u.insn.x)
1495 /* oops, really a semaphore op (cmpxchg, etc) */
1496 goto failure;
1497 fallthrough;
1498 case ST_IMM_OP:
1499 case STREL_IMM_OP:
1500 ret = emulate_store_int(ifa, u.insn, regs, kernel_mode);
1501 break;
1502
1503 case LDF_OP:
1504 case LDFA_OP:
1505 case LDFCCLR_OP:
1506 case LDFCNC_OP:
1507 if (u.insn.x)
1508 ret = emulate_load_floatpair(ifa, u.insn, regs, kernel_mode);
1509 else
1510 ret = emulate_load_float(ifa, u.insn, regs, kernel_mode);
1511 break;
1512
1513 case LDF_IMM_OP:
1514 case LDFA_IMM_OP:
1515 case LDFCCLR_IMM_OP:
1516 case LDFCNC_IMM_OP:
1517 ret = emulate_load_float(ifa, u.insn, regs, kernel_mode);
1518 break;
1519
1520 case STF_OP:
1521 case STF_IMM_OP:
1522 ret = emulate_store_float(ifa, u.insn, regs, kernel_mode);
1523 break;
1524
1525 default:
1526 goto failure;
1527 }
1528 DPRINT("ret=%d\n", ret);
1529 if (ret)
1530 goto failure;
1531
1532 if (ipsr->ri == 2)
1533 /*
1534 * given today's architecture this case is not likely to happen because a
1535 * memory access instruction (M) can never be in the last slot of a
1536 * bundle. But let's keep it for now.
1537 */
1538 regs->cr_iip += 16;
1539 ipsr->ri = (ipsr->ri + 1) & 0x3;
1540
1541 DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip);
1542 done:
1543 return;
1544
1545 failure:
1546 /* something went wrong... */
1547 if (!user_mode(regs)) {
1548 if (eh) {
1549 ia64_handle_exception(regs, eh);
1550 goto done;
1551 }
1552 if (die_if_kernel("error during unaligned kernel access\n", regs, ret))
1553 return;
1554 /* NOT_REACHED */
1555 }
1556 force_sigbus:
1557 force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) ifa,
1558 0, 0, 0);
1559 goto done;
1560}
1/*
2 * Architecture-specific unaligned trap handling.
3 *
4 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
5 * Stephane Eranian <eranian@hpl.hp.com>
6 * David Mosberger-Tang <davidm@hpl.hp.com>
7 *
8 * 2002/12/09 Fix rotating register handling (off-by-1 error, missing fr-rotation). Fix
9 * get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
10 * stacked register returns an undefined value; it does NOT trigger a
11 * "rsvd register fault").
12 * 2001/10/11 Fix unaligned access to rotating registers in s/w pipelined loops.
13 * 2001/08/13 Correct size of extended floats (float_fsz) from 16 to 10 bytes.
14 * 2001/01/17 Add support emulation of unaligned kernel accesses.
15 */
16#include <linux/jiffies.h>
17#include <linux/kernel.h>
18#include <linux/sched.h>
19#include <linux/tty.h>
20#include <linux/ratelimit.h>
21
22#include <asm/intrinsics.h>
23#include <asm/processor.h>
24#include <asm/rse.h>
25#include <asm/uaccess.h>
26#include <asm/unaligned.h>
27
28extern int die_if_kernel(char *str, struct pt_regs *regs, long err);
29
30#undef DEBUG_UNALIGNED_TRAP
31
32#ifdef DEBUG_UNALIGNED_TRAP
33# define DPRINT(a...) do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0)
34# define DDUMP(str,vp,len) dump(str, vp, len)
35
36static void
37dump (const char *str, void *vp, size_t len)
38{
39 unsigned char *cp = vp;
40 int i;
41
42 printk("%s", str);
43 for (i = 0; i < len; ++i)
44 printk (" %02x", *cp++);
45 printk("\n");
46}
47#else
48# define DPRINT(a...)
49# define DDUMP(str,vp,len)
50#endif
51
52#define IA64_FIRST_STACKED_GR 32
53#define IA64_FIRST_ROTATING_FR 32
54#define SIGN_EXT9 0xffffffffffffff00ul
55
56/*
57 * sysctl settable hook which tells the kernel whether to honor the
58 * IA64_THREAD_UAC_NOPRINT prctl. Because this is user settable, we want
59 * to allow the super user to enable/disable this for security reasons
60 * (i.e. don't allow attacker to fill up logs with unaligned accesses).
61 */
62int no_unaligned_warning;
63int unaligned_dump_stack;
64
65/*
66 * For M-unit:
67 *
68 * opcode | m | x6 |
69 * --------|------|---------|
70 * [40-37] | [36] | [35:30] |
71 * --------|------|---------|
72 * 4 | 1 | 6 | = 11 bits
73 * --------------------------
74 * However bits [31:30] are not directly useful to distinguish between
75 * load/store so we can use [35:32] instead, which gives the following
76 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
77 * checking the m-bit until later in the load/store emulation.
78 */
79#define IA64_OPCODE_MASK 0x1ef
80#define IA64_OPCODE_SHIFT 32
81
82/*
83 * Table C-28 Integer Load/Store
84 *
85 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
86 *
87 * ld8.fill, st8.fill MUST be aligned because the RNATs are based on
88 * the address (bits [8:3]), so we must failed.
89 */
90#define LD_OP 0x080
91#define LDS_OP 0x081
92#define LDA_OP 0x082
93#define LDSA_OP 0x083
94#define LDBIAS_OP 0x084
95#define LDACQ_OP 0x085
96/* 0x086, 0x087 are not relevant */
97#define LDCCLR_OP 0x088
98#define LDCNC_OP 0x089
99#define LDCCLRACQ_OP 0x08a
100#define ST_OP 0x08c
101#define STREL_OP 0x08d
102/* 0x08e,0x8f are not relevant */
103
104/*
105 * Table C-29 Integer Load +Reg
106 *
107 * we use the ld->m (bit [36:36]) field to determine whether or not we have
108 * a load/store of this form.
109 */
110
111/*
112 * Table C-30 Integer Load/Store +Imm
113 *
114 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
115 *
116 * ld8.fill, st8.fill must be aligned because the Nat register are based on
117 * the address, so we must fail and the program must be fixed.
118 */
119#define LD_IMM_OP 0x0a0
120#define LDS_IMM_OP 0x0a1
121#define LDA_IMM_OP 0x0a2
122#define LDSA_IMM_OP 0x0a3
123#define LDBIAS_IMM_OP 0x0a4
124#define LDACQ_IMM_OP 0x0a5
125/* 0x0a6, 0xa7 are not relevant */
126#define LDCCLR_IMM_OP 0x0a8
127#define LDCNC_IMM_OP 0x0a9
128#define LDCCLRACQ_IMM_OP 0x0aa
129#define ST_IMM_OP 0x0ac
130#define STREL_IMM_OP 0x0ad
131/* 0x0ae,0xaf are not relevant */
132
133/*
134 * Table C-32 Floating-point Load/Store
135 */
136#define LDF_OP 0x0c0
137#define LDFS_OP 0x0c1
138#define LDFA_OP 0x0c2
139#define LDFSA_OP 0x0c3
140/* 0x0c6 is irrelevant */
141#define LDFCCLR_OP 0x0c8
142#define LDFCNC_OP 0x0c9
143/* 0x0cb is irrelevant */
144#define STF_OP 0x0cc
145
146/*
147 * Table C-33 Floating-point Load +Reg
148 *
149 * we use the ld->m (bit [36:36]) field to determine whether or not we have
150 * a load/store of this form.
151 */
152
153/*
154 * Table C-34 Floating-point Load/Store +Imm
155 */
156#define LDF_IMM_OP 0x0e0
157#define LDFS_IMM_OP 0x0e1
158#define LDFA_IMM_OP 0x0e2
159#define LDFSA_IMM_OP 0x0e3
160/* 0x0e6 is irrelevant */
161#define LDFCCLR_IMM_OP 0x0e8
162#define LDFCNC_IMM_OP 0x0e9
163#define STF_IMM_OP 0x0ec
164
165typedef struct {
166 unsigned long qp:6; /* [0:5] */
167 unsigned long r1:7; /* [6:12] */
168 unsigned long imm:7; /* [13:19] */
169 unsigned long r3:7; /* [20:26] */
170 unsigned long x:1; /* [27:27] */
171 unsigned long hint:2; /* [28:29] */
172 unsigned long x6_sz:2; /* [30:31] */
173 unsigned long x6_op:4; /* [32:35], x6 = x6_sz|x6_op */
174 unsigned long m:1; /* [36:36] */
175 unsigned long op:4; /* [37:40] */
176 unsigned long pad:23; /* [41:63] */
177} load_store_t;
178
179
180typedef enum {
181 UPD_IMMEDIATE, /* ldXZ r1=[r3],imm(9) */
182 UPD_REG /* ldXZ r1=[r3],r2 */
183} update_t;
184
185/*
186 * We use tables to keep track of the offsets of registers in the saved state.
187 * This way we save having big switch/case statements.
188 *
189 * We use bit 0 to indicate switch_stack or pt_regs.
190 * The offset is simply shifted by 1 bit.
191 * A 2-byte value should be enough to hold any kind of offset
192 *
193 * In case the calling convention changes (and thus pt_regs/switch_stack)
194 * simply use RSW instead of RPT or vice-versa.
195 */
196
197#define RPO(x) ((size_t) &((struct pt_regs *)0)->x)
198#define RSO(x) ((size_t) &((struct switch_stack *)0)->x)
199
200#define RPT(x) (RPO(x) << 1)
201#define RSW(x) (1| RSO(x)<<1)
202
203#define GR_OFFS(x) (gr_info[x]>>1)
204#define GR_IN_SW(x) (gr_info[x] & 0x1)
205
206#define FR_OFFS(x) (fr_info[x]>>1)
207#define FR_IN_SW(x) (fr_info[x] & 0x1)
208
209static u16 gr_info[32]={
210 0, /* r0 is read-only : WE SHOULD NEVER GET THIS */
211
212 RPT(r1), RPT(r2), RPT(r3),
213
214 RSW(r4), RSW(r5), RSW(r6), RSW(r7),
215
216 RPT(r8), RPT(r9), RPT(r10), RPT(r11),
217 RPT(r12), RPT(r13), RPT(r14), RPT(r15),
218
219 RPT(r16), RPT(r17), RPT(r18), RPT(r19),
220 RPT(r20), RPT(r21), RPT(r22), RPT(r23),
221 RPT(r24), RPT(r25), RPT(r26), RPT(r27),
222 RPT(r28), RPT(r29), RPT(r30), RPT(r31)
223};
224
225static u16 fr_info[32]={
226 0, /* constant : WE SHOULD NEVER GET THIS */
227 0, /* constant : WE SHOULD NEVER GET THIS */
228
229 RSW(f2), RSW(f3), RSW(f4), RSW(f5),
230
231 RPT(f6), RPT(f7), RPT(f8), RPT(f9),
232 RPT(f10), RPT(f11),
233
234 RSW(f12), RSW(f13), RSW(f14),
235 RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19),
236 RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24),
237 RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29),
238 RSW(f30), RSW(f31)
239};
240
241/* Invalidate ALAT entry for integer register REGNO. */
242static void
243invala_gr (int regno)
244{
245# define F(reg) case reg: ia64_invala_gr(reg); break
246
247 switch (regno) {
248 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7);
249 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
250 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
251 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
252 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
253 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
254 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
255 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
256 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
257 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
258 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
259 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
260 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
261 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
262 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
263 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
264 }
265# undef F
266}
267
268/* Invalidate ALAT entry for floating-point register REGNO. */
269static void
270invala_fr (int regno)
271{
272# define F(reg) case reg: ia64_invala_fr(reg); break
273
274 switch (regno) {
275 F( 0); F( 1); F( 2); F( 3); F( 4); F( 5); F( 6); F( 7);
276 F( 8); F( 9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
277 F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
278 F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
279 F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
280 F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
281 F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
282 F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
283 F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
284 F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
285 F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
286 F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
287 F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
288 F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
289 F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
290 F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
291 }
292# undef F
293}
294
295static inline unsigned long
296rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg)
297{
298 reg += rrb;
299 if (reg >= sor)
300 reg -= sor;
301 return reg;
302}
303
304static void
305set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat)
306{
307 struct switch_stack *sw = (struct switch_stack *) regs - 1;
308 unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end;
309 unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
310 unsigned long rnats, nat_mask;
311 unsigned long on_kbs;
312 long sof = (regs->cr_ifs) & 0x7f;
313 long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
314 long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
315 long ridx = r1 - 32;
316
317 if (ridx >= sof) {
318 /* this should never happen, as the "rsvd register fault" has higher priority */
319 DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof);
320 return;
321 }
322
323 if (ridx < sor)
324 ridx = rotate_reg(sor, rrb_gr, ridx);
325
326 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
327 r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
328
329 on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
330 addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
331 if (addr >= kbs) {
332 /* the register is on the kernel backing store: easy... */
333 rnat_addr = ia64_rse_rnat_addr(addr);
334 if ((unsigned long) rnat_addr >= sw->ar_bspstore)
335 rnat_addr = &sw->ar_rnat;
336 nat_mask = 1UL << ia64_rse_slot_num(addr);
337
338 *addr = val;
339 if (nat)
340 *rnat_addr |= nat_mask;
341 else
342 *rnat_addr &= ~nat_mask;
343 return;
344 }
345
346 if (!user_stack(current, regs)) {
347 DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1);
348 return;
349 }
350
351 bspstore = (unsigned long *)regs->ar_bspstore;
352 ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
353 bsp = ia64_rse_skip_regs(ubs_end, -sof);
354 addr = ia64_rse_skip_regs(bsp, ridx);
355
356 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
357
358 ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
359
360 rnat_addr = ia64_rse_rnat_addr(addr);
361
362 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
363 DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n",
364 (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1);
365
366 nat_mask = 1UL << ia64_rse_slot_num(addr);
367 if (nat)
368 rnats |= nat_mask;
369 else
370 rnats &= ~nat_mask;
371 ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats);
372
373 DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats);
374}
375
376
377static void
378get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat)
379{
380 struct switch_stack *sw = (struct switch_stack *) regs - 1;
381 unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore;
382 unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
383 unsigned long rnats, nat_mask;
384 unsigned long on_kbs;
385 long sof = (regs->cr_ifs) & 0x7f;
386 long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
387 long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
388 long ridx = r1 - 32;
389
390 if (ridx >= sof) {
391 /* read of out-of-frame register returns an undefined value; 0 in our case. */
392 DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof);
393 goto fail;
394 }
395
396 if (ridx < sor)
397 ridx = rotate_reg(sor, rrb_gr, ridx);
398
399 DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
400 r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
401
402 on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
403 addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
404 if (addr >= kbs) {
405 /* the register is on the kernel backing store: easy... */
406 *val = *addr;
407 if (nat) {
408 rnat_addr = ia64_rse_rnat_addr(addr);
409 if ((unsigned long) rnat_addr >= sw->ar_bspstore)
410 rnat_addr = &sw->ar_rnat;
411 nat_mask = 1UL << ia64_rse_slot_num(addr);
412 *nat = (*rnat_addr & nat_mask) != 0;
413 }
414 return;
415 }
416
417 if (!user_stack(current, regs)) {
418 DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1);
419 goto fail;
420 }
421
422 bspstore = (unsigned long *)regs->ar_bspstore;
423 ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
424 bsp = ia64_rse_skip_regs(ubs_end, -sof);
425 addr = ia64_rse_skip_regs(bsp, ridx);
426
427 DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
428
429 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
430
431 if (nat) {
432 rnat_addr = ia64_rse_rnat_addr(addr);
433 nat_mask = 1UL << ia64_rse_slot_num(addr);
434
435 DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats);
436
437 ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
438 *nat = (rnats & nat_mask) != 0;
439 }
440 return;
441
442 fail:
443 *val = 0;
444 if (nat)
445 *nat = 0;
446 return;
447}
448
449
450static void
451setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs)
452{
453 struct switch_stack *sw = (struct switch_stack *) regs - 1;
454 unsigned long addr;
455 unsigned long bitmask;
456 unsigned long *unat;
457
458 /*
459 * First takes care of stacked registers
460 */
461 if (regnum >= IA64_FIRST_STACKED_GR) {
462 set_rse_reg(regs, regnum, val, nat);
463 return;
464 }
465
466 /*
467 * Using r0 as a target raises a General Exception fault which has higher priority
468 * than the Unaligned Reference fault.
469 */
470
471 /*
472 * Now look at registers in [0-31] range and init correct UNAT
473 */
474 if (GR_IN_SW(regnum)) {
475 addr = (unsigned long)sw;
476 unat = &sw->ar_unat;
477 } else {
478 addr = (unsigned long)regs;
479 unat = &sw->caller_unat;
480 }
481 DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n",
482 addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum));
483 /*
484 * add offset from base of struct
485 * and do it !
486 */
487 addr += GR_OFFS(regnum);
488
489 *(unsigned long *)addr = val;
490
491 /*
492 * We need to clear the corresponding UNAT bit to fully emulate the load
493 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
494 */
495 bitmask = 1UL << (addr >> 3 & 0x3f);
496 DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat);
497 if (nat) {
498 *unat |= bitmask;
499 } else {
500 *unat &= ~bitmask;
501 }
502 DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat);
503}
504
505/*
506 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
507 * range from 32-127, result is in the range from 0-95.
508 */
509static inline unsigned long
510fph_index (struct pt_regs *regs, long regnum)
511{
512 unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;
513 return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));
514}
515
516static void
517setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
518{
519 struct switch_stack *sw = (struct switch_stack *)regs - 1;
520 unsigned long addr;
521
522 /*
523 * From EAS-2.5: FPDisableFault has higher priority than Unaligned
524 * Fault. Thus, when we get here, we know the partition is enabled.
525 * To update f32-f127, there are three choices:
526 *
527 * (1) save f32-f127 to thread.fph and update the values there
528 * (2) use a gigantic switch statement to directly access the registers
529 * (3) generate code on the fly to update the desired register
530 *
531 * For now, we are using approach (1).
532 */
533 if (regnum >= IA64_FIRST_ROTATING_FR) {
534 ia64_sync_fph(current);
535 current->thread.fph[fph_index(regs, regnum)] = *fpval;
536 } else {
537 /*
538 * pt_regs or switch_stack ?
539 */
540 if (FR_IN_SW(regnum)) {
541 addr = (unsigned long)sw;
542 } else {
543 addr = (unsigned long)regs;
544 }
545
546 DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum));
547
548 addr += FR_OFFS(regnum);
549 *(struct ia64_fpreg *)addr = *fpval;
550
551 /*
552 * mark the low partition as being used now
553 *
554 * It is highly unlikely that this bit is not already set, but
555 * let's do it for safety.
556 */
557 regs->cr_ipsr |= IA64_PSR_MFL;
558 }
559}
560
561/*
562 * Those 2 inline functions generate the spilled versions of the constant floating point
563 * registers which can be used with stfX
564 */
565static inline void
566float_spill_f0 (struct ia64_fpreg *final)
567{
568 ia64_stf_spill(final, 0);
569}
570
571static inline void
572float_spill_f1 (struct ia64_fpreg *final)
573{
574 ia64_stf_spill(final, 1);
575}
576
577static void
578getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
579{
580 struct switch_stack *sw = (struct switch_stack *) regs - 1;
581 unsigned long addr;
582
583 /*
584 * From EAS-2.5: FPDisableFault has higher priority than
585 * Unaligned Fault. Thus, when we get here, we know the partition is
586 * enabled.
587 *
588 * When regnum > 31, the register is still live and we need to force a save
589 * to current->thread.fph to get access to it. See discussion in setfpreg()
590 * for reasons and other ways of doing this.
591 */
592 if (regnum >= IA64_FIRST_ROTATING_FR) {
593 ia64_flush_fph(current);
594 *fpval = current->thread.fph[fph_index(regs, regnum)];
595 } else {
596 /*
597 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
598 * not saved, we must generate their spilled form on the fly
599 */
600 switch(regnum) {
601 case 0:
602 float_spill_f0(fpval);
603 break;
604 case 1:
605 float_spill_f1(fpval);
606 break;
607 default:
608 /*
609 * pt_regs or switch_stack ?
610 */
611 addr = FR_IN_SW(regnum) ? (unsigned long)sw
612 : (unsigned long)regs;
613
614 DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n",
615 FR_IN_SW(regnum), addr, FR_OFFS(regnum));
616
617 addr += FR_OFFS(regnum);
618 *fpval = *(struct ia64_fpreg *)addr;
619 }
620 }
621}
622
623
624static void
625getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs)
626{
627 struct switch_stack *sw = (struct switch_stack *) regs - 1;
628 unsigned long addr, *unat;
629
630 if (regnum >= IA64_FIRST_STACKED_GR) {
631 get_rse_reg(regs, regnum, val, nat);
632 return;
633 }
634
635 /*
636 * take care of r0 (read-only always evaluate to 0)
637 */
638 if (regnum == 0) {
639 *val = 0;
640 if (nat)
641 *nat = 0;
642 return;
643 }
644
645 /*
646 * Now look at registers in [0-31] range and init correct UNAT
647 */
648 if (GR_IN_SW(regnum)) {
649 addr = (unsigned long)sw;
650 unat = &sw->ar_unat;
651 } else {
652 addr = (unsigned long)regs;
653 unat = &sw->caller_unat;
654 }
655
656 DPRINT("addr_base=%lx offset=0x%x\n", addr, GR_OFFS(regnum));
657
658 addr += GR_OFFS(regnum);
659
660 *val = *(unsigned long *)addr;
661
662 /*
663 * do it only when requested
664 */
665 if (nat)
666 *nat = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL;
667}
668
669static void
670emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa)
671{
672 /*
673 * IMPORTANT:
674 * Given the way we handle unaligned speculative loads, we should
675 * not get to this point in the code but we keep this sanity check,
676 * just in case.
677 */
678 if (ld.x6_op == 1 || ld.x6_op == 3) {
679 printk(KERN_ERR "%s: register update on speculative load, error\n", __func__);
680 if (die_if_kernel("unaligned reference on speculative load with register update\n",
681 regs, 30))
682 return;
683 }
684
685
686 /*
687 * at this point, we know that the base register to update is valid i.e.,
688 * it's not r0
689 */
690 if (type == UPD_IMMEDIATE) {
691 unsigned long imm;
692
693 /*
694 * Load +Imm: ldXZ r1=[r3],imm(9)
695 *
696 *
697 * form imm9: [13:19] contain the first 7 bits
698 */
699 imm = ld.x << 7 | ld.imm;
700
701 /*
702 * sign extend (1+8bits) if m set
703 */
704 if (ld.m) imm |= SIGN_EXT9;
705
706 /*
707 * ifa == r3 and we know that the NaT bit on r3 was clear so
708 * we can directly use ifa.
709 */
710 ifa += imm;
711
712 setreg(ld.r3, ifa, 0, regs);
713
714 DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa);
715
716 } else if (ld.m) {
717 unsigned long r2;
718 int nat_r2;
719
720 /*
721 * Load +Reg Opcode: ldXZ r1=[r3],r2
722 *
723 * Note: that we update r3 even in the case of ldfX.a
724 * (where the load does not happen)
725 *
726 * The way the load algorithm works, we know that r3 does not
727 * have its NaT bit set (would have gotten NaT consumption
728 * before getting the unaligned fault). So we can use ifa
729 * which equals r3 at this point.
730 *
731 * IMPORTANT:
732 * The above statement holds ONLY because we know that we
733 * never reach this code when trying to do a ldX.s.
734 * If we ever make it to here on an ldfX.s then
735 */
736 getreg(ld.imm, &r2, &nat_r2, regs);
737
738 ifa += r2;
739
740 /*
741 * propagate Nat r2 -> r3
742 */
743 setreg(ld.r3, ifa, nat_r2, regs);
744
745 DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2);
746 }
747}
748
749
750static int
751emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
752{
753 unsigned int len = 1 << ld.x6_sz;
754 unsigned long val = 0;
755
756 /*
757 * r0, as target, doesn't need to be checked because Illegal Instruction
758 * faults have higher priority than unaligned faults.
759 *
760 * r0 cannot be found as the base as it would never generate an
761 * unaligned reference.
762 */
763
764 /*
765 * ldX.a we will emulate load and also invalidate the ALAT entry.
766 * See comment below for explanation on how we handle ldX.a
767 */
768
769 if (len != 2 && len != 4 && len != 8) {
770 DPRINT("unknown size: x6=%d\n", ld.x6_sz);
771 return -1;
772 }
773 /* this assumes little-endian byte-order: */
774 if (copy_from_user(&val, (void __user *) ifa, len))
775 return -1;
776 setreg(ld.r1, val, 0, regs);
777
778 /*
779 * check for updates on any kind of loads
780 */
781 if (ld.op == 0x5 || ld.m)
782 emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
783
784 /*
785 * handling of various loads (based on EAS2.4):
786 *
787 * ldX.acq (ordered load):
788 * - acquire semantics would have been used, so force fence instead.
789 *
790 * ldX.c.clr (check load and clear):
791 * - if we get to this handler, it's because the entry was not in the ALAT.
792 * Therefore the operation reverts to a normal load
793 *
794 * ldX.c.nc (check load no clear):
795 * - same as previous one
796 *
797 * ldX.c.clr.acq (ordered check load and clear):
798 * - same as above for c.clr part. The load needs to have acquire semantics. So
799 * we use the fence semantics which is stronger and thus ensures correctness.
800 *
801 * ldX.a (advanced load):
802 * - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
803 * address doesn't match requested size alignment. This means that we would
804 * possibly need more than one load to get the result.
805 *
806 * The load part can be handled just like a normal load, however the difficult
807 * part is to get the right thing into the ALAT. The critical piece of information
808 * in the base address of the load & size. To do that, a ld.a must be executed,
809 * clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
810 * if we use the same target register, we will be okay for the check.a instruction.
811 * If we look at the store, basically a stX [r3]=r1 checks the ALAT for any entry
812 * which would overlap within [r3,r3+X] (the size of the load was store in the
813 * ALAT). If such an entry is found the entry is invalidated. But this is not good
814 * enough, take the following example:
815 * r3=3
816 * ld4.a r1=[r3]
817 *
818 * Could be emulated by doing:
819 * ld1.a r1=[r3],1
820 * store to temporary;
821 * ld1.a r1=[r3],1
822 * store & shift to temporary;
823 * ld1.a r1=[r3],1
824 * store & shift to temporary;
825 * ld1.a r1=[r3]
826 * store & shift to temporary;
827 * r1=temporary
828 *
829 * So in this case, you would get the right value is r1 but the wrong info in
830 * the ALAT. Notice that you could do it in reverse to finish with address 3
831 * but you would still get the size wrong. To get the size right, one needs to
832 * execute exactly the same kind of load. You could do it from a aligned
833 * temporary location, but you would get the address wrong.
834 *
835 * So no matter what, it is not possible to emulate an advanced load
836 * correctly. But is that really critical ?
837 *
838 * We will always convert ld.a into a normal load with ALAT invalidated. This
839 * will enable compiler to do optimization where certain code path after ld.a
840 * is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
841 *
842 * If there is a store after the advanced load, one must either do a ld.c.* or
843 * chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
844 * entry found in ALAT), and that's perfectly ok because:
845 *
846 * - ld.c.*, if the entry is not present a normal load is executed
847 * - chk.a.*, if the entry is not present, execution jumps to recovery code
848 *
849 * In either case, the load can be potentially retried in another form.
850 *
851 * ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
852 * up a stale entry later). The register base update MUST also be performed.
853 */
854
855 /*
856 * when the load has the .acq completer then
857 * use ordering fence.
858 */
859 if (ld.x6_op == 0x5 || ld.x6_op == 0xa)
860 mb();
861
862 /*
863 * invalidate ALAT entry in case of advanced load
864 */
865 if (ld.x6_op == 0x2)
866 invala_gr(ld.r1);
867
868 return 0;
869}
870
871static int
872emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
873{
874 unsigned long r2;
875 unsigned int len = 1 << ld.x6_sz;
876
877 /*
878 * if we get to this handler, Nat bits on both r3 and r2 have already
879 * been checked. so we don't need to do it
880 *
881 * extract the value to be stored
882 */
883 getreg(ld.imm, &r2, NULL, regs);
884
885 /*
886 * we rely on the macros in unaligned.h for now i.e.,
887 * we let the compiler figure out how to read memory gracefully.
888 *
889 * We need this switch/case because the way the inline function
890 * works. The code is optimized by the compiler and looks like
891 * a single switch/case.
892 */
893 DPRINT("st%d [%lx]=%lx\n", len, ifa, r2);
894
895 if (len != 2 && len != 4 && len != 8) {
896 DPRINT("unknown size: x6=%d\n", ld.x6_sz);
897 return -1;
898 }
899
900 /* this assumes little-endian byte-order: */
901 if (copy_to_user((void __user *) ifa, &r2, len))
902 return -1;
903
904 /*
905 * stX [r3]=r2,imm(9)
906 *
907 * NOTE:
908 * ld.r3 can never be r0, because r0 would not generate an
909 * unaligned access.
910 */
911 if (ld.op == 0x5) {
912 unsigned long imm;
913
914 /*
915 * form imm9: [12:6] contain first 7bits
916 */
917 imm = ld.x << 7 | ld.r1;
918 /*
919 * sign extend (8bits) if m set
920 */
921 if (ld.m) imm |= SIGN_EXT9;
922 /*
923 * ifa == r3 (NaT is necessarily cleared)
924 */
925 ifa += imm;
926
927 DPRINT("imm=%lx r3=%lx\n", imm, ifa);
928
929 setreg(ld.r3, ifa, 0, regs);
930 }
931 /*
932 * we don't have alat_invalidate_multiple() so we need
933 * to do the complete flush :-<<
934 */
935 ia64_invala();
936
937 /*
938 * stX.rel: use fence instead of release
939 */
940 if (ld.x6_op == 0xd)
941 mb();
942
943 return 0;
944}
945
946/*
947 * floating point operations sizes in bytes
948 */
949static const unsigned char float_fsz[4]={
950 10, /* extended precision (e) */
951 8, /* integer (8) */
952 4, /* single precision (s) */
953 8 /* double precision (d) */
954};
955
956static inline void
957mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
958{
959 ia64_ldfe(6, init);
960 ia64_stop();
961 ia64_stf_spill(final, 6);
962}
963
964static inline void
965mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
966{
967 ia64_ldf8(6, init);
968 ia64_stop();
969 ia64_stf_spill(final, 6);
970}
971
972static inline void
973mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
974{
975 ia64_ldfs(6, init);
976 ia64_stop();
977 ia64_stf_spill(final, 6);
978}
979
980static inline void
981mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
982{
983 ia64_ldfd(6, init);
984 ia64_stop();
985 ia64_stf_spill(final, 6);
986}
987
988static inline void
989float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
990{
991 ia64_ldf_fill(6, init);
992 ia64_stop();
993 ia64_stfe(final, 6);
994}
995
996static inline void
997float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
998{
999 ia64_ldf_fill(6, init);
1000 ia64_stop();
1001 ia64_stf8(final, 6);
1002}
1003
1004static inline void
1005float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
1006{
1007 ia64_ldf_fill(6, init);
1008 ia64_stop();
1009 ia64_stfs(final, 6);
1010}
1011
1012static inline void
1013float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1014{
1015 ia64_ldf_fill(6, init);
1016 ia64_stop();
1017 ia64_stfd(final, 6);
1018}
1019
1020static int
1021emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1022{
1023 struct ia64_fpreg fpr_init[2];
1024 struct ia64_fpreg fpr_final[2];
1025 unsigned long len = float_fsz[ld.x6_sz];
1026
1027 /*
1028 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1029 * higher priority than unaligned faults.
1030 *
1031 * r0 cannot be found as the base as it would never generate an unaligned
1032 * reference.
1033 */
1034
1035 /*
1036 * make sure we get clean buffers
1037 */
1038 memset(&fpr_init, 0, sizeof(fpr_init));
1039 memset(&fpr_final, 0, sizeof(fpr_final));
1040
1041 /*
1042 * ldfpX.a: we don't try to emulate anything but we must
1043 * invalidate the ALAT entry and execute updates, if any.
1044 */
1045 if (ld.x6_op != 0x2) {
1046 /*
1047 * This assumes little-endian byte-order. Note that there is no "ldfpe"
1048 * instruction:
1049 */
1050 if (copy_from_user(&fpr_init[0], (void __user *) ifa, len)
1051 || copy_from_user(&fpr_init[1], (void __user *) (ifa + len), len))
1052 return -1;
1053
1054 DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz);
1055 DDUMP("frp_init =", &fpr_init, 2*len);
1056 /*
1057 * XXX fixme
1058 * Could optimize inlines by using ldfpX & 2 spills
1059 */
1060 switch( ld.x6_sz ) {
1061 case 0:
1062 mem2float_extended(&fpr_init[0], &fpr_final[0]);
1063 mem2float_extended(&fpr_init[1], &fpr_final[1]);
1064 break;
1065 case 1:
1066 mem2float_integer(&fpr_init[0], &fpr_final[0]);
1067 mem2float_integer(&fpr_init[1], &fpr_final[1]);
1068 break;
1069 case 2:
1070 mem2float_single(&fpr_init[0], &fpr_final[0]);
1071 mem2float_single(&fpr_init[1], &fpr_final[1]);
1072 break;
1073 case 3:
1074 mem2float_double(&fpr_init[0], &fpr_final[0]);
1075 mem2float_double(&fpr_init[1], &fpr_final[1]);
1076 break;
1077 }
1078 DDUMP("fpr_final =", &fpr_final, 2*len);
1079 /*
1080 * XXX fixme
1081 *
1082 * A possible optimization would be to drop fpr_final and directly
1083 * use the storage from the saved context i.e., the actual final
1084 * destination (pt_regs, switch_stack or thread structure).
1085 */
1086 setfpreg(ld.r1, &fpr_final[0], regs);
1087 setfpreg(ld.imm, &fpr_final[1], regs);
1088 }
1089
1090 /*
1091 * Check for updates: only immediate updates are available for this
1092 * instruction.
1093 */
1094 if (ld.m) {
1095 /*
1096 * the immediate is implicit given the ldsz of the operation:
1097 * single: 8 (2x4) and for all others it's 16 (2x8)
1098 */
1099 ifa += len<<1;
1100
1101 /*
1102 * IMPORTANT:
1103 * the fact that we force the NaT of r3 to zero is ONLY valid
1104 * as long as we don't come here with a ldfpX.s.
1105 * For this reason we keep this sanity check
1106 */
1107 if (ld.x6_op == 1 || ld.x6_op == 3)
1108 printk(KERN_ERR "%s: register update on speculative load pair, error\n",
1109 __func__);
1110
1111 setreg(ld.r3, ifa, 0, regs);
1112 }
1113
1114 /*
1115 * Invalidate ALAT entries, if any, for both registers.
1116 */
1117 if (ld.x6_op == 0x2) {
1118 invala_fr(ld.r1);
1119 invala_fr(ld.imm);
1120 }
1121 return 0;
1122}
1123
1124
1125static int
1126emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1127{
1128 struct ia64_fpreg fpr_init;
1129 struct ia64_fpreg fpr_final;
1130 unsigned long len = float_fsz[ld.x6_sz];
1131
1132 /*
1133 * fr0 & fr1 don't need to be checked because Illegal Instruction
1134 * faults have higher priority than unaligned faults.
1135 *
1136 * r0 cannot be found as the base as it would never generate an
1137 * unaligned reference.
1138 */
1139
1140 /*
1141 * make sure we get clean buffers
1142 */
1143 memset(&fpr_init,0, sizeof(fpr_init));
1144 memset(&fpr_final,0, sizeof(fpr_final));
1145
1146 /*
1147 * ldfX.a we don't try to emulate anything but we must
1148 * invalidate the ALAT entry.
1149 * See comments in ldX for descriptions on how the various loads are handled.
1150 */
1151 if (ld.x6_op != 0x2) {
1152 if (copy_from_user(&fpr_init, (void __user *) ifa, len))
1153 return -1;
1154
1155 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1156 DDUMP("fpr_init =", &fpr_init, len);
1157 /*
1158 * we only do something for x6_op={0,8,9}
1159 */
1160 switch( ld.x6_sz ) {
1161 case 0:
1162 mem2float_extended(&fpr_init, &fpr_final);
1163 break;
1164 case 1:
1165 mem2float_integer(&fpr_init, &fpr_final);
1166 break;
1167 case 2:
1168 mem2float_single(&fpr_init, &fpr_final);
1169 break;
1170 case 3:
1171 mem2float_double(&fpr_init, &fpr_final);
1172 break;
1173 }
1174 DDUMP("fpr_final =", &fpr_final, len);
1175 /*
1176 * XXX fixme
1177 *
1178 * A possible optimization would be to drop fpr_final and directly
1179 * use the storage from the saved context i.e., the actual final
1180 * destination (pt_regs, switch_stack or thread structure).
1181 */
1182 setfpreg(ld.r1, &fpr_final, regs);
1183 }
1184
1185 /*
1186 * check for updates on any loads
1187 */
1188 if (ld.op == 0x7 || ld.m)
1189 emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
1190
1191 /*
1192 * invalidate ALAT entry in case of advanced floating point loads
1193 */
1194 if (ld.x6_op == 0x2)
1195 invala_fr(ld.r1);
1196
1197 return 0;
1198}
1199
1200
1201static int
1202emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1203{
1204 struct ia64_fpreg fpr_init;
1205 struct ia64_fpreg fpr_final;
1206 unsigned long len = float_fsz[ld.x6_sz];
1207
1208 /*
1209 * make sure we get clean buffers
1210 */
1211 memset(&fpr_init,0, sizeof(fpr_init));
1212 memset(&fpr_final,0, sizeof(fpr_final));
1213
1214 /*
1215 * if we get to this handler, Nat bits on both r3 and r2 have already
1216 * been checked. so we don't need to do it
1217 *
1218 * extract the value to be stored
1219 */
1220 getfpreg(ld.imm, &fpr_init, regs);
1221 /*
1222 * during this step, we extract the spilled registers from the saved
1223 * context i.e., we refill. Then we store (no spill) to temporary
1224 * aligned location
1225 */
1226 switch( ld.x6_sz ) {
1227 case 0:
1228 float2mem_extended(&fpr_init, &fpr_final);
1229 break;
1230 case 1:
1231 float2mem_integer(&fpr_init, &fpr_final);
1232 break;
1233 case 2:
1234 float2mem_single(&fpr_init, &fpr_final);
1235 break;
1236 case 3:
1237 float2mem_double(&fpr_init, &fpr_final);
1238 break;
1239 }
1240 DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1241 DDUMP("fpr_init =", &fpr_init, len);
1242 DDUMP("fpr_final =", &fpr_final, len);
1243
1244 if (copy_to_user((void __user *) ifa, &fpr_final, len))
1245 return -1;
1246
1247 /*
1248 * stfX [r3]=r2,imm(9)
1249 *
1250 * NOTE:
1251 * ld.r3 can never be r0, because r0 would not generate an
1252 * unaligned access.
1253 */
1254 if (ld.op == 0x7) {
1255 unsigned long imm;
1256
1257 /*
1258 * form imm9: [12:6] contain first 7bits
1259 */
1260 imm = ld.x << 7 | ld.r1;
1261 /*
1262 * sign extend (8bits) if m set
1263 */
1264 if (ld.m)
1265 imm |= SIGN_EXT9;
1266 /*
1267 * ifa == r3 (NaT is necessarily cleared)
1268 */
1269 ifa += imm;
1270
1271 DPRINT("imm=%lx r3=%lx\n", imm, ifa);
1272
1273 setreg(ld.r3, ifa, 0, regs);
1274 }
1275 /*
1276 * we don't have alat_invalidate_multiple() so we need
1277 * to do the complete flush :-<<
1278 */
1279 ia64_invala();
1280
1281 return 0;
1282}
1283
1284/*
1285 * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1286 * eventually fix the program. However, we don't want to do that for every access so we
1287 * pace it with jiffies.
1288 */
1289static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5);
1290
1291void
1292ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs)
1293{
1294 struct ia64_psr *ipsr = ia64_psr(regs);
1295 mm_segment_t old_fs = get_fs();
1296 unsigned long bundle[2];
1297 unsigned long opcode;
1298 struct siginfo si;
1299 const struct exception_table_entry *eh = NULL;
1300 union {
1301 unsigned long l;
1302 load_store_t insn;
1303 } u;
1304 int ret = -1;
1305
1306 if (ia64_psr(regs)->be) {
1307 /* we don't support big-endian accesses */
1308 if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0))
1309 return;
1310 goto force_sigbus;
1311 }
1312
1313 /*
1314 * Treat kernel accesses for which there is an exception handler entry the same as
1315 * user-level unaligned accesses. Otherwise, a clever program could trick this
1316 * handler into reading an arbitrary kernel addresses...
1317 */
1318 if (!user_mode(regs))
1319 eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri);
1320 if (user_mode(regs) || eh) {
1321 if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0)
1322 goto force_sigbus;
1323
1324 if (!no_unaligned_warning &&
1325 !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) &&
1326 __ratelimit(&logging_rate_limit))
1327 {
1328 char buf[200]; /* comm[] is at most 16 bytes... */
1329 size_t len;
1330
1331 len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, "
1332 "ip=0x%016lx\n\r", current->comm,
1333 task_pid_nr(current),
1334 ifa, regs->cr_iip + ipsr->ri);
1335 /*
1336 * Don't call tty_write_message() if we're in the kernel; we might
1337 * be holding locks...
1338 */
1339 if (user_mode(regs))
1340 tty_write_message(current->signal->tty, buf);
1341 buf[len-1] = '\0'; /* drop '\r' */
1342 /* watch for command names containing %s */
1343 printk(KERN_WARNING "%s", buf);
1344 } else {
1345 if (no_unaligned_warning) {
1346 printk_once(KERN_WARNING "%s(%d) encountered an "
1347 "unaligned exception which required\n"
1348 "kernel assistance, which degrades "
1349 "the performance of the application.\n"
1350 "Unaligned exception warnings have "
1351 "been disabled by the system "
1352 "administrator\n"
1353 "echo 0 > /proc/sys/kernel/ignore-"
1354 "unaligned-usertrap to re-enable\n",
1355 current->comm, task_pid_nr(current));
1356 }
1357 }
1358 } else {
1359 if (__ratelimit(&logging_rate_limit)) {
1360 printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n",
1361 ifa, regs->cr_iip + ipsr->ri);
1362 if (unaligned_dump_stack)
1363 dump_stack();
1364 }
1365 set_fs(KERNEL_DS);
1366 }
1367
1368 DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n",
1369 regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it);
1370
1371 if (__copy_from_user(bundle, (void __user *) regs->cr_iip, 16))
1372 goto failure;
1373
1374 /*
1375 * extract the instruction from the bundle given the slot number
1376 */
1377 switch (ipsr->ri) {
1378 case 0: u.l = (bundle[0] >> 5); break;
1379 case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break;
1380 case 2: u.l = (bundle[1] >> 23); break;
1381 }
1382 opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK;
1383
1384 DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d "
1385 "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm,
1386 u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op);
1387
1388 /*
1389 * IMPORTANT:
1390 * Notice that the switch statement DOES not cover all possible instructions
1391 * that DO generate unaligned references. This is made on purpose because for some
1392 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1393 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1394 * the program will get a signal and die:
1395 *
1396 * load/store:
1397 * - ldX.spill
1398 * - stX.spill
1399 * Reason: RNATs are based on addresses
1400 * - ld16
1401 * - st16
1402 * Reason: ld16 and st16 are supposed to occur in a single
1403 * memory op
1404 *
1405 * synchronization:
1406 * - cmpxchg
1407 * - fetchadd
1408 * - xchg
1409 * Reason: ATOMIC operations cannot be emulated properly using multiple
1410 * instructions.
1411 *
1412 * speculative loads:
1413 * - ldX.sZ
1414 * Reason: side effects, code must be ready to deal with failure so simpler
1415 * to let the load fail.
1416 * ---------------------------------------------------------------------------------
1417 * XXX fixme
1418 *
1419 * I would like to get rid of this switch case and do something
1420 * more elegant.
1421 */
1422 switch (opcode) {
1423 case LDS_OP:
1424 case LDSA_OP:
1425 if (u.insn.x)
1426 /* oops, really a semaphore op (cmpxchg, etc) */
1427 goto failure;
1428 /* no break */
1429 case LDS_IMM_OP:
1430 case LDSA_IMM_OP:
1431 case LDFS_OP:
1432 case LDFSA_OP:
1433 case LDFS_IMM_OP:
1434 /*
1435 * The instruction will be retried with deferred exceptions turned on, and
1436 * we should get Nat bit installed
1437 *
1438 * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1439 * are actually executed even though the operation failed. So we don't
1440 * need to take care of this.
1441 */
1442 DPRINT("forcing PSR_ED\n");
1443 regs->cr_ipsr |= IA64_PSR_ED;
1444 goto done;
1445
1446 case LD_OP:
1447 case LDA_OP:
1448 case LDBIAS_OP:
1449 case LDACQ_OP:
1450 case LDCCLR_OP:
1451 case LDCNC_OP:
1452 case LDCCLRACQ_OP:
1453 if (u.insn.x)
1454 /* oops, really a semaphore op (cmpxchg, etc) */
1455 goto failure;
1456 /* no break */
1457 case LD_IMM_OP:
1458 case LDA_IMM_OP:
1459 case LDBIAS_IMM_OP:
1460 case LDACQ_IMM_OP:
1461 case LDCCLR_IMM_OP:
1462 case LDCNC_IMM_OP:
1463 case LDCCLRACQ_IMM_OP:
1464 ret = emulate_load_int(ifa, u.insn, regs);
1465 break;
1466
1467 case ST_OP:
1468 case STREL_OP:
1469 if (u.insn.x)
1470 /* oops, really a semaphore op (cmpxchg, etc) */
1471 goto failure;
1472 /* no break */
1473 case ST_IMM_OP:
1474 case STREL_IMM_OP:
1475 ret = emulate_store_int(ifa, u.insn, regs);
1476 break;
1477
1478 case LDF_OP:
1479 case LDFA_OP:
1480 case LDFCCLR_OP:
1481 case LDFCNC_OP:
1482 if (u.insn.x)
1483 ret = emulate_load_floatpair(ifa, u.insn, regs);
1484 else
1485 ret = emulate_load_float(ifa, u.insn, regs);
1486 break;
1487
1488 case LDF_IMM_OP:
1489 case LDFA_IMM_OP:
1490 case LDFCCLR_IMM_OP:
1491 case LDFCNC_IMM_OP:
1492 ret = emulate_load_float(ifa, u.insn, regs);
1493 break;
1494
1495 case STF_OP:
1496 case STF_IMM_OP:
1497 ret = emulate_store_float(ifa, u.insn, regs);
1498 break;
1499
1500 default:
1501 goto failure;
1502 }
1503 DPRINT("ret=%d\n", ret);
1504 if (ret)
1505 goto failure;
1506
1507 if (ipsr->ri == 2)
1508 /*
1509 * given today's architecture this case is not likely to happen because a
1510 * memory access instruction (M) can never be in the last slot of a
1511 * bundle. But let's keep it for now.
1512 */
1513 regs->cr_iip += 16;
1514 ipsr->ri = (ipsr->ri + 1) & 0x3;
1515
1516 DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip);
1517 done:
1518 set_fs(old_fs); /* restore original address limit */
1519 return;
1520
1521 failure:
1522 /* something went wrong... */
1523 if (!user_mode(regs)) {
1524 if (eh) {
1525 ia64_handle_exception(regs, eh);
1526 goto done;
1527 }
1528 if (die_if_kernel("error during unaligned kernel access\n", regs, ret))
1529 return;
1530 /* NOT_REACHED */
1531 }
1532 force_sigbus:
1533 si.si_signo = SIGBUS;
1534 si.si_errno = 0;
1535 si.si_code = BUS_ADRALN;
1536 si.si_addr = (void __user *) ifa;
1537 si.si_flags = 0;
1538 si.si_isr = 0;
1539 si.si_imm = 0;
1540 force_sig_info(SIGBUS, &si, current);
1541 goto done;
1542}