Linux Audio

Check our new training course

Loading...
v5.9
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Architecture-specific unaligned trap handling.
   4 *
   5 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
   6 *	Stephane Eranian <eranian@hpl.hp.com>
   7 *	David Mosberger-Tang <davidm@hpl.hp.com>
   8 *
   9 * 2002/12/09   Fix rotating register handling (off-by-1 error, missing fr-rotation).  Fix
  10 *		get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
  11 *		stacked register returns an undefined value; it does NOT trigger a
  12 *		"rsvd register fault").
  13 * 2001/10/11	Fix unaligned access to rotating registers in s/w pipelined loops.
  14 * 2001/08/13	Correct size of extended floats (float_fsz) from 16 to 10 bytes.
  15 * 2001/01/17	Add support emulation of unaligned kernel accesses.
  16 */
  17#include <linux/jiffies.h>
  18#include <linux/kernel.h>
  19#include <linux/sched/signal.h>
  20#include <linux/tty.h>
  21#include <linux/extable.h>
  22#include <linux/ratelimit.h>
  23#include <linux/uaccess.h>
  24
  25#include <asm/intrinsics.h>
  26#include <asm/processor.h>
  27#include <asm/rse.h>
  28#include <asm/exception.h>
  29#include <asm/unaligned.h>
  30
  31extern int die_if_kernel(char *str, struct pt_regs *regs, long err);
  32
  33#undef DEBUG_UNALIGNED_TRAP
  34
  35#ifdef DEBUG_UNALIGNED_TRAP
  36# define DPRINT(a...)	do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0)
  37# define DDUMP(str,vp,len)	dump(str, vp, len)
  38
  39static void
  40dump (const char *str, void *vp, size_t len)
  41{
  42	unsigned char *cp = vp;
  43	int i;
  44
  45	printk("%s", str);
  46	for (i = 0; i < len; ++i)
  47		printk (" %02x", *cp++);
  48	printk("\n");
  49}
  50#else
  51# define DPRINT(a...)
  52# define DDUMP(str,vp,len)
  53#endif
  54
  55#define IA64_FIRST_STACKED_GR	32
  56#define IA64_FIRST_ROTATING_FR	32
  57#define SIGN_EXT9		0xffffffffffffff00ul
  58
  59/*
  60 *  sysctl settable hook which tells the kernel whether to honor the
  61 *  IA64_THREAD_UAC_NOPRINT prctl.  Because this is user settable, we want
  62 *  to allow the super user to enable/disable this for security reasons
  63 *  (i.e. don't allow attacker to fill up logs with unaligned accesses).
  64 */
  65int no_unaligned_warning;
  66int unaligned_dump_stack;
  67
  68/*
  69 * For M-unit:
  70 *
  71 *  opcode |   m  |   x6    |
  72 * --------|------|---------|
  73 * [40-37] | [36] | [35:30] |
  74 * --------|------|---------|
  75 *     4   |   1  |    6    | = 11 bits
  76 * --------------------------
  77 * However bits [31:30] are not directly useful to distinguish between
  78 * load/store so we can use [35:32] instead, which gives the following
  79 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
  80 * checking the m-bit until later in the load/store emulation.
  81 */
  82#define IA64_OPCODE_MASK	0x1ef
  83#define IA64_OPCODE_SHIFT	32
  84
  85/*
  86 * Table C-28 Integer Load/Store
  87 *
  88 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
  89 *
  90 * ld8.fill, st8.fill  MUST be aligned because the RNATs are based on
  91 * the address (bits [8:3]), so we must failed.
  92 */
  93#define LD_OP            0x080
  94#define LDS_OP           0x081
  95#define LDA_OP           0x082
  96#define LDSA_OP          0x083
  97#define LDBIAS_OP        0x084
  98#define LDACQ_OP         0x085
  99/* 0x086, 0x087 are not relevant */
 100#define LDCCLR_OP        0x088
 101#define LDCNC_OP         0x089
 102#define LDCCLRACQ_OP     0x08a
 103#define ST_OP            0x08c
 104#define STREL_OP         0x08d
 105/* 0x08e,0x8f are not relevant */
 106
 107/*
 108 * Table C-29 Integer Load +Reg
 109 *
 110 * we use the ld->m (bit [36:36]) field to determine whether or not we have
 111 * a load/store of this form.
 112 */
 113
 114/*
 115 * Table C-30 Integer Load/Store +Imm
 116 *
 117 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
 118 *
 119 * ld8.fill, st8.fill  must be aligned because the Nat register are based on
 120 * the address, so we must fail and the program must be fixed.
 121 */
 122#define LD_IMM_OP            0x0a0
 123#define LDS_IMM_OP           0x0a1
 124#define LDA_IMM_OP           0x0a2
 125#define LDSA_IMM_OP          0x0a3
 126#define LDBIAS_IMM_OP        0x0a4
 127#define LDACQ_IMM_OP         0x0a5
 128/* 0x0a6, 0xa7 are not relevant */
 129#define LDCCLR_IMM_OP        0x0a8
 130#define LDCNC_IMM_OP         0x0a9
 131#define LDCCLRACQ_IMM_OP     0x0aa
 132#define ST_IMM_OP            0x0ac
 133#define STREL_IMM_OP         0x0ad
 134/* 0x0ae,0xaf are not relevant */
 135
 136/*
 137 * Table C-32 Floating-point Load/Store
 138 */
 139#define LDF_OP           0x0c0
 140#define LDFS_OP          0x0c1
 141#define LDFA_OP          0x0c2
 142#define LDFSA_OP         0x0c3
 143/* 0x0c6 is irrelevant */
 144#define LDFCCLR_OP       0x0c8
 145#define LDFCNC_OP        0x0c9
 146/* 0x0cb is irrelevant  */
 147#define STF_OP           0x0cc
 148
 149/*
 150 * Table C-33 Floating-point Load +Reg
 151 *
 152 * we use the ld->m (bit [36:36]) field to determine whether or not we have
 153 * a load/store of this form.
 154 */
 155
 156/*
 157 * Table C-34 Floating-point Load/Store +Imm
 158 */
 159#define LDF_IMM_OP       0x0e0
 160#define LDFS_IMM_OP      0x0e1
 161#define LDFA_IMM_OP      0x0e2
 162#define LDFSA_IMM_OP     0x0e3
 163/* 0x0e6 is irrelevant */
 164#define LDFCCLR_IMM_OP   0x0e8
 165#define LDFCNC_IMM_OP    0x0e9
 166#define STF_IMM_OP       0x0ec
 167
 168typedef struct {
 169	unsigned long	 qp:6;	/* [0:5]   */
 170	unsigned long    r1:7;	/* [6:12]  */
 171	unsigned long   imm:7;	/* [13:19] */
 172	unsigned long    r3:7;	/* [20:26] */
 173	unsigned long     x:1;  /* [27:27] */
 174	unsigned long  hint:2;	/* [28:29] */
 175	unsigned long x6_sz:2;	/* [30:31] */
 176	unsigned long x6_op:4;	/* [32:35], x6 = x6_sz|x6_op */
 177	unsigned long     m:1;	/* [36:36] */
 178	unsigned long    op:4;	/* [37:40] */
 179	unsigned long   pad:23; /* [41:63] */
 180} load_store_t;
 181
 182
 183typedef enum {
 184	UPD_IMMEDIATE,	/* ldXZ r1=[r3],imm(9) */
 185	UPD_REG		/* ldXZ r1=[r3],r2     */
 186} update_t;
 187
 188/*
 189 * We use tables to keep track of the offsets of registers in the saved state.
 190 * This way we save having big switch/case statements.
 191 *
 192 * We use bit 0 to indicate switch_stack or pt_regs.
 193 * The offset is simply shifted by 1 bit.
 194 * A 2-byte value should be enough to hold any kind of offset
 195 *
 196 * In case the calling convention changes (and thus pt_regs/switch_stack)
 197 * simply use RSW instead of RPT or vice-versa.
 198 */
 199
 200#define RPO(x)	((size_t) &((struct pt_regs *)0)->x)
 201#define RSO(x)	((size_t) &((struct switch_stack *)0)->x)
 202
 203#define RPT(x)		(RPO(x) << 1)
 204#define RSW(x)		(1| RSO(x)<<1)
 205
 206#define GR_OFFS(x)	(gr_info[x]>>1)
 207#define GR_IN_SW(x)	(gr_info[x] & 0x1)
 208
 209#define FR_OFFS(x)	(fr_info[x]>>1)
 210#define FR_IN_SW(x)	(fr_info[x] & 0x1)
 211
 212static u16 gr_info[32]={
 213	0,			/* r0 is read-only : WE SHOULD NEVER GET THIS */
 214
 215	RPT(r1), RPT(r2), RPT(r3),
 216
 217	RSW(r4), RSW(r5), RSW(r6), RSW(r7),
 218
 219	RPT(r8), RPT(r9), RPT(r10), RPT(r11),
 220	RPT(r12), RPT(r13), RPT(r14), RPT(r15),
 221
 222	RPT(r16), RPT(r17), RPT(r18), RPT(r19),
 223	RPT(r20), RPT(r21), RPT(r22), RPT(r23),
 224	RPT(r24), RPT(r25), RPT(r26), RPT(r27),
 225	RPT(r28), RPT(r29), RPT(r30), RPT(r31)
 226};
 227
 228static u16 fr_info[32]={
 229	0,			/* constant : WE SHOULD NEVER GET THIS */
 230	0,			/* constant : WE SHOULD NEVER GET THIS */
 231
 232	RSW(f2), RSW(f3), RSW(f4), RSW(f5),
 233
 234	RPT(f6), RPT(f7), RPT(f8), RPT(f9),
 235	RPT(f10), RPT(f11),
 236
 237	RSW(f12), RSW(f13), RSW(f14),
 238	RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19),
 239	RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24),
 240	RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29),
 241	RSW(f30), RSW(f31)
 242};
 243
 244/* Invalidate ALAT entry for integer register REGNO.  */
 245static void
 246invala_gr (int regno)
 247{
 248#	define F(reg)	case reg: ia64_invala_gr(reg); break
 249
 250	switch (regno) {
 251		F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
 252		F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
 253		F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
 254		F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
 255		F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
 256		F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
 257		F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
 258		F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
 259		F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
 260		F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
 261		F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
 262		F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
 263		F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
 264		F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
 265		F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
 266		F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
 267	}
 268#	undef F
 269}
 270
 271/* Invalidate ALAT entry for floating-point register REGNO.  */
 272static void
 273invala_fr (int regno)
 274{
 275#	define F(reg)	case reg: ia64_invala_fr(reg); break
 276
 277	switch (regno) {
 278		F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
 279		F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
 280		F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
 281		F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
 282		F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
 283		F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
 284		F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
 285		F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
 286		F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
 287		F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
 288		F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
 289		F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
 290		F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
 291		F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
 292		F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
 293		F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
 294	}
 295#	undef F
 296}
 297
 298static inline unsigned long
 299rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg)
 300{
 301	reg += rrb;
 302	if (reg >= sor)
 303		reg -= sor;
 304	return reg;
 305}
 306
 307static void
 308set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat)
 309{
 310	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 311	unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end;
 312	unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
 313	unsigned long rnats, nat_mask;
 314	unsigned long on_kbs;
 315	long sof = (regs->cr_ifs) & 0x7f;
 316	long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
 317	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
 318	long ridx = r1 - 32;
 319
 320	if (ridx >= sof) {
 321		/* this should never happen, as the "rsvd register fault" has higher priority */
 322		DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof);
 323		return;
 324	}
 325
 326	if (ridx < sor)
 327		ridx = rotate_reg(sor, rrb_gr, ridx);
 328
 329	DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
 330	       r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
 331
 332	on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
 333	addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
 334	if (addr >= kbs) {
 335		/* the register is on the kernel backing store: easy... */
 336		rnat_addr = ia64_rse_rnat_addr(addr);
 337		if ((unsigned long) rnat_addr >= sw->ar_bspstore)
 338			rnat_addr = &sw->ar_rnat;
 339		nat_mask = 1UL << ia64_rse_slot_num(addr);
 340
 341		*addr = val;
 342		if (nat)
 343			*rnat_addr |=  nat_mask;
 344		else
 345			*rnat_addr &= ~nat_mask;
 346		return;
 347	}
 348
 349	if (!user_stack(current, regs)) {
 350		DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1);
 351		return;
 352	}
 353
 354	bspstore = (unsigned long *)regs->ar_bspstore;
 355	ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
 356	bsp     = ia64_rse_skip_regs(ubs_end, -sof);
 357	addr    = ia64_rse_skip_regs(bsp, ridx);
 358
 359	DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
 360
 361	ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
 362
 363	rnat_addr = ia64_rse_rnat_addr(addr);
 364
 365	ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
 366	DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n",
 367	       (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1);
 368
 369	nat_mask = 1UL << ia64_rse_slot_num(addr);
 370	if (nat)
 371		rnats |=  nat_mask;
 372	else
 373		rnats &= ~nat_mask;
 374	ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats);
 375
 376	DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats);
 377}
 378
 379
 380static void
 381get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat)
 382{
 383	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 384	unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore;
 385	unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
 386	unsigned long rnats, nat_mask;
 387	unsigned long on_kbs;
 388	long sof = (regs->cr_ifs) & 0x7f;
 389	long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
 390	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
 391	long ridx = r1 - 32;
 392
 393	if (ridx >= sof) {
 394		/* read of out-of-frame register returns an undefined value; 0 in our case.  */
 395		DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof);
 396		goto fail;
 397	}
 398
 399	if (ridx < sor)
 400		ridx = rotate_reg(sor, rrb_gr, ridx);
 401
 402	DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
 403	       r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
 404
 405	on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
 406	addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
 407	if (addr >= kbs) {
 408		/* the register is on the kernel backing store: easy... */
 409		*val = *addr;
 410		if (nat) {
 411			rnat_addr = ia64_rse_rnat_addr(addr);
 412			if ((unsigned long) rnat_addr >= sw->ar_bspstore)
 413				rnat_addr = &sw->ar_rnat;
 414			nat_mask = 1UL << ia64_rse_slot_num(addr);
 415			*nat = (*rnat_addr & nat_mask) != 0;
 416		}
 417		return;
 418	}
 419
 420	if (!user_stack(current, regs)) {
 421		DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1);
 422		goto fail;
 423	}
 424
 425	bspstore = (unsigned long *)regs->ar_bspstore;
 426	ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
 427	bsp     = ia64_rse_skip_regs(ubs_end, -sof);
 428	addr    = ia64_rse_skip_regs(bsp, ridx);
 429
 430	DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
 431
 432	ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
 433
 434	if (nat) {
 435		rnat_addr = ia64_rse_rnat_addr(addr);
 436		nat_mask = 1UL << ia64_rse_slot_num(addr);
 437
 438		DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats);
 439
 440		ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
 441		*nat = (rnats & nat_mask) != 0;
 442	}
 443	return;
 444
 445  fail:
 446	*val = 0;
 447	if (nat)
 448		*nat = 0;
 449	return;
 450}
 451
 452
 453static void
 454setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs)
 455{
 456	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 457	unsigned long addr;
 458	unsigned long bitmask;
 459	unsigned long *unat;
 460
 461	/*
 462	 * First takes care of stacked registers
 463	 */
 464	if (regnum >= IA64_FIRST_STACKED_GR) {
 465		set_rse_reg(regs, regnum, val, nat);
 466		return;
 467	}
 468
 469	/*
 470	 * Using r0 as a target raises a General Exception fault which has higher priority
 471	 * than the Unaligned Reference fault.
 472	 */
 473
 474	/*
 475	 * Now look at registers in [0-31] range and init correct UNAT
 476	 */
 477	if (GR_IN_SW(regnum)) {
 478		addr = (unsigned long)sw;
 479		unat = &sw->ar_unat;
 480	} else {
 481		addr = (unsigned long)regs;
 482		unat = &sw->caller_unat;
 483	}
 484	DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n",
 485	       addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum));
 486	/*
 487	 * add offset from base of struct
 488	 * and do it !
 489	 */
 490	addr += GR_OFFS(regnum);
 491
 492	*(unsigned long *)addr = val;
 493
 494	/*
 495	 * We need to clear the corresponding UNAT bit to fully emulate the load
 496	 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
 497	 */
 498	bitmask   = 1UL << (addr >> 3 & 0x3f);
 499	DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat);
 500	if (nat) {
 501		*unat |= bitmask;
 502	} else {
 503		*unat &= ~bitmask;
 504	}
 505	DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat);
 506}
 507
 508/*
 509 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
 510 * range from 32-127, result is in the range from 0-95.
 511 */
 512static inline unsigned long
 513fph_index (struct pt_regs *regs, long regnum)
 514{
 515	unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;
 516	return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));
 517}
 518
 519static void
 520setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
 521{
 522	struct switch_stack *sw = (struct switch_stack *)regs - 1;
 523	unsigned long addr;
 524
 525	/*
 526	 * From EAS-2.5: FPDisableFault has higher priority than Unaligned
 527	 * Fault. Thus, when we get here, we know the partition is enabled.
 528	 * To update f32-f127, there are three choices:
 529	 *
 530	 *	(1) save f32-f127 to thread.fph and update the values there
 531	 *	(2) use a gigantic switch statement to directly access the registers
 532	 *	(3) generate code on the fly to update the desired register
 533	 *
 534	 * For now, we are using approach (1).
 535	 */
 536	if (regnum >= IA64_FIRST_ROTATING_FR) {
 537		ia64_sync_fph(current);
 538		current->thread.fph[fph_index(regs, regnum)] = *fpval;
 539	} else {
 540		/*
 541		 * pt_regs or switch_stack ?
 542		 */
 543		if (FR_IN_SW(regnum)) {
 544			addr = (unsigned long)sw;
 545		} else {
 546			addr = (unsigned long)regs;
 547		}
 548
 549		DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum));
 550
 551		addr += FR_OFFS(regnum);
 552		*(struct ia64_fpreg *)addr = *fpval;
 553
 554		/*
 555		 * mark the low partition as being used now
 556		 *
 557		 * It is highly unlikely that this bit is not already set, but
 558		 * let's do it for safety.
 559		 */
 560		regs->cr_ipsr |= IA64_PSR_MFL;
 561	}
 562}
 563
 564/*
 565 * Those 2 inline functions generate the spilled versions of the constant floating point
 566 * registers which can be used with stfX
 567 */
 568static inline void
 569float_spill_f0 (struct ia64_fpreg *final)
 570{
 571	ia64_stf_spill(final, 0);
 572}
 573
 574static inline void
 575float_spill_f1 (struct ia64_fpreg *final)
 576{
 577	ia64_stf_spill(final, 1);
 578}
 579
 580static void
 581getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
 582{
 583	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 584	unsigned long addr;
 585
 586	/*
 587	 * From EAS-2.5: FPDisableFault has higher priority than
 588	 * Unaligned Fault. Thus, when we get here, we know the partition is
 589	 * enabled.
 590	 *
 591	 * When regnum > 31, the register is still live and we need to force a save
 592	 * to current->thread.fph to get access to it.  See discussion in setfpreg()
 593	 * for reasons and other ways of doing this.
 594	 */
 595	if (regnum >= IA64_FIRST_ROTATING_FR) {
 596		ia64_flush_fph(current);
 597		*fpval = current->thread.fph[fph_index(regs, regnum)];
 598	} else {
 599		/*
 600		 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
 601		 * not saved, we must generate their spilled form on the fly
 602		 */
 603		switch(regnum) {
 604		case 0:
 605			float_spill_f0(fpval);
 606			break;
 607		case 1:
 608			float_spill_f1(fpval);
 609			break;
 610		default:
 611			/*
 612			 * pt_regs or switch_stack ?
 613			 */
 614			addr =  FR_IN_SW(regnum) ? (unsigned long)sw
 615						 : (unsigned long)regs;
 616
 617			DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n",
 618			       FR_IN_SW(regnum), addr, FR_OFFS(regnum));
 619
 620			addr  += FR_OFFS(regnum);
 621			*fpval = *(struct ia64_fpreg *)addr;
 622		}
 623	}
 624}
 625
 626
 627static void
 628getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs)
 629{
 630	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 631	unsigned long addr, *unat;
 632
 633	if (regnum >= IA64_FIRST_STACKED_GR) {
 634		get_rse_reg(regs, regnum, val, nat);
 635		return;
 636	}
 637
 638	/*
 639	 * take care of r0 (read-only always evaluate to 0)
 640	 */
 641	if (regnum == 0) {
 642		*val = 0;
 643		if (nat)
 644			*nat = 0;
 645		return;
 646	}
 647
 648	/*
 649	 * Now look at registers in [0-31] range and init correct UNAT
 650	 */
 651	if (GR_IN_SW(regnum)) {
 652		addr = (unsigned long)sw;
 653		unat = &sw->ar_unat;
 654	} else {
 655		addr = (unsigned long)regs;
 656		unat = &sw->caller_unat;
 657	}
 658
 659	DPRINT("addr_base=%lx offset=0x%x\n", addr,  GR_OFFS(regnum));
 660
 661	addr += GR_OFFS(regnum);
 662
 663	*val  = *(unsigned long *)addr;
 664
 665	/*
 666	 * do it only when requested
 667	 */
 668	if (nat)
 669		*nat  = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL;
 670}
 671
 672static void
 673emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa)
 674{
 675	/*
 676	 * IMPORTANT:
 677	 * Given the way we handle unaligned speculative loads, we should
 678	 * not get to this point in the code but we keep this sanity check,
 679	 * just in case.
 680	 */
 681	if (ld.x6_op == 1 || ld.x6_op == 3) {
 682		printk(KERN_ERR "%s: register update on speculative load, error\n", __func__);
 683		if (die_if_kernel("unaligned reference on speculative load with register update\n",
 684				  regs, 30))
 685			return;
 686	}
 687
 688
 689	/*
 690	 * at this point, we know that the base register to update is valid i.e.,
 691	 * it's not r0
 692	 */
 693	if (type == UPD_IMMEDIATE) {
 694		unsigned long imm;
 695
 696		/*
 697		 * Load +Imm: ldXZ r1=[r3],imm(9)
 698		 *
 699		 *
 700		 * form imm9: [13:19] contain the first 7 bits
 701		 */
 702		imm = ld.x << 7 | ld.imm;
 703
 704		/*
 705		 * sign extend (1+8bits) if m set
 706		 */
 707		if (ld.m) imm |= SIGN_EXT9;
 708
 709		/*
 710		 * ifa == r3 and we know that the NaT bit on r3 was clear so
 711		 * we can directly use ifa.
 712		 */
 713		ifa += imm;
 714
 715		setreg(ld.r3, ifa, 0, regs);
 716
 717		DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa);
 718
 719	} else if (ld.m) {
 720		unsigned long r2;
 721		int nat_r2;
 722
 723		/*
 724		 * Load +Reg Opcode: ldXZ r1=[r3],r2
 725		 *
 726		 * Note: that we update r3 even in the case of ldfX.a
 727		 * (where the load does not happen)
 728		 *
 729		 * The way the load algorithm works, we know that r3 does not
 730		 * have its NaT bit set (would have gotten NaT consumption
 731		 * before getting the unaligned fault). So we can use ifa
 732		 * which equals r3 at this point.
 733		 *
 734		 * IMPORTANT:
 735		 * The above statement holds ONLY because we know that we
 736		 * never reach this code when trying to do a ldX.s.
 737		 * If we ever make it to here on an ldfX.s then
 738		 */
 739		getreg(ld.imm, &r2, &nat_r2, regs);
 740
 741		ifa += r2;
 742
 743		/*
 744		 * propagate Nat r2 -> r3
 745		 */
 746		setreg(ld.r3, ifa, nat_r2, regs);
 747
 748		DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2);
 749	}
 750}
 751
 752
 753static int
 754emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
 755{
 756	unsigned int len = 1 << ld.x6_sz;
 757	unsigned long val = 0;
 758
 759	/*
 760	 * r0, as target, doesn't need to be checked because Illegal Instruction
 761	 * faults have higher priority than unaligned faults.
 762	 *
 763	 * r0 cannot be found as the base as it would never generate an
 764	 * unaligned reference.
 765	 */
 766
 767	/*
 768	 * ldX.a we will emulate load and also invalidate the ALAT entry.
 769	 * See comment below for explanation on how we handle ldX.a
 770	 */
 771
 772	if (len != 2 && len != 4 && len != 8) {
 773		DPRINT("unknown size: x6=%d\n", ld.x6_sz);
 774		return -1;
 775	}
 776	/* this assumes little-endian byte-order: */
 777	if (copy_from_user(&val, (void __user *) ifa, len))
 778		return -1;
 779	setreg(ld.r1, val, 0, regs);
 780
 781	/*
 782	 * check for updates on any kind of loads
 783	 */
 784	if (ld.op == 0x5 || ld.m)
 785		emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
 786
 787	/*
 788	 * handling of various loads (based on EAS2.4):
 789	 *
 790	 * ldX.acq (ordered load):
 791	 *	- acquire semantics would have been used, so force fence instead.
 792	 *
 793	 * ldX.c.clr (check load and clear):
 794	 *	- if we get to this handler, it's because the entry was not in the ALAT.
 795	 *	  Therefore the operation reverts to a normal load
 796	 *
 797	 * ldX.c.nc (check load no clear):
 798	 *	- same as previous one
 799	 *
 800	 * ldX.c.clr.acq (ordered check load and clear):
 801	 *	- same as above for c.clr part. The load needs to have acquire semantics. So
 802	 *	  we use the fence semantics which is stronger and thus ensures correctness.
 803	 *
 804	 * ldX.a (advanced load):
 805	 *	- suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
 806	 *	  address doesn't match requested size alignment. This means that we would
 807	 *	  possibly need more than one load to get the result.
 808	 *
 809	 *	  The load part can be handled just like a normal load, however the difficult
 810	 *	  part is to get the right thing into the ALAT. The critical piece of information
 811	 *	  in the base address of the load & size. To do that, a ld.a must be executed,
 812	 *	  clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
 813	 *	  if we use the same target register, we will be okay for the check.a instruction.
 814	 *	  If we look at the store, basically a stX [r3]=r1 checks the ALAT  for any entry
 815	 *	  which would overlap within [r3,r3+X] (the size of the load was store in the
 816	 *	  ALAT). If such an entry is found the entry is invalidated. But this is not good
 817	 *	  enough, take the following example:
 818	 *		r3=3
 819	 *		ld4.a r1=[r3]
 820	 *
 821	 *	  Could be emulated by doing:
 822	 *		ld1.a r1=[r3],1
 823	 *		store to temporary;
 824	 *		ld1.a r1=[r3],1
 825	 *		store & shift to temporary;
 826	 *		ld1.a r1=[r3],1
 827	 *		store & shift to temporary;
 828	 *		ld1.a r1=[r3]
 829	 *		store & shift to temporary;
 830	 *		r1=temporary
 831	 *
 832	 *	  So in this case, you would get the right value is r1 but the wrong info in
 833	 *	  the ALAT.  Notice that you could do it in reverse to finish with address 3
 834	 *	  but you would still get the size wrong.  To get the size right, one needs to
 835	 *	  execute exactly the same kind of load. You could do it from a aligned
 836	 *	  temporary location, but you would get the address wrong.
 837	 *
 838	 *	  So no matter what, it is not possible to emulate an advanced load
 839	 *	  correctly. But is that really critical ?
 840	 *
 841	 *	  We will always convert ld.a into a normal load with ALAT invalidated.  This
 842	 *	  will enable compiler to do optimization where certain code path after ld.a
 843	 *	  is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
 844	 *
 845	 *	  If there is a store after the advanced load, one must either do a ld.c.* or
 846	 *	  chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
 847	 *	  entry found in ALAT), and that's perfectly ok because:
 848	 *
 849	 *		- ld.c.*, if the entry is not present a  normal load is executed
 850	 *		- chk.a.*, if the entry is not present, execution jumps to recovery code
 851	 *
 852	 *	  In either case, the load can be potentially retried in another form.
 853	 *
 854	 *	  ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
 855	 *	  up a stale entry later). The register base update MUST also be performed.
 856	 */
 857
 858	/*
 859	 * when the load has the .acq completer then
 860	 * use ordering fence.
 861	 */
 862	if (ld.x6_op == 0x5 || ld.x6_op == 0xa)
 863		mb();
 864
 865	/*
 866	 * invalidate ALAT entry in case of advanced load
 867	 */
 868	if (ld.x6_op == 0x2)
 869		invala_gr(ld.r1);
 870
 871	return 0;
 872}
 873
 874static int
 875emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
 876{
 877	unsigned long r2;
 878	unsigned int len = 1 << ld.x6_sz;
 879
 880	/*
 881	 * if we get to this handler, Nat bits on both r3 and r2 have already
 882	 * been checked. so we don't need to do it
 883	 *
 884	 * extract the value to be stored
 885	 */
 886	getreg(ld.imm, &r2, NULL, regs);
 887
 888	/*
 889	 * we rely on the macros in unaligned.h for now i.e.,
 890	 * we let the compiler figure out how to read memory gracefully.
 891	 *
 892	 * We need this switch/case because the way the inline function
 893	 * works. The code is optimized by the compiler and looks like
 894	 * a single switch/case.
 895	 */
 896	DPRINT("st%d [%lx]=%lx\n", len, ifa, r2);
 897
 898	if (len != 2 && len != 4 && len != 8) {
 899		DPRINT("unknown size: x6=%d\n", ld.x6_sz);
 900		return -1;
 901	}
 902
 903	/* this assumes little-endian byte-order: */
 904	if (copy_to_user((void __user *) ifa, &r2, len))
 905		return -1;
 906
 907	/*
 908	 * stX [r3]=r2,imm(9)
 909	 *
 910	 * NOTE:
 911	 * ld.r3 can never be r0, because r0 would not generate an
 912	 * unaligned access.
 913	 */
 914	if (ld.op == 0x5) {
 915		unsigned long imm;
 916
 917		/*
 918		 * form imm9: [12:6] contain first 7bits
 919		 */
 920		imm = ld.x << 7 | ld.r1;
 921		/*
 922		 * sign extend (8bits) if m set
 923		 */
 924		if (ld.m) imm |= SIGN_EXT9;
 925		/*
 926		 * ifa == r3 (NaT is necessarily cleared)
 927		 */
 928		ifa += imm;
 929
 930		DPRINT("imm=%lx r3=%lx\n", imm, ifa);
 931
 932		setreg(ld.r3, ifa, 0, regs);
 933	}
 934	/*
 935	 * we don't have alat_invalidate_multiple() so we need
 936	 * to do the complete flush :-<<
 937	 */
 938	ia64_invala();
 939
 940	/*
 941	 * stX.rel: use fence instead of release
 942	 */
 943	if (ld.x6_op == 0xd)
 944		mb();
 945
 946	return 0;
 947}
 948
 949/*
 950 * floating point operations sizes in bytes
 951 */
 952static const unsigned char float_fsz[4]={
 953	10, /* extended precision (e) */
 954	8,  /* integer (8)            */
 955	4,  /* single precision (s)   */
 956	8   /* double precision (d)   */
 957};
 958
 959static inline void
 960mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
 961{
 962	ia64_ldfe(6, init);
 963	ia64_stop();
 964	ia64_stf_spill(final, 6);
 965}
 966
 967static inline void
 968mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
 969{
 970	ia64_ldf8(6, init);
 971	ia64_stop();
 972	ia64_stf_spill(final, 6);
 973}
 974
 975static inline void
 976mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
 977{
 978	ia64_ldfs(6, init);
 979	ia64_stop();
 980	ia64_stf_spill(final, 6);
 981}
 982
 983static inline void
 984mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
 985{
 986	ia64_ldfd(6, init);
 987	ia64_stop();
 988	ia64_stf_spill(final, 6);
 989}
 990
 991static inline void
 992float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
 993{
 994	ia64_ldf_fill(6, init);
 995	ia64_stop();
 996	ia64_stfe(final, 6);
 997}
 998
 999static inline void
1000float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
1001{
1002	ia64_ldf_fill(6, init);
1003	ia64_stop();
1004	ia64_stf8(final, 6);
1005}
1006
1007static inline void
1008float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
1009{
1010	ia64_ldf_fill(6, init);
1011	ia64_stop();
1012	ia64_stfs(final, 6);
1013}
1014
1015static inline void
1016float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1017{
1018	ia64_ldf_fill(6, init);
1019	ia64_stop();
1020	ia64_stfd(final, 6);
1021}
1022
1023static int
1024emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1025{
1026	struct ia64_fpreg fpr_init[2];
1027	struct ia64_fpreg fpr_final[2];
1028	unsigned long len = float_fsz[ld.x6_sz];
1029
1030	/*
1031	 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1032	 * higher priority than unaligned faults.
1033	 *
1034	 * r0 cannot be found as the base as it would never generate an unaligned
1035	 * reference.
1036	 */
1037
1038	/*
1039	 * make sure we get clean buffers
1040	 */
1041	memset(&fpr_init, 0, sizeof(fpr_init));
1042	memset(&fpr_final, 0, sizeof(fpr_final));
1043
1044	/*
1045	 * ldfpX.a: we don't try to emulate anything but we must
1046	 * invalidate the ALAT entry and execute updates, if any.
1047	 */
1048	if (ld.x6_op != 0x2) {
1049		/*
1050		 * This assumes little-endian byte-order.  Note that there is no "ldfpe"
1051		 * instruction:
1052		 */
1053		if (copy_from_user(&fpr_init[0], (void __user *) ifa, len)
1054		    || copy_from_user(&fpr_init[1], (void __user *) (ifa + len), len))
1055			return -1;
1056
1057		DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz);
1058		DDUMP("frp_init =", &fpr_init, 2*len);
1059		/*
1060		 * XXX fixme
1061		 * Could optimize inlines by using ldfpX & 2 spills
1062		 */
1063		switch( ld.x6_sz ) {
1064			case 0:
1065				mem2float_extended(&fpr_init[0], &fpr_final[0]);
1066				mem2float_extended(&fpr_init[1], &fpr_final[1]);
1067				break;
1068			case 1:
1069				mem2float_integer(&fpr_init[0], &fpr_final[0]);
1070				mem2float_integer(&fpr_init[1], &fpr_final[1]);
1071				break;
1072			case 2:
1073				mem2float_single(&fpr_init[0], &fpr_final[0]);
1074				mem2float_single(&fpr_init[1], &fpr_final[1]);
1075				break;
1076			case 3:
1077				mem2float_double(&fpr_init[0], &fpr_final[0]);
1078				mem2float_double(&fpr_init[1], &fpr_final[1]);
1079				break;
1080		}
1081		DDUMP("fpr_final =", &fpr_final, 2*len);
1082		/*
1083		 * XXX fixme
1084		 *
1085		 * A possible optimization would be to drop fpr_final and directly
1086		 * use the storage from the saved context i.e., the actual final
1087		 * destination (pt_regs, switch_stack or thread structure).
1088		 */
1089		setfpreg(ld.r1, &fpr_final[0], regs);
1090		setfpreg(ld.imm, &fpr_final[1], regs);
1091	}
1092
1093	/*
1094	 * Check for updates: only immediate updates are available for this
1095	 * instruction.
1096	 */
1097	if (ld.m) {
1098		/*
1099		 * the immediate is implicit given the ldsz of the operation:
1100		 * single: 8 (2x4) and for  all others it's 16 (2x8)
1101		 */
1102		ifa += len<<1;
1103
1104		/*
1105		 * IMPORTANT:
1106		 * the fact that we force the NaT of r3 to zero is ONLY valid
1107		 * as long as we don't come here with a ldfpX.s.
1108		 * For this reason we keep this sanity check
1109		 */
1110		if (ld.x6_op == 1 || ld.x6_op == 3)
1111			printk(KERN_ERR "%s: register update on speculative load pair, error\n",
1112			       __func__);
1113
1114		setreg(ld.r3, ifa, 0, regs);
1115	}
1116
1117	/*
1118	 * Invalidate ALAT entries, if any, for both registers.
1119	 */
1120	if (ld.x6_op == 0x2) {
1121		invala_fr(ld.r1);
1122		invala_fr(ld.imm);
1123	}
1124	return 0;
1125}
1126
1127
1128static int
1129emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1130{
1131	struct ia64_fpreg fpr_init;
1132	struct ia64_fpreg fpr_final;
1133	unsigned long len = float_fsz[ld.x6_sz];
1134
1135	/*
1136	 * fr0 & fr1 don't need to be checked because Illegal Instruction
1137	 * faults have higher priority than unaligned faults.
1138	 *
1139	 * r0 cannot be found as the base as it would never generate an
1140	 * unaligned reference.
1141	 */
1142
1143	/*
1144	 * make sure we get clean buffers
1145	 */
1146	memset(&fpr_init,0, sizeof(fpr_init));
1147	memset(&fpr_final,0, sizeof(fpr_final));
1148
1149	/*
1150	 * ldfX.a we don't try to emulate anything but we must
1151	 * invalidate the ALAT entry.
1152	 * See comments in ldX for descriptions on how the various loads are handled.
1153	 */
1154	if (ld.x6_op != 0x2) {
1155		if (copy_from_user(&fpr_init, (void __user *) ifa, len))
1156			return -1;
1157
1158		DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1159		DDUMP("fpr_init =", &fpr_init, len);
1160		/*
1161		 * we only do something for x6_op={0,8,9}
1162		 */
1163		switch( ld.x6_sz ) {
1164			case 0:
1165				mem2float_extended(&fpr_init, &fpr_final);
1166				break;
1167			case 1:
1168				mem2float_integer(&fpr_init, &fpr_final);
1169				break;
1170			case 2:
1171				mem2float_single(&fpr_init, &fpr_final);
1172				break;
1173			case 3:
1174				mem2float_double(&fpr_init, &fpr_final);
1175				break;
1176		}
1177		DDUMP("fpr_final =", &fpr_final, len);
1178		/*
1179		 * XXX fixme
1180		 *
1181		 * A possible optimization would be to drop fpr_final and directly
1182		 * use the storage from the saved context i.e., the actual final
1183		 * destination (pt_regs, switch_stack or thread structure).
1184		 */
1185		setfpreg(ld.r1, &fpr_final, regs);
1186	}
1187
1188	/*
1189	 * check for updates on any loads
1190	 */
1191	if (ld.op == 0x7 || ld.m)
1192		emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
1193
1194	/*
1195	 * invalidate ALAT entry in case of advanced floating point loads
1196	 */
1197	if (ld.x6_op == 0x2)
1198		invala_fr(ld.r1);
1199
1200	return 0;
1201}
1202
1203
1204static int
1205emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1206{
1207	struct ia64_fpreg fpr_init;
1208	struct ia64_fpreg fpr_final;
1209	unsigned long len = float_fsz[ld.x6_sz];
1210
1211	/*
1212	 * make sure we get clean buffers
1213	 */
1214	memset(&fpr_init,0, sizeof(fpr_init));
1215	memset(&fpr_final,0, sizeof(fpr_final));
1216
1217	/*
1218	 * if we get to this handler, Nat bits on both r3 and r2 have already
1219	 * been checked. so we don't need to do it
1220	 *
1221	 * extract the value to be stored
1222	 */
1223	getfpreg(ld.imm, &fpr_init, regs);
1224	/*
1225	 * during this step, we extract the spilled registers from the saved
1226	 * context i.e., we refill. Then we store (no spill) to temporary
1227	 * aligned location
1228	 */
1229	switch( ld.x6_sz ) {
1230		case 0:
1231			float2mem_extended(&fpr_init, &fpr_final);
1232			break;
1233		case 1:
1234			float2mem_integer(&fpr_init, &fpr_final);
1235			break;
1236		case 2:
1237			float2mem_single(&fpr_init, &fpr_final);
1238			break;
1239		case 3:
1240			float2mem_double(&fpr_init, &fpr_final);
1241			break;
1242	}
1243	DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1244	DDUMP("fpr_init =", &fpr_init, len);
1245	DDUMP("fpr_final =", &fpr_final, len);
1246
1247	if (copy_to_user((void __user *) ifa, &fpr_final, len))
1248		return -1;
1249
1250	/*
1251	 * stfX [r3]=r2,imm(9)
1252	 *
1253	 * NOTE:
1254	 * ld.r3 can never be r0, because r0 would not generate an
1255	 * unaligned access.
1256	 */
1257	if (ld.op == 0x7) {
1258		unsigned long imm;
1259
1260		/*
1261		 * form imm9: [12:6] contain first 7bits
1262		 */
1263		imm = ld.x << 7 | ld.r1;
1264		/*
1265		 * sign extend (8bits) if m set
1266		 */
1267		if (ld.m)
1268			imm |= SIGN_EXT9;
1269		/*
1270		 * ifa == r3 (NaT is necessarily cleared)
1271		 */
1272		ifa += imm;
1273
1274		DPRINT("imm=%lx r3=%lx\n", imm, ifa);
1275
1276		setreg(ld.r3, ifa, 0, regs);
1277	}
1278	/*
1279	 * we don't have alat_invalidate_multiple() so we need
1280	 * to do the complete flush :-<<
1281	 */
1282	ia64_invala();
1283
1284	return 0;
1285}
1286
1287/*
1288 * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1289 * eventually fix the program.  However, we don't want to do that for every access so we
1290 * pace it with jiffies.
1291 */
1292static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5);
1293
1294void
1295ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs)
1296{
1297	struct ia64_psr *ipsr = ia64_psr(regs);
1298	mm_segment_t old_fs = get_fs();
1299	unsigned long bundle[2];
1300	unsigned long opcode;
 
1301	const struct exception_table_entry *eh = NULL;
1302	union {
1303		unsigned long l;
1304		load_store_t insn;
1305	} u;
1306	int ret = -1;
1307
1308	if (ia64_psr(regs)->be) {
1309		/* we don't support big-endian accesses */
1310		if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0))
1311			return;
1312		goto force_sigbus;
1313	}
1314
1315	/*
1316	 * Treat kernel accesses for which there is an exception handler entry the same as
1317	 * user-level unaligned accesses.  Otherwise, a clever program could trick this
1318	 * handler into reading an arbitrary kernel addresses...
1319	 */
1320	if (!user_mode(regs))
1321		eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri);
1322	if (user_mode(regs) || eh) {
1323		if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0)
1324			goto force_sigbus;
1325
1326		if (!no_unaligned_warning &&
1327		    !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) &&
1328		    __ratelimit(&logging_rate_limit))
1329		{
1330			char buf[200];	/* comm[] is at most 16 bytes... */
1331			size_t len;
1332
1333			len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, "
1334				      "ip=0x%016lx\n\r", current->comm,
1335				      task_pid_nr(current),
1336				      ifa, regs->cr_iip + ipsr->ri);
1337			/*
1338			 * Don't call tty_write_message() if we're in the kernel; we might
1339			 * be holding locks...
1340			 */
1341			if (user_mode(regs)) {
1342				struct tty_struct *tty = get_current_tty();
1343				tty_write_message(tty, buf);
1344				tty_kref_put(tty);
1345			}
1346			buf[len-1] = '\0';	/* drop '\r' */
1347			/* watch for command names containing %s */
1348			printk(KERN_WARNING "%s", buf);
1349		} else {
1350			if (no_unaligned_warning) {
1351				printk_once(KERN_WARNING "%s(%d) encountered an "
1352				       "unaligned exception which required\n"
1353				       "kernel assistance, which degrades "
1354				       "the performance of the application.\n"
1355				       "Unaligned exception warnings have "
1356				       "been disabled by the system "
1357				       "administrator\n"
1358				       "echo 0 > /proc/sys/kernel/ignore-"
1359				       "unaligned-usertrap to re-enable\n",
1360				       current->comm, task_pid_nr(current));
1361			}
1362		}
1363	} else {
1364		if (__ratelimit(&logging_rate_limit)) {
1365			printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n",
1366			       ifa, regs->cr_iip + ipsr->ri);
1367			if (unaligned_dump_stack)
1368				dump_stack();
1369		}
1370		set_fs(KERNEL_DS);
1371	}
1372
1373	DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n",
1374	       regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it);
1375
1376	if (__copy_from_user(bundle, (void __user *) regs->cr_iip, 16))
1377		goto failure;
1378
1379	/*
1380	 * extract the instruction from the bundle given the slot number
1381	 */
1382	switch (ipsr->ri) {
1383	      default:
1384	      case 0: u.l = (bundle[0] >>  5); break;
1385	      case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break;
1386	      case 2: u.l = (bundle[1] >> 23); break;
1387	}
1388	opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK;
1389
1390	DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d "
1391	       "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm,
1392	       u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op);
1393
1394	/*
1395	 * IMPORTANT:
1396	 * Notice that the switch statement DOES not cover all possible instructions
1397	 * that DO generate unaligned references. This is made on purpose because for some
1398	 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1399	 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1400	 * the program will get a signal and die:
1401	 *
1402	 *	load/store:
1403	 *		- ldX.spill
1404	 *		- stX.spill
1405	 *	Reason: RNATs are based on addresses
1406	 *		- ld16
1407	 *		- st16
1408	 *	Reason: ld16 and st16 are supposed to occur in a single
1409	 *		memory op
1410	 *
1411	 *	synchronization:
1412	 *		- cmpxchg
1413	 *		- fetchadd
1414	 *		- xchg
1415	 *	Reason: ATOMIC operations cannot be emulated properly using multiple
1416	 *	        instructions.
1417	 *
1418	 *	speculative loads:
1419	 *		- ldX.sZ
1420	 *	Reason: side effects, code must be ready to deal with failure so simpler
1421	 *		to let the load fail.
1422	 * ---------------------------------------------------------------------------------
1423	 * XXX fixme
1424	 *
1425	 * I would like to get rid of this switch case and do something
1426	 * more elegant.
1427	 */
1428	switch (opcode) {
1429	      case LDS_OP:
1430	      case LDSA_OP:
1431		if (u.insn.x)
1432			/* oops, really a semaphore op (cmpxchg, etc) */
1433			goto failure;
1434		fallthrough;
1435	      case LDS_IMM_OP:
1436	      case LDSA_IMM_OP:
1437	      case LDFS_OP:
1438	      case LDFSA_OP:
1439	      case LDFS_IMM_OP:
1440		/*
1441		 * The instruction will be retried with deferred exceptions turned on, and
1442		 * we should get Nat bit installed
1443		 *
1444		 * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1445		 * are actually executed even though the operation failed. So we don't
1446		 * need to take care of this.
1447		 */
1448		DPRINT("forcing PSR_ED\n");
1449		regs->cr_ipsr |= IA64_PSR_ED;
1450		goto done;
1451
1452	      case LD_OP:
1453	      case LDA_OP:
1454	      case LDBIAS_OP:
1455	      case LDACQ_OP:
1456	      case LDCCLR_OP:
1457	      case LDCNC_OP:
1458	      case LDCCLRACQ_OP:
1459		if (u.insn.x)
1460			/* oops, really a semaphore op (cmpxchg, etc) */
1461			goto failure;
1462		fallthrough;
1463	      case LD_IMM_OP:
1464	      case LDA_IMM_OP:
1465	      case LDBIAS_IMM_OP:
1466	      case LDACQ_IMM_OP:
1467	      case LDCCLR_IMM_OP:
1468	      case LDCNC_IMM_OP:
1469	      case LDCCLRACQ_IMM_OP:
1470		ret = emulate_load_int(ifa, u.insn, regs);
1471		break;
1472
1473	      case ST_OP:
1474	      case STREL_OP:
1475		if (u.insn.x)
1476			/* oops, really a semaphore op (cmpxchg, etc) */
1477			goto failure;
1478		fallthrough;
1479	      case ST_IMM_OP:
1480	      case STREL_IMM_OP:
1481		ret = emulate_store_int(ifa, u.insn, regs);
1482		break;
1483
1484	      case LDF_OP:
1485	      case LDFA_OP:
1486	      case LDFCCLR_OP:
1487	      case LDFCNC_OP:
1488		if (u.insn.x)
1489			ret = emulate_load_floatpair(ifa, u.insn, regs);
1490		else
1491			ret = emulate_load_float(ifa, u.insn, regs);
1492		break;
1493
1494	      case LDF_IMM_OP:
1495	      case LDFA_IMM_OP:
1496	      case LDFCCLR_IMM_OP:
1497	      case LDFCNC_IMM_OP:
1498		ret = emulate_load_float(ifa, u.insn, regs);
1499		break;
1500
1501	      case STF_OP:
1502	      case STF_IMM_OP:
1503		ret = emulate_store_float(ifa, u.insn, regs);
1504		break;
1505
1506	      default:
1507		goto failure;
1508	}
1509	DPRINT("ret=%d\n", ret);
1510	if (ret)
1511		goto failure;
1512
1513	if (ipsr->ri == 2)
1514		/*
1515		 * given today's architecture this case is not likely to happen because a
1516		 * memory access instruction (M) can never be in the last slot of a
1517		 * bundle. But let's keep it for now.
1518		 */
1519		regs->cr_iip += 16;
1520	ipsr->ri = (ipsr->ri + 1) & 0x3;
1521
1522	DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip);
1523  done:
1524	set_fs(old_fs);		/* restore original address limit */
1525	return;
1526
1527  failure:
1528	/* something went wrong... */
1529	if (!user_mode(regs)) {
1530		if (eh) {
1531			ia64_handle_exception(regs, eh);
1532			goto done;
1533		}
1534		if (die_if_kernel("error during unaligned kernel access\n", regs, ret))
1535			return;
1536		/* NOT_REACHED */
1537	}
1538  force_sigbus:
1539	force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) ifa,
1540			0, 0, 0);
 
 
 
 
 
 
1541	goto done;
1542}
v4.6
 
   1/*
   2 * Architecture-specific unaligned trap handling.
   3 *
   4 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
   5 *	Stephane Eranian <eranian@hpl.hp.com>
   6 *	David Mosberger-Tang <davidm@hpl.hp.com>
   7 *
   8 * 2002/12/09   Fix rotating register handling (off-by-1 error, missing fr-rotation).  Fix
   9 *		get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
  10 *		stacked register returns an undefined value; it does NOT trigger a
  11 *		"rsvd register fault").
  12 * 2001/10/11	Fix unaligned access to rotating registers in s/w pipelined loops.
  13 * 2001/08/13	Correct size of extended floats (float_fsz) from 16 to 10 bytes.
  14 * 2001/01/17	Add support emulation of unaligned kernel accesses.
  15 */
  16#include <linux/jiffies.h>
  17#include <linux/kernel.h>
  18#include <linux/sched.h>
  19#include <linux/tty.h>
 
  20#include <linux/ratelimit.h>
 
  21
  22#include <asm/intrinsics.h>
  23#include <asm/processor.h>
  24#include <asm/rse.h>
  25#include <asm/uaccess.h>
  26#include <asm/unaligned.h>
  27
  28extern int die_if_kernel(char *str, struct pt_regs *regs, long err);
  29
  30#undef DEBUG_UNALIGNED_TRAP
  31
  32#ifdef DEBUG_UNALIGNED_TRAP
  33# define DPRINT(a...)	do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0)
  34# define DDUMP(str,vp,len)	dump(str, vp, len)
  35
  36static void
  37dump (const char *str, void *vp, size_t len)
  38{
  39	unsigned char *cp = vp;
  40	int i;
  41
  42	printk("%s", str);
  43	for (i = 0; i < len; ++i)
  44		printk (" %02x", *cp++);
  45	printk("\n");
  46}
  47#else
  48# define DPRINT(a...)
  49# define DDUMP(str,vp,len)
  50#endif
  51
  52#define IA64_FIRST_STACKED_GR	32
  53#define IA64_FIRST_ROTATING_FR	32
  54#define SIGN_EXT9		0xffffffffffffff00ul
  55
  56/*
  57 *  sysctl settable hook which tells the kernel whether to honor the
  58 *  IA64_THREAD_UAC_NOPRINT prctl.  Because this is user settable, we want
  59 *  to allow the super user to enable/disable this for security reasons
  60 *  (i.e. don't allow attacker to fill up logs with unaligned accesses).
  61 */
  62int no_unaligned_warning;
  63int unaligned_dump_stack;
  64
  65/*
  66 * For M-unit:
  67 *
  68 *  opcode |   m  |   x6    |
  69 * --------|------|---------|
  70 * [40-37] | [36] | [35:30] |
  71 * --------|------|---------|
  72 *     4   |   1  |    6    | = 11 bits
  73 * --------------------------
  74 * However bits [31:30] are not directly useful to distinguish between
  75 * load/store so we can use [35:32] instead, which gives the following
  76 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
  77 * checking the m-bit until later in the load/store emulation.
  78 */
  79#define IA64_OPCODE_MASK	0x1ef
  80#define IA64_OPCODE_SHIFT	32
  81
  82/*
  83 * Table C-28 Integer Load/Store
  84 *
  85 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
  86 *
  87 * ld8.fill, st8.fill  MUST be aligned because the RNATs are based on
  88 * the address (bits [8:3]), so we must failed.
  89 */
  90#define LD_OP            0x080
  91#define LDS_OP           0x081
  92#define LDA_OP           0x082
  93#define LDSA_OP          0x083
  94#define LDBIAS_OP        0x084
  95#define LDACQ_OP         0x085
  96/* 0x086, 0x087 are not relevant */
  97#define LDCCLR_OP        0x088
  98#define LDCNC_OP         0x089
  99#define LDCCLRACQ_OP     0x08a
 100#define ST_OP            0x08c
 101#define STREL_OP         0x08d
 102/* 0x08e,0x8f are not relevant */
 103
 104/*
 105 * Table C-29 Integer Load +Reg
 106 *
 107 * we use the ld->m (bit [36:36]) field to determine whether or not we have
 108 * a load/store of this form.
 109 */
 110
 111/*
 112 * Table C-30 Integer Load/Store +Imm
 113 *
 114 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
 115 *
 116 * ld8.fill, st8.fill  must be aligned because the Nat register are based on
 117 * the address, so we must fail and the program must be fixed.
 118 */
 119#define LD_IMM_OP            0x0a0
 120#define LDS_IMM_OP           0x0a1
 121#define LDA_IMM_OP           0x0a2
 122#define LDSA_IMM_OP          0x0a3
 123#define LDBIAS_IMM_OP        0x0a4
 124#define LDACQ_IMM_OP         0x0a5
 125/* 0x0a6, 0xa7 are not relevant */
 126#define LDCCLR_IMM_OP        0x0a8
 127#define LDCNC_IMM_OP         0x0a9
 128#define LDCCLRACQ_IMM_OP     0x0aa
 129#define ST_IMM_OP            0x0ac
 130#define STREL_IMM_OP         0x0ad
 131/* 0x0ae,0xaf are not relevant */
 132
 133/*
 134 * Table C-32 Floating-point Load/Store
 135 */
 136#define LDF_OP           0x0c0
 137#define LDFS_OP          0x0c1
 138#define LDFA_OP          0x0c2
 139#define LDFSA_OP         0x0c3
 140/* 0x0c6 is irrelevant */
 141#define LDFCCLR_OP       0x0c8
 142#define LDFCNC_OP        0x0c9
 143/* 0x0cb is irrelevant  */
 144#define STF_OP           0x0cc
 145
 146/*
 147 * Table C-33 Floating-point Load +Reg
 148 *
 149 * we use the ld->m (bit [36:36]) field to determine whether or not we have
 150 * a load/store of this form.
 151 */
 152
 153/*
 154 * Table C-34 Floating-point Load/Store +Imm
 155 */
 156#define LDF_IMM_OP       0x0e0
 157#define LDFS_IMM_OP      0x0e1
 158#define LDFA_IMM_OP      0x0e2
 159#define LDFSA_IMM_OP     0x0e3
 160/* 0x0e6 is irrelevant */
 161#define LDFCCLR_IMM_OP   0x0e8
 162#define LDFCNC_IMM_OP    0x0e9
 163#define STF_IMM_OP       0x0ec
 164
 165typedef struct {
 166	unsigned long	 qp:6;	/* [0:5]   */
 167	unsigned long    r1:7;	/* [6:12]  */
 168	unsigned long   imm:7;	/* [13:19] */
 169	unsigned long    r3:7;	/* [20:26] */
 170	unsigned long     x:1;  /* [27:27] */
 171	unsigned long  hint:2;	/* [28:29] */
 172	unsigned long x6_sz:2;	/* [30:31] */
 173	unsigned long x6_op:4;	/* [32:35], x6 = x6_sz|x6_op */
 174	unsigned long     m:1;	/* [36:36] */
 175	unsigned long    op:4;	/* [37:40] */
 176	unsigned long   pad:23; /* [41:63] */
 177} load_store_t;
 178
 179
 180typedef enum {
 181	UPD_IMMEDIATE,	/* ldXZ r1=[r3],imm(9) */
 182	UPD_REG		/* ldXZ r1=[r3],r2     */
 183} update_t;
 184
 185/*
 186 * We use tables to keep track of the offsets of registers in the saved state.
 187 * This way we save having big switch/case statements.
 188 *
 189 * We use bit 0 to indicate switch_stack or pt_regs.
 190 * The offset is simply shifted by 1 bit.
 191 * A 2-byte value should be enough to hold any kind of offset
 192 *
 193 * In case the calling convention changes (and thus pt_regs/switch_stack)
 194 * simply use RSW instead of RPT or vice-versa.
 195 */
 196
 197#define RPO(x)	((size_t) &((struct pt_regs *)0)->x)
 198#define RSO(x)	((size_t) &((struct switch_stack *)0)->x)
 199
 200#define RPT(x)		(RPO(x) << 1)
 201#define RSW(x)		(1| RSO(x)<<1)
 202
 203#define GR_OFFS(x)	(gr_info[x]>>1)
 204#define GR_IN_SW(x)	(gr_info[x] & 0x1)
 205
 206#define FR_OFFS(x)	(fr_info[x]>>1)
 207#define FR_IN_SW(x)	(fr_info[x] & 0x1)
 208
 209static u16 gr_info[32]={
 210	0,			/* r0 is read-only : WE SHOULD NEVER GET THIS */
 211
 212	RPT(r1), RPT(r2), RPT(r3),
 213
 214	RSW(r4), RSW(r5), RSW(r6), RSW(r7),
 215
 216	RPT(r8), RPT(r9), RPT(r10), RPT(r11),
 217	RPT(r12), RPT(r13), RPT(r14), RPT(r15),
 218
 219	RPT(r16), RPT(r17), RPT(r18), RPT(r19),
 220	RPT(r20), RPT(r21), RPT(r22), RPT(r23),
 221	RPT(r24), RPT(r25), RPT(r26), RPT(r27),
 222	RPT(r28), RPT(r29), RPT(r30), RPT(r31)
 223};
 224
 225static u16 fr_info[32]={
 226	0,			/* constant : WE SHOULD NEVER GET THIS */
 227	0,			/* constant : WE SHOULD NEVER GET THIS */
 228
 229	RSW(f2), RSW(f3), RSW(f4), RSW(f5),
 230
 231	RPT(f6), RPT(f7), RPT(f8), RPT(f9),
 232	RPT(f10), RPT(f11),
 233
 234	RSW(f12), RSW(f13), RSW(f14),
 235	RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19),
 236	RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24),
 237	RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29),
 238	RSW(f30), RSW(f31)
 239};
 240
 241/* Invalidate ALAT entry for integer register REGNO.  */
 242static void
 243invala_gr (int regno)
 244{
 245#	define F(reg)	case reg: ia64_invala_gr(reg); break
 246
 247	switch (regno) {
 248		F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
 249		F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
 250		F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
 251		F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
 252		F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
 253		F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
 254		F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
 255		F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
 256		F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
 257		F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
 258		F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
 259		F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
 260		F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
 261		F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
 262		F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
 263		F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
 264	}
 265#	undef F
 266}
 267
 268/* Invalidate ALAT entry for floating-point register REGNO.  */
 269static void
 270invala_fr (int regno)
 271{
 272#	define F(reg)	case reg: ia64_invala_fr(reg); break
 273
 274	switch (regno) {
 275		F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
 276		F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
 277		F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
 278		F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
 279		F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
 280		F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
 281		F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
 282		F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
 283		F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
 284		F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
 285		F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
 286		F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
 287		F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
 288		F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
 289		F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
 290		F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
 291	}
 292#	undef F
 293}
 294
 295static inline unsigned long
 296rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg)
 297{
 298	reg += rrb;
 299	if (reg >= sor)
 300		reg -= sor;
 301	return reg;
 302}
 303
 304static void
 305set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat)
 306{
 307	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 308	unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end;
 309	unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
 310	unsigned long rnats, nat_mask;
 311	unsigned long on_kbs;
 312	long sof = (regs->cr_ifs) & 0x7f;
 313	long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
 314	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
 315	long ridx = r1 - 32;
 316
 317	if (ridx >= sof) {
 318		/* this should never happen, as the "rsvd register fault" has higher priority */
 319		DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof);
 320		return;
 321	}
 322
 323	if (ridx < sor)
 324		ridx = rotate_reg(sor, rrb_gr, ridx);
 325
 326	DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
 327	       r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
 328
 329	on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
 330	addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
 331	if (addr >= kbs) {
 332		/* the register is on the kernel backing store: easy... */
 333		rnat_addr = ia64_rse_rnat_addr(addr);
 334		if ((unsigned long) rnat_addr >= sw->ar_bspstore)
 335			rnat_addr = &sw->ar_rnat;
 336		nat_mask = 1UL << ia64_rse_slot_num(addr);
 337
 338		*addr = val;
 339		if (nat)
 340			*rnat_addr |=  nat_mask;
 341		else
 342			*rnat_addr &= ~nat_mask;
 343		return;
 344	}
 345
 346	if (!user_stack(current, regs)) {
 347		DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1);
 348		return;
 349	}
 350
 351	bspstore = (unsigned long *)regs->ar_bspstore;
 352	ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
 353	bsp     = ia64_rse_skip_regs(ubs_end, -sof);
 354	addr    = ia64_rse_skip_regs(bsp, ridx);
 355
 356	DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
 357
 358	ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
 359
 360	rnat_addr = ia64_rse_rnat_addr(addr);
 361
 362	ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
 363	DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n",
 364	       (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1);
 365
 366	nat_mask = 1UL << ia64_rse_slot_num(addr);
 367	if (nat)
 368		rnats |=  nat_mask;
 369	else
 370		rnats &= ~nat_mask;
 371	ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats);
 372
 373	DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats);
 374}
 375
 376
 377static void
 378get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat)
 379{
 380	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 381	unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore;
 382	unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
 383	unsigned long rnats, nat_mask;
 384	unsigned long on_kbs;
 385	long sof = (regs->cr_ifs) & 0x7f;
 386	long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
 387	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
 388	long ridx = r1 - 32;
 389
 390	if (ridx >= sof) {
 391		/* read of out-of-frame register returns an undefined value; 0 in our case.  */
 392		DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof);
 393		goto fail;
 394	}
 395
 396	if (ridx < sor)
 397		ridx = rotate_reg(sor, rrb_gr, ridx);
 398
 399	DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
 400	       r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
 401
 402	on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
 403	addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
 404	if (addr >= kbs) {
 405		/* the register is on the kernel backing store: easy... */
 406		*val = *addr;
 407		if (nat) {
 408			rnat_addr = ia64_rse_rnat_addr(addr);
 409			if ((unsigned long) rnat_addr >= sw->ar_bspstore)
 410				rnat_addr = &sw->ar_rnat;
 411			nat_mask = 1UL << ia64_rse_slot_num(addr);
 412			*nat = (*rnat_addr & nat_mask) != 0;
 413		}
 414		return;
 415	}
 416
 417	if (!user_stack(current, regs)) {
 418		DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1);
 419		goto fail;
 420	}
 421
 422	bspstore = (unsigned long *)regs->ar_bspstore;
 423	ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
 424	bsp     = ia64_rse_skip_regs(ubs_end, -sof);
 425	addr    = ia64_rse_skip_regs(bsp, ridx);
 426
 427	DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
 428
 429	ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
 430
 431	if (nat) {
 432		rnat_addr = ia64_rse_rnat_addr(addr);
 433		nat_mask = 1UL << ia64_rse_slot_num(addr);
 434
 435		DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats);
 436
 437		ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
 438		*nat = (rnats & nat_mask) != 0;
 439	}
 440	return;
 441
 442  fail:
 443	*val = 0;
 444	if (nat)
 445		*nat = 0;
 446	return;
 447}
 448
 449
 450static void
 451setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs)
 452{
 453	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 454	unsigned long addr;
 455	unsigned long bitmask;
 456	unsigned long *unat;
 457
 458	/*
 459	 * First takes care of stacked registers
 460	 */
 461	if (regnum >= IA64_FIRST_STACKED_GR) {
 462		set_rse_reg(regs, regnum, val, nat);
 463		return;
 464	}
 465
 466	/*
 467	 * Using r0 as a target raises a General Exception fault which has higher priority
 468	 * than the Unaligned Reference fault.
 469	 */
 470
 471	/*
 472	 * Now look at registers in [0-31] range and init correct UNAT
 473	 */
 474	if (GR_IN_SW(regnum)) {
 475		addr = (unsigned long)sw;
 476		unat = &sw->ar_unat;
 477	} else {
 478		addr = (unsigned long)regs;
 479		unat = &sw->caller_unat;
 480	}
 481	DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n",
 482	       addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum));
 483	/*
 484	 * add offset from base of struct
 485	 * and do it !
 486	 */
 487	addr += GR_OFFS(regnum);
 488
 489	*(unsigned long *)addr = val;
 490
 491	/*
 492	 * We need to clear the corresponding UNAT bit to fully emulate the load
 493	 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
 494	 */
 495	bitmask   = 1UL << (addr >> 3 & 0x3f);
 496	DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat);
 497	if (nat) {
 498		*unat |= bitmask;
 499	} else {
 500		*unat &= ~bitmask;
 501	}
 502	DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat);
 503}
 504
 505/*
 506 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
 507 * range from 32-127, result is in the range from 0-95.
 508 */
 509static inline unsigned long
 510fph_index (struct pt_regs *regs, long regnum)
 511{
 512	unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;
 513	return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));
 514}
 515
 516static void
 517setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
 518{
 519	struct switch_stack *sw = (struct switch_stack *)regs - 1;
 520	unsigned long addr;
 521
 522	/*
 523	 * From EAS-2.5: FPDisableFault has higher priority than Unaligned
 524	 * Fault. Thus, when we get here, we know the partition is enabled.
 525	 * To update f32-f127, there are three choices:
 526	 *
 527	 *	(1) save f32-f127 to thread.fph and update the values there
 528	 *	(2) use a gigantic switch statement to directly access the registers
 529	 *	(3) generate code on the fly to update the desired register
 530	 *
 531	 * For now, we are using approach (1).
 532	 */
 533	if (regnum >= IA64_FIRST_ROTATING_FR) {
 534		ia64_sync_fph(current);
 535		current->thread.fph[fph_index(regs, regnum)] = *fpval;
 536	} else {
 537		/*
 538		 * pt_regs or switch_stack ?
 539		 */
 540		if (FR_IN_SW(regnum)) {
 541			addr = (unsigned long)sw;
 542		} else {
 543			addr = (unsigned long)regs;
 544		}
 545
 546		DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum));
 547
 548		addr += FR_OFFS(regnum);
 549		*(struct ia64_fpreg *)addr = *fpval;
 550
 551		/*
 552		 * mark the low partition as being used now
 553		 *
 554		 * It is highly unlikely that this bit is not already set, but
 555		 * let's do it for safety.
 556		 */
 557		regs->cr_ipsr |= IA64_PSR_MFL;
 558	}
 559}
 560
 561/*
 562 * Those 2 inline functions generate the spilled versions of the constant floating point
 563 * registers which can be used with stfX
 564 */
 565static inline void
 566float_spill_f0 (struct ia64_fpreg *final)
 567{
 568	ia64_stf_spill(final, 0);
 569}
 570
 571static inline void
 572float_spill_f1 (struct ia64_fpreg *final)
 573{
 574	ia64_stf_spill(final, 1);
 575}
 576
 577static void
 578getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
 579{
 580	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 581	unsigned long addr;
 582
 583	/*
 584	 * From EAS-2.5: FPDisableFault has higher priority than
 585	 * Unaligned Fault. Thus, when we get here, we know the partition is
 586	 * enabled.
 587	 *
 588	 * When regnum > 31, the register is still live and we need to force a save
 589	 * to current->thread.fph to get access to it.  See discussion in setfpreg()
 590	 * for reasons and other ways of doing this.
 591	 */
 592	if (regnum >= IA64_FIRST_ROTATING_FR) {
 593		ia64_flush_fph(current);
 594		*fpval = current->thread.fph[fph_index(regs, regnum)];
 595	} else {
 596		/*
 597		 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
 598		 * not saved, we must generate their spilled form on the fly
 599		 */
 600		switch(regnum) {
 601		case 0:
 602			float_spill_f0(fpval);
 603			break;
 604		case 1:
 605			float_spill_f1(fpval);
 606			break;
 607		default:
 608			/*
 609			 * pt_regs or switch_stack ?
 610			 */
 611			addr =  FR_IN_SW(regnum) ? (unsigned long)sw
 612						 : (unsigned long)regs;
 613
 614			DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n",
 615			       FR_IN_SW(regnum), addr, FR_OFFS(regnum));
 616
 617			addr  += FR_OFFS(regnum);
 618			*fpval = *(struct ia64_fpreg *)addr;
 619		}
 620	}
 621}
 622
 623
 624static void
 625getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs)
 626{
 627	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 628	unsigned long addr, *unat;
 629
 630	if (regnum >= IA64_FIRST_STACKED_GR) {
 631		get_rse_reg(regs, regnum, val, nat);
 632		return;
 633	}
 634
 635	/*
 636	 * take care of r0 (read-only always evaluate to 0)
 637	 */
 638	if (regnum == 0) {
 639		*val = 0;
 640		if (nat)
 641			*nat = 0;
 642		return;
 643	}
 644
 645	/*
 646	 * Now look at registers in [0-31] range and init correct UNAT
 647	 */
 648	if (GR_IN_SW(regnum)) {
 649		addr = (unsigned long)sw;
 650		unat = &sw->ar_unat;
 651	} else {
 652		addr = (unsigned long)regs;
 653		unat = &sw->caller_unat;
 654	}
 655
 656	DPRINT("addr_base=%lx offset=0x%x\n", addr,  GR_OFFS(regnum));
 657
 658	addr += GR_OFFS(regnum);
 659
 660	*val  = *(unsigned long *)addr;
 661
 662	/*
 663	 * do it only when requested
 664	 */
 665	if (nat)
 666		*nat  = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL;
 667}
 668
 669static void
 670emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa)
 671{
 672	/*
 673	 * IMPORTANT:
 674	 * Given the way we handle unaligned speculative loads, we should
 675	 * not get to this point in the code but we keep this sanity check,
 676	 * just in case.
 677	 */
 678	if (ld.x6_op == 1 || ld.x6_op == 3) {
 679		printk(KERN_ERR "%s: register update on speculative load, error\n", __func__);
 680		if (die_if_kernel("unaligned reference on speculative load with register update\n",
 681				  regs, 30))
 682			return;
 683	}
 684
 685
 686	/*
 687	 * at this point, we know that the base register to update is valid i.e.,
 688	 * it's not r0
 689	 */
 690	if (type == UPD_IMMEDIATE) {
 691		unsigned long imm;
 692
 693		/*
 694		 * Load +Imm: ldXZ r1=[r3],imm(9)
 695		 *
 696		 *
 697		 * form imm9: [13:19] contain the first 7 bits
 698		 */
 699		imm = ld.x << 7 | ld.imm;
 700
 701		/*
 702		 * sign extend (1+8bits) if m set
 703		 */
 704		if (ld.m) imm |= SIGN_EXT9;
 705
 706		/*
 707		 * ifa == r3 and we know that the NaT bit on r3 was clear so
 708		 * we can directly use ifa.
 709		 */
 710		ifa += imm;
 711
 712		setreg(ld.r3, ifa, 0, regs);
 713
 714		DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa);
 715
 716	} else if (ld.m) {
 717		unsigned long r2;
 718		int nat_r2;
 719
 720		/*
 721		 * Load +Reg Opcode: ldXZ r1=[r3],r2
 722		 *
 723		 * Note: that we update r3 even in the case of ldfX.a
 724		 * (where the load does not happen)
 725		 *
 726		 * The way the load algorithm works, we know that r3 does not
 727		 * have its NaT bit set (would have gotten NaT consumption
 728		 * before getting the unaligned fault). So we can use ifa
 729		 * which equals r3 at this point.
 730		 *
 731		 * IMPORTANT:
 732		 * The above statement holds ONLY because we know that we
 733		 * never reach this code when trying to do a ldX.s.
 734		 * If we ever make it to here on an ldfX.s then
 735		 */
 736		getreg(ld.imm, &r2, &nat_r2, regs);
 737
 738		ifa += r2;
 739
 740		/*
 741		 * propagate Nat r2 -> r3
 742		 */
 743		setreg(ld.r3, ifa, nat_r2, regs);
 744
 745		DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2);
 746	}
 747}
 748
 749
 750static int
 751emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
 752{
 753	unsigned int len = 1 << ld.x6_sz;
 754	unsigned long val = 0;
 755
 756	/*
 757	 * r0, as target, doesn't need to be checked because Illegal Instruction
 758	 * faults have higher priority than unaligned faults.
 759	 *
 760	 * r0 cannot be found as the base as it would never generate an
 761	 * unaligned reference.
 762	 */
 763
 764	/*
 765	 * ldX.a we will emulate load and also invalidate the ALAT entry.
 766	 * See comment below for explanation on how we handle ldX.a
 767	 */
 768
 769	if (len != 2 && len != 4 && len != 8) {
 770		DPRINT("unknown size: x6=%d\n", ld.x6_sz);
 771		return -1;
 772	}
 773	/* this assumes little-endian byte-order: */
 774	if (copy_from_user(&val, (void __user *) ifa, len))
 775		return -1;
 776	setreg(ld.r1, val, 0, regs);
 777
 778	/*
 779	 * check for updates on any kind of loads
 780	 */
 781	if (ld.op == 0x5 || ld.m)
 782		emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
 783
 784	/*
 785	 * handling of various loads (based on EAS2.4):
 786	 *
 787	 * ldX.acq (ordered load):
 788	 *	- acquire semantics would have been used, so force fence instead.
 789	 *
 790	 * ldX.c.clr (check load and clear):
 791	 *	- if we get to this handler, it's because the entry was not in the ALAT.
 792	 *	  Therefore the operation reverts to a normal load
 793	 *
 794	 * ldX.c.nc (check load no clear):
 795	 *	- same as previous one
 796	 *
 797	 * ldX.c.clr.acq (ordered check load and clear):
 798	 *	- same as above for c.clr part. The load needs to have acquire semantics. So
 799	 *	  we use the fence semantics which is stronger and thus ensures correctness.
 800	 *
 801	 * ldX.a (advanced load):
 802	 *	- suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
 803	 *	  address doesn't match requested size alignment. This means that we would
 804	 *	  possibly need more than one load to get the result.
 805	 *
 806	 *	  The load part can be handled just like a normal load, however the difficult
 807	 *	  part is to get the right thing into the ALAT. The critical piece of information
 808	 *	  in the base address of the load & size. To do that, a ld.a must be executed,
 809	 *	  clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
 810	 *	  if we use the same target register, we will be okay for the check.a instruction.
 811	 *	  If we look at the store, basically a stX [r3]=r1 checks the ALAT  for any entry
 812	 *	  which would overlap within [r3,r3+X] (the size of the load was store in the
 813	 *	  ALAT). If such an entry is found the entry is invalidated. But this is not good
 814	 *	  enough, take the following example:
 815	 *		r3=3
 816	 *		ld4.a r1=[r3]
 817	 *
 818	 *	  Could be emulated by doing:
 819	 *		ld1.a r1=[r3],1
 820	 *		store to temporary;
 821	 *		ld1.a r1=[r3],1
 822	 *		store & shift to temporary;
 823	 *		ld1.a r1=[r3],1
 824	 *		store & shift to temporary;
 825	 *		ld1.a r1=[r3]
 826	 *		store & shift to temporary;
 827	 *		r1=temporary
 828	 *
 829	 *	  So in this case, you would get the right value is r1 but the wrong info in
 830	 *	  the ALAT.  Notice that you could do it in reverse to finish with address 3
 831	 *	  but you would still get the size wrong.  To get the size right, one needs to
 832	 *	  execute exactly the same kind of load. You could do it from a aligned
 833	 *	  temporary location, but you would get the address wrong.
 834	 *
 835	 *	  So no matter what, it is not possible to emulate an advanced load
 836	 *	  correctly. But is that really critical ?
 837	 *
 838	 *	  We will always convert ld.a into a normal load with ALAT invalidated.  This
 839	 *	  will enable compiler to do optimization where certain code path after ld.a
 840	 *	  is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
 841	 *
 842	 *	  If there is a store after the advanced load, one must either do a ld.c.* or
 843	 *	  chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
 844	 *	  entry found in ALAT), and that's perfectly ok because:
 845	 *
 846	 *		- ld.c.*, if the entry is not present a  normal load is executed
 847	 *		- chk.a.*, if the entry is not present, execution jumps to recovery code
 848	 *
 849	 *	  In either case, the load can be potentially retried in another form.
 850	 *
 851	 *	  ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
 852	 *	  up a stale entry later). The register base update MUST also be performed.
 853	 */
 854
 855	/*
 856	 * when the load has the .acq completer then
 857	 * use ordering fence.
 858	 */
 859	if (ld.x6_op == 0x5 || ld.x6_op == 0xa)
 860		mb();
 861
 862	/*
 863	 * invalidate ALAT entry in case of advanced load
 864	 */
 865	if (ld.x6_op == 0x2)
 866		invala_gr(ld.r1);
 867
 868	return 0;
 869}
 870
 871static int
 872emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
 873{
 874	unsigned long r2;
 875	unsigned int len = 1 << ld.x6_sz;
 876
 877	/*
 878	 * if we get to this handler, Nat bits on both r3 and r2 have already
 879	 * been checked. so we don't need to do it
 880	 *
 881	 * extract the value to be stored
 882	 */
 883	getreg(ld.imm, &r2, NULL, regs);
 884
 885	/*
 886	 * we rely on the macros in unaligned.h for now i.e.,
 887	 * we let the compiler figure out how to read memory gracefully.
 888	 *
 889	 * We need this switch/case because the way the inline function
 890	 * works. The code is optimized by the compiler and looks like
 891	 * a single switch/case.
 892	 */
 893	DPRINT("st%d [%lx]=%lx\n", len, ifa, r2);
 894
 895	if (len != 2 && len != 4 && len != 8) {
 896		DPRINT("unknown size: x6=%d\n", ld.x6_sz);
 897		return -1;
 898	}
 899
 900	/* this assumes little-endian byte-order: */
 901	if (copy_to_user((void __user *) ifa, &r2, len))
 902		return -1;
 903
 904	/*
 905	 * stX [r3]=r2,imm(9)
 906	 *
 907	 * NOTE:
 908	 * ld.r3 can never be r0, because r0 would not generate an
 909	 * unaligned access.
 910	 */
 911	if (ld.op == 0x5) {
 912		unsigned long imm;
 913
 914		/*
 915		 * form imm9: [12:6] contain first 7bits
 916		 */
 917		imm = ld.x << 7 | ld.r1;
 918		/*
 919		 * sign extend (8bits) if m set
 920		 */
 921		if (ld.m) imm |= SIGN_EXT9;
 922		/*
 923		 * ifa == r3 (NaT is necessarily cleared)
 924		 */
 925		ifa += imm;
 926
 927		DPRINT("imm=%lx r3=%lx\n", imm, ifa);
 928
 929		setreg(ld.r3, ifa, 0, regs);
 930	}
 931	/*
 932	 * we don't have alat_invalidate_multiple() so we need
 933	 * to do the complete flush :-<<
 934	 */
 935	ia64_invala();
 936
 937	/*
 938	 * stX.rel: use fence instead of release
 939	 */
 940	if (ld.x6_op == 0xd)
 941		mb();
 942
 943	return 0;
 944}
 945
 946/*
 947 * floating point operations sizes in bytes
 948 */
 949static const unsigned char float_fsz[4]={
 950	10, /* extended precision (e) */
 951	8,  /* integer (8)            */
 952	4,  /* single precision (s)   */
 953	8   /* double precision (d)   */
 954};
 955
 956static inline void
 957mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
 958{
 959	ia64_ldfe(6, init);
 960	ia64_stop();
 961	ia64_stf_spill(final, 6);
 962}
 963
 964static inline void
 965mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
 966{
 967	ia64_ldf8(6, init);
 968	ia64_stop();
 969	ia64_stf_spill(final, 6);
 970}
 971
 972static inline void
 973mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
 974{
 975	ia64_ldfs(6, init);
 976	ia64_stop();
 977	ia64_stf_spill(final, 6);
 978}
 979
 980static inline void
 981mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
 982{
 983	ia64_ldfd(6, init);
 984	ia64_stop();
 985	ia64_stf_spill(final, 6);
 986}
 987
 988static inline void
 989float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
 990{
 991	ia64_ldf_fill(6, init);
 992	ia64_stop();
 993	ia64_stfe(final, 6);
 994}
 995
 996static inline void
 997float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
 998{
 999	ia64_ldf_fill(6, init);
1000	ia64_stop();
1001	ia64_stf8(final, 6);
1002}
1003
1004static inline void
1005float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
1006{
1007	ia64_ldf_fill(6, init);
1008	ia64_stop();
1009	ia64_stfs(final, 6);
1010}
1011
1012static inline void
1013float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1014{
1015	ia64_ldf_fill(6, init);
1016	ia64_stop();
1017	ia64_stfd(final, 6);
1018}
1019
1020static int
1021emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1022{
1023	struct ia64_fpreg fpr_init[2];
1024	struct ia64_fpreg fpr_final[2];
1025	unsigned long len = float_fsz[ld.x6_sz];
1026
1027	/*
1028	 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1029	 * higher priority than unaligned faults.
1030	 *
1031	 * r0 cannot be found as the base as it would never generate an unaligned
1032	 * reference.
1033	 */
1034
1035	/*
1036	 * make sure we get clean buffers
1037	 */
1038	memset(&fpr_init, 0, sizeof(fpr_init));
1039	memset(&fpr_final, 0, sizeof(fpr_final));
1040
1041	/*
1042	 * ldfpX.a: we don't try to emulate anything but we must
1043	 * invalidate the ALAT entry and execute updates, if any.
1044	 */
1045	if (ld.x6_op != 0x2) {
1046		/*
1047		 * This assumes little-endian byte-order.  Note that there is no "ldfpe"
1048		 * instruction:
1049		 */
1050		if (copy_from_user(&fpr_init[0], (void __user *) ifa, len)
1051		    || copy_from_user(&fpr_init[1], (void __user *) (ifa + len), len))
1052			return -1;
1053
1054		DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz);
1055		DDUMP("frp_init =", &fpr_init, 2*len);
1056		/*
1057		 * XXX fixme
1058		 * Could optimize inlines by using ldfpX & 2 spills
1059		 */
1060		switch( ld.x6_sz ) {
1061			case 0:
1062				mem2float_extended(&fpr_init[0], &fpr_final[0]);
1063				mem2float_extended(&fpr_init[1], &fpr_final[1]);
1064				break;
1065			case 1:
1066				mem2float_integer(&fpr_init[0], &fpr_final[0]);
1067				mem2float_integer(&fpr_init[1], &fpr_final[1]);
1068				break;
1069			case 2:
1070				mem2float_single(&fpr_init[0], &fpr_final[0]);
1071				mem2float_single(&fpr_init[1], &fpr_final[1]);
1072				break;
1073			case 3:
1074				mem2float_double(&fpr_init[0], &fpr_final[0]);
1075				mem2float_double(&fpr_init[1], &fpr_final[1]);
1076				break;
1077		}
1078		DDUMP("fpr_final =", &fpr_final, 2*len);
1079		/*
1080		 * XXX fixme
1081		 *
1082		 * A possible optimization would be to drop fpr_final and directly
1083		 * use the storage from the saved context i.e., the actual final
1084		 * destination (pt_regs, switch_stack or thread structure).
1085		 */
1086		setfpreg(ld.r1, &fpr_final[0], regs);
1087		setfpreg(ld.imm, &fpr_final[1], regs);
1088	}
1089
1090	/*
1091	 * Check for updates: only immediate updates are available for this
1092	 * instruction.
1093	 */
1094	if (ld.m) {
1095		/*
1096		 * the immediate is implicit given the ldsz of the operation:
1097		 * single: 8 (2x4) and for  all others it's 16 (2x8)
1098		 */
1099		ifa += len<<1;
1100
1101		/*
1102		 * IMPORTANT:
1103		 * the fact that we force the NaT of r3 to zero is ONLY valid
1104		 * as long as we don't come here with a ldfpX.s.
1105		 * For this reason we keep this sanity check
1106		 */
1107		if (ld.x6_op == 1 || ld.x6_op == 3)
1108			printk(KERN_ERR "%s: register update on speculative load pair, error\n",
1109			       __func__);
1110
1111		setreg(ld.r3, ifa, 0, regs);
1112	}
1113
1114	/*
1115	 * Invalidate ALAT entries, if any, for both registers.
1116	 */
1117	if (ld.x6_op == 0x2) {
1118		invala_fr(ld.r1);
1119		invala_fr(ld.imm);
1120	}
1121	return 0;
1122}
1123
1124
1125static int
1126emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1127{
1128	struct ia64_fpreg fpr_init;
1129	struct ia64_fpreg fpr_final;
1130	unsigned long len = float_fsz[ld.x6_sz];
1131
1132	/*
1133	 * fr0 & fr1 don't need to be checked because Illegal Instruction
1134	 * faults have higher priority than unaligned faults.
1135	 *
1136	 * r0 cannot be found as the base as it would never generate an
1137	 * unaligned reference.
1138	 */
1139
1140	/*
1141	 * make sure we get clean buffers
1142	 */
1143	memset(&fpr_init,0, sizeof(fpr_init));
1144	memset(&fpr_final,0, sizeof(fpr_final));
1145
1146	/*
1147	 * ldfX.a we don't try to emulate anything but we must
1148	 * invalidate the ALAT entry.
1149	 * See comments in ldX for descriptions on how the various loads are handled.
1150	 */
1151	if (ld.x6_op != 0x2) {
1152		if (copy_from_user(&fpr_init, (void __user *) ifa, len))
1153			return -1;
1154
1155		DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1156		DDUMP("fpr_init =", &fpr_init, len);
1157		/*
1158		 * we only do something for x6_op={0,8,9}
1159		 */
1160		switch( ld.x6_sz ) {
1161			case 0:
1162				mem2float_extended(&fpr_init, &fpr_final);
1163				break;
1164			case 1:
1165				mem2float_integer(&fpr_init, &fpr_final);
1166				break;
1167			case 2:
1168				mem2float_single(&fpr_init, &fpr_final);
1169				break;
1170			case 3:
1171				mem2float_double(&fpr_init, &fpr_final);
1172				break;
1173		}
1174		DDUMP("fpr_final =", &fpr_final, len);
1175		/*
1176		 * XXX fixme
1177		 *
1178		 * A possible optimization would be to drop fpr_final and directly
1179		 * use the storage from the saved context i.e., the actual final
1180		 * destination (pt_regs, switch_stack or thread structure).
1181		 */
1182		setfpreg(ld.r1, &fpr_final, regs);
1183	}
1184
1185	/*
1186	 * check for updates on any loads
1187	 */
1188	if (ld.op == 0x7 || ld.m)
1189		emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
1190
1191	/*
1192	 * invalidate ALAT entry in case of advanced floating point loads
1193	 */
1194	if (ld.x6_op == 0x2)
1195		invala_fr(ld.r1);
1196
1197	return 0;
1198}
1199
1200
1201static int
1202emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs)
1203{
1204	struct ia64_fpreg fpr_init;
1205	struct ia64_fpreg fpr_final;
1206	unsigned long len = float_fsz[ld.x6_sz];
1207
1208	/*
1209	 * make sure we get clean buffers
1210	 */
1211	memset(&fpr_init,0, sizeof(fpr_init));
1212	memset(&fpr_final,0, sizeof(fpr_final));
1213
1214	/*
1215	 * if we get to this handler, Nat bits on both r3 and r2 have already
1216	 * been checked. so we don't need to do it
1217	 *
1218	 * extract the value to be stored
1219	 */
1220	getfpreg(ld.imm, &fpr_init, regs);
1221	/*
1222	 * during this step, we extract the spilled registers from the saved
1223	 * context i.e., we refill. Then we store (no spill) to temporary
1224	 * aligned location
1225	 */
1226	switch( ld.x6_sz ) {
1227		case 0:
1228			float2mem_extended(&fpr_init, &fpr_final);
1229			break;
1230		case 1:
1231			float2mem_integer(&fpr_init, &fpr_final);
1232			break;
1233		case 2:
1234			float2mem_single(&fpr_init, &fpr_final);
1235			break;
1236		case 3:
1237			float2mem_double(&fpr_init, &fpr_final);
1238			break;
1239	}
1240	DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1241	DDUMP("fpr_init =", &fpr_init, len);
1242	DDUMP("fpr_final =", &fpr_final, len);
1243
1244	if (copy_to_user((void __user *) ifa, &fpr_final, len))
1245		return -1;
1246
1247	/*
1248	 * stfX [r3]=r2,imm(9)
1249	 *
1250	 * NOTE:
1251	 * ld.r3 can never be r0, because r0 would not generate an
1252	 * unaligned access.
1253	 */
1254	if (ld.op == 0x7) {
1255		unsigned long imm;
1256
1257		/*
1258		 * form imm9: [12:6] contain first 7bits
1259		 */
1260		imm = ld.x << 7 | ld.r1;
1261		/*
1262		 * sign extend (8bits) if m set
1263		 */
1264		if (ld.m)
1265			imm |= SIGN_EXT9;
1266		/*
1267		 * ifa == r3 (NaT is necessarily cleared)
1268		 */
1269		ifa += imm;
1270
1271		DPRINT("imm=%lx r3=%lx\n", imm, ifa);
1272
1273		setreg(ld.r3, ifa, 0, regs);
1274	}
1275	/*
1276	 * we don't have alat_invalidate_multiple() so we need
1277	 * to do the complete flush :-<<
1278	 */
1279	ia64_invala();
1280
1281	return 0;
1282}
1283
1284/*
1285 * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1286 * eventually fix the program.  However, we don't want to do that for every access so we
1287 * pace it with jiffies.
1288 */
1289static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5);
1290
1291void
1292ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs)
1293{
1294	struct ia64_psr *ipsr = ia64_psr(regs);
1295	mm_segment_t old_fs = get_fs();
1296	unsigned long bundle[2];
1297	unsigned long opcode;
1298	struct siginfo si;
1299	const struct exception_table_entry *eh = NULL;
1300	union {
1301		unsigned long l;
1302		load_store_t insn;
1303	} u;
1304	int ret = -1;
1305
1306	if (ia64_psr(regs)->be) {
1307		/* we don't support big-endian accesses */
1308		if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0))
1309			return;
1310		goto force_sigbus;
1311	}
1312
1313	/*
1314	 * Treat kernel accesses for which there is an exception handler entry the same as
1315	 * user-level unaligned accesses.  Otherwise, a clever program could trick this
1316	 * handler into reading an arbitrary kernel addresses...
1317	 */
1318	if (!user_mode(regs))
1319		eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri);
1320	if (user_mode(regs) || eh) {
1321		if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0)
1322			goto force_sigbus;
1323
1324		if (!no_unaligned_warning &&
1325		    !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) &&
1326		    __ratelimit(&logging_rate_limit))
1327		{
1328			char buf[200];	/* comm[] is at most 16 bytes... */
1329			size_t len;
1330
1331			len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, "
1332				      "ip=0x%016lx\n\r", current->comm,
1333				      task_pid_nr(current),
1334				      ifa, regs->cr_iip + ipsr->ri);
1335			/*
1336			 * Don't call tty_write_message() if we're in the kernel; we might
1337			 * be holding locks...
1338			 */
1339			if (user_mode(regs)) {
1340				struct tty_struct *tty = get_current_tty();
1341				tty_write_message(tty, buf);
1342				tty_kref_put(tty);
1343			}
1344			buf[len-1] = '\0';	/* drop '\r' */
1345			/* watch for command names containing %s */
1346			printk(KERN_WARNING "%s", buf);
1347		} else {
1348			if (no_unaligned_warning) {
1349				printk_once(KERN_WARNING "%s(%d) encountered an "
1350				       "unaligned exception which required\n"
1351				       "kernel assistance, which degrades "
1352				       "the performance of the application.\n"
1353				       "Unaligned exception warnings have "
1354				       "been disabled by the system "
1355				       "administrator\n"
1356				       "echo 0 > /proc/sys/kernel/ignore-"
1357				       "unaligned-usertrap to re-enable\n",
1358				       current->comm, task_pid_nr(current));
1359			}
1360		}
1361	} else {
1362		if (__ratelimit(&logging_rate_limit)) {
1363			printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n",
1364			       ifa, regs->cr_iip + ipsr->ri);
1365			if (unaligned_dump_stack)
1366				dump_stack();
1367		}
1368		set_fs(KERNEL_DS);
1369	}
1370
1371	DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n",
1372	       regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it);
1373
1374	if (__copy_from_user(bundle, (void __user *) regs->cr_iip, 16))
1375		goto failure;
1376
1377	/*
1378	 * extract the instruction from the bundle given the slot number
1379	 */
1380	switch (ipsr->ri) {
 
1381	      case 0: u.l = (bundle[0] >>  5); break;
1382	      case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break;
1383	      case 2: u.l = (bundle[1] >> 23); break;
1384	}
1385	opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK;
1386
1387	DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d "
1388	       "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm,
1389	       u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op);
1390
1391	/*
1392	 * IMPORTANT:
1393	 * Notice that the switch statement DOES not cover all possible instructions
1394	 * that DO generate unaligned references. This is made on purpose because for some
1395	 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1396	 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1397	 * the program will get a signal and die:
1398	 *
1399	 *	load/store:
1400	 *		- ldX.spill
1401	 *		- stX.spill
1402	 *	Reason: RNATs are based on addresses
1403	 *		- ld16
1404	 *		- st16
1405	 *	Reason: ld16 and st16 are supposed to occur in a single
1406	 *		memory op
1407	 *
1408	 *	synchronization:
1409	 *		- cmpxchg
1410	 *		- fetchadd
1411	 *		- xchg
1412	 *	Reason: ATOMIC operations cannot be emulated properly using multiple
1413	 *	        instructions.
1414	 *
1415	 *	speculative loads:
1416	 *		- ldX.sZ
1417	 *	Reason: side effects, code must be ready to deal with failure so simpler
1418	 *		to let the load fail.
1419	 * ---------------------------------------------------------------------------------
1420	 * XXX fixme
1421	 *
1422	 * I would like to get rid of this switch case and do something
1423	 * more elegant.
1424	 */
1425	switch (opcode) {
1426	      case LDS_OP:
1427	      case LDSA_OP:
1428		if (u.insn.x)
1429			/* oops, really a semaphore op (cmpxchg, etc) */
1430			goto failure;
1431		/* no break */
1432	      case LDS_IMM_OP:
1433	      case LDSA_IMM_OP:
1434	      case LDFS_OP:
1435	      case LDFSA_OP:
1436	      case LDFS_IMM_OP:
1437		/*
1438		 * The instruction will be retried with deferred exceptions turned on, and
1439		 * we should get Nat bit installed
1440		 *
1441		 * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1442		 * are actually executed even though the operation failed. So we don't
1443		 * need to take care of this.
1444		 */
1445		DPRINT("forcing PSR_ED\n");
1446		regs->cr_ipsr |= IA64_PSR_ED;
1447		goto done;
1448
1449	      case LD_OP:
1450	      case LDA_OP:
1451	      case LDBIAS_OP:
1452	      case LDACQ_OP:
1453	      case LDCCLR_OP:
1454	      case LDCNC_OP:
1455	      case LDCCLRACQ_OP:
1456		if (u.insn.x)
1457			/* oops, really a semaphore op (cmpxchg, etc) */
1458			goto failure;
1459		/* no break */
1460	      case LD_IMM_OP:
1461	      case LDA_IMM_OP:
1462	      case LDBIAS_IMM_OP:
1463	      case LDACQ_IMM_OP:
1464	      case LDCCLR_IMM_OP:
1465	      case LDCNC_IMM_OP:
1466	      case LDCCLRACQ_IMM_OP:
1467		ret = emulate_load_int(ifa, u.insn, regs);
1468		break;
1469
1470	      case ST_OP:
1471	      case STREL_OP:
1472		if (u.insn.x)
1473			/* oops, really a semaphore op (cmpxchg, etc) */
1474			goto failure;
1475		/* no break */
1476	      case ST_IMM_OP:
1477	      case STREL_IMM_OP:
1478		ret = emulate_store_int(ifa, u.insn, regs);
1479		break;
1480
1481	      case LDF_OP:
1482	      case LDFA_OP:
1483	      case LDFCCLR_OP:
1484	      case LDFCNC_OP:
1485		if (u.insn.x)
1486			ret = emulate_load_floatpair(ifa, u.insn, regs);
1487		else
1488			ret = emulate_load_float(ifa, u.insn, regs);
1489		break;
1490
1491	      case LDF_IMM_OP:
1492	      case LDFA_IMM_OP:
1493	      case LDFCCLR_IMM_OP:
1494	      case LDFCNC_IMM_OP:
1495		ret = emulate_load_float(ifa, u.insn, regs);
1496		break;
1497
1498	      case STF_OP:
1499	      case STF_IMM_OP:
1500		ret = emulate_store_float(ifa, u.insn, regs);
1501		break;
1502
1503	      default:
1504		goto failure;
1505	}
1506	DPRINT("ret=%d\n", ret);
1507	if (ret)
1508		goto failure;
1509
1510	if (ipsr->ri == 2)
1511		/*
1512		 * given today's architecture this case is not likely to happen because a
1513		 * memory access instruction (M) can never be in the last slot of a
1514		 * bundle. But let's keep it for now.
1515		 */
1516		regs->cr_iip += 16;
1517	ipsr->ri = (ipsr->ri + 1) & 0x3;
1518
1519	DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip);
1520  done:
1521	set_fs(old_fs);		/* restore original address limit */
1522	return;
1523
1524  failure:
1525	/* something went wrong... */
1526	if (!user_mode(regs)) {
1527		if (eh) {
1528			ia64_handle_exception(regs, eh);
1529			goto done;
1530		}
1531		if (die_if_kernel("error during unaligned kernel access\n", regs, ret))
1532			return;
1533		/* NOT_REACHED */
1534	}
1535  force_sigbus:
1536	si.si_signo = SIGBUS;
1537	si.si_errno = 0;
1538	si.si_code = BUS_ADRALN;
1539	si.si_addr = (void __user *) ifa;
1540	si.si_flags = 0;
1541	si.si_isr = 0;
1542	si.si_imm = 0;
1543	force_sig_info(SIGBUS, &si, current);
1544	goto done;
1545}