1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Architecture-specific unaligned trap handling.
   4 *
   5 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
   6 *	Stephane Eranian <eranian@hpl.hp.com>
   7 *	David Mosberger-Tang <davidm@hpl.hp.com>
   8 *
   9 * 2002/12/09   Fix rotating register handling (off-by-1 error, missing fr-rotation).  Fix
  10 *		get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
  11 *		stacked register returns an undefined value; it does NOT trigger a
  12 *		"rsvd register fault").
  13 * 2001/10/11	Fix unaligned access to rotating registers in s/w pipelined loops.
  14 * 2001/08/13	Correct size of extended floats (float_fsz) from 16 to 10 bytes.
  15 * 2001/01/17	Add support emulation of unaligned kernel accesses.
  16 */
  17#include <linux/jiffies.h>
  18#include <linux/kernel.h>
  19#include <linux/sched/signal.h>
  20#include <linux/tty.h>
  21#include <linux/extable.h>
  22#include <linux/ratelimit.h>
  23#include <linux/uaccess.h>
  24
  25#include <asm/intrinsics.h>
  26#include <asm/processor.h>
  27#include <asm/rse.h>
  28#include <asm/exception.h>
  29#include <asm/unaligned.h>
  30
  31extern int die_if_kernel(char *str, struct pt_regs *regs, long err);
  32
  33#undef DEBUG_UNALIGNED_TRAP
  34
  35#ifdef DEBUG_UNALIGNED_TRAP
  36# define DPRINT(a...)	do { printk("%s %u: ", __func__, __LINE__); printk (a); } while (0)
  37# define DDUMP(str,vp,len)	dump(str, vp, len)
  38
  39static void
  40dump (const char *str, void *vp, size_t len)
  41{
  42	unsigned char *cp = vp;
  43	int i;
  44
  45	printk("%s", str);
  46	for (i = 0; i < len; ++i)
  47		printk (" %02x", *cp++);
  48	printk("\n");
  49}
  50#else
  51# define DPRINT(a...)
  52# define DDUMP(str,vp,len)
  53#endif
  54
  55#define IA64_FIRST_STACKED_GR	32
  56#define IA64_FIRST_ROTATING_FR	32
  57#define SIGN_EXT9		0xffffffffffffff00ul
  58
  59/*
  60 *  sysctl settable hook which tells the kernel whether to honor the
  61 *  IA64_THREAD_UAC_NOPRINT prctl.  Because this is user settable, we want
  62 *  to allow the super user to enable/disable this for security reasons
  63 *  (i.e. don't allow attacker to fill up logs with unaligned accesses).
  64 */
  65int no_unaligned_warning;
  66int unaligned_dump_stack;
  67
  68/*
  69 * For M-unit:
  70 *
  71 *  opcode |   m  |   x6    |
  72 * --------|------|---------|
  73 * [40-37] | [36] | [35:30] |
  74 * --------|------|---------|
  75 *     4   |   1  |    6    | = 11 bits
  76 * --------------------------
  77 * However bits [31:30] are not directly useful to distinguish between
  78 * load/store so we can use [35:32] instead, which gives the following
  79 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
  80 * checking the m-bit until later in the load/store emulation.
  81 */
  82#define IA64_OPCODE_MASK	0x1ef
  83#define IA64_OPCODE_SHIFT	32
  84
  85/*
  86 * Table C-28 Integer Load/Store
  87 *
  88 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
  89 *
  90 * ld8.fill, st8.fill  MUST be aligned because the RNATs are based on
  91 * the address (bits [8:3]), so we must failed.
  92 */
  93#define LD_OP            0x080
  94#define LDS_OP           0x081
  95#define LDA_OP           0x082
  96#define LDSA_OP          0x083
  97#define LDBIAS_OP        0x084
  98#define LDACQ_OP         0x085
  99/* 0x086, 0x087 are not relevant */
 100#define LDCCLR_OP        0x088
 101#define LDCNC_OP         0x089
 102#define LDCCLRACQ_OP     0x08a
 103#define ST_OP            0x08c
 104#define STREL_OP         0x08d
 105/* 0x08e,0x8f are not relevant */
 106
 107/*
 108 * Table C-29 Integer Load +Reg
 109 *
 110 * we use the ld->m (bit [36:36]) field to determine whether or not we have
 111 * a load/store of this form.
 112 */
 113
 114/*
 115 * Table C-30 Integer Load/Store +Imm
 116 *
 117 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
 118 *
 119 * ld8.fill, st8.fill  must be aligned because the Nat register are based on
 120 * the address, so we must fail and the program must be fixed.
 121 */
 122#define LD_IMM_OP            0x0a0
 123#define LDS_IMM_OP           0x0a1
 124#define LDA_IMM_OP           0x0a2
 125#define LDSA_IMM_OP          0x0a3
 126#define LDBIAS_IMM_OP        0x0a4
 127#define LDACQ_IMM_OP         0x0a5
 128/* 0x0a6, 0xa7 are not relevant */
 129#define LDCCLR_IMM_OP        0x0a8
 130#define LDCNC_IMM_OP         0x0a9
 131#define LDCCLRACQ_IMM_OP     0x0aa
 132#define ST_IMM_OP            0x0ac
 133#define STREL_IMM_OP         0x0ad
 134/* 0x0ae,0xaf are not relevant */
 135
 136/*
 137 * Table C-32 Floating-point Load/Store
 138 */
 139#define LDF_OP           0x0c0
 140#define LDFS_OP          0x0c1
 141#define LDFA_OP          0x0c2
 142#define LDFSA_OP         0x0c3
 143/* 0x0c6 is irrelevant */
 144#define LDFCCLR_OP       0x0c8
 145#define LDFCNC_OP        0x0c9
 146/* 0x0cb is irrelevant  */
 147#define STF_OP           0x0cc
 148
 149/*
 150 * Table C-33 Floating-point Load +Reg
 151 *
 152 * we use the ld->m (bit [36:36]) field to determine whether or not we have
 153 * a load/store of this form.
 154 */
 155
 156/*
 157 * Table C-34 Floating-point Load/Store +Imm
 158 */
 159#define LDF_IMM_OP       0x0e0
 160#define LDFS_IMM_OP      0x0e1
 161#define LDFA_IMM_OP      0x0e2
 162#define LDFSA_IMM_OP     0x0e3
 163/* 0x0e6 is irrelevant */
 164#define LDFCCLR_IMM_OP   0x0e8
 165#define LDFCNC_IMM_OP    0x0e9
 166#define STF_IMM_OP       0x0ec
 167
 168typedef struct {
 169	unsigned long	 qp:6;	/* [0:5]   */
 170	unsigned long    r1:7;	/* [6:12]  */
 171	unsigned long   imm:7;	/* [13:19] */
 172	unsigned long    r3:7;	/* [20:26] */
 173	unsigned long     x:1;  /* [27:27] */
 174	unsigned long  hint:2;	/* [28:29] */
 175	unsigned long x6_sz:2;	/* [30:31] */
 176	unsigned long x6_op:4;	/* [32:35], x6 = x6_sz|x6_op */
 177	unsigned long     m:1;	/* [36:36] */
 178	unsigned long    op:4;	/* [37:40] */
 179	unsigned long   pad:23; /* [41:63] */
 180} load_store_t;
 181
 182
 183typedef enum {
 184	UPD_IMMEDIATE,	/* ldXZ r1=[r3],imm(9) */
 185	UPD_REG		/* ldXZ r1=[r3],r2     */
 186} update_t;
 187
 188/*
 189 * We use tables to keep track of the offsets of registers in the saved state.
 190 * This way we save having big switch/case statements.
 191 *
 192 * We use bit 0 to indicate switch_stack or pt_regs.
 193 * The offset is simply shifted by 1 bit.
 194 * A 2-byte value should be enough to hold any kind of offset
 195 *
 196 * In case the calling convention changes (and thus pt_regs/switch_stack)
 197 * simply use RSW instead of RPT or vice-versa.
 198 */
 199
 200#define RPO(x)	((size_t) &((struct pt_regs *)0)->x)
 201#define RSO(x)	((size_t) &((struct switch_stack *)0)->x)
 202
 203#define RPT(x)		(RPO(x) << 1)
 204#define RSW(x)		(1| RSO(x)<<1)
 205
 206#define GR_OFFS(x)	(gr_info[x]>>1)
 207#define GR_IN_SW(x)	(gr_info[x] & 0x1)
 208
 209#define FR_OFFS(x)	(fr_info[x]>>1)
 210#define FR_IN_SW(x)	(fr_info[x] & 0x1)
 211
 212static u16 gr_info[32]={
 213	0,			/* r0 is read-only : WE SHOULD NEVER GET THIS */
 214
 215	RPT(r1), RPT(r2), RPT(r3),
 216
 217	RSW(r4), RSW(r5), RSW(r6), RSW(r7),
 218
 219	RPT(r8), RPT(r9), RPT(r10), RPT(r11),
 220	RPT(r12), RPT(r13), RPT(r14), RPT(r15),
 221
 222	RPT(r16), RPT(r17), RPT(r18), RPT(r19),
 223	RPT(r20), RPT(r21), RPT(r22), RPT(r23),
 224	RPT(r24), RPT(r25), RPT(r26), RPT(r27),
 225	RPT(r28), RPT(r29), RPT(r30), RPT(r31)
 226};
 227
 228static u16 fr_info[32]={
 229	0,			/* constant : WE SHOULD NEVER GET THIS */
 230	0,			/* constant : WE SHOULD NEVER GET THIS */
 231
 232	RSW(f2), RSW(f3), RSW(f4), RSW(f5),
 233
 234	RPT(f6), RPT(f7), RPT(f8), RPT(f9),
 235	RPT(f10), RPT(f11),
 236
 237	RSW(f12), RSW(f13), RSW(f14),
 238	RSW(f15), RSW(f16), RSW(f17), RSW(f18), RSW(f19),
 239	RSW(f20), RSW(f21), RSW(f22), RSW(f23), RSW(f24),
 240	RSW(f25), RSW(f26), RSW(f27), RSW(f28), RSW(f29),
 241	RSW(f30), RSW(f31)
 242};
 243
 244/* Invalidate ALAT entry for integer register REGNO.  */
 245static void
 246invala_gr (int regno)
 247{
 248#	define F(reg)	case reg: ia64_invala_gr(reg); break
 249
 250	switch (regno) {
 251		F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
 252		F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
 253		F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
 254		F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
 255		F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
 256		F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
 257		F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
 258		F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
 259		F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
 260		F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
 261		F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
 262		F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
 263		F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
 264		F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
 265		F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
 266		F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
 267	}
 268#	undef F
 269}
 270
 271/* Invalidate ALAT entry for floating-point register REGNO.  */
 272static void
 273invala_fr (int regno)
 274{
 275#	define F(reg)	case reg: ia64_invala_fr(reg); break
 276
 277	switch (regno) {
 278		F(  0); F(  1); F(  2); F(  3); F(  4); F(  5); F(  6); F(  7);
 279		F(  8); F(  9); F( 10); F( 11); F( 12); F( 13); F( 14); F( 15);
 280		F( 16); F( 17); F( 18); F( 19); F( 20); F( 21); F( 22); F( 23);
 281		F( 24); F( 25); F( 26); F( 27); F( 28); F( 29); F( 30); F( 31);
 282		F( 32); F( 33); F( 34); F( 35); F( 36); F( 37); F( 38); F( 39);
 283		F( 40); F( 41); F( 42); F( 43); F( 44); F( 45); F( 46); F( 47);
 284		F( 48); F( 49); F( 50); F( 51); F( 52); F( 53); F( 54); F( 55);
 285		F( 56); F( 57); F( 58); F( 59); F( 60); F( 61); F( 62); F( 63);
 286		F( 64); F( 65); F( 66); F( 67); F( 68); F( 69); F( 70); F( 71);
 287		F( 72); F( 73); F( 74); F( 75); F( 76); F( 77); F( 78); F( 79);
 288		F( 80); F( 81); F( 82); F( 83); F( 84); F( 85); F( 86); F( 87);
 289		F( 88); F( 89); F( 90); F( 91); F( 92); F( 93); F( 94); F( 95);
 290		F( 96); F( 97); F( 98); F( 99); F(100); F(101); F(102); F(103);
 291		F(104); F(105); F(106); F(107); F(108); F(109); F(110); F(111);
 292		F(112); F(113); F(114); F(115); F(116); F(117); F(118); F(119);
 293		F(120); F(121); F(122); F(123); F(124); F(125); F(126); F(127);
 294	}
 295#	undef F
 296}
 297
 298static inline unsigned long
 299rotate_reg (unsigned long sor, unsigned long rrb, unsigned long reg)
 300{
 301	reg += rrb;
 302	if (reg >= sor)
 303		reg -= sor;
 304	return reg;
 305}
 306
 307static void
 308set_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long val, int nat)
 309{
 310	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 311	unsigned long *bsp, *bspstore, *addr, *rnat_addr, *ubs_end;
 312	unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
 313	unsigned long rnats, nat_mask;
 314	unsigned long on_kbs;
 315	long sof = (regs->cr_ifs) & 0x7f;
 316	long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
 317	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
 318	long ridx = r1 - 32;
 319
 320	if (ridx >= sof) {
 321		/* this should never happen, as the "rsvd register fault" has higher priority */
 322		DPRINT("ignoring write to r%lu; only %lu registers are allocated!\n", r1, sof);
 323		return;
 324	}
 325
 326	if (ridx < sor)
 327		ridx = rotate_reg(sor, rrb_gr, ridx);
 328
 329	DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
 330	       r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
 331
 332	on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
 333	addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
 334	if (addr >= kbs) {
 335		/* the register is on the kernel backing store: easy... */
 336		rnat_addr = ia64_rse_rnat_addr(addr);
 337		if ((unsigned long) rnat_addr >= sw->ar_bspstore)
 338			rnat_addr = &sw->ar_rnat;
 339		nat_mask = 1UL << ia64_rse_slot_num(addr);
 340
 341		*addr = val;
 342		if (nat)
 343			*rnat_addr |=  nat_mask;
 344		else
 345			*rnat_addr &= ~nat_mask;
 346		return;
 347	}
 348
 349	if (!user_stack(current, regs)) {
 350		DPRINT("ignoring kernel write to r%lu; register isn't on the kernel RBS!", r1);
 351		return;
 352	}
 353
 354	bspstore = (unsigned long *)regs->ar_bspstore;
 355	ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
 356	bsp     = ia64_rse_skip_regs(ubs_end, -sof);
 357	addr    = ia64_rse_skip_regs(bsp, ridx);
 358
 359	DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
 360
 361	ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
 362
 363	rnat_addr = ia64_rse_rnat_addr(addr);
 364
 365	ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
 366	DPRINT("rnat @%p = 0x%lx nat=%d old nat=%ld\n",
 367	       (void *) rnat_addr, rnats, nat, (rnats >> ia64_rse_slot_num(addr)) & 1);
 368
 369	nat_mask = 1UL << ia64_rse_slot_num(addr);
 370	if (nat)
 371		rnats |=  nat_mask;
 372	else
 373		rnats &= ~nat_mask;
 374	ia64_poke(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, rnats);
 375
 376	DPRINT("rnat changed to @%p = 0x%lx\n", (void *) rnat_addr, rnats);
 377}
 378
 379
 380static void
 381get_rse_reg (struct pt_regs *regs, unsigned long r1, unsigned long *val, int *nat)
 382{
 383	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 384	unsigned long *bsp, *addr, *rnat_addr, *ubs_end, *bspstore;
 385	unsigned long *kbs = (void *) current + IA64_RBS_OFFSET;
 386	unsigned long rnats, nat_mask;
 387	unsigned long on_kbs;
 388	long sof = (regs->cr_ifs) & 0x7f;
 389	long sor = 8 * ((regs->cr_ifs >> 14) & 0xf);
 390	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
 391	long ridx = r1 - 32;
 392
 393	if (ridx >= sof) {
 394		/* read of out-of-frame register returns an undefined value; 0 in our case.  */
 395		DPRINT("ignoring read from r%lu; only %lu registers are allocated!\n", r1, sof);
 396		goto fail;
 397	}
 398
 399	if (ridx < sor)
 400		ridx = rotate_reg(sor, rrb_gr, ridx);
 401
 402	DPRINT("r%lu, sw.bspstore=%lx pt.bspstore=%lx sof=%ld sol=%ld ridx=%ld\n",
 403	       r1, sw->ar_bspstore, regs->ar_bspstore, sof, (regs->cr_ifs >> 7) & 0x7f, ridx);
 404
 405	on_kbs = ia64_rse_num_regs(kbs, (unsigned long *) sw->ar_bspstore);
 406	addr = ia64_rse_skip_regs((unsigned long *) sw->ar_bspstore, -sof + ridx);
 407	if (addr >= kbs) {
 408		/* the register is on the kernel backing store: easy... */
 409		*val = *addr;
 410		if (nat) {
 411			rnat_addr = ia64_rse_rnat_addr(addr);
 412			if ((unsigned long) rnat_addr >= sw->ar_bspstore)
 413				rnat_addr = &sw->ar_rnat;
 414			nat_mask = 1UL << ia64_rse_slot_num(addr);
 415			*nat = (*rnat_addr & nat_mask) != 0;
 416		}
 417		return;
 418	}
 419
 420	if (!user_stack(current, regs)) {
 421		DPRINT("ignoring kernel read of r%lu; register isn't on the RBS!", r1);
 422		goto fail;
 423	}
 424
 425	bspstore = (unsigned long *)regs->ar_bspstore;
 426	ubs_end = ia64_rse_skip_regs(bspstore, on_kbs);
 427	bsp     = ia64_rse_skip_regs(ubs_end, -sof);
 428	addr    = ia64_rse_skip_regs(bsp, ridx);
 429
 430	DPRINT("ubs_end=%p bsp=%p addr=%p\n", (void *) ubs_end, (void *) bsp, (void *) addr);
 431
 432	ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) addr, val);
 433
 434	if (nat) {
 435		rnat_addr = ia64_rse_rnat_addr(addr);
 436		nat_mask = 1UL << ia64_rse_slot_num(addr);
 437
 438		DPRINT("rnat @%p = 0x%lx\n", (void *) rnat_addr, rnats);
 439
 440		ia64_peek(current, sw, (unsigned long) ubs_end, (unsigned long) rnat_addr, &rnats);
 441		*nat = (rnats & nat_mask) != 0;
 442	}
 443	return;
 444
 445  fail:
 446	*val = 0;
 447	if (nat)
 448		*nat = 0;
 449	return;
 450}
 451
 452
 453static void
 454setreg (unsigned long regnum, unsigned long val, int nat, struct pt_regs *regs)
 455{
 456	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 457	unsigned long addr;
 458	unsigned long bitmask;
 459	unsigned long *unat;
 460
 461	/*
 462	 * First takes care of stacked registers
 463	 */
 464	if (regnum >= IA64_FIRST_STACKED_GR) {
 465		set_rse_reg(regs, regnum, val, nat);
 466		return;
 467	}
 468
 469	/*
 470	 * Using r0 as a target raises a General Exception fault which has higher priority
 471	 * than the Unaligned Reference fault.
 472	 */
 473
 474	/*
 475	 * Now look at registers in [0-31] range and init correct UNAT
 476	 */
 477	if (GR_IN_SW(regnum)) {
 478		addr = (unsigned long)sw;
 479		unat = &sw->ar_unat;
 480	} else {
 481		addr = (unsigned long)regs;
 482		unat = &sw->caller_unat;
 483	}
 484	DPRINT("tmp_base=%lx switch_stack=%s offset=%d\n",
 485	       addr, unat==&sw->ar_unat ? "yes":"no", GR_OFFS(regnum));
 486	/*
 487	 * add offset from base of struct
 488	 * and do it !
 489	 */
 490	addr += GR_OFFS(regnum);
 491
 492	*(unsigned long *)addr = val;
 493
 494	/*
 495	 * We need to clear the corresponding UNAT bit to fully emulate the load
 496	 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
 497	 */
 498	bitmask   = 1UL << (addr >> 3 & 0x3f);
 499	DPRINT("*0x%lx=0x%lx NaT=%d prev_unat @%p=%lx\n", addr, val, nat, (void *) unat, *unat);
 500	if (nat) {
 501		*unat |= bitmask;
 502	} else {
 503		*unat &= ~bitmask;
 504	}
 505	DPRINT("*0x%lx=0x%lx NaT=%d new unat: %p=%lx\n", addr, val, nat, (void *) unat,*unat);
 506}
 507
 508/*
 509 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
 510 * range from 32-127, result is in the range from 0-95.
 511 */
 512static inline unsigned long
 513fph_index (struct pt_regs *regs, long regnum)
 514{
 515	unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;
 516	return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));
 517}
 518
 519static void
 520setfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
 521{
 522	struct switch_stack *sw = (struct switch_stack *)regs - 1;
 523	unsigned long addr;
 524
 525	/*
 526	 * From EAS-2.5: FPDisableFault has higher priority than Unaligned
 527	 * Fault. Thus, when we get here, we know the partition is enabled.
 528	 * To update f32-f127, there are three choices:
 529	 *
 530	 *	(1) save f32-f127 to thread.fph and update the values there
 531	 *	(2) use a gigantic switch statement to directly access the registers
 532	 *	(3) generate code on the fly to update the desired register
 533	 *
 534	 * For now, we are using approach (1).
 535	 */
 536	if (regnum >= IA64_FIRST_ROTATING_FR) {
 537		ia64_sync_fph(current);
 538		current->thread.fph[fph_index(regs, regnum)] = *fpval;
 539	} else {
 540		/*
 541		 * pt_regs or switch_stack ?
 542		 */
 543		if (FR_IN_SW(regnum)) {
 544			addr = (unsigned long)sw;
 545		} else {
 546			addr = (unsigned long)regs;
 547		}
 548
 549		DPRINT("tmp_base=%lx offset=%d\n", addr, FR_OFFS(regnum));
 550
 551		addr += FR_OFFS(regnum);
 552		*(struct ia64_fpreg *)addr = *fpval;
 553
 554		/*
 555		 * mark the low partition as being used now
 556		 *
 557		 * It is highly unlikely that this bit is not already set, but
 558		 * let's do it for safety.
 559		 */
 560		regs->cr_ipsr |= IA64_PSR_MFL;
 561	}
 562}
 563
 564/*
 565 * Those 2 inline functions generate the spilled versions of the constant floating point
 566 * registers which can be used with stfX
 567 */
 568static inline void
 569float_spill_f0 (struct ia64_fpreg *final)
 570{
 571	ia64_stf_spill(final, 0);
 572}
 573
 574static inline void
 575float_spill_f1 (struct ia64_fpreg *final)
 576{
 577	ia64_stf_spill(final, 1);
 578}
 579
 580static void
 581getfpreg (unsigned long regnum, struct ia64_fpreg *fpval, struct pt_regs *regs)
 582{
 583	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 584	unsigned long addr;
 585
 586	/*
 587	 * From EAS-2.5: FPDisableFault has higher priority than
 588	 * Unaligned Fault. Thus, when we get here, we know the partition is
 589	 * enabled.
 590	 *
 591	 * When regnum > 31, the register is still live and we need to force a save
 592	 * to current->thread.fph to get access to it.  See discussion in setfpreg()
 593	 * for reasons and other ways of doing this.
 594	 */
 595	if (regnum >= IA64_FIRST_ROTATING_FR) {
 596		ia64_flush_fph(current);
 597		*fpval = current->thread.fph[fph_index(regs, regnum)];
 598	} else {
 599		/*
 600		 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
 601		 * not saved, we must generate their spilled form on the fly
 602		 */
 603		switch(regnum) {
 604		case 0:
 605			float_spill_f0(fpval);
 606			break;
 607		case 1:
 608			float_spill_f1(fpval);
 609			break;
 610		default:
 611			/*
 612			 * pt_regs or switch_stack ?
 613			 */
 614			addr =  FR_IN_SW(regnum) ? (unsigned long)sw
 615						 : (unsigned long)regs;
 616
 617			DPRINT("is_sw=%d tmp_base=%lx offset=0x%x\n",
 618			       FR_IN_SW(regnum), addr, FR_OFFS(regnum));
 619
 620			addr  += FR_OFFS(regnum);
 621			*fpval = *(struct ia64_fpreg *)addr;
 622		}
 623	}
 624}
 625
 626
 627static void
 628getreg (unsigned long regnum, unsigned long *val, int *nat, struct pt_regs *regs)
 629{
 630	struct switch_stack *sw = (struct switch_stack *) regs - 1;
 631	unsigned long addr, *unat;
 632
 633	if (regnum >= IA64_FIRST_STACKED_GR) {
 634		get_rse_reg(regs, regnum, val, nat);
 635		return;
 636	}
 637
 638	/*
 639	 * take care of r0 (read-only always evaluate to 0)
 640	 */
 641	if (regnum == 0) {
 642		*val = 0;
 643		if (nat)
 644			*nat = 0;
 645		return;
 646	}
 647
 648	/*
 649	 * Now look at registers in [0-31] range and init correct UNAT
 650	 */
 651	if (GR_IN_SW(regnum)) {
 652		addr = (unsigned long)sw;
 653		unat = &sw->ar_unat;
 654	} else {
 655		addr = (unsigned long)regs;
 656		unat = &sw->caller_unat;
 657	}
 658
 659	DPRINT("addr_base=%lx offset=0x%x\n", addr,  GR_OFFS(regnum));
 660
 661	addr += GR_OFFS(regnum);
 662
 663	*val  = *(unsigned long *)addr;
 664
 665	/*
 666	 * do it only when requested
 667	 */
 668	if (nat)
 669		*nat  = (*unat >> (addr >> 3 & 0x3f)) & 0x1UL;
 670}
 671
 672static void
 673emulate_load_updates (update_t type, load_store_t ld, struct pt_regs *regs, unsigned long ifa)
 674{
 675	/*
 676	 * IMPORTANT:
 677	 * Given the way we handle unaligned speculative loads, we should
 678	 * not get to this point in the code but we keep this sanity check,
 679	 * just in case.
 680	 */
 681	if (ld.x6_op == 1 || ld.x6_op == 3) {
 682		printk(KERN_ERR "%s: register update on speculative load, error\n", __func__);
 683		if (die_if_kernel("unaligned reference on speculative load with register update\n",
 684				  regs, 30))
 685			return;
 686	}
 687
 688
 689	/*
 690	 * at this point, we know that the base register to update is valid i.e.,
 691	 * it's not r0
 692	 */
 693	if (type == UPD_IMMEDIATE) {
 694		unsigned long imm;
 695
 696		/*
 697		 * Load +Imm: ldXZ r1=[r3],imm(9)
 698		 *
 699		 *
 700		 * form imm9: [13:19] contain the first 7 bits
 701		 */
 702		imm = ld.x << 7 | ld.imm;
 703
 704		/*
 705		 * sign extend (1+8bits) if m set
 706		 */
 707		if (ld.m) imm |= SIGN_EXT9;
 708
 709		/*
 710		 * ifa == r3 and we know that the NaT bit on r3 was clear so
 711		 * we can directly use ifa.
 712		 */
 713		ifa += imm;
 714
 715		setreg(ld.r3, ifa, 0, regs);
 716
 717		DPRINT("ld.x=%d ld.m=%d imm=%ld r3=0x%lx\n", ld.x, ld.m, imm, ifa);
 718
 719	} else if (ld.m) {
 720		unsigned long r2;
 721		int nat_r2;
 722
 723		/*
 724		 * Load +Reg Opcode: ldXZ r1=[r3],r2
 725		 *
 726		 * Note: that we update r3 even in the case of ldfX.a
 727		 * (where the load does not happen)
 728		 *
 729		 * The way the load algorithm works, we know that r3 does not
 730		 * have its NaT bit set (would have gotten NaT consumption
 731		 * before getting the unaligned fault). So we can use ifa
 732		 * which equals r3 at this point.
 733		 *
 734		 * IMPORTANT:
 735		 * The above statement holds ONLY because we know that we
 736		 * never reach this code when trying to do a ldX.s.
 737		 * If we ever make it to here on an ldfX.s then
 738		 */
 739		getreg(ld.imm, &r2, &nat_r2, regs);
 740
 741		ifa += r2;
 742
 743		/*
 744		 * propagate Nat r2 -> r3
 745		 */
 746		setreg(ld.r3, ifa, nat_r2, regs);
 747
 748		DPRINT("imm=%d r2=%ld r3=0x%lx nat_r2=%d\n",ld.imm, r2, ifa, nat_r2);
 749	}
 750}
 751
 752static int emulate_store(unsigned long ifa, void *val, int len, bool kernel_mode)
 753{
 754	if (kernel_mode)
 755		return copy_to_kernel_nofault((void *)ifa, val, len);
 756
 757	return copy_to_user((void __user *)ifa, val, len);
 758}
 759
 760static int emulate_load(void *val, unsigned long ifa, int len, bool kernel_mode)
 761{
 762	if (kernel_mode)
 763	       return copy_from_kernel_nofault(val, (void *)ifa, len);
 764
 765	return copy_from_user(val, (void __user *)ifa, len);
 766}
 767
 768static int
 769emulate_load_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs,
 770		  bool kernel_mode)
 771{
 772	unsigned int len = 1 << ld.x6_sz;
 773	unsigned long val = 0;
 774
 775	/*
 776	 * r0, as target, doesn't need to be checked because Illegal Instruction
 777	 * faults have higher priority than unaligned faults.
 778	 *
 779	 * r0 cannot be found as the base as it would never generate an
 780	 * unaligned reference.
 781	 */
 782
 783	/*
 784	 * ldX.a we will emulate load and also invalidate the ALAT entry.
 785	 * See comment below for explanation on how we handle ldX.a
 786	 */
 787
 788	if (len != 2 && len != 4 && len != 8) {
 789		DPRINT("unknown size: x6=%d\n", ld.x6_sz);
 790		return -1;
 791	}
 792	/* this assumes little-endian byte-order: */
 793	if (emulate_load(&val, ifa, len, kernel_mode))
 794		return -1;
 795	setreg(ld.r1, val, 0, regs);
 796
 797	/*
 798	 * check for updates on any kind of loads
 799	 */
 800	if (ld.op == 0x5 || ld.m)
 801		emulate_load_updates(ld.op == 0x5 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
 802
 803	/*
 804	 * handling of various loads (based on EAS2.4):
 805	 *
 806	 * ldX.acq (ordered load):
 807	 *	- acquire semantics would have been used, so force fence instead.
 808	 *
 809	 * ldX.c.clr (check load and clear):
 810	 *	- if we get to this handler, it's because the entry was not in the ALAT.
 811	 *	  Therefore the operation reverts to a normal load
 812	 *
 813	 * ldX.c.nc (check load no clear):
 814	 *	- same as previous one
 815	 *
 816	 * ldX.c.clr.acq (ordered check load and clear):
 817	 *	- same as above for c.clr part. The load needs to have acquire semantics. So
 818	 *	  we use the fence semantics which is stronger and thus ensures correctness.
 819	 *
 820	 * ldX.a (advanced load):
 821	 *	- suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
 822	 *	  address doesn't match requested size alignment. This means that we would
 823	 *	  possibly need more than one load to get the result.
 824	 *
 825	 *	  The load part can be handled just like a normal load, however the difficult
 826	 *	  part is to get the right thing into the ALAT. The critical piece of information
 827	 *	  in the base address of the load & size. To do that, a ld.a must be executed,
 828	 *	  clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
 829	 *	  if we use the same target register, we will be okay for the check.a instruction.
 830	 *	  If we look at the store, basically a stX [r3]=r1 checks the ALAT  for any entry
 831	 *	  which would overlap within [r3,r3+X] (the size of the load was store in the
 832	 *	  ALAT). If such an entry is found the entry is invalidated. But this is not good
 833	 *	  enough, take the following example:
 834	 *		r3=3
 835	 *		ld4.a r1=[r3]
 836	 *
 837	 *	  Could be emulated by doing:
 838	 *		ld1.a r1=[r3],1
 839	 *		store to temporary;
 840	 *		ld1.a r1=[r3],1
 841	 *		store & shift to temporary;
 842	 *		ld1.a r1=[r3],1
 843	 *		store & shift to temporary;
 844	 *		ld1.a r1=[r3]
 845	 *		store & shift to temporary;
 846	 *		r1=temporary
 847	 *
 848	 *	  So in this case, you would get the right value is r1 but the wrong info in
 849	 *	  the ALAT.  Notice that you could do it in reverse to finish with address 3
 850	 *	  but you would still get the size wrong.  To get the size right, one needs to
 851	 *	  execute exactly the same kind of load. You could do it from a aligned
 852	 *	  temporary location, but you would get the address wrong.
 853	 *
 854	 *	  So no matter what, it is not possible to emulate an advanced load
 855	 *	  correctly. But is that really critical ?
 856	 *
 857	 *	  We will always convert ld.a into a normal load with ALAT invalidated.  This
 858	 *	  will enable compiler to do optimization where certain code path after ld.a
 859	 *	  is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
 860	 *
 861	 *	  If there is a store after the advanced load, one must either do a ld.c.* or
 862	 *	  chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
 863	 *	  entry found in ALAT), and that's perfectly ok because:
 864	 *
 865	 *		- ld.c.*, if the entry is not present a  normal load is executed
 866	 *		- chk.a.*, if the entry is not present, execution jumps to recovery code
 867	 *
 868	 *	  In either case, the load can be potentially retried in another form.
 869	 *
 870	 *	  ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
 871	 *	  up a stale entry later). The register base update MUST also be performed.
 872	 */
 873
 874	/*
 875	 * when the load has the .acq completer then
 876	 * use ordering fence.
 877	 */
 878	if (ld.x6_op == 0x5 || ld.x6_op == 0xa)
 879		mb();
 880
 881	/*
 882	 * invalidate ALAT entry in case of advanced load
 883	 */
 884	if (ld.x6_op == 0x2)
 885		invala_gr(ld.r1);
 886
 887	return 0;
 888}
 889
 890static int
 891emulate_store_int (unsigned long ifa, load_store_t ld, struct pt_regs *regs,
 892		   bool kernel_mode)
 893{
 894	unsigned long r2;
 895	unsigned int len = 1 << ld.x6_sz;
 896
 897	/*
 898	 * if we get to this handler, Nat bits on both r3 and r2 have already
 899	 * been checked. so we don't need to do it
 900	 *
 901	 * extract the value to be stored
 902	 */
 903	getreg(ld.imm, &r2, NULL, regs);
 904
 905	/*
 906	 * we rely on the macros in unaligned.h for now i.e.,
 907	 * we let the compiler figure out how to read memory gracefully.
 908	 *
 909	 * We need this switch/case because the way the inline function
 910	 * works. The code is optimized by the compiler and looks like
 911	 * a single switch/case.
 912	 */
 913	DPRINT("st%d [%lx]=%lx\n", len, ifa, r2);
 914
 915	if (len != 2 && len != 4 && len != 8) {
 916		DPRINT("unknown size: x6=%d\n", ld.x6_sz);
 917		return -1;
 918	}
 919
 920	/* this assumes little-endian byte-order: */
 921	if (emulate_store(ifa, &r2, len, kernel_mode))
 922		return -1;
 923
 924	/*
 925	 * stX [r3]=r2,imm(9)
 926	 *
 927	 * NOTE:
 928	 * ld.r3 can never be r0, because r0 would not generate an
 929	 * unaligned access.
 930	 */
 931	if (ld.op == 0x5) {
 932		unsigned long imm;
 933
 934		/*
 935		 * form imm9: [12:6] contain first 7bits
 936		 */
 937		imm = ld.x << 7 | ld.r1;
 938		/*
 939		 * sign extend (8bits) if m set
 940		 */
 941		if (ld.m) imm |= SIGN_EXT9;
 942		/*
 943		 * ifa == r3 (NaT is necessarily cleared)
 944		 */
 945		ifa += imm;
 946
 947		DPRINT("imm=%lx r3=%lx\n", imm, ifa);
 948
 949		setreg(ld.r3, ifa, 0, regs);
 950	}
 951	/*
 952	 * we don't have alat_invalidate_multiple() so we need
 953	 * to do the complete flush :-<<
 954	 */
 955	ia64_invala();
 956
 957	/*
 958	 * stX.rel: use fence instead of release
 959	 */
 960	if (ld.x6_op == 0xd)
 961		mb();
 962
 963	return 0;
 964}
 965
 966/*
 967 * floating point operations sizes in bytes
 968 */
 969static const unsigned char float_fsz[4]={
 970	10, /* extended precision (e) */
 971	8,  /* integer (8)            */
 972	4,  /* single precision (s)   */
 973	8   /* double precision (d)   */
 974};
 975
 976static inline void
 977mem2float_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
 978{
 979	ia64_ldfe(6, init);
 980	ia64_stop();
 981	ia64_stf_spill(final, 6);
 982}
 983
 984static inline void
 985mem2float_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
 986{
 987	ia64_ldf8(6, init);
 988	ia64_stop();
 989	ia64_stf_spill(final, 6);
 990}
 991
 992static inline void
 993mem2float_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
 994{
 995	ia64_ldfs(6, init);
 996	ia64_stop();
 997	ia64_stf_spill(final, 6);
 998}
 999
1000static inline void
1001mem2float_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1002{
1003	ia64_ldfd(6, init);
1004	ia64_stop();
1005	ia64_stf_spill(final, 6);
1006}
1007
1008static inline void
1009float2mem_extended (struct ia64_fpreg *init, struct ia64_fpreg *final)
1010{
1011	ia64_ldf_fill(6, init);
1012	ia64_stop();
1013	ia64_stfe(final, 6);
1014}
1015
1016static inline void
1017float2mem_integer (struct ia64_fpreg *init, struct ia64_fpreg *final)
1018{
1019	ia64_ldf_fill(6, init);
1020	ia64_stop();
1021	ia64_stf8(final, 6);
1022}
1023
1024static inline void
1025float2mem_single (struct ia64_fpreg *init, struct ia64_fpreg *final)
1026{
1027	ia64_ldf_fill(6, init);
1028	ia64_stop();
1029	ia64_stfs(final, 6);
1030}
1031
1032static inline void
1033float2mem_double (struct ia64_fpreg *init, struct ia64_fpreg *final)
1034{
1035	ia64_ldf_fill(6, init);
1036	ia64_stop();
1037	ia64_stfd(final, 6);
1038}
1039
1040static int
1041emulate_load_floatpair (unsigned long ifa, load_store_t ld, struct pt_regs *regs, bool kernel_mode)
1042{
1043	struct ia64_fpreg fpr_init[2];
1044	struct ia64_fpreg fpr_final[2];
1045	unsigned long len = float_fsz[ld.x6_sz];
1046
1047	/*
1048	 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1049	 * higher priority than unaligned faults.
1050	 *
1051	 * r0 cannot be found as the base as it would never generate an unaligned
1052	 * reference.
1053	 */
1054
1055	/*
1056	 * make sure we get clean buffers
1057	 */
1058	memset(&fpr_init, 0, sizeof(fpr_init));
1059	memset(&fpr_final, 0, sizeof(fpr_final));
1060
1061	/*
1062	 * ldfpX.a: we don't try to emulate anything but we must
1063	 * invalidate the ALAT entry and execute updates, if any.
1064	 */
1065	if (ld.x6_op != 0x2) {
1066		/*
1067		 * This assumes little-endian byte-order.  Note that there is no "ldfpe"
1068		 * instruction:
1069		 */
1070		if (emulate_load(&fpr_init[0], ifa, len, kernel_mode)
1071		    || emulate_load(&fpr_init[1], (ifa + len), len, kernel_mode))
1072			return -1;
1073
1074		DPRINT("ld.r1=%d ld.imm=%d x6_sz=%d\n", ld.r1, ld.imm, ld.x6_sz);
1075		DDUMP("frp_init =", &fpr_init, 2*len);
1076		/*
1077		 * XXX fixme
1078		 * Could optimize inlines by using ldfpX & 2 spills
1079		 */
1080		switch( ld.x6_sz ) {
1081			case 0:
1082				mem2float_extended(&fpr_init[0], &fpr_final[0]);
1083				mem2float_extended(&fpr_init[1], &fpr_final[1]);
1084				break;
1085			case 1:
1086				mem2float_integer(&fpr_init[0], &fpr_final[0]);
1087				mem2float_integer(&fpr_init[1], &fpr_final[1]);
1088				break;
1089			case 2:
1090				mem2float_single(&fpr_init[0], &fpr_final[0]);
1091				mem2float_single(&fpr_init[1], &fpr_final[1]);
1092				break;
1093			case 3:
1094				mem2float_double(&fpr_init[0], &fpr_final[0]);
1095				mem2float_double(&fpr_init[1], &fpr_final[1]);
1096				break;
1097		}
1098		DDUMP("fpr_final =", &fpr_final, 2*len);
1099		/*
1100		 * XXX fixme
1101		 *
1102		 * A possible optimization would be to drop fpr_final and directly
1103		 * use the storage from the saved context i.e., the actual final
1104		 * destination (pt_regs, switch_stack or thread structure).
1105		 */
1106		setfpreg(ld.r1, &fpr_final[0], regs);
1107		setfpreg(ld.imm, &fpr_final[1], regs);
1108	}
1109
1110	/*
1111	 * Check for updates: only immediate updates are available for this
1112	 * instruction.
1113	 */
1114	if (ld.m) {
1115		/*
1116		 * the immediate is implicit given the ldsz of the operation:
1117		 * single: 8 (2x4) and for  all others it's 16 (2x8)
1118		 */
1119		ifa += len<<1;
1120
1121		/*
1122		 * IMPORTANT:
1123		 * the fact that we force the NaT of r3 to zero is ONLY valid
1124		 * as long as we don't come here with a ldfpX.s.
1125		 * For this reason we keep this sanity check
1126		 */
1127		if (ld.x6_op == 1 || ld.x6_op == 3)
1128			printk(KERN_ERR "%s: register update on speculative load pair, error\n",
1129			       __func__);
1130
1131		setreg(ld.r3, ifa, 0, regs);
1132	}
1133
1134	/*
1135	 * Invalidate ALAT entries, if any, for both registers.
1136	 */
1137	if (ld.x6_op == 0x2) {
1138		invala_fr(ld.r1);
1139		invala_fr(ld.imm);
1140	}
1141	return 0;
1142}
1143
1144
1145static int
1146emulate_load_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs,
1147	            bool kernel_mode)
1148{
1149	struct ia64_fpreg fpr_init;
1150	struct ia64_fpreg fpr_final;
1151	unsigned long len = float_fsz[ld.x6_sz];
1152
1153	/*
1154	 * fr0 & fr1 don't need to be checked because Illegal Instruction
1155	 * faults have higher priority than unaligned faults.
1156	 *
1157	 * r0 cannot be found as the base as it would never generate an
1158	 * unaligned reference.
1159	 */
1160
1161	/*
1162	 * make sure we get clean buffers
1163	 */
1164	memset(&fpr_init,0, sizeof(fpr_init));
1165	memset(&fpr_final,0, sizeof(fpr_final));
1166
1167	/*
1168	 * ldfX.a we don't try to emulate anything but we must
1169	 * invalidate the ALAT entry.
1170	 * See comments in ldX for descriptions on how the various loads are handled.
1171	 */
1172	if (ld.x6_op != 0x2) {
1173		if (emulate_load(&fpr_init, ifa, len, kernel_mode))
1174			return -1;
1175
1176		DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1177		DDUMP("fpr_init =", &fpr_init, len);
1178		/*
1179		 * we only do something for x6_op={0,8,9}
1180		 */
1181		switch( ld.x6_sz ) {
1182			case 0:
1183				mem2float_extended(&fpr_init, &fpr_final);
1184				break;
1185			case 1:
1186				mem2float_integer(&fpr_init, &fpr_final);
1187				break;
1188			case 2:
1189				mem2float_single(&fpr_init, &fpr_final);
1190				break;
1191			case 3:
1192				mem2float_double(&fpr_init, &fpr_final);
1193				break;
1194		}
1195		DDUMP("fpr_final =", &fpr_final, len);
1196		/*
1197		 * XXX fixme
1198		 *
1199		 * A possible optimization would be to drop fpr_final and directly
1200		 * use the storage from the saved context i.e., the actual final
1201		 * destination (pt_regs, switch_stack or thread structure).
1202		 */
1203		setfpreg(ld.r1, &fpr_final, regs);
1204	}
1205
1206	/*
1207	 * check for updates on any loads
1208	 */
1209	if (ld.op == 0x7 || ld.m)
1210		emulate_load_updates(ld.op == 0x7 ? UPD_IMMEDIATE: UPD_REG, ld, regs, ifa);
1211
1212	/*
1213	 * invalidate ALAT entry in case of advanced floating point loads
1214	 */
1215	if (ld.x6_op == 0x2)
1216		invala_fr(ld.r1);
1217
1218	return 0;
1219}
1220
1221
1222static int
1223emulate_store_float (unsigned long ifa, load_store_t ld, struct pt_regs *regs,
1224		     bool kernel_mode)
1225{
1226	struct ia64_fpreg fpr_init;
1227	struct ia64_fpreg fpr_final;
1228	unsigned long len = float_fsz[ld.x6_sz];
1229
1230	/*
1231	 * make sure we get clean buffers
1232	 */
1233	memset(&fpr_init,0, sizeof(fpr_init));
1234	memset(&fpr_final,0, sizeof(fpr_final));
1235
1236	/*
1237	 * if we get to this handler, Nat bits on both r3 and r2 have already
1238	 * been checked. so we don't need to do it
1239	 *
1240	 * extract the value to be stored
1241	 */
1242	getfpreg(ld.imm, &fpr_init, regs);
1243	/*
1244	 * during this step, we extract the spilled registers from the saved
1245	 * context i.e., we refill. Then we store (no spill) to temporary
1246	 * aligned location
1247	 */
1248	switch( ld.x6_sz ) {
1249		case 0:
1250			float2mem_extended(&fpr_init, &fpr_final);
1251			break;
1252		case 1:
1253			float2mem_integer(&fpr_init, &fpr_final);
1254			break;
1255		case 2:
1256			float2mem_single(&fpr_init, &fpr_final);
1257			break;
1258		case 3:
1259			float2mem_double(&fpr_init, &fpr_final);
1260			break;
1261	}
1262	DPRINT("ld.r1=%d x6_sz=%d\n", ld.r1, ld.x6_sz);
1263	DDUMP("fpr_init =", &fpr_init, len);
1264	DDUMP("fpr_final =", &fpr_final, len);
1265
1266	if (emulate_store(ifa, &fpr_final, len, kernel_mode))
1267		return -1;
1268
1269	/*
1270	 * stfX [r3]=r2,imm(9)
1271	 *
1272	 * NOTE:
1273	 * ld.r3 can never be r0, because r0 would not generate an
1274	 * unaligned access.
1275	 */
1276	if (ld.op == 0x7) {
1277		unsigned long imm;
1278
1279		/*
1280		 * form imm9: [12:6] contain first 7bits
1281		 */
1282		imm = ld.x << 7 | ld.r1;
1283		/*
1284		 * sign extend (8bits) if m set
1285		 */
1286		if (ld.m)
1287			imm |= SIGN_EXT9;
1288		/*
1289		 * ifa == r3 (NaT is necessarily cleared)
1290		 */
1291		ifa += imm;
1292
1293		DPRINT("imm=%lx r3=%lx\n", imm, ifa);
1294
1295		setreg(ld.r3, ifa, 0, regs);
1296	}
1297	/*
1298	 * we don't have alat_invalidate_multiple() so we need
1299	 * to do the complete flush :-<<
1300	 */
1301	ia64_invala();
1302
1303	return 0;
1304}
1305
1306/*
1307 * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1308 * eventually fix the program.  However, we don't want to do that for every access so we
1309 * pace it with jiffies.
1310 */
1311static DEFINE_RATELIMIT_STATE(logging_rate_limit, 5 * HZ, 5);
1312
1313void
1314ia64_handle_unaligned (unsigned long ifa, struct pt_regs *regs)
1315{
1316	struct ia64_psr *ipsr = ia64_psr(regs);
 
1317	unsigned long bundle[2];
1318	unsigned long opcode;
 
1319	const struct exception_table_entry *eh = NULL;
1320	union {
1321		unsigned long l;
1322		load_store_t insn;
1323	} u;
1324	int ret = -1;
1325	bool kernel_mode = false;
1326
1327	if (ia64_psr(regs)->be) {
1328		/* we don't support big-endian accesses */
1329		if (die_if_kernel("big-endian unaligned accesses are not supported", regs, 0))
1330			return;
1331		goto force_sigbus;
1332	}
1333
1334	/*
1335	 * Treat kernel accesses for which there is an exception handler entry the same as
1336	 * user-level unaligned accesses.  Otherwise, a clever program could trick this
1337	 * handler into reading an arbitrary kernel addresses...
1338	 */
1339	if (!user_mode(regs))
1340		eh = search_exception_tables(regs->cr_iip + ia64_psr(regs)->ri);
1341	if (user_mode(regs) || eh) {
1342		if ((current->thread.flags & IA64_THREAD_UAC_SIGBUS) != 0)
1343			goto force_sigbus;
1344
1345		if (!no_unaligned_warning &&
1346		    !(current->thread.flags & IA64_THREAD_UAC_NOPRINT) &&
1347		    __ratelimit(&logging_rate_limit))
1348		{
1349			char buf[200];	/* comm[] is at most 16 bytes... */
1350			size_t len;
1351
1352			len = sprintf(buf, "%s(%d): unaligned access to 0x%016lx, "
1353				      "ip=0x%016lx\n\r", current->comm,
1354				      task_pid_nr(current),
1355				      ifa, regs->cr_iip + ipsr->ri);
1356			/*
1357			 * Don't call tty_write_message() if we're in the kernel; we might
1358			 * be holding locks...
1359			 */
1360			if (user_mode(regs)) {
1361				struct tty_struct *tty = get_current_tty();
1362				tty_write_message(tty, buf);
1363				tty_kref_put(tty);
1364			}
1365			buf[len-1] = '\0';	/* drop '\r' */
1366			/* watch for command names containing %s */
1367			printk(KERN_WARNING "%s", buf);
1368		} else {
1369			if (no_unaligned_warning) {
1370				printk_once(KERN_WARNING "%s(%d) encountered an "
1371				       "unaligned exception which required\n"
1372				       "kernel assistance, which degrades "
1373				       "the performance of the application.\n"
1374				       "Unaligned exception warnings have "
1375				       "been disabled by the system "
1376				       "administrator\n"
1377				       "echo 0 > /proc/sys/kernel/ignore-"
1378				       "unaligned-usertrap to re-enable\n",
1379				       current->comm, task_pid_nr(current));
1380			}
1381		}
1382	} else {
1383		if (__ratelimit(&logging_rate_limit)) {
1384			printk(KERN_WARNING "kernel unaligned access to 0x%016lx, ip=0x%016lx\n",
1385			       ifa, regs->cr_iip + ipsr->ri);
1386			if (unaligned_dump_stack)
1387				dump_stack();
1388		}
1389		kernel_mode = true;
1390	}
1391
1392	DPRINT("iip=%lx ifa=%lx isr=%lx (ei=%d, sp=%d)\n",
1393	       regs->cr_iip, ifa, regs->cr_ipsr, ipsr->ri, ipsr->it);
1394
1395	if (emulate_load(bundle, regs->cr_iip, 16, kernel_mode))
1396		goto failure;
1397
1398	/*
1399	 * extract the instruction from the bundle given the slot number
1400	 */
1401	switch (ipsr->ri) {
1402	      default:
1403	      case 0: u.l = (bundle[0] >>  5); break;
1404	      case 1: u.l = (bundle[0] >> 46) | (bundle[1] << 18); break;
1405	      case 2: u.l = (bundle[1] >> 23); break;
1406	}
1407	opcode = (u.l >> IA64_OPCODE_SHIFT) & IA64_OPCODE_MASK;
1408
1409	DPRINT("opcode=%lx ld.qp=%d ld.r1=%d ld.imm=%d ld.r3=%d ld.x=%d ld.hint=%d "
1410	       "ld.x6=0x%x ld.m=%d ld.op=%d\n", opcode, u.insn.qp, u.insn.r1, u.insn.imm,
1411	       u.insn.r3, u.insn.x, u.insn.hint, u.insn.x6_sz, u.insn.m, u.insn.op);
1412
1413	/*
1414	 * IMPORTANT:
1415	 * Notice that the switch statement DOES not cover all possible instructions
1416	 * that DO generate unaligned references. This is made on purpose because for some
1417	 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1418	 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1419	 * the program will get a signal and die:
1420	 *
1421	 *	load/store:
1422	 *		- ldX.spill
1423	 *		- stX.spill
1424	 *	Reason: RNATs are based on addresses
1425	 *		- ld16
1426	 *		- st16
1427	 *	Reason: ld16 and st16 are supposed to occur in a single
1428	 *		memory op
1429	 *
1430	 *	synchronization:
1431	 *		- cmpxchg
1432	 *		- fetchadd
1433	 *		- xchg
1434	 *	Reason: ATOMIC operations cannot be emulated properly using multiple
1435	 *	        instructions.
1436	 *
1437	 *	speculative loads:
1438	 *		- ldX.sZ
1439	 *	Reason: side effects, code must be ready to deal with failure so simpler
1440	 *		to let the load fail.
1441	 * ---------------------------------------------------------------------------------
1442	 * XXX fixme
1443	 *
1444	 * I would like to get rid of this switch case and do something
1445	 * more elegant.
1446	 */
1447	switch (opcode) {
1448	      case LDS_OP:
1449	      case LDSA_OP:
1450		if (u.insn.x)
1451			/* oops, really a semaphore op (cmpxchg, etc) */
1452			goto failure;
1453		fallthrough;
1454	      case LDS_IMM_OP:
1455	      case LDSA_IMM_OP:
1456	      case LDFS_OP:
1457	      case LDFSA_OP:
1458	      case LDFS_IMM_OP:
1459		/*
1460		 * The instruction will be retried with deferred exceptions turned on, and
1461		 * we should get Nat bit installed
1462		 *
1463		 * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1464		 * are actually executed even though the operation failed. So we don't
1465		 * need to take care of this.
1466		 */
1467		DPRINT("forcing PSR_ED\n");
1468		regs->cr_ipsr |= IA64_PSR_ED;
1469		goto done;
1470
1471	      case LD_OP:
1472	      case LDA_OP:
1473	      case LDBIAS_OP:
1474	      case LDACQ_OP:
1475	      case LDCCLR_OP:
1476	      case LDCNC_OP:
1477	      case LDCCLRACQ_OP:
1478		if (u.insn.x)
1479			/* oops, really a semaphore op (cmpxchg, etc) */
1480			goto failure;
1481		fallthrough;
1482	      case LD_IMM_OP:
1483	      case LDA_IMM_OP:
1484	      case LDBIAS_IMM_OP:
1485	      case LDACQ_IMM_OP:
1486	      case LDCCLR_IMM_OP:
1487	      case LDCNC_IMM_OP:
1488	      case LDCCLRACQ_IMM_OP:
1489		ret = emulate_load_int(ifa, u.insn, regs, kernel_mode);
1490		break;
1491
1492	      case ST_OP:
1493	      case STREL_OP:
1494		if (u.insn.x)
1495			/* oops, really a semaphore op (cmpxchg, etc) */
1496			goto failure;
1497		fallthrough;
1498	      case ST_IMM_OP:
1499	      case STREL_IMM_OP:
1500		ret = emulate_store_int(ifa, u.insn, regs, kernel_mode);
1501		break;
1502
1503	      case LDF_OP:
1504	      case LDFA_OP:
1505	      case LDFCCLR_OP:
1506	      case LDFCNC_OP:
1507		if (u.insn.x)
1508			ret = emulate_load_floatpair(ifa, u.insn, regs, kernel_mode);
1509		else
1510			ret = emulate_load_float(ifa, u.insn, regs, kernel_mode);
1511		break;
1512
1513	      case LDF_IMM_OP:
1514	      case LDFA_IMM_OP:
1515	      case LDFCCLR_IMM_OP:
1516	      case LDFCNC_IMM_OP:
1517		ret = emulate_load_float(ifa, u.insn, regs, kernel_mode);
1518		break;
1519
1520	      case STF_OP:
1521	      case STF_IMM_OP:
1522		ret = emulate_store_float(ifa, u.insn, regs, kernel_mode);
1523		break;
1524
1525	      default:
1526		goto failure;
1527	}
1528	DPRINT("ret=%d\n", ret);
1529	if (ret)
1530		goto failure;
1531
1532	if (ipsr->ri == 2)
1533		/*
1534		 * given today's architecture this case is not likely to happen because a
1535		 * memory access instruction (M) can never be in the last slot of a
1536		 * bundle. But let's keep it for now.
1537		 */
1538		regs->cr_iip += 16;
1539	ipsr->ri = (ipsr->ri + 1) & 0x3;
1540
1541	DPRINT("ipsr->ri=%d iip=%lx\n", ipsr->ri, regs->cr_iip);
1542  done:
 
1543	return;
1544
1545  failure:
1546	/* something went wrong... */
1547	if (!user_mode(regs)) {
1548		if (eh) {
1549			ia64_handle_exception(regs, eh);
1550			goto done;
1551		}
1552		if (die_if_kernel("error during unaligned kernel access\n", regs, ret))
1553			return;
1554		/* NOT_REACHED */
1555	}
1556  force_sigbus:
1557	force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) ifa,
1558			0, 0, 0);
 
 
 
 
 
 
1559	goto done;
1560}