1/*
   2 * Handle unaligned accesses by emulation.
   3 *
   4 * This file is subject to the terms and conditions of the GNU General Public
   5 * License.  See the file "COPYING" in the main directory of this archive
   6 * for more details.
   7 *
   8 * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
   9 * Copyright (C) 1999 Silicon Graphics, Inc.
  10 * Copyright (C) 2014 Imagination Technologies Ltd.
  11 *
  12 * This file contains exception handler for address error exception with the
  13 * special capability to execute faulting instructions in software.  The
  14 * handler does not try to handle the case when the program counter points
  15 * to an address not aligned to a word boundary.
  16 *
  17 * Putting data to unaligned addresses is a bad practice even on Intel where
  18 * only the performance is affected.  Much worse is that such code is non-
  19 * portable.  Due to several programs that die on MIPS due to alignment
  20 * problems I decided to implement this handler anyway though I originally
  21 * didn't intend to do this at all for user code.
  22 *
  23 * For now I enable fixing of address errors by default to make life easier.
  24 * I however intend to disable this somewhen in the future when the alignment
  25 * problems with user programs have been fixed.	 For programmers this is the
  26 * right way to go.
  27 *
  28 * Fixing address errors is a per process option.  The option is inherited
  29 * across fork(2) and execve(2) calls.	If you really want to use the
  30 * option in your user programs - I discourage the use of the software
  31 * emulation strongly - use the following code in your userland stuff:
  32 *
  33 * #include <sys/sysmips.h>
  34 *
  35 * ...
  36 * sysmips(MIPS_FIXADE, x);
  37 * ...
  38 *
  39 * The argument x is 0 for disabling software emulation, enabled otherwise.
  40 *
  41 * Below a little program to play around with this feature.
  42 *
  43 * #include <stdio.h>
  44 * #include <sys/sysmips.h>
  45 *
  46 * struct foo {
  47 *	   unsigned char bar[8];
  48 * };
  49 *
  50 * main(int argc, char *argv[])
  51 * {
  52 *	   struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
  53 *	   unsigned int *p = (unsigned int *) (x.bar + 3);
  54 *	   int i;
  55 *
  56 *	   if (argc > 1)
  57 *		   sysmips(MIPS_FIXADE, atoi(argv[1]));
  58 *
  59 *	   printf("*p = %08lx\n", *p);
  60 *
  61 *	   *p = 0xdeadface;
  62 *
  63 *	   for(i = 0; i <= 7; i++)
  64 *	   printf("%02x ", x.bar[i]);
  65 *	   printf("\n");
  66 * }
  67 *
  68 * Coprocessor loads are not supported; I think this case is unimportant
  69 * in the practice.
  70 *
  71 * TODO: Handle ndc (attempted store to doubleword in uncached memory)
  72 *	 exception for the R6000.
  73 *	 A store crossing a page boundary might be executed only partially.
  74 *	 Undo the partial store in this case.
  75 */
  76#include <linux/context_tracking.h>
  77#include <linux/mm.h>
  78#include <linux/signal.h>
  79#include <linux/smp.h>
  80#include <linux/sched.h>
  81#include <linux/debugfs.h>
  82#include <linux/perf_event.h>
  83
  84#include <asm/asm.h>
  85#include <asm/branch.h>
  86#include <asm/byteorder.h>
  87#include <asm/cop2.h>
  88#include <asm/debug.h>
  89#include <asm/fpu.h>
  90#include <asm/fpu_emulator.h>
  91#include <asm/inst.h>
  92#include <asm/uaccess.h>
 
 
  93
  94#define STR(x)	__STR(x)
  95#define __STR(x)  #x
  96
  97enum {
  98	UNALIGNED_ACTION_QUIET,
  99	UNALIGNED_ACTION_SIGNAL,
 100	UNALIGNED_ACTION_SHOW,
 101};
 102#ifdef CONFIG_DEBUG_FS
 103static u32 unaligned_instructions;
 104static u32 unaligned_action;
 105#else
 106#define unaligned_action UNALIGNED_ACTION_QUIET
 107#endif
 108extern void show_registers(struct pt_regs *regs);
 109
 110#ifdef __BIG_ENDIAN
 111#define     _LoadHW(addr, value, res, type)  \
 112do {                                                        \
 113		__asm__ __volatile__ (".set\tnoat\n"        \
 114			"1:\t"type##_lb("%0", "0(%2)")"\n"  \
 115			"2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 116			"sll\t%0, 0x8\n\t"                  \
 117			"or\t%0, $1\n\t"                    \
 118			"li\t%1, 0\n"                       \
 119			"3:\t.set\tat\n\t"                  \
 120			".insn\n\t"                         \
 121			".section\t.fixup,\"ax\"\n\t"       \
 122			"4:\tli\t%1, %3\n\t"                \
 123			"j\t3b\n\t"                         \
 124			".previous\n\t"                     \
 125			".section\t__ex_table,\"a\"\n\t"    \
 126			STR(PTR)"\t1b, 4b\n\t"              \
 127			STR(PTR)"\t2b, 4b\n\t"              \
 128			".previous"                         \
 129			: "=&r" (value), "=r" (res)         \
 130			: "r" (addr), "i" (-EFAULT));       \
 131} while(0)
 132
 133#ifndef CONFIG_CPU_MIPSR6
 134#define     _LoadW(addr, value, res, type)   \
 135do {                                                        \
 136		__asm__ __volatile__ (                      \
 137			"1:\t"type##_lwl("%0", "(%2)")"\n"   \
 138			"2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 139			"li\t%1, 0\n"                       \
 140			"3:\n\t"                            \
 141			".insn\n\t"                         \
 142			".section\t.fixup,\"ax\"\n\t"       \
 143			"4:\tli\t%1, %3\n\t"                \
 144			"j\t3b\n\t"                         \
 145			".previous\n\t"                     \
 146			".section\t__ex_table,\"a\"\n\t"    \
 147			STR(PTR)"\t1b, 4b\n\t"              \
 148			STR(PTR)"\t2b, 4b\n\t"              \
 149			".previous"                         \
 150			: "=&r" (value), "=r" (res)         \
 151			: "r" (addr), "i" (-EFAULT));       \
 152} while(0)
 153
 154#else
 155/* MIPSR6 has no lwl instruction */
 156#define     _LoadW(addr, value, res, type) \
 157do {                                                        \
 158		__asm__ __volatile__ (			    \
 159			".set\tpush\n"			    \
 160			".set\tnoat\n\t"		    \
 161			"1:"type##_lb("%0", "0(%2)")"\n\t"  \
 162			"2:"type##_lbu("$1", "1(%2)")"\n\t" \
 163			"sll\t%0, 0x8\n\t"		    \
 164			"or\t%0, $1\n\t"		    \
 165			"3:"type##_lbu("$1", "2(%2)")"\n\t" \
 166			"sll\t%0, 0x8\n\t"		    \
 167			"or\t%0, $1\n\t"		    \
 168			"4:"type##_lbu("$1", "3(%2)")"\n\t" \
 169			"sll\t%0, 0x8\n\t"		    \
 170			"or\t%0, $1\n\t"		    \
 171			"li\t%1, 0\n"			    \
 172			".set\tpop\n"			    \
 173			"10:\n\t"			    \
 174			".insn\n\t"			    \
 175			".section\t.fixup,\"ax\"\n\t"	    \
 176			"11:\tli\t%1, %3\n\t"		    \
 177			"j\t10b\n\t"			    \
 178			".previous\n\t"			    \
 179			".section\t__ex_table,\"a\"\n\t"    \
 180			STR(PTR)"\t1b, 11b\n\t"		    \
 181			STR(PTR)"\t2b, 11b\n\t"		    \
 182			STR(PTR)"\t3b, 11b\n\t"		    \
 183			STR(PTR)"\t4b, 11b\n\t"		    \
 184			".previous"			    \
 185			: "=&r" (value), "=r" (res)	    \
 186			: "r" (addr), "i" (-EFAULT));       \
 187} while(0)
 188
 189#endif /* CONFIG_CPU_MIPSR6 */
 190
 191#define     _LoadHWU(addr, value, res, type) \
 192do {                                                        \
 193		__asm__ __volatile__ (                      \
 194			".set\tnoat\n"                      \
 195			"1:\t"type##_lbu("%0", "0(%2)")"\n" \
 196			"2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 197			"sll\t%0, 0x8\n\t"                  \
 198			"or\t%0, $1\n\t"                    \
 199			"li\t%1, 0\n"                       \
 200			"3:\n\t"                            \
 201			".insn\n\t"                         \
 202			".set\tat\n\t"                      \
 203			".section\t.fixup,\"ax\"\n\t"       \
 204			"4:\tli\t%1, %3\n\t"                \
 205			"j\t3b\n\t"                         \
 206			".previous\n\t"                     \
 207			".section\t__ex_table,\"a\"\n\t"    \
 208			STR(PTR)"\t1b, 4b\n\t"              \
 209			STR(PTR)"\t2b, 4b\n\t"              \
 210			".previous"                         \
 211			: "=&r" (value), "=r" (res)         \
 212			: "r" (addr), "i" (-EFAULT));       \
 213} while(0)
 214
 215#ifndef CONFIG_CPU_MIPSR6
 216#define     _LoadWU(addr, value, res, type)  \
 217do {                                                        \
 218		__asm__ __volatile__ (                      \
 219			"1:\t"type##_lwl("%0", "(%2)")"\n"  \
 220			"2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 221			"dsll\t%0, %0, 32\n\t"              \
 222			"dsrl\t%0, %0, 32\n\t"              \
 223			"li\t%1, 0\n"                       \
 224			"3:\n\t"                            \
 225			".insn\n\t"                         \
 226			"\t.section\t.fixup,\"ax\"\n\t"     \
 227			"4:\tli\t%1, %3\n\t"                \
 228			"j\t3b\n\t"                         \
 229			".previous\n\t"                     \
 230			".section\t__ex_table,\"a\"\n\t"    \
 231			STR(PTR)"\t1b, 4b\n\t"              \
 232			STR(PTR)"\t2b, 4b\n\t"              \
 233			".previous"                         \
 234			: "=&r" (value), "=r" (res)         \
 235			: "r" (addr), "i" (-EFAULT));       \
 236} while(0)
 237
 238#define     _LoadDW(addr, value, res)  \
 239do {                                                        \
 240		__asm__ __volatile__ (                      \
 241			"1:\tldl\t%0, (%2)\n"               \
 242			"2:\tldr\t%0, 7(%2)\n\t"            \
 243			"li\t%1, 0\n"                       \
 244			"3:\n\t"                            \
 245			".insn\n\t"                         \
 246			"\t.section\t.fixup,\"ax\"\n\t"     \
 247			"4:\tli\t%1, %3\n\t"                \
 248			"j\t3b\n\t"                         \
 249			".previous\n\t"                     \
 250			".section\t__ex_table,\"a\"\n\t"    \
 251			STR(PTR)"\t1b, 4b\n\t"              \
 252			STR(PTR)"\t2b, 4b\n\t"              \
 253			".previous"                         \
 254			: "=&r" (value), "=r" (res)         \
 255			: "r" (addr), "i" (-EFAULT));       \
 256} while(0)
 257
 258#else
 259/* MIPSR6 has not lwl and ldl instructions */
 260#define	    _LoadWU(addr, value, res, type) \
 261do {                                                        \
 262		__asm__ __volatile__ (			    \
 263			".set\tpush\n\t"		    \
 264			".set\tnoat\n\t"		    \
 265			"1:"type##_lbu("%0", "0(%2)")"\n\t" \
 266			"2:"type##_lbu("$1", "1(%2)")"\n\t" \
 267			"sll\t%0, 0x8\n\t"		    \
 268			"or\t%0, $1\n\t"		    \
 269			"3:"type##_lbu("$1", "2(%2)")"\n\t" \
 270			"sll\t%0, 0x8\n\t"		    \
 271			"or\t%0, $1\n\t"		    \
 272			"4:"type##_lbu("$1", "3(%2)")"\n\t" \
 273			"sll\t%0, 0x8\n\t"		    \
 274			"or\t%0, $1\n\t"		    \
 275			"li\t%1, 0\n"			    \
 276			".set\tpop\n"			    \
 277			"10:\n\t"			    \
 278			".insn\n\t"			    \
 279			".section\t.fixup,\"ax\"\n\t"	    \
 280			"11:\tli\t%1, %3\n\t"		    \
 281			"j\t10b\n\t"			    \
 282			".previous\n\t"			    \
 283			".section\t__ex_table,\"a\"\n\t"    \
 284			STR(PTR)"\t1b, 11b\n\t"		    \
 285			STR(PTR)"\t2b, 11b\n\t"		    \
 286			STR(PTR)"\t3b, 11b\n\t"		    \
 287			STR(PTR)"\t4b, 11b\n\t"		    \
 288			".previous"			    \
 289			: "=&r" (value), "=r" (res)	    \
 290			: "r" (addr), "i" (-EFAULT));       \
 291} while(0)
 292
 293#define     _LoadDW(addr, value, res)  \
 294do {                                                        \
 295		__asm__ __volatile__ (			    \
 296			".set\tpush\n\t"		    \
 297			".set\tnoat\n\t"		    \
 298			"1:lb\t%0, 0(%2)\n\t"    	    \
 299			"2:lbu\t $1, 1(%2)\n\t"   	    \
 300			"dsll\t%0, 0x8\n\t"		    \
 301			"or\t%0, $1\n\t"		    \
 302			"3:lbu\t$1, 2(%2)\n\t"   	    \
 303			"dsll\t%0, 0x8\n\t"		    \
 304			"or\t%0, $1\n\t"		    \
 305			"4:lbu\t$1, 3(%2)\n\t"   	    \
 306			"dsll\t%0, 0x8\n\t"		    \
 307			"or\t%0, $1\n\t"		    \
 308			"5:lbu\t$1, 4(%2)\n\t"   	    \
 309			"dsll\t%0, 0x8\n\t"		    \
 310			"or\t%0, $1\n\t"		    \
 311			"6:lbu\t$1, 5(%2)\n\t"   	    \
 312			"dsll\t%0, 0x8\n\t"		    \
 313			"or\t%0, $1\n\t"		    \
 314			"7:lbu\t$1, 6(%2)\n\t"   	    \
 315			"dsll\t%0, 0x8\n\t"		    \
 316			"or\t%0, $1\n\t"		    \
 317			"8:lbu\t$1, 7(%2)\n\t"   	    \
 318			"dsll\t%0, 0x8\n\t"		    \
 319			"or\t%0, $1\n\t"		    \
 320			"li\t%1, 0\n"			    \
 321			".set\tpop\n\t"			    \
 322			"10:\n\t"			    \
 323			".insn\n\t"			    \
 324			".section\t.fixup,\"ax\"\n\t"	    \
 325			"11:\tli\t%1, %3\n\t"		    \
 326			"j\t10b\n\t"			    \
 327			".previous\n\t"			    \
 328			".section\t__ex_table,\"a\"\n\t"    \
 329			STR(PTR)"\t1b, 11b\n\t"		    \
 330			STR(PTR)"\t2b, 11b\n\t"		    \
 331			STR(PTR)"\t3b, 11b\n\t"		    \
 332			STR(PTR)"\t4b, 11b\n\t"		    \
 333			STR(PTR)"\t5b, 11b\n\t"		    \
 334			STR(PTR)"\t6b, 11b\n\t"		    \
 335			STR(PTR)"\t7b, 11b\n\t"		    \
 336			STR(PTR)"\t8b, 11b\n\t"		    \
 337			".previous"			    \
 338			: "=&r" (value), "=r" (res)	    \
 339			: "r" (addr), "i" (-EFAULT));       \
 340} while(0)
 341
 342#endif /* CONFIG_CPU_MIPSR6 */
 343
 344
 345#define     _StoreHW(addr, value, res, type) \
 346do {                                                        \
 347		__asm__ __volatile__ (                      \
 348			".set\tnoat\n"                      \
 349			"1:\t"type##_sb("%1", "1(%2)")"\n"  \
 350			"srl\t$1, %1, 0x8\n"                \
 351			"2:\t"type##_sb("$1", "0(%2)")"\n"  \
 352			".set\tat\n\t"                      \
 353			"li\t%0, 0\n"                       \
 354			"3:\n\t"                            \
 355			".insn\n\t"                         \
 356			".section\t.fixup,\"ax\"\n\t"       \
 357			"4:\tli\t%0, %3\n\t"                \
 358			"j\t3b\n\t"                         \
 359			".previous\n\t"                     \
 360			".section\t__ex_table,\"a\"\n\t"    \
 361			STR(PTR)"\t1b, 4b\n\t"              \
 362			STR(PTR)"\t2b, 4b\n\t"              \
 363			".previous"                         \
 364			: "=r" (res)                        \
 365			: "r" (value), "r" (addr), "i" (-EFAULT));\
 366} while(0)
 367
 368#ifndef CONFIG_CPU_MIPSR6
 369#define     _StoreW(addr, value, res, type)  \
 370do {                                                        \
 371		__asm__ __volatile__ (                      \
 372			"1:\t"type##_swl("%1", "(%2)")"\n"  \
 373			"2:\t"type##_swr("%1", "3(%2)")"\n\t"\
 374			"li\t%0, 0\n"                       \
 375			"3:\n\t"                            \
 376			".insn\n\t"                         \
 377			".section\t.fixup,\"ax\"\n\t"       \
 378			"4:\tli\t%0, %3\n\t"                \
 379			"j\t3b\n\t"                         \
 380			".previous\n\t"                     \
 381			".section\t__ex_table,\"a\"\n\t"    \
 382			STR(PTR)"\t1b, 4b\n\t"              \
 383			STR(PTR)"\t2b, 4b\n\t"              \
 384			".previous"                         \
 385		: "=r" (res)                                \
 386		: "r" (value), "r" (addr), "i" (-EFAULT));  \
 387} while(0)
 388
 389#define     _StoreDW(addr, value, res) \
 390do {                                                        \
 391		__asm__ __volatile__ (                      \
 392			"1:\tsdl\t%1,(%2)\n"                \
 393			"2:\tsdr\t%1, 7(%2)\n\t"            \
 394			"li\t%0, 0\n"                       \
 395			"3:\n\t"                            \
 396			".insn\n\t"                         \
 397			".section\t.fixup,\"ax\"\n\t"       \
 398			"4:\tli\t%0, %3\n\t"                \
 399			"j\t3b\n\t"                         \
 400			".previous\n\t"                     \
 401			".section\t__ex_table,\"a\"\n\t"    \
 402			STR(PTR)"\t1b, 4b\n\t"              \
 403			STR(PTR)"\t2b, 4b\n\t"              \
 404			".previous"                         \
 405		: "=r" (res)                                \
 406		: "r" (value), "r" (addr), "i" (-EFAULT));  \
 407} while(0)
 408
 409#else
 410/* MIPSR6 has no swl and sdl instructions */
 411#define     _StoreW(addr, value, res, type)  \
 412do {                                                        \
 413		__asm__ __volatile__ (                      \
 414			".set\tpush\n\t"		    \
 415			".set\tnoat\n\t"		    \
 416			"1:"type##_sb("%1", "3(%2)")"\n\t"  \
 417			"srl\t$1, %1, 0x8\n\t"		    \
 418			"2:"type##_sb("$1", "2(%2)")"\n\t"  \
 419			"srl\t$1, $1,  0x8\n\t"		    \
 420			"3:"type##_sb("$1", "1(%2)")"\n\t"  \
 421			"srl\t$1, $1, 0x8\n\t"		    \
 422			"4:"type##_sb("$1", "0(%2)")"\n\t"  \
 423			".set\tpop\n\t"			    \
 424			"li\t%0, 0\n"			    \
 425			"10:\n\t"			    \
 426			".insn\n\t"			    \
 427			".section\t.fixup,\"ax\"\n\t"	    \
 428			"11:\tli\t%0, %3\n\t"		    \
 429			"j\t10b\n\t"			    \
 430			".previous\n\t"			    \
 431			".section\t__ex_table,\"a\"\n\t"    \
 432			STR(PTR)"\t1b, 11b\n\t"		    \
 433			STR(PTR)"\t2b, 11b\n\t"		    \
 434			STR(PTR)"\t3b, 11b\n\t"		    \
 435			STR(PTR)"\t4b, 11b\n\t"		    \
 436			".previous"			    \
 437		: "=&r" (res)			    	    \
 438		: "r" (value), "r" (addr), "i" (-EFAULT)    \
 439		: "memory");                                \
 440} while(0)
 441
 442#define     _StoreDW(addr, value, res) \
 443do {                                                        \
 444		__asm__ __volatile__ (                      \
 445			".set\tpush\n\t"		    \
 446			".set\tnoat\n\t"		    \
 447			"1:sb\t%1, 7(%2)\n\t"    	    \
 448			"dsrl\t$1, %1, 0x8\n\t"		    \
 449			"2:sb\t$1, 6(%2)\n\t"    	    \
 450			"dsrl\t$1, $1, 0x8\n\t"		    \
 451			"3:sb\t$1, 5(%2)\n\t"    	    \
 452			"dsrl\t$1, $1, 0x8\n\t"		    \
 453			"4:sb\t$1, 4(%2)\n\t"    	    \
 454			"dsrl\t$1, $1, 0x8\n\t"		    \
 455			"5:sb\t$1, 3(%2)\n\t"    	    \
 456			"dsrl\t$1, $1, 0x8\n\t"		    \
 457			"6:sb\t$1, 2(%2)\n\t"    	    \
 458			"dsrl\t$1, $1, 0x8\n\t"		    \
 459			"7:sb\t$1, 1(%2)\n\t"    	    \
 460			"dsrl\t$1, $1, 0x8\n\t"		    \
 461			"8:sb\t$1, 0(%2)\n\t"    	    \
 462			"dsrl\t$1, $1, 0x8\n\t"		    \
 463			".set\tpop\n\t"			    \
 464			"li\t%0, 0\n"			    \
 465			"10:\n\t"			    \
 466			".insn\n\t"			    \
 467			".section\t.fixup,\"ax\"\n\t"	    \
 468			"11:\tli\t%0, %3\n\t"		    \
 469			"j\t10b\n\t"			    \
 470			".previous\n\t"			    \
 471			".section\t__ex_table,\"a\"\n\t"    \
 472			STR(PTR)"\t1b, 11b\n\t"		    \
 473			STR(PTR)"\t2b, 11b\n\t"		    \
 474			STR(PTR)"\t3b, 11b\n\t"		    \
 475			STR(PTR)"\t4b, 11b\n\t"		    \
 476			STR(PTR)"\t5b, 11b\n\t"		    \
 477			STR(PTR)"\t6b, 11b\n\t"		    \
 478			STR(PTR)"\t7b, 11b\n\t"		    \
 479			STR(PTR)"\t8b, 11b\n\t"		    \
 480			".previous"			    \
 481		: "=&r" (res)			    	    \
 482		: "r" (value), "r" (addr), "i" (-EFAULT)    \
 483		: "memory");                                \
 484} while(0)
 485
 486#endif /* CONFIG_CPU_MIPSR6 */
 487
 488#else /* __BIG_ENDIAN */
 489
 490#define     _LoadHW(addr, value, res, type)  \
 491do {                                                        \
 492		__asm__ __volatile__ (".set\tnoat\n"        \
 493			"1:\t"type##_lb("%0", "1(%2)")"\n"  \
 494			"2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 495			"sll\t%0, 0x8\n\t"                  \
 496			"or\t%0, $1\n\t"                    \
 497			"li\t%1, 0\n"                       \
 498			"3:\t.set\tat\n\t"                  \
 499			".insn\n\t"                         \
 500			".section\t.fixup,\"ax\"\n\t"       \
 501			"4:\tli\t%1, %3\n\t"                \
 502			"j\t3b\n\t"                         \
 503			".previous\n\t"                     \
 504			".section\t__ex_table,\"a\"\n\t"    \
 505			STR(PTR)"\t1b, 4b\n\t"              \
 506			STR(PTR)"\t2b, 4b\n\t"              \
 507			".previous"                         \
 508			: "=&r" (value), "=r" (res)         \
 509			: "r" (addr), "i" (-EFAULT));       \
 510} while(0)
 511
 512#ifndef CONFIG_CPU_MIPSR6
 513#define     _LoadW(addr, value, res, type)   \
 514do {                                                        \
 515		__asm__ __volatile__ (                      \
 516			"1:\t"type##_lwl("%0", "3(%2)")"\n" \
 517			"2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 518			"li\t%1, 0\n"                       \
 519			"3:\n\t"                            \
 520			".insn\n\t"                         \
 521			".section\t.fixup,\"ax\"\n\t"       \
 522			"4:\tli\t%1, %3\n\t"                \
 523			"j\t3b\n\t"                         \
 524			".previous\n\t"                     \
 525			".section\t__ex_table,\"a\"\n\t"    \
 526			STR(PTR)"\t1b, 4b\n\t"              \
 527			STR(PTR)"\t2b, 4b\n\t"              \
 528			".previous"                         \
 529			: "=&r" (value), "=r" (res)         \
 530			: "r" (addr), "i" (-EFAULT));       \
 531} while(0)
 532
 533#else
 534/* MIPSR6 has no lwl instruction */
 535#define     _LoadW(addr, value, res, type) \
 536do {                                                        \
 537		__asm__ __volatile__ (			    \
 538			".set\tpush\n"			    \
 539			".set\tnoat\n\t"		    \
 540			"1:"type##_lb("%0", "3(%2)")"\n\t"  \
 541			"2:"type##_lbu("$1", "2(%2)")"\n\t" \
 542			"sll\t%0, 0x8\n\t"		    \
 543			"or\t%0, $1\n\t"		    \
 544			"3:"type##_lbu("$1", "1(%2)")"\n\t" \
 545			"sll\t%0, 0x8\n\t"		    \
 546			"or\t%0, $1\n\t"		    \
 547			"4:"type##_lbu("$1", "0(%2)")"\n\t" \
 548			"sll\t%0, 0x8\n\t"		    \
 549			"or\t%0, $1\n\t"		    \
 550			"li\t%1, 0\n"			    \
 551			".set\tpop\n"			    \
 552			"10:\n\t"			    \
 553			".insn\n\t"			    \
 554			".section\t.fixup,\"ax\"\n\t"	    \
 555			"11:\tli\t%1, %3\n\t"		    \
 556			"j\t10b\n\t"			    \
 557			".previous\n\t"			    \
 558			".section\t__ex_table,\"a\"\n\t"    \
 559			STR(PTR)"\t1b, 11b\n\t"		    \
 560			STR(PTR)"\t2b, 11b\n\t"		    \
 561			STR(PTR)"\t3b, 11b\n\t"		    \
 562			STR(PTR)"\t4b, 11b\n\t"		    \
 563			".previous"			    \
 564			: "=&r" (value), "=r" (res)	    \
 565			: "r" (addr), "i" (-EFAULT));       \
 566} while(0)
 567
 568#endif /* CONFIG_CPU_MIPSR6 */
 569
 570
 571#define     _LoadHWU(addr, value, res, type) \
 572do {                                                        \
 573		__asm__ __volatile__ (                      \
 574			".set\tnoat\n"                      \
 575			"1:\t"type##_lbu("%0", "1(%2)")"\n" \
 576			"2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 577			"sll\t%0, 0x8\n\t"                  \
 578			"or\t%0, $1\n\t"                    \
 579			"li\t%1, 0\n"                       \
 580			"3:\n\t"                            \
 581			".insn\n\t"                         \
 582			".set\tat\n\t"                      \
 583			".section\t.fixup,\"ax\"\n\t"       \
 584			"4:\tli\t%1, %3\n\t"                \
 585			"j\t3b\n\t"                         \
 586			".previous\n\t"                     \
 587			".section\t__ex_table,\"a\"\n\t"    \
 588			STR(PTR)"\t1b, 4b\n\t"              \
 589			STR(PTR)"\t2b, 4b\n\t"              \
 590			".previous"                         \
 591			: "=&r" (value), "=r" (res)         \
 592			: "r" (addr), "i" (-EFAULT));       \
 593} while(0)
 594
 595#ifndef CONFIG_CPU_MIPSR6
 596#define     _LoadWU(addr, value, res, type)  \
 597do {                                                        \
 598		__asm__ __volatile__ (                      \
 599			"1:\t"type##_lwl("%0", "3(%2)")"\n" \
 600			"2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 601			"dsll\t%0, %0, 32\n\t"              \
 602			"dsrl\t%0, %0, 32\n\t"              \
 603			"li\t%1, 0\n"                       \
 604			"3:\n\t"                            \
 605			".insn\n\t"                         \
 606			"\t.section\t.fixup,\"ax\"\n\t"     \
 607			"4:\tli\t%1, %3\n\t"                \
 608			"j\t3b\n\t"                         \
 609			".previous\n\t"                     \
 610			".section\t__ex_table,\"a\"\n\t"    \
 611			STR(PTR)"\t1b, 4b\n\t"              \
 612			STR(PTR)"\t2b, 4b\n\t"              \
 613			".previous"                         \
 614			: "=&r" (value), "=r" (res)         \
 615			: "r" (addr), "i" (-EFAULT));       \
 616} while(0)
 617
 618#define     _LoadDW(addr, value, res)  \
 619do {                                                        \
 620		__asm__ __volatile__ (                      \
 621			"1:\tldl\t%0, 7(%2)\n"              \
 622			"2:\tldr\t%0, (%2)\n\t"             \
 623			"li\t%1, 0\n"                       \
 624			"3:\n\t"                            \
 625			".insn\n\t"                         \
 626			"\t.section\t.fixup,\"ax\"\n\t"     \
 627			"4:\tli\t%1, %3\n\t"                \
 628			"j\t3b\n\t"                         \
 629			".previous\n\t"                     \
 630			".section\t__ex_table,\"a\"\n\t"    \
 631			STR(PTR)"\t1b, 4b\n\t"              \
 632			STR(PTR)"\t2b, 4b\n\t"              \
 633			".previous"                         \
 634			: "=&r" (value), "=r" (res)         \
 635			: "r" (addr), "i" (-EFAULT));       \
 636} while(0)
 637
 638#else
 639/* MIPSR6 has not lwl and ldl instructions */
 640#define	    _LoadWU(addr, value, res, type) \
 641do {                                                        \
 642		__asm__ __volatile__ (			    \
 643			".set\tpush\n\t"		    \
 644			".set\tnoat\n\t"		    \
 645			"1:"type##_lbu("%0", "3(%2)")"\n\t" \
 646			"2:"type##_lbu("$1", "2(%2)")"\n\t" \
 647			"sll\t%0, 0x8\n\t"		    \
 648			"or\t%0, $1\n\t"		    \
 649			"3:"type##_lbu("$1", "1(%2)")"\n\t" \
 650			"sll\t%0, 0x8\n\t"		    \
 651			"or\t%0, $1\n\t"		    \
 652			"4:"type##_lbu("$1", "0(%2)")"\n\t" \
 653			"sll\t%0, 0x8\n\t"		    \
 654			"or\t%0, $1\n\t"		    \
 655			"li\t%1, 0\n"			    \
 656			".set\tpop\n"			    \
 657			"10:\n\t"			    \
 658			".insn\n\t"			    \
 659			".section\t.fixup,\"ax\"\n\t"	    \
 660			"11:\tli\t%1, %3\n\t"		    \
 661			"j\t10b\n\t"			    \
 662			".previous\n\t"			    \
 663			".section\t__ex_table,\"a\"\n\t"    \
 664			STR(PTR)"\t1b, 11b\n\t"		    \
 665			STR(PTR)"\t2b, 11b\n\t"		    \
 666			STR(PTR)"\t3b, 11b\n\t"		    \
 667			STR(PTR)"\t4b, 11b\n\t"		    \
 668			".previous"			    \
 669			: "=&r" (value), "=r" (res)	    \
 670			: "r" (addr), "i" (-EFAULT));       \
 671} while(0)
 672
 673#define     _LoadDW(addr, value, res)  \
 674do {                                                        \
 675		__asm__ __volatile__ (			    \
 676			".set\tpush\n\t"		    \
 677			".set\tnoat\n\t"		    \
 678			"1:lb\t%0, 7(%2)\n\t"    	    \
 679			"2:lbu\t$1, 6(%2)\n\t"   	    \
 680			"dsll\t%0, 0x8\n\t"		    \
 681			"or\t%0, $1\n\t"		    \
 682			"3:lbu\t$1, 5(%2)\n\t"   	    \
 683			"dsll\t%0, 0x8\n\t"		    \
 684			"or\t%0, $1\n\t"		    \
 685			"4:lbu\t$1, 4(%2)\n\t"   	    \
 686			"dsll\t%0, 0x8\n\t"		    \
 687			"or\t%0, $1\n\t"		    \
 688			"5:lbu\t$1, 3(%2)\n\t"   	    \
 689			"dsll\t%0, 0x8\n\t"		    \
 690			"or\t%0, $1\n\t"		    \
 691			"6:lbu\t$1, 2(%2)\n\t"   	    \
 692			"dsll\t%0, 0x8\n\t"		    \
 693			"or\t%0, $1\n\t"		    \
 694			"7:lbu\t$1, 1(%2)\n\t"   	    \
 695			"dsll\t%0, 0x8\n\t"		    \
 696			"or\t%0, $1\n\t"		    \
 697			"8:lbu\t$1, 0(%2)\n\t"   	    \
 698			"dsll\t%0, 0x8\n\t"		    \
 699			"or\t%0, $1\n\t"		    \
 700			"li\t%1, 0\n"			    \
 701			".set\tpop\n\t"			    \
 702			"10:\n\t"			    \
 703			".insn\n\t"			    \
 704			".section\t.fixup,\"ax\"\n\t"	    \
 705			"11:\tli\t%1, %3\n\t"		    \
 706			"j\t10b\n\t"			    \
 707			".previous\n\t"			    \
 708			".section\t__ex_table,\"a\"\n\t"    \
 709			STR(PTR)"\t1b, 11b\n\t"		    \
 710			STR(PTR)"\t2b, 11b\n\t"		    \
 711			STR(PTR)"\t3b, 11b\n\t"		    \
 712			STR(PTR)"\t4b, 11b\n\t"		    \
 713			STR(PTR)"\t5b, 11b\n\t"		    \
 714			STR(PTR)"\t6b, 11b\n\t"		    \
 715			STR(PTR)"\t7b, 11b\n\t"		    \
 716			STR(PTR)"\t8b, 11b\n\t"		    \
 717			".previous"			    \
 718			: "=&r" (value), "=r" (res)	    \
 719			: "r" (addr), "i" (-EFAULT));       \
 720} while(0)
 721#endif /* CONFIG_CPU_MIPSR6 */
 722
 723#define     _StoreHW(addr, value, res, type) \
 724do {                                                        \
 725		__asm__ __volatile__ (                      \
 726			".set\tnoat\n"                      \
 727			"1:\t"type##_sb("%1", "0(%2)")"\n"  \
 728			"srl\t$1,%1, 0x8\n"                 \
 729			"2:\t"type##_sb("$1", "1(%2)")"\n"  \
 730			".set\tat\n\t"                      \
 731			"li\t%0, 0\n"                       \
 732			"3:\n\t"                            \
 733			".insn\n\t"                         \
 734			".section\t.fixup,\"ax\"\n\t"       \
 735			"4:\tli\t%0, %3\n\t"                \
 736			"j\t3b\n\t"                         \
 737			".previous\n\t"                     \
 738			".section\t__ex_table,\"a\"\n\t"    \
 739			STR(PTR)"\t1b, 4b\n\t"              \
 740			STR(PTR)"\t2b, 4b\n\t"              \
 741			".previous"                         \
 742			: "=r" (res)                        \
 743			: "r" (value), "r" (addr), "i" (-EFAULT));\
 744} while(0)
 745
 746#ifndef CONFIG_CPU_MIPSR6
 747#define     _StoreW(addr, value, res, type)  \
 748do {                                                        \
 749		__asm__ __volatile__ (                      \
 750			"1:\t"type##_swl("%1", "3(%2)")"\n" \
 751			"2:\t"type##_swr("%1", "(%2)")"\n\t"\
 752			"li\t%0, 0\n"                       \
 753			"3:\n\t"                            \
 754			".insn\n\t"                         \
 755			".section\t.fixup,\"ax\"\n\t"       \
 756			"4:\tli\t%0, %3\n\t"                \
 757			"j\t3b\n\t"                         \
 758			".previous\n\t"                     \
 759			".section\t__ex_table,\"a\"\n\t"    \
 760			STR(PTR)"\t1b, 4b\n\t"              \
 761			STR(PTR)"\t2b, 4b\n\t"              \
 762			".previous"                         \
 763		: "=r" (res)                                \
 764		: "r" (value), "r" (addr), "i" (-EFAULT));  \
 765} while(0)
 766
 767#define     _StoreDW(addr, value, res) \
 768do {                                                        \
 769		__asm__ __volatile__ (                      \
 770			"1:\tsdl\t%1, 7(%2)\n"              \
 771			"2:\tsdr\t%1, (%2)\n\t"             \
 772			"li\t%0, 0\n"                       \
 773			"3:\n\t"                            \
 774			".insn\n\t"                         \
 775			".section\t.fixup,\"ax\"\n\t"       \
 776			"4:\tli\t%0, %3\n\t"                \
 777			"j\t3b\n\t"                         \
 778			".previous\n\t"                     \
 779			".section\t__ex_table,\"a\"\n\t"    \
 780			STR(PTR)"\t1b, 4b\n\t"              \
 781			STR(PTR)"\t2b, 4b\n\t"              \
 782			".previous"                         \
 783		: "=r" (res)                                \
 784		: "r" (value), "r" (addr), "i" (-EFAULT));  \
 785} while(0)
 786
 787#else
 788/* MIPSR6 has no swl and sdl instructions */
 789#define     _StoreW(addr, value, res, type)  \
 790do {                                                        \
 791		__asm__ __volatile__ (                      \
 792			".set\tpush\n\t"		    \
 793			".set\tnoat\n\t"		    \
 794			"1:"type##_sb("%1", "0(%2)")"\n\t"  \
 795			"srl\t$1, %1, 0x8\n\t"		    \
 796			"2:"type##_sb("$1", "1(%2)")"\n\t"  \
 797			"srl\t$1, $1,  0x8\n\t"		    \
 798			"3:"type##_sb("$1", "2(%2)")"\n\t"  \
 799			"srl\t$1, $1, 0x8\n\t"		    \
 800			"4:"type##_sb("$1", "3(%2)")"\n\t"  \
 801			".set\tpop\n\t"			    \
 802			"li\t%0, 0\n"			    \
 803			"10:\n\t"			    \
 804			".insn\n\t"			    \
 805			".section\t.fixup,\"ax\"\n\t"	    \
 806			"11:\tli\t%0, %3\n\t"		    \
 807			"j\t10b\n\t"			    \
 808			".previous\n\t"			    \
 809			".section\t__ex_table,\"a\"\n\t"    \
 810			STR(PTR)"\t1b, 11b\n\t"		    \
 811			STR(PTR)"\t2b, 11b\n\t"		    \
 812			STR(PTR)"\t3b, 11b\n\t"		    \
 813			STR(PTR)"\t4b, 11b\n\t"		    \
 814			".previous"			    \
 815		: "=&r" (res)			    	    \
 816		: "r" (value), "r" (addr), "i" (-EFAULT)    \
 817		: "memory");                                \
 818} while(0)
 819
 820#define     _StoreDW(addr, value, res) \
 821do {                                                        \
 822		__asm__ __volatile__ (                      \
 823			".set\tpush\n\t"		    \
 824			".set\tnoat\n\t"		    \
 825			"1:sb\t%1, 0(%2)\n\t"    	    \
 826			"dsrl\t$1, %1, 0x8\n\t"		    \
 827			"2:sb\t$1, 1(%2)\n\t"    	    \
 828			"dsrl\t$1, $1, 0x8\n\t"		    \
 829			"3:sb\t$1, 2(%2)\n\t"    	    \
 830			"dsrl\t$1, $1, 0x8\n\t"		    \
 831			"4:sb\t$1, 3(%2)\n\t"    	    \
 832			"dsrl\t$1, $1, 0x8\n\t"		    \
 833			"5:sb\t$1, 4(%2)\n\t"    	    \
 834			"dsrl\t$1, $1, 0x8\n\t"		    \
 835			"6:sb\t$1, 5(%2)\n\t"    	    \
 836			"dsrl\t$1, $1, 0x8\n\t"		    \
 837			"7:sb\t$1, 6(%2)\n\t"    	    \
 838			"dsrl\t$1, $1, 0x8\n\t"		    \
 839			"8:sb\t$1, 7(%2)\n\t"    	    \
 840			"dsrl\t$1, $1, 0x8\n\t"		    \
 841			".set\tpop\n\t"			    \
 842			"li\t%0, 0\n"			    \
 843			"10:\n\t"			    \
 844			".insn\n\t"			    \
 845			".section\t.fixup,\"ax\"\n\t"	    \
 846			"11:\tli\t%0, %3\n\t"		    \
 847			"j\t10b\n\t"			    \
 848			".previous\n\t"			    \
 849			".section\t__ex_table,\"a\"\n\t"    \
 850			STR(PTR)"\t1b, 11b\n\t"		    \
 851			STR(PTR)"\t2b, 11b\n\t"		    \
 852			STR(PTR)"\t3b, 11b\n\t"		    \
 853			STR(PTR)"\t4b, 11b\n\t"		    \
 854			STR(PTR)"\t5b, 11b\n\t"		    \
 855			STR(PTR)"\t6b, 11b\n\t"		    \
 856			STR(PTR)"\t7b, 11b\n\t"		    \
 857			STR(PTR)"\t8b, 11b\n\t"		    \
 858			".previous"			    \
 859		: "=&r" (res)			    	    \
 860		: "r" (value), "r" (addr), "i" (-EFAULT)    \
 861		: "memory");                                \
 862} while(0)
 863
 864#endif /* CONFIG_CPU_MIPSR6 */
 865#endif
 866
 867#define LoadHWU(addr, value, res)	_LoadHWU(addr, value, res, kernel)
 868#define LoadHWUE(addr, value, res)	_LoadHWU(addr, value, res, user)
 869#define LoadWU(addr, value, res)	_LoadWU(addr, value, res, kernel)
 870#define LoadWUE(addr, value, res)	_LoadWU(addr, value, res, user)
 871#define LoadHW(addr, value, res)	_LoadHW(addr, value, res, kernel)
 872#define LoadHWE(addr, value, res)	_LoadHW(addr, value, res, user)
 873#define LoadW(addr, value, res)		_LoadW(addr, value, res, kernel)
 874#define LoadWE(addr, value, res)	_LoadW(addr, value, res, user)
 875#define LoadDW(addr, value, res)	_LoadDW(addr, value, res)
 876
 877#define StoreHW(addr, value, res)	_StoreHW(addr, value, res, kernel)
 878#define StoreHWE(addr, value, res)	_StoreHW(addr, value, res, user)
 879#define StoreW(addr, value, res)	_StoreW(addr, value, res, kernel)
 880#define StoreWE(addr, value, res)	_StoreW(addr, value, res, user)
 881#define StoreDW(addr, value, res)	_StoreDW(addr, value, res)
 882
 883static void emulate_load_store_insn(struct pt_regs *regs,
 884	void __user *addr, unsigned int __user *pc)
 885{
 886	union mips_instruction insn;
 887	unsigned long value;
 888	unsigned int res, preempted;
 889	unsigned long origpc;
 890	unsigned long orig31;
 891	void __user *fault_addr = NULL;
 892#ifdef	CONFIG_EVA
 893	mm_segment_t seg;
 894#endif
 895	union fpureg *fpr;
 896	enum msa_2b_fmt df;
 897	unsigned int wd;
 898	origpc = (unsigned long)pc;
 899	orig31 = regs->regs[31];
 900
 901	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
 902
 903	/*
 904	 * This load never faults.
 905	 */
 906	__get_user(insn.word, pc);
 907
 908	switch (insn.i_format.opcode) {
 909		/*
 910		 * These are instructions that a compiler doesn't generate.  We
 911		 * can assume therefore that the code is MIPS-aware and
 912		 * really buggy.  Emulating these instructions would break the
 913		 * semantics anyway.
 914		 */
 915	case ll_op:
 916	case lld_op:
 917	case sc_op:
 918	case scd_op:
 919
 920		/*
 921		 * For these instructions the only way to create an address
 922		 * error is an attempted access to kernel/supervisor address
 923		 * space.
 924		 */
 925	case ldl_op:
 926	case ldr_op:
 927	case lwl_op:
 928	case lwr_op:
 929	case sdl_op:
 930	case sdr_op:
 931	case swl_op:
 932	case swr_op:
 933	case lb_op:
 934	case lbu_op:
 935	case sb_op:
 936		goto sigbus;
 937
 938		/*
 939		 * The remaining opcodes are the ones that are really of
 940		 * interest.
 941		 */
 942#ifdef CONFIG_EVA
 943	case spec3_op:
 944		/*
 945		 * we can land here only from kernel accessing user memory,
 946		 * so we need to "switch" the address limit to user space, so
 947		 * address check can work properly.
 948		 */
 949		seg = get_fs();
 950		set_fs(USER_DS);
 951		switch (insn.spec3_format.func) {
 952		case lhe_op:
 953			if (!access_ok(VERIFY_READ, addr, 2)) {
 954				set_fs(seg);
 955				goto sigbus;
 956			}
 957			LoadHWE(addr, value, res);
 958			if (res) {
 959				set_fs(seg);
 960				goto fault;
 961			}
 962			compute_return_epc(regs);
 963			regs->regs[insn.spec3_format.rt] = value;
 964			break;
 965		case lwe_op:
 966			if (!access_ok(VERIFY_READ, addr, 4)) {
 967				set_fs(seg);
 968				goto sigbus;
 969			}
 970				LoadWE(addr, value, res);
 971			if (res) {
 972				set_fs(seg);
 973				goto fault;
 974			}
 975			compute_return_epc(regs);
 976			regs->regs[insn.spec3_format.rt] = value;
 977			break;
 978		case lhue_op:
 979			if (!access_ok(VERIFY_READ, addr, 2)) {
 980				set_fs(seg);
 981				goto sigbus;
 982			}
 983			LoadHWUE(addr, value, res);
 984			if (res) {
 985				set_fs(seg);
 986				goto fault;
 987			}
 988			compute_return_epc(regs);
 989			regs->regs[insn.spec3_format.rt] = value;
 990			break;
 991		case she_op:
 992			if (!access_ok(VERIFY_WRITE, addr, 2)) {
 993				set_fs(seg);
 994				goto sigbus;
 995			}
 996			compute_return_epc(regs);
 997			value = regs->regs[insn.spec3_format.rt];
 998			StoreHWE(addr, value, res);
 999			if (res) {
1000				set_fs(seg);
1001				goto fault;
1002			}
1003			break;
1004		case swe_op:
1005			if (!access_ok(VERIFY_WRITE, addr, 4)) {
1006				set_fs(seg);
1007				goto sigbus;
1008			}
1009			compute_return_epc(regs);
1010			value = regs->regs[insn.spec3_format.rt];
1011			StoreWE(addr, value, res);
1012			if (res) {
1013				set_fs(seg);
1014				goto fault;
1015			}
1016			break;
1017		default:
1018			set_fs(seg);
1019			goto sigill;
1020		}
1021		set_fs(seg);
1022		break;
1023#endif
1024	case lh_op:
1025		if (!access_ok(VERIFY_READ, addr, 2))
1026			goto sigbus;
1027
1028		if (config_enabled(CONFIG_EVA)) {
1029			if (segment_eq(get_fs(), get_ds()))
1030				LoadHW(addr, value, res);
1031			else
1032				LoadHWE(addr, value, res);
1033		} else {
1034			LoadHW(addr, value, res);
1035		}
1036
1037		if (res)
1038			goto fault;
1039		compute_return_epc(regs);
1040		regs->regs[insn.i_format.rt] = value;
1041		break;
1042
1043	case lw_op:
1044		if (!access_ok(VERIFY_READ, addr, 4))
1045			goto sigbus;
1046
1047		if (config_enabled(CONFIG_EVA)) {
1048			if (segment_eq(get_fs(), get_ds()))
1049				LoadW(addr, value, res);
1050			else
1051				LoadWE(addr, value, res);
1052		} else {
1053			LoadW(addr, value, res);
1054		}
1055
1056		if (res)
1057			goto fault;
1058		compute_return_epc(regs);
1059		regs->regs[insn.i_format.rt] = value;
1060		break;
1061
1062	case lhu_op:
1063		if (!access_ok(VERIFY_READ, addr, 2))
1064			goto sigbus;
1065
1066		if (config_enabled(CONFIG_EVA)) {
1067			if (segment_eq(get_fs(), get_ds()))
1068				LoadHWU(addr, value, res);
1069			else
1070				LoadHWUE(addr, value, res);
1071		} else {
1072			LoadHWU(addr, value, res);
1073		}
1074
1075		if (res)
1076			goto fault;
1077		compute_return_epc(regs);
1078		regs->regs[insn.i_format.rt] = value;
1079		break;
1080
1081	case lwu_op:
1082#ifdef CONFIG_64BIT
1083		/*
1084		 * A 32-bit kernel might be running on a 64-bit processor.  But
1085		 * if we're on a 32-bit processor and an i-cache incoherency
1086		 * or race makes us see a 64-bit instruction here the sdl/sdr
1087		 * would blow up, so for now we don't handle unaligned 64-bit
1088		 * instructions on 32-bit kernels.
1089		 */
1090		if (!access_ok(VERIFY_READ, addr, 4))
1091			goto sigbus;
1092
1093		LoadWU(addr, value, res);
1094		if (res)
1095			goto fault;
1096		compute_return_epc(regs);
1097		regs->regs[insn.i_format.rt] = value;
1098		break;
1099#endif /* CONFIG_64BIT */
1100
1101		/* Cannot handle 64-bit instructions in 32-bit kernel */
1102		goto sigill;
1103
1104	case ld_op:
1105#ifdef CONFIG_64BIT
1106		/*
1107		 * A 32-bit kernel might be running on a 64-bit processor.  But
1108		 * if we're on a 32-bit processor and an i-cache incoherency
1109		 * or race makes us see a 64-bit instruction here the sdl/sdr
1110		 * would blow up, so for now we don't handle unaligned 64-bit
1111		 * instructions on 32-bit kernels.
1112		 */
1113		if (!access_ok(VERIFY_READ, addr, 8))
1114			goto sigbus;
1115
1116		LoadDW(addr, value, res);
1117		if (res)
1118			goto fault;
1119		compute_return_epc(regs);
1120		regs->regs[insn.i_format.rt] = value;
1121		break;
1122#endif /* CONFIG_64BIT */
1123
1124		/* Cannot handle 64-bit instructions in 32-bit kernel */
1125		goto sigill;
1126
1127	case sh_op:
1128		if (!access_ok(VERIFY_WRITE, addr, 2))
1129			goto sigbus;
1130
1131		compute_return_epc(regs);
1132		value = regs->regs[insn.i_format.rt];
1133
1134		if (config_enabled(CONFIG_EVA)) {
1135			if (segment_eq(get_fs(), get_ds()))
1136				StoreHW(addr, value, res);
1137			else
1138				StoreHWE(addr, value, res);
1139		} else {
1140			StoreHW(addr, value, res);
1141		}
1142
1143		if (res)
1144			goto fault;
1145		break;
1146
1147	case sw_op:
1148		if (!access_ok(VERIFY_WRITE, addr, 4))
1149			goto sigbus;
1150
1151		compute_return_epc(regs);
1152		value = regs->regs[insn.i_format.rt];
1153
1154		if (config_enabled(CONFIG_EVA)) {
1155			if (segment_eq(get_fs(), get_ds()))
1156				StoreW(addr, value, res);
1157			else
1158				StoreWE(addr, value, res);
1159		} else {
1160			StoreW(addr, value, res);
1161		}
1162
1163		if (res)
1164			goto fault;
1165		break;
1166
1167	case sd_op:
1168#ifdef CONFIG_64BIT
1169		/*
1170		 * A 32-bit kernel might be running on a 64-bit processor.  But
1171		 * if we're on a 32-bit processor and an i-cache incoherency
1172		 * or race makes us see a 64-bit instruction here the sdl/sdr
1173		 * would blow up, so for now we don't handle unaligned 64-bit
1174		 * instructions on 32-bit kernels.
1175		 */
1176		if (!access_ok(VERIFY_WRITE, addr, 8))
1177			goto sigbus;
1178
1179		compute_return_epc(regs);
1180		value = regs->regs[insn.i_format.rt];
1181		StoreDW(addr, value, res);
1182		if (res)
1183			goto fault;
1184		break;
1185#endif /* CONFIG_64BIT */
1186
1187		/* Cannot handle 64-bit instructions in 32-bit kernel */
1188		goto sigill;
1189
1190	case lwc1_op:
1191	case ldc1_op:
1192	case swc1_op:
1193	case sdc1_op:
1194		die_if_kernel("Unaligned FP access in kernel code", regs);
1195		BUG_ON(!used_math());
 
1196
1197		lose_fpu(1);	/* Save FPU state for the emulator. */
1198		res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1199					       &fault_addr);
1200		own_fpu(1);	/* Restore FPU state. */
1201
1202		/* Signal if something went wrong. */
1203		process_fpemu_return(res, fault_addr, 0);
1204
1205		if (res == 0)
1206			break;
1207		return;
1208
1209	case msa_op:
1210		if (!cpu_has_msa)
1211			goto sigill;
1212
1213		/*
1214		 * If we've reached this point then userland should have taken
1215		 * the MSA disabled exception & initialised vector context at
1216		 * some point in the past.
1217		 */
1218		BUG_ON(!thread_msa_context_live());
1219
1220		df = insn.msa_mi10_format.df;
1221		wd = insn.msa_mi10_format.wd;
1222		fpr = &current->thread.fpu.fpr[wd];
1223
1224		switch (insn.msa_mi10_format.func) {
1225		case msa_ld_op:
1226			if (!access_ok(VERIFY_READ, addr, sizeof(*fpr)))
1227				goto sigbus;
1228
1229			do {
1230				/*
1231				 * If we have live MSA context keep track of
1232				 * whether we get preempted in order to avoid
1233				 * the register context we load being clobbered
1234				 * by the live context as it's saved during
1235				 * preemption. If we don't have live context
1236				 * then it can't be saved to clobber the value
1237				 * we load.
1238				 */
1239				preempted = test_thread_flag(TIF_USEDMSA);
1240
1241				res = __copy_from_user_inatomic(fpr, addr,
1242								sizeof(*fpr));
1243				if (res)
1244					goto fault;
1245
1246				/*
1247				 * Update the hardware register if it is in use
1248				 * by the task in this quantum, in order to
1249				 * avoid having to save & restore the whole
1250				 * vector context.
1251				 */
1252				preempt_disable();
1253				if (test_thread_flag(TIF_USEDMSA)) {
1254					write_msa_wr(wd, fpr, df);
1255					preempted = 0;
1256				}
1257				preempt_enable();
1258			} while (preempted);
1259			break;
1260
1261		case msa_st_op:
1262			if (!access_ok(VERIFY_WRITE, addr, sizeof(*fpr)))
1263				goto sigbus;
1264
1265			/*
1266			 * Update from the hardware register if it is in use by
1267			 * the task in this quantum, in order to avoid having to
1268			 * save & restore the whole vector context.
1269			 */
1270			preempt_disable();
1271			if (test_thread_flag(TIF_USEDMSA))
1272				read_msa_wr(wd, fpr, df);
1273			preempt_enable();
1274
1275			res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr));
1276			if (res)
1277				goto fault;
1278			break;
1279
1280		default:
1281			goto sigbus;
1282		}
1283
1284		compute_return_epc(regs);
1285		break;
1286
1287#ifndef CONFIG_CPU_MIPSR6
1288	/*
1289	 * COP2 is available to implementor for application specific use.
1290	 * It's up to applications to register a notifier chain and do
1291	 * whatever they have to do, including possible sending of signals.
1292	 *
1293	 * This instruction has been reallocated in Release 6
1294	 */
1295	case lwc2_op:
1296		cu2_notifier_call_chain(CU2_LWC2_OP, regs);
1297		break;
1298
1299	case ldc2_op:
1300		cu2_notifier_call_chain(CU2_LDC2_OP, regs);
1301		break;
1302
1303	case swc2_op:
1304		cu2_notifier_call_chain(CU2_SWC2_OP, regs);
1305		break;
1306
1307	case sdc2_op:
1308		cu2_notifier_call_chain(CU2_SDC2_OP, regs);
1309		break;
1310#endif
1311	default:
1312		/*
1313		 * Pheeee...  We encountered an yet unknown instruction or
1314		 * cache coherence problem.  Die sucker, die ...
1315		 */
1316		goto sigill;
1317	}
1318
1319#ifdef CONFIG_DEBUG_FS
1320	unaligned_instructions++;
1321#endif
1322
1323	return;
1324
1325fault:
1326	/* roll back jump/branch */
1327	regs->cp0_epc = origpc;
1328	regs->regs[31] = orig31;
1329	/* Did we have an exception handler installed? */
1330	if (fixup_exception(regs))
1331		return;
1332
1333	die_if_kernel("Unhandled kernel unaligned access", regs);
1334	force_sig(SIGSEGV, current);
1335
1336	return;
1337
1338sigbus:
1339	die_if_kernel("Unhandled kernel unaligned access", regs);
1340	force_sig(SIGBUS, current);
1341
1342	return;
1343
1344sigill:
1345	die_if_kernel
1346	    ("Unhandled kernel unaligned access or invalid instruction", regs);
1347	force_sig(SIGILL, current);
1348}
1349
1350/* Recode table from 16-bit register notation to 32-bit GPR. */
1351const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
1352
1353/* Recode table from 16-bit STORE register notation to 32-bit GPR. */
1354const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
1355
1356static void emulate_load_store_microMIPS(struct pt_regs *regs,
1357					 void __user *addr)
1358{
1359	unsigned long value;
1360	unsigned int res;
1361	int i;
1362	unsigned int reg = 0, rvar;
1363	unsigned long orig31;
1364	u16 __user *pc16;
1365	u16 halfword;
1366	unsigned int word;
1367	unsigned long origpc, contpc;
1368	union mips_instruction insn;
1369	struct mm_decoded_insn mminsn;
1370	void __user *fault_addr = NULL;
1371
1372	origpc = regs->cp0_epc;
1373	orig31 = regs->regs[31];
1374
1375	mminsn.micro_mips_mode = 1;
1376
1377	/*
1378	 * This load never faults.
1379	 */
1380	pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
1381	__get_user(halfword, pc16);
1382	pc16++;
1383	contpc = regs->cp0_epc + 2;
1384	word = ((unsigned int)halfword << 16);
1385	mminsn.pc_inc = 2;
1386
1387	if (!mm_insn_16bit(halfword)) {
1388		__get_user(halfword, pc16);
1389		pc16++;
1390		contpc = regs->cp0_epc + 4;
1391		mminsn.pc_inc = 4;
1392		word |= halfword;
1393	}
1394	mminsn.insn = word;
1395
1396	if (get_user(halfword, pc16))
1397		goto fault;
1398	mminsn.next_pc_inc = 2;
1399	word = ((unsigned int)halfword << 16);
1400
1401	if (!mm_insn_16bit(halfword)) {
1402		pc16++;
1403		if (get_user(halfword, pc16))
1404			goto fault;
1405		mminsn.next_pc_inc = 4;
1406		word |= halfword;
1407	}
1408	mminsn.next_insn = word;
1409
1410	insn = (union mips_instruction)(mminsn.insn);
1411	if (mm_isBranchInstr(regs, mminsn, &contpc))
1412		insn = (union mips_instruction)(mminsn.next_insn);
1413
1414	/*  Parse instruction to find what to do */
1415
1416	switch (insn.mm_i_format.opcode) {
1417
1418	case mm_pool32a_op:
1419		switch (insn.mm_x_format.func) {
1420		case mm_lwxs_op:
1421			reg = insn.mm_x_format.rd;
1422			goto loadW;
1423		}
1424
1425		goto sigbus;
1426
1427	case mm_pool32b_op:
1428		switch (insn.mm_m_format.func) {
1429		case mm_lwp_func:
1430			reg = insn.mm_m_format.rd;
1431			if (reg == 31)
1432				goto sigbus;
1433
1434			if (!access_ok(VERIFY_READ, addr, 8))
1435				goto sigbus;
1436
1437			LoadW(addr, value, res);
1438			if (res)
1439				goto fault;
1440			regs->regs[reg] = value;
1441			addr += 4;
1442			LoadW(addr, value, res);
1443			if (res)
1444				goto fault;
1445			regs->regs[reg + 1] = value;
1446			goto success;
1447
1448		case mm_swp_func:
1449			reg = insn.mm_m_format.rd;
1450			if (reg == 31)
1451				goto sigbus;
1452
1453			if (!access_ok(VERIFY_WRITE, addr, 8))
1454				goto sigbus;
1455
1456			value = regs->regs[reg];
1457			StoreW(addr, value, res);
1458			if (res)
1459				goto fault;
1460			addr += 4;
1461			value = regs->regs[reg + 1];
1462			StoreW(addr, value, res);
1463			if (res)
1464				goto fault;
1465			goto success;
1466
1467		case mm_ldp_func:
1468#ifdef CONFIG_64BIT
1469			reg = insn.mm_m_format.rd;
1470			if (reg == 31)
1471				goto sigbus;
1472
1473			if (!access_ok(VERIFY_READ, addr, 16))
1474				goto sigbus;
1475
1476			LoadDW(addr, value, res);
1477			if (res)
1478				goto fault;
1479			regs->regs[reg] = value;
1480			addr += 8;
1481			LoadDW(addr, value, res);
1482			if (res)
1483				goto fault;
1484			regs->regs[reg + 1] = value;
1485			goto success;
1486#endif /* CONFIG_64BIT */
1487
1488			goto sigill;
1489
1490		case mm_sdp_func:
1491#ifdef CONFIG_64BIT
1492			reg = insn.mm_m_format.rd;
1493			if (reg == 31)
1494				goto sigbus;
1495
1496			if (!access_ok(VERIFY_WRITE, addr, 16))
1497				goto sigbus;
1498
1499			value = regs->regs[reg];
1500			StoreDW(addr, value, res);
1501			if (res)
1502				goto fault;
1503			addr += 8;
1504			value = regs->regs[reg + 1];
1505			StoreDW(addr, value, res);
1506			if (res)
1507				goto fault;
1508			goto success;
1509#endif /* CONFIG_64BIT */
1510
1511			goto sigill;
1512
1513		case mm_lwm32_func:
1514			reg = insn.mm_m_format.rd;
1515			rvar = reg & 0xf;
1516			if ((rvar > 9) || !reg)
1517				goto sigill;
1518			if (reg & 0x10) {
1519				if (!access_ok
1520				    (VERIFY_READ, addr, 4 * (rvar + 1)))
1521					goto sigbus;
1522			} else {
1523				if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1524					goto sigbus;
1525			}
1526			if (rvar == 9)
1527				rvar = 8;
1528			for (i = 16; rvar; rvar--, i++) {
1529				LoadW(addr, value, res);
1530				if (res)
1531					goto fault;
1532				addr += 4;
1533				regs->regs[i] = value;
1534			}
1535			if ((reg & 0xf) == 9) {
1536				LoadW(addr, value, res);
1537				if (res)
1538					goto fault;
1539				addr += 4;
1540				regs->regs[30] = value;
1541			}
1542			if (reg & 0x10) {
1543				LoadW(addr, value, res);
1544				if (res)
1545					goto fault;
1546				regs->regs[31] = value;
1547			}
1548			goto success;
1549
1550		case mm_swm32_func:
1551			reg = insn.mm_m_format.rd;
1552			rvar = reg & 0xf;
1553			if ((rvar > 9) || !reg)
1554				goto sigill;
1555			if (reg & 0x10) {
1556				if (!access_ok
1557				    (VERIFY_WRITE, addr, 4 * (rvar + 1)))
1558					goto sigbus;
1559			} else {
1560				if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1561					goto sigbus;
1562			}
1563			if (rvar == 9)
1564				rvar = 8;
1565			for (i = 16; rvar; rvar--, i++) {
1566				value = regs->regs[i];
1567				StoreW(addr, value, res);
1568				if (res)
1569					goto fault;
1570				addr += 4;
1571			}
1572			if ((reg & 0xf) == 9) {
1573				value = regs->regs[30];
1574				StoreW(addr, value, res);
1575				if (res)
1576					goto fault;
1577				addr += 4;
1578			}
1579			if (reg & 0x10) {
1580				value = regs->regs[31];
1581				StoreW(addr, value, res);
1582				if (res)
1583					goto fault;
1584			}
1585			goto success;
1586
1587		case mm_ldm_func:
1588#ifdef CONFIG_64BIT
1589			reg = insn.mm_m_format.rd;
1590			rvar = reg & 0xf;
1591			if ((rvar > 9) || !reg)
1592				goto sigill;
1593			if (reg & 0x10) {
1594				if (!access_ok
1595				    (VERIFY_READ, addr, 8 * (rvar + 1)))
1596					goto sigbus;
1597			} else {
1598				if (!access_ok(VERIFY_READ, addr, 8 * rvar))
1599					goto sigbus;
1600			}
1601			if (rvar == 9)
1602				rvar = 8;
1603
1604			for (i = 16; rvar; rvar--, i++) {
1605				LoadDW(addr, value, res);
1606				if (res)
1607					goto fault;
1608				addr += 4;
1609				regs->regs[i] = value;
1610			}
1611			if ((reg & 0xf) == 9) {
1612				LoadDW(addr, value, res);
1613				if (res)
1614					goto fault;
1615				addr += 8;
1616				regs->regs[30] = value;
1617			}
1618			if (reg & 0x10) {
1619				LoadDW(addr, value, res);
1620				if (res)
1621					goto fault;
1622				regs->regs[31] = value;
1623			}
1624			goto success;
1625#endif /* CONFIG_64BIT */
1626
1627			goto sigill;
1628
1629		case mm_sdm_func:
1630#ifdef CONFIG_64BIT
1631			reg = insn.mm_m_format.rd;
1632			rvar = reg & 0xf;
1633			if ((rvar > 9) || !reg)
1634				goto sigill;
1635			if (reg & 0x10) {
1636				if (!access_ok
1637				    (VERIFY_WRITE, addr, 8 * (rvar + 1)))
1638					goto sigbus;
1639			} else {
1640				if (!access_ok(VERIFY_WRITE, addr, 8 * rvar))
1641					goto sigbus;
1642			}
1643			if (rvar == 9)
1644				rvar = 8;
1645
1646			for (i = 16; rvar; rvar--, i++) {
1647				value = regs->regs[i];
1648				StoreDW(addr, value, res);
1649				if (res)
1650					goto fault;
1651				addr += 8;
1652			}
1653			if ((reg & 0xf) == 9) {
1654				value = regs->regs[30];
1655				StoreDW(addr, value, res);
1656				if (res)
1657					goto fault;
1658				addr += 8;
1659			}
1660			if (reg & 0x10) {
1661				value = regs->regs[31];
1662				StoreDW(addr, value, res);
1663				if (res)
1664					goto fault;
1665			}
1666			goto success;
1667#endif /* CONFIG_64BIT */
1668
1669			goto sigill;
1670
1671			/*  LWC2, SWC2, LDC2, SDC2 are not serviced */
1672		}
1673
1674		goto sigbus;
1675
1676	case mm_pool32c_op:
1677		switch (insn.mm_m_format.func) {
1678		case mm_lwu_func:
1679			reg = insn.mm_m_format.rd;
1680			goto loadWU;
1681		}
1682
1683		/*  LL,SC,LLD,SCD are not serviced */
1684		goto sigbus;
1685
1686	case mm_pool32f_op:
1687		switch (insn.mm_x_format.func) {
1688		case mm_lwxc1_func:
1689		case mm_swxc1_func:
1690		case mm_ldxc1_func:
1691		case mm_sdxc1_func:
1692			goto fpu_emul;
1693		}
1694
1695		goto sigbus;
1696
1697	case mm_ldc132_op:
1698	case mm_sdc132_op:
1699	case mm_lwc132_op:
1700	case mm_swc132_op:
1701fpu_emul:
1702		/* roll back jump/branch */
1703		regs->cp0_epc = origpc;
1704		regs->regs[31] = orig31;
1705
1706		die_if_kernel("Unaligned FP access in kernel code", regs);
1707		BUG_ON(!used_math());
1708		BUG_ON(!is_fpu_owner());
1709
1710		lose_fpu(1);	/* save the FPU state for the emulator */
1711		res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1712					       &fault_addr);
1713		own_fpu(1);	/* restore FPU state */
1714
1715		/* If something went wrong, signal */
1716		process_fpemu_return(res, fault_addr, 0);
1717
1718		if (res == 0)
1719			goto success;
1720		return;
1721
1722	case mm_lh32_op:
1723		reg = insn.mm_i_format.rt;
1724		goto loadHW;
1725
1726	case mm_lhu32_op:
1727		reg = insn.mm_i_format.rt;
1728		goto loadHWU;
1729
1730	case mm_lw32_op:
1731		reg = insn.mm_i_format.rt;
1732		goto loadW;
1733
1734	case mm_sh32_op:
1735		reg = insn.mm_i_format.rt;
1736		goto storeHW;
1737
1738	case mm_sw32_op:
1739		reg = insn.mm_i_format.rt;
1740		goto storeW;
1741
1742	case mm_ld32_op:
1743		reg = insn.mm_i_format.rt;
1744		goto loadDW;
1745
1746	case mm_sd32_op:
1747		reg = insn.mm_i_format.rt;
1748		goto storeDW;
1749
1750	case mm_pool16c_op:
1751		switch (insn.mm16_m_format.func) {
1752		case mm_lwm16_op:
1753			reg = insn.mm16_m_format.rlist;
1754			rvar = reg + 1;
1755			if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1756				goto sigbus;
1757
1758			for (i = 16; rvar; rvar--, i++) {
1759				LoadW(addr, value, res);
1760				if (res)
1761					goto fault;
1762				addr += 4;
1763				regs->regs[i] = value;
1764			}
1765			LoadW(addr, value, res);
1766			if (res)
1767				goto fault;
1768			regs->regs[31] = value;
1769
1770			goto success;
1771
1772		case mm_swm16_op:
1773			reg = insn.mm16_m_format.rlist;
1774			rvar = reg + 1;
1775			if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1776				goto sigbus;
1777
1778			for (i = 16; rvar; rvar--, i++) {
1779				value = regs->regs[i];
1780				StoreW(addr, value, res);
1781				if (res)
1782					goto fault;
1783				addr += 4;
1784			}
1785			value = regs->regs[31];
1786			StoreW(addr, value, res);
1787			if (res)
1788				goto fault;
1789
1790			goto success;
1791
1792		}
1793
1794		goto sigbus;
1795
1796	case mm_lhu16_op:
1797		reg = reg16to32[insn.mm16_rb_format.rt];
1798		goto loadHWU;
1799
1800	case mm_lw16_op:
1801		reg = reg16to32[insn.mm16_rb_format.rt];
1802		goto loadW;
1803
1804	case mm_sh16_op:
1805		reg = reg16to32st[insn.mm16_rb_format.rt];
1806		goto storeHW;
1807
1808	case mm_sw16_op:
1809		reg = reg16to32st[insn.mm16_rb_format.rt];
1810		goto storeW;
1811
1812	case mm_lwsp16_op:
1813		reg = insn.mm16_r5_format.rt;
1814		goto loadW;
1815
1816	case mm_swsp16_op:
1817		reg = insn.mm16_r5_format.rt;
1818		goto storeW;
1819
1820	case mm_lwgp16_op:
1821		reg = reg16to32[insn.mm16_r3_format.rt];
1822		goto loadW;
1823
1824	default:
1825		goto sigill;
1826	}
1827
1828loadHW:
1829	if (!access_ok(VERIFY_READ, addr, 2))
1830		goto sigbus;
1831
1832	LoadHW(addr, value, res);
1833	if (res)
1834		goto fault;
1835	regs->regs[reg] = value;
1836	goto success;
1837
1838loadHWU:
1839	if (!access_ok(VERIFY_READ, addr, 2))
1840		goto sigbus;
1841
1842	LoadHWU(addr, value, res);
1843	if (res)
1844		goto fault;
1845	regs->regs[reg] = value;
1846	goto success;
1847
1848loadW:
1849	if (!access_ok(VERIFY_READ, addr, 4))
1850		goto sigbus;
1851
1852	LoadW(addr, value, res);
1853	if (res)
1854		goto fault;
1855	regs->regs[reg] = value;
1856	goto success;
1857
1858loadWU:
1859#ifdef CONFIG_64BIT
1860	/*
1861	 * A 32-bit kernel might be running on a 64-bit processor.  But
1862	 * if we're on a 32-bit processor and an i-cache incoherency
1863	 * or race makes us see a 64-bit instruction here the sdl/sdr
1864	 * would blow up, so for now we don't handle unaligned 64-bit
1865	 * instructions on 32-bit kernels.
1866	 */
1867	if (!access_ok(VERIFY_READ, addr, 4))
1868		goto sigbus;
1869
1870	LoadWU(addr, value, res);
1871	if (res)
1872		goto fault;
1873	regs->regs[reg] = value;
1874	goto success;
1875#endif /* CONFIG_64BIT */
1876
1877	/* Cannot handle 64-bit instructions in 32-bit kernel */
1878	goto sigill;
1879
1880loadDW:
1881#ifdef CONFIG_64BIT
1882	/*
1883	 * A 32-bit kernel might be running on a 64-bit processor.  But
1884	 * if we're on a 32-bit processor and an i-cache incoherency
1885	 * or race makes us see a 64-bit instruction here the sdl/sdr
1886	 * would blow up, so for now we don't handle unaligned 64-bit
1887	 * instructions on 32-bit kernels.
1888	 */
1889	if (!access_ok(VERIFY_READ, addr, 8))
1890		goto sigbus;
1891
1892	LoadDW(addr, value, res);
1893	if (res)
1894		goto fault;
1895	regs->regs[reg] = value;
1896	goto success;
1897#endif /* CONFIG_64BIT */
1898
1899	/* Cannot handle 64-bit instructions in 32-bit kernel */
1900	goto sigill;
1901
1902storeHW:
1903	if (!access_ok(VERIFY_WRITE, addr, 2))
1904		goto sigbus;
1905
1906	value = regs->regs[reg];
1907	StoreHW(addr, value, res);
1908	if (res)
1909		goto fault;
1910	goto success;
1911
1912storeW:
1913	if (!access_ok(VERIFY_WRITE, addr, 4))
1914		goto sigbus;
1915
1916	value = regs->regs[reg];
1917	StoreW(addr, value, res);
1918	if (res)
1919		goto fault;
1920	goto success;
1921
1922storeDW:
1923#ifdef CONFIG_64BIT
1924	/*
1925	 * A 32-bit kernel might be running on a 64-bit processor.  But
1926	 * if we're on a 32-bit processor and an i-cache incoherency
1927	 * or race makes us see a 64-bit instruction here the sdl/sdr
1928	 * would blow up, so for now we don't handle unaligned 64-bit
1929	 * instructions on 32-bit kernels.
1930	 */
1931	if (!access_ok(VERIFY_WRITE, addr, 8))
1932		goto sigbus;
1933
1934	value = regs->regs[reg];
1935	StoreDW(addr, value, res);
1936	if (res)
1937		goto fault;
1938	goto success;
1939#endif /* CONFIG_64BIT */
1940
1941	/* Cannot handle 64-bit instructions in 32-bit kernel */
1942	goto sigill;
1943
1944success:
1945	regs->cp0_epc = contpc;	/* advance or branch */
1946
1947#ifdef CONFIG_DEBUG_FS
1948	unaligned_instructions++;
1949#endif
1950	return;
1951
1952fault:
1953	/* roll back jump/branch */
1954	regs->cp0_epc = origpc;
1955	regs->regs[31] = orig31;
1956	/* Did we have an exception handler installed? */
1957	if (fixup_exception(regs))
1958		return;
1959
1960	die_if_kernel("Unhandled kernel unaligned access", regs);
1961	force_sig(SIGSEGV, current);
1962
1963	return;
1964
1965sigbus:
1966	die_if_kernel("Unhandled kernel unaligned access", regs);
1967	force_sig(SIGBUS, current);
1968
1969	return;
1970
1971sigill:
1972	die_if_kernel
1973	    ("Unhandled kernel unaligned access or invalid instruction", regs);
1974	force_sig(SIGILL, current);
1975}
1976
1977static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
1978{
1979	unsigned long value;
1980	unsigned int res;
1981	int reg;
1982	unsigned long orig31;
1983	u16 __user *pc16;
1984	unsigned long origpc;
1985	union mips16e_instruction mips16inst, oldinst;
1986
1987	origpc = regs->cp0_epc;
1988	orig31 = regs->regs[31];
1989	pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
1990	/*
1991	 * This load never faults.
1992	 */
1993	__get_user(mips16inst.full, pc16);
1994	oldinst = mips16inst;
1995
1996	/* skip EXTEND instruction */
1997	if (mips16inst.ri.opcode == MIPS16e_extend_op) {
1998		pc16++;
1999		__get_user(mips16inst.full, pc16);
2000	} else if (delay_slot(regs)) {
2001		/*  skip jump instructions */
2002		/*  JAL/JALX are 32 bits but have OPCODE in first short int */
2003		if (mips16inst.ri.opcode == MIPS16e_jal_op)
2004			pc16++;
2005		pc16++;
2006		if (get_user(mips16inst.full, pc16))
2007			goto sigbus;
2008	}
2009
2010	switch (mips16inst.ri.opcode) {
2011	case MIPS16e_i64_op:	/* I64 or RI64 instruction */
2012		switch (mips16inst.i64.func) {	/* I64/RI64 func field check */
2013		case MIPS16e_ldpc_func:
2014		case MIPS16e_ldsp_func:
2015			reg = reg16to32[mips16inst.ri64.ry];
2016			goto loadDW;
2017
2018		case MIPS16e_sdsp_func:
2019			reg = reg16to32[mips16inst.ri64.ry];
2020			goto writeDW;
2021
2022		case MIPS16e_sdrasp_func:
2023			reg = 29;	/* GPRSP */
2024			goto writeDW;
2025		}
2026
2027		goto sigbus;
2028
2029	case MIPS16e_swsp_op:
2030	case MIPS16e_lwpc_op:
2031	case MIPS16e_lwsp_op:
2032		reg = reg16to32[mips16inst.ri.rx];
2033		break;
2034
2035	case MIPS16e_i8_op:
2036		if (mips16inst.i8.func != MIPS16e_swrasp_func)
2037			goto sigbus;
2038		reg = 29;	/* GPRSP */
2039		break;
2040
2041	default:
2042		reg = reg16to32[mips16inst.rri.ry];
2043		break;
2044	}
2045
2046	switch (mips16inst.ri.opcode) {
2047
2048	case MIPS16e_lb_op:
2049	case MIPS16e_lbu_op:
2050	case MIPS16e_sb_op:
2051		goto sigbus;
2052
2053	case MIPS16e_lh_op:
2054		if (!access_ok(VERIFY_READ, addr, 2))
2055			goto sigbus;
2056
2057		LoadHW(addr, value, res);
2058		if (res)
2059			goto fault;
2060		MIPS16e_compute_return_epc(regs, &oldinst);
2061		regs->regs[reg] = value;
2062		break;
2063
2064	case MIPS16e_lhu_op:
2065		if (!access_ok(VERIFY_READ, addr, 2))
2066			goto sigbus;
2067
2068		LoadHWU(addr, value, res);
2069		if (res)
2070			goto fault;
2071		MIPS16e_compute_return_epc(regs, &oldinst);
2072		regs->regs[reg] = value;
2073		break;
2074
2075	case MIPS16e_lw_op:
2076	case MIPS16e_lwpc_op:
2077	case MIPS16e_lwsp_op:
2078		if (!access_ok(VERIFY_READ, addr, 4))
2079			goto sigbus;
2080
2081		LoadW(addr, value, res);
2082		if (res)
2083			goto fault;
2084		MIPS16e_compute_return_epc(regs, &oldinst);
2085		regs->regs[reg] = value;
2086		break;
2087
2088	case MIPS16e_lwu_op:
2089#ifdef CONFIG_64BIT
2090		/*
2091		 * A 32-bit kernel might be running on a 64-bit processor.  But
2092		 * if we're on a 32-bit processor and an i-cache incoherency
2093		 * or race makes us see a 64-bit instruction here the sdl/sdr
2094		 * would blow up, so for now we don't handle unaligned 64-bit
2095		 * instructions on 32-bit kernels.
2096		 */
2097		if (!access_ok(VERIFY_READ, addr, 4))
2098			goto sigbus;
2099
2100		LoadWU(addr, value, res);
2101		if (res)
2102			goto fault;
2103		MIPS16e_compute_return_epc(regs, &oldinst);
2104		regs->regs[reg] = value;
2105		break;
2106#endif /* CONFIG_64BIT */
2107
2108		/* Cannot handle 64-bit instructions in 32-bit kernel */
2109		goto sigill;
2110
2111	case MIPS16e_ld_op:
2112loadDW:
2113#ifdef CONFIG_64BIT
2114		/*
2115		 * A 32-bit kernel might be running on a 64-bit processor.  But
2116		 * if we're on a 32-bit processor and an i-cache incoherency
2117		 * or race makes us see a 64-bit instruction here the sdl/sdr
2118		 * would blow up, so for now we don't handle unaligned 64-bit
2119		 * instructions on 32-bit kernels.
2120		 */
2121		if (!access_ok(VERIFY_READ, addr, 8))
2122			goto sigbus;
2123
2124		LoadDW(addr, value, res);
2125		if (res)
2126			goto fault;
2127		MIPS16e_compute_return_epc(regs, &oldinst);
2128		regs->regs[reg] = value;
2129		break;
2130#endif /* CONFIG_64BIT */
2131
2132		/* Cannot handle 64-bit instructions in 32-bit kernel */
2133		goto sigill;
2134
2135	case MIPS16e_sh_op:
2136		if (!access_ok(VERIFY_WRITE, addr, 2))
2137			goto sigbus;
2138
2139		MIPS16e_compute_return_epc(regs, &oldinst);
2140		value = regs->regs[reg];
2141		StoreHW(addr, value, res);
2142		if (res)
2143			goto fault;
2144		break;
2145
2146	case MIPS16e_sw_op:
2147	case MIPS16e_swsp_op:
2148	case MIPS16e_i8_op:	/* actually - MIPS16e_swrasp_func */
2149		if (!access_ok(VERIFY_WRITE, addr, 4))
2150			goto sigbus;
2151
2152		MIPS16e_compute_return_epc(regs, &oldinst);
2153		value = regs->regs[reg];
2154		StoreW(addr, value, res);
2155		if (res)
2156			goto fault;
2157		break;
2158
2159	case MIPS16e_sd_op:
2160writeDW:
2161#ifdef CONFIG_64BIT
2162		/*
2163		 * A 32-bit kernel might be running on a 64-bit processor.  But
2164		 * if we're on a 32-bit processor and an i-cache incoherency
2165		 * or race makes us see a 64-bit instruction here the sdl/sdr
2166		 * would blow up, so for now we don't handle unaligned 64-bit
2167		 * instructions on 32-bit kernels.
2168		 */
2169		if (!access_ok(VERIFY_WRITE, addr, 8))
2170			goto sigbus;
2171
2172		MIPS16e_compute_return_epc(regs, &oldinst);
2173		value = regs->regs[reg];
2174		StoreDW(addr, value, res);
2175		if (res)
2176			goto fault;
2177		break;
2178#endif /* CONFIG_64BIT */
2179
2180		/* Cannot handle 64-bit instructions in 32-bit kernel */
2181		goto sigill;
2182
2183	default:
2184		/*
2185		 * Pheeee...  We encountered an yet unknown instruction or
2186		 * cache coherence problem.  Die sucker, die ...
2187		 */
2188		goto sigill;
2189	}
2190
2191#ifdef CONFIG_DEBUG_FS
2192	unaligned_instructions++;
2193#endif
2194
2195	return;
2196
2197fault:
2198	/* roll back jump/branch */
2199	regs->cp0_epc = origpc;
2200	regs->regs[31] = orig31;
2201	/* Did we have an exception handler installed? */
2202	if (fixup_exception(regs))
2203		return;
2204
2205	die_if_kernel("Unhandled kernel unaligned access", regs);
2206	force_sig(SIGSEGV, current);
2207
2208	return;
2209
2210sigbus:
2211	die_if_kernel("Unhandled kernel unaligned access", regs);
2212	force_sig(SIGBUS, current);
2213
2214	return;
2215
2216sigill:
2217	die_if_kernel
2218	    ("Unhandled kernel unaligned access or invalid instruction", regs);
2219	force_sig(SIGILL, current);
2220}
2221
2222asmlinkage void do_ade(struct pt_regs *regs)
2223{
2224	enum ctx_state prev_state;
2225	unsigned int __user *pc;
2226	mm_segment_t seg;
2227
2228	prev_state = exception_enter();
2229	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
2230			1, regs, regs->cp0_badvaddr);
2231	/*
2232	 * Did we catch a fault trying to load an instruction?
2233	 */
2234	if (regs->cp0_badvaddr == regs->cp0_epc)
2235		goto sigbus;
2236
2237	if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
2238		goto sigbus;
2239	if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
2240		goto sigbus;
2241
2242	/*
2243	 * Do branch emulation only if we didn't forward the exception.
2244	 * This is all so but ugly ...
2245	 */
2246
2247	/*
2248	 * Are we running in microMIPS mode?
2249	 */
2250	if (get_isa16_mode(regs->cp0_epc)) {
2251		/*
2252		 * Did we catch a fault trying to load an instruction in
2253		 * 16-bit mode?
2254		 */
2255		if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
2256			goto sigbus;
2257		if (unaligned_action == UNALIGNED_ACTION_SHOW)
2258			show_registers(regs);
2259
2260		if (cpu_has_mmips) {
2261			seg = get_fs();
2262			if (!user_mode(regs))
2263				set_fs(KERNEL_DS);
2264			emulate_load_store_microMIPS(regs,
2265				(void __user *)regs->cp0_badvaddr);
2266			set_fs(seg);
2267
2268			return;
2269		}
2270
2271		if (cpu_has_mips16) {
2272			seg = get_fs();
2273			if (!user_mode(regs))
2274				set_fs(KERNEL_DS);
2275			emulate_load_store_MIPS16e(regs,
2276				(void __user *)regs->cp0_badvaddr);
2277			set_fs(seg);
2278
2279			return;
2280	}
2281
2282		goto sigbus;
2283	}
2284
2285	if (unaligned_action == UNALIGNED_ACTION_SHOW)
2286		show_registers(regs);
2287	pc = (unsigned int __user *)exception_epc(regs);
2288
2289	seg = get_fs();
2290	if (!user_mode(regs))
2291		set_fs(KERNEL_DS);
2292	emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
2293	set_fs(seg);
2294
2295	return;
2296
2297sigbus:
2298	die_if_kernel("Kernel unaligned instruction access", regs);
2299	force_sig(SIGBUS, current);
2300
2301	/*
2302	 * XXX On return from the signal handler we should advance the epc
2303	 */
2304	exception_exit(prev_state);
2305}
2306
2307#ifdef CONFIG_DEBUG_FS
 
2308static int __init debugfs_unaligned(void)
2309{
2310	struct dentry *d;
2311
2312	if (!mips_debugfs_dir)
2313		return -ENODEV;
2314	d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
2315			       mips_debugfs_dir, &unaligned_instructions);
2316	if (!d)
2317		return -ENOMEM;
2318	d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
2319			       mips_debugfs_dir, &unaligned_action);
2320	if (!d)
2321		return -ENOMEM;
2322	return 0;
2323}
2324arch_initcall(debugfs_unaligned);
2325#endif