1/*
   2 * Handle unaligned accesses by emulation.
   3 *
   4 * This file is subject to the terms and conditions of the GNU General Public
   5 * License.  See the file "COPYING" in the main directory of this archive
   6 * for more details.
   7 *
   8 * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
   9 * Copyright (C) 1999 Silicon Graphics, Inc.
  10 * Copyright (C) 2014 Imagination Technologies Ltd.
  11 *
  12 * This file contains exception handler for address error exception with the
  13 * special capability to execute faulting instructions in software.  The
  14 * handler does not try to handle the case when the program counter points
  15 * to an address not aligned to a word boundary.
  16 *
  17 * Putting data to unaligned addresses is a bad practice even on Intel where
  18 * only the performance is affected.  Much worse is that such code is non-
  19 * portable.  Due to several programs that die on MIPS due to alignment
  20 * problems I decided to implement this handler anyway though I originally
  21 * didn't intend to do this at all for user code.
  22 *
  23 * For now I enable fixing of address errors by default to make life easier.
  24 * I however intend to disable this somewhen in the future when the alignment
  25 * problems with user programs have been fixed.	 For programmers this is the
  26 * right way to go.
  27 *
  28 * Fixing address errors is a per process option.  The option is inherited
  29 * across fork(2) and execve(2) calls.	If you really want to use the
  30 * option in your user programs - I discourage the use of the software
  31 * emulation strongly - use the following code in your userland stuff:
  32 *
  33 * #include <sys/sysmips.h>
  34 *
  35 * ...
  36 * sysmips(MIPS_FIXADE, x);
  37 * ...
  38 *
  39 * The argument x is 0 for disabling software emulation, enabled otherwise.
  40 *
  41 * Below a little program to play around with this feature.
  42 *
  43 * #include <stdio.h>
  44 * #include <sys/sysmips.h>
  45 *
  46 * struct foo {
  47 *	   unsigned char bar[8];
  48 * };
  49 *
  50 * main(int argc, char *argv[])
  51 * {
  52 *	   struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
  53 *	   unsigned int *p = (unsigned int *) (x.bar + 3);
  54 *	   int i;
  55 *
  56 *	   if (argc > 1)
  57 *		   sysmips(MIPS_FIXADE, atoi(argv[1]));
  58 *
  59 *	   printf("*p = %08lx\n", *p);
  60 *
  61 *	   *p = 0xdeadface;
  62 *
  63 *	   for(i = 0; i <= 7; i++)
  64 *	   printf("%02x ", x.bar[i]);
  65 *	   printf("\n");
  66 * }
  67 *
  68 * Coprocessor loads are not supported; I think this case is unimportant
  69 * in the practice.
  70 *
  71 * TODO: Handle ndc (attempted store to doubleword in uncached memory)
  72 *	 exception for the R6000.
  73 *	 A store crossing a page boundary might be executed only partially.
  74 *	 Undo the partial store in this case.
  75 */
  76#include <linux/context_tracking.h>
  77#include <linux/mm.h>
  78#include <linux/signal.h>
  79#include <linux/smp.h>
  80#include <linux/sched.h>
  81#include <linux/debugfs.h>
  82#include <linux/perf_event.h>
  83
  84#include <asm/asm.h>
  85#include <asm/branch.h>
  86#include <asm/byteorder.h>
  87#include <asm/cop2.h>
  88#include <asm/debug.h>
  89#include <asm/fpu.h>
  90#include <asm/fpu_emulator.h>
  91#include <asm/inst.h>
 
  92#include <asm/mmu_context.h>
  93#include <linux/uaccess.h>
  94
  95#define STR(x)	__STR(x)
  96#define __STR(x)  #x
  97
  98enum {
  99	UNALIGNED_ACTION_QUIET,
 100	UNALIGNED_ACTION_SIGNAL,
 101	UNALIGNED_ACTION_SHOW,
 102};
 103#ifdef CONFIG_DEBUG_FS
 104static u32 unaligned_instructions;
 105static u32 unaligned_action;
 106#else
 107#define unaligned_action UNALIGNED_ACTION_QUIET
 108#endif
 109extern void show_registers(struct pt_regs *regs);
 110
 111#ifdef __BIG_ENDIAN
 112#define     _LoadHW(addr, value, res, type)  \
 113do {                                                        \
 114		__asm__ __volatile__ (".set\tnoat\n"        \
 115			"1:\t"type##_lb("%0", "0(%2)")"\n"  \
 116			"2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 117			"sll\t%0, 0x8\n\t"                  \
 118			"or\t%0, $1\n\t"                    \
 119			"li\t%1, 0\n"                       \
 120			"3:\t.set\tat\n\t"                  \
 121			".insn\n\t"                         \
 122			".section\t.fixup,\"ax\"\n\t"       \
 123			"4:\tli\t%1, %3\n\t"                \
 124			"j\t3b\n\t"                         \
 125			".previous\n\t"                     \
 126			".section\t__ex_table,\"a\"\n\t"    \
 127			STR(PTR)"\t1b, 4b\n\t"              \
 128			STR(PTR)"\t2b, 4b\n\t"              \
 129			".previous"                         \
 130			: "=&r" (value), "=r" (res)         \
 131			: "r" (addr), "i" (-EFAULT));       \
 132} while(0)
 133
 134#ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 135#define     _LoadW(addr, value, res, type)   \
 136do {                                                        \
 137		__asm__ __volatile__ (                      \
 138			"1:\t"type##_lwl("%0", "(%2)")"\n"   \
 139			"2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 140			"li\t%1, 0\n"                       \
 141			"3:\n\t"                            \
 142			".insn\n\t"                         \
 143			".section\t.fixup,\"ax\"\n\t"       \
 144			"4:\tli\t%1, %3\n\t"                \
 145			"j\t3b\n\t"                         \
 146			".previous\n\t"                     \
 147			".section\t__ex_table,\"a\"\n\t"    \
 148			STR(PTR)"\t1b, 4b\n\t"              \
 149			STR(PTR)"\t2b, 4b\n\t"              \
 150			".previous"                         \
 151			: "=&r" (value), "=r" (res)         \
 152			: "r" (addr), "i" (-EFAULT));       \
 153} while(0)
 154
 155#else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 156/* For CPUs without lwl instruction */
 157#define     _LoadW(addr, value, res, type) \
 158do {                                                        \
 159		__asm__ __volatile__ (			    \
 160			".set\tpush\n"			    \
 161			".set\tnoat\n\t"		    \
 162			"1:"type##_lb("%0", "0(%2)")"\n\t"  \
 163			"2:"type##_lbu("$1", "1(%2)")"\n\t" \
 164			"sll\t%0, 0x8\n\t"		    \
 165			"or\t%0, $1\n\t"		    \
 166			"3:"type##_lbu("$1", "2(%2)")"\n\t" \
 167			"sll\t%0, 0x8\n\t"		    \
 168			"or\t%0, $1\n\t"		    \
 169			"4:"type##_lbu("$1", "3(%2)")"\n\t" \
 170			"sll\t%0, 0x8\n\t"		    \
 171			"or\t%0, $1\n\t"		    \
 172			"li\t%1, 0\n"			    \
 173			".set\tpop\n"			    \
 174			"10:\n\t"			    \
 175			".insn\n\t"			    \
 176			".section\t.fixup,\"ax\"\n\t"	    \
 177			"11:\tli\t%1, %3\n\t"		    \
 178			"j\t10b\n\t"			    \
 179			".previous\n\t"			    \
 180			".section\t__ex_table,\"a\"\n\t"    \
 181			STR(PTR)"\t1b, 11b\n\t"		    \
 182			STR(PTR)"\t2b, 11b\n\t"		    \
 183			STR(PTR)"\t3b, 11b\n\t"		    \
 184			STR(PTR)"\t4b, 11b\n\t"		    \
 185			".previous"			    \
 186			: "=&r" (value), "=r" (res)	    \
 187			: "r" (addr), "i" (-EFAULT));       \
 188} while(0)
 189
 190#endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 191
 192#define     _LoadHWU(addr, value, res, type) \
 193do {                                                        \
 194		__asm__ __volatile__ (                      \
 195			".set\tnoat\n"                      \
 196			"1:\t"type##_lbu("%0", "0(%2)")"\n" \
 197			"2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 198			"sll\t%0, 0x8\n\t"                  \
 199			"or\t%0, $1\n\t"                    \
 200			"li\t%1, 0\n"                       \
 201			"3:\n\t"                            \
 202			".insn\n\t"                         \
 203			".set\tat\n\t"                      \
 204			".section\t.fixup,\"ax\"\n\t"       \
 205			"4:\tli\t%1, %3\n\t"                \
 206			"j\t3b\n\t"                         \
 207			".previous\n\t"                     \
 208			".section\t__ex_table,\"a\"\n\t"    \
 209			STR(PTR)"\t1b, 4b\n\t"              \
 210			STR(PTR)"\t2b, 4b\n\t"              \
 211			".previous"                         \
 212			: "=&r" (value), "=r" (res)         \
 213			: "r" (addr), "i" (-EFAULT));       \
 214} while(0)
 215
 216#ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 217#define     _LoadWU(addr, value, res, type)  \
 218do {                                                        \
 219		__asm__ __volatile__ (                      \
 220			"1:\t"type##_lwl("%0", "(%2)")"\n"  \
 221			"2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 222			"dsll\t%0, %0, 32\n\t"              \
 223			"dsrl\t%0, %0, 32\n\t"              \
 224			"li\t%1, 0\n"                       \
 225			"3:\n\t"                            \
 226			".insn\n\t"                         \
 227			"\t.section\t.fixup,\"ax\"\n\t"     \
 228			"4:\tli\t%1, %3\n\t"                \
 229			"j\t3b\n\t"                         \
 230			".previous\n\t"                     \
 231			".section\t__ex_table,\"a\"\n\t"    \
 232			STR(PTR)"\t1b, 4b\n\t"              \
 233			STR(PTR)"\t2b, 4b\n\t"              \
 234			".previous"                         \
 235			: "=&r" (value), "=r" (res)         \
 236			: "r" (addr), "i" (-EFAULT));       \
 237} while(0)
 238
 239#define     _LoadDW(addr, value, res)  \
 240do {                                                        \
 241		__asm__ __volatile__ (                      \
 242			"1:\tldl\t%0, (%2)\n"               \
 243			"2:\tldr\t%0, 7(%2)\n\t"            \
 244			"li\t%1, 0\n"                       \
 245			"3:\n\t"                            \
 246			".insn\n\t"                         \
 247			"\t.section\t.fixup,\"ax\"\n\t"     \
 248			"4:\tli\t%1, %3\n\t"                \
 249			"j\t3b\n\t"                         \
 250			".previous\n\t"                     \
 251			".section\t__ex_table,\"a\"\n\t"    \
 252			STR(PTR)"\t1b, 4b\n\t"              \
 253			STR(PTR)"\t2b, 4b\n\t"              \
 254			".previous"                         \
 255			: "=&r" (value), "=r" (res)         \
 256			: "r" (addr), "i" (-EFAULT));       \
 257} while(0)
 258
 259#else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 260/* For CPUs without lwl and ldl instructions */
 261#define	    _LoadWU(addr, value, res, type) \
 262do {                                                        \
 263		__asm__ __volatile__ (			    \
 264			".set\tpush\n\t"		    \
 265			".set\tnoat\n\t"		    \
 266			"1:"type##_lbu("%0", "0(%2)")"\n\t" \
 267			"2:"type##_lbu("$1", "1(%2)")"\n\t" \
 268			"sll\t%0, 0x8\n\t"		    \
 269			"or\t%0, $1\n\t"		    \
 270			"3:"type##_lbu("$1", "2(%2)")"\n\t" \
 271			"sll\t%0, 0x8\n\t"		    \
 272			"or\t%0, $1\n\t"		    \
 273			"4:"type##_lbu("$1", "3(%2)")"\n\t" \
 274			"sll\t%0, 0x8\n\t"		    \
 275			"or\t%0, $1\n\t"		    \
 276			"li\t%1, 0\n"			    \
 277			".set\tpop\n"			    \
 278			"10:\n\t"			    \
 279			".insn\n\t"			    \
 280			".section\t.fixup,\"ax\"\n\t"	    \
 281			"11:\tli\t%1, %3\n\t"		    \
 282			"j\t10b\n\t"			    \
 283			".previous\n\t"			    \
 284			".section\t__ex_table,\"a\"\n\t"    \
 285			STR(PTR)"\t1b, 11b\n\t"		    \
 286			STR(PTR)"\t2b, 11b\n\t"		    \
 287			STR(PTR)"\t3b, 11b\n\t"		    \
 288			STR(PTR)"\t4b, 11b\n\t"		    \
 289			".previous"			    \
 290			: "=&r" (value), "=r" (res)	    \
 291			: "r" (addr), "i" (-EFAULT));       \
 292} while(0)
 293
 294#define     _LoadDW(addr, value, res)  \
 295do {                                                        \
 296		__asm__ __volatile__ (			    \
 297			".set\tpush\n\t"		    \
 298			".set\tnoat\n\t"		    \
 299			"1:lb\t%0, 0(%2)\n\t"    	    \
 300			"2:lbu\t $1, 1(%2)\n\t"   	    \
 301			"dsll\t%0, 0x8\n\t"		    \
 302			"or\t%0, $1\n\t"		    \
 303			"3:lbu\t$1, 2(%2)\n\t"   	    \
 304			"dsll\t%0, 0x8\n\t"		    \
 305			"or\t%0, $1\n\t"		    \
 306			"4:lbu\t$1, 3(%2)\n\t"   	    \
 307			"dsll\t%0, 0x8\n\t"		    \
 308			"or\t%0, $1\n\t"		    \
 309			"5:lbu\t$1, 4(%2)\n\t"   	    \
 310			"dsll\t%0, 0x8\n\t"		    \
 311			"or\t%0, $1\n\t"		    \
 312			"6:lbu\t$1, 5(%2)\n\t"   	    \
 313			"dsll\t%0, 0x8\n\t"		    \
 314			"or\t%0, $1\n\t"		    \
 315			"7:lbu\t$1, 6(%2)\n\t"   	    \
 316			"dsll\t%0, 0x8\n\t"		    \
 317			"or\t%0, $1\n\t"		    \
 318			"8:lbu\t$1, 7(%2)\n\t"   	    \
 319			"dsll\t%0, 0x8\n\t"		    \
 320			"or\t%0, $1\n\t"		    \
 321			"li\t%1, 0\n"			    \
 322			".set\tpop\n\t"			    \
 323			"10:\n\t"			    \
 324			".insn\n\t"			    \
 325			".section\t.fixup,\"ax\"\n\t"	    \
 326			"11:\tli\t%1, %3\n\t"		    \
 327			"j\t10b\n\t"			    \
 328			".previous\n\t"			    \
 329			".section\t__ex_table,\"a\"\n\t"    \
 330			STR(PTR)"\t1b, 11b\n\t"		    \
 331			STR(PTR)"\t2b, 11b\n\t"		    \
 332			STR(PTR)"\t3b, 11b\n\t"		    \
 333			STR(PTR)"\t4b, 11b\n\t"		    \
 334			STR(PTR)"\t5b, 11b\n\t"		    \
 335			STR(PTR)"\t6b, 11b\n\t"		    \
 336			STR(PTR)"\t7b, 11b\n\t"		    \
 337			STR(PTR)"\t8b, 11b\n\t"		    \
 338			".previous"			    \
 339			: "=&r" (value), "=r" (res)	    \
 340			: "r" (addr), "i" (-EFAULT));       \
 341} while(0)
 342
 343#endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 344
 345
 346#define     _StoreHW(addr, value, res, type) \
 347do {                                                        \
 348		__asm__ __volatile__ (                      \
 349			".set\tnoat\n"                      \
 350			"1:\t"type##_sb("%1", "1(%2)")"\n"  \
 351			"srl\t$1, %1, 0x8\n"                \
 352			"2:\t"type##_sb("$1", "0(%2)")"\n"  \
 353			".set\tat\n\t"                      \
 354			"li\t%0, 0\n"                       \
 355			"3:\n\t"                            \
 356			".insn\n\t"                         \
 357			".section\t.fixup,\"ax\"\n\t"       \
 358			"4:\tli\t%0, %3\n\t"                \
 359			"j\t3b\n\t"                         \
 360			".previous\n\t"                     \
 361			".section\t__ex_table,\"a\"\n\t"    \
 362			STR(PTR)"\t1b, 4b\n\t"              \
 363			STR(PTR)"\t2b, 4b\n\t"              \
 364			".previous"                         \
 365			: "=r" (res)                        \
 366			: "r" (value), "r" (addr), "i" (-EFAULT));\
 367} while(0)
 368
 369#ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 370#define     _StoreW(addr, value, res, type)  \
 371do {                                                        \
 372		__asm__ __volatile__ (                      \
 373			"1:\t"type##_swl("%1", "(%2)")"\n"  \
 374			"2:\t"type##_swr("%1", "3(%2)")"\n\t"\
 375			"li\t%0, 0\n"                       \
 376			"3:\n\t"                            \
 377			".insn\n\t"                         \
 378			".section\t.fixup,\"ax\"\n\t"       \
 379			"4:\tli\t%0, %3\n\t"                \
 380			"j\t3b\n\t"                         \
 381			".previous\n\t"                     \
 382			".section\t__ex_table,\"a\"\n\t"    \
 383			STR(PTR)"\t1b, 4b\n\t"              \
 384			STR(PTR)"\t2b, 4b\n\t"              \
 385			".previous"                         \
 386		: "=r" (res)                                \
 387		: "r" (value), "r" (addr), "i" (-EFAULT));  \
 388} while(0)
 389
 390#define     _StoreDW(addr, value, res) \
 391do {                                                        \
 392		__asm__ __volatile__ (                      \
 393			"1:\tsdl\t%1,(%2)\n"                \
 394			"2:\tsdr\t%1, 7(%2)\n\t"            \
 395			"li\t%0, 0\n"                       \
 396			"3:\n\t"                            \
 397			".insn\n\t"                         \
 398			".section\t.fixup,\"ax\"\n\t"       \
 399			"4:\tli\t%0, %3\n\t"                \
 400			"j\t3b\n\t"                         \
 401			".previous\n\t"                     \
 402			".section\t__ex_table,\"a\"\n\t"    \
 403			STR(PTR)"\t1b, 4b\n\t"              \
 404			STR(PTR)"\t2b, 4b\n\t"              \
 405			".previous"                         \
 406		: "=r" (res)                                \
 407		: "r" (value), "r" (addr), "i" (-EFAULT));  \
 408} while(0)
 409
 410#else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 411#define     _StoreW(addr, value, res, type)  \
 412do {                                                        \
 413		__asm__ __volatile__ (                      \
 414			".set\tpush\n\t"		    \
 415			".set\tnoat\n\t"		    \
 416			"1:"type##_sb("%1", "3(%2)")"\n\t"  \
 417			"srl\t$1, %1, 0x8\n\t"		    \
 418			"2:"type##_sb("$1", "2(%2)")"\n\t"  \
 419			"srl\t$1, $1,  0x8\n\t"		    \
 420			"3:"type##_sb("$1", "1(%2)")"\n\t"  \
 421			"srl\t$1, $1, 0x8\n\t"		    \
 422			"4:"type##_sb("$1", "0(%2)")"\n\t"  \
 423			".set\tpop\n\t"			    \
 424			"li\t%0, 0\n"			    \
 425			"10:\n\t"			    \
 426			".insn\n\t"			    \
 427			".section\t.fixup,\"ax\"\n\t"	    \
 428			"11:\tli\t%0, %3\n\t"		    \
 429			"j\t10b\n\t"			    \
 430			".previous\n\t"			    \
 431			".section\t__ex_table,\"a\"\n\t"    \
 432			STR(PTR)"\t1b, 11b\n\t"		    \
 433			STR(PTR)"\t2b, 11b\n\t"		    \
 434			STR(PTR)"\t3b, 11b\n\t"		    \
 435			STR(PTR)"\t4b, 11b\n\t"		    \
 436			".previous"			    \
 437		: "=&r" (res)			    	    \
 438		: "r" (value), "r" (addr), "i" (-EFAULT)    \
 439		: "memory");                                \
 440} while(0)
 441
 442#define     _StoreDW(addr, value, res) \
 443do {                                                        \
 444		__asm__ __volatile__ (                      \
 445			".set\tpush\n\t"		    \
 446			".set\tnoat\n\t"		    \
 447			"1:sb\t%1, 7(%2)\n\t"    	    \
 448			"dsrl\t$1, %1, 0x8\n\t"		    \
 449			"2:sb\t$1, 6(%2)\n\t"    	    \
 450			"dsrl\t$1, $1, 0x8\n\t"		    \
 451			"3:sb\t$1, 5(%2)\n\t"    	    \
 452			"dsrl\t$1, $1, 0x8\n\t"		    \
 453			"4:sb\t$1, 4(%2)\n\t"    	    \
 454			"dsrl\t$1, $1, 0x8\n\t"		    \
 455			"5:sb\t$1, 3(%2)\n\t"    	    \
 456			"dsrl\t$1, $1, 0x8\n\t"		    \
 457			"6:sb\t$1, 2(%2)\n\t"    	    \
 458			"dsrl\t$1, $1, 0x8\n\t"		    \
 459			"7:sb\t$1, 1(%2)\n\t"    	    \
 460			"dsrl\t$1, $1, 0x8\n\t"		    \
 461			"8:sb\t$1, 0(%2)\n\t"    	    \
 462			"dsrl\t$1, $1, 0x8\n\t"		    \
 463			".set\tpop\n\t"			    \
 464			"li\t%0, 0\n"			    \
 465			"10:\n\t"			    \
 466			".insn\n\t"			    \
 467			".section\t.fixup,\"ax\"\n\t"	    \
 468			"11:\tli\t%0, %3\n\t"		    \
 469			"j\t10b\n\t"			    \
 470			".previous\n\t"			    \
 471			".section\t__ex_table,\"a\"\n\t"    \
 472			STR(PTR)"\t1b, 11b\n\t"		    \
 473			STR(PTR)"\t2b, 11b\n\t"		    \
 474			STR(PTR)"\t3b, 11b\n\t"		    \
 475			STR(PTR)"\t4b, 11b\n\t"		    \
 476			STR(PTR)"\t5b, 11b\n\t"		    \
 477			STR(PTR)"\t6b, 11b\n\t"		    \
 478			STR(PTR)"\t7b, 11b\n\t"		    \
 479			STR(PTR)"\t8b, 11b\n\t"		    \
 480			".previous"			    \
 481		: "=&r" (res)			    	    \
 482		: "r" (value), "r" (addr), "i" (-EFAULT)    \
 483		: "memory");                                \
 484} while(0)
 485
 486#endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 487
 488#else /* __BIG_ENDIAN */
 489
 490#define     _LoadHW(addr, value, res, type)  \
 491do {                                                        \
 492		__asm__ __volatile__ (".set\tnoat\n"        \
 493			"1:\t"type##_lb("%0", "1(%2)")"\n"  \
 494			"2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 495			"sll\t%0, 0x8\n\t"                  \
 496			"or\t%0, $1\n\t"                    \
 497			"li\t%1, 0\n"                       \
 498			"3:\t.set\tat\n\t"                  \
 499			".insn\n\t"                         \
 500			".section\t.fixup,\"ax\"\n\t"       \
 501			"4:\tli\t%1, %3\n\t"                \
 502			"j\t3b\n\t"                         \
 503			".previous\n\t"                     \
 504			".section\t__ex_table,\"a\"\n\t"    \
 505			STR(PTR)"\t1b, 4b\n\t"              \
 506			STR(PTR)"\t2b, 4b\n\t"              \
 507			".previous"                         \
 508			: "=&r" (value), "=r" (res)         \
 509			: "r" (addr), "i" (-EFAULT));       \
 510} while(0)
 511
 512#ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 513#define     _LoadW(addr, value, res, type)   \
 514do {                                                        \
 515		__asm__ __volatile__ (                      \
 516			"1:\t"type##_lwl("%0", "3(%2)")"\n" \
 517			"2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 518			"li\t%1, 0\n"                       \
 519			"3:\n\t"                            \
 520			".insn\n\t"                         \
 521			".section\t.fixup,\"ax\"\n\t"       \
 522			"4:\tli\t%1, %3\n\t"                \
 523			"j\t3b\n\t"                         \
 524			".previous\n\t"                     \
 525			".section\t__ex_table,\"a\"\n\t"    \
 526			STR(PTR)"\t1b, 4b\n\t"              \
 527			STR(PTR)"\t2b, 4b\n\t"              \
 528			".previous"                         \
 529			: "=&r" (value), "=r" (res)         \
 530			: "r" (addr), "i" (-EFAULT));       \
 531} while(0)
 532
 533#else  /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 534/* For CPUs without lwl instruction */
 535#define     _LoadW(addr, value, res, type) \
 536do {                                                        \
 537		__asm__ __volatile__ (			    \
 538			".set\tpush\n"			    \
 539			".set\tnoat\n\t"		    \
 540			"1:"type##_lb("%0", "3(%2)")"\n\t"  \
 541			"2:"type##_lbu("$1", "2(%2)")"\n\t" \
 542			"sll\t%0, 0x8\n\t"		    \
 543			"or\t%0, $1\n\t"		    \
 544			"3:"type##_lbu("$1", "1(%2)")"\n\t" \
 545			"sll\t%0, 0x8\n\t"		    \
 546			"or\t%0, $1\n\t"		    \
 547			"4:"type##_lbu("$1", "0(%2)")"\n\t" \
 548			"sll\t%0, 0x8\n\t"		    \
 549			"or\t%0, $1\n\t"		    \
 550			"li\t%1, 0\n"			    \
 551			".set\tpop\n"			    \
 552			"10:\n\t"			    \
 553			".insn\n\t"			    \
 554			".section\t.fixup,\"ax\"\n\t"	    \
 555			"11:\tli\t%1, %3\n\t"		    \
 556			"j\t10b\n\t"			    \
 557			".previous\n\t"			    \
 558			".section\t__ex_table,\"a\"\n\t"    \
 559			STR(PTR)"\t1b, 11b\n\t"		    \
 560			STR(PTR)"\t2b, 11b\n\t"		    \
 561			STR(PTR)"\t3b, 11b\n\t"		    \
 562			STR(PTR)"\t4b, 11b\n\t"		    \
 563			".previous"			    \
 564			: "=&r" (value), "=r" (res)	    \
 565			: "r" (addr), "i" (-EFAULT));       \
 566} while(0)
 567
 568#endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 569
 570
 571#define     _LoadHWU(addr, value, res, type) \
 572do {                                                        \
 573		__asm__ __volatile__ (                      \
 574			".set\tnoat\n"                      \
 575			"1:\t"type##_lbu("%0", "1(%2)")"\n" \
 576			"2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 577			"sll\t%0, 0x8\n\t"                  \
 578			"or\t%0, $1\n\t"                    \
 579			"li\t%1, 0\n"                       \
 580			"3:\n\t"                            \
 581			".insn\n\t"                         \
 582			".set\tat\n\t"                      \
 583			".section\t.fixup,\"ax\"\n\t"       \
 584			"4:\tli\t%1, %3\n\t"                \
 585			"j\t3b\n\t"                         \
 586			".previous\n\t"                     \
 587			".section\t__ex_table,\"a\"\n\t"    \
 588			STR(PTR)"\t1b, 4b\n\t"              \
 589			STR(PTR)"\t2b, 4b\n\t"              \
 590			".previous"                         \
 591			: "=&r" (value), "=r" (res)         \
 592			: "r" (addr), "i" (-EFAULT));       \
 593} while(0)
 594
 595#ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 596#define     _LoadWU(addr, value, res, type)  \
 597do {                                                        \
 598		__asm__ __volatile__ (                      \
 599			"1:\t"type##_lwl("%0", "3(%2)")"\n" \
 600			"2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 601			"dsll\t%0, %0, 32\n\t"              \
 602			"dsrl\t%0, %0, 32\n\t"              \
 603			"li\t%1, 0\n"                       \
 604			"3:\n\t"                            \
 605			".insn\n\t"                         \
 606			"\t.section\t.fixup,\"ax\"\n\t"     \
 607			"4:\tli\t%1, %3\n\t"                \
 608			"j\t3b\n\t"                         \
 609			".previous\n\t"                     \
 610			".section\t__ex_table,\"a\"\n\t"    \
 611			STR(PTR)"\t1b, 4b\n\t"              \
 612			STR(PTR)"\t2b, 4b\n\t"              \
 613			".previous"                         \
 614			: "=&r" (value), "=r" (res)         \
 615			: "r" (addr), "i" (-EFAULT));       \
 616} while(0)
 617
 618#define     _LoadDW(addr, value, res)  \
 619do {                                                        \
 620		__asm__ __volatile__ (                      \
 621			"1:\tldl\t%0, 7(%2)\n"              \
 622			"2:\tldr\t%0, (%2)\n\t"             \
 623			"li\t%1, 0\n"                       \
 624			"3:\n\t"                            \
 625			".insn\n\t"                         \
 626			"\t.section\t.fixup,\"ax\"\n\t"     \
 627			"4:\tli\t%1, %3\n\t"                \
 628			"j\t3b\n\t"                         \
 629			".previous\n\t"                     \
 630			".section\t__ex_table,\"a\"\n\t"    \
 631			STR(PTR)"\t1b, 4b\n\t"              \
 632			STR(PTR)"\t2b, 4b\n\t"              \
 633			".previous"                         \
 634			: "=&r" (value), "=r" (res)         \
 635			: "r" (addr), "i" (-EFAULT));       \
 636} while(0)
 637
 638#else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 639/* For CPUs without lwl and ldl instructions */
 640#define	    _LoadWU(addr, value, res, type) \
 641do {                                                        \
 642		__asm__ __volatile__ (			    \
 643			".set\tpush\n\t"		    \
 644			".set\tnoat\n\t"		    \
 645			"1:"type##_lbu("%0", "3(%2)")"\n\t" \
 646			"2:"type##_lbu("$1", "2(%2)")"\n\t" \
 647			"sll\t%0, 0x8\n\t"		    \
 648			"or\t%0, $1\n\t"		    \
 649			"3:"type##_lbu("$1", "1(%2)")"\n\t" \
 650			"sll\t%0, 0x8\n\t"		    \
 651			"or\t%0, $1\n\t"		    \
 652			"4:"type##_lbu("$1", "0(%2)")"\n\t" \
 653			"sll\t%0, 0x8\n\t"		    \
 654			"or\t%0, $1\n\t"		    \
 655			"li\t%1, 0\n"			    \
 656			".set\tpop\n"			    \
 657			"10:\n\t"			    \
 658			".insn\n\t"			    \
 659			".section\t.fixup,\"ax\"\n\t"	    \
 660			"11:\tli\t%1, %3\n\t"		    \
 661			"j\t10b\n\t"			    \
 662			".previous\n\t"			    \
 663			".section\t__ex_table,\"a\"\n\t"    \
 664			STR(PTR)"\t1b, 11b\n\t"		    \
 665			STR(PTR)"\t2b, 11b\n\t"		    \
 666			STR(PTR)"\t3b, 11b\n\t"		    \
 667			STR(PTR)"\t4b, 11b\n\t"		    \
 668			".previous"			    \
 669			: "=&r" (value), "=r" (res)	    \
 670			: "r" (addr), "i" (-EFAULT));       \
 671} while(0)
 672
 673#define     _LoadDW(addr, value, res)  \
 674do {                                                        \
 675		__asm__ __volatile__ (			    \
 676			".set\tpush\n\t"		    \
 677			".set\tnoat\n\t"		    \
 678			"1:lb\t%0, 7(%2)\n\t"    	    \
 679			"2:lbu\t$1, 6(%2)\n\t"   	    \
 680			"dsll\t%0, 0x8\n\t"		    \
 681			"or\t%0, $1\n\t"		    \
 682			"3:lbu\t$1, 5(%2)\n\t"   	    \
 683			"dsll\t%0, 0x8\n\t"		    \
 684			"or\t%0, $1\n\t"		    \
 685			"4:lbu\t$1, 4(%2)\n\t"   	    \
 686			"dsll\t%0, 0x8\n\t"		    \
 687			"or\t%0, $1\n\t"		    \
 688			"5:lbu\t$1, 3(%2)\n\t"   	    \
 689			"dsll\t%0, 0x8\n\t"		    \
 690			"or\t%0, $1\n\t"		    \
 691			"6:lbu\t$1, 2(%2)\n\t"   	    \
 692			"dsll\t%0, 0x8\n\t"		    \
 693			"or\t%0, $1\n\t"		    \
 694			"7:lbu\t$1, 1(%2)\n\t"   	    \
 695			"dsll\t%0, 0x8\n\t"		    \
 696			"or\t%0, $1\n\t"		    \
 697			"8:lbu\t$1, 0(%2)\n\t"   	    \
 698			"dsll\t%0, 0x8\n\t"		    \
 699			"or\t%0, $1\n\t"		    \
 700			"li\t%1, 0\n"			    \
 701			".set\tpop\n\t"			    \
 702			"10:\n\t"			    \
 703			".insn\n\t"			    \
 704			".section\t.fixup,\"ax\"\n\t"	    \
 705			"11:\tli\t%1, %3\n\t"		    \
 706			"j\t10b\n\t"			    \
 707			".previous\n\t"			    \
 708			".section\t__ex_table,\"a\"\n\t"    \
 709			STR(PTR)"\t1b, 11b\n\t"		    \
 710			STR(PTR)"\t2b, 11b\n\t"		    \
 711			STR(PTR)"\t3b, 11b\n\t"		    \
 712			STR(PTR)"\t4b, 11b\n\t"		    \
 713			STR(PTR)"\t5b, 11b\n\t"		    \
 714			STR(PTR)"\t6b, 11b\n\t"		    \
 715			STR(PTR)"\t7b, 11b\n\t"		    \
 716			STR(PTR)"\t8b, 11b\n\t"		    \
 717			".previous"			    \
 718			: "=&r" (value), "=r" (res)	    \
 719			: "r" (addr), "i" (-EFAULT));       \
 720} while(0)
 721#endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 722
 723#define     _StoreHW(addr, value, res, type) \
 724do {                                                        \
 725		__asm__ __volatile__ (                      \
 726			".set\tnoat\n"                      \
 727			"1:\t"type##_sb("%1", "0(%2)")"\n"  \
 728			"srl\t$1,%1, 0x8\n"                 \
 729			"2:\t"type##_sb("$1", "1(%2)")"\n"  \
 730			".set\tat\n\t"                      \
 731			"li\t%0, 0\n"                       \
 732			"3:\n\t"                            \
 733			".insn\n\t"                         \
 734			".section\t.fixup,\"ax\"\n\t"       \
 735			"4:\tli\t%0, %3\n\t"                \
 736			"j\t3b\n\t"                         \
 737			".previous\n\t"                     \
 738			".section\t__ex_table,\"a\"\n\t"    \
 739			STR(PTR)"\t1b, 4b\n\t"              \
 740			STR(PTR)"\t2b, 4b\n\t"              \
 741			".previous"                         \
 742			: "=r" (res)                        \
 743			: "r" (value), "r" (addr), "i" (-EFAULT));\
 744} while(0)
 745
 746#ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 747#define     _StoreW(addr, value, res, type)  \
 748do {                                                        \
 749		__asm__ __volatile__ (                      \
 750			"1:\t"type##_swl("%1", "3(%2)")"\n" \
 751			"2:\t"type##_swr("%1", "(%2)")"\n\t"\
 752			"li\t%0, 0\n"                       \
 753			"3:\n\t"                            \
 754			".insn\n\t"                         \
 755			".section\t.fixup,\"ax\"\n\t"       \
 756			"4:\tli\t%0, %3\n\t"                \
 757			"j\t3b\n\t"                         \
 758			".previous\n\t"                     \
 759			".section\t__ex_table,\"a\"\n\t"    \
 760			STR(PTR)"\t1b, 4b\n\t"              \
 761			STR(PTR)"\t2b, 4b\n\t"              \
 762			".previous"                         \
 763		: "=r" (res)                                \
 764		: "r" (value), "r" (addr), "i" (-EFAULT));  \
 765} while(0)
 766
 767#define     _StoreDW(addr, value, res) \
 768do {                                                        \
 769		__asm__ __volatile__ (                      \
 770			"1:\tsdl\t%1, 7(%2)\n"              \
 771			"2:\tsdr\t%1, (%2)\n\t"             \
 772			"li\t%0, 0\n"                       \
 773			"3:\n\t"                            \
 774			".insn\n\t"                         \
 775			".section\t.fixup,\"ax\"\n\t"       \
 776			"4:\tli\t%0, %3\n\t"                \
 777			"j\t3b\n\t"                         \
 778			".previous\n\t"                     \
 779			".section\t__ex_table,\"a\"\n\t"    \
 780			STR(PTR)"\t1b, 4b\n\t"              \
 781			STR(PTR)"\t2b, 4b\n\t"              \
 782			".previous"                         \
 783		: "=r" (res)                                \
 784		: "r" (value), "r" (addr), "i" (-EFAULT));  \
 785} while(0)
 786
 787#else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 788/* For CPUs without swl and sdl instructions */
 789#define     _StoreW(addr, value, res, type)  \
 790do {                                                        \
 791		__asm__ __volatile__ (                      \
 792			".set\tpush\n\t"		    \
 793			".set\tnoat\n\t"		    \
 794			"1:"type##_sb("%1", "0(%2)")"\n\t"  \
 795			"srl\t$1, %1, 0x8\n\t"		    \
 796			"2:"type##_sb("$1", "1(%2)")"\n\t"  \
 797			"srl\t$1, $1,  0x8\n\t"		    \
 798			"3:"type##_sb("$1", "2(%2)")"\n\t"  \
 799			"srl\t$1, $1, 0x8\n\t"		    \
 800			"4:"type##_sb("$1", "3(%2)")"\n\t"  \
 801			".set\tpop\n\t"			    \
 802			"li\t%0, 0\n"			    \
 803			"10:\n\t"			    \
 804			".insn\n\t"			    \
 805			".section\t.fixup,\"ax\"\n\t"	    \
 806			"11:\tli\t%0, %3\n\t"		    \
 807			"j\t10b\n\t"			    \
 808			".previous\n\t"			    \
 809			".section\t__ex_table,\"a\"\n\t"    \
 810			STR(PTR)"\t1b, 11b\n\t"		    \
 811			STR(PTR)"\t2b, 11b\n\t"		    \
 812			STR(PTR)"\t3b, 11b\n\t"		    \
 813			STR(PTR)"\t4b, 11b\n\t"		    \
 814			".previous"			    \
 815		: "=&r" (res)			    	    \
 816		: "r" (value), "r" (addr), "i" (-EFAULT)    \
 817		: "memory");                                \
 818} while(0)
 819
 820#define     _StoreDW(addr, value, res) \
 821do {                                                        \
 822		__asm__ __volatile__ (                      \
 823			".set\tpush\n\t"		    \
 824			".set\tnoat\n\t"		    \
 825			"1:sb\t%1, 0(%2)\n\t"    	    \
 826			"dsrl\t$1, %1, 0x8\n\t"		    \
 827			"2:sb\t$1, 1(%2)\n\t"    	    \
 828			"dsrl\t$1, $1, 0x8\n\t"		    \
 829			"3:sb\t$1, 2(%2)\n\t"    	    \
 830			"dsrl\t$1, $1, 0x8\n\t"		    \
 831			"4:sb\t$1, 3(%2)\n\t"    	    \
 832			"dsrl\t$1, $1, 0x8\n\t"		    \
 833			"5:sb\t$1, 4(%2)\n\t"    	    \
 834			"dsrl\t$1, $1, 0x8\n\t"		    \
 835			"6:sb\t$1, 5(%2)\n\t"    	    \
 836			"dsrl\t$1, $1, 0x8\n\t"		    \
 837			"7:sb\t$1, 6(%2)\n\t"    	    \
 838			"dsrl\t$1, $1, 0x8\n\t"		    \
 839			"8:sb\t$1, 7(%2)\n\t"    	    \
 840			"dsrl\t$1, $1, 0x8\n\t"		    \
 841			".set\tpop\n\t"			    \
 842			"li\t%0, 0\n"			    \
 843			"10:\n\t"			    \
 844			".insn\n\t"			    \
 845			".section\t.fixup,\"ax\"\n\t"	    \
 846			"11:\tli\t%0, %3\n\t"		    \
 847			"j\t10b\n\t"			    \
 848			".previous\n\t"			    \
 849			".section\t__ex_table,\"a\"\n\t"    \
 850			STR(PTR)"\t1b, 11b\n\t"		    \
 851			STR(PTR)"\t2b, 11b\n\t"		    \
 852			STR(PTR)"\t3b, 11b\n\t"		    \
 853			STR(PTR)"\t4b, 11b\n\t"		    \
 854			STR(PTR)"\t5b, 11b\n\t"		    \
 855			STR(PTR)"\t6b, 11b\n\t"		    \
 856			STR(PTR)"\t7b, 11b\n\t"		    \
 857			STR(PTR)"\t8b, 11b\n\t"		    \
 858			".previous"			    \
 859		: "=&r" (res)			    	    \
 860		: "r" (value), "r" (addr), "i" (-EFAULT)    \
 861		: "memory");                                \
 862} while(0)
 863
 864#endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 865#endif
 866
 867#define LoadHWU(addr, value, res)	_LoadHWU(addr, value, res, kernel)
 868#define LoadHWUE(addr, value, res)	_LoadHWU(addr, value, res, user)
 869#define LoadWU(addr, value, res)	_LoadWU(addr, value, res, kernel)
 870#define LoadWUE(addr, value, res)	_LoadWU(addr, value, res, user)
 871#define LoadHW(addr, value, res)	_LoadHW(addr, value, res, kernel)
 872#define LoadHWE(addr, value, res)	_LoadHW(addr, value, res, user)
 873#define LoadW(addr, value, res)		_LoadW(addr, value, res, kernel)
 874#define LoadWE(addr, value, res)	_LoadW(addr, value, res, user)
 875#define LoadDW(addr, value, res)	_LoadDW(addr, value, res)
 876
 877#define StoreHW(addr, value, res)	_StoreHW(addr, value, res, kernel)
 878#define StoreHWE(addr, value, res)	_StoreHW(addr, value, res, user)
 879#define StoreW(addr, value, res)	_StoreW(addr, value, res, kernel)
 880#define StoreWE(addr, value, res)	_StoreW(addr, value, res, user)
 881#define StoreDW(addr, value, res)	_StoreDW(addr, value, res)
 882
 883static void emulate_load_store_insn(struct pt_regs *regs,
 884	void __user *addr, unsigned int __user *pc)
 885{
 886	unsigned long origpc, orig31, value;
 887	union mips_instruction insn;
 888	unsigned int res;
 889#ifdef	CONFIG_EVA
 890	mm_segment_t seg;
 891#endif
 892	origpc = (unsigned long)pc;
 893	orig31 = regs->regs[31];
 894
 895	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
 896
 897	/*
 898	 * This load never faults.
 899	 */
 900	__get_user(insn.word, pc);
 901
 902	switch (insn.i_format.opcode) {
 903		/*
 904		 * These are instructions that a compiler doesn't generate.  We
 905		 * can assume therefore that the code is MIPS-aware and
 906		 * really buggy.  Emulating these instructions would break the
 907		 * semantics anyway.
 908		 */
 909	case ll_op:
 910	case lld_op:
 911	case sc_op:
 912	case scd_op:
 913
 914		/*
 915		 * For these instructions the only way to create an address
 916		 * error is an attempted access to kernel/supervisor address
 917		 * space.
 918		 */
 919	case ldl_op:
 920	case ldr_op:
 921	case lwl_op:
 922	case lwr_op:
 923	case sdl_op:
 924	case sdr_op:
 925	case swl_op:
 926	case swr_op:
 927	case lb_op:
 928	case lbu_op:
 929	case sb_op:
 930		goto sigbus;
 931
 932		/*
 933		 * The remaining opcodes are the ones that are really of
 934		 * interest.
 935		 */
 936	case spec3_op:
 937		if (insn.dsp_format.func == lx_op) {
 938			switch (insn.dsp_format.op) {
 939			case lwx_op:
 940				if (!access_ok(addr, 4))
 941					goto sigbus;
 942				LoadW(addr, value, res);
 943				if (res)
 944					goto fault;
 945				compute_return_epc(regs);
 946				regs->regs[insn.dsp_format.rd] = value;
 947				break;
 948			case lhx_op:
 949				if (!access_ok(addr, 2))
 950					goto sigbus;
 951				LoadHW(addr, value, res);
 952				if (res)
 953					goto fault;
 954				compute_return_epc(regs);
 955				regs->regs[insn.dsp_format.rd] = value;
 956				break;
 957			default:
 958				goto sigill;
 959			}
 960		}
 961#ifdef CONFIG_EVA
 962		else {
 963			/*
 964			 * we can land here only from kernel accessing user
 965			 * memory, so we need to "switch" the address limit to
 966			 * user space, so that address check can work properly.
 967			 */
 968			seg = get_fs();
 969			set_fs(USER_DS);
 970			switch (insn.spec3_format.func) {
 971			case lhe_op:
 972				if (!access_ok(addr, 2)) {
 973					set_fs(seg);
 974					goto sigbus;
 975				}
 976				LoadHWE(addr, value, res);
 977				if (res) {
 978					set_fs(seg);
 979					goto fault;
 980				}
 981				compute_return_epc(regs);
 982				regs->regs[insn.spec3_format.rt] = value;
 983				break;
 984			case lwe_op:
 985				if (!access_ok(addr, 4)) {
 986					set_fs(seg);
 987					goto sigbus;
 988				}
 989				LoadWE(addr, value, res);
 990				if (res) {
 991					set_fs(seg);
 992					goto fault;
 993				}
 994				compute_return_epc(regs);
 995				regs->regs[insn.spec3_format.rt] = value;
 996				break;
 997			case lhue_op:
 998				if (!access_ok(addr, 2)) {
 999					set_fs(seg);
1000					goto sigbus;
1001				}
1002				LoadHWUE(addr, value, res);
1003				if (res) {
1004					set_fs(seg);
1005					goto fault;
1006				}
1007				compute_return_epc(regs);
1008				regs->regs[insn.spec3_format.rt] = value;
1009				break;
1010			case she_op:
1011				if (!access_ok(addr, 2)) {
1012					set_fs(seg);
1013					goto sigbus;
1014				}
1015				compute_return_epc(regs);
1016				value = regs->regs[insn.spec3_format.rt];
1017				StoreHWE(addr, value, res);
1018				if (res) {
1019					set_fs(seg);
1020					goto fault;
1021				}
1022				break;
1023			case swe_op:
1024				if (!access_ok(addr, 4)) {
1025					set_fs(seg);
1026					goto sigbus;
1027				}
1028				compute_return_epc(regs);
1029				value = regs->regs[insn.spec3_format.rt];
1030				StoreWE(addr, value, res);
1031				if (res) {
1032					set_fs(seg);
1033					goto fault;
1034				}
1035				break;
1036			default:
1037				set_fs(seg);
1038				goto sigill;
1039			}
1040			set_fs(seg);
1041		}
1042#endif
1043		break;
1044	case lh_op:
1045		if (!access_ok(addr, 2))
1046			goto sigbus;
1047
1048		if (IS_ENABLED(CONFIG_EVA)) {
1049			if (uaccess_kernel())
1050				LoadHW(addr, value, res);
1051			else
1052				LoadHWE(addr, value, res);
1053		} else {
1054			LoadHW(addr, value, res);
1055		}
1056
1057		if (res)
1058			goto fault;
1059		compute_return_epc(regs);
1060		regs->regs[insn.i_format.rt] = value;
1061		break;
1062
1063	case lw_op:
1064		if (!access_ok(addr, 4))
1065			goto sigbus;
1066
1067		if (IS_ENABLED(CONFIG_EVA)) {
1068			if (uaccess_kernel())
1069				LoadW(addr, value, res);
1070			else
1071				LoadWE(addr, value, res);
1072		} else {
1073			LoadW(addr, value, res);
1074		}
1075
1076		if (res)
1077			goto fault;
1078		compute_return_epc(regs);
1079		regs->regs[insn.i_format.rt] = value;
1080		break;
1081
1082	case lhu_op:
1083		if (!access_ok(addr, 2))
1084			goto sigbus;
1085
1086		if (IS_ENABLED(CONFIG_EVA)) {
1087			if (uaccess_kernel())
1088				LoadHWU(addr, value, res);
1089			else
1090				LoadHWUE(addr, value, res);
1091		} else {
1092			LoadHWU(addr, value, res);
1093		}
1094
1095		if (res)
1096			goto fault;
1097		compute_return_epc(regs);
1098		regs->regs[insn.i_format.rt] = value;
1099		break;
1100
1101	case lwu_op:
1102#ifdef CONFIG_64BIT
1103		/*
1104		 * A 32-bit kernel might be running on a 64-bit processor.  But
1105		 * if we're on a 32-bit processor and an i-cache incoherency
1106		 * or race makes us see a 64-bit instruction here the sdl/sdr
1107		 * would blow up, so for now we don't handle unaligned 64-bit
1108		 * instructions on 32-bit kernels.
1109		 */
1110		if (!access_ok(addr, 4))
1111			goto sigbus;
1112
1113		LoadWU(addr, value, res);
1114		if (res)
1115			goto fault;
1116		compute_return_epc(regs);
1117		regs->regs[insn.i_format.rt] = value;
1118		break;
1119#endif /* CONFIG_64BIT */
1120
1121		/* Cannot handle 64-bit instructions in 32-bit kernel */
1122		goto sigill;
1123
1124	case ld_op:
1125#ifdef CONFIG_64BIT
1126		/*
1127		 * A 32-bit kernel might be running on a 64-bit processor.  But
1128		 * if we're on a 32-bit processor and an i-cache incoherency
1129		 * or race makes us see a 64-bit instruction here the sdl/sdr
1130		 * would blow up, so for now we don't handle unaligned 64-bit
1131		 * instructions on 32-bit kernels.
1132		 */
1133		if (!access_ok(addr, 8))
1134			goto sigbus;
1135
1136		LoadDW(addr, value, res);
1137		if (res)
1138			goto fault;
1139		compute_return_epc(regs);
1140		regs->regs[insn.i_format.rt] = value;
1141		break;
1142#endif /* CONFIG_64BIT */
1143
1144		/* Cannot handle 64-bit instructions in 32-bit kernel */
1145		goto sigill;
1146
1147	case sh_op:
1148		if (!access_ok(addr, 2))
1149			goto sigbus;
1150
1151		compute_return_epc(regs);
1152		value = regs->regs[insn.i_format.rt];
1153
1154		if (IS_ENABLED(CONFIG_EVA)) {
1155			if (uaccess_kernel())
1156				StoreHW(addr, value, res);
1157			else
1158				StoreHWE(addr, value, res);
1159		} else {
1160			StoreHW(addr, value, res);
1161		}
1162
1163		if (res)
1164			goto fault;
1165		break;
1166
1167	case sw_op:
1168		if (!access_ok(addr, 4))
1169			goto sigbus;
1170
1171		compute_return_epc(regs);
1172		value = regs->regs[insn.i_format.rt];
1173
1174		if (IS_ENABLED(CONFIG_EVA)) {
1175			if (uaccess_kernel())
1176				StoreW(addr, value, res);
1177			else
1178				StoreWE(addr, value, res);
1179		} else {
1180			StoreW(addr, value, res);
1181		}
1182
1183		if (res)
1184			goto fault;
1185		break;
1186
1187	case sd_op:
1188#ifdef CONFIG_64BIT
1189		/*
1190		 * A 32-bit kernel might be running on a 64-bit processor.  But
1191		 * if we're on a 32-bit processor and an i-cache incoherency
1192		 * or race makes us see a 64-bit instruction here the sdl/sdr
1193		 * would blow up, so for now we don't handle unaligned 64-bit
1194		 * instructions on 32-bit kernels.
1195		 */
1196		if (!access_ok(addr, 8))
1197			goto sigbus;
1198
1199		compute_return_epc(regs);
1200		value = regs->regs[insn.i_format.rt];
1201		StoreDW(addr, value, res);
1202		if (res)
1203			goto fault;
1204		break;
1205#endif /* CONFIG_64BIT */
1206
1207		/* Cannot handle 64-bit instructions in 32-bit kernel */
1208		goto sigill;
1209
1210#ifdef CONFIG_MIPS_FP_SUPPORT
1211
1212	case lwc1_op:
1213	case ldc1_op:
1214	case swc1_op:
1215	case sdc1_op:
1216	case cop1x_op: {
1217		void __user *fault_addr = NULL;
1218
1219		die_if_kernel("Unaligned FP access in kernel code", regs);
1220		BUG_ON(!used_math());
1221
1222		res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1223					       &fault_addr);
1224		own_fpu(1);	/* Restore FPU state. */
1225
1226		/* Signal if something went wrong. */
1227		process_fpemu_return(res, fault_addr, 0);
1228
1229		if (res == 0)
1230			break;
1231		return;
1232	}
1233#endif /* CONFIG_MIPS_FP_SUPPORT */
1234
1235#ifdef CONFIG_CPU_HAS_MSA
1236
1237	case msa_op: {
1238		unsigned int wd, preempted;
1239		enum msa_2b_fmt df;
1240		union fpureg *fpr;
1241
1242		if (!cpu_has_msa)
1243			goto sigill;
1244
1245		/*
1246		 * If we've reached this point then userland should have taken
1247		 * the MSA disabled exception & initialised vector context at
1248		 * some point in the past.
1249		 */
1250		BUG_ON(!thread_msa_context_live());
1251
1252		df = insn.msa_mi10_format.df;
1253		wd = insn.msa_mi10_format.wd;
1254		fpr = &current->thread.fpu.fpr[wd];
1255
1256		switch (insn.msa_mi10_format.func) {
1257		case msa_ld_op:
1258			if (!access_ok(addr, sizeof(*fpr)))
1259				goto sigbus;
1260
1261			do {
1262				/*
1263				 * If we have live MSA context keep track of
1264				 * whether we get preempted in order to avoid
1265				 * the register context we load being clobbered
1266				 * by the live context as it's saved during
1267				 * preemption. If we don't have live context
1268				 * then it can't be saved to clobber the value
1269				 * we load.
1270				 */
1271				preempted = test_thread_flag(TIF_USEDMSA);
1272
1273				res = __copy_from_user_inatomic(fpr, addr,
1274								sizeof(*fpr));
1275				if (res)
1276					goto fault;
1277
1278				/*
1279				 * Update the hardware register if it is in use
1280				 * by the task in this quantum, in order to
1281				 * avoid having to save & restore the whole
1282				 * vector context.
1283				 */
1284				preempt_disable();
1285				if (test_thread_flag(TIF_USEDMSA)) {
1286					write_msa_wr(wd, fpr, df);
1287					preempted = 0;
1288				}
1289				preempt_enable();
1290			} while (preempted);
1291			break;
1292
1293		case msa_st_op:
1294			if (!access_ok(addr, sizeof(*fpr)))
1295				goto sigbus;
1296
1297			/*
1298			 * Update from the hardware register if it is in use by
1299			 * the task in this quantum, in order to avoid having to
1300			 * save & restore the whole vector context.
1301			 */
1302			preempt_disable();
1303			if (test_thread_flag(TIF_USEDMSA))
1304				read_msa_wr(wd, fpr, df);
1305			preempt_enable();
1306
1307			res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr));
1308			if (res)
1309				goto fault;
1310			break;
1311
1312		default:
1313			goto sigbus;
1314		}
1315
1316		compute_return_epc(regs);
1317		break;
1318	}
1319#endif /* CONFIG_CPU_HAS_MSA */
1320
1321#ifndef CONFIG_CPU_MIPSR6
1322	/*
1323	 * COP2 is available to implementor for application specific use.
1324	 * It's up to applications to register a notifier chain and do
1325	 * whatever they have to do, including possible sending of signals.
1326	 *
1327	 * This instruction has been reallocated in Release 6
1328	 */
1329	case lwc2_op:
1330		cu2_notifier_call_chain(CU2_LWC2_OP, regs);
1331		break;
1332
1333	case ldc2_op:
1334		cu2_notifier_call_chain(CU2_LDC2_OP, regs);
1335		break;
1336
1337	case swc2_op:
1338		cu2_notifier_call_chain(CU2_SWC2_OP, regs);
1339		break;
1340
1341	case sdc2_op:
1342		cu2_notifier_call_chain(CU2_SDC2_OP, regs);
1343		break;
1344#endif
1345	default:
1346		/*
1347		 * Pheeee...  We encountered an yet unknown instruction or
1348		 * cache coherence problem.  Die sucker, die ...
1349		 */
1350		goto sigill;
1351	}
1352
1353#ifdef CONFIG_DEBUG_FS
1354	unaligned_instructions++;
1355#endif
1356
1357	return;
1358
1359fault:
1360	/* roll back jump/branch */
1361	regs->cp0_epc = origpc;
1362	regs->regs[31] = orig31;
1363	/* Did we have an exception handler installed? */
1364	if (fixup_exception(regs))
1365		return;
1366
1367	die_if_kernel("Unhandled kernel unaligned access", regs);
1368	force_sig(SIGSEGV);
1369
1370	return;
1371
1372sigbus:
1373	die_if_kernel("Unhandled kernel unaligned access", regs);
1374	force_sig(SIGBUS);
1375
1376	return;
1377
1378sigill:
1379	die_if_kernel
1380	    ("Unhandled kernel unaligned access or invalid instruction", regs);
1381	force_sig(SIGILL);
1382}
1383
1384/* Recode table from 16-bit register notation to 32-bit GPR. */
1385const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
1386
1387/* Recode table from 16-bit STORE register notation to 32-bit GPR. */
1388static const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
1389
1390static void emulate_load_store_microMIPS(struct pt_regs *regs,
1391					 void __user *addr)
1392{
1393	unsigned long value;
1394	unsigned int res;
1395	int i;
1396	unsigned int reg = 0, rvar;
1397	unsigned long orig31;
1398	u16 __user *pc16;
1399	u16 halfword;
1400	unsigned int word;
1401	unsigned long origpc, contpc;
1402	union mips_instruction insn;
1403	struct mm_decoded_insn mminsn;
 
1404
1405	origpc = regs->cp0_epc;
1406	orig31 = regs->regs[31];
1407
1408	mminsn.micro_mips_mode = 1;
1409
1410	/*
1411	 * This load never faults.
1412	 */
1413	pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
1414	__get_user(halfword, pc16);
1415	pc16++;
1416	contpc = regs->cp0_epc + 2;
1417	word = ((unsigned int)halfword << 16);
1418	mminsn.pc_inc = 2;
1419
1420	if (!mm_insn_16bit(halfword)) {
1421		__get_user(halfword, pc16);
1422		pc16++;
1423		contpc = regs->cp0_epc + 4;
1424		mminsn.pc_inc = 4;
1425		word |= halfword;
1426	}
1427	mminsn.insn = word;
1428
1429	if (get_user(halfword, pc16))
1430		goto fault;
1431	mminsn.next_pc_inc = 2;
1432	word = ((unsigned int)halfword << 16);
1433
1434	if (!mm_insn_16bit(halfword)) {
1435		pc16++;
1436		if (get_user(halfword, pc16))
1437			goto fault;
1438		mminsn.next_pc_inc = 4;
1439		word |= halfword;
1440	}
1441	mminsn.next_insn = word;
1442
1443	insn = (union mips_instruction)(mminsn.insn);
1444	if (mm_isBranchInstr(regs, mminsn, &contpc))
1445		insn = (union mips_instruction)(mminsn.next_insn);
1446
1447	/*  Parse instruction to find what to do */
1448
1449	switch (insn.mm_i_format.opcode) {
1450
1451	case mm_pool32a_op:
1452		switch (insn.mm_x_format.func) {
1453		case mm_lwxs_op:
1454			reg = insn.mm_x_format.rd;
1455			goto loadW;
1456		}
1457
1458		goto sigbus;
1459
1460	case mm_pool32b_op:
1461		switch (insn.mm_m_format.func) {
1462		case mm_lwp_func:
1463			reg = insn.mm_m_format.rd;
1464			if (reg == 31)
1465				goto sigbus;
1466
1467			if (!access_ok(addr, 8))
1468				goto sigbus;
1469
1470			LoadW(addr, value, res);
1471			if (res)
1472				goto fault;
1473			regs->regs[reg] = value;
1474			addr += 4;
1475			LoadW(addr, value, res);
1476			if (res)
1477				goto fault;
1478			regs->regs[reg + 1] = value;
1479			goto success;
1480
1481		case mm_swp_func:
1482			reg = insn.mm_m_format.rd;
1483			if (reg == 31)
1484				goto sigbus;
1485
1486			if (!access_ok(addr, 8))
1487				goto sigbus;
1488
1489			value = regs->regs[reg];
1490			StoreW(addr, value, res);
1491			if (res)
1492				goto fault;
1493			addr += 4;
1494			value = regs->regs[reg + 1];
1495			StoreW(addr, value, res);
1496			if (res)
1497				goto fault;
1498			goto success;
1499
1500		case mm_ldp_func:
1501#ifdef CONFIG_64BIT
1502			reg = insn.mm_m_format.rd;
1503			if (reg == 31)
1504				goto sigbus;
1505
1506			if (!access_ok(addr, 16))
1507				goto sigbus;
1508
1509			LoadDW(addr, value, res);
1510			if (res)
1511				goto fault;
1512			regs->regs[reg] = value;
1513			addr += 8;
1514			LoadDW(addr, value, res);
1515			if (res)
1516				goto fault;
1517			regs->regs[reg + 1] = value;
1518			goto success;
1519#endif /* CONFIG_64BIT */
1520
1521			goto sigill;
1522
1523		case mm_sdp_func:
1524#ifdef CONFIG_64BIT
1525			reg = insn.mm_m_format.rd;
1526			if (reg == 31)
1527				goto sigbus;
1528
1529			if (!access_ok(addr, 16))
1530				goto sigbus;
1531
1532			value = regs->regs[reg];
1533			StoreDW(addr, value, res);
1534			if (res)
1535				goto fault;
1536			addr += 8;
1537			value = regs->regs[reg + 1];
1538			StoreDW(addr, value, res);
1539			if (res)
1540				goto fault;
1541			goto success;
1542#endif /* CONFIG_64BIT */
1543
1544			goto sigill;
1545
1546		case mm_lwm32_func:
1547			reg = insn.mm_m_format.rd;
1548			rvar = reg & 0xf;
1549			if ((rvar > 9) || !reg)
1550				goto sigill;
1551			if (reg & 0x10) {
1552				if (!access_ok(addr, 4 * (rvar + 1)))
1553					goto sigbus;
1554			} else {
1555				if (!access_ok(addr, 4 * rvar))
1556					goto sigbus;
1557			}
1558			if (rvar == 9)
1559				rvar = 8;
1560			for (i = 16; rvar; rvar--, i++) {
1561				LoadW(addr, value, res);
1562				if (res)
1563					goto fault;
1564				addr += 4;
1565				regs->regs[i] = value;
1566			}
1567			if ((reg & 0xf) == 9) {
1568				LoadW(addr, value, res);
1569				if (res)
1570					goto fault;
1571				addr += 4;
1572				regs->regs[30] = value;
1573			}
1574			if (reg & 0x10) {
1575				LoadW(addr, value, res);
1576				if (res)
1577					goto fault;
1578				regs->regs[31] = value;
1579			}
1580			goto success;
1581
1582		case mm_swm32_func:
1583			reg = insn.mm_m_format.rd;
1584			rvar = reg & 0xf;
1585			if ((rvar > 9) || !reg)
1586				goto sigill;
1587			if (reg & 0x10) {
1588				if (!access_ok(addr, 4 * (rvar + 1)))
1589					goto sigbus;
1590			} else {
1591				if (!access_ok(addr, 4 * rvar))
1592					goto sigbus;
1593			}
1594			if (rvar == 9)
1595				rvar = 8;
1596			for (i = 16; rvar; rvar--, i++) {
1597				value = regs->regs[i];
1598				StoreW(addr, value, res);
1599				if (res)
1600					goto fault;
1601				addr += 4;
1602			}
1603			if ((reg & 0xf) == 9) {
1604				value = regs->regs[30];
1605				StoreW(addr, value, res);
1606				if (res)
1607					goto fault;
1608				addr += 4;
1609			}
1610			if (reg & 0x10) {
1611				value = regs->regs[31];
1612				StoreW(addr, value, res);
1613				if (res)
1614					goto fault;
1615			}
1616			goto success;
1617
1618		case mm_ldm_func:
1619#ifdef CONFIG_64BIT
1620			reg = insn.mm_m_format.rd;
1621			rvar = reg & 0xf;
1622			if ((rvar > 9) || !reg)
1623				goto sigill;
1624			if (reg & 0x10) {
1625				if (!access_ok(addr, 8 * (rvar + 1)))
1626					goto sigbus;
1627			} else {
1628				if (!access_ok(addr, 8 * rvar))
1629					goto sigbus;
1630			}
1631			if (rvar == 9)
1632				rvar = 8;
1633
1634			for (i = 16; rvar; rvar--, i++) {
1635				LoadDW(addr, value, res);
1636				if (res)
1637					goto fault;
1638				addr += 4;
1639				regs->regs[i] = value;
1640			}
1641			if ((reg & 0xf) == 9) {
1642				LoadDW(addr, value, res);
1643				if (res)
1644					goto fault;
1645				addr += 8;
1646				regs->regs[30] = value;
1647			}
1648			if (reg & 0x10) {
1649				LoadDW(addr, value, res);
1650				if (res)
1651					goto fault;
1652				regs->regs[31] = value;
1653			}
1654			goto success;
1655#endif /* CONFIG_64BIT */
1656
1657			goto sigill;
1658
1659		case mm_sdm_func:
1660#ifdef CONFIG_64BIT
1661			reg = insn.mm_m_format.rd;
1662			rvar = reg & 0xf;
1663			if ((rvar > 9) || !reg)
1664				goto sigill;
1665			if (reg & 0x10) {
1666				if (!access_ok(addr, 8 * (rvar + 1)))
1667					goto sigbus;
1668			} else {
1669				if (!access_ok(addr, 8 * rvar))
1670					goto sigbus;
1671			}
1672			if (rvar == 9)
1673				rvar = 8;
1674
1675			for (i = 16; rvar; rvar--, i++) {
1676				value = regs->regs[i];
1677				StoreDW(addr, value, res);
1678				if (res)
1679					goto fault;
1680				addr += 8;
1681			}
1682			if ((reg & 0xf) == 9) {
1683				value = regs->regs[30];
1684				StoreDW(addr, value, res);
1685				if (res)
1686					goto fault;
1687				addr += 8;
1688			}
1689			if (reg & 0x10) {
1690				value = regs->regs[31];
1691				StoreDW(addr, value, res);
1692				if (res)
1693					goto fault;
1694			}
1695			goto success;
1696#endif /* CONFIG_64BIT */
1697
1698			goto sigill;
1699
1700			/*  LWC2, SWC2, LDC2, SDC2 are not serviced */
1701		}
1702
1703		goto sigbus;
1704
1705	case mm_pool32c_op:
1706		switch (insn.mm_m_format.func) {
1707		case mm_lwu_func:
1708			reg = insn.mm_m_format.rd;
1709			goto loadWU;
1710		}
1711
1712		/*  LL,SC,LLD,SCD are not serviced */
1713		goto sigbus;
1714
1715#ifdef CONFIG_MIPS_FP_SUPPORT
1716	case mm_pool32f_op:
1717		switch (insn.mm_x_format.func) {
1718		case mm_lwxc1_func:
1719		case mm_swxc1_func:
1720		case mm_ldxc1_func:
1721		case mm_sdxc1_func:
1722			goto fpu_emul;
1723		}
1724
1725		goto sigbus;
1726
1727	case mm_ldc132_op:
1728	case mm_sdc132_op:
1729	case mm_lwc132_op:
1730	case mm_swc132_op: {
1731		void __user *fault_addr = NULL;
1732
1733fpu_emul:
1734		/* roll back jump/branch */
1735		regs->cp0_epc = origpc;
1736		regs->regs[31] = orig31;
1737
1738		die_if_kernel("Unaligned FP access in kernel code", regs);
1739		BUG_ON(!used_math());
1740		BUG_ON(!is_fpu_owner());
1741
1742		res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1743					       &fault_addr);
1744		own_fpu(1);	/* restore FPU state */
1745
1746		/* If something went wrong, signal */
1747		process_fpemu_return(res, fault_addr, 0);
1748
1749		if (res == 0)
1750			goto success;
1751		return;
1752	}
1753#endif /* CONFIG_MIPS_FP_SUPPORT */
1754
1755	case mm_lh32_op:
1756		reg = insn.mm_i_format.rt;
1757		goto loadHW;
1758
1759	case mm_lhu32_op:
1760		reg = insn.mm_i_format.rt;
1761		goto loadHWU;
1762
1763	case mm_lw32_op:
1764		reg = insn.mm_i_format.rt;
1765		goto loadW;
1766
1767	case mm_sh32_op:
1768		reg = insn.mm_i_format.rt;
1769		goto storeHW;
1770
1771	case mm_sw32_op:
1772		reg = insn.mm_i_format.rt;
1773		goto storeW;
1774
1775	case mm_ld32_op:
1776		reg = insn.mm_i_format.rt;
1777		goto loadDW;
1778
1779	case mm_sd32_op:
1780		reg = insn.mm_i_format.rt;
1781		goto storeDW;
1782
1783	case mm_pool16c_op:
1784		switch (insn.mm16_m_format.func) {
1785		case mm_lwm16_op:
1786			reg = insn.mm16_m_format.rlist;
1787			rvar = reg + 1;
1788			if (!access_ok(addr, 4 * rvar))
1789				goto sigbus;
1790
1791			for (i = 16; rvar; rvar--, i++) {
1792				LoadW(addr, value, res);
1793				if (res)
1794					goto fault;
1795				addr += 4;
1796				regs->regs[i] = value;
1797			}
1798			LoadW(addr, value, res);
1799			if (res)
1800				goto fault;
1801			regs->regs[31] = value;
1802
1803			goto success;
1804
1805		case mm_swm16_op:
1806			reg = insn.mm16_m_format.rlist;
1807			rvar = reg + 1;
1808			if (!access_ok(addr, 4 * rvar))
1809				goto sigbus;
1810
1811			for (i = 16; rvar; rvar--, i++) {
1812				value = regs->regs[i];
1813				StoreW(addr, value, res);
1814				if (res)
1815					goto fault;
1816				addr += 4;
1817			}
1818			value = regs->regs[31];
1819			StoreW(addr, value, res);
1820			if (res)
1821				goto fault;
1822
1823			goto success;
1824
1825		}
1826
1827		goto sigbus;
1828
1829	case mm_lhu16_op:
1830		reg = reg16to32[insn.mm16_rb_format.rt];
1831		goto loadHWU;
1832
1833	case mm_lw16_op:
1834		reg = reg16to32[insn.mm16_rb_format.rt];
1835		goto loadW;
1836
1837	case mm_sh16_op:
1838		reg = reg16to32st[insn.mm16_rb_format.rt];
1839		goto storeHW;
1840
1841	case mm_sw16_op:
1842		reg = reg16to32st[insn.mm16_rb_format.rt];
1843		goto storeW;
1844
1845	case mm_lwsp16_op:
1846		reg = insn.mm16_r5_format.rt;
1847		goto loadW;
1848
1849	case mm_swsp16_op:
1850		reg = insn.mm16_r5_format.rt;
1851		goto storeW;
1852
1853	case mm_lwgp16_op:
1854		reg = reg16to32[insn.mm16_r3_format.rt];
1855		goto loadW;
1856
1857	default:
1858		goto sigill;
1859	}
1860
1861loadHW:
1862	if (!access_ok(addr, 2))
1863		goto sigbus;
1864
1865	LoadHW(addr, value, res);
1866	if (res)
1867		goto fault;
1868	regs->regs[reg] = value;
1869	goto success;
1870
1871loadHWU:
1872	if (!access_ok(addr, 2))
1873		goto sigbus;
1874
1875	LoadHWU(addr, value, res);
1876	if (res)
1877		goto fault;
1878	regs->regs[reg] = value;
1879	goto success;
1880
1881loadW:
1882	if (!access_ok(addr, 4))
1883		goto sigbus;
1884
1885	LoadW(addr, value, res);
1886	if (res)
1887		goto fault;
1888	regs->regs[reg] = value;
1889	goto success;
1890
1891loadWU:
1892#ifdef CONFIG_64BIT
1893	/*
1894	 * A 32-bit kernel might be running on a 64-bit processor.  But
1895	 * if we're on a 32-bit processor and an i-cache incoherency
1896	 * or race makes us see a 64-bit instruction here the sdl/sdr
1897	 * would blow up, so for now we don't handle unaligned 64-bit
1898	 * instructions on 32-bit kernels.
1899	 */
1900	if (!access_ok(addr, 4))
1901		goto sigbus;
1902
1903	LoadWU(addr, value, res);
1904	if (res)
1905		goto fault;
1906	regs->regs[reg] = value;
1907	goto success;
1908#endif /* CONFIG_64BIT */
1909
1910	/* Cannot handle 64-bit instructions in 32-bit kernel */
1911	goto sigill;
1912
1913loadDW:
1914#ifdef CONFIG_64BIT
1915	/*
1916	 * A 32-bit kernel might be running on a 64-bit processor.  But
1917	 * if we're on a 32-bit processor and an i-cache incoherency
1918	 * or race makes us see a 64-bit instruction here the sdl/sdr
1919	 * would blow up, so for now we don't handle unaligned 64-bit
1920	 * instructions on 32-bit kernels.
1921	 */
1922	if (!access_ok(addr, 8))
1923		goto sigbus;
1924
1925	LoadDW(addr, value, res);
1926	if (res)
1927		goto fault;
1928	regs->regs[reg] = value;
1929	goto success;
1930#endif /* CONFIG_64BIT */
1931
1932	/* Cannot handle 64-bit instructions in 32-bit kernel */
1933	goto sigill;
1934
1935storeHW:
1936	if (!access_ok(addr, 2))
1937		goto sigbus;
1938
1939	value = regs->regs[reg];
1940	StoreHW(addr, value, res);
1941	if (res)
1942		goto fault;
1943	goto success;
1944
1945storeW:
1946	if (!access_ok(addr, 4))
1947		goto sigbus;
1948
1949	value = regs->regs[reg];
1950	StoreW(addr, value, res);
1951	if (res)
1952		goto fault;
1953	goto success;
1954
1955storeDW:
1956#ifdef CONFIG_64BIT
1957	/*
1958	 * A 32-bit kernel might be running on a 64-bit processor.  But
1959	 * if we're on a 32-bit processor and an i-cache incoherency
1960	 * or race makes us see a 64-bit instruction here the sdl/sdr
1961	 * would blow up, so for now we don't handle unaligned 64-bit
1962	 * instructions on 32-bit kernels.
1963	 */
1964	if (!access_ok(addr, 8))
1965		goto sigbus;
1966
1967	value = regs->regs[reg];
1968	StoreDW(addr, value, res);
1969	if (res)
1970		goto fault;
1971	goto success;
1972#endif /* CONFIG_64BIT */
1973
1974	/* Cannot handle 64-bit instructions in 32-bit kernel */
1975	goto sigill;
1976
1977success:
1978	regs->cp0_epc = contpc;	/* advance or branch */
1979
1980#ifdef CONFIG_DEBUG_FS
1981	unaligned_instructions++;
1982#endif
1983	return;
1984
1985fault:
1986	/* roll back jump/branch */
1987	regs->cp0_epc = origpc;
1988	regs->regs[31] = orig31;
1989	/* Did we have an exception handler installed? */
1990	if (fixup_exception(regs))
1991		return;
1992
1993	die_if_kernel("Unhandled kernel unaligned access", regs);
1994	force_sig(SIGSEGV);
1995
1996	return;
1997
1998sigbus:
1999	die_if_kernel("Unhandled kernel unaligned access", regs);
2000	force_sig(SIGBUS);
2001
2002	return;
2003
2004sigill:
2005	die_if_kernel
2006	    ("Unhandled kernel unaligned access or invalid instruction", regs);
2007	force_sig(SIGILL);
2008}
2009
2010static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
2011{
2012	unsigned long value;
2013	unsigned int res;
2014	int reg;
2015	unsigned long orig31;
2016	u16 __user *pc16;
2017	unsigned long origpc;
2018	union mips16e_instruction mips16inst, oldinst;
2019	unsigned int opcode;
2020	int extended = 0;
 
2021
2022	origpc = regs->cp0_epc;
2023	orig31 = regs->regs[31];
2024	pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
2025	/*
2026	 * This load never faults.
2027	 */
2028	__get_user(mips16inst.full, pc16);
2029	oldinst = mips16inst;
2030
2031	/* skip EXTEND instruction */
2032	if (mips16inst.ri.opcode == MIPS16e_extend_op) {
2033		extended = 1;
2034		pc16++;
2035		__get_user(mips16inst.full, pc16);
2036	} else if (delay_slot(regs)) {
2037		/*  skip jump instructions */
2038		/*  JAL/JALX are 32 bits but have OPCODE in first short int */
2039		if (mips16inst.ri.opcode == MIPS16e_jal_op)
2040			pc16++;
2041		pc16++;
2042		if (get_user(mips16inst.full, pc16))
2043			goto sigbus;
2044	}
2045
2046	opcode = mips16inst.ri.opcode;
2047	switch (opcode) {
2048	case MIPS16e_i64_op:	/* I64 or RI64 instruction */
2049		switch (mips16inst.i64.func) {	/* I64/RI64 func field check */
2050		case MIPS16e_ldpc_func:
2051		case MIPS16e_ldsp_func:
2052			reg = reg16to32[mips16inst.ri64.ry];
2053			goto loadDW;
2054
2055		case MIPS16e_sdsp_func:
2056			reg = reg16to32[mips16inst.ri64.ry];
2057			goto writeDW;
2058
2059		case MIPS16e_sdrasp_func:
2060			reg = 29;	/* GPRSP */
2061			goto writeDW;
2062		}
2063
2064		goto sigbus;
2065
2066	case MIPS16e_swsp_op:
2067		reg = reg16to32[mips16inst.ri.rx];
2068		if (extended && cpu_has_mips16e2)
2069			switch (mips16inst.ri.imm >> 5) {
2070			case 0:		/* SWSP */
2071			case 1:		/* SWGP */
2072				break;
2073			case 2:		/* SHGP */
2074				opcode = MIPS16e_sh_op;
2075				break;
2076			default:
2077				goto sigbus;
2078			}
2079		break;
2080
2081	case MIPS16e_lwpc_op:
2082		reg = reg16to32[mips16inst.ri.rx];
2083		break;
2084
2085	case MIPS16e_lwsp_op:
2086		reg = reg16to32[mips16inst.ri.rx];
2087		if (extended && cpu_has_mips16e2)
2088			switch (mips16inst.ri.imm >> 5) {
2089			case 0:		/* LWSP */
2090			case 1:		/* LWGP */
2091				break;
2092			case 2:		/* LHGP */
2093				opcode = MIPS16e_lh_op;
2094				break;
2095			case 4:		/* LHUGP */
2096				opcode = MIPS16e_lhu_op;
2097				break;
2098			default:
2099				goto sigbus;
2100			}
2101		break;
2102
2103	case MIPS16e_i8_op:
2104		if (mips16inst.i8.func != MIPS16e_swrasp_func)
2105			goto sigbus;
2106		reg = 29;	/* GPRSP */
2107		break;
2108
2109	default:
2110		reg = reg16to32[mips16inst.rri.ry];
2111		break;
2112	}
2113
2114	switch (opcode) {
2115
2116	case MIPS16e_lb_op:
2117	case MIPS16e_lbu_op:
2118	case MIPS16e_sb_op:
2119		goto sigbus;
2120
2121	case MIPS16e_lh_op:
2122		if (!access_ok(addr, 2))
2123			goto sigbus;
2124
2125		LoadHW(addr, value, res);
2126		if (res)
2127			goto fault;
2128		MIPS16e_compute_return_epc(regs, &oldinst);
2129		regs->regs[reg] = value;
2130		break;
2131
2132	case MIPS16e_lhu_op:
2133		if (!access_ok(addr, 2))
2134			goto sigbus;
2135
2136		LoadHWU(addr, value, res);
2137		if (res)
2138			goto fault;
2139		MIPS16e_compute_return_epc(regs, &oldinst);
2140		regs->regs[reg] = value;
2141		break;
2142
2143	case MIPS16e_lw_op:
2144	case MIPS16e_lwpc_op:
2145	case MIPS16e_lwsp_op:
2146		if (!access_ok(addr, 4))
2147			goto sigbus;
2148
2149		LoadW(addr, value, res);
2150		if (res)
2151			goto fault;
2152		MIPS16e_compute_return_epc(regs, &oldinst);
2153		regs->regs[reg] = value;
2154		break;
2155
2156	case MIPS16e_lwu_op:
2157#ifdef CONFIG_64BIT
2158		/*
2159		 * A 32-bit kernel might be running on a 64-bit processor.  But
2160		 * if we're on a 32-bit processor and an i-cache incoherency
2161		 * or race makes us see a 64-bit instruction here the sdl/sdr
2162		 * would blow up, so for now we don't handle unaligned 64-bit
2163		 * instructions on 32-bit kernels.
2164		 */
2165		if (!access_ok(addr, 4))
2166			goto sigbus;
2167
2168		LoadWU(addr, value, res);
2169		if (res)
2170			goto fault;
2171		MIPS16e_compute_return_epc(regs, &oldinst);
2172		regs->regs[reg] = value;
2173		break;
2174#endif /* CONFIG_64BIT */
2175
2176		/* Cannot handle 64-bit instructions in 32-bit kernel */
2177		goto sigill;
2178
2179	case MIPS16e_ld_op:
2180loadDW:
2181#ifdef CONFIG_64BIT
2182		/*
2183		 * A 32-bit kernel might be running on a 64-bit processor.  But
2184		 * if we're on a 32-bit processor and an i-cache incoherency
2185		 * or race makes us see a 64-bit instruction here the sdl/sdr
2186		 * would blow up, so for now we don't handle unaligned 64-bit
2187		 * instructions on 32-bit kernels.
2188		 */
2189		if (!access_ok(addr, 8))
2190			goto sigbus;
2191
2192		LoadDW(addr, value, res);
2193		if (res)
2194			goto fault;
2195		MIPS16e_compute_return_epc(regs, &oldinst);
2196		regs->regs[reg] = value;
2197		break;
2198#endif /* CONFIG_64BIT */
2199
2200		/* Cannot handle 64-bit instructions in 32-bit kernel */
2201		goto sigill;
2202
2203	case MIPS16e_sh_op:
2204		if (!access_ok(addr, 2))
2205			goto sigbus;
2206
2207		MIPS16e_compute_return_epc(regs, &oldinst);
2208		value = regs->regs[reg];
2209		StoreHW(addr, value, res);
2210		if (res)
2211			goto fault;
2212		break;
2213
2214	case MIPS16e_sw_op:
2215	case MIPS16e_swsp_op:
2216	case MIPS16e_i8_op:	/* actually - MIPS16e_swrasp_func */
2217		if (!access_ok(addr, 4))
2218			goto sigbus;
2219
2220		MIPS16e_compute_return_epc(regs, &oldinst);
2221		value = regs->regs[reg];
2222		StoreW(addr, value, res);
2223		if (res)
2224			goto fault;
2225		break;
2226
2227	case MIPS16e_sd_op:
2228writeDW:
2229#ifdef CONFIG_64BIT
2230		/*
2231		 * A 32-bit kernel might be running on a 64-bit processor.  But
2232		 * if we're on a 32-bit processor and an i-cache incoherency
2233		 * or race makes us see a 64-bit instruction here the sdl/sdr
2234		 * would blow up, so for now we don't handle unaligned 64-bit
2235		 * instructions on 32-bit kernels.
2236		 */
2237		if (!access_ok(addr, 8))
2238			goto sigbus;
2239
2240		MIPS16e_compute_return_epc(regs, &oldinst);
2241		value = regs->regs[reg];
2242		StoreDW(addr, value, res);
2243		if (res)
2244			goto fault;
2245		break;
2246#endif /* CONFIG_64BIT */
2247
2248		/* Cannot handle 64-bit instructions in 32-bit kernel */
2249		goto sigill;
2250
2251	default:
2252		/*
2253		 * Pheeee...  We encountered an yet unknown instruction or
2254		 * cache coherence problem.  Die sucker, die ...
2255		 */
2256		goto sigill;
2257	}
2258
2259#ifdef CONFIG_DEBUG_FS
2260	unaligned_instructions++;
2261#endif
2262
2263	return;
2264
2265fault:
2266	/* roll back jump/branch */
2267	regs->cp0_epc = origpc;
2268	regs->regs[31] = orig31;
2269	/* Did we have an exception handler installed? */
2270	if (fixup_exception(regs))
2271		return;
2272
2273	die_if_kernel("Unhandled kernel unaligned access", regs);
2274	force_sig(SIGSEGV);
2275
2276	return;
2277
2278sigbus:
2279	die_if_kernel("Unhandled kernel unaligned access", regs);
2280	force_sig(SIGBUS);
2281
2282	return;
2283
2284sigill:
2285	die_if_kernel
2286	    ("Unhandled kernel unaligned access or invalid instruction", regs);
2287	force_sig(SIGILL);
2288}
2289
2290asmlinkage void do_ade(struct pt_regs *regs)
2291{
2292	enum ctx_state prev_state;
2293	unsigned int __user *pc;
2294	mm_segment_t seg;
2295
2296	prev_state = exception_enter();
2297	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
2298			1, regs, regs->cp0_badvaddr);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2299	/*
2300	 * Did we catch a fault trying to load an instruction?
2301	 */
2302	if (regs->cp0_badvaddr == regs->cp0_epc)
2303		goto sigbus;
2304
2305	if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
2306		goto sigbus;
2307	if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
2308		goto sigbus;
2309
2310	/*
2311	 * Do branch emulation only if we didn't forward the exception.
2312	 * This is all so but ugly ...
2313	 */
2314
2315	/*
2316	 * Are we running in microMIPS mode?
2317	 */
2318	if (get_isa16_mode(regs->cp0_epc)) {
2319		/*
2320		 * Did we catch a fault trying to load an instruction in
2321		 * 16-bit mode?
2322		 */
2323		if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
2324			goto sigbus;
2325		if (unaligned_action == UNALIGNED_ACTION_SHOW)
2326			show_registers(regs);
2327
2328		if (cpu_has_mmips) {
2329			seg = get_fs();
2330			if (!user_mode(regs))
2331				set_fs(KERNEL_DS);
2332			emulate_load_store_microMIPS(regs,
2333				(void __user *)regs->cp0_badvaddr);
2334			set_fs(seg);
2335
2336			return;
2337		}
2338
2339		if (cpu_has_mips16) {
2340			seg = get_fs();
2341			if (!user_mode(regs))
2342				set_fs(KERNEL_DS);
2343			emulate_load_store_MIPS16e(regs,
2344				(void __user *)regs->cp0_badvaddr);
2345			set_fs(seg);
2346
2347			return;
2348		}
2349
2350		goto sigbus;
2351	}
2352
2353	if (unaligned_action == UNALIGNED_ACTION_SHOW)
2354		show_registers(regs);
2355	pc = (unsigned int __user *)exception_epc(regs);
2356
2357	seg = get_fs();
2358	if (!user_mode(regs))
2359		set_fs(KERNEL_DS);
2360	emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
2361	set_fs(seg);
2362
2363	return;
2364
2365sigbus:
2366	die_if_kernel("Kernel unaligned instruction access", regs);
2367	force_sig(SIGBUS);
2368
2369	/*
2370	 * XXX On return from the signal handler we should advance the epc
2371	 */
2372	exception_exit(prev_state);
2373}
2374
2375#ifdef CONFIG_DEBUG_FS
2376static int __init debugfs_unaligned(void)
2377{
2378	debugfs_create_u32("unaligned_instructions", S_IRUGO, mips_debugfs_dir,
2379			   &unaligned_instructions);
2380	debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
2381			   mips_debugfs_dir, &unaligned_action);
2382	return 0;
2383}
2384arch_initcall(debugfs_unaligned);
2385#endif