1/*
   2 * Copyright 2018 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 */
  22
  23/* To compile this assembly code:
  24 *
  25 * Navi1x:
  26 *   cpp -DASIC_FAMILY=CHIP_NAVI10 cwsr_trap_handler_gfx10.asm -P -o nv1x.sp3
  27 *   sp3 nv1x.sp3 -hex nv1x.hex
  28 *
  29 * gfx10:
  30 *   cpp -DASIC_FAMILY=CHIP_SIENNA_CICHLID cwsr_trap_handler_gfx10.asm -P -o gfx10.sp3
  31 *   sp3 gfx10.sp3 -hex gfx10.hex
  32 *
  33 * gfx11:
  34 *   cpp -DASIC_FAMILY=CHIP_PLUM_BONITO cwsr_trap_handler_gfx10.asm -P -o gfx11.sp3
  35 *   sp3 gfx11.sp3 -hex gfx11.hex
  36 */
  37
  38#define CHIP_NAVI10 26
  39#define CHIP_SIENNA_CICHLID 30
  40#define CHIP_PLUM_BONITO 36
  41
  42#define NO_SQC_STORE (ASIC_FAMILY >= CHIP_SIENNA_CICHLID)
  43#define HAVE_XNACK (ASIC_FAMILY < CHIP_SIENNA_CICHLID)
  44#define HAVE_SENDMSG_RTN (ASIC_FAMILY >= CHIP_PLUM_BONITO)
  45#define HAVE_BUFFER_LDS_LOAD (ASIC_FAMILY < CHIP_PLUM_BONITO)
  46#define SW_SA_TRAP (ASIC_FAMILY >= CHIP_PLUM_BONITO)
  47#define SAVE_AFTER_XNACK_ERROR (HAVE_XNACK && !NO_SQC_STORE) // workaround for TCP store failure after XNACK error when ALLOW_REPLAY=0, for debugger
  48
  49var SINGLE_STEP_MISSED_WORKAROUND		= 1	//workaround for lost MODE.DEBUG_EN exception when SAVECTX raised
  50
 
 
  51var SQ_WAVE_STATUS_SPI_PRIO_MASK		= 0x00000006
  52var SQ_WAVE_STATUS_HALT_MASK			= 0x2000
  53var SQ_WAVE_STATUS_ECC_ERR_MASK			= 0x20000
  54var SQ_WAVE_STATUS_TRAP_EN_SHIFT		= 6
  55
  56var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT		= 12
  57var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE		= 9
  58var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE		= 8
 
 
 
  59var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT	= 24
  60var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE	= 4
  61var SQ_WAVE_IB_STS2_WAVE64_SHIFT		= 11
  62var SQ_WAVE_IB_STS2_WAVE64_SIZE			= 1
  63
  64#if ASIC_FAMILY < CHIP_PLUM_BONITO
  65var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT		= 8
  66#else
  67var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT		= 12
  68#endif
  69
  70var SQ_WAVE_TRAPSTS_SAVECTX_MASK		= 0x400
  71var SQ_WAVE_TRAPSTS_EXCP_MASK			= 0x1FF
  72var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT		= 10
  73var SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK		= 0x80
  74var SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT		= 7
  75var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK		= 0x100
  76var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT		= 8
  77var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK		= 0x3FF
  78var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT		= 0x0
  79var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE		= 10
  80var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK		= 0xFFFFF800
  81var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT		= 11
  82var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE		= 21
  83var SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK		= 0x800
  84var SQ_WAVE_TRAPSTS_EXCP_HI_MASK		= 0x7000
  85#if ASIC_FAMILY >= CHIP_PLUM_BONITO
  86var SQ_WAVE_TRAPSTS_WAVE_START_MASK		= 0x20000
  87var SQ_WAVE_TRAPSTS_WAVE_END_MASK		= 0x40000
  88var SQ_WAVE_TRAPSTS_TRAP_AFTER_INST_MASK	= 0x100000
  89#endif
  90var SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK		= 0x10000000
  91
  92var SQ_WAVE_MODE_EXCP_EN_SHIFT			= 12
  93var SQ_WAVE_MODE_EXCP_EN_ADDR_WATCH_SHIFT	= 19
  94
 
  95var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT		= 15
  96var SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT		= 25
 
  97var SQ_WAVE_IB_STS_REPLAY_W64H_MASK		= 0x02000000
 
 
  98var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK	= 0x003F8000
 
  99
 100var SQ_WAVE_MODE_DEBUG_EN_MASK			= 0x800
 101
 102#if ASIC_FAMILY < CHIP_PLUM_BONITO
 103var S_TRAPSTS_NON_MASKABLE_EXCP_MASK		= SQ_WAVE_TRAPSTS_MEM_VIOL_MASK|SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK
 104#else
 105var S_TRAPSTS_NON_MASKABLE_EXCP_MASK		= SQ_WAVE_TRAPSTS_MEM_VIOL_MASK		|\
 106						  SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK	|\
 107						  SQ_WAVE_TRAPSTS_WAVE_START_MASK	|\
 108						  SQ_WAVE_TRAPSTS_WAVE_END_MASK		|\
 109						  SQ_WAVE_TRAPSTS_TRAP_AFTER_INST_MASK
 110#endif
 111
 112// bits [31:24] unused by SPI debug data
 113var TTMP11_SAVE_REPLAY_W64H_SHIFT		= 31
 114var TTMP11_SAVE_REPLAY_W64H_MASK		= 0x80000000
 115var TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT		= 24
 116var TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK		= 0x7F000000
 117var TTMP11_DEBUG_TRAP_ENABLED_SHIFT		= 23
 118var TTMP11_DEBUG_TRAP_ENABLED_MASK		= 0x800000
 119
 120// SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14]
 121// when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
 122var S_SAVE_BUF_RSRC_WORD1_STRIDE		= 0x00040000
 123var S_SAVE_BUF_RSRC_WORD3_MISC			= 0x10807FAC
 124var S_SAVE_PC_HI_TRAP_ID_MASK			= 0x00FF0000
 125var S_SAVE_PC_HI_HT_MASK			= 0x01000000
 
 
 
 126var S_SAVE_SPI_INIT_FIRST_WAVE_MASK		= 0x04000000
 127var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT		= 26
 128
 129var S_SAVE_PC_HI_FIRST_WAVE_MASK		= 0x80000000
 130var S_SAVE_PC_HI_FIRST_WAVE_SHIFT		= 31
 
 
 
 
 131
 132var s_sgpr_save_num				= 108
 133
 134var s_save_spi_init_lo				= exec_lo
 135var s_save_spi_init_hi				= exec_hi
 136var s_save_pc_lo				= ttmp0
 137var s_save_pc_hi				= ttmp1
 138var s_save_exec_lo				= ttmp2
 139var s_save_exec_hi				= ttmp3
 140var s_save_status				= ttmp12
 141var s_save_trapsts				= ttmp15
 142var s_save_xnack_mask				= s_save_trapsts
 143var s_wave_size					= ttmp7
 144var s_save_buf_rsrc0				= ttmp8
 145var s_save_buf_rsrc1				= ttmp9
 146var s_save_buf_rsrc2				= ttmp10
 147var s_save_buf_rsrc3				= ttmp11
 148var s_save_mem_offset				= ttmp4
 149var s_save_alloc_size				= s_save_trapsts
 150var s_save_tmp					= ttmp14
 151var s_save_m0					= ttmp5
 152var s_save_ttmps_lo				= s_save_tmp
 153var s_save_ttmps_hi				= s_save_trapsts
 154
 155var S_RESTORE_BUF_RSRC_WORD1_STRIDE		= S_SAVE_BUF_RSRC_WORD1_STRIDE
 156var S_RESTORE_BUF_RSRC_WORD3_MISC		= S_SAVE_BUF_RSRC_WORD3_MISC
 157
 
 
 
 
 158var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK		= 0x04000000
 159var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT		= 26
 160var S_WAVE_SIZE					= 25
 161
 
 
 
 
 
 162var s_restore_spi_init_lo			= exec_lo
 163var s_restore_spi_init_hi			= exec_hi
 164var s_restore_mem_offset			= ttmp12
 165var s_restore_alloc_size			= ttmp3
 166var s_restore_tmp				= ttmp2
 167var s_restore_mem_offset_save			= s_restore_tmp
 168var s_restore_m0				= s_restore_alloc_size
 169var s_restore_mode				= ttmp7
 170var s_restore_flat_scratch			= s_restore_tmp
 171var s_restore_pc_lo				= ttmp0
 172var s_restore_pc_hi				= ttmp1
 173var s_restore_exec_lo				= ttmp4
 174var s_restore_exec_hi				= ttmp5
 175var s_restore_status				= ttmp14
 176var s_restore_trapsts				= ttmp15
 177var s_restore_xnack_mask			= ttmp13
 178var s_restore_buf_rsrc0				= ttmp8
 179var s_restore_buf_rsrc1				= ttmp9
 180var s_restore_buf_rsrc2				= ttmp10
 181var s_restore_buf_rsrc3				= ttmp11
 182var s_restore_size				= ttmp6
 183var s_restore_ttmps_lo				= s_restore_tmp
 184var s_restore_ttmps_hi				= s_restore_alloc_size
 185
 186shader main
 187	asic(DEFAULT)
 188	type(CS)
 189	wave_size(32)
 190
 191	s_branch	L_SKIP_RESTORE						//NOT restore. might be a regular trap or save
 192
 193L_JUMP_TO_RESTORE:
 194	s_branch	L_RESTORE
 195
 196L_SKIP_RESTORE:
 197	s_getreg_b32	s_save_status, hwreg(HW_REG_STATUS)			//save STATUS since we will change SCC
 
 198
 199	// Clear SPI_PRIO: do not save with elevated priority.
 200	// Clear ECC_ERR: prevents SQC store and triggers FATAL_HALT if setreg'd.
 201	s_andn2_b32	s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK|SQ_WAVE_STATUS_ECC_ERR_MASK
 202
 203	s_getreg_b32	s_save_trapsts, hwreg(HW_REG_TRAPSTS)
 204
 205#if SW_SA_TRAP
 206	// If ttmp1[30] is set then issue s_barrier to unblock dependent waves.
 207	s_bitcmp1_b32	s_save_pc_hi, 30
 208	s_cbranch_scc0	L_TRAP_NO_BARRIER
 209	s_barrier
 210
 211L_TRAP_NO_BARRIER:
 212	// If ttmp1[31] is set then trap may occur early.
 213	// Spin wait until SAVECTX exception is raised.
 214	s_bitcmp1_b32	s_save_pc_hi, 31
 215	s_cbranch_scc1  L_CHECK_SAVE
 216#endif
 217
 
 218	s_and_b32       ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK
 219	s_cbranch_scc0	L_NOT_HALTED
 220
 221L_HALTED:
 222	// Host trap may occur while wave is halted.
 223	s_and_b32	ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
 224	s_cbranch_scc1	L_FETCH_2ND_TRAP
 225
 226L_CHECK_SAVE:
 227	s_and_b32	ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK
 228	s_cbranch_scc1	L_SAVE
 229
 230	// Wave is halted but neither host trap nor SAVECTX is raised.
 231	// Caused by instruction fetch memory violation.
 232	// Spin wait until context saved to prevent interrupt storm.
 233	s_sleep		0x10
 234	s_getreg_b32	s_save_trapsts, hwreg(HW_REG_TRAPSTS)
 235	s_branch	L_CHECK_SAVE
 236
 237L_NOT_HALTED:
 238	// Let second-level handle non-SAVECTX exception or trap.
 239	// Any concurrent SAVECTX will be handled upon re-entry once halted.
 240
 241	// Check non-maskable exceptions. memory_violation, illegal_instruction
 242	// and xnack_error exceptions always cause the wave to enter the trap
 243	// handler.
 244	s_and_b32	ttmp2, s_save_trapsts, S_TRAPSTS_NON_MASKABLE_EXCP_MASK
 245	s_cbranch_scc1	L_FETCH_2ND_TRAP
 246
 247	// Check for maskable exceptions in trapsts.excp and trapsts.excp_hi.
 248	// Maskable exceptions only cause the wave to enter the trap handler if
 249	// their respective bit in mode.excp_en is set.
 250	s_and_b32	ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCP_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK
 251	s_cbranch_scc0	L_CHECK_TRAP_ID
 252
 253	s_and_b32	ttmp3, s_save_trapsts, SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK
 254	s_cbranch_scc0	L_NOT_ADDR_WATCH
 255	s_bitset1_b32	ttmp2, SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT // Check all addr_watch[123] exceptions against excp_en.addr_watch
 256
 257L_NOT_ADDR_WATCH:
 258	s_getreg_b32	ttmp3, hwreg(HW_REG_MODE)
 259	s_lshl_b32	ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT
 260	s_and_b32	ttmp2, ttmp2, ttmp3
 261	s_cbranch_scc1	L_FETCH_2ND_TRAP
 262
 263L_CHECK_TRAP_ID:
 264	// Check trap_id != 0
 265	s_and_b32	ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
 266	s_cbranch_scc1	L_FETCH_2ND_TRAP
 267
 268if SINGLE_STEP_MISSED_WORKAROUND
 269	// Prioritize single step exception over context save.
 270	// Second-level trap will halt wave and RFE, re-entering for SAVECTX.
 271	s_getreg_b32	ttmp2, hwreg(HW_REG_MODE)
 272	s_and_b32	ttmp2, ttmp2, SQ_WAVE_MODE_DEBUG_EN_MASK
 273	s_cbranch_scc1	L_FETCH_2ND_TRAP
 274end
 275
 276	s_and_b32	ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK
 
 
 277	s_cbranch_scc1	L_SAVE
 278
 
 
 
 
 
 
 279L_FETCH_2ND_TRAP:
 280#if HAVE_XNACK
 281	save_and_clear_ib_sts(ttmp14, ttmp15)
 
 
 
 
 
 
 
 
 
 
 
 
 
 282#endif
 283
 284	// Read second-level TBA/TMA from first-level TMA and jump if available.
 285	// ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data)
 286	// ttmp12 holds SQ_WAVE_STATUS
 287#if HAVE_SENDMSG_RTN
 288	s_sendmsg_rtn_b64       [ttmp14, ttmp15], sendmsg(MSG_RTN_GET_TMA)
 289	s_waitcnt       lgkmcnt(0)
 290#else
 291	s_getreg_b32	ttmp14, hwreg(HW_REG_SHADER_TMA_LO)
 292	s_getreg_b32	ttmp15, hwreg(HW_REG_SHADER_TMA_HI)
 293#endif
 294	s_lshl_b64	[ttmp14, ttmp15], [ttmp14, ttmp15], 0x8
 295
 296	s_bitcmp1_b32	ttmp15, 0xF
 297	s_cbranch_scc0	L_NO_SIGN_EXTEND_TMA
 298	s_or_b32	ttmp15, ttmp15, 0xFFFF0000
 299L_NO_SIGN_EXTEND_TMA:
 300
 301	s_load_dword    ttmp2, [ttmp14, ttmp15], 0x10 glc:1			// debug trap enabled flag
 302	s_waitcnt       lgkmcnt(0)
 303	s_lshl_b32      ttmp2, ttmp2, TTMP11_DEBUG_TRAP_ENABLED_SHIFT
 304	s_andn2_b32     ttmp11, ttmp11, TTMP11_DEBUG_TRAP_ENABLED_MASK
 305	s_or_b32        ttmp11, ttmp11, ttmp2
 306
 307	s_load_dwordx2	[ttmp2, ttmp3], [ttmp14, ttmp15], 0x0 glc:1		// second-level TBA
 308	s_waitcnt	lgkmcnt(0)
 309	s_load_dwordx2	[ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 glc:1		// second-level TMA
 310	s_waitcnt	lgkmcnt(0)
 311
 312	s_and_b64	[ttmp2, ttmp3], [ttmp2, ttmp3], [ttmp2, ttmp3]
 313	s_cbranch_scc0	L_NO_NEXT_TRAP						// second-level trap handler not been set
 314	s_setpc_b64	[ttmp2, ttmp3]						// jump to second-level trap handler
 315
 316L_NO_NEXT_TRAP:
 317	// If not caused by trap then halt wave to prevent re-entry.
 318	s_and_b32	ttmp2, s_save_pc_hi, (S_SAVE_PC_HI_TRAP_ID_MASK|S_SAVE_PC_HI_HT_MASK)
 319	s_cbranch_scc1	L_TRAP_CASE
 320	s_or_b32	s_save_status, s_save_status, SQ_WAVE_STATUS_HALT_MASK
 321
 322	// If the PC points to S_ENDPGM then context save will fail if STATUS.HALT is set.
 323	// Rewind the PC to prevent this from occurring.
 324	s_sub_u32	ttmp0, ttmp0, 0x8
 325	s_subb_u32	ttmp1, ttmp1, 0x0
 326
 327	s_branch	L_EXIT_TRAP
 328
 329L_TRAP_CASE:
 330	// Host trap will not cause trap re-entry.
 331	s_and_b32	ttmp2, s_save_pc_hi, S_SAVE_PC_HI_HT_MASK
 332	s_cbranch_scc1	L_EXIT_TRAP
 333
 334	// Advance past trap instruction to prevent re-entry.
 335	s_add_u32	ttmp0, ttmp0, 0x4
 336	s_addc_u32	ttmp1, ttmp1, 0x0
 337
 338L_EXIT_TRAP:
 339	s_and_b32	ttmp1, ttmp1, 0xFFFF
 340
 341#if HAVE_XNACK
 342	restore_ib_sts(ttmp14, ttmp15)
 
 
 
 
 
 
 343#endif
 344
 345	// Restore SQ_WAVE_STATUS.
 346	s_and_b64	exec, exec, exec					// Restore STATUS.EXECZ, not writable by s_setreg_b32
 347	s_and_b64	vcc, vcc, vcc						// Restore STATUS.VCCZ, not writable by s_setreg_b32
 348	s_setreg_b32	hwreg(HW_REG_STATUS), s_save_status
 349
 350	s_rfe_b64	[ttmp0, ttmp1]
 351
 352L_SAVE:
 353	s_and_b32	s_save_pc_hi, s_save_pc_hi, 0x0000ffff			//pc[47:32]
 354	s_mov_b32	s_save_tmp, 0
 355	s_setreg_b32	hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp	//clear saveCtx bit
 356
 357#if HAVE_XNACK
 358	save_and_clear_ib_sts(s_save_tmp, s_save_trapsts)
 
 
 
 
 
 
 
 
 
 
 
 
 359#endif
 360
 361	/* inform SPI the readiness and wait for SPI's go signal */
 362	s_mov_b32	s_save_exec_lo, exec_lo					//save EXEC and use EXEC for the go signal from SPI
 363	s_mov_b32	s_save_exec_hi, exec_hi
 364	s_mov_b64	exec, 0x0						//clear EXEC to get ready to receive
 365
 366#if HAVE_SENDMSG_RTN
 367	s_sendmsg_rtn_b64       [exec_lo, exec_hi], sendmsg(MSG_RTN_SAVE_WAVE)
 368#else
 369	s_sendmsg	sendmsg(MSG_SAVEWAVE)					//send SPI a message and wait for SPI's write to EXEC
 370#endif
 371
 372#if ASIC_FAMILY < CHIP_SIENNA_CICHLID
 373L_SLEEP:
 374	// sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause
 375	// SQ hang, since the 7,8th wave could not get arbit to exec inst, while
 376	// other waves are stuck into the sleep-loop and waiting for wrexec!=0
 377	s_sleep		0x2
 378	s_cbranch_execz	L_SLEEP
 379#else
 380	s_waitcnt	lgkmcnt(0)
 381#endif
 382
 383	// Save first_wave flag so we can clear high bits of save address.
 384	s_and_b32	s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK
 385	s_lshl_b32	s_save_tmp, s_save_tmp, (S_SAVE_PC_HI_FIRST_WAVE_SHIFT - S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT)
 386	s_or_b32	s_save_pc_hi, s_save_pc_hi, s_save_tmp
 387
 388#if NO_SQC_STORE
 389#if ASIC_FAMILY <= CHIP_SIENNA_CICHLID
 390	// gfx10: If there was a VALU exception, the exception state must be
 391	// cleared before executing the VALU instructions below.
 392	v_clrexcp
 393#endif
 394
 395	// Trap temporaries must be saved via VGPR but all VGPRs are in use.
 396	// There is no ttmp space to hold the resource constant for VGPR save.
 397	// Save v0 by itself since it requires only two SGPRs.
 398	s_mov_b32	s_save_ttmps_lo, exec_lo
 399	s_and_b32	s_save_ttmps_hi, exec_hi, 0xFFFF
 400	s_mov_b32	exec_lo, 0xFFFFFFFF
 401	s_mov_b32	exec_hi, 0xFFFFFFFF
 402	global_store_dword_addtid	v0, [s_save_ttmps_lo, s_save_ttmps_hi] slc:1 glc:1
 403	v_mov_b32	v0, 0x0
 404	s_mov_b32	exec_lo, s_save_ttmps_lo
 405	s_mov_b32	exec_hi, s_save_ttmps_hi
 406#endif
 407
 408	// Save trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
 409	// ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40
 410	get_wave_size(s_save_ttmps_hi)
 411	get_vgpr_size_bytes(s_save_ttmps_lo, s_save_ttmps_hi)
 412	get_svgpr_size_bytes(s_save_ttmps_hi)
 413	s_add_u32	s_save_ttmps_lo, s_save_ttmps_lo, s_save_ttmps_hi
 414	s_and_b32	s_save_ttmps_hi, s_save_spi_init_hi, 0xFFFF
 415	s_add_u32	s_save_ttmps_lo, s_save_ttmps_lo, get_sgpr_size_bytes()
 416	s_add_u32	s_save_ttmps_lo, s_save_ttmps_lo, s_save_spi_init_lo
 417	s_addc_u32	s_save_ttmps_hi, s_save_ttmps_hi, 0x0
 418
 419#if NO_SQC_STORE
 420	v_writelane_b32	v0, ttmp4, 0x4
 421	v_writelane_b32	v0, ttmp5, 0x5
 422	v_writelane_b32	v0, ttmp6, 0x6
 423	v_writelane_b32	v0, ttmp7, 0x7
 424	v_writelane_b32	v0, ttmp8, 0x8
 425	v_writelane_b32	v0, ttmp9, 0x9
 426	v_writelane_b32	v0, ttmp10, 0xA
 427	v_writelane_b32	v0, ttmp11, 0xB
 428	v_writelane_b32	v0, ttmp13, 0xD
 429	v_writelane_b32	v0, exec_lo, 0xE
 430	v_writelane_b32	v0, exec_hi, 0xF
 431
 432	s_mov_b32	exec_lo, 0x3FFF
 433	s_mov_b32	exec_hi, 0x0
 434	global_store_dword_addtid	v0, [s_save_ttmps_lo, s_save_ttmps_hi] inst_offset:0x40 slc:1 glc:1
 435	v_readlane_b32	ttmp14, v0, 0xE
 436	v_readlane_b32	ttmp15, v0, 0xF
 437	s_mov_b32	exec_lo, ttmp14
 438	s_mov_b32	exec_hi, ttmp15
 439#else
 440	s_store_dwordx4	[ttmp4, ttmp5, ttmp6, ttmp7], [s_save_ttmps_lo, s_save_ttmps_hi], 0x50 glc:1
 441	s_store_dwordx4	[ttmp8, ttmp9, ttmp10, ttmp11], [s_save_ttmps_lo, s_save_ttmps_hi], 0x60 glc:1
 442	s_store_dword   ttmp13, [s_save_ttmps_lo, s_save_ttmps_hi], 0x74 glc:1
 443#endif
 444
 445	/* setup Resource Contants */
 446	s_mov_b32	s_save_buf_rsrc0, s_save_spi_init_lo			//base_addr_lo
 447	s_and_b32	s_save_buf_rsrc1, s_save_spi_init_hi, 0x0000FFFF	//base_addr_hi
 448	s_or_b32	s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE
 449	s_mov_b32	s_save_buf_rsrc2, 0					//NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
 450	s_mov_b32	s_save_buf_rsrc3, S_SAVE_BUF_RSRC_WORD3_MISC
 
 
 
 
 
 
 451
 452	s_mov_b32	s_save_m0, m0
 453
 454	/* global mem offset */
 455	s_mov_b32	s_save_mem_offset, 0x0
 456	get_wave_size(s_wave_size)
 457
 458#if HAVE_XNACK
 459	// Save and clear vector XNACK state late to free up SGPRs.
 460	s_getreg_b32	s_save_xnack_mask, hwreg(HW_REG_SHADER_XNACK_MASK)
 461	s_setreg_imm32_b32	hwreg(HW_REG_SHADER_XNACK_MASK), 0x0
 462#endif
 463
 464	/* save first 4 VGPRs, needed for SGPR save */
 465	s_mov_b32	exec_lo, 0xFFFFFFFF					//need every thread from now on
 466	s_lshr_b32	m0, s_wave_size, S_WAVE_SIZE
 467	s_and_b32	m0, m0, 1
 468	s_cmp_eq_u32	m0, 1
 469	s_cbranch_scc1	L_ENABLE_SAVE_4VGPR_EXEC_HI
 470	s_mov_b32	exec_hi, 0x00000000
 471	s_branch	L_SAVE_4VGPR_WAVE32
 472L_ENABLE_SAVE_4VGPR_EXEC_HI:
 473	s_mov_b32	exec_hi, 0xFFFFFFFF
 474	s_branch	L_SAVE_4VGPR_WAVE64
 475L_SAVE_4VGPR_WAVE32:
 476	s_mov_b32	s_save_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
 477
 478	// VGPR Allocated in 4-GPR granularity
 479
 480#if SAVE_AFTER_XNACK_ERROR
 481	check_if_tcp_store_ok()
 482	s_cbranch_scc1 L_SAVE_FIRST_VGPRS32_WITH_TCP
 483
 484	write_vgprs_to_mem_with_sqc_w32(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)
 485	s_branch L_SAVE_HWREG
 486
 487L_SAVE_FIRST_VGPRS32_WITH_TCP:
 488#endif
 489
 490#if !NO_SQC_STORE
 491	buffer_store_dword	v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 492#endif
 493	buffer_store_dword	v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128
 494	buffer_store_dword	v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*2
 495	buffer_store_dword	v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*3
 496	s_branch	L_SAVE_HWREG
 497
 498L_SAVE_4VGPR_WAVE64:
 499	s_mov_b32	s_save_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
 500
 501	// VGPR Allocated in 4-GPR granularity
 502
 503#if  SAVE_AFTER_XNACK_ERROR
 504	check_if_tcp_store_ok()
 505	s_cbranch_scc1 L_SAVE_FIRST_VGPRS64_WITH_TCP
 506
 507	write_vgprs_to_mem_with_sqc_w64(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)
 508	s_branch L_SAVE_HWREG
 509
 510L_SAVE_FIRST_VGPRS64_WITH_TCP:
 511#endif
 512
 513#if !NO_SQC_STORE
 514	buffer_store_dword	v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 515#endif
 516	buffer_store_dword	v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
 517	buffer_store_dword	v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
 518	buffer_store_dword	v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
 519
 520	/* save HW registers */
 521
 522L_SAVE_HWREG:
 523	// HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)
 524	get_vgpr_size_bytes(s_save_mem_offset, s_wave_size)
 525	get_svgpr_size_bytes(s_save_tmp)
 526	s_add_u32	s_save_mem_offset, s_save_mem_offset, s_save_tmp
 527	s_add_u32	s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes()
 528
 529	s_mov_b32	s_save_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
 530
 531#if NO_SQC_STORE
 532	v_mov_b32	v0, 0x0							//Offset[31:0] from buffer resource
 533	v_mov_b32	v1, 0x0							//Offset[63:32] from buffer resource
 534	v_mov_b32	v2, 0x0							//Set of SGPRs for TCP store
 535	s_mov_b32	m0, 0x0							//Next lane of v2 to write to
 536#endif
 537
 538	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
 539	write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset)
 540	s_andn2_b32	s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
 541	write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)
 542	write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset)
 543	write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
 544	write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset)
 545
 546	s_getreg_b32	s_save_tmp, hwreg(HW_REG_TRAPSTS)
 547	write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)
 548
 549	// Not used on Sienna_Cichlid but keep layout same for debugger.
 550	write_hwreg_to_mem(s_save_xnack_mask, s_save_buf_rsrc0, s_save_mem_offset)
 551
 552	s_getreg_b32	s_save_m0, hwreg(HW_REG_MODE)
 553	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
 554
 555	s_getreg_b32	s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO)
 556	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
 557
 558	s_getreg_b32	s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI)
 559	write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
 560
 561#if NO_SQC_STORE
 562	// Write HWREGs with 16 VGPR lanes. TTMPs occupy space after this.
 563	s_mov_b32       exec_lo, 0xFFFF
 564	s_mov_b32	exec_hi, 0x0
 565	buffer_store_dword	v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 566
 567	// Write SGPRs with 32 VGPR lanes. This works in wave32 and wave64 mode.
 568	s_mov_b32       exec_lo, 0xFFFFFFFF
 569#endif
 570
 571	/* save SGPRs */
 572	// Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...
 573
 574	// SGPR SR memory offset : size(VGPR)+size(SVGPR)
 575	get_vgpr_size_bytes(s_save_mem_offset, s_wave_size)
 576	get_svgpr_size_bytes(s_save_tmp)
 577	s_add_u32	s_save_mem_offset, s_save_mem_offset, s_save_tmp
 578	s_mov_b32	s_save_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
 579
 580#if NO_SQC_STORE
 581	s_mov_b32	ttmp13, 0x0						//next VGPR lane to copy SGPR into
 582#else
 583	// backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0
 584	s_mov_b32	s_save_xnack_mask, s_save_buf_rsrc0
 585	s_add_u32	s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset
 586	s_addc_u32	s_save_buf_rsrc1, s_save_buf_rsrc1, 0
 587#endif
 588
 589	s_mov_b32	m0, 0x0							//SGPR initial index value =0
 590	s_nop		0x0							//Manually inserted wait states
 591L_SAVE_SGPR_LOOP:
 592	// SGPR is allocated in 16 SGPR granularity
 593	s_movrels_b64	s0, s0							//s0 = s[0+m0], s1 = s[1+m0]
 594	s_movrels_b64	s2, s2							//s2 = s[2+m0], s3 = s[3+m0]
 595	s_movrels_b64	s4, s4							//s4 = s[4+m0], s5 = s[5+m0]
 596	s_movrels_b64	s6, s6							//s6 = s[6+m0], s7 = s[7+m0]
 597	s_movrels_b64	s8, s8							//s8 = s[8+m0], s9 = s[9+m0]
 598	s_movrels_b64	s10, s10						//s10 = s[10+m0], s11 = s[11+m0]
 599	s_movrels_b64	s12, s12						//s12 = s[12+m0], s13 = s[13+m0]
 600	s_movrels_b64	s14, s14						//s14 = s[14+m0], s15 = s[15+m0]
 601
 602	write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset)
 603
 604#if NO_SQC_STORE
 605	s_cmp_eq_u32	ttmp13, 0x20						//have 32 VGPR lanes filled?
 606	s_cbranch_scc0	L_SAVE_SGPR_SKIP_TCP_STORE
 607
 608	buffer_store_dword	v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 609	s_add_u32	s_save_mem_offset, s_save_mem_offset, 0x80
 610	s_mov_b32	ttmp13, 0x0
 611	v_mov_b32	v2, 0x0
 612L_SAVE_SGPR_SKIP_TCP_STORE:
 613#endif
 614
 615	s_add_u32	m0, m0, 16						//next sgpr index
 616	s_cmp_lt_u32	m0, 96							//scc = (m0 < first 96 SGPR) ? 1 : 0
 617	s_cbranch_scc1	L_SAVE_SGPR_LOOP					//first 96 SGPR save is complete?
 618
 619	//save the rest 12 SGPR
 620	s_movrels_b64	s0, s0							//s0 = s[0+m0], s1 = s[1+m0]
 621	s_movrels_b64	s2, s2							//s2 = s[2+m0], s3 = s[3+m0]
 622	s_movrels_b64	s4, s4							//s4 = s[4+m0], s5 = s[5+m0]
 623	s_movrels_b64	s6, s6							//s6 = s[6+m0], s7 = s[7+m0]
 624	s_movrels_b64	s8, s8							//s8 = s[8+m0], s9 = s[9+m0]
 625	s_movrels_b64	s10, s10						//s10 = s[10+m0], s11 = s[11+m0]
 626	write_12sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset)
 627
 628#if NO_SQC_STORE
 629	buffer_store_dword	v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 630#else
 631	// restore s_save_buf_rsrc0,1
 632	s_mov_b32	s_save_buf_rsrc0, s_save_xnack_mask
 633#endif
 634
 635	/* save LDS */
 636
 637L_SAVE_LDS:
 638	// Change EXEC to all threads...
 639	s_mov_b32	exec_lo, 0xFFFFFFFF					//need every thread from now on
 640	s_lshr_b32	m0, s_wave_size, S_WAVE_SIZE
 641	s_and_b32	m0, m0, 1
 642	s_cmp_eq_u32	m0, 1
 643	s_cbranch_scc1	L_ENABLE_SAVE_LDS_EXEC_HI
 644	s_mov_b32	exec_hi, 0x00000000
 645	s_branch	L_SAVE_LDS_NORMAL
 646L_ENABLE_SAVE_LDS_EXEC_HI:
 647	s_mov_b32	exec_hi, 0xFFFFFFFF
 648L_SAVE_LDS_NORMAL:
 649	s_getreg_b32	s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)
 650	s_and_b32	s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF	//lds_size is zero?
 651	s_cbranch_scc0	L_SAVE_LDS_DONE						//no lds used? jump to L_SAVE_DONE
 652
 653	s_barrier								//LDS is used? wait for other waves in the same TG
 654	s_and_b32	s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
 655	s_cbranch_scc0	L_SAVE_LDS_DONE
 656
 657	// first wave do LDS save;
 658
 659	s_lshl_b32	s_save_alloc_size, s_save_alloc_size, 6			//LDS size in dwords = lds_size * 64dw
 660	s_lshl_b32	s_save_alloc_size, s_save_alloc_size, 2			//LDS size in bytes
 661	s_mov_b32	s_save_buf_rsrc2, s_save_alloc_size			//NUM_RECORDS in bytes
 662
 663	// LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG)
 664	//
 665	get_vgpr_size_bytes(s_save_mem_offset, s_wave_size)
 666	get_svgpr_size_bytes(s_save_tmp)
 667	s_add_u32	s_save_mem_offset, s_save_mem_offset, s_save_tmp
 668	s_add_u32	s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes()
 669	s_add_u32	s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()
 670
 671	s_mov_b32	s_save_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
 672
 673	//load 0~63*4(byte address) to vgpr v0
 674	v_mbcnt_lo_u32_b32	v0, -1, 0
 675	v_mbcnt_hi_u32_b32	v0, -1, v0
 676	v_mul_u32_u24	v0, 4, v0
 677
 678	s_lshr_b32	m0, s_wave_size, S_WAVE_SIZE
 679	s_and_b32	m0, m0, 1
 680	s_cmp_eq_u32	m0, 1
 681	s_mov_b32	m0, 0x0
 682	s_cbranch_scc1	L_SAVE_LDS_W64
 683
 684L_SAVE_LDS_W32:
 685#if SAVE_AFTER_XNACK_ERROR
 686	check_if_tcp_store_ok()
 687	s_cbranch_scc1 L_SAVE_LDS_WITH_TCP_W32
 688
 689L_SAVE_LDS_LOOP_SQC_W32:
 690	ds_read_b32	v1, v0
 691	s_waitcnt	0
 692
 693	write_vgprs_to_mem_with_sqc_w32(v1, 1, s_save_buf_rsrc0, s_save_mem_offset)
 694
 695	s_add_u32	m0, m0, 128						//every buffer_store_lds does 128 bytes
 696	v_add_nc_u32	v0, v0, 128						//mem offset increased by 128 bytes
 697	s_cmp_lt_u32	m0, s_save_alloc_size					//scc=(m0 < s_save_alloc_size) ? 1 : 0
 698	s_cbranch_scc1	L_SAVE_LDS_LOOP_SQC_W32					//LDS save is complete?
 699
 700	s_branch	L_SAVE_LDS_DONE
 701
 702L_SAVE_LDS_WITH_TCP_W32:
 703#endif
 704
 705	s_mov_b32	s3, 128
 706	s_nop		0
 707	s_nop		0
 708	s_nop		0
 709L_SAVE_LDS_LOOP_W32:
 710	ds_read_b32	v1, v0
 711	s_waitcnt	0
 712	buffer_store_dword	v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 713
 714	s_add_u32	m0, m0, s3						//every buffer_store_lds does 128 bytes
 715	s_add_u32	s_save_mem_offset, s_save_mem_offset, s3
 716	v_add_nc_u32	v0, v0, 128						//mem offset increased by 128 bytes
 717	s_cmp_lt_u32	m0, s_save_alloc_size					//scc=(m0 < s_save_alloc_size) ? 1 : 0
 718	s_cbranch_scc1	L_SAVE_LDS_LOOP_W32					//LDS save is complete?
 719
 720	s_branch	L_SAVE_LDS_DONE
 721
 722L_SAVE_LDS_W64:
 723#if  SAVE_AFTER_XNACK_ERROR
 724	check_if_tcp_store_ok()
 725	s_cbranch_scc1 L_SAVE_LDS_WITH_TCP_W64
 726
 727L_SAVE_LDS_LOOP_SQC_W64:
 728	ds_read_b32	v1, v0
 729	s_waitcnt	0
 730
 731	write_vgprs_to_mem_with_sqc_w64(v1, 1, s_save_buf_rsrc0, s_save_mem_offset)
 732
 733	s_add_u32	m0, m0, 256						//every buffer_store_lds does 256 bytes
 734	v_add_nc_u32	v0, v0, 256						//mem offset increased by 256 bytes
 735	s_cmp_lt_u32	m0, s_save_alloc_size					//scc=(m0 < s_save_alloc_size) ? 1 : 0
 736	s_cbranch_scc1	L_SAVE_LDS_LOOP_SQC_W64					//LDS save is complete?
 737
 738	s_branch	L_SAVE_LDS_DONE
 739
 740L_SAVE_LDS_WITH_TCP_W64:
 741#endif
 742
 743	s_mov_b32	s3, 256
 744	s_nop		0
 745	s_nop		0
 746	s_nop		0
 747L_SAVE_LDS_LOOP_W64:
 748	ds_read_b32	v1, v0
 749	s_waitcnt	0
 750	buffer_store_dword	v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 751
 752	s_add_u32	m0, m0, s3						//every buffer_store_lds does 256 bytes
 753	s_add_u32	s_save_mem_offset, s_save_mem_offset, s3
 754	v_add_nc_u32	v0, v0, 256						//mem offset increased by 256 bytes
 755	s_cmp_lt_u32	m0, s_save_alloc_size					//scc=(m0 < s_save_alloc_size) ? 1 : 0
 756	s_cbranch_scc1	L_SAVE_LDS_LOOP_W64					//LDS save is complete?
 757
 758L_SAVE_LDS_DONE:
 759	/* save VGPRs  - set the Rest VGPRs */
 760L_SAVE_VGPR:
 761	// VGPR SR memory offset: 0
 762	s_mov_b32	exec_lo, 0xFFFFFFFF					//need every thread from now on
 763	s_lshr_b32	m0, s_wave_size, S_WAVE_SIZE
 764	s_and_b32	m0, m0, 1
 765	s_cmp_eq_u32	m0, 1
 766	s_cbranch_scc1	L_ENABLE_SAVE_VGPR_EXEC_HI
 767	s_mov_b32	s_save_mem_offset, (0+128*4)				// for the rest VGPRs
 768	s_mov_b32	exec_hi, 0x00000000
 769	s_branch	L_SAVE_VGPR_NORMAL
 770L_ENABLE_SAVE_VGPR_EXEC_HI:
 771	s_mov_b32	s_save_mem_offset, (0+256*4)				// for the rest VGPRs
 772	s_mov_b32	exec_hi, 0xFFFFFFFF
 773L_SAVE_VGPR_NORMAL:
 774	s_getreg_b32	s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)
 775	s_add_u32	s_save_alloc_size, s_save_alloc_size, 1
 776	s_lshl_b32	s_save_alloc_size, s_save_alloc_size, 2			//Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)
 777	//determine it is wave32 or wave64
 778	s_lshr_b32	m0, s_wave_size, S_WAVE_SIZE
 779	s_and_b32	m0, m0, 1
 780	s_cmp_eq_u32	m0, 1
 781	s_cbranch_scc1	L_SAVE_VGPR_WAVE64
 782
 783	s_mov_b32	s_save_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
 784
 785	// VGPR Allocated in 4-GPR granularity
 786
 787	// VGPR store using dw burst
 788	s_mov_b32	m0, 0x4							//VGPR initial index value =4
 789	s_cmp_lt_u32	m0, s_save_alloc_size
 790	s_cbranch_scc0	L_SAVE_VGPR_END
 791
 792#if  SAVE_AFTER_XNACK_ERROR
 793	check_if_tcp_store_ok()
 794	s_cbranch_scc1 L_SAVE_VGPR_W32_LOOP
 795
 796L_SAVE_VGPR_LOOP_SQC_W32:
 797	v_movrels_b32	v0, v0							//v0 = v[0+m0]
 798	v_movrels_b32	v1, v1							//v1 = v[1+m0]
 799	v_movrels_b32	v2, v2							//v2 = v[2+m0]
 800	v_movrels_b32	v3, v3							//v3 = v[3+m0]
 801
 802	write_vgprs_to_mem_with_sqc_w32(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)
 803
 804	s_add_u32 m0, m0, 4
 805	s_cmp_lt_u32 m0, s_save_alloc_size
 806	s_cbranch_scc1 L_SAVE_VGPR_LOOP_SQC_W32
 807
 808	s_branch L_SAVE_VGPR_END
 809#endif
 810
 811L_SAVE_VGPR_W32_LOOP:
 812	v_movrels_b32	v0, v0							//v0 = v[0+m0]
 813	v_movrels_b32	v1, v1							//v1 = v[1+m0]
 814	v_movrels_b32	v2, v2							//v2 = v[2+m0]
 815	v_movrels_b32	v3, v3							//v3 = v[3+m0]
 816
 817	buffer_store_dword	v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 818	buffer_store_dword	v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128
 819	buffer_store_dword	v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*2
 820	buffer_store_dword	v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*3
 821
 822	s_add_u32	m0, m0, 4						//next vgpr index
 823	s_add_u32	s_save_mem_offset, s_save_mem_offset, 128*4		//every buffer_store_dword does 128 bytes
 824	s_cmp_lt_u32	m0, s_save_alloc_size					//scc = (m0 < s_save_alloc_size) ? 1 : 0
 825	s_cbranch_scc1	L_SAVE_VGPR_W32_LOOP					//VGPR save is complete?
 826
 827	s_branch	L_SAVE_VGPR_END
 828
 829L_SAVE_VGPR_WAVE64:
 830	s_mov_b32	s_save_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
 831
 832	// VGPR store using dw burst
 833	s_mov_b32	m0, 0x4							//VGPR initial index value =4
 834	s_cmp_lt_u32	m0, s_save_alloc_size
 835	s_cbranch_scc0	L_SAVE_SHARED_VGPR
 836
 837#if  SAVE_AFTER_XNACK_ERROR
 838	check_if_tcp_store_ok()
 839	s_cbranch_scc1 L_SAVE_VGPR_W64_LOOP
 840
 841L_SAVE_VGPR_LOOP_SQC_W64:
 842	v_movrels_b32	v0, v0							//v0 = v[0+m0]
 843	v_movrels_b32	v1, v1							//v1 = v[1+m0]
 844	v_movrels_b32	v2, v2							//v2 = v[2+m0]
 845	v_movrels_b32	v3, v3							//v3 = v[3+m0]
 846
 847	write_vgprs_to_mem_with_sqc_w64(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)
 848
 849	s_add_u32 m0, m0, 4
 850	s_cmp_lt_u32 m0, s_save_alloc_size
 851	s_cbranch_scc1 L_SAVE_VGPR_LOOP_SQC_W64
 852
 853	s_branch L_SAVE_VGPR_END
 854#endif
 855
 856L_SAVE_VGPR_W64_LOOP:
 857	v_movrels_b32	v0, v0							//v0 = v[0+m0]
 858	v_movrels_b32	v1, v1							//v1 = v[1+m0]
 859	v_movrels_b32	v2, v2							//v2 = v[2+m0]
 860	v_movrels_b32	v3, v3							//v3 = v[3+m0]
 861
 862	buffer_store_dword	v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 863	buffer_store_dword	v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
 864	buffer_store_dword	v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
 865	buffer_store_dword	v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
 866
 867	s_add_u32	m0, m0, 4						//next vgpr index
 868	s_add_u32	s_save_mem_offset, s_save_mem_offset, 256*4		//every buffer_store_dword does 256 bytes
 869	s_cmp_lt_u32	m0, s_save_alloc_size					//scc = (m0 < s_save_alloc_size) ? 1 : 0
 870	s_cbranch_scc1	L_SAVE_VGPR_W64_LOOP					//VGPR save is complete?
 871
 872L_SAVE_SHARED_VGPR:
 873	//Below part will be the save shared vgpr part (new for gfx10)
 874	s_getreg_b32	s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE)
 875	s_and_b32	s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF	//shared_vgpr_size is zero?
 876	s_cbranch_scc0	L_SAVE_VGPR_END						//no shared_vgpr used? jump to L_SAVE_LDS
 877	s_lshl_b32	s_save_alloc_size, s_save_alloc_size, 3			//Number of SHARED_VGPRs = shared_vgpr_size * 8    (non-zero value)
 878	//m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count.
 879	//save shared_vgpr will start from the index of m0
 880	s_add_u32	s_save_alloc_size, s_save_alloc_size, m0
 881	s_mov_b32	exec_lo, 0xFFFFFFFF
 882	s_mov_b32	exec_hi, 0x00000000
 883
 884#if  SAVE_AFTER_XNACK_ERROR
 885	check_if_tcp_store_ok()
 886	s_cbranch_scc1 L_SAVE_SHARED_VGPR_WAVE64_LOOP
 887
 888L_SAVE_SHARED_VGPR_WAVE64_LOOP_SQC:
 889	v_movrels_b32	v0, v0
 890
 891	write_vgprs_to_mem_with_sqc_w64(v0, 1, s_save_buf_rsrc0, s_save_mem_offset)
 892
 893	s_add_u32 m0, m0, 1
 894	s_cmp_lt_u32 m0, s_save_alloc_size
 895	s_cbranch_scc1 L_SAVE_SHARED_VGPR_WAVE64_LOOP_SQC
 896
 897	s_branch L_SAVE_VGPR_END
 898#endif
 899
 900L_SAVE_SHARED_VGPR_WAVE64_LOOP:
 901	v_movrels_b32	v0, v0							//v0 = v[0+m0]
 902	buffer_store_dword	v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 903	s_add_u32	m0, m0, 1						//next vgpr index
 904	s_add_u32	s_save_mem_offset, s_save_mem_offset, 128
 905	s_cmp_lt_u32	m0, s_save_alloc_size					//scc = (m0 < s_save_alloc_size) ? 1 : 0
 906	s_cbranch_scc1	L_SAVE_SHARED_VGPR_WAVE64_LOOP				//SHARED_VGPR save is complete?
 907
 908L_SAVE_VGPR_END:
 909	s_branch	L_END_PGM
 910
 911L_RESTORE:
 912	/* Setup Resource Contants */
 913	s_mov_b32	s_restore_buf_rsrc0, s_restore_spi_init_lo		//base_addr_lo
 914	s_and_b32	s_restore_buf_rsrc1, s_restore_spi_init_hi, 0x0000FFFF	//base_addr_hi
 915	s_or_b32	s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE
 916	s_mov_b32	s_restore_buf_rsrc2, 0					//NUM_RECORDS initial value = 0 (in bytes)
 917	s_mov_b32	s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC
 918
 
 
 
 
 
 919	//determine it is wave32 or wave64
 920	get_wave_size(s_restore_size)
 921
 922	s_and_b32	s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
 923	s_cbranch_scc0	L_RESTORE_VGPR
 924
 925	/* restore LDS */
 926L_RESTORE_LDS:
 927	s_mov_b32	exec_lo, 0xFFFFFFFF					//need every thread from now on
 928	s_lshr_b32	m0, s_restore_size, S_WAVE_SIZE
 929	s_and_b32	m0, m0, 1
 930	s_cmp_eq_u32	m0, 1
 931	s_cbranch_scc1	L_ENABLE_RESTORE_LDS_EXEC_HI
 932	s_mov_b32	exec_hi, 0x00000000
 933	s_branch	L_RESTORE_LDS_NORMAL
 934L_ENABLE_RESTORE_LDS_EXEC_HI:
 935	s_mov_b32	exec_hi, 0xFFFFFFFF
 936L_RESTORE_LDS_NORMAL:
 937	s_getreg_b32	s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)
 938	s_and_b32	s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF	//lds_size is zero?
 939	s_cbranch_scc0	L_RESTORE_VGPR						//no lds used? jump to L_RESTORE_VGPR
 940	s_lshl_b32	s_restore_alloc_size, s_restore_alloc_size, 6		//LDS size in dwords = lds_size * 64dw
 941	s_lshl_b32	s_restore_alloc_size, s_restore_alloc_size, 2		//LDS size in bytes
 942	s_mov_b32	s_restore_buf_rsrc2, s_restore_alloc_size		//NUM_RECORDS in bytes
 943
 944	// LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG)
 945	//
 946	get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size)
 947	get_svgpr_size_bytes(s_restore_tmp)
 948	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
 949	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes()
 950	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes()
 951
 952	s_mov_b32	s_restore_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
 953
 954	s_lshr_b32	m0, s_restore_size, S_WAVE_SIZE
 955	s_and_b32	m0, m0, 1
 956	s_cmp_eq_u32	m0, 1
 957	s_mov_b32	m0, 0x0
 958	s_cbranch_scc1	L_RESTORE_LDS_LOOP_W64
 959
 960L_RESTORE_LDS_LOOP_W32:
 961#if HAVE_BUFFER_LDS_LOAD
 962	buffer_load_dword	v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1	// first 64DW
 963#else
 964	buffer_load_dword       v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset
 965	s_waitcnt	vmcnt(0)
 966	ds_store_addtid_b32     v0
 967#endif
 968	s_add_u32	m0, m0, 128						// 128 DW
 969	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, 128		//mem offset increased by 128DW
 970	s_cmp_lt_u32	m0, s_restore_alloc_size				//scc=(m0 < s_restore_alloc_size) ? 1 : 0
 971	s_cbranch_scc1	L_RESTORE_LDS_LOOP_W32					//LDS restore is complete?
 972	s_branch	L_RESTORE_VGPR
 973
 974L_RESTORE_LDS_LOOP_W64:
 975#if HAVE_BUFFER_LDS_LOAD
 976	buffer_load_dword	v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1	// first 64DW
 977#else
 978	buffer_load_dword       v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset
 979	s_waitcnt	vmcnt(0)
 980	ds_store_addtid_b32     v0
 981#endif
 982	s_add_u32	m0, m0, 256						// 256 DW
 983	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, 256		//mem offset increased by 256DW
 984	s_cmp_lt_u32	m0, s_restore_alloc_size				//scc=(m0 < s_restore_alloc_size) ? 1 : 0
 985	s_cbranch_scc1	L_RESTORE_LDS_LOOP_W64					//LDS restore is complete?
 986
 987	/* restore VGPRs */
 988L_RESTORE_VGPR:
 989	// VGPR SR memory offset : 0
 990	s_mov_b32	s_restore_mem_offset, 0x0
 991 	s_mov_b32	exec_lo, 0xFFFFFFFF					//need every thread from now on
 992	s_lshr_b32	m0, s_restore_size, S_WAVE_SIZE
 993	s_and_b32	m0, m0, 1
 994	s_cmp_eq_u32	m0, 1
 995	s_cbranch_scc1	L_ENABLE_RESTORE_VGPR_EXEC_HI
 996	s_mov_b32	exec_hi, 0x00000000
 997	s_branch	L_RESTORE_VGPR_NORMAL
 998L_ENABLE_RESTORE_VGPR_EXEC_HI:
 999	s_mov_b32	exec_hi, 0xFFFFFFFF
1000L_RESTORE_VGPR_NORMAL:
1001	s_getreg_b32	s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)
1002	s_add_u32	s_restore_alloc_size, s_restore_alloc_size, 1
1003	s_lshl_b32	s_restore_alloc_size, s_restore_alloc_size, 2		//Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)
1004	//determine it is wave32 or wave64
1005	s_lshr_b32	m0, s_restore_size, S_WAVE_SIZE
1006	s_and_b32	m0, m0, 1
1007	s_cmp_eq_u32	m0, 1
1008	s_cbranch_scc1	L_RESTORE_VGPR_WAVE64
1009
1010	s_mov_b32	s_restore_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
1011
1012	// VGPR load using dw burst
1013	s_mov_b32	s_restore_mem_offset_save, s_restore_mem_offset		// restore start with v1, v0 will be the last
1014	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, 128*4
1015	s_mov_b32	m0, 4							//VGPR initial index value = 4
1016	s_cmp_lt_u32	m0, s_restore_alloc_size
1017	s_cbranch_scc0	L_RESTORE_SGPR
1018
1019L_RESTORE_VGPR_WAVE32_LOOP:
1020	buffer_load_dword	v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
1021	buffer_load_dword	v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:128
1022	buffer_load_dword	v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:128*2
1023	buffer_load_dword	v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:128*3
1024	s_waitcnt	vmcnt(0)
1025	v_movreld_b32	v0, v0							//v[0+m0] = v0
1026	v_movreld_b32	v1, v1
1027	v_movreld_b32	v2, v2
1028	v_movreld_b32	v3, v3
1029	s_add_u32	m0, m0, 4						//next vgpr index
1030	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, 128*4	//every buffer_load_dword does 128 bytes
1031	s_cmp_lt_u32	m0, s_restore_alloc_size				//scc = (m0 < s_restore_alloc_size) ? 1 : 0
1032	s_cbranch_scc1	L_RESTORE_VGPR_WAVE32_LOOP				//VGPR restore (except v0) is complete?
1033
1034	/* VGPR restore on v0 */
1035	buffer_load_dword	v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
1036	buffer_load_dword	v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:128
1037	buffer_load_dword	v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:128*2
1038	buffer_load_dword	v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:128*3
1039	s_waitcnt	vmcnt(0)
1040
1041	s_branch	L_RESTORE_SGPR
1042
1043L_RESTORE_VGPR_WAVE64:
1044	s_mov_b32	s_restore_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
1045
1046	// VGPR load using dw burst
1047	s_mov_b32	s_restore_mem_offset_save, s_restore_mem_offset		// restore start with v4, v0 will be the last
1048	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, 256*4
1049	s_mov_b32	m0, 4							//VGPR initial index value = 4
1050	s_cmp_lt_u32	m0, s_restore_alloc_size
1051	s_cbranch_scc0	L_RESTORE_SHARED_VGPR
1052
1053L_RESTORE_VGPR_WAVE64_LOOP:
1054	buffer_load_dword	v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
1055	buffer_load_dword	v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256
1056	buffer_load_dword	v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*2
1057	buffer_load_dword	v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*3
1058	s_waitcnt	vmcnt(0)
1059	v_movreld_b32	v0, v0							//v[0+m0] = v0
1060	v_movreld_b32	v1, v1
1061	v_movreld_b32	v2, v2
1062	v_movreld_b32	v3, v3
1063	s_add_u32	m0, m0, 4						//next vgpr index
1064	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, 256*4	//every buffer_load_dword does 256 bytes
1065	s_cmp_lt_u32	m0, s_restore_alloc_size				//scc = (m0 < s_restore_alloc_size) ? 1 : 0
1066	s_cbranch_scc1	L_RESTORE_VGPR_WAVE64_LOOP				//VGPR restore (except v0) is complete?
1067
1068L_RESTORE_SHARED_VGPR:
1069	//Below part will be the restore shared vgpr part (new for gfx10)
1070	s_getreg_b32	s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE)	//shared_vgpr_size
1071	s_and_b32	s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF	//shared_vgpr_size is zero?
1072	s_cbranch_scc0	L_RESTORE_V0						//no shared_vgpr used?
1073	s_lshl_b32	s_restore_alloc_size, s_restore_alloc_size, 3		//Number of SHARED_VGPRs = shared_vgpr_size * 8    (non-zero value)
1074	//m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count.
1075	//restore shared_vgpr will start from the index of m0
1076	s_add_u32	s_restore_alloc_size, s_restore_alloc_size, m0
1077	s_mov_b32	exec_lo, 0xFFFFFFFF
1078	s_mov_b32	exec_hi, 0x00000000
1079L_RESTORE_SHARED_VGPR_WAVE64_LOOP:
1080	buffer_load_dword	v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
1081	s_waitcnt	vmcnt(0)
1082	v_movreld_b32	v0, v0							//v[0+m0] = v0
1083	s_add_u32	m0, m0, 1						//next vgpr index
1084	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, 128
1085	s_cmp_lt_u32	m0, s_restore_alloc_size				//scc = (m0 < s_restore_alloc_size) ? 1 : 0
1086	s_cbranch_scc1	L_RESTORE_SHARED_VGPR_WAVE64_LOOP			//VGPR restore (except v0) is complete?
1087
1088	s_mov_b32	exec_hi, 0xFFFFFFFF					//restore back exec_hi before restoring V0!!
1089
1090	/* VGPR restore on v0 */
1091L_RESTORE_V0:
1092	buffer_load_dword	v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
1093	buffer_load_dword	v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256
1094	buffer_load_dword	v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*2
1095	buffer_load_dword	v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*3
1096	s_waitcnt	vmcnt(0)
1097
1098	/* restore SGPRs */
1099	//will be 2+8+16*6
1100	// SGPR SR memory offset : size(VGPR)+size(SVGPR)
1101L_RESTORE_SGPR:
1102	get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size)
1103	get_svgpr_size_bytes(s_restore_tmp)
1104	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
1105	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes()
1106	s_sub_u32	s_restore_mem_offset, s_restore_mem_offset, 20*4	//s108~s127 is not saved
1107
1108	s_mov_b32	s_restore_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
1109
1110	s_mov_b32	m0, s_sgpr_save_num
1111
1112	read_4sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)
1113	s_waitcnt	lgkmcnt(0)
1114
1115	s_sub_u32	m0, m0, 4						// Restore from S[0] to S[104]
1116	s_nop		0							// hazard SALU M0=> S_MOVREL
1117
1118	s_movreld_b64	s0, s0							//s[0+m0] = s0
1119	s_movreld_b64	s2, s2
1120
1121	read_8sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)
1122	s_waitcnt	lgkmcnt(0)
1123
1124	s_sub_u32	m0, m0, 8						// Restore from S[0] to S[96]
1125	s_nop		0							// hazard SALU M0=> S_MOVREL
1126
1127	s_movreld_b64	s0, s0							//s[0+m0] = s0
1128	s_movreld_b64	s2, s2
1129	s_movreld_b64	s4, s4
1130	s_movreld_b64	s6, s6
1131
1132 L_RESTORE_SGPR_LOOP:
1133	read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)
1134	s_waitcnt	lgkmcnt(0)
1135
1136	s_sub_u32	m0, m0, 16						// Restore from S[n] to S[0]
1137	s_nop		0							// hazard SALU M0=> S_MOVREL
1138
1139	s_movreld_b64	s0, s0							//s[0+m0] = s0
1140	s_movreld_b64	s2, s2
1141	s_movreld_b64	s4, s4
1142	s_movreld_b64	s6, s6
1143	s_movreld_b64	s8, s8
1144	s_movreld_b64	s10, s10
1145	s_movreld_b64	s12, s12
1146	s_movreld_b64	s14, s14
1147
1148	s_cmp_eq_u32	m0, 0							//scc = (m0 < s_sgpr_save_num) ? 1 : 0
1149	s_cbranch_scc0	L_RESTORE_SGPR_LOOP
1150
1151	// s_barrier with MODE.DEBUG_EN=1, STATUS.PRIV=1 incorrectly asserts debug exception.
1152	// Clear DEBUG_EN before and restore MODE after the barrier.
1153	s_setreg_imm32_b32	hwreg(HW_REG_MODE), 0
1154	s_barrier								//barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG
1155
1156	/* restore HW registers */
1157L_RESTORE_HWREG:
1158	// HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)
1159	get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size)
1160	get_svgpr_size_bytes(s_restore_tmp)
1161	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
1162	s_add_u32	s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes()
1163
1164	s_mov_b32	s_restore_buf_rsrc2, 0x1000000				//NUM_RECORDS in bytes
1165
1166	read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset)
1167	read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset)
1168	read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
1169	read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset)
1170	read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
1171	read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset)
1172	read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset)
1173	read_hwreg_from_mem(s_restore_xnack_mask, s_restore_buf_rsrc0, s_restore_mem_offset)
1174	read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset)
1175	read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset)
1176	s_waitcnt	lgkmcnt(0)
1177
1178	s_setreg_b32	hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO), s_restore_flat_scratch
1179
1180	read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset)
1181	s_waitcnt	lgkmcnt(0)						//from now on, it is safe to restore STATUS and IB_STS
1182
1183	s_setreg_b32	hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI), s_restore_flat_scratch
1184
1185	s_mov_b32	m0, s_restore_m0
1186	s_mov_b32	exec_lo, s_restore_exec_lo
1187	s_mov_b32	exec_hi, s_restore_exec_hi
1188
1189	s_and_b32	s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
1190	s_setreg_b32	hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
1191
1192#if HAVE_XNACK
1193	s_setreg_b32	hwreg(HW_REG_SHADER_XNACK_MASK), s_restore_xnack_mask
1194#endif
1195
1196	s_and_b32	s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
1197	s_lshr_b32	s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
1198	s_setreg_b32	hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
1199	s_setreg_b32	hwreg(HW_REG_MODE), s_restore_mode
1200
1201	// Restore trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
1202	// ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40
1203	get_vgpr_size_bytes(s_restore_ttmps_lo, s_restore_size)
1204	get_svgpr_size_bytes(s_restore_ttmps_hi)
1205	s_add_u32	s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_ttmps_hi
1206	s_add_u32	s_restore_ttmps_lo, s_restore_ttmps_lo, get_sgpr_size_bytes()
1207	s_add_u32	s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_buf_rsrc0
1208	s_addc_u32	s_restore_ttmps_hi, s_restore_buf_rsrc1, 0x0
1209	s_and_b32	s_restore_ttmps_hi, s_restore_ttmps_hi, 0xFFFF
1210	s_load_dwordx4	[ttmp4, ttmp5, ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x50 glc:1
1211	s_load_dwordx4	[ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x60 glc:1
1212	s_load_dword	ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x74 glc:1
1213	s_waitcnt	lgkmcnt(0)
1214
1215#if HAVE_XNACK
1216	restore_ib_sts(s_restore_tmp, s_restore_m0)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1217#endif
1218
1219	s_and_b32	s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff		//pc[47:32] //Do it here in order not to affect STATUS
1220	s_and_b64	exec, exec, exec					// Restore STATUS.EXECZ, not writable by s_setreg_b32
1221	s_and_b64	vcc, vcc, vcc						// Restore STATUS.VCCZ, not writable by s_setreg_b32
1222
1223#if SW_SA_TRAP
1224	// If traps are enabled then return to the shader with PRIV=0.
1225	// Otherwise retain PRIV=1 for subsequent context save requests.
1226	s_getreg_b32	s_restore_tmp, hwreg(HW_REG_STATUS)
1227	s_bitcmp1_b32	s_restore_tmp, SQ_WAVE_STATUS_TRAP_EN_SHIFT
1228	s_cbranch_scc1	L_RETURN_WITHOUT_PRIV
1229
1230	s_setreg_b32	hwreg(HW_REG_STATUS), s_restore_status			// SCC is included, which is changed by previous salu
1231	s_setpc_b64	[s_restore_pc_lo, s_restore_pc_hi]
1232L_RETURN_WITHOUT_PRIV:
1233#endif
1234
1235	s_setreg_b32	hwreg(HW_REG_STATUS), s_restore_status			// SCC is included, which is changed by previous salu
 
1236	s_rfe_b64	s_restore_pc_lo						//Return to the main shader program and resume execution
1237
1238L_END_PGM:
1239	s_endpgm_saved
1240end
1241
1242function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)
1243#if NO_SQC_STORE
1244	// Copy into VGPR for later TCP store.
1245	v_writelane_b32	v2, s, m0
1246	s_add_u32	m0, m0, 0x1
1247#else
1248	s_mov_b32	exec_lo, m0
1249	s_mov_b32	m0, s_mem_offset
1250	s_buffer_store_dword	s, s_rsrc, m0 glc:1
1251	s_add_u32	s_mem_offset, s_mem_offset, 4
1252	s_mov_b32	m0, exec_lo
1253#endif
1254end
1255
1256
1257function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)
1258#if NO_SQC_STORE
1259	// Copy into VGPR for later TCP store.
1260	for var sgpr_idx = 0; sgpr_idx < 16; sgpr_idx ++
1261		v_writelane_b32	v2, s[sgpr_idx], ttmp13
1262		s_add_u32	ttmp13, ttmp13, 0x1
1263	end
1264#else
1265	s_buffer_store_dwordx4	s[0], s_rsrc, 0 glc:1
1266	s_buffer_store_dwordx4	s[4], s_rsrc, 16 glc:1
1267	s_buffer_store_dwordx4	s[8], s_rsrc, 32 glc:1
1268	s_buffer_store_dwordx4	s[12], s_rsrc, 48 glc:1
1269	s_add_u32	s_rsrc[0], s_rsrc[0], 4*16
1270	s_addc_u32	s_rsrc[1], s_rsrc[1], 0x0
1271#endif
1272end
1273
1274function write_12sgpr_to_mem(s, s_rsrc, s_mem_offset)
1275#if NO_SQC_STORE
1276	// Copy into VGPR for later TCP store.
1277	for var sgpr_idx = 0; sgpr_idx < 12; sgpr_idx ++
1278		v_writelane_b32	v2, s[sgpr_idx], ttmp13
1279		s_add_u32	ttmp13, ttmp13, 0x1
1280	end
1281#else
1282	s_buffer_store_dwordx4	s[0], s_rsrc, 0 glc:1
1283	s_buffer_store_dwordx4	s[4], s_rsrc, 16 glc:1
1284	s_buffer_store_dwordx4	s[8], s_rsrc, 32 glc:1
1285	s_add_u32	s_rsrc[0], s_rsrc[0], 4*12
1286	s_addc_u32	s_rsrc[1], s_rsrc[1], 0x0
1287#endif
1288end
1289
1290function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
1291	s_buffer_load_dword	s, s_rsrc, s_mem_offset glc:1
1292	s_add_u32	s_mem_offset, s_mem_offset, 4
1293end
1294
1295function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)
1296	s_sub_u32	s_mem_offset, s_mem_offset, 4*16
1297	s_buffer_load_dwordx16	s, s_rsrc, s_mem_offset glc:1
1298end
1299
1300function read_8sgpr_from_mem(s, s_rsrc, s_mem_offset)
1301	s_sub_u32	s_mem_offset, s_mem_offset, 4*8
1302	s_buffer_load_dwordx8	s, s_rsrc, s_mem_offset glc:1
1303end
1304
1305function read_4sgpr_from_mem(s, s_rsrc, s_mem_offset)
1306	s_sub_u32	s_mem_offset, s_mem_offset, 4*4
1307	s_buffer_load_dwordx4	s, s_rsrc, s_mem_offset glc:1
1308end
1309
1310#if SAVE_AFTER_XNACK_ERROR
1311function check_if_tcp_store_ok
1312	// If TRAPSTS.XNACK_ERROR=1 then TCP stores will fail.
1313	s_getreg_b32 s_save_tmp, hwreg(HW_REG_TRAPSTS)
1314	s_andn2_b32 s_save_tmp, SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK, s_save_tmp
1315
1316L_TCP_STORE_CHECK_DONE:
1317end
1318
1319function write_vgpr_to_mem_with_sqc(vgpr, n_lanes, s_rsrc, s_mem_offset)
1320	s_mov_b32 s4, 0
1321
1322L_WRITE_VGPR_LANE_LOOP:
1323	for var lane = 0; lane < 4; ++lane
1324		v_readlane_b32 s[lane], vgpr, s4
1325		s_add_u32 s4, s4, 1
1326	end
1327
1328	s_buffer_store_dwordx4 s[0:3], s_rsrc, s_mem_offset glc:1
1329
1330	s_add_u32 s_mem_offset, s_mem_offset, 0x10
1331	s_cmp_eq_u32 s4, n_lanes
1332	s_cbranch_scc0 L_WRITE_VGPR_LANE_LOOP
1333end
1334
1335function write_vgprs_to_mem_with_sqc_w32(vgpr0, n_vgprs, s_rsrc, s_mem_offset)
1336	for var vgpr = 0; vgpr < n_vgprs; ++vgpr
1337		write_vgpr_to_mem_with_sqc(vgpr0[vgpr], 32, s_rsrc, s_mem_offset)
1338	end
1339end
1340
1341function write_vgprs_to_mem_with_sqc_w64(vgpr0, n_vgprs, s_rsrc, s_mem_offset)
1342	for var vgpr = 0; vgpr < n_vgprs; ++vgpr
1343		write_vgpr_to_mem_with_sqc(vgpr0[vgpr], 64, s_rsrc, s_mem_offset)
1344	end
1345end
1346#endif
1347
1348function get_lds_size_bytes(s_lds_size_byte)
1349	s_getreg_b32	s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)
1350	s_lshl_b32	s_lds_size_byte, s_lds_size_byte, 8			//LDS size in dwords = lds_size * 64 *4Bytes // granularity 64DW
1351end
1352
1353function get_vgpr_size_bytes(s_vgpr_size_byte, s_size)
1354	s_getreg_b32	s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)
1355	s_add_u32	s_vgpr_size_byte, s_vgpr_size_byte, 1
1356	s_bitcmp1_b32	s_size, S_WAVE_SIZE
1357	s_cbranch_scc1	L_ENABLE_SHIFT_W64
1358	s_lshl_b32	s_vgpr_size_byte, s_vgpr_size_byte, (2+7)		//Number of VGPRs = (vgpr_size + 1) * 4 * 32 * 4   (non-zero value)
1359	s_branch	L_SHIFT_DONE
1360L_ENABLE_SHIFT_W64:
1361	s_lshl_b32	s_vgpr_size_byte, s_vgpr_size_byte, (2+8)		//Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4   (non-zero value)
1362L_SHIFT_DONE:
1363end
1364
1365function get_svgpr_size_bytes(s_svgpr_size_byte)
1366	s_getreg_b32	s_svgpr_size_byte, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE)
1367	s_lshl_b32	s_svgpr_size_byte, s_svgpr_size_byte, (3+7)
1368end
1369
1370function get_sgpr_size_bytes
1371	return 512
1372end
1373
1374function get_hwreg_size_bytes
1375	return 128
1376end
1377
1378function get_wave_size(s_reg)
1379	s_getreg_b32	s_reg, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE)
1380	s_lshl_b32	s_reg, s_reg, S_WAVE_SIZE
1381end
1382
1383function save_and_clear_ib_sts(tmp1, tmp2)
1384	// Preserve and clear scalar XNACK state before issuing scalar loads.
1385	// Save IB_STS.REPLAY_W64H[25], RCNT[21:16], FIRST_REPLAY[15] into
1386	// unused space ttmp11[31:24].
1387	s_andn2_b32	ttmp11, ttmp11, (TTMP11_SAVE_REPLAY_W64H_MASK | TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK)
1388	s_getreg_b32	tmp1, hwreg(HW_REG_IB_STS)
1389	s_and_b32	tmp2, tmp1, SQ_WAVE_IB_STS_REPLAY_W64H_MASK
1390	s_lshl_b32	tmp2, tmp2, (TTMP11_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT)
1391	s_or_b32	ttmp11, ttmp11, tmp2
1392	s_and_b32	tmp2, tmp1, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
1393	s_lshl_b32	tmp2, tmp2, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
1394	s_or_b32	ttmp11, ttmp11, tmp2
1395	s_andn2_b32	tmp1, tmp1, (SQ_WAVE_IB_STS_REPLAY_W64H_MASK | SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK)
1396	s_setreg_b32	hwreg(HW_REG_IB_STS), tmp1
1397end
1398
1399function restore_ib_sts(tmp1, tmp2)
1400	s_lshr_b32	tmp1, ttmp11, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
1401	s_and_b32	tmp2, tmp1, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
1402	s_lshr_b32	tmp1, ttmp11, (TTMP11_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT)
1403	s_and_b32	tmp1, tmp1, SQ_WAVE_IB_STS_REPLAY_W64H_MASK
1404	s_or_b32	tmp1, tmp1, tmp2
1405	s_setreg_b32	hwreg(HW_REG_IB_STS), tmp1
1406end