Loading...
Note: File does not exist in v4.6.
1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23/* To compile this assembly code:
24 *
25 * Navi1x:
26 * cpp -DASIC_FAMILY=CHIP_NAVI10 cwsr_trap_handler_gfx10.asm -P -o nv1x.sp3
27 * sp3 nv1x.sp3 -hex nv1x.hex
28 *
29 * gfx10:
30 * cpp -DASIC_FAMILY=CHIP_SIENNA_CICHLID cwsr_trap_handler_gfx10.asm -P -o gfx10.sp3
31 * sp3 gfx10.sp3 -hex gfx10.hex
32 *
33 * gfx11:
34 * cpp -DASIC_FAMILY=CHIP_PLUM_BONITO cwsr_trap_handler_gfx10.asm -P -o gfx11.sp3
35 * sp3 gfx11.sp3 -hex gfx11.hex
36 */
37
38#define CHIP_NAVI10 26
39#define CHIP_SIENNA_CICHLID 30
40#define CHIP_PLUM_BONITO 36
41
42#define NO_SQC_STORE (ASIC_FAMILY >= CHIP_SIENNA_CICHLID)
43#define HAVE_XNACK (ASIC_FAMILY < CHIP_SIENNA_CICHLID)
44#define HAVE_SENDMSG_RTN (ASIC_FAMILY >= CHIP_PLUM_BONITO)
45#define HAVE_BUFFER_LDS_LOAD (ASIC_FAMILY < CHIP_PLUM_BONITO)
46#define SW_SA_TRAP (ASIC_FAMILY >= CHIP_PLUM_BONITO)
47
48var SINGLE_STEP_MISSED_WORKAROUND = 1 //workaround for lost MODE.DEBUG_EN exception when SAVECTX raised
49
50var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006
51var SQ_WAVE_STATUS_HALT_MASK = 0x2000
52var SQ_WAVE_STATUS_ECC_ERR_MASK = 0x20000
53var SQ_WAVE_STATUS_TRAP_EN_SHIFT = 6
54
55var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12
56var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9
57var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 8
58var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT = 24
59var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE = 4
60var SQ_WAVE_IB_STS2_WAVE64_SHIFT = 11
61var SQ_WAVE_IB_STS2_WAVE64_SIZE = 1
62
63#if ASIC_FAMILY < CHIP_PLUM_BONITO
64var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 8
65#else
66var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 12
67#endif
68
69var SQ_WAVE_TRAPSTS_SAVECTX_MASK = 0x400
70var SQ_WAVE_TRAPSTS_EXCP_MASK = 0x1FF
71var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT = 10
72var SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK = 0x80
73var SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT = 7
74var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK = 0x100
75var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT = 8
76var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK = 0x3FF
77var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT = 0x0
78var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE = 10
79var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK = 0xFFFFF800
80var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT = 11
81var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE = 21
82var SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK = 0x800
83var SQ_WAVE_TRAPSTS_EXCP_HI_MASK = 0x7000
84
85var SQ_WAVE_MODE_EXCP_EN_SHIFT = 12
86var SQ_WAVE_MODE_EXCP_EN_ADDR_WATCH_SHIFT = 19
87
88var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT = 15
89var SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT = 25
90var SQ_WAVE_IB_STS_REPLAY_W64H_MASK = 0x02000000
91var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK = 0x003F8000
92
93var SQ_WAVE_MODE_DEBUG_EN_MASK = 0x800
94
95// bits [31:24] unused by SPI debug data
96var TTMP11_SAVE_REPLAY_W64H_SHIFT = 31
97var TTMP11_SAVE_REPLAY_W64H_MASK = 0x80000000
98var TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT = 24
99var TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK = 0x7F000000
100var TTMP11_DEBUG_TRAP_ENABLED_SHIFT = 23
101var TTMP11_DEBUG_TRAP_ENABLED_MASK = 0x800000
102
103// SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14]
104// when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
105var S_SAVE_BUF_RSRC_WORD1_STRIDE = 0x00040000
106var S_SAVE_BUF_RSRC_WORD3_MISC = 0x10807FAC
107var S_SAVE_PC_HI_TRAP_ID_MASK = 0x00FF0000
108var S_SAVE_PC_HI_HT_MASK = 0x01000000
109var S_SAVE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000
110var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26
111
112var S_SAVE_PC_HI_FIRST_WAVE_MASK = 0x80000000
113var S_SAVE_PC_HI_FIRST_WAVE_SHIFT = 31
114
115var s_sgpr_save_num = 108
116
117var s_save_spi_init_lo = exec_lo
118var s_save_spi_init_hi = exec_hi
119var s_save_pc_lo = ttmp0
120var s_save_pc_hi = ttmp1
121var s_save_exec_lo = ttmp2
122var s_save_exec_hi = ttmp3
123var s_save_status = ttmp12
124var s_save_trapsts = ttmp15
125var s_save_xnack_mask = s_save_trapsts
126var s_wave_size = ttmp7
127var s_save_buf_rsrc0 = ttmp8
128var s_save_buf_rsrc1 = ttmp9
129var s_save_buf_rsrc2 = ttmp10
130var s_save_buf_rsrc3 = ttmp11
131var s_save_mem_offset = ttmp4
132var s_save_alloc_size = s_save_trapsts
133var s_save_tmp = ttmp14
134var s_save_m0 = ttmp5
135var s_save_ttmps_lo = s_save_tmp
136var s_save_ttmps_hi = s_save_trapsts
137
138var S_RESTORE_BUF_RSRC_WORD1_STRIDE = S_SAVE_BUF_RSRC_WORD1_STRIDE
139var S_RESTORE_BUF_RSRC_WORD3_MISC = S_SAVE_BUF_RSRC_WORD3_MISC
140
141var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000
142var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT = 26
143var S_WAVE_SIZE = 25
144
145var s_restore_spi_init_lo = exec_lo
146var s_restore_spi_init_hi = exec_hi
147var s_restore_mem_offset = ttmp12
148var s_restore_alloc_size = ttmp3
149var s_restore_tmp = ttmp2
150var s_restore_mem_offset_save = s_restore_tmp
151var s_restore_m0 = s_restore_alloc_size
152var s_restore_mode = ttmp7
153var s_restore_flat_scratch = s_restore_tmp
154var s_restore_pc_lo = ttmp0
155var s_restore_pc_hi = ttmp1
156var s_restore_exec_lo = ttmp4
157var s_restore_exec_hi = ttmp5
158var s_restore_status = ttmp14
159var s_restore_trapsts = ttmp15
160var s_restore_xnack_mask = ttmp13
161var s_restore_buf_rsrc0 = ttmp8
162var s_restore_buf_rsrc1 = ttmp9
163var s_restore_buf_rsrc2 = ttmp10
164var s_restore_buf_rsrc3 = ttmp11
165var s_restore_size = ttmp6
166var s_restore_ttmps_lo = s_restore_tmp
167var s_restore_ttmps_hi = s_restore_alloc_size
168
169shader main
170 asic(DEFAULT)
171 type(CS)
172 wave_size(32)
173
174 s_branch L_SKIP_RESTORE //NOT restore. might be a regular trap or save
175
176L_JUMP_TO_RESTORE:
177 s_branch L_RESTORE
178
179L_SKIP_RESTORE:
180 s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC
181
182 // Clear SPI_PRIO: do not save with elevated priority.
183 // Clear ECC_ERR: prevents SQC store and triggers FATAL_HALT if setreg'd.
184 s_andn2_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK|SQ_WAVE_STATUS_ECC_ERR_MASK
185
186 s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
187
188#if SW_SA_TRAP
189 // If ttmp1[30] is set then issue s_barrier to unblock dependent waves.
190 s_bitcmp1_b32 s_save_pc_hi, 30
191 s_cbranch_scc0 L_TRAP_NO_BARRIER
192 s_barrier
193
194L_TRAP_NO_BARRIER:
195 // If ttmp1[31] is set then trap may occur early.
196 // Spin wait until SAVECTX exception is raised.
197 s_bitcmp1_b32 s_save_pc_hi, 31
198 s_cbranch_scc1 L_CHECK_SAVE
199#endif
200
201 s_and_b32 ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK
202 s_cbranch_scc0 L_NOT_HALTED
203
204L_HALTED:
205 // Host trap may occur while wave is halted.
206 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
207 s_cbranch_scc1 L_FETCH_2ND_TRAP
208
209L_CHECK_SAVE:
210 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK
211 s_cbranch_scc1 L_SAVE
212
213 // Wave is halted but neither host trap nor SAVECTX is raised.
214 // Caused by instruction fetch memory violation.
215 // Spin wait until context saved to prevent interrupt storm.
216 s_sleep 0x10
217 s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
218 s_branch L_CHECK_SAVE
219
220L_NOT_HALTED:
221 // Let second-level handle non-SAVECTX exception or trap.
222 // Any concurrent SAVECTX will be handled upon re-entry once halted.
223
224 // Check non-maskable exceptions. memory_violation, illegal_instruction
225 // and xnack_error exceptions always cause the wave to enter the trap
226 // handler.
227 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK|SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK
228 s_cbranch_scc1 L_FETCH_2ND_TRAP
229
230 // Check for maskable exceptions in trapsts.excp and trapsts.excp_hi.
231 // Maskable exceptions only cause the wave to enter the trap handler if
232 // their respective bit in mode.excp_en is set.
233 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCP_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK
234 s_cbranch_scc0 L_CHECK_TRAP_ID
235
236 s_and_b32 ttmp3, s_save_trapsts, SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK
237 s_cbranch_scc0 L_NOT_ADDR_WATCH
238 s_bitset1_b32 ttmp2, SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT // Check all addr_watch[123] exceptions against excp_en.addr_watch
239
240L_NOT_ADDR_WATCH:
241 s_getreg_b32 ttmp3, hwreg(HW_REG_MODE)
242 s_lshl_b32 ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT
243 s_and_b32 ttmp2, ttmp2, ttmp3
244 s_cbranch_scc1 L_FETCH_2ND_TRAP
245
246L_CHECK_TRAP_ID:
247 // Check trap_id != 0
248 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
249 s_cbranch_scc1 L_FETCH_2ND_TRAP
250
251if SINGLE_STEP_MISSED_WORKAROUND
252 // Prioritize single step exception over context save.
253 // Second-level trap will halt wave and RFE, re-entering for SAVECTX.
254 s_getreg_b32 ttmp2, hwreg(HW_REG_MODE)
255 s_and_b32 ttmp2, ttmp2, SQ_WAVE_MODE_DEBUG_EN_MASK
256 s_cbranch_scc1 L_FETCH_2ND_TRAP
257end
258
259 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK
260 s_cbranch_scc1 L_SAVE
261
262L_FETCH_2ND_TRAP:
263#if HAVE_XNACK
264 save_and_clear_ib_sts(ttmp14, ttmp15)
265#endif
266
267 // Read second-level TBA/TMA from first-level TMA and jump if available.
268 // ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data)
269 // ttmp12 holds SQ_WAVE_STATUS
270#if HAVE_SENDMSG_RTN
271 s_sendmsg_rtn_b64 [ttmp14, ttmp15], sendmsg(MSG_RTN_GET_TMA)
272 s_waitcnt lgkmcnt(0)
273#else
274 s_getreg_b32 ttmp14, hwreg(HW_REG_SHADER_TMA_LO)
275 s_getreg_b32 ttmp15, hwreg(HW_REG_SHADER_TMA_HI)
276#endif
277 s_lshl_b64 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8
278
279 s_bitcmp1_b32 ttmp15, 0xF
280 s_cbranch_scc0 L_NO_SIGN_EXTEND_TMA
281 s_or_b32 ttmp15, ttmp15, 0xFFFF0000
282L_NO_SIGN_EXTEND_TMA:
283
284 s_load_dword ttmp2, [ttmp14, ttmp15], 0x10 glc:1 // debug trap enabled flag
285 s_waitcnt lgkmcnt(0)
286 s_lshl_b32 ttmp2, ttmp2, TTMP11_DEBUG_TRAP_ENABLED_SHIFT
287 s_andn2_b32 ttmp11, ttmp11, TTMP11_DEBUG_TRAP_ENABLED_MASK
288 s_or_b32 ttmp11, ttmp11, ttmp2
289
290 s_load_dwordx2 [ttmp2, ttmp3], [ttmp14, ttmp15], 0x0 glc:1 // second-level TBA
291 s_waitcnt lgkmcnt(0)
292 s_load_dwordx2 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 glc:1 // second-level TMA
293 s_waitcnt lgkmcnt(0)
294
295 s_and_b64 [ttmp2, ttmp3], [ttmp2, ttmp3], [ttmp2, ttmp3]
296 s_cbranch_scc0 L_NO_NEXT_TRAP // second-level trap handler not been set
297 s_setpc_b64 [ttmp2, ttmp3] // jump to second-level trap handler
298
299L_NO_NEXT_TRAP:
300 // If not caused by trap then halt wave to prevent re-entry.
301 s_and_b32 ttmp2, s_save_pc_hi, (S_SAVE_PC_HI_TRAP_ID_MASK|S_SAVE_PC_HI_HT_MASK)
302 s_cbranch_scc1 L_TRAP_CASE
303 s_or_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_HALT_MASK
304
305 // If the PC points to S_ENDPGM then context save will fail if STATUS.HALT is set.
306 // Rewind the PC to prevent this from occurring.
307 s_sub_u32 ttmp0, ttmp0, 0x8
308 s_subb_u32 ttmp1, ttmp1, 0x0
309
310 s_branch L_EXIT_TRAP
311
312L_TRAP_CASE:
313 // Host trap will not cause trap re-entry.
314 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_HT_MASK
315 s_cbranch_scc1 L_EXIT_TRAP
316
317 // Advance past trap instruction to prevent re-entry.
318 s_add_u32 ttmp0, ttmp0, 0x4
319 s_addc_u32 ttmp1, ttmp1, 0x0
320
321L_EXIT_TRAP:
322 s_and_b32 ttmp1, ttmp1, 0xFFFF
323
324#if HAVE_XNACK
325 restore_ib_sts(ttmp14, ttmp15)
326#endif
327
328 // Restore SQ_WAVE_STATUS.
329 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
330 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
331 s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status
332
333 s_rfe_b64 [ttmp0, ttmp1]
334
335L_SAVE:
336 s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
337 s_mov_b32 s_save_tmp, 0
338 s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp //clear saveCtx bit
339
340#if HAVE_XNACK
341 save_and_clear_ib_sts(s_save_tmp, s_save_trapsts)
342#endif
343
344 /* inform SPI the readiness and wait for SPI's go signal */
345 s_mov_b32 s_save_exec_lo, exec_lo //save EXEC and use EXEC for the go signal from SPI
346 s_mov_b32 s_save_exec_hi, exec_hi
347 s_mov_b64 exec, 0x0 //clear EXEC to get ready to receive
348
349#if HAVE_SENDMSG_RTN
350 s_sendmsg_rtn_b64 [exec_lo, exec_hi], sendmsg(MSG_RTN_SAVE_WAVE)
351#else
352 s_sendmsg sendmsg(MSG_SAVEWAVE) //send SPI a message and wait for SPI's write to EXEC
353#endif
354
355#if ASIC_FAMILY < CHIP_SIENNA_CICHLID
356L_SLEEP:
357 // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause
358 // SQ hang, since the 7,8th wave could not get arbit to exec inst, while
359 // other waves are stuck into the sleep-loop and waiting for wrexec!=0
360 s_sleep 0x2
361 s_cbranch_execz L_SLEEP
362#else
363 s_waitcnt lgkmcnt(0)
364#endif
365
366 // Save first_wave flag so we can clear high bits of save address.
367 s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK
368 s_lshl_b32 s_save_tmp, s_save_tmp, (S_SAVE_PC_HI_FIRST_WAVE_SHIFT - S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT)
369 s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp
370
371#if NO_SQC_STORE
372#if ASIC_FAMILY <= CHIP_SIENNA_CICHLID
373 // gfx10: If there was a VALU exception, the exception state must be
374 // cleared before executing the VALU instructions below.
375 v_clrexcp
376#endif
377
378 // Trap temporaries must be saved via VGPR but all VGPRs are in use.
379 // There is no ttmp space to hold the resource constant for VGPR save.
380 // Save v0 by itself since it requires only two SGPRs.
381 s_mov_b32 s_save_ttmps_lo, exec_lo
382 s_and_b32 s_save_ttmps_hi, exec_hi, 0xFFFF
383 s_mov_b32 exec_lo, 0xFFFFFFFF
384 s_mov_b32 exec_hi, 0xFFFFFFFF
385 global_store_dword_addtid v0, [s_save_ttmps_lo, s_save_ttmps_hi] slc:1 glc:1
386 v_mov_b32 v0, 0x0
387 s_mov_b32 exec_lo, s_save_ttmps_lo
388 s_mov_b32 exec_hi, s_save_ttmps_hi
389#endif
390
391 // Save trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
392 // ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40
393 get_wave_size(s_save_ttmps_hi)
394 get_vgpr_size_bytes(s_save_ttmps_lo, s_save_ttmps_hi)
395 get_svgpr_size_bytes(s_save_ttmps_hi)
396 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_ttmps_hi
397 s_and_b32 s_save_ttmps_hi, s_save_spi_init_hi, 0xFFFF
398 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, get_sgpr_size_bytes()
399 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_spi_init_lo
400 s_addc_u32 s_save_ttmps_hi, s_save_ttmps_hi, 0x0
401
402#if NO_SQC_STORE
403 v_writelane_b32 v0, ttmp4, 0x4
404 v_writelane_b32 v0, ttmp5, 0x5
405 v_writelane_b32 v0, ttmp6, 0x6
406 v_writelane_b32 v0, ttmp7, 0x7
407 v_writelane_b32 v0, ttmp8, 0x8
408 v_writelane_b32 v0, ttmp9, 0x9
409 v_writelane_b32 v0, ttmp10, 0xA
410 v_writelane_b32 v0, ttmp11, 0xB
411 v_writelane_b32 v0, ttmp13, 0xD
412 v_writelane_b32 v0, exec_lo, 0xE
413 v_writelane_b32 v0, exec_hi, 0xF
414
415 s_mov_b32 exec_lo, 0x3FFF
416 s_mov_b32 exec_hi, 0x0
417 global_store_dword_addtid v0, [s_save_ttmps_lo, s_save_ttmps_hi] inst_offset:0x40 slc:1 glc:1
418 v_readlane_b32 ttmp14, v0, 0xE
419 v_readlane_b32 ttmp15, v0, 0xF
420 s_mov_b32 exec_lo, ttmp14
421 s_mov_b32 exec_hi, ttmp15
422#else
423 s_store_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_save_ttmps_lo, s_save_ttmps_hi], 0x50 glc:1
424 s_store_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_save_ttmps_lo, s_save_ttmps_hi], 0x60 glc:1
425 s_store_dword ttmp13, [s_save_ttmps_lo, s_save_ttmps_hi], 0x74 glc:1
426#endif
427
428 /* setup Resource Contants */
429 s_mov_b32 s_save_buf_rsrc0, s_save_spi_init_lo //base_addr_lo
430 s_and_b32 s_save_buf_rsrc1, s_save_spi_init_hi, 0x0000FFFF //base_addr_hi
431 s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE
432 s_mov_b32 s_save_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
433 s_mov_b32 s_save_buf_rsrc3, S_SAVE_BUF_RSRC_WORD3_MISC
434
435 s_mov_b32 s_save_m0, m0
436
437 /* global mem offset */
438 s_mov_b32 s_save_mem_offset, 0x0
439 get_wave_size(s_wave_size)
440
441#if HAVE_XNACK
442 // Save and clear vector XNACK state late to free up SGPRs.
443 s_getreg_b32 s_save_xnack_mask, hwreg(HW_REG_SHADER_XNACK_MASK)
444 s_setreg_imm32_b32 hwreg(HW_REG_SHADER_XNACK_MASK), 0x0
445#endif
446
447 /* save first 4 VGPRs, needed for SGPR save */
448 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
449 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
450 s_and_b32 m0, m0, 1
451 s_cmp_eq_u32 m0, 1
452 s_cbranch_scc1 L_ENABLE_SAVE_4VGPR_EXEC_HI
453 s_mov_b32 exec_hi, 0x00000000
454 s_branch L_SAVE_4VGPR_WAVE32
455L_ENABLE_SAVE_4VGPR_EXEC_HI:
456 s_mov_b32 exec_hi, 0xFFFFFFFF
457 s_branch L_SAVE_4VGPR_WAVE64
458L_SAVE_4VGPR_WAVE32:
459 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
460
461 // VGPR Allocated in 4-GPR granularity
462
463#if !NO_SQC_STORE
464 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
465#endif
466 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128
467 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*2
468 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*3
469 s_branch L_SAVE_HWREG
470
471L_SAVE_4VGPR_WAVE64:
472 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
473
474 // VGPR Allocated in 4-GPR granularity
475
476#if !NO_SQC_STORE
477 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
478#endif
479 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
480 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
481 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
482
483 /* save HW registers */
484
485L_SAVE_HWREG:
486 // HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)
487 get_vgpr_size_bytes(s_save_mem_offset, s_wave_size)
488 get_svgpr_size_bytes(s_save_tmp)
489 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
490 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes()
491
492 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
493
494#if NO_SQC_STORE
495 v_mov_b32 v0, 0x0 //Offset[31:0] from buffer resource
496 v_mov_b32 v1, 0x0 //Offset[63:32] from buffer resource
497 v_mov_b32 v2, 0x0 //Set of SGPRs for TCP store
498 s_mov_b32 m0, 0x0 //Next lane of v2 to write to
499#endif
500
501 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
502 write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset)
503 s_andn2_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
504 write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)
505 write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset)
506 write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
507 write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset)
508
509 s_getreg_b32 s_save_tmp, hwreg(HW_REG_TRAPSTS)
510 write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)
511
512 // Not used on Sienna_Cichlid but keep layout same for debugger.
513 write_hwreg_to_mem(s_save_xnack_mask, s_save_buf_rsrc0, s_save_mem_offset)
514
515 s_getreg_b32 s_save_m0, hwreg(HW_REG_MODE)
516 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
517
518 s_getreg_b32 s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO)
519 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
520
521 s_getreg_b32 s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI)
522 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
523
524#if NO_SQC_STORE
525 // Write HWREGs with 16 VGPR lanes. TTMPs occupy space after this.
526 s_mov_b32 exec_lo, 0xFFFF
527 s_mov_b32 exec_hi, 0x0
528 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
529
530 // Write SGPRs with 32 VGPR lanes. This works in wave32 and wave64 mode.
531 s_mov_b32 exec_lo, 0xFFFFFFFF
532#endif
533
534 /* save SGPRs */
535 // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...
536
537 // SGPR SR memory offset : size(VGPR)+size(SVGPR)
538 get_vgpr_size_bytes(s_save_mem_offset, s_wave_size)
539 get_svgpr_size_bytes(s_save_tmp)
540 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
541 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
542
543#if NO_SQC_STORE
544 s_mov_b32 ttmp13, 0x0 //next VGPR lane to copy SGPR into
545#else
546 // backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0
547 s_mov_b32 s_save_xnack_mask, s_save_buf_rsrc0
548 s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset
549 s_addc_u32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0
550#endif
551
552 s_mov_b32 m0, 0x0 //SGPR initial index value =0
553 s_nop 0x0 //Manually inserted wait states
554L_SAVE_SGPR_LOOP:
555 // SGPR is allocated in 16 SGPR granularity
556 s_movrels_b64 s0, s0 //s0 = s[0+m0], s1 = s[1+m0]
557 s_movrels_b64 s2, s2 //s2 = s[2+m0], s3 = s[3+m0]
558 s_movrels_b64 s4, s4 //s4 = s[4+m0], s5 = s[5+m0]
559 s_movrels_b64 s6, s6 //s6 = s[6+m0], s7 = s[7+m0]
560 s_movrels_b64 s8, s8 //s8 = s[8+m0], s9 = s[9+m0]
561 s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0]
562 s_movrels_b64 s12, s12 //s12 = s[12+m0], s13 = s[13+m0]
563 s_movrels_b64 s14, s14 //s14 = s[14+m0], s15 = s[15+m0]
564
565 write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset)
566
567#if NO_SQC_STORE
568 s_cmp_eq_u32 ttmp13, 0x20 //have 32 VGPR lanes filled?
569 s_cbranch_scc0 L_SAVE_SGPR_SKIP_TCP_STORE
570
571 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
572 s_add_u32 s_save_mem_offset, s_save_mem_offset, 0x80
573 s_mov_b32 ttmp13, 0x0
574 v_mov_b32 v2, 0x0
575L_SAVE_SGPR_SKIP_TCP_STORE:
576#endif
577
578 s_add_u32 m0, m0, 16 //next sgpr index
579 s_cmp_lt_u32 m0, 96 //scc = (m0 < first 96 SGPR) ? 1 : 0
580 s_cbranch_scc1 L_SAVE_SGPR_LOOP //first 96 SGPR save is complete?
581
582 //save the rest 12 SGPR
583 s_movrels_b64 s0, s0 //s0 = s[0+m0], s1 = s[1+m0]
584 s_movrels_b64 s2, s2 //s2 = s[2+m0], s3 = s[3+m0]
585 s_movrels_b64 s4, s4 //s4 = s[4+m0], s5 = s[5+m0]
586 s_movrels_b64 s6, s6 //s6 = s[6+m0], s7 = s[7+m0]
587 s_movrels_b64 s8, s8 //s8 = s[8+m0], s9 = s[9+m0]
588 s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0]
589 write_12sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset)
590
591#if NO_SQC_STORE
592 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
593#else
594 // restore s_save_buf_rsrc0,1
595 s_mov_b32 s_save_buf_rsrc0, s_save_xnack_mask
596#endif
597
598 /* save LDS */
599
600L_SAVE_LDS:
601 // Change EXEC to all threads...
602 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
603 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
604 s_and_b32 m0, m0, 1
605 s_cmp_eq_u32 m0, 1
606 s_cbranch_scc1 L_ENABLE_SAVE_LDS_EXEC_HI
607 s_mov_b32 exec_hi, 0x00000000
608 s_branch L_SAVE_LDS_NORMAL
609L_ENABLE_SAVE_LDS_EXEC_HI:
610 s_mov_b32 exec_hi, 0xFFFFFFFF
611L_SAVE_LDS_NORMAL:
612 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)
613 s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //lds_size is zero?
614 s_cbranch_scc0 L_SAVE_LDS_DONE //no lds used? jump to L_SAVE_DONE
615
616 s_barrier //LDS is used? wait for other waves in the same TG
617 s_and_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
618 s_cbranch_scc0 L_SAVE_LDS_DONE
619
620 // first wave do LDS save;
621
622 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
623 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //LDS size in bytes
624 s_mov_b32 s_save_buf_rsrc2, s_save_alloc_size //NUM_RECORDS in bytes
625
626 // LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG)
627 //
628 get_vgpr_size_bytes(s_save_mem_offset, s_wave_size)
629 get_svgpr_size_bytes(s_save_tmp)
630 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
631 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes()
632 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()
633
634 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
635
636 //load 0~63*4(byte address) to vgpr v0
637 v_mbcnt_lo_u32_b32 v0, -1, 0
638 v_mbcnt_hi_u32_b32 v0, -1, v0
639 v_mul_u32_u24 v0, 4, v0
640
641 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
642 s_and_b32 m0, m0, 1
643 s_cmp_eq_u32 m0, 1
644 s_mov_b32 m0, 0x0
645 s_cbranch_scc1 L_SAVE_LDS_W64
646
647L_SAVE_LDS_W32:
648 s_mov_b32 s3, 128
649 s_nop 0
650 s_nop 0
651 s_nop 0
652L_SAVE_LDS_LOOP_W32:
653 ds_read_b32 v1, v0
654 s_waitcnt 0
655 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
656
657 s_add_u32 m0, m0, s3 //every buffer_store_lds does 256 bytes
658 s_add_u32 s_save_mem_offset, s_save_mem_offset, s3
659 v_add_nc_u32 v0, v0, 128 //mem offset increased by 128 bytes
660 s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0
661 s_cbranch_scc1 L_SAVE_LDS_LOOP_W32 //LDS save is complete?
662
663 s_branch L_SAVE_LDS_DONE
664
665L_SAVE_LDS_W64:
666 s_mov_b32 s3, 256
667 s_nop 0
668 s_nop 0
669 s_nop 0
670L_SAVE_LDS_LOOP_W64:
671 ds_read_b32 v1, v0
672 s_waitcnt 0
673 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
674
675 s_add_u32 m0, m0, s3 //every buffer_store_lds does 256 bytes
676 s_add_u32 s_save_mem_offset, s_save_mem_offset, s3
677 v_add_nc_u32 v0, v0, 256 //mem offset increased by 256 bytes
678 s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0
679 s_cbranch_scc1 L_SAVE_LDS_LOOP_W64 //LDS save is complete?
680
681L_SAVE_LDS_DONE:
682 /* save VGPRs - set the Rest VGPRs */
683L_SAVE_VGPR:
684 // VGPR SR memory offset: 0
685 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
686 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
687 s_and_b32 m0, m0, 1
688 s_cmp_eq_u32 m0, 1
689 s_cbranch_scc1 L_ENABLE_SAVE_VGPR_EXEC_HI
690 s_mov_b32 s_save_mem_offset, (0+128*4) // for the rest VGPRs
691 s_mov_b32 exec_hi, 0x00000000
692 s_branch L_SAVE_VGPR_NORMAL
693L_ENABLE_SAVE_VGPR_EXEC_HI:
694 s_mov_b32 s_save_mem_offset, (0+256*4) // for the rest VGPRs
695 s_mov_b32 exec_hi, 0xFFFFFFFF
696L_SAVE_VGPR_NORMAL:
697 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)
698 s_add_u32 s_save_alloc_size, s_save_alloc_size, 1
699 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value)
700 //determine it is wave32 or wave64
701 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
702 s_and_b32 m0, m0, 1
703 s_cmp_eq_u32 m0, 1
704 s_cbranch_scc1 L_SAVE_VGPR_WAVE64
705
706 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
707
708 // VGPR Allocated in 4-GPR granularity
709
710 // VGPR store using dw burst
711 s_mov_b32 m0, 0x4 //VGPR initial index value =4
712 s_cmp_lt_u32 m0, s_save_alloc_size
713 s_cbranch_scc0 L_SAVE_VGPR_END
714
715L_SAVE_VGPR_W32_LOOP:
716 v_movrels_b32 v0, v0 //v0 = v[0+m0]
717 v_movrels_b32 v1, v1 //v1 = v[1+m0]
718 v_movrels_b32 v2, v2 //v2 = v[2+m0]
719 v_movrels_b32 v3, v3 //v3 = v[3+m0]
720
721 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
722 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128
723 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*2
724 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*3
725
726 s_add_u32 m0, m0, 4 //next vgpr index
727 s_add_u32 s_save_mem_offset, s_save_mem_offset, 128*4 //every buffer_store_dword does 128 bytes
728 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
729 s_cbranch_scc1 L_SAVE_VGPR_W32_LOOP //VGPR save is complete?
730
731 s_branch L_SAVE_VGPR_END
732
733L_SAVE_VGPR_WAVE64:
734 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
735
736 // VGPR store using dw burst
737 s_mov_b32 m0, 0x4 //VGPR initial index value =4
738 s_cmp_lt_u32 m0, s_save_alloc_size
739 s_cbranch_scc0 L_SAVE_SHARED_VGPR
740
741L_SAVE_VGPR_W64_LOOP:
742 v_movrels_b32 v0, v0 //v0 = v[0+m0]
743 v_movrels_b32 v1, v1 //v1 = v[1+m0]
744 v_movrels_b32 v2, v2 //v2 = v[2+m0]
745 v_movrels_b32 v3, v3 //v3 = v[3+m0]
746
747 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
748 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
749 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
750 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
751
752 s_add_u32 m0, m0, 4 //next vgpr index
753 s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4 //every buffer_store_dword does 256 bytes
754 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
755 s_cbranch_scc1 L_SAVE_VGPR_W64_LOOP //VGPR save is complete?
756
757L_SAVE_SHARED_VGPR:
758 //Below part will be the save shared vgpr part (new for gfx10)
759 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE)
760 s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero?
761 s_cbranch_scc0 L_SAVE_VGPR_END //no shared_vgpr used? jump to L_SAVE_LDS
762 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value)
763 //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count.
764 //save shared_vgpr will start from the index of m0
765 s_add_u32 s_save_alloc_size, s_save_alloc_size, m0
766 s_mov_b32 exec_lo, 0xFFFFFFFF
767 s_mov_b32 exec_hi, 0x00000000
768L_SAVE_SHARED_VGPR_WAVE64_LOOP:
769 v_movrels_b32 v0, v0 //v0 = v[0+m0]
770 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
771 s_add_u32 m0, m0, 1 //next vgpr index
772 s_add_u32 s_save_mem_offset, s_save_mem_offset, 128
773 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
774 s_cbranch_scc1 L_SAVE_SHARED_VGPR_WAVE64_LOOP //SHARED_VGPR save is complete?
775
776L_SAVE_VGPR_END:
777 s_branch L_END_PGM
778
779L_RESTORE:
780 /* Setup Resource Contants */
781 s_mov_b32 s_restore_buf_rsrc0, s_restore_spi_init_lo //base_addr_lo
782 s_and_b32 s_restore_buf_rsrc1, s_restore_spi_init_hi, 0x0000FFFF //base_addr_hi
783 s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE
784 s_mov_b32 s_restore_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes)
785 s_mov_b32 s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC
786
787 //determine it is wave32 or wave64
788 get_wave_size(s_restore_size)
789
790 s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
791 s_cbranch_scc0 L_RESTORE_VGPR
792
793 /* restore LDS */
794L_RESTORE_LDS:
795 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
796 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE
797 s_and_b32 m0, m0, 1
798 s_cmp_eq_u32 m0, 1
799 s_cbranch_scc1 L_ENABLE_RESTORE_LDS_EXEC_HI
800 s_mov_b32 exec_hi, 0x00000000
801 s_branch L_RESTORE_LDS_NORMAL
802L_ENABLE_RESTORE_LDS_EXEC_HI:
803 s_mov_b32 exec_hi, 0xFFFFFFFF
804L_RESTORE_LDS_NORMAL:
805 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)
806 s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //lds_size is zero?
807 s_cbranch_scc0 L_RESTORE_VGPR //no lds used? jump to L_RESTORE_VGPR
808 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
809 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //LDS size in bytes
810 s_mov_b32 s_restore_buf_rsrc2, s_restore_alloc_size //NUM_RECORDS in bytes
811
812 // LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG)
813 //
814 get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size)
815 get_svgpr_size_bytes(s_restore_tmp)
816 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
817 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes()
818 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes()
819
820 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
821
822 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE
823 s_and_b32 m0, m0, 1
824 s_cmp_eq_u32 m0, 1
825 s_mov_b32 m0, 0x0
826 s_cbranch_scc1 L_RESTORE_LDS_LOOP_W64
827
828L_RESTORE_LDS_LOOP_W32:
829#if HAVE_BUFFER_LDS_LOAD
830 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 // first 64DW
831#else
832 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset
833 s_waitcnt vmcnt(0)
834 ds_store_addtid_b32 v0
835#endif
836 s_add_u32 m0, m0, 128 // 128 DW
837 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128 //mem offset increased by 128DW
838 s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0
839 s_cbranch_scc1 L_RESTORE_LDS_LOOP_W32 //LDS restore is complete?
840 s_branch L_RESTORE_VGPR
841
842L_RESTORE_LDS_LOOP_W64:
843#if HAVE_BUFFER_LDS_LOAD
844 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 // first 64DW
845#else
846 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset
847 s_waitcnt vmcnt(0)
848 ds_store_addtid_b32 v0
849#endif
850 s_add_u32 m0, m0, 256 // 256 DW
851 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256 //mem offset increased by 256DW
852 s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0
853 s_cbranch_scc1 L_RESTORE_LDS_LOOP_W64 //LDS restore is complete?
854
855 /* restore VGPRs */
856L_RESTORE_VGPR:
857 // VGPR SR memory offset : 0
858 s_mov_b32 s_restore_mem_offset, 0x0
859 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
860 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE
861 s_and_b32 m0, m0, 1
862 s_cmp_eq_u32 m0, 1
863 s_cbranch_scc1 L_ENABLE_RESTORE_VGPR_EXEC_HI
864 s_mov_b32 exec_hi, 0x00000000
865 s_branch L_RESTORE_VGPR_NORMAL
866L_ENABLE_RESTORE_VGPR_EXEC_HI:
867 s_mov_b32 exec_hi, 0xFFFFFFFF
868L_RESTORE_VGPR_NORMAL:
869 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)
870 s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1
871 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value)
872 //determine it is wave32 or wave64
873 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE
874 s_and_b32 m0, m0, 1
875 s_cmp_eq_u32 m0, 1
876 s_cbranch_scc1 L_RESTORE_VGPR_WAVE64
877
878 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
879
880 // VGPR load using dw burst
881 s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v1, v0 will be the last
882 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128*4
883 s_mov_b32 m0, 4 //VGPR initial index value = 4
884 s_cmp_lt_u32 m0, s_restore_alloc_size
885 s_cbranch_scc0 L_RESTORE_SGPR
886
887L_RESTORE_VGPR_WAVE32_LOOP:
888 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
889 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:128
890 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:128*2
891 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:128*3
892 s_waitcnt vmcnt(0)
893 v_movreld_b32 v0, v0 //v[0+m0] = v0
894 v_movreld_b32 v1, v1
895 v_movreld_b32 v2, v2
896 v_movreld_b32 v3, v3
897 s_add_u32 m0, m0, 4 //next vgpr index
898 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128*4 //every buffer_load_dword does 128 bytes
899 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
900 s_cbranch_scc1 L_RESTORE_VGPR_WAVE32_LOOP //VGPR restore (except v0) is complete?
901
902 /* VGPR restore on v0 */
903 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
904 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:128
905 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:128*2
906 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:128*3
907 s_waitcnt vmcnt(0)
908
909 s_branch L_RESTORE_SGPR
910
911L_RESTORE_VGPR_WAVE64:
912 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
913
914 // VGPR load using dw burst
915 s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v4, v0 will be the last
916 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4
917 s_mov_b32 m0, 4 //VGPR initial index value = 4
918 s_cmp_lt_u32 m0, s_restore_alloc_size
919 s_cbranch_scc0 L_RESTORE_SHARED_VGPR
920
921L_RESTORE_VGPR_WAVE64_LOOP:
922 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
923 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256
924 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*2
925 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*3
926 s_waitcnt vmcnt(0)
927 v_movreld_b32 v0, v0 //v[0+m0] = v0
928 v_movreld_b32 v1, v1
929 v_movreld_b32 v2, v2
930 v_movreld_b32 v3, v3
931 s_add_u32 m0, m0, 4 //next vgpr index
932 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4 //every buffer_load_dword does 256 bytes
933 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
934 s_cbranch_scc1 L_RESTORE_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete?
935
936L_RESTORE_SHARED_VGPR:
937 //Below part will be the restore shared vgpr part (new for gfx10)
938 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) //shared_vgpr_size
939 s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero?
940 s_cbranch_scc0 L_RESTORE_V0 //no shared_vgpr used?
941 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value)
942 //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count.
943 //restore shared_vgpr will start from the index of m0
944 s_add_u32 s_restore_alloc_size, s_restore_alloc_size, m0
945 s_mov_b32 exec_lo, 0xFFFFFFFF
946 s_mov_b32 exec_hi, 0x00000000
947L_RESTORE_SHARED_VGPR_WAVE64_LOOP:
948 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
949 s_waitcnt vmcnt(0)
950 v_movreld_b32 v0, v0 //v[0+m0] = v0
951 s_add_u32 m0, m0, 1 //next vgpr index
952 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128
953 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
954 s_cbranch_scc1 L_RESTORE_SHARED_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete?
955
956 s_mov_b32 exec_hi, 0xFFFFFFFF //restore back exec_hi before restoring V0!!
957
958 /* VGPR restore on v0 */
959L_RESTORE_V0:
960 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
961 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256
962 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*2
963 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*3
964 s_waitcnt vmcnt(0)
965
966 /* restore SGPRs */
967 //will be 2+8+16*6
968 // SGPR SR memory offset : size(VGPR)+size(SVGPR)
969L_RESTORE_SGPR:
970 get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size)
971 get_svgpr_size_bytes(s_restore_tmp)
972 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
973 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes()
974 s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 20*4 //s108~s127 is not saved
975
976 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
977
978 s_mov_b32 m0, s_sgpr_save_num
979
980 read_4sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)
981 s_waitcnt lgkmcnt(0)
982
983 s_sub_u32 m0, m0, 4 // Restore from S[0] to S[104]
984 s_nop 0 // hazard SALU M0=> S_MOVREL
985
986 s_movreld_b64 s0, s0 //s[0+m0] = s0
987 s_movreld_b64 s2, s2
988
989 read_8sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)
990 s_waitcnt lgkmcnt(0)
991
992 s_sub_u32 m0, m0, 8 // Restore from S[0] to S[96]
993 s_nop 0 // hazard SALU M0=> S_MOVREL
994
995 s_movreld_b64 s0, s0 //s[0+m0] = s0
996 s_movreld_b64 s2, s2
997 s_movreld_b64 s4, s4
998 s_movreld_b64 s6, s6
999
1000 L_RESTORE_SGPR_LOOP:
1001 read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)
1002 s_waitcnt lgkmcnt(0)
1003
1004 s_sub_u32 m0, m0, 16 // Restore from S[n] to S[0]
1005 s_nop 0 // hazard SALU M0=> S_MOVREL
1006
1007 s_movreld_b64 s0, s0 //s[0+m0] = s0
1008 s_movreld_b64 s2, s2
1009 s_movreld_b64 s4, s4
1010 s_movreld_b64 s6, s6
1011 s_movreld_b64 s8, s8
1012 s_movreld_b64 s10, s10
1013 s_movreld_b64 s12, s12
1014 s_movreld_b64 s14, s14
1015
1016 s_cmp_eq_u32 m0, 0 //scc = (m0 < s_sgpr_save_num) ? 1 : 0
1017 s_cbranch_scc0 L_RESTORE_SGPR_LOOP
1018
1019 // s_barrier with MODE.DEBUG_EN=1, STATUS.PRIV=1 incorrectly asserts debug exception.
1020 // Clear DEBUG_EN before and restore MODE after the barrier.
1021 s_setreg_imm32_b32 hwreg(HW_REG_MODE), 0
1022 s_barrier //barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG
1023
1024 /* restore HW registers */
1025L_RESTORE_HWREG:
1026 // HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)
1027 get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size)
1028 get_svgpr_size_bytes(s_restore_tmp)
1029 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
1030 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes()
1031
1032 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
1033
1034 read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset)
1035 read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset)
1036 read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
1037 read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset)
1038 read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
1039 read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset)
1040 read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset)
1041 read_hwreg_from_mem(s_restore_xnack_mask, s_restore_buf_rsrc0, s_restore_mem_offset)
1042 read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset)
1043 read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset)
1044 s_waitcnt lgkmcnt(0)
1045
1046 s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO), s_restore_flat_scratch
1047
1048 read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset)
1049 s_waitcnt lgkmcnt(0) //from now on, it is safe to restore STATUS and IB_STS
1050
1051 s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI), s_restore_flat_scratch
1052
1053 s_mov_b32 m0, s_restore_m0
1054 s_mov_b32 exec_lo, s_restore_exec_lo
1055 s_mov_b32 exec_hi, s_restore_exec_hi
1056
1057 s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
1058 s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
1059
1060#if HAVE_XNACK
1061 s_setreg_b32 hwreg(HW_REG_SHADER_XNACK_MASK), s_restore_xnack_mask
1062#endif
1063
1064 s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
1065 s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
1066 s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
1067 s_setreg_b32 hwreg(HW_REG_MODE), s_restore_mode
1068
1069 // Restore trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
1070 // ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40
1071 get_vgpr_size_bytes(s_restore_ttmps_lo, s_restore_size)
1072 get_svgpr_size_bytes(s_restore_ttmps_hi)
1073 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_ttmps_hi
1074 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, get_sgpr_size_bytes()
1075 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_buf_rsrc0
1076 s_addc_u32 s_restore_ttmps_hi, s_restore_buf_rsrc1, 0x0
1077 s_and_b32 s_restore_ttmps_hi, s_restore_ttmps_hi, 0xFFFF
1078 s_load_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x50 glc:1
1079 s_load_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x60 glc:1
1080 s_load_dword ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x74 glc:1
1081 s_waitcnt lgkmcnt(0)
1082
1083#if HAVE_XNACK
1084 restore_ib_sts(s_restore_tmp, s_restore_m0)
1085#endif
1086
1087 s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS
1088 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
1089 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
1090
1091#if SW_SA_TRAP
1092 // If traps are enabled then return to the shader with PRIV=0.
1093 // Otherwise retain PRIV=1 for subsequent context save requests.
1094 s_getreg_b32 s_restore_tmp, hwreg(HW_REG_STATUS)
1095 s_bitcmp1_b32 s_restore_tmp, SQ_WAVE_STATUS_TRAP_EN_SHIFT
1096 s_cbranch_scc1 L_RETURN_WITHOUT_PRIV
1097
1098 s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu
1099 s_setpc_b64 [s_restore_pc_lo, s_restore_pc_hi]
1100L_RETURN_WITHOUT_PRIV:
1101#endif
1102
1103 s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu
1104 s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution
1105
1106L_END_PGM:
1107 s_endpgm_saved
1108end
1109
1110function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)
1111#if NO_SQC_STORE
1112 // Copy into VGPR for later TCP store.
1113 v_writelane_b32 v2, s, m0
1114 s_add_u32 m0, m0, 0x1
1115#else
1116 s_mov_b32 exec_lo, m0
1117 s_mov_b32 m0, s_mem_offset
1118 s_buffer_store_dword s, s_rsrc, m0 glc:1
1119 s_add_u32 s_mem_offset, s_mem_offset, 4
1120 s_mov_b32 m0, exec_lo
1121#endif
1122end
1123
1124
1125function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)
1126#if NO_SQC_STORE
1127 // Copy into VGPR for later TCP store.
1128 for var sgpr_idx = 0; sgpr_idx < 16; sgpr_idx ++
1129 v_writelane_b32 v2, s[sgpr_idx], ttmp13
1130 s_add_u32 ttmp13, ttmp13, 0x1
1131 end
1132#else
1133 s_buffer_store_dwordx4 s[0], s_rsrc, 0 glc:1
1134 s_buffer_store_dwordx4 s[4], s_rsrc, 16 glc:1
1135 s_buffer_store_dwordx4 s[8], s_rsrc, 32 glc:1
1136 s_buffer_store_dwordx4 s[12], s_rsrc, 48 glc:1
1137 s_add_u32 s_rsrc[0], s_rsrc[0], 4*16
1138 s_addc_u32 s_rsrc[1], s_rsrc[1], 0x0
1139#endif
1140end
1141
1142function write_12sgpr_to_mem(s, s_rsrc, s_mem_offset)
1143#if NO_SQC_STORE
1144 // Copy into VGPR for later TCP store.
1145 for var sgpr_idx = 0; sgpr_idx < 12; sgpr_idx ++
1146 v_writelane_b32 v2, s[sgpr_idx], ttmp13
1147 s_add_u32 ttmp13, ttmp13, 0x1
1148 end
1149#else
1150 s_buffer_store_dwordx4 s[0], s_rsrc, 0 glc:1
1151 s_buffer_store_dwordx4 s[4], s_rsrc, 16 glc:1
1152 s_buffer_store_dwordx4 s[8], s_rsrc, 32 glc:1
1153 s_add_u32 s_rsrc[0], s_rsrc[0], 4*12
1154 s_addc_u32 s_rsrc[1], s_rsrc[1], 0x0
1155#endif
1156end
1157
1158function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
1159 s_buffer_load_dword s, s_rsrc, s_mem_offset glc:1
1160 s_add_u32 s_mem_offset, s_mem_offset, 4
1161end
1162
1163function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)
1164 s_sub_u32 s_mem_offset, s_mem_offset, 4*16
1165 s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset glc:1
1166end
1167
1168function read_8sgpr_from_mem(s, s_rsrc, s_mem_offset)
1169 s_sub_u32 s_mem_offset, s_mem_offset, 4*8
1170 s_buffer_load_dwordx8 s, s_rsrc, s_mem_offset glc:1
1171end
1172
1173function read_4sgpr_from_mem(s, s_rsrc, s_mem_offset)
1174 s_sub_u32 s_mem_offset, s_mem_offset, 4*4
1175 s_buffer_load_dwordx4 s, s_rsrc, s_mem_offset glc:1
1176end
1177
1178
1179function get_lds_size_bytes(s_lds_size_byte)
1180 s_getreg_b32 s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)
1181 s_lshl_b32 s_lds_size_byte, s_lds_size_byte, 8 //LDS size in dwords = lds_size * 64 *4Bytes // granularity 64DW
1182end
1183
1184function get_vgpr_size_bytes(s_vgpr_size_byte, s_size)
1185 s_getreg_b32 s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)
1186 s_add_u32 s_vgpr_size_byte, s_vgpr_size_byte, 1
1187 s_bitcmp1_b32 s_size, S_WAVE_SIZE
1188 s_cbranch_scc1 L_ENABLE_SHIFT_W64
1189 s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, (2+7) //Number of VGPRs = (vgpr_size + 1) * 4 * 32 * 4 (non-zero value)
1190 s_branch L_SHIFT_DONE
1191L_ENABLE_SHIFT_W64:
1192 s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4 (non-zero value)
1193L_SHIFT_DONE:
1194end
1195
1196function get_svgpr_size_bytes(s_svgpr_size_byte)
1197 s_getreg_b32 s_svgpr_size_byte, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE)
1198 s_lshl_b32 s_svgpr_size_byte, s_svgpr_size_byte, (3+7)
1199end
1200
1201function get_sgpr_size_bytes
1202 return 512
1203end
1204
1205function get_hwreg_size_bytes
1206 return 128
1207end
1208
1209function get_wave_size(s_reg)
1210 s_getreg_b32 s_reg, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE)
1211 s_lshl_b32 s_reg, s_reg, S_WAVE_SIZE
1212end
1213
1214function save_and_clear_ib_sts(tmp1, tmp2)
1215 // Preserve and clear scalar XNACK state before issuing scalar loads.
1216 // Save IB_STS.REPLAY_W64H[25], RCNT[21:16], FIRST_REPLAY[15] into
1217 // unused space ttmp11[31:24].
1218 s_andn2_b32 ttmp11, ttmp11, (TTMP11_SAVE_REPLAY_W64H_MASK | TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK)
1219 s_getreg_b32 tmp1, hwreg(HW_REG_IB_STS)
1220 s_and_b32 tmp2, tmp1, SQ_WAVE_IB_STS_REPLAY_W64H_MASK
1221 s_lshl_b32 tmp2, tmp2, (TTMP11_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT)
1222 s_or_b32 ttmp11, ttmp11, tmp2
1223 s_and_b32 tmp2, tmp1, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
1224 s_lshl_b32 tmp2, tmp2, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
1225 s_or_b32 ttmp11, ttmp11, tmp2
1226 s_andn2_b32 tmp1, tmp1, (SQ_WAVE_IB_STS_REPLAY_W64H_MASK | SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK)
1227 s_setreg_b32 hwreg(HW_REG_IB_STS), tmp1
1228end
1229
1230function restore_ib_sts(tmp1, tmp2)
1231 s_lshr_b32 tmp1, ttmp11, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
1232 s_and_b32 tmp2, tmp1, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
1233 s_lshr_b32 tmp1, ttmp11, (TTMP11_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT)
1234 s_and_b32 tmp1, tmp1, SQ_WAVE_IB_STS_REPLAY_W64H_MASK
1235 s_or_b32 tmp1, tmp1, tmp2
1236 s_setreg_b32 hwreg(HW_REG_IB_STS), tmp1
1237end