Loading...
Note: File does not exist in v4.6.
1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23/* To compile this assembly code:
24 *
25 * Navi1x:
26 * cpp -DASIC_FAMILY=CHIP_NAVI10 cwsr_trap_handler_gfx10.asm -P -o nv1x.sp3
27 * sp3 nv1x.sp3 -hex nv1x.hex
28 *
29 * gfx10:
30 * cpp -DASIC_FAMILY=CHIP_SIENNA_CICHLID cwsr_trap_handler_gfx10.asm -P -o gfx10.sp3
31 * sp3 gfx10.sp3 -hex gfx10.hex
32 *
33 * gfx11:
34 * cpp -DASIC_FAMILY=CHIP_PLUM_BONITO cwsr_trap_handler_gfx10.asm -P -o gfx11.sp3
35 * sp3 gfx11.sp3 -hex gfx11.hex
36 */
37
38#define CHIP_NAVI10 26
39#define CHIP_SIENNA_CICHLID 30
40#define CHIP_PLUM_BONITO 36
41
42#define NO_SQC_STORE (ASIC_FAMILY >= CHIP_SIENNA_CICHLID)
43#define HAVE_XNACK (ASIC_FAMILY < CHIP_SIENNA_CICHLID)
44#define HAVE_SENDMSG_RTN (ASIC_FAMILY >= CHIP_PLUM_BONITO)
45#define HAVE_BUFFER_LDS_LOAD (ASIC_FAMILY < CHIP_PLUM_BONITO)
46#define SW_SA_TRAP (ASIC_FAMILY >= CHIP_PLUM_BONITO)
47
48var SINGLE_STEP_MISSED_WORKAROUND = 1 //workaround for lost MODE.DEBUG_EN exception when SAVECTX raised
49
50var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006
51var SQ_WAVE_STATUS_HALT_MASK = 0x2000
52var SQ_WAVE_STATUS_ECC_ERR_MASK = 0x20000
53var SQ_WAVE_STATUS_TRAP_EN_SHIFT = 6
54
55var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12
56var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9
57var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 8
58var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT = 24
59var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE = 4
60var SQ_WAVE_IB_STS2_WAVE64_SHIFT = 11
61var SQ_WAVE_IB_STS2_WAVE64_SIZE = 1
62
63#if ASIC_FAMILY < CHIP_PLUM_BONITO
64var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 8
65#else
66var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 12
67#endif
68
69var SQ_WAVE_TRAPSTS_SAVECTX_MASK = 0x400
70var SQ_WAVE_TRAPSTS_EXCP_MASK = 0x1FF
71var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT = 10
72var SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK = 0x80
73var SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT = 7
74var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK = 0x100
75var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT = 8
76var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK = 0x3FF
77var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT = 0x0
78var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE = 10
79var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK = 0xFFFFF800
80var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT = 11
81var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE = 21
82var SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK = 0x800
83var SQ_WAVE_TRAPSTS_EXCP_HI_MASK = 0x7000
84
85var SQ_WAVE_MODE_EXCP_EN_SHIFT = 12
86var SQ_WAVE_MODE_EXCP_EN_ADDR_WATCH_SHIFT = 19
87
88var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT = 15
89var SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT = 25
90var SQ_WAVE_IB_STS_REPLAY_W64H_MASK = 0x02000000
91var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK = 0x003F8000
92
93var SQ_WAVE_MODE_DEBUG_EN_MASK = 0x800
94
95// bits [31:24] unused by SPI debug data
96var TTMP11_SAVE_REPLAY_W64H_SHIFT = 31
97var TTMP11_SAVE_REPLAY_W64H_MASK = 0x80000000
98var TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT = 24
99var TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK = 0x7F000000
100var TTMP11_DEBUG_TRAP_ENABLED_SHIFT = 23
101var TTMP11_DEBUG_TRAP_ENABLED_MASK = 0x800000
102
103// SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14]
104// when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
105var S_SAVE_BUF_RSRC_WORD1_STRIDE = 0x00040000
106var S_SAVE_BUF_RSRC_WORD3_MISC = 0x10807FAC
107var S_SAVE_PC_HI_TRAP_ID_MASK = 0x00FF0000
108var S_SAVE_PC_HI_HT_MASK = 0x01000000
109var S_SAVE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000
110var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26
111
112var S_SAVE_PC_HI_FIRST_WAVE_MASK = 0x80000000
113var S_SAVE_PC_HI_FIRST_WAVE_SHIFT = 31
114
115var s_sgpr_save_num = 108
116
117var s_save_spi_init_lo = exec_lo
118var s_save_spi_init_hi = exec_hi
119var s_save_pc_lo = ttmp0
120var s_save_pc_hi = ttmp1
121var s_save_exec_lo = ttmp2
122var s_save_exec_hi = ttmp3
123var s_save_status = ttmp12
124var s_save_trapsts = ttmp15
125var s_save_xnack_mask = s_save_trapsts
126var s_wave_size = ttmp7
127var s_save_buf_rsrc0 = ttmp8
128var s_save_buf_rsrc1 = ttmp9
129var s_save_buf_rsrc2 = ttmp10
130var s_save_buf_rsrc3 = ttmp11
131var s_save_mem_offset = ttmp4
132var s_save_alloc_size = s_save_trapsts
133var s_save_tmp = ttmp14
134var s_save_m0 = ttmp5
135var s_save_ttmps_lo = s_save_tmp
136var s_save_ttmps_hi = s_save_trapsts
137
138var S_RESTORE_BUF_RSRC_WORD1_STRIDE = S_SAVE_BUF_RSRC_WORD1_STRIDE
139var S_RESTORE_BUF_RSRC_WORD3_MISC = S_SAVE_BUF_RSRC_WORD3_MISC
140
141var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000
142var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT = 26
143var S_WAVE_SIZE = 25
144
145var s_restore_spi_init_lo = exec_lo
146var s_restore_spi_init_hi = exec_hi
147var s_restore_mem_offset = ttmp12
148var s_restore_alloc_size = ttmp3
149var s_restore_tmp = ttmp2
150var s_restore_mem_offset_save = s_restore_tmp
151var s_restore_m0 = s_restore_alloc_size
152var s_restore_mode = ttmp7
153var s_restore_flat_scratch = s_restore_tmp
154var s_restore_pc_lo = ttmp0
155var s_restore_pc_hi = ttmp1
156var s_restore_exec_lo = ttmp4
157var s_restore_exec_hi = ttmp5
158var s_restore_status = ttmp14
159var s_restore_trapsts = ttmp15
160var s_restore_xnack_mask = ttmp13
161var s_restore_buf_rsrc0 = ttmp8
162var s_restore_buf_rsrc1 = ttmp9
163var s_restore_buf_rsrc2 = ttmp10
164var s_restore_buf_rsrc3 = ttmp11
165var s_restore_size = ttmp6
166var s_restore_ttmps_lo = s_restore_tmp
167var s_restore_ttmps_hi = s_restore_alloc_size
168
169shader main
170 asic(DEFAULT)
171 type(CS)
172 wave_size(32)
173
174 s_branch L_SKIP_RESTORE //NOT restore. might be a regular trap or save
175
176L_JUMP_TO_RESTORE:
177 s_branch L_RESTORE
178
179L_SKIP_RESTORE:
180 s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC
181
182 // Clear SPI_PRIO: do not save with elevated priority.
183 // Clear ECC_ERR: prevents SQC store and triggers FATAL_HALT if setreg'd.
184 s_andn2_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK|SQ_WAVE_STATUS_ECC_ERR_MASK
185
186 s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
187
188#if SW_SA_TRAP
189 // If ttmp1[30] is set then issue s_barrier to unblock dependent waves.
190 s_bitcmp1_b32 s_save_pc_hi, 30
191 s_cbranch_scc0 L_TRAP_NO_BARRIER
192 s_barrier
193
194L_TRAP_NO_BARRIER:
195 // If ttmp1[31] is set then trap may occur early.
196 // Spin wait until SAVECTX exception is raised.
197 s_bitcmp1_b32 s_save_pc_hi, 31
198 s_cbranch_scc1 L_CHECK_SAVE
199#endif
200
201 s_and_b32 ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK
202 s_cbranch_scc0 L_NOT_HALTED
203
204L_HALTED:
205 // Host trap may occur while wave is halted.
206 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
207 s_cbranch_scc1 L_FETCH_2ND_TRAP
208
209L_CHECK_SAVE:
210 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK
211 s_cbranch_scc1 L_SAVE
212
213 // Wave is halted but neither host trap nor SAVECTX is raised.
214 // Caused by instruction fetch memory violation.
215 // Spin wait until context saved to prevent interrupt storm.
216 s_sleep 0x10
217 s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
218 s_branch L_CHECK_SAVE
219
220L_NOT_HALTED:
221 // Let second-level handle non-SAVECTX exception or trap.
222 // Any concurrent SAVECTX will be handled upon re-entry once halted.
223
224 // Check non-maskable exceptions. memory_violation, illegal_instruction
225 // and xnack_error exceptions always cause the wave to enter the trap
226 // handler.
227 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK|SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK
228 s_cbranch_scc1 L_FETCH_2ND_TRAP
229
230 // Check for maskable exceptions in trapsts.excp and trapsts.excp_hi.
231 // Maskable exceptions only cause the wave to enter the trap handler if
232 // their respective bit in mode.excp_en is set.
233 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCP_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK
234 s_cbranch_scc0 L_CHECK_TRAP_ID
235
236 s_and_b32 ttmp3, s_save_trapsts, SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK
237 s_cbranch_scc0 L_NOT_ADDR_WATCH
238 s_bitset1_b32 ttmp2, SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT // Check all addr_watch[123] exceptions against excp_en.addr_watch
239
240L_NOT_ADDR_WATCH:
241 s_getreg_b32 ttmp3, hwreg(HW_REG_MODE)
242 s_lshl_b32 ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT
243 s_and_b32 ttmp2, ttmp2, ttmp3
244 s_cbranch_scc1 L_FETCH_2ND_TRAP
245
246L_CHECK_TRAP_ID:
247 // Check trap_id != 0
248 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
249 s_cbranch_scc1 L_FETCH_2ND_TRAP
250
251if SINGLE_STEP_MISSED_WORKAROUND
252 // Prioritize single step exception over context save.
253 // Second-level trap will halt wave and RFE, re-entering for SAVECTX.
254 s_getreg_b32 ttmp2, hwreg(HW_REG_MODE)
255 s_and_b32 ttmp2, ttmp2, SQ_WAVE_MODE_DEBUG_EN_MASK
256 s_cbranch_scc1 L_FETCH_2ND_TRAP
257end
258
259 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK
260 s_cbranch_scc1 L_SAVE
261
262L_FETCH_2ND_TRAP:
263#if HAVE_XNACK
264 save_and_clear_ib_sts(ttmp14, ttmp15)
265#endif
266
267 // Read second-level TBA/TMA from first-level TMA and jump if available.
268 // ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data)
269 // ttmp12 holds SQ_WAVE_STATUS
270#if HAVE_SENDMSG_RTN
271 s_sendmsg_rtn_b64 [ttmp14, ttmp15], sendmsg(MSG_RTN_GET_TMA)
272 s_waitcnt lgkmcnt(0)
273#else
274 s_getreg_b32 ttmp14, hwreg(HW_REG_SHADER_TMA_LO)
275 s_getreg_b32 ttmp15, hwreg(HW_REG_SHADER_TMA_HI)
276#endif
277 s_lshl_b64 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8
278
279 s_load_dword ttmp2, [ttmp14, ttmp15], 0x10 glc:1 // debug trap enabled flag
280 s_waitcnt lgkmcnt(0)
281 s_lshl_b32 ttmp2, ttmp2, TTMP11_DEBUG_TRAP_ENABLED_SHIFT
282 s_andn2_b32 ttmp11, ttmp11, TTMP11_DEBUG_TRAP_ENABLED_MASK
283 s_or_b32 ttmp11, ttmp11, ttmp2
284
285 s_load_dwordx2 [ttmp2, ttmp3], [ttmp14, ttmp15], 0x0 glc:1 // second-level TBA
286 s_waitcnt lgkmcnt(0)
287 s_load_dwordx2 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 glc:1 // second-level TMA
288 s_waitcnt lgkmcnt(0)
289
290 s_and_b64 [ttmp2, ttmp3], [ttmp2, ttmp3], [ttmp2, ttmp3]
291 s_cbranch_scc0 L_NO_NEXT_TRAP // second-level trap handler not been set
292 s_setpc_b64 [ttmp2, ttmp3] // jump to second-level trap handler
293
294L_NO_NEXT_TRAP:
295 // If not caused by trap then halt wave to prevent re-entry.
296 s_and_b32 ttmp2, s_save_pc_hi, (S_SAVE_PC_HI_TRAP_ID_MASK|S_SAVE_PC_HI_HT_MASK)
297 s_cbranch_scc1 L_TRAP_CASE
298 s_or_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_HALT_MASK
299
300 // If the PC points to S_ENDPGM then context save will fail if STATUS.HALT is set.
301 // Rewind the PC to prevent this from occurring.
302 s_sub_u32 ttmp0, ttmp0, 0x8
303 s_subb_u32 ttmp1, ttmp1, 0x0
304
305 s_branch L_EXIT_TRAP
306
307L_TRAP_CASE:
308 // Host trap will not cause trap re-entry.
309 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_HT_MASK
310 s_cbranch_scc1 L_EXIT_TRAP
311
312 // Advance past trap instruction to prevent re-entry.
313 s_add_u32 ttmp0, ttmp0, 0x4
314 s_addc_u32 ttmp1, ttmp1, 0x0
315
316L_EXIT_TRAP:
317 s_and_b32 ttmp1, ttmp1, 0xFFFF
318
319#if HAVE_XNACK
320 restore_ib_sts(ttmp14, ttmp15)
321#endif
322
323 // Restore SQ_WAVE_STATUS.
324 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
325 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
326 s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status
327
328 s_rfe_b64 [ttmp0, ttmp1]
329
330L_SAVE:
331 s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
332 s_mov_b32 s_save_tmp, 0
333 s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp //clear saveCtx bit
334
335#if HAVE_XNACK
336 save_and_clear_ib_sts(s_save_tmp, s_save_trapsts)
337#endif
338
339 /* inform SPI the readiness and wait for SPI's go signal */
340 s_mov_b32 s_save_exec_lo, exec_lo //save EXEC and use EXEC for the go signal from SPI
341 s_mov_b32 s_save_exec_hi, exec_hi
342 s_mov_b64 exec, 0x0 //clear EXEC to get ready to receive
343
344#if HAVE_SENDMSG_RTN
345 s_sendmsg_rtn_b64 [exec_lo, exec_hi], sendmsg(MSG_RTN_SAVE_WAVE)
346#else
347 s_sendmsg sendmsg(MSG_SAVEWAVE) //send SPI a message and wait for SPI's write to EXEC
348#endif
349
350#if ASIC_FAMILY < CHIP_SIENNA_CICHLID
351L_SLEEP:
352 // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause
353 // SQ hang, since the 7,8th wave could not get arbit to exec inst, while
354 // other waves are stuck into the sleep-loop and waiting for wrexec!=0
355 s_sleep 0x2
356 s_cbranch_execz L_SLEEP
357#else
358 s_waitcnt lgkmcnt(0)
359#endif
360
361 // Save first_wave flag so we can clear high bits of save address.
362 s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK
363 s_lshl_b32 s_save_tmp, s_save_tmp, (S_SAVE_PC_HI_FIRST_WAVE_SHIFT - S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT)
364 s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp
365
366#if NO_SQC_STORE
367 // Trap temporaries must be saved via VGPR but all VGPRs are in use.
368 // There is no ttmp space to hold the resource constant for VGPR save.
369 // Save v0 by itself since it requires only two SGPRs.
370 s_mov_b32 s_save_ttmps_lo, exec_lo
371 s_and_b32 s_save_ttmps_hi, exec_hi, 0xFFFF
372 s_mov_b32 exec_lo, 0xFFFFFFFF
373 s_mov_b32 exec_hi, 0xFFFFFFFF
374 global_store_dword_addtid v0, [s_save_ttmps_lo, s_save_ttmps_hi] slc:1 glc:1
375 v_mov_b32 v0, 0x0
376 s_mov_b32 exec_lo, s_save_ttmps_lo
377 s_mov_b32 exec_hi, s_save_ttmps_hi
378#endif
379
380 // Save trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
381 // ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40
382 get_wave_size(s_save_ttmps_hi)
383 get_vgpr_size_bytes(s_save_ttmps_lo, s_save_ttmps_hi)
384 get_svgpr_size_bytes(s_save_ttmps_hi)
385 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_ttmps_hi
386 s_and_b32 s_save_ttmps_hi, s_save_spi_init_hi, 0xFFFF
387 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, get_sgpr_size_bytes()
388 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_spi_init_lo
389 s_addc_u32 s_save_ttmps_hi, s_save_ttmps_hi, 0x0
390
391#if NO_SQC_STORE
392 v_writelane_b32 v0, ttmp4, 0x4
393 v_writelane_b32 v0, ttmp5, 0x5
394 v_writelane_b32 v0, ttmp6, 0x6
395 v_writelane_b32 v0, ttmp7, 0x7
396 v_writelane_b32 v0, ttmp8, 0x8
397 v_writelane_b32 v0, ttmp9, 0x9
398 v_writelane_b32 v0, ttmp10, 0xA
399 v_writelane_b32 v0, ttmp11, 0xB
400 v_writelane_b32 v0, ttmp13, 0xD
401 v_writelane_b32 v0, exec_lo, 0xE
402 v_writelane_b32 v0, exec_hi, 0xF
403
404 s_mov_b32 exec_lo, 0x3FFF
405 s_mov_b32 exec_hi, 0x0
406 global_store_dword_addtid v0, [s_save_ttmps_lo, s_save_ttmps_hi] inst_offset:0x40 slc:1 glc:1
407 v_readlane_b32 ttmp14, v0, 0xE
408 v_readlane_b32 ttmp15, v0, 0xF
409 s_mov_b32 exec_lo, ttmp14
410 s_mov_b32 exec_hi, ttmp15
411#else
412 s_store_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_save_ttmps_lo, s_save_ttmps_hi], 0x50 glc:1
413 s_store_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_save_ttmps_lo, s_save_ttmps_hi], 0x60 glc:1
414 s_store_dword ttmp13, [s_save_ttmps_lo, s_save_ttmps_hi], 0x74 glc:1
415#endif
416
417 /* setup Resource Contants */
418 s_mov_b32 s_save_buf_rsrc0, s_save_spi_init_lo //base_addr_lo
419 s_and_b32 s_save_buf_rsrc1, s_save_spi_init_hi, 0x0000FFFF //base_addr_hi
420 s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE
421 s_mov_b32 s_save_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
422 s_mov_b32 s_save_buf_rsrc3, S_SAVE_BUF_RSRC_WORD3_MISC
423
424 s_mov_b32 s_save_m0, m0
425
426 /* global mem offset */
427 s_mov_b32 s_save_mem_offset, 0x0
428 get_wave_size(s_wave_size)
429
430#if HAVE_XNACK
431 // Save and clear vector XNACK state late to free up SGPRs.
432 s_getreg_b32 s_save_xnack_mask, hwreg(HW_REG_SHADER_XNACK_MASK)
433 s_setreg_imm32_b32 hwreg(HW_REG_SHADER_XNACK_MASK), 0x0
434#endif
435
436 /* save first 4 VGPRs, needed for SGPR save */
437 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
438 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
439 s_and_b32 m0, m0, 1
440 s_cmp_eq_u32 m0, 1
441 s_cbranch_scc1 L_ENABLE_SAVE_4VGPR_EXEC_HI
442 s_mov_b32 exec_hi, 0x00000000
443 s_branch L_SAVE_4VGPR_WAVE32
444L_ENABLE_SAVE_4VGPR_EXEC_HI:
445 s_mov_b32 exec_hi, 0xFFFFFFFF
446 s_branch L_SAVE_4VGPR_WAVE64
447L_SAVE_4VGPR_WAVE32:
448 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
449
450 // VGPR Allocated in 4-GPR granularity
451
452#if !NO_SQC_STORE
453 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
454#endif
455 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128
456 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*2
457 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*3
458 s_branch L_SAVE_HWREG
459
460L_SAVE_4VGPR_WAVE64:
461 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
462
463 // VGPR Allocated in 4-GPR granularity
464
465#if !NO_SQC_STORE
466 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
467#endif
468 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
469 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
470 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
471
472 /* save HW registers */
473
474L_SAVE_HWREG:
475 // HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)
476 get_vgpr_size_bytes(s_save_mem_offset, s_wave_size)
477 get_svgpr_size_bytes(s_save_tmp)
478 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
479 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes()
480
481 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
482
483#if NO_SQC_STORE
484 v_mov_b32 v0, 0x0 //Offset[31:0] from buffer resource
485 v_mov_b32 v1, 0x0 //Offset[63:32] from buffer resource
486 v_mov_b32 v2, 0x0 //Set of SGPRs for TCP store
487 s_mov_b32 m0, 0x0 //Next lane of v2 to write to
488#endif
489
490 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
491 write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset)
492 s_andn2_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
493 write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)
494 write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset)
495 write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
496 write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset)
497
498 s_getreg_b32 s_save_tmp, hwreg(HW_REG_TRAPSTS)
499 write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)
500
501 // Not used on Sienna_Cichlid but keep layout same for debugger.
502 write_hwreg_to_mem(s_save_xnack_mask, s_save_buf_rsrc0, s_save_mem_offset)
503
504 s_getreg_b32 s_save_m0, hwreg(HW_REG_MODE)
505 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
506
507 s_getreg_b32 s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO)
508 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
509
510 s_getreg_b32 s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI)
511 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
512
513#if NO_SQC_STORE
514 // Write HWREGs with 16 VGPR lanes. TTMPs occupy space after this.
515 s_mov_b32 exec_lo, 0xFFFF
516 s_mov_b32 exec_hi, 0x0
517 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
518
519 // Write SGPRs with 32 VGPR lanes. This works in wave32 and wave64 mode.
520 s_mov_b32 exec_lo, 0xFFFFFFFF
521#endif
522
523 /* save SGPRs */
524 // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...
525
526 // SGPR SR memory offset : size(VGPR)+size(SVGPR)
527 get_vgpr_size_bytes(s_save_mem_offset, s_wave_size)
528 get_svgpr_size_bytes(s_save_tmp)
529 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
530 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
531
532#if NO_SQC_STORE
533 s_mov_b32 ttmp13, 0x0 //next VGPR lane to copy SGPR into
534#else
535 // backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0
536 s_mov_b32 s_save_xnack_mask, s_save_buf_rsrc0
537 s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset
538 s_addc_u32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0
539#endif
540
541 s_mov_b32 m0, 0x0 //SGPR initial index value =0
542 s_nop 0x0 //Manually inserted wait states
543L_SAVE_SGPR_LOOP:
544 // SGPR is allocated in 16 SGPR granularity
545 s_movrels_b64 s0, s0 //s0 = s[0+m0], s1 = s[1+m0]
546 s_movrels_b64 s2, s2 //s2 = s[2+m0], s3 = s[3+m0]
547 s_movrels_b64 s4, s4 //s4 = s[4+m0], s5 = s[5+m0]
548 s_movrels_b64 s6, s6 //s6 = s[6+m0], s7 = s[7+m0]
549 s_movrels_b64 s8, s8 //s8 = s[8+m0], s9 = s[9+m0]
550 s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0]
551 s_movrels_b64 s12, s12 //s12 = s[12+m0], s13 = s[13+m0]
552 s_movrels_b64 s14, s14 //s14 = s[14+m0], s15 = s[15+m0]
553
554 write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset)
555
556#if NO_SQC_STORE
557 s_cmp_eq_u32 ttmp13, 0x20 //have 32 VGPR lanes filled?
558 s_cbranch_scc0 L_SAVE_SGPR_SKIP_TCP_STORE
559
560 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
561 s_add_u32 s_save_mem_offset, s_save_mem_offset, 0x80
562 s_mov_b32 ttmp13, 0x0
563 v_mov_b32 v2, 0x0
564L_SAVE_SGPR_SKIP_TCP_STORE:
565#endif
566
567 s_add_u32 m0, m0, 16 //next sgpr index
568 s_cmp_lt_u32 m0, 96 //scc = (m0 < first 96 SGPR) ? 1 : 0
569 s_cbranch_scc1 L_SAVE_SGPR_LOOP //first 96 SGPR save is complete?
570
571 //save the rest 12 SGPR
572 s_movrels_b64 s0, s0 //s0 = s[0+m0], s1 = s[1+m0]
573 s_movrels_b64 s2, s2 //s2 = s[2+m0], s3 = s[3+m0]
574 s_movrels_b64 s4, s4 //s4 = s[4+m0], s5 = s[5+m0]
575 s_movrels_b64 s6, s6 //s6 = s[6+m0], s7 = s[7+m0]
576 s_movrels_b64 s8, s8 //s8 = s[8+m0], s9 = s[9+m0]
577 s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0]
578 write_12sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset)
579
580#if NO_SQC_STORE
581 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
582#else
583 // restore s_save_buf_rsrc0,1
584 s_mov_b32 s_save_buf_rsrc0, s_save_xnack_mask
585#endif
586
587 /* save LDS */
588
589L_SAVE_LDS:
590 // Change EXEC to all threads...
591 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
592 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
593 s_and_b32 m0, m0, 1
594 s_cmp_eq_u32 m0, 1
595 s_cbranch_scc1 L_ENABLE_SAVE_LDS_EXEC_HI
596 s_mov_b32 exec_hi, 0x00000000
597 s_branch L_SAVE_LDS_NORMAL
598L_ENABLE_SAVE_LDS_EXEC_HI:
599 s_mov_b32 exec_hi, 0xFFFFFFFF
600L_SAVE_LDS_NORMAL:
601 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)
602 s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //lds_size is zero?
603 s_cbranch_scc0 L_SAVE_LDS_DONE //no lds used? jump to L_SAVE_DONE
604
605 s_barrier //LDS is used? wait for other waves in the same TG
606 s_and_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
607 s_cbranch_scc0 L_SAVE_LDS_DONE
608
609 // first wave do LDS save;
610
611 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
612 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //LDS size in bytes
613 s_mov_b32 s_save_buf_rsrc2, s_save_alloc_size //NUM_RECORDS in bytes
614
615 // LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG)
616 //
617 get_vgpr_size_bytes(s_save_mem_offset, s_wave_size)
618 get_svgpr_size_bytes(s_save_tmp)
619 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
620 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes()
621 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()
622
623 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
624
625 //load 0~63*4(byte address) to vgpr v0
626 v_mbcnt_lo_u32_b32 v0, -1, 0
627 v_mbcnt_hi_u32_b32 v0, -1, v0
628 v_mul_u32_u24 v0, 4, v0
629
630 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
631 s_and_b32 m0, m0, 1
632 s_cmp_eq_u32 m0, 1
633 s_mov_b32 m0, 0x0
634 s_cbranch_scc1 L_SAVE_LDS_W64
635
636L_SAVE_LDS_W32:
637 s_mov_b32 s3, 128
638 s_nop 0
639 s_nop 0
640 s_nop 0
641L_SAVE_LDS_LOOP_W32:
642 ds_read_b32 v1, v0
643 s_waitcnt 0
644 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
645
646 s_add_u32 m0, m0, s3 //every buffer_store_lds does 256 bytes
647 s_add_u32 s_save_mem_offset, s_save_mem_offset, s3
648 v_add_nc_u32 v0, v0, 128 //mem offset increased by 128 bytes
649 s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0
650 s_cbranch_scc1 L_SAVE_LDS_LOOP_W32 //LDS save is complete?
651
652 s_branch L_SAVE_LDS_DONE
653
654L_SAVE_LDS_W64:
655 s_mov_b32 s3, 256
656 s_nop 0
657 s_nop 0
658 s_nop 0
659L_SAVE_LDS_LOOP_W64:
660 ds_read_b32 v1, v0
661 s_waitcnt 0
662 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
663
664 s_add_u32 m0, m0, s3 //every buffer_store_lds does 256 bytes
665 s_add_u32 s_save_mem_offset, s_save_mem_offset, s3
666 v_add_nc_u32 v0, v0, 256 //mem offset increased by 256 bytes
667 s_cmp_lt_u32 m0, s_save_alloc_size //scc=(m0 < s_save_alloc_size) ? 1 : 0
668 s_cbranch_scc1 L_SAVE_LDS_LOOP_W64 //LDS save is complete?
669
670L_SAVE_LDS_DONE:
671 /* save VGPRs - set the Rest VGPRs */
672L_SAVE_VGPR:
673 // VGPR SR memory offset: 0
674 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
675 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
676 s_and_b32 m0, m0, 1
677 s_cmp_eq_u32 m0, 1
678 s_cbranch_scc1 L_ENABLE_SAVE_VGPR_EXEC_HI
679 s_mov_b32 s_save_mem_offset, (0+128*4) // for the rest VGPRs
680 s_mov_b32 exec_hi, 0x00000000
681 s_branch L_SAVE_VGPR_NORMAL
682L_ENABLE_SAVE_VGPR_EXEC_HI:
683 s_mov_b32 s_save_mem_offset, (0+256*4) // for the rest VGPRs
684 s_mov_b32 exec_hi, 0xFFFFFFFF
685L_SAVE_VGPR_NORMAL:
686 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)
687 s_add_u32 s_save_alloc_size, s_save_alloc_size, 1
688 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value)
689 //determine it is wave32 or wave64
690 s_lshr_b32 m0, s_wave_size, S_WAVE_SIZE
691 s_and_b32 m0, m0, 1
692 s_cmp_eq_u32 m0, 1
693 s_cbranch_scc1 L_SAVE_VGPR_WAVE64
694
695 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
696
697 // VGPR Allocated in 4-GPR granularity
698
699 // VGPR store using dw burst
700 s_mov_b32 m0, 0x4 //VGPR initial index value =4
701 s_cmp_lt_u32 m0, s_save_alloc_size
702 s_cbranch_scc0 L_SAVE_VGPR_END
703
704L_SAVE_VGPR_W32_LOOP:
705 v_movrels_b32 v0, v0 //v0 = v[0+m0]
706 v_movrels_b32 v1, v1 //v1 = v[1+m0]
707 v_movrels_b32 v2, v2 //v2 = v[2+m0]
708 v_movrels_b32 v3, v3 //v3 = v[3+m0]
709
710 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
711 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128
712 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*2
713 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:128*3
714
715 s_add_u32 m0, m0, 4 //next vgpr index
716 s_add_u32 s_save_mem_offset, s_save_mem_offset, 128*4 //every buffer_store_dword does 128 bytes
717 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
718 s_cbranch_scc1 L_SAVE_VGPR_W32_LOOP //VGPR save is complete?
719
720 s_branch L_SAVE_VGPR_END
721
722L_SAVE_VGPR_WAVE64:
723 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
724
725 // VGPR store using dw burst
726 s_mov_b32 m0, 0x4 //VGPR initial index value =4
727 s_cmp_lt_u32 m0, s_save_alloc_size
728 s_cbranch_scc0 L_SAVE_SHARED_VGPR
729
730L_SAVE_VGPR_W64_LOOP:
731 v_movrels_b32 v0, v0 //v0 = v[0+m0]
732 v_movrels_b32 v1, v1 //v1 = v[1+m0]
733 v_movrels_b32 v2, v2 //v2 = v[2+m0]
734 v_movrels_b32 v3, v3 //v3 = v[3+m0]
735
736 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
737 buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
738 buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
739 buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
740
741 s_add_u32 m0, m0, 4 //next vgpr index
742 s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4 //every buffer_store_dword does 256 bytes
743 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
744 s_cbranch_scc1 L_SAVE_VGPR_W64_LOOP //VGPR save is complete?
745
746L_SAVE_SHARED_VGPR:
747 //Below part will be the save shared vgpr part (new for gfx10)
748 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE)
749 s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero?
750 s_cbranch_scc0 L_SAVE_VGPR_END //no shared_vgpr used? jump to L_SAVE_LDS
751 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value)
752 //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count.
753 //save shared_vgpr will start from the index of m0
754 s_add_u32 s_save_alloc_size, s_save_alloc_size, m0
755 s_mov_b32 exec_lo, 0xFFFFFFFF
756 s_mov_b32 exec_hi, 0x00000000
757L_SAVE_SHARED_VGPR_WAVE64_LOOP:
758 v_movrels_b32 v0, v0 //v0 = v[0+m0]
759 buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
760 s_add_u32 m0, m0, 1 //next vgpr index
761 s_add_u32 s_save_mem_offset, s_save_mem_offset, 128
762 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0
763 s_cbranch_scc1 L_SAVE_SHARED_VGPR_WAVE64_LOOP //SHARED_VGPR save is complete?
764
765L_SAVE_VGPR_END:
766 s_branch L_END_PGM
767
768L_RESTORE:
769 /* Setup Resource Contants */
770 s_mov_b32 s_restore_buf_rsrc0, s_restore_spi_init_lo //base_addr_lo
771 s_and_b32 s_restore_buf_rsrc1, s_restore_spi_init_hi, 0x0000FFFF //base_addr_hi
772 s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE
773 s_mov_b32 s_restore_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes)
774 s_mov_b32 s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC
775
776 //determine it is wave32 or wave64
777 get_wave_size(s_restore_size)
778
779 s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
780 s_cbranch_scc0 L_RESTORE_VGPR
781
782 /* restore LDS */
783L_RESTORE_LDS:
784 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
785 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE
786 s_and_b32 m0, m0, 1
787 s_cmp_eq_u32 m0, 1
788 s_cbranch_scc1 L_ENABLE_RESTORE_LDS_EXEC_HI
789 s_mov_b32 exec_hi, 0x00000000
790 s_branch L_RESTORE_LDS_NORMAL
791L_ENABLE_RESTORE_LDS_EXEC_HI:
792 s_mov_b32 exec_hi, 0xFFFFFFFF
793L_RESTORE_LDS_NORMAL:
794 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)
795 s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //lds_size is zero?
796 s_cbranch_scc0 L_RESTORE_VGPR //no lds used? jump to L_RESTORE_VGPR
797 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 6 //LDS size in dwords = lds_size * 64dw
798 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //LDS size in bytes
799 s_mov_b32 s_restore_buf_rsrc2, s_restore_alloc_size //NUM_RECORDS in bytes
800
801 // LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG)
802 //
803 get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size)
804 get_svgpr_size_bytes(s_restore_tmp)
805 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
806 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes()
807 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes()
808
809 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
810
811 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE
812 s_and_b32 m0, m0, 1
813 s_cmp_eq_u32 m0, 1
814 s_mov_b32 m0, 0x0
815 s_cbranch_scc1 L_RESTORE_LDS_LOOP_W64
816
817L_RESTORE_LDS_LOOP_W32:
818#if HAVE_BUFFER_LDS_LOAD
819 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 // first 64DW
820#else
821 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset
822 s_waitcnt vmcnt(0)
823 ds_store_addtid_b32 v0
824#endif
825 s_add_u32 m0, m0, 128 // 128 DW
826 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128 //mem offset increased by 128DW
827 s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0
828 s_cbranch_scc1 L_RESTORE_LDS_LOOP_W32 //LDS restore is complete?
829 s_branch L_RESTORE_VGPR
830
831L_RESTORE_LDS_LOOP_W64:
832#if HAVE_BUFFER_LDS_LOAD
833 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 // first 64DW
834#else
835 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset
836 s_waitcnt vmcnt(0)
837 ds_store_addtid_b32 v0
838#endif
839 s_add_u32 m0, m0, 256 // 256 DW
840 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256 //mem offset increased by 256DW
841 s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0
842 s_cbranch_scc1 L_RESTORE_LDS_LOOP_W64 //LDS restore is complete?
843
844 /* restore VGPRs */
845L_RESTORE_VGPR:
846 // VGPR SR memory offset : 0
847 s_mov_b32 s_restore_mem_offset, 0x0
848 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on
849 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE
850 s_and_b32 m0, m0, 1
851 s_cmp_eq_u32 m0, 1
852 s_cbranch_scc1 L_ENABLE_RESTORE_VGPR_EXEC_HI
853 s_mov_b32 exec_hi, 0x00000000
854 s_branch L_RESTORE_VGPR_NORMAL
855L_ENABLE_RESTORE_VGPR_EXEC_HI:
856 s_mov_b32 exec_hi, 0xFFFFFFFF
857L_RESTORE_VGPR_NORMAL:
858 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)
859 s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1
860 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //Number of VGPRs = (vgpr_size + 1) * 4 (non-zero value)
861 //determine it is wave32 or wave64
862 s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE
863 s_and_b32 m0, m0, 1
864 s_cmp_eq_u32 m0, 1
865 s_cbranch_scc1 L_RESTORE_VGPR_WAVE64
866
867 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
868
869 // VGPR load using dw burst
870 s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v1, v0 will be the last
871 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128*4
872 s_mov_b32 m0, 4 //VGPR initial index value = 4
873 s_cmp_lt_u32 m0, s_restore_alloc_size
874 s_cbranch_scc0 L_RESTORE_SGPR
875
876L_RESTORE_VGPR_WAVE32_LOOP:
877 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
878 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:128
879 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:128*2
880 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:128*3
881 s_waitcnt vmcnt(0)
882 v_movreld_b32 v0, v0 //v[0+m0] = v0
883 v_movreld_b32 v1, v1
884 v_movreld_b32 v2, v2
885 v_movreld_b32 v3, v3
886 s_add_u32 m0, m0, 4 //next vgpr index
887 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128*4 //every buffer_load_dword does 128 bytes
888 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
889 s_cbranch_scc1 L_RESTORE_VGPR_WAVE32_LOOP //VGPR restore (except v0) is complete?
890
891 /* VGPR restore on v0 */
892 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
893 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:128
894 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:128*2
895 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:128*3
896 s_waitcnt vmcnt(0)
897
898 s_branch L_RESTORE_SGPR
899
900L_RESTORE_VGPR_WAVE64:
901 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
902
903 // VGPR load using dw burst
904 s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v4, v0 will be the last
905 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4
906 s_mov_b32 m0, 4 //VGPR initial index value = 4
907 s_cmp_lt_u32 m0, s_restore_alloc_size
908 s_cbranch_scc0 L_RESTORE_SHARED_VGPR
909
910L_RESTORE_VGPR_WAVE64_LOOP:
911 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
912 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256
913 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*2
914 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*3
915 s_waitcnt vmcnt(0)
916 v_movreld_b32 v0, v0 //v[0+m0] = v0
917 v_movreld_b32 v1, v1
918 v_movreld_b32 v2, v2
919 v_movreld_b32 v3, v3
920 s_add_u32 m0, m0, 4 //next vgpr index
921 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4 //every buffer_load_dword does 256 bytes
922 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
923 s_cbranch_scc1 L_RESTORE_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete?
924
925L_RESTORE_SHARED_VGPR:
926 //Below part will be the restore shared vgpr part (new for gfx10)
927 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) //shared_vgpr_size
928 s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero?
929 s_cbranch_scc0 L_RESTORE_V0 //no shared_vgpr used?
930 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value)
931 //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count.
932 //restore shared_vgpr will start from the index of m0
933 s_add_u32 s_restore_alloc_size, s_restore_alloc_size, m0
934 s_mov_b32 exec_lo, 0xFFFFFFFF
935 s_mov_b32 exec_hi, 0x00000000
936L_RESTORE_SHARED_VGPR_WAVE64_LOOP:
937 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
938 s_waitcnt vmcnt(0)
939 v_movreld_b32 v0, v0 //v[0+m0] = v0
940 s_add_u32 m0, m0, 1 //next vgpr index
941 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128
942 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0
943 s_cbranch_scc1 L_RESTORE_SHARED_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete?
944
945 s_mov_b32 exec_hi, 0xFFFFFFFF //restore back exec_hi before restoring V0!!
946
947 /* VGPR restore on v0 */
948L_RESTORE_V0:
949 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1
950 buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256
951 buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*2
952 buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save slc:1 glc:1 offset:256*3
953 s_waitcnt vmcnt(0)
954
955 /* restore SGPRs */
956 //will be 2+8+16*6
957 // SGPR SR memory offset : size(VGPR)+size(SVGPR)
958L_RESTORE_SGPR:
959 get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size)
960 get_svgpr_size_bytes(s_restore_tmp)
961 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
962 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes()
963 s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 20*4 //s108~s127 is not saved
964
965 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
966
967 s_mov_b32 m0, s_sgpr_save_num
968
969 read_4sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)
970 s_waitcnt lgkmcnt(0)
971
972 s_sub_u32 m0, m0, 4 // Restore from S[0] to S[104]
973 s_nop 0 // hazard SALU M0=> S_MOVREL
974
975 s_movreld_b64 s0, s0 //s[0+m0] = s0
976 s_movreld_b64 s2, s2
977
978 read_8sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)
979 s_waitcnt lgkmcnt(0)
980
981 s_sub_u32 m0, m0, 8 // Restore from S[0] to S[96]
982 s_nop 0 // hazard SALU M0=> S_MOVREL
983
984 s_movreld_b64 s0, s0 //s[0+m0] = s0
985 s_movreld_b64 s2, s2
986 s_movreld_b64 s4, s4
987 s_movreld_b64 s6, s6
988
989 L_RESTORE_SGPR_LOOP:
990 read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)
991 s_waitcnt lgkmcnt(0)
992
993 s_sub_u32 m0, m0, 16 // Restore from S[n] to S[0]
994 s_nop 0 // hazard SALU M0=> S_MOVREL
995
996 s_movreld_b64 s0, s0 //s[0+m0] = s0
997 s_movreld_b64 s2, s2
998 s_movreld_b64 s4, s4
999 s_movreld_b64 s6, s6
1000 s_movreld_b64 s8, s8
1001 s_movreld_b64 s10, s10
1002 s_movreld_b64 s12, s12
1003 s_movreld_b64 s14, s14
1004
1005 s_cmp_eq_u32 m0, 0 //scc = (m0 < s_sgpr_save_num) ? 1 : 0
1006 s_cbranch_scc0 L_RESTORE_SGPR_LOOP
1007
1008 // s_barrier with MODE.DEBUG_EN=1, STATUS.PRIV=1 incorrectly asserts debug exception.
1009 // Clear DEBUG_EN before and restore MODE after the barrier.
1010 s_setreg_imm32_b32 hwreg(HW_REG_MODE), 0
1011 s_barrier //barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG
1012
1013 /* restore HW registers */
1014L_RESTORE_HWREG:
1015 // HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)
1016 get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size)
1017 get_svgpr_size_bytes(s_restore_tmp)
1018 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
1019 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes()
1020
1021 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes
1022
1023 read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset)
1024 read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset)
1025 read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
1026 read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset)
1027 read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
1028 read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset)
1029 read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset)
1030 read_hwreg_from_mem(s_restore_xnack_mask, s_restore_buf_rsrc0, s_restore_mem_offset)
1031 read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset)
1032 read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset)
1033 s_waitcnt lgkmcnt(0)
1034
1035 s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO), s_restore_flat_scratch
1036
1037 read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset)
1038 s_waitcnt lgkmcnt(0) //from now on, it is safe to restore STATUS and IB_STS
1039
1040 s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI), s_restore_flat_scratch
1041
1042 s_mov_b32 m0, s_restore_m0
1043 s_mov_b32 exec_lo, s_restore_exec_lo
1044 s_mov_b32 exec_hi, s_restore_exec_hi
1045
1046 s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
1047 s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
1048
1049#if HAVE_XNACK
1050 s_setreg_b32 hwreg(HW_REG_SHADER_XNACK_MASK), s_restore_xnack_mask
1051#endif
1052
1053 s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
1054 s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
1055 s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
1056 s_setreg_b32 hwreg(HW_REG_MODE), s_restore_mode
1057
1058 // Restore trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
1059 // ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40
1060 get_vgpr_size_bytes(s_restore_ttmps_lo, s_restore_size)
1061 get_svgpr_size_bytes(s_restore_ttmps_hi)
1062 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_ttmps_hi
1063 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, get_sgpr_size_bytes()
1064 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_buf_rsrc0
1065 s_addc_u32 s_restore_ttmps_hi, s_restore_buf_rsrc1, 0x0
1066 s_and_b32 s_restore_ttmps_hi, s_restore_ttmps_hi, 0xFFFF
1067 s_load_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x50 glc:1
1068 s_load_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x60 glc:1
1069 s_load_dword ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x74 glc:1
1070 s_waitcnt lgkmcnt(0)
1071
1072#if HAVE_XNACK
1073 restore_ib_sts(s_restore_tmp, s_restore_m0)
1074#endif
1075
1076 s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS
1077 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
1078 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
1079
1080#if SW_SA_TRAP
1081 // If traps are enabled then return to the shader with PRIV=0.
1082 // Otherwise retain PRIV=1 for subsequent context save requests.
1083 s_getreg_b32 s_restore_tmp, hwreg(HW_REG_STATUS)
1084 s_bitcmp1_b32 s_restore_tmp, SQ_WAVE_STATUS_TRAP_EN_SHIFT
1085 s_cbranch_scc1 L_RETURN_WITHOUT_PRIV
1086
1087 s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu
1088 s_setpc_b64 [s_restore_pc_lo, s_restore_pc_hi]
1089L_RETURN_WITHOUT_PRIV:
1090#endif
1091
1092 s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu
1093 s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution
1094
1095L_END_PGM:
1096 s_endpgm
1097end
1098
1099function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)
1100#if NO_SQC_STORE
1101 // Copy into VGPR for later TCP store.
1102 v_writelane_b32 v2, s, m0
1103 s_add_u32 m0, m0, 0x1
1104#else
1105 s_mov_b32 exec_lo, m0
1106 s_mov_b32 m0, s_mem_offset
1107 s_buffer_store_dword s, s_rsrc, m0 glc:1
1108 s_add_u32 s_mem_offset, s_mem_offset, 4
1109 s_mov_b32 m0, exec_lo
1110#endif
1111end
1112
1113
1114function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)
1115#if NO_SQC_STORE
1116 // Copy into VGPR for later TCP store.
1117 for var sgpr_idx = 0; sgpr_idx < 16; sgpr_idx ++
1118 v_writelane_b32 v2, s[sgpr_idx], ttmp13
1119 s_add_u32 ttmp13, ttmp13, 0x1
1120 end
1121#else
1122 s_buffer_store_dwordx4 s[0], s_rsrc, 0 glc:1
1123 s_buffer_store_dwordx4 s[4], s_rsrc, 16 glc:1
1124 s_buffer_store_dwordx4 s[8], s_rsrc, 32 glc:1
1125 s_buffer_store_dwordx4 s[12], s_rsrc, 48 glc:1
1126 s_add_u32 s_rsrc[0], s_rsrc[0], 4*16
1127 s_addc_u32 s_rsrc[1], s_rsrc[1], 0x0
1128#endif
1129end
1130
1131function write_12sgpr_to_mem(s, s_rsrc, s_mem_offset)
1132#if NO_SQC_STORE
1133 // Copy into VGPR for later TCP store.
1134 for var sgpr_idx = 0; sgpr_idx < 12; sgpr_idx ++
1135 v_writelane_b32 v2, s[sgpr_idx], ttmp13
1136 s_add_u32 ttmp13, ttmp13, 0x1
1137 end
1138#else
1139 s_buffer_store_dwordx4 s[0], s_rsrc, 0 glc:1
1140 s_buffer_store_dwordx4 s[4], s_rsrc, 16 glc:1
1141 s_buffer_store_dwordx4 s[8], s_rsrc, 32 glc:1
1142 s_add_u32 s_rsrc[0], s_rsrc[0], 4*12
1143 s_addc_u32 s_rsrc[1], s_rsrc[1], 0x0
1144#endif
1145end
1146
1147function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
1148 s_buffer_load_dword s, s_rsrc, s_mem_offset glc:1
1149 s_add_u32 s_mem_offset, s_mem_offset, 4
1150end
1151
1152function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)
1153 s_sub_u32 s_mem_offset, s_mem_offset, 4*16
1154 s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset glc:1
1155end
1156
1157function read_8sgpr_from_mem(s, s_rsrc, s_mem_offset)
1158 s_sub_u32 s_mem_offset, s_mem_offset, 4*8
1159 s_buffer_load_dwordx8 s, s_rsrc, s_mem_offset glc:1
1160end
1161
1162function read_4sgpr_from_mem(s, s_rsrc, s_mem_offset)
1163 s_sub_u32 s_mem_offset, s_mem_offset, 4*4
1164 s_buffer_load_dwordx4 s, s_rsrc, s_mem_offset glc:1
1165end
1166
1167
1168function get_lds_size_bytes(s_lds_size_byte)
1169 s_getreg_b32 s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)
1170 s_lshl_b32 s_lds_size_byte, s_lds_size_byte, 8 //LDS size in dwords = lds_size * 64 *4Bytes // granularity 64DW
1171end
1172
1173function get_vgpr_size_bytes(s_vgpr_size_byte, s_size)
1174 s_getreg_b32 s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)
1175 s_add_u32 s_vgpr_size_byte, s_vgpr_size_byte, 1
1176 s_bitcmp1_b32 s_size, S_WAVE_SIZE
1177 s_cbranch_scc1 L_ENABLE_SHIFT_W64
1178 s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, (2+7) //Number of VGPRs = (vgpr_size + 1) * 4 * 32 * 4 (non-zero value)
1179 s_branch L_SHIFT_DONE
1180L_ENABLE_SHIFT_W64:
1181 s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4 (non-zero value)
1182L_SHIFT_DONE:
1183end
1184
1185function get_svgpr_size_bytes(s_svgpr_size_byte)
1186 s_getreg_b32 s_svgpr_size_byte, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE)
1187 s_lshl_b32 s_svgpr_size_byte, s_svgpr_size_byte, (3+7)
1188end
1189
1190function get_sgpr_size_bytes
1191 return 512
1192end
1193
1194function get_hwreg_size_bytes
1195 return 128
1196end
1197
1198function get_wave_size(s_reg)
1199 s_getreg_b32 s_reg, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE)
1200 s_lshl_b32 s_reg, s_reg, S_WAVE_SIZE
1201end
1202
1203function save_and_clear_ib_sts(tmp1, tmp2)
1204 // Preserve and clear scalar XNACK state before issuing scalar loads.
1205 // Save IB_STS.REPLAY_W64H[25], RCNT[21:16], FIRST_REPLAY[15] into
1206 // unused space ttmp11[31:24].
1207 s_andn2_b32 ttmp11, ttmp11, (TTMP11_SAVE_REPLAY_W64H_MASK | TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK)
1208 s_getreg_b32 tmp1, hwreg(HW_REG_IB_STS)
1209 s_and_b32 tmp2, tmp1, SQ_WAVE_IB_STS_REPLAY_W64H_MASK
1210 s_lshl_b32 tmp2, tmp2, (TTMP11_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT)
1211 s_or_b32 ttmp11, ttmp11, tmp2
1212 s_and_b32 tmp2, tmp1, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
1213 s_lshl_b32 tmp2, tmp2, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
1214 s_or_b32 ttmp11, ttmp11, tmp2
1215 s_andn2_b32 tmp1, tmp1, (SQ_WAVE_IB_STS_REPLAY_W64H_MASK | SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK)
1216 s_setreg_b32 hwreg(HW_REG_IB_STS), tmp1
1217end
1218
1219function restore_ib_sts(tmp1, tmp2)
1220 s_lshr_b32 tmp1, ttmp11, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
1221 s_and_b32 tmp2, tmp1, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
1222 s_lshr_b32 tmp1, ttmp11, (TTMP11_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT)
1223 s_and_b32 tmp1, tmp1, SQ_WAVE_IB_STS_REPLAY_W64H_MASK
1224 s_or_b32 tmp1, tmp1, tmp2
1225 s_setreg_b32 hwreg(HW_REG_IB_STS), tmp1
1226end