Loading...
Note: File does not exist in v4.6.
1/*
2 * Copyright 2016-2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include "kfd_priv.h"
24#include "kfd_events.h"
25#include "soc15_int.h"
26#include "kfd_device_queue_manager.h"
27#include "kfd_smi_events.h"
28
29enum SQ_INTERRUPT_WORD_ENCODING {
30 SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,
31 SQ_INTERRUPT_WORD_ENCODING_INST,
32 SQ_INTERRUPT_WORD_ENCODING_ERROR,
33};
34
35enum SQ_INTERRUPT_ERROR_TYPE {
36 SQ_INTERRUPT_ERROR_TYPE_EDC_FUE = 0x0,
37 SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST,
38 SQ_INTERRUPT_ERROR_TYPE_MEMVIOL,
39 SQ_INTERRUPT_ERROR_TYPE_EDC_FED,
40};
41
42/* SQ_INTERRUPT_WORD_AUTO_CTXID */
43#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE__SHIFT 0
44#define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT__SHIFT 1
45#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL__SHIFT 2
46#define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP__SHIFT 3
47#define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP__SHIFT 4
48#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW__SHIFT 5
49#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW__SHIFT 6
50#define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW__SHIFT 7
51#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR__SHIFT 8
52#define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID__SHIFT 24
53#define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING__SHIFT 26
54
55#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_MASK 0x00000001
56#define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT_MASK 0x00000002
57#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL_MASK 0x00000004
58#define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP_MASK 0x00000008
59#define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP_MASK 0x00000010
60#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW_MASK 0x00000020
61#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW_MASK 0x00000040
62#define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW_MASK 0x00000080
63#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR_MASK 0x00000100
64#define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID_MASK 0x03000000
65#define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING_MASK 0x0c000000
66
67/* SQ_INTERRUPT_WORD_WAVE_CTXID */
68#define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA__SHIFT 0
69#define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID__SHIFT 12
70#define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV__SHIFT 13
71#define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID__SHIFT 14
72#define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID__SHIFT 18
73#define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID__SHIFT 20
74#define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID__SHIFT 24
75#define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING__SHIFT 26
76
77#define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA_MASK 0x00000fff
78#define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID_MASK 0x00001000
79#define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK 0x00002000
80#define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID_MASK 0x0003c000
81#define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID_MASK 0x000c0000
82#define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID_MASK 0x00f00000
83#define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID_MASK 0x03000000
84#define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING_MASK 0x0c000000
85
86#define KFD_CONTEXT_ID_GET_SQ_INT_DATA(ctx0, ctx1) \
87 ((ctx0 & 0xfff) | ((ctx0 >> 16) & 0xf000) | ((ctx1 << 16) & 0xff0000))
88
89#define KFD_SQ_INT_DATA__ERR_TYPE_MASK 0xF00000
90#define KFD_SQ_INT_DATA__ERR_TYPE__SHIFT 20
91
92static bool event_interrupt_isr_v9(struct kfd_dev *dev,
93 const uint32_t *ih_ring_entry,
94 uint32_t *patched_ihre,
95 bool *patched_flag)
96{
97 uint16_t source_id, client_id, pasid, vmid;
98 const uint32_t *data = ih_ring_entry;
99
100 /* Only handle interrupts from KFD VMIDs */
101 vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
102 if (vmid < dev->vm_info.first_vmid_kfd ||
103 vmid > dev->vm_info.last_vmid_kfd)
104 return false;
105
106 source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
107 client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
108 pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
109
110 /* Only handle clients we care about */
111 if (client_id != SOC15_IH_CLIENTID_GRBM_CP &&
112 client_id != SOC15_IH_CLIENTID_SDMA0 &&
113 client_id != SOC15_IH_CLIENTID_SDMA1 &&
114 client_id != SOC15_IH_CLIENTID_SDMA2 &&
115 client_id != SOC15_IH_CLIENTID_SDMA3 &&
116 client_id != SOC15_IH_CLIENTID_SDMA4 &&
117 client_id != SOC15_IH_CLIENTID_SDMA5 &&
118 client_id != SOC15_IH_CLIENTID_SDMA6 &&
119 client_id != SOC15_IH_CLIENTID_SDMA7 &&
120 client_id != SOC15_IH_CLIENTID_VMC &&
121 client_id != SOC15_IH_CLIENTID_VMC1 &&
122 client_id != SOC15_IH_CLIENTID_UTCL2 &&
123 client_id != SOC15_IH_CLIENTID_SE0SH &&
124 client_id != SOC15_IH_CLIENTID_SE1SH &&
125 client_id != SOC15_IH_CLIENTID_SE2SH &&
126 client_id != SOC15_IH_CLIENTID_SE3SH)
127 return false;
128
129 /* This is a known issue for gfx9. Under non HWS, pasid is not set
130 * in the interrupt payload, so we need to find out the pasid on our
131 * own.
132 */
133 if (!pasid && dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
134 const uint32_t pasid_mask = 0xffff;
135
136 *patched_flag = true;
137 memcpy(patched_ihre, ih_ring_entry,
138 dev->device_info->ih_ring_entry_size);
139
140 pasid = dev->dqm->vmid_pasid[vmid];
141
142 /* Patch the pasid field */
143 patched_ihre[3] = cpu_to_le32((le32_to_cpu(patched_ihre[3])
144 & ~pasid_mask) | pasid);
145 }
146
147 pr_debug("client id 0x%x, source id %d, vmid %d, pasid 0x%x. raw data:\n",
148 client_id, source_id, vmid, pasid);
149 pr_debug("%8X, %8X, %8X, %8X, %8X, %8X, %8X, %8X.\n",
150 data[0], data[1], data[2], data[3],
151 data[4], data[5], data[6], data[7]);
152
153 /* If there is no valid PASID, it's likely a bug */
154 if (WARN_ONCE(pasid == 0, "Bug: No PASID in KFD interrupt"))
155 return false;
156
157 /* Interrupt types we care about: various signals and faults.
158 * They will be forwarded to a work queue (see below).
159 */
160 return source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
161 source_id == SOC15_INTSRC_SDMA_TRAP ||
162 source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
163 source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
164 ((client_id == SOC15_IH_CLIENTID_VMC ||
165 client_id == SOC15_IH_CLIENTID_VMC1 ||
166 client_id == SOC15_IH_CLIENTID_UTCL2) &&
167 !amdgpu_no_queue_eviction_on_vm_fault);
168}
169
170static void event_interrupt_wq_v9(struct kfd_dev *dev,
171 const uint32_t *ih_ring_entry)
172{
173 uint16_t source_id, client_id, pasid, vmid;
174 uint32_t context_id0, context_id1;
175 uint32_t sq_intr_err, sq_int_data, encoding;
176
177 source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
178 client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
179 pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
180 vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
181 context_id0 = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry);
182 context_id1 = SOC15_CONTEXT_ID1_FROM_IH_ENTRY(ih_ring_entry);
183
184 if (client_id == SOC15_IH_CLIENTID_GRBM_CP ||
185 client_id == SOC15_IH_CLIENTID_SE0SH ||
186 client_id == SOC15_IH_CLIENTID_SE1SH ||
187 client_id == SOC15_IH_CLIENTID_SE2SH ||
188 client_id == SOC15_IH_CLIENTID_SE3SH) {
189 if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
190 kfd_signal_event_interrupt(pasid, context_id0, 32);
191 else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) {
192 sq_int_data = KFD_CONTEXT_ID_GET_SQ_INT_DATA(context_id0, context_id1);
193 encoding = REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, ENCODING);
194 switch (encoding) {
195 case SQ_INTERRUPT_WORD_ENCODING_AUTO:
196 pr_debug(
197 "sq_intr: auto, se %d, ttrace %d, wlt %d, ttrac_buf_full %d, reg_tms %d, cmd_tms %d, host_cmd_ovf %d, host_reg_ovf %d, immed_ovf %d, ttrace_utc_err %d\n",
198 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, SE_ID),
199 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE),
200 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, WLT),
201 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_BUF_FULL),
202 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, REG_TIMESTAMP),
203 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, CMD_TIMESTAMP),
204 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_CMD_OVERFLOW),
205 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_REG_OVERFLOW),
206 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, IMMED_OVERFLOW),
207 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_UTC_ERROR));
208 break;
209 case SQ_INTERRUPT_WORD_ENCODING_INST:
210 pr_debug("sq_intr: inst, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, intr_data 0x%x\n",
211 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID),
212 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA),
213 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID),
214 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV),
215 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID),
216 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID),
217 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID),
218 sq_int_data);
219 break;
220 case SQ_INTERRUPT_WORD_ENCODING_ERROR:
221 sq_intr_err = REG_GET_FIELD(sq_int_data, KFD_SQ_INT_DATA, ERR_TYPE);
222 pr_warn("sq_intr: error, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, err_type %d\n",
223 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID),
224 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA),
225 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID),
226 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV),
227 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID),
228 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID),
229 REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID),
230 sq_intr_err);
231 if (sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST &&
232 sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) {
233 kfd_signal_poison_consumed_event(dev, pasid);
234 amdgpu_amdkfd_gpu_reset(dev->kgd);
235 return;
236 }
237 break;
238 default:
239 break;
240 }
241 kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24);
242 } else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE)
243 kfd_signal_hw_exception_event(pasid);
244 } else if (client_id == SOC15_IH_CLIENTID_SDMA0 ||
245 client_id == SOC15_IH_CLIENTID_SDMA1 ||
246 client_id == SOC15_IH_CLIENTID_SDMA2 ||
247 client_id == SOC15_IH_CLIENTID_SDMA3 ||
248 client_id == SOC15_IH_CLIENTID_SDMA4 ||
249 client_id == SOC15_IH_CLIENTID_SDMA5 ||
250 client_id == SOC15_IH_CLIENTID_SDMA6 ||
251 client_id == SOC15_IH_CLIENTID_SDMA7) {
252 if (source_id == SOC15_INTSRC_SDMA_TRAP) {
253 kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
254 } else if (source_id == SOC15_INTSRC_SDMA_ECC) {
255 kfd_signal_poison_consumed_event(dev, pasid);
256 amdgpu_amdkfd_gpu_reset(dev->kgd);
257 return;
258 }
259 } else if (client_id == SOC15_IH_CLIENTID_VMC ||
260 client_id == SOC15_IH_CLIENTID_VMC1 ||
261 client_id == SOC15_IH_CLIENTID_UTCL2) {
262 struct kfd_vm_fault_info info = {0};
263 uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
264
265 info.vmid = vmid;
266 info.mc_id = client_id;
267 info.page_addr = ih_ring_entry[4] |
268 (uint64_t)(ih_ring_entry[5] & 0xf) << 32;
269 info.prot_valid = ring_id & 0x08;
270 info.prot_read = ring_id & 0x10;
271 info.prot_write = ring_id & 0x20;
272
273 kfd_smi_event_update_vmfault(dev, pasid);
274 kfd_process_vm_fault(dev->dqm, pasid);
275 kfd_signal_vm_fault_event(dev, pasid, &info);
276 }
277}
278
279const struct kfd_event_interrupt_class event_interrupt_class_v9 = {
280 .interrupt_isr = event_interrupt_isr_v9,
281 .interrupt_wq = event_interrupt_wq_v9,
282};