Loading...
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#ifndef KFD_DEVICE_QUEUE_MANAGER_H_
25#define KFD_DEVICE_QUEUE_MANAGER_H_
26
27#include <linux/rwsem.h>
28#include <linux/list.h>
29#include <linux/mutex.h>
30#include <linux/sched/mm.h>
31#include "kfd_priv.h"
32#include "kfd_mqd_manager.h"
33
34
35#define VMID_NUM 16
36
37struct device_process_node {
38 struct qcm_process_device *qpd;
39 struct list_head list;
40};
41
42/**
43 * struct device_queue_manager_ops
44 *
45 * @create_queue: Queue creation routine.
46 *
47 * @destroy_queue: Queue destruction routine.
48 *
49 * @update_queue: Queue update routine.
50 *
51 * @exeute_queues: Dispatches the queues list to the H/W.
52 *
53 * @register_process: This routine associates a specific process with device.
54 *
55 * @unregister_process: destroys the associations between process to device.
56 *
57 * @initialize: Initializes the pipelines and memory module for that device.
58 *
59 * @start: Initializes the resources/modules the the device needs for queues
60 * execution. This function is called on device initialization and after the
61 * system woke up after suspension.
62 *
63 * @stop: This routine stops execution of all the active queue running on the
64 * H/W and basically this function called on system suspend.
65 *
66 * @uninitialize: Destroys all the device queue manager resources allocated in
67 * initialize routine.
68 *
69 * @create_kernel_queue: Creates kernel queue. Used for debug queue.
70 *
71 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
72 *
73 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
74 * memory apertures.
75 *
76 * @process_termination: Clears all process queues belongs to that device.
77 *
78 * @evict_process_queues: Evict all active queues of a process
79 *
80 * @restore_process_queues: Restore all evicted queues queues of a process
81 *
82 * @get_wave_state: Retrieves context save state and optionally copies the
83 * control stack, if kept in the MQD, to the given userspace address.
84 */
85
86struct device_queue_manager_ops {
87 int (*create_queue)(struct device_queue_manager *dqm,
88 struct queue *q,
89 struct qcm_process_device *qpd);
90
91 int (*destroy_queue)(struct device_queue_manager *dqm,
92 struct qcm_process_device *qpd,
93 struct queue *q);
94
95 int (*update_queue)(struct device_queue_manager *dqm,
96 struct queue *q);
97
98 int (*register_process)(struct device_queue_manager *dqm,
99 struct qcm_process_device *qpd);
100
101 int (*unregister_process)(struct device_queue_manager *dqm,
102 struct qcm_process_device *qpd);
103
104 int (*initialize)(struct device_queue_manager *dqm);
105 int (*start)(struct device_queue_manager *dqm);
106 int (*stop)(struct device_queue_manager *dqm);
107 void (*pre_reset)(struct device_queue_manager *dqm);
108 void (*uninitialize)(struct device_queue_manager *dqm);
109 int (*create_kernel_queue)(struct device_queue_manager *dqm,
110 struct kernel_queue *kq,
111 struct qcm_process_device *qpd);
112
113 void (*destroy_kernel_queue)(struct device_queue_manager *dqm,
114 struct kernel_queue *kq,
115 struct qcm_process_device *qpd);
116
117 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
118 struct qcm_process_device *qpd,
119 enum cache_policy default_policy,
120 enum cache_policy alternate_policy,
121 void __user *alternate_aperture_base,
122 uint64_t alternate_aperture_size);
123
124 int (*set_trap_handler)(struct device_queue_manager *dqm,
125 struct qcm_process_device *qpd,
126 uint64_t tba_addr,
127 uint64_t tma_addr);
128
129 int (*process_termination)(struct device_queue_manager *dqm,
130 struct qcm_process_device *qpd);
131
132 int (*evict_process_queues)(struct device_queue_manager *dqm,
133 struct qcm_process_device *qpd);
134 int (*restore_process_queues)(struct device_queue_manager *dqm,
135 struct qcm_process_device *qpd);
136
137 int (*get_wave_state)(struct device_queue_manager *dqm,
138 struct queue *q,
139 void __user *ctl_stack,
140 u32 *ctl_stack_used_size,
141 u32 *save_area_used_size);
142};
143
144struct device_queue_manager_asic_ops {
145 int (*update_qpd)(struct device_queue_manager *dqm,
146 struct qcm_process_device *qpd);
147 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
148 struct qcm_process_device *qpd,
149 enum cache_policy default_policy,
150 enum cache_policy alternate_policy,
151 void __user *alternate_aperture_base,
152 uint64_t alternate_aperture_size);
153 void (*init_sdma_vm)(struct device_queue_manager *dqm,
154 struct queue *q,
155 struct qcm_process_device *qpd);
156 struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type,
157 struct kfd_dev *dev);
158};
159
160/**
161 * struct device_queue_manager
162 *
163 * This struct is a base class for the kfd queues scheduler in the
164 * device level. The device base class should expose the basic operations
165 * for queue creation and queue destruction. This base class hides the
166 * scheduling mode of the driver and the specific implementation of the
167 * concrete device. This class is the only class in the queues scheduler
168 * that configures the H/W.
169 *
170 */
171
172struct device_queue_manager {
173 struct device_queue_manager_ops ops;
174 struct device_queue_manager_asic_ops asic_ops;
175
176 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];
177 struct packet_manager packets;
178 struct kfd_dev *dev;
179 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
180 struct list_head queues;
181 unsigned int saved_flags;
182 unsigned int processes_count;
183 unsigned int active_queue_count;
184 unsigned int active_cp_queue_count;
185 unsigned int gws_queue_count;
186 unsigned int total_queue_count;
187 unsigned int next_pipe_to_allocate;
188 unsigned int *allocated_queues;
189 uint64_t sdma_bitmap;
190 uint64_t xgmi_sdma_bitmap;
191 /* the pasid mapping for each kfd vmid */
192 uint16_t vmid_pasid[VMID_NUM];
193 uint64_t pipelines_addr;
194 uint64_t fence_gpu_addr;
195 unsigned int *fence_addr;
196 struct kfd_mem_obj *fence_mem;
197 bool active_runlist;
198 int sched_policy;
199
200 /* hw exception */
201 bool is_hws_hang;
202 bool is_resetting;
203 struct work_struct hw_exception_work;
204 struct kfd_mem_obj hiq_sdma_mqd;
205 bool sched_running;
206};
207
208void device_queue_manager_init_cik(
209 struct device_queue_manager_asic_ops *asic_ops);
210void device_queue_manager_init_cik_hawaii(
211 struct device_queue_manager_asic_ops *asic_ops);
212void device_queue_manager_init_vi(
213 struct device_queue_manager_asic_ops *asic_ops);
214void device_queue_manager_init_vi_tonga(
215 struct device_queue_manager_asic_ops *asic_ops);
216void device_queue_manager_init_v9(
217 struct device_queue_manager_asic_ops *asic_ops);
218void device_queue_manager_init_v10_navi10(
219 struct device_queue_manager_asic_ops *asic_ops);
220void program_sh_mem_settings(struct device_queue_manager *dqm,
221 struct qcm_process_device *qpd);
222unsigned int get_cp_queues_num(struct device_queue_manager *dqm);
223unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
224unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
225unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
226unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);
227
228static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
229{
230 return (pdd->lds_base >> 16) & 0xFF;
231}
232
233static inline unsigned int
234get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
235{
236 return (pdd->lds_base >> 60) & 0x0E;
237}
238
239/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
240 * happens while holding this lock anywhere to prevent deadlocks when
241 * an MMU notifier runs in reclaim-FS context.
242 */
243static inline void dqm_lock(struct device_queue_manager *dqm)
244{
245 mutex_lock(&dqm->lock_hidden);
246 dqm->saved_flags = memalloc_nofs_save();
247}
248static inline void dqm_unlock(struct device_queue_manager *dqm)
249{
250 memalloc_nofs_restore(dqm->saved_flags);
251 mutex_unlock(&dqm->lock_hidden);
252}
253
254int read_sdma_queue_counter(uint64_t q_rptr, uint64_t *val);
255#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
1/* SPDX-License-Identifier: GPL-2.0 OR MIT */
2/*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#ifndef KFD_DEVICE_QUEUE_MANAGER_H_
26#define KFD_DEVICE_QUEUE_MANAGER_H_
27
28#include <linux/rwsem.h>
29#include <linux/list.h>
30#include <linux/mutex.h>
31#include <linux/sched/mm.h>
32#include "kfd_priv.h"
33#include "kfd_mqd_manager.h"
34
35
36#define VMID_NUM 16
37
38#define KFD_MES_PROCESS_QUANTUM 100000
39#define KFD_MES_GANG_QUANTUM 10000
40#define USE_DEFAULT_GRACE_PERIOD 0xffffffff
41
42struct device_process_node {
43 struct qcm_process_device *qpd;
44 struct list_head list;
45};
46
47union SQ_CMD_BITS {
48 struct {
49 uint32_t cmd:3;
50 uint32_t:1;
51 uint32_t mode:3;
52 uint32_t check_vmid:1;
53 uint32_t trap_id:3;
54 uint32_t:5;
55 uint32_t wave_id:4;
56 uint32_t simd_id:2;
57 uint32_t:2;
58 uint32_t queue_id:3;
59 uint32_t:1;
60 uint32_t vm_id:4;
61 } bitfields, bits;
62 uint32_t u32All;
63 signed int i32All;
64 float f32All;
65};
66
67union GRBM_GFX_INDEX_BITS {
68 struct {
69 uint32_t instance_index:8;
70 uint32_t sh_index:8;
71 uint32_t se_index:8;
72 uint32_t:5;
73 uint32_t sh_broadcast_writes:1;
74 uint32_t instance_broadcast_writes:1;
75 uint32_t se_broadcast_writes:1;
76 } bitfields, bits;
77 uint32_t u32All;
78 signed int i32All;
79 float f32All;
80};
81
82/**
83 * struct device_queue_manager_ops
84 *
85 * @create_queue: Queue creation routine.
86 *
87 * @destroy_queue: Queue destruction routine.
88 *
89 * @update_queue: Queue update routine.
90 *
91 * @exeute_queues: Dispatches the queues list to the H/W.
92 *
93 * @register_process: This routine associates a specific process with device.
94 *
95 * @unregister_process: destroys the associations between process to device.
96 *
97 * @initialize: Initializes the pipelines and memory module for that device.
98 *
99 * @start: Initializes the resources/modules the device needs for queues
100 * execution. This function is called on device initialization and after the
101 * system woke up after suspension.
102 *
103 * @stop: This routine stops execution of all the active queue running on the
104 * H/W and basically this function called on system suspend.
105 *
106 * @uninitialize: Destroys all the device queue manager resources allocated in
107 * initialize routine.
108 *
109 * @create_kernel_queue: Creates kernel queue. Used for debug queue.
110 *
111 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
112 *
113 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
114 * memory apertures.
115 *
116 * @process_termination: Clears all process queues belongs to that device.
117 *
118 * @evict_process_queues: Evict all active queues of a process
119 *
120 * @restore_process_queues: Restore all evicted queues of a process
121 *
122 * @get_wave_state: Retrieves context save state and optionally copies the
123 * control stack, if kept in the MQD, to the given userspace address.
124 *
125 * @reset_queues: reset queues which consume RAS poison
126 * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint.
127 *
128 * @checkpoint_mqd: checkpoint queue MQD contents for CRIU.
129 */
130
131struct device_queue_manager_ops {
132 int (*create_queue)(struct device_queue_manager *dqm,
133 struct queue *q,
134 struct qcm_process_device *qpd,
135 const struct kfd_criu_queue_priv_data *qd,
136 const void *restore_mqd,
137 const void *restore_ctl_stack);
138
139 int (*destroy_queue)(struct device_queue_manager *dqm,
140 struct qcm_process_device *qpd,
141 struct queue *q);
142
143 int (*update_queue)(struct device_queue_manager *dqm,
144 struct queue *q, struct mqd_update_info *minfo);
145
146 int (*register_process)(struct device_queue_manager *dqm,
147 struct qcm_process_device *qpd);
148
149 int (*unregister_process)(struct device_queue_manager *dqm,
150 struct qcm_process_device *qpd);
151
152 int (*initialize)(struct device_queue_manager *dqm);
153 int (*start)(struct device_queue_manager *dqm);
154 int (*stop)(struct device_queue_manager *dqm);
155 void (*pre_reset)(struct device_queue_manager *dqm);
156 void (*uninitialize)(struct device_queue_manager *dqm);
157 int (*create_kernel_queue)(struct device_queue_manager *dqm,
158 struct kernel_queue *kq,
159 struct qcm_process_device *qpd);
160
161 void (*destroy_kernel_queue)(struct device_queue_manager *dqm,
162 struct kernel_queue *kq,
163 struct qcm_process_device *qpd);
164
165 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
166 struct qcm_process_device *qpd,
167 enum cache_policy default_policy,
168 enum cache_policy alternate_policy,
169 void __user *alternate_aperture_base,
170 uint64_t alternate_aperture_size);
171
172 int (*process_termination)(struct device_queue_manager *dqm,
173 struct qcm_process_device *qpd);
174
175 int (*evict_process_queues)(struct device_queue_manager *dqm,
176 struct qcm_process_device *qpd);
177 int (*restore_process_queues)(struct device_queue_manager *dqm,
178 struct qcm_process_device *qpd);
179
180 int (*get_wave_state)(struct device_queue_manager *dqm,
181 struct queue *q,
182 void __user *ctl_stack,
183 u32 *ctl_stack_used_size,
184 u32 *save_area_used_size);
185
186 int (*reset_queues)(struct device_queue_manager *dqm,
187 uint16_t pasid);
188 void (*get_queue_checkpoint_info)(struct device_queue_manager *dqm,
189 const struct queue *q, u32 *mqd_size,
190 u32 *ctl_stack_size);
191
192 int (*checkpoint_mqd)(struct device_queue_manager *dqm,
193 const struct queue *q,
194 void *mqd,
195 void *ctl_stack);
196};
197
198struct device_queue_manager_asic_ops {
199 int (*update_qpd)(struct device_queue_manager *dqm,
200 struct qcm_process_device *qpd);
201 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
202 struct qcm_process_device *qpd,
203 enum cache_policy default_policy,
204 enum cache_policy alternate_policy,
205 void __user *alternate_aperture_base,
206 uint64_t alternate_aperture_size);
207 void (*init_sdma_vm)(struct device_queue_manager *dqm,
208 struct queue *q,
209 struct qcm_process_device *qpd);
210 struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type,
211 struct kfd_node *dev);
212};
213
214/**
215 * struct device_queue_manager
216 *
217 * This struct is a base class for the kfd queues scheduler in the
218 * device level. The device base class should expose the basic operations
219 * for queue creation and queue destruction. This base class hides the
220 * scheduling mode of the driver and the specific implementation of the
221 * concrete device. This class is the only class in the queues scheduler
222 * that configures the H/W.
223 *
224 */
225
226struct device_queue_manager {
227 struct device_queue_manager_ops ops;
228 struct device_queue_manager_asic_ops asic_ops;
229
230 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];
231 struct packet_manager packet_mgr;
232 struct kfd_node *dev;
233 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
234 struct list_head queues;
235 unsigned int saved_flags;
236 unsigned int processes_count;
237 unsigned int active_queue_count;
238 unsigned int active_cp_queue_count;
239 unsigned int gws_queue_count;
240 unsigned int total_queue_count;
241 unsigned int next_pipe_to_allocate;
242 unsigned int *allocated_queues;
243 DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES);
244 DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
245 /* the pasid mapping for each kfd vmid */
246 uint16_t vmid_pasid[VMID_NUM];
247 uint64_t pipelines_addr;
248 uint64_t fence_gpu_addr;
249 uint64_t *fence_addr;
250 struct kfd_mem_obj *fence_mem;
251 bool active_runlist;
252 int sched_policy;
253 uint32_t trap_debug_vmid;
254
255 /* hw exception */
256 bool is_hws_hang;
257 bool is_resetting;
258 struct work_struct hw_exception_work;
259 struct kfd_mem_obj hiq_sdma_mqd;
260 bool sched_running;
261
262 /* used for GFX 9.4.3 only */
263 uint32_t current_logical_xcc_start;
264
265 uint32_t wait_times;
266
267 wait_queue_head_t destroy_wait;
268};
269
270void device_queue_manager_init_cik(
271 struct device_queue_manager_asic_ops *asic_ops);
272void device_queue_manager_init_vi(
273 struct device_queue_manager_asic_ops *asic_ops);
274void device_queue_manager_init_v9(
275 struct device_queue_manager_asic_ops *asic_ops);
276void device_queue_manager_init_v10(
277 struct device_queue_manager_asic_ops *asic_ops);
278void device_queue_manager_init_v11(
279 struct device_queue_manager_asic_ops *asic_ops);
280void program_sh_mem_settings(struct device_queue_manager *dqm,
281 struct qcm_process_device *qpd);
282unsigned int get_cp_queues_num(struct device_queue_manager *dqm);
283unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
284unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
285unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
286unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);
287int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
288 struct qcm_process_device *qpd);
289int release_debug_trap_vmid(struct device_queue_manager *dqm,
290 struct qcm_process_device *qpd);
291int suspend_queues(struct kfd_process *p,
292 uint32_t num_queues,
293 uint32_t grace_period,
294 uint64_t exception_clear_mask,
295 uint32_t *usr_queue_id_array);
296int resume_queues(struct kfd_process *p,
297 uint32_t num_queues,
298 uint32_t *usr_queue_id_array);
299void set_queue_snapshot_entry(struct queue *q,
300 uint64_t exception_clear_mask,
301 struct kfd_queue_snapshot_entry *qss_entry);
302int debug_lock_and_unmap(struct device_queue_manager *dqm);
303int debug_map_and_unlock(struct device_queue_manager *dqm);
304int debug_refresh_runlist(struct device_queue_manager *dqm);
305
306static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
307{
308 return (pdd->lds_base >> 16) & 0xFF;
309}
310
311static inline unsigned int
312get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
313{
314 return (pdd->lds_base >> 60) & 0x0E;
315}
316
317/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
318 * happens while holding this lock anywhere to prevent deadlocks when
319 * an MMU notifier runs in reclaim-FS context.
320 */
321static inline void dqm_lock(struct device_queue_manager *dqm)
322{
323 mutex_lock(&dqm->lock_hidden);
324 dqm->saved_flags = memalloc_noreclaim_save();
325}
326static inline void dqm_unlock(struct device_queue_manager *dqm)
327{
328 memalloc_noreclaim_restore(dqm->saved_flags);
329 mutex_unlock(&dqm->lock_hidden);
330}
331
332static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val)
333{
334 /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */
335 return get_user(*val, q_rptr + 1);
336}
337#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */