Loading...
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/slab.h>
25#include <linux/list.h>
26#include <linux/types.h>
27#include <linux/printk.h>
28#include <linux/bitops.h>
29#include <linux/sched.h>
30#include "kfd_priv.h"
31#include "kfd_device_queue_manager.h"
32#include "kfd_mqd_manager.h"
33#include "cik_regs.h"
34#include "kfd_kernel_queue.h"
35
36/* Size of the per-pipe EOP queue */
37#define CIK_HPD_EOP_BYTES_LOG2 11
38#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
39
40static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
41 unsigned int pasid, unsigned int vmid);
42
43static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
44 struct queue *q,
45 struct qcm_process_device *qpd);
46
47static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
48static int destroy_queues_cpsch(struct device_queue_manager *dqm,
49 bool preempt_static_queues, bool lock);
50
51static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
52 struct queue *q,
53 struct qcm_process_device *qpd);
54
55static void deallocate_sdma_queue(struct device_queue_manager *dqm,
56 unsigned int sdma_queue_id);
57
58static inline
59enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
60{
61 if (type == KFD_QUEUE_TYPE_SDMA)
62 return KFD_MQD_TYPE_SDMA;
63 return KFD_MQD_TYPE_CP;
64}
65
66unsigned int get_first_pipe(struct device_queue_manager *dqm)
67{
68 BUG_ON(!dqm || !dqm->dev);
69 return dqm->dev->shared_resources.first_compute_pipe;
70}
71
72unsigned int get_pipes_num(struct device_queue_manager *dqm)
73{
74 BUG_ON(!dqm || !dqm->dev);
75 return dqm->dev->shared_resources.compute_pipe_count;
76}
77
78static inline unsigned int get_pipes_num_cpsch(void)
79{
80 return PIPE_PER_ME_CP_SCHEDULING;
81}
82
83void program_sh_mem_settings(struct device_queue_manager *dqm,
84 struct qcm_process_device *qpd)
85{
86 return dqm->dev->kfd2kgd->program_sh_mem_settings(
87 dqm->dev->kgd, qpd->vmid,
88 qpd->sh_mem_config,
89 qpd->sh_mem_ape1_base,
90 qpd->sh_mem_ape1_limit,
91 qpd->sh_mem_bases);
92}
93
94static int allocate_vmid(struct device_queue_manager *dqm,
95 struct qcm_process_device *qpd,
96 struct queue *q)
97{
98 int bit, allocated_vmid;
99
100 if (dqm->vmid_bitmap == 0)
101 return -ENOMEM;
102
103 bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
104 clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
105
106 /* Kaveri kfd vmid's starts from vmid 8 */
107 allocated_vmid = bit + KFD_VMID_START_OFFSET;
108 pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
109 qpd->vmid = allocated_vmid;
110 q->properties.vmid = allocated_vmid;
111
112 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
113 program_sh_mem_settings(dqm, qpd);
114
115 return 0;
116}
117
118static void deallocate_vmid(struct device_queue_manager *dqm,
119 struct qcm_process_device *qpd,
120 struct queue *q)
121{
122 int bit = qpd->vmid - KFD_VMID_START_OFFSET;
123
124 /* Release the vmid mapping */
125 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
126
127 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
128 qpd->vmid = 0;
129 q->properties.vmid = 0;
130}
131
132static int create_queue_nocpsch(struct device_queue_manager *dqm,
133 struct queue *q,
134 struct qcm_process_device *qpd,
135 int *allocated_vmid)
136{
137 int retval;
138
139 BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
140
141 pr_debug("kfd: In func %s\n", __func__);
142 print_queue(q);
143
144 mutex_lock(&dqm->lock);
145
146 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
147 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
148 dqm->total_queue_count);
149 mutex_unlock(&dqm->lock);
150 return -EPERM;
151 }
152
153 if (list_empty(&qpd->queues_list)) {
154 retval = allocate_vmid(dqm, qpd, q);
155 if (retval != 0) {
156 mutex_unlock(&dqm->lock);
157 return retval;
158 }
159 }
160 *allocated_vmid = qpd->vmid;
161 q->properties.vmid = qpd->vmid;
162
163 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
164 retval = create_compute_queue_nocpsch(dqm, q, qpd);
165 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
166 retval = create_sdma_queue_nocpsch(dqm, q, qpd);
167
168 if (retval != 0) {
169 if (list_empty(&qpd->queues_list)) {
170 deallocate_vmid(dqm, qpd, q);
171 *allocated_vmid = 0;
172 }
173 mutex_unlock(&dqm->lock);
174 return retval;
175 }
176
177 list_add(&q->list, &qpd->queues_list);
178 if (q->properties.is_active)
179 dqm->queue_count++;
180
181 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
182 dqm->sdma_queue_count++;
183
184 /*
185 * Unconditionally increment this counter, regardless of the queue's
186 * type or whether the queue is active.
187 */
188 dqm->total_queue_count++;
189 pr_debug("Total of %d queues are accountable so far\n",
190 dqm->total_queue_count);
191
192 mutex_unlock(&dqm->lock);
193 return 0;
194}
195
196static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
197{
198 bool set;
199 int pipe, bit, i;
200
201 set = false;
202
203 for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm);
204 pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) {
205 if (dqm->allocated_queues[pipe] != 0) {
206 bit = find_first_bit(
207 (unsigned long *)&dqm->allocated_queues[pipe],
208 QUEUES_PER_PIPE);
209
210 clear_bit(bit,
211 (unsigned long *)&dqm->allocated_queues[pipe]);
212 q->pipe = pipe;
213 q->queue = bit;
214 set = true;
215 break;
216 }
217 }
218
219 if (set == false)
220 return -EBUSY;
221
222 pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
223 __func__, q->pipe, q->queue);
224 /* horizontal hqd allocation */
225 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm);
226
227 return 0;
228}
229
230static inline void deallocate_hqd(struct device_queue_manager *dqm,
231 struct queue *q)
232{
233 set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
234}
235
236static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
237 struct queue *q,
238 struct qcm_process_device *qpd)
239{
240 int retval;
241 struct mqd_manager *mqd;
242
243 BUG_ON(!dqm || !q || !qpd);
244
245 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
246 if (mqd == NULL)
247 return -ENOMEM;
248
249 retval = allocate_hqd(dqm, q);
250 if (retval != 0)
251 return retval;
252
253 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
254 &q->gart_mqd_addr, &q->properties);
255 if (retval != 0) {
256 deallocate_hqd(dqm, q);
257 return retval;
258 }
259
260 pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",
261 q->pipe,
262 q->queue);
263
264 retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
265 q->queue, (uint32_t __user *) q->properties.write_ptr);
266 if (retval != 0) {
267 deallocate_hqd(dqm, q);
268 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
269 return retval;
270 }
271
272 return 0;
273}
274
275static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
276 struct qcm_process_device *qpd,
277 struct queue *q)
278{
279 int retval;
280 struct mqd_manager *mqd;
281
282 BUG_ON(!dqm || !q || !q->mqd || !qpd);
283
284 retval = 0;
285
286 pr_debug("kfd: In Func %s\n", __func__);
287
288 mutex_lock(&dqm->lock);
289
290 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
291 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
292 if (mqd == NULL) {
293 retval = -ENOMEM;
294 goto out;
295 }
296 deallocate_hqd(dqm, q);
297 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
298 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
299 if (mqd == NULL) {
300 retval = -ENOMEM;
301 goto out;
302 }
303 dqm->sdma_queue_count--;
304 deallocate_sdma_queue(dqm, q->sdma_id);
305 } else {
306 pr_debug("q->properties.type is invalid (%d)\n",
307 q->properties.type);
308 retval = -EINVAL;
309 goto out;
310 }
311
312 retval = mqd->destroy_mqd(mqd, q->mqd,
313 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
314 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
315 q->pipe, q->queue);
316
317 if (retval != 0)
318 goto out;
319
320 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
321
322 list_del(&q->list);
323 if (list_empty(&qpd->queues_list))
324 deallocate_vmid(dqm, qpd, q);
325 if (q->properties.is_active)
326 dqm->queue_count--;
327
328 /*
329 * Unconditionally decrement this counter, regardless of the queue's
330 * type
331 */
332 dqm->total_queue_count--;
333 pr_debug("Total of %d queues are accountable so far\n",
334 dqm->total_queue_count);
335
336out:
337 mutex_unlock(&dqm->lock);
338 return retval;
339}
340
341static int update_queue(struct device_queue_manager *dqm, struct queue *q)
342{
343 int retval;
344 struct mqd_manager *mqd;
345 bool prev_active = false;
346
347 BUG_ON(!dqm || !q || !q->mqd);
348
349 mutex_lock(&dqm->lock);
350 mqd = dqm->ops.get_mqd_manager(dqm,
351 get_mqd_type_from_queue_type(q->properties.type));
352 if (mqd == NULL) {
353 mutex_unlock(&dqm->lock);
354 return -ENOMEM;
355 }
356
357 if (q->properties.is_active == true)
358 prev_active = true;
359
360 /*
361 *
362 * check active state vs. the previous state
363 * and modify counter accordingly
364 */
365 retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
366 if ((q->properties.is_active == true) && (prev_active == false))
367 dqm->queue_count++;
368 else if ((q->properties.is_active == false) && (prev_active == true))
369 dqm->queue_count--;
370
371 if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
372 retval = execute_queues_cpsch(dqm, false);
373
374 mutex_unlock(&dqm->lock);
375 return retval;
376}
377
378static struct mqd_manager *get_mqd_manager_nocpsch(
379 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
380{
381 struct mqd_manager *mqd;
382
383 BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
384
385 pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
386
387 mqd = dqm->mqds[type];
388 if (!mqd) {
389 mqd = mqd_manager_init(type, dqm->dev);
390 if (mqd == NULL)
391 pr_err("kfd: mqd manager is NULL");
392 dqm->mqds[type] = mqd;
393 }
394
395 return mqd;
396}
397
398static int register_process_nocpsch(struct device_queue_manager *dqm,
399 struct qcm_process_device *qpd)
400{
401 struct device_process_node *n;
402 int retval;
403
404 BUG_ON(!dqm || !qpd);
405
406 pr_debug("kfd: In func %s\n", __func__);
407
408 n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
409 if (!n)
410 return -ENOMEM;
411
412 n->qpd = qpd;
413
414 mutex_lock(&dqm->lock);
415 list_add(&n->list, &dqm->queues);
416
417 retval = dqm->ops_asic_specific.register_process(dqm, qpd);
418
419 dqm->processes_count++;
420
421 mutex_unlock(&dqm->lock);
422
423 return retval;
424}
425
426static int unregister_process_nocpsch(struct device_queue_manager *dqm,
427 struct qcm_process_device *qpd)
428{
429 int retval;
430 struct device_process_node *cur, *next;
431
432 BUG_ON(!dqm || !qpd);
433
434 pr_debug("In func %s\n", __func__);
435
436 pr_debug("qpd->queues_list is %s\n",
437 list_empty(&qpd->queues_list) ? "empty" : "not empty");
438
439 retval = 0;
440 mutex_lock(&dqm->lock);
441
442 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
443 if (qpd == cur->qpd) {
444 list_del(&cur->list);
445 kfree(cur);
446 dqm->processes_count--;
447 goto out;
448 }
449 }
450 /* qpd not found in dqm list */
451 retval = 1;
452out:
453 mutex_unlock(&dqm->lock);
454 return retval;
455}
456
457static int
458set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
459 unsigned int vmid)
460{
461 uint32_t pasid_mapping;
462
463 pasid_mapping = (pasid == 0) ? 0 :
464 (uint32_t)pasid |
465 ATC_VMID_PASID_MAPPING_VALID;
466
467 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
468 dqm->dev->kgd, pasid_mapping,
469 vmid);
470}
471
472int init_pipelines(struct device_queue_manager *dqm,
473 unsigned int pipes_num, unsigned int first_pipe)
474{
475 void *hpdptr;
476 struct mqd_manager *mqd;
477 unsigned int i, err, inx;
478 uint64_t pipe_hpd_addr;
479
480 BUG_ON(!dqm || !dqm->dev);
481
482 pr_debug("kfd: In func %s\n", __func__);
483
484 /*
485 * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
486 * The driver never accesses this memory after zeroing it.
487 * It doesn't even have to be saved/restored on suspend/resume
488 * because it contains no data when there are no active queues.
489 */
490
491 err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num,
492 &dqm->pipeline_mem);
493
494 if (err) {
495 pr_err("kfd: error allocate vidmem num pipes: %d\n",
496 pipes_num);
497 return -ENOMEM;
498 }
499
500 hpdptr = dqm->pipeline_mem->cpu_ptr;
501 dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr;
502
503 memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num);
504
505 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
506 if (mqd == NULL) {
507 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
508 return -ENOMEM;
509 }
510
511 for (i = 0; i < pipes_num; i++) {
512 inx = i + first_pipe;
513 /*
514 * HPD buffer on GTT is allocated by amdkfd, no need to waste
515 * space in GTT for pipelines we don't initialize
516 */
517 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES;
518 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr);
519 /* = log2(bytes/4)-1 */
520 dqm->dev->kfd2kgd->init_pipeline(dqm->dev->kgd, inx,
521 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr);
522 }
523
524 return 0;
525}
526
527static void init_interrupts(struct device_queue_manager *dqm)
528{
529 unsigned int i;
530
531 BUG_ON(dqm == NULL);
532
533 for (i = 0 ; i < get_pipes_num(dqm) ; i++)
534 dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd,
535 i + get_first_pipe(dqm));
536}
537
538static int init_scheduler(struct device_queue_manager *dqm)
539{
540 int retval;
541
542 BUG_ON(!dqm);
543
544 pr_debug("kfd: In %s\n", __func__);
545
546 retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm));
547 return retval;
548}
549
550static int initialize_nocpsch(struct device_queue_manager *dqm)
551{
552 int i;
553
554 BUG_ON(!dqm);
555
556 pr_debug("kfd: In func %s num of pipes: %d\n",
557 __func__, get_pipes_num(dqm));
558
559 mutex_init(&dqm->lock);
560 INIT_LIST_HEAD(&dqm->queues);
561 dqm->queue_count = dqm->next_pipe_to_allocate = 0;
562 dqm->sdma_queue_count = 0;
563 dqm->allocated_queues = kcalloc(get_pipes_num(dqm),
564 sizeof(unsigned int), GFP_KERNEL);
565 if (!dqm->allocated_queues) {
566 mutex_destroy(&dqm->lock);
567 return -ENOMEM;
568 }
569
570 for (i = 0; i < get_pipes_num(dqm); i++)
571 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1;
572
573 dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
574 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
575
576 init_scheduler(dqm);
577 return 0;
578}
579
580static void uninitialize_nocpsch(struct device_queue_manager *dqm)
581{
582 int i;
583
584 BUG_ON(!dqm);
585
586 BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
587
588 kfree(dqm->allocated_queues);
589 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
590 kfree(dqm->mqds[i]);
591 mutex_destroy(&dqm->lock);
592 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
593}
594
595static int start_nocpsch(struct device_queue_manager *dqm)
596{
597 init_interrupts(dqm);
598 return 0;
599}
600
601static int stop_nocpsch(struct device_queue_manager *dqm)
602{
603 return 0;
604}
605
606static int allocate_sdma_queue(struct device_queue_manager *dqm,
607 unsigned int *sdma_queue_id)
608{
609 int bit;
610
611 if (dqm->sdma_bitmap == 0)
612 return -ENOMEM;
613
614 bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap,
615 CIK_SDMA_QUEUES);
616
617 clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap);
618 *sdma_queue_id = bit;
619
620 return 0;
621}
622
623static void deallocate_sdma_queue(struct device_queue_manager *dqm,
624 unsigned int sdma_queue_id)
625{
626 if (sdma_queue_id >= CIK_SDMA_QUEUES)
627 return;
628 set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap);
629}
630
631static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
632 struct queue *q,
633 struct qcm_process_device *qpd)
634{
635 struct mqd_manager *mqd;
636 int retval;
637
638 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
639 if (!mqd)
640 return -ENOMEM;
641
642 retval = allocate_sdma_queue(dqm, &q->sdma_id);
643 if (retval != 0)
644 return retval;
645
646 q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
647 q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
648
649 pr_debug("kfd: sdma id is: %d\n", q->sdma_id);
650 pr_debug(" sdma queue id: %d\n", q->properties.sdma_queue_id);
651 pr_debug(" sdma engine id: %d\n", q->properties.sdma_engine_id);
652
653 dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
654 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
655 &q->gart_mqd_addr, &q->properties);
656 if (retval != 0) {
657 deallocate_sdma_queue(dqm, q->sdma_id);
658 return retval;
659 }
660
661 retval = mqd->load_mqd(mqd, q->mqd, 0,
662 0, NULL);
663 if (retval != 0) {
664 deallocate_sdma_queue(dqm, q->sdma_id);
665 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
666 return retval;
667 }
668
669 return 0;
670}
671
672/*
673 * Device Queue Manager implementation for cp scheduler
674 */
675
676static int set_sched_resources(struct device_queue_manager *dqm)
677{
678 struct scheduling_resources res;
679 unsigned int queue_num, queue_mask;
680
681 BUG_ON(!dqm);
682
683 pr_debug("kfd: In func %s\n", __func__);
684
685 queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE;
686 queue_mask = (1 << queue_num) - 1;
687 res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
688 res.vmid_mask <<= KFD_VMID_START_OFFSET;
689 res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE);
690 res.gws_mask = res.oac_mask = res.gds_heap_base =
691 res.gds_heap_size = 0;
692
693 pr_debug("kfd: scheduling resources:\n"
694 " vmid mask: 0x%8X\n"
695 " queue mask: 0x%8llX\n",
696 res.vmid_mask, res.queue_mask);
697
698 return pm_send_set_resources(&dqm->packets, &res);
699}
700
701static int initialize_cpsch(struct device_queue_manager *dqm)
702{
703 int retval;
704
705 BUG_ON(!dqm);
706
707 pr_debug("kfd: In func %s num of pipes: %d\n",
708 __func__, get_pipes_num_cpsch());
709
710 mutex_init(&dqm->lock);
711 INIT_LIST_HEAD(&dqm->queues);
712 dqm->queue_count = dqm->processes_count = 0;
713 dqm->sdma_queue_count = 0;
714 dqm->active_runlist = false;
715 retval = dqm->ops_asic_specific.initialize(dqm);
716 if (retval != 0)
717 goto fail_init_pipelines;
718
719 return 0;
720
721fail_init_pipelines:
722 mutex_destroy(&dqm->lock);
723 return retval;
724}
725
726static int start_cpsch(struct device_queue_manager *dqm)
727{
728 struct device_process_node *node;
729 int retval;
730
731 BUG_ON(!dqm);
732
733 retval = 0;
734
735 retval = pm_init(&dqm->packets, dqm);
736 if (retval != 0)
737 goto fail_packet_manager_init;
738
739 retval = set_sched_resources(dqm);
740 if (retval != 0)
741 goto fail_set_sched_resources;
742
743 pr_debug("kfd: allocating fence memory\n");
744
745 /* allocate fence memory on the gart */
746 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
747 &dqm->fence_mem);
748
749 if (retval != 0)
750 goto fail_allocate_vidmem;
751
752 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
753 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
754
755 init_interrupts(dqm);
756
757 list_for_each_entry(node, &dqm->queues, list)
758 if (node->qpd->pqm->process && dqm->dev)
759 kfd_bind_process_to_device(dqm->dev,
760 node->qpd->pqm->process);
761
762 execute_queues_cpsch(dqm, true);
763
764 return 0;
765fail_allocate_vidmem:
766fail_set_sched_resources:
767 pm_uninit(&dqm->packets);
768fail_packet_manager_init:
769 return retval;
770}
771
772static int stop_cpsch(struct device_queue_manager *dqm)
773{
774 struct device_process_node *node;
775 struct kfd_process_device *pdd;
776
777 BUG_ON(!dqm);
778
779 destroy_queues_cpsch(dqm, true, true);
780
781 list_for_each_entry(node, &dqm->queues, list) {
782 pdd = qpd_to_pdd(node->qpd);
783 pdd->bound = false;
784 }
785 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
786 pm_uninit(&dqm->packets);
787
788 return 0;
789}
790
791static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
792 struct kernel_queue *kq,
793 struct qcm_process_device *qpd)
794{
795 BUG_ON(!dqm || !kq || !qpd);
796
797 pr_debug("kfd: In func %s\n", __func__);
798
799 mutex_lock(&dqm->lock);
800 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
801 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n",
802 dqm->total_queue_count);
803 mutex_unlock(&dqm->lock);
804 return -EPERM;
805 }
806
807 /*
808 * Unconditionally increment this counter, regardless of the queue's
809 * type or whether the queue is active.
810 */
811 dqm->total_queue_count++;
812 pr_debug("Total of %d queues are accountable so far\n",
813 dqm->total_queue_count);
814
815 list_add(&kq->list, &qpd->priv_queue_list);
816 dqm->queue_count++;
817 qpd->is_debug = true;
818 execute_queues_cpsch(dqm, false);
819 mutex_unlock(&dqm->lock);
820
821 return 0;
822}
823
824static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
825 struct kernel_queue *kq,
826 struct qcm_process_device *qpd)
827{
828 BUG_ON(!dqm || !kq);
829
830 pr_debug("kfd: In %s\n", __func__);
831
832 mutex_lock(&dqm->lock);
833 /* here we actually preempt the DIQ */
834 destroy_queues_cpsch(dqm, true, false);
835 list_del(&kq->list);
836 dqm->queue_count--;
837 qpd->is_debug = false;
838 execute_queues_cpsch(dqm, false);
839 /*
840 * Unconditionally decrement this counter, regardless of the queue's
841 * type.
842 */
843 dqm->total_queue_count--;
844 pr_debug("Total of %d queues are accountable so far\n",
845 dqm->total_queue_count);
846 mutex_unlock(&dqm->lock);
847}
848
849static void select_sdma_engine_id(struct queue *q)
850{
851 static int sdma_id;
852
853 q->sdma_id = sdma_id;
854 sdma_id = (sdma_id + 1) % 2;
855}
856
857static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
858 struct qcm_process_device *qpd, int *allocate_vmid)
859{
860 int retval;
861 struct mqd_manager *mqd;
862
863 BUG_ON(!dqm || !q || !qpd);
864
865 retval = 0;
866
867 if (allocate_vmid)
868 *allocate_vmid = 0;
869
870 mutex_lock(&dqm->lock);
871
872 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
873 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
874 dqm->total_queue_count);
875 retval = -EPERM;
876 goto out;
877 }
878
879 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
880 select_sdma_engine_id(q);
881
882 mqd = dqm->ops.get_mqd_manager(dqm,
883 get_mqd_type_from_queue_type(q->properties.type));
884
885 if (mqd == NULL) {
886 mutex_unlock(&dqm->lock);
887 return -ENOMEM;
888 }
889
890 dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
891 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
892 &q->gart_mqd_addr, &q->properties);
893 if (retval != 0)
894 goto out;
895
896 list_add(&q->list, &qpd->queues_list);
897 if (q->properties.is_active) {
898 dqm->queue_count++;
899 retval = execute_queues_cpsch(dqm, false);
900 }
901
902 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
903 dqm->sdma_queue_count++;
904 /*
905 * Unconditionally increment this counter, regardless of the queue's
906 * type or whether the queue is active.
907 */
908 dqm->total_queue_count++;
909
910 pr_debug("Total of %d queues are accountable so far\n",
911 dqm->total_queue_count);
912
913out:
914 mutex_unlock(&dqm->lock);
915 return retval;
916}
917
918int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
919 unsigned int fence_value,
920 unsigned long timeout)
921{
922 BUG_ON(!fence_addr);
923 timeout += jiffies;
924
925 while (*fence_addr != fence_value) {
926 if (time_after(jiffies, timeout)) {
927 pr_err("kfd: qcm fence wait loop timeout expired\n");
928 return -ETIME;
929 }
930 schedule();
931 }
932
933 return 0;
934}
935
936static int destroy_sdma_queues(struct device_queue_manager *dqm,
937 unsigned int sdma_engine)
938{
939 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
940 KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES, 0, false,
941 sdma_engine);
942}
943
944static int destroy_queues_cpsch(struct device_queue_manager *dqm,
945 bool preempt_static_queues, bool lock)
946{
947 int retval;
948 enum kfd_preempt_type_filter preempt_type;
949 struct kfd_process_device *pdd;
950
951 BUG_ON(!dqm);
952
953 retval = 0;
954
955 if (lock)
956 mutex_lock(&dqm->lock);
957 if (dqm->active_runlist == false)
958 goto out;
959
960 pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n",
961 dqm->sdma_queue_count);
962
963 if (dqm->sdma_queue_count > 0) {
964 destroy_sdma_queues(dqm, 0);
965 destroy_sdma_queues(dqm, 1);
966 }
967
968 preempt_type = preempt_static_queues ?
969 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES :
970 KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES;
971
972 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
973 preempt_type, 0, false, 0);
974 if (retval != 0)
975 goto out;
976
977 *dqm->fence_addr = KFD_FENCE_INIT;
978 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
979 KFD_FENCE_COMPLETED);
980 /* should be timed out */
981 retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
982 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
983 if (retval != 0) {
984 pdd = kfd_get_process_device_data(dqm->dev,
985 kfd_get_process(current));
986 pdd->reset_wavefronts = true;
987 goto out;
988 }
989 pm_release_ib(&dqm->packets);
990 dqm->active_runlist = false;
991
992out:
993 if (lock)
994 mutex_unlock(&dqm->lock);
995 return retval;
996}
997
998static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
999{
1000 int retval;
1001
1002 BUG_ON(!dqm);
1003
1004 if (lock)
1005 mutex_lock(&dqm->lock);
1006
1007 retval = destroy_queues_cpsch(dqm, false, false);
1008 if (retval != 0) {
1009 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
1010 goto out;
1011 }
1012
1013 if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
1014 retval = 0;
1015 goto out;
1016 }
1017
1018 if (dqm->active_runlist) {
1019 retval = 0;
1020 goto out;
1021 }
1022
1023 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1024 if (retval != 0) {
1025 pr_err("kfd: failed to execute runlist");
1026 goto out;
1027 }
1028 dqm->active_runlist = true;
1029
1030out:
1031 if (lock)
1032 mutex_unlock(&dqm->lock);
1033 return retval;
1034}
1035
1036static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1037 struct qcm_process_device *qpd,
1038 struct queue *q)
1039{
1040 int retval;
1041 struct mqd_manager *mqd;
1042 bool preempt_all_queues;
1043
1044 BUG_ON(!dqm || !qpd || !q);
1045
1046 preempt_all_queues = false;
1047
1048 retval = 0;
1049
1050 /* remove queue from list to prevent rescheduling after preemption */
1051 mutex_lock(&dqm->lock);
1052
1053 if (qpd->is_debug) {
1054 /*
1055 * error, currently we do not allow to destroy a queue
1056 * of a currently debugged process
1057 */
1058 retval = -EBUSY;
1059 goto failed_try_destroy_debugged_queue;
1060
1061 }
1062
1063 mqd = dqm->ops.get_mqd_manager(dqm,
1064 get_mqd_type_from_queue_type(q->properties.type));
1065 if (!mqd) {
1066 retval = -ENOMEM;
1067 goto failed;
1068 }
1069
1070 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1071 dqm->sdma_queue_count--;
1072
1073 list_del(&q->list);
1074 if (q->properties.is_active)
1075 dqm->queue_count--;
1076
1077 execute_queues_cpsch(dqm, false);
1078
1079 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1080
1081 /*
1082 * Unconditionally decrement this counter, regardless of the queue's
1083 * type
1084 */
1085 dqm->total_queue_count--;
1086 pr_debug("Total of %d queues are accountable so far\n",
1087 dqm->total_queue_count);
1088
1089 mutex_unlock(&dqm->lock);
1090
1091 return 0;
1092
1093failed:
1094failed_try_destroy_debugged_queue:
1095
1096 mutex_unlock(&dqm->lock);
1097 return retval;
1098}
1099
1100/*
1101 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1102 * stay in user mode.
1103 */
1104#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1105/* APE1 limit is inclusive and 64K aligned. */
1106#define APE1_LIMIT_ALIGNMENT 0xFFFF
1107
1108static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1109 struct qcm_process_device *qpd,
1110 enum cache_policy default_policy,
1111 enum cache_policy alternate_policy,
1112 void __user *alternate_aperture_base,
1113 uint64_t alternate_aperture_size)
1114{
1115 bool retval;
1116
1117 pr_debug("kfd: In func %s\n", __func__);
1118
1119 mutex_lock(&dqm->lock);
1120
1121 if (alternate_aperture_size == 0) {
1122 /* base > limit disables APE1 */
1123 qpd->sh_mem_ape1_base = 1;
1124 qpd->sh_mem_ape1_limit = 0;
1125 } else {
1126 /*
1127 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1128 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1129 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1130 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1131 * Verify that the base and size parameters can be
1132 * represented in this format and convert them.
1133 * Additionally restrict APE1 to user-mode addresses.
1134 */
1135
1136 uint64_t base = (uintptr_t)alternate_aperture_base;
1137 uint64_t limit = base + alternate_aperture_size - 1;
1138
1139 if (limit <= base)
1140 goto out;
1141
1142 if ((base & APE1_FIXED_BITS_MASK) != 0)
1143 goto out;
1144
1145 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
1146 goto out;
1147
1148 qpd->sh_mem_ape1_base = base >> 16;
1149 qpd->sh_mem_ape1_limit = limit >> 16;
1150 }
1151
1152 retval = dqm->ops_asic_specific.set_cache_memory_policy(
1153 dqm,
1154 qpd,
1155 default_policy,
1156 alternate_policy,
1157 alternate_aperture_base,
1158 alternate_aperture_size);
1159
1160 if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
1161 program_sh_mem_settings(dqm, qpd);
1162
1163 pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
1164 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1165 qpd->sh_mem_ape1_limit);
1166
1167 mutex_unlock(&dqm->lock);
1168 return retval;
1169
1170out:
1171 mutex_unlock(&dqm->lock);
1172 return false;
1173}
1174
1175struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1176{
1177 struct device_queue_manager *dqm;
1178
1179 BUG_ON(!dev);
1180
1181 pr_debug("kfd: loading device queue manager\n");
1182
1183 dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
1184 if (!dqm)
1185 return NULL;
1186
1187 dqm->dev = dev;
1188 switch (sched_policy) {
1189 case KFD_SCHED_POLICY_HWS:
1190 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1191 /* initialize dqm for cp scheduling */
1192 dqm->ops.create_queue = create_queue_cpsch;
1193 dqm->ops.initialize = initialize_cpsch;
1194 dqm->ops.start = start_cpsch;
1195 dqm->ops.stop = stop_cpsch;
1196 dqm->ops.destroy_queue = destroy_queue_cpsch;
1197 dqm->ops.update_queue = update_queue;
1198 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1199 dqm->ops.register_process = register_process_nocpsch;
1200 dqm->ops.unregister_process = unregister_process_nocpsch;
1201 dqm->ops.uninitialize = uninitialize_nocpsch;
1202 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1203 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1204 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1205 break;
1206 case KFD_SCHED_POLICY_NO_HWS:
1207 /* initialize dqm for no cp scheduling */
1208 dqm->ops.start = start_nocpsch;
1209 dqm->ops.stop = stop_nocpsch;
1210 dqm->ops.create_queue = create_queue_nocpsch;
1211 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1212 dqm->ops.update_queue = update_queue;
1213 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1214 dqm->ops.register_process = register_process_nocpsch;
1215 dqm->ops.unregister_process = unregister_process_nocpsch;
1216 dqm->ops.initialize = initialize_nocpsch;
1217 dqm->ops.uninitialize = uninitialize_nocpsch;
1218 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1219 break;
1220 default:
1221 BUG();
1222 break;
1223 }
1224
1225 switch (dev->device_info->asic_family) {
1226 case CHIP_CARRIZO:
1227 device_queue_manager_init_vi(&dqm->ops_asic_specific);
1228 break;
1229
1230 case CHIP_KAVERI:
1231 device_queue_manager_init_cik(&dqm->ops_asic_specific);
1232 break;
1233 }
1234
1235 if (dqm->ops.initialize(dqm) != 0) {
1236 kfree(dqm);
1237 return NULL;
1238 }
1239
1240 return dqm;
1241}
1242
1243void device_queue_manager_uninit(struct device_queue_manager *dqm)
1244{
1245 BUG_ON(!dqm);
1246
1247 dqm->ops.uninitialize(dqm);
1248 kfree(dqm);
1249}
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/ratelimit.h>
25#include <linux/printk.h>
26#include <linux/slab.h>
27#include <linux/list.h>
28#include <linux/types.h>
29#include <linux/bitops.h>
30#include <linux/sched.h>
31#include "kfd_priv.h"
32#include "kfd_device_queue_manager.h"
33#include "kfd_mqd_manager.h"
34#include "cik_regs.h"
35#include "kfd_kernel_queue.h"
36
37/* Size of the per-pipe EOP queue */
38#define CIK_HPD_EOP_BYTES_LOG2 11
39#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
40
41static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
42 unsigned int pasid, unsigned int vmid);
43
44static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
45 struct queue *q,
46 struct qcm_process_device *qpd);
47
48static int execute_queues_cpsch(struct device_queue_manager *dqm,
49 enum kfd_unmap_queues_filter filter,
50 uint32_t filter_param);
51static int unmap_queues_cpsch(struct device_queue_manager *dqm,
52 enum kfd_unmap_queues_filter filter,
53 uint32_t filter_param);
54
55static int map_queues_cpsch(struct device_queue_manager *dqm);
56
57static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
58 struct queue *q,
59 struct qcm_process_device *qpd);
60
61static void deallocate_sdma_queue(struct device_queue_manager *dqm,
62 unsigned int sdma_queue_id);
63
64static inline
65enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
66{
67 if (type == KFD_QUEUE_TYPE_SDMA)
68 return KFD_MQD_TYPE_SDMA;
69 return KFD_MQD_TYPE_CP;
70}
71
72static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
73{
74 int i;
75 int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec
76 + pipe * dqm->dev->shared_resources.num_queue_per_pipe;
77
78 /* queue is available for KFD usage if bit is 1 */
79 for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i)
80 if (test_bit(pipe_offset + i,
81 dqm->dev->shared_resources.queue_bitmap))
82 return true;
83 return false;
84}
85
86unsigned int get_queues_num(struct device_queue_manager *dqm)
87{
88 return bitmap_weight(dqm->dev->shared_resources.queue_bitmap,
89 KGD_MAX_QUEUES);
90}
91
92unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
93{
94 return dqm->dev->shared_resources.num_queue_per_pipe;
95}
96
97unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
98{
99 return dqm->dev->shared_resources.num_pipe_per_mec;
100}
101
102void program_sh_mem_settings(struct device_queue_manager *dqm,
103 struct qcm_process_device *qpd)
104{
105 return dqm->dev->kfd2kgd->program_sh_mem_settings(
106 dqm->dev->kgd, qpd->vmid,
107 qpd->sh_mem_config,
108 qpd->sh_mem_ape1_base,
109 qpd->sh_mem_ape1_limit,
110 qpd->sh_mem_bases);
111}
112
113static int allocate_vmid(struct device_queue_manager *dqm,
114 struct qcm_process_device *qpd,
115 struct queue *q)
116{
117 int bit, allocated_vmid;
118
119 if (dqm->vmid_bitmap == 0)
120 return -ENOMEM;
121
122 bit = ffs(dqm->vmid_bitmap) - 1;
123 dqm->vmid_bitmap &= ~(1 << bit);
124
125 allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd;
126 pr_debug("vmid allocation %d\n", allocated_vmid);
127 qpd->vmid = allocated_vmid;
128 q->properties.vmid = allocated_vmid;
129
130 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
131 program_sh_mem_settings(dqm, qpd);
132
133 /* qpd->page_table_base is set earlier when register_process()
134 * is called, i.e. when the first queue is created.
135 */
136 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd,
137 qpd->vmid,
138 qpd->page_table_base);
139 /* invalidate the VM context after pasid and vmid mapping is set up */
140 kfd_flush_tlb(qpd_to_pdd(qpd));
141
142 return 0;
143}
144
145static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
146 struct qcm_process_device *qpd)
147{
148 uint32_t len;
149
150 if (!qpd->ib_kaddr)
151 return -ENOMEM;
152
153 len = pm_create_release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
154
155 return kdev->kfd2kgd->submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid,
156 qpd->ib_base, (uint32_t *)qpd->ib_kaddr, len);
157}
158
159static void deallocate_vmid(struct device_queue_manager *dqm,
160 struct qcm_process_device *qpd,
161 struct queue *q)
162{
163 int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd;
164
165 /* On GFX v7, CP doesn't flush TC at dequeue */
166 if (q->device->device_info->asic_family == CHIP_HAWAII)
167 if (flush_texture_cache_nocpsch(q->device, qpd))
168 pr_err("Failed to flush TC\n");
169
170 kfd_flush_tlb(qpd_to_pdd(qpd));
171
172 /* Release the vmid mapping */
173 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
174
175 dqm->vmid_bitmap |= (1 << bit);
176 qpd->vmid = 0;
177 q->properties.vmid = 0;
178}
179
180static int create_queue_nocpsch(struct device_queue_manager *dqm,
181 struct queue *q,
182 struct qcm_process_device *qpd)
183{
184 int retval;
185
186 print_queue(q);
187
188 mutex_lock(&dqm->lock);
189
190 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
191 pr_warn("Can't create new usermode queue because %d queues were already created\n",
192 dqm->total_queue_count);
193 retval = -EPERM;
194 goto out_unlock;
195 }
196
197 if (list_empty(&qpd->queues_list)) {
198 retval = allocate_vmid(dqm, qpd, q);
199 if (retval)
200 goto out_unlock;
201 }
202 q->properties.vmid = qpd->vmid;
203 /*
204 * Eviction state logic: we only mark active queues as evicted
205 * to avoid the overhead of restoring inactive queues later
206 */
207 if (qpd->evicted)
208 q->properties.is_evicted = (q->properties.queue_size > 0 &&
209 q->properties.queue_percent > 0 &&
210 q->properties.queue_address != 0);
211
212 q->properties.tba_addr = qpd->tba_addr;
213 q->properties.tma_addr = qpd->tma_addr;
214
215 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
216 retval = create_compute_queue_nocpsch(dqm, q, qpd);
217 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
218 retval = create_sdma_queue_nocpsch(dqm, q, qpd);
219 else
220 retval = -EINVAL;
221
222 if (retval) {
223 if (list_empty(&qpd->queues_list))
224 deallocate_vmid(dqm, qpd, q);
225 goto out_unlock;
226 }
227
228 list_add(&q->list, &qpd->queues_list);
229 qpd->queue_count++;
230 if (q->properties.is_active)
231 dqm->queue_count++;
232
233 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
234 dqm->sdma_queue_count++;
235
236 /*
237 * Unconditionally increment this counter, regardless of the queue's
238 * type or whether the queue is active.
239 */
240 dqm->total_queue_count++;
241 pr_debug("Total of %d queues are accountable so far\n",
242 dqm->total_queue_count);
243
244out_unlock:
245 mutex_unlock(&dqm->lock);
246 return retval;
247}
248
249static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
250{
251 bool set;
252 int pipe, bit, i;
253
254 set = false;
255
256 for (pipe = dqm->next_pipe_to_allocate, i = 0;
257 i < get_pipes_per_mec(dqm);
258 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
259
260 if (!is_pipe_enabled(dqm, 0, pipe))
261 continue;
262
263 if (dqm->allocated_queues[pipe] != 0) {
264 bit = ffs(dqm->allocated_queues[pipe]) - 1;
265 dqm->allocated_queues[pipe] &= ~(1 << bit);
266 q->pipe = pipe;
267 q->queue = bit;
268 set = true;
269 break;
270 }
271 }
272
273 if (!set)
274 return -EBUSY;
275
276 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
277 /* horizontal hqd allocation */
278 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
279
280 return 0;
281}
282
283static inline void deallocate_hqd(struct device_queue_manager *dqm,
284 struct queue *q)
285{
286 dqm->allocated_queues[q->pipe] |= (1 << q->queue);
287}
288
289static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
290 struct queue *q,
291 struct qcm_process_device *qpd)
292{
293 int retval;
294 struct mqd_manager *mqd;
295
296 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
297 if (!mqd)
298 return -ENOMEM;
299
300 retval = allocate_hqd(dqm, q);
301 if (retval)
302 return retval;
303
304 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
305 &q->gart_mqd_addr, &q->properties);
306 if (retval)
307 goto out_deallocate_hqd;
308
309 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
310 q->pipe, q->queue);
311
312 dqm->dev->kfd2kgd->set_scratch_backing_va(
313 dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
314
315 if (!q->properties.is_active)
316 return 0;
317
318 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue, &q->properties,
319 q->process->mm);
320 if (retval)
321 goto out_uninit_mqd;
322
323 return 0;
324
325out_uninit_mqd:
326 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
327out_deallocate_hqd:
328 deallocate_hqd(dqm, q);
329
330 return retval;
331}
332
333/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
334 * to avoid asynchronized access
335 */
336static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
337 struct qcm_process_device *qpd,
338 struct queue *q)
339{
340 int retval;
341 struct mqd_manager *mqd;
342
343 mqd = dqm->ops.get_mqd_manager(dqm,
344 get_mqd_type_from_queue_type(q->properties.type));
345 if (!mqd)
346 return -ENOMEM;
347
348 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
349 deallocate_hqd(dqm, q);
350 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
351 dqm->sdma_queue_count--;
352 deallocate_sdma_queue(dqm, q->sdma_id);
353 } else {
354 pr_debug("q->properties.type %d is invalid\n",
355 q->properties.type);
356 return -EINVAL;
357 }
358 dqm->total_queue_count--;
359
360 retval = mqd->destroy_mqd(mqd, q->mqd,
361 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
362 KFD_UNMAP_LATENCY_MS,
363 q->pipe, q->queue);
364 if (retval == -ETIME)
365 qpd->reset_wavefronts = true;
366
367 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
368
369 list_del(&q->list);
370 if (list_empty(&qpd->queues_list)) {
371 if (qpd->reset_wavefronts) {
372 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
373 dqm->dev);
374 /* dbgdev_wave_reset_wavefronts has to be called before
375 * deallocate_vmid(), i.e. when vmid is still in use.
376 */
377 dbgdev_wave_reset_wavefronts(dqm->dev,
378 qpd->pqm->process);
379 qpd->reset_wavefronts = false;
380 }
381
382 deallocate_vmid(dqm, qpd, q);
383 }
384 qpd->queue_count--;
385 if (q->properties.is_active)
386 dqm->queue_count--;
387
388 return retval;
389}
390
391static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
392 struct qcm_process_device *qpd,
393 struct queue *q)
394{
395 int retval;
396
397 mutex_lock(&dqm->lock);
398 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
399 mutex_unlock(&dqm->lock);
400
401 return retval;
402}
403
404static int update_queue(struct device_queue_manager *dqm, struct queue *q)
405{
406 int retval;
407 struct mqd_manager *mqd;
408 struct kfd_process_device *pdd;
409 bool prev_active = false;
410
411 mutex_lock(&dqm->lock);
412 pdd = kfd_get_process_device_data(q->device, q->process);
413 if (!pdd) {
414 retval = -ENODEV;
415 goto out_unlock;
416 }
417 mqd = dqm->ops.get_mqd_manager(dqm,
418 get_mqd_type_from_queue_type(q->properties.type));
419 if (!mqd) {
420 retval = -ENOMEM;
421 goto out_unlock;
422 }
423 /*
424 * Eviction state logic: we only mark active queues as evicted
425 * to avoid the overhead of restoring inactive queues later
426 */
427 if (pdd->qpd.evicted)
428 q->properties.is_evicted = (q->properties.queue_size > 0 &&
429 q->properties.queue_percent > 0 &&
430 q->properties.queue_address != 0);
431
432 /* Save previous activity state for counters */
433 prev_active = q->properties.is_active;
434
435 /* Make sure the queue is unmapped before updating the MQD */
436 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
437 retval = unmap_queues_cpsch(dqm,
438 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
439 if (retval) {
440 pr_err("unmap queue failed\n");
441 goto out_unlock;
442 }
443 } else if (prev_active &&
444 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
445 q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
446 retval = mqd->destroy_mqd(mqd, q->mqd,
447 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
448 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
449 if (retval) {
450 pr_err("destroy mqd failed\n");
451 goto out_unlock;
452 }
453 }
454
455 retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
456
457 /*
458 * check active state vs. the previous state and modify
459 * counter accordingly. map_queues_cpsch uses the
460 * dqm->queue_count to determine whether a new runlist must be
461 * uploaded.
462 */
463 if (q->properties.is_active && !prev_active)
464 dqm->queue_count++;
465 else if (!q->properties.is_active && prev_active)
466 dqm->queue_count--;
467
468 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
469 retval = map_queues_cpsch(dqm);
470 else if (q->properties.is_active &&
471 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
472 q->properties.type == KFD_QUEUE_TYPE_SDMA))
473 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue,
474 &q->properties, q->process->mm);
475
476out_unlock:
477 mutex_unlock(&dqm->lock);
478 return retval;
479}
480
481static struct mqd_manager *get_mqd_manager(
482 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
483{
484 struct mqd_manager *mqd;
485
486 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
487 return NULL;
488
489 pr_debug("mqd type %d\n", type);
490
491 mqd = dqm->mqds[type];
492 if (!mqd) {
493 mqd = mqd_manager_init(type, dqm->dev);
494 if (!mqd)
495 pr_err("mqd manager is NULL");
496 dqm->mqds[type] = mqd;
497 }
498
499 return mqd;
500}
501
502static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
503 struct qcm_process_device *qpd)
504{
505 struct queue *q;
506 struct mqd_manager *mqd;
507 struct kfd_process_device *pdd;
508 int retval = 0;
509
510 mutex_lock(&dqm->lock);
511 if (qpd->evicted++ > 0) /* already evicted, do nothing */
512 goto out;
513
514 pdd = qpd_to_pdd(qpd);
515 pr_info_ratelimited("Evicting PASID %u queues\n",
516 pdd->process->pasid);
517
518 /* unactivate all active queues on the qpd */
519 list_for_each_entry(q, &qpd->queues_list, list) {
520 if (!q->properties.is_active)
521 continue;
522 mqd = dqm->ops.get_mqd_manager(dqm,
523 get_mqd_type_from_queue_type(q->properties.type));
524 if (!mqd) { /* should not be here */
525 pr_err("Cannot evict queue, mqd mgr is NULL\n");
526 retval = -ENOMEM;
527 goto out;
528 }
529 q->properties.is_evicted = true;
530 q->properties.is_active = false;
531 retval = mqd->destroy_mqd(mqd, q->mqd,
532 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
533 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
534 if (retval)
535 goto out;
536 dqm->queue_count--;
537 }
538
539out:
540 mutex_unlock(&dqm->lock);
541 return retval;
542}
543
544static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
545 struct qcm_process_device *qpd)
546{
547 struct queue *q;
548 struct kfd_process_device *pdd;
549 int retval = 0;
550
551 mutex_lock(&dqm->lock);
552 if (qpd->evicted++ > 0) /* already evicted, do nothing */
553 goto out;
554
555 pdd = qpd_to_pdd(qpd);
556 pr_info_ratelimited("Evicting PASID %u queues\n",
557 pdd->process->pasid);
558
559 /* unactivate all active queues on the qpd */
560 list_for_each_entry(q, &qpd->queues_list, list) {
561 if (!q->properties.is_active)
562 continue;
563 q->properties.is_evicted = true;
564 q->properties.is_active = false;
565 dqm->queue_count--;
566 }
567 retval = execute_queues_cpsch(dqm,
568 qpd->is_debug ?
569 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
570 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
571
572out:
573 mutex_unlock(&dqm->lock);
574 return retval;
575}
576
577static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
578 struct qcm_process_device *qpd)
579{
580 struct queue *q;
581 struct mqd_manager *mqd;
582 struct kfd_process_device *pdd;
583 uint32_t pd_base;
584 int retval = 0;
585
586 pdd = qpd_to_pdd(qpd);
587 /* Retrieve PD base */
588 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
589
590 mutex_lock(&dqm->lock);
591 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
592 goto out;
593 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
594 qpd->evicted--;
595 goto out;
596 }
597
598 pr_info_ratelimited("Restoring PASID %u queues\n",
599 pdd->process->pasid);
600
601 /* Update PD Base in QPD */
602 qpd->page_table_base = pd_base;
603 pr_debug("Updated PD address to 0x%08x\n", pd_base);
604
605 if (!list_empty(&qpd->queues_list)) {
606 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
607 dqm->dev->kgd,
608 qpd->vmid,
609 qpd->page_table_base);
610 kfd_flush_tlb(pdd);
611 }
612
613 /* activate all active queues on the qpd */
614 list_for_each_entry(q, &qpd->queues_list, list) {
615 if (!q->properties.is_evicted)
616 continue;
617 mqd = dqm->ops.get_mqd_manager(dqm,
618 get_mqd_type_from_queue_type(q->properties.type));
619 if (!mqd) { /* should not be here */
620 pr_err("Cannot restore queue, mqd mgr is NULL\n");
621 retval = -ENOMEM;
622 goto out;
623 }
624 q->properties.is_evicted = false;
625 q->properties.is_active = true;
626 retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
627 q->queue, &q->properties,
628 q->process->mm);
629 if (retval)
630 goto out;
631 dqm->queue_count++;
632 }
633 qpd->evicted = 0;
634out:
635 mutex_unlock(&dqm->lock);
636 return retval;
637}
638
639static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
640 struct qcm_process_device *qpd)
641{
642 struct queue *q;
643 struct kfd_process_device *pdd;
644 uint32_t pd_base;
645 int retval = 0;
646
647 pdd = qpd_to_pdd(qpd);
648 /* Retrieve PD base */
649 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
650
651 mutex_lock(&dqm->lock);
652 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
653 goto out;
654 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
655 qpd->evicted--;
656 goto out;
657 }
658
659 pr_info_ratelimited("Restoring PASID %u queues\n",
660 pdd->process->pasid);
661
662 /* Update PD Base in QPD */
663 qpd->page_table_base = pd_base;
664 pr_debug("Updated PD address to 0x%08x\n", pd_base);
665
666 /* activate all active queues on the qpd */
667 list_for_each_entry(q, &qpd->queues_list, list) {
668 if (!q->properties.is_evicted)
669 continue;
670 q->properties.is_evicted = false;
671 q->properties.is_active = true;
672 dqm->queue_count++;
673 }
674 retval = execute_queues_cpsch(dqm,
675 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
676 if (!retval)
677 qpd->evicted = 0;
678out:
679 mutex_unlock(&dqm->lock);
680 return retval;
681}
682
683static int register_process(struct device_queue_manager *dqm,
684 struct qcm_process_device *qpd)
685{
686 struct device_process_node *n;
687 struct kfd_process_device *pdd;
688 uint32_t pd_base;
689 int retval;
690
691 n = kzalloc(sizeof(*n), GFP_KERNEL);
692 if (!n)
693 return -ENOMEM;
694
695 n->qpd = qpd;
696
697 pdd = qpd_to_pdd(qpd);
698 /* Retrieve PD base */
699 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
700
701 mutex_lock(&dqm->lock);
702 list_add(&n->list, &dqm->queues);
703
704 /* Update PD Base in QPD */
705 qpd->page_table_base = pd_base;
706
707 retval = dqm->asic_ops.update_qpd(dqm, qpd);
708
709 dqm->processes_count++;
710
711 mutex_unlock(&dqm->lock);
712
713 return retval;
714}
715
716static int unregister_process(struct device_queue_manager *dqm,
717 struct qcm_process_device *qpd)
718{
719 int retval;
720 struct device_process_node *cur, *next;
721
722 pr_debug("qpd->queues_list is %s\n",
723 list_empty(&qpd->queues_list) ? "empty" : "not empty");
724
725 retval = 0;
726 mutex_lock(&dqm->lock);
727
728 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
729 if (qpd == cur->qpd) {
730 list_del(&cur->list);
731 kfree(cur);
732 dqm->processes_count--;
733 goto out;
734 }
735 }
736 /* qpd not found in dqm list */
737 retval = 1;
738out:
739 mutex_unlock(&dqm->lock);
740 return retval;
741}
742
743static int
744set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
745 unsigned int vmid)
746{
747 uint32_t pasid_mapping;
748
749 pasid_mapping = (pasid == 0) ? 0 :
750 (uint32_t)pasid |
751 ATC_VMID_PASID_MAPPING_VALID;
752
753 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
754 dqm->dev->kgd, pasid_mapping,
755 vmid);
756}
757
758static void init_interrupts(struct device_queue_manager *dqm)
759{
760 unsigned int i;
761
762 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
763 if (is_pipe_enabled(dqm, 0, i))
764 dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
765}
766
767static int initialize_nocpsch(struct device_queue_manager *dqm)
768{
769 int pipe, queue;
770
771 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
772
773 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
774 sizeof(unsigned int), GFP_KERNEL);
775 if (!dqm->allocated_queues)
776 return -ENOMEM;
777
778 mutex_init(&dqm->lock);
779 INIT_LIST_HEAD(&dqm->queues);
780 dqm->queue_count = dqm->next_pipe_to_allocate = 0;
781 dqm->sdma_queue_count = 0;
782
783 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
784 int pipe_offset = pipe * get_queues_per_pipe(dqm);
785
786 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
787 if (test_bit(pipe_offset + queue,
788 dqm->dev->shared_resources.queue_bitmap))
789 dqm->allocated_queues[pipe] |= 1 << queue;
790 }
791
792 dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
793 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
794
795 return 0;
796}
797
798static void uninitialize(struct device_queue_manager *dqm)
799{
800 int i;
801
802 WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
803
804 kfree(dqm->allocated_queues);
805 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
806 kfree(dqm->mqds[i]);
807 mutex_destroy(&dqm->lock);
808 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
809}
810
811static int start_nocpsch(struct device_queue_manager *dqm)
812{
813 init_interrupts(dqm);
814 return pm_init(&dqm->packets, dqm);
815}
816
817static int stop_nocpsch(struct device_queue_manager *dqm)
818{
819 pm_uninit(&dqm->packets);
820 return 0;
821}
822
823static int allocate_sdma_queue(struct device_queue_manager *dqm,
824 unsigned int *sdma_queue_id)
825{
826 int bit;
827
828 if (dqm->sdma_bitmap == 0)
829 return -ENOMEM;
830
831 bit = ffs(dqm->sdma_bitmap) - 1;
832 dqm->sdma_bitmap &= ~(1 << bit);
833 *sdma_queue_id = bit;
834
835 return 0;
836}
837
838static void deallocate_sdma_queue(struct device_queue_manager *dqm,
839 unsigned int sdma_queue_id)
840{
841 if (sdma_queue_id >= CIK_SDMA_QUEUES)
842 return;
843 dqm->sdma_bitmap |= (1 << sdma_queue_id);
844}
845
846static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
847 struct queue *q,
848 struct qcm_process_device *qpd)
849{
850 struct mqd_manager *mqd;
851 int retval;
852
853 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
854 if (!mqd)
855 return -ENOMEM;
856
857 retval = allocate_sdma_queue(dqm, &q->sdma_id);
858 if (retval)
859 return retval;
860
861 q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
862 q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
863
864 pr_debug("SDMA id is: %d\n", q->sdma_id);
865 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
866 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
867
868 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
869 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
870 &q->gart_mqd_addr, &q->properties);
871 if (retval)
872 goto out_deallocate_sdma_queue;
873
874 retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);
875 if (retval)
876 goto out_uninit_mqd;
877
878 return 0;
879
880out_uninit_mqd:
881 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
882out_deallocate_sdma_queue:
883 deallocate_sdma_queue(dqm, q->sdma_id);
884
885 return retval;
886}
887
888/*
889 * Device Queue Manager implementation for cp scheduler
890 */
891
892static int set_sched_resources(struct device_queue_manager *dqm)
893{
894 int i, mec;
895 struct scheduling_resources res;
896
897 res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
898
899 res.queue_mask = 0;
900 for (i = 0; i < KGD_MAX_QUEUES; ++i) {
901 mec = (i / dqm->dev->shared_resources.num_queue_per_pipe)
902 / dqm->dev->shared_resources.num_pipe_per_mec;
903
904 if (!test_bit(i, dqm->dev->shared_resources.queue_bitmap))
905 continue;
906
907 /* only acquire queues from the first MEC */
908 if (mec > 0)
909 continue;
910
911 /* This situation may be hit in the future if a new HW
912 * generation exposes more than 64 queues. If so, the
913 * definition of res.queue_mask needs updating
914 */
915 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
916 pr_err("Invalid queue enabled by amdgpu: %d\n", i);
917 break;
918 }
919
920 res.queue_mask |= (1ull << i);
921 }
922 res.gws_mask = res.oac_mask = res.gds_heap_base =
923 res.gds_heap_size = 0;
924
925 pr_debug("Scheduling resources:\n"
926 "vmid mask: 0x%8X\n"
927 "queue mask: 0x%8llX\n",
928 res.vmid_mask, res.queue_mask);
929
930 return pm_send_set_resources(&dqm->packets, &res);
931}
932
933static int initialize_cpsch(struct device_queue_manager *dqm)
934{
935 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
936
937 mutex_init(&dqm->lock);
938 INIT_LIST_HEAD(&dqm->queues);
939 dqm->queue_count = dqm->processes_count = 0;
940 dqm->sdma_queue_count = 0;
941 dqm->active_runlist = false;
942 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
943
944 return 0;
945}
946
947static int start_cpsch(struct device_queue_manager *dqm)
948{
949 int retval;
950
951 retval = 0;
952
953 retval = pm_init(&dqm->packets, dqm);
954 if (retval)
955 goto fail_packet_manager_init;
956
957 retval = set_sched_resources(dqm);
958 if (retval)
959 goto fail_set_sched_resources;
960
961 pr_debug("Allocating fence memory\n");
962
963 /* allocate fence memory on the gart */
964 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
965 &dqm->fence_mem);
966
967 if (retval)
968 goto fail_allocate_vidmem;
969
970 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
971 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
972
973 init_interrupts(dqm);
974
975 mutex_lock(&dqm->lock);
976 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
977 mutex_unlock(&dqm->lock);
978
979 return 0;
980fail_allocate_vidmem:
981fail_set_sched_resources:
982 pm_uninit(&dqm->packets);
983fail_packet_manager_init:
984 return retval;
985}
986
987static int stop_cpsch(struct device_queue_manager *dqm)
988{
989 mutex_lock(&dqm->lock);
990 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
991 mutex_unlock(&dqm->lock);
992
993 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
994 pm_uninit(&dqm->packets);
995
996 return 0;
997}
998
999static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1000 struct kernel_queue *kq,
1001 struct qcm_process_device *qpd)
1002{
1003 mutex_lock(&dqm->lock);
1004 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1005 pr_warn("Can't create new kernel queue because %d queues were already created\n",
1006 dqm->total_queue_count);
1007 mutex_unlock(&dqm->lock);
1008 return -EPERM;
1009 }
1010
1011 /*
1012 * Unconditionally increment this counter, regardless of the queue's
1013 * type or whether the queue is active.
1014 */
1015 dqm->total_queue_count++;
1016 pr_debug("Total of %d queues are accountable so far\n",
1017 dqm->total_queue_count);
1018
1019 list_add(&kq->list, &qpd->priv_queue_list);
1020 dqm->queue_count++;
1021 qpd->is_debug = true;
1022 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
1023 mutex_unlock(&dqm->lock);
1024
1025 return 0;
1026}
1027
1028static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1029 struct kernel_queue *kq,
1030 struct qcm_process_device *qpd)
1031{
1032 mutex_lock(&dqm->lock);
1033 list_del(&kq->list);
1034 dqm->queue_count--;
1035 qpd->is_debug = false;
1036 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
1037 /*
1038 * Unconditionally decrement this counter, regardless of the queue's
1039 * type.
1040 */
1041 dqm->total_queue_count--;
1042 pr_debug("Total of %d queues are accountable so far\n",
1043 dqm->total_queue_count);
1044 mutex_unlock(&dqm->lock);
1045}
1046
1047static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
1048 struct qcm_process_device *qpd)
1049{
1050 int retval;
1051 struct mqd_manager *mqd;
1052
1053 retval = 0;
1054
1055 mutex_lock(&dqm->lock);
1056
1057 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1058 pr_warn("Can't create new usermode queue because %d queues were already created\n",
1059 dqm->total_queue_count);
1060 retval = -EPERM;
1061 goto out_unlock;
1062 }
1063
1064 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1065 retval = allocate_sdma_queue(dqm, &q->sdma_id);
1066 if (retval)
1067 goto out_unlock;
1068 q->properties.sdma_queue_id =
1069 q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
1070 q->properties.sdma_engine_id =
1071 q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
1072 }
1073 mqd = dqm->ops.get_mqd_manager(dqm,
1074 get_mqd_type_from_queue_type(q->properties.type));
1075
1076 if (!mqd) {
1077 retval = -ENOMEM;
1078 goto out_deallocate_sdma_queue;
1079 }
1080 /*
1081 * Eviction state logic: we only mark active queues as evicted
1082 * to avoid the overhead of restoring inactive queues later
1083 */
1084 if (qpd->evicted)
1085 q->properties.is_evicted = (q->properties.queue_size > 0 &&
1086 q->properties.queue_percent > 0 &&
1087 q->properties.queue_address != 0);
1088
1089 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
1090
1091 q->properties.tba_addr = qpd->tba_addr;
1092 q->properties.tma_addr = qpd->tma_addr;
1093 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
1094 &q->gart_mqd_addr, &q->properties);
1095 if (retval)
1096 goto out_deallocate_sdma_queue;
1097
1098 list_add(&q->list, &qpd->queues_list);
1099 qpd->queue_count++;
1100 if (q->properties.is_active) {
1101 dqm->queue_count++;
1102 retval = execute_queues_cpsch(dqm,
1103 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
1104 }
1105
1106 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1107 dqm->sdma_queue_count++;
1108 /*
1109 * Unconditionally increment this counter, regardless of the queue's
1110 * type or whether the queue is active.
1111 */
1112 dqm->total_queue_count++;
1113
1114 pr_debug("Total of %d queues are accountable so far\n",
1115 dqm->total_queue_count);
1116
1117 mutex_unlock(&dqm->lock);
1118 return retval;
1119
1120out_deallocate_sdma_queue:
1121 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1122 deallocate_sdma_queue(dqm, q->sdma_id);
1123out_unlock:
1124 mutex_unlock(&dqm->lock);
1125 return retval;
1126}
1127
1128int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
1129 unsigned int fence_value,
1130 unsigned int timeout_ms)
1131{
1132 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
1133
1134 while (*fence_addr != fence_value) {
1135 if (time_after(jiffies, end_jiffies)) {
1136 pr_err("qcm fence wait loop timeout expired\n");
1137 return -ETIME;
1138 }
1139 schedule();
1140 }
1141
1142 return 0;
1143}
1144
1145static int unmap_sdma_queues(struct device_queue_manager *dqm,
1146 unsigned int sdma_engine)
1147{
1148 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
1149 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false,
1150 sdma_engine);
1151}
1152
1153/* dqm->lock mutex has to be locked before calling this function */
1154static int map_queues_cpsch(struct device_queue_manager *dqm)
1155{
1156 int retval;
1157
1158 if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
1159 return 0;
1160
1161 if (dqm->active_runlist)
1162 return 0;
1163
1164 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1165 if (retval) {
1166 pr_err("failed to execute runlist\n");
1167 return retval;
1168 }
1169 dqm->active_runlist = true;
1170
1171 return retval;
1172}
1173
1174/* dqm->lock mutex has to be locked before calling this function */
1175static int unmap_queues_cpsch(struct device_queue_manager *dqm,
1176 enum kfd_unmap_queues_filter filter,
1177 uint32_t filter_param)
1178{
1179 int retval = 0;
1180
1181 if (!dqm->active_runlist)
1182 return retval;
1183
1184 pr_debug("Before destroying queues, sdma queue count is : %u\n",
1185 dqm->sdma_queue_count);
1186
1187 if (dqm->sdma_queue_count > 0) {
1188 unmap_sdma_queues(dqm, 0);
1189 unmap_sdma_queues(dqm, 1);
1190 }
1191
1192 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
1193 filter, filter_param, false, 0);
1194 if (retval)
1195 return retval;
1196
1197 *dqm->fence_addr = KFD_FENCE_INIT;
1198 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
1199 KFD_FENCE_COMPLETED);
1200 /* should be timed out */
1201 retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
1202 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
1203 if (retval)
1204 return retval;
1205
1206 pm_release_ib(&dqm->packets);
1207 dqm->active_runlist = false;
1208
1209 return retval;
1210}
1211
1212/* dqm->lock mutex has to be locked before calling this function */
1213static int execute_queues_cpsch(struct device_queue_manager *dqm,
1214 enum kfd_unmap_queues_filter filter,
1215 uint32_t filter_param)
1216{
1217 int retval;
1218
1219 retval = unmap_queues_cpsch(dqm, filter, filter_param);
1220 if (retval) {
1221 pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
1222 return retval;
1223 }
1224
1225 return map_queues_cpsch(dqm);
1226}
1227
1228static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1229 struct qcm_process_device *qpd,
1230 struct queue *q)
1231{
1232 int retval;
1233 struct mqd_manager *mqd;
1234 bool preempt_all_queues;
1235
1236 preempt_all_queues = false;
1237
1238 retval = 0;
1239
1240 /* remove queue from list to prevent rescheduling after preemption */
1241 mutex_lock(&dqm->lock);
1242
1243 if (qpd->is_debug) {
1244 /*
1245 * error, currently we do not allow to destroy a queue
1246 * of a currently debugged process
1247 */
1248 retval = -EBUSY;
1249 goto failed_try_destroy_debugged_queue;
1250
1251 }
1252
1253 mqd = dqm->ops.get_mqd_manager(dqm,
1254 get_mqd_type_from_queue_type(q->properties.type));
1255 if (!mqd) {
1256 retval = -ENOMEM;
1257 goto failed;
1258 }
1259
1260 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1261 dqm->sdma_queue_count--;
1262 deallocate_sdma_queue(dqm, q->sdma_id);
1263 }
1264
1265 list_del(&q->list);
1266 qpd->queue_count--;
1267 if (q->properties.is_active) {
1268 dqm->queue_count--;
1269 retval = execute_queues_cpsch(dqm,
1270 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
1271 if (retval == -ETIME)
1272 qpd->reset_wavefronts = true;
1273 }
1274
1275 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1276
1277 /*
1278 * Unconditionally decrement this counter, regardless of the queue's
1279 * type
1280 */
1281 dqm->total_queue_count--;
1282 pr_debug("Total of %d queues are accountable so far\n",
1283 dqm->total_queue_count);
1284
1285 mutex_unlock(&dqm->lock);
1286
1287 return retval;
1288
1289failed:
1290failed_try_destroy_debugged_queue:
1291
1292 mutex_unlock(&dqm->lock);
1293 return retval;
1294}
1295
1296/*
1297 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1298 * stay in user mode.
1299 */
1300#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1301/* APE1 limit is inclusive and 64K aligned. */
1302#define APE1_LIMIT_ALIGNMENT 0xFFFF
1303
1304static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1305 struct qcm_process_device *qpd,
1306 enum cache_policy default_policy,
1307 enum cache_policy alternate_policy,
1308 void __user *alternate_aperture_base,
1309 uint64_t alternate_aperture_size)
1310{
1311 bool retval;
1312
1313 mutex_lock(&dqm->lock);
1314
1315 if (alternate_aperture_size == 0) {
1316 /* base > limit disables APE1 */
1317 qpd->sh_mem_ape1_base = 1;
1318 qpd->sh_mem_ape1_limit = 0;
1319 } else {
1320 /*
1321 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1322 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1323 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1324 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1325 * Verify that the base and size parameters can be
1326 * represented in this format and convert them.
1327 * Additionally restrict APE1 to user-mode addresses.
1328 */
1329
1330 uint64_t base = (uintptr_t)alternate_aperture_base;
1331 uint64_t limit = base + alternate_aperture_size - 1;
1332
1333 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
1334 (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
1335 retval = false;
1336 goto out;
1337 }
1338
1339 qpd->sh_mem_ape1_base = base >> 16;
1340 qpd->sh_mem_ape1_limit = limit >> 16;
1341 }
1342
1343 retval = dqm->asic_ops.set_cache_memory_policy(
1344 dqm,
1345 qpd,
1346 default_policy,
1347 alternate_policy,
1348 alternate_aperture_base,
1349 alternate_aperture_size);
1350
1351 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
1352 program_sh_mem_settings(dqm, qpd);
1353
1354 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
1355 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1356 qpd->sh_mem_ape1_limit);
1357
1358out:
1359 mutex_unlock(&dqm->lock);
1360 return retval;
1361}
1362
1363static int set_trap_handler(struct device_queue_manager *dqm,
1364 struct qcm_process_device *qpd,
1365 uint64_t tba_addr,
1366 uint64_t tma_addr)
1367{
1368 uint64_t *tma;
1369
1370 if (dqm->dev->cwsr_enabled) {
1371 /* Jump from CWSR trap handler to user trap */
1372 tma = (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1373 tma[0] = tba_addr;
1374 tma[1] = tma_addr;
1375 } else {
1376 qpd->tba_addr = tba_addr;
1377 qpd->tma_addr = tma_addr;
1378 }
1379
1380 return 0;
1381}
1382
1383static int process_termination_nocpsch(struct device_queue_manager *dqm,
1384 struct qcm_process_device *qpd)
1385{
1386 struct queue *q, *next;
1387 struct device_process_node *cur, *next_dpn;
1388 int retval = 0;
1389
1390 mutex_lock(&dqm->lock);
1391
1392 /* Clear all user mode queues */
1393 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1394 int ret;
1395
1396 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
1397 if (ret)
1398 retval = ret;
1399 }
1400
1401 /* Unregister process */
1402 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1403 if (qpd == cur->qpd) {
1404 list_del(&cur->list);
1405 kfree(cur);
1406 dqm->processes_count--;
1407 break;
1408 }
1409 }
1410
1411 mutex_unlock(&dqm->lock);
1412 return retval;
1413}
1414
1415
1416static int process_termination_cpsch(struct device_queue_manager *dqm,
1417 struct qcm_process_device *qpd)
1418{
1419 int retval;
1420 struct queue *q, *next;
1421 struct kernel_queue *kq, *kq_next;
1422 struct mqd_manager *mqd;
1423 struct device_process_node *cur, *next_dpn;
1424 enum kfd_unmap_queues_filter filter =
1425 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
1426
1427 retval = 0;
1428
1429 mutex_lock(&dqm->lock);
1430
1431 /* Clean all kernel queues */
1432 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
1433 list_del(&kq->list);
1434 dqm->queue_count--;
1435 qpd->is_debug = false;
1436 dqm->total_queue_count--;
1437 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
1438 }
1439
1440 /* Clear all user mode queues */
1441 list_for_each_entry(q, &qpd->queues_list, list) {
1442 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1443 dqm->sdma_queue_count--;
1444 deallocate_sdma_queue(dqm, q->sdma_id);
1445 }
1446
1447 if (q->properties.is_active)
1448 dqm->queue_count--;
1449
1450 dqm->total_queue_count--;
1451 }
1452
1453 /* Unregister process */
1454 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1455 if (qpd == cur->qpd) {
1456 list_del(&cur->list);
1457 kfree(cur);
1458 dqm->processes_count--;
1459 break;
1460 }
1461 }
1462
1463 retval = execute_queues_cpsch(dqm, filter, 0);
1464 if (retval || qpd->reset_wavefronts) {
1465 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
1466 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
1467 qpd->reset_wavefronts = false;
1468 }
1469
1470 /* lastly, free mqd resources */
1471 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1472 mqd = dqm->ops.get_mqd_manager(dqm,
1473 get_mqd_type_from_queue_type(q->properties.type));
1474 if (!mqd) {
1475 retval = -ENOMEM;
1476 goto out;
1477 }
1478 list_del(&q->list);
1479 qpd->queue_count--;
1480 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1481 }
1482
1483out:
1484 mutex_unlock(&dqm->lock);
1485 return retval;
1486}
1487
1488struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1489{
1490 struct device_queue_manager *dqm;
1491
1492 pr_debug("Loading device queue manager\n");
1493
1494 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
1495 if (!dqm)
1496 return NULL;
1497
1498 switch (dev->device_info->asic_family) {
1499 /* HWS is not available on Hawaii. */
1500 case CHIP_HAWAII:
1501 /* HWS depends on CWSR for timely dequeue. CWSR is not
1502 * available on Tonga.
1503 *
1504 * FIXME: This argument also applies to Kaveri.
1505 */
1506 case CHIP_TONGA:
1507 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
1508 break;
1509 default:
1510 dqm->sched_policy = sched_policy;
1511 break;
1512 }
1513
1514 dqm->dev = dev;
1515 switch (dqm->sched_policy) {
1516 case KFD_SCHED_POLICY_HWS:
1517 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1518 /* initialize dqm for cp scheduling */
1519 dqm->ops.create_queue = create_queue_cpsch;
1520 dqm->ops.initialize = initialize_cpsch;
1521 dqm->ops.start = start_cpsch;
1522 dqm->ops.stop = stop_cpsch;
1523 dqm->ops.destroy_queue = destroy_queue_cpsch;
1524 dqm->ops.update_queue = update_queue;
1525 dqm->ops.get_mqd_manager = get_mqd_manager;
1526 dqm->ops.register_process = register_process;
1527 dqm->ops.unregister_process = unregister_process;
1528 dqm->ops.uninitialize = uninitialize;
1529 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1530 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1531 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1532 dqm->ops.set_trap_handler = set_trap_handler;
1533 dqm->ops.process_termination = process_termination_cpsch;
1534 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
1535 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
1536 break;
1537 case KFD_SCHED_POLICY_NO_HWS:
1538 /* initialize dqm for no cp scheduling */
1539 dqm->ops.start = start_nocpsch;
1540 dqm->ops.stop = stop_nocpsch;
1541 dqm->ops.create_queue = create_queue_nocpsch;
1542 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1543 dqm->ops.update_queue = update_queue;
1544 dqm->ops.get_mqd_manager = get_mqd_manager;
1545 dqm->ops.register_process = register_process;
1546 dqm->ops.unregister_process = unregister_process;
1547 dqm->ops.initialize = initialize_nocpsch;
1548 dqm->ops.uninitialize = uninitialize;
1549 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1550 dqm->ops.set_trap_handler = set_trap_handler;
1551 dqm->ops.process_termination = process_termination_nocpsch;
1552 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
1553 dqm->ops.restore_process_queues =
1554 restore_process_queues_nocpsch;
1555 break;
1556 default:
1557 pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
1558 goto out_free;
1559 }
1560
1561 switch (dev->device_info->asic_family) {
1562 case CHIP_CARRIZO:
1563 device_queue_manager_init_vi(&dqm->asic_ops);
1564 break;
1565
1566 case CHIP_KAVERI:
1567 device_queue_manager_init_cik(&dqm->asic_ops);
1568 break;
1569
1570 case CHIP_HAWAII:
1571 device_queue_manager_init_cik_hawaii(&dqm->asic_ops);
1572 break;
1573
1574 case CHIP_TONGA:
1575 case CHIP_FIJI:
1576 case CHIP_POLARIS10:
1577 case CHIP_POLARIS11:
1578 device_queue_manager_init_vi_tonga(&dqm->asic_ops);
1579 break;
1580 default:
1581 WARN(1, "Unexpected ASIC family %u",
1582 dev->device_info->asic_family);
1583 goto out_free;
1584 }
1585
1586 if (!dqm->ops.initialize(dqm))
1587 return dqm;
1588
1589out_free:
1590 kfree(dqm);
1591 return NULL;
1592}
1593
1594void device_queue_manager_uninit(struct device_queue_manager *dqm)
1595{
1596 dqm->ops.uninitialize(dqm);
1597 kfree(dqm);
1598}
1599
1600#if defined(CONFIG_DEBUG_FS)
1601
1602static void seq_reg_dump(struct seq_file *m,
1603 uint32_t (*dump)[2], uint32_t n_regs)
1604{
1605 uint32_t i, count;
1606
1607 for (i = 0, count = 0; i < n_regs; i++) {
1608 if (count == 0 ||
1609 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
1610 seq_printf(m, "%s %08x: %08x",
1611 i ? "\n" : "",
1612 dump[i][0], dump[i][1]);
1613 count = 7;
1614 } else {
1615 seq_printf(m, " %08x", dump[i][1]);
1616 count--;
1617 }
1618 }
1619
1620 seq_puts(m, "\n");
1621}
1622
1623int dqm_debugfs_hqds(struct seq_file *m, void *data)
1624{
1625 struct device_queue_manager *dqm = data;
1626 uint32_t (*dump)[2], n_regs;
1627 int pipe, queue;
1628 int r = 0;
1629
1630 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1631 int pipe_offset = pipe * get_queues_per_pipe(dqm);
1632
1633 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
1634 if (!test_bit(pipe_offset + queue,
1635 dqm->dev->shared_resources.queue_bitmap))
1636 continue;
1637
1638 r = dqm->dev->kfd2kgd->hqd_dump(
1639 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
1640 if (r)
1641 break;
1642
1643 seq_printf(m, " CP Pipe %d, Queue %d\n",
1644 pipe, queue);
1645 seq_reg_dump(m, dump, n_regs);
1646
1647 kfree(dump);
1648 }
1649 }
1650
1651 for (pipe = 0; pipe < CIK_SDMA_ENGINE_NUM; pipe++) {
1652 for (queue = 0; queue < CIK_SDMA_QUEUES_PER_ENGINE; queue++) {
1653 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
1654 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
1655 if (r)
1656 break;
1657
1658 seq_printf(m, " SDMA Engine %d, RLC %d\n",
1659 pipe, queue);
1660 seq_reg_dump(m, dump, n_regs);
1661
1662 kfree(dump);
1663 }
1664 }
1665
1666 return r;
1667}
1668
1669#endif