Loading...
Note: File does not exist in v5.14.15.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_gt_pagefault.h"
7
8#include <linux/bitfield.h>
9#include <linux/circ_buf.h>
10
11#include <drm/drm_exec.h>
12#include <drm/drm_managed.h>
13#include <drm/ttm/ttm_execbuf_util.h>
14
15#include "abi/guc_actions_abi.h"
16#include "xe_bo.h"
17#include "xe_gt.h"
18#include "xe_gt_tlb_invalidation.h"
19#include "xe_guc.h"
20#include "xe_guc_ct.h"
21#include "xe_migrate.h"
22#include "xe_trace_bo.h"
23#include "xe_vm.h"
24
25struct pagefault {
26 u64 page_addr;
27 u32 asid;
28 u16 pdata;
29 u8 vfid;
30 u8 access_type;
31 u8 fault_type;
32 u8 fault_level;
33 u8 engine_class;
34 u8 engine_instance;
35 u8 fault_unsuccessful;
36 bool trva_fault;
37};
38
39enum access_type {
40 ACCESS_TYPE_READ = 0,
41 ACCESS_TYPE_WRITE = 1,
42 ACCESS_TYPE_ATOMIC = 2,
43 ACCESS_TYPE_RESERVED = 3,
44};
45
46enum fault_type {
47 NOT_PRESENT = 0,
48 WRITE_ACCESS_VIOLATION = 1,
49 ATOMIC_ACCESS_VIOLATION = 2,
50};
51
52struct acc {
53 u64 va_range_base;
54 u32 asid;
55 u32 sub_granularity;
56 u8 granularity;
57 u8 vfid;
58 u8 access_type;
59 u8 engine_class;
60 u8 engine_instance;
61};
62
63static bool access_is_atomic(enum access_type access_type)
64{
65 return access_type == ACCESS_TYPE_ATOMIC;
66}
67
68static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma)
69{
70 return BIT(tile->id) & vma->tile_present &&
71 !(BIT(tile->id) & vma->tile_invalidated);
72}
73
74static bool vma_matches(struct xe_vma *vma, u64 page_addr)
75{
76 if (page_addr > xe_vma_end(vma) - 1 ||
77 page_addr + SZ_4K - 1 < xe_vma_start(vma))
78 return false;
79
80 return true;
81}
82
83static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr)
84{
85 struct xe_vma *vma = NULL;
86
87 if (vm->usm.last_fault_vma) { /* Fast lookup */
88 if (vma_matches(vm->usm.last_fault_vma, page_addr))
89 vma = vm->usm.last_fault_vma;
90 }
91 if (!vma)
92 vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);
93
94 return vma;
95}
96
97static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,
98 bool atomic, unsigned int id)
99{
100 struct xe_bo *bo = xe_vma_bo(vma);
101 struct xe_vm *vm = xe_vma_vm(vma);
102 int err;
103
104 err = xe_vm_lock_vma(exec, vma);
105 if (err)
106 return err;
107
108 if (atomic && IS_DGFX(vm->xe)) {
109 if (xe_vma_is_userptr(vma)) {
110 err = -EACCES;
111 return err;
112 }
113
114 /* Migrate to VRAM, move should invalidate the VMA first */
115 err = xe_bo_migrate(bo, XE_PL_VRAM0 + id);
116 if (err)
117 return err;
118 } else if (bo) {
119 /* Create backing store if needed */
120 err = xe_bo_validate(bo, vm, true);
121 if (err)
122 return err;
123 }
124
125 return 0;
126}
127
128static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
129 struct xe_vma *vma)
130{
131 struct xe_vm *vm = xe_vma_vm(vma);
132 struct drm_exec exec;
133 struct dma_fence *fence;
134 ktime_t end = 0;
135 int err;
136 bool atomic;
137
138 trace_xe_vma_pagefault(vma);
139 atomic = access_is_atomic(pf->access_type);
140
141 /* Check if VMA is valid */
142 if (vma_is_valid(tile, vma) && !atomic)
143 return 0;
144
145retry_userptr:
146 if (xe_vma_is_userptr(vma) &&
147 xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
148 struct xe_userptr_vma *uvma = to_userptr_vma(vma);
149
150 err = xe_vma_userptr_pin_pages(uvma);
151 if (err)
152 return err;
153 }
154
155 /* Lock VM and BOs dma-resv */
156 drm_exec_init(&exec, 0, 0);
157 drm_exec_until_all_locked(&exec) {
158 err = xe_pf_begin(&exec, vma, atomic, tile->id);
159 drm_exec_retry_on_contention(&exec);
160 if (xe_vm_validate_should_retry(&exec, err, &end))
161 err = -EAGAIN;
162 if (err)
163 goto unlock_dma_resv;
164
165 /* Bind VMA only to the GT that has faulted */
166 trace_xe_vma_pf_bind(vma);
167 fence = xe_vma_rebind(vm, vma, BIT(tile->id));
168 if (IS_ERR(fence)) {
169 err = PTR_ERR(fence);
170 if (xe_vm_validate_should_retry(&exec, err, &end))
171 err = -EAGAIN;
172 goto unlock_dma_resv;
173 }
174 }
175
176 dma_fence_wait(fence, false);
177 dma_fence_put(fence);
178 vma->tile_invalidated &= ~BIT(tile->id);
179
180unlock_dma_resv:
181 drm_exec_fini(&exec);
182 if (err == -EAGAIN)
183 goto retry_userptr;
184
185 return err;
186}
187
188static struct xe_vm *asid_to_vm(struct xe_device *xe, u32 asid)
189{
190 struct xe_vm *vm;
191
192 down_read(&xe->usm.lock);
193 vm = xa_load(&xe->usm.asid_to_vm, asid);
194 if (vm && xe_vm_in_fault_mode(vm))
195 xe_vm_get(vm);
196 else
197 vm = ERR_PTR(-EINVAL);
198 up_read(&xe->usm.lock);
199
200 return vm;
201}
202
203static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
204{
205 struct xe_device *xe = gt_to_xe(gt);
206 struct xe_tile *tile = gt_to_tile(gt);
207 struct xe_vm *vm;
208 struct xe_vma *vma = NULL;
209 int err;
210
211 /* SW isn't expected to handle TRTT faults */
212 if (pf->trva_fault)
213 return -EFAULT;
214
215 vm = asid_to_vm(xe, pf->asid);
216 if (IS_ERR(vm))
217 return PTR_ERR(vm);
218
219 /*
220 * TODO: Change to read lock? Using write lock for simplicity.
221 */
222 down_write(&vm->lock);
223
224 if (xe_vm_is_closed(vm)) {
225 err = -ENOENT;
226 goto unlock_vm;
227 }
228
229 vma = lookup_vma(vm, pf->page_addr);
230 if (!vma) {
231 err = -EINVAL;
232 goto unlock_vm;
233 }
234
235 err = handle_vma_pagefault(tile, pf, vma);
236
237unlock_vm:
238 if (!err)
239 vm->usm.last_fault_vma = vma;
240 up_write(&vm->lock);
241 xe_vm_put(vm);
242
243 return err;
244}
245
246static int send_pagefault_reply(struct xe_guc *guc,
247 struct xe_guc_pagefault_reply *reply)
248{
249 u32 action[] = {
250 XE_GUC_ACTION_PAGE_FAULT_RES_DESC,
251 reply->dw0,
252 reply->dw1,
253 };
254
255 return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
256}
257
258static void print_pagefault(struct xe_device *xe, struct pagefault *pf)
259{
260 drm_dbg(&xe->drm, "\n\tASID: %d\n"
261 "\tVFID: %d\n"
262 "\tPDATA: 0x%04x\n"
263 "\tFaulted Address: 0x%08x%08x\n"
264 "\tFaultType: %d\n"
265 "\tAccessType: %d\n"
266 "\tFaultLevel: %d\n"
267 "\tEngineClass: %d\n"
268 "\tEngineInstance: %d\n",
269 pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr),
270 lower_32_bits(pf->page_addr),
271 pf->fault_type, pf->access_type, pf->fault_level,
272 pf->engine_class, pf->engine_instance);
273}
274
275#define PF_MSG_LEN_DW 4
276
277static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf)
278{
279 const struct xe_guc_pagefault_desc *desc;
280 bool ret = false;
281
282 spin_lock_irq(&pf_queue->lock);
283 if (pf_queue->tail != pf_queue->head) {
284 desc = (const struct xe_guc_pagefault_desc *)
285 (pf_queue->data + pf_queue->tail);
286
287 pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0);
288 pf->trva_fault = FIELD_GET(XE2_PFD_TRVA_FAULT, desc->dw0);
289 pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0);
290 pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0);
291 pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) <<
292 PFD_PDATA_HI_SHIFT;
293 pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0);
294 pf->asid = FIELD_GET(PFD_ASID, desc->dw1);
295 pf->vfid = FIELD_GET(PFD_VFID, desc->dw2);
296 pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2);
297 pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2);
298 pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) <<
299 PFD_VIRTUAL_ADDR_HI_SHIFT;
300 pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) <<
301 PFD_VIRTUAL_ADDR_LO_SHIFT;
302
303 pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) %
304 pf_queue->num_dw;
305 ret = true;
306 }
307 spin_unlock_irq(&pf_queue->lock);
308
309 return ret;
310}
311
312static bool pf_queue_full(struct pf_queue *pf_queue)
313{
314 lockdep_assert_held(&pf_queue->lock);
315
316 return CIRC_SPACE(pf_queue->head, pf_queue->tail,
317 pf_queue->num_dw) <=
318 PF_MSG_LEN_DW;
319}
320
321int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len)
322{
323 struct xe_gt *gt = guc_to_gt(guc);
324 struct xe_device *xe = gt_to_xe(gt);
325 struct pf_queue *pf_queue;
326 unsigned long flags;
327 u32 asid;
328 bool full;
329
330 if (unlikely(len != PF_MSG_LEN_DW))
331 return -EPROTO;
332
333 asid = FIELD_GET(PFD_ASID, msg[1]);
334 pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE);
335
336 /*
337 * The below logic doesn't work unless PF_QUEUE_NUM_DW % PF_MSG_LEN_DW == 0
338 */
339 xe_gt_assert(gt, !(pf_queue->num_dw % PF_MSG_LEN_DW));
340
341 spin_lock_irqsave(&pf_queue->lock, flags);
342 full = pf_queue_full(pf_queue);
343 if (!full) {
344 memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32));
345 pf_queue->head = (pf_queue->head + len) %
346 pf_queue->num_dw;
347 queue_work(gt->usm.pf_wq, &pf_queue->worker);
348 } else {
349 drm_warn(&xe->drm, "PF Queue full, shouldn't be possible");
350 }
351 spin_unlock_irqrestore(&pf_queue->lock, flags);
352
353 return full ? -ENOSPC : 0;
354}
355
356#define USM_QUEUE_MAX_RUNTIME_MS 20
357
358static void pf_queue_work_func(struct work_struct *w)
359{
360 struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker);
361 struct xe_gt *gt = pf_queue->gt;
362 struct xe_device *xe = gt_to_xe(gt);
363 struct xe_guc_pagefault_reply reply = {};
364 struct pagefault pf = {};
365 unsigned long threshold;
366 int ret;
367
368 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
369
370 while (get_pagefault(pf_queue, &pf)) {
371 ret = handle_pagefault(gt, &pf);
372 if (unlikely(ret)) {
373 print_pagefault(xe, &pf);
374 pf.fault_unsuccessful = 1;
375 drm_dbg(&xe->drm, "Fault response: Unsuccessful %d\n", ret);
376 }
377
378 reply.dw0 = FIELD_PREP(PFR_VALID, 1) |
379 FIELD_PREP(PFR_SUCCESS, pf.fault_unsuccessful) |
380 FIELD_PREP(PFR_REPLY, PFR_ACCESS) |
381 FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) |
382 FIELD_PREP(PFR_ASID, pf.asid);
383
384 reply.dw1 = FIELD_PREP(PFR_VFID, pf.vfid) |
385 FIELD_PREP(PFR_ENG_INSTANCE, pf.engine_instance) |
386 FIELD_PREP(PFR_ENG_CLASS, pf.engine_class) |
387 FIELD_PREP(PFR_PDATA, pf.pdata);
388
389 send_pagefault_reply(>->uc.guc, &reply);
390
391 if (time_after(jiffies, threshold) &&
392 pf_queue->tail != pf_queue->head) {
393 queue_work(gt->usm.pf_wq, w);
394 break;
395 }
396 }
397}
398
399static void acc_queue_work_func(struct work_struct *w);
400
401static void pagefault_fini(void *arg)
402{
403 struct xe_gt *gt = arg;
404 struct xe_device *xe = gt_to_xe(gt);
405
406 if (!xe->info.has_usm)
407 return;
408
409 destroy_workqueue(gt->usm.acc_wq);
410 destroy_workqueue(gt->usm.pf_wq);
411}
412
413static int xe_alloc_pf_queue(struct xe_gt *gt, struct pf_queue *pf_queue)
414{
415 struct xe_device *xe = gt_to_xe(gt);
416 xe_dss_mask_t all_dss;
417 int num_dss, num_eus;
418
419 bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask,
420 XE_MAX_DSS_FUSE_BITS);
421
422 num_dss = bitmap_weight(all_dss, XE_MAX_DSS_FUSE_BITS);
423 num_eus = bitmap_weight(gt->fuse_topo.eu_mask_per_dss,
424 XE_MAX_EU_FUSE_BITS) * num_dss;
425
426 /* user can issue separate page faults per EU and per CS */
427 pf_queue->num_dw =
428 (num_eus + XE_NUM_HW_ENGINES) * PF_MSG_LEN_DW;
429
430 pf_queue->gt = gt;
431 pf_queue->data = devm_kcalloc(xe->drm.dev, pf_queue->num_dw,
432 sizeof(u32), GFP_KERNEL);
433 if (!pf_queue->data)
434 return -ENOMEM;
435
436 spin_lock_init(&pf_queue->lock);
437 INIT_WORK(&pf_queue->worker, pf_queue_work_func);
438
439 return 0;
440}
441
442int xe_gt_pagefault_init(struct xe_gt *gt)
443{
444 struct xe_device *xe = gt_to_xe(gt);
445 int i, ret = 0;
446
447 if (!xe->info.has_usm)
448 return 0;
449
450 for (i = 0; i < NUM_PF_QUEUE; ++i) {
451 ret = xe_alloc_pf_queue(gt, >->usm.pf_queue[i]);
452 if (ret)
453 return ret;
454 }
455 for (i = 0; i < NUM_ACC_QUEUE; ++i) {
456 gt->usm.acc_queue[i].gt = gt;
457 spin_lock_init(>->usm.acc_queue[i].lock);
458 INIT_WORK(>->usm.acc_queue[i].worker, acc_queue_work_func);
459 }
460
461 gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue",
462 WQ_UNBOUND | WQ_HIGHPRI, NUM_PF_QUEUE);
463 if (!gt->usm.pf_wq)
464 return -ENOMEM;
465
466 gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue",
467 WQ_UNBOUND | WQ_HIGHPRI,
468 NUM_ACC_QUEUE);
469 if (!gt->usm.acc_wq) {
470 destroy_workqueue(gt->usm.pf_wq);
471 return -ENOMEM;
472 }
473
474 return devm_add_action_or_reset(xe->drm.dev, pagefault_fini, gt);
475}
476
477void xe_gt_pagefault_reset(struct xe_gt *gt)
478{
479 struct xe_device *xe = gt_to_xe(gt);
480 int i;
481
482 if (!xe->info.has_usm)
483 return;
484
485 for (i = 0; i < NUM_PF_QUEUE; ++i) {
486 spin_lock_irq(>->usm.pf_queue[i].lock);
487 gt->usm.pf_queue[i].head = 0;
488 gt->usm.pf_queue[i].tail = 0;
489 spin_unlock_irq(>->usm.pf_queue[i].lock);
490 }
491
492 for (i = 0; i < NUM_ACC_QUEUE; ++i) {
493 spin_lock(>->usm.acc_queue[i].lock);
494 gt->usm.acc_queue[i].head = 0;
495 gt->usm.acc_queue[i].tail = 0;
496 spin_unlock(>->usm.acc_queue[i].lock);
497 }
498}
499
500static int granularity_in_byte(int val)
501{
502 switch (val) {
503 case 0:
504 return SZ_128K;
505 case 1:
506 return SZ_2M;
507 case 2:
508 return SZ_16M;
509 case 3:
510 return SZ_64M;
511 default:
512 return 0;
513 }
514}
515
516static int sub_granularity_in_byte(int val)
517{
518 return (granularity_in_byte(val) / 32);
519}
520
521static void print_acc(struct xe_device *xe, struct acc *acc)
522{
523 drm_warn(&xe->drm, "Access counter request:\n"
524 "\tType: %s\n"
525 "\tASID: %d\n"
526 "\tVFID: %d\n"
527 "\tEngine: %d:%d\n"
528 "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n"
529 "\tSub_Granularity Vector: 0x%08x\n"
530 "\tVA Range base: 0x%016llx\n",
531 acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL",
532 acc->asid, acc->vfid, acc->engine_class, acc->engine_instance,
533 granularity_in_byte(acc->granularity) / SZ_1K,
534 sub_granularity_in_byte(acc->granularity) / SZ_1K,
535 acc->sub_granularity, acc->va_range_base);
536}
537
538static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc)
539{
540 u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) *
541 sub_granularity_in_byte(acc->granularity);
542
543 return xe_vm_find_overlapping_vma(vm, page_va, SZ_4K);
544}
545
546static int handle_acc(struct xe_gt *gt, struct acc *acc)
547{
548 struct xe_device *xe = gt_to_xe(gt);
549 struct xe_tile *tile = gt_to_tile(gt);
550 struct drm_exec exec;
551 struct xe_vm *vm;
552 struct xe_vma *vma;
553 int ret = 0;
554
555 /* We only support ACC_TRIGGER at the moment */
556 if (acc->access_type != ACC_TRIGGER)
557 return -EINVAL;
558
559 vm = asid_to_vm(xe, acc->asid);
560 if (IS_ERR(vm))
561 return PTR_ERR(vm);
562
563 down_read(&vm->lock);
564
565 /* Lookup VMA */
566 vma = get_acc_vma(vm, acc);
567 if (!vma) {
568 ret = -EINVAL;
569 goto unlock_vm;
570 }
571
572 trace_xe_vma_acc(vma);
573
574 /* Userptr or null can't be migrated, nothing to do */
575 if (xe_vma_has_no_bo(vma))
576 goto unlock_vm;
577
578 /* Lock VM and BOs dma-resv */
579 drm_exec_init(&exec, 0, 0);
580 drm_exec_until_all_locked(&exec) {
581 ret = xe_pf_begin(&exec, vma, true, tile->id);
582 drm_exec_retry_on_contention(&exec);
583 if (ret)
584 break;
585 }
586
587 drm_exec_fini(&exec);
588unlock_vm:
589 up_read(&vm->lock);
590 xe_vm_put(vm);
591
592 return ret;
593}
594
595#define make_u64(hi__, low__) ((u64)(hi__) << 32 | (u64)(low__))
596
597#define ACC_MSG_LEN_DW 4
598
599static bool get_acc(struct acc_queue *acc_queue, struct acc *acc)
600{
601 const struct xe_guc_acc_desc *desc;
602 bool ret = false;
603
604 spin_lock(&acc_queue->lock);
605 if (acc_queue->tail != acc_queue->head) {
606 desc = (const struct xe_guc_acc_desc *)
607 (acc_queue->data + acc_queue->tail);
608
609 acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2);
610 acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 |
611 FIELD_GET(ACC_SUBG_LO, desc->dw0);
612 acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1);
613 acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1);
614 acc->asid = FIELD_GET(ACC_ASID, desc->dw1);
615 acc->vfid = FIELD_GET(ACC_VFID, desc->dw2);
616 acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0);
617 acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI,
618 desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO);
619
620 acc_queue->tail = (acc_queue->tail + ACC_MSG_LEN_DW) %
621 ACC_QUEUE_NUM_DW;
622 ret = true;
623 }
624 spin_unlock(&acc_queue->lock);
625
626 return ret;
627}
628
629static void acc_queue_work_func(struct work_struct *w)
630{
631 struct acc_queue *acc_queue = container_of(w, struct acc_queue, worker);
632 struct xe_gt *gt = acc_queue->gt;
633 struct xe_device *xe = gt_to_xe(gt);
634 struct acc acc = {};
635 unsigned long threshold;
636 int ret;
637
638 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
639
640 while (get_acc(acc_queue, &acc)) {
641 ret = handle_acc(gt, &acc);
642 if (unlikely(ret)) {
643 print_acc(xe, &acc);
644 drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret);
645 }
646
647 if (time_after(jiffies, threshold) &&
648 acc_queue->tail != acc_queue->head) {
649 queue_work(gt->usm.acc_wq, w);
650 break;
651 }
652 }
653}
654
655static bool acc_queue_full(struct acc_queue *acc_queue)
656{
657 lockdep_assert_held(&acc_queue->lock);
658
659 return CIRC_SPACE(acc_queue->head, acc_queue->tail, ACC_QUEUE_NUM_DW) <=
660 ACC_MSG_LEN_DW;
661}
662
663int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len)
664{
665 struct xe_gt *gt = guc_to_gt(guc);
666 struct acc_queue *acc_queue;
667 u32 asid;
668 bool full;
669
670 /*
671 * The below logic doesn't work unless ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW == 0
672 */
673 BUILD_BUG_ON(ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW);
674
675 if (unlikely(len != ACC_MSG_LEN_DW))
676 return -EPROTO;
677
678 asid = FIELD_GET(ACC_ASID, msg[1]);
679 acc_queue = >->usm.acc_queue[asid % NUM_ACC_QUEUE];
680
681 spin_lock(&acc_queue->lock);
682 full = acc_queue_full(acc_queue);
683 if (!full) {
684 memcpy(acc_queue->data + acc_queue->head, msg,
685 len * sizeof(u32));
686 acc_queue->head = (acc_queue->head + len) % ACC_QUEUE_NUM_DW;
687 queue_work(gt->usm.acc_wq, &acc_queue->worker);
688 } else {
689 drm_warn(>_to_xe(gt)->drm, "ACC Queue full, dropping ACC");
690 }
691 spin_unlock(&acc_queue->lock);
692
693 return full ? -ENOSPC : 0;
694}