Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_gt_pagefault.h"
7
8#include <linux/bitfield.h>
9#include <linux/circ_buf.h>
10
11#include <drm/drm_exec.h>
12#include <drm/drm_managed.h>
13#include <drm/ttm/ttm_execbuf_util.h>
14
15#include "abi/guc_actions_abi.h"
16#include "xe_bo.h"
17#include "xe_gt.h"
18#include "xe_gt_tlb_invalidation.h"
19#include "xe_guc.h"
20#include "xe_guc_ct.h"
21#include "xe_migrate.h"
22#include "xe_pt.h"
23#include "xe_trace.h"
24#include "xe_vm.h"
25
26struct pagefault {
27 u64 page_addr;
28 u32 asid;
29 u16 pdata;
30 u8 vfid;
31 u8 access_type;
32 u8 fault_type;
33 u8 fault_level;
34 u8 engine_class;
35 u8 engine_instance;
36 u8 fault_unsuccessful;
37 bool trva_fault;
38};
39
40enum access_type {
41 ACCESS_TYPE_READ = 0,
42 ACCESS_TYPE_WRITE = 1,
43 ACCESS_TYPE_ATOMIC = 2,
44 ACCESS_TYPE_RESERVED = 3,
45};
46
47enum fault_type {
48 NOT_PRESENT = 0,
49 WRITE_ACCESS_VIOLATION = 1,
50 ATOMIC_ACCESS_VIOLATION = 2,
51};
52
53struct acc {
54 u64 va_range_base;
55 u32 asid;
56 u32 sub_granularity;
57 u8 granularity;
58 u8 vfid;
59 u8 access_type;
60 u8 engine_class;
61 u8 engine_instance;
62};
63
64static bool access_is_atomic(enum access_type access_type)
65{
66 return access_type == ACCESS_TYPE_ATOMIC;
67}
68
69static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma)
70{
71 return BIT(tile->id) & vma->tile_present &&
72 !(BIT(tile->id) & vma->tile_invalidated);
73}
74
75static bool vma_matches(struct xe_vma *vma, u64 page_addr)
76{
77 if (page_addr > xe_vma_end(vma) - 1 ||
78 page_addr + SZ_4K - 1 < xe_vma_start(vma))
79 return false;
80
81 return true;
82}
83
84static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr)
85{
86 struct xe_vma *vma = NULL;
87
88 if (vm->usm.last_fault_vma) { /* Fast lookup */
89 if (vma_matches(vm->usm.last_fault_vma, page_addr))
90 vma = vm->usm.last_fault_vma;
91 }
92 if (!vma)
93 vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);
94
95 return vma;
96}
97
98static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,
99 bool atomic, unsigned int id)
100{
101 struct xe_bo *bo = xe_vma_bo(vma);
102 struct xe_vm *vm = xe_vma_vm(vma);
103 int err;
104
105 err = xe_vm_lock_vma(exec, vma);
106 if (err)
107 return err;
108
109 if (atomic && IS_DGFX(vm->xe)) {
110 if (xe_vma_is_userptr(vma)) {
111 err = -EACCES;
112 return err;
113 }
114
115 /* Migrate to VRAM, move should invalidate the VMA first */
116 err = xe_bo_migrate(bo, XE_PL_VRAM0 + id);
117 if (err)
118 return err;
119 } else if (bo) {
120 /* Create backing store if needed */
121 err = xe_bo_validate(bo, vm, true);
122 if (err)
123 return err;
124 }
125
126 return 0;
127}
128
129static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
130{
131 struct xe_device *xe = gt_to_xe(gt);
132 struct xe_tile *tile = gt_to_tile(gt);
133 struct drm_exec exec;
134 struct xe_vm *vm;
135 struct xe_vma *vma = NULL;
136 struct dma_fence *fence;
137 bool write_locked;
138 int ret = 0;
139 bool atomic;
140
141 /* SW isn't expected to handle TRTT faults */
142 if (pf->trva_fault)
143 return -EFAULT;
144
145 /* ASID to VM */
146 mutex_lock(&xe->usm.lock);
147 vm = xa_load(&xe->usm.asid_to_vm, pf->asid);
148 if (vm && xe_vm_in_fault_mode(vm))
149 xe_vm_get(vm);
150 else
151 vm = NULL;
152 mutex_unlock(&xe->usm.lock);
153 if (!vm)
154 return -EINVAL;
155
156retry_userptr:
157 /*
158 * TODO: Avoid exclusive lock if VM doesn't have userptrs, or
159 * start out read-locked?
160 */
161 down_write(&vm->lock);
162 write_locked = true;
163 vma = lookup_vma(vm, pf->page_addr);
164 if (!vma) {
165 ret = -EINVAL;
166 goto unlock_vm;
167 }
168
169 if (!xe_vma_is_userptr(vma) ||
170 !xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
171 downgrade_write(&vm->lock);
172 write_locked = false;
173 }
174
175 trace_xe_vma_pagefault(vma);
176
177 atomic = access_is_atomic(pf->access_type);
178
179 /* Check if VMA is valid */
180 if (vma_is_valid(tile, vma) && !atomic)
181 goto unlock_vm;
182
183 /* TODO: Validate fault */
184
185 if (xe_vma_is_userptr(vma) && write_locked) {
186 struct xe_userptr_vma *uvma = to_userptr_vma(vma);
187
188 spin_lock(&vm->userptr.invalidated_lock);
189 list_del_init(&uvma->userptr.invalidate_link);
190 spin_unlock(&vm->userptr.invalidated_lock);
191
192 ret = xe_vma_userptr_pin_pages(uvma);
193 if (ret)
194 goto unlock_vm;
195
196 downgrade_write(&vm->lock);
197 write_locked = false;
198 }
199
200 /* Lock VM and BOs dma-resv */
201 drm_exec_init(&exec, 0, 0);
202 drm_exec_until_all_locked(&exec) {
203 ret = xe_pf_begin(&exec, vma, atomic, tile->id);
204 drm_exec_retry_on_contention(&exec);
205 if (ret)
206 goto unlock_dma_resv;
207 }
208
209 /* Bind VMA only to the GT that has faulted */
210 trace_xe_vma_pf_bind(vma);
211 fence = __xe_pt_bind_vma(tile, vma, xe_tile_migrate_engine(tile), NULL, 0,
212 vma->tile_present & BIT(tile->id));
213 if (IS_ERR(fence)) {
214 ret = PTR_ERR(fence);
215 goto unlock_dma_resv;
216 }
217
218 /*
219 * XXX: Should we drop the lock before waiting? This only helps if doing
220 * GPU binds which is currently only done if we have to wait for more
221 * than 10ms on a move.
222 */
223 dma_fence_wait(fence, false);
224 dma_fence_put(fence);
225
226 if (xe_vma_is_userptr(vma))
227 ret = xe_vma_userptr_check_repin(to_userptr_vma(vma));
228 vma->tile_invalidated &= ~BIT(tile->id);
229
230unlock_dma_resv:
231 drm_exec_fini(&exec);
232unlock_vm:
233 if (!ret)
234 vm->usm.last_fault_vma = vma;
235 if (write_locked)
236 up_write(&vm->lock);
237 else
238 up_read(&vm->lock);
239 if (ret == -EAGAIN)
240 goto retry_userptr;
241
242 if (!ret) {
243 ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma);
244 if (ret >= 0)
245 ret = 0;
246 }
247 xe_vm_put(vm);
248
249 return ret;
250}
251
252static int send_pagefault_reply(struct xe_guc *guc,
253 struct xe_guc_pagefault_reply *reply)
254{
255 u32 action[] = {
256 XE_GUC_ACTION_PAGE_FAULT_RES_DESC,
257 reply->dw0,
258 reply->dw1,
259 };
260
261 return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
262}
263
264static void print_pagefault(struct xe_device *xe, struct pagefault *pf)
265{
266 drm_dbg(&xe->drm, "\n\tASID: %d\n"
267 "\tVFID: %d\n"
268 "\tPDATA: 0x%04x\n"
269 "\tFaulted Address: 0x%08x%08x\n"
270 "\tFaultType: %d\n"
271 "\tAccessType: %d\n"
272 "\tFaultLevel: %d\n"
273 "\tEngineClass: %d\n"
274 "\tEngineInstance: %d\n",
275 pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr),
276 lower_32_bits(pf->page_addr),
277 pf->fault_type, pf->access_type, pf->fault_level,
278 pf->engine_class, pf->engine_instance);
279}
280
281#define PF_MSG_LEN_DW 4
282
283static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf)
284{
285 const struct xe_guc_pagefault_desc *desc;
286 bool ret = false;
287
288 spin_lock_irq(&pf_queue->lock);
289 if (pf_queue->tail != pf_queue->head) {
290 desc = (const struct xe_guc_pagefault_desc *)
291 (pf_queue->data + pf_queue->tail);
292
293 pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0);
294 pf->trva_fault = FIELD_GET(XE2_PFD_TRVA_FAULT, desc->dw0);
295 pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0);
296 pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0);
297 pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) <<
298 PFD_PDATA_HI_SHIFT;
299 pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0);
300 pf->asid = FIELD_GET(PFD_ASID, desc->dw1);
301 pf->vfid = FIELD_GET(PFD_VFID, desc->dw2);
302 pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2);
303 pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2);
304 pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) <<
305 PFD_VIRTUAL_ADDR_HI_SHIFT;
306 pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) <<
307 PFD_VIRTUAL_ADDR_LO_SHIFT;
308
309 pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) %
310 PF_QUEUE_NUM_DW;
311 ret = true;
312 }
313 spin_unlock_irq(&pf_queue->lock);
314
315 return ret;
316}
317
318static bool pf_queue_full(struct pf_queue *pf_queue)
319{
320 lockdep_assert_held(&pf_queue->lock);
321
322 return CIRC_SPACE(pf_queue->head, pf_queue->tail, PF_QUEUE_NUM_DW) <=
323 PF_MSG_LEN_DW;
324}
325
326int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len)
327{
328 struct xe_gt *gt = guc_to_gt(guc);
329 struct xe_device *xe = gt_to_xe(gt);
330 struct pf_queue *pf_queue;
331 unsigned long flags;
332 u32 asid;
333 bool full;
334
335 /*
336 * The below logic doesn't work unless PF_QUEUE_NUM_DW % PF_MSG_LEN_DW == 0
337 */
338 BUILD_BUG_ON(PF_QUEUE_NUM_DW % PF_MSG_LEN_DW);
339
340 if (unlikely(len != PF_MSG_LEN_DW))
341 return -EPROTO;
342
343 asid = FIELD_GET(PFD_ASID, msg[1]);
344 pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE);
345
346 spin_lock_irqsave(&pf_queue->lock, flags);
347 full = pf_queue_full(pf_queue);
348 if (!full) {
349 memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32));
350 pf_queue->head = (pf_queue->head + len) % PF_QUEUE_NUM_DW;
351 queue_work(gt->usm.pf_wq, &pf_queue->worker);
352 } else {
353 drm_warn(&xe->drm, "PF Queue full, shouldn't be possible");
354 }
355 spin_unlock_irqrestore(&pf_queue->lock, flags);
356
357 return full ? -ENOSPC : 0;
358}
359
360#define USM_QUEUE_MAX_RUNTIME_MS 20
361
362static void pf_queue_work_func(struct work_struct *w)
363{
364 struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker);
365 struct xe_gt *gt = pf_queue->gt;
366 struct xe_device *xe = gt_to_xe(gt);
367 struct xe_guc_pagefault_reply reply = {};
368 struct pagefault pf = {};
369 unsigned long threshold;
370 int ret;
371
372 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
373
374 while (get_pagefault(pf_queue, &pf)) {
375 ret = handle_pagefault(gt, &pf);
376 if (unlikely(ret)) {
377 print_pagefault(xe, &pf);
378 pf.fault_unsuccessful = 1;
379 drm_dbg(&xe->drm, "Fault response: Unsuccessful %d\n", ret);
380 }
381
382 reply.dw0 = FIELD_PREP(PFR_VALID, 1) |
383 FIELD_PREP(PFR_SUCCESS, pf.fault_unsuccessful) |
384 FIELD_PREP(PFR_REPLY, PFR_ACCESS) |
385 FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) |
386 FIELD_PREP(PFR_ASID, pf.asid);
387
388 reply.dw1 = FIELD_PREP(PFR_VFID, pf.vfid) |
389 FIELD_PREP(PFR_ENG_INSTANCE, pf.engine_instance) |
390 FIELD_PREP(PFR_ENG_CLASS, pf.engine_class) |
391 FIELD_PREP(PFR_PDATA, pf.pdata);
392
393 send_pagefault_reply(>->uc.guc, &reply);
394
395 if (time_after(jiffies, threshold) &&
396 pf_queue->tail != pf_queue->head) {
397 queue_work(gt->usm.pf_wq, w);
398 break;
399 }
400 }
401}
402
403static void acc_queue_work_func(struct work_struct *w);
404
405int xe_gt_pagefault_init(struct xe_gt *gt)
406{
407 struct xe_device *xe = gt_to_xe(gt);
408 int i;
409
410 if (!xe->info.has_usm)
411 return 0;
412
413 for (i = 0; i < NUM_PF_QUEUE; ++i) {
414 gt->usm.pf_queue[i].gt = gt;
415 spin_lock_init(>->usm.pf_queue[i].lock);
416 INIT_WORK(>->usm.pf_queue[i].worker, pf_queue_work_func);
417 }
418 for (i = 0; i < NUM_ACC_QUEUE; ++i) {
419 gt->usm.acc_queue[i].gt = gt;
420 spin_lock_init(>->usm.acc_queue[i].lock);
421 INIT_WORK(>->usm.acc_queue[i].worker, acc_queue_work_func);
422 }
423
424 gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue",
425 WQ_UNBOUND | WQ_HIGHPRI, NUM_PF_QUEUE);
426 if (!gt->usm.pf_wq)
427 return -ENOMEM;
428
429 gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue",
430 WQ_UNBOUND | WQ_HIGHPRI,
431 NUM_ACC_QUEUE);
432 if (!gt->usm.acc_wq)
433 return -ENOMEM;
434
435 return 0;
436}
437
438void xe_gt_pagefault_reset(struct xe_gt *gt)
439{
440 struct xe_device *xe = gt_to_xe(gt);
441 int i;
442
443 if (!xe->info.has_usm)
444 return;
445
446 for (i = 0; i < NUM_PF_QUEUE; ++i) {
447 spin_lock_irq(>->usm.pf_queue[i].lock);
448 gt->usm.pf_queue[i].head = 0;
449 gt->usm.pf_queue[i].tail = 0;
450 spin_unlock_irq(>->usm.pf_queue[i].lock);
451 }
452
453 for (i = 0; i < NUM_ACC_QUEUE; ++i) {
454 spin_lock(>->usm.acc_queue[i].lock);
455 gt->usm.acc_queue[i].head = 0;
456 gt->usm.acc_queue[i].tail = 0;
457 spin_unlock(>->usm.acc_queue[i].lock);
458 }
459}
460
461static int granularity_in_byte(int val)
462{
463 switch (val) {
464 case 0:
465 return SZ_128K;
466 case 1:
467 return SZ_2M;
468 case 2:
469 return SZ_16M;
470 case 3:
471 return SZ_64M;
472 default:
473 return 0;
474 }
475}
476
477static int sub_granularity_in_byte(int val)
478{
479 return (granularity_in_byte(val) / 32);
480}
481
482static void print_acc(struct xe_device *xe, struct acc *acc)
483{
484 drm_warn(&xe->drm, "Access counter request:\n"
485 "\tType: %s\n"
486 "\tASID: %d\n"
487 "\tVFID: %d\n"
488 "\tEngine: %d:%d\n"
489 "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n"
490 "\tSub_Granularity Vector: 0x%08x\n"
491 "\tVA Range base: 0x%016llx\n",
492 acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL",
493 acc->asid, acc->vfid, acc->engine_class, acc->engine_instance,
494 granularity_in_byte(acc->granularity) / SZ_1K,
495 sub_granularity_in_byte(acc->granularity) / SZ_1K,
496 acc->sub_granularity, acc->va_range_base);
497}
498
499static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc)
500{
501 u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) *
502 sub_granularity_in_byte(acc->granularity);
503
504 return xe_vm_find_overlapping_vma(vm, page_va, SZ_4K);
505}
506
507static int handle_acc(struct xe_gt *gt, struct acc *acc)
508{
509 struct xe_device *xe = gt_to_xe(gt);
510 struct xe_tile *tile = gt_to_tile(gt);
511 struct drm_exec exec;
512 struct xe_vm *vm;
513 struct xe_vma *vma;
514 int ret = 0;
515
516 /* We only support ACC_TRIGGER at the moment */
517 if (acc->access_type != ACC_TRIGGER)
518 return -EINVAL;
519
520 /* ASID to VM */
521 mutex_lock(&xe->usm.lock);
522 vm = xa_load(&xe->usm.asid_to_vm, acc->asid);
523 if (vm)
524 xe_vm_get(vm);
525 mutex_unlock(&xe->usm.lock);
526 if (!vm || !xe_vm_in_fault_mode(vm))
527 return -EINVAL;
528
529 down_read(&vm->lock);
530
531 /* Lookup VMA */
532 vma = get_acc_vma(vm, acc);
533 if (!vma) {
534 ret = -EINVAL;
535 goto unlock_vm;
536 }
537
538 trace_xe_vma_acc(vma);
539
540 /* Userptr or null can't be migrated, nothing to do */
541 if (xe_vma_has_no_bo(vma))
542 goto unlock_vm;
543
544 /* Lock VM and BOs dma-resv */
545 drm_exec_init(&exec, 0, 0);
546 drm_exec_until_all_locked(&exec) {
547 ret = xe_pf_begin(&exec, vma, true, tile->id);
548 drm_exec_retry_on_contention(&exec);
549 if (ret)
550 break;
551 }
552
553 drm_exec_fini(&exec);
554unlock_vm:
555 up_read(&vm->lock);
556 xe_vm_put(vm);
557
558 return ret;
559}
560
561#define make_u64(hi__, low__) ((u64)(hi__) << 32 | (u64)(low__))
562
563#define ACC_MSG_LEN_DW 4
564
565static bool get_acc(struct acc_queue *acc_queue, struct acc *acc)
566{
567 const struct xe_guc_acc_desc *desc;
568 bool ret = false;
569
570 spin_lock(&acc_queue->lock);
571 if (acc_queue->tail != acc_queue->head) {
572 desc = (const struct xe_guc_acc_desc *)
573 (acc_queue->data + acc_queue->tail);
574
575 acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2);
576 acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 |
577 FIELD_GET(ACC_SUBG_LO, desc->dw0);
578 acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1);
579 acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1);
580 acc->asid = FIELD_GET(ACC_ASID, desc->dw1);
581 acc->vfid = FIELD_GET(ACC_VFID, desc->dw2);
582 acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0);
583 acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI,
584 desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO);
585
586 acc_queue->tail = (acc_queue->tail + ACC_MSG_LEN_DW) %
587 ACC_QUEUE_NUM_DW;
588 ret = true;
589 }
590 spin_unlock(&acc_queue->lock);
591
592 return ret;
593}
594
595static void acc_queue_work_func(struct work_struct *w)
596{
597 struct acc_queue *acc_queue = container_of(w, struct acc_queue, worker);
598 struct xe_gt *gt = acc_queue->gt;
599 struct xe_device *xe = gt_to_xe(gt);
600 struct acc acc = {};
601 unsigned long threshold;
602 int ret;
603
604 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
605
606 while (get_acc(acc_queue, &acc)) {
607 ret = handle_acc(gt, &acc);
608 if (unlikely(ret)) {
609 print_acc(xe, &acc);
610 drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret);
611 }
612
613 if (time_after(jiffies, threshold) &&
614 acc_queue->tail != acc_queue->head) {
615 queue_work(gt->usm.acc_wq, w);
616 break;
617 }
618 }
619}
620
621static bool acc_queue_full(struct acc_queue *acc_queue)
622{
623 lockdep_assert_held(&acc_queue->lock);
624
625 return CIRC_SPACE(acc_queue->head, acc_queue->tail, ACC_QUEUE_NUM_DW) <=
626 ACC_MSG_LEN_DW;
627}
628
629int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len)
630{
631 struct xe_gt *gt = guc_to_gt(guc);
632 struct acc_queue *acc_queue;
633 u32 asid;
634 bool full;
635
636 /*
637 * The below logic doesn't work unless ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW == 0
638 */
639 BUILD_BUG_ON(ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW);
640
641 if (unlikely(len != ACC_MSG_LEN_DW))
642 return -EPROTO;
643
644 asid = FIELD_GET(ACC_ASID, msg[1]);
645 acc_queue = >->usm.acc_queue[asid % NUM_ACC_QUEUE];
646
647 spin_lock(&acc_queue->lock);
648 full = acc_queue_full(acc_queue);
649 if (!full) {
650 memcpy(acc_queue->data + acc_queue->head, msg,
651 len * sizeof(u32));
652 acc_queue->head = (acc_queue->head + len) % ACC_QUEUE_NUM_DW;
653 queue_work(gt->usm.acc_wq, &acc_queue->worker);
654 } else {
655 drm_warn(>_to_xe(gt)->drm, "ACC Queue full, dropping ACC");
656 }
657 spin_unlock(&acc_queue->lock);
658
659 return full ? -ENOSPC : 0;
660}