Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/pci.h>
7#include <linux/device.h>
8#include <linux/sched/task.h>
9#include <linux/io-64-nonatomic-lo-hi.h>
10#include <linux/cdev.h>
11#include <linux/fs.h>
12#include <linux/poll.h>
13#include <linux/iommu.h>
14#include <linux/highmem.h>
15#include <uapi/linux/idxd.h>
16#include <linux/xarray.h>
17#include "registers.h"
18#include "idxd.h"
19
20struct idxd_cdev_context {
21 const char *name;
22 dev_t devt;
23 struct ida minor_ida;
24};
25
26/*
27 * Since user file names are global in DSA devices, define their ida's as
28 * global to avoid conflict file names.
29 */
30static DEFINE_IDA(file_ida);
31static DEFINE_MUTEX(ida_lock);
32
33/*
34 * ictx is an array based off of accelerator types. enum idxd_type
35 * is used as index
36 */
37static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
38 { .name = "dsa" },
39 { .name = "iax" }
40};
41
42struct idxd_user_context {
43 struct idxd_wq *wq;
44 struct task_struct *task;
45 unsigned int pasid;
46 struct mm_struct *mm;
47 unsigned int flags;
48 struct iommu_sva *sva;
49 struct idxd_dev idxd_dev;
50 u64 counters[COUNTER_MAX];
51 int id;
52 pid_t pid;
53};
54
55static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid);
56static void idxd_xa_pasid_remove(struct idxd_user_context *ctx);
57
58static inline struct idxd_user_context *dev_to_uctx(struct device *dev)
59{
60 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
61
62 return container_of(idxd_dev, struct idxd_user_context, idxd_dev);
63}
64
65static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf)
66{
67 struct idxd_user_context *ctx = dev_to_uctx(dev);
68
69 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]);
70}
71static DEVICE_ATTR_RO(cr_faults);
72
73static ssize_t cr_fault_failures_show(struct device *dev,
74 struct device_attribute *attr, char *buf)
75{
76 struct idxd_user_context *ctx = dev_to_uctx(dev);
77
78 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]);
79}
80static DEVICE_ATTR_RO(cr_fault_failures);
81
82static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf)
83{
84 struct idxd_user_context *ctx = dev_to_uctx(dev);
85
86 return sysfs_emit(buf, "%u\n", ctx->pid);
87}
88static DEVICE_ATTR_RO(pid);
89
90static struct attribute *cdev_file_attributes[] = {
91 &dev_attr_cr_faults.attr,
92 &dev_attr_cr_fault_failures.attr,
93 &dev_attr_pid.attr,
94 NULL
95};
96
97static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n)
98{
99 struct device *dev = container_of(kobj, typeof(*dev), kobj);
100 struct idxd_user_context *ctx = dev_to_uctx(dev);
101 struct idxd_wq *wq = ctx->wq;
102
103 if (!wq_pasid_enabled(wq))
104 return 0;
105
106 return a->mode;
107}
108
109static const struct attribute_group cdev_file_attribute_group = {
110 .attrs = cdev_file_attributes,
111 .is_visible = cdev_file_attr_visible,
112};
113
114static const struct attribute_group *cdev_file_attribute_groups[] = {
115 &cdev_file_attribute_group,
116 NULL
117};
118
119static void idxd_file_dev_release(struct device *dev)
120{
121 struct idxd_user_context *ctx = dev_to_uctx(dev);
122 struct idxd_wq *wq = ctx->wq;
123 struct idxd_device *idxd = wq->idxd;
124 int rc;
125
126 mutex_lock(&ida_lock);
127 ida_free(&file_ida, ctx->id);
128 mutex_unlock(&ida_lock);
129
130 /* Wait for in-flight operations to complete. */
131 if (wq_shared(wq)) {
132 idxd_device_drain_pasid(idxd, ctx->pasid);
133 } else {
134 if (device_user_pasid_enabled(idxd)) {
135 /* The wq disable in the disable pasid function will drain the wq */
136 rc = idxd_wq_disable_pasid(wq);
137 if (rc < 0)
138 dev_err(dev, "wq disable pasid failed.\n");
139 } else {
140 idxd_wq_drain(wq);
141 }
142 }
143
144 if (ctx->sva) {
145 idxd_cdev_evl_drain_pasid(wq, ctx->pasid);
146 iommu_sva_unbind_device(ctx->sva);
147 idxd_xa_pasid_remove(ctx);
148 }
149 kfree(ctx);
150 mutex_lock(&wq->wq_lock);
151 idxd_wq_put(wq);
152 mutex_unlock(&wq->wq_lock);
153}
154
155static const struct device_type idxd_cdev_file_type = {
156 .name = "idxd_file",
157 .release = idxd_file_dev_release,
158 .groups = cdev_file_attribute_groups,
159};
160
161static void idxd_cdev_dev_release(struct device *dev)
162{
163 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
164 struct idxd_cdev_context *cdev_ctx;
165 struct idxd_wq *wq = idxd_cdev->wq;
166
167 cdev_ctx = &ictx[wq->idxd->data->type];
168 ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor);
169 kfree(idxd_cdev);
170}
171
172static const struct device_type idxd_cdev_device_type = {
173 .name = "idxd_cdev",
174 .release = idxd_cdev_dev_release,
175};
176
177static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
178{
179 struct cdev *cdev = inode->i_cdev;
180
181 return container_of(cdev, struct idxd_cdev, cdev);
182}
183
184static inline struct idxd_wq *inode_wq(struct inode *inode)
185{
186 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
187
188 return idxd_cdev->wq;
189}
190
191static void idxd_xa_pasid_remove(struct idxd_user_context *ctx)
192{
193 struct idxd_wq *wq = ctx->wq;
194 void *ptr;
195
196 mutex_lock(&wq->uc_lock);
197 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL);
198 if (ptr != (void *)ctx)
199 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n",
200 ctx->pasid);
201 mutex_unlock(&wq->uc_lock);
202}
203
204void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index)
205{
206 struct idxd_user_context *ctx;
207
208 if (index >= COUNTER_MAX)
209 return;
210
211 mutex_lock(&wq->uc_lock);
212 ctx = xa_load(&wq->upasid_xa, pasid);
213 if (!ctx) {
214 mutex_unlock(&wq->uc_lock);
215 return;
216 }
217 ctx->counters[index]++;
218 mutex_unlock(&wq->uc_lock);
219}
220
221static int idxd_cdev_open(struct inode *inode, struct file *filp)
222{
223 struct idxd_user_context *ctx;
224 struct idxd_device *idxd;
225 struct idxd_wq *wq;
226 struct device *dev, *fdev;
227 int rc = 0;
228 struct iommu_sva *sva;
229 unsigned int pasid;
230 struct idxd_cdev *idxd_cdev;
231
232 wq = inode_wq(inode);
233 idxd = wq->idxd;
234 dev = &idxd->pdev->dev;
235
236 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
237
238 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
239 if (!ctx)
240 return -ENOMEM;
241
242 mutex_lock(&wq->wq_lock);
243
244 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
245 rc = -EBUSY;
246 goto failed;
247 }
248
249 ctx->wq = wq;
250 filp->private_data = ctx;
251 ctx->pid = current->pid;
252
253 if (device_user_pasid_enabled(idxd)) {
254 sva = iommu_sva_bind_device(dev, current->mm);
255 if (IS_ERR(sva)) {
256 rc = PTR_ERR(sva);
257 dev_err(dev, "pasid allocation failed: %d\n", rc);
258 goto failed;
259 }
260
261 pasid = iommu_sva_get_pasid(sva);
262 if (pasid == IOMMU_PASID_INVALID) {
263 rc = -EINVAL;
264 goto failed_get_pasid;
265 }
266
267 ctx->sva = sva;
268 ctx->pasid = pasid;
269 ctx->mm = current->mm;
270
271 mutex_lock(&wq->uc_lock);
272 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL);
273 mutex_unlock(&wq->uc_lock);
274 if (rc < 0)
275 dev_warn(dev, "PASID entry already exist in xarray.\n");
276
277 if (wq_dedicated(wq)) {
278 rc = idxd_wq_set_pasid(wq, pasid);
279 if (rc < 0) {
280 dev_err(dev, "wq set pasid failed: %d\n", rc);
281 goto failed_set_pasid;
282 }
283 }
284 }
285
286 idxd_cdev = wq->idxd_cdev;
287 mutex_lock(&ida_lock);
288 ctx->id = ida_alloc(&file_ida, GFP_KERNEL);
289 mutex_unlock(&ida_lock);
290 if (ctx->id < 0) {
291 dev_warn(dev, "ida alloc failure\n");
292 goto failed_ida;
293 }
294 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE;
295 fdev = user_ctx_dev(ctx);
296 device_initialize(fdev);
297 fdev->parent = cdev_dev(idxd_cdev);
298 fdev->bus = &dsa_bus_type;
299 fdev->type = &idxd_cdev_file_type;
300
301 rc = dev_set_name(fdev, "file%d", ctx->id);
302 if (rc < 0) {
303 dev_warn(dev, "set name failure\n");
304 goto failed_dev_name;
305 }
306
307 rc = device_add(fdev);
308 if (rc < 0) {
309 dev_warn(dev, "file device add failure\n");
310 goto failed_dev_add;
311 }
312
313 idxd_wq_get(wq);
314 mutex_unlock(&wq->wq_lock);
315 return 0;
316
317failed_dev_add:
318failed_dev_name:
319 put_device(fdev);
320failed_ida:
321failed_set_pasid:
322 if (device_user_pasid_enabled(idxd))
323 idxd_xa_pasid_remove(ctx);
324failed_get_pasid:
325 if (device_user_pasid_enabled(idxd))
326 iommu_sva_unbind_device(sva);
327failed:
328 mutex_unlock(&wq->wq_lock);
329 kfree(ctx);
330 return rc;
331}
332
333static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid)
334{
335 struct idxd_device *idxd = wq->idxd;
336 struct idxd_evl *evl = idxd->evl;
337 union evl_status_reg status;
338 u16 h, t, size;
339 int ent_size = evl_ent_size(idxd);
340 struct __evl_entry *entry_head;
341
342 if (!evl)
343 return;
344
345 mutex_lock(&evl->lock);
346 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET);
347 t = status.tail;
348 h = status.head;
349 size = evl->size;
350
351 while (h != t) {
352 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size));
353 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id)
354 set_bit(h, evl->bmap);
355 h = (h + 1) % size;
356 }
357 drain_workqueue(wq->wq);
358 mutex_unlock(&evl->lock);
359}
360
361static int idxd_cdev_release(struct inode *node, struct file *filep)
362{
363 struct idxd_user_context *ctx = filep->private_data;
364 struct idxd_wq *wq = ctx->wq;
365 struct idxd_device *idxd = wq->idxd;
366 struct device *dev = &idxd->pdev->dev;
367
368 dev_dbg(dev, "%s called\n", __func__);
369 filep->private_data = NULL;
370
371 device_unregister(user_ctx_dev(ctx));
372
373 return 0;
374}
375
376static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
377 const char *func)
378{
379 struct device *dev = &wq->idxd->pdev->dev;
380
381 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
382 dev_info_ratelimited(dev,
383 "%s: %s: mapping too large: %lu\n",
384 current->comm, func,
385 vma->vm_end - vma->vm_start);
386 return -EINVAL;
387 }
388
389 return 0;
390}
391
392static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
393{
394 struct idxd_user_context *ctx = filp->private_data;
395 struct idxd_wq *wq = ctx->wq;
396 struct idxd_device *idxd = wq->idxd;
397 struct pci_dev *pdev = idxd->pdev;
398 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
399 unsigned long pfn;
400 int rc;
401
402 dev_dbg(&pdev->dev, "%s called\n", __func__);
403
404 /*
405 * Due to an erratum in some of the devices supported by the driver,
406 * direct user submission to the device can be unsafe.
407 * (See the INTEL-SA-01084 security advisory)
408 *
409 * For the devices that exhibit this behavior, require that the user
410 * has CAP_SYS_RAWIO capabilities.
411 */
412 if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO))
413 return -EPERM;
414
415 rc = check_vma(wq, vma, __func__);
416 if (rc < 0)
417 return rc;
418
419 vm_flags_set(vma, VM_DONTCOPY);
420 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
421 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
422 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
423 vma->vm_private_data = ctx;
424
425 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
426 vma->vm_page_prot);
427}
428
429static int idxd_submit_user_descriptor(struct idxd_user_context *ctx,
430 struct dsa_hw_desc __user *udesc)
431{
432 struct idxd_wq *wq = ctx->wq;
433 struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev;
434 const uint64_t comp_addr_align = is_dsa_dev(idxd_dev) ? 0x20 : 0x40;
435 void __iomem *portal = idxd_wq_portal_addr(wq);
436 struct dsa_hw_desc descriptor __aligned(64);
437 int rc;
438
439 rc = copy_from_user(&descriptor, udesc, sizeof(descriptor));
440 if (rc)
441 return -EFAULT;
442
443 /*
444 * DSA devices are capable of indirect ("batch") command submission.
445 * On devices where direct user submissions are not safe, we cannot
446 * allow this since there is no good way for us to verify these
447 * indirect commands.
448 */
449 if (is_dsa_dev(idxd_dev) && descriptor.opcode == DSA_OPCODE_BATCH &&
450 !wq->idxd->user_submission_safe)
451 return -EINVAL;
452 /*
453 * As per the programming specification, the completion address must be
454 * aligned to 32 or 64 bytes. If this is violated the hardware
455 * engine can get very confused (security issue).
456 */
457 if (!IS_ALIGNED(descriptor.completion_addr, comp_addr_align))
458 return -EINVAL;
459
460 if (wq_dedicated(wq))
461 iosubmit_cmds512(portal, &descriptor, 1);
462 else {
463 descriptor.priv = 0;
464 descriptor.pasid = ctx->pasid;
465 rc = idxd_enqcmds(wq, portal, &descriptor);
466 if (rc < 0)
467 return rc;
468 }
469
470 return 0;
471}
472
473static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t len,
474 loff_t *unused)
475{
476 struct dsa_hw_desc __user *udesc = (struct dsa_hw_desc __user *)buf;
477 struct idxd_user_context *ctx = filp->private_data;
478 ssize_t written = 0;
479 int i;
480
481 for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) {
482 int rc = idxd_submit_user_descriptor(ctx, udesc + i);
483
484 if (rc)
485 return written ? written : rc;
486
487 written += sizeof(struct dsa_hw_desc);
488 }
489
490 return written;
491}
492
493static __poll_t idxd_cdev_poll(struct file *filp,
494 struct poll_table_struct *wait)
495{
496 struct idxd_user_context *ctx = filp->private_data;
497 struct idxd_wq *wq = ctx->wq;
498 struct idxd_device *idxd = wq->idxd;
499 __poll_t out = 0;
500
501 poll_wait(filp, &wq->err_queue, wait);
502 spin_lock(&idxd->dev_lock);
503 if (idxd->sw_err.valid)
504 out = EPOLLIN | EPOLLRDNORM;
505 spin_unlock(&idxd->dev_lock);
506
507 return out;
508}
509
510static const struct file_operations idxd_cdev_fops = {
511 .owner = THIS_MODULE,
512 .open = idxd_cdev_open,
513 .release = idxd_cdev_release,
514 .mmap = idxd_cdev_mmap,
515 .write = idxd_cdev_write,
516 .poll = idxd_cdev_poll,
517};
518
519int idxd_cdev_get_major(struct idxd_device *idxd)
520{
521 return MAJOR(ictx[idxd->data->type].devt);
522}
523
524int idxd_wq_add_cdev(struct idxd_wq *wq)
525{
526 struct idxd_device *idxd = wq->idxd;
527 struct idxd_cdev *idxd_cdev;
528 struct cdev *cdev;
529 struct device *dev;
530 struct idxd_cdev_context *cdev_ctx;
531 int rc, minor;
532
533 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
534 if (!idxd_cdev)
535 return -ENOMEM;
536
537 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
538 idxd_cdev->wq = wq;
539 cdev = &idxd_cdev->cdev;
540 dev = cdev_dev(idxd_cdev);
541 cdev_ctx = &ictx[wq->idxd->data->type];
542 minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL);
543 if (minor < 0) {
544 kfree(idxd_cdev);
545 return minor;
546 }
547 idxd_cdev->minor = minor;
548
549 device_initialize(dev);
550 dev->parent = wq_confdev(wq);
551 dev->bus = &dsa_bus_type;
552 dev->type = &idxd_cdev_device_type;
553 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
554
555 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
556 if (rc < 0)
557 goto err;
558
559 wq->idxd_cdev = idxd_cdev;
560 cdev_init(cdev, &idxd_cdev_fops);
561 rc = cdev_device_add(cdev, dev);
562 if (rc) {
563 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
564 goto err;
565 }
566
567 return 0;
568
569 err:
570 put_device(dev);
571 wq->idxd_cdev = NULL;
572 return rc;
573}
574
575void idxd_wq_del_cdev(struct idxd_wq *wq)
576{
577 struct idxd_cdev *idxd_cdev;
578
579 idxd_cdev = wq->idxd_cdev;
580 wq->idxd_cdev = NULL;
581 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
582 put_device(cdev_dev(idxd_cdev));
583}
584
585static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
586{
587 struct device *dev = &idxd_dev->conf_dev;
588 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
589 struct idxd_device *idxd = wq->idxd;
590 int rc;
591
592 if (idxd->state != IDXD_DEV_ENABLED)
593 return -ENXIO;
594
595 mutex_lock(&wq->wq_lock);
596
597 if (!idxd_wq_driver_name_match(wq, dev)) {
598 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
599 rc = -ENODEV;
600 goto wq_err;
601 }
602
603 /*
604 * User type WQ is enabled only when SVA is enabled for two reasons:
605 * - If no IOMMU or IOMMU Passthrough without SVA, userspace
606 * can directly access physical address through the WQ.
607 * - The IDXD cdev driver does not provide any ways to pin
608 * user pages and translate the address from user VA to IOVA or
609 * PA without IOMMU SVA. Therefore the application has no way
610 * to instruct the device to perform DMA function. This makes
611 * the cdev not usable for normal application usage.
612 */
613 if (!device_user_pasid_enabled(idxd)) {
614 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU;
615 dev_dbg(&idxd->pdev->dev,
616 "User type WQ cannot be enabled without SVA.\n");
617
618 rc = -EOPNOTSUPP;
619 goto wq_err;
620 }
621
622 wq->wq = create_workqueue(dev_name(wq_confdev(wq)));
623 if (!wq->wq) {
624 rc = -ENOMEM;
625 goto wq_err;
626 }
627
628 wq->type = IDXD_WQT_USER;
629 rc = idxd_drv_enable_wq(wq);
630 if (rc < 0)
631 goto err;
632
633 rc = idxd_wq_add_cdev(wq);
634 if (rc < 0) {
635 idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
636 goto err_cdev;
637 }
638
639 idxd->cmd_status = 0;
640 mutex_unlock(&wq->wq_lock);
641 return 0;
642
643err_cdev:
644 idxd_drv_disable_wq(wq);
645err:
646 destroy_workqueue(wq->wq);
647 wq->type = IDXD_WQT_NONE;
648wq_err:
649 mutex_unlock(&wq->wq_lock);
650 return rc;
651}
652
653static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
654{
655 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
656
657 mutex_lock(&wq->wq_lock);
658 idxd_wq_del_cdev(wq);
659 idxd_drv_disable_wq(wq);
660 wq->type = IDXD_WQT_NONE;
661 destroy_workqueue(wq->wq);
662 wq->wq = NULL;
663 mutex_unlock(&wq->wq_lock);
664}
665
666static enum idxd_dev_type dev_types[] = {
667 IDXD_DEV_WQ,
668 IDXD_DEV_NONE,
669};
670
671struct idxd_device_driver idxd_user_drv = {
672 .probe = idxd_user_drv_probe,
673 .remove = idxd_user_drv_remove,
674 .name = "user",
675 .type = dev_types,
676};
677EXPORT_SYMBOL_GPL(idxd_user_drv);
678
679int idxd_cdev_register(void)
680{
681 int rc, i;
682
683 for (i = 0; i < IDXD_TYPE_MAX; i++) {
684 ida_init(&ictx[i].minor_ida);
685 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
686 ictx[i].name);
687 if (rc)
688 goto err_free_chrdev_region;
689 }
690
691 return 0;
692
693err_free_chrdev_region:
694 for (i--; i >= 0; i--)
695 unregister_chrdev_region(ictx[i].devt, MINORMASK);
696
697 return rc;
698}
699
700void idxd_cdev_remove(void)
701{
702 int i;
703
704 for (i = 0; i < IDXD_TYPE_MAX; i++) {
705 unregister_chrdev_region(ictx[i].devt, MINORMASK);
706 ida_destroy(&ictx[i].minor_ida);
707 }
708}
709
710/**
711 * idxd_copy_cr - copy completion record to user address space found by wq and
712 * PASID
713 * @wq: work queue
714 * @pasid: PASID
715 * @addr: user fault address to write
716 * @cr: completion record
717 * @len: number of bytes to copy
718 *
719 * This is called by a work that handles completion record fault.
720 *
721 * Return: number of bytes copied.
722 */
723int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,
724 void *cr, int len)
725{
726 struct device *dev = &wq->idxd->pdev->dev;
727 int left = len, status_size = 1;
728 struct idxd_user_context *ctx;
729 struct mm_struct *mm;
730
731 mutex_lock(&wq->uc_lock);
732
733 ctx = xa_load(&wq->upasid_xa, pasid);
734 if (!ctx) {
735 dev_warn(dev, "No user context\n");
736 goto out;
737 }
738
739 mm = ctx->mm;
740 /*
741 * The completion record fault handling work is running in kernel
742 * thread context. It temporarily switches to the mm to copy cr
743 * to addr in the mm.
744 */
745 kthread_use_mm(mm);
746 left = copy_to_user((void __user *)addr + status_size, cr + status_size,
747 len - status_size);
748 /*
749 * Copy status only after the rest of completion record is copied
750 * successfully so that the user gets the complete completion record
751 * when a non-zero status is polled.
752 */
753 if (!left) {
754 u8 status;
755
756 /*
757 * Ensure that the completion record's status field is written
758 * after the rest of the completion record has been written.
759 * This ensures that the user receives the correct completion
760 * record information once polling for a non-zero status.
761 */
762 wmb();
763 status = *(u8 *)cr;
764 if (put_user(status, (u8 __user *)addr))
765 left += status_size;
766 } else {
767 left += status_size;
768 }
769 kthread_unuse_mm(mm);
770
771out:
772 mutex_unlock(&wq->uc_lock);
773
774 return len - left;
775}