Loading...
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/pci.h>
7#include <linux/device.h>
8#include <linux/sched/task.h>
9#include <linux/io-64-nonatomic-lo-hi.h>
10#include <linux/cdev.h>
11#include <linux/fs.h>
12#include <linux/poll.h>
13#include <linux/iommu.h>
14#include <linux/highmem.h>
15#include <uapi/linux/idxd.h>
16#include <linux/xarray.h>
17#include "registers.h"
18#include "idxd.h"
19
20struct idxd_cdev_context {
21 const char *name;
22 dev_t devt;
23 struct ida minor_ida;
24};
25
26/*
27 * Since user file names are global in DSA devices, define their ida's as
28 * global to avoid conflict file names.
29 */
30static DEFINE_IDA(file_ida);
31static DEFINE_MUTEX(ida_lock);
32
33/*
34 * ictx is an array based off of accelerator types. enum idxd_type
35 * is used as index
36 */
37static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
38 { .name = "dsa" },
39 { .name = "iax" }
40};
41
42struct idxd_user_context {
43 struct idxd_wq *wq;
44 struct task_struct *task;
45 unsigned int pasid;
46 struct mm_struct *mm;
47 unsigned int flags;
48 struct iommu_sva *sva;
49 struct idxd_dev idxd_dev;
50 u64 counters[COUNTER_MAX];
51 int id;
52 pid_t pid;
53};
54
55static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid);
56static void idxd_xa_pasid_remove(struct idxd_user_context *ctx);
57
58static inline struct idxd_user_context *dev_to_uctx(struct device *dev)
59{
60 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
61
62 return container_of(idxd_dev, struct idxd_user_context, idxd_dev);
63}
64
65static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf)
66{
67 struct idxd_user_context *ctx = dev_to_uctx(dev);
68
69 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]);
70}
71static DEVICE_ATTR_RO(cr_faults);
72
73static ssize_t cr_fault_failures_show(struct device *dev,
74 struct device_attribute *attr, char *buf)
75{
76 struct idxd_user_context *ctx = dev_to_uctx(dev);
77
78 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]);
79}
80static DEVICE_ATTR_RO(cr_fault_failures);
81
82static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf)
83{
84 struct idxd_user_context *ctx = dev_to_uctx(dev);
85
86 return sysfs_emit(buf, "%u\n", ctx->pid);
87}
88static DEVICE_ATTR_RO(pid);
89
90static struct attribute *cdev_file_attributes[] = {
91 &dev_attr_cr_faults.attr,
92 &dev_attr_cr_fault_failures.attr,
93 &dev_attr_pid.attr,
94 NULL
95};
96
97static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n)
98{
99 struct device *dev = container_of(kobj, typeof(*dev), kobj);
100 struct idxd_user_context *ctx = dev_to_uctx(dev);
101 struct idxd_wq *wq = ctx->wq;
102
103 if (!wq_pasid_enabled(wq))
104 return 0;
105
106 return a->mode;
107}
108
109static const struct attribute_group cdev_file_attribute_group = {
110 .attrs = cdev_file_attributes,
111 .is_visible = cdev_file_attr_visible,
112};
113
114static const struct attribute_group *cdev_file_attribute_groups[] = {
115 &cdev_file_attribute_group,
116 NULL
117};
118
119static void idxd_file_dev_release(struct device *dev)
120{
121 struct idxd_user_context *ctx = dev_to_uctx(dev);
122 struct idxd_wq *wq = ctx->wq;
123 struct idxd_device *idxd = wq->idxd;
124 int rc;
125
126 mutex_lock(&ida_lock);
127 ida_free(&file_ida, ctx->id);
128 mutex_unlock(&ida_lock);
129
130 /* Wait for in-flight operations to complete. */
131 if (wq_shared(wq)) {
132 idxd_device_drain_pasid(idxd, ctx->pasid);
133 } else {
134 if (device_user_pasid_enabled(idxd)) {
135 /* The wq disable in the disable pasid function will drain the wq */
136 rc = idxd_wq_disable_pasid(wq);
137 if (rc < 0)
138 dev_err(dev, "wq disable pasid failed.\n");
139 } else {
140 idxd_wq_drain(wq);
141 }
142 }
143
144 if (ctx->sva) {
145 idxd_cdev_evl_drain_pasid(wq, ctx->pasid);
146 iommu_sva_unbind_device(ctx->sva);
147 idxd_xa_pasid_remove(ctx);
148 }
149 kfree(ctx);
150 mutex_lock(&wq->wq_lock);
151 idxd_wq_put(wq);
152 mutex_unlock(&wq->wq_lock);
153}
154
155static struct device_type idxd_cdev_file_type = {
156 .name = "idxd_file",
157 .release = idxd_file_dev_release,
158 .groups = cdev_file_attribute_groups,
159};
160
161static void idxd_cdev_dev_release(struct device *dev)
162{
163 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
164 struct idxd_cdev_context *cdev_ctx;
165 struct idxd_wq *wq = idxd_cdev->wq;
166
167 cdev_ctx = &ictx[wq->idxd->data->type];
168 ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor);
169 kfree(idxd_cdev);
170}
171
172static struct device_type idxd_cdev_device_type = {
173 .name = "idxd_cdev",
174 .release = idxd_cdev_dev_release,
175};
176
177static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
178{
179 struct cdev *cdev = inode->i_cdev;
180
181 return container_of(cdev, struct idxd_cdev, cdev);
182}
183
184static inline struct idxd_wq *inode_wq(struct inode *inode)
185{
186 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
187
188 return idxd_cdev->wq;
189}
190
191static void idxd_xa_pasid_remove(struct idxd_user_context *ctx)
192{
193 struct idxd_wq *wq = ctx->wq;
194 void *ptr;
195
196 mutex_lock(&wq->uc_lock);
197 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL);
198 if (ptr != (void *)ctx)
199 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n",
200 ctx->pasid);
201 mutex_unlock(&wq->uc_lock);
202}
203
204void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index)
205{
206 struct idxd_user_context *ctx;
207
208 if (index >= COUNTER_MAX)
209 return;
210
211 mutex_lock(&wq->uc_lock);
212 ctx = xa_load(&wq->upasid_xa, pasid);
213 if (!ctx) {
214 mutex_unlock(&wq->uc_lock);
215 return;
216 }
217 ctx->counters[index]++;
218 mutex_unlock(&wq->uc_lock);
219}
220
221static int idxd_cdev_open(struct inode *inode, struct file *filp)
222{
223 struct idxd_user_context *ctx;
224 struct idxd_device *idxd;
225 struct idxd_wq *wq;
226 struct device *dev, *fdev;
227 int rc = 0;
228 struct iommu_sva *sva;
229 unsigned int pasid;
230 struct idxd_cdev *idxd_cdev;
231
232 wq = inode_wq(inode);
233 idxd = wq->idxd;
234 dev = &idxd->pdev->dev;
235
236 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
237
238 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
239 if (!ctx)
240 return -ENOMEM;
241
242 mutex_lock(&wq->wq_lock);
243
244 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
245 rc = -EBUSY;
246 goto failed;
247 }
248
249 ctx->wq = wq;
250 filp->private_data = ctx;
251 ctx->pid = current->pid;
252
253 if (device_user_pasid_enabled(idxd)) {
254 sva = iommu_sva_bind_device(dev, current->mm);
255 if (IS_ERR(sva)) {
256 rc = PTR_ERR(sva);
257 dev_err(dev, "pasid allocation failed: %d\n", rc);
258 goto failed;
259 }
260
261 pasid = iommu_sva_get_pasid(sva);
262 if (pasid == IOMMU_PASID_INVALID) {
263 rc = -EINVAL;
264 goto failed_get_pasid;
265 }
266
267 ctx->sva = sva;
268 ctx->pasid = pasid;
269 ctx->mm = current->mm;
270
271 mutex_lock(&wq->uc_lock);
272 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL);
273 mutex_unlock(&wq->uc_lock);
274 if (rc < 0)
275 dev_warn(dev, "PASID entry already exist in xarray.\n");
276
277 if (wq_dedicated(wq)) {
278 rc = idxd_wq_set_pasid(wq, pasid);
279 if (rc < 0) {
280 dev_err(dev, "wq set pasid failed: %d\n", rc);
281 goto failed_set_pasid;
282 }
283 }
284 }
285
286 idxd_cdev = wq->idxd_cdev;
287 mutex_lock(&ida_lock);
288 ctx->id = ida_alloc(&file_ida, GFP_KERNEL);
289 mutex_unlock(&ida_lock);
290 if (ctx->id < 0) {
291 dev_warn(dev, "ida alloc failure\n");
292 goto failed_ida;
293 }
294 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE;
295 fdev = user_ctx_dev(ctx);
296 device_initialize(fdev);
297 fdev->parent = cdev_dev(idxd_cdev);
298 fdev->bus = &dsa_bus_type;
299 fdev->type = &idxd_cdev_file_type;
300
301 rc = dev_set_name(fdev, "file%d", ctx->id);
302 if (rc < 0) {
303 dev_warn(dev, "set name failure\n");
304 goto failed_dev_name;
305 }
306
307 rc = device_add(fdev);
308 if (rc < 0) {
309 dev_warn(dev, "file device add failure\n");
310 goto failed_dev_add;
311 }
312
313 idxd_wq_get(wq);
314 mutex_unlock(&wq->wq_lock);
315 return 0;
316
317failed_dev_add:
318failed_dev_name:
319 put_device(fdev);
320failed_ida:
321failed_set_pasid:
322 if (device_user_pasid_enabled(idxd))
323 idxd_xa_pasid_remove(ctx);
324failed_get_pasid:
325 if (device_user_pasid_enabled(idxd))
326 iommu_sva_unbind_device(sva);
327failed:
328 mutex_unlock(&wq->wq_lock);
329 kfree(ctx);
330 return rc;
331}
332
333static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid)
334{
335 struct idxd_device *idxd = wq->idxd;
336 struct idxd_evl *evl = idxd->evl;
337 union evl_status_reg status;
338 u16 h, t, size;
339 int ent_size = evl_ent_size(idxd);
340 struct __evl_entry *entry_head;
341
342 if (!evl)
343 return;
344
345 spin_lock(&evl->lock);
346 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET);
347 t = status.tail;
348 h = status.head;
349 size = evl->size;
350
351 while (h != t) {
352 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size));
353 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id)
354 set_bit(h, evl->bmap);
355 h = (h + 1) % size;
356 }
357 spin_unlock(&evl->lock);
358
359 drain_workqueue(wq->wq);
360}
361
362static int idxd_cdev_release(struct inode *node, struct file *filep)
363{
364 struct idxd_user_context *ctx = filep->private_data;
365 struct idxd_wq *wq = ctx->wq;
366 struct idxd_device *idxd = wq->idxd;
367 struct device *dev = &idxd->pdev->dev;
368
369 dev_dbg(dev, "%s called\n", __func__);
370 filep->private_data = NULL;
371
372 device_unregister(user_ctx_dev(ctx));
373
374 return 0;
375}
376
377static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
378 const char *func)
379{
380 struct device *dev = &wq->idxd->pdev->dev;
381
382 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
383 dev_info_ratelimited(dev,
384 "%s: %s: mapping too large: %lu\n",
385 current->comm, func,
386 vma->vm_end - vma->vm_start);
387 return -EINVAL;
388 }
389
390 return 0;
391}
392
393static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
394{
395 struct idxd_user_context *ctx = filp->private_data;
396 struct idxd_wq *wq = ctx->wq;
397 struct idxd_device *idxd = wq->idxd;
398 struct pci_dev *pdev = idxd->pdev;
399 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
400 unsigned long pfn;
401 int rc;
402
403 dev_dbg(&pdev->dev, "%s called\n", __func__);
404 rc = check_vma(wq, vma, __func__);
405 if (rc < 0)
406 return rc;
407
408 vm_flags_set(vma, VM_DONTCOPY);
409 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
410 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
411 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
412 vma->vm_private_data = ctx;
413
414 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
415 vma->vm_page_prot);
416}
417
418static __poll_t idxd_cdev_poll(struct file *filp,
419 struct poll_table_struct *wait)
420{
421 struct idxd_user_context *ctx = filp->private_data;
422 struct idxd_wq *wq = ctx->wq;
423 struct idxd_device *idxd = wq->idxd;
424 __poll_t out = 0;
425
426 poll_wait(filp, &wq->err_queue, wait);
427 spin_lock(&idxd->dev_lock);
428 if (idxd->sw_err.valid)
429 out = EPOLLIN | EPOLLRDNORM;
430 spin_unlock(&idxd->dev_lock);
431
432 return out;
433}
434
435static const struct file_operations idxd_cdev_fops = {
436 .owner = THIS_MODULE,
437 .open = idxd_cdev_open,
438 .release = idxd_cdev_release,
439 .mmap = idxd_cdev_mmap,
440 .poll = idxd_cdev_poll,
441};
442
443int idxd_cdev_get_major(struct idxd_device *idxd)
444{
445 return MAJOR(ictx[idxd->data->type].devt);
446}
447
448int idxd_wq_add_cdev(struct idxd_wq *wq)
449{
450 struct idxd_device *idxd = wq->idxd;
451 struct idxd_cdev *idxd_cdev;
452 struct cdev *cdev;
453 struct device *dev;
454 struct idxd_cdev_context *cdev_ctx;
455 int rc, minor;
456
457 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
458 if (!idxd_cdev)
459 return -ENOMEM;
460
461 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
462 idxd_cdev->wq = wq;
463 cdev = &idxd_cdev->cdev;
464 dev = cdev_dev(idxd_cdev);
465 cdev_ctx = &ictx[wq->idxd->data->type];
466 minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL);
467 if (minor < 0) {
468 kfree(idxd_cdev);
469 return minor;
470 }
471 idxd_cdev->minor = minor;
472
473 device_initialize(dev);
474 dev->parent = wq_confdev(wq);
475 dev->bus = &dsa_bus_type;
476 dev->type = &idxd_cdev_device_type;
477 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
478
479 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
480 if (rc < 0)
481 goto err;
482
483 wq->idxd_cdev = idxd_cdev;
484 cdev_init(cdev, &idxd_cdev_fops);
485 rc = cdev_device_add(cdev, dev);
486 if (rc) {
487 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
488 goto err;
489 }
490
491 return 0;
492
493 err:
494 put_device(dev);
495 wq->idxd_cdev = NULL;
496 return rc;
497}
498
499void idxd_wq_del_cdev(struct idxd_wq *wq)
500{
501 struct idxd_cdev *idxd_cdev;
502
503 idxd_cdev = wq->idxd_cdev;
504 ida_destroy(&file_ida);
505 wq->idxd_cdev = NULL;
506 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
507 put_device(cdev_dev(idxd_cdev));
508}
509
510static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
511{
512 struct device *dev = &idxd_dev->conf_dev;
513 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
514 struct idxd_device *idxd = wq->idxd;
515 int rc;
516
517 if (idxd->state != IDXD_DEV_ENABLED)
518 return -ENXIO;
519
520 /*
521 * User type WQ is enabled only when SVA is enabled for two reasons:
522 * - If no IOMMU or IOMMU Passthrough without SVA, userspace
523 * can directly access physical address through the WQ.
524 * - The IDXD cdev driver does not provide any ways to pin
525 * user pages and translate the address from user VA to IOVA or
526 * PA without IOMMU SVA. Therefore the application has no way
527 * to instruct the device to perform DMA function. This makes
528 * the cdev not usable for normal application usage.
529 */
530 if (!device_user_pasid_enabled(idxd)) {
531 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU;
532 dev_dbg(&idxd->pdev->dev,
533 "User type WQ cannot be enabled without SVA.\n");
534
535 return -EOPNOTSUPP;
536 }
537
538 mutex_lock(&wq->wq_lock);
539
540 if (!idxd_wq_driver_name_match(wq, dev)) {
541 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
542 rc = -ENODEV;
543 goto wq_err;
544 }
545
546 wq->wq = create_workqueue(dev_name(wq_confdev(wq)));
547 if (!wq->wq) {
548 rc = -ENOMEM;
549 goto wq_err;
550 }
551
552 wq->type = IDXD_WQT_USER;
553 rc = idxd_drv_enable_wq(wq);
554 if (rc < 0)
555 goto err;
556
557 rc = idxd_wq_add_cdev(wq);
558 if (rc < 0) {
559 idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
560 goto err_cdev;
561 }
562
563 idxd->cmd_status = 0;
564 mutex_unlock(&wq->wq_lock);
565 return 0;
566
567err_cdev:
568 idxd_drv_disable_wq(wq);
569err:
570 destroy_workqueue(wq->wq);
571 wq->type = IDXD_WQT_NONE;
572wq_err:
573 mutex_unlock(&wq->wq_lock);
574 return rc;
575}
576
577static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
578{
579 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
580
581 mutex_lock(&wq->wq_lock);
582 idxd_wq_del_cdev(wq);
583 idxd_drv_disable_wq(wq);
584 wq->type = IDXD_WQT_NONE;
585 destroy_workqueue(wq->wq);
586 wq->wq = NULL;
587 mutex_unlock(&wq->wq_lock);
588}
589
590static enum idxd_dev_type dev_types[] = {
591 IDXD_DEV_WQ,
592 IDXD_DEV_NONE,
593};
594
595struct idxd_device_driver idxd_user_drv = {
596 .probe = idxd_user_drv_probe,
597 .remove = idxd_user_drv_remove,
598 .name = "user",
599 .type = dev_types,
600};
601EXPORT_SYMBOL_GPL(idxd_user_drv);
602
603int idxd_cdev_register(void)
604{
605 int rc, i;
606
607 for (i = 0; i < IDXD_TYPE_MAX; i++) {
608 ida_init(&ictx[i].minor_ida);
609 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
610 ictx[i].name);
611 if (rc)
612 goto err_free_chrdev_region;
613 }
614
615 return 0;
616
617err_free_chrdev_region:
618 for (i--; i >= 0; i--)
619 unregister_chrdev_region(ictx[i].devt, MINORMASK);
620
621 return rc;
622}
623
624void idxd_cdev_remove(void)
625{
626 int i;
627
628 for (i = 0; i < IDXD_TYPE_MAX; i++) {
629 unregister_chrdev_region(ictx[i].devt, MINORMASK);
630 ida_destroy(&ictx[i].minor_ida);
631 }
632}
633
634/**
635 * idxd_copy_cr - copy completion record to user address space found by wq and
636 * PASID
637 * @wq: work queue
638 * @pasid: PASID
639 * @addr: user fault address to write
640 * @cr: completion record
641 * @len: number of bytes to copy
642 *
643 * This is called by a work that handles completion record fault.
644 *
645 * Return: number of bytes copied.
646 */
647int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,
648 void *cr, int len)
649{
650 struct device *dev = &wq->idxd->pdev->dev;
651 int left = len, status_size = 1;
652 struct idxd_user_context *ctx;
653 struct mm_struct *mm;
654
655 mutex_lock(&wq->uc_lock);
656
657 ctx = xa_load(&wq->upasid_xa, pasid);
658 if (!ctx) {
659 dev_warn(dev, "No user context\n");
660 goto out;
661 }
662
663 mm = ctx->mm;
664 /*
665 * The completion record fault handling work is running in kernel
666 * thread context. It temporarily switches to the mm to copy cr
667 * to addr in the mm.
668 */
669 kthread_use_mm(mm);
670 left = copy_to_user((void __user *)addr + status_size, cr + status_size,
671 len - status_size);
672 /*
673 * Copy status only after the rest of completion record is copied
674 * successfully so that the user gets the complete completion record
675 * when a non-zero status is polled.
676 */
677 if (!left) {
678 u8 status;
679
680 /*
681 * Ensure that the completion record's status field is written
682 * after the rest of the completion record has been written.
683 * This ensures that the user receives the correct completion
684 * record information once polling for a non-zero status.
685 */
686 wmb();
687 status = *(u8 *)cr;
688 if (put_user(status, (u8 __user *)addr))
689 left += status_size;
690 } else {
691 left += status_size;
692 }
693 kthread_unuse_mm(mm);
694
695out:
696 mutex_unlock(&wq->uc_lock);
697
698 return len - left;
699}
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/pci.h>
7#include <linux/device.h>
8#include <linux/sched/task.h>
9#include <linux/intel-svm.h>
10#include <linux/io-64-nonatomic-lo-hi.h>
11#include <linux/cdev.h>
12#include <linux/fs.h>
13#include <linux/poll.h>
14#include <linux/iommu.h>
15#include <uapi/linux/idxd.h>
16#include "registers.h"
17#include "idxd.h"
18
19struct idxd_cdev_context {
20 const char *name;
21 dev_t devt;
22 struct ida minor_ida;
23};
24
25/*
26 * ictx is an array based off of accelerator types. enum idxd_type
27 * is used as index
28 */
29static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
30 { .name = "dsa" },
31 { .name = "iax" }
32};
33
34struct idxd_user_context {
35 struct idxd_wq *wq;
36 struct task_struct *task;
37 unsigned int pasid;
38 unsigned int flags;
39 struct iommu_sva *sva;
40};
41
42static void idxd_cdev_dev_release(struct device *dev)
43{
44 struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev);
45 struct idxd_cdev_context *cdev_ctx;
46 struct idxd_wq *wq = idxd_cdev->wq;
47
48 cdev_ctx = &ictx[wq->idxd->data->type];
49 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
50 kfree(idxd_cdev);
51}
52
53static struct device_type idxd_cdev_device_type = {
54 .name = "idxd_cdev",
55 .release = idxd_cdev_dev_release,
56};
57
58static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
59{
60 struct cdev *cdev = inode->i_cdev;
61
62 return container_of(cdev, struct idxd_cdev, cdev);
63}
64
65static inline struct idxd_wq *inode_wq(struct inode *inode)
66{
67 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
68
69 return idxd_cdev->wq;
70}
71
72static int idxd_cdev_open(struct inode *inode, struct file *filp)
73{
74 struct idxd_user_context *ctx;
75 struct idxd_device *idxd;
76 struct idxd_wq *wq;
77 struct device *dev;
78 int rc = 0;
79 struct iommu_sva *sva;
80 unsigned int pasid;
81
82 wq = inode_wq(inode);
83 idxd = wq->idxd;
84 dev = &idxd->pdev->dev;
85
86 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
87
88 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
89 if (!ctx)
90 return -ENOMEM;
91
92 mutex_lock(&wq->wq_lock);
93
94 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
95 rc = -EBUSY;
96 goto failed;
97 }
98
99 ctx->wq = wq;
100 filp->private_data = ctx;
101
102 if (device_pasid_enabled(idxd)) {
103 sva = iommu_sva_bind_device(dev, current->mm, NULL);
104 if (IS_ERR(sva)) {
105 rc = PTR_ERR(sva);
106 dev_err(dev, "pasid allocation failed: %d\n", rc);
107 goto failed;
108 }
109
110 pasid = iommu_sva_get_pasid(sva);
111 if (pasid == IOMMU_PASID_INVALID) {
112 iommu_sva_unbind_device(sva);
113 rc = -EINVAL;
114 goto failed;
115 }
116
117 ctx->sva = sva;
118 ctx->pasid = pasid;
119
120 if (wq_dedicated(wq)) {
121 rc = idxd_wq_set_pasid(wq, pasid);
122 if (rc < 0) {
123 iommu_sva_unbind_device(sva);
124 dev_err(dev, "wq set pasid failed: %d\n", rc);
125 goto failed;
126 }
127 }
128 }
129
130 idxd_wq_get(wq);
131 mutex_unlock(&wq->wq_lock);
132 return 0;
133
134 failed:
135 mutex_unlock(&wq->wq_lock);
136 kfree(ctx);
137 return rc;
138}
139
140static int idxd_cdev_release(struct inode *node, struct file *filep)
141{
142 struct idxd_user_context *ctx = filep->private_data;
143 struct idxd_wq *wq = ctx->wq;
144 struct idxd_device *idxd = wq->idxd;
145 struct device *dev = &idxd->pdev->dev;
146 int rc;
147
148 dev_dbg(dev, "%s called\n", __func__);
149 filep->private_data = NULL;
150
151 /* Wait for in-flight operations to complete. */
152 if (wq_shared(wq)) {
153 idxd_device_drain_pasid(idxd, ctx->pasid);
154 } else {
155 if (device_pasid_enabled(idxd)) {
156 /* The wq disable in the disable pasid function will drain the wq */
157 rc = idxd_wq_disable_pasid(wq);
158 if (rc < 0)
159 dev_err(dev, "wq disable pasid failed.\n");
160 } else {
161 idxd_wq_drain(wq);
162 }
163 }
164
165 if (ctx->sva)
166 iommu_sva_unbind_device(ctx->sva);
167 kfree(ctx);
168 mutex_lock(&wq->wq_lock);
169 idxd_wq_put(wq);
170 mutex_unlock(&wq->wq_lock);
171 return 0;
172}
173
174static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
175 const char *func)
176{
177 struct device *dev = &wq->idxd->pdev->dev;
178
179 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
180 dev_info_ratelimited(dev,
181 "%s: %s: mapping too large: %lu\n",
182 current->comm, func,
183 vma->vm_end - vma->vm_start);
184 return -EINVAL;
185 }
186
187 return 0;
188}
189
190static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
191{
192 struct idxd_user_context *ctx = filp->private_data;
193 struct idxd_wq *wq = ctx->wq;
194 struct idxd_device *idxd = wq->idxd;
195 struct pci_dev *pdev = idxd->pdev;
196 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
197 unsigned long pfn;
198 int rc;
199
200 dev_dbg(&pdev->dev, "%s called\n", __func__);
201 rc = check_vma(wq, vma, __func__);
202 if (rc < 0)
203 return rc;
204
205 vma->vm_flags |= VM_DONTCOPY;
206 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
207 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
208 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
209 vma->vm_private_data = ctx;
210
211 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
212 vma->vm_page_prot);
213}
214
215static __poll_t idxd_cdev_poll(struct file *filp,
216 struct poll_table_struct *wait)
217{
218 struct idxd_user_context *ctx = filp->private_data;
219 struct idxd_wq *wq = ctx->wq;
220 struct idxd_device *idxd = wq->idxd;
221 unsigned long flags;
222 __poll_t out = 0;
223
224 poll_wait(filp, &wq->err_queue, wait);
225 spin_lock_irqsave(&idxd->dev_lock, flags);
226 if (idxd->sw_err.valid)
227 out = EPOLLIN | EPOLLRDNORM;
228 spin_unlock_irqrestore(&idxd->dev_lock, flags);
229
230 return out;
231}
232
233static const struct file_operations idxd_cdev_fops = {
234 .owner = THIS_MODULE,
235 .open = idxd_cdev_open,
236 .release = idxd_cdev_release,
237 .mmap = idxd_cdev_mmap,
238 .poll = idxd_cdev_poll,
239};
240
241int idxd_cdev_get_major(struct idxd_device *idxd)
242{
243 return MAJOR(ictx[idxd->data->type].devt);
244}
245
246int idxd_wq_add_cdev(struct idxd_wq *wq)
247{
248 struct idxd_device *idxd = wq->idxd;
249 struct idxd_cdev *idxd_cdev;
250 struct cdev *cdev;
251 struct device *dev;
252 struct idxd_cdev_context *cdev_ctx;
253 int rc, minor;
254
255 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
256 if (!idxd_cdev)
257 return -ENOMEM;
258
259 idxd_cdev->wq = wq;
260 cdev = &idxd_cdev->cdev;
261 dev = &idxd_cdev->dev;
262 cdev_ctx = &ictx[wq->idxd->data->type];
263 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
264 if (minor < 0) {
265 kfree(idxd_cdev);
266 return minor;
267 }
268 idxd_cdev->minor = minor;
269
270 device_initialize(dev);
271 dev->parent = &wq->conf_dev;
272 dev->bus = &dsa_bus_type;
273 dev->type = &idxd_cdev_device_type;
274 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
275
276 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
277 if (rc < 0)
278 goto err;
279
280 wq->idxd_cdev = idxd_cdev;
281 cdev_init(cdev, &idxd_cdev_fops);
282 rc = cdev_device_add(cdev, dev);
283 if (rc) {
284 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
285 goto err;
286 }
287
288 return 0;
289
290 err:
291 put_device(dev);
292 wq->idxd_cdev = NULL;
293 return rc;
294}
295
296void idxd_wq_del_cdev(struct idxd_wq *wq)
297{
298 struct idxd_cdev *idxd_cdev;
299
300 idxd_cdev = wq->idxd_cdev;
301 wq->idxd_cdev = NULL;
302 cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev);
303 put_device(&idxd_cdev->dev);
304}
305
306int idxd_cdev_register(void)
307{
308 int rc, i;
309
310 for (i = 0; i < IDXD_TYPE_MAX; i++) {
311 ida_init(&ictx[i].minor_ida);
312 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
313 ictx[i].name);
314 if (rc)
315 return rc;
316 }
317
318 return 0;
319}
320
321void idxd_cdev_remove(void)
322{
323 int i;
324
325 for (i = 0; i < IDXD_TYPE_MAX; i++) {
326 unregister_chrdev_region(ictx[i].devt, MINORMASK);
327 ida_destroy(&ictx[i].minor_ida);
328 }
329}