Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 */
6
7#define pr_fmt(fmt) "iommu: " fmt
8
9#include <linux/amba/bus.h>
10#include <linux/device.h>
11#include <linux/kernel.h>
12#include <linux/bits.h>
13#include <linux/bug.h>
14#include <linux/types.h>
15#include <linux/init.h>
16#include <linux/export.h>
17#include <linux/slab.h>
18#include <linux/errno.h>
19#include <linux/host1x_context_bus.h>
20#include <linux/iommu.h>
21#include <linux/idr.h>
22#include <linux/err.h>
23#include <linux/pci.h>
24#include <linux/pci-ats.h>
25#include <linux/bitops.h>
26#include <linux/platform_device.h>
27#include <linux/property.h>
28#include <linux/fsl/mc.h>
29#include <linux/module.h>
30#include <linux/cc_platform.h>
31#include <trace/events/iommu.h>
32#include <linux/sched/mm.h>
33
34#include "dma-iommu.h"
35
36#include "iommu-sva.h"
37
38static struct kset *iommu_group_kset;
39static DEFINE_IDA(iommu_group_ida);
40
41static unsigned int iommu_def_domain_type __read_mostly;
42static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_DMA_STRICT);
43static u32 iommu_cmd_line __read_mostly;
44
45struct iommu_group {
46 struct kobject kobj;
47 struct kobject *devices_kobj;
48 struct list_head devices;
49 struct xarray pasid_array;
50 struct mutex mutex;
51 void *iommu_data;
52 void (*iommu_data_release)(void *iommu_data);
53 char *name;
54 int id;
55 struct iommu_domain *default_domain;
56 struct iommu_domain *blocking_domain;
57 struct iommu_domain *domain;
58 struct list_head entry;
59 unsigned int owner_cnt;
60 void *owner;
61};
62
63struct group_device {
64 struct list_head list;
65 struct device *dev;
66 char *name;
67};
68
69struct iommu_group_attribute {
70 struct attribute attr;
71 ssize_t (*show)(struct iommu_group *group, char *buf);
72 ssize_t (*store)(struct iommu_group *group,
73 const char *buf, size_t count);
74};
75
76static const char * const iommu_group_resv_type_string[] = {
77 [IOMMU_RESV_DIRECT] = "direct",
78 [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable",
79 [IOMMU_RESV_RESERVED] = "reserved",
80 [IOMMU_RESV_MSI] = "msi",
81 [IOMMU_RESV_SW_MSI] = "msi",
82};
83
84#define IOMMU_CMD_LINE_DMA_API BIT(0)
85#define IOMMU_CMD_LINE_STRICT BIT(1)
86
87static int iommu_bus_notifier(struct notifier_block *nb,
88 unsigned long action, void *data);
89static int iommu_alloc_default_domain(struct iommu_group *group,
90 struct device *dev);
91static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
92 unsigned type);
93static int __iommu_attach_device(struct iommu_domain *domain,
94 struct device *dev);
95static int __iommu_attach_group(struct iommu_domain *domain,
96 struct iommu_group *group);
97static int __iommu_group_set_domain(struct iommu_group *group,
98 struct iommu_domain *new_domain);
99static int iommu_create_device_direct_mappings(struct iommu_group *group,
100 struct device *dev);
101static struct iommu_group *iommu_group_get_for_dev(struct device *dev);
102static ssize_t iommu_group_store_type(struct iommu_group *group,
103 const char *buf, size_t count);
104
105#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
106struct iommu_group_attribute iommu_group_attr_##_name = \
107 __ATTR(_name, _mode, _show, _store)
108
109#define to_iommu_group_attr(_attr) \
110 container_of(_attr, struct iommu_group_attribute, attr)
111#define to_iommu_group(_kobj) \
112 container_of(_kobj, struct iommu_group, kobj)
113
114static LIST_HEAD(iommu_device_list);
115static DEFINE_SPINLOCK(iommu_device_lock);
116
117static struct bus_type * const iommu_buses[] = {
118 &platform_bus_type,
119#ifdef CONFIG_PCI
120 &pci_bus_type,
121#endif
122#ifdef CONFIG_ARM_AMBA
123 &amba_bustype,
124#endif
125#ifdef CONFIG_FSL_MC_BUS
126 &fsl_mc_bus_type,
127#endif
128#ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS
129 &host1x_context_device_bus_type,
130#endif
131};
132
133/*
134 * Use a function instead of an array here because the domain-type is a
135 * bit-field, so an array would waste memory.
136 */
137static const char *iommu_domain_type_str(unsigned int t)
138{
139 switch (t) {
140 case IOMMU_DOMAIN_BLOCKED:
141 return "Blocked";
142 case IOMMU_DOMAIN_IDENTITY:
143 return "Passthrough";
144 case IOMMU_DOMAIN_UNMANAGED:
145 return "Unmanaged";
146 case IOMMU_DOMAIN_DMA:
147 case IOMMU_DOMAIN_DMA_FQ:
148 return "Translated";
149 default:
150 return "Unknown";
151 }
152}
153
154static int __init iommu_subsys_init(void)
155{
156 struct notifier_block *nb;
157
158 if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) {
159 if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
160 iommu_set_default_passthrough(false);
161 else
162 iommu_set_default_translated(false);
163
164 if (iommu_default_passthrough() && cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
165 pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
166 iommu_set_default_translated(false);
167 }
168 }
169
170 if (!iommu_default_passthrough() && !iommu_dma_strict)
171 iommu_def_domain_type = IOMMU_DOMAIN_DMA_FQ;
172
173 pr_info("Default domain type: %s %s\n",
174 iommu_domain_type_str(iommu_def_domain_type),
175 (iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ?
176 "(set via kernel command line)" : "");
177
178 if (!iommu_default_passthrough())
179 pr_info("DMA domain TLB invalidation policy: %s mode %s\n",
180 iommu_dma_strict ? "strict" : "lazy",
181 (iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ?
182 "(set via kernel command line)" : "");
183
184 nb = kcalloc(ARRAY_SIZE(iommu_buses), sizeof(*nb), GFP_KERNEL);
185 if (!nb)
186 return -ENOMEM;
187
188 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
189 nb[i].notifier_call = iommu_bus_notifier;
190 bus_register_notifier(iommu_buses[i], &nb[i]);
191 }
192
193 return 0;
194}
195subsys_initcall(iommu_subsys_init);
196
197static int remove_iommu_group(struct device *dev, void *data)
198{
199 if (dev->iommu && dev->iommu->iommu_dev == data)
200 iommu_release_device(dev);
201
202 return 0;
203}
204
205/**
206 * iommu_device_register() - Register an IOMMU hardware instance
207 * @iommu: IOMMU handle for the instance
208 * @ops: IOMMU ops to associate with the instance
209 * @hwdev: (optional) actual instance device, used for fwnode lookup
210 *
211 * Return: 0 on success, or an error.
212 */
213int iommu_device_register(struct iommu_device *iommu,
214 const struct iommu_ops *ops, struct device *hwdev)
215{
216 int err = 0;
217
218 /* We need to be able to take module references appropriately */
219 if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner))
220 return -EINVAL;
221 /*
222 * Temporarily enforce global restriction to a single driver. This was
223 * already the de-facto behaviour, since any possible combination of
224 * existing drivers would compete for at least the PCI or platform bus.
225 */
226 if (iommu_buses[0]->iommu_ops && iommu_buses[0]->iommu_ops != ops)
227 return -EBUSY;
228
229 iommu->ops = ops;
230 if (hwdev)
231 iommu->fwnode = dev_fwnode(hwdev);
232
233 spin_lock(&iommu_device_lock);
234 list_add_tail(&iommu->list, &iommu_device_list);
235 spin_unlock(&iommu_device_lock);
236
237 for (int i = 0; i < ARRAY_SIZE(iommu_buses) && !err; i++) {
238 iommu_buses[i]->iommu_ops = ops;
239 err = bus_iommu_probe(iommu_buses[i]);
240 }
241 if (err)
242 iommu_device_unregister(iommu);
243 return err;
244}
245EXPORT_SYMBOL_GPL(iommu_device_register);
246
247void iommu_device_unregister(struct iommu_device *iommu)
248{
249 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++)
250 bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group);
251
252 spin_lock(&iommu_device_lock);
253 list_del(&iommu->list);
254 spin_unlock(&iommu_device_lock);
255}
256EXPORT_SYMBOL_GPL(iommu_device_unregister);
257
258static struct dev_iommu *dev_iommu_get(struct device *dev)
259{
260 struct dev_iommu *param = dev->iommu;
261
262 if (param)
263 return param;
264
265 param = kzalloc(sizeof(*param), GFP_KERNEL);
266 if (!param)
267 return NULL;
268
269 mutex_init(¶m->lock);
270 dev->iommu = param;
271 return param;
272}
273
274static void dev_iommu_free(struct device *dev)
275{
276 struct dev_iommu *param = dev->iommu;
277
278 dev->iommu = NULL;
279 if (param->fwspec) {
280 fwnode_handle_put(param->fwspec->iommu_fwnode);
281 kfree(param->fwspec);
282 }
283 kfree(param);
284}
285
286static u32 dev_iommu_get_max_pasids(struct device *dev)
287{
288 u32 max_pasids = 0, bits = 0;
289 int ret;
290
291 if (dev_is_pci(dev)) {
292 ret = pci_max_pasids(to_pci_dev(dev));
293 if (ret > 0)
294 max_pasids = ret;
295 } else {
296 ret = device_property_read_u32(dev, "pasid-num-bits", &bits);
297 if (!ret)
298 max_pasids = 1UL << bits;
299 }
300
301 return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);
302}
303
304static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
305{
306 const struct iommu_ops *ops = dev->bus->iommu_ops;
307 struct iommu_device *iommu_dev;
308 struct iommu_group *group;
309 static DEFINE_MUTEX(iommu_probe_device_lock);
310 int ret;
311
312 if (!ops)
313 return -ENODEV;
314 /*
315 * Serialise to avoid races between IOMMU drivers registering in
316 * parallel and/or the "replay" calls from ACPI/OF code via client
317 * driver probe. Once the latter have been cleaned up we should
318 * probably be able to use device_lock() here to minimise the scope,
319 * but for now enforcing a simple global ordering is fine.
320 */
321 mutex_lock(&iommu_probe_device_lock);
322 if (!dev_iommu_get(dev)) {
323 ret = -ENOMEM;
324 goto err_unlock;
325 }
326
327 if (!try_module_get(ops->owner)) {
328 ret = -EINVAL;
329 goto err_free;
330 }
331
332 iommu_dev = ops->probe_device(dev);
333 if (IS_ERR(iommu_dev)) {
334 ret = PTR_ERR(iommu_dev);
335 goto out_module_put;
336 }
337
338 dev->iommu->iommu_dev = iommu_dev;
339 dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
340
341 group = iommu_group_get_for_dev(dev);
342 if (IS_ERR(group)) {
343 ret = PTR_ERR(group);
344 goto out_release;
345 }
346
347 mutex_lock(&group->mutex);
348 if (group_list && !group->default_domain && list_empty(&group->entry))
349 list_add_tail(&group->entry, group_list);
350 mutex_unlock(&group->mutex);
351 iommu_group_put(group);
352
353 mutex_unlock(&iommu_probe_device_lock);
354 iommu_device_link(iommu_dev, dev);
355
356 return 0;
357
358out_release:
359 if (ops->release_device)
360 ops->release_device(dev);
361
362out_module_put:
363 module_put(ops->owner);
364
365err_free:
366 dev_iommu_free(dev);
367
368err_unlock:
369 mutex_unlock(&iommu_probe_device_lock);
370
371 return ret;
372}
373
374int iommu_probe_device(struct device *dev)
375{
376 const struct iommu_ops *ops;
377 struct iommu_group *group;
378 int ret;
379
380 ret = __iommu_probe_device(dev, NULL);
381 if (ret)
382 goto err_out;
383
384 group = iommu_group_get(dev);
385 if (!group) {
386 ret = -ENODEV;
387 goto err_release;
388 }
389
390 /*
391 * Try to allocate a default domain - needs support from the
392 * IOMMU driver. There are still some drivers which don't
393 * support default domains, so the return value is not yet
394 * checked.
395 */
396 mutex_lock(&group->mutex);
397 iommu_alloc_default_domain(group, dev);
398
399 /*
400 * If device joined an existing group which has been claimed, don't
401 * attach the default domain.
402 */
403 if (group->default_domain && !group->owner) {
404 ret = __iommu_attach_device(group->default_domain, dev);
405 if (ret) {
406 mutex_unlock(&group->mutex);
407 iommu_group_put(group);
408 goto err_release;
409 }
410 }
411
412 iommu_create_device_direct_mappings(group, dev);
413
414 mutex_unlock(&group->mutex);
415 iommu_group_put(group);
416
417 ops = dev_iommu_ops(dev);
418 if (ops->probe_finalize)
419 ops->probe_finalize(dev);
420
421 return 0;
422
423err_release:
424 iommu_release_device(dev);
425
426err_out:
427 return ret;
428
429}
430
431void iommu_release_device(struct device *dev)
432{
433 const struct iommu_ops *ops;
434
435 if (!dev->iommu)
436 return;
437
438 iommu_device_unlink(dev->iommu->iommu_dev, dev);
439
440 ops = dev_iommu_ops(dev);
441 if (ops->release_device)
442 ops->release_device(dev);
443
444 iommu_group_remove_device(dev);
445 module_put(ops->owner);
446 dev_iommu_free(dev);
447}
448
449static int __init iommu_set_def_domain_type(char *str)
450{
451 bool pt;
452 int ret;
453
454 ret = kstrtobool(str, &pt);
455 if (ret)
456 return ret;
457
458 if (pt)
459 iommu_set_default_passthrough(true);
460 else
461 iommu_set_default_translated(true);
462
463 return 0;
464}
465early_param("iommu.passthrough", iommu_set_def_domain_type);
466
467static int __init iommu_dma_setup(char *str)
468{
469 int ret = kstrtobool(str, &iommu_dma_strict);
470
471 if (!ret)
472 iommu_cmd_line |= IOMMU_CMD_LINE_STRICT;
473 return ret;
474}
475early_param("iommu.strict", iommu_dma_setup);
476
477void iommu_set_dma_strict(void)
478{
479 iommu_dma_strict = true;
480 if (iommu_def_domain_type == IOMMU_DOMAIN_DMA_FQ)
481 iommu_def_domain_type = IOMMU_DOMAIN_DMA;
482}
483
484static ssize_t iommu_group_attr_show(struct kobject *kobj,
485 struct attribute *__attr, char *buf)
486{
487 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
488 struct iommu_group *group = to_iommu_group(kobj);
489 ssize_t ret = -EIO;
490
491 if (attr->show)
492 ret = attr->show(group, buf);
493 return ret;
494}
495
496static ssize_t iommu_group_attr_store(struct kobject *kobj,
497 struct attribute *__attr,
498 const char *buf, size_t count)
499{
500 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
501 struct iommu_group *group = to_iommu_group(kobj);
502 ssize_t ret = -EIO;
503
504 if (attr->store)
505 ret = attr->store(group, buf, count);
506 return ret;
507}
508
509static const struct sysfs_ops iommu_group_sysfs_ops = {
510 .show = iommu_group_attr_show,
511 .store = iommu_group_attr_store,
512};
513
514static int iommu_group_create_file(struct iommu_group *group,
515 struct iommu_group_attribute *attr)
516{
517 return sysfs_create_file(&group->kobj, &attr->attr);
518}
519
520static void iommu_group_remove_file(struct iommu_group *group,
521 struct iommu_group_attribute *attr)
522{
523 sysfs_remove_file(&group->kobj, &attr->attr);
524}
525
526static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
527{
528 return sprintf(buf, "%s\n", group->name);
529}
530
531/**
532 * iommu_insert_resv_region - Insert a new region in the
533 * list of reserved regions.
534 * @new: new region to insert
535 * @regions: list of regions
536 *
537 * Elements are sorted by start address and overlapping segments
538 * of the same type are merged.
539 */
540static int iommu_insert_resv_region(struct iommu_resv_region *new,
541 struct list_head *regions)
542{
543 struct iommu_resv_region *iter, *tmp, *nr, *top;
544 LIST_HEAD(stack);
545
546 nr = iommu_alloc_resv_region(new->start, new->length,
547 new->prot, new->type, GFP_KERNEL);
548 if (!nr)
549 return -ENOMEM;
550
551 /* First add the new element based on start address sorting */
552 list_for_each_entry(iter, regions, list) {
553 if (nr->start < iter->start ||
554 (nr->start == iter->start && nr->type <= iter->type))
555 break;
556 }
557 list_add_tail(&nr->list, &iter->list);
558
559 /* Merge overlapping segments of type nr->type in @regions, if any */
560 list_for_each_entry_safe(iter, tmp, regions, list) {
561 phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
562
563 /* no merge needed on elements of different types than @new */
564 if (iter->type != new->type) {
565 list_move_tail(&iter->list, &stack);
566 continue;
567 }
568
569 /* look for the last stack element of same type as @iter */
570 list_for_each_entry_reverse(top, &stack, list)
571 if (top->type == iter->type)
572 goto check_overlap;
573
574 list_move_tail(&iter->list, &stack);
575 continue;
576
577check_overlap:
578 top_end = top->start + top->length - 1;
579
580 if (iter->start > top_end + 1) {
581 list_move_tail(&iter->list, &stack);
582 } else {
583 top->length = max(top_end, iter_end) - top->start + 1;
584 list_del(&iter->list);
585 kfree(iter);
586 }
587 }
588 list_splice(&stack, regions);
589 return 0;
590}
591
592static int
593iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
594 struct list_head *group_resv_regions)
595{
596 struct iommu_resv_region *entry;
597 int ret = 0;
598
599 list_for_each_entry(entry, dev_resv_regions, list) {
600 ret = iommu_insert_resv_region(entry, group_resv_regions);
601 if (ret)
602 break;
603 }
604 return ret;
605}
606
607int iommu_get_group_resv_regions(struct iommu_group *group,
608 struct list_head *head)
609{
610 struct group_device *device;
611 int ret = 0;
612
613 mutex_lock(&group->mutex);
614 list_for_each_entry(device, &group->devices, list) {
615 struct list_head dev_resv_regions;
616
617 /*
618 * Non-API groups still expose reserved_regions in sysfs,
619 * so filter out calls that get here that way.
620 */
621 if (!device->dev->iommu)
622 break;
623
624 INIT_LIST_HEAD(&dev_resv_regions);
625 iommu_get_resv_regions(device->dev, &dev_resv_regions);
626 ret = iommu_insert_device_resv_regions(&dev_resv_regions, head);
627 iommu_put_resv_regions(device->dev, &dev_resv_regions);
628 if (ret)
629 break;
630 }
631 mutex_unlock(&group->mutex);
632 return ret;
633}
634EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
635
636static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
637 char *buf)
638{
639 struct iommu_resv_region *region, *next;
640 struct list_head group_resv_regions;
641 char *str = buf;
642
643 INIT_LIST_HEAD(&group_resv_regions);
644 iommu_get_group_resv_regions(group, &group_resv_regions);
645
646 list_for_each_entry_safe(region, next, &group_resv_regions, list) {
647 str += sprintf(str, "0x%016llx 0x%016llx %s\n",
648 (long long int)region->start,
649 (long long int)(region->start +
650 region->length - 1),
651 iommu_group_resv_type_string[region->type]);
652 kfree(region);
653 }
654
655 return (str - buf);
656}
657
658static ssize_t iommu_group_show_type(struct iommu_group *group,
659 char *buf)
660{
661 char *type = "unknown\n";
662
663 mutex_lock(&group->mutex);
664 if (group->default_domain) {
665 switch (group->default_domain->type) {
666 case IOMMU_DOMAIN_BLOCKED:
667 type = "blocked\n";
668 break;
669 case IOMMU_DOMAIN_IDENTITY:
670 type = "identity\n";
671 break;
672 case IOMMU_DOMAIN_UNMANAGED:
673 type = "unmanaged\n";
674 break;
675 case IOMMU_DOMAIN_DMA:
676 type = "DMA\n";
677 break;
678 case IOMMU_DOMAIN_DMA_FQ:
679 type = "DMA-FQ\n";
680 break;
681 }
682 }
683 mutex_unlock(&group->mutex);
684 strcpy(buf, type);
685
686 return strlen(type);
687}
688
689static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
690
691static IOMMU_GROUP_ATTR(reserved_regions, 0444,
692 iommu_group_show_resv_regions, NULL);
693
694static IOMMU_GROUP_ATTR(type, 0644, iommu_group_show_type,
695 iommu_group_store_type);
696
697static void iommu_group_release(struct kobject *kobj)
698{
699 struct iommu_group *group = to_iommu_group(kobj);
700
701 pr_debug("Releasing group %d\n", group->id);
702
703 if (group->iommu_data_release)
704 group->iommu_data_release(group->iommu_data);
705
706 ida_free(&iommu_group_ida, group->id);
707
708 if (group->default_domain)
709 iommu_domain_free(group->default_domain);
710 if (group->blocking_domain)
711 iommu_domain_free(group->blocking_domain);
712
713 kfree(group->name);
714 kfree(group);
715}
716
717static struct kobj_type iommu_group_ktype = {
718 .sysfs_ops = &iommu_group_sysfs_ops,
719 .release = iommu_group_release,
720};
721
722/**
723 * iommu_group_alloc - Allocate a new group
724 *
725 * This function is called by an iommu driver to allocate a new iommu
726 * group. The iommu group represents the minimum granularity of the iommu.
727 * Upon successful return, the caller holds a reference to the supplied
728 * group in order to hold the group until devices are added. Use
729 * iommu_group_put() to release this extra reference count, allowing the
730 * group to be automatically reclaimed once it has no devices or external
731 * references.
732 */
733struct iommu_group *iommu_group_alloc(void)
734{
735 struct iommu_group *group;
736 int ret;
737
738 group = kzalloc(sizeof(*group), GFP_KERNEL);
739 if (!group)
740 return ERR_PTR(-ENOMEM);
741
742 group->kobj.kset = iommu_group_kset;
743 mutex_init(&group->mutex);
744 INIT_LIST_HEAD(&group->devices);
745 INIT_LIST_HEAD(&group->entry);
746 xa_init(&group->pasid_array);
747
748 ret = ida_alloc(&iommu_group_ida, GFP_KERNEL);
749 if (ret < 0) {
750 kfree(group);
751 return ERR_PTR(ret);
752 }
753 group->id = ret;
754
755 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
756 NULL, "%d", group->id);
757 if (ret) {
758 kobject_put(&group->kobj);
759 return ERR_PTR(ret);
760 }
761
762 group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
763 if (!group->devices_kobj) {
764 kobject_put(&group->kobj); /* triggers .release & free */
765 return ERR_PTR(-ENOMEM);
766 }
767
768 /*
769 * The devices_kobj holds a reference on the group kobject, so
770 * as long as that exists so will the group. We can therefore
771 * use the devices_kobj for reference counting.
772 */
773 kobject_put(&group->kobj);
774
775 ret = iommu_group_create_file(group,
776 &iommu_group_attr_reserved_regions);
777 if (ret)
778 return ERR_PTR(ret);
779
780 ret = iommu_group_create_file(group, &iommu_group_attr_type);
781 if (ret)
782 return ERR_PTR(ret);
783
784 pr_debug("Allocated group %d\n", group->id);
785
786 return group;
787}
788EXPORT_SYMBOL_GPL(iommu_group_alloc);
789
790struct iommu_group *iommu_group_get_by_id(int id)
791{
792 struct kobject *group_kobj;
793 struct iommu_group *group;
794 const char *name;
795
796 if (!iommu_group_kset)
797 return NULL;
798
799 name = kasprintf(GFP_KERNEL, "%d", id);
800 if (!name)
801 return NULL;
802
803 group_kobj = kset_find_obj(iommu_group_kset, name);
804 kfree(name);
805
806 if (!group_kobj)
807 return NULL;
808
809 group = container_of(group_kobj, struct iommu_group, kobj);
810 BUG_ON(group->id != id);
811
812 kobject_get(group->devices_kobj);
813 kobject_put(&group->kobj);
814
815 return group;
816}
817EXPORT_SYMBOL_GPL(iommu_group_get_by_id);
818
819/**
820 * iommu_group_get_iommudata - retrieve iommu_data registered for a group
821 * @group: the group
822 *
823 * iommu drivers can store data in the group for use when doing iommu
824 * operations. This function provides a way to retrieve it. Caller
825 * should hold a group reference.
826 */
827void *iommu_group_get_iommudata(struct iommu_group *group)
828{
829 return group->iommu_data;
830}
831EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
832
833/**
834 * iommu_group_set_iommudata - set iommu_data for a group
835 * @group: the group
836 * @iommu_data: new data
837 * @release: release function for iommu_data
838 *
839 * iommu drivers can store data in the group for use when doing iommu
840 * operations. This function provides a way to set the data after
841 * the group has been allocated. Caller should hold a group reference.
842 */
843void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
844 void (*release)(void *iommu_data))
845{
846 group->iommu_data = iommu_data;
847 group->iommu_data_release = release;
848}
849EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
850
851/**
852 * iommu_group_set_name - set name for a group
853 * @group: the group
854 * @name: name
855 *
856 * Allow iommu driver to set a name for a group. When set it will
857 * appear in a name attribute file under the group in sysfs.
858 */
859int iommu_group_set_name(struct iommu_group *group, const char *name)
860{
861 int ret;
862
863 if (group->name) {
864 iommu_group_remove_file(group, &iommu_group_attr_name);
865 kfree(group->name);
866 group->name = NULL;
867 if (!name)
868 return 0;
869 }
870
871 group->name = kstrdup(name, GFP_KERNEL);
872 if (!group->name)
873 return -ENOMEM;
874
875 ret = iommu_group_create_file(group, &iommu_group_attr_name);
876 if (ret) {
877 kfree(group->name);
878 group->name = NULL;
879 return ret;
880 }
881
882 return 0;
883}
884EXPORT_SYMBOL_GPL(iommu_group_set_name);
885
886static int iommu_create_device_direct_mappings(struct iommu_group *group,
887 struct device *dev)
888{
889 struct iommu_domain *domain = group->default_domain;
890 struct iommu_resv_region *entry;
891 struct list_head mappings;
892 unsigned long pg_size;
893 int ret = 0;
894
895 if (!domain || !iommu_is_dma_domain(domain))
896 return 0;
897
898 BUG_ON(!domain->pgsize_bitmap);
899
900 pg_size = 1UL << __ffs(domain->pgsize_bitmap);
901 INIT_LIST_HEAD(&mappings);
902
903 iommu_get_resv_regions(dev, &mappings);
904
905 /* We need to consider overlapping regions for different devices */
906 list_for_each_entry(entry, &mappings, list) {
907 dma_addr_t start, end, addr;
908 size_t map_size = 0;
909
910 start = ALIGN(entry->start, pg_size);
911 end = ALIGN(entry->start + entry->length, pg_size);
912
913 if (entry->type != IOMMU_RESV_DIRECT &&
914 entry->type != IOMMU_RESV_DIRECT_RELAXABLE)
915 continue;
916
917 for (addr = start; addr <= end; addr += pg_size) {
918 phys_addr_t phys_addr;
919
920 if (addr == end)
921 goto map_end;
922
923 phys_addr = iommu_iova_to_phys(domain, addr);
924 if (!phys_addr) {
925 map_size += pg_size;
926 continue;
927 }
928
929map_end:
930 if (map_size) {
931 ret = iommu_map(domain, addr - map_size,
932 addr - map_size, map_size,
933 entry->prot);
934 if (ret)
935 goto out;
936 map_size = 0;
937 }
938 }
939
940 }
941
942 iommu_flush_iotlb_all(domain);
943
944out:
945 iommu_put_resv_regions(dev, &mappings);
946
947 return ret;
948}
949
950static bool iommu_is_attach_deferred(struct device *dev)
951{
952 const struct iommu_ops *ops = dev_iommu_ops(dev);
953
954 if (ops->is_attach_deferred)
955 return ops->is_attach_deferred(dev);
956
957 return false;
958}
959
960/**
961 * iommu_group_add_device - add a device to an iommu group
962 * @group: the group into which to add the device (reference should be held)
963 * @dev: the device
964 *
965 * This function is called by an iommu driver to add a device into a
966 * group. Adding a device increments the group reference count.
967 */
968int iommu_group_add_device(struct iommu_group *group, struct device *dev)
969{
970 int ret, i = 0;
971 struct group_device *device;
972
973 device = kzalloc(sizeof(*device), GFP_KERNEL);
974 if (!device)
975 return -ENOMEM;
976
977 device->dev = dev;
978
979 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
980 if (ret)
981 goto err_free_device;
982
983 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
984rename:
985 if (!device->name) {
986 ret = -ENOMEM;
987 goto err_remove_link;
988 }
989
990 ret = sysfs_create_link_nowarn(group->devices_kobj,
991 &dev->kobj, device->name);
992 if (ret) {
993 if (ret == -EEXIST && i >= 0) {
994 /*
995 * Account for the slim chance of collision
996 * and append an instance to the name.
997 */
998 kfree(device->name);
999 device->name = kasprintf(GFP_KERNEL, "%s.%d",
1000 kobject_name(&dev->kobj), i++);
1001 goto rename;
1002 }
1003 goto err_free_name;
1004 }
1005
1006 kobject_get(group->devices_kobj);
1007
1008 dev->iommu_group = group;
1009
1010 mutex_lock(&group->mutex);
1011 list_add_tail(&device->list, &group->devices);
1012 if (group->domain && !iommu_is_attach_deferred(dev))
1013 ret = __iommu_attach_device(group->domain, dev);
1014 mutex_unlock(&group->mutex);
1015 if (ret)
1016 goto err_put_group;
1017
1018 trace_add_device_to_group(group->id, dev);
1019
1020 dev_info(dev, "Adding to iommu group %d\n", group->id);
1021
1022 return 0;
1023
1024err_put_group:
1025 mutex_lock(&group->mutex);
1026 list_del(&device->list);
1027 mutex_unlock(&group->mutex);
1028 dev->iommu_group = NULL;
1029 kobject_put(group->devices_kobj);
1030 sysfs_remove_link(group->devices_kobj, device->name);
1031err_free_name:
1032 kfree(device->name);
1033err_remove_link:
1034 sysfs_remove_link(&dev->kobj, "iommu_group");
1035err_free_device:
1036 kfree(device);
1037 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);
1038 return ret;
1039}
1040EXPORT_SYMBOL_GPL(iommu_group_add_device);
1041
1042/**
1043 * iommu_group_remove_device - remove a device from it's current group
1044 * @dev: device to be removed
1045 *
1046 * This function is called by an iommu driver to remove the device from
1047 * it's current group. This decrements the iommu group reference count.
1048 */
1049void iommu_group_remove_device(struct device *dev)
1050{
1051 struct iommu_group *group = dev->iommu_group;
1052 struct group_device *tmp_device, *device = NULL;
1053
1054 if (!group)
1055 return;
1056
1057 dev_info(dev, "Removing from iommu group %d\n", group->id);
1058
1059 mutex_lock(&group->mutex);
1060 list_for_each_entry(tmp_device, &group->devices, list) {
1061 if (tmp_device->dev == dev) {
1062 device = tmp_device;
1063 list_del(&device->list);
1064 break;
1065 }
1066 }
1067 mutex_unlock(&group->mutex);
1068
1069 if (!device)
1070 return;
1071
1072 sysfs_remove_link(group->devices_kobj, device->name);
1073 sysfs_remove_link(&dev->kobj, "iommu_group");
1074
1075 trace_remove_device_from_group(group->id, dev);
1076
1077 kfree(device->name);
1078 kfree(device);
1079 dev->iommu_group = NULL;
1080 kobject_put(group->devices_kobj);
1081}
1082EXPORT_SYMBOL_GPL(iommu_group_remove_device);
1083
1084static int iommu_group_device_count(struct iommu_group *group)
1085{
1086 struct group_device *entry;
1087 int ret = 0;
1088
1089 list_for_each_entry(entry, &group->devices, list)
1090 ret++;
1091
1092 return ret;
1093}
1094
1095static int __iommu_group_for_each_dev(struct iommu_group *group, void *data,
1096 int (*fn)(struct device *, void *))
1097{
1098 struct group_device *device;
1099 int ret = 0;
1100
1101 list_for_each_entry(device, &group->devices, list) {
1102 ret = fn(device->dev, data);
1103 if (ret)
1104 break;
1105 }
1106 return ret;
1107}
1108
1109/**
1110 * iommu_group_for_each_dev - iterate over each device in the group
1111 * @group: the group
1112 * @data: caller opaque data to be passed to callback function
1113 * @fn: caller supplied callback function
1114 *
1115 * This function is called by group users to iterate over group devices.
1116 * Callers should hold a reference count to the group during callback.
1117 * The group->mutex is held across callbacks, which will block calls to
1118 * iommu_group_add/remove_device.
1119 */
1120int iommu_group_for_each_dev(struct iommu_group *group, void *data,
1121 int (*fn)(struct device *, void *))
1122{
1123 int ret;
1124
1125 mutex_lock(&group->mutex);
1126 ret = __iommu_group_for_each_dev(group, data, fn);
1127 mutex_unlock(&group->mutex);
1128
1129 return ret;
1130}
1131EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
1132
1133/**
1134 * iommu_group_get - Return the group for a device and increment reference
1135 * @dev: get the group that this device belongs to
1136 *
1137 * This function is called by iommu drivers and users to get the group
1138 * for the specified device. If found, the group is returned and the group
1139 * reference in incremented, else NULL.
1140 */
1141struct iommu_group *iommu_group_get(struct device *dev)
1142{
1143 struct iommu_group *group = dev->iommu_group;
1144
1145 if (group)
1146 kobject_get(group->devices_kobj);
1147
1148 return group;
1149}
1150EXPORT_SYMBOL_GPL(iommu_group_get);
1151
1152/**
1153 * iommu_group_ref_get - Increment reference on a group
1154 * @group: the group to use, must not be NULL
1155 *
1156 * This function is called by iommu drivers to take additional references on an
1157 * existing group. Returns the given group for convenience.
1158 */
1159struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
1160{
1161 kobject_get(group->devices_kobj);
1162 return group;
1163}
1164EXPORT_SYMBOL_GPL(iommu_group_ref_get);
1165
1166/**
1167 * iommu_group_put - Decrement group reference
1168 * @group: the group to use
1169 *
1170 * This function is called by iommu drivers and users to release the
1171 * iommu group. Once the reference count is zero, the group is released.
1172 */
1173void iommu_group_put(struct iommu_group *group)
1174{
1175 if (group)
1176 kobject_put(group->devices_kobj);
1177}
1178EXPORT_SYMBOL_GPL(iommu_group_put);
1179
1180/**
1181 * iommu_register_device_fault_handler() - Register a device fault handler
1182 * @dev: the device
1183 * @handler: the fault handler
1184 * @data: private data passed as argument to the handler
1185 *
1186 * When an IOMMU fault event is received, this handler gets called with the
1187 * fault event and data as argument. The handler should return 0 on success. If
1188 * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also
1189 * complete the fault by calling iommu_page_response() with one of the following
1190 * response code:
1191 * - IOMMU_PAGE_RESP_SUCCESS: retry the translation
1192 * - IOMMU_PAGE_RESP_INVALID: terminate the fault
1193 * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting
1194 * page faults if possible.
1195 *
1196 * Return 0 if the fault handler was installed successfully, or an error.
1197 */
1198int iommu_register_device_fault_handler(struct device *dev,
1199 iommu_dev_fault_handler_t handler,
1200 void *data)
1201{
1202 struct dev_iommu *param = dev->iommu;
1203 int ret = 0;
1204
1205 if (!param)
1206 return -EINVAL;
1207
1208 mutex_lock(¶m->lock);
1209 /* Only allow one fault handler registered for each device */
1210 if (param->fault_param) {
1211 ret = -EBUSY;
1212 goto done_unlock;
1213 }
1214
1215 get_device(dev);
1216 param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL);
1217 if (!param->fault_param) {
1218 put_device(dev);
1219 ret = -ENOMEM;
1220 goto done_unlock;
1221 }
1222 param->fault_param->handler = handler;
1223 param->fault_param->data = data;
1224 mutex_init(¶m->fault_param->lock);
1225 INIT_LIST_HEAD(¶m->fault_param->faults);
1226
1227done_unlock:
1228 mutex_unlock(¶m->lock);
1229
1230 return ret;
1231}
1232EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler);
1233
1234/**
1235 * iommu_unregister_device_fault_handler() - Unregister the device fault handler
1236 * @dev: the device
1237 *
1238 * Remove the device fault handler installed with
1239 * iommu_register_device_fault_handler().
1240 *
1241 * Return 0 on success, or an error.
1242 */
1243int iommu_unregister_device_fault_handler(struct device *dev)
1244{
1245 struct dev_iommu *param = dev->iommu;
1246 int ret = 0;
1247
1248 if (!param)
1249 return -EINVAL;
1250
1251 mutex_lock(¶m->lock);
1252
1253 if (!param->fault_param)
1254 goto unlock;
1255
1256 /* we cannot unregister handler if there are pending faults */
1257 if (!list_empty(¶m->fault_param->faults)) {
1258 ret = -EBUSY;
1259 goto unlock;
1260 }
1261
1262 kfree(param->fault_param);
1263 param->fault_param = NULL;
1264 put_device(dev);
1265unlock:
1266 mutex_unlock(¶m->lock);
1267
1268 return ret;
1269}
1270EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler);
1271
1272/**
1273 * iommu_report_device_fault() - Report fault event to device driver
1274 * @dev: the device
1275 * @evt: fault event data
1276 *
1277 * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ
1278 * handler. When this function fails and the fault is recoverable, it is the
1279 * caller's responsibility to complete the fault.
1280 *
1281 * Return 0 on success, or an error.
1282 */
1283int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt)
1284{
1285 struct dev_iommu *param = dev->iommu;
1286 struct iommu_fault_event *evt_pending = NULL;
1287 struct iommu_fault_param *fparam;
1288 int ret = 0;
1289
1290 if (!param || !evt)
1291 return -EINVAL;
1292
1293 /* we only report device fault if there is a handler registered */
1294 mutex_lock(¶m->lock);
1295 fparam = param->fault_param;
1296 if (!fparam || !fparam->handler) {
1297 ret = -EINVAL;
1298 goto done_unlock;
1299 }
1300
1301 if (evt->fault.type == IOMMU_FAULT_PAGE_REQ &&
1302 (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) {
1303 evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event),
1304 GFP_KERNEL);
1305 if (!evt_pending) {
1306 ret = -ENOMEM;
1307 goto done_unlock;
1308 }
1309 mutex_lock(&fparam->lock);
1310 list_add_tail(&evt_pending->list, &fparam->faults);
1311 mutex_unlock(&fparam->lock);
1312 }
1313
1314 ret = fparam->handler(&evt->fault, fparam->data);
1315 if (ret && evt_pending) {
1316 mutex_lock(&fparam->lock);
1317 list_del(&evt_pending->list);
1318 mutex_unlock(&fparam->lock);
1319 kfree(evt_pending);
1320 }
1321done_unlock:
1322 mutex_unlock(¶m->lock);
1323 return ret;
1324}
1325EXPORT_SYMBOL_GPL(iommu_report_device_fault);
1326
1327int iommu_page_response(struct device *dev,
1328 struct iommu_page_response *msg)
1329{
1330 bool needs_pasid;
1331 int ret = -EINVAL;
1332 struct iommu_fault_event *evt;
1333 struct iommu_fault_page_request *prm;
1334 struct dev_iommu *param = dev->iommu;
1335 const struct iommu_ops *ops = dev_iommu_ops(dev);
1336 bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID;
1337
1338 if (!ops->page_response)
1339 return -ENODEV;
1340
1341 if (!param || !param->fault_param)
1342 return -EINVAL;
1343
1344 if (msg->version != IOMMU_PAGE_RESP_VERSION_1 ||
1345 msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID)
1346 return -EINVAL;
1347
1348 /* Only send response if there is a fault report pending */
1349 mutex_lock(¶m->fault_param->lock);
1350 if (list_empty(¶m->fault_param->faults)) {
1351 dev_warn_ratelimited(dev, "no pending PRQ, drop response\n");
1352 goto done_unlock;
1353 }
1354 /*
1355 * Check if we have a matching page request pending to respond,
1356 * otherwise return -EINVAL
1357 */
1358 list_for_each_entry(evt, ¶m->fault_param->faults, list) {
1359 prm = &evt->fault.prm;
1360 if (prm->grpid != msg->grpid)
1361 continue;
1362
1363 /*
1364 * If the PASID is required, the corresponding request is
1365 * matched using the group ID, the PASID valid bit and the PASID
1366 * value. Otherwise only the group ID matches request and
1367 * response.
1368 */
1369 needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID;
1370 if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid))
1371 continue;
1372
1373 if (!needs_pasid && has_pasid) {
1374 /* No big deal, just clear it. */
1375 msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID;
1376 msg->pasid = 0;
1377 }
1378
1379 ret = ops->page_response(dev, evt, msg);
1380 list_del(&evt->list);
1381 kfree(evt);
1382 break;
1383 }
1384
1385done_unlock:
1386 mutex_unlock(¶m->fault_param->lock);
1387 return ret;
1388}
1389EXPORT_SYMBOL_GPL(iommu_page_response);
1390
1391/**
1392 * iommu_group_id - Return ID for a group
1393 * @group: the group to ID
1394 *
1395 * Return the unique ID for the group matching the sysfs group number.
1396 */
1397int iommu_group_id(struct iommu_group *group)
1398{
1399 return group->id;
1400}
1401EXPORT_SYMBOL_GPL(iommu_group_id);
1402
1403static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1404 unsigned long *devfns);
1405
1406/*
1407 * To consider a PCI device isolated, we require ACS to support Source
1408 * Validation, Request Redirection, Completer Redirection, and Upstream
1409 * Forwarding. This effectively means that devices cannot spoof their
1410 * requester ID, requests and completions cannot be redirected, and all
1411 * transactions are forwarded upstream, even as it passes through a
1412 * bridge where the target device is downstream.
1413 */
1414#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
1415
1416/*
1417 * For multifunction devices which are not isolated from each other, find
1418 * all the other non-isolated functions and look for existing groups. For
1419 * each function, we also need to look for aliases to or from other devices
1420 * that may already have a group.
1421 */
1422static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
1423 unsigned long *devfns)
1424{
1425 struct pci_dev *tmp = NULL;
1426 struct iommu_group *group;
1427
1428 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
1429 return NULL;
1430
1431 for_each_pci_dev(tmp) {
1432 if (tmp == pdev || tmp->bus != pdev->bus ||
1433 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
1434 pci_acs_enabled(tmp, REQ_ACS_FLAGS))
1435 continue;
1436
1437 group = get_pci_alias_group(tmp, devfns);
1438 if (group) {
1439 pci_dev_put(tmp);
1440 return group;
1441 }
1442 }
1443
1444 return NULL;
1445}
1446
1447/*
1448 * Look for aliases to or from the given device for existing groups. DMA
1449 * aliases are only supported on the same bus, therefore the search
1450 * space is quite small (especially since we're really only looking at pcie
1451 * device, and therefore only expect multiple slots on the root complex or
1452 * downstream switch ports). It's conceivable though that a pair of
1453 * multifunction devices could have aliases between them that would cause a
1454 * loop. To prevent this, we use a bitmap to track where we've been.
1455 */
1456static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1457 unsigned long *devfns)
1458{
1459 struct pci_dev *tmp = NULL;
1460 struct iommu_group *group;
1461
1462 if (test_and_set_bit(pdev->devfn & 0xff, devfns))
1463 return NULL;
1464
1465 group = iommu_group_get(&pdev->dev);
1466 if (group)
1467 return group;
1468
1469 for_each_pci_dev(tmp) {
1470 if (tmp == pdev || tmp->bus != pdev->bus)
1471 continue;
1472
1473 /* We alias them or they alias us */
1474 if (pci_devs_are_dma_aliases(pdev, tmp)) {
1475 group = get_pci_alias_group(tmp, devfns);
1476 if (group) {
1477 pci_dev_put(tmp);
1478 return group;
1479 }
1480
1481 group = get_pci_function_alias_group(tmp, devfns);
1482 if (group) {
1483 pci_dev_put(tmp);
1484 return group;
1485 }
1486 }
1487 }
1488
1489 return NULL;
1490}
1491
1492struct group_for_pci_data {
1493 struct pci_dev *pdev;
1494 struct iommu_group *group;
1495};
1496
1497/*
1498 * DMA alias iterator callback, return the last seen device. Stop and return
1499 * the IOMMU group if we find one along the way.
1500 */
1501static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
1502{
1503 struct group_for_pci_data *data = opaque;
1504
1505 data->pdev = pdev;
1506 data->group = iommu_group_get(&pdev->dev);
1507
1508 return data->group != NULL;
1509}
1510
1511/*
1512 * Generic device_group call-back function. It just allocates one
1513 * iommu-group per device.
1514 */
1515struct iommu_group *generic_device_group(struct device *dev)
1516{
1517 return iommu_group_alloc();
1518}
1519EXPORT_SYMBOL_GPL(generic_device_group);
1520
1521/*
1522 * Use standard PCI bus topology, isolation features, and DMA alias quirks
1523 * to find or create an IOMMU group for a device.
1524 */
1525struct iommu_group *pci_device_group(struct device *dev)
1526{
1527 struct pci_dev *pdev = to_pci_dev(dev);
1528 struct group_for_pci_data data;
1529 struct pci_bus *bus;
1530 struct iommu_group *group = NULL;
1531 u64 devfns[4] = { 0 };
1532
1533 if (WARN_ON(!dev_is_pci(dev)))
1534 return ERR_PTR(-EINVAL);
1535
1536 /*
1537 * Find the upstream DMA alias for the device. A device must not
1538 * be aliased due to topology in order to have its own IOMMU group.
1539 * If we find an alias along the way that already belongs to a
1540 * group, use it.
1541 */
1542 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))
1543 return data.group;
1544
1545 pdev = data.pdev;
1546
1547 /*
1548 * Continue upstream from the point of minimum IOMMU granularity
1549 * due to aliases to the point where devices are protected from
1550 * peer-to-peer DMA by PCI ACS. Again, if we find an existing
1551 * group, use it.
1552 */
1553 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {
1554 if (!bus->self)
1555 continue;
1556
1557 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
1558 break;
1559
1560 pdev = bus->self;
1561
1562 group = iommu_group_get(&pdev->dev);
1563 if (group)
1564 return group;
1565 }
1566
1567 /*
1568 * Look for existing groups on device aliases. If we alias another
1569 * device or another device aliases us, use the same group.
1570 */
1571 group = get_pci_alias_group(pdev, (unsigned long *)devfns);
1572 if (group)
1573 return group;
1574
1575 /*
1576 * Look for existing groups on non-isolated functions on the same
1577 * slot and aliases of those funcions, if any. No need to clear
1578 * the search bitmap, the tested devfns are still valid.
1579 */
1580 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
1581 if (group)
1582 return group;
1583
1584 /* No shared group found, allocate new */
1585 return iommu_group_alloc();
1586}
1587EXPORT_SYMBOL_GPL(pci_device_group);
1588
1589/* Get the IOMMU group for device on fsl-mc bus */
1590struct iommu_group *fsl_mc_device_group(struct device *dev)
1591{
1592 struct device *cont_dev = fsl_mc_cont_dev(dev);
1593 struct iommu_group *group;
1594
1595 group = iommu_group_get(cont_dev);
1596 if (!group)
1597 group = iommu_group_alloc();
1598 return group;
1599}
1600EXPORT_SYMBOL_GPL(fsl_mc_device_group);
1601
1602static int iommu_get_def_domain_type(struct device *dev)
1603{
1604 const struct iommu_ops *ops = dev_iommu_ops(dev);
1605
1606 if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
1607 return IOMMU_DOMAIN_DMA;
1608
1609 if (ops->def_domain_type)
1610 return ops->def_domain_type(dev);
1611
1612 return 0;
1613}
1614
1615static int iommu_group_alloc_default_domain(struct bus_type *bus,
1616 struct iommu_group *group,
1617 unsigned int type)
1618{
1619 struct iommu_domain *dom;
1620
1621 dom = __iommu_domain_alloc(bus, type);
1622 if (!dom && type != IOMMU_DOMAIN_DMA) {
1623 dom = __iommu_domain_alloc(bus, IOMMU_DOMAIN_DMA);
1624 if (dom)
1625 pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA",
1626 type, group->name);
1627 }
1628
1629 if (!dom)
1630 return -ENOMEM;
1631
1632 group->default_domain = dom;
1633 if (!group->domain)
1634 group->domain = dom;
1635 return 0;
1636}
1637
1638static int iommu_alloc_default_domain(struct iommu_group *group,
1639 struct device *dev)
1640{
1641 unsigned int type;
1642
1643 if (group->default_domain)
1644 return 0;
1645
1646 type = iommu_get_def_domain_type(dev) ? : iommu_def_domain_type;
1647
1648 return iommu_group_alloc_default_domain(dev->bus, group, type);
1649}
1650
1651/**
1652 * iommu_group_get_for_dev - Find or create the IOMMU group for a device
1653 * @dev: target device
1654 *
1655 * This function is intended to be called by IOMMU drivers and extended to
1656 * support common, bus-defined algorithms when determining or creating the
1657 * IOMMU group for a device. On success, the caller will hold a reference
1658 * to the returned IOMMU group, which will already include the provided
1659 * device. The reference should be released with iommu_group_put().
1660 */
1661static struct iommu_group *iommu_group_get_for_dev(struct device *dev)
1662{
1663 const struct iommu_ops *ops = dev_iommu_ops(dev);
1664 struct iommu_group *group;
1665 int ret;
1666
1667 group = iommu_group_get(dev);
1668 if (group)
1669 return group;
1670
1671 group = ops->device_group(dev);
1672 if (WARN_ON_ONCE(group == NULL))
1673 return ERR_PTR(-EINVAL);
1674
1675 if (IS_ERR(group))
1676 return group;
1677
1678 ret = iommu_group_add_device(group, dev);
1679 if (ret)
1680 goto out_put_group;
1681
1682 return group;
1683
1684out_put_group:
1685 iommu_group_put(group);
1686
1687 return ERR_PTR(ret);
1688}
1689
1690struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
1691{
1692 return group->default_domain;
1693}
1694
1695static int probe_iommu_group(struct device *dev, void *data)
1696{
1697 struct list_head *group_list = data;
1698 struct iommu_group *group;
1699 int ret;
1700
1701 /* Device is probed already if in a group */
1702 group = iommu_group_get(dev);
1703 if (group) {
1704 iommu_group_put(group);
1705 return 0;
1706 }
1707
1708 ret = __iommu_probe_device(dev, group_list);
1709 if (ret == -ENODEV)
1710 ret = 0;
1711
1712 return ret;
1713}
1714
1715static int iommu_bus_notifier(struct notifier_block *nb,
1716 unsigned long action, void *data)
1717{
1718 struct device *dev = data;
1719
1720 if (action == BUS_NOTIFY_ADD_DEVICE) {
1721 int ret;
1722
1723 ret = iommu_probe_device(dev);
1724 return (ret) ? NOTIFY_DONE : NOTIFY_OK;
1725 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
1726 iommu_release_device(dev);
1727 return NOTIFY_OK;
1728 }
1729
1730 return 0;
1731}
1732
1733struct __group_domain_type {
1734 struct device *dev;
1735 unsigned int type;
1736};
1737
1738static int probe_get_default_domain_type(struct device *dev, void *data)
1739{
1740 struct __group_domain_type *gtype = data;
1741 unsigned int type = iommu_get_def_domain_type(dev);
1742
1743 if (type) {
1744 if (gtype->type && gtype->type != type) {
1745 dev_warn(dev, "Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n",
1746 iommu_domain_type_str(type),
1747 dev_name(gtype->dev),
1748 iommu_domain_type_str(gtype->type));
1749 gtype->type = 0;
1750 }
1751
1752 if (!gtype->dev) {
1753 gtype->dev = dev;
1754 gtype->type = type;
1755 }
1756 }
1757
1758 return 0;
1759}
1760
1761static void probe_alloc_default_domain(struct bus_type *bus,
1762 struct iommu_group *group)
1763{
1764 struct __group_domain_type gtype;
1765
1766 memset(>ype, 0, sizeof(gtype));
1767
1768 /* Ask for default domain requirements of all devices in the group */
1769 __iommu_group_for_each_dev(group, >ype,
1770 probe_get_default_domain_type);
1771
1772 if (!gtype.type)
1773 gtype.type = iommu_def_domain_type;
1774
1775 iommu_group_alloc_default_domain(bus, group, gtype.type);
1776
1777}
1778
1779static int iommu_group_do_dma_attach(struct device *dev, void *data)
1780{
1781 struct iommu_domain *domain = data;
1782 int ret = 0;
1783
1784 if (!iommu_is_attach_deferred(dev))
1785 ret = __iommu_attach_device(domain, dev);
1786
1787 return ret;
1788}
1789
1790static int __iommu_group_dma_attach(struct iommu_group *group)
1791{
1792 return __iommu_group_for_each_dev(group, group->default_domain,
1793 iommu_group_do_dma_attach);
1794}
1795
1796static int iommu_group_do_probe_finalize(struct device *dev, void *data)
1797{
1798 const struct iommu_ops *ops = dev_iommu_ops(dev);
1799
1800 if (ops->probe_finalize)
1801 ops->probe_finalize(dev);
1802
1803 return 0;
1804}
1805
1806static void __iommu_group_dma_finalize(struct iommu_group *group)
1807{
1808 __iommu_group_for_each_dev(group, group->default_domain,
1809 iommu_group_do_probe_finalize);
1810}
1811
1812static int iommu_do_create_direct_mappings(struct device *dev, void *data)
1813{
1814 struct iommu_group *group = data;
1815
1816 iommu_create_device_direct_mappings(group, dev);
1817
1818 return 0;
1819}
1820
1821static int iommu_group_create_direct_mappings(struct iommu_group *group)
1822{
1823 return __iommu_group_for_each_dev(group, group,
1824 iommu_do_create_direct_mappings);
1825}
1826
1827int bus_iommu_probe(struct bus_type *bus)
1828{
1829 struct iommu_group *group, *next;
1830 LIST_HEAD(group_list);
1831 int ret;
1832
1833 /*
1834 * This code-path does not allocate the default domain when
1835 * creating the iommu group, so do it after the groups are
1836 * created.
1837 */
1838 ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group);
1839 if (ret)
1840 return ret;
1841
1842 list_for_each_entry_safe(group, next, &group_list, entry) {
1843 mutex_lock(&group->mutex);
1844
1845 /* Remove item from the list */
1846 list_del_init(&group->entry);
1847
1848 /* Try to allocate default domain */
1849 probe_alloc_default_domain(bus, group);
1850
1851 if (!group->default_domain) {
1852 mutex_unlock(&group->mutex);
1853 continue;
1854 }
1855
1856 iommu_group_create_direct_mappings(group);
1857
1858 ret = __iommu_group_dma_attach(group);
1859
1860 mutex_unlock(&group->mutex);
1861
1862 if (ret)
1863 break;
1864
1865 __iommu_group_dma_finalize(group);
1866 }
1867
1868 return ret;
1869}
1870
1871bool iommu_present(struct bus_type *bus)
1872{
1873 return bus->iommu_ops != NULL;
1874}
1875EXPORT_SYMBOL_GPL(iommu_present);
1876
1877/**
1878 * device_iommu_capable() - check for a general IOMMU capability
1879 * @dev: device to which the capability would be relevant, if available
1880 * @cap: IOMMU capability
1881 *
1882 * Return: true if an IOMMU is present and supports the given capability
1883 * for the given device, otherwise false.
1884 */
1885bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
1886{
1887 const struct iommu_ops *ops;
1888
1889 if (!dev->iommu || !dev->iommu->iommu_dev)
1890 return false;
1891
1892 ops = dev_iommu_ops(dev);
1893 if (!ops->capable)
1894 return false;
1895
1896 return ops->capable(dev, cap);
1897}
1898EXPORT_SYMBOL_GPL(device_iommu_capable);
1899
1900/**
1901 * iommu_set_fault_handler() - set a fault handler for an iommu domain
1902 * @domain: iommu domain
1903 * @handler: fault handler
1904 * @token: user data, will be passed back to the fault handler
1905 *
1906 * This function should be used by IOMMU users which want to be notified
1907 * whenever an IOMMU fault happens.
1908 *
1909 * The fault handler itself should return 0 on success, and an appropriate
1910 * error code otherwise.
1911 */
1912void iommu_set_fault_handler(struct iommu_domain *domain,
1913 iommu_fault_handler_t handler,
1914 void *token)
1915{
1916 BUG_ON(!domain);
1917
1918 domain->handler = handler;
1919 domain->handler_token = token;
1920}
1921EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
1922
1923static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
1924 unsigned type)
1925{
1926 struct iommu_domain *domain;
1927
1928 if (bus == NULL || bus->iommu_ops == NULL)
1929 return NULL;
1930
1931 domain = bus->iommu_ops->domain_alloc(type);
1932 if (!domain)
1933 return NULL;
1934
1935 domain->type = type;
1936 /* Assume all sizes by default; the driver may override this later */
1937 domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
1938 if (!domain->ops)
1939 domain->ops = bus->iommu_ops->default_domain_ops;
1940
1941 if (iommu_is_dma_domain(domain) && iommu_get_dma_cookie(domain)) {
1942 iommu_domain_free(domain);
1943 domain = NULL;
1944 }
1945 return domain;
1946}
1947
1948struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
1949{
1950 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED);
1951}
1952EXPORT_SYMBOL_GPL(iommu_domain_alloc);
1953
1954void iommu_domain_free(struct iommu_domain *domain)
1955{
1956 if (domain->type == IOMMU_DOMAIN_SVA)
1957 mmdrop(domain->mm);
1958 iommu_put_dma_cookie(domain);
1959 domain->ops->free(domain);
1960}
1961EXPORT_SYMBOL_GPL(iommu_domain_free);
1962
1963/*
1964 * Put the group's domain back to the appropriate core-owned domain - either the
1965 * standard kernel-mode DMA configuration or an all-DMA-blocked domain.
1966 */
1967static void __iommu_group_set_core_domain(struct iommu_group *group)
1968{
1969 struct iommu_domain *new_domain;
1970 int ret;
1971
1972 if (group->owner)
1973 new_domain = group->blocking_domain;
1974 else
1975 new_domain = group->default_domain;
1976
1977 ret = __iommu_group_set_domain(group, new_domain);
1978 WARN(ret, "iommu driver failed to attach the default/blocking domain");
1979}
1980
1981static int __iommu_attach_device(struct iommu_domain *domain,
1982 struct device *dev)
1983{
1984 int ret;
1985
1986 if (unlikely(domain->ops->attach_dev == NULL))
1987 return -ENODEV;
1988
1989 ret = domain->ops->attach_dev(domain, dev);
1990 if (!ret)
1991 trace_attach_device_to_domain(dev);
1992 return ret;
1993}
1994
1995/**
1996 * iommu_attach_device - Attach an IOMMU domain to a device
1997 * @domain: IOMMU domain to attach
1998 * @dev: Device that will be attached
1999 *
2000 * Returns 0 on success and error code on failure
2001 *
2002 * Note that EINVAL can be treated as a soft failure, indicating
2003 * that certain configuration of the domain is incompatible with
2004 * the device. In this case attaching a different domain to the
2005 * device may succeed.
2006 */
2007int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
2008{
2009 struct iommu_group *group;
2010 int ret;
2011
2012 group = iommu_group_get(dev);
2013 if (!group)
2014 return -ENODEV;
2015
2016 /*
2017 * Lock the group to make sure the device-count doesn't
2018 * change while we are attaching
2019 */
2020 mutex_lock(&group->mutex);
2021 ret = -EINVAL;
2022 if (iommu_group_device_count(group) != 1)
2023 goto out_unlock;
2024
2025 ret = __iommu_attach_group(domain, group);
2026
2027out_unlock:
2028 mutex_unlock(&group->mutex);
2029 iommu_group_put(group);
2030
2031 return ret;
2032}
2033EXPORT_SYMBOL_GPL(iommu_attach_device);
2034
2035int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
2036{
2037 if (iommu_is_attach_deferred(dev))
2038 return __iommu_attach_device(domain, dev);
2039
2040 return 0;
2041}
2042
2043static void __iommu_detach_device(struct iommu_domain *domain,
2044 struct device *dev)
2045{
2046 if (iommu_is_attach_deferred(dev))
2047 return;
2048
2049 domain->ops->detach_dev(domain, dev);
2050 trace_detach_device_from_domain(dev);
2051}
2052
2053void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
2054{
2055 struct iommu_group *group;
2056
2057 group = iommu_group_get(dev);
2058 if (!group)
2059 return;
2060
2061 mutex_lock(&group->mutex);
2062 if (WARN_ON(domain != group->domain) ||
2063 WARN_ON(iommu_group_device_count(group) != 1))
2064 goto out_unlock;
2065 __iommu_group_set_core_domain(group);
2066
2067out_unlock:
2068 mutex_unlock(&group->mutex);
2069 iommu_group_put(group);
2070}
2071EXPORT_SYMBOL_GPL(iommu_detach_device);
2072
2073struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
2074{
2075 struct iommu_domain *domain;
2076 struct iommu_group *group;
2077
2078 group = iommu_group_get(dev);
2079 if (!group)
2080 return NULL;
2081
2082 domain = group->domain;
2083
2084 iommu_group_put(group);
2085
2086 return domain;
2087}
2088EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
2089
2090/*
2091 * For IOMMU_DOMAIN_DMA implementations which already provide their own
2092 * guarantees that the group and its default domain are valid and correct.
2093 */
2094struct iommu_domain *iommu_get_dma_domain(struct device *dev)
2095{
2096 return dev->iommu_group->default_domain;
2097}
2098
2099/*
2100 * IOMMU groups are really the natural working unit of the IOMMU, but
2101 * the IOMMU API works on domains and devices. Bridge that gap by
2102 * iterating over the devices in a group. Ideally we'd have a single
2103 * device which represents the requestor ID of the group, but we also
2104 * allow IOMMU drivers to create policy defined minimum sets, where
2105 * the physical hardware may be able to distiguish members, but we
2106 * wish to group them at a higher level (ex. untrusted multi-function
2107 * PCI devices). Thus we attach each device.
2108 */
2109static int iommu_group_do_attach_device(struct device *dev, void *data)
2110{
2111 struct iommu_domain *domain = data;
2112
2113 return __iommu_attach_device(domain, dev);
2114}
2115
2116static int __iommu_attach_group(struct iommu_domain *domain,
2117 struct iommu_group *group)
2118{
2119 int ret;
2120
2121 if (group->domain && group->domain != group->default_domain &&
2122 group->domain != group->blocking_domain)
2123 return -EBUSY;
2124
2125 ret = __iommu_group_for_each_dev(group, domain,
2126 iommu_group_do_attach_device);
2127 if (ret == 0)
2128 group->domain = domain;
2129
2130 return ret;
2131}
2132
2133/**
2134 * iommu_attach_group - Attach an IOMMU domain to an IOMMU group
2135 * @domain: IOMMU domain to attach
2136 * @group: IOMMU group that will be attached
2137 *
2138 * Returns 0 on success and error code on failure
2139 *
2140 * Note that EINVAL can be treated as a soft failure, indicating
2141 * that certain configuration of the domain is incompatible with
2142 * the group. In this case attaching a different domain to the
2143 * group may succeed.
2144 */
2145int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
2146{
2147 int ret;
2148
2149 mutex_lock(&group->mutex);
2150 ret = __iommu_attach_group(domain, group);
2151 mutex_unlock(&group->mutex);
2152
2153 return ret;
2154}
2155EXPORT_SYMBOL_GPL(iommu_attach_group);
2156
2157static int iommu_group_do_detach_device(struct device *dev, void *data)
2158{
2159 struct iommu_domain *domain = data;
2160
2161 __iommu_detach_device(domain, dev);
2162
2163 return 0;
2164}
2165
2166static int __iommu_group_set_domain(struct iommu_group *group,
2167 struct iommu_domain *new_domain)
2168{
2169 int ret;
2170
2171 if (group->domain == new_domain)
2172 return 0;
2173
2174 /*
2175 * New drivers should support default domains and so the detach_dev() op
2176 * will never be called. Otherwise the NULL domain represents some
2177 * platform specific behavior.
2178 */
2179 if (!new_domain) {
2180 if (WARN_ON(!group->domain->ops->detach_dev))
2181 return -EINVAL;
2182 __iommu_group_for_each_dev(group, group->domain,
2183 iommu_group_do_detach_device);
2184 group->domain = NULL;
2185 return 0;
2186 }
2187
2188 /*
2189 * Changing the domain is done by calling attach_dev() on the new
2190 * domain. This switch does not have to be atomic and DMA can be
2191 * discarded during the transition. DMA must only be able to access
2192 * either new_domain or group->domain, never something else.
2193 *
2194 * Note that this is called in error unwind paths, attaching to a
2195 * domain that has already been attached cannot fail.
2196 */
2197 ret = __iommu_group_for_each_dev(group, new_domain,
2198 iommu_group_do_attach_device);
2199 if (ret)
2200 return ret;
2201 group->domain = new_domain;
2202 return 0;
2203}
2204
2205void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
2206{
2207 mutex_lock(&group->mutex);
2208 __iommu_group_set_core_domain(group);
2209 mutex_unlock(&group->mutex);
2210}
2211EXPORT_SYMBOL_GPL(iommu_detach_group);
2212
2213phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2214{
2215 if (domain->type == IOMMU_DOMAIN_IDENTITY)
2216 return iova;
2217
2218 if (domain->type == IOMMU_DOMAIN_BLOCKED)
2219 return 0;
2220
2221 return domain->ops->iova_to_phys(domain, iova);
2222}
2223EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
2224
2225static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova,
2226 phys_addr_t paddr, size_t size, size_t *count)
2227{
2228 unsigned int pgsize_idx, pgsize_idx_next;
2229 unsigned long pgsizes;
2230 size_t offset, pgsize, pgsize_next;
2231 unsigned long addr_merge = paddr | iova;
2232
2233 /* Page sizes supported by the hardware and small enough for @size */
2234 pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0);
2235
2236 /* Constrain the page sizes further based on the maximum alignment */
2237 if (likely(addr_merge))
2238 pgsizes &= GENMASK(__ffs(addr_merge), 0);
2239
2240 /* Make sure we have at least one suitable page size */
2241 BUG_ON(!pgsizes);
2242
2243 /* Pick the biggest page size remaining */
2244 pgsize_idx = __fls(pgsizes);
2245 pgsize = BIT(pgsize_idx);
2246 if (!count)
2247 return pgsize;
2248
2249 /* Find the next biggest support page size, if it exists */
2250 pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
2251 if (!pgsizes)
2252 goto out_set_count;
2253
2254 pgsize_idx_next = __ffs(pgsizes);
2255 pgsize_next = BIT(pgsize_idx_next);
2256
2257 /*
2258 * There's no point trying a bigger page size unless the virtual
2259 * and physical addresses are similarly offset within the larger page.
2260 */
2261 if ((iova ^ paddr) & (pgsize_next - 1))
2262 goto out_set_count;
2263
2264 /* Calculate the offset to the next page size alignment boundary */
2265 offset = pgsize_next - (addr_merge & (pgsize_next - 1));
2266
2267 /*
2268 * If size is big enough to accommodate the larger page, reduce
2269 * the number of smaller pages.
2270 */
2271 if (offset + pgsize_next <= size)
2272 size = offset;
2273
2274out_set_count:
2275 *count = size >> pgsize_idx;
2276 return pgsize;
2277}
2278
2279static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova,
2280 phys_addr_t paddr, size_t size, int prot,
2281 gfp_t gfp, size_t *mapped)
2282{
2283 const struct iommu_domain_ops *ops = domain->ops;
2284 size_t pgsize, count;
2285 int ret;
2286
2287 pgsize = iommu_pgsize(domain, iova, paddr, size, &count);
2288
2289 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
2290 iova, &paddr, pgsize, count);
2291
2292 if (ops->map_pages) {
2293 ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
2294 gfp, mapped);
2295 } else {
2296 ret = ops->map(domain, iova, paddr, pgsize, prot, gfp);
2297 *mapped = ret ? 0 : pgsize;
2298 }
2299
2300 return ret;
2301}
2302
2303static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
2304 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2305{
2306 const struct iommu_domain_ops *ops = domain->ops;
2307 unsigned long orig_iova = iova;
2308 unsigned int min_pagesz;
2309 size_t orig_size = size;
2310 phys_addr_t orig_paddr = paddr;
2311 int ret = 0;
2312
2313 if (unlikely(!(ops->map || ops->map_pages) ||
2314 domain->pgsize_bitmap == 0UL))
2315 return -ENODEV;
2316
2317 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2318 return -EINVAL;
2319
2320 /* find out the minimum page size supported */
2321 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2322
2323 /*
2324 * both the virtual address and the physical one, as well as
2325 * the size of the mapping, must be aligned (at least) to the
2326 * size of the smallest page supported by the hardware
2327 */
2328 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
2329 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
2330 iova, &paddr, size, min_pagesz);
2331 return -EINVAL;
2332 }
2333
2334 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
2335
2336 while (size) {
2337 size_t mapped = 0;
2338
2339 ret = __iommu_map_pages(domain, iova, paddr, size, prot, gfp,
2340 &mapped);
2341 /*
2342 * Some pages may have been mapped, even if an error occurred,
2343 * so we should account for those so they can be unmapped.
2344 */
2345 size -= mapped;
2346
2347 if (ret)
2348 break;
2349
2350 iova += mapped;
2351 paddr += mapped;
2352 }
2353
2354 /* unroll mapping in case something went wrong */
2355 if (ret)
2356 iommu_unmap(domain, orig_iova, orig_size - size);
2357 else
2358 trace_map(orig_iova, orig_paddr, orig_size);
2359
2360 return ret;
2361}
2362
2363static int _iommu_map(struct iommu_domain *domain, unsigned long iova,
2364 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2365{
2366 const struct iommu_domain_ops *ops = domain->ops;
2367 int ret;
2368
2369 ret = __iommu_map(domain, iova, paddr, size, prot, gfp);
2370 if (ret == 0 && ops->iotlb_sync_map)
2371 ops->iotlb_sync_map(domain, iova, size);
2372
2373 return ret;
2374}
2375
2376int iommu_map(struct iommu_domain *domain, unsigned long iova,
2377 phys_addr_t paddr, size_t size, int prot)
2378{
2379 might_sleep();
2380 return _iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL);
2381}
2382EXPORT_SYMBOL_GPL(iommu_map);
2383
2384int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
2385 phys_addr_t paddr, size_t size, int prot)
2386{
2387 return _iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC);
2388}
2389EXPORT_SYMBOL_GPL(iommu_map_atomic);
2390
2391static size_t __iommu_unmap_pages(struct iommu_domain *domain,
2392 unsigned long iova, size_t size,
2393 struct iommu_iotlb_gather *iotlb_gather)
2394{
2395 const struct iommu_domain_ops *ops = domain->ops;
2396 size_t pgsize, count;
2397
2398 pgsize = iommu_pgsize(domain, iova, iova, size, &count);
2399 return ops->unmap_pages ?
2400 ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather) :
2401 ops->unmap(domain, iova, pgsize, iotlb_gather);
2402}
2403
2404static size_t __iommu_unmap(struct iommu_domain *domain,
2405 unsigned long iova, size_t size,
2406 struct iommu_iotlb_gather *iotlb_gather)
2407{
2408 const struct iommu_domain_ops *ops = domain->ops;
2409 size_t unmapped_page, unmapped = 0;
2410 unsigned long orig_iova = iova;
2411 unsigned int min_pagesz;
2412
2413 if (unlikely(!(ops->unmap || ops->unmap_pages) ||
2414 domain->pgsize_bitmap == 0UL))
2415 return 0;
2416
2417 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2418 return 0;
2419
2420 /* find out the minimum page size supported */
2421 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2422
2423 /*
2424 * The virtual address, as well as the size of the mapping, must be
2425 * aligned (at least) to the size of the smallest page supported
2426 * by the hardware
2427 */
2428 if (!IS_ALIGNED(iova | size, min_pagesz)) {
2429 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
2430 iova, size, min_pagesz);
2431 return 0;
2432 }
2433
2434 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
2435
2436 /*
2437 * Keep iterating until we either unmap 'size' bytes (or more)
2438 * or we hit an area that isn't mapped.
2439 */
2440 while (unmapped < size) {
2441 unmapped_page = __iommu_unmap_pages(domain, iova,
2442 size - unmapped,
2443 iotlb_gather);
2444 if (!unmapped_page)
2445 break;
2446
2447 pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
2448 iova, unmapped_page);
2449
2450 iova += unmapped_page;
2451 unmapped += unmapped_page;
2452 }
2453
2454 trace_unmap(orig_iova, size, unmapped);
2455 return unmapped;
2456}
2457
2458size_t iommu_unmap(struct iommu_domain *domain,
2459 unsigned long iova, size_t size)
2460{
2461 struct iommu_iotlb_gather iotlb_gather;
2462 size_t ret;
2463
2464 iommu_iotlb_gather_init(&iotlb_gather);
2465 ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
2466 iommu_iotlb_sync(domain, &iotlb_gather);
2467
2468 return ret;
2469}
2470EXPORT_SYMBOL_GPL(iommu_unmap);
2471
2472size_t iommu_unmap_fast(struct iommu_domain *domain,
2473 unsigned long iova, size_t size,
2474 struct iommu_iotlb_gather *iotlb_gather)
2475{
2476 return __iommu_unmap(domain, iova, size, iotlb_gather);
2477}
2478EXPORT_SYMBOL_GPL(iommu_unmap_fast);
2479
2480static ssize_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
2481 struct scatterlist *sg, unsigned int nents, int prot,
2482 gfp_t gfp)
2483{
2484 const struct iommu_domain_ops *ops = domain->ops;
2485 size_t len = 0, mapped = 0;
2486 phys_addr_t start;
2487 unsigned int i = 0;
2488 int ret;
2489
2490 while (i <= nents) {
2491 phys_addr_t s_phys = sg_phys(sg);
2492
2493 if (len && s_phys != start + len) {
2494 ret = __iommu_map(domain, iova + mapped, start,
2495 len, prot, gfp);
2496
2497 if (ret)
2498 goto out_err;
2499
2500 mapped += len;
2501 len = 0;
2502 }
2503
2504 if (sg_is_dma_bus_address(sg))
2505 goto next;
2506
2507 if (len) {
2508 len += sg->length;
2509 } else {
2510 len = sg->length;
2511 start = s_phys;
2512 }
2513
2514next:
2515 if (++i < nents)
2516 sg = sg_next(sg);
2517 }
2518
2519 if (ops->iotlb_sync_map)
2520 ops->iotlb_sync_map(domain, iova, mapped);
2521 return mapped;
2522
2523out_err:
2524 /* undo mappings already done */
2525 iommu_unmap(domain, iova, mapped);
2526
2527 return ret;
2528}
2529
2530ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
2531 struct scatterlist *sg, unsigned int nents, int prot)
2532{
2533 might_sleep();
2534 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL);
2535}
2536EXPORT_SYMBOL_GPL(iommu_map_sg);
2537
2538ssize_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova,
2539 struct scatterlist *sg, unsigned int nents, int prot)
2540{
2541 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC);
2542}
2543
2544/**
2545 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
2546 * @domain: the iommu domain where the fault has happened
2547 * @dev: the device where the fault has happened
2548 * @iova: the faulting address
2549 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...)
2550 *
2551 * This function should be called by the low-level IOMMU implementations
2552 * whenever IOMMU faults happen, to allow high-level users, that are
2553 * interested in such events, to know about them.
2554 *
2555 * This event may be useful for several possible use cases:
2556 * - mere logging of the event
2557 * - dynamic TLB/PTE loading
2558 * - if restarting of the faulting device is required
2559 *
2560 * Returns 0 on success and an appropriate error code otherwise (if dynamic
2561 * PTE/TLB loading will one day be supported, implementations will be able
2562 * to tell whether it succeeded or not according to this return value).
2563 *
2564 * Specifically, -ENOSYS is returned if a fault handler isn't installed
2565 * (though fault handlers can also return -ENOSYS, in case they want to
2566 * elicit the default behavior of the IOMMU drivers).
2567 */
2568int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
2569 unsigned long iova, int flags)
2570{
2571 int ret = -ENOSYS;
2572
2573 /*
2574 * if upper layers showed interest and installed a fault handler,
2575 * invoke it.
2576 */
2577 if (domain->handler)
2578 ret = domain->handler(domain, dev, iova, flags,
2579 domain->handler_token);
2580
2581 trace_io_page_fault(dev, iova, flags);
2582 return ret;
2583}
2584EXPORT_SYMBOL_GPL(report_iommu_fault);
2585
2586static int __init iommu_init(void)
2587{
2588 iommu_group_kset = kset_create_and_add("iommu_groups",
2589 NULL, kernel_kobj);
2590 BUG_ON(!iommu_group_kset);
2591
2592 iommu_debugfs_setup();
2593
2594 return 0;
2595}
2596core_initcall(iommu_init);
2597
2598int iommu_enable_nesting(struct iommu_domain *domain)
2599{
2600 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
2601 return -EINVAL;
2602 if (!domain->ops->enable_nesting)
2603 return -EINVAL;
2604 return domain->ops->enable_nesting(domain);
2605}
2606EXPORT_SYMBOL_GPL(iommu_enable_nesting);
2607
2608int iommu_set_pgtable_quirks(struct iommu_domain *domain,
2609 unsigned long quirk)
2610{
2611 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
2612 return -EINVAL;
2613 if (!domain->ops->set_pgtable_quirks)
2614 return -EINVAL;
2615 return domain->ops->set_pgtable_quirks(domain, quirk);
2616}
2617EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);
2618
2619void iommu_get_resv_regions(struct device *dev, struct list_head *list)
2620{
2621 const struct iommu_ops *ops = dev_iommu_ops(dev);
2622
2623 if (ops->get_resv_regions)
2624 ops->get_resv_regions(dev, list);
2625}
2626
2627/**
2628 * iommu_put_resv_regions - release resered regions
2629 * @dev: device for which to free reserved regions
2630 * @list: reserved region list for device
2631 *
2632 * This releases a reserved region list acquired by iommu_get_resv_regions().
2633 */
2634void iommu_put_resv_regions(struct device *dev, struct list_head *list)
2635{
2636 struct iommu_resv_region *entry, *next;
2637
2638 list_for_each_entry_safe(entry, next, list, list) {
2639 if (entry->free)
2640 entry->free(dev, entry);
2641 else
2642 kfree(entry);
2643 }
2644}
2645EXPORT_SYMBOL(iommu_put_resv_regions);
2646
2647struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
2648 size_t length, int prot,
2649 enum iommu_resv_type type,
2650 gfp_t gfp)
2651{
2652 struct iommu_resv_region *region;
2653
2654 region = kzalloc(sizeof(*region), gfp);
2655 if (!region)
2656 return NULL;
2657
2658 INIT_LIST_HEAD(®ion->list);
2659 region->start = start;
2660 region->length = length;
2661 region->prot = prot;
2662 region->type = type;
2663 return region;
2664}
2665EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
2666
2667void iommu_set_default_passthrough(bool cmd_line)
2668{
2669 if (cmd_line)
2670 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
2671 iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
2672}
2673
2674void iommu_set_default_translated(bool cmd_line)
2675{
2676 if (cmd_line)
2677 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
2678 iommu_def_domain_type = IOMMU_DOMAIN_DMA;
2679}
2680
2681bool iommu_default_passthrough(void)
2682{
2683 return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
2684}
2685EXPORT_SYMBOL_GPL(iommu_default_passthrough);
2686
2687const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
2688{
2689 const struct iommu_ops *ops = NULL;
2690 struct iommu_device *iommu;
2691
2692 spin_lock(&iommu_device_lock);
2693 list_for_each_entry(iommu, &iommu_device_list, list)
2694 if (iommu->fwnode == fwnode) {
2695 ops = iommu->ops;
2696 break;
2697 }
2698 spin_unlock(&iommu_device_lock);
2699 return ops;
2700}
2701
2702int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
2703 const struct iommu_ops *ops)
2704{
2705 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2706
2707 if (fwspec)
2708 return ops == fwspec->ops ? 0 : -EINVAL;
2709
2710 if (!dev_iommu_get(dev))
2711 return -ENOMEM;
2712
2713 /* Preallocate for the overwhelmingly common case of 1 ID */
2714 fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL);
2715 if (!fwspec)
2716 return -ENOMEM;
2717
2718 of_node_get(to_of_node(iommu_fwnode));
2719 fwspec->iommu_fwnode = iommu_fwnode;
2720 fwspec->ops = ops;
2721 dev_iommu_fwspec_set(dev, fwspec);
2722 return 0;
2723}
2724EXPORT_SYMBOL_GPL(iommu_fwspec_init);
2725
2726void iommu_fwspec_free(struct device *dev)
2727{
2728 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2729
2730 if (fwspec) {
2731 fwnode_handle_put(fwspec->iommu_fwnode);
2732 kfree(fwspec);
2733 dev_iommu_fwspec_set(dev, NULL);
2734 }
2735}
2736EXPORT_SYMBOL_GPL(iommu_fwspec_free);
2737
2738int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids)
2739{
2740 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2741 int i, new_num;
2742
2743 if (!fwspec)
2744 return -EINVAL;
2745
2746 new_num = fwspec->num_ids + num_ids;
2747 if (new_num > 1) {
2748 fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num),
2749 GFP_KERNEL);
2750 if (!fwspec)
2751 return -ENOMEM;
2752
2753 dev_iommu_fwspec_set(dev, fwspec);
2754 }
2755
2756 for (i = 0; i < num_ids; i++)
2757 fwspec->ids[fwspec->num_ids + i] = ids[i];
2758
2759 fwspec->num_ids = new_num;
2760 return 0;
2761}
2762EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
2763
2764/*
2765 * Per device IOMMU features.
2766 */
2767int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
2768{
2769 if (dev->iommu && dev->iommu->iommu_dev) {
2770 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;
2771
2772 if (ops->dev_enable_feat)
2773 return ops->dev_enable_feat(dev, feat);
2774 }
2775
2776 return -ENODEV;
2777}
2778EXPORT_SYMBOL_GPL(iommu_dev_enable_feature);
2779
2780/*
2781 * The device drivers should do the necessary cleanups before calling this.
2782 */
2783int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
2784{
2785 if (dev->iommu && dev->iommu->iommu_dev) {
2786 const struct iommu_ops *ops = dev->iommu->iommu_dev->ops;
2787
2788 if (ops->dev_disable_feat)
2789 return ops->dev_disable_feat(dev, feat);
2790 }
2791
2792 return -EBUSY;
2793}
2794EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
2795
2796/*
2797 * Changes the default domain of an iommu group that has *only* one device
2798 *
2799 * @group: The group for which the default domain should be changed
2800 * @prev_dev: The device in the group (this is used to make sure that the device
2801 * hasn't changed after the caller has called this function)
2802 * @type: The type of the new default domain that gets associated with the group
2803 *
2804 * Returns 0 on success and error code on failure
2805 *
2806 * Note:
2807 * 1. Presently, this function is called only when user requests to change the
2808 * group's default domain type through /sys/kernel/iommu_groups/<grp_id>/type
2809 * Please take a closer look if intended to use for other purposes.
2810 */
2811static int iommu_change_dev_def_domain(struct iommu_group *group,
2812 struct device *prev_dev, int type)
2813{
2814 struct iommu_domain *prev_dom;
2815 struct group_device *grp_dev;
2816 int ret, dev_def_dom;
2817 struct device *dev;
2818
2819 mutex_lock(&group->mutex);
2820
2821 if (group->default_domain != group->domain) {
2822 dev_err_ratelimited(prev_dev, "Group not assigned to default domain\n");
2823 ret = -EBUSY;
2824 goto out;
2825 }
2826
2827 /*
2828 * iommu group wasn't locked while acquiring device lock in
2829 * iommu_group_store_type(). So, make sure that the device count hasn't
2830 * changed while acquiring device lock.
2831 *
2832 * Changing default domain of an iommu group with two or more devices
2833 * isn't supported because there could be a potential deadlock. Consider
2834 * the following scenario. T1 is trying to acquire device locks of all
2835 * the devices in the group and before it could acquire all of them,
2836 * there could be another thread T2 (from different sub-system and use
2837 * case) that has already acquired some of the device locks and might be
2838 * waiting for T1 to release other device locks.
2839 */
2840 if (iommu_group_device_count(group) != 1) {
2841 dev_err_ratelimited(prev_dev, "Cannot change default domain: Group has more than one device\n");
2842 ret = -EINVAL;
2843 goto out;
2844 }
2845
2846 /* Since group has only one device */
2847 grp_dev = list_first_entry(&group->devices, struct group_device, list);
2848 dev = grp_dev->dev;
2849
2850 if (prev_dev != dev) {
2851 dev_err_ratelimited(prev_dev, "Cannot change default domain: Device has been changed\n");
2852 ret = -EBUSY;
2853 goto out;
2854 }
2855
2856 prev_dom = group->default_domain;
2857 if (!prev_dom) {
2858 ret = -EINVAL;
2859 goto out;
2860 }
2861
2862 dev_def_dom = iommu_get_def_domain_type(dev);
2863 if (!type) {
2864 /*
2865 * If the user hasn't requested any specific type of domain and
2866 * if the device supports both the domains, then default to the
2867 * domain the device was booted with
2868 */
2869 type = dev_def_dom ? : iommu_def_domain_type;
2870 } else if (dev_def_dom && type != dev_def_dom) {
2871 dev_err_ratelimited(prev_dev, "Device cannot be in %s domain\n",
2872 iommu_domain_type_str(type));
2873 ret = -EINVAL;
2874 goto out;
2875 }
2876
2877 /*
2878 * Switch to a new domain only if the requested domain type is different
2879 * from the existing default domain type
2880 */
2881 if (prev_dom->type == type) {
2882 ret = 0;
2883 goto out;
2884 }
2885
2886 /* We can bring up a flush queue without tearing down the domain */
2887 if (type == IOMMU_DOMAIN_DMA_FQ && prev_dom->type == IOMMU_DOMAIN_DMA) {
2888 ret = iommu_dma_init_fq(prev_dom);
2889 if (!ret)
2890 prev_dom->type = IOMMU_DOMAIN_DMA_FQ;
2891 goto out;
2892 }
2893
2894 /* Sets group->default_domain to the newly allocated domain */
2895 ret = iommu_group_alloc_default_domain(dev->bus, group, type);
2896 if (ret)
2897 goto out;
2898
2899 ret = iommu_create_device_direct_mappings(group, dev);
2900 if (ret)
2901 goto free_new_domain;
2902
2903 ret = __iommu_attach_device(group->default_domain, dev);
2904 if (ret)
2905 goto free_new_domain;
2906
2907 group->domain = group->default_domain;
2908
2909 /*
2910 * Release the mutex here because ops->probe_finalize() call-back of
2911 * some vendor IOMMU drivers calls arm_iommu_attach_device() which
2912 * in-turn might call back into IOMMU core code, where it tries to take
2913 * group->mutex, resulting in a deadlock.
2914 */
2915 mutex_unlock(&group->mutex);
2916
2917 /* Make sure dma_ops is appropriatley set */
2918 iommu_group_do_probe_finalize(dev, group->default_domain);
2919 iommu_domain_free(prev_dom);
2920 return 0;
2921
2922free_new_domain:
2923 iommu_domain_free(group->default_domain);
2924 group->default_domain = prev_dom;
2925 group->domain = prev_dom;
2926
2927out:
2928 mutex_unlock(&group->mutex);
2929
2930 return ret;
2931}
2932
2933/*
2934 * Changing the default domain through sysfs requires the users to unbind the
2935 * drivers from the devices in the iommu group, except for a DMA -> DMA-FQ
2936 * transition. Return failure if this isn't met.
2937 *
2938 * We need to consider the race between this and the device release path.
2939 * device_lock(dev) is used here to guarantee that the device release path
2940 * will not be entered at the same time.
2941 */
2942static ssize_t iommu_group_store_type(struct iommu_group *group,
2943 const char *buf, size_t count)
2944{
2945 struct group_device *grp_dev;
2946 struct device *dev;
2947 int ret, req_type;
2948
2949 if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO))
2950 return -EACCES;
2951
2952 if (WARN_ON(!group) || !group->default_domain)
2953 return -EINVAL;
2954
2955 if (sysfs_streq(buf, "identity"))
2956 req_type = IOMMU_DOMAIN_IDENTITY;
2957 else if (sysfs_streq(buf, "DMA"))
2958 req_type = IOMMU_DOMAIN_DMA;
2959 else if (sysfs_streq(buf, "DMA-FQ"))
2960 req_type = IOMMU_DOMAIN_DMA_FQ;
2961 else if (sysfs_streq(buf, "auto"))
2962 req_type = 0;
2963 else
2964 return -EINVAL;
2965
2966 /*
2967 * Lock/Unlock the group mutex here before device lock to
2968 * 1. Make sure that the iommu group has only one device (this is a
2969 * prerequisite for step 2)
2970 * 2. Get struct *dev which is needed to lock device
2971 */
2972 mutex_lock(&group->mutex);
2973 if (iommu_group_device_count(group) != 1) {
2974 mutex_unlock(&group->mutex);
2975 pr_err_ratelimited("Cannot change default domain: Group has more than one device\n");
2976 return -EINVAL;
2977 }
2978
2979 /* Since group has only one device */
2980 grp_dev = list_first_entry(&group->devices, struct group_device, list);
2981 dev = grp_dev->dev;
2982 get_device(dev);
2983
2984 /*
2985 * Don't hold the group mutex because taking group mutex first and then
2986 * the device lock could potentially cause a deadlock as below. Assume
2987 * two threads T1 and T2. T1 is trying to change default domain of an
2988 * iommu group and T2 is trying to hot unplug a device or release [1] VF
2989 * of a PCIe device which is in the same iommu group. T1 takes group
2990 * mutex and before it could take device lock assume T2 has taken device
2991 * lock and is yet to take group mutex. Now, both the threads will be
2992 * waiting for the other thread to release lock. Below, lock order was
2993 * suggested.
2994 * device_lock(dev);
2995 * mutex_lock(&group->mutex);
2996 * iommu_change_dev_def_domain();
2997 * mutex_unlock(&group->mutex);
2998 * device_unlock(dev);
2999 *
3000 * [1] Typical device release path
3001 * device_lock() from device/driver core code
3002 * -> bus_notifier()
3003 * -> iommu_bus_notifier()
3004 * -> iommu_release_device()
3005 * -> ops->release_device() vendor driver calls back iommu core code
3006 * -> mutex_lock() from iommu core code
3007 */
3008 mutex_unlock(&group->mutex);
3009
3010 /* Check if the device in the group still has a driver bound to it */
3011 device_lock(dev);
3012 if (device_is_bound(dev) && !(req_type == IOMMU_DOMAIN_DMA_FQ &&
3013 group->default_domain->type == IOMMU_DOMAIN_DMA)) {
3014 pr_err_ratelimited("Device is still bound to driver\n");
3015 ret = -EBUSY;
3016 goto out;
3017 }
3018
3019 ret = iommu_change_dev_def_domain(group, dev, req_type);
3020 ret = ret ?: count;
3021
3022out:
3023 device_unlock(dev);
3024 put_device(dev);
3025
3026 return ret;
3027}
3028
3029static bool iommu_is_default_domain(struct iommu_group *group)
3030{
3031 if (group->domain == group->default_domain)
3032 return true;
3033
3034 /*
3035 * If the default domain was set to identity and it is still an identity
3036 * domain then we consider this a pass. This happens because of
3037 * amd_iommu_init_device() replacing the default idenytity domain with an
3038 * identity domain that has a different configuration for AMDGPU.
3039 */
3040 if (group->default_domain &&
3041 group->default_domain->type == IOMMU_DOMAIN_IDENTITY &&
3042 group->domain && group->domain->type == IOMMU_DOMAIN_IDENTITY)
3043 return true;
3044 return false;
3045}
3046
3047/**
3048 * iommu_device_use_default_domain() - Device driver wants to handle device
3049 * DMA through the kernel DMA API.
3050 * @dev: The device.
3051 *
3052 * The device driver about to bind @dev wants to do DMA through the kernel
3053 * DMA API. Return 0 if it is allowed, otherwise an error.
3054 */
3055int iommu_device_use_default_domain(struct device *dev)
3056{
3057 struct iommu_group *group = iommu_group_get(dev);
3058 int ret = 0;
3059
3060 if (!group)
3061 return 0;
3062
3063 mutex_lock(&group->mutex);
3064 if (group->owner_cnt) {
3065 if (group->owner || !iommu_is_default_domain(group) ||
3066 !xa_empty(&group->pasid_array)) {
3067 ret = -EBUSY;
3068 goto unlock_out;
3069 }
3070 }
3071
3072 group->owner_cnt++;
3073
3074unlock_out:
3075 mutex_unlock(&group->mutex);
3076 iommu_group_put(group);
3077
3078 return ret;
3079}
3080
3081/**
3082 * iommu_device_unuse_default_domain() - Device driver stops handling device
3083 * DMA through the kernel DMA API.
3084 * @dev: The device.
3085 *
3086 * The device driver doesn't want to do DMA through kernel DMA API anymore.
3087 * It must be called after iommu_device_use_default_domain().
3088 */
3089void iommu_device_unuse_default_domain(struct device *dev)
3090{
3091 struct iommu_group *group = iommu_group_get(dev);
3092
3093 if (!group)
3094 return;
3095
3096 mutex_lock(&group->mutex);
3097 if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array)))
3098 group->owner_cnt--;
3099
3100 mutex_unlock(&group->mutex);
3101 iommu_group_put(group);
3102}
3103
3104static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
3105{
3106 struct group_device *dev =
3107 list_first_entry(&group->devices, struct group_device, list);
3108
3109 if (group->blocking_domain)
3110 return 0;
3111
3112 group->blocking_domain =
3113 __iommu_domain_alloc(dev->dev->bus, IOMMU_DOMAIN_BLOCKED);
3114 if (!group->blocking_domain) {
3115 /*
3116 * For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED
3117 * create an empty domain instead.
3118 */
3119 group->blocking_domain = __iommu_domain_alloc(
3120 dev->dev->bus, IOMMU_DOMAIN_UNMANAGED);
3121 if (!group->blocking_domain)
3122 return -EINVAL;
3123 }
3124 return 0;
3125}
3126
3127static int __iommu_take_dma_ownership(struct iommu_group *group, void *owner)
3128{
3129 int ret;
3130
3131 if ((group->domain && group->domain != group->default_domain) ||
3132 !xa_empty(&group->pasid_array))
3133 return -EBUSY;
3134
3135 ret = __iommu_group_alloc_blocking_domain(group);
3136 if (ret)
3137 return ret;
3138 ret = __iommu_group_set_domain(group, group->blocking_domain);
3139 if (ret)
3140 return ret;
3141
3142 group->owner = owner;
3143 group->owner_cnt++;
3144 return 0;
3145}
3146
3147/**
3148 * iommu_group_claim_dma_owner() - Set DMA ownership of a group
3149 * @group: The group.
3150 * @owner: Caller specified pointer. Used for exclusive ownership.
3151 *
3152 * This is to support backward compatibility for vfio which manages the dma
3153 * ownership in iommu_group level. New invocations on this interface should be
3154 * prohibited. Only a single owner may exist for a group.
3155 */
3156int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
3157{
3158 int ret = 0;
3159
3160 if (WARN_ON(!owner))
3161 return -EINVAL;
3162
3163 mutex_lock(&group->mutex);
3164 if (group->owner_cnt) {
3165 ret = -EPERM;
3166 goto unlock_out;
3167 }
3168
3169 ret = __iommu_take_dma_ownership(group, owner);
3170unlock_out:
3171 mutex_unlock(&group->mutex);
3172
3173 return ret;
3174}
3175EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);
3176
3177/**
3178 * iommu_device_claim_dma_owner() - Set DMA ownership of a device
3179 * @dev: The device.
3180 * @owner: Caller specified pointer. Used for exclusive ownership.
3181 *
3182 * Claim the DMA ownership of a device. Multiple devices in the same group may
3183 * concurrently claim ownership if they present the same owner value. Returns 0
3184 * on success and error code on failure
3185 */
3186int iommu_device_claim_dma_owner(struct device *dev, void *owner)
3187{
3188 struct iommu_group *group;
3189 int ret = 0;
3190
3191 if (WARN_ON(!owner))
3192 return -EINVAL;
3193
3194 group = iommu_group_get(dev);
3195 if (!group)
3196 return -ENODEV;
3197
3198 mutex_lock(&group->mutex);
3199 if (group->owner_cnt) {
3200 if (group->owner != owner) {
3201 ret = -EPERM;
3202 goto unlock_out;
3203 }
3204 group->owner_cnt++;
3205 goto unlock_out;
3206 }
3207
3208 ret = __iommu_take_dma_ownership(group, owner);
3209unlock_out:
3210 mutex_unlock(&group->mutex);
3211 iommu_group_put(group);
3212
3213 return ret;
3214}
3215EXPORT_SYMBOL_GPL(iommu_device_claim_dma_owner);
3216
3217static void __iommu_release_dma_ownership(struct iommu_group *group)
3218{
3219 int ret;
3220
3221 if (WARN_ON(!group->owner_cnt || !group->owner ||
3222 !xa_empty(&group->pasid_array)))
3223 return;
3224
3225 group->owner_cnt = 0;
3226 group->owner = NULL;
3227 ret = __iommu_group_set_domain(group, group->default_domain);
3228 WARN(ret, "iommu driver failed to attach the default domain");
3229}
3230
3231/**
3232 * iommu_group_release_dma_owner() - Release DMA ownership of a group
3233 * @dev: The device
3234 *
3235 * Release the DMA ownership claimed by iommu_group_claim_dma_owner().
3236 */
3237void iommu_group_release_dma_owner(struct iommu_group *group)
3238{
3239 mutex_lock(&group->mutex);
3240 __iommu_release_dma_ownership(group);
3241 mutex_unlock(&group->mutex);
3242}
3243EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);
3244
3245/**
3246 * iommu_device_release_dma_owner() - Release DMA ownership of a device
3247 * @group: The device.
3248 *
3249 * Release the DMA ownership claimed by iommu_device_claim_dma_owner().
3250 */
3251void iommu_device_release_dma_owner(struct device *dev)
3252{
3253 struct iommu_group *group = iommu_group_get(dev);
3254
3255 mutex_lock(&group->mutex);
3256 if (group->owner_cnt > 1)
3257 group->owner_cnt--;
3258 else
3259 __iommu_release_dma_ownership(group);
3260 mutex_unlock(&group->mutex);
3261 iommu_group_put(group);
3262}
3263EXPORT_SYMBOL_GPL(iommu_device_release_dma_owner);
3264
3265/**
3266 * iommu_group_dma_owner_claimed() - Query group dma ownership status
3267 * @group: The group.
3268 *
3269 * This provides status query on a given group. It is racy and only for
3270 * non-binding status reporting.
3271 */
3272bool iommu_group_dma_owner_claimed(struct iommu_group *group)
3273{
3274 unsigned int user;
3275
3276 mutex_lock(&group->mutex);
3277 user = group->owner_cnt;
3278 mutex_unlock(&group->mutex);
3279
3280 return user;
3281}
3282EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
3283
3284static int __iommu_set_group_pasid(struct iommu_domain *domain,
3285 struct iommu_group *group, ioasid_t pasid)
3286{
3287 struct group_device *device;
3288 int ret = 0;
3289
3290 list_for_each_entry(device, &group->devices, list) {
3291 ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
3292 if (ret)
3293 break;
3294 }
3295
3296 return ret;
3297}
3298
3299static void __iommu_remove_group_pasid(struct iommu_group *group,
3300 ioasid_t pasid)
3301{
3302 struct group_device *device;
3303 const struct iommu_ops *ops;
3304
3305 list_for_each_entry(device, &group->devices, list) {
3306 ops = dev_iommu_ops(device->dev);
3307 ops->remove_dev_pasid(device->dev, pasid);
3308 }
3309}
3310
3311/*
3312 * iommu_attach_device_pasid() - Attach a domain to pasid of device
3313 * @domain: the iommu domain.
3314 * @dev: the attached device.
3315 * @pasid: the pasid of the device.
3316 *
3317 * Return: 0 on success, or an error.
3318 */
3319int iommu_attach_device_pasid(struct iommu_domain *domain,
3320 struct device *dev, ioasid_t pasid)
3321{
3322 struct iommu_group *group;
3323 void *curr;
3324 int ret;
3325
3326 if (!domain->ops->set_dev_pasid)
3327 return -EOPNOTSUPP;
3328
3329 group = iommu_group_get(dev);
3330 if (!group)
3331 return -ENODEV;
3332
3333 mutex_lock(&group->mutex);
3334 curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
3335 if (curr) {
3336 ret = xa_err(curr) ? : -EBUSY;
3337 goto out_unlock;
3338 }
3339
3340 ret = __iommu_set_group_pasid(domain, group, pasid);
3341 if (ret) {
3342 __iommu_remove_group_pasid(group, pasid);
3343 xa_erase(&group->pasid_array, pasid);
3344 }
3345out_unlock:
3346 mutex_unlock(&group->mutex);
3347 iommu_group_put(group);
3348
3349 return ret;
3350}
3351EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
3352
3353/*
3354 * iommu_detach_device_pasid() - Detach the domain from pasid of device
3355 * @domain: the iommu domain.
3356 * @dev: the attached device.
3357 * @pasid: the pasid of the device.
3358 *
3359 * The @domain must have been attached to @pasid of the @dev with
3360 * iommu_attach_device_pasid().
3361 */
3362void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
3363 ioasid_t pasid)
3364{
3365 struct iommu_group *group = iommu_group_get(dev);
3366
3367 mutex_lock(&group->mutex);
3368 __iommu_remove_group_pasid(group, pasid);
3369 WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
3370 mutex_unlock(&group->mutex);
3371
3372 iommu_group_put(group);
3373}
3374EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
3375
3376/*
3377 * iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
3378 * @dev: the queried device
3379 * @pasid: the pasid of the device
3380 * @type: matched domain type, 0 for any match
3381 *
3382 * This is a variant of iommu_get_domain_for_dev(). It returns the existing
3383 * domain attached to pasid of a device. Callers must hold a lock around this
3384 * function, and both iommu_attach/detach_dev_pasid() whenever a domain of
3385 * type is being manipulated. This API does not internally resolve races with
3386 * attach/detach.
3387 *
3388 * Return: attached domain on success, NULL otherwise.
3389 */
3390struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
3391 ioasid_t pasid,
3392 unsigned int type)
3393{
3394 struct iommu_domain *domain;
3395 struct iommu_group *group;
3396
3397 group = iommu_group_get(dev);
3398 if (!group)
3399 return NULL;
3400
3401 xa_lock(&group->pasid_array);
3402 domain = xa_load(&group->pasid_array, pasid);
3403 if (type && domain && domain->type != type)
3404 domain = ERR_PTR(-EBUSY);
3405 xa_unlock(&group->pasid_array);
3406 iommu_group_put(group);
3407
3408 return domain;
3409}
3410EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
3411
3412struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
3413 struct mm_struct *mm)
3414{
3415 const struct iommu_ops *ops = dev_iommu_ops(dev);
3416 struct iommu_domain *domain;
3417
3418 domain = ops->domain_alloc(IOMMU_DOMAIN_SVA);
3419 if (!domain)
3420 return NULL;
3421
3422 domain->type = IOMMU_DOMAIN_SVA;
3423 mmgrab(mm);
3424 domain->mm = mm;
3425 domain->iopf_handler = iommu_sva_handle_iopf;
3426 domain->fault_data = mm;
3427
3428 return domain;
3429}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 */
6
7#define pr_fmt(fmt) "iommu: " fmt
8
9#include <linux/device.h>
10#include <linux/kernel.h>
11#include <linux/bug.h>
12#include <linux/types.h>
13#include <linux/init.h>
14#include <linux/export.h>
15#include <linux/slab.h>
16#include <linux/errno.h>
17#include <linux/iommu.h>
18#include <linux/idr.h>
19#include <linux/notifier.h>
20#include <linux/err.h>
21#include <linux/pci.h>
22#include <linux/bitops.h>
23#include <linux/property.h>
24#include <linux/fsl/mc.h>
25#include <linux/module.h>
26#include <trace/events/iommu.h>
27
28static struct kset *iommu_group_kset;
29static DEFINE_IDA(iommu_group_ida);
30
31static unsigned int iommu_def_domain_type __read_mostly;
32static bool iommu_dma_strict __read_mostly = true;
33static u32 iommu_cmd_line __read_mostly;
34
35struct iommu_group {
36 struct kobject kobj;
37 struct kobject *devices_kobj;
38 struct list_head devices;
39 struct mutex mutex;
40 struct blocking_notifier_head notifier;
41 void *iommu_data;
42 void (*iommu_data_release)(void *iommu_data);
43 char *name;
44 int id;
45 struct iommu_domain *default_domain;
46 struct iommu_domain *domain;
47 struct list_head entry;
48};
49
50struct group_device {
51 struct list_head list;
52 struct device *dev;
53 char *name;
54};
55
56struct iommu_group_attribute {
57 struct attribute attr;
58 ssize_t (*show)(struct iommu_group *group, char *buf);
59 ssize_t (*store)(struct iommu_group *group,
60 const char *buf, size_t count);
61};
62
63static const char * const iommu_group_resv_type_string[] = {
64 [IOMMU_RESV_DIRECT] = "direct",
65 [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable",
66 [IOMMU_RESV_RESERVED] = "reserved",
67 [IOMMU_RESV_MSI] = "msi",
68 [IOMMU_RESV_SW_MSI] = "msi",
69};
70
71#define IOMMU_CMD_LINE_DMA_API BIT(0)
72
73static void iommu_set_cmd_line_dma_api(void)
74{
75 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
76}
77
78static bool iommu_cmd_line_dma_api(void)
79{
80 return !!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API);
81}
82
83static int iommu_alloc_default_domain(struct iommu_group *group,
84 struct device *dev);
85static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
86 unsigned type);
87static int __iommu_attach_device(struct iommu_domain *domain,
88 struct device *dev);
89static int __iommu_attach_group(struct iommu_domain *domain,
90 struct iommu_group *group);
91static void __iommu_detach_group(struct iommu_domain *domain,
92 struct iommu_group *group);
93static int iommu_create_device_direct_mappings(struct iommu_group *group,
94 struct device *dev);
95static struct iommu_group *iommu_group_get_for_dev(struct device *dev);
96
97#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
98struct iommu_group_attribute iommu_group_attr_##_name = \
99 __ATTR(_name, _mode, _show, _store)
100
101#define to_iommu_group_attr(_attr) \
102 container_of(_attr, struct iommu_group_attribute, attr)
103#define to_iommu_group(_kobj) \
104 container_of(_kobj, struct iommu_group, kobj)
105
106static LIST_HEAD(iommu_device_list);
107static DEFINE_SPINLOCK(iommu_device_lock);
108
109/*
110 * Use a function instead of an array here because the domain-type is a
111 * bit-field, so an array would waste memory.
112 */
113static const char *iommu_domain_type_str(unsigned int t)
114{
115 switch (t) {
116 case IOMMU_DOMAIN_BLOCKED:
117 return "Blocked";
118 case IOMMU_DOMAIN_IDENTITY:
119 return "Passthrough";
120 case IOMMU_DOMAIN_UNMANAGED:
121 return "Unmanaged";
122 case IOMMU_DOMAIN_DMA:
123 return "Translated";
124 default:
125 return "Unknown";
126 }
127}
128
129static int __init iommu_subsys_init(void)
130{
131 bool cmd_line = iommu_cmd_line_dma_api();
132
133 if (!cmd_line) {
134 if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
135 iommu_set_default_passthrough(false);
136 else
137 iommu_set_default_translated(false);
138
139 if (iommu_default_passthrough() && mem_encrypt_active()) {
140 pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
141 iommu_set_default_translated(false);
142 }
143 }
144
145 pr_info("Default domain type: %s %s\n",
146 iommu_domain_type_str(iommu_def_domain_type),
147 cmd_line ? "(set via kernel command line)" : "");
148
149 return 0;
150}
151subsys_initcall(iommu_subsys_init);
152
153int iommu_device_register(struct iommu_device *iommu)
154{
155 spin_lock(&iommu_device_lock);
156 list_add_tail(&iommu->list, &iommu_device_list);
157 spin_unlock(&iommu_device_lock);
158 return 0;
159}
160EXPORT_SYMBOL_GPL(iommu_device_register);
161
162void iommu_device_unregister(struct iommu_device *iommu)
163{
164 spin_lock(&iommu_device_lock);
165 list_del(&iommu->list);
166 spin_unlock(&iommu_device_lock);
167}
168EXPORT_SYMBOL_GPL(iommu_device_unregister);
169
170static struct dev_iommu *dev_iommu_get(struct device *dev)
171{
172 struct dev_iommu *param = dev->iommu;
173
174 if (param)
175 return param;
176
177 param = kzalloc(sizeof(*param), GFP_KERNEL);
178 if (!param)
179 return NULL;
180
181 mutex_init(¶m->lock);
182 dev->iommu = param;
183 return param;
184}
185
186static void dev_iommu_free(struct device *dev)
187{
188 iommu_fwspec_free(dev);
189 kfree(dev->iommu);
190 dev->iommu = NULL;
191}
192
193static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
194{
195 const struct iommu_ops *ops = dev->bus->iommu_ops;
196 struct iommu_device *iommu_dev;
197 struct iommu_group *group;
198 int ret;
199
200 if (!ops)
201 return -ENODEV;
202
203 if (!dev_iommu_get(dev))
204 return -ENOMEM;
205
206 if (!try_module_get(ops->owner)) {
207 ret = -EINVAL;
208 goto err_free;
209 }
210
211 iommu_dev = ops->probe_device(dev);
212 if (IS_ERR(iommu_dev)) {
213 ret = PTR_ERR(iommu_dev);
214 goto out_module_put;
215 }
216
217 dev->iommu->iommu_dev = iommu_dev;
218
219 group = iommu_group_get_for_dev(dev);
220 if (IS_ERR(group)) {
221 ret = PTR_ERR(group);
222 goto out_release;
223 }
224 iommu_group_put(group);
225
226 if (group_list && !group->default_domain && list_empty(&group->entry))
227 list_add_tail(&group->entry, group_list);
228
229 iommu_device_link(iommu_dev, dev);
230
231 return 0;
232
233out_release:
234 ops->release_device(dev);
235
236out_module_put:
237 module_put(ops->owner);
238
239err_free:
240 dev_iommu_free(dev);
241
242 return ret;
243}
244
245int iommu_probe_device(struct device *dev)
246{
247 const struct iommu_ops *ops = dev->bus->iommu_ops;
248 struct iommu_group *group;
249 int ret;
250
251 ret = __iommu_probe_device(dev, NULL);
252 if (ret)
253 goto err_out;
254
255 group = iommu_group_get(dev);
256 if (!group)
257 goto err_release;
258
259 /*
260 * Try to allocate a default domain - needs support from the
261 * IOMMU driver. There are still some drivers which don't
262 * support default domains, so the return value is not yet
263 * checked.
264 */
265 iommu_alloc_default_domain(group, dev);
266
267 if (group->default_domain)
268 ret = __iommu_attach_device(group->default_domain, dev);
269
270 iommu_create_device_direct_mappings(group, dev);
271
272 iommu_group_put(group);
273
274 if (ret)
275 goto err_release;
276
277 if (ops->probe_finalize)
278 ops->probe_finalize(dev);
279
280 return 0;
281
282err_release:
283 iommu_release_device(dev);
284
285err_out:
286 return ret;
287
288}
289
290void iommu_release_device(struct device *dev)
291{
292 const struct iommu_ops *ops = dev->bus->iommu_ops;
293
294 if (!dev->iommu)
295 return;
296
297 iommu_device_unlink(dev->iommu->iommu_dev, dev);
298
299 ops->release_device(dev);
300
301 iommu_group_remove_device(dev);
302 module_put(ops->owner);
303 dev_iommu_free(dev);
304}
305
306static int __init iommu_set_def_domain_type(char *str)
307{
308 bool pt;
309 int ret;
310
311 ret = kstrtobool(str, &pt);
312 if (ret)
313 return ret;
314
315 if (pt)
316 iommu_set_default_passthrough(true);
317 else
318 iommu_set_default_translated(true);
319
320 return 0;
321}
322early_param("iommu.passthrough", iommu_set_def_domain_type);
323
324static int __init iommu_dma_setup(char *str)
325{
326 return kstrtobool(str, &iommu_dma_strict);
327}
328early_param("iommu.strict", iommu_dma_setup);
329
330static ssize_t iommu_group_attr_show(struct kobject *kobj,
331 struct attribute *__attr, char *buf)
332{
333 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
334 struct iommu_group *group = to_iommu_group(kobj);
335 ssize_t ret = -EIO;
336
337 if (attr->show)
338 ret = attr->show(group, buf);
339 return ret;
340}
341
342static ssize_t iommu_group_attr_store(struct kobject *kobj,
343 struct attribute *__attr,
344 const char *buf, size_t count)
345{
346 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
347 struct iommu_group *group = to_iommu_group(kobj);
348 ssize_t ret = -EIO;
349
350 if (attr->store)
351 ret = attr->store(group, buf, count);
352 return ret;
353}
354
355static const struct sysfs_ops iommu_group_sysfs_ops = {
356 .show = iommu_group_attr_show,
357 .store = iommu_group_attr_store,
358};
359
360static int iommu_group_create_file(struct iommu_group *group,
361 struct iommu_group_attribute *attr)
362{
363 return sysfs_create_file(&group->kobj, &attr->attr);
364}
365
366static void iommu_group_remove_file(struct iommu_group *group,
367 struct iommu_group_attribute *attr)
368{
369 sysfs_remove_file(&group->kobj, &attr->attr);
370}
371
372static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
373{
374 return sprintf(buf, "%s\n", group->name);
375}
376
377/**
378 * iommu_insert_resv_region - Insert a new region in the
379 * list of reserved regions.
380 * @new: new region to insert
381 * @regions: list of regions
382 *
383 * Elements are sorted by start address and overlapping segments
384 * of the same type are merged.
385 */
386static int iommu_insert_resv_region(struct iommu_resv_region *new,
387 struct list_head *regions)
388{
389 struct iommu_resv_region *iter, *tmp, *nr, *top;
390 LIST_HEAD(stack);
391
392 nr = iommu_alloc_resv_region(new->start, new->length,
393 new->prot, new->type);
394 if (!nr)
395 return -ENOMEM;
396
397 /* First add the new element based on start address sorting */
398 list_for_each_entry(iter, regions, list) {
399 if (nr->start < iter->start ||
400 (nr->start == iter->start && nr->type <= iter->type))
401 break;
402 }
403 list_add_tail(&nr->list, &iter->list);
404
405 /* Merge overlapping segments of type nr->type in @regions, if any */
406 list_for_each_entry_safe(iter, tmp, regions, list) {
407 phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
408
409 /* no merge needed on elements of different types than @new */
410 if (iter->type != new->type) {
411 list_move_tail(&iter->list, &stack);
412 continue;
413 }
414
415 /* look for the last stack element of same type as @iter */
416 list_for_each_entry_reverse(top, &stack, list)
417 if (top->type == iter->type)
418 goto check_overlap;
419
420 list_move_tail(&iter->list, &stack);
421 continue;
422
423check_overlap:
424 top_end = top->start + top->length - 1;
425
426 if (iter->start > top_end + 1) {
427 list_move_tail(&iter->list, &stack);
428 } else {
429 top->length = max(top_end, iter_end) - top->start + 1;
430 list_del(&iter->list);
431 kfree(iter);
432 }
433 }
434 list_splice(&stack, regions);
435 return 0;
436}
437
438static int
439iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
440 struct list_head *group_resv_regions)
441{
442 struct iommu_resv_region *entry;
443 int ret = 0;
444
445 list_for_each_entry(entry, dev_resv_regions, list) {
446 ret = iommu_insert_resv_region(entry, group_resv_regions);
447 if (ret)
448 break;
449 }
450 return ret;
451}
452
453int iommu_get_group_resv_regions(struct iommu_group *group,
454 struct list_head *head)
455{
456 struct group_device *device;
457 int ret = 0;
458
459 mutex_lock(&group->mutex);
460 list_for_each_entry(device, &group->devices, list) {
461 struct list_head dev_resv_regions;
462
463 INIT_LIST_HEAD(&dev_resv_regions);
464 iommu_get_resv_regions(device->dev, &dev_resv_regions);
465 ret = iommu_insert_device_resv_regions(&dev_resv_regions, head);
466 iommu_put_resv_regions(device->dev, &dev_resv_regions);
467 if (ret)
468 break;
469 }
470 mutex_unlock(&group->mutex);
471 return ret;
472}
473EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
474
475static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
476 char *buf)
477{
478 struct iommu_resv_region *region, *next;
479 struct list_head group_resv_regions;
480 char *str = buf;
481
482 INIT_LIST_HEAD(&group_resv_regions);
483 iommu_get_group_resv_regions(group, &group_resv_regions);
484
485 list_for_each_entry_safe(region, next, &group_resv_regions, list) {
486 str += sprintf(str, "0x%016llx 0x%016llx %s\n",
487 (long long int)region->start,
488 (long long int)(region->start +
489 region->length - 1),
490 iommu_group_resv_type_string[region->type]);
491 kfree(region);
492 }
493
494 return (str - buf);
495}
496
497static ssize_t iommu_group_show_type(struct iommu_group *group,
498 char *buf)
499{
500 char *type = "unknown\n";
501
502 if (group->default_domain) {
503 switch (group->default_domain->type) {
504 case IOMMU_DOMAIN_BLOCKED:
505 type = "blocked\n";
506 break;
507 case IOMMU_DOMAIN_IDENTITY:
508 type = "identity\n";
509 break;
510 case IOMMU_DOMAIN_UNMANAGED:
511 type = "unmanaged\n";
512 break;
513 case IOMMU_DOMAIN_DMA:
514 type = "DMA\n";
515 break;
516 }
517 }
518 strcpy(buf, type);
519
520 return strlen(type);
521}
522
523static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
524
525static IOMMU_GROUP_ATTR(reserved_regions, 0444,
526 iommu_group_show_resv_regions, NULL);
527
528static IOMMU_GROUP_ATTR(type, 0444, iommu_group_show_type, NULL);
529
530static void iommu_group_release(struct kobject *kobj)
531{
532 struct iommu_group *group = to_iommu_group(kobj);
533
534 pr_debug("Releasing group %d\n", group->id);
535
536 if (group->iommu_data_release)
537 group->iommu_data_release(group->iommu_data);
538
539 ida_simple_remove(&iommu_group_ida, group->id);
540
541 if (group->default_domain)
542 iommu_domain_free(group->default_domain);
543
544 kfree(group->name);
545 kfree(group);
546}
547
548static struct kobj_type iommu_group_ktype = {
549 .sysfs_ops = &iommu_group_sysfs_ops,
550 .release = iommu_group_release,
551};
552
553/**
554 * iommu_group_alloc - Allocate a new group
555 *
556 * This function is called by an iommu driver to allocate a new iommu
557 * group. The iommu group represents the minimum granularity of the iommu.
558 * Upon successful return, the caller holds a reference to the supplied
559 * group in order to hold the group until devices are added. Use
560 * iommu_group_put() to release this extra reference count, allowing the
561 * group to be automatically reclaimed once it has no devices or external
562 * references.
563 */
564struct iommu_group *iommu_group_alloc(void)
565{
566 struct iommu_group *group;
567 int ret;
568
569 group = kzalloc(sizeof(*group), GFP_KERNEL);
570 if (!group)
571 return ERR_PTR(-ENOMEM);
572
573 group->kobj.kset = iommu_group_kset;
574 mutex_init(&group->mutex);
575 INIT_LIST_HEAD(&group->devices);
576 INIT_LIST_HEAD(&group->entry);
577 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
578
579 ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL);
580 if (ret < 0) {
581 kfree(group);
582 return ERR_PTR(ret);
583 }
584 group->id = ret;
585
586 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
587 NULL, "%d", group->id);
588 if (ret) {
589 ida_simple_remove(&iommu_group_ida, group->id);
590 kobject_put(&group->kobj);
591 return ERR_PTR(ret);
592 }
593
594 group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
595 if (!group->devices_kobj) {
596 kobject_put(&group->kobj); /* triggers .release & free */
597 return ERR_PTR(-ENOMEM);
598 }
599
600 /*
601 * The devices_kobj holds a reference on the group kobject, so
602 * as long as that exists so will the group. We can therefore
603 * use the devices_kobj for reference counting.
604 */
605 kobject_put(&group->kobj);
606
607 ret = iommu_group_create_file(group,
608 &iommu_group_attr_reserved_regions);
609 if (ret)
610 return ERR_PTR(ret);
611
612 ret = iommu_group_create_file(group, &iommu_group_attr_type);
613 if (ret)
614 return ERR_PTR(ret);
615
616 pr_debug("Allocated group %d\n", group->id);
617
618 return group;
619}
620EXPORT_SYMBOL_GPL(iommu_group_alloc);
621
622struct iommu_group *iommu_group_get_by_id(int id)
623{
624 struct kobject *group_kobj;
625 struct iommu_group *group;
626 const char *name;
627
628 if (!iommu_group_kset)
629 return NULL;
630
631 name = kasprintf(GFP_KERNEL, "%d", id);
632 if (!name)
633 return NULL;
634
635 group_kobj = kset_find_obj(iommu_group_kset, name);
636 kfree(name);
637
638 if (!group_kobj)
639 return NULL;
640
641 group = container_of(group_kobj, struct iommu_group, kobj);
642 BUG_ON(group->id != id);
643
644 kobject_get(group->devices_kobj);
645 kobject_put(&group->kobj);
646
647 return group;
648}
649EXPORT_SYMBOL_GPL(iommu_group_get_by_id);
650
651/**
652 * iommu_group_get_iommudata - retrieve iommu_data registered for a group
653 * @group: the group
654 *
655 * iommu drivers can store data in the group for use when doing iommu
656 * operations. This function provides a way to retrieve it. Caller
657 * should hold a group reference.
658 */
659void *iommu_group_get_iommudata(struct iommu_group *group)
660{
661 return group->iommu_data;
662}
663EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
664
665/**
666 * iommu_group_set_iommudata - set iommu_data for a group
667 * @group: the group
668 * @iommu_data: new data
669 * @release: release function for iommu_data
670 *
671 * iommu drivers can store data in the group for use when doing iommu
672 * operations. This function provides a way to set the data after
673 * the group has been allocated. Caller should hold a group reference.
674 */
675void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
676 void (*release)(void *iommu_data))
677{
678 group->iommu_data = iommu_data;
679 group->iommu_data_release = release;
680}
681EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
682
683/**
684 * iommu_group_set_name - set name for a group
685 * @group: the group
686 * @name: name
687 *
688 * Allow iommu driver to set a name for a group. When set it will
689 * appear in a name attribute file under the group in sysfs.
690 */
691int iommu_group_set_name(struct iommu_group *group, const char *name)
692{
693 int ret;
694
695 if (group->name) {
696 iommu_group_remove_file(group, &iommu_group_attr_name);
697 kfree(group->name);
698 group->name = NULL;
699 if (!name)
700 return 0;
701 }
702
703 group->name = kstrdup(name, GFP_KERNEL);
704 if (!group->name)
705 return -ENOMEM;
706
707 ret = iommu_group_create_file(group, &iommu_group_attr_name);
708 if (ret) {
709 kfree(group->name);
710 group->name = NULL;
711 return ret;
712 }
713
714 return 0;
715}
716EXPORT_SYMBOL_GPL(iommu_group_set_name);
717
718static int iommu_create_device_direct_mappings(struct iommu_group *group,
719 struct device *dev)
720{
721 struct iommu_domain *domain = group->default_domain;
722 struct iommu_resv_region *entry;
723 struct list_head mappings;
724 unsigned long pg_size;
725 int ret = 0;
726
727 if (!domain || domain->type != IOMMU_DOMAIN_DMA)
728 return 0;
729
730 BUG_ON(!domain->pgsize_bitmap);
731
732 pg_size = 1UL << __ffs(domain->pgsize_bitmap);
733 INIT_LIST_HEAD(&mappings);
734
735 iommu_get_resv_regions(dev, &mappings);
736
737 /* We need to consider overlapping regions for different devices */
738 list_for_each_entry(entry, &mappings, list) {
739 dma_addr_t start, end, addr;
740
741 if (domain->ops->apply_resv_region)
742 domain->ops->apply_resv_region(dev, domain, entry);
743
744 start = ALIGN(entry->start, pg_size);
745 end = ALIGN(entry->start + entry->length, pg_size);
746
747 if (entry->type != IOMMU_RESV_DIRECT &&
748 entry->type != IOMMU_RESV_DIRECT_RELAXABLE)
749 continue;
750
751 for (addr = start; addr < end; addr += pg_size) {
752 phys_addr_t phys_addr;
753
754 phys_addr = iommu_iova_to_phys(domain, addr);
755 if (phys_addr)
756 continue;
757
758 ret = iommu_map(domain, addr, addr, pg_size, entry->prot);
759 if (ret)
760 goto out;
761 }
762
763 }
764
765 iommu_flush_tlb_all(domain);
766
767out:
768 iommu_put_resv_regions(dev, &mappings);
769
770 return ret;
771}
772
773static bool iommu_is_attach_deferred(struct iommu_domain *domain,
774 struct device *dev)
775{
776 if (domain->ops->is_attach_deferred)
777 return domain->ops->is_attach_deferred(domain, dev);
778
779 return false;
780}
781
782/**
783 * iommu_group_add_device - add a device to an iommu group
784 * @group: the group into which to add the device (reference should be held)
785 * @dev: the device
786 *
787 * This function is called by an iommu driver to add a device into a
788 * group. Adding a device increments the group reference count.
789 */
790int iommu_group_add_device(struct iommu_group *group, struct device *dev)
791{
792 int ret, i = 0;
793 struct group_device *device;
794
795 device = kzalloc(sizeof(*device), GFP_KERNEL);
796 if (!device)
797 return -ENOMEM;
798
799 device->dev = dev;
800
801 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
802 if (ret)
803 goto err_free_device;
804
805 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
806rename:
807 if (!device->name) {
808 ret = -ENOMEM;
809 goto err_remove_link;
810 }
811
812 ret = sysfs_create_link_nowarn(group->devices_kobj,
813 &dev->kobj, device->name);
814 if (ret) {
815 if (ret == -EEXIST && i >= 0) {
816 /*
817 * Account for the slim chance of collision
818 * and append an instance to the name.
819 */
820 kfree(device->name);
821 device->name = kasprintf(GFP_KERNEL, "%s.%d",
822 kobject_name(&dev->kobj), i++);
823 goto rename;
824 }
825 goto err_free_name;
826 }
827
828 kobject_get(group->devices_kobj);
829
830 dev->iommu_group = group;
831
832 mutex_lock(&group->mutex);
833 list_add_tail(&device->list, &group->devices);
834 if (group->domain && !iommu_is_attach_deferred(group->domain, dev))
835 ret = __iommu_attach_device(group->domain, dev);
836 mutex_unlock(&group->mutex);
837 if (ret)
838 goto err_put_group;
839
840 /* Notify any listeners about change to group. */
841 blocking_notifier_call_chain(&group->notifier,
842 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev);
843
844 trace_add_device_to_group(group->id, dev);
845
846 dev_info(dev, "Adding to iommu group %d\n", group->id);
847
848 return 0;
849
850err_put_group:
851 mutex_lock(&group->mutex);
852 list_del(&device->list);
853 mutex_unlock(&group->mutex);
854 dev->iommu_group = NULL;
855 kobject_put(group->devices_kobj);
856 sysfs_remove_link(group->devices_kobj, device->name);
857err_free_name:
858 kfree(device->name);
859err_remove_link:
860 sysfs_remove_link(&dev->kobj, "iommu_group");
861err_free_device:
862 kfree(device);
863 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);
864 return ret;
865}
866EXPORT_SYMBOL_GPL(iommu_group_add_device);
867
868/**
869 * iommu_group_remove_device - remove a device from it's current group
870 * @dev: device to be removed
871 *
872 * This function is called by an iommu driver to remove the device from
873 * it's current group. This decrements the iommu group reference count.
874 */
875void iommu_group_remove_device(struct device *dev)
876{
877 struct iommu_group *group = dev->iommu_group;
878 struct group_device *tmp_device, *device = NULL;
879
880 dev_info(dev, "Removing from iommu group %d\n", group->id);
881
882 /* Pre-notify listeners that a device is being removed. */
883 blocking_notifier_call_chain(&group->notifier,
884 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev);
885
886 mutex_lock(&group->mutex);
887 list_for_each_entry(tmp_device, &group->devices, list) {
888 if (tmp_device->dev == dev) {
889 device = tmp_device;
890 list_del(&device->list);
891 break;
892 }
893 }
894 mutex_unlock(&group->mutex);
895
896 if (!device)
897 return;
898
899 sysfs_remove_link(group->devices_kobj, device->name);
900 sysfs_remove_link(&dev->kobj, "iommu_group");
901
902 trace_remove_device_from_group(group->id, dev);
903
904 kfree(device->name);
905 kfree(device);
906 dev->iommu_group = NULL;
907 kobject_put(group->devices_kobj);
908}
909EXPORT_SYMBOL_GPL(iommu_group_remove_device);
910
911static int iommu_group_device_count(struct iommu_group *group)
912{
913 struct group_device *entry;
914 int ret = 0;
915
916 list_for_each_entry(entry, &group->devices, list)
917 ret++;
918
919 return ret;
920}
921
922/**
923 * iommu_group_for_each_dev - iterate over each device in the group
924 * @group: the group
925 * @data: caller opaque data to be passed to callback function
926 * @fn: caller supplied callback function
927 *
928 * This function is called by group users to iterate over group devices.
929 * Callers should hold a reference count to the group during callback.
930 * The group->mutex is held across callbacks, which will block calls to
931 * iommu_group_add/remove_device.
932 */
933static int __iommu_group_for_each_dev(struct iommu_group *group, void *data,
934 int (*fn)(struct device *, void *))
935{
936 struct group_device *device;
937 int ret = 0;
938
939 list_for_each_entry(device, &group->devices, list) {
940 ret = fn(device->dev, data);
941 if (ret)
942 break;
943 }
944 return ret;
945}
946
947
948int iommu_group_for_each_dev(struct iommu_group *group, void *data,
949 int (*fn)(struct device *, void *))
950{
951 int ret;
952
953 mutex_lock(&group->mutex);
954 ret = __iommu_group_for_each_dev(group, data, fn);
955 mutex_unlock(&group->mutex);
956
957 return ret;
958}
959EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
960
961/**
962 * iommu_group_get - Return the group for a device and increment reference
963 * @dev: get the group that this device belongs to
964 *
965 * This function is called by iommu drivers and users to get the group
966 * for the specified device. If found, the group is returned and the group
967 * reference in incremented, else NULL.
968 */
969struct iommu_group *iommu_group_get(struct device *dev)
970{
971 struct iommu_group *group = dev->iommu_group;
972
973 if (group)
974 kobject_get(group->devices_kobj);
975
976 return group;
977}
978EXPORT_SYMBOL_GPL(iommu_group_get);
979
980/**
981 * iommu_group_ref_get - Increment reference on a group
982 * @group: the group to use, must not be NULL
983 *
984 * This function is called by iommu drivers to take additional references on an
985 * existing group. Returns the given group for convenience.
986 */
987struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
988{
989 kobject_get(group->devices_kobj);
990 return group;
991}
992EXPORT_SYMBOL_GPL(iommu_group_ref_get);
993
994/**
995 * iommu_group_put - Decrement group reference
996 * @group: the group to use
997 *
998 * This function is called by iommu drivers and users to release the
999 * iommu group. Once the reference count is zero, the group is released.
1000 */
1001void iommu_group_put(struct iommu_group *group)
1002{
1003 if (group)
1004 kobject_put(group->devices_kobj);
1005}
1006EXPORT_SYMBOL_GPL(iommu_group_put);
1007
1008/**
1009 * iommu_group_register_notifier - Register a notifier for group changes
1010 * @group: the group to watch
1011 * @nb: notifier block to signal
1012 *
1013 * This function allows iommu group users to track changes in a group.
1014 * See include/linux/iommu.h for actions sent via this notifier. Caller
1015 * should hold a reference to the group throughout notifier registration.
1016 */
1017int iommu_group_register_notifier(struct iommu_group *group,
1018 struct notifier_block *nb)
1019{
1020 return blocking_notifier_chain_register(&group->notifier, nb);
1021}
1022EXPORT_SYMBOL_GPL(iommu_group_register_notifier);
1023
1024/**
1025 * iommu_group_unregister_notifier - Unregister a notifier
1026 * @group: the group to watch
1027 * @nb: notifier block to signal
1028 *
1029 * Unregister a previously registered group notifier block.
1030 */
1031int iommu_group_unregister_notifier(struct iommu_group *group,
1032 struct notifier_block *nb)
1033{
1034 return blocking_notifier_chain_unregister(&group->notifier, nb);
1035}
1036EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier);
1037
1038/**
1039 * iommu_register_device_fault_handler() - Register a device fault handler
1040 * @dev: the device
1041 * @handler: the fault handler
1042 * @data: private data passed as argument to the handler
1043 *
1044 * When an IOMMU fault event is received, this handler gets called with the
1045 * fault event and data as argument. The handler should return 0 on success. If
1046 * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also
1047 * complete the fault by calling iommu_page_response() with one of the following
1048 * response code:
1049 * - IOMMU_PAGE_RESP_SUCCESS: retry the translation
1050 * - IOMMU_PAGE_RESP_INVALID: terminate the fault
1051 * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting
1052 * page faults if possible.
1053 *
1054 * Return 0 if the fault handler was installed successfully, or an error.
1055 */
1056int iommu_register_device_fault_handler(struct device *dev,
1057 iommu_dev_fault_handler_t handler,
1058 void *data)
1059{
1060 struct dev_iommu *param = dev->iommu;
1061 int ret = 0;
1062
1063 if (!param)
1064 return -EINVAL;
1065
1066 mutex_lock(¶m->lock);
1067 /* Only allow one fault handler registered for each device */
1068 if (param->fault_param) {
1069 ret = -EBUSY;
1070 goto done_unlock;
1071 }
1072
1073 get_device(dev);
1074 param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL);
1075 if (!param->fault_param) {
1076 put_device(dev);
1077 ret = -ENOMEM;
1078 goto done_unlock;
1079 }
1080 param->fault_param->handler = handler;
1081 param->fault_param->data = data;
1082 mutex_init(¶m->fault_param->lock);
1083 INIT_LIST_HEAD(¶m->fault_param->faults);
1084
1085done_unlock:
1086 mutex_unlock(¶m->lock);
1087
1088 return ret;
1089}
1090EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler);
1091
1092/**
1093 * iommu_unregister_device_fault_handler() - Unregister the device fault handler
1094 * @dev: the device
1095 *
1096 * Remove the device fault handler installed with
1097 * iommu_register_device_fault_handler().
1098 *
1099 * Return 0 on success, or an error.
1100 */
1101int iommu_unregister_device_fault_handler(struct device *dev)
1102{
1103 struct dev_iommu *param = dev->iommu;
1104 int ret = 0;
1105
1106 if (!param)
1107 return -EINVAL;
1108
1109 mutex_lock(¶m->lock);
1110
1111 if (!param->fault_param)
1112 goto unlock;
1113
1114 /* we cannot unregister handler if there are pending faults */
1115 if (!list_empty(¶m->fault_param->faults)) {
1116 ret = -EBUSY;
1117 goto unlock;
1118 }
1119
1120 kfree(param->fault_param);
1121 param->fault_param = NULL;
1122 put_device(dev);
1123unlock:
1124 mutex_unlock(¶m->lock);
1125
1126 return ret;
1127}
1128EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler);
1129
1130/**
1131 * iommu_report_device_fault() - Report fault event to device driver
1132 * @dev: the device
1133 * @evt: fault event data
1134 *
1135 * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ
1136 * handler. When this function fails and the fault is recoverable, it is the
1137 * caller's responsibility to complete the fault.
1138 *
1139 * Return 0 on success, or an error.
1140 */
1141int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt)
1142{
1143 struct dev_iommu *param = dev->iommu;
1144 struct iommu_fault_event *evt_pending = NULL;
1145 struct iommu_fault_param *fparam;
1146 int ret = 0;
1147
1148 if (!param || !evt)
1149 return -EINVAL;
1150
1151 /* we only report device fault if there is a handler registered */
1152 mutex_lock(¶m->lock);
1153 fparam = param->fault_param;
1154 if (!fparam || !fparam->handler) {
1155 ret = -EINVAL;
1156 goto done_unlock;
1157 }
1158
1159 if (evt->fault.type == IOMMU_FAULT_PAGE_REQ &&
1160 (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) {
1161 evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event),
1162 GFP_KERNEL);
1163 if (!evt_pending) {
1164 ret = -ENOMEM;
1165 goto done_unlock;
1166 }
1167 mutex_lock(&fparam->lock);
1168 list_add_tail(&evt_pending->list, &fparam->faults);
1169 mutex_unlock(&fparam->lock);
1170 }
1171
1172 ret = fparam->handler(&evt->fault, fparam->data);
1173 if (ret && evt_pending) {
1174 mutex_lock(&fparam->lock);
1175 list_del(&evt_pending->list);
1176 mutex_unlock(&fparam->lock);
1177 kfree(evt_pending);
1178 }
1179done_unlock:
1180 mutex_unlock(¶m->lock);
1181 return ret;
1182}
1183EXPORT_SYMBOL_GPL(iommu_report_device_fault);
1184
1185int iommu_page_response(struct device *dev,
1186 struct iommu_page_response *msg)
1187{
1188 bool needs_pasid;
1189 int ret = -EINVAL;
1190 struct iommu_fault_event *evt;
1191 struct iommu_fault_page_request *prm;
1192 struct dev_iommu *param = dev->iommu;
1193 bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID;
1194 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1195
1196 if (!domain || !domain->ops->page_response)
1197 return -ENODEV;
1198
1199 if (!param || !param->fault_param)
1200 return -EINVAL;
1201
1202 if (msg->version != IOMMU_PAGE_RESP_VERSION_1 ||
1203 msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID)
1204 return -EINVAL;
1205
1206 /* Only send response if there is a fault report pending */
1207 mutex_lock(¶m->fault_param->lock);
1208 if (list_empty(¶m->fault_param->faults)) {
1209 dev_warn_ratelimited(dev, "no pending PRQ, drop response\n");
1210 goto done_unlock;
1211 }
1212 /*
1213 * Check if we have a matching page request pending to respond,
1214 * otherwise return -EINVAL
1215 */
1216 list_for_each_entry(evt, ¶m->fault_param->faults, list) {
1217 prm = &evt->fault.prm;
1218 if (prm->grpid != msg->grpid)
1219 continue;
1220
1221 /*
1222 * If the PASID is required, the corresponding request is
1223 * matched using the group ID, the PASID valid bit and the PASID
1224 * value. Otherwise only the group ID matches request and
1225 * response.
1226 */
1227 needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID;
1228 if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid))
1229 continue;
1230
1231 if (!needs_pasid && has_pasid) {
1232 /* No big deal, just clear it. */
1233 msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID;
1234 msg->pasid = 0;
1235 }
1236
1237 ret = domain->ops->page_response(dev, evt, msg);
1238 list_del(&evt->list);
1239 kfree(evt);
1240 break;
1241 }
1242
1243done_unlock:
1244 mutex_unlock(¶m->fault_param->lock);
1245 return ret;
1246}
1247EXPORT_SYMBOL_GPL(iommu_page_response);
1248
1249/**
1250 * iommu_group_id - Return ID for a group
1251 * @group: the group to ID
1252 *
1253 * Return the unique ID for the group matching the sysfs group number.
1254 */
1255int iommu_group_id(struct iommu_group *group)
1256{
1257 return group->id;
1258}
1259EXPORT_SYMBOL_GPL(iommu_group_id);
1260
1261static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1262 unsigned long *devfns);
1263
1264/*
1265 * To consider a PCI device isolated, we require ACS to support Source
1266 * Validation, Request Redirection, Completer Redirection, and Upstream
1267 * Forwarding. This effectively means that devices cannot spoof their
1268 * requester ID, requests and completions cannot be redirected, and all
1269 * transactions are forwarded upstream, even as it passes through a
1270 * bridge where the target device is downstream.
1271 */
1272#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
1273
1274/*
1275 * For multifunction devices which are not isolated from each other, find
1276 * all the other non-isolated functions and look for existing groups. For
1277 * each function, we also need to look for aliases to or from other devices
1278 * that may already have a group.
1279 */
1280static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
1281 unsigned long *devfns)
1282{
1283 struct pci_dev *tmp = NULL;
1284 struct iommu_group *group;
1285
1286 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
1287 return NULL;
1288
1289 for_each_pci_dev(tmp) {
1290 if (tmp == pdev || tmp->bus != pdev->bus ||
1291 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
1292 pci_acs_enabled(tmp, REQ_ACS_FLAGS))
1293 continue;
1294
1295 group = get_pci_alias_group(tmp, devfns);
1296 if (group) {
1297 pci_dev_put(tmp);
1298 return group;
1299 }
1300 }
1301
1302 return NULL;
1303}
1304
1305/*
1306 * Look for aliases to or from the given device for existing groups. DMA
1307 * aliases are only supported on the same bus, therefore the search
1308 * space is quite small (especially since we're really only looking at pcie
1309 * device, and therefore only expect multiple slots on the root complex or
1310 * downstream switch ports). It's conceivable though that a pair of
1311 * multifunction devices could have aliases between them that would cause a
1312 * loop. To prevent this, we use a bitmap to track where we've been.
1313 */
1314static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1315 unsigned long *devfns)
1316{
1317 struct pci_dev *tmp = NULL;
1318 struct iommu_group *group;
1319
1320 if (test_and_set_bit(pdev->devfn & 0xff, devfns))
1321 return NULL;
1322
1323 group = iommu_group_get(&pdev->dev);
1324 if (group)
1325 return group;
1326
1327 for_each_pci_dev(tmp) {
1328 if (tmp == pdev || tmp->bus != pdev->bus)
1329 continue;
1330
1331 /* We alias them or they alias us */
1332 if (pci_devs_are_dma_aliases(pdev, tmp)) {
1333 group = get_pci_alias_group(tmp, devfns);
1334 if (group) {
1335 pci_dev_put(tmp);
1336 return group;
1337 }
1338
1339 group = get_pci_function_alias_group(tmp, devfns);
1340 if (group) {
1341 pci_dev_put(tmp);
1342 return group;
1343 }
1344 }
1345 }
1346
1347 return NULL;
1348}
1349
1350struct group_for_pci_data {
1351 struct pci_dev *pdev;
1352 struct iommu_group *group;
1353};
1354
1355/*
1356 * DMA alias iterator callback, return the last seen device. Stop and return
1357 * the IOMMU group if we find one along the way.
1358 */
1359static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
1360{
1361 struct group_for_pci_data *data = opaque;
1362
1363 data->pdev = pdev;
1364 data->group = iommu_group_get(&pdev->dev);
1365
1366 return data->group != NULL;
1367}
1368
1369/*
1370 * Generic device_group call-back function. It just allocates one
1371 * iommu-group per device.
1372 */
1373struct iommu_group *generic_device_group(struct device *dev)
1374{
1375 return iommu_group_alloc();
1376}
1377EXPORT_SYMBOL_GPL(generic_device_group);
1378
1379/*
1380 * Use standard PCI bus topology, isolation features, and DMA alias quirks
1381 * to find or create an IOMMU group for a device.
1382 */
1383struct iommu_group *pci_device_group(struct device *dev)
1384{
1385 struct pci_dev *pdev = to_pci_dev(dev);
1386 struct group_for_pci_data data;
1387 struct pci_bus *bus;
1388 struct iommu_group *group = NULL;
1389 u64 devfns[4] = { 0 };
1390
1391 if (WARN_ON(!dev_is_pci(dev)))
1392 return ERR_PTR(-EINVAL);
1393
1394 /*
1395 * Find the upstream DMA alias for the device. A device must not
1396 * be aliased due to topology in order to have its own IOMMU group.
1397 * If we find an alias along the way that already belongs to a
1398 * group, use it.
1399 */
1400 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))
1401 return data.group;
1402
1403 pdev = data.pdev;
1404
1405 /*
1406 * Continue upstream from the point of minimum IOMMU granularity
1407 * due to aliases to the point where devices are protected from
1408 * peer-to-peer DMA by PCI ACS. Again, if we find an existing
1409 * group, use it.
1410 */
1411 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {
1412 if (!bus->self)
1413 continue;
1414
1415 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
1416 break;
1417
1418 pdev = bus->self;
1419
1420 group = iommu_group_get(&pdev->dev);
1421 if (group)
1422 return group;
1423 }
1424
1425 /*
1426 * Look for existing groups on device aliases. If we alias another
1427 * device or another device aliases us, use the same group.
1428 */
1429 group = get_pci_alias_group(pdev, (unsigned long *)devfns);
1430 if (group)
1431 return group;
1432
1433 /*
1434 * Look for existing groups on non-isolated functions on the same
1435 * slot and aliases of those funcions, if any. No need to clear
1436 * the search bitmap, the tested devfns are still valid.
1437 */
1438 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
1439 if (group)
1440 return group;
1441
1442 /* No shared group found, allocate new */
1443 return iommu_group_alloc();
1444}
1445EXPORT_SYMBOL_GPL(pci_device_group);
1446
1447/* Get the IOMMU group for device on fsl-mc bus */
1448struct iommu_group *fsl_mc_device_group(struct device *dev)
1449{
1450 struct device *cont_dev = fsl_mc_cont_dev(dev);
1451 struct iommu_group *group;
1452
1453 group = iommu_group_get(cont_dev);
1454 if (!group)
1455 group = iommu_group_alloc();
1456 return group;
1457}
1458EXPORT_SYMBOL_GPL(fsl_mc_device_group);
1459
1460static int iommu_get_def_domain_type(struct device *dev)
1461{
1462 const struct iommu_ops *ops = dev->bus->iommu_ops;
1463 unsigned int type = 0;
1464
1465 if (ops->def_domain_type)
1466 type = ops->def_domain_type(dev);
1467
1468 return (type == 0) ? iommu_def_domain_type : type;
1469}
1470
1471static int iommu_group_alloc_default_domain(struct bus_type *bus,
1472 struct iommu_group *group,
1473 unsigned int type)
1474{
1475 struct iommu_domain *dom;
1476
1477 dom = __iommu_domain_alloc(bus, type);
1478 if (!dom && type != IOMMU_DOMAIN_DMA) {
1479 dom = __iommu_domain_alloc(bus, IOMMU_DOMAIN_DMA);
1480 if (dom)
1481 pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA",
1482 type, group->name);
1483 }
1484
1485 if (!dom)
1486 return -ENOMEM;
1487
1488 group->default_domain = dom;
1489 if (!group->domain)
1490 group->domain = dom;
1491
1492 if (!iommu_dma_strict) {
1493 int attr = 1;
1494 iommu_domain_set_attr(dom,
1495 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
1496 &attr);
1497 }
1498
1499 return 0;
1500}
1501
1502static int iommu_alloc_default_domain(struct iommu_group *group,
1503 struct device *dev)
1504{
1505 unsigned int type;
1506
1507 if (group->default_domain)
1508 return 0;
1509
1510 type = iommu_get_def_domain_type(dev);
1511
1512 return iommu_group_alloc_default_domain(dev->bus, group, type);
1513}
1514
1515/**
1516 * iommu_group_get_for_dev - Find or create the IOMMU group for a device
1517 * @dev: target device
1518 *
1519 * This function is intended to be called by IOMMU drivers and extended to
1520 * support common, bus-defined algorithms when determining or creating the
1521 * IOMMU group for a device. On success, the caller will hold a reference
1522 * to the returned IOMMU group, which will already include the provided
1523 * device. The reference should be released with iommu_group_put().
1524 */
1525static struct iommu_group *iommu_group_get_for_dev(struct device *dev)
1526{
1527 const struct iommu_ops *ops = dev->bus->iommu_ops;
1528 struct iommu_group *group;
1529 int ret;
1530
1531 group = iommu_group_get(dev);
1532 if (group)
1533 return group;
1534
1535 if (!ops)
1536 return ERR_PTR(-EINVAL);
1537
1538 group = ops->device_group(dev);
1539 if (WARN_ON_ONCE(group == NULL))
1540 return ERR_PTR(-EINVAL);
1541
1542 if (IS_ERR(group))
1543 return group;
1544
1545 ret = iommu_group_add_device(group, dev);
1546 if (ret)
1547 goto out_put_group;
1548
1549 return group;
1550
1551out_put_group:
1552 iommu_group_put(group);
1553
1554 return ERR_PTR(ret);
1555}
1556
1557struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
1558{
1559 return group->default_domain;
1560}
1561
1562static int probe_iommu_group(struct device *dev, void *data)
1563{
1564 struct list_head *group_list = data;
1565 struct iommu_group *group;
1566 int ret;
1567
1568 /* Device is probed already if in a group */
1569 group = iommu_group_get(dev);
1570 if (group) {
1571 iommu_group_put(group);
1572 return 0;
1573 }
1574
1575 ret = __iommu_probe_device(dev, group_list);
1576 if (ret == -ENODEV)
1577 ret = 0;
1578
1579 return ret;
1580}
1581
1582static int remove_iommu_group(struct device *dev, void *data)
1583{
1584 iommu_release_device(dev);
1585
1586 return 0;
1587}
1588
1589static int iommu_bus_notifier(struct notifier_block *nb,
1590 unsigned long action, void *data)
1591{
1592 unsigned long group_action = 0;
1593 struct device *dev = data;
1594 struct iommu_group *group;
1595
1596 /*
1597 * ADD/DEL call into iommu driver ops if provided, which may
1598 * result in ADD/DEL notifiers to group->notifier
1599 */
1600 if (action == BUS_NOTIFY_ADD_DEVICE) {
1601 int ret;
1602
1603 ret = iommu_probe_device(dev);
1604 return (ret) ? NOTIFY_DONE : NOTIFY_OK;
1605 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
1606 iommu_release_device(dev);
1607 return NOTIFY_OK;
1608 }
1609
1610 /*
1611 * Remaining BUS_NOTIFYs get filtered and republished to the
1612 * group, if anyone is listening
1613 */
1614 group = iommu_group_get(dev);
1615 if (!group)
1616 return 0;
1617
1618 switch (action) {
1619 case BUS_NOTIFY_BIND_DRIVER:
1620 group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER;
1621 break;
1622 case BUS_NOTIFY_BOUND_DRIVER:
1623 group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER;
1624 break;
1625 case BUS_NOTIFY_UNBIND_DRIVER:
1626 group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER;
1627 break;
1628 case BUS_NOTIFY_UNBOUND_DRIVER:
1629 group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER;
1630 break;
1631 }
1632
1633 if (group_action)
1634 blocking_notifier_call_chain(&group->notifier,
1635 group_action, dev);
1636
1637 iommu_group_put(group);
1638 return 0;
1639}
1640
1641struct __group_domain_type {
1642 struct device *dev;
1643 unsigned int type;
1644};
1645
1646static int probe_get_default_domain_type(struct device *dev, void *data)
1647{
1648 const struct iommu_ops *ops = dev->bus->iommu_ops;
1649 struct __group_domain_type *gtype = data;
1650 unsigned int type = 0;
1651
1652 if (ops->def_domain_type)
1653 type = ops->def_domain_type(dev);
1654
1655 if (type) {
1656 if (gtype->type && gtype->type != type) {
1657 dev_warn(dev, "Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n",
1658 iommu_domain_type_str(type),
1659 dev_name(gtype->dev),
1660 iommu_domain_type_str(gtype->type));
1661 gtype->type = 0;
1662 }
1663
1664 if (!gtype->dev) {
1665 gtype->dev = dev;
1666 gtype->type = type;
1667 }
1668 }
1669
1670 return 0;
1671}
1672
1673static void probe_alloc_default_domain(struct bus_type *bus,
1674 struct iommu_group *group)
1675{
1676 struct __group_domain_type gtype;
1677
1678 memset(>ype, 0, sizeof(gtype));
1679
1680 /* Ask for default domain requirements of all devices in the group */
1681 __iommu_group_for_each_dev(group, >ype,
1682 probe_get_default_domain_type);
1683
1684 if (!gtype.type)
1685 gtype.type = iommu_def_domain_type;
1686
1687 iommu_group_alloc_default_domain(bus, group, gtype.type);
1688
1689}
1690
1691static int iommu_group_do_dma_attach(struct device *dev, void *data)
1692{
1693 struct iommu_domain *domain = data;
1694 int ret = 0;
1695
1696 if (!iommu_is_attach_deferred(domain, dev))
1697 ret = __iommu_attach_device(domain, dev);
1698
1699 return ret;
1700}
1701
1702static int __iommu_group_dma_attach(struct iommu_group *group)
1703{
1704 return __iommu_group_for_each_dev(group, group->default_domain,
1705 iommu_group_do_dma_attach);
1706}
1707
1708static int iommu_group_do_probe_finalize(struct device *dev, void *data)
1709{
1710 struct iommu_domain *domain = data;
1711
1712 if (domain->ops->probe_finalize)
1713 domain->ops->probe_finalize(dev);
1714
1715 return 0;
1716}
1717
1718static void __iommu_group_dma_finalize(struct iommu_group *group)
1719{
1720 __iommu_group_for_each_dev(group, group->default_domain,
1721 iommu_group_do_probe_finalize);
1722}
1723
1724static int iommu_do_create_direct_mappings(struct device *dev, void *data)
1725{
1726 struct iommu_group *group = data;
1727
1728 iommu_create_device_direct_mappings(group, dev);
1729
1730 return 0;
1731}
1732
1733static int iommu_group_create_direct_mappings(struct iommu_group *group)
1734{
1735 return __iommu_group_for_each_dev(group, group,
1736 iommu_do_create_direct_mappings);
1737}
1738
1739int bus_iommu_probe(struct bus_type *bus)
1740{
1741 struct iommu_group *group, *next;
1742 LIST_HEAD(group_list);
1743 int ret;
1744
1745 /*
1746 * This code-path does not allocate the default domain when
1747 * creating the iommu group, so do it after the groups are
1748 * created.
1749 */
1750 ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group);
1751 if (ret)
1752 return ret;
1753
1754 list_for_each_entry_safe(group, next, &group_list, entry) {
1755 /* Remove item from the list */
1756 list_del_init(&group->entry);
1757
1758 mutex_lock(&group->mutex);
1759
1760 /* Try to allocate default domain */
1761 probe_alloc_default_domain(bus, group);
1762
1763 if (!group->default_domain) {
1764 mutex_unlock(&group->mutex);
1765 continue;
1766 }
1767
1768 iommu_group_create_direct_mappings(group);
1769
1770 ret = __iommu_group_dma_attach(group);
1771
1772 mutex_unlock(&group->mutex);
1773
1774 if (ret)
1775 break;
1776
1777 __iommu_group_dma_finalize(group);
1778 }
1779
1780 return ret;
1781}
1782
1783static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
1784{
1785 struct notifier_block *nb;
1786 int err;
1787
1788 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
1789 if (!nb)
1790 return -ENOMEM;
1791
1792 nb->notifier_call = iommu_bus_notifier;
1793
1794 err = bus_register_notifier(bus, nb);
1795 if (err)
1796 goto out_free;
1797
1798 err = bus_iommu_probe(bus);
1799 if (err)
1800 goto out_err;
1801
1802
1803 return 0;
1804
1805out_err:
1806 /* Clean up */
1807 bus_for_each_dev(bus, NULL, NULL, remove_iommu_group);
1808 bus_unregister_notifier(bus, nb);
1809
1810out_free:
1811 kfree(nb);
1812
1813 return err;
1814}
1815
1816/**
1817 * bus_set_iommu - set iommu-callbacks for the bus
1818 * @bus: bus.
1819 * @ops: the callbacks provided by the iommu-driver
1820 *
1821 * This function is called by an iommu driver to set the iommu methods
1822 * used for a particular bus. Drivers for devices on that bus can use
1823 * the iommu-api after these ops are registered.
1824 * This special function is needed because IOMMUs are usually devices on
1825 * the bus itself, so the iommu drivers are not initialized when the bus
1826 * is set up. With this function the iommu-driver can set the iommu-ops
1827 * afterwards.
1828 */
1829int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops)
1830{
1831 int err;
1832
1833 if (ops == NULL) {
1834 bus->iommu_ops = NULL;
1835 return 0;
1836 }
1837
1838 if (bus->iommu_ops != NULL)
1839 return -EBUSY;
1840
1841 bus->iommu_ops = ops;
1842
1843 /* Do IOMMU specific setup for this bus-type */
1844 err = iommu_bus_init(bus, ops);
1845 if (err)
1846 bus->iommu_ops = NULL;
1847
1848 return err;
1849}
1850EXPORT_SYMBOL_GPL(bus_set_iommu);
1851
1852bool iommu_present(struct bus_type *bus)
1853{
1854 return bus->iommu_ops != NULL;
1855}
1856EXPORT_SYMBOL_GPL(iommu_present);
1857
1858bool iommu_capable(struct bus_type *bus, enum iommu_cap cap)
1859{
1860 if (!bus->iommu_ops || !bus->iommu_ops->capable)
1861 return false;
1862
1863 return bus->iommu_ops->capable(cap);
1864}
1865EXPORT_SYMBOL_GPL(iommu_capable);
1866
1867/**
1868 * iommu_set_fault_handler() - set a fault handler for an iommu domain
1869 * @domain: iommu domain
1870 * @handler: fault handler
1871 * @token: user data, will be passed back to the fault handler
1872 *
1873 * This function should be used by IOMMU users which want to be notified
1874 * whenever an IOMMU fault happens.
1875 *
1876 * The fault handler itself should return 0 on success, and an appropriate
1877 * error code otherwise.
1878 */
1879void iommu_set_fault_handler(struct iommu_domain *domain,
1880 iommu_fault_handler_t handler,
1881 void *token)
1882{
1883 BUG_ON(!domain);
1884
1885 domain->handler = handler;
1886 domain->handler_token = token;
1887}
1888EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
1889
1890static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
1891 unsigned type)
1892{
1893 struct iommu_domain *domain;
1894
1895 if (bus == NULL || bus->iommu_ops == NULL)
1896 return NULL;
1897
1898 domain = bus->iommu_ops->domain_alloc(type);
1899 if (!domain)
1900 return NULL;
1901
1902 domain->ops = bus->iommu_ops;
1903 domain->type = type;
1904 /* Assume all sizes by default; the driver may override this later */
1905 domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
1906
1907 return domain;
1908}
1909
1910struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
1911{
1912 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED);
1913}
1914EXPORT_SYMBOL_GPL(iommu_domain_alloc);
1915
1916void iommu_domain_free(struct iommu_domain *domain)
1917{
1918 domain->ops->domain_free(domain);
1919}
1920EXPORT_SYMBOL_GPL(iommu_domain_free);
1921
1922static int __iommu_attach_device(struct iommu_domain *domain,
1923 struct device *dev)
1924{
1925 int ret;
1926
1927 if (unlikely(domain->ops->attach_dev == NULL))
1928 return -ENODEV;
1929
1930 ret = domain->ops->attach_dev(domain, dev);
1931 if (!ret)
1932 trace_attach_device_to_domain(dev);
1933 return ret;
1934}
1935
1936int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
1937{
1938 struct iommu_group *group;
1939 int ret;
1940
1941 group = iommu_group_get(dev);
1942 if (!group)
1943 return -ENODEV;
1944
1945 /*
1946 * Lock the group to make sure the device-count doesn't
1947 * change while we are attaching
1948 */
1949 mutex_lock(&group->mutex);
1950 ret = -EINVAL;
1951 if (iommu_group_device_count(group) != 1)
1952 goto out_unlock;
1953
1954 ret = __iommu_attach_group(domain, group);
1955
1956out_unlock:
1957 mutex_unlock(&group->mutex);
1958 iommu_group_put(group);
1959
1960 return ret;
1961}
1962EXPORT_SYMBOL_GPL(iommu_attach_device);
1963
1964int iommu_cache_invalidate(struct iommu_domain *domain, struct device *dev,
1965 struct iommu_cache_invalidate_info *inv_info)
1966{
1967 if (unlikely(!domain->ops->cache_invalidate))
1968 return -ENODEV;
1969
1970 return domain->ops->cache_invalidate(domain, dev, inv_info);
1971}
1972EXPORT_SYMBOL_GPL(iommu_cache_invalidate);
1973
1974int iommu_sva_bind_gpasid(struct iommu_domain *domain,
1975 struct device *dev, struct iommu_gpasid_bind_data *data)
1976{
1977 if (unlikely(!domain->ops->sva_bind_gpasid))
1978 return -ENODEV;
1979
1980 return domain->ops->sva_bind_gpasid(domain, dev, data);
1981}
1982EXPORT_SYMBOL_GPL(iommu_sva_bind_gpasid);
1983
1984int iommu_sva_unbind_gpasid(struct iommu_domain *domain, struct device *dev,
1985 ioasid_t pasid)
1986{
1987 if (unlikely(!domain->ops->sva_unbind_gpasid))
1988 return -ENODEV;
1989
1990 return domain->ops->sva_unbind_gpasid(dev, pasid);
1991}
1992EXPORT_SYMBOL_GPL(iommu_sva_unbind_gpasid);
1993
1994static void __iommu_detach_device(struct iommu_domain *domain,
1995 struct device *dev)
1996{
1997 if (iommu_is_attach_deferred(domain, dev))
1998 return;
1999
2000 if (unlikely(domain->ops->detach_dev == NULL))
2001 return;
2002
2003 domain->ops->detach_dev(domain, dev);
2004 trace_detach_device_from_domain(dev);
2005}
2006
2007void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
2008{
2009 struct iommu_group *group;
2010
2011 group = iommu_group_get(dev);
2012 if (!group)
2013 return;
2014
2015 mutex_lock(&group->mutex);
2016 if (iommu_group_device_count(group) != 1) {
2017 WARN_ON(1);
2018 goto out_unlock;
2019 }
2020
2021 __iommu_detach_group(domain, group);
2022
2023out_unlock:
2024 mutex_unlock(&group->mutex);
2025 iommu_group_put(group);
2026}
2027EXPORT_SYMBOL_GPL(iommu_detach_device);
2028
2029struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
2030{
2031 struct iommu_domain *domain;
2032 struct iommu_group *group;
2033
2034 group = iommu_group_get(dev);
2035 if (!group)
2036 return NULL;
2037
2038 domain = group->domain;
2039
2040 iommu_group_put(group);
2041
2042 return domain;
2043}
2044EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
2045
2046/*
2047 * For IOMMU_DOMAIN_DMA implementations which already provide their own
2048 * guarantees that the group and its default domain are valid and correct.
2049 */
2050struct iommu_domain *iommu_get_dma_domain(struct device *dev)
2051{
2052 return dev->iommu_group->default_domain;
2053}
2054
2055/*
2056 * IOMMU groups are really the natural working unit of the IOMMU, but
2057 * the IOMMU API works on domains and devices. Bridge that gap by
2058 * iterating over the devices in a group. Ideally we'd have a single
2059 * device which represents the requestor ID of the group, but we also
2060 * allow IOMMU drivers to create policy defined minimum sets, where
2061 * the physical hardware may be able to distiguish members, but we
2062 * wish to group them at a higher level (ex. untrusted multi-function
2063 * PCI devices). Thus we attach each device.
2064 */
2065static int iommu_group_do_attach_device(struct device *dev, void *data)
2066{
2067 struct iommu_domain *domain = data;
2068
2069 return __iommu_attach_device(domain, dev);
2070}
2071
2072static int __iommu_attach_group(struct iommu_domain *domain,
2073 struct iommu_group *group)
2074{
2075 int ret;
2076
2077 if (group->default_domain && group->domain != group->default_domain)
2078 return -EBUSY;
2079
2080 ret = __iommu_group_for_each_dev(group, domain,
2081 iommu_group_do_attach_device);
2082 if (ret == 0)
2083 group->domain = domain;
2084
2085 return ret;
2086}
2087
2088int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
2089{
2090 int ret;
2091
2092 mutex_lock(&group->mutex);
2093 ret = __iommu_attach_group(domain, group);
2094 mutex_unlock(&group->mutex);
2095
2096 return ret;
2097}
2098EXPORT_SYMBOL_GPL(iommu_attach_group);
2099
2100static int iommu_group_do_detach_device(struct device *dev, void *data)
2101{
2102 struct iommu_domain *domain = data;
2103
2104 __iommu_detach_device(domain, dev);
2105
2106 return 0;
2107}
2108
2109static void __iommu_detach_group(struct iommu_domain *domain,
2110 struct iommu_group *group)
2111{
2112 int ret;
2113
2114 if (!group->default_domain) {
2115 __iommu_group_for_each_dev(group, domain,
2116 iommu_group_do_detach_device);
2117 group->domain = NULL;
2118 return;
2119 }
2120
2121 if (group->domain == group->default_domain)
2122 return;
2123
2124 /* Detach by re-attaching to the default domain */
2125 ret = __iommu_group_for_each_dev(group, group->default_domain,
2126 iommu_group_do_attach_device);
2127 if (ret != 0)
2128 WARN_ON(1);
2129 else
2130 group->domain = group->default_domain;
2131}
2132
2133void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
2134{
2135 mutex_lock(&group->mutex);
2136 __iommu_detach_group(domain, group);
2137 mutex_unlock(&group->mutex);
2138}
2139EXPORT_SYMBOL_GPL(iommu_detach_group);
2140
2141phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2142{
2143 if (unlikely(domain->ops->iova_to_phys == NULL))
2144 return 0;
2145
2146 return domain->ops->iova_to_phys(domain, iova);
2147}
2148EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
2149
2150static size_t iommu_pgsize(struct iommu_domain *domain,
2151 unsigned long addr_merge, size_t size)
2152{
2153 unsigned int pgsize_idx;
2154 size_t pgsize;
2155
2156 /* Max page size that still fits into 'size' */
2157 pgsize_idx = __fls(size);
2158
2159 /* need to consider alignment requirements ? */
2160 if (likely(addr_merge)) {
2161 /* Max page size allowed by address */
2162 unsigned int align_pgsize_idx = __ffs(addr_merge);
2163 pgsize_idx = min(pgsize_idx, align_pgsize_idx);
2164 }
2165
2166 /* build a mask of acceptable page sizes */
2167 pgsize = (1UL << (pgsize_idx + 1)) - 1;
2168
2169 /* throw away page sizes not supported by the hardware */
2170 pgsize &= domain->pgsize_bitmap;
2171
2172 /* make sure we're still sane */
2173 BUG_ON(!pgsize);
2174
2175 /* pick the biggest page */
2176 pgsize_idx = __fls(pgsize);
2177 pgsize = 1UL << pgsize_idx;
2178
2179 return pgsize;
2180}
2181
2182static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
2183 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2184{
2185 const struct iommu_ops *ops = domain->ops;
2186 unsigned long orig_iova = iova;
2187 unsigned int min_pagesz;
2188 size_t orig_size = size;
2189 phys_addr_t orig_paddr = paddr;
2190 int ret = 0;
2191
2192 if (unlikely(ops->map == NULL ||
2193 domain->pgsize_bitmap == 0UL))
2194 return -ENODEV;
2195
2196 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2197 return -EINVAL;
2198
2199 /* find out the minimum page size supported */
2200 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2201
2202 /*
2203 * both the virtual address and the physical one, as well as
2204 * the size of the mapping, must be aligned (at least) to the
2205 * size of the smallest page supported by the hardware
2206 */
2207 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
2208 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
2209 iova, &paddr, size, min_pagesz);
2210 return -EINVAL;
2211 }
2212
2213 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
2214
2215 while (size) {
2216 size_t pgsize = iommu_pgsize(domain, iova | paddr, size);
2217
2218 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
2219 iova, &paddr, pgsize);
2220 ret = ops->map(domain, iova, paddr, pgsize, prot, gfp);
2221
2222 if (ret)
2223 break;
2224
2225 iova += pgsize;
2226 paddr += pgsize;
2227 size -= pgsize;
2228 }
2229
2230 if (ops->iotlb_sync_map)
2231 ops->iotlb_sync_map(domain);
2232
2233 /* unroll mapping in case something went wrong */
2234 if (ret)
2235 iommu_unmap(domain, orig_iova, orig_size - size);
2236 else
2237 trace_map(orig_iova, orig_paddr, orig_size);
2238
2239 return ret;
2240}
2241
2242int iommu_map(struct iommu_domain *domain, unsigned long iova,
2243 phys_addr_t paddr, size_t size, int prot)
2244{
2245 might_sleep();
2246 return __iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL);
2247}
2248EXPORT_SYMBOL_GPL(iommu_map);
2249
2250int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
2251 phys_addr_t paddr, size_t size, int prot)
2252{
2253 return __iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC);
2254}
2255EXPORT_SYMBOL_GPL(iommu_map_atomic);
2256
2257static size_t __iommu_unmap(struct iommu_domain *domain,
2258 unsigned long iova, size_t size,
2259 struct iommu_iotlb_gather *iotlb_gather)
2260{
2261 const struct iommu_ops *ops = domain->ops;
2262 size_t unmapped_page, unmapped = 0;
2263 unsigned long orig_iova = iova;
2264 unsigned int min_pagesz;
2265
2266 if (unlikely(ops->unmap == NULL ||
2267 domain->pgsize_bitmap == 0UL))
2268 return 0;
2269
2270 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2271 return 0;
2272
2273 /* find out the minimum page size supported */
2274 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2275
2276 /*
2277 * The virtual address, as well as the size of the mapping, must be
2278 * aligned (at least) to the size of the smallest page supported
2279 * by the hardware
2280 */
2281 if (!IS_ALIGNED(iova | size, min_pagesz)) {
2282 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
2283 iova, size, min_pagesz);
2284 return 0;
2285 }
2286
2287 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
2288
2289 /*
2290 * Keep iterating until we either unmap 'size' bytes (or more)
2291 * or we hit an area that isn't mapped.
2292 */
2293 while (unmapped < size) {
2294 size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
2295
2296 unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather);
2297 if (!unmapped_page)
2298 break;
2299
2300 pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
2301 iova, unmapped_page);
2302
2303 iova += unmapped_page;
2304 unmapped += unmapped_page;
2305 }
2306
2307 trace_unmap(orig_iova, size, unmapped);
2308 return unmapped;
2309}
2310
2311size_t iommu_unmap(struct iommu_domain *domain,
2312 unsigned long iova, size_t size)
2313{
2314 struct iommu_iotlb_gather iotlb_gather;
2315 size_t ret;
2316
2317 iommu_iotlb_gather_init(&iotlb_gather);
2318 ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
2319 iommu_tlb_sync(domain, &iotlb_gather);
2320
2321 return ret;
2322}
2323EXPORT_SYMBOL_GPL(iommu_unmap);
2324
2325size_t iommu_unmap_fast(struct iommu_domain *domain,
2326 unsigned long iova, size_t size,
2327 struct iommu_iotlb_gather *iotlb_gather)
2328{
2329 return __iommu_unmap(domain, iova, size, iotlb_gather);
2330}
2331EXPORT_SYMBOL_GPL(iommu_unmap_fast);
2332
2333static size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
2334 struct scatterlist *sg, unsigned int nents, int prot,
2335 gfp_t gfp)
2336{
2337 size_t len = 0, mapped = 0;
2338 phys_addr_t start;
2339 unsigned int i = 0;
2340 int ret;
2341
2342 while (i <= nents) {
2343 phys_addr_t s_phys = sg_phys(sg);
2344
2345 if (len && s_phys != start + len) {
2346 ret = __iommu_map(domain, iova + mapped, start,
2347 len, prot, gfp);
2348
2349 if (ret)
2350 goto out_err;
2351
2352 mapped += len;
2353 len = 0;
2354 }
2355
2356 if (len) {
2357 len += sg->length;
2358 } else {
2359 len = sg->length;
2360 start = s_phys;
2361 }
2362
2363 if (++i < nents)
2364 sg = sg_next(sg);
2365 }
2366
2367 return mapped;
2368
2369out_err:
2370 /* undo mappings already done */
2371 iommu_unmap(domain, iova, mapped);
2372
2373 return 0;
2374
2375}
2376
2377size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
2378 struct scatterlist *sg, unsigned int nents, int prot)
2379{
2380 might_sleep();
2381 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL);
2382}
2383EXPORT_SYMBOL_GPL(iommu_map_sg);
2384
2385size_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova,
2386 struct scatterlist *sg, unsigned int nents, int prot)
2387{
2388 return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC);
2389}
2390EXPORT_SYMBOL_GPL(iommu_map_sg_atomic);
2391
2392int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
2393 phys_addr_t paddr, u64 size, int prot)
2394{
2395 if (unlikely(domain->ops->domain_window_enable == NULL))
2396 return -ENODEV;
2397
2398 return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size,
2399 prot);
2400}
2401EXPORT_SYMBOL_GPL(iommu_domain_window_enable);
2402
2403void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr)
2404{
2405 if (unlikely(domain->ops->domain_window_disable == NULL))
2406 return;
2407
2408 return domain->ops->domain_window_disable(domain, wnd_nr);
2409}
2410EXPORT_SYMBOL_GPL(iommu_domain_window_disable);
2411
2412/**
2413 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
2414 * @domain: the iommu domain where the fault has happened
2415 * @dev: the device where the fault has happened
2416 * @iova: the faulting address
2417 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...)
2418 *
2419 * This function should be called by the low-level IOMMU implementations
2420 * whenever IOMMU faults happen, to allow high-level users, that are
2421 * interested in such events, to know about them.
2422 *
2423 * This event may be useful for several possible use cases:
2424 * - mere logging of the event
2425 * - dynamic TLB/PTE loading
2426 * - if restarting of the faulting device is required
2427 *
2428 * Returns 0 on success and an appropriate error code otherwise (if dynamic
2429 * PTE/TLB loading will one day be supported, implementations will be able
2430 * to tell whether it succeeded or not according to this return value).
2431 *
2432 * Specifically, -ENOSYS is returned if a fault handler isn't installed
2433 * (though fault handlers can also return -ENOSYS, in case they want to
2434 * elicit the default behavior of the IOMMU drivers).
2435 */
2436int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
2437 unsigned long iova, int flags)
2438{
2439 int ret = -ENOSYS;
2440
2441 /*
2442 * if upper layers showed interest and installed a fault handler,
2443 * invoke it.
2444 */
2445 if (domain->handler)
2446 ret = domain->handler(domain, dev, iova, flags,
2447 domain->handler_token);
2448
2449 trace_io_page_fault(dev, iova, flags);
2450 return ret;
2451}
2452EXPORT_SYMBOL_GPL(report_iommu_fault);
2453
2454static int __init iommu_init(void)
2455{
2456 iommu_group_kset = kset_create_and_add("iommu_groups",
2457 NULL, kernel_kobj);
2458 BUG_ON(!iommu_group_kset);
2459
2460 iommu_debugfs_setup();
2461
2462 return 0;
2463}
2464core_initcall(iommu_init);
2465
2466int iommu_domain_get_attr(struct iommu_domain *domain,
2467 enum iommu_attr attr, void *data)
2468{
2469 struct iommu_domain_geometry *geometry;
2470 bool *paging;
2471 int ret = 0;
2472
2473 switch (attr) {
2474 case DOMAIN_ATTR_GEOMETRY:
2475 geometry = data;
2476 *geometry = domain->geometry;
2477
2478 break;
2479 case DOMAIN_ATTR_PAGING:
2480 paging = data;
2481 *paging = (domain->pgsize_bitmap != 0UL);
2482 break;
2483 default:
2484 if (!domain->ops->domain_get_attr)
2485 return -EINVAL;
2486
2487 ret = domain->ops->domain_get_attr(domain, attr, data);
2488 }
2489
2490 return ret;
2491}
2492EXPORT_SYMBOL_GPL(iommu_domain_get_attr);
2493
2494int iommu_domain_set_attr(struct iommu_domain *domain,
2495 enum iommu_attr attr, void *data)
2496{
2497 int ret = 0;
2498
2499 switch (attr) {
2500 default:
2501 if (domain->ops->domain_set_attr == NULL)
2502 return -EINVAL;
2503
2504 ret = domain->ops->domain_set_attr(domain, attr, data);
2505 }
2506
2507 return ret;
2508}
2509EXPORT_SYMBOL_GPL(iommu_domain_set_attr);
2510
2511void iommu_get_resv_regions(struct device *dev, struct list_head *list)
2512{
2513 const struct iommu_ops *ops = dev->bus->iommu_ops;
2514
2515 if (ops && ops->get_resv_regions)
2516 ops->get_resv_regions(dev, list);
2517}
2518
2519void iommu_put_resv_regions(struct device *dev, struct list_head *list)
2520{
2521 const struct iommu_ops *ops = dev->bus->iommu_ops;
2522
2523 if (ops && ops->put_resv_regions)
2524 ops->put_resv_regions(dev, list);
2525}
2526
2527/**
2528 * generic_iommu_put_resv_regions - Reserved region driver helper
2529 * @dev: device for which to free reserved regions
2530 * @list: reserved region list for device
2531 *
2532 * IOMMU drivers can use this to implement their .put_resv_regions() callback
2533 * for simple reservations. Memory allocated for each reserved region will be
2534 * freed. If an IOMMU driver allocates additional resources per region, it is
2535 * going to have to implement a custom callback.
2536 */
2537void generic_iommu_put_resv_regions(struct device *dev, struct list_head *list)
2538{
2539 struct iommu_resv_region *entry, *next;
2540
2541 list_for_each_entry_safe(entry, next, list, list)
2542 kfree(entry);
2543}
2544EXPORT_SYMBOL(generic_iommu_put_resv_regions);
2545
2546struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
2547 size_t length, int prot,
2548 enum iommu_resv_type type)
2549{
2550 struct iommu_resv_region *region;
2551
2552 region = kzalloc(sizeof(*region), GFP_KERNEL);
2553 if (!region)
2554 return NULL;
2555
2556 INIT_LIST_HEAD(®ion->list);
2557 region->start = start;
2558 region->length = length;
2559 region->prot = prot;
2560 region->type = type;
2561 return region;
2562}
2563EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
2564
2565void iommu_set_default_passthrough(bool cmd_line)
2566{
2567 if (cmd_line)
2568 iommu_set_cmd_line_dma_api();
2569
2570 iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
2571}
2572
2573void iommu_set_default_translated(bool cmd_line)
2574{
2575 if (cmd_line)
2576 iommu_set_cmd_line_dma_api();
2577
2578 iommu_def_domain_type = IOMMU_DOMAIN_DMA;
2579}
2580
2581bool iommu_default_passthrough(void)
2582{
2583 return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
2584}
2585EXPORT_SYMBOL_GPL(iommu_default_passthrough);
2586
2587const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
2588{
2589 const struct iommu_ops *ops = NULL;
2590 struct iommu_device *iommu;
2591
2592 spin_lock(&iommu_device_lock);
2593 list_for_each_entry(iommu, &iommu_device_list, list)
2594 if (iommu->fwnode == fwnode) {
2595 ops = iommu->ops;
2596 break;
2597 }
2598 spin_unlock(&iommu_device_lock);
2599 return ops;
2600}
2601
2602int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
2603 const struct iommu_ops *ops)
2604{
2605 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2606
2607 if (fwspec)
2608 return ops == fwspec->ops ? 0 : -EINVAL;
2609
2610 if (!dev_iommu_get(dev))
2611 return -ENOMEM;
2612
2613 /* Preallocate for the overwhelmingly common case of 1 ID */
2614 fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL);
2615 if (!fwspec)
2616 return -ENOMEM;
2617
2618 of_node_get(to_of_node(iommu_fwnode));
2619 fwspec->iommu_fwnode = iommu_fwnode;
2620 fwspec->ops = ops;
2621 dev_iommu_fwspec_set(dev, fwspec);
2622 return 0;
2623}
2624EXPORT_SYMBOL_GPL(iommu_fwspec_init);
2625
2626void iommu_fwspec_free(struct device *dev)
2627{
2628 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2629
2630 if (fwspec) {
2631 fwnode_handle_put(fwspec->iommu_fwnode);
2632 kfree(fwspec);
2633 dev_iommu_fwspec_set(dev, NULL);
2634 }
2635}
2636EXPORT_SYMBOL_GPL(iommu_fwspec_free);
2637
2638int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids)
2639{
2640 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2641 int i, new_num;
2642
2643 if (!fwspec)
2644 return -EINVAL;
2645
2646 new_num = fwspec->num_ids + num_ids;
2647 if (new_num > 1) {
2648 fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num),
2649 GFP_KERNEL);
2650 if (!fwspec)
2651 return -ENOMEM;
2652
2653 dev_iommu_fwspec_set(dev, fwspec);
2654 }
2655
2656 for (i = 0; i < num_ids; i++)
2657 fwspec->ids[fwspec->num_ids + i] = ids[i];
2658
2659 fwspec->num_ids = new_num;
2660 return 0;
2661}
2662EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
2663
2664/*
2665 * Per device IOMMU features.
2666 */
2667bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat)
2668{
2669 const struct iommu_ops *ops = dev->bus->iommu_ops;
2670
2671 if (ops && ops->dev_has_feat)
2672 return ops->dev_has_feat(dev, feat);
2673
2674 return false;
2675}
2676EXPORT_SYMBOL_GPL(iommu_dev_has_feature);
2677
2678int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
2679{
2680 const struct iommu_ops *ops = dev->bus->iommu_ops;
2681
2682 if (ops && ops->dev_enable_feat)
2683 return ops->dev_enable_feat(dev, feat);
2684
2685 return -ENODEV;
2686}
2687EXPORT_SYMBOL_GPL(iommu_dev_enable_feature);
2688
2689/*
2690 * The device drivers should do the necessary cleanups before calling this.
2691 * For example, before disabling the aux-domain feature, the device driver
2692 * should detach all aux-domains. Otherwise, this will return -EBUSY.
2693 */
2694int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
2695{
2696 const struct iommu_ops *ops = dev->bus->iommu_ops;
2697
2698 if (ops && ops->dev_disable_feat)
2699 return ops->dev_disable_feat(dev, feat);
2700
2701 return -EBUSY;
2702}
2703EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
2704
2705bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat)
2706{
2707 const struct iommu_ops *ops = dev->bus->iommu_ops;
2708
2709 if (ops && ops->dev_feat_enabled)
2710 return ops->dev_feat_enabled(dev, feat);
2711
2712 return false;
2713}
2714EXPORT_SYMBOL_GPL(iommu_dev_feature_enabled);
2715
2716/*
2717 * Aux-domain specific attach/detach.
2718 *
2719 * Only works if iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX) returns
2720 * true. Also, as long as domains are attached to a device through this
2721 * interface, any tries to call iommu_attach_device() should fail
2722 * (iommu_detach_device() can't fail, so we fail when trying to re-attach).
2723 * This should make us safe against a device being attached to a guest as a
2724 * whole while there are still pasid users on it (aux and sva).
2725 */
2726int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev)
2727{
2728 int ret = -ENODEV;
2729
2730 if (domain->ops->aux_attach_dev)
2731 ret = domain->ops->aux_attach_dev(domain, dev);
2732
2733 if (!ret)
2734 trace_attach_device_to_domain(dev);
2735
2736 return ret;
2737}
2738EXPORT_SYMBOL_GPL(iommu_aux_attach_device);
2739
2740void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev)
2741{
2742 if (domain->ops->aux_detach_dev) {
2743 domain->ops->aux_detach_dev(domain, dev);
2744 trace_detach_device_from_domain(dev);
2745 }
2746}
2747EXPORT_SYMBOL_GPL(iommu_aux_detach_device);
2748
2749int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
2750{
2751 int ret = -ENODEV;
2752
2753 if (domain->ops->aux_get_pasid)
2754 ret = domain->ops->aux_get_pasid(domain, dev);
2755
2756 return ret;
2757}
2758EXPORT_SYMBOL_GPL(iommu_aux_get_pasid);
2759
2760/**
2761 * iommu_sva_bind_device() - Bind a process address space to a device
2762 * @dev: the device
2763 * @mm: the mm to bind, caller must hold a reference to it
2764 *
2765 * Create a bond between device and address space, allowing the device to access
2766 * the mm using the returned PASID. If a bond already exists between @device and
2767 * @mm, it is returned and an additional reference is taken. Caller must call
2768 * iommu_sva_unbind_device() to release each reference.
2769 *
2770 * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
2771 * initialize the required SVA features.
2772 *
2773 * On error, returns an ERR_PTR value.
2774 */
2775struct iommu_sva *
2776iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
2777{
2778 struct iommu_group *group;
2779 struct iommu_sva *handle = ERR_PTR(-EINVAL);
2780 const struct iommu_ops *ops = dev->bus->iommu_ops;
2781
2782 if (!ops || !ops->sva_bind)
2783 return ERR_PTR(-ENODEV);
2784
2785 group = iommu_group_get(dev);
2786 if (!group)
2787 return ERR_PTR(-ENODEV);
2788
2789 /* Ensure device count and domain don't change while we're binding */
2790 mutex_lock(&group->mutex);
2791
2792 /*
2793 * To keep things simple, SVA currently doesn't support IOMMU groups
2794 * with more than one device. Existing SVA-capable systems are not
2795 * affected by the problems that required IOMMU groups (lack of ACS
2796 * isolation, device ID aliasing and other hardware issues).
2797 */
2798 if (iommu_group_device_count(group) != 1)
2799 goto out_unlock;
2800
2801 handle = ops->sva_bind(dev, mm, drvdata);
2802
2803out_unlock:
2804 mutex_unlock(&group->mutex);
2805 iommu_group_put(group);
2806
2807 return handle;
2808}
2809EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
2810
2811/**
2812 * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
2813 * @handle: the handle returned by iommu_sva_bind_device()
2814 *
2815 * Put reference to a bond between device and address space. The device should
2816 * not be issuing any more transaction for this PASID. All outstanding page
2817 * requests for this PASID must have been flushed to the IOMMU.
2818 *
2819 * Returns 0 on success, or an error value
2820 */
2821void iommu_sva_unbind_device(struct iommu_sva *handle)
2822{
2823 struct iommu_group *group;
2824 struct device *dev = handle->dev;
2825 const struct iommu_ops *ops = dev->bus->iommu_ops;
2826
2827 if (!ops || !ops->sva_unbind)
2828 return;
2829
2830 group = iommu_group_get(dev);
2831 if (!group)
2832 return;
2833
2834 mutex_lock(&group->mutex);
2835 ops->sva_unbind(handle);
2836 mutex_unlock(&group->mutex);
2837
2838 iommu_group_put(group);
2839}
2840EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
2841
2842int iommu_sva_get_pasid(struct iommu_sva *handle)
2843{
2844 const struct iommu_ops *ops = handle->dev->bus->iommu_ops;
2845
2846 if (!ops || !ops->sva_get_pasid)
2847 return IOMMU_PASID_INVALID;
2848
2849 return ops->sva_get_pasid(handle);
2850}
2851EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);