Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 */
6
7#define pr_fmt(fmt) "iommu: " fmt
8
9#include <linux/amba/bus.h>
10#include <linux/device.h>
11#include <linux/kernel.h>
12#include <linux/bits.h>
13#include <linux/bug.h>
14#include <linux/types.h>
15#include <linux/init.h>
16#include <linux/export.h>
17#include <linux/slab.h>
18#include <linux/errno.h>
19#include <linux/host1x_context_bus.h>
20#include <linux/iommu.h>
21#include <linux/idr.h>
22#include <linux/err.h>
23#include <linux/pci.h>
24#include <linux/pci-ats.h>
25#include <linux/bitops.h>
26#include <linux/platform_device.h>
27#include <linux/property.h>
28#include <linux/fsl/mc.h>
29#include <linux/module.h>
30#include <linux/cc_platform.h>
31#include <linux/cdx/cdx_bus.h>
32#include <trace/events/iommu.h>
33#include <linux/sched/mm.h>
34#include <linux/msi.h>
35
36#include "dma-iommu.h"
37#include "iommu-priv.h"
38
39static struct kset *iommu_group_kset;
40static DEFINE_IDA(iommu_group_ida);
41static DEFINE_IDA(iommu_global_pasid_ida);
42
43static unsigned int iommu_def_domain_type __read_mostly;
44static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_DMA_STRICT);
45static u32 iommu_cmd_line __read_mostly;
46
47struct iommu_group {
48 struct kobject kobj;
49 struct kobject *devices_kobj;
50 struct list_head devices;
51 struct xarray pasid_array;
52 struct mutex mutex;
53 void *iommu_data;
54 void (*iommu_data_release)(void *iommu_data);
55 char *name;
56 int id;
57 struct iommu_domain *default_domain;
58 struct iommu_domain *blocking_domain;
59 struct iommu_domain *domain;
60 struct list_head entry;
61 unsigned int owner_cnt;
62 void *owner;
63};
64
65struct group_device {
66 struct list_head list;
67 struct device *dev;
68 char *name;
69};
70
71/* Iterate over each struct group_device in a struct iommu_group */
72#define for_each_group_device(group, pos) \
73 list_for_each_entry(pos, &(group)->devices, list)
74
75struct iommu_group_attribute {
76 struct attribute attr;
77 ssize_t (*show)(struct iommu_group *group, char *buf);
78 ssize_t (*store)(struct iommu_group *group,
79 const char *buf, size_t count);
80};
81
82static const char * const iommu_group_resv_type_string[] = {
83 [IOMMU_RESV_DIRECT] = "direct",
84 [IOMMU_RESV_DIRECT_RELAXABLE] = "direct-relaxable",
85 [IOMMU_RESV_RESERVED] = "reserved",
86 [IOMMU_RESV_MSI] = "msi",
87 [IOMMU_RESV_SW_MSI] = "msi",
88};
89
90#define IOMMU_CMD_LINE_DMA_API BIT(0)
91#define IOMMU_CMD_LINE_STRICT BIT(1)
92
93static int iommu_bus_notifier(struct notifier_block *nb,
94 unsigned long action, void *data);
95static void iommu_release_device(struct device *dev);
96static struct iommu_domain *
97__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type);
98static int __iommu_attach_device(struct iommu_domain *domain,
99 struct device *dev);
100static int __iommu_attach_group(struct iommu_domain *domain,
101 struct iommu_group *group);
102
103enum {
104 IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0,
105};
106
107static int __iommu_device_set_domain(struct iommu_group *group,
108 struct device *dev,
109 struct iommu_domain *new_domain,
110 unsigned int flags);
111static int __iommu_group_set_domain_internal(struct iommu_group *group,
112 struct iommu_domain *new_domain,
113 unsigned int flags);
114static int __iommu_group_set_domain(struct iommu_group *group,
115 struct iommu_domain *new_domain)
116{
117 return __iommu_group_set_domain_internal(group, new_domain, 0);
118}
119static void __iommu_group_set_domain_nofail(struct iommu_group *group,
120 struct iommu_domain *new_domain)
121{
122 WARN_ON(__iommu_group_set_domain_internal(
123 group, new_domain, IOMMU_SET_DOMAIN_MUST_SUCCEED));
124}
125
126static int iommu_setup_default_domain(struct iommu_group *group,
127 int target_type);
128static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
129 struct device *dev);
130static ssize_t iommu_group_store_type(struct iommu_group *group,
131 const char *buf, size_t count);
132static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
133 struct device *dev);
134static void __iommu_group_free_device(struct iommu_group *group,
135 struct group_device *grp_dev);
136
137#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
138struct iommu_group_attribute iommu_group_attr_##_name = \
139 __ATTR(_name, _mode, _show, _store)
140
141#define to_iommu_group_attr(_attr) \
142 container_of(_attr, struct iommu_group_attribute, attr)
143#define to_iommu_group(_kobj) \
144 container_of(_kobj, struct iommu_group, kobj)
145
146static LIST_HEAD(iommu_device_list);
147static DEFINE_SPINLOCK(iommu_device_lock);
148
149static const struct bus_type * const iommu_buses[] = {
150 &platform_bus_type,
151#ifdef CONFIG_PCI
152 &pci_bus_type,
153#endif
154#ifdef CONFIG_ARM_AMBA
155 &amba_bustype,
156#endif
157#ifdef CONFIG_FSL_MC_BUS
158 &fsl_mc_bus_type,
159#endif
160#ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS
161 &host1x_context_device_bus_type,
162#endif
163#ifdef CONFIG_CDX_BUS
164 &cdx_bus_type,
165#endif
166};
167
168/*
169 * Use a function instead of an array here because the domain-type is a
170 * bit-field, so an array would waste memory.
171 */
172static const char *iommu_domain_type_str(unsigned int t)
173{
174 switch (t) {
175 case IOMMU_DOMAIN_BLOCKED:
176 return "Blocked";
177 case IOMMU_DOMAIN_IDENTITY:
178 return "Passthrough";
179 case IOMMU_DOMAIN_UNMANAGED:
180 return "Unmanaged";
181 case IOMMU_DOMAIN_DMA:
182 case IOMMU_DOMAIN_DMA_FQ:
183 return "Translated";
184 case IOMMU_DOMAIN_PLATFORM:
185 return "Platform";
186 default:
187 return "Unknown";
188 }
189}
190
191static int __init iommu_subsys_init(void)
192{
193 struct notifier_block *nb;
194
195 if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) {
196 if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
197 iommu_set_default_passthrough(false);
198 else
199 iommu_set_default_translated(false);
200
201 if (iommu_default_passthrough() && cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
202 pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
203 iommu_set_default_translated(false);
204 }
205 }
206
207 if (!iommu_default_passthrough() && !iommu_dma_strict)
208 iommu_def_domain_type = IOMMU_DOMAIN_DMA_FQ;
209
210 pr_info("Default domain type: %s%s\n",
211 iommu_domain_type_str(iommu_def_domain_type),
212 (iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ?
213 " (set via kernel command line)" : "");
214
215 if (!iommu_default_passthrough())
216 pr_info("DMA domain TLB invalidation policy: %s mode%s\n",
217 iommu_dma_strict ? "strict" : "lazy",
218 (iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ?
219 " (set via kernel command line)" : "");
220
221 nb = kcalloc(ARRAY_SIZE(iommu_buses), sizeof(*nb), GFP_KERNEL);
222 if (!nb)
223 return -ENOMEM;
224
225 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
226 nb[i].notifier_call = iommu_bus_notifier;
227 bus_register_notifier(iommu_buses[i], &nb[i]);
228 }
229
230 return 0;
231}
232subsys_initcall(iommu_subsys_init);
233
234static int remove_iommu_group(struct device *dev, void *data)
235{
236 if (dev->iommu && dev->iommu->iommu_dev == data)
237 iommu_release_device(dev);
238
239 return 0;
240}
241
242/**
243 * iommu_device_register() - Register an IOMMU hardware instance
244 * @iommu: IOMMU handle for the instance
245 * @ops: IOMMU ops to associate with the instance
246 * @hwdev: (optional) actual instance device, used for fwnode lookup
247 *
248 * Return: 0 on success, or an error.
249 */
250int iommu_device_register(struct iommu_device *iommu,
251 const struct iommu_ops *ops, struct device *hwdev)
252{
253 int err = 0;
254
255 /* We need to be able to take module references appropriately */
256 if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner))
257 return -EINVAL;
258
259 iommu->ops = ops;
260 if (hwdev)
261 iommu->fwnode = dev_fwnode(hwdev);
262
263 spin_lock(&iommu_device_lock);
264 list_add_tail(&iommu->list, &iommu_device_list);
265 spin_unlock(&iommu_device_lock);
266
267 for (int i = 0; i < ARRAY_SIZE(iommu_buses) && !err; i++)
268 err = bus_iommu_probe(iommu_buses[i]);
269 if (err)
270 iommu_device_unregister(iommu);
271 return err;
272}
273EXPORT_SYMBOL_GPL(iommu_device_register);
274
275void iommu_device_unregister(struct iommu_device *iommu)
276{
277 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++)
278 bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group);
279
280 spin_lock(&iommu_device_lock);
281 list_del(&iommu->list);
282 spin_unlock(&iommu_device_lock);
283
284 /* Pairs with the alloc in generic_single_device_group() */
285 iommu_group_put(iommu->singleton_group);
286 iommu->singleton_group = NULL;
287}
288EXPORT_SYMBOL_GPL(iommu_device_unregister);
289
290#if IS_ENABLED(CONFIG_IOMMUFD_TEST)
291void iommu_device_unregister_bus(struct iommu_device *iommu,
292 const struct bus_type *bus,
293 struct notifier_block *nb)
294{
295 bus_unregister_notifier(bus, nb);
296 iommu_device_unregister(iommu);
297}
298EXPORT_SYMBOL_GPL(iommu_device_unregister_bus);
299
300/*
301 * Register an iommu driver against a single bus. This is only used by iommufd
302 * selftest to create a mock iommu driver. The caller must provide
303 * some memory to hold a notifier_block.
304 */
305int iommu_device_register_bus(struct iommu_device *iommu,
306 const struct iommu_ops *ops,
307 const struct bus_type *bus,
308 struct notifier_block *nb)
309{
310 int err;
311
312 iommu->ops = ops;
313 nb->notifier_call = iommu_bus_notifier;
314 err = bus_register_notifier(bus, nb);
315 if (err)
316 return err;
317
318 spin_lock(&iommu_device_lock);
319 list_add_tail(&iommu->list, &iommu_device_list);
320 spin_unlock(&iommu_device_lock);
321
322 err = bus_iommu_probe(bus);
323 if (err) {
324 iommu_device_unregister_bus(iommu, bus, nb);
325 return err;
326 }
327 return 0;
328}
329EXPORT_SYMBOL_GPL(iommu_device_register_bus);
330#endif
331
332static struct dev_iommu *dev_iommu_get(struct device *dev)
333{
334 struct dev_iommu *param = dev->iommu;
335
336 lockdep_assert_held(&iommu_probe_device_lock);
337
338 if (param)
339 return param;
340
341 param = kzalloc(sizeof(*param), GFP_KERNEL);
342 if (!param)
343 return NULL;
344
345 mutex_init(¶m->lock);
346 dev->iommu = param;
347 return param;
348}
349
350static void dev_iommu_free(struct device *dev)
351{
352 struct dev_iommu *param = dev->iommu;
353
354 dev->iommu = NULL;
355 if (param->fwspec) {
356 fwnode_handle_put(param->fwspec->iommu_fwnode);
357 kfree(param->fwspec);
358 }
359 kfree(param);
360}
361
362/*
363 * Internal equivalent of device_iommu_mapped() for when we care that a device
364 * actually has API ops, and don't want false positives from VFIO-only groups.
365 */
366static bool dev_has_iommu(struct device *dev)
367{
368 return dev->iommu && dev->iommu->iommu_dev;
369}
370
371static u32 dev_iommu_get_max_pasids(struct device *dev)
372{
373 u32 max_pasids = 0, bits = 0;
374 int ret;
375
376 if (dev_is_pci(dev)) {
377 ret = pci_max_pasids(to_pci_dev(dev));
378 if (ret > 0)
379 max_pasids = ret;
380 } else {
381 ret = device_property_read_u32(dev, "pasid-num-bits", &bits);
382 if (!ret)
383 max_pasids = 1UL << bits;
384 }
385
386 return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);
387}
388
389void dev_iommu_priv_set(struct device *dev, void *priv)
390{
391 /* FSL_PAMU does something weird */
392 if (!IS_ENABLED(CONFIG_FSL_PAMU))
393 lockdep_assert_held(&iommu_probe_device_lock);
394 dev->iommu->priv = priv;
395}
396EXPORT_SYMBOL_GPL(dev_iommu_priv_set);
397
398/*
399 * Init the dev->iommu and dev->iommu_group in the struct device and get the
400 * driver probed
401 */
402static int iommu_init_device(struct device *dev, const struct iommu_ops *ops)
403{
404 struct iommu_device *iommu_dev;
405 struct iommu_group *group;
406 int ret;
407
408 if (!dev_iommu_get(dev))
409 return -ENOMEM;
410
411 if (!try_module_get(ops->owner)) {
412 ret = -EINVAL;
413 goto err_free;
414 }
415
416 iommu_dev = ops->probe_device(dev);
417 if (IS_ERR(iommu_dev)) {
418 ret = PTR_ERR(iommu_dev);
419 goto err_module_put;
420 }
421 dev->iommu->iommu_dev = iommu_dev;
422
423 ret = iommu_device_link(iommu_dev, dev);
424 if (ret)
425 goto err_release;
426
427 group = ops->device_group(dev);
428 if (WARN_ON_ONCE(group == NULL))
429 group = ERR_PTR(-EINVAL);
430 if (IS_ERR(group)) {
431 ret = PTR_ERR(group);
432 goto err_unlink;
433 }
434 dev->iommu_group = group;
435
436 dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
437 if (ops->is_attach_deferred)
438 dev->iommu->attach_deferred = ops->is_attach_deferred(dev);
439 return 0;
440
441err_unlink:
442 iommu_device_unlink(iommu_dev, dev);
443err_release:
444 if (ops->release_device)
445 ops->release_device(dev);
446err_module_put:
447 module_put(ops->owner);
448err_free:
449 dev->iommu->iommu_dev = NULL;
450 dev_iommu_free(dev);
451 return ret;
452}
453
454static void iommu_deinit_device(struct device *dev)
455{
456 struct iommu_group *group = dev->iommu_group;
457 const struct iommu_ops *ops = dev_iommu_ops(dev);
458
459 lockdep_assert_held(&group->mutex);
460
461 iommu_device_unlink(dev->iommu->iommu_dev, dev);
462
463 /*
464 * release_device() must stop using any attached domain on the device.
465 * If there are still other devices in the group, they are not affected
466 * by this callback.
467 *
468 * If the iommu driver provides release_domain, the core code ensures
469 * that domain is attached prior to calling release_device. Drivers can
470 * use this to enforce a translation on the idle iommu. Typically, the
471 * global static blocked_domain is a good choice.
472 *
473 * Otherwise, the iommu driver must set the device to either an identity
474 * or a blocking translation in release_device() and stop using any
475 * domain pointer, as it is going to be freed.
476 *
477 * Regardless, if a delayed attach never occurred, then the release
478 * should still avoid touching any hardware configuration either.
479 */
480 if (!dev->iommu->attach_deferred && ops->release_domain)
481 ops->release_domain->ops->attach_dev(ops->release_domain, dev);
482
483 if (ops->release_device)
484 ops->release_device(dev);
485
486 /*
487 * If this is the last driver to use the group then we must free the
488 * domains before we do the module_put().
489 */
490 if (list_empty(&group->devices)) {
491 if (group->default_domain) {
492 iommu_domain_free(group->default_domain);
493 group->default_domain = NULL;
494 }
495 if (group->blocking_domain) {
496 iommu_domain_free(group->blocking_domain);
497 group->blocking_domain = NULL;
498 }
499 group->domain = NULL;
500 }
501
502 /* Caller must put iommu_group */
503 dev->iommu_group = NULL;
504 module_put(ops->owner);
505 dev_iommu_free(dev);
506}
507
508DEFINE_MUTEX(iommu_probe_device_lock);
509
510static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
511{
512 const struct iommu_ops *ops;
513 struct iommu_fwspec *fwspec;
514 struct iommu_group *group;
515 struct group_device *gdev;
516 int ret;
517
518 /*
519 * For FDT-based systems and ACPI IORT/VIOT, drivers register IOMMU
520 * instances with non-NULL fwnodes, and client devices should have been
521 * identified with a fwspec by this point. Otherwise, we can currently
522 * assume that only one of Intel, AMD, s390, PAMU or legacy SMMUv2 can
523 * be present, and that any of their registered instances has suitable
524 * ops for probing, and thus cheekily co-opt the same mechanism.
525 */
526 fwspec = dev_iommu_fwspec_get(dev);
527 if (fwspec && fwspec->ops)
528 ops = fwspec->ops;
529 else
530 ops = iommu_ops_from_fwnode(NULL);
531
532 if (!ops)
533 return -ENODEV;
534 /*
535 * Serialise to avoid races between IOMMU drivers registering in
536 * parallel and/or the "replay" calls from ACPI/OF code via client
537 * driver probe. Once the latter have been cleaned up we should
538 * probably be able to use device_lock() here to minimise the scope,
539 * but for now enforcing a simple global ordering is fine.
540 */
541 lockdep_assert_held(&iommu_probe_device_lock);
542
543 /* Device is probed already if in a group */
544 if (dev->iommu_group)
545 return 0;
546
547 ret = iommu_init_device(dev, ops);
548 if (ret)
549 return ret;
550
551 group = dev->iommu_group;
552 gdev = iommu_group_alloc_device(group, dev);
553 mutex_lock(&group->mutex);
554 if (IS_ERR(gdev)) {
555 ret = PTR_ERR(gdev);
556 goto err_put_group;
557 }
558
559 /*
560 * The gdev must be in the list before calling
561 * iommu_setup_default_domain()
562 */
563 list_add_tail(&gdev->list, &group->devices);
564 WARN_ON(group->default_domain && !group->domain);
565 if (group->default_domain)
566 iommu_create_device_direct_mappings(group->default_domain, dev);
567 if (group->domain) {
568 ret = __iommu_device_set_domain(group, dev, group->domain, 0);
569 if (ret)
570 goto err_remove_gdev;
571 } else if (!group->default_domain && !group_list) {
572 ret = iommu_setup_default_domain(group, 0);
573 if (ret)
574 goto err_remove_gdev;
575 } else if (!group->default_domain) {
576 /*
577 * With a group_list argument we defer the default_domain setup
578 * to the caller by providing a de-duplicated list of groups
579 * that need further setup.
580 */
581 if (list_empty(&group->entry))
582 list_add_tail(&group->entry, group_list);
583 }
584 mutex_unlock(&group->mutex);
585
586 if (dev_is_pci(dev))
587 iommu_dma_set_pci_32bit_workaround(dev);
588
589 return 0;
590
591err_remove_gdev:
592 list_del(&gdev->list);
593 __iommu_group_free_device(group, gdev);
594err_put_group:
595 iommu_deinit_device(dev);
596 mutex_unlock(&group->mutex);
597 iommu_group_put(group);
598
599 return ret;
600}
601
602int iommu_probe_device(struct device *dev)
603{
604 const struct iommu_ops *ops;
605 int ret;
606
607 mutex_lock(&iommu_probe_device_lock);
608 ret = __iommu_probe_device(dev, NULL);
609 mutex_unlock(&iommu_probe_device_lock);
610 if (ret)
611 return ret;
612
613 ops = dev_iommu_ops(dev);
614 if (ops->probe_finalize)
615 ops->probe_finalize(dev);
616
617 return 0;
618}
619
620static void __iommu_group_free_device(struct iommu_group *group,
621 struct group_device *grp_dev)
622{
623 struct device *dev = grp_dev->dev;
624
625 sysfs_remove_link(group->devices_kobj, grp_dev->name);
626 sysfs_remove_link(&dev->kobj, "iommu_group");
627
628 trace_remove_device_from_group(group->id, dev);
629
630 /*
631 * If the group has become empty then ownership must have been
632 * released, and the current domain must be set back to NULL or
633 * the default domain.
634 */
635 if (list_empty(&group->devices))
636 WARN_ON(group->owner_cnt ||
637 group->domain != group->default_domain);
638
639 kfree(grp_dev->name);
640 kfree(grp_dev);
641}
642
643/* Remove the iommu_group from the struct device. */
644static void __iommu_group_remove_device(struct device *dev)
645{
646 struct iommu_group *group = dev->iommu_group;
647 struct group_device *device;
648
649 mutex_lock(&group->mutex);
650 for_each_group_device(group, device) {
651 if (device->dev != dev)
652 continue;
653
654 list_del(&device->list);
655 __iommu_group_free_device(group, device);
656 if (dev_has_iommu(dev))
657 iommu_deinit_device(dev);
658 else
659 dev->iommu_group = NULL;
660 break;
661 }
662 mutex_unlock(&group->mutex);
663
664 /*
665 * Pairs with the get in iommu_init_device() or
666 * iommu_group_add_device()
667 */
668 iommu_group_put(group);
669}
670
671static void iommu_release_device(struct device *dev)
672{
673 struct iommu_group *group = dev->iommu_group;
674
675 if (group)
676 __iommu_group_remove_device(dev);
677
678 /* Free any fwspec if no iommu_driver was ever attached */
679 if (dev->iommu)
680 dev_iommu_free(dev);
681}
682
683static int __init iommu_set_def_domain_type(char *str)
684{
685 bool pt;
686 int ret;
687
688 ret = kstrtobool(str, &pt);
689 if (ret)
690 return ret;
691
692 if (pt)
693 iommu_set_default_passthrough(true);
694 else
695 iommu_set_default_translated(true);
696
697 return 0;
698}
699early_param("iommu.passthrough", iommu_set_def_domain_type);
700
701static int __init iommu_dma_setup(char *str)
702{
703 int ret = kstrtobool(str, &iommu_dma_strict);
704
705 if (!ret)
706 iommu_cmd_line |= IOMMU_CMD_LINE_STRICT;
707 return ret;
708}
709early_param("iommu.strict", iommu_dma_setup);
710
711void iommu_set_dma_strict(void)
712{
713 iommu_dma_strict = true;
714 if (iommu_def_domain_type == IOMMU_DOMAIN_DMA_FQ)
715 iommu_def_domain_type = IOMMU_DOMAIN_DMA;
716}
717
718static ssize_t iommu_group_attr_show(struct kobject *kobj,
719 struct attribute *__attr, char *buf)
720{
721 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
722 struct iommu_group *group = to_iommu_group(kobj);
723 ssize_t ret = -EIO;
724
725 if (attr->show)
726 ret = attr->show(group, buf);
727 return ret;
728}
729
730static ssize_t iommu_group_attr_store(struct kobject *kobj,
731 struct attribute *__attr,
732 const char *buf, size_t count)
733{
734 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
735 struct iommu_group *group = to_iommu_group(kobj);
736 ssize_t ret = -EIO;
737
738 if (attr->store)
739 ret = attr->store(group, buf, count);
740 return ret;
741}
742
743static const struct sysfs_ops iommu_group_sysfs_ops = {
744 .show = iommu_group_attr_show,
745 .store = iommu_group_attr_store,
746};
747
748static int iommu_group_create_file(struct iommu_group *group,
749 struct iommu_group_attribute *attr)
750{
751 return sysfs_create_file(&group->kobj, &attr->attr);
752}
753
754static void iommu_group_remove_file(struct iommu_group *group,
755 struct iommu_group_attribute *attr)
756{
757 sysfs_remove_file(&group->kobj, &attr->attr);
758}
759
760static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
761{
762 return sysfs_emit(buf, "%s\n", group->name);
763}
764
765/**
766 * iommu_insert_resv_region - Insert a new region in the
767 * list of reserved regions.
768 * @new: new region to insert
769 * @regions: list of regions
770 *
771 * Elements are sorted by start address and overlapping segments
772 * of the same type are merged.
773 */
774static int iommu_insert_resv_region(struct iommu_resv_region *new,
775 struct list_head *regions)
776{
777 struct iommu_resv_region *iter, *tmp, *nr, *top;
778 LIST_HEAD(stack);
779
780 nr = iommu_alloc_resv_region(new->start, new->length,
781 new->prot, new->type, GFP_KERNEL);
782 if (!nr)
783 return -ENOMEM;
784
785 /* First add the new element based on start address sorting */
786 list_for_each_entry(iter, regions, list) {
787 if (nr->start < iter->start ||
788 (nr->start == iter->start && nr->type <= iter->type))
789 break;
790 }
791 list_add_tail(&nr->list, &iter->list);
792
793 /* Merge overlapping segments of type nr->type in @regions, if any */
794 list_for_each_entry_safe(iter, tmp, regions, list) {
795 phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
796
797 /* no merge needed on elements of different types than @new */
798 if (iter->type != new->type) {
799 list_move_tail(&iter->list, &stack);
800 continue;
801 }
802
803 /* look for the last stack element of same type as @iter */
804 list_for_each_entry_reverse(top, &stack, list)
805 if (top->type == iter->type)
806 goto check_overlap;
807
808 list_move_tail(&iter->list, &stack);
809 continue;
810
811check_overlap:
812 top_end = top->start + top->length - 1;
813
814 if (iter->start > top_end + 1) {
815 list_move_tail(&iter->list, &stack);
816 } else {
817 top->length = max(top_end, iter_end) - top->start + 1;
818 list_del(&iter->list);
819 kfree(iter);
820 }
821 }
822 list_splice(&stack, regions);
823 return 0;
824}
825
826static int
827iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
828 struct list_head *group_resv_regions)
829{
830 struct iommu_resv_region *entry;
831 int ret = 0;
832
833 list_for_each_entry(entry, dev_resv_regions, list) {
834 ret = iommu_insert_resv_region(entry, group_resv_regions);
835 if (ret)
836 break;
837 }
838 return ret;
839}
840
841int iommu_get_group_resv_regions(struct iommu_group *group,
842 struct list_head *head)
843{
844 struct group_device *device;
845 int ret = 0;
846
847 mutex_lock(&group->mutex);
848 for_each_group_device(group, device) {
849 struct list_head dev_resv_regions;
850
851 /*
852 * Non-API groups still expose reserved_regions in sysfs,
853 * so filter out calls that get here that way.
854 */
855 if (!dev_has_iommu(device->dev))
856 break;
857
858 INIT_LIST_HEAD(&dev_resv_regions);
859 iommu_get_resv_regions(device->dev, &dev_resv_regions);
860 ret = iommu_insert_device_resv_regions(&dev_resv_regions, head);
861 iommu_put_resv_regions(device->dev, &dev_resv_regions);
862 if (ret)
863 break;
864 }
865 mutex_unlock(&group->mutex);
866 return ret;
867}
868EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
869
870static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
871 char *buf)
872{
873 struct iommu_resv_region *region, *next;
874 struct list_head group_resv_regions;
875 int offset = 0;
876
877 INIT_LIST_HEAD(&group_resv_regions);
878 iommu_get_group_resv_regions(group, &group_resv_regions);
879
880 list_for_each_entry_safe(region, next, &group_resv_regions, list) {
881 offset += sysfs_emit_at(buf, offset, "0x%016llx 0x%016llx %s\n",
882 (long long)region->start,
883 (long long)(region->start +
884 region->length - 1),
885 iommu_group_resv_type_string[region->type]);
886 kfree(region);
887 }
888
889 return offset;
890}
891
892static ssize_t iommu_group_show_type(struct iommu_group *group,
893 char *buf)
894{
895 char *type = "unknown";
896
897 mutex_lock(&group->mutex);
898 if (group->default_domain) {
899 switch (group->default_domain->type) {
900 case IOMMU_DOMAIN_BLOCKED:
901 type = "blocked";
902 break;
903 case IOMMU_DOMAIN_IDENTITY:
904 type = "identity";
905 break;
906 case IOMMU_DOMAIN_UNMANAGED:
907 type = "unmanaged";
908 break;
909 case IOMMU_DOMAIN_DMA:
910 type = "DMA";
911 break;
912 case IOMMU_DOMAIN_DMA_FQ:
913 type = "DMA-FQ";
914 break;
915 }
916 }
917 mutex_unlock(&group->mutex);
918
919 return sysfs_emit(buf, "%s\n", type);
920}
921
922static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
923
924static IOMMU_GROUP_ATTR(reserved_regions, 0444,
925 iommu_group_show_resv_regions, NULL);
926
927static IOMMU_GROUP_ATTR(type, 0644, iommu_group_show_type,
928 iommu_group_store_type);
929
930static void iommu_group_release(struct kobject *kobj)
931{
932 struct iommu_group *group = to_iommu_group(kobj);
933
934 pr_debug("Releasing group %d\n", group->id);
935
936 if (group->iommu_data_release)
937 group->iommu_data_release(group->iommu_data);
938
939 ida_free(&iommu_group_ida, group->id);
940
941 /* Domains are free'd by iommu_deinit_device() */
942 WARN_ON(group->default_domain);
943 WARN_ON(group->blocking_domain);
944
945 kfree(group->name);
946 kfree(group);
947}
948
949static const struct kobj_type iommu_group_ktype = {
950 .sysfs_ops = &iommu_group_sysfs_ops,
951 .release = iommu_group_release,
952};
953
954/**
955 * iommu_group_alloc - Allocate a new group
956 *
957 * This function is called by an iommu driver to allocate a new iommu
958 * group. The iommu group represents the minimum granularity of the iommu.
959 * Upon successful return, the caller holds a reference to the supplied
960 * group in order to hold the group until devices are added. Use
961 * iommu_group_put() to release this extra reference count, allowing the
962 * group to be automatically reclaimed once it has no devices or external
963 * references.
964 */
965struct iommu_group *iommu_group_alloc(void)
966{
967 struct iommu_group *group;
968 int ret;
969
970 group = kzalloc(sizeof(*group), GFP_KERNEL);
971 if (!group)
972 return ERR_PTR(-ENOMEM);
973
974 group->kobj.kset = iommu_group_kset;
975 mutex_init(&group->mutex);
976 INIT_LIST_HEAD(&group->devices);
977 INIT_LIST_HEAD(&group->entry);
978 xa_init(&group->pasid_array);
979
980 ret = ida_alloc(&iommu_group_ida, GFP_KERNEL);
981 if (ret < 0) {
982 kfree(group);
983 return ERR_PTR(ret);
984 }
985 group->id = ret;
986
987 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
988 NULL, "%d", group->id);
989 if (ret) {
990 kobject_put(&group->kobj);
991 return ERR_PTR(ret);
992 }
993
994 group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
995 if (!group->devices_kobj) {
996 kobject_put(&group->kobj); /* triggers .release & free */
997 return ERR_PTR(-ENOMEM);
998 }
999
1000 /*
1001 * The devices_kobj holds a reference on the group kobject, so
1002 * as long as that exists so will the group. We can therefore
1003 * use the devices_kobj for reference counting.
1004 */
1005 kobject_put(&group->kobj);
1006
1007 ret = iommu_group_create_file(group,
1008 &iommu_group_attr_reserved_regions);
1009 if (ret) {
1010 kobject_put(group->devices_kobj);
1011 return ERR_PTR(ret);
1012 }
1013
1014 ret = iommu_group_create_file(group, &iommu_group_attr_type);
1015 if (ret) {
1016 kobject_put(group->devices_kobj);
1017 return ERR_PTR(ret);
1018 }
1019
1020 pr_debug("Allocated group %d\n", group->id);
1021
1022 return group;
1023}
1024EXPORT_SYMBOL_GPL(iommu_group_alloc);
1025
1026/**
1027 * iommu_group_get_iommudata - retrieve iommu_data registered for a group
1028 * @group: the group
1029 *
1030 * iommu drivers can store data in the group for use when doing iommu
1031 * operations. This function provides a way to retrieve it. Caller
1032 * should hold a group reference.
1033 */
1034void *iommu_group_get_iommudata(struct iommu_group *group)
1035{
1036 return group->iommu_data;
1037}
1038EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
1039
1040/**
1041 * iommu_group_set_iommudata - set iommu_data for a group
1042 * @group: the group
1043 * @iommu_data: new data
1044 * @release: release function for iommu_data
1045 *
1046 * iommu drivers can store data in the group for use when doing iommu
1047 * operations. This function provides a way to set the data after
1048 * the group has been allocated. Caller should hold a group reference.
1049 */
1050void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
1051 void (*release)(void *iommu_data))
1052{
1053 group->iommu_data = iommu_data;
1054 group->iommu_data_release = release;
1055}
1056EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
1057
1058/**
1059 * iommu_group_set_name - set name for a group
1060 * @group: the group
1061 * @name: name
1062 *
1063 * Allow iommu driver to set a name for a group. When set it will
1064 * appear in a name attribute file under the group in sysfs.
1065 */
1066int iommu_group_set_name(struct iommu_group *group, const char *name)
1067{
1068 int ret;
1069
1070 if (group->name) {
1071 iommu_group_remove_file(group, &iommu_group_attr_name);
1072 kfree(group->name);
1073 group->name = NULL;
1074 if (!name)
1075 return 0;
1076 }
1077
1078 group->name = kstrdup(name, GFP_KERNEL);
1079 if (!group->name)
1080 return -ENOMEM;
1081
1082 ret = iommu_group_create_file(group, &iommu_group_attr_name);
1083 if (ret) {
1084 kfree(group->name);
1085 group->name = NULL;
1086 return ret;
1087 }
1088
1089 return 0;
1090}
1091EXPORT_SYMBOL_GPL(iommu_group_set_name);
1092
1093static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
1094 struct device *dev)
1095{
1096 struct iommu_resv_region *entry;
1097 struct list_head mappings;
1098 unsigned long pg_size;
1099 int ret = 0;
1100
1101 pg_size = domain->pgsize_bitmap ? 1UL << __ffs(domain->pgsize_bitmap) : 0;
1102 INIT_LIST_HEAD(&mappings);
1103
1104 if (WARN_ON_ONCE(iommu_is_dma_domain(domain) && !pg_size))
1105 return -EINVAL;
1106
1107 iommu_get_resv_regions(dev, &mappings);
1108
1109 /* We need to consider overlapping regions for different devices */
1110 list_for_each_entry(entry, &mappings, list) {
1111 dma_addr_t start, end, addr;
1112 size_t map_size = 0;
1113
1114 if (entry->type == IOMMU_RESV_DIRECT)
1115 dev->iommu->require_direct = 1;
1116
1117 if ((entry->type != IOMMU_RESV_DIRECT &&
1118 entry->type != IOMMU_RESV_DIRECT_RELAXABLE) ||
1119 !iommu_is_dma_domain(domain))
1120 continue;
1121
1122 start = ALIGN(entry->start, pg_size);
1123 end = ALIGN(entry->start + entry->length, pg_size);
1124
1125 for (addr = start; addr <= end; addr += pg_size) {
1126 phys_addr_t phys_addr;
1127
1128 if (addr == end)
1129 goto map_end;
1130
1131 phys_addr = iommu_iova_to_phys(domain, addr);
1132 if (!phys_addr) {
1133 map_size += pg_size;
1134 continue;
1135 }
1136
1137map_end:
1138 if (map_size) {
1139 ret = iommu_map(domain, addr - map_size,
1140 addr - map_size, map_size,
1141 entry->prot, GFP_KERNEL);
1142 if (ret)
1143 goto out;
1144 map_size = 0;
1145 }
1146 }
1147
1148 }
1149
1150 if (!list_empty(&mappings) && iommu_is_dma_domain(domain))
1151 iommu_flush_iotlb_all(domain);
1152
1153out:
1154 iommu_put_resv_regions(dev, &mappings);
1155
1156 return ret;
1157}
1158
1159/* This is undone by __iommu_group_free_device() */
1160static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
1161 struct device *dev)
1162{
1163 int ret, i = 0;
1164 struct group_device *device;
1165
1166 device = kzalloc(sizeof(*device), GFP_KERNEL);
1167 if (!device)
1168 return ERR_PTR(-ENOMEM);
1169
1170 device->dev = dev;
1171
1172 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
1173 if (ret)
1174 goto err_free_device;
1175
1176 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
1177rename:
1178 if (!device->name) {
1179 ret = -ENOMEM;
1180 goto err_remove_link;
1181 }
1182
1183 ret = sysfs_create_link_nowarn(group->devices_kobj,
1184 &dev->kobj, device->name);
1185 if (ret) {
1186 if (ret == -EEXIST && i >= 0) {
1187 /*
1188 * Account for the slim chance of collision
1189 * and append an instance to the name.
1190 */
1191 kfree(device->name);
1192 device->name = kasprintf(GFP_KERNEL, "%s.%d",
1193 kobject_name(&dev->kobj), i++);
1194 goto rename;
1195 }
1196 goto err_free_name;
1197 }
1198
1199 trace_add_device_to_group(group->id, dev);
1200
1201 dev_info(dev, "Adding to iommu group %d\n", group->id);
1202
1203 return device;
1204
1205err_free_name:
1206 kfree(device->name);
1207err_remove_link:
1208 sysfs_remove_link(&dev->kobj, "iommu_group");
1209err_free_device:
1210 kfree(device);
1211 dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);
1212 return ERR_PTR(ret);
1213}
1214
1215/**
1216 * iommu_group_add_device - add a device to an iommu group
1217 * @group: the group into which to add the device (reference should be held)
1218 * @dev: the device
1219 *
1220 * This function is called by an iommu driver to add a device into a
1221 * group. Adding a device increments the group reference count.
1222 */
1223int iommu_group_add_device(struct iommu_group *group, struct device *dev)
1224{
1225 struct group_device *gdev;
1226
1227 gdev = iommu_group_alloc_device(group, dev);
1228 if (IS_ERR(gdev))
1229 return PTR_ERR(gdev);
1230
1231 iommu_group_ref_get(group);
1232 dev->iommu_group = group;
1233
1234 mutex_lock(&group->mutex);
1235 list_add_tail(&gdev->list, &group->devices);
1236 mutex_unlock(&group->mutex);
1237 return 0;
1238}
1239EXPORT_SYMBOL_GPL(iommu_group_add_device);
1240
1241/**
1242 * iommu_group_remove_device - remove a device from it's current group
1243 * @dev: device to be removed
1244 *
1245 * This function is called by an iommu driver to remove the device from
1246 * it's current group. This decrements the iommu group reference count.
1247 */
1248void iommu_group_remove_device(struct device *dev)
1249{
1250 struct iommu_group *group = dev->iommu_group;
1251
1252 if (!group)
1253 return;
1254
1255 dev_info(dev, "Removing from iommu group %d\n", group->id);
1256
1257 __iommu_group_remove_device(dev);
1258}
1259EXPORT_SYMBOL_GPL(iommu_group_remove_device);
1260
1261#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API)
1262/**
1263 * iommu_group_mutex_assert - Check device group mutex lock
1264 * @dev: the device that has group param set
1265 *
1266 * This function is called by an iommu driver to check whether it holds
1267 * group mutex lock for the given device or not.
1268 *
1269 * Note that this function must be called after device group param is set.
1270 */
1271void iommu_group_mutex_assert(struct device *dev)
1272{
1273 struct iommu_group *group = dev->iommu_group;
1274
1275 lockdep_assert_held(&group->mutex);
1276}
1277EXPORT_SYMBOL_GPL(iommu_group_mutex_assert);
1278#endif
1279
1280static struct device *iommu_group_first_dev(struct iommu_group *group)
1281{
1282 lockdep_assert_held(&group->mutex);
1283 return list_first_entry(&group->devices, struct group_device, list)->dev;
1284}
1285
1286/**
1287 * iommu_group_for_each_dev - iterate over each device in the group
1288 * @group: the group
1289 * @data: caller opaque data to be passed to callback function
1290 * @fn: caller supplied callback function
1291 *
1292 * This function is called by group users to iterate over group devices.
1293 * Callers should hold a reference count to the group during callback.
1294 * The group->mutex is held across callbacks, which will block calls to
1295 * iommu_group_add/remove_device.
1296 */
1297int iommu_group_for_each_dev(struct iommu_group *group, void *data,
1298 int (*fn)(struct device *, void *))
1299{
1300 struct group_device *device;
1301 int ret = 0;
1302
1303 mutex_lock(&group->mutex);
1304 for_each_group_device(group, device) {
1305 ret = fn(device->dev, data);
1306 if (ret)
1307 break;
1308 }
1309 mutex_unlock(&group->mutex);
1310
1311 return ret;
1312}
1313EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
1314
1315/**
1316 * iommu_group_get - Return the group for a device and increment reference
1317 * @dev: get the group that this device belongs to
1318 *
1319 * This function is called by iommu drivers and users to get the group
1320 * for the specified device. If found, the group is returned and the group
1321 * reference in incremented, else NULL.
1322 */
1323struct iommu_group *iommu_group_get(struct device *dev)
1324{
1325 struct iommu_group *group = dev->iommu_group;
1326
1327 if (group)
1328 kobject_get(group->devices_kobj);
1329
1330 return group;
1331}
1332EXPORT_SYMBOL_GPL(iommu_group_get);
1333
1334/**
1335 * iommu_group_ref_get - Increment reference on a group
1336 * @group: the group to use, must not be NULL
1337 *
1338 * This function is called by iommu drivers to take additional references on an
1339 * existing group. Returns the given group for convenience.
1340 */
1341struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
1342{
1343 kobject_get(group->devices_kobj);
1344 return group;
1345}
1346EXPORT_SYMBOL_GPL(iommu_group_ref_get);
1347
1348/**
1349 * iommu_group_put - Decrement group reference
1350 * @group: the group to use
1351 *
1352 * This function is called by iommu drivers and users to release the
1353 * iommu group. Once the reference count is zero, the group is released.
1354 */
1355void iommu_group_put(struct iommu_group *group)
1356{
1357 if (group)
1358 kobject_put(group->devices_kobj);
1359}
1360EXPORT_SYMBOL_GPL(iommu_group_put);
1361
1362/**
1363 * iommu_group_id - Return ID for a group
1364 * @group: the group to ID
1365 *
1366 * Return the unique ID for the group matching the sysfs group number.
1367 */
1368int iommu_group_id(struct iommu_group *group)
1369{
1370 return group->id;
1371}
1372EXPORT_SYMBOL_GPL(iommu_group_id);
1373
1374static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1375 unsigned long *devfns);
1376
1377/*
1378 * To consider a PCI device isolated, we require ACS to support Source
1379 * Validation, Request Redirection, Completer Redirection, and Upstream
1380 * Forwarding. This effectively means that devices cannot spoof their
1381 * requester ID, requests and completions cannot be redirected, and all
1382 * transactions are forwarded upstream, even as it passes through a
1383 * bridge where the target device is downstream.
1384 */
1385#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
1386
1387/*
1388 * For multifunction devices which are not isolated from each other, find
1389 * all the other non-isolated functions and look for existing groups. For
1390 * each function, we also need to look for aliases to or from other devices
1391 * that may already have a group.
1392 */
1393static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
1394 unsigned long *devfns)
1395{
1396 struct pci_dev *tmp = NULL;
1397 struct iommu_group *group;
1398
1399 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
1400 return NULL;
1401
1402 for_each_pci_dev(tmp) {
1403 if (tmp == pdev || tmp->bus != pdev->bus ||
1404 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
1405 pci_acs_enabled(tmp, REQ_ACS_FLAGS))
1406 continue;
1407
1408 group = get_pci_alias_group(tmp, devfns);
1409 if (group) {
1410 pci_dev_put(tmp);
1411 return group;
1412 }
1413 }
1414
1415 return NULL;
1416}
1417
1418/*
1419 * Look for aliases to or from the given device for existing groups. DMA
1420 * aliases are only supported on the same bus, therefore the search
1421 * space is quite small (especially since we're really only looking at pcie
1422 * device, and therefore only expect multiple slots on the root complex or
1423 * downstream switch ports). It's conceivable though that a pair of
1424 * multifunction devices could have aliases between them that would cause a
1425 * loop. To prevent this, we use a bitmap to track where we've been.
1426 */
1427static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1428 unsigned long *devfns)
1429{
1430 struct pci_dev *tmp = NULL;
1431 struct iommu_group *group;
1432
1433 if (test_and_set_bit(pdev->devfn & 0xff, devfns))
1434 return NULL;
1435
1436 group = iommu_group_get(&pdev->dev);
1437 if (group)
1438 return group;
1439
1440 for_each_pci_dev(tmp) {
1441 if (tmp == pdev || tmp->bus != pdev->bus)
1442 continue;
1443
1444 /* We alias them or they alias us */
1445 if (pci_devs_are_dma_aliases(pdev, tmp)) {
1446 group = get_pci_alias_group(tmp, devfns);
1447 if (group) {
1448 pci_dev_put(tmp);
1449 return group;
1450 }
1451
1452 group = get_pci_function_alias_group(tmp, devfns);
1453 if (group) {
1454 pci_dev_put(tmp);
1455 return group;
1456 }
1457 }
1458 }
1459
1460 return NULL;
1461}
1462
1463struct group_for_pci_data {
1464 struct pci_dev *pdev;
1465 struct iommu_group *group;
1466};
1467
1468/*
1469 * DMA alias iterator callback, return the last seen device. Stop and return
1470 * the IOMMU group if we find one along the way.
1471 */
1472static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
1473{
1474 struct group_for_pci_data *data = opaque;
1475
1476 data->pdev = pdev;
1477 data->group = iommu_group_get(&pdev->dev);
1478
1479 return data->group != NULL;
1480}
1481
1482/*
1483 * Generic device_group call-back function. It just allocates one
1484 * iommu-group per device.
1485 */
1486struct iommu_group *generic_device_group(struct device *dev)
1487{
1488 return iommu_group_alloc();
1489}
1490EXPORT_SYMBOL_GPL(generic_device_group);
1491
1492/*
1493 * Generic device_group call-back function. It just allocates one
1494 * iommu-group per iommu driver instance shared by every device
1495 * probed by that iommu driver.
1496 */
1497struct iommu_group *generic_single_device_group(struct device *dev)
1498{
1499 struct iommu_device *iommu = dev->iommu->iommu_dev;
1500
1501 if (!iommu->singleton_group) {
1502 struct iommu_group *group;
1503
1504 group = iommu_group_alloc();
1505 if (IS_ERR(group))
1506 return group;
1507 iommu->singleton_group = group;
1508 }
1509 return iommu_group_ref_get(iommu->singleton_group);
1510}
1511EXPORT_SYMBOL_GPL(generic_single_device_group);
1512
1513/*
1514 * Use standard PCI bus topology, isolation features, and DMA alias quirks
1515 * to find or create an IOMMU group for a device.
1516 */
1517struct iommu_group *pci_device_group(struct device *dev)
1518{
1519 struct pci_dev *pdev = to_pci_dev(dev);
1520 struct group_for_pci_data data;
1521 struct pci_bus *bus;
1522 struct iommu_group *group = NULL;
1523 u64 devfns[4] = { 0 };
1524
1525 if (WARN_ON(!dev_is_pci(dev)))
1526 return ERR_PTR(-EINVAL);
1527
1528 /*
1529 * Find the upstream DMA alias for the device. A device must not
1530 * be aliased due to topology in order to have its own IOMMU group.
1531 * If we find an alias along the way that already belongs to a
1532 * group, use it.
1533 */
1534 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))
1535 return data.group;
1536
1537 pdev = data.pdev;
1538
1539 /*
1540 * Continue upstream from the point of minimum IOMMU granularity
1541 * due to aliases to the point where devices are protected from
1542 * peer-to-peer DMA by PCI ACS. Again, if we find an existing
1543 * group, use it.
1544 */
1545 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {
1546 if (!bus->self)
1547 continue;
1548
1549 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
1550 break;
1551
1552 pdev = bus->self;
1553
1554 group = iommu_group_get(&pdev->dev);
1555 if (group)
1556 return group;
1557 }
1558
1559 /*
1560 * Look for existing groups on device aliases. If we alias another
1561 * device or another device aliases us, use the same group.
1562 */
1563 group = get_pci_alias_group(pdev, (unsigned long *)devfns);
1564 if (group)
1565 return group;
1566
1567 /*
1568 * Look for existing groups on non-isolated functions on the same
1569 * slot and aliases of those funcions, if any. No need to clear
1570 * the search bitmap, the tested devfns are still valid.
1571 */
1572 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
1573 if (group)
1574 return group;
1575
1576 /* No shared group found, allocate new */
1577 return iommu_group_alloc();
1578}
1579EXPORT_SYMBOL_GPL(pci_device_group);
1580
1581/* Get the IOMMU group for device on fsl-mc bus */
1582struct iommu_group *fsl_mc_device_group(struct device *dev)
1583{
1584 struct device *cont_dev = fsl_mc_cont_dev(dev);
1585 struct iommu_group *group;
1586
1587 group = iommu_group_get(cont_dev);
1588 if (!group)
1589 group = iommu_group_alloc();
1590 return group;
1591}
1592EXPORT_SYMBOL_GPL(fsl_mc_device_group);
1593
1594static struct iommu_domain *
1595__iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
1596{
1597 if (group->default_domain && group->default_domain->type == req_type)
1598 return group->default_domain;
1599 return __iommu_group_domain_alloc(group, req_type);
1600}
1601
1602/*
1603 * req_type of 0 means "auto" which means to select a domain based on
1604 * iommu_def_domain_type or what the driver actually supports.
1605 */
1606static struct iommu_domain *
1607iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
1608{
1609 const struct iommu_ops *ops = dev_iommu_ops(iommu_group_first_dev(group));
1610 struct iommu_domain *dom;
1611
1612 lockdep_assert_held(&group->mutex);
1613
1614 /*
1615 * Allow legacy drivers to specify the domain that will be the default
1616 * domain. This should always be either an IDENTITY/BLOCKED/PLATFORM
1617 * domain. Do not use in new drivers.
1618 */
1619 if (ops->default_domain) {
1620 if (req_type != ops->default_domain->type)
1621 return ERR_PTR(-EINVAL);
1622 return ops->default_domain;
1623 }
1624
1625 if (req_type)
1626 return __iommu_group_alloc_default_domain(group, req_type);
1627
1628 /* The driver gave no guidance on what type to use, try the default */
1629 dom = __iommu_group_alloc_default_domain(group, iommu_def_domain_type);
1630 if (!IS_ERR(dom))
1631 return dom;
1632
1633 /* Otherwise IDENTITY and DMA_FQ defaults will try DMA */
1634 if (iommu_def_domain_type == IOMMU_DOMAIN_DMA)
1635 return ERR_PTR(-EINVAL);
1636 dom = __iommu_group_alloc_default_domain(group, IOMMU_DOMAIN_DMA);
1637 if (IS_ERR(dom))
1638 return dom;
1639
1640 pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA",
1641 iommu_def_domain_type, group->name);
1642 return dom;
1643}
1644
1645struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
1646{
1647 return group->default_domain;
1648}
1649
1650static int probe_iommu_group(struct device *dev, void *data)
1651{
1652 struct list_head *group_list = data;
1653 int ret;
1654
1655 mutex_lock(&iommu_probe_device_lock);
1656 ret = __iommu_probe_device(dev, group_list);
1657 mutex_unlock(&iommu_probe_device_lock);
1658 if (ret == -ENODEV)
1659 ret = 0;
1660
1661 return ret;
1662}
1663
1664static int iommu_bus_notifier(struct notifier_block *nb,
1665 unsigned long action, void *data)
1666{
1667 struct device *dev = data;
1668
1669 if (action == BUS_NOTIFY_ADD_DEVICE) {
1670 int ret;
1671
1672 ret = iommu_probe_device(dev);
1673 return (ret) ? NOTIFY_DONE : NOTIFY_OK;
1674 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
1675 iommu_release_device(dev);
1676 return NOTIFY_OK;
1677 }
1678
1679 return 0;
1680}
1681
1682/*
1683 * Combine the driver's chosen def_domain_type across all the devices in a
1684 * group. Drivers must give a consistent result.
1685 */
1686static int iommu_get_def_domain_type(struct iommu_group *group,
1687 struct device *dev, int cur_type)
1688{
1689 const struct iommu_ops *ops = dev_iommu_ops(dev);
1690 int type;
1691
1692 if (ops->default_domain) {
1693 /*
1694 * Drivers that declare a global static default_domain will
1695 * always choose that.
1696 */
1697 type = ops->default_domain->type;
1698 } else {
1699 if (ops->def_domain_type)
1700 type = ops->def_domain_type(dev);
1701 else
1702 return cur_type;
1703 }
1704 if (!type || cur_type == type)
1705 return cur_type;
1706 if (!cur_type)
1707 return type;
1708
1709 dev_err_ratelimited(
1710 dev,
1711 "IOMMU driver error, requesting conflicting def_domain_type, %s and %s, for devices in group %u.\n",
1712 iommu_domain_type_str(cur_type), iommu_domain_type_str(type),
1713 group->id);
1714
1715 /*
1716 * Try to recover, drivers are allowed to force IDENITY or DMA, IDENTITY
1717 * takes precedence.
1718 */
1719 if (type == IOMMU_DOMAIN_IDENTITY)
1720 return type;
1721 return cur_type;
1722}
1723
1724/*
1725 * A target_type of 0 will select the best domain type. 0 can be returned in
1726 * this case meaning the global default should be used.
1727 */
1728static int iommu_get_default_domain_type(struct iommu_group *group,
1729 int target_type)
1730{
1731 struct device *untrusted = NULL;
1732 struct group_device *gdev;
1733 int driver_type = 0;
1734
1735 lockdep_assert_held(&group->mutex);
1736
1737 /*
1738 * ARM32 drivers supporting CONFIG_ARM_DMA_USE_IOMMU can declare an
1739 * identity_domain and it will automatically become their default
1740 * domain. Later on ARM_DMA_USE_IOMMU will install its UNMANAGED domain.
1741 * Override the selection to IDENTITY.
1742 */
1743 if (IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)) {
1744 static_assert(!(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) &&
1745 IS_ENABLED(CONFIG_IOMMU_DMA)));
1746 driver_type = IOMMU_DOMAIN_IDENTITY;
1747 }
1748
1749 for_each_group_device(group, gdev) {
1750 driver_type = iommu_get_def_domain_type(group, gdev->dev,
1751 driver_type);
1752
1753 if (dev_is_pci(gdev->dev) && to_pci_dev(gdev->dev)->untrusted) {
1754 /*
1755 * No ARM32 using systems will set untrusted, it cannot
1756 * work.
1757 */
1758 if (WARN_ON(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)))
1759 return -1;
1760 untrusted = gdev->dev;
1761 }
1762 }
1763
1764 /*
1765 * If the common dma ops are not selected in kconfig then we cannot use
1766 * IOMMU_DOMAIN_DMA at all. Force IDENTITY if nothing else has been
1767 * selected.
1768 */
1769 if (!IS_ENABLED(CONFIG_IOMMU_DMA)) {
1770 if (WARN_ON(driver_type == IOMMU_DOMAIN_DMA))
1771 return -1;
1772 if (!driver_type)
1773 driver_type = IOMMU_DOMAIN_IDENTITY;
1774 }
1775
1776 if (untrusted) {
1777 if (driver_type && driver_type != IOMMU_DOMAIN_DMA) {
1778 dev_err_ratelimited(
1779 untrusted,
1780 "Device is not trusted, but driver is overriding group %u to %s, refusing to probe.\n",
1781 group->id, iommu_domain_type_str(driver_type));
1782 return -1;
1783 }
1784 driver_type = IOMMU_DOMAIN_DMA;
1785 }
1786
1787 if (target_type) {
1788 if (driver_type && target_type != driver_type)
1789 return -1;
1790 return target_type;
1791 }
1792 return driver_type;
1793}
1794
1795static void iommu_group_do_probe_finalize(struct device *dev)
1796{
1797 const struct iommu_ops *ops = dev_iommu_ops(dev);
1798
1799 if (ops->probe_finalize)
1800 ops->probe_finalize(dev);
1801}
1802
1803int bus_iommu_probe(const struct bus_type *bus)
1804{
1805 struct iommu_group *group, *next;
1806 LIST_HEAD(group_list);
1807 int ret;
1808
1809 ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group);
1810 if (ret)
1811 return ret;
1812
1813 list_for_each_entry_safe(group, next, &group_list, entry) {
1814 struct group_device *gdev;
1815
1816 mutex_lock(&group->mutex);
1817
1818 /* Remove item from the list */
1819 list_del_init(&group->entry);
1820
1821 /*
1822 * We go to the trouble of deferred default domain creation so
1823 * that the cross-group default domain type and the setup of the
1824 * IOMMU_RESV_DIRECT will work correctly in non-hotpug scenarios.
1825 */
1826 ret = iommu_setup_default_domain(group, 0);
1827 if (ret) {
1828 mutex_unlock(&group->mutex);
1829 return ret;
1830 }
1831 mutex_unlock(&group->mutex);
1832
1833 /*
1834 * FIXME: Mis-locked because the ops->probe_finalize() call-back
1835 * of some IOMMU drivers calls arm_iommu_attach_device() which
1836 * in-turn might call back into IOMMU core code, where it tries
1837 * to take group->mutex, resulting in a deadlock.
1838 */
1839 for_each_group_device(group, gdev)
1840 iommu_group_do_probe_finalize(gdev->dev);
1841 }
1842
1843 return 0;
1844}
1845
1846/**
1847 * iommu_present() - make platform-specific assumptions about an IOMMU
1848 * @bus: bus to check
1849 *
1850 * Do not use this function. You want device_iommu_mapped() instead.
1851 *
1852 * Return: true if some IOMMU is present and aware of devices on the given bus;
1853 * in general it may not be the only IOMMU, and it may not have anything to do
1854 * with whatever device you are ultimately interested in.
1855 */
1856bool iommu_present(const struct bus_type *bus)
1857{
1858 bool ret = false;
1859
1860 for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
1861 if (iommu_buses[i] == bus) {
1862 spin_lock(&iommu_device_lock);
1863 ret = !list_empty(&iommu_device_list);
1864 spin_unlock(&iommu_device_lock);
1865 }
1866 }
1867 return ret;
1868}
1869EXPORT_SYMBOL_GPL(iommu_present);
1870
1871/**
1872 * device_iommu_capable() - check for a general IOMMU capability
1873 * @dev: device to which the capability would be relevant, if available
1874 * @cap: IOMMU capability
1875 *
1876 * Return: true if an IOMMU is present and supports the given capability
1877 * for the given device, otherwise false.
1878 */
1879bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
1880{
1881 const struct iommu_ops *ops;
1882
1883 if (!dev_has_iommu(dev))
1884 return false;
1885
1886 ops = dev_iommu_ops(dev);
1887 if (!ops->capable)
1888 return false;
1889
1890 return ops->capable(dev, cap);
1891}
1892EXPORT_SYMBOL_GPL(device_iommu_capable);
1893
1894/**
1895 * iommu_group_has_isolated_msi() - Compute msi_device_has_isolated_msi()
1896 * for a group
1897 * @group: Group to query
1898 *
1899 * IOMMU groups should not have differing values of
1900 * msi_device_has_isolated_msi() for devices in a group. However nothing
1901 * directly prevents this, so ensure mistakes don't result in isolation failures
1902 * by checking that all the devices are the same.
1903 */
1904bool iommu_group_has_isolated_msi(struct iommu_group *group)
1905{
1906 struct group_device *group_dev;
1907 bool ret = true;
1908
1909 mutex_lock(&group->mutex);
1910 for_each_group_device(group, group_dev)
1911 ret &= msi_device_has_isolated_msi(group_dev->dev);
1912 mutex_unlock(&group->mutex);
1913 return ret;
1914}
1915EXPORT_SYMBOL_GPL(iommu_group_has_isolated_msi);
1916
1917/**
1918 * iommu_set_fault_handler() - set a fault handler for an iommu domain
1919 * @domain: iommu domain
1920 * @handler: fault handler
1921 * @token: user data, will be passed back to the fault handler
1922 *
1923 * This function should be used by IOMMU users which want to be notified
1924 * whenever an IOMMU fault happens.
1925 *
1926 * The fault handler itself should return 0 on success, and an appropriate
1927 * error code otherwise.
1928 */
1929void iommu_set_fault_handler(struct iommu_domain *domain,
1930 iommu_fault_handler_t handler,
1931 void *token)
1932{
1933 BUG_ON(!domain);
1934
1935 domain->handler = handler;
1936 domain->handler_token = token;
1937}
1938EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
1939
1940static struct iommu_domain *__iommu_domain_alloc(const struct iommu_ops *ops,
1941 struct device *dev,
1942 unsigned int type)
1943{
1944 struct iommu_domain *domain;
1945 unsigned int alloc_type = type & IOMMU_DOMAIN_ALLOC_FLAGS;
1946
1947 if (alloc_type == IOMMU_DOMAIN_IDENTITY && ops->identity_domain)
1948 return ops->identity_domain;
1949 else if (alloc_type == IOMMU_DOMAIN_BLOCKED && ops->blocked_domain)
1950 return ops->blocked_domain;
1951 else if (type & __IOMMU_DOMAIN_PAGING && ops->domain_alloc_paging)
1952 domain = ops->domain_alloc_paging(dev);
1953 else if (ops->domain_alloc)
1954 domain = ops->domain_alloc(alloc_type);
1955 else
1956 return ERR_PTR(-EOPNOTSUPP);
1957
1958 /*
1959 * Many domain_alloc ops now return ERR_PTR, make things easier for the
1960 * driver by accepting ERR_PTR from all domain_alloc ops instead of
1961 * having two rules.
1962 */
1963 if (IS_ERR(domain))
1964 return domain;
1965 if (!domain)
1966 return ERR_PTR(-ENOMEM);
1967
1968 domain->type = type;
1969 domain->owner = ops;
1970 /*
1971 * If not already set, assume all sizes by default; the driver
1972 * may override this later
1973 */
1974 if (!domain->pgsize_bitmap)
1975 domain->pgsize_bitmap = ops->pgsize_bitmap;
1976
1977 if (!domain->ops)
1978 domain->ops = ops->default_domain_ops;
1979
1980 if (iommu_is_dma_domain(domain)) {
1981 int rc;
1982
1983 rc = iommu_get_dma_cookie(domain);
1984 if (rc) {
1985 iommu_domain_free(domain);
1986 return ERR_PTR(rc);
1987 }
1988 }
1989 return domain;
1990}
1991
1992static struct iommu_domain *
1993__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type)
1994{
1995 struct device *dev = iommu_group_first_dev(group);
1996
1997 return __iommu_domain_alloc(dev_iommu_ops(dev), dev, type);
1998}
1999
2000static int __iommu_domain_alloc_dev(struct device *dev, void *data)
2001{
2002 const struct iommu_ops **ops = data;
2003
2004 if (!dev_has_iommu(dev))
2005 return 0;
2006
2007 if (WARN_ONCE(*ops && *ops != dev_iommu_ops(dev),
2008 "Multiple IOMMU drivers present for bus %s, which the public IOMMU API can't fully support yet. You will still need to disable one or more for this to work, sorry!\n",
2009 dev_bus_name(dev)))
2010 return -EBUSY;
2011
2012 *ops = dev_iommu_ops(dev);
2013 return 0;
2014}
2015
2016struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
2017{
2018 const struct iommu_ops *ops = NULL;
2019 int err = bus_for_each_dev(bus, NULL, &ops, __iommu_domain_alloc_dev);
2020 struct iommu_domain *domain;
2021
2022 if (err || !ops)
2023 return NULL;
2024
2025 domain = __iommu_domain_alloc(ops, NULL, IOMMU_DOMAIN_UNMANAGED);
2026 if (IS_ERR(domain))
2027 return NULL;
2028 return domain;
2029}
2030EXPORT_SYMBOL_GPL(iommu_domain_alloc);
2031
2032void iommu_domain_free(struct iommu_domain *domain)
2033{
2034 if (domain->type == IOMMU_DOMAIN_SVA)
2035 mmdrop(domain->mm);
2036 iommu_put_dma_cookie(domain);
2037 if (domain->ops->free)
2038 domain->ops->free(domain);
2039}
2040EXPORT_SYMBOL_GPL(iommu_domain_free);
2041
2042/*
2043 * Put the group's domain back to the appropriate core-owned domain - either the
2044 * standard kernel-mode DMA configuration or an all-DMA-blocked domain.
2045 */
2046static void __iommu_group_set_core_domain(struct iommu_group *group)
2047{
2048 struct iommu_domain *new_domain;
2049
2050 if (group->owner)
2051 new_domain = group->blocking_domain;
2052 else
2053 new_domain = group->default_domain;
2054
2055 __iommu_group_set_domain_nofail(group, new_domain);
2056}
2057
2058static int __iommu_attach_device(struct iommu_domain *domain,
2059 struct device *dev)
2060{
2061 int ret;
2062
2063 if (unlikely(domain->ops->attach_dev == NULL))
2064 return -ENODEV;
2065
2066 ret = domain->ops->attach_dev(domain, dev);
2067 if (ret)
2068 return ret;
2069 dev->iommu->attach_deferred = 0;
2070 trace_attach_device_to_domain(dev);
2071 return 0;
2072}
2073
2074/**
2075 * iommu_attach_device - Attach an IOMMU domain to a device
2076 * @domain: IOMMU domain to attach
2077 * @dev: Device that will be attached
2078 *
2079 * Returns 0 on success and error code on failure
2080 *
2081 * Note that EINVAL can be treated as a soft failure, indicating
2082 * that certain configuration of the domain is incompatible with
2083 * the device. In this case attaching a different domain to the
2084 * device may succeed.
2085 */
2086int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
2087{
2088 /* Caller must be a probed driver on dev */
2089 struct iommu_group *group = dev->iommu_group;
2090 int ret;
2091
2092 if (!group)
2093 return -ENODEV;
2094
2095 /*
2096 * Lock the group to make sure the device-count doesn't
2097 * change while we are attaching
2098 */
2099 mutex_lock(&group->mutex);
2100 ret = -EINVAL;
2101 if (list_count_nodes(&group->devices) != 1)
2102 goto out_unlock;
2103
2104 ret = __iommu_attach_group(domain, group);
2105
2106out_unlock:
2107 mutex_unlock(&group->mutex);
2108 return ret;
2109}
2110EXPORT_SYMBOL_GPL(iommu_attach_device);
2111
2112int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
2113{
2114 if (dev->iommu && dev->iommu->attach_deferred)
2115 return __iommu_attach_device(domain, dev);
2116
2117 return 0;
2118}
2119
2120void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
2121{
2122 /* Caller must be a probed driver on dev */
2123 struct iommu_group *group = dev->iommu_group;
2124
2125 if (!group)
2126 return;
2127
2128 mutex_lock(&group->mutex);
2129 if (WARN_ON(domain != group->domain) ||
2130 WARN_ON(list_count_nodes(&group->devices) != 1))
2131 goto out_unlock;
2132 __iommu_group_set_core_domain(group);
2133
2134out_unlock:
2135 mutex_unlock(&group->mutex);
2136}
2137EXPORT_SYMBOL_GPL(iommu_detach_device);
2138
2139struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
2140{
2141 /* Caller must be a probed driver on dev */
2142 struct iommu_group *group = dev->iommu_group;
2143
2144 if (!group)
2145 return NULL;
2146
2147 return group->domain;
2148}
2149EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
2150
2151/*
2152 * For IOMMU_DOMAIN_DMA implementations which already provide their own
2153 * guarantees that the group and its default domain are valid and correct.
2154 */
2155struct iommu_domain *iommu_get_dma_domain(struct device *dev)
2156{
2157 return dev->iommu_group->default_domain;
2158}
2159
2160static int __iommu_attach_group(struct iommu_domain *domain,
2161 struct iommu_group *group)
2162{
2163 struct device *dev;
2164
2165 if (group->domain && group->domain != group->default_domain &&
2166 group->domain != group->blocking_domain)
2167 return -EBUSY;
2168
2169 dev = iommu_group_first_dev(group);
2170 if (!dev_has_iommu(dev) || dev_iommu_ops(dev) != domain->owner)
2171 return -EINVAL;
2172
2173 return __iommu_group_set_domain(group, domain);
2174}
2175
2176/**
2177 * iommu_attach_group - Attach an IOMMU domain to an IOMMU group
2178 * @domain: IOMMU domain to attach
2179 * @group: IOMMU group that will be attached
2180 *
2181 * Returns 0 on success and error code on failure
2182 *
2183 * Note that EINVAL can be treated as a soft failure, indicating
2184 * that certain configuration of the domain is incompatible with
2185 * the group. In this case attaching a different domain to the
2186 * group may succeed.
2187 */
2188int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
2189{
2190 int ret;
2191
2192 mutex_lock(&group->mutex);
2193 ret = __iommu_attach_group(domain, group);
2194 mutex_unlock(&group->mutex);
2195
2196 return ret;
2197}
2198EXPORT_SYMBOL_GPL(iommu_attach_group);
2199
2200/**
2201 * iommu_group_replace_domain - replace the domain that a group is attached to
2202 * @new_domain: new IOMMU domain to replace with
2203 * @group: IOMMU group that will be attached to the new domain
2204 *
2205 * This API allows the group to switch domains without being forced to go to
2206 * the blocking domain in-between.
2207 *
2208 * If the currently attached domain is a core domain (e.g. a default_domain),
2209 * it will act just like the iommu_attach_group().
2210 */
2211int iommu_group_replace_domain(struct iommu_group *group,
2212 struct iommu_domain *new_domain)
2213{
2214 int ret;
2215
2216 if (!new_domain)
2217 return -EINVAL;
2218
2219 mutex_lock(&group->mutex);
2220 ret = __iommu_group_set_domain(group, new_domain);
2221 mutex_unlock(&group->mutex);
2222 return ret;
2223}
2224EXPORT_SYMBOL_NS_GPL(iommu_group_replace_domain, IOMMUFD_INTERNAL);
2225
2226static int __iommu_device_set_domain(struct iommu_group *group,
2227 struct device *dev,
2228 struct iommu_domain *new_domain,
2229 unsigned int flags)
2230{
2231 int ret;
2232
2233 /*
2234 * If the device requires IOMMU_RESV_DIRECT then we cannot allow
2235 * the blocking domain to be attached as it does not contain the
2236 * required 1:1 mapping. This test effectively excludes the device
2237 * being used with iommu_group_claim_dma_owner() which will block
2238 * vfio and iommufd as well.
2239 */
2240 if (dev->iommu->require_direct &&
2241 (new_domain->type == IOMMU_DOMAIN_BLOCKED ||
2242 new_domain == group->blocking_domain)) {
2243 dev_warn(dev,
2244 "Firmware has requested this device have a 1:1 IOMMU mapping, rejecting configuring the device without a 1:1 mapping. Contact your platform vendor.\n");
2245 return -EINVAL;
2246 }
2247
2248 if (dev->iommu->attach_deferred) {
2249 if (new_domain == group->default_domain)
2250 return 0;
2251 dev->iommu->attach_deferred = 0;
2252 }
2253
2254 ret = __iommu_attach_device(new_domain, dev);
2255 if (ret) {
2256 /*
2257 * If we have a blocking domain then try to attach that in hopes
2258 * of avoiding a UAF. Modern drivers should implement blocking
2259 * domains as global statics that cannot fail.
2260 */
2261 if ((flags & IOMMU_SET_DOMAIN_MUST_SUCCEED) &&
2262 group->blocking_domain &&
2263 group->blocking_domain != new_domain)
2264 __iommu_attach_device(group->blocking_domain, dev);
2265 return ret;
2266 }
2267 return 0;
2268}
2269
2270/*
2271 * If 0 is returned the group's domain is new_domain. If an error is returned
2272 * then the group's domain will be set back to the existing domain unless
2273 * IOMMU_SET_DOMAIN_MUST_SUCCEED, otherwise an error is returned and the group's
2274 * domains is left inconsistent. This is a driver bug to fail attach with a
2275 * previously good domain. We try to avoid a kernel UAF because of this.
2276 *
2277 * IOMMU groups are really the natural working unit of the IOMMU, but the IOMMU
2278 * API works on domains and devices. Bridge that gap by iterating over the
2279 * devices in a group. Ideally we'd have a single device which represents the
2280 * requestor ID of the group, but we also allow IOMMU drivers to create policy
2281 * defined minimum sets, where the physical hardware may be able to distiguish
2282 * members, but we wish to group them at a higher level (ex. untrusted
2283 * multi-function PCI devices). Thus we attach each device.
2284 */
2285static int __iommu_group_set_domain_internal(struct iommu_group *group,
2286 struct iommu_domain *new_domain,
2287 unsigned int flags)
2288{
2289 struct group_device *last_gdev;
2290 struct group_device *gdev;
2291 int result;
2292 int ret;
2293
2294 lockdep_assert_held(&group->mutex);
2295
2296 if (group->domain == new_domain)
2297 return 0;
2298
2299 if (WARN_ON(!new_domain))
2300 return -EINVAL;
2301
2302 /*
2303 * Changing the domain is done by calling attach_dev() on the new
2304 * domain. This switch does not have to be atomic and DMA can be
2305 * discarded during the transition. DMA must only be able to access
2306 * either new_domain or group->domain, never something else.
2307 */
2308 result = 0;
2309 for_each_group_device(group, gdev) {
2310 ret = __iommu_device_set_domain(group, gdev->dev, new_domain,
2311 flags);
2312 if (ret) {
2313 result = ret;
2314 /*
2315 * Keep trying the other devices in the group. If a
2316 * driver fails attach to an otherwise good domain, and
2317 * does not support blocking domains, it should at least
2318 * drop its reference on the current domain so we don't
2319 * UAF.
2320 */
2321 if (flags & IOMMU_SET_DOMAIN_MUST_SUCCEED)
2322 continue;
2323 goto err_revert;
2324 }
2325 }
2326 group->domain = new_domain;
2327 return result;
2328
2329err_revert:
2330 /*
2331 * This is called in error unwind paths. A well behaved driver should
2332 * always allow us to attach to a domain that was already attached.
2333 */
2334 last_gdev = gdev;
2335 for_each_group_device(group, gdev) {
2336 /*
2337 * A NULL domain can happen only for first probe, in which case
2338 * we leave group->domain as NULL and let release clean
2339 * everything up.
2340 */
2341 if (group->domain)
2342 WARN_ON(__iommu_device_set_domain(
2343 group, gdev->dev, group->domain,
2344 IOMMU_SET_DOMAIN_MUST_SUCCEED));
2345 if (gdev == last_gdev)
2346 break;
2347 }
2348 return ret;
2349}
2350
2351void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
2352{
2353 mutex_lock(&group->mutex);
2354 __iommu_group_set_core_domain(group);
2355 mutex_unlock(&group->mutex);
2356}
2357EXPORT_SYMBOL_GPL(iommu_detach_group);
2358
2359phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2360{
2361 if (domain->type == IOMMU_DOMAIN_IDENTITY)
2362 return iova;
2363
2364 if (domain->type == IOMMU_DOMAIN_BLOCKED)
2365 return 0;
2366
2367 return domain->ops->iova_to_phys(domain, iova);
2368}
2369EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
2370
2371static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova,
2372 phys_addr_t paddr, size_t size, size_t *count)
2373{
2374 unsigned int pgsize_idx, pgsize_idx_next;
2375 unsigned long pgsizes;
2376 size_t offset, pgsize, pgsize_next;
2377 unsigned long addr_merge = paddr | iova;
2378
2379 /* Page sizes supported by the hardware and small enough for @size */
2380 pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0);
2381
2382 /* Constrain the page sizes further based on the maximum alignment */
2383 if (likely(addr_merge))
2384 pgsizes &= GENMASK(__ffs(addr_merge), 0);
2385
2386 /* Make sure we have at least one suitable page size */
2387 BUG_ON(!pgsizes);
2388
2389 /* Pick the biggest page size remaining */
2390 pgsize_idx = __fls(pgsizes);
2391 pgsize = BIT(pgsize_idx);
2392 if (!count)
2393 return pgsize;
2394
2395 /* Find the next biggest support page size, if it exists */
2396 pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
2397 if (!pgsizes)
2398 goto out_set_count;
2399
2400 pgsize_idx_next = __ffs(pgsizes);
2401 pgsize_next = BIT(pgsize_idx_next);
2402
2403 /*
2404 * There's no point trying a bigger page size unless the virtual
2405 * and physical addresses are similarly offset within the larger page.
2406 */
2407 if ((iova ^ paddr) & (pgsize_next - 1))
2408 goto out_set_count;
2409
2410 /* Calculate the offset to the next page size alignment boundary */
2411 offset = pgsize_next - (addr_merge & (pgsize_next - 1));
2412
2413 /*
2414 * If size is big enough to accommodate the larger page, reduce
2415 * the number of smaller pages.
2416 */
2417 if (offset + pgsize_next <= size)
2418 size = offset;
2419
2420out_set_count:
2421 *count = size >> pgsize_idx;
2422 return pgsize;
2423}
2424
2425static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
2426 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2427{
2428 const struct iommu_domain_ops *ops = domain->ops;
2429 unsigned long orig_iova = iova;
2430 unsigned int min_pagesz;
2431 size_t orig_size = size;
2432 phys_addr_t orig_paddr = paddr;
2433 int ret = 0;
2434
2435 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2436 return -EINVAL;
2437
2438 if (WARN_ON(!ops->map_pages || domain->pgsize_bitmap == 0UL))
2439 return -ENODEV;
2440
2441 /* find out the minimum page size supported */
2442 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2443
2444 /*
2445 * both the virtual address and the physical one, as well as
2446 * the size of the mapping, must be aligned (at least) to the
2447 * size of the smallest page supported by the hardware
2448 */
2449 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
2450 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
2451 iova, &paddr, size, min_pagesz);
2452 return -EINVAL;
2453 }
2454
2455 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
2456
2457 while (size) {
2458 size_t pgsize, count, mapped = 0;
2459
2460 pgsize = iommu_pgsize(domain, iova, paddr, size, &count);
2461
2462 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
2463 iova, &paddr, pgsize, count);
2464 ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
2465 gfp, &mapped);
2466 /*
2467 * Some pages may have been mapped, even if an error occurred,
2468 * so we should account for those so they can be unmapped.
2469 */
2470 size -= mapped;
2471
2472 if (ret)
2473 break;
2474
2475 iova += mapped;
2476 paddr += mapped;
2477 }
2478
2479 /* unroll mapping in case something went wrong */
2480 if (ret)
2481 iommu_unmap(domain, orig_iova, orig_size - size);
2482 else
2483 trace_map(orig_iova, orig_paddr, orig_size);
2484
2485 return ret;
2486}
2487
2488int iommu_map(struct iommu_domain *domain, unsigned long iova,
2489 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2490{
2491 const struct iommu_domain_ops *ops = domain->ops;
2492 int ret;
2493
2494 might_sleep_if(gfpflags_allow_blocking(gfp));
2495
2496 /* Discourage passing strange GFP flags */
2497 if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 |
2498 __GFP_HIGHMEM)))
2499 return -EINVAL;
2500
2501 ret = __iommu_map(domain, iova, paddr, size, prot, gfp);
2502 if (ret == 0 && ops->iotlb_sync_map) {
2503 ret = ops->iotlb_sync_map(domain, iova, size);
2504 if (ret)
2505 goto out_err;
2506 }
2507
2508 return ret;
2509
2510out_err:
2511 /* undo mappings already done */
2512 iommu_unmap(domain, iova, size);
2513
2514 return ret;
2515}
2516EXPORT_SYMBOL_GPL(iommu_map);
2517
2518static size_t __iommu_unmap(struct iommu_domain *domain,
2519 unsigned long iova, size_t size,
2520 struct iommu_iotlb_gather *iotlb_gather)
2521{
2522 const struct iommu_domain_ops *ops = domain->ops;
2523 size_t unmapped_page, unmapped = 0;
2524 unsigned long orig_iova = iova;
2525 unsigned int min_pagesz;
2526
2527 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2528 return 0;
2529
2530 if (WARN_ON(!ops->unmap_pages || domain->pgsize_bitmap == 0UL))
2531 return 0;
2532
2533 /* find out the minimum page size supported */
2534 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2535
2536 /*
2537 * The virtual address, as well as the size of the mapping, must be
2538 * aligned (at least) to the size of the smallest page supported
2539 * by the hardware
2540 */
2541 if (!IS_ALIGNED(iova | size, min_pagesz)) {
2542 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
2543 iova, size, min_pagesz);
2544 return 0;
2545 }
2546
2547 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
2548
2549 /*
2550 * Keep iterating until we either unmap 'size' bytes (or more)
2551 * or we hit an area that isn't mapped.
2552 */
2553 while (unmapped < size) {
2554 size_t pgsize, count;
2555
2556 pgsize = iommu_pgsize(domain, iova, iova, size - unmapped, &count);
2557 unmapped_page = ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather);
2558 if (!unmapped_page)
2559 break;
2560
2561 pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
2562 iova, unmapped_page);
2563
2564 iova += unmapped_page;
2565 unmapped += unmapped_page;
2566 }
2567
2568 trace_unmap(orig_iova, size, unmapped);
2569 return unmapped;
2570}
2571
2572size_t iommu_unmap(struct iommu_domain *domain,
2573 unsigned long iova, size_t size)
2574{
2575 struct iommu_iotlb_gather iotlb_gather;
2576 size_t ret;
2577
2578 iommu_iotlb_gather_init(&iotlb_gather);
2579 ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
2580 iommu_iotlb_sync(domain, &iotlb_gather);
2581
2582 return ret;
2583}
2584EXPORT_SYMBOL_GPL(iommu_unmap);
2585
2586size_t iommu_unmap_fast(struct iommu_domain *domain,
2587 unsigned long iova, size_t size,
2588 struct iommu_iotlb_gather *iotlb_gather)
2589{
2590 return __iommu_unmap(domain, iova, size, iotlb_gather);
2591}
2592EXPORT_SYMBOL_GPL(iommu_unmap_fast);
2593
2594ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
2595 struct scatterlist *sg, unsigned int nents, int prot,
2596 gfp_t gfp)
2597{
2598 const struct iommu_domain_ops *ops = domain->ops;
2599 size_t len = 0, mapped = 0;
2600 phys_addr_t start;
2601 unsigned int i = 0;
2602 int ret;
2603
2604 might_sleep_if(gfpflags_allow_blocking(gfp));
2605
2606 /* Discourage passing strange GFP flags */
2607 if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 |
2608 __GFP_HIGHMEM)))
2609 return -EINVAL;
2610
2611 while (i <= nents) {
2612 phys_addr_t s_phys = sg_phys(sg);
2613
2614 if (len && s_phys != start + len) {
2615 ret = __iommu_map(domain, iova + mapped, start,
2616 len, prot, gfp);
2617
2618 if (ret)
2619 goto out_err;
2620
2621 mapped += len;
2622 len = 0;
2623 }
2624
2625 if (sg_dma_is_bus_address(sg))
2626 goto next;
2627
2628 if (len) {
2629 len += sg->length;
2630 } else {
2631 len = sg->length;
2632 start = s_phys;
2633 }
2634
2635next:
2636 if (++i < nents)
2637 sg = sg_next(sg);
2638 }
2639
2640 if (ops->iotlb_sync_map) {
2641 ret = ops->iotlb_sync_map(domain, iova, mapped);
2642 if (ret)
2643 goto out_err;
2644 }
2645 return mapped;
2646
2647out_err:
2648 /* undo mappings already done */
2649 iommu_unmap(domain, iova, mapped);
2650
2651 return ret;
2652}
2653EXPORT_SYMBOL_GPL(iommu_map_sg);
2654
2655/**
2656 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
2657 * @domain: the iommu domain where the fault has happened
2658 * @dev: the device where the fault has happened
2659 * @iova: the faulting address
2660 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...)
2661 *
2662 * This function should be called by the low-level IOMMU implementations
2663 * whenever IOMMU faults happen, to allow high-level users, that are
2664 * interested in such events, to know about them.
2665 *
2666 * This event may be useful for several possible use cases:
2667 * - mere logging of the event
2668 * - dynamic TLB/PTE loading
2669 * - if restarting of the faulting device is required
2670 *
2671 * Returns 0 on success and an appropriate error code otherwise (if dynamic
2672 * PTE/TLB loading will one day be supported, implementations will be able
2673 * to tell whether it succeeded or not according to this return value).
2674 *
2675 * Specifically, -ENOSYS is returned if a fault handler isn't installed
2676 * (though fault handlers can also return -ENOSYS, in case they want to
2677 * elicit the default behavior of the IOMMU drivers).
2678 */
2679int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
2680 unsigned long iova, int flags)
2681{
2682 int ret = -ENOSYS;
2683
2684 /*
2685 * if upper layers showed interest and installed a fault handler,
2686 * invoke it.
2687 */
2688 if (domain->handler)
2689 ret = domain->handler(domain, dev, iova, flags,
2690 domain->handler_token);
2691
2692 trace_io_page_fault(dev, iova, flags);
2693 return ret;
2694}
2695EXPORT_SYMBOL_GPL(report_iommu_fault);
2696
2697static int __init iommu_init(void)
2698{
2699 iommu_group_kset = kset_create_and_add("iommu_groups",
2700 NULL, kernel_kobj);
2701 BUG_ON(!iommu_group_kset);
2702
2703 iommu_debugfs_setup();
2704
2705 return 0;
2706}
2707core_initcall(iommu_init);
2708
2709int iommu_enable_nesting(struct iommu_domain *domain)
2710{
2711 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
2712 return -EINVAL;
2713 if (!domain->ops->enable_nesting)
2714 return -EINVAL;
2715 return domain->ops->enable_nesting(domain);
2716}
2717EXPORT_SYMBOL_GPL(iommu_enable_nesting);
2718
2719int iommu_set_pgtable_quirks(struct iommu_domain *domain,
2720 unsigned long quirk)
2721{
2722 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
2723 return -EINVAL;
2724 if (!domain->ops->set_pgtable_quirks)
2725 return -EINVAL;
2726 return domain->ops->set_pgtable_quirks(domain, quirk);
2727}
2728EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);
2729
2730/**
2731 * iommu_get_resv_regions - get reserved regions
2732 * @dev: device for which to get reserved regions
2733 * @list: reserved region list for device
2734 *
2735 * This returns a list of reserved IOVA regions specific to this device.
2736 * A domain user should not map IOVA in these ranges.
2737 */
2738void iommu_get_resv_regions(struct device *dev, struct list_head *list)
2739{
2740 const struct iommu_ops *ops = dev_iommu_ops(dev);
2741
2742 if (ops->get_resv_regions)
2743 ops->get_resv_regions(dev, list);
2744}
2745EXPORT_SYMBOL_GPL(iommu_get_resv_regions);
2746
2747/**
2748 * iommu_put_resv_regions - release reserved regions
2749 * @dev: device for which to free reserved regions
2750 * @list: reserved region list for device
2751 *
2752 * This releases a reserved region list acquired by iommu_get_resv_regions().
2753 */
2754void iommu_put_resv_regions(struct device *dev, struct list_head *list)
2755{
2756 struct iommu_resv_region *entry, *next;
2757
2758 list_for_each_entry_safe(entry, next, list, list) {
2759 if (entry->free)
2760 entry->free(dev, entry);
2761 else
2762 kfree(entry);
2763 }
2764}
2765EXPORT_SYMBOL(iommu_put_resv_regions);
2766
2767struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
2768 size_t length, int prot,
2769 enum iommu_resv_type type,
2770 gfp_t gfp)
2771{
2772 struct iommu_resv_region *region;
2773
2774 region = kzalloc(sizeof(*region), gfp);
2775 if (!region)
2776 return NULL;
2777
2778 INIT_LIST_HEAD(®ion->list);
2779 region->start = start;
2780 region->length = length;
2781 region->prot = prot;
2782 region->type = type;
2783 return region;
2784}
2785EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
2786
2787void iommu_set_default_passthrough(bool cmd_line)
2788{
2789 if (cmd_line)
2790 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
2791 iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
2792}
2793
2794void iommu_set_default_translated(bool cmd_line)
2795{
2796 if (cmd_line)
2797 iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
2798 iommu_def_domain_type = IOMMU_DOMAIN_DMA;
2799}
2800
2801bool iommu_default_passthrough(void)
2802{
2803 return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
2804}
2805EXPORT_SYMBOL_GPL(iommu_default_passthrough);
2806
2807const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode)
2808{
2809 const struct iommu_ops *ops = NULL;
2810 struct iommu_device *iommu;
2811
2812 spin_lock(&iommu_device_lock);
2813 list_for_each_entry(iommu, &iommu_device_list, list)
2814 if (iommu->fwnode == fwnode) {
2815 ops = iommu->ops;
2816 break;
2817 }
2818 spin_unlock(&iommu_device_lock);
2819 return ops;
2820}
2821
2822int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
2823 const struct iommu_ops *ops)
2824{
2825 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2826
2827 if (fwspec)
2828 return ops == fwspec->ops ? 0 : -EINVAL;
2829
2830 if (!dev_iommu_get(dev))
2831 return -ENOMEM;
2832
2833 /* Preallocate for the overwhelmingly common case of 1 ID */
2834 fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL);
2835 if (!fwspec)
2836 return -ENOMEM;
2837
2838 of_node_get(to_of_node(iommu_fwnode));
2839 fwspec->iommu_fwnode = iommu_fwnode;
2840 fwspec->ops = ops;
2841 dev_iommu_fwspec_set(dev, fwspec);
2842 return 0;
2843}
2844EXPORT_SYMBOL_GPL(iommu_fwspec_init);
2845
2846void iommu_fwspec_free(struct device *dev)
2847{
2848 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2849
2850 if (fwspec) {
2851 fwnode_handle_put(fwspec->iommu_fwnode);
2852 kfree(fwspec);
2853 dev_iommu_fwspec_set(dev, NULL);
2854 }
2855}
2856EXPORT_SYMBOL_GPL(iommu_fwspec_free);
2857
2858int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids)
2859{
2860 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2861 int i, new_num;
2862
2863 if (!fwspec)
2864 return -EINVAL;
2865
2866 new_num = fwspec->num_ids + num_ids;
2867 if (new_num > 1) {
2868 fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num),
2869 GFP_KERNEL);
2870 if (!fwspec)
2871 return -ENOMEM;
2872
2873 dev_iommu_fwspec_set(dev, fwspec);
2874 }
2875
2876 for (i = 0; i < num_ids; i++)
2877 fwspec->ids[fwspec->num_ids + i] = ids[i];
2878
2879 fwspec->num_ids = new_num;
2880 return 0;
2881}
2882EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
2883
2884/*
2885 * Per device IOMMU features.
2886 */
2887int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
2888{
2889 if (dev_has_iommu(dev)) {
2890 const struct iommu_ops *ops = dev_iommu_ops(dev);
2891
2892 if (ops->dev_enable_feat)
2893 return ops->dev_enable_feat(dev, feat);
2894 }
2895
2896 return -ENODEV;
2897}
2898EXPORT_SYMBOL_GPL(iommu_dev_enable_feature);
2899
2900/*
2901 * The device drivers should do the necessary cleanups before calling this.
2902 */
2903int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
2904{
2905 if (dev_has_iommu(dev)) {
2906 const struct iommu_ops *ops = dev_iommu_ops(dev);
2907
2908 if (ops->dev_disable_feat)
2909 return ops->dev_disable_feat(dev, feat);
2910 }
2911
2912 return -EBUSY;
2913}
2914EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
2915
2916/**
2917 * iommu_setup_default_domain - Set the default_domain for the group
2918 * @group: Group to change
2919 * @target_type: Domain type to set as the default_domain
2920 *
2921 * Allocate a default domain and set it as the current domain on the group. If
2922 * the group already has a default domain it will be changed to the target_type.
2923 * When target_type is 0 the default domain is selected based on driver and
2924 * system preferences.
2925 */
2926static int iommu_setup_default_domain(struct iommu_group *group,
2927 int target_type)
2928{
2929 struct iommu_domain *old_dom = group->default_domain;
2930 struct group_device *gdev;
2931 struct iommu_domain *dom;
2932 bool direct_failed;
2933 int req_type;
2934 int ret;
2935
2936 lockdep_assert_held(&group->mutex);
2937
2938 req_type = iommu_get_default_domain_type(group, target_type);
2939 if (req_type < 0)
2940 return -EINVAL;
2941
2942 dom = iommu_group_alloc_default_domain(group, req_type);
2943 if (IS_ERR(dom))
2944 return PTR_ERR(dom);
2945
2946 if (group->default_domain == dom)
2947 return 0;
2948
2949 /*
2950 * IOMMU_RESV_DIRECT and IOMMU_RESV_DIRECT_RELAXABLE regions must be
2951 * mapped before their device is attached, in order to guarantee
2952 * continuity with any FW activity
2953 */
2954 direct_failed = false;
2955 for_each_group_device(group, gdev) {
2956 if (iommu_create_device_direct_mappings(dom, gdev->dev)) {
2957 direct_failed = true;
2958 dev_warn_once(
2959 gdev->dev->iommu->iommu_dev->dev,
2960 "IOMMU driver was not able to establish FW requested direct mapping.");
2961 }
2962 }
2963
2964 /* We must set default_domain early for __iommu_device_set_domain */
2965 group->default_domain = dom;
2966 if (!group->domain) {
2967 /*
2968 * Drivers are not allowed to fail the first domain attach.
2969 * The only way to recover from this is to fail attaching the
2970 * iommu driver and call ops->release_device. Put the domain
2971 * in group->default_domain so it is freed after.
2972 */
2973 ret = __iommu_group_set_domain_internal(
2974 group, dom, IOMMU_SET_DOMAIN_MUST_SUCCEED);
2975 if (WARN_ON(ret))
2976 goto out_free_old;
2977 } else {
2978 ret = __iommu_group_set_domain(group, dom);
2979 if (ret)
2980 goto err_restore_def_domain;
2981 }
2982
2983 /*
2984 * Drivers are supposed to allow mappings to be installed in a domain
2985 * before device attachment, but some don't. Hack around this defect by
2986 * trying again after attaching. If this happens it means the device
2987 * will not continuously have the IOMMU_RESV_DIRECT map.
2988 */
2989 if (direct_failed) {
2990 for_each_group_device(group, gdev) {
2991 ret = iommu_create_device_direct_mappings(dom, gdev->dev);
2992 if (ret)
2993 goto err_restore_domain;
2994 }
2995 }
2996
2997out_free_old:
2998 if (old_dom)
2999 iommu_domain_free(old_dom);
3000 return ret;
3001
3002err_restore_domain:
3003 if (old_dom)
3004 __iommu_group_set_domain_internal(
3005 group, old_dom, IOMMU_SET_DOMAIN_MUST_SUCCEED);
3006err_restore_def_domain:
3007 if (old_dom) {
3008 iommu_domain_free(dom);
3009 group->default_domain = old_dom;
3010 }
3011 return ret;
3012}
3013
3014/*
3015 * Changing the default domain through sysfs requires the users to unbind the
3016 * drivers from the devices in the iommu group, except for a DMA -> DMA-FQ
3017 * transition. Return failure if this isn't met.
3018 *
3019 * We need to consider the race between this and the device release path.
3020 * group->mutex is used here to guarantee that the device release path
3021 * will not be entered at the same time.
3022 */
3023static ssize_t iommu_group_store_type(struct iommu_group *group,
3024 const char *buf, size_t count)
3025{
3026 struct group_device *gdev;
3027 int ret, req_type;
3028
3029 if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO))
3030 return -EACCES;
3031
3032 if (WARN_ON(!group) || !group->default_domain)
3033 return -EINVAL;
3034
3035 if (sysfs_streq(buf, "identity"))
3036 req_type = IOMMU_DOMAIN_IDENTITY;
3037 else if (sysfs_streq(buf, "DMA"))
3038 req_type = IOMMU_DOMAIN_DMA;
3039 else if (sysfs_streq(buf, "DMA-FQ"))
3040 req_type = IOMMU_DOMAIN_DMA_FQ;
3041 else if (sysfs_streq(buf, "auto"))
3042 req_type = 0;
3043 else
3044 return -EINVAL;
3045
3046 mutex_lock(&group->mutex);
3047 /* We can bring up a flush queue without tearing down the domain. */
3048 if (req_type == IOMMU_DOMAIN_DMA_FQ &&
3049 group->default_domain->type == IOMMU_DOMAIN_DMA) {
3050 ret = iommu_dma_init_fq(group->default_domain);
3051 if (ret)
3052 goto out_unlock;
3053
3054 group->default_domain->type = IOMMU_DOMAIN_DMA_FQ;
3055 ret = count;
3056 goto out_unlock;
3057 }
3058
3059 /* Otherwise, ensure that device exists and no driver is bound. */
3060 if (list_empty(&group->devices) || group->owner_cnt) {
3061 ret = -EPERM;
3062 goto out_unlock;
3063 }
3064
3065 ret = iommu_setup_default_domain(group, req_type);
3066 if (ret)
3067 goto out_unlock;
3068
3069 /*
3070 * Release the mutex here because ops->probe_finalize() call-back of
3071 * some vendor IOMMU drivers calls arm_iommu_attach_device() which
3072 * in-turn might call back into IOMMU core code, where it tries to take
3073 * group->mutex, resulting in a deadlock.
3074 */
3075 mutex_unlock(&group->mutex);
3076
3077 /* Make sure dma_ops is appropriatley set */
3078 for_each_group_device(group, gdev)
3079 iommu_group_do_probe_finalize(gdev->dev);
3080 return count;
3081
3082out_unlock:
3083 mutex_unlock(&group->mutex);
3084 return ret ?: count;
3085}
3086
3087/**
3088 * iommu_device_use_default_domain() - Device driver wants to handle device
3089 * DMA through the kernel DMA API.
3090 * @dev: The device.
3091 *
3092 * The device driver about to bind @dev wants to do DMA through the kernel
3093 * DMA API. Return 0 if it is allowed, otherwise an error.
3094 */
3095int iommu_device_use_default_domain(struct device *dev)
3096{
3097 /* Caller is the driver core during the pre-probe path */
3098 struct iommu_group *group = dev->iommu_group;
3099 int ret = 0;
3100
3101 if (!group)
3102 return 0;
3103
3104 mutex_lock(&group->mutex);
3105 if (group->owner_cnt) {
3106 if (group->domain != group->default_domain || group->owner ||
3107 !xa_empty(&group->pasid_array)) {
3108 ret = -EBUSY;
3109 goto unlock_out;
3110 }
3111 }
3112
3113 group->owner_cnt++;
3114
3115unlock_out:
3116 mutex_unlock(&group->mutex);
3117 return ret;
3118}
3119
3120/**
3121 * iommu_device_unuse_default_domain() - Device driver stops handling device
3122 * DMA through the kernel DMA API.
3123 * @dev: The device.
3124 *
3125 * The device driver doesn't want to do DMA through kernel DMA API anymore.
3126 * It must be called after iommu_device_use_default_domain().
3127 */
3128void iommu_device_unuse_default_domain(struct device *dev)
3129{
3130 /* Caller is the driver core during the post-probe path */
3131 struct iommu_group *group = dev->iommu_group;
3132
3133 if (!group)
3134 return;
3135
3136 mutex_lock(&group->mutex);
3137 if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array)))
3138 group->owner_cnt--;
3139
3140 mutex_unlock(&group->mutex);
3141}
3142
3143static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
3144{
3145 struct iommu_domain *domain;
3146
3147 if (group->blocking_domain)
3148 return 0;
3149
3150 domain = __iommu_group_domain_alloc(group, IOMMU_DOMAIN_BLOCKED);
3151 if (IS_ERR(domain)) {
3152 /*
3153 * For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED
3154 * create an empty domain instead.
3155 */
3156 domain = __iommu_group_domain_alloc(group,
3157 IOMMU_DOMAIN_UNMANAGED);
3158 if (IS_ERR(domain))
3159 return PTR_ERR(domain);
3160 }
3161 group->blocking_domain = domain;
3162 return 0;
3163}
3164
3165static int __iommu_take_dma_ownership(struct iommu_group *group, void *owner)
3166{
3167 int ret;
3168
3169 if ((group->domain && group->domain != group->default_domain) ||
3170 !xa_empty(&group->pasid_array))
3171 return -EBUSY;
3172
3173 ret = __iommu_group_alloc_blocking_domain(group);
3174 if (ret)
3175 return ret;
3176 ret = __iommu_group_set_domain(group, group->blocking_domain);
3177 if (ret)
3178 return ret;
3179
3180 group->owner = owner;
3181 group->owner_cnt++;
3182 return 0;
3183}
3184
3185/**
3186 * iommu_group_claim_dma_owner() - Set DMA ownership of a group
3187 * @group: The group.
3188 * @owner: Caller specified pointer. Used for exclusive ownership.
3189 *
3190 * This is to support backward compatibility for vfio which manages the dma
3191 * ownership in iommu_group level. New invocations on this interface should be
3192 * prohibited. Only a single owner may exist for a group.
3193 */
3194int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
3195{
3196 int ret = 0;
3197
3198 if (WARN_ON(!owner))
3199 return -EINVAL;
3200
3201 mutex_lock(&group->mutex);
3202 if (group->owner_cnt) {
3203 ret = -EPERM;
3204 goto unlock_out;
3205 }
3206
3207 ret = __iommu_take_dma_ownership(group, owner);
3208unlock_out:
3209 mutex_unlock(&group->mutex);
3210
3211 return ret;
3212}
3213EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);
3214
3215/**
3216 * iommu_device_claim_dma_owner() - Set DMA ownership of a device
3217 * @dev: The device.
3218 * @owner: Caller specified pointer. Used for exclusive ownership.
3219 *
3220 * Claim the DMA ownership of a device. Multiple devices in the same group may
3221 * concurrently claim ownership if they present the same owner value. Returns 0
3222 * on success and error code on failure
3223 */
3224int iommu_device_claim_dma_owner(struct device *dev, void *owner)
3225{
3226 /* Caller must be a probed driver on dev */
3227 struct iommu_group *group = dev->iommu_group;
3228 int ret = 0;
3229
3230 if (WARN_ON(!owner))
3231 return -EINVAL;
3232
3233 if (!group)
3234 return -ENODEV;
3235
3236 mutex_lock(&group->mutex);
3237 if (group->owner_cnt) {
3238 if (group->owner != owner) {
3239 ret = -EPERM;
3240 goto unlock_out;
3241 }
3242 group->owner_cnt++;
3243 goto unlock_out;
3244 }
3245
3246 ret = __iommu_take_dma_ownership(group, owner);
3247unlock_out:
3248 mutex_unlock(&group->mutex);
3249 return ret;
3250}
3251EXPORT_SYMBOL_GPL(iommu_device_claim_dma_owner);
3252
3253static void __iommu_release_dma_ownership(struct iommu_group *group)
3254{
3255 if (WARN_ON(!group->owner_cnt || !group->owner ||
3256 !xa_empty(&group->pasid_array)))
3257 return;
3258
3259 group->owner_cnt = 0;
3260 group->owner = NULL;
3261 __iommu_group_set_domain_nofail(group, group->default_domain);
3262}
3263
3264/**
3265 * iommu_group_release_dma_owner() - Release DMA ownership of a group
3266 * @group: The group
3267 *
3268 * Release the DMA ownership claimed by iommu_group_claim_dma_owner().
3269 */
3270void iommu_group_release_dma_owner(struct iommu_group *group)
3271{
3272 mutex_lock(&group->mutex);
3273 __iommu_release_dma_ownership(group);
3274 mutex_unlock(&group->mutex);
3275}
3276EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);
3277
3278/**
3279 * iommu_device_release_dma_owner() - Release DMA ownership of a device
3280 * @dev: The device.
3281 *
3282 * Release the DMA ownership claimed by iommu_device_claim_dma_owner().
3283 */
3284void iommu_device_release_dma_owner(struct device *dev)
3285{
3286 /* Caller must be a probed driver on dev */
3287 struct iommu_group *group = dev->iommu_group;
3288
3289 mutex_lock(&group->mutex);
3290 if (group->owner_cnt > 1)
3291 group->owner_cnt--;
3292 else
3293 __iommu_release_dma_ownership(group);
3294 mutex_unlock(&group->mutex);
3295}
3296EXPORT_SYMBOL_GPL(iommu_device_release_dma_owner);
3297
3298/**
3299 * iommu_group_dma_owner_claimed() - Query group dma ownership status
3300 * @group: The group.
3301 *
3302 * This provides status query on a given group. It is racy and only for
3303 * non-binding status reporting.
3304 */
3305bool iommu_group_dma_owner_claimed(struct iommu_group *group)
3306{
3307 unsigned int user;
3308
3309 mutex_lock(&group->mutex);
3310 user = group->owner_cnt;
3311 mutex_unlock(&group->mutex);
3312
3313 return user;
3314}
3315EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
3316
3317static int __iommu_set_group_pasid(struct iommu_domain *domain,
3318 struct iommu_group *group, ioasid_t pasid)
3319{
3320 struct group_device *device, *last_gdev;
3321 int ret;
3322
3323 for_each_group_device(group, device) {
3324 ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
3325 if (ret)
3326 goto err_revert;
3327 }
3328
3329 return 0;
3330
3331err_revert:
3332 last_gdev = device;
3333 for_each_group_device(group, device) {
3334 const struct iommu_ops *ops = dev_iommu_ops(device->dev);
3335
3336 if (device == last_gdev)
3337 break;
3338 ops->remove_dev_pasid(device->dev, pasid);
3339 }
3340 return ret;
3341}
3342
3343static void __iommu_remove_group_pasid(struct iommu_group *group,
3344 ioasid_t pasid)
3345{
3346 struct group_device *device;
3347 const struct iommu_ops *ops;
3348
3349 for_each_group_device(group, device) {
3350 ops = dev_iommu_ops(device->dev);
3351 ops->remove_dev_pasid(device->dev, pasid);
3352 }
3353}
3354
3355/*
3356 * iommu_attach_device_pasid() - Attach a domain to pasid of device
3357 * @domain: the iommu domain.
3358 * @dev: the attached device.
3359 * @pasid: the pasid of the device.
3360 *
3361 * Return: 0 on success, or an error.
3362 */
3363int iommu_attach_device_pasid(struct iommu_domain *domain,
3364 struct device *dev, ioasid_t pasid)
3365{
3366 /* Caller must be a probed driver on dev */
3367 struct iommu_group *group = dev->iommu_group;
3368 struct group_device *device;
3369 void *curr;
3370 int ret;
3371
3372 if (!domain->ops->set_dev_pasid)
3373 return -EOPNOTSUPP;
3374
3375 if (!group)
3376 return -ENODEV;
3377
3378 if (!dev_has_iommu(dev) || dev_iommu_ops(dev) != domain->owner ||
3379 pasid == IOMMU_NO_PASID)
3380 return -EINVAL;
3381
3382 mutex_lock(&group->mutex);
3383 for_each_group_device(group, device) {
3384 if (pasid >= device->dev->iommu->max_pasids) {
3385 ret = -EINVAL;
3386 goto out_unlock;
3387 }
3388 }
3389
3390 curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
3391 if (curr) {
3392 ret = xa_err(curr) ? : -EBUSY;
3393 goto out_unlock;
3394 }
3395
3396 ret = __iommu_set_group_pasid(domain, group, pasid);
3397 if (ret)
3398 xa_erase(&group->pasid_array, pasid);
3399out_unlock:
3400 mutex_unlock(&group->mutex);
3401 return ret;
3402}
3403EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
3404
3405/*
3406 * iommu_detach_device_pasid() - Detach the domain from pasid of device
3407 * @domain: the iommu domain.
3408 * @dev: the attached device.
3409 * @pasid: the pasid of the device.
3410 *
3411 * The @domain must have been attached to @pasid of the @dev with
3412 * iommu_attach_device_pasid().
3413 */
3414void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
3415 ioasid_t pasid)
3416{
3417 /* Caller must be a probed driver on dev */
3418 struct iommu_group *group = dev->iommu_group;
3419
3420 mutex_lock(&group->mutex);
3421 __iommu_remove_group_pasid(group, pasid);
3422 WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
3423 mutex_unlock(&group->mutex);
3424}
3425EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
3426
3427/*
3428 * iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
3429 * @dev: the queried device
3430 * @pasid: the pasid of the device
3431 * @type: matched domain type, 0 for any match
3432 *
3433 * This is a variant of iommu_get_domain_for_dev(). It returns the existing
3434 * domain attached to pasid of a device. Callers must hold a lock around this
3435 * function, and both iommu_attach/detach_dev_pasid() whenever a domain of
3436 * type is being manipulated. This API does not internally resolve races with
3437 * attach/detach.
3438 *
3439 * Return: attached domain on success, NULL otherwise.
3440 */
3441struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
3442 ioasid_t pasid,
3443 unsigned int type)
3444{
3445 /* Caller must be a probed driver on dev */
3446 struct iommu_group *group = dev->iommu_group;
3447 struct iommu_domain *domain;
3448
3449 if (!group)
3450 return NULL;
3451
3452 xa_lock(&group->pasid_array);
3453 domain = xa_load(&group->pasid_array, pasid);
3454 if (type && domain && domain->type != type)
3455 domain = ERR_PTR(-EBUSY);
3456 xa_unlock(&group->pasid_array);
3457
3458 return domain;
3459}
3460EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
3461
3462ioasid_t iommu_alloc_global_pasid(struct device *dev)
3463{
3464 int ret;
3465
3466 /* max_pasids == 0 means that the device does not support PASID */
3467 if (!dev->iommu->max_pasids)
3468 return IOMMU_PASID_INVALID;
3469
3470 /*
3471 * max_pasids is set up by vendor driver based on number of PASID bits
3472 * supported but the IDA allocation is inclusive.
3473 */
3474 ret = ida_alloc_range(&iommu_global_pasid_ida, IOMMU_FIRST_GLOBAL_PASID,
3475 dev->iommu->max_pasids - 1, GFP_KERNEL);
3476 return ret < 0 ? IOMMU_PASID_INVALID : ret;
3477}
3478EXPORT_SYMBOL_GPL(iommu_alloc_global_pasid);
3479
3480void iommu_free_global_pasid(ioasid_t pasid)
3481{
3482 if (WARN_ON(pasid == IOMMU_PASID_INVALID))
3483 return;
3484
3485 ida_free(&iommu_global_pasid_ida, pasid);
3486}
3487EXPORT_SYMBOL_GPL(iommu_free_global_pasid);
1/*
2 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <jroedel@suse.de>
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */
18
19#define pr_fmt(fmt) "iommu: " fmt
20
21#include <linux/device.h>
22#include <linux/kernel.h>
23#include <linux/bug.h>
24#include <linux/types.h>
25#include <linux/module.h>
26#include <linux/slab.h>
27#include <linux/errno.h>
28#include <linux/iommu.h>
29#include <linux/idr.h>
30#include <linux/notifier.h>
31#include <linux/err.h>
32#include <linux/pci.h>
33#include <linux/bitops.h>
34#include <linux/property.h>
35#include <trace/events/iommu.h>
36
37static struct kset *iommu_group_kset;
38static DEFINE_IDA(iommu_group_ida);
39static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA;
40
41struct iommu_callback_data {
42 const struct iommu_ops *ops;
43};
44
45struct iommu_group {
46 struct kobject kobj;
47 struct kobject *devices_kobj;
48 struct list_head devices;
49 struct mutex mutex;
50 struct blocking_notifier_head notifier;
51 void *iommu_data;
52 void (*iommu_data_release)(void *iommu_data);
53 char *name;
54 int id;
55 struct iommu_domain *default_domain;
56 struct iommu_domain *domain;
57};
58
59struct group_device {
60 struct list_head list;
61 struct device *dev;
62 char *name;
63};
64
65struct iommu_group_attribute {
66 struct attribute attr;
67 ssize_t (*show)(struct iommu_group *group, char *buf);
68 ssize_t (*store)(struct iommu_group *group,
69 const char *buf, size_t count);
70};
71
72static const char * const iommu_group_resv_type_string[] = {
73 [IOMMU_RESV_DIRECT] = "direct",
74 [IOMMU_RESV_RESERVED] = "reserved",
75 [IOMMU_RESV_MSI] = "msi",
76 [IOMMU_RESV_SW_MSI] = "msi",
77};
78
79#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
80struct iommu_group_attribute iommu_group_attr_##_name = \
81 __ATTR(_name, _mode, _show, _store)
82
83#define to_iommu_group_attr(_attr) \
84 container_of(_attr, struct iommu_group_attribute, attr)
85#define to_iommu_group(_kobj) \
86 container_of(_kobj, struct iommu_group, kobj)
87
88static LIST_HEAD(iommu_device_list);
89static DEFINE_SPINLOCK(iommu_device_lock);
90
91int iommu_device_register(struct iommu_device *iommu)
92{
93 spin_lock(&iommu_device_lock);
94 list_add_tail(&iommu->list, &iommu_device_list);
95 spin_unlock(&iommu_device_lock);
96
97 return 0;
98}
99
100void iommu_device_unregister(struct iommu_device *iommu)
101{
102 spin_lock(&iommu_device_lock);
103 list_del(&iommu->list);
104 spin_unlock(&iommu_device_lock);
105}
106
107static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
108 unsigned type);
109static int __iommu_attach_device(struct iommu_domain *domain,
110 struct device *dev);
111static int __iommu_attach_group(struct iommu_domain *domain,
112 struct iommu_group *group);
113static void __iommu_detach_group(struct iommu_domain *domain,
114 struct iommu_group *group);
115
116static int __init iommu_set_def_domain_type(char *str)
117{
118 bool pt;
119
120 if (!str || strtobool(str, &pt))
121 return -EINVAL;
122
123 iommu_def_domain_type = pt ? IOMMU_DOMAIN_IDENTITY : IOMMU_DOMAIN_DMA;
124 return 0;
125}
126early_param("iommu.passthrough", iommu_set_def_domain_type);
127
128static ssize_t iommu_group_attr_show(struct kobject *kobj,
129 struct attribute *__attr, char *buf)
130{
131 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
132 struct iommu_group *group = to_iommu_group(kobj);
133 ssize_t ret = -EIO;
134
135 if (attr->show)
136 ret = attr->show(group, buf);
137 return ret;
138}
139
140static ssize_t iommu_group_attr_store(struct kobject *kobj,
141 struct attribute *__attr,
142 const char *buf, size_t count)
143{
144 struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
145 struct iommu_group *group = to_iommu_group(kobj);
146 ssize_t ret = -EIO;
147
148 if (attr->store)
149 ret = attr->store(group, buf, count);
150 return ret;
151}
152
153static const struct sysfs_ops iommu_group_sysfs_ops = {
154 .show = iommu_group_attr_show,
155 .store = iommu_group_attr_store,
156};
157
158static int iommu_group_create_file(struct iommu_group *group,
159 struct iommu_group_attribute *attr)
160{
161 return sysfs_create_file(&group->kobj, &attr->attr);
162}
163
164static void iommu_group_remove_file(struct iommu_group *group,
165 struct iommu_group_attribute *attr)
166{
167 sysfs_remove_file(&group->kobj, &attr->attr);
168}
169
170static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
171{
172 return sprintf(buf, "%s\n", group->name);
173}
174
175/**
176 * iommu_insert_resv_region - Insert a new region in the
177 * list of reserved regions.
178 * @new: new region to insert
179 * @regions: list of regions
180 *
181 * The new element is sorted by address with respect to the other
182 * regions of the same type. In case it overlaps with another
183 * region of the same type, regions are merged. In case it
184 * overlaps with another region of different type, regions are
185 * not merged.
186 */
187static int iommu_insert_resv_region(struct iommu_resv_region *new,
188 struct list_head *regions)
189{
190 struct iommu_resv_region *region;
191 phys_addr_t start = new->start;
192 phys_addr_t end = new->start + new->length - 1;
193 struct list_head *pos = regions->next;
194
195 while (pos != regions) {
196 struct iommu_resv_region *entry =
197 list_entry(pos, struct iommu_resv_region, list);
198 phys_addr_t a = entry->start;
199 phys_addr_t b = entry->start + entry->length - 1;
200 int type = entry->type;
201
202 if (end < a) {
203 goto insert;
204 } else if (start > b) {
205 pos = pos->next;
206 } else if ((start >= a) && (end <= b)) {
207 if (new->type == type)
208 goto done;
209 else
210 pos = pos->next;
211 } else {
212 if (new->type == type) {
213 phys_addr_t new_start = min(a, start);
214 phys_addr_t new_end = max(b, end);
215
216 list_del(&entry->list);
217 entry->start = new_start;
218 entry->length = new_end - new_start + 1;
219 iommu_insert_resv_region(entry, regions);
220 } else {
221 pos = pos->next;
222 }
223 }
224 }
225insert:
226 region = iommu_alloc_resv_region(new->start, new->length,
227 new->prot, new->type);
228 if (!region)
229 return -ENOMEM;
230
231 list_add_tail(®ion->list, pos);
232done:
233 return 0;
234}
235
236static int
237iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
238 struct list_head *group_resv_regions)
239{
240 struct iommu_resv_region *entry;
241 int ret = 0;
242
243 list_for_each_entry(entry, dev_resv_regions, list) {
244 ret = iommu_insert_resv_region(entry, group_resv_regions);
245 if (ret)
246 break;
247 }
248 return ret;
249}
250
251int iommu_get_group_resv_regions(struct iommu_group *group,
252 struct list_head *head)
253{
254 struct group_device *device;
255 int ret = 0;
256
257 mutex_lock(&group->mutex);
258 list_for_each_entry(device, &group->devices, list) {
259 struct list_head dev_resv_regions;
260
261 INIT_LIST_HEAD(&dev_resv_regions);
262 iommu_get_resv_regions(device->dev, &dev_resv_regions);
263 ret = iommu_insert_device_resv_regions(&dev_resv_regions, head);
264 iommu_put_resv_regions(device->dev, &dev_resv_regions);
265 if (ret)
266 break;
267 }
268 mutex_unlock(&group->mutex);
269 return ret;
270}
271EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
272
273static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
274 char *buf)
275{
276 struct iommu_resv_region *region, *next;
277 struct list_head group_resv_regions;
278 char *str = buf;
279
280 INIT_LIST_HEAD(&group_resv_regions);
281 iommu_get_group_resv_regions(group, &group_resv_regions);
282
283 list_for_each_entry_safe(region, next, &group_resv_regions, list) {
284 str += sprintf(str, "0x%016llx 0x%016llx %s\n",
285 (long long int)region->start,
286 (long long int)(region->start +
287 region->length - 1),
288 iommu_group_resv_type_string[region->type]);
289 kfree(region);
290 }
291
292 return (str - buf);
293}
294
295static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
296
297static IOMMU_GROUP_ATTR(reserved_regions, 0444,
298 iommu_group_show_resv_regions, NULL);
299
300static void iommu_group_release(struct kobject *kobj)
301{
302 struct iommu_group *group = to_iommu_group(kobj);
303
304 pr_debug("Releasing group %d\n", group->id);
305
306 if (group->iommu_data_release)
307 group->iommu_data_release(group->iommu_data);
308
309 ida_simple_remove(&iommu_group_ida, group->id);
310
311 if (group->default_domain)
312 iommu_domain_free(group->default_domain);
313
314 kfree(group->name);
315 kfree(group);
316}
317
318static struct kobj_type iommu_group_ktype = {
319 .sysfs_ops = &iommu_group_sysfs_ops,
320 .release = iommu_group_release,
321};
322
323/**
324 * iommu_group_alloc - Allocate a new group
325 * @name: Optional name to associate with group, visible in sysfs
326 *
327 * This function is called by an iommu driver to allocate a new iommu
328 * group. The iommu group represents the minimum granularity of the iommu.
329 * Upon successful return, the caller holds a reference to the supplied
330 * group in order to hold the group until devices are added. Use
331 * iommu_group_put() to release this extra reference count, allowing the
332 * group to be automatically reclaimed once it has no devices or external
333 * references.
334 */
335struct iommu_group *iommu_group_alloc(void)
336{
337 struct iommu_group *group;
338 int ret;
339
340 group = kzalloc(sizeof(*group), GFP_KERNEL);
341 if (!group)
342 return ERR_PTR(-ENOMEM);
343
344 group->kobj.kset = iommu_group_kset;
345 mutex_init(&group->mutex);
346 INIT_LIST_HEAD(&group->devices);
347 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
348
349 ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL);
350 if (ret < 0) {
351 kfree(group);
352 return ERR_PTR(ret);
353 }
354 group->id = ret;
355
356 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
357 NULL, "%d", group->id);
358 if (ret) {
359 ida_simple_remove(&iommu_group_ida, group->id);
360 kfree(group);
361 return ERR_PTR(ret);
362 }
363
364 group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
365 if (!group->devices_kobj) {
366 kobject_put(&group->kobj); /* triggers .release & free */
367 return ERR_PTR(-ENOMEM);
368 }
369
370 /*
371 * The devices_kobj holds a reference on the group kobject, so
372 * as long as that exists so will the group. We can therefore
373 * use the devices_kobj for reference counting.
374 */
375 kobject_put(&group->kobj);
376
377 ret = iommu_group_create_file(group,
378 &iommu_group_attr_reserved_regions);
379 if (ret)
380 return ERR_PTR(ret);
381
382 pr_debug("Allocated group %d\n", group->id);
383
384 return group;
385}
386EXPORT_SYMBOL_GPL(iommu_group_alloc);
387
388struct iommu_group *iommu_group_get_by_id(int id)
389{
390 struct kobject *group_kobj;
391 struct iommu_group *group;
392 const char *name;
393
394 if (!iommu_group_kset)
395 return NULL;
396
397 name = kasprintf(GFP_KERNEL, "%d", id);
398 if (!name)
399 return NULL;
400
401 group_kobj = kset_find_obj(iommu_group_kset, name);
402 kfree(name);
403
404 if (!group_kobj)
405 return NULL;
406
407 group = container_of(group_kobj, struct iommu_group, kobj);
408 BUG_ON(group->id != id);
409
410 kobject_get(group->devices_kobj);
411 kobject_put(&group->kobj);
412
413 return group;
414}
415EXPORT_SYMBOL_GPL(iommu_group_get_by_id);
416
417/**
418 * iommu_group_get_iommudata - retrieve iommu_data registered for a group
419 * @group: the group
420 *
421 * iommu drivers can store data in the group for use when doing iommu
422 * operations. This function provides a way to retrieve it. Caller
423 * should hold a group reference.
424 */
425void *iommu_group_get_iommudata(struct iommu_group *group)
426{
427 return group->iommu_data;
428}
429EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
430
431/**
432 * iommu_group_set_iommudata - set iommu_data for a group
433 * @group: the group
434 * @iommu_data: new data
435 * @release: release function for iommu_data
436 *
437 * iommu drivers can store data in the group for use when doing iommu
438 * operations. This function provides a way to set the data after
439 * the group has been allocated. Caller should hold a group reference.
440 */
441void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
442 void (*release)(void *iommu_data))
443{
444 group->iommu_data = iommu_data;
445 group->iommu_data_release = release;
446}
447EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
448
449/**
450 * iommu_group_set_name - set name for a group
451 * @group: the group
452 * @name: name
453 *
454 * Allow iommu driver to set a name for a group. When set it will
455 * appear in a name attribute file under the group in sysfs.
456 */
457int iommu_group_set_name(struct iommu_group *group, const char *name)
458{
459 int ret;
460
461 if (group->name) {
462 iommu_group_remove_file(group, &iommu_group_attr_name);
463 kfree(group->name);
464 group->name = NULL;
465 if (!name)
466 return 0;
467 }
468
469 group->name = kstrdup(name, GFP_KERNEL);
470 if (!group->name)
471 return -ENOMEM;
472
473 ret = iommu_group_create_file(group, &iommu_group_attr_name);
474 if (ret) {
475 kfree(group->name);
476 group->name = NULL;
477 return ret;
478 }
479
480 return 0;
481}
482EXPORT_SYMBOL_GPL(iommu_group_set_name);
483
484static int iommu_group_create_direct_mappings(struct iommu_group *group,
485 struct device *dev)
486{
487 struct iommu_domain *domain = group->default_domain;
488 struct iommu_resv_region *entry;
489 struct list_head mappings;
490 unsigned long pg_size;
491 int ret = 0;
492
493 if (!domain || domain->type != IOMMU_DOMAIN_DMA)
494 return 0;
495
496 BUG_ON(!domain->pgsize_bitmap);
497
498 pg_size = 1UL << __ffs(domain->pgsize_bitmap);
499 INIT_LIST_HEAD(&mappings);
500
501 iommu_get_resv_regions(dev, &mappings);
502
503 /* We need to consider overlapping regions for different devices */
504 list_for_each_entry(entry, &mappings, list) {
505 dma_addr_t start, end, addr;
506
507 if (domain->ops->apply_resv_region)
508 domain->ops->apply_resv_region(dev, domain, entry);
509
510 start = ALIGN(entry->start, pg_size);
511 end = ALIGN(entry->start + entry->length, pg_size);
512
513 if (entry->type != IOMMU_RESV_DIRECT)
514 continue;
515
516 for (addr = start; addr < end; addr += pg_size) {
517 phys_addr_t phys_addr;
518
519 phys_addr = iommu_iova_to_phys(domain, addr);
520 if (phys_addr)
521 continue;
522
523 ret = iommu_map(domain, addr, addr, pg_size, entry->prot);
524 if (ret)
525 goto out;
526 }
527
528 }
529
530 iommu_flush_tlb_all(domain);
531
532out:
533 iommu_put_resv_regions(dev, &mappings);
534
535 return ret;
536}
537
538/**
539 * iommu_group_add_device - add a device to an iommu group
540 * @group: the group into which to add the device (reference should be held)
541 * @dev: the device
542 *
543 * This function is called by an iommu driver to add a device into a
544 * group. Adding a device increments the group reference count.
545 */
546int iommu_group_add_device(struct iommu_group *group, struct device *dev)
547{
548 int ret, i = 0;
549 struct group_device *device;
550
551 device = kzalloc(sizeof(*device), GFP_KERNEL);
552 if (!device)
553 return -ENOMEM;
554
555 device->dev = dev;
556
557 ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
558 if (ret)
559 goto err_free_device;
560
561 device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
562rename:
563 if (!device->name) {
564 ret = -ENOMEM;
565 goto err_remove_link;
566 }
567
568 ret = sysfs_create_link_nowarn(group->devices_kobj,
569 &dev->kobj, device->name);
570 if (ret) {
571 if (ret == -EEXIST && i >= 0) {
572 /*
573 * Account for the slim chance of collision
574 * and append an instance to the name.
575 */
576 kfree(device->name);
577 device->name = kasprintf(GFP_KERNEL, "%s.%d",
578 kobject_name(&dev->kobj), i++);
579 goto rename;
580 }
581 goto err_free_name;
582 }
583
584 kobject_get(group->devices_kobj);
585
586 dev->iommu_group = group;
587
588 iommu_group_create_direct_mappings(group, dev);
589
590 mutex_lock(&group->mutex);
591 list_add_tail(&device->list, &group->devices);
592 if (group->domain)
593 ret = __iommu_attach_device(group->domain, dev);
594 mutex_unlock(&group->mutex);
595 if (ret)
596 goto err_put_group;
597
598 /* Notify any listeners about change to group. */
599 blocking_notifier_call_chain(&group->notifier,
600 IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev);
601
602 trace_add_device_to_group(group->id, dev);
603
604 pr_info("Adding device %s to group %d\n", dev_name(dev), group->id);
605
606 return 0;
607
608err_put_group:
609 mutex_lock(&group->mutex);
610 list_del(&device->list);
611 mutex_unlock(&group->mutex);
612 dev->iommu_group = NULL;
613 kobject_put(group->devices_kobj);
614err_free_name:
615 kfree(device->name);
616err_remove_link:
617 sysfs_remove_link(&dev->kobj, "iommu_group");
618err_free_device:
619 kfree(device);
620 pr_err("Failed to add device %s to group %d: %d\n", dev_name(dev), group->id, ret);
621 return ret;
622}
623EXPORT_SYMBOL_GPL(iommu_group_add_device);
624
625/**
626 * iommu_group_remove_device - remove a device from it's current group
627 * @dev: device to be removed
628 *
629 * This function is called by an iommu driver to remove the device from
630 * it's current group. This decrements the iommu group reference count.
631 */
632void iommu_group_remove_device(struct device *dev)
633{
634 struct iommu_group *group = dev->iommu_group;
635 struct group_device *tmp_device, *device = NULL;
636
637 pr_info("Removing device %s from group %d\n", dev_name(dev), group->id);
638
639 /* Pre-notify listeners that a device is being removed. */
640 blocking_notifier_call_chain(&group->notifier,
641 IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev);
642
643 mutex_lock(&group->mutex);
644 list_for_each_entry(tmp_device, &group->devices, list) {
645 if (tmp_device->dev == dev) {
646 device = tmp_device;
647 list_del(&device->list);
648 break;
649 }
650 }
651 mutex_unlock(&group->mutex);
652
653 if (!device)
654 return;
655
656 sysfs_remove_link(group->devices_kobj, device->name);
657 sysfs_remove_link(&dev->kobj, "iommu_group");
658
659 trace_remove_device_from_group(group->id, dev);
660
661 kfree(device->name);
662 kfree(device);
663 dev->iommu_group = NULL;
664 kobject_put(group->devices_kobj);
665}
666EXPORT_SYMBOL_GPL(iommu_group_remove_device);
667
668static int iommu_group_device_count(struct iommu_group *group)
669{
670 struct group_device *entry;
671 int ret = 0;
672
673 list_for_each_entry(entry, &group->devices, list)
674 ret++;
675
676 return ret;
677}
678
679/**
680 * iommu_group_for_each_dev - iterate over each device in the group
681 * @group: the group
682 * @data: caller opaque data to be passed to callback function
683 * @fn: caller supplied callback function
684 *
685 * This function is called by group users to iterate over group devices.
686 * Callers should hold a reference count to the group during callback.
687 * The group->mutex is held across callbacks, which will block calls to
688 * iommu_group_add/remove_device.
689 */
690static int __iommu_group_for_each_dev(struct iommu_group *group, void *data,
691 int (*fn)(struct device *, void *))
692{
693 struct group_device *device;
694 int ret = 0;
695
696 list_for_each_entry(device, &group->devices, list) {
697 ret = fn(device->dev, data);
698 if (ret)
699 break;
700 }
701 return ret;
702}
703
704
705int iommu_group_for_each_dev(struct iommu_group *group, void *data,
706 int (*fn)(struct device *, void *))
707{
708 int ret;
709
710 mutex_lock(&group->mutex);
711 ret = __iommu_group_for_each_dev(group, data, fn);
712 mutex_unlock(&group->mutex);
713
714 return ret;
715}
716EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
717
718/**
719 * iommu_group_get - Return the group for a device and increment reference
720 * @dev: get the group that this device belongs to
721 *
722 * This function is called by iommu drivers and users to get the group
723 * for the specified device. If found, the group is returned and the group
724 * reference in incremented, else NULL.
725 */
726struct iommu_group *iommu_group_get(struct device *dev)
727{
728 struct iommu_group *group = dev->iommu_group;
729
730 if (group)
731 kobject_get(group->devices_kobj);
732
733 return group;
734}
735EXPORT_SYMBOL_GPL(iommu_group_get);
736
737/**
738 * iommu_group_ref_get - Increment reference on a group
739 * @group: the group to use, must not be NULL
740 *
741 * This function is called by iommu drivers to take additional references on an
742 * existing group. Returns the given group for convenience.
743 */
744struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
745{
746 kobject_get(group->devices_kobj);
747 return group;
748}
749
750/**
751 * iommu_group_put - Decrement group reference
752 * @group: the group to use
753 *
754 * This function is called by iommu drivers and users to release the
755 * iommu group. Once the reference count is zero, the group is released.
756 */
757void iommu_group_put(struct iommu_group *group)
758{
759 if (group)
760 kobject_put(group->devices_kobj);
761}
762EXPORT_SYMBOL_GPL(iommu_group_put);
763
764/**
765 * iommu_group_register_notifier - Register a notifier for group changes
766 * @group: the group to watch
767 * @nb: notifier block to signal
768 *
769 * This function allows iommu group users to track changes in a group.
770 * See include/linux/iommu.h for actions sent via this notifier. Caller
771 * should hold a reference to the group throughout notifier registration.
772 */
773int iommu_group_register_notifier(struct iommu_group *group,
774 struct notifier_block *nb)
775{
776 return blocking_notifier_chain_register(&group->notifier, nb);
777}
778EXPORT_SYMBOL_GPL(iommu_group_register_notifier);
779
780/**
781 * iommu_group_unregister_notifier - Unregister a notifier
782 * @group: the group to watch
783 * @nb: notifier block to signal
784 *
785 * Unregister a previously registered group notifier block.
786 */
787int iommu_group_unregister_notifier(struct iommu_group *group,
788 struct notifier_block *nb)
789{
790 return blocking_notifier_chain_unregister(&group->notifier, nb);
791}
792EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier);
793
794/**
795 * iommu_group_id - Return ID for a group
796 * @group: the group to ID
797 *
798 * Return the unique ID for the group matching the sysfs group number.
799 */
800int iommu_group_id(struct iommu_group *group)
801{
802 return group->id;
803}
804EXPORT_SYMBOL_GPL(iommu_group_id);
805
806static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
807 unsigned long *devfns);
808
809/*
810 * To consider a PCI device isolated, we require ACS to support Source
811 * Validation, Request Redirection, Completer Redirection, and Upstream
812 * Forwarding. This effectively means that devices cannot spoof their
813 * requester ID, requests and completions cannot be redirected, and all
814 * transactions are forwarded upstream, even as it passes through a
815 * bridge where the target device is downstream.
816 */
817#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
818
819/*
820 * For multifunction devices which are not isolated from each other, find
821 * all the other non-isolated functions and look for existing groups. For
822 * each function, we also need to look for aliases to or from other devices
823 * that may already have a group.
824 */
825static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
826 unsigned long *devfns)
827{
828 struct pci_dev *tmp = NULL;
829 struct iommu_group *group;
830
831 if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
832 return NULL;
833
834 for_each_pci_dev(tmp) {
835 if (tmp == pdev || tmp->bus != pdev->bus ||
836 PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
837 pci_acs_enabled(tmp, REQ_ACS_FLAGS))
838 continue;
839
840 group = get_pci_alias_group(tmp, devfns);
841 if (group) {
842 pci_dev_put(tmp);
843 return group;
844 }
845 }
846
847 return NULL;
848}
849
850/*
851 * Look for aliases to or from the given device for existing groups. DMA
852 * aliases are only supported on the same bus, therefore the search
853 * space is quite small (especially since we're really only looking at pcie
854 * device, and therefore only expect multiple slots on the root complex or
855 * downstream switch ports). It's conceivable though that a pair of
856 * multifunction devices could have aliases between them that would cause a
857 * loop. To prevent this, we use a bitmap to track where we've been.
858 */
859static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
860 unsigned long *devfns)
861{
862 struct pci_dev *tmp = NULL;
863 struct iommu_group *group;
864
865 if (test_and_set_bit(pdev->devfn & 0xff, devfns))
866 return NULL;
867
868 group = iommu_group_get(&pdev->dev);
869 if (group)
870 return group;
871
872 for_each_pci_dev(tmp) {
873 if (tmp == pdev || tmp->bus != pdev->bus)
874 continue;
875
876 /* We alias them or they alias us */
877 if (pci_devs_are_dma_aliases(pdev, tmp)) {
878 group = get_pci_alias_group(tmp, devfns);
879 if (group) {
880 pci_dev_put(tmp);
881 return group;
882 }
883
884 group = get_pci_function_alias_group(tmp, devfns);
885 if (group) {
886 pci_dev_put(tmp);
887 return group;
888 }
889 }
890 }
891
892 return NULL;
893}
894
895struct group_for_pci_data {
896 struct pci_dev *pdev;
897 struct iommu_group *group;
898};
899
900/*
901 * DMA alias iterator callback, return the last seen device. Stop and return
902 * the IOMMU group if we find one along the way.
903 */
904static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
905{
906 struct group_for_pci_data *data = opaque;
907
908 data->pdev = pdev;
909 data->group = iommu_group_get(&pdev->dev);
910
911 return data->group != NULL;
912}
913
914/*
915 * Generic device_group call-back function. It just allocates one
916 * iommu-group per device.
917 */
918struct iommu_group *generic_device_group(struct device *dev)
919{
920 return iommu_group_alloc();
921}
922
923/*
924 * Use standard PCI bus topology, isolation features, and DMA alias quirks
925 * to find or create an IOMMU group for a device.
926 */
927struct iommu_group *pci_device_group(struct device *dev)
928{
929 struct pci_dev *pdev = to_pci_dev(dev);
930 struct group_for_pci_data data;
931 struct pci_bus *bus;
932 struct iommu_group *group = NULL;
933 u64 devfns[4] = { 0 };
934
935 if (WARN_ON(!dev_is_pci(dev)))
936 return ERR_PTR(-EINVAL);
937
938 /*
939 * Find the upstream DMA alias for the device. A device must not
940 * be aliased due to topology in order to have its own IOMMU group.
941 * If we find an alias along the way that already belongs to a
942 * group, use it.
943 */
944 if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))
945 return data.group;
946
947 pdev = data.pdev;
948
949 /*
950 * Continue upstream from the point of minimum IOMMU granularity
951 * due to aliases to the point where devices are protected from
952 * peer-to-peer DMA by PCI ACS. Again, if we find an existing
953 * group, use it.
954 */
955 for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {
956 if (!bus->self)
957 continue;
958
959 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
960 break;
961
962 pdev = bus->self;
963
964 group = iommu_group_get(&pdev->dev);
965 if (group)
966 return group;
967 }
968
969 /*
970 * Look for existing groups on device aliases. If we alias another
971 * device or another device aliases us, use the same group.
972 */
973 group = get_pci_alias_group(pdev, (unsigned long *)devfns);
974 if (group)
975 return group;
976
977 /*
978 * Look for existing groups on non-isolated functions on the same
979 * slot and aliases of those funcions, if any. No need to clear
980 * the search bitmap, the tested devfns are still valid.
981 */
982 group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
983 if (group)
984 return group;
985
986 /* No shared group found, allocate new */
987 return iommu_group_alloc();
988}
989
990/**
991 * iommu_group_get_for_dev - Find or create the IOMMU group for a device
992 * @dev: target device
993 *
994 * This function is intended to be called by IOMMU drivers and extended to
995 * support common, bus-defined algorithms when determining or creating the
996 * IOMMU group for a device. On success, the caller will hold a reference
997 * to the returned IOMMU group, which will already include the provided
998 * device. The reference should be released with iommu_group_put().
999 */
1000struct iommu_group *iommu_group_get_for_dev(struct device *dev)
1001{
1002 const struct iommu_ops *ops = dev->bus->iommu_ops;
1003 struct iommu_group *group;
1004 int ret;
1005
1006 group = iommu_group_get(dev);
1007 if (group)
1008 return group;
1009
1010 if (!ops)
1011 return ERR_PTR(-EINVAL);
1012
1013 group = ops->device_group(dev);
1014 if (WARN_ON_ONCE(group == NULL))
1015 return ERR_PTR(-EINVAL);
1016
1017 if (IS_ERR(group))
1018 return group;
1019
1020 /*
1021 * Try to allocate a default domain - needs support from the
1022 * IOMMU driver.
1023 */
1024 if (!group->default_domain) {
1025 struct iommu_domain *dom;
1026
1027 dom = __iommu_domain_alloc(dev->bus, iommu_def_domain_type);
1028 if (!dom && iommu_def_domain_type != IOMMU_DOMAIN_DMA) {
1029 dev_warn(dev,
1030 "failed to allocate default IOMMU domain of type %u; falling back to IOMMU_DOMAIN_DMA",
1031 iommu_def_domain_type);
1032 dom = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_DMA);
1033 }
1034
1035 group->default_domain = dom;
1036 if (!group->domain)
1037 group->domain = dom;
1038 }
1039
1040 ret = iommu_group_add_device(group, dev);
1041 if (ret) {
1042 iommu_group_put(group);
1043 return ERR_PTR(ret);
1044 }
1045
1046 return group;
1047}
1048
1049struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
1050{
1051 return group->default_domain;
1052}
1053
1054static int add_iommu_group(struct device *dev, void *data)
1055{
1056 struct iommu_callback_data *cb = data;
1057 const struct iommu_ops *ops = cb->ops;
1058 int ret;
1059
1060 if (!ops->add_device)
1061 return 0;
1062
1063 WARN_ON(dev->iommu_group);
1064
1065 ret = ops->add_device(dev);
1066
1067 /*
1068 * We ignore -ENODEV errors for now, as they just mean that the
1069 * device is not translated by an IOMMU. We still care about
1070 * other errors and fail to initialize when they happen.
1071 */
1072 if (ret == -ENODEV)
1073 ret = 0;
1074
1075 return ret;
1076}
1077
1078static int remove_iommu_group(struct device *dev, void *data)
1079{
1080 struct iommu_callback_data *cb = data;
1081 const struct iommu_ops *ops = cb->ops;
1082
1083 if (ops->remove_device && dev->iommu_group)
1084 ops->remove_device(dev);
1085
1086 return 0;
1087}
1088
1089static int iommu_bus_notifier(struct notifier_block *nb,
1090 unsigned long action, void *data)
1091{
1092 struct device *dev = data;
1093 const struct iommu_ops *ops = dev->bus->iommu_ops;
1094 struct iommu_group *group;
1095 unsigned long group_action = 0;
1096
1097 /*
1098 * ADD/DEL call into iommu driver ops if provided, which may
1099 * result in ADD/DEL notifiers to group->notifier
1100 */
1101 if (action == BUS_NOTIFY_ADD_DEVICE) {
1102 if (ops->add_device) {
1103 int ret;
1104
1105 ret = ops->add_device(dev);
1106 return (ret) ? NOTIFY_DONE : NOTIFY_OK;
1107 }
1108 } else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
1109 if (ops->remove_device && dev->iommu_group) {
1110 ops->remove_device(dev);
1111 return 0;
1112 }
1113 }
1114
1115 /*
1116 * Remaining BUS_NOTIFYs get filtered and republished to the
1117 * group, if anyone is listening
1118 */
1119 group = iommu_group_get(dev);
1120 if (!group)
1121 return 0;
1122
1123 switch (action) {
1124 case BUS_NOTIFY_BIND_DRIVER:
1125 group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER;
1126 break;
1127 case BUS_NOTIFY_BOUND_DRIVER:
1128 group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER;
1129 break;
1130 case BUS_NOTIFY_UNBIND_DRIVER:
1131 group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER;
1132 break;
1133 case BUS_NOTIFY_UNBOUND_DRIVER:
1134 group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER;
1135 break;
1136 }
1137
1138 if (group_action)
1139 blocking_notifier_call_chain(&group->notifier,
1140 group_action, dev);
1141
1142 iommu_group_put(group);
1143 return 0;
1144}
1145
1146static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
1147{
1148 int err;
1149 struct notifier_block *nb;
1150 struct iommu_callback_data cb = {
1151 .ops = ops,
1152 };
1153
1154 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
1155 if (!nb)
1156 return -ENOMEM;
1157
1158 nb->notifier_call = iommu_bus_notifier;
1159
1160 err = bus_register_notifier(bus, nb);
1161 if (err)
1162 goto out_free;
1163
1164 err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
1165 if (err)
1166 goto out_err;
1167
1168
1169 return 0;
1170
1171out_err:
1172 /* Clean up */
1173 bus_for_each_dev(bus, NULL, &cb, remove_iommu_group);
1174 bus_unregister_notifier(bus, nb);
1175
1176out_free:
1177 kfree(nb);
1178
1179 return err;
1180}
1181
1182/**
1183 * bus_set_iommu - set iommu-callbacks for the bus
1184 * @bus: bus.
1185 * @ops: the callbacks provided by the iommu-driver
1186 *
1187 * This function is called by an iommu driver to set the iommu methods
1188 * used for a particular bus. Drivers for devices on that bus can use
1189 * the iommu-api after these ops are registered.
1190 * This special function is needed because IOMMUs are usually devices on
1191 * the bus itself, so the iommu drivers are not initialized when the bus
1192 * is set up. With this function the iommu-driver can set the iommu-ops
1193 * afterwards.
1194 */
1195int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops)
1196{
1197 int err;
1198
1199 if (bus->iommu_ops != NULL)
1200 return -EBUSY;
1201
1202 bus->iommu_ops = ops;
1203
1204 /* Do IOMMU specific setup for this bus-type */
1205 err = iommu_bus_init(bus, ops);
1206 if (err)
1207 bus->iommu_ops = NULL;
1208
1209 return err;
1210}
1211EXPORT_SYMBOL_GPL(bus_set_iommu);
1212
1213bool iommu_present(struct bus_type *bus)
1214{
1215 return bus->iommu_ops != NULL;
1216}
1217EXPORT_SYMBOL_GPL(iommu_present);
1218
1219bool iommu_capable(struct bus_type *bus, enum iommu_cap cap)
1220{
1221 if (!bus->iommu_ops || !bus->iommu_ops->capable)
1222 return false;
1223
1224 return bus->iommu_ops->capable(cap);
1225}
1226EXPORT_SYMBOL_GPL(iommu_capable);
1227
1228/**
1229 * iommu_set_fault_handler() - set a fault handler for an iommu domain
1230 * @domain: iommu domain
1231 * @handler: fault handler
1232 * @token: user data, will be passed back to the fault handler
1233 *
1234 * This function should be used by IOMMU users which want to be notified
1235 * whenever an IOMMU fault happens.
1236 *
1237 * The fault handler itself should return 0 on success, and an appropriate
1238 * error code otherwise.
1239 */
1240void iommu_set_fault_handler(struct iommu_domain *domain,
1241 iommu_fault_handler_t handler,
1242 void *token)
1243{
1244 BUG_ON(!domain);
1245
1246 domain->handler = handler;
1247 domain->handler_token = token;
1248}
1249EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
1250
1251static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
1252 unsigned type)
1253{
1254 struct iommu_domain *domain;
1255
1256 if (bus == NULL || bus->iommu_ops == NULL)
1257 return NULL;
1258
1259 domain = bus->iommu_ops->domain_alloc(type);
1260 if (!domain)
1261 return NULL;
1262
1263 domain->ops = bus->iommu_ops;
1264 domain->type = type;
1265 /* Assume all sizes by default; the driver may override this later */
1266 domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
1267
1268 return domain;
1269}
1270
1271struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
1272{
1273 return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED);
1274}
1275EXPORT_SYMBOL_GPL(iommu_domain_alloc);
1276
1277void iommu_domain_free(struct iommu_domain *domain)
1278{
1279 domain->ops->domain_free(domain);
1280}
1281EXPORT_SYMBOL_GPL(iommu_domain_free);
1282
1283static int __iommu_attach_device(struct iommu_domain *domain,
1284 struct device *dev)
1285{
1286 int ret;
1287 if ((domain->ops->is_attach_deferred != NULL) &&
1288 domain->ops->is_attach_deferred(domain, dev))
1289 return 0;
1290
1291 if (unlikely(domain->ops->attach_dev == NULL))
1292 return -ENODEV;
1293
1294 ret = domain->ops->attach_dev(domain, dev);
1295 if (!ret)
1296 trace_attach_device_to_domain(dev);
1297 return ret;
1298}
1299
1300int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
1301{
1302 struct iommu_group *group;
1303 int ret;
1304
1305 group = iommu_group_get(dev);
1306 if (!group)
1307 return -ENODEV;
1308
1309 /*
1310 * Lock the group to make sure the device-count doesn't
1311 * change while we are attaching
1312 */
1313 mutex_lock(&group->mutex);
1314 ret = -EINVAL;
1315 if (iommu_group_device_count(group) != 1)
1316 goto out_unlock;
1317
1318 ret = __iommu_attach_group(domain, group);
1319
1320out_unlock:
1321 mutex_unlock(&group->mutex);
1322 iommu_group_put(group);
1323
1324 return ret;
1325}
1326EXPORT_SYMBOL_GPL(iommu_attach_device);
1327
1328static void __iommu_detach_device(struct iommu_domain *domain,
1329 struct device *dev)
1330{
1331 if ((domain->ops->is_attach_deferred != NULL) &&
1332 domain->ops->is_attach_deferred(domain, dev))
1333 return;
1334
1335 if (unlikely(domain->ops->detach_dev == NULL))
1336 return;
1337
1338 domain->ops->detach_dev(domain, dev);
1339 trace_detach_device_from_domain(dev);
1340}
1341
1342void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
1343{
1344 struct iommu_group *group;
1345
1346 group = iommu_group_get(dev);
1347 if (!group)
1348 return;
1349
1350 mutex_lock(&group->mutex);
1351 if (iommu_group_device_count(group) != 1) {
1352 WARN_ON(1);
1353 goto out_unlock;
1354 }
1355
1356 __iommu_detach_group(domain, group);
1357
1358out_unlock:
1359 mutex_unlock(&group->mutex);
1360 iommu_group_put(group);
1361}
1362EXPORT_SYMBOL_GPL(iommu_detach_device);
1363
1364struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
1365{
1366 struct iommu_domain *domain;
1367 struct iommu_group *group;
1368
1369 group = iommu_group_get(dev);
1370 if (!group)
1371 return NULL;
1372
1373 domain = group->domain;
1374
1375 iommu_group_put(group);
1376
1377 return domain;
1378}
1379EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
1380
1381/*
1382 * IOMMU groups are really the natrual working unit of the IOMMU, but
1383 * the IOMMU API works on domains and devices. Bridge that gap by
1384 * iterating over the devices in a group. Ideally we'd have a single
1385 * device which represents the requestor ID of the group, but we also
1386 * allow IOMMU drivers to create policy defined minimum sets, where
1387 * the physical hardware may be able to distiguish members, but we
1388 * wish to group them at a higher level (ex. untrusted multi-function
1389 * PCI devices). Thus we attach each device.
1390 */
1391static int iommu_group_do_attach_device(struct device *dev, void *data)
1392{
1393 struct iommu_domain *domain = data;
1394
1395 return __iommu_attach_device(domain, dev);
1396}
1397
1398static int __iommu_attach_group(struct iommu_domain *domain,
1399 struct iommu_group *group)
1400{
1401 int ret;
1402
1403 if (group->default_domain && group->domain != group->default_domain)
1404 return -EBUSY;
1405
1406 ret = __iommu_group_for_each_dev(group, domain,
1407 iommu_group_do_attach_device);
1408 if (ret == 0)
1409 group->domain = domain;
1410
1411 return ret;
1412}
1413
1414int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
1415{
1416 int ret;
1417
1418 mutex_lock(&group->mutex);
1419 ret = __iommu_attach_group(domain, group);
1420 mutex_unlock(&group->mutex);
1421
1422 return ret;
1423}
1424EXPORT_SYMBOL_GPL(iommu_attach_group);
1425
1426static int iommu_group_do_detach_device(struct device *dev, void *data)
1427{
1428 struct iommu_domain *domain = data;
1429
1430 __iommu_detach_device(domain, dev);
1431
1432 return 0;
1433}
1434
1435static void __iommu_detach_group(struct iommu_domain *domain,
1436 struct iommu_group *group)
1437{
1438 int ret;
1439
1440 if (!group->default_domain) {
1441 __iommu_group_for_each_dev(group, domain,
1442 iommu_group_do_detach_device);
1443 group->domain = NULL;
1444 return;
1445 }
1446
1447 if (group->domain == group->default_domain)
1448 return;
1449
1450 /* Detach by re-attaching to the default domain */
1451 ret = __iommu_group_for_each_dev(group, group->default_domain,
1452 iommu_group_do_attach_device);
1453 if (ret != 0)
1454 WARN_ON(1);
1455 else
1456 group->domain = group->default_domain;
1457}
1458
1459void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
1460{
1461 mutex_lock(&group->mutex);
1462 __iommu_detach_group(domain, group);
1463 mutex_unlock(&group->mutex);
1464}
1465EXPORT_SYMBOL_GPL(iommu_detach_group);
1466
1467phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1468{
1469 if (unlikely(domain->ops->iova_to_phys == NULL))
1470 return 0;
1471
1472 return domain->ops->iova_to_phys(domain, iova);
1473}
1474EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
1475
1476static size_t iommu_pgsize(struct iommu_domain *domain,
1477 unsigned long addr_merge, size_t size)
1478{
1479 unsigned int pgsize_idx;
1480 size_t pgsize;
1481
1482 /* Max page size that still fits into 'size' */
1483 pgsize_idx = __fls(size);
1484
1485 /* need to consider alignment requirements ? */
1486 if (likely(addr_merge)) {
1487 /* Max page size allowed by address */
1488 unsigned int align_pgsize_idx = __ffs(addr_merge);
1489 pgsize_idx = min(pgsize_idx, align_pgsize_idx);
1490 }
1491
1492 /* build a mask of acceptable page sizes */
1493 pgsize = (1UL << (pgsize_idx + 1)) - 1;
1494
1495 /* throw away page sizes not supported by the hardware */
1496 pgsize &= domain->pgsize_bitmap;
1497
1498 /* make sure we're still sane */
1499 BUG_ON(!pgsize);
1500
1501 /* pick the biggest page */
1502 pgsize_idx = __fls(pgsize);
1503 pgsize = 1UL << pgsize_idx;
1504
1505 return pgsize;
1506}
1507
1508int iommu_map(struct iommu_domain *domain, unsigned long iova,
1509 phys_addr_t paddr, size_t size, int prot)
1510{
1511 unsigned long orig_iova = iova;
1512 unsigned int min_pagesz;
1513 size_t orig_size = size;
1514 phys_addr_t orig_paddr = paddr;
1515 int ret = 0;
1516
1517 if (unlikely(domain->ops->map == NULL ||
1518 domain->pgsize_bitmap == 0UL))
1519 return -ENODEV;
1520
1521 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
1522 return -EINVAL;
1523
1524 /* find out the minimum page size supported */
1525 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
1526
1527 /*
1528 * both the virtual address and the physical one, as well as
1529 * the size of the mapping, must be aligned (at least) to the
1530 * size of the smallest page supported by the hardware
1531 */
1532 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
1533 pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
1534 iova, &paddr, size, min_pagesz);
1535 return -EINVAL;
1536 }
1537
1538 pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
1539
1540 while (size) {
1541 size_t pgsize = iommu_pgsize(domain, iova | paddr, size);
1542
1543 pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
1544 iova, &paddr, pgsize);
1545
1546 ret = domain->ops->map(domain, iova, paddr, pgsize, prot);
1547 if (ret)
1548 break;
1549
1550 iova += pgsize;
1551 paddr += pgsize;
1552 size -= pgsize;
1553 }
1554
1555 /* unroll mapping in case something went wrong */
1556 if (ret)
1557 iommu_unmap(domain, orig_iova, orig_size - size);
1558 else
1559 trace_map(orig_iova, orig_paddr, orig_size);
1560
1561 return ret;
1562}
1563EXPORT_SYMBOL_GPL(iommu_map);
1564
1565static size_t __iommu_unmap(struct iommu_domain *domain,
1566 unsigned long iova, size_t size,
1567 bool sync)
1568{
1569 const struct iommu_ops *ops = domain->ops;
1570 size_t unmapped_page, unmapped = 0;
1571 unsigned long orig_iova = iova;
1572 unsigned int min_pagesz;
1573
1574 if (unlikely(ops->unmap == NULL ||
1575 domain->pgsize_bitmap == 0UL))
1576 return 0;
1577
1578 if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
1579 return 0;
1580
1581 /* find out the minimum page size supported */
1582 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
1583
1584 /*
1585 * The virtual address, as well as the size of the mapping, must be
1586 * aligned (at least) to the size of the smallest page supported
1587 * by the hardware
1588 */
1589 if (!IS_ALIGNED(iova | size, min_pagesz)) {
1590 pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
1591 iova, size, min_pagesz);
1592 return 0;
1593 }
1594
1595 pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
1596
1597 /*
1598 * Keep iterating until we either unmap 'size' bytes (or more)
1599 * or we hit an area that isn't mapped.
1600 */
1601 while (unmapped < size) {
1602 size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
1603
1604 unmapped_page = ops->unmap(domain, iova, pgsize);
1605 if (!unmapped_page)
1606 break;
1607
1608 if (sync && ops->iotlb_range_add)
1609 ops->iotlb_range_add(domain, iova, pgsize);
1610
1611 pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
1612 iova, unmapped_page);
1613
1614 iova += unmapped_page;
1615 unmapped += unmapped_page;
1616 }
1617
1618 if (sync && ops->iotlb_sync)
1619 ops->iotlb_sync(domain);
1620
1621 trace_unmap(orig_iova, size, unmapped);
1622 return unmapped;
1623}
1624
1625size_t iommu_unmap(struct iommu_domain *domain,
1626 unsigned long iova, size_t size)
1627{
1628 return __iommu_unmap(domain, iova, size, true);
1629}
1630EXPORT_SYMBOL_GPL(iommu_unmap);
1631
1632size_t iommu_unmap_fast(struct iommu_domain *domain,
1633 unsigned long iova, size_t size)
1634{
1635 return __iommu_unmap(domain, iova, size, false);
1636}
1637EXPORT_SYMBOL_GPL(iommu_unmap_fast);
1638
1639size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
1640 struct scatterlist *sg, unsigned int nents, int prot)
1641{
1642 struct scatterlist *s;
1643 size_t mapped = 0;
1644 unsigned int i, min_pagesz;
1645 int ret;
1646
1647 if (unlikely(domain->pgsize_bitmap == 0UL))
1648 return 0;
1649
1650 min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
1651
1652 for_each_sg(sg, s, nents, i) {
1653 phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
1654
1655 /*
1656 * We are mapping on IOMMU page boundaries, so offset within
1657 * the page must be 0. However, the IOMMU may support pages
1658 * smaller than PAGE_SIZE, so s->offset may still represent
1659 * an offset of that boundary within the CPU page.
1660 */
1661 if (!IS_ALIGNED(s->offset, min_pagesz))
1662 goto out_err;
1663
1664 ret = iommu_map(domain, iova + mapped, phys, s->length, prot);
1665 if (ret)
1666 goto out_err;
1667
1668 mapped += s->length;
1669 }
1670
1671 return mapped;
1672
1673out_err:
1674 /* undo mappings already done */
1675 iommu_unmap(domain, iova, mapped);
1676
1677 return 0;
1678
1679}
1680EXPORT_SYMBOL_GPL(default_iommu_map_sg);
1681
1682int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
1683 phys_addr_t paddr, u64 size, int prot)
1684{
1685 if (unlikely(domain->ops->domain_window_enable == NULL))
1686 return -ENODEV;
1687
1688 return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size,
1689 prot);
1690}
1691EXPORT_SYMBOL_GPL(iommu_domain_window_enable);
1692
1693void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr)
1694{
1695 if (unlikely(domain->ops->domain_window_disable == NULL))
1696 return;
1697
1698 return domain->ops->domain_window_disable(domain, wnd_nr);
1699}
1700EXPORT_SYMBOL_GPL(iommu_domain_window_disable);
1701
1702/**
1703 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
1704 * @domain: the iommu domain where the fault has happened
1705 * @dev: the device where the fault has happened
1706 * @iova: the faulting address
1707 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...)
1708 *
1709 * This function should be called by the low-level IOMMU implementations
1710 * whenever IOMMU faults happen, to allow high-level users, that are
1711 * interested in such events, to know about them.
1712 *
1713 * This event may be useful for several possible use cases:
1714 * - mere logging of the event
1715 * - dynamic TLB/PTE loading
1716 * - if restarting of the faulting device is required
1717 *
1718 * Returns 0 on success and an appropriate error code otherwise (if dynamic
1719 * PTE/TLB loading will one day be supported, implementations will be able
1720 * to tell whether it succeeded or not according to this return value).
1721 *
1722 * Specifically, -ENOSYS is returned if a fault handler isn't installed
1723 * (though fault handlers can also return -ENOSYS, in case they want to
1724 * elicit the default behavior of the IOMMU drivers).
1725 */
1726int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
1727 unsigned long iova, int flags)
1728{
1729 int ret = -ENOSYS;
1730
1731 /*
1732 * if upper layers showed interest and installed a fault handler,
1733 * invoke it.
1734 */
1735 if (domain->handler)
1736 ret = domain->handler(domain, dev, iova, flags,
1737 domain->handler_token);
1738
1739 trace_io_page_fault(dev, iova, flags);
1740 return ret;
1741}
1742EXPORT_SYMBOL_GPL(report_iommu_fault);
1743
1744static int __init iommu_init(void)
1745{
1746 iommu_group_kset = kset_create_and_add("iommu_groups",
1747 NULL, kernel_kobj);
1748 BUG_ON(!iommu_group_kset);
1749
1750 return 0;
1751}
1752core_initcall(iommu_init);
1753
1754int iommu_domain_get_attr(struct iommu_domain *domain,
1755 enum iommu_attr attr, void *data)
1756{
1757 struct iommu_domain_geometry *geometry;
1758 bool *paging;
1759 int ret = 0;
1760 u32 *count;
1761
1762 switch (attr) {
1763 case DOMAIN_ATTR_GEOMETRY:
1764 geometry = data;
1765 *geometry = domain->geometry;
1766
1767 break;
1768 case DOMAIN_ATTR_PAGING:
1769 paging = data;
1770 *paging = (domain->pgsize_bitmap != 0UL);
1771 break;
1772 case DOMAIN_ATTR_WINDOWS:
1773 count = data;
1774
1775 if (domain->ops->domain_get_windows != NULL)
1776 *count = domain->ops->domain_get_windows(domain);
1777 else
1778 ret = -ENODEV;
1779
1780 break;
1781 default:
1782 if (!domain->ops->domain_get_attr)
1783 return -EINVAL;
1784
1785 ret = domain->ops->domain_get_attr(domain, attr, data);
1786 }
1787
1788 return ret;
1789}
1790EXPORT_SYMBOL_GPL(iommu_domain_get_attr);
1791
1792int iommu_domain_set_attr(struct iommu_domain *domain,
1793 enum iommu_attr attr, void *data)
1794{
1795 int ret = 0;
1796 u32 *count;
1797
1798 switch (attr) {
1799 case DOMAIN_ATTR_WINDOWS:
1800 count = data;
1801
1802 if (domain->ops->domain_set_windows != NULL)
1803 ret = domain->ops->domain_set_windows(domain, *count);
1804 else
1805 ret = -ENODEV;
1806
1807 break;
1808 default:
1809 if (domain->ops->domain_set_attr == NULL)
1810 return -EINVAL;
1811
1812 ret = domain->ops->domain_set_attr(domain, attr, data);
1813 }
1814
1815 return ret;
1816}
1817EXPORT_SYMBOL_GPL(iommu_domain_set_attr);
1818
1819void iommu_get_resv_regions(struct device *dev, struct list_head *list)
1820{
1821 const struct iommu_ops *ops = dev->bus->iommu_ops;
1822
1823 if (ops && ops->get_resv_regions)
1824 ops->get_resv_regions(dev, list);
1825}
1826
1827void iommu_put_resv_regions(struct device *dev, struct list_head *list)
1828{
1829 const struct iommu_ops *ops = dev->bus->iommu_ops;
1830
1831 if (ops && ops->put_resv_regions)
1832 ops->put_resv_regions(dev, list);
1833}
1834
1835struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
1836 size_t length, int prot,
1837 enum iommu_resv_type type)
1838{
1839 struct iommu_resv_region *region;
1840
1841 region = kzalloc(sizeof(*region), GFP_KERNEL);
1842 if (!region)
1843 return NULL;
1844
1845 INIT_LIST_HEAD(®ion->list);
1846 region->start = start;
1847 region->length = length;
1848 region->prot = prot;
1849 region->type = type;
1850 return region;
1851}
1852
1853/* Request that a device is direct mapped by the IOMMU */
1854int iommu_request_dm_for_dev(struct device *dev)
1855{
1856 struct iommu_domain *dm_domain;
1857 struct iommu_group *group;
1858 int ret;
1859
1860 /* Device must already be in a group before calling this function */
1861 group = iommu_group_get_for_dev(dev);
1862 if (IS_ERR(group))
1863 return PTR_ERR(group);
1864
1865 mutex_lock(&group->mutex);
1866
1867 /* Check if the default domain is already direct mapped */
1868 ret = 0;
1869 if (group->default_domain &&
1870 group->default_domain->type == IOMMU_DOMAIN_IDENTITY)
1871 goto out;
1872
1873 /* Don't change mappings of existing devices */
1874 ret = -EBUSY;
1875 if (iommu_group_device_count(group) != 1)
1876 goto out;
1877
1878 /* Allocate a direct mapped domain */
1879 ret = -ENOMEM;
1880 dm_domain = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_IDENTITY);
1881 if (!dm_domain)
1882 goto out;
1883
1884 /* Attach the device to the domain */
1885 ret = __iommu_attach_group(dm_domain, group);
1886 if (ret) {
1887 iommu_domain_free(dm_domain);
1888 goto out;
1889 }
1890
1891 /* Make the direct mapped domain the default for this group */
1892 if (group->default_domain)
1893 iommu_domain_free(group->default_domain);
1894 group->default_domain = dm_domain;
1895
1896 pr_info("Using direct mapping for device %s\n", dev_name(dev));
1897
1898 ret = 0;
1899out:
1900 mutex_unlock(&group->mutex);
1901 iommu_group_put(group);
1902
1903 return ret;
1904}
1905
1906const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
1907{
1908 const struct iommu_ops *ops = NULL;
1909 struct iommu_device *iommu;
1910
1911 spin_lock(&iommu_device_lock);
1912 list_for_each_entry(iommu, &iommu_device_list, list)
1913 if (iommu->fwnode == fwnode) {
1914 ops = iommu->ops;
1915 break;
1916 }
1917 spin_unlock(&iommu_device_lock);
1918 return ops;
1919}
1920
1921int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
1922 const struct iommu_ops *ops)
1923{
1924 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1925
1926 if (fwspec)
1927 return ops == fwspec->ops ? 0 : -EINVAL;
1928
1929 fwspec = kzalloc(sizeof(*fwspec), GFP_KERNEL);
1930 if (!fwspec)
1931 return -ENOMEM;
1932
1933 of_node_get(to_of_node(iommu_fwnode));
1934 fwspec->iommu_fwnode = iommu_fwnode;
1935 fwspec->ops = ops;
1936 dev->iommu_fwspec = fwspec;
1937 return 0;
1938}
1939EXPORT_SYMBOL_GPL(iommu_fwspec_init);
1940
1941void iommu_fwspec_free(struct device *dev)
1942{
1943 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1944
1945 if (fwspec) {
1946 fwnode_handle_put(fwspec->iommu_fwnode);
1947 kfree(fwspec);
1948 dev->iommu_fwspec = NULL;
1949 }
1950}
1951EXPORT_SYMBOL_GPL(iommu_fwspec_free);
1952
1953int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids)
1954{
1955 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1956 size_t size;
1957 int i;
1958
1959 if (!fwspec)
1960 return -EINVAL;
1961
1962 size = offsetof(struct iommu_fwspec, ids[fwspec->num_ids + num_ids]);
1963 if (size > sizeof(*fwspec)) {
1964 fwspec = krealloc(dev->iommu_fwspec, size, GFP_KERNEL);
1965 if (!fwspec)
1966 return -ENOMEM;
1967
1968 dev->iommu_fwspec = fwspec;
1969 }
1970
1971 for (i = 0; i < num_ids; i++)
1972 fwspec->ids[fwspec->num_ids + i] = ids[i];
1973
1974 fwspec->num_ids += num_ids;
1975 return 0;
1976}
1977EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);