1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * nested.c - nested mode translation support
  4 *
  5 * Copyright (C) 2023 Intel Corporation
  6 *
  7 * Author: Lu Baolu <baolu.lu@linux.intel.com>
  8 *         Jacob Pan <jacob.jun.pan@linux.intel.com>
  9 *         Yi Liu <yi.l.liu@intel.com>
 10 */
 11
 12#define pr_fmt(fmt)	"DMAR: " fmt
 13
 14#include <linux/iommu.h>
 15#include <linux/pci.h>
 16#include <linux/pci-ats.h>
 17
 18#include "iommu.h"
 19#include "pasid.h"
 20
 21static int intel_nested_attach_dev(struct iommu_domain *domain,
 22				   struct device *dev)
 23{
 24	struct device_domain_info *info = dev_iommu_priv_get(dev);
 25	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
 26	struct intel_iommu *iommu = info->iommu;
 27	unsigned long flags;
 28	int ret = 0;
 29
 30	if (info->domain)
 31		device_block_translation(dev);
 32
 33	if (iommu->agaw < dmar_domain->s2_domain->agaw) {
 34		dev_err_ratelimited(dev, "Adjusted guest address width not compatible\n");
 35		return -ENODEV;
 36	}
 37
 38	/*
 39	 * Stage-1 domain cannot work alone, it is nested on a s2_domain.
 40	 * The s2_domain will be used in nested translation, hence needs
 41	 * to ensure the s2_domain is compatible with this IOMMU.
 42	 */
 43	ret = prepare_domain_attach_device(&dmar_domain->s2_domain->domain, dev);
 44	if (ret) {
 45		dev_err_ratelimited(dev, "s2 domain is not compatible\n");
 46		return ret;
 47	}
 48
 49	ret = domain_attach_iommu(dmar_domain, iommu);
 50	if (ret) {
 51		dev_err_ratelimited(dev, "Failed to attach domain to iommu\n");
 52		return ret;
 53	}
 54
 55	ret = intel_pasid_setup_nested(iommu, dev,
 56				       IOMMU_NO_PASID, dmar_domain);
 57	if (ret) {
 58		domain_detach_iommu(dmar_domain, iommu);
 59		dev_err_ratelimited(dev, "Failed to setup pasid entry\n");
 60		return ret;
 61	}
 62
 63	info->domain = dmar_domain;
 64	spin_lock_irqsave(&dmar_domain->lock, flags);
 65	list_add(&info->link, &dmar_domain->devices);
 66	spin_unlock_irqrestore(&dmar_domain->lock, flags);
 67
 68	domain_update_iotlb(dmar_domain);
 69
 70	return 0;
 71}
 72
 73static void intel_nested_domain_free(struct iommu_domain *domain)
 74{
 75	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
 76	struct dmar_domain *s2_domain = dmar_domain->s2_domain;
 77
 78	spin_lock(&s2_domain->s1_lock);
 79	list_del(&dmar_domain->s2_link);
 80	spin_unlock(&s2_domain->s1_lock);
 81	kfree(dmar_domain);
 82}
 83
 84static void nested_flush_dev_iotlb(struct dmar_domain *domain, u64 addr,
 85				   unsigned int mask)
 86{
 87	struct device_domain_info *info;
 88	unsigned long flags;
 89	u16 sid, qdep;
 90
 91	spin_lock_irqsave(&domain->lock, flags);
 92	list_for_each_entry(info, &domain->devices, link) {
 93		if (!info->ats_enabled)
 94			continue;
 95		sid = info->bus << 8 | info->devfn;
 96		qdep = info->ats_qdep;
 97		qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
 98				   qdep, addr, mask);
 99		quirk_extra_dev_tlb_flush(info, addr, mask,
100					  IOMMU_NO_PASID, qdep);
101	}
102	spin_unlock_irqrestore(&domain->lock, flags);
103}
104
105static void intel_nested_flush_cache(struct dmar_domain *domain, u64 addr,
106				     u64 npages, bool ih)
107{
108	struct iommu_domain_info *info;
109	unsigned int mask;
110	unsigned long i;
111
112	xa_for_each(&domain->iommu_array, i, info)
113		qi_flush_piotlb(info->iommu,
114				domain_id_iommu(domain, info->iommu),
115				IOMMU_NO_PASID, addr, npages, ih);
116
117	if (!domain->has_iotlb_device)
118		return;
119
120	if (npages == U64_MAX)
121		mask = 64 - VTD_PAGE_SHIFT;
122	else
123		mask = ilog2(__roundup_pow_of_two(npages));
124
125	nested_flush_dev_iotlb(domain, addr, mask);
126}
127
128static int intel_nested_cache_invalidate_user(struct iommu_domain *domain,
129					      struct iommu_user_data_array *array)
130{
131	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
132	struct iommu_hwpt_vtd_s1_invalidate inv_entry;
133	u32 index, processed = 0;
134	int ret = 0;
135
136	if (array->type != IOMMU_HWPT_INVALIDATE_DATA_VTD_S1) {
137		ret = -EINVAL;
138		goto out;
139	}
140
141	for (index = 0; index < array->entry_num; index++) {
142		ret = iommu_copy_struct_from_user_array(&inv_entry, array,
143							IOMMU_HWPT_INVALIDATE_DATA_VTD_S1,
144							index, __reserved);
145		if (ret)
146			break;
147
148		if ((inv_entry.flags & ~IOMMU_VTD_INV_FLAGS_LEAF) ||
149		    inv_entry.__reserved) {
150			ret = -EOPNOTSUPP;
151			break;
152		}
153
154		if (!IS_ALIGNED(inv_entry.addr, VTD_PAGE_SIZE) ||
155		    ((inv_entry.npages == U64_MAX) && inv_entry.addr)) {
156			ret = -EINVAL;
157			break;
158		}
159
160		intel_nested_flush_cache(dmar_domain, inv_entry.addr,
161					 inv_entry.npages,
162					 inv_entry.flags & IOMMU_VTD_INV_FLAGS_LEAF);
163		processed++;
164	}
165
166out:
167	array->entry_num = processed;
168	return ret;
169}
170
171static const struct iommu_domain_ops intel_nested_domain_ops = {
172	.attach_dev		= intel_nested_attach_dev,
173	.free			= intel_nested_domain_free,
174	.cache_invalidate_user	= intel_nested_cache_invalidate_user,
175};
176
177struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent,
178					       const struct iommu_user_data *user_data)
179{
180	struct dmar_domain *s2_domain = to_dmar_domain(parent);
181	struct iommu_hwpt_vtd_s1 vtd;
182	struct dmar_domain *domain;
183	int ret;
184
185	/* Must be nested domain */
186	if (user_data->type != IOMMU_HWPT_DATA_VTD_S1)
187		return ERR_PTR(-EOPNOTSUPP);
188	if (parent->ops != intel_iommu_ops.default_domain_ops ||
189	    !s2_domain->nested_parent)
190		return ERR_PTR(-EINVAL);
191
192	ret = iommu_copy_struct_from_user(&vtd, user_data,
193					  IOMMU_HWPT_DATA_VTD_S1, __reserved);
194	if (ret)
195		return ERR_PTR(ret);
196
197	domain = kzalloc(sizeof(*domain), GFP_KERNEL_ACCOUNT);
198	if (!domain)
199		return ERR_PTR(-ENOMEM);
200
201	domain->use_first_level = true;
202	domain->s2_domain = s2_domain;
203	domain->s1_pgtbl = vtd.pgtbl_addr;
204	domain->s1_cfg = vtd;
205	domain->domain.ops = &intel_nested_domain_ops;
206	domain->domain.type = IOMMU_DOMAIN_NESTED;
207	INIT_LIST_HEAD(&domain->devices);
208	INIT_LIST_HEAD(&domain->dev_pasids);
209	spin_lock_init(&domain->lock);
210	xa_init(&domain->iommu_array);
211
212	spin_lock(&s2_domain->s1_lock);
213	list_add(&domain->s2_link, &s2_domain->s1_domains);
214	spin_unlock(&s2_domain->s1_lock);
215
216	return &domain->domain;
217}