1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2014-2016 Intel Corporation
  5 */
  6
  7#include <linux/highmem.h>
  8#include <linux/shmem_fs.h>
  9#include <linux/swap.h>
 10
 11#include <drm/drm.h> /* for drm_legacy.h! */
 12#include <drm/drm_cache.h>
 13
 14#include "gt/intel_gt.h"
 15#include "i915_drv.h"
 16#include "i915_gem_object.h"
 17#include "i915_gem_region.h"
 18#include "i915_scatterlist.h"
 19
 20static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 21{
 22	struct address_space *mapping = obj->base.filp->f_mapping;
 23	struct scatterlist *sg;
 24	struct sg_table *st;
 25	dma_addr_t dma;
 26	void *vaddr;
 27	void *dst;
 28	int i;
 29
 30	if (GEM_WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 31		return -EINVAL;
 32
 33	/*
 34	 * Always aligning to the object size, allows a single allocation
 35	 * to handle all possible callers, and given typical object sizes,
 36	 * the alignment of the buddy allocation will naturally match.
 37	 */
 38	vaddr = dma_alloc_coherent(&obj->base.dev->pdev->dev,
 39				   roundup_pow_of_two(obj->base.size),
 40				   &dma, GFP_KERNEL);
 41	if (!vaddr)
 42		return -ENOMEM;
 43
 44	st = kmalloc(sizeof(*st), GFP_KERNEL);
 45	if (!st)
 46		goto err_pci;
 47
 48	if (sg_alloc_table(st, 1, GFP_KERNEL))
 49		goto err_st;
 50
 51	sg = st->sgl;
 52	sg->offset = 0;
 53	sg->length = obj->base.size;
 54
 55	sg_assign_page(sg, (struct page *)vaddr);
 56	sg_dma_address(sg) = dma;
 57	sg_dma_len(sg) = obj->base.size;
 58
 59	dst = vaddr;
 60	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 61		struct page *page;
 62		void *src;
 63
 64		page = shmem_read_mapping_page(mapping, i);
 65		if (IS_ERR(page))
 66			goto err_st;
 67
 68		src = kmap_atomic(page);
 69		memcpy(dst, src, PAGE_SIZE);
 70		drm_clflush_virt_range(dst, PAGE_SIZE);
 71		kunmap_atomic(src);
 72
 73		put_page(page);
 74		dst += PAGE_SIZE;
 75	}
 76
 77	intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt);
 78
 79	__i915_gem_object_set_pages(obj, st, sg->length);
 80
 81	return 0;
 82
 83err_st:
 84	kfree(st);
 85err_pci:
 86	dma_free_coherent(&obj->base.dev->pdev->dev,
 87			  roundup_pow_of_two(obj->base.size),
 88			  vaddr, dma);
 89	return -ENOMEM;
 90}
 91
 92static void
 93i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
 94			       struct sg_table *pages)
 95{
 96	dma_addr_t dma = sg_dma_address(pages->sgl);
 97	void *vaddr = sg_page(pages->sgl);
 98
 99	__i915_gem_object_release_shmem(obj, pages, false);
100
101	if (obj->mm.dirty) {
102		struct address_space *mapping = obj->base.filp->f_mapping;
103		void *src = vaddr;
104		int i;
105
106		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
107			struct page *page;
108			char *dst;
109
110			page = shmem_read_mapping_page(mapping, i);
111			if (IS_ERR(page))
112				continue;
113
114			dst = kmap_atomic(page);
115			drm_clflush_virt_range(src, PAGE_SIZE);
116			memcpy(dst, src, PAGE_SIZE);
117			kunmap_atomic(dst);
118
119			set_page_dirty(page);
120			if (obj->mm.madv == I915_MADV_WILLNEED)
121				mark_page_accessed(page);
122			put_page(page);
123
124			src += PAGE_SIZE;
125		}
126		obj->mm.dirty = false;
127	}
128
129	sg_free_table(pages);
130	kfree(pages);
131
132	dma_free_coherent(&obj->base.dev->pdev->dev,
133			  roundup_pow_of_two(obj->base.size),
134			  vaddr, dma);
135}
136
137static void phys_release(struct drm_i915_gem_object *obj)
138{
139	fput(obj->base.filp);
140}
141
142static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
143	.name = "i915_gem_object_phys",
144	.get_pages = i915_gem_object_get_pages_phys,
145	.put_pages = i915_gem_object_put_pages_phys,
146
147	.release = phys_release,
148};
149
150int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
151{
152	struct sg_table *pages;
153	int err;
154
155	if (align > obj->base.size)
156		return -EINVAL;
157
158	if (obj->ops == &i915_gem_phys_ops)
159		return 0;
160
161	if (obj->ops != &i915_gem_shmem_ops)
162		return -EINVAL;
163
164	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
165	if (err)
166		return err;
167
168	mutex_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES);
169
170	if (obj->mm.madv != I915_MADV_WILLNEED) {
171		err = -EFAULT;
172		goto err_unlock;
173	}
174
175	if (obj->mm.quirked) {
176		err = -EFAULT;
177		goto err_unlock;
178	}
179
180	if (obj->mm.mapping) {
181		err = -EBUSY;
182		goto err_unlock;
183	}
184
185	pages = __i915_gem_object_unset_pages(obj);
186
187	obj->ops = &i915_gem_phys_ops;
188
189	err = ____i915_gem_object_get_pages(obj);
190	if (err)
191		goto err_xfer;
192
193	/* Perma-pin (until release) the physical set of pages */
194	__i915_gem_object_pin_pages(obj);
195
196	if (!IS_ERR_OR_NULL(pages))
197		i915_gem_shmem_ops.put_pages(obj, pages);
198
199	i915_gem_object_release_memory_region(obj);
200
201	mutex_unlock(&obj->mm.lock);
202	return 0;
203
204err_xfer:
205	obj->ops = &i915_gem_shmem_ops;
206	if (!IS_ERR_OR_NULL(pages)) {
207		unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl);
208
209		__i915_gem_object_set_pages(obj, pages, sg_page_sizes);
210	}
211err_unlock:
212	mutex_unlock(&obj->mm.lock);
213	return err;
214}
215
216#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
217#include "selftests/i915_gem_phys.c"
218#endif