1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2014-2016 Intel Corporation
  5 */
  6
  7#include <linux/highmem.h>
  8#include <linux/shmem_fs.h>
  9#include <linux/swap.h>
 10
 11#include <drm/drm.h> /* for drm_legacy.h! */
 12#include <drm/drm_cache.h>
 13#include <drm/drm_legacy.h> /* for drm_pci.h! */
 14#include <drm/drm_pci.h>
 15
 16#include "gt/intel_gt.h"
 17#include "i915_drv.h"
 18#include "i915_gem_object.h"
 19#include "i915_scatterlist.h"
 20
 21static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 22{
 23	struct address_space *mapping = obj->base.filp->f_mapping;
 24	struct drm_dma_handle *phys;
 25	struct sg_table *st;
 26	struct scatterlist *sg;
 27	char *vaddr;
 28	int i;
 29	int err;
 30
 31	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 32		return -EINVAL;
 33
 34	/* Always aligning to the object size, allows a single allocation
 35	 * to handle all possible callers, and given typical object sizes,
 36	 * the alignment of the buddy allocation will naturally match.
 37	 */
 38	phys = drm_pci_alloc(obj->base.dev,
 39			     roundup_pow_of_two(obj->base.size),
 40			     roundup_pow_of_two(obj->base.size));
 41	if (!phys)
 42		return -ENOMEM;
 43
 44	vaddr = phys->vaddr;
 45	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 46		struct page *page;
 47		char *src;
 48
 49		page = shmem_read_mapping_page(mapping, i);
 50		if (IS_ERR(page)) {
 51			err = PTR_ERR(page);
 52			goto err_phys;
 53		}
 54
 55		src = kmap_atomic(page);
 56		memcpy(vaddr, src, PAGE_SIZE);
 57		drm_clflush_virt_range(vaddr, PAGE_SIZE);
 58		kunmap_atomic(src);
 59
 60		put_page(page);
 61		vaddr += PAGE_SIZE;
 62	}
 63
 64	intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt);
 65
 66	st = kmalloc(sizeof(*st), GFP_KERNEL);
 67	if (!st) {
 68		err = -ENOMEM;
 69		goto err_phys;
 70	}
 71
 72	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 73		kfree(st);
 74		err = -ENOMEM;
 75		goto err_phys;
 76	}
 77
 78	sg = st->sgl;
 79	sg->offset = 0;
 80	sg->length = obj->base.size;
 81
 82	sg_dma_address(sg) = phys->busaddr;
 83	sg_dma_len(sg) = obj->base.size;
 84
 85	obj->phys_handle = phys;
 86
 87	__i915_gem_object_set_pages(obj, st, sg->length);
 88
 89	return 0;
 90
 91err_phys:
 92	drm_pci_free(obj->base.dev, phys);
 93
 94	return err;
 95}
 96
 97static void
 98i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
 99			       struct sg_table *pages)
100{
101	__i915_gem_object_release_shmem(obj, pages, false);
102
103	if (obj->mm.dirty) {
104		struct address_space *mapping = obj->base.filp->f_mapping;
105		char *vaddr = obj->phys_handle->vaddr;
106		int i;
107
108		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
109			struct page *page;
110			char *dst;
111
112			page = shmem_read_mapping_page(mapping, i);
113			if (IS_ERR(page))
114				continue;
115
116			dst = kmap_atomic(page);
117			drm_clflush_virt_range(vaddr, PAGE_SIZE);
118			memcpy(dst, vaddr, PAGE_SIZE);
119			kunmap_atomic(dst);
120
121			set_page_dirty(page);
122			if (obj->mm.madv == I915_MADV_WILLNEED)
123				mark_page_accessed(page);
124			put_page(page);
125			vaddr += PAGE_SIZE;
126		}
127		obj->mm.dirty = false;
128	}
129
130	sg_free_table(pages);
131	kfree(pages);
132
133	drm_pci_free(obj->base.dev, obj->phys_handle);
134}
135
136static void phys_release(struct drm_i915_gem_object *obj)
137{
138	fput(obj->base.filp);
139}
140
141static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
142	.get_pages = i915_gem_object_get_pages_phys,
143	.put_pages = i915_gem_object_put_pages_phys,
144
145	.release = phys_release,
146};
147
148int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
149{
150	struct sg_table *pages;
151	int err;
152
153	if (align > obj->base.size)
154		return -EINVAL;
155
156	if (obj->ops == &i915_gem_phys_ops)
157		return 0;
158
159	if (obj->ops != &i915_gem_shmem_ops)
160		return -EINVAL;
161
162	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
163	if (err)
164		return err;
165
166	mutex_lock(&obj->mm.lock);
167
168	if (obj->mm.madv != I915_MADV_WILLNEED) {
169		err = -EFAULT;
170		goto err_unlock;
171	}
172
173	if (obj->mm.quirked) {
174		err = -EFAULT;
175		goto err_unlock;
176	}
177
178	if (obj->mm.mapping) {
179		err = -EBUSY;
180		goto err_unlock;
181	}
182
183	pages = __i915_gem_object_unset_pages(obj);
184
185	obj->ops = &i915_gem_phys_ops;
186
187	err = ____i915_gem_object_get_pages(obj);
188	if (err)
189		goto err_xfer;
190
191	/* Perma-pin (until release) the physical set of pages */
192	__i915_gem_object_pin_pages(obj);
193
194	if (!IS_ERR_OR_NULL(pages))
195		i915_gem_shmem_ops.put_pages(obj, pages);
196	mutex_unlock(&obj->mm.lock);
197	return 0;
198
199err_xfer:
200	obj->ops = &i915_gem_shmem_ops;
201	if (!IS_ERR_OR_NULL(pages)) {
202		unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl);
203
204		__i915_gem_object_set_pages(obj, pages, sg_page_sizes);
205	}
206err_unlock:
207	mutex_unlock(&obj->mm.lock);
208	return err;
209}
210
211#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
212#include "selftests/i915_gem_phys.c"
213#endif