1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2014-2016 Intel Corporation
  5 */
  6
  7#include "i915_drv.h"
  8#include "i915_gem_object.h"
  9#include "i915_scatterlist.h"
 10#include "i915_gem_lmem.h"
 11#include "i915_gem_mman.h"
 12
 13void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
 14				 struct sg_table *pages,
 15				 unsigned int sg_page_sizes)
 16{
 17	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 18	unsigned long supported = INTEL_INFO(i915)->page_sizes;
 19	bool shrinkable;
 20	int i;
 21
 22	assert_object_held_shared(obj);
 23
 24	if (i915_gem_object_is_volatile(obj))
 25		obj->mm.madv = I915_MADV_DONTNEED;
 26
 27	/* Make the pages coherent with the GPU (flushing any swapin). */
 28	if (obj->cache_dirty) {
 29		obj->write_domain = 0;
 30		if (i915_gem_object_has_struct_page(obj))
 31			drm_clflush_sg(pages);
 32		obj->cache_dirty = false;
 33	}
 34
 35	obj->mm.get_page.sg_pos = pages->sgl;
 36	obj->mm.get_page.sg_idx = 0;
 37	obj->mm.get_dma_page.sg_pos = pages->sgl;
 38	obj->mm.get_dma_page.sg_idx = 0;
 39
 40	obj->mm.pages = pages;
 41
 42	GEM_BUG_ON(!sg_page_sizes);
 43	obj->mm.page_sizes.phys = sg_page_sizes;
 44
 45	/*
 46	 * Calculate the supported page-sizes which fit into the given
 47	 * sg_page_sizes. This will give us the page-sizes which we may be able
 48	 * to use opportunistically when later inserting into the GTT. For
 49	 * example if phys=2G, then in theory we should be able to use 1G, 2M,
 50	 * 64K or 4K pages, although in practice this will depend on a number of
 51	 * other factors.
 52	 */
 53	obj->mm.page_sizes.sg = 0;
 54	for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 55		if (obj->mm.page_sizes.phys & ~0u << i)
 56			obj->mm.page_sizes.sg |= BIT(i);
 57	}
 58	GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
 59
 60	shrinkable = i915_gem_object_is_shrinkable(obj);
 61
 62	if (i915_gem_object_is_tiled(obj) &&
 63	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
 64		GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
 65		i915_gem_object_set_tiling_quirk(obj);
 66		GEM_BUG_ON(!list_empty(&obj->mm.link));
 67		atomic_inc(&obj->mm.shrink_pin);
 68		shrinkable = false;
 69	}
 70
 71	if (shrinkable) {
 72		struct list_head *list;
 73		unsigned long flags;
 74
 75		assert_object_held(obj);
 76		spin_lock_irqsave(&i915->mm.obj_lock, flags);
 77
 78		i915->mm.shrink_count++;
 79		i915->mm.shrink_memory += obj->base.size;
 80
 81		if (obj->mm.madv != I915_MADV_WILLNEED)
 82			list = &i915->mm.purge_list;
 83		else
 84			list = &i915->mm.shrink_list;
 85		list_add_tail(&obj->mm.link, list);
 86
 87		atomic_set(&obj->mm.shrink_pin, 0);
 88		spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
 89	}
 90}
 91
 92int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
 93{
 94	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 95	int err;
 96
 97	assert_object_held_shared(obj);
 98
 99	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
100		drm_dbg(&i915->drm,
101			"Attempting to obtain a purgeable object\n");
102		return -EFAULT;
103	}
104
105	err = obj->ops->get_pages(obj);
106	GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
107
108	return err;
109}
110
111/* Ensure that the associated pages are gathered from the backing storage
112 * and pinned into our object. i915_gem_object_pin_pages() may be called
113 * multiple times before they are released by a single call to
114 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
115 * either as a result of memory pressure (reaping pages under the shrinker)
116 * or as the object is itself released.
117 */
118int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
119{
120	int err;
121
122	assert_object_held(obj);
123
124	assert_object_held_shared(obj);
125
126	if (unlikely(!i915_gem_object_has_pages(obj))) {
127		GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
128
129		err = ____i915_gem_object_get_pages(obj);
130		if (err)
131			return err;
132
133		smp_mb__before_atomic();
134	}
135	atomic_inc(&obj->mm.pages_pin_count);
136
137	return 0;
138}
139
140int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)
141{
142	struct i915_gem_ww_ctx ww;
143	int err;
144
145	i915_gem_ww_ctx_init(&ww, true);
146retry:
147	err = i915_gem_object_lock(obj, &ww);
148	if (!err)
149		err = i915_gem_object_pin_pages(obj);
150
151	if (err == -EDEADLK) {
152		err = i915_gem_ww_ctx_backoff(&ww);
153		if (!err)
154			goto retry;
155	}
156	i915_gem_ww_ctx_fini(&ww);
157	return err;
158}
159
160/* Immediately discard the backing storage */
161void i915_gem_object_truncate(struct drm_i915_gem_object *obj)
162{
163	drm_gem_free_mmap_offset(&obj->base);
164	if (obj->ops->truncate)
165		obj->ops->truncate(obj);
166}
167
168/* Try to discard unwanted pages */
169void i915_gem_object_writeback(struct drm_i915_gem_object *obj)
170{
171	assert_object_held_shared(obj);
172	GEM_BUG_ON(i915_gem_object_has_pages(obj));
173
174	if (obj->ops->writeback)
175		obj->ops->writeback(obj);
176}
177
178static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
179{
180	struct radix_tree_iter iter;
181	void __rcu **slot;
182
183	rcu_read_lock();
184	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
185		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
186	radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0)
187		radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index);
188	rcu_read_unlock();
189}
190
191static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
192{
193	if (is_vmalloc_addr(ptr))
194		vunmap(ptr);
195}
196
197struct sg_table *
198__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
199{
200	struct sg_table *pages;
201
202	assert_object_held_shared(obj);
203
204	pages = fetch_and_zero(&obj->mm.pages);
205	if (IS_ERR_OR_NULL(pages))
206		return pages;
207
208	if (i915_gem_object_is_volatile(obj))
209		obj->mm.madv = I915_MADV_WILLNEED;
210
211	i915_gem_object_make_unshrinkable(obj);
212
213	if (obj->mm.mapping) {
214		unmap_object(obj, page_mask_bits(obj->mm.mapping));
215		obj->mm.mapping = NULL;
216	}
217
218	__i915_gem_object_reset_page_iter(obj);
219	obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
220
221	return pages;
222}
223
224int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
225{
226	struct sg_table *pages;
227
228	if (i915_gem_object_has_pinned_pages(obj))
229		return -EBUSY;
230
231	/* May be called by shrinker from within get_pages() (on another bo) */
232	assert_object_held_shared(obj);
233
234	i915_gem_object_release_mmap_offset(obj);
235
236	/*
237	 * ->put_pages might need to allocate memory for the bit17 swizzle
238	 * array, hence protect them from being reaped by removing them from gtt
239	 * lists early.
240	 */
241	pages = __i915_gem_object_unset_pages(obj);
242
243	/*
244	 * XXX Temporary hijinx to avoid updating all backends to handle
245	 * NULL pages. In the future, when we have more asynchronous
246	 * get_pages backends we should be better able to handle the
247	 * cancellation of the async task in a more uniform manner.
248	 */
249	if (!IS_ERR_OR_NULL(pages))
250		obj->ops->put_pages(obj, pages);
251
252	return 0;
253}
254
255/* The 'mapping' part of i915_gem_object_pin_map() below */
256static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,
257				      enum i915_map_type type)
258{
259	unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
260	struct page *stack[32], **pages = stack, *page;
261	struct sgt_iter iter;
262	pgprot_t pgprot;
263	void *vaddr;
264
265	switch (type) {
266	default:
267		MISSING_CASE(type);
268		fallthrough;	/* to use PAGE_KERNEL anyway */
269	case I915_MAP_WB:
270		/*
271		 * On 32b, highmem using a finite set of indirect PTE (i.e.
272		 * vmap) to provide virtual mappings of the high pages.
273		 * As these are finite, map_new_virtual() must wait for some
274		 * other kmap() to finish when it runs out. If we map a large
275		 * number of objects, there is no method for it to tell us
276		 * to release the mappings, and we deadlock.
277		 *
278		 * However, if we make an explicit vmap of the page, that
279		 * uses a larger vmalloc arena, and also has the ability
280		 * to tell us to release unwanted mappings. Most importantly,
281		 * it will fail and propagate an error instead of waiting
282		 * forever.
283		 *
284		 * So if the page is beyond the 32b boundary, make an explicit
285		 * vmap.
286		 */
287		if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl)))
288			return page_address(sg_page(obj->mm.pages->sgl));
289		pgprot = PAGE_KERNEL;
290		break;
291	case I915_MAP_WC:
292		pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
293		break;
294	}
295
296	if (n_pages > ARRAY_SIZE(stack)) {
297		/* Too big for stack -- allocate temporary array instead */
298		pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
299		if (!pages)
300			return ERR_PTR(-ENOMEM);
301	}
302
303	i = 0;
304	for_each_sgt_page(page, iter, obj->mm.pages)
305		pages[i++] = page;
306	vaddr = vmap(pages, n_pages, 0, pgprot);
307	if (pages != stack)
308		kvfree(pages);
309
310	return vaddr ?: ERR_PTR(-ENOMEM);
311}
312
313static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
314				     enum i915_map_type type)
315{
316	resource_size_t iomap = obj->mm.region->iomap.base -
317		obj->mm.region->region.start;
318	unsigned long n_pfn = obj->base.size >> PAGE_SHIFT;
319	unsigned long stack[32], *pfns = stack, i;
320	struct sgt_iter iter;
321	dma_addr_t addr;
322	void *vaddr;
323
324	if (type != I915_MAP_WC)
325		return ERR_PTR(-ENODEV);
326
327	if (n_pfn > ARRAY_SIZE(stack)) {
328		/* Too big for stack -- allocate temporary array instead */
329		pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL);
330		if (!pfns)
331			return ERR_PTR(-ENOMEM);
332	}
333
334	i = 0;
335	for_each_sgt_daddr(addr, iter, obj->mm.pages)
336		pfns[i++] = (iomap + addr) >> PAGE_SHIFT;
337	vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO));
338	if (pfns != stack)
339		kvfree(pfns);
340
341	return vaddr ?: ERR_PTR(-ENOMEM);
342}
343
344/* get, pin, and map the pages of the object into kernel space */
345void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
346			      enum i915_map_type type)
347{
348	enum i915_map_type has_type;
349	bool pinned;
350	void *ptr;
351	int err;
352
353	if (!i915_gem_object_has_struct_page(obj) &&
354	    !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM))
355		return ERR_PTR(-ENXIO);
356
357	assert_object_held(obj);
358
359	pinned = !(type & I915_MAP_OVERRIDE);
360	type &= ~I915_MAP_OVERRIDE;
361
362	if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
363		if (unlikely(!i915_gem_object_has_pages(obj))) {
364			GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
365
366			err = ____i915_gem_object_get_pages(obj);
367			if (err)
368				return ERR_PTR(err);
369
370			smp_mb__before_atomic();
371		}
372		atomic_inc(&obj->mm.pages_pin_count);
373		pinned = false;
374	}
375	GEM_BUG_ON(!i915_gem_object_has_pages(obj));
376
377	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
378	if (ptr && has_type != type) {
379		if (pinned) {
380			ptr = ERR_PTR(-EBUSY);
381			goto err_unpin;
382		}
383
384		unmap_object(obj, ptr);
385
386		ptr = obj->mm.mapping = NULL;
387	}
388
389	if (!ptr) {
390		if (GEM_WARN_ON(type == I915_MAP_WC &&
391				!static_cpu_has(X86_FEATURE_PAT)))
392			ptr = ERR_PTR(-ENODEV);
393		else if (i915_gem_object_has_struct_page(obj))
394			ptr = i915_gem_object_map_page(obj, type);
395		else
396			ptr = i915_gem_object_map_pfn(obj, type);
397		if (IS_ERR(ptr))
398			goto err_unpin;
399
400		obj->mm.mapping = page_pack_bits(ptr, type);
401	}
402
403	return ptr;
404
405err_unpin:
406	atomic_dec(&obj->mm.pages_pin_count);
407	return ptr;
408}
409
410void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
411				       enum i915_map_type type)
412{
413	void *ret;
414
415	i915_gem_object_lock(obj, NULL);
416	ret = i915_gem_object_pin_map(obj, type);
417	i915_gem_object_unlock(obj);
418
419	return ret;
420}
421
422void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
423				 unsigned long offset,
424				 unsigned long size)
425{
426	enum i915_map_type has_type;
427	void *ptr;
428
429	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
430	GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
431				     offset, size, obj->base.size));
432
433	wmb(); /* let all previous writes be visible to coherent partners */
434	obj->mm.dirty = true;
435
436	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
437		return;
438
439	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
440	if (has_type == I915_MAP_WC)
441		return;
442
443	drm_clflush_virt_range(ptr + offset, size);
444	if (size == obj->base.size) {
445		obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
446		obj->cache_dirty = false;
447	}
448}
449
450void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
451{
452	GEM_BUG_ON(!obj->mm.mapping);
453
454	/*
455	 * We allow removing the mapping from underneath pinned pages!
456	 *
457	 * Furthermore, since this is an unsafe operation reserved only
458	 * for construction time manipulation, we ignore locking prudence.
459	 */
460	unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));
461
462	i915_gem_object_unpin_map(obj);
463}
464
465struct scatterlist *
466__i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
467			 struct i915_gem_object_page_iter *iter,
468			 unsigned int n,
469			 unsigned int *offset,
470			 bool allow_alloc)
471{
472	const bool dma = iter == &obj->mm.get_dma_page;
473	struct scatterlist *sg;
474	unsigned int idx, count;
475
476	might_sleep();
477	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
478	if (!i915_gem_object_has_pinned_pages(obj))
479		assert_object_held(obj);
480
481	/* As we iterate forward through the sg, we record each entry in a
482	 * radixtree for quick repeated (backwards) lookups. If we have seen
483	 * this index previously, we will have an entry for it.
484	 *
485	 * Initial lookup is O(N), but this is amortized to O(1) for
486	 * sequential page access (where each new request is consecutive
487	 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
488	 * i.e. O(1) with a large constant!
489	 */
490	if (n < READ_ONCE(iter->sg_idx))
491		goto lookup;
492
493	if (!allow_alloc)
494		goto manual_lookup;
495
496	mutex_lock(&iter->lock);
497
498	/* We prefer to reuse the last sg so that repeated lookup of this
499	 * (or the subsequent) sg are fast - comparing against the last
500	 * sg is faster than going through the radixtree.
501	 */
502
503	sg = iter->sg_pos;
504	idx = iter->sg_idx;
505	count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
506
507	while (idx + count <= n) {
508		void *entry;
509		unsigned long i;
510		int ret;
511
512		/* If we cannot allocate and insert this entry, or the
513		 * individual pages from this range, cancel updating the
514		 * sg_idx so that on this lookup we are forced to linearly
515		 * scan onwards, but on future lookups we will try the
516		 * insertion again (in which case we need to be careful of
517		 * the error return reporting that we have already inserted
518		 * this index).
519		 */
520		ret = radix_tree_insert(&iter->radix, idx, sg);
521		if (ret && ret != -EEXIST)
522			goto scan;
523
524		entry = xa_mk_value(idx);
525		for (i = 1; i < count; i++) {
526			ret = radix_tree_insert(&iter->radix, idx + i, entry);
527			if (ret && ret != -EEXIST)
528				goto scan;
529		}
530
531		idx += count;
532		sg = ____sg_next(sg);
533		count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
534	}
535
536scan:
537	iter->sg_pos = sg;
538	iter->sg_idx = idx;
539
540	mutex_unlock(&iter->lock);
541
542	if (unlikely(n < idx)) /* insertion completed by another thread */
543		goto lookup;
544
545	goto manual_walk;
546
547manual_lookup:
548	idx = 0;
549	sg = obj->mm.pages->sgl;
550	count = __sg_page_count(sg);
551
552manual_walk:
553	/*
554	 * In case we failed to insert the entry into the radixtree, we need
555	 * to look beyond the current sg.
556	 */
557	while (idx + count <= n) {
558		idx += count;
559		sg = ____sg_next(sg);
560		count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
561	}
562
563	*offset = n - idx;
564	return sg;
565
566lookup:
567	rcu_read_lock();
568
569	sg = radix_tree_lookup(&iter->radix, n);
570	GEM_BUG_ON(!sg);
571
572	/* If this index is in the middle of multi-page sg entry,
573	 * the radix tree will contain a value entry that points
574	 * to the start of that range. We will return the pointer to
575	 * the base page and the offset of this page within the
576	 * sg entry's range.
577	 */
578	*offset = 0;
579	if (unlikely(xa_is_value(sg))) {
580		unsigned long base = xa_to_value(sg);
581
582		sg = radix_tree_lookup(&iter->radix, base);
583		GEM_BUG_ON(!sg);
584
585		*offset = n - base;
586	}
587
588	rcu_read_unlock();
589
590	return sg;
591}
592
593struct page *
594i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
595{
596	struct scatterlist *sg;
597	unsigned int offset;
598
599	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
600
601	sg = i915_gem_object_get_sg(obj, n, &offset, true);
602	return nth_page(sg_page(sg), offset);
603}
604
605/* Like i915_gem_object_get_page(), but mark the returned page dirty */
606struct page *
607i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
608			       unsigned int n)
609{
610	struct page *page;
611
612	page = i915_gem_object_get_page(obj, n);
613	if (!obj->mm.dirty)
614		set_page_dirty(page);
615
616	return page;
617}
618
619dma_addr_t
620i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
621				    unsigned long n,
622				    unsigned int *len)
623{
624	struct scatterlist *sg;
625	unsigned int offset;
626
627	sg = i915_gem_object_get_sg_dma(obj, n, &offset, true);
628
629	if (len)
630		*len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
631
632	return sg_dma_address(sg) + (offset << PAGE_SHIFT);
633}
634
635dma_addr_t
636i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
637				unsigned long n)
638{
639	return i915_gem_object_get_dma_address_len(obj, n, NULL);
640}