1/*
  2 *  IOMMU helpers in MMU context.
  3 *
  4 *  Copyright (C) 2015 IBM Corp. <aik@ozlabs.ru>
  5 *
  6 *  This program is free software; you can redistribute it and/or
  7 *  modify it under the terms of the GNU General Public License
  8 *  as published by the Free Software Foundation; either version
  9 *  2 of the License, or (at your option) any later version.
 10 *
 11 */
 12
 13#include <linux/sched/signal.h>
 14#include <linux/slab.h>
 15#include <linux/rculist.h>
 16#include <linux/vmalloc.h>
 17#include <linux/mutex.h>
 18#include <linux/migrate.h>
 19#include <linux/hugetlb.h>
 20#include <linux/swap.h>
 21#include <asm/mmu_context.h>
 22
 23static DEFINE_MUTEX(mem_list_mutex);
 24
 25struct mm_iommu_table_group_mem_t {
 26	struct list_head next;
 27	struct rcu_head rcu;
 28	unsigned long used;
 29	atomic64_t mapped;
 30	u64 ua;			/* userspace address */
 31	u64 entries;		/* number of entries in hpas[] */
 32	u64 *hpas;		/* vmalloc'ed */
 33};
 34
 35static long mm_iommu_adjust_locked_vm(struct mm_struct *mm,
 36		unsigned long npages, bool incr)
 37{
 38	long ret = 0, locked, lock_limit;
 39
 40	if (!npages)
 41		return 0;
 42
 43	down_write(&mm->mmap_sem);
 44
 45	if (incr) {
 46		locked = mm->locked_vm + npages;
 47		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 48		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
 49			ret = -ENOMEM;
 50		else
 51			mm->locked_vm += npages;
 52	} else {
 53		if (WARN_ON_ONCE(npages > mm->locked_vm))
 54			npages = mm->locked_vm;
 55		mm->locked_vm -= npages;
 56	}
 57
 58	pr_debug("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n",
 59			current ? current->pid : 0,
 60			incr ? '+' : '-',
 61			npages << PAGE_SHIFT,
 62			mm->locked_vm << PAGE_SHIFT,
 63			rlimit(RLIMIT_MEMLOCK));
 64	up_write(&mm->mmap_sem);
 65
 66	return ret;
 67}
 68
 69bool mm_iommu_preregistered(struct mm_struct *mm)
 70{
 71	return !list_empty(&mm->context.iommu_group_mem_list);
 72}
 73EXPORT_SYMBOL_GPL(mm_iommu_preregistered);
 74
 75/*
 76 * Taken from alloc_migrate_target with changes to remove CMA allocations
 77 */
 78struct page *new_iommu_non_cma_page(struct page *page, unsigned long private)
 79{
 80	gfp_t gfp_mask = GFP_USER;
 81	struct page *new_page;
 82
 83	if (PageCompound(page))
 84		return NULL;
 85
 86	if (PageHighMem(page))
 87		gfp_mask |= __GFP_HIGHMEM;
 88
 89	/*
 90	 * We don't want the allocation to force an OOM if possibe
 91	 */
 92	new_page = alloc_page(gfp_mask | __GFP_NORETRY | __GFP_NOWARN);
 93	return new_page;
 94}
 95
 96static int mm_iommu_move_page_from_cma(struct page *page)
 97{
 98	int ret = 0;
 99	LIST_HEAD(cma_migrate_pages);
100
101	/* Ignore huge pages for now */
102	if (PageCompound(page))
103		return -EBUSY;
104
105	lru_add_drain();
106	ret = isolate_lru_page(page);
107	if (ret)
108		return ret;
109
110	list_add(&page->lru, &cma_migrate_pages);
111	put_page(page); /* Drop the gup reference */
112
113	ret = migrate_pages(&cma_migrate_pages, new_iommu_non_cma_page,
114				NULL, 0, MIGRATE_SYNC, MR_CONTIG_RANGE);
115	if (ret) {
116		if (!list_empty(&cma_migrate_pages))
117			putback_movable_pages(&cma_migrate_pages);
118	}
119
120	return 0;
121}
122
123long mm_iommu_get(struct mm_struct *mm, unsigned long ua, unsigned long entries,
124		struct mm_iommu_table_group_mem_t **pmem)
125{
126	struct mm_iommu_table_group_mem_t *mem;
127	long i, j, ret = 0, locked_entries = 0;
128	struct page *page = NULL;
129
130	mutex_lock(&mem_list_mutex);
131
132	list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list,
133			next) {
134		if ((mem->ua == ua) && (mem->entries == entries)) {
135			++mem->used;
136			*pmem = mem;
137			goto unlock_exit;
138		}
139
140		/* Overlap? */
141		if ((mem->ua < (ua + (entries << PAGE_SHIFT))) &&
142				(ua < (mem->ua +
143				       (mem->entries << PAGE_SHIFT)))) {
144			ret = -EINVAL;
145			goto unlock_exit;
146		}
147
148	}
149
150	ret = mm_iommu_adjust_locked_vm(mm, entries, true);
151	if (ret)
152		goto unlock_exit;
153
154	locked_entries = entries;
155
156	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
157	if (!mem) {
158		ret = -ENOMEM;
159		goto unlock_exit;
160	}
161
162	mem->hpas = vzalloc(entries * sizeof(mem->hpas[0]));
163	if (!mem->hpas) {
164		kfree(mem);
165		ret = -ENOMEM;
166		goto unlock_exit;
167	}
168
169	for (i = 0; i < entries; ++i) {
170		if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT),
171					1/* pages */, 1/* iswrite */, &page)) {
172			ret = -EFAULT;
173			for (j = 0; j < i; ++j)
174				put_page(pfn_to_page(mem->hpas[j] >>
175						PAGE_SHIFT));
176			vfree(mem->hpas);
177			kfree(mem);
178			goto unlock_exit;
179		}
180		/*
181		 * If we get a page from the CMA zone, since we are going to
182		 * be pinning these entries, we might as well move them out
183		 * of the CMA zone if possible. NOTE: faulting in + migration
184		 * can be expensive. Batching can be considered later
185		 */
186		if (is_migrate_cma_page(page)) {
187			if (mm_iommu_move_page_from_cma(page))
188				goto populate;
189			if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT),
190						1/* pages */, 1/* iswrite */,
191						&page)) {
192				ret = -EFAULT;
193				for (j = 0; j < i; ++j)
194					put_page(pfn_to_page(mem->hpas[j] >>
195								PAGE_SHIFT));
196				vfree(mem->hpas);
197				kfree(mem);
198				goto unlock_exit;
199			}
200		}
201populate:
202		mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT;
203	}
204
205	atomic64_set(&mem->mapped, 1);
206	mem->used = 1;
207	mem->ua = ua;
208	mem->entries = entries;
209	*pmem = mem;
210
211	list_add_rcu(&mem->next, &mm->context.iommu_group_mem_list);
212
213unlock_exit:
214	if (locked_entries && ret)
215		mm_iommu_adjust_locked_vm(mm, locked_entries, false);
216
217	mutex_unlock(&mem_list_mutex);
218
219	return ret;
220}
221EXPORT_SYMBOL_GPL(mm_iommu_get);
222
223static void mm_iommu_unpin(struct mm_iommu_table_group_mem_t *mem)
224{
225	long i;
226	struct page *page = NULL;
227
228	for (i = 0; i < mem->entries; ++i) {
229		if (!mem->hpas[i])
230			continue;
231
232		page = pfn_to_page(mem->hpas[i] >> PAGE_SHIFT);
233		if (!page)
234			continue;
235
236		put_page(page);
237		mem->hpas[i] = 0;
238	}
239}
240
241static void mm_iommu_do_free(struct mm_iommu_table_group_mem_t *mem)
242{
243
244	mm_iommu_unpin(mem);
245	vfree(mem->hpas);
246	kfree(mem);
247}
248
249static void mm_iommu_free(struct rcu_head *head)
250{
251	struct mm_iommu_table_group_mem_t *mem = container_of(head,
252			struct mm_iommu_table_group_mem_t, rcu);
253
254	mm_iommu_do_free(mem);
255}
256
257static void mm_iommu_release(struct mm_iommu_table_group_mem_t *mem)
258{
259	list_del_rcu(&mem->next);
260	call_rcu(&mem->rcu, mm_iommu_free);
261}
262
263long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem)
264{
265	long ret = 0;
266
267	mutex_lock(&mem_list_mutex);
268
269	if (mem->used == 0) {
270		ret = -ENOENT;
271		goto unlock_exit;
272	}
273
274	--mem->used;
275	/* There are still users, exit */
276	if (mem->used)
277		goto unlock_exit;
278
279	/* Are there still mappings? */
280	if (atomic_cmpxchg(&mem->mapped, 1, 0) != 1) {
281		++mem->used;
282		ret = -EBUSY;
283		goto unlock_exit;
284	}
285
286	/* @mapped became 0 so now mappings are disabled, release the region */
287	mm_iommu_release(mem);
288
289	mm_iommu_adjust_locked_vm(mm, mem->entries, false);
290
291unlock_exit:
292	mutex_unlock(&mem_list_mutex);
293
294	return ret;
295}
296EXPORT_SYMBOL_GPL(mm_iommu_put);
297
298struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
299		unsigned long ua, unsigned long size)
300{
301	struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
302
303	list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
304		if ((mem->ua <= ua) &&
305				(ua + size <= mem->ua +
306				 (mem->entries << PAGE_SHIFT))) {
307			ret = mem;
308			break;
309		}
310	}
311
312	return ret;
313}
314EXPORT_SYMBOL_GPL(mm_iommu_lookup);
315
316struct mm_iommu_table_group_mem_t *mm_iommu_lookup_rm(struct mm_struct *mm,
317		unsigned long ua, unsigned long size)
318{
319	struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
320
321	list_for_each_entry_lockless(mem, &mm->context.iommu_group_mem_list,
322			next) {
323		if ((mem->ua <= ua) &&
324				(ua + size <= mem->ua +
325				 (mem->entries << PAGE_SHIFT))) {
326			ret = mem;
327			break;
328		}
329	}
330
331	return ret;
332}
333EXPORT_SYMBOL_GPL(mm_iommu_lookup_rm);
334
335struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm,
336		unsigned long ua, unsigned long entries)
337{
338	struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
339
340	list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
341		if ((mem->ua == ua) && (mem->entries == entries)) {
342			ret = mem;
343			break;
344		}
345	}
346
347	return ret;
348}
349EXPORT_SYMBOL_GPL(mm_iommu_find);
350
351long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
352		unsigned long ua, unsigned long *hpa)
353{
354	const long entry = (ua - mem->ua) >> PAGE_SHIFT;
355	u64 *va = &mem->hpas[entry];
356
357	if (entry >= mem->entries)
358		return -EFAULT;
359
360	*hpa = *va | (ua & ~PAGE_MASK);
361
362	return 0;
363}
364EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa);
365
366long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem,
367		unsigned long ua, unsigned long *hpa)
368{
369	const long entry = (ua - mem->ua) >> PAGE_SHIFT;
370	void *va = &mem->hpas[entry];
371	unsigned long *pa;
372
373	if (entry >= mem->entries)
374		return -EFAULT;
375
376	pa = (void *) vmalloc_to_phys(va);
377	if (!pa)
378		return -EFAULT;
379
380	*hpa = *pa | (ua & ~PAGE_MASK);
381
382	return 0;
383}
384EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa_rm);
385
386long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem)
387{
388	if (atomic64_inc_not_zero(&mem->mapped))
389		return 0;
390
391	/* Last mm_iommu_put() has been called, no more mappings allowed() */
392	return -ENXIO;
393}
394EXPORT_SYMBOL_GPL(mm_iommu_mapped_inc);
395
396void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem)
397{
398	atomic64_add_unless(&mem->mapped, -1, 1);
399}
400EXPORT_SYMBOL_GPL(mm_iommu_mapped_dec);
401
402void mm_iommu_init(struct mm_struct *mm)
403{
404	INIT_LIST_HEAD_RCU(&mm->context.iommu_group_mem_list);
405}