1/*
  2 *   linux/mm/fremap.c
  3 * 
  4 * Explicit pagetable population and nonlinear (random) mappings support.
  5 *
  6 * started by Ingo Molnar, Copyright (C) 2002, 2003
  7 */
  8#include <linux/backing-dev.h>
  9#include <linux/mm.h>
 10#include <linux/swap.h>
 11#include <linux/file.h>
 12#include <linux/mman.h>
 13#include <linux/pagemap.h>
 14#include <linux/swapops.h>
 15#include <linux/rmap.h>
 16#include <linux/module.h>
 17#include <linux/syscalls.h>
 18#include <linux/mmu_notifier.h>
 19
 20#include <asm/mmu_context.h>
 21#include <asm/cacheflush.h>
 22#include <asm/tlbflush.h>
 23
 24#include "internal.h"
 25
 26static void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma,
 27			unsigned long addr, pte_t *ptep)
 28{
 29	pte_t pte = *ptep;
 30
 31	if (pte_present(pte)) {
 32		struct page *page;
 33
 34		flush_cache_page(vma, addr, pte_pfn(pte));
 35		pte = ptep_clear_flush(vma, addr, ptep);
 36		page = vm_normal_page(vma, addr, pte);
 37		if (page) {
 38			if (pte_dirty(pte))
 39				set_page_dirty(page);
 40			page_remove_rmap(page);
 41			page_cache_release(page);
 42			update_hiwater_rss(mm);
 43			dec_mm_counter(mm, MM_FILEPAGES);
 44		}
 45	} else {
 46		if (!pte_file(pte))
 47			free_swap_and_cache(pte_to_swp_entry(pte));
 48		pte_clear_not_present_full(mm, addr, ptep, 0);
 49	}
 50}
 51
 52/*
 53 * Install a file pte to a given virtual memory address, release any
 54 * previously existing mapping.
 55 */
 56static int install_file_pte(struct mm_struct *mm, struct vm_area_struct *vma,
 57		unsigned long addr, unsigned long pgoff, pgprot_t prot)
 58{
 59	int err = -ENOMEM;
 60	pte_t *pte;
 61	spinlock_t *ptl;
 62
 63	pte = get_locked_pte(mm, addr, &ptl);
 64	if (!pte)
 65		goto out;
 66
 67	if (!pte_none(*pte))
 68		zap_pte(mm, vma, addr, pte);
 69
 70	set_pte_at(mm, addr, pte, pgoff_to_pte(pgoff));
 71	/*
 72	 * We don't need to run update_mmu_cache() here because the "file pte"
 73	 * being installed by install_file_pte() is not a real pte - it's a
 74	 * non-present entry (like a swap entry), noting what file offset should
 75	 * be mapped there when there's a fault (in a non-linear vma where
 76	 * that's not obvious).
 77	 */
 78	pte_unmap_unlock(pte, ptl);
 79	err = 0;
 80out:
 81	return err;
 82}
 83
 84static int populate_range(struct mm_struct *mm, struct vm_area_struct *vma,
 85			unsigned long addr, unsigned long size, pgoff_t pgoff)
 86{
 87	int err;
 88
 89	do {
 90		err = install_file_pte(mm, vma, addr, pgoff, vma->vm_page_prot);
 91		if (err)
 92			return err;
 93
 94		size -= PAGE_SIZE;
 95		addr += PAGE_SIZE;
 96		pgoff++;
 97	} while (size);
 98
 99        return 0;
100
101}
102
103/**
104 * sys_remap_file_pages - remap arbitrary pages of an existing VM_SHARED vma
105 * @start: start of the remapped virtual memory range
106 * @size: size of the remapped virtual memory range
107 * @prot: new protection bits of the range (see NOTE)
108 * @pgoff: to-be-mapped page of the backing store file
109 * @flags: 0 or MAP_NONBLOCKED - the later will cause no IO.
110 *
111 * sys_remap_file_pages remaps arbitrary pages of an existing VM_SHARED vma
112 * (shared backing store file).
113 *
114 * This syscall works purely via pagetables, so it's the most efficient
115 * way to map the same (large) file into a given virtual window. Unlike
116 * mmap()/mremap() it does not create any new vmas. The new mappings are
117 * also safe across swapout.
118 *
119 * NOTE: the @prot parameter right now is ignored (but must be zero),
120 * and the vma's default protection is used. Arbitrary protections
121 * might be implemented in the future.
122 */
123SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
124		unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
125{
126	struct mm_struct *mm = current->mm;
127	struct address_space *mapping;
128	struct vm_area_struct *vma;
129	int err = -EINVAL;
130	int has_write_lock = 0;
131
132	if (prot)
133		return err;
134	/*
135	 * Sanitize the syscall parameters:
136	 */
137	start = start & PAGE_MASK;
138	size = size & PAGE_MASK;
139
140	/* Does the address range wrap, or is the span zero-sized? */
141	if (start + size <= start)
142		return err;
143
144	/* Does pgoff wrap? */
145	if (pgoff + (size >> PAGE_SHIFT) < pgoff)
146		return err;
147
148	/* Can we represent this offset inside this architecture's pte's? */
149#if PTE_FILE_MAX_BITS < BITS_PER_LONG
150	if (pgoff + (size >> PAGE_SHIFT) >= (1UL << PTE_FILE_MAX_BITS))
151		return err;
152#endif
153
154	/* We need down_write() to change vma->vm_flags. */
155	down_read(&mm->mmap_sem);
156 retry:
157	vma = find_vma(mm, start);
158
159	/*
160	 * Make sure the vma is shared, that it supports prefaulting,
161	 * and that the remapped range is valid and fully within
162	 * the single existing vma.  vm_private_data is used as a
163	 * swapout cursor in a VM_NONLINEAR vma.
164	 */
165	if (!vma || !(vma->vm_flags & VM_SHARED))
166		goto out;
167
168	if (vma->vm_private_data && !(vma->vm_flags & VM_NONLINEAR))
169		goto out;
170
171	if (!(vma->vm_flags & VM_CAN_NONLINEAR))
172		goto out;
173
174	if (start < vma->vm_start || start + size > vma->vm_end)
175		goto out;
176
177	/* Must set VM_NONLINEAR before any pages are populated. */
178	if (!(vma->vm_flags & VM_NONLINEAR)) {
179		/* Don't need a nonlinear mapping, exit success */
180		if (pgoff == linear_page_index(vma, start)) {
181			err = 0;
182			goto out;
183		}
184
185		if (!has_write_lock) {
186			up_read(&mm->mmap_sem);
187			down_write(&mm->mmap_sem);
188			has_write_lock = 1;
189			goto retry;
190		}
191		mapping = vma->vm_file->f_mapping;
192		/*
193		 * page_mkclean doesn't work on nonlinear vmas, so if
194		 * dirty pages need to be accounted, emulate with linear
195		 * vmas.
196		 */
197		if (mapping_cap_account_dirty(mapping)) {
198			unsigned long addr;
199			struct file *file = vma->vm_file;
200
201			flags &= MAP_NONBLOCK;
202			get_file(file);
203			addr = mmap_region(file, start, size,
204					flags, vma->vm_flags, pgoff);
205			fput(file);
206			if (IS_ERR_VALUE(addr)) {
207				err = addr;
208			} else {
209				BUG_ON(addr != start);
210				err = 0;
211			}
212			goto out;
213		}
214		mutex_lock(&mapping->i_mmap_mutex);
215		flush_dcache_mmap_lock(mapping);
216		vma->vm_flags |= VM_NONLINEAR;
217		vma_prio_tree_remove(vma, &mapping->i_mmap);
218		vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
219		flush_dcache_mmap_unlock(mapping);
220		mutex_unlock(&mapping->i_mmap_mutex);
221	}
222
223	if (vma->vm_flags & VM_LOCKED) {
224		/*
225		 * drop PG_Mlocked flag for over-mapped range
226		 */
227		vm_flags_t saved_flags = vma->vm_flags;
228		munlock_vma_pages_range(vma, start, start + size);
229		vma->vm_flags = saved_flags;
230	}
231
232	mmu_notifier_invalidate_range_start(mm, start, start + size);
233	err = populate_range(mm, vma, start, size, pgoff);
234	mmu_notifier_invalidate_range_end(mm, start, start + size);
235	if (!err && !(flags & MAP_NONBLOCK)) {
236		if (vma->vm_flags & VM_LOCKED) {
237			/*
238			 * might be mapping previously unmapped range of file
239			 */
240			mlock_vma_pages_range(vma, start, start + size);
241		} else {
242			if (unlikely(has_write_lock)) {
243				downgrade_write(&mm->mmap_sem);
244				has_write_lock = 0;
245			}
246			make_pages_present(start, start+size);
247		}
248	}
249
250	/*
251	 * We can't clear VM_NONLINEAR because we'd have to do
252	 * it after ->populate completes, and that would prevent
253	 * downgrading the lock.  (Locks can't be upgraded).
254	 */
255
256out:
257	if (likely(!has_write_lock))
258		up_read(&mm->mmap_sem);
259	else
260		up_write(&mm->mmap_sem);
261
262	return err;
263}