1/*
  2 * Copyright (C) 2014 Davidlohr Bueso.
  3 */
  4#include <linux/sched.h>
  5#include <linux/mm.h>
  6#include <linux/vmacache.h>
  7
  8/*
  9 * Flush vma caches for threads that share a given mm.
 10 *
 11 * The operation is safe because the caller holds the mmap_sem
 12 * exclusively and other threads accessing the vma cache will
 13 * have mmap_sem held at least for read, so no extra locking
 14 * is required to maintain the vma cache.
 15 */
 16void vmacache_flush_all(struct mm_struct *mm)
 17{
 18	struct task_struct *g, *p;
 19
 20	rcu_read_lock();
 21	for_each_process_thread(g, p) {
 22		/*
 23		 * Only flush the vmacache pointers as the
 24		 * mm seqnum is already set and curr's will
 25		 * be set upon invalidation when the next
 26		 * lookup is done.
 27		 */
 28		if (mm == p->mm)
 29			vmacache_flush(p);
 30	}
 31	rcu_read_unlock();
 32}
 33
 34/*
 35 * This task may be accessing a foreign mm via (for example)
 36 * get_user_pages()->find_vma().  The vmacache is task-local and this
 37 * task's vmacache pertains to a different mm (ie, its own).  There is
 38 * nothing we can do here.
 39 *
 40 * Also handle the case where a kernel thread has adopted this mm via use_mm().
 41 * That kernel thread's vmacache is not applicable to this mm.
 42 */
 43static bool vmacache_valid_mm(struct mm_struct *mm)
 44{
 45	return current->mm == mm && !(current->flags & PF_KTHREAD);
 46}
 47
 48void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
 49{
 50	if (vmacache_valid_mm(newvma->vm_mm))
 51		current->vmacache[VMACACHE_HASH(addr)] = newvma;
 52}
 53
 54static bool vmacache_valid(struct mm_struct *mm)
 55{
 56	struct task_struct *curr;
 57
 58	if (!vmacache_valid_mm(mm))
 59		return false;
 60
 61	curr = current;
 62	if (mm->vmacache_seqnum != curr->vmacache_seqnum) {
 63		/*
 64		 * First attempt will always be invalid, initialize
 65		 * the new cache for this task here.
 66		 */
 67		curr->vmacache_seqnum = mm->vmacache_seqnum;
 68		vmacache_flush(curr);
 69		return false;
 70	}
 71	return true;
 72}
 73
 74struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
 75{
 76	int i;
 77
 78	if (!vmacache_valid(mm))
 79		return NULL;
 80
 81	for (i = 0; i < VMACACHE_SIZE; i++) {
 82		struct vm_area_struct *vma = current->vmacache[i];
 83
 84		if (!vma)
 85			continue;
 86		if (WARN_ON_ONCE(vma->vm_mm != mm))
 87			break;
 88		if (vma->vm_start <= addr && vma->vm_end > addr)
 89			return vma;
 90	}
 91
 92	return NULL;
 93}
 94
 95#ifndef CONFIG_MMU
 96struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
 97					   unsigned long start,
 98					   unsigned long end)
 99{
100	int i;
101
102	if (!vmacache_valid(mm))
103		return NULL;
104
105	for (i = 0; i < VMACACHE_SIZE; i++) {
106		struct vm_area_struct *vma = current->vmacache[i];
107
108		if (vma && vma->vm_start == start && vma->vm_end == end)
109			return vma;
110	}
111
112	return NULL;
113}
114#endif