1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2014 Davidlohr Bueso.
  4 */
  5#include <linux/sched/signal.h>
  6#include <linux/sched/task.h>
  7#include <linux/mm.h>
  8#include <linux/vmacache.h>
  9
 10/*
 11 * Hash based on the pmd of addr if configured with MMU, which provides a good
 12 * hit rate for workloads with spatial locality.  Otherwise, use pages.
 13 */
 14#ifdef CONFIG_MMU
 15#define VMACACHE_SHIFT	PMD_SHIFT
 16#else
 17#define VMACACHE_SHIFT	PAGE_SHIFT
 18#endif
 19#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
 20
 21/*
 22 * This task may be accessing a foreign mm via (for example)
 23 * get_user_pages()->find_vma().  The vmacache is task-local and this
 24 * task's vmacache pertains to a different mm (ie, its own).  There is
 25 * nothing we can do here.
 26 *
 27 * Also handle the case where a kernel thread has adopted this mm via
 28 * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm.
 29 */
 30static inline bool vmacache_valid_mm(struct mm_struct *mm)
 31{
 32	return current->mm == mm && !(current->flags & PF_KTHREAD);
 33}
 34
 35void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
 36{
 37	if (vmacache_valid_mm(newvma->vm_mm))
 38		current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma;
 39}
 40
 41static bool vmacache_valid(struct mm_struct *mm)
 42{
 43	struct task_struct *curr;
 44
 45	if (!vmacache_valid_mm(mm))
 46		return false;
 47
 48	curr = current;
 49	if (mm->vmacache_seqnum != curr->vmacache.seqnum) {
 50		/*
 51		 * First attempt will always be invalid, initialize
 52		 * the new cache for this task here.
 53		 */
 54		curr->vmacache.seqnum = mm->vmacache_seqnum;
 55		vmacache_flush(curr);
 56		return false;
 57	}
 58	return true;
 59}
 60
 61struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
 62{
 63	int idx = VMACACHE_HASH(addr);
 64	int i;
 65
 66	count_vm_vmacache_event(VMACACHE_FIND_CALLS);
 67
 68	if (!vmacache_valid(mm))
 69		return NULL;
 70
 71	for (i = 0; i < VMACACHE_SIZE; i++) {
 72		struct vm_area_struct *vma = current->vmacache.vmas[idx];
 73
 74		if (vma) {
 75#ifdef CONFIG_DEBUG_VM_VMACACHE
 76			if (WARN_ON_ONCE(vma->vm_mm != mm))
 77				break;
 78#endif
 79			if (vma->vm_start <= addr && vma->vm_end > addr) {
 80				count_vm_vmacache_event(VMACACHE_FIND_HITS);
 81				return vma;
 82			}
 83		}
 84		if (++idx == VMACACHE_SIZE)
 85			idx = 0;
 86	}
 87
 88	return NULL;
 89}
 90
 91#ifndef CONFIG_MMU
 92struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
 93					   unsigned long start,
 94					   unsigned long end)
 95{
 96	int idx = VMACACHE_HASH(start);
 97	int i;
 98
 99	count_vm_vmacache_event(VMACACHE_FIND_CALLS);
100
101	if (!vmacache_valid(mm))
102		return NULL;
103
104	for (i = 0; i < VMACACHE_SIZE; i++) {
105		struct vm_area_struct *vma = current->vmacache.vmas[idx];
106
107		if (vma && vma->vm_start == start && vma->vm_end == end) {
108			count_vm_vmacache_event(VMACACHE_FIND_HITS);
109			return vma;
110		}
111		if (++idx == VMACACHE_SIZE)
112			idx = 0;
113	}
114
115	return NULL;
116}
117#endif