1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) Gao Xiang <xiang@kernel.org>
  4 *
  5 * For low-latency decompression algorithms (e.g. lz4), reserve consecutive
  6 * per-CPU virtual memory (in pages) in advance to store such inplace I/O
  7 * data if inplace decompression is failed (due to unmet inplace margin for
  8 * example).
  9 */
 10#include "internal.h"
 11
 12struct erofs_pcpubuf {
 13	raw_spinlock_t lock;
 14	void *ptr;
 15	struct page **pages;
 16	unsigned int nrpages;
 17};
 18
 19static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
 20
 21void *erofs_get_pcpubuf(unsigned int requiredpages)
 22	__acquires(pcb->lock)
 23{
 24	struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
 25
 26	raw_spin_lock(&pcb->lock);
 27	/* check if the per-CPU buffer is too small */
 28	if (requiredpages > pcb->nrpages) {
 29		raw_spin_unlock(&pcb->lock);
 30		put_cpu_var(erofs_pcb);
 31		/* (for sparse checker) pretend pcb->lock is still taken */
 32		__acquire(pcb->lock);
 33		return NULL;
 34	}
 35	return pcb->ptr;
 36}
 37
 38void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
 39{
 40	struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
 41
 42	DBG_BUGON(pcb->ptr != ptr);
 43	raw_spin_unlock(&pcb->lock);
 44	put_cpu_var(erofs_pcb);
 45}
 46
 47/* the next step: support per-CPU page buffers hotplug */
 48int erofs_pcpubuf_growsize(unsigned int nrpages)
 49{
 50	static DEFINE_MUTEX(pcb_resize_mutex);
 51	static unsigned int pcb_nrpages;
 52	struct page *pagepool = NULL;
 53	int delta, cpu, ret, i;
 54
 55	mutex_lock(&pcb_resize_mutex);
 56	delta = nrpages - pcb_nrpages;
 57	ret = 0;
 58	/* avoid shrinking pcpubuf, since no idea how many fses rely on */
 59	if (delta <= 0)
 60		goto out;
 61
 62	for_each_possible_cpu(cpu) {
 63		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
 64		struct page **pages, **oldpages;
 65		void *ptr, *old_ptr;
 66
 67		pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
 68		if (!pages) {
 69			ret = -ENOMEM;
 70			break;
 71		}
 72
 73		for (i = 0; i < nrpages; ++i) {
 74			pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
 75			if (!pages[i]) {
 76				ret = -ENOMEM;
 77				oldpages = pages;
 78				goto free_pagearray;
 79			}
 80		}
 81		ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
 82		if (!ptr) {
 83			ret = -ENOMEM;
 84			oldpages = pages;
 85			goto free_pagearray;
 86		}
 87		raw_spin_lock(&pcb->lock);
 88		old_ptr = pcb->ptr;
 89		pcb->ptr = ptr;
 90		oldpages = pcb->pages;
 91		pcb->pages = pages;
 92		i = pcb->nrpages;
 93		pcb->nrpages = nrpages;
 94		raw_spin_unlock(&pcb->lock);
 95
 96		if (!oldpages) {
 97			DBG_BUGON(old_ptr);
 98			continue;
 99		}
100
101		if (old_ptr)
102			vunmap(old_ptr);
103free_pagearray:
104		while (i)
105			erofs_pagepool_add(&pagepool, oldpages[--i]);
106		kfree(oldpages);
107		if (ret)
108			break;
109	}
110	pcb_nrpages = nrpages;
111	erofs_release_pages(&pagepool);
112out:
113	mutex_unlock(&pcb_resize_mutex);
114	return ret;
115}
116
117void __init erofs_pcpubuf_init(void)
118{
119	int cpu;
120
121	for_each_possible_cpu(cpu) {
122		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
123
124		raw_spin_lock_init(&pcb->lock);
125	}
126}
127
128void erofs_pcpubuf_exit(void)
129{
130	int cpu, i;
131
132	for_each_possible_cpu(cpu) {
133		struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
134
135		if (pcb->ptr) {
136			vunmap(pcb->ptr);
137			pcb->ptr = NULL;
138		}
139		if (!pcb->pages)
140			continue;
141
142		for (i = 0; i < pcb->nrpages; ++i)
143			if (pcb->pages[i])
144				put_page(pcb->pages[i]);
145		kfree(pcb->pages);
146		pcb->pages = NULL;
147	}
148}