1// SPDX-License-Identifier: GPL-2.0-only
  2/* Page fragment allocator
  3 *
  4 * Page Fragment:
  5 *  An arbitrary-length arbitrary-offset area of memory which resides within a
  6 *  0 or higher order page.  Multiple fragments within that page are
  7 *  individually refcounted, in the page's reference counter.
  8 *
  9 * The page_frag functions provide a simple allocation framework for page
 10 * fragments.  This is used by the network stack and network device drivers to
 11 * provide a backing region of memory for use as either an sk_buff->head, or to
 12 * be used in the "frags" portion of skb_shared_info.
 13 */
 14
 15#include <linux/build_bug.h>
 16#include <linux/export.h>
 17#include <linux/gfp_types.h>
 18#include <linux/init.h>
 19#include <linux/mm.h>
 20#include <linux/page_frag_cache.h>
 21#include "internal.h"
 22
 23static unsigned long encoded_page_create(struct page *page, unsigned int order,
 24					 bool pfmemalloc)
 25{
 26	BUILD_BUG_ON(PAGE_FRAG_CACHE_MAX_ORDER > PAGE_FRAG_CACHE_ORDER_MASK);
 27	BUILD_BUG_ON(PAGE_FRAG_CACHE_PFMEMALLOC_BIT >= PAGE_SIZE);
 28
 29	return (unsigned long)page_address(page) |
 30		(order & PAGE_FRAG_CACHE_ORDER_MASK) |
 31		((unsigned long)pfmemalloc * PAGE_FRAG_CACHE_PFMEMALLOC_BIT);
 32}
 33
 34static unsigned long encoded_page_decode_order(unsigned long encoded_page)
 35{
 36	return encoded_page & PAGE_FRAG_CACHE_ORDER_MASK;
 37}
 38
 39static void *encoded_page_decode_virt(unsigned long encoded_page)
 40{
 41	return (void *)(encoded_page & PAGE_MASK);
 42}
 43
 44static struct page *encoded_page_decode_page(unsigned long encoded_page)
 45{
 46	return virt_to_page((void *)encoded_page);
 47}
 48
 49static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
 50					     gfp_t gfp_mask)
 51{
 52	unsigned long order = PAGE_FRAG_CACHE_MAX_ORDER;
 53	struct page *page = NULL;
 54	gfp_t gfp = gfp_mask;
 55
 56#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
 57	gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) |  __GFP_COMP |
 58		   __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC;
 59	page = __alloc_pages(gfp_mask, PAGE_FRAG_CACHE_MAX_ORDER,
 60			     numa_mem_id(), NULL);
 61#endif
 62	if (unlikely(!page)) {
 63		page = __alloc_pages(gfp, 0, numa_mem_id(), NULL);
 64		order = 0;
 65	}
 66
 67	nc->encoded_page = page ?
 68		encoded_page_create(page, order, page_is_pfmemalloc(page)) : 0;
 69
 70	return page;
 71}
 72
 73void page_frag_cache_drain(struct page_frag_cache *nc)
 74{
 75	if (!nc->encoded_page)
 76		return;
 77
 78	__page_frag_cache_drain(encoded_page_decode_page(nc->encoded_page),
 79				nc->pagecnt_bias);
 80	nc->encoded_page = 0;
 81}
 82EXPORT_SYMBOL(page_frag_cache_drain);
 83
 84void __page_frag_cache_drain(struct page *page, unsigned int count)
 85{
 86	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
 87
 88	if (page_ref_sub_and_test(page, count))
 89		free_unref_page(page, compound_order(page));
 90}
 91EXPORT_SYMBOL(__page_frag_cache_drain);
 92
 93void *__page_frag_alloc_align(struct page_frag_cache *nc,
 94			      unsigned int fragsz, gfp_t gfp_mask,
 95			      unsigned int align_mask)
 96{
 97	unsigned long encoded_page = nc->encoded_page;
 98	unsigned int size, offset;
 99	struct page *page;
100
101	if (unlikely(!encoded_page)) {
102refill:
103		page = __page_frag_cache_refill(nc, gfp_mask);
104		if (!page)
105			return NULL;
106
107		encoded_page = nc->encoded_page;
108
109		/* Even if we own the page, we do not use atomic_set().
110		 * This would break get_page_unless_zero() users.
111		 */
112		page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
113
114		/* reset page count bias and offset to start of new frag */
115		nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
116		nc->offset = 0;
117	}
118
119	size = PAGE_SIZE << encoded_page_decode_order(encoded_page);
120	offset = __ALIGN_KERNEL_MASK(nc->offset, ~align_mask);
121	if (unlikely(offset + fragsz > size)) {
122		if (unlikely(fragsz > PAGE_SIZE)) {
123			/*
124			 * The caller is trying to allocate a fragment
125			 * with fragsz > PAGE_SIZE but the cache isn't big
126			 * enough to satisfy the request, this may
127			 * happen in low memory conditions.
128			 * We don't release the cache page because
129			 * it could make memory pressure worse
130			 * so we simply return NULL here.
131			 */
132			return NULL;
133		}
134
135		page = encoded_page_decode_page(encoded_page);
136
137		if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
138			goto refill;
139
140		if (unlikely(encoded_page_decode_pfmemalloc(encoded_page))) {
141			free_unref_page(page,
142					encoded_page_decode_order(encoded_page));
143			goto refill;
144		}
145
146		/* OK, page count is 0, we can safely set it */
147		set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
148
149		/* reset page count bias and offset to start of new frag */
150		nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
151		offset = 0;
152	}
153
154	nc->pagecnt_bias--;
155	nc->offset = offset + fragsz;
156
157	return encoded_page_decode_virt(encoded_page) + offset;
158}
159EXPORT_SYMBOL(__page_frag_alloc_align);
160
161/*
162 * Frees a page fragment allocated out of either a compound or order 0 page.
163 */
164void page_frag_free(void *addr)
165{
166	struct page *page = virt_to_head_page(addr);
167
168	if (unlikely(put_page_testzero(page)))
169		free_unref_page(page, compound_order(page));
170}
171EXPORT_SYMBOL(page_frag_free);