Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef PAGE_FLAGS_LAYOUT_H
3#define PAGE_FLAGS_LAYOUT_H
4
5#include <linux/numa.h>
6#include <generated/bounds.h>
7
8/*
9 * When a memory allocation must conform to specific limitations (such
10 * as being suitable for DMA) the caller will pass in hints to the
11 * allocator in the gfp_mask, in the zone modifier bits. These bits
12 * are used to select a priority ordered list of memory zones which
13 * match the requested limits. See gfp_zone() in include/linux/gfp.h
14 */
15#if MAX_NR_ZONES < 2
16#define ZONES_SHIFT 0
17#elif MAX_NR_ZONES <= 2
18#define ZONES_SHIFT 1
19#elif MAX_NR_ZONES <= 4
20#define ZONES_SHIFT 2
21#elif MAX_NR_ZONES <= 8
22#define ZONES_SHIFT 3
23#else
24#error ZONES_SHIFT "Too many zones configured"
25#endif
26
27#define ZONES_WIDTH ZONES_SHIFT
28
29#ifdef CONFIG_SPARSEMEM
30#include <asm/sparsemem.h>
31#define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS)
32#else
33#define SECTIONS_SHIFT 0
34#endif
35
36#ifndef BUILD_VDSO32_64
37/*
38 * page->flags layout:
39 *
40 * There are five possibilities for how page->flags get laid out. The first
41 * pair is for the normal case without sparsemem. The second pair is for
42 * sparsemem when there is plenty of space for node and section information.
43 * The last is when there is insufficient space in page->flags and a separate
44 * lookup is necessary.
45 *
46 * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS |
47 * " plus space for last_cpupid: | NODE | ZONE | LAST_CPUPID ... | FLAGS |
48 * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS |
49 * " plus space for last_cpupid: | SECTION | NODE | ZONE | LAST_CPUPID ... | FLAGS |
50 * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS |
51 */
52#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
53#define SECTIONS_WIDTH SECTIONS_SHIFT
54#else
55#define SECTIONS_WIDTH 0
56#endif
57
58#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
59#define NODES_WIDTH NODES_SHIFT
60#elif defined(CONFIG_SPARSEMEM_VMEMMAP)
61#error "Vmemmap: No space for nodes field in page flags"
62#else
63#define NODES_WIDTH 0
64#endif
65
66/*
67 * Note that this #define MUST have a value so that it can be tested with
68 * the IS_ENABLED() macro.
69 */
70#if NODES_SHIFT != 0 && NODES_WIDTH == 0
71#define NODE_NOT_IN_PAGE_FLAGS 1
72#endif
73
74#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
75#define KASAN_TAG_WIDTH 8
76#else
77#define KASAN_TAG_WIDTH 0
78#endif
79
80#ifdef CONFIG_NUMA_BALANCING
81#define LAST__PID_SHIFT 8
82#define LAST__PID_MASK ((1 << LAST__PID_SHIFT)-1)
83
84#define LAST__CPU_SHIFT NR_CPUS_BITS
85#define LAST__CPU_MASK ((1 << LAST__CPU_SHIFT)-1)
86
87#define LAST_CPUPID_SHIFT (LAST__PID_SHIFT+LAST__CPU_SHIFT)
88#else
89#define LAST_CPUPID_SHIFT 0
90#endif
91
92#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \
93 <= BITS_PER_LONG - NR_PAGEFLAGS
94#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT
95#else
96#define LAST_CPUPID_WIDTH 0
97#endif
98
99#if LAST_CPUPID_SHIFT != 0 && LAST_CPUPID_WIDTH == 0
100#define LAST_CPUPID_NOT_IN_PAGE_FLAGS
101#endif
102
103#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \
104 > BITS_PER_LONG - NR_PAGEFLAGS
105#error "Not enough bits in page flags"
106#endif
107
108#endif
109#endif /* _LINUX_PAGE_FLAGS_LAYOUT */
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef PAGE_FLAGS_LAYOUT_H
3#define PAGE_FLAGS_LAYOUT_H
4
5#include <linux/numa.h>
6#include <generated/bounds.h>
7
8/*
9 * When a memory allocation must conform to specific limitations (such
10 * as being suitable for DMA) the caller will pass in hints to the
11 * allocator in the gfp_mask, in the zone modifier bits. These bits
12 * are used to select a priority ordered list of memory zones which
13 * match the requested limits. See gfp_zone() in include/linux/gfp.h
14 */
15#if MAX_NR_ZONES < 2
16#define ZONES_SHIFT 0
17#elif MAX_NR_ZONES <= 2
18#define ZONES_SHIFT 1
19#elif MAX_NR_ZONES <= 4
20#define ZONES_SHIFT 2
21#elif MAX_NR_ZONES <= 8
22#define ZONES_SHIFT 3
23#else
24#error ZONES_SHIFT "Too many zones configured"
25#endif
26
27#define ZONES_WIDTH ZONES_SHIFT
28
29#ifdef CONFIG_SPARSEMEM
30#include <asm/sparsemem.h>
31#define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS)
32#else
33#define SECTIONS_SHIFT 0
34#endif
35
36#ifndef BUILD_VDSO32_64
37/*
38 * page->flags layout:
39 *
40 * There are five possibilities for how page->flags get laid out. The first
41 * pair is for the normal case without sparsemem. The second pair is for
42 * sparsemem when there is plenty of space for node and section information.
43 * The last is when there is insufficient space in page->flags and a separate
44 * lookup is necessary.
45 *
46 * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS |
47 * " plus space for last_cpupid: | NODE | ZONE | LAST_CPUPID ... | FLAGS |
48 * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS |
49 * " plus space for last_cpupid: | SECTION | NODE | ZONE | LAST_CPUPID ... | FLAGS |
50 * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS |
51 */
52#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
53#define SECTIONS_WIDTH SECTIONS_SHIFT
54#else
55#define SECTIONS_WIDTH 0
56#endif
57
58#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \
59 <= BITS_PER_LONG - NR_PAGEFLAGS
60#define NODES_WIDTH NODES_SHIFT
61#elif defined(CONFIG_SPARSEMEM_VMEMMAP)
62#error "Vmemmap: No space for nodes field in page flags"
63#else
64#define NODES_WIDTH 0
65#endif
66
67/*
68 * Note that this #define MUST have a value so that it can be tested with
69 * the IS_ENABLED() macro.
70 */
71#if NODES_SHIFT != 0 && NODES_WIDTH == 0
72#define NODE_NOT_IN_PAGE_FLAGS 1
73#endif
74
75#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
76#define KASAN_TAG_WIDTH 8
77#else
78#define KASAN_TAG_WIDTH 0
79#endif
80
81#ifdef CONFIG_NUMA_BALANCING
82#define LAST__PID_SHIFT 8
83#define LAST__PID_MASK ((1 << LAST__PID_SHIFT)-1)
84
85#define LAST__CPU_SHIFT NR_CPUS_BITS
86#define LAST__CPU_MASK ((1 << LAST__CPU_SHIFT)-1)
87
88#define LAST_CPUPID_SHIFT (LAST__PID_SHIFT+LAST__CPU_SHIFT)
89#else
90#define LAST_CPUPID_SHIFT 0
91#endif
92
93#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
94 KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
95#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT
96#else
97#define LAST_CPUPID_WIDTH 0
98#endif
99
100#if LAST_CPUPID_SHIFT != 0 && LAST_CPUPID_WIDTH == 0
101#define LAST_CPUPID_NOT_IN_PAGE_FLAGS
102#endif
103
104#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
105 KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
106#error "Not enough bits in page flags"
107#endif
108
109/* see the comment on MAX_NR_TIERS */
110#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \
111 ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \
112 NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH)
113
114#define NR_NON_PAGEFLAG_BITS (SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH + \
115 LAST_CPUPID_SHIFT + KASAN_TAG_WIDTH + \
116 LRU_GEN_WIDTH + LRU_REFS_WIDTH)
117
118#define NR_UNUSED_PAGEFLAG_BITS (BITS_PER_LONG - \
119 (NR_NON_PAGEFLAG_BITS + NR_PAGEFLAGS))
120
121#endif
122#endif /* _LINUX_PAGE_FLAGS_LAYOUT */