1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _MM_PERCPU_INTERNAL_H
  3#define _MM_PERCPU_INTERNAL_H
  4
  5#include <linux/types.h>
  6#include <linux/percpu.h>
  7
  8/*
  9 * pcpu_block_md is the metadata block struct.
 10 * Each chunk's bitmap is split into a number of full blocks.
 11 * All units are in terms of bits.
 12 *
 13 * The scan hint is the largest known contiguous area before the contig hint.
 14 * It is not necessarily the actual largest contig hint though.  There is an
 15 * invariant that the scan_hint_start > contig_hint_start iff
 16 * scan_hint == contig_hint.  This is necessary because when scanning forward,
 17 * we don't know if a new contig hint would be better than the current one.
 18 */
 19struct pcpu_block_md {
 20	int			scan_hint;	/* scan hint for block */
 21	int			scan_hint_start; /* block relative starting
 22						    position of the scan hint */
 23	int                     contig_hint;    /* contig hint for block */
 24	int                     contig_hint_start; /* block relative starting
 25						      position of the contig hint */
 26	int                     left_free;      /* size of free space along
 27						   the left side of the block */
 28	int                     right_free;     /* size of free space along
 29						   the right side of the block */
 30	int                     first_free;     /* block position of first free */
 31	int			nr_bits;	/* total bits responsible for */
 32};
 33
 34struct pcpu_chunk {
 35#ifdef CONFIG_PERCPU_STATS
 36	int			nr_alloc;	/* # of allocations */
 37	size_t			max_alloc_size; /* largest allocation size */
 38#endif
 39
 40	struct list_head	list;		/* linked to pcpu_slot lists */
 41	int			free_bytes;	/* free bytes in the chunk */
 42	struct pcpu_block_md	chunk_md;
 43	void			*base_addr;	/* base address of this chunk */
 44
 45	unsigned long		*alloc_map;	/* allocation map */
 46	unsigned long		*bound_map;	/* boundary map */
 47	struct pcpu_block_md	*md_blocks;	/* metadata blocks */
 48
 49	void			*data;		/* chunk data */
 50	bool			immutable;	/* no [de]population allowed */
 51	bool			isolated;	/* isolated from active chunk
 52						   slots */
 53	int			start_offset;	/* the overlap with the previous
 54						   region to have a page aligned
 55						   base_addr */
 56	int			end_offset;	/* additional area required to
 57						   have the region end page
 58						   aligned */
 59#ifdef CONFIG_MEMCG_KMEM
 60	struct obj_cgroup	**obj_cgroups;	/* vector of object cgroups */
 61#endif
 62
 63	int			nr_pages;	/* # of pages served by this chunk */
 64	int			nr_populated;	/* # of populated pages */
 65	int                     nr_empty_pop_pages; /* # of empty populated pages */
 66	unsigned long		populated[];	/* populated bitmap */
 67};
 68
 69extern spinlock_t pcpu_lock;
 70
 71extern struct list_head *pcpu_chunk_lists;
 72extern int pcpu_nr_slots;
 73extern int pcpu_sidelined_slot;
 74extern int pcpu_to_depopulate_slot;
 75extern int pcpu_nr_empty_pop_pages;
 76
 77extern struct pcpu_chunk *pcpu_first_chunk;
 78extern struct pcpu_chunk *pcpu_reserved_chunk;
 79
 80/**
 81 * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks
 82 * @chunk: chunk of interest
 83 *
 84 * This conversion is from the number of physical pages that the chunk
 85 * serves to the number of bitmap blocks used.
 86 */
 87static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk)
 88{
 89	return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE;
 90}
 91
 92/**
 93 * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap
 94 * @pages: number of physical pages
 95 *
 96 * This conversion is from physical pages to the number of bits
 97 * required in the bitmap.
 98 */
 99static inline int pcpu_nr_pages_to_map_bits(int pages)
100{
101	return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
102}
103
104/**
105 * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap
106 * @chunk: chunk of interest
107 *
108 * This conversion is from the number of physical pages that the chunk
109 * serves to the number of bits in the bitmap.
110 */
111static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk)
112{
113	return pcpu_nr_pages_to_map_bits(chunk->nr_pages);
114}
115
116#ifdef CONFIG_PERCPU_STATS
117
118#include <linux/spinlock.h>
119
120struct percpu_stats {
121	u64 nr_alloc;		/* lifetime # of allocations */
122	u64 nr_dealloc;		/* lifetime # of deallocations */
123	u64 nr_cur_alloc;	/* current # of allocations */
124	u64 nr_max_alloc;	/* max # of live allocations */
125	u32 nr_chunks;		/* current # of live chunks */
126	u32 nr_max_chunks;	/* max # of live chunks */
127	size_t min_alloc_size;	/* min allocation size */
128	size_t max_alloc_size;	/* max allocation size */
129};
130
131extern struct percpu_stats pcpu_stats;
132extern struct pcpu_alloc_info pcpu_stats_ai;
133
134/*
135 * For debug purposes. We don't care about the flexible array.
136 */
137static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
138{
139	memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info));
140
141	/* initialize min_alloc_size to unit_size */
142	pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size;
143}
144
145/*
146 * pcpu_stats_area_alloc - increment area allocation stats
147 * @chunk: the location of the area being allocated
148 * @size: size of area to allocate in bytes
149 *
150 * CONTEXT:
151 * pcpu_lock.
152 */
153static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
154{
155	lockdep_assert_held(&pcpu_lock);
156
157	pcpu_stats.nr_alloc++;
158	pcpu_stats.nr_cur_alloc++;
159	pcpu_stats.nr_max_alloc =
160		max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc);
161	pcpu_stats.min_alloc_size =
162		min(pcpu_stats.min_alloc_size, size);
163	pcpu_stats.max_alloc_size =
164		max(pcpu_stats.max_alloc_size, size);
165
166	chunk->nr_alloc++;
167	chunk->max_alloc_size = max(chunk->max_alloc_size, size);
168}
169
170/*
171 * pcpu_stats_area_dealloc - decrement allocation stats
172 * @chunk: the location of the area being deallocated
173 *
174 * CONTEXT:
175 * pcpu_lock.
176 */
177static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
178{
179	lockdep_assert_held(&pcpu_lock);
180
181	pcpu_stats.nr_dealloc++;
182	pcpu_stats.nr_cur_alloc--;
183
184	chunk->nr_alloc--;
185}
186
187/*
188 * pcpu_stats_chunk_alloc - increment chunk stats
189 */
190static inline void pcpu_stats_chunk_alloc(void)
191{
192	unsigned long flags;
193	spin_lock_irqsave(&pcpu_lock, flags);
194
195	pcpu_stats.nr_chunks++;
196	pcpu_stats.nr_max_chunks =
197		max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks);
198
199	spin_unlock_irqrestore(&pcpu_lock, flags);
200}
201
202/*
203 * pcpu_stats_chunk_dealloc - decrement chunk stats
204 */
205static inline void pcpu_stats_chunk_dealloc(void)
206{
207	unsigned long flags;
208	spin_lock_irqsave(&pcpu_lock, flags);
209
210	pcpu_stats.nr_chunks--;
211
212	spin_unlock_irqrestore(&pcpu_lock, flags);
213}
214
215#else
216
217static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
218{
219}
220
221static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
222{
223}
224
225static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
226{
227}
228
229static inline void pcpu_stats_chunk_alloc(void)
230{
231}
232
233static inline void pcpu_stats_chunk_dealloc(void)
234{
235}
236
237#endif /* !CONFIG_PERCPU_STATS */
238
239#endif