1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * mm/percpu-debug.c
  4 *
  5 * Copyright (C) 2017		Facebook Inc.
  6 * Copyright (C) 2017		Dennis Zhou <dennis@kernel.org>
  7 *
  8 * Prints statistics about the percpu allocator and backing chunks.
  9 */
 10#include <linux/debugfs.h>
 11#include <linux/list.h>
 12#include <linux/percpu.h>
 13#include <linux/seq_file.h>
 14#include <linux/sort.h>
 15#include <linux/vmalloc.h>
 16
 17#include "percpu-internal.h"
 18
 19#define P(X, Y) \
 20	seq_printf(m, "  %-20s: %12lld\n", X, (long long int)Y)
 21
 22struct percpu_stats pcpu_stats;
 23struct pcpu_alloc_info pcpu_stats_ai;
 24
 25static int cmpint(const void *a, const void *b)
 26{
 27	return *(int *)a - *(int *)b;
 28}
 29
 30/*
 31 * Iterates over all chunks to find the max nr_alloc entries.
 32 */
 33static int find_max_nr_alloc(void)
 34{
 35	struct pcpu_chunk *chunk;
 36	int slot, max_nr_alloc;
 37	enum pcpu_chunk_type type;
 38
 39	max_nr_alloc = 0;
 40	for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++)
 41		for (slot = 0; slot < pcpu_nr_slots; slot++)
 42			list_for_each_entry(chunk, &pcpu_chunk_list(type)[slot],
 43					    list)
 44				max_nr_alloc = max(max_nr_alloc,
 45						   chunk->nr_alloc);
 46
 47	return max_nr_alloc;
 48}
 49
 50/*
 51 * Prints out chunk state. Fragmentation is considered between
 52 * the beginning of the chunk to the last allocation.
 53 *
 54 * All statistics are in bytes unless stated otherwise.
 55 */
 56static void chunk_map_stats(struct seq_file *m, struct pcpu_chunk *chunk,
 57			    int *buffer)
 58{
 59	struct pcpu_block_md *chunk_md = &chunk->chunk_md;
 60	int i, last_alloc, as_len, start, end;
 61	int *alloc_sizes, *p;
 62	/* statistics */
 63	int sum_frag = 0, max_frag = 0;
 64	int cur_min_alloc = 0, cur_med_alloc = 0, cur_max_alloc = 0;
 65
 66	alloc_sizes = buffer;
 67
 68	/*
 69	 * find_last_bit returns the start value if nothing found.
 70	 * Therefore, we must determine if it is a failure of find_last_bit
 71	 * and set the appropriate value.
 72	 */
 73	last_alloc = find_last_bit(chunk->alloc_map,
 74				   pcpu_chunk_map_bits(chunk) -
 75				   chunk->end_offset / PCPU_MIN_ALLOC_SIZE - 1);
 76	last_alloc = test_bit(last_alloc, chunk->alloc_map) ?
 77		     last_alloc + 1 : 0;
 78
 79	as_len = 0;
 80	start = chunk->start_offset / PCPU_MIN_ALLOC_SIZE;
 81
 82	/*
 83	 * If a bit is set in the allocation map, the bound_map identifies
 84	 * where the allocation ends.  If the allocation is not set, the
 85	 * bound_map does not identify free areas as it is only kept accurate
 86	 * on allocation, not free.
 87	 *
 88	 * Positive values are allocations and negative values are free
 89	 * fragments.
 90	 */
 91	while (start < last_alloc) {
 92		if (test_bit(start, chunk->alloc_map)) {
 93			end = find_next_bit(chunk->bound_map, last_alloc,
 94					    start + 1);
 95			alloc_sizes[as_len] = 1;
 96		} else {
 97			end = find_next_bit(chunk->alloc_map, last_alloc,
 98					    start + 1);
 99			alloc_sizes[as_len] = -1;
100		}
101
102		alloc_sizes[as_len++] *= (end - start) * PCPU_MIN_ALLOC_SIZE;
103
104		start = end;
105	}
106
107	/*
108	 * The negative values are free fragments and thus sorting gives the
109	 * free fragments at the beginning in largest first order.
110	 */
111	if (as_len > 0) {
112		sort(alloc_sizes, as_len, sizeof(int), cmpint, NULL);
113
114		/* iterate through the unallocated fragments */
115		for (i = 0, p = alloc_sizes; *p < 0 && i < as_len; i++, p++) {
116			sum_frag -= *p;
117			max_frag = max(max_frag, -1 * (*p));
118		}
119
120		cur_min_alloc = alloc_sizes[i];
121		cur_med_alloc = alloc_sizes[(i + as_len - 1) / 2];
122		cur_max_alloc = alloc_sizes[as_len - 1];
123	}
124
125	P("nr_alloc", chunk->nr_alloc);
126	P("max_alloc_size", chunk->max_alloc_size);
127	P("empty_pop_pages", chunk->nr_empty_pop_pages);
128	P("first_bit", chunk_md->first_free);
129	P("free_bytes", chunk->free_bytes);
130	P("contig_bytes", chunk_md->contig_hint * PCPU_MIN_ALLOC_SIZE);
131	P("sum_frag", sum_frag);
132	P("max_frag", max_frag);
133	P("cur_min_alloc", cur_min_alloc);
134	P("cur_med_alloc", cur_med_alloc);
135	P("cur_max_alloc", cur_max_alloc);
136#ifdef CONFIG_MEMCG_KMEM
137	P("memcg_aware", pcpu_is_memcg_chunk(pcpu_chunk_type(chunk)));
138#endif
139	seq_putc(m, '\n');
140}
141
142static int percpu_stats_show(struct seq_file *m, void *v)
143{
144	struct pcpu_chunk *chunk;
145	int slot, max_nr_alloc;
146	int *buffer;
147	enum pcpu_chunk_type type;
148
149alloc_buffer:
150	spin_lock_irq(&pcpu_lock);
151	max_nr_alloc = find_max_nr_alloc();
152	spin_unlock_irq(&pcpu_lock);
153
154	/* there can be at most this many free and allocated fragments */
155	buffer = vmalloc(array_size(sizeof(int), (2 * max_nr_alloc + 1)));
156	if (!buffer)
157		return -ENOMEM;
158
159	spin_lock_irq(&pcpu_lock);
160
161	/* if the buffer allocated earlier is too small */
162	if (max_nr_alloc < find_max_nr_alloc()) {
163		spin_unlock_irq(&pcpu_lock);
164		vfree(buffer);
165		goto alloc_buffer;
166	}
167
168#define PL(X) \
169	seq_printf(m, "  %-20s: %12lld\n", #X, (long long int)pcpu_stats_ai.X)
170
171	seq_printf(m,
172			"Percpu Memory Statistics\n"
173			"Allocation Info:\n"
174			"----------------------------------------\n");
175	PL(unit_size);
176	PL(static_size);
177	PL(reserved_size);
178	PL(dyn_size);
179	PL(atom_size);
180	PL(alloc_size);
181	seq_putc(m, '\n');
182
183#undef PL
184
185#define PU(X) \
186	seq_printf(m, "  %-20s: %12llu\n", #X, (unsigned long long)pcpu_stats.X)
187
188	seq_printf(m,
189			"Global Stats:\n"
190			"----------------------------------------\n");
191	PU(nr_alloc);
192	PU(nr_dealloc);
193	PU(nr_cur_alloc);
194	PU(nr_max_alloc);
195	PU(nr_chunks);
196	PU(nr_max_chunks);
197	PU(min_alloc_size);
198	PU(max_alloc_size);
199	P("empty_pop_pages", pcpu_nr_empty_pop_pages);
200	seq_putc(m, '\n');
201
202#undef PU
203
204	seq_printf(m,
205			"Per Chunk Stats:\n"
206			"----------------------------------------\n");
207
208	if (pcpu_reserved_chunk) {
209		seq_puts(m, "Chunk: <- Reserved Chunk\n");
210		chunk_map_stats(m, pcpu_reserved_chunk, buffer);
211	}
212
213	for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++) {
214		for (slot = 0; slot < pcpu_nr_slots; slot++) {
215			list_for_each_entry(chunk, &pcpu_chunk_list(type)[slot],
216					    list) {
217				if (chunk == pcpu_first_chunk) {
218					seq_puts(m, "Chunk: <- First Chunk\n");
219					chunk_map_stats(m, chunk, buffer);
220				} else {
221					seq_puts(m, "Chunk:\n");
222					chunk_map_stats(m, chunk, buffer);
223				}
224			}
225		}
226	}
227
228	spin_unlock_irq(&pcpu_lock);
229
230	vfree(buffer);
231
232	return 0;
233}
234DEFINE_SHOW_ATTRIBUTE(percpu_stats);
235
236static int __init init_percpu_stats_debugfs(void)
237{
238	debugfs_create_file("percpu_stats", 0444, NULL, NULL,
239			&percpu_stats_fops);
240
241	return 0;
242}
243
244late_initcall(init_percpu_stats_debugfs);