1/*
   2 * mm/kmemleak.c
   3 *
   4 * Copyright (C) 2008 ARM Limited
   5 * Written by Catalin Marinas <catalin.marinas@arm.com>
   6 *
   7 * This program is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License version 2 as
   9 * published by the Free Software Foundation.
  10 *
  11 * This program is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with this program; if not, write to the Free Software
  18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19 *
  20 *
  21 * For more information on the algorithm and kmemleak usage, please see
  22 * Documentation/kmemleak.txt.
  23 *
  24 * Notes on locking
  25 * ----------------
  26 *
  27 * The following locks and mutexes are used by kmemleak:
  28 *
  29 * - kmemleak_lock (rwlock): protects the object_list modifications and
  30 *   accesses to the object_tree_root. The object_list is the main list
  31 *   holding the metadata (struct kmemleak_object) for the allocated memory
  32 *   blocks. The object_tree_root is a red black tree used to look-up
  33 *   metadata based on a pointer to the corresponding memory block.  The
  34 *   kmemleak_object structures are added to the object_list and
  35 *   object_tree_root in the create_object() function called from the
  36 *   kmemleak_alloc() callback and removed in delete_object() called from the
  37 *   kmemleak_free() callback
  38 * - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to
  39 *   the metadata (e.g. count) are protected by this lock. Note that some
  40 *   members of this structure may be protected by other means (atomic or
  41 *   kmemleak_lock). This lock is also held when scanning the corresponding
  42 *   memory block to avoid the kernel freeing it via the kmemleak_free()
  43 *   callback. This is less heavyweight than holding a global lock like
  44 *   kmemleak_lock during scanning
  45 * - scan_mutex (mutex): ensures that only one thread may scan the memory for
  46 *   unreferenced objects at a time. The gray_list contains the objects which
  47 *   are already referenced or marked as false positives and need to be
  48 *   scanned. This list is only modified during a scanning episode when the
  49 *   scan_mutex is held. At the end of a scan, the gray_list is always empty.
  50 *   Note that the kmemleak_object.use_count is incremented when an object is
  51 *   added to the gray_list and therefore cannot be freed. This mutex also
  52 *   prevents multiple users of the "kmemleak" debugfs file together with
  53 *   modifications to the memory scanning parameters including the scan_thread
  54 *   pointer
  55 *
 
 
 
 
 
 
 
  56 * The kmemleak_object structures have a use_count incremented or decremented
  57 * using the get_object()/put_object() functions. When the use_count becomes
  58 * 0, this count can no longer be incremented and put_object() schedules the
  59 * kmemleak_object freeing via an RCU callback. All calls to the get_object()
  60 * function must be protected by rcu_read_lock() to avoid accessing a freed
  61 * structure.
  62 */
  63
  64#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  65
  66#include <linux/init.h>
  67#include <linux/kernel.h>
  68#include <linux/list.h>
  69#include <linux/sched.h>
  70#include <linux/jiffies.h>
  71#include <linux/delay.h>
  72#include <linux/export.h>
  73#include <linux/kthread.h>
  74#include <linux/rbtree.h>
  75#include <linux/fs.h>
  76#include <linux/debugfs.h>
  77#include <linux/seq_file.h>
  78#include <linux/cpumask.h>
  79#include <linux/spinlock.h>
  80#include <linux/mutex.h>
  81#include <linux/rcupdate.h>
  82#include <linux/stacktrace.h>
  83#include <linux/cache.h>
  84#include <linux/percpu.h>
  85#include <linux/hardirq.h>
  86#include <linux/mmzone.h>
  87#include <linux/slab.h>
  88#include <linux/thread_info.h>
  89#include <linux/err.h>
  90#include <linux/uaccess.h>
  91#include <linux/string.h>
  92#include <linux/nodemask.h>
  93#include <linux/mm.h>
  94#include <linux/workqueue.h>
  95#include <linux/crc32.h>
  96
  97#include <asm/sections.h>
  98#include <asm/processor.h>
  99#include <linux/atomic.h>
 100
 
 101#include <linux/kmemcheck.h>
 102#include <linux/kmemleak.h>
 103#include <linux/memory_hotplug.h>
 104
 105/*
 106 * Kmemleak configuration and common defines.
 107 */
 108#define MAX_TRACE		16	/* stack trace length */
 109#define MSECS_MIN_AGE		5000	/* minimum object age for reporting */
 110#define SECS_FIRST_SCAN		60	/* delay before the first scan */
 111#define SECS_SCAN_WAIT		600	/* subsequent auto scanning delay */
 112#define MAX_SCAN_SIZE		4096	/* maximum size of a scanned block */
 113
 114#define BYTES_PER_POINTER	sizeof(void *)
 115
 116/* GFP bitmask for kmemleak internal allocations */
 117#define gfp_kmemleak_mask(gfp)	(((gfp) & (GFP_KERNEL | GFP_ATOMIC)) | \
 118				 __GFP_NORETRY | __GFP_NOMEMALLOC | \
 119				 __GFP_NOWARN)
 120
 121/* scanning area inside a memory block */
 122struct kmemleak_scan_area {
 123	struct hlist_node node;
 124	unsigned long start;
 125	size_t size;
 126};
 127
 128#define KMEMLEAK_GREY	0
 129#define KMEMLEAK_BLACK	-1
 130
 131/*
 132 * Structure holding the metadata for each allocated memory block.
 133 * Modifications to such objects should be made while holding the
 134 * object->lock. Insertions or deletions from object_list, gray_list or
 135 * rb_node are already protected by the corresponding locks or mutex (see
 136 * the notes on locking above). These objects are reference-counted
 137 * (use_count) and freed using the RCU mechanism.
 138 */
 139struct kmemleak_object {
 140	spinlock_t lock;
 141	unsigned long flags;		/* object status flags */
 142	struct list_head object_list;
 143	struct list_head gray_list;
 144	struct rb_node rb_node;
 145	struct rcu_head rcu;		/* object_list lockless traversal */
 146	/* object usage count; object freed when use_count == 0 */
 147	atomic_t use_count;
 148	unsigned long pointer;
 149	size_t size;
 150	/* minimum number of a pointers found before it is considered leak */
 151	int min_count;
 152	/* the total number of pointers found pointing to this object */
 153	int count;
 154	/* checksum for detecting modified objects */
 155	u32 checksum;
 156	/* memory ranges to be scanned inside an object (empty for all) */
 157	struct hlist_head area_list;
 158	unsigned long trace[MAX_TRACE];
 159	unsigned int trace_len;
 160	unsigned long jiffies;		/* creation timestamp */
 161	pid_t pid;			/* pid of the current task */
 162	char comm[TASK_COMM_LEN];	/* executable name */
 163};
 164
 165/* flag representing the memory block allocation status */
 166#define OBJECT_ALLOCATED	(1 << 0)
 167/* flag set after the first reporting of an unreference object */
 168#define OBJECT_REPORTED		(1 << 1)
 169/* flag set to not scan the object */
 170#define OBJECT_NO_SCAN		(1 << 2)
 171
 172/* number of bytes to print per line; must be 16 or 32 */
 173#define HEX_ROW_SIZE		16
 174/* number of bytes to print at a time (1, 2, 4, 8) */
 175#define HEX_GROUP_SIZE		1
 176/* include ASCII after the hex output */
 177#define HEX_ASCII		1
 178/* max number of lines to be printed */
 179#define HEX_MAX_LINES		2
 180
 181/* the list of all allocated objects */
 182static LIST_HEAD(object_list);
 183/* the list of gray-colored objects (see color_gray comment below) */
 184static LIST_HEAD(gray_list);
 185/* search tree for object boundaries */
 186static struct rb_root object_tree_root = RB_ROOT;
 187/* rw_lock protecting the access to object_list and object_tree_root */
 188static DEFINE_RWLOCK(kmemleak_lock);
 189
 190/* allocation caches for kmemleak internal data */
 191static struct kmem_cache *object_cache;
 192static struct kmem_cache *scan_area_cache;
 193
 194/* set if tracing memory operations is enabled */
 195static int kmemleak_enabled;
 
 
 196/* set in the late_initcall if there were no errors */
 197static int kmemleak_initialized;
 198/* enables or disables early logging of the memory operations */
 199static int kmemleak_early_log = 1;
 200/* set if a kmemleak warning was issued */
 201static int kmemleak_warning;
 202/* set if a fatal kmemleak error has occurred */
 203static int kmemleak_error;
 204
 205/* minimum and maximum address that may be valid pointers */
 206static unsigned long min_addr = ULONG_MAX;
 207static unsigned long max_addr;
 208
 209static struct task_struct *scan_thread;
 210/* used to avoid reporting of recently allocated objects */
 211static unsigned long jiffies_min_age;
 212static unsigned long jiffies_last_scan;
 213/* delay between automatic memory scannings */
 214static signed long jiffies_scan_wait;
 215/* enables or disables the task stacks scanning */
 216static int kmemleak_stack_scan = 1;
 217/* protects the memory scanning, parameters and debug/kmemleak file access */
 218static DEFINE_MUTEX(scan_mutex);
 219/* setting kmemleak=on, will set this var, skipping the disable */
 220static int kmemleak_skip_disable;
 221/* If there are leaks that can be reported */
 222static bool kmemleak_found_leaks;
 223
 224/*
 225 * Early object allocation/freeing logging. Kmemleak is initialized after the
 226 * kernel allocator. However, both the kernel allocator and kmemleak may
 227 * allocate memory blocks which need to be tracked. Kmemleak defines an
 228 * arbitrary buffer to hold the allocation/freeing information before it is
 229 * fully initialized.
 230 */
 231
 232/* kmemleak operation type for early logging */
 233enum {
 234	KMEMLEAK_ALLOC,
 235	KMEMLEAK_ALLOC_PERCPU,
 236	KMEMLEAK_FREE,
 237	KMEMLEAK_FREE_PART,
 238	KMEMLEAK_FREE_PERCPU,
 239	KMEMLEAK_NOT_LEAK,
 240	KMEMLEAK_IGNORE,
 241	KMEMLEAK_SCAN_AREA,
 242	KMEMLEAK_NO_SCAN
 243};
 244
 245/*
 246 * Structure holding the information passed to kmemleak callbacks during the
 247 * early logging.
 248 */
 249struct early_log {
 250	int op_type;			/* kmemleak operation type */
 251	const void *ptr;		/* allocated/freed memory block */
 252	size_t size;			/* memory block size */
 253	int min_count;			/* minimum reference count */
 254	unsigned long trace[MAX_TRACE];	/* stack trace */
 255	unsigned int trace_len;		/* stack trace length */
 256};
 257
 258/* early logging buffer and current position */
 259static struct early_log
 260	early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
 261static int crt_early_log __initdata;
 262
 263static void kmemleak_disable(void);
 264
 265/*
 266 * Print a warning and dump the stack trace.
 267 */
 268#define kmemleak_warn(x...)	do {		\
 269	pr_warning(x);				\
 270	dump_stack();				\
 271	kmemleak_warning = 1;			\
 272} while (0)
 273
 274/*
 275 * Macro invoked when a serious kmemleak condition occurred and cannot be
 276 * recovered from. Kmemleak will be disabled and further allocation/freeing
 277 * tracing no longer available.
 278 */
 279#define kmemleak_stop(x...)	do {	\
 280	kmemleak_warn(x);		\
 281	kmemleak_disable();		\
 282} while (0)
 283
 284/*
 285 * Printing of the objects hex dump to the seq file. The number of lines to be
 286 * printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
 287 * actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
 288 * with the object->lock held.
 289 */
 290static void hex_dump_object(struct seq_file *seq,
 291			    struct kmemleak_object *object)
 292{
 293	const u8 *ptr = (const u8 *)object->pointer;
 294	int i, len, remaining;
 295	unsigned char linebuf[HEX_ROW_SIZE * 5];
 296
 297	/* limit the number of lines to HEX_MAX_LINES */
 298	remaining = len =
 299		min(object->size, (size_t)(HEX_MAX_LINES * HEX_ROW_SIZE));
 300
 301	seq_printf(seq, "  hex dump (first %d bytes):\n", len);
 302	for (i = 0; i < len; i += HEX_ROW_SIZE) {
 303		int linelen = min(remaining, HEX_ROW_SIZE);
 304
 305		remaining -= HEX_ROW_SIZE;
 306		hex_dump_to_buffer(ptr + i, linelen, HEX_ROW_SIZE,
 307				   HEX_GROUP_SIZE, linebuf, sizeof(linebuf),
 308				   HEX_ASCII);
 309		seq_printf(seq, "    %s\n", linebuf);
 310	}
 311}
 312
 313/*
 314 * Object colors, encoded with count and min_count:
 315 * - white - orphan object, not enough references to it (count < min_count)
 316 * - gray  - not orphan, not marked as false positive (min_count == 0) or
 317 *		sufficient references to it (count >= min_count)
 318 * - black - ignore, it doesn't contain references (e.g. text section)
 319 *		(min_count == -1). No function defined for this color.
 320 * Newly created objects don't have any color assigned (object->count == -1)
 321 * before the next memory scan when they become white.
 322 */
 323static bool color_white(const struct kmemleak_object *object)
 324{
 325	return object->count != KMEMLEAK_BLACK &&
 326		object->count < object->min_count;
 327}
 328
 329static bool color_gray(const struct kmemleak_object *object)
 330{
 331	return object->min_count != KMEMLEAK_BLACK &&
 332		object->count >= object->min_count;
 333}
 334
 335/*
 336 * Objects are considered unreferenced only if their color is white, they have
 337 * not be deleted and have a minimum age to avoid false positives caused by
 338 * pointers temporarily stored in CPU registers.
 339 */
 340static bool unreferenced_object(struct kmemleak_object *object)
 341{
 342	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
 343		time_before_eq(object->jiffies + jiffies_min_age,
 344			       jiffies_last_scan);
 345}
 346
 347/*
 348 * Printing of the unreferenced objects information to the seq file. The
 349 * print_unreferenced function must be called with the object->lock held.
 350 */
 351static void print_unreferenced(struct seq_file *seq,
 352			       struct kmemleak_object *object)
 353{
 354	int i;
 355	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
 356
 357	seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
 358		   object->pointer, object->size);
 359	seq_printf(seq, "  comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
 360		   object->comm, object->pid, object->jiffies,
 361		   msecs_age / 1000, msecs_age % 1000);
 362	hex_dump_object(seq, object);
 363	seq_printf(seq, "  backtrace:\n");
 364
 365	for (i = 0; i < object->trace_len; i++) {
 366		void *ptr = (void *)object->trace[i];
 367		seq_printf(seq, "    [<%p>] %pS\n", ptr, ptr);
 368	}
 369}
 370
 371/*
 372 * Print the kmemleak_object information. This function is used mainly for
 373 * debugging special cases when kmemleak operations. It must be called with
 374 * the object->lock held.
 375 */
 376static void dump_object_info(struct kmemleak_object *object)
 377{
 378	struct stack_trace trace;
 379
 380	trace.nr_entries = object->trace_len;
 381	trace.entries = object->trace;
 382
 383	pr_notice("Object 0x%08lx (size %zu):\n",
 384		  object->pointer, object->size);
 385	pr_notice("  comm \"%s\", pid %d, jiffies %lu\n",
 386		  object->comm, object->pid, object->jiffies);
 387	pr_notice("  min_count = %d\n", object->min_count);
 388	pr_notice("  count = %d\n", object->count);
 389	pr_notice("  flags = 0x%lx\n", object->flags);
 390	pr_notice("  checksum = %d\n", object->checksum);
 391	pr_notice("  backtrace:\n");
 392	print_stack_trace(&trace, 4);
 393}
 394
 395/*
 396 * Look-up a memory block metadata (kmemleak_object) in the object search
 397 * tree based on a pointer value. If alias is 0, only values pointing to the
 398 * beginning of the memory block are allowed. The kmemleak_lock must be held
 399 * when calling this function.
 400 */
 401static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
 402{
 403	struct rb_node *rb = object_tree_root.rb_node;
 404
 405	while (rb) {
 406		struct kmemleak_object *object =
 407			rb_entry(rb, struct kmemleak_object, rb_node);
 408		if (ptr < object->pointer)
 409			rb = object->rb_node.rb_left;
 410		else if (object->pointer + object->size <= ptr)
 411			rb = object->rb_node.rb_right;
 412		else if (object->pointer == ptr || alias)
 413			return object;
 414		else {
 415			kmemleak_warn("Found object by alias at 0x%08lx\n",
 416				      ptr);
 417			dump_object_info(object);
 418			break;
 419		}
 420	}
 421	return NULL;
 422}
 423
 424/*
 425 * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
 426 * that once an object's use_count reached 0, the RCU freeing was already
 427 * registered and the object should no longer be used. This function must be
 428 * called under the protection of rcu_read_lock().
 429 */
 430static int get_object(struct kmemleak_object *object)
 431{
 432	return atomic_inc_not_zero(&object->use_count);
 433}
 434
 435/*
 436 * RCU callback to free a kmemleak_object.
 437 */
 438static void free_object_rcu(struct rcu_head *rcu)
 439{
 440	struct hlist_node *tmp;
 441	struct kmemleak_scan_area *area;
 442	struct kmemleak_object *object =
 443		container_of(rcu, struct kmemleak_object, rcu);
 444
 445	/*
 446	 * Once use_count is 0 (guaranteed by put_object), there is no other
 447	 * code accessing this object, hence no need for locking.
 448	 */
 449	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
 450		hlist_del(&area->node);
 451		kmem_cache_free(scan_area_cache, area);
 452	}
 453	kmem_cache_free(object_cache, object);
 454}
 455
 456/*
 457 * Decrement the object use_count. Once the count is 0, free the object using
 458 * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
 459 * delete_object() path, the delayed RCU freeing ensures that there is no
 460 * recursive call to the kernel allocator. Lock-less RCU object_list traversal
 461 * is also possible.
 462 */
 463static void put_object(struct kmemleak_object *object)
 464{
 465	if (!atomic_dec_and_test(&object->use_count))
 466		return;
 467
 468	/* should only get here after delete_object was called */
 469	WARN_ON(object->flags & OBJECT_ALLOCATED);
 470
 471	call_rcu(&object->rcu, free_object_rcu);
 472}
 473
 474/*
 475 * Look up an object in the object search tree and increase its use_count.
 476 */
 477static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
 478{
 479	unsigned long flags;
 480	struct kmemleak_object *object = NULL;
 481
 482	rcu_read_lock();
 483	read_lock_irqsave(&kmemleak_lock, flags);
 484	if (ptr >= min_addr && ptr < max_addr)
 485		object = lookup_object(ptr, alias);
 486	read_unlock_irqrestore(&kmemleak_lock, flags);
 487
 488	/* check whether the object is still available */
 489	if (object && !get_object(object))
 490		object = NULL;
 491	rcu_read_unlock();
 492
 493	return object;
 494}
 495
 496/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 497 * Save stack trace to the given array of MAX_TRACE size.
 498 */
 499static int __save_stack_trace(unsigned long *trace)
 500{
 501	struct stack_trace stack_trace;
 502
 503	stack_trace.max_entries = MAX_TRACE;
 504	stack_trace.nr_entries = 0;
 505	stack_trace.entries = trace;
 506	stack_trace.skip = 2;
 507	save_stack_trace(&stack_trace);
 508
 509	return stack_trace.nr_entries;
 510}
 511
 512/*
 513 * Create the metadata (struct kmemleak_object) corresponding to an allocated
 514 * memory block and add it to the object_list and object_tree_root.
 515 */
 516static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
 517					     int min_count, gfp_t gfp)
 518{
 519	unsigned long flags;
 520	struct kmemleak_object *object, *parent;
 521	struct rb_node **link, *rb_parent;
 522
 523	object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
 524	if (!object) {
 525		pr_warning("Cannot allocate a kmemleak_object structure\n");
 526		kmemleak_disable();
 527		return NULL;
 528	}
 529
 530	INIT_LIST_HEAD(&object->object_list);
 531	INIT_LIST_HEAD(&object->gray_list);
 532	INIT_HLIST_HEAD(&object->area_list);
 533	spin_lock_init(&object->lock);
 534	atomic_set(&object->use_count, 1);
 535	object->flags = OBJECT_ALLOCATED;
 536	object->pointer = ptr;
 537	object->size = size;
 538	object->min_count = min_count;
 539	object->count = 0;			/* white color initially */
 540	object->jiffies = jiffies;
 541	object->checksum = 0;
 542
 543	/* task information */
 544	if (in_irq()) {
 545		object->pid = 0;
 546		strncpy(object->comm, "hardirq", sizeof(object->comm));
 547	} else if (in_softirq()) {
 548		object->pid = 0;
 549		strncpy(object->comm, "softirq", sizeof(object->comm));
 550	} else {
 551		object->pid = current->pid;
 552		/*
 553		 * There is a small chance of a race with set_task_comm(),
 554		 * however using get_task_comm() here may cause locking
 555		 * dependency issues with current->alloc_lock. In the worst
 556		 * case, the command line is not correct.
 557		 */
 558		strncpy(object->comm, current->comm, sizeof(object->comm));
 559	}
 560
 561	/* kernel backtrace */
 562	object->trace_len = __save_stack_trace(object->trace);
 563
 564	write_lock_irqsave(&kmemleak_lock, flags);
 565
 566	min_addr = min(min_addr, ptr);
 567	max_addr = max(max_addr, ptr + size);
 568	link = &object_tree_root.rb_node;
 569	rb_parent = NULL;
 570	while (*link) {
 571		rb_parent = *link;
 572		parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
 573		if (ptr + size <= parent->pointer)
 574			link = &parent->rb_node.rb_left;
 575		else if (parent->pointer + parent->size <= ptr)
 576			link = &parent->rb_node.rb_right;
 577		else {
 578			kmemleak_stop("Cannot insert 0x%lx into the object "
 579				      "search tree (overlaps existing)\n",
 580				      ptr);
 
 
 
 
 
 581			kmem_cache_free(object_cache, object);
 582			object = parent;
 583			spin_lock(&object->lock);
 584			dump_object_info(object);
 585			spin_unlock(&object->lock);
 586			goto out;
 587		}
 588	}
 589	rb_link_node(&object->rb_node, rb_parent, link);
 590	rb_insert_color(&object->rb_node, &object_tree_root);
 591
 592	list_add_tail_rcu(&object->object_list, &object_list);
 593out:
 594	write_unlock_irqrestore(&kmemleak_lock, flags);
 595	return object;
 596}
 597
 598/*
 599 * Remove the metadata (struct kmemleak_object) for a memory block from the
 600 * object_list and object_tree_root and decrement its use_count.
 601 */
 602static void __delete_object(struct kmemleak_object *object)
 603{
 604	unsigned long flags;
 605
 606	write_lock_irqsave(&kmemleak_lock, flags);
 607	rb_erase(&object->rb_node, &object_tree_root);
 608	list_del_rcu(&object->object_list);
 609	write_unlock_irqrestore(&kmemleak_lock, flags);
 610
 611	WARN_ON(!(object->flags & OBJECT_ALLOCATED));
 612	WARN_ON(atomic_read(&object->use_count) < 2);
 613
 614	/*
 615	 * Locking here also ensures that the corresponding memory block
 616	 * cannot be freed when it is being scanned.
 617	 */
 618	spin_lock_irqsave(&object->lock, flags);
 619	object->flags &= ~OBJECT_ALLOCATED;
 620	spin_unlock_irqrestore(&object->lock, flags);
 621	put_object(object);
 622}
 623
 624/*
 625 * Look up the metadata (struct kmemleak_object) corresponding to ptr and
 626 * delete it.
 627 */
 628static void delete_object_full(unsigned long ptr)
 629{
 630	struct kmemleak_object *object;
 631
 632	object = find_and_get_object(ptr, 0);
 633	if (!object) {
 634#ifdef DEBUG
 635		kmemleak_warn("Freeing unknown object at 0x%08lx\n",
 636			      ptr);
 637#endif
 638		return;
 639	}
 640	__delete_object(object);
 641	put_object(object);
 642}
 643
 644/*
 645 * Look up the metadata (struct kmemleak_object) corresponding to ptr and
 646 * delete it. If the memory block is partially freed, the function may create
 647 * additional metadata for the remaining parts of the block.
 648 */
 649static void delete_object_part(unsigned long ptr, size_t size)
 650{
 651	struct kmemleak_object *object;
 652	unsigned long start, end;
 653
 654	object = find_and_get_object(ptr, 1);
 655	if (!object) {
 656#ifdef DEBUG
 657		kmemleak_warn("Partially freeing unknown object at 0x%08lx "
 658			      "(size %zu)\n", ptr, size);
 659#endif
 660		return;
 661	}
 662	__delete_object(object);
 663
 664	/*
 665	 * Create one or two objects that may result from the memory block
 666	 * split. Note that partial freeing is only done by free_bootmem() and
 667	 * this happens before kmemleak_init() is called. The path below is
 668	 * only executed during early log recording in kmemleak_init(), so
 669	 * GFP_KERNEL is enough.
 670	 */
 671	start = object->pointer;
 672	end = object->pointer + object->size;
 673	if (ptr > start)
 674		create_object(start, ptr - start, object->min_count,
 675			      GFP_KERNEL);
 676	if (ptr + size < end)
 677		create_object(ptr + size, end - ptr - size, object->min_count,
 678			      GFP_KERNEL);
 679
 680	put_object(object);
 681}
 682
 683static void __paint_it(struct kmemleak_object *object, int color)
 684{
 685	object->min_count = color;
 686	if (color == KMEMLEAK_BLACK)
 687		object->flags |= OBJECT_NO_SCAN;
 688}
 689
 690static void paint_it(struct kmemleak_object *object, int color)
 691{
 692	unsigned long flags;
 693
 694	spin_lock_irqsave(&object->lock, flags);
 695	__paint_it(object, color);
 696	spin_unlock_irqrestore(&object->lock, flags);
 697}
 698
 699static void paint_ptr(unsigned long ptr, int color)
 700{
 701	struct kmemleak_object *object;
 702
 703	object = find_and_get_object(ptr, 0);
 704	if (!object) {
 705		kmemleak_warn("Trying to color unknown object "
 706			      "at 0x%08lx as %s\n", ptr,
 707			      (color == KMEMLEAK_GREY) ? "Grey" :
 708			      (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
 709		return;
 710	}
 711	paint_it(object, color);
 712	put_object(object);
 713}
 714
 715/*
 716 * Mark an object permanently as gray-colored so that it can no longer be
 717 * reported as a leak. This is used in general to mark a false positive.
 718 */
 719static void make_gray_object(unsigned long ptr)
 720{
 721	paint_ptr(ptr, KMEMLEAK_GREY);
 722}
 723
 724/*
 725 * Mark the object as black-colored so that it is ignored from scans and
 726 * reporting.
 727 */
 728static void make_black_object(unsigned long ptr)
 729{
 730	paint_ptr(ptr, KMEMLEAK_BLACK);
 731}
 732
 733/*
 734 * Add a scanning area to the object. If at least one such area is added,
 735 * kmemleak will only scan these ranges rather than the whole memory block.
 736 */
 737static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
 738{
 739	unsigned long flags;
 740	struct kmemleak_object *object;
 741	struct kmemleak_scan_area *area;
 742
 743	object = find_and_get_object(ptr, 1);
 744	if (!object) {
 745		kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
 746			      ptr);
 747		return;
 748	}
 749
 750	area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
 751	if (!area) {
 752		pr_warning("Cannot allocate a scan area\n");
 753		goto out;
 754	}
 755
 756	spin_lock_irqsave(&object->lock, flags);
 757	if (size == SIZE_MAX) {
 758		size = object->pointer + object->size - ptr;
 759	} else if (ptr + size > object->pointer + object->size) {
 760		kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
 761		dump_object_info(object);
 762		kmem_cache_free(scan_area_cache, area);
 763		goto out_unlock;
 764	}
 765
 766	INIT_HLIST_NODE(&area->node);
 767	area->start = ptr;
 768	area->size = size;
 769
 770	hlist_add_head(&area->node, &object->area_list);
 771out_unlock:
 772	spin_unlock_irqrestore(&object->lock, flags);
 773out:
 774	put_object(object);
 775}
 776
 777/*
 778 * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
 779 * pointer. Such object will not be scanned by kmemleak but references to it
 780 * are searched.
 781 */
 782static void object_no_scan(unsigned long ptr)
 783{
 784	unsigned long flags;
 785	struct kmemleak_object *object;
 786
 787	object = find_and_get_object(ptr, 0);
 788	if (!object) {
 789		kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
 790		return;
 791	}
 792
 793	spin_lock_irqsave(&object->lock, flags);
 794	object->flags |= OBJECT_NO_SCAN;
 795	spin_unlock_irqrestore(&object->lock, flags);
 796	put_object(object);
 797}
 798
 799/*
 800 * Log an early kmemleak_* call to the early_log buffer. These calls will be
 801 * processed later once kmemleak is fully initialized.
 802 */
 803static void __init log_early(int op_type, const void *ptr, size_t size,
 804			     int min_count)
 805{
 806	unsigned long flags;
 807	struct early_log *log;
 808
 809	if (kmemleak_error) {
 810		/* kmemleak stopped recording, just count the requests */
 811		crt_early_log++;
 812		return;
 813	}
 814
 815	if (crt_early_log >= ARRAY_SIZE(early_log)) {
 
 816		kmemleak_disable();
 817		return;
 818	}
 819
 820	/*
 821	 * There is no need for locking since the kernel is still in UP mode
 822	 * at this stage. Disabling the IRQs is enough.
 823	 */
 824	local_irq_save(flags);
 825	log = &early_log[crt_early_log];
 826	log->op_type = op_type;
 827	log->ptr = ptr;
 828	log->size = size;
 829	log->min_count = min_count;
 830	log->trace_len = __save_stack_trace(log->trace);
 831	crt_early_log++;
 832	local_irq_restore(flags);
 833}
 834
 835/*
 836 * Log an early allocated block and populate the stack trace.
 837 */
 838static void early_alloc(struct early_log *log)
 839{
 840	struct kmemleak_object *object;
 841	unsigned long flags;
 842	int i;
 843
 844	if (!kmemleak_enabled || !log->ptr || IS_ERR(log->ptr))
 845		return;
 846
 847	/*
 848	 * RCU locking needed to ensure object is not freed via put_object().
 849	 */
 850	rcu_read_lock();
 851	object = create_object((unsigned long)log->ptr, log->size,
 852			       log->min_count, GFP_ATOMIC);
 853	if (!object)
 854		goto out;
 855	spin_lock_irqsave(&object->lock, flags);
 856	for (i = 0; i < log->trace_len; i++)
 857		object->trace[i] = log->trace[i];
 858	object->trace_len = log->trace_len;
 859	spin_unlock_irqrestore(&object->lock, flags);
 860out:
 861	rcu_read_unlock();
 862}
 863
 864/*
 865 * Log an early allocated block and populate the stack trace.
 866 */
 867static void early_alloc_percpu(struct early_log *log)
 868{
 869	unsigned int cpu;
 870	const void __percpu *ptr = log->ptr;
 871
 872	for_each_possible_cpu(cpu) {
 873		log->ptr = per_cpu_ptr(ptr, cpu);
 874		early_alloc(log);
 875	}
 876}
 877
 878/**
 879 * kmemleak_alloc - register a newly allocated object
 880 * @ptr:	pointer to beginning of the object
 881 * @size:	size of the object
 882 * @min_count:	minimum number of references to this object. If during memory
 883 *		scanning a number of references less than @min_count is found,
 884 *		the object is reported as a memory leak. If @min_count is 0,
 885 *		the object is never reported as a leak. If @min_count is -1,
 886 *		the object is ignored (not scanned and not reported as a leak)
 887 * @gfp:	kmalloc() flags used for kmemleak internal memory allocations
 888 *
 889 * This function is called from the kernel allocators when a new object
 890 * (memory block) is allocated (kmem_cache_alloc, kmalloc, vmalloc etc.).
 891 */
 892void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
 893			  gfp_t gfp)
 894{
 895	pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
 896
 897	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
 898		create_object((unsigned long)ptr, size, min_count, gfp);
 899	else if (kmemleak_early_log)
 900		log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
 901}
 902EXPORT_SYMBOL_GPL(kmemleak_alloc);
 903
 904/**
 905 * kmemleak_alloc_percpu - register a newly allocated __percpu object
 906 * @ptr:	__percpu pointer to beginning of the object
 907 * @size:	size of the object
 
 908 *
 909 * This function is called from the kernel percpu allocator when a new object
 910 * (memory block) is allocated (alloc_percpu). It assumes GFP_KERNEL
 911 * allocation.
 912 */
 913void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size)
 
 914{
 915	unsigned int cpu;
 916
 917	pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
 918
 919	/*
 920	 * Percpu allocations are only scanned and not reported as leaks
 921	 * (min_count is set to 0).
 922	 */
 923	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
 924		for_each_possible_cpu(cpu)
 925			create_object((unsigned long)per_cpu_ptr(ptr, cpu),
 926				      size, 0, GFP_KERNEL);
 927	else if (kmemleak_early_log)
 928		log_early(KMEMLEAK_ALLOC_PERCPU, ptr, size, 0);
 929}
 930EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
 931
 932/**
 933 * kmemleak_free - unregister a previously registered object
 934 * @ptr:	pointer to beginning of the object
 935 *
 936 * This function is called from the kernel allocators when an object (memory
 937 * block) is freed (kmem_cache_free, kfree, vfree etc.).
 938 */
 939void __ref kmemleak_free(const void *ptr)
 940{
 941	pr_debug("%s(0x%p)\n", __func__, ptr);
 942
 943	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
 944		delete_object_full((unsigned long)ptr);
 945	else if (kmemleak_early_log)
 946		log_early(KMEMLEAK_FREE, ptr, 0, 0);
 947}
 948EXPORT_SYMBOL_GPL(kmemleak_free);
 949
 950/**
 951 * kmemleak_free_part - partially unregister a previously registered object
 952 * @ptr:	pointer to the beginning or inside the object. This also
 953 *		represents the start of the range to be freed
 954 * @size:	size to be unregistered
 955 *
 956 * This function is called when only a part of a memory block is freed
 957 * (usually from the bootmem allocator).
 958 */
 959void __ref kmemleak_free_part(const void *ptr, size_t size)
 960{
 961	pr_debug("%s(0x%p)\n", __func__, ptr);
 962
 963	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
 964		delete_object_part((unsigned long)ptr, size);
 965	else if (kmemleak_early_log)
 966		log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
 967}
 968EXPORT_SYMBOL_GPL(kmemleak_free_part);
 969
 970/**
 971 * kmemleak_free_percpu - unregister a previously registered __percpu object
 972 * @ptr:	__percpu pointer to beginning of the object
 973 *
 974 * This function is called from the kernel percpu allocator when an object
 975 * (memory block) is freed (free_percpu).
 976 */
 977void __ref kmemleak_free_percpu(const void __percpu *ptr)
 978{
 979	unsigned int cpu;
 980
 981	pr_debug("%s(0x%p)\n", __func__, ptr);
 982
 983	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
 984		for_each_possible_cpu(cpu)
 985			delete_object_full((unsigned long)per_cpu_ptr(ptr,
 986								      cpu));
 987	else if (kmemleak_early_log)
 988		log_early(KMEMLEAK_FREE_PERCPU, ptr, 0, 0);
 989}
 990EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
 991
 992/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 993 * kmemleak_not_leak - mark an allocated object as false positive
 994 * @ptr:	pointer to beginning of the object
 995 *
 996 * Calling this function on an object will cause the memory block to no longer
 997 * be reported as leak and always be scanned.
 998 */
 999void __ref kmemleak_not_leak(const void *ptr)
1000{
1001	pr_debug("%s(0x%p)\n", __func__, ptr);
1002
1003	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1004		make_gray_object((unsigned long)ptr);
1005	else if (kmemleak_early_log)
1006		log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
1007}
1008EXPORT_SYMBOL(kmemleak_not_leak);
1009
1010/**
1011 * kmemleak_ignore - ignore an allocated object
1012 * @ptr:	pointer to beginning of the object
1013 *
1014 * Calling this function on an object will cause the memory block to be
1015 * ignored (not scanned and not reported as a leak). This is usually done when
1016 * it is known that the corresponding block is not a leak and does not contain
1017 * any references to other allocated memory blocks.
1018 */
1019void __ref kmemleak_ignore(const void *ptr)
1020{
1021	pr_debug("%s(0x%p)\n", __func__, ptr);
1022
1023	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1024		make_black_object((unsigned long)ptr);
1025	else if (kmemleak_early_log)
1026		log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
1027}
1028EXPORT_SYMBOL(kmemleak_ignore);
1029
1030/**
1031 * kmemleak_scan_area - limit the range to be scanned in an allocated object
1032 * @ptr:	pointer to beginning or inside the object. This also
1033 *		represents the start of the scan area
1034 * @size:	size of the scan area
1035 * @gfp:	kmalloc() flags used for kmemleak internal memory allocations
1036 *
1037 * This function is used when it is known that only certain parts of an object
1038 * contain references to other objects. Kmemleak will only scan these areas
1039 * reducing the number false negatives.
1040 */
1041void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1042{
1043	pr_debug("%s(0x%p)\n", __func__, ptr);
1044
1045	if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
1046		add_scan_area((unsigned long)ptr, size, gfp);
1047	else if (kmemleak_early_log)
1048		log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
1049}
1050EXPORT_SYMBOL(kmemleak_scan_area);
1051
1052/**
1053 * kmemleak_no_scan - do not scan an allocated object
1054 * @ptr:	pointer to beginning of the object
1055 *
1056 * This function notifies kmemleak not to scan the given memory block. Useful
1057 * in situations where it is known that the given object does not contain any
1058 * references to other objects. Kmemleak will not scan such objects reducing
1059 * the number of false negatives.
1060 */
1061void __ref kmemleak_no_scan(const void *ptr)
1062{
1063	pr_debug("%s(0x%p)\n", __func__, ptr);
1064
1065	if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1066		object_no_scan((unsigned long)ptr);
1067	else if (kmemleak_early_log)
1068		log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
1069}
1070EXPORT_SYMBOL(kmemleak_no_scan);
1071
1072/*
1073 * Update an object's checksum and return true if it was modified.
1074 */
1075static bool update_checksum(struct kmemleak_object *object)
1076{
1077	u32 old_csum = object->checksum;
1078
1079	if (!kmemcheck_is_obj_initialized(object->pointer, object->size))
1080		return false;
1081
 
1082	object->checksum = crc32(0, (void *)object->pointer, object->size);
 
 
1083	return object->checksum != old_csum;
1084}
1085
1086/*
1087 * Memory scanning is a long process and it needs to be interruptable. This
1088 * function checks whether such interrupt condition occurred.
1089 */
1090static int scan_should_stop(void)
1091{
1092	if (!kmemleak_enabled)
1093		return 1;
1094
1095	/*
1096	 * This function may be called from either process or kthread context,
1097	 * hence the need to check for both stop conditions.
1098	 */
1099	if (current->mm)
1100		return signal_pending(current);
1101	else
1102		return kthread_should_stop();
1103
1104	return 0;
1105}
1106
1107/*
1108 * Scan a memory block (exclusive range) for valid pointers and add those
1109 * found to the gray list.
1110 */
1111static void scan_block(void *_start, void *_end,
1112		       struct kmemleak_object *scanned, int allow_resched)
1113{
1114	unsigned long *ptr;
1115	unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1116	unsigned long *end = _end - (BYTES_PER_POINTER - 1);
 
1117
 
1118	for (ptr = start; ptr < end; ptr++) {
1119		struct kmemleak_object *object;
1120		unsigned long flags;
1121		unsigned long pointer;
1122
1123		if (allow_resched)
1124			cond_resched();
1125		if (scan_should_stop())
1126			break;
1127
1128		/* don't scan uninitialized memory */
1129		if (!kmemcheck_is_obj_initialized((unsigned long)ptr,
1130						  BYTES_PER_POINTER))
1131			continue;
1132
 
1133		pointer = *ptr;
 
1134
1135		object = find_and_get_object(pointer, 1);
 
 
 
 
 
 
 
 
 
1136		if (!object)
1137			continue;
1138		if (object == scanned) {
1139			/* self referenced, ignore */
1140			put_object(object);
1141			continue;
1142		}
1143
1144		/*
1145		 * Avoid the lockdep recursive warning on object->lock being
1146		 * previously acquired in scan_object(). These locks are
1147		 * enclosed by scan_mutex.
1148		 */
1149		spin_lock_irqsave_nested(&object->lock, flags,
1150					 SINGLE_DEPTH_NESTING);
1151		if (!color_white(object)) {
1152			/* non-orphan, ignored or new */
1153			spin_unlock_irqrestore(&object->lock, flags);
1154			put_object(object);
1155			continue;
1156		}
1157
1158		/*
1159		 * Increase the object's reference count (number of pointers
1160		 * to the memory block). If this count reaches the required
1161		 * minimum, the object's color will become gray and it will be
1162		 * added to the gray_list.
1163		 */
1164		object->count++;
1165		if (color_gray(object)) {
 
 
1166			list_add_tail(&object->gray_list, &gray_list);
1167			spin_unlock_irqrestore(&object->lock, flags);
1168			continue;
1169		}
 
 
 
 
1170
1171		spin_unlock_irqrestore(&object->lock, flags);
1172		put_object(object);
 
 
 
 
 
 
 
 
 
 
1173	}
1174}
1175
1176/*
1177 * Scan a memory block corresponding to a kmemleak_object. A condition is
1178 * that object->use_count >= 1.
1179 */
1180static void scan_object(struct kmemleak_object *object)
1181{
1182	struct kmemleak_scan_area *area;
1183	unsigned long flags;
1184
1185	/*
1186	 * Once the object->lock is acquired, the corresponding memory block
1187	 * cannot be freed (the same lock is acquired in delete_object).
1188	 */
1189	spin_lock_irqsave(&object->lock, flags);
1190	if (object->flags & OBJECT_NO_SCAN)
1191		goto out;
1192	if (!(object->flags & OBJECT_ALLOCATED))
1193		/* already freed object */
1194		goto out;
1195	if (hlist_empty(&object->area_list)) {
1196		void *start = (void *)object->pointer;
1197		void *end = (void *)(object->pointer + object->size);
 
1198
1199		while (start < end && (object->flags & OBJECT_ALLOCATED) &&
1200		       !(object->flags & OBJECT_NO_SCAN)) {
1201			scan_block(start, min(start + MAX_SCAN_SIZE, end),
1202				   object, 0);
1203			start += MAX_SCAN_SIZE;
 
 
1204
1205			spin_unlock_irqrestore(&object->lock, flags);
1206			cond_resched();
1207			spin_lock_irqsave(&object->lock, flags);
1208		}
1209	} else
1210		hlist_for_each_entry(area, &object->area_list, node)
1211			scan_block((void *)area->start,
1212				   (void *)(area->start + area->size),
1213				   object, 0);
1214out:
1215	spin_unlock_irqrestore(&object->lock, flags);
1216}
1217
1218/*
1219 * Scan the objects already referenced (gray objects). More objects will be
1220 * referenced and, if there are no memory leaks, all the objects are scanned.
1221 */
1222static void scan_gray_list(void)
1223{
1224	struct kmemleak_object *object, *tmp;
1225
1226	/*
1227	 * The list traversal is safe for both tail additions and removals
1228	 * from inside the loop. The kmemleak objects cannot be freed from
1229	 * outside the loop because their use_count was incremented.
1230	 */
1231	object = list_entry(gray_list.next, typeof(*object), gray_list);
1232	while (&object->gray_list != &gray_list) {
1233		cond_resched();
1234
1235		/* may add new objects to the list */
1236		if (!scan_should_stop())
1237			scan_object(object);
1238
1239		tmp = list_entry(object->gray_list.next, typeof(*object),
1240				 gray_list);
1241
1242		/* remove the object from the list and release it */
1243		list_del(&object->gray_list);
1244		put_object(object);
1245
1246		object = tmp;
1247	}
1248	WARN_ON(!list_empty(&gray_list));
1249}
1250
1251/*
1252 * Scan data sections and all the referenced memory blocks allocated via the
1253 * kernel's standard allocators. This function must be called with the
1254 * scan_mutex held.
1255 */
1256static void kmemleak_scan(void)
1257{
1258	unsigned long flags;
1259	struct kmemleak_object *object;
1260	int i;
1261	int new_leaks = 0;
1262
1263	jiffies_last_scan = jiffies;
1264
1265	/* prepare the kmemleak_object's */
1266	rcu_read_lock();
1267	list_for_each_entry_rcu(object, &object_list, object_list) {
1268		spin_lock_irqsave(&object->lock, flags);
1269#ifdef DEBUG
1270		/*
1271		 * With a few exceptions there should be a maximum of
1272		 * 1 reference to any object at this point.
1273		 */
1274		if (atomic_read(&object->use_count) > 1) {
1275			pr_debug("object->use_count = %d\n",
1276				 atomic_read(&object->use_count));
1277			dump_object_info(object);
1278		}
1279#endif
1280		/* reset the reference count (whiten the object) */
1281		object->count = 0;
1282		if (color_gray(object) && get_object(object))
1283			list_add_tail(&object->gray_list, &gray_list);
1284
1285		spin_unlock_irqrestore(&object->lock, flags);
1286	}
1287	rcu_read_unlock();
1288
1289	/* data/bss scanning */
1290	scan_block(_sdata, _edata, NULL, 1);
1291	scan_block(__bss_start, __bss_stop, NULL, 1);
1292
1293#ifdef CONFIG_SMP
1294	/* per-cpu sections scanning */
1295	for_each_possible_cpu(i)
1296		scan_block(__per_cpu_start + per_cpu_offset(i),
1297			   __per_cpu_end + per_cpu_offset(i), NULL, 1);
1298#endif
1299
1300	/*
1301	 * Struct page scanning for each node.
1302	 */
1303	lock_memory_hotplug();
1304	for_each_online_node(i) {
1305		unsigned long start_pfn = node_start_pfn(i);
1306		unsigned long end_pfn = node_end_pfn(i);
1307		unsigned long pfn;
1308
1309		for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1310			struct page *page;
1311
1312			if (!pfn_valid(pfn))
1313				continue;
1314			page = pfn_to_page(pfn);
1315			/* only scan if page is in use */
1316			if (page_count(page) == 0)
1317				continue;
1318			scan_block(page, page + 1, NULL, 1);
1319		}
1320	}
1321	unlock_memory_hotplug();
1322
1323	/*
1324	 * Scanning the task stacks (may introduce false negatives).
1325	 */
1326	if (kmemleak_stack_scan) {
1327		struct task_struct *p, *g;
1328
1329		read_lock(&tasklist_lock);
1330		do_each_thread(g, p) {
1331			scan_block(task_stack_page(p), task_stack_page(p) +
1332				   THREAD_SIZE, NULL, 0);
1333		} while_each_thread(g, p);
1334		read_unlock(&tasklist_lock);
1335	}
1336
1337	/*
1338	 * Scan the objects already referenced from the sections scanned
1339	 * above.
1340	 */
1341	scan_gray_list();
1342
1343	/*
1344	 * Check for new or unreferenced objects modified since the previous
1345	 * scan and color them gray until the next scan.
1346	 */
1347	rcu_read_lock();
1348	list_for_each_entry_rcu(object, &object_list, object_list) {
1349		spin_lock_irqsave(&object->lock, flags);
1350		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1351		    && update_checksum(object) && get_object(object)) {
1352			/* color it gray temporarily */
1353			object->count = object->min_count;
1354			list_add_tail(&object->gray_list, &gray_list);
1355		}
1356		spin_unlock_irqrestore(&object->lock, flags);
1357	}
1358	rcu_read_unlock();
1359
1360	/*
1361	 * Re-scan the gray list for modified unreferenced objects.
1362	 */
1363	scan_gray_list();
1364
1365	/*
1366	 * If scanning was stopped do not report any new unreferenced objects.
1367	 */
1368	if (scan_should_stop())
1369		return;
1370
1371	/*
1372	 * Scanning result reporting.
1373	 */
1374	rcu_read_lock();
1375	list_for_each_entry_rcu(object, &object_list, object_list) {
1376		spin_lock_irqsave(&object->lock, flags);
1377		if (unreferenced_object(object) &&
1378		    !(object->flags & OBJECT_REPORTED)) {
1379			object->flags |= OBJECT_REPORTED;
1380			new_leaks++;
1381		}
1382		spin_unlock_irqrestore(&object->lock, flags);
1383	}
1384	rcu_read_unlock();
1385
1386	if (new_leaks) {
1387		kmemleak_found_leaks = true;
1388
1389		pr_info("%d new suspected memory leaks (see "
1390			"/sys/kernel/debug/kmemleak)\n", new_leaks);
1391	}
1392
1393}
1394
1395/*
1396 * Thread function performing automatic memory scanning. Unreferenced objects
1397 * at the end of a memory scan are reported but only the first time.
1398 */
1399static int kmemleak_scan_thread(void *arg)
1400{
1401	static int first_run = 1;
1402
1403	pr_info("Automatic memory scanning thread started\n");
1404	set_user_nice(current, 10);
1405
1406	/*
1407	 * Wait before the first scan to allow the system to fully initialize.
1408	 */
1409	if (first_run) {
1410		first_run = 0;
1411		ssleep(SECS_FIRST_SCAN);
1412	}
1413
1414	while (!kthread_should_stop()) {
1415		signed long timeout = jiffies_scan_wait;
1416
1417		mutex_lock(&scan_mutex);
1418		kmemleak_scan();
1419		mutex_unlock(&scan_mutex);
1420
1421		/* wait before the next scan */
1422		while (timeout && !kthread_should_stop())
1423			timeout = schedule_timeout_interruptible(timeout);
1424	}
1425
1426	pr_info("Automatic memory scanning thread ended\n");
1427
1428	return 0;
1429}
1430
1431/*
1432 * Start the automatic memory scanning thread. This function must be called
1433 * with the scan_mutex held.
1434 */
1435static void start_scan_thread(void)
1436{
1437	if (scan_thread)
1438		return;
1439	scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1440	if (IS_ERR(scan_thread)) {
1441		pr_warning("Failed to create the scan thread\n");
1442		scan_thread = NULL;
1443	}
1444}
1445
1446/*
1447 * Stop the automatic memory scanning thread. This function must be called
1448 * with the scan_mutex held.
1449 */
1450static void stop_scan_thread(void)
1451{
1452	if (scan_thread) {
1453		kthread_stop(scan_thread);
1454		scan_thread = NULL;
1455	}
1456}
1457
1458/*
1459 * Iterate over the object_list and return the first valid object at or after
1460 * the required position with its use_count incremented. The function triggers
1461 * a memory scanning when the pos argument points to the first position.
1462 */
1463static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1464{
1465	struct kmemleak_object *object;
1466	loff_t n = *pos;
1467	int err;
1468
1469	err = mutex_lock_interruptible(&scan_mutex);
1470	if (err < 0)
1471		return ERR_PTR(err);
1472
1473	rcu_read_lock();
1474	list_for_each_entry_rcu(object, &object_list, object_list) {
1475		if (n-- > 0)
1476			continue;
1477		if (get_object(object))
1478			goto out;
1479	}
1480	object = NULL;
1481out:
1482	return object;
1483}
1484
1485/*
1486 * Return the next object in the object_list. The function decrements the
1487 * use_count of the previous object and increases that of the next one.
1488 */
1489static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1490{
1491	struct kmemleak_object *prev_obj = v;
1492	struct kmemleak_object *next_obj = NULL;
1493	struct kmemleak_object *obj = prev_obj;
1494
1495	++(*pos);
1496
1497	list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
1498		if (get_object(obj)) {
1499			next_obj = obj;
1500			break;
1501		}
1502	}
1503
1504	put_object(prev_obj);
1505	return next_obj;
1506}
1507
1508/*
1509 * Decrement the use_count of the last object required, if any.
1510 */
1511static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1512{
1513	if (!IS_ERR(v)) {
1514		/*
1515		 * kmemleak_seq_start may return ERR_PTR if the scan_mutex
1516		 * waiting was interrupted, so only release it if !IS_ERR.
1517		 */
1518		rcu_read_unlock();
1519		mutex_unlock(&scan_mutex);
1520		if (v)
1521			put_object(v);
1522	}
1523}
1524
1525/*
1526 * Print the information for an unreferenced object to the seq file.
1527 */
1528static int kmemleak_seq_show(struct seq_file *seq, void *v)
1529{
1530	struct kmemleak_object *object = v;
1531	unsigned long flags;
1532
1533	spin_lock_irqsave(&object->lock, flags);
1534	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
1535		print_unreferenced(seq, object);
1536	spin_unlock_irqrestore(&object->lock, flags);
1537	return 0;
1538}
1539
1540static const struct seq_operations kmemleak_seq_ops = {
1541	.start = kmemleak_seq_start,
1542	.next  = kmemleak_seq_next,
1543	.stop  = kmemleak_seq_stop,
1544	.show  = kmemleak_seq_show,
1545};
1546
1547static int kmemleak_open(struct inode *inode, struct file *file)
1548{
1549	return seq_open(file, &kmemleak_seq_ops);
1550}
1551
1552static int dump_str_object_info(const char *str)
1553{
1554	unsigned long flags;
1555	struct kmemleak_object *object;
1556	unsigned long addr;
1557
1558	if (kstrtoul(str, 0, &addr))
1559		return -EINVAL;
1560	object = find_and_get_object(addr, 0);
1561	if (!object) {
1562		pr_info("Unknown object at 0x%08lx\n", addr);
1563		return -EINVAL;
1564	}
1565
1566	spin_lock_irqsave(&object->lock, flags);
1567	dump_object_info(object);
1568	spin_unlock_irqrestore(&object->lock, flags);
1569
1570	put_object(object);
1571	return 0;
1572}
1573
1574/*
1575 * We use grey instead of black to ensure we can do future scans on the same
1576 * objects. If we did not do future scans these black objects could
1577 * potentially contain references to newly allocated objects in the future and
1578 * we'd end up with false positives.
1579 */
1580static void kmemleak_clear(void)
1581{
1582	struct kmemleak_object *object;
1583	unsigned long flags;
1584
1585	rcu_read_lock();
1586	list_for_each_entry_rcu(object, &object_list, object_list) {
1587		spin_lock_irqsave(&object->lock, flags);
1588		if ((object->flags & OBJECT_REPORTED) &&
1589		    unreferenced_object(object))
1590			__paint_it(object, KMEMLEAK_GREY);
1591		spin_unlock_irqrestore(&object->lock, flags);
1592	}
1593	rcu_read_unlock();
1594
1595	kmemleak_found_leaks = false;
1596}
1597
1598static void __kmemleak_do_cleanup(void);
1599
1600/*
1601 * File write operation to configure kmemleak at run-time. The following
1602 * commands can be written to the /sys/kernel/debug/kmemleak file:
1603 *   off	- disable kmemleak (irreversible)
1604 *   stack=on	- enable the task stacks scanning
1605 *   stack=off	- disable the tasks stacks scanning
1606 *   scan=on	- start the automatic memory scanning thread
1607 *   scan=off	- stop the automatic memory scanning thread
1608 *   scan=...	- set the automatic memory scanning period in seconds (0 to
1609 *		  disable it)
1610 *   scan	- trigger a memory scan
1611 *   clear	- mark all current reported unreferenced kmemleak objects as
1612 *		  grey to ignore printing them, or free all kmemleak objects
1613 *		  if kmemleak has been disabled.
1614 *   dump=...	- dump information about the object found at the given address
1615 */
1616static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1617			      size_t size, loff_t *ppos)
1618{
1619	char buf[64];
1620	int buf_size;
1621	int ret;
1622
1623	buf_size = min(size, (sizeof(buf) - 1));
1624	if (strncpy_from_user(buf, user_buf, buf_size) < 0)
1625		return -EFAULT;
1626	buf[buf_size] = 0;
1627
1628	ret = mutex_lock_interruptible(&scan_mutex);
1629	if (ret < 0)
1630		return ret;
1631
1632	if (strncmp(buf, "clear", 5) == 0) {
1633		if (kmemleak_enabled)
1634			kmemleak_clear();
1635		else
1636			__kmemleak_do_cleanup();
1637		goto out;
1638	}
1639
1640	if (!kmemleak_enabled) {
1641		ret = -EBUSY;
1642		goto out;
1643	}
1644
1645	if (strncmp(buf, "off", 3) == 0)
1646		kmemleak_disable();
1647	else if (strncmp(buf, "stack=on", 8) == 0)
1648		kmemleak_stack_scan = 1;
1649	else if (strncmp(buf, "stack=off", 9) == 0)
1650		kmemleak_stack_scan = 0;
1651	else if (strncmp(buf, "scan=on", 7) == 0)
1652		start_scan_thread();
1653	else if (strncmp(buf, "scan=off", 8) == 0)
1654		stop_scan_thread();
1655	else if (strncmp(buf, "scan=", 5) == 0) {
1656		unsigned long secs;
1657
1658		ret = kstrtoul(buf + 5, 0, &secs);
1659		if (ret < 0)
1660			goto out;
1661		stop_scan_thread();
1662		if (secs) {
1663			jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
1664			start_scan_thread();
1665		}
1666	} else if (strncmp(buf, "scan", 4) == 0)
1667		kmemleak_scan();
1668	else if (strncmp(buf, "dump=", 5) == 0)
1669		ret = dump_str_object_info(buf + 5);
1670	else
1671		ret = -EINVAL;
1672
1673out:
1674	mutex_unlock(&scan_mutex);
1675	if (ret < 0)
1676		return ret;
1677
1678	/* ignore the rest of the buffer, only one command at a time */
1679	*ppos += size;
1680	return size;
1681}
1682
1683static const struct file_operations kmemleak_fops = {
1684	.owner		= THIS_MODULE,
1685	.open		= kmemleak_open,
1686	.read		= seq_read,
1687	.write		= kmemleak_write,
1688	.llseek		= seq_lseek,
1689	.release	= seq_release,
1690};
1691
1692static void __kmemleak_do_cleanup(void)
1693{
1694	struct kmemleak_object *object;
1695
1696	rcu_read_lock();
1697	list_for_each_entry_rcu(object, &object_list, object_list)
1698		delete_object_full(object->pointer);
1699	rcu_read_unlock();
1700}
1701
1702/*
1703 * Stop the memory scanning thread and free the kmemleak internal objects if
1704 * no previous scan thread (otherwise, kmemleak may still have some useful
1705 * information on memory leaks).
1706 */
1707static void kmemleak_do_cleanup(struct work_struct *work)
1708{
1709	mutex_lock(&scan_mutex);
1710	stop_scan_thread();
1711
 
 
 
 
 
 
 
1712	if (!kmemleak_found_leaks)
1713		__kmemleak_do_cleanup();
1714	else
1715		pr_info("Kmemleak disabled without freeing internal data. "
1716			"Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\"\n");
1717	mutex_unlock(&scan_mutex);
1718}
1719
1720static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
1721
1722/*
1723 * Disable kmemleak. No memory allocation/freeing will be traced once this
1724 * function is called. Disabling kmemleak is an irreversible operation.
1725 */
1726static void kmemleak_disable(void)
1727{
1728	/* atomically check whether it was already invoked */
1729	if (cmpxchg(&kmemleak_error, 0, 1))
1730		return;
1731
1732	/* stop any memory operation tracing */
1733	kmemleak_enabled = 0;
1734
1735	/* check whether it is too early for a kernel thread */
1736	if (kmemleak_initialized)
1737		schedule_work(&cleanup_work);
 
 
1738
1739	pr_info("Kernel memory leak detector disabled\n");
1740}
1741
1742/*
1743 * Allow boot-time kmemleak disabling (enabled by default).
1744 */
1745static int kmemleak_boot_config(char *str)
1746{
1747	if (!str)
1748		return -EINVAL;
1749	if (strcmp(str, "off") == 0)
1750		kmemleak_disable();
1751	else if (strcmp(str, "on") == 0)
1752		kmemleak_skip_disable = 1;
1753	else
1754		return -EINVAL;
1755	return 0;
1756}
1757early_param("kmemleak", kmemleak_boot_config);
1758
1759static void __init print_log_trace(struct early_log *log)
1760{
1761	struct stack_trace trace;
1762
1763	trace.nr_entries = log->trace_len;
1764	trace.entries = log->trace;
1765
1766	pr_notice("Early log backtrace:\n");
1767	print_stack_trace(&trace, 2);
1768}
1769
1770/*
1771 * Kmemleak initialization.
1772 */
1773void __init kmemleak_init(void)
1774{
1775	int i;
1776	unsigned long flags;
1777
1778#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
1779	if (!kmemleak_skip_disable) {
1780		kmemleak_early_log = 0;
1781		kmemleak_disable();
1782		return;
1783	}
1784#endif
1785
1786	jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
1787	jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
1788
1789	object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
1790	scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
1791
1792	if (crt_early_log >= ARRAY_SIZE(early_log))
1793		pr_warning("Early log buffer exceeded (%d), please increase "
1794			   "DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n", crt_early_log);
1795
1796	/* the kernel is still in UP mode, so disabling the IRQs is enough */
1797	local_irq_save(flags);
1798	kmemleak_early_log = 0;
1799	if (kmemleak_error) {
1800		local_irq_restore(flags);
1801		return;
1802	} else
1803		kmemleak_enabled = 1;
 
 
1804	local_irq_restore(flags);
1805
1806	/*
1807	 * This is the point where tracking allocations is safe. Automatic
1808	 * scanning is started during the late initcall. Add the early logged
1809	 * callbacks to the kmemleak infrastructure.
1810	 */
1811	for (i = 0; i < crt_early_log; i++) {
1812		struct early_log *log = &early_log[i];
1813
1814		switch (log->op_type) {
1815		case KMEMLEAK_ALLOC:
1816			early_alloc(log);
1817			break;
1818		case KMEMLEAK_ALLOC_PERCPU:
1819			early_alloc_percpu(log);
1820			break;
1821		case KMEMLEAK_FREE:
1822			kmemleak_free(log->ptr);
1823			break;
1824		case KMEMLEAK_FREE_PART:
1825			kmemleak_free_part(log->ptr, log->size);
1826			break;
1827		case KMEMLEAK_FREE_PERCPU:
1828			kmemleak_free_percpu(log->ptr);
1829			break;
1830		case KMEMLEAK_NOT_LEAK:
1831			kmemleak_not_leak(log->ptr);
1832			break;
1833		case KMEMLEAK_IGNORE:
1834			kmemleak_ignore(log->ptr);
1835			break;
1836		case KMEMLEAK_SCAN_AREA:
1837			kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
1838			break;
1839		case KMEMLEAK_NO_SCAN:
1840			kmemleak_no_scan(log->ptr);
1841			break;
1842		default:
1843			kmemleak_warn("Unknown early log operation: %d\n",
1844				      log->op_type);
1845		}
1846
1847		if (kmemleak_warning) {
1848			print_log_trace(log);
1849			kmemleak_warning = 0;
1850		}
1851	}
1852}
1853
1854/*
1855 * Late initialization function.
1856 */
1857static int __init kmemleak_late_init(void)
1858{
1859	struct dentry *dentry;
1860
1861	kmemleak_initialized = 1;
1862
1863	if (kmemleak_error) {
1864		/*
1865		 * Some error occurred and kmemleak was disabled. There is a
1866		 * small chance that kmemleak_disable() was called immediately
1867		 * after setting kmemleak_initialized and we may end up with
1868		 * two clean-up threads but serialized by scan_mutex.
1869		 */
1870		schedule_work(&cleanup_work);
1871		return -ENOMEM;
1872	}
1873
1874	dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
1875				     &kmemleak_fops);
1876	if (!dentry)
1877		pr_warning("Failed to create the debugfs kmemleak file\n");
1878	mutex_lock(&scan_mutex);
1879	start_scan_thread();
1880	mutex_unlock(&scan_mutex);
1881
1882	pr_info("Kernel memory leak detector initialized\n");
1883
1884	return 0;
1885}
1886late_initcall(kmemleak_late_init);