1/*
   2 * User-space Probes (UProbes)
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License as published by
   6 * the Free Software Foundation; either version 2 of the License, or
   7 * (at your option) any later version.
   8 *
   9 * This program is distributed in the hope that it will be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 *
  14 * You should have received a copy of the GNU General Public License
  15 * along with this program; if not, write to the Free Software
  16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17 *
  18 * Copyright (C) IBM Corporation, 2008-2012
  19 * Authors:
  20 *	Srikar Dronamraju
  21 *	Jim Keniston
  22 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  23 */
  24
  25#include <linux/kernel.h>
  26#include <linux/highmem.h>
  27#include <linux/pagemap.h>	/* read_mapping_page */
  28#include <linux/slab.h>
  29#include <linux/sched.h>
  30#include <linux/export.h>
  31#include <linux/rmap.h>		/* anon_vma_prepare */
  32#include <linux/mmu_notifier.h>	/* set_pte_at_notify */
  33#include <linux/swap.h>		/* try_to_free_swap */
  34#include <linux/ptrace.h>	/* user_enable_single_step */
  35#include <linux/kdebug.h>	/* notifier mechanism */
  36#include "../../mm/internal.h"	/* munlock_vma_page */
  37#include <linux/percpu-rwsem.h>
  38#include <linux/task_work.h>
 
  39
  40#include <linux/uprobes.h>
  41
  42#define UINSNS_PER_PAGE			(PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
  43#define MAX_UPROBE_XOL_SLOTS		UINSNS_PER_PAGE
  44
  45static struct rb_root uprobes_tree = RB_ROOT;
  46/*
  47 * allows us to skip the uprobe_mmap if there are no uprobe events active
  48 * at this time.  Probably a fine grained per inode count is better?
  49 */
  50#define no_uprobe_events()	RB_EMPTY_ROOT(&uprobes_tree)
  51
  52static DEFINE_SPINLOCK(uprobes_treelock);	/* serialize rbtree access */
  53
  54#define UPROBES_HASH_SZ	13
  55/* serialize uprobe->pending_list */
  56static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
  57#define uprobes_mmap_hash(v)	(&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
  58
  59static struct percpu_rw_semaphore dup_mmap_sem;
  60
  61/* Have a copy of original instruction */
  62#define UPROBE_COPY_INSN	0
  63/* Can skip singlestep */
  64#define UPROBE_SKIP_SSTEP	1
  65
  66struct uprobe {
  67	struct rb_node		rb_node;	/* node in the rb tree */
  68	atomic_t		ref;
  69	struct rw_semaphore	register_rwsem;
  70	struct rw_semaphore	consumer_rwsem;
  71	struct list_head	pending_list;
  72	struct uprobe_consumer	*consumers;
  73	struct inode		*inode;		/* Also hold a ref to inode */
  74	loff_t			offset;
  75	unsigned long		flags;
  76
  77	/*
  78	 * The generic code assumes that it has two members of unknown type
  79	 * owned by the arch-specific code:
  80	 *
  81	 * 	insn -	copy_insn() saves the original instruction here for
  82	 *		arch_uprobe_analyze_insn().
  83	 *
  84	 *	ixol -	potentially modified instruction to execute out of
  85	 *		line, copied to xol_area by xol_get_insn_slot().
  86	 */
  87	struct arch_uprobe	arch;
  88};
  89
  90struct return_instance {
  91	struct uprobe		*uprobe;
  92	unsigned long		func;
  93	unsigned long		orig_ret_vaddr; /* original return address */
  94	bool			chained;	/* true, if instance is nested */
  95
  96	struct return_instance	*next;		/* keep as stack */
  97};
  98
  99/*
 100 * Execute out of line area: anonymous executable mapping installed
 101 * by the probed task to execute the copy of the original instruction
 102 * mangled by set_swbp().
 103 *
 104 * On a breakpoint hit, thread contests for a slot.  It frees the
 105 * slot after singlestep. Currently a fixed number of slots are
 106 * allocated.
 107 */
 108struct xol_area {
 109	wait_queue_head_t 	wq;		/* if all slots are busy */
 110	atomic_t 		slot_count;	/* number of in-use slots */
 111	unsigned long 		*bitmap;	/* 0 = free slot */
 112	struct page 		*page;
 113
 
 
 114	/*
 115	 * We keep the vma's vm_start rather than a pointer to the vma
 116	 * itself.  The probed process or a naughty kernel module could make
 117	 * the vma go away, and we must handle that reasonably gracefully.
 118	 */
 119	unsigned long 		vaddr;		/* Page(s) of instruction slots */
 120};
 121
 122/*
 123 * valid_vma: Verify if the specified vma is an executable vma
 124 * Relax restrictions while unregistering: vm_flags might have
 125 * changed after breakpoint was inserted.
 126 *	- is_register: indicates if we are in register context.
 127 *	- Return 1 if the specified virtual address is in an
 128 *	  executable vma.
 129 */
 130static bool valid_vma(struct vm_area_struct *vma, bool is_register)
 131{
 132	vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_SHARED;
 133
 134	if (is_register)
 135		flags |= VM_WRITE;
 136
 137	return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
 138}
 139
 140static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
 141{
 142	return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
 143}
 144
 145static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
 146{
 147	return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
 148}
 149
 150/**
 151 * __replace_page - replace page in vma by new page.
 152 * based on replace_page in mm/ksm.c
 153 *
 154 * @vma:      vma that holds the pte pointing to page
 155 * @addr:     address the old @page is mapped at
 156 * @page:     the cowed page we are replacing by kpage
 157 * @kpage:    the modified page we replace page by
 158 *
 159 * Returns 0 on success, -EFAULT on failure.
 160 */
 161static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 162				struct page *page, struct page *kpage)
 163{
 164	struct mm_struct *mm = vma->vm_mm;
 165	spinlock_t *ptl;
 166	pte_t *ptep;
 167	int err;
 168	/* For mmu_notifiers */
 169	const unsigned long mmun_start = addr;
 170	const unsigned long mmun_end   = addr + PAGE_SIZE;
 
 
 
 
 
 
 171
 172	/* For try_to_free_swap() and munlock_vma_page() below */
 173	lock_page(page);
 174
 175	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
 176	err = -EAGAIN;
 177	ptep = page_check_address(page, mm, addr, &ptl, 0);
 178	if (!ptep)
 179		goto unlock;
 180
 181	get_page(kpage);
 182	page_add_new_anon_rmap(kpage, vma, addr);
 
 
 183
 184	if (!PageAnon(page)) {
 185		dec_mm_counter(mm, MM_FILEPAGES);
 186		inc_mm_counter(mm, MM_ANONPAGES);
 187	}
 188
 189	flush_cache_page(vma, addr, pte_pfn(*ptep));
 190	ptep_clear_flush(vma, addr, ptep);
 191	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
 192
 193	page_remove_rmap(page);
 194	if (!page_mapped(page))
 195		try_to_free_swap(page);
 196	pte_unmap_unlock(ptep, ptl);
 197
 198	if (vma->vm_flags & VM_LOCKED)
 199		munlock_vma_page(page);
 200	put_page(page);
 201
 202	err = 0;
 203 unlock:
 
 204	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
 205	unlock_page(page);
 206	return err;
 207}
 208
 209/**
 210 * is_swbp_insn - check if instruction is breakpoint instruction.
 211 * @insn: instruction to be checked.
 212 * Default implementation of is_swbp_insn
 213 * Returns true if @insn is a breakpoint instruction.
 214 */
 215bool __weak is_swbp_insn(uprobe_opcode_t *insn)
 216{
 217	return *insn == UPROBE_SWBP_INSN;
 218}
 219
 220/**
 221 * is_trap_insn - check if instruction is breakpoint instruction.
 222 * @insn: instruction to be checked.
 223 * Default implementation of is_trap_insn
 224 * Returns true if @insn is a breakpoint instruction.
 225 *
 226 * This function is needed for the case where an architecture has multiple
 227 * trap instructions (like powerpc).
 228 */
 229bool __weak is_trap_insn(uprobe_opcode_t *insn)
 230{
 231	return is_swbp_insn(insn);
 232}
 233
 234static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
 235{
 236	void *kaddr = kmap_atomic(page);
 237	memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
 238	kunmap_atomic(kaddr);
 239}
 240
 241static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
 242{
 243	void *kaddr = kmap_atomic(page);
 244	memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
 245	kunmap_atomic(kaddr);
 246}
 247
 248static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
 249{
 250	uprobe_opcode_t old_opcode;
 251	bool is_swbp;
 252
 253	/*
 254	 * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
 255	 * We do not check if it is any other 'trap variant' which could
 256	 * be conditional trap instruction such as the one powerpc supports.
 257	 *
 258	 * The logic is that we do not care if the underlying instruction
 259	 * is a trap variant; uprobes always wins over any other (gdb)
 260	 * breakpoint.
 261	 */
 262	copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
 263	is_swbp = is_swbp_insn(&old_opcode);
 264
 265	if (is_swbp_insn(new_opcode)) {
 266		if (is_swbp)		/* register: already installed? */
 267			return 0;
 268	} else {
 269		if (!is_swbp)		/* unregister: was it changed by us? */
 270			return 0;
 271	}
 272
 273	return 1;
 274}
 275
 276/*
 277 * NOTE:
 278 * Expect the breakpoint instruction to be the smallest size instruction for
 279 * the architecture. If an arch has variable length instruction and the
 280 * breakpoint instruction is not of the smallest length instruction
 281 * supported by that architecture then we need to modify is_trap_at_addr and
 282 * uprobe_write_opcode accordingly. This would never be a problem for archs
 283 * that have fixed length instructions.
 284 */
 285
 286/*
 287 * uprobe_write_opcode - write the opcode at a given virtual address.
 288 * @mm: the probed process address space.
 289 * @vaddr: the virtual address to store the opcode.
 290 * @opcode: opcode to be written at @vaddr.
 291 *
 292 * Called with mm->mmap_sem held (for read and with a reference to
 293 * mm).
 294 *
 295 * For mm @mm, write the opcode at @vaddr.
 296 * Return 0 (success) or a negative errno.
 297 */
 298int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
 299			uprobe_opcode_t opcode)
 300{
 301	struct page *old_page, *new_page;
 302	struct vm_area_struct *vma;
 303	int ret;
 304
 305retry:
 306	/* Read the page with vaddr into memory */
 307	ret = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &old_page, &vma);
 308	if (ret <= 0)
 309		return ret;
 310
 311	ret = verify_opcode(old_page, vaddr, &opcode);
 312	if (ret <= 0)
 313		goto put_old;
 314
 
 
 
 
 315	ret = -ENOMEM;
 316	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
 317	if (!new_page)
 318		goto put_old;
 319
 320	__SetPageUptodate(new_page);
 321
 322	copy_highpage(new_page, old_page);
 323	copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
 324
 325	ret = anon_vma_prepare(vma);
 326	if (ret)
 327		goto put_new;
 328
 329	ret = __replace_page(vma, vaddr, old_page, new_page);
 330
 331put_new:
 332	page_cache_release(new_page);
 333put_old:
 334	put_page(old_page);
 335
 336	if (unlikely(ret == -EAGAIN))
 337		goto retry;
 338	return ret;
 339}
 340
 341/**
 342 * set_swbp - store breakpoint at a given address.
 343 * @auprobe: arch specific probepoint information.
 344 * @mm: the probed process address space.
 345 * @vaddr: the virtual address to insert the opcode.
 346 *
 347 * For mm @mm, store the breakpoint instruction at @vaddr.
 348 * Return 0 (success) or a negative errno.
 349 */
 350int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
 351{
 352	return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN);
 353}
 354
 355/**
 356 * set_orig_insn - Restore the original instruction.
 357 * @mm: the probed process address space.
 358 * @auprobe: arch specific probepoint information.
 359 * @vaddr: the virtual address to insert the opcode.
 360 *
 361 * For mm @mm, restore the original opcode (opcode) at @vaddr.
 362 * Return 0 (success) or a negative errno.
 363 */
 364int __weak
 365set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
 366{
 367	return uprobe_write_opcode(mm, vaddr, *(uprobe_opcode_t *)&auprobe->insn);
 368}
 369
 
 
 
 
 
 
 
 
 
 
 
 
 370static int match_uprobe(struct uprobe *l, struct uprobe *r)
 371{
 372	if (l->inode < r->inode)
 373		return -1;
 374
 375	if (l->inode > r->inode)
 376		return 1;
 377
 378	if (l->offset < r->offset)
 379		return -1;
 380
 381	if (l->offset > r->offset)
 382		return 1;
 383
 384	return 0;
 385}
 386
 387static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
 388{
 389	struct uprobe u = { .inode = inode, .offset = offset };
 390	struct rb_node *n = uprobes_tree.rb_node;
 391	struct uprobe *uprobe;
 392	int match;
 393
 394	while (n) {
 395		uprobe = rb_entry(n, struct uprobe, rb_node);
 396		match = match_uprobe(&u, uprobe);
 397		if (!match) {
 398			atomic_inc(&uprobe->ref);
 399			return uprobe;
 400		}
 401
 402		if (match < 0)
 403			n = n->rb_left;
 404		else
 405			n = n->rb_right;
 406	}
 407	return NULL;
 408}
 409
 410/*
 411 * Find a uprobe corresponding to a given inode:offset
 412 * Acquires uprobes_treelock
 413 */
 414static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
 415{
 416	struct uprobe *uprobe;
 417
 418	spin_lock(&uprobes_treelock);
 419	uprobe = __find_uprobe(inode, offset);
 420	spin_unlock(&uprobes_treelock);
 421
 422	return uprobe;
 423}
 424
 425static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
 426{
 427	struct rb_node **p = &uprobes_tree.rb_node;
 428	struct rb_node *parent = NULL;
 429	struct uprobe *u;
 430	int match;
 431
 432	while (*p) {
 433		parent = *p;
 434		u = rb_entry(parent, struct uprobe, rb_node);
 435		match = match_uprobe(uprobe, u);
 436		if (!match) {
 437			atomic_inc(&u->ref);
 438			return u;
 439		}
 440
 441		if (match < 0)
 442			p = &parent->rb_left;
 443		else
 444			p = &parent->rb_right;
 445
 446	}
 447
 448	u = NULL;
 449	rb_link_node(&uprobe->rb_node, parent, p);
 450	rb_insert_color(&uprobe->rb_node, &uprobes_tree);
 451	/* get access + creation ref */
 452	atomic_set(&uprobe->ref, 2);
 453
 454	return u;
 455}
 456
 457/*
 458 * Acquire uprobes_treelock.
 459 * Matching uprobe already exists in rbtree;
 460 *	increment (access refcount) and return the matching uprobe.
 461 *
 462 * No matching uprobe; insert the uprobe in rb_tree;
 463 *	get a double refcount (access + creation) and return NULL.
 464 */
 465static struct uprobe *insert_uprobe(struct uprobe *uprobe)
 466{
 467	struct uprobe *u;
 468
 469	spin_lock(&uprobes_treelock);
 470	u = __insert_uprobe(uprobe);
 471	spin_unlock(&uprobes_treelock);
 472
 473	return u;
 474}
 475
 476static void put_uprobe(struct uprobe *uprobe)
 477{
 478	if (atomic_dec_and_test(&uprobe->ref))
 479		kfree(uprobe);
 480}
 481
 482static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
 483{
 484	struct uprobe *uprobe, *cur_uprobe;
 485
 486	uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
 487	if (!uprobe)
 488		return NULL;
 489
 490	uprobe->inode = igrab(inode);
 491	uprobe->offset = offset;
 492	init_rwsem(&uprobe->register_rwsem);
 493	init_rwsem(&uprobe->consumer_rwsem);
 494	/* For now assume that the instruction need not be single-stepped */
 495	__set_bit(UPROBE_SKIP_SSTEP, &uprobe->flags);
 496
 497	/* add to uprobes_tree, sorted on inode:offset */
 498	cur_uprobe = insert_uprobe(uprobe);
 499
 500	/* a uprobe exists for this inode:offset combination */
 501	if (cur_uprobe) {
 502		kfree(uprobe);
 503		uprobe = cur_uprobe;
 504		iput(inode);
 505	}
 506
 507	return uprobe;
 508}
 509
 510static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
 511{
 512	down_write(&uprobe->consumer_rwsem);
 513	uc->next = uprobe->consumers;
 514	uprobe->consumers = uc;
 515	up_write(&uprobe->consumer_rwsem);
 516}
 517
 518/*
 519 * For uprobe @uprobe, delete the consumer @uc.
 520 * Return true if the @uc is deleted successfully
 521 * or return false.
 522 */
 523static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
 524{
 525	struct uprobe_consumer **con;
 526	bool ret = false;
 527
 528	down_write(&uprobe->consumer_rwsem);
 529	for (con = &uprobe->consumers; *con; con = &(*con)->next) {
 530		if (*con == uc) {
 531			*con = uc->next;
 532			ret = true;
 533			break;
 534		}
 535	}
 536	up_write(&uprobe->consumer_rwsem);
 537
 538	return ret;
 539}
 540
 541static int __copy_insn(struct address_space *mapping, struct file *filp,
 542			void *insn, int nbytes, loff_t offset)
 543{
 544	struct page *page;
 545
 546	if (!mapping->a_ops->readpage)
 547		return -EIO;
 548	/*
 549	 * Ensure that the page that has the original instruction is
 550	 * populated and in page-cache.
 
 551	 */
 552	page = read_mapping_page(mapping, offset >> PAGE_CACHE_SHIFT, filp);
 
 
 
 553	if (IS_ERR(page))
 554		return PTR_ERR(page);
 555
 556	copy_from_page(page, offset, insn, nbytes);
 557	page_cache_release(page);
 558
 559	return 0;
 560}
 561
 562static int copy_insn(struct uprobe *uprobe, struct file *filp)
 563{
 564	struct address_space *mapping = uprobe->inode->i_mapping;
 565	loff_t offs = uprobe->offset;
 566	void *insn = &uprobe->arch.insn;
 567	int size = sizeof(uprobe->arch.insn);
 568	int len, err = -EIO;
 569
 570	/* Copy only available bytes, -EIO if nothing was read */
 571	do {
 572		if (offs >= i_size_read(uprobe->inode))
 573			break;
 574
 575		len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
 576		err = __copy_insn(mapping, filp, insn, len, offs);
 577		if (err)
 578			break;
 579
 580		insn += len;
 581		offs += len;
 582		size -= len;
 583	} while (size);
 584
 585	return err;
 586}
 587
 588static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
 589				struct mm_struct *mm, unsigned long vaddr)
 590{
 591	int ret = 0;
 592
 593	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
 594		return ret;
 595
 596	/* TODO: move this into _register, until then we abuse this sem. */
 597	down_write(&uprobe->consumer_rwsem);
 598	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
 599		goto out;
 600
 601	ret = copy_insn(uprobe, file);
 602	if (ret)
 603		goto out;
 604
 605	ret = -ENOTSUPP;
 606	if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
 607		goto out;
 608
 609	ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
 610	if (ret)
 611		goto out;
 612
 613	/* uprobe_write_opcode() assumes we don't cross page boundary */
 614	BUG_ON((uprobe->offset & ~PAGE_MASK) +
 615			UPROBE_SWBP_INSN_SIZE > PAGE_SIZE);
 616
 617	smp_wmb(); /* pairs with rmb() in find_active_uprobe() */
 618	set_bit(UPROBE_COPY_INSN, &uprobe->flags);
 619
 620 out:
 621	up_write(&uprobe->consumer_rwsem);
 622
 623	return ret;
 624}
 625
 626static inline bool consumer_filter(struct uprobe_consumer *uc,
 627				   enum uprobe_filter_ctx ctx, struct mm_struct *mm)
 628{
 629	return !uc->filter || uc->filter(uc, ctx, mm);
 630}
 631
 632static bool filter_chain(struct uprobe *uprobe,
 633			 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
 634{
 635	struct uprobe_consumer *uc;
 636	bool ret = false;
 637
 638	down_read(&uprobe->consumer_rwsem);
 639	for (uc = uprobe->consumers; uc; uc = uc->next) {
 640		ret = consumer_filter(uc, ctx, mm);
 641		if (ret)
 642			break;
 643	}
 644	up_read(&uprobe->consumer_rwsem);
 645
 646	return ret;
 647}
 648
 649static int
 650install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
 651			struct vm_area_struct *vma, unsigned long vaddr)
 652{
 653	bool first_uprobe;
 654	int ret;
 655
 656	ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
 657	if (ret)
 658		return ret;
 659
 660	/*
 661	 * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
 662	 * the task can hit this breakpoint right after __replace_page().
 663	 */
 664	first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
 665	if (first_uprobe)
 666		set_bit(MMF_HAS_UPROBES, &mm->flags);
 667
 668	ret = set_swbp(&uprobe->arch, mm, vaddr);
 669	if (!ret)
 670		clear_bit(MMF_RECALC_UPROBES, &mm->flags);
 671	else if (first_uprobe)
 672		clear_bit(MMF_HAS_UPROBES, &mm->flags);
 673
 674	return ret;
 675}
 676
 677static int
 678remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
 679{
 680	set_bit(MMF_RECALC_UPROBES, &mm->flags);
 681	return set_orig_insn(&uprobe->arch, mm, vaddr);
 682}
 683
 684static inline bool uprobe_is_active(struct uprobe *uprobe)
 685{
 686	return !RB_EMPTY_NODE(&uprobe->rb_node);
 687}
 688/*
 689 * There could be threads that have already hit the breakpoint. They
 690 * will recheck the current insn and restart if find_uprobe() fails.
 691 * See find_active_uprobe().
 692 */
 693static void delete_uprobe(struct uprobe *uprobe)
 694{
 695	if (WARN_ON(!uprobe_is_active(uprobe)))
 696		return;
 697
 698	spin_lock(&uprobes_treelock);
 699	rb_erase(&uprobe->rb_node, &uprobes_tree);
 700	spin_unlock(&uprobes_treelock);
 701	RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
 702	iput(uprobe->inode);
 703	put_uprobe(uprobe);
 704}
 705
 706struct map_info {
 707	struct map_info *next;
 708	struct mm_struct *mm;
 709	unsigned long vaddr;
 710};
 711
 712static inline struct map_info *free_map_info(struct map_info *info)
 713{
 714	struct map_info *next = info->next;
 715	kfree(info);
 716	return next;
 717}
 718
 719static struct map_info *
 720build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
 721{
 722	unsigned long pgoff = offset >> PAGE_SHIFT;
 723	struct vm_area_struct *vma;
 724	struct map_info *curr = NULL;
 725	struct map_info *prev = NULL;
 726	struct map_info *info;
 727	int more = 0;
 728
 729 again:
 730	mutex_lock(&mapping->i_mmap_mutex);
 731	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
 732		if (!valid_vma(vma, is_register))
 733			continue;
 734
 735		if (!prev && !more) {
 736			/*
 737			 * Needs GFP_NOWAIT to avoid i_mmap_mutex recursion through
 738			 * reclaim. This is optimistic, no harm done if it fails.
 739			 */
 740			prev = kmalloc(sizeof(struct map_info),
 741					GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
 742			if (prev)
 743				prev->next = NULL;
 744		}
 745		if (!prev) {
 746			more++;
 747			continue;
 748		}
 749
 750		if (!atomic_inc_not_zero(&vma->vm_mm->mm_users))
 751			continue;
 752
 753		info = prev;
 754		prev = prev->next;
 755		info->next = curr;
 756		curr = info;
 757
 758		info->mm = vma->vm_mm;
 759		info->vaddr = offset_to_vaddr(vma, offset);
 760	}
 761	mutex_unlock(&mapping->i_mmap_mutex);
 762
 763	if (!more)
 764		goto out;
 765
 766	prev = curr;
 767	while (curr) {
 768		mmput(curr->mm);
 769		curr = curr->next;
 770	}
 771
 772	do {
 773		info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
 774		if (!info) {
 775			curr = ERR_PTR(-ENOMEM);
 776			goto out;
 777		}
 778		info->next = prev;
 779		prev = info;
 780	} while (--more);
 781
 782	goto again;
 783 out:
 784	while (prev)
 785		prev = free_map_info(prev);
 786	return curr;
 787}
 788
 789static int
 790register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
 791{
 792	bool is_register = !!new;
 793	struct map_info *info;
 794	int err = 0;
 795
 796	percpu_down_write(&dup_mmap_sem);
 797	info = build_map_info(uprobe->inode->i_mapping,
 798					uprobe->offset, is_register);
 799	if (IS_ERR(info)) {
 800		err = PTR_ERR(info);
 801		goto out;
 802	}
 803
 804	while (info) {
 805		struct mm_struct *mm = info->mm;
 806		struct vm_area_struct *vma;
 807
 808		if (err && is_register)
 809			goto free;
 810
 811		down_write(&mm->mmap_sem);
 812		vma = find_vma(mm, info->vaddr);
 813		if (!vma || !valid_vma(vma, is_register) ||
 814		    file_inode(vma->vm_file) != uprobe->inode)
 815			goto unlock;
 816
 817		if (vma->vm_start > info->vaddr ||
 818		    vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
 819			goto unlock;
 820
 821		if (is_register) {
 822			/* consult only the "caller", new consumer. */
 823			if (consumer_filter(new,
 824					UPROBE_FILTER_REGISTER, mm))
 825				err = install_breakpoint(uprobe, mm, vma, info->vaddr);
 826		} else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
 827			if (!filter_chain(uprobe,
 828					UPROBE_FILTER_UNREGISTER, mm))
 829				err |= remove_breakpoint(uprobe, mm, info->vaddr);
 830		}
 831
 832 unlock:
 833		up_write(&mm->mmap_sem);
 834 free:
 835		mmput(mm);
 836		info = free_map_info(info);
 837	}
 838 out:
 839	percpu_up_write(&dup_mmap_sem);
 840	return err;
 841}
 842
 843static int __uprobe_register(struct uprobe *uprobe, struct uprobe_consumer *uc)
 844{
 845	consumer_add(uprobe, uc);
 846	return register_for_each_vma(uprobe, uc);
 847}
 848
 849static void __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)
 850{
 851	int err;
 852
 853	if (!consumer_del(uprobe, uc))	/* WARN? */
 854		return;
 855
 856	err = register_for_each_vma(uprobe, NULL);
 857	/* TODO : cant unregister? schedule a worker thread */
 858	if (!uprobe->consumers && !err)
 859		delete_uprobe(uprobe);
 860}
 861
 862/*
 863 * uprobe_register - register a probe
 864 * @inode: the file in which the probe has to be placed.
 865 * @offset: offset from the start of the file.
 866 * @uc: information on howto handle the probe..
 867 *
 868 * Apart from the access refcount, uprobe_register() takes a creation
 869 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
 870 * inserted into the rbtree (i.e first consumer for a @inode:@offset
 871 * tuple).  Creation refcount stops uprobe_unregister from freeing the
 872 * @uprobe even before the register operation is complete. Creation
 873 * refcount is released when the last @uc for the @uprobe
 874 * unregisters.
 875 *
 876 * Return errno if it cannot successully install probes
 877 * else return 0 (success)
 878 */
 879int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
 880{
 881	struct uprobe *uprobe;
 882	int ret;
 883
 884	/* Uprobe must have at least one set consumer */
 885	if (!uc->handler && !uc->ret_handler)
 886		return -EINVAL;
 887
 
 
 
 888	/* Racy, just to catch the obvious mistakes */
 889	if (offset > i_size_read(inode))
 890		return -EINVAL;
 891
 892 retry:
 893	uprobe = alloc_uprobe(inode, offset);
 894	if (!uprobe)
 895		return -ENOMEM;
 896	/*
 897	 * We can race with uprobe_unregister()->delete_uprobe().
 898	 * Check uprobe_is_active() and retry if it is false.
 899	 */
 900	down_write(&uprobe->register_rwsem);
 901	ret = -EAGAIN;
 902	if (likely(uprobe_is_active(uprobe))) {
 903		ret = __uprobe_register(uprobe, uc);
 904		if (ret)
 905			__uprobe_unregister(uprobe, uc);
 906	}
 907	up_write(&uprobe->register_rwsem);
 908	put_uprobe(uprobe);
 909
 910	if (unlikely(ret == -EAGAIN))
 911		goto retry;
 912	return ret;
 913}
 914EXPORT_SYMBOL_GPL(uprobe_register);
 915
 916/*
 917 * uprobe_apply - unregister a already registered probe.
 918 * @inode: the file in which the probe has to be removed.
 919 * @offset: offset from the start of the file.
 920 * @uc: consumer which wants to add more or remove some breakpoints
 921 * @add: add or remove the breakpoints
 922 */
 923int uprobe_apply(struct inode *inode, loff_t offset,
 924			struct uprobe_consumer *uc, bool add)
 925{
 926	struct uprobe *uprobe;
 927	struct uprobe_consumer *con;
 928	int ret = -ENOENT;
 929
 930	uprobe = find_uprobe(inode, offset);
 931	if (!uprobe)
 932		return ret;
 933
 934	down_write(&uprobe->register_rwsem);
 935	for (con = uprobe->consumers; con && con != uc ; con = con->next)
 936		;
 937	if (con)
 938		ret = register_for_each_vma(uprobe, add ? uc : NULL);
 939	up_write(&uprobe->register_rwsem);
 940	put_uprobe(uprobe);
 941
 942	return ret;
 943}
 944
 945/*
 946 * uprobe_unregister - unregister a already registered probe.
 947 * @inode: the file in which the probe has to be removed.
 948 * @offset: offset from the start of the file.
 949 * @uc: identify which probe if multiple probes are colocated.
 950 */
 951void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
 952{
 953	struct uprobe *uprobe;
 954
 955	uprobe = find_uprobe(inode, offset);
 956	if (!uprobe)
 957		return;
 958
 959	down_write(&uprobe->register_rwsem);
 960	__uprobe_unregister(uprobe, uc);
 961	up_write(&uprobe->register_rwsem);
 962	put_uprobe(uprobe);
 963}
 964EXPORT_SYMBOL_GPL(uprobe_unregister);
 965
 966static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
 967{
 968	struct vm_area_struct *vma;
 969	int err = 0;
 970
 971	down_read(&mm->mmap_sem);
 972	for (vma = mm->mmap; vma; vma = vma->vm_next) {
 973		unsigned long vaddr;
 974		loff_t offset;
 975
 976		if (!valid_vma(vma, false) ||
 977		    file_inode(vma->vm_file) != uprobe->inode)
 978			continue;
 979
 980		offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
 981		if (uprobe->offset <  offset ||
 982		    uprobe->offset >= offset + vma->vm_end - vma->vm_start)
 983			continue;
 984
 985		vaddr = offset_to_vaddr(vma, uprobe->offset);
 986		err |= remove_breakpoint(uprobe, mm, vaddr);
 987	}
 988	up_read(&mm->mmap_sem);
 989
 990	return err;
 991}
 992
 993static struct rb_node *
 994find_node_in_range(struct inode *inode, loff_t min, loff_t max)
 995{
 996	struct rb_node *n = uprobes_tree.rb_node;
 997
 998	while (n) {
 999		struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
1000
1001		if (inode < u->inode) {
1002			n = n->rb_left;
1003		} else if (inode > u->inode) {
1004			n = n->rb_right;
1005		} else {
1006			if (max < u->offset)
1007				n = n->rb_left;
1008			else if (min > u->offset)
1009				n = n->rb_right;
1010			else
1011				break;
1012		}
1013	}
1014
1015	return n;
1016}
1017
1018/*
1019 * For a given range in vma, build a list of probes that need to be inserted.
1020 */
1021static void build_probe_list(struct inode *inode,
1022				struct vm_area_struct *vma,
1023				unsigned long start, unsigned long end,
1024				struct list_head *head)
1025{
1026	loff_t min, max;
1027	struct rb_node *n, *t;
1028	struct uprobe *u;
1029
1030	INIT_LIST_HEAD(head);
1031	min = vaddr_to_offset(vma, start);
1032	max = min + (end - start) - 1;
1033
1034	spin_lock(&uprobes_treelock);
1035	n = find_node_in_range(inode, min, max);
1036	if (n) {
1037		for (t = n; t; t = rb_prev(t)) {
1038			u = rb_entry(t, struct uprobe, rb_node);
1039			if (u->inode != inode || u->offset < min)
1040				break;
1041			list_add(&u->pending_list, head);
1042			atomic_inc(&u->ref);
1043		}
1044		for (t = n; (t = rb_next(t)); ) {
1045			u = rb_entry(t, struct uprobe, rb_node);
1046			if (u->inode != inode || u->offset > max)
1047				break;
1048			list_add(&u->pending_list, head);
1049			atomic_inc(&u->ref);
1050		}
1051	}
1052	spin_unlock(&uprobes_treelock);
1053}
1054
1055/*
1056 * Called from mmap_region/vma_adjust with mm->mmap_sem acquired.
1057 *
1058 * Currently we ignore all errors and always return 0, the callers
1059 * can't handle the failure anyway.
1060 */
1061int uprobe_mmap(struct vm_area_struct *vma)
1062{
1063	struct list_head tmp_list;
1064	struct uprobe *uprobe, *u;
1065	struct inode *inode;
1066
1067	if (no_uprobe_events() || !valid_vma(vma, true))
1068		return 0;
1069
1070	inode = file_inode(vma->vm_file);
1071	if (!inode)
1072		return 0;
1073
1074	mutex_lock(uprobes_mmap_hash(inode));
1075	build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
1076	/*
1077	 * We can race with uprobe_unregister(), this uprobe can be already
1078	 * removed. But in this case filter_chain() must return false, all
1079	 * consumers have gone away.
1080	 */
1081	list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
1082		if (!fatal_signal_pending(current) &&
1083		    filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
1084			unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
1085			install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
1086		}
1087		put_uprobe(uprobe);
1088	}
1089	mutex_unlock(uprobes_mmap_hash(inode));
1090
1091	return 0;
1092}
1093
1094static bool
1095vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1096{
1097	loff_t min, max;
1098	struct inode *inode;
1099	struct rb_node *n;
1100
1101	inode = file_inode(vma->vm_file);
1102
1103	min = vaddr_to_offset(vma, start);
1104	max = min + (end - start) - 1;
1105
1106	spin_lock(&uprobes_treelock);
1107	n = find_node_in_range(inode, min, max);
1108	spin_unlock(&uprobes_treelock);
1109
1110	return !!n;
1111}
1112
1113/*
1114 * Called in context of a munmap of a vma.
1115 */
1116void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1117{
1118	if (no_uprobe_events() || !valid_vma(vma, false))
1119		return;
1120
1121	if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
1122		return;
1123
1124	if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
1125	     test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
1126		return;
1127
1128	if (vma_has_uprobes(vma, start, end))
1129		set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
1130}
1131
1132/* Slot allocation for XOL */
1133static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
1134{
1135	int ret = -EALREADY;
 
1136
1137	down_write(&mm->mmap_sem);
1138	if (mm->uprobes_state.xol_area)
 
1139		goto fail;
 
1140
1141	if (!area->vaddr) {
1142		/* Try to map as high as possible, this is only a hint. */
1143		area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
1144						PAGE_SIZE, 0, 0);
1145		if (area->vaddr & ~PAGE_MASK) {
1146			ret = area->vaddr;
1147			goto fail;
1148		}
1149	}
1150
1151	ret = install_special_mapping(mm, area->vaddr, PAGE_SIZE,
1152				VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, &area->page);
1153	if (ret)
 
 
1154		goto fail;
 
1155
 
1156	smp_wmb();	/* pairs with get_xol_area() */
1157	mm->uprobes_state.xol_area = area;
1158 fail:
1159	up_write(&mm->mmap_sem);
1160
1161	return ret;
1162}
1163
1164static struct xol_area *__create_xol_area(unsigned long vaddr)
1165{
1166	struct mm_struct *mm = current->mm;
1167	uprobe_opcode_t insn = UPROBE_SWBP_INSN;
1168	struct xol_area *area;
1169
1170	area = kmalloc(sizeof(*area), GFP_KERNEL);
1171	if (unlikely(!area))
1172		goto out;
1173
1174	area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL);
1175	if (!area->bitmap)
1176		goto free_area;
1177
1178	area->page = alloc_page(GFP_HIGHUSER);
1179	if (!area->page)
 
 
 
1180		goto free_bitmap;
 
1181
1182	area->vaddr = vaddr;
1183	init_waitqueue_head(&area->wq);
1184	/* Reserve the 1st slot for get_trampoline_vaddr() */
1185	set_bit(0, area->bitmap);
1186	atomic_set(&area->slot_count, 1);
1187	copy_to_page(area->page, 0, &insn, UPROBE_SWBP_INSN_SIZE);
1188
1189	if (!xol_add_vma(mm, area))
1190		return area;
1191
1192	__free_page(area->page);
1193 free_bitmap:
1194	kfree(area->bitmap);
1195 free_area:
1196	kfree(area);
1197 out:
1198	return NULL;
1199}
1200
1201/*
1202 * get_xol_area - Allocate process's xol_area if necessary.
1203 * This area will be used for storing instructions for execution out of line.
1204 *
1205 * Returns the allocated area or NULL.
1206 */
1207static struct xol_area *get_xol_area(void)
1208{
1209	struct mm_struct *mm = current->mm;
1210	struct xol_area *area;
1211
1212	if (!mm->uprobes_state.xol_area)
1213		__create_xol_area(0);
1214
1215	area = mm->uprobes_state.xol_area;
1216	smp_read_barrier_depends();	/* pairs with wmb in xol_add_vma() */
1217	return area;
1218}
1219
1220/*
1221 * uprobe_clear_state - Free the area allocated for slots.
1222 */
1223void uprobe_clear_state(struct mm_struct *mm)
1224{
1225	struct xol_area *area = mm->uprobes_state.xol_area;
1226
1227	if (!area)
1228		return;
1229
1230	put_page(area->page);
1231	kfree(area->bitmap);
1232	kfree(area);
1233}
1234
1235void uprobe_start_dup_mmap(void)
1236{
1237	percpu_down_read(&dup_mmap_sem);
1238}
1239
1240void uprobe_end_dup_mmap(void)
1241{
1242	percpu_up_read(&dup_mmap_sem);
1243}
1244
1245void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
1246{
1247	newmm->uprobes_state.xol_area = NULL;
1248
1249	if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
1250		set_bit(MMF_HAS_UPROBES, &newmm->flags);
1251		/* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
1252		set_bit(MMF_RECALC_UPROBES, &newmm->flags);
1253	}
1254}
1255
1256/*
1257 *  - search for a free slot.
1258 */
1259static unsigned long xol_take_insn_slot(struct xol_area *area)
1260{
1261	unsigned long slot_addr;
1262	int slot_nr;
1263
1264	do {
1265		slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1266		if (slot_nr < UINSNS_PER_PAGE) {
1267			if (!test_and_set_bit(slot_nr, area->bitmap))
1268				break;
1269
1270			slot_nr = UINSNS_PER_PAGE;
1271			continue;
1272		}
1273		wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
1274	} while (slot_nr >= UINSNS_PER_PAGE);
1275
1276	slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
1277	atomic_inc(&area->slot_count);
1278
1279	return slot_addr;
1280}
1281
1282/*
1283 * xol_get_insn_slot - allocate a slot for xol.
1284 * Returns the allocated slot address or 0.
1285 */
1286static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
1287{
1288	struct xol_area *area;
1289	unsigned long xol_vaddr;
1290
1291	area = get_xol_area();
1292	if (!area)
1293		return 0;
1294
1295	xol_vaddr = xol_take_insn_slot(area);
1296	if (unlikely(!xol_vaddr))
1297		return 0;
1298
1299	/* Initialize the slot */
1300	copy_to_page(area->page, xol_vaddr,
1301			&uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
1302	/*
1303	 * We probably need flush_icache_user_range() but it needs vma.
1304	 * This should work on supported architectures too.
1305	 */
1306	flush_dcache_page(area->page);
1307
1308	return xol_vaddr;
1309}
1310
1311/*
1312 * xol_free_insn_slot - If slot was earlier allocated by
1313 * @xol_get_insn_slot(), make the slot available for
1314 * subsequent requests.
1315 */
1316static void xol_free_insn_slot(struct task_struct *tsk)
1317{
1318	struct xol_area *area;
1319	unsigned long vma_end;
1320	unsigned long slot_addr;
1321
1322	if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
1323		return;
1324
1325	slot_addr = tsk->utask->xol_vaddr;
1326	if (unlikely(!slot_addr))
1327		return;
1328
1329	area = tsk->mm->uprobes_state.xol_area;
1330	vma_end = area->vaddr + PAGE_SIZE;
1331	if (area->vaddr <= slot_addr && slot_addr < vma_end) {
1332		unsigned long offset;
1333		int slot_nr;
1334
1335		offset = slot_addr - area->vaddr;
1336		slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
1337		if (slot_nr >= UINSNS_PER_PAGE)
1338			return;
1339
1340		clear_bit(slot_nr, area->bitmap);
1341		atomic_dec(&area->slot_count);
 
1342		if (waitqueue_active(&area->wq))
1343			wake_up(&area->wq);
1344
1345		tsk->utask->xol_vaddr = 0;
1346	}
1347}
1348
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1349/**
1350 * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
1351 * @regs: Reflects the saved state of the task after it has hit a breakpoint
1352 * instruction.
1353 * Return the address of the breakpoint instruction.
1354 */
1355unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
1356{
1357	return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
1358}
1359
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1360/*
1361 * Called with no locks held.
1362 * Called in context of a exiting or a exec-ing thread.
1363 */
1364void uprobe_free_utask(struct task_struct *t)
1365{
1366	struct uprobe_task *utask = t->utask;
1367	struct return_instance *ri, *tmp;
1368
1369	if (!utask)
1370		return;
1371
1372	if (utask->active_uprobe)
1373		put_uprobe(utask->active_uprobe);
1374
1375	ri = utask->return_instances;
1376	while (ri) {
1377		tmp = ri;
1378		ri = ri->next;
1379
1380		put_uprobe(tmp->uprobe);
1381		kfree(tmp);
1382	}
1383
1384	xol_free_insn_slot(t);
1385	kfree(utask);
1386	t->utask = NULL;
1387}
1388
1389/*
1390 * Allocate a uprobe_task object for the task if if necessary.
1391 * Called when the thread hits a breakpoint.
1392 *
1393 * Returns:
1394 * - pointer to new uprobe_task on success
1395 * - NULL otherwise
1396 */
1397static struct uprobe_task *get_utask(void)
1398{
1399	if (!current->utask)
1400		current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1401	return current->utask;
1402}
1403
1404static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
1405{
1406	struct uprobe_task *n_utask;
1407	struct return_instance **p, *o, *n;
1408
1409	n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1410	if (!n_utask)
1411		return -ENOMEM;
1412	t->utask = n_utask;
1413
1414	p = &n_utask->return_instances;
1415	for (o = o_utask->return_instances; o; o = o->next) {
1416		n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1417		if (!n)
1418			return -ENOMEM;
1419
1420		*n = *o;
1421		atomic_inc(&n->uprobe->ref);
1422		n->next = NULL;
1423
1424		*p = n;
1425		p = &n->next;
1426		n_utask->depth++;
1427	}
1428
1429	return 0;
1430}
1431
1432static void uprobe_warn(struct task_struct *t, const char *msg)
1433{
1434	pr_warn("uprobe: %s:%d failed to %s\n",
1435			current->comm, current->pid, msg);
1436}
1437
1438static void dup_xol_work(struct callback_head *work)
1439{
1440	if (current->flags & PF_EXITING)
1441		return;
1442
1443	if (!__create_xol_area(current->utask->dup_xol_addr))
1444		uprobe_warn(current, "dup xol area");
1445}
1446
1447/*
1448 * Called in context of a new clone/fork from copy_process.
1449 */
1450void uprobe_copy_process(struct task_struct *t, unsigned long flags)
1451{
1452	struct uprobe_task *utask = current->utask;
1453	struct mm_struct *mm = current->mm;
1454	struct xol_area *area;
1455
1456	t->utask = NULL;
1457
1458	if (!utask || !utask->return_instances)
1459		return;
1460
1461	if (mm == t->mm && !(flags & CLONE_VFORK))
1462		return;
1463
1464	if (dup_utask(t, utask))
1465		return uprobe_warn(t, "dup ret instances");
1466
1467	/* The task can fork() after dup_xol_work() fails */
1468	area = mm->uprobes_state.xol_area;
1469	if (!area)
1470		return uprobe_warn(t, "dup xol area");
1471
1472	if (mm == t->mm)
1473		return;
1474
1475	t->utask->dup_xol_addr = area->vaddr;
1476	init_task_work(&t->utask->dup_xol_work, dup_xol_work);
1477	task_work_add(t, &t->utask->dup_xol_work, true);
1478}
1479
1480/*
1481 * Current area->vaddr notion assume the trampoline address is always
1482 * equal area->vaddr.
1483 *
1484 * Returns -1 in case the xol_area is not allocated.
1485 */
1486static unsigned long get_trampoline_vaddr(void)
1487{
1488	struct xol_area *area;
1489	unsigned long trampoline_vaddr = -1;
1490
1491	area = current->mm->uprobes_state.xol_area;
1492	smp_read_barrier_depends();
1493	if (area)
1494		trampoline_vaddr = area->vaddr;
1495
1496	return trampoline_vaddr;
1497}
1498
 
 
 
 
 
 
 
 
 
 
 
 
 
1499static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
1500{
1501	struct return_instance *ri;
1502	struct uprobe_task *utask;
1503	unsigned long orig_ret_vaddr, trampoline_vaddr;
1504	bool chained = false;
1505
1506	if (!get_xol_area())
1507		return;
1508
1509	utask = get_utask();
1510	if (!utask)
1511		return;
1512
1513	if (utask->depth >= MAX_URETPROBE_DEPTH) {
1514		printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
1515				" nestedness limit pid/tgid=%d/%d\n",
1516				current->pid, current->tgid);
1517		return;
1518	}
1519
1520	ri = kzalloc(sizeof(struct return_instance), GFP_KERNEL);
1521	if (!ri)
1522		goto fail;
1523
1524	trampoline_vaddr = get_trampoline_vaddr();
1525	orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
1526	if (orig_ret_vaddr == -1)
1527		goto fail;
1528
 
 
 
 
1529	/*
1530	 * We don't want to keep trampoline address in stack, rather keep the
1531	 * original return address of first caller thru all the consequent
1532	 * instances. This also makes breakpoint unwrapping easier.
1533	 */
1534	if (orig_ret_vaddr == trampoline_vaddr) {
1535		if (!utask->return_instances) {
1536			/*
1537			 * This situation is not possible. Likely we have an
1538			 * attack from user-space.
1539			 */
1540			pr_warn("uprobe: unable to set uretprobe pid/tgid=%d/%d\n",
1541						current->pid, current->tgid);
1542			goto fail;
1543		}
1544
1545		chained = true;
1546		orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
1547	}
1548
1549	atomic_inc(&uprobe->ref);
1550	ri->uprobe = uprobe;
1551	ri->func = instruction_pointer(regs);
 
1552	ri->orig_ret_vaddr = orig_ret_vaddr;
1553	ri->chained = chained;
1554
1555	utask->depth++;
1556
1557	/* add instance to the stack */
1558	ri->next = utask->return_instances;
1559	utask->return_instances = ri;
1560
1561	return;
1562
1563 fail:
1564	kfree(ri);
1565}
1566
1567/* Prepare to single-step probed instruction out of line. */
1568static int
1569pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
1570{
1571	struct uprobe_task *utask;
1572	unsigned long xol_vaddr;
1573	int err;
1574
1575	utask = get_utask();
1576	if (!utask)
1577		return -ENOMEM;
1578
1579	xol_vaddr = xol_get_insn_slot(uprobe);
1580	if (!xol_vaddr)
1581		return -ENOMEM;
1582
1583	utask->xol_vaddr = xol_vaddr;
1584	utask->vaddr = bp_vaddr;
1585
1586	err = arch_uprobe_pre_xol(&uprobe->arch, regs);
1587	if (unlikely(err)) {
1588		xol_free_insn_slot(current);
1589		return err;
1590	}
1591
1592	utask->active_uprobe = uprobe;
1593	utask->state = UTASK_SSTEP;
1594	return 0;
1595}
1596
1597/*
1598 * If we are singlestepping, then ensure this thread is not connected to
1599 * non-fatal signals until completion of singlestep.  When xol insn itself
1600 * triggers the signal,  restart the original insn even if the task is
1601 * already SIGKILL'ed (since coredump should report the correct ip).  This
1602 * is even more important if the task has a handler for SIGSEGV/etc, The
1603 * _same_ instruction should be repeated again after return from the signal
1604 * handler, and SSTEP can never finish in this case.
1605 */
1606bool uprobe_deny_signal(void)
1607{
1608	struct task_struct *t = current;
1609	struct uprobe_task *utask = t->utask;
1610
1611	if (likely(!utask || !utask->active_uprobe))
1612		return false;
1613
1614	WARN_ON_ONCE(utask->state != UTASK_SSTEP);
1615
1616	if (signal_pending(t)) {
1617		spin_lock_irq(&t->sighand->siglock);
1618		clear_tsk_thread_flag(t, TIF_SIGPENDING);
1619		spin_unlock_irq(&t->sighand->siglock);
1620
1621		if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
1622			utask->state = UTASK_SSTEP_TRAPPED;
1623			set_tsk_thread_flag(t, TIF_UPROBE);
1624			set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
1625		}
1626	}
1627
1628	return true;
1629}
1630
1631/*
1632 * Avoid singlestepping the original instruction if the original instruction
1633 * is a NOP or can be emulated.
1634 */
1635static bool can_skip_sstep(struct uprobe *uprobe, struct pt_regs *regs)
1636{
1637	if (test_bit(UPROBE_SKIP_SSTEP, &uprobe->flags)) {
1638		if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
1639			return true;
1640		clear_bit(UPROBE_SKIP_SSTEP, &uprobe->flags);
1641	}
1642	return false;
1643}
1644
1645static void mmf_recalc_uprobes(struct mm_struct *mm)
1646{
1647	struct vm_area_struct *vma;
1648
1649	for (vma = mm->mmap; vma; vma = vma->vm_next) {
1650		if (!valid_vma(vma, false))
1651			continue;
1652		/*
1653		 * This is not strictly accurate, we can race with
1654		 * uprobe_unregister() and see the already removed
1655		 * uprobe if delete_uprobe() was not yet called.
1656		 * Or this uprobe can be filtered out.
1657		 */
1658		if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
1659			return;
1660	}
1661
1662	clear_bit(MMF_HAS_UPROBES, &mm->flags);
1663}
1664
1665static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
1666{
1667	struct page *page;
1668	uprobe_opcode_t opcode;
1669	int result;
1670
1671	pagefault_disable();
1672	result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr,
1673							sizeof(opcode));
1674	pagefault_enable();
1675
1676	if (likely(result == 0))
1677		goto out;
1678
1679	result = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &page, NULL);
 
 
 
 
 
 
1680	if (result < 0)
1681		return result;
1682
1683	copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
1684	put_page(page);
1685 out:
1686	/* This needs to return true for any variant of the trap insn */
1687	return is_trap_insn(&opcode);
1688}
1689
1690static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
1691{
1692	struct mm_struct *mm = current->mm;
1693	struct uprobe *uprobe = NULL;
1694	struct vm_area_struct *vma;
1695
1696	down_read(&mm->mmap_sem);
1697	vma = find_vma(mm, bp_vaddr);
1698	if (vma && vma->vm_start <= bp_vaddr) {
1699		if (valid_vma(vma, false)) {
1700			struct inode *inode = file_inode(vma->vm_file);
1701			loff_t offset = vaddr_to_offset(vma, bp_vaddr);
1702
1703			uprobe = find_uprobe(inode, offset);
1704		}
1705
1706		if (!uprobe)
1707			*is_swbp = is_trap_at_addr(mm, bp_vaddr);
1708	} else {
1709		*is_swbp = -EFAULT;
1710	}
1711
1712	if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
1713		mmf_recalc_uprobes(mm);
1714	up_read(&mm->mmap_sem);
1715
1716	return uprobe;
1717}
1718
1719static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
1720{
1721	struct uprobe_consumer *uc;
1722	int remove = UPROBE_HANDLER_REMOVE;
1723	bool need_prep = false; /* prepare return uprobe, when needed */
1724
1725	down_read(&uprobe->register_rwsem);
1726	for (uc = uprobe->consumers; uc; uc = uc->next) {
1727		int rc = 0;
1728
1729		if (uc->handler) {
1730			rc = uc->handler(uc, regs);
1731			WARN(rc & ~UPROBE_HANDLER_MASK,
1732				"bad rc=0x%x from %pf()\n", rc, uc->handler);
1733		}
1734
1735		if (uc->ret_handler)
1736			need_prep = true;
1737
1738		remove &= rc;
1739	}
1740
1741	if (need_prep && !remove)
1742		prepare_uretprobe(uprobe, regs); /* put bp at return */
1743
1744	if (remove && uprobe->consumers) {
1745		WARN_ON(!uprobe_is_active(uprobe));
1746		unapply_uprobe(uprobe, current->mm);
1747	}
1748	up_read(&uprobe->register_rwsem);
1749}
1750
1751static void
1752handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
1753{
1754	struct uprobe *uprobe = ri->uprobe;
1755	struct uprobe_consumer *uc;
1756
1757	down_read(&uprobe->register_rwsem);
1758	for (uc = uprobe->consumers; uc; uc = uc->next) {
1759		if (uc->ret_handler)
1760			uc->ret_handler(uc, ri->func, regs);
1761	}
1762	up_read(&uprobe->register_rwsem);
1763}
1764
1765static bool handle_trampoline(struct pt_regs *regs)
1766{
1767	struct uprobe_task *utask;
1768	struct return_instance *ri, *tmp;
1769	bool chained;
1770
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1771	utask = current->utask;
1772	if (!utask)
1773		return false;
1774
1775	ri = utask->return_instances;
1776	if (!ri)
1777		return false;
1778
1779	/*
1780	 * TODO: we should throw out return_instance's invalidated by
1781	 * longjmp(), currently we assume that the probed function always
1782	 * returns.
1783	 */
1784	instruction_pointer_set(regs, ri->orig_ret_vaddr);
1785
1786	for (;;) {
1787		handle_uretprobe_chain(ri, regs);
1788
1789		chained = ri->chained;
1790		put_uprobe(ri->uprobe);
1791
1792		tmp = ri;
1793		ri = ri->next;
1794		kfree(tmp);
1795		utask->depth--;
1796
1797		if (!chained)
1798			break;
1799		BUG_ON(!ri);
1800	}
 
 
 
 
1801
1802	utask->return_instances = ri;
 
 
 
 
 
1803
1804	return true;
1805}
1806
1807bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
1808{
1809	return false;
1810}
1811
 
 
 
 
 
 
1812/*
1813 * Run handler and ask thread to singlestep.
1814 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
1815 */
1816static void handle_swbp(struct pt_regs *regs)
1817{
1818	struct uprobe *uprobe;
1819	unsigned long bp_vaddr;
1820	int uninitialized_var(is_swbp);
1821
1822	bp_vaddr = uprobe_get_swbp_addr(regs);
1823	if (bp_vaddr == get_trampoline_vaddr()) {
1824		if (handle_trampoline(regs))
1825			return;
1826
1827		pr_warn("uprobe: unable to handle uretprobe pid/tgid=%d/%d\n",
1828						current->pid, current->tgid);
1829	}
1830
1831	uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
1832	if (!uprobe) {
1833		if (is_swbp > 0) {
1834			/* No matching uprobe; signal SIGTRAP. */
1835			send_sig(SIGTRAP, current, 0);
1836		} else {
1837			/*
1838			 * Either we raced with uprobe_unregister() or we can't
1839			 * access this memory. The latter is only possible if
1840			 * another thread plays with our ->mm. In both cases
1841			 * we can simply restart. If this vma was unmapped we
1842			 * can pretend this insn was not executed yet and get
1843			 * the (correct) SIGSEGV after restart.
1844			 */
1845			instruction_pointer_set(regs, bp_vaddr);
1846		}
1847		return;
1848	}
1849
1850	/* change it in advance for ->handler() and restart */
1851	instruction_pointer_set(regs, bp_vaddr);
1852
1853	/*
1854	 * TODO: move copy_insn/etc into _register and remove this hack.
1855	 * After we hit the bp, _unregister + _register can install the
1856	 * new and not-yet-analyzed uprobe at the same address, restart.
1857	 */
1858	smp_rmb(); /* pairs with wmb() in install_breakpoint() */
1859	if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
1860		goto out;
1861
1862	/* Tracing handlers use ->utask to communicate with fetch methods */
1863	if (!get_utask())
1864		goto out;
1865
1866	if (arch_uprobe_ignore(&uprobe->arch, regs))
1867		goto out;
1868
1869	handler_chain(uprobe, regs);
1870
1871	if (can_skip_sstep(uprobe, regs))
1872		goto out;
1873
1874	if (!pre_ssout(uprobe, regs, bp_vaddr))
1875		return;
1876
1877	/* can_skip_sstep() succeeded, or restart if can't singlestep */
1878out:
1879	put_uprobe(uprobe);
1880}
1881
1882/*
1883 * Perform required fix-ups and disable singlestep.
1884 * Allow pending signals to take effect.
1885 */
1886static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
1887{
1888	struct uprobe *uprobe;
 
1889
1890	uprobe = utask->active_uprobe;
1891	if (utask->state == UTASK_SSTEP_ACK)
1892		arch_uprobe_post_xol(&uprobe->arch, regs);
1893	else if (utask->state == UTASK_SSTEP_TRAPPED)
1894		arch_uprobe_abort_xol(&uprobe->arch, regs);
1895	else
1896		WARN_ON_ONCE(1);
1897
1898	put_uprobe(uprobe);
1899	utask->active_uprobe = NULL;
1900	utask->state = UTASK_RUNNING;
1901	xol_free_insn_slot(current);
1902
1903	spin_lock_irq(&current->sighand->siglock);
1904	recalc_sigpending(); /* see uprobe_deny_signal() */
1905	spin_unlock_irq(&current->sighand->siglock);
 
 
 
 
 
1906}
1907
1908/*
1909 * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
1910 * allows the thread to return from interrupt. After that handle_swbp()
1911 * sets utask->active_uprobe.
1912 *
1913 * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
1914 * and allows the thread to return from interrupt.
1915 *
1916 * While returning to userspace, thread notices the TIF_UPROBE flag and calls
1917 * uprobe_notify_resume().
1918 */
1919void uprobe_notify_resume(struct pt_regs *regs)
1920{
1921	struct uprobe_task *utask;
1922
1923	clear_thread_flag(TIF_UPROBE);
1924
1925	utask = current->utask;
1926	if (utask && utask->active_uprobe)
1927		handle_singlestep(utask, regs);
1928	else
1929		handle_swbp(regs);
1930}
1931
1932/*
1933 * uprobe_pre_sstep_notifier gets called from interrupt context as part of
1934 * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
1935 */
1936int uprobe_pre_sstep_notifier(struct pt_regs *regs)
1937{
1938	if (!current->mm)
1939		return 0;
1940
1941	if (!test_bit(MMF_HAS_UPROBES, &current->mm->flags) &&
1942	    (!current->utask || !current->utask->return_instances))
1943		return 0;
1944
1945	set_thread_flag(TIF_UPROBE);
1946	return 1;
1947}
1948
1949/*
1950 * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
1951 * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
1952 */
1953int uprobe_post_sstep_notifier(struct pt_regs *regs)
1954{
1955	struct uprobe_task *utask = current->utask;
1956
1957	if (!current->mm || !utask || !utask->active_uprobe)
1958		/* task is currently not uprobed */
1959		return 0;
1960
1961	utask->state = UTASK_SSTEP_ACK;
1962	set_thread_flag(TIF_UPROBE);
1963	return 1;
1964}
1965
1966static struct notifier_block uprobe_exception_nb = {
1967	.notifier_call		= arch_uprobe_exception_notify,
1968	.priority		= INT_MAX-1,	/* notified after kprobes, kgdb */
1969};
1970
1971static int __init init_uprobes(void)
1972{
1973	int i;
1974
1975	for (i = 0; i < UPROBES_HASH_SZ; i++)
1976		mutex_init(&uprobes_mmap_mutex[i]);
1977
1978	if (percpu_init_rwsem(&dup_mmap_sem))
1979		return -ENOMEM;
1980
1981	return register_die_notifier(&uprobe_exception_nb);
1982}
1983__initcall(init_uprobes);