1/*
   2 * User-space Probes (UProbes)
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License as published by
   6 * the Free Software Foundation; either version 2 of the License, or
   7 * (at your option) any later version.
   8 *
   9 * This program is distributed in the hope that it will be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 *
  14 * You should have received a copy of the GNU General Public License
  15 * along with this program; if not, write to the Free Software
  16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17 *
  18 * Copyright (C) IBM Corporation, 2008-2012
  19 * Authors:
  20 *	Srikar Dronamraju
  21 *	Jim Keniston
  22 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
  23 */
  24
  25#include <linux/kernel.h>
  26#include <linux/highmem.h>
  27#include <linux/pagemap.h>	/* read_mapping_page */
  28#include <linux/slab.h>
  29#include <linux/sched.h>
 
 
  30#include <linux/export.h>
  31#include <linux/rmap.h>		/* anon_vma_prepare */
  32#include <linux/mmu_notifier.h>	/* set_pte_at_notify */
  33#include <linux/swap.h>		/* try_to_free_swap */
  34#include <linux/ptrace.h>	/* user_enable_single_step */
  35#include <linux/kdebug.h>	/* notifier mechanism */
  36#include "../../mm/internal.h"	/* munlock_vma_page */
  37#include <linux/percpu-rwsem.h>
  38#include <linux/task_work.h>
  39#include <linux/shmem_fs.h>
 
  40
  41#include <linux/uprobes.h>
  42
  43#define UINSNS_PER_PAGE			(PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
  44#define MAX_UPROBE_XOL_SLOTS		UINSNS_PER_PAGE
  45
  46static struct rb_root uprobes_tree = RB_ROOT;
  47/*
  48 * allows us to skip the uprobe_mmap if there are no uprobe events active
  49 * at this time.  Probably a fine grained per inode count is better?
  50 */
  51#define no_uprobe_events()	RB_EMPTY_ROOT(&uprobes_tree)
  52
  53static DEFINE_SPINLOCK(uprobes_treelock);	/* serialize rbtree access */
  54
  55#define UPROBES_HASH_SZ	13
  56/* serialize uprobe->pending_list */
  57static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
  58#define uprobes_mmap_hash(v)	(&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
  59
  60static struct percpu_rw_semaphore dup_mmap_sem;
  61
  62/* Have a copy of original instruction */
  63#define UPROBE_COPY_INSN	0
  64
  65struct uprobe {
  66	struct rb_node		rb_node;	/* node in the rb tree */
  67	atomic_t		ref;
  68	struct rw_semaphore	register_rwsem;
  69	struct rw_semaphore	consumer_rwsem;
  70	struct list_head	pending_list;
  71	struct uprobe_consumer	*consumers;
  72	struct inode		*inode;		/* Also hold a ref to inode */
  73	loff_t			offset;
 
  74	unsigned long		flags;
  75
  76	/*
  77	 * The generic code assumes that it has two members of unknown type
  78	 * owned by the arch-specific code:
  79	 *
  80	 * 	insn -	copy_insn() saves the original instruction here for
  81	 *		arch_uprobe_analyze_insn().
  82	 *
  83	 *	ixol -	potentially modified instruction to execute out of
  84	 *		line, copied to xol_area by xol_get_insn_slot().
  85	 */
  86	struct arch_uprobe	arch;
  87};
  88
 
 
 
 
 
 
 
 
 
  89/*
  90 * Execute out of line area: anonymous executable mapping installed
  91 * by the probed task to execute the copy of the original instruction
  92 * mangled by set_swbp().
  93 *
  94 * On a breakpoint hit, thread contests for a slot.  It frees the
  95 * slot after singlestep. Currently a fixed number of slots are
  96 * allocated.
  97 */
  98struct xol_area {
  99	wait_queue_head_t 		wq;		/* if all slots are busy */
 100	atomic_t 			slot_count;	/* number of in-use slots */
 101	unsigned long 			*bitmap;	/* 0 = free slot */
 102
 103	struct vm_special_mapping	xol_mapping;
 104	struct page 			*pages[2];
 105	/*
 106	 * We keep the vma's vm_start rather than a pointer to the vma
 107	 * itself.  The probed process or a naughty kernel module could make
 108	 * the vma go away, and we must handle that reasonably gracefully.
 109	 */
 110	unsigned long 			vaddr;		/* Page(s) of instruction slots */
 111};
 112
 113/*
 114 * valid_vma: Verify if the specified vma is an executable vma
 115 * Relax restrictions while unregistering: vm_flags might have
 116 * changed after breakpoint was inserted.
 117 *	- is_register: indicates if we are in register context.
 118 *	- Return 1 if the specified virtual address is in an
 119 *	  executable vma.
 120 */
 121static bool valid_vma(struct vm_area_struct *vma, bool is_register)
 122{
 123	vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
 124
 125	if (is_register)
 126		flags |= VM_WRITE;
 127
 128	return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
 129}
 130
 131static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
 132{
 133	return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
 134}
 135
 136static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
 137{
 138	return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
 139}
 140
 141/**
 142 * __replace_page - replace page in vma by new page.
 143 * based on replace_page in mm/ksm.c
 144 *
 145 * @vma:      vma that holds the pte pointing to page
 146 * @addr:     address the old @page is mapped at
 147 * @page:     the cowed page we are replacing by kpage
 148 * @kpage:    the modified page we replace page by
 149 *
 150 * Returns 0 on success, -EFAULT on failure.
 
 
 151 */
 152static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 153				struct page *page, struct page *kpage)
 154{
 155	struct mm_struct *mm = vma->vm_mm;
 156	spinlock_t *ptl;
 157	pte_t *ptep;
 
 
 
 158	int err;
 159	/* For mmu_notifiers */
 160	const unsigned long mmun_start = addr;
 161	const unsigned long mmun_end   = addr + PAGE_SIZE;
 162	struct mem_cgroup *memcg;
 163
 164	err = mem_cgroup_try_charge(kpage, vma->vm_mm, GFP_KERNEL, &memcg,
 165			false);
 166	if (err)
 167		return err;
 
 168
 169	/* For try_to_free_swap() and munlock_vma_page() below */
 170	lock_page(page);
 171
 172	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
 173	err = -EAGAIN;
 174	ptep = page_check_address(page, mm, addr, &ptl, 0);
 175	if (!ptep)
 176		goto unlock;
 
 177
 178	get_page(kpage);
 179	page_add_new_anon_rmap(kpage, vma, addr, false);
 180	mem_cgroup_commit_charge(kpage, memcg, false, false);
 181	lru_cache_add_active_or_unevictable(kpage, vma);
 
 
 
 182
 183	if (!PageAnon(page)) {
 184		dec_mm_counter(mm, mm_counter_file(page));
 185		inc_mm_counter(mm, MM_ANONPAGES);
 186	}
 187
 188	flush_cache_page(vma, addr, pte_pfn(*ptep));
 189	ptep_clear_flush_notify(vma, addr, ptep);
 190	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
 191
 192	page_remove_rmap(page, false);
 193	if (!page_mapped(page))
 194		try_to_free_swap(page);
 195	pte_unmap_unlock(ptep, ptl);
 
 
 196
 197	if (vma->vm_flags & VM_LOCKED)
 198		munlock_vma_page(page);
 199	put_page(page);
 200
 201	err = 0;
 202 unlock:
 203	mem_cgroup_cancel_charge(kpage, memcg, false);
 204	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
 205	unlock_page(page);
 206	return err;
 207}
 208
 209/**
 210 * is_swbp_insn - check if instruction is breakpoint instruction.
 211 * @insn: instruction to be checked.
 212 * Default implementation of is_swbp_insn
 213 * Returns true if @insn is a breakpoint instruction.
 214 */
 215bool __weak is_swbp_insn(uprobe_opcode_t *insn)
 216{
 217	return *insn == UPROBE_SWBP_INSN;
 218}
 219
 220/**
 221 * is_trap_insn - check if instruction is breakpoint instruction.
 222 * @insn: instruction to be checked.
 223 * Default implementation of is_trap_insn
 224 * Returns true if @insn is a breakpoint instruction.
 225 *
 226 * This function is needed for the case where an architecture has multiple
 227 * trap instructions (like powerpc).
 228 */
 229bool __weak is_trap_insn(uprobe_opcode_t *insn)
 230{
 231	return is_swbp_insn(insn);
 232}
 233
 234static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
 235{
 236	void *kaddr = kmap_atomic(page);
 237	memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
 238	kunmap_atomic(kaddr);
 239}
 240
 241static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
 242{
 243	void *kaddr = kmap_atomic(page);
 244	memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
 245	kunmap_atomic(kaddr);
 246}
 247
 248static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
 249{
 250	uprobe_opcode_t old_opcode;
 251	bool is_swbp;
 252
 253	/*
 254	 * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
 255	 * We do not check if it is any other 'trap variant' which could
 256	 * be conditional trap instruction such as the one powerpc supports.
 257	 *
 258	 * The logic is that we do not care if the underlying instruction
 259	 * is a trap variant; uprobes always wins over any other (gdb)
 260	 * breakpoint.
 261	 */
 262	copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
 263	is_swbp = is_swbp_insn(&old_opcode);
 264
 265	if (is_swbp_insn(new_opcode)) {
 266		if (is_swbp)		/* register: already installed? */
 267			return 0;
 268	} else {
 269		if (!is_swbp)		/* unregister: was it changed by us? */
 270			return 0;
 271	}
 272
 273	return 1;
 274}
 275
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 276/*
 277 * NOTE:
 278 * Expect the breakpoint instruction to be the smallest size instruction for
 279 * the architecture. If an arch has variable length instruction and the
 280 * breakpoint instruction is not of the smallest length instruction
 281 * supported by that architecture then we need to modify is_trap_at_addr and
 282 * uprobe_write_opcode accordingly. This would never be a problem for archs
 283 * that have fixed length instructions.
 284 *
 285 * uprobe_write_opcode - write the opcode at a given virtual address.
 
 286 * @mm: the probed process address space.
 287 * @vaddr: the virtual address to store the opcode.
 288 * @opcode: opcode to be written at @vaddr.
 289 *
 290 * Called with mm->mmap_sem held for write.
 291 * Return 0 (success) or a negative errno.
 292 */
 293int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
 294			uprobe_opcode_t opcode)
 295{
 
 296	struct page *old_page, *new_page;
 297	struct vm_area_struct *vma;
 298	int ret;
 
 
 
 
 
 299
 300retry:
 
 
 301	/* Read the page with vaddr into memory */
 302	ret = get_user_pages_remote(NULL, mm, vaddr, 1, 0, 1, &old_page, &vma);
 
 303	if (ret <= 0)
 304		return ret;
 305
 306	ret = verify_opcode(old_page, vaddr, &opcode);
 307	if (ret <= 0)
 308		goto put_old;
 309
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 310	ret = anon_vma_prepare(vma);
 311	if (ret)
 312		goto put_old;
 313
 314	ret = -ENOMEM;
 315	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
 316	if (!new_page)
 317		goto put_old;
 318
 319	__SetPageUptodate(new_page);
 320	copy_highpage(new_page, old_page);
 321	copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
 322
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 323	ret = __replace_page(vma, vaddr, old_page, new_page);
 324	put_page(new_page);
 
 325put_old:
 326	put_page(old_page);
 327
 328	if (unlikely(ret == -EAGAIN))
 329		goto retry;
 
 
 
 
 
 
 
 
 
 330	return ret;
 331}
 332
 333/**
 334 * set_swbp - store breakpoint at a given address.
 335 * @auprobe: arch specific probepoint information.
 336 * @mm: the probed process address space.
 337 * @vaddr: the virtual address to insert the opcode.
 338 *
 339 * For mm @mm, store the breakpoint instruction at @vaddr.
 340 * Return 0 (success) or a negative errno.
 341 */
 342int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
 343{
 344	return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN);
 345}
 346
 347/**
 348 * set_orig_insn - Restore the original instruction.
 349 * @mm: the probed process address space.
 350 * @auprobe: arch specific probepoint information.
 351 * @vaddr: the virtual address to insert the opcode.
 352 *
 353 * For mm @mm, restore the original opcode (opcode) at @vaddr.
 354 * Return 0 (success) or a negative errno.
 355 */
 356int __weak
 357set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
 358{
 359	return uprobe_write_opcode(mm, vaddr, *(uprobe_opcode_t *)&auprobe->insn);
 
 360}
 361
 362static struct uprobe *get_uprobe(struct uprobe *uprobe)
 363{
 364	atomic_inc(&uprobe->ref);
 365	return uprobe;
 366}
 367
 368static void put_uprobe(struct uprobe *uprobe)
 369{
 370	if (atomic_dec_and_test(&uprobe->ref))
 
 
 
 
 
 
 
 
 371		kfree(uprobe);
 
 372}
 373
 374static int match_uprobe(struct uprobe *l, struct uprobe *r)
 
 
 375{
 376	if (l->inode < r->inode)
 377		return -1;
 378
 379	if (l->inode > r->inode)
 380		return 1;
 381
 382	if (l->offset < r->offset)
 383		return -1;
 384
 385	if (l->offset > r->offset)
 386		return 1;
 387
 388	return 0;
 389}
 390
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 391static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
 392{
 393	struct uprobe u = { .inode = inode, .offset = offset };
 394	struct rb_node *n = uprobes_tree.rb_node;
 395	struct uprobe *uprobe;
 396	int match;
 
 397
 398	while (n) {
 399		uprobe = rb_entry(n, struct uprobe, rb_node);
 400		match = match_uprobe(&u, uprobe);
 401		if (!match)
 402			return get_uprobe(uprobe);
 403
 404		if (match < 0)
 405			n = n->rb_left;
 406		else
 407			n = n->rb_right;
 408	}
 409	return NULL;
 410}
 411
 412/*
 413 * Find a uprobe corresponding to a given inode:offset
 414 * Acquires uprobes_treelock
 415 */
 416static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
 417{
 418	struct uprobe *uprobe;
 419
 420	spin_lock(&uprobes_treelock);
 421	uprobe = __find_uprobe(inode, offset);
 422	spin_unlock(&uprobes_treelock);
 423
 424	return uprobe;
 425}
 426
 427static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
 428{
 429	struct rb_node **p = &uprobes_tree.rb_node;
 430	struct rb_node *parent = NULL;
 431	struct uprobe *u;
 432	int match;
 433
 434	while (*p) {
 435		parent = *p;
 436		u = rb_entry(parent, struct uprobe, rb_node);
 437		match = match_uprobe(uprobe, u);
 438		if (!match)
 439			return get_uprobe(u);
 440
 441		if (match < 0)
 442			p = &parent->rb_left;
 443		else
 444			p = &parent->rb_right;
 445
 446	}
 
 
 447
 448	u = NULL;
 449	rb_link_node(&uprobe->rb_node, parent, p);
 450	rb_insert_color(&uprobe->rb_node, &uprobes_tree);
 451	/* get access + creation ref */
 452	atomic_set(&uprobe->ref, 2);
 453
 454	return u;
 455}
 456
 457/*
 458 * Acquire uprobes_treelock.
 459 * Matching uprobe already exists in rbtree;
 460 *	increment (access refcount) and return the matching uprobe.
 461 *
 462 * No matching uprobe; insert the uprobe in rb_tree;
 463 *	get a double refcount (access + creation) and return NULL.
 464 */
 465static struct uprobe *insert_uprobe(struct uprobe *uprobe)
 466{
 467	struct uprobe *u;
 468
 469	spin_lock(&uprobes_treelock);
 470	u = __insert_uprobe(uprobe);
 471	spin_unlock(&uprobes_treelock);
 472
 473	return u;
 474}
 475
 476static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
 
 
 
 
 
 
 
 
 
 
 
 477{
 478	struct uprobe *uprobe, *cur_uprobe;
 479
 480	uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
 481	if (!uprobe)
 482		return NULL;
 483
 484	uprobe->inode = igrab(inode);
 485	uprobe->offset = offset;
 
 486	init_rwsem(&uprobe->register_rwsem);
 487	init_rwsem(&uprobe->consumer_rwsem);
 488
 489	/* add to uprobes_tree, sorted on inode:offset */
 490	cur_uprobe = insert_uprobe(uprobe);
 491	/* a uprobe exists for this inode:offset combination */
 492	if (cur_uprobe) {
 
 
 
 
 
 
 493		kfree(uprobe);
 494		uprobe = cur_uprobe;
 495		iput(inode);
 496	}
 497
 498	return uprobe;
 499}
 500
 501static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
 502{
 503	down_write(&uprobe->consumer_rwsem);
 504	uc->next = uprobe->consumers;
 505	uprobe->consumers = uc;
 506	up_write(&uprobe->consumer_rwsem);
 507}
 508
 509/*
 510 * For uprobe @uprobe, delete the consumer @uc.
 511 * Return true if the @uc is deleted successfully
 512 * or return false.
 513 */
 514static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
 515{
 516	struct uprobe_consumer **con;
 517	bool ret = false;
 518
 519	down_write(&uprobe->consumer_rwsem);
 520	for (con = &uprobe->consumers; *con; con = &(*con)->next) {
 521		if (*con == uc) {
 522			*con = uc->next;
 523			ret = true;
 524			break;
 525		}
 526	}
 527	up_write(&uprobe->consumer_rwsem);
 528
 529	return ret;
 530}
 531
 532static int __copy_insn(struct address_space *mapping, struct file *filp,
 533			void *insn, int nbytes, loff_t offset)
 534{
 535	struct page *page;
 536	/*
 537	 * Ensure that the page that has the original instruction is populated
 538	 * and in page-cache. If ->readpage == NULL it must be shmem_mapping(),
 539	 * see uprobe_register().
 540	 */
 541	if (mapping->a_ops->readpage)
 542		page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp);
 543	else
 544		page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
 545	if (IS_ERR(page))
 546		return PTR_ERR(page);
 547
 548	copy_from_page(page, offset, insn, nbytes);
 549	put_page(page);
 550
 551	return 0;
 552}
 553
 554static int copy_insn(struct uprobe *uprobe, struct file *filp)
 555{
 556	struct address_space *mapping = uprobe->inode->i_mapping;
 557	loff_t offs = uprobe->offset;
 558	void *insn = &uprobe->arch.insn;
 559	int size = sizeof(uprobe->arch.insn);
 560	int len, err = -EIO;
 561
 562	/* Copy only available bytes, -EIO if nothing was read */
 563	do {
 564		if (offs >= i_size_read(uprobe->inode))
 565			break;
 566
 567		len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
 568		err = __copy_insn(mapping, filp, insn, len, offs);
 569		if (err)
 570			break;
 571
 572		insn += len;
 573		offs += len;
 574		size -= len;
 575	} while (size);
 576
 577	return err;
 578}
 579
 580static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
 581				struct mm_struct *mm, unsigned long vaddr)
 582{
 583	int ret = 0;
 584
 585	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
 586		return ret;
 587
 588	/* TODO: move this into _register, until then we abuse this sem. */
 589	down_write(&uprobe->consumer_rwsem);
 590	if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
 591		goto out;
 592
 593	ret = copy_insn(uprobe, file);
 594	if (ret)
 595		goto out;
 596
 597	ret = -ENOTSUPP;
 598	if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
 599		goto out;
 600
 601	ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
 602	if (ret)
 603		goto out;
 604
 605	/* uprobe_write_opcode() assumes we don't cross page boundary */
 606	BUG_ON((uprobe->offset & ~PAGE_MASK) +
 607			UPROBE_SWBP_INSN_SIZE > PAGE_SIZE);
 608
 609	smp_wmb(); /* pairs with rmb() in find_active_uprobe() */
 610	set_bit(UPROBE_COPY_INSN, &uprobe->flags);
 611
 612 out:
 613	up_write(&uprobe->consumer_rwsem);
 614
 615	return ret;
 616}
 617
 618static inline bool consumer_filter(struct uprobe_consumer *uc,
 619				   enum uprobe_filter_ctx ctx, struct mm_struct *mm)
 620{
 621	return !uc->filter || uc->filter(uc, ctx, mm);
 622}
 623
 624static bool filter_chain(struct uprobe *uprobe,
 625			 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
 626{
 627	struct uprobe_consumer *uc;
 628	bool ret = false;
 629
 630	down_read(&uprobe->consumer_rwsem);
 631	for (uc = uprobe->consumers; uc; uc = uc->next) {
 632		ret = consumer_filter(uc, ctx, mm);
 633		if (ret)
 634			break;
 635	}
 636	up_read(&uprobe->consumer_rwsem);
 637
 638	return ret;
 639}
 640
 641static int
 642install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
 643			struct vm_area_struct *vma, unsigned long vaddr)
 644{
 645	bool first_uprobe;
 646	int ret;
 647
 648	ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
 649	if (ret)
 650		return ret;
 651
 652	/*
 653	 * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
 654	 * the task can hit this breakpoint right after __replace_page().
 655	 */
 656	first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
 657	if (first_uprobe)
 658		set_bit(MMF_HAS_UPROBES, &mm->flags);
 659
 660	ret = set_swbp(&uprobe->arch, mm, vaddr);
 661	if (!ret)
 662		clear_bit(MMF_RECALC_UPROBES, &mm->flags);
 663	else if (first_uprobe)
 664		clear_bit(MMF_HAS_UPROBES, &mm->flags);
 665
 666	return ret;
 667}
 668
 669static int
 670remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
 671{
 672	set_bit(MMF_RECALC_UPROBES, &mm->flags);
 673	return set_orig_insn(&uprobe->arch, mm, vaddr);
 674}
 675
 676static inline bool uprobe_is_active(struct uprobe *uprobe)
 677{
 678	return !RB_EMPTY_NODE(&uprobe->rb_node);
 679}
 680/*
 681 * There could be threads that have already hit the breakpoint. They
 682 * will recheck the current insn and restart if find_uprobe() fails.
 683 * See find_active_uprobe().
 684 */
 685static void delete_uprobe(struct uprobe *uprobe)
 686{
 687	if (WARN_ON(!uprobe_is_active(uprobe)))
 688		return;
 689
 690	spin_lock(&uprobes_treelock);
 691	rb_erase(&uprobe->rb_node, &uprobes_tree);
 692	spin_unlock(&uprobes_treelock);
 693	RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
 694	iput(uprobe->inode);
 695	put_uprobe(uprobe);
 696}
 697
 698struct map_info {
 699	struct map_info *next;
 700	struct mm_struct *mm;
 701	unsigned long vaddr;
 702};
 703
 704static inline struct map_info *free_map_info(struct map_info *info)
 705{
 706	struct map_info *next = info->next;
 707	kfree(info);
 708	return next;
 709}
 710
 711static struct map_info *
 712build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
 713{
 714	unsigned long pgoff = offset >> PAGE_SHIFT;
 715	struct vm_area_struct *vma;
 716	struct map_info *curr = NULL;
 717	struct map_info *prev = NULL;
 718	struct map_info *info;
 719	int more = 0;
 720
 721 again:
 722	i_mmap_lock_read(mapping);
 723	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
 724		if (!valid_vma(vma, is_register))
 725			continue;
 726
 727		if (!prev && !more) {
 728			/*
 729			 * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through
 730			 * reclaim. This is optimistic, no harm done if it fails.
 731			 */
 732			prev = kmalloc(sizeof(struct map_info),
 733					GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
 734			if (prev)
 735				prev->next = NULL;
 736		}
 737		if (!prev) {
 738			more++;
 739			continue;
 740		}
 741
 742		if (!atomic_inc_not_zero(&vma->vm_mm->mm_users))
 743			continue;
 744
 745		info = prev;
 746		prev = prev->next;
 747		info->next = curr;
 748		curr = info;
 749
 750		info->mm = vma->vm_mm;
 751		info->vaddr = offset_to_vaddr(vma, offset);
 752	}
 753	i_mmap_unlock_read(mapping);
 754
 755	if (!more)
 756		goto out;
 757
 758	prev = curr;
 759	while (curr) {
 760		mmput(curr->mm);
 761		curr = curr->next;
 762	}
 763
 764	do {
 765		info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
 766		if (!info) {
 767			curr = ERR_PTR(-ENOMEM);
 768			goto out;
 769		}
 770		info->next = prev;
 771		prev = info;
 772	} while (--more);
 773
 774	goto again;
 775 out:
 776	while (prev)
 777		prev = free_map_info(prev);
 778	return curr;
 779}
 780
 781static int
 782register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
 783{
 784	bool is_register = !!new;
 785	struct map_info *info;
 786	int err = 0;
 787
 788	percpu_down_write(&dup_mmap_sem);
 789	info = build_map_info(uprobe->inode->i_mapping,
 790					uprobe->offset, is_register);
 791	if (IS_ERR(info)) {
 792		err = PTR_ERR(info);
 793		goto out;
 794	}
 795
 796	while (info) {
 797		struct mm_struct *mm = info->mm;
 798		struct vm_area_struct *vma;
 799
 800		if (err && is_register)
 801			goto free;
 802
 803		down_write(&mm->mmap_sem);
 804		vma = find_vma(mm, info->vaddr);
 805		if (!vma || !valid_vma(vma, is_register) ||
 806		    file_inode(vma->vm_file) != uprobe->inode)
 807			goto unlock;
 808
 809		if (vma->vm_start > info->vaddr ||
 810		    vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
 811			goto unlock;
 812
 813		if (is_register) {
 814			/* consult only the "caller", new consumer. */
 815			if (consumer_filter(new,
 816					UPROBE_FILTER_REGISTER, mm))
 817				err = install_breakpoint(uprobe, mm, vma, info->vaddr);
 818		} else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
 819			if (!filter_chain(uprobe,
 820					UPROBE_FILTER_UNREGISTER, mm))
 821				err |= remove_breakpoint(uprobe, mm, info->vaddr);
 822		}
 823
 824 unlock:
 825		up_write(&mm->mmap_sem);
 826 free:
 827		mmput(mm);
 828		info = free_map_info(info);
 829	}
 830 out:
 831	percpu_up_write(&dup_mmap_sem);
 832	return err;
 833}
 834
 835static int __uprobe_register(struct uprobe *uprobe, struct uprobe_consumer *uc)
 836{
 837	consumer_add(uprobe, uc);
 838	return register_for_each_vma(uprobe, uc);
 839}
 840
 841static void __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)
 842{
 843	int err;
 844
 845	if (WARN_ON(!consumer_del(uprobe, uc)))
 846		return;
 847
 848	err = register_for_each_vma(uprobe, NULL);
 849	/* TODO : cant unregister? schedule a worker thread */
 850	if (!uprobe->consumers && !err)
 851		delete_uprobe(uprobe);
 852}
 853
 854/*
 855 * uprobe_register - register a probe
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 856 * @inode: the file in which the probe has to be placed.
 857 * @offset: offset from the start of the file.
 858 * @uc: information on howto handle the probe..
 859 *
 860 * Apart from the access refcount, uprobe_register() takes a creation
 861 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
 862 * inserted into the rbtree (i.e first consumer for a @inode:@offset
 863 * tuple).  Creation refcount stops uprobe_unregister from freeing the
 864 * @uprobe even before the register operation is complete. Creation
 865 * refcount is released when the last @uc for the @uprobe
 866 * unregisters.
 
 867 *
 868 * Return errno if it cannot successully install probes
 869 * else return 0 (success)
 870 */
 871int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
 
 872{
 873	struct uprobe *uprobe;
 874	int ret;
 875
 876	/* Uprobe must have at least one set consumer */
 877	if (!uc->handler && !uc->ret_handler)
 878		return -EINVAL;
 879
 880	/* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
 881	if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping))
 882		return -EIO;
 883	/* Racy, just to catch the obvious mistakes */
 884	if (offset > i_size_read(inode))
 885		return -EINVAL;
 886
 
 
 
 
 
 
 
 
 
 887 retry:
 888	uprobe = alloc_uprobe(inode, offset);
 889	if (!uprobe)
 890		return -ENOMEM;
 
 
 
 891	/*
 892	 * We can race with uprobe_unregister()->delete_uprobe().
 893	 * Check uprobe_is_active() and retry if it is false.
 894	 */
 895	down_write(&uprobe->register_rwsem);
 896	ret = -EAGAIN;
 897	if (likely(uprobe_is_active(uprobe))) {
 898		ret = __uprobe_register(uprobe, uc);
 
 899		if (ret)
 900			__uprobe_unregister(uprobe, uc);
 901	}
 902	up_write(&uprobe->register_rwsem);
 903	put_uprobe(uprobe);
 904
 905	if (unlikely(ret == -EAGAIN))
 906		goto retry;
 907	return ret;
 908}
 
 
 
 
 
 
 909EXPORT_SYMBOL_GPL(uprobe_register);
 910
 
 
 
 
 
 
 
 911/*
 912 * uprobe_apply - unregister a already registered probe.
 913 * @inode: the file in which the probe has to be removed.
 914 * @offset: offset from the start of the file.
 915 * @uc: consumer which wants to add more or remove some breakpoints
 916 * @add: add or remove the breakpoints
 917 */
 918int uprobe_apply(struct inode *inode, loff_t offset,
 919			struct uprobe_consumer *uc, bool add)
 920{
 921	struct uprobe *uprobe;
 922	struct uprobe_consumer *con;
 923	int ret = -ENOENT;
 924
 925	uprobe = find_uprobe(inode, offset);
 926	if (WARN_ON(!uprobe))
 927		return ret;
 928
 929	down_write(&uprobe->register_rwsem);
 930	for (con = uprobe->consumers; con && con != uc ; con = con->next)
 931		;
 932	if (con)
 933		ret = register_for_each_vma(uprobe, add ? uc : NULL);
 934	up_write(&uprobe->register_rwsem);
 935	put_uprobe(uprobe);
 936
 937	return ret;
 938}
 939
 940/*
 941 * uprobe_unregister - unregister a already registered probe.
 942 * @inode: the file in which the probe has to be removed.
 943 * @offset: offset from the start of the file.
 944 * @uc: identify which probe if multiple probes are colocated.
 945 */
 946void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
 947{
 948	struct uprobe *uprobe;
 949
 950	uprobe = find_uprobe(inode, offset);
 951	if (WARN_ON(!uprobe))
 952		return;
 953
 954	down_write(&uprobe->register_rwsem);
 955	__uprobe_unregister(uprobe, uc);
 956	up_write(&uprobe->register_rwsem);
 957	put_uprobe(uprobe);
 958}
 959EXPORT_SYMBOL_GPL(uprobe_unregister);
 960
 961static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
 962{
 963	struct vm_area_struct *vma;
 964	int err = 0;
 965
 966	down_read(&mm->mmap_sem);
 967	for (vma = mm->mmap; vma; vma = vma->vm_next) {
 968		unsigned long vaddr;
 969		loff_t offset;
 970
 971		if (!valid_vma(vma, false) ||
 972		    file_inode(vma->vm_file) != uprobe->inode)
 973			continue;
 974
 975		offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
 976		if (uprobe->offset <  offset ||
 977		    uprobe->offset >= offset + vma->vm_end - vma->vm_start)
 978			continue;
 979
 980		vaddr = offset_to_vaddr(vma, uprobe->offset);
 981		err |= remove_breakpoint(uprobe, mm, vaddr);
 982	}
 983	up_read(&mm->mmap_sem);
 984
 985	return err;
 986}
 987
 988static struct rb_node *
 989find_node_in_range(struct inode *inode, loff_t min, loff_t max)
 990{
 991	struct rb_node *n = uprobes_tree.rb_node;
 992
 993	while (n) {
 994		struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
 995
 996		if (inode < u->inode) {
 997			n = n->rb_left;
 998		} else if (inode > u->inode) {
 999			n = n->rb_right;
1000		} else {
1001			if (max < u->offset)
1002				n = n->rb_left;
1003			else if (min > u->offset)
1004				n = n->rb_right;
1005			else
1006				break;
1007		}
1008	}
1009
1010	return n;
1011}
1012
1013/*
1014 * For a given range in vma, build a list of probes that need to be inserted.
1015 */
1016static void build_probe_list(struct inode *inode,
1017				struct vm_area_struct *vma,
1018				unsigned long start, unsigned long end,
1019				struct list_head *head)
1020{
1021	loff_t min, max;
1022	struct rb_node *n, *t;
1023	struct uprobe *u;
1024
1025	INIT_LIST_HEAD(head);
1026	min = vaddr_to_offset(vma, start);
1027	max = min + (end - start) - 1;
1028
1029	spin_lock(&uprobes_treelock);
1030	n = find_node_in_range(inode, min, max);
1031	if (n) {
1032		for (t = n; t; t = rb_prev(t)) {
1033			u = rb_entry(t, struct uprobe, rb_node);
1034			if (u->inode != inode || u->offset < min)
1035				break;
1036			list_add(&u->pending_list, head);
1037			get_uprobe(u);
1038		}
1039		for (t = n; (t = rb_next(t)); ) {
1040			u = rb_entry(t, struct uprobe, rb_node);
1041			if (u->inode != inode || u->offset > max)
1042				break;
1043			list_add(&u->pending_list, head);
1044			get_uprobe(u);
1045		}
1046	}
1047	spin_unlock(&uprobes_treelock);
1048}
1049
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1050/*
1051 * Called from mmap_region/vma_adjust with mm->mmap_sem acquired.
1052 *
1053 * Currently we ignore all errors and always return 0, the callers
1054 * can't handle the failure anyway.
1055 */
1056int uprobe_mmap(struct vm_area_struct *vma)
1057{
1058	struct list_head tmp_list;
1059	struct uprobe *uprobe, *u;
1060	struct inode *inode;
1061
1062	if (no_uprobe_events() || !valid_vma(vma, true))
 
 
 
 
 
 
 
 
1063		return 0;
1064
1065	inode = file_inode(vma->vm_file);
1066	if (!inode)
1067		return 0;
1068
1069	mutex_lock(uprobes_mmap_hash(inode));
1070	build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
1071	/*
1072	 * We can race with uprobe_unregister(), this uprobe can be already
1073	 * removed. But in this case filter_chain() must return false, all
1074	 * consumers have gone away.
1075	 */
1076	list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
1077		if (!fatal_signal_pending(current) &&
1078		    filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
1079			unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
1080			install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
1081		}
1082		put_uprobe(uprobe);
1083	}
1084	mutex_unlock(uprobes_mmap_hash(inode));
1085
1086	return 0;
1087}
1088
1089static bool
1090vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1091{
1092	loff_t min, max;
1093	struct inode *inode;
1094	struct rb_node *n;
1095
1096	inode = file_inode(vma->vm_file);
1097
1098	min = vaddr_to_offset(vma, start);
1099	max = min + (end - start) - 1;
1100
1101	spin_lock(&uprobes_treelock);
1102	n = find_node_in_range(inode, min, max);
1103	spin_unlock(&uprobes_treelock);
1104
1105	return !!n;
1106}
1107
1108/*
1109 * Called in context of a munmap of a vma.
1110 */
1111void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1112{
1113	if (no_uprobe_events() || !valid_vma(vma, false))
1114		return;
1115
1116	if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
1117		return;
1118
1119	if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
1120	     test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
1121		return;
1122
1123	if (vma_has_uprobes(vma, start, end))
1124		set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
1125}
1126
1127/* Slot allocation for XOL */
1128static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
1129{
1130	struct vm_area_struct *vma;
1131	int ret;
1132
1133	down_write(&mm->mmap_sem);
 
 
1134	if (mm->uprobes_state.xol_area) {
1135		ret = -EALREADY;
1136		goto fail;
1137	}
1138
1139	if (!area->vaddr) {
1140		/* Try to map as high as possible, this is only a hint. */
1141		area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
1142						PAGE_SIZE, 0, 0);
1143		if (area->vaddr & ~PAGE_MASK) {
1144			ret = area->vaddr;
1145			goto fail;
1146		}
1147	}
1148
1149	vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
1150				VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO,
1151				&area->xol_mapping);
1152	if (IS_ERR(vma)) {
1153		ret = PTR_ERR(vma);
1154		goto fail;
1155	}
1156
1157	ret = 0;
1158	smp_wmb();	/* pairs with get_xol_area() */
1159	mm->uprobes_state.xol_area = area;
1160 fail:
1161	up_write(&mm->mmap_sem);
1162
1163	return ret;
1164}
1165
1166static struct xol_area *__create_xol_area(unsigned long vaddr)
1167{
1168	struct mm_struct *mm = current->mm;
1169	uprobe_opcode_t insn = UPROBE_SWBP_INSN;
1170	struct xol_area *area;
1171
1172	area = kmalloc(sizeof(*area), GFP_KERNEL);
1173	if (unlikely(!area))
1174		goto out;
1175
1176	area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL);
 
1177	if (!area->bitmap)
1178		goto free_area;
1179
1180	area->xol_mapping.name = "[uprobes]";
1181	area->xol_mapping.fault = NULL;
1182	area->xol_mapping.pages = area->pages;
1183	area->pages[0] = alloc_page(GFP_HIGHUSER);
1184	if (!area->pages[0])
1185		goto free_bitmap;
1186	area->pages[1] = NULL;
1187
1188	area->vaddr = vaddr;
1189	init_waitqueue_head(&area->wq);
1190	/* Reserve the 1st slot for get_trampoline_vaddr() */
1191	set_bit(0, area->bitmap);
1192	atomic_set(&area->slot_count, 1);
1193	copy_to_page(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
1194
1195	if (!xol_add_vma(mm, area))
1196		return area;
1197
1198	__free_page(area->pages[0]);
1199 free_bitmap:
1200	kfree(area->bitmap);
1201 free_area:
1202	kfree(area);
1203 out:
1204	return NULL;
1205}
1206
1207/*
1208 * get_xol_area - Allocate process's xol_area if necessary.
1209 * This area will be used for storing instructions for execution out of line.
1210 *
1211 * Returns the allocated area or NULL.
1212 */
1213static struct xol_area *get_xol_area(void)
1214{
1215	struct mm_struct *mm = current->mm;
1216	struct xol_area *area;
1217
1218	if (!mm->uprobes_state.xol_area)
1219		__create_xol_area(0);
1220
1221	area = mm->uprobes_state.xol_area;
1222	smp_read_barrier_depends();	/* pairs with wmb in xol_add_vma() */
1223	return area;
1224}
1225
1226/*
1227 * uprobe_clear_state - Free the area allocated for slots.
1228 */
1229void uprobe_clear_state(struct mm_struct *mm)
1230{
1231	struct xol_area *area = mm->uprobes_state.xol_area;
1232
 
 
 
 
1233	if (!area)
1234		return;
1235
1236	put_page(area->pages[0]);
1237	kfree(area->bitmap);
1238	kfree(area);
1239}
1240
1241void uprobe_start_dup_mmap(void)
1242{
1243	percpu_down_read(&dup_mmap_sem);
1244}
1245
1246void uprobe_end_dup_mmap(void)
1247{
1248	percpu_up_read(&dup_mmap_sem);
1249}
1250
1251void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
1252{
1253	newmm->uprobes_state.xol_area = NULL;
1254
1255	if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
1256		set_bit(MMF_HAS_UPROBES, &newmm->flags);
1257		/* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
1258		set_bit(MMF_RECALC_UPROBES, &newmm->flags);
1259	}
1260}
1261
1262/*
1263 *  - search for a free slot.
1264 */
1265static unsigned long xol_take_insn_slot(struct xol_area *area)
1266{
1267	unsigned long slot_addr;
1268	int slot_nr;
1269
1270	do {
1271		slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
1272		if (slot_nr < UINSNS_PER_PAGE) {
1273			if (!test_and_set_bit(slot_nr, area->bitmap))
1274				break;
1275
1276			slot_nr = UINSNS_PER_PAGE;
1277			continue;
1278		}
1279		wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
1280	} while (slot_nr >= UINSNS_PER_PAGE);
1281
1282	slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
1283	atomic_inc(&area->slot_count);
1284
1285	return slot_addr;
1286}
1287
1288/*
1289 * xol_get_insn_slot - allocate a slot for xol.
1290 * Returns the allocated slot address or 0.
1291 */
1292static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
1293{
1294	struct xol_area *area;
1295	unsigned long xol_vaddr;
1296
1297	area = get_xol_area();
1298	if (!area)
1299		return 0;
1300
1301	xol_vaddr = xol_take_insn_slot(area);
1302	if (unlikely(!xol_vaddr))
1303		return 0;
1304
1305	arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,
1306			      &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
1307
1308	return xol_vaddr;
1309}
1310
1311/*
1312 * xol_free_insn_slot - If slot was earlier allocated by
1313 * @xol_get_insn_slot(), make the slot available for
1314 * subsequent requests.
1315 */
1316static void xol_free_insn_slot(struct task_struct *tsk)
1317{
1318	struct xol_area *area;
1319	unsigned long vma_end;
1320	unsigned long slot_addr;
1321
1322	if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
1323		return;
1324
1325	slot_addr = tsk->utask->xol_vaddr;
1326	if (unlikely(!slot_addr))
1327		return;
1328
1329	area = tsk->mm->uprobes_state.xol_area;
1330	vma_end = area->vaddr + PAGE_SIZE;
1331	if (area->vaddr <= slot_addr && slot_addr < vma_end) {
1332		unsigned long offset;
1333		int slot_nr;
1334
1335		offset = slot_addr - area->vaddr;
1336		slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
1337		if (slot_nr >= UINSNS_PER_PAGE)
1338			return;
1339
1340		clear_bit(slot_nr, area->bitmap);
1341		atomic_dec(&area->slot_count);
1342		smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
1343		if (waitqueue_active(&area->wq))
1344			wake_up(&area->wq);
1345
1346		tsk->utask->xol_vaddr = 0;
1347	}
1348}
1349
1350void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
1351				  void *src, unsigned long len)
1352{
1353	/* Initialize the slot */
1354	copy_to_page(page, vaddr, src, len);
1355
1356	/*
1357	 * We probably need flush_icache_user_range() but it needs vma.
1358	 * This should work on most of architectures by default. If
1359	 * architecture needs to do something different it can define
1360	 * its own version of the function.
1361	 */
1362	flush_dcache_page(page);
1363}
1364
1365/**
1366 * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
1367 * @regs: Reflects the saved state of the task after it has hit a breakpoint
1368 * instruction.
1369 * Return the address of the breakpoint instruction.
1370 */
1371unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
1372{
1373	return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
1374}
1375
1376unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
1377{
1378	struct uprobe_task *utask = current->utask;
1379
1380	if (unlikely(utask && utask->active_uprobe))
1381		return utask->vaddr;
1382
1383	return instruction_pointer(regs);
1384}
1385
1386static struct return_instance *free_ret_instance(struct return_instance *ri)
1387{
1388	struct return_instance *next = ri->next;
1389	put_uprobe(ri->uprobe);
1390	kfree(ri);
1391	return next;
1392}
1393
1394/*
1395 * Called with no locks held.
1396 * Called in context of a exiting or a exec-ing thread.
1397 */
1398void uprobe_free_utask(struct task_struct *t)
1399{
1400	struct uprobe_task *utask = t->utask;
1401	struct return_instance *ri;
1402
1403	if (!utask)
1404		return;
1405
1406	if (utask->active_uprobe)
1407		put_uprobe(utask->active_uprobe);
1408
1409	ri = utask->return_instances;
1410	while (ri)
1411		ri = free_ret_instance(ri);
1412
1413	xol_free_insn_slot(t);
1414	kfree(utask);
1415	t->utask = NULL;
1416}
1417
1418/*
1419 * Allocate a uprobe_task object for the task if if necessary.
1420 * Called when the thread hits a breakpoint.
1421 *
1422 * Returns:
1423 * - pointer to new uprobe_task on success
1424 * - NULL otherwise
1425 */
1426static struct uprobe_task *get_utask(void)
1427{
1428	if (!current->utask)
1429		current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1430	return current->utask;
1431}
1432
1433static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
1434{
1435	struct uprobe_task *n_utask;
1436	struct return_instance **p, *o, *n;
1437
1438	n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
1439	if (!n_utask)
1440		return -ENOMEM;
1441	t->utask = n_utask;
1442
1443	p = &n_utask->return_instances;
1444	for (o = o_utask->return_instances; o; o = o->next) {
1445		n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1446		if (!n)
1447			return -ENOMEM;
1448
1449		*n = *o;
1450		get_uprobe(n->uprobe);
1451		n->next = NULL;
1452
1453		*p = n;
1454		p = &n->next;
1455		n_utask->depth++;
1456	}
1457
1458	return 0;
1459}
1460
1461static void uprobe_warn(struct task_struct *t, const char *msg)
1462{
1463	pr_warn("uprobe: %s:%d failed to %s\n",
1464			current->comm, current->pid, msg);
1465}
1466
1467static void dup_xol_work(struct callback_head *work)
1468{
1469	if (current->flags & PF_EXITING)
1470		return;
1471
1472	if (!__create_xol_area(current->utask->dup_xol_addr))
 
1473		uprobe_warn(current, "dup xol area");
1474}
1475
1476/*
1477 * Called in context of a new clone/fork from copy_process.
1478 */
1479void uprobe_copy_process(struct task_struct *t, unsigned long flags)
1480{
1481	struct uprobe_task *utask = current->utask;
1482	struct mm_struct *mm = current->mm;
1483	struct xol_area *area;
1484
1485	t->utask = NULL;
1486
1487	if (!utask || !utask->return_instances)
1488		return;
1489
1490	if (mm == t->mm && !(flags & CLONE_VFORK))
1491		return;
1492
1493	if (dup_utask(t, utask))
1494		return uprobe_warn(t, "dup ret instances");
1495
1496	/* The task can fork() after dup_xol_work() fails */
1497	area = mm->uprobes_state.xol_area;
1498	if (!area)
1499		return uprobe_warn(t, "dup xol area");
1500
1501	if (mm == t->mm)
1502		return;
1503
1504	t->utask->dup_xol_addr = area->vaddr;
1505	init_task_work(&t->utask->dup_xol_work, dup_xol_work);
1506	task_work_add(t, &t->utask->dup_xol_work, true);
1507}
1508
1509/*
1510 * Current area->vaddr notion assume the trampoline address is always
1511 * equal area->vaddr.
1512 *
1513 * Returns -1 in case the xol_area is not allocated.
1514 */
1515static unsigned long get_trampoline_vaddr(void)
1516{
1517	struct xol_area *area;
1518	unsigned long trampoline_vaddr = -1;
1519
1520	area = current->mm->uprobes_state.xol_area;
1521	smp_read_barrier_depends();
1522	if (area)
1523		trampoline_vaddr = area->vaddr;
1524
1525	return trampoline_vaddr;
1526}
1527
1528static void cleanup_return_instances(struct uprobe_task *utask, bool chained,
1529					struct pt_regs *regs)
1530{
1531	struct return_instance *ri = utask->return_instances;
1532	enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
1533
1534	while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
1535		ri = free_ret_instance(ri);
1536		utask->depth--;
1537	}
1538	utask->return_instances = ri;
1539}
1540
1541static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
1542{
1543	struct return_instance *ri;
1544	struct uprobe_task *utask;
1545	unsigned long orig_ret_vaddr, trampoline_vaddr;
1546	bool chained;
1547
1548	if (!get_xol_area())
1549		return;
1550
1551	utask = get_utask();
1552	if (!utask)
1553		return;
1554
1555	if (utask->depth >= MAX_URETPROBE_DEPTH) {
1556		printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
1557				" nestedness limit pid/tgid=%d/%d\n",
1558				current->pid, current->tgid);
1559		return;
1560	}
1561
1562	ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
1563	if (!ri)
1564		return;
1565
1566	trampoline_vaddr = get_trampoline_vaddr();
1567	orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
1568	if (orig_ret_vaddr == -1)
1569		goto fail;
1570
1571	/* drop the entries invalidated by longjmp() */
1572	chained = (orig_ret_vaddr == trampoline_vaddr);
1573	cleanup_return_instances(utask, chained, regs);
1574
1575	/*
1576	 * We don't want to keep trampoline address in stack, rather keep the
1577	 * original return address of first caller thru all the consequent
1578	 * instances. This also makes breakpoint unwrapping easier.
1579	 */
1580	if (chained) {
1581		if (!utask->return_instances) {
1582			/*
1583			 * This situation is not possible. Likely we have an
1584			 * attack from user-space.
1585			 */
1586			uprobe_warn(current, "handle tail call");
1587			goto fail;
1588		}
1589		orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
1590	}
1591
1592	ri->uprobe = get_uprobe(uprobe);
1593	ri->func = instruction_pointer(regs);
1594	ri->stack = user_stack_pointer(regs);
1595	ri->orig_ret_vaddr = orig_ret_vaddr;
1596	ri->chained = chained;
1597
1598	utask->depth++;
1599	ri->next = utask->return_instances;
1600	utask->return_instances = ri;
1601
1602	return;
1603 fail:
1604	kfree(ri);
1605}
1606
1607/* Prepare to single-step probed instruction out of line. */
1608static int
1609pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
1610{
1611	struct uprobe_task *utask;
1612	unsigned long xol_vaddr;
1613	int err;
1614
1615	utask = get_utask();
1616	if (!utask)
1617		return -ENOMEM;
1618
1619	xol_vaddr = xol_get_insn_slot(uprobe);
1620	if (!xol_vaddr)
1621		return -ENOMEM;
1622
1623	utask->xol_vaddr = xol_vaddr;
1624	utask->vaddr = bp_vaddr;
1625
1626	err = arch_uprobe_pre_xol(&uprobe->arch, regs);
1627	if (unlikely(err)) {
1628		xol_free_insn_slot(current);
1629		return err;
1630	}
1631
1632	utask->active_uprobe = uprobe;
1633	utask->state = UTASK_SSTEP;
1634	return 0;
1635}
1636
1637/*
1638 * If we are singlestepping, then ensure this thread is not connected to
1639 * non-fatal signals until completion of singlestep.  When xol insn itself
1640 * triggers the signal,  restart the original insn even if the task is
1641 * already SIGKILL'ed (since coredump should report the correct ip).  This
1642 * is even more important if the task has a handler for SIGSEGV/etc, The
1643 * _same_ instruction should be repeated again after return from the signal
1644 * handler, and SSTEP can never finish in this case.
1645 */
1646bool uprobe_deny_signal(void)
1647{
1648	struct task_struct *t = current;
1649	struct uprobe_task *utask = t->utask;
1650
1651	if (likely(!utask || !utask->active_uprobe))
1652		return false;
1653
1654	WARN_ON_ONCE(utask->state != UTASK_SSTEP);
1655
1656	if (signal_pending(t)) {
1657		spin_lock_irq(&t->sighand->siglock);
1658		clear_tsk_thread_flag(t, TIF_SIGPENDING);
1659		spin_unlock_irq(&t->sighand->siglock);
1660
1661		if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
1662			utask->state = UTASK_SSTEP_TRAPPED;
1663			set_tsk_thread_flag(t, TIF_UPROBE);
1664		}
1665	}
1666
1667	return true;
1668}
1669
1670static void mmf_recalc_uprobes(struct mm_struct *mm)
1671{
1672	struct vm_area_struct *vma;
1673
1674	for (vma = mm->mmap; vma; vma = vma->vm_next) {
1675		if (!valid_vma(vma, false))
1676			continue;
1677		/*
1678		 * This is not strictly accurate, we can race with
1679		 * uprobe_unregister() and see the already removed
1680		 * uprobe if delete_uprobe() was not yet called.
1681		 * Or this uprobe can be filtered out.
1682		 */
1683		if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
1684			return;
1685	}
1686
1687	clear_bit(MMF_HAS_UPROBES, &mm->flags);
1688}
1689
1690static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
1691{
1692	struct page *page;
1693	uprobe_opcode_t opcode;
1694	int result;
1695
 
 
 
1696	pagefault_disable();
1697	result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr,
1698							sizeof(opcode));
1699	pagefault_enable();
1700
1701	if (likely(result == 0))
1702		goto out;
1703
1704	/*
1705	 * The NULL 'tsk' here ensures that any faults that occur here
1706	 * will not be accounted to the task.  'mm' *is* current->mm,
1707	 * but we treat this as a 'remote' access since it is
1708	 * essentially a kernel access to the memory.
1709	 */
1710	result = get_user_pages_remote(NULL, mm, vaddr, 1, 0, 1, &page, NULL);
 
1711	if (result < 0)
1712		return result;
1713
1714	copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
1715	put_page(page);
1716 out:
1717	/* This needs to return true for any variant of the trap insn */
1718	return is_trap_insn(&opcode);
1719}
1720
1721static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
1722{
1723	struct mm_struct *mm = current->mm;
1724	struct uprobe *uprobe = NULL;
1725	struct vm_area_struct *vma;
1726
1727	down_read(&mm->mmap_sem);
1728	vma = find_vma(mm, bp_vaddr);
1729	if (vma && vma->vm_start <= bp_vaddr) {
1730		if (valid_vma(vma, false)) {
1731			struct inode *inode = file_inode(vma->vm_file);
1732			loff_t offset = vaddr_to_offset(vma, bp_vaddr);
1733
1734			uprobe = find_uprobe(inode, offset);
1735		}
1736
1737		if (!uprobe)
1738			*is_swbp = is_trap_at_addr(mm, bp_vaddr);
1739	} else {
1740		*is_swbp = -EFAULT;
1741	}
1742
1743	if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
1744		mmf_recalc_uprobes(mm);
1745	up_read(&mm->mmap_sem);
1746
1747	return uprobe;
1748}
1749
1750static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
1751{
1752	struct uprobe_consumer *uc;
1753	int remove = UPROBE_HANDLER_REMOVE;
1754	bool need_prep = false; /* prepare return uprobe, when needed */
1755
1756	down_read(&uprobe->register_rwsem);
1757	for (uc = uprobe->consumers; uc; uc = uc->next) {
1758		int rc = 0;
1759
1760		if (uc->handler) {
1761			rc = uc->handler(uc, regs);
1762			WARN(rc & ~UPROBE_HANDLER_MASK,
1763				"bad rc=0x%x from %pf()\n", rc, uc->handler);
1764		}
1765
1766		if (uc->ret_handler)
1767			need_prep = true;
1768
1769		remove &= rc;
1770	}
1771
1772	if (need_prep && !remove)
1773		prepare_uretprobe(uprobe, regs); /* put bp at return */
1774
1775	if (remove && uprobe->consumers) {
1776		WARN_ON(!uprobe_is_active(uprobe));
1777		unapply_uprobe(uprobe, current->mm);
1778	}
1779	up_read(&uprobe->register_rwsem);
1780}
1781
1782static void
1783handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
1784{
1785	struct uprobe *uprobe = ri->uprobe;
1786	struct uprobe_consumer *uc;
1787
1788	down_read(&uprobe->register_rwsem);
1789	for (uc = uprobe->consumers; uc; uc = uc->next) {
1790		if (uc->ret_handler)
1791			uc->ret_handler(uc, ri->func, regs);
1792	}
1793	up_read(&uprobe->register_rwsem);
1794}
1795
1796static struct return_instance *find_next_ret_chain(struct return_instance *ri)
1797{
1798	bool chained;
1799
1800	do {
1801		chained = ri->chained;
1802		ri = ri->next;	/* can't be NULL if chained */
1803	} while (chained);
1804
1805	return ri;
1806}
1807
1808static void handle_trampoline(struct pt_regs *regs)
1809{
1810	struct uprobe_task *utask;
1811	struct return_instance *ri, *next;
1812	bool valid;
1813
1814	utask = current->utask;
1815	if (!utask)
1816		goto sigill;
1817
1818	ri = utask->return_instances;
1819	if (!ri)
1820		goto sigill;
1821
1822	do {
1823		/*
1824		 * We should throw out the frames invalidated by longjmp().
1825		 * If this chain is valid, then the next one should be alive
1826		 * or NULL; the latter case means that nobody but ri->func
1827		 * could hit this trampoline on return. TODO: sigaltstack().
1828		 */
1829		next = find_next_ret_chain(ri);
1830		valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs);
1831
1832		instruction_pointer_set(regs, ri->orig_ret_vaddr);
1833		do {
1834			if (valid)
1835				handle_uretprobe_chain(ri, regs);
1836			ri = free_ret_instance(ri);
1837			utask->depth--;
1838		} while (ri != next);
1839	} while (!valid);
1840
1841	utask->return_instances = ri;
1842	return;
1843
1844 sigill:
1845	uprobe_warn(current, "handle uretprobe, sending SIGILL.");
1846	force_sig_info(SIGILL, SEND_SIG_FORCED, current);
1847
1848}
1849
1850bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
1851{
1852	return false;
1853}
1854
1855bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
1856					struct pt_regs *regs)
1857{
1858	return true;
1859}
1860
1861/*
1862 * Run handler and ask thread to singlestep.
1863 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
1864 */
1865static void handle_swbp(struct pt_regs *regs)
1866{
1867	struct uprobe *uprobe;
1868	unsigned long bp_vaddr;
1869	int uninitialized_var(is_swbp);
1870
1871	bp_vaddr = uprobe_get_swbp_addr(regs);
1872	if (bp_vaddr == get_trampoline_vaddr())
1873		return handle_trampoline(regs);
1874
1875	uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
1876	if (!uprobe) {
1877		if (is_swbp > 0) {
1878			/* No matching uprobe; signal SIGTRAP. */
1879			send_sig(SIGTRAP, current, 0);
1880		} else {
1881			/*
1882			 * Either we raced with uprobe_unregister() or we can't
1883			 * access this memory. The latter is only possible if
1884			 * another thread plays with our ->mm. In both cases
1885			 * we can simply restart. If this vma was unmapped we
1886			 * can pretend this insn was not executed yet and get
1887			 * the (correct) SIGSEGV after restart.
1888			 */
1889			instruction_pointer_set(regs, bp_vaddr);
1890		}
1891		return;
1892	}
1893
1894	/* change it in advance for ->handler() and restart */
1895	instruction_pointer_set(regs, bp_vaddr);
1896
1897	/*
1898	 * TODO: move copy_insn/etc into _register and remove this hack.
1899	 * After we hit the bp, _unregister + _register can install the
1900	 * new and not-yet-analyzed uprobe at the same address, restart.
1901	 */
1902	smp_rmb(); /* pairs with wmb() in install_breakpoint() */
1903	if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
1904		goto out;
1905
 
 
 
 
 
 
 
 
 
1906	/* Tracing handlers use ->utask to communicate with fetch methods */
1907	if (!get_utask())
1908		goto out;
1909
1910	if (arch_uprobe_ignore(&uprobe->arch, regs))
1911		goto out;
1912
1913	handler_chain(uprobe, regs);
1914
1915	if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
1916		goto out;
1917
1918	if (!pre_ssout(uprobe, regs, bp_vaddr))
1919		return;
1920
1921	/* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
1922out:
1923	put_uprobe(uprobe);
1924}
1925
1926/*
1927 * Perform required fix-ups and disable singlestep.
1928 * Allow pending signals to take effect.
1929 */
1930static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
1931{
1932	struct uprobe *uprobe;
1933	int err = 0;
1934
1935	uprobe = utask->active_uprobe;
1936	if (utask->state == UTASK_SSTEP_ACK)
1937		err = arch_uprobe_post_xol(&uprobe->arch, regs);
1938	else if (utask->state == UTASK_SSTEP_TRAPPED)
1939		arch_uprobe_abort_xol(&uprobe->arch, regs);
1940	else
1941		WARN_ON_ONCE(1);
1942
1943	put_uprobe(uprobe);
1944	utask->active_uprobe = NULL;
1945	utask->state = UTASK_RUNNING;
1946	xol_free_insn_slot(current);
1947
1948	spin_lock_irq(&current->sighand->siglock);
1949	recalc_sigpending(); /* see uprobe_deny_signal() */
1950	spin_unlock_irq(&current->sighand->siglock);
1951
1952	if (unlikely(err)) {
1953		uprobe_warn(current, "execute the probed insn, sending SIGILL.");
1954		force_sig_info(SIGILL, SEND_SIG_FORCED, current);
1955	}
1956}
1957
1958/*
1959 * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
1960 * allows the thread to return from interrupt. After that handle_swbp()
1961 * sets utask->active_uprobe.
1962 *
1963 * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
1964 * and allows the thread to return from interrupt.
1965 *
1966 * While returning to userspace, thread notices the TIF_UPROBE flag and calls
1967 * uprobe_notify_resume().
1968 */
1969void uprobe_notify_resume(struct pt_regs *regs)
1970{
1971	struct uprobe_task *utask;
1972
1973	clear_thread_flag(TIF_UPROBE);
1974
1975	utask = current->utask;
1976	if (utask && utask->active_uprobe)
1977		handle_singlestep(utask, regs);
1978	else
1979		handle_swbp(regs);
1980}
1981
1982/*
1983 * uprobe_pre_sstep_notifier gets called from interrupt context as part of
1984 * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
1985 */
1986int uprobe_pre_sstep_notifier(struct pt_regs *regs)
1987{
1988	if (!current->mm)
1989		return 0;
1990
1991	if (!test_bit(MMF_HAS_UPROBES, &current->mm->flags) &&
1992	    (!current->utask || !current->utask->return_instances))
1993		return 0;
1994
1995	set_thread_flag(TIF_UPROBE);
1996	return 1;
1997}
1998
1999/*
2000 * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
2001 * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
2002 */
2003int uprobe_post_sstep_notifier(struct pt_regs *regs)
2004{
2005	struct uprobe_task *utask = current->utask;
2006
2007	if (!current->mm || !utask || !utask->active_uprobe)
2008		/* task is currently not uprobed */
2009		return 0;
2010
2011	utask->state = UTASK_SSTEP_ACK;
2012	set_thread_flag(TIF_UPROBE);
2013	return 1;
2014}
2015
2016static struct notifier_block uprobe_exception_nb = {
2017	.notifier_call		= arch_uprobe_exception_notify,
2018	.priority		= INT_MAX-1,	/* notified after kprobes, kgdb */
2019};
2020
2021static int __init init_uprobes(void)
2022{
2023	int i;
2024
2025	for (i = 0; i < UPROBES_HASH_SZ; i++)
2026		mutex_init(&uprobes_mmap_mutex[i]);
2027
2028	if (percpu_init_rwsem(&dup_mmap_sem))
2029		return -ENOMEM;
2030
2031	return register_die_notifier(&uprobe_exception_nb);
2032}
2033__initcall(init_uprobes);