1/*
   2 * This program is free software; you can redistribute it and/or modify
   3 * it under the terms of the GNU General Public License version 2 as
   4 * published by the Free Software Foundation.
   5 *
   6 * This program is distributed in the hope that it will be useful,
   7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9 * GNU General Public License for more details.
  10 *
  11 * You should have received a copy of the GNU General Public License
  12 * along with this program; if not, write to the Free Software
  13 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  14 *
  15 * Copyright (C) 2009, 2010 ARM Limited
  16 *
  17 * Author: Will Deacon <will.deacon@arm.com>
  18 */
  19
  20/*
  21 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
  22 * using the CPU's debug registers.
  23 */
  24#define pr_fmt(fmt) "hw-breakpoint: " fmt
  25
  26#include <linux/errno.h>
  27#include <linux/hardirq.h>
  28#include <linux/perf_event.h>
  29#include <linux/hw_breakpoint.h>
  30#include <linux/smp.h>
  31#include <linux/cpu_pm.h>
  32#include <linux/coresight.h>
  33
  34#include <asm/cacheflush.h>
  35#include <asm/cputype.h>
  36#include <asm/current.h>
  37#include <asm/hw_breakpoint.h>
  38#include <asm/traps.h>
  39
  40/* Breakpoint currently in use for each BRP. */
  41static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
  42
  43/* Watchpoint currently in use for each WRP. */
  44static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
  45
  46/* Number of BRP/WRP registers on this CPU. */
  47static int core_num_brps;
  48static int core_num_wrps;
  49
  50/* Debug architecture version. */
  51static u8 debug_arch;
  52
  53/* Does debug architecture support OS Save and Restore? */
  54static bool has_ossr;
  55
  56/* Maximum supported watchpoint length. */
  57static u8 max_watchpoint_len;
  58
  59#define READ_WB_REG_CASE(OP2, M, VAL)			\
  60	case ((OP2 << 4) + M):				\
  61		ARM_DBG_READ(c0, c ## M, OP2, VAL);	\
  62		break
  63
  64#define WRITE_WB_REG_CASE(OP2, M, VAL)			\
  65	case ((OP2 << 4) + M):				\
  66		ARM_DBG_WRITE(c0, c ## M, OP2, VAL);	\
  67		break
  68
  69#define GEN_READ_WB_REG_CASES(OP2, VAL)		\
  70	READ_WB_REG_CASE(OP2, 0, VAL);		\
  71	READ_WB_REG_CASE(OP2, 1, VAL);		\
  72	READ_WB_REG_CASE(OP2, 2, VAL);		\
  73	READ_WB_REG_CASE(OP2, 3, VAL);		\
  74	READ_WB_REG_CASE(OP2, 4, VAL);		\
  75	READ_WB_REG_CASE(OP2, 5, VAL);		\
  76	READ_WB_REG_CASE(OP2, 6, VAL);		\
  77	READ_WB_REG_CASE(OP2, 7, VAL);		\
  78	READ_WB_REG_CASE(OP2, 8, VAL);		\
  79	READ_WB_REG_CASE(OP2, 9, VAL);		\
  80	READ_WB_REG_CASE(OP2, 10, VAL);		\
  81	READ_WB_REG_CASE(OP2, 11, VAL);		\
  82	READ_WB_REG_CASE(OP2, 12, VAL);		\
  83	READ_WB_REG_CASE(OP2, 13, VAL);		\
  84	READ_WB_REG_CASE(OP2, 14, VAL);		\
  85	READ_WB_REG_CASE(OP2, 15, VAL)
  86
  87#define GEN_WRITE_WB_REG_CASES(OP2, VAL)	\
  88	WRITE_WB_REG_CASE(OP2, 0, VAL);		\
  89	WRITE_WB_REG_CASE(OP2, 1, VAL);		\
  90	WRITE_WB_REG_CASE(OP2, 2, VAL);		\
  91	WRITE_WB_REG_CASE(OP2, 3, VAL);		\
  92	WRITE_WB_REG_CASE(OP2, 4, VAL);		\
  93	WRITE_WB_REG_CASE(OP2, 5, VAL);		\
  94	WRITE_WB_REG_CASE(OP2, 6, VAL);		\
  95	WRITE_WB_REG_CASE(OP2, 7, VAL);		\
  96	WRITE_WB_REG_CASE(OP2, 8, VAL);		\
  97	WRITE_WB_REG_CASE(OP2, 9, VAL);		\
  98	WRITE_WB_REG_CASE(OP2, 10, VAL);	\
  99	WRITE_WB_REG_CASE(OP2, 11, VAL);	\
 100	WRITE_WB_REG_CASE(OP2, 12, VAL);	\
 101	WRITE_WB_REG_CASE(OP2, 13, VAL);	\
 102	WRITE_WB_REG_CASE(OP2, 14, VAL);	\
 103	WRITE_WB_REG_CASE(OP2, 15, VAL)
 104
 105static u32 read_wb_reg(int n)
 106{
 107	u32 val = 0;
 108
 109	switch (n) {
 110	GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val);
 111	GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val);
 112	GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val);
 113	GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val);
 114	default:
 115		pr_warn("attempt to read from unknown breakpoint register %d\n",
 116			n);
 117	}
 118
 119	return val;
 120}
 121
 122static void write_wb_reg(int n, u32 val)
 123{
 124	switch (n) {
 125	GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val);
 126	GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val);
 127	GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val);
 128	GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val);
 129	default:
 130		pr_warn("attempt to write to unknown breakpoint register %d\n",
 131			n);
 132	}
 133	isb();
 134}
 135
 136/* Determine debug architecture. */
 137static u8 get_debug_arch(void)
 138{
 139	u32 didr;
 140
 141	/* Do we implement the extended CPUID interface? */
 142	if (((read_cpuid_id() >> 16) & 0xf) != 0xf) {
 143		pr_warn_once("CPUID feature registers not supported. "
 144			     "Assuming v6 debug is present.\n");
 145		return ARM_DEBUG_ARCH_V6;
 146	}
 147
 148	ARM_DBG_READ(c0, c0, 0, didr);
 149	return (didr >> 16) & 0xf;
 150}
 151
 152u8 arch_get_debug_arch(void)
 153{
 154	return debug_arch;
 155}
 156
 157static int debug_arch_supported(void)
 158{
 159	u8 arch = get_debug_arch();
 160
 161	/* We don't support the memory-mapped interface. */
 162	return (arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14) ||
 163		arch >= ARM_DEBUG_ARCH_V7_1;
 164}
 165
 166/* Can we determine the watchpoint access type from the fsr? */
 167static int debug_exception_updates_fsr(void)
 168{
 169	return get_debug_arch() >= ARM_DEBUG_ARCH_V8;
 170}
 171
 172/* Determine number of WRP registers available. */
 173static int get_num_wrp_resources(void)
 174{
 175	u32 didr;
 176	ARM_DBG_READ(c0, c0, 0, didr);
 177	return ((didr >> 28) & 0xf) + 1;
 178}
 179
 180/* Determine number of BRP registers available. */
 181static int get_num_brp_resources(void)
 182{
 183	u32 didr;
 184	ARM_DBG_READ(c0, c0, 0, didr);
 185	return ((didr >> 24) & 0xf) + 1;
 186}
 187
 188/* Does this core support mismatch breakpoints? */
 189static int core_has_mismatch_brps(void)
 190{
 191	return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 &&
 192		get_num_brp_resources() > 1);
 193}
 194
 195/* Determine number of usable WRPs available. */
 196static int get_num_wrps(void)
 197{
 198	/*
 199	 * On debug architectures prior to 7.1, when a watchpoint fires, the
 200	 * only way to work out which watchpoint it was is by disassembling
 201	 * the faulting instruction and working out the address of the memory
 202	 * access.
 203	 *
 204	 * Furthermore, we can only do this if the watchpoint was precise
 205	 * since imprecise watchpoints prevent us from calculating register
 206	 * based addresses.
 207	 *
 208	 * Providing we have more than 1 breakpoint register, we only report
 209	 * a single watchpoint register for the time being. This way, we always
 210	 * know which watchpoint fired. In the future we can either add a
 211	 * disassembler and address generation emulator, or we can insert a
 212	 * check to see if the DFAR is set on watchpoint exception entry
 213	 * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows
 214	 * that it is set on some implementations].
 215	 */
 216	if (get_debug_arch() < ARM_DEBUG_ARCH_V7_1)
 217		return 1;
 218
 219	return get_num_wrp_resources();
 220}
 221
 222/* Determine number of usable BRPs available. */
 223static int get_num_brps(void)
 224{
 225	int brps = get_num_brp_resources();
 226	return core_has_mismatch_brps() ? brps - 1 : brps;
 227}
 228
 229/*
 230 * In order to access the breakpoint/watchpoint control registers,
 231 * we must be running in debug monitor mode. Unfortunately, we can
 232 * be put into halting debug mode at any time by an external debugger
 233 * but there is nothing we can do to prevent that.
 234 */
 235static int monitor_mode_enabled(void)
 236{
 237	u32 dscr;
 238	ARM_DBG_READ(c0, c1, 0, dscr);
 239	return !!(dscr & ARM_DSCR_MDBGEN);
 240}
 241
 242static int enable_monitor_mode(void)
 243{
 244	u32 dscr;
 245	ARM_DBG_READ(c0, c1, 0, dscr);
 246
 247	/* If monitor mode is already enabled, just return. */
 248	if (dscr & ARM_DSCR_MDBGEN)
 249		goto out;
 250
 251	/* Write to the corresponding DSCR. */
 252	switch (get_debug_arch()) {
 253	case ARM_DEBUG_ARCH_V6:
 254	case ARM_DEBUG_ARCH_V6_1:
 255		ARM_DBG_WRITE(c0, c1, 0, (dscr | ARM_DSCR_MDBGEN));
 256		break;
 257	case ARM_DEBUG_ARCH_V7_ECP14:
 258	case ARM_DEBUG_ARCH_V7_1:
 259	case ARM_DEBUG_ARCH_V8:
 260		ARM_DBG_WRITE(c0, c2, 2, (dscr | ARM_DSCR_MDBGEN));
 261		isb();
 262		break;
 263	default:
 264		return -ENODEV;
 265	}
 266
 267	/* Check that the write made it through. */
 268	ARM_DBG_READ(c0, c1, 0, dscr);
 269	if (!(dscr & ARM_DSCR_MDBGEN)) {
 270		pr_warn_once("Failed to enable monitor mode on CPU %d.\n",
 271				smp_processor_id());
 272		return -EPERM;
 273	}
 274
 275out:
 276	return 0;
 277}
 278
 279int hw_breakpoint_slots(int type)
 280{
 281	if (!debug_arch_supported())
 282		return 0;
 283
 284	/*
 285	 * We can be called early, so don't rely on
 286	 * our static variables being initialised.
 287	 */
 288	switch (type) {
 289	case TYPE_INST:
 290		return get_num_brps();
 291	case TYPE_DATA:
 292		return get_num_wrps();
 293	default:
 294		pr_warn("unknown slot type: %d\n", type);
 295		return 0;
 296	}
 297}
 298
 299/*
 300 * Check if 8-bit byte-address select is available.
 301 * This clobbers WRP 0.
 302 */
 303static u8 get_max_wp_len(void)
 304{
 305	u32 ctrl_reg;
 306	struct arch_hw_breakpoint_ctrl ctrl;
 307	u8 size = 4;
 308
 309	if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14)
 310		goto out;
 311
 312	memset(&ctrl, 0, sizeof(ctrl));
 313	ctrl.len = ARM_BREAKPOINT_LEN_8;
 314	ctrl_reg = encode_ctrl_reg(ctrl);
 315
 316	write_wb_reg(ARM_BASE_WVR, 0);
 317	write_wb_reg(ARM_BASE_WCR, ctrl_reg);
 318	if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg)
 319		size = 8;
 320
 321out:
 322	return size;
 323}
 324
 325u8 arch_get_max_wp_len(void)
 326{
 327	return max_watchpoint_len;
 328}
 329
 330/*
 331 * Install a perf counter breakpoint.
 332 */
 333int arch_install_hw_breakpoint(struct perf_event *bp)
 334{
 335	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 336	struct perf_event **slot, **slots;
 337	int i, max_slots, ctrl_base, val_base;
 338	u32 addr, ctrl;
 339
 340	addr = info->address;
 341	ctrl = encode_ctrl_reg(info->ctrl) | 0x1;
 342
 343	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
 344		/* Breakpoint */
 345		ctrl_base = ARM_BASE_BCR;
 346		val_base = ARM_BASE_BVR;
 347		slots = this_cpu_ptr(bp_on_reg);
 348		max_slots = core_num_brps;
 349	} else {
 350		/* Watchpoint */
 351		ctrl_base = ARM_BASE_WCR;
 352		val_base = ARM_BASE_WVR;
 353		slots = this_cpu_ptr(wp_on_reg);
 354		max_slots = core_num_wrps;
 355	}
 356
 357	for (i = 0; i < max_slots; ++i) {
 358		slot = &slots[i];
 359
 360		if (!*slot) {
 361			*slot = bp;
 362			break;
 363		}
 364	}
 365
 366	if (i == max_slots) {
 367		pr_warn("Can't find any breakpoint slot\n");
 368		return -EBUSY;
 369	}
 370
 371	/* Override the breakpoint data with the step data. */
 372	if (info->step_ctrl.enabled) {
 373		addr = info->trigger & ~0x3;
 374		ctrl = encode_ctrl_reg(info->step_ctrl);
 375		if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE) {
 376			i = 0;
 377			ctrl_base = ARM_BASE_BCR + core_num_brps;
 378			val_base = ARM_BASE_BVR + core_num_brps;
 379		}
 380	}
 381
 382	/* Setup the address register. */
 383	write_wb_reg(val_base + i, addr);
 384
 385	/* Setup the control register. */
 386	write_wb_reg(ctrl_base + i, ctrl);
 387	return 0;
 388}
 389
 390void arch_uninstall_hw_breakpoint(struct perf_event *bp)
 391{
 392	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 393	struct perf_event **slot, **slots;
 394	int i, max_slots, base;
 395
 396	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
 397		/* Breakpoint */
 398		base = ARM_BASE_BCR;
 399		slots = this_cpu_ptr(bp_on_reg);
 400		max_slots = core_num_brps;
 401	} else {
 402		/* Watchpoint */
 403		base = ARM_BASE_WCR;
 404		slots = this_cpu_ptr(wp_on_reg);
 405		max_slots = core_num_wrps;
 406	}
 407
 408	/* Remove the breakpoint. */
 409	for (i = 0; i < max_slots; ++i) {
 410		slot = &slots[i];
 411
 412		if (*slot == bp) {
 413			*slot = NULL;
 414			break;
 415		}
 416	}
 417
 418	if (i == max_slots) {
 419		pr_warn("Can't find any breakpoint slot\n");
 420		return;
 421	}
 422
 423	/* Ensure that we disable the mismatch breakpoint. */
 424	if (info->ctrl.type != ARM_BREAKPOINT_EXECUTE &&
 425	    info->step_ctrl.enabled) {
 426		i = 0;
 427		base = ARM_BASE_BCR + core_num_brps;
 428	}
 429
 430	/* Reset the control register. */
 431	write_wb_reg(base + i, 0);
 432}
 433
 434static int get_hbp_len(u8 hbp_len)
 435{
 436	unsigned int len_in_bytes = 0;
 437
 438	switch (hbp_len) {
 439	case ARM_BREAKPOINT_LEN_1:
 440		len_in_bytes = 1;
 441		break;
 442	case ARM_BREAKPOINT_LEN_2:
 443		len_in_bytes = 2;
 444		break;
 445	case ARM_BREAKPOINT_LEN_4:
 446		len_in_bytes = 4;
 447		break;
 448	case ARM_BREAKPOINT_LEN_8:
 449		len_in_bytes = 8;
 450		break;
 451	}
 452
 453	return len_in_bytes;
 454}
 455
 456/*
 457 * Check whether bp virtual address is in kernel space.
 458 */
 459int arch_check_bp_in_kernelspace(struct perf_event *bp)
 460{
 461	unsigned int len;
 462	unsigned long va;
 463	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 464
 465	va = info->address;
 466	len = get_hbp_len(info->ctrl.len);
 467
 468	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
 469}
 470
 471/*
 472 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
 473 * Hopefully this will disappear when ptrace can bypass the conversion
 474 * to generic breakpoint descriptions.
 475 */
 476int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
 477			   int *gen_len, int *gen_type)
 478{
 479	/* Type */
 480	switch (ctrl.type) {
 481	case ARM_BREAKPOINT_EXECUTE:
 482		*gen_type = HW_BREAKPOINT_X;
 483		break;
 484	case ARM_BREAKPOINT_LOAD:
 485		*gen_type = HW_BREAKPOINT_R;
 486		break;
 487	case ARM_BREAKPOINT_STORE:
 488		*gen_type = HW_BREAKPOINT_W;
 489		break;
 490	case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
 491		*gen_type = HW_BREAKPOINT_RW;
 492		break;
 493	default:
 494		return -EINVAL;
 495	}
 496
 497	/* Len */
 498	switch (ctrl.len) {
 499	case ARM_BREAKPOINT_LEN_1:
 500		*gen_len = HW_BREAKPOINT_LEN_1;
 501		break;
 502	case ARM_BREAKPOINT_LEN_2:
 503		*gen_len = HW_BREAKPOINT_LEN_2;
 504		break;
 505	case ARM_BREAKPOINT_LEN_4:
 506		*gen_len = HW_BREAKPOINT_LEN_4;
 507		break;
 508	case ARM_BREAKPOINT_LEN_8:
 509		*gen_len = HW_BREAKPOINT_LEN_8;
 510		break;
 511	default:
 512		return -EINVAL;
 513	}
 514
 515	return 0;
 516}
 517
 518/*
 519 * Construct an arch_hw_breakpoint from a perf_event.
 520 */
 521static int arch_build_bp_info(struct perf_event *bp)
 522{
 523	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 524
 525	/* Type */
 526	switch (bp->attr.bp_type) {
 527	case HW_BREAKPOINT_X:
 528		info->ctrl.type = ARM_BREAKPOINT_EXECUTE;
 529		break;
 530	case HW_BREAKPOINT_R:
 531		info->ctrl.type = ARM_BREAKPOINT_LOAD;
 532		break;
 533	case HW_BREAKPOINT_W:
 534		info->ctrl.type = ARM_BREAKPOINT_STORE;
 535		break;
 536	case HW_BREAKPOINT_RW:
 537		info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
 538		break;
 539	default:
 540		return -EINVAL;
 541	}
 542
 543	/* Len */
 544	switch (bp->attr.bp_len) {
 545	case HW_BREAKPOINT_LEN_1:
 546		info->ctrl.len = ARM_BREAKPOINT_LEN_1;
 547		break;
 548	case HW_BREAKPOINT_LEN_2:
 549		info->ctrl.len = ARM_BREAKPOINT_LEN_2;
 550		break;
 551	case HW_BREAKPOINT_LEN_4:
 552		info->ctrl.len = ARM_BREAKPOINT_LEN_4;
 553		break;
 554	case HW_BREAKPOINT_LEN_8:
 555		info->ctrl.len = ARM_BREAKPOINT_LEN_8;
 556		if ((info->ctrl.type != ARM_BREAKPOINT_EXECUTE)
 557			&& max_watchpoint_len >= 8)
 558			break;
 559	default:
 560		return -EINVAL;
 561	}
 562
 563	/*
 564	 * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes.
 565	 * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported
 566	 * by the hardware and must be aligned to the appropriate number of
 567	 * bytes.
 568	 */
 569	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE &&
 570	    info->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
 571	    info->ctrl.len != ARM_BREAKPOINT_LEN_4)
 572		return -EINVAL;
 573
 574	/* Address */
 575	info->address = bp->attr.bp_addr;
 576
 577	/* Privilege */
 578	info->ctrl.privilege = ARM_BREAKPOINT_USER;
 579	if (arch_check_bp_in_kernelspace(bp))
 580		info->ctrl.privilege |= ARM_BREAKPOINT_PRIV;
 581
 582	/* Enabled? */
 583	info->ctrl.enabled = !bp->attr.disabled;
 584
 585	/* Mismatch */
 586	info->ctrl.mismatch = 0;
 587
 588	return 0;
 589}
 590
 591/*
 592 * Validate the arch-specific HW Breakpoint register settings.
 593 */
 594int arch_validate_hwbkpt_settings(struct perf_event *bp)
 595{
 596	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 597	int ret = 0;
 598	u32 offset, alignment_mask = 0x3;
 599
 600	/* Ensure that we are in monitor debug mode. */
 601	if (!monitor_mode_enabled())
 602		return -ENODEV;
 603
 604	/* Build the arch_hw_breakpoint. */
 605	ret = arch_build_bp_info(bp);
 606	if (ret)
 607		goto out;
 608
 609	/* Check address alignment. */
 610	if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
 611		alignment_mask = 0x7;
 612	offset = info->address & alignment_mask;
 613	switch (offset) {
 614	case 0:
 615		/* Aligned */
 616		break;
 617	case 1:
 618	case 2:
 619		/* Allow halfword watchpoints and breakpoints. */
 620		if (info->ctrl.len == ARM_BREAKPOINT_LEN_2)
 621			break;
 622	case 3:
 623		/* Allow single byte watchpoint. */
 624		if (info->ctrl.len == ARM_BREAKPOINT_LEN_1)
 625			break;
 626	default:
 627		ret = -EINVAL;
 628		goto out;
 629	}
 630
 631	info->address &= ~alignment_mask;
 632	info->ctrl.len <<= offset;
 633
 634	if (!bp->overflow_handler) {
 635		/*
 636		 * Mismatch breakpoints are required for single-stepping
 637		 * breakpoints.
 638		 */
 639		if (!core_has_mismatch_brps())
 640			return -EINVAL;
 641
 642		/* We don't allow mismatch breakpoints in kernel space. */
 643		if (arch_check_bp_in_kernelspace(bp))
 644			return -EPERM;
 645
 646		/*
 647		 * Per-cpu breakpoints are not supported by our stepping
 648		 * mechanism.
 649		 */
 650		if (!bp->hw.target)
 651			return -EINVAL;
 652
 653		/*
 654		 * We only support specific access types if the fsr
 655		 * reports them.
 656		 */
 657		if (!debug_exception_updates_fsr() &&
 658		    (info->ctrl.type == ARM_BREAKPOINT_LOAD ||
 659		     info->ctrl.type == ARM_BREAKPOINT_STORE))
 660			return -EINVAL;
 661	}
 662
 663out:
 664	return ret;
 665}
 666
 667/*
 668 * Enable/disable single-stepping over the breakpoint bp at address addr.
 669 */
 670static void enable_single_step(struct perf_event *bp, u32 addr)
 671{
 672	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
 673
 674	arch_uninstall_hw_breakpoint(bp);
 675	info->step_ctrl.mismatch  = 1;
 676	info->step_ctrl.len	  = ARM_BREAKPOINT_LEN_4;
 677	info->step_ctrl.type	  = ARM_BREAKPOINT_EXECUTE;
 678	info->step_ctrl.privilege = info->ctrl.privilege;
 679	info->step_ctrl.enabled	  = 1;
 680	info->trigger		  = addr;
 681	arch_install_hw_breakpoint(bp);
 682}
 683
 684static void disable_single_step(struct perf_event *bp)
 685{
 686	arch_uninstall_hw_breakpoint(bp);
 687	counter_arch_bp(bp)->step_ctrl.enabled = 0;
 688	arch_install_hw_breakpoint(bp);
 689}
 690
 691static void watchpoint_handler(unsigned long addr, unsigned int fsr,
 692			       struct pt_regs *regs)
 693{
 694	int i, access;
 695	u32 val, ctrl_reg, alignment_mask;
 696	struct perf_event *wp, **slots;
 697	struct arch_hw_breakpoint *info;
 698	struct arch_hw_breakpoint_ctrl ctrl;
 699
 700	slots = this_cpu_ptr(wp_on_reg);
 701
 702	for (i = 0; i < core_num_wrps; ++i) {
 703		rcu_read_lock();
 704
 705		wp = slots[i];
 706
 707		if (wp == NULL)
 708			goto unlock;
 709
 710		info = counter_arch_bp(wp);
 711		/*
 712		 * The DFAR is an unknown value on debug architectures prior
 713		 * to 7.1. Since we only allow a single watchpoint on these
 714		 * older CPUs, we can set the trigger to the lowest possible
 715		 * faulting address.
 716		 */
 717		if (debug_arch < ARM_DEBUG_ARCH_V7_1) {
 718			BUG_ON(i > 0);
 719			info->trigger = wp->attr.bp_addr;
 720		} else {
 721			if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
 722				alignment_mask = 0x7;
 723			else
 724				alignment_mask = 0x3;
 725
 726			/* Check if the watchpoint value matches. */
 727			val = read_wb_reg(ARM_BASE_WVR + i);
 728			if (val != (addr & ~alignment_mask))
 729				goto unlock;
 730
 731			/* Possible match, check the byte address select. */
 732			ctrl_reg = read_wb_reg(ARM_BASE_WCR + i);
 733			decode_ctrl_reg(ctrl_reg, &ctrl);
 734			if (!((1 << (addr & alignment_mask)) & ctrl.len))
 735				goto unlock;
 736
 737			/* Check that the access type matches. */
 738			if (debug_exception_updates_fsr()) {
 739				access = (fsr & ARM_FSR_ACCESS_MASK) ?
 740					  HW_BREAKPOINT_W : HW_BREAKPOINT_R;
 741				if (!(access & hw_breakpoint_type(wp)))
 742					goto unlock;
 743			}
 744
 745			/* We have a winner. */
 746			info->trigger = addr;
 747		}
 748
 749		pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);
 750		perf_bp_event(wp, regs);
 751
 752		/*
 753		 * If no overflow handler is present, insert a temporary
 754		 * mismatch breakpoint so we can single-step over the
 755		 * watchpoint trigger.
 756		 */
 757		if (!wp->overflow_handler)
 758			enable_single_step(wp, instruction_pointer(regs));
 759
 760unlock:
 761		rcu_read_unlock();
 762	}
 763}
 764
 765static void watchpoint_single_step_handler(unsigned long pc)
 766{
 767	int i;
 768	struct perf_event *wp, **slots;
 769	struct arch_hw_breakpoint *info;
 770
 771	slots = this_cpu_ptr(wp_on_reg);
 772
 773	for (i = 0; i < core_num_wrps; ++i) {
 774		rcu_read_lock();
 775
 776		wp = slots[i];
 777
 778		if (wp == NULL)
 779			goto unlock;
 780
 781		info = counter_arch_bp(wp);
 782		if (!info->step_ctrl.enabled)
 783			goto unlock;
 784
 785		/*
 786		 * Restore the original watchpoint if we've completed the
 787		 * single-step.
 788		 */
 789		if (info->trigger != pc)
 790			disable_single_step(wp);
 791
 792unlock:
 793		rcu_read_unlock();
 794	}
 795}
 796
 797static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs)
 798{
 799	int i;
 800	u32 ctrl_reg, val, addr;
 801	struct perf_event *bp, **slots;
 802	struct arch_hw_breakpoint *info;
 803	struct arch_hw_breakpoint_ctrl ctrl;
 804
 805	slots = this_cpu_ptr(bp_on_reg);
 806
 807	/* The exception entry code places the amended lr in the PC. */
 808	addr = regs->ARM_pc;
 809
 810	/* Check the currently installed breakpoints first. */
 811	for (i = 0; i < core_num_brps; ++i) {
 812		rcu_read_lock();
 813
 814		bp = slots[i];
 815
 816		if (bp == NULL)
 817			goto unlock;
 818
 819		info = counter_arch_bp(bp);
 820
 821		/* Check if the breakpoint value matches. */
 822		val = read_wb_reg(ARM_BASE_BVR + i);
 823		if (val != (addr & ~0x3))
 824			goto mismatch;
 825
 826		/* Possible match, check the byte address select to confirm. */
 827		ctrl_reg = read_wb_reg(ARM_BASE_BCR + i);
 828		decode_ctrl_reg(ctrl_reg, &ctrl);
 829		if ((1 << (addr & 0x3)) & ctrl.len) {
 830			info->trigger = addr;
 831			pr_debug("breakpoint fired: address = 0x%x\n", addr);
 832			perf_bp_event(bp, regs);
 833			if (!bp->overflow_handler)
 834				enable_single_step(bp, addr);
 835			goto unlock;
 836		}
 837
 838mismatch:
 839		/* If we're stepping a breakpoint, it can now be restored. */
 840		if (info->step_ctrl.enabled)
 841			disable_single_step(bp);
 842unlock:
 843		rcu_read_unlock();
 844	}
 845
 846	/* Handle any pending watchpoint single-step breakpoints. */
 847	watchpoint_single_step_handler(addr);
 848}
 849
 850/*
 851 * Called from either the Data Abort Handler [watchpoint] or the
 852 * Prefetch Abort Handler [breakpoint] with interrupts disabled.
 853 */
 854static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr,
 855				 struct pt_regs *regs)
 856{
 857	int ret = 0;
 858	u32 dscr;
 859
 860	preempt_disable();
 861
 862	if (interrupts_enabled(regs))
 863		local_irq_enable();
 864
 865	/* We only handle watchpoints and hardware breakpoints. */
 866	ARM_DBG_READ(c0, c1, 0, dscr);
 867
 868	/* Perform perf callbacks. */
 869	switch (ARM_DSCR_MOE(dscr)) {
 870	case ARM_ENTRY_BREAKPOINT:
 871		breakpoint_handler(addr, regs);
 872		break;
 873	case ARM_ENTRY_ASYNC_WATCHPOINT:
 874		WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n");
 875	case ARM_ENTRY_SYNC_WATCHPOINT:
 876		watchpoint_handler(addr, fsr, regs);
 877		break;
 878	default:
 879		ret = 1; /* Unhandled fault. */
 880	}
 881
 882	preempt_enable();
 883
 884	return ret;
 885}
 886
 887/*
 888 * One-time initialisation.
 889 */
 890static cpumask_t debug_err_mask;
 891
 892static int debug_reg_trap(struct pt_regs *regs, unsigned int instr)
 893{
 894	int cpu = smp_processor_id();
 895
 896	pr_warn("Debug register access (0x%x) caused undefined instruction on CPU %d\n",
 897		instr, cpu);
 898
 899	/* Set the error flag for this CPU and skip the faulting instruction. */
 900	cpumask_set_cpu(cpu, &debug_err_mask);
 901	instruction_pointer(regs) += 4;
 902	return 0;
 903}
 904
 905static struct undef_hook debug_reg_hook = {
 906	.instr_mask	= 0x0fe80f10,
 907	.instr_val	= 0x0e000e10,
 908	.fn		= debug_reg_trap,
 909};
 910
 911/* Does this core support OS Save and Restore? */
 912static bool core_has_os_save_restore(void)
 913{
 914	u32 oslsr;
 915
 916	switch (get_debug_arch()) {
 917	case ARM_DEBUG_ARCH_V7_1:
 918		return true;
 919	case ARM_DEBUG_ARCH_V7_ECP14:
 920		ARM_DBG_READ(c1, c1, 4, oslsr);
 921		if (oslsr & ARM_OSLSR_OSLM0)
 922			return true;
 923	default:
 924		return false;
 925	}
 926}
 927
 928static void reset_ctrl_regs(void *unused)
 929{
 930	int i, raw_num_brps, err = 0, cpu = smp_processor_id();
 931	u32 val;
 932
 933	/*
 934	 * v7 debug contains save and restore registers so that debug state
 935	 * can be maintained across low-power modes without leaving the debug
 936	 * logic powered up. It is IMPLEMENTATION DEFINED whether we can access
 937	 * the debug registers out of reset, so we must unlock the OS Lock
 938	 * Access Register to avoid taking undefined instruction exceptions
 939	 * later on.
 940	 */
 941	switch (debug_arch) {
 942	case ARM_DEBUG_ARCH_V6:
 943	case ARM_DEBUG_ARCH_V6_1:
 944		/* ARMv6 cores clear the registers out of reset. */
 945		goto out_mdbgen;
 946	case ARM_DEBUG_ARCH_V7_ECP14:
 947		/*
 948		 * Ensure sticky power-down is clear (i.e. debug logic is
 949		 * powered up).
 950		 */
 951		ARM_DBG_READ(c1, c5, 4, val);
 952		if ((val & 0x1) == 0)
 953			err = -EPERM;
 954
 955		if (!has_ossr)
 956			goto clear_vcr;
 957		break;
 958	case ARM_DEBUG_ARCH_V7_1:
 959		/*
 960		 * Ensure the OS double lock is clear.
 961		 */
 962		ARM_DBG_READ(c1, c3, 4, val);
 963		if ((val & 0x1) == 1)
 964			err = -EPERM;
 965		break;
 966	}
 967
 968	if (err) {
 969		pr_warn_once("CPU %d debug is powered down!\n", cpu);
 970		cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
 971		return;
 972	}
 973
 974	/*
 975	 * Unconditionally clear the OS lock by writing a value
 976	 * other than CS_LAR_KEY to the access register.
 977	 */
 978	ARM_DBG_WRITE(c1, c0, 4, ~CORESIGHT_UNLOCK);
 979	isb();
 980
 981	/*
 982	 * Clear any configured vector-catch events before
 983	 * enabling monitor mode.
 984	 */
 985clear_vcr:
 986	ARM_DBG_WRITE(c0, c7, 0, 0);
 987	isb();
 988
 989	if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
 990		pr_warn_once("CPU %d failed to disable vector catch\n", cpu);
 991		return;
 992	}
 993
 994	/*
 995	 * The control/value register pairs are UNKNOWN out of reset so
 996	 * clear them to avoid spurious debug events.
 997	 */
 998	raw_num_brps = get_num_brp_resources();
 999	for (i = 0; i < raw_num_brps; ++i) {
1000		write_wb_reg(ARM_BASE_BCR + i, 0UL);
1001		write_wb_reg(ARM_BASE_BVR + i, 0UL);
1002	}
1003
1004	for (i = 0; i < core_num_wrps; ++i) {
1005		write_wb_reg(ARM_BASE_WCR + i, 0UL);
1006		write_wb_reg(ARM_BASE_WVR + i, 0UL);
1007	}
1008
1009	if (cpumask_intersects(&debug_err_mask, cpumask_of(cpu))) {
1010		pr_warn_once("CPU %d failed to clear debug register pairs\n", cpu);
1011		return;
1012	}
1013
1014	/*
1015	 * Have a crack at enabling monitor mode. We don't actually need
1016	 * it yet, but reporting an error early is useful if it fails.
1017	 */
1018out_mdbgen:
1019	if (enable_monitor_mode())
1020		cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
1021}
1022
1023static int dbg_reset_notify(struct notifier_block *self,
1024				      unsigned long action, void *cpu)
1025{
1026	if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE)
1027		smp_call_function_single((int)cpu, reset_ctrl_regs, NULL, 1);
1028
1029	return NOTIFY_OK;
1030}
1031
1032static struct notifier_block dbg_reset_nb = {
1033	.notifier_call = dbg_reset_notify,
1034};
1035
1036#ifdef CONFIG_CPU_PM
1037static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action,
1038			     void *v)
1039{
1040	if (action == CPU_PM_EXIT)
1041		reset_ctrl_regs(NULL);
1042
1043	return NOTIFY_OK;
1044}
1045
1046static struct notifier_block dbg_cpu_pm_nb = {
1047	.notifier_call = dbg_cpu_pm_notify,
1048};
1049
1050static void __init pm_init(void)
1051{
1052	cpu_pm_register_notifier(&dbg_cpu_pm_nb);
1053}
1054#else
1055static inline void pm_init(void)
1056{
1057}
1058#endif
1059
1060static int __init arch_hw_breakpoint_init(void)
1061{
 
 
1062	debug_arch = get_debug_arch();
1063
1064	if (!debug_arch_supported()) {
1065		pr_info("debug architecture 0x%x unsupported.\n", debug_arch);
1066		return 0;
1067	}
1068
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1069	has_ossr = core_has_os_save_restore();
1070
1071	/* Determine how many BRPs/WRPs are available. */
1072	core_num_brps = get_num_brps();
1073	core_num_wrps = get_num_wrps();
1074
1075	cpu_notifier_register_begin();
1076
1077	/*
1078	 * We need to tread carefully here because DBGSWENABLE may be
1079	 * driven low on this core and there isn't an architected way to
1080	 * determine that.
1081	 */
 
1082	register_undef_hook(&debug_reg_hook);
1083
1084	/*
1085	 * Reset the breakpoint resources. We assume that a halting
1086	 * debugger will leave the world in a nice state for us.
 
1087	 */
1088	on_each_cpu(reset_ctrl_regs, NULL, 1);
 
 
1089	unregister_undef_hook(&debug_reg_hook);
1090	if (!cpumask_empty(&debug_err_mask)) {
1091		core_num_brps = 0;
1092		core_num_wrps = 0;
1093		cpu_notifier_register_done();
 
 
1094		return 0;
1095	}
1096
1097	pr_info("found %d " "%s" "breakpoint and %d watchpoint registers.\n",
1098		core_num_brps, core_has_mismatch_brps() ? "(+1 reserved) " :
1099		"", core_num_wrps);
1100
1101	/* Work out the maximum supported watchpoint length. */
1102	max_watchpoint_len = get_max_wp_len();
1103	pr_info("maximum watchpoint size is %u bytes.\n",
1104			max_watchpoint_len);
1105
1106	/* Register debug fault handler. */
1107	hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1108			TRAP_HWBKPT, "watchpoint debug exception");
1109	hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1110			TRAP_HWBKPT, "breakpoint debug exception");
 
1111
1112	/* Register hotplug and PM notifiers. */
1113	__register_cpu_notifier(&dbg_reset_nb);
1114
1115	cpu_notifier_register_done();
1116
1117	pm_init();
1118	return 0;
1119}
1120arch_initcall(arch_hw_breakpoint_init);
1121
1122void hw_breakpoint_pmu_read(struct perf_event *bp)
1123{
1124}
1125
1126/*
1127 * Dummy function to register with die_notifier.
1128 */
1129int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1130					unsigned long val, void *data)
1131{
1132	return NOTIFY_DONE;
1133}