1/*
   2 * lib/locking-selftest.c
   3 *
   4 * Testsuite for various locking APIs: spinlocks, rwlocks,
   5 * mutexes and rw-semaphores.
   6 *
   7 * It is checking both false positives and false negatives.
   8 *
   9 * Started by Ingo Molnar:
  10 *
  11 *  Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  12 */
  13#include <linux/rwsem.h>
  14#include <linux/mutex.h>
  15#include <linux/ww_mutex.h>
  16#include <linux/sched.h>
  17#include <linux/delay.h>
  18#include <linux/lockdep.h>
  19#include <linux/spinlock.h>
  20#include <linux/kallsyms.h>
  21#include <linux/interrupt.h>
  22#include <linux/debug_locks.h>
  23#include <linux/irqflags.h>
  24
  25/*
  26 * Change this to 1 if you want to see the failure printouts:
  27 */
  28static unsigned int debug_locks_verbose;
  29
  30static DEFINE_WW_CLASS(ww_lockdep);
  31
  32static int __init setup_debug_locks_verbose(char *str)
  33{
  34	get_option(&str, &debug_locks_verbose);
  35
  36	return 1;
  37}
  38
  39__setup("debug_locks_verbose=", setup_debug_locks_verbose);
  40
  41#define FAILURE		0
  42#define SUCCESS		1
  43
  44#define LOCKTYPE_SPIN	0x1
  45#define LOCKTYPE_RWLOCK	0x2
  46#define LOCKTYPE_MUTEX	0x4
  47#define LOCKTYPE_RWSEM	0x8
  48#define LOCKTYPE_WW	0x10
  49
  50static struct ww_acquire_ctx t, t2;
  51static struct ww_mutex o, o2, o3;
  52
  53/*
  54 * Normal standalone locks, for the circular and irq-context
  55 * dependency tests:
  56 */
  57static DEFINE_RAW_SPINLOCK(lock_A);
  58static DEFINE_RAW_SPINLOCK(lock_B);
  59static DEFINE_RAW_SPINLOCK(lock_C);
  60static DEFINE_RAW_SPINLOCK(lock_D);
  61
  62static DEFINE_RWLOCK(rwlock_A);
  63static DEFINE_RWLOCK(rwlock_B);
  64static DEFINE_RWLOCK(rwlock_C);
  65static DEFINE_RWLOCK(rwlock_D);
  66
  67static DEFINE_MUTEX(mutex_A);
  68static DEFINE_MUTEX(mutex_B);
  69static DEFINE_MUTEX(mutex_C);
  70static DEFINE_MUTEX(mutex_D);
  71
  72static DECLARE_RWSEM(rwsem_A);
  73static DECLARE_RWSEM(rwsem_B);
  74static DECLARE_RWSEM(rwsem_C);
  75static DECLARE_RWSEM(rwsem_D);
  76
  77/*
  78 * Locks that we initialize dynamically as well so that
  79 * e.g. X1 and X2 becomes two instances of the same class,
  80 * but X* and Y* are different classes. We do this so that
  81 * we do not trigger a real lockup:
  82 */
  83static DEFINE_RAW_SPINLOCK(lock_X1);
  84static DEFINE_RAW_SPINLOCK(lock_X2);
  85static DEFINE_RAW_SPINLOCK(lock_Y1);
  86static DEFINE_RAW_SPINLOCK(lock_Y2);
  87static DEFINE_RAW_SPINLOCK(lock_Z1);
  88static DEFINE_RAW_SPINLOCK(lock_Z2);
  89
  90static DEFINE_RWLOCK(rwlock_X1);
  91static DEFINE_RWLOCK(rwlock_X2);
  92static DEFINE_RWLOCK(rwlock_Y1);
  93static DEFINE_RWLOCK(rwlock_Y2);
  94static DEFINE_RWLOCK(rwlock_Z1);
  95static DEFINE_RWLOCK(rwlock_Z2);
  96
  97static DEFINE_MUTEX(mutex_X1);
  98static DEFINE_MUTEX(mutex_X2);
  99static DEFINE_MUTEX(mutex_Y1);
 100static DEFINE_MUTEX(mutex_Y2);
 101static DEFINE_MUTEX(mutex_Z1);
 102static DEFINE_MUTEX(mutex_Z2);
 103
 104static DECLARE_RWSEM(rwsem_X1);
 105static DECLARE_RWSEM(rwsem_X2);
 106static DECLARE_RWSEM(rwsem_Y1);
 107static DECLARE_RWSEM(rwsem_Y2);
 108static DECLARE_RWSEM(rwsem_Z1);
 109static DECLARE_RWSEM(rwsem_Z2);
 110
 111/*
 112 * non-inlined runtime initializers, to let separate locks share
 113 * the same lock-class:
 114 */
 115#define INIT_CLASS_FUNC(class) 				\
 116static noinline void					\
 117init_class_##class(raw_spinlock_t *lock, rwlock_t *rwlock, \
 118	struct mutex *mutex, struct rw_semaphore *rwsem)\
 119{							\
 120	raw_spin_lock_init(lock);			\
 121	rwlock_init(rwlock);				\
 122	mutex_init(mutex);				\
 123	init_rwsem(rwsem);				\
 124}
 125
 126INIT_CLASS_FUNC(X)
 127INIT_CLASS_FUNC(Y)
 128INIT_CLASS_FUNC(Z)
 129
 130static void init_shared_classes(void)
 131{
 132	init_class_X(&lock_X1, &rwlock_X1, &mutex_X1, &rwsem_X1);
 133	init_class_X(&lock_X2, &rwlock_X2, &mutex_X2, &rwsem_X2);
 134
 135	init_class_Y(&lock_Y1, &rwlock_Y1, &mutex_Y1, &rwsem_Y1);
 136	init_class_Y(&lock_Y2, &rwlock_Y2, &mutex_Y2, &rwsem_Y2);
 137
 138	init_class_Z(&lock_Z1, &rwlock_Z1, &mutex_Z1, &rwsem_Z1);
 139	init_class_Z(&lock_Z2, &rwlock_Z2, &mutex_Z2, &rwsem_Z2);
 140}
 141
 142/*
 143 * For spinlocks and rwlocks we also do hardirq-safe / softirq-safe tests.
 144 * The following functions use a lock from a simulated hardirq/softirq
 145 * context, causing the locks to be marked as hardirq-safe/softirq-safe:
 146 */
 147
 148#define HARDIRQ_DISABLE		local_irq_disable
 149#define HARDIRQ_ENABLE		local_irq_enable
 150
 151#define HARDIRQ_ENTER()				\
 152	local_irq_disable();			\
 153	__irq_enter();				\
 154	WARN_ON(!in_irq());
 155
 156#define HARDIRQ_EXIT()				\
 157	__irq_exit();				\
 158	local_irq_enable();
 159
 160#define SOFTIRQ_DISABLE		local_bh_disable
 161#define SOFTIRQ_ENABLE		local_bh_enable
 162
 163#define SOFTIRQ_ENTER()				\
 164		local_bh_disable();		\
 165		local_irq_disable();		\
 166		lockdep_softirq_enter();	\
 167		WARN_ON(!in_softirq());
 168
 169#define SOFTIRQ_EXIT()				\
 170		lockdep_softirq_exit();		\
 171		local_irq_enable();		\
 172		local_bh_enable();
 173
 174/*
 175 * Shortcuts for lock/unlock API variants, to keep
 176 * the testcases compact:
 177 */
 178#define L(x)			raw_spin_lock(&lock_##x)
 179#define U(x)			raw_spin_unlock(&lock_##x)
 180#define LU(x)			L(x); U(x)
 181#define SI(x)			raw_spin_lock_init(&lock_##x)
 182
 183#define WL(x)			write_lock(&rwlock_##x)
 184#define WU(x)			write_unlock(&rwlock_##x)
 185#define WLU(x)			WL(x); WU(x)
 186
 187#define RL(x)			read_lock(&rwlock_##x)
 188#define RU(x)			read_unlock(&rwlock_##x)
 189#define RLU(x)			RL(x); RU(x)
 190#define RWI(x)			rwlock_init(&rwlock_##x)
 191
 192#define ML(x)			mutex_lock(&mutex_##x)
 193#define MU(x)			mutex_unlock(&mutex_##x)
 194#define MI(x)			mutex_init(&mutex_##x)
 195
 196#define WSL(x)			down_write(&rwsem_##x)
 197#define WSU(x)			up_write(&rwsem_##x)
 198
 199#define RSL(x)			down_read(&rwsem_##x)
 200#define RSU(x)			up_read(&rwsem_##x)
 201#define RWSI(x)			init_rwsem(&rwsem_##x)
 202
 203#ifndef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
 204#define WWAI(x)			ww_acquire_init(x, &ww_lockdep)
 205#else
 206#define WWAI(x)			do { ww_acquire_init(x, &ww_lockdep); (x)->deadlock_inject_countdown = ~0U; } while (0)
 207#endif
 208#define WWAD(x)			ww_acquire_done(x)
 209#define WWAF(x)			ww_acquire_fini(x)
 210
 211#define WWL(x, c)		ww_mutex_lock(x, c)
 212#define WWT(x)			ww_mutex_trylock(x)
 213#define WWL1(x)			ww_mutex_lock(x, NULL)
 214#define WWU(x)			ww_mutex_unlock(x)
 215
 216
 217#define LOCK_UNLOCK_2(x,y)	LOCK(x); LOCK(y); UNLOCK(y); UNLOCK(x)
 218
 219/*
 220 * Generate different permutations of the same testcase, using
 221 * the same basic lock-dependency/state events:
 222 */
 223
 224#define GENERATE_TESTCASE(name)			\
 225						\
 226static void name(void) { E(); }
 227
 228#define GENERATE_PERMUTATIONS_2_EVENTS(name)	\
 229						\
 230static void name##_12(void) { E1(); E2(); }	\
 231static void name##_21(void) { E2(); E1(); }
 232
 233#define GENERATE_PERMUTATIONS_3_EVENTS(name)		\
 234							\
 235static void name##_123(void) { E1(); E2(); E3(); }	\
 236static void name##_132(void) { E1(); E3(); E2(); }	\
 237static void name##_213(void) { E2(); E1(); E3(); }	\
 238static void name##_231(void) { E2(); E3(); E1(); }	\
 239static void name##_312(void) { E3(); E1(); E2(); }	\
 240static void name##_321(void) { E3(); E2(); E1(); }
 241
 242/*
 243 * AA deadlock:
 244 */
 245
 246#define E()					\
 247						\
 248	LOCK(X1);				\
 249	LOCK(X2); /* this one should fail */
 250
 251/*
 252 * 6 testcases:
 253 */
 254#include "locking-selftest-spin.h"
 255GENERATE_TESTCASE(AA_spin)
 256#include "locking-selftest-wlock.h"
 257GENERATE_TESTCASE(AA_wlock)
 258#include "locking-selftest-rlock.h"
 259GENERATE_TESTCASE(AA_rlock)
 260#include "locking-selftest-mutex.h"
 261GENERATE_TESTCASE(AA_mutex)
 262#include "locking-selftest-wsem.h"
 263GENERATE_TESTCASE(AA_wsem)
 264#include "locking-selftest-rsem.h"
 265GENERATE_TESTCASE(AA_rsem)
 266
 267#undef E
 268
 269/*
 270 * Special-case for read-locking, they are
 271 * allowed to recurse on the same lock class:
 272 */
 273static void rlock_AA1(void)
 274{
 275	RL(X1);
 276	RL(X1); // this one should NOT fail
 277}
 278
 279static void rlock_AA1B(void)
 280{
 281	RL(X1);
 282	RL(X2); // this one should NOT fail
 283}
 284
 285static void rsem_AA1(void)
 286{
 287	RSL(X1);
 288	RSL(X1); // this one should fail
 289}
 290
 291static void rsem_AA1B(void)
 292{
 293	RSL(X1);
 294	RSL(X2); // this one should fail
 295}
 296/*
 297 * The mixing of read and write locks is not allowed:
 298 */
 299static void rlock_AA2(void)
 300{
 301	RL(X1);
 302	WL(X2); // this one should fail
 303}
 304
 305static void rsem_AA2(void)
 306{
 307	RSL(X1);
 308	WSL(X2); // this one should fail
 309}
 310
 311static void rlock_AA3(void)
 312{
 313	WL(X1);
 314	RL(X2); // this one should fail
 315}
 316
 317static void rsem_AA3(void)
 318{
 319	WSL(X1);
 320	RSL(X2); // this one should fail
 321}
 322
 323/*
 324 * ABBA deadlock:
 325 */
 326
 327#define E()					\
 328						\
 329	LOCK_UNLOCK_2(A, B);			\
 330	LOCK_UNLOCK_2(B, A); /* fail */
 331
 332/*
 333 * 6 testcases:
 334 */
 335#include "locking-selftest-spin.h"
 336GENERATE_TESTCASE(ABBA_spin)
 337#include "locking-selftest-wlock.h"
 338GENERATE_TESTCASE(ABBA_wlock)
 339#include "locking-selftest-rlock.h"
 340GENERATE_TESTCASE(ABBA_rlock)
 341#include "locking-selftest-mutex.h"
 342GENERATE_TESTCASE(ABBA_mutex)
 343#include "locking-selftest-wsem.h"
 344GENERATE_TESTCASE(ABBA_wsem)
 345#include "locking-selftest-rsem.h"
 346GENERATE_TESTCASE(ABBA_rsem)
 347
 348#undef E
 349
 350/*
 351 * AB BC CA deadlock:
 352 */
 353
 354#define E()					\
 355						\
 356	LOCK_UNLOCK_2(A, B);			\
 357	LOCK_UNLOCK_2(B, C);			\
 358	LOCK_UNLOCK_2(C, A); /* fail */
 359
 360/*
 361 * 6 testcases:
 362 */
 363#include "locking-selftest-spin.h"
 364GENERATE_TESTCASE(ABBCCA_spin)
 365#include "locking-selftest-wlock.h"
 366GENERATE_TESTCASE(ABBCCA_wlock)
 367#include "locking-selftest-rlock.h"
 368GENERATE_TESTCASE(ABBCCA_rlock)
 369#include "locking-selftest-mutex.h"
 370GENERATE_TESTCASE(ABBCCA_mutex)
 371#include "locking-selftest-wsem.h"
 372GENERATE_TESTCASE(ABBCCA_wsem)
 373#include "locking-selftest-rsem.h"
 374GENERATE_TESTCASE(ABBCCA_rsem)
 375
 376#undef E
 377
 378/*
 379 * AB CA BC deadlock:
 380 */
 381
 382#define E()					\
 383						\
 384	LOCK_UNLOCK_2(A, B);			\
 385	LOCK_UNLOCK_2(C, A);			\
 386	LOCK_UNLOCK_2(B, C); /* fail */
 387
 388/*
 389 * 6 testcases:
 390 */
 391#include "locking-selftest-spin.h"
 392GENERATE_TESTCASE(ABCABC_spin)
 393#include "locking-selftest-wlock.h"
 394GENERATE_TESTCASE(ABCABC_wlock)
 395#include "locking-selftest-rlock.h"
 396GENERATE_TESTCASE(ABCABC_rlock)
 397#include "locking-selftest-mutex.h"
 398GENERATE_TESTCASE(ABCABC_mutex)
 399#include "locking-selftest-wsem.h"
 400GENERATE_TESTCASE(ABCABC_wsem)
 401#include "locking-selftest-rsem.h"
 402GENERATE_TESTCASE(ABCABC_rsem)
 403
 404#undef E
 405
 406/*
 407 * AB BC CD DA deadlock:
 408 */
 409
 410#define E()					\
 411						\
 412	LOCK_UNLOCK_2(A, B);			\
 413	LOCK_UNLOCK_2(B, C);			\
 414	LOCK_UNLOCK_2(C, D);			\
 415	LOCK_UNLOCK_2(D, A); /* fail */
 416
 417/*
 418 * 6 testcases:
 419 */
 420#include "locking-selftest-spin.h"
 421GENERATE_TESTCASE(ABBCCDDA_spin)
 422#include "locking-selftest-wlock.h"
 423GENERATE_TESTCASE(ABBCCDDA_wlock)
 424#include "locking-selftest-rlock.h"
 425GENERATE_TESTCASE(ABBCCDDA_rlock)
 426#include "locking-selftest-mutex.h"
 427GENERATE_TESTCASE(ABBCCDDA_mutex)
 428#include "locking-selftest-wsem.h"
 429GENERATE_TESTCASE(ABBCCDDA_wsem)
 430#include "locking-selftest-rsem.h"
 431GENERATE_TESTCASE(ABBCCDDA_rsem)
 432
 433#undef E
 434
 435/*
 436 * AB CD BD DA deadlock:
 437 */
 438#define E()					\
 439						\
 440	LOCK_UNLOCK_2(A, B);			\
 441	LOCK_UNLOCK_2(C, D);			\
 442	LOCK_UNLOCK_2(B, D);			\
 443	LOCK_UNLOCK_2(D, A); /* fail */
 444
 445/*
 446 * 6 testcases:
 447 */
 448#include "locking-selftest-spin.h"
 449GENERATE_TESTCASE(ABCDBDDA_spin)
 450#include "locking-selftest-wlock.h"
 451GENERATE_TESTCASE(ABCDBDDA_wlock)
 452#include "locking-selftest-rlock.h"
 453GENERATE_TESTCASE(ABCDBDDA_rlock)
 454#include "locking-selftest-mutex.h"
 455GENERATE_TESTCASE(ABCDBDDA_mutex)
 456#include "locking-selftest-wsem.h"
 457GENERATE_TESTCASE(ABCDBDDA_wsem)
 458#include "locking-selftest-rsem.h"
 459GENERATE_TESTCASE(ABCDBDDA_rsem)
 460
 461#undef E
 462
 463/*
 464 * AB CD BC DA deadlock:
 465 */
 466#define E()					\
 467						\
 468	LOCK_UNLOCK_2(A, B);			\
 469	LOCK_UNLOCK_2(C, D);			\
 470	LOCK_UNLOCK_2(B, C);			\
 471	LOCK_UNLOCK_2(D, A); /* fail */
 472
 473/*
 474 * 6 testcases:
 475 */
 476#include "locking-selftest-spin.h"
 477GENERATE_TESTCASE(ABCDBCDA_spin)
 478#include "locking-selftest-wlock.h"
 479GENERATE_TESTCASE(ABCDBCDA_wlock)
 480#include "locking-selftest-rlock.h"
 481GENERATE_TESTCASE(ABCDBCDA_rlock)
 482#include "locking-selftest-mutex.h"
 483GENERATE_TESTCASE(ABCDBCDA_mutex)
 484#include "locking-selftest-wsem.h"
 485GENERATE_TESTCASE(ABCDBCDA_wsem)
 486#include "locking-selftest-rsem.h"
 487GENERATE_TESTCASE(ABCDBCDA_rsem)
 488
 489#undef E
 490
 491/*
 492 * Double unlock:
 493 */
 494#define E()					\
 495						\
 496	LOCK(A);				\
 497	UNLOCK(A);				\
 498	UNLOCK(A); /* fail */
 499
 500/*
 501 * 6 testcases:
 502 */
 503#include "locking-selftest-spin.h"
 504GENERATE_TESTCASE(double_unlock_spin)
 505#include "locking-selftest-wlock.h"
 506GENERATE_TESTCASE(double_unlock_wlock)
 507#include "locking-selftest-rlock.h"
 508GENERATE_TESTCASE(double_unlock_rlock)
 509#include "locking-selftest-mutex.h"
 510GENERATE_TESTCASE(double_unlock_mutex)
 511#include "locking-selftest-wsem.h"
 512GENERATE_TESTCASE(double_unlock_wsem)
 513#include "locking-selftest-rsem.h"
 514GENERATE_TESTCASE(double_unlock_rsem)
 515
 516#undef E
 517
 518/*
 519 * Bad unlock ordering:
 520 */
 521#define E()					\
 522						\
 523	LOCK(A);				\
 524	LOCK(B);				\
 525	UNLOCK(A); /* fail */			\
 526	UNLOCK(B);
 527
 528/*
 529 * 6 testcases:
 530 */
 531#include "locking-selftest-spin.h"
 532GENERATE_TESTCASE(bad_unlock_order_spin)
 533#include "locking-selftest-wlock.h"
 534GENERATE_TESTCASE(bad_unlock_order_wlock)
 535#include "locking-selftest-rlock.h"
 536GENERATE_TESTCASE(bad_unlock_order_rlock)
 537#include "locking-selftest-mutex.h"
 538GENERATE_TESTCASE(bad_unlock_order_mutex)
 539#include "locking-selftest-wsem.h"
 540GENERATE_TESTCASE(bad_unlock_order_wsem)
 541#include "locking-selftest-rsem.h"
 542GENERATE_TESTCASE(bad_unlock_order_rsem)
 543
 544#undef E
 545
 546/*
 547 * initializing a held lock:
 548 */
 549#define E()					\
 550						\
 551	LOCK(A);				\
 552	INIT(A); /* fail */
 553
 554/*
 555 * 6 testcases:
 556 */
 557#include "locking-selftest-spin.h"
 558GENERATE_TESTCASE(init_held_spin)
 559#include "locking-selftest-wlock.h"
 560GENERATE_TESTCASE(init_held_wlock)
 561#include "locking-selftest-rlock.h"
 562GENERATE_TESTCASE(init_held_rlock)
 563#include "locking-selftest-mutex.h"
 564GENERATE_TESTCASE(init_held_mutex)
 565#include "locking-selftest-wsem.h"
 566GENERATE_TESTCASE(init_held_wsem)
 567#include "locking-selftest-rsem.h"
 568GENERATE_TESTCASE(init_held_rsem)
 569
 570#undef E
 571
 572/*
 573 * locking an irq-safe lock with irqs enabled:
 574 */
 575#define E1()				\
 576					\
 577	IRQ_ENTER();			\
 578	LOCK(A);			\
 579	UNLOCK(A);			\
 580	IRQ_EXIT();
 581
 582#define E2()				\
 583					\
 584	LOCK(A);			\
 585	UNLOCK(A);
 586
 587/*
 588 * Generate 24 testcases:
 589 */
 590#include "locking-selftest-spin-hardirq.h"
 591GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin)
 592
 593#include "locking-selftest-rlock-hardirq.h"
 594GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock)
 595
 596#include "locking-selftest-wlock-hardirq.h"
 597GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_wlock)
 598
 599#include "locking-selftest-spin-softirq.h"
 600GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_spin)
 601
 602#include "locking-selftest-rlock-softirq.h"
 603GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_rlock)
 604
 605#include "locking-selftest-wlock-softirq.h"
 606GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock)
 607
 608#undef E1
 609#undef E2
 610
 611/*
 612 * Enabling hardirqs with a softirq-safe lock held:
 613 */
 614#define E1()				\
 615					\
 616	SOFTIRQ_ENTER();		\
 617	LOCK(A);			\
 618	UNLOCK(A);			\
 619	SOFTIRQ_EXIT();
 620
 621#define E2()				\
 622					\
 623	HARDIRQ_DISABLE();		\
 624	LOCK(A);			\
 625	HARDIRQ_ENABLE();		\
 626	UNLOCK(A);
 627
 628/*
 629 * Generate 12 testcases:
 630 */
 631#include "locking-selftest-spin.h"
 632GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_spin)
 633
 634#include "locking-selftest-wlock.h"
 635GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_wlock)
 636
 637#include "locking-selftest-rlock.h"
 638GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_rlock)
 639
 640#undef E1
 641#undef E2
 642
 643/*
 644 * Enabling irqs with an irq-safe lock held:
 645 */
 646#define E1()				\
 647					\
 648	IRQ_ENTER();			\
 649	LOCK(A);			\
 650	UNLOCK(A);			\
 651	IRQ_EXIT();
 652
 653#define E2()				\
 654					\
 655	IRQ_DISABLE();			\
 656	LOCK(A);			\
 657	IRQ_ENABLE();			\
 658	UNLOCK(A);
 659
 660/*
 661 * Generate 24 testcases:
 662 */
 663#include "locking-selftest-spin-hardirq.h"
 664GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin)
 665
 666#include "locking-selftest-rlock-hardirq.h"
 667GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock)
 668
 669#include "locking-selftest-wlock-hardirq.h"
 670GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_wlock)
 671
 672#include "locking-selftest-spin-softirq.h"
 673GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_spin)
 674
 675#include "locking-selftest-rlock-softirq.h"
 676GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_rlock)
 677
 678#include "locking-selftest-wlock-softirq.h"
 679GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock)
 680
 681#undef E1
 682#undef E2
 683
 684/*
 685 * Acquiring a irq-unsafe lock while holding an irq-safe-lock:
 686 */
 687#define E1()				\
 688					\
 689	LOCK(A);			\
 690	LOCK(B);			\
 691	UNLOCK(B);			\
 692	UNLOCK(A);			\
 693
 694#define E2()				\
 695					\
 696	LOCK(B);			\
 697	UNLOCK(B);
 698
 699#define E3()				\
 700					\
 701	IRQ_ENTER();			\
 702	LOCK(A);			\
 703	UNLOCK(A);			\
 704	IRQ_EXIT();
 705
 706/*
 707 * Generate 36 testcases:
 708 */
 709#include "locking-selftest-spin-hardirq.h"
 710GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin)
 711
 712#include "locking-selftest-rlock-hardirq.h"
 713GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock)
 714
 715#include "locking-selftest-wlock-hardirq.h"
 716GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_wlock)
 717
 718#include "locking-selftest-spin-softirq.h"
 719GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_spin)
 720
 721#include "locking-selftest-rlock-softirq.h"
 722GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_rlock)
 723
 724#include "locking-selftest-wlock-softirq.h"
 725GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock)
 726
 727#undef E1
 728#undef E2
 729#undef E3
 730
 731/*
 732 * If a lock turns into softirq-safe, but earlier it took
 733 * a softirq-unsafe lock:
 734 */
 735
 736#define E1()				\
 737	IRQ_DISABLE();			\
 738	LOCK(A);			\
 739	LOCK(B);			\
 740	UNLOCK(B);			\
 741	UNLOCK(A);			\
 742	IRQ_ENABLE();
 743
 744#define E2()				\
 745	LOCK(B);			\
 746	UNLOCK(B);
 747
 748#define E3()				\
 749	IRQ_ENTER();			\
 750	LOCK(A);			\
 751	UNLOCK(A);			\
 752	IRQ_EXIT();
 753
 754/*
 755 * Generate 36 testcases:
 756 */
 757#include "locking-selftest-spin-hardirq.h"
 758GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin)
 759
 760#include "locking-selftest-rlock-hardirq.h"
 761GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock)
 762
 763#include "locking-selftest-wlock-hardirq.h"
 764GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_wlock)
 765
 766#include "locking-selftest-spin-softirq.h"
 767GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_spin)
 768
 769#include "locking-selftest-rlock-softirq.h"
 770GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_rlock)
 771
 772#include "locking-selftest-wlock-softirq.h"
 773GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock)
 774
 775#undef E1
 776#undef E2
 777#undef E3
 778
 779/*
 780 * read-lock / write-lock irq inversion.
 781 *
 782 * Deadlock scenario:
 783 *
 784 * CPU#1 is at #1, i.e. it has write-locked A, but has not
 785 * taken B yet.
 786 *
 787 * CPU#2 is at #2, i.e. it has locked B.
 788 *
 789 * Hardirq hits CPU#2 at point #2 and is trying to read-lock A.
 790 *
 791 * The deadlock occurs because CPU#1 will spin on B, and CPU#2
 792 * will spin on A.
 793 */
 794
 795#define E1()				\
 796					\
 797	IRQ_DISABLE();			\
 798	WL(A);				\
 799	LOCK(B);			\
 800	UNLOCK(B);			\
 801	WU(A);				\
 802	IRQ_ENABLE();
 803
 804#define E2()				\
 805					\
 806	LOCK(B);			\
 807	UNLOCK(B);
 808
 809#define E3()				\
 810					\
 811	IRQ_ENTER();			\
 812	RL(A);				\
 813	RU(A);				\
 814	IRQ_EXIT();
 815
 816/*
 817 * Generate 36 testcases:
 818 */
 819#include "locking-selftest-spin-hardirq.h"
 820GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_spin)
 821
 822#include "locking-selftest-rlock-hardirq.h"
 823GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_rlock)
 824
 825#include "locking-selftest-wlock-hardirq.h"
 826GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_hard_wlock)
 827
 828#include "locking-selftest-spin-softirq.h"
 829GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_spin)
 830
 831#include "locking-selftest-rlock-softirq.h"
 832GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_rlock)
 833
 834#include "locking-selftest-wlock-softirq.h"
 835GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_wlock)
 836
 837#undef E1
 838#undef E2
 839#undef E3
 840
 841/*
 842 * read-lock / write-lock recursion that is actually safe.
 843 */
 844
 845#define E1()				\
 846					\
 847	IRQ_DISABLE();			\
 848	WL(A);				\
 849	WU(A);				\
 850	IRQ_ENABLE();
 851
 852#define E2()				\
 853					\
 854	RL(A);				\
 855	RU(A);				\
 856
 857#define E3()				\
 858					\
 859	IRQ_ENTER();			\
 860	RL(A);				\
 861	L(B);				\
 862	U(B);				\
 863	RU(A);				\
 864	IRQ_EXIT();
 865
 866/*
 867 * Generate 12 testcases:
 868 */
 869#include "locking-selftest-hardirq.h"
 870GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard)
 871
 872#include "locking-selftest-softirq.h"
 873GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft)
 874
 875#undef E1
 876#undef E2
 877#undef E3
 878
 879/*
 880 * read-lock / write-lock recursion that is unsafe.
 881 */
 882
 883#define E1()				\
 884					\
 885	IRQ_DISABLE();			\
 886	L(B);				\
 887	WL(A);				\
 888	WU(A);				\
 889	U(B);				\
 890	IRQ_ENABLE();
 891
 892#define E2()				\
 893					\
 894	RL(A);				\
 895	RU(A);				\
 896
 897#define E3()				\
 898					\
 899	IRQ_ENTER();			\
 900	L(B);				\
 901	U(B);				\
 902	IRQ_EXIT();
 903
 904/*
 905 * Generate 12 testcases:
 906 */
 907#include "locking-selftest-hardirq.h"
 908// GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard)
 909
 910#include "locking-selftest-softirq.h"
 911// GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft)
 912
 913#ifdef CONFIG_DEBUG_LOCK_ALLOC
 914# define I_SPINLOCK(x)	lockdep_reset_lock(&lock_##x.dep_map)
 915# define I_RWLOCK(x)	lockdep_reset_lock(&rwlock_##x.dep_map)
 916# define I_MUTEX(x)	lockdep_reset_lock(&mutex_##x.dep_map)
 917# define I_RWSEM(x)	lockdep_reset_lock(&rwsem_##x.dep_map)
 918# define I_WW(x)	lockdep_reset_lock(&x.dep_map)
 919#else
 920# define I_SPINLOCK(x)
 921# define I_RWLOCK(x)
 922# define I_MUTEX(x)
 923# define I_RWSEM(x)
 924# define I_WW(x)
 925#endif
 926
 927#define I1(x)					\
 928	do {					\
 929		I_SPINLOCK(x);			\
 930		I_RWLOCK(x);			\
 931		I_MUTEX(x);			\
 932		I_RWSEM(x);			\
 933	} while (0)
 934
 935#define I2(x)					\
 936	do {					\
 937		raw_spin_lock_init(&lock_##x);	\
 938		rwlock_init(&rwlock_##x);	\
 939		mutex_init(&mutex_##x);		\
 940		init_rwsem(&rwsem_##x);		\
 941	} while (0)
 942
 943static void reset_locks(void)
 944{
 945	local_irq_disable();
 946	lockdep_free_key_range(&ww_lockdep.acquire_key, 1);
 947	lockdep_free_key_range(&ww_lockdep.mutex_key, 1);
 948
 949	I1(A); I1(B); I1(C); I1(D);
 950	I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2);
 951	I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); I_WW(o3.base);
 952	lockdep_reset();
 953	I2(A); I2(B); I2(C); I2(D);
 954	init_shared_classes();
 955
 956	ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); ww_mutex_init(&o3, &ww_lockdep);
 957	memset(&t, 0, sizeof(t)); memset(&t2, 0, sizeof(t2));
 958	memset(&ww_lockdep.acquire_key, 0, sizeof(ww_lockdep.acquire_key));
 959	memset(&ww_lockdep.mutex_key, 0, sizeof(ww_lockdep.mutex_key));
 960	local_irq_enable();
 961}
 962
 963#undef I
 964
 965static int testcase_total;
 966static int testcase_successes;
 967static int expected_testcase_failures;
 968static int unexpected_testcase_failures;
 969
 970static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
 971{
 972	unsigned long saved_preempt_count = preempt_count();
 973
 974	WARN_ON(irqs_disabled());
 975
 976	testcase_fn();
 977	/*
 978	 * Filter out expected failures:
 979	 */
 980#ifndef CONFIG_PROVE_LOCKING
 981	if (expected == FAILURE && debug_locks) {
 982		expected_testcase_failures++;
 983		printk("failed|");
 984	}
 985	else
 986#endif
 987	if (debug_locks != expected) {
 988		unexpected_testcase_failures++;
 989		printk("FAILED|");
 990
 991		dump_stack();
 992	} else {
 993		testcase_successes++;
 994		printk("  ok  |");
 995	}
 996	testcase_total++;
 997
 998	if (debug_locks_verbose)
 999		printk(" lockclass mask: %x, debug_locks: %d, expected: %d\n",
1000			lockclass_mask, debug_locks, expected);
1001	/*
1002	 * Some tests (e.g. double-unlock) might corrupt the preemption
1003	 * count, so restore it:
1004	 */
1005	preempt_count_set(saved_preempt_count);
1006#ifdef CONFIG_TRACE_IRQFLAGS
1007	if (softirq_count())
1008		current->softirqs_enabled = 0;
1009	else
1010		current->softirqs_enabled = 1;
1011#endif
1012
1013	reset_locks();
1014}
1015
1016static inline void print_testname(const char *testname)
1017{
1018	printk("%33s:", testname);
1019}
1020
1021#define DO_TESTCASE_1(desc, name, nr)				\
1022	print_testname(desc"/"#nr);				\
1023	dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK);		\
1024	printk("\n");
1025
1026#define DO_TESTCASE_1B(desc, name, nr)				\
1027	print_testname(desc"/"#nr);				\
1028	dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK);		\
1029	printk("\n");
1030
1031#define DO_TESTCASE_3(desc, name, nr)				\
1032	print_testname(desc"/"#nr);				\
1033	dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN);	\
1034	dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);	\
1035	dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);	\
1036	printk("\n");
1037
1038#define DO_TESTCASE_3RW(desc, name, nr)				\
1039	print_testname(desc"/"#nr);				\
1040	dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN|LOCKTYPE_RWLOCK);\
1041	dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK);	\
1042	dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK);	\
1043	printk("\n");
1044
1045#define DO_TESTCASE_6(desc, name)				\
1046	print_testname(desc);					\
1047	dotest(name##_spin, FAILURE, LOCKTYPE_SPIN);		\
1048	dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK);		\
1049	dotest(name##_rlock, FAILURE, LOCKTYPE_RWLOCK);		\
1050	dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX);		\
1051	dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM);		\
1052	dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM);		\
1053	printk("\n");
1054
1055#define DO_TESTCASE_6_SUCCESS(desc, name)			\
1056	print_testname(desc);					\
1057	dotest(name##_spin, SUCCESS, LOCKTYPE_SPIN);		\
1058	dotest(name##_wlock, SUCCESS, LOCKTYPE_RWLOCK);		\
1059	dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK);		\
1060	dotest(name##_mutex, SUCCESS, LOCKTYPE_MUTEX);		\
1061	dotest(name##_wsem, SUCCESS, LOCKTYPE_RWSEM);		\
1062	dotest(name##_rsem, SUCCESS, LOCKTYPE_RWSEM);		\
1063	printk("\n");
1064
1065/*
1066 * 'read' variant: rlocks must not trigger.
1067 */
1068#define DO_TESTCASE_6R(desc, name)				\
1069	print_testname(desc);					\
1070	dotest(name##_spin, FAILURE, LOCKTYPE_SPIN);		\
1071	dotest(name##_wlock, FAILURE, LOCKTYPE_RWLOCK);		\
1072	dotest(name##_rlock, SUCCESS, LOCKTYPE_RWLOCK);		\
1073	dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX);		\
1074	dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM);		\
1075	dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM);		\
1076	printk("\n");
1077
1078#define DO_TESTCASE_2I(desc, name, nr)				\
1079	DO_TESTCASE_1("hard-"desc, name##_hard, nr);		\
1080	DO_TESTCASE_1("soft-"desc, name##_soft, nr);
1081
1082#define DO_TESTCASE_2IB(desc, name, nr)				\
1083	DO_TESTCASE_1B("hard-"desc, name##_hard, nr);		\
1084	DO_TESTCASE_1B("soft-"desc, name##_soft, nr);
1085
1086#define DO_TESTCASE_6I(desc, name, nr)				\
1087	DO_TESTCASE_3("hard-"desc, name##_hard, nr);		\
1088	DO_TESTCASE_3("soft-"desc, name##_soft, nr);
1089
1090#define DO_TESTCASE_6IRW(desc, name, nr)			\
1091	DO_TESTCASE_3RW("hard-"desc, name##_hard, nr);		\
1092	DO_TESTCASE_3RW("soft-"desc, name##_soft, nr);
1093
1094#define DO_TESTCASE_2x3(desc, name)				\
1095	DO_TESTCASE_3(desc, name, 12);				\
1096	DO_TESTCASE_3(desc, name, 21);
1097
1098#define DO_TESTCASE_2x6(desc, name)				\
1099	DO_TESTCASE_6I(desc, name, 12);				\
1100	DO_TESTCASE_6I(desc, name, 21);
1101
1102#define DO_TESTCASE_6x2(desc, name)				\
1103	DO_TESTCASE_2I(desc, name, 123);			\
1104	DO_TESTCASE_2I(desc, name, 132);			\
1105	DO_TESTCASE_2I(desc, name, 213);			\
1106	DO_TESTCASE_2I(desc, name, 231);			\
1107	DO_TESTCASE_2I(desc, name, 312);			\
1108	DO_TESTCASE_2I(desc, name, 321);
1109
1110#define DO_TESTCASE_6x2B(desc, name)				\
1111	DO_TESTCASE_2IB(desc, name, 123);			\
1112	DO_TESTCASE_2IB(desc, name, 132);			\
1113	DO_TESTCASE_2IB(desc, name, 213);			\
1114	DO_TESTCASE_2IB(desc, name, 231);			\
1115	DO_TESTCASE_2IB(desc, name, 312);			\
1116	DO_TESTCASE_2IB(desc, name, 321);
1117
1118#define DO_TESTCASE_6x6(desc, name)				\
1119	DO_TESTCASE_6I(desc, name, 123);			\
1120	DO_TESTCASE_6I(desc, name, 132);			\
1121	DO_TESTCASE_6I(desc, name, 213);			\
1122	DO_TESTCASE_6I(desc, name, 231);			\
1123	DO_TESTCASE_6I(desc, name, 312);			\
1124	DO_TESTCASE_6I(desc, name, 321);
1125
1126#define DO_TESTCASE_6x6RW(desc, name)				\
1127	DO_TESTCASE_6IRW(desc, name, 123);			\
1128	DO_TESTCASE_6IRW(desc, name, 132);			\
1129	DO_TESTCASE_6IRW(desc, name, 213);			\
1130	DO_TESTCASE_6IRW(desc, name, 231);			\
1131	DO_TESTCASE_6IRW(desc, name, 312);			\
1132	DO_TESTCASE_6IRW(desc, name, 321);
1133
1134static void ww_test_fail_acquire(void)
1135{
1136	int ret;
1137
1138	WWAI(&t);
1139	t.stamp++;
1140
1141	ret = WWL(&o, &t);
1142
1143	if (WARN_ON(!o.ctx) ||
1144	    WARN_ON(ret))
1145		return;
1146
1147	/* No lockdep test, pure API */
1148	ret = WWL(&o, &t);
1149	WARN_ON(ret != -EALREADY);
1150
1151	ret = WWT(&o);
1152	WARN_ON(ret);
1153
1154	t2 = t;
1155	t2.stamp++;
1156	ret = WWL(&o, &t2);
1157	WARN_ON(ret != -EDEADLK);
1158	WWU(&o);
1159
1160	if (WWT(&o))
1161		WWU(&o);
1162#ifdef CONFIG_DEBUG_LOCK_ALLOC
1163	else
1164		DEBUG_LOCKS_WARN_ON(1);
1165#endif
1166}
1167
1168static void ww_test_normal(void)
1169{
1170	int ret;
1171
1172	WWAI(&t);
1173
1174	/*
1175	 * None of the ww_mutex codepaths should be taken in the 'normal'
1176	 * mutex calls. The easiest way to verify this is by using the
1177	 * normal mutex calls, and making sure o.ctx is unmodified.
1178	 */
1179
1180	/* mutex_lock (and indirectly, mutex_lock_nested) */
1181	o.ctx = (void *)~0UL;
1182	mutex_lock(&o.base);
1183	mutex_unlock(&o.base);
1184	WARN_ON(o.ctx != (void *)~0UL);
1185
1186	/* mutex_lock_interruptible (and *_nested) */
1187	o.ctx = (void *)~0UL;
1188	ret = mutex_lock_interruptible(&o.base);
1189	if (!ret)
1190		mutex_unlock(&o.base);
1191	else
1192		WARN_ON(1);
1193	WARN_ON(o.ctx != (void *)~0UL);
1194
1195	/* mutex_lock_killable (and *_nested) */
1196	o.ctx = (void *)~0UL;
1197	ret = mutex_lock_killable(&o.base);
1198	if (!ret)
1199		mutex_unlock(&o.base);
1200	else
1201		WARN_ON(1);
1202	WARN_ON(o.ctx != (void *)~0UL);
1203
1204	/* trylock, succeeding */
1205	o.ctx = (void *)~0UL;
1206	ret = mutex_trylock(&o.base);
1207	WARN_ON(!ret);
1208	if (ret)
1209		mutex_unlock(&o.base);
1210	else
1211		WARN_ON(1);
1212	WARN_ON(o.ctx != (void *)~0UL);
1213
1214	/* trylock, failing */
1215	o.ctx = (void *)~0UL;
1216	mutex_lock(&o.base);
1217	ret = mutex_trylock(&o.base);
1218	WARN_ON(ret);
1219	mutex_unlock(&o.base);
1220	WARN_ON(o.ctx != (void *)~0UL);
1221
1222	/* nest_lock */
1223	o.ctx = (void *)~0UL;
1224	mutex_lock_nest_lock(&o.base, &t);
1225	mutex_unlock(&o.base);
1226	WARN_ON(o.ctx != (void *)~0UL);
1227}
1228
1229static void ww_test_two_contexts(void)
1230{
1231	WWAI(&t);
1232	WWAI(&t2);
1233}
1234
1235static void ww_test_diff_class(void)
1236{
1237	WWAI(&t);
1238#ifdef CONFIG_DEBUG_MUTEXES
1239	t.ww_class = NULL;
1240#endif
1241	WWL(&o, &t);
1242}
1243
1244static void ww_test_context_done_twice(void)
1245{
1246	WWAI(&t);
1247	WWAD(&t);
1248	WWAD(&t);
1249	WWAF(&t);
1250}
1251
1252static void ww_test_context_unlock_twice(void)
1253{
1254	WWAI(&t);
1255	WWAD(&t);
1256	WWAF(&t);
1257	WWAF(&t);
1258}
1259
1260static void ww_test_context_fini_early(void)
1261{
1262	WWAI(&t);
1263	WWL(&o, &t);
1264	WWAD(&t);
1265	WWAF(&t);
1266}
1267
1268static void ww_test_context_lock_after_done(void)
1269{
1270	WWAI(&t);
1271	WWAD(&t);
1272	WWL(&o, &t);
1273}
1274
1275static void ww_test_object_unlock_twice(void)
1276{
1277	WWL1(&o);
1278	WWU(&o);
1279	WWU(&o);
1280}
1281
1282static void ww_test_object_lock_unbalanced(void)
1283{
1284	WWAI(&t);
1285	WWL(&o, &t);
1286	t.acquired = 0;
1287	WWU(&o);
1288	WWAF(&t);
1289}
1290
1291static void ww_test_object_lock_stale_context(void)
1292{
1293	WWAI(&t);
1294	o.ctx = &t2;
1295	WWL(&o, &t);
1296}
1297
1298static void ww_test_edeadlk_normal(void)
1299{
1300	int ret;
1301
1302	mutex_lock(&o2.base);
1303	o2.ctx = &t2;
1304	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1305
1306	WWAI(&t);
1307	t2 = t;
1308	t2.stamp--;
1309
1310	ret = WWL(&o, &t);
1311	WARN_ON(ret);
1312
1313	ret = WWL(&o2, &t);
1314	WARN_ON(ret != -EDEADLK);
1315
1316	o2.ctx = NULL;
1317	mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1318	mutex_unlock(&o2.base);
1319	WWU(&o);
1320
1321	WWL(&o2, &t);
1322}
1323
1324static void ww_test_edeadlk_normal_slow(void)
1325{
1326	int ret;
1327
1328	mutex_lock(&o2.base);
1329	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1330	o2.ctx = &t2;
1331
1332	WWAI(&t);
1333	t2 = t;
1334	t2.stamp--;
1335
1336	ret = WWL(&o, &t);
1337	WARN_ON(ret);
1338
1339	ret = WWL(&o2, &t);
1340	WARN_ON(ret != -EDEADLK);
1341
1342	o2.ctx = NULL;
1343	mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1344	mutex_unlock(&o2.base);
1345	WWU(&o);
1346
1347	ww_mutex_lock_slow(&o2, &t);
1348}
1349
1350static void ww_test_edeadlk_no_unlock(void)
1351{
1352	int ret;
1353
1354	mutex_lock(&o2.base);
1355	o2.ctx = &t2;
1356	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1357
1358	WWAI(&t);
1359	t2 = t;
1360	t2.stamp--;
1361
1362	ret = WWL(&o, &t);
1363	WARN_ON(ret);
1364
1365	ret = WWL(&o2, &t);
1366	WARN_ON(ret != -EDEADLK);
1367
1368	o2.ctx = NULL;
1369	mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1370	mutex_unlock(&o2.base);
1371
1372	WWL(&o2, &t);
1373}
1374
1375static void ww_test_edeadlk_no_unlock_slow(void)
1376{
1377	int ret;
1378
1379	mutex_lock(&o2.base);
1380	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1381	o2.ctx = &t2;
1382
1383	WWAI(&t);
1384	t2 = t;
1385	t2.stamp--;
1386
1387	ret = WWL(&o, &t);
1388	WARN_ON(ret);
1389
1390	ret = WWL(&o2, &t);
1391	WARN_ON(ret != -EDEADLK);
1392
1393	o2.ctx = NULL;
1394	mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_);
1395	mutex_unlock(&o2.base);
1396
1397	ww_mutex_lock_slow(&o2, &t);
1398}
1399
1400static void ww_test_edeadlk_acquire_more(void)
1401{
1402	int ret;
1403
1404	mutex_lock(&o2.base);
1405	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1406	o2.ctx = &t2;
1407
1408	WWAI(&t);
1409	t2 = t;
1410	t2.stamp--;
1411
1412	ret = WWL(&o, &t);
1413	WARN_ON(ret);
1414
1415	ret = WWL(&o2, &t);
1416	WARN_ON(ret != -EDEADLK);
1417
1418	ret = WWL(&o3, &t);
1419}
1420
1421static void ww_test_edeadlk_acquire_more_slow(void)
1422{
1423	int ret;
1424
1425	mutex_lock(&o2.base);
1426	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1427	o2.ctx = &t2;
1428
1429	WWAI(&t);
1430	t2 = t;
1431	t2.stamp--;
1432
1433	ret = WWL(&o, &t);
1434	WARN_ON(ret);
1435
1436	ret = WWL(&o2, &t);
1437	WARN_ON(ret != -EDEADLK);
1438
1439	ww_mutex_lock_slow(&o3, &t);
1440}
1441
1442static void ww_test_edeadlk_acquire_more_edeadlk(void)
1443{
1444	int ret;
1445
1446	mutex_lock(&o2.base);
1447	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1448	o2.ctx = &t2;
1449
1450	mutex_lock(&o3.base);
1451	mutex_release(&o3.base.dep_map, 1, _THIS_IP_);
1452	o3.ctx = &t2;
1453
1454	WWAI(&t);
1455	t2 = t;
1456	t2.stamp--;
1457
1458	ret = WWL(&o, &t);
1459	WARN_ON(ret);
1460
1461	ret = WWL(&o2, &t);
1462	WARN_ON(ret != -EDEADLK);
1463
1464	ret = WWL(&o3, &t);
1465	WARN_ON(ret != -EDEADLK);
1466}
1467
1468static void ww_test_edeadlk_acquire_more_edeadlk_slow(void)
1469{
1470	int ret;
1471
1472	mutex_lock(&o2.base);
1473	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1474	o2.ctx = &t2;
1475
1476	mutex_lock(&o3.base);
1477	mutex_release(&o3.base.dep_map, 1, _THIS_IP_);
1478	o3.ctx = &t2;
1479
1480	WWAI(&t);
1481	t2 = t;
1482	t2.stamp--;
1483
1484	ret = WWL(&o, &t);
1485	WARN_ON(ret);
1486
1487	ret = WWL(&o2, &t);
1488	WARN_ON(ret != -EDEADLK);
1489
1490	ww_mutex_lock_slow(&o3, &t);
1491}
1492
1493static void ww_test_edeadlk_acquire_wrong(void)
1494{
1495	int ret;
1496
1497	mutex_lock(&o2.base);
1498	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1499	o2.ctx = &t2;
1500
1501	WWAI(&t);
1502	t2 = t;
1503	t2.stamp--;
1504
1505	ret = WWL(&o, &t);
1506	WARN_ON(ret);
1507
1508	ret = WWL(&o2, &t);
1509	WARN_ON(ret != -EDEADLK);
1510	if (!ret)
1511		WWU(&o2);
1512
1513	WWU(&o);
1514
1515	ret = WWL(&o3, &t);
1516}
1517
1518static void ww_test_edeadlk_acquire_wrong_slow(void)
1519{
1520	int ret;
1521
1522	mutex_lock(&o2.base);
1523	mutex_release(&o2.base.dep_map, 1, _THIS_IP_);
1524	o2.ctx = &t2;
1525
1526	WWAI(&t);
1527	t2 = t;
1528	t2.stamp--;
1529
1530	ret = WWL(&o, &t);
1531	WARN_ON(ret);
1532
1533	ret = WWL(&o2, &t);
1534	WARN_ON(ret != -EDEADLK);
1535	if (!ret)
1536		WWU(&o2);
1537
1538	WWU(&o);
1539
1540	ww_mutex_lock_slow(&o3, &t);
1541}
1542
1543static void ww_test_spin_nest_unlocked(void)
1544{
1545	raw_spin_lock_nest_lock(&lock_A, &o.base);
1546	U(A);
1547}
1548
1549static void ww_test_unneeded_slow(void)
1550{
1551	WWAI(&t);
1552
1553	ww_mutex_lock_slow(&o, &t);
1554}
1555
1556static void ww_test_context_block(void)
1557{
1558	int ret;
1559
1560	WWAI(&t);
1561
1562	ret = WWL(&o, &t);
1563	WARN_ON(ret);
1564	WWL1(&o2);
1565}
1566
1567static void ww_test_context_try(void)
1568{
1569	int ret;
1570
1571	WWAI(&t);
1572
1573	ret = WWL(&o, &t);
1574	WARN_ON(ret);
1575
1576	ret = WWT(&o2);
1577	WARN_ON(!ret);
1578	WWU(&o2);
1579	WWU(&o);
1580}
1581
1582static void ww_test_context_context(void)
1583{
1584	int ret;
1585
1586	WWAI(&t);
1587
1588	ret = WWL(&o, &t);
1589	WARN_ON(ret);
1590
1591	ret = WWL(&o2, &t);
1592	WARN_ON(ret);
1593
1594	WWU(&o2);
1595	WWU(&o);
1596}
1597
1598static void ww_test_try_block(void)
1599{
1600	bool ret;
1601
1602	ret = WWT(&o);
1603	WARN_ON(!ret);
1604
1605	WWL1(&o2);
1606	WWU(&o2);
1607	WWU(&o);
1608}
1609
1610static void ww_test_try_try(void)
1611{
1612	bool ret;
1613
1614	ret = WWT(&o);
1615	WARN_ON(!ret);
1616	ret = WWT(&o2);
1617	WARN_ON(!ret);
1618	WWU(&o2);
1619	WWU(&o);
1620}
1621
1622static void ww_test_try_context(void)
1623{
1624	int ret;
1625
1626	ret = WWT(&o);
1627	WARN_ON(!ret);
1628
1629	WWAI(&t);
1630
1631	ret = WWL(&o2, &t);
1632	WARN_ON(ret);
1633}
1634
1635static void ww_test_block_block(void)
1636{
1637	WWL1(&o);
1638	WWL1(&o2);
1639}
1640
1641static void ww_test_block_try(void)
1642{
1643	bool ret;
1644
1645	WWL1(&o);
1646	ret = WWT(&o2);
1647	WARN_ON(!ret);
1648}
1649
1650static void ww_test_block_context(void)
1651{
1652	int ret;
1653
1654	WWL1(&o);
1655	WWAI(&t);
1656
1657	ret = WWL(&o2, &t);
1658	WARN_ON(ret);
1659}
1660
1661static void ww_test_spin_block(void)
1662{
1663	L(A);
1664	U(A);
1665
1666	WWL1(&o);
1667	L(A);
1668	U(A);
1669	WWU(&o);
1670
1671	L(A);
1672	WWL1(&o);
1673	WWU(&o);
1674	U(A);
1675}
1676
1677static void ww_test_spin_try(void)
1678{
1679	bool ret;
1680
1681	L(A);
1682	U(A);
1683
1684	ret = WWT(&o);
1685	WARN_ON(!ret);
1686	L(A);
1687	U(A);
1688	WWU(&o);
1689
1690	L(A);
1691	ret = WWT(&o);
1692	WARN_ON(!ret);
1693	WWU(&o);
1694	U(A);
1695}
1696
1697static void ww_test_spin_context(void)
1698{
1699	int ret;
1700
1701	L(A);
1702	U(A);
1703
1704	WWAI(&t);
1705
1706	ret = WWL(&o, &t);
1707	WARN_ON(ret);
1708	L(A);
1709	U(A);
1710	WWU(&o);
1711
1712	L(A);
1713	ret = WWL(&o, &t);
1714	WARN_ON(ret);
1715	WWU(&o);
1716	U(A);
1717}
1718
1719static void ww_tests(void)
1720{
1721	printk("  --------------------------------------------------------------------------\n");
1722	printk("  | Wound/wait tests |\n");
1723	printk("  ---------------------\n");
1724
1725	print_testname("ww api failures");
1726	dotest(ww_test_fail_acquire, SUCCESS, LOCKTYPE_WW);
1727	dotest(ww_test_normal, SUCCESS, LOCKTYPE_WW);
1728	dotest(ww_test_unneeded_slow, FAILURE, LOCKTYPE_WW);
1729	printk("\n");
1730
1731	print_testname("ww contexts mixing");
1732	dotest(ww_test_two_contexts, FAILURE, LOCKTYPE_WW);
1733	dotest(ww_test_diff_class, FAILURE, LOCKTYPE_WW);
1734	printk("\n");
1735
1736	print_testname("finishing ww context");
1737	dotest(ww_test_context_done_twice, FAILURE, LOCKTYPE_WW);
1738	dotest(ww_test_context_unlock_twice, FAILURE, LOCKTYPE_WW);
1739	dotest(ww_test_context_fini_early, FAILURE, LOCKTYPE_WW);
1740	dotest(ww_test_context_lock_after_done, FAILURE, LOCKTYPE_WW);
1741	printk("\n");
1742
1743	print_testname("locking mismatches");
1744	dotest(ww_test_object_unlock_twice, FAILURE, LOCKTYPE_WW);
1745	dotest(ww_test_object_lock_unbalanced, FAILURE, LOCKTYPE_WW);
1746	dotest(ww_test_object_lock_stale_context, FAILURE, LOCKTYPE_WW);
1747	printk("\n");
1748
1749	print_testname("EDEADLK handling");
1750	dotest(ww_test_edeadlk_normal, SUCCESS, LOCKTYPE_WW);
1751	dotest(ww_test_edeadlk_normal_slow, SUCCESS, LOCKTYPE_WW);
1752	dotest(ww_test_edeadlk_no_unlock, FAILURE, LOCKTYPE_WW);
1753	dotest(ww_test_edeadlk_no_unlock_slow, FAILURE, LOCKTYPE_WW);
1754	dotest(ww_test_edeadlk_acquire_more, FAILURE, LOCKTYPE_WW);
1755	dotest(ww_test_edeadlk_acquire_more_slow, FAILURE, LOCKTYPE_WW);
1756	dotest(ww_test_edeadlk_acquire_more_edeadlk, FAILURE, LOCKTYPE_WW);
1757	dotest(ww_test_edeadlk_acquire_more_edeadlk_slow, FAILURE, LOCKTYPE_WW);
1758	dotest(ww_test_edeadlk_acquire_wrong, FAILURE, LOCKTYPE_WW);
1759	dotest(ww_test_edeadlk_acquire_wrong_slow, FAILURE, LOCKTYPE_WW);
1760	printk("\n");
1761
1762	print_testname("spinlock nest unlocked");
1763	dotest(ww_test_spin_nest_unlocked, FAILURE, LOCKTYPE_WW);
1764	printk("\n");
1765
1766	printk("  -----------------------------------------------------\n");
1767	printk("                                 |block | try  |context|\n");
1768	printk("  -----------------------------------------------------\n");
1769
1770	print_testname("context");
1771	dotest(ww_test_context_block, FAILURE, LOCKTYPE_WW);
1772	dotest(ww_test_context_try, SUCCESS, LOCKTYPE_WW);
1773	dotest(ww_test_context_context, SUCCESS, LOCKTYPE_WW);
1774	printk("\n");
1775
1776	print_testname("try");
1777	dotest(ww_test_try_block, FAILURE, LOCKTYPE_WW);
1778	dotest(ww_test_try_try, SUCCESS, LOCKTYPE_WW);
1779	dotest(ww_test_try_context, FAILURE, LOCKTYPE_WW);
1780	printk("\n");
1781
1782	print_testname("block");
1783	dotest(ww_test_block_block, FAILURE, LOCKTYPE_WW);
1784	dotest(ww_test_block_try, SUCCESS, LOCKTYPE_WW);
1785	dotest(ww_test_block_context, FAILURE, LOCKTYPE_WW);
1786	printk("\n");
1787
1788	print_testname("spinlock");
1789	dotest(ww_test_spin_block, FAILURE, LOCKTYPE_WW);
1790	dotest(ww_test_spin_try, SUCCESS, LOCKTYPE_WW);
1791	dotest(ww_test_spin_context, FAILURE, LOCKTYPE_WW);
1792	printk("\n");
1793}
1794
1795void locking_selftest(void)
1796{
1797	/*
1798	 * Got a locking failure before the selftest ran?
1799	 */
1800	if (!debug_locks) {
1801		printk("----------------------------------\n");
1802		printk("| Locking API testsuite disabled |\n");
1803		printk("----------------------------------\n");
1804		return;
1805	}
1806
1807	/*
1808	 * Run the testsuite:
1809	 */
1810	printk("------------------------\n");
1811	printk("| Locking API testsuite:\n");
1812	printk("----------------------------------------------------------------------------\n");
1813	printk("                                 | spin |wlock |rlock |mutex | wsem | rsem |\n");
1814	printk("  --------------------------------------------------------------------------\n");
1815
1816	init_shared_classes();
1817	debug_locks_silent = !debug_locks_verbose;
1818
1819	DO_TESTCASE_6R("A-A deadlock", AA);
1820	DO_TESTCASE_6R("A-B-B-A deadlock", ABBA);
1821	DO_TESTCASE_6R("A-B-B-C-C-A deadlock", ABBCCA);
1822	DO_TESTCASE_6R("A-B-C-A-B-C deadlock", ABCABC);
1823	DO_TESTCASE_6R("A-B-B-C-C-D-D-A deadlock", ABBCCDDA);
1824	DO_TESTCASE_6R("A-B-C-D-B-D-D-A deadlock", ABCDBDDA);
1825	DO_TESTCASE_6R("A-B-C-D-B-C-D-A deadlock", ABCDBCDA);
1826	DO_TESTCASE_6("double unlock", double_unlock);
1827	DO_TESTCASE_6("initialize held", init_held);
1828	DO_TESTCASE_6_SUCCESS("bad unlock order", bad_unlock_order);
1829
1830	printk("  --------------------------------------------------------------------------\n");
1831	print_testname("recursive read-lock");
1832	printk("             |");
1833	dotest(rlock_AA1, SUCCESS, LOCKTYPE_RWLOCK);
1834	printk("             |");
1835	dotest(rsem_AA1, FAILURE, LOCKTYPE_RWSEM);
1836	printk("\n");
1837
1838	print_testname("recursive read-lock #2");
1839	printk("             |");
1840	dotest(rlock_AA1B, SUCCESS, LOCKTYPE_RWLOCK);
1841	printk("             |");
1842	dotest(rsem_AA1B, FAILURE, LOCKTYPE_RWSEM);
1843	printk("\n");
1844
1845	print_testname("mixed read-write-lock");
1846	printk("             |");
1847	dotest(rlock_AA2, FAILURE, LOCKTYPE_RWLOCK);
1848	printk("             |");
1849	dotest(rsem_AA2, FAILURE, LOCKTYPE_RWSEM);
1850	printk("\n");
1851
1852	print_testname("mixed write-read-lock");
1853	printk("             |");
1854	dotest(rlock_AA3, FAILURE, LOCKTYPE_RWLOCK);
1855	printk("             |");
1856	dotest(rsem_AA3, FAILURE, LOCKTYPE_RWSEM);
1857	printk("\n");
1858
1859	printk("  --------------------------------------------------------------------------\n");
1860
1861	/*
1862	 * irq-context testcases:
1863	 */
1864	DO_TESTCASE_2x6("irqs-on + irq-safe-A", irqsafe1);
1865	DO_TESTCASE_2x3("sirq-safe-A => hirqs-on", irqsafe2A);
1866	DO_TESTCASE_2x6("safe-A + irqs-on", irqsafe2B);
1867	DO_TESTCASE_6x6("safe-A + unsafe-B #1", irqsafe3);
1868	DO_TESTCASE_6x6("safe-A + unsafe-B #2", irqsafe4);
1869	DO_TESTCASE_6x6RW("irq lock-inversion", irq_inversion);
1870
1871	DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion);
1872//	DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2);
1873
1874	ww_tests();
1875
1876	if (unexpected_testcase_failures) {
1877		printk("-----------------------------------------------------------------\n");
1878		debug_locks = 0;
1879		printk("BUG: %3d unexpected failures (out of %3d) - debugging disabled! |\n",
1880			unexpected_testcase_failures, testcase_total);
1881		printk("-----------------------------------------------------------------\n");
1882	} else if (expected_testcase_failures && testcase_successes) {
1883		printk("--------------------------------------------------------\n");
1884		printk("%3d out of %3d testcases failed, as expected. |\n",
1885			expected_testcase_failures, testcase_total);
1886		printk("----------------------------------------------------\n");
1887		debug_locks = 1;
1888	} else if (expected_testcase_failures && !testcase_successes) {
1889		printk("--------------------------------------------------------\n");
1890		printk("All %3d testcases failed, as expected. |\n",
1891			expected_testcase_failures);
1892		printk("----------------------------------------\n");
1893		debug_locks = 1;
1894	} else {
1895		printk("-------------------------------------------------------\n");
1896		printk("Good, all %3d testcases passed! |\n",
1897			testcase_successes);
1898		printk("---------------------------------\n");
1899		debug_locks = 1;
1900	}
1901	debug_locks_silent = 0;
1902}