1/*
  2 * Copyright 2005, Red Hat, Inc., Ingo Molnar
  3 * Released under the General Public License (GPL).
  4 *
  5 * This file contains the spinlock/rwlock implementations for
  6 * DEBUG_SPINLOCK.
  7 */
  8
  9#include <linux/spinlock.h>
 10#include <linux/nmi.h>
 11#include <linux/interrupt.h>
 12#include <linux/debug_locks.h>
 13#include <linux/delay.h>
 14#include <linux/export.h>
 15
 16void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
 17			  struct lock_class_key *key)
 18{
 19#ifdef CONFIG_DEBUG_LOCK_ALLOC
 20	/*
 21	 * Make sure we are not reinitializing a held lock:
 22	 */
 23	debug_check_no_locks_freed((void *)lock, sizeof(*lock));
 24	lockdep_init_map(&lock->dep_map, name, key, 0);
 25#endif
 26	lock->raw_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
 27	lock->magic = SPINLOCK_MAGIC;
 28	lock->owner = SPINLOCK_OWNER_INIT;
 29	lock->owner_cpu = -1;
 30}
 31
 32EXPORT_SYMBOL(__raw_spin_lock_init);
 33
 34void __rwlock_init(rwlock_t *lock, const char *name,
 35		   struct lock_class_key *key)
 36{
 37#ifdef CONFIG_DEBUG_LOCK_ALLOC
 38	/*
 39	 * Make sure we are not reinitializing a held lock:
 40	 */
 41	debug_check_no_locks_freed((void *)lock, sizeof(*lock));
 42	lockdep_init_map(&lock->dep_map, name, key, 0);
 43#endif
 44	lock->raw_lock = (arch_rwlock_t) __ARCH_RW_LOCK_UNLOCKED;
 45	lock->magic = RWLOCK_MAGIC;
 46	lock->owner = SPINLOCK_OWNER_INIT;
 47	lock->owner_cpu = -1;
 48}
 49
 50EXPORT_SYMBOL(__rwlock_init);
 51
 52static void spin_dump(raw_spinlock_t *lock, const char *msg)
 53{
 54	struct task_struct *owner = NULL;
 55
 56	if (lock->owner && lock->owner != SPINLOCK_OWNER_INIT)
 57		owner = lock->owner;
 58	printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n",
 59		msg, raw_smp_processor_id(),
 60		current->comm, task_pid_nr(current));
 61	printk(KERN_EMERG " lock: %pS, .magic: %08x, .owner: %s/%d, "
 62			".owner_cpu: %d\n",
 63		lock, lock->magic,
 64		owner ? owner->comm : "<none>",
 65		owner ? task_pid_nr(owner) : -1,
 66		lock->owner_cpu);
 67	dump_stack();
 68}
 69
 70static void spin_bug(raw_spinlock_t *lock, const char *msg)
 71{
 72	if (!debug_locks_off())
 73		return;
 74
 75	spin_dump(lock, msg);
 76}
 77
 78#define SPIN_BUG_ON(cond, lock, msg) if (unlikely(cond)) spin_bug(lock, msg)
 79
 80static inline void
 81debug_spin_lock_before(raw_spinlock_t *lock)
 82{
 83	SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
 84	SPIN_BUG_ON(lock->owner == current, lock, "recursion");
 85	SPIN_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
 86							lock, "cpu recursion");
 87}
 88
 89static inline void debug_spin_lock_after(raw_spinlock_t *lock)
 90{
 91	lock->owner_cpu = raw_smp_processor_id();
 92	lock->owner = current;
 93}
 94
 95static inline void debug_spin_unlock(raw_spinlock_t *lock)
 96{
 97	SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
 98	SPIN_BUG_ON(!raw_spin_is_locked(lock), lock, "already unlocked");
 99	SPIN_BUG_ON(lock->owner != current, lock, "wrong owner");
100	SPIN_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
101							lock, "wrong CPU");
102	lock->owner = SPINLOCK_OWNER_INIT;
103	lock->owner_cpu = -1;
104}
105
106/*
107 * We are now relying on the NMI watchdog to detect lockup instead of doing
108 * the detection here with an unfair lock which can cause problem of its own.
109 */
110void do_raw_spin_lock(raw_spinlock_t *lock)
111{
112	debug_spin_lock_before(lock);
113	arch_spin_lock(&lock->raw_lock);
114	mmiowb_spin_lock();
115	debug_spin_lock_after(lock);
116}
117
118int do_raw_spin_trylock(raw_spinlock_t *lock)
119{
120	int ret = arch_spin_trylock(&lock->raw_lock);
121
122	if (ret) {
123		mmiowb_spin_lock();
124		debug_spin_lock_after(lock);
125	}
126#ifndef CONFIG_SMP
127	/*
128	 * Must not happen on UP:
129	 */
130	SPIN_BUG_ON(!ret, lock, "trylock failure on UP");
131#endif
132	return ret;
133}
134
135void do_raw_spin_unlock(raw_spinlock_t *lock)
136{
137	mmiowb_spin_unlock();
138	debug_spin_unlock(lock);
139	arch_spin_unlock(&lock->raw_lock);
140}
141
142static void rwlock_bug(rwlock_t *lock, const char *msg)
143{
144	if (!debug_locks_off())
145		return;
146
147	printk(KERN_EMERG "BUG: rwlock %s on CPU#%d, %s/%d, %p\n",
148		msg, raw_smp_processor_id(), current->comm,
149		task_pid_nr(current), lock);
150	dump_stack();
151}
152
153#define RWLOCK_BUG_ON(cond, lock, msg) if (unlikely(cond)) rwlock_bug(lock, msg)
154
155void do_raw_read_lock(rwlock_t *lock)
156{
157	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
158	arch_read_lock(&lock->raw_lock);
159}
160
161int do_raw_read_trylock(rwlock_t *lock)
162{
163	int ret = arch_read_trylock(&lock->raw_lock);
164
165#ifndef CONFIG_SMP
166	/*
167	 * Must not happen on UP:
168	 */
169	RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
170#endif
171	return ret;
172}
173
174void do_raw_read_unlock(rwlock_t *lock)
175{
176	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
177	arch_read_unlock(&lock->raw_lock);
178}
179
180static inline void debug_write_lock_before(rwlock_t *lock)
181{
182	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
183	RWLOCK_BUG_ON(lock->owner == current, lock, "recursion");
184	RWLOCK_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
185							lock, "cpu recursion");
186}
187
188static inline void debug_write_lock_after(rwlock_t *lock)
189{
190	lock->owner_cpu = raw_smp_processor_id();
191	lock->owner = current;
192}
193
194static inline void debug_write_unlock(rwlock_t *lock)
195{
196	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
197	RWLOCK_BUG_ON(lock->owner != current, lock, "wrong owner");
198	RWLOCK_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
199							lock, "wrong CPU");
200	lock->owner = SPINLOCK_OWNER_INIT;
201	lock->owner_cpu = -1;
202}
203
204void do_raw_write_lock(rwlock_t *lock)
205{
206	debug_write_lock_before(lock);
207	arch_write_lock(&lock->raw_lock);
208	debug_write_lock_after(lock);
209}
210
211int do_raw_write_trylock(rwlock_t *lock)
212{
213	int ret = arch_write_trylock(&lock->raw_lock);
214
215	if (ret)
216		debug_write_lock_after(lock);
217#ifndef CONFIG_SMP
218	/*
219	 * Must not happen on UP:
220	 */
221	RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
222#endif
223	return ret;
224}
225
226void do_raw_write_unlock(rwlock_t *lock)
227{
228	debug_write_unlock(lock);
229	arch_write_unlock(&lock->raw_lock);
230}