1/*
  2 * net/core/dst.c	Protocol independent destination cache.
  3 *
  4 * Authors:		Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  5 *
  6 */
  7
  8#include <linux/bitops.h>
  9#include <linux/errno.h>
 10#include <linux/init.h>
 11#include <linux/kernel.h>
 12#include <linux/workqueue.h>
 13#include <linux/mm.h>
 14#include <linux/module.h>
 15#include <linux/slab.h>
 16#include <linux/netdevice.h>
 17#include <linux/skbuff.h>
 18#include <linux/string.h>
 19#include <linux/types.h>
 20#include <net/net_namespace.h>
 21#include <linux/sched.h>
 22#include <linux/prefetch.h>
 23
 24#include <net/dst.h>
 25
 26/*
 27 * Theory of operations:
 28 * 1) We use a list, protected by a spinlock, to add
 29 *    new entries from both BH and non-BH context.
 30 * 2) In order to keep spinlock held for a small delay,
 31 *    we use a second list where are stored long lived
 32 *    entries, that are handled by the garbage collect thread
 33 *    fired by a workqueue.
 34 * 3) This list is guarded by a mutex,
 35 *    so that the gc_task and dst_dev_event() can be synchronized.
 36 */
 37
 38/*
 39 * We want to keep lock & list close together
 40 * to dirty as few cache lines as possible in __dst_free().
 41 * As this is not a very strong hint, we dont force an alignment on SMP.
 42 */
 43static struct {
 44	spinlock_t		lock;
 45	struct dst_entry	*list;
 46	unsigned long		timer_inc;
 47	unsigned long		timer_expires;
 48} dst_garbage = {
 49	.lock = __SPIN_LOCK_UNLOCKED(dst_garbage.lock),
 50	.timer_inc = DST_GC_MAX,
 51};
 52static void dst_gc_task(struct work_struct *work);
 53static void ___dst_free(struct dst_entry *dst);
 54
 55static DECLARE_DELAYED_WORK(dst_gc_work, dst_gc_task);
 56
 57static DEFINE_MUTEX(dst_gc_mutex);
 58/*
 59 * long lived entries are maintained in this list, guarded by dst_gc_mutex
 60 */
 61static struct dst_entry         *dst_busy_list;
 62
 63static void dst_gc_task(struct work_struct *work)
 64{
 65	int    delayed = 0;
 66	int    work_performed = 0;
 67	unsigned long expires = ~0L;
 68	struct dst_entry *dst, *next, head;
 69	struct dst_entry *last = &head;
 70
 71	mutex_lock(&dst_gc_mutex);
 72	next = dst_busy_list;
 73
 74loop:
 75	while ((dst = next) != NULL) {
 76		next = dst->next;
 77		prefetch(&next->next);
 78		cond_resched();
 79		if (likely(atomic_read(&dst->__refcnt))) {
 80			last->next = dst;
 81			last = dst;
 82			delayed++;
 83			continue;
 84		}
 85		work_performed++;
 86
 87		dst = dst_destroy(dst);
 88		if (dst) {
 89			/* NOHASH and still referenced. Unless it is already
 90			 * on gc list, invalidate it and add to gc list.
 91			 *
 92			 * Note: this is temporary. Actually, NOHASH dst's
 93			 * must be obsoleted when parent is obsoleted.
 94			 * But we do not have state "obsoleted, but
 95			 * referenced by parent", so it is right.
 96			 */
 97			if (dst->obsolete > 1)
 98				continue;
 99
100			___dst_free(dst);
101			dst->next = next;
102			next = dst;
103		}
104	}
105
106	spin_lock_bh(&dst_garbage.lock);
107	next = dst_garbage.list;
108	if (next) {
109		dst_garbage.list = NULL;
110		spin_unlock_bh(&dst_garbage.lock);
111		goto loop;
112	}
113	last->next = NULL;
114	dst_busy_list = head.next;
115	if (!dst_busy_list)
116		dst_garbage.timer_inc = DST_GC_MAX;
117	else {
118		/*
119		 * if we freed less than 1/10 of delayed entries,
120		 * we can sleep longer.
121		 */
122		if (work_performed <= delayed/10) {
123			dst_garbage.timer_expires += dst_garbage.timer_inc;
124			if (dst_garbage.timer_expires > DST_GC_MAX)
125				dst_garbage.timer_expires = DST_GC_MAX;
126			dst_garbage.timer_inc += DST_GC_INC;
127		} else {
128			dst_garbage.timer_inc = DST_GC_INC;
129			dst_garbage.timer_expires = DST_GC_MIN;
130		}
131		expires = dst_garbage.timer_expires;
132		/*
133		 * if the next desired timer is more than 4 seconds in the
134		 * future then round the timer to whole seconds
135		 */
136		if (expires > 4*HZ)
137			expires = round_jiffies_relative(expires);
138		schedule_delayed_work(&dst_gc_work, expires);
139	}
140
141	spin_unlock_bh(&dst_garbage.lock);
142	mutex_unlock(&dst_gc_mutex);
143}
144
145int dst_discard(struct sk_buff *skb)
146{
147	kfree_skb(skb);
148	return 0;
149}
150EXPORT_SYMBOL(dst_discard);
151
152const u32 dst_default_metrics[RTAX_MAX];
153
154void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
155		int initial_ref, int initial_obsolete, int flags)
156{
157	struct dst_entry *dst;
158
159	if (ops->gc && dst_entries_get_fast(ops) > ops->gc_thresh) {
160		if (ops->gc(ops))
161			return NULL;
162	}
163	dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
164	if (!dst)
165		return NULL;
166	dst->child = NULL;
167	dst->dev = dev;
168	if (dev)
169		dev_hold(dev);
170	dst->ops = ops;
171	dst_init_metrics(dst, dst_default_metrics, true);
172	dst->expires = 0UL;
173	dst->path = dst;
174	dst->_neighbour = NULL;
175#ifdef CONFIG_XFRM
176	dst->xfrm = NULL;
177#endif
178	dst->input = dst_discard;
179	dst->output = dst_discard;
180	dst->error = 0;
181	dst->obsolete = initial_obsolete;
182	dst->header_len = 0;
183	dst->trailer_len = 0;
184#ifdef CONFIG_IP_ROUTE_CLASSID
185	dst->tclassid = 0;
186#endif
187	atomic_set(&dst->__refcnt, initial_ref);
188	dst->__use = 0;
189	dst->lastuse = jiffies;
190	dst->flags = flags;
191	dst->next = NULL;
192	if (!(flags & DST_NOCOUNT))
193		dst_entries_add(ops, 1);
194	return dst;
195}
196EXPORT_SYMBOL(dst_alloc);
197
198static void ___dst_free(struct dst_entry *dst)
199{
200	/* The first case (dev==NULL) is required, when
201	   protocol module is unloaded.
202	 */
203	if (dst->dev == NULL || !(dst->dev->flags&IFF_UP))
204		dst->input = dst->output = dst_discard;
205	dst->obsolete = 2;
206}
207
208void __dst_free(struct dst_entry *dst)
209{
210	spin_lock_bh(&dst_garbage.lock);
211	___dst_free(dst);
212	dst->next = dst_garbage.list;
213	dst_garbage.list = dst;
214	if (dst_garbage.timer_inc > DST_GC_INC) {
215		dst_garbage.timer_inc = DST_GC_INC;
216		dst_garbage.timer_expires = DST_GC_MIN;
217		cancel_delayed_work(&dst_gc_work);
218		schedule_delayed_work(&dst_gc_work, dst_garbage.timer_expires);
219	}
220	spin_unlock_bh(&dst_garbage.lock);
221}
222EXPORT_SYMBOL(__dst_free);
223
224struct dst_entry *dst_destroy(struct dst_entry * dst)
225{
226	struct dst_entry *child;
227	struct neighbour *neigh;
228
229	smp_rmb();
230
231again:
232	neigh = dst->_neighbour;
233	child = dst->child;
234
235	if (neigh) {
236		dst->_neighbour = NULL;
237		neigh_release(neigh);
238	}
239
240	if (!(dst->flags & DST_NOCOUNT))
241		dst_entries_add(dst->ops, -1);
242
243	if (dst->ops->destroy)
244		dst->ops->destroy(dst);
245	if (dst->dev)
246		dev_put(dst->dev);
247	kmem_cache_free(dst->ops->kmem_cachep, dst);
248
249	dst = child;
250	if (dst) {
251		int nohash = dst->flags & DST_NOHASH;
252
253		if (atomic_dec_and_test(&dst->__refcnt)) {
254			/* We were real parent of this dst, so kill child. */
255			if (nohash)
256				goto again;
257		} else {
258			/* Child is still referenced, return it for freeing. */
259			if (nohash)
260				return dst;
261			/* Child is still in his hash table */
262		}
263	}
264	return NULL;
265}
266EXPORT_SYMBOL(dst_destroy);
267
268void dst_release(struct dst_entry *dst)
269{
270	if (dst) {
271		int newrefcnt;
272
273		newrefcnt = atomic_dec_return(&dst->__refcnt);
274		WARN_ON(newrefcnt < 0);
275		if (unlikely(dst->flags & DST_NOCACHE) && !newrefcnt) {
276			dst = dst_destroy(dst);
277			if (dst)
278				__dst_free(dst);
279		}
280	}
281}
282EXPORT_SYMBOL(dst_release);
283
284u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old)
285{
286	u32 *p = kmalloc(sizeof(u32) * RTAX_MAX, GFP_ATOMIC);
287
288	if (p) {
289		u32 *old_p = __DST_METRICS_PTR(old);
290		unsigned long prev, new;
291
292		memcpy(p, old_p, sizeof(u32) * RTAX_MAX);
293
294		new = (unsigned long) p;
295		prev = cmpxchg(&dst->_metrics, old, new);
296
297		if (prev != old) {
298			kfree(p);
299			p = __DST_METRICS_PTR(prev);
300			if (prev & DST_METRICS_READ_ONLY)
301				p = NULL;
302		}
303	}
304	return p;
305}
306EXPORT_SYMBOL(dst_cow_metrics_generic);
307
308/* Caller asserts that dst_metrics_read_only(dst) is false.  */
309void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old)
310{
311	unsigned long prev, new;
312
313	new = ((unsigned long) dst_default_metrics) | DST_METRICS_READ_ONLY;
314	prev = cmpxchg(&dst->_metrics, old, new);
315	if (prev == old)
316		kfree(__DST_METRICS_PTR(old));
317}
318EXPORT_SYMBOL(__dst_destroy_metrics_generic);
319
320/**
321 * skb_dst_set_noref - sets skb dst, without a reference
322 * @skb: buffer
323 * @dst: dst entry
324 *
325 * Sets skb dst, assuming a reference was not taken on dst
326 * skb_dst_drop() should not dst_release() this dst
327 */
328void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
329{
330	WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
331	/* If dst not in cache, we must take a reference, because
332	 * dst_release() will destroy dst as soon as its refcount becomes zero
333	 */
334	if (unlikely(dst->flags & DST_NOCACHE)) {
335		dst_hold(dst);
336		skb_dst_set(skb, dst);
337	} else {
338		skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
339	}
340}
341EXPORT_SYMBOL(skb_dst_set_noref);
342
343/* Dirty hack. We did it in 2.2 (in __dst_free),
344 * we have _very_ good reasons not to repeat
345 * this mistake in 2.3, but we have no choice
346 * now. _It_ _is_ _explicit_ _deliberate_
347 * _race_ _condition_.
348 *
349 * Commented and originally written by Alexey.
350 */
351static void dst_ifdown(struct dst_entry *dst, struct net_device *dev,
352		       int unregister)
353{
354	if (dst->ops->ifdown)
355		dst->ops->ifdown(dst, dev, unregister);
356
357	if (dev != dst->dev)
358		return;
359
360	if (!unregister) {
361		dst->input = dst->output = dst_discard;
362	} else {
 
 
363		dst->dev = dev_net(dst->dev)->loopback_dev;
364		dev_hold(dst->dev);
365		dev_put(dev);
366		if (dst->_neighbour && dst->_neighbour->dev == dev) {
367			dst->_neighbour->dev = dst->dev;
 
 
368			dev_hold(dst->dev);
369			dev_put(dev);
370		}
 
371	}
372}
373
374static int dst_dev_event(struct notifier_block *this, unsigned long event,
375			 void *ptr)
376{
377	struct net_device *dev = ptr;
378	struct dst_entry *dst, *last = NULL;
379
380	switch (event) {
381	case NETDEV_UNREGISTER:
382	case NETDEV_DOWN:
383		mutex_lock(&dst_gc_mutex);
384		for (dst = dst_busy_list; dst; dst = dst->next) {
385			last = dst;
386			dst_ifdown(dst, dev, event != NETDEV_DOWN);
387		}
388
389		spin_lock_bh(&dst_garbage.lock);
390		dst = dst_garbage.list;
391		dst_garbage.list = NULL;
392		spin_unlock_bh(&dst_garbage.lock);
393
394		if (last)
395			last->next = dst;
396		else
397			dst_busy_list = dst;
398		for (; dst; dst = dst->next)
399			dst_ifdown(dst, dev, event != NETDEV_DOWN);
400		mutex_unlock(&dst_gc_mutex);
401		break;
402	}
403	return NOTIFY_DONE;
404}
405
406static struct notifier_block dst_dev_notifier = {
407	.notifier_call	= dst_dev_event,
408	.priority = -10, /* must be called after other network notifiers */
409};
410
411void __init dst_init(void)
412{
413	register_netdevice_notifier(&dst_dev_notifier);
414}

  1/*
  2 * net/core/dst.c	Protocol independent destination cache.
  3 *
  4 * Authors:		Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  5 *
  6 */
  7
  8#include <linux/bitops.h>
  9#include <linux/errno.h>
 10#include <linux/init.h>
 11#include <linux/kernel.h>
 12#include <linux/workqueue.h>
 13#include <linux/mm.h>
 14#include <linux/module.h>
 15#include <linux/slab.h>
 16#include <linux/netdevice.h>
 17#include <linux/skbuff.h>
 18#include <linux/string.h>
 19#include <linux/types.h>
 20#include <net/net_namespace.h>
 21#include <linux/sched.h>
 22#include <linux/prefetch.h>
 23
 24#include <net/dst.h>
 25
 26/*
 27 * Theory of operations:
 28 * 1) We use a list, protected by a spinlock, to add
 29 *    new entries from both BH and non-BH context.
 30 * 2) In order to keep spinlock held for a small delay,
 31 *    we use a second list where are stored long lived
 32 *    entries, that are handled by the garbage collect thread
 33 *    fired by a workqueue.
 34 * 3) This list is guarded by a mutex,
 35 *    so that the gc_task and dst_dev_event() can be synchronized.
 36 */
 37
 38/*
 39 * We want to keep lock & list close together
 40 * to dirty as few cache lines as possible in __dst_free().
 41 * As this is not a very strong hint, we dont force an alignment on SMP.
 42 */
 43static struct {
 44	spinlock_t		lock;
 45	struct dst_entry	*list;
 46	unsigned long		timer_inc;
 47	unsigned long		timer_expires;
 48} dst_garbage = {
 49	.lock = __SPIN_LOCK_UNLOCKED(dst_garbage.lock),
 50	.timer_inc = DST_GC_MAX,
 51};
 52static void dst_gc_task(struct work_struct *work);
 53static void ___dst_free(struct dst_entry *dst);
 54
 55static DECLARE_DELAYED_WORK(dst_gc_work, dst_gc_task);
 56
 57static DEFINE_MUTEX(dst_gc_mutex);
 58/*
 59 * long lived entries are maintained in this list, guarded by dst_gc_mutex
 60 */
 61static struct dst_entry         *dst_busy_list;
 62
 63static void dst_gc_task(struct work_struct *work)
 64{
 65	int    delayed = 0;
 66	int    work_performed = 0;
 67	unsigned long expires = ~0L;
 68	struct dst_entry *dst, *next, head;
 69	struct dst_entry *last = &head;
 70
 71	mutex_lock(&dst_gc_mutex);
 72	next = dst_busy_list;
 73
 74loop:
 75	while ((dst = next) != NULL) {
 76		next = dst->next;
 77		prefetch(&next->next);
 78		cond_resched();
 79		if (likely(atomic_read(&dst->__refcnt))) {
 80			last->next = dst;
 81			last = dst;
 82			delayed++;
 83			continue;
 84		}
 85		work_performed++;
 86
 87		dst = dst_destroy(dst);
 88		if (dst) {
 89			/* NOHASH and still referenced. Unless it is already
 90			 * on gc list, invalidate it and add to gc list.
 91			 *
 92			 * Note: this is temporary. Actually, NOHASH dst's
 93			 * must be obsoleted when parent is obsoleted.
 94			 * But we do not have state "obsoleted, but
 95			 * referenced by parent", so it is right.
 96			 */
 97			if (dst->obsolete > 1)
 98				continue;
 99
100			___dst_free(dst);
101			dst->next = next;
102			next = dst;
103		}
104	}
105
106	spin_lock_bh(&dst_garbage.lock);
107	next = dst_garbage.list;
108	if (next) {
109		dst_garbage.list = NULL;
110		spin_unlock_bh(&dst_garbage.lock);
111		goto loop;
112	}
113	last->next = NULL;
114	dst_busy_list = head.next;
115	if (!dst_busy_list)
116		dst_garbage.timer_inc = DST_GC_MAX;
117	else {
118		/*
119		 * if we freed less than 1/10 of delayed entries,
120		 * we can sleep longer.
121		 */
122		if (work_performed <= delayed/10) {
123			dst_garbage.timer_expires += dst_garbage.timer_inc;
124			if (dst_garbage.timer_expires > DST_GC_MAX)
125				dst_garbage.timer_expires = DST_GC_MAX;
126			dst_garbage.timer_inc += DST_GC_INC;
127		} else {
128			dst_garbage.timer_inc = DST_GC_INC;
129			dst_garbage.timer_expires = DST_GC_MIN;
130		}
131		expires = dst_garbage.timer_expires;
132		/*
133		 * if the next desired timer is more than 4 seconds in the
134		 * future then round the timer to whole seconds
135		 */
136		if (expires > 4*HZ)
137			expires = round_jiffies_relative(expires);
138		schedule_delayed_work(&dst_gc_work, expires);
139	}
140
141	spin_unlock_bh(&dst_garbage.lock);
142	mutex_unlock(&dst_gc_mutex);
143}
144
145int dst_discard(struct sk_buff *skb)
146{
147	kfree_skb(skb);
148	return 0;
149}
150EXPORT_SYMBOL(dst_discard);
151
152const u32 dst_default_metrics[RTAX_MAX];
153
154void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
155		int initial_ref, int initial_obsolete, int flags)
156{
157	struct dst_entry *dst;
158
159	if (ops->gc && dst_entries_get_fast(ops) > ops->gc_thresh) {
160		if (ops->gc(ops))
161			return NULL;
162	}
163	dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
164	if (!dst)
165		return NULL;
166	dst->child = NULL;
167	dst->dev = dev;
168	if (dev)
169		dev_hold(dev);
170	dst->ops = ops;
171	dst_init_metrics(dst, dst_default_metrics, true);
172	dst->expires = 0UL;
173	dst->path = dst;
174	RCU_INIT_POINTER(dst->_neighbour, NULL);
175#ifdef CONFIG_XFRM
176	dst->xfrm = NULL;
177#endif
178	dst->input = dst_discard;
179	dst->output = dst_discard;
180	dst->error = 0;
181	dst->obsolete = initial_obsolete;
182	dst->header_len = 0;
183	dst->trailer_len = 0;
184#ifdef CONFIG_IP_ROUTE_CLASSID
185	dst->tclassid = 0;
186#endif
187	atomic_set(&dst->__refcnt, initial_ref);
188	dst->__use = 0;
189	dst->lastuse = jiffies;
190	dst->flags = flags;
191	dst->next = NULL;
192	if (!(flags & DST_NOCOUNT))
193		dst_entries_add(ops, 1);
194	return dst;
195}
196EXPORT_SYMBOL(dst_alloc);
197
198static void ___dst_free(struct dst_entry *dst)
199{
200	/* The first case (dev==NULL) is required, when
201	   protocol module is unloaded.
202	 */
203	if (dst->dev == NULL || !(dst->dev->flags&IFF_UP))
204		dst->input = dst->output = dst_discard;
205	dst->obsolete = 2;
206}
207
208void __dst_free(struct dst_entry *dst)
209{
210	spin_lock_bh(&dst_garbage.lock);
211	___dst_free(dst);
212	dst->next = dst_garbage.list;
213	dst_garbage.list = dst;
214	if (dst_garbage.timer_inc > DST_GC_INC) {
215		dst_garbage.timer_inc = DST_GC_INC;
216		dst_garbage.timer_expires = DST_GC_MIN;
217		cancel_delayed_work(&dst_gc_work);
218		schedule_delayed_work(&dst_gc_work, dst_garbage.timer_expires);
219	}
220	spin_unlock_bh(&dst_garbage.lock);
221}
222EXPORT_SYMBOL(__dst_free);
223
224struct dst_entry *dst_destroy(struct dst_entry * dst)
225{
226	struct dst_entry *child;
227	struct neighbour *neigh;
228
229	smp_rmb();
230
231again:
232	neigh = rcu_dereference_protected(dst->_neighbour, 1);
233	child = dst->child;
234
235	if (neigh) {
236		RCU_INIT_POINTER(dst->_neighbour, NULL);
237		neigh_release(neigh);
238	}
239
240	if (!(dst->flags & DST_NOCOUNT))
241		dst_entries_add(dst->ops, -1);
242
243	if (dst->ops->destroy)
244		dst->ops->destroy(dst);
245	if (dst->dev)
246		dev_put(dst->dev);
247	kmem_cache_free(dst->ops->kmem_cachep, dst);
248
249	dst = child;
250	if (dst) {
251		int nohash = dst->flags & DST_NOHASH;
252
253		if (atomic_dec_and_test(&dst->__refcnt)) {
254			/* We were real parent of this dst, so kill child. */
255			if (nohash)
256				goto again;
257		} else {
258			/* Child is still referenced, return it for freeing. */
259			if (nohash)
260				return dst;
261			/* Child is still in his hash table */
262		}
263	}
264	return NULL;
265}
266EXPORT_SYMBOL(dst_destroy);
267
268void dst_release(struct dst_entry *dst)
269{
270	if (dst) {
271		int newrefcnt;
272
273		newrefcnt = atomic_dec_return(&dst->__refcnt);
274		WARN_ON(newrefcnt < 0);
275		if (unlikely(dst->flags & DST_NOCACHE) && !newrefcnt) {
276			dst = dst_destroy(dst);
277			if (dst)
278				__dst_free(dst);
279		}
280	}
281}
282EXPORT_SYMBOL(dst_release);
283
284u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old)
285{
286	u32 *p = kmalloc(sizeof(u32) * RTAX_MAX, GFP_ATOMIC);
287
288	if (p) {
289		u32 *old_p = __DST_METRICS_PTR(old);
290		unsigned long prev, new;
291
292		memcpy(p, old_p, sizeof(u32) * RTAX_MAX);
293
294		new = (unsigned long) p;
295		prev = cmpxchg(&dst->_metrics, old, new);
296
297		if (prev != old) {
298			kfree(p);
299			p = __DST_METRICS_PTR(prev);
300			if (prev & DST_METRICS_READ_ONLY)
301				p = NULL;
302		}
303	}
304	return p;
305}
306EXPORT_SYMBOL(dst_cow_metrics_generic);
307
308/* Caller asserts that dst_metrics_read_only(dst) is false.  */
309void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old)
310{
311	unsigned long prev, new;
312
313	new = ((unsigned long) dst_default_metrics) | DST_METRICS_READ_ONLY;
314	prev = cmpxchg(&dst->_metrics, old, new);
315	if (prev == old)
316		kfree(__DST_METRICS_PTR(old));
317}
318EXPORT_SYMBOL(__dst_destroy_metrics_generic);
319
320/**
321 * skb_dst_set_noref - sets skb dst, without a reference
322 * @skb: buffer
323 * @dst: dst entry
324 *
325 * Sets skb dst, assuming a reference was not taken on dst
326 * skb_dst_drop() should not dst_release() this dst
327 */
328void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
329{
330	WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
331	/* If dst not in cache, we must take a reference, because
332	 * dst_release() will destroy dst as soon as its refcount becomes zero
333	 */
334	if (unlikely(dst->flags & DST_NOCACHE)) {
335		dst_hold(dst);
336		skb_dst_set(skb, dst);
337	} else {
338		skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
339	}
340}
341EXPORT_SYMBOL(skb_dst_set_noref);
342
343/* Dirty hack. We did it in 2.2 (in __dst_free),
344 * we have _very_ good reasons not to repeat
345 * this mistake in 2.3, but we have no choice
346 * now. _It_ _is_ _explicit_ _deliberate_
347 * _race_ _condition_.
348 *
349 * Commented and originally written by Alexey.
350 */
351static void dst_ifdown(struct dst_entry *dst, struct net_device *dev,
352		       int unregister)
353{
354	if (dst->ops->ifdown)
355		dst->ops->ifdown(dst, dev, unregister);
356
357	if (dev != dst->dev)
358		return;
359
360	if (!unregister) {
361		dst->input = dst->output = dst_discard;
362	} else {
363		struct neighbour *neigh;
364
365		dst->dev = dev_net(dst->dev)->loopback_dev;
366		dev_hold(dst->dev);
367		dev_put(dev);
368		rcu_read_lock();
369		neigh = dst_get_neighbour_noref(dst);
370		if (neigh && neigh->dev == dev) {
371			neigh->dev = dst->dev;
372			dev_hold(dst->dev);
373			dev_put(dev);
374		}
375		rcu_read_unlock();
376	}
377}
378
379static int dst_dev_event(struct notifier_block *this, unsigned long event,
380			 void *ptr)
381{
382	struct net_device *dev = ptr;
383	struct dst_entry *dst, *last = NULL;
384
385	switch (event) {
386	case NETDEV_UNREGISTER:
387	case NETDEV_DOWN:
388		mutex_lock(&dst_gc_mutex);
389		for (dst = dst_busy_list; dst; dst = dst->next) {
390			last = dst;
391			dst_ifdown(dst, dev, event != NETDEV_DOWN);
392		}
393
394		spin_lock_bh(&dst_garbage.lock);
395		dst = dst_garbage.list;
396		dst_garbage.list = NULL;
397		spin_unlock_bh(&dst_garbage.lock);
398
399		if (last)
400			last->next = dst;
401		else
402			dst_busy_list = dst;
403		for (; dst; dst = dst->next)
404			dst_ifdown(dst, dev, event != NETDEV_DOWN);
405		mutex_unlock(&dst_gc_mutex);
406		break;
407	}
408	return NOTIFY_DONE;
409}
410
411static struct notifier_block dst_dev_notifier = {
412	.notifier_call	= dst_dev_event,
413	.priority = -10, /* must be called after other network notifiers */
414};
415
416void __init dst_init(void)
417{
418	register_netdevice_notifier(&dst_dev_notifier);
419}