Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * NET3 Protocol independent device support routines.
4 *
5 * Derived from the non IP parts of dev.c 1.0.19
6 * Authors: Ross Biro
7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8 * Mark Evans, <evansmp@uhura.aston.ac.uk>
9 *
10 * Additional Authors:
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
12 * Alan Cox <gw4pts@gw4pts.ampr.org>
13 * David Hinds <dahinds@users.sourceforge.net>
14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15 * Adam Sulmicki <adam@cfar.umd.edu>
16 * Pekka Riikonen <priikone@poesidon.pspt.fi>
17 *
18 * Changes:
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
20 * to 2 if register_netdev gets called
21 * before net_dev_init & also removed a
22 * few lines of code in the process.
23 * Alan Cox : device private ioctl copies fields back.
24 * Alan Cox : Transmit queue code does relevant
25 * stunts to keep the queue safe.
26 * Alan Cox : Fixed double lock.
27 * Alan Cox : Fixed promisc NULL pointer trap
28 * ???????? : Support the full private ioctl range
29 * Alan Cox : Moved ioctl permission check into
30 * drivers
31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
32 * Alan Cox : 100 backlog just doesn't cut it when
33 * you start doing multicast video 8)
34 * Alan Cox : Rewrote net_bh and list manager.
35 * Alan Cox : Fix ETH_P_ALL echoback lengths.
36 * Alan Cox : Took out transmit every packet pass
37 * Saved a few bytes in the ioctl handler
38 * Alan Cox : Network driver sets packet type before
39 * calling netif_rx. Saves a function
40 * call a packet.
41 * Alan Cox : Hashed net_bh()
42 * Richard Kooijman: Timestamp fixes.
43 * Alan Cox : Wrong field in SIOCGIFDSTADDR
44 * Alan Cox : Device lock protection.
45 * Alan Cox : Fixed nasty side effect of device close
46 * changes.
47 * Rudi Cilibrasi : Pass the right thing to
48 * set_mac_address()
49 * Dave Miller : 32bit quantity for the device lock to
50 * make it work out on a Sparc.
51 * Bjorn Ekwall : Added KERNELD hack.
52 * Alan Cox : Cleaned up the backlog initialise.
53 * Craig Metz : SIOCGIFCONF fix if space for under
54 * 1 device.
55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
56 * is no device open function.
57 * Andi Kleen : Fix error reporting for SIOCGIFCONF
58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
59 * Cyrus Durgin : Cleaned for KMOD
60 * Adam Sulmicki : Bug Fix : Network Device Unload
61 * A network device unload needs to purge
62 * the backlog queue.
63 * Paul Rusty Russell : SIOCSIFNAME
64 * Pekka Riikonen : Netdev boot-time settings code
65 * Andrew Morton : Make unregister_netdevice wait
66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
68 * - netif_rx() feedback
69 */
70
71#include <linux/uaccess.h>
72#include <linux/bitmap.h>
73#include <linux/capability.h>
74#include <linux/cpu.h>
75#include <linux/types.h>
76#include <linux/kernel.h>
77#include <linux/hash.h>
78#include <linux/slab.h>
79#include <linux/sched.h>
80#include <linux/sched/mm.h>
81#include <linux/mutex.h>
82#include <linux/rwsem.h>
83#include <linux/string.h>
84#include <linux/mm.h>
85#include <linux/socket.h>
86#include <linux/sockios.h>
87#include <linux/errno.h>
88#include <linux/interrupt.h>
89#include <linux/if_ether.h>
90#include <linux/netdevice.h>
91#include <linux/etherdevice.h>
92#include <linux/ethtool.h>
93#include <linux/skbuff.h>
94#include <linux/kthread.h>
95#include <linux/bpf.h>
96#include <linux/bpf_trace.h>
97#include <net/net_namespace.h>
98#include <net/sock.h>
99#include <net/busy_poll.h>
100#include <linux/rtnetlink.h>
101#include <linux/stat.h>
102#include <net/dsa.h>
103#include <net/dst.h>
104#include <net/dst_metadata.h>
105#include <net/gro.h>
106#include <net/pkt_sched.h>
107#include <net/pkt_cls.h>
108#include <net/checksum.h>
109#include <net/xfrm.h>
110#include <net/tcx.h>
111#include <linux/highmem.h>
112#include <linux/init.h>
113#include <linux/module.h>
114#include <linux/netpoll.h>
115#include <linux/rcupdate.h>
116#include <linux/delay.h>
117#include <net/iw_handler.h>
118#include <asm/current.h>
119#include <linux/audit.h>
120#include <linux/dmaengine.h>
121#include <linux/err.h>
122#include <linux/ctype.h>
123#include <linux/if_arp.h>
124#include <linux/if_vlan.h>
125#include <linux/ip.h>
126#include <net/ip.h>
127#include <net/mpls.h>
128#include <linux/ipv6.h>
129#include <linux/in.h>
130#include <linux/jhash.h>
131#include <linux/random.h>
132#include <trace/events/napi.h>
133#include <trace/events/net.h>
134#include <trace/events/skb.h>
135#include <trace/events/qdisc.h>
136#include <trace/events/xdp.h>
137#include <linux/inetdevice.h>
138#include <linux/cpu_rmap.h>
139#include <linux/static_key.h>
140#include <linux/hashtable.h>
141#include <linux/vmalloc.h>
142#include <linux/if_macvlan.h>
143#include <linux/errqueue.h>
144#include <linux/hrtimer.h>
145#include <linux/netfilter_netdev.h>
146#include <linux/crash_dump.h>
147#include <linux/sctp.h>
148#include <net/udp_tunnel.h>
149#include <linux/net_namespace.h>
150#include <linux/indirect_call_wrapper.h>
151#include <net/devlink.h>
152#include <linux/pm_runtime.h>
153#include <linux/prandom.h>
154#include <linux/once_lite.h>
155#include <net/netdev_rx_queue.h>
156
157#include "dev.h"
158#include "net-sysfs.h"
159
160static DEFINE_SPINLOCK(ptype_lock);
161struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
162struct list_head ptype_all __read_mostly; /* Taps */
163
164static int netif_rx_internal(struct sk_buff *skb);
165static int call_netdevice_notifiers_extack(unsigned long val,
166 struct net_device *dev,
167 struct netlink_ext_ack *extack);
168
169/*
170 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
171 * semaphore.
172 *
173 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
174 *
175 * Writers must hold the rtnl semaphore while they loop through the
176 * dev_base_head list, and hold dev_base_lock for writing when they do the
177 * actual updates. This allows pure readers to access the list even
178 * while a writer is preparing to update it.
179 *
180 * To put it another way, dev_base_lock is held for writing only to
181 * protect against pure readers; the rtnl semaphore provides the
182 * protection against other writers.
183 *
184 * See, for example usages, register_netdevice() and
185 * unregister_netdevice(), which must be called with the rtnl
186 * semaphore held.
187 */
188DEFINE_RWLOCK(dev_base_lock);
189EXPORT_SYMBOL(dev_base_lock);
190
191static DEFINE_MUTEX(ifalias_mutex);
192
193/* protects napi_hash addition/deletion and napi_gen_id */
194static DEFINE_SPINLOCK(napi_hash_lock);
195
196static unsigned int napi_gen_id = NR_CPUS;
197static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
198
199static DECLARE_RWSEM(devnet_rename_sem);
200
201static inline void dev_base_seq_inc(struct net *net)
202{
203 while (++net->dev_base_seq == 0)
204 ;
205}
206
207static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
208{
209 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
210
211 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
212}
213
214static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
215{
216 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
217}
218
219static inline void rps_lock_irqsave(struct softnet_data *sd,
220 unsigned long *flags)
221{
222 if (IS_ENABLED(CONFIG_RPS))
223 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
224 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
225 local_irq_save(*flags);
226}
227
228static inline void rps_lock_irq_disable(struct softnet_data *sd)
229{
230 if (IS_ENABLED(CONFIG_RPS))
231 spin_lock_irq(&sd->input_pkt_queue.lock);
232 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
233 local_irq_disable();
234}
235
236static inline void rps_unlock_irq_restore(struct softnet_data *sd,
237 unsigned long *flags)
238{
239 if (IS_ENABLED(CONFIG_RPS))
240 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
241 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
242 local_irq_restore(*flags);
243}
244
245static inline void rps_unlock_irq_enable(struct softnet_data *sd)
246{
247 if (IS_ENABLED(CONFIG_RPS))
248 spin_unlock_irq(&sd->input_pkt_queue.lock);
249 else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
250 local_irq_enable();
251}
252
253static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
254 const char *name)
255{
256 struct netdev_name_node *name_node;
257
258 name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
259 if (!name_node)
260 return NULL;
261 INIT_HLIST_NODE(&name_node->hlist);
262 name_node->dev = dev;
263 name_node->name = name;
264 return name_node;
265}
266
267static struct netdev_name_node *
268netdev_name_node_head_alloc(struct net_device *dev)
269{
270 struct netdev_name_node *name_node;
271
272 name_node = netdev_name_node_alloc(dev, dev->name);
273 if (!name_node)
274 return NULL;
275 INIT_LIST_HEAD(&name_node->list);
276 return name_node;
277}
278
279static void netdev_name_node_free(struct netdev_name_node *name_node)
280{
281 kfree(name_node);
282}
283
284static void netdev_name_node_add(struct net *net,
285 struct netdev_name_node *name_node)
286{
287 hlist_add_head_rcu(&name_node->hlist,
288 dev_name_hash(net, name_node->name));
289}
290
291static void netdev_name_node_del(struct netdev_name_node *name_node)
292{
293 hlist_del_rcu(&name_node->hlist);
294}
295
296static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
297 const char *name)
298{
299 struct hlist_head *head = dev_name_hash(net, name);
300 struct netdev_name_node *name_node;
301
302 hlist_for_each_entry(name_node, head, hlist)
303 if (!strcmp(name_node->name, name))
304 return name_node;
305 return NULL;
306}
307
308static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
309 const char *name)
310{
311 struct hlist_head *head = dev_name_hash(net, name);
312 struct netdev_name_node *name_node;
313
314 hlist_for_each_entry_rcu(name_node, head, hlist)
315 if (!strcmp(name_node->name, name))
316 return name_node;
317 return NULL;
318}
319
320bool netdev_name_in_use(struct net *net, const char *name)
321{
322 return netdev_name_node_lookup(net, name);
323}
324EXPORT_SYMBOL(netdev_name_in_use);
325
326int netdev_name_node_alt_create(struct net_device *dev, const char *name)
327{
328 struct netdev_name_node *name_node;
329 struct net *net = dev_net(dev);
330
331 name_node = netdev_name_node_lookup(net, name);
332 if (name_node)
333 return -EEXIST;
334 name_node = netdev_name_node_alloc(dev, name);
335 if (!name_node)
336 return -ENOMEM;
337 netdev_name_node_add(net, name_node);
338 /* The node that holds dev->name acts as a head of per-device list. */
339 list_add_tail_rcu(&name_node->list, &dev->name_node->list);
340
341 return 0;
342}
343
344static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
345{
346 list_del(&name_node->list);
347 kfree(name_node->name);
348 netdev_name_node_free(name_node);
349}
350
351int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
352{
353 struct netdev_name_node *name_node;
354 struct net *net = dev_net(dev);
355
356 name_node = netdev_name_node_lookup(net, name);
357 if (!name_node)
358 return -ENOENT;
359 /* lookup might have found our primary name or a name belonging
360 * to another device.
361 */
362 if (name_node == dev->name_node || name_node->dev != dev)
363 return -EINVAL;
364
365 netdev_name_node_del(name_node);
366 synchronize_rcu();
367 __netdev_name_node_alt_destroy(name_node);
368
369 return 0;
370}
371
372static void netdev_name_node_alt_flush(struct net_device *dev)
373{
374 struct netdev_name_node *name_node, *tmp;
375
376 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
377 __netdev_name_node_alt_destroy(name_node);
378}
379
380/* Device list insertion */
381static void list_netdevice(struct net_device *dev)
382{
383 struct netdev_name_node *name_node;
384 struct net *net = dev_net(dev);
385
386 ASSERT_RTNL();
387
388 write_lock(&dev_base_lock);
389 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
390 netdev_name_node_add(net, dev->name_node);
391 hlist_add_head_rcu(&dev->index_hlist,
392 dev_index_hash(net, dev->ifindex));
393 write_unlock(&dev_base_lock);
394
395 netdev_for_each_altname(dev, name_node)
396 netdev_name_node_add(net, name_node);
397
398 /* We reserved the ifindex, this can't fail */
399 WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL));
400
401 dev_base_seq_inc(net);
402}
403
404/* Device list removal
405 * caller must respect a RCU grace period before freeing/reusing dev
406 */
407static void unlist_netdevice(struct net_device *dev, bool lock)
408{
409 struct netdev_name_node *name_node;
410 struct net *net = dev_net(dev);
411
412 ASSERT_RTNL();
413
414 xa_erase(&net->dev_by_index, dev->ifindex);
415
416 netdev_for_each_altname(dev, name_node)
417 netdev_name_node_del(name_node);
418
419 /* Unlink dev from the device chain */
420 if (lock)
421 write_lock(&dev_base_lock);
422 list_del_rcu(&dev->dev_list);
423 netdev_name_node_del(dev->name_node);
424 hlist_del_rcu(&dev->index_hlist);
425 if (lock)
426 write_unlock(&dev_base_lock);
427
428 dev_base_seq_inc(dev_net(dev));
429}
430
431/*
432 * Our notifier list
433 */
434
435static RAW_NOTIFIER_HEAD(netdev_chain);
436
437/*
438 * Device drivers call our routines to queue packets here. We empty the
439 * queue in the local softnet handler.
440 */
441
442DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
443EXPORT_PER_CPU_SYMBOL(softnet_data);
444
445#ifdef CONFIG_LOCKDEP
446/*
447 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
448 * according to dev->type
449 */
450static const unsigned short netdev_lock_type[] = {
451 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
452 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
453 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
454 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
455 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
456 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
457 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
458 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
459 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
460 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
461 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
462 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
463 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
464 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
465 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
466
467static const char *const netdev_lock_name[] = {
468 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
469 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
470 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
471 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
472 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
473 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
474 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
475 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
476 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
477 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
478 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
479 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
480 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
481 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
482 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
483
484static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
485static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
486
487static inline unsigned short netdev_lock_pos(unsigned short dev_type)
488{
489 int i;
490
491 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
492 if (netdev_lock_type[i] == dev_type)
493 return i;
494 /* the last key is used by default */
495 return ARRAY_SIZE(netdev_lock_type) - 1;
496}
497
498static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
499 unsigned short dev_type)
500{
501 int i;
502
503 i = netdev_lock_pos(dev_type);
504 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
505 netdev_lock_name[i]);
506}
507
508static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
509{
510 int i;
511
512 i = netdev_lock_pos(dev->type);
513 lockdep_set_class_and_name(&dev->addr_list_lock,
514 &netdev_addr_lock_key[i],
515 netdev_lock_name[i]);
516}
517#else
518static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
519 unsigned short dev_type)
520{
521}
522
523static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
524{
525}
526#endif
527
528/*******************************************************************************
529 *
530 * Protocol management and registration routines
531 *
532 *******************************************************************************/
533
534
535/*
536 * Add a protocol ID to the list. Now that the input handler is
537 * smarter we can dispense with all the messy stuff that used to be
538 * here.
539 *
540 * BEWARE!!! Protocol handlers, mangling input packets,
541 * MUST BE last in hash buckets and checking protocol handlers
542 * MUST start from promiscuous ptype_all chain in net_bh.
543 * It is true now, do not change it.
544 * Explanation follows: if protocol handler, mangling packet, will
545 * be the first on list, it is not able to sense, that packet
546 * is cloned and should be copied-on-write, so that it will
547 * change it and subsequent readers will get broken packet.
548 * --ANK (980803)
549 */
550
551static inline struct list_head *ptype_head(const struct packet_type *pt)
552{
553 if (pt->type == htons(ETH_P_ALL))
554 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
555 else
556 return pt->dev ? &pt->dev->ptype_specific :
557 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
558}
559
560/**
561 * dev_add_pack - add packet handler
562 * @pt: packet type declaration
563 *
564 * Add a protocol handler to the networking stack. The passed &packet_type
565 * is linked into kernel lists and may not be freed until it has been
566 * removed from the kernel lists.
567 *
568 * This call does not sleep therefore it can not
569 * guarantee all CPU's that are in middle of receiving packets
570 * will see the new packet type (until the next received packet).
571 */
572
573void dev_add_pack(struct packet_type *pt)
574{
575 struct list_head *head = ptype_head(pt);
576
577 spin_lock(&ptype_lock);
578 list_add_rcu(&pt->list, head);
579 spin_unlock(&ptype_lock);
580}
581EXPORT_SYMBOL(dev_add_pack);
582
583/**
584 * __dev_remove_pack - remove packet handler
585 * @pt: packet type declaration
586 *
587 * Remove a protocol handler that was previously added to the kernel
588 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
589 * from the kernel lists and can be freed or reused once this function
590 * returns.
591 *
592 * The packet type might still be in use by receivers
593 * and must not be freed until after all the CPU's have gone
594 * through a quiescent state.
595 */
596void __dev_remove_pack(struct packet_type *pt)
597{
598 struct list_head *head = ptype_head(pt);
599 struct packet_type *pt1;
600
601 spin_lock(&ptype_lock);
602
603 list_for_each_entry(pt1, head, list) {
604 if (pt == pt1) {
605 list_del_rcu(&pt->list);
606 goto out;
607 }
608 }
609
610 pr_warn("dev_remove_pack: %p not found\n", pt);
611out:
612 spin_unlock(&ptype_lock);
613}
614EXPORT_SYMBOL(__dev_remove_pack);
615
616/**
617 * dev_remove_pack - remove packet handler
618 * @pt: packet type declaration
619 *
620 * Remove a protocol handler that was previously added to the kernel
621 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
622 * from the kernel lists and can be freed or reused once this function
623 * returns.
624 *
625 * This call sleeps to guarantee that no CPU is looking at the packet
626 * type after return.
627 */
628void dev_remove_pack(struct packet_type *pt)
629{
630 __dev_remove_pack(pt);
631
632 synchronize_net();
633}
634EXPORT_SYMBOL(dev_remove_pack);
635
636
637/*******************************************************************************
638 *
639 * Device Interface Subroutines
640 *
641 *******************************************************************************/
642
643/**
644 * dev_get_iflink - get 'iflink' value of a interface
645 * @dev: targeted interface
646 *
647 * Indicates the ifindex the interface is linked to.
648 * Physical interfaces have the same 'ifindex' and 'iflink' values.
649 */
650
651int dev_get_iflink(const struct net_device *dev)
652{
653 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
654 return dev->netdev_ops->ndo_get_iflink(dev);
655
656 return dev->ifindex;
657}
658EXPORT_SYMBOL(dev_get_iflink);
659
660/**
661 * dev_fill_metadata_dst - Retrieve tunnel egress information.
662 * @dev: targeted interface
663 * @skb: The packet.
664 *
665 * For better visibility of tunnel traffic OVS needs to retrieve
666 * egress tunnel information for a packet. Following API allows
667 * user to get this info.
668 */
669int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
670{
671 struct ip_tunnel_info *info;
672
673 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
674 return -EINVAL;
675
676 info = skb_tunnel_info_unclone(skb);
677 if (!info)
678 return -ENOMEM;
679 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
680 return -EINVAL;
681
682 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
683}
684EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
685
686static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
687{
688 int k = stack->num_paths++;
689
690 if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
691 return NULL;
692
693 return &stack->path[k];
694}
695
696int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
697 struct net_device_path_stack *stack)
698{
699 const struct net_device *last_dev;
700 struct net_device_path_ctx ctx = {
701 .dev = dev,
702 };
703 struct net_device_path *path;
704 int ret = 0;
705
706 memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
707 stack->num_paths = 0;
708 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
709 last_dev = ctx.dev;
710 path = dev_fwd_path(stack);
711 if (!path)
712 return -1;
713
714 memset(path, 0, sizeof(struct net_device_path));
715 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
716 if (ret < 0)
717 return -1;
718
719 if (WARN_ON_ONCE(last_dev == ctx.dev))
720 return -1;
721 }
722
723 if (!ctx.dev)
724 return ret;
725
726 path = dev_fwd_path(stack);
727 if (!path)
728 return -1;
729 path->type = DEV_PATH_ETHERNET;
730 path->dev = ctx.dev;
731
732 return ret;
733}
734EXPORT_SYMBOL_GPL(dev_fill_forward_path);
735
736/**
737 * __dev_get_by_name - find a device by its name
738 * @net: the applicable net namespace
739 * @name: name to find
740 *
741 * Find an interface by name. Must be called under RTNL semaphore
742 * or @dev_base_lock. If the name is found a pointer to the device
743 * is returned. If the name is not found then %NULL is returned. The
744 * reference counters are not incremented so the caller must be
745 * careful with locks.
746 */
747
748struct net_device *__dev_get_by_name(struct net *net, const char *name)
749{
750 struct netdev_name_node *node_name;
751
752 node_name = netdev_name_node_lookup(net, name);
753 return node_name ? node_name->dev : NULL;
754}
755EXPORT_SYMBOL(__dev_get_by_name);
756
757/**
758 * dev_get_by_name_rcu - find a device by its name
759 * @net: the applicable net namespace
760 * @name: name to find
761 *
762 * Find an interface by name.
763 * If the name is found a pointer to the device is returned.
764 * If the name is not found then %NULL is returned.
765 * The reference counters are not incremented so the caller must be
766 * careful with locks. The caller must hold RCU lock.
767 */
768
769struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
770{
771 struct netdev_name_node *node_name;
772
773 node_name = netdev_name_node_lookup_rcu(net, name);
774 return node_name ? node_name->dev : NULL;
775}
776EXPORT_SYMBOL(dev_get_by_name_rcu);
777
778/* Deprecated for new users, call netdev_get_by_name() instead */
779struct net_device *dev_get_by_name(struct net *net, const char *name)
780{
781 struct net_device *dev;
782
783 rcu_read_lock();
784 dev = dev_get_by_name_rcu(net, name);
785 dev_hold(dev);
786 rcu_read_unlock();
787 return dev;
788}
789EXPORT_SYMBOL(dev_get_by_name);
790
791/**
792 * netdev_get_by_name() - find a device by its name
793 * @net: the applicable net namespace
794 * @name: name to find
795 * @tracker: tracking object for the acquired reference
796 * @gfp: allocation flags for the tracker
797 *
798 * Find an interface by name. This can be called from any
799 * context and does its own locking. The returned handle has
800 * the usage count incremented and the caller must use netdev_put() to
801 * release it when it is no longer needed. %NULL is returned if no
802 * matching device is found.
803 */
804struct net_device *netdev_get_by_name(struct net *net, const char *name,
805 netdevice_tracker *tracker, gfp_t gfp)
806{
807 struct net_device *dev;
808
809 dev = dev_get_by_name(net, name);
810 if (dev)
811 netdev_tracker_alloc(dev, tracker, gfp);
812 return dev;
813}
814EXPORT_SYMBOL(netdev_get_by_name);
815
816/**
817 * __dev_get_by_index - find a device by its ifindex
818 * @net: the applicable net namespace
819 * @ifindex: index of device
820 *
821 * Search for an interface by index. Returns %NULL if the device
822 * is not found or a pointer to the device. The device has not
823 * had its reference counter increased so the caller must be careful
824 * about locking. The caller must hold either the RTNL semaphore
825 * or @dev_base_lock.
826 */
827
828struct net_device *__dev_get_by_index(struct net *net, int ifindex)
829{
830 struct net_device *dev;
831 struct hlist_head *head = dev_index_hash(net, ifindex);
832
833 hlist_for_each_entry(dev, head, index_hlist)
834 if (dev->ifindex == ifindex)
835 return dev;
836
837 return NULL;
838}
839EXPORT_SYMBOL(__dev_get_by_index);
840
841/**
842 * dev_get_by_index_rcu - find a device by its ifindex
843 * @net: the applicable net namespace
844 * @ifindex: index of device
845 *
846 * Search for an interface by index. Returns %NULL if the device
847 * is not found or a pointer to the device. The device has not
848 * had its reference counter increased so the caller must be careful
849 * about locking. The caller must hold RCU lock.
850 */
851
852struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
853{
854 struct net_device *dev;
855 struct hlist_head *head = dev_index_hash(net, ifindex);
856
857 hlist_for_each_entry_rcu(dev, head, index_hlist)
858 if (dev->ifindex == ifindex)
859 return dev;
860
861 return NULL;
862}
863EXPORT_SYMBOL(dev_get_by_index_rcu);
864
865/* Deprecated for new users, call netdev_get_by_index() instead */
866struct net_device *dev_get_by_index(struct net *net, int ifindex)
867{
868 struct net_device *dev;
869
870 rcu_read_lock();
871 dev = dev_get_by_index_rcu(net, ifindex);
872 dev_hold(dev);
873 rcu_read_unlock();
874 return dev;
875}
876EXPORT_SYMBOL(dev_get_by_index);
877
878/**
879 * netdev_get_by_index() - find a device by its ifindex
880 * @net: the applicable net namespace
881 * @ifindex: index of device
882 * @tracker: tracking object for the acquired reference
883 * @gfp: allocation flags for the tracker
884 *
885 * Search for an interface by index. Returns NULL if the device
886 * is not found or a pointer to the device. The device returned has
887 * had a reference added and the pointer is safe until the user calls
888 * netdev_put() to indicate they have finished with it.
889 */
890struct net_device *netdev_get_by_index(struct net *net, int ifindex,
891 netdevice_tracker *tracker, gfp_t gfp)
892{
893 struct net_device *dev;
894
895 dev = dev_get_by_index(net, ifindex);
896 if (dev)
897 netdev_tracker_alloc(dev, tracker, gfp);
898 return dev;
899}
900EXPORT_SYMBOL(netdev_get_by_index);
901
902/**
903 * dev_get_by_napi_id - find a device by napi_id
904 * @napi_id: ID of the NAPI struct
905 *
906 * Search for an interface by NAPI ID. Returns %NULL if the device
907 * is not found or a pointer to the device. The device has not had
908 * its reference counter increased so the caller must be careful
909 * about locking. The caller must hold RCU lock.
910 */
911
912struct net_device *dev_get_by_napi_id(unsigned int napi_id)
913{
914 struct napi_struct *napi;
915
916 WARN_ON_ONCE(!rcu_read_lock_held());
917
918 if (napi_id < MIN_NAPI_ID)
919 return NULL;
920
921 napi = napi_by_id(napi_id);
922
923 return napi ? napi->dev : NULL;
924}
925EXPORT_SYMBOL(dev_get_by_napi_id);
926
927/**
928 * netdev_get_name - get a netdevice name, knowing its ifindex.
929 * @net: network namespace
930 * @name: a pointer to the buffer where the name will be stored.
931 * @ifindex: the ifindex of the interface to get the name from.
932 */
933int netdev_get_name(struct net *net, char *name, int ifindex)
934{
935 struct net_device *dev;
936 int ret;
937
938 down_read(&devnet_rename_sem);
939 rcu_read_lock();
940
941 dev = dev_get_by_index_rcu(net, ifindex);
942 if (!dev) {
943 ret = -ENODEV;
944 goto out;
945 }
946
947 strcpy(name, dev->name);
948
949 ret = 0;
950out:
951 rcu_read_unlock();
952 up_read(&devnet_rename_sem);
953 return ret;
954}
955
956/**
957 * dev_getbyhwaddr_rcu - find a device by its hardware address
958 * @net: the applicable net namespace
959 * @type: media type of device
960 * @ha: hardware address
961 *
962 * Search for an interface by MAC address. Returns NULL if the device
963 * is not found or a pointer to the device.
964 * The caller must hold RCU or RTNL.
965 * The returned device has not had its ref count increased
966 * and the caller must therefore be careful about locking
967 *
968 */
969
970struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
971 const char *ha)
972{
973 struct net_device *dev;
974
975 for_each_netdev_rcu(net, dev)
976 if (dev->type == type &&
977 !memcmp(dev->dev_addr, ha, dev->addr_len))
978 return dev;
979
980 return NULL;
981}
982EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
983
984struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
985{
986 struct net_device *dev, *ret = NULL;
987
988 rcu_read_lock();
989 for_each_netdev_rcu(net, dev)
990 if (dev->type == type) {
991 dev_hold(dev);
992 ret = dev;
993 break;
994 }
995 rcu_read_unlock();
996 return ret;
997}
998EXPORT_SYMBOL(dev_getfirstbyhwtype);
999
1000/**
1001 * __dev_get_by_flags - find any device with given flags
1002 * @net: the applicable net namespace
1003 * @if_flags: IFF_* values
1004 * @mask: bitmask of bits in if_flags to check
1005 *
1006 * Search for any interface with the given flags. Returns NULL if a device
1007 * is not found or a pointer to the device. Must be called inside
1008 * rtnl_lock(), and result refcount is unchanged.
1009 */
1010
1011struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1012 unsigned short mask)
1013{
1014 struct net_device *dev, *ret;
1015
1016 ASSERT_RTNL();
1017
1018 ret = NULL;
1019 for_each_netdev(net, dev) {
1020 if (((dev->flags ^ if_flags) & mask) == 0) {
1021 ret = dev;
1022 break;
1023 }
1024 }
1025 return ret;
1026}
1027EXPORT_SYMBOL(__dev_get_by_flags);
1028
1029/**
1030 * dev_valid_name - check if name is okay for network device
1031 * @name: name string
1032 *
1033 * Network device names need to be valid file names to
1034 * allow sysfs to work. We also disallow any kind of
1035 * whitespace.
1036 */
1037bool dev_valid_name(const char *name)
1038{
1039 if (*name == '\0')
1040 return false;
1041 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1042 return false;
1043 if (!strcmp(name, ".") || !strcmp(name, ".."))
1044 return false;
1045
1046 while (*name) {
1047 if (*name == '/' || *name == ':' || isspace(*name))
1048 return false;
1049 name++;
1050 }
1051 return true;
1052}
1053EXPORT_SYMBOL(dev_valid_name);
1054
1055/**
1056 * __dev_alloc_name - allocate a name for a device
1057 * @net: network namespace to allocate the device name in
1058 * @name: name format string
1059 * @res: result name string
1060 *
1061 * Passed a format string - eg "lt%d" it will try and find a suitable
1062 * id. It scans list of devices to build up a free map, then chooses
1063 * the first empty slot. The caller must hold the dev_base or rtnl lock
1064 * while allocating the name and adding the device in order to avoid
1065 * duplicates.
1066 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1067 * Returns the number of the unit assigned or a negative errno code.
1068 */
1069
1070static int __dev_alloc_name(struct net *net, const char *name, char *res)
1071{
1072 int i = 0;
1073 const char *p;
1074 const int max_netdevices = 8*PAGE_SIZE;
1075 unsigned long *inuse;
1076 struct net_device *d;
1077 char buf[IFNAMSIZ];
1078
1079 /* Verify the string as this thing may have come from the user.
1080 * There must be one "%d" and no other "%" characters.
1081 */
1082 p = strchr(name, '%');
1083 if (!p || p[1] != 'd' || strchr(p + 2, '%'))
1084 return -EINVAL;
1085
1086 /* Use one page as a bit array of possible slots */
1087 inuse = bitmap_zalloc(max_netdevices, GFP_ATOMIC);
1088 if (!inuse)
1089 return -ENOMEM;
1090
1091 for_each_netdev(net, d) {
1092 struct netdev_name_node *name_node;
1093
1094 netdev_for_each_altname(d, name_node) {
1095 if (!sscanf(name_node->name, name, &i))
1096 continue;
1097 if (i < 0 || i >= max_netdevices)
1098 continue;
1099
1100 /* avoid cases where sscanf is not exact inverse of printf */
1101 snprintf(buf, IFNAMSIZ, name, i);
1102 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1103 __set_bit(i, inuse);
1104 }
1105 if (!sscanf(d->name, name, &i))
1106 continue;
1107 if (i < 0 || i >= max_netdevices)
1108 continue;
1109
1110 /* avoid cases where sscanf is not exact inverse of printf */
1111 snprintf(buf, IFNAMSIZ, name, i);
1112 if (!strncmp(buf, d->name, IFNAMSIZ))
1113 __set_bit(i, inuse);
1114 }
1115
1116 i = find_first_zero_bit(inuse, max_netdevices);
1117 bitmap_free(inuse);
1118 if (i == max_netdevices)
1119 return -ENFILE;
1120
1121 /* 'res' and 'name' could overlap, use 'buf' as an intermediate buffer */
1122 strscpy(buf, name, IFNAMSIZ);
1123 snprintf(res, IFNAMSIZ, buf, i);
1124 return i;
1125}
1126
1127/* Returns negative errno or allocated unit id (see __dev_alloc_name()) */
1128static int dev_prep_valid_name(struct net *net, struct net_device *dev,
1129 const char *want_name, char *out_name,
1130 int dup_errno)
1131{
1132 if (!dev_valid_name(want_name))
1133 return -EINVAL;
1134
1135 if (strchr(want_name, '%'))
1136 return __dev_alloc_name(net, want_name, out_name);
1137
1138 if (netdev_name_in_use(net, want_name))
1139 return -dup_errno;
1140 if (out_name != want_name)
1141 strscpy(out_name, want_name, IFNAMSIZ);
1142 return 0;
1143}
1144
1145/**
1146 * dev_alloc_name - allocate a name for a device
1147 * @dev: device
1148 * @name: name format string
1149 *
1150 * Passed a format string - eg "lt%d" it will try and find a suitable
1151 * id. It scans list of devices to build up a free map, then chooses
1152 * the first empty slot. The caller must hold the dev_base or rtnl lock
1153 * while allocating the name and adding the device in order to avoid
1154 * duplicates.
1155 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1156 * Returns the number of the unit assigned or a negative errno code.
1157 */
1158
1159int dev_alloc_name(struct net_device *dev, const char *name)
1160{
1161 return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE);
1162}
1163EXPORT_SYMBOL(dev_alloc_name);
1164
1165static int dev_get_valid_name(struct net *net, struct net_device *dev,
1166 const char *name)
1167{
1168 int ret;
1169
1170 ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST);
1171 return ret < 0 ? ret : 0;
1172}
1173
1174/**
1175 * dev_change_name - change name of a device
1176 * @dev: device
1177 * @newname: name (or format string) must be at least IFNAMSIZ
1178 *
1179 * Change name of a device, can pass format strings "eth%d".
1180 * for wildcarding.
1181 */
1182int dev_change_name(struct net_device *dev, const char *newname)
1183{
1184 unsigned char old_assign_type;
1185 char oldname[IFNAMSIZ];
1186 int err = 0;
1187 int ret;
1188 struct net *net;
1189
1190 ASSERT_RTNL();
1191 BUG_ON(!dev_net(dev));
1192
1193 net = dev_net(dev);
1194
1195 down_write(&devnet_rename_sem);
1196
1197 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1198 up_write(&devnet_rename_sem);
1199 return 0;
1200 }
1201
1202 memcpy(oldname, dev->name, IFNAMSIZ);
1203
1204 err = dev_get_valid_name(net, dev, newname);
1205 if (err < 0) {
1206 up_write(&devnet_rename_sem);
1207 return err;
1208 }
1209
1210 if (oldname[0] && !strchr(oldname, '%'))
1211 netdev_info(dev, "renamed from %s%s\n", oldname,
1212 dev->flags & IFF_UP ? " (while UP)" : "");
1213
1214 old_assign_type = dev->name_assign_type;
1215 dev->name_assign_type = NET_NAME_RENAMED;
1216
1217rollback:
1218 ret = device_rename(&dev->dev, dev->name);
1219 if (ret) {
1220 memcpy(dev->name, oldname, IFNAMSIZ);
1221 dev->name_assign_type = old_assign_type;
1222 up_write(&devnet_rename_sem);
1223 return ret;
1224 }
1225
1226 up_write(&devnet_rename_sem);
1227
1228 netdev_adjacent_rename_links(dev, oldname);
1229
1230 write_lock(&dev_base_lock);
1231 netdev_name_node_del(dev->name_node);
1232 write_unlock(&dev_base_lock);
1233
1234 synchronize_rcu();
1235
1236 write_lock(&dev_base_lock);
1237 netdev_name_node_add(net, dev->name_node);
1238 write_unlock(&dev_base_lock);
1239
1240 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1241 ret = notifier_to_errno(ret);
1242
1243 if (ret) {
1244 /* err >= 0 after dev_alloc_name() or stores the first errno */
1245 if (err >= 0) {
1246 err = ret;
1247 down_write(&devnet_rename_sem);
1248 memcpy(dev->name, oldname, IFNAMSIZ);
1249 memcpy(oldname, newname, IFNAMSIZ);
1250 dev->name_assign_type = old_assign_type;
1251 old_assign_type = NET_NAME_RENAMED;
1252 goto rollback;
1253 } else {
1254 netdev_err(dev, "name change rollback failed: %d\n",
1255 ret);
1256 }
1257 }
1258
1259 return err;
1260}
1261
1262/**
1263 * dev_set_alias - change ifalias of a device
1264 * @dev: device
1265 * @alias: name up to IFALIASZ
1266 * @len: limit of bytes to copy from info
1267 *
1268 * Set ifalias for a device,
1269 */
1270int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1271{
1272 struct dev_ifalias *new_alias = NULL;
1273
1274 if (len >= IFALIASZ)
1275 return -EINVAL;
1276
1277 if (len) {
1278 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1279 if (!new_alias)
1280 return -ENOMEM;
1281
1282 memcpy(new_alias->ifalias, alias, len);
1283 new_alias->ifalias[len] = 0;
1284 }
1285
1286 mutex_lock(&ifalias_mutex);
1287 new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1288 mutex_is_locked(&ifalias_mutex));
1289 mutex_unlock(&ifalias_mutex);
1290
1291 if (new_alias)
1292 kfree_rcu(new_alias, rcuhead);
1293
1294 return len;
1295}
1296EXPORT_SYMBOL(dev_set_alias);
1297
1298/**
1299 * dev_get_alias - get ifalias of a device
1300 * @dev: device
1301 * @name: buffer to store name of ifalias
1302 * @len: size of buffer
1303 *
1304 * get ifalias for a device. Caller must make sure dev cannot go
1305 * away, e.g. rcu read lock or own a reference count to device.
1306 */
1307int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1308{
1309 const struct dev_ifalias *alias;
1310 int ret = 0;
1311
1312 rcu_read_lock();
1313 alias = rcu_dereference(dev->ifalias);
1314 if (alias)
1315 ret = snprintf(name, len, "%s", alias->ifalias);
1316 rcu_read_unlock();
1317
1318 return ret;
1319}
1320
1321/**
1322 * netdev_features_change - device changes features
1323 * @dev: device to cause notification
1324 *
1325 * Called to indicate a device has changed features.
1326 */
1327void netdev_features_change(struct net_device *dev)
1328{
1329 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1330}
1331EXPORT_SYMBOL(netdev_features_change);
1332
1333/**
1334 * netdev_state_change - device changes state
1335 * @dev: device to cause notification
1336 *
1337 * Called to indicate a device has changed state. This function calls
1338 * the notifier chains for netdev_chain and sends a NEWLINK message
1339 * to the routing socket.
1340 */
1341void netdev_state_change(struct net_device *dev)
1342{
1343 if (dev->flags & IFF_UP) {
1344 struct netdev_notifier_change_info change_info = {
1345 .info.dev = dev,
1346 };
1347
1348 call_netdevice_notifiers_info(NETDEV_CHANGE,
1349 &change_info.info);
1350 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL);
1351 }
1352}
1353EXPORT_SYMBOL(netdev_state_change);
1354
1355/**
1356 * __netdev_notify_peers - notify network peers about existence of @dev,
1357 * to be called when rtnl lock is already held.
1358 * @dev: network device
1359 *
1360 * Generate traffic such that interested network peers are aware of
1361 * @dev, such as by generating a gratuitous ARP. This may be used when
1362 * a device wants to inform the rest of the network about some sort of
1363 * reconfiguration such as a failover event or virtual machine
1364 * migration.
1365 */
1366void __netdev_notify_peers(struct net_device *dev)
1367{
1368 ASSERT_RTNL();
1369 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1370 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1371}
1372EXPORT_SYMBOL(__netdev_notify_peers);
1373
1374/**
1375 * netdev_notify_peers - notify network peers about existence of @dev
1376 * @dev: network device
1377 *
1378 * Generate traffic such that interested network peers are aware of
1379 * @dev, such as by generating a gratuitous ARP. This may be used when
1380 * a device wants to inform the rest of the network about some sort of
1381 * reconfiguration such as a failover event or virtual machine
1382 * migration.
1383 */
1384void netdev_notify_peers(struct net_device *dev)
1385{
1386 rtnl_lock();
1387 __netdev_notify_peers(dev);
1388 rtnl_unlock();
1389}
1390EXPORT_SYMBOL(netdev_notify_peers);
1391
1392static int napi_threaded_poll(void *data);
1393
1394static int napi_kthread_create(struct napi_struct *n)
1395{
1396 int err = 0;
1397
1398 /* Create and wake up the kthread once to put it in
1399 * TASK_INTERRUPTIBLE mode to avoid the blocked task
1400 * warning and work with loadavg.
1401 */
1402 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1403 n->dev->name, n->napi_id);
1404 if (IS_ERR(n->thread)) {
1405 err = PTR_ERR(n->thread);
1406 pr_err("kthread_run failed with err %d\n", err);
1407 n->thread = NULL;
1408 }
1409
1410 return err;
1411}
1412
1413static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1414{
1415 const struct net_device_ops *ops = dev->netdev_ops;
1416 int ret;
1417
1418 ASSERT_RTNL();
1419 dev_addr_check(dev);
1420
1421 if (!netif_device_present(dev)) {
1422 /* may be detached because parent is runtime-suspended */
1423 if (dev->dev.parent)
1424 pm_runtime_resume(dev->dev.parent);
1425 if (!netif_device_present(dev))
1426 return -ENODEV;
1427 }
1428
1429 /* Block netpoll from trying to do any rx path servicing.
1430 * If we don't do this there is a chance ndo_poll_controller
1431 * or ndo_poll may be running while we open the device
1432 */
1433 netpoll_poll_disable(dev);
1434
1435 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1436 ret = notifier_to_errno(ret);
1437 if (ret)
1438 return ret;
1439
1440 set_bit(__LINK_STATE_START, &dev->state);
1441
1442 if (ops->ndo_validate_addr)
1443 ret = ops->ndo_validate_addr(dev);
1444
1445 if (!ret && ops->ndo_open)
1446 ret = ops->ndo_open(dev);
1447
1448 netpoll_poll_enable(dev);
1449
1450 if (ret)
1451 clear_bit(__LINK_STATE_START, &dev->state);
1452 else {
1453 dev->flags |= IFF_UP;
1454 dev_set_rx_mode(dev);
1455 dev_activate(dev);
1456 add_device_randomness(dev->dev_addr, dev->addr_len);
1457 }
1458
1459 return ret;
1460}
1461
1462/**
1463 * dev_open - prepare an interface for use.
1464 * @dev: device to open
1465 * @extack: netlink extended ack
1466 *
1467 * Takes a device from down to up state. The device's private open
1468 * function is invoked and then the multicast lists are loaded. Finally
1469 * the device is moved into the up state and a %NETDEV_UP message is
1470 * sent to the netdev notifier chain.
1471 *
1472 * Calling this function on an active interface is a nop. On a failure
1473 * a negative errno code is returned.
1474 */
1475int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1476{
1477 int ret;
1478
1479 if (dev->flags & IFF_UP)
1480 return 0;
1481
1482 ret = __dev_open(dev, extack);
1483 if (ret < 0)
1484 return ret;
1485
1486 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1487 call_netdevice_notifiers(NETDEV_UP, dev);
1488
1489 return ret;
1490}
1491EXPORT_SYMBOL(dev_open);
1492
1493static void __dev_close_many(struct list_head *head)
1494{
1495 struct net_device *dev;
1496
1497 ASSERT_RTNL();
1498 might_sleep();
1499
1500 list_for_each_entry(dev, head, close_list) {
1501 /* Temporarily disable netpoll until the interface is down */
1502 netpoll_poll_disable(dev);
1503
1504 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1505
1506 clear_bit(__LINK_STATE_START, &dev->state);
1507
1508 /* Synchronize to scheduled poll. We cannot touch poll list, it
1509 * can be even on different cpu. So just clear netif_running().
1510 *
1511 * dev->stop() will invoke napi_disable() on all of it's
1512 * napi_struct instances on this device.
1513 */
1514 smp_mb__after_atomic(); /* Commit netif_running(). */
1515 }
1516
1517 dev_deactivate_many(head);
1518
1519 list_for_each_entry(dev, head, close_list) {
1520 const struct net_device_ops *ops = dev->netdev_ops;
1521
1522 /*
1523 * Call the device specific close. This cannot fail.
1524 * Only if device is UP
1525 *
1526 * We allow it to be called even after a DETACH hot-plug
1527 * event.
1528 */
1529 if (ops->ndo_stop)
1530 ops->ndo_stop(dev);
1531
1532 dev->flags &= ~IFF_UP;
1533 netpoll_poll_enable(dev);
1534 }
1535}
1536
1537static void __dev_close(struct net_device *dev)
1538{
1539 LIST_HEAD(single);
1540
1541 list_add(&dev->close_list, &single);
1542 __dev_close_many(&single);
1543 list_del(&single);
1544}
1545
1546void dev_close_many(struct list_head *head, bool unlink)
1547{
1548 struct net_device *dev, *tmp;
1549
1550 /* Remove the devices that don't need to be closed */
1551 list_for_each_entry_safe(dev, tmp, head, close_list)
1552 if (!(dev->flags & IFF_UP))
1553 list_del_init(&dev->close_list);
1554
1555 __dev_close_many(head);
1556
1557 list_for_each_entry_safe(dev, tmp, head, close_list) {
1558 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1559 call_netdevice_notifiers(NETDEV_DOWN, dev);
1560 if (unlink)
1561 list_del_init(&dev->close_list);
1562 }
1563}
1564EXPORT_SYMBOL(dev_close_many);
1565
1566/**
1567 * dev_close - shutdown an interface.
1568 * @dev: device to shutdown
1569 *
1570 * This function moves an active device into down state. A
1571 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1572 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1573 * chain.
1574 */
1575void dev_close(struct net_device *dev)
1576{
1577 if (dev->flags & IFF_UP) {
1578 LIST_HEAD(single);
1579
1580 list_add(&dev->close_list, &single);
1581 dev_close_many(&single, true);
1582 list_del(&single);
1583 }
1584}
1585EXPORT_SYMBOL(dev_close);
1586
1587
1588/**
1589 * dev_disable_lro - disable Large Receive Offload on a device
1590 * @dev: device
1591 *
1592 * Disable Large Receive Offload (LRO) on a net device. Must be
1593 * called under RTNL. This is needed if received packets may be
1594 * forwarded to another interface.
1595 */
1596void dev_disable_lro(struct net_device *dev)
1597{
1598 struct net_device *lower_dev;
1599 struct list_head *iter;
1600
1601 dev->wanted_features &= ~NETIF_F_LRO;
1602 netdev_update_features(dev);
1603
1604 if (unlikely(dev->features & NETIF_F_LRO))
1605 netdev_WARN(dev, "failed to disable LRO!\n");
1606
1607 netdev_for_each_lower_dev(dev, lower_dev, iter)
1608 dev_disable_lro(lower_dev);
1609}
1610EXPORT_SYMBOL(dev_disable_lro);
1611
1612/**
1613 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1614 * @dev: device
1615 *
1616 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1617 * called under RTNL. This is needed if Generic XDP is installed on
1618 * the device.
1619 */
1620static void dev_disable_gro_hw(struct net_device *dev)
1621{
1622 dev->wanted_features &= ~NETIF_F_GRO_HW;
1623 netdev_update_features(dev);
1624
1625 if (unlikely(dev->features & NETIF_F_GRO_HW))
1626 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1627}
1628
1629const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1630{
1631#define N(val) \
1632 case NETDEV_##val: \
1633 return "NETDEV_" __stringify(val);
1634 switch (cmd) {
1635 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1636 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1637 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1638 N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN)
1639 N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA)
1640 N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE)
1641 N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1642 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1643 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1644 N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1645 N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1646 N(XDP_FEAT_CHANGE)
1647 }
1648#undef N
1649 return "UNKNOWN_NETDEV_EVENT";
1650}
1651EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1652
1653static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1654 struct net_device *dev)
1655{
1656 struct netdev_notifier_info info = {
1657 .dev = dev,
1658 };
1659
1660 return nb->notifier_call(nb, val, &info);
1661}
1662
1663static int call_netdevice_register_notifiers(struct notifier_block *nb,
1664 struct net_device *dev)
1665{
1666 int err;
1667
1668 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1669 err = notifier_to_errno(err);
1670 if (err)
1671 return err;
1672
1673 if (!(dev->flags & IFF_UP))
1674 return 0;
1675
1676 call_netdevice_notifier(nb, NETDEV_UP, dev);
1677 return 0;
1678}
1679
1680static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1681 struct net_device *dev)
1682{
1683 if (dev->flags & IFF_UP) {
1684 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1685 dev);
1686 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1687 }
1688 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1689}
1690
1691static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1692 struct net *net)
1693{
1694 struct net_device *dev;
1695 int err;
1696
1697 for_each_netdev(net, dev) {
1698 err = call_netdevice_register_notifiers(nb, dev);
1699 if (err)
1700 goto rollback;
1701 }
1702 return 0;
1703
1704rollback:
1705 for_each_netdev_continue_reverse(net, dev)
1706 call_netdevice_unregister_notifiers(nb, dev);
1707 return err;
1708}
1709
1710static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1711 struct net *net)
1712{
1713 struct net_device *dev;
1714
1715 for_each_netdev(net, dev)
1716 call_netdevice_unregister_notifiers(nb, dev);
1717}
1718
1719static int dev_boot_phase = 1;
1720
1721/**
1722 * register_netdevice_notifier - register a network notifier block
1723 * @nb: notifier
1724 *
1725 * Register a notifier to be called when network device events occur.
1726 * The notifier passed is linked into the kernel structures and must
1727 * not be reused until it has been unregistered. A negative errno code
1728 * is returned on a failure.
1729 *
1730 * When registered all registration and up events are replayed
1731 * to the new notifier to allow device to have a race free
1732 * view of the network device list.
1733 */
1734
1735int register_netdevice_notifier(struct notifier_block *nb)
1736{
1737 struct net *net;
1738 int err;
1739
1740 /* Close race with setup_net() and cleanup_net() */
1741 down_write(&pernet_ops_rwsem);
1742 rtnl_lock();
1743 err = raw_notifier_chain_register(&netdev_chain, nb);
1744 if (err)
1745 goto unlock;
1746 if (dev_boot_phase)
1747 goto unlock;
1748 for_each_net(net) {
1749 err = call_netdevice_register_net_notifiers(nb, net);
1750 if (err)
1751 goto rollback;
1752 }
1753
1754unlock:
1755 rtnl_unlock();
1756 up_write(&pernet_ops_rwsem);
1757 return err;
1758
1759rollback:
1760 for_each_net_continue_reverse(net)
1761 call_netdevice_unregister_net_notifiers(nb, net);
1762
1763 raw_notifier_chain_unregister(&netdev_chain, nb);
1764 goto unlock;
1765}
1766EXPORT_SYMBOL(register_netdevice_notifier);
1767
1768/**
1769 * unregister_netdevice_notifier - unregister a network notifier block
1770 * @nb: notifier
1771 *
1772 * Unregister a notifier previously registered by
1773 * register_netdevice_notifier(). The notifier is unlinked into the
1774 * kernel structures and may then be reused. A negative errno code
1775 * is returned on a failure.
1776 *
1777 * After unregistering unregister and down device events are synthesized
1778 * for all devices on the device list to the removed notifier to remove
1779 * the need for special case cleanup code.
1780 */
1781
1782int unregister_netdevice_notifier(struct notifier_block *nb)
1783{
1784 struct net *net;
1785 int err;
1786
1787 /* Close race with setup_net() and cleanup_net() */
1788 down_write(&pernet_ops_rwsem);
1789 rtnl_lock();
1790 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1791 if (err)
1792 goto unlock;
1793
1794 for_each_net(net)
1795 call_netdevice_unregister_net_notifiers(nb, net);
1796
1797unlock:
1798 rtnl_unlock();
1799 up_write(&pernet_ops_rwsem);
1800 return err;
1801}
1802EXPORT_SYMBOL(unregister_netdevice_notifier);
1803
1804static int __register_netdevice_notifier_net(struct net *net,
1805 struct notifier_block *nb,
1806 bool ignore_call_fail)
1807{
1808 int err;
1809
1810 err = raw_notifier_chain_register(&net->netdev_chain, nb);
1811 if (err)
1812 return err;
1813 if (dev_boot_phase)
1814 return 0;
1815
1816 err = call_netdevice_register_net_notifiers(nb, net);
1817 if (err && !ignore_call_fail)
1818 goto chain_unregister;
1819
1820 return 0;
1821
1822chain_unregister:
1823 raw_notifier_chain_unregister(&net->netdev_chain, nb);
1824 return err;
1825}
1826
1827static int __unregister_netdevice_notifier_net(struct net *net,
1828 struct notifier_block *nb)
1829{
1830 int err;
1831
1832 err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1833 if (err)
1834 return err;
1835
1836 call_netdevice_unregister_net_notifiers(nb, net);
1837 return 0;
1838}
1839
1840/**
1841 * register_netdevice_notifier_net - register a per-netns network notifier block
1842 * @net: network namespace
1843 * @nb: notifier
1844 *
1845 * Register a notifier to be called when network device events occur.
1846 * The notifier passed is linked into the kernel structures and must
1847 * not be reused until it has been unregistered. A negative errno code
1848 * is returned on a failure.
1849 *
1850 * When registered all registration and up events are replayed
1851 * to the new notifier to allow device to have a race free
1852 * view of the network device list.
1853 */
1854
1855int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1856{
1857 int err;
1858
1859 rtnl_lock();
1860 err = __register_netdevice_notifier_net(net, nb, false);
1861 rtnl_unlock();
1862 return err;
1863}
1864EXPORT_SYMBOL(register_netdevice_notifier_net);
1865
1866/**
1867 * unregister_netdevice_notifier_net - unregister a per-netns
1868 * network notifier block
1869 * @net: network namespace
1870 * @nb: notifier
1871 *
1872 * Unregister a notifier previously registered by
1873 * register_netdevice_notifier_net(). The notifier is unlinked from the
1874 * kernel structures and may then be reused. A negative errno code
1875 * is returned on a failure.
1876 *
1877 * After unregistering unregister and down device events are synthesized
1878 * for all devices on the device list to the removed notifier to remove
1879 * the need for special case cleanup code.
1880 */
1881
1882int unregister_netdevice_notifier_net(struct net *net,
1883 struct notifier_block *nb)
1884{
1885 int err;
1886
1887 rtnl_lock();
1888 err = __unregister_netdevice_notifier_net(net, nb);
1889 rtnl_unlock();
1890 return err;
1891}
1892EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1893
1894static void __move_netdevice_notifier_net(struct net *src_net,
1895 struct net *dst_net,
1896 struct notifier_block *nb)
1897{
1898 __unregister_netdevice_notifier_net(src_net, nb);
1899 __register_netdevice_notifier_net(dst_net, nb, true);
1900}
1901
1902int register_netdevice_notifier_dev_net(struct net_device *dev,
1903 struct notifier_block *nb,
1904 struct netdev_net_notifier *nn)
1905{
1906 int err;
1907
1908 rtnl_lock();
1909 err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
1910 if (!err) {
1911 nn->nb = nb;
1912 list_add(&nn->list, &dev->net_notifier_list);
1913 }
1914 rtnl_unlock();
1915 return err;
1916}
1917EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
1918
1919int unregister_netdevice_notifier_dev_net(struct net_device *dev,
1920 struct notifier_block *nb,
1921 struct netdev_net_notifier *nn)
1922{
1923 int err;
1924
1925 rtnl_lock();
1926 list_del(&nn->list);
1927 err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
1928 rtnl_unlock();
1929 return err;
1930}
1931EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
1932
1933static void move_netdevice_notifiers_dev_net(struct net_device *dev,
1934 struct net *net)
1935{
1936 struct netdev_net_notifier *nn;
1937
1938 list_for_each_entry(nn, &dev->net_notifier_list, list)
1939 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb);
1940}
1941
1942/**
1943 * call_netdevice_notifiers_info - call all network notifier blocks
1944 * @val: value passed unmodified to notifier function
1945 * @info: notifier information data
1946 *
1947 * Call all network notifier blocks. Parameters and return value
1948 * are as for raw_notifier_call_chain().
1949 */
1950
1951int call_netdevice_notifiers_info(unsigned long val,
1952 struct netdev_notifier_info *info)
1953{
1954 struct net *net = dev_net(info->dev);
1955 int ret;
1956
1957 ASSERT_RTNL();
1958
1959 /* Run per-netns notifier block chain first, then run the global one.
1960 * Hopefully, one day, the global one is going to be removed after
1961 * all notifier block registrators get converted to be per-netns.
1962 */
1963 ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
1964 if (ret & NOTIFY_STOP_MASK)
1965 return ret;
1966 return raw_notifier_call_chain(&netdev_chain, val, info);
1967}
1968
1969/**
1970 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks
1971 * for and rollback on error
1972 * @val_up: value passed unmodified to notifier function
1973 * @val_down: value passed unmodified to the notifier function when
1974 * recovering from an error on @val_up
1975 * @info: notifier information data
1976 *
1977 * Call all per-netns network notifier blocks, but not notifier blocks on
1978 * the global notifier chain. Parameters and return value are as for
1979 * raw_notifier_call_chain_robust().
1980 */
1981
1982static int
1983call_netdevice_notifiers_info_robust(unsigned long val_up,
1984 unsigned long val_down,
1985 struct netdev_notifier_info *info)
1986{
1987 struct net *net = dev_net(info->dev);
1988
1989 ASSERT_RTNL();
1990
1991 return raw_notifier_call_chain_robust(&net->netdev_chain,
1992 val_up, val_down, info);
1993}
1994
1995static int call_netdevice_notifiers_extack(unsigned long val,
1996 struct net_device *dev,
1997 struct netlink_ext_ack *extack)
1998{
1999 struct netdev_notifier_info info = {
2000 .dev = dev,
2001 .extack = extack,
2002 };
2003
2004 return call_netdevice_notifiers_info(val, &info);
2005}
2006
2007/**
2008 * call_netdevice_notifiers - call all network notifier blocks
2009 * @val: value passed unmodified to notifier function
2010 * @dev: net_device pointer passed unmodified to notifier function
2011 *
2012 * Call all network notifier blocks. Parameters and return value
2013 * are as for raw_notifier_call_chain().
2014 */
2015
2016int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
2017{
2018 return call_netdevice_notifiers_extack(val, dev, NULL);
2019}
2020EXPORT_SYMBOL(call_netdevice_notifiers);
2021
2022/**
2023 * call_netdevice_notifiers_mtu - call all network notifier blocks
2024 * @val: value passed unmodified to notifier function
2025 * @dev: net_device pointer passed unmodified to notifier function
2026 * @arg: additional u32 argument passed to the notifier function
2027 *
2028 * Call all network notifier blocks. Parameters and return value
2029 * are as for raw_notifier_call_chain().
2030 */
2031static int call_netdevice_notifiers_mtu(unsigned long val,
2032 struct net_device *dev, u32 arg)
2033{
2034 struct netdev_notifier_info_ext info = {
2035 .info.dev = dev,
2036 .ext.mtu = arg,
2037 };
2038
2039 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2040
2041 return call_netdevice_notifiers_info(val, &info.info);
2042}
2043
2044#ifdef CONFIG_NET_INGRESS
2045static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2046
2047void net_inc_ingress_queue(void)
2048{
2049 static_branch_inc(&ingress_needed_key);
2050}
2051EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2052
2053void net_dec_ingress_queue(void)
2054{
2055 static_branch_dec(&ingress_needed_key);
2056}
2057EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2058#endif
2059
2060#ifdef CONFIG_NET_EGRESS
2061static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2062
2063void net_inc_egress_queue(void)
2064{
2065 static_branch_inc(&egress_needed_key);
2066}
2067EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2068
2069void net_dec_egress_queue(void)
2070{
2071 static_branch_dec(&egress_needed_key);
2072}
2073EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2074#endif
2075
2076DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2077EXPORT_SYMBOL(netstamp_needed_key);
2078#ifdef CONFIG_JUMP_LABEL
2079static atomic_t netstamp_needed_deferred;
2080static atomic_t netstamp_wanted;
2081static void netstamp_clear(struct work_struct *work)
2082{
2083 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2084 int wanted;
2085
2086 wanted = atomic_add_return(deferred, &netstamp_wanted);
2087 if (wanted > 0)
2088 static_branch_enable(&netstamp_needed_key);
2089 else
2090 static_branch_disable(&netstamp_needed_key);
2091}
2092static DECLARE_WORK(netstamp_work, netstamp_clear);
2093#endif
2094
2095void net_enable_timestamp(void)
2096{
2097#ifdef CONFIG_JUMP_LABEL
2098 int wanted = atomic_read(&netstamp_wanted);
2099
2100 while (wanted > 0) {
2101 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1))
2102 return;
2103 }
2104 atomic_inc(&netstamp_needed_deferred);
2105 schedule_work(&netstamp_work);
2106#else
2107 static_branch_inc(&netstamp_needed_key);
2108#endif
2109}
2110EXPORT_SYMBOL(net_enable_timestamp);
2111
2112void net_disable_timestamp(void)
2113{
2114#ifdef CONFIG_JUMP_LABEL
2115 int wanted = atomic_read(&netstamp_wanted);
2116
2117 while (wanted > 1) {
2118 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1))
2119 return;
2120 }
2121 atomic_dec(&netstamp_needed_deferred);
2122 schedule_work(&netstamp_work);
2123#else
2124 static_branch_dec(&netstamp_needed_key);
2125#endif
2126}
2127EXPORT_SYMBOL(net_disable_timestamp);
2128
2129static inline void net_timestamp_set(struct sk_buff *skb)
2130{
2131 skb->tstamp = 0;
2132 skb->mono_delivery_time = 0;
2133 if (static_branch_unlikely(&netstamp_needed_key))
2134 skb->tstamp = ktime_get_real();
2135}
2136
2137#define net_timestamp_check(COND, SKB) \
2138 if (static_branch_unlikely(&netstamp_needed_key)) { \
2139 if ((COND) && !(SKB)->tstamp) \
2140 (SKB)->tstamp = ktime_get_real(); \
2141 } \
2142
2143bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2144{
2145 return __is_skb_forwardable(dev, skb, true);
2146}
2147EXPORT_SYMBOL_GPL(is_skb_forwardable);
2148
2149static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2150 bool check_mtu)
2151{
2152 int ret = ____dev_forward_skb(dev, skb, check_mtu);
2153
2154 if (likely(!ret)) {
2155 skb->protocol = eth_type_trans(skb, dev);
2156 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2157 }
2158
2159 return ret;
2160}
2161
2162int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2163{
2164 return __dev_forward_skb2(dev, skb, true);
2165}
2166EXPORT_SYMBOL_GPL(__dev_forward_skb);
2167
2168/**
2169 * dev_forward_skb - loopback an skb to another netif
2170 *
2171 * @dev: destination network device
2172 * @skb: buffer to forward
2173 *
2174 * return values:
2175 * NET_RX_SUCCESS (no congestion)
2176 * NET_RX_DROP (packet was dropped, but freed)
2177 *
2178 * dev_forward_skb can be used for injecting an skb from the
2179 * start_xmit function of one device into the receive queue
2180 * of another device.
2181 *
2182 * The receiving device may be in another namespace, so
2183 * we have to clear all information in the skb that could
2184 * impact namespace isolation.
2185 */
2186int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2187{
2188 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2189}
2190EXPORT_SYMBOL_GPL(dev_forward_skb);
2191
2192int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2193{
2194 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2195}
2196
2197static inline int deliver_skb(struct sk_buff *skb,
2198 struct packet_type *pt_prev,
2199 struct net_device *orig_dev)
2200{
2201 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2202 return -ENOMEM;
2203 refcount_inc(&skb->users);
2204 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2205}
2206
2207static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2208 struct packet_type **pt,
2209 struct net_device *orig_dev,
2210 __be16 type,
2211 struct list_head *ptype_list)
2212{
2213 struct packet_type *ptype, *pt_prev = *pt;
2214
2215 list_for_each_entry_rcu(ptype, ptype_list, list) {
2216 if (ptype->type != type)
2217 continue;
2218 if (pt_prev)
2219 deliver_skb(skb, pt_prev, orig_dev);
2220 pt_prev = ptype;
2221 }
2222 *pt = pt_prev;
2223}
2224
2225static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2226{
2227 if (!ptype->af_packet_priv || !skb->sk)
2228 return false;
2229
2230 if (ptype->id_match)
2231 return ptype->id_match(ptype, skb->sk);
2232 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2233 return true;
2234
2235 return false;
2236}
2237
2238/**
2239 * dev_nit_active - return true if any network interface taps are in use
2240 *
2241 * @dev: network device to check for the presence of taps
2242 */
2243bool dev_nit_active(struct net_device *dev)
2244{
2245 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2246}
2247EXPORT_SYMBOL_GPL(dev_nit_active);
2248
2249/*
2250 * Support routine. Sends outgoing frames to any network
2251 * taps currently in use.
2252 */
2253
2254void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2255{
2256 struct packet_type *ptype;
2257 struct sk_buff *skb2 = NULL;
2258 struct packet_type *pt_prev = NULL;
2259 struct list_head *ptype_list = &ptype_all;
2260
2261 rcu_read_lock();
2262again:
2263 list_for_each_entry_rcu(ptype, ptype_list, list) {
2264 if (ptype->ignore_outgoing)
2265 continue;
2266
2267 /* Never send packets back to the socket
2268 * they originated from - MvS (miquels@drinkel.ow.org)
2269 */
2270 if (skb_loop_sk(ptype, skb))
2271 continue;
2272
2273 if (pt_prev) {
2274 deliver_skb(skb2, pt_prev, skb->dev);
2275 pt_prev = ptype;
2276 continue;
2277 }
2278
2279 /* need to clone skb, done only once */
2280 skb2 = skb_clone(skb, GFP_ATOMIC);
2281 if (!skb2)
2282 goto out_unlock;
2283
2284 net_timestamp_set(skb2);
2285
2286 /* skb->nh should be correctly
2287 * set by sender, so that the second statement is
2288 * just protection against buggy protocols.
2289 */
2290 skb_reset_mac_header(skb2);
2291
2292 if (skb_network_header(skb2) < skb2->data ||
2293 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2294 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2295 ntohs(skb2->protocol),
2296 dev->name);
2297 skb_reset_network_header(skb2);
2298 }
2299
2300 skb2->transport_header = skb2->network_header;
2301 skb2->pkt_type = PACKET_OUTGOING;
2302 pt_prev = ptype;
2303 }
2304
2305 if (ptype_list == &ptype_all) {
2306 ptype_list = &dev->ptype_all;
2307 goto again;
2308 }
2309out_unlock:
2310 if (pt_prev) {
2311 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2312 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2313 else
2314 kfree_skb(skb2);
2315 }
2316 rcu_read_unlock();
2317}
2318EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2319
2320/**
2321 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2322 * @dev: Network device
2323 * @txq: number of queues available
2324 *
2325 * If real_num_tx_queues is changed the tc mappings may no longer be
2326 * valid. To resolve this verify the tc mapping remains valid and if
2327 * not NULL the mapping. With no priorities mapping to this
2328 * offset/count pair it will no longer be used. In the worst case TC0
2329 * is invalid nothing can be done so disable priority mappings. If is
2330 * expected that drivers will fix this mapping if they can before
2331 * calling netif_set_real_num_tx_queues.
2332 */
2333static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2334{
2335 int i;
2336 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2337
2338 /* If TC0 is invalidated disable TC mapping */
2339 if (tc->offset + tc->count > txq) {
2340 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2341 dev->num_tc = 0;
2342 return;
2343 }
2344
2345 /* Invalidated prio to tc mappings set to TC0 */
2346 for (i = 1; i < TC_BITMASK + 1; i++) {
2347 int q = netdev_get_prio_tc_map(dev, i);
2348
2349 tc = &dev->tc_to_txq[q];
2350 if (tc->offset + tc->count > txq) {
2351 netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2352 i, q);
2353 netdev_set_prio_tc_map(dev, i, 0);
2354 }
2355 }
2356}
2357
2358int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2359{
2360 if (dev->num_tc) {
2361 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2362 int i;
2363
2364 /* walk through the TCs and see if it falls into any of them */
2365 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2366 if ((txq - tc->offset) < tc->count)
2367 return i;
2368 }
2369
2370 /* didn't find it, just return -1 to indicate no match */
2371 return -1;
2372 }
2373
2374 return 0;
2375}
2376EXPORT_SYMBOL(netdev_txq_to_tc);
2377
2378#ifdef CONFIG_XPS
2379static struct static_key xps_needed __read_mostly;
2380static struct static_key xps_rxqs_needed __read_mostly;
2381static DEFINE_MUTEX(xps_map_mutex);
2382#define xmap_dereference(P) \
2383 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2384
2385static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2386 struct xps_dev_maps *old_maps, int tci, u16 index)
2387{
2388 struct xps_map *map = NULL;
2389 int pos;
2390
2391 map = xmap_dereference(dev_maps->attr_map[tci]);
2392 if (!map)
2393 return false;
2394
2395 for (pos = map->len; pos--;) {
2396 if (map->queues[pos] != index)
2397 continue;
2398
2399 if (map->len > 1) {
2400 map->queues[pos] = map->queues[--map->len];
2401 break;
2402 }
2403
2404 if (old_maps)
2405 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2406 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2407 kfree_rcu(map, rcu);
2408 return false;
2409 }
2410
2411 return true;
2412}
2413
2414static bool remove_xps_queue_cpu(struct net_device *dev,
2415 struct xps_dev_maps *dev_maps,
2416 int cpu, u16 offset, u16 count)
2417{
2418 int num_tc = dev_maps->num_tc;
2419 bool active = false;
2420 int tci;
2421
2422 for (tci = cpu * num_tc; num_tc--; tci++) {
2423 int i, j;
2424
2425 for (i = count, j = offset; i--; j++) {
2426 if (!remove_xps_queue(dev_maps, NULL, tci, j))
2427 break;
2428 }
2429
2430 active |= i < 0;
2431 }
2432
2433 return active;
2434}
2435
2436static void reset_xps_maps(struct net_device *dev,
2437 struct xps_dev_maps *dev_maps,
2438 enum xps_map_type type)
2439{
2440 static_key_slow_dec_cpuslocked(&xps_needed);
2441 if (type == XPS_RXQS)
2442 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2443
2444 RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2445
2446 kfree_rcu(dev_maps, rcu);
2447}
2448
2449static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2450 u16 offset, u16 count)
2451{
2452 struct xps_dev_maps *dev_maps;
2453 bool active = false;
2454 int i, j;
2455
2456 dev_maps = xmap_dereference(dev->xps_maps[type]);
2457 if (!dev_maps)
2458 return;
2459
2460 for (j = 0; j < dev_maps->nr_ids; j++)
2461 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2462 if (!active)
2463 reset_xps_maps(dev, dev_maps, type);
2464
2465 if (type == XPS_CPUS) {
2466 for (i = offset + (count - 1); count--; i--)
2467 netdev_queue_numa_node_write(
2468 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2469 }
2470}
2471
2472static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2473 u16 count)
2474{
2475 if (!static_key_false(&xps_needed))
2476 return;
2477
2478 cpus_read_lock();
2479 mutex_lock(&xps_map_mutex);
2480
2481 if (static_key_false(&xps_rxqs_needed))
2482 clean_xps_maps(dev, XPS_RXQS, offset, count);
2483
2484 clean_xps_maps(dev, XPS_CPUS, offset, count);
2485
2486 mutex_unlock(&xps_map_mutex);
2487 cpus_read_unlock();
2488}
2489
2490static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2491{
2492 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2493}
2494
2495static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2496 u16 index, bool is_rxqs_map)
2497{
2498 struct xps_map *new_map;
2499 int alloc_len = XPS_MIN_MAP_ALLOC;
2500 int i, pos;
2501
2502 for (pos = 0; map && pos < map->len; pos++) {
2503 if (map->queues[pos] != index)
2504 continue;
2505 return map;
2506 }
2507
2508 /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2509 if (map) {
2510 if (pos < map->alloc_len)
2511 return map;
2512
2513 alloc_len = map->alloc_len * 2;
2514 }
2515
2516 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2517 * map
2518 */
2519 if (is_rxqs_map)
2520 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2521 else
2522 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2523 cpu_to_node(attr_index));
2524 if (!new_map)
2525 return NULL;
2526
2527 for (i = 0; i < pos; i++)
2528 new_map->queues[i] = map->queues[i];
2529 new_map->alloc_len = alloc_len;
2530 new_map->len = pos;
2531
2532 return new_map;
2533}
2534
2535/* Copy xps maps at a given index */
2536static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2537 struct xps_dev_maps *new_dev_maps, int index,
2538 int tc, bool skip_tc)
2539{
2540 int i, tci = index * dev_maps->num_tc;
2541 struct xps_map *map;
2542
2543 /* copy maps belonging to foreign traffic classes */
2544 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2545 if (i == tc && skip_tc)
2546 continue;
2547
2548 /* fill in the new device map from the old device map */
2549 map = xmap_dereference(dev_maps->attr_map[tci]);
2550 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2551 }
2552}
2553
2554/* Must be called under cpus_read_lock */
2555int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2556 u16 index, enum xps_map_type type)
2557{
2558 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2559 const unsigned long *online_mask = NULL;
2560 bool active = false, copy = false;
2561 int i, j, tci, numa_node_id = -2;
2562 int maps_sz, num_tc = 1, tc = 0;
2563 struct xps_map *map, *new_map;
2564 unsigned int nr_ids;
2565
2566 WARN_ON_ONCE(index >= dev->num_tx_queues);
2567
2568 if (dev->num_tc) {
2569 /* Do not allow XPS on subordinate device directly */
2570 num_tc = dev->num_tc;
2571 if (num_tc < 0)
2572 return -EINVAL;
2573
2574 /* If queue belongs to subordinate dev use its map */
2575 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2576
2577 tc = netdev_txq_to_tc(dev, index);
2578 if (tc < 0)
2579 return -EINVAL;
2580 }
2581
2582 mutex_lock(&xps_map_mutex);
2583
2584 dev_maps = xmap_dereference(dev->xps_maps[type]);
2585 if (type == XPS_RXQS) {
2586 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2587 nr_ids = dev->num_rx_queues;
2588 } else {
2589 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2590 if (num_possible_cpus() > 1)
2591 online_mask = cpumask_bits(cpu_online_mask);
2592 nr_ids = nr_cpu_ids;
2593 }
2594
2595 if (maps_sz < L1_CACHE_BYTES)
2596 maps_sz = L1_CACHE_BYTES;
2597
2598 /* The old dev_maps could be larger or smaller than the one we're
2599 * setting up now, as dev->num_tc or nr_ids could have been updated in
2600 * between. We could try to be smart, but let's be safe instead and only
2601 * copy foreign traffic classes if the two map sizes match.
2602 */
2603 if (dev_maps &&
2604 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2605 copy = true;
2606
2607 /* allocate memory for queue storage */
2608 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2609 j < nr_ids;) {
2610 if (!new_dev_maps) {
2611 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2612 if (!new_dev_maps) {
2613 mutex_unlock(&xps_map_mutex);
2614 return -ENOMEM;
2615 }
2616
2617 new_dev_maps->nr_ids = nr_ids;
2618 new_dev_maps->num_tc = num_tc;
2619 }
2620
2621 tci = j * num_tc + tc;
2622 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2623
2624 map = expand_xps_map(map, j, index, type == XPS_RXQS);
2625 if (!map)
2626 goto error;
2627
2628 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2629 }
2630
2631 if (!new_dev_maps)
2632 goto out_no_new_maps;
2633
2634 if (!dev_maps) {
2635 /* Increment static keys at most once per type */
2636 static_key_slow_inc_cpuslocked(&xps_needed);
2637 if (type == XPS_RXQS)
2638 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2639 }
2640
2641 for (j = 0; j < nr_ids; j++) {
2642 bool skip_tc = false;
2643
2644 tci = j * num_tc + tc;
2645 if (netif_attr_test_mask(j, mask, nr_ids) &&
2646 netif_attr_test_online(j, online_mask, nr_ids)) {
2647 /* add tx-queue to CPU/rx-queue maps */
2648 int pos = 0;
2649
2650 skip_tc = true;
2651
2652 map = xmap_dereference(new_dev_maps->attr_map[tci]);
2653 while ((pos < map->len) && (map->queues[pos] != index))
2654 pos++;
2655
2656 if (pos == map->len)
2657 map->queues[map->len++] = index;
2658#ifdef CONFIG_NUMA
2659 if (type == XPS_CPUS) {
2660 if (numa_node_id == -2)
2661 numa_node_id = cpu_to_node(j);
2662 else if (numa_node_id != cpu_to_node(j))
2663 numa_node_id = -1;
2664 }
2665#endif
2666 }
2667
2668 if (copy)
2669 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2670 skip_tc);
2671 }
2672
2673 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2674
2675 /* Cleanup old maps */
2676 if (!dev_maps)
2677 goto out_no_old_maps;
2678
2679 for (j = 0; j < dev_maps->nr_ids; j++) {
2680 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2681 map = xmap_dereference(dev_maps->attr_map[tci]);
2682 if (!map)
2683 continue;
2684
2685 if (copy) {
2686 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2687 if (map == new_map)
2688 continue;
2689 }
2690
2691 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2692 kfree_rcu(map, rcu);
2693 }
2694 }
2695
2696 old_dev_maps = dev_maps;
2697
2698out_no_old_maps:
2699 dev_maps = new_dev_maps;
2700 active = true;
2701
2702out_no_new_maps:
2703 if (type == XPS_CPUS)
2704 /* update Tx queue numa node */
2705 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2706 (numa_node_id >= 0) ?
2707 numa_node_id : NUMA_NO_NODE);
2708
2709 if (!dev_maps)
2710 goto out_no_maps;
2711
2712 /* removes tx-queue from unused CPUs/rx-queues */
2713 for (j = 0; j < dev_maps->nr_ids; j++) {
2714 tci = j * dev_maps->num_tc;
2715
2716 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2717 if (i == tc &&
2718 netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2719 netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2720 continue;
2721
2722 active |= remove_xps_queue(dev_maps,
2723 copy ? old_dev_maps : NULL,
2724 tci, index);
2725 }
2726 }
2727
2728 if (old_dev_maps)
2729 kfree_rcu(old_dev_maps, rcu);
2730
2731 /* free map if not active */
2732 if (!active)
2733 reset_xps_maps(dev, dev_maps, type);
2734
2735out_no_maps:
2736 mutex_unlock(&xps_map_mutex);
2737
2738 return 0;
2739error:
2740 /* remove any maps that we added */
2741 for (j = 0; j < nr_ids; j++) {
2742 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2743 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2744 map = copy ?
2745 xmap_dereference(dev_maps->attr_map[tci]) :
2746 NULL;
2747 if (new_map && new_map != map)
2748 kfree(new_map);
2749 }
2750 }
2751
2752 mutex_unlock(&xps_map_mutex);
2753
2754 kfree(new_dev_maps);
2755 return -ENOMEM;
2756}
2757EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2758
2759int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2760 u16 index)
2761{
2762 int ret;
2763
2764 cpus_read_lock();
2765 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2766 cpus_read_unlock();
2767
2768 return ret;
2769}
2770EXPORT_SYMBOL(netif_set_xps_queue);
2771
2772#endif
2773static void netdev_unbind_all_sb_channels(struct net_device *dev)
2774{
2775 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2776
2777 /* Unbind any subordinate channels */
2778 while (txq-- != &dev->_tx[0]) {
2779 if (txq->sb_dev)
2780 netdev_unbind_sb_channel(dev, txq->sb_dev);
2781 }
2782}
2783
2784void netdev_reset_tc(struct net_device *dev)
2785{
2786#ifdef CONFIG_XPS
2787 netif_reset_xps_queues_gt(dev, 0);
2788#endif
2789 netdev_unbind_all_sb_channels(dev);
2790
2791 /* Reset TC configuration of device */
2792 dev->num_tc = 0;
2793 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2794 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2795}
2796EXPORT_SYMBOL(netdev_reset_tc);
2797
2798int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2799{
2800 if (tc >= dev->num_tc)
2801 return -EINVAL;
2802
2803#ifdef CONFIG_XPS
2804 netif_reset_xps_queues(dev, offset, count);
2805#endif
2806 dev->tc_to_txq[tc].count = count;
2807 dev->tc_to_txq[tc].offset = offset;
2808 return 0;
2809}
2810EXPORT_SYMBOL(netdev_set_tc_queue);
2811
2812int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2813{
2814 if (num_tc > TC_MAX_QUEUE)
2815 return -EINVAL;
2816
2817#ifdef CONFIG_XPS
2818 netif_reset_xps_queues_gt(dev, 0);
2819#endif
2820 netdev_unbind_all_sb_channels(dev);
2821
2822 dev->num_tc = num_tc;
2823 return 0;
2824}
2825EXPORT_SYMBOL(netdev_set_num_tc);
2826
2827void netdev_unbind_sb_channel(struct net_device *dev,
2828 struct net_device *sb_dev)
2829{
2830 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2831
2832#ifdef CONFIG_XPS
2833 netif_reset_xps_queues_gt(sb_dev, 0);
2834#endif
2835 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2836 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2837
2838 while (txq-- != &dev->_tx[0]) {
2839 if (txq->sb_dev == sb_dev)
2840 txq->sb_dev = NULL;
2841 }
2842}
2843EXPORT_SYMBOL(netdev_unbind_sb_channel);
2844
2845int netdev_bind_sb_channel_queue(struct net_device *dev,
2846 struct net_device *sb_dev,
2847 u8 tc, u16 count, u16 offset)
2848{
2849 /* Make certain the sb_dev and dev are already configured */
2850 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2851 return -EINVAL;
2852
2853 /* We cannot hand out queues we don't have */
2854 if ((offset + count) > dev->real_num_tx_queues)
2855 return -EINVAL;
2856
2857 /* Record the mapping */
2858 sb_dev->tc_to_txq[tc].count = count;
2859 sb_dev->tc_to_txq[tc].offset = offset;
2860
2861 /* Provide a way for Tx queue to find the tc_to_txq map or
2862 * XPS map for itself.
2863 */
2864 while (count--)
2865 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2866
2867 return 0;
2868}
2869EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2870
2871int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2872{
2873 /* Do not use a multiqueue device to represent a subordinate channel */
2874 if (netif_is_multiqueue(dev))
2875 return -ENODEV;
2876
2877 /* We allow channels 1 - 32767 to be used for subordinate channels.
2878 * Channel 0 is meant to be "native" mode and used only to represent
2879 * the main root device. We allow writing 0 to reset the device back
2880 * to normal mode after being used as a subordinate channel.
2881 */
2882 if (channel > S16_MAX)
2883 return -EINVAL;
2884
2885 dev->num_tc = -channel;
2886
2887 return 0;
2888}
2889EXPORT_SYMBOL(netdev_set_sb_channel);
2890
2891/*
2892 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2893 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2894 */
2895int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2896{
2897 bool disabling;
2898 int rc;
2899
2900 disabling = txq < dev->real_num_tx_queues;
2901
2902 if (txq < 1 || txq > dev->num_tx_queues)
2903 return -EINVAL;
2904
2905 if (dev->reg_state == NETREG_REGISTERED ||
2906 dev->reg_state == NETREG_UNREGISTERING) {
2907 ASSERT_RTNL();
2908
2909 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2910 txq);
2911 if (rc)
2912 return rc;
2913
2914 if (dev->num_tc)
2915 netif_setup_tc(dev, txq);
2916
2917 dev_qdisc_change_real_num_tx(dev, txq);
2918
2919 dev->real_num_tx_queues = txq;
2920
2921 if (disabling) {
2922 synchronize_net();
2923 qdisc_reset_all_tx_gt(dev, txq);
2924#ifdef CONFIG_XPS
2925 netif_reset_xps_queues_gt(dev, txq);
2926#endif
2927 }
2928 } else {
2929 dev->real_num_tx_queues = txq;
2930 }
2931
2932 return 0;
2933}
2934EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2935
2936#ifdef CONFIG_SYSFS
2937/**
2938 * netif_set_real_num_rx_queues - set actual number of RX queues used
2939 * @dev: Network device
2940 * @rxq: Actual number of RX queues
2941 *
2942 * This must be called either with the rtnl_lock held or before
2943 * registration of the net device. Returns 0 on success, or a
2944 * negative error code. If called before registration, it always
2945 * succeeds.
2946 */
2947int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2948{
2949 int rc;
2950
2951 if (rxq < 1 || rxq > dev->num_rx_queues)
2952 return -EINVAL;
2953
2954 if (dev->reg_state == NETREG_REGISTERED) {
2955 ASSERT_RTNL();
2956
2957 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2958 rxq);
2959 if (rc)
2960 return rc;
2961 }
2962
2963 dev->real_num_rx_queues = rxq;
2964 return 0;
2965}
2966EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2967#endif
2968
2969/**
2970 * netif_set_real_num_queues - set actual number of RX and TX queues used
2971 * @dev: Network device
2972 * @txq: Actual number of TX queues
2973 * @rxq: Actual number of RX queues
2974 *
2975 * Set the real number of both TX and RX queues.
2976 * Does nothing if the number of queues is already correct.
2977 */
2978int netif_set_real_num_queues(struct net_device *dev,
2979 unsigned int txq, unsigned int rxq)
2980{
2981 unsigned int old_rxq = dev->real_num_rx_queues;
2982 int err;
2983
2984 if (txq < 1 || txq > dev->num_tx_queues ||
2985 rxq < 1 || rxq > dev->num_rx_queues)
2986 return -EINVAL;
2987
2988 /* Start from increases, so the error path only does decreases -
2989 * decreases can't fail.
2990 */
2991 if (rxq > dev->real_num_rx_queues) {
2992 err = netif_set_real_num_rx_queues(dev, rxq);
2993 if (err)
2994 return err;
2995 }
2996 if (txq > dev->real_num_tx_queues) {
2997 err = netif_set_real_num_tx_queues(dev, txq);
2998 if (err)
2999 goto undo_rx;
3000 }
3001 if (rxq < dev->real_num_rx_queues)
3002 WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
3003 if (txq < dev->real_num_tx_queues)
3004 WARN_ON(netif_set_real_num_tx_queues(dev, txq));
3005
3006 return 0;
3007undo_rx:
3008 WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
3009 return err;
3010}
3011EXPORT_SYMBOL(netif_set_real_num_queues);
3012
3013/**
3014 * netif_set_tso_max_size() - set the max size of TSO frames supported
3015 * @dev: netdev to update
3016 * @size: max skb->len of a TSO frame
3017 *
3018 * Set the limit on the size of TSO super-frames the device can handle.
3019 * Unless explicitly set the stack will assume the value of
3020 * %GSO_LEGACY_MAX_SIZE.
3021 */
3022void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
3023{
3024 dev->tso_max_size = min(GSO_MAX_SIZE, size);
3025 if (size < READ_ONCE(dev->gso_max_size))
3026 netif_set_gso_max_size(dev, size);
3027 if (size < READ_ONCE(dev->gso_ipv4_max_size))
3028 netif_set_gso_ipv4_max_size(dev, size);
3029}
3030EXPORT_SYMBOL(netif_set_tso_max_size);
3031
3032/**
3033 * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3034 * @dev: netdev to update
3035 * @segs: max number of TCP segments
3036 *
3037 * Set the limit on the number of TCP segments the device can generate from
3038 * a single TSO super-frame.
3039 * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3040 */
3041void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3042{
3043 dev->tso_max_segs = segs;
3044 if (segs < READ_ONCE(dev->gso_max_segs))
3045 netif_set_gso_max_segs(dev, segs);
3046}
3047EXPORT_SYMBOL(netif_set_tso_max_segs);
3048
3049/**
3050 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3051 * @to: netdev to update
3052 * @from: netdev from which to copy the limits
3053 */
3054void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
3055{
3056 netif_set_tso_max_size(to, from->tso_max_size);
3057 netif_set_tso_max_segs(to, from->tso_max_segs);
3058}
3059EXPORT_SYMBOL(netif_inherit_tso_max);
3060
3061/**
3062 * netif_get_num_default_rss_queues - default number of RSS queues
3063 *
3064 * Default value is the number of physical cores if there are only 1 or 2, or
3065 * divided by 2 if there are more.
3066 */
3067int netif_get_num_default_rss_queues(void)
3068{
3069 cpumask_var_t cpus;
3070 int cpu, count = 0;
3071
3072 if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3073 return 1;
3074
3075 cpumask_copy(cpus, cpu_online_mask);
3076 for_each_cpu(cpu, cpus) {
3077 ++count;
3078 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3079 }
3080 free_cpumask_var(cpus);
3081
3082 return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3083}
3084EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3085
3086static void __netif_reschedule(struct Qdisc *q)
3087{
3088 struct softnet_data *sd;
3089 unsigned long flags;
3090
3091 local_irq_save(flags);
3092 sd = this_cpu_ptr(&softnet_data);
3093 q->next_sched = NULL;
3094 *sd->output_queue_tailp = q;
3095 sd->output_queue_tailp = &q->next_sched;
3096 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3097 local_irq_restore(flags);
3098}
3099
3100void __netif_schedule(struct Qdisc *q)
3101{
3102 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3103 __netif_reschedule(q);
3104}
3105EXPORT_SYMBOL(__netif_schedule);
3106
3107struct dev_kfree_skb_cb {
3108 enum skb_drop_reason reason;
3109};
3110
3111static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3112{
3113 return (struct dev_kfree_skb_cb *)skb->cb;
3114}
3115
3116void netif_schedule_queue(struct netdev_queue *txq)
3117{
3118 rcu_read_lock();
3119 if (!netif_xmit_stopped(txq)) {
3120 struct Qdisc *q = rcu_dereference(txq->qdisc);
3121
3122 __netif_schedule(q);
3123 }
3124 rcu_read_unlock();
3125}
3126EXPORT_SYMBOL(netif_schedule_queue);
3127
3128void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3129{
3130 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3131 struct Qdisc *q;
3132
3133 rcu_read_lock();
3134 q = rcu_dereference(dev_queue->qdisc);
3135 __netif_schedule(q);
3136 rcu_read_unlock();
3137 }
3138}
3139EXPORT_SYMBOL(netif_tx_wake_queue);
3140
3141void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3142{
3143 unsigned long flags;
3144
3145 if (unlikely(!skb))
3146 return;
3147
3148 if (likely(refcount_read(&skb->users) == 1)) {
3149 smp_rmb();
3150 refcount_set(&skb->users, 0);
3151 } else if (likely(!refcount_dec_and_test(&skb->users))) {
3152 return;
3153 }
3154 get_kfree_skb_cb(skb)->reason = reason;
3155 local_irq_save(flags);
3156 skb->next = __this_cpu_read(softnet_data.completion_queue);
3157 __this_cpu_write(softnet_data.completion_queue, skb);
3158 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3159 local_irq_restore(flags);
3160}
3161EXPORT_SYMBOL(dev_kfree_skb_irq_reason);
3162
3163void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3164{
3165 if (in_hardirq() || irqs_disabled())
3166 dev_kfree_skb_irq_reason(skb, reason);
3167 else
3168 kfree_skb_reason(skb, reason);
3169}
3170EXPORT_SYMBOL(dev_kfree_skb_any_reason);
3171
3172
3173/**
3174 * netif_device_detach - mark device as removed
3175 * @dev: network device
3176 *
3177 * Mark device as removed from system and therefore no longer available.
3178 */
3179void netif_device_detach(struct net_device *dev)
3180{
3181 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3182 netif_running(dev)) {
3183 netif_tx_stop_all_queues(dev);
3184 }
3185}
3186EXPORT_SYMBOL(netif_device_detach);
3187
3188/**
3189 * netif_device_attach - mark device as attached
3190 * @dev: network device
3191 *
3192 * Mark device as attached from system and restart if needed.
3193 */
3194void netif_device_attach(struct net_device *dev)
3195{
3196 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3197 netif_running(dev)) {
3198 netif_tx_wake_all_queues(dev);
3199 __netdev_watchdog_up(dev);
3200 }
3201}
3202EXPORT_SYMBOL(netif_device_attach);
3203
3204/*
3205 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3206 * to be used as a distribution range.
3207 */
3208static u16 skb_tx_hash(const struct net_device *dev,
3209 const struct net_device *sb_dev,
3210 struct sk_buff *skb)
3211{
3212 u32 hash;
3213 u16 qoffset = 0;
3214 u16 qcount = dev->real_num_tx_queues;
3215
3216 if (dev->num_tc) {
3217 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3218
3219 qoffset = sb_dev->tc_to_txq[tc].offset;
3220 qcount = sb_dev->tc_to_txq[tc].count;
3221 if (unlikely(!qcount)) {
3222 net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3223 sb_dev->name, qoffset, tc);
3224 qoffset = 0;
3225 qcount = dev->real_num_tx_queues;
3226 }
3227 }
3228
3229 if (skb_rx_queue_recorded(skb)) {
3230 DEBUG_NET_WARN_ON_ONCE(qcount == 0);
3231 hash = skb_get_rx_queue(skb);
3232 if (hash >= qoffset)
3233 hash -= qoffset;
3234 while (unlikely(hash >= qcount))
3235 hash -= qcount;
3236 return hash + qoffset;
3237 }
3238
3239 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3240}
3241
3242void skb_warn_bad_offload(const struct sk_buff *skb)
3243{
3244 static const netdev_features_t null_features;
3245 struct net_device *dev = skb->dev;
3246 const char *name = "";
3247
3248 if (!net_ratelimit())
3249 return;
3250
3251 if (dev) {
3252 if (dev->dev.parent)
3253 name = dev_driver_string(dev->dev.parent);
3254 else
3255 name = netdev_name(dev);
3256 }
3257 skb_dump(KERN_WARNING, skb, false);
3258 WARN(1, "%s: caps=(%pNF, %pNF)\n",
3259 name, dev ? &dev->features : &null_features,
3260 skb->sk ? &skb->sk->sk_route_caps : &null_features);
3261}
3262
3263/*
3264 * Invalidate hardware checksum when packet is to be mangled, and
3265 * complete checksum manually on outgoing path.
3266 */
3267int skb_checksum_help(struct sk_buff *skb)
3268{
3269 __wsum csum;
3270 int ret = 0, offset;
3271
3272 if (skb->ip_summed == CHECKSUM_COMPLETE)
3273 goto out_set_summed;
3274
3275 if (unlikely(skb_is_gso(skb))) {
3276 skb_warn_bad_offload(skb);
3277 return -EINVAL;
3278 }
3279
3280 /* Before computing a checksum, we should make sure no frag could
3281 * be modified by an external entity : checksum could be wrong.
3282 */
3283 if (skb_has_shared_frag(skb)) {
3284 ret = __skb_linearize(skb);
3285 if (ret)
3286 goto out;
3287 }
3288
3289 offset = skb_checksum_start_offset(skb);
3290 ret = -EINVAL;
3291 if (unlikely(offset >= skb_headlen(skb))) {
3292 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3293 WARN_ONCE(true, "offset (%d) >= skb_headlen() (%u)\n",
3294 offset, skb_headlen(skb));
3295 goto out;
3296 }
3297 csum = skb_checksum(skb, offset, skb->len - offset, 0);
3298
3299 offset += skb->csum_offset;
3300 if (unlikely(offset + sizeof(__sum16) > skb_headlen(skb))) {
3301 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3302 WARN_ONCE(true, "offset+2 (%zu) > skb_headlen() (%u)\n",
3303 offset + sizeof(__sum16), skb_headlen(skb));
3304 goto out;
3305 }
3306 ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3307 if (ret)
3308 goto out;
3309
3310 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3311out_set_summed:
3312 skb->ip_summed = CHECKSUM_NONE;
3313out:
3314 return ret;
3315}
3316EXPORT_SYMBOL(skb_checksum_help);
3317
3318int skb_crc32c_csum_help(struct sk_buff *skb)
3319{
3320 __le32 crc32c_csum;
3321 int ret = 0, offset, start;
3322
3323 if (skb->ip_summed != CHECKSUM_PARTIAL)
3324 goto out;
3325
3326 if (unlikely(skb_is_gso(skb)))
3327 goto out;
3328
3329 /* Before computing a checksum, we should make sure no frag could
3330 * be modified by an external entity : checksum could be wrong.
3331 */
3332 if (unlikely(skb_has_shared_frag(skb))) {
3333 ret = __skb_linearize(skb);
3334 if (ret)
3335 goto out;
3336 }
3337 start = skb_checksum_start_offset(skb);
3338 offset = start + offsetof(struct sctphdr, checksum);
3339 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3340 ret = -EINVAL;
3341 goto out;
3342 }
3343
3344 ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3345 if (ret)
3346 goto out;
3347
3348 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3349 skb->len - start, ~(__u32)0,
3350 crc32c_csum_stub));
3351 *(__le32 *)(skb->data + offset) = crc32c_csum;
3352 skb_reset_csum_not_inet(skb);
3353out:
3354 return ret;
3355}
3356
3357__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3358{
3359 __be16 type = skb->protocol;
3360
3361 /* Tunnel gso handlers can set protocol to ethernet. */
3362 if (type == htons(ETH_P_TEB)) {
3363 struct ethhdr *eth;
3364
3365 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3366 return 0;
3367
3368 eth = (struct ethhdr *)skb->data;
3369 type = eth->h_proto;
3370 }
3371
3372 return vlan_get_protocol_and_depth(skb, type, depth);
3373}
3374
3375
3376/* Take action when hardware reception checksum errors are detected. */
3377#ifdef CONFIG_BUG
3378static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3379{
3380 netdev_err(dev, "hw csum failure\n");
3381 skb_dump(KERN_ERR, skb, true);
3382 dump_stack();
3383}
3384
3385void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3386{
3387 DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3388}
3389EXPORT_SYMBOL(netdev_rx_csum_fault);
3390#endif
3391
3392/* XXX: check that highmem exists at all on the given machine. */
3393static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3394{
3395#ifdef CONFIG_HIGHMEM
3396 int i;
3397
3398 if (!(dev->features & NETIF_F_HIGHDMA)) {
3399 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3400 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3401
3402 if (PageHighMem(skb_frag_page(frag)))
3403 return 1;
3404 }
3405 }
3406#endif
3407 return 0;
3408}
3409
3410/* If MPLS offload request, verify we are testing hardware MPLS features
3411 * instead of standard features for the netdev.
3412 */
3413#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3414static netdev_features_t net_mpls_features(struct sk_buff *skb,
3415 netdev_features_t features,
3416 __be16 type)
3417{
3418 if (eth_p_mpls(type))
3419 features &= skb->dev->mpls_features;
3420
3421 return features;
3422}
3423#else
3424static netdev_features_t net_mpls_features(struct sk_buff *skb,
3425 netdev_features_t features,
3426 __be16 type)
3427{
3428 return features;
3429}
3430#endif
3431
3432static netdev_features_t harmonize_features(struct sk_buff *skb,
3433 netdev_features_t features)
3434{
3435 __be16 type;
3436
3437 type = skb_network_protocol(skb, NULL);
3438 features = net_mpls_features(skb, features, type);
3439
3440 if (skb->ip_summed != CHECKSUM_NONE &&
3441 !can_checksum_protocol(features, type)) {
3442 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3443 }
3444 if (illegal_highdma(skb->dev, skb))
3445 features &= ~NETIF_F_SG;
3446
3447 return features;
3448}
3449
3450netdev_features_t passthru_features_check(struct sk_buff *skb,
3451 struct net_device *dev,
3452 netdev_features_t features)
3453{
3454 return features;
3455}
3456EXPORT_SYMBOL(passthru_features_check);
3457
3458static netdev_features_t dflt_features_check(struct sk_buff *skb,
3459 struct net_device *dev,
3460 netdev_features_t features)
3461{
3462 return vlan_features_check(skb, features);
3463}
3464
3465static netdev_features_t gso_features_check(const struct sk_buff *skb,
3466 struct net_device *dev,
3467 netdev_features_t features)
3468{
3469 u16 gso_segs = skb_shinfo(skb)->gso_segs;
3470
3471 if (gso_segs > READ_ONCE(dev->gso_max_segs))
3472 return features & ~NETIF_F_GSO_MASK;
3473
3474 if (unlikely(skb->len >= READ_ONCE(dev->gso_max_size)))
3475 return features & ~NETIF_F_GSO_MASK;
3476
3477 if (!skb_shinfo(skb)->gso_type) {
3478 skb_warn_bad_offload(skb);
3479 return features & ~NETIF_F_GSO_MASK;
3480 }
3481
3482 /* Support for GSO partial features requires software
3483 * intervention before we can actually process the packets
3484 * so we need to strip support for any partial features now
3485 * and we can pull them back in after we have partially
3486 * segmented the frame.
3487 */
3488 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3489 features &= ~dev->gso_partial_features;
3490
3491 /* Make sure to clear the IPv4 ID mangling feature if the
3492 * IPv4 header has the potential to be fragmented.
3493 */
3494 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3495 struct iphdr *iph = skb->encapsulation ?
3496 inner_ip_hdr(skb) : ip_hdr(skb);
3497
3498 if (!(iph->frag_off & htons(IP_DF)))
3499 features &= ~NETIF_F_TSO_MANGLEID;
3500 }
3501
3502 return features;
3503}
3504
3505netdev_features_t netif_skb_features(struct sk_buff *skb)
3506{
3507 struct net_device *dev = skb->dev;
3508 netdev_features_t features = dev->features;
3509
3510 if (skb_is_gso(skb))
3511 features = gso_features_check(skb, dev, features);
3512
3513 /* If encapsulation offload request, verify we are testing
3514 * hardware encapsulation features instead of standard
3515 * features for the netdev
3516 */
3517 if (skb->encapsulation)
3518 features &= dev->hw_enc_features;
3519
3520 if (skb_vlan_tagged(skb))
3521 features = netdev_intersect_features(features,
3522 dev->vlan_features |
3523 NETIF_F_HW_VLAN_CTAG_TX |
3524 NETIF_F_HW_VLAN_STAG_TX);
3525
3526 if (dev->netdev_ops->ndo_features_check)
3527 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3528 features);
3529 else
3530 features &= dflt_features_check(skb, dev, features);
3531
3532 return harmonize_features(skb, features);
3533}
3534EXPORT_SYMBOL(netif_skb_features);
3535
3536static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3537 struct netdev_queue *txq, bool more)
3538{
3539 unsigned int len;
3540 int rc;
3541
3542 if (dev_nit_active(dev))
3543 dev_queue_xmit_nit(skb, dev);
3544
3545 len = skb->len;
3546 trace_net_dev_start_xmit(skb, dev);
3547 rc = netdev_start_xmit(skb, dev, txq, more);
3548 trace_net_dev_xmit(skb, rc, dev, len);
3549
3550 return rc;
3551}
3552
3553struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3554 struct netdev_queue *txq, int *ret)
3555{
3556 struct sk_buff *skb = first;
3557 int rc = NETDEV_TX_OK;
3558
3559 while (skb) {
3560 struct sk_buff *next = skb->next;
3561
3562 skb_mark_not_on_list(skb);
3563 rc = xmit_one(skb, dev, txq, next != NULL);
3564 if (unlikely(!dev_xmit_complete(rc))) {
3565 skb->next = next;
3566 goto out;
3567 }
3568
3569 skb = next;
3570 if (netif_tx_queue_stopped(txq) && skb) {
3571 rc = NETDEV_TX_BUSY;
3572 break;
3573 }
3574 }
3575
3576out:
3577 *ret = rc;
3578 return skb;
3579}
3580
3581static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3582 netdev_features_t features)
3583{
3584 if (skb_vlan_tag_present(skb) &&
3585 !vlan_hw_offload_capable(features, skb->vlan_proto))
3586 skb = __vlan_hwaccel_push_inside(skb);
3587 return skb;
3588}
3589
3590int skb_csum_hwoffload_help(struct sk_buff *skb,
3591 const netdev_features_t features)
3592{
3593 if (unlikely(skb_csum_is_sctp(skb)))
3594 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3595 skb_crc32c_csum_help(skb);
3596
3597 if (features & NETIF_F_HW_CSUM)
3598 return 0;
3599
3600 if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3601 switch (skb->csum_offset) {
3602 case offsetof(struct tcphdr, check):
3603 case offsetof(struct udphdr, check):
3604 return 0;
3605 }
3606 }
3607
3608 return skb_checksum_help(skb);
3609}
3610EXPORT_SYMBOL(skb_csum_hwoffload_help);
3611
3612static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3613{
3614 netdev_features_t features;
3615
3616 features = netif_skb_features(skb);
3617 skb = validate_xmit_vlan(skb, features);
3618 if (unlikely(!skb))
3619 goto out_null;
3620
3621 skb = sk_validate_xmit_skb(skb, dev);
3622 if (unlikely(!skb))
3623 goto out_null;
3624
3625 if (netif_needs_gso(skb, features)) {
3626 struct sk_buff *segs;
3627
3628 segs = skb_gso_segment(skb, features);
3629 if (IS_ERR(segs)) {
3630 goto out_kfree_skb;
3631 } else if (segs) {
3632 consume_skb(skb);
3633 skb = segs;
3634 }
3635 } else {
3636 if (skb_needs_linearize(skb, features) &&
3637 __skb_linearize(skb))
3638 goto out_kfree_skb;
3639
3640 /* If packet is not checksummed and device does not
3641 * support checksumming for this protocol, complete
3642 * checksumming here.
3643 */
3644 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3645 if (skb->encapsulation)
3646 skb_set_inner_transport_header(skb,
3647 skb_checksum_start_offset(skb));
3648 else
3649 skb_set_transport_header(skb,
3650 skb_checksum_start_offset(skb));
3651 if (skb_csum_hwoffload_help(skb, features))
3652 goto out_kfree_skb;
3653 }
3654 }
3655
3656 skb = validate_xmit_xfrm(skb, features, again);
3657
3658 return skb;
3659
3660out_kfree_skb:
3661 kfree_skb(skb);
3662out_null:
3663 dev_core_stats_tx_dropped_inc(dev);
3664 return NULL;
3665}
3666
3667struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3668{
3669 struct sk_buff *next, *head = NULL, *tail;
3670
3671 for (; skb != NULL; skb = next) {
3672 next = skb->next;
3673 skb_mark_not_on_list(skb);
3674
3675 /* in case skb wont be segmented, point to itself */
3676 skb->prev = skb;
3677
3678 skb = validate_xmit_skb(skb, dev, again);
3679 if (!skb)
3680 continue;
3681
3682 if (!head)
3683 head = skb;
3684 else
3685 tail->next = skb;
3686 /* If skb was segmented, skb->prev points to
3687 * the last segment. If not, it still contains skb.
3688 */
3689 tail = skb->prev;
3690 }
3691 return head;
3692}
3693EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3694
3695static void qdisc_pkt_len_init(struct sk_buff *skb)
3696{
3697 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3698
3699 qdisc_skb_cb(skb)->pkt_len = skb->len;
3700
3701 /* To get more precise estimation of bytes sent on wire,
3702 * we add to pkt_len the headers size of all segments
3703 */
3704 if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3705 u16 gso_segs = shinfo->gso_segs;
3706 unsigned int hdr_len;
3707
3708 /* mac layer + network layer */
3709 hdr_len = skb_transport_offset(skb);
3710
3711 /* + transport layer */
3712 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3713 const struct tcphdr *th;
3714 struct tcphdr _tcphdr;
3715
3716 th = skb_header_pointer(skb, hdr_len,
3717 sizeof(_tcphdr), &_tcphdr);
3718 if (likely(th))
3719 hdr_len += __tcp_hdrlen(th);
3720 } else {
3721 struct udphdr _udphdr;
3722
3723 if (skb_header_pointer(skb, hdr_len,
3724 sizeof(_udphdr), &_udphdr))
3725 hdr_len += sizeof(struct udphdr);
3726 }
3727
3728 if (shinfo->gso_type & SKB_GSO_DODGY)
3729 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3730 shinfo->gso_size);
3731
3732 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3733 }
3734}
3735
3736static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3737 struct sk_buff **to_free,
3738 struct netdev_queue *txq)
3739{
3740 int rc;
3741
3742 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3743 if (rc == NET_XMIT_SUCCESS)
3744 trace_qdisc_enqueue(q, txq, skb);
3745 return rc;
3746}
3747
3748static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3749 struct net_device *dev,
3750 struct netdev_queue *txq)
3751{
3752 spinlock_t *root_lock = qdisc_lock(q);
3753 struct sk_buff *to_free = NULL;
3754 bool contended;
3755 int rc;
3756
3757 qdisc_calculate_pkt_len(skb, q);
3758
3759 tcf_set_drop_reason(skb, SKB_DROP_REASON_QDISC_DROP);
3760
3761 if (q->flags & TCQ_F_NOLOCK) {
3762 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3763 qdisc_run_begin(q)) {
3764 /* Retest nolock_qdisc_is_empty() within the protection
3765 * of q->seqlock to protect from racing with requeuing.
3766 */
3767 if (unlikely(!nolock_qdisc_is_empty(q))) {
3768 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3769 __qdisc_run(q);
3770 qdisc_run_end(q);
3771
3772 goto no_lock_out;
3773 }
3774
3775 qdisc_bstats_cpu_update(q, skb);
3776 if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3777 !nolock_qdisc_is_empty(q))
3778 __qdisc_run(q);
3779
3780 qdisc_run_end(q);
3781 return NET_XMIT_SUCCESS;
3782 }
3783
3784 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3785 qdisc_run(q);
3786
3787no_lock_out:
3788 if (unlikely(to_free))
3789 kfree_skb_list_reason(to_free,
3790 tcf_get_drop_reason(to_free));
3791 return rc;
3792 }
3793
3794 /*
3795 * Heuristic to force contended enqueues to serialize on a
3796 * separate lock before trying to get qdisc main lock.
3797 * This permits qdisc->running owner to get the lock more
3798 * often and dequeue packets faster.
3799 * On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
3800 * and then other tasks will only enqueue packets. The packets will be
3801 * sent after the qdisc owner is scheduled again. To prevent this
3802 * scenario the task always serialize on the lock.
3803 */
3804 contended = qdisc_is_running(q) || IS_ENABLED(CONFIG_PREEMPT_RT);
3805 if (unlikely(contended))
3806 spin_lock(&q->busylock);
3807
3808 spin_lock(root_lock);
3809 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3810 __qdisc_drop(skb, &to_free);
3811 rc = NET_XMIT_DROP;
3812 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3813 qdisc_run_begin(q)) {
3814 /*
3815 * This is a work-conserving queue; there are no old skbs
3816 * waiting to be sent out; and the qdisc is not running -
3817 * xmit the skb directly.
3818 */
3819
3820 qdisc_bstats_update(q, skb);
3821
3822 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3823 if (unlikely(contended)) {
3824 spin_unlock(&q->busylock);
3825 contended = false;
3826 }
3827 __qdisc_run(q);
3828 }
3829
3830 qdisc_run_end(q);
3831 rc = NET_XMIT_SUCCESS;
3832 } else {
3833 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3834 if (qdisc_run_begin(q)) {
3835 if (unlikely(contended)) {
3836 spin_unlock(&q->busylock);
3837 contended = false;
3838 }
3839 __qdisc_run(q);
3840 qdisc_run_end(q);
3841 }
3842 }
3843 spin_unlock(root_lock);
3844 if (unlikely(to_free))
3845 kfree_skb_list_reason(to_free,
3846 tcf_get_drop_reason(to_free));
3847 if (unlikely(contended))
3848 spin_unlock(&q->busylock);
3849 return rc;
3850}
3851
3852#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3853static void skb_update_prio(struct sk_buff *skb)
3854{
3855 const struct netprio_map *map;
3856 const struct sock *sk;
3857 unsigned int prioidx;
3858
3859 if (skb->priority)
3860 return;
3861 map = rcu_dereference_bh(skb->dev->priomap);
3862 if (!map)
3863 return;
3864 sk = skb_to_full_sk(skb);
3865 if (!sk)
3866 return;
3867
3868 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3869
3870 if (prioidx < map->priomap_len)
3871 skb->priority = map->priomap[prioidx];
3872}
3873#else
3874#define skb_update_prio(skb)
3875#endif
3876
3877/**
3878 * dev_loopback_xmit - loop back @skb
3879 * @net: network namespace this loopback is happening in
3880 * @sk: sk needed to be a netfilter okfn
3881 * @skb: buffer to transmit
3882 */
3883int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3884{
3885 skb_reset_mac_header(skb);
3886 __skb_pull(skb, skb_network_offset(skb));
3887 skb->pkt_type = PACKET_LOOPBACK;
3888 if (skb->ip_summed == CHECKSUM_NONE)
3889 skb->ip_summed = CHECKSUM_UNNECESSARY;
3890 DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
3891 skb_dst_force(skb);
3892 netif_rx(skb);
3893 return 0;
3894}
3895EXPORT_SYMBOL(dev_loopback_xmit);
3896
3897#ifdef CONFIG_NET_EGRESS
3898static struct netdev_queue *
3899netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
3900{
3901 int qm = skb_get_queue_mapping(skb);
3902
3903 return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
3904}
3905
3906static bool netdev_xmit_txqueue_skipped(void)
3907{
3908 return __this_cpu_read(softnet_data.xmit.skip_txqueue);
3909}
3910
3911void netdev_xmit_skip_txqueue(bool skip)
3912{
3913 __this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
3914}
3915EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
3916#endif /* CONFIG_NET_EGRESS */
3917
3918#ifdef CONFIG_NET_XGRESS
3919static int tc_run(struct tcx_entry *entry, struct sk_buff *skb,
3920 enum skb_drop_reason *drop_reason)
3921{
3922 int ret = TC_ACT_UNSPEC;
3923#ifdef CONFIG_NET_CLS_ACT
3924 struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq);
3925 struct tcf_result res;
3926
3927 if (!miniq)
3928 return ret;
3929
3930 tc_skb_cb(skb)->mru = 0;
3931 tc_skb_cb(skb)->post_ct = false;
3932 tcf_set_drop_reason(skb, *drop_reason);
3933
3934 mini_qdisc_bstats_cpu_update(miniq, skb);
3935 ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false);
3936 /* Only tcf related quirks below. */
3937 switch (ret) {
3938 case TC_ACT_SHOT:
3939 *drop_reason = tcf_get_drop_reason(skb);
3940 mini_qdisc_qstats_cpu_drop(miniq);
3941 break;
3942 case TC_ACT_OK:
3943 case TC_ACT_RECLASSIFY:
3944 skb->tc_index = TC_H_MIN(res.classid);
3945 break;
3946 }
3947#endif /* CONFIG_NET_CLS_ACT */
3948 return ret;
3949}
3950
3951static DEFINE_STATIC_KEY_FALSE(tcx_needed_key);
3952
3953void tcx_inc(void)
3954{
3955 static_branch_inc(&tcx_needed_key);
3956}
3957
3958void tcx_dec(void)
3959{
3960 static_branch_dec(&tcx_needed_key);
3961}
3962
3963static __always_inline enum tcx_action_base
3964tcx_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
3965 const bool needs_mac)
3966{
3967 const struct bpf_mprog_fp *fp;
3968 const struct bpf_prog *prog;
3969 int ret = TCX_NEXT;
3970
3971 if (needs_mac)
3972 __skb_push(skb, skb->mac_len);
3973 bpf_mprog_foreach_prog(entry, fp, prog) {
3974 bpf_compute_data_pointers(skb);
3975 ret = bpf_prog_run(prog, skb);
3976 if (ret != TCX_NEXT)
3977 break;
3978 }
3979 if (needs_mac)
3980 __skb_pull(skb, skb->mac_len);
3981 return tcx_action_code(skb, ret);
3982}
3983
3984static __always_inline struct sk_buff *
3985sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3986 struct net_device *orig_dev, bool *another)
3987{
3988 struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress);
3989 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_INGRESS;
3990 int sch_ret;
3991
3992 if (!entry)
3993 return skb;
3994 if (*pt_prev) {
3995 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3996 *pt_prev = NULL;
3997 }
3998
3999 qdisc_skb_cb(skb)->pkt_len = skb->len;
4000 tcx_set_ingress(skb, true);
4001
4002 if (static_branch_unlikely(&tcx_needed_key)) {
4003 sch_ret = tcx_run(entry, skb, true);
4004 if (sch_ret != TC_ACT_UNSPEC)
4005 goto ingress_verdict;
4006 }
4007 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4008ingress_verdict:
4009 switch (sch_ret) {
4010 case TC_ACT_REDIRECT:
4011 /* skb_mac_header check was done by BPF, so we can safely
4012 * push the L2 header back before redirecting to another
4013 * netdev.
4014 */
4015 __skb_push(skb, skb->mac_len);
4016 if (skb_do_redirect(skb) == -EAGAIN) {
4017 __skb_pull(skb, skb->mac_len);
4018 *another = true;
4019 break;
4020 }
4021 *ret = NET_RX_SUCCESS;
4022 return NULL;
4023 case TC_ACT_SHOT:
4024 kfree_skb_reason(skb, drop_reason);
4025 *ret = NET_RX_DROP;
4026 return NULL;
4027 /* used by tc_run */
4028 case TC_ACT_STOLEN:
4029 case TC_ACT_QUEUED:
4030 case TC_ACT_TRAP:
4031 consume_skb(skb);
4032 fallthrough;
4033 case TC_ACT_CONSUMED:
4034 *ret = NET_RX_SUCCESS;
4035 return NULL;
4036 }
4037
4038 return skb;
4039}
4040
4041static __always_inline struct sk_buff *
4042sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4043{
4044 struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress);
4045 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_EGRESS;
4046 int sch_ret;
4047
4048 if (!entry)
4049 return skb;
4050
4051 /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was
4052 * already set by the caller.
4053 */
4054 if (static_branch_unlikely(&tcx_needed_key)) {
4055 sch_ret = tcx_run(entry, skb, false);
4056 if (sch_ret != TC_ACT_UNSPEC)
4057 goto egress_verdict;
4058 }
4059 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4060egress_verdict:
4061 switch (sch_ret) {
4062 case TC_ACT_REDIRECT:
4063 /* No need to push/pop skb's mac_header here on egress! */
4064 skb_do_redirect(skb);
4065 *ret = NET_XMIT_SUCCESS;
4066 return NULL;
4067 case TC_ACT_SHOT:
4068 kfree_skb_reason(skb, drop_reason);
4069 *ret = NET_XMIT_DROP;
4070 return NULL;
4071 /* used by tc_run */
4072 case TC_ACT_STOLEN:
4073 case TC_ACT_QUEUED:
4074 case TC_ACT_TRAP:
4075 consume_skb(skb);
4076 fallthrough;
4077 case TC_ACT_CONSUMED:
4078 *ret = NET_XMIT_SUCCESS;
4079 return NULL;
4080 }
4081
4082 return skb;
4083}
4084#else
4085static __always_inline struct sk_buff *
4086sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4087 struct net_device *orig_dev, bool *another)
4088{
4089 return skb;
4090}
4091
4092static __always_inline struct sk_buff *
4093sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4094{
4095 return skb;
4096}
4097#endif /* CONFIG_NET_XGRESS */
4098
4099#ifdef CONFIG_XPS
4100static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4101 struct xps_dev_maps *dev_maps, unsigned int tci)
4102{
4103 int tc = netdev_get_prio_tc_map(dev, skb->priority);
4104 struct xps_map *map;
4105 int queue_index = -1;
4106
4107 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
4108 return queue_index;
4109
4110 tci *= dev_maps->num_tc;
4111 tci += tc;
4112
4113 map = rcu_dereference(dev_maps->attr_map[tci]);
4114 if (map) {
4115 if (map->len == 1)
4116 queue_index = map->queues[0];
4117 else
4118 queue_index = map->queues[reciprocal_scale(
4119 skb_get_hash(skb), map->len)];
4120 if (unlikely(queue_index >= dev->real_num_tx_queues))
4121 queue_index = -1;
4122 }
4123 return queue_index;
4124}
4125#endif
4126
4127static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4128 struct sk_buff *skb)
4129{
4130#ifdef CONFIG_XPS
4131 struct xps_dev_maps *dev_maps;
4132 struct sock *sk = skb->sk;
4133 int queue_index = -1;
4134
4135 if (!static_key_false(&xps_needed))
4136 return -1;
4137
4138 rcu_read_lock();
4139 if (!static_key_false(&xps_rxqs_needed))
4140 goto get_cpus_map;
4141
4142 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4143 if (dev_maps) {
4144 int tci = sk_rx_queue_get(sk);
4145
4146 if (tci >= 0)
4147 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4148 tci);
4149 }
4150
4151get_cpus_map:
4152 if (queue_index < 0) {
4153 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4154 if (dev_maps) {
4155 unsigned int tci = skb->sender_cpu - 1;
4156
4157 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4158 tci);
4159 }
4160 }
4161 rcu_read_unlock();
4162
4163 return queue_index;
4164#else
4165 return -1;
4166#endif
4167}
4168
4169u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4170 struct net_device *sb_dev)
4171{
4172 return 0;
4173}
4174EXPORT_SYMBOL(dev_pick_tx_zero);
4175
4176u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
4177 struct net_device *sb_dev)
4178{
4179 return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4180}
4181EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4182
4183u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4184 struct net_device *sb_dev)
4185{
4186 struct sock *sk = skb->sk;
4187 int queue_index = sk_tx_queue_get(sk);
4188
4189 sb_dev = sb_dev ? : dev;
4190
4191 if (queue_index < 0 || skb->ooo_okay ||
4192 queue_index >= dev->real_num_tx_queues) {
4193 int new_index = get_xps_queue(dev, sb_dev, skb);
4194
4195 if (new_index < 0)
4196 new_index = skb_tx_hash(dev, sb_dev, skb);
4197
4198 if (queue_index != new_index && sk &&
4199 sk_fullsock(sk) &&
4200 rcu_access_pointer(sk->sk_dst_cache))
4201 sk_tx_queue_set(sk, new_index);
4202
4203 queue_index = new_index;
4204 }
4205
4206 return queue_index;
4207}
4208EXPORT_SYMBOL(netdev_pick_tx);
4209
4210struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4211 struct sk_buff *skb,
4212 struct net_device *sb_dev)
4213{
4214 int queue_index = 0;
4215
4216#ifdef CONFIG_XPS
4217 u32 sender_cpu = skb->sender_cpu - 1;
4218
4219 if (sender_cpu >= (u32)NR_CPUS)
4220 skb->sender_cpu = raw_smp_processor_id() + 1;
4221#endif
4222
4223 if (dev->real_num_tx_queues != 1) {
4224 const struct net_device_ops *ops = dev->netdev_ops;
4225
4226 if (ops->ndo_select_queue)
4227 queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4228 else
4229 queue_index = netdev_pick_tx(dev, skb, sb_dev);
4230
4231 queue_index = netdev_cap_txqueue(dev, queue_index);
4232 }
4233
4234 skb_set_queue_mapping(skb, queue_index);
4235 return netdev_get_tx_queue(dev, queue_index);
4236}
4237
4238/**
4239 * __dev_queue_xmit() - transmit a buffer
4240 * @skb: buffer to transmit
4241 * @sb_dev: suboordinate device used for L2 forwarding offload
4242 *
4243 * Queue a buffer for transmission to a network device. The caller must
4244 * have set the device and priority and built the buffer before calling
4245 * this function. The function can be called from an interrupt.
4246 *
4247 * When calling this method, interrupts MUST be enabled. This is because
4248 * the BH enable code must have IRQs enabled so that it will not deadlock.
4249 *
4250 * Regardless of the return value, the skb is consumed, so it is currently
4251 * difficult to retry a send to this method. (You can bump the ref count
4252 * before sending to hold a reference for retry if you are careful.)
4253 *
4254 * Return:
4255 * * 0 - buffer successfully transmitted
4256 * * positive qdisc return code - NET_XMIT_DROP etc.
4257 * * negative errno - other errors
4258 */
4259int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4260{
4261 struct net_device *dev = skb->dev;
4262 struct netdev_queue *txq = NULL;
4263 struct Qdisc *q;
4264 int rc = -ENOMEM;
4265 bool again = false;
4266
4267 skb_reset_mac_header(skb);
4268 skb_assert_len(skb);
4269
4270 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4271 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4272
4273 /* Disable soft irqs for various locks below. Also
4274 * stops preemption for RCU.
4275 */
4276 rcu_read_lock_bh();
4277
4278 skb_update_prio(skb);
4279
4280 qdisc_pkt_len_init(skb);
4281 tcx_set_ingress(skb, false);
4282#ifdef CONFIG_NET_EGRESS
4283 if (static_branch_unlikely(&egress_needed_key)) {
4284 if (nf_hook_egress_active()) {
4285 skb = nf_hook_egress(skb, &rc, dev);
4286 if (!skb)
4287 goto out;
4288 }
4289
4290 netdev_xmit_skip_txqueue(false);
4291
4292 nf_skip_egress(skb, true);
4293 skb = sch_handle_egress(skb, &rc, dev);
4294 if (!skb)
4295 goto out;
4296 nf_skip_egress(skb, false);
4297
4298 if (netdev_xmit_txqueue_skipped())
4299 txq = netdev_tx_queue_mapping(dev, skb);
4300 }
4301#endif
4302 /* If device/qdisc don't need skb->dst, release it right now while
4303 * its hot in this cpu cache.
4304 */
4305 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4306 skb_dst_drop(skb);
4307 else
4308 skb_dst_force(skb);
4309
4310 if (!txq)
4311 txq = netdev_core_pick_tx(dev, skb, sb_dev);
4312
4313 q = rcu_dereference_bh(txq->qdisc);
4314
4315 trace_net_dev_queue(skb);
4316 if (q->enqueue) {
4317 rc = __dev_xmit_skb(skb, q, dev, txq);
4318 goto out;
4319 }
4320
4321 /* The device has no queue. Common case for software devices:
4322 * loopback, all the sorts of tunnels...
4323
4324 * Really, it is unlikely that netif_tx_lock protection is necessary
4325 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
4326 * counters.)
4327 * However, it is possible, that they rely on protection
4328 * made by us here.
4329
4330 * Check this and shot the lock. It is not prone from deadlocks.
4331 *Either shot noqueue qdisc, it is even simpler 8)
4332 */
4333 if (dev->flags & IFF_UP) {
4334 int cpu = smp_processor_id(); /* ok because BHs are off */
4335
4336 /* Other cpus might concurrently change txq->xmit_lock_owner
4337 * to -1 or to their cpu id, but not to our id.
4338 */
4339 if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
4340 if (dev_xmit_recursion())
4341 goto recursion_alert;
4342
4343 skb = validate_xmit_skb(skb, dev, &again);
4344 if (!skb)
4345 goto out;
4346
4347 HARD_TX_LOCK(dev, txq, cpu);
4348
4349 if (!netif_xmit_stopped(txq)) {
4350 dev_xmit_recursion_inc();
4351 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4352 dev_xmit_recursion_dec();
4353 if (dev_xmit_complete(rc)) {
4354 HARD_TX_UNLOCK(dev, txq);
4355 goto out;
4356 }
4357 }
4358 HARD_TX_UNLOCK(dev, txq);
4359 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4360 dev->name);
4361 } else {
4362 /* Recursion is detected! It is possible,
4363 * unfortunately
4364 */
4365recursion_alert:
4366 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4367 dev->name);
4368 }
4369 }
4370
4371 rc = -ENETDOWN;
4372 rcu_read_unlock_bh();
4373
4374 dev_core_stats_tx_dropped_inc(dev);
4375 kfree_skb_list(skb);
4376 return rc;
4377out:
4378 rcu_read_unlock_bh();
4379 return rc;
4380}
4381EXPORT_SYMBOL(__dev_queue_xmit);
4382
4383int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4384{
4385 struct net_device *dev = skb->dev;
4386 struct sk_buff *orig_skb = skb;
4387 struct netdev_queue *txq;
4388 int ret = NETDEV_TX_BUSY;
4389 bool again = false;
4390
4391 if (unlikely(!netif_running(dev) ||
4392 !netif_carrier_ok(dev)))
4393 goto drop;
4394
4395 skb = validate_xmit_skb_list(skb, dev, &again);
4396 if (skb != orig_skb)
4397 goto drop;
4398
4399 skb_set_queue_mapping(skb, queue_id);
4400 txq = skb_get_tx_queue(dev, skb);
4401
4402 local_bh_disable();
4403
4404 dev_xmit_recursion_inc();
4405 HARD_TX_LOCK(dev, txq, smp_processor_id());
4406 if (!netif_xmit_frozen_or_drv_stopped(txq))
4407 ret = netdev_start_xmit(skb, dev, txq, false);
4408 HARD_TX_UNLOCK(dev, txq);
4409 dev_xmit_recursion_dec();
4410
4411 local_bh_enable();
4412 return ret;
4413drop:
4414 dev_core_stats_tx_dropped_inc(dev);
4415 kfree_skb_list(skb);
4416 return NET_XMIT_DROP;
4417}
4418EXPORT_SYMBOL(__dev_direct_xmit);
4419
4420/*************************************************************************
4421 * Receiver routines
4422 *************************************************************************/
4423
4424int netdev_max_backlog __read_mostly = 1000;
4425EXPORT_SYMBOL(netdev_max_backlog);
4426
4427int netdev_tstamp_prequeue __read_mostly = 1;
4428unsigned int sysctl_skb_defer_max __read_mostly = 64;
4429int netdev_budget __read_mostly = 300;
4430/* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4431unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
4432int weight_p __read_mostly = 64; /* old backlog weight */
4433int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
4434int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
4435int dev_rx_weight __read_mostly = 64;
4436int dev_tx_weight __read_mostly = 64;
4437
4438/* Called with irq disabled */
4439static inline void ____napi_schedule(struct softnet_data *sd,
4440 struct napi_struct *napi)
4441{
4442 struct task_struct *thread;
4443
4444 lockdep_assert_irqs_disabled();
4445
4446 if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4447 /* Paired with smp_mb__before_atomic() in
4448 * napi_enable()/dev_set_threaded().
4449 * Use READ_ONCE() to guarantee a complete
4450 * read on napi->thread. Only call
4451 * wake_up_process() when it's not NULL.
4452 */
4453 thread = READ_ONCE(napi->thread);
4454 if (thread) {
4455 /* Avoid doing set_bit() if the thread is in
4456 * INTERRUPTIBLE state, cause napi_thread_wait()
4457 * makes sure to proceed with napi polling
4458 * if the thread is explicitly woken from here.
4459 */
4460 if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4461 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4462 wake_up_process(thread);
4463 return;
4464 }
4465 }
4466
4467 list_add_tail(&napi->poll_list, &sd->poll_list);
4468 WRITE_ONCE(napi->list_owner, smp_processor_id());
4469 /* If not called from net_rx_action()
4470 * we have to raise NET_RX_SOFTIRQ.
4471 */
4472 if (!sd->in_net_rx_action)
4473 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4474}
4475
4476#ifdef CONFIG_RPS
4477
4478/* One global table that all flow-based protocols share. */
4479struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4480EXPORT_SYMBOL(rps_sock_flow_table);
4481u32 rps_cpu_mask __read_mostly;
4482EXPORT_SYMBOL(rps_cpu_mask);
4483
4484struct static_key_false rps_needed __read_mostly;
4485EXPORT_SYMBOL(rps_needed);
4486struct static_key_false rfs_needed __read_mostly;
4487EXPORT_SYMBOL(rfs_needed);
4488
4489static struct rps_dev_flow *
4490set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4491 struct rps_dev_flow *rflow, u16 next_cpu)
4492{
4493 if (next_cpu < nr_cpu_ids) {
4494#ifdef CONFIG_RFS_ACCEL
4495 struct netdev_rx_queue *rxqueue;
4496 struct rps_dev_flow_table *flow_table;
4497 struct rps_dev_flow *old_rflow;
4498 u32 flow_id;
4499 u16 rxq_index;
4500 int rc;
4501
4502 /* Should we steer this flow to a different hardware queue? */
4503 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4504 !(dev->features & NETIF_F_NTUPLE))
4505 goto out;
4506 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4507 if (rxq_index == skb_get_rx_queue(skb))
4508 goto out;
4509
4510 rxqueue = dev->_rx + rxq_index;
4511 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4512 if (!flow_table)
4513 goto out;
4514 flow_id = skb_get_hash(skb) & flow_table->mask;
4515 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4516 rxq_index, flow_id);
4517 if (rc < 0)
4518 goto out;
4519 old_rflow = rflow;
4520 rflow = &flow_table->flows[flow_id];
4521 rflow->filter = rc;
4522 if (old_rflow->filter == rflow->filter)
4523 old_rflow->filter = RPS_NO_FILTER;
4524 out:
4525#endif
4526 rflow->last_qtail =
4527 per_cpu(softnet_data, next_cpu).input_queue_head;
4528 }
4529
4530 rflow->cpu = next_cpu;
4531 return rflow;
4532}
4533
4534/*
4535 * get_rps_cpu is called from netif_receive_skb and returns the target
4536 * CPU from the RPS map of the receiving queue for a given skb.
4537 * rcu_read_lock must be held on entry.
4538 */
4539static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4540 struct rps_dev_flow **rflowp)
4541{
4542 const struct rps_sock_flow_table *sock_flow_table;
4543 struct netdev_rx_queue *rxqueue = dev->_rx;
4544 struct rps_dev_flow_table *flow_table;
4545 struct rps_map *map;
4546 int cpu = -1;
4547 u32 tcpu;
4548 u32 hash;
4549
4550 if (skb_rx_queue_recorded(skb)) {
4551 u16 index = skb_get_rx_queue(skb);
4552
4553 if (unlikely(index >= dev->real_num_rx_queues)) {
4554 WARN_ONCE(dev->real_num_rx_queues > 1,
4555 "%s received packet on queue %u, but number "
4556 "of RX queues is %u\n",
4557 dev->name, index, dev->real_num_rx_queues);
4558 goto done;
4559 }
4560 rxqueue += index;
4561 }
4562
4563 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4564
4565 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4566 map = rcu_dereference(rxqueue->rps_map);
4567 if (!flow_table && !map)
4568 goto done;
4569
4570 skb_reset_network_header(skb);
4571 hash = skb_get_hash(skb);
4572 if (!hash)
4573 goto done;
4574
4575 sock_flow_table = rcu_dereference(rps_sock_flow_table);
4576 if (flow_table && sock_flow_table) {
4577 struct rps_dev_flow *rflow;
4578 u32 next_cpu;
4579 u32 ident;
4580
4581 /* First check into global flow table if there is a match.
4582 * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
4583 */
4584 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);
4585 if ((ident ^ hash) & ~rps_cpu_mask)
4586 goto try_rps;
4587
4588 next_cpu = ident & rps_cpu_mask;
4589
4590 /* OK, now we know there is a match,
4591 * we can look at the local (per receive queue) flow table
4592 */
4593 rflow = &flow_table->flows[hash & flow_table->mask];
4594 tcpu = rflow->cpu;
4595
4596 /*
4597 * If the desired CPU (where last recvmsg was done) is
4598 * different from current CPU (one in the rx-queue flow
4599 * table entry), switch if one of the following holds:
4600 * - Current CPU is unset (>= nr_cpu_ids).
4601 * - Current CPU is offline.
4602 * - The current CPU's queue tail has advanced beyond the
4603 * last packet that was enqueued using this table entry.
4604 * This guarantees that all previous packets for the flow
4605 * have been dequeued, thus preserving in order delivery.
4606 */
4607 if (unlikely(tcpu != next_cpu) &&
4608 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4609 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4610 rflow->last_qtail)) >= 0)) {
4611 tcpu = next_cpu;
4612 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4613 }
4614
4615 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4616 *rflowp = rflow;
4617 cpu = tcpu;
4618 goto done;
4619 }
4620 }
4621
4622try_rps:
4623
4624 if (map) {
4625 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4626 if (cpu_online(tcpu)) {
4627 cpu = tcpu;
4628 goto done;
4629 }
4630 }
4631
4632done:
4633 return cpu;
4634}
4635
4636#ifdef CONFIG_RFS_ACCEL
4637
4638/**
4639 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4640 * @dev: Device on which the filter was set
4641 * @rxq_index: RX queue index
4642 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4643 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4644 *
4645 * Drivers that implement ndo_rx_flow_steer() should periodically call
4646 * this function for each installed filter and remove the filters for
4647 * which it returns %true.
4648 */
4649bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4650 u32 flow_id, u16 filter_id)
4651{
4652 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4653 struct rps_dev_flow_table *flow_table;
4654 struct rps_dev_flow *rflow;
4655 bool expire = true;
4656 unsigned int cpu;
4657
4658 rcu_read_lock();
4659 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4660 if (flow_table && flow_id <= flow_table->mask) {
4661 rflow = &flow_table->flows[flow_id];
4662 cpu = READ_ONCE(rflow->cpu);
4663 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4664 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4665 rflow->last_qtail) <
4666 (int)(10 * flow_table->mask)))
4667 expire = false;
4668 }
4669 rcu_read_unlock();
4670 return expire;
4671}
4672EXPORT_SYMBOL(rps_may_expire_flow);
4673
4674#endif /* CONFIG_RFS_ACCEL */
4675
4676/* Called from hardirq (IPI) context */
4677static void rps_trigger_softirq(void *data)
4678{
4679 struct softnet_data *sd = data;
4680
4681 ____napi_schedule(sd, &sd->backlog);
4682 sd->received_rps++;
4683}
4684
4685#endif /* CONFIG_RPS */
4686
4687/* Called from hardirq (IPI) context */
4688static void trigger_rx_softirq(void *data)
4689{
4690 struct softnet_data *sd = data;
4691
4692 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4693 smp_store_release(&sd->defer_ipi_scheduled, 0);
4694}
4695
4696/*
4697 * After we queued a packet into sd->input_pkt_queue,
4698 * we need to make sure this queue is serviced soon.
4699 *
4700 * - If this is another cpu queue, link it to our rps_ipi_list,
4701 * and make sure we will process rps_ipi_list from net_rx_action().
4702 *
4703 * - If this is our own queue, NAPI schedule our backlog.
4704 * Note that this also raises NET_RX_SOFTIRQ.
4705 */
4706static void napi_schedule_rps(struct softnet_data *sd)
4707{
4708 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4709
4710#ifdef CONFIG_RPS
4711 if (sd != mysd) {
4712 sd->rps_ipi_next = mysd->rps_ipi_list;
4713 mysd->rps_ipi_list = sd;
4714
4715 /* If not called from net_rx_action() or napi_threaded_poll()
4716 * we have to raise NET_RX_SOFTIRQ.
4717 */
4718 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll)
4719 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4720 return;
4721 }
4722#endif /* CONFIG_RPS */
4723 __napi_schedule_irqoff(&mysd->backlog);
4724}
4725
4726#ifdef CONFIG_NET_FLOW_LIMIT
4727int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4728#endif
4729
4730static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4731{
4732#ifdef CONFIG_NET_FLOW_LIMIT
4733 struct sd_flow_limit *fl;
4734 struct softnet_data *sd;
4735 unsigned int old_flow, new_flow;
4736
4737 if (qlen < (READ_ONCE(netdev_max_backlog) >> 1))
4738 return false;
4739
4740 sd = this_cpu_ptr(&softnet_data);
4741
4742 rcu_read_lock();
4743 fl = rcu_dereference(sd->flow_limit);
4744 if (fl) {
4745 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4746 old_flow = fl->history[fl->history_head];
4747 fl->history[fl->history_head] = new_flow;
4748
4749 fl->history_head++;
4750 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4751
4752 if (likely(fl->buckets[old_flow]))
4753 fl->buckets[old_flow]--;
4754
4755 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4756 fl->count++;
4757 rcu_read_unlock();
4758 return true;
4759 }
4760 }
4761 rcu_read_unlock();
4762#endif
4763 return false;
4764}
4765
4766/*
4767 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4768 * queue (may be a remote CPU queue).
4769 */
4770static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4771 unsigned int *qtail)
4772{
4773 enum skb_drop_reason reason;
4774 struct softnet_data *sd;
4775 unsigned long flags;
4776 unsigned int qlen;
4777
4778 reason = SKB_DROP_REASON_NOT_SPECIFIED;
4779 sd = &per_cpu(softnet_data, cpu);
4780
4781 rps_lock_irqsave(sd, &flags);
4782 if (!netif_running(skb->dev))
4783 goto drop;
4784 qlen = skb_queue_len(&sd->input_pkt_queue);
4785 if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) {
4786 if (qlen) {
4787enqueue:
4788 __skb_queue_tail(&sd->input_pkt_queue, skb);
4789 input_queue_tail_incr_save(sd, qtail);
4790 rps_unlock_irq_restore(sd, &flags);
4791 return NET_RX_SUCCESS;
4792 }
4793
4794 /* Schedule NAPI for backlog device
4795 * We can use non atomic operation since we own the queue lock
4796 */
4797 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
4798 napi_schedule_rps(sd);
4799 goto enqueue;
4800 }
4801 reason = SKB_DROP_REASON_CPU_BACKLOG;
4802
4803drop:
4804 sd->dropped++;
4805 rps_unlock_irq_restore(sd, &flags);
4806
4807 dev_core_stats_rx_dropped_inc(skb->dev);
4808 kfree_skb_reason(skb, reason);
4809 return NET_RX_DROP;
4810}
4811
4812static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4813{
4814 struct net_device *dev = skb->dev;
4815 struct netdev_rx_queue *rxqueue;
4816
4817 rxqueue = dev->_rx;
4818
4819 if (skb_rx_queue_recorded(skb)) {
4820 u16 index = skb_get_rx_queue(skb);
4821
4822 if (unlikely(index >= dev->real_num_rx_queues)) {
4823 WARN_ONCE(dev->real_num_rx_queues > 1,
4824 "%s received packet on queue %u, but number "
4825 "of RX queues is %u\n",
4826 dev->name, index, dev->real_num_rx_queues);
4827
4828 return rxqueue; /* Return first rxqueue */
4829 }
4830 rxqueue += index;
4831 }
4832 return rxqueue;
4833}
4834
4835u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4836 struct bpf_prog *xdp_prog)
4837{
4838 void *orig_data, *orig_data_end, *hard_start;
4839 struct netdev_rx_queue *rxqueue;
4840 bool orig_bcast, orig_host;
4841 u32 mac_len, frame_sz;
4842 __be16 orig_eth_type;
4843 struct ethhdr *eth;
4844 u32 metalen, act;
4845 int off;
4846
4847 /* The XDP program wants to see the packet starting at the MAC
4848 * header.
4849 */
4850 mac_len = skb->data - skb_mac_header(skb);
4851 hard_start = skb->data - skb_headroom(skb);
4852
4853 /* SKB "head" area always have tailroom for skb_shared_info */
4854 frame_sz = (void *)skb_end_pointer(skb) - hard_start;
4855 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4856
4857 rxqueue = netif_get_rxqueue(skb);
4858 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4859 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4860 skb_headlen(skb) + mac_len, true);
4861
4862 orig_data_end = xdp->data_end;
4863 orig_data = xdp->data;
4864 eth = (struct ethhdr *)xdp->data;
4865 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
4866 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4867 orig_eth_type = eth->h_proto;
4868
4869 act = bpf_prog_run_xdp(xdp_prog, xdp);
4870
4871 /* check if bpf_xdp_adjust_head was used */
4872 off = xdp->data - orig_data;
4873 if (off) {
4874 if (off > 0)
4875 __skb_pull(skb, off);
4876 else if (off < 0)
4877 __skb_push(skb, -off);
4878
4879 skb->mac_header += off;
4880 skb_reset_network_header(skb);
4881 }
4882
4883 /* check if bpf_xdp_adjust_tail was used */
4884 off = xdp->data_end - orig_data_end;
4885 if (off != 0) {
4886 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4887 skb->len += off; /* positive on grow, negative on shrink */
4888 }
4889
4890 /* check if XDP changed eth hdr such SKB needs update */
4891 eth = (struct ethhdr *)xdp->data;
4892 if ((orig_eth_type != eth->h_proto) ||
4893 (orig_host != ether_addr_equal_64bits(eth->h_dest,
4894 skb->dev->dev_addr)) ||
4895 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4896 __skb_push(skb, ETH_HLEN);
4897 skb->pkt_type = PACKET_HOST;
4898 skb->protocol = eth_type_trans(skb, skb->dev);
4899 }
4900
4901 /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
4902 * before calling us again on redirect path. We do not call do_redirect
4903 * as we leave that up to the caller.
4904 *
4905 * Caller is responsible for managing lifetime of skb (i.e. calling
4906 * kfree_skb in response to actions it cannot handle/XDP_DROP).
4907 */
4908 switch (act) {
4909 case XDP_REDIRECT:
4910 case XDP_TX:
4911 __skb_push(skb, mac_len);
4912 break;
4913 case XDP_PASS:
4914 metalen = xdp->data - xdp->data_meta;
4915 if (metalen)
4916 skb_metadata_set(skb, metalen);
4917 break;
4918 }
4919
4920 return act;
4921}
4922
4923static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4924 struct xdp_buff *xdp,
4925 struct bpf_prog *xdp_prog)
4926{
4927 u32 act = XDP_DROP;
4928
4929 /* Reinjected packets coming from act_mirred or similar should
4930 * not get XDP generic processing.
4931 */
4932 if (skb_is_redirected(skb))
4933 return XDP_PASS;
4934
4935 /* XDP packets must be linear and must have sufficient headroom
4936 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4937 * native XDP provides, thus we need to do it here as well.
4938 */
4939 if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4940 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4941 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4942 int troom = skb->tail + skb->data_len - skb->end;
4943
4944 /* In case we have to go down the path and also linearize,
4945 * then lets do the pskb_expand_head() work just once here.
4946 */
4947 if (pskb_expand_head(skb,
4948 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4949 troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4950 goto do_drop;
4951 if (skb_linearize(skb))
4952 goto do_drop;
4953 }
4954
4955 act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4956 switch (act) {
4957 case XDP_REDIRECT:
4958 case XDP_TX:
4959 case XDP_PASS:
4960 break;
4961 default:
4962 bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act);
4963 fallthrough;
4964 case XDP_ABORTED:
4965 trace_xdp_exception(skb->dev, xdp_prog, act);
4966 fallthrough;
4967 case XDP_DROP:
4968 do_drop:
4969 kfree_skb(skb);
4970 break;
4971 }
4972
4973 return act;
4974}
4975
4976/* When doing generic XDP we have to bypass the qdisc layer and the
4977 * network taps in order to match in-driver-XDP behavior. This also means
4978 * that XDP packets are able to starve other packets going through a qdisc,
4979 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
4980 * queues, so they do not have this starvation issue.
4981 */
4982void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4983{
4984 struct net_device *dev = skb->dev;
4985 struct netdev_queue *txq;
4986 bool free_skb = true;
4987 int cpu, rc;
4988
4989 txq = netdev_core_pick_tx(dev, skb, NULL);
4990 cpu = smp_processor_id();
4991 HARD_TX_LOCK(dev, txq, cpu);
4992 if (!netif_xmit_frozen_or_drv_stopped(txq)) {
4993 rc = netdev_start_xmit(skb, dev, txq, 0);
4994 if (dev_xmit_complete(rc))
4995 free_skb = false;
4996 }
4997 HARD_TX_UNLOCK(dev, txq);
4998 if (free_skb) {
4999 trace_xdp_exception(dev, xdp_prog, XDP_TX);
5000 dev_core_stats_tx_dropped_inc(dev);
5001 kfree_skb(skb);
5002 }
5003}
5004
5005static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
5006
5007int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
5008{
5009 if (xdp_prog) {
5010 struct xdp_buff xdp;
5011 u32 act;
5012 int err;
5013
5014 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
5015 if (act != XDP_PASS) {
5016 switch (act) {
5017 case XDP_REDIRECT:
5018 err = xdp_do_generic_redirect(skb->dev, skb,
5019 &xdp, xdp_prog);
5020 if (err)
5021 goto out_redir;
5022 break;
5023 case XDP_TX:
5024 generic_xdp_tx(skb, xdp_prog);
5025 break;
5026 }
5027 return XDP_DROP;
5028 }
5029 }
5030 return XDP_PASS;
5031out_redir:
5032 kfree_skb_reason(skb, SKB_DROP_REASON_XDP);
5033 return XDP_DROP;
5034}
5035EXPORT_SYMBOL_GPL(do_xdp_generic);
5036
5037static int netif_rx_internal(struct sk_buff *skb)
5038{
5039 int ret;
5040
5041 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5042
5043 trace_netif_rx(skb);
5044
5045#ifdef CONFIG_RPS
5046 if (static_branch_unlikely(&rps_needed)) {
5047 struct rps_dev_flow voidflow, *rflow = &voidflow;
5048 int cpu;
5049
5050 rcu_read_lock();
5051
5052 cpu = get_rps_cpu(skb->dev, skb, &rflow);
5053 if (cpu < 0)
5054 cpu = smp_processor_id();
5055
5056 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5057
5058 rcu_read_unlock();
5059 } else
5060#endif
5061 {
5062 unsigned int qtail;
5063
5064 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
5065 }
5066 return ret;
5067}
5068
5069/**
5070 * __netif_rx - Slightly optimized version of netif_rx
5071 * @skb: buffer to post
5072 *
5073 * This behaves as netif_rx except that it does not disable bottom halves.
5074 * As a result this function may only be invoked from the interrupt context
5075 * (either hard or soft interrupt).
5076 */
5077int __netif_rx(struct sk_buff *skb)
5078{
5079 int ret;
5080
5081 lockdep_assert_once(hardirq_count() | softirq_count());
5082
5083 trace_netif_rx_entry(skb);
5084 ret = netif_rx_internal(skb);
5085 trace_netif_rx_exit(ret);
5086 return ret;
5087}
5088EXPORT_SYMBOL(__netif_rx);
5089
5090/**
5091 * netif_rx - post buffer to the network code
5092 * @skb: buffer to post
5093 *
5094 * This function receives a packet from a device driver and queues it for
5095 * the upper (protocol) levels to process via the backlog NAPI device. It
5096 * always succeeds. The buffer may be dropped during processing for
5097 * congestion control or by the protocol layers.
5098 * The network buffer is passed via the backlog NAPI device. Modern NIC
5099 * driver should use NAPI and GRO.
5100 * This function can used from interrupt and from process context. The
5101 * caller from process context must not disable interrupts before invoking
5102 * this function.
5103 *
5104 * return values:
5105 * NET_RX_SUCCESS (no congestion)
5106 * NET_RX_DROP (packet was dropped)
5107 *
5108 */
5109int netif_rx(struct sk_buff *skb)
5110{
5111 bool need_bh_off = !(hardirq_count() | softirq_count());
5112 int ret;
5113
5114 if (need_bh_off)
5115 local_bh_disable();
5116 trace_netif_rx_entry(skb);
5117 ret = netif_rx_internal(skb);
5118 trace_netif_rx_exit(ret);
5119 if (need_bh_off)
5120 local_bh_enable();
5121 return ret;
5122}
5123EXPORT_SYMBOL(netif_rx);
5124
5125static __latent_entropy void net_tx_action(struct softirq_action *h)
5126{
5127 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5128
5129 if (sd->completion_queue) {
5130 struct sk_buff *clist;
5131
5132 local_irq_disable();
5133 clist = sd->completion_queue;
5134 sd->completion_queue = NULL;
5135 local_irq_enable();
5136
5137 while (clist) {
5138 struct sk_buff *skb = clist;
5139
5140 clist = clist->next;
5141
5142 WARN_ON(refcount_read(&skb->users));
5143 if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED))
5144 trace_consume_skb(skb, net_tx_action);
5145 else
5146 trace_kfree_skb(skb, net_tx_action,
5147 get_kfree_skb_cb(skb)->reason);
5148
5149 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5150 __kfree_skb(skb);
5151 else
5152 __napi_kfree_skb(skb,
5153 get_kfree_skb_cb(skb)->reason);
5154 }
5155 }
5156
5157 if (sd->output_queue) {
5158 struct Qdisc *head;
5159
5160 local_irq_disable();
5161 head = sd->output_queue;
5162 sd->output_queue = NULL;
5163 sd->output_queue_tailp = &sd->output_queue;
5164 local_irq_enable();
5165
5166 rcu_read_lock();
5167
5168 while (head) {
5169 struct Qdisc *q = head;
5170 spinlock_t *root_lock = NULL;
5171
5172 head = head->next_sched;
5173
5174 /* We need to make sure head->next_sched is read
5175 * before clearing __QDISC_STATE_SCHED
5176 */
5177 smp_mb__before_atomic();
5178
5179 if (!(q->flags & TCQ_F_NOLOCK)) {
5180 root_lock = qdisc_lock(q);
5181 spin_lock(root_lock);
5182 } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5183 &q->state))) {
5184 /* There is a synchronize_net() between
5185 * STATE_DEACTIVATED flag being set and
5186 * qdisc_reset()/some_qdisc_is_busy() in
5187 * dev_deactivate(), so we can safely bail out
5188 * early here to avoid data race between
5189 * qdisc_deactivate() and some_qdisc_is_busy()
5190 * for lockless qdisc.
5191 */
5192 clear_bit(__QDISC_STATE_SCHED, &q->state);
5193 continue;
5194 }
5195
5196 clear_bit(__QDISC_STATE_SCHED, &q->state);
5197 qdisc_run(q);
5198 if (root_lock)
5199 spin_unlock(root_lock);
5200 }
5201
5202 rcu_read_unlock();
5203 }
5204
5205 xfrm_dev_backlog(sd);
5206}
5207
5208#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
5209/* This hook is defined here for ATM LANE */
5210int (*br_fdb_test_addr_hook)(struct net_device *dev,
5211 unsigned char *addr) __read_mostly;
5212EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
5213#endif
5214
5215/**
5216 * netdev_is_rx_handler_busy - check if receive handler is registered
5217 * @dev: device to check
5218 *
5219 * Check if a receive handler is already registered for a given device.
5220 * Return true if there one.
5221 *
5222 * The caller must hold the rtnl_mutex.
5223 */
5224bool netdev_is_rx_handler_busy(struct net_device *dev)
5225{
5226 ASSERT_RTNL();
5227 return dev && rtnl_dereference(dev->rx_handler);
5228}
5229EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5230
5231/**
5232 * netdev_rx_handler_register - register receive handler
5233 * @dev: device to register a handler for
5234 * @rx_handler: receive handler to register
5235 * @rx_handler_data: data pointer that is used by rx handler
5236 *
5237 * Register a receive handler for a device. This handler will then be
5238 * called from __netif_receive_skb. A negative errno code is returned
5239 * on a failure.
5240 *
5241 * The caller must hold the rtnl_mutex.
5242 *
5243 * For a general description of rx_handler, see enum rx_handler_result.
5244 */
5245int netdev_rx_handler_register(struct net_device *dev,
5246 rx_handler_func_t *rx_handler,
5247 void *rx_handler_data)
5248{
5249 if (netdev_is_rx_handler_busy(dev))
5250 return -EBUSY;
5251
5252 if (dev->priv_flags & IFF_NO_RX_HANDLER)
5253 return -EINVAL;
5254
5255 /* Note: rx_handler_data must be set before rx_handler */
5256 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
5257 rcu_assign_pointer(dev->rx_handler, rx_handler);
5258
5259 return 0;
5260}
5261EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5262
5263/**
5264 * netdev_rx_handler_unregister - unregister receive handler
5265 * @dev: device to unregister a handler from
5266 *
5267 * Unregister a receive handler from a device.
5268 *
5269 * The caller must hold the rtnl_mutex.
5270 */
5271void netdev_rx_handler_unregister(struct net_device *dev)
5272{
5273
5274 ASSERT_RTNL();
5275 RCU_INIT_POINTER(dev->rx_handler, NULL);
5276 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5277 * section has a guarantee to see a non NULL rx_handler_data
5278 * as well.
5279 */
5280 synchronize_net();
5281 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
5282}
5283EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5284
5285/*
5286 * Limit the use of PFMEMALLOC reserves to those protocols that implement
5287 * the special handling of PFMEMALLOC skbs.
5288 */
5289static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5290{
5291 switch (skb->protocol) {
5292 case htons(ETH_P_ARP):
5293 case htons(ETH_P_IP):
5294 case htons(ETH_P_IPV6):
5295 case htons(ETH_P_8021Q):
5296 case htons(ETH_P_8021AD):
5297 return true;
5298 default:
5299 return false;
5300 }
5301}
5302
5303static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5304 int *ret, struct net_device *orig_dev)
5305{
5306 if (nf_hook_ingress_active(skb)) {
5307 int ingress_retval;
5308
5309 if (*pt_prev) {
5310 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5311 *pt_prev = NULL;
5312 }
5313
5314 rcu_read_lock();
5315 ingress_retval = nf_hook_ingress(skb);
5316 rcu_read_unlock();
5317 return ingress_retval;
5318 }
5319 return 0;
5320}
5321
5322static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
5323 struct packet_type **ppt_prev)
5324{
5325 struct packet_type *ptype, *pt_prev;
5326 rx_handler_func_t *rx_handler;
5327 struct sk_buff *skb = *pskb;
5328 struct net_device *orig_dev;
5329 bool deliver_exact = false;
5330 int ret = NET_RX_DROP;
5331 __be16 type;
5332
5333 net_timestamp_check(!READ_ONCE(netdev_tstamp_prequeue), skb);
5334
5335 trace_netif_receive_skb(skb);
5336
5337 orig_dev = skb->dev;
5338
5339 skb_reset_network_header(skb);
5340 if (!skb_transport_header_was_set(skb))
5341 skb_reset_transport_header(skb);
5342 skb_reset_mac_len(skb);
5343
5344 pt_prev = NULL;
5345
5346another_round:
5347 skb->skb_iif = skb->dev->ifindex;
5348
5349 __this_cpu_inc(softnet_data.processed);
5350
5351 if (static_branch_unlikely(&generic_xdp_needed_key)) {
5352 int ret2;
5353
5354 migrate_disable();
5355 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5356 migrate_enable();
5357
5358 if (ret2 != XDP_PASS) {
5359 ret = NET_RX_DROP;
5360 goto out;
5361 }
5362 }
5363
5364 if (eth_type_vlan(skb->protocol)) {
5365 skb = skb_vlan_untag(skb);
5366 if (unlikely(!skb))
5367 goto out;
5368 }
5369
5370 if (skb_skip_tc_classify(skb))
5371 goto skip_classify;
5372
5373 if (pfmemalloc)
5374 goto skip_taps;
5375
5376 list_for_each_entry_rcu(ptype, &ptype_all, list) {
5377 if (pt_prev)
5378 ret = deliver_skb(skb, pt_prev, orig_dev);
5379 pt_prev = ptype;
5380 }
5381
5382 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5383 if (pt_prev)
5384 ret = deliver_skb(skb, pt_prev, orig_dev);
5385 pt_prev = ptype;
5386 }
5387
5388skip_taps:
5389#ifdef CONFIG_NET_INGRESS
5390 if (static_branch_unlikely(&ingress_needed_key)) {
5391 bool another = false;
5392
5393 nf_skip_egress(skb, true);
5394 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5395 &another);
5396 if (another)
5397 goto another_round;
5398 if (!skb)
5399 goto out;
5400
5401 nf_skip_egress(skb, false);
5402 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
5403 goto out;
5404 }
5405#endif
5406 skb_reset_redirect(skb);
5407skip_classify:
5408 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
5409 goto drop;
5410
5411 if (skb_vlan_tag_present(skb)) {
5412 if (pt_prev) {
5413 ret = deliver_skb(skb, pt_prev, orig_dev);
5414 pt_prev = NULL;
5415 }
5416 if (vlan_do_receive(&skb))
5417 goto another_round;
5418 else if (unlikely(!skb))
5419 goto out;
5420 }
5421
5422 rx_handler = rcu_dereference(skb->dev->rx_handler);
5423 if (rx_handler) {
5424 if (pt_prev) {
5425 ret = deliver_skb(skb, pt_prev, orig_dev);
5426 pt_prev = NULL;
5427 }
5428 switch (rx_handler(&skb)) {
5429 case RX_HANDLER_CONSUMED:
5430 ret = NET_RX_SUCCESS;
5431 goto out;
5432 case RX_HANDLER_ANOTHER:
5433 goto another_round;
5434 case RX_HANDLER_EXACT:
5435 deliver_exact = true;
5436 break;
5437 case RX_HANDLER_PASS:
5438 break;
5439 default:
5440 BUG();
5441 }
5442 }
5443
5444 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
5445check_vlan_id:
5446 if (skb_vlan_tag_get_id(skb)) {
5447 /* Vlan id is non 0 and vlan_do_receive() above couldn't
5448 * find vlan device.
5449 */
5450 skb->pkt_type = PACKET_OTHERHOST;
5451 } else if (eth_type_vlan(skb->protocol)) {
5452 /* Outer header is 802.1P with vlan 0, inner header is
5453 * 802.1Q or 802.1AD and vlan_do_receive() above could
5454 * not find vlan dev for vlan id 0.
5455 */
5456 __vlan_hwaccel_clear_tag(skb);
5457 skb = skb_vlan_untag(skb);
5458 if (unlikely(!skb))
5459 goto out;
5460 if (vlan_do_receive(&skb))
5461 /* After stripping off 802.1P header with vlan 0
5462 * vlan dev is found for inner header.
5463 */
5464 goto another_round;
5465 else if (unlikely(!skb))
5466 goto out;
5467 else
5468 /* We have stripped outer 802.1P vlan 0 header.
5469 * But could not find vlan dev.
5470 * check again for vlan id to set OTHERHOST.
5471 */
5472 goto check_vlan_id;
5473 }
5474 /* Note: we might in the future use prio bits
5475 * and set skb->priority like in vlan_do_receive()
5476 * For the time being, just ignore Priority Code Point
5477 */
5478 __vlan_hwaccel_clear_tag(skb);
5479 }
5480
5481 type = skb->protocol;
5482
5483 /* deliver only exact match when indicated */
5484 if (likely(!deliver_exact)) {
5485 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5486 &ptype_base[ntohs(type) &
5487 PTYPE_HASH_MASK]);
5488 }
5489
5490 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5491 &orig_dev->ptype_specific);
5492
5493 if (unlikely(skb->dev != orig_dev)) {
5494 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5495 &skb->dev->ptype_specific);
5496 }
5497
5498 if (pt_prev) {
5499 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
5500 goto drop;
5501 *ppt_prev = pt_prev;
5502 } else {
5503drop:
5504 if (!deliver_exact)
5505 dev_core_stats_rx_dropped_inc(skb->dev);
5506 else
5507 dev_core_stats_rx_nohandler_inc(skb->dev);
5508 kfree_skb_reason(skb, SKB_DROP_REASON_UNHANDLED_PROTO);
5509 /* Jamal, now you will not able to escape explaining
5510 * me how you were going to use this. :-)
5511 */
5512 ret = NET_RX_DROP;
5513 }
5514
5515out:
5516 /* The invariant here is that if *ppt_prev is not NULL
5517 * then skb should also be non-NULL.
5518 *
5519 * Apparently *ppt_prev assignment above holds this invariant due to
5520 * skb dereferencing near it.
5521 */
5522 *pskb = skb;
5523 return ret;
5524}
5525
5526static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5527{
5528 struct net_device *orig_dev = skb->dev;
5529 struct packet_type *pt_prev = NULL;
5530 int ret;
5531
5532 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5533 if (pt_prev)
5534 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5535 skb->dev, pt_prev, orig_dev);
5536 return ret;
5537}
5538
5539/**
5540 * netif_receive_skb_core - special purpose version of netif_receive_skb
5541 * @skb: buffer to process
5542 *
5543 * More direct receive version of netif_receive_skb(). It should
5544 * only be used by callers that have a need to skip RPS and Generic XDP.
5545 * Caller must also take care of handling if ``(page_is_)pfmemalloc``.
5546 *
5547 * This function may only be called from softirq context and interrupts
5548 * should be enabled.
5549 *
5550 * Return values (usually ignored):
5551 * NET_RX_SUCCESS: no congestion
5552 * NET_RX_DROP: packet was dropped
5553 */
5554int netif_receive_skb_core(struct sk_buff *skb)
5555{
5556 int ret;
5557
5558 rcu_read_lock();
5559 ret = __netif_receive_skb_one_core(skb, false);
5560 rcu_read_unlock();
5561
5562 return ret;
5563}
5564EXPORT_SYMBOL(netif_receive_skb_core);
5565
5566static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5567 struct packet_type *pt_prev,
5568 struct net_device *orig_dev)
5569{
5570 struct sk_buff *skb, *next;
5571
5572 if (!pt_prev)
5573 return;
5574 if (list_empty(head))
5575 return;
5576 if (pt_prev->list_func != NULL)
5577 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5578 ip_list_rcv, head, pt_prev, orig_dev);
5579 else
5580 list_for_each_entry_safe(skb, next, head, list) {
5581 skb_list_del_init(skb);
5582 pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5583 }
5584}
5585
5586static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5587{
5588 /* Fast-path assumptions:
5589 * - There is no RX handler.
5590 * - Only one packet_type matches.
5591 * If either of these fails, we will end up doing some per-packet
5592 * processing in-line, then handling the 'last ptype' for the whole
5593 * sublist. This can't cause out-of-order delivery to any single ptype,
5594 * because the 'last ptype' must be constant across the sublist, and all
5595 * other ptypes are handled per-packet.
5596 */
5597 /* Current (common) ptype of sublist */
5598 struct packet_type *pt_curr = NULL;
5599 /* Current (common) orig_dev of sublist */
5600 struct net_device *od_curr = NULL;
5601 struct list_head sublist;
5602 struct sk_buff *skb, *next;
5603
5604 INIT_LIST_HEAD(&sublist);
5605 list_for_each_entry_safe(skb, next, head, list) {
5606 struct net_device *orig_dev = skb->dev;
5607 struct packet_type *pt_prev = NULL;
5608
5609 skb_list_del_init(skb);
5610 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5611 if (!pt_prev)
5612 continue;
5613 if (pt_curr != pt_prev || od_curr != orig_dev) {
5614 /* dispatch old sublist */
5615 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5616 /* start new sublist */
5617 INIT_LIST_HEAD(&sublist);
5618 pt_curr = pt_prev;
5619 od_curr = orig_dev;
5620 }
5621 list_add_tail(&skb->list, &sublist);
5622 }
5623
5624 /* dispatch final sublist */
5625 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5626}
5627
5628static int __netif_receive_skb(struct sk_buff *skb)
5629{
5630 int ret;
5631
5632 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5633 unsigned int noreclaim_flag;
5634
5635 /*
5636 * PFMEMALLOC skbs are special, they should
5637 * - be delivered to SOCK_MEMALLOC sockets only
5638 * - stay away from userspace
5639 * - have bounded memory usage
5640 *
5641 * Use PF_MEMALLOC as this saves us from propagating the allocation
5642 * context down to all allocation sites.
5643 */
5644 noreclaim_flag = memalloc_noreclaim_save();
5645 ret = __netif_receive_skb_one_core(skb, true);
5646 memalloc_noreclaim_restore(noreclaim_flag);
5647 } else
5648 ret = __netif_receive_skb_one_core(skb, false);
5649
5650 return ret;
5651}
5652
5653static void __netif_receive_skb_list(struct list_head *head)
5654{
5655 unsigned long noreclaim_flag = 0;
5656 struct sk_buff *skb, *next;
5657 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5658
5659 list_for_each_entry_safe(skb, next, head, list) {
5660 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5661 struct list_head sublist;
5662
5663 /* Handle the previous sublist */
5664 list_cut_before(&sublist, head, &skb->list);
5665 if (!list_empty(&sublist))
5666 __netif_receive_skb_list_core(&sublist, pfmemalloc);
5667 pfmemalloc = !pfmemalloc;
5668 /* See comments in __netif_receive_skb */
5669 if (pfmemalloc)
5670 noreclaim_flag = memalloc_noreclaim_save();
5671 else
5672 memalloc_noreclaim_restore(noreclaim_flag);
5673 }
5674 }
5675 /* Handle the remaining sublist */
5676 if (!list_empty(head))
5677 __netif_receive_skb_list_core(head, pfmemalloc);
5678 /* Restore pflags */
5679 if (pfmemalloc)
5680 memalloc_noreclaim_restore(noreclaim_flag);
5681}
5682
5683static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5684{
5685 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5686 struct bpf_prog *new = xdp->prog;
5687 int ret = 0;
5688
5689 switch (xdp->command) {
5690 case XDP_SETUP_PROG:
5691 rcu_assign_pointer(dev->xdp_prog, new);
5692 if (old)
5693 bpf_prog_put(old);
5694
5695 if (old && !new) {
5696 static_branch_dec(&generic_xdp_needed_key);
5697 } else if (new && !old) {
5698 static_branch_inc(&generic_xdp_needed_key);
5699 dev_disable_lro(dev);
5700 dev_disable_gro_hw(dev);
5701 }
5702 break;
5703
5704 default:
5705 ret = -EINVAL;
5706 break;
5707 }
5708
5709 return ret;
5710}
5711
5712static int netif_receive_skb_internal(struct sk_buff *skb)
5713{
5714 int ret;
5715
5716 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5717
5718 if (skb_defer_rx_timestamp(skb))
5719 return NET_RX_SUCCESS;
5720
5721 rcu_read_lock();
5722#ifdef CONFIG_RPS
5723 if (static_branch_unlikely(&rps_needed)) {
5724 struct rps_dev_flow voidflow, *rflow = &voidflow;
5725 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5726
5727 if (cpu >= 0) {
5728 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5729 rcu_read_unlock();
5730 return ret;
5731 }
5732 }
5733#endif
5734 ret = __netif_receive_skb(skb);
5735 rcu_read_unlock();
5736 return ret;
5737}
5738
5739void netif_receive_skb_list_internal(struct list_head *head)
5740{
5741 struct sk_buff *skb, *next;
5742 struct list_head sublist;
5743
5744 INIT_LIST_HEAD(&sublist);
5745 list_for_each_entry_safe(skb, next, head, list) {
5746 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5747 skb_list_del_init(skb);
5748 if (!skb_defer_rx_timestamp(skb))
5749 list_add_tail(&skb->list, &sublist);
5750 }
5751 list_splice_init(&sublist, head);
5752
5753 rcu_read_lock();
5754#ifdef CONFIG_RPS
5755 if (static_branch_unlikely(&rps_needed)) {
5756 list_for_each_entry_safe(skb, next, head, list) {
5757 struct rps_dev_flow voidflow, *rflow = &voidflow;
5758 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5759
5760 if (cpu >= 0) {
5761 /* Will be handled, remove from list */
5762 skb_list_del_init(skb);
5763 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5764 }
5765 }
5766 }
5767#endif
5768 __netif_receive_skb_list(head);
5769 rcu_read_unlock();
5770}
5771
5772/**
5773 * netif_receive_skb - process receive buffer from network
5774 * @skb: buffer to process
5775 *
5776 * netif_receive_skb() is the main receive data processing function.
5777 * It always succeeds. The buffer may be dropped during processing
5778 * for congestion control or by the protocol layers.
5779 *
5780 * This function may only be called from softirq context and interrupts
5781 * should be enabled.
5782 *
5783 * Return values (usually ignored):
5784 * NET_RX_SUCCESS: no congestion
5785 * NET_RX_DROP: packet was dropped
5786 */
5787int netif_receive_skb(struct sk_buff *skb)
5788{
5789 int ret;
5790
5791 trace_netif_receive_skb_entry(skb);
5792
5793 ret = netif_receive_skb_internal(skb);
5794 trace_netif_receive_skb_exit(ret);
5795
5796 return ret;
5797}
5798EXPORT_SYMBOL(netif_receive_skb);
5799
5800/**
5801 * netif_receive_skb_list - process many receive buffers from network
5802 * @head: list of skbs to process.
5803 *
5804 * Since return value of netif_receive_skb() is normally ignored, and
5805 * wouldn't be meaningful for a list, this function returns void.
5806 *
5807 * This function may only be called from softirq context and interrupts
5808 * should be enabled.
5809 */
5810void netif_receive_skb_list(struct list_head *head)
5811{
5812 struct sk_buff *skb;
5813
5814 if (list_empty(head))
5815 return;
5816 if (trace_netif_receive_skb_list_entry_enabled()) {
5817 list_for_each_entry(skb, head, list)
5818 trace_netif_receive_skb_list_entry(skb);
5819 }
5820 netif_receive_skb_list_internal(head);
5821 trace_netif_receive_skb_list_exit(0);
5822}
5823EXPORT_SYMBOL(netif_receive_skb_list);
5824
5825static DEFINE_PER_CPU(struct work_struct, flush_works);
5826
5827/* Network device is going away, flush any packets still pending */
5828static void flush_backlog(struct work_struct *work)
5829{
5830 struct sk_buff *skb, *tmp;
5831 struct softnet_data *sd;
5832
5833 local_bh_disable();
5834 sd = this_cpu_ptr(&softnet_data);
5835
5836 rps_lock_irq_disable(sd);
5837 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5838 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5839 __skb_unlink(skb, &sd->input_pkt_queue);
5840 dev_kfree_skb_irq(skb);
5841 input_queue_head_incr(sd);
5842 }
5843 }
5844 rps_unlock_irq_enable(sd);
5845
5846 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5847 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5848 __skb_unlink(skb, &sd->process_queue);
5849 kfree_skb(skb);
5850 input_queue_head_incr(sd);
5851 }
5852 }
5853 local_bh_enable();
5854}
5855
5856static bool flush_required(int cpu)
5857{
5858#if IS_ENABLED(CONFIG_RPS)
5859 struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5860 bool do_flush;
5861
5862 rps_lock_irq_disable(sd);
5863
5864 /* as insertion into process_queue happens with the rps lock held,
5865 * process_queue access may race only with dequeue
5866 */
5867 do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
5868 !skb_queue_empty_lockless(&sd->process_queue);
5869 rps_unlock_irq_enable(sd);
5870
5871 return do_flush;
5872#endif
5873 /* without RPS we can't safely check input_pkt_queue: during a
5874 * concurrent remote skb_queue_splice() we can detect as empty both
5875 * input_pkt_queue and process_queue even if the latter could end-up
5876 * containing a lot of packets.
5877 */
5878 return true;
5879}
5880
5881static void flush_all_backlogs(void)
5882{
5883 static cpumask_t flush_cpus;
5884 unsigned int cpu;
5885
5886 /* since we are under rtnl lock protection we can use static data
5887 * for the cpumask and avoid allocating on stack the possibly
5888 * large mask
5889 */
5890 ASSERT_RTNL();
5891
5892 cpus_read_lock();
5893
5894 cpumask_clear(&flush_cpus);
5895 for_each_online_cpu(cpu) {
5896 if (flush_required(cpu)) {
5897 queue_work_on(cpu, system_highpri_wq,
5898 per_cpu_ptr(&flush_works, cpu));
5899 cpumask_set_cpu(cpu, &flush_cpus);
5900 }
5901 }
5902
5903 /* we can have in flight packet[s] on the cpus we are not flushing,
5904 * synchronize_net() in unregister_netdevice_many() will take care of
5905 * them
5906 */
5907 for_each_cpu(cpu, &flush_cpus)
5908 flush_work(per_cpu_ptr(&flush_works, cpu));
5909
5910 cpus_read_unlock();
5911}
5912
5913static void net_rps_send_ipi(struct softnet_data *remsd)
5914{
5915#ifdef CONFIG_RPS
5916 while (remsd) {
5917 struct softnet_data *next = remsd->rps_ipi_next;
5918
5919 if (cpu_online(remsd->cpu))
5920 smp_call_function_single_async(remsd->cpu, &remsd->csd);
5921 remsd = next;
5922 }
5923#endif
5924}
5925
5926/*
5927 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5928 * Note: called with local irq disabled, but exits with local irq enabled.
5929 */
5930static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5931{
5932#ifdef CONFIG_RPS
5933 struct softnet_data *remsd = sd->rps_ipi_list;
5934
5935 if (remsd) {
5936 sd->rps_ipi_list = NULL;
5937
5938 local_irq_enable();
5939
5940 /* Send pending IPI's to kick RPS processing on remote cpus. */
5941 net_rps_send_ipi(remsd);
5942 } else
5943#endif
5944 local_irq_enable();
5945}
5946
5947static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5948{
5949#ifdef CONFIG_RPS
5950 return sd->rps_ipi_list != NULL;
5951#else
5952 return false;
5953#endif
5954}
5955
5956static int process_backlog(struct napi_struct *napi, int quota)
5957{
5958 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5959 bool again = true;
5960 int work = 0;
5961
5962 /* Check if we have pending ipi, its better to send them now,
5963 * not waiting net_rx_action() end.
5964 */
5965 if (sd_has_rps_ipi_waiting(sd)) {
5966 local_irq_disable();
5967 net_rps_action_and_irq_enable(sd);
5968 }
5969
5970 napi->weight = READ_ONCE(dev_rx_weight);
5971 while (again) {
5972 struct sk_buff *skb;
5973
5974 while ((skb = __skb_dequeue(&sd->process_queue))) {
5975 rcu_read_lock();
5976 __netif_receive_skb(skb);
5977 rcu_read_unlock();
5978 input_queue_head_incr(sd);
5979 if (++work >= quota)
5980 return work;
5981
5982 }
5983
5984 rps_lock_irq_disable(sd);
5985 if (skb_queue_empty(&sd->input_pkt_queue)) {
5986 /*
5987 * Inline a custom version of __napi_complete().
5988 * only current cpu owns and manipulates this napi,
5989 * and NAPI_STATE_SCHED is the only possible flag set
5990 * on backlog.
5991 * We can use a plain write instead of clear_bit(),
5992 * and we dont need an smp_mb() memory barrier.
5993 */
5994 napi->state = 0;
5995 again = false;
5996 } else {
5997 skb_queue_splice_tail_init(&sd->input_pkt_queue,
5998 &sd->process_queue);
5999 }
6000 rps_unlock_irq_enable(sd);
6001 }
6002
6003 return work;
6004}
6005
6006/**
6007 * __napi_schedule - schedule for receive
6008 * @n: entry to schedule
6009 *
6010 * The entry's receive function will be scheduled to run.
6011 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
6012 */
6013void __napi_schedule(struct napi_struct *n)
6014{
6015 unsigned long flags;
6016
6017 local_irq_save(flags);
6018 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6019 local_irq_restore(flags);
6020}
6021EXPORT_SYMBOL(__napi_schedule);
6022
6023/**
6024 * napi_schedule_prep - check if napi can be scheduled
6025 * @n: napi context
6026 *
6027 * Test if NAPI routine is already running, and if not mark
6028 * it as running. This is used as a condition variable to
6029 * insure only one NAPI poll instance runs. We also make
6030 * sure there is no pending NAPI disable.
6031 */
6032bool napi_schedule_prep(struct napi_struct *n)
6033{
6034 unsigned long new, val = READ_ONCE(n->state);
6035
6036 do {
6037 if (unlikely(val & NAPIF_STATE_DISABLE))
6038 return false;
6039 new = val | NAPIF_STATE_SCHED;
6040
6041 /* Sets STATE_MISSED bit if STATE_SCHED was already set
6042 * This was suggested by Alexander Duyck, as compiler
6043 * emits better code than :
6044 * if (val & NAPIF_STATE_SCHED)
6045 * new |= NAPIF_STATE_MISSED;
6046 */
6047 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6048 NAPIF_STATE_MISSED;
6049 } while (!try_cmpxchg(&n->state, &val, new));
6050
6051 return !(val & NAPIF_STATE_SCHED);
6052}
6053EXPORT_SYMBOL(napi_schedule_prep);
6054
6055/**
6056 * __napi_schedule_irqoff - schedule for receive
6057 * @n: entry to schedule
6058 *
6059 * Variant of __napi_schedule() assuming hard irqs are masked.
6060 *
6061 * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6062 * because the interrupt disabled assumption might not be true
6063 * due to force-threaded interrupts and spinlock substitution.
6064 */
6065void __napi_schedule_irqoff(struct napi_struct *n)
6066{
6067 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6068 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6069 else
6070 __napi_schedule(n);
6071}
6072EXPORT_SYMBOL(__napi_schedule_irqoff);
6073
6074bool napi_complete_done(struct napi_struct *n, int work_done)
6075{
6076 unsigned long flags, val, new, timeout = 0;
6077 bool ret = true;
6078
6079 /*
6080 * 1) Don't let napi dequeue from the cpu poll list
6081 * just in case its running on a different cpu.
6082 * 2) If we are busy polling, do nothing here, we have
6083 * the guarantee we will be called later.
6084 */
6085 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6086 NAPIF_STATE_IN_BUSY_POLL)))
6087 return false;
6088
6089 if (work_done) {
6090 if (n->gro_bitmask)
6091 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6092 n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
6093 }
6094 if (n->defer_hard_irqs_count > 0) {
6095 n->defer_hard_irqs_count--;
6096 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6097 if (timeout)
6098 ret = false;
6099 }
6100 if (n->gro_bitmask) {
6101 /* When the NAPI instance uses a timeout and keeps postponing
6102 * it, we need to bound somehow the time packets are kept in
6103 * the GRO layer
6104 */
6105 napi_gro_flush(n, !!timeout);
6106 }
6107
6108 gro_normal_list(n);
6109
6110 if (unlikely(!list_empty(&n->poll_list))) {
6111 /* If n->poll_list is not empty, we need to mask irqs */
6112 local_irq_save(flags);
6113 list_del_init(&n->poll_list);
6114 local_irq_restore(flags);
6115 }
6116 WRITE_ONCE(n->list_owner, -1);
6117
6118 val = READ_ONCE(n->state);
6119 do {
6120 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6121
6122 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
6123 NAPIF_STATE_SCHED_THREADED |
6124 NAPIF_STATE_PREFER_BUSY_POLL);
6125
6126 /* If STATE_MISSED was set, leave STATE_SCHED set,
6127 * because we will call napi->poll() one more time.
6128 * This C code was suggested by Alexander Duyck to help gcc.
6129 */
6130 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6131 NAPIF_STATE_SCHED;
6132 } while (!try_cmpxchg(&n->state, &val, new));
6133
6134 if (unlikely(val & NAPIF_STATE_MISSED)) {
6135 __napi_schedule(n);
6136 return false;
6137 }
6138
6139 if (timeout)
6140 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6141 HRTIMER_MODE_REL_PINNED);
6142 return ret;
6143}
6144EXPORT_SYMBOL(napi_complete_done);
6145
6146/* must be called under rcu_read_lock(), as we dont take a reference */
6147struct napi_struct *napi_by_id(unsigned int napi_id)
6148{
6149 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6150 struct napi_struct *napi;
6151
6152 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6153 if (napi->napi_id == napi_id)
6154 return napi;
6155
6156 return NULL;
6157}
6158
6159#if defined(CONFIG_NET_RX_BUSY_POLL)
6160
6161static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
6162{
6163 if (!skip_schedule) {
6164 gro_normal_list(napi);
6165 __napi_schedule(napi);
6166 return;
6167 }
6168
6169 if (napi->gro_bitmask) {
6170 /* flush too old packets
6171 * If HZ < 1000, flush all packets.
6172 */
6173 napi_gro_flush(napi, HZ >= 1000);
6174 }
6175
6176 gro_normal_list(napi);
6177 clear_bit(NAPI_STATE_SCHED, &napi->state);
6178}
6179
6180static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
6181 u16 budget)
6182{
6183 bool skip_schedule = false;
6184 unsigned long timeout;
6185 int rc;
6186
6187 /* Busy polling means there is a high chance device driver hard irq
6188 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6189 * set in napi_schedule_prep().
6190 * Since we are about to call napi->poll() once more, we can safely
6191 * clear NAPI_STATE_MISSED.
6192 *
6193 * Note: x86 could use a single "lock and ..." instruction
6194 * to perform these two clear_bit()
6195 */
6196 clear_bit(NAPI_STATE_MISSED, &napi->state);
6197 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6198
6199 local_bh_disable();
6200
6201 if (prefer_busy_poll) {
6202 napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
6203 timeout = READ_ONCE(napi->dev->gro_flush_timeout);
6204 if (napi->defer_hard_irqs_count && timeout) {
6205 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6206 skip_schedule = true;
6207 }
6208 }
6209
6210 /* All we really want here is to re-enable device interrupts.
6211 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6212 */
6213 rc = napi->poll(napi, budget);
6214 /* We can't gro_normal_list() here, because napi->poll() might have
6215 * rearmed the napi (napi_complete_done()) in which case it could
6216 * already be running on another CPU.
6217 */
6218 trace_napi_poll(napi, rc, budget);
6219 netpoll_poll_unlock(have_poll_lock);
6220 if (rc == budget)
6221 __busy_poll_stop(napi, skip_schedule);
6222 local_bh_enable();
6223}
6224
6225void napi_busy_loop(unsigned int napi_id,
6226 bool (*loop_end)(void *, unsigned long),
6227 void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6228{
6229 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6230 int (*napi_poll)(struct napi_struct *napi, int budget);
6231 void *have_poll_lock = NULL;
6232 struct napi_struct *napi;
6233
6234restart:
6235 napi_poll = NULL;
6236
6237 rcu_read_lock();
6238
6239 napi = napi_by_id(napi_id);
6240 if (!napi)
6241 goto out;
6242
6243 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6244 preempt_disable();
6245 for (;;) {
6246 int work = 0;
6247
6248 local_bh_disable();
6249 if (!napi_poll) {
6250 unsigned long val = READ_ONCE(napi->state);
6251
6252 /* If multiple threads are competing for this napi,
6253 * we avoid dirtying napi->state as much as we can.
6254 */
6255 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6256 NAPIF_STATE_IN_BUSY_POLL)) {
6257 if (prefer_busy_poll)
6258 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6259 goto count;
6260 }
6261 if (cmpxchg(&napi->state, val,
6262 val | NAPIF_STATE_IN_BUSY_POLL |
6263 NAPIF_STATE_SCHED) != val) {
6264 if (prefer_busy_poll)
6265 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6266 goto count;
6267 }
6268 have_poll_lock = netpoll_poll_lock(napi);
6269 napi_poll = napi->poll;
6270 }
6271 work = napi_poll(napi, budget);
6272 trace_napi_poll(napi, work, budget);
6273 gro_normal_list(napi);
6274count:
6275 if (work > 0)
6276 __NET_ADD_STATS(dev_net(napi->dev),
6277 LINUX_MIB_BUSYPOLLRXPACKETS, work);
6278 local_bh_enable();
6279
6280 if (!loop_end || loop_end(loop_end_arg, start_time))
6281 break;
6282
6283 if (unlikely(need_resched())) {
6284 if (napi_poll)
6285 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6286 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6287 preempt_enable();
6288 rcu_read_unlock();
6289 cond_resched();
6290 if (loop_end(loop_end_arg, start_time))
6291 return;
6292 goto restart;
6293 }
6294 cpu_relax();
6295 }
6296 if (napi_poll)
6297 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6298 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6299 preempt_enable();
6300out:
6301 rcu_read_unlock();
6302}
6303EXPORT_SYMBOL(napi_busy_loop);
6304
6305#endif /* CONFIG_NET_RX_BUSY_POLL */
6306
6307static void napi_hash_add(struct napi_struct *napi)
6308{
6309 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
6310 return;
6311
6312 spin_lock(&napi_hash_lock);
6313
6314 /* 0..NR_CPUS range is reserved for sender_cpu use */
6315 do {
6316 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6317 napi_gen_id = MIN_NAPI_ID;
6318 } while (napi_by_id(napi_gen_id));
6319 napi->napi_id = napi_gen_id;
6320
6321 hlist_add_head_rcu(&napi->napi_hash_node,
6322 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
6323
6324 spin_unlock(&napi_hash_lock);
6325}
6326
6327/* Warning : caller is responsible to make sure rcu grace period
6328 * is respected before freeing memory containing @napi
6329 */
6330static void napi_hash_del(struct napi_struct *napi)
6331{
6332 spin_lock(&napi_hash_lock);
6333
6334 hlist_del_init_rcu(&napi->napi_hash_node);
6335
6336 spin_unlock(&napi_hash_lock);
6337}
6338
6339static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6340{
6341 struct napi_struct *napi;
6342
6343 napi = container_of(timer, struct napi_struct, timer);
6344
6345 /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6346 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6347 */
6348 if (!napi_disable_pending(napi) &&
6349 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6350 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6351 __napi_schedule_irqoff(napi);
6352 }
6353
6354 return HRTIMER_NORESTART;
6355}
6356
6357static void init_gro_hash(struct napi_struct *napi)
6358{
6359 int i;
6360
6361 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6362 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6363 napi->gro_hash[i].count = 0;
6364 }
6365 napi->gro_bitmask = 0;
6366}
6367
6368int dev_set_threaded(struct net_device *dev, bool threaded)
6369{
6370 struct napi_struct *napi;
6371 int err = 0;
6372
6373 if (dev->threaded == threaded)
6374 return 0;
6375
6376 if (threaded) {
6377 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6378 if (!napi->thread) {
6379 err = napi_kthread_create(napi);
6380 if (err) {
6381 threaded = false;
6382 break;
6383 }
6384 }
6385 }
6386 }
6387
6388 dev->threaded = threaded;
6389
6390 /* Make sure kthread is created before THREADED bit
6391 * is set.
6392 */
6393 smp_mb__before_atomic();
6394
6395 /* Setting/unsetting threaded mode on a napi might not immediately
6396 * take effect, if the current napi instance is actively being
6397 * polled. In this case, the switch between threaded mode and
6398 * softirq mode will happen in the next round of napi_schedule().
6399 * This should not cause hiccups/stalls to the live traffic.
6400 */
6401 list_for_each_entry(napi, &dev->napi_list, dev_list)
6402 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded);
6403
6404 return err;
6405}
6406EXPORT_SYMBOL(dev_set_threaded);
6407
6408/**
6409 * netif_queue_set_napi - Associate queue with the napi
6410 * @dev: device to which NAPI and queue belong
6411 * @queue_index: Index of queue
6412 * @type: queue type as RX or TX
6413 * @napi: NAPI context, pass NULL to clear previously set NAPI
6414 *
6415 * Set queue with its corresponding napi context. This should be done after
6416 * registering the NAPI handler for the queue-vector and the queues have been
6417 * mapped to the corresponding interrupt vector.
6418 */
6419void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
6420 enum netdev_queue_type type, struct napi_struct *napi)
6421{
6422 struct netdev_rx_queue *rxq;
6423 struct netdev_queue *txq;
6424
6425 if (WARN_ON_ONCE(napi && !napi->dev))
6426 return;
6427 if (dev->reg_state >= NETREG_REGISTERED)
6428 ASSERT_RTNL();
6429
6430 switch (type) {
6431 case NETDEV_QUEUE_TYPE_RX:
6432 rxq = __netif_get_rx_queue(dev, queue_index);
6433 rxq->napi = napi;
6434 return;
6435 case NETDEV_QUEUE_TYPE_TX:
6436 txq = netdev_get_tx_queue(dev, queue_index);
6437 txq->napi = napi;
6438 return;
6439 default:
6440 return;
6441 }
6442}
6443EXPORT_SYMBOL(netif_queue_set_napi);
6444
6445void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
6446 int (*poll)(struct napi_struct *, int), int weight)
6447{
6448 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6449 return;
6450
6451 INIT_LIST_HEAD(&napi->poll_list);
6452 INIT_HLIST_NODE(&napi->napi_hash_node);
6453 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6454 napi->timer.function = napi_watchdog;
6455 init_gro_hash(napi);
6456 napi->skb = NULL;
6457 INIT_LIST_HEAD(&napi->rx_list);
6458 napi->rx_count = 0;
6459 napi->poll = poll;
6460 if (weight > NAPI_POLL_WEIGHT)
6461 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6462 weight);
6463 napi->weight = weight;
6464 napi->dev = dev;
6465#ifdef CONFIG_NETPOLL
6466 napi->poll_owner = -1;
6467#endif
6468 napi->list_owner = -1;
6469 set_bit(NAPI_STATE_SCHED, &napi->state);
6470 set_bit(NAPI_STATE_NPSVC, &napi->state);
6471 list_add_rcu(&napi->dev_list, &dev->napi_list);
6472 napi_hash_add(napi);
6473 napi_get_frags_check(napi);
6474 /* Create kthread for this napi if dev->threaded is set.
6475 * Clear dev->threaded if kthread creation failed so that
6476 * threaded mode will not be enabled in napi_enable().
6477 */
6478 if (dev->threaded && napi_kthread_create(napi))
6479 dev->threaded = 0;
6480 netif_napi_set_irq(napi, -1);
6481}
6482EXPORT_SYMBOL(netif_napi_add_weight);
6483
6484void napi_disable(struct napi_struct *n)
6485{
6486 unsigned long val, new;
6487
6488 might_sleep();
6489 set_bit(NAPI_STATE_DISABLE, &n->state);
6490
6491 val = READ_ONCE(n->state);
6492 do {
6493 while (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) {
6494 usleep_range(20, 200);
6495 val = READ_ONCE(n->state);
6496 }
6497
6498 new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC;
6499 new &= ~(NAPIF_STATE_THREADED | NAPIF_STATE_PREFER_BUSY_POLL);
6500 } while (!try_cmpxchg(&n->state, &val, new));
6501
6502 hrtimer_cancel(&n->timer);
6503
6504 clear_bit(NAPI_STATE_DISABLE, &n->state);
6505}
6506EXPORT_SYMBOL(napi_disable);
6507
6508/**
6509 * napi_enable - enable NAPI scheduling
6510 * @n: NAPI context
6511 *
6512 * Resume NAPI from being scheduled on this context.
6513 * Must be paired with napi_disable.
6514 */
6515void napi_enable(struct napi_struct *n)
6516{
6517 unsigned long new, val = READ_ONCE(n->state);
6518
6519 do {
6520 BUG_ON(!test_bit(NAPI_STATE_SCHED, &val));
6521
6522 new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC);
6523 if (n->dev->threaded && n->thread)
6524 new |= NAPIF_STATE_THREADED;
6525 } while (!try_cmpxchg(&n->state, &val, new));
6526}
6527EXPORT_SYMBOL(napi_enable);
6528
6529static void flush_gro_hash(struct napi_struct *napi)
6530{
6531 int i;
6532
6533 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6534 struct sk_buff *skb, *n;
6535
6536 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
6537 kfree_skb(skb);
6538 napi->gro_hash[i].count = 0;
6539 }
6540}
6541
6542/* Must be called in process context */
6543void __netif_napi_del(struct napi_struct *napi)
6544{
6545 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
6546 return;
6547
6548 napi_hash_del(napi);
6549 list_del_rcu(&napi->dev_list);
6550 napi_free_frags(napi);
6551
6552 flush_gro_hash(napi);
6553 napi->gro_bitmask = 0;
6554
6555 if (napi->thread) {
6556 kthread_stop(napi->thread);
6557 napi->thread = NULL;
6558 }
6559}
6560EXPORT_SYMBOL(__netif_napi_del);
6561
6562static int __napi_poll(struct napi_struct *n, bool *repoll)
6563{
6564 int work, weight;
6565
6566 weight = n->weight;
6567
6568 /* This NAPI_STATE_SCHED test is for avoiding a race
6569 * with netpoll's poll_napi(). Only the entity which
6570 * obtains the lock and sees NAPI_STATE_SCHED set will
6571 * actually make the ->poll() call. Therefore we avoid
6572 * accidentally calling ->poll() when NAPI is not scheduled.
6573 */
6574 work = 0;
6575 if (napi_is_scheduled(n)) {
6576 work = n->poll(n, weight);
6577 trace_napi_poll(n, work, weight);
6578
6579 xdp_do_check_flushed(n);
6580 }
6581
6582 if (unlikely(work > weight))
6583 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
6584 n->poll, work, weight);
6585
6586 if (likely(work < weight))
6587 return work;
6588
6589 /* Drivers must not modify the NAPI state if they
6590 * consume the entire weight. In such cases this code
6591 * still "owns" the NAPI instance and therefore can
6592 * move the instance around on the list at-will.
6593 */
6594 if (unlikely(napi_disable_pending(n))) {
6595 napi_complete(n);
6596 return work;
6597 }
6598
6599 /* The NAPI context has more processing work, but busy-polling
6600 * is preferred. Exit early.
6601 */
6602 if (napi_prefer_busy_poll(n)) {
6603 if (napi_complete_done(n, work)) {
6604 /* If timeout is not set, we need to make sure
6605 * that the NAPI is re-scheduled.
6606 */
6607 napi_schedule(n);
6608 }
6609 return work;
6610 }
6611
6612 if (n->gro_bitmask) {
6613 /* flush too old packets
6614 * If HZ < 1000, flush all packets.
6615 */
6616 napi_gro_flush(n, HZ >= 1000);
6617 }
6618
6619 gro_normal_list(n);
6620
6621 /* Some drivers may have called napi_schedule
6622 * prior to exhausting their budget.
6623 */
6624 if (unlikely(!list_empty(&n->poll_list))) {
6625 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6626 n->dev ? n->dev->name : "backlog");
6627 return work;
6628 }
6629
6630 *repoll = true;
6631
6632 return work;
6633}
6634
6635static int napi_poll(struct napi_struct *n, struct list_head *repoll)
6636{
6637 bool do_repoll = false;
6638 void *have;
6639 int work;
6640
6641 list_del_init(&n->poll_list);
6642
6643 have = netpoll_poll_lock(n);
6644
6645 work = __napi_poll(n, &do_repoll);
6646
6647 if (do_repoll)
6648 list_add_tail(&n->poll_list, repoll);
6649
6650 netpoll_poll_unlock(have);
6651
6652 return work;
6653}
6654
6655static int napi_thread_wait(struct napi_struct *napi)
6656{
6657 bool woken = false;
6658
6659 set_current_state(TASK_INTERRUPTIBLE);
6660
6661 while (!kthread_should_stop()) {
6662 /* Testing SCHED_THREADED bit here to make sure the current
6663 * kthread owns this napi and could poll on this napi.
6664 * Testing SCHED bit is not enough because SCHED bit might be
6665 * set by some other busy poll thread or by napi_disable().
6666 */
6667 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) {
6668 WARN_ON(!list_empty(&napi->poll_list));
6669 __set_current_state(TASK_RUNNING);
6670 return 0;
6671 }
6672
6673 schedule();
6674 /* woken being true indicates this thread owns this napi. */
6675 woken = true;
6676 set_current_state(TASK_INTERRUPTIBLE);
6677 }
6678 __set_current_state(TASK_RUNNING);
6679
6680 return -1;
6681}
6682
6683static void skb_defer_free_flush(struct softnet_data *sd)
6684{
6685 struct sk_buff *skb, *next;
6686
6687 /* Paired with WRITE_ONCE() in skb_attempt_defer_free() */
6688 if (!READ_ONCE(sd->defer_list))
6689 return;
6690
6691 spin_lock(&sd->defer_lock);
6692 skb = sd->defer_list;
6693 sd->defer_list = NULL;
6694 sd->defer_count = 0;
6695 spin_unlock(&sd->defer_lock);
6696
6697 while (skb != NULL) {
6698 next = skb->next;
6699 napi_consume_skb(skb, 1);
6700 skb = next;
6701 }
6702}
6703
6704static int napi_threaded_poll(void *data)
6705{
6706 struct napi_struct *napi = data;
6707 struct softnet_data *sd;
6708 void *have;
6709
6710 while (!napi_thread_wait(napi)) {
6711 for (;;) {
6712 bool repoll = false;
6713
6714 local_bh_disable();
6715 sd = this_cpu_ptr(&softnet_data);
6716 sd->in_napi_threaded_poll = true;
6717
6718 have = netpoll_poll_lock(napi);
6719 __napi_poll(napi, &repoll);
6720 netpoll_poll_unlock(have);
6721
6722 sd->in_napi_threaded_poll = false;
6723 barrier();
6724
6725 if (sd_has_rps_ipi_waiting(sd)) {
6726 local_irq_disable();
6727 net_rps_action_and_irq_enable(sd);
6728 }
6729 skb_defer_free_flush(sd);
6730 local_bh_enable();
6731
6732 if (!repoll)
6733 break;
6734
6735 cond_resched();
6736 }
6737 }
6738 return 0;
6739}
6740
6741static __latent_entropy void net_rx_action(struct softirq_action *h)
6742{
6743 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6744 unsigned long time_limit = jiffies +
6745 usecs_to_jiffies(READ_ONCE(netdev_budget_usecs));
6746 int budget = READ_ONCE(netdev_budget);
6747 LIST_HEAD(list);
6748 LIST_HEAD(repoll);
6749
6750start:
6751 sd->in_net_rx_action = true;
6752 local_irq_disable();
6753 list_splice_init(&sd->poll_list, &list);
6754 local_irq_enable();
6755
6756 for (;;) {
6757 struct napi_struct *n;
6758
6759 skb_defer_free_flush(sd);
6760
6761 if (list_empty(&list)) {
6762 if (list_empty(&repoll)) {
6763 sd->in_net_rx_action = false;
6764 barrier();
6765 /* We need to check if ____napi_schedule()
6766 * had refilled poll_list while
6767 * sd->in_net_rx_action was true.
6768 */
6769 if (!list_empty(&sd->poll_list))
6770 goto start;
6771 if (!sd_has_rps_ipi_waiting(sd))
6772 goto end;
6773 }
6774 break;
6775 }
6776
6777 n = list_first_entry(&list, struct napi_struct, poll_list);
6778 budget -= napi_poll(n, &repoll);
6779
6780 /* If softirq window is exhausted then punt.
6781 * Allow this to run for 2 jiffies since which will allow
6782 * an average latency of 1.5/HZ.
6783 */
6784 if (unlikely(budget <= 0 ||
6785 time_after_eq(jiffies, time_limit))) {
6786 sd->time_squeeze++;
6787 break;
6788 }
6789 }
6790
6791 local_irq_disable();
6792
6793 list_splice_tail_init(&sd->poll_list, &list);
6794 list_splice_tail(&repoll, &list);
6795 list_splice(&list, &sd->poll_list);
6796 if (!list_empty(&sd->poll_list))
6797 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
6798 else
6799 sd->in_net_rx_action = false;
6800
6801 net_rps_action_and_irq_enable(sd);
6802end:;
6803}
6804
6805struct netdev_adjacent {
6806 struct net_device *dev;
6807 netdevice_tracker dev_tracker;
6808
6809 /* upper master flag, there can only be one master device per list */
6810 bool master;
6811
6812 /* lookup ignore flag */
6813 bool ignore;
6814
6815 /* counter for the number of times this device was added to us */
6816 u16 ref_nr;
6817
6818 /* private field for the users */
6819 void *private;
6820
6821 struct list_head list;
6822 struct rcu_head rcu;
6823};
6824
6825static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
6826 struct list_head *adj_list)
6827{
6828 struct netdev_adjacent *adj;
6829
6830 list_for_each_entry(adj, adj_list, list) {
6831 if (adj->dev == adj_dev)
6832 return adj;
6833 }
6834 return NULL;
6835}
6836
6837static int ____netdev_has_upper_dev(struct net_device *upper_dev,
6838 struct netdev_nested_priv *priv)
6839{
6840 struct net_device *dev = (struct net_device *)priv->data;
6841
6842 return upper_dev == dev;
6843}
6844
6845/**
6846 * netdev_has_upper_dev - Check if device is linked to an upper device
6847 * @dev: device
6848 * @upper_dev: upper device to check
6849 *
6850 * Find out if a device is linked to specified upper device and return true
6851 * in case it is. Note that this checks only immediate upper device,
6852 * not through a complete stack of devices. The caller must hold the RTNL lock.
6853 */
6854bool netdev_has_upper_dev(struct net_device *dev,
6855 struct net_device *upper_dev)
6856{
6857 struct netdev_nested_priv priv = {
6858 .data = (void *)upper_dev,
6859 };
6860
6861 ASSERT_RTNL();
6862
6863 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
6864 &priv);
6865}
6866EXPORT_SYMBOL(netdev_has_upper_dev);
6867
6868/**
6869 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
6870 * @dev: device
6871 * @upper_dev: upper device to check
6872 *
6873 * Find out if a device is linked to specified upper device and return true
6874 * in case it is. Note that this checks the entire upper device chain.
6875 * The caller must hold rcu lock.
6876 */
6877
6878bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
6879 struct net_device *upper_dev)
6880{
6881 struct netdev_nested_priv priv = {
6882 .data = (void *)upper_dev,
6883 };
6884
6885 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
6886 &priv);
6887}
6888EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
6889
6890/**
6891 * netdev_has_any_upper_dev - Check if device is linked to some device
6892 * @dev: device
6893 *
6894 * Find out if a device is linked to an upper device and return true in case
6895 * it is. The caller must hold the RTNL lock.
6896 */
6897bool netdev_has_any_upper_dev(struct net_device *dev)
6898{
6899 ASSERT_RTNL();
6900
6901 return !list_empty(&dev->adj_list.upper);
6902}
6903EXPORT_SYMBOL(netdev_has_any_upper_dev);
6904
6905/**
6906 * netdev_master_upper_dev_get - Get master upper device
6907 * @dev: device
6908 *
6909 * Find a master upper device and return pointer to it or NULL in case
6910 * it's not there. The caller must hold the RTNL lock.
6911 */
6912struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
6913{
6914 struct netdev_adjacent *upper;
6915
6916 ASSERT_RTNL();
6917
6918 if (list_empty(&dev->adj_list.upper))
6919 return NULL;
6920
6921 upper = list_first_entry(&dev->adj_list.upper,
6922 struct netdev_adjacent, list);
6923 if (likely(upper->master))
6924 return upper->dev;
6925 return NULL;
6926}
6927EXPORT_SYMBOL(netdev_master_upper_dev_get);
6928
6929static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
6930{
6931 struct netdev_adjacent *upper;
6932
6933 ASSERT_RTNL();
6934
6935 if (list_empty(&dev->adj_list.upper))
6936 return NULL;
6937
6938 upper = list_first_entry(&dev->adj_list.upper,
6939 struct netdev_adjacent, list);
6940 if (likely(upper->master) && !upper->ignore)
6941 return upper->dev;
6942 return NULL;
6943}
6944
6945/**
6946 * netdev_has_any_lower_dev - Check if device is linked to some device
6947 * @dev: device
6948 *
6949 * Find out if a device is linked to a lower device and return true in case
6950 * it is. The caller must hold the RTNL lock.
6951 */
6952static bool netdev_has_any_lower_dev(struct net_device *dev)
6953{
6954 ASSERT_RTNL();
6955
6956 return !list_empty(&dev->adj_list.lower);
6957}
6958
6959void *netdev_adjacent_get_private(struct list_head *adj_list)
6960{
6961 struct netdev_adjacent *adj;
6962
6963 adj = list_entry(adj_list, struct netdev_adjacent, list);
6964
6965 return adj->private;
6966}
6967EXPORT_SYMBOL(netdev_adjacent_get_private);
6968
6969/**
6970 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
6971 * @dev: device
6972 * @iter: list_head ** of the current position
6973 *
6974 * Gets the next device from the dev's upper list, starting from iter
6975 * position. The caller must hold RCU read lock.
6976 */
6977struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
6978 struct list_head **iter)
6979{
6980 struct netdev_adjacent *upper;
6981
6982 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6983
6984 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6985
6986 if (&upper->list == &dev->adj_list.upper)
6987 return NULL;
6988
6989 *iter = &upper->list;
6990
6991 return upper->dev;
6992}
6993EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
6994
6995static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
6996 struct list_head **iter,
6997 bool *ignore)
6998{
6999 struct netdev_adjacent *upper;
7000
7001 upper = list_entry((*iter)->next, struct netdev_adjacent, list);
7002
7003 if (&upper->list == &dev->adj_list.upper)
7004 return NULL;
7005
7006 *iter = &upper->list;
7007 *ignore = upper->ignore;
7008
7009 return upper->dev;
7010}
7011
7012static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
7013 struct list_head **iter)
7014{
7015 struct netdev_adjacent *upper;
7016
7017 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7018
7019 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7020
7021 if (&upper->list == &dev->adj_list.upper)
7022 return NULL;
7023
7024 *iter = &upper->list;
7025
7026 return upper->dev;
7027}
7028
7029static int __netdev_walk_all_upper_dev(struct net_device *dev,
7030 int (*fn)(struct net_device *dev,
7031 struct netdev_nested_priv *priv),
7032 struct netdev_nested_priv *priv)
7033{
7034 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7035 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7036 int ret, cur = 0;
7037 bool ignore;
7038
7039 now = dev;
7040 iter = &dev->adj_list.upper;
7041
7042 while (1) {
7043 if (now != dev) {
7044 ret = fn(now, priv);
7045 if (ret)
7046 return ret;
7047 }
7048
7049 next = NULL;
7050 while (1) {
7051 udev = __netdev_next_upper_dev(now, &iter, &ignore);
7052 if (!udev)
7053 break;
7054 if (ignore)
7055 continue;
7056
7057 next = udev;
7058 niter = &udev->adj_list.upper;
7059 dev_stack[cur] = now;
7060 iter_stack[cur++] = iter;
7061 break;
7062 }
7063
7064 if (!next) {
7065 if (!cur)
7066 return 0;
7067 next = dev_stack[--cur];
7068 niter = iter_stack[cur];
7069 }
7070
7071 now = next;
7072 iter = niter;
7073 }
7074
7075 return 0;
7076}
7077
7078int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
7079 int (*fn)(struct net_device *dev,
7080 struct netdev_nested_priv *priv),
7081 struct netdev_nested_priv *priv)
7082{
7083 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7084 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7085 int ret, cur = 0;
7086
7087 now = dev;
7088 iter = &dev->adj_list.upper;
7089
7090 while (1) {
7091 if (now != dev) {
7092 ret = fn(now, priv);
7093 if (ret)
7094 return ret;
7095 }
7096
7097 next = NULL;
7098 while (1) {
7099 udev = netdev_next_upper_dev_rcu(now, &iter);
7100 if (!udev)
7101 break;
7102
7103 next = udev;
7104 niter = &udev->adj_list.upper;
7105 dev_stack[cur] = now;
7106 iter_stack[cur++] = iter;
7107 break;
7108 }
7109
7110 if (!next) {
7111 if (!cur)
7112 return 0;
7113 next = dev_stack[--cur];
7114 niter = iter_stack[cur];
7115 }
7116
7117 now = next;
7118 iter = niter;
7119 }
7120
7121 return 0;
7122}
7123EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
7124
7125static bool __netdev_has_upper_dev(struct net_device *dev,
7126 struct net_device *upper_dev)
7127{
7128 struct netdev_nested_priv priv = {
7129 .flags = 0,
7130 .data = (void *)upper_dev,
7131 };
7132
7133 ASSERT_RTNL();
7134
7135 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
7136 &priv);
7137}
7138
7139/**
7140 * netdev_lower_get_next_private - Get the next ->private from the
7141 * lower neighbour list
7142 * @dev: device
7143 * @iter: list_head ** of the current position
7144 *
7145 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7146 * list, starting from iter position. The caller must hold either hold the
7147 * RTNL lock or its own locking that guarantees that the neighbour lower
7148 * list will remain unchanged.
7149 */
7150void *netdev_lower_get_next_private(struct net_device *dev,
7151 struct list_head **iter)
7152{
7153 struct netdev_adjacent *lower;
7154
7155 lower = list_entry(*iter, struct netdev_adjacent, list);
7156
7157 if (&lower->list == &dev->adj_list.lower)
7158 return NULL;
7159
7160 *iter = lower->list.next;
7161
7162 return lower->private;
7163}
7164EXPORT_SYMBOL(netdev_lower_get_next_private);
7165
7166/**
7167 * netdev_lower_get_next_private_rcu - Get the next ->private from the
7168 * lower neighbour list, RCU
7169 * variant
7170 * @dev: device
7171 * @iter: list_head ** of the current position
7172 *
7173 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7174 * list, starting from iter position. The caller must hold RCU read lock.
7175 */
7176void *netdev_lower_get_next_private_rcu(struct net_device *dev,
7177 struct list_head **iter)
7178{
7179 struct netdev_adjacent *lower;
7180
7181 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
7182
7183 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7184
7185 if (&lower->list == &dev->adj_list.lower)
7186 return NULL;
7187
7188 *iter = &lower->list;
7189
7190 return lower->private;
7191}
7192EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
7193
7194/**
7195 * netdev_lower_get_next - Get the next device from the lower neighbour
7196 * list
7197 * @dev: device
7198 * @iter: list_head ** of the current position
7199 *
7200 * Gets the next netdev_adjacent from the dev's lower neighbour
7201 * list, starting from iter position. The caller must hold RTNL lock or
7202 * its own locking that guarantees that the neighbour lower
7203 * list will remain unchanged.
7204 */
7205void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
7206{
7207 struct netdev_adjacent *lower;
7208
7209 lower = list_entry(*iter, struct netdev_adjacent, list);
7210
7211 if (&lower->list == &dev->adj_list.lower)
7212 return NULL;
7213
7214 *iter = lower->list.next;
7215
7216 return lower->dev;
7217}
7218EXPORT_SYMBOL(netdev_lower_get_next);
7219
7220static struct net_device *netdev_next_lower_dev(struct net_device *dev,
7221 struct list_head **iter)
7222{
7223 struct netdev_adjacent *lower;
7224
7225 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7226
7227 if (&lower->list == &dev->adj_list.lower)
7228 return NULL;
7229
7230 *iter = &lower->list;
7231
7232 return lower->dev;
7233}
7234
7235static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
7236 struct list_head **iter,
7237 bool *ignore)
7238{
7239 struct netdev_adjacent *lower;
7240
7241 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7242
7243 if (&lower->list == &dev->adj_list.lower)
7244 return NULL;
7245
7246 *iter = &lower->list;
7247 *ignore = lower->ignore;
7248
7249 return lower->dev;
7250}
7251
7252int netdev_walk_all_lower_dev(struct net_device *dev,
7253 int (*fn)(struct net_device *dev,
7254 struct netdev_nested_priv *priv),
7255 struct netdev_nested_priv *priv)
7256{
7257 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7258 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7259 int ret, cur = 0;
7260
7261 now = dev;
7262 iter = &dev->adj_list.lower;
7263
7264 while (1) {
7265 if (now != dev) {
7266 ret = fn(now, priv);
7267 if (ret)
7268 return ret;
7269 }
7270
7271 next = NULL;
7272 while (1) {
7273 ldev = netdev_next_lower_dev(now, &iter);
7274 if (!ldev)
7275 break;
7276
7277 next = ldev;
7278 niter = &ldev->adj_list.lower;
7279 dev_stack[cur] = now;
7280 iter_stack[cur++] = iter;
7281 break;
7282 }
7283
7284 if (!next) {
7285 if (!cur)
7286 return 0;
7287 next = dev_stack[--cur];
7288 niter = iter_stack[cur];
7289 }
7290
7291 now = next;
7292 iter = niter;
7293 }
7294
7295 return 0;
7296}
7297EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
7298
7299static int __netdev_walk_all_lower_dev(struct net_device *dev,
7300 int (*fn)(struct net_device *dev,
7301 struct netdev_nested_priv *priv),
7302 struct netdev_nested_priv *priv)
7303{
7304 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7305 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7306 int ret, cur = 0;
7307 bool ignore;
7308
7309 now = dev;
7310 iter = &dev->adj_list.lower;
7311
7312 while (1) {
7313 if (now != dev) {
7314 ret = fn(now, priv);
7315 if (ret)
7316 return ret;
7317 }
7318
7319 next = NULL;
7320 while (1) {
7321 ldev = __netdev_next_lower_dev(now, &iter, &ignore);
7322 if (!ldev)
7323 break;
7324 if (ignore)
7325 continue;
7326
7327 next = ldev;
7328 niter = &ldev->adj_list.lower;
7329 dev_stack[cur] = now;
7330 iter_stack[cur++] = iter;
7331 break;
7332 }
7333
7334 if (!next) {
7335 if (!cur)
7336 return 0;
7337 next = dev_stack[--cur];
7338 niter = iter_stack[cur];
7339 }
7340
7341 now = next;
7342 iter = niter;
7343 }
7344
7345 return 0;
7346}
7347
7348struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
7349 struct list_head **iter)
7350{
7351 struct netdev_adjacent *lower;
7352
7353 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7354 if (&lower->list == &dev->adj_list.lower)
7355 return NULL;
7356
7357 *iter = &lower->list;
7358
7359 return lower->dev;
7360}
7361EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
7362
7363static u8 __netdev_upper_depth(struct net_device *dev)
7364{
7365 struct net_device *udev;
7366 struct list_head *iter;
7367 u8 max_depth = 0;
7368 bool ignore;
7369
7370 for (iter = &dev->adj_list.upper,
7371 udev = __netdev_next_upper_dev(dev, &iter, &ignore);
7372 udev;
7373 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
7374 if (ignore)
7375 continue;
7376 if (max_depth < udev->upper_level)
7377 max_depth = udev->upper_level;
7378 }
7379
7380 return max_depth;
7381}
7382
7383static u8 __netdev_lower_depth(struct net_device *dev)
7384{
7385 struct net_device *ldev;
7386 struct list_head *iter;
7387 u8 max_depth = 0;
7388 bool ignore;
7389
7390 for (iter = &dev->adj_list.lower,
7391 ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
7392 ldev;
7393 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
7394 if (ignore)
7395 continue;
7396 if (max_depth < ldev->lower_level)
7397 max_depth = ldev->lower_level;
7398 }
7399
7400 return max_depth;
7401}
7402
7403static int __netdev_update_upper_level(struct net_device *dev,
7404 struct netdev_nested_priv *__unused)
7405{
7406 dev->upper_level = __netdev_upper_depth(dev) + 1;
7407 return 0;
7408}
7409
7410#ifdef CONFIG_LOCKDEP
7411static LIST_HEAD(net_unlink_list);
7412
7413static void net_unlink_todo(struct net_device *dev)
7414{
7415 if (list_empty(&dev->unlink_list))
7416 list_add_tail(&dev->unlink_list, &net_unlink_list);
7417}
7418#endif
7419
7420static int __netdev_update_lower_level(struct net_device *dev,
7421 struct netdev_nested_priv *priv)
7422{
7423 dev->lower_level = __netdev_lower_depth(dev) + 1;
7424
7425#ifdef CONFIG_LOCKDEP
7426 if (!priv)
7427 return 0;
7428
7429 if (priv->flags & NESTED_SYNC_IMM)
7430 dev->nested_level = dev->lower_level - 1;
7431 if (priv->flags & NESTED_SYNC_TODO)
7432 net_unlink_todo(dev);
7433#endif
7434 return 0;
7435}
7436
7437int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
7438 int (*fn)(struct net_device *dev,
7439 struct netdev_nested_priv *priv),
7440 struct netdev_nested_priv *priv)
7441{
7442 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7443 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7444 int ret, cur = 0;
7445
7446 now = dev;
7447 iter = &dev->adj_list.lower;
7448
7449 while (1) {
7450 if (now != dev) {
7451 ret = fn(now, priv);
7452 if (ret)
7453 return ret;
7454 }
7455
7456 next = NULL;
7457 while (1) {
7458 ldev = netdev_next_lower_dev_rcu(now, &iter);
7459 if (!ldev)
7460 break;
7461
7462 next = ldev;
7463 niter = &ldev->adj_list.lower;
7464 dev_stack[cur] = now;
7465 iter_stack[cur++] = iter;
7466 break;
7467 }
7468
7469 if (!next) {
7470 if (!cur)
7471 return 0;
7472 next = dev_stack[--cur];
7473 niter = iter_stack[cur];
7474 }
7475
7476 now = next;
7477 iter = niter;
7478 }
7479
7480 return 0;
7481}
7482EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
7483
7484/**
7485 * netdev_lower_get_first_private_rcu - Get the first ->private from the
7486 * lower neighbour list, RCU
7487 * variant
7488 * @dev: device
7489 *
7490 * Gets the first netdev_adjacent->private from the dev's lower neighbour
7491 * list. The caller must hold RCU read lock.
7492 */
7493void *netdev_lower_get_first_private_rcu(struct net_device *dev)
7494{
7495 struct netdev_adjacent *lower;
7496
7497 lower = list_first_or_null_rcu(&dev->adj_list.lower,
7498 struct netdev_adjacent, list);
7499 if (lower)
7500 return lower->private;
7501 return NULL;
7502}
7503EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
7504
7505/**
7506 * netdev_master_upper_dev_get_rcu - Get master upper device
7507 * @dev: device
7508 *
7509 * Find a master upper device and return pointer to it or NULL in case
7510 * it's not there. The caller must hold the RCU read lock.
7511 */
7512struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
7513{
7514 struct netdev_adjacent *upper;
7515
7516 upper = list_first_or_null_rcu(&dev->adj_list.upper,
7517 struct netdev_adjacent, list);
7518 if (upper && likely(upper->master))
7519 return upper->dev;
7520 return NULL;
7521}
7522EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
7523
7524static int netdev_adjacent_sysfs_add(struct net_device *dev,
7525 struct net_device *adj_dev,
7526 struct list_head *dev_list)
7527{
7528 char linkname[IFNAMSIZ+7];
7529
7530 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7531 "upper_%s" : "lower_%s", adj_dev->name);
7532 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
7533 linkname);
7534}
7535static void netdev_adjacent_sysfs_del(struct net_device *dev,
7536 char *name,
7537 struct list_head *dev_list)
7538{
7539 char linkname[IFNAMSIZ+7];
7540
7541 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7542 "upper_%s" : "lower_%s", name);
7543 sysfs_remove_link(&(dev->dev.kobj), linkname);
7544}
7545
7546static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
7547 struct net_device *adj_dev,
7548 struct list_head *dev_list)
7549{
7550 return (dev_list == &dev->adj_list.upper ||
7551 dev_list == &dev->adj_list.lower) &&
7552 net_eq(dev_net(dev), dev_net(adj_dev));
7553}
7554
7555static int __netdev_adjacent_dev_insert(struct net_device *dev,
7556 struct net_device *adj_dev,
7557 struct list_head *dev_list,
7558 void *private, bool master)
7559{
7560 struct netdev_adjacent *adj;
7561 int ret;
7562
7563 adj = __netdev_find_adj(adj_dev, dev_list);
7564
7565 if (adj) {
7566 adj->ref_nr += 1;
7567 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
7568 dev->name, adj_dev->name, adj->ref_nr);
7569
7570 return 0;
7571 }
7572
7573 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
7574 if (!adj)
7575 return -ENOMEM;
7576
7577 adj->dev = adj_dev;
7578 adj->master = master;
7579 adj->ref_nr = 1;
7580 adj->private = private;
7581 adj->ignore = false;
7582 netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL);
7583
7584 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
7585 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
7586
7587 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
7588 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
7589 if (ret)
7590 goto free_adj;
7591 }
7592
7593 /* Ensure that master link is always the first item in list. */
7594 if (master) {
7595 ret = sysfs_create_link(&(dev->dev.kobj),
7596 &(adj_dev->dev.kobj), "master");
7597 if (ret)
7598 goto remove_symlinks;
7599
7600 list_add_rcu(&adj->list, dev_list);
7601 } else {
7602 list_add_tail_rcu(&adj->list, dev_list);
7603 }
7604
7605 return 0;
7606
7607remove_symlinks:
7608 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7609 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7610free_adj:
7611 netdev_put(adj_dev, &adj->dev_tracker);
7612 kfree(adj);
7613
7614 return ret;
7615}
7616
7617static void __netdev_adjacent_dev_remove(struct net_device *dev,
7618 struct net_device *adj_dev,
7619 u16 ref_nr,
7620 struct list_head *dev_list)
7621{
7622 struct netdev_adjacent *adj;
7623
7624 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
7625 dev->name, adj_dev->name, ref_nr);
7626
7627 adj = __netdev_find_adj(adj_dev, dev_list);
7628
7629 if (!adj) {
7630 pr_err("Adjacency does not exist for device %s from %s\n",
7631 dev->name, adj_dev->name);
7632 WARN_ON(1);
7633 return;
7634 }
7635
7636 if (adj->ref_nr > ref_nr) {
7637 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
7638 dev->name, adj_dev->name, ref_nr,
7639 adj->ref_nr - ref_nr);
7640 adj->ref_nr -= ref_nr;
7641 return;
7642 }
7643
7644 if (adj->master)
7645 sysfs_remove_link(&(dev->dev.kobj), "master");
7646
7647 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7648 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7649
7650 list_del_rcu(&adj->list);
7651 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
7652 adj_dev->name, dev->name, adj_dev->name);
7653 netdev_put(adj_dev, &adj->dev_tracker);
7654 kfree_rcu(adj, rcu);
7655}
7656
7657static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
7658 struct net_device *upper_dev,
7659 struct list_head *up_list,
7660 struct list_head *down_list,
7661 void *private, bool master)
7662{
7663 int ret;
7664
7665 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
7666 private, master);
7667 if (ret)
7668 return ret;
7669
7670 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
7671 private, false);
7672 if (ret) {
7673 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
7674 return ret;
7675 }
7676
7677 return 0;
7678}
7679
7680static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
7681 struct net_device *upper_dev,
7682 u16 ref_nr,
7683 struct list_head *up_list,
7684 struct list_head *down_list)
7685{
7686 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
7687 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
7688}
7689
7690static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
7691 struct net_device *upper_dev,
7692 void *private, bool master)
7693{
7694 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
7695 &dev->adj_list.upper,
7696 &upper_dev->adj_list.lower,
7697 private, master);
7698}
7699
7700static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
7701 struct net_device *upper_dev)
7702{
7703 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
7704 &dev->adj_list.upper,
7705 &upper_dev->adj_list.lower);
7706}
7707
7708static int __netdev_upper_dev_link(struct net_device *dev,
7709 struct net_device *upper_dev, bool master,
7710 void *upper_priv, void *upper_info,
7711 struct netdev_nested_priv *priv,
7712 struct netlink_ext_ack *extack)
7713{
7714 struct netdev_notifier_changeupper_info changeupper_info = {
7715 .info = {
7716 .dev = dev,
7717 .extack = extack,
7718 },
7719 .upper_dev = upper_dev,
7720 .master = master,
7721 .linking = true,
7722 .upper_info = upper_info,
7723 };
7724 struct net_device *master_dev;
7725 int ret = 0;
7726
7727 ASSERT_RTNL();
7728
7729 if (dev == upper_dev)
7730 return -EBUSY;
7731
7732 /* To prevent loops, check if dev is not upper device to upper_dev. */
7733 if (__netdev_has_upper_dev(upper_dev, dev))
7734 return -EBUSY;
7735
7736 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
7737 return -EMLINK;
7738
7739 if (!master) {
7740 if (__netdev_has_upper_dev(dev, upper_dev))
7741 return -EEXIST;
7742 } else {
7743 master_dev = __netdev_master_upper_dev_get(dev);
7744 if (master_dev)
7745 return master_dev == upper_dev ? -EEXIST : -EBUSY;
7746 }
7747
7748 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
7749 &changeupper_info.info);
7750 ret = notifier_to_errno(ret);
7751 if (ret)
7752 return ret;
7753
7754 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
7755 master);
7756 if (ret)
7757 return ret;
7758
7759 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7760 &changeupper_info.info);
7761 ret = notifier_to_errno(ret);
7762 if (ret)
7763 goto rollback;
7764
7765 __netdev_update_upper_level(dev, NULL);
7766 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7767
7768 __netdev_update_lower_level(upper_dev, priv);
7769 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
7770 priv);
7771
7772 return 0;
7773
7774rollback:
7775 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7776
7777 return ret;
7778}
7779
7780/**
7781 * netdev_upper_dev_link - Add a link to the upper device
7782 * @dev: device
7783 * @upper_dev: new upper device
7784 * @extack: netlink extended ack
7785 *
7786 * Adds a link to device which is upper to this one. The caller must hold
7787 * the RTNL lock. On a failure a negative errno code is returned.
7788 * On success the reference counts are adjusted and the function
7789 * returns zero.
7790 */
7791int netdev_upper_dev_link(struct net_device *dev,
7792 struct net_device *upper_dev,
7793 struct netlink_ext_ack *extack)
7794{
7795 struct netdev_nested_priv priv = {
7796 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7797 .data = NULL,
7798 };
7799
7800 return __netdev_upper_dev_link(dev, upper_dev, false,
7801 NULL, NULL, &priv, extack);
7802}
7803EXPORT_SYMBOL(netdev_upper_dev_link);
7804
7805/**
7806 * netdev_master_upper_dev_link - Add a master link to the upper device
7807 * @dev: device
7808 * @upper_dev: new upper device
7809 * @upper_priv: upper device private
7810 * @upper_info: upper info to be passed down via notifier
7811 * @extack: netlink extended ack
7812 *
7813 * Adds a link to device which is upper to this one. In this case, only
7814 * one master upper device can be linked, although other non-master devices
7815 * might be linked as well. The caller must hold the RTNL lock.
7816 * On a failure a negative errno code is returned. On success the reference
7817 * counts are adjusted and the function returns zero.
7818 */
7819int netdev_master_upper_dev_link(struct net_device *dev,
7820 struct net_device *upper_dev,
7821 void *upper_priv, void *upper_info,
7822 struct netlink_ext_ack *extack)
7823{
7824 struct netdev_nested_priv priv = {
7825 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7826 .data = NULL,
7827 };
7828
7829 return __netdev_upper_dev_link(dev, upper_dev, true,
7830 upper_priv, upper_info, &priv, extack);
7831}
7832EXPORT_SYMBOL(netdev_master_upper_dev_link);
7833
7834static void __netdev_upper_dev_unlink(struct net_device *dev,
7835 struct net_device *upper_dev,
7836 struct netdev_nested_priv *priv)
7837{
7838 struct netdev_notifier_changeupper_info changeupper_info = {
7839 .info = {
7840 .dev = dev,
7841 },
7842 .upper_dev = upper_dev,
7843 .linking = false,
7844 };
7845
7846 ASSERT_RTNL();
7847
7848 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
7849
7850 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
7851 &changeupper_info.info);
7852
7853 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7854
7855 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7856 &changeupper_info.info);
7857
7858 __netdev_update_upper_level(dev, NULL);
7859 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7860
7861 __netdev_update_lower_level(upper_dev, priv);
7862 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
7863 priv);
7864}
7865
7866/**
7867 * netdev_upper_dev_unlink - Removes a link to upper device
7868 * @dev: device
7869 * @upper_dev: new upper device
7870 *
7871 * Removes a link to device which is upper to this one. The caller must hold
7872 * the RTNL lock.
7873 */
7874void netdev_upper_dev_unlink(struct net_device *dev,
7875 struct net_device *upper_dev)
7876{
7877 struct netdev_nested_priv priv = {
7878 .flags = NESTED_SYNC_TODO,
7879 .data = NULL,
7880 };
7881
7882 __netdev_upper_dev_unlink(dev, upper_dev, &priv);
7883}
7884EXPORT_SYMBOL(netdev_upper_dev_unlink);
7885
7886static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
7887 struct net_device *lower_dev,
7888 bool val)
7889{
7890 struct netdev_adjacent *adj;
7891
7892 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
7893 if (adj)
7894 adj->ignore = val;
7895
7896 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
7897 if (adj)
7898 adj->ignore = val;
7899}
7900
7901static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
7902 struct net_device *lower_dev)
7903{
7904 __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
7905}
7906
7907static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
7908 struct net_device *lower_dev)
7909{
7910 __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
7911}
7912
7913int netdev_adjacent_change_prepare(struct net_device *old_dev,
7914 struct net_device *new_dev,
7915 struct net_device *dev,
7916 struct netlink_ext_ack *extack)
7917{
7918 struct netdev_nested_priv priv = {
7919 .flags = 0,
7920 .data = NULL,
7921 };
7922 int err;
7923
7924 if (!new_dev)
7925 return 0;
7926
7927 if (old_dev && new_dev != old_dev)
7928 netdev_adjacent_dev_disable(dev, old_dev);
7929 err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
7930 extack);
7931 if (err) {
7932 if (old_dev && new_dev != old_dev)
7933 netdev_adjacent_dev_enable(dev, old_dev);
7934 return err;
7935 }
7936
7937 return 0;
7938}
7939EXPORT_SYMBOL(netdev_adjacent_change_prepare);
7940
7941void netdev_adjacent_change_commit(struct net_device *old_dev,
7942 struct net_device *new_dev,
7943 struct net_device *dev)
7944{
7945 struct netdev_nested_priv priv = {
7946 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7947 .data = NULL,
7948 };
7949
7950 if (!new_dev || !old_dev)
7951 return;
7952
7953 if (new_dev == old_dev)
7954 return;
7955
7956 netdev_adjacent_dev_enable(dev, old_dev);
7957 __netdev_upper_dev_unlink(old_dev, dev, &priv);
7958}
7959EXPORT_SYMBOL(netdev_adjacent_change_commit);
7960
7961void netdev_adjacent_change_abort(struct net_device *old_dev,
7962 struct net_device *new_dev,
7963 struct net_device *dev)
7964{
7965 struct netdev_nested_priv priv = {
7966 .flags = 0,
7967 .data = NULL,
7968 };
7969
7970 if (!new_dev)
7971 return;
7972
7973 if (old_dev && new_dev != old_dev)
7974 netdev_adjacent_dev_enable(dev, old_dev);
7975
7976 __netdev_upper_dev_unlink(new_dev, dev, &priv);
7977}
7978EXPORT_SYMBOL(netdev_adjacent_change_abort);
7979
7980/**
7981 * netdev_bonding_info_change - Dispatch event about slave change
7982 * @dev: device
7983 * @bonding_info: info to dispatch
7984 *
7985 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
7986 * The caller must hold the RTNL lock.
7987 */
7988void netdev_bonding_info_change(struct net_device *dev,
7989 struct netdev_bonding_info *bonding_info)
7990{
7991 struct netdev_notifier_bonding_info info = {
7992 .info.dev = dev,
7993 };
7994
7995 memcpy(&info.bonding_info, bonding_info,
7996 sizeof(struct netdev_bonding_info));
7997 call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
7998 &info.info);
7999}
8000EXPORT_SYMBOL(netdev_bonding_info_change);
8001
8002static int netdev_offload_xstats_enable_l3(struct net_device *dev,
8003 struct netlink_ext_ack *extack)
8004{
8005 struct netdev_notifier_offload_xstats_info info = {
8006 .info.dev = dev,
8007 .info.extack = extack,
8008 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8009 };
8010 int err;
8011 int rc;
8012
8013 dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3),
8014 GFP_KERNEL);
8015 if (!dev->offload_xstats_l3)
8016 return -ENOMEM;
8017
8018 rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE,
8019 NETDEV_OFFLOAD_XSTATS_DISABLE,
8020 &info.info);
8021 err = notifier_to_errno(rc);
8022 if (err)
8023 goto free_stats;
8024
8025 return 0;
8026
8027free_stats:
8028 kfree(dev->offload_xstats_l3);
8029 dev->offload_xstats_l3 = NULL;
8030 return err;
8031}
8032
8033int netdev_offload_xstats_enable(struct net_device *dev,
8034 enum netdev_offload_xstats_type type,
8035 struct netlink_ext_ack *extack)
8036{
8037 ASSERT_RTNL();
8038
8039 if (netdev_offload_xstats_enabled(dev, type))
8040 return -EALREADY;
8041
8042 switch (type) {
8043 case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8044 return netdev_offload_xstats_enable_l3(dev, extack);
8045 }
8046
8047 WARN_ON(1);
8048 return -EINVAL;
8049}
8050EXPORT_SYMBOL(netdev_offload_xstats_enable);
8051
8052static void netdev_offload_xstats_disable_l3(struct net_device *dev)
8053{
8054 struct netdev_notifier_offload_xstats_info info = {
8055 .info.dev = dev,
8056 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8057 };
8058
8059 call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE,
8060 &info.info);
8061 kfree(dev->offload_xstats_l3);
8062 dev->offload_xstats_l3 = NULL;
8063}
8064
8065int netdev_offload_xstats_disable(struct net_device *dev,
8066 enum netdev_offload_xstats_type type)
8067{
8068 ASSERT_RTNL();
8069
8070 if (!netdev_offload_xstats_enabled(dev, type))
8071 return -EALREADY;
8072
8073 switch (type) {
8074 case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8075 netdev_offload_xstats_disable_l3(dev);
8076 return 0;
8077 }
8078
8079 WARN_ON(1);
8080 return -EINVAL;
8081}
8082EXPORT_SYMBOL(netdev_offload_xstats_disable);
8083
8084static void netdev_offload_xstats_disable_all(struct net_device *dev)
8085{
8086 netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
8087}
8088
8089static struct rtnl_hw_stats64 *
8090netdev_offload_xstats_get_ptr(const struct net_device *dev,
8091 enum netdev_offload_xstats_type type)
8092{
8093 switch (type) {
8094 case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8095 return dev->offload_xstats_l3;
8096 }
8097
8098 WARN_ON(1);
8099 return NULL;
8100}
8101
8102bool netdev_offload_xstats_enabled(const struct net_device *dev,
8103 enum netdev_offload_xstats_type type)
8104{
8105 ASSERT_RTNL();
8106
8107 return netdev_offload_xstats_get_ptr(dev, type);
8108}
8109EXPORT_SYMBOL(netdev_offload_xstats_enabled);
8110
8111struct netdev_notifier_offload_xstats_ru {
8112 bool used;
8113};
8114
8115struct netdev_notifier_offload_xstats_rd {
8116 struct rtnl_hw_stats64 stats;
8117 bool used;
8118};
8119
8120static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest,
8121 const struct rtnl_hw_stats64 *src)
8122{
8123 dest->rx_packets += src->rx_packets;
8124 dest->tx_packets += src->tx_packets;
8125 dest->rx_bytes += src->rx_bytes;
8126 dest->tx_bytes += src->tx_bytes;
8127 dest->rx_errors += src->rx_errors;
8128 dest->tx_errors += src->tx_errors;
8129 dest->rx_dropped += src->rx_dropped;
8130 dest->tx_dropped += src->tx_dropped;
8131 dest->multicast += src->multicast;
8132}
8133
8134static int netdev_offload_xstats_get_used(struct net_device *dev,
8135 enum netdev_offload_xstats_type type,
8136 bool *p_used,
8137 struct netlink_ext_ack *extack)
8138{
8139 struct netdev_notifier_offload_xstats_ru report_used = {};
8140 struct netdev_notifier_offload_xstats_info info = {
8141 .info.dev = dev,
8142 .info.extack = extack,
8143 .type = type,
8144 .report_used = &report_used,
8145 };
8146 int rc;
8147
8148 WARN_ON(!netdev_offload_xstats_enabled(dev, type));
8149 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED,
8150 &info.info);
8151 *p_used = report_used.used;
8152 return notifier_to_errno(rc);
8153}
8154
8155static int netdev_offload_xstats_get_stats(struct net_device *dev,
8156 enum netdev_offload_xstats_type type,
8157 struct rtnl_hw_stats64 *p_stats,
8158 bool *p_used,
8159 struct netlink_ext_ack *extack)
8160{
8161 struct netdev_notifier_offload_xstats_rd report_delta = {};
8162 struct netdev_notifier_offload_xstats_info info = {
8163 .info.dev = dev,
8164 .info.extack = extack,
8165 .type = type,
8166 .report_delta = &report_delta,
8167 };
8168 struct rtnl_hw_stats64 *stats;
8169 int rc;
8170
8171 stats = netdev_offload_xstats_get_ptr(dev, type);
8172 if (WARN_ON(!stats))
8173 return -EINVAL;
8174
8175 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
8176 &info.info);
8177
8178 /* Cache whatever we got, even if there was an error, otherwise the
8179 * successful stats retrievals would get lost.
8180 */
8181 netdev_hw_stats64_add(stats, &report_delta.stats);
8182
8183 if (p_stats)
8184 *p_stats = *stats;
8185 *p_used = report_delta.used;
8186
8187 return notifier_to_errno(rc);
8188}
8189
8190int netdev_offload_xstats_get(struct net_device *dev,
8191 enum netdev_offload_xstats_type type,
8192 struct rtnl_hw_stats64 *p_stats, bool *p_used,
8193 struct netlink_ext_ack *extack)
8194{
8195 ASSERT_RTNL();
8196
8197 if (p_stats)
8198 return netdev_offload_xstats_get_stats(dev, type, p_stats,
8199 p_used, extack);
8200 else
8201 return netdev_offload_xstats_get_used(dev, type, p_used,
8202 extack);
8203}
8204EXPORT_SYMBOL(netdev_offload_xstats_get);
8205
8206void
8207netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta,
8208 const struct rtnl_hw_stats64 *stats)
8209{
8210 report_delta->used = true;
8211 netdev_hw_stats64_add(&report_delta->stats, stats);
8212}
8213EXPORT_SYMBOL(netdev_offload_xstats_report_delta);
8214
8215void
8216netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used)
8217{
8218 report_used->used = true;
8219}
8220EXPORT_SYMBOL(netdev_offload_xstats_report_used);
8221
8222void netdev_offload_xstats_push_delta(struct net_device *dev,
8223 enum netdev_offload_xstats_type type,
8224 const struct rtnl_hw_stats64 *p_stats)
8225{
8226 struct rtnl_hw_stats64 *stats;
8227
8228 ASSERT_RTNL();
8229
8230 stats = netdev_offload_xstats_get_ptr(dev, type);
8231 if (WARN_ON(!stats))
8232 return;
8233
8234 netdev_hw_stats64_add(stats, p_stats);
8235}
8236EXPORT_SYMBOL(netdev_offload_xstats_push_delta);
8237
8238/**
8239 * netdev_get_xmit_slave - Get the xmit slave of master device
8240 * @dev: device
8241 * @skb: The packet
8242 * @all_slaves: assume all the slaves are active
8243 *
8244 * The reference counters are not incremented so the caller must be
8245 * careful with locks. The caller must hold RCU lock.
8246 * %NULL is returned if no slave is found.
8247 */
8248
8249struct net_device *netdev_get_xmit_slave(struct net_device *dev,
8250 struct sk_buff *skb,
8251 bool all_slaves)
8252{
8253 const struct net_device_ops *ops = dev->netdev_ops;
8254
8255 if (!ops->ndo_get_xmit_slave)
8256 return NULL;
8257 return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
8258}
8259EXPORT_SYMBOL(netdev_get_xmit_slave);
8260
8261static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
8262 struct sock *sk)
8263{
8264 const struct net_device_ops *ops = dev->netdev_ops;
8265
8266 if (!ops->ndo_sk_get_lower_dev)
8267 return NULL;
8268 return ops->ndo_sk_get_lower_dev(dev, sk);
8269}
8270
8271/**
8272 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8273 * @dev: device
8274 * @sk: the socket
8275 *
8276 * %NULL is returned if no lower device is found.
8277 */
8278
8279struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
8280 struct sock *sk)
8281{
8282 struct net_device *lower;
8283
8284 lower = netdev_sk_get_lower_dev(dev, sk);
8285 while (lower) {
8286 dev = lower;
8287 lower = netdev_sk_get_lower_dev(dev, sk);
8288 }
8289
8290 return dev;
8291}
8292EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
8293
8294static void netdev_adjacent_add_links(struct net_device *dev)
8295{
8296 struct netdev_adjacent *iter;
8297
8298 struct net *net = dev_net(dev);
8299
8300 list_for_each_entry(iter, &dev->adj_list.upper, list) {
8301 if (!net_eq(net, dev_net(iter->dev)))
8302 continue;
8303 netdev_adjacent_sysfs_add(iter->dev, dev,
8304 &iter->dev->adj_list.lower);
8305 netdev_adjacent_sysfs_add(dev, iter->dev,
8306 &dev->adj_list.upper);
8307 }
8308
8309 list_for_each_entry(iter, &dev->adj_list.lower, list) {
8310 if (!net_eq(net, dev_net(iter->dev)))
8311 continue;
8312 netdev_adjacent_sysfs_add(iter->dev, dev,
8313 &iter->dev->adj_list.upper);
8314 netdev_adjacent_sysfs_add(dev, iter->dev,
8315 &dev->adj_list.lower);
8316 }
8317}
8318
8319static void netdev_adjacent_del_links(struct net_device *dev)
8320{
8321 struct netdev_adjacent *iter;
8322
8323 struct net *net = dev_net(dev);
8324
8325 list_for_each_entry(iter, &dev->adj_list.upper, list) {
8326 if (!net_eq(net, dev_net(iter->dev)))
8327 continue;
8328 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8329 &iter->dev->adj_list.lower);
8330 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8331 &dev->adj_list.upper);
8332 }
8333
8334 list_for_each_entry(iter, &dev->adj_list.lower, list) {
8335 if (!net_eq(net, dev_net(iter->dev)))
8336 continue;
8337 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8338 &iter->dev->adj_list.upper);
8339 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8340 &dev->adj_list.lower);
8341 }
8342}
8343
8344void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
8345{
8346 struct netdev_adjacent *iter;
8347
8348 struct net *net = dev_net(dev);
8349
8350 list_for_each_entry(iter, &dev->adj_list.upper, list) {
8351 if (!net_eq(net, dev_net(iter->dev)))
8352 continue;
8353 netdev_adjacent_sysfs_del(iter->dev, oldname,
8354 &iter->dev->adj_list.lower);
8355 netdev_adjacent_sysfs_add(iter->dev, dev,
8356 &iter->dev->adj_list.lower);
8357 }
8358
8359 list_for_each_entry(iter, &dev->adj_list.lower, list) {
8360 if (!net_eq(net, dev_net(iter->dev)))
8361 continue;
8362 netdev_adjacent_sysfs_del(iter->dev, oldname,
8363 &iter->dev->adj_list.upper);
8364 netdev_adjacent_sysfs_add(iter->dev, dev,
8365 &iter->dev->adj_list.upper);
8366 }
8367}
8368
8369void *netdev_lower_dev_get_private(struct net_device *dev,
8370 struct net_device *lower_dev)
8371{
8372 struct netdev_adjacent *lower;
8373
8374 if (!lower_dev)
8375 return NULL;
8376 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
8377 if (!lower)
8378 return NULL;
8379
8380 return lower->private;
8381}
8382EXPORT_SYMBOL(netdev_lower_dev_get_private);
8383
8384
8385/**
8386 * netdev_lower_state_changed - Dispatch event about lower device state change
8387 * @lower_dev: device
8388 * @lower_state_info: state to dispatch
8389 *
8390 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
8391 * The caller must hold the RTNL lock.
8392 */
8393void netdev_lower_state_changed(struct net_device *lower_dev,
8394 void *lower_state_info)
8395{
8396 struct netdev_notifier_changelowerstate_info changelowerstate_info = {
8397 .info.dev = lower_dev,
8398 };
8399
8400 ASSERT_RTNL();
8401 changelowerstate_info.lower_state_info = lower_state_info;
8402 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
8403 &changelowerstate_info.info);
8404}
8405EXPORT_SYMBOL(netdev_lower_state_changed);
8406
8407static void dev_change_rx_flags(struct net_device *dev, int flags)
8408{
8409 const struct net_device_ops *ops = dev->netdev_ops;
8410
8411 if (ops->ndo_change_rx_flags)
8412 ops->ndo_change_rx_flags(dev, flags);
8413}
8414
8415static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
8416{
8417 unsigned int old_flags = dev->flags;
8418 kuid_t uid;
8419 kgid_t gid;
8420
8421 ASSERT_RTNL();
8422
8423 dev->flags |= IFF_PROMISC;
8424 dev->promiscuity += inc;
8425 if (dev->promiscuity == 0) {
8426 /*
8427 * Avoid overflow.
8428 * If inc causes overflow, untouch promisc and return error.
8429 */
8430 if (inc < 0)
8431 dev->flags &= ~IFF_PROMISC;
8432 else {
8433 dev->promiscuity -= inc;
8434 netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n");
8435 return -EOVERFLOW;
8436 }
8437 }
8438 if (dev->flags != old_flags) {
8439 netdev_info(dev, "%s promiscuous mode\n",
8440 dev->flags & IFF_PROMISC ? "entered" : "left");
8441 if (audit_enabled) {
8442 current_uid_gid(&uid, &gid);
8443 audit_log(audit_context(), GFP_ATOMIC,
8444 AUDIT_ANOM_PROMISCUOUS,
8445 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
8446 dev->name, (dev->flags & IFF_PROMISC),
8447 (old_flags & IFF_PROMISC),
8448 from_kuid(&init_user_ns, audit_get_loginuid(current)),
8449 from_kuid(&init_user_ns, uid),
8450 from_kgid(&init_user_ns, gid),
8451 audit_get_sessionid(current));
8452 }
8453
8454 dev_change_rx_flags(dev, IFF_PROMISC);
8455 }
8456 if (notify)
8457 __dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL);
8458 return 0;
8459}
8460
8461/**
8462 * dev_set_promiscuity - update promiscuity count on a device
8463 * @dev: device
8464 * @inc: modifier
8465 *
8466 * Add or remove promiscuity from a device. While the count in the device
8467 * remains above zero the interface remains promiscuous. Once it hits zero
8468 * the device reverts back to normal filtering operation. A negative inc
8469 * value is used to drop promiscuity on the device.
8470 * Return 0 if successful or a negative errno code on error.
8471 */
8472int dev_set_promiscuity(struct net_device *dev, int inc)
8473{
8474 unsigned int old_flags = dev->flags;
8475 int err;
8476
8477 err = __dev_set_promiscuity(dev, inc, true);
8478 if (err < 0)
8479 return err;
8480 if (dev->flags != old_flags)
8481 dev_set_rx_mode(dev);
8482 return err;
8483}
8484EXPORT_SYMBOL(dev_set_promiscuity);
8485
8486static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
8487{
8488 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
8489
8490 ASSERT_RTNL();
8491
8492 dev->flags |= IFF_ALLMULTI;
8493 dev->allmulti += inc;
8494 if (dev->allmulti == 0) {
8495 /*
8496 * Avoid overflow.
8497 * If inc causes overflow, untouch allmulti and return error.
8498 */
8499 if (inc < 0)
8500 dev->flags &= ~IFF_ALLMULTI;
8501 else {
8502 dev->allmulti -= inc;
8503 netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n");
8504 return -EOVERFLOW;
8505 }
8506 }
8507 if (dev->flags ^ old_flags) {
8508 netdev_info(dev, "%s allmulticast mode\n",
8509 dev->flags & IFF_ALLMULTI ? "entered" : "left");
8510 dev_change_rx_flags(dev, IFF_ALLMULTI);
8511 dev_set_rx_mode(dev);
8512 if (notify)
8513 __dev_notify_flags(dev, old_flags,
8514 dev->gflags ^ old_gflags, 0, NULL);
8515 }
8516 return 0;
8517}
8518
8519/**
8520 * dev_set_allmulti - update allmulti count on a device
8521 * @dev: device
8522 * @inc: modifier
8523 *
8524 * Add or remove reception of all multicast frames to a device. While the
8525 * count in the device remains above zero the interface remains listening
8526 * to all interfaces. Once it hits zero the device reverts back to normal
8527 * filtering operation. A negative @inc value is used to drop the counter
8528 * when releasing a resource needing all multicasts.
8529 * Return 0 if successful or a negative errno code on error.
8530 */
8531
8532int dev_set_allmulti(struct net_device *dev, int inc)
8533{
8534 return __dev_set_allmulti(dev, inc, true);
8535}
8536EXPORT_SYMBOL(dev_set_allmulti);
8537
8538/*
8539 * Upload unicast and multicast address lists to device and
8540 * configure RX filtering. When the device doesn't support unicast
8541 * filtering it is put in promiscuous mode while unicast addresses
8542 * are present.
8543 */
8544void __dev_set_rx_mode(struct net_device *dev)
8545{
8546 const struct net_device_ops *ops = dev->netdev_ops;
8547
8548 /* dev_open will call this function so the list will stay sane. */
8549 if (!(dev->flags&IFF_UP))
8550 return;
8551
8552 if (!netif_device_present(dev))
8553 return;
8554
8555 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
8556 /* Unicast addresses changes may only happen under the rtnl,
8557 * therefore calling __dev_set_promiscuity here is safe.
8558 */
8559 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
8560 __dev_set_promiscuity(dev, 1, false);
8561 dev->uc_promisc = true;
8562 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
8563 __dev_set_promiscuity(dev, -1, false);
8564 dev->uc_promisc = false;
8565 }
8566 }
8567
8568 if (ops->ndo_set_rx_mode)
8569 ops->ndo_set_rx_mode(dev);
8570}
8571
8572void dev_set_rx_mode(struct net_device *dev)
8573{
8574 netif_addr_lock_bh(dev);
8575 __dev_set_rx_mode(dev);
8576 netif_addr_unlock_bh(dev);
8577}
8578
8579/**
8580 * dev_get_flags - get flags reported to userspace
8581 * @dev: device
8582 *
8583 * Get the combination of flag bits exported through APIs to userspace.
8584 */
8585unsigned int dev_get_flags(const struct net_device *dev)
8586{
8587 unsigned int flags;
8588
8589 flags = (dev->flags & ~(IFF_PROMISC |
8590 IFF_ALLMULTI |
8591 IFF_RUNNING |
8592 IFF_LOWER_UP |
8593 IFF_DORMANT)) |
8594 (dev->gflags & (IFF_PROMISC |
8595 IFF_ALLMULTI));
8596
8597 if (netif_running(dev)) {
8598 if (netif_oper_up(dev))
8599 flags |= IFF_RUNNING;
8600 if (netif_carrier_ok(dev))
8601 flags |= IFF_LOWER_UP;
8602 if (netif_dormant(dev))
8603 flags |= IFF_DORMANT;
8604 }
8605
8606 return flags;
8607}
8608EXPORT_SYMBOL(dev_get_flags);
8609
8610int __dev_change_flags(struct net_device *dev, unsigned int flags,
8611 struct netlink_ext_ack *extack)
8612{
8613 unsigned int old_flags = dev->flags;
8614 int ret;
8615
8616 ASSERT_RTNL();
8617
8618 /*
8619 * Set the flags on our device.
8620 */
8621
8622 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
8623 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
8624 IFF_AUTOMEDIA)) |
8625 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
8626 IFF_ALLMULTI));
8627
8628 /*
8629 * Load in the correct multicast list now the flags have changed.
8630 */
8631
8632 if ((old_flags ^ flags) & IFF_MULTICAST)
8633 dev_change_rx_flags(dev, IFF_MULTICAST);
8634
8635 dev_set_rx_mode(dev);
8636
8637 /*
8638 * Have we downed the interface. We handle IFF_UP ourselves
8639 * according to user attempts to set it, rather than blindly
8640 * setting it.
8641 */
8642
8643 ret = 0;
8644 if ((old_flags ^ flags) & IFF_UP) {
8645 if (old_flags & IFF_UP)
8646 __dev_close(dev);
8647 else
8648 ret = __dev_open(dev, extack);
8649 }
8650
8651 if ((flags ^ dev->gflags) & IFF_PROMISC) {
8652 int inc = (flags & IFF_PROMISC) ? 1 : -1;
8653 unsigned int old_flags = dev->flags;
8654
8655 dev->gflags ^= IFF_PROMISC;
8656
8657 if (__dev_set_promiscuity(dev, inc, false) >= 0)
8658 if (dev->flags != old_flags)
8659 dev_set_rx_mode(dev);
8660 }
8661
8662 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
8663 * is important. Some (broken) drivers set IFF_PROMISC, when
8664 * IFF_ALLMULTI is requested not asking us and not reporting.
8665 */
8666 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
8667 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
8668
8669 dev->gflags ^= IFF_ALLMULTI;
8670 __dev_set_allmulti(dev, inc, false);
8671 }
8672
8673 return ret;
8674}
8675
8676void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
8677 unsigned int gchanges, u32 portid,
8678 const struct nlmsghdr *nlh)
8679{
8680 unsigned int changes = dev->flags ^ old_flags;
8681
8682 if (gchanges)
8683 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh);
8684
8685 if (changes & IFF_UP) {
8686 if (dev->flags & IFF_UP)
8687 call_netdevice_notifiers(NETDEV_UP, dev);
8688 else
8689 call_netdevice_notifiers(NETDEV_DOWN, dev);
8690 }
8691
8692 if (dev->flags & IFF_UP &&
8693 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
8694 struct netdev_notifier_change_info change_info = {
8695 .info = {
8696 .dev = dev,
8697 },
8698 .flags_changed = changes,
8699 };
8700
8701 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
8702 }
8703}
8704
8705/**
8706 * dev_change_flags - change device settings
8707 * @dev: device
8708 * @flags: device state flags
8709 * @extack: netlink extended ack
8710 *
8711 * Change settings on device based state flags. The flags are
8712 * in the userspace exported format.
8713 */
8714int dev_change_flags(struct net_device *dev, unsigned int flags,
8715 struct netlink_ext_ack *extack)
8716{
8717 int ret;
8718 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
8719
8720 ret = __dev_change_flags(dev, flags, extack);
8721 if (ret < 0)
8722 return ret;
8723
8724 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
8725 __dev_notify_flags(dev, old_flags, changes, 0, NULL);
8726 return ret;
8727}
8728EXPORT_SYMBOL(dev_change_flags);
8729
8730int __dev_set_mtu(struct net_device *dev, int new_mtu)
8731{
8732 const struct net_device_ops *ops = dev->netdev_ops;
8733
8734 if (ops->ndo_change_mtu)
8735 return ops->ndo_change_mtu(dev, new_mtu);
8736
8737 /* Pairs with all the lockless reads of dev->mtu in the stack */
8738 WRITE_ONCE(dev->mtu, new_mtu);
8739 return 0;
8740}
8741EXPORT_SYMBOL(__dev_set_mtu);
8742
8743int dev_validate_mtu(struct net_device *dev, int new_mtu,
8744 struct netlink_ext_ack *extack)
8745{
8746 /* MTU must be positive, and in range */
8747 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
8748 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
8749 return -EINVAL;
8750 }
8751
8752 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
8753 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
8754 return -EINVAL;
8755 }
8756 return 0;
8757}
8758
8759/**
8760 * dev_set_mtu_ext - Change maximum transfer unit
8761 * @dev: device
8762 * @new_mtu: new transfer unit
8763 * @extack: netlink extended ack
8764 *
8765 * Change the maximum transfer size of the network device.
8766 */
8767int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
8768 struct netlink_ext_ack *extack)
8769{
8770 int err, orig_mtu;
8771
8772 if (new_mtu == dev->mtu)
8773 return 0;
8774
8775 err = dev_validate_mtu(dev, new_mtu, extack);
8776 if (err)
8777 return err;
8778
8779 if (!netif_device_present(dev))
8780 return -ENODEV;
8781
8782 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
8783 err = notifier_to_errno(err);
8784 if (err)
8785 return err;
8786
8787 orig_mtu = dev->mtu;
8788 err = __dev_set_mtu(dev, new_mtu);
8789
8790 if (!err) {
8791 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8792 orig_mtu);
8793 err = notifier_to_errno(err);
8794 if (err) {
8795 /* setting mtu back and notifying everyone again,
8796 * so that they have a chance to revert changes.
8797 */
8798 __dev_set_mtu(dev, orig_mtu);
8799 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8800 new_mtu);
8801 }
8802 }
8803 return err;
8804}
8805
8806int dev_set_mtu(struct net_device *dev, int new_mtu)
8807{
8808 struct netlink_ext_ack extack;
8809 int err;
8810
8811 memset(&extack, 0, sizeof(extack));
8812 err = dev_set_mtu_ext(dev, new_mtu, &extack);
8813 if (err && extack._msg)
8814 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
8815 return err;
8816}
8817EXPORT_SYMBOL(dev_set_mtu);
8818
8819/**
8820 * dev_change_tx_queue_len - Change TX queue length of a netdevice
8821 * @dev: device
8822 * @new_len: new tx queue length
8823 */
8824int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
8825{
8826 unsigned int orig_len = dev->tx_queue_len;
8827 int res;
8828
8829 if (new_len != (unsigned int)new_len)
8830 return -ERANGE;
8831
8832 if (new_len != orig_len) {
8833 dev->tx_queue_len = new_len;
8834 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
8835 res = notifier_to_errno(res);
8836 if (res)
8837 goto err_rollback;
8838 res = dev_qdisc_change_tx_queue_len(dev);
8839 if (res)
8840 goto err_rollback;
8841 }
8842
8843 return 0;
8844
8845err_rollback:
8846 netdev_err(dev, "refused to change device tx_queue_len\n");
8847 dev->tx_queue_len = orig_len;
8848 return res;
8849}
8850
8851/**
8852 * dev_set_group - Change group this device belongs to
8853 * @dev: device
8854 * @new_group: group this device should belong to
8855 */
8856void dev_set_group(struct net_device *dev, int new_group)
8857{
8858 dev->group = new_group;
8859}
8860
8861/**
8862 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
8863 * @dev: device
8864 * @addr: new address
8865 * @extack: netlink extended ack
8866 */
8867int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
8868 struct netlink_ext_ack *extack)
8869{
8870 struct netdev_notifier_pre_changeaddr_info info = {
8871 .info.dev = dev,
8872 .info.extack = extack,
8873 .dev_addr = addr,
8874 };
8875 int rc;
8876
8877 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
8878 return notifier_to_errno(rc);
8879}
8880EXPORT_SYMBOL(dev_pre_changeaddr_notify);
8881
8882/**
8883 * dev_set_mac_address - Change Media Access Control Address
8884 * @dev: device
8885 * @sa: new address
8886 * @extack: netlink extended ack
8887 *
8888 * Change the hardware (MAC) address of the device
8889 */
8890int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
8891 struct netlink_ext_ack *extack)
8892{
8893 const struct net_device_ops *ops = dev->netdev_ops;
8894 int err;
8895
8896 if (!ops->ndo_set_mac_address)
8897 return -EOPNOTSUPP;
8898 if (sa->sa_family != dev->type)
8899 return -EINVAL;
8900 if (!netif_device_present(dev))
8901 return -ENODEV;
8902 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
8903 if (err)
8904 return err;
8905 if (memcmp(dev->dev_addr, sa->sa_data, dev->addr_len)) {
8906 err = ops->ndo_set_mac_address(dev, sa);
8907 if (err)
8908 return err;
8909 }
8910 dev->addr_assign_type = NET_ADDR_SET;
8911 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
8912 add_device_randomness(dev->dev_addr, dev->addr_len);
8913 return 0;
8914}
8915EXPORT_SYMBOL(dev_set_mac_address);
8916
8917static DECLARE_RWSEM(dev_addr_sem);
8918
8919int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
8920 struct netlink_ext_ack *extack)
8921{
8922 int ret;
8923
8924 down_write(&dev_addr_sem);
8925 ret = dev_set_mac_address(dev, sa, extack);
8926 up_write(&dev_addr_sem);
8927 return ret;
8928}
8929EXPORT_SYMBOL(dev_set_mac_address_user);
8930
8931int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
8932{
8933 size_t size = sizeof(sa->sa_data_min);
8934 struct net_device *dev;
8935 int ret = 0;
8936
8937 down_read(&dev_addr_sem);
8938 rcu_read_lock();
8939
8940 dev = dev_get_by_name_rcu(net, dev_name);
8941 if (!dev) {
8942 ret = -ENODEV;
8943 goto unlock;
8944 }
8945 if (!dev->addr_len)
8946 memset(sa->sa_data, 0, size);
8947 else
8948 memcpy(sa->sa_data, dev->dev_addr,
8949 min_t(size_t, size, dev->addr_len));
8950 sa->sa_family = dev->type;
8951
8952unlock:
8953 rcu_read_unlock();
8954 up_read(&dev_addr_sem);
8955 return ret;
8956}
8957EXPORT_SYMBOL(dev_get_mac_address);
8958
8959/**
8960 * dev_change_carrier - Change device carrier
8961 * @dev: device
8962 * @new_carrier: new value
8963 *
8964 * Change device carrier
8965 */
8966int dev_change_carrier(struct net_device *dev, bool new_carrier)
8967{
8968 const struct net_device_ops *ops = dev->netdev_ops;
8969
8970 if (!ops->ndo_change_carrier)
8971 return -EOPNOTSUPP;
8972 if (!netif_device_present(dev))
8973 return -ENODEV;
8974 return ops->ndo_change_carrier(dev, new_carrier);
8975}
8976
8977/**
8978 * dev_get_phys_port_id - Get device physical port ID
8979 * @dev: device
8980 * @ppid: port ID
8981 *
8982 * Get device physical port ID
8983 */
8984int dev_get_phys_port_id(struct net_device *dev,
8985 struct netdev_phys_item_id *ppid)
8986{
8987 const struct net_device_ops *ops = dev->netdev_ops;
8988
8989 if (!ops->ndo_get_phys_port_id)
8990 return -EOPNOTSUPP;
8991 return ops->ndo_get_phys_port_id(dev, ppid);
8992}
8993
8994/**
8995 * dev_get_phys_port_name - Get device physical port name
8996 * @dev: device
8997 * @name: port name
8998 * @len: limit of bytes to copy to name
8999 *
9000 * Get device physical port name
9001 */
9002int dev_get_phys_port_name(struct net_device *dev,
9003 char *name, size_t len)
9004{
9005 const struct net_device_ops *ops = dev->netdev_ops;
9006 int err;
9007
9008 if (ops->ndo_get_phys_port_name) {
9009 err = ops->ndo_get_phys_port_name(dev, name, len);
9010 if (err != -EOPNOTSUPP)
9011 return err;
9012 }
9013 return devlink_compat_phys_port_name_get(dev, name, len);
9014}
9015
9016/**
9017 * dev_get_port_parent_id - Get the device's port parent identifier
9018 * @dev: network device
9019 * @ppid: pointer to a storage for the port's parent identifier
9020 * @recurse: allow/disallow recursion to lower devices
9021 *
9022 * Get the devices's port parent identifier
9023 */
9024int dev_get_port_parent_id(struct net_device *dev,
9025 struct netdev_phys_item_id *ppid,
9026 bool recurse)
9027{
9028 const struct net_device_ops *ops = dev->netdev_ops;
9029 struct netdev_phys_item_id first = { };
9030 struct net_device *lower_dev;
9031 struct list_head *iter;
9032 int err;
9033
9034 if (ops->ndo_get_port_parent_id) {
9035 err = ops->ndo_get_port_parent_id(dev, ppid);
9036 if (err != -EOPNOTSUPP)
9037 return err;
9038 }
9039
9040 err = devlink_compat_switch_id_get(dev, ppid);
9041 if (!recurse || err != -EOPNOTSUPP)
9042 return err;
9043
9044 netdev_for_each_lower_dev(dev, lower_dev, iter) {
9045 err = dev_get_port_parent_id(lower_dev, ppid, true);
9046 if (err)
9047 break;
9048 if (!first.id_len)
9049 first = *ppid;
9050 else if (memcmp(&first, ppid, sizeof(*ppid)))
9051 return -EOPNOTSUPP;
9052 }
9053
9054 return err;
9055}
9056EXPORT_SYMBOL(dev_get_port_parent_id);
9057
9058/**
9059 * netdev_port_same_parent_id - Indicate if two network devices have
9060 * the same port parent identifier
9061 * @a: first network device
9062 * @b: second network device
9063 */
9064bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
9065{
9066 struct netdev_phys_item_id a_id = { };
9067 struct netdev_phys_item_id b_id = { };
9068
9069 if (dev_get_port_parent_id(a, &a_id, true) ||
9070 dev_get_port_parent_id(b, &b_id, true))
9071 return false;
9072
9073 return netdev_phys_item_id_same(&a_id, &b_id);
9074}
9075EXPORT_SYMBOL(netdev_port_same_parent_id);
9076
9077/**
9078 * dev_change_proto_down - set carrier according to proto_down.
9079 *
9080 * @dev: device
9081 * @proto_down: new value
9082 */
9083int dev_change_proto_down(struct net_device *dev, bool proto_down)
9084{
9085 if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN))
9086 return -EOPNOTSUPP;
9087 if (!netif_device_present(dev))
9088 return -ENODEV;
9089 if (proto_down)
9090 netif_carrier_off(dev);
9091 else
9092 netif_carrier_on(dev);
9093 dev->proto_down = proto_down;
9094 return 0;
9095}
9096
9097/**
9098 * dev_change_proto_down_reason - proto down reason
9099 *
9100 * @dev: device
9101 * @mask: proto down mask
9102 * @value: proto down value
9103 */
9104void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
9105 u32 value)
9106{
9107 int b;
9108
9109 if (!mask) {
9110 dev->proto_down_reason = value;
9111 } else {
9112 for_each_set_bit(b, &mask, 32) {
9113 if (value & (1 << b))
9114 dev->proto_down_reason |= BIT(b);
9115 else
9116 dev->proto_down_reason &= ~BIT(b);
9117 }
9118 }
9119}
9120
9121struct bpf_xdp_link {
9122 struct bpf_link link;
9123 struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
9124 int flags;
9125};
9126
9127static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
9128{
9129 if (flags & XDP_FLAGS_HW_MODE)
9130 return XDP_MODE_HW;
9131 if (flags & XDP_FLAGS_DRV_MODE)
9132 return XDP_MODE_DRV;
9133 if (flags & XDP_FLAGS_SKB_MODE)
9134 return XDP_MODE_SKB;
9135 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
9136}
9137
9138static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
9139{
9140 switch (mode) {
9141 case XDP_MODE_SKB:
9142 return generic_xdp_install;
9143 case XDP_MODE_DRV:
9144 case XDP_MODE_HW:
9145 return dev->netdev_ops->ndo_bpf;
9146 default:
9147 return NULL;
9148 }
9149}
9150
9151static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
9152 enum bpf_xdp_mode mode)
9153{
9154 return dev->xdp_state[mode].link;
9155}
9156
9157static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
9158 enum bpf_xdp_mode mode)
9159{
9160 struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
9161
9162 if (link)
9163 return link->link.prog;
9164 return dev->xdp_state[mode].prog;
9165}
9166
9167u8 dev_xdp_prog_count(struct net_device *dev)
9168{
9169 u8 count = 0;
9170 int i;
9171
9172 for (i = 0; i < __MAX_XDP_MODE; i++)
9173 if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
9174 count++;
9175 return count;
9176}
9177EXPORT_SYMBOL_GPL(dev_xdp_prog_count);
9178
9179u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
9180{
9181 struct bpf_prog *prog = dev_xdp_prog(dev, mode);
9182
9183 return prog ? prog->aux->id : 0;
9184}
9185
9186static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
9187 struct bpf_xdp_link *link)
9188{
9189 dev->xdp_state[mode].link = link;
9190 dev->xdp_state[mode].prog = NULL;
9191}
9192
9193static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
9194 struct bpf_prog *prog)
9195{
9196 dev->xdp_state[mode].link = NULL;
9197 dev->xdp_state[mode].prog = prog;
9198}
9199
9200static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
9201 bpf_op_t bpf_op, struct netlink_ext_ack *extack,
9202 u32 flags, struct bpf_prog *prog)
9203{
9204 struct netdev_bpf xdp;
9205 int err;
9206
9207 memset(&xdp, 0, sizeof(xdp));
9208 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
9209 xdp.extack = extack;
9210 xdp.flags = flags;
9211 xdp.prog = prog;
9212
9213 /* Drivers assume refcnt is already incremented (i.e, prog pointer is
9214 * "moved" into driver), so they don't increment it on their own, but
9215 * they do decrement refcnt when program is detached or replaced.
9216 * Given net_device also owns link/prog, we need to bump refcnt here
9217 * to prevent drivers from underflowing it.
9218 */
9219 if (prog)
9220 bpf_prog_inc(prog);
9221 err = bpf_op(dev, &xdp);
9222 if (err) {
9223 if (prog)
9224 bpf_prog_put(prog);
9225 return err;
9226 }
9227
9228 if (mode != XDP_MODE_HW)
9229 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
9230
9231 return 0;
9232}
9233
9234static void dev_xdp_uninstall(struct net_device *dev)
9235{
9236 struct bpf_xdp_link *link;
9237 struct bpf_prog *prog;
9238 enum bpf_xdp_mode mode;
9239 bpf_op_t bpf_op;
9240
9241 ASSERT_RTNL();
9242
9243 for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
9244 prog = dev_xdp_prog(dev, mode);
9245 if (!prog)
9246 continue;
9247
9248 bpf_op = dev_xdp_bpf_op(dev, mode);
9249 if (!bpf_op)
9250 continue;
9251
9252 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9253
9254 /* auto-detach link from net device */
9255 link = dev_xdp_link(dev, mode);
9256 if (link)
9257 link->dev = NULL;
9258 else
9259 bpf_prog_put(prog);
9260
9261 dev_xdp_set_link(dev, mode, NULL);
9262 }
9263}
9264
9265static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
9266 struct bpf_xdp_link *link, struct bpf_prog *new_prog,
9267 struct bpf_prog *old_prog, u32 flags)
9268{
9269 unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
9270 struct bpf_prog *cur_prog;
9271 struct net_device *upper;
9272 struct list_head *iter;
9273 enum bpf_xdp_mode mode;
9274 bpf_op_t bpf_op;
9275 int err;
9276
9277 ASSERT_RTNL();
9278
9279 /* either link or prog attachment, never both */
9280 if (link && (new_prog || old_prog))
9281 return -EINVAL;
9282 /* link supports only XDP mode flags */
9283 if (link && (flags & ~XDP_FLAGS_MODES)) {
9284 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
9285 return -EINVAL;
9286 }
9287 /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
9288 if (num_modes > 1) {
9289 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
9290 return -EINVAL;
9291 }
9292 /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
9293 if (!num_modes && dev_xdp_prog_count(dev) > 1) {
9294 NL_SET_ERR_MSG(extack,
9295 "More than one program loaded, unset mode is ambiguous");
9296 return -EINVAL;
9297 }
9298 /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
9299 if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
9300 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
9301 return -EINVAL;
9302 }
9303
9304 mode = dev_xdp_mode(dev, flags);
9305 /* can't replace attached link */
9306 if (dev_xdp_link(dev, mode)) {
9307 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
9308 return -EBUSY;
9309 }
9310
9311 /* don't allow if an upper device already has a program */
9312 netdev_for_each_upper_dev_rcu(dev, upper, iter) {
9313 if (dev_xdp_prog_count(upper) > 0) {
9314 NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program");
9315 return -EEXIST;
9316 }
9317 }
9318
9319 cur_prog = dev_xdp_prog(dev, mode);
9320 /* can't replace attached prog with link */
9321 if (link && cur_prog) {
9322 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
9323 return -EBUSY;
9324 }
9325 if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
9326 NL_SET_ERR_MSG(extack, "Active program does not match expected");
9327 return -EEXIST;
9328 }
9329
9330 /* put effective new program into new_prog */
9331 if (link)
9332 new_prog = link->link.prog;
9333
9334 if (new_prog) {
9335 bool offload = mode == XDP_MODE_HW;
9336 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
9337 ? XDP_MODE_DRV : XDP_MODE_SKB;
9338
9339 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
9340 NL_SET_ERR_MSG(extack, "XDP program already attached");
9341 return -EBUSY;
9342 }
9343 if (!offload && dev_xdp_prog(dev, other_mode)) {
9344 NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
9345 return -EEXIST;
9346 }
9347 if (!offload && bpf_prog_is_offloaded(new_prog->aux)) {
9348 NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported");
9349 return -EINVAL;
9350 }
9351 if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) {
9352 NL_SET_ERR_MSG(extack, "Program bound to different device");
9353 return -EINVAL;
9354 }
9355 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
9356 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
9357 return -EINVAL;
9358 }
9359 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
9360 NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
9361 return -EINVAL;
9362 }
9363 }
9364
9365 /* don't call drivers if the effective program didn't change */
9366 if (new_prog != cur_prog) {
9367 bpf_op = dev_xdp_bpf_op(dev, mode);
9368 if (!bpf_op) {
9369 NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
9370 return -EOPNOTSUPP;
9371 }
9372
9373 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
9374 if (err)
9375 return err;
9376 }
9377
9378 if (link)
9379 dev_xdp_set_link(dev, mode, link);
9380 else
9381 dev_xdp_set_prog(dev, mode, new_prog);
9382 if (cur_prog)
9383 bpf_prog_put(cur_prog);
9384
9385 return 0;
9386}
9387
9388static int dev_xdp_attach_link(struct net_device *dev,
9389 struct netlink_ext_ack *extack,
9390 struct bpf_xdp_link *link)
9391{
9392 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
9393}
9394
9395static int dev_xdp_detach_link(struct net_device *dev,
9396 struct netlink_ext_ack *extack,
9397 struct bpf_xdp_link *link)
9398{
9399 enum bpf_xdp_mode mode;
9400 bpf_op_t bpf_op;
9401
9402 ASSERT_RTNL();
9403
9404 mode = dev_xdp_mode(dev, link->flags);
9405 if (dev_xdp_link(dev, mode) != link)
9406 return -EINVAL;
9407
9408 bpf_op = dev_xdp_bpf_op(dev, mode);
9409 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9410 dev_xdp_set_link(dev, mode, NULL);
9411 return 0;
9412}
9413
9414static void bpf_xdp_link_release(struct bpf_link *link)
9415{
9416 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9417
9418 rtnl_lock();
9419
9420 /* if racing with net_device's tear down, xdp_link->dev might be
9421 * already NULL, in which case link was already auto-detached
9422 */
9423 if (xdp_link->dev) {
9424 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
9425 xdp_link->dev = NULL;
9426 }
9427
9428 rtnl_unlock();
9429}
9430
9431static int bpf_xdp_link_detach(struct bpf_link *link)
9432{
9433 bpf_xdp_link_release(link);
9434 return 0;
9435}
9436
9437static void bpf_xdp_link_dealloc(struct bpf_link *link)
9438{
9439 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9440
9441 kfree(xdp_link);
9442}
9443
9444static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
9445 struct seq_file *seq)
9446{
9447 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9448 u32 ifindex = 0;
9449
9450 rtnl_lock();
9451 if (xdp_link->dev)
9452 ifindex = xdp_link->dev->ifindex;
9453 rtnl_unlock();
9454
9455 seq_printf(seq, "ifindex:\t%u\n", ifindex);
9456}
9457
9458static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
9459 struct bpf_link_info *info)
9460{
9461 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9462 u32 ifindex = 0;
9463
9464 rtnl_lock();
9465 if (xdp_link->dev)
9466 ifindex = xdp_link->dev->ifindex;
9467 rtnl_unlock();
9468
9469 info->xdp.ifindex = ifindex;
9470 return 0;
9471}
9472
9473static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
9474 struct bpf_prog *old_prog)
9475{
9476 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9477 enum bpf_xdp_mode mode;
9478 bpf_op_t bpf_op;
9479 int err = 0;
9480
9481 rtnl_lock();
9482
9483 /* link might have been auto-released already, so fail */
9484 if (!xdp_link->dev) {
9485 err = -ENOLINK;
9486 goto out_unlock;
9487 }
9488
9489 if (old_prog && link->prog != old_prog) {
9490 err = -EPERM;
9491 goto out_unlock;
9492 }
9493 old_prog = link->prog;
9494 if (old_prog->type != new_prog->type ||
9495 old_prog->expected_attach_type != new_prog->expected_attach_type) {
9496 err = -EINVAL;
9497 goto out_unlock;
9498 }
9499
9500 if (old_prog == new_prog) {
9501 /* no-op, don't disturb drivers */
9502 bpf_prog_put(new_prog);
9503 goto out_unlock;
9504 }
9505
9506 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
9507 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
9508 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
9509 xdp_link->flags, new_prog);
9510 if (err)
9511 goto out_unlock;
9512
9513 old_prog = xchg(&link->prog, new_prog);
9514 bpf_prog_put(old_prog);
9515
9516out_unlock:
9517 rtnl_unlock();
9518 return err;
9519}
9520
9521static const struct bpf_link_ops bpf_xdp_link_lops = {
9522 .release = bpf_xdp_link_release,
9523 .dealloc = bpf_xdp_link_dealloc,
9524 .detach = bpf_xdp_link_detach,
9525 .show_fdinfo = bpf_xdp_link_show_fdinfo,
9526 .fill_link_info = bpf_xdp_link_fill_link_info,
9527 .update_prog = bpf_xdp_link_update,
9528};
9529
9530int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
9531{
9532 struct net *net = current->nsproxy->net_ns;
9533 struct bpf_link_primer link_primer;
9534 struct netlink_ext_ack extack = {};
9535 struct bpf_xdp_link *link;
9536 struct net_device *dev;
9537 int err, fd;
9538
9539 rtnl_lock();
9540 dev = dev_get_by_index(net, attr->link_create.target_ifindex);
9541 if (!dev) {
9542 rtnl_unlock();
9543 return -EINVAL;
9544 }
9545
9546 link = kzalloc(sizeof(*link), GFP_USER);
9547 if (!link) {
9548 err = -ENOMEM;
9549 goto unlock;
9550 }
9551
9552 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
9553 link->dev = dev;
9554 link->flags = attr->link_create.flags;
9555
9556 err = bpf_link_prime(&link->link, &link_primer);
9557 if (err) {
9558 kfree(link);
9559 goto unlock;
9560 }
9561
9562 err = dev_xdp_attach_link(dev, &extack, link);
9563 rtnl_unlock();
9564
9565 if (err) {
9566 link->dev = NULL;
9567 bpf_link_cleanup(&link_primer);
9568 trace_bpf_xdp_link_attach_failed(extack._msg);
9569 goto out_put_dev;
9570 }
9571
9572 fd = bpf_link_settle(&link_primer);
9573 /* link itself doesn't hold dev's refcnt to not complicate shutdown */
9574 dev_put(dev);
9575 return fd;
9576
9577unlock:
9578 rtnl_unlock();
9579
9580out_put_dev:
9581 dev_put(dev);
9582 return err;
9583}
9584
9585/**
9586 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
9587 * @dev: device
9588 * @extack: netlink extended ack
9589 * @fd: new program fd or negative value to clear
9590 * @expected_fd: old program fd that userspace expects to replace or clear
9591 * @flags: xdp-related flags
9592 *
9593 * Set or clear a bpf program for a device
9594 */
9595int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
9596 int fd, int expected_fd, u32 flags)
9597{
9598 enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
9599 struct bpf_prog *new_prog = NULL, *old_prog = NULL;
9600 int err;
9601
9602 ASSERT_RTNL();
9603
9604 if (fd >= 0) {
9605 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
9606 mode != XDP_MODE_SKB);
9607 if (IS_ERR(new_prog))
9608 return PTR_ERR(new_prog);
9609 }
9610
9611 if (expected_fd >= 0) {
9612 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
9613 mode != XDP_MODE_SKB);
9614 if (IS_ERR(old_prog)) {
9615 err = PTR_ERR(old_prog);
9616 old_prog = NULL;
9617 goto err_out;
9618 }
9619 }
9620
9621 err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
9622
9623err_out:
9624 if (err && new_prog)
9625 bpf_prog_put(new_prog);
9626 if (old_prog)
9627 bpf_prog_put(old_prog);
9628 return err;
9629}
9630
9631/**
9632 * dev_index_reserve() - allocate an ifindex in a namespace
9633 * @net: the applicable net namespace
9634 * @ifindex: requested ifindex, pass %0 to get one allocated
9635 *
9636 * Allocate a ifindex for a new device. Caller must either use the ifindex
9637 * to store the device (via list_netdevice()) or call dev_index_release()
9638 * to give the index up.
9639 *
9640 * Return: a suitable unique value for a new device interface number or -errno.
9641 */
9642static int dev_index_reserve(struct net *net, u32 ifindex)
9643{
9644 int err;
9645
9646 if (ifindex > INT_MAX) {
9647 DEBUG_NET_WARN_ON_ONCE(1);
9648 return -EINVAL;
9649 }
9650
9651 if (!ifindex)
9652 err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL,
9653 xa_limit_31b, &net->ifindex, GFP_KERNEL);
9654 else
9655 err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL);
9656 if (err < 0)
9657 return err;
9658
9659 return ifindex;
9660}
9661
9662static void dev_index_release(struct net *net, int ifindex)
9663{
9664 /* Expect only unused indexes, unlist_netdevice() removes the used */
9665 WARN_ON(xa_erase(&net->dev_by_index, ifindex));
9666}
9667
9668/* Delayed registration/unregisteration */
9669LIST_HEAD(net_todo_list);
9670DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
9671
9672static void net_set_todo(struct net_device *dev)
9673{
9674 list_add_tail(&dev->todo_list, &net_todo_list);
9675 atomic_inc(&dev_net(dev)->dev_unreg_count);
9676}
9677
9678static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
9679 struct net_device *upper, netdev_features_t features)
9680{
9681 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9682 netdev_features_t feature;
9683 int feature_bit;
9684
9685 for_each_netdev_feature(upper_disables, feature_bit) {
9686 feature = __NETIF_F_BIT(feature_bit);
9687 if (!(upper->wanted_features & feature)
9688 && (features & feature)) {
9689 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
9690 &feature, upper->name);
9691 features &= ~feature;
9692 }
9693 }
9694
9695 return features;
9696}
9697
9698static void netdev_sync_lower_features(struct net_device *upper,
9699 struct net_device *lower, netdev_features_t features)
9700{
9701 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9702 netdev_features_t feature;
9703 int feature_bit;
9704
9705 for_each_netdev_feature(upper_disables, feature_bit) {
9706 feature = __NETIF_F_BIT(feature_bit);
9707 if (!(features & feature) && (lower->features & feature)) {
9708 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
9709 &feature, lower->name);
9710 lower->wanted_features &= ~feature;
9711 __netdev_update_features(lower);
9712
9713 if (unlikely(lower->features & feature))
9714 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
9715 &feature, lower->name);
9716 else
9717 netdev_features_change(lower);
9718 }
9719 }
9720}
9721
9722static netdev_features_t netdev_fix_features(struct net_device *dev,
9723 netdev_features_t features)
9724{
9725 /* Fix illegal checksum combinations */
9726 if ((features & NETIF_F_HW_CSUM) &&
9727 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
9728 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
9729 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
9730 }
9731
9732 /* TSO requires that SG is present as well. */
9733 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
9734 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
9735 features &= ~NETIF_F_ALL_TSO;
9736 }
9737
9738 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
9739 !(features & NETIF_F_IP_CSUM)) {
9740 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
9741 features &= ~NETIF_F_TSO;
9742 features &= ~NETIF_F_TSO_ECN;
9743 }
9744
9745 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
9746 !(features & NETIF_F_IPV6_CSUM)) {
9747 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
9748 features &= ~NETIF_F_TSO6;
9749 }
9750
9751 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
9752 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
9753 features &= ~NETIF_F_TSO_MANGLEID;
9754
9755 /* TSO ECN requires that TSO is present as well. */
9756 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
9757 features &= ~NETIF_F_TSO_ECN;
9758
9759 /* Software GSO depends on SG. */
9760 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
9761 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
9762 features &= ~NETIF_F_GSO;
9763 }
9764
9765 /* GSO partial features require GSO partial be set */
9766 if ((features & dev->gso_partial_features) &&
9767 !(features & NETIF_F_GSO_PARTIAL)) {
9768 netdev_dbg(dev,
9769 "Dropping partially supported GSO features since no GSO partial.\n");
9770 features &= ~dev->gso_partial_features;
9771 }
9772
9773 if (!(features & NETIF_F_RXCSUM)) {
9774 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
9775 * successfully merged by hardware must also have the
9776 * checksum verified by hardware. If the user does not
9777 * want to enable RXCSUM, logically, we should disable GRO_HW.
9778 */
9779 if (features & NETIF_F_GRO_HW) {
9780 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
9781 features &= ~NETIF_F_GRO_HW;
9782 }
9783 }
9784
9785 /* LRO/HW-GRO features cannot be combined with RX-FCS */
9786 if (features & NETIF_F_RXFCS) {
9787 if (features & NETIF_F_LRO) {
9788 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
9789 features &= ~NETIF_F_LRO;
9790 }
9791
9792 if (features & NETIF_F_GRO_HW) {
9793 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
9794 features &= ~NETIF_F_GRO_HW;
9795 }
9796 }
9797
9798 if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) {
9799 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n");
9800 features &= ~NETIF_F_LRO;
9801 }
9802
9803 if (features & NETIF_F_HW_TLS_TX) {
9804 bool ip_csum = (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) ==
9805 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
9806 bool hw_csum = features & NETIF_F_HW_CSUM;
9807
9808 if (!ip_csum && !hw_csum) {
9809 netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
9810 features &= ~NETIF_F_HW_TLS_TX;
9811 }
9812 }
9813
9814 if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
9815 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
9816 features &= ~NETIF_F_HW_TLS_RX;
9817 }
9818
9819 return features;
9820}
9821
9822int __netdev_update_features(struct net_device *dev)
9823{
9824 struct net_device *upper, *lower;
9825 netdev_features_t features;
9826 struct list_head *iter;
9827 int err = -1;
9828
9829 ASSERT_RTNL();
9830
9831 features = netdev_get_wanted_features(dev);
9832
9833 if (dev->netdev_ops->ndo_fix_features)
9834 features = dev->netdev_ops->ndo_fix_features(dev, features);
9835
9836 /* driver might be less strict about feature dependencies */
9837 features = netdev_fix_features(dev, features);
9838
9839 /* some features can't be enabled if they're off on an upper device */
9840 netdev_for_each_upper_dev_rcu(dev, upper, iter)
9841 features = netdev_sync_upper_features(dev, upper, features);
9842
9843 if (dev->features == features)
9844 goto sync_lower;
9845
9846 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
9847 &dev->features, &features);
9848
9849 if (dev->netdev_ops->ndo_set_features)
9850 err = dev->netdev_ops->ndo_set_features(dev, features);
9851 else
9852 err = 0;
9853
9854 if (unlikely(err < 0)) {
9855 netdev_err(dev,
9856 "set_features() failed (%d); wanted %pNF, left %pNF\n",
9857 err, &features, &dev->features);
9858 /* return non-0 since some features might have changed and
9859 * it's better to fire a spurious notification than miss it
9860 */
9861 return -1;
9862 }
9863
9864sync_lower:
9865 /* some features must be disabled on lower devices when disabled
9866 * on an upper device (think: bonding master or bridge)
9867 */
9868 netdev_for_each_lower_dev(dev, lower, iter)
9869 netdev_sync_lower_features(dev, lower, features);
9870
9871 if (!err) {
9872 netdev_features_t diff = features ^ dev->features;
9873
9874 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
9875 /* udp_tunnel_{get,drop}_rx_info both need
9876 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
9877 * device, or they won't do anything.
9878 * Thus we need to update dev->features
9879 * *before* calling udp_tunnel_get_rx_info,
9880 * but *after* calling udp_tunnel_drop_rx_info.
9881 */
9882 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
9883 dev->features = features;
9884 udp_tunnel_get_rx_info(dev);
9885 } else {
9886 udp_tunnel_drop_rx_info(dev);
9887 }
9888 }
9889
9890 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
9891 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
9892 dev->features = features;
9893 err |= vlan_get_rx_ctag_filter_info(dev);
9894 } else {
9895 vlan_drop_rx_ctag_filter_info(dev);
9896 }
9897 }
9898
9899 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
9900 if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
9901 dev->features = features;
9902 err |= vlan_get_rx_stag_filter_info(dev);
9903 } else {
9904 vlan_drop_rx_stag_filter_info(dev);
9905 }
9906 }
9907
9908 dev->features = features;
9909 }
9910
9911 return err < 0 ? 0 : 1;
9912}
9913
9914/**
9915 * netdev_update_features - recalculate device features
9916 * @dev: the device to check
9917 *
9918 * Recalculate dev->features set and send notifications if it
9919 * has changed. Should be called after driver or hardware dependent
9920 * conditions might have changed that influence the features.
9921 */
9922void netdev_update_features(struct net_device *dev)
9923{
9924 if (__netdev_update_features(dev))
9925 netdev_features_change(dev);
9926}
9927EXPORT_SYMBOL(netdev_update_features);
9928
9929/**
9930 * netdev_change_features - recalculate device features
9931 * @dev: the device to check
9932 *
9933 * Recalculate dev->features set and send notifications even
9934 * if they have not changed. Should be called instead of
9935 * netdev_update_features() if also dev->vlan_features might
9936 * have changed to allow the changes to be propagated to stacked
9937 * VLAN devices.
9938 */
9939void netdev_change_features(struct net_device *dev)
9940{
9941 __netdev_update_features(dev);
9942 netdev_features_change(dev);
9943}
9944EXPORT_SYMBOL(netdev_change_features);
9945
9946/**
9947 * netif_stacked_transfer_operstate - transfer operstate
9948 * @rootdev: the root or lower level device to transfer state from
9949 * @dev: the device to transfer operstate to
9950 *
9951 * Transfer operational state from root to device. This is normally
9952 * called when a stacking relationship exists between the root
9953 * device and the device(a leaf device).
9954 */
9955void netif_stacked_transfer_operstate(const struct net_device *rootdev,
9956 struct net_device *dev)
9957{
9958 if (rootdev->operstate == IF_OPER_DORMANT)
9959 netif_dormant_on(dev);
9960 else
9961 netif_dormant_off(dev);
9962
9963 if (rootdev->operstate == IF_OPER_TESTING)
9964 netif_testing_on(dev);
9965 else
9966 netif_testing_off(dev);
9967
9968 if (netif_carrier_ok(rootdev))
9969 netif_carrier_on(dev);
9970 else
9971 netif_carrier_off(dev);
9972}
9973EXPORT_SYMBOL(netif_stacked_transfer_operstate);
9974
9975static int netif_alloc_rx_queues(struct net_device *dev)
9976{
9977 unsigned int i, count = dev->num_rx_queues;
9978 struct netdev_rx_queue *rx;
9979 size_t sz = count * sizeof(*rx);
9980 int err = 0;
9981
9982 BUG_ON(count < 1);
9983
9984 rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
9985 if (!rx)
9986 return -ENOMEM;
9987
9988 dev->_rx = rx;
9989
9990 for (i = 0; i < count; i++) {
9991 rx[i].dev = dev;
9992
9993 /* XDP RX-queue setup */
9994 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
9995 if (err < 0)
9996 goto err_rxq_info;
9997 }
9998 return 0;
9999
10000err_rxq_info:
10001 /* Rollback successful reg's and free other resources */
10002 while (i--)
10003 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
10004 kvfree(dev->_rx);
10005 dev->_rx = NULL;
10006 return err;
10007}
10008
10009static void netif_free_rx_queues(struct net_device *dev)
10010{
10011 unsigned int i, count = dev->num_rx_queues;
10012
10013 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
10014 if (!dev->_rx)
10015 return;
10016
10017 for (i = 0; i < count; i++)
10018 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
10019
10020 kvfree(dev->_rx);
10021}
10022
10023static void netdev_init_one_queue(struct net_device *dev,
10024 struct netdev_queue *queue, void *_unused)
10025{
10026 /* Initialize queue lock */
10027 spin_lock_init(&queue->_xmit_lock);
10028 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
10029 queue->xmit_lock_owner = -1;
10030 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
10031 queue->dev = dev;
10032#ifdef CONFIG_BQL
10033 dql_init(&queue->dql, HZ);
10034#endif
10035}
10036
10037static void netif_free_tx_queues(struct net_device *dev)
10038{
10039 kvfree(dev->_tx);
10040}
10041
10042static int netif_alloc_netdev_queues(struct net_device *dev)
10043{
10044 unsigned int count = dev->num_tx_queues;
10045 struct netdev_queue *tx;
10046 size_t sz = count * sizeof(*tx);
10047
10048 if (count < 1 || count > 0xffff)
10049 return -EINVAL;
10050
10051 tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
10052 if (!tx)
10053 return -ENOMEM;
10054
10055 dev->_tx = tx;
10056
10057 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
10058 spin_lock_init(&dev->tx_global_lock);
10059
10060 return 0;
10061}
10062
10063void netif_tx_stop_all_queues(struct net_device *dev)
10064{
10065 unsigned int i;
10066
10067 for (i = 0; i < dev->num_tx_queues; i++) {
10068 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
10069
10070 netif_tx_stop_queue(txq);
10071 }
10072}
10073EXPORT_SYMBOL(netif_tx_stop_all_queues);
10074
10075static int netdev_do_alloc_pcpu_stats(struct net_device *dev)
10076{
10077 void __percpu *v;
10078
10079 /* Drivers implementing ndo_get_peer_dev must support tstat
10080 * accounting, so that skb_do_redirect() can bump the dev's
10081 * RX stats upon network namespace switch.
10082 */
10083 if (dev->netdev_ops->ndo_get_peer_dev &&
10084 dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS)
10085 return -EOPNOTSUPP;
10086
10087 switch (dev->pcpu_stat_type) {
10088 case NETDEV_PCPU_STAT_NONE:
10089 return 0;
10090 case NETDEV_PCPU_STAT_LSTATS:
10091 v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
10092 break;
10093 case NETDEV_PCPU_STAT_TSTATS:
10094 v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
10095 break;
10096 case NETDEV_PCPU_STAT_DSTATS:
10097 v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats);
10098 break;
10099 default:
10100 return -EINVAL;
10101 }
10102
10103 return v ? 0 : -ENOMEM;
10104}
10105
10106static void netdev_do_free_pcpu_stats(struct net_device *dev)
10107{
10108 switch (dev->pcpu_stat_type) {
10109 case NETDEV_PCPU_STAT_NONE:
10110 return;
10111 case NETDEV_PCPU_STAT_LSTATS:
10112 free_percpu(dev->lstats);
10113 break;
10114 case NETDEV_PCPU_STAT_TSTATS:
10115 free_percpu(dev->tstats);
10116 break;
10117 case NETDEV_PCPU_STAT_DSTATS:
10118 free_percpu(dev->dstats);
10119 break;
10120 }
10121}
10122
10123/**
10124 * register_netdevice() - register a network device
10125 * @dev: device to register
10126 *
10127 * Take a prepared network device structure and make it externally accessible.
10128 * A %NETDEV_REGISTER message is sent to the netdev notifier chain.
10129 * Callers must hold the rtnl lock - you may want register_netdev()
10130 * instead of this.
10131 */
10132int register_netdevice(struct net_device *dev)
10133{
10134 int ret;
10135 struct net *net = dev_net(dev);
10136
10137 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
10138 NETDEV_FEATURE_COUNT);
10139 BUG_ON(dev_boot_phase);
10140 ASSERT_RTNL();
10141
10142 might_sleep();
10143
10144 /* When net_device's are persistent, this will be fatal. */
10145 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
10146 BUG_ON(!net);
10147
10148 ret = ethtool_check_ops(dev->ethtool_ops);
10149 if (ret)
10150 return ret;
10151
10152 spin_lock_init(&dev->addr_list_lock);
10153 netdev_set_addr_lockdep_class(dev);
10154
10155 ret = dev_get_valid_name(net, dev, dev->name);
10156 if (ret < 0)
10157 goto out;
10158
10159 ret = -ENOMEM;
10160 dev->name_node = netdev_name_node_head_alloc(dev);
10161 if (!dev->name_node)
10162 goto out;
10163
10164 /* Init, if this function is available */
10165 if (dev->netdev_ops->ndo_init) {
10166 ret = dev->netdev_ops->ndo_init(dev);
10167 if (ret) {
10168 if (ret > 0)
10169 ret = -EIO;
10170 goto err_free_name;
10171 }
10172 }
10173
10174 if (((dev->hw_features | dev->features) &
10175 NETIF_F_HW_VLAN_CTAG_FILTER) &&
10176 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
10177 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
10178 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
10179 ret = -EINVAL;
10180 goto err_uninit;
10181 }
10182
10183 ret = netdev_do_alloc_pcpu_stats(dev);
10184 if (ret)
10185 goto err_uninit;
10186
10187 ret = dev_index_reserve(net, dev->ifindex);
10188 if (ret < 0)
10189 goto err_free_pcpu;
10190 dev->ifindex = ret;
10191
10192 /* Transfer changeable features to wanted_features and enable
10193 * software offloads (GSO and GRO).
10194 */
10195 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
10196 dev->features |= NETIF_F_SOFT_FEATURES;
10197
10198 if (dev->udp_tunnel_nic_info) {
10199 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10200 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10201 }
10202
10203 dev->wanted_features = dev->features & dev->hw_features;
10204
10205 if (!(dev->flags & IFF_LOOPBACK))
10206 dev->hw_features |= NETIF_F_NOCACHE_COPY;
10207
10208 /* If IPv4 TCP segmentation offload is supported we should also
10209 * allow the device to enable segmenting the frame with the option
10210 * of ignoring a static IP ID value. This doesn't enable the
10211 * feature itself but allows the user to enable it later.
10212 */
10213 if (dev->hw_features & NETIF_F_TSO)
10214 dev->hw_features |= NETIF_F_TSO_MANGLEID;
10215 if (dev->vlan_features & NETIF_F_TSO)
10216 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
10217 if (dev->mpls_features & NETIF_F_TSO)
10218 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
10219 if (dev->hw_enc_features & NETIF_F_TSO)
10220 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
10221
10222 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
10223 */
10224 dev->vlan_features |= NETIF_F_HIGHDMA;
10225
10226 /* Make NETIF_F_SG inheritable to tunnel devices.
10227 */
10228 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
10229
10230 /* Make NETIF_F_SG inheritable to MPLS.
10231 */
10232 dev->mpls_features |= NETIF_F_SG;
10233
10234 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
10235 ret = notifier_to_errno(ret);
10236 if (ret)
10237 goto err_ifindex_release;
10238
10239 ret = netdev_register_kobject(dev);
10240 write_lock(&dev_base_lock);
10241 dev->reg_state = ret ? NETREG_UNREGISTERED : NETREG_REGISTERED;
10242 write_unlock(&dev_base_lock);
10243 if (ret)
10244 goto err_uninit_notify;
10245
10246 __netdev_update_features(dev);
10247
10248 /*
10249 * Default initial state at registry is that the
10250 * device is present.
10251 */
10252
10253 set_bit(__LINK_STATE_PRESENT, &dev->state);
10254
10255 linkwatch_init_dev(dev);
10256
10257 dev_init_scheduler(dev);
10258
10259 netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL);
10260 list_netdevice(dev);
10261
10262 add_device_randomness(dev->dev_addr, dev->addr_len);
10263
10264 /* If the device has permanent device address, driver should
10265 * set dev_addr and also addr_assign_type should be set to
10266 * NET_ADDR_PERM (default value).
10267 */
10268 if (dev->addr_assign_type == NET_ADDR_PERM)
10269 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10270
10271 /* Notify protocols, that a new device appeared. */
10272 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
10273 ret = notifier_to_errno(ret);
10274 if (ret) {
10275 /* Expect explicit free_netdev() on failure */
10276 dev->needs_free_netdev = false;
10277 unregister_netdevice_queue(dev, NULL);
10278 goto out;
10279 }
10280 /*
10281 * Prevent userspace races by waiting until the network
10282 * device is fully setup before sending notifications.
10283 */
10284 if (!dev->rtnl_link_ops ||
10285 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10286 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
10287
10288out:
10289 return ret;
10290
10291err_uninit_notify:
10292 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
10293err_ifindex_release:
10294 dev_index_release(net, dev->ifindex);
10295err_free_pcpu:
10296 netdev_do_free_pcpu_stats(dev);
10297err_uninit:
10298 if (dev->netdev_ops->ndo_uninit)
10299 dev->netdev_ops->ndo_uninit(dev);
10300 if (dev->priv_destructor)
10301 dev->priv_destructor(dev);
10302err_free_name:
10303 netdev_name_node_free(dev->name_node);
10304 goto out;
10305}
10306EXPORT_SYMBOL(register_netdevice);
10307
10308/**
10309 * init_dummy_netdev - init a dummy network device for NAPI
10310 * @dev: device to init
10311 *
10312 * This takes a network device structure and initialize the minimum
10313 * amount of fields so it can be used to schedule NAPI polls without
10314 * registering a full blown interface. This is to be used by drivers
10315 * that need to tie several hardware interfaces to a single NAPI
10316 * poll scheduler due to HW limitations.
10317 */
10318int init_dummy_netdev(struct net_device *dev)
10319{
10320 /* Clear everything. Note we don't initialize spinlocks
10321 * are they aren't supposed to be taken by any of the
10322 * NAPI code and this dummy netdev is supposed to be
10323 * only ever used for NAPI polls
10324 */
10325 memset(dev, 0, sizeof(struct net_device));
10326
10327 /* make sure we BUG if trying to hit standard
10328 * register/unregister code path
10329 */
10330 dev->reg_state = NETREG_DUMMY;
10331
10332 /* NAPI wants this */
10333 INIT_LIST_HEAD(&dev->napi_list);
10334
10335 /* a dummy interface is started by default */
10336 set_bit(__LINK_STATE_PRESENT, &dev->state);
10337 set_bit(__LINK_STATE_START, &dev->state);
10338
10339 /* napi_busy_loop stats accounting wants this */
10340 dev_net_set(dev, &init_net);
10341
10342 /* Note : We dont allocate pcpu_refcnt for dummy devices,
10343 * because users of this 'device' dont need to change
10344 * its refcount.
10345 */
10346
10347 return 0;
10348}
10349EXPORT_SYMBOL_GPL(init_dummy_netdev);
10350
10351
10352/**
10353 * register_netdev - register a network device
10354 * @dev: device to register
10355 *
10356 * Take a completed network device structure and add it to the kernel
10357 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10358 * chain. 0 is returned on success. A negative errno code is returned
10359 * on a failure to set up the device, or if the name is a duplicate.
10360 *
10361 * This is a wrapper around register_netdevice that takes the rtnl semaphore
10362 * and expands the device name if you passed a format string to
10363 * alloc_netdev.
10364 */
10365int register_netdev(struct net_device *dev)
10366{
10367 int err;
10368
10369 if (rtnl_lock_killable())
10370 return -EINTR;
10371 err = register_netdevice(dev);
10372 rtnl_unlock();
10373 return err;
10374}
10375EXPORT_SYMBOL(register_netdev);
10376
10377int netdev_refcnt_read(const struct net_device *dev)
10378{
10379#ifdef CONFIG_PCPU_DEV_REFCNT
10380 int i, refcnt = 0;
10381
10382 for_each_possible_cpu(i)
10383 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10384 return refcnt;
10385#else
10386 return refcount_read(&dev->dev_refcnt);
10387#endif
10388}
10389EXPORT_SYMBOL(netdev_refcnt_read);
10390
10391int netdev_unregister_timeout_secs __read_mostly = 10;
10392
10393#define WAIT_REFS_MIN_MSECS 1
10394#define WAIT_REFS_MAX_MSECS 250
10395/**
10396 * netdev_wait_allrefs_any - wait until all references are gone.
10397 * @list: list of net_devices to wait on
10398 *
10399 * This is called when unregistering network devices.
10400 *
10401 * Any protocol or device that holds a reference should register
10402 * for netdevice notification, and cleanup and put back the
10403 * reference if they receive an UNREGISTER event.
10404 * We can get stuck here if buggy protocols don't correctly
10405 * call dev_put.
10406 */
10407static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
10408{
10409 unsigned long rebroadcast_time, warning_time;
10410 struct net_device *dev;
10411 int wait = 0;
10412
10413 rebroadcast_time = warning_time = jiffies;
10414
10415 list_for_each_entry(dev, list, todo_list)
10416 if (netdev_refcnt_read(dev) == 1)
10417 return dev;
10418
10419 while (true) {
10420 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
10421 rtnl_lock();
10422
10423 /* Rebroadcast unregister notification */
10424 list_for_each_entry(dev, list, todo_list)
10425 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10426
10427 __rtnl_unlock();
10428 rcu_barrier();
10429 rtnl_lock();
10430
10431 list_for_each_entry(dev, list, todo_list)
10432 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
10433 &dev->state)) {
10434 /* We must not have linkwatch events
10435 * pending on unregister. If this
10436 * happens, we simply run the queue
10437 * unscheduled, resulting in a noop
10438 * for this device.
10439 */
10440 linkwatch_run_queue();
10441 break;
10442 }
10443
10444 __rtnl_unlock();
10445
10446 rebroadcast_time = jiffies;
10447 }
10448
10449 if (!wait) {
10450 rcu_barrier();
10451 wait = WAIT_REFS_MIN_MSECS;
10452 } else {
10453 msleep(wait);
10454 wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10455 }
10456
10457 list_for_each_entry(dev, list, todo_list)
10458 if (netdev_refcnt_read(dev) == 1)
10459 return dev;
10460
10461 if (time_after(jiffies, warning_time +
10462 READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
10463 list_for_each_entry(dev, list, todo_list) {
10464 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10465 dev->name, netdev_refcnt_read(dev));
10466 ref_tracker_dir_print(&dev->refcnt_tracker, 10);
10467 }
10468
10469 warning_time = jiffies;
10470 }
10471 }
10472}
10473
10474/* The sequence is:
10475 *
10476 * rtnl_lock();
10477 * ...
10478 * register_netdevice(x1);
10479 * register_netdevice(x2);
10480 * ...
10481 * unregister_netdevice(y1);
10482 * unregister_netdevice(y2);
10483 * ...
10484 * rtnl_unlock();
10485 * free_netdev(y1);
10486 * free_netdev(y2);
10487 *
10488 * We are invoked by rtnl_unlock().
10489 * This allows us to deal with problems:
10490 * 1) We can delete sysfs objects which invoke hotplug
10491 * without deadlocking with linkwatch via keventd.
10492 * 2) Since we run with the RTNL semaphore not held, we can sleep
10493 * safely in order to wait for the netdev refcnt to drop to zero.
10494 *
10495 * We must not return until all unregister events added during
10496 * the interval the lock was held have been completed.
10497 */
10498void netdev_run_todo(void)
10499{
10500 struct net_device *dev, *tmp;
10501 struct list_head list;
10502#ifdef CONFIG_LOCKDEP
10503 struct list_head unlink_list;
10504
10505 list_replace_init(&net_unlink_list, &unlink_list);
10506
10507 while (!list_empty(&unlink_list)) {
10508 struct net_device *dev = list_first_entry(&unlink_list,
10509 struct net_device,
10510 unlink_list);
10511 list_del_init(&dev->unlink_list);
10512 dev->nested_level = dev->lower_level - 1;
10513 }
10514#endif
10515
10516 /* Snapshot list, allow later requests */
10517 list_replace_init(&net_todo_list, &list);
10518
10519 __rtnl_unlock();
10520
10521 /* Wait for rcu callbacks to finish before next phase */
10522 if (!list_empty(&list))
10523 rcu_barrier();
10524
10525 list_for_each_entry_safe(dev, tmp, &list, todo_list) {
10526 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
10527 netdev_WARN(dev, "run_todo but not unregistering\n");
10528 list_del(&dev->todo_list);
10529 continue;
10530 }
10531
10532 write_lock(&dev_base_lock);
10533 dev->reg_state = NETREG_UNREGISTERED;
10534 write_unlock(&dev_base_lock);
10535 linkwatch_sync_dev(dev);
10536 }
10537
10538 while (!list_empty(&list)) {
10539 dev = netdev_wait_allrefs_any(&list);
10540 list_del(&dev->todo_list);
10541
10542 /* paranoia */
10543 BUG_ON(netdev_refcnt_read(dev) != 1);
10544 BUG_ON(!list_empty(&dev->ptype_all));
10545 BUG_ON(!list_empty(&dev->ptype_specific));
10546 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10547 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
10548
10549 netdev_do_free_pcpu_stats(dev);
10550 if (dev->priv_destructor)
10551 dev->priv_destructor(dev);
10552 if (dev->needs_free_netdev)
10553 free_netdev(dev);
10554
10555 if (atomic_dec_and_test(&dev_net(dev)->dev_unreg_count))
10556 wake_up(&netdev_unregistering_wq);
10557
10558 /* Free network device */
10559 kobject_put(&dev->dev.kobj);
10560 }
10561}
10562
10563/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
10564 * all the same fields in the same order as net_device_stats, with only
10565 * the type differing, but rtnl_link_stats64 may have additional fields
10566 * at the end for newer counters.
10567 */
10568void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
10569 const struct net_device_stats *netdev_stats)
10570{
10571 size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t);
10572 const atomic_long_t *src = (atomic_long_t *)netdev_stats;
10573 u64 *dst = (u64 *)stats64;
10574
10575 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
10576 for (i = 0; i < n; i++)
10577 dst[i] = (unsigned long)atomic_long_read(&src[i]);
10578 /* zero out counters that only exist in rtnl_link_stats64 */
10579 memset((char *)stats64 + n * sizeof(u64), 0,
10580 sizeof(*stats64) - n * sizeof(u64));
10581}
10582EXPORT_SYMBOL(netdev_stats_to_stats64);
10583
10584static __cold struct net_device_core_stats __percpu *netdev_core_stats_alloc(
10585 struct net_device *dev)
10586{
10587 struct net_device_core_stats __percpu *p;
10588
10589 p = alloc_percpu_gfp(struct net_device_core_stats,
10590 GFP_ATOMIC | __GFP_NOWARN);
10591
10592 if (p && cmpxchg(&dev->core_stats, NULL, p))
10593 free_percpu(p);
10594
10595 /* This READ_ONCE() pairs with the cmpxchg() above */
10596 return READ_ONCE(dev->core_stats);
10597}
10598
10599noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset)
10600{
10601 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
10602 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats);
10603 unsigned long __percpu *field;
10604
10605 if (unlikely(!p)) {
10606 p = netdev_core_stats_alloc(dev);
10607 if (!p)
10608 return;
10609 }
10610
10611 field = (__force unsigned long __percpu *)((__force void *)p + offset);
10612 this_cpu_inc(*field);
10613}
10614EXPORT_SYMBOL_GPL(netdev_core_stats_inc);
10615
10616/**
10617 * dev_get_stats - get network device statistics
10618 * @dev: device to get statistics from
10619 * @storage: place to store stats
10620 *
10621 * Get network statistics from device. Return @storage.
10622 * The device driver may provide its own method by setting
10623 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
10624 * otherwise the internal statistics structure is used.
10625 */
10626struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
10627 struct rtnl_link_stats64 *storage)
10628{
10629 const struct net_device_ops *ops = dev->netdev_ops;
10630 const struct net_device_core_stats __percpu *p;
10631
10632 if (ops->ndo_get_stats64) {
10633 memset(storage, 0, sizeof(*storage));
10634 ops->ndo_get_stats64(dev, storage);
10635 } else if (ops->ndo_get_stats) {
10636 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
10637 } else {
10638 netdev_stats_to_stats64(storage, &dev->stats);
10639 }
10640
10641 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
10642 p = READ_ONCE(dev->core_stats);
10643 if (p) {
10644 const struct net_device_core_stats *core_stats;
10645 int i;
10646
10647 for_each_possible_cpu(i) {
10648 core_stats = per_cpu_ptr(p, i);
10649 storage->rx_dropped += READ_ONCE(core_stats->rx_dropped);
10650 storage->tx_dropped += READ_ONCE(core_stats->tx_dropped);
10651 storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler);
10652 storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped);
10653 }
10654 }
10655 return storage;
10656}
10657EXPORT_SYMBOL(dev_get_stats);
10658
10659/**
10660 * dev_fetch_sw_netstats - get per-cpu network device statistics
10661 * @s: place to store stats
10662 * @netstats: per-cpu network stats to read from
10663 *
10664 * Read per-cpu network statistics and populate the related fields in @s.
10665 */
10666void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
10667 const struct pcpu_sw_netstats __percpu *netstats)
10668{
10669 int cpu;
10670
10671 for_each_possible_cpu(cpu) {
10672 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
10673 const struct pcpu_sw_netstats *stats;
10674 unsigned int start;
10675
10676 stats = per_cpu_ptr(netstats, cpu);
10677 do {
10678 start = u64_stats_fetch_begin(&stats->syncp);
10679 rx_packets = u64_stats_read(&stats->rx_packets);
10680 rx_bytes = u64_stats_read(&stats->rx_bytes);
10681 tx_packets = u64_stats_read(&stats->tx_packets);
10682 tx_bytes = u64_stats_read(&stats->tx_bytes);
10683 } while (u64_stats_fetch_retry(&stats->syncp, start));
10684
10685 s->rx_packets += rx_packets;
10686 s->rx_bytes += rx_bytes;
10687 s->tx_packets += tx_packets;
10688 s->tx_bytes += tx_bytes;
10689 }
10690}
10691EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
10692
10693/**
10694 * dev_get_tstats64 - ndo_get_stats64 implementation
10695 * @dev: device to get statistics from
10696 * @s: place to store stats
10697 *
10698 * Populate @s from dev->stats and dev->tstats. Can be used as
10699 * ndo_get_stats64() callback.
10700 */
10701void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
10702{
10703 netdev_stats_to_stats64(s, &dev->stats);
10704 dev_fetch_sw_netstats(s, dev->tstats);
10705}
10706EXPORT_SYMBOL_GPL(dev_get_tstats64);
10707
10708struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
10709{
10710 struct netdev_queue *queue = dev_ingress_queue(dev);
10711
10712#ifdef CONFIG_NET_CLS_ACT
10713 if (queue)
10714 return queue;
10715 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
10716 if (!queue)
10717 return NULL;
10718 netdev_init_one_queue(dev, queue, NULL);
10719 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
10720 RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc);
10721 rcu_assign_pointer(dev->ingress_queue, queue);
10722#endif
10723 return queue;
10724}
10725
10726static const struct ethtool_ops default_ethtool_ops;
10727
10728void netdev_set_default_ethtool_ops(struct net_device *dev,
10729 const struct ethtool_ops *ops)
10730{
10731 if (dev->ethtool_ops == &default_ethtool_ops)
10732 dev->ethtool_ops = ops;
10733}
10734EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
10735
10736/**
10737 * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
10738 * @dev: netdev to enable the IRQ coalescing on
10739 *
10740 * Sets a conservative default for SW IRQ coalescing. Users can use
10741 * sysfs attributes to override the default values.
10742 */
10743void netdev_sw_irq_coalesce_default_on(struct net_device *dev)
10744{
10745 WARN_ON(dev->reg_state == NETREG_REGISTERED);
10746
10747 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
10748 dev->gro_flush_timeout = 20000;
10749 dev->napi_defer_hard_irqs = 1;
10750 }
10751}
10752EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on);
10753
10754void netdev_freemem(struct net_device *dev)
10755{
10756 char *addr = (char *)dev - dev->padded;
10757
10758 kvfree(addr);
10759}
10760
10761/**
10762 * alloc_netdev_mqs - allocate network device
10763 * @sizeof_priv: size of private data to allocate space for
10764 * @name: device name format string
10765 * @name_assign_type: origin of device name
10766 * @setup: callback to initialize device
10767 * @txqs: the number of TX subqueues to allocate
10768 * @rxqs: the number of RX subqueues to allocate
10769 *
10770 * Allocates a struct net_device with private data area for driver use
10771 * and performs basic initialization. Also allocates subqueue structs
10772 * for each queue on the device.
10773 */
10774struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
10775 unsigned char name_assign_type,
10776 void (*setup)(struct net_device *),
10777 unsigned int txqs, unsigned int rxqs)
10778{
10779 struct net_device *dev;
10780 unsigned int alloc_size;
10781 struct net_device *p;
10782
10783 BUG_ON(strlen(name) >= sizeof(dev->name));
10784
10785 if (txqs < 1) {
10786 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
10787 return NULL;
10788 }
10789
10790 if (rxqs < 1) {
10791 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
10792 return NULL;
10793 }
10794
10795 alloc_size = sizeof(struct net_device);
10796 if (sizeof_priv) {
10797 /* ensure 32-byte alignment of private area */
10798 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
10799 alloc_size += sizeof_priv;
10800 }
10801 /* ensure 32-byte alignment of whole construct */
10802 alloc_size += NETDEV_ALIGN - 1;
10803
10804 p = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
10805 if (!p)
10806 return NULL;
10807
10808 dev = PTR_ALIGN(p, NETDEV_ALIGN);
10809 dev->padded = (char *)dev - (char *)p;
10810
10811 ref_tracker_dir_init(&dev->refcnt_tracker, 128, name);
10812#ifdef CONFIG_PCPU_DEV_REFCNT
10813 dev->pcpu_refcnt = alloc_percpu(int);
10814 if (!dev->pcpu_refcnt)
10815 goto free_dev;
10816 __dev_hold(dev);
10817#else
10818 refcount_set(&dev->dev_refcnt, 1);
10819#endif
10820
10821 if (dev_addr_init(dev))
10822 goto free_pcpu;
10823
10824 dev_mc_init(dev);
10825 dev_uc_init(dev);
10826
10827 dev_net_set(dev, &init_net);
10828
10829 dev->gso_max_size = GSO_LEGACY_MAX_SIZE;
10830 dev->xdp_zc_max_segs = 1;
10831 dev->gso_max_segs = GSO_MAX_SEGS;
10832 dev->gro_max_size = GRO_LEGACY_MAX_SIZE;
10833 dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE;
10834 dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE;
10835 dev->tso_max_size = TSO_LEGACY_MAX_SIZE;
10836 dev->tso_max_segs = TSO_MAX_SEGS;
10837 dev->upper_level = 1;
10838 dev->lower_level = 1;
10839#ifdef CONFIG_LOCKDEP
10840 dev->nested_level = 0;
10841 INIT_LIST_HEAD(&dev->unlink_list);
10842#endif
10843
10844 INIT_LIST_HEAD(&dev->napi_list);
10845 INIT_LIST_HEAD(&dev->unreg_list);
10846 INIT_LIST_HEAD(&dev->close_list);
10847 INIT_LIST_HEAD(&dev->link_watch_list);
10848 INIT_LIST_HEAD(&dev->adj_list.upper);
10849 INIT_LIST_HEAD(&dev->adj_list.lower);
10850 INIT_LIST_HEAD(&dev->ptype_all);
10851 INIT_LIST_HEAD(&dev->ptype_specific);
10852 INIT_LIST_HEAD(&dev->net_notifier_list);
10853#ifdef CONFIG_NET_SCHED
10854 hash_init(dev->qdisc_hash);
10855#endif
10856 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
10857 setup(dev);
10858
10859 if (!dev->tx_queue_len) {
10860 dev->priv_flags |= IFF_NO_QUEUE;
10861 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
10862 }
10863
10864 dev->num_tx_queues = txqs;
10865 dev->real_num_tx_queues = txqs;
10866 if (netif_alloc_netdev_queues(dev))
10867 goto free_all;
10868
10869 dev->num_rx_queues = rxqs;
10870 dev->real_num_rx_queues = rxqs;
10871 if (netif_alloc_rx_queues(dev))
10872 goto free_all;
10873
10874 strcpy(dev->name, name);
10875 dev->name_assign_type = name_assign_type;
10876 dev->group = INIT_NETDEV_GROUP;
10877 if (!dev->ethtool_ops)
10878 dev->ethtool_ops = &default_ethtool_ops;
10879
10880 nf_hook_netdev_init(dev);
10881
10882 return dev;
10883
10884free_all:
10885 free_netdev(dev);
10886 return NULL;
10887
10888free_pcpu:
10889#ifdef CONFIG_PCPU_DEV_REFCNT
10890 free_percpu(dev->pcpu_refcnt);
10891free_dev:
10892#endif
10893 netdev_freemem(dev);
10894 return NULL;
10895}
10896EXPORT_SYMBOL(alloc_netdev_mqs);
10897
10898/**
10899 * free_netdev - free network device
10900 * @dev: device
10901 *
10902 * This function does the last stage of destroying an allocated device
10903 * interface. The reference to the device object is released. If this
10904 * is the last reference then it will be freed.Must be called in process
10905 * context.
10906 */
10907void free_netdev(struct net_device *dev)
10908{
10909 struct napi_struct *p, *n;
10910
10911 might_sleep();
10912
10913 /* When called immediately after register_netdevice() failed the unwind
10914 * handling may still be dismantling the device. Handle that case by
10915 * deferring the free.
10916 */
10917 if (dev->reg_state == NETREG_UNREGISTERING) {
10918 ASSERT_RTNL();
10919 dev->needs_free_netdev = true;
10920 return;
10921 }
10922
10923 netif_free_tx_queues(dev);
10924 netif_free_rx_queues(dev);
10925
10926 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
10927
10928 /* Flush device addresses */
10929 dev_addr_flush(dev);
10930
10931 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
10932 netif_napi_del(p);
10933
10934 ref_tracker_dir_exit(&dev->refcnt_tracker);
10935#ifdef CONFIG_PCPU_DEV_REFCNT
10936 free_percpu(dev->pcpu_refcnt);
10937 dev->pcpu_refcnt = NULL;
10938#endif
10939 free_percpu(dev->core_stats);
10940 dev->core_stats = NULL;
10941 free_percpu(dev->xdp_bulkq);
10942 dev->xdp_bulkq = NULL;
10943
10944 /* Compatibility with error handling in drivers */
10945 if (dev->reg_state == NETREG_UNINITIALIZED) {
10946 netdev_freemem(dev);
10947 return;
10948 }
10949
10950 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
10951 dev->reg_state = NETREG_RELEASED;
10952
10953 /* will free via device release */
10954 put_device(&dev->dev);
10955}
10956EXPORT_SYMBOL(free_netdev);
10957
10958/**
10959 * synchronize_net - Synchronize with packet receive processing
10960 *
10961 * Wait for packets currently being received to be done.
10962 * Does not block later packets from starting.
10963 */
10964void synchronize_net(void)
10965{
10966 might_sleep();
10967 if (rtnl_is_locked())
10968 synchronize_rcu_expedited();
10969 else
10970 synchronize_rcu();
10971}
10972EXPORT_SYMBOL(synchronize_net);
10973
10974/**
10975 * unregister_netdevice_queue - remove device from the kernel
10976 * @dev: device
10977 * @head: list
10978 *
10979 * This function shuts down a device interface and removes it
10980 * from the kernel tables.
10981 * If head not NULL, device is queued to be unregistered later.
10982 *
10983 * Callers must hold the rtnl semaphore. You may want
10984 * unregister_netdev() instead of this.
10985 */
10986
10987void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
10988{
10989 ASSERT_RTNL();
10990
10991 if (head) {
10992 list_move_tail(&dev->unreg_list, head);
10993 } else {
10994 LIST_HEAD(single);
10995
10996 list_add(&dev->unreg_list, &single);
10997 unregister_netdevice_many(&single);
10998 }
10999}
11000EXPORT_SYMBOL(unregister_netdevice_queue);
11001
11002void unregister_netdevice_many_notify(struct list_head *head,
11003 u32 portid, const struct nlmsghdr *nlh)
11004{
11005 struct net_device *dev, *tmp;
11006 LIST_HEAD(close_head);
11007
11008 BUG_ON(dev_boot_phase);
11009 ASSERT_RTNL();
11010
11011 if (list_empty(head))
11012 return;
11013
11014 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
11015 /* Some devices call without registering
11016 * for initialization unwind. Remove those
11017 * devices and proceed with the remaining.
11018 */
11019 if (dev->reg_state == NETREG_UNINITIALIZED) {
11020 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
11021 dev->name, dev);
11022
11023 WARN_ON(1);
11024 list_del(&dev->unreg_list);
11025 continue;
11026 }
11027 dev->dismantle = true;
11028 BUG_ON(dev->reg_state != NETREG_REGISTERED);
11029 }
11030
11031 /* If device is running, close it first. */
11032 list_for_each_entry(dev, head, unreg_list)
11033 list_add_tail(&dev->close_list, &close_head);
11034 dev_close_many(&close_head, true);
11035
11036 list_for_each_entry(dev, head, unreg_list) {
11037 /* And unlink it from device chain. */
11038 write_lock(&dev_base_lock);
11039 unlist_netdevice(dev, false);
11040 dev->reg_state = NETREG_UNREGISTERING;
11041 write_unlock(&dev_base_lock);
11042 }
11043 flush_all_backlogs();
11044
11045 synchronize_net();
11046
11047 list_for_each_entry(dev, head, unreg_list) {
11048 struct sk_buff *skb = NULL;
11049
11050 /* Shutdown queueing discipline. */
11051 dev_shutdown(dev);
11052 dev_tcx_uninstall(dev);
11053 dev_xdp_uninstall(dev);
11054 bpf_dev_bound_netdev_unregister(dev);
11055
11056 netdev_offload_xstats_disable_all(dev);
11057
11058 /* Notify protocols, that we are about to destroy
11059 * this device. They should clean all the things.
11060 */
11061 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11062
11063 if (!dev->rtnl_link_ops ||
11064 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
11065 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
11066 GFP_KERNEL, NULL, 0,
11067 portid, nlh);
11068
11069 /*
11070 * Flush the unicast and multicast chains
11071 */
11072 dev_uc_flush(dev);
11073 dev_mc_flush(dev);
11074
11075 netdev_name_node_alt_flush(dev);
11076 netdev_name_node_free(dev->name_node);
11077
11078 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
11079
11080 if (dev->netdev_ops->ndo_uninit)
11081 dev->netdev_ops->ndo_uninit(dev);
11082
11083 if (skb)
11084 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh);
11085
11086 /* Notifier chain MUST detach us all upper devices. */
11087 WARN_ON(netdev_has_any_upper_dev(dev));
11088 WARN_ON(netdev_has_any_lower_dev(dev));
11089
11090 /* Remove entries from kobject tree */
11091 netdev_unregister_kobject(dev);
11092#ifdef CONFIG_XPS
11093 /* Remove XPS queueing entries */
11094 netif_reset_xps_queues_gt(dev, 0);
11095#endif
11096 }
11097
11098 synchronize_net();
11099
11100 list_for_each_entry(dev, head, unreg_list) {
11101 netdev_put(dev, &dev->dev_registered_tracker);
11102 net_set_todo(dev);
11103 }
11104
11105 list_del(head);
11106}
11107
11108/**
11109 * unregister_netdevice_many - unregister many devices
11110 * @head: list of devices
11111 *
11112 * Note: As most callers use a stack allocated list_head,
11113 * we force a list_del() to make sure stack wont be corrupted later.
11114 */
11115void unregister_netdevice_many(struct list_head *head)
11116{
11117 unregister_netdevice_many_notify(head, 0, NULL);
11118}
11119EXPORT_SYMBOL(unregister_netdevice_many);
11120
11121/**
11122 * unregister_netdev - remove device from the kernel
11123 * @dev: device
11124 *
11125 * This function shuts down a device interface and removes it
11126 * from the kernel tables.
11127 *
11128 * This is just a wrapper for unregister_netdevice that takes
11129 * the rtnl semaphore. In general you want to use this and not
11130 * unregister_netdevice.
11131 */
11132void unregister_netdev(struct net_device *dev)
11133{
11134 rtnl_lock();
11135 unregister_netdevice(dev);
11136 rtnl_unlock();
11137}
11138EXPORT_SYMBOL(unregister_netdev);
11139
11140/**
11141 * __dev_change_net_namespace - move device to different nethost namespace
11142 * @dev: device
11143 * @net: network namespace
11144 * @pat: If not NULL name pattern to try if the current device name
11145 * is already taken in the destination network namespace.
11146 * @new_ifindex: If not zero, specifies device index in the target
11147 * namespace.
11148 *
11149 * This function shuts down a device interface and moves it
11150 * to a new network namespace. On success 0 is returned, on
11151 * a failure a netagive errno code is returned.
11152 *
11153 * Callers must hold the rtnl semaphore.
11154 */
11155
11156int __dev_change_net_namespace(struct net_device *dev, struct net *net,
11157 const char *pat, int new_ifindex)
11158{
11159 struct netdev_name_node *name_node;
11160 struct net *net_old = dev_net(dev);
11161 char new_name[IFNAMSIZ] = {};
11162 int err, new_nsid;
11163
11164 ASSERT_RTNL();
11165
11166 /* Don't allow namespace local devices to be moved. */
11167 err = -EINVAL;
11168 if (dev->features & NETIF_F_NETNS_LOCAL)
11169 goto out;
11170
11171 /* Ensure the device has been registrered */
11172 if (dev->reg_state != NETREG_REGISTERED)
11173 goto out;
11174
11175 /* Get out if there is nothing todo */
11176 err = 0;
11177 if (net_eq(net_old, net))
11178 goto out;
11179
11180 /* Pick the destination device name, and ensure
11181 * we can use it in the destination network namespace.
11182 */
11183 err = -EEXIST;
11184 if (netdev_name_in_use(net, dev->name)) {
11185 /* We get here if we can't use the current device name */
11186 if (!pat)
11187 goto out;
11188 err = dev_prep_valid_name(net, dev, pat, new_name, EEXIST);
11189 if (err < 0)
11190 goto out;
11191 }
11192 /* Check that none of the altnames conflicts. */
11193 err = -EEXIST;
11194 netdev_for_each_altname(dev, name_node)
11195 if (netdev_name_in_use(net, name_node->name))
11196 goto out;
11197
11198 /* Check that new_ifindex isn't used yet. */
11199 if (new_ifindex) {
11200 err = dev_index_reserve(net, new_ifindex);
11201 if (err < 0)
11202 goto out;
11203 } else {
11204 /* If there is an ifindex conflict assign a new one */
11205 err = dev_index_reserve(net, dev->ifindex);
11206 if (err == -EBUSY)
11207 err = dev_index_reserve(net, 0);
11208 if (err < 0)
11209 goto out;
11210 new_ifindex = err;
11211 }
11212
11213 /*
11214 * And now a mini version of register_netdevice unregister_netdevice.
11215 */
11216
11217 /* If device is running close it first. */
11218 dev_close(dev);
11219
11220 /* And unlink it from device chain */
11221 unlist_netdevice(dev, true);
11222
11223 synchronize_net();
11224
11225 /* Shutdown queueing discipline. */
11226 dev_shutdown(dev);
11227
11228 /* Notify protocols, that we are about to destroy
11229 * this device. They should clean all the things.
11230 *
11231 * Note that dev->reg_state stays at NETREG_REGISTERED.
11232 * This is wanted because this way 8021q and macvlan know
11233 * the device is just moving and can keep their slaves up.
11234 */
11235 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11236 rcu_barrier();
11237
11238 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
11239
11240 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11241 new_ifindex);
11242
11243 /*
11244 * Flush the unicast and multicast chains
11245 */
11246 dev_uc_flush(dev);
11247 dev_mc_flush(dev);
11248
11249 /* Send a netdev-removed uevent to the old namespace */
11250 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
11251 netdev_adjacent_del_links(dev);
11252
11253 /* Move per-net netdevice notifiers that are following the netdevice */
11254 move_netdevice_notifiers_dev_net(dev, net);
11255
11256 /* Actually switch the network namespace */
11257 dev_net_set(dev, net);
11258 dev->ifindex = new_ifindex;
11259
11260 if (new_name[0]) /* Rename the netdev to prepared name */
11261 strscpy(dev->name, new_name, IFNAMSIZ);
11262
11263 /* Fixup kobjects */
11264 dev_set_uevent_suppress(&dev->dev, 1);
11265 err = device_rename(&dev->dev, dev->name);
11266 dev_set_uevent_suppress(&dev->dev, 0);
11267 WARN_ON(err);
11268
11269 /* Send a netdev-add uevent to the new namespace */
11270 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
11271 netdev_adjacent_add_links(dev);
11272
11273 /* Adapt owner in case owning user namespace of target network
11274 * namespace is different from the original one.
11275 */
11276 err = netdev_change_owner(dev, net_old, net);
11277 WARN_ON(err);
11278
11279 /* Add the device back in the hashes */
11280 list_netdevice(dev);
11281
11282 /* Notify protocols, that a new device appeared. */
11283 call_netdevice_notifiers(NETDEV_REGISTER, dev);
11284
11285 /*
11286 * Prevent userspace races by waiting until the network
11287 * device is fully setup before sending notifications.
11288 */
11289 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
11290
11291 synchronize_net();
11292 err = 0;
11293out:
11294 return err;
11295}
11296EXPORT_SYMBOL_GPL(__dev_change_net_namespace);
11297
11298static int dev_cpu_dead(unsigned int oldcpu)
11299{
11300 struct sk_buff **list_skb;
11301 struct sk_buff *skb;
11302 unsigned int cpu;
11303 struct softnet_data *sd, *oldsd, *remsd = NULL;
11304
11305 local_irq_disable();
11306 cpu = smp_processor_id();
11307 sd = &per_cpu(softnet_data, cpu);
11308 oldsd = &per_cpu(softnet_data, oldcpu);
11309
11310 /* Find end of our completion_queue. */
11311 list_skb = &sd->completion_queue;
11312 while (*list_skb)
11313 list_skb = &(*list_skb)->next;
11314 /* Append completion queue from offline CPU. */
11315 *list_skb = oldsd->completion_queue;
11316 oldsd->completion_queue = NULL;
11317
11318 /* Append output queue from offline CPU. */
11319 if (oldsd->output_queue) {
11320 *sd->output_queue_tailp = oldsd->output_queue;
11321 sd->output_queue_tailp = oldsd->output_queue_tailp;
11322 oldsd->output_queue = NULL;
11323 oldsd->output_queue_tailp = &oldsd->output_queue;
11324 }
11325 /* Append NAPI poll list from offline CPU, with one exception :
11326 * process_backlog() must be called by cpu owning percpu backlog.
11327 * We properly handle process_queue & input_pkt_queue later.
11328 */
11329 while (!list_empty(&oldsd->poll_list)) {
11330 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11331 struct napi_struct,
11332 poll_list);
11333
11334 list_del_init(&napi->poll_list);
11335 if (napi->poll == process_backlog)
11336 napi->state = 0;
11337 else
11338 ____napi_schedule(sd, napi);
11339 }
11340
11341 raise_softirq_irqoff(NET_TX_SOFTIRQ);
11342 local_irq_enable();
11343
11344#ifdef CONFIG_RPS
11345 remsd = oldsd->rps_ipi_list;
11346 oldsd->rps_ipi_list = NULL;
11347#endif
11348 /* send out pending IPI's on offline CPU */
11349 net_rps_send_ipi(remsd);
11350
11351 /* Process offline CPU's input_pkt_queue */
11352 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
11353 netif_rx(skb);
11354 input_queue_head_incr(oldsd);
11355 }
11356 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
11357 netif_rx(skb);
11358 input_queue_head_incr(oldsd);
11359 }
11360
11361 return 0;
11362}
11363
11364/**
11365 * netdev_increment_features - increment feature set by one
11366 * @all: current feature set
11367 * @one: new feature set
11368 * @mask: mask feature set
11369 *
11370 * Computes a new feature set after adding a device with feature set
11371 * @one to the master device with current feature set @all. Will not
11372 * enable anything that is off in @mask. Returns the new feature set.
11373 */
11374netdev_features_t netdev_increment_features(netdev_features_t all,
11375 netdev_features_t one, netdev_features_t mask)
11376{
11377 if (mask & NETIF_F_HW_CSUM)
11378 mask |= NETIF_F_CSUM_MASK;
11379 mask |= NETIF_F_VLAN_CHALLENGED;
11380
11381 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
11382 all &= one | ~NETIF_F_ALL_FOR_ALL;
11383
11384 /* If one device supports hw checksumming, set for all. */
11385 if (all & NETIF_F_HW_CSUM)
11386 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
11387
11388 return all;
11389}
11390EXPORT_SYMBOL(netdev_increment_features);
11391
11392static struct hlist_head * __net_init netdev_create_hash(void)
11393{
11394 int i;
11395 struct hlist_head *hash;
11396
11397 hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
11398 if (hash != NULL)
11399 for (i = 0; i < NETDEV_HASHENTRIES; i++)
11400 INIT_HLIST_HEAD(&hash[i]);
11401
11402 return hash;
11403}
11404
11405/* Initialize per network namespace state */
11406static int __net_init netdev_init(struct net *net)
11407{
11408 BUILD_BUG_ON(GRO_HASH_BUCKETS >
11409 8 * sizeof_field(struct napi_struct, gro_bitmask));
11410
11411 INIT_LIST_HEAD(&net->dev_base_head);
11412
11413 net->dev_name_head = netdev_create_hash();
11414 if (net->dev_name_head == NULL)
11415 goto err_name;
11416
11417 net->dev_index_head = netdev_create_hash();
11418 if (net->dev_index_head == NULL)
11419 goto err_idx;
11420
11421 xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1);
11422
11423 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
11424
11425 return 0;
11426
11427err_idx:
11428 kfree(net->dev_name_head);
11429err_name:
11430 return -ENOMEM;
11431}
11432
11433/**
11434 * netdev_drivername - network driver for the device
11435 * @dev: network device
11436 *
11437 * Determine network driver for device.
11438 */
11439const char *netdev_drivername(const struct net_device *dev)
11440{
11441 const struct device_driver *driver;
11442 const struct device *parent;
11443 const char *empty = "";
11444
11445 parent = dev->dev.parent;
11446 if (!parent)
11447 return empty;
11448
11449 driver = parent->driver;
11450 if (driver && driver->name)
11451 return driver->name;
11452 return empty;
11453}
11454
11455static void __netdev_printk(const char *level, const struct net_device *dev,
11456 struct va_format *vaf)
11457{
11458 if (dev && dev->dev.parent) {
11459 dev_printk_emit(level[1] - '0',
11460 dev->dev.parent,
11461 "%s %s %s%s: %pV",
11462 dev_driver_string(dev->dev.parent),
11463 dev_name(dev->dev.parent),
11464 netdev_name(dev), netdev_reg_state(dev),
11465 vaf);
11466 } else if (dev) {
11467 printk("%s%s%s: %pV",
11468 level, netdev_name(dev), netdev_reg_state(dev), vaf);
11469 } else {
11470 printk("%s(NULL net_device): %pV", level, vaf);
11471 }
11472}
11473
11474void netdev_printk(const char *level, const struct net_device *dev,
11475 const char *format, ...)
11476{
11477 struct va_format vaf;
11478 va_list args;
11479
11480 va_start(args, format);
11481
11482 vaf.fmt = format;
11483 vaf.va = &args;
11484
11485 __netdev_printk(level, dev, &vaf);
11486
11487 va_end(args);
11488}
11489EXPORT_SYMBOL(netdev_printk);
11490
11491#define define_netdev_printk_level(func, level) \
11492void func(const struct net_device *dev, const char *fmt, ...) \
11493{ \
11494 struct va_format vaf; \
11495 va_list args; \
11496 \
11497 va_start(args, fmt); \
11498 \
11499 vaf.fmt = fmt; \
11500 vaf.va = &args; \
11501 \
11502 __netdev_printk(level, dev, &vaf); \
11503 \
11504 va_end(args); \
11505} \
11506EXPORT_SYMBOL(func);
11507
11508define_netdev_printk_level(netdev_emerg, KERN_EMERG);
11509define_netdev_printk_level(netdev_alert, KERN_ALERT);
11510define_netdev_printk_level(netdev_crit, KERN_CRIT);
11511define_netdev_printk_level(netdev_err, KERN_ERR);
11512define_netdev_printk_level(netdev_warn, KERN_WARNING);
11513define_netdev_printk_level(netdev_notice, KERN_NOTICE);
11514define_netdev_printk_level(netdev_info, KERN_INFO);
11515
11516static void __net_exit netdev_exit(struct net *net)
11517{
11518 kfree(net->dev_name_head);
11519 kfree(net->dev_index_head);
11520 xa_destroy(&net->dev_by_index);
11521 if (net != &init_net)
11522 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
11523}
11524
11525static struct pernet_operations __net_initdata netdev_net_ops = {
11526 .init = netdev_init,
11527 .exit = netdev_exit,
11528};
11529
11530static void __net_exit default_device_exit_net(struct net *net)
11531{
11532 struct netdev_name_node *name_node, *tmp;
11533 struct net_device *dev, *aux;
11534 /*
11535 * Push all migratable network devices back to the
11536 * initial network namespace
11537 */
11538 ASSERT_RTNL();
11539 for_each_netdev_safe(net, dev, aux) {
11540 int err;
11541 char fb_name[IFNAMSIZ];
11542
11543 /* Ignore unmoveable devices (i.e. loopback) */
11544 if (dev->features & NETIF_F_NETNS_LOCAL)
11545 continue;
11546
11547 /* Leave virtual devices for the generic cleanup */
11548 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
11549 continue;
11550
11551 /* Push remaining network devices to init_net */
11552 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
11553 if (netdev_name_in_use(&init_net, fb_name))
11554 snprintf(fb_name, IFNAMSIZ, "dev%%d");
11555
11556 netdev_for_each_altname_safe(dev, name_node, tmp)
11557 if (netdev_name_in_use(&init_net, name_node->name)) {
11558 netdev_name_node_del(name_node);
11559 synchronize_rcu();
11560 __netdev_name_node_alt_destroy(name_node);
11561 }
11562
11563 err = dev_change_net_namespace(dev, &init_net, fb_name);
11564 if (err) {
11565 pr_emerg("%s: failed to move %s to init_net: %d\n",
11566 __func__, dev->name, err);
11567 BUG();
11568 }
11569 }
11570}
11571
11572static void __net_exit default_device_exit_batch(struct list_head *net_list)
11573{
11574 /* At exit all network devices most be removed from a network
11575 * namespace. Do this in the reverse order of registration.
11576 * Do this across as many network namespaces as possible to
11577 * improve batching efficiency.
11578 */
11579 struct net_device *dev;
11580 struct net *net;
11581 LIST_HEAD(dev_kill_list);
11582
11583 rtnl_lock();
11584 list_for_each_entry(net, net_list, exit_list) {
11585 default_device_exit_net(net);
11586 cond_resched();
11587 }
11588
11589 list_for_each_entry(net, net_list, exit_list) {
11590 for_each_netdev_reverse(net, dev) {
11591 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
11592 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
11593 else
11594 unregister_netdevice_queue(dev, &dev_kill_list);
11595 }
11596 }
11597 unregister_netdevice_many(&dev_kill_list);
11598 rtnl_unlock();
11599}
11600
11601static struct pernet_operations __net_initdata default_device_ops = {
11602 .exit_batch = default_device_exit_batch,
11603};
11604
11605static void __init net_dev_struct_check(void)
11606{
11607 /* TX read-mostly hotpath */
11608 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags);
11609 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops);
11610 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops);
11611 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx);
11612 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, real_num_tx_queues);
11613 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size);
11614 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size);
11615 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs);
11616 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_partial_features);
11617 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc);
11618 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu);
11619 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom);
11620 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tc_to_txq);
11621#ifdef CONFIG_XPS
11622 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, xps_maps);
11623#endif
11624#ifdef CONFIG_NETFILTER_EGRESS
11625 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, nf_hooks_egress);
11626#endif
11627#ifdef CONFIG_NET_XGRESS
11628 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress);
11629#endif
11630 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 160);
11631
11632 /* TXRX read-mostly hotpath */
11633 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, lstats);
11634 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags);
11635 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len);
11636 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features);
11637 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr);
11638 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 38);
11639
11640 /* RX read-mostly hotpath */
11641 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific);
11642 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ifindex);
11643 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, real_num_rx_queues);
11644 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, _rx);
11645 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_flush_timeout);
11646 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, napi_defer_hard_irqs);
11647 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_max_size);
11648 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_ipv4_max_size);
11649 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler);
11650 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler_data);
11651 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, nd_net);
11652#ifdef CONFIG_NETPOLL
11653 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, npinfo);
11654#endif
11655#ifdef CONFIG_NET_XGRESS
11656 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress);
11657#endif
11658 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 104);
11659}
11660
11661/*
11662 * Initialize the DEV module. At boot time this walks the device list and
11663 * unhooks any devices that fail to initialise (normally hardware not
11664 * present) and leaves us with a valid list of present and active devices.
11665 *
11666 */
11667
11668/*
11669 * This is called single threaded during boot, so no need
11670 * to take the rtnl semaphore.
11671 */
11672static int __init net_dev_init(void)
11673{
11674 int i, rc = -ENOMEM;
11675
11676 BUG_ON(!dev_boot_phase);
11677
11678 net_dev_struct_check();
11679
11680 if (dev_proc_init())
11681 goto out;
11682
11683 if (netdev_kobject_init())
11684 goto out;
11685
11686 INIT_LIST_HEAD(&ptype_all);
11687 for (i = 0; i < PTYPE_HASH_SIZE; i++)
11688 INIT_LIST_HEAD(&ptype_base[i]);
11689
11690 if (register_pernet_subsys(&netdev_net_ops))
11691 goto out;
11692
11693 /*
11694 * Initialise the packet receive queues.
11695 */
11696
11697 for_each_possible_cpu(i) {
11698 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
11699 struct softnet_data *sd = &per_cpu(softnet_data, i);
11700
11701 INIT_WORK(flush, flush_backlog);
11702
11703 skb_queue_head_init(&sd->input_pkt_queue);
11704 skb_queue_head_init(&sd->process_queue);
11705#ifdef CONFIG_XFRM_OFFLOAD
11706 skb_queue_head_init(&sd->xfrm_backlog);
11707#endif
11708 INIT_LIST_HEAD(&sd->poll_list);
11709 sd->output_queue_tailp = &sd->output_queue;
11710#ifdef CONFIG_RPS
11711 INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
11712 sd->cpu = i;
11713#endif
11714 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
11715 spin_lock_init(&sd->defer_lock);
11716
11717 init_gro_hash(&sd->backlog);
11718 sd->backlog.poll = process_backlog;
11719 sd->backlog.weight = weight_p;
11720 }
11721
11722 dev_boot_phase = 0;
11723
11724 /* The loopback device is special if any other network devices
11725 * is present in a network namespace the loopback device must
11726 * be present. Since we now dynamically allocate and free the
11727 * loopback device ensure this invariant is maintained by
11728 * keeping the loopback device as the first device on the
11729 * list of network devices. Ensuring the loopback devices
11730 * is the first device that appears and the last network device
11731 * that disappears.
11732 */
11733 if (register_pernet_device(&loopback_net_ops))
11734 goto out;
11735
11736 if (register_pernet_device(&default_device_ops))
11737 goto out;
11738
11739 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
11740 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
11741
11742 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
11743 NULL, dev_cpu_dead);
11744 WARN_ON(rc < 0);
11745 rc = 0;
11746out:
11747 return rc;
11748}
11749
11750subsys_initcall(net_dev_init);
1/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
76#include <linux/bitops.h>
77#include <linux/capability.h>
78#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
81#include <linux/hash.h>
82#include <linux/slab.h>
83#include <linux/sched.h>
84#include <linux/mutex.h>
85#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
94#include <linux/ethtool.h>
95#include <linux/notifier.h>
96#include <linux/skbuff.h>
97#include <net/net_namespace.h>
98#include <net/sock.h>
99#include <linux/rtnetlink.h>
100#include <linux/stat.h>
101#include <net/dst.h>
102#include <net/pkt_sched.h>
103#include <net/checksum.h>
104#include <net/xfrm.h>
105#include <linux/highmem.h>
106#include <linux/init.h>
107#include <linux/module.h>
108#include <linux/netpoll.h>
109#include <linux/rcupdate.h>
110#include <linux/delay.h>
111#include <net/iw_handler.h>
112#include <asm/current.h>
113#include <linux/audit.h>
114#include <linux/dmaengine.h>
115#include <linux/err.h>
116#include <linux/ctype.h>
117#include <linux/if_arp.h>
118#include <linux/if_vlan.h>
119#include <linux/ip.h>
120#include <net/ip.h>
121#include <linux/ipv6.h>
122#include <linux/in.h>
123#include <linux/jhash.h>
124#include <linux/random.h>
125#include <trace/events/napi.h>
126#include <trace/events/net.h>
127#include <trace/events/skb.h>
128#include <linux/pci.h>
129#include <linux/inetdevice.h>
130#include <linux/cpu_rmap.h>
131#include <linux/static_key.h>
132#include <linux/hashtable.h>
133#include <linux/vmalloc.h>
134#include <linux/if_macvlan.h>
135
136#include "net-sysfs.h"
137
138/* Instead of increasing this, you should create a hash table. */
139#define MAX_GRO_SKBS 8
140
141/* This should be increased if a protocol with a bigger head is added. */
142#define GRO_MAX_HEAD (MAX_HEADER + 128)
143
144static DEFINE_SPINLOCK(ptype_lock);
145static DEFINE_SPINLOCK(offload_lock);
146struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
147struct list_head ptype_all __read_mostly; /* Taps */
148static struct list_head offload_base __read_mostly;
149
150static int netif_rx_internal(struct sk_buff *skb);
151
152/*
153 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
154 * semaphore.
155 *
156 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
157 *
158 * Writers must hold the rtnl semaphore while they loop through the
159 * dev_base_head list, and hold dev_base_lock for writing when they do the
160 * actual updates. This allows pure readers to access the list even
161 * while a writer is preparing to update it.
162 *
163 * To put it another way, dev_base_lock is held for writing only to
164 * protect against pure readers; the rtnl semaphore provides the
165 * protection against other writers.
166 *
167 * See, for example usages, register_netdevice() and
168 * unregister_netdevice(), which must be called with the rtnl
169 * semaphore held.
170 */
171DEFINE_RWLOCK(dev_base_lock);
172EXPORT_SYMBOL(dev_base_lock);
173
174/* protects napi_hash addition/deletion and napi_gen_id */
175static DEFINE_SPINLOCK(napi_hash_lock);
176
177static unsigned int napi_gen_id;
178static DEFINE_HASHTABLE(napi_hash, 8);
179
180static seqcount_t devnet_rename_seq;
181
182static inline void dev_base_seq_inc(struct net *net)
183{
184 while (++net->dev_base_seq == 0);
185}
186
187static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
188{
189 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
190
191 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
192}
193
194static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
195{
196 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
197}
198
199static inline void rps_lock(struct softnet_data *sd)
200{
201#ifdef CONFIG_RPS
202 spin_lock(&sd->input_pkt_queue.lock);
203#endif
204}
205
206static inline void rps_unlock(struct softnet_data *sd)
207{
208#ifdef CONFIG_RPS
209 spin_unlock(&sd->input_pkt_queue.lock);
210#endif
211}
212
213/* Device list insertion */
214static void list_netdevice(struct net_device *dev)
215{
216 struct net *net = dev_net(dev);
217
218 ASSERT_RTNL();
219
220 write_lock_bh(&dev_base_lock);
221 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
222 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
223 hlist_add_head_rcu(&dev->index_hlist,
224 dev_index_hash(net, dev->ifindex));
225 write_unlock_bh(&dev_base_lock);
226
227 dev_base_seq_inc(net);
228}
229
230/* Device list removal
231 * caller must respect a RCU grace period before freeing/reusing dev
232 */
233static void unlist_netdevice(struct net_device *dev)
234{
235 ASSERT_RTNL();
236
237 /* Unlink dev from the device chain */
238 write_lock_bh(&dev_base_lock);
239 list_del_rcu(&dev->dev_list);
240 hlist_del_rcu(&dev->name_hlist);
241 hlist_del_rcu(&dev->index_hlist);
242 write_unlock_bh(&dev_base_lock);
243
244 dev_base_seq_inc(dev_net(dev));
245}
246
247/*
248 * Our notifier list
249 */
250
251static RAW_NOTIFIER_HEAD(netdev_chain);
252
253/*
254 * Device drivers call our routines to queue packets here. We empty the
255 * queue in the local softnet handler.
256 */
257
258DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
259EXPORT_PER_CPU_SYMBOL(softnet_data);
260
261#ifdef CONFIG_LOCKDEP
262/*
263 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
264 * according to dev->type
265 */
266static const unsigned short netdev_lock_type[] =
267 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
268 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
269 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
270 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
271 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
272 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
273 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
274 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
275 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
276 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
277 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
278 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
279 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
280 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
281 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
282
283static const char *const netdev_lock_name[] =
284 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
285 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
286 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
287 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
288 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
289 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
290 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
291 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
292 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
293 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
294 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
295 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
296 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
297 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
298 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
299
300static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
301static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
302
303static inline unsigned short netdev_lock_pos(unsigned short dev_type)
304{
305 int i;
306
307 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
308 if (netdev_lock_type[i] == dev_type)
309 return i;
310 /* the last key is used by default */
311 return ARRAY_SIZE(netdev_lock_type) - 1;
312}
313
314static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
315 unsigned short dev_type)
316{
317 int i;
318
319 i = netdev_lock_pos(dev_type);
320 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
321 netdev_lock_name[i]);
322}
323
324static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
325{
326 int i;
327
328 i = netdev_lock_pos(dev->type);
329 lockdep_set_class_and_name(&dev->addr_list_lock,
330 &netdev_addr_lock_key[i],
331 netdev_lock_name[i]);
332}
333#else
334static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
335 unsigned short dev_type)
336{
337}
338static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
339{
340}
341#endif
342
343/*******************************************************************************
344
345 Protocol management and registration routines
346
347*******************************************************************************/
348
349/*
350 * Add a protocol ID to the list. Now that the input handler is
351 * smarter we can dispense with all the messy stuff that used to be
352 * here.
353 *
354 * BEWARE!!! Protocol handlers, mangling input packets,
355 * MUST BE last in hash buckets and checking protocol handlers
356 * MUST start from promiscuous ptype_all chain in net_bh.
357 * It is true now, do not change it.
358 * Explanation follows: if protocol handler, mangling packet, will
359 * be the first on list, it is not able to sense, that packet
360 * is cloned and should be copied-on-write, so that it will
361 * change it and subsequent readers will get broken packet.
362 * --ANK (980803)
363 */
364
365static inline struct list_head *ptype_head(const struct packet_type *pt)
366{
367 if (pt->type == htons(ETH_P_ALL))
368 return &ptype_all;
369 else
370 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
371}
372
373/**
374 * dev_add_pack - add packet handler
375 * @pt: packet type declaration
376 *
377 * Add a protocol handler to the networking stack. The passed &packet_type
378 * is linked into kernel lists and may not be freed until it has been
379 * removed from the kernel lists.
380 *
381 * This call does not sleep therefore it can not
382 * guarantee all CPU's that are in middle of receiving packets
383 * will see the new packet type (until the next received packet).
384 */
385
386void dev_add_pack(struct packet_type *pt)
387{
388 struct list_head *head = ptype_head(pt);
389
390 spin_lock(&ptype_lock);
391 list_add_rcu(&pt->list, head);
392 spin_unlock(&ptype_lock);
393}
394EXPORT_SYMBOL(dev_add_pack);
395
396/**
397 * __dev_remove_pack - remove packet handler
398 * @pt: packet type declaration
399 *
400 * Remove a protocol handler that was previously added to the kernel
401 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
402 * from the kernel lists and can be freed or reused once this function
403 * returns.
404 *
405 * The packet type might still be in use by receivers
406 * and must not be freed until after all the CPU's have gone
407 * through a quiescent state.
408 */
409void __dev_remove_pack(struct packet_type *pt)
410{
411 struct list_head *head = ptype_head(pt);
412 struct packet_type *pt1;
413
414 spin_lock(&ptype_lock);
415
416 list_for_each_entry(pt1, head, list) {
417 if (pt == pt1) {
418 list_del_rcu(&pt->list);
419 goto out;
420 }
421 }
422
423 pr_warn("dev_remove_pack: %p not found\n", pt);
424out:
425 spin_unlock(&ptype_lock);
426}
427EXPORT_SYMBOL(__dev_remove_pack);
428
429/**
430 * dev_remove_pack - remove packet handler
431 * @pt: packet type declaration
432 *
433 * Remove a protocol handler that was previously added to the kernel
434 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
435 * from the kernel lists and can be freed or reused once this function
436 * returns.
437 *
438 * This call sleeps to guarantee that no CPU is looking at the packet
439 * type after return.
440 */
441void dev_remove_pack(struct packet_type *pt)
442{
443 __dev_remove_pack(pt);
444
445 synchronize_net();
446}
447EXPORT_SYMBOL(dev_remove_pack);
448
449
450/**
451 * dev_add_offload - register offload handlers
452 * @po: protocol offload declaration
453 *
454 * Add protocol offload handlers to the networking stack. The passed
455 * &proto_offload is linked into kernel lists and may not be freed until
456 * it has been removed from the kernel lists.
457 *
458 * This call does not sleep therefore it can not
459 * guarantee all CPU's that are in middle of receiving packets
460 * will see the new offload handlers (until the next received packet).
461 */
462void dev_add_offload(struct packet_offload *po)
463{
464 struct list_head *head = &offload_base;
465
466 spin_lock(&offload_lock);
467 list_add_rcu(&po->list, head);
468 spin_unlock(&offload_lock);
469}
470EXPORT_SYMBOL(dev_add_offload);
471
472/**
473 * __dev_remove_offload - remove offload handler
474 * @po: packet offload declaration
475 *
476 * Remove a protocol offload handler that was previously added to the
477 * kernel offload handlers by dev_add_offload(). The passed &offload_type
478 * is removed from the kernel lists and can be freed or reused once this
479 * function returns.
480 *
481 * The packet type might still be in use by receivers
482 * and must not be freed until after all the CPU's have gone
483 * through a quiescent state.
484 */
485static void __dev_remove_offload(struct packet_offload *po)
486{
487 struct list_head *head = &offload_base;
488 struct packet_offload *po1;
489
490 spin_lock(&offload_lock);
491
492 list_for_each_entry(po1, head, list) {
493 if (po == po1) {
494 list_del_rcu(&po->list);
495 goto out;
496 }
497 }
498
499 pr_warn("dev_remove_offload: %p not found\n", po);
500out:
501 spin_unlock(&offload_lock);
502}
503
504/**
505 * dev_remove_offload - remove packet offload handler
506 * @po: packet offload declaration
507 *
508 * Remove a packet offload handler that was previously added to the kernel
509 * offload handlers by dev_add_offload(). The passed &offload_type is
510 * removed from the kernel lists and can be freed or reused once this
511 * function returns.
512 *
513 * This call sleeps to guarantee that no CPU is looking at the packet
514 * type after return.
515 */
516void dev_remove_offload(struct packet_offload *po)
517{
518 __dev_remove_offload(po);
519
520 synchronize_net();
521}
522EXPORT_SYMBOL(dev_remove_offload);
523
524/******************************************************************************
525
526 Device Boot-time Settings Routines
527
528*******************************************************************************/
529
530/* Boot time configuration table */
531static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
532
533/**
534 * netdev_boot_setup_add - add new setup entry
535 * @name: name of the device
536 * @map: configured settings for the device
537 *
538 * Adds new setup entry to the dev_boot_setup list. The function
539 * returns 0 on error and 1 on success. This is a generic routine to
540 * all netdevices.
541 */
542static int netdev_boot_setup_add(char *name, struct ifmap *map)
543{
544 struct netdev_boot_setup *s;
545 int i;
546
547 s = dev_boot_setup;
548 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
549 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
550 memset(s[i].name, 0, sizeof(s[i].name));
551 strlcpy(s[i].name, name, IFNAMSIZ);
552 memcpy(&s[i].map, map, sizeof(s[i].map));
553 break;
554 }
555 }
556
557 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
558}
559
560/**
561 * netdev_boot_setup_check - check boot time settings
562 * @dev: the netdevice
563 *
564 * Check boot time settings for the device.
565 * The found settings are set for the device to be used
566 * later in the device probing.
567 * Returns 0 if no settings found, 1 if they are.
568 */
569int netdev_boot_setup_check(struct net_device *dev)
570{
571 struct netdev_boot_setup *s = dev_boot_setup;
572 int i;
573
574 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
575 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
576 !strcmp(dev->name, s[i].name)) {
577 dev->irq = s[i].map.irq;
578 dev->base_addr = s[i].map.base_addr;
579 dev->mem_start = s[i].map.mem_start;
580 dev->mem_end = s[i].map.mem_end;
581 return 1;
582 }
583 }
584 return 0;
585}
586EXPORT_SYMBOL(netdev_boot_setup_check);
587
588
589/**
590 * netdev_boot_base - get address from boot time settings
591 * @prefix: prefix for network device
592 * @unit: id for network device
593 *
594 * Check boot time settings for the base address of device.
595 * The found settings are set for the device to be used
596 * later in the device probing.
597 * Returns 0 if no settings found.
598 */
599unsigned long netdev_boot_base(const char *prefix, int unit)
600{
601 const struct netdev_boot_setup *s = dev_boot_setup;
602 char name[IFNAMSIZ];
603 int i;
604
605 sprintf(name, "%s%d", prefix, unit);
606
607 /*
608 * If device already registered then return base of 1
609 * to indicate not to probe for this interface
610 */
611 if (__dev_get_by_name(&init_net, name))
612 return 1;
613
614 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
615 if (!strcmp(name, s[i].name))
616 return s[i].map.base_addr;
617 return 0;
618}
619
620/*
621 * Saves at boot time configured settings for any netdevice.
622 */
623int __init netdev_boot_setup(char *str)
624{
625 int ints[5];
626 struct ifmap map;
627
628 str = get_options(str, ARRAY_SIZE(ints), ints);
629 if (!str || !*str)
630 return 0;
631
632 /* Save settings */
633 memset(&map, 0, sizeof(map));
634 if (ints[0] > 0)
635 map.irq = ints[1];
636 if (ints[0] > 1)
637 map.base_addr = ints[2];
638 if (ints[0] > 2)
639 map.mem_start = ints[3];
640 if (ints[0] > 3)
641 map.mem_end = ints[4];
642
643 /* Add new entry to the list */
644 return netdev_boot_setup_add(str, &map);
645}
646
647__setup("netdev=", netdev_boot_setup);
648
649/*******************************************************************************
650
651 Device Interface Subroutines
652
653*******************************************************************************/
654
655/**
656 * __dev_get_by_name - find a device by its name
657 * @net: the applicable net namespace
658 * @name: name to find
659 *
660 * Find an interface by name. Must be called under RTNL semaphore
661 * or @dev_base_lock. If the name is found a pointer to the device
662 * is returned. If the name is not found then %NULL is returned. The
663 * reference counters are not incremented so the caller must be
664 * careful with locks.
665 */
666
667struct net_device *__dev_get_by_name(struct net *net, const char *name)
668{
669 struct net_device *dev;
670 struct hlist_head *head = dev_name_hash(net, name);
671
672 hlist_for_each_entry(dev, head, name_hlist)
673 if (!strncmp(dev->name, name, IFNAMSIZ))
674 return dev;
675
676 return NULL;
677}
678EXPORT_SYMBOL(__dev_get_by_name);
679
680/**
681 * dev_get_by_name_rcu - find a device by its name
682 * @net: the applicable net namespace
683 * @name: name to find
684 *
685 * Find an interface by name.
686 * If the name is found a pointer to the device is returned.
687 * If the name is not found then %NULL is returned.
688 * The reference counters are not incremented so the caller must be
689 * careful with locks. The caller must hold RCU lock.
690 */
691
692struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
693{
694 struct net_device *dev;
695 struct hlist_head *head = dev_name_hash(net, name);
696
697 hlist_for_each_entry_rcu(dev, head, name_hlist)
698 if (!strncmp(dev->name, name, IFNAMSIZ))
699 return dev;
700
701 return NULL;
702}
703EXPORT_SYMBOL(dev_get_by_name_rcu);
704
705/**
706 * dev_get_by_name - find a device by its name
707 * @net: the applicable net namespace
708 * @name: name to find
709 *
710 * Find an interface by name. This can be called from any
711 * context and does its own locking. The returned handle has
712 * the usage count incremented and the caller must use dev_put() to
713 * release it when it is no longer needed. %NULL is returned if no
714 * matching device is found.
715 */
716
717struct net_device *dev_get_by_name(struct net *net, const char *name)
718{
719 struct net_device *dev;
720
721 rcu_read_lock();
722 dev = dev_get_by_name_rcu(net, name);
723 if (dev)
724 dev_hold(dev);
725 rcu_read_unlock();
726 return dev;
727}
728EXPORT_SYMBOL(dev_get_by_name);
729
730/**
731 * __dev_get_by_index - find a device by its ifindex
732 * @net: the applicable net namespace
733 * @ifindex: index of device
734 *
735 * Search for an interface by index. Returns %NULL if the device
736 * is not found or a pointer to the device. The device has not
737 * had its reference counter increased so the caller must be careful
738 * about locking. The caller must hold either the RTNL semaphore
739 * or @dev_base_lock.
740 */
741
742struct net_device *__dev_get_by_index(struct net *net, int ifindex)
743{
744 struct net_device *dev;
745 struct hlist_head *head = dev_index_hash(net, ifindex);
746
747 hlist_for_each_entry(dev, head, index_hlist)
748 if (dev->ifindex == ifindex)
749 return dev;
750
751 return NULL;
752}
753EXPORT_SYMBOL(__dev_get_by_index);
754
755/**
756 * dev_get_by_index_rcu - find a device by its ifindex
757 * @net: the applicable net namespace
758 * @ifindex: index of device
759 *
760 * Search for an interface by index. Returns %NULL if the device
761 * is not found or a pointer to the device. The device has not
762 * had its reference counter increased so the caller must be careful
763 * about locking. The caller must hold RCU lock.
764 */
765
766struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
767{
768 struct net_device *dev;
769 struct hlist_head *head = dev_index_hash(net, ifindex);
770
771 hlist_for_each_entry_rcu(dev, head, index_hlist)
772 if (dev->ifindex == ifindex)
773 return dev;
774
775 return NULL;
776}
777EXPORT_SYMBOL(dev_get_by_index_rcu);
778
779
780/**
781 * dev_get_by_index - find a device by its ifindex
782 * @net: the applicable net namespace
783 * @ifindex: index of device
784 *
785 * Search for an interface by index. Returns NULL if the device
786 * is not found or a pointer to the device. The device returned has
787 * had a reference added and the pointer is safe until the user calls
788 * dev_put to indicate they have finished with it.
789 */
790
791struct net_device *dev_get_by_index(struct net *net, int ifindex)
792{
793 struct net_device *dev;
794
795 rcu_read_lock();
796 dev = dev_get_by_index_rcu(net, ifindex);
797 if (dev)
798 dev_hold(dev);
799 rcu_read_unlock();
800 return dev;
801}
802EXPORT_SYMBOL(dev_get_by_index);
803
804/**
805 * netdev_get_name - get a netdevice name, knowing its ifindex.
806 * @net: network namespace
807 * @name: a pointer to the buffer where the name will be stored.
808 * @ifindex: the ifindex of the interface to get the name from.
809 *
810 * The use of raw_seqcount_begin() and cond_resched() before
811 * retrying is required as we want to give the writers a chance
812 * to complete when CONFIG_PREEMPT is not set.
813 */
814int netdev_get_name(struct net *net, char *name, int ifindex)
815{
816 struct net_device *dev;
817 unsigned int seq;
818
819retry:
820 seq = raw_seqcount_begin(&devnet_rename_seq);
821 rcu_read_lock();
822 dev = dev_get_by_index_rcu(net, ifindex);
823 if (!dev) {
824 rcu_read_unlock();
825 return -ENODEV;
826 }
827
828 strcpy(name, dev->name);
829 rcu_read_unlock();
830 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
831 cond_resched();
832 goto retry;
833 }
834
835 return 0;
836}
837
838/**
839 * dev_getbyhwaddr_rcu - find a device by its hardware address
840 * @net: the applicable net namespace
841 * @type: media type of device
842 * @ha: hardware address
843 *
844 * Search for an interface by MAC address. Returns NULL if the device
845 * is not found or a pointer to the device.
846 * The caller must hold RCU or RTNL.
847 * The returned device has not had its ref count increased
848 * and the caller must therefore be careful about locking
849 *
850 */
851
852struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
853 const char *ha)
854{
855 struct net_device *dev;
856
857 for_each_netdev_rcu(net, dev)
858 if (dev->type == type &&
859 !memcmp(dev->dev_addr, ha, dev->addr_len))
860 return dev;
861
862 return NULL;
863}
864EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
865
866struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
867{
868 struct net_device *dev;
869
870 ASSERT_RTNL();
871 for_each_netdev(net, dev)
872 if (dev->type == type)
873 return dev;
874
875 return NULL;
876}
877EXPORT_SYMBOL(__dev_getfirstbyhwtype);
878
879struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
880{
881 struct net_device *dev, *ret = NULL;
882
883 rcu_read_lock();
884 for_each_netdev_rcu(net, dev)
885 if (dev->type == type) {
886 dev_hold(dev);
887 ret = dev;
888 break;
889 }
890 rcu_read_unlock();
891 return ret;
892}
893EXPORT_SYMBOL(dev_getfirstbyhwtype);
894
895/**
896 * dev_get_by_flags_rcu - find any device with given flags
897 * @net: the applicable net namespace
898 * @if_flags: IFF_* values
899 * @mask: bitmask of bits in if_flags to check
900 *
901 * Search for any interface with the given flags. Returns NULL if a device
902 * is not found or a pointer to the device. Must be called inside
903 * rcu_read_lock(), and result refcount is unchanged.
904 */
905
906struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
907 unsigned short mask)
908{
909 struct net_device *dev, *ret;
910
911 ret = NULL;
912 for_each_netdev_rcu(net, dev) {
913 if (((dev->flags ^ if_flags) & mask) == 0) {
914 ret = dev;
915 break;
916 }
917 }
918 return ret;
919}
920EXPORT_SYMBOL(dev_get_by_flags_rcu);
921
922/**
923 * dev_valid_name - check if name is okay for network device
924 * @name: name string
925 *
926 * Network device names need to be valid file names to
927 * to allow sysfs to work. We also disallow any kind of
928 * whitespace.
929 */
930bool dev_valid_name(const char *name)
931{
932 if (*name == '\0')
933 return false;
934 if (strlen(name) >= IFNAMSIZ)
935 return false;
936 if (!strcmp(name, ".") || !strcmp(name, ".."))
937 return false;
938
939 while (*name) {
940 if (*name == '/' || isspace(*name))
941 return false;
942 name++;
943 }
944 return true;
945}
946EXPORT_SYMBOL(dev_valid_name);
947
948/**
949 * __dev_alloc_name - allocate a name for a device
950 * @net: network namespace to allocate the device name in
951 * @name: name format string
952 * @buf: scratch buffer and result name string
953 *
954 * Passed a format string - eg "lt%d" it will try and find a suitable
955 * id. It scans list of devices to build up a free map, then chooses
956 * the first empty slot. The caller must hold the dev_base or rtnl lock
957 * while allocating the name and adding the device in order to avoid
958 * duplicates.
959 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
960 * Returns the number of the unit assigned or a negative errno code.
961 */
962
963static int __dev_alloc_name(struct net *net, const char *name, char *buf)
964{
965 int i = 0;
966 const char *p;
967 const int max_netdevices = 8*PAGE_SIZE;
968 unsigned long *inuse;
969 struct net_device *d;
970
971 p = strnchr(name, IFNAMSIZ-1, '%');
972 if (p) {
973 /*
974 * Verify the string as this thing may have come from
975 * the user. There must be either one "%d" and no other "%"
976 * characters.
977 */
978 if (p[1] != 'd' || strchr(p + 2, '%'))
979 return -EINVAL;
980
981 /* Use one page as a bit array of possible slots */
982 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
983 if (!inuse)
984 return -ENOMEM;
985
986 for_each_netdev(net, d) {
987 if (!sscanf(d->name, name, &i))
988 continue;
989 if (i < 0 || i >= max_netdevices)
990 continue;
991
992 /* avoid cases where sscanf is not exact inverse of printf */
993 snprintf(buf, IFNAMSIZ, name, i);
994 if (!strncmp(buf, d->name, IFNAMSIZ))
995 set_bit(i, inuse);
996 }
997
998 i = find_first_zero_bit(inuse, max_netdevices);
999 free_page((unsigned long) inuse);
1000 }
1001
1002 if (buf != name)
1003 snprintf(buf, IFNAMSIZ, name, i);
1004 if (!__dev_get_by_name(net, buf))
1005 return i;
1006
1007 /* It is possible to run out of possible slots
1008 * when the name is long and there isn't enough space left
1009 * for the digits, or if all bits are used.
1010 */
1011 return -ENFILE;
1012}
1013
1014/**
1015 * dev_alloc_name - allocate a name for a device
1016 * @dev: device
1017 * @name: name format string
1018 *
1019 * Passed a format string - eg "lt%d" it will try and find a suitable
1020 * id. It scans list of devices to build up a free map, then chooses
1021 * the first empty slot. The caller must hold the dev_base or rtnl lock
1022 * while allocating the name and adding the device in order to avoid
1023 * duplicates.
1024 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1025 * Returns the number of the unit assigned or a negative errno code.
1026 */
1027
1028int dev_alloc_name(struct net_device *dev, const char *name)
1029{
1030 char buf[IFNAMSIZ];
1031 struct net *net;
1032 int ret;
1033
1034 BUG_ON(!dev_net(dev));
1035 net = dev_net(dev);
1036 ret = __dev_alloc_name(net, name, buf);
1037 if (ret >= 0)
1038 strlcpy(dev->name, buf, IFNAMSIZ);
1039 return ret;
1040}
1041EXPORT_SYMBOL(dev_alloc_name);
1042
1043static int dev_alloc_name_ns(struct net *net,
1044 struct net_device *dev,
1045 const char *name)
1046{
1047 char buf[IFNAMSIZ];
1048 int ret;
1049
1050 ret = __dev_alloc_name(net, name, buf);
1051 if (ret >= 0)
1052 strlcpy(dev->name, buf, IFNAMSIZ);
1053 return ret;
1054}
1055
1056static int dev_get_valid_name(struct net *net,
1057 struct net_device *dev,
1058 const char *name)
1059{
1060 BUG_ON(!net);
1061
1062 if (!dev_valid_name(name))
1063 return -EINVAL;
1064
1065 if (strchr(name, '%'))
1066 return dev_alloc_name_ns(net, dev, name);
1067 else if (__dev_get_by_name(net, name))
1068 return -EEXIST;
1069 else if (dev->name != name)
1070 strlcpy(dev->name, name, IFNAMSIZ);
1071
1072 return 0;
1073}
1074
1075/**
1076 * dev_change_name - change name of a device
1077 * @dev: device
1078 * @newname: name (or format string) must be at least IFNAMSIZ
1079 *
1080 * Change name of a device, can pass format strings "eth%d".
1081 * for wildcarding.
1082 */
1083int dev_change_name(struct net_device *dev, const char *newname)
1084{
1085 char oldname[IFNAMSIZ];
1086 int err = 0;
1087 int ret;
1088 struct net *net;
1089
1090 ASSERT_RTNL();
1091 BUG_ON(!dev_net(dev));
1092
1093 net = dev_net(dev);
1094 if (dev->flags & IFF_UP)
1095 return -EBUSY;
1096
1097 write_seqcount_begin(&devnet_rename_seq);
1098
1099 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1100 write_seqcount_end(&devnet_rename_seq);
1101 return 0;
1102 }
1103
1104 memcpy(oldname, dev->name, IFNAMSIZ);
1105
1106 err = dev_get_valid_name(net, dev, newname);
1107 if (err < 0) {
1108 write_seqcount_end(&devnet_rename_seq);
1109 return err;
1110 }
1111
1112rollback:
1113 ret = device_rename(&dev->dev, dev->name);
1114 if (ret) {
1115 memcpy(dev->name, oldname, IFNAMSIZ);
1116 write_seqcount_end(&devnet_rename_seq);
1117 return ret;
1118 }
1119
1120 write_seqcount_end(&devnet_rename_seq);
1121
1122 netdev_adjacent_rename_links(dev, oldname);
1123
1124 write_lock_bh(&dev_base_lock);
1125 hlist_del_rcu(&dev->name_hlist);
1126 write_unlock_bh(&dev_base_lock);
1127
1128 synchronize_rcu();
1129
1130 write_lock_bh(&dev_base_lock);
1131 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1132 write_unlock_bh(&dev_base_lock);
1133
1134 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1135 ret = notifier_to_errno(ret);
1136
1137 if (ret) {
1138 /* err >= 0 after dev_alloc_name() or stores the first errno */
1139 if (err >= 0) {
1140 err = ret;
1141 write_seqcount_begin(&devnet_rename_seq);
1142 memcpy(dev->name, oldname, IFNAMSIZ);
1143 memcpy(oldname, newname, IFNAMSIZ);
1144 goto rollback;
1145 } else {
1146 pr_err("%s: name change rollback failed: %d\n",
1147 dev->name, ret);
1148 }
1149 }
1150
1151 return err;
1152}
1153
1154/**
1155 * dev_set_alias - change ifalias of a device
1156 * @dev: device
1157 * @alias: name up to IFALIASZ
1158 * @len: limit of bytes to copy from info
1159 *
1160 * Set ifalias for a device,
1161 */
1162int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1163{
1164 char *new_ifalias;
1165
1166 ASSERT_RTNL();
1167
1168 if (len >= IFALIASZ)
1169 return -EINVAL;
1170
1171 if (!len) {
1172 kfree(dev->ifalias);
1173 dev->ifalias = NULL;
1174 return 0;
1175 }
1176
1177 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1178 if (!new_ifalias)
1179 return -ENOMEM;
1180 dev->ifalias = new_ifalias;
1181
1182 strlcpy(dev->ifalias, alias, len+1);
1183 return len;
1184}
1185
1186
1187/**
1188 * netdev_features_change - device changes features
1189 * @dev: device to cause notification
1190 *
1191 * Called to indicate a device has changed features.
1192 */
1193void netdev_features_change(struct net_device *dev)
1194{
1195 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1196}
1197EXPORT_SYMBOL(netdev_features_change);
1198
1199/**
1200 * netdev_state_change - device changes state
1201 * @dev: device to cause notification
1202 *
1203 * Called to indicate a device has changed state. This function calls
1204 * the notifier chains for netdev_chain and sends a NEWLINK message
1205 * to the routing socket.
1206 */
1207void netdev_state_change(struct net_device *dev)
1208{
1209 if (dev->flags & IFF_UP) {
1210 call_netdevice_notifiers(NETDEV_CHANGE, dev);
1211 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1212 }
1213}
1214EXPORT_SYMBOL(netdev_state_change);
1215
1216/**
1217 * netdev_notify_peers - notify network peers about existence of @dev
1218 * @dev: network device
1219 *
1220 * Generate traffic such that interested network peers are aware of
1221 * @dev, such as by generating a gratuitous ARP. This may be used when
1222 * a device wants to inform the rest of the network about some sort of
1223 * reconfiguration such as a failover event or virtual machine
1224 * migration.
1225 */
1226void netdev_notify_peers(struct net_device *dev)
1227{
1228 rtnl_lock();
1229 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1230 rtnl_unlock();
1231}
1232EXPORT_SYMBOL(netdev_notify_peers);
1233
1234static int __dev_open(struct net_device *dev)
1235{
1236 const struct net_device_ops *ops = dev->netdev_ops;
1237 int ret;
1238
1239 ASSERT_RTNL();
1240
1241 if (!netif_device_present(dev))
1242 return -ENODEV;
1243
1244 /* Block netpoll from trying to do any rx path servicing.
1245 * If we don't do this there is a chance ndo_poll_controller
1246 * or ndo_poll may be running while we open the device
1247 */
1248 netpoll_poll_disable(dev);
1249
1250 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1251 ret = notifier_to_errno(ret);
1252 if (ret)
1253 return ret;
1254
1255 set_bit(__LINK_STATE_START, &dev->state);
1256
1257 if (ops->ndo_validate_addr)
1258 ret = ops->ndo_validate_addr(dev);
1259
1260 if (!ret && ops->ndo_open)
1261 ret = ops->ndo_open(dev);
1262
1263 netpoll_poll_enable(dev);
1264
1265 if (ret)
1266 clear_bit(__LINK_STATE_START, &dev->state);
1267 else {
1268 dev->flags |= IFF_UP;
1269 net_dmaengine_get();
1270 dev_set_rx_mode(dev);
1271 dev_activate(dev);
1272 add_device_randomness(dev->dev_addr, dev->addr_len);
1273 }
1274
1275 return ret;
1276}
1277
1278/**
1279 * dev_open - prepare an interface for use.
1280 * @dev: device to open
1281 *
1282 * Takes a device from down to up state. The device's private open
1283 * function is invoked and then the multicast lists are loaded. Finally
1284 * the device is moved into the up state and a %NETDEV_UP message is
1285 * sent to the netdev notifier chain.
1286 *
1287 * Calling this function on an active interface is a nop. On a failure
1288 * a negative errno code is returned.
1289 */
1290int dev_open(struct net_device *dev)
1291{
1292 int ret;
1293
1294 if (dev->flags & IFF_UP)
1295 return 0;
1296
1297 ret = __dev_open(dev);
1298 if (ret < 0)
1299 return ret;
1300
1301 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1302 call_netdevice_notifiers(NETDEV_UP, dev);
1303
1304 return ret;
1305}
1306EXPORT_SYMBOL(dev_open);
1307
1308static int __dev_close_many(struct list_head *head)
1309{
1310 struct net_device *dev;
1311
1312 ASSERT_RTNL();
1313 might_sleep();
1314
1315 list_for_each_entry(dev, head, close_list) {
1316 /* Temporarily disable netpoll until the interface is down */
1317 netpoll_poll_disable(dev);
1318
1319 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1320
1321 clear_bit(__LINK_STATE_START, &dev->state);
1322
1323 /* Synchronize to scheduled poll. We cannot touch poll list, it
1324 * can be even on different cpu. So just clear netif_running().
1325 *
1326 * dev->stop() will invoke napi_disable() on all of it's
1327 * napi_struct instances on this device.
1328 */
1329 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1330 }
1331
1332 dev_deactivate_many(head);
1333
1334 list_for_each_entry(dev, head, close_list) {
1335 const struct net_device_ops *ops = dev->netdev_ops;
1336
1337 /*
1338 * Call the device specific close. This cannot fail.
1339 * Only if device is UP
1340 *
1341 * We allow it to be called even after a DETACH hot-plug
1342 * event.
1343 */
1344 if (ops->ndo_stop)
1345 ops->ndo_stop(dev);
1346
1347 dev->flags &= ~IFF_UP;
1348 net_dmaengine_put();
1349 netpoll_poll_enable(dev);
1350 }
1351
1352 return 0;
1353}
1354
1355static int __dev_close(struct net_device *dev)
1356{
1357 int retval;
1358 LIST_HEAD(single);
1359
1360 list_add(&dev->close_list, &single);
1361 retval = __dev_close_many(&single);
1362 list_del(&single);
1363
1364 return retval;
1365}
1366
1367static int dev_close_many(struct list_head *head)
1368{
1369 struct net_device *dev, *tmp;
1370
1371 /* Remove the devices that don't need to be closed */
1372 list_for_each_entry_safe(dev, tmp, head, close_list)
1373 if (!(dev->flags & IFF_UP))
1374 list_del_init(&dev->close_list);
1375
1376 __dev_close_many(head);
1377
1378 list_for_each_entry_safe(dev, tmp, head, close_list) {
1379 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1380 call_netdevice_notifiers(NETDEV_DOWN, dev);
1381 list_del_init(&dev->close_list);
1382 }
1383
1384 return 0;
1385}
1386
1387/**
1388 * dev_close - shutdown an interface.
1389 * @dev: device to shutdown
1390 *
1391 * This function moves an active device into down state. A
1392 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1393 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1394 * chain.
1395 */
1396int dev_close(struct net_device *dev)
1397{
1398 if (dev->flags & IFF_UP) {
1399 LIST_HEAD(single);
1400
1401 list_add(&dev->close_list, &single);
1402 dev_close_many(&single);
1403 list_del(&single);
1404 }
1405 return 0;
1406}
1407EXPORT_SYMBOL(dev_close);
1408
1409
1410/**
1411 * dev_disable_lro - disable Large Receive Offload on a device
1412 * @dev: device
1413 *
1414 * Disable Large Receive Offload (LRO) on a net device. Must be
1415 * called under RTNL. This is needed if received packets may be
1416 * forwarded to another interface.
1417 */
1418void dev_disable_lro(struct net_device *dev)
1419{
1420 /*
1421 * If we're trying to disable lro on a vlan device
1422 * use the underlying physical device instead
1423 */
1424 if (is_vlan_dev(dev))
1425 dev = vlan_dev_real_dev(dev);
1426
1427 /* the same for macvlan devices */
1428 if (netif_is_macvlan(dev))
1429 dev = macvlan_dev_real_dev(dev);
1430
1431 dev->wanted_features &= ~NETIF_F_LRO;
1432 netdev_update_features(dev);
1433
1434 if (unlikely(dev->features & NETIF_F_LRO))
1435 netdev_WARN(dev, "failed to disable LRO!\n");
1436}
1437EXPORT_SYMBOL(dev_disable_lro);
1438
1439static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1440 struct net_device *dev)
1441{
1442 struct netdev_notifier_info info;
1443
1444 netdev_notifier_info_init(&info, dev);
1445 return nb->notifier_call(nb, val, &info);
1446}
1447
1448static int dev_boot_phase = 1;
1449
1450/**
1451 * register_netdevice_notifier - register a network notifier block
1452 * @nb: notifier
1453 *
1454 * Register a notifier to be called when network device events occur.
1455 * The notifier passed is linked into the kernel structures and must
1456 * not be reused until it has been unregistered. A negative errno code
1457 * is returned on a failure.
1458 *
1459 * When registered all registration and up events are replayed
1460 * to the new notifier to allow device to have a race free
1461 * view of the network device list.
1462 */
1463
1464int register_netdevice_notifier(struct notifier_block *nb)
1465{
1466 struct net_device *dev;
1467 struct net_device *last;
1468 struct net *net;
1469 int err;
1470
1471 rtnl_lock();
1472 err = raw_notifier_chain_register(&netdev_chain, nb);
1473 if (err)
1474 goto unlock;
1475 if (dev_boot_phase)
1476 goto unlock;
1477 for_each_net(net) {
1478 for_each_netdev(net, dev) {
1479 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1480 err = notifier_to_errno(err);
1481 if (err)
1482 goto rollback;
1483
1484 if (!(dev->flags & IFF_UP))
1485 continue;
1486
1487 call_netdevice_notifier(nb, NETDEV_UP, dev);
1488 }
1489 }
1490
1491unlock:
1492 rtnl_unlock();
1493 return err;
1494
1495rollback:
1496 last = dev;
1497 for_each_net(net) {
1498 for_each_netdev(net, dev) {
1499 if (dev == last)
1500 goto outroll;
1501
1502 if (dev->flags & IFF_UP) {
1503 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1504 dev);
1505 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1506 }
1507 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1508 }
1509 }
1510
1511outroll:
1512 raw_notifier_chain_unregister(&netdev_chain, nb);
1513 goto unlock;
1514}
1515EXPORT_SYMBOL(register_netdevice_notifier);
1516
1517/**
1518 * unregister_netdevice_notifier - unregister a network notifier block
1519 * @nb: notifier
1520 *
1521 * Unregister a notifier previously registered by
1522 * register_netdevice_notifier(). The notifier is unlinked into the
1523 * kernel structures and may then be reused. A negative errno code
1524 * is returned on a failure.
1525 *
1526 * After unregistering unregister and down device events are synthesized
1527 * for all devices on the device list to the removed notifier to remove
1528 * the need for special case cleanup code.
1529 */
1530
1531int unregister_netdevice_notifier(struct notifier_block *nb)
1532{
1533 struct net_device *dev;
1534 struct net *net;
1535 int err;
1536
1537 rtnl_lock();
1538 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1539 if (err)
1540 goto unlock;
1541
1542 for_each_net(net) {
1543 for_each_netdev(net, dev) {
1544 if (dev->flags & IFF_UP) {
1545 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1546 dev);
1547 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1548 }
1549 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1550 }
1551 }
1552unlock:
1553 rtnl_unlock();
1554 return err;
1555}
1556EXPORT_SYMBOL(unregister_netdevice_notifier);
1557
1558/**
1559 * call_netdevice_notifiers_info - call all network notifier blocks
1560 * @val: value passed unmodified to notifier function
1561 * @dev: net_device pointer passed unmodified to notifier function
1562 * @info: notifier information data
1563 *
1564 * Call all network notifier blocks. Parameters and return value
1565 * are as for raw_notifier_call_chain().
1566 */
1567
1568static int call_netdevice_notifiers_info(unsigned long val,
1569 struct net_device *dev,
1570 struct netdev_notifier_info *info)
1571{
1572 ASSERT_RTNL();
1573 netdev_notifier_info_init(info, dev);
1574 return raw_notifier_call_chain(&netdev_chain, val, info);
1575}
1576
1577/**
1578 * call_netdevice_notifiers - call all network notifier blocks
1579 * @val: value passed unmodified to notifier function
1580 * @dev: net_device pointer passed unmodified to notifier function
1581 *
1582 * Call all network notifier blocks. Parameters and return value
1583 * are as for raw_notifier_call_chain().
1584 */
1585
1586int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1587{
1588 struct netdev_notifier_info info;
1589
1590 return call_netdevice_notifiers_info(val, dev, &info);
1591}
1592EXPORT_SYMBOL(call_netdevice_notifiers);
1593
1594static struct static_key netstamp_needed __read_mostly;
1595#ifdef HAVE_JUMP_LABEL
1596/* We are not allowed to call static_key_slow_dec() from irq context
1597 * If net_disable_timestamp() is called from irq context, defer the
1598 * static_key_slow_dec() calls.
1599 */
1600static atomic_t netstamp_needed_deferred;
1601#endif
1602
1603void net_enable_timestamp(void)
1604{
1605#ifdef HAVE_JUMP_LABEL
1606 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1607
1608 if (deferred) {
1609 while (--deferred)
1610 static_key_slow_dec(&netstamp_needed);
1611 return;
1612 }
1613#endif
1614 static_key_slow_inc(&netstamp_needed);
1615}
1616EXPORT_SYMBOL(net_enable_timestamp);
1617
1618void net_disable_timestamp(void)
1619{
1620#ifdef HAVE_JUMP_LABEL
1621 if (in_interrupt()) {
1622 atomic_inc(&netstamp_needed_deferred);
1623 return;
1624 }
1625#endif
1626 static_key_slow_dec(&netstamp_needed);
1627}
1628EXPORT_SYMBOL(net_disable_timestamp);
1629
1630static inline void net_timestamp_set(struct sk_buff *skb)
1631{
1632 skb->tstamp.tv64 = 0;
1633 if (static_key_false(&netstamp_needed))
1634 __net_timestamp(skb);
1635}
1636
1637#define net_timestamp_check(COND, SKB) \
1638 if (static_key_false(&netstamp_needed)) { \
1639 if ((COND) && !(SKB)->tstamp.tv64) \
1640 __net_timestamp(SKB); \
1641 } \
1642
1643bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
1644{
1645 unsigned int len;
1646
1647 if (!(dev->flags & IFF_UP))
1648 return false;
1649
1650 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1651 if (skb->len <= len)
1652 return true;
1653
1654 /* if TSO is enabled, we don't care about the length as the packet
1655 * could be forwarded without being segmented before
1656 */
1657 if (skb_is_gso(skb))
1658 return true;
1659
1660 return false;
1661}
1662EXPORT_SYMBOL_GPL(is_skb_forwardable);
1663
1664/**
1665 * dev_forward_skb - loopback an skb to another netif
1666 *
1667 * @dev: destination network device
1668 * @skb: buffer to forward
1669 *
1670 * return values:
1671 * NET_RX_SUCCESS (no congestion)
1672 * NET_RX_DROP (packet was dropped, but freed)
1673 *
1674 * dev_forward_skb can be used for injecting an skb from the
1675 * start_xmit function of one device into the receive queue
1676 * of another device.
1677 *
1678 * The receiving device may be in another namespace, so
1679 * we have to clear all information in the skb that could
1680 * impact namespace isolation.
1681 */
1682int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1683{
1684 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1685 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1686 atomic_long_inc(&dev->rx_dropped);
1687 kfree_skb(skb);
1688 return NET_RX_DROP;
1689 }
1690 }
1691
1692 if (unlikely(!is_skb_forwardable(dev, skb))) {
1693 atomic_long_inc(&dev->rx_dropped);
1694 kfree_skb(skb);
1695 return NET_RX_DROP;
1696 }
1697
1698 skb_scrub_packet(skb, true);
1699 skb->protocol = eth_type_trans(skb, dev);
1700
1701 return netif_rx_internal(skb);
1702}
1703EXPORT_SYMBOL_GPL(dev_forward_skb);
1704
1705static inline int deliver_skb(struct sk_buff *skb,
1706 struct packet_type *pt_prev,
1707 struct net_device *orig_dev)
1708{
1709 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1710 return -ENOMEM;
1711 atomic_inc(&skb->users);
1712 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1713}
1714
1715static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1716{
1717 if (!ptype->af_packet_priv || !skb->sk)
1718 return false;
1719
1720 if (ptype->id_match)
1721 return ptype->id_match(ptype, skb->sk);
1722 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1723 return true;
1724
1725 return false;
1726}
1727
1728/*
1729 * Support routine. Sends outgoing frames to any network
1730 * taps currently in use.
1731 */
1732
1733static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1734{
1735 struct packet_type *ptype;
1736 struct sk_buff *skb2 = NULL;
1737 struct packet_type *pt_prev = NULL;
1738
1739 rcu_read_lock();
1740 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1741 /* Never send packets back to the socket
1742 * they originated from - MvS (miquels@drinkel.ow.org)
1743 */
1744 if ((ptype->dev == dev || !ptype->dev) &&
1745 (!skb_loop_sk(ptype, skb))) {
1746 if (pt_prev) {
1747 deliver_skb(skb2, pt_prev, skb->dev);
1748 pt_prev = ptype;
1749 continue;
1750 }
1751
1752 skb2 = skb_clone(skb, GFP_ATOMIC);
1753 if (!skb2)
1754 break;
1755
1756 net_timestamp_set(skb2);
1757
1758 /* skb->nh should be correctly
1759 set by sender, so that the second statement is
1760 just protection against buggy protocols.
1761 */
1762 skb_reset_mac_header(skb2);
1763
1764 if (skb_network_header(skb2) < skb2->data ||
1765 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1766 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1767 ntohs(skb2->protocol),
1768 dev->name);
1769 skb_reset_network_header(skb2);
1770 }
1771
1772 skb2->transport_header = skb2->network_header;
1773 skb2->pkt_type = PACKET_OUTGOING;
1774 pt_prev = ptype;
1775 }
1776 }
1777 if (pt_prev)
1778 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1779 rcu_read_unlock();
1780}
1781
1782/**
1783 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1784 * @dev: Network device
1785 * @txq: number of queues available
1786 *
1787 * If real_num_tx_queues is changed the tc mappings may no longer be
1788 * valid. To resolve this verify the tc mapping remains valid and if
1789 * not NULL the mapping. With no priorities mapping to this
1790 * offset/count pair it will no longer be used. In the worst case TC0
1791 * is invalid nothing can be done so disable priority mappings. If is
1792 * expected that drivers will fix this mapping if they can before
1793 * calling netif_set_real_num_tx_queues.
1794 */
1795static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1796{
1797 int i;
1798 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1799
1800 /* If TC0 is invalidated disable TC mapping */
1801 if (tc->offset + tc->count > txq) {
1802 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1803 dev->num_tc = 0;
1804 return;
1805 }
1806
1807 /* Invalidated prio to tc mappings set to TC0 */
1808 for (i = 1; i < TC_BITMASK + 1; i++) {
1809 int q = netdev_get_prio_tc_map(dev, i);
1810
1811 tc = &dev->tc_to_txq[q];
1812 if (tc->offset + tc->count > txq) {
1813 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1814 i, q);
1815 netdev_set_prio_tc_map(dev, i, 0);
1816 }
1817 }
1818}
1819
1820#ifdef CONFIG_XPS
1821static DEFINE_MUTEX(xps_map_mutex);
1822#define xmap_dereference(P) \
1823 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1824
1825static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1826 int cpu, u16 index)
1827{
1828 struct xps_map *map = NULL;
1829 int pos;
1830
1831 if (dev_maps)
1832 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1833
1834 for (pos = 0; map && pos < map->len; pos++) {
1835 if (map->queues[pos] == index) {
1836 if (map->len > 1) {
1837 map->queues[pos] = map->queues[--map->len];
1838 } else {
1839 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1840 kfree_rcu(map, rcu);
1841 map = NULL;
1842 }
1843 break;
1844 }
1845 }
1846
1847 return map;
1848}
1849
1850static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1851{
1852 struct xps_dev_maps *dev_maps;
1853 int cpu, i;
1854 bool active = false;
1855
1856 mutex_lock(&xps_map_mutex);
1857 dev_maps = xmap_dereference(dev->xps_maps);
1858
1859 if (!dev_maps)
1860 goto out_no_maps;
1861
1862 for_each_possible_cpu(cpu) {
1863 for (i = index; i < dev->num_tx_queues; i++) {
1864 if (!remove_xps_queue(dev_maps, cpu, i))
1865 break;
1866 }
1867 if (i == dev->num_tx_queues)
1868 active = true;
1869 }
1870
1871 if (!active) {
1872 RCU_INIT_POINTER(dev->xps_maps, NULL);
1873 kfree_rcu(dev_maps, rcu);
1874 }
1875
1876 for (i = index; i < dev->num_tx_queues; i++)
1877 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1878 NUMA_NO_NODE);
1879
1880out_no_maps:
1881 mutex_unlock(&xps_map_mutex);
1882}
1883
1884static struct xps_map *expand_xps_map(struct xps_map *map,
1885 int cpu, u16 index)
1886{
1887 struct xps_map *new_map;
1888 int alloc_len = XPS_MIN_MAP_ALLOC;
1889 int i, pos;
1890
1891 for (pos = 0; map && pos < map->len; pos++) {
1892 if (map->queues[pos] != index)
1893 continue;
1894 return map;
1895 }
1896
1897 /* Need to add queue to this CPU's existing map */
1898 if (map) {
1899 if (pos < map->alloc_len)
1900 return map;
1901
1902 alloc_len = map->alloc_len * 2;
1903 }
1904
1905 /* Need to allocate new map to store queue on this CPU's map */
1906 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
1907 cpu_to_node(cpu));
1908 if (!new_map)
1909 return NULL;
1910
1911 for (i = 0; i < pos; i++)
1912 new_map->queues[i] = map->queues[i];
1913 new_map->alloc_len = alloc_len;
1914 new_map->len = pos;
1915
1916 return new_map;
1917}
1918
1919int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
1920 u16 index)
1921{
1922 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
1923 struct xps_map *map, *new_map;
1924 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
1925 int cpu, numa_node_id = -2;
1926 bool active = false;
1927
1928 mutex_lock(&xps_map_mutex);
1929
1930 dev_maps = xmap_dereference(dev->xps_maps);
1931
1932 /* allocate memory for queue storage */
1933 for_each_online_cpu(cpu) {
1934 if (!cpumask_test_cpu(cpu, mask))
1935 continue;
1936
1937 if (!new_dev_maps)
1938 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
1939 if (!new_dev_maps) {
1940 mutex_unlock(&xps_map_mutex);
1941 return -ENOMEM;
1942 }
1943
1944 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
1945 NULL;
1946
1947 map = expand_xps_map(map, cpu, index);
1948 if (!map)
1949 goto error;
1950
1951 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
1952 }
1953
1954 if (!new_dev_maps)
1955 goto out_no_new_maps;
1956
1957 for_each_possible_cpu(cpu) {
1958 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
1959 /* add queue to CPU maps */
1960 int pos = 0;
1961
1962 map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1963 while ((pos < map->len) && (map->queues[pos] != index))
1964 pos++;
1965
1966 if (pos == map->len)
1967 map->queues[map->len++] = index;
1968#ifdef CONFIG_NUMA
1969 if (numa_node_id == -2)
1970 numa_node_id = cpu_to_node(cpu);
1971 else if (numa_node_id != cpu_to_node(cpu))
1972 numa_node_id = -1;
1973#endif
1974 } else if (dev_maps) {
1975 /* fill in the new device map from the old device map */
1976 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1977 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
1978 }
1979
1980 }
1981
1982 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
1983
1984 /* Cleanup old maps */
1985 if (dev_maps) {
1986 for_each_possible_cpu(cpu) {
1987 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
1988 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1989 if (map && map != new_map)
1990 kfree_rcu(map, rcu);
1991 }
1992
1993 kfree_rcu(dev_maps, rcu);
1994 }
1995
1996 dev_maps = new_dev_maps;
1997 active = true;
1998
1999out_no_new_maps:
2000 /* update Tx queue numa node */
2001 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2002 (numa_node_id >= 0) ? numa_node_id :
2003 NUMA_NO_NODE);
2004
2005 if (!dev_maps)
2006 goto out_no_maps;
2007
2008 /* removes queue from unused CPUs */
2009 for_each_possible_cpu(cpu) {
2010 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2011 continue;
2012
2013 if (remove_xps_queue(dev_maps, cpu, index))
2014 active = true;
2015 }
2016
2017 /* free map if not active */
2018 if (!active) {
2019 RCU_INIT_POINTER(dev->xps_maps, NULL);
2020 kfree_rcu(dev_maps, rcu);
2021 }
2022
2023out_no_maps:
2024 mutex_unlock(&xps_map_mutex);
2025
2026 return 0;
2027error:
2028 /* remove any maps that we added */
2029 for_each_possible_cpu(cpu) {
2030 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2031 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2032 NULL;
2033 if (new_map && new_map != map)
2034 kfree(new_map);
2035 }
2036
2037 mutex_unlock(&xps_map_mutex);
2038
2039 kfree(new_dev_maps);
2040 return -ENOMEM;
2041}
2042EXPORT_SYMBOL(netif_set_xps_queue);
2043
2044#endif
2045/*
2046 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2047 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2048 */
2049int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2050{
2051 int rc;
2052
2053 if (txq < 1 || txq > dev->num_tx_queues)
2054 return -EINVAL;
2055
2056 if (dev->reg_state == NETREG_REGISTERED ||
2057 dev->reg_state == NETREG_UNREGISTERING) {
2058 ASSERT_RTNL();
2059
2060 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2061 txq);
2062 if (rc)
2063 return rc;
2064
2065 if (dev->num_tc)
2066 netif_setup_tc(dev, txq);
2067
2068 if (txq < dev->real_num_tx_queues) {
2069 qdisc_reset_all_tx_gt(dev, txq);
2070#ifdef CONFIG_XPS
2071 netif_reset_xps_queues_gt(dev, txq);
2072#endif
2073 }
2074 }
2075
2076 dev->real_num_tx_queues = txq;
2077 return 0;
2078}
2079EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2080
2081#ifdef CONFIG_SYSFS
2082/**
2083 * netif_set_real_num_rx_queues - set actual number of RX queues used
2084 * @dev: Network device
2085 * @rxq: Actual number of RX queues
2086 *
2087 * This must be called either with the rtnl_lock held or before
2088 * registration of the net device. Returns 0 on success, or a
2089 * negative error code. If called before registration, it always
2090 * succeeds.
2091 */
2092int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2093{
2094 int rc;
2095
2096 if (rxq < 1 || rxq > dev->num_rx_queues)
2097 return -EINVAL;
2098
2099 if (dev->reg_state == NETREG_REGISTERED) {
2100 ASSERT_RTNL();
2101
2102 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2103 rxq);
2104 if (rc)
2105 return rc;
2106 }
2107
2108 dev->real_num_rx_queues = rxq;
2109 return 0;
2110}
2111EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2112#endif
2113
2114/**
2115 * netif_get_num_default_rss_queues - default number of RSS queues
2116 *
2117 * This routine should set an upper limit on the number of RSS queues
2118 * used by default by multiqueue devices.
2119 */
2120int netif_get_num_default_rss_queues(void)
2121{
2122 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2123}
2124EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2125
2126static inline void __netif_reschedule(struct Qdisc *q)
2127{
2128 struct softnet_data *sd;
2129 unsigned long flags;
2130
2131 local_irq_save(flags);
2132 sd = &__get_cpu_var(softnet_data);
2133 q->next_sched = NULL;
2134 *sd->output_queue_tailp = q;
2135 sd->output_queue_tailp = &q->next_sched;
2136 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2137 local_irq_restore(flags);
2138}
2139
2140void __netif_schedule(struct Qdisc *q)
2141{
2142 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2143 __netif_reschedule(q);
2144}
2145EXPORT_SYMBOL(__netif_schedule);
2146
2147struct dev_kfree_skb_cb {
2148 enum skb_free_reason reason;
2149};
2150
2151static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2152{
2153 return (struct dev_kfree_skb_cb *)skb->cb;
2154}
2155
2156void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2157{
2158 unsigned long flags;
2159
2160 if (likely(atomic_read(&skb->users) == 1)) {
2161 smp_rmb();
2162 atomic_set(&skb->users, 0);
2163 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2164 return;
2165 }
2166 get_kfree_skb_cb(skb)->reason = reason;
2167 local_irq_save(flags);
2168 skb->next = __this_cpu_read(softnet_data.completion_queue);
2169 __this_cpu_write(softnet_data.completion_queue, skb);
2170 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2171 local_irq_restore(flags);
2172}
2173EXPORT_SYMBOL(__dev_kfree_skb_irq);
2174
2175void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2176{
2177 if (in_irq() || irqs_disabled())
2178 __dev_kfree_skb_irq(skb, reason);
2179 else
2180 dev_kfree_skb(skb);
2181}
2182EXPORT_SYMBOL(__dev_kfree_skb_any);
2183
2184
2185/**
2186 * netif_device_detach - mark device as removed
2187 * @dev: network device
2188 *
2189 * Mark device as removed from system and therefore no longer available.
2190 */
2191void netif_device_detach(struct net_device *dev)
2192{
2193 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2194 netif_running(dev)) {
2195 netif_tx_stop_all_queues(dev);
2196 }
2197}
2198EXPORT_SYMBOL(netif_device_detach);
2199
2200/**
2201 * netif_device_attach - mark device as attached
2202 * @dev: network device
2203 *
2204 * Mark device as attached from system and restart if needed.
2205 */
2206void netif_device_attach(struct net_device *dev)
2207{
2208 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2209 netif_running(dev)) {
2210 netif_tx_wake_all_queues(dev);
2211 __netdev_watchdog_up(dev);
2212 }
2213}
2214EXPORT_SYMBOL(netif_device_attach);
2215
2216static void skb_warn_bad_offload(const struct sk_buff *skb)
2217{
2218 static const netdev_features_t null_features = 0;
2219 struct net_device *dev = skb->dev;
2220 const char *driver = "";
2221
2222 if (!net_ratelimit())
2223 return;
2224
2225 if (dev && dev->dev.parent)
2226 driver = dev_driver_string(dev->dev.parent);
2227
2228 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2229 "gso_type=%d ip_summed=%d\n",
2230 driver, dev ? &dev->features : &null_features,
2231 skb->sk ? &skb->sk->sk_route_caps : &null_features,
2232 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2233 skb_shinfo(skb)->gso_type, skb->ip_summed);
2234}
2235
2236/*
2237 * Invalidate hardware checksum when packet is to be mangled, and
2238 * complete checksum manually on outgoing path.
2239 */
2240int skb_checksum_help(struct sk_buff *skb)
2241{
2242 __wsum csum;
2243 int ret = 0, offset;
2244
2245 if (skb->ip_summed == CHECKSUM_COMPLETE)
2246 goto out_set_summed;
2247
2248 if (unlikely(skb_shinfo(skb)->gso_size)) {
2249 skb_warn_bad_offload(skb);
2250 return -EINVAL;
2251 }
2252
2253 /* Before computing a checksum, we should make sure no frag could
2254 * be modified by an external entity : checksum could be wrong.
2255 */
2256 if (skb_has_shared_frag(skb)) {
2257 ret = __skb_linearize(skb);
2258 if (ret)
2259 goto out;
2260 }
2261
2262 offset = skb_checksum_start_offset(skb);
2263 BUG_ON(offset >= skb_headlen(skb));
2264 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2265
2266 offset += skb->csum_offset;
2267 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2268
2269 if (skb_cloned(skb) &&
2270 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2271 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2272 if (ret)
2273 goto out;
2274 }
2275
2276 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
2277out_set_summed:
2278 skb->ip_summed = CHECKSUM_NONE;
2279out:
2280 return ret;
2281}
2282EXPORT_SYMBOL(skb_checksum_help);
2283
2284__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2285{
2286 unsigned int vlan_depth = skb->mac_len;
2287 __be16 type = skb->protocol;
2288
2289 /* Tunnel gso handlers can set protocol to ethernet. */
2290 if (type == htons(ETH_P_TEB)) {
2291 struct ethhdr *eth;
2292
2293 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2294 return 0;
2295
2296 eth = (struct ethhdr *)skb_mac_header(skb);
2297 type = eth->h_proto;
2298 }
2299
2300 /* if skb->protocol is 802.1Q/AD then the header should already be
2301 * present at mac_len - VLAN_HLEN (if mac_len > 0), or at
2302 * ETH_HLEN otherwise
2303 */
2304 if (type == htons(ETH_P_8021Q) || type == htons(ETH_P_8021AD)) {
2305 if (vlan_depth) {
2306 if (unlikely(WARN_ON(vlan_depth < VLAN_HLEN)))
2307 return 0;
2308 vlan_depth -= VLAN_HLEN;
2309 } else {
2310 vlan_depth = ETH_HLEN;
2311 }
2312 do {
2313 struct vlan_hdr *vh;
2314
2315 if (unlikely(!pskb_may_pull(skb,
2316 vlan_depth + VLAN_HLEN)))
2317 return 0;
2318
2319 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
2320 type = vh->h_vlan_encapsulated_proto;
2321 vlan_depth += VLAN_HLEN;
2322 } while (type == htons(ETH_P_8021Q) ||
2323 type == htons(ETH_P_8021AD));
2324 }
2325
2326 *depth = vlan_depth;
2327
2328 return type;
2329}
2330
2331/**
2332 * skb_mac_gso_segment - mac layer segmentation handler.
2333 * @skb: buffer to segment
2334 * @features: features for the output path (see dev->features)
2335 */
2336struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2337 netdev_features_t features)
2338{
2339 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2340 struct packet_offload *ptype;
2341 int vlan_depth = skb->mac_len;
2342 __be16 type = skb_network_protocol(skb, &vlan_depth);
2343
2344 if (unlikely(!type))
2345 return ERR_PTR(-EINVAL);
2346
2347 __skb_pull(skb, vlan_depth);
2348
2349 rcu_read_lock();
2350 list_for_each_entry_rcu(ptype, &offload_base, list) {
2351 if (ptype->type == type && ptype->callbacks.gso_segment) {
2352 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
2353 int err;
2354
2355 err = ptype->callbacks.gso_send_check(skb);
2356 segs = ERR_PTR(err);
2357 if (err || skb_gso_ok(skb, features))
2358 break;
2359 __skb_push(skb, (skb->data -
2360 skb_network_header(skb)));
2361 }
2362 segs = ptype->callbacks.gso_segment(skb, features);
2363 break;
2364 }
2365 }
2366 rcu_read_unlock();
2367
2368 __skb_push(skb, skb->data - skb_mac_header(skb));
2369
2370 return segs;
2371}
2372EXPORT_SYMBOL(skb_mac_gso_segment);
2373
2374
2375/* openvswitch calls this on rx path, so we need a different check.
2376 */
2377static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2378{
2379 if (tx_path)
2380 return skb->ip_summed != CHECKSUM_PARTIAL;
2381 else
2382 return skb->ip_summed == CHECKSUM_NONE;
2383}
2384
2385/**
2386 * __skb_gso_segment - Perform segmentation on skb.
2387 * @skb: buffer to segment
2388 * @features: features for the output path (see dev->features)
2389 * @tx_path: whether it is called in TX path
2390 *
2391 * This function segments the given skb and returns a list of segments.
2392 *
2393 * It may return NULL if the skb requires no segmentation. This is
2394 * only possible when GSO is used for verifying header integrity.
2395 */
2396struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2397 netdev_features_t features, bool tx_path)
2398{
2399 if (unlikely(skb_needs_check(skb, tx_path))) {
2400 int err;
2401
2402 skb_warn_bad_offload(skb);
2403
2404 if (skb_header_cloned(skb) &&
2405 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
2406 return ERR_PTR(err);
2407 }
2408
2409 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2410 SKB_GSO_CB(skb)->encap_level = 0;
2411
2412 skb_reset_mac_header(skb);
2413 skb_reset_mac_len(skb);
2414
2415 return skb_mac_gso_segment(skb, features);
2416}
2417EXPORT_SYMBOL(__skb_gso_segment);
2418
2419/* Take action when hardware reception checksum errors are detected. */
2420#ifdef CONFIG_BUG
2421void netdev_rx_csum_fault(struct net_device *dev)
2422{
2423 if (net_ratelimit()) {
2424 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2425 dump_stack();
2426 }
2427}
2428EXPORT_SYMBOL(netdev_rx_csum_fault);
2429#endif
2430
2431/* Actually, we should eliminate this check as soon as we know, that:
2432 * 1. IOMMU is present and allows to map all the memory.
2433 * 2. No high memory really exists on this machine.
2434 */
2435
2436static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2437{
2438#ifdef CONFIG_HIGHMEM
2439 int i;
2440 if (!(dev->features & NETIF_F_HIGHDMA)) {
2441 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2442 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2443 if (PageHighMem(skb_frag_page(frag)))
2444 return 1;
2445 }
2446 }
2447
2448 if (PCI_DMA_BUS_IS_PHYS) {
2449 struct device *pdev = dev->dev.parent;
2450
2451 if (!pdev)
2452 return 0;
2453 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2454 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2455 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2456 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2457 return 1;
2458 }
2459 }
2460#endif
2461 return 0;
2462}
2463
2464struct dev_gso_cb {
2465 void (*destructor)(struct sk_buff *skb);
2466};
2467
2468#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
2469
2470static void dev_gso_skb_destructor(struct sk_buff *skb)
2471{
2472 struct dev_gso_cb *cb;
2473
2474 kfree_skb_list(skb->next);
2475 skb->next = NULL;
2476
2477 cb = DEV_GSO_CB(skb);
2478 if (cb->destructor)
2479 cb->destructor(skb);
2480}
2481
2482/**
2483 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2484 * @skb: buffer to segment
2485 * @features: device features as applicable to this skb
2486 *
2487 * This function segments the given skb and stores the list of segments
2488 * in skb->next.
2489 */
2490static int dev_gso_segment(struct sk_buff *skb, netdev_features_t features)
2491{
2492 struct sk_buff *segs;
2493
2494 segs = skb_gso_segment(skb, features);
2495
2496 /* Verifying header integrity only. */
2497 if (!segs)
2498 return 0;
2499
2500 if (IS_ERR(segs))
2501 return PTR_ERR(segs);
2502
2503 skb->next = segs;
2504 DEV_GSO_CB(skb)->destructor = skb->destructor;
2505 skb->destructor = dev_gso_skb_destructor;
2506
2507 return 0;
2508}
2509
2510static netdev_features_t harmonize_features(struct sk_buff *skb,
2511 netdev_features_t features)
2512{
2513 int tmp;
2514
2515 if (skb->ip_summed != CHECKSUM_NONE &&
2516 !can_checksum_protocol(features, skb_network_protocol(skb, &tmp))) {
2517 features &= ~NETIF_F_ALL_CSUM;
2518 } else if (illegal_highdma(skb->dev, skb)) {
2519 features &= ~NETIF_F_SG;
2520 }
2521
2522 return features;
2523}
2524
2525netdev_features_t netif_skb_features(struct sk_buff *skb)
2526{
2527 __be16 protocol = skb->protocol;
2528 netdev_features_t features = skb->dev->features;
2529
2530 if (skb_shinfo(skb)->gso_segs > skb->dev->gso_max_segs)
2531 features &= ~NETIF_F_GSO_MASK;
2532
2533 if (protocol == htons(ETH_P_8021Q) || protocol == htons(ETH_P_8021AD)) {
2534 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2535 protocol = veh->h_vlan_encapsulated_proto;
2536 } else if (!vlan_tx_tag_present(skb)) {
2537 return harmonize_features(skb, features);
2538 }
2539
2540 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_CTAG_TX |
2541 NETIF_F_HW_VLAN_STAG_TX);
2542
2543 if (protocol == htons(ETH_P_8021Q) || protocol == htons(ETH_P_8021AD))
2544 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
2545 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2546 NETIF_F_HW_VLAN_STAG_TX;
2547
2548 return harmonize_features(skb, features);
2549}
2550EXPORT_SYMBOL(netif_skb_features);
2551
2552int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2553 struct netdev_queue *txq)
2554{
2555 const struct net_device_ops *ops = dev->netdev_ops;
2556 int rc = NETDEV_TX_OK;
2557 unsigned int skb_len;
2558
2559 if (likely(!skb->next)) {
2560 netdev_features_t features;
2561
2562 /*
2563 * If device doesn't need skb->dst, release it right now while
2564 * its hot in this cpu cache
2565 */
2566 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2567 skb_dst_drop(skb);
2568
2569 features = netif_skb_features(skb);
2570
2571 if (vlan_tx_tag_present(skb) &&
2572 !vlan_hw_offload_capable(features, skb->vlan_proto)) {
2573 skb = __vlan_put_tag(skb, skb->vlan_proto,
2574 vlan_tx_tag_get(skb));
2575 if (unlikely(!skb))
2576 goto out;
2577
2578 skb->vlan_tci = 0;
2579 }
2580
2581 /* If encapsulation offload request, verify we are testing
2582 * hardware encapsulation features instead of standard
2583 * features for the netdev
2584 */
2585 if (skb->encapsulation)
2586 features &= dev->hw_enc_features;
2587
2588 if (netif_needs_gso(skb, features)) {
2589 if (unlikely(dev_gso_segment(skb, features)))
2590 goto out_kfree_skb;
2591 if (skb->next)
2592 goto gso;
2593 } else {
2594 if (skb_needs_linearize(skb, features) &&
2595 __skb_linearize(skb))
2596 goto out_kfree_skb;
2597
2598 /* If packet is not checksummed and device does not
2599 * support checksumming for this protocol, complete
2600 * checksumming here.
2601 */
2602 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2603 if (skb->encapsulation)
2604 skb_set_inner_transport_header(skb,
2605 skb_checksum_start_offset(skb));
2606 else
2607 skb_set_transport_header(skb,
2608 skb_checksum_start_offset(skb));
2609 if (!(features & NETIF_F_ALL_CSUM) &&
2610 skb_checksum_help(skb))
2611 goto out_kfree_skb;
2612 }
2613 }
2614
2615 if (!list_empty(&ptype_all))
2616 dev_queue_xmit_nit(skb, dev);
2617
2618 skb_len = skb->len;
2619 trace_net_dev_start_xmit(skb, dev);
2620 rc = ops->ndo_start_xmit(skb, dev);
2621 trace_net_dev_xmit(skb, rc, dev, skb_len);
2622 if (rc == NETDEV_TX_OK)
2623 txq_trans_update(txq);
2624 return rc;
2625 }
2626
2627gso:
2628 do {
2629 struct sk_buff *nskb = skb->next;
2630
2631 skb->next = nskb->next;
2632 nskb->next = NULL;
2633
2634 if (!list_empty(&ptype_all))
2635 dev_queue_xmit_nit(nskb, dev);
2636
2637 skb_len = nskb->len;
2638 trace_net_dev_start_xmit(nskb, dev);
2639 rc = ops->ndo_start_xmit(nskb, dev);
2640 trace_net_dev_xmit(nskb, rc, dev, skb_len);
2641 if (unlikely(rc != NETDEV_TX_OK)) {
2642 if (rc & ~NETDEV_TX_MASK)
2643 goto out_kfree_gso_skb;
2644 nskb->next = skb->next;
2645 skb->next = nskb;
2646 return rc;
2647 }
2648 txq_trans_update(txq);
2649 if (unlikely(netif_xmit_stopped(txq) && skb->next))
2650 return NETDEV_TX_BUSY;
2651 } while (skb->next);
2652
2653out_kfree_gso_skb:
2654 if (likely(skb->next == NULL)) {
2655 skb->destructor = DEV_GSO_CB(skb)->destructor;
2656 consume_skb(skb);
2657 return rc;
2658 }
2659out_kfree_skb:
2660 kfree_skb(skb);
2661out:
2662 return rc;
2663}
2664EXPORT_SYMBOL_GPL(dev_hard_start_xmit);
2665
2666static void qdisc_pkt_len_init(struct sk_buff *skb)
2667{
2668 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2669
2670 qdisc_skb_cb(skb)->pkt_len = skb->len;
2671
2672 /* To get more precise estimation of bytes sent on wire,
2673 * we add to pkt_len the headers size of all segments
2674 */
2675 if (shinfo->gso_size) {
2676 unsigned int hdr_len;
2677 u16 gso_segs = shinfo->gso_segs;
2678
2679 /* mac layer + network layer */
2680 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2681
2682 /* + transport layer */
2683 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2684 hdr_len += tcp_hdrlen(skb);
2685 else
2686 hdr_len += sizeof(struct udphdr);
2687
2688 if (shinfo->gso_type & SKB_GSO_DODGY)
2689 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
2690 shinfo->gso_size);
2691
2692 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
2693 }
2694}
2695
2696static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2697 struct net_device *dev,
2698 struct netdev_queue *txq)
2699{
2700 spinlock_t *root_lock = qdisc_lock(q);
2701 bool contended;
2702 int rc;
2703
2704 qdisc_pkt_len_init(skb);
2705 qdisc_calculate_pkt_len(skb, q);
2706 /*
2707 * Heuristic to force contended enqueues to serialize on a
2708 * separate lock before trying to get qdisc main lock.
2709 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2710 * and dequeue packets faster.
2711 */
2712 contended = qdisc_is_running(q);
2713 if (unlikely(contended))
2714 spin_lock(&q->busylock);
2715
2716 spin_lock(root_lock);
2717 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2718 kfree_skb(skb);
2719 rc = NET_XMIT_DROP;
2720 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2721 qdisc_run_begin(q)) {
2722 /*
2723 * This is a work-conserving queue; there are no old skbs
2724 * waiting to be sent out; and the qdisc is not running -
2725 * xmit the skb directly.
2726 */
2727 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2728 skb_dst_force(skb);
2729
2730 qdisc_bstats_update(q, skb);
2731
2732 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2733 if (unlikely(contended)) {
2734 spin_unlock(&q->busylock);
2735 contended = false;
2736 }
2737 __qdisc_run(q);
2738 } else
2739 qdisc_run_end(q);
2740
2741 rc = NET_XMIT_SUCCESS;
2742 } else {
2743 skb_dst_force(skb);
2744 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
2745 if (qdisc_run_begin(q)) {
2746 if (unlikely(contended)) {
2747 spin_unlock(&q->busylock);
2748 contended = false;
2749 }
2750 __qdisc_run(q);
2751 }
2752 }
2753 spin_unlock(root_lock);
2754 if (unlikely(contended))
2755 spin_unlock(&q->busylock);
2756 return rc;
2757}
2758
2759#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2760static void skb_update_prio(struct sk_buff *skb)
2761{
2762 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
2763
2764 if (!skb->priority && skb->sk && map) {
2765 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2766
2767 if (prioidx < map->priomap_len)
2768 skb->priority = map->priomap[prioidx];
2769 }
2770}
2771#else
2772#define skb_update_prio(skb)
2773#endif
2774
2775static DEFINE_PER_CPU(int, xmit_recursion);
2776#define RECURSION_LIMIT 10
2777
2778/**
2779 * dev_loopback_xmit - loop back @skb
2780 * @skb: buffer to transmit
2781 */
2782int dev_loopback_xmit(struct sk_buff *skb)
2783{
2784 skb_reset_mac_header(skb);
2785 __skb_pull(skb, skb_network_offset(skb));
2786 skb->pkt_type = PACKET_LOOPBACK;
2787 skb->ip_summed = CHECKSUM_UNNECESSARY;
2788 WARN_ON(!skb_dst(skb));
2789 skb_dst_force(skb);
2790 netif_rx_ni(skb);
2791 return 0;
2792}
2793EXPORT_SYMBOL(dev_loopback_xmit);
2794
2795/**
2796 * __dev_queue_xmit - transmit a buffer
2797 * @skb: buffer to transmit
2798 * @accel_priv: private data used for L2 forwarding offload
2799 *
2800 * Queue a buffer for transmission to a network device. The caller must
2801 * have set the device and priority and built the buffer before calling
2802 * this function. The function can be called from an interrupt.
2803 *
2804 * A negative errno code is returned on a failure. A success does not
2805 * guarantee the frame will be transmitted as it may be dropped due
2806 * to congestion or traffic shaping.
2807 *
2808 * -----------------------------------------------------------------------------------
2809 * I notice this method can also return errors from the queue disciplines,
2810 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2811 * be positive.
2812 *
2813 * Regardless of the return value, the skb is consumed, so it is currently
2814 * difficult to retry a send to this method. (You can bump the ref count
2815 * before sending to hold a reference for retry if you are careful.)
2816 *
2817 * When calling this method, interrupts MUST be enabled. This is because
2818 * the BH enable code must have IRQs enabled so that it will not deadlock.
2819 * --BLG
2820 */
2821static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
2822{
2823 struct net_device *dev = skb->dev;
2824 struct netdev_queue *txq;
2825 struct Qdisc *q;
2826 int rc = -ENOMEM;
2827
2828 skb_reset_mac_header(skb);
2829
2830 /* Disable soft irqs for various locks below. Also
2831 * stops preemption for RCU.
2832 */
2833 rcu_read_lock_bh();
2834
2835 skb_update_prio(skb);
2836
2837 txq = netdev_pick_tx(dev, skb, accel_priv);
2838 q = rcu_dereference_bh(txq->qdisc);
2839
2840#ifdef CONFIG_NET_CLS_ACT
2841 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
2842#endif
2843 trace_net_dev_queue(skb);
2844 if (q->enqueue) {
2845 rc = __dev_xmit_skb(skb, q, dev, txq);
2846 goto out;
2847 }
2848
2849 /* The device has no queue. Common case for software devices:
2850 loopback, all the sorts of tunnels...
2851
2852 Really, it is unlikely that netif_tx_lock protection is necessary
2853 here. (f.e. loopback and IP tunnels are clean ignoring statistics
2854 counters.)
2855 However, it is possible, that they rely on protection
2856 made by us here.
2857
2858 Check this and shot the lock. It is not prone from deadlocks.
2859 Either shot noqueue qdisc, it is even simpler 8)
2860 */
2861 if (dev->flags & IFF_UP) {
2862 int cpu = smp_processor_id(); /* ok because BHs are off */
2863
2864 if (txq->xmit_lock_owner != cpu) {
2865
2866 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2867 goto recursion_alert;
2868
2869 HARD_TX_LOCK(dev, txq, cpu);
2870
2871 if (!netif_xmit_stopped(txq)) {
2872 __this_cpu_inc(xmit_recursion);
2873 rc = dev_hard_start_xmit(skb, dev, txq);
2874 __this_cpu_dec(xmit_recursion);
2875 if (dev_xmit_complete(rc)) {
2876 HARD_TX_UNLOCK(dev, txq);
2877 goto out;
2878 }
2879 }
2880 HARD_TX_UNLOCK(dev, txq);
2881 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
2882 dev->name);
2883 } else {
2884 /* Recursion is detected! It is possible,
2885 * unfortunately
2886 */
2887recursion_alert:
2888 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
2889 dev->name);
2890 }
2891 }
2892
2893 rc = -ENETDOWN;
2894 rcu_read_unlock_bh();
2895
2896 atomic_long_inc(&dev->tx_dropped);
2897 kfree_skb(skb);
2898 return rc;
2899out:
2900 rcu_read_unlock_bh();
2901 return rc;
2902}
2903
2904int dev_queue_xmit(struct sk_buff *skb)
2905{
2906 return __dev_queue_xmit(skb, NULL);
2907}
2908EXPORT_SYMBOL(dev_queue_xmit);
2909
2910int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
2911{
2912 return __dev_queue_xmit(skb, accel_priv);
2913}
2914EXPORT_SYMBOL(dev_queue_xmit_accel);
2915
2916
2917/*=======================================================================
2918 Receiver routines
2919 =======================================================================*/
2920
2921int netdev_max_backlog __read_mostly = 1000;
2922EXPORT_SYMBOL(netdev_max_backlog);
2923
2924int netdev_tstamp_prequeue __read_mostly = 1;
2925int netdev_budget __read_mostly = 300;
2926int weight_p __read_mostly = 64; /* old backlog weight */
2927
2928/* Called with irq disabled */
2929static inline void ____napi_schedule(struct softnet_data *sd,
2930 struct napi_struct *napi)
2931{
2932 list_add_tail(&napi->poll_list, &sd->poll_list);
2933 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2934}
2935
2936#ifdef CONFIG_RPS
2937
2938/* One global table that all flow-based protocols share. */
2939struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
2940EXPORT_SYMBOL(rps_sock_flow_table);
2941
2942struct static_key rps_needed __read_mostly;
2943
2944static struct rps_dev_flow *
2945set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2946 struct rps_dev_flow *rflow, u16 next_cpu)
2947{
2948 if (next_cpu != RPS_NO_CPU) {
2949#ifdef CONFIG_RFS_ACCEL
2950 struct netdev_rx_queue *rxqueue;
2951 struct rps_dev_flow_table *flow_table;
2952 struct rps_dev_flow *old_rflow;
2953 u32 flow_id;
2954 u16 rxq_index;
2955 int rc;
2956
2957 /* Should we steer this flow to a different hardware queue? */
2958 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2959 !(dev->features & NETIF_F_NTUPLE))
2960 goto out;
2961 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2962 if (rxq_index == skb_get_rx_queue(skb))
2963 goto out;
2964
2965 rxqueue = dev->_rx + rxq_index;
2966 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2967 if (!flow_table)
2968 goto out;
2969 flow_id = skb_get_hash(skb) & flow_table->mask;
2970 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2971 rxq_index, flow_id);
2972 if (rc < 0)
2973 goto out;
2974 old_rflow = rflow;
2975 rflow = &flow_table->flows[flow_id];
2976 rflow->filter = rc;
2977 if (old_rflow->filter == rflow->filter)
2978 old_rflow->filter = RPS_NO_FILTER;
2979 out:
2980#endif
2981 rflow->last_qtail =
2982 per_cpu(softnet_data, next_cpu).input_queue_head;
2983 }
2984
2985 rflow->cpu = next_cpu;
2986 return rflow;
2987}
2988
2989/*
2990 * get_rps_cpu is called from netif_receive_skb and returns the target
2991 * CPU from the RPS map of the receiving queue for a given skb.
2992 * rcu_read_lock must be held on entry.
2993 */
2994static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2995 struct rps_dev_flow **rflowp)
2996{
2997 struct netdev_rx_queue *rxqueue;
2998 struct rps_map *map;
2999 struct rps_dev_flow_table *flow_table;
3000 struct rps_sock_flow_table *sock_flow_table;
3001 int cpu = -1;
3002 u16 tcpu;
3003 u32 hash;
3004
3005 if (skb_rx_queue_recorded(skb)) {
3006 u16 index = skb_get_rx_queue(skb);
3007 if (unlikely(index >= dev->real_num_rx_queues)) {
3008 WARN_ONCE(dev->real_num_rx_queues > 1,
3009 "%s received packet on queue %u, but number "
3010 "of RX queues is %u\n",
3011 dev->name, index, dev->real_num_rx_queues);
3012 goto done;
3013 }
3014 rxqueue = dev->_rx + index;
3015 } else
3016 rxqueue = dev->_rx;
3017
3018 map = rcu_dereference(rxqueue->rps_map);
3019 if (map) {
3020 if (map->len == 1 &&
3021 !rcu_access_pointer(rxqueue->rps_flow_table)) {
3022 tcpu = map->cpus[0];
3023 if (cpu_online(tcpu))
3024 cpu = tcpu;
3025 goto done;
3026 }
3027 } else if (!rcu_access_pointer(rxqueue->rps_flow_table)) {
3028 goto done;
3029 }
3030
3031 skb_reset_network_header(skb);
3032 hash = skb_get_hash(skb);
3033 if (!hash)
3034 goto done;
3035
3036 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3037 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3038 if (flow_table && sock_flow_table) {
3039 u16 next_cpu;
3040 struct rps_dev_flow *rflow;
3041
3042 rflow = &flow_table->flows[hash & flow_table->mask];
3043 tcpu = rflow->cpu;
3044
3045 next_cpu = sock_flow_table->ents[hash & sock_flow_table->mask];
3046
3047 /*
3048 * If the desired CPU (where last recvmsg was done) is
3049 * different from current CPU (one in the rx-queue flow
3050 * table entry), switch if one of the following holds:
3051 * - Current CPU is unset (equal to RPS_NO_CPU).
3052 * - Current CPU is offline.
3053 * - The current CPU's queue tail has advanced beyond the
3054 * last packet that was enqueued using this table entry.
3055 * This guarantees that all previous packets for the flow
3056 * have been dequeued, thus preserving in order delivery.
3057 */
3058 if (unlikely(tcpu != next_cpu) &&
3059 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
3060 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3061 rflow->last_qtail)) >= 0)) {
3062 tcpu = next_cpu;
3063 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3064 }
3065
3066 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
3067 *rflowp = rflow;
3068 cpu = tcpu;
3069 goto done;
3070 }
3071 }
3072
3073 if (map) {
3074 tcpu = map->cpus[((u64) hash * map->len) >> 32];
3075
3076 if (cpu_online(tcpu)) {
3077 cpu = tcpu;
3078 goto done;
3079 }
3080 }
3081
3082done:
3083 return cpu;
3084}
3085
3086#ifdef CONFIG_RFS_ACCEL
3087
3088/**
3089 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3090 * @dev: Device on which the filter was set
3091 * @rxq_index: RX queue index
3092 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3093 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3094 *
3095 * Drivers that implement ndo_rx_flow_steer() should periodically call
3096 * this function for each installed filter and remove the filters for
3097 * which it returns %true.
3098 */
3099bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3100 u32 flow_id, u16 filter_id)
3101{
3102 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3103 struct rps_dev_flow_table *flow_table;
3104 struct rps_dev_flow *rflow;
3105 bool expire = true;
3106 int cpu;
3107
3108 rcu_read_lock();
3109 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3110 if (flow_table && flow_id <= flow_table->mask) {
3111 rflow = &flow_table->flows[flow_id];
3112 cpu = ACCESS_ONCE(rflow->cpu);
3113 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
3114 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3115 rflow->last_qtail) <
3116 (int)(10 * flow_table->mask)))
3117 expire = false;
3118 }
3119 rcu_read_unlock();
3120 return expire;
3121}
3122EXPORT_SYMBOL(rps_may_expire_flow);
3123
3124#endif /* CONFIG_RFS_ACCEL */
3125
3126/* Called from hardirq (IPI) context */
3127static void rps_trigger_softirq(void *data)
3128{
3129 struct softnet_data *sd = data;
3130
3131 ____napi_schedule(sd, &sd->backlog);
3132 sd->received_rps++;
3133}
3134
3135#endif /* CONFIG_RPS */
3136
3137/*
3138 * Check if this softnet_data structure is another cpu one
3139 * If yes, queue it to our IPI list and return 1
3140 * If no, return 0
3141 */
3142static int rps_ipi_queued(struct softnet_data *sd)
3143{
3144#ifdef CONFIG_RPS
3145 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
3146
3147 if (sd != mysd) {
3148 sd->rps_ipi_next = mysd->rps_ipi_list;
3149 mysd->rps_ipi_list = sd;
3150
3151 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3152 return 1;
3153 }
3154#endif /* CONFIG_RPS */
3155 return 0;
3156}
3157
3158#ifdef CONFIG_NET_FLOW_LIMIT
3159int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3160#endif
3161
3162static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3163{
3164#ifdef CONFIG_NET_FLOW_LIMIT
3165 struct sd_flow_limit *fl;
3166 struct softnet_data *sd;
3167 unsigned int old_flow, new_flow;
3168
3169 if (qlen < (netdev_max_backlog >> 1))
3170 return false;
3171
3172 sd = &__get_cpu_var(softnet_data);
3173
3174 rcu_read_lock();
3175 fl = rcu_dereference(sd->flow_limit);
3176 if (fl) {
3177 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3178 old_flow = fl->history[fl->history_head];
3179 fl->history[fl->history_head] = new_flow;
3180
3181 fl->history_head++;
3182 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3183
3184 if (likely(fl->buckets[old_flow]))
3185 fl->buckets[old_flow]--;
3186
3187 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3188 fl->count++;
3189 rcu_read_unlock();
3190 return true;
3191 }
3192 }
3193 rcu_read_unlock();
3194#endif
3195 return false;
3196}
3197
3198/*
3199 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3200 * queue (may be a remote CPU queue).
3201 */
3202static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3203 unsigned int *qtail)
3204{
3205 struct softnet_data *sd;
3206 unsigned long flags;
3207 unsigned int qlen;
3208
3209 sd = &per_cpu(softnet_data, cpu);
3210
3211 local_irq_save(flags);
3212
3213 rps_lock(sd);
3214 qlen = skb_queue_len(&sd->input_pkt_queue);
3215 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3216 if (skb_queue_len(&sd->input_pkt_queue)) {
3217enqueue:
3218 __skb_queue_tail(&sd->input_pkt_queue, skb);
3219 input_queue_tail_incr_save(sd, qtail);
3220 rps_unlock(sd);
3221 local_irq_restore(flags);
3222 return NET_RX_SUCCESS;
3223 }
3224
3225 /* Schedule NAPI for backlog device
3226 * We can use non atomic operation since we own the queue lock
3227 */
3228 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3229 if (!rps_ipi_queued(sd))
3230 ____napi_schedule(sd, &sd->backlog);
3231 }
3232 goto enqueue;
3233 }
3234
3235 sd->dropped++;
3236 rps_unlock(sd);
3237
3238 local_irq_restore(flags);
3239
3240 atomic_long_inc(&skb->dev->rx_dropped);
3241 kfree_skb(skb);
3242 return NET_RX_DROP;
3243}
3244
3245static int netif_rx_internal(struct sk_buff *skb)
3246{
3247 int ret;
3248
3249 net_timestamp_check(netdev_tstamp_prequeue, skb);
3250
3251 trace_netif_rx(skb);
3252#ifdef CONFIG_RPS
3253 if (static_key_false(&rps_needed)) {
3254 struct rps_dev_flow voidflow, *rflow = &voidflow;
3255 int cpu;
3256
3257 preempt_disable();
3258 rcu_read_lock();
3259
3260 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3261 if (cpu < 0)
3262 cpu = smp_processor_id();
3263
3264 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3265
3266 rcu_read_unlock();
3267 preempt_enable();
3268 } else
3269#endif
3270 {
3271 unsigned int qtail;
3272 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3273 put_cpu();
3274 }
3275 return ret;
3276}
3277
3278/**
3279 * netif_rx - post buffer to the network code
3280 * @skb: buffer to post
3281 *
3282 * This function receives a packet from a device driver and queues it for
3283 * the upper (protocol) levels to process. It always succeeds. The buffer
3284 * may be dropped during processing for congestion control or by the
3285 * protocol layers.
3286 *
3287 * return values:
3288 * NET_RX_SUCCESS (no congestion)
3289 * NET_RX_DROP (packet was dropped)
3290 *
3291 */
3292
3293int netif_rx(struct sk_buff *skb)
3294{
3295 trace_netif_rx_entry(skb);
3296
3297 return netif_rx_internal(skb);
3298}
3299EXPORT_SYMBOL(netif_rx);
3300
3301int netif_rx_ni(struct sk_buff *skb)
3302{
3303 int err;
3304
3305 trace_netif_rx_ni_entry(skb);
3306
3307 preempt_disable();
3308 err = netif_rx_internal(skb);
3309 if (local_softirq_pending())
3310 do_softirq();
3311 preempt_enable();
3312
3313 return err;
3314}
3315EXPORT_SYMBOL(netif_rx_ni);
3316
3317static void net_tx_action(struct softirq_action *h)
3318{
3319 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3320
3321 if (sd->completion_queue) {
3322 struct sk_buff *clist;
3323
3324 local_irq_disable();
3325 clist = sd->completion_queue;
3326 sd->completion_queue = NULL;
3327 local_irq_enable();
3328
3329 while (clist) {
3330 struct sk_buff *skb = clist;
3331 clist = clist->next;
3332
3333 WARN_ON(atomic_read(&skb->users));
3334 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3335 trace_consume_skb(skb);
3336 else
3337 trace_kfree_skb(skb, net_tx_action);
3338 __kfree_skb(skb);
3339 }
3340 }
3341
3342 if (sd->output_queue) {
3343 struct Qdisc *head;
3344
3345 local_irq_disable();
3346 head = sd->output_queue;
3347 sd->output_queue = NULL;
3348 sd->output_queue_tailp = &sd->output_queue;
3349 local_irq_enable();
3350
3351 while (head) {
3352 struct Qdisc *q = head;
3353 spinlock_t *root_lock;
3354
3355 head = head->next_sched;
3356
3357 root_lock = qdisc_lock(q);
3358 if (spin_trylock(root_lock)) {
3359 smp_mb__before_clear_bit();
3360 clear_bit(__QDISC_STATE_SCHED,
3361 &q->state);
3362 qdisc_run(q);
3363 spin_unlock(root_lock);
3364 } else {
3365 if (!test_bit(__QDISC_STATE_DEACTIVATED,
3366 &q->state)) {
3367 __netif_reschedule(q);
3368 } else {
3369 smp_mb__before_clear_bit();
3370 clear_bit(__QDISC_STATE_SCHED,
3371 &q->state);
3372 }
3373 }
3374 }
3375 }
3376}
3377
3378#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3379 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3380/* This hook is defined here for ATM LANE */
3381int (*br_fdb_test_addr_hook)(struct net_device *dev,
3382 unsigned char *addr) __read_mostly;
3383EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3384#endif
3385
3386#ifdef CONFIG_NET_CLS_ACT
3387/* TODO: Maybe we should just force sch_ingress to be compiled in
3388 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3389 * a compare and 2 stores extra right now if we dont have it on
3390 * but have CONFIG_NET_CLS_ACT
3391 * NOTE: This doesn't stop any functionality; if you dont have
3392 * the ingress scheduler, you just can't add policies on ingress.
3393 *
3394 */
3395static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
3396{
3397 struct net_device *dev = skb->dev;
3398 u32 ttl = G_TC_RTTL(skb->tc_verd);
3399 int result = TC_ACT_OK;
3400 struct Qdisc *q;
3401
3402 if (unlikely(MAX_RED_LOOP < ttl++)) {
3403 net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n",
3404 skb->skb_iif, dev->ifindex);
3405 return TC_ACT_SHOT;
3406 }
3407
3408 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3409 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3410
3411 q = rxq->qdisc;
3412 if (q != &noop_qdisc) {
3413 spin_lock(qdisc_lock(q));
3414 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3415 result = qdisc_enqueue_root(skb, q);
3416 spin_unlock(qdisc_lock(q));
3417 }
3418
3419 return result;
3420}
3421
3422static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3423 struct packet_type **pt_prev,
3424 int *ret, struct net_device *orig_dev)
3425{
3426 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3427
3428 if (!rxq || rxq->qdisc == &noop_qdisc)
3429 goto out;
3430
3431 if (*pt_prev) {
3432 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3433 *pt_prev = NULL;
3434 }
3435
3436 switch (ing_filter(skb, rxq)) {
3437 case TC_ACT_SHOT:
3438 case TC_ACT_STOLEN:
3439 kfree_skb(skb);
3440 return NULL;
3441 }
3442
3443out:
3444 skb->tc_verd = 0;
3445 return skb;
3446}
3447#endif
3448
3449/**
3450 * netdev_rx_handler_register - register receive handler
3451 * @dev: device to register a handler for
3452 * @rx_handler: receive handler to register
3453 * @rx_handler_data: data pointer that is used by rx handler
3454 *
3455 * Register a receive handler for a device. This handler will then be
3456 * called from __netif_receive_skb. A negative errno code is returned
3457 * on a failure.
3458 *
3459 * The caller must hold the rtnl_mutex.
3460 *
3461 * For a general description of rx_handler, see enum rx_handler_result.
3462 */
3463int netdev_rx_handler_register(struct net_device *dev,
3464 rx_handler_func_t *rx_handler,
3465 void *rx_handler_data)
3466{
3467 ASSERT_RTNL();
3468
3469 if (dev->rx_handler)
3470 return -EBUSY;
3471
3472 /* Note: rx_handler_data must be set before rx_handler */
3473 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3474 rcu_assign_pointer(dev->rx_handler, rx_handler);
3475
3476 return 0;
3477}
3478EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3479
3480/**
3481 * netdev_rx_handler_unregister - unregister receive handler
3482 * @dev: device to unregister a handler from
3483 *
3484 * Unregister a receive handler from a device.
3485 *
3486 * The caller must hold the rtnl_mutex.
3487 */
3488void netdev_rx_handler_unregister(struct net_device *dev)
3489{
3490
3491 ASSERT_RTNL();
3492 RCU_INIT_POINTER(dev->rx_handler, NULL);
3493 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3494 * section has a guarantee to see a non NULL rx_handler_data
3495 * as well.
3496 */
3497 synchronize_net();
3498 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
3499}
3500EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3501
3502/*
3503 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3504 * the special handling of PFMEMALLOC skbs.
3505 */
3506static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3507{
3508 switch (skb->protocol) {
3509 case htons(ETH_P_ARP):
3510 case htons(ETH_P_IP):
3511 case htons(ETH_P_IPV6):
3512 case htons(ETH_P_8021Q):
3513 case htons(ETH_P_8021AD):
3514 return true;
3515 default:
3516 return false;
3517 }
3518}
3519
3520static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
3521{
3522 struct packet_type *ptype, *pt_prev;
3523 rx_handler_func_t *rx_handler;
3524 struct net_device *orig_dev;
3525 struct net_device *null_or_dev;
3526 bool deliver_exact = false;
3527 int ret = NET_RX_DROP;
3528 __be16 type;
3529
3530 net_timestamp_check(!netdev_tstamp_prequeue, skb);
3531
3532 trace_netif_receive_skb(skb);
3533
3534 orig_dev = skb->dev;
3535
3536 skb_reset_network_header(skb);
3537 if (!skb_transport_header_was_set(skb))
3538 skb_reset_transport_header(skb);
3539 skb_reset_mac_len(skb);
3540
3541 pt_prev = NULL;
3542
3543 rcu_read_lock();
3544
3545another_round:
3546 skb->skb_iif = skb->dev->ifindex;
3547
3548 __this_cpu_inc(softnet_data.processed);
3549
3550 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
3551 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
3552 skb = vlan_untag(skb);
3553 if (unlikely(!skb))
3554 goto unlock;
3555 }
3556
3557#ifdef CONFIG_NET_CLS_ACT
3558 if (skb->tc_verd & TC_NCLS) {
3559 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3560 goto ncls;
3561 }
3562#endif
3563
3564 if (pfmemalloc)
3565 goto skip_taps;
3566
3567 list_for_each_entry_rcu(ptype, &ptype_all, list) {
3568 if (!ptype->dev || ptype->dev == skb->dev) {
3569 if (pt_prev)
3570 ret = deliver_skb(skb, pt_prev, orig_dev);
3571 pt_prev = ptype;
3572 }
3573 }
3574
3575skip_taps:
3576#ifdef CONFIG_NET_CLS_ACT
3577 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3578 if (!skb)
3579 goto unlock;
3580ncls:
3581#endif
3582
3583 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
3584 goto drop;
3585
3586 if (vlan_tx_tag_present(skb)) {
3587 if (pt_prev) {
3588 ret = deliver_skb(skb, pt_prev, orig_dev);
3589 pt_prev = NULL;
3590 }
3591 if (vlan_do_receive(&skb))
3592 goto another_round;
3593 else if (unlikely(!skb))
3594 goto unlock;
3595 }
3596
3597 rx_handler = rcu_dereference(skb->dev->rx_handler);
3598 if (rx_handler) {
3599 if (pt_prev) {
3600 ret = deliver_skb(skb, pt_prev, orig_dev);
3601 pt_prev = NULL;
3602 }
3603 switch (rx_handler(&skb)) {
3604 case RX_HANDLER_CONSUMED:
3605 ret = NET_RX_SUCCESS;
3606 goto unlock;
3607 case RX_HANDLER_ANOTHER:
3608 goto another_round;
3609 case RX_HANDLER_EXACT:
3610 deliver_exact = true;
3611 case RX_HANDLER_PASS:
3612 break;
3613 default:
3614 BUG();
3615 }
3616 }
3617
3618 if (unlikely(vlan_tx_tag_present(skb))) {
3619 if (vlan_tx_tag_get_id(skb))
3620 skb->pkt_type = PACKET_OTHERHOST;
3621 /* Note: we might in the future use prio bits
3622 * and set skb->priority like in vlan_do_receive()
3623 * For the time being, just ignore Priority Code Point
3624 */
3625 skb->vlan_tci = 0;
3626 }
3627
3628 /* deliver only exact match when indicated */
3629 null_or_dev = deliver_exact ? skb->dev : NULL;
3630
3631 type = skb->protocol;
3632 list_for_each_entry_rcu(ptype,
3633 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
3634 if (ptype->type == type &&
3635 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3636 ptype->dev == orig_dev)) {
3637 if (pt_prev)
3638 ret = deliver_skb(skb, pt_prev, orig_dev);
3639 pt_prev = ptype;
3640 }
3641 }
3642
3643 if (pt_prev) {
3644 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
3645 goto drop;
3646 else
3647 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
3648 } else {
3649drop:
3650 atomic_long_inc(&skb->dev->rx_dropped);
3651 kfree_skb(skb);
3652 /* Jamal, now you will not able to escape explaining
3653 * me how you were going to use this. :-)
3654 */
3655 ret = NET_RX_DROP;
3656 }
3657
3658unlock:
3659 rcu_read_unlock();
3660 return ret;
3661}
3662
3663static int __netif_receive_skb(struct sk_buff *skb)
3664{
3665 int ret;
3666
3667 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3668 unsigned long pflags = current->flags;
3669
3670 /*
3671 * PFMEMALLOC skbs are special, they should
3672 * - be delivered to SOCK_MEMALLOC sockets only
3673 * - stay away from userspace
3674 * - have bounded memory usage
3675 *
3676 * Use PF_MEMALLOC as this saves us from propagating the allocation
3677 * context down to all allocation sites.
3678 */
3679 current->flags |= PF_MEMALLOC;
3680 ret = __netif_receive_skb_core(skb, true);
3681 tsk_restore_flags(current, pflags, PF_MEMALLOC);
3682 } else
3683 ret = __netif_receive_skb_core(skb, false);
3684
3685 return ret;
3686}
3687
3688static int netif_receive_skb_internal(struct sk_buff *skb)
3689{
3690 net_timestamp_check(netdev_tstamp_prequeue, skb);
3691
3692 if (skb_defer_rx_timestamp(skb))
3693 return NET_RX_SUCCESS;
3694
3695#ifdef CONFIG_RPS
3696 if (static_key_false(&rps_needed)) {
3697 struct rps_dev_flow voidflow, *rflow = &voidflow;
3698 int cpu, ret;
3699
3700 rcu_read_lock();
3701
3702 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3703
3704 if (cpu >= 0) {
3705 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3706 rcu_read_unlock();
3707 return ret;
3708 }
3709 rcu_read_unlock();
3710 }
3711#endif
3712 return __netif_receive_skb(skb);
3713}
3714
3715/**
3716 * netif_receive_skb - process receive buffer from network
3717 * @skb: buffer to process
3718 *
3719 * netif_receive_skb() is the main receive data processing function.
3720 * It always succeeds. The buffer may be dropped during processing
3721 * for congestion control or by the protocol layers.
3722 *
3723 * This function may only be called from softirq context and interrupts
3724 * should be enabled.
3725 *
3726 * Return values (usually ignored):
3727 * NET_RX_SUCCESS: no congestion
3728 * NET_RX_DROP: packet was dropped
3729 */
3730int netif_receive_skb(struct sk_buff *skb)
3731{
3732 trace_netif_receive_skb_entry(skb);
3733
3734 return netif_receive_skb_internal(skb);
3735}
3736EXPORT_SYMBOL(netif_receive_skb);
3737
3738/* Network device is going away, flush any packets still pending
3739 * Called with irqs disabled.
3740 */
3741static void flush_backlog(void *arg)
3742{
3743 struct net_device *dev = arg;
3744 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3745 struct sk_buff *skb, *tmp;
3746
3747 rps_lock(sd);
3748 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3749 if (skb->dev == dev) {
3750 __skb_unlink(skb, &sd->input_pkt_queue);
3751 kfree_skb(skb);
3752 input_queue_head_incr(sd);
3753 }
3754 }
3755 rps_unlock(sd);
3756
3757 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3758 if (skb->dev == dev) {
3759 __skb_unlink(skb, &sd->process_queue);
3760 kfree_skb(skb);
3761 input_queue_head_incr(sd);
3762 }
3763 }
3764}
3765
3766static int napi_gro_complete(struct sk_buff *skb)
3767{
3768 struct packet_offload *ptype;
3769 __be16 type = skb->protocol;
3770 struct list_head *head = &offload_base;
3771 int err = -ENOENT;
3772
3773 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
3774
3775 if (NAPI_GRO_CB(skb)->count == 1) {
3776 skb_shinfo(skb)->gso_size = 0;
3777 goto out;
3778 }
3779
3780 rcu_read_lock();
3781 list_for_each_entry_rcu(ptype, head, list) {
3782 if (ptype->type != type || !ptype->callbacks.gro_complete)
3783 continue;
3784
3785 err = ptype->callbacks.gro_complete(skb, 0);
3786 break;
3787 }
3788 rcu_read_unlock();
3789
3790 if (err) {
3791 WARN_ON(&ptype->list == head);
3792 kfree_skb(skb);
3793 return NET_RX_SUCCESS;
3794 }
3795
3796out:
3797 return netif_receive_skb_internal(skb);
3798}
3799
3800/* napi->gro_list contains packets ordered by age.
3801 * youngest packets at the head of it.
3802 * Complete skbs in reverse order to reduce latencies.
3803 */
3804void napi_gro_flush(struct napi_struct *napi, bool flush_old)
3805{
3806 struct sk_buff *skb, *prev = NULL;
3807
3808 /* scan list and build reverse chain */
3809 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
3810 skb->prev = prev;
3811 prev = skb;
3812 }
3813
3814 for (skb = prev; skb; skb = prev) {
3815 skb->next = NULL;
3816
3817 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
3818 return;
3819
3820 prev = skb->prev;
3821 napi_gro_complete(skb);
3822 napi->gro_count--;
3823 }
3824
3825 napi->gro_list = NULL;
3826}
3827EXPORT_SYMBOL(napi_gro_flush);
3828
3829static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
3830{
3831 struct sk_buff *p;
3832 unsigned int maclen = skb->dev->hard_header_len;
3833 u32 hash = skb_get_hash_raw(skb);
3834
3835 for (p = napi->gro_list; p; p = p->next) {
3836 unsigned long diffs;
3837
3838 NAPI_GRO_CB(p)->flush = 0;
3839
3840 if (hash != skb_get_hash_raw(p)) {
3841 NAPI_GRO_CB(p)->same_flow = 0;
3842 continue;
3843 }
3844
3845 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3846 diffs |= p->vlan_tci ^ skb->vlan_tci;
3847 if (maclen == ETH_HLEN)
3848 diffs |= compare_ether_header(skb_mac_header(p),
3849 skb_mac_header(skb));
3850 else if (!diffs)
3851 diffs = memcmp(skb_mac_header(p),
3852 skb_mac_header(skb),
3853 maclen);
3854 NAPI_GRO_CB(p)->same_flow = !diffs;
3855 }
3856}
3857
3858static void skb_gro_reset_offset(struct sk_buff *skb)
3859{
3860 const struct skb_shared_info *pinfo = skb_shinfo(skb);
3861 const skb_frag_t *frag0 = &pinfo->frags[0];
3862
3863 NAPI_GRO_CB(skb)->data_offset = 0;
3864 NAPI_GRO_CB(skb)->frag0 = NULL;
3865 NAPI_GRO_CB(skb)->frag0_len = 0;
3866
3867 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
3868 pinfo->nr_frags &&
3869 !PageHighMem(skb_frag_page(frag0))) {
3870 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
3871 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
3872 }
3873}
3874
3875static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
3876{
3877 struct skb_shared_info *pinfo = skb_shinfo(skb);
3878
3879 BUG_ON(skb->end - skb->tail < grow);
3880
3881 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3882
3883 skb->data_len -= grow;
3884 skb->tail += grow;
3885
3886 pinfo->frags[0].page_offset += grow;
3887 skb_frag_size_sub(&pinfo->frags[0], grow);
3888
3889 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
3890 skb_frag_unref(skb, 0);
3891 memmove(pinfo->frags, pinfo->frags + 1,
3892 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
3893 }
3894}
3895
3896static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3897{
3898 struct sk_buff **pp = NULL;
3899 struct packet_offload *ptype;
3900 __be16 type = skb->protocol;
3901 struct list_head *head = &offload_base;
3902 int same_flow;
3903 enum gro_result ret;
3904 int grow;
3905
3906 if (!(skb->dev->features & NETIF_F_GRO))
3907 goto normal;
3908
3909 if (skb_is_gso(skb) || skb_has_frag_list(skb))
3910 goto normal;
3911
3912 gro_list_prepare(napi, skb);
3913 NAPI_GRO_CB(skb)->csum = skb->csum; /* Needed for CHECKSUM_COMPLETE */
3914
3915 rcu_read_lock();
3916 list_for_each_entry_rcu(ptype, head, list) {
3917 if (ptype->type != type || !ptype->callbacks.gro_receive)
3918 continue;
3919
3920 skb_set_network_header(skb, skb_gro_offset(skb));
3921 skb_reset_mac_len(skb);
3922 NAPI_GRO_CB(skb)->same_flow = 0;
3923 NAPI_GRO_CB(skb)->flush = 0;
3924 NAPI_GRO_CB(skb)->free = 0;
3925 NAPI_GRO_CB(skb)->udp_mark = 0;
3926
3927 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
3928 break;
3929 }
3930 rcu_read_unlock();
3931
3932 if (&ptype->list == head)
3933 goto normal;
3934
3935 same_flow = NAPI_GRO_CB(skb)->same_flow;
3936 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
3937
3938 if (pp) {
3939 struct sk_buff *nskb = *pp;
3940
3941 *pp = nskb->next;
3942 nskb->next = NULL;
3943 napi_gro_complete(nskb);
3944 napi->gro_count--;
3945 }
3946
3947 if (same_flow)
3948 goto ok;
3949
3950 if (NAPI_GRO_CB(skb)->flush)
3951 goto normal;
3952
3953 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
3954 struct sk_buff *nskb = napi->gro_list;
3955
3956 /* locate the end of the list to select the 'oldest' flow */
3957 while (nskb->next) {
3958 pp = &nskb->next;
3959 nskb = *pp;
3960 }
3961 *pp = NULL;
3962 nskb->next = NULL;
3963 napi_gro_complete(nskb);
3964 } else {
3965 napi->gro_count++;
3966 }
3967 NAPI_GRO_CB(skb)->count = 1;
3968 NAPI_GRO_CB(skb)->age = jiffies;
3969 NAPI_GRO_CB(skb)->last = skb;
3970 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
3971 skb->next = napi->gro_list;
3972 napi->gro_list = skb;
3973 ret = GRO_HELD;
3974
3975pull:
3976 grow = skb_gro_offset(skb) - skb_headlen(skb);
3977 if (grow > 0)
3978 gro_pull_from_frag0(skb, grow);
3979ok:
3980 return ret;
3981
3982normal:
3983 ret = GRO_NORMAL;
3984 goto pull;
3985}
3986
3987struct packet_offload *gro_find_receive_by_type(__be16 type)
3988{
3989 struct list_head *offload_head = &offload_base;
3990 struct packet_offload *ptype;
3991
3992 list_for_each_entry_rcu(ptype, offload_head, list) {
3993 if (ptype->type != type || !ptype->callbacks.gro_receive)
3994 continue;
3995 return ptype;
3996 }
3997 return NULL;
3998}
3999EXPORT_SYMBOL(gro_find_receive_by_type);
4000
4001struct packet_offload *gro_find_complete_by_type(__be16 type)
4002{
4003 struct list_head *offload_head = &offload_base;
4004 struct packet_offload *ptype;
4005
4006 list_for_each_entry_rcu(ptype, offload_head, list) {
4007 if (ptype->type != type || !ptype->callbacks.gro_complete)
4008 continue;
4009 return ptype;
4010 }
4011 return NULL;
4012}
4013EXPORT_SYMBOL(gro_find_complete_by_type);
4014
4015static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4016{
4017 switch (ret) {
4018 case GRO_NORMAL:
4019 if (netif_receive_skb_internal(skb))
4020 ret = GRO_DROP;
4021 break;
4022
4023 case GRO_DROP:
4024 kfree_skb(skb);
4025 break;
4026
4027 case GRO_MERGED_FREE:
4028 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4029 kmem_cache_free(skbuff_head_cache, skb);
4030 else
4031 __kfree_skb(skb);
4032 break;
4033
4034 case GRO_HELD:
4035 case GRO_MERGED:
4036 break;
4037 }
4038
4039 return ret;
4040}
4041
4042gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4043{
4044 trace_napi_gro_receive_entry(skb);
4045
4046 skb_gro_reset_offset(skb);
4047
4048 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4049}
4050EXPORT_SYMBOL(napi_gro_receive);
4051
4052static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4053{
4054 __skb_pull(skb, skb_headlen(skb));
4055 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4056 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4057 skb->vlan_tci = 0;
4058 skb->dev = napi->dev;
4059 skb->skb_iif = 0;
4060 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4061
4062 napi->skb = skb;
4063}
4064
4065struct sk_buff *napi_get_frags(struct napi_struct *napi)
4066{
4067 struct sk_buff *skb = napi->skb;
4068
4069 if (!skb) {
4070 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
4071 napi->skb = skb;
4072 }
4073 return skb;
4074}
4075EXPORT_SYMBOL(napi_get_frags);
4076
4077static gro_result_t napi_frags_finish(struct napi_struct *napi,
4078 struct sk_buff *skb,
4079 gro_result_t ret)
4080{
4081 switch (ret) {
4082 case GRO_NORMAL:
4083 case GRO_HELD:
4084 __skb_push(skb, ETH_HLEN);
4085 skb->protocol = eth_type_trans(skb, skb->dev);
4086 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4087 ret = GRO_DROP;
4088 break;
4089
4090 case GRO_DROP:
4091 case GRO_MERGED_FREE:
4092 napi_reuse_skb(napi, skb);
4093 break;
4094
4095 case GRO_MERGED:
4096 break;
4097 }
4098
4099 return ret;
4100}
4101
4102/* Upper GRO stack assumes network header starts at gro_offset=0
4103 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4104 * We copy ethernet header into skb->data to have a common layout.
4105 */
4106static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4107{
4108 struct sk_buff *skb = napi->skb;
4109 const struct ethhdr *eth;
4110 unsigned int hlen = sizeof(*eth);
4111
4112 napi->skb = NULL;
4113
4114 skb_reset_mac_header(skb);
4115 skb_gro_reset_offset(skb);
4116
4117 eth = skb_gro_header_fast(skb, 0);
4118 if (unlikely(skb_gro_header_hard(skb, hlen))) {
4119 eth = skb_gro_header_slow(skb, hlen, 0);
4120 if (unlikely(!eth)) {
4121 napi_reuse_skb(napi, skb);
4122 return NULL;
4123 }
4124 } else {
4125 gro_pull_from_frag0(skb, hlen);
4126 NAPI_GRO_CB(skb)->frag0 += hlen;
4127 NAPI_GRO_CB(skb)->frag0_len -= hlen;
4128 }
4129 __skb_pull(skb, hlen);
4130
4131 /*
4132 * This works because the only protocols we care about don't require
4133 * special handling.
4134 * We'll fix it up properly in napi_frags_finish()
4135 */
4136 skb->protocol = eth->h_proto;
4137
4138 return skb;
4139}
4140
4141gro_result_t napi_gro_frags(struct napi_struct *napi)
4142{
4143 struct sk_buff *skb = napi_frags_skb(napi);
4144
4145 if (!skb)
4146 return GRO_DROP;
4147
4148 trace_napi_gro_frags_entry(skb);
4149
4150 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
4151}
4152EXPORT_SYMBOL(napi_gro_frags);
4153
4154/*
4155 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4156 * Note: called with local irq disabled, but exits with local irq enabled.
4157 */
4158static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4159{
4160#ifdef CONFIG_RPS
4161 struct softnet_data *remsd = sd->rps_ipi_list;
4162
4163 if (remsd) {
4164 sd->rps_ipi_list = NULL;
4165
4166 local_irq_enable();
4167
4168 /* Send pending IPI's to kick RPS processing on remote cpus. */
4169 while (remsd) {
4170 struct softnet_data *next = remsd->rps_ipi_next;
4171
4172 if (cpu_online(remsd->cpu))
4173 smp_call_function_single_async(remsd->cpu,
4174 &remsd->csd);
4175 remsd = next;
4176 }
4177 } else
4178#endif
4179 local_irq_enable();
4180}
4181
4182static int process_backlog(struct napi_struct *napi, int quota)
4183{
4184 int work = 0;
4185 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4186
4187#ifdef CONFIG_RPS
4188 /* Check if we have pending ipi, its better to send them now,
4189 * not waiting net_rx_action() end.
4190 */
4191 if (sd->rps_ipi_list) {
4192 local_irq_disable();
4193 net_rps_action_and_irq_enable(sd);
4194 }
4195#endif
4196 napi->weight = weight_p;
4197 local_irq_disable();
4198 while (work < quota) {
4199 struct sk_buff *skb;
4200 unsigned int qlen;
4201
4202 while ((skb = __skb_dequeue(&sd->process_queue))) {
4203 local_irq_enable();
4204 __netif_receive_skb(skb);
4205 local_irq_disable();
4206 input_queue_head_incr(sd);
4207 if (++work >= quota) {
4208 local_irq_enable();
4209 return work;
4210 }
4211 }
4212
4213 rps_lock(sd);
4214 qlen = skb_queue_len(&sd->input_pkt_queue);
4215 if (qlen)
4216 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4217 &sd->process_queue);
4218
4219 if (qlen < quota - work) {
4220 /*
4221 * Inline a custom version of __napi_complete().
4222 * only current cpu owns and manipulates this napi,
4223 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
4224 * we can use a plain write instead of clear_bit(),
4225 * and we dont need an smp_mb() memory barrier.
4226 */
4227 list_del(&napi->poll_list);
4228 napi->state = 0;
4229
4230 quota = work + qlen;
4231 }
4232 rps_unlock(sd);
4233 }
4234 local_irq_enable();
4235
4236 return work;
4237}
4238
4239/**
4240 * __napi_schedule - schedule for receive
4241 * @n: entry to schedule
4242 *
4243 * The entry's receive function will be scheduled to run
4244 */
4245void __napi_schedule(struct napi_struct *n)
4246{
4247 unsigned long flags;
4248
4249 local_irq_save(flags);
4250 ____napi_schedule(&__get_cpu_var(softnet_data), n);
4251 local_irq_restore(flags);
4252}
4253EXPORT_SYMBOL(__napi_schedule);
4254
4255void __napi_complete(struct napi_struct *n)
4256{
4257 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4258 BUG_ON(n->gro_list);
4259
4260 list_del(&n->poll_list);
4261 smp_mb__before_clear_bit();
4262 clear_bit(NAPI_STATE_SCHED, &n->state);
4263}
4264EXPORT_SYMBOL(__napi_complete);
4265
4266void napi_complete(struct napi_struct *n)
4267{
4268 unsigned long flags;
4269
4270 /*
4271 * don't let napi dequeue from the cpu poll list
4272 * just in case its running on a different cpu
4273 */
4274 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4275 return;
4276
4277 napi_gro_flush(n, false);
4278 local_irq_save(flags);
4279 __napi_complete(n);
4280 local_irq_restore(flags);
4281}
4282EXPORT_SYMBOL(napi_complete);
4283
4284/* must be called under rcu_read_lock(), as we dont take a reference */
4285struct napi_struct *napi_by_id(unsigned int napi_id)
4286{
4287 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4288 struct napi_struct *napi;
4289
4290 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4291 if (napi->napi_id == napi_id)
4292 return napi;
4293
4294 return NULL;
4295}
4296EXPORT_SYMBOL_GPL(napi_by_id);
4297
4298void napi_hash_add(struct napi_struct *napi)
4299{
4300 if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) {
4301
4302 spin_lock(&napi_hash_lock);
4303
4304 /* 0 is not a valid id, we also skip an id that is taken
4305 * we expect both events to be extremely rare
4306 */
4307 napi->napi_id = 0;
4308 while (!napi->napi_id) {
4309 napi->napi_id = ++napi_gen_id;
4310 if (napi_by_id(napi->napi_id))
4311 napi->napi_id = 0;
4312 }
4313
4314 hlist_add_head_rcu(&napi->napi_hash_node,
4315 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4316
4317 spin_unlock(&napi_hash_lock);
4318 }
4319}
4320EXPORT_SYMBOL_GPL(napi_hash_add);
4321
4322/* Warning : caller is responsible to make sure rcu grace period
4323 * is respected before freeing memory containing @napi
4324 */
4325void napi_hash_del(struct napi_struct *napi)
4326{
4327 spin_lock(&napi_hash_lock);
4328
4329 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
4330 hlist_del_rcu(&napi->napi_hash_node);
4331
4332 spin_unlock(&napi_hash_lock);
4333}
4334EXPORT_SYMBOL_GPL(napi_hash_del);
4335
4336void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4337 int (*poll)(struct napi_struct *, int), int weight)
4338{
4339 INIT_LIST_HEAD(&napi->poll_list);
4340 napi->gro_count = 0;
4341 napi->gro_list = NULL;
4342 napi->skb = NULL;
4343 napi->poll = poll;
4344 if (weight > NAPI_POLL_WEIGHT)
4345 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4346 weight, dev->name);
4347 napi->weight = weight;
4348 list_add(&napi->dev_list, &dev->napi_list);
4349 napi->dev = dev;
4350#ifdef CONFIG_NETPOLL
4351 spin_lock_init(&napi->poll_lock);
4352 napi->poll_owner = -1;
4353#endif
4354 set_bit(NAPI_STATE_SCHED, &napi->state);
4355}
4356EXPORT_SYMBOL(netif_napi_add);
4357
4358void netif_napi_del(struct napi_struct *napi)
4359{
4360 list_del_init(&napi->dev_list);
4361 napi_free_frags(napi);
4362
4363 kfree_skb_list(napi->gro_list);
4364 napi->gro_list = NULL;
4365 napi->gro_count = 0;
4366}
4367EXPORT_SYMBOL(netif_napi_del);
4368
4369static void net_rx_action(struct softirq_action *h)
4370{
4371 struct softnet_data *sd = &__get_cpu_var(softnet_data);
4372 unsigned long time_limit = jiffies + 2;
4373 int budget = netdev_budget;
4374 void *have;
4375
4376 local_irq_disable();
4377
4378 while (!list_empty(&sd->poll_list)) {
4379 struct napi_struct *n;
4380 int work, weight;
4381
4382 /* If softirq window is exhuasted then punt.
4383 * Allow this to run for 2 jiffies since which will allow
4384 * an average latency of 1.5/HZ.
4385 */
4386 if (unlikely(budget <= 0 || time_after_eq(jiffies, time_limit)))
4387 goto softnet_break;
4388
4389 local_irq_enable();
4390
4391 /* Even though interrupts have been re-enabled, this
4392 * access is safe because interrupts can only add new
4393 * entries to the tail of this list, and only ->poll()
4394 * calls can remove this head entry from the list.
4395 */
4396 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
4397
4398 have = netpoll_poll_lock(n);
4399
4400 weight = n->weight;
4401
4402 /* This NAPI_STATE_SCHED test is for avoiding a race
4403 * with netpoll's poll_napi(). Only the entity which
4404 * obtains the lock and sees NAPI_STATE_SCHED set will
4405 * actually make the ->poll() call. Therefore we avoid
4406 * accidentally calling ->poll() when NAPI is not scheduled.
4407 */
4408 work = 0;
4409 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4410 work = n->poll(n, weight);
4411 trace_napi_poll(n);
4412 }
4413
4414 WARN_ON_ONCE(work > weight);
4415
4416 budget -= work;
4417
4418 local_irq_disable();
4419
4420 /* Drivers must not modify the NAPI state if they
4421 * consume the entire weight. In such cases this code
4422 * still "owns" the NAPI instance and therefore can
4423 * move the instance around on the list at-will.
4424 */
4425 if (unlikely(work == weight)) {
4426 if (unlikely(napi_disable_pending(n))) {
4427 local_irq_enable();
4428 napi_complete(n);
4429 local_irq_disable();
4430 } else {
4431 if (n->gro_list) {
4432 /* flush too old packets
4433 * If HZ < 1000, flush all packets.
4434 */
4435 local_irq_enable();
4436 napi_gro_flush(n, HZ >= 1000);
4437 local_irq_disable();
4438 }
4439 list_move_tail(&n->poll_list, &sd->poll_list);
4440 }
4441 }
4442
4443 netpoll_poll_unlock(have);
4444 }
4445out:
4446 net_rps_action_and_irq_enable(sd);
4447
4448#ifdef CONFIG_NET_DMA
4449 /*
4450 * There may not be any more sk_buffs coming right now, so push
4451 * any pending DMA copies to hardware
4452 */
4453 dma_issue_pending_all();
4454#endif
4455
4456 return;
4457
4458softnet_break:
4459 sd->time_squeeze++;
4460 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4461 goto out;
4462}
4463
4464struct netdev_adjacent {
4465 struct net_device *dev;
4466
4467 /* upper master flag, there can only be one master device per list */
4468 bool master;
4469
4470 /* counter for the number of times this device was added to us */
4471 u16 ref_nr;
4472
4473 /* private field for the users */
4474 void *private;
4475
4476 struct list_head list;
4477 struct rcu_head rcu;
4478};
4479
4480static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev,
4481 struct net_device *adj_dev,
4482 struct list_head *adj_list)
4483{
4484 struct netdev_adjacent *adj;
4485
4486 list_for_each_entry(adj, adj_list, list) {
4487 if (adj->dev == adj_dev)
4488 return adj;
4489 }
4490 return NULL;
4491}
4492
4493/**
4494 * netdev_has_upper_dev - Check if device is linked to an upper device
4495 * @dev: device
4496 * @upper_dev: upper device to check
4497 *
4498 * Find out if a device is linked to specified upper device and return true
4499 * in case it is. Note that this checks only immediate upper device,
4500 * not through a complete stack of devices. The caller must hold the RTNL lock.
4501 */
4502bool netdev_has_upper_dev(struct net_device *dev,
4503 struct net_device *upper_dev)
4504{
4505 ASSERT_RTNL();
4506
4507 return __netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper);
4508}
4509EXPORT_SYMBOL(netdev_has_upper_dev);
4510
4511/**
4512 * netdev_has_any_upper_dev - Check if device is linked to some device
4513 * @dev: device
4514 *
4515 * Find out if a device is linked to an upper device and return true in case
4516 * it is. The caller must hold the RTNL lock.
4517 */
4518static bool netdev_has_any_upper_dev(struct net_device *dev)
4519{
4520 ASSERT_RTNL();
4521
4522 return !list_empty(&dev->all_adj_list.upper);
4523}
4524
4525/**
4526 * netdev_master_upper_dev_get - Get master upper device
4527 * @dev: device
4528 *
4529 * Find a master upper device and return pointer to it or NULL in case
4530 * it's not there. The caller must hold the RTNL lock.
4531 */
4532struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4533{
4534 struct netdev_adjacent *upper;
4535
4536 ASSERT_RTNL();
4537
4538 if (list_empty(&dev->adj_list.upper))
4539 return NULL;
4540
4541 upper = list_first_entry(&dev->adj_list.upper,
4542 struct netdev_adjacent, list);
4543 if (likely(upper->master))
4544 return upper->dev;
4545 return NULL;
4546}
4547EXPORT_SYMBOL(netdev_master_upper_dev_get);
4548
4549void *netdev_adjacent_get_private(struct list_head *adj_list)
4550{
4551 struct netdev_adjacent *adj;
4552
4553 adj = list_entry(adj_list, struct netdev_adjacent, list);
4554
4555 return adj->private;
4556}
4557EXPORT_SYMBOL(netdev_adjacent_get_private);
4558
4559/**
4560 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
4561 * @dev: device
4562 * @iter: list_head ** of the current position
4563 *
4564 * Gets the next device from the dev's upper list, starting from iter
4565 * position. The caller must hold RCU read lock.
4566 */
4567struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4568 struct list_head **iter)
4569{
4570 struct netdev_adjacent *upper;
4571
4572 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4573
4574 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4575
4576 if (&upper->list == &dev->adj_list.upper)
4577 return NULL;
4578
4579 *iter = &upper->list;
4580
4581 return upper->dev;
4582}
4583EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
4584
4585/**
4586 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
4587 * @dev: device
4588 * @iter: list_head ** of the current position
4589 *
4590 * Gets the next device from the dev's upper list, starting from iter
4591 * position. The caller must hold RCU read lock.
4592 */
4593struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
4594 struct list_head **iter)
4595{
4596 struct netdev_adjacent *upper;
4597
4598 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4599
4600 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4601
4602 if (&upper->list == &dev->all_adj_list.upper)
4603 return NULL;
4604
4605 *iter = &upper->list;
4606
4607 return upper->dev;
4608}
4609EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
4610
4611/**
4612 * netdev_lower_get_next_private - Get the next ->private from the
4613 * lower neighbour list
4614 * @dev: device
4615 * @iter: list_head ** of the current position
4616 *
4617 * Gets the next netdev_adjacent->private from the dev's lower neighbour
4618 * list, starting from iter position. The caller must hold either hold the
4619 * RTNL lock or its own locking that guarantees that the neighbour lower
4620 * list will remain unchainged.
4621 */
4622void *netdev_lower_get_next_private(struct net_device *dev,
4623 struct list_head **iter)
4624{
4625 struct netdev_adjacent *lower;
4626
4627 lower = list_entry(*iter, struct netdev_adjacent, list);
4628
4629 if (&lower->list == &dev->adj_list.lower)
4630 return NULL;
4631
4632 *iter = lower->list.next;
4633
4634 return lower->private;
4635}
4636EXPORT_SYMBOL(netdev_lower_get_next_private);
4637
4638/**
4639 * netdev_lower_get_next_private_rcu - Get the next ->private from the
4640 * lower neighbour list, RCU
4641 * variant
4642 * @dev: device
4643 * @iter: list_head ** of the current position
4644 *
4645 * Gets the next netdev_adjacent->private from the dev's lower neighbour
4646 * list, starting from iter position. The caller must hold RCU read lock.
4647 */
4648void *netdev_lower_get_next_private_rcu(struct net_device *dev,
4649 struct list_head **iter)
4650{
4651 struct netdev_adjacent *lower;
4652
4653 WARN_ON_ONCE(!rcu_read_lock_held());
4654
4655 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4656
4657 if (&lower->list == &dev->adj_list.lower)
4658 return NULL;
4659
4660 *iter = &lower->list;
4661
4662 return lower->private;
4663}
4664EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
4665
4666/**
4667 * netdev_lower_get_next - Get the next device from the lower neighbour
4668 * list
4669 * @dev: device
4670 * @iter: list_head ** of the current position
4671 *
4672 * Gets the next netdev_adjacent from the dev's lower neighbour
4673 * list, starting from iter position. The caller must hold RTNL lock or
4674 * its own locking that guarantees that the neighbour lower
4675 * list will remain unchainged.
4676 */
4677void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
4678{
4679 struct netdev_adjacent *lower;
4680
4681 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
4682
4683 if (&lower->list == &dev->adj_list.lower)
4684 return NULL;
4685
4686 *iter = &lower->list;
4687
4688 return lower->dev;
4689}
4690EXPORT_SYMBOL(netdev_lower_get_next);
4691
4692/**
4693 * netdev_lower_get_first_private_rcu - Get the first ->private from the
4694 * lower neighbour list, RCU
4695 * variant
4696 * @dev: device
4697 *
4698 * Gets the first netdev_adjacent->private from the dev's lower neighbour
4699 * list. The caller must hold RCU read lock.
4700 */
4701void *netdev_lower_get_first_private_rcu(struct net_device *dev)
4702{
4703 struct netdev_adjacent *lower;
4704
4705 lower = list_first_or_null_rcu(&dev->adj_list.lower,
4706 struct netdev_adjacent, list);
4707 if (lower)
4708 return lower->private;
4709 return NULL;
4710}
4711EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
4712
4713/**
4714 * netdev_master_upper_dev_get_rcu - Get master upper device
4715 * @dev: device
4716 *
4717 * Find a master upper device and return pointer to it or NULL in case
4718 * it's not there. The caller must hold the RCU read lock.
4719 */
4720struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
4721{
4722 struct netdev_adjacent *upper;
4723
4724 upper = list_first_or_null_rcu(&dev->adj_list.upper,
4725 struct netdev_adjacent, list);
4726 if (upper && likely(upper->master))
4727 return upper->dev;
4728 return NULL;
4729}
4730EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
4731
4732static int netdev_adjacent_sysfs_add(struct net_device *dev,
4733 struct net_device *adj_dev,
4734 struct list_head *dev_list)
4735{
4736 char linkname[IFNAMSIZ+7];
4737 sprintf(linkname, dev_list == &dev->adj_list.upper ?
4738 "upper_%s" : "lower_%s", adj_dev->name);
4739 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
4740 linkname);
4741}
4742static void netdev_adjacent_sysfs_del(struct net_device *dev,
4743 char *name,
4744 struct list_head *dev_list)
4745{
4746 char linkname[IFNAMSIZ+7];
4747 sprintf(linkname, dev_list == &dev->adj_list.upper ?
4748 "upper_%s" : "lower_%s", name);
4749 sysfs_remove_link(&(dev->dev.kobj), linkname);
4750}
4751
4752#define netdev_adjacent_is_neigh_list(dev, dev_list) \
4753 (dev_list == &dev->adj_list.upper || \
4754 dev_list == &dev->adj_list.lower)
4755
4756static int __netdev_adjacent_dev_insert(struct net_device *dev,
4757 struct net_device *adj_dev,
4758 struct list_head *dev_list,
4759 void *private, bool master)
4760{
4761 struct netdev_adjacent *adj;
4762 int ret;
4763
4764 adj = __netdev_find_adj(dev, adj_dev, dev_list);
4765
4766 if (adj) {
4767 adj->ref_nr++;
4768 return 0;
4769 }
4770
4771 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
4772 if (!adj)
4773 return -ENOMEM;
4774
4775 adj->dev = adj_dev;
4776 adj->master = master;
4777 adj->ref_nr = 1;
4778 adj->private = private;
4779 dev_hold(adj_dev);
4780
4781 pr_debug("dev_hold for %s, because of link added from %s to %s\n",
4782 adj_dev->name, dev->name, adj_dev->name);
4783
4784 if (netdev_adjacent_is_neigh_list(dev, dev_list)) {
4785 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
4786 if (ret)
4787 goto free_adj;
4788 }
4789
4790 /* Ensure that master link is always the first item in list. */
4791 if (master) {
4792 ret = sysfs_create_link(&(dev->dev.kobj),
4793 &(adj_dev->dev.kobj), "master");
4794 if (ret)
4795 goto remove_symlinks;
4796
4797 list_add_rcu(&adj->list, dev_list);
4798 } else {
4799 list_add_tail_rcu(&adj->list, dev_list);
4800 }
4801
4802 return 0;
4803
4804remove_symlinks:
4805 if (netdev_adjacent_is_neigh_list(dev, dev_list))
4806 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
4807free_adj:
4808 kfree(adj);
4809 dev_put(adj_dev);
4810
4811 return ret;
4812}
4813
4814static void __netdev_adjacent_dev_remove(struct net_device *dev,
4815 struct net_device *adj_dev,
4816 struct list_head *dev_list)
4817{
4818 struct netdev_adjacent *adj;
4819
4820 adj = __netdev_find_adj(dev, adj_dev, dev_list);
4821
4822 if (!adj) {
4823 pr_err("tried to remove device %s from %s\n",
4824 dev->name, adj_dev->name);
4825 BUG();
4826 }
4827
4828 if (adj->ref_nr > 1) {
4829 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
4830 adj->ref_nr-1);
4831 adj->ref_nr--;
4832 return;
4833 }
4834
4835 if (adj->master)
4836 sysfs_remove_link(&(dev->dev.kobj), "master");
4837
4838 if (netdev_adjacent_is_neigh_list(dev, dev_list))
4839 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
4840
4841 list_del_rcu(&adj->list);
4842 pr_debug("dev_put for %s, because link removed from %s to %s\n",
4843 adj_dev->name, dev->name, adj_dev->name);
4844 dev_put(adj_dev);
4845 kfree_rcu(adj, rcu);
4846}
4847
4848static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
4849 struct net_device *upper_dev,
4850 struct list_head *up_list,
4851 struct list_head *down_list,
4852 void *private, bool master)
4853{
4854 int ret;
4855
4856 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
4857 master);
4858 if (ret)
4859 return ret;
4860
4861 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
4862 false);
4863 if (ret) {
4864 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
4865 return ret;
4866 }
4867
4868 return 0;
4869}
4870
4871static int __netdev_adjacent_dev_link(struct net_device *dev,
4872 struct net_device *upper_dev)
4873{
4874 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
4875 &dev->all_adj_list.upper,
4876 &upper_dev->all_adj_list.lower,
4877 NULL, false);
4878}
4879
4880static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
4881 struct net_device *upper_dev,
4882 struct list_head *up_list,
4883 struct list_head *down_list)
4884{
4885 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
4886 __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
4887}
4888
4889static void __netdev_adjacent_dev_unlink(struct net_device *dev,
4890 struct net_device *upper_dev)
4891{
4892 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
4893 &dev->all_adj_list.upper,
4894 &upper_dev->all_adj_list.lower);
4895}
4896
4897static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
4898 struct net_device *upper_dev,
4899 void *private, bool master)
4900{
4901 int ret = __netdev_adjacent_dev_link(dev, upper_dev);
4902
4903 if (ret)
4904 return ret;
4905
4906 ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
4907 &dev->adj_list.upper,
4908 &upper_dev->adj_list.lower,
4909 private, master);
4910 if (ret) {
4911 __netdev_adjacent_dev_unlink(dev, upper_dev);
4912 return ret;
4913 }
4914
4915 return 0;
4916}
4917
4918static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
4919 struct net_device *upper_dev)
4920{
4921 __netdev_adjacent_dev_unlink(dev, upper_dev);
4922 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
4923 &dev->adj_list.upper,
4924 &upper_dev->adj_list.lower);
4925}
4926
4927static int __netdev_upper_dev_link(struct net_device *dev,
4928 struct net_device *upper_dev, bool master,
4929 void *private)
4930{
4931 struct netdev_adjacent *i, *j, *to_i, *to_j;
4932 int ret = 0;
4933
4934 ASSERT_RTNL();
4935
4936 if (dev == upper_dev)
4937 return -EBUSY;
4938
4939 /* To prevent loops, check if dev is not upper device to upper_dev. */
4940 if (__netdev_find_adj(upper_dev, dev, &upper_dev->all_adj_list.upper))
4941 return -EBUSY;
4942
4943 if (__netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper))
4944 return -EEXIST;
4945
4946 if (master && netdev_master_upper_dev_get(dev))
4947 return -EBUSY;
4948
4949 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private,
4950 master);
4951 if (ret)
4952 return ret;
4953
4954 /* Now that we linked these devs, make all the upper_dev's
4955 * all_adj_list.upper visible to every dev's all_adj_list.lower an
4956 * versa, and don't forget the devices itself. All of these
4957 * links are non-neighbours.
4958 */
4959 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
4960 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
4961 pr_debug("Interlinking %s with %s, non-neighbour\n",
4962 i->dev->name, j->dev->name);
4963 ret = __netdev_adjacent_dev_link(i->dev, j->dev);
4964 if (ret)
4965 goto rollback_mesh;
4966 }
4967 }
4968
4969 /* add dev to every upper_dev's upper device */
4970 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
4971 pr_debug("linking %s's upper device %s with %s\n",
4972 upper_dev->name, i->dev->name, dev->name);
4973 ret = __netdev_adjacent_dev_link(dev, i->dev);
4974 if (ret)
4975 goto rollback_upper_mesh;
4976 }
4977
4978 /* add upper_dev to every dev's lower device */
4979 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
4980 pr_debug("linking %s's lower device %s with %s\n", dev->name,
4981 i->dev->name, upper_dev->name);
4982 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
4983 if (ret)
4984 goto rollback_lower_mesh;
4985 }
4986
4987 call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
4988 return 0;
4989
4990rollback_lower_mesh:
4991 to_i = i;
4992 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
4993 if (i == to_i)
4994 break;
4995 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
4996 }
4997
4998 i = NULL;
4999
5000rollback_upper_mesh:
5001 to_i = i;
5002 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5003 if (i == to_i)
5004 break;
5005 __netdev_adjacent_dev_unlink(dev, i->dev);
5006 }
5007
5008 i = j = NULL;
5009
5010rollback_mesh:
5011 to_i = i;
5012 to_j = j;
5013 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5014 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5015 if (i == to_i && j == to_j)
5016 break;
5017 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5018 }
5019 if (i == to_i)
5020 break;
5021 }
5022
5023 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5024
5025 return ret;
5026}
5027
5028/**
5029 * netdev_upper_dev_link - Add a link to the upper device
5030 * @dev: device
5031 * @upper_dev: new upper device
5032 *
5033 * Adds a link to device which is upper to this one. The caller must hold
5034 * the RTNL lock. On a failure a negative errno code is returned.
5035 * On success the reference counts are adjusted and the function
5036 * returns zero.
5037 */
5038int netdev_upper_dev_link(struct net_device *dev,
5039 struct net_device *upper_dev)
5040{
5041 return __netdev_upper_dev_link(dev, upper_dev, false, NULL);
5042}
5043EXPORT_SYMBOL(netdev_upper_dev_link);
5044
5045/**
5046 * netdev_master_upper_dev_link - Add a master link to the upper device
5047 * @dev: device
5048 * @upper_dev: new upper device
5049 *
5050 * Adds a link to device which is upper to this one. In this case, only
5051 * one master upper device can be linked, although other non-master devices
5052 * might be linked as well. The caller must hold the RTNL lock.
5053 * On a failure a negative errno code is returned. On success the reference
5054 * counts are adjusted and the function returns zero.
5055 */
5056int netdev_master_upper_dev_link(struct net_device *dev,
5057 struct net_device *upper_dev)
5058{
5059 return __netdev_upper_dev_link(dev, upper_dev, true, NULL);
5060}
5061EXPORT_SYMBOL(netdev_master_upper_dev_link);
5062
5063int netdev_master_upper_dev_link_private(struct net_device *dev,
5064 struct net_device *upper_dev,
5065 void *private)
5066{
5067 return __netdev_upper_dev_link(dev, upper_dev, true, private);
5068}
5069EXPORT_SYMBOL(netdev_master_upper_dev_link_private);
5070
5071/**
5072 * netdev_upper_dev_unlink - Removes a link to upper device
5073 * @dev: device
5074 * @upper_dev: new upper device
5075 *
5076 * Removes a link to device which is upper to this one. The caller must hold
5077 * the RTNL lock.
5078 */
5079void netdev_upper_dev_unlink(struct net_device *dev,
5080 struct net_device *upper_dev)
5081{
5082 struct netdev_adjacent *i, *j;
5083 ASSERT_RTNL();
5084
5085 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5086
5087 /* Here is the tricky part. We must remove all dev's lower
5088 * devices from all upper_dev's upper devices and vice
5089 * versa, to maintain the graph relationship.
5090 */
5091 list_for_each_entry(i, &dev->all_adj_list.lower, list)
5092 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
5093 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5094
5095 /* remove also the devices itself from lower/upper device
5096 * list
5097 */
5098 list_for_each_entry(i, &dev->all_adj_list.lower, list)
5099 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5100
5101 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
5102 __netdev_adjacent_dev_unlink(dev, i->dev);
5103
5104 call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev);
5105}
5106EXPORT_SYMBOL(netdev_upper_dev_unlink);
5107
5108void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
5109{
5110 struct netdev_adjacent *iter;
5111
5112 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5113 netdev_adjacent_sysfs_del(iter->dev, oldname,
5114 &iter->dev->adj_list.lower);
5115 netdev_adjacent_sysfs_add(iter->dev, dev,
5116 &iter->dev->adj_list.lower);
5117 }
5118
5119 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5120 netdev_adjacent_sysfs_del(iter->dev, oldname,
5121 &iter->dev->adj_list.upper);
5122 netdev_adjacent_sysfs_add(iter->dev, dev,
5123 &iter->dev->adj_list.upper);
5124 }
5125}
5126
5127void *netdev_lower_dev_get_private(struct net_device *dev,
5128 struct net_device *lower_dev)
5129{
5130 struct netdev_adjacent *lower;
5131
5132 if (!lower_dev)
5133 return NULL;
5134 lower = __netdev_find_adj(dev, lower_dev, &dev->adj_list.lower);
5135 if (!lower)
5136 return NULL;
5137
5138 return lower->private;
5139}
5140EXPORT_SYMBOL(netdev_lower_dev_get_private);
5141
5142
5143int dev_get_nest_level(struct net_device *dev,
5144 bool (*type_check)(struct net_device *dev))
5145{
5146 struct net_device *lower = NULL;
5147 struct list_head *iter;
5148 int max_nest = -1;
5149 int nest;
5150
5151 ASSERT_RTNL();
5152
5153 netdev_for_each_lower_dev(dev, lower, iter) {
5154 nest = dev_get_nest_level(lower, type_check);
5155 if (max_nest < nest)
5156 max_nest = nest;
5157 }
5158
5159 if (type_check(dev))
5160 max_nest++;
5161
5162 return max_nest;
5163}
5164EXPORT_SYMBOL(dev_get_nest_level);
5165
5166static void dev_change_rx_flags(struct net_device *dev, int flags)
5167{
5168 const struct net_device_ops *ops = dev->netdev_ops;
5169
5170 if (ops->ndo_change_rx_flags)
5171 ops->ndo_change_rx_flags(dev, flags);
5172}
5173
5174static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
5175{
5176 unsigned int old_flags = dev->flags;
5177 kuid_t uid;
5178 kgid_t gid;
5179
5180 ASSERT_RTNL();
5181
5182 dev->flags |= IFF_PROMISC;
5183 dev->promiscuity += inc;
5184 if (dev->promiscuity == 0) {
5185 /*
5186 * Avoid overflow.
5187 * If inc causes overflow, untouch promisc and return error.
5188 */
5189 if (inc < 0)
5190 dev->flags &= ~IFF_PROMISC;
5191 else {
5192 dev->promiscuity -= inc;
5193 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
5194 dev->name);
5195 return -EOVERFLOW;
5196 }
5197 }
5198 if (dev->flags != old_flags) {
5199 pr_info("device %s %s promiscuous mode\n",
5200 dev->name,
5201 dev->flags & IFF_PROMISC ? "entered" : "left");
5202 if (audit_enabled) {
5203 current_uid_gid(&uid, &gid);
5204 audit_log(current->audit_context, GFP_ATOMIC,
5205 AUDIT_ANOM_PROMISCUOUS,
5206 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
5207 dev->name, (dev->flags & IFF_PROMISC),
5208 (old_flags & IFF_PROMISC),
5209 from_kuid(&init_user_ns, audit_get_loginuid(current)),
5210 from_kuid(&init_user_ns, uid),
5211 from_kgid(&init_user_ns, gid),
5212 audit_get_sessionid(current));
5213 }
5214
5215 dev_change_rx_flags(dev, IFF_PROMISC);
5216 }
5217 if (notify)
5218 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
5219 return 0;
5220}
5221
5222/**
5223 * dev_set_promiscuity - update promiscuity count on a device
5224 * @dev: device
5225 * @inc: modifier
5226 *
5227 * Add or remove promiscuity from a device. While the count in the device
5228 * remains above zero the interface remains promiscuous. Once it hits zero
5229 * the device reverts back to normal filtering operation. A negative inc
5230 * value is used to drop promiscuity on the device.
5231 * Return 0 if successful or a negative errno code on error.
5232 */
5233int dev_set_promiscuity(struct net_device *dev, int inc)
5234{
5235 unsigned int old_flags = dev->flags;
5236 int err;
5237
5238 err = __dev_set_promiscuity(dev, inc, true);
5239 if (err < 0)
5240 return err;
5241 if (dev->flags != old_flags)
5242 dev_set_rx_mode(dev);
5243 return err;
5244}
5245EXPORT_SYMBOL(dev_set_promiscuity);
5246
5247static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
5248{
5249 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
5250
5251 ASSERT_RTNL();
5252
5253 dev->flags |= IFF_ALLMULTI;
5254 dev->allmulti += inc;
5255 if (dev->allmulti == 0) {
5256 /*
5257 * Avoid overflow.
5258 * If inc causes overflow, untouch allmulti and return error.
5259 */
5260 if (inc < 0)
5261 dev->flags &= ~IFF_ALLMULTI;
5262 else {
5263 dev->allmulti -= inc;
5264 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
5265 dev->name);
5266 return -EOVERFLOW;
5267 }
5268 }
5269 if (dev->flags ^ old_flags) {
5270 dev_change_rx_flags(dev, IFF_ALLMULTI);
5271 dev_set_rx_mode(dev);
5272 if (notify)
5273 __dev_notify_flags(dev, old_flags,
5274 dev->gflags ^ old_gflags);
5275 }
5276 return 0;
5277}
5278
5279/**
5280 * dev_set_allmulti - update allmulti count on a device
5281 * @dev: device
5282 * @inc: modifier
5283 *
5284 * Add or remove reception of all multicast frames to a device. While the
5285 * count in the device remains above zero the interface remains listening
5286 * to all interfaces. Once it hits zero the device reverts back to normal
5287 * filtering operation. A negative @inc value is used to drop the counter
5288 * when releasing a resource needing all multicasts.
5289 * Return 0 if successful or a negative errno code on error.
5290 */
5291
5292int dev_set_allmulti(struct net_device *dev, int inc)
5293{
5294 return __dev_set_allmulti(dev, inc, true);
5295}
5296EXPORT_SYMBOL(dev_set_allmulti);
5297
5298/*
5299 * Upload unicast and multicast address lists to device and
5300 * configure RX filtering. When the device doesn't support unicast
5301 * filtering it is put in promiscuous mode while unicast addresses
5302 * are present.
5303 */
5304void __dev_set_rx_mode(struct net_device *dev)
5305{
5306 const struct net_device_ops *ops = dev->netdev_ops;
5307
5308 /* dev_open will call this function so the list will stay sane. */
5309 if (!(dev->flags&IFF_UP))
5310 return;
5311
5312 if (!netif_device_present(dev))
5313 return;
5314
5315 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
5316 /* Unicast addresses changes may only happen under the rtnl,
5317 * therefore calling __dev_set_promiscuity here is safe.
5318 */
5319 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
5320 __dev_set_promiscuity(dev, 1, false);
5321 dev->uc_promisc = true;
5322 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
5323 __dev_set_promiscuity(dev, -1, false);
5324 dev->uc_promisc = false;
5325 }
5326 }
5327
5328 if (ops->ndo_set_rx_mode)
5329 ops->ndo_set_rx_mode(dev);
5330}
5331
5332void dev_set_rx_mode(struct net_device *dev)
5333{
5334 netif_addr_lock_bh(dev);
5335 __dev_set_rx_mode(dev);
5336 netif_addr_unlock_bh(dev);
5337}
5338
5339/**
5340 * dev_get_flags - get flags reported to userspace
5341 * @dev: device
5342 *
5343 * Get the combination of flag bits exported through APIs to userspace.
5344 */
5345unsigned int dev_get_flags(const struct net_device *dev)
5346{
5347 unsigned int flags;
5348
5349 flags = (dev->flags & ~(IFF_PROMISC |
5350 IFF_ALLMULTI |
5351 IFF_RUNNING |
5352 IFF_LOWER_UP |
5353 IFF_DORMANT)) |
5354 (dev->gflags & (IFF_PROMISC |
5355 IFF_ALLMULTI));
5356
5357 if (netif_running(dev)) {
5358 if (netif_oper_up(dev))
5359 flags |= IFF_RUNNING;
5360 if (netif_carrier_ok(dev))
5361 flags |= IFF_LOWER_UP;
5362 if (netif_dormant(dev))
5363 flags |= IFF_DORMANT;
5364 }
5365
5366 return flags;
5367}
5368EXPORT_SYMBOL(dev_get_flags);
5369
5370int __dev_change_flags(struct net_device *dev, unsigned int flags)
5371{
5372 unsigned int old_flags = dev->flags;
5373 int ret;
5374
5375 ASSERT_RTNL();
5376
5377 /*
5378 * Set the flags on our device.
5379 */
5380
5381 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
5382 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
5383 IFF_AUTOMEDIA)) |
5384 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
5385 IFF_ALLMULTI));
5386
5387 /*
5388 * Load in the correct multicast list now the flags have changed.
5389 */
5390
5391 if ((old_flags ^ flags) & IFF_MULTICAST)
5392 dev_change_rx_flags(dev, IFF_MULTICAST);
5393
5394 dev_set_rx_mode(dev);
5395
5396 /*
5397 * Have we downed the interface. We handle IFF_UP ourselves
5398 * according to user attempts to set it, rather than blindly
5399 * setting it.
5400 */
5401
5402 ret = 0;
5403 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
5404 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
5405
5406 if (!ret)
5407 dev_set_rx_mode(dev);
5408 }
5409
5410 if ((flags ^ dev->gflags) & IFF_PROMISC) {
5411 int inc = (flags & IFF_PROMISC) ? 1 : -1;
5412 unsigned int old_flags = dev->flags;
5413
5414 dev->gflags ^= IFF_PROMISC;
5415
5416 if (__dev_set_promiscuity(dev, inc, false) >= 0)
5417 if (dev->flags != old_flags)
5418 dev_set_rx_mode(dev);
5419 }
5420
5421 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
5422 is important. Some (broken) drivers set IFF_PROMISC, when
5423 IFF_ALLMULTI is requested not asking us and not reporting.
5424 */
5425 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
5426 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
5427
5428 dev->gflags ^= IFF_ALLMULTI;
5429 __dev_set_allmulti(dev, inc, false);
5430 }
5431
5432 return ret;
5433}
5434
5435void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
5436 unsigned int gchanges)
5437{
5438 unsigned int changes = dev->flags ^ old_flags;
5439
5440 if (gchanges)
5441 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
5442
5443 if (changes & IFF_UP) {
5444 if (dev->flags & IFF_UP)
5445 call_netdevice_notifiers(NETDEV_UP, dev);
5446 else
5447 call_netdevice_notifiers(NETDEV_DOWN, dev);
5448 }
5449
5450 if (dev->flags & IFF_UP &&
5451 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
5452 struct netdev_notifier_change_info change_info;
5453
5454 change_info.flags_changed = changes;
5455 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
5456 &change_info.info);
5457 }
5458}
5459
5460/**
5461 * dev_change_flags - change device settings
5462 * @dev: device
5463 * @flags: device state flags
5464 *
5465 * Change settings on device based state flags. The flags are
5466 * in the userspace exported format.
5467 */
5468int dev_change_flags(struct net_device *dev, unsigned int flags)
5469{
5470 int ret;
5471 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
5472
5473 ret = __dev_change_flags(dev, flags);
5474 if (ret < 0)
5475 return ret;
5476
5477 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
5478 __dev_notify_flags(dev, old_flags, changes);
5479 return ret;
5480}
5481EXPORT_SYMBOL(dev_change_flags);
5482
5483static int __dev_set_mtu(struct net_device *dev, int new_mtu)
5484{
5485 const struct net_device_ops *ops = dev->netdev_ops;
5486
5487 if (ops->ndo_change_mtu)
5488 return ops->ndo_change_mtu(dev, new_mtu);
5489
5490 dev->mtu = new_mtu;
5491 return 0;
5492}
5493
5494/**
5495 * dev_set_mtu - Change maximum transfer unit
5496 * @dev: device
5497 * @new_mtu: new transfer unit
5498 *
5499 * Change the maximum transfer size of the network device.
5500 */
5501int dev_set_mtu(struct net_device *dev, int new_mtu)
5502{
5503 int err, orig_mtu;
5504
5505 if (new_mtu == dev->mtu)
5506 return 0;
5507
5508 /* MTU must be positive. */
5509 if (new_mtu < 0)
5510 return -EINVAL;
5511
5512 if (!netif_device_present(dev))
5513 return -ENODEV;
5514
5515 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
5516 err = notifier_to_errno(err);
5517 if (err)
5518 return err;
5519
5520 orig_mtu = dev->mtu;
5521 err = __dev_set_mtu(dev, new_mtu);
5522
5523 if (!err) {
5524 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
5525 err = notifier_to_errno(err);
5526 if (err) {
5527 /* setting mtu back and notifying everyone again,
5528 * so that they have a chance to revert changes.
5529 */
5530 __dev_set_mtu(dev, orig_mtu);
5531 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
5532 }
5533 }
5534 return err;
5535}
5536EXPORT_SYMBOL(dev_set_mtu);
5537
5538/**
5539 * dev_set_group - Change group this device belongs to
5540 * @dev: device
5541 * @new_group: group this device should belong to
5542 */
5543void dev_set_group(struct net_device *dev, int new_group)
5544{
5545 dev->group = new_group;
5546}
5547EXPORT_SYMBOL(dev_set_group);
5548
5549/**
5550 * dev_set_mac_address - Change Media Access Control Address
5551 * @dev: device
5552 * @sa: new address
5553 *
5554 * Change the hardware (MAC) address of the device
5555 */
5556int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
5557{
5558 const struct net_device_ops *ops = dev->netdev_ops;
5559 int err;
5560
5561 if (!ops->ndo_set_mac_address)
5562 return -EOPNOTSUPP;
5563 if (sa->sa_family != dev->type)
5564 return -EINVAL;
5565 if (!netif_device_present(dev))
5566 return -ENODEV;
5567 err = ops->ndo_set_mac_address(dev, sa);
5568 if (err)
5569 return err;
5570 dev->addr_assign_type = NET_ADDR_SET;
5571 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
5572 add_device_randomness(dev->dev_addr, dev->addr_len);
5573 return 0;
5574}
5575EXPORT_SYMBOL(dev_set_mac_address);
5576
5577/**
5578 * dev_change_carrier - Change device carrier
5579 * @dev: device
5580 * @new_carrier: new value
5581 *
5582 * Change device carrier
5583 */
5584int dev_change_carrier(struct net_device *dev, bool new_carrier)
5585{
5586 const struct net_device_ops *ops = dev->netdev_ops;
5587
5588 if (!ops->ndo_change_carrier)
5589 return -EOPNOTSUPP;
5590 if (!netif_device_present(dev))
5591 return -ENODEV;
5592 return ops->ndo_change_carrier(dev, new_carrier);
5593}
5594EXPORT_SYMBOL(dev_change_carrier);
5595
5596/**
5597 * dev_get_phys_port_id - Get device physical port ID
5598 * @dev: device
5599 * @ppid: port ID
5600 *
5601 * Get device physical port ID
5602 */
5603int dev_get_phys_port_id(struct net_device *dev,
5604 struct netdev_phys_port_id *ppid)
5605{
5606 const struct net_device_ops *ops = dev->netdev_ops;
5607
5608 if (!ops->ndo_get_phys_port_id)
5609 return -EOPNOTSUPP;
5610 return ops->ndo_get_phys_port_id(dev, ppid);
5611}
5612EXPORT_SYMBOL(dev_get_phys_port_id);
5613
5614/**
5615 * dev_new_index - allocate an ifindex
5616 * @net: the applicable net namespace
5617 *
5618 * Returns a suitable unique value for a new device interface
5619 * number. The caller must hold the rtnl semaphore or the
5620 * dev_base_lock to be sure it remains unique.
5621 */
5622static int dev_new_index(struct net *net)
5623{
5624 int ifindex = net->ifindex;
5625 for (;;) {
5626 if (++ifindex <= 0)
5627 ifindex = 1;
5628 if (!__dev_get_by_index(net, ifindex))
5629 return net->ifindex = ifindex;
5630 }
5631}
5632
5633/* Delayed registration/unregisteration */
5634static LIST_HEAD(net_todo_list);
5635DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
5636
5637static void net_set_todo(struct net_device *dev)
5638{
5639 list_add_tail(&dev->todo_list, &net_todo_list);
5640 dev_net(dev)->dev_unreg_count++;
5641}
5642
5643static void rollback_registered_many(struct list_head *head)
5644{
5645 struct net_device *dev, *tmp;
5646 LIST_HEAD(close_head);
5647
5648 BUG_ON(dev_boot_phase);
5649 ASSERT_RTNL();
5650
5651 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
5652 /* Some devices call without registering
5653 * for initialization unwind. Remove those
5654 * devices and proceed with the remaining.
5655 */
5656 if (dev->reg_state == NETREG_UNINITIALIZED) {
5657 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
5658 dev->name, dev);
5659
5660 WARN_ON(1);
5661 list_del(&dev->unreg_list);
5662 continue;
5663 }
5664 dev->dismantle = true;
5665 BUG_ON(dev->reg_state != NETREG_REGISTERED);
5666 }
5667
5668 /* If device is running, close it first. */
5669 list_for_each_entry(dev, head, unreg_list)
5670 list_add_tail(&dev->close_list, &close_head);
5671 dev_close_many(&close_head);
5672
5673 list_for_each_entry(dev, head, unreg_list) {
5674 /* And unlink it from device chain. */
5675 unlist_netdevice(dev);
5676
5677 dev->reg_state = NETREG_UNREGISTERING;
5678 }
5679
5680 synchronize_net();
5681
5682 list_for_each_entry(dev, head, unreg_list) {
5683 /* Shutdown queueing discipline. */
5684 dev_shutdown(dev);
5685
5686
5687 /* Notify protocols, that we are about to destroy
5688 this device. They should clean all the things.
5689 */
5690 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5691
5692 if (!dev->rtnl_link_ops ||
5693 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5694 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
5695
5696 /*
5697 * Flush the unicast and multicast chains
5698 */
5699 dev_uc_flush(dev);
5700 dev_mc_flush(dev);
5701
5702 if (dev->netdev_ops->ndo_uninit)
5703 dev->netdev_ops->ndo_uninit(dev);
5704
5705 /* Notifier chain MUST detach us all upper devices. */
5706 WARN_ON(netdev_has_any_upper_dev(dev));
5707
5708 /* Remove entries from kobject tree */
5709 netdev_unregister_kobject(dev);
5710#ifdef CONFIG_XPS
5711 /* Remove XPS queueing entries */
5712 netif_reset_xps_queues_gt(dev, 0);
5713#endif
5714 }
5715
5716 synchronize_net();
5717
5718 list_for_each_entry(dev, head, unreg_list)
5719 dev_put(dev);
5720}
5721
5722static void rollback_registered(struct net_device *dev)
5723{
5724 LIST_HEAD(single);
5725
5726 list_add(&dev->unreg_list, &single);
5727 rollback_registered_many(&single);
5728 list_del(&single);
5729}
5730
5731static netdev_features_t netdev_fix_features(struct net_device *dev,
5732 netdev_features_t features)
5733{
5734 /* Fix illegal checksum combinations */
5735 if ((features & NETIF_F_HW_CSUM) &&
5736 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5737 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
5738 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5739 }
5740
5741 /* TSO requires that SG is present as well. */
5742 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
5743 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
5744 features &= ~NETIF_F_ALL_TSO;
5745 }
5746
5747 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
5748 !(features & NETIF_F_IP_CSUM)) {
5749 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
5750 features &= ~NETIF_F_TSO;
5751 features &= ~NETIF_F_TSO_ECN;
5752 }
5753
5754 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
5755 !(features & NETIF_F_IPV6_CSUM)) {
5756 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
5757 features &= ~NETIF_F_TSO6;
5758 }
5759
5760 /* TSO ECN requires that TSO is present as well. */
5761 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5762 features &= ~NETIF_F_TSO_ECN;
5763
5764 /* Software GSO depends on SG. */
5765 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
5766 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
5767 features &= ~NETIF_F_GSO;
5768 }
5769
5770 /* UFO needs SG and checksumming */
5771 if (features & NETIF_F_UFO) {
5772 /* maybe split UFO into V4 and V6? */
5773 if (!((features & NETIF_F_GEN_CSUM) ||
5774 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5775 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5776 netdev_dbg(dev,
5777 "Dropping NETIF_F_UFO since no checksum offload features.\n");
5778 features &= ~NETIF_F_UFO;
5779 }
5780
5781 if (!(features & NETIF_F_SG)) {
5782 netdev_dbg(dev,
5783 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
5784 features &= ~NETIF_F_UFO;
5785 }
5786 }
5787
5788#ifdef CONFIG_NET_RX_BUSY_POLL
5789 if (dev->netdev_ops->ndo_busy_poll)
5790 features |= NETIF_F_BUSY_POLL;
5791 else
5792#endif
5793 features &= ~NETIF_F_BUSY_POLL;
5794
5795 return features;
5796}
5797
5798int __netdev_update_features(struct net_device *dev)
5799{
5800 netdev_features_t features;
5801 int err = 0;
5802
5803 ASSERT_RTNL();
5804
5805 features = netdev_get_wanted_features(dev);
5806
5807 if (dev->netdev_ops->ndo_fix_features)
5808 features = dev->netdev_ops->ndo_fix_features(dev, features);
5809
5810 /* driver might be less strict about feature dependencies */
5811 features = netdev_fix_features(dev, features);
5812
5813 if (dev->features == features)
5814 return 0;
5815
5816 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
5817 &dev->features, &features);
5818
5819 if (dev->netdev_ops->ndo_set_features)
5820 err = dev->netdev_ops->ndo_set_features(dev, features);
5821
5822 if (unlikely(err < 0)) {
5823 netdev_err(dev,
5824 "set_features() failed (%d); wanted %pNF, left %pNF\n",
5825 err, &features, &dev->features);
5826 return -1;
5827 }
5828
5829 if (!err)
5830 dev->features = features;
5831
5832 return 1;
5833}
5834
5835/**
5836 * netdev_update_features - recalculate device features
5837 * @dev: the device to check
5838 *
5839 * Recalculate dev->features set and send notifications if it
5840 * has changed. Should be called after driver or hardware dependent
5841 * conditions might have changed that influence the features.
5842 */
5843void netdev_update_features(struct net_device *dev)
5844{
5845 if (__netdev_update_features(dev))
5846 netdev_features_change(dev);
5847}
5848EXPORT_SYMBOL(netdev_update_features);
5849
5850/**
5851 * netdev_change_features - recalculate device features
5852 * @dev: the device to check
5853 *
5854 * Recalculate dev->features set and send notifications even
5855 * if they have not changed. Should be called instead of
5856 * netdev_update_features() if also dev->vlan_features might
5857 * have changed to allow the changes to be propagated to stacked
5858 * VLAN devices.
5859 */
5860void netdev_change_features(struct net_device *dev)
5861{
5862 __netdev_update_features(dev);
5863 netdev_features_change(dev);
5864}
5865EXPORT_SYMBOL(netdev_change_features);
5866
5867/**
5868 * netif_stacked_transfer_operstate - transfer operstate
5869 * @rootdev: the root or lower level device to transfer state from
5870 * @dev: the device to transfer operstate to
5871 *
5872 * Transfer operational state from root to device. This is normally
5873 * called when a stacking relationship exists between the root
5874 * device and the device(a leaf device).
5875 */
5876void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5877 struct net_device *dev)
5878{
5879 if (rootdev->operstate == IF_OPER_DORMANT)
5880 netif_dormant_on(dev);
5881 else
5882 netif_dormant_off(dev);
5883
5884 if (netif_carrier_ok(rootdev)) {
5885 if (!netif_carrier_ok(dev))
5886 netif_carrier_on(dev);
5887 } else {
5888 if (netif_carrier_ok(dev))
5889 netif_carrier_off(dev);
5890 }
5891}
5892EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5893
5894#ifdef CONFIG_SYSFS
5895static int netif_alloc_rx_queues(struct net_device *dev)
5896{
5897 unsigned int i, count = dev->num_rx_queues;
5898 struct netdev_rx_queue *rx;
5899
5900 BUG_ON(count < 1);
5901
5902 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5903 if (!rx)
5904 return -ENOMEM;
5905
5906 dev->_rx = rx;
5907
5908 for (i = 0; i < count; i++)
5909 rx[i].dev = dev;
5910 return 0;
5911}
5912#endif
5913
5914static void netdev_init_one_queue(struct net_device *dev,
5915 struct netdev_queue *queue, void *_unused)
5916{
5917 /* Initialize queue lock */
5918 spin_lock_init(&queue->_xmit_lock);
5919 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5920 queue->xmit_lock_owner = -1;
5921 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
5922 queue->dev = dev;
5923#ifdef CONFIG_BQL
5924 dql_init(&queue->dql, HZ);
5925#endif
5926}
5927
5928static void netif_free_tx_queues(struct net_device *dev)
5929{
5930 if (is_vmalloc_addr(dev->_tx))
5931 vfree(dev->_tx);
5932 else
5933 kfree(dev->_tx);
5934}
5935
5936static int netif_alloc_netdev_queues(struct net_device *dev)
5937{
5938 unsigned int count = dev->num_tx_queues;
5939 struct netdev_queue *tx;
5940 size_t sz = count * sizeof(*tx);
5941
5942 BUG_ON(count < 1 || count > 0xffff);
5943
5944 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
5945 if (!tx) {
5946 tx = vzalloc(sz);
5947 if (!tx)
5948 return -ENOMEM;
5949 }
5950 dev->_tx = tx;
5951
5952 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5953 spin_lock_init(&dev->tx_global_lock);
5954
5955 return 0;
5956}
5957
5958/**
5959 * register_netdevice - register a network device
5960 * @dev: device to register
5961 *
5962 * Take a completed network device structure and add it to the kernel
5963 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5964 * chain. 0 is returned on success. A negative errno code is returned
5965 * on a failure to set up the device, or if the name is a duplicate.
5966 *
5967 * Callers must hold the rtnl semaphore. You may want
5968 * register_netdev() instead of this.
5969 *
5970 * BUGS:
5971 * The locking appears insufficient to guarantee two parallel registers
5972 * will not get the same name.
5973 */
5974
5975int register_netdevice(struct net_device *dev)
5976{
5977 int ret;
5978 struct net *net = dev_net(dev);
5979
5980 BUG_ON(dev_boot_phase);
5981 ASSERT_RTNL();
5982
5983 might_sleep();
5984
5985 /* When net_device's are persistent, this will be fatal. */
5986 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
5987 BUG_ON(!net);
5988
5989 spin_lock_init(&dev->addr_list_lock);
5990 netdev_set_addr_lockdep_class(dev);
5991
5992 dev->iflink = -1;
5993
5994 ret = dev_get_valid_name(net, dev, dev->name);
5995 if (ret < 0)
5996 goto out;
5997
5998 /* Init, if this function is available */
5999 if (dev->netdev_ops->ndo_init) {
6000 ret = dev->netdev_ops->ndo_init(dev);
6001 if (ret) {
6002 if (ret > 0)
6003 ret = -EIO;
6004 goto out;
6005 }
6006 }
6007
6008 if (((dev->hw_features | dev->features) &
6009 NETIF_F_HW_VLAN_CTAG_FILTER) &&
6010 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6011 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6012 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6013 ret = -EINVAL;
6014 goto err_uninit;
6015 }
6016
6017 ret = -EBUSY;
6018 if (!dev->ifindex)
6019 dev->ifindex = dev_new_index(net);
6020 else if (__dev_get_by_index(net, dev->ifindex))
6021 goto err_uninit;
6022
6023 if (dev->iflink == -1)
6024 dev->iflink = dev->ifindex;
6025
6026 /* Transfer changeable features to wanted_features and enable
6027 * software offloads (GSO and GRO).
6028 */
6029 dev->hw_features |= NETIF_F_SOFT_FEATURES;
6030 dev->features |= NETIF_F_SOFT_FEATURES;
6031 dev->wanted_features = dev->features & dev->hw_features;
6032
6033 if (!(dev->flags & IFF_LOOPBACK)) {
6034 dev->hw_features |= NETIF_F_NOCACHE_COPY;
6035 }
6036
6037 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
6038 */
6039 dev->vlan_features |= NETIF_F_HIGHDMA;
6040
6041 /* Make NETIF_F_SG inheritable to tunnel devices.
6042 */
6043 dev->hw_enc_features |= NETIF_F_SG;
6044
6045 /* Make NETIF_F_SG inheritable to MPLS.
6046 */
6047 dev->mpls_features |= NETIF_F_SG;
6048
6049 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6050 ret = notifier_to_errno(ret);
6051 if (ret)
6052 goto err_uninit;
6053
6054 ret = netdev_register_kobject(dev);
6055 if (ret)
6056 goto err_uninit;
6057 dev->reg_state = NETREG_REGISTERED;
6058
6059 __netdev_update_features(dev);
6060
6061 /*
6062 * Default initial state at registry is that the
6063 * device is present.
6064 */
6065
6066 set_bit(__LINK_STATE_PRESENT, &dev->state);
6067
6068 linkwatch_init_dev(dev);
6069
6070 dev_init_scheduler(dev);
6071 dev_hold(dev);
6072 list_netdevice(dev);
6073 add_device_randomness(dev->dev_addr, dev->addr_len);
6074
6075 /* If the device has permanent device address, driver should
6076 * set dev_addr and also addr_assign_type should be set to
6077 * NET_ADDR_PERM (default value).
6078 */
6079 if (dev->addr_assign_type == NET_ADDR_PERM)
6080 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
6081
6082 /* Notify protocols, that a new device appeared. */
6083 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
6084 ret = notifier_to_errno(ret);
6085 if (ret) {
6086 rollback_registered(dev);
6087 dev->reg_state = NETREG_UNREGISTERED;
6088 }
6089 /*
6090 * Prevent userspace races by waiting until the network
6091 * device is fully setup before sending notifications.
6092 */
6093 if (!dev->rtnl_link_ops ||
6094 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6095 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
6096
6097out:
6098 return ret;
6099
6100err_uninit:
6101 if (dev->netdev_ops->ndo_uninit)
6102 dev->netdev_ops->ndo_uninit(dev);
6103 goto out;
6104}
6105EXPORT_SYMBOL(register_netdevice);
6106
6107/**
6108 * init_dummy_netdev - init a dummy network device for NAPI
6109 * @dev: device to init
6110 *
6111 * This takes a network device structure and initialize the minimum
6112 * amount of fields so it can be used to schedule NAPI polls without
6113 * registering a full blown interface. This is to be used by drivers
6114 * that need to tie several hardware interfaces to a single NAPI
6115 * poll scheduler due to HW limitations.
6116 */
6117int init_dummy_netdev(struct net_device *dev)
6118{
6119 /* Clear everything. Note we don't initialize spinlocks
6120 * are they aren't supposed to be taken by any of the
6121 * NAPI code and this dummy netdev is supposed to be
6122 * only ever used for NAPI polls
6123 */
6124 memset(dev, 0, sizeof(struct net_device));
6125
6126 /* make sure we BUG if trying to hit standard
6127 * register/unregister code path
6128 */
6129 dev->reg_state = NETREG_DUMMY;
6130
6131 /* NAPI wants this */
6132 INIT_LIST_HEAD(&dev->napi_list);
6133
6134 /* a dummy interface is started by default */
6135 set_bit(__LINK_STATE_PRESENT, &dev->state);
6136 set_bit(__LINK_STATE_START, &dev->state);
6137
6138 /* Note : We dont allocate pcpu_refcnt for dummy devices,
6139 * because users of this 'device' dont need to change
6140 * its refcount.
6141 */
6142
6143 return 0;
6144}
6145EXPORT_SYMBOL_GPL(init_dummy_netdev);
6146
6147
6148/**
6149 * register_netdev - register a network device
6150 * @dev: device to register
6151 *
6152 * Take a completed network device structure and add it to the kernel
6153 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6154 * chain. 0 is returned on success. A negative errno code is returned
6155 * on a failure to set up the device, or if the name is a duplicate.
6156 *
6157 * This is a wrapper around register_netdevice that takes the rtnl semaphore
6158 * and expands the device name if you passed a format string to
6159 * alloc_netdev.
6160 */
6161int register_netdev(struct net_device *dev)
6162{
6163 int err;
6164
6165 rtnl_lock();
6166 err = register_netdevice(dev);
6167 rtnl_unlock();
6168 return err;
6169}
6170EXPORT_SYMBOL(register_netdev);
6171
6172int netdev_refcnt_read(const struct net_device *dev)
6173{
6174 int i, refcnt = 0;
6175
6176 for_each_possible_cpu(i)
6177 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
6178 return refcnt;
6179}
6180EXPORT_SYMBOL(netdev_refcnt_read);
6181
6182/**
6183 * netdev_wait_allrefs - wait until all references are gone.
6184 * @dev: target net_device
6185 *
6186 * This is called when unregistering network devices.
6187 *
6188 * Any protocol or device that holds a reference should register
6189 * for netdevice notification, and cleanup and put back the
6190 * reference if they receive an UNREGISTER event.
6191 * We can get stuck here if buggy protocols don't correctly
6192 * call dev_put.
6193 */
6194static void netdev_wait_allrefs(struct net_device *dev)
6195{
6196 unsigned long rebroadcast_time, warning_time;
6197 int refcnt;
6198
6199 linkwatch_forget_dev(dev);
6200
6201 rebroadcast_time = warning_time = jiffies;
6202 refcnt = netdev_refcnt_read(dev);
6203
6204 while (refcnt != 0) {
6205 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
6206 rtnl_lock();
6207
6208 /* Rebroadcast unregister notification */
6209 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6210
6211 __rtnl_unlock();
6212 rcu_barrier();
6213 rtnl_lock();
6214
6215 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
6216 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
6217 &dev->state)) {
6218 /* We must not have linkwatch events
6219 * pending on unregister. If this
6220 * happens, we simply run the queue
6221 * unscheduled, resulting in a noop
6222 * for this device.
6223 */
6224 linkwatch_run_queue();
6225 }
6226
6227 __rtnl_unlock();
6228
6229 rebroadcast_time = jiffies;
6230 }
6231
6232 msleep(250);
6233
6234 refcnt = netdev_refcnt_read(dev);
6235
6236 if (time_after(jiffies, warning_time + 10 * HZ)) {
6237 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
6238 dev->name, refcnt);
6239 warning_time = jiffies;
6240 }
6241 }
6242}
6243
6244/* The sequence is:
6245 *
6246 * rtnl_lock();
6247 * ...
6248 * register_netdevice(x1);
6249 * register_netdevice(x2);
6250 * ...
6251 * unregister_netdevice(y1);
6252 * unregister_netdevice(y2);
6253 * ...
6254 * rtnl_unlock();
6255 * free_netdev(y1);
6256 * free_netdev(y2);
6257 *
6258 * We are invoked by rtnl_unlock().
6259 * This allows us to deal with problems:
6260 * 1) We can delete sysfs objects which invoke hotplug
6261 * without deadlocking with linkwatch via keventd.
6262 * 2) Since we run with the RTNL semaphore not held, we can sleep
6263 * safely in order to wait for the netdev refcnt to drop to zero.
6264 *
6265 * We must not return until all unregister events added during
6266 * the interval the lock was held have been completed.
6267 */
6268void netdev_run_todo(void)
6269{
6270 struct list_head list;
6271
6272 /* Snapshot list, allow later requests */
6273 list_replace_init(&net_todo_list, &list);
6274
6275 __rtnl_unlock();
6276
6277
6278 /* Wait for rcu callbacks to finish before next phase */
6279 if (!list_empty(&list))
6280 rcu_barrier();
6281
6282 while (!list_empty(&list)) {
6283 struct net_device *dev
6284 = list_first_entry(&list, struct net_device, todo_list);
6285 list_del(&dev->todo_list);
6286
6287 rtnl_lock();
6288 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
6289 __rtnl_unlock();
6290
6291 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
6292 pr_err("network todo '%s' but state %d\n",
6293 dev->name, dev->reg_state);
6294 dump_stack();
6295 continue;
6296 }
6297
6298 dev->reg_state = NETREG_UNREGISTERED;
6299
6300 on_each_cpu(flush_backlog, dev, 1);
6301
6302 netdev_wait_allrefs(dev);
6303
6304 /* paranoia */
6305 BUG_ON(netdev_refcnt_read(dev));
6306 WARN_ON(rcu_access_pointer(dev->ip_ptr));
6307 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
6308 WARN_ON(dev->dn_ptr);
6309
6310 if (dev->destructor)
6311 dev->destructor(dev);
6312
6313 /* Report a network device has been unregistered */
6314 rtnl_lock();
6315 dev_net(dev)->dev_unreg_count--;
6316 __rtnl_unlock();
6317 wake_up(&netdev_unregistering_wq);
6318
6319 /* Free network device */
6320 kobject_put(&dev->dev.kobj);
6321 }
6322}
6323
6324/* Convert net_device_stats to rtnl_link_stats64. They have the same
6325 * fields in the same order, with only the type differing.
6326 */
6327void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
6328 const struct net_device_stats *netdev_stats)
6329{
6330#if BITS_PER_LONG == 64
6331 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
6332 memcpy(stats64, netdev_stats, sizeof(*stats64));
6333#else
6334 size_t i, n = sizeof(*stats64) / sizeof(u64);
6335 const unsigned long *src = (const unsigned long *)netdev_stats;
6336 u64 *dst = (u64 *)stats64;
6337
6338 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
6339 sizeof(*stats64) / sizeof(u64));
6340 for (i = 0; i < n; i++)
6341 dst[i] = src[i];
6342#endif
6343}
6344EXPORT_SYMBOL(netdev_stats_to_stats64);
6345
6346/**
6347 * dev_get_stats - get network device statistics
6348 * @dev: device to get statistics from
6349 * @storage: place to store stats
6350 *
6351 * Get network statistics from device. Return @storage.
6352 * The device driver may provide its own method by setting
6353 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
6354 * otherwise the internal statistics structure is used.
6355 */
6356struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
6357 struct rtnl_link_stats64 *storage)
6358{
6359 const struct net_device_ops *ops = dev->netdev_ops;
6360
6361 if (ops->ndo_get_stats64) {
6362 memset(storage, 0, sizeof(*storage));
6363 ops->ndo_get_stats64(dev, storage);
6364 } else if (ops->ndo_get_stats) {
6365 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
6366 } else {
6367 netdev_stats_to_stats64(storage, &dev->stats);
6368 }
6369 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
6370 storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
6371 return storage;
6372}
6373EXPORT_SYMBOL(dev_get_stats);
6374
6375struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
6376{
6377 struct netdev_queue *queue = dev_ingress_queue(dev);
6378
6379#ifdef CONFIG_NET_CLS_ACT
6380 if (queue)
6381 return queue;
6382 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
6383 if (!queue)
6384 return NULL;
6385 netdev_init_one_queue(dev, queue, NULL);
6386 queue->qdisc = &noop_qdisc;
6387 queue->qdisc_sleeping = &noop_qdisc;
6388 rcu_assign_pointer(dev->ingress_queue, queue);
6389#endif
6390 return queue;
6391}
6392
6393static const struct ethtool_ops default_ethtool_ops;
6394
6395void netdev_set_default_ethtool_ops(struct net_device *dev,
6396 const struct ethtool_ops *ops)
6397{
6398 if (dev->ethtool_ops == &default_ethtool_ops)
6399 dev->ethtool_ops = ops;
6400}
6401EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
6402
6403void netdev_freemem(struct net_device *dev)
6404{
6405 char *addr = (char *)dev - dev->padded;
6406
6407 if (is_vmalloc_addr(addr))
6408 vfree(addr);
6409 else
6410 kfree(addr);
6411}
6412
6413/**
6414 * alloc_netdev_mqs - allocate network device
6415 * @sizeof_priv: size of private data to allocate space for
6416 * @name: device name format string
6417 * @setup: callback to initialize device
6418 * @txqs: the number of TX subqueues to allocate
6419 * @rxqs: the number of RX subqueues to allocate
6420 *
6421 * Allocates a struct net_device with private data area for driver use
6422 * and performs basic initialization. Also allocates subqueue structs
6423 * for each queue on the device.
6424 */
6425struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
6426 void (*setup)(struct net_device *),
6427 unsigned int txqs, unsigned int rxqs)
6428{
6429 struct net_device *dev;
6430 size_t alloc_size;
6431 struct net_device *p;
6432
6433 BUG_ON(strlen(name) >= sizeof(dev->name));
6434
6435 if (txqs < 1) {
6436 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
6437 return NULL;
6438 }
6439
6440#ifdef CONFIG_SYSFS
6441 if (rxqs < 1) {
6442 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
6443 return NULL;
6444 }
6445#endif
6446
6447 alloc_size = sizeof(struct net_device);
6448 if (sizeof_priv) {
6449 /* ensure 32-byte alignment of private area */
6450 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
6451 alloc_size += sizeof_priv;
6452 }
6453 /* ensure 32-byte alignment of whole construct */
6454 alloc_size += NETDEV_ALIGN - 1;
6455
6456 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6457 if (!p)
6458 p = vzalloc(alloc_size);
6459 if (!p)
6460 return NULL;
6461
6462 dev = PTR_ALIGN(p, NETDEV_ALIGN);
6463 dev->padded = (char *)dev - (char *)p;
6464
6465 dev->pcpu_refcnt = alloc_percpu(int);
6466 if (!dev->pcpu_refcnt)
6467 goto free_dev;
6468
6469 if (dev_addr_init(dev))
6470 goto free_pcpu;
6471
6472 dev_mc_init(dev);
6473 dev_uc_init(dev);
6474
6475 dev_net_set(dev, &init_net);
6476
6477 dev->gso_max_size = GSO_MAX_SIZE;
6478 dev->gso_max_segs = GSO_MAX_SEGS;
6479
6480 INIT_LIST_HEAD(&dev->napi_list);
6481 INIT_LIST_HEAD(&dev->unreg_list);
6482 INIT_LIST_HEAD(&dev->close_list);
6483 INIT_LIST_HEAD(&dev->link_watch_list);
6484 INIT_LIST_HEAD(&dev->adj_list.upper);
6485 INIT_LIST_HEAD(&dev->adj_list.lower);
6486 INIT_LIST_HEAD(&dev->all_adj_list.upper);
6487 INIT_LIST_HEAD(&dev->all_adj_list.lower);
6488 dev->priv_flags = IFF_XMIT_DST_RELEASE;
6489 setup(dev);
6490
6491 dev->num_tx_queues = txqs;
6492 dev->real_num_tx_queues = txqs;
6493 if (netif_alloc_netdev_queues(dev))
6494 goto free_all;
6495
6496#ifdef CONFIG_SYSFS
6497 dev->num_rx_queues = rxqs;
6498 dev->real_num_rx_queues = rxqs;
6499 if (netif_alloc_rx_queues(dev))
6500 goto free_all;
6501#endif
6502
6503 strcpy(dev->name, name);
6504 dev->group = INIT_NETDEV_GROUP;
6505 if (!dev->ethtool_ops)
6506 dev->ethtool_ops = &default_ethtool_ops;
6507 return dev;
6508
6509free_all:
6510 free_netdev(dev);
6511 return NULL;
6512
6513free_pcpu:
6514 free_percpu(dev->pcpu_refcnt);
6515 netif_free_tx_queues(dev);
6516#ifdef CONFIG_SYSFS
6517 kfree(dev->_rx);
6518#endif
6519
6520free_dev:
6521 netdev_freemem(dev);
6522 return NULL;
6523}
6524EXPORT_SYMBOL(alloc_netdev_mqs);
6525
6526/**
6527 * free_netdev - free network device
6528 * @dev: device
6529 *
6530 * This function does the last stage of destroying an allocated device
6531 * interface. The reference to the device object is released.
6532 * If this is the last reference then it will be freed.
6533 */
6534void free_netdev(struct net_device *dev)
6535{
6536 struct napi_struct *p, *n;
6537
6538 release_net(dev_net(dev));
6539
6540 netif_free_tx_queues(dev);
6541#ifdef CONFIG_SYSFS
6542 kfree(dev->_rx);
6543#endif
6544
6545 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
6546
6547 /* Flush device addresses */
6548 dev_addr_flush(dev);
6549
6550 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
6551 netif_napi_del(p);
6552
6553 free_percpu(dev->pcpu_refcnt);
6554 dev->pcpu_refcnt = NULL;
6555
6556 /* Compatibility with error handling in drivers */
6557 if (dev->reg_state == NETREG_UNINITIALIZED) {
6558 netdev_freemem(dev);
6559 return;
6560 }
6561
6562 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
6563 dev->reg_state = NETREG_RELEASED;
6564
6565 /* will free via device release */
6566 put_device(&dev->dev);
6567}
6568EXPORT_SYMBOL(free_netdev);
6569
6570/**
6571 * synchronize_net - Synchronize with packet receive processing
6572 *
6573 * Wait for packets currently being received to be done.
6574 * Does not block later packets from starting.
6575 */
6576void synchronize_net(void)
6577{
6578 might_sleep();
6579 if (rtnl_is_locked())
6580 synchronize_rcu_expedited();
6581 else
6582 synchronize_rcu();
6583}
6584EXPORT_SYMBOL(synchronize_net);
6585
6586/**
6587 * unregister_netdevice_queue - remove device from the kernel
6588 * @dev: device
6589 * @head: list
6590 *
6591 * This function shuts down a device interface and removes it
6592 * from the kernel tables.
6593 * If head not NULL, device is queued to be unregistered later.
6594 *
6595 * Callers must hold the rtnl semaphore. You may want
6596 * unregister_netdev() instead of this.
6597 */
6598
6599void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
6600{
6601 ASSERT_RTNL();
6602
6603 if (head) {
6604 list_move_tail(&dev->unreg_list, head);
6605 } else {
6606 rollback_registered(dev);
6607 /* Finish processing unregister after unlock */
6608 net_set_todo(dev);
6609 }
6610}
6611EXPORT_SYMBOL(unregister_netdevice_queue);
6612
6613/**
6614 * unregister_netdevice_many - unregister many devices
6615 * @head: list of devices
6616 */
6617void unregister_netdevice_many(struct list_head *head)
6618{
6619 struct net_device *dev;
6620
6621 if (!list_empty(head)) {
6622 rollback_registered_many(head);
6623 list_for_each_entry(dev, head, unreg_list)
6624 net_set_todo(dev);
6625 }
6626}
6627EXPORT_SYMBOL(unregister_netdevice_many);
6628
6629/**
6630 * unregister_netdev - remove device from the kernel
6631 * @dev: device
6632 *
6633 * This function shuts down a device interface and removes it
6634 * from the kernel tables.
6635 *
6636 * This is just a wrapper for unregister_netdevice that takes
6637 * the rtnl semaphore. In general you want to use this and not
6638 * unregister_netdevice.
6639 */
6640void unregister_netdev(struct net_device *dev)
6641{
6642 rtnl_lock();
6643 unregister_netdevice(dev);
6644 rtnl_unlock();
6645}
6646EXPORT_SYMBOL(unregister_netdev);
6647
6648/**
6649 * dev_change_net_namespace - move device to different nethost namespace
6650 * @dev: device
6651 * @net: network namespace
6652 * @pat: If not NULL name pattern to try if the current device name
6653 * is already taken in the destination network namespace.
6654 *
6655 * This function shuts down a device interface and moves it
6656 * to a new network namespace. On success 0 is returned, on
6657 * a failure a netagive errno code is returned.
6658 *
6659 * Callers must hold the rtnl semaphore.
6660 */
6661
6662int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6663{
6664 int err;
6665
6666 ASSERT_RTNL();
6667
6668 /* Don't allow namespace local devices to be moved. */
6669 err = -EINVAL;
6670 if (dev->features & NETIF_F_NETNS_LOCAL)
6671 goto out;
6672
6673 /* Ensure the device has been registrered */
6674 if (dev->reg_state != NETREG_REGISTERED)
6675 goto out;
6676
6677 /* Get out if there is nothing todo */
6678 err = 0;
6679 if (net_eq(dev_net(dev), net))
6680 goto out;
6681
6682 /* Pick the destination device name, and ensure
6683 * we can use it in the destination network namespace.
6684 */
6685 err = -EEXIST;
6686 if (__dev_get_by_name(net, dev->name)) {
6687 /* We get here if we can't use the current device name */
6688 if (!pat)
6689 goto out;
6690 if (dev_get_valid_name(net, dev, pat) < 0)
6691 goto out;
6692 }
6693
6694 /*
6695 * And now a mini version of register_netdevice unregister_netdevice.
6696 */
6697
6698 /* If device is running close it first. */
6699 dev_close(dev);
6700
6701 /* And unlink it from device chain */
6702 err = -ENODEV;
6703 unlist_netdevice(dev);
6704
6705 synchronize_net();
6706
6707 /* Shutdown queueing discipline. */
6708 dev_shutdown(dev);
6709
6710 /* Notify protocols, that we are about to destroy
6711 this device. They should clean all the things.
6712
6713 Note that dev->reg_state stays at NETREG_REGISTERED.
6714 This is wanted because this way 8021q and macvlan know
6715 the device is just moving and can keep their slaves up.
6716 */
6717 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6718 rcu_barrier();
6719 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
6720 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
6721
6722 /*
6723 * Flush the unicast and multicast chains
6724 */
6725 dev_uc_flush(dev);
6726 dev_mc_flush(dev);
6727
6728 /* Send a netdev-removed uevent to the old namespace */
6729 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
6730
6731 /* Actually switch the network namespace */
6732 dev_net_set(dev, net);
6733
6734 /* If there is an ifindex conflict assign a new one */
6735 if (__dev_get_by_index(net, dev->ifindex)) {
6736 int iflink = (dev->iflink == dev->ifindex);
6737 dev->ifindex = dev_new_index(net);
6738 if (iflink)
6739 dev->iflink = dev->ifindex;
6740 }
6741
6742 /* Send a netdev-add uevent to the new namespace */
6743 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
6744
6745 /* Fixup kobjects */
6746 err = device_rename(&dev->dev, dev->name);
6747 WARN_ON(err);
6748
6749 /* Add the device back in the hashes */
6750 list_netdevice(dev);
6751
6752 /* Notify protocols, that a new device appeared. */
6753 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6754
6755 /*
6756 * Prevent userspace races by waiting until the network
6757 * device is fully setup before sending notifications.
6758 */
6759 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
6760
6761 synchronize_net();
6762 err = 0;
6763out:
6764 return err;
6765}
6766EXPORT_SYMBOL_GPL(dev_change_net_namespace);
6767
6768static int dev_cpu_callback(struct notifier_block *nfb,
6769 unsigned long action,
6770 void *ocpu)
6771{
6772 struct sk_buff **list_skb;
6773 struct sk_buff *skb;
6774 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6775 struct softnet_data *sd, *oldsd;
6776
6777 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
6778 return NOTIFY_OK;
6779
6780 local_irq_disable();
6781 cpu = smp_processor_id();
6782 sd = &per_cpu(softnet_data, cpu);
6783 oldsd = &per_cpu(softnet_data, oldcpu);
6784
6785 /* Find end of our completion_queue. */
6786 list_skb = &sd->completion_queue;
6787 while (*list_skb)
6788 list_skb = &(*list_skb)->next;
6789 /* Append completion queue from offline CPU. */
6790 *list_skb = oldsd->completion_queue;
6791 oldsd->completion_queue = NULL;
6792
6793 /* Append output queue from offline CPU. */
6794 if (oldsd->output_queue) {
6795 *sd->output_queue_tailp = oldsd->output_queue;
6796 sd->output_queue_tailp = oldsd->output_queue_tailp;
6797 oldsd->output_queue = NULL;
6798 oldsd->output_queue_tailp = &oldsd->output_queue;
6799 }
6800 /* Append NAPI poll list from offline CPU. */
6801 if (!list_empty(&oldsd->poll_list)) {
6802 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6803 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6804 }
6805
6806 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6807 local_irq_enable();
6808
6809 /* Process offline CPU's input_pkt_queue */
6810 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6811 netif_rx_internal(skb);
6812 input_queue_head_incr(oldsd);
6813 }
6814 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
6815 netif_rx_internal(skb);
6816 input_queue_head_incr(oldsd);
6817 }
6818
6819 return NOTIFY_OK;
6820}
6821
6822
6823/**
6824 * netdev_increment_features - increment feature set by one
6825 * @all: current feature set
6826 * @one: new feature set
6827 * @mask: mask feature set
6828 *
6829 * Computes a new feature set after adding a device with feature set
6830 * @one to the master device with current feature set @all. Will not
6831 * enable anything that is off in @mask. Returns the new feature set.
6832 */
6833netdev_features_t netdev_increment_features(netdev_features_t all,
6834 netdev_features_t one, netdev_features_t mask)
6835{
6836 if (mask & NETIF_F_GEN_CSUM)
6837 mask |= NETIF_F_ALL_CSUM;
6838 mask |= NETIF_F_VLAN_CHALLENGED;
6839
6840 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6841 all &= one | ~NETIF_F_ALL_FOR_ALL;
6842
6843 /* If one device supports hw checksumming, set for all. */
6844 if (all & NETIF_F_GEN_CSUM)
6845 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
6846
6847 return all;
6848}
6849EXPORT_SYMBOL(netdev_increment_features);
6850
6851static struct hlist_head * __net_init netdev_create_hash(void)
6852{
6853 int i;
6854 struct hlist_head *hash;
6855
6856 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6857 if (hash != NULL)
6858 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6859 INIT_HLIST_HEAD(&hash[i]);
6860
6861 return hash;
6862}
6863
6864/* Initialize per network namespace state */
6865static int __net_init netdev_init(struct net *net)
6866{
6867 if (net != &init_net)
6868 INIT_LIST_HEAD(&net->dev_base_head);
6869
6870 net->dev_name_head = netdev_create_hash();
6871 if (net->dev_name_head == NULL)
6872 goto err_name;
6873
6874 net->dev_index_head = netdev_create_hash();
6875 if (net->dev_index_head == NULL)
6876 goto err_idx;
6877
6878 return 0;
6879
6880err_idx:
6881 kfree(net->dev_name_head);
6882err_name:
6883 return -ENOMEM;
6884}
6885
6886/**
6887 * netdev_drivername - network driver for the device
6888 * @dev: network device
6889 *
6890 * Determine network driver for device.
6891 */
6892const char *netdev_drivername(const struct net_device *dev)
6893{
6894 const struct device_driver *driver;
6895 const struct device *parent;
6896 const char *empty = "";
6897
6898 parent = dev->dev.parent;
6899 if (!parent)
6900 return empty;
6901
6902 driver = parent->driver;
6903 if (driver && driver->name)
6904 return driver->name;
6905 return empty;
6906}
6907
6908static int __netdev_printk(const char *level, const struct net_device *dev,
6909 struct va_format *vaf)
6910{
6911 int r;
6912
6913 if (dev && dev->dev.parent) {
6914 r = dev_printk_emit(level[1] - '0',
6915 dev->dev.parent,
6916 "%s %s %s: %pV",
6917 dev_driver_string(dev->dev.parent),
6918 dev_name(dev->dev.parent),
6919 netdev_name(dev), vaf);
6920 } else if (dev) {
6921 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6922 } else {
6923 r = printk("%s(NULL net_device): %pV", level, vaf);
6924 }
6925
6926 return r;
6927}
6928
6929int netdev_printk(const char *level, const struct net_device *dev,
6930 const char *format, ...)
6931{
6932 struct va_format vaf;
6933 va_list args;
6934 int r;
6935
6936 va_start(args, format);
6937
6938 vaf.fmt = format;
6939 vaf.va = &args;
6940
6941 r = __netdev_printk(level, dev, &vaf);
6942
6943 va_end(args);
6944
6945 return r;
6946}
6947EXPORT_SYMBOL(netdev_printk);
6948
6949#define define_netdev_printk_level(func, level) \
6950int func(const struct net_device *dev, const char *fmt, ...) \
6951{ \
6952 int r; \
6953 struct va_format vaf; \
6954 va_list args; \
6955 \
6956 va_start(args, fmt); \
6957 \
6958 vaf.fmt = fmt; \
6959 vaf.va = &args; \
6960 \
6961 r = __netdev_printk(level, dev, &vaf); \
6962 \
6963 va_end(args); \
6964 \
6965 return r; \
6966} \
6967EXPORT_SYMBOL(func);
6968
6969define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6970define_netdev_printk_level(netdev_alert, KERN_ALERT);
6971define_netdev_printk_level(netdev_crit, KERN_CRIT);
6972define_netdev_printk_level(netdev_err, KERN_ERR);
6973define_netdev_printk_level(netdev_warn, KERN_WARNING);
6974define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6975define_netdev_printk_level(netdev_info, KERN_INFO);
6976
6977static void __net_exit netdev_exit(struct net *net)
6978{
6979 kfree(net->dev_name_head);
6980 kfree(net->dev_index_head);
6981}
6982
6983static struct pernet_operations __net_initdata netdev_net_ops = {
6984 .init = netdev_init,
6985 .exit = netdev_exit,
6986};
6987
6988static void __net_exit default_device_exit(struct net *net)
6989{
6990 struct net_device *dev, *aux;
6991 /*
6992 * Push all migratable network devices back to the
6993 * initial network namespace
6994 */
6995 rtnl_lock();
6996 for_each_netdev_safe(net, dev, aux) {
6997 int err;
6998 char fb_name[IFNAMSIZ];
6999
7000 /* Ignore unmoveable devices (i.e. loopback) */
7001 if (dev->features & NETIF_F_NETNS_LOCAL)
7002 continue;
7003
7004 /* Leave virtual devices for the generic cleanup */
7005 if (dev->rtnl_link_ops)
7006 continue;
7007
7008 /* Push remaining network devices to init_net */
7009 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7010 err = dev_change_net_namespace(dev, &init_net, fb_name);
7011 if (err) {
7012 pr_emerg("%s: failed to move %s to init_net: %d\n",
7013 __func__, dev->name, err);
7014 BUG();
7015 }
7016 }
7017 rtnl_unlock();
7018}
7019
7020static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7021{
7022 /* Return with the rtnl_lock held when there are no network
7023 * devices unregistering in any network namespace in net_list.
7024 */
7025 struct net *net;
7026 bool unregistering;
7027 DEFINE_WAIT(wait);
7028
7029 for (;;) {
7030 prepare_to_wait(&netdev_unregistering_wq, &wait,
7031 TASK_UNINTERRUPTIBLE);
7032 unregistering = false;
7033 rtnl_lock();
7034 list_for_each_entry(net, net_list, exit_list) {
7035 if (net->dev_unreg_count > 0) {
7036 unregistering = true;
7037 break;
7038 }
7039 }
7040 if (!unregistering)
7041 break;
7042 __rtnl_unlock();
7043 schedule();
7044 }
7045 finish_wait(&netdev_unregistering_wq, &wait);
7046}
7047
7048static void __net_exit default_device_exit_batch(struct list_head *net_list)
7049{
7050 /* At exit all network devices most be removed from a network
7051 * namespace. Do this in the reverse order of registration.
7052 * Do this across as many network namespaces as possible to
7053 * improve batching efficiency.
7054 */
7055 struct net_device *dev;
7056 struct net *net;
7057 LIST_HEAD(dev_kill_list);
7058
7059 /* To prevent network device cleanup code from dereferencing
7060 * loopback devices or network devices that have been freed
7061 * wait here for all pending unregistrations to complete,
7062 * before unregistring the loopback device and allowing the
7063 * network namespace be freed.
7064 *
7065 * The netdev todo list containing all network devices
7066 * unregistrations that happen in default_device_exit_batch
7067 * will run in the rtnl_unlock() at the end of
7068 * default_device_exit_batch.
7069 */
7070 rtnl_lock_unregistering(net_list);
7071 list_for_each_entry(net, net_list, exit_list) {
7072 for_each_netdev_reverse(net, dev) {
7073 if (dev->rtnl_link_ops)
7074 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
7075 else
7076 unregister_netdevice_queue(dev, &dev_kill_list);
7077 }
7078 }
7079 unregister_netdevice_many(&dev_kill_list);
7080 list_del(&dev_kill_list);
7081 rtnl_unlock();
7082}
7083
7084static struct pernet_operations __net_initdata default_device_ops = {
7085 .exit = default_device_exit,
7086 .exit_batch = default_device_exit_batch,
7087};
7088
7089/*
7090 * Initialize the DEV module. At boot time this walks the device list and
7091 * unhooks any devices that fail to initialise (normally hardware not
7092 * present) and leaves us with a valid list of present and active devices.
7093 *
7094 */
7095
7096/*
7097 * This is called single threaded during boot, so no need
7098 * to take the rtnl semaphore.
7099 */
7100static int __init net_dev_init(void)
7101{
7102 int i, rc = -ENOMEM;
7103
7104 BUG_ON(!dev_boot_phase);
7105
7106 if (dev_proc_init())
7107 goto out;
7108
7109 if (netdev_kobject_init())
7110 goto out;
7111
7112 INIT_LIST_HEAD(&ptype_all);
7113 for (i = 0; i < PTYPE_HASH_SIZE; i++)
7114 INIT_LIST_HEAD(&ptype_base[i]);
7115
7116 INIT_LIST_HEAD(&offload_base);
7117
7118 if (register_pernet_subsys(&netdev_net_ops))
7119 goto out;
7120
7121 /*
7122 * Initialise the packet receive queues.
7123 */
7124
7125 for_each_possible_cpu(i) {
7126 struct softnet_data *sd = &per_cpu(softnet_data, i);
7127
7128 skb_queue_head_init(&sd->input_pkt_queue);
7129 skb_queue_head_init(&sd->process_queue);
7130 INIT_LIST_HEAD(&sd->poll_list);
7131 sd->output_queue_tailp = &sd->output_queue;
7132#ifdef CONFIG_RPS
7133 sd->csd.func = rps_trigger_softirq;
7134 sd->csd.info = sd;
7135 sd->cpu = i;
7136#endif
7137
7138 sd->backlog.poll = process_backlog;
7139 sd->backlog.weight = weight_p;
7140 }
7141
7142 dev_boot_phase = 0;
7143
7144 /* The loopback device is special if any other network devices
7145 * is present in a network namespace the loopback device must
7146 * be present. Since we now dynamically allocate and free the
7147 * loopback device ensure this invariant is maintained by
7148 * keeping the loopback device as the first device on the
7149 * list of network devices. Ensuring the loopback devices
7150 * is the first device that appears and the last network device
7151 * that disappears.
7152 */
7153 if (register_pernet_device(&loopback_net_ops))
7154 goto out;
7155
7156 if (register_pernet_device(&default_device_ops))
7157 goto out;
7158
7159 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
7160 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
7161
7162 hotcpu_notifier(dev_cpu_callback, 0);
7163 dst_init();
7164 rc = 0;
7165out:
7166 return rc;
7167}
7168
7169subsys_initcall(net_dev_init);