Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * NET3 Protocol independent device support routines.
4 *
5 * Derived from the non IP parts of dev.c 1.0.19
6 * Authors: Ross Biro
7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8 * Mark Evans, <evansmp@uhura.aston.ac.uk>
9 *
10 * Additional Authors:
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
12 * Alan Cox <gw4pts@gw4pts.ampr.org>
13 * David Hinds <dahinds@users.sourceforge.net>
14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15 * Adam Sulmicki <adam@cfar.umd.edu>
16 * Pekka Riikonen <priikone@poesidon.pspt.fi>
17 *
18 * Changes:
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
20 * to 2 if register_netdev gets called
21 * before net_dev_init & also removed a
22 * few lines of code in the process.
23 * Alan Cox : device private ioctl copies fields back.
24 * Alan Cox : Transmit queue code does relevant
25 * stunts to keep the queue safe.
26 * Alan Cox : Fixed double lock.
27 * Alan Cox : Fixed promisc NULL pointer trap
28 * ???????? : Support the full private ioctl range
29 * Alan Cox : Moved ioctl permission check into
30 * drivers
31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
32 * Alan Cox : 100 backlog just doesn't cut it when
33 * you start doing multicast video 8)
34 * Alan Cox : Rewrote net_bh and list manager.
35 * Alan Cox : Fix ETH_P_ALL echoback lengths.
36 * Alan Cox : Took out transmit every packet pass
37 * Saved a few bytes in the ioctl handler
38 * Alan Cox : Network driver sets packet type before
39 * calling netif_rx. Saves a function
40 * call a packet.
41 * Alan Cox : Hashed net_bh()
42 * Richard Kooijman: Timestamp fixes.
43 * Alan Cox : Wrong field in SIOCGIFDSTADDR
44 * Alan Cox : Device lock protection.
45 * Alan Cox : Fixed nasty side effect of device close
46 * changes.
47 * Rudi Cilibrasi : Pass the right thing to
48 * set_mac_address()
49 * Dave Miller : 32bit quantity for the device lock to
50 * make it work out on a Sparc.
51 * Bjorn Ekwall : Added KERNELD hack.
52 * Alan Cox : Cleaned up the backlog initialise.
53 * Craig Metz : SIOCGIFCONF fix if space for under
54 * 1 device.
55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
56 * is no device open function.
57 * Andi Kleen : Fix error reporting for SIOCGIFCONF
58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
59 * Cyrus Durgin : Cleaned for KMOD
60 * Adam Sulmicki : Bug Fix : Network Device Unload
61 * A network device unload needs to purge
62 * the backlog queue.
63 * Paul Rusty Russell : SIOCSIFNAME
64 * Pekka Riikonen : Netdev boot-time settings code
65 * Andrew Morton : Make unregister_netdevice wait
66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
68 * - netif_rx() feedback
69 */
70
71#include <linux/uaccess.h>
72#include <linux/bitmap.h>
73#include <linux/capability.h>
74#include <linux/cpu.h>
75#include <linux/types.h>
76#include <linux/kernel.h>
77#include <linux/hash.h>
78#include <linux/slab.h>
79#include <linux/sched.h>
80#include <linux/sched/isolation.h>
81#include <linux/sched/mm.h>
82#include <linux/smpboot.h>
83#include <linux/mutex.h>
84#include <linux/rwsem.h>
85#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
94#include <linux/ethtool.h>
95#include <linux/skbuff.h>
96#include <linux/kthread.h>
97#include <linux/bpf.h>
98#include <linux/bpf_trace.h>
99#include <net/net_namespace.h>
100#include <net/sock.h>
101#include <net/busy_poll.h>
102#include <linux/rtnetlink.h>
103#include <linux/stat.h>
104#include <net/dsa.h>
105#include <net/dst.h>
106#include <net/dst_metadata.h>
107#include <net/gro.h>
108#include <net/pkt_sched.h>
109#include <net/pkt_cls.h>
110#include <net/checksum.h>
111#include <net/xfrm.h>
112#include <net/tcx.h>
113#include <linux/highmem.h>
114#include <linux/init.h>
115#include <linux/module.h>
116#include <linux/netpoll.h>
117#include <linux/rcupdate.h>
118#include <linux/delay.h>
119#include <net/iw_handler.h>
120#include <asm/current.h>
121#include <linux/audit.h>
122#include <linux/dmaengine.h>
123#include <linux/err.h>
124#include <linux/ctype.h>
125#include <linux/if_arp.h>
126#include <linux/if_vlan.h>
127#include <linux/ip.h>
128#include <net/ip.h>
129#include <net/mpls.h>
130#include <linux/ipv6.h>
131#include <linux/in.h>
132#include <linux/jhash.h>
133#include <linux/random.h>
134#include <trace/events/napi.h>
135#include <trace/events/net.h>
136#include <trace/events/skb.h>
137#include <trace/events/qdisc.h>
138#include <trace/events/xdp.h>
139#include <linux/inetdevice.h>
140#include <linux/cpu_rmap.h>
141#include <linux/static_key.h>
142#include <linux/hashtable.h>
143#include <linux/vmalloc.h>
144#include <linux/if_macvlan.h>
145#include <linux/errqueue.h>
146#include <linux/hrtimer.h>
147#include <linux/netfilter_netdev.h>
148#include <linux/crash_dump.h>
149#include <linux/sctp.h>
150#include <net/udp_tunnel.h>
151#include <linux/net_namespace.h>
152#include <linux/indirect_call_wrapper.h>
153#include <net/devlink.h>
154#include <linux/pm_runtime.h>
155#include <linux/prandom.h>
156#include <linux/once_lite.h>
157#include <net/netdev_rx_queue.h>
158#include <net/page_pool/types.h>
159#include <net/page_pool/helpers.h>
160#include <net/rps.h>
161#include <linux/phy_link_topology.h>
162
163#include "dev.h"
164#include "devmem.h"
165#include "net-sysfs.h"
166
167static DEFINE_SPINLOCK(ptype_lock);
168struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
169
170static int netif_rx_internal(struct sk_buff *skb);
171static int call_netdevice_notifiers_extack(unsigned long val,
172 struct net_device *dev,
173 struct netlink_ext_ack *extack);
174
175static DEFINE_MUTEX(ifalias_mutex);
176
177/* protects napi_hash addition/deletion and napi_gen_id */
178static DEFINE_SPINLOCK(napi_hash_lock);
179
180static unsigned int napi_gen_id = NR_CPUS;
181static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
182
183static DECLARE_RWSEM(devnet_rename_sem);
184
185static inline void dev_base_seq_inc(struct net *net)
186{
187 unsigned int val = net->dev_base_seq + 1;
188
189 WRITE_ONCE(net->dev_base_seq, val ?: 1);
190}
191
192static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
193{
194 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
195
196 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
197}
198
199static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
200{
201 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
202}
203
204#ifndef CONFIG_PREEMPT_RT
205
206static DEFINE_STATIC_KEY_FALSE(use_backlog_threads_key);
207
208static int __init setup_backlog_napi_threads(char *arg)
209{
210 static_branch_enable(&use_backlog_threads_key);
211 return 0;
212}
213early_param("thread_backlog_napi", setup_backlog_napi_threads);
214
215static bool use_backlog_threads(void)
216{
217 return static_branch_unlikely(&use_backlog_threads_key);
218}
219
220#else
221
222static bool use_backlog_threads(void)
223{
224 return true;
225}
226
227#endif
228
229static inline void backlog_lock_irq_save(struct softnet_data *sd,
230 unsigned long *flags)
231{
232 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
233 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
234 else
235 local_irq_save(*flags);
236}
237
238static inline void backlog_lock_irq_disable(struct softnet_data *sd)
239{
240 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
241 spin_lock_irq(&sd->input_pkt_queue.lock);
242 else
243 local_irq_disable();
244}
245
246static inline void backlog_unlock_irq_restore(struct softnet_data *sd,
247 unsigned long *flags)
248{
249 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
250 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
251 else
252 local_irq_restore(*flags);
253}
254
255static inline void backlog_unlock_irq_enable(struct softnet_data *sd)
256{
257 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
258 spin_unlock_irq(&sd->input_pkt_queue.lock);
259 else
260 local_irq_enable();
261}
262
263static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
264 const char *name)
265{
266 struct netdev_name_node *name_node;
267
268 name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
269 if (!name_node)
270 return NULL;
271 INIT_HLIST_NODE(&name_node->hlist);
272 name_node->dev = dev;
273 name_node->name = name;
274 return name_node;
275}
276
277static struct netdev_name_node *
278netdev_name_node_head_alloc(struct net_device *dev)
279{
280 struct netdev_name_node *name_node;
281
282 name_node = netdev_name_node_alloc(dev, dev->name);
283 if (!name_node)
284 return NULL;
285 INIT_LIST_HEAD(&name_node->list);
286 return name_node;
287}
288
289static void netdev_name_node_free(struct netdev_name_node *name_node)
290{
291 kfree(name_node);
292}
293
294static void netdev_name_node_add(struct net *net,
295 struct netdev_name_node *name_node)
296{
297 hlist_add_head_rcu(&name_node->hlist,
298 dev_name_hash(net, name_node->name));
299}
300
301static void netdev_name_node_del(struct netdev_name_node *name_node)
302{
303 hlist_del_rcu(&name_node->hlist);
304}
305
306static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
307 const char *name)
308{
309 struct hlist_head *head = dev_name_hash(net, name);
310 struct netdev_name_node *name_node;
311
312 hlist_for_each_entry(name_node, head, hlist)
313 if (!strcmp(name_node->name, name))
314 return name_node;
315 return NULL;
316}
317
318static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
319 const char *name)
320{
321 struct hlist_head *head = dev_name_hash(net, name);
322 struct netdev_name_node *name_node;
323
324 hlist_for_each_entry_rcu(name_node, head, hlist)
325 if (!strcmp(name_node->name, name))
326 return name_node;
327 return NULL;
328}
329
330bool netdev_name_in_use(struct net *net, const char *name)
331{
332 return netdev_name_node_lookup(net, name);
333}
334EXPORT_SYMBOL(netdev_name_in_use);
335
336int netdev_name_node_alt_create(struct net_device *dev, const char *name)
337{
338 struct netdev_name_node *name_node;
339 struct net *net = dev_net(dev);
340
341 name_node = netdev_name_node_lookup(net, name);
342 if (name_node)
343 return -EEXIST;
344 name_node = netdev_name_node_alloc(dev, name);
345 if (!name_node)
346 return -ENOMEM;
347 netdev_name_node_add(net, name_node);
348 /* The node that holds dev->name acts as a head of per-device list. */
349 list_add_tail_rcu(&name_node->list, &dev->name_node->list);
350
351 return 0;
352}
353
354static void netdev_name_node_alt_free(struct rcu_head *head)
355{
356 struct netdev_name_node *name_node =
357 container_of(head, struct netdev_name_node, rcu);
358
359 kfree(name_node->name);
360 netdev_name_node_free(name_node);
361}
362
363static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
364{
365 netdev_name_node_del(name_node);
366 list_del(&name_node->list);
367 call_rcu(&name_node->rcu, netdev_name_node_alt_free);
368}
369
370int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
371{
372 struct netdev_name_node *name_node;
373 struct net *net = dev_net(dev);
374
375 name_node = netdev_name_node_lookup(net, name);
376 if (!name_node)
377 return -ENOENT;
378 /* lookup might have found our primary name or a name belonging
379 * to another device.
380 */
381 if (name_node == dev->name_node || name_node->dev != dev)
382 return -EINVAL;
383
384 __netdev_name_node_alt_destroy(name_node);
385 return 0;
386}
387
388static void netdev_name_node_alt_flush(struct net_device *dev)
389{
390 struct netdev_name_node *name_node, *tmp;
391
392 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) {
393 list_del(&name_node->list);
394 netdev_name_node_alt_free(&name_node->rcu);
395 }
396}
397
398/* Device list insertion */
399static void list_netdevice(struct net_device *dev)
400{
401 struct netdev_name_node *name_node;
402 struct net *net = dev_net(dev);
403
404 ASSERT_RTNL();
405
406 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
407 netdev_name_node_add(net, dev->name_node);
408 hlist_add_head_rcu(&dev->index_hlist,
409 dev_index_hash(net, dev->ifindex));
410
411 netdev_for_each_altname(dev, name_node)
412 netdev_name_node_add(net, name_node);
413
414 /* We reserved the ifindex, this can't fail */
415 WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL));
416
417 dev_base_seq_inc(net);
418}
419
420/* Device list removal
421 * caller must respect a RCU grace period before freeing/reusing dev
422 */
423static void unlist_netdevice(struct net_device *dev)
424{
425 struct netdev_name_node *name_node;
426 struct net *net = dev_net(dev);
427
428 ASSERT_RTNL();
429
430 xa_erase(&net->dev_by_index, dev->ifindex);
431
432 netdev_for_each_altname(dev, name_node)
433 netdev_name_node_del(name_node);
434
435 /* Unlink dev from the device chain */
436 list_del_rcu(&dev->dev_list);
437 netdev_name_node_del(dev->name_node);
438 hlist_del_rcu(&dev->index_hlist);
439
440 dev_base_seq_inc(dev_net(dev));
441}
442
443/*
444 * Our notifier list
445 */
446
447static RAW_NOTIFIER_HEAD(netdev_chain);
448
449/*
450 * Device drivers call our routines to queue packets here. We empty the
451 * queue in the local softnet handler.
452 */
453
454DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data) = {
455 .process_queue_bh_lock = INIT_LOCAL_LOCK(process_queue_bh_lock),
456};
457EXPORT_PER_CPU_SYMBOL(softnet_data);
458
459/* Page_pool has a lockless array/stack to alloc/recycle pages.
460 * PP consumers must pay attention to run APIs in the appropriate context
461 * (e.g. NAPI context).
462 */
463static DEFINE_PER_CPU(struct page_pool *, system_page_pool);
464
465#ifdef CONFIG_LOCKDEP
466/*
467 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
468 * according to dev->type
469 */
470static const unsigned short netdev_lock_type[] = {
471 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
472 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
473 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
474 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
475 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
476 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
477 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
478 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
479 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
480 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
481 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
482 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
483 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
484 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
485 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
486
487static const char *const netdev_lock_name[] = {
488 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
489 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
490 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
491 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
492 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
493 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
494 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
495 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
496 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
497 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
498 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
499 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
500 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
501 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
502 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
503
504static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
505static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
506
507static inline unsigned short netdev_lock_pos(unsigned short dev_type)
508{
509 int i;
510
511 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
512 if (netdev_lock_type[i] == dev_type)
513 return i;
514 /* the last key is used by default */
515 return ARRAY_SIZE(netdev_lock_type) - 1;
516}
517
518static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
519 unsigned short dev_type)
520{
521 int i;
522
523 i = netdev_lock_pos(dev_type);
524 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
525 netdev_lock_name[i]);
526}
527
528static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
529{
530 int i;
531
532 i = netdev_lock_pos(dev->type);
533 lockdep_set_class_and_name(&dev->addr_list_lock,
534 &netdev_addr_lock_key[i],
535 netdev_lock_name[i]);
536}
537#else
538static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
539 unsigned short dev_type)
540{
541}
542
543static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
544{
545}
546#endif
547
548/*******************************************************************************
549 *
550 * Protocol management and registration routines
551 *
552 *******************************************************************************/
553
554
555/*
556 * Add a protocol ID to the list. Now that the input handler is
557 * smarter we can dispense with all the messy stuff that used to be
558 * here.
559 *
560 * BEWARE!!! Protocol handlers, mangling input packets,
561 * MUST BE last in hash buckets and checking protocol handlers
562 * MUST start from promiscuous ptype_all chain in net_bh.
563 * It is true now, do not change it.
564 * Explanation follows: if protocol handler, mangling packet, will
565 * be the first on list, it is not able to sense, that packet
566 * is cloned and should be copied-on-write, so that it will
567 * change it and subsequent readers will get broken packet.
568 * --ANK (980803)
569 */
570
571static inline struct list_head *ptype_head(const struct packet_type *pt)
572{
573 if (pt->type == htons(ETH_P_ALL))
574 return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all;
575 else
576 return pt->dev ? &pt->dev->ptype_specific :
577 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
578}
579
580/**
581 * dev_add_pack - add packet handler
582 * @pt: packet type declaration
583 *
584 * Add a protocol handler to the networking stack. The passed &packet_type
585 * is linked into kernel lists and may not be freed until it has been
586 * removed from the kernel lists.
587 *
588 * This call does not sleep therefore it can not
589 * guarantee all CPU's that are in middle of receiving packets
590 * will see the new packet type (until the next received packet).
591 */
592
593void dev_add_pack(struct packet_type *pt)
594{
595 struct list_head *head = ptype_head(pt);
596
597 spin_lock(&ptype_lock);
598 list_add_rcu(&pt->list, head);
599 spin_unlock(&ptype_lock);
600}
601EXPORT_SYMBOL(dev_add_pack);
602
603/**
604 * __dev_remove_pack - remove packet handler
605 * @pt: packet type declaration
606 *
607 * Remove a protocol handler that was previously added to the kernel
608 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
609 * from the kernel lists and can be freed or reused once this function
610 * returns.
611 *
612 * The packet type might still be in use by receivers
613 * and must not be freed until after all the CPU's have gone
614 * through a quiescent state.
615 */
616void __dev_remove_pack(struct packet_type *pt)
617{
618 struct list_head *head = ptype_head(pt);
619 struct packet_type *pt1;
620
621 spin_lock(&ptype_lock);
622
623 list_for_each_entry(pt1, head, list) {
624 if (pt == pt1) {
625 list_del_rcu(&pt->list);
626 goto out;
627 }
628 }
629
630 pr_warn("dev_remove_pack: %p not found\n", pt);
631out:
632 spin_unlock(&ptype_lock);
633}
634EXPORT_SYMBOL(__dev_remove_pack);
635
636/**
637 * dev_remove_pack - remove packet handler
638 * @pt: packet type declaration
639 *
640 * Remove a protocol handler that was previously added to the kernel
641 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
642 * from the kernel lists and can be freed or reused once this function
643 * returns.
644 *
645 * This call sleeps to guarantee that no CPU is looking at the packet
646 * type after return.
647 */
648void dev_remove_pack(struct packet_type *pt)
649{
650 __dev_remove_pack(pt);
651
652 synchronize_net();
653}
654EXPORT_SYMBOL(dev_remove_pack);
655
656
657/*******************************************************************************
658 *
659 * Device Interface Subroutines
660 *
661 *******************************************************************************/
662
663/**
664 * dev_get_iflink - get 'iflink' value of a interface
665 * @dev: targeted interface
666 *
667 * Indicates the ifindex the interface is linked to.
668 * Physical interfaces have the same 'ifindex' and 'iflink' values.
669 */
670
671int dev_get_iflink(const struct net_device *dev)
672{
673 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
674 return dev->netdev_ops->ndo_get_iflink(dev);
675
676 return READ_ONCE(dev->ifindex);
677}
678EXPORT_SYMBOL(dev_get_iflink);
679
680/**
681 * dev_fill_metadata_dst - Retrieve tunnel egress information.
682 * @dev: targeted interface
683 * @skb: The packet.
684 *
685 * For better visibility of tunnel traffic OVS needs to retrieve
686 * egress tunnel information for a packet. Following API allows
687 * user to get this info.
688 */
689int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
690{
691 struct ip_tunnel_info *info;
692
693 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
694 return -EINVAL;
695
696 info = skb_tunnel_info_unclone(skb);
697 if (!info)
698 return -ENOMEM;
699 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
700 return -EINVAL;
701
702 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
703}
704EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
705
706static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
707{
708 int k = stack->num_paths++;
709
710 if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
711 return NULL;
712
713 return &stack->path[k];
714}
715
716int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
717 struct net_device_path_stack *stack)
718{
719 const struct net_device *last_dev;
720 struct net_device_path_ctx ctx = {
721 .dev = dev,
722 };
723 struct net_device_path *path;
724 int ret = 0;
725
726 memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
727 stack->num_paths = 0;
728 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
729 last_dev = ctx.dev;
730 path = dev_fwd_path(stack);
731 if (!path)
732 return -1;
733
734 memset(path, 0, sizeof(struct net_device_path));
735 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
736 if (ret < 0)
737 return -1;
738
739 if (WARN_ON_ONCE(last_dev == ctx.dev))
740 return -1;
741 }
742
743 if (!ctx.dev)
744 return ret;
745
746 path = dev_fwd_path(stack);
747 if (!path)
748 return -1;
749 path->type = DEV_PATH_ETHERNET;
750 path->dev = ctx.dev;
751
752 return ret;
753}
754EXPORT_SYMBOL_GPL(dev_fill_forward_path);
755
756/* must be called under rcu_read_lock(), as we dont take a reference */
757static struct napi_struct *napi_by_id(unsigned int napi_id)
758{
759 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
760 struct napi_struct *napi;
761
762 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
763 if (napi->napi_id == napi_id)
764 return napi;
765
766 return NULL;
767}
768
769/* must be called under rcu_read_lock(), as we dont take a reference */
770struct napi_struct *netdev_napi_by_id(struct net *net, unsigned int napi_id)
771{
772 struct napi_struct *napi;
773
774 napi = napi_by_id(napi_id);
775 if (!napi)
776 return NULL;
777
778 if (WARN_ON_ONCE(!napi->dev))
779 return NULL;
780 if (!net_eq(net, dev_net(napi->dev)))
781 return NULL;
782
783 return napi;
784}
785
786/**
787 * __dev_get_by_name - find a device by its name
788 * @net: the applicable net namespace
789 * @name: name to find
790 *
791 * Find an interface by name. Must be called under RTNL semaphore.
792 * If the name is found a pointer to the device is returned.
793 * If the name is not found then %NULL is returned. The
794 * reference counters are not incremented so the caller must be
795 * careful with locks.
796 */
797
798struct net_device *__dev_get_by_name(struct net *net, const char *name)
799{
800 struct netdev_name_node *node_name;
801
802 node_name = netdev_name_node_lookup(net, name);
803 return node_name ? node_name->dev : NULL;
804}
805EXPORT_SYMBOL(__dev_get_by_name);
806
807/**
808 * dev_get_by_name_rcu - find a device by its name
809 * @net: the applicable net namespace
810 * @name: name to find
811 *
812 * Find an interface by name.
813 * If the name is found a pointer to the device is returned.
814 * If the name is not found then %NULL is returned.
815 * The reference counters are not incremented so the caller must be
816 * careful with locks. The caller must hold RCU lock.
817 */
818
819struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
820{
821 struct netdev_name_node *node_name;
822
823 node_name = netdev_name_node_lookup_rcu(net, name);
824 return node_name ? node_name->dev : NULL;
825}
826EXPORT_SYMBOL(dev_get_by_name_rcu);
827
828/* Deprecated for new users, call netdev_get_by_name() instead */
829struct net_device *dev_get_by_name(struct net *net, const char *name)
830{
831 struct net_device *dev;
832
833 rcu_read_lock();
834 dev = dev_get_by_name_rcu(net, name);
835 dev_hold(dev);
836 rcu_read_unlock();
837 return dev;
838}
839EXPORT_SYMBOL(dev_get_by_name);
840
841/**
842 * netdev_get_by_name() - find a device by its name
843 * @net: the applicable net namespace
844 * @name: name to find
845 * @tracker: tracking object for the acquired reference
846 * @gfp: allocation flags for the tracker
847 *
848 * Find an interface by name. This can be called from any
849 * context and does its own locking. The returned handle has
850 * the usage count incremented and the caller must use netdev_put() to
851 * release it when it is no longer needed. %NULL is returned if no
852 * matching device is found.
853 */
854struct net_device *netdev_get_by_name(struct net *net, const char *name,
855 netdevice_tracker *tracker, gfp_t gfp)
856{
857 struct net_device *dev;
858
859 dev = dev_get_by_name(net, name);
860 if (dev)
861 netdev_tracker_alloc(dev, tracker, gfp);
862 return dev;
863}
864EXPORT_SYMBOL(netdev_get_by_name);
865
866/**
867 * __dev_get_by_index - find a device by its ifindex
868 * @net: the applicable net namespace
869 * @ifindex: index of device
870 *
871 * Search for an interface by index. Returns %NULL if the device
872 * is not found or a pointer to the device. The device has not
873 * had its reference counter increased so the caller must be careful
874 * about locking. The caller must hold the RTNL semaphore.
875 */
876
877struct net_device *__dev_get_by_index(struct net *net, int ifindex)
878{
879 struct net_device *dev;
880 struct hlist_head *head = dev_index_hash(net, ifindex);
881
882 hlist_for_each_entry(dev, head, index_hlist)
883 if (dev->ifindex == ifindex)
884 return dev;
885
886 return NULL;
887}
888EXPORT_SYMBOL(__dev_get_by_index);
889
890/**
891 * dev_get_by_index_rcu - find a device by its ifindex
892 * @net: the applicable net namespace
893 * @ifindex: index of device
894 *
895 * Search for an interface by index. Returns %NULL if the device
896 * is not found or a pointer to the device. The device has not
897 * had its reference counter increased so the caller must be careful
898 * about locking. The caller must hold RCU lock.
899 */
900
901struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
902{
903 struct net_device *dev;
904 struct hlist_head *head = dev_index_hash(net, ifindex);
905
906 hlist_for_each_entry_rcu(dev, head, index_hlist)
907 if (dev->ifindex == ifindex)
908 return dev;
909
910 return NULL;
911}
912EXPORT_SYMBOL(dev_get_by_index_rcu);
913
914/* Deprecated for new users, call netdev_get_by_index() instead */
915struct net_device *dev_get_by_index(struct net *net, int ifindex)
916{
917 struct net_device *dev;
918
919 rcu_read_lock();
920 dev = dev_get_by_index_rcu(net, ifindex);
921 dev_hold(dev);
922 rcu_read_unlock();
923 return dev;
924}
925EXPORT_SYMBOL(dev_get_by_index);
926
927/**
928 * netdev_get_by_index() - find a device by its ifindex
929 * @net: the applicable net namespace
930 * @ifindex: index of device
931 * @tracker: tracking object for the acquired reference
932 * @gfp: allocation flags for the tracker
933 *
934 * Search for an interface by index. Returns NULL if the device
935 * is not found or a pointer to the device. The device returned has
936 * had a reference added and the pointer is safe until the user calls
937 * netdev_put() to indicate they have finished with it.
938 */
939struct net_device *netdev_get_by_index(struct net *net, int ifindex,
940 netdevice_tracker *tracker, gfp_t gfp)
941{
942 struct net_device *dev;
943
944 dev = dev_get_by_index(net, ifindex);
945 if (dev)
946 netdev_tracker_alloc(dev, tracker, gfp);
947 return dev;
948}
949EXPORT_SYMBOL(netdev_get_by_index);
950
951/**
952 * dev_get_by_napi_id - find a device by napi_id
953 * @napi_id: ID of the NAPI struct
954 *
955 * Search for an interface by NAPI ID. Returns %NULL if the device
956 * is not found or a pointer to the device. The device has not had
957 * its reference counter increased so the caller must be careful
958 * about locking. The caller must hold RCU lock.
959 */
960
961struct net_device *dev_get_by_napi_id(unsigned int napi_id)
962{
963 struct napi_struct *napi;
964
965 WARN_ON_ONCE(!rcu_read_lock_held());
966
967 if (napi_id < MIN_NAPI_ID)
968 return NULL;
969
970 napi = napi_by_id(napi_id);
971
972 return napi ? napi->dev : NULL;
973}
974EXPORT_SYMBOL(dev_get_by_napi_id);
975
976static DEFINE_SEQLOCK(netdev_rename_lock);
977
978void netdev_copy_name(struct net_device *dev, char *name)
979{
980 unsigned int seq;
981
982 do {
983 seq = read_seqbegin(&netdev_rename_lock);
984 strscpy(name, dev->name, IFNAMSIZ);
985 } while (read_seqretry(&netdev_rename_lock, seq));
986}
987
988/**
989 * netdev_get_name - get a netdevice name, knowing its ifindex.
990 * @net: network namespace
991 * @name: a pointer to the buffer where the name will be stored.
992 * @ifindex: the ifindex of the interface to get the name from.
993 */
994int netdev_get_name(struct net *net, char *name, int ifindex)
995{
996 struct net_device *dev;
997 int ret;
998
999 rcu_read_lock();
1000
1001 dev = dev_get_by_index_rcu(net, ifindex);
1002 if (!dev) {
1003 ret = -ENODEV;
1004 goto out;
1005 }
1006
1007 netdev_copy_name(dev, name);
1008
1009 ret = 0;
1010out:
1011 rcu_read_unlock();
1012 return ret;
1013}
1014
1015static bool dev_addr_cmp(struct net_device *dev, unsigned short type,
1016 const char *ha)
1017{
1018 return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len);
1019}
1020
1021/**
1022 * dev_getbyhwaddr_rcu - find a device by its hardware address
1023 * @net: the applicable net namespace
1024 * @type: media type of device
1025 * @ha: hardware address
1026 *
1027 * Search for an interface by MAC address. Returns NULL if the device
1028 * is not found or a pointer to the device.
1029 * The caller must hold RCU.
1030 * The returned device has not had its ref count increased
1031 * and the caller must therefore be careful about locking
1032 *
1033 */
1034
1035struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
1036 const char *ha)
1037{
1038 struct net_device *dev;
1039
1040 for_each_netdev_rcu(net, dev)
1041 if (dev_addr_cmp(dev, type, ha))
1042 return dev;
1043
1044 return NULL;
1045}
1046EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
1047
1048/**
1049 * dev_getbyhwaddr() - find a device by its hardware address
1050 * @net: the applicable net namespace
1051 * @type: media type of device
1052 * @ha: hardware address
1053 *
1054 * Similar to dev_getbyhwaddr_rcu(), but the owner needs to hold
1055 * rtnl_lock.
1056 *
1057 * Context: rtnl_lock() must be held.
1058 * Return: pointer to the net_device, or NULL if not found
1059 */
1060struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type,
1061 const char *ha)
1062{
1063 struct net_device *dev;
1064
1065 ASSERT_RTNL();
1066 for_each_netdev(net, dev)
1067 if (dev_addr_cmp(dev, type, ha))
1068 return dev;
1069
1070 return NULL;
1071}
1072EXPORT_SYMBOL(dev_getbyhwaddr);
1073
1074struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
1075{
1076 struct net_device *dev, *ret = NULL;
1077
1078 rcu_read_lock();
1079 for_each_netdev_rcu(net, dev)
1080 if (dev->type == type) {
1081 dev_hold(dev);
1082 ret = dev;
1083 break;
1084 }
1085 rcu_read_unlock();
1086 return ret;
1087}
1088EXPORT_SYMBOL(dev_getfirstbyhwtype);
1089
1090/**
1091 * __dev_get_by_flags - find any device with given flags
1092 * @net: the applicable net namespace
1093 * @if_flags: IFF_* values
1094 * @mask: bitmask of bits in if_flags to check
1095 *
1096 * Search for any interface with the given flags. Returns NULL if a device
1097 * is not found or a pointer to the device. Must be called inside
1098 * rtnl_lock(), and result refcount is unchanged.
1099 */
1100
1101struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1102 unsigned short mask)
1103{
1104 struct net_device *dev, *ret;
1105
1106 ASSERT_RTNL();
1107
1108 ret = NULL;
1109 for_each_netdev(net, dev) {
1110 if (((dev->flags ^ if_flags) & mask) == 0) {
1111 ret = dev;
1112 break;
1113 }
1114 }
1115 return ret;
1116}
1117EXPORT_SYMBOL(__dev_get_by_flags);
1118
1119/**
1120 * dev_valid_name - check if name is okay for network device
1121 * @name: name string
1122 *
1123 * Network device names need to be valid file names to
1124 * allow sysfs to work. We also disallow any kind of
1125 * whitespace.
1126 */
1127bool dev_valid_name(const char *name)
1128{
1129 if (*name == '\0')
1130 return false;
1131 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1132 return false;
1133 if (!strcmp(name, ".") || !strcmp(name, ".."))
1134 return false;
1135
1136 while (*name) {
1137 if (*name == '/' || *name == ':' || isspace(*name))
1138 return false;
1139 name++;
1140 }
1141 return true;
1142}
1143EXPORT_SYMBOL(dev_valid_name);
1144
1145/**
1146 * __dev_alloc_name - allocate a name for a device
1147 * @net: network namespace to allocate the device name in
1148 * @name: name format string
1149 * @res: result name string
1150 *
1151 * Passed a format string - eg "lt%d" it will try and find a suitable
1152 * id. It scans list of devices to build up a free map, then chooses
1153 * the first empty slot. The caller must hold the dev_base or rtnl lock
1154 * while allocating the name and adding the device in order to avoid
1155 * duplicates.
1156 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1157 * Returns the number of the unit assigned or a negative errno code.
1158 */
1159
1160static int __dev_alloc_name(struct net *net, const char *name, char *res)
1161{
1162 int i = 0;
1163 const char *p;
1164 const int max_netdevices = 8*PAGE_SIZE;
1165 unsigned long *inuse;
1166 struct net_device *d;
1167 char buf[IFNAMSIZ];
1168
1169 /* Verify the string as this thing may have come from the user.
1170 * There must be one "%d" and no other "%" characters.
1171 */
1172 p = strchr(name, '%');
1173 if (!p || p[1] != 'd' || strchr(p + 2, '%'))
1174 return -EINVAL;
1175
1176 /* Use one page as a bit array of possible slots */
1177 inuse = bitmap_zalloc(max_netdevices, GFP_ATOMIC);
1178 if (!inuse)
1179 return -ENOMEM;
1180
1181 for_each_netdev(net, d) {
1182 struct netdev_name_node *name_node;
1183
1184 netdev_for_each_altname(d, name_node) {
1185 if (!sscanf(name_node->name, name, &i))
1186 continue;
1187 if (i < 0 || i >= max_netdevices)
1188 continue;
1189
1190 /* avoid cases where sscanf is not exact inverse of printf */
1191 snprintf(buf, IFNAMSIZ, name, i);
1192 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1193 __set_bit(i, inuse);
1194 }
1195 if (!sscanf(d->name, name, &i))
1196 continue;
1197 if (i < 0 || i >= max_netdevices)
1198 continue;
1199
1200 /* avoid cases where sscanf is not exact inverse of printf */
1201 snprintf(buf, IFNAMSIZ, name, i);
1202 if (!strncmp(buf, d->name, IFNAMSIZ))
1203 __set_bit(i, inuse);
1204 }
1205
1206 i = find_first_zero_bit(inuse, max_netdevices);
1207 bitmap_free(inuse);
1208 if (i == max_netdevices)
1209 return -ENFILE;
1210
1211 /* 'res' and 'name' could overlap, use 'buf' as an intermediate buffer */
1212 strscpy(buf, name, IFNAMSIZ);
1213 snprintf(res, IFNAMSIZ, buf, i);
1214 return i;
1215}
1216
1217/* Returns negative errno or allocated unit id (see __dev_alloc_name()) */
1218static int dev_prep_valid_name(struct net *net, struct net_device *dev,
1219 const char *want_name, char *out_name,
1220 int dup_errno)
1221{
1222 if (!dev_valid_name(want_name))
1223 return -EINVAL;
1224
1225 if (strchr(want_name, '%'))
1226 return __dev_alloc_name(net, want_name, out_name);
1227
1228 if (netdev_name_in_use(net, want_name))
1229 return -dup_errno;
1230 if (out_name != want_name)
1231 strscpy(out_name, want_name, IFNAMSIZ);
1232 return 0;
1233}
1234
1235/**
1236 * dev_alloc_name - allocate a name for a device
1237 * @dev: device
1238 * @name: name format string
1239 *
1240 * Passed a format string - eg "lt%d" it will try and find a suitable
1241 * id. It scans list of devices to build up a free map, then chooses
1242 * the first empty slot. The caller must hold the dev_base or rtnl lock
1243 * while allocating the name and adding the device in order to avoid
1244 * duplicates.
1245 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1246 * Returns the number of the unit assigned or a negative errno code.
1247 */
1248
1249int dev_alloc_name(struct net_device *dev, const char *name)
1250{
1251 return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE);
1252}
1253EXPORT_SYMBOL(dev_alloc_name);
1254
1255static int dev_get_valid_name(struct net *net, struct net_device *dev,
1256 const char *name)
1257{
1258 int ret;
1259
1260 ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST);
1261 return ret < 0 ? ret : 0;
1262}
1263
1264/**
1265 * dev_change_name - change name of a device
1266 * @dev: device
1267 * @newname: name (or format string) must be at least IFNAMSIZ
1268 *
1269 * Change name of a device, can pass format strings "eth%d".
1270 * for wildcarding.
1271 */
1272int dev_change_name(struct net_device *dev, const char *newname)
1273{
1274 unsigned char old_assign_type;
1275 char oldname[IFNAMSIZ];
1276 int err = 0;
1277 int ret;
1278 struct net *net;
1279
1280 ASSERT_RTNL();
1281 BUG_ON(!dev_net(dev));
1282
1283 net = dev_net(dev);
1284
1285 down_write(&devnet_rename_sem);
1286
1287 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1288 up_write(&devnet_rename_sem);
1289 return 0;
1290 }
1291
1292 memcpy(oldname, dev->name, IFNAMSIZ);
1293
1294 write_seqlock_bh(&netdev_rename_lock);
1295 err = dev_get_valid_name(net, dev, newname);
1296 write_sequnlock_bh(&netdev_rename_lock);
1297
1298 if (err < 0) {
1299 up_write(&devnet_rename_sem);
1300 return err;
1301 }
1302
1303 if (oldname[0] && !strchr(oldname, '%'))
1304 netdev_info(dev, "renamed from %s%s\n", oldname,
1305 dev->flags & IFF_UP ? " (while UP)" : "");
1306
1307 old_assign_type = dev->name_assign_type;
1308 WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED);
1309
1310rollback:
1311 ret = device_rename(&dev->dev, dev->name);
1312 if (ret) {
1313 write_seqlock_bh(&netdev_rename_lock);
1314 memcpy(dev->name, oldname, IFNAMSIZ);
1315 write_sequnlock_bh(&netdev_rename_lock);
1316 WRITE_ONCE(dev->name_assign_type, old_assign_type);
1317 up_write(&devnet_rename_sem);
1318 return ret;
1319 }
1320
1321 up_write(&devnet_rename_sem);
1322
1323 netdev_adjacent_rename_links(dev, oldname);
1324
1325 netdev_name_node_del(dev->name_node);
1326
1327 synchronize_net();
1328
1329 netdev_name_node_add(net, dev->name_node);
1330
1331 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1332 ret = notifier_to_errno(ret);
1333
1334 if (ret) {
1335 /* err >= 0 after dev_alloc_name() or stores the first errno */
1336 if (err >= 0) {
1337 err = ret;
1338 down_write(&devnet_rename_sem);
1339 write_seqlock_bh(&netdev_rename_lock);
1340 memcpy(dev->name, oldname, IFNAMSIZ);
1341 write_sequnlock_bh(&netdev_rename_lock);
1342 memcpy(oldname, newname, IFNAMSIZ);
1343 WRITE_ONCE(dev->name_assign_type, old_assign_type);
1344 old_assign_type = NET_NAME_RENAMED;
1345 goto rollback;
1346 } else {
1347 netdev_err(dev, "name change rollback failed: %d\n",
1348 ret);
1349 }
1350 }
1351
1352 return err;
1353}
1354
1355/**
1356 * dev_set_alias - change ifalias of a device
1357 * @dev: device
1358 * @alias: name up to IFALIASZ
1359 * @len: limit of bytes to copy from info
1360 *
1361 * Set ifalias for a device,
1362 */
1363int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1364{
1365 struct dev_ifalias *new_alias = NULL;
1366
1367 if (len >= IFALIASZ)
1368 return -EINVAL;
1369
1370 if (len) {
1371 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1372 if (!new_alias)
1373 return -ENOMEM;
1374
1375 memcpy(new_alias->ifalias, alias, len);
1376 new_alias->ifalias[len] = 0;
1377 }
1378
1379 mutex_lock(&ifalias_mutex);
1380 new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1381 mutex_is_locked(&ifalias_mutex));
1382 mutex_unlock(&ifalias_mutex);
1383
1384 if (new_alias)
1385 kfree_rcu(new_alias, rcuhead);
1386
1387 return len;
1388}
1389EXPORT_SYMBOL(dev_set_alias);
1390
1391/**
1392 * dev_get_alias - get ifalias of a device
1393 * @dev: device
1394 * @name: buffer to store name of ifalias
1395 * @len: size of buffer
1396 *
1397 * get ifalias for a device. Caller must make sure dev cannot go
1398 * away, e.g. rcu read lock or own a reference count to device.
1399 */
1400int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1401{
1402 const struct dev_ifalias *alias;
1403 int ret = 0;
1404
1405 rcu_read_lock();
1406 alias = rcu_dereference(dev->ifalias);
1407 if (alias)
1408 ret = snprintf(name, len, "%s", alias->ifalias);
1409 rcu_read_unlock();
1410
1411 return ret;
1412}
1413
1414/**
1415 * netdev_features_change - device changes features
1416 * @dev: device to cause notification
1417 *
1418 * Called to indicate a device has changed features.
1419 */
1420void netdev_features_change(struct net_device *dev)
1421{
1422 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1423}
1424EXPORT_SYMBOL(netdev_features_change);
1425
1426/**
1427 * netdev_state_change - device changes state
1428 * @dev: device to cause notification
1429 *
1430 * Called to indicate a device has changed state. This function calls
1431 * the notifier chains for netdev_chain and sends a NEWLINK message
1432 * to the routing socket.
1433 */
1434void netdev_state_change(struct net_device *dev)
1435{
1436 if (dev->flags & IFF_UP) {
1437 struct netdev_notifier_change_info change_info = {
1438 .info.dev = dev,
1439 };
1440
1441 call_netdevice_notifiers_info(NETDEV_CHANGE,
1442 &change_info.info);
1443 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL);
1444 }
1445}
1446EXPORT_SYMBOL(netdev_state_change);
1447
1448/**
1449 * __netdev_notify_peers - notify network peers about existence of @dev,
1450 * to be called when rtnl lock is already held.
1451 * @dev: network device
1452 *
1453 * Generate traffic such that interested network peers are aware of
1454 * @dev, such as by generating a gratuitous ARP. This may be used when
1455 * a device wants to inform the rest of the network about some sort of
1456 * reconfiguration such as a failover event or virtual machine
1457 * migration.
1458 */
1459void __netdev_notify_peers(struct net_device *dev)
1460{
1461 ASSERT_RTNL();
1462 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1463 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1464}
1465EXPORT_SYMBOL(__netdev_notify_peers);
1466
1467/**
1468 * netdev_notify_peers - notify network peers about existence of @dev
1469 * @dev: network device
1470 *
1471 * Generate traffic such that interested network peers are aware of
1472 * @dev, such as by generating a gratuitous ARP. This may be used when
1473 * a device wants to inform the rest of the network about some sort of
1474 * reconfiguration such as a failover event or virtual machine
1475 * migration.
1476 */
1477void netdev_notify_peers(struct net_device *dev)
1478{
1479 rtnl_lock();
1480 __netdev_notify_peers(dev);
1481 rtnl_unlock();
1482}
1483EXPORT_SYMBOL(netdev_notify_peers);
1484
1485static int napi_threaded_poll(void *data);
1486
1487static int napi_kthread_create(struct napi_struct *n)
1488{
1489 int err = 0;
1490
1491 /* Create and wake up the kthread once to put it in
1492 * TASK_INTERRUPTIBLE mode to avoid the blocked task
1493 * warning and work with loadavg.
1494 */
1495 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1496 n->dev->name, n->napi_id);
1497 if (IS_ERR(n->thread)) {
1498 err = PTR_ERR(n->thread);
1499 pr_err("kthread_run failed with err %d\n", err);
1500 n->thread = NULL;
1501 }
1502
1503 return err;
1504}
1505
1506static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1507{
1508 const struct net_device_ops *ops = dev->netdev_ops;
1509 int ret;
1510
1511 ASSERT_RTNL();
1512 dev_addr_check(dev);
1513
1514 if (!netif_device_present(dev)) {
1515 /* may be detached because parent is runtime-suspended */
1516 if (dev->dev.parent)
1517 pm_runtime_resume(dev->dev.parent);
1518 if (!netif_device_present(dev))
1519 return -ENODEV;
1520 }
1521
1522 /* Block netpoll from trying to do any rx path servicing.
1523 * If we don't do this there is a chance ndo_poll_controller
1524 * or ndo_poll may be running while we open the device
1525 */
1526 netpoll_poll_disable(dev);
1527
1528 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1529 ret = notifier_to_errno(ret);
1530 if (ret)
1531 return ret;
1532
1533 set_bit(__LINK_STATE_START, &dev->state);
1534
1535 if (ops->ndo_validate_addr)
1536 ret = ops->ndo_validate_addr(dev);
1537
1538 if (!ret && ops->ndo_open)
1539 ret = ops->ndo_open(dev);
1540
1541 netpoll_poll_enable(dev);
1542
1543 if (ret)
1544 clear_bit(__LINK_STATE_START, &dev->state);
1545 else {
1546 dev->flags |= IFF_UP;
1547 dev_set_rx_mode(dev);
1548 dev_activate(dev);
1549 add_device_randomness(dev->dev_addr, dev->addr_len);
1550 }
1551
1552 return ret;
1553}
1554
1555/**
1556 * dev_open - prepare an interface for use.
1557 * @dev: device to open
1558 * @extack: netlink extended ack
1559 *
1560 * Takes a device from down to up state. The device's private open
1561 * function is invoked and then the multicast lists are loaded. Finally
1562 * the device is moved into the up state and a %NETDEV_UP message is
1563 * sent to the netdev notifier chain.
1564 *
1565 * Calling this function on an active interface is a nop. On a failure
1566 * a negative errno code is returned.
1567 */
1568int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1569{
1570 int ret;
1571
1572 if (dev->flags & IFF_UP)
1573 return 0;
1574
1575 ret = __dev_open(dev, extack);
1576 if (ret < 0)
1577 return ret;
1578
1579 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1580 call_netdevice_notifiers(NETDEV_UP, dev);
1581
1582 return ret;
1583}
1584EXPORT_SYMBOL(dev_open);
1585
1586static void __dev_close_many(struct list_head *head)
1587{
1588 struct net_device *dev;
1589
1590 ASSERT_RTNL();
1591 might_sleep();
1592
1593 list_for_each_entry(dev, head, close_list) {
1594 /* Temporarily disable netpoll until the interface is down */
1595 netpoll_poll_disable(dev);
1596
1597 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1598
1599 clear_bit(__LINK_STATE_START, &dev->state);
1600
1601 /* Synchronize to scheduled poll. We cannot touch poll list, it
1602 * can be even on different cpu. So just clear netif_running().
1603 *
1604 * dev->stop() will invoke napi_disable() on all of it's
1605 * napi_struct instances on this device.
1606 */
1607 smp_mb__after_atomic(); /* Commit netif_running(). */
1608 }
1609
1610 dev_deactivate_many(head);
1611
1612 list_for_each_entry(dev, head, close_list) {
1613 const struct net_device_ops *ops = dev->netdev_ops;
1614
1615 /*
1616 * Call the device specific close. This cannot fail.
1617 * Only if device is UP
1618 *
1619 * We allow it to be called even after a DETACH hot-plug
1620 * event.
1621 */
1622 if (ops->ndo_stop)
1623 ops->ndo_stop(dev);
1624
1625 dev->flags &= ~IFF_UP;
1626 netpoll_poll_enable(dev);
1627 }
1628}
1629
1630static void __dev_close(struct net_device *dev)
1631{
1632 LIST_HEAD(single);
1633
1634 list_add(&dev->close_list, &single);
1635 __dev_close_many(&single);
1636 list_del(&single);
1637}
1638
1639void dev_close_many(struct list_head *head, bool unlink)
1640{
1641 struct net_device *dev, *tmp;
1642
1643 /* Remove the devices that don't need to be closed */
1644 list_for_each_entry_safe(dev, tmp, head, close_list)
1645 if (!(dev->flags & IFF_UP))
1646 list_del_init(&dev->close_list);
1647
1648 __dev_close_many(head);
1649
1650 list_for_each_entry_safe(dev, tmp, head, close_list) {
1651 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1652 call_netdevice_notifiers(NETDEV_DOWN, dev);
1653 if (unlink)
1654 list_del_init(&dev->close_list);
1655 }
1656}
1657EXPORT_SYMBOL(dev_close_many);
1658
1659/**
1660 * dev_close - shutdown an interface.
1661 * @dev: device to shutdown
1662 *
1663 * This function moves an active device into down state. A
1664 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1665 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1666 * chain.
1667 */
1668void dev_close(struct net_device *dev)
1669{
1670 if (dev->flags & IFF_UP) {
1671 LIST_HEAD(single);
1672
1673 list_add(&dev->close_list, &single);
1674 dev_close_many(&single, true);
1675 list_del(&single);
1676 }
1677}
1678EXPORT_SYMBOL(dev_close);
1679
1680
1681/**
1682 * dev_disable_lro - disable Large Receive Offload on a device
1683 * @dev: device
1684 *
1685 * Disable Large Receive Offload (LRO) on a net device. Must be
1686 * called under RTNL. This is needed if received packets may be
1687 * forwarded to another interface.
1688 */
1689void dev_disable_lro(struct net_device *dev)
1690{
1691 struct net_device *lower_dev;
1692 struct list_head *iter;
1693
1694 dev->wanted_features &= ~NETIF_F_LRO;
1695 netdev_update_features(dev);
1696
1697 if (unlikely(dev->features & NETIF_F_LRO))
1698 netdev_WARN(dev, "failed to disable LRO!\n");
1699
1700 netdev_for_each_lower_dev(dev, lower_dev, iter)
1701 dev_disable_lro(lower_dev);
1702}
1703EXPORT_SYMBOL(dev_disable_lro);
1704
1705/**
1706 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1707 * @dev: device
1708 *
1709 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1710 * called under RTNL. This is needed if Generic XDP is installed on
1711 * the device.
1712 */
1713static void dev_disable_gro_hw(struct net_device *dev)
1714{
1715 dev->wanted_features &= ~NETIF_F_GRO_HW;
1716 netdev_update_features(dev);
1717
1718 if (unlikely(dev->features & NETIF_F_GRO_HW))
1719 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1720}
1721
1722const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1723{
1724#define N(val) \
1725 case NETDEV_##val: \
1726 return "NETDEV_" __stringify(val);
1727 switch (cmd) {
1728 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1729 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1730 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1731 N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN)
1732 N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA)
1733 N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE)
1734 N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1735 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1736 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1737 N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1738 N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1739 N(XDP_FEAT_CHANGE)
1740 }
1741#undef N
1742 return "UNKNOWN_NETDEV_EVENT";
1743}
1744EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1745
1746static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1747 struct net_device *dev)
1748{
1749 struct netdev_notifier_info info = {
1750 .dev = dev,
1751 };
1752
1753 return nb->notifier_call(nb, val, &info);
1754}
1755
1756static int call_netdevice_register_notifiers(struct notifier_block *nb,
1757 struct net_device *dev)
1758{
1759 int err;
1760
1761 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1762 err = notifier_to_errno(err);
1763 if (err)
1764 return err;
1765
1766 if (!(dev->flags & IFF_UP))
1767 return 0;
1768
1769 call_netdevice_notifier(nb, NETDEV_UP, dev);
1770 return 0;
1771}
1772
1773static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1774 struct net_device *dev)
1775{
1776 if (dev->flags & IFF_UP) {
1777 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1778 dev);
1779 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1780 }
1781 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1782}
1783
1784static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1785 struct net *net)
1786{
1787 struct net_device *dev;
1788 int err;
1789
1790 for_each_netdev(net, dev) {
1791 err = call_netdevice_register_notifiers(nb, dev);
1792 if (err)
1793 goto rollback;
1794 }
1795 return 0;
1796
1797rollback:
1798 for_each_netdev_continue_reverse(net, dev)
1799 call_netdevice_unregister_notifiers(nb, dev);
1800 return err;
1801}
1802
1803static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1804 struct net *net)
1805{
1806 struct net_device *dev;
1807
1808 for_each_netdev(net, dev)
1809 call_netdevice_unregister_notifiers(nb, dev);
1810}
1811
1812static int dev_boot_phase = 1;
1813
1814/**
1815 * register_netdevice_notifier - register a network notifier block
1816 * @nb: notifier
1817 *
1818 * Register a notifier to be called when network device events occur.
1819 * The notifier passed is linked into the kernel structures and must
1820 * not be reused until it has been unregistered. A negative errno code
1821 * is returned on a failure.
1822 *
1823 * When registered all registration and up events are replayed
1824 * to the new notifier to allow device to have a race free
1825 * view of the network device list.
1826 */
1827
1828int register_netdevice_notifier(struct notifier_block *nb)
1829{
1830 struct net *net;
1831 int err;
1832
1833 /* Close race with setup_net() and cleanup_net() */
1834 down_write(&pernet_ops_rwsem);
1835 rtnl_lock();
1836 err = raw_notifier_chain_register(&netdev_chain, nb);
1837 if (err)
1838 goto unlock;
1839 if (dev_boot_phase)
1840 goto unlock;
1841 for_each_net(net) {
1842 err = call_netdevice_register_net_notifiers(nb, net);
1843 if (err)
1844 goto rollback;
1845 }
1846
1847unlock:
1848 rtnl_unlock();
1849 up_write(&pernet_ops_rwsem);
1850 return err;
1851
1852rollback:
1853 for_each_net_continue_reverse(net)
1854 call_netdevice_unregister_net_notifiers(nb, net);
1855
1856 raw_notifier_chain_unregister(&netdev_chain, nb);
1857 goto unlock;
1858}
1859EXPORT_SYMBOL(register_netdevice_notifier);
1860
1861/**
1862 * unregister_netdevice_notifier - unregister a network notifier block
1863 * @nb: notifier
1864 *
1865 * Unregister a notifier previously registered by
1866 * register_netdevice_notifier(). The notifier is unlinked into the
1867 * kernel structures and may then be reused. A negative errno code
1868 * is returned on a failure.
1869 *
1870 * After unregistering unregister and down device events are synthesized
1871 * for all devices on the device list to the removed notifier to remove
1872 * the need for special case cleanup code.
1873 */
1874
1875int unregister_netdevice_notifier(struct notifier_block *nb)
1876{
1877 struct net *net;
1878 int err;
1879
1880 /* Close race with setup_net() and cleanup_net() */
1881 down_write(&pernet_ops_rwsem);
1882 rtnl_lock();
1883 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1884 if (err)
1885 goto unlock;
1886
1887 for_each_net(net)
1888 call_netdevice_unregister_net_notifiers(nb, net);
1889
1890unlock:
1891 rtnl_unlock();
1892 up_write(&pernet_ops_rwsem);
1893 return err;
1894}
1895EXPORT_SYMBOL(unregister_netdevice_notifier);
1896
1897static int __register_netdevice_notifier_net(struct net *net,
1898 struct notifier_block *nb,
1899 bool ignore_call_fail)
1900{
1901 int err;
1902
1903 err = raw_notifier_chain_register(&net->netdev_chain, nb);
1904 if (err)
1905 return err;
1906 if (dev_boot_phase)
1907 return 0;
1908
1909 err = call_netdevice_register_net_notifiers(nb, net);
1910 if (err && !ignore_call_fail)
1911 goto chain_unregister;
1912
1913 return 0;
1914
1915chain_unregister:
1916 raw_notifier_chain_unregister(&net->netdev_chain, nb);
1917 return err;
1918}
1919
1920static int __unregister_netdevice_notifier_net(struct net *net,
1921 struct notifier_block *nb)
1922{
1923 int err;
1924
1925 err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1926 if (err)
1927 return err;
1928
1929 call_netdevice_unregister_net_notifiers(nb, net);
1930 return 0;
1931}
1932
1933/**
1934 * register_netdevice_notifier_net - register a per-netns network notifier block
1935 * @net: network namespace
1936 * @nb: notifier
1937 *
1938 * Register a notifier to be called when network device events occur.
1939 * The notifier passed is linked into the kernel structures and must
1940 * not be reused until it has been unregistered. A negative errno code
1941 * is returned on a failure.
1942 *
1943 * When registered all registration and up events are replayed
1944 * to the new notifier to allow device to have a race free
1945 * view of the network device list.
1946 */
1947
1948int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1949{
1950 int err;
1951
1952 rtnl_lock();
1953 err = __register_netdevice_notifier_net(net, nb, false);
1954 rtnl_unlock();
1955 return err;
1956}
1957EXPORT_SYMBOL(register_netdevice_notifier_net);
1958
1959/**
1960 * unregister_netdevice_notifier_net - unregister a per-netns
1961 * network notifier block
1962 * @net: network namespace
1963 * @nb: notifier
1964 *
1965 * Unregister a notifier previously registered by
1966 * register_netdevice_notifier_net(). The notifier is unlinked from the
1967 * kernel structures and may then be reused. A negative errno code
1968 * is returned on a failure.
1969 *
1970 * After unregistering unregister and down device events are synthesized
1971 * for all devices on the device list to the removed notifier to remove
1972 * the need for special case cleanup code.
1973 */
1974
1975int unregister_netdevice_notifier_net(struct net *net,
1976 struct notifier_block *nb)
1977{
1978 int err;
1979
1980 rtnl_lock();
1981 err = __unregister_netdevice_notifier_net(net, nb);
1982 rtnl_unlock();
1983 return err;
1984}
1985EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1986
1987static void __move_netdevice_notifier_net(struct net *src_net,
1988 struct net *dst_net,
1989 struct notifier_block *nb)
1990{
1991 __unregister_netdevice_notifier_net(src_net, nb);
1992 __register_netdevice_notifier_net(dst_net, nb, true);
1993}
1994
1995int register_netdevice_notifier_dev_net(struct net_device *dev,
1996 struct notifier_block *nb,
1997 struct netdev_net_notifier *nn)
1998{
1999 int err;
2000
2001 rtnl_lock();
2002 err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
2003 if (!err) {
2004 nn->nb = nb;
2005 list_add(&nn->list, &dev->net_notifier_list);
2006 }
2007 rtnl_unlock();
2008 return err;
2009}
2010EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
2011
2012int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2013 struct notifier_block *nb,
2014 struct netdev_net_notifier *nn)
2015{
2016 int err;
2017
2018 rtnl_lock();
2019 list_del(&nn->list);
2020 err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
2021 rtnl_unlock();
2022 return err;
2023}
2024EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
2025
2026static void move_netdevice_notifiers_dev_net(struct net_device *dev,
2027 struct net *net)
2028{
2029 struct netdev_net_notifier *nn;
2030
2031 list_for_each_entry(nn, &dev->net_notifier_list, list)
2032 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb);
2033}
2034
2035/**
2036 * call_netdevice_notifiers_info - call all network notifier blocks
2037 * @val: value passed unmodified to notifier function
2038 * @info: notifier information data
2039 *
2040 * Call all network notifier blocks. Parameters and return value
2041 * are as for raw_notifier_call_chain().
2042 */
2043
2044int call_netdevice_notifiers_info(unsigned long val,
2045 struct netdev_notifier_info *info)
2046{
2047 struct net *net = dev_net(info->dev);
2048 int ret;
2049
2050 ASSERT_RTNL();
2051
2052 /* Run per-netns notifier block chain first, then run the global one.
2053 * Hopefully, one day, the global one is going to be removed after
2054 * all notifier block registrators get converted to be per-netns.
2055 */
2056 ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
2057 if (ret & NOTIFY_STOP_MASK)
2058 return ret;
2059 return raw_notifier_call_chain(&netdev_chain, val, info);
2060}
2061
2062/**
2063 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks
2064 * for and rollback on error
2065 * @val_up: value passed unmodified to notifier function
2066 * @val_down: value passed unmodified to the notifier function when
2067 * recovering from an error on @val_up
2068 * @info: notifier information data
2069 *
2070 * Call all per-netns network notifier blocks, but not notifier blocks on
2071 * the global notifier chain. Parameters and return value are as for
2072 * raw_notifier_call_chain_robust().
2073 */
2074
2075static int
2076call_netdevice_notifiers_info_robust(unsigned long val_up,
2077 unsigned long val_down,
2078 struct netdev_notifier_info *info)
2079{
2080 struct net *net = dev_net(info->dev);
2081
2082 ASSERT_RTNL();
2083
2084 return raw_notifier_call_chain_robust(&net->netdev_chain,
2085 val_up, val_down, info);
2086}
2087
2088static int call_netdevice_notifiers_extack(unsigned long val,
2089 struct net_device *dev,
2090 struct netlink_ext_ack *extack)
2091{
2092 struct netdev_notifier_info info = {
2093 .dev = dev,
2094 .extack = extack,
2095 };
2096
2097 return call_netdevice_notifiers_info(val, &info);
2098}
2099
2100/**
2101 * call_netdevice_notifiers - call all network notifier blocks
2102 * @val: value passed unmodified to notifier function
2103 * @dev: net_device pointer passed unmodified to notifier function
2104 *
2105 * Call all network notifier blocks. Parameters and return value
2106 * are as for raw_notifier_call_chain().
2107 */
2108
2109int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
2110{
2111 return call_netdevice_notifiers_extack(val, dev, NULL);
2112}
2113EXPORT_SYMBOL(call_netdevice_notifiers);
2114
2115/**
2116 * call_netdevice_notifiers_mtu - call all network notifier blocks
2117 * @val: value passed unmodified to notifier function
2118 * @dev: net_device pointer passed unmodified to notifier function
2119 * @arg: additional u32 argument passed to the notifier function
2120 *
2121 * Call all network notifier blocks. Parameters and return value
2122 * are as for raw_notifier_call_chain().
2123 */
2124static int call_netdevice_notifiers_mtu(unsigned long val,
2125 struct net_device *dev, u32 arg)
2126{
2127 struct netdev_notifier_info_ext info = {
2128 .info.dev = dev,
2129 .ext.mtu = arg,
2130 };
2131
2132 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2133
2134 return call_netdevice_notifiers_info(val, &info.info);
2135}
2136
2137#ifdef CONFIG_NET_INGRESS
2138static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2139
2140void net_inc_ingress_queue(void)
2141{
2142 static_branch_inc(&ingress_needed_key);
2143}
2144EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2145
2146void net_dec_ingress_queue(void)
2147{
2148 static_branch_dec(&ingress_needed_key);
2149}
2150EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2151#endif
2152
2153#ifdef CONFIG_NET_EGRESS
2154static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2155
2156void net_inc_egress_queue(void)
2157{
2158 static_branch_inc(&egress_needed_key);
2159}
2160EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2161
2162void net_dec_egress_queue(void)
2163{
2164 static_branch_dec(&egress_needed_key);
2165}
2166EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2167#endif
2168
2169#ifdef CONFIG_NET_CLS_ACT
2170DEFINE_STATIC_KEY_FALSE(tcf_sw_enabled_key);
2171EXPORT_SYMBOL(tcf_sw_enabled_key);
2172#endif
2173
2174DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2175EXPORT_SYMBOL(netstamp_needed_key);
2176#ifdef CONFIG_JUMP_LABEL
2177static atomic_t netstamp_needed_deferred;
2178static atomic_t netstamp_wanted;
2179static void netstamp_clear(struct work_struct *work)
2180{
2181 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2182 int wanted;
2183
2184 wanted = atomic_add_return(deferred, &netstamp_wanted);
2185 if (wanted > 0)
2186 static_branch_enable(&netstamp_needed_key);
2187 else
2188 static_branch_disable(&netstamp_needed_key);
2189}
2190static DECLARE_WORK(netstamp_work, netstamp_clear);
2191#endif
2192
2193void net_enable_timestamp(void)
2194{
2195#ifdef CONFIG_JUMP_LABEL
2196 int wanted = atomic_read(&netstamp_wanted);
2197
2198 while (wanted > 0) {
2199 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1))
2200 return;
2201 }
2202 atomic_inc(&netstamp_needed_deferred);
2203 schedule_work(&netstamp_work);
2204#else
2205 static_branch_inc(&netstamp_needed_key);
2206#endif
2207}
2208EXPORT_SYMBOL(net_enable_timestamp);
2209
2210void net_disable_timestamp(void)
2211{
2212#ifdef CONFIG_JUMP_LABEL
2213 int wanted = atomic_read(&netstamp_wanted);
2214
2215 while (wanted > 1) {
2216 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1))
2217 return;
2218 }
2219 atomic_dec(&netstamp_needed_deferred);
2220 schedule_work(&netstamp_work);
2221#else
2222 static_branch_dec(&netstamp_needed_key);
2223#endif
2224}
2225EXPORT_SYMBOL(net_disable_timestamp);
2226
2227static inline void net_timestamp_set(struct sk_buff *skb)
2228{
2229 skb->tstamp = 0;
2230 skb->tstamp_type = SKB_CLOCK_REALTIME;
2231 if (static_branch_unlikely(&netstamp_needed_key))
2232 skb->tstamp = ktime_get_real();
2233}
2234
2235#define net_timestamp_check(COND, SKB) \
2236 if (static_branch_unlikely(&netstamp_needed_key)) { \
2237 if ((COND) && !(SKB)->tstamp) \
2238 (SKB)->tstamp = ktime_get_real(); \
2239 } \
2240
2241bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2242{
2243 return __is_skb_forwardable(dev, skb, true);
2244}
2245EXPORT_SYMBOL_GPL(is_skb_forwardable);
2246
2247static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2248 bool check_mtu)
2249{
2250 int ret = ____dev_forward_skb(dev, skb, check_mtu);
2251
2252 if (likely(!ret)) {
2253 skb->protocol = eth_type_trans(skb, dev);
2254 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2255 }
2256
2257 return ret;
2258}
2259
2260int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2261{
2262 return __dev_forward_skb2(dev, skb, true);
2263}
2264EXPORT_SYMBOL_GPL(__dev_forward_skb);
2265
2266/**
2267 * dev_forward_skb - loopback an skb to another netif
2268 *
2269 * @dev: destination network device
2270 * @skb: buffer to forward
2271 *
2272 * return values:
2273 * NET_RX_SUCCESS (no congestion)
2274 * NET_RX_DROP (packet was dropped, but freed)
2275 *
2276 * dev_forward_skb can be used for injecting an skb from the
2277 * start_xmit function of one device into the receive queue
2278 * of another device.
2279 *
2280 * The receiving device may be in another namespace, so
2281 * we have to clear all information in the skb that could
2282 * impact namespace isolation.
2283 */
2284int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2285{
2286 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2287}
2288EXPORT_SYMBOL_GPL(dev_forward_skb);
2289
2290int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2291{
2292 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2293}
2294
2295static inline int deliver_skb(struct sk_buff *skb,
2296 struct packet_type *pt_prev,
2297 struct net_device *orig_dev)
2298{
2299 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2300 return -ENOMEM;
2301 refcount_inc(&skb->users);
2302 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2303}
2304
2305static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2306 struct packet_type **pt,
2307 struct net_device *orig_dev,
2308 __be16 type,
2309 struct list_head *ptype_list)
2310{
2311 struct packet_type *ptype, *pt_prev = *pt;
2312
2313 list_for_each_entry_rcu(ptype, ptype_list, list) {
2314 if (ptype->type != type)
2315 continue;
2316 if (pt_prev)
2317 deliver_skb(skb, pt_prev, orig_dev);
2318 pt_prev = ptype;
2319 }
2320 *pt = pt_prev;
2321}
2322
2323static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2324{
2325 if (!ptype->af_packet_priv || !skb->sk)
2326 return false;
2327
2328 if (ptype->id_match)
2329 return ptype->id_match(ptype, skb->sk);
2330 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2331 return true;
2332
2333 return false;
2334}
2335
2336/**
2337 * dev_nit_active - return true if any network interface taps are in use
2338 *
2339 * @dev: network device to check for the presence of taps
2340 */
2341bool dev_nit_active(struct net_device *dev)
2342{
2343 return !list_empty(&net_hotdata.ptype_all) ||
2344 !list_empty(&dev->ptype_all);
2345}
2346EXPORT_SYMBOL_GPL(dev_nit_active);
2347
2348/*
2349 * Support routine. Sends outgoing frames to any network
2350 * taps currently in use.
2351 */
2352
2353void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2354{
2355 struct list_head *ptype_list = &net_hotdata.ptype_all;
2356 struct packet_type *ptype, *pt_prev = NULL;
2357 struct sk_buff *skb2 = NULL;
2358
2359 rcu_read_lock();
2360again:
2361 list_for_each_entry_rcu(ptype, ptype_list, list) {
2362 if (READ_ONCE(ptype->ignore_outgoing))
2363 continue;
2364
2365 /* Never send packets back to the socket
2366 * they originated from - MvS (miquels@drinkel.ow.org)
2367 */
2368 if (skb_loop_sk(ptype, skb))
2369 continue;
2370
2371 if (pt_prev) {
2372 deliver_skb(skb2, pt_prev, skb->dev);
2373 pt_prev = ptype;
2374 continue;
2375 }
2376
2377 /* need to clone skb, done only once */
2378 skb2 = skb_clone(skb, GFP_ATOMIC);
2379 if (!skb2)
2380 goto out_unlock;
2381
2382 net_timestamp_set(skb2);
2383
2384 /* skb->nh should be correctly
2385 * set by sender, so that the second statement is
2386 * just protection against buggy protocols.
2387 */
2388 skb_reset_mac_header(skb2);
2389
2390 if (skb_network_header(skb2) < skb2->data ||
2391 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2392 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2393 ntohs(skb2->protocol),
2394 dev->name);
2395 skb_reset_network_header(skb2);
2396 }
2397
2398 skb2->transport_header = skb2->network_header;
2399 skb2->pkt_type = PACKET_OUTGOING;
2400 pt_prev = ptype;
2401 }
2402
2403 if (ptype_list == &net_hotdata.ptype_all) {
2404 ptype_list = &dev->ptype_all;
2405 goto again;
2406 }
2407out_unlock:
2408 if (pt_prev) {
2409 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2410 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2411 else
2412 kfree_skb(skb2);
2413 }
2414 rcu_read_unlock();
2415}
2416EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2417
2418/**
2419 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2420 * @dev: Network device
2421 * @txq: number of queues available
2422 *
2423 * If real_num_tx_queues is changed the tc mappings may no longer be
2424 * valid. To resolve this verify the tc mapping remains valid and if
2425 * not NULL the mapping. With no priorities mapping to this
2426 * offset/count pair it will no longer be used. In the worst case TC0
2427 * is invalid nothing can be done so disable priority mappings. If is
2428 * expected that drivers will fix this mapping if they can before
2429 * calling netif_set_real_num_tx_queues.
2430 */
2431static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2432{
2433 int i;
2434 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2435
2436 /* If TC0 is invalidated disable TC mapping */
2437 if (tc->offset + tc->count > txq) {
2438 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2439 dev->num_tc = 0;
2440 return;
2441 }
2442
2443 /* Invalidated prio to tc mappings set to TC0 */
2444 for (i = 1; i < TC_BITMASK + 1; i++) {
2445 int q = netdev_get_prio_tc_map(dev, i);
2446
2447 tc = &dev->tc_to_txq[q];
2448 if (tc->offset + tc->count > txq) {
2449 netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2450 i, q);
2451 netdev_set_prio_tc_map(dev, i, 0);
2452 }
2453 }
2454}
2455
2456int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2457{
2458 if (dev->num_tc) {
2459 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2460 int i;
2461
2462 /* walk through the TCs and see if it falls into any of them */
2463 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2464 if ((txq - tc->offset) < tc->count)
2465 return i;
2466 }
2467
2468 /* didn't find it, just return -1 to indicate no match */
2469 return -1;
2470 }
2471
2472 return 0;
2473}
2474EXPORT_SYMBOL(netdev_txq_to_tc);
2475
2476#ifdef CONFIG_XPS
2477static struct static_key xps_needed __read_mostly;
2478static struct static_key xps_rxqs_needed __read_mostly;
2479static DEFINE_MUTEX(xps_map_mutex);
2480#define xmap_dereference(P) \
2481 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2482
2483static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2484 struct xps_dev_maps *old_maps, int tci, u16 index)
2485{
2486 struct xps_map *map = NULL;
2487 int pos;
2488
2489 map = xmap_dereference(dev_maps->attr_map[tci]);
2490 if (!map)
2491 return false;
2492
2493 for (pos = map->len; pos--;) {
2494 if (map->queues[pos] != index)
2495 continue;
2496
2497 if (map->len > 1) {
2498 map->queues[pos] = map->queues[--map->len];
2499 break;
2500 }
2501
2502 if (old_maps)
2503 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2504 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2505 kfree_rcu(map, rcu);
2506 return false;
2507 }
2508
2509 return true;
2510}
2511
2512static bool remove_xps_queue_cpu(struct net_device *dev,
2513 struct xps_dev_maps *dev_maps,
2514 int cpu, u16 offset, u16 count)
2515{
2516 int num_tc = dev_maps->num_tc;
2517 bool active = false;
2518 int tci;
2519
2520 for (tci = cpu * num_tc; num_tc--; tci++) {
2521 int i, j;
2522
2523 for (i = count, j = offset; i--; j++) {
2524 if (!remove_xps_queue(dev_maps, NULL, tci, j))
2525 break;
2526 }
2527
2528 active |= i < 0;
2529 }
2530
2531 return active;
2532}
2533
2534static void reset_xps_maps(struct net_device *dev,
2535 struct xps_dev_maps *dev_maps,
2536 enum xps_map_type type)
2537{
2538 static_key_slow_dec_cpuslocked(&xps_needed);
2539 if (type == XPS_RXQS)
2540 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2541
2542 RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2543
2544 kfree_rcu(dev_maps, rcu);
2545}
2546
2547static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2548 u16 offset, u16 count)
2549{
2550 struct xps_dev_maps *dev_maps;
2551 bool active = false;
2552 int i, j;
2553
2554 dev_maps = xmap_dereference(dev->xps_maps[type]);
2555 if (!dev_maps)
2556 return;
2557
2558 for (j = 0; j < dev_maps->nr_ids; j++)
2559 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2560 if (!active)
2561 reset_xps_maps(dev, dev_maps, type);
2562
2563 if (type == XPS_CPUS) {
2564 for (i = offset + (count - 1); count--; i--)
2565 netdev_queue_numa_node_write(
2566 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2567 }
2568}
2569
2570static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2571 u16 count)
2572{
2573 if (!static_key_false(&xps_needed))
2574 return;
2575
2576 cpus_read_lock();
2577 mutex_lock(&xps_map_mutex);
2578
2579 if (static_key_false(&xps_rxqs_needed))
2580 clean_xps_maps(dev, XPS_RXQS, offset, count);
2581
2582 clean_xps_maps(dev, XPS_CPUS, offset, count);
2583
2584 mutex_unlock(&xps_map_mutex);
2585 cpus_read_unlock();
2586}
2587
2588static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2589{
2590 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2591}
2592
2593static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2594 u16 index, bool is_rxqs_map)
2595{
2596 struct xps_map *new_map;
2597 int alloc_len = XPS_MIN_MAP_ALLOC;
2598 int i, pos;
2599
2600 for (pos = 0; map && pos < map->len; pos++) {
2601 if (map->queues[pos] != index)
2602 continue;
2603 return map;
2604 }
2605
2606 /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2607 if (map) {
2608 if (pos < map->alloc_len)
2609 return map;
2610
2611 alloc_len = map->alloc_len * 2;
2612 }
2613
2614 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2615 * map
2616 */
2617 if (is_rxqs_map)
2618 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2619 else
2620 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2621 cpu_to_node(attr_index));
2622 if (!new_map)
2623 return NULL;
2624
2625 for (i = 0; i < pos; i++)
2626 new_map->queues[i] = map->queues[i];
2627 new_map->alloc_len = alloc_len;
2628 new_map->len = pos;
2629
2630 return new_map;
2631}
2632
2633/* Copy xps maps at a given index */
2634static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2635 struct xps_dev_maps *new_dev_maps, int index,
2636 int tc, bool skip_tc)
2637{
2638 int i, tci = index * dev_maps->num_tc;
2639 struct xps_map *map;
2640
2641 /* copy maps belonging to foreign traffic classes */
2642 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2643 if (i == tc && skip_tc)
2644 continue;
2645
2646 /* fill in the new device map from the old device map */
2647 map = xmap_dereference(dev_maps->attr_map[tci]);
2648 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2649 }
2650}
2651
2652/* Must be called under cpus_read_lock */
2653int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2654 u16 index, enum xps_map_type type)
2655{
2656 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2657 const unsigned long *online_mask = NULL;
2658 bool active = false, copy = false;
2659 int i, j, tci, numa_node_id = -2;
2660 int maps_sz, num_tc = 1, tc = 0;
2661 struct xps_map *map, *new_map;
2662 unsigned int nr_ids;
2663
2664 WARN_ON_ONCE(index >= dev->num_tx_queues);
2665
2666 if (dev->num_tc) {
2667 /* Do not allow XPS on subordinate device directly */
2668 num_tc = dev->num_tc;
2669 if (num_tc < 0)
2670 return -EINVAL;
2671
2672 /* If queue belongs to subordinate dev use its map */
2673 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2674
2675 tc = netdev_txq_to_tc(dev, index);
2676 if (tc < 0)
2677 return -EINVAL;
2678 }
2679
2680 mutex_lock(&xps_map_mutex);
2681
2682 dev_maps = xmap_dereference(dev->xps_maps[type]);
2683 if (type == XPS_RXQS) {
2684 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2685 nr_ids = dev->num_rx_queues;
2686 } else {
2687 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2688 if (num_possible_cpus() > 1)
2689 online_mask = cpumask_bits(cpu_online_mask);
2690 nr_ids = nr_cpu_ids;
2691 }
2692
2693 if (maps_sz < L1_CACHE_BYTES)
2694 maps_sz = L1_CACHE_BYTES;
2695
2696 /* The old dev_maps could be larger or smaller than the one we're
2697 * setting up now, as dev->num_tc or nr_ids could have been updated in
2698 * between. We could try to be smart, but let's be safe instead and only
2699 * copy foreign traffic classes if the two map sizes match.
2700 */
2701 if (dev_maps &&
2702 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2703 copy = true;
2704
2705 /* allocate memory for queue storage */
2706 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2707 j < nr_ids;) {
2708 if (!new_dev_maps) {
2709 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2710 if (!new_dev_maps) {
2711 mutex_unlock(&xps_map_mutex);
2712 return -ENOMEM;
2713 }
2714
2715 new_dev_maps->nr_ids = nr_ids;
2716 new_dev_maps->num_tc = num_tc;
2717 }
2718
2719 tci = j * num_tc + tc;
2720 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2721
2722 map = expand_xps_map(map, j, index, type == XPS_RXQS);
2723 if (!map)
2724 goto error;
2725
2726 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2727 }
2728
2729 if (!new_dev_maps)
2730 goto out_no_new_maps;
2731
2732 if (!dev_maps) {
2733 /* Increment static keys at most once per type */
2734 static_key_slow_inc_cpuslocked(&xps_needed);
2735 if (type == XPS_RXQS)
2736 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2737 }
2738
2739 for (j = 0; j < nr_ids; j++) {
2740 bool skip_tc = false;
2741
2742 tci = j * num_tc + tc;
2743 if (netif_attr_test_mask(j, mask, nr_ids) &&
2744 netif_attr_test_online(j, online_mask, nr_ids)) {
2745 /* add tx-queue to CPU/rx-queue maps */
2746 int pos = 0;
2747
2748 skip_tc = true;
2749
2750 map = xmap_dereference(new_dev_maps->attr_map[tci]);
2751 while ((pos < map->len) && (map->queues[pos] != index))
2752 pos++;
2753
2754 if (pos == map->len)
2755 map->queues[map->len++] = index;
2756#ifdef CONFIG_NUMA
2757 if (type == XPS_CPUS) {
2758 if (numa_node_id == -2)
2759 numa_node_id = cpu_to_node(j);
2760 else if (numa_node_id != cpu_to_node(j))
2761 numa_node_id = -1;
2762 }
2763#endif
2764 }
2765
2766 if (copy)
2767 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2768 skip_tc);
2769 }
2770
2771 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2772
2773 /* Cleanup old maps */
2774 if (!dev_maps)
2775 goto out_no_old_maps;
2776
2777 for (j = 0; j < dev_maps->nr_ids; j++) {
2778 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2779 map = xmap_dereference(dev_maps->attr_map[tci]);
2780 if (!map)
2781 continue;
2782
2783 if (copy) {
2784 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2785 if (map == new_map)
2786 continue;
2787 }
2788
2789 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2790 kfree_rcu(map, rcu);
2791 }
2792 }
2793
2794 old_dev_maps = dev_maps;
2795
2796out_no_old_maps:
2797 dev_maps = new_dev_maps;
2798 active = true;
2799
2800out_no_new_maps:
2801 if (type == XPS_CPUS)
2802 /* update Tx queue numa node */
2803 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2804 (numa_node_id >= 0) ?
2805 numa_node_id : NUMA_NO_NODE);
2806
2807 if (!dev_maps)
2808 goto out_no_maps;
2809
2810 /* removes tx-queue from unused CPUs/rx-queues */
2811 for (j = 0; j < dev_maps->nr_ids; j++) {
2812 tci = j * dev_maps->num_tc;
2813
2814 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2815 if (i == tc &&
2816 netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2817 netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2818 continue;
2819
2820 active |= remove_xps_queue(dev_maps,
2821 copy ? old_dev_maps : NULL,
2822 tci, index);
2823 }
2824 }
2825
2826 if (old_dev_maps)
2827 kfree_rcu(old_dev_maps, rcu);
2828
2829 /* free map if not active */
2830 if (!active)
2831 reset_xps_maps(dev, dev_maps, type);
2832
2833out_no_maps:
2834 mutex_unlock(&xps_map_mutex);
2835
2836 return 0;
2837error:
2838 /* remove any maps that we added */
2839 for (j = 0; j < nr_ids; j++) {
2840 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2841 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2842 map = copy ?
2843 xmap_dereference(dev_maps->attr_map[tci]) :
2844 NULL;
2845 if (new_map && new_map != map)
2846 kfree(new_map);
2847 }
2848 }
2849
2850 mutex_unlock(&xps_map_mutex);
2851
2852 kfree(new_dev_maps);
2853 return -ENOMEM;
2854}
2855EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2856
2857int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2858 u16 index)
2859{
2860 int ret;
2861
2862 cpus_read_lock();
2863 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2864 cpus_read_unlock();
2865
2866 return ret;
2867}
2868EXPORT_SYMBOL(netif_set_xps_queue);
2869
2870#endif
2871static void netdev_unbind_all_sb_channels(struct net_device *dev)
2872{
2873 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2874
2875 /* Unbind any subordinate channels */
2876 while (txq-- != &dev->_tx[0]) {
2877 if (txq->sb_dev)
2878 netdev_unbind_sb_channel(dev, txq->sb_dev);
2879 }
2880}
2881
2882void netdev_reset_tc(struct net_device *dev)
2883{
2884#ifdef CONFIG_XPS
2885 netif_reset_xps_queues_gt(dev, 0);
2886#endif
2887 netdev_unbind_all_sb_channels(dev);
2888
2889 /* Reset TC configuration of device */
2890 dev->num_tc = 0;
2891 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2892 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2893}
2894EXPORT_SYMBOL(netdev_reset_tc);
2895
2896int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2897{
2898 if (tc >= dev->num_tc)
2899 return -EINVAL;
2900
2901#ifdef CONFIG_XPS
2902 netif_reset_xps_queues(dev, offset, count);
2903#endif
2904 dev->tc_to_txq[tc].count = count;
2905 dev->tc_to_txq[tc].offset = offset;
2906 return 0;
2907}
2908EXPORT_SYMBOL(netdev_set_tc_queue);
2909
2910int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2911{
2912 if (num_tc > TC_MAX_QUEUE)
2913 return -EINVAL;
2914
2915#ifdef CONFIG_XPS
2916 netif_reset_xps_queues_gt(dev, 0);
2917#endif
2918 netdev_unbind_all_sb_channels(dev);
2919
2920 dev->num_tc = num_tc;
2921 return 0;
2922}
2923EXPORT_SYMBOL(netdev_set_num_tc);
2924
2925void netdev_unbind_sb_channel(struct net_device *dev,
2926 struct net_device *sb_dev)
2927{
2928 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2929
2930#ifdef CONFIG_XPS
2931 netif_reset_xps_queues_gt(sb_dev, 0);
2932#endif
2933 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2934 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2935
2936 while (txq-- != &dev->_tx[0]) {
2937 if (txq->sb_dev == sb_dev)
2938 txq->sb_dev = NULL;
2939 }
2940}
2941EXPORT_SYMBOL(netdev_unbind_sb_channel);
2942
2943int netdev_bind_sb_channel_queue(struct net_device *dev,
2944 struct net_device *sb_dev,
2945 u8 tc, u16 count, u16 offset)
2946{
2947 /* Make certain the sb_dev and dev are already configured */
2948 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2949 return -EINVAL;
2950
2951 /* We cannot hand out queues we don't have */
2952 if ((offset + count) > dev->real_num_tx_queues)
2953 return -EINVAL;
2954
2955 /* Record the mapping */
2956 sb_dev->tc_to_txq[tc].count = count;
2957 sb_dev->tc_to_txq[tc].offset = offset;
2958
2959 /* Provide a way for Tx queue to find the tc_to_txq map or
2960 * XPS map for itself.
2961 */
2962 while (count--)
2963 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2964
2965 return 0;
2966}
2967EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2968
2969int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2970{
2971 /* Do not use a multiqueue device to represent a subordinate channel */
2972 if (netif_is_multiqueue(dev))
2973 return -ENODEV;
2974
2975 /* We allow channels 1 - 32767 to be used for subordinate channels.
2976 * Channel 0 is meant to be "native" mode and used only to represent
2977 * the main root device. We allow writing 0 to reset the device back
2978 * to normal mode after being used as a subordinate channel.
2979 */
2980 if (channel > S16_MAX)
2981 return -EINVAL;
2982
2983 dev->num_tc = -channel;
2984
2985 return 0;
2986}
2987EXPORT_SYMBOL(netdev_set_sb_channel);
2988
2989/*
2990 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2991 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2992 */
2993int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2994{
2995 bool disabling;
2996 int rc;
2997
2998 disabling = txq < dev->real_num_tx_queues;
2999
3000 if (txq < 1 || txq > dev->num_tx_queues)
3001 return -EINVAL;
3002
3003 if (dev->reg_state == NETREG_REGISTERED ||
3004 dev->reg_state == NETREG_UNREGISTERING) {
3005 ASSERT_RTNL();
3006
3007 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
3008 txq);
3009 if (rc)
3010 return rc;
3011
3012 if (dev->num_tc)
3013 netif_setup_tc(dev, txq);
3014
3015 net_shaper_set_real_num_tx_queues(dev, txq);
3016
3017 dev_qdisc_change_real_num_tx(dev, txq);
3018
3019 dev->real_num_tx_queues = txq;
3020
3021 if (disabling) {
3022 synchronize_net();
3023 qdisc_reset_all_tx_gt(dev, txq);
3024#ifdef CONFIG_XPS
3025 netif_reset_xps_queues_gt(dev, txq);
3026#endif
3027 }
3028 } else {
3029 dev->real_num_tx_queues = txq;
3030 }
3031
3032 return 0;
3033}
3034EXPORT_SYMBOL(netif_set_real_num_tx_queues);
3035
3036#ifdef CONFIG_SYSFS
3037/**
3038 * netif_set_real_num_rx_queues - set actual number of RX queues used
3039 * @dev: Network device
3040 * @rxq: Actual number of RX queues
3041 *
3042 * This must be called either with the rtnl_lock held or before
3043 * registration of the net device. Returns 0 on success, or a
3044 * negative error code. If called before registration, it always
3045 * succeeds.
3046 */
3047int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
3048{
3049 int rc;
3050
3051 if (rxq < 1 || rxq > dev->num_rx_queues)
3052 return -EINVAL;
3053
3054 if (dev->reg_state == NETREG_REGISTERED) {
3055 ASSERT_RTNL();
3056
3057 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
3058 rxq);
3059 if (rc)
3060 return rc;
3061 }
3062
3063 dev->real_num_rx_queues = rxq;
3064 return 0;
3065}
3066EXPORT_SYMBOL(netif_set_real_num_rx_queues);
3067#endif
3068
3069/**
3070 * netif_set_real_num_queues - set actual number of RX and TX queues used
3071 * @dev: Network device
3072 * @txq: Actual number of TX queues
3073 * @rxq: Actual number of RX queues
3074 *
3075 * Set the real number of both TX and RX queues.
3076 * Does nothing if the number of queues is already correct.
3077 */
3078int netif_set_real_num_queues(struct net_device *dev,
3079 unsigned int txq, unsigned int rxq)
3080{
3081 unsigned int old_rxq = dev->real_num_rx_queues;
3082 int err;
3083
3084 if (txq < 1 || txq > dev->num_tx_queues ||
3085 rxq < 1 || rxq > dev->num_rx_queues)
3086 return -EINVAL;
3087
3088 /* Start from increases, so the error path only does decreases -
3089 * decreases can't fail.
3090 */
3091 if (rxq > dev->real_num_rx_queues) {
3092 err = netif_set_real_num_rx_queues(dev, rxq);
3093 if (err)
3094 return err;
3095 }
3096 if (txq > dev->real_num_tx_queues) {
3097 err = netif_set_real_num_tx_queues(dev, txq);
3098 if (err)
3099 goto undo_rx;
3100 }
3101 if (rxq < dev->real_num_rx_queues)
3102 WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
3103 if (txq < dev->real_num_tx_queues)
3104 WARN_ON(netif_set_real_num_tx_queues(dev, txq));
3105
3106 return 0;
3107undo_rx:
3108 WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
3109 return err;
3110}
3111EXPORT_SYMBOL(netif_set_real_num_queues);
3112
3113/**
3114 * netif_set_tso_max_size() - set the max size of TSO frames supported
3115 * @dev: netdev to update
3116 * @size: max skb->len of a TSO frame
3117 *
3118 * Set the limit on the size of TSO super-frames the device can handle.
3119 * Unless explicitly set the stack will assume the value of
3120 * %GSO_LEGACY_MAX_SIZE.
3121 */
3122void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
3123{
3124 dev->tso_max_size = min(GSO_MAX_SIZE, size);
3125 if (size < READ_ONCE(dev->gso_max_size))
3126 netif_set_gso_max_size(dev, size);
3127 if (size < READ_ONCE(dev->gso_ipv4_max_size))
3128 netif_set_gso_ipv4_max_size(dev, size);
3129}
3130EXPORT_SYMBOL(netif_set_tso_max_size);
3131
3132/**
3133 * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3134 * @dev: netdev to update
3135 * @segs: max number of TCP segments
3136 *
3137 * Set the limit on the number of TCP segments the device can generate from
3138 * a single TSO super-frame.
3139 * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3140 */
3141void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3142{
3143 dev->tso_max_segs = segs;
3144 if (segs < READ_ONCE(dev->gso_max_segs))
3145 netif_set_gso_max_segs(dev, segs);
3146}
3147EXPORT_SYMBOL(netif_set_tso_max_segs);
3148
3149/**
3150 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3151 * @to: netdev to update
3152 * @from: netdev from which to copy the limits
3153 */
3154void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
3155{
3156 netif_set_tso_max_size(to, from->tso_max_size);
3157 netif_set_tso_max_segs(to, from->tso_max_segs);
3158}
3159EXPORT_SYMBOL(netif_inherit_tso_max);
3160
3161/**
3162 * netif_get_num_default_rss_queues - default number of RSS queues
3163 *
3164 * Default value is the number of physical cores if there are only 1 or 2, or
3165 * divided by 2 if there are more.
3166 */
3167int netif_get_num_default_rss_queues(void)
3168{
3169 cpumask_var_t cpus;
3170 int cpu, count = 0;
3171
3172 if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3173 return 1;
3174
3175 cpumask_copy(cpus, cpu_online_mask);
3176 for_each_cpu(cpu, cpus) {
3177 ++count;
3178 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3179 }
3180 free_cpumask_var(cpus);
3181
3182 return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3183}
3184EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3185
3186static void __netif_reschedule(struct Qdisc *q)
3187{
3188 struct softnet_data *sd;
3189 unsigned long flags;
3190
3191 local_irq_save(flags);
3192 sd = this_cpu_ptr(&softnet_data);
3193 q->next_sched = NULL;
3194 *sd->output_queue_tailp = q;
3195 sd->output_queue_tailp = &q->next_sched;
3196 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3197 local_irq_restore(flags);
3198}
3199
3200void __netif_schedule(struct Qdisc *q)
3201{
3202 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3203 __netif_reschedule(q);
3204}
3205EXPORT_SYMBOL(__netif_schedule);
3206
3207struct dev_kfree_skb_cb {
3208 enum skb_drop_reason reason;
3209};
3210
3211static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3212{
3213 return (struct dev_kfree_skb_cb *)skb->cb;
3214}
3215
3216void netif_schedule_queue(struct netdev_queue *txq)
3217{
3218 rcu_read_lock();
3219 if (!netif_xmit_stopped(txq)) {
3220 struct Qdisc *q = rcu_dereference(txq->qdisc);
3221
3222 __netif_schedule(q);
3223 }
3224 rcu_read_unlock();
3225}
3226EXPORT_SYMBOL(netif_schedule_queue);
3227
3228void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3229{
3230 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3231 struct Qdisc *q;
3232
3233 rcu_read_lock();
3234 q = rcu_dereference(dev_queue->qdisc);
3235 __netif_schedule(q);
3236 rcu_read_unlock();
3237 }
3238}
3239EXPORT_SYMBOL(netif_tx_wake_queue);
3240
3241void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3242{
3243 unsigned long flags;
3244
3245 if (unlikely(!skb))
3246 return;
3247
3248 if (likely(refcount_read(&skb->users) == 1)) {
3249 smp_rmb();
3250 refcount_set(&skb->users, 0);
3251 } else if (likely(!refcount_dec_and_test(&skb->users))) {
3252 return;
3253 }
3254 get_kfree_skb_cb(skb)->reason = reason;
3255 local_irq_save(flags);
3256 skb->next = __this_cpu_read(softnet_data.completion_queue);
3257 __this_cpu_write(softnet_data.completion_queue, skb);
3258 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3259 local_irq_restore(flags);
3260}
3261EXPORT_SYMBOL(dev_kfree_skb_irq_reason);
3262
3263void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3264{
3265 if (in_hardirq() || irqs_disabled())
3266 dev_kfree_skb_irq_reason(skb, reason);
3267 else
3268 kfree_skb_reason(skb, reason);
3269}
3270EXPORT_SYMBOL(dev_kfree_skb_any_reason);
3271
3272
3273/**
3274 * netif_device_detach - mark device as removed
3275 * @dev: network device
3276 *
3277 * Mark device as removed from system and therefore no longer available.
3278 */
3279void netif_device_detach(struct net_device *dev)
3280{
3281 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3282 netif_running(dev)) {
3283 netif_tx_stop_all_queues(dev);
3284 }
3285}
3286EXPORT_SYMBOL(netif_device_detach);
3287
3288/**
3289 * netif_device_attach - mark device as attached
3290 * @dev: network device
3291 *
3292 * Mark device as attached from system and restart if needed.
3293 */
3294void netif_device_attach(struct net_device *dev)
3295{
3296 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3297 netif_running(dev)) {
3298 netif_tx_wake_all_queues(dev);
3299 __netdev_watchdog_up(dev);
3300 }
3301}
3302EXPORT_SYMBOL(netif_device_attach);
3303
3304/*
3305 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3306 * to be used as a distribution range.
3307 */
3308static u16 skb_tx_hash(const struct net_device *dev,
3309 const struct net_device *sb_dev,
3310 struct sk_buff *skb)
3311{
3312 u32 hash;
3313 u16 qoffset = 0;
3314 u16 qcount = dev->real_num_tx_queues;
3315
3316 if (dev->num_tc) {
3317 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3318
3319 qoffset = sb_dev->tc_to_txq[tc].offset;
3320 qcount = sb_dev->tc_to_txq[tc].count;
3321 if (unlikely(!qcount)) {
3322 net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3323 sb_dev->name, qoffset, tc);
3324 qoffset = 0;
3325 qcount = dev->real_num_tx_queues;
3326 }
3327 }
3328
3329 if (skb_rx_queue_recorded(skb)) {
3330 DEBUG_NET_WARN_ON_ONCE(qcount == 0);
3331 hash = skb_get_rx_queue(skb);
3332 if (hash >= qoffset)
3333 hash -= qoffset;
3334 while (unlikely(hash >= qcount))
3335 hash -= qcount;
3336 return hash + qoffset;
3337 }
3338
3339 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3340}
3341
3342void skb_warn_bad_offload(const struct sk_buff *skb)
3343{
3344 static const netdev_features_t null_features;
3345 struct net_device *dev = skb->dev;
3346 const char *name = "";
3347
3348 if (!net_ratelimit())
3349 return;
3350
3351 if (dev) {
3352 if (dev->dev.parent)
3353 name = dev_driver_string(dev->dev.parent);
3354 else
3355 name = netdev_name(dev);
3356 }
3357 skb_dump(KERN_WARNING, skb, false);
3358 WARN(1, "%s: caps=(%pNF, %pNF)\n",
3359 name, dev ? &dev->features : &null_features,
3360 skb->sk ? &skb->sk->sk_route_caps : &null_features);
3361}
3362
3363/*
3364 * Invalidate hardware checksum when packet is to be mangled, and
3365 * complete checksum manually on outgoing path.
3366 */
3367int skb_checksum_help(struct sk_buff *skb)
3368{
3369 __wsum csum;
3370 int ret = 0, offset;
3371
3372 if (skb->ip_summed == CHECKSUM_COMPLETE)
3373 goto out_set_summed;
3374
3375 if (unlikely(skb_is_gso(skb))) {
3376 skb_warn_bad_offload(skb);
3377 return -EINVAL;
3378 }
3379
3380 if (!skb_frags_readable(skb)) {
3381 return -EFAULT;
3382 }
3383
3384 /* Before computing a checksum, we should make sure no frag could
3385 * be modified by an external entity : checksum could be wrong.
3386 */
3387 if (skb_has_shared_frag(skb)) {
3388 ret = __skb_linearize(skb);
3389 if (ret)
3390 goto out;
3391 }
3392
3393 offset = skb_checksum_start_offset(skb);
3394 ret = -EINVAL;
3395 if (unlikely(offset >= skb_headlen(skb))) {
3396 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3397 WARN_ONCE(true, "offset (%d) >= skb_headlen() (%u)\n",
3398 offset, skb_headlen(skb));
3399 goto out;
3400 }
3401 csum = skb_checksum(skb, offset, skb->len - offset, 0);
3402
3403 offset += skb->csum_offset;
3404 if (unlikely(offset + sizeof(__sum16) > skb_headlen(skb))) {
3405 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3406 WARN_ONCE(true, "offset+2 (%zu) > skb_headlen() (%u)\n",
3407 offset + sizeof(__sum16), skb_headlen(skb));
3408 goto out;
3409 }
3410 ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3411 if (ret)
3412 goto out;
3413
3414 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3415out_set_summed:
3416 skb->ip_summed = CHECKSUM_NONE;
3417out:
3418 return ret;
3419}
3420EXPORT_SYMBOL(skb_checksum_help);
3421
3422int skb_crc32c_csum_help(struct sk_buff *skb)
3423{
3424 __le32 crc32c_csum;
3425 int ret = 0, offset, start;
3426
3427 if (skb->ip_summed != CHECKSUM_PARTIAL)
3428 goto out;
3429
3430 if (unlikely(skb_is_gso(skb)))
3431 goto out;
3432
3433 /* Before computing a checksum, we should make sure no frag could
3434 * be modified by an external entity : checksum could be wrong.
3435 */
3436 if (unlikely(skb_has_shared_frag(skb))) {
3437 ret = __skb_linearize(skb);
3438 if (ret)
3439 goto out;
3440 }
3441 start = skb_checksum_start_offset(skb);
3442 offset = start + offsetof(struct sctphdr, checksum);
3443 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3444 ret = -EINVAL;
3445 goto out;
3446 }
3447
3448 ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3449 if (ret)
3450 goto out;
3451
3452 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3453 skb->len - start, ~(__u32)0,
3454 crc32c_csum_stub));
3455 *(__le32 *)(skb->data + offset) = crc32c_csum;
3456 skb_reset_csum_not_inet(skb);
3457out:
3458 return ret;
3459}
3460EXPORT_SYMBOL(skb_crc32c_csum_help);
3461
3462__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3463{
3464 __be16 type = skb->protocol;
3465
3466 /* Tunnel gso handlers can set protocol to ethernet. */
3467 if (type == htons(ETH_P_TEB)) {
3468 struct ethhdr *eth;
3469
3470 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3471 return 0;
3472
3473 eth = (struct ethhdr *)skb->data;
3474 type = eth->h_proto;
3475 }
3476
3477 return vlan_get_protocol_and_depth(skb, type, depth);
3478}
3479
3480
3481/* Take action when hardware reception checksum errors are detected. */
3482#ifdef CONFIG_BUG
3483static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3484{
3485 netdev_err(dev, "hw csum failure\n");
3486 skb_dump(KERN_ERR, skb, true);
3487 dump_stack();
3488}
3489
3490void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3491{
3492 DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3493}
3494EXPORT_SYMBOL(netdev_rx_csum_fault);
3495#endif
3496
3497/* XXX: check that highmem exists at all on the given machine. */
3498static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3499{
3500#ifdef CONFIG_HIGHMEM
3501 int i;
3502
3503 if (!(dev->features & NETIF_F_HIGHDMA)) {
3504 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3505 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3506 struct page *page = skb_frag_page(frag);
3507
3508 if (page && PageHighMem(page))
3509 return 1;
3510 }
3511 }
3512#endif
3513 return 0;
3514}
3515
3516/* If MPLS offload request, verify we are testing hardware MPLS features
3517 * instead of standard features for the netdev.
3518 */
3519#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3520static netdev_features_t net_mpls_features(struct sk_buff *skb,
3521 netdev_features_t features,
3522 __be16 type)
3523{
3524 if (eth_p_mpls(type))
3525 features &= skb->dev->mpls_features;
3526
3527 return features;
3528}
3529#else
3530static netdev_features_t net_mpls_features(struct sk_buff *skb,
3531 netdev_features_t features,
3532 __be16 type)
3533{
3534 return features;
3535}
3536#endif
3537
3538static netdev_features_t harmonize_features(struct sk_buff *skb,
3539 netdev_features_t features)
3540{
3541 __be16 type;
3542
3543 type = skb_network_protocol(skb, NULL);
3544 features = net_mpls_features(skb, features, type);
3545
3546 if (skb->ip_summed != CHECKSUM_NONE &&
3547 !can_checksum_protocol(features, type)) {
3548 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3549 }
3550 if (illegal_highdma(skb->dev, skb))
3551 features &= ~NETIF_F_SG;
3552
3553 return features;
3554}
3555
3556netdev_features_t passthru_features_check(struct sk_buff *skb,
3557 struct net_device *dev,
3558 netdev_features_t features)
3559{
3560 return features;
3561}
3562EXPORT_SYMBOL(passthru_features_check);
3563
3564static netdev_features_t dflt_features_check(struct sk_buff *skb,
3565 struct net_device *dev,
3566 netdev_features_t features)
3567{
3568 return vlan_features_check(skb, features);
3569}
3570
3571static netdev_features_t gso_features_check(const struct sk_buff *skb,
3572 struct net_device *dev,
3573 netdev_features_t features)
3574{
3575 u16 gso_segs = skb_shinfo(skb)->gso_segs;
3576
3577 if (gso_segs > READ_ONCE(dev->gso_max_segs))
3578 return features & ~NETIF_F_GSO_MASK;
3579
3580 if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb)))
3581 return features & ~NETIF_F_GSO_MASK;
3582
3583 if (!skb_shinfo(skb)->gso_type) {
3584 skb_warn_bad_offload(skb);
3585 return features & ~NETIF_F_GSO_MASK;
3586 }
3587
3588 /* Support for GSO partial features requires software
3589 * intervention before we can actually process the packets
3590 * so we need to strip support for any partial features now
3591 * and we can pull them back in after we have partially
3592 * segmented the frame.
3593 */
3594 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3595 features &= ~dev->gso_partial_features;
3596
3597 /* Make sure to clear the IPv4 ID mangling feature if the
3598 * IPv4 header has the potential to be fragmented.
3599 */
3600 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3601 struct iphdr *iph = skb->encapsulation ?
3602 inner_ip_hdr(skb) : ip_hdr(skb);
3603
3604 if (!(iph->frag_off & htons(IP_DF)))
3605 features &= ~NETIF_F_TSO_MANGLEID;
3606 }
3607
3608 return features;
3609}
3610
3611netdev_features_t netif_skb_features(struct sk_buff *skb)
3612{
3613 struct net_device *dev = skb->dev;
3614 netdev_features_t features = dev->features;
3615
3616 if (skb_is_gso(skb))
3617 features = gso_features_check(skb, dev, features);
3618
3619 /* If encapsulation offload request, verify we are testing
3620 * hardware encapsulation features instead of standard
3621 * features for the netdev
3622 */
3623 if (skb->encapsulation)
3624 features &= dev->hw_enc_features;
3625
3626 if (skb_vlan_tagged(skb))
3627 features = netdev_intersect_features(features,
3628 dev->vlan_features |
3629 NETIF_F_HW_VLAN_CTAG_TX |
3630 NETIF_F_HW_VLAN_STAG_TX);
3631
3632 if (dev->netdev_ops->ndo_features_check)
3633 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3634 features);
3635 else
3636 features &= dflt_features_check(skb, dev, features);
3637
3638 return harmonize_features(skb, features);
3639}
3640EXPORT_SYMBOL(netif_skb_features);
3641
3642static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3643 struct netdev_queue *txq, bool more)
3644{
3645 unsigned int len;
3646 int rc;
3647
3648 if (dev_nit_active(dev))
3649 dev_queue_xmit_nit(skb, dev);
3650
3651 len = skb->len;
3652 trace_net_dev_start_xmit(skb, dev);
3653 rc = netdev_start_xmit(skb, dev, txq, more);
3654 trace_net_dev_xmit(skb, rc, dev, len);
3655
3656 return rc;
3657}
3658
3659struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3660 struct netdev_queue *txq, int *ret)
3661{
3662 struct sk_buff *skb = first;
3663 int rc = NETDEV_TX_OK;
3664
3665 while (skb) {
3666 struct sk_buff *next = skb->next;
3667
3668 skb_mark_not_on_list(skb);
3669 rc = xmit_one(skb, dev, txq, next != NULL);
3670 if (unlikely(!dev_xmit_complete(rc))) {
3671 skb->next = next;
3672 goto out;
3673 }
3674
3675 skb = next;
3676 if (netif_tx_queue_stopped(txq) && skb) {
3677 rc = NETDEV_TX_BUSY;
3678 break;
3679 }
3680 }
3681
3682out:
3683 *ret = rc;
3684 return skb;
3685}
3686
3687static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3688 netdev_features_t features)
3689{
3690 if (skb_vlan_tag_present(skb) &&
3691 !vlan_hw_offload_capable(features, skb->vlan_proto))
3692 skb = __vlan_hwaccel_push_inside(skb);
3693 return skb;
3694}
3695
3696int skb_csum_hwoffload_help(struct sk_buff *skb,
3697 const netdev_features_t features)
3698{
3699 if (unlikely(skb_csum_is_sctp(skb)))
3700 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3701 skb_crc32c_csum_help(skb);
3702
3703 if (features & NETIF_F_HW_CSUM)
3704 return 0;
3705
3706 if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3707 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6) &&
3708 skb_network_header_len(skb) != sizeof(struct ipv6hdr) &&
3709 !ipv6_has_hopopt_jumbo(skb))
3710 goto sw_checksum;
3711
3712 switch (skb->csum_offset) {
3713 case offsetof(struct tcphdr, check):
3714 case offsetof(struct udphdr, check):
3715 return 0;
3716 }
3717 }
3718
3719sw_checksum:
3720 return skb_checksum_help(skb);
3721}
3722EXPORT_SYMBOL(skb_csum_hwoffload_help);
3723
3724static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3725{
3726 netdev_features_t features;
3727
3728 features = netif_skb_features(skb);
3729 skb = validate_xmit_vlan(skb, features);
3730 if (unlikely(!skb))
3731 goto out_null;
3732
3733 skb = sk_validate_xmit_skb(skb, dev);
3734 if (unlikely(!skb))
3735 goto out_null;
3736
3737 if (netif_needs_gso(skb, features)) {
3738 struct sk_buff *segs;
3739
3740 segs = skb_gso_segment(skb, features);
3741 if (IS_ERR(segs)) {
3742 goto out_kfree_skb;
3743 } else if (segs) {
3744 consume_skb(skb);
3745 skb = segs;
3746 }
3747 } else {
3748 if (skb_needs_linearize(skb, features) &&
3749 __skb_linearize(skb))
3750 goto out_kfree_skb;
3751
3752 /* If packet is not checksummed and device does not
3753 * support checksumming for this protocol, complete
3754 * checksumming here.
3755 */
3756 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3757 if (skb->encapsulation)
3758 skb_set_inner_transport_header(skb,
3759 skb_checksum_start_offset(skb));
3760 else
3761 skb_set_transport_header(skb,
3762 skb_checksum_start_offset(skb));
3763 if (skb_csum_hwoffload_help(skb, features))
3764 goto out_kfree_skb;
3765 }
3766 }
3767
3768 skb = validate_xmit_xfrm(skb, features, again);
3769
3770 return skb;
3771
3772out_kfree_skb:
3773 kfree_skb(skb);
3774out_null:
3775 dev_core_stats_tx_dropped_inc(dev);
3776 return NULL;
3777}
3778
3779struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3780{
3781 struct sk_buff *next, *head = NULL, *tail;
3782
3783 for (; skb != NULL; skb = next) {
3784 next = skb->next;
3785 skb_mark_not_on_list(skb);
3786
3787 /* in case skb won't be segmented, point to itself */
3788 skb->prev = skb;
3789
3790 skb = validate_xmit_skb(skb, dev, again);
3791 if (!skb)
3792 continue;
3793
3794 if (!head)
3795 head = skb;
3796 else
3797 tail->next = skb;
3798 /* If skb was segmented, skb->prev points to
3799 * the last segment. If not, it still contains skb.
3800 */
3801 tail = skb->prev;
3802 }
3803 return head;
3804}
3805EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3806
3807static void qdisc_pkt_len_init(struct sk_buff *skb)
3808{
3809 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3810
3811 qdisc_skb_cb(skb)->pkt_len = skb->len;
3812
3813 /* To get more precise estimation of bytes sent on wire,
3814 * we add to pkt_len the headers size of all segments
3815 */
3816 if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3817 u16 gso_segs = shinfo->gso_segs;
3818 unsigned int hdr_len;
3819
3820 /* mac layer + network layer */
3821 hdr_len = skb_transport_offset(skb);
3822
3823 /* + transport layer */
3824 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3825 const struct tcphdr *th;
3826 struct tcphdr _tcphdr;
3827
3828 th = skb_header_pointer(skb, hdr_len,
3829 sizeof(_tcphdr), &_tcphdr);
3830 if (likely(th))
3831 hdr_len += __tcp_hdrlen(th);
3832 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
3833 struct udphdr _udphdr;
3834
3835 if (skb_header_pointer(skb, hdr_len,
3836 sizeof(_udphdr), &_udphdr))
3837 hdr_len += sizeof(struct udphdr);
3838 }
3839
3840 if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) {
3841 int payload = skb->len - hdr_len;
3842
3843 /* Malicious packet. */
3844 if (payload <= 0)
3845 return;
3846 gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size);
3847 }
3848 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3849 }
3850}
3851
3852static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3853 struct sk_buff **to_free,
3854 struct netdev_queue *txq)
3855{
3856 int rc;
3857
3858 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3859 if (rc == NET_XMIT_SUCCESS)
3860 trace_qdisc_enqueue(q, txq, skb);
3861 return rc;
3862}
3863
3864static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3865 struct net_device *dev,
3866 struct netdev_queue *txq)
3867{
3868 spinlock_t *root_lock = qdisc_lock(q);
3869 struct sk_buff *to_free = NULL;
3870 bool contended;
3871 int rc;
3872
3873 qdisc_calculate_pkt_len(skb, q);
3874
3875 tcf_set_drop_reason(skb, SKB_DROP_REASON_QDISC_DROP);
3876
3877 if (q->flags & TCQ_F_NOLOCK) {
3878 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3879 qdisc_run_begin(q)) {
3880 /* Retest nolock_qdisc_is_empty() within the protection
3881 * of q->seqlock to protect from racing with requeuing.
3882 */
3883 if (unlikely(!nolock_qdisc_is_empty(q))) {
3884 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3885 __qdisc_run(q);
3886 qdisc_run_end(q);
3887
3888 goto no_lock_out;
3889 }
3890
3891 qdisc_bstats_cpu_update(q, skb);
3892 if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3893 !nolock_qdisc_is_empty(q))
3894 __qdisc_run(q);
3895
3896 qdisc_run_end(q);
3897 return NET_XMIT_SUCCESS;
3898 }
3899
3900 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3901 qdisc_run(q);
3902
3903no_lock_out:
3904 if (unlikely(to_free))
3905 kfree_skb_list_reason(to_free,
3906 tcf_get_drop_reason(to_free));
3907 return rc;
3908 }
3909
3910 if (unlikely(READ_ONCE(q->owner) == smp_processor_id())) {
3911 kfree_skb_reason(skb, SKB_DROP_REASON_TC_RECLASSIFY_LOOP);
3912 return NET_XMIT_DROP;
3913 }
3914 /*
3915 * Heuristic to force contended enqueues to serialize on a
3916 * separate lock before trying to get qdisc main lock.
3917 * This permits qdisc->running owner to get the lock more
3918 * often and dequeue packets faster.
3919 * On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
3920 * and then other tasks will only enqueue packets. The packets will be
3921 * sent after the qdisc owner is scheduled again. To prevent this
3922 * scenario the task always serialize on the lock.
3923 */
3924 contended = qdisc_is_running(q) || IS_ENABLED(CONFIG_PREEMPT_RT);
3925 if (unlikely(contended))
3926 spin_lock(&q->busylock);
3927
3928 spin_lock(root_lock);
3929 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3930 __qdisc_drop(skb, &to_free);
3931 rc = NET_XMIT_DROP;
3932 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3933 qdisc_run_begin(q)) {
3934 /*
3935 * This is a work-conserving queue; there are no old skbs
3936 * waiting to be sent out; and the qdisc is not running -
3937 * xmit the skb directly.
3938 */
3939
3940 qdisc_bstats_update(q, skb);
3941
3942 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3943 if (unlikely(contended)) {
3944 spin_unlock(&q->busylock);
3945 contended = false;
3946 }
3947 __qdisc_run(q);
3948 }
3949
3950 qdisc_run_end(q);
3951 rc = NET_XMIT_SUCCESS;
3952 } else {
3953 WRITE_ONCE(q->owner, smp_processor_id());
3954 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3955 WRITE_ONCE(q->owner, -1);
3956 if (qdisc_run_begin(q)) {
3957 if (unlikely(contended)) {
3958 spin_unlock(&q->busylock);
3959 contended = false;
3960 }
3961 __qdisc_run(q);
3962 qdisc_run_end(q);
3963 }
3964 }
3965 spin_unlock(root_lock);
3966 if (unlikely(to_free))
3967 kfree_skb_list_reason(to_free,
3968 tcf_get_drop_reason(to_free));
3969 if (unlikely(contended))
3970 spin_unlock(&q->busylock);
3971 return rc;
3972}
3973
3974#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3975static void skb_update_prio(struct sk_buff *skb)
3976{
3977 const struct netprio_map *map;
3978 const struct sock *sk;
3979 unsigned int prioidx;
3980
3981 if (skb->priority)
3982 return;
3983 map = rcu_dereference_bh(skb->dev->priomap);
3984 if (!map)
3985 return;
3986 sk = skb_to_full_sk(skb);
3987 if (!sk)
3988 return;
3989
3990 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3991
3992 if (prioidx < map->priomap_len)
3993 skb->priority = map->priomap[prioidx];
3994}
3995#else
3996#define skb_update_prio(skb)
3997#endif
3998
3999/**
4000 * dev_loopback_xmit - loop back @skb
4001 * @net: network namespace this loopback is happening in
4002 * @sk: sk needed to be a netfilter okfn
4003 * @skb: buffer to transmit
4004 */
4005int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
4006{
4007 skb_reset_mac_header(skb);
4008 __skb_pull(skb, skb_network_offset(skb));
4009 skb->pkt_type = PACKET_LOOPBACK;
4010 if (skb->ip_summed == CHECKSUM_NONE)
4011 skb->ip_summed = CHECKSUM_UNNECESSARY;
4012 DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
4013 skb_dst_force(skb);
4014 netif_rx(skb);
4015 return 0;
4016}
4017EXPORT_SYMBOL(dev_loopback_xmit);
4018
4019#ifdef CONFIG_NET_EGRESS
4020static struct netdev_queue *
4021netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
4022{
4023 int qm = skb_get_queue_mapping(skb);
4024
4025 return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
4026}
4027
4028#ifndef CONFIG_PREEMPT_RT
4029static bool netdev_xmit_txqueue_skipped(void)
4030{
4031 return __this_cpu_read(softnet_data.xmit.skip_txqueue);
4032}
4033
4034void netdev_xmit_skip_txqueue(bool skip)
4035{
4036 __this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
4037}
4038EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4039
4040#else
4041static bool netdev_xmit_txqueue_skipped(void)
4042{
4043 return current->net_xmit.skip_txqueue;
4044}
4045
4046void netdev_xmit_skip_txqueue(bool skip)
4047{
4048 current->net_xmit.skip_txqueue = skip;
4049}
4050EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4051#endif
4052#endif /* CONFIG_NET_EGRESS */
4053
4054#ifdef CONFIG_NET_XGRESS
4055static int tc_run(struct tcx_entry *entry, struct sk_buff *skb,
4056 enum skb_drop_reason *drop_reason)
4057{
4058 int ret = TC_ACT_UNSPEC;
4059#ifdef CONFIG_NET_CLS_ACT
4060 struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq);
4061 struct tcf_result res;
4062
4063 if (!miniq)
4064 return ret;
4065
4066 /* Global bypass */
4067 if (!static_branch_likely(&tcf_sw_enabled_key))
4068 return ret;
4069
4070 /* Block-wise bypass */
4071 if (tcf_block_bypass_sw(miniq->block))
4072 return ret;
4073
4074 tc_skb_cb(skb)->mru = 0;
4075 tc_skb_cb(skb)->post_ct = false;
4076 tcf_set_drop_reason(skb, *drop_reason);
4077
4078 mini_qdisc_bstats_cpu_update(miniq, skb);
4079 ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false);
4080 /* Only tcf related quirks below. */
4081 switch (ret) {
4082 case TC_ACT_SHOT:
4083 *drop_reason = tcf_get_drop_reason(skb);
4084 mini_qdisc_qstats_cpu_drop(miniq);
4085 break;
4086 case TC_ACT_OK:
4087 case TC_ACT_RECLASSIFY:
4088 skb->tc_index = TC_H_MIN(res.classid);
4089 break;
4090 }
4091#endif /* CONFIG_NET_CLS_ACT */
4092 return ret;
4093}
4094
4095static DEFINE_STATIC_KEY_FALSE(tcx_needed_key);
4096
4097void tcx_inc(void)
4098{
4099 static_branch_inc(&tcx_needed_key);
4100}
4101
4102void tcx_dec(void)
4103{
4104 static_branch_dec(&tcx_needed_key);
4105}
4106
4107static __always_inline enum tcx_action_base
4108tcx_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
4109 const bool needs_mac)
4110{
4111 const struct bpf_mprog_fp *fp;
4112 const struct bpf_prog *prog;
4113 int ret = TCX_NEXT;
4114
4115 if (needs_mac)
4116 __skb_push(skb, skb->mac_len);
4117 bpf_mprog_foreach_prog(entry, fp, prog) {
4118 bpf_compute_data_pointers(skb);
4119 ret = bpf_prog_run(prog, skb);
4120 if (ret != TCX_NEXT)
4121 break;
4122 }
4123 if (needs_mac)
4124 __skb_pull(skb, skb->mac_len);
4125 return tcx_action_code(skb, ret);
4126}
4127
4128static __always_inline struct sk_buff *
4129sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4130 struct net_device *orig_dev, bool *another)
4131{
4132 struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress);
4133 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_INGRESS;
4134 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
4135 int sch_ret;
4136
4137 if (!entry)
4138 return skb;
4139
4140 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
4141 if (*pt_prev) {
4142 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4143 *pt_prev = NULL;
4144 }
4145
4146 qdisc_skb_cb(skb)->pkt_len = skb->len;
4147 tcx_set_ingress(skb, true);
4148
4149 if (static_branch_unlikely(&tcx_needed_key)) {
4150 sch_ret = tcx_run(entry, skb, true);
4151 if (sch_ret != TC_ACT_UNSPEC)
4152 goto ingress_verdict;
4153 }
4154 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4155ingress_verdict:
4156 switch (sch_ret) {
4157 case TC_ACT_REDIRECT:
4158 /* skb_mac_header check was done by BPF, so we can safely
4159 * push the L2 header back before redirecting to another
4160 * netdev.
4161 */
4162 __skb_push(skb, skb->mac_len);
4163 if (skb_do_redirect(skb) == -EAGAIN) {
4164 __skb_pull(skb, skb->mac_len);
4165 *another = true;
4166 break;
4167 }
4168 *ret = NET_RX_SUCCESS;
4169 bpf_net_ctx_clear(bpf_net_ctx);
4170 return NULL;
4171 case TC_ACT_SHOT:
4172 kfree_skb_reason(skb, drop_reason);
4173 *ret = NET_RX_DROP;
4174 bpf_net_ctx_clear(bpf_net_ctx);
4175 return NULL;
4176 /* used by tc_run */
4177 case TC_ACT_STOLEN:
4178 case TC_ACT_QUEUED:
4179 case TC_ACT_TRAP:
4180 consume_skb(skb);
4181 fallthrough;
4182 case TC_ACT_CONSUMED:
4183 *ret = NET_RX_SUCCESS;
4184 bpf_net_ctx_clear(bpf_net_ctx);
4185 return NULL;
4186 }
4187 bpf_net_ctx_clear(bpf_net_ctx);
4188
4189 return skb;
4190}
4191
4192static __always_inline struct sk_buff *
4193sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4194{
4195 struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress);
4196 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_EGRESS;
4197 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
4198 int sch_ret;
4199
4200 if (!entry)
4201 return skb;
4202
4203 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
4204
4205 /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was
4206 * already set by the caller.
4207 */
4208 if (static_branch_unlikely(&tcx_needed_key)) {
4209 sch_ret = tcx_run(entry, skb, false);
4210 if (sch_ret != TC_ACT_UNSPEC)
4211 goto egress_verdict;
4212 }
4213 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4214egress_verdict:
4215 switch (sch_ret) {
4216 case TC_ACT_REDIRECT:
4217 /* No need to push/pop skb's mac_header here on egress! */
4218 skb_do_redirect(skb);
4219 *ret = NET_XMIT_SUCCESS;
4220 bpf_net_ctx_clear(bpf_net_ctx);
4221 return NULL;
4222 case TC_ACT_SHOT:
4223 kfree_skb_reason(skb, drop_reason);
4224 *ret = NET_XMIT_DROP;
4225 bpf_net_ctx_clear(bpf_net_ctx);
4226 return NULL;
4227 /* used by tc_run */
4228 case TC_ACT_STOLEN:
4229 case TC_ACT_QUEUED:
4230 case TC_ACT_TRAP:
4231 consume_skb(skb);
4232 fallthrough;
4233 case TC_ACT_CONSUMED:
4234 *ret = NET_XMIT_SUCCESS;
4235 bpf_net_ctx_clear(bpf_net_ctx);
4236 return NULL;
4237 }
4238 bpf_net_ctx_clear(bpf_net_ctx);
4239
4240 return skb;
4241}
4242#else
4243static __always_inline struct sk_buff *
4244sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4245 struct net_device *orig_dev, bool *another)
4246{
4247 return skb;
4248}
4249
4250static __always_inline struct sk_buff *
4251sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4252{
4253 return skb;
4254}
4255#endif /* CONFIG_NET_XGRESS */
4256
4257#ifdef CONFIG_XPS
4258static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4259 struct xps_dev_maps *dev_maps, unsigned int tci)
4260{
4261 int tc = netdev_get_prio_tc_map(dev, skb->priority);
4262 struct xps_map *map;
4263 int queue_index = -1;
4264
4265 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
4266 return queue_index;
4267
4268 tci *= dev_maps->num_tc;
4269 tci += tc;
4270
4271 map = rcu_dereference(dev_maps->attr_map[tci]);
4272 if (map) {
4273 if (map->len == 1)
4274 queue_index = map->queues[0];
4275 else
4276 queue_index = map->queues[reciprocal_scale(
4277 skb_get_hash(skb), map->len)];
4278 if (unlikely(queue_index >= dev->real_num_tx_queues))
4279 queue_index = -1;
4280 }
4281 return queue_index;
4282}
4283#endif
4284
4285static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4286 struct sk_buff *skb)
4287{
4288#ifdef CONFIG_XPS
4289 struct xps_dev_maps *dev_maps;
4290 struct sock *sk = skb->sk;
4291 int queue_index = -1;
4292
4293 if (!static_key_false(&xps_needed))
4294 return -1;
4295
4296 rcu_read_lock();
4297 if (!static_key_false(&xps_rxqs_needed))
4298 goto get_cpus_map;
4299
4300 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4301 if (dev_maps) {
4302 int tci = sk_rx_queue_get(sk);
4303
4304 if (tci >= 0)
4305 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4306 tci);
4307 }
4308
4309get_cpus_map:
4310 if (queue_index < 0) {
4311 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4312 if (dev_maps) {
4313 unsigned int tci = skb->sender_cpu - 1;
4314
4315 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4316 tci);
4317 }
4318 }
4319 rcu_read_unlock();
4320
4321 return queue_index;
4322#else
4323 return -1;
4324#endif
4325}
4326
4327u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4328 struct net_device *sb_dev)
4329{
4330 return 0;
4331}
4332EXPORT_SYMBOL(dev_pick_tx_zero);
4333
4334u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4335 struct net_device *sb_dev)
4336{
4337 struct sock *sk = skb->sk;
4338 int queue_index = sk_tx_queue_get(sk);
4339
4340 sb_dev = sb_dev ? : dev;
4341
4342 if (queue_index < 0 || skb->ooo_okay ||
4343 queue_index >= dev->real_num_tx_queues) {
4344 int new_index = get_xps_queue(dev, sb_dev, skb);
4345
4346 if (new_index < 0)
4347 new_index = skb_tx_hash(dev, sb_dev, skb);
4348
4349 if (queue_index != new_index && sk &&
4350 sk_fullsock(sk) &&
4351 rcu_access_pointer(sk->sk_dst_cache))
4352 sk_tx_queue_set(sk, new_index);
4353
4354 queue_index = new_index;
4355 }
4356
4357 return queue_index;
4358}
4359EXPORT_SYMBOL(netdev_pick_tx);
4360
4361struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4362 struct sk_buff *skb,
4363 struct net_device *sb_dev)
4364{
4365 int queue_index = 0;
4366
4367#ifdef CONFIG_XPS
4368 u32 sender_cpu = skb->sender_cpu - 1;
4369
4370 if (sender_cpu >= (u32)NR_CPUS)
4371 skb->sender_cpu = raw_smp_processor_id() + 1;
4372#endif
4373
4374 if (dev->real_num_tx_queues != 1) {
4375 const struct net_device_ops *ops = dev->netdev_ops;
4376
4377 if (ops->ndo_select_queue)
4378 queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4379 else
4380 queue_index = netdev_pick_tx(dev, skb, sb_dev);
4381
4382 queue_index = netdev_cap_txqueue(dev, queue_index);
4383 }
4384
4385 skb_set_queue_mapping(skb, queue_index);
4386 return netdev_get_tx_queue(dev, queue_index);
4387}
4388
4389/**
4390 * __dev_queue_xmit() - transmit a buffer
4391 * @skb: buffer to transmit
4392 * @sb_dev: suboordinate device used for L2 forwarding offload
4393 *
4394 * Queue a buffer for transmission to a network device. The caller must
4395 * have set the device and priority and built the buffer before calling
4396 * this function. The function can be called from an interrupt.
4397 *
4398 * When calling this method, interrupts MUST be enabled. This is because
4399 * the BH enable code must have IRQs enabled so that it will not deadlock.
4400 *
4401 * Regardless of the return value, the skb is consumed, so it is currently
4402 * difficult to retry a send to this method. (You can bump the ref count
4403 * before sending to hold a reference for retry if you are careful.)
4404 *
4405 * Return:
4406 * * 0 - buffer successfully transmitted
4407 * * positive qdisc return code - NET_XMIT_DROP etc.
4408 * * negative errno - other errors
4409 */
4410int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4411{
4412 struct net_device *dev = skb->dev;
4413 struct netdev_queue *txq = NULL;
4414 struct Qdisc *q;
4415 int rc = -ENOMEM;
4416 bool again = false;
4417
4418 skb_reset_mac_header(skb);
4419 skb_assert_len(skb);
4420
4421 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4422 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4423
4424 /* Disable soft irqs for various locks below. Also
4425 * stops preemption for RCU.
4426 */
4427 rcu_read_lock_bh();
4428
4429 skb_update_prio(skb);
4430
4431 qdisc_pkt_len_init(skb);
4432 tcx_set_ingress(skb, false);
4433#ifdef CONFIG_NET_EGRESS
4434 if (static_branch_unlikely(&egress_needed_key)) {
4435 if (nf_hook_egress_active()) {
4436 skb = nf_hook_egress(skb, &rc, dev);
4437 if (!skb)
4438 goto out;
4439 }
4440
4441 netdev_xmit_skip_txqueue(false);
4442
4443 nf_skip_egress(skb, true);
4444 skb = sch_handle_egress(skb, &rc, dev);
4445 if (!skb)
4446 goto out;
4447 nf_skip_egress(skb, false);
4448
4449 if (netdev_xmit_txqueue_skipped())
4450 txq = netdev_tx_queue_mapping(dev, skb);
4451 }
4452#endif
4453 /* If device/qdisc don't need skb->dst, release it right now while
4454 * its hot in this cpu cache.
4455 */
4456 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4457 skb_dst_drop(skb);
4458 else
4459 skb_dst_force(skb);
4460
4461 if (!txq)
4462 txq = netdev_core_pick_tx(dev, skb, sb_dev);
4463
4464 q = rcu_dereference_bh(txq->qdisc);
4465
4466 trace_net_dev_queue(skb);
4467 if (q->enqueue) {
4468 rc = __dev_xmit_skb(skb, q, dev, txq);
4469 goto out;
4470 }
4471
4472 /* The device has no queue. Common case for software devices:
4473 * loopback, all the sorts of tunnels...
4474
4475 * Really, it is unlikely that netif_tx_lock protection is necessary
4476 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
4477 * counters.)
4478 * However, it is possible, that they rely on protection
4479 * made by us here.
4480
4481 * Check this and shot the lock. It is not prone from deadlocks.
4482 *Either shot noqueue qdisc, it is even simpler 8)
4483 */
4484 if (dev->flags & IFF_UP) {
4485 int cpu = smp_processor_id(); /* ok because BHs are off */
4486
4487 /* Other cpus might concurrently change txq->xmit_lock_owner
4488 * to -1 or to their cpu id, but not to our id.
4489 */
4490 if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
4491 if (dev_xmit_recursion())
4492 goto recursion_alert;
4493
4494 skb = validate_xmit_skb(skb, dev, &again);
4495 if (!skb)
4496 goto out;
4497
4498 HARD_TX_LOCK(dev, txq, cpu);
4499
4500 if (!netif_xmit_stopped(txq)) {
4501 dev_xmit_recursion_inc();
4502 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4503 dev_xmit_recursion_dec();
4504 if (dev_xmit_complete(rc)) {
4505 HARD_TX_UNLOCK(dev, txq);
4506 goto out;
4507 }
4508 }
4509 HARD_TX_UNLOCK(dev, txq);
4510 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4511 dev->name);
4512 } else {
4513 /* Recursion is detected! It is possible,
4514 * unfortunately
4515 */
4516recursion_alert:
4517 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4518 dev->name);
4519 }
4520 }
4521
4522 rc = -ENETDOWN;
4523 rcu_read_unlock_bh();
4524
4525 dev_core_stats_tx_dropped_inc(dev);
4526 kfree_skb_list(skb);
4527 return rc;
4528out:
4529 rcu_read_unlock_bh();
4530 return rc;
4531}
4532EXPORT_SYMBOL(__dev_queue_xmit);
4533
4534int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4535{
4536 struct net_device *dev = skb->dev;
4537 struct sk_buff *orig_skb = skb;
4538 struct netdev_queue *txq;
4539 int ret = NETDEV_TX_BUSY;
4540 bool again = false;
4541
4542 if (unlikely(!netif_running(dev) ||
4543 !netif_carrier_ok(dev)))
4544 goto drop;
4545
4546 skb = validate_xmit_skb_list(skb, dev, &again);
4547 if (skb != orig_skb)
4548 goto drop;
4549
4550 skb_set_queue_mapping(skb, queue_id);
4551 txq = skb_get_tx_queue(dev, skb);
4552
4553 local_bh_disable();
4554
4555 dev_xmit_recursion_inc();
4556 HARD_TX_LOCK(dev, txq, smp_processor_id());
4557 if (!netif_xmit_frozen_or_drv_stopped(txq))
4558 ret = netdev_start_xmit(skb, dev, txq, false);
4559 HARD_TX_UNLOCK(dev, txq);
4560 dev_xmit_recursion_dec();
4561
4562 local_bh_enable();
4563 return ret;
4564drop:
4565 dev_core_stats_tx_dropped_inc(dev);
4566 kfree_skb_list(skb);
4567 return NET_XMIT_DROP;
4568}
4569EXPORT_SYMBOL(__dev_direct_xmit);
4570
4571/*************************************************************************
4572 * Receiver routines
4573 *************************************************************************/
4574static DEFINE_PER_CPU(struct task_struct *, backlog_napi);
4575
4576int weight_p __read_mostly = 64; /* old backlog weight */
4577int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
4578int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
4579
4580/* Called with irq disabled */
4581static inline void ____napi_schedule(struct softnet_data *sd,
4582 struct napi_struct *napi)
4583{
4584 struct task_struct *thread;
4585
4586 lockdep_assert_irqs_disabled();
4587
4588 if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4589 /* Paired with smp_mb__before_atomic() in
4590 * napi_enable()/dev_set_threaded().
4591 * Use READ_ONCE() to guarantee a complete
4592 * read on napi->thread. Only call
4593 * wake_up_process() when it's not NULL.
4594 */
4595 thread = READ_ONCE(napi->thread);
4596 if (thread) {
4597 if (use_backlog_threads() && thread == raw_cpu_read(backlog_napi))
4598 goto use_local_napi;
4599
4600 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4601 wake_up_process(thread);
4602 return;
4603 }
4604 }
4605
4606use_local_napi:
4607 list_add_tail(&napi->poll_list, &sd->poll_list);
4608 WRITE_ONCE(napi->list_owner, smp_processor_id());
4609 /* If not called from net_rx_action()
4610 * we have to raise NET_RX_SOFTIRQ.
4611 */
4612 if (!sd->in_net_rx_action)
4613 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4614}
4615
4616#ifdef CONFIG_RPS
4617
4618struct static_key_false rps_needed __read_mostly;
4619EXPORT_SYMBOL(rps_needed);
4620struct static_key_false rfs_needed __read_mostly;
4621EXPORT_SYMBOL(rfs_needed);
4622
4623static struct rps_dev_flow *
4624set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4625 struct rps_dev_flow *rflow, u16 next_cpu)
4626{
4627 if (next_cpu < nr_cpu_ids) {
4628 u32 head;
4629#ifdef CONFIG_RFS_ACCEL
4630 struct netdev_rx_queue *rxqueue;
4631 struct rps_dev_flow_table *flow_table;
4632 struct rps_dev_flow *old_rflow;
4633 u16 rxq_index;
4634 u32 flow_id;
4635 int rc;
4636
4637 /* Should we steer this flow to a different hardware queue? */
4638 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4639 !(dev->features & NETIF_F_NTUPLE))
4640 goto out;
4641 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4642 if (rxq_index == skb_get_rx_queue(skb))
4643 goto out;
4644
4645 rxqueue = dev->_rx + rxq_index;
4646 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4647 if (!flow_table)
4648 goto out;
4649 flow_id = skb_get_hash(skb) & flow_table->mask;
4650 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4651 rxq_index, flow_id);
4652 if (rc < 0)
4653 goto out;
4654 old_rflow = rflow;
4655 rflow = &flow_table->flows[flow_id];
4656 WRITE_ONCE(rflow->filter, rc);
4657 if (old_rflow->filter == rc)
4658 WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER);
4659 out:
4660#endif
4661 head = READ_ONCE(per_cpu(softnet_data, next_cpu).input_queue_head);
4662 rps_input_queue_tail_save(&rflow->last_qtail, head);
4663 }
4664
4665 WRITE_ONCE(rflow->cpu, next_cpu);
4666 return rflow;
4667}
4668
4669/*
4670 * get_rps_cpu is called from netif_receive_skb and returns the target
4671 * CPU from the RPS map of the receiving queue for a given skb.
4672 * rcu_read_lock must be held on entry.
4673 */
4674static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4675 struct rps_dev_flow **rflowp)
4676{
4677 const struct rps_sock_flow_table *sock_flow_table;
4678 struct netdev_rx_queue *rxqueue = dev->_rx;
4679 struct rps_dev_flow_table *flow_table;
4680 struct rps_map *map;
4681 int cpu = -1;
4682 u32 tcpu;
4683 u32 hash;
4684
4685 if (skb_rx_queue_recorded(skb)) {
4686 u16 index = skb_get_rx_queue(skb);
4687
4688 if (unlikely(index >= dev->real_num_rx_queues)) {
4689 WARN_ONCE(dev->real_num_rx_queues > 1,
4690 "%s received packet on queue %u, but number "
4691 "of RX queues is %u\n",
4692 dev->name, index, dev->real_num_rx_queues);
4693 goto done;
4694 }
4695 rxqueue += index;
4696 }
4697
4698 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4699
4700 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4701 map = rcu_dereference(rxqueue->rps_map);
4702 if (!flow_table && !map)
4703 goto done;
4704
4705 skb_reset_network_header(skb);
4706 hash = skb_get_hash(skb);
4707 if (!hash)
4708 goto done;
4709
4710 sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
4711 if (flow_table && sock_flow_table) {
4712 struct rps_dev_flow *rflow;
4713 u32 next_cpu;
4714 u32 ident;
4715
4716 /* First check into global flow table if there is a match.
4717 * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
4718 */
4719 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);
4720 if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask)
4721 goto try_rps;
4722
4723 next_cpu = ident & net_hotdata.rps_cpu_mask;
4724
4725 /* OK, now we know there is a match,
4726 * we can look at the local (per receive queue) flow table
4727 */
4728 rflow = &flow_table->flows[hash & flow_table->mask];
4729 tcpu = rflow->cpu;
4730
4731 /*
4732 * If the desired CPU (where last recvmsg was done) is
4733 * different from current CPU (one in the rx-queue flow
4734 * table entry), switch if one of the following holds:
4735 * - Current CPU is unset (>= nr_cpu_ids).
4736 * - Current CPU is offline.
4737 * - The current CPU's queue tail has advanced beyond the
4738 * last packet that was enqueued using this table entry.
4739 * This guarantees that all previous packets for the flow
4740 * have been dequeued, thus preserving in order delivery.
4741 */
4742 if (unlikely(tcpu != next_cpu) &&
4743 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4744 ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) -
4745 rflow->last_qtail)) >= 0)) {
4746 tcpu = next_cpu;
4747 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4748 }
4749
4750 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4751 *rflowp = rflow;
4752 cpu = tcpu;
4753 goto done;
4754 }
4755 }
4756
4757try_rps:
4758
4759 if (map) {
4760 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4761 if (cpu_online(tcpu)) {
4762 cpu = tcpu;
4763 goto done;
4764 }
4765 }
4766
4767done:
4768 return cpu;
4769}
4770
4771#ifdef CONFIG_RFS_ACCEL
4772
4773/**
4774 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4775 * @dev: Device on which the filter was set
4776 * @rxq_index: RX queue index
4777 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4778 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4779 *
4780 * Drivers that implement ndo_rx_flow_steer() should periodically call
4781 * this function for each installed filter and remove the filters for
4782 * which it returns %true.
4783 */
4784bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4785 u32 flow_id, u16 filter_id)
4786{
4787 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4788 struct rps_dev_flow_table *flow_table;
4789 struct rps_dev_flow *rflow;
4790 bool expire = true;
4791 unsigned int cpu;
4792
4793 rcu_read_lock();
4794 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4795 if (flow_table && flow_id <= flow_table->mask) {
4796 rflow = &flow_table->flows[flow_id];
4797 cpu = READ_ONCE(rflow->cpu);
4798 if (READ_ONCE(rflow->filter) == filter_id && cpu < nr_cpu_ids &&
4799 ((int)(READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head) -
4800 READ_ONCE(rflow->last_qtail)) <
4801 (int)(10 * flow_table->mask)))
4802 expire = false;
4803 }
4804 rcu_read_unlock();
4805 return expire;
4806}
4807EXPORT_SYMBOL(rps_may_expire_flow);
4808
4809#endif /* CONFIG_RFS_ACCEL */
4810
4811/* Called from hardirq (IPI) context */
4812static void rps_trigger_softirq(void *data)
4813{
4814 struct softnet_data *sd = data;
4815
4816 ____napi_schedule(sd, &sd->backlog);
4817 sd->received_rps++;
4818}
4819
4820#endif /* CONFIG_RPS */
4821
4822/* Called from hardirq (IPI) context */
4823static void trigger_rx_softirq(void *data)
4824{
4825 struct softnet_data *sd = data;
4826
4827 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4828 smp_store_release(&sd->defer_ipi_scheduled, 0);
4829}
4830
4831/*
4832 * After we queued a packet into sd->input_pkt_queue,
4833 * we need to make sure this queue is serviced soon.
4834 *
4835 * - If this is another cpu queue, link it to our rps_ipi_list,
4836 * and make sure we will process rps_ipi_list from net_rx_action().
4837 *
4838 * - If this is our own queue, NAPI schedule our backlog.
4839 * Note that this also raises NET_RX_SOFTIRQ.
4840 */
4841static void napi_schedule_rps(struct softnet_data *sd)
4842{
4843 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4844
4845#ifdef CONFIG_RPS
4846 if (sd != mysd) {
4847 if (use_backlog_threads()) {
4848 __napi_schedule_irqoff(&sd->backlog);
4849 return;
4850 }
4851
4852 sd->rps_ipi_next = mysd->rps_ipi_list;
4853 mysd->rps_ipi_list = sd;
4854
4855 /* If not called from net_rx_action() or napi_threaded_poll()
4856 * we have to raise NET_RX_SOFTIRQ.
4857 */
4858 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll)
4859 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4860 return;
4861 }
4862#endif /* CONFIG_RPS */
4863 __napi_schedule_irqoff(&mysd->backlog);
4864}
4865
4866void kick_defer_list_purge(struct softnet_data *sd, unsigned int cpu)
4867{
4868 unsigned long flags;
4869
4870 if (use_backlog_threads()) {
4871 backlog_lock_irq_save(sd, &flags);
4872
4873 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
4874 __napi_schedule_irqoff(&sd->backlog);
4875
4876 backlog_unlock_irq_restore(sd, &flags);
4877
4878 } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) {
4879 smp_call_function_single_async(cpu, &sd->defer_csd);
4880 }
4881}
4882
4883#ifdef CONFIG_NET_FLOW_LIMIT
4884int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4885#endif
4886
4887static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4888{
4889#ifdef CONFIG_NET_FLOW_LIMIT
4890 struct sd_flow_limit *fl;
4891 struct softnet_data *sd;
4892 unsigned int old_flow, new_flow;
4893
4894 if (qlen < (READ_ONCE(net_hotdata.max_backlog) >> 1))
4895 return false;
4896
4897 sd = this_cpu_ptr(&softnet_data);
4898
4899 rcu_read_lock();
4900 fl = rcu_dereference(sd->flow_limit);
4901 if (fl) {
4902 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4903 old_flow = fl->history[fl->history_head];
4904 fl->history[fl->history_head] = new_flow;
4905
4906 fl->history_head++;
4907 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4908
4909 if (likely(fl->buckets[old_flow]))
4910 fl->buckets[old_flow]--;
4911
4912 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4913 fl->count++;
4914 rcu_read_unlock();
4915 return true;
4916 }
4917 }
4918 rcu_read_unlock();
4919#endif
4920 return false;
4921}
4922
4923/*
4924 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4925 * queue (may be a remote CPU queue).
4926 */
4927static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4928 unsigned int *qtail)
4929{
4930 enum skb_drop_reason reason;
4931 struct softnet_data *sd;
4932 unsigned long flags;
4933 unsigned int qlen;
4934 int max_backlog;
4935 u32 tail;
4936
4937 reason = SKB_DROP_REASON_DEV_READY;
4938 if (!netif_running(skb->dev))
4939 goto bad_dev;
4940
4941 reason = SKB_DROP_REASON_CPU_BACKLOG;
4942 sd = &per_cpu(softnet_data, cpu);
4943
4944 qlen = skb_queue_len_lockless(&sd->input_pkt_queue);
4945 max_backlog = READ_ONCE(net_hotdata.max_backlog);
4946 if (unlikely(qlen > max_backlog))
4947 goto cpu_backlog_drop;
4948 backlog_lock_irq_save(sd, &flags);
4949 qlen = skb_queue_len(&sd->input_pkt_queue);
4950 if (qlen <= max_backlog && !skb_flow_limit(skb, qlen)) {
4951 if (!qlen) {
4952 /* Schedule NAPI for backlog device. We can use
4953 * non atomic operation as we own the queue lock.
4954 */
4955 if (!__test_and_set_bit(NAPI_STATE_SCHED,
4956 &sd->backlog.state))
4957 napi_schedule_rps(sd);
4958 }
4959 __skb_queue_tail(&sd->input_pkt_queue, skb);
4960 tail = rps_input_queue_tail_incr(sd);
4961 backlog_unlock_irq_restore(sd, &flags);
4962
4963 /* save the tail outside of the critical section */
4964 rps_input_queue_tail_save(qtail, tail);
4965 return NET_RX_SUCCESS;
4966 }
4967
4968 backlog_unlock_irq_restore(sd, &flags);
4969
4970cpu_backlog_drop:
4971 atomic_inc(&sd->dropped);
4972bad_dev:
4973 dev_core_stats_rx_dropped_inc(skb->dev);
4974 kfree_skb_reason(skb, reason);
4975 return NET_RX_DROP;
4976}
4977
4978static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4979{
4980 struct net_device *dev = skb->dev;
4981 struct netdev_rx_queue *rxqueue;
4982
4983 rxqueue = dev->_rx;
4984
4985 if (skb_rx_queue_recorded(skb)) {
4986 u16 index = skb_get_rx_queue(skb);
4987
4988 if (unlikely(index >= dev->real_num_rx_queues)) {
4989 WARN_ONCE(dev->real_num_rx_queues > 1,
4990 "%s received packet on queue %u, but number "
4991 "of RX queues is %u\n",
4992 dev->name, index, dev->real_num_rx_queues);
4993
4994 return rxqueue; /* Return first rxqueue */
4995 }
4996 rxqueue += index;
4997 }
4998 return rxqueue;
4999}
5000
5001u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
5002 struct bpf_prog *xdp_prog)
5003{
5004 void *orig_data, *orig_data_end, *hard_start;
5005 struct netdev_rx_queue *rxqueue;
5006 bool orig_bcast, orig_host;
5007 u32 mac_len, frame_sz;
5008 __be16 orig_eth_type;
5009 struct ethhdr *eth;
5010 u32 metalen, act;
5011 int off;
5012
5013 /* The XDP program wants to see the packet starting at the MAC
5014 * header.
5015 */
5016 mac_len = skb->data - skb_mac_header(skb);
5017 hard_start = skb->data - skb_headroom(skb);
5018
5019 /* SKB "head" area always have tailroom for skb_shared_info */
5020 frame_sz = (void *)skb_end_pointer(skb) - hard_start;
5021 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
5022
5023 rxqueue = netif_get_rxqueue(skb);
5024 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
5025 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
5026 skb_headlen(skb) + mac_len, true);
5027 if (skb_is_nonlinear(skb)) {
5028 skb_shinfo(skb)->xdp_frags_size = skb->data_len;
5029 xdp_buff_set_frags_flag(xdp);
5030 } else {
5031 xdp_buff_clear_frags_flag(xdp);
5032 }
5033
5034 orig_data_end = xdp->data_end;
5035 orig_data = xdp->data;
5036 eth = (struct ethhdr *)xdp->data;
5037 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
5038 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
5039 orig_eth_type = eth->h_proto;
5040
5041 act = bpf_prog_run_xdp(xdp_prog, xdp);
5042
5043 /* check if bpf_xdp_adjust_head was used */
5044 off = xdp->data - orig_data;
5045 if (off) {
5046 if (off > 0)
5047 __skb_pull(skb, off);
5048 else if (off < 0)
5049 __skb_push(skb, -off);
5050
5051 skb->mac_header += off;
5052 skb_reset_network_header(skb);
5053 }
5054
5055 /* check if bpf_xdp_adjust_tail was used */
5056 off = xdp->data_end - orig_data_end;
5057 if (off != 0) {
5058 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
5059 skb->len += off; /* positive on grow, negative on shrink */
5060 }
5061
5062 /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers
5063 * (e.g. bpf_xdp_adjust_tail), we need to update data_len here.
5064 */
5065 if (xdp_buff_has_frags(xdp))
5066 skb->data_len = skb_shinfo(skb)->xdp_frags_size;
5067 else
5068 skb->data_len = 0;
5069
5070 /* check if XDP changed eth hdr such SKB needs update */
5071 eth = (struct ethhdr *)xdp->data;
5072 if ((orig_eth_type != eth->h_proto) ||
5073 (orig_host != ether_addr_equal_64bits(eth->h_dest,
5074 skb->dev->dev_addr)) ||
5075 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
5076 __skb_push(skb, ETH_HLEN);
5077 skb->pkt_type = PACKET_HOST;
5078 skb->protocol = eth_type_trans(skb, skb->dev);
5079 }
5080
5081 /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
5082 * before calling us again on redirect path. We do not call do_redirect
5083 * as we leave that up to the caller.
5084 *
5085 * Caller is responsible for managing lifetime of skb (i.e. calling
5086 * kfree_skb in response to actions it cannot handle/XDP_DROP).
5087 */
5088 switch (act) {
5089 case XDP_REDIRECT:
5090 case XDP_TX:
5091 __skb_push(skb, mac_len);
5092 break;
5093 case XDP_PASS:
5094 metalen = xdp->data - xdp->data_meta;
5095 if (metalen)
5096 skb_metadata_set(skb, metalen);
5097 break;
5098 }
5099
5100 return act;
5101}
5102
5103static int
5104netif_skb_check_for_xdp(struct sk_buff **pskb, struct bpf_prog *prog)
5105{
5106 struct sk_buff *skb = *pskb;
5107 int err, hroom, troom;
5108
5109 if (!skb_cow_data_for_xdp(this_cpu_read(system_page_pool), pskb, prog))
5110 return 0;
5111
5112 /* In case we have to go down the path and also linearize,
5113 * then lets do the pskb_expand_head() work just once here.
5114 */
5115 hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
5116 troom = skb->tail + skb->data_len - skb->end;
5117 err = pskb_expand_head(skb,
5118 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
5119 troom > 0 ? troom + 128 : 0, GFP_ATOMIC);
5120 if (err)
5121 return err;
5122
5123 return skb_linearize(skb);
5124}
5125
5126static u32 netif_receive_generic_xdp(struct sk_buff **pskb,
5127 struct xdp_buff *xdp,
5128 struct bpf_prog *xdp_prog)
5129{
5130 struct sk_buff *skb = *pskb;
5131 u32 mac_len, act = XDP_DROP;
5132
5133 /* Reinjected packets coming from act_mirred or similar should
5134 * not get XDP generic processing.
5135 */
5136 if (skb_is_redirected(skb))
5137 return XDP_PASS;
5138
5139 /* XDP packets must have sufficient headroom of XDP_PACKET_HEADROOM
5140 * bytes. This is the guarantee that also native XDP provides,
5141 * thus we need to do it here as well.
5142 */
5143 mac_len = skb->data - skb_mac_header(skb);
5144 __skb_push(skb, mac_len);
5145
5146 if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
5147 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
5148 if (netif_skb_check_for_xdp(pskb, xdp_prog))
5149 goto do_drop;
5150 }
5151
5152 __skb_pull(*pskb, mac_len);
5153
5154 act = bpf_prog_run_generic_xdp(*pskb, xdp, xdp_prog);
5155 switch (act) {
5156 case XDP_REDIRECT:
5157 case XDP_TX:
5158 case XDP_PASS:
5159 break;
5160 default:
5161 bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act);
5162 fallthrough;
5163 case XDP_ABORTED:
5164 trace_xdp_exception((*pskb)->dev, xdp_prog, act);
5165 fallthrough;
5166 case XDP_DROP:
5167 do_drop:
5168 kfree_skb(*pskb);
5169 break;
5170 }
5171
5172 return act;
5173}
5174
5175/* When doing generic XDP we have to bypass the qdisc layer and the
5176 * network taps in order to match in-driver-XDP behavior. This also means
5177 * that XDP packets are able to starve other packets going through a qdisc,
5178 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
5179 * queues, so they do not have this starvation issue.
5180 */
5181void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
5182{
5183 struct net_device *dev = skb->dev;
5184 struct netdev_queue *txq;
5185 bool free_skb = true;
5186 int cpu, rc;
5187
5188 txq = netdev_core_pick_tx(dev, skb, NULL);
5189 cpu = smp_processor_id();
5190 HARD_TX_LOCK(dev, txq, cpu);
5191 if (!netif_xmit_frozen_or_drv_stopped(txq)) {
5192 rc = netdev_start_xmit(skb, dev, txq, 0);
5193 if (dev_xmit_complete(rc))
5194 free_skb = false;
5195 }
5196 HARD_TX_UNLOCK(dev, txq);
5197 if (free_skb) {
5198 trace_xdp_exception(dev, xdp_prog, XDP_TX);
5199 dev_core_stats_tx_dropped_inc(dev);
5200 kfree_skb(skb);
5201 }
5202}
5203
5204static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
5205
5206int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb)
5207{
5208 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
5209
5210 if (xdp_prog) {
5211 struct xdp_buff xdp;
5212 u32 act;
5213 int err;
5214
5215 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
5216 act = netif_receive_generic_xdp(pskb, &xdp, xdp_prog);
5217 if (act != XDP_PASS) {
5218 switch (act) {
5219 case XDP_REDIRECT:
5220 err = xdp_do_generic_redirect((*pskb)->dev, *pskb,
5221 &xdp, xdp_prog);
5222 if (err)
5223 goto out_redir;
5224 break;
5225 case XDP_TX:
5226 generic_xdp_tx(*pskb, xdp_prog);
5227 break;
5228 }
5229 bpf_net_ctx_clear(bpf_net_ctx);
5230 return XDP_DROP;
5231 }
5232 bpf_net_ctx_clear(bpf_net_ctx);
5233 }
5234 return XDP_PASS;
5235out_redir:
5236 bpf_net_ctx_clear(bpf_net_ctx);
5237 kfree_skb_reason(*pskb, SKB_DROP_REASON_XDP);
5238 return XDP_DROP;
5239}
5240EXPORT_SYMBOL_GPL(do_xdp_generic);
5241
5242static int netif_rx_internal(struct sk_buff *skb)
5243{
5244 int ret;
5245
5246 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5247
5248 trace_netif_rx(skb);
5249
5250#ifdef CONFIG_RPS
5251 if (static_branch_unlikely(&rps_needed)) {
5252 struct rps_dev_flow voidflow, *rflow = &voidflow;
5253 int cpu;
5254
5255 rcu_read_lock();
5256
5257 cpu = get_rps_cpu(skb->dev, skb, &rflow);
5258 if (cpu < 0)
5259 cpu = smp_processor_id();
5260
5261 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5262
5263 rcu_read_unlock();
5264 } else
5265#endif
5266 {
5267 unsigned int qtail;
5268
5269 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
5270 }
5271 return ret;
5272}
5273
5274/**
5275 * __netif_rx - Slightly optimized version of netif_rx
5276 * @skb: buffer to post
5277 *
5278 * This behaves as netif_rx except that it does not disable bottom halves.
5279 * As a result this function may only be invoked from the interrupt context
5280 * (either hard or soft interrupt).
5281 */
5282int __netif_rx(struct sk_buff *skb)
5283{
5284 int ret;
5285
5286 lockdep_assert_once(hardirq_count() | softirq_count());
5287
5288 trace_netif_rx_entry(skb);
5289 ret = netif_rx_internal(skb);
5290 trace_netif_rx_exit(ret);
5291 return ret;
5292}
5293EXPORT_SYMBOL(__netif_rx);
5294
5295/**
5296 * netif_rx - post buffer to the network code
5297 * @skb: buffer to post
5298 *
5299 * This function receives a packet from a device driver and queues it for
5300 * the upper (protocol) levels to process via the backlog NAPI device. It
5301 * always succeeds. The buffer may be dropped during processing for
5302 * congestion control or by the protocol layers.
5303 * The network buffer is passed via the backlog NAPI device. Modern NIC
5304 * driver should use NAPI and GRO.
5305 * This function can used from interrupt and from process context. The
5306 * caller from process context must not disable interrupts before invoking
5307 * this function.
5308 *
5309 * return values:
5310 * NET_RX_SUCCESS (no congestion)
5311 * NET_RX_DROP (packet was dropped)
5312 *
5313 */
5314int netif_rx(struct sk_buff *skb)
5315{
5316 bool need_bh_off = !(hardirq_count() | softirq_count());
5317 int ret;
5318
5319 if (need_bh_off)
5320 local_bh_disable();
5321 trace_netif_rx_entry(skb);
5322 ret = netif_rx_internal(skb);
5323 trace_netif_rx_exit(ret);
5324 if (need_bh_off)
5325 local_bh_enable();
5326 return ret;
5327}
5328EXPORT_SYMBOL(netif_rx);
5329
5330static __latent_entropy void net_tx_action(void)
5331{
5332 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5333
5334 if (sd->completion_queue) {
5335 struct sk_buff *clist;
5336
5337 local_irq_disable();
5338 clist = sd->completion_queue;
5339 sd->completion_queue = NULL;
5340 local_irq_enable();
5341
5342 while (clist) {
5343 struct sk_buff *skb = clist;
5344
5345 clist = clist->next;
5346
5347 WARN_ON(refcount_read(&skb->users));
5348 if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED))
5349 trace_consume_skb(skb, net_tx_action);
5350 else
5351 trace_kfree_skb(skb, net_tx_action,
5352 get_kfree_skb_cb(skb)->reason, NULL);
5353
5354 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5355 __kfree_skb(skb);
5356 else
5357 __napi_kfree_skb(skb,
5358 get_kfree_skb_cb(skb)->reason);
5359 }
5360 }
5361
5362 if (sd->output_queue) {
5363 struct Qdisc *head;
5364
5365 local_irq_disable();
5366 head = sd->output_queue;
5367 sd->output_queue = NULL;
5368 sd->output_queue_tailp = &sd->output_queue;
5369 local_irq_enable();
5370
5371 rcu_read_lock();
5372
5373 while (head) {
5374 struct Qdisc *q = head;
5375 spinlock_t *root_lock = NULL;
5376
5377 head = head->next_sched;
5378
5379 /* We need to make sure head->next_sched is read
5380 * before clearing __QDISC_STATE_SCHED
5381 */
5382 smp_mb__before_atomic();
5383
5384 if (!(q->flags & TCQ_F_NOLOCK)) {
5385 root_lock = qdisc_lock(q);
5386 spin_lock(root_lock);
5387 } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5388 &q->state))) {
5389 /* There is a synchronize_net() between
5390 * STATE_DEACTIVATED flag being set and
5391 * qdisc_reset()/some_qdisc_is_busy() in
5392 * dev_deactivate(), so we can safely bail out
5393 * early here to avoid data race between
5394 * qdisc_deactivate() and some_qdisc_is_busy()
5395 * for lockless qdisc.
5396 */
5397 clear_bit(__QDISC_STATE_SCHED, &q->state);
5398 continue;
5399 }
5400
5401 clear_bit(__QDISC_STATE_SCHED, &q->state);
5402 qdisc_run(q);
5403 if (root_lock)
5404 spin_unlock(root_lock);
5405 }
5406
5407 rcu_read_unlock();
5408 }
5409
5410 xfrm_dev_backlog(sd);
5411}
5412
5413#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
5414/* This hook is defined here for ATM LANE */
5415int (*br_fdb_test_addr_hook)(struct net_device *dev,
5416 unsigned char *addr) __read_mostly;
5417EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
5418#endif
5419
5420/**
5421 * netdev_is_rx_handler_busy - check if receive handler is registered
5422 * @dev: device to check
5423 *
5424 * Check if a receive handler is already registered for a given device.
5425 * Return true if there one.
5426 *
5427 * The caller must hold the rtnl_mutex.
5428 */
5429bool netdev_is_rx_handler_busy(struct net_device *dev)
5430{
5431 ASSERT_RTNL();
5432 return dev && rtnl_dereference(dev->rx_handler);
5433}
5434EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5435
5436/**
5437 * netdev_rx_handler_register - register receive handler
5438 * @dev: device to register a handler for
5439 * @rx_handler: receive handler to register
5440 * @rx_handler_data: data pointer that is used by rx handler
5441 *
5442 * Register a receive handler for a device. This handler will then be
5443 * called from __netif_receive_skb. A negative errno code is returned
5444 * on a failure.
5445 *
5446 * The caller must hold the rtnl_mutex.
5447 *
5448 * For a general description of rx_handler, see enum rx_handler_result.
5449 */
5450int netdev_rx_handler_register(struct net_device *dev,
5451 rx_handler_func_t *rx_handler,
5452 void *rx_handler_data)
5453{
5454 if (netdev_is_rx_handler_busy(dev))
5455 return -EBUSY;
5456
5457 if (dev->priv_flags & IFF_NO_RX_HANDLER)
5458 return -EINVAL;
5459
5460 /* Note: rx_handler_data must be set before rx_handler */
5461 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
5462 rcu_assign_pointer(dev->rx_handler, rx_handler);
5463
5464 return 0;
5465}
5466EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5467
5468/**
5469 * netdev_rx_handler_unregister - unregister receive handler
5470 * @dev: device to unregister a handler from
5471 *
5472 * Unregister a receive handler from a device.
5473 *
5474 * The caller must hold the rtnl_mutex.
5475 */
5476void netdev_rx_handler_unregister(struct net_device *dev)
5477{
5478
5479 ASSERT_RTNL();
5480 RCU_INIT_POINTER(dev->rx_handler, NULL);
5481 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5482 * section has a guarantee to see a non NULL rx_handler_data
5483 * as well.
5484 */
5485 synchronize_net();
5486 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
5487}
5488EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5489
5490/*
5491 * Limit the use of PFMEMALLOC reserves to those protocols that implement
5492 * the special handling of PFMEMALLOC skbs.
5493 */
5494static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5495{
5496 switch (skb->protocol) {
5497 case htons(ETH_P_ARP):
5498 case htons(ETH_P_IP):
5499 case htons(ETH_P_IPV6):
5500 case htons(ETH_P_8021Q):
5501 case htons(ETH_P_8021AD):
5502 return true;
5503 default:
5504 return false;
5505 }
5506}
5507
5508static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5509 int *ret, struct net_device *orig_dev)
5510{
5511 if (nf_hook_ingress_active(skb)) {
5512 int ingress_retval;
5513
5514 if (*pt_prev) {
5515 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5516 *pt_prev = NULL;
5517 }
5518
5519 rcu_read_lock();
5520 ingress_retval = nf_hook_ingress(skb);
5521 rcu_read_unlock();
5522 return ingress_retval;
5523 }
5524 return 0;
5525}
5526
5527static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
5528 struct packet_type **ppt_prev)
5529{
5530 struct packet_type *ptype, *pt_prev;
5531 rx_handler_func_t *rx_handler;
5532 struct sk_buff *skb = *pskb;
5533 struct net_device *orig_dev;
5534 bool deliver_exact = false;
5535 int ret = NET_RX_DROP;
5536 __be16 type;
5537
5538 net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5539
5540 trace_netif_receive_skb(skb);
5541
5542 orig_dev = skb->dev;
5543
5544 skb_reset_network_header(skb);
5545 if (!skb_transport_header_was_set(skb))
5546 skb_reset_transport_header(skb);
5547 skb_reset_mac_len(skb);
5548
5549 pt_prev = NULL;
5550
5551another_round:
5552 skb->skb_iif = skb->dev->ifindex;
5553
5554 __this_cpu_inc(softnet_data.processed);
5555
5556 if (static_branch_unlikely(&generic_xdp_needed_key)) {
5557 int ret2;
5558
5559 migrate_disable();
5560 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog),
5561 &skb);
5562 migrate_enable();
5563
5564 if (ret2 != XDP_PASS) {
5565 ret = NET_RX_DROP;
5566 goto out;
5567 }
5568 }
5569
5570 if (eth_type_vlan(skb->protocol)) {
5571 skb = skb_vlan_untag(skb);
5572 if (unlikely(!skb))
5573 goto out;
5574 }
5575
5576 if (skb_skip_tc_classify(skb))
5577 goto skip_classify;
5578
5579 if (pfmemalloc)
5580 goto skip_taps;
5581
5582 list_for_each_entry_rcu(ptype, &net_hotdata.ptype_all, list) {
5583 if (pt_prev)
5584 ret = deliver_skb(skb, pt_prev, orig_dev);
5585 pt_prev = ptype;
5586 }
5587
5588 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5589 if (pt_prev)
5590 ret = deliver_skb(skb, pt_prev, orig_dev);
5591 pt_prev = ptype;
5592 }
5593
5594skip_taps:
5595#ifdef CONFIG_NET_INGRESS
5596 if (static_branch_unlikely(&ingress_needed_key)) {
5597 bool another = false;
5598
5599 nf_skip_egress(skb, true);
5600 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5601 &another);
5602 if (another)
5603 goto another_round;
5604 if (!skb)
5605 goto out;
5606
5607 nf_skip_egress(skb, false);
5608 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
5609 goto out;
5610 }
5611#endif
5612 skb_reset_redirect(skb);
5613skip_classify:
5614 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
5615 goto drop;
5616
5617 if (skb_vlan_tag_present(skb)) {
5618 if (pt_prev) {
5619 ret = deliver_skb(skb, pt_prev, orig_dev);
5620 pt_prev = NULL;
5621 }
5622 if (vlan_do_receive(&skb))
5623 goto another_round;
5624 else if (unlikely(!skb))
5625 goto out;
5626 }
5627
5628 rx_handler = rcu_dereference(skb->dev->rx_handler);
5629 if (rx_handler) {
5630 if (pt_prev) {
5631 ret = deliver_skb(skb, pt_prev, orig_dev);
5632 pt_prev = NULL;
5633 }
5634 switch (rx_handler(&skb)) {
5635 case RX_HANDLER_CONSUMED:
5636 ret = NET_RX_SUCCESS;
5637 goto out;
5638 case RX_HANDLER_ANOTHER:
5639 goto another_round;
5640 case RX_HANDLER_EXACT:
5641 deliver_exact = true;
5642 break;
5643 case RX_HANDLER_PASS:
5644 break;
5645 default:
5646 BUG();
5647 }
5648 }
5649
5650 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
5651check_vlan_id:
5652 if (skb_vlan_tag_get_id(skb)) {
5653 /* Vlan id is non 0 and vlan_do_receive() above couldn't
5654 * find vlan device.
5655 */
5656 skb->pkt_type = PACKET_OTHERHOST;
5657 } else if (eth_type_vlan(skb->protocol)) {
5658 /* Outer header is 802.1P with vlan 0, inner header is
5659 * 802.1Q or 802.1AD and vlan_do_receive() above could
5660 * not find vlan dev for vlan id 0.
5661 */
5662 __vlan_hwaccel_clear_tag(skb);
5663 skb = skb_vlan_untag(skb);
5664 if (unlikely(!skb))
5665 goto out;
5666 if (vlan_do_receive(&skb))
5667 /* After stripping off 802.1P header with vlan 0
5668 * vlan dev is found for inner header.
5669 */
5670 goto another_round;
5671 else if (unlikely(!skb))
5672 goto out;
5673 else
5674 /* We have stripped outer 802.1P vlan 0 header.
5675 * But could not find vlan dev.
5676 * check again for vlan id to set OTHERHOST.
5677 */
5678 goto check_vlan_id;
5679 }
5680 /* Note: we might in the future use prio bits
5681 * and set skb->priority like in vlan_do_receive()
5682 * For the time being, just ignore Priority Code Point
5683 */
5684 __vlan_hwaccel_clear_tag(skb);
5685 }
5686
5687 type = skb->protocol;
5688
5689 /* deliver only exact match when indicated */
5690 if (likely(!deliver_exact)) {
5691 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5692 &ptype_base[ntohs(type) &
5693 PTYPE_HASH_MASK]);
5694 }
5695
5696 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5697 &orig_dev->ptype_specific);
5698
5699 if (unlikely(skb->dev != orig_dev)) {
5700 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5701 &skb->dev->ptype_specific);
5702 }
5703
5704 if (pt_prev) {
5705 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
5706 goto drop;
5707 *ppt_prev = pt_prev;
5708 } else {
5709drop:
5710 if (!deliver_exact)
5711 dev_core_stats_rx_dropped_inc(skb->dev);
5712 else
5713 dev_core_stats_rx_nohandler_inc(skb->dev);
5714 kfree_skb_reason(skb, SKB_DROP_REASON_UNHANDLED_PROTO);
5715 /* Jamal, now you will not able to escape explaining
5716 * me how you were going to use this. :-)
5717 */
5718 ret = NET_RX_DROP;
5719 }
5720
5721out:
5722 /* The invariant here is that if *ppt_prev is not NULL
5723 * then skb should also be non-NULL.
5724 *
5725 * Apparently *ppt_prev assignment above holds this invariant due to
5726 * skb dereferencing near it.
5727 */
5728 *pskb = skb;
5729 return ret;
5730}
5731
5732static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5733{
5734 struct net_device *orig_dev = skb->dev;
5735 struct packet_type *pt_prev = NULL;
5736 int ret;
5737
5738 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5739 if (pt_prev)
5740 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5741 skb->dev, pt_prev, orig_dev);
5742 return ret;
5743}
5744
5745/**
5746 * netif_receive_skb_core - special purpose version of netif_receive_skb
5747 * @skb: buffer to process
5748 *
5749 * More direct receive version of netif_receive_skb(). It should
5750 * only be used by callers that have a need to skip RPS and Generic XDP.
5751 * Caller must also take care of handling if ``(page_is_)pfmemalloc``.
5752 *
5753 * This function may only be called from softirq context and interrupts
5754 * should be enabled.
5755 *
5756 * Return values (usually ignored):
5757 * NET_RX_SUCCESS: no congestion
5758 * NET_RX_DROP: packet was dropped
5759 */
5760int netif_receive_skb_core(struct sk_buff *skb)
5761{
5762 int ret;
5763
5764 rcu_read_lock();
5765 ret = __netif_receive_skb_one_core(skb, false);
5766 rcu_read_unlock();
5767
5768 return ret;
5769}
5770EXPORT_SYMBOL(netif_receive_skb_core);
5771
5772static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5773 struct packet_type *pt_prev,
5774 struct net_device *orig_dev)
5775{
5776 struct sk_buff *skb, *next;
5777
5778 if (!pt_prev)
5779 return;
5780 if (list_empty(head))
5781 return;
5782 if (pt_prev->list_func != NULL)
5783 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5784 ip_list_rcv, head, pt_prev, orig_dev);
5785 else
5786 list_for_each_entry_safe(skb, next, head, list) {
5787 skb_list_del_init(skb);
5788 pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5789 }
5790}
5791
5792static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5793{
5794 /* Fast-path assumptions:
5795 * - There is no RX handler.
5796 * - Only one packet_type matches.
5797 * If either of these fails, we will end up doing some per-packet
5798 * processing in-line, then handling the 'last ptype' for the whole
5799 * sublist. This can't cause out-of-order delivery to any single ptype,
5800 * because the 'last ptype' must be constant across the sublist, and all
5801 * other ptypes are handled per-packet.
5802 */
5803 /* Current (common) ptype of sublist */
5804 struct packet_type *pt_curr = NULL;
5805 /* Current (common) orig_dev of sublist */
5806 struct net_device *od_curr = NULL;
5807 struct sk_buff *skb, *next;
5808 LIST_HEAD(sublist);
5809
5810 list_for_each_entry_safe(skb, next, head, list) {
5811 struct net_device *orig_dev = skb->dev;
5812 struct packet_type *pt_prev = NULL;
5813
5814 skb_list_del_init(skb);
5815 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5816 if (!pt_prev)
5817 continue;
5818 if (pt_curr != pt_prev || od_curr != orig_dev) {
5819 /* dispatch old sublist */
5820 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5821 /* start new sublist */
5822 INIT_LIST_HEAD(&sublist);
5823 pt_curr = pt_prev;
5824 od_curr = orig_dev;
5825 }
5826 list_add_tail(&skb->list, &sublist);
5827 }
5828
5829 /* dispatch final sublist */
5830 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5831}
5832
5833static int __netif_receive_skb(struct sk_buff *skb)
5834{
5835 int ret;
5836
5837 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5838 unsigned int noreclaim_flag;
5839
5840 /*
5841 * PFMEMALLOC skbs are special, they should
5842 * - be delivered to SOCK_MEMALLOC sockets only
5843 * - stay away from userspace
5844 * - have bounded memory usage
5845 *
5846 * Use PF_MEMALLOC as this saves us from propagating the allocation
5847 * context down to all allocation sites.
5848 */
5849 noreclaim_flag = memalloc_noreclaim_save();
5850 ret = __netif_receive_skb_one_core(skb, true);
5851 memalloc_noreclaim_restore(noreclaim_flag);
5852 } else
5853 ret = __netif_receive_skb_one_core(skb, false);
5854
5855 return ret;
5856}
5857
5858static void __netif_receive_skb_list(struct list_head *head)
5859{
5860 unsigned long noreclaim_flag = 0;
5861 struct sk_buff *skb, *next;
5862 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5863
5864 list_for_each_entry_safe(skb, next, head, list) {
5865 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5866 struct list_head sublist;
5867
5868 /* Handle the previous sublist */
5869 list_cut_before(&sublist, head, &skb->list);
5870 if (!list_empty(&sublist))
5871 __netif_receive_skb_list_core(&sublist, pfmemalloc);
5872 pfmemalloc = !pfmemalloc;
5873 /* See comments in __netif_receive_skb */
5874 if (pfmemalloc)
5875 noreclaim_flag = memalloc_noreclaim_save();
5876 else
5877 memalloc_noreclaim_restore(noreclaim_flag);
5878 }
5879 }
5880 /* Handle the remaining sublist */
5881 if (!list_empty(head))
5882 __netif_receive_skb_list_core(head, pfmemalloc);
5883 /* Restore pflags */
5884 if (pfmemalloc)
5885 memalloc_noreclaim_restore(noreclaim_flag);
5886}
5887
5888static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5889{
5890 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5891 struct bpf_prog *new = xdp->prog;
5892 int ret = 0;
5893
5894 switch (xdp->command) {
5895 case XDP_SETUP_PROG:
5896 rcu_assign_pointer(dev->xdp_prog, new);
5897 if (old)
5898 bpf_prog_put(old);
5899
5900 if (old && !new) {
5901 static_branch_dec(&generic_xdp_needed_key);
5902 } else if (new && !old) {
5903 static_branch_inc(&generic_xdp_needed_key);
5904 dev_disable_lro(dev);
5905 dev_disable_gro_hw(dev);
5906 }
5907 break;
5908
5909 default:
5910 ret = -EINVAL;
5911 break;
5912 }
5913
5914 return ret;
5915}
5916
5917static int netif_receive_skb_internal(struct sk_buff *skb)
5918{
5919 int ret;
5920
5921 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5922
5923 if (skb_defer_rx_timestamp(skb))
5924 return NET_RX_SUCCESS;
5925
5926 rcu_read_lock();
5927#ifdef CONFIG_RPS
5928 if (static_branch_unlikely(&rps_needed)) {
5929 struct rps_dev_flow voidflow, *rflow = &voidflow;
5930 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5931
5932 if (cpu >= 0) {
5933 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5934 rcu_read_unlock();
5935 return ret;
5936 }
5937 }
5938#endif
5939 ret = __netif_receive_skb(skb);
5940 rcu_read_unlock();
5941 return ret;
5942}
5943
5944void netif_receive_skb_list_internal(struct list_head *head)
5945{
5946 struct sk_buff *skb, *next;
5947 LIST_HEAD(sublist);
5948
5949 list_for_each_entry_safe(skb, next, head, list) {
5950 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue),
5951 skb);
5952 skb_list_del_init(skb);
5953 if (!skb_defer_rx_timestamp(skb))
5954 list_add_tail(&skb->list, &sublist);
5955 }
5956 list_splice_init(&sublist, head);
5957
5958 rcu_read_lock();
5959#ifdef CONFIG_RPS
5960 if (static_branch_unlikely(&rps_needed)) {
5961 list_for_each_entry_safe(skb, next, head, list) {
5962 struct rps_dev_flow voidflow, *rflow = &voidflow;
5963 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5964
5965 if (cpu >= 0) {
5966 /* Will be handled, remove from list */
5967 skb_list_del_init(skb);
5968 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5969 }
5970 }
5971 }
5972#endif
5973 __netif_receive_skb_list(head);
5974 rcu_read_unlock();
5975}
5976
5977/**
5978 * netif_receive_skb - process receive buffer from network
5979 * @skb: buffer to process
5980 *
5981 * netif_receive_skb() is the main receive data processing function.
5982 * It always succeeds. The buffer may be dropped during processing
5983 * for congestion control or by the protocol layers.
5984 *
5985 * This function may only be called from softirq context and interrupts
5986 * should be enabled.
5987 *
5988 * Return values (usually ignored):
5989 * NET_RX_SUCCESS: no congestion
5990 * NET_RX_DROP: packet was dropped
5991 */
5992int netif_receive_skb(struct sk_buff *skb)
5993{
5994 int ret;
5995
5996 trace_netif_receive_skb_entry(skb);
5997
5998 ret = netif_receive_skb_internal(skb);
5999 trace_netif_receive_skb_exit(ret);
6000
6001 return ret;
6002}
6003EXPORT_SYMBOL(netif_receive_skb);
6004
6005/**
6006 * netif_receive_skb_list - process many receive buffers from network
6007 * @head: list of skbs to process.
6008 *
6009 * Since return value of netif_receive_skb() is normally ignored, and
6010 * wouldn't be meaningful for a list, this function returns void.
6011 *
6012 * This function may only be called from softirq context and interrupts
6013 * should be enabled.
6014 */
6015void netif_receive_skb_list(struct list_head *head)
6016{
6017 struct sk_buff *skb;
6018
6019 if (list_empty(head))
6020 return;
6021 if (trace_netif_receive_skb_list_entry_enabled()) {
6022 list_for_each_entry(skb, head, list)
6023 trace_netif_receive_skb_list_entry(skb);
6024 }
6025 netif_receive_skb_list_internal(head);
6026 trace_netif_receive_skb_list_exit(0);
6027}
6028EXPORT_SYMBOL(netif_receive_skb_list);
6029
6030static DEFINE_PER_CPU(struct work_struct, flush_works);
6031
6032/* Network device is going away, flush any packets still pending */
6033static void flush_backlog(struct work_struct *work)
6034{
6035 struct sk_buff *skb, *tmp;
6036 struct softnet_data *sd;
6037
6038 local_bh_disable();
6039 sd = this_cpu_ptr(&softnet_data);
6040
6041 backlog_lock_irq_disable(sd);
6042 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
6043 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
6044 __skb_unlink(skb, &sd->input_pkt_queue);
6045 dev_kfree_skb_irq(skb);
6046 rps_input_queue_head_incr(sd);
6047 }
6048 }
6049 backlog_unlock_irq_enable(sd);
6050
6051 local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6052 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
6053 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
6054 __skb_unlink(skb, &sd->process_queue);
6055 kfree_skb(skb);
6056 rps_input_queue_head_incr(sd);
6057 }
6058 }
6059 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6060 local_bh_enable();
6061}
6062
6063static bool flush_required(int cpu)
6064{
6065#if IS_ENABLED(CONFIG_RPS)
6066 struct softnet_data *sd = &per_cpu(softnet_data, cpu);
6067 bool do_flush;
6068
6069 backlog_lock_irq_disable(sd);
6070
6071 /* as insertion into process_queue happens with the rps lock held,
6072 * process_queue access may race only with dequeue
6073 */
6074 do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
6075 !skb_queue_empty_lockless(&sd->process_queue);
6076 backlog_unlock_irq_enable(sd);
6077
6078 return do_flush;
6079#endif
6080 /* without RPS we can't safely check input_pkt_queue: during a
6081 * concurrent remote skb_queue_splice() we can detect as empty both
6082 * input_pkt_queue and process_queue even if the latter could end-up
6083 * containing a lot of packets.
6084 */
6085 return true;
6086}
6087
6088static void flush_all_backlogs(void)
6089{
6090 static cpumask_t flush_cpus;
6091 unsigned int cpu;
6092
6093 /* since we are under rtnl lock protection we can use static data
6094 * for the cpumask and avoid allocating on stack the possibly
6095 * large mask
6096 */
6097 ASSERT_RTNL();
6098
6099 cpus_read_lock();
6100
6101 cpumask_clear(&flush_cpus);
6102 for_each_online_cpu(cpu) {
6103 if (flush_required(cpu)) {
6104 queue_work_on(cpu, system_highpri_wq,
6105 per_cpu_ptr(&flush_works, cpu));
6106 cpumask_set_cpu(cpu, &flush_cpus);
6107 }
6108 }
6109
6110 /* we can have in flight packet[s] on the cpus we are not flushing,
6111 * synchronize_net() in unregister_netdevice_many() will take care of
6112 * them
6113 */
6114 for_each_cpu(cpu, &flush_cpus)
6115 flush_work(per_cpu_ptr(&flush_works, cpu));
6116
6117 cpus_read_unlock();
6118}
6119
6120static void net_rps_send_ipi(struct softnet_data *remsd)
6121{
6122#ifdef CONFIG_RPS
6123 while (remsd) {
6124 struct softnet_data *next = remsd->rps_ipi_next;
6125
6126 if (cpu_online(remsd->cpu))
6127 smp_call_function_single_async(remsd->cpu, &remsd->csd);
6128 remsd = next;
6129 }
6130#endif
6131}
6132
6133/*
6134 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
6135 * Note: called with local irq disabled, but exits with local irq enabled.
6136 */
6137static void net_rps_action_and_irq_enable(struct softnet_data *sd)
6138{
6139#ifdef CONFIG_RPS
6140 struct softnet_data *remsd = sd->rps_ipi_list;
6141
6142 if (!use_backlog_threads() && remsd) {
6143 sd->rps_ipi_list = NULL;
6144
6145 local_irq_enable();
6146
6147 /* Send pending IPI's to kick RPS processing on remote cpus. */
6148 net_rps_send_ipi(remsd);
6149 } else
6150#endif
6151 local_irq_enable();
6152}
6153
6154static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
6155{
6156#ifdef CONFIG_RPS
6157 return !use_backlog_threads() && sd->rps_ipi_list;
6158#else
6159 return false;
6160#endif
6161}
6162
6163static int process_backlog(struct napi_struct *napi, int quota)
6164{
6165 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
6166 bool again = true;
6167 int work = 0;
6168
6169 /* Check if we have pending ipi, its better to send them now,
6170 * not waiting net_rx_action() end.
6171 */
6172 if (sd_has_rps_ipi_waiting(sd)) {
6173 local_irq_disable();
6174 net_rps_action_and_irq_enable(sd);
6175 }
6176
6177 napi->weight = READ_ONCE(net_hotdata.dev_rx_weight);
6178 while (again) {
6179 struct sk_buff *skb;
6180
6181 local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6182 while ((skb = __skb_dequeue(&sd->process_queue))) {
6183 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6184 rcu_read_lock();
6185 __netif_receive_skb(skb);
6186 rcu_read_unlock();
6187 if (++work >= quota) {
6188 rps_input_queue_head_add(sd, work);
6189 return work;
6190 }
6191
6192 local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6193 }
6194 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6195
6196 backlog_lock_irq_disable(sd);
6197 if (skb_queue_empty(&sd->input_pkt_queue)) {
6198 /*
6199 * Inline a custom version of __napi_complete().
6200 * only current cpu owns and manipulates this napi,
6201 * and NAPI_STATE_SCHED is the only possible flag set
6202 * on backlog.
6203 * We can use a plain write instead of clear_bit(),
6204 * and we dont need an smp_mb() memory barrier.
6205 */
6206 napi->state &= NAPIF_STATE_THREADED;
6207 again = false;
6208 } else {
6209 local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6210 skb_queue_splice_tail_init(&sd->input_pkt_queue,
6211 &sd->process_queue);
6212 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6213 }
6214 backlog_unlock_irq_enable(sd);
6215 }
6216
6217 if (work)
6218 rps_input_queue_head_add(sd, work);
6219 return work;
6220}
6221
6222/**
6223 * __napi_schedule - schedule for receive
6224 * @n: entry to schedule
6225 *
6226 * The entry's receive function will be scheduled to run.
6227 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
6228 */
6229void __napi_schedule(struct napi_struct *n)
6230{
6231 unsigned long flags;
6232
6233 local_irq_save(flags);
6234 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6235 local_irq_restore(flags);
6236}
6237EXPORT_SYMBOL(__napi_schedule);
6238
6239/**
6240 * napi_schedule_prep - check if napi can be scheduled
6241 * @n: napi context
6242 *
6243 * Test if NAPI routine is already running, and if not mark
6244 * it as running. This is used as a condition variable to
6245 * insure only one NAPI poll instance runs. We also make
6246 * sure there is no pending NAPI disable.
6247 */
6248bool napi_schedule_prep(struct napi_struct *n)
6249{
6250 unsigned long new, val = READ_ONCE(n->state);
6251
6252 do {
6253 if (unlikely(val & NAPIF_STATE_DISABLE))
6254 return false;
6255 new = val | NAPIF_STATE_SCHED;
6256
6257 /* Sets STATE_MISSED bit if STATE_SCHED was already set
6258 * This was suggested by Alexander Duyck, as compiler
6259 * emits better code than :
6260 * if (val & NAPIF_STATE_SCHED)
6261 * new |= NAPIF_STATE_MISSED;
6262 */
6263 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6264 NAPIF_STATE_MISSED;
6265 } while (!try_cmpxchg(&n->state, &val, new));
6266
6267 return !(val & NAPIF_STATE_SCHED);
6268}
6269EXPORT_SYMBOL(napi_schedule_prep);
6270
6271/**
6272 * __napi_schedule_irqoff - schedule for receive
6273 * @n: entry to schedule
6274 *
6275 * Variant of __napi_schedule() assuming hard irqs are masked.
6276 *
6277 * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6278 * because the interrupt disabled assumption might not be true
6279 * due to force-threaded interrupts and spinlock substitution.
6280 */
6281void __napi_schedule_irqoff(struct napi_struct *n)
6282{
6283 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6284 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6285 else
6286 __napi_schedule(n);
6287}
6288EXPORT_SYMBOL(__napi_schedule_irqoff);
6289
6290bool napi_complete_done(struct napi_struct *n, int work_done)
6291{
6292 unsigned long flags, val, new, timeout = 0;
6293 bool ret = true;
6294
6295 /*
6296 * 1) Don't let napi dequeue from the cpu poll list
6297 * just in case its running on a different cpu.
6298 * 2) If we are busy polling, do nothing here, we have
6299 * the guarantee we will be called later.
6300 */
6301 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6302 NAPIF_STATE_IN_BUSY_POLL)))
6303 return false;
6304
6305 if (work_done) {
6306 if (n->gro_bitmask)
6307 timeout = napi_get_gro_flush_timeout(n);
6308 n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n);
6309 }
6310 if (n->defer_hard_irqs_count > 0) {
6311 n->defer_hard_irqs_count--;
6312 timeout = napi_get_gro_flush_timeout(n);
6313 if (timeout)
6314 ret = false;
6315 }
6316 if (n->gro_bitmask) {
6317 /* When the NAPI instance uses a timeout and keeps postponing
6318 * it, we need to bound somehow the time packets are kept in
6319 * the GRO layer
6320 */
6321 napi_gro_flush(n, !!timeout);
6322 }
6323
6324 gro_normal_list(n);
6325
6326 if (unlikely(!list_empty(&n->poll_list))) {
6327 /* If n->poll_list is not empty, we need to mask irqs */
6328 local_irq_save(flags);
6329 list_del_init(&n->poll_list);
6330 local_irq_restore(flags);
6331 }
6332 WRITE_ONCE(n->list_owner, -1);
6333
6334 val = READ_ONCE(n->state);
6335 do {
6336 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6337
6338 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
6339 NAPIF_STATE_SCHED_THREADED |
6340 NAPIF_STATE_PREFER_BUSY_POLL);
6341
6342 /* If STATE_MISSED was set, leave STATE_SCHED set,
6343 * because we will call napi->poll() one more time.
6344 * This C code was suggested by Alexander Duyck to help gcc.
6345 */
6346 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6347 NAPIF_STATE_SCHED;
6348 } while (!try_cmpxchg(&n->state, &val, new));
6349
6350 if (unlikely(val & NAPIF_STATE_MISSED)) {
6351 __napi_schedule(n);
6352 return false;
6353 }
6354
6355 if (timeout)
6356 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6357 HRTIMER_MODE_REL_PINNED);
6358 return ret;
6359}
6360EXPORT_SYMBOL(napi_complete_done);
6361
6362static void skb_defer_free_flush(struct softnet_data *sd)
6363{
6364 struct sk_buff *skb, *next;
6365
6366 /* Paired with WRITE_ONCE() in skb_attempt_defer_free() */
6367 if (!READ_ONCE(sd->defer_list))
6368 return;
6369
6370 spin_lock(&sd->defer_lock);
6371 skb = sd->defer_list;
6372 sd->defer_list = NULL;
6373 sd->defer_count = 0;
6374 spin_unlock(&sd->defer_lock);
6375
6376 while (skb != NULL) {
6377 next = skb->next;
6378 napi_consume_skb(skb, 1);
6379 skb = next;
6380 }
6381}
6382
6383#if defined(CONFIG_NET_RX_BUSY_POLL)
6384
6385static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
6386{
6387 if (!skip_schedule) {
6388 gro_normal_list(napi);
6389 __napi_schedule(napi);
6390 return;
6391 }
6392
6393 if (napi->gro_bitmask) {
6394 /* flush too old packets
6395 * If HZ < 1000, flush all packets.
6396 */
6397 napi_gro_flush(napi, HZ >= 1000);
6398 }
6399
6400 gro_normal_list(napi);
6401 clear_bit(NAPI_STATE_SCHED, &napi->state);
6402}
6403
6404enum {
6405 NAPI_F_PREFER_BUSY_POLL = 1,
6406 NAPI_F_END_ON_RESCHED = 2,
6407};
6408
6409static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
6410 unsigned flags, u16 budget)
6411{
6412 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
6413 bool skip_schedule = false;
6414 unsigned long timeout;
6415 int rc;
6416
6417 /* Busy polling means there is a high chance device driver hard irq
6418 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6419 * set in napi_schedule_prep().
6420 * Since we are about to call napi->poll() once more, we can safely
6421 * clear NAPI_STATE_MISSED.
6422 *
6423 * Note: x86 could use a single "lock and ..." instruction
6424 * to perform these two clear_bit()
6425 */
6426 clear_bit(NAPI_STATE_MISSED, &napi->state);
6427 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6428
6429 local_bh_disable();
6430 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
6431
6432 if (flags & NAPI_F_PREFER_BUSY_POLL) {
6433 napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi);
6434 timeout = napi_get_gro_flush_timeout(napi);
6435 if (napi->defer_hard_irqs_count && timeout) {
6436 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6437 skip_schedule = true;
6438 }
6439 }
6440
6441 /* All we really want here is to re-enable device interrupts.
6442 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6443 */
6444 rc = napi->poll(napi, budget);
6445 /* We can't gro_normal_list() here, because napi->poll() might have
6446 * rearmed the napi (napi_complete_done()) in which case it could
6447 * already be running on another CPU.
6448 */
6449 trace_napi_poll(napi, rc, budget);
6450 netpoll_poll_unlock(have_poll_lock);
6451 if (rc == budget)
6452 __busy_poll_stop(napi, skip_schedule);
6453 bpf_net_ctx_clear(bpf_net_ctx);
6454 local_bh_enable();
6455}
6456
6457static void __napi_busy_loop(unsigned int napi_id,
6458 bool (*loop_end)(void *, unsigned long),
6459 void *loop_end_arg, unsigned flags, u16 budget)
6460{
6461 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6462 int (*napi_poll)(struct napi_struct *napi, int budget);
6463 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
6464 void *have_poll_lock = NULL;
6465 struct napi_struct *napi;
6466
6467 WARN_ON_ONCE(!rcu_read_lock_held());
6468
6469restart:
6470 napi_poll = NULL;
6471
6472 napi = napi_by_id(napi_id);
6473 if (!napi)
6474 return;
6475
6476 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6477 preempt_disable();
6478 for (;;) {
6479 int work = 0;
6480
6481 local_bh_disable();
6482 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
6483 if (!napi_poll) {
6484 unsigned long val = READ_ONCE(napi->state);
6485
6486 /* If multiple threads are competing for this napi,
6487 * we avoid dirtying napi->state as much as we can.
6488 */
6489 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6490 NAPIF_STATE_IN_BUSY_POLL)) {
6491 if (flags & NAPI_F_PREFER_BUSY_POLL)
6492 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6493 goto count;
6494 }
6495 if (cmpxchg(&napi->state, val,
6496 val | NAPIF_STATE_IN_BUSY_POLL |
6497 NAPIF_STATE_SCHED) != val) {
6498 if (flags & NAPI_F_PREFER_BUSY_POLL)
6499 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6500 goto count;
6501 }
6502 have_poll_lock = netpoll_poll_lock(napi);
6503 napi_poll = napi->poll;
6504 }
6505 work = napi_poll(napi, budget);
6506 trace_napi_poll(napi, work, budget);
6507 gro_normal_list(napi);
6508count:
6509 if (work > 0)
6510 __NET_ADD_STATS(dev_net(napi->dev),
6511 LINUX_MIB_BUSYPOLLRXPACKETS, work);
6512 skb_defer_free_flush(this_cpu_ptr(&softnet_data));
6513 bpf_net_ctx_clear(bpf_net_ctx);
6514 local_bh_enable();
6515
6516 if (!loop_end || loop_end(loop_end_arg, start_time))
6517 break;
6518
6519 if (unlikely(need_resched())) {
6520 if (flags & NAPI_F_END_ON_RESCHED)
6521 break;
6522 if (napi_poll)
6523 busy_poll_stop(napi, have_poll_lock, flags, budget);
6524 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6525 preempt_enable();
6526 rcu_read_unlock();
6527 cond_resched();
6528 rcu_read_lock();
6529 if (loop_end(loop_end_arg, start_time))
6530 return;
6531 goto restart;
6532 }
6533 cpu_relax();
6534 }
6535 if (napi_poll)
6536 busy_poll_stop(napi, have_poll_lock, flags, budget);
6537 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6538 preempt_enable();
6539}
6540
6541void napi_busy_loop_rcu(unsigned int napi_id,
6542 bool (*loop_end)(void *, unsigned long),
6543 void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6544{
6545 unsigned flags = NAPI_F_END_ON_RESCHED;
6546
6547 if (prefer_busy_poll)
6548 flags |= NAPI_F_PREFER_BUSY_POLL;
6549
6550 __napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
6551}
6552
6553void napi_busy_loop(unsigned int napi_id,
6554 bool (*loop_end)(void *, unsigned long),
6555 void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6556{
6557 unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0;
6558
6559 rcu_read_lock();
6560 __napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
6561 rcu_read_unlock();
6562}
6563EXPORT_SYMBOL(napi_busy_loop);
6564
6565void napi_suspend_irqs(unsigned int napi_id)
6566{
6567 struct napi_struct *napi;
6568
6569 rcu_read_lock();
6570 napi = napi_by_id(napi_id);
6571 if (napi) {
6572 unsigned long timeout = napi_get_irq_suspend_timeout(napi);
6573
6574 if (timeout)
6575 hrtimer_start(&napi->timer, ns_to_ktime(timeout),
6576 HRTIMER_MODE_REL_PINNED);
6577 }
6578 rcu_read_unlock();
6579}
6580
6581void napi_resume_irqs(unsigned int napi_id)
6582{
6583 struct napi_struct *napi;
6584
6585 rcu_read_lock();
6586 napi = napi_by_id(napi_id);
6587 if (napi) {
6588 /* If irq_suspend_timeout is set to 0 between the call to
6589 * napi_suspend_irqs and now, the original value still
6590 * determines the safety timeout as intended and napi_watchdog
6591 * will resume irq processing.
6592 */
6593 if (napi_get_irq_suspend_timeout(napi)) {
6594 local_bh_disable();
6595 napi_schedule(napi);
6596 local_bh_enable();
6597 }
6598 }
6599 rcu_read_unlock();
6600}
6601
6602#endif /* CONFIG_NET_RX_BUSY_POLL */
6603
6604static void __napi_hash_add_with_id(struct napi_struct *napi,
6605 unsigned int napi_id)
6606{
6607 WRITE_ONCE(napi->napi_id, napi_id);
6608 hlist_add_head_rcu(&napi->napi_hash_node,
6609 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
6610}
6611
6612static void napi_hash_add_with_id(struct napi_struct *napi,
6613 unsigned int napi_id)
6614{
6615 unsigned long flags;
6616
6617 spin_lock_irqsave(&napi_hash_lock, flags);
6618 WARN_ON_ONCE(napi_by_id(napi_id));
6619 __napi_hash_add_with_id(napi, napi_id);
6620 spin_unlock_irqrestore(&napi_hash_lock, flags);
6621}
6622
6623static void napi_hash_add(struct napi_struct *napi)
6624{
6625 unsigned long flags;
6626
6627 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
6628 return;
6629
6630 spin_lock_irqsave(&napi_hash_lock, flags);
6631
6632 /* 0..NR_CPUS range is reserved for sender_cpu use */
6633 do {
6634 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6635 napi_gen_id = MIN_NAPI_ID;
6636 } while (napi_by_id(napi_gen_id));
6637
6638 __napi_hash_add_with_id(napi, napi_gen_id);
6639
6640 spin_unlock_irqrestore(&napi_hash_lock, flags);
6641}
6642
6643/* Warning : caller is responsible to make sure rcu grace period
6644 * is respected before freeing memory containing @napi
6645 */
6646static void napi_hash_del(struct napi_struct *napi)
6647{
6648 unsigned long flags;
6649
6650 spin_lock_irqsave(&napi_hash_lock, flags);
6651
6652 hlist_del_init_rcu(&napi->napi_hash_node);
6653
6654 spin_unlock_irqrestore(&napi_hash_lock, flags);
6655}
6656
6657static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6658{
6659 struct napi_struct *napi;
6660
6661 napi = container_of(timer, struct napi_struct, timer);
6662
6663 /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6664 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6665 */
6666 if (!napi_disable_pending(napi) &&
6667 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6668 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6669 __napi_schedule_irqoff(napi);
6670 }
6671
6672 return HRTIMER_NORESTART;
6673}
6674
6675static void init_gro_hash(struct napi_struct *napi)
6676{
6677 int i;
6678
6679 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6680 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6681 napi->gro_hash[i].count = 0;
6682 }
6683 napi->gro_bitmask = 0;
6684}
6685
6686int dev_set_threaded(struct net_device *dev, bool threaded)
6687{
6688 struct napi_struct *napi;
6689 int err = 0;
6690
6691 if (dev->threaded == threaded)
6692 return 0;
6693
6694 if (threaded) {
6695 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6696 if (!napi->thread) {
6697 err = napi_kthread_create(napi);
6698 if (err) {
6699 threaded = false;
6700 break;
6701 }
6702 }
6703 }
6704 }
6705
6706 WRITE_ONCE(dev->threaded, threaded);
6707
6708 /* Make sure kthread is created before THREADED bit
6709 * is set.
6710 */
6711 smp_mb__before_atomic();
6712
6713 /* Setting/unsetting threaded mode on a napi might not immediately
6714 * take effect, if the current napi instance is actively being
6715 * polled. In this case, the switch between threaded mode and
6716 * softirq mode will happen in the next round of napi_schedule().
6717 * This should not cause hiccups/stalls to the live traffic.
6718 */
6719 list_for_each_entry(napi, &dev->napi_list, dev_list)
6720 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded);
6721
6722 return err;
6723}
6724EXPORT_SYMBOL(dev_set_threaded);
6725
6726/**
6727 * netif_queue_set_napi - Associate queue with the napi
6728 * @dev: device to which NAPI and queue belong
6729 * @queue_index: Index of queue
6730 * @type: queue type as RX or TX
6731 * @napi: NAPI context, pass NULL to clear previously set NAPI
6732 *
6733 * Set queue with its corresponding napi context. This should be done after
6734 * registering the NAPI handler for the queue-vector and the queues have been
6735 * mapped to the corresponding interrupt vector.
6736 */
6737void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
6738 enum netdev_queue_type type, struct napi_struct *napi)
6739{
6740 struct netdev_rx_queue *rxq;
6741 struct netdev_queue *txq;
6742
6743 if (WARN_ON_ONCE(napi && !napi->dev))
6744 return;
6745 if (dev->reg_state >= NETREG_REGISTERED)
6746 ASSERT_RTNL();
6747
6748 switch (type) {
6749 case NETDEV_QUEUE_TYPE_RX:
6750 rxq = __netif_get_rx_queue(dev, queue_index);
6751 rxq->napi = napi;
6752 return;
6753 case NETDEV_QUEUE_TYPE_TX:
6754 txq = netdev_get_tx_queue(dev, queue_index);
6755 txq->napi = napi;
6756 return;
6757 default:
6758 return;
6759 }
6760}
6761EXPORT_SYMBOL(netif_queue_set_napi);
6762
6763static void napi_restore_config(struct napi_struct *n)
6764{
6765 n->defer_hard_irqs = n->config->defer_hard_irqs;
6766 n->gro_flush_timeout = n->config->gro_flush_timeout;
6767 n->irq_suspend_timeout = n->config->irq_suspend_timeout;
6768 /* a NAPI ID might be stored in the config, if so use it. if not, use
6769 * napi_hash_add to generate one for us. It will be saved to the config
6770 * in napi_disable.
6771 */
6772 if (n->config->napi_id)
6773 napi_hash_add_with_id(n, n->config->napi_id);
6774 else
6775 napi_hash_add(n);
6776}
6777
6778static void napi_save_config(struct napi_struct *n)
6779{
6780 n->config->defer_hard_irqs = n->defer_hard_irqs;
6781 n->config->gro_flush_timeout = n->gro_flush_timeout;
6782 n->config->irq_suspend_timeout = n->irq_suspend_timeout;
6783 n->config->napi_id = n->napi_id;
6784 napi_hash_del(n);
6785}
6786
6787void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
6788 int (*poll)(struct napi_struct *, int), int weight)
6789{
6790 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6791 return;
6792
6793 INIT_LIST_HEAD(&napi->poll_list);
6794 INIT_HLIST_NODE(&napi->napi_hash_node);
6795 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6796 napi->timer.function = napi_watchdog;
6797 init_gro_hash(napi);
6798 napi->skb = NULL;
6799 INIT_LIST_HEAD(&napi->rx_list);
6800 napi->rx_count = 0;
6801 napi->poll = poll;
6802 if (weight > NAPI_POLL_WEIGHT)
6803 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6804 weight);
6805 napi->weight = weight;
6806 napi->dev = dev;
6807#ifdef CONFIG_NETPOLL
6808 napi->poll_owner = -1;
6809#endif
6810 napi->list_owner = -1;
6811 set_bit(NAPI_STATE_SCHED, &napi->state);
6812 set_bit(NAPI_STATE_NPSVC, &napi->state);
6813 list_add_rcu(&napi->dev_list, &dev->napi_list);
6814
6815 /* default settings from sysfs are applied to all NAPIs. any per-NAPI
6816 * configuration will be loaded in napi_enable
6817 */
6818 napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs));
6819 napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout));
6820
6821 napi_get_frags_check(napi);
6822 /* Create kthread for this napi if dev->threaded is set.
6823 * Clear dev->threaded if kthread creation failed so that
6824 * threaded mode will not be enabled in napi_enable().
6825 */
6826 if (dev->threaded && napi_kthread_create(napi))
6827 dev->threaded = false;
6828 netif_napi_set_irq(napi, -1);
6829}
6830EXPORT_SYMBOL(netif_napi_add_weight);
6831
6832void napi_disable(struct napi_struct *n)
6833{
6834 unsigned long val, new;
6835
6836 might_sleep();
6837 set_bit(NAPI_STATE_DISABLE, &n->state);
6838
6839 val = READ_ONCE(n->state);
6840 do {
6841 while (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) {
6842 usleep_range(20, 200);
6843 val = READ_ONCE(n->state);
6844 }
6845
6846 new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC;
6847 new &= ~(NAPIF_STATE_THREADED | NAPIF_STATE_PREFER_BUSY_POLL);
6848 } while (!try_cmpxchg(&n->state, &val, new));
6849
6850 hrtimer_cancel(&n->timer);
6851
6852 if (n->config)
6853 napi_save_config(n);
6854 else
6855 napi_hash_del(n);
6856
6857 clear_bit(NAPI_STATE_DISABLE, &n->state);
6858}
6859EXPORT_SYMBOL(napi_disable);
6860
6861/**
6862 * napi_enable - enable NAPI scheduling
6863 * @n: NAPI context
6864 *
6865 * Resume NAPI from being scheduled on this context.
6866 * Must be paired with napi_disable.
6867 */
6868void napi_enable(struct napi_struct *n)
6869{
6870 unsigned long new, val = READ_ONCE(n->state);
6871
6872 if (n->config)
6873 napi_restore_config(n);
6874 else
6875 napi_hash_add(n);
6876
6877 do {
6878 BUG_ON(!test_bit(NAPI_STATE_SCHED, &val));
6879
6880 new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC);
6881 if (n->dev->threaded && n->thread)
6882 new |= NAPIF_STATE_THREADED;
6883 } while (!try_cmpxchg(&n->state, &val, new));
6884}
6885EXPORT_SYMBOL(napi_enable);
6886
6887static void flush_gro_hash(struct napi_struct *napi)
6888{
6889 int i;
6890
6891 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6892 struct sk_buff *skb, *n;
6893
6894 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
6895 kfree_skb(skb);
6896 napi->gro_hash[i].count = 0;
6897 }
6898}
6899
6900/* Must be called in process context */
6901void __netif_napi_del(struct napi_struct *napi)
6902{
6903 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
6904 return;
6905
6906 if (napi->config) {
6907 napi->index = -1;
6908 napi->config = NULL;
6909 }
6910
6911 list_del_rcu(&napi->dev_list);
6912 napi_free_frags(napi);
6913
6914 flush_gro_hash(napi);
6915 napi->gro_bitmask = 0;
6916
6917 if (napi->thread) {
6918 kthread_stop(napi->thread);
6919 napi->thread = NULL;
6920 }
6921}
6922EXPORT_SYMBOL(__netif_napi_del);
6923
6924static int __napi_poll(struct napi_struct *n, bool *repoll)
6925{
6926 int work, weight;
6927
6928 weight = n->weight;
6929
6930 /* This NAPI_STATE_SCHED test is for avoiding a race
6931 * with netpoll's poll_napi(). Only the entity which
6932 * obtains the lock and sees NAPI_STATE_SCHED set will
6933 * actually make the ->poll() call. Therefore we avoid
6934 * accidentally calling ->poll() when NAPI is not scheduled.
6935 */
6936 work = 0;
6937 if (napi_is_scheduled(n)) {
6938 work = n->poll(n, weight);
6939 trace_napi_poll(n, work, weight);
6940
6941 xdp_do_check_flushed(n);
6942 }
6943
6944 if (unlikely(work > weight))
6945 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
6946 n->poll, work, weight);
6947
6948 if (likely(work < weight))
6949 return work;
6950
6951 /* Drivers must not modify the NAPI state if they
6952 * consume the entire weight. In such cases this code
6953 * still "owns" the NAPI instance and therefore can
6954 * move the instance around on the list at-will.
6955 */
6956 if (unlikely(napi_disable_pending(n))) {
6957 napi_complete(n);
6958 return work;
6959 }
6960
6961 /* The NAPI context has more processing work, but busy-polling
6962 * is preferred. Exit early.
6963 */
6964 if (napi_prefer_busy_poll(n)) {
6965 if (napi_complete_done(n, work)) {
6966 /* If timeout is not set, we need to make sure
6967 * that the NAPI is re-scheduled.
6968 */
6969 napi_schedule(n);
6970 }
6971 return work;
6972 }
6973
6974 if (n->gro_bitmask) {
6975 /* flush too old packets
6976 * If HZ < 1000, flush all packets.
6977 */
6978 napi_gro_flush(n, HZ >= 1000);
6979 }
6980
6981 gro_normal_list(n);
6982
6983 /* Some drivers may have called napi_schedule
6984 * prior to exhausting their budget.
6985 */
6986 if (unlikely(!list_empty(&n->poll_list))) {
6987 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6988 n->dev ? n->dev->name : "backlog");
6989 return work;
6990 }
6991
6992 *repoll = true;
6993
6994 return work;
6995}
6996
6997static int napi_poll(struct napi_struct *n, struct list_head *repoll)
6998{
6999 bool do_repoll = false;
7000 void *have;
7001 int work;
7002
7003 list_del_init(&n->poll_list);
7004
7005 have = netpoll_poll_lock(n);
7006
7007 work = __napi_poll(n, &do_repoll);
7008
7009 if (do_repoll)
7010 list_add_tail(&n->poll_list, repoll);
7011
7012 netpoll_poll_unlock(have);
7013
7014 return work;
7015}
7016
7017static int napi_thread_wait(struct napi_struct *napi)
7018{
7019 set_current_state(TASK_INTERRUPTIBLE);
7020
7021 while (!kthread_should_stop()) {
7022 /* Testing SCHED_THREADED bit here to make sure the current
7023 * kthread owns this napi and could poll on this napi.
7024 * Testing SCHED bit is not enough because SCHED bit might be
7025 * set by some other busy poll thread or by napi_disable().
7026 */
7027 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) {
7028 WARN_ON(!list_empty(&napi->poll_list));
7029 __set_current_state(TASK_RUNNING);
7030 return 0;
7031 }
7032
7033 schedule();
7034 set_current_state(TASK_INTERRUPTIBLE);
7035 }
7036 __set_current_state(TASK_RUNNING);
7037
7038 return -1;
7039}
7040
7041static void napi_threaded_poll_loop(struct napi_struct *napi)
7042{
7043 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
7044 struct softnet_data *sd;
7045 unsigned long last_qs = jiffies;
7046
7047 for (;;) {
7048 bool repoll = false;
7049 void *have;
7050
7051 local_bh_disable();
7052 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
7053
7054 sd = this_cpu_ptr(&softnet_data);
7055 sd->in_napi_threaded_poll = true;
7056
7057 have = netpoll_poll_lock(napi);
7058 __napi_poll(napi, &repoll);
7059 netpoll_poll_unlock(have);
7060
7061 sd->in_napi_threaded_poll = false;
7062 barrier();
7063
7064 if (sd_has_rps_ipi_waiting(sd)) {
7065 local_irq_disable();
7066 net_rps_action_and_irq_enable(sd);
7067 }
7068 skb_defer_free_flush(sd);
7069 bpf_net_ctx_clear(bpf_net_ctx);
7070 local_bh_enable();
7071
7072 if (!repoll)
7073 break;
7074
7075 rcu_softirq_qs_periodic(last_qs);
7076 cond_resched();
7077 }
7078}
7079
7080static int napi_threaded_poll(void *data)
7081{
7082 struct napi_struct *napi = data;
7083
7084 while (!napi_thread_wait(napi))
7085 napi_threaded_poll_loop(napi);
7086
7087 return 0;
7088}
7089
7090static __latent_entropy void net_rx_action(void)
7091{
7092 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
7093 unsigned long time_limit = jiffies +
7094 usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs));
7095 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
7096 int budget = READ_ONCE(net_hotdata.netdev_budget);
7097 LIST_HEAD(list);
7098 LIST_HEAD(repoll);
7099
7100 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
7101start:
7102 sd->in_net_rx_action = true;
7103 local_irq_disable();
7104 list_splice_init(&sd->poll_list, &list);
7105 local_irq_enable();
7106
7107 for (;;) {
7108 struct napi_struct *n;
7109
7110 skb_defer_free_flush(sd);
7111
7112 if (list_empty(&list)) {
7113 if (list_empty(&repoll)) {
7114 sd->in_net_rx_action = false;
7115 barrier();
7116 /* We need to check if ____napi_schedule()
7117 * had refilled poll_list while
7118 * sd->in_net_rx_action was true.
7119 */
7120 if (!list_empty(&sd->poll_list))
7121 goto start;
7122 if (!sd_has_rps_ipi_waiting(sd))
7123 goto end;
7124 }
7125 break;
7126 }
7127
7128 n = list_first_entry(&list, struct napi_struct, poll_list);
7129 budget -= napi_poll(n, &repoll);
7130
7131 /* If softirq window is exhausted then punt.
7132 * Allow this to run for 2 jiffies since which will allow
7133 * an average latency of 1.5/HZ.
7134 */
7135 if (unlikely(budget <= 0 ||
7136 time_after_eq(jiffies, time_limit))) {
7137 sd->time_squeeze++;
7138 break;
7139 }
7140 }
7141
7142 local_irq_disable();
7143
7144 list_splice_tail_init(&sd->poll_list, &list);
7145 list_splice_tail(&repoll, &list);
7146 list_splice(&list, &sd->poll_list);
7147 if (!list_empty(&sd->poll_list))
7148 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
7149 else
7150 sd->in_net_rx_action = false;
7151
7152 net_rps_action_and_irq_enable(sd);
7153end:
7154 bpf_net_ctx_clear(bpf_net_ctx);
7155}
7156
7157struct netdev_adjacent {
7158 struct net_device *dev;
7159 netdevice_tracker dev_tracker;
7160
7161 /* upper master flag, there can only be one master device per list */
7162 bool master;
7163
7164 /* lookup ignore flag */
7165 bool ignore;
7166
7167 /* counter for the number of times this device was added to us */
7168 u16 ref_nr;
7169
7170 /* private field for the users */
7171 void *private;
7172
7173 struct list_head list;
7174 struct rcu_head rcu;
7175};
7176
7177static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
7178 struct list_head *adj_list)
7179{
7180 struct netdev_adjacent *adj;
7181
7182 list_for_each_entry(adj, adj_list, list) {
7183 if (adj->dev == adj_dev)
7184 return adj;
7185 }
7186 return NULL;
7187}
7188
7189static int ____netdev_has_upper_dev(struct net_device *upper_dev,
7190 struct netdev_nested_priv *priv)
7191{
7192 struct net_device *dev = (struct net_device *)priv->data;
7193
7194 return upper_dev == dev;
7195}
7196
7197/**
7198 * netdev_has_upper_dev - Check if device is linked to an upper device
7199 * @dev: device
7200 * @upper_dev: upper device to check
7201 *
7202 * Find out if a device is linked to specified upper device and return true
7203 * in case it is. Note that this checks only immediate upper device,
7204 * not through a complete stack of devices. The caller must hold the RTNL lock.
7205 */
7206bool netdev_has_upper_dev(struct net_device *dev,
7207 struct net_device *upper_dev)
7208{
7209 struct netdev_nested_priv priv = {
7210 .data = (void *)upper_dev,
7211 };
7212
7213 ASSERT_RTNL();
7214
7215 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
7216 &priv);
7217}
7218EXPORT_SYMBOL(netdev_has_upper_dev);
7219
7220/**
7221 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
7222 * @dev: device
7223 * @upper_dev: upper device to check
7224 *
7225 * Find out if a device is linked to specified upper device and return true
7226 * in case it is. Note that this checks the entire upper device chain.
7227 * The caller must hold rcu lock.
7228 */
7229
7230bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
7231 struct net_device *upper_dev)
7232{
7233 struct netdev_nested_priv priv = {
7234 .data = (void *)upper_dev,
7235 };
7236
7237 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
7238 &priv);
7239}
7240EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
7241
7242/**
7243 * netdev_has_any_upper_dev - Check if device is linked to some device
7244 * @dev: device
7245 *
7246 * Find out if a device is linked to an upper device and return true in case
7247 * it is. The caller must hold the RTNL lock.
7248 */
7249bool netdev_has_any_upper_dev(struct net_device *dev)
7250{
7251 ASSERT_RTNL();
7252
7253 return !list_empty(&dev->adj_list.upper);
7254}
7255EXPORT_SYMBOL(netdev_has_any_upper_dev);
7256
7257/**
7258 * netdev_master_upper_dev_get - Get master upper device
7259 * @dev: device
7260 *
7261 * Find a master upper device and return pointer to it or NULL in case
7262 * it's not there. The caller must hold the RTNL lock.
7263 */
7264struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
7265{
7266 struct netdev_adjacent *upper;
7267
7268 ASSERT_RTNL();
7269
7270 if (list_empty(&dev->adj_list.upper))
7271 return NULL;
7272
7273 upper = list_first_entry(&dev->adj_list.upper,
7274 struct netdev_adjacent, list);
7275 if (likely(upper->master))
7276 return upper->dev;
7277 return NULL;
7278}
7279EXPORT_SYMBOL(netdev_master_upper_dev_get);
7280
7281static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
7282{
7283 struct netdev_adjacent *upper;
7284
7285 ASSERT_RTNL();
7286
7287 if (list_empty(&dev->adj_list.upper))
7288 return NULL;
7289
7290 upper = list_first_entry(&dev->adj_list.upper,
7291 struct netdev_adjacent, list);
7292 if (likely(upper->master) && !upper->ignore)
7293 return upper->dev;
7294 return NULL;
7295}
7296
7297/**
7298 * netdev_has_any_lower_dev - Check if device is linked to some device
7299 * @dev: device
7300 *
7301 * Find out if a device is linked to a lower device and return true in case
7302 * it is. The caller must hold the RTNL lock.
7303 */
7304static bool netdev_has_any_lower_dev(struct net_device *dev)
7305{
7306 ASSERT_RTNL();
7307
7308 return !list_empty(&dev->adj_list.lower);
7309}
7310
7311void *netdev_adjacent_get_private(struct list_head *adj_list)
7312{
7313 struct netdev_adjacent *adj;
7314
7315 adj = list_entry(adj_list, struct netdev_adjacent, list);
7316
7317 return adj->private;
7318}
7319EXPORT_SYMBOL(netdev_adjacent_get_private);
7320
7321/**
7322 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
7323 * @dev: device
7324 * @iter: list_head ** of the current position
7325 *
7326 * Gets the next device from the dev's upper list, starting from iter
7327 * position. The caller must hold RCU read lock.
7328 */
7329struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
7330 struct list_head **iter)
7331{
7332 struct netdev_adjacent *upper;
7333
7334 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7335
7336 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7337
7338 if (&upper->list == &dev->adj_list.upper)
7339 return NULL;
7340
7341 *iter = &upper->list;
7342
7343 return upper->dev;
7344}
7345EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
7346
7347static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
7348 struct list_head **iter,
7349 bool *ignore)
7350{
7351 struct netdev_adjacent *upper;
7352
7353 upper = list_entry((*iter)->next, struct netdev_adjacent, list);
7354
7355 if (&upper->list == &dev->adj_list.upper)
7356 return NULL;
7357
7358 *iter = &upper->list;
7359 *ignore = upper->ignore;
7360
7361 return upper->dev;
7362}
7363
7364static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
7365 struct list_head **iter)
7366{
7367 struct netdev_adjacent *upper;
7368
7369 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7370
7371 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7372
7373 if (&upper->list == &dev->adj_list.upper)
7374 return NULL;
7375
7376 *iter = &upper->list;
7377
7378 return upper->dev;
7379}
7380
7381static int __netdev_walk_all_upper_dev(struct net_device *dev,
7382 int (*fn)(struct net_device *dev,
7383 struct netdev_nested_priv *priv),
7384 struct netdev_nested_priv *priv)
7385{
7386 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7387 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7388 int ret, cur = 0;
7389 bool ignore;
7390
7391 now = dev;
7392 iter = &dev->adj_list.upper;
7393
7394 while (1) {
7395 if (now != dev) {
7396 ret = fn(now, priv);
7397 if (ret)
7398 return ret;
7399 }
7400
7401 next = NULL;
7402 while (1) {
7403 udev = __netdev_next_upper_dev(now, &iter, &ignore);
7404 if (!udev)
7405 break;
7406 if (ignore)
7407 continue;
7408
7409 next = udev;
7410 niter = &udev->adj_list.upper;
7411 dev_stack[cur] = now;
7412 iter_stack[cur++] = iter;
7413 break;
7414 }
7415
7416 if (!next) {
7417 if (!cur)
7418 return 0;
7419 next = dev_stack[--cur];
7420 niter = iter_stack[cur];
7421 }
7422
7423 now = next;
7424 iter = niter;
7425 }
7426
7427 return 0;
7428}
7429
7430int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
7431 int (*fn)(struct net_device *dev,
7432 struct netdev_nested_priv *priv),
7433 struct netdev_nested_priv *priv)
7434{
7435 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7436 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7437 int ret, cur = 0;
7438
7439 now = dev;
7440 iter = &dev->adj_list.upper;
7441
7442 while (1) {
7443 if (now != dev) {
7444 ret = fn(now, priv);
7445 if (ret)
7446 return ret;
7447 }
7448
7449 next = NULL;
7450 while (1) {
7451 udev = netdev_next_upper_dev_rcu(now, &iter);
7452 if (!udev)
7453 break;
7454
7455 next = udev;
7456 niter = &udev->adj_list.upper;
7457 dev_stack[cur] = now;
7458 iter_stack[cur++] = iter;
7459 break;
7460 }
7461
7462 if (!next) {
7463 if (!cur)
7464 return 0;
7465 next = dev_stack[--cur];
7466 niter = iter_stack[cur];
7467 }
7468
7469 now = next;
7470 iter = niter;
7471 }
7472
7473 return 0;
7474}
7475EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
7476
7477static bool __netdev_has_upper_dev(struct net_device *dev,
7478 struct net_device *upper_dev)
7479{
7480 struct netdev_nested_priv priv = {
7481 .flags = 0,
7482 .data = (void *)upper_dev,
7483 };
7484
7485 ASSERT_RTNL();
7486
7487 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
7488 &priv);
7489}
7490
7491/**
7492 * netdev_lower_get_next_private - Get the next ->private from the
7493 * lower neighbour list
7494 * @dev: device
7495 * @iter: list_head ** of the current position
7496 *
7497 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7498 * list, starting from iter position. The caller must hold either hold the
7499 * RTNL lock or its own locking that guarantees that the neighbour lower
7500 * list will remain unchanged.
7501 */
7502void *netdev_lower_get_next_private(struct net_device *dev,
7503 struct list_head **iter)
7504{
7505 struct netdev_adjacent *lower;
7506
7507 lower = list_entry(*iter, struct netdev_adjacent, list);
7508
7509 if (&lower->list == &dev->adj_list.lower)
7510 return NULL;
7511
7512 *iter = lower->list.next;
7513
7514 return lower->private;
7515}
7516EXPORT_SYMBOL(netdev_lower_get_next_private);
7517
7518/**
7519 * netdev_lower_get_next_private_rcu - Get the next ->private from the
7520 * lower neighbour list, RCU
7521 * variant
7522 * @dev: device
7523 * @iter: list_head ** of the current position
7524 *
7525 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7526 * list, starting from iter position. The caller must hold RCU read lock.
7527 */
7528void *netdev_lower_get_next_private_rcu(struct net_device *dev,
7529 struct list_head **iter)
7530{
7531 struct netdev_adjacent *lower;
7532
7533 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
7534
7535 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7536
7537 if (&lower->list == &dev->adj_list.lower)
7538 return NULL;
7539
7540 *iter = &lower->list;
7541
7542 return lower->private;
7543}
7544EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
7545
7546/**
7547 * netdev_lower_get_next - Get the next device from the lower neighbour
7548 * list
7549 * @dev: device
7550 * @iter: list_head ** of the current position
7551 *
7552 * Gets the next netdev_adjacent from the dev's lower neighbour
7553 * list, starting from iter position. The caller must hold RTNL lock or
7554 * its own locking that guarantees that the neighbour lower
7555 * list will remain unchanged.
7556 */
7557void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
7558{
7559 struct netdev_adjacent *lower;
7560
7561 lower = list_entry(*iter, struct netdev_adjacent, list);
7562
7563 if (&lower->list == &dev->adj_list.lower)
7564 return NULL;
7565
7566 *iter = lower->list.next;
7567
7568 return lower->dev;
7569}
7570EXPORT_SYMBOL(netdev_lower_get_next);
7571
7572static struct net_device *netdev_next_lower_dev(struct net_device *dev,
7573 struct list_head **iter)
7574{
7575 struct netdev_adjacent *lower;
7576
7577 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7578
7579 if (&lower->list == &dev->adj_list.lower)
7580 return NULL;
7581
7582 *iter = &lower->list;
7583
7584 return lower->dev;
7585}
7586
7587static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
7588 struct list_head **iter,
7589 bool *ignore)
7590{
7591 struct netdev_adjacent *lower;
7592
7593 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7594
7595 if (&lower->list == &dev->adj_list.lower)
7596 return NULL;
7597
7598 *iter = &lower->list;
7599 *ignore = lower->ignore;
7600
7601 return lower->dev;
7602}
7603
7604int netdev_walk_all_lower_dev(struct net_device *dev,
7605 int (*fn)(struct net_device *dev,
7606 struct netdev_nested_priv *priv),
7607 struct netdev_nested_priv *priv)
7608{
7609 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7610 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7611 int ret, cur = 0;
7612
7613 now = dev;
7614 iter = &dev->adj_list.lower;
7615
7616 while (1) {
7617 if (now != dev) {
7618 ret = fn(now, priv);
7619 if (ret)
7620 return ret;
7621 }
7622
7623 next = NULL;
7624 while (1) {
7625 ldev = netdev_next_lower_dev(now, &iter);
7626 if (!ldev)
7627 break;
7628
7629 next = ldev;
7630 niter = &ldev->adj_list.lower;
7631 dev_stack[cur] = now;
7632 iter_stack[cur++] = iter;
7633 break;
7634 }
7635
7636 if (!next) {
7637 if (!cur)
7638 return 0;
7639 next = dev_stack[--cur];
7640 niter = iter_stack[cur];
7641 }
7642
7643 now = next;
7644 iter = niter;
7645 }
7646
7647 return 0;
7648}
7649EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
7650
7651static int __netdev_walk_all_lower_dev(struct net_device *dev,
7652 int (*fn)(struct net_device *dev,
7653 struct netdev_nested_priv *priv),
7654 struct netdev_nested_priv *priv)
7655{
7656 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7657 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7658 int ret, cur = 0;
7659 bool ignore;
7660
7661 now = dev;
7662 iter = &dev->adj_list.lower;
7663
7664 while (1) {
7665 if (now != dev) {
7666 ret = fn(now, priv);
7667 if (ret)
7668 return ret;
7669 }
7670
7671 next = NULL;
7672 while (1) {
7673 ldev = __netdev_next_lower_dev(now, &iter, &ignore);
7674 if (!ldev)
7675 break;
7676 if (ignore)
7677 continue;
7678
7679 next = ldev;
7680 niter = &ldev->adj_list.lower;
7681 dev_stack[cur] = now;
7682 iter_stack[cur++] = iter;
7683 break;
7684 }
7685
7686 if (!next) {
7687 if (!cur)
7688 return 0;
7689 next = dev_stack[--cur];
7690 niter = iter_stack[cur];
7691 }
7692
7693 now = next;
7694 iter = niter;
7695 }
7696
7697 return 0;
7698}
7699
7700struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
7701 struct list_head **iter)
7702{
7703 struct netdev_adjacent *lower;
7704
7705 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7706 if (&lower->list == &dev->adj_list.lower)
7707 return NULL;
7708
7709 *iter = &lower->list;
7710
7711 return lower->dev;
7712}
7713EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
7714
7715static u8 __netdev_upper_depth(struct net_device *dev)
7716{
7717 struct net_device *udev;
7718 struct list_head *iter;
7719 u8 max_depth = 0;
7720 bool ignore;
7721
7722 for (iter = &dev->adj_list.upper,
7723 udev = __netdev_next_upper_dev(dev, &iter, &ignore);
7724 udev;
7725 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
7726 if (ignore)
7727 continue;
7728 if (max_depth < udev->upper_level)
7729 max_depth = udev->upper_level;
7730 }
7731
7732 return max_depth;
7733}
7734
7735static u8 __netdev_lower_depth(struct net_device *dev)
7736{
7737 struct net_device *ldev;
7738 struct list_head *iter;
7739 u8 max_depth = 0;
7740 bool ignore;
7741
7742 for (iter = &dev->adj_list.lower,
7743 ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
7744 ldev;
7745 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
7746 if (ignore)
7747 continue;
7748 if (max_depth < ldev->lower_level)
7749 max_depth = ldev->lower_level;
7750 }
7751
7752 return max_depth;
7753}
7754
7755static int __netdev_update_upper_level(struct net_device *dev,
7756 struct netdev_nested_priv *__unused)
7757{
7758 dev->upper_level = __netdev_upper_depth(dev) + 1;
7759 return 0;
7760}
7761
7762#ifdef CONFIG_LOCKDEP
7763static LIST_HEAD(net_unlink_list);
7764
7765static void net_unlink_todo(struct net_device *dev)
7766{
7767 if (list_empty(&dev->unlink_list))
7768 list_add_tail(&dev->unlink_list, &net_unlink_list);
7769}
7770#endif
7771
7772static int __netdev_update_lower_level(struct net_device *dev,
7773 struct netdev_nested_priv *priv)
7774{
7775 dev->lower_level = __netdev_lower_depth(dev) + 1;
7776
7777#ifdef CONFIG_LOCKDEP
7778 if (!priv)
7779 return 0;
7780
7781 if (priv->flags & NESTED_SYNC_IMM)
7782 dev->nested_level = dev->lower_level - 1;
7783 if (priv->flags & NESTED_SYNC_TODO)
7784 net_unlink_todo(dev);
7785#endif
7786 return 0;
7787}
7788
7789int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
7790 int (*fn)(struct net_device *dev,
7791 struct netdev_nested_priv *priv),
7792 struct netdev_nested_priv *priv)
7793{
7794 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7795 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7796 int ret, cur = 0;
7797
7798 now = dev;
7799 iter = &dev->adj_list.lower;
7800
7801 while (1) {
7802 if (now != dev) {
7803 ret = fn(now, priv);
7804 if (ret)
7805 return ret;
7806 }
7807
7808 next = NULL;
7809 while (1) {
7810 ldev = netdev_next_lower_dev_rcu(now, &iter);
7811 if (!ldev)
7812 break;
7813
7814 next = ldev;
7815 niter = &ldev->adj_list.lower;
7816 dev_stack[cur] = now;
7817 iter_stack[cur++] = iter;
7818 break;
7819 }
7820
7821 if (!next) {
7822 if (!cur)
7823 return 0;
7824 next = dev_stack[--cur];
7825 niter = iter_stack[cur];
7826 }
7827
7828 now = next;
7829 iter = niter;
7830 }
7831
7832 return 0;
7833}
7834EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
7835
7836/**
7837 * netdev_lower_get_first_private_rcu - Get the first ->private from the
7838 * lower neighbour list, RCU
7839 * variant
7840 * @dev: device
7841 *
7842 * Gets the first netdev_adjacent->private from the dev's lower neighbour
7843 * list. The caller must hold RCU read lock.
7844 */
7845void *netdev_lower_get_first_private_rcu(struct net_device *dev)
7846{
7847 struct netdev_adjacent *lower;
7848
7849 lower = list_first_or_null_rcu(&dev->adj_list.lower,
7850 struct netdev_adjacent, list);
7851 if (lower)
7852 return lower->private;
7853 return NULL;
7854}
7855EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
7856
7857/**
7858 * netdev_master_upper_dev_get_rcu - Get master upper device
7859 * @dev: device
7860 *
7861 * Find a master upper device and return pointer to it or NULL in case
7862 * it's not there. The caller must hold the RCU read lock.
7863 */
7864struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
7865{
7866 struct netdev_adjacent *upper;
7867
7868 upper = list_first_or_null_rcu(&dev->adj_list.upper,
7869 struct netdev_adjacent, list);
7870 if (upper && likely(upper->master))
7871 return upper->dev;
7872 return NULL;
7873}
7874EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
7875
7876static int netdev_adjacent_sysfs_add(struct net_device *dev,
7877 struct net_device *adj_dev,
7878 struct list_head *dev_list)
7879{
7880 char linkname[IFNAMSIZ+7];
7881
7882 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7883 "upper_%s" : "lower_%s", adj_dev->name);
7884 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
7885 linkname);
7886}
7887static void netdev_adjacent_sysfs_del(struct net_device *dev,
7888 char *name,
7889 struct list_head *dev_list)
7890{
7891 char linkname[IFNAMSIZ+7];
7892
7893 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7894 "upper_%s" : "lower_%s", name);
7895 sysfs_remove_link(&(dev->dev.kobj), linkname);
7896}
7897
7898static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
7899 struct net_device *adj_dev,
7900 struct list_head *dev_list)
7901{
7902 return (dev_list == &dev->adj_list.upper ||
7903 dev_list == &dev->adj_list.lower) &&
7904 net_eq(dev_net(dev), dev_net(adj_dev));
7905}
7906
7907static int __netdev_adjacent_dev_insert(struct net_device *dev,
7908 struct net_device *adj_dev,
7909 struct list_head *dev_list,
7910 void *private, bool master)
7911{
7912 struct netdev_adjacent *adj;
7913 int ret;
7914
7915 adj = __netdev_find_adj(adj_dev, dev_list);
7916
7917 if (adj) {
7918 adj->ref_nr += 1;
7919 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
7920 dev->name, adj_dev->name, adj->ref_nr);
7921
7922 return 0;
7923 }
7924
7925 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
7926 if (!adj)
7927 return -ENOMEM;
7928
7929 adj->dev = adj_dev;
7930 adj->master = master;
7931 adj->ref_nr = 1;
7932 adj->private = private;
7933 adj->ignore = false;
7934 netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL);
7935
7936 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
7937 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
7938
7939 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
7940 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
7941 if (ret)
7942 goto free_adj;
7943 }
7944
7945 /* Ensure that master link is always the first item in list. */
7946 if (master) {
7947 ret = sysfs_create_link(&(dev->dev.kobj),
7948 &(adj_dev->dev.kobj), "master");
7949 if (ret)
7950 goto remove_symlinks;
7951
7952 list_add_rcu(&adj->list, dev_list);
7953 } else {
7954 list_add_tail_rcu(&adj->list, dev_list);
7955 }
7956
7957 return 0;
7958
7959remove_symlinks:
7960 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7961 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7962free_adj:
7963 netdev_put(adj_dev, &adj->dev_tracker);
7964 kfree(adj);
7965
7966 return ret;
7967}
7968
7969static void __netdev_adjacent_dev_remove(struct net_device *dev,
7970 struct net_device *adj_dev,
7971 u16 ref_nr,
7972 struct list_head *dev_list)
7973{
7974 struct netdev_adjacent *adj;
7975
7976 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
7977 dev->name, adj_dev->name, ref_nr);
7978
7979 adj = __netdev_find_adj(adj_dev, dev_list);
7980
7981 if (!adj) {
7982 pr_err("Adjacency does not exist for device %s from %s\n",
7983 dev->name, adj_dev->name);
7984 WARN_ON(1);
7985 return;
7986 }
7987
7988 if (adj->ref_nr > ref_nr) {
7989 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
7990 dev->name, adj_dev->name, ref_nr,
7991 adj->ref_nr - ref_nr);
7992 adj->ref_nr -= ref_nr;
7993 return;
7994 }
7995
7996 if (adj->master)
7997 sysfs_remove_link(&(dev->dev.kobj), "master");
7998
7999 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
8000 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
8001
8002 list_del_rcu(&adj->list);
8003 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
8004 adj_dev->name, dev->name, adj_dev->name);
8005 netdev_put(adj_dev, &adj->dev_tracker);
8006 kfree_rcu(adj, rcu);
8007}
8008
8009static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
8010 struct net_device *upper_dev,
8011 struct list_head *up_list,
8012 struct list_head *down_list,
8013 void *private, bool master)
8014{
8015 int ret;
8016
8017 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
8018 private, master);
8019 if (ret)
8020 return ret;
8021
8022 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
8023 private, false);
8024 if (ret) {
8025 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
8026 return ret;
8027 }
8028
8029 return 0;
8030}
8031
8032static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
8033 struct net_device *upper_dev,
8034 u16 ref_nr,
8035 struct list_head *up_list,
8036 struct list_head *down_list)
8037{
8038 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
8039 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
8040}
8041
8042static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
8043 struct net_device *upper_dev,
8044 void *private, bool master)
8045{
8046 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
8047 &dev->adj_list.upper,
8048 &upper_dev->adj_list.lower,
8049 private, master);
8050}
8051
8052static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
8053 struct net_device *upper_dev)
8054{
8055 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
8056 &dev->adj_list.upper,
8057 &upper_dev->adj_list.lower);
8058}
8059
8060static int __netdev_upper_dev_link(struct net_device *dev,
8061 struct net_device *upper_dev, bool master,
8062 void *upper_priv, void *upper_info,
8063 struct netdev_nested_priv *priv,
8064 struct netlink_ext_ack *extack)
8065{
8066 struct netdev_notifier_changeupper_info changeupper_info = {
8067 .info = {
8068 .dev = dev,
8069 .extack = extack,
8070 },
8071 .upper_dev = upper_dev,
8072 .master = master,
8073 .linking = true,
8074 .upper_info = upper_info,
8075 };
8076 struct net_device *master_dev;
8077 int ret = 0;
8078
8079 ASSERT_RTNL();
8080
8081 if (dev == upper_dev)
8082 return -EBUSY;
8083
8084 /* To prevent loops, check if dev is not upper device to upper_dev. */
8085 if (__netdev_has_upper_dev(upper_dev, dev))
8086 return -EBUSY;
8087
8088 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
8089 return -EMLINK;
8090
8091 if (!master) {
8092 if (__netdev_has_upper_dev(dev, upper_dev))
8093 return -EEXIST;
8094 } else {
8095 master_dev = __netdev_master_upper_dev_get(dev);
8096 if (master_dev)
8097 return master_dev == upper_dev ? -EEXIST : -EBUSY;
8098 }
8099
8100 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
8101 &changeupper_info.info);
8102 ret = notifier_to_errno(ret);
8103 if (ret)
8104 return ret;
8105
8106 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
8107 master);
8108 if (ret)
8109 return ret;
8110
8111 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
8112 &changeupper_info.info);
8113 ret = notifier_to_errno(ret);
8114 if (ret)
8115 goto rollback;
8116
8117 __netdev_update_upper_level(dev, NULL);
8118 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
8119
8120 __netdev_update_lower_level(upper_dev, priv);
8121 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
8122 priv);
8123
8124 return 0;
8125
8126rollback:
8127 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
8128
8129 return ret;
8130}
8131
8132/**
8133 * netdev_upper_dev_link - Add a link to the upper device
8134 * @dev: device
8135 * @upper_dev: new upper device
8136 * @extack: netlink extended ack
8137 *
8138 * Adds a link to device which is upper to this one. The caller must hold
8139 * the RTNL lock. On a failure a negative errno code is returned.
8140 * On success the reference counts are adjusted and the function
8141 * returns zero.
8142 */
8143int netdev_upper_dev_link(struct net_device *dev,
8144 struct net_device *upper_dev,
8145 struct netlink_ext_ack *extack)
8146{
8147 struct netdev_nested_priv priv = {
8148 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8149 .data = NULL,
8150 };
8151
8152 return __netdev_upper_dev_link(dev, upper_dev, false,
8153 NULL, NULL, &priv, extack);
8154}
8155EXPORT_SYMBOL(netdev_upper_dev_link);
8156
8157/**
8158 * netdev_master_upper_dev_link - Add a master link to the upper device
8159 * @dev: device
8160 * @upper_dev: new upper device
8161 * @upper_priv: upper device private
8162 * @upper_info: upper info to be passed down via notifier
8163 * @extack: netlink extended ack
8164 *
8165 * Adds a link to device which is upper to this one. In this case, only
8166 * one master upper device can be linked, although other non-master devices
8167 * might be linked as well. The caller must hold the RTNL lock.
8168 * On a failure a negative errno code is returned. On success the reference
8169 * counts are adjusted and the function returns zero.
8170 */
8171int netdev_master_upper_dev_link(struct net_device *dev,
8172 struct net_device *upper_dev,
8173 void *upper_priv, void *upper_info,
8174 struct netlink_ext_ack *extack)
8175{
8176 struct netdev_nested_priv priv = {
8177 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8178 .data = NULL,
8179 };
8180
8181 return __netdev_upper_dev_link(dev, upper_dev, true,
8182 upper_priv, upper_info, &priv, extack);
8183}
8184EXPORT_SYMBOL(netdev_master_upper_dev_link);
8185
8186static void __netdev_upper_dev_unlink(struct net_device *dev,
8187 struct net_device *upper_dev,
8188 struct netdev_nested_priv *priv)
8189{
8190 struct netdev_notifier_changeupper_info changeupper_info = {
8191 .info = {
8192 .dev = dev,
8193 },
8194 .upper_dev = upper_dev,
8195 .linking = false,
8196 };
8197
8198 ASSERT_RTNL();
8199
8200 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
8201
8202 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
8203 &changeupper_info.info);
8204
8205 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
8206
8207 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
8208 &changeupper_info.info);
8209
8210 __netdev_update_upper_level(dev, NULL);
8211 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
8212
8213 __netdev_update_lower_level(upper_dev, priv);
8214 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
8215 priv);
8216}
8217
8218/**
8219 * netdev_upper_dev_unlink - Removes a link to upper device
8220 * @dev: device
8221 * @upper_dev: new upper device
8222 *
8223 * Removes a link to device which is upper to this one. The caller must hold
8224 * the RTNL lock.
8225 */
8226void netdev_upper_dev_unlink(struct net_device *dev,
8227 struct net_device *upper_dev)
8228{
8229 struct netdev_nested_priv priv = {
8230 .flags = NESTED_SYNC_TODO,
8231 .data = NULL,
8232 };
8233
8234 __netdev_upper_dev_unlink(dev, upper_dev, &priv);
8235}
8236EXPORT_SYMBOL(netdev_upper_dev_unlink);
8237
8238static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
8239 struct net_device *lower_dev,
8240 bool val)
8241{
8242 struct netdev_adjacent *adj;
8243
8244 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
8245 if (adj)
8246 adj->ignore = val;
8247
8248 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
8249 if (adj)
8250 adj->ignore = val;
8251}
8252
8253static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
8254 struct net_device *lower_dev)
8255{
8256 __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
8257}
8258
8259static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
8260 struct net_device *lower_dev)
8261{
8262 __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
8263}
8264
8265int netdev_adjacent_change_prepare(struct net_device *old_dev,
8266 struct net_device *new_dev,
8267 struct net_device *dev,
8268 struct netlink_ext_ack *extack)
8269{
8270 struct netdev_nested_priv priv = {
8271 .flags = 0,
8272 .data = NULL,
8273 };
8274 int err;
8275
8276 if (!new_dev)
8277 return 0;
8278
8279 if (old_dev && new_dev != old_dev)
8280 netdev_adjacent_dev_disable(dev, old_dev);
8281 err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
8282 extack);
8283 if (err) {
8284 if (old_dev && new_dev != old_dev)
8285 netdev_adjacent_dev_enable(dev, old_dev);
8286 return err;
8287 }
8288
8289 return 0;
8290}
8291EXPORT_SYMBOL(netdev_adjacent_change_prepare);
8292
8293void netdev_adjacent_change_commit(struct net_device *old_dev,
8294 struct net_device *new_dev,
8295 struct net_device *dev)
8296{
8297 struct netdev_nested_priv priv = {
8298 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8299 .data = NULL,
8300 };
8301
8302 if (!new_dev || !old_dev)
8303 return;
8304
8305 if (new_dev == old_dev)
8306 return;
8307
8308 netdev_adjacent_dev_enable(dev, old_dev);
8309 __netdev_upper_dev_unlink(old_dev, dev, &priv);
8310}
8311EXPORT_SYMBOL(netdev_adjacent_change_commit);
8312
8313void netdev_adjacent_change_abort(struct net_device *old_dev,
8314 struct net_device *new_dev,
8315 struct net_device *dev)
8316{
8317 struct netdev_nested_priv priv = {
8318 .flags = 0,
8319 .data = NULL,
8320 };
8321
8322 if (!new_dev)
8323 return;
8324
8325 if (old_dev && new_dev != old_dev)
8326 netdev_adjacent_dev_enable(dev, old_dev);
8327
8328 __netdev_upper_dev_unlink(new_dev, dev, &priv);
8329}
8330EXPORT_SYMBOL(netdev_adjacent_change_abort);
8331
8332/**
8333 * netdev_bonding_info_change - Dispatch event about slave change
8334 * @dev: device
8335 * @bonding_info: info to dispatch
8336 *
8337 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
8338 * The caller must hold the RTNL lock.
8339 */
8340void netdev_bonding_info_change(struct net_device *dev,
8341 struct netdev_bonding_info *bonding_info)
8342{
8343 struct netdev_notifier_bonding_info info = {
8344 .info.dev = dev,
8345 };
8346
8347 memcpy(&info.bonding_info, bonding_info,
8348 sizeof(struct netdev_bonding_info));
8349 call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
8350 &info.info);
8351}
8352EXPORT_SYMBOL(netdev_bonding_info_change);
8353
8354static int netdev_offload_xstats_enable_l3(struct net_device *dev,
8355 struct netlink_ext_ack *extack)
8356{
8357 struct netdev_notifier_offload_xstats_info info = {
8358 .info.dev = dev,
8359 .info.extack = extack,
8360 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8361 };
8362 int err;
8363 int rc;
8364
8365 dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3),
8366 GFP_KERNEL);
8367 if (!dev->offload_xstats_l3)
8368 return -ENOMEM;
8369
8370 rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE,
8371 NETDEV_OFFLOAD_XSTATS_DISABLE,
8372 &info.info);
8373 err = notifier_to_errno(rc);
8374 if (err)
8375 goto free_stats;
8376
8377 return 0;
8378
8379free_stats:
8380 kfree(dev->offload_xstats_l3);
8381 dev->offload_xstats_l3 = NULL;
8382 return err;
8383}
8384
8385int netdev_offload_xstats_enable(struct net_device *dev,
8386 enum netdev_offload_xstats_type type,
8387 struct netlink_ext_ack *extack)
8388{
8389 ASSERT_RTNL();
8390
8391 if (netdev_offload_xstats_enabled(dev, type))
8392 return -EALREADY;
8393
8394 switch (type) {
8395 case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8396 return netdev_offload_xstats_enable_l3(dev, extack);
8397 }
8398
8399 WARN_ON(1);
8400 return -EINVAL;
8401}
8402EXPORT_SYMBOL(netdev_offload_xstats_enable);
8403
8404static void netdev_offload_xstats_disable_l3(struct net_device *dev)
8405{
8406 struct netdev_notifier_offload_xstats_info info = {
8407 .info.dev = dev,
8408 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8409 };
8410
8411 call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE,
8412 &info.info);
8413 kfree(dev->offload_xstats_l3);
8414 dev->offload_xstats_l3 = NULL;
8415}
8416
8417int netdev_offload_xstats_disable(struct net_device *dev,
8418 enum netdev_offload_xstats_type type)
8419{
8420 ASSERT_RTNL();
8421
8422 if (!netdev_offload_xstats_enabled(dev, type))
8423 return -EALREADY;
8424
8425 switch (type) {
8426 case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8427 netdev_offload_xstats_disable_l3(dev);
8428 return 0;
8429 }
8430
8431 WARN_ON(1);
8432 return -EINVAL;
8433}
8434EXPORT_SYMBOL(netdev_offload_xstats_disable);
8435
8436static void netdev_offload_xstats_disable_all(struct net_device *dev)
8437{
8438 netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
8439}
8440
8441static struct rtnl_hw_stats64 *
8442netdev_offload_xstats_get_ptr(const struct net_device *dev,
8443 enum netdev_offload_xstats_type type)
8444{
8445 switch (type) {
8446 case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8447 return dev->offload_xstats_l3;
8448 }
8449
8450 WARN_ON(1);
8451 return NULL;
8452}
8453
8454bool netdev_offload_xstats_enabled(const struct net_device *dev,
8455 enum netdev_offload_xstats_type type)
8456{
8457 ASSERT_RTNL();
8458
8459 return netdev_offload_xstats_get_ptr(dev, type);
8460}
8461EXPORT_SYMBOL(netdev_offload_xstats_enabled);
8462
8463struct netdev_notifier_offload_xstats_ru {
8464 bool used;
8465};
8466
8467struct netdev_notifier_offload_xstats_rd {
8468 struct rtnl_hw_stats64 stats;
8469 bool used;
8470};
8471
8472static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest,
8473 const struct rtnl_hw_stats64 *src)
8474{
8475 dest->rx_packets += src->rx_packets;
8476 dest->tx_packets += src->tx_packets;
8477 dest->rx_bytes += src->rx_bytes;
8478 dest->tx_bytes += src->tx_bytes;
8479 dest->rx_errors += src->rx_errors;
8480 dest->tx_errors += src->tx_errors;
8481 dest->rx_dropped += src->rx_dropped;
8482 dest->tx_dropped += src->tx_dropped;
8483 dest->multicast += src->multicast;
8484}
8485
8486static int netdev_offload_xstats_get_used(struct net_device *dev,
8487 enum netdev_offload_xstats_type type,
8488 bool *p_used,
8489 struct netlink_ext_ack *extack)
8490{
8491 struct netdev_notifier_offload_xstats_ru report_used = {};
8492 struct netdev_notifier_offload_xstats_info info = {
8493 .info.dev = dev,
8494 .info.extack = extack,
8495 .type = type,
8496 .report_used = &report_used,
8497 };
8498 int rc;
8499
8500 WARN_ON(!netdev_offload_xstats_enabled(dev, type));
8501 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED,
8502 &info.info);
8503 *p_used = report_used.used;
8504 return notifier_to_errno(rc);
8505}
8506
8507static int netdev_offload_xstats_get_stats(struct net_device *dev,
8508 enum netdev_offload_xstats_type type,
8509 struct rtnl_hw_stats64 *p_stats,
8510 bool *p_used,
8511 struct netlink_ext_ack *extack)
8512{
8513 struct netdev_notifier_offload_xstats_rd report_delta = {};
8514 struct netdev_notifier_offload_xstats_info info = {
8515 .info.dev = dev,
8516 .info.extack = extack,
8517 .type = type,
8518 .report_delta = &report_delta,
8519 };
8520 struct rtnl_hw_stats64 *stats;
8521 int rc;
8522
8523 stats = netdev_offload_xstats_get_ptr(dev, type);
8524 if (WARN_ON(!stats))
8525 return -EINVAL;
8526
8527 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
8528 &info.info);
8529
8530 /* Cache whatever we got, even if there was an error, otherwise the
8531 * successful stats retrievals would get lost.
8532 */
8533 netdev_hw_stats64_add(stats, &report_delta.stats);
8534
8535 if (p_stats)
8536 *p_stats = *stats;
8537 *p_used = report_delta.used;
8538
8539 return notifier_to_errno(rc);
8540}
8541
8542int netdev_offload_xstats_get(struct net_device *dev,
8543 enum netdev_offload_xstats_type type,
8544 struct rtnl_hw_stats64 *p_stats, bool *p_used,
8545 struct netlink_ext_ack *extack)
8546{
8547 ASSERT_RTNL();
8548
8549 if (p_stats)
8550 return netdev_offload_xstats_get_stats(dev, type, p_stats,
8551 p_used, extack);
8552 else
8553 return netdev_offload_xstats_get_used(dev, type, p_used,
8554 extack);
8555}
8556EXPORT_SYMBOL(netdev_offload_xstats_get);
8557
8558void
8559netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta,
8560 const struct rtnl_hw_stats64 *stats)
8561{
8562 report_delta->used = true;
8563 netdev_hw_stats64_add(&report_delta->stats, stats);
8564}
8565EXPORT_SYMBOL(netdev_offload_xstats_report_delta);
8566
8567void
8568netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used)
8569{
8570 report_used->used = true;
8571}
8572EXPORT_SYMBOL(netdev_offload_xstats_report_used);
8573
8574void netdev_offload_xstats_push_delta(struct net_device *dev,
8575 enum netdev_offload_xstats_type type,
8576 const struct rtnl_hw_stats64 *p_stats)
8577{
8578 struct rtnl_hw_stats64 *stats;
8579
8580 ASSERT_RTNL();
8581
8582 stats = netdev_offload_xstats_get_ptr(dev, type);
8583 if (WARN_ON(!stats))
8584 return;
8585
8586 netdev_hw_stats64_add(stats, p_stats);
8587}
8588EXPORT_SYMBOL(netdev_offload_xstats_push_delta);
8589
8590/**
8591 * netdev_get_xmit_slave - Get the xmit slave of master device
8592 * @dev: device
8593 * @skb: The packet
8594 * @all_slaves: assume all the slaves are active
8595 *
8596 * The reference counters are not incremented so the caller must be
8597 * careful with locks. The caller must hold RCU lock.
8598 * %NULL is returned if no slave is found.
8599 */
8600
8601struct net_device *netdev_get_xmit_slave(struct net_device *dev,
8602 struct sk_buff *skb,
8603 bool all_slaves)
8604{
8605 const struct net_device_ops *ops = dev->netdev_ops;
8606
8607 if (!ops->ndo_get_xmit_slave)
8608 return NULL;
8609 return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
8610}
8611EXPORT_SYMBOL(netdev_get_xmit_slave);
8612
8613static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
8614 struct sock *sk)
8615{
8616 const struct net_device_ops *ops = dev->netdev_ops;
8617
8618 if (!ops->ndo_sk_get_lower_dev)
8619 return NULL;
8620 return ops->ndo_sk_get_lower_dev(dev, sk);
8621}
8622
8623/**
8624 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8625 * @dev: device
8626 * @sk: the socket
8627 *
8628 * %NULL is returned if no lower device is found.
8629 */
8630
8631struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
8632 struct sock *sk)
8633{
8634 struct net_device *lower;
8635
8636 lower = netdev_sk_get_lower_dev(dev, sk);
8637 while (lower) {
8638 dev = lower;
8639 lower = netdev_sk_get_lower_dev(dev, sk);
8640 }
8641
8642 return dev;
8643}
8644EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
8645
8646static void netdev_adjacent_add_links(struct net_device *dev)
8647{
8648 struct netdev_adjacent *iter;
8649
8650 struct net *net = dev_net(dev);
8651
8652 list_for_each_entry(iter, &dev->adj_list.upper, list) {
8653 if (!net_eq(net, dev_net(iter->dev)))
8654 continue;
8655 netdev_adjacent_sysfs_add(iter->dev, dev,
8656 &iter->dev->adj_list.lower);
8657 netdev_adjacent_sysfs_add(dev, iter->dev,
8658 &dev->adj_list.upper);
8659 }
8660
8661 list_for_each_entry(iter, &dev->adj_list.lower, list) {
8662 if (!net_eq(net, dev_net(iter->dev)))
8663 continue;
8664 netdev_adjacent_sysfs_add(iter->dev, dev,
8665 &iter->dev->adj_list.upper);
8666 netdev_adjacent_sysfs_add(dev, iter->dev,
8667 &dev->adj_list.lower);
8668 }
8669}
8670
8671static void netdev_adjacent_del_links(struct net_device *dev)
8672{
8673 struct netdev_adjacent *iter;
8674
8675 struct net *net = dev_net(dev);
8676
8677 list_for_each_entry(iter, &dev->adj_list.upper, list) {
8678 if (!net_eq(net, dev_net(iter->dev)))
8679 continue;
8680 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8681 &iter->dev->adj_list.lower);
8682 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8683 &dev->adj_list.upper);
8684 }
8685
8686 list_for_each_entry(iter, &dev->adj_list.lower, list) {
8687 if (!net_eq(net, dev_net(iter->dev)))
8688 continue;
8689 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8690 &iter->dev->adj_list.upper);
8691 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8692 &dev->adj_list.lower);
8693 }
8694}
8695
8696void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
8697{
8698 struct netdev_adjacent *iter;
8699
8700 struct net *net = dev_net(dev);
8701
8702 list_for_each_entry(iter, &dev->adj_list.upper, list) {
8703 if (!net_eq(net, dev_net(iter->dev)))
8704 continue;
8705 netdev_adjacent_sysfs_del(iter->dev, oldname,
8706 &iter->dev->adj_list.lower);
8707 netdev_adjacent_sysfs_add(iter->dev, dev,
8708 &iter->dev->adj_list.lower);
8709 }
8710
8711 list_for_each_entry(iter, &dev->adj_list.lower, list) {
8712 if (!net_eq(net, dev_net(iter->dev)))
8713 continue;
8714 netdev_adjacent_sysfs_del(iter->dev, oldname,
8715 &iter->dev->adj_list.upper);
8716 netdev_adjacent_sysfs_add(iter->dev, dev,
8717 &iter->dev->adj_list.upper);
8718 }
8719}
8720
8721void *netdev_lower_dev_get_private(struct net_device *dev,
8722 struct net_device *lower_dev)
8723{
8724 struct netdev_adjacent *lower;
8725
8726 if (!lower_dev)
8727 return NULL;
8728 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
8729 if (!lower)
8730 return NULL;
8731
8732 return lower->private;
8733}
8734EXPORT_SYMBOL(netdev_lower_dev_get_private);
8735
8736
8737/**
8738 * netdev_lower_state_changed - Dispatch event about lower device state change
8739 * @lower_dev: device
8740 * @lower_state_info: state to dispatch
8741 *
8742 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
8743 * The caller must hold the RTNL lock.
8744 */
8745void netdev_lower_state_changed(struct net_device *lower_dev,
8746 void *lower_state_info)
8747{
8748 struct netdev_notifier_changelowerstate_info changelowerstate_info = {
8749 .info.dev = lower_dev,
8750 };
8751
8752 ASSERT_RTNL();
8753 changelowerstate_info.lower_state_info = lower_state_info;
8754 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
8755 &changelowerstate_info.info);
8756}
8757EXPORT_SYMBOL(netdev_lower_state_changed);
8758
8759static void dev_change_rx_flags(struct net_device *dev, int flags)
8760{
8761 const struct net_device_ops *ops = dev->netdev_ops;
8762
8763 if (ops->ndo_change_rx_flags)
8764 ops->ndo_change_rx_flags(dev, flags);
8765}
8766
8767static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
8768{
8769 unsigned int old_flags = dev->flags;
8770 unsigned int promiscuity, flags;
8771 kuid_t uid;
8772 kgid_t gid;
8773
8774 ASSERT_RTNL();
8775
8776 promiscuity = dev->promiscuity + inc;
8777 if (promiscuity == 0) {
8778 /*
8779 * Avoid overflow.
8780 * If inc causes overflow, untouch promisc and return error.
8781 */
8782 if (unlikely(inc > 0)) {
8783 netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n");
8784 return -EOVERFLOW;
8785 }
8786 flags = old_flags & ~IFF_PROMISC;
8787 } else {
8788 flags = old_flags | IFF_PROMISC;
8789 }
8790 WRITE_ONCE(dev->promiscuity, promiscuity);
8791 if (flags != old_flags) {
8792 WRITE_ONCE(dev->flags, flags);
8793 netdev_info(dev, "%s promiscuous mode\n",
8794 dev->flags & IFF_PROMISC ? "entered" : "left");
8795 if (audit_enabled) {
8796 current_uid_gid(&uid, &gid);
8797 audit_log(audit_context(), GFP_ATOMIC,
8798 AUDIT_ANOM_PROMISCUOUS,
8799 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
8800 dev->name, (dev->flags & IFF_PROMISC),
8801 (old_flags & IFF_PROMISC),
8802 from_kuid(&init_user_ns, audit_get_loginuid(current)),
8803 from_kuid(&init_user_ns, uid),
8804 from_kgid(&init_user_ns, gid),
8805 audit_get_sessionid(current));
8806 }
8807
8808 dev_change_rx_flags(dev, IFF_PROMISC);
8809 }
8810 if (notify)
8811 __dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL);
8812 return 0;
8813}
8814
8815/**
8816 * dev_set_promiscuity - update promiscuity count on a device
8817 * @dev: device
8818 * @inc: modifier
8819 *
8820 * Add or remove promiscuity from a device. While the count in the device
8821 * remains above zero the interface remains promiscuous. Once it hits zero
8822 * the device reverts back to normal filtering operation. A negative inc
8823 * value is used to drop promiscuity on the device.
8824 * Return 0 if successful or a negative errno code on error.
8825 */
8826int dev_set_promiscuity(struct net_device *dev, int inc)
8827{
8828 unsigned int old_flags = dev->flags;
8829 int err;
8830
8831 err = __dev_set_promiscuity(dev, inc, true);
8832 if (err < 0)
8833 return err;
8834 if (dev->flags != old_flags)
8835 dev_set_rx_mode(dev);
8836 return err;
8837}
8838EXPORT_SYMBOL(dev_set_promiscuity);
8839
8840static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
8841{
8842 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
8843 unsigned int allmulti, flags;
8844
8845 ASSERT_RTNL();
8846
8847 allmulti = dev->allmulti + inc;
8848 if (allmulti == 0) {
8849 /*
8850 * Avoid overflow.
8851 * If inc causes overflow, untouch allmulti and return error.
8852 */
8853 if (unlikely(inc > 0)) {
8854 netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n");
8855 return -EOVERFLOW;
8856 }
8857 flags = old_flags & ~IFF_ALLMULTI;
8858 } else {
8859 flags = old_flags | IFF_ALLMULTI;
8860 }
8861 WRITE_ONCE(dev->allmulti, allmulti);
8862 if (flags != old_flags) {
8863 WRITE_ONCE(dev->flags, flags);
8864 netdev_info(dev, "%s allmulticast mode\n",
8865 dev->flags & IFF_ALLMULTI ? "entered" : "left");
8866 dev_change_rx_flags(dev, IFF_ALLMULTI);
8867 dev_set_rx_mode(dev);
8868 if (notify)
8869 __dev_notify_flags(dev, old_flags,
8870 dev->gflags ^ old_gflags, 0, NULL);
8871 }
8872 return 0;
8873}
8874
8875/**
8876 * dev_set_allmulti - update allmulti count on a device
8877 * @dev: device
8878 * @inc: modifier
8879 *
8880 * Add or remove reception of all multicast frames to a device. While the
8881 * count in the device remains above zero the interface remains listening
8882 * to all interfaces. Once it hits zero the device reverts back to normal
8883 * filtering operation. A negative @inc value is used to drop the counter
8884 * when releasing a resource needing all multicasts.
8885 * Return 0 if successful or a negative errno code on error.
8886 */
8887
8888int dev_set_allmulti(struct net_device *dev, int inc)
8889{
8890 return __dev_set_allmulti(dev, inc, true);
8891}
8892EXPORT_SYMBOL(dev_set_allmulti);
8893
8894/*
8895 * Upload unicast and multicast address lists to device and
8896 * configure RX filtering. When the device doesn't support unicast
8897 * filtering it is put in promiscuous mode while unicast addresses
8898 * are present.
8899 */
8900void __dev_set_rx_mode(struct net_device *dev)
8901{
8902 const struct net_device_ops *ops = dev->netdev_ops;
8903
8904 /* dev_open will call this function so the list will stay sane. */
8905 if (!(dev->flags&IFF_UP))
8906 return;
8907
8908 if (!netif_device_present(dev))
8909 return;
8910
8911 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
8912 /* Unicast addresses changes may only happen under the rtnl,
8913 * therefore calling __dev_set_promiscuity here is safe.
8914 */
8915 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
8916 __dev_set_promiscuity(dev, 1, false);
8917 dev->uc_promisc = true;
8918 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
8919 __dev_set_promiscuity(dev, -1, false);
8920 dev->uc_promisc = false;
8921 }
8922 }
8923
8924 if (ops->ndo_set_rx_mode)
8925 ops->ndo_set_rx_mode(dev);
8926}
8927
8928void dev_set_rx_mode(struct net_device *dev)
8929{
8930 netif_addr_lock_bh(dev);
8931 __dev_set_rx_mode(dev);
8932 netif_addr_unlock_bh(dev);
8933}
8934
8935/**
8936 * dev_get_flags - get flags reported to userspace
8937 * @dev: device
8938 *
8939 * Get the combination of flag bits exported through APIs to userspace.
8940 */
8941unsigned int dev_get_flags(const struct net_device *dev)
8942{
8943 unsigned int flags;
8944
8945 flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC |
8946 IFF_ALLMULTI |
8947 IFF_RUNNING |
8948 IFF_LOWER_UP |
8949 IFF_DORMANT)) |
8950 (READ_ONCE(dev->gflags) & (IFF_PROMISC |
8951 IFF_ALLMULTI));
8952
8953 if (netif_running(dev)) {
8954 if (netif_oper_up(dev))
8955 flags |= IFF_RUNNING;
8956 if (netif_carrier_ok(dev))
8957 flags |= IFF_LOWER_UP;
8958 if (netif_dormant(dev))
8959 flags |= IFF_DORMANT;
8960 }
8961
8962 return flags;
8963}
8964EXPORT_SYMBOL(dev_get_flags);
8965
8966int __dev_change_flags(struct net_device *dev, unsigned int flags,
8967 struct netlink_ext_ack *extack)
8968{
8969 unsigned int old_flags = dev->flags;
8970 int ret;
8971
8972 ASSERT_RTNL();
8973
8974 /*
8975 * Set the flags on our device.
8976 */
8977
8978 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
8979 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
8980 IFF_AUTOMEDIA)) |
8981 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
8982 IFF_ALLMULTI));
8983
8984 /*
8985 * Load in the correct multicast list now the flags have changed.
8986 */
8987
8988 if ((old_flags ^ flags) & IFF_MULTICAST)
8989 dev_change_rx_flags(dev, IFF_MULTICAST);
8990
8991 dev_set_rx_mode(dev);
8992
8993 /*
8994 * Have we downed the interface. We handle IFF_UP ourselves
8995 * according to user attempts to set it, rather than blindly
8996 * setting it.
8997 */
8998
8999 ret = 0;
9000 if ((old_flags ^ flags) & IFF_UP) {
9001 if (old_flags & IFF_UP)
9002 __dev_close(dev);
9003 else
9004 ret = __dev_open(dev, extack);
9005 }
9006
9007 if ((flags ^ dev->gflags) & IFF_PROMISC) {
9008 int inc = (flags & IFF_PROMISC) ? 1 : -1;
9009 unsigned int old_flags = dev->flags;
9010
9011 dev->gflags ^= IFF_PROMISC;
9012
9013 if (__dev_set_promiscuity(dev, inc, false) >= 0)
9014 if (dev->flags != old_flags)
9015 dev_set_rx_mode(dev);
9016 }
9017
9018 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
9019 * is important. Some (broken) drivers set IFF_PROMISC, when
9020 * IFF_ALLMULTI is requested not asking us and not reporting.
9021 */
9022 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
9023 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
9024
9025 dev->gflags ^= IFF_ALLMULTI;
9026 __dev_set_allmulti(dev, inc, false);
9027 }
9028
9029 return ret;
9030}
9031
9032void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
9033 unsigned int gchanges, u32 portid,
9034 const struct nlmsghdr *nlh)
9035{
9036 unsigned int changes = dev->flags ^ old_flags;
9037
9038 if (gchanges)
9039 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh);
9040
9041 if (changes & IFF_UP) {
9042 if (dev->flags & IFF_UP)
9043 call_netdevice_notifiers(NETDEV_UP, dev);
9044 else
9045 call_netdevice_notifiers(NETDEV_DOWN, dev);
9046 }
9047
9048 if (dev->flags & IFF_UP &&
9049 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
9050 struct netdev_notifier_change_info change_info = {
9051 .info = {
9052 .dev = dev,
9053 },
9054 .flags_changed = changes,
9055 };
9056
9057 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
9058 }
9059}
9060
9061/**
9062 * dev_change_flags - change device settings
9063 * @dev: device
9064 * @flags: device state flags
9065 * @extack: netlink extended ack
9066 *
9067 * Change settings on device based state flags. The flags are
9068 * in the userspace exported format.
9069 */
9070int dev_change_flags(struct net_device *dev, unsigned int flags,
9071 struct netlink_ext_ack *extack)
9072{
9073 int ret;
9074 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
9075
9076 ret = __dev_change_flags(dev, flags, extack);
9077 if (ret < 0)
9078 return ret;
9079
9080 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
9081 __dev_notify_flags(dev, old_flags, changes, 0, NULL);
9082 return ret;
9083}
9084EXPORT_SYMBOL(dev_change_flags);
9085
9086int __dev_set_mtu(struct net_device *dev, int new_mtu)
9087{
9088 const struct net_device_ops *ops = dev->netdev_ops;
9089
9090 if (ops->ndo_change_mtu)
9091 return ops->ndo_change_mtu(dev, new_mtu);
9092
9093 /* Pairs with all the lockless reads of dev->mtu in the stack */
9094 WRITE_ONCE(dev->mtu, new_mtu);
9095 return 0;
9096}
9097EXPORT_SYMBOL(__dev_set_mtu);
9098
9099int dev_validate_mtu(struct net_device *dev, int new_mtu,
9100 struct netlink_ext_ack *extack)
9101{
9102 /* MTU must be positive, and in range */
9103 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
9104 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
9105 return -EINVAL;
9106 }
9107
9108 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
9109 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
9110 return -EINVAL;
9111 }
9112 return 0;
9113}
9114
9115/**
9116 * dev_set_mtu_ext - Change maximum transfer unit
9117 * @dev: device
9118 * @new_mtu: new transfer unit
9119 * @extack: netlink extended ack
9120 *
9121 * Change the maximum transfer size of the network device.
9122 */
9123int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
9124 struct netlink_ext_ack *extack)
9125{
9126 int err, orig_mtu;
9127
9128 if (new_mtu == dev->mtu)
9129 return 0;
9130
9131 err = dev_validate_mtu(dev, new_mtu, extack);
9132 if (err)
9133 return err;
9134
9135 if (!netif_device_present(dev))
9136 return -ENODEV;
9137
9138 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
9139 err = notifier_to_errno(err);
9140 if (err)
9141 return err;
9142
9143 orig_mtu = dev->mtu;
9144 err = __dev_set_mtu(dev, new_mtu);
9145
9146 if (!err) {
9147 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
9148 orig_mtu);
9149 err = notifier_to_errno(err);
9150 if (err) {
9151 /* setting mtu back and notifying everyone again,
9152 * so that they have a chance to revert changes.
9153 */
9154 __dev_set_mtu(dev, orig_mtu);
9155 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
9156 new_mtu);
9157 }
9158 }
9159 return err;
9160}
9161
9162int dev_set_mtu(struct net_device *dev, int new_mtu)
9163{
9164 struct netlink_ext_ack extack;
9165 int err;
9166
9167 memset(&extack, 0, sizeof(extack));
9168 err = dev_set_mtu_ext(dev, new_mtu, &extack);
9169 if (err && extack._msg)
9170 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
9171 return err;
9172}
9173EXPORT_SYMBOL(dev_set_mtu);
9174
9175/**
9176 * dev_change_tx_queue_len - Change TX queue length of a netdevice
9177 * @dev: device
9178 * @new_len: new tx queue length
9179 */
9180int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
9181{
9182 unsigned int orig_len = dev->tx_queue_len;
9183 int res;
9184
9185 if (new_len != (unsigned int)new_len)
9186 return -ERANGE;
9187
9188 if (new_len != orig_len) {
9189 WRITE_ONCE(dev->tx_queue_len, new_len);
9190 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
9191 res = notifier_to_errno(res);
9192 if (res)
9193 goto err_rollback;
9194 res = dev_qdisc_change_tx_queue_len(dev);
9195 if (res)
9196 goto err_rollback;
9197 }
9198
9199 return 0;
9200
9201err_rollback:
9202 netdev_err(dev, "refused to change device tx_queue_len\n");
9203 WRITE_ONCE(dev->tx_queue_len, orig_len);
9204 return res;
9205}
9206
9207/**
9208 * dev_set_group - Change group this device belongs to
9209 * @dev: device
9210 * @new_group: group this device should belong to
9211 */
9212void dev_set_group(struct net_device *dev, int new_group)
9213{
9214 dev->group = new_group;
9215}
9216
9217/**
9218 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
9219 * @dev: device
9220 * @addr: new address
9221 * @extack: netlink extended ack
9222 */
9223int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
9224 struct netlink_ext_ack *extack)
9225{
9226 struct netdev_notifier_pre_changeaddr_info info = {
9227 .info.dev = dev,
9228 .info.extack = extack,
9229 .dev_addr = addr,
9230 };
9231 int rc;
9232
9233 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
9234 return notifier_to_errno(rc);
9235}
9236EXPORT_SYMBOL(dev_pre_changeaddr_notify);
9237
9238/**
9239 * dev_set_mac_address - Change Media Access Control Address
9240 * @dev: device
9241 * @sa: new address
9242 * @extack: netlink extended ack
9243 *
9244 * Change the hardware (MAC) address of the device
9245 */
9246int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
9247 struct netlink_ext_ack *extack)
9248{
9249 const struct net_device_ops *ops = dev->netdev_ops;
9250 int err;
9251
9252 if (!ops->ndo_set_mac_address)
9253 return -EOPNOTSUPP;
9254 if (sa->sa_family != dev->type)
9255 return -EINVAL;
9256 if (!netif_device_present(dev))
9257 return -ENODEV;
9258 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
9259 if (err)
9260 return err;
9261 if (memcmp(dev->dev_addr, sa->sa_data, dev->addr_len)) {
9262 err = ops->ndo_set_mac_address(dev, sa);
9263 if (err)
9264 return err;
9265 }
9266 dev->addr_assign_type = NET_ADDR_SET;
9267 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
9268 add_device_randomness(dev->dev_addr, dev->addr_len);
9269 return 0;
9270}
9271EXPORT_SYMBOL(dev_set_mac_address);
9272
9273DECLARE_RWSEM(dev_addr_sem);
9274
9275int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
9276 struct netlink_ext_ack *extack)
9277{
9278 int ret;
9279
9280 down_write(&dev_addr_sem);
9281 ret = dev_set_mac_address(dev, sa, extack);
9282 up_write(&dev_addr_sem);
9283 return ret;
9284}
9285EXPORT_SYMBOL(dev_set_mac_address_user);
9286
9287int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
9288{
9289 size_t size = sizeof(sa->sa_data_min);
9290 struct net_device *dev;
9291 int ret = 0;
9292
9293 down_read(&dev_addr_sem);
9294 rcu_read_lock();
9295
9296 dev = dev_get_by_name_rcu(net, dev_name);
9297 if (!dev) {
9298 ret = -ENODEV;
9299 goto unlock;
9300 }
9301 if (!dev->addr_len)
9302 memset(sa->sa_data, 0, size);
9303 else
9304 memcpy(sa->sa_data, dev->dev_addr,
9305 min_t(size_t, size, dev->addr_len));
9306 sa->sa_family = dev->type;
9307
9308unlock:
9309 rcu_read_unlock();
9310 up_read(&dev_addr_sem);
9311 return ret;
9312}
9313EXPORT_SYMBOL(dev_get_mac_address);
9314
9315/**
9316 * dev_change_carrier - Change device carrier
9317 * @dev: device
9318 * @new_carrier: new value
9319 *
9320 * Change device carrier
9321 */
9322int dev_change_carrier(struct net_device *dev, bool new_carrier)
9323{
9324 const struct net_device_ops *ops = dev->netdev_ops;
9325
9326 if (!ops->ndo_change_carrier)
9327 return -EOPNOTSUPP;
9328 if (!netif_device_present(dev))
9329 return -ENODEV;
9330 return ops->ndo_change_carrier(dev, new_carrier);
9331}
9332
9333/**
9334 * dev_get_phys_port_id - Get device physical port ID
9335 * @dev: device
9336 * @ppid: port ID
9337 *
9338 * Get device physical port ID
9339 */
9340int dev_get_phys_port_id(struct net_device *dev,
9341 struct netdev_phys_item_id *ppid)
9342{
9343 const struct net_device_ops *ops = dev->netdev_ops;
9344
9345 if (!ops->ndo_get_phys_port_id)
9346 return -EOPNOTSUPP;
9347 return ops->ndo_get_phys_port_id(dev, ppid);
9348}
9349
9350/**
9351 * dev_get_phys_port_name - Get device physical port name
9352 * @dev: device
9353 * @name: port name
9354 * @len: limit of bytes to copy to name
9355 *
9356 * Get device physical port name
9357 */
9358int dev_get_phys_port_name(struct net_device *dev,
9359 char *name, size_t len)
9360{
9361 const struct net_device_ops *ops = dev->netdev_ops;
9362 int err;
9363
9364 if (ops->ndo_get_phys_port_name) {
9365 err = ops->ndo_get_phys_port_name(dev, name, len);
9366 if (err != -EOPNOTSUPP)
9367 return err;
9368 }
9369 return devlink_compat_phys_port_name_get(dev, name, len);
9370}
9371
9372/**
9373 * dev_get_port_parent_id - Get the device's port parent identifier
9374 * @dev: network device
9375 * @ppid: pointer to a storage for the port's parent identifier
9376 * @recurse: allow/disallow recursion to lower devices
9377 *
9378 * Get the devices's port parent identifier
9379 */
9380int dev_get_port_parent_id(struct net_device *dev,
9381 struct netdev_phys_item_id *ppid,
9382 bool recurse)
9383{
9384 const struct net_device_ops *ops = dev->netdev_ops;
9385 struct netdev_phys_item_id first = { };
9386 struct net_device *lower_dev;
9387 struct list_head *iter;
9388 int err;
9389
9390 if (ops->ndo_get_port_parent_id) {
9391 err = ops->ndo_get_port_parent_id(dev, ppid);
9392 if (err != -EOPNOTSUPP)
9393 return err;
9394 }
9395
9396 err = devlink_compat_switch_id_get(dev, ppid);
9397 if (!recurse || err != -EOPNOTSUPP)
9398 return err;
9399
9400 netdev_for_each_lower_dev(dev, lower_dev, iter) {
9401 err = dev_get_port_parent_id(lower_dev, ppid, true);
9402 if (err)
9403 break;
9404 if (!first.id_len)
9405 first = *ppid;
9406 else if (memcmp(&first, ppid, sizeof(*ppid)))
9407 return -EOPNOTSUPP;
9408 }
9409
9410 return err;
9411}
9412EXPORT_SYMBOL(dev_get_port_parent_id);
9413
9414/**
9415 * netdev_port_same_parent_id - Indicate if two network devices have
9416 * the same port parent identifier
9417 * @a: first network device
9418 * @b: second network device
9419 */
9420bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
9421{
9422 struct netdev_phys_item_id a_id = { };
9423 struct netdev_phys_item_id b_id = { };
9424
9425 if (dev_get_port_parent_id(a, &a_id, true) ||
9426 dev_get_port_parent_id(b, &b_id, true))
9427 return false;
9428
9429 return netdev_phys_item_id_same(&a_id, &b_id);
9430}
9431EXPORT_SYMBOL(netdev_port_same_parent_id);
9432
9433/**
9434 * dev_change_proto_down - set carrier according to proto_down.
9435 *
9436 * @dev: device
9437 * @proto_down: new value
9438 */
9439int dev_change_proto_down(struct net_device *dev, bool proto_down)
9440{
9441 if (!dev->change_proto_down)
9442 return -EOPNOTSUPP;
9443 if (!netif_device_present(dev))
9444 return -ENODEV;
9445 if (proto_down)
9446 netif_carrier_off(dev);
9447 else
9448 netif_carrier_on(dev);
9449 WRITE_ONCE(dev->proto_down, proto_down);
9450 return 0;
9451}
9452
9453/**
9454 * dev_change_proto_down_reason - proto down reason
9455 *
9456 * @dev: device
9457 * @mask: proto down mask
9458 * @value: proto down value
9459 */
9460void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
9461 u32 value)
9462{
9463 u32 proto_down_reason;
9464 int b;
9465
9466 if (!mask) {
9467 proto_down_reason = value;
9468 } else {
9469 proto_down_reason = dev->proto_down_reason;
9470 for_each_set_bit(b, &mask, 32) {
9471 if (value & (1 << b))
9472 proto_down_reason |= BIT(b);
9473 else
9474 proto_down_reason &= ~BIT(b);
9475 }
9476 }
9477 WRITE_ONCE(dev->proto_down_reason, proto_down_reason);
9478}
9479
9480struct bpf_xdp_link {
9481 struct bpf_link link;
9482 struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
9483 int flags;
9484};
9485
9486static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
9487{
9488 if (flags & XDP_FLAGS_HW_MODE)
9489 return XDP_MODE_HW;
9490 if (flags & XDP_FLAGS_DRV_MODE)
9491 return XDP_MODE_DRV;
9492 if (flags & XDP_FLAGS_SKB_MODE)
9493 return XDP_MODE_SKB;
9494 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
9495}
9496
9497static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
9498{
9499 switch (mode) {
9500 case XDP_MODE_SKB:
9501 return generic_xdp_install;
9502 case XDP_MODE_DRV:
9503 case XDP_MODE_HW:
9504 return dev->netdev_ops->ndo_bpf;
9505 default:
9506 return NULL;
9507 }
9508}
9509
9510static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
9511 enum bpf_xdp_mode mode)
9512{
9513 return dev->xdp_state[mode].link;
9514}
9515
9516static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
9517 enum bpf_xdp_mode mode)
9518{
9519 struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
9520
9521 if (link)
9522 return link->link.prog;
9523 return dev->xdp_state[mode].prog;
9524}
9525
9526u8 dev_xdp_prog_count(struct net_device *dev)
9527{
9528 u8 count = 0;
9529 int i;
9530
9531 for (i = 0; i < __MAX_XDP_MODE; i++)
9532 if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
9533 count++;
9534 return count;
9535}
9536EXPORT_SYMBOL_GPL(dev_xdp_prog_count);
9537
9538int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
9539{
9540 if (!dev->netdev_ops->ndo_bpf)
9541 return -EOPNOTSUPP;
9542
9543 if (dev_get_min_mp_channel_count(dev)) {
9544 NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider");
9545 return -EBUSY;
9546 }
9547
9548 return dev->netdev_ops->ndo_bpf(dev, bpf);
9549}
9550EXPORT_SYMBOL_GPL(dev_xdp_propagate);
9551
9552u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
9553{
9554 struct bpf_prog *prog = dev_xdp_prog(dev, mode);
9555
9556 return prog ? prog->aux->id : 0;
9557}
9558
9559static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
9560 struct bpf_xdp_link *link)
9561{
9562 dev->xdp_state[mode].link = link;
9563 dev->xdp_state[mode].prog = NULL;
9564}
9565
9566static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
9567 struct bpf_prog *prog)
9568{
9569 dev->xdp_state[mode].link = NULL;
9570 dev->xdp_state[mode].prog = prog;
9571}
9572
9573static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
9574 bpf_op_t bpf_op, struct netlink_ext_ack *extack,
9575 u32 flags, struct bpf_prog *prog)
9576{
9577 struct netdev_bpf xdp;
9578 int err;
9579
9580 if (dev_get_min_mp_channel_count(dev)) {
9581 NL_SET_ERR_MSG(extack, "unable to install XDP to device using memory provider");
9582 return -EBUSY;
9583 }
9584
9585 memset(&xdp, 0, sizeof(xdp));
9586 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
9587 xdp.extack = extack;
9588 xdp.flags = flags;
9589 xdp.prog = prog;
9590
9591 /* Drivers assume refcnt is already incremented (i.e, prog pointer is
9592 * "moved" into driver), so they don't increment it on their own, but
9593 * they do decrement refcnt when program is detached or replaced.
9594 * Given net_device also owns link/prog, we need to bump refcnt here
9595 * to prevent drivers from underflowing it.
9596 */
9597 if (prog)
9598 bpf_prog_inc(prog);
9599 err = bpf_op(dev, &xdp);
9600 if (err) {
9601 if (prog)
9602 bpf_prog_put(prog);
9603 return err;
9604 }
9605
9606 if (mode != XDP_MODE_HW)
9607 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
9608
9609 return 0;
9610}
9611
9612static void dev_xdp_uninstall(struct net_device *dev)
9613{
9614 struct bpf_xdp_link *link;
9615 struct bpf_prog *prog;
9616 enum bpf_xdp_mode mode;
9617 bpf_op_t bpf_op;
9618
9619 ASSERT_RTNL();
9620
9621 for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
9622 prog = dev_xdp_prog(dev, mode);
9623 if (!prog)
9624 continue;
9625
9626 bpf_op = dev_xdp_bpf_op(dev, mode);
9627 if (!bpf_op)
9628 continue;
9629
9630 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9631
9632 /* auto-detach link from net device */
9633 link = dev_xdp_link(dev, mode);
9634 if (link)
9635 link->dev = NULL;
9636 else
9637 bpf_prog_put(prog);
9638
9639 dev_xdp_set_link(dev, mode, NULL);
9640 }
9641}
9642
9643static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
9644 struct bpf_xdp_link *link, struct bpf_prog *new_prog,
9645 struct bpf_prog *old_prog, u32 flags)
9646{
9647 unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
9648 struct bpf_prog *cur_prog;
9649 struct net_device *upper;
9650 struct list_head *iter;
9651 enum bpf_xdp_mode mode;
9652 bpf_op_t bpf_op;
9653 int err;
9654
9655 ASSERT_RTNL();
9656
9657 /* either link or prog attachment, never both */
9658 if (link && (new_prog || old_prog))
9659 return -EINVAL;
9660 /* link supports only XDP mode flags */
9661 if (link && (flags & ~XDP_FLAGS_MODES)) {
9662 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
9663 return -EINVAL;
9664 }
9665 /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
9666 if (num_modes > 1) {
9667 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
9668 return -EINVAL;
9669 }
9670 /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
9671 if (!num_modes && dev_xdp_prog_count(dev) > 1) {
9672 NL_SET_ERR_MSG(extack,
9673 "More than one program loaded, unset mode is ambiguous");
9674 return -EINVAL;
9675 }
9676 /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
9677 if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
9678 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
9679 return -EINVAL;
9680 }
9681
9682 mode = dev_xdp_mode(dev, flags);
9683 /* can't replace attached link */
9684 if (dev_xdp_link(dev, mode)) {
9685 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
9686 return -EBUSY;
9687 }
9688
9689 /* don't allow if an upper device already has a program */
9690 netdev_for_each_upper_dev_rcu(dev, upper, iter) {
9691 if (dev_xdp_prog_count(upper) > 0) {
9692 NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program");
9693 return -EEXIST;
9694 }
9695 }
9696
9697 cur_prog = dev_xdp_prog(dev, mode);
9698 /* can't replace attached prog with link */
9699 if (link && cur_prog) {
9700 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
9701 return -EBUSY;
9702 }
9703 if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
9704 NL_SET_ERR_MSG(extack, "Active program does not match expected");
9705 return -EEXIST;
9706 }
9707
9708 /* put effective new program into new_prog */
9709 if (link)
9710 new_prog = link->link.prog;
9711
9712 if (new_prog) {
9713 bool offload = mode == XDP_MODE_HW;
9714 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
9715 ? XDP_MODE_DRV : XDP_MODE_SKB;
9716
9717 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
9718 NL_SET_ERR_MSG(extack, "XDP program already attached");
9719 return -EBUSY;
9720 }
9721 if (!offload && dev_xdp_prog(dev, other_mode)) {
9722 NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
9723 return -EEXIST;
9724 }
9725 if (!offload && bpf_prog_is_offloaded(new_prog->aux)) {
9726 NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported");
9727 return -EINVAL;
9728 }
9729 if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) {
9730 NL_SET_ERR_MSG(extack, "Program bound to different device");
9731 return -EINVAL;
9732 }
9733 if (bpf_prog_is_dev_bound(new_prog->aux) && mode == XDP_MODE_SKB) {
9734 NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode");
9735 return -EINVAL;
9736 }
9737 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
9738 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
9739 return -EINVAL;
9740 }
9741 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
9742 NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
9743 return -EINVAL;
9744 }
9745 }
9746
9747 /* don't call drivers if the effective program didn't change */
9748 if (new_prog != cur_prog) {
9749 bpf_op = dev_xdp_bpf_op(dev, mode);
9750 if (!bpf_op) {
9751 NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
9752 return -EOPNOTSUPP;
9753 }
9754
9755 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
9756 if (err)
9757 return err;
9758 }
9759
9760 if (link)
9761 dev_xdp_set_link(dev, mode, link);
9762 else
9763 dev_xdp_set_prog(dev, mode, new_prog);
9764 if (cur_prog)
9765 bpf_prog_put(cur_prog);
9766
9767 return 0;
9768}
9769
9770static int dev_xdp_attach_link(struct net_device *dev,
9771 struct netlink_ext_ack *extack,
9772 struct bpf_xdp_link *link)
9773{
9774 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
9775}
9776
9777static int dev_xdp_detach_link(struct net_device *dev,
9778 struct netlink_ext_ack *extack,
9779 struct bpf_xdp_link *link)
9780{
9781 enum bpf_xdp_mode mode;
9782 bpf_op_t bpf_op;
9783
9784 ASSERT_RTNL();
9785
9786 mode = dev_xdp_mode(dev, link->flags);
9787 if (dev_xdp_link(dev, mode) != link)
9788 return -EINVAL;
9789
9790 bpf_op = dev_xdp_bpf_op(dev, mode);
9791 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9792 dev_xdp_set_link(dev, mode, NULL);
9793 return 0;
9794}
9795
9796static void bpf_xdp_link_release(struct bpf_link *link)
9797{
9798 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9799
9800 rtnl_lock();
9801
9802 /* if racing with net_device's tear down, xdp_link->dev might be
9803 * already NULL, in which case link was already auto-detached
9804 */
9805 if (xdp_link->dev) {
9806 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
9807 xdp_link->dev = NULL;
9808 }
9809
9810 rtnl_unlock();
9811}
9812
9813static int bpf_xdp_link_detach(struct bpf_link *link)
9814{
9815 bpf_xdp_link_release(link);
9816 return 0;
9817}
9818
9819static void bpf_xdp_link_dealloc(struct bpf_link *link)
9820{
9821 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9822
9823 kfree(xdp_link);
9824}
9825
9826static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
9827 struct seq_file *seq)
9828{
9829 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9830 u32 ifindex = 0;
9831
9832 rtnl_lock();
9833 if (xdp_link->dev)
9834 ifindex = xdp_link->dev->ifindex;
9835 rtnl_unlock();
9836
9837 seq_printf(seq, "ifindex:\t%u\n", ifindex);
9838}
9839
9840static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
9841 struct bpf_link_info *info)
9842{
9843 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9844 u32 ifindex = 0;
9845
9846 rtnl_lock();
9847 if (xdp_link->dev)
9848 ifindex = xdp_link->dev->ifindex;
9849 rtnl_unlock();
9850
9851 info->xdp.ifindex = ifindex;
9852 return 0;
9853}
9854
9855static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
9856 struct bpf_prog *old_prog)
9857{
9858 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9859 enum bpf_xdp_mode mode;
9860 bpf_op_t bpf_op;
9861 int err = 0;
9862
9863 rtnl_lock();
9864
9865 /* link might have been auto-released already, so fail */
9866 if (!xdp_link->dev) {
9867 err = -ENOLINK;
9868 goto out_unlock;
9869 }
9870
9871 if (old_prog && link->prog != old_prog) {
9872 err = -EPERM;
9873 goto out_unlock;
9874 }
9875 old_prog = link->prog;
9876 if (old_prog->type != new_prog->type ||
9877 old_prog->expected_attach_type != new_prog->expected_attach_type) {
9878 err = -EINVAL;
9879 goto out_unlock;
9880 }
9881
9882 if (old_prog == new_prog) {
9883 /* no-op, don't disturb drivers */
9884 bpf_prog_put(new_prog);
9885 goto out_unlock;
9886 }
9887
9888 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
9889 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
9890 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
9891 xdp_link->flags, new_prog);
9892 if (err)
9893 goto out_unlock;
9894
9895 old_prog = xchg(&link->prog, new_prog);
9896 bpf_prog_put(old_prog);
9897
9898out_unlock:
9899 rtnl_unlock();
9900 return err;
9901}
9902
9903static const struct bpf_link_ops bpf_xdp_link_lops = {
9904 .release = bpf_xdp_link_release,
9905 .dealloc = bpf_xdp_link_dealloc,
9906 .detach = bpf_xdp_link_detach,
9907 .show_fdinfo = bpf_xdp_link_show_fdinfo,
9908 .fill_link_info = bpf_xdp_link_fill_link_info,
9909 .update_prog = bpf_xdp_link_update,
9910};
9911
9912int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
9913{
9914 struct net *net = current->nsproxy->net_ns;
9915 struct bpf_link_primer link_primer;
9916 struct netlink_ext_ack extack = {};
9917 struct bpf_xdp_link *link;
9918 struct net_device *dev;
9919 int err, fd;
9920
9921 rtnl_lock();
9922 dev = dev_get_by_index(net, attr->link_create.target_ifindex);
9923 if (!dev) {
9924 rtnl_unlock();
9925 return -EINVAL;
9926 }
9927
9928 link = kzalloc(sizeof(*link), GFP_USER);
9929 if (!link) {
9930 err = -ENOMEM;
9931 goto unlock;
9932 }
9933
9934 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
9935 link->dev = dev;
9936 link->flags = attr->link_create.flags;
9937
9938 err = bpf_link_prime(&link->link, &link_primer);
9939 if (err) {
9940 kfree(link);
9941 goto unlock;
9942 }
9943
9944 err = dev_xdp_attach_link(dev, &extack, link);
9945 rtnl_unlock();
9946
9947 if (err) {
9948 link->dev = NULL;
9949 bpf_link_cleanup(&link_primer);
9950 trace_bpf_xdp_link_attach_failed(extack._msg);
9951 goto out_put_dev;
9952 }
9953
9954 fd = bpf_link_settle(&link_primer);
9955 /* link itself doesn't hold dev's refcnt to not complicate shutdown */
9956 dev_put(dev);
9957 return fd;
9958
9959unlock:
9960 rtnl_unlock();
9961
9962out_put_dev:
9963 dev_put(dev);
9964 return err;
9965}
9966
9967/**
9968 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
9969 * @dev: device
9970 * @extack: netlink extended ack
9971 * @fd: new program fd or negative value to clear
9972 * @expected_fd: old program fd that userspace expects to replace or clear
9973 * @flags: xdp-related flags
9974 *
9975 * Set or clear a bpf program for a device
9976 */
9977int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
9978 int fd, int expected_fd, u32 flags)
9979{
9980 enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
9981 struct bpf_prog *new_prog = NULL, *old_prog = NULL;
9982 int err;
9983
9984 ASSERT_RTNL();
9985
9986 if (fd >= 0) {
9987 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
9988 mode != XDP_MODE_SKB);
9989 if (IS_ERR(new_prog))
9990 return PTR_ERR(new_prog);
9991 }
9992
9993 if (expected_fd >= 0) {
9994 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
9995 mode != XDP_MODE_SKB);
9996 if (IS_ERR(old_prog)) {
9997 err = PTR_ERR(old_prog);
9998 old_prog = NULL;
9999 goto err_out;
10000 }
10001 }
10002
10003 err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
10004
10005err_out:
10006 if (err && new_prog)
10007 bpf_prog_put(new_prog);
10008 if (old_prog)
10009 bpf_prog_put(old_prog);
10010 return err;
10011}
10012
10013u32 dev_get_min_mp_channel_count(const struct net_device *dev)
10014{
10015 int i;
10016
10017 ASSERT_RTNL();
10018
10019 for (i = dev->real_num_rx_queues - 1; i >= 0; i--)
10020 if (dev->_rx[i].mp_params.mp_priv)
10021 /* The channel count is the idx plus 1. */
10022 return i + 1;
10023
10024 return 0;
10025}
10026
10027/**
10028 * dev_index_reserve() - allocate an ifindex in a namespace
10029 * @net: the applicable net namespace
10030 * @ifindex: requested ifindex, pass %0 to get one allocated
10031 *
10032 * Allocate a ifindex for a new device. Caller must either use the ifindex
10033 * to store the device (via list_netdevice()) or call dev_index_release()
10034 * to give the index up.
10035 *
10036 * Return: a suitable unique value for a new device interface number or -errno.
10037 */
10038static int dev_index_reserve(struct net *net, u32 ifindex)
10039{
10040 int err;
10041
10042 if (ifindex > INT_MAX) {
10043 DEBUG_NET_WARN_ON_ONCE(1);
10044 return -EINVAL;
10045 }
10046
10047 if (!ifindex)
10048 err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL,
10049 xa_limit_31b, &net->ifindex, GFP_KERNEL);
10050 else
10051 err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL);
10052 if (err < 0)
10053 return err;
10054
10055 return ifindex;
10056}
10057
10058static void dev_index_release(struct net *net, int ifindex)
10059{
10060 /* Expect only unused indexes, unlist_netdevice() removes the used */
10061 WARN_ON(xa_erase(&net->dev_by_index, ifindex));
10062}
10063
10064/* Delayed registration/unregisteration */
10065LIST_HEAD(net_todo_list);
10066DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
10067atomic_t dev_unreg_count = ATOMIC_INIT(0);
10068
10069static void net_set_todo(struct net_device *dev)
10070{
10071 list_add_tail(&dev->todo_list, &net_todo_list);
10072}
10073
10074static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
10075 struct net_device *upper, netdev_features_t features)
10076{
10077 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
10078 netdev_features_t feature;
10079 int feature_bit;
10080
10081 for_each_netdev_feature(upper_disables, feature_bit) {
10082 feature = __NETIF_F_BIT(feature_bit);
10083 if (!(upper->wanted_features & feature)
10084 && (features & feature)) {
10085 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
10086 &feature, upper->name);
10087 features &= ~feature;
10088 }
10089 }
10090
10091 return features;
10092}
10093
10094static void netdev_sync_lower_features(struct net_device *upper,
10095 struct net_device *lower, netdev_features_t features)
10096{
10097 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
10098 netdev_features_t feature;
10099 int feature_bit;
10100
10101 for_each_netdev_feature(upper_disables, feature_bit) {
10102 feature = __NETIF_F_BIT(feature_bit);
10103 if (!(features & feature) && (lower->features & feature)) {
10104 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
10105 &feature, lower->name);
10106 lower->wanted_features &= ~feature;
10107 __netdev_update_features(lower);
10108
10109 if (unlikely(lower->features & feature))
10110 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
10111 &feature, lower->name);
10112 else
10113 netdev_features_change(lower);
10114 }
10115 }
10116}
10117
10118static bool netdev_has_ip_or_hw_csum(netdev_features_t features)
10119{
10120 netdev_features_t ip_csum_mask = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
10121 bool ip_csum = (features & ip_csum_mask) == ip_csum_mask;
10122 bool hw_csum = features & NETIF_F_HW_CSUM;
10123
10124 return ip_csum || hw_csum;
10125}
10126
10127static netdev_features_t netdev_fix_features(struct net_device *dev,
10128 netdev_features_t features)
10129{
10130 /* Fix illegal checksum combinations */
10131 if ((features & NETIF_F_HW_CSUM) &&
10132 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
10133 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
10134 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
10135 }
10136
10137 /* TSO requires that SG is present as well. */
10138 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
10139 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
10140 features &= ~NETIF_F_ALL_TSO;
10141 }
10142
10143 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
10144 !(features & NETIF_F_IP_CSUM)) {
10145 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
10146 features &= ~NETIF_F_TSO;
10147 features &= ~NETIF_F_TSO_ECN;
10148 }
10149
10150 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
10151 !(features & NETIF_F_IPV6_CSUM)) {
10152 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
10153 features &= ~NETIF_F_TSO6;
10154 }
10155
10156 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
10157 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
10158 features &= ~NETIF_F_TSO_MANGLEID;
10159
10160 /* TSO ECN requires that TSO is present as well. */
10161 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
10162 features &= ~NETIF_F_TSO_ECN;
10163
10164 /* Software GSO depends on SG. */
10165 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
10166 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
10167 features &= ~NETIF_F_GSO;
10168 }
10169
10170 /* GSO partial features require GSO partial be set */
10171 if ((features & dev->gso_partial_features) &&
10172 !(features & NETIF_F_GSO_PARTIAL)) {
10173 netdev_dbg(dev,
10174 "Dropping partially supported GSO features since no GSO partial.\n");
10175 features &= ~dev->gso_partial_features;
10176 }
10177
10178 if (!(features & NETIF_F_RXCSUM)) {
10179 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
10180 * successfully merged by hardware must also have the
10181 * checksum verified by hardware. If the user does not
10182 * want to enable RXCSUM, logically, we should disable GRO_HW.
10183 */
10184 if (features & NETIF_F_GRO_HW) {
10185 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
10186 features &= ~NETIF_F_GRO_HW;
10187 }
10188 }
10189
10190 /* LRO/HW-GRO features cannot be combined with RX-FCS */
10191 if (features & NETIF_F_RXFCS) {
10192 if (features & NETIF_F_LRO) {
10193 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
10194 features &= ~NETIF_F_LRO;
10195 }
10196
10197 if (features & NETIF_F_GRO_HW) {
10198 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
10199 features &= ~NETIF_F_GRO_HW;
10200 }
10201 }
10202
10203 if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) {
10204 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n");
10205 features &= ~NETIF_F_LRO;
10206 }
10207
10208 if ((features & NETIF_F_HW_TLS_TX) && !netdev_has_ip_or_hw_csum(features)) {
10209 netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
10210 features &= ~NETIF_F_HW_TLS_TX;
10211 }
10212
10213 if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
10214 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
10215 features &= ~NETIF_F_HW_TLS_RX;
10216 }
10217
10218 if ((features & NETIF_F_GSO_UDP_L4) && !netdev_has_ip_or_hw_csum(features)) {
10219 netdev_dbg(dev, "Dropping USO feature since no CSUM feature.\n");
10220 features &= ~NETIF_F_GSO_UDP_L4;
10221 }
10222
10223 return features;
10224}
10225
10226int __netdev_update_features(struct net_device *dev)
10227{
10228 struct net_device *upper, *lower;
10229 netdev_features_t features;
10230 struct list_head *iter;
10231 int err = -1;
10232
10233 ASSERT_RTNL();
10234
10235 features = netdev_get_wanted_features(dev);
10236
10237 if (dev->netdev_ops->ndo_fix_features)
10238 features = dev->netdev_ops->ndo_fix_features(dev, features);
10239
10240 /* driver might be less strict about feature dependencies */
10241 features = netdev_fix_features(dev, features);
10242
10243 /* some features can't be enabled if they're off on an upper device */
10244 netdev_for_each_upper_dev_rcu(dev, upper, iter)
10245 features = netdev_sync_upper_features(dev, upper, features);
10246
10247 if (dev->features == features)
10248 goto sync_lower;
10249
10250 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
10251 &dev->features, &features);
10252
10253 if (dev->netdev_ops->ndo_set_features)
10254 err = dev->netdev_ops->ndo_set_features(dev, features);
10255 else
10256 err = 0;
10257
10258 if (unlikely(err < 0)) {
10259 netdev_err(dev,
10260 "set_features() failed (%d); wanted %pNF, left %pNF\n",
10261 err, &features, &dev->features);
10262 /* return non-0 since some features might have changed and
10263 * it's better to fire a spurious notification than miss it
10264 */
10265 return -1;
10266 }
10267
10268sync_lower:
10269 /* some features must be disabled on lower devices when disabled
10270 * on an upper device (think: bonding master or bridge)
10271 */
10272 netdev_for_each_lower_dev(dev, lower, iter)
10273 netdev_sync_lower_features(dev, lower, features);
10274
10275 if (!err) {
10276 netdev_features_t diff = features ^ dev->features;
10277
10278 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
10279 /* udp_tunnel_{get,drop}_rx_info both need
10280 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
10281 * device, or they won't do anything.
10282 * Thus we need to update dev->features
10283 * *before* calling udp_tunnel_get_rx_info,
10284 * but *after* calling udp_tunnel_drop_rx_info.
10285 */
10286 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
10287 dev->features = features;
10288 udp_tunnel_get_rx_info(dev);
10289 } else {
10290 udp_tunnel_drop_rx_info(dev);
10291 }
10292 }
10293
10294 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
10295 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
10296 dev->features = features;
10297 err |= vlan_get_rx_ctag_filter_info(dev);
10298 } else {
10299 vlan_drop_rx_ctag_filter_info(dev);
10300 }
10301 }
10302
10303 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
10304 if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
10305 dev->features = features;
10306 err |= vlan_get_rx_stag_filter_info(dev);
10307 } else {
10308 vlan_drop_rx_stag_filter_info(dev);
10309 }
10310 }
10311
10312 dev->features = features;
10313 }
10314
10315 return err < 0 ? 0 : 1;
10316}
10317
10318/**
10319 * netdev_update_features - recalculate device features
10320 * @dev: the device to check
10321 *
10322 * Recalculate dev->features set and send notifications if it
10323 * has changed. Should be called after driver or hardware dependent
10324 * conditions might have changed that influence the features.
10325 */
10326void netdev_update_features(struct net_device *dev)
10327{
10328 if (__netdev_update_features(dev))
10329 netdev_features_change(dev);
10330}
10331EXPORT_SYMBOL(netdev_update_features);
10332
10333/**
10334 * netdev_change_features - recalculate device features
10335 * @dev: the device to check
10336 *
10337 * Recalculate dev->features set and send notifications even
10338 * if they have not changed. Should be called instead of
10339 * netdev_update_features() if also dev->vlan_features might
10340 * have changed to allow the changes to be propagated to stacked
10341 * VLAN devices.
10342 */
10343void netdev_change_features(struct net_device *dev)
10344{
10345 __netdev_update_features(dev);
10346 netdev_features_change(dev);
10347}
10348EXPORT_SYMBOL(netdev_change_features);
10349
10350/**
10351 * netif_stacked_transfer_operstate - transfer operstate
10352 * @rootdev: the root or lower level device to transfer state from
10353 * @dev: the device to transfer operstate to
10354 *
10355 * Transfer operational state from root to device. This is normally
10356 * called when a stacking relationship exists between the root
10357 * device and the device(a leaf device).
10358 */
10359void netif_stacked_transfer_operstate(const struct net_device *rootdev,
10360 struct net_device *dev)
10361{
10362 if (rootdev->operstate == IF_OPER_DORMANT)
10363 netif_dormant_on(dev);
10364 else
10365 netif_dormant_off(dev);
10366
10367 if (rootdev->operstate == IF_OPER_TESTING)
10368 netif_testing_on(dev);
10369 else
10370 netif_testing_off(dev);
10371
10372 if (netif_carrier_ok(rootdev))
10373 netif_carrier_on(dev);
10374 else
10375 netif_carrier_off(dev);
10376}
10377EXPORT_SYMBOL(netif_stacked_transfer_operstate);
10378
10379static int netif_alloc_rx_queues(struct net_device *dev)
10380{
10381 unsigned int i, count = dev->num_rx_queues;
10382 struct netdev_rx_queue *rx;
10383 size_t sz = count * sizeof(*rx);
10384 int err = 0;
10385
10386 BUG_ON(count < 1);
10387
10388 rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
10389 if (!rx)
10390 return -ENOMEM;
10391
10392 dev->_rx = rx;
10393
10394 for (i = 0; i < count; i++) {
10395 rx[i].dev = dev;
10396
10397 /* XDP RX-queue setup */
10398 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
10399 if (err < 0)
10400 goto err_rxq_info;
10401 }
10402 return 0;
10403
10404err_rxq_info:
10405 /* Rollback successful reg's and free other resources */
10406 while (i--)
10407 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
10408 kvfree(dev->_rx);
10409 dev->_rx = NULL;
10410 return err;
10411}
10412
10413static void netif_free_rx_queues(struct net_device *dev)
10414{
10415 unsigned int i, count = dev->num_rx_queues;
10416
10417 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
10418 if (!dev->_rx)
10419 return;
10420
10421 for (i = 0; i < count; i++)
10422 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
10423
10424 kvfree(dev->_rx);
10425}
10426
10427static void netdev_init_one_queue(struct net_device *dev,
10428 struct netdev_queue *queue, void *_unused)
10429{
10430 /* Initialize queue lock */
10431 spin_lock_init(&queue->_xmit_lock);
10432 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
10433 queue->xmit_lock_owner = -1;
10434 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
10435 queue->dev = dev;
10436#ifdef CONFIG_BQL
10437 dql_init(&queue->dql, HZ);
10438#endif
10439}
10440
10441static void netif_free_tx_queues(struct net_device *dev)
10442{
10443 kvfree(dev->_tx);
10444}
10445
10446static int netif_alloc_netdev_queues(struct net_device *dev)
10447{
10448 unsigned int count = dev->num_tx_queues;
10449 struct netdev_queue *tx;
10450 size_t sz = count * sizeof(*tx);
10451
10452 if (count < 1 || count > 0xffff)
10453 return -EINVAL;
10454
10455 tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
10456 if (!tx)
10457 return -ENOMEM;
10458
10459 dev->_tx = tx;
10460
10461 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
10462 spin_lock_init(&dev->tx_global_lock);
10463
10464 return 0;
10465}
10466
10467void netif_tx_stop_all_queues(struct net_device *dev)
10468{
10469 unsigned int i;
10470
10471 for (i = 0; i < dev->num_tx_queues; i++) {
10472 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
10473
10474 netif_tx_stop_queue(txq);
10475 }
10476}
10477EXPORT_SYMBOL(netif_tx_stop_all_queues);
10478
10479static int netdev_do_alloc_pcpu_stats(struct net_device *dev)
10480{
10481 void __percpu *v;
10482
10483 /* Drivers implementing ndo_get_peer_dev must support tstat
10484 * accounting, so that skb_do_redirect() can bump the dev's
10485 * RX stats upon network namespace switch.
10486 */
10487 if (dev->netdev_ops->ndo_get_peer_dev &&
10488 dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS)
10489 return -EOPNOTSUPP;
10490
10491 switch (dev->pcpu_stat_type) {
10492 case NETDEV_PCPU_STAT_NONE:
10493 return 0;
10494 case NETDEV_PCPU_STAT_LSTATS:
10495 v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
10496 break;
10497 case NETDEV_PCPU_STAT_TSTATS:
10498 v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
10499 break;
10500 case NETDEV_PCPU_STAT_DSTATS:
10501 v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats);
10502 break;
10503 default:
10504 return -EINVAL;
10505 }
10506
10507 return v ? 0 : -ENOMEM;
10508}
10509
10510static void netdev_do_free_pcpu_stats(struct net_device *dev)
10511{
10512 switch (dev->pcpu_stat_type) {
10513 case NETDEV_PCPU_STAT_NONE:
10514 return;
10515 case NETDEV_PCPU_STAT_LSTATS:
10516 free_percpu(dev->lstats);
10517 break;
10518 case NETDEV_PCPU_STAT_TSTATS:
10519 free_percpu(dev->tstats);
10520 break;
10521 case NETDEV_PCPU_STAT_DSTATS:
10522 free_percpu(dev->dstats);
10523 break;
10524 }
10525}
10526
10527static void netdev_free_phy_link_topology(struct net_device *dev)
10528{
10529 struct phy_link_topology *topo = dev->link_topo;
10530
10531 if (IS_ENABLED(CONFIG_PHYLIB) && topo) {
10532 xa_destroy(&topo->phys);
10533 kfree(topo);
10534 dev->link_topo = NULL;
10535 }
10536}
10537
10538/**
10539 * register_netdevice() - register a network device
10540 * @dev: device to register
10541 *
10542 * Take a prepared network device structure and make it externally accessible.
10543 * A %NETDEV_REGISTER message is sent to the netdev notifier chain.
10544 * Callers must hold the rtnl lock - you may want register_netdev()
10545 * instead of this.
10546 */
10547int register_netdevice(struct net_device *dev)
10548{
10549 int ret;
10550 struct net *net = dev_net(dev);
10551
10552 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
10553 NETDEV_FEATURE_COUNT);
10554 BUG_ON(dev_boot_phase);
10555 ASSERT_RTNL();
10556
10557 might_sleep();
10558
10559 /* When net_device's are persistent, this will be fatal. */
10560 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
10561 BUG_ON(!net);
10562
10563 ret = ethtool_check_ops(dev->ethtool_ops);
10564 if (ret)
10565 return ret;
10566
10567 /* rss ctx ID 0 is reserved for the default context, start from 1 */
10568 xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1);
10569 mutex_init(&dev->ethtool->rss_lock);
10570
10571 spin_lock_init(&dev->addr_list_lock);
10572 netdev_set_addr_lockdep_class(dev);
10573
10574 ret = dev_get_valid_name(net, dev, dev->name);
10575 if (ret < 0)
10576 goto out;
10577
10578 ret = -ENOMEM;
10579 dev->name_node = netdev_name_node_head_alloc(dev);
10580 if (!dev->name_node)
10581 goto out;
10582
10583 /* Init, if this function is available */
10584 if (dev->netdev_ops->ndo_init) {
10585 ret = dev->netdev_ops->ndo_init(dev);
10586 if (ret) {
10587 if (ret > 0)
10588 ret = -EIO;
10589 goto err_free_name;
10590 }
10591 }
10592
10593 if (((dev->hw_features | dev->features) &
10594 NETIF_F_HW_VLAN_CTAG_FILTER) &&
10595 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
10596 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
10597 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
10598 ret = -EINVAL;
10599 goto err_uninit;
10600 }
10601
10602 ret = netdev_do_alloc_pcpu_stats(dev);
10603 if (ret)
10604 goto err_uninit;
10605
10606 ret = dev_index_reserve(net, dev->ifindex);
10607 if (ret < 0)
10608 goto err_free_pcpu;
10609 dev->ifindex = ret;
10610
10611 /* Transfer changeable features to wanted_features and enable
10612 * software offloads (GSO and GRO).
10613 */
10614 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
10615 dev->features |= NETIF_F_SOFT_FEATURES;
10616
10617 if (dev->udp_tunnel_nic_info) {
10618 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10619 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10620 }
10621
10622 dev->wanted_features = dev->features & dev->hw_features;
10623
10624 if (!(dev->flags & IFF_LOOPBACK))
10625 dev->hw_features |= NETIF_F_NOCACHE_COPY;
10626
10627 /* If IPv4 TCP segmentation offload is supported we should also
10628 * allow the device to enable segmenting the frame with the option
10629 * of ignoring a static IP ID value. This doesn't enable the
10630 * feature itself but allows the user to enable it later.
10631 */
10632 if (dev->hw_features & NETIF_F_TSO)
10633 dev->hw_features |= NETIF_F_TSO_MANGLEID;
10634 if (dev->vlan_features & NETIF_F_TSO)
10635 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
10636 if (dev->mpls_features & NETIF_F_TSO)
10637 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
10638 if (dev->hw_enc_features & NETIF_F_TSO)
10639 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
10640
10641 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
10642 */
10643 dev->vlan_features |= NETIF_F_HIGHDMA;
10644
10645 /* Make NETIF_F_SG inheritable to tunnel devices.
10646 */
10647 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
10648
10649 /* Make NETIF_F_SG inheritable to MPLS.
10650 */
10651 dev->mpls_features |= NETIF_F_SG;
10652
10653 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
10654 ret = notifier_to_errno(ret);
10655 if (ret)
10656 goto err_ifindex_release;
10657
10658 ret = netdev_register_kobject(dev);
10659
10660 WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED);
10661
10662 if (ret)
10663 goto err_uninit_notify;
10664
10665 __netdev_update_features(dev);
10666
10667 /*
10668 * Default initial state at registry is that the
10669 * device is present.
10670 */
10671
10672 set_bit(__LINK_STATE_PRESENT, &dev->state);
10673
10674 linkwatch_init_dev(dev);
10675
10676 dev_init_scheduler(dev);
10677
10678 netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL);
10679 list_netdevice(dev);
10680
10681 add_device_randomness(dev->dev_addr, dev->addr_len);
10682
10683 /* If the device has permanent device address, driver should
10684 * set dev_addr and also addr_assign_type should be set to
10685 * NET_ADDR_PERM (default value).
10686 */
10687 if (dev->addr_assign_type == NET_ADDR_PERM)
10688 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10689
10690 /* Notify protocols, that a new device appeared. */
10691 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
10692 ret = notifier_to_errno(ret);
10693 if (ret) {
10694 /* Expect explicit free_netdev() on failure */
10695 dev->needs_free_netdev = false;
10696 unregister_netdevice_queue(dev, NULL);
10697 goto out;
10698 }
10699 /*
10700 * Prevent userspace races by waiting until the network
10701 * device is fully setup before sending notifications.
10702 */
10703 if (!dev->rtnl_link_ops ||
10704 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10705 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
10706
10707out:
10708 return ret;
10709
10710err_uninit_notify:
10711 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
10712err_ifindex_release:
10713 dev_index_release(net, dev->ifindex);
10714err_free_pcpu:
10715 netdev_do_free_pcpu_stats(dev);
10716err_uninit:
10717 if (dev->netdev_ops->ndo_uninit)
10718 dev->netdev_ops->ndo_uninit(dev);
10719 if (dev->priv_destructor)
10720 dev->priv_destructor(dev);
10721err_free_name:
10722 netdev_name_node_free(dev->name_node);
10723 goto out;
10724}
10725EXPORT_SYMBOL(register_netdevice);
10726
10727/* Initialize the core of a dummy net device.
10728 * This is useful if you are calling this function after alloc_netdev(),
10729 * since it does not memset the net_device fields.
10730 */
10731static void init_dummy_netdev_core(struct net_device *dev)
10732{
10733 /* make sure we BUG if trying to hit standard
10734 * register/unregister code path
10735 */
10736 dev->reg_state = NETREG_DUMMY;
10737
10738 /* NAPI wants this */
10739 INIT_LIST_HEAD(&dev->napi_list);
10740
10741 /* a dummy interface is started by default */
10742 set_bit(__LINK_STATE_PRESENT, &dev->state);
10743 set_bit(__LINK_STATE_START, &dev->state);
10744
10745 /* napi_busy_loop stats accounting wants this */
10746 dev_net_set(dev, &init_net);
10747
10748 /* Note : We dont allocate pcpu_refcnt for dummy devices,
10749 * because users of this 'device' dont need to change
10750 * its refcount.
10751 */
10752}
10753
10754/**
10755 * init_dummy_netdev - init a dummy network device for NAPI
10756 * @dev: device to init
10757 *
10758 * This takes a network device structure and initializes the minimum
10759 * amount of fields so it can be used to schedule NAPI polls without
10760 * registering a full blown interface. This is to be used by drivers
10761 * that need to tie several hardware interfaces to a single NAPI
10762 * poll scheduler due to HW limitations.
10763 */
10764void init_dummy_netdev(struct net_device *dev)
10765{
10766 /* Clear everything. Note we don't initialize spinlocks
10767 * as they aren't supposed to be taken by any of the
10768 * NAPI code and this dummy netdev is supposed to be
10769 * only ever used for NAPI polls
10770 */
10771 memset(dev, 0, sizeof(struct net_device));
10772 init_dummy_netdev_core(dev);
10773}
10774EXPORT_SYMBOL_GPL(init_dummy_netdev);
10775
10776/**
10777 * register_netdev - register a network device
10778 * @dev: device to register
10779 *
10780 * Take a completed network device structure and add it to the kernel
10781 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10782 * chain. 0 is returned on success. A negative errno code is returned
10783 * on a failure to set up the device, or if the name is a duplicate.
10784 *
10785 * This is a wrapper around register_netdevice that takes the rtnl semaphore
10786 * and expands the device name if you passed a format string to
10787 * alloc_netdev.
10788 */
10789int register_netdev(struct net_device *dev)
10790{
10791 int err;
10792
10793 if (rtnl_lock_killable())
10794 return -EINTR;
10795 err = register_netdevice(dev);
10796 rtnl_unlock();
10797 return err;
10798}
10799EXPORT_SYMBOL(register_netdev);
10800
10801int netdev_refcnt_read(const struct net_device *dev)
10802{
10803#ifdef CONFIG_PCPU_DEV_REFCNT
10804 int i, refcnt = 0;
10805
10806 for_each_possible_cpu(i)
10807 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10808 return refcnt;
10809#else
10810 return refcount_read(&dev->dev_refcnt);
10811#endif
10812}
10813EXPORT_SYMBOL(netdev_refcnt_read);
10814
10815int netdev_unregister_timeout_secs __read_mostly = 10;
10816
10817#define WAIT_REFS_MIN_MSECS 1
10818#define WAIT_REFS_MAX_MSECS 250
10819/**
10820 * netdev_wait_allrefs_any - wait until all references are gone.
10821 * @list: list of net_devices to wait on
10822 *
10823 * This is called when unregistering network devices.
10824 *
10825 * Any protocol or device that holds a reference should register
10826 * for netdevice notification, and cleanup and put back the
10827 * reference if they receive an UNREGISTER event.
10828 * We can get stuck here if buggy protocols don't correctly
10829 * call dev_put.
10830 */
10831static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
10832{
10833 unsigned long rebroadcast_time, warning_time;
10834 struct net_device *dev;
10835 int wait = 0;
10836
10837 rebroadcast_time = warning_time = jiffies;
10838
10839 list_for_each_entry(dev, list, todo_list)
10840 if (netdev_refcnt_read(dev) == 1)
10841 return dev;
10842
10843 while (true) {
10844 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
10845 rtnl_lock();
10846
10847 /* Rebroadcast unregister notification */
10848 list_for_each_entry(dev, list, todo_list)
10849 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10850
10851 __rtnl_unlock();
10852 rcu_barrier();
10853 rtnl_lock();
10854
10855 list_for_each_entry(dev, list, todo_list)
10856 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
10857 &dev->state)) {
10858 /* We must not have linkwatch events
10859 * pending on unregister. If this
10860 * happens, we simply run the queue
10861 * unscheduled, resulting in a noop
10862 * for this device.
10863 */
10864 linkwatch_run_queue();
10865 break;
10866 }
10867
10868 __rtnl_unlock();
10869
10870 rebroadcast_time = jiffies;
10871 }
10872
10873 rcu_barrier();
10874
10875 if (!wait) {
10876 wait = WAIT_REFS_MIN_MSECS;
10877 } else {
10878 msleep(wait);
10879 wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10880 }
10881
10882 list_for_each_entry(dev, list, todo_list)
10883 if (netdev_refcnt_read(dev) == 1)
10884 return dev;
10885
10886 if (time_after(jiffies, warning_time +
10887 READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
10888 list_for_each_entry(dev, list, todo_list) {
10889 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10890 dev->name, netdev_refcnt_read(dev));
10891 ref_tracker_dir_print(&dev->refcnt_tracker, 10);
10892 }
10893
10894 warning_time = jiffies;
10895 }
10896 }
10897}
10898
10899/* The sequence is:
10900 *
10901 * rtnl_lock();
10902 * ...
10903 * register_netdevice(x1);
10904 * register_netdevice(x2);
10905 * ...
10906 * unregister_netdevice(y1);
10907 * unregister_netdevice(y2);
10908 * ...
10909 * rtnl_unlock();
10910 * free_netdev(y1);
10911 * free_netdev(y2);
10912 *
10913 * We are invoked by rtnl_unlock().
10914 * This allows us to deal with problems:
10915 * 1) We can delete sysfs objects which invoke hotplug
10916 * without deadlocking with linkwatch via keventd.
10917 * 2) Since we run with the RTNL semaphore not held, we can sleep
10918 * safely in order to wait for the netdev refcnt to drop to zero.
10919 *
10920 * We must not return until all unregister events added during
10921 * the interval the lock was held have been completed.
10922 */
10923void netdev_run_todo(void)
10924{
10925 struct net_device *dev, *tmp;
10926 struct list_head list;
10927 int cnt;
10928#ifdef CONFIG_LOCKDEP
10929 struct list_head unlink_list;
10930
10931 list_replace_init(&net_unlink_list, &unlink_list);
10932
10933 while (!list_empty(&unlink_list)) {
10934 struct net_device *dev = list_first_entry(&unlink_list,
10935 struct net_device,
10936 unlink_list);
10937 list_del_init(&dev->unlink_list);
10938 dev->nested_level = dev->lower_level - 1;
10939 }
10940#endif
10941
10942 /* Snapshot list, allow later requests */
10943 list_replace_init(&net_todo_list, &list);
10944
10945 __rtnl_unlock();
10946
10947 /* Wait for rcu callbacks to finish before next phase */
10948 if (!list_empty(&list))
10949 rcu_barrier();
10950
10951 list_for_each_entry_safe(dev, tmp, &list, todo_list) {
10952 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
10953 netdev_WARN(dev, "run_todo but not unregistering\n");
10954 list_del(&dev->todo_list);
10955 continue;
10956 }
10957
10958 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED);
10959 linkwatch_sync_dev(dev);
10960 }
10961
10962 cnt = 0;
10963 while (!list_empty(&list)) {
10964 dev = netdev_wait_allrefs_any(&list);
10965 list_del(&dev->todo_list);
10966
10967 /* paranoia */
10968 BUG_ON(netdev_refcnt_read(dev) != 1);
10969 BUG_ON(!list_empty(&dev->ptype_all));
10970 BUG_ON(!list_empty(&dev->ptype_specific));
10971 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10972 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
10973
10974 netdev_do_free_pcpu_stats(dev);
10975 if (dev->priv_destructor)
10976 dev->priv_destructor(dev);
10977 if (dev->needs_free_netdev)
10978 free_netdev(dev);
10979
10980 cnt++;
10981
10982 /* Free network device */
10983 kobject_put(&dev->dev.kobj);
10984 }
10985 if (cnt && atomic_sub_and_test(cnt, &dev_unreg_count))
10986 wake_up(&netdev_unregistering_wq);
10987}
10988
10989/* Collate per-cpu network dstats statistics
10990 *
10991 * Read per-cpu network statistics from dev->dstats and populate the related
10992 * fields in @s.
10993 */
10994static void dev_fetch_dstats(struct rtnl_link_stats64 *s,
10995 const struct pcpu_dstats __percpu *dstats)
10996{
10997 int cpu;
10998
10999 for_each_possible_cpu(cpu) {
11000 u64 rx_packets, rx_bytes, rx_drops;
11001 u64 tx_packets, tx_bytes, tx_drops;
11002 const struct pcpu_dstats *stats;
11003 unsigned int start;
11004
11005 stats = per_cpu_ptr(dstats, cpu);
11006 do {
11007 start = u64_stats_fetch_begin(&stats->syncp);
11008 rx_packets = u64_stats_read(&stats->rx_packets);
11009 rx_bytes = u64_stats_read(&stats->rx_bytes);
11010 rx_drops = u64_stats_read(&stats->rx_drops);
11011 tx_packets = u64_stats_read(&stats->tx_packets);
11012 tx_bytes = u64_stats_read(&stats->tx_bytes);
11013 tx_drops = u64_stats_read(&stats->tx_drops);
11014 } while (u64_stats_fetch_retry(&stats->syncp, start));
11015
11016 s->rx_packets += rx_packets;
11017 s->rx_bytes += rx_bytes;
11018 s->rx_dropped += rx_drops;
11019 s->tx_packets += tx_packets;
11020 s->tx_bytes += tx_bytes;
11021 s->tx_dropped += tx_drops;
11022 }
11023}
11024
11025/* ndo_get_stats64 implementation for dtstats-based accounting.
11026 *
11027 * Populate @s from dev->stats and dev->dstats. This is used internally by the
11028 * core for NETDEV_PCPU_STAT_DSTAT-type stats collection.
11029 */
11030static void dev_get_dstats64(const struct net_device *dev,
11031 struct rtnl_link_stats64 *s)
11032{
11033 netdev_stats_to_stats64(s, &dev->stats);
11034 dev_fetch_dstats(s, dev->dstats);
11035}
11036
11037/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
11038 * all the same fields in the same order as net_device_stats, with only
11039 * the type differing, but rtnl_link_stats64 may have additional fields
11040 * at the end for newer counters.
11041 */
11042void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
11043 const struct net_device_stats *netdev_stats)
11044{
11045 size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t);
11046 const atomic_long_t *src = (atomic_long_t *)netdev_stats;
11047 u64 *dst = (u64 *)stats64;
11048
11049 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
11050 for (i = 0; i < n; i++)
11051 dst[i] = (unsigned long)atomic_long_read(&src[i]);
11052 /* zero out counters that only exist in rtnl_link_stats64 */
11053 memset((char *)stats64 + n * sizeof(u64), 0,
11054 sizeof(*stats64) - n * sizeof(u64));
11055}
11056EXPORT_SYMBOL(netdev_stats_to_stats64);
11057
11058static __cold struct net_device_core_stats __percpu *netdev_core_stats_alloc(
11059 struct net_device *dev)
11060{
11061 struct net_device_core_stats __percpu *p;
11062
11063 p = alloc_percpu_gfp(struct net_device_core_stats,
11064 GFP_ATOMIC | __GFP_NOWARN);
11065
11066 if (p && cmpxchg(&dev->core_stats, NULL, p))
11067 free_percpu(p);
11068
11069 /* This READ_ONCE() pairs with the cmpxchg() above */
11070 return READ_ONCE(dev->core_stats);
11071}
11072
11073noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset)
11074{
11075 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
11076 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats);
11077 unsigned long __percpu *field;
11078
11079 if (unlikely(!p)) {
11080 p = netdev_core_stats_alloc(dev);
11081 if (!p)
11082 return;
11083 }
11084
11085 field = (unsigned long __percpu *)((void __percpu *)p + offset);
11086 this_cpu_inc(*field);
11087}
11088EXPORT_SYMBOL_GPL(netdev_core_stats_inc);
11089
11090/**
11091 * dev_get_stats - get network device statistics
11092 * @dev: device to get statistics from
11093 * @storage: place to store stats
11094 *
11095 * Get network statistics from device. Return @storage.
11096 * The device driver may provide its own method by setting
11097 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
11098 * otherwise the internal statistics structure is used.
11099 */
11100struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
11101 struct rtnl_link_stats64 *storage)
11102{
11103 const struct net_device_ops *ops = dev->netdev_ops;
11104 const struct net_device_core_stats __percpu *p;
11105
11106 if (ops->ndo_get_stats64) {
11107 memset(storage, 0, sizeof(*storage));
11108 ops->ndo_get_stats64(dev, storage);
11109 } else if (ops->ndo_get_stats) {
11110 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
11111 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) {
11112 dev_get_tstats64(dev, storage);
11113 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) {
11114 dev_get_dstats64(dev, storage);
11115 } else {
11116 netdev_stats_to_stats64(storage, &dev->stats);
11117 }
11118
11119 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
11120 p = READ_ONCE(dev->core_stats);
11121 if (p) {
11122 const struct net_device_core_stats *core_stats;
11123 int i;
11124
11125 for_each_possible_cpu(i) {
11126 core_stats = per_cpu_ptr(p, i);
11127 storage->rx_dropped += READ_ONCE(core_stats->rx_dropped);
11128 storage->tx_dropped += READ_ONCE(core_stats->tx_dropped);
11129 storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler);
11130 storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped);
11131 }
11132 }
11133 return storage;
11134}
11135EXPORT_SYMBOL(dev_get_stats);
11136
11137/**
11138 * dev_fetch_sw_netstats - get per-cpu network device statistics
11139 * @s: place to store stats
11140 * @netstats: per-cpu network stats to read from
11141 *
11142 * Read per-cpu network statistics and populate the related fields in @s.
11143 */
11144void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
11145 const struct pcpu_sw_netstats __percpu *netstats)
11146{
11147 int cpu;
11148
11149 for_each_possible_cpu(cpu) {
11150 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
11151 const struct pcpu_sw_netstats *stats;
11152 unsigned int start;
11153
11154 stats = per_cpu_ptr(netstats, cpu);
11155 do {
11156 start = u64_stats_fetch_begin(&stats->syncp);
11157 rx_packets = u64_stats_read(&stats->rx_packets);
11158 rx_bytes = u64_stats_read(&stats->rx_bytes);
11159 tx_packets = u64_stats_read(&stats->tx_packets);
11160 tx_bytes = u64_stats_read(&stats->tx_bytes);
11161 } while (u64_stats_fetch_retry(&stats->syncp, start));
11162
11163 s->rx_packets += rx_packets;
11164 s->rx_bytes += rx_bytes;
11165 s->tx_packets += tx_packets;
11166 s->tx_bytes += tx_bytes;
11167 }
11168}
11169EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
11170
11171/**
11172 * dev_get_tstats64 - ndo_get_stats64 implementation
11173 * @dev: device to get statistics from
11174 * @s: place to store stats
11175 *
11176 * Populate @s from dev->stats and dev->tstats. Can be used as
11177 * ndo_get_stats64() callback.
11178 */
11179void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
11180{
11181 netdev_stats_to_stats64(s, &dev->stats);
11182 dev_fetch_sw_netstats(s, dev->tstats);
11183}
11184EXPORT_SYMBOL_GPL(dev_get_tstats64);
11185
11186struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
11187{
11188 struct netdev_queue *queue = dev_ingress_queue(dev);
11189
11190#ifdef CONFIG_NET_CLS_ACT
11191 if (queue)
11192 return queue;
11193 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
11194 if (!queue)
11195 return NULL;
11196 netdev_init_one_queue(dev, queue, NULL);
11197 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
11198 RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc);
11199 rcu_assign_pointer(dev->ingress_queue, queue);
11200#endif
11201 return queue;
11202}
11203
11204static const struct ethtool_ops default_ethtool_ops;
11205
11206void netdev_set_default_ethtool_ops(struct net_device *dev,
11207 const struct ethtool_ops *ops)
11208{
11209 if (dev->ethtool_ops == &default_ethtool_ops)
11210 dev->ethtool_ops = ops;
11211}
11212EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
11213
11214/**
11215 * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
11216 * @dev: netdev to enable the IRQ coalescing on
11217 *
11218 * Sets a conservative default for SW IRQ coalescing. Users can use
11219 * sysfs attributes to override the default values.
11220 */
11221void netdev_sw_irq_coalesce_default_on(struct net_device *dev)
11222{
11223 WARN_ON(dev->reg_state == NETREG_REGISTERED);
11224
11225 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
11226 netdev_set_gro_flush_timeout(dev, 20000);
11227 netdev_set_defer_hard_irqs(dev, 1);
11228 }
11229}
11230EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on);
11231
11232/**
11233 * alloc_netdev_mqs - allocate network device
11234 * @sizeof_priv: size of private data to allocate space for
11235 * @name: device name format string
11236 * @name_assign_type: origin of device name
11237 * @setup: callback to initialize device
11238 * @txqs: the number of TX subqueues to allocate
11239 * @rxqs: the number of RX subqueues to allocate
11240 *
11241 * Allocates a struct net_device with private data area for driver use
11242 * and performs basic initialization. Also allocates subqueue structs
11243 * for each queue on the device.
11244 */
11245struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
11246 unsigned char name_assign_type,
11247 void (*setup)(struct net_device *),
11248 unsigned int txqs, unsigned int rxqs)
11249{
11250 struct net_device *dev;
11251 size_t napi_config_sz;
11252 unsigned int maxqs;
11253
11254 BUG_ON(strlen(name) >= sizeof(dev->name));
11255
11256 if (txqs < 1) {
11257 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
11258 return NULL;
11259 }
11260
11261 if (rxqs < 1) {
11262 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
11263 return NULL;
11264 }
11265
11266 maxqs = max(txqs, rxqs);
11267
11268 dev = kvzalloc(struct_size(dev, priv, sizeof_priv),
11269 GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
11270 if (!dev)
11271 return NULL;
11272
11273 dev->priv_len = sizeof_priv;
11274
11275 ref_tracker_dir_init(&dev->refcnt_tracker, 128, name);
11276#ifdef CONFIG_PCPU_DEV_REFCNT
11277 dev->pcpu_refcnt = alloc_percpu(int);
11278 if (!dev->pcpu_refcnt)
11279 goto free_dev;
11280 __dev_hold(dev);
11281#else
11282 refcount_set(&dev->dev_refcnt, 1);
11283#endif
11284
11285 if (dev_addr_init(dev))
11286 goto free_pcpu;
11287
11288 dev_mc_init(dev);
11289 dev_uc_init(dev);
11290
11291 dev_net_set(dev, &init_net);
11292
11293 dev->gso_max_size = GSO_LEGACY_MAX_SIZE;
11294 dev->xdp_zc_max_segs = 1;
11295 dev->gso_max_segs = GSO_MAX_SEGS;
11296 dev->gro_max_size = GRO_LEGACY_MAX_SIZE;
11297 dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE;
11298 dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE;
11299 dev->tso_max_size = TSO_LEGACY_MAX_SIZE;
11300 dev->tso_max_segs = TSO_MAX_SEGS;
11301 dev->upper_level = 1;
11302 dev->lower_level = 1;
11303#ifdef CONFIG_LOCKDEP
11304 dev->nested_level = 0;
11305 INIT_LIST_HEAD(&dev->unlink_list);
11306#endif
11307
11308 INIT_LIST_HEAD(&dev->napi_list);
11309 INIT_LIST_HEAD(&dev->unreg_list);
11310 INIT_LIST_HEAD(&dev->close_list);
11311 INIT_LIST_HEAD(&dev->link_watch_list);
11312 INIT_LIST_HEAD(&dev->adj_list.upper);
11313 INIT_LIST_HEAD(&dev->adj_list.lower);
11314 INIT_LIST_HEAD(&dev->ptype_all);
11315 INIT_LIST_HEAD(&dev->ptype_specific);
11316 INIT_LIST_HEAD(&dev->net_notifier_list);
11317#ifdef CONFIG_NET_SCHED
11318 hash_init(dev->qdisc_hash);
11319#endif
11320
11321 mutex_init(&dev->lock);
11322
11323 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
11324 setup(dev);
11325
11326 if (!dev->tx_queue_len) {
11327 dev->priv_flags |= IFF_NO_QUEUE;
11328 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
11329 }
11330
11331 dev->num_tx_queues = txqs;
11332 dev->real_num_tx_queues = txqs;
11333 if (netif_alloc_netdev_queues(dev))
11334 goto free_all;
11335
11336 dev->num_rx_queues = rxqs;
11337 dev->real_num_rx_queues = rxqs;
11338 if (netif_alloc_rx_queues(dev))
11339 goto free_all;
11340 dev->ethtool = kzalloc(sizeof(*dev->ethtool), GFP_KERNEL_ACCOUNT);
11341 if (!dev->ethtool)
11342 goto free_all;
11343
11344 napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config));
11345 dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT);
11346 if (!dev->napi_config)
11347 goto free_all;
11348
11349 strscpy(dev->name, name);
11350 dev->name_assign_type = name_assign_type;
11351 dev->group = INIT_NETDEV_GROUP;
11352 if (!dev->ethtool_ops)
11353 dev->ethtool_ops = &default_ethtool_ops;
11354
11355 nf_hook_netdev_init(dev);
11356
11357 return dev;
11358
11359free_all:
11360 free_netdev(dev);
11361 return NULL;
11362
11363free_pcpu:
11364#ifdef CONFIG_PCPU_DEV_REFCNT
11365 free_percpu(dev->pcpu_refcnt);
11366free_dev:
11367#endif
11368 kvfree(dev);
11369 return NULL;
11370}
11371EXPORT_SYMBOL(alloc_netdev_mqs);
11372
11373/**
11374 * free_netdev - free network device
11375 * @dev: device
11376 *
11377 * This function does the last stage of destroying an allocated device
11378 * interface. The reference to the device object is released. If this
11379 * is the last reference then it will be freed.Must be called in process
11380 * context.
11381 */
11382void free_netdev(struct net_device *dev)
11383{
11384 struct napi_struct *p, *n;
11385
11386 might_sleep();
11387
11388 /* When called immediately after register_netdevice() failed the unwind
11389 * handling may still be dismantling the device. Handle that case by
11390 * deferring the free.
11391 */
11392 if (dev->reg_state == NETREG_UNREGISTERING) {
11393 ASSERT_RTNL();
11394 dev->needs_free_netdev = true;
11395 return;
11396 }
11397
11398 mutex_destroy(&dev->lock);
11399
11400 kfree(dev->ethtool);
11401 netif_free_tx_queues(dev);
11402 netif_free_rx_queues(dev);
11403
11404 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
11405
11406 /* Flush device addresses */
11407 dev_addr_flush(dev);
11408
11409 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
11410 netif_napi_del(p);
11411
11412 kvfree(dev->napi_config);
11413
11414 ref_tracker_dir_exit(&dev->refcnt_tracker);
11415#ifdef CONFIG_PCPU_DEV_REFCNT
11416 free_percpu(dev->pcpu_refcnt);
11417 dev->pcpu_refcnt = NULL;
11418#endif
11419 free_percpu(dev->core_stats);
11420 dev->core_stats = NULL;
11421 free_percpu(dev->xdp_bulkq);
11422 dev->xdp_bulkq = NULL;
11423
11424 netdev_free_phy_link_topology(dev);
11425
11426 /* Compatibility with error handling in drivers */
11427 if (dev->reg_state == NETREG_UNINITIALIZED ||
11428 dev->reg_state == NETREG_DUMMY) {
11429 kvfree(dev);
11430 return;
11431 }
11432
11433 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
11434 WRITE_ONCE(dev->reg_state, NETREG_RELEASED);
11435
11436 /* will free via device release */
11437 put_device(&dev->dev);
11438}
11439EXPORT_SYMBOL(free_netdev);
11440
11441/**
11442 * alloc_netdev_dummy - Allocate and initialize a dummy net device.
11443 * @sizeof_priv: size of private data to allocate space for
11444 *
11445 * Return: the allocated net_device on success, NULL otherwise
11446 */
11447struct net_device *alloc_netdev_dummy(int sizeof_priv)
11448{
11449 return alloc_netdev(sizeof_priv, "dummy#", NET_NAME_UNKNOWN,
11450 init_dummy_netdev_core);
11451}
11452EXPORT_SYMBOL_GPL(alloc_netdev_dummy);
11453
11454/**
11455 * synchronize_net - Synchronize with packet receive processing
11456 *
11457 * Wait for packets currently being received to be done.
11458 * Does not block later packets from starting.
11459 */
11460void synchronize_net(void)
11461{
11462 might_sleep();
11463 if (rtnl_is_locked())
11464 synchronize_rcu_expedited();
11465 else
11466 synchronize_rcu();
11467}
11468EXPORT_SYMBOL(synchronize_net);
11469
11470static void netdev_rss_contexts_free(struct net_device *dev)
11471{
11472 struct ethtool_rxfh_context *ctx;
11473 unsigned long context;
11474
11475 mutex_lock(&dev->ethtool->rss_lock);
11476 xa_for_each(&dev->ethtool->rss_ctx, context, ctx) {
11477 struct ethtool_rxfh_param rxfh;
11478
11479 rxfh.indir = ethtool_rxfh_context_indir(ctx);
11480 rxfh.key = ethtool_rxfh_context_key(ctx);
11481 rxfh.hfunc = ctx->hfunc;
11482 rxfh.input_xfrm = ctx->input_xfrm;
11483 rxfh.rss_context = context;
11484 rxfh.rss_delete = true;
11485
11486 xa_erase(&dev->ethtool->rss_ctx, context);
11487 if (dev->ethtool_ops->create_rxfh_context)
11488 dev->ethtool_ops->remove_rxfh_context(dev, ctx,
11489 context, NULL);
11490 else
11491 dev->ethtool_ops->set_rxfh(dev, &rxfh, NULL);
11492 kfree(ctx);
11493 }
11494 xa_destroy(&dev->ethtool->rss_ctx);
11495 mutex_unlock(&dev->ethtool->rss_lock);
11496}
11497
11498/**
11499 * unregister_netdevice_queue - remove device from the kernel
11500 * @dev: device
11501 * @head: list
11502 *
11503 * This function shuts down a device interface and removes it
11504 * from the kernel tables.
11505 * If head not NULL, device is queued to be unregistered later.
11506 *
11507 * Callers must hold the rtnl semaphore. You may want
11508 * unregister_netdev() instead of this.
11509 */
11510
11511void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
11512{
11513 ASSERT_RTNL();
11514
11515 if (head) {
11516 list_move_tail(&dev->unreg_list, head);
11517 } else {
11518 LIST_HEAD(single);
11519
11520 list_add(&dev->unreg_list, &single);
11521 unregister_netdevice_many(&single);
11522 }
11523}
11524EXPORT_SYMBOL(unregister_netdevice_queue);
11525
11526void unregister_netdevice_many_notify(struct list_head *head,
11527 u32 portid, const struct nlmsghdr *nlh)
11528{
11529 struct net_device *dev, *tmp;
11530 LIST_HEAD(close_head);
11531 int cnt = 0;
11532
11533 BUG_ON(dev_boot_phase);
11534 ASSERT_RTNL();
11535
11536 if (list_empty(head))
11537 return;
11538
11539 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
11540 /* Some devices call without registering
11541 * for initialization unwind. Remove those
11542 * devices and proceed with the remaining.
11543 */
11544 if (dev->reg_state == NETREG_UNINITIALIZED) {
11545 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
11546 dev->name, dev);
11547
11548 WARN_ON(1);
11549 list_del(&dev->unreg_list);
11550 continue;
11551 }
11552 dev->dismantle = true;
11553 BUG_ON(dev->reg_state != NETREG_REGISTERED);
11554 }
11555
11556 /* If device is running, close it first. */
11557 list_for_each_entry(dev, head, unreg_list)
11558 list_add_tail(&dev->close_list, &close_head);
11559 dev_close_many(&close_head, true);
11560
11561 list_for_each_entry(dev, head, unreg_list) {
11562 /* And unlink it from device chain. */
11563 unlist_netdevice(dev);
11564 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING);
11565 }
11566 flush_all_backlogs();
11567
11568 synchronize_net();
11569
11570 list_for_each_entry(dev, head, unreg_list) {
11571 struct sk_buff *skb = NULL;
11572
11573 /* Shutdown queueing discipline. */
11574 dev_shutdown(dev);
11575 dev_tcx_uninstall(dev);
11576 dev_xdp_uninstall(dev);
11577 bpf_dev_bound_netdev_unregister(dev);
11578 dev_dmabuf_uninstall(dev);
11579
11580 netdev_offload_xstats_disable_all(dev);
11581
11582 /* Notify protocols, that we are about to destroy
11583 * this device. They should clean all the things.
11584 */
11585 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11586
11587 if (!dev->rtnl_link_ops ||
11588 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
11589 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
11590 GFP_KERNEL, NULL, 0,
11591 portid, nlh);
11592
11593 /*
11594 * Flush the unicast and multicast chains
11595 */
11596 dev_uc_flush(dev);
11597 dev_mc_flush(dev);
11598
11599 netdev_name_node_alt_flush(dev);
11600 netdev_name_node_free(dev->name_node);
11601
11602 netdev_rss_contexts_free(dev);
11603
11604 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
11605
11606 if (dev->netdev_ops->ndo_uninit)
11607 dev->netdev_ops->ndo_uninit(dev);
11608
11609 mutex_destroy(&dev->ethtool->rss_lock);
11610
11611 net_shaper_flush_netdev(dev);
11612
11613 if (skb)
11614 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh);
11615
11616 /* Notifier chain MUST detach us all upper devices. */
11617 WARN_ON(netdev_has_any_upper_dev(dev));
11618 WARN_ON(netdev_has_any_lower_dev(dev));
11619
11620 /* Remove entries from kobject tree */
11621 netdev_unregister_kobject(dev);
11622#ifdef CONFIG_XPS
11623 /* Remove XPS queueing entries */
11624 netif_reset_xps_queues_gt(dev, 0);
11625#endif
11626 }
11627
11628 synchronize_net();
11629
11630 list_for_each_entry(dev, head, unreg_list) {
11631 netdev_put(dev, &dev->dev_registered_tracker);
11632 net_set_todo(dev);
11633 cnt++;
11634 }
11635 atomic_add(cnt, &dev_unreg_count);
11636
11637 list_del(head);
11638}
11639
11640/**
11641 * unregister_netdevice_many - unregister many devices
11642 * @head: list of devices
11643 *
11644 * Note: As most callers use a stack allocated list_head,
11645 * we force a list_del() to make sure stack won't be corrupted later.
11646 */
11647void unregister_netdevice_many(struct list_head *head)
11648{
11649 unregister_netdevice_many_notify(head, 0, NULL);
11650}
11651EXPORT_SYMBOL(unregister_netdevice_many);
11652
11653/**
11654 * unregister_netdev - remove device from the kernel
11655 * @dev: device
11656 *
11657 * This function shuts down a device interface and removes it
11658 * from the kernel tables.
11659 *
11660 * This is just a wrapper for unregister_netdevice that takes
11661 * the rtnl semaphore. In general you want to use this and not
11662 * unregister_netdevice.
11663 */
11664void unregister_netdev(struct net_device *dev)
11665{
11666 rtnl_lock();
11667 unregister_netdevice(dev);
11668 rtnl_unlock();
11669}
11670EXPORT_SYMBOL(unregister_netdev);
11671
11672/**
11673 * __dev_change_net_namespace - move device to different nethost namespace
11674 * @dev: device
11675 * @net: network namespace
11676 * @pat: If not NULL name pattern to try if the current device name
11677 * is already taken in the destination network namespace.
11678 * @new_ifindex: If not zero, specifies device index in the target
11679 * namespace.
11680 *
11681 * This function shuts down a device interface and moves it
11682 * to a new network namespace. On success 0 is returned, on
11683 * a failure a netagive errno code is returned.
11684 *
11685 * Callers must hold the rtnl semaphore.
11686 */
11687
11688int __dev_change_net_namespace(struct net_device *dev, struct net *net,
11689 const char *pat, int new_ifindex)
11690{
11691 struct netdev_name_node *name_node;
11692 struct net *net_old = dev_net(dev);
11693 char new_name[IFNAMSIZ] = {};
11694 int err, new_nsid;
11695
11696 ASSERT_RTNL();
11697
11698 /* Don't allow namespace local devices to be moved. */
11699 err = -EINVAL;
11700 if (dev->netns_local)
11701 goto out;
11702
11703 /* Ensure the device has been registered */
11704 if (dev->reg_state != NETREG_REGISTERED)
11705 goto out;
11706
11707 /* Get out if there is nothing todo */
11708 err = 0;
11709 if (net_eq(net_old, net))
11710 goto out;
11711
11712 /* Pick the destination device name, and ensure
11713 * we can use it in the destination network namespace.
11714 */
11715 err = -EEXIST;
11716 if (netdev_name_in_use(net, dev->name)) {
11717 /* We get here if we can't use the current device name */
11718 if (!pat)
11719 goto out;
11720 err = dev_prep_valid_name(net, dev, pat, new_name, EEXIST);
11721 if (err < 0)
11722 goto out;
11723 }
11724 /* Check that none of the altnames conflicts. */
11725 err = -EEXIST;
11726 netdev_for_each_altname(dev, name_node)
11727 if (netdev_name_in_use(net, name_node->name))
11728 goto out;
11729
11730 /* Check that new_ifindex isn't used yet. */
11731 if (new_ifindex) {
11732 err = dev_index_reserve(net, new_ifindex);
11733 if (err < 0)
11734 goto out;
11735 } else {
11736 /* If there is an ifindex conflict assign a new one */
11737 err = dev_index_reserve(net, dev->ifindex);
11738 if (err == -EBUSY)
11739 err = dev_index_reserve(net, 0);
11740 if (err < 0)
11741 goto out;
11742 new_ifindex = err;
11743 }
11744
11745 /*
11746 * And now a mini version of register_netdevice unregister_netdevice.
11747 */
11748
11749 /* If device is running close it first. */
11750 dev_close(dev);
11751
11752 /* And unlink it from device chain */
11753 unlist_netdevice(dev);
11754
11755 synchronize_net();
11756
11757 /* Shutdown queueing discipline. */
11758 dev_shutdown(dev);
11759
11760 /* Notify protocols, that we are about to destroy
11761 * this device. They should clean all the things.
11762 *
11763 * Note that dev->reg_state stays at NETREG_REGISTERED.
11764 * This is wanted because this way 8021q and macvlan know
11765 * the device is just moving and can keep their slaves up.
11766 */
11767 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11768 rcu_barrier();
11769
11770 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
11771
11772 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11773 new_ifindex);
11774
11775 /*
11776 * Flush the unicast and multicast chains
11777 */
11778 dev_uc_flush(dev);
11779 dev_mc_flush(dev);
11780
11781 /* Send a netdev-removed uevent to the old namespace */
11782 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
11783 netdev_adjacent_del_links(dev);
11784
11785 /* Move per-net netdevice notifiers that are following the netdevice */
11786 move_netdevice_notifiers_dev_net(dev, net);
11787
11788 /* Actually switch the network namespace */
11789 dev_net_set(dev, net);
11790 dev->ifindex = new_ifindex;
11791
11792 if (new_name[0]) {
11793 /* Rename the netdev to prepared name */
11794 write_seqlock_bh(&netdev_rename_lock);
11795 strscpy(dev->name, new_name, IFNAMSIZ);
11796 write_sequnlock_bh(&netdev_rename_lock);
11797 }
11798
11799 /* Fixup kobjects */
11800 dev_set_uevent_suppress(&dev->dev, 1);
11801 err = device_rename(&dev->dev, dev->name);
11802 dev_set_uevent_suppress(&dev->dev, 0);
11803 WARN_ON(err);
11804
11805 /* Send a netdev-add uevent to the new namespace */
11806 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
11807 netdev_adjacent_add_links(dev);
11808
11809 /* Adapt owner in case owning user namespace of target network
11810 * namespace is different from the original one.
11811 */
11812 err = netdev_change_owner(dev, net_old, net);
11813 WARN_ON(err);
11814
11815 /* Add the device back in the hashes */
11816 list_netdevice(dev);
11817
11818 /* Notify protocols, that a new device appeared. */
11819 call_netdevice_notifiers(NETDEV_REGISTER, dev);
11820
11821 /*
11822 * Prevent userspace races by waiting until the network
11823 * device is fully setup before sending notifications.
11824 */
11825 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
11826
11827 synchronize_net();
11828 err = 0;
11829out:
11830 return err;
11831}
11832EXPORT_SYMBOL_GPL(__dev_change_net_namespace);
11833
11834static int dev_cpu_dead(unsigned int oldcpu)
11835{
11836 struct sk_buff **list_skb;
11837 struct sk_buff *skb;
11838 unsigned int cpu;
11839 struct softnet_data *sd, *oldsd, *remsd = NULL;
11840
11841 local_irq_disable();
11842 cpu = smp_processor_id();
11843 sd = &per_cpu(softnet_data, cpu);
11844 oldsd = &per_cpu(softnet_data, oldcpu);
11845
11846 /* Find end of our completion_queue. */
11847 list_skb = &sd->completion_queue;
11848 while (*list_skb)
11849 list_skb = &(*list_skb)->next;
11850 /* Append completion queue from offline CPU. */
11851 *list_skb = oldsd->completion_queue;
11852 oldsd->completion_queue = NULL;
11853
11854 /* Append output queue from offline CPU. */
11855 if (oldsd->output_queue) {
11856 *sd->output_queue_tailp = oldsd->output_queue;
11857 sd->output_queue_tailp = oldsd->output_queue_tailp;
11858 oldsd->output_queue = NULL;
11859 oldsd->output_queue_tailp = &oldsd->output_queue;
11860 }
11861 /* Append NAPI poll list from offline CPU, with one exception :
11862 * process_backlog() must be called by cpu owning percpu backlog.
11863 * We properly handle process_queue & input_pkt_queue later.
11864 */
11865 while (!list_empty(&oldsd->poll_list)) {
11866 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11867 struct napi_struct,
11868 poll_list);
11869
11870 list_del_init(&napi->poll_list);
11871 if (napi->poll == process_backlog)
11872 napi->state &= NAPIF_STATE_THREADED;
11873 else
11874 ____napi_schedule(sd, napi);
11875 }
11876
11877 raise_softirq_irqoff(NET_TX_SOFTIRQ);
11878 local_irq_enable();
11879
11880 if (!use_backlog_threads()) {
11881#ifdef CONFIG_RPS
11882 remsd = oldsd->rps_ipi_list;
11883 oldsd->rps_ipi_list = NULL;
11884#endif
11885 /* send out pending IPI's on offline CPU */
11886 net_rps_send_ipi(remsd);
11887 }
11888
11889 /* Process offline CPU's input_pkt_queue */
11890 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
11891 netif_rx(skb);
11892 rps_input_queue_head_incr(oldsd);
11893 }
11894 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
11895 netif_rx(skb);
11896 rps_input_queue_head_incr(oldsd);
11897 }
11898
11899 return 0;
11900}
11901
11902/**
11903 * netdev_increment_features - increment feature set by one
11904 * @all: current feature set
11905 * @one: new feature set
11906 * @mask: mask feature set
11907 *
11908 * Computes a new feature set after adding a device with feature set
11909 * @one to the master device with current feature set @all. Will not
11910 * enable anything that is off in @mask. Returns the new feature set.
11911 */
11912netdev_features_t netdev_increment_features(netdev_features_t all,
11913 netdev_features_t one, netdev_features_t mask)
11914{
11915 if (mask & NETIF_F_HW_CSUM)
11916 mask |= NETIF_F_CSUM_MASK;
11917 mask |= NETIF_F_VLAN_CHALLENGED;
11918
11919 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
11920 all &= one | ~NETIF_F_ALL_FOR_ALL;
11921
11922 /* If one device supports hw checksumming, set for all. */
11923 if (all & NETIF_F_HW_CSUM)
11924 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
11925
11926 return all;
11927}
11928EXPORT_SYMBOL(netdev_increment_features);
11929
11930static struct hlist_head * __net_init netdev_create_hash(void)
11931{
11932 int i;
11933 struct hlist_head *hash;
11934
11935 hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
11936 if (hash != NULL)
11937 for (i = 0; i < NETDEV_HASHENTRIES; i++)
11938 INIT_HLIST_HEAD(&hash[i]);
11939
11940 return hash;
11941}
11942
11943/* Initialize per network namespace state */
11944static int __net_init netdev_init(struct net *net)
11945{
11946 BUILD_BUG_ON(GRO_HASH_BUCKETS >
11947 8 * sizeof_field(struct napi_struct, gro_bitmask));
11948
11949 INIT_LIST_HEAD(&net->dev_base_head);
11950
11951 net->dev_name_head = netdev_create_hash();
11952 if (net->dev_name_head == NULL)
11953 goto err_name;
11954
11955 net->dev_index_head = netdev_create_hash();
11956 if (net->dev_index_head == NULL)
11957 goto err_idx;
11958
11959 xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1);
11960
11961 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
11962
11963 return 0;
11964
11965err_idx:
11966 kfree(net->dev_name_head);
11967err_name:
11968 return -ENOMEM;
11969}
11970
11971/**
11972 * netdev_drivername - network driver for the device
11973 * @dev: network device
11974 *
11975 * Determine network driver for device.
11976 */
11977const char *netdev_drivername(const struct net_device *dev)
11978{
11979 const struct device_driver *driver;
11980 const struct device *parent;
11981 const char *empty = "";
11982
11983 parent = dev->dev.parent;
11984 if (!parent)
11985 return empty;
11986
11987 driver = parent->driver;
11988 if (driver && driver->name)
11989 return driver->name;
11990 return empty;
11991}
11992
11993static void __netdev_printk(const char *level, const struct net_device *dev,
11994 struct va_format *vaf)
11995{
11996 if (dev && dev->dev.parent) {
11997 dev_printk_emit(level[1] - '0',
11998 dev->dev.parent,
11999 "%s %s %s%s: %pV",
12000 dev_driver_string(dev->dev.parent),
12001 dev_name(dev->dev.parent),
12002 netdev_name(dev), netdev_reg_state(dev),
12003 vaf);
12004 } else if (dev) {
12005 printk("%s%s%s: %pV",
12006 level, netdev_name(dev), netdev_reg_state(dev), vaf);
12007 } else {
12008 printk("%s(NULL net_device): %pV", level, vaf);
12009 }
12010}
12011
12012void netdev_printk(const char *level, const struct net_device *dev,
12013 const char *format, ...)
12014{
12015 struct va_format vaf;
12016 va_list args;
12017
12018 va_start(args, format);
12019
12020 vaf.fmt = format;
12021 vaf.va = &args;
12022
12023 __netdev_printk(level, dev, &vaf);
12024
12025 va_end(args);
12026}
12027EXPORT_SYMBOL(netdev_printk);
12028
12029#define define_netdev_printk_level(func, level) \
12030void func(const struct net_device *dev, const char *fmt, ...) \
12031{ \
12032 struct va_format vaf; \
12033 va_list args; \
12034 \
12035 va_start(args, fmt); \
12036 \
12037 vaf.fmt = fmt; \
12038 vaf.va = &args; \
12039 \
12040 __netdev_printk(level, dev, &vaf); \
12041 \
12042 va_end(args); \
12043} \
12044EXPORT_SYMBOL(func);
12045
12046define_netdev_printk_level(netdev_emerg, KERN_EMERG);
12047define_netdev_printk_level(netdev_alert, KERN_ALERT);
12048define_netdev_printk_level(netdev_crit, KERN_CRIT);
12049define_netdev_printk_level(netdev_err, KERN_ERR);
12050define_netdev_printk_level(netdev_warn, KERN_WARNING);
12051define_netdev_printk_level(netdev_notice, KERN_NOTICE);
12052define_netdev_printk_level(netdev_info, KERN_INFO);
12053
12054static void __net_exit netdev_exit(struct net *net)
12055{
12056 kfree(net->dev_name_head);
12057 kfree(net->dev_index_head);
12058 xa_destroy(&net->dev_by_index);
12059 if (net != &init_net)
12060 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
12061}
12062
12063static struct pernet_operations __net_initdata netdev_net_ops = {
12064 .init = netdev_init,
12065 .exit = netdev_exit,
12066};
12067
12068static void __net_exit default_device_exit_net(struct net *net)
12069{
12070 struct netdev_name_node *name_node, *tmp;
12071 struct net_device *dev, *aux;
12072 /*
12073 * Push all migratable network devices back to the
12074 * initial network namespace
12075 */
12076 ASSERT_RTNL();
12077 for_each_netdev_safe(net, dev, aux) {
12078 int err;
12079 char fb_name[IFNAMSIZ];
12080
12081 /* Ignore unmoveable devices (i.e. loopback) */
12082 if (dev->netns_local)
12083 continue;
12084
12085 /* Leave virtual devices for the generic cleanup */
12086 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
12087 continue;
12088
12089 /* Push remaining network devices to init_net */
12090 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
12091 if (netdev_name_in_use(&init_net, fb_name))
12092 snprintf(fb_name, IFNAMSIZ, "dev%%d");
12093
12094 netdev_for_each_altname_safe(dev, name_node, tmp)
12095 if (netdev_name_in_use(&init_net, name_node->name))
12096 __netdev_name_node_alt_destroy(name_node);
12097
12098 err = dev_change_net_namespace(dev, &init_net, fb_name);
12099 if (err) {
12100 pr_emerg("%s: failed to move %s to init_net: %d\n",
12101 __func__, dev->name, err);
12102 BUG();
12103 }
12104 }
12105}
12106
12107static void __net_exit default_device_exit_batch(struct list_head *net_list)
12108{
12109 /* At exit all network devices most be removed from a network
12110 * namespace. Do this in the reverse order of registration.
12111 * Do this across as many network namespaces as possible to
12112 * improve batching efficiency.
12113 */
12114 struct net_device *dev;
12115 struct net *net;
12116 LIST_HEAD(dev_kill_list);
12117
12118 rtnl_lock();
12119 list_for_each_entry(net, net_list, exit_list) {
12120 default_device_exit_net(net);
12121 cond_resched();
12122 }
12123
12124 list_for_each_entry(net, net_list, exit_list) {
12125 for_each_netdev_reverse(net, dev) {
12126 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
12127 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
12128 else
12129 unregister_netdevice_queue(dev, &dev_kill_list);
12130 }
12131 }
12132 unregister_netdevice_many(&dev_kill_list);
12133 rtnl_unlock();
12134}
12135
12136static struct pernet_operations __net_initdata default_device_ops = {
12137 .exit_batch = default_device_exit_batch,
12138};
12139
12140static void __init net_dev_struct_check(void)
12141{
12142 /* TX read-mostly hotpath */
12143 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags_fast);
12144 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops);
12145 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops);
12146 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx);
12147 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, real_num_tx_queues);
12148 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size);
12149 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size);
12150 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs);
12151 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_partial_features);
12152 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc);
12153 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu);
12154 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom);
12155 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tc_to_txq);
12156#ifdef CONFIG_XPS
12157 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, xps_maps);
12158#endif
12159#ifdef CONFIG_NETFILTER_EGRESS
12160 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, nf_hooks_egress);
12161#endif
12162#ifdef CONFIG_NET_XGRESS
12163 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress);
12164#endif
12165 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 160);
12166
12167 /* TXRX read-mostly hotpath */
12168 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, lstats);
12169 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, state);
12170 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags);
12171 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len);
12172 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features);
12173 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr);
12174 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 46);
12175
12176 /* RX read-mostly hotpath */
12177 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific);
12178 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ifindex);
12179 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, real_num_rx_queues);
12180 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, _rx);
12181 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_max_size);
12182 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_ipv4_max_size);
12183 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler);
12184 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler_data);
12185 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, nd_net);
12186#ifdef CONFIG_NETPOLL
12187 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, npinfo);
12188#endif
12189#ifdef CONFIG_NET_XGRESS
12190 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress);
12191#endif
12192 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 92);
12193}
12194
12195/*
12196 * Initialize the DEV module. At boot time this walks the device list and
12197 * unhooks any devices that fail to initialise (normally hardware not
12198 * present) and leaves us with a valid list of present and active devices.
12199 *
12200 */
12201
12202/* We allocate 256 pages for each CPU if PAGE_SHIFT is 12 */
12203#define SYSTEM_PERCPU_PAGE_POOL_SIZE ((1 << 20) / PAGE_SIZE)
12204
12205static int net_page_pool_create(int cpuid)
12206{
12207#if IS_ENABLED(CONFIG_PAGE_POOL)
12208 struct page_pool_params page_pool_params = {
12209 .pool_size = SYSTEM_PERCPU_PAGE_POOL_SIZE,
12210 .flags = PP_FLAG_SYSTEM_POOL,
12211 .nid = cpu_to_mem(cpuid),
12212 };
12213 struct page_pool *pp_ptr;
12214
12215 pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid);
12216 if (IS_ERR(pp_ptr))
12217 return -ENOMEM;
12218
12219 per_cpu(system_page_pool, cpuid) = pp_ptr;
12220#endif
12221 return 0;
12222}
12223
12224static int backlog_napi_should_run(unsigned int cpu)
12225{
12226 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
12227 struct napi_struct *napi = &sd->backlog;
12228
12229 return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
12230}
12231
12232static void run_backlog_napi(unsigned int cpu)
12233{
12234 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
12235
12236 napi_threaded_poll_loop(&sd->backlog);
12237}
12238
12239static void backlog_napi_setup(unsigned int cpu)
12240{
12241 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
12242 struct napi_struct *napi = &sd->backlog;
12243
12244 napi->thread = this_cpu_read(backlog_napi);
12245 set_bit(NAPI_STATE_THREADED, &napi->state);
12246}
12247
12248static struct smp_hotplug_thread backlog_threads = {
12249 .store = &backlog_napi,
12250 .thread_should_run = backlog_napi_should_run,
12251 .thread_fn = run_backlog_napi,
12252 .thread_comm = "backlog_napi/%u",
12253 .setup = backlog_napi_setup,
12254};
12255
12256/*
12257 * This is called single threaded during boot, so no need
12258 * to take the rtnl semaphore.
12259 */
12260static int __init net_dev_init(void)
12261{
12262 int i, rc = -ENOMEM;
12263
12264 BUG_ON(!dev_boot_phase);
12265
12266 net_dev_struct_check();
12267
12268 if (dev_proc_init())
12269 goto out;
12270
12271 if (netdev_kobject_init())
12272 goto out;
12273
12274 for (i = 0; i < PTYPE_HASH_SIZE; i++)
12275 INIT_LIST_HEAD(&ptype_base[i]);
12276
12277 if (register_pernet_subsys(&netdev_net_ops))
12278 goto out;
12279
12280 /*
12281 * Initialise the packet receive queues.
12282 */
12283
12284 for_each_possible_cpu(i) {
12285 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
12286 struct softnet_data *sd = &per_cpu(softnet_data, i);
12287
12288 INIT_WORK(flush, flush_backlog);
12289
12290 skb_queue_head_init(&sd->input_pkt_queue);
12291 skb_queue_head_init(&sd->process_queue);
12292#ifdef CONFIG_XFRM_OFFLOAD
12293 skb_queue_head_init(&sd->xfrm_backlog);
12294#endif
12295 INIT_LIST_HEAD(&sd->poll_list);
12296 sd->output_queue_tailp = &sd->output_queue;
12297#ifdef CONFIG_RPS
12298 INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
12299 sd->cpu = i;
12300#endif
12301 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
12302 spin_lock_init(&sd->defer_lock);
12303
12304 init_gro_hash(&sd->backlog);
12305 sd->backlog.poll = process_backlog;
12306 sd->backlog.weight = weight_p;
12307 INIT_LIST_HEAD(&sd->backlog.poll_list);
12308
12309 if (net_page_pool_create(i))
12310 goto out;
12311 }
12312 if (use_backlog_threads())
12313 smpboot_register_percpu_thread(&backlog_threads);
12314
12315 dev_boot_phase = 0;
12316
12317 /* The loopback device is special if any other network devices
12318 * is present in a network namespace the loopback device must
12319 * be present. Since we now dynamically allocate and free the
12320 * loopback device ensure this invariant is maintained by
12321 * keeping the loopback device as the first device on the
12322 * list of network devices. Ensuring the loopback devices
12323 * is the first device that appears and the last network device
12324 * that disappears.
12325 */
12326 if (register_pernet_device(&loopback_net_ops))
12327 goto out;
12328
12329 if (register_pernet_device(&default_device_ops))
12330 goto out;
12331
12332 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
12333 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
12334
12335 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
12336 NULL, dev_cpu_dead);
12337 WARN_ON(rc < 0);
12338 rc = 0;
12339
12340 /* avoid static key IPIs to isolated CPUs */
12341 if (housekeeping_enabled(HK_TYPE_MISC))
12342 net_enable_timestamp();
12343out:
12344 if (rc < 0) {
12345 for_each_possible_cpu(i) {
12346 struct page_pool *pp_ptr;
12347
12348 pp_ptr = per_cpu(system_page_pool, i);
12349 if (!pp_ptr)
12350 continue;
12351
12352 page_pool_destroy(pp_ptr);
12353 per_cpu(system_page_pool, i) = NULL;
12354 }
12355 }
12356
12357 return rc;
12358}
12359
12360subsys_initcall(net_dev_init);
1/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
76#include <linux/bitops.h>
77#include <linux/capability.h>
78#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
81#include <linux/hash.h>
82#include <linux/slab.h>
83#include <linux/sched.h>
84#include <linux/mutex.h>
85#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
94#include <linux/ethtool.h>
95#include <linux/notifier.h>
96#include <linux/skbuff.h>
97#include <net/net_namespace.h>
98#include <net/sock.h>
99#include <net/busy_poll.h>
100#include <linux/rtnetlink.h>
101#include <linux/stat.h>
102#include <net/dst.h>
103#include <net/dst_metadata.h>
104#include <net/pkt_sched.h>
105#include <net/checksum.h>
106#include <net/xfrm.h>
107#include <linux/highmem.h>
108#include <linux/init.h>
109#include <linux/module.h>
110#include <linux/netpoll.h>
111#include <linux/rcupdate.h>
112#include <linux/delay.h>
113#include <net/iw_handler.h>
114#include <asm/current.h>
115#include <linux/audit.h>
116#include <linux/dmaengine.h>
117#include <linux/err.h>
118#include <linux/ctype.h>
119#include <linux/if_arp.h>
120#include <linux/if_vlan.h>
121#include <linux/ip.h>
122#include <net/ip.h>
123#include <net/mpls.h>
124#include <linux/ipv6.h>
125#include <linux/in.h>
126#include <linux/jhash.h>
127#include <linux/random.h>
128#include <trace/events/napi.h>
129#include <trace/events/net.h>
130#include <trace/events/skb.h>
131#include <linux/pci.h>
132#include <linux/inetdevice.h>
133#include <linux/cpu_rmap.h>
134#include <linux/static_key.h>
135#include <linux/hashtable.h>
136#include <linux/vmalloc.h>
137#include <linux/if_macvlan.h>
138#include <linux/errqueue.h>
139#include <linux/hrtimer.h>
140#include <linux/netfilter_ingress.h>
141#include <linux/sctp.h>
142
143#include "net-sysfs.h"
144
145/* Instead of increasing this, you should create a hash table. */
146#define MAX_GRO_SKBS 8
147
148/* This should be increased if a protocol with a bigger head is added. */
149#define GRO_MAX_HEAD (MAX_HEADER + 128)
150
151static DEFINE_SPINLOCK(ptype_lock);
152static DEFINE_SPINLOCK(offload_lock);
153struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
154struct list_head ptype_all __read_mostly; /* Taps */
155static struct list_head offload_base __read_mostly;
156
157static int netif_rx_internal(struct sk_buff *skb);
158static int call_netdevice_notifiers_info(unsigned long val,
159 struct net_device *dev,
160 struct netdev_notifier_info *info);
161
162/*
163 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
164 * semaphore.
165 *
166 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
167 *
168 * Writers must hold the rtnl semaphore while they loop through the
169 * dev_base_head list, and hold dev_base_lock for writing when they do the
170 * actual updates. This allows pure readers to access the list even
171 * while a writer is preparing to update it.
172 *
173 * To put it another way, dev_base_lock is held for writing only to
174 * protect against pure readers; the rtnl semaphore provides the
175 * protection against other writers.
176 *
177 * See, for example usages, register_netdevice() and
178 * unregister_netdevice(), which must be called with the rtnl
179 * semaphore held.
180 */
181DEFINE_RWLOCK(dev_base_lock);
182EXPORT_SYMBOL(dev_base_lock);
183
184/* protects napi_hash addition/deletion and napi_gen_id */
185static DEFINE_SPINLOCK(napi_hash_lock);
186
187static unsigned int napi_gen_id = NR_CPUS;
188static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
189
190static seqcount_t devnet_rename_seq;
191
192static inline void dev_base_seq_inc(struct net *net)
193{
194 while (++net->dev_base_seq == 0);
195}
196
197static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
198{
199 unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
200
201 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
202}
203
204static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
205{
206 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
207}
208
209static inline void rps_lock(struct softnet_data *sd)
210{
211#ifdef CONFIG_RPS
212 spin_lock(&sd->input_pkt_queue.lock);
213#endif
214}
215
216static inline void rps_unlock(struct softnet_data *sd)
217{
218#ifdef CONFIG_RPS
219 spin_unlock(&sd->input_pkt_queue.lock);
220#endif
221}
222
223/* Device list insertion */
224static void list_netdevice(struct net_device *dev)
225{
226 struct net *net = dev_net(dev);
227
228 ASSERT_RTNL();
229
230 write_lock_bh(&dev_base_lock);
231 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
232 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
233 hlist_add_head_rcu(&dev->index_hlist,
234 dev_index_hash(net, dev->ifindex));
235 write_unlock_bh(&dev_base_lock);
236
237 dev_base_seq_inc(net);
238}
239
240/* Device list removal
241 * caller must respect a RCU grace period before freeing/reusing dev
242 */
243static void unlist_netdevice(struct net_device *dev)
244{
245 ASSERT_RTNL();
246
247 /* Unlink dev from the device chain */
248 write_lock_bh(&dev_base_lock);
249 list_del_rcu(&dev->dev_list);
250 hlist_del_rcu(&dev->name_hlist);
251 hlist_del_rcu(&dev->index_hlist);
252 write_unlock_bh(&dev_base_lock);
253
254 dev_base_seq_inc(dev_net(dev));
255}
256
257/*
258 * Our notifier list
259 */
260
261static RAW_NOTIFIER_HEAD(netdev_chain);
262
263/*
264 * Device drivers call our routines to queue packets here. We empty the
265 * queue in the local softnet handler.
266 */
267
268DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
269EXPORT_PER_CPU_SYMBOL(softnet_data);
270
271#ifdef CONFIG_LOCKDEP
272/*
273 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
274 * according to dev->type
275 */
276static const unsigned short netdev_lock_type[] =
277 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
278 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
279 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
280 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
281 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
282 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
283 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
284 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
285 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
286 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
287 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
288 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
289 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
290 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
291 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
292
293static const char *const netdev_lock_name[] =
294 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
295 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
296 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
297 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
298 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
299 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
300 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
301 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
302 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
303 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
304 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
305 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
306 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
307 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
308 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
309
310static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
311static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
312
313static inline unsigned short netdev_lock_pos(unsigned short dev_type)
314{
315 int i;
316
317 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
318 if (netdev_lock_type[i] == dev_type)
319 return i;
320 /* the last key is used by default */
321 return ARRAY_SIZE(netdev_lock_type) - 1;
322}
323
324static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
325 unsigned short dev_type)
326{
327 int i;
328
329 i = netdev_lock_pos(dev_type);
330 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
331 netdev_lock_name[i]);
332}
333
334static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
335{
336 int i;
337
338 i = netdev_lock_pos(dev->type);
339 lockdep_set_class_and_name(&dev->addr_list_lock,
340 &netdev_addr_lock_key[i],
341 netdev_lock_name[i]);
342}
343#else
344static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
345 unsigned short dev_type)
346{
347}
348static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
349{
350}
351#endif
352
353/*******************************************************************************
354
355 Protocol management and registration routines
356
357*******************************************************************************/
358
359/*
360 * Add a protocol ID to the list. Now that the input handler is
361 * smarter we can dispense with all the messy stuff that used to be
362 * here.
363 *
364 * BEWARE!!! Protocol handlers, mangling input packets,
365 * MUST BE last in hash buckets and checking protocol handlers
366 * MUST start from promiscuous ptype_all chain in net_bh.
367 * It is true now, do not change it.
368 * Explanation follows: if protocol handler, mangling packet, will
369 * be the first on list, it is not able to sense, that packet
370 * is cloned and should be copied-on-write, so that it will
371 * change it and subsequent readers will get broken packet.
372 * --ANK (980803)
373 */
374
375static inline struct list_head *ptype_head(const struct packet_type *pt)
376{
377 if (pt->type == htons(ETH_P_ALL))
378 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
379 else
380 return pt->dev ? &pt->dev->ptype_specific :
381 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
382}
383
384/**
385 * dev_add_pack - add packet handler
386 * @pt: packet type declaration
387 *
388 * Add a protocol handler to the networking stack. The passed &packet_type
389 * is linked into kernel lists and may not be freed until it has been
390 * removed from the kernel lists.
391 *
392 * This call does not sleep therefore it can not
393 * guarantee all CPU's that are in middle of receiving packets
394 * will see the new packet type (until the next received packet).
395 */
396
397void dev_add_pack(struct packet_type *pt)
398{
399 struct list_head *head = ptype_head(pt);
400
401 spin_lock(&ptype_lock);
402 list_add_rcu(&pt->list, head);
403 spin_unlock(&ptype_lock);
404}
405EXPORT_SYMBOL(dev_add_pack);
406
407/**
408 * __dev_remove_pack - remove packet handler
409 * @pt: packet type declaration
410 *
411 * Remove a protocol handler that was previously added to the kernel
412 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
413 * from the kernel lists and can be freed or reused once this function
414 * returns.
415 *
416 * The packet type might still be in use by receivers
417 * and must not be freed until after all the CPU's have gone
418 * through a quiescent state.
419 */
420void __dev_remove_pack(struct packet_type *pt)
421{
422 struct list_head *head = ptype_head(pt);
423 struct packet_type *pt1;
424
425 spin_lock(&ptype_lock);
426
427 list_for_each_entry(pt1, head, list) {
428 if (pt == pt1) {
429 list_del_rcu(&pt->list);
430 goto out;
431 }
432 }
433
434 pr_warn("dev_remove_pack: %p not found\n", pt);
435out:
436 spin_unlock(&ptype_lock);
437}
438EXPORT_SYMBOL(__dev_remove_pack);
439
440/**
441 * dev_remove_pack - remove packet handler
442 * @pt: packet type declaration
443 *
444 * Remove a protocol handler that was previously added to the kernel
445 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
446 * from the kernel lists and can be freed or reused once this function
447 * returns.
448 *
449 * This call sleeps to guarantee that no CPU is looking at the packet
450 * type after return.
451 */
452void dev_remove_pack(struct packet_type *pt)
453{
454 __dev_remove_pack(pt);
455
456 synchronize_net();
457}
458EXPORT_SYMBOL(dev_remove_pack);
459
460
461/**
462 * dev_add_offload - register offload handlers
463 * @po: protocol offload declaration
464 *
465 * Add protocol offload handlers to the networking stack. The passed
466 * &proto_offload is linked into kernel lists and may not be freed until
467 * it has been removed from the kernel lists.
468 *
469 * This call does not sleep therefore it can not
470 * guarantee all CPU's that are in middle of receiving packets
471 * will see the new offload handlers (until the next received packet).
472 */
473void dev_add_offload(struct packet_offload *po)
474{
475 struct packet_offload *elem;
476
477 spin_lock(&offload_lock);
478 list_for_each_entry(elem, &offload_base, list) {
479 if (po->priority < elem->priority)
480 break;
481 }
482 list_add_rcu(&po->list, elem->list.prev);
483 spin_unlock(&offload_lock);
484}
485EXPORT_SYMBOL(dev_add_offload);
486
487/**
488 * __dev_remove_offload - remove offload handler
489 * @po: packet offload declaration
490 *
491 * Remove a protocol offload handler that was previously added to the
492 * kernel offload handlers by dev_add_offload(). The passed &offload_type
493 * is removed from the kernel lists and can be freed or reused once this
494 * function returns.
495 *
496 * The packet type might still be in use by receivers
497 * and must not be freed until after all the CPU's have gone
498 * through a quiescent state.
499 */
500static void __dev_remove_offload(struct packet_offload *po)
501{
502 struct list_head *head = &offload_base;
503 struct packet_offload *po1;
504
505 spin_lock(&offload_lock);
506
507 list_for_each_entry(po1, head, list) {
508 if (po == po1) {
509 list_del_rcu(&po->list);
510 goto out;
511 }
512 }
513
514 pr_warn("dev_remove_offload: %p not found\n", po);
515out:
516 spin_unlock(&offload_lock);
517}
518
519/**
520 * dev_remove_offload - remove packet offload handler
521 * @po: packet offload declaration
522 *
523 * Remove a packet offload handler that was previously added to the kernel
524 * offload handlers by dev_add_offload(). The passed &offload_type is
525 * removed from the kernel lists and can be freed or reused once this
526 * function returns.
527 *
528 * This call sleeps to guarantee that no CPU is looking at the packet
529 * type after return.
530 */
531void dev_remove_offload(struct packet_offload *po)
532{
533 __dev_remove_offload(po);
534
535 synchronize_net();
536}
537EXPORT_SYMBOL(dev_remove_offload);
538
539/******************************************************************************
540
541 Device Boot-time Settings Routines
542
543*******************************************************************************/
544
545/* Boot time configuration table */
546static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
547
548/**
549 * netdev_boot_setup_add - add new setup entry
550 * @name: name of the device
551 * @map: configured settings for the device
552 *
553 * Adds new setup entry to the dev_boot_setup list. The function
554 * returns 0 on error and 1 on success. This is a generic routine to
555 * all netdevices.
556 */
557static int netdev_boot_setup_add(char *name, struct ifmap *map)
558{
559 struct netdev_boot_setup *s;
560 int i;
561
562 s = dev_boot_setup;
563 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
564 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
565 memset(s[i].name, 0, sizeof(s[i].name));
566 strlcpy(s[i].name, name, IFNAMSIZ);
567 memcpy(&s[i].map, map, sizeof(s[i].map));
568 break;
569 }
570 }
571
572 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
573}
574
575/**
576 * netdev_boot_setup_check - check boot time settings
577 * @dev: the netdevice
578 *
579 * Check boot time settings for the device.
580 * The found settings are set for the device to be used
581 * later in the device probing.
582 * Returns 0 if no settings found, 1 if they are.
583 */
584int netdev_boot_setup_check(struct net_device *dev)
585{
586 struct netdev_boot_setup *s = dev_boot_setup;
587 int i;
588
589 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
590 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
591 !strcmp(dev->name, s[i].name)) {
592 dev->irq = s[i].map.irq;
593 dev->base_addr = s[i].map.base_addr;
594 dev->mem_start = s[i].map.mem_start;
595 dev->mem_end = s[i].map.mem_end;
596 return 1;
597 }
598 }
599 return 0;
600}
601EXPORT_SYMBOL(netdev_boot_setup_check);
602
603
604/**
605 * netdev_boot_base - get address from boot time settings
606 * @prefix: prefix for network device
607 * @unit: id for network device
608 *
609 * Check boot time settings for the base address of device.
610 * The found settings are set for the device to be used
611 * later in the device probing.
612 * Returns 0 if no settings found.
613 */
614unsigned long netdev_boot_base(const char *prefix, int unit)
615{
616 const struct netdev_boot_setup *s = dev_boot_setup;
617 char name[IFNAMSIZ];
618 int i;
619
620 sprintf(name, "%s%d", prefix, unit);
621
622 /*
623 * If device already registered then return base of 1
624 * to indicate not to probe for this interface
625 */
626 if (__dev_get_by_name(&init_net, name))
627 return 1;
628
629 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
630 if (!strcmp(name, s[i].name))
631 return s[i].map.base_addr;
632 return 0;
633}
634
635/*
636 * Saves at boot time configured settings for any netdevice.
637 */
638int __init netdev_boot_setup(char *str)
639{
640 int ints[5];
641 struct ifmap map;
642
643 str = get_options(str, ARRAY_SIZE(ints), ints);
644 if (!str || !*str)
645 return 0;
646
647 /* Save settings */
648 memset(&map, 0, sizeof(map));
649 if (ints[0] > 0)
650 map.irq = ints[1];
651 if (ints[0] > 1)
652 map.base_addr = ints[2];
653 if (ints[0] > 2)
654 map.mem_start = ints[3];
655 if (ints[0] > 3)
656 map.mem_end = ints[4];
657
658 /* Add new entry to the list */
659 return netdev_boot_setup_add(str, &map);
660}
661
662__setup("netdev=", netdev_boot_setup);
663
664/*******************************************************************************
665
666 Device Interface Subroutines
667
668*******************************************************************************/
669
670/**
671 * dev_get_iflink - get 'iflink' value of a interface
672 * @dev: targeted interface
673 *
674 * Indicates the ifindex the interface is linked to.
675 * Physical interfaces have the same 'ifindex' and 'iflink' values.
676 */
677
678int dev_get_iflink(const struct net_device *dev)
679{
680 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
681 return dev->netdev_ops->ndo_get_iflink(dev);
682
683 return dev->ifindex;
684}
685EXPORT_SYMBOL(dev_get_iflink);
686
687/**
688 * dev_fill_metadata_dst - Retrieve tunnel egress information.
689 * @dev: targeted interface
690 * @skb: The packet.
691 *
692 * For better visibility of tunnel traffic OVS needs to retrieve
693 * egress tunnel information for a packet. Following API allows
694 * user to get this info.
695 */
696int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
697{
698 struct ip_tunnel_info *info;
699
700 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
701 return -EINVAL;
702
703 info = skb_tunnel_info_unclone(skb);
704 if (!info)
705 return -ENOMEM;
706 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
707 return -EINVAL;
708
709 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
710}
711EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
712
713/**
714 * __dev_get_by_name - find a device by its name
715 * @net: the applicable net namespace
716 * @name: name to find
717 *
718 * Find an interface by name. Must be called under RTNL semaphore
719 * or @dev_base_lock. If the name is found a pointer to the device
720 * is returned. If the name is not found then %NULL is returned. The
721 * reference counters are not incremented so the caller must be
722 * careful with locks.
723 */
724
725struct net_device *__dev_get_by_name(struct net *net, const char *name)
726{
727 struct net_device *dev;
728 struct hlist_head *head = dev_name_hash(net, name);
729
730 hlist_for_each_entry(dev, head, name_hlist)
731 if (!strncmp(dev->name, name, IFNAMSIZ))
732 return dev;
733
734 return NULL;
735}
736EXPORT_SYMBOL(__dev_get_by_name);
737
738/**
739 * dev_get_by_name_rcu - find a device by its name
740 * @net: the applicable net namespace
741 * @name: name to find
742 *
743 * Find an interface by name.
744 * If the name is found a pointer to the device is returned.
745 * If the name is not found then %NULL is returned.
746 * The reference counters are not incremented so the caller must be
747 * careful with locks. The caller must hold RCU lock.
748 */
749
750struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
751{
752 struct net_device *dev;
753 struct hlist_head *head = dev_name_hash(net, name);
754
755 hlist_for_each_entry_rcu(dev, head, name_hlist)
756 if (!strncmp(dev->name, name, IFNAMSIZ))
757 return dev;
758
759 return NULL;
760}
761EXPORT_SYMBOL(dev_get_by_name_rcu);
762
763/**
764 * dev_get_by_name - find a device by its name
765 * @net: the applicable net namespace
766 * @name: name to find
767 *
768 * Find an interface by name. This can be called from any
769 * context and does its own locking. The returned handle has
770 * the usage count incremented and the caller must use dev_put() to
771 * release it when it is no longer needed. %NULL is returned if no
772 * matching device is found.
773 */
774
775struct net_device *dev_get_by_name(struct net *net, const char *name)
776{
777 struct net_device *dev;
778
779 rcu_read_lock();
780 dev = dev_get_by_name_rcu(net, name);
781 if (dev)
782 dev_hold(dev);
783 rcu_read_unlock();
784 return dev;
785}
786EXPORT_SYMBOL(dev_get_by_name);
787
788/**
789 * __dev_get_by_index - find a device by its ifindex
790 * @net: the applicable net namespace
791 * @ifindex: index of device
792 *
793 * Search for an interface by index. Returns %NULL if the device
794 * is not found or a pointer to the device. The device has not
795 * had its reference counter increased so the caller must be careful
796 * about locking. The caller must hold either the RTNL semaphore
797 * or @dev_base_lock.
798 */
799
800struct net_device *__dev_get_by_index(struct net *net, int ifindex)
801{
802 struct net_device *dev;
803 struct hlist_head *head = dev_index_hash(net, ifindex);
804
805 hlist_for_each_entry(dev, head, index_hlist)
806 if (dev->ifindex == ifindex)
807 return dev;
808
809 return NULL;
810}
811EXPORT_SYMBOL(__dev_get_by_index);
812
813/**
814 * dev_get_by_index_rcu - find a device by its ifindex
815 * @net: the applicable net namespace
816 * @ifindex: index of device
817 *
818 * Search for an interface by index. Returns %NULL if the device
819 * is not found or a pointer to the device. The device has not
820 * had its reference counter increased so the caller must be careful
821 * about locking. The caller must hold RCU lock.
822 */
823
824struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
825{
826 struct net_device *dev;
827 struct hlist_head *head = dev_index_hash(net, ifindex);
828
829 hlist_for_each_entry_rcu(dev, head, index_hlist)
830 if (dev->ifindex == ifindex)
831 return dev;
832
833 return NULL;
834}
835EXPORT_SYMBOL(dev_get_by_index_rcu);
836
837
838/**
839 * dev_get_by_index - find a device by its ifindex
840 * @net: the applicable net namespace
841 * @ifindex: index of device
842 *
843 * Search for an interface by index. Returns NULL if the device
844 * is not found or a pointer to the device. The device returned has
845 * had a reference added and the pointer is safe until the user calls
846 * dev_put to indicate they have finished with it.
847 */
848
849struct net_device *dev_get_by_index(struct net *net, int ifindex)
850{
851 struct net_device *dev;
852
853 rcu_read_lock();
854 dev = dev_get_by_index_rcu(net, ifindex);
855 if (dev)
856 dev_hold(dev);
857 rcu_read_unlock();
858 return dev;
859}
860EXPORT_SYMBOL(dev_get_by_index);
861
862/**
863 * netdev_get_name - get a netdevice name, knowing its ifindex.
864 * @net: network namespace
865 * @name: a pointer to the buffer where the name will be stored.
866 * @ifindex: the ifindex of the interface to get the name from.
867 *
868 * The use of raw_seqcount_begin() and cond_resched() before
869 * retrying is required as we want to give the writers a chance
870 * to complete when CONFIG_PREEMPT is not set.
871 */
872int netdev_get_name(struct net *net, char *name, int ifindex)
873{
874 struct net_device *dev;
875 unsigned int seq;
876
877retry:
878 seq = raw_seqcount_begin(&devnet_rename_seq);
879 rcu_read_lock();
880 dev = dev_get_by_index_rcu(net, ifindex);
881 if (!dev) {
882 rcu_read_unlock();
883 return -ENODEV;
884 }
885
886 strcpy(name, dev->name);
887 rcu_read_unlock();
888 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
889 cond_resched();
890 goto retry;
891 }
892
893 return 0;
894}
895
896/**
897 * dev_getbyhwaddr_rcu - find a device by its hardware address
898 * @net: the applicable net namespace
899 * @type: media type of device
900 * @ha: hardware address
901 *
902 * Search for an interface by MAC address. Returns NULL if the device
903 * is not found or a pointer to the device.
904 * The caller must hold RCU or RTNL.
905 * The returned device has not had its ref count increased
906 * and the caller must therefore be careful about locking
907 *
908 */
909
910struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
911 const char *ha)
912{
913 struct net_device *dev;
914
915 for_each_netdev_rcu(net, dev)
916 if (dev->type == type &&
917 !memcmp(dev->dev_addr, ha, dev->addr_len))
918 return dev;
919
920 return NULL;
921}
922EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
923
924struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
925{
926 struct net_device *dev;
927
928 ASSERT_RTNL();
929 for_each_netdev(net, dev)
930 if (dev->type == type)
931 return dev;
932
933 return NULL;
934}
935EXPORT_SYMBOL(__dev_getfirstbyhwtype);
936
937struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
938{
939 struct net_device *dev, *ret = NULL;
940
941 rcu_read_lock();
942 for_each_netdev_rcu(net, dev)
943 if (dev->type == type) {
944 dev_hold(dev);
945 ret = dev;
946 break;
947 }
948 rcu_read_unlock();
949 return ret;
950}
951EXPORT_SYMBOL(dev_getfirstbyhwtype);
952
953/**
954 * __dev_get_by_flags - find any device with given flags
955 * @net: the applicable net namespace
956 * @if_flags: IFF_* values
957 * @mask: bitmask of bits in if_flags to check
958 *
959 * Search for any interface with the given flags. Returns NULL if a device
960 * is not found or a pointer to the device. Must be called inside
961 * rtnl_lock(), and result refcount is unchanged.
962 */
963
964struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
965 unsigned short mask)
966{
967 struct net_device *dev, *ret;
968
969 ASSERT_RTNL();
970
971 ret = NULL;
972 for_each_netdev(net, dev) {
973 if (((dev->flags ^ if_flags) & mask) == 0) {
974 ret = dev;
975 break;
976 }
977 }
978 return ret;
979}
980EXPORT_SYMBOL(__dev_get_by_flags);
981
982/**
983 * dev_valid_name - check if name is okay for network device
984 * @name: name string
985 *
986 * Network device names need to be valid file names to
987 * to allow sysfs to work. We also disallow any kind of
988 * whitespace.
989 */
990bool dev_valid_name(const char *name)
991{
992 if (*name == '\0')
993 return false;
994 if (strlen(name) >= IFNAMSIZ)
995 return false;
996 if (!strcmp(name, ".") || !strcmp(name, ".."))
997 return false;
998
999 while (*name) {
1000 if (*name == '/' || *name == ':' || isspace(*name))
1001 return false;
1002 name++;
1003 }
1004 return true;
1005}
1006EXPORT_SYMBOL(dev_valid_name);
1007
1008/**
1009 * __dev_alloc_name - allocate a name for a device
1010 * @net: network namespace to allocate the device name in
1011 * @name: name format string
1012 * @buf: scratch buffer and result name string
1013 *
1014 * Passed a format string - eg "lt%d" it will try and find a suitable
1015 * id. It scans list of devices to build up a free map, then chooses
1016 * the first empty slot. The caller must hold the dev_base or rtnl lock
1017 * while allocating the name and adding the device in order to avoid
1018 * duplicates.
1019 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1020 * Returns the number of the unit assigned or a negative errno code.
1021 */
1022
1023static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1024{
1025 int i = 0;
1026 const char *p;
1027 const int max_netdevices = 8*PAGE_SIZE;
1028 unsigned long *inuse;
1029 struct net_device *d;
1030
1031 p = strnchr(name, IFNAMSIZ-1, '%');
1032 if (p) {
1033 /*
1034 * Verify the string as this thing may have come from
1035 * the user. There must be either one "%d" and no other "%"
1036 * characters.
1037 */
1038 if (p[1] != 'd' || strchr(p + 2, '%'))
1039 return -EINVAL;
1040
1041 /* Use one page as a bit array of possible slots */
1042 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1043 if (!inuse)
1044 return -ENOMEM;
1045
1046 for_each_netdev(net, d) {
1047 if (!sscanf(d->name, name, &i))
1048 continue;
1049 if (i < 0 || i >= max_netdevices)
1050 continue;
1051
1052 /* avoid cases where sscanf is not exact inverse of printf */
1053 snprintf(buf, IFNAMSIZ, name, i);
1054 if (!strncmp(buf, d->name, IFNAMSIZ))
1055 set_bit(i, inuse);
1056 }
1057
1058 i = find_first_zero_bit(inuse, max_netdevices);
1059 free_page((unsigned long) inuse);
1060 }
1061
1062 if (buf != name)
1063 snprintf(buf, IFNAMSIZ, name, i);
1064 if (!__dev_get_by_name(net, buf))
1065 return i;
1066
1067 /* It is possible to run out of possible slots
1068 * when the name is long and there isn't enough space left
1069 * for the digits, or if all bits are used.
1070 */
1071 return -ENFILE;
1072}
1073
1074/**
1075 * dev_alloc_name - allocate a name for a device
1076 * @dev: device
1077 * @name: name format string
1078 *
1079 * Passed a format string - eg "lt%d" it will try and find a suitable
1080 * id. It scans list of devices to build up a free map, then chooses
1081 * the first empty slot. The caller must hold the dev_base or rtnl lock
1082 * while allocating the name and adding the device in order to avoid
1083 * duplicates.
1084 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1085 * Returns the number of the unit assigned or a negative errno code.
1086 */
1087
1088int dev_alloc_name(struct net_device *dev, const char *name)
1089{
1090 char buf[IFNAMSIZ];
1091 struct net *net;
1092 int ret;
1093
1094 BUG_ON(!dev_net(dev));
1095 net = dev_net(dev);
1096 ret = __dev_alloc_name(net, name, buf);
1097 if (ret >= 0)
1098 strlcpy(dev->name, buf, IFNAMSIZ);
1099 return ret;
1100}
1101EXPORT_SYMBOL(dev_alloc_name);
1102
1103static int dev_alloc_name_ns(struct net *net,
1104 struct net_device *dev,
1105 const char *name)
1106{
1107 char buf[IFNAMSIZ];
1108 int ret;
1109
1110 ret = __dev_alloc_name(net, name, buf);
1111 if (ret >= 0)
1112 strlcpy(dev->name, buf, IFNAMSIZ);
1113 return ret;
1114}
1115
1116static int dev_get_valid_name(struct net *net,
1117 struct net_device *dev,
1118 const char *name)
1119{
1120 BUG_ON(!net);
1121
1122 if (!dev_valid_name(name))
1123 return -EINVAL;
1124
1125 if (strchr(name, '%'))
1126 return dev_alloc_name_ns(net, dev, name);
1127 else if (__dev_get_by_name(net, name))
1128 return -EEXIST;
1129 else if (dev->name != name)
1130 strlcpy(dev->name, name, IFNAMSIZ);
1131
1132 return 0;
1133}
1134
1135/**
1136 * dev_change_name - change name of a device
1137 * @dev: device
1138 * @newname: name (or format string) must be at least IFNAMSIZ
1139 *
1140 * Change name of a device, can pass format strings "eth%d".
1141 * for wildcarding.
1142 */
1143int dev_change_name(struct net_device *dev, const char *newname)
1144{
1145 unsigned char old_assign_type;
1146 char oldname[IFNAMSIZ];
1147 int err = 0;
1148 int ret;
1149 struct net *net;
1150
1151 ASSERT_RTNL();
1152 BUG_ON(!dev_net(dev));
1153
1154 net = dev_net(dev);
1155 if (dev->flags & IFF_UP)
1156 return -EBUSY;
1157
1158 write_seqcount_begin(&devnet_rename_seq);
1159
1160 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1161 write_seqcount_end(&devnet_rename_seq);
1162 return 0;
1163 }
1164
1165 memcpy(oldname, dev->name, IFNAMSIZ);
1166
1167 err = dev_get_valid_name(net, dev, newname);
1168 if (err < 0) {
1169 write_seqcount_end(&devnet_rename_seq);
1170 return err;
1171 }
1172
1173 if (oldname[0] && !strchr(oldname, '%'))
1174 netdev_info(dev, "renamed from %s\n", oldname);
1175
1176 old_assign_type = dev->name_assign_type;
1177 dev->name_assign_type = NET_NAME_RENAMED;
1178
1179rollback:
1180 ret = device_rename(&dev->dev, dev->name);
1181 if (ret) {
1182 memcpy(dev->name, oldname, IFNAMSIZ);
1183 dev->name_assign_type = old_assign_type;
1184 write_seqcount_end(&devnet_rename_seq);
1185 return ret;
1186 }
1187
1188 write_seqcount_end(&devnet_rename_seq);
1189
1190 netdev_adjacent_rename_links(dev, oldname);
1191
1192 write_lock_bh(&dev_base_lock);
1193 hlist_del_rcu(&dev->name_hlist);
1194 write_unlock_bh(&dev_base_lock);
1195
1196 synchronize_rcu();
1197
1198 write_lock_bh(&dev_base_lock);
1199 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1200 write_unlock_bh(&dev_base_lock);
1201
1202 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1203 ret = notifier_to_errno(ret);
1204
1205 if (ret) {
1206 /* err >= 0 after dev_alloc_name() or stores the first errno */
1207 if (err >= 0) {
1208 err = ret;
1209 write_seqcount_begin(&devnet_rename_seq);
1210 memcpy(dev->name, oldname, IFNAMSIZ);
1211 memcpy(oldname, newname, IFNAMSIZ);
1212 dev->name_assign_type = old_assign_type;
1213 old_assign_type = NET_NAME_RENAMED;
1214 goto rollback;
1215 } else {
1216 pr_err("%s: name change rollback failed: %d\n",
1217 dev->name, ret);
1218 }
1219 }
1220
1221 return err;
1222}
1223
1224/**
1225 * dev_set_alias - change ifalias of a device
1226 * @dev: device
1227 * @alias: name up to IFALIASZ
1228 * @len: limit of bytes to copy from info
1229 *
1230 * Set ifalias for a device,
1231 */
1232int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1233{
1234 char *new_ifalias;
1235
1236 ASSERT_RTNL();
1237
1238 if (len >= IFALIASZ)
1239 return -EINVAL;
1240
1241 if (!len) {
1242 kfree(dev->ifalias);
1243 dev->ifalias = NULL;
1244 return 0;
1245 }
1246
1247 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1248 if (!new_ifalias)
1249 return -ENOMEM;
1250 dev->ifalias = new_ifalias;
1251
1252 strlcpy(dev->ifalias, alias, len+1);
1253 return len;
1254}
1255
1256
1257/**
1258 * netdev_features_change - device changes features
1259 * @dev: device to cause notification
1260 *
1261 * Called to indicate a device has changed features.
1262 */
1263void netdev_features_change(struct net_device *dev)
1264{
1265 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1266}
1267EXPORT_SYMBOL(netdev_features_change);
1268
1269/**
1270 * netdev_state_change - device changes state
1271 * @dev: device to cause notification
1272 *
1273 * Called to indicate a device has changed state. This function calls
1274 * the notifier chains for netdev_chain and sends a NEWLINK message
1275 * to the routing socket.
1276 */
1277void netdev_state_change(struct net_device *dev)
1278{
1279 if (dev->flags & IFF_UP) {
1280 struct netdev_notifier_change_info change_info;
1281
1282 change_info.flags_changed = 0;
1283 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1284 &change_info.info);
1285 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1286 }
1287}
1288EXPORT_SYMBOL(netdev_state_change);
1289
1290/**
1291 * netdev_notify_peers - notify network peers about existence of @dev
1292 * @dev: network device
1293 *
1294 * Generate traffic such that interested network peers are aware of
1295 * @dev, such as by generating a gratuitous ARP. This may be used when
1296 * a device wants to inform the rest of the network about some sort of
1297 * reconfiguration such as a failover event or virtual machine
1298 * migration.
1299 */
1300void netdev_notify_peers(struct net_device *dev)
1301{
1302 rtnl_lock();
1303 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1304 rtnl_unlock();
1305}
1306EXPORT_SYMBOL(netdev_notify_peers);
1307
1308static int __dev_open(struct net_device *dev)
1309{
1310 const struct net_device_ops *ops = dev->netdev_ops;
1311 int ret;
1312
1313 ASSERT_RTNL();
1314
1315 if (!netif_device_present(dev))
1316 return -ENODEV;
1317
1318 /* Block netpoll from trying to do any rx path servicing.
1319 * If we don't do this there is a chance ndo_poll_controller
1320 * or ndo_poll may be running while we open the device
1321 */
1322 netpoll_poll_disable(dev);
1323
1324 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1325 ret = notifier_to_errno(ret);
1326 if (ret)
1327 return ret;
1328
1329 set_bit(__LINK_STATE_START, &dev->state);
1330
1331 if (ops->ndo_validate_addr)
1332 ret = ops->ndo_validate_addr(dev);
1333
1334 if (!ret && ops->ndo_open)
1335 ret = ops->ndo_open(dev);
1336
1337 netpoll_poll_enable(dev);
1338
1339 if (ret)
1340 clear_bit(__LINK_STATE_START, &dev->state);
1341 else {
1342 dev->flags |= IFF_UP;
1343 dev_set_rx_mode(dev);
1344 dev_activate(dev);
1345 add_device_randomness(dev->dev_addr, dev->addr_len);
1346 }
1347
1348 return ret;
1349}
1350
1351/**
1352 * dev_open - prepare an interface for use.
1353 * @dev: device to open
1354 *
1355 * Takes a device from down to up state. The device's private open
1356 * function is invoked and then the multicast lists are loaded. Finally
1357 * the device is moved into the up state and a %NETDEV_UP message is
1358 * sent to the netdev notifier chain.
1359 *
1360 * Calling this function on an active interface is a nop. On a failure
1361 * a negative errno code is returned.
1362 */
1363int dev_open(struct net_device *dev)
1364{
1365 int ret;
1366
1367 if (dev->flags & IFF_UP)
1368 return 0;
1369
1370 ret = __dev_open(dev);
1371 if (ret < 0)
1372 return ret;
1373
1374 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1375 call_netdevice_notifiers(NETDEV_UP, dev);
1376
1377 return ret;
1378}
1379EXPORT_SYMBOL(dev_open);
1380
1381static int __dev_close_many(struct list_head *head)
1382{
1383 struct net_device *dev;
1384
1385 ASSERT_RTNL();
1386 might_sleep();
1387
1388 list_for_each_entry(dev, head, close_list) {
1389 /* Temporarily disable netpoll until the interface is down */
1390 netpoll_poll_disable(dev);
1391
1392 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1393
1394 clear_bit(__LINK_STATE_START, &dev->state);
1395
1396 /* Synchronize to scheduled poll. We cannot touch poll list, it
1397 * can be even on different cpu. So just clear netif_running().
1398 *
1399 * dev->stop() will invoke napi_disable() on all of it's
1400 * napi_struct instances on this device.
1401 */
1402 smp_mb__after_atomic(); /* Commit netif_running(). */
1403 }
1404
1405 dev_deactivate_many(head);
1406
1407 list_for_each_entry(dev, head, close_list) {
1408 const struct net_device_ops *ops = dev->netdev_ops;
1409
1410 /*
1411 * Call the device specific close. This cannot fail.
1412 * Only if device is UP
1413 *
1414 * We allow it to be called even after a DETACH hot-plug
1415 * event.
1416 */
1417 if (ops->ndo_stop)
1418 ops->ndo_stop(dev);
1419
1420 dev->flags &= ~IFF_UP;
1421 netpoll_poll_enable(dev);
1422 }
1423
1424 return 0;
1425}
1426
1427static int __dev_close(struct net_device *dev)
1428{
1429 int retval;
1430 LIST_HEAD(single);
1431
1432 list_add(&dev->close_list, &single);
1433 retval = __dev_close_many(&single);
1434 list_del(&single);
1435
1436 return retval;
1437}
1438
1439int dev_close_many(struct list_head *head, bool unlink)
1440{
1441 struct net_device *dev, *tmp;
1442
1443 /* Remove the devices that don't need to be closed */
1444 list_for_each_entry_safe(dev, tmp, head, close_list)
1445 if (!(dev->flags & IFF_UP))
1446 list_del_init(&dev->close_list);
1447
1448 __dev_close_many(head);
1449
1450 list_for_each_entry_safe(dev, tmp, head, close_list) {
1451 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1452 call_netdevice_notifiers(NETDEV_DOWN, dev);
1453 if (unlink)
1454 list_del_init(&dev->close_list);
1455 }
1456
1457 return 0;
1458}
1459EXPORT_SYMBOL(dev_close_many);
1460
1461/**
1462 * dev_close - shutdown an interface.
1463 * @dev: device to shutdown
1464 *
1465 * This function moves an active device into down state. A
1466 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1467 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1468 * chain.
1469 */
1470int dev_close(struct net_device *dev)
1471{
1472 if (dev->flags & IFF_UP) {
1473 LIST_HEAD(single);
1474
1475 list_add(&dev->close_list, &single);
1476 dev_close_many(&single, true);
1477 list_del(&single);
1478 }
1479 return 0;
1480}
1481EXPORT_SYMBOL(dev_close);
1482
1483
1484/**
1485 * dev_disable_lro - disable Large Receive Offload on a device
1486 * @dev: device
1487 *
1488 * Disable Large Receive Offload (LRO) on a net device. Must be
1489 * called under RTNL. This is needed if received packets may be
1490 * forwarded to another interface.
1491 */
1492void dev_disable_lro(struct net_device *dev)
1493{
1494 struct net_device *lower_dev;
1495 struct list_head *iter;
1496
1497 dev->wanted_features &= ~NETIF_F_LRO;
1498 netdev_update_features(dev);
1499
1500 if (unlikely(dev->features & NETIF_F_LRO))
1501 netdev_WARN(dev, "failed to disable LRO!\n");
1502
1503 netdev_for_each_lower_dev(dev, lower_dev, iter)
1504 dev_disable_lro(lower_dev);
1505}
1506EXPORT_SYMBOL(dev_disable_lro);
1507
1508static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1509 struct net_device *dev)
1510{
1511 struct netdev_notifier_info info;
1512
1513 netdev_notifier_info_init(&info, dev);
1514 return nb->notifier_call(nb, val, &info);
1515}
1516
1517static int dev_boot_phase = 1;
1518
1519/**
1520 * register_netdevice_notifier - register a network notifier block
1521 * @nb: notifier
1522 *
1523 * Register a notifier to be called when network device events occur.
1524 * The notifier passed is linked into the kernel structures and must
1525 * not be reused until it has been unregistered. A negative errno code
1526 * is returned on a failure.
1527 *
1528 * When registered all registration and up events are replayed
1529 * to the new notifier to allow device to have a race free
1530 * view of the network device list.
1531 */
1532
1533int register_netdevice_notifier(struct notifier_block *nb)
1534{
1535 struct net_device *dev;
1536 struct net_device *last;
1537 struct net *net;
1538 int err;
1539
1540 rtnl_lock();
1541 err = raw_notifier_chain_register(&netdev_chain, nb);
1542 if (err)
1543 goto unlock;
1544 if (dev_boot_phase)
1545 goto unlock;
1546 for_each_net(net) {
1547 for_each_netdev(net, dev) {
1548 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1549 err = notifier_to_errno(err);
1550 if (err)
1551 goto rollback;
1552
1553 if (!(dev->flags & IFF_UP))
1554 continue;
1555
1556 call_netdevice_notifier(nb, NETDEV_UP, dev);
1557 }
1558 }
1559
1560unlock:
1561 rtnl_unlock();
1562 return err;
1563
1564rollback:
1565 last = dev;
1566 for_each_net(net) {
1567 for_each_netdev(net, dev) {
1568 if (dev == last)
1569 goto outroll;
1570
1571 if (dev->flags & IFF_UP) {
1572 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1573 dev);
1574 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1575 }
1576 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1577 }
1578 }
1579
1580outroll:
1581 raw_notifier_chain_unregister(&netdev_chain, nb);
1582 goto unlock;
1583}
1584EXPORT_SYMBOL(register_netdevice_notifier);
1585
1586/**
1587 * unregister_netdevice_notifier - unregister a network notifier block
1588 * @nb: notifier
1589 *
1590 * Unregister a notifier previously registered by
1591 * register_netdevice_notifier(). The notifier is unlinked into the
1592 * kernel structures and may then be reused. A negative errno code
1593 * is returned on a failure.
1594 *
1595 * After unregistering unregister and down device events are synthesized
1596 * for all devices on the device list to the removed notifier to remove
1597 * the need for special case cleanup code.
1598 */
1599
1600int unregister_netdevice_notifier(struct notifier_block *nb)
1601{
1602 struct net_device *dev;
1603 struct net *net;
1604 int err;
1605
1606 rtnl_lock();
1607 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1608 if (err)
1609 goto unlock;
1610
1611 for_each_net(net) {
1612 for_each_netdev(net, dev) {
1613 if (dev->flags & IFF_UP) {
1614 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1615 dev);
1616 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1617 }
1618 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1619 }
1620 }
1621unlock:
1622 rtnl_unlock();
1623 return err;
1624}
1625EXPORT_SYMBOL(unregister_netdevice_notifier);
1626
1627/**
1628 * call_netdevice_notifiers_info - call all network notifier blocks
1629 * @val: value passed unmodified to notifier function
1630 * @dev: net_device pointer passed unmodified to notifier function
1631 * @info: notifier information data
1632 *
1633 * Call all network notifier blocks. Parameters and return value
1634 * are as for raw_notifier_call_chain().
1635 */
1636
1637static int call_netdevice_notifiers_info(unsigned long val,
1638 struct net_device *dev,
1639 struct netdev_notifier_info *info)
1640{
1641 ASSERT_RTNL();
1642 netdev_notifier_info_init(info, dev);
1643 return raw_notifier_call_chain(&netdev_chain, val, info);
1644}
1645
1646/**
1647 * call_netdevice_notifiers - call all network notifier blocks
1648 * @val: value passed unmodified to notifier function
1649 * @dev: net_device pointer passed unmodified to notifier function
1650 *
1651 * Call all network notifier blocks. Parameters and return value
1652 * are as for raw_notifier_call_chain().
1653 */
1654
1655int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1656{
1657 struct netdev_notifier_info info;
1658
1659 return call_netdevice_notifiers_info(val, dev, &info);
1660}
1661EXPORT_SYMBOL(call_netdevice_notifiers);
1662
1663#ifdef CONFIG_NET_INGRESS
1664static struct static_key ingress_needed __read_mostly;
1665
1666void net_inc_ingress_queue(void)
1667{
1668 static_key_slow_inc(&ingress_needed);
1669}
1670EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1671
1672void net_dec_ingress_queue(void)
1673{
1674 static_key_slow_dec(&ingress_needed);
1675}
1676EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1677#endif
1678
1679#ifdef CONFIG_NET_EGRESS
1680static struct static_key egress_needed __read_mostly;
1681
1682void net_inc_egress_queue(void)
1683{
1684 static_key_slow_inc(&egress_needed);
1685}
1686EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1687
1688void net_dec_egress_queue(void)
1689{
1690 static_key_slow_dec(&egress_needed);
1691}
1692EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1693#endif
1694
1695static struct static_key netstamp_needed __read_mostly;
1696#ifdef HAVE_JUMP_LABEL
1697/* We are not allowed to call static_key_slow_dec() from irq context
1698 * If net_disable_timestamp() is called from irq context, defer the
1699 * static_key_slow_dec() calls.
1700 */
1701static atomic_t netstamp_needed_deferred;
1702#endif
1703
1704void net_enable_timestamp(void)
1705{
1706#ifdef HAVE_JUMP_LABEL
1707 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1708
1709 if (deferred) {
1710 while (--deferred)
1711 static_key_slow_dec(&netstamp_needed);
1712 return;
1713 }
1714#endif
1715 static_key_slow_inc(&netstamp_needed);
1716}
1717EXPORT_SYMBOL(net_enable_timestamp);
1718
1719void net_disable_timestamp(void)
1720{
1721#ifdef HAVE_JUMP_LABEL
1722 if (in_interrupt()) {
1723 atomic_inc(&netstamp_needed_deferred);
1724 return;
1725 }
1726#endif
1727 static_key_slow_dec(&netstamp_needed);
1728}
1729EXPORT_SYMBOL(net_disable_timestamp);
1730
1731static inline void net_timestamp_set(struct sk_buff *skb)
1732{
1733 skb->tstamp.tv64 = 0;
1734 if (static_key_false(&netstamp_needed))
1735 __net_timestamp(skb);
1736}
1737
1738#define net_timestamp_check(COND, SKB) \
1739 if (static_key_false(&netstamp_needed)) { \
1740 if ((COND) && !(SKB)->tstamp.tv64) \
1741 __net_timestamp(SKB); \
1742 } \
1743
1744bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
1745{
1746 unsigned int len;
1747
1748 if (!(dev->flags & IFF_UP))
1749 return false;
1750
1751 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1752 if (skb->len <= len)
1753 return true;
1754
1755 /* if TSO is enabled, we don't care about the length as the packet
1756 * could be forwarded without being segmented before
1757 */
1758 if (skb_is_gso(skb))
1759 return true;
1760
1761 return false;
1762}
1763EXPORT_SYMBOL_GPL(is_skb_forwardable);
1764
1765int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1766{
1767 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
1768 unlikely(!is_skb_forwardable(dev, skb))) {
1769 atomic_long_inc(&dev->rx_dropped);
1770 kfree_skb(skb);
1771 return NET_RX_DROP;
1772 }
1773
1774 skb_scrub_packet(skb, true);
1775 skb->priority = 0;
1776 skb->protocol = eth_type_trans(skb, dev);
1777 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1778
1779 return 0;
1780}
1781EXPORT_SYMBOL_GPL(__dev_forward_skb);
1782
1783/**
1784 * dev_forward_skb - loopback an skb to another netif
1785 *
1786 * @dev: destination network device
1787 * @skb: buffer to forward
1788 *
1789 * return values:
1790 * NET_RX_SUCCESS (no congestion)
1791 * NET_RX_DROP (packet was dropped, but freed)
1792 *
1793 * dev_forward_skb can be used for injecting an skb from the
1794 * start_xmit function of one device into the receive queue
1795 * of another device.
1796 *
1797 * The receiving device may be in another namespace, so
1798 * we have to clear all information in the skb that could
1799 * impact namespace isolation.
1800 */
1801int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1802{
1803 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1804}
1805EXPORT_SYMBOL_GPL(dev_forward_skb);
1806
1807static inline int deliver_skb(struct sk_buff *skb,
1808 struct packet_type *pt_prev,
1809 struct net_device *orig_dev)
1810{
1811 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1812 return -ENOMEM;
1813 atomic_inc(&skb->users);
1814 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1815}
1816
1817static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1818 struct packet_type **pt,
1819 struct net_device *orig_dev,
1820 __be16 type,
1821 struct list_head *ptype_list)
1822{
1823 struct packet_type *ptype, *pt_prev = *pt;
1824
1825 list_for_each_entry_rcu(ptype, ptype_list, list) {
1826 if (ptype->type != type)
1827 continue;
1828 if (pt_prev)
1829 deliver_skb(skb, pt_prev, orig_dev);
1830 pt_prev = ptype;
1831 }
1832 *pt = pt_prev;
1833}
1834
1835static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1836{
1837 if (!ptype->af_packet_priv || !skb->sk)
1838 return false;
1839
1840 if (ptype->id_match)
1841 return ptype->id_match(ptype, skb->sk);
1842 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1843 return true;
1844
1845 return false;
1846}
1847
1848/*
1849 * Support routine. Sends outgoing frames to any network
1850 * taps currently in use.
1851 */
1852
1853static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1854{
1855 struct packet_type *ptype;
1856 struct sk_buff *skb2 = NULL;
1857 struct packet_type *pt_prev = NULL;
1858 struct list_head *ptype_list = &ptype_all;
1859
1860 rcu_read_lock();
1861again:
1862 list_for_each_entry_rcu(ptype, ptype_list, list) {
1863 /* Never send packets back to the socket
1864 * they originated from - MvS (miquels@drinkel.ow.org)
1865 */
1866 if (skb_loop_sk(ptype, skb))
1867 continue;
1868
1869 if (pt_prev) {
1870 deliver_skb(skb2, pt_prev, skb->dev);
1871 pt_prev = ptype;
1872 continue;
1873 }
1874
1875 /* need to clone skb, done only once */
1876 skb2 = skb_clone(skb, GFP_ATOMIC);
1877 if (!skb2)
1878 goto out_unlock;
1879
1880 net_timestamp_set(skb2);
1881
1882 /* skb->nh should be correctly
1883 * set by sender, so that the second statement is
1884 * just protection against buggy protocols.
1885 */
1886 skb_reset_mac_header(skb2);
1887
1888 if (skb_network_header(skb2) < skb2->data ||
1889 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1890 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1891 ntohs(skb2->protocol),
1892 dev->name);
1893 skb_reset_network_header(skb2);
1894 }
1895
1896 skb2->transport_header = skb2->network_header;
1897 skb2->pkt_type = PACKET_OUTGOING;
1898 pt_prev = ptype;
1899 }
1900
1901 if (ptype_list == &ptype_all) {
1902 ptype_list = &dev->ptype_all;
1903 goto again;
1904 }
1905out_unlock:
1906 if (pt_prev)
1907 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1908 rcu_read_unlock();
1909}
1910
1911/**
1912 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1913 * @dev: Network device
1914 * @txq: number of queues available
1915 *
1916 * If real_num_tx_queues is changed the tc mappings may no longer be
1917 * valid. To resolve this verify the tc mapping remains valid and if
1918 * not NULL the mapping. With no priorities mapping to this
1919 * offset/count pair it will no longer be used. In the worst case TC0
1920 * is invalid nothing can be done so disable priority mappings. If is
1921 * expected that drivers will fix this mapping if they can before
1922 * calling netif_set_real_num_tx_queues.
1923 */
1924static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1925{
1926 int i;
1927 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1928
1929 /* If TC0 is invalidated disable TC mapping */
1930 if (tc->offset + tc->count > txq) {
1931 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1932 dev->num_tc = 0;
1933 return;
1934 }
1935
1936 /* Invalidated prio to tc mappings set to TC0 */
1937 for (i = 1; i < TC_BITMASK + 1; i++) {
1938 int q = netdev_get_prio_tc_map(dev, i);
1939
1940 tc = &dev->tc_to_txq[q];
1941 if (tc->offset + tc->count > txq) {
1942 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1943 i, q);
1944 netdev_set_prio_tc_map(dev, i, 0);
1945 }
1946 }
1947}
1948
1949#ifdef CONFIG_XPS
1950static DEFINE_MUTEX(xps_map_mutex);
1951#define xmap_dereference(P) \
1952 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1953
1954static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1955 int cpu, u16 index)
1956{
1957 struct xps_map *map = NULL;
1958 int pos;
1959
1960 if (dev_maps)
1961 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1962
1963 for (pos = 0; map && pos < map->len; pos++) {
1964 if (map->queues[pos] == index) {
1965 if (map->len > 1) {
1966 map->queues[pos] = map->queues[--map->len];
1967 } else {
1968 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1969 kfree_rcu(map, rcu);
1970 map = NULL;
1971 }
1972 break;
1973 }
1974 }
1975
1976 return map;
1977}
1978
1979static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1980{
1981 struct xps_dev_maps *dev_maps;
1982 int cpu, i;
1983 bool active = false;
1984
1985 mutex_lock(&xps_map_mutex);
1986 dev_maps = xmap_dereference(dev->xps_maps);
1987
1988 if (!dev_maps)
1989 goto out_no_maps;
1990
1991 for_each_possible_cpu(cpu) {
1992 for (i = index; i < dev->num_tx_queues; i++) {
1993 if (!remove_xps_queue(dev_maps, cpu, i))
1994 break;
1995 }
1996 if (i == dev->num_tx_queues)
1997 active = true;
1998 }
1999
2000 if (!active) {
2001 RCU_INIT_POINTER(dev->xps_maps, NULL);
2002 kfree_rcu(dev_maps, rcu);
2003 }
2004
2005 for (i = index; i < dev->num_tx_queues; i++)
2006 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2007 NUMA_NO_NODE);
2008
2009out_no_maps:
2010 mutex_unlock(&xps_map_mutex);
2011}
2012
2013static struct xps_map *expand_xps_map(struct xps_map *map,
2014 int cpu, u16 index)
2015{
2016 struct xps_map *new_map;
2017 int alloc_len = XPS_MIN_MAP_ALLOC;
2018 int i, pos;
2019
2020 for (pos = 0; map && pos < map->len; pos++) {
2021 if (map->queues[pos] != index)
2022 continue;
2023 return map;
2024 }
2025
2026 /* Need to add queue to this CPU's existing map */
2027 if (map) {
2028 if (pos < map->alloc_len)
2029 return map;
2030
2031 alloc_len = map->alloc_len * 2;
2032 }
2033
2034 /* Need to allocate new map to store queue on this CPU's map */
2035 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2036 cpu_to_node(cpu));
2037 if (!new_map)
2038 return NULL;
2039
2040 for (i = 0; i < pos; i++)
2041 new_map->queues[i] = map->queues[i];
2042 new_map->alloc_len = alloc_len;
2043 new_map->len = pos;
2044
2045 return new_map;
2046}
2047
2048int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2049 u16 index)
2050{
2051 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2052 struct xps_map *map, *new_map;
2053 int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
2054 int cpu, numa_node_id = -2;
2055 bool active = false;
2056
2057 mutex_lock(&xps_map_mutex);
2058
2059 dev_maps = xmap_dereference(dev->xps_maps);
2060
2061 /* allocate memory for queue storage */
2062 for_each_online_cpu(cpu) {
2063 if (!cpumask_test_cpu(cpu, mask))
2064 continue;
2065
2066 if (!new_dev_maps)
2067 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2068 if (!new_dev_maps) {
2069 mutex_unlock(&xps_map_mutex);
2070 return -ENOMEM;
2071 }
2072
2073 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2074 NULL;
2075
2076 map = expand_xps_map(map, cpu, index);
2077 if (!map)
2078 goto error;
2079
2080 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2081 }
2082
2083 if (!new_dev_maps)
2084 goto out_no_new_maps;
2085
2086 for_each_possible_cpu(cpu) {
2087 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2088 /* add queue to CPU maps */
2089 int pos = 0;
2090
2091 map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2092 while ((pos < map->len) && (map->queues[pos] != index))
2093 pos++;
2094
2095 if (pos == map->len)
2096 map->queues[map->len++] = index;
2097#ifdef CONFIG_NUMA
2098 if (numa_node_id == -2)
2099 numa_node_id = cpu_to_node(cpu);
2100 else if (numa_node_id != cpu_to_node(cpu))
2101 numa_node_id = -1;
2102#endif
2103 } else if (dev_maps) {
2104 /* fill in the new device map from the old device map */
2105 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2106 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2107 }
2108
2109 }
2110
2111 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2112
2113 /* Cleanup old maps */
2114 if (dev_maps) {
2115 for_each_possible_cpu(cpu) {
2116 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2117 map = xmap_dereference(dev_maps->cpu_map[cpu]);
2118 if (map && map != new_map)
2119 kfree_rcu(map, rcu);
2120 }
2121
2122 kfree_rcu(dev_maps, rcu);
2123 }
2124
2125 dev_maps = new_dev_maps;
2126 active = true;
2127
2128out_no_new_maps:
2129 /* update Tx queue numa node */
2130 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2131 (numa_node_id >= 0) ? numa_node_id :
2132 NUMA_NO_NODE);
2133
2134 if (!dev_maps)
2135 goto out_no_maps;
2136
2137 /* removes queue from unused CPUs */
2138 for_each_possible_cpu(cpu) {
2139 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2140 continue;
2141
2142 if (remove_xps_queue(dev_maps, cpu, index))
2143 active = true;
2144 }
2145
2146 /* free map if not active */
2147 if (!active) {
2148 RCU_INIT_POINTER(dev->xps_maps, NULL);
2149 kfree_rcu(dev_maps, rcu);
2150 }
2151
2152out_no_maps:
2153 mutex_unlock(&xps_map_mutex);
2154
2155 return 0;
2156error:
2157 /* remove any maps that we added */
2158 for_each_possible_cpu(cpu) {
2159 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2160 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2161 NULL;
2162 if (new_map && new_map != map)
2163 kfree(new_map);
2164 }
2165
2166 mutex_unlock(&xps_map_mutex);
2167
2168 kfree(new_dev_maps);
2169 return -ENOMEM;
2170}
2171EXPORT_SYMBOL(netif_set_xps_queue);
2172
2173#endif
2174/*
2175 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2176 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2177 */
2178int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2179{
2180 int rc;
2181
2182 if (txq < 1 || txq > dev->num_tx_queues)
2183 return -EINVAL;
2184
2185 if (dev->reg_state == NETREG_REGISTERED ||
2186 dev->reg_state == NETREG_UNREGISTERING) {
2187 ASSERT_RTNL();
2188
2189 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2190 txq);
2191 if (rc)
2192 return rc;
2193
2194 if (dev->num_tc)
2195 netif_setup_tc(dev, txq);
2196
2197 if (txq < dev->real_num_tx_queues) {
2198 qdisc_reset_all_tx_gt(dev, txq);
2199#ifdef CONFIG_XPS
2200 netif_reset_xps_queues_gt(dev, txq);
2201#endif
2202 }
2203 }
2204
2205 dev->real_num_tx_queues = txq;
2206 return 0;
2207}
2208EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2209
2210#ifdef CONFIG_SYSFS
2211/**
2212 * netif_set_real_num_rx_queues - set actual number of RX queues used
2213 * @dev: Network device
2214 * @rxq: Actual number of RX queues
2215 *
2216 * This must be called either with the rtnl_lock held or before
2217 * registration of the net device. Returns 0 on success, or a
2218 * negative error code. If called before registration, it always
2219 * succeeds.
2220 */
2221int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2222{
2223 int rc;
2224
2225 if (rxq < 1 || rxq > dev->num_rx_queues)
2226 return -EINVAL;
2227
2228 if (dev->reg_state == NETREG_REGISTERED) {
2229 ASSERT_RTNL();
2230
2231 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2232 rxq);
2233 if (rc)
2234 return rc;
2235 }
2236
2237 dev->real_num_rx_queues = rxq;
2238 return 0;
2239}
2240EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2241#endif
2242
2243/**
2244 * netif_get_num_default_rss_queues - default number of RSS queues
2245 *
2246 * This routine should set an upper limit on the number of RSS queues
2247 * used by default by multiqueue devices.
2248 */
2249int netif_get_num_default_rss_queues(void)
2250{
2251 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2252}
2253EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2254
2255static inline void __netif_reschedule(struct Qdisc *q)
2256{
2257 struct softnet_data *sd;
2258 unsigned long flags;
2259
2260 local_irq_save(flags);
2261 sd = this_cpu_ptr(&softnet_data);
2262 q->next_sched = NULL;
2263 *sd->output_queue_tailp = q;
2264 sd->output_queue_tailp = &q->next_sched;
2265 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2266 local_irq_restore(flags);
2267}
2268
2269void __netif_schedule(struct Qdisc *q)
2270{
2271 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2272 __netif_reschedule(q);
2273}
2274EXPORT_SYMBOL(__netif_schedule);
2275
2276struct dev_kfree_skb_cb {
2277 enum skb_free_reason reason;
2278};
2279
2280static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2281{
2282 return (struct dev_kfree_skb_cb *)skb->cb;
2283}
2284
2285void netif_schedule_queue(struct netdev_queue *txq)
2286{
2287 rcu_read_lock();
2288 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2289 struct Qdisc *q = rcu_dereference(txq->qdisc);
2290
2291 __netif_schedule(q);
2292 }
2293 rcu_read_unlock();
2294}
2295EXPORT_SYMBOL(netif_schedule_queue);
2296
2297/**
2298 * netif_wake_subqueue - allow sending packets on subqueue
2299 * @dev: network device
2300 * @queue_index: sub queue index
2301 *
2302 * Resume individual transmit queue of a device with multiple transmit queues.
2303 */
2304void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2305{
2306 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2307
2308 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2309 struct Qdisc *q;
2310
2311 rcu_read_lock();
2312 q = rcu_dereference(txq->qdisc);
2313 __netif_schedule(q);
2314 rcu_read_unlock();
2315 }
2316}
2317EXPORT_SYMBOL(netif_wake_subqueue);
2318
2319void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2320{
2321 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2322 struct Qdisc *q;
2323
2324 rcu_read_lock();
2325 q = rcu_dereference(dev_queue->qdisc);
2326 __netif_schedule(q);
2327 rcu_read_unlock();
2328 }
2329}
2330EXPORT_SYMBOL(netif_tx_wake_queue);
2331
2332void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2333{
2334 unsigned long flags;
2335
2336 if (likely(atomic_read(&skb->users) == 1)) {
2337 smp_rmb();
2338 atomic_set(&skb->users, 0);
2339 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2340 return;
2341 }
2342 get_kfree_skb_cb(skb)->reason = reason;
2343 local_irq_save(flags);
2344 skb->next = __this_cpu_read(softnet_data.completion_queue);
2345 __this_cpu_write(softnet_data.completion_queue, skb);
2346 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2347 local_irq_restore(flags);
2348}
2349EXPORT_SYMBOL(__dev_kfree_skb_irq);
2350
2351void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2352{
2353 if (in_irq() || irqs_disabled())
2354 __dev_kfree_skb_irq(skb, reason);
2355 else
2356 dev_kfree_skb(skb);
2357}
2358EXPORT_SYMBOL(__dev_kfree_skb_any);
2359
2360
2361/**
2362 * netif_device_detach - mark device as removed
2363 * @dev: network device
2364 *
2365 * Mark device as removed from system and therefore no longer available.
2366 */
2367void netif_device_detach(struct net_device *dev)
2368{
2369 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2370 netif_running(dev)) {
2371 netif_tx_stop_all_queues(dev);
2372 }
2373}
2374EXPORT_SYMBOL(netif_device_detach);
2375
2376/**
2377 * netif_device_attach - mark device as attached
2378 * @dev: network device
2379 *
2380 * Mark device as attached from system and restart if needed.
2381 */
2382void netif_device_attach(struct net_device *dev)
2383{
2384 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2385 netif_running(dev)) {
2386 netif_tx_wake_all_queues(dev);
2387 __netdev_watchdog_up(dev);
2388 }
2389}
2390EXPORT_SYMBOL(netif_device_attach);
2391
2392/*
2393 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2394 * to be used as a distribution range.
2395 */
2396u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2397 unsigned int num_tx_queues)
2398{
2399 u32 hash;
2400 u16 qoffset = 0;
2401 u16 qcount = num_tx_queues;
2402
2403 if (skb_rx_queue_recorded(skb)) {
2404 hash = skb_get_rx_queue(skb);
2405 while (unlikely(hash >= num_tx_queues))
2406 hash -= num_tx_queues;
2407 return hash;
2408 }
2409
2410 if (dev->num_tc) {
2411 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2412 qoffset = dev->tc_to_txq[tc].offset;
2413 qcount = dev->tc_to_txq[tc].count;
2414 }
2415
2416 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2417}
2418EXPORT_SYMBOL(__skb_tx_hash);
2419
2420static void skb_warn_bad_offload(const struct sk_buff *skb)
2421{
2422 static const netdev_features_t null_features = 0;
2423 struct net_device *dev = skb->dev;
2424 const char *name = "";
2425
2426 if (!net_ratelimit())
2427 return;
2428
2429 if (dev) {
2430 if (dev->dev.parent)
2431 name = dev_driver_string(dev->dev.parent);
2432 else
2433 name = netdev_name(dev);
2434 }
2435 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2436 "gso_type=%d ip_summed=%d\n",
2437 name, dev ? &dev->features : &null_features,
2438 skb->sk ? &skb->sk->sk_route_caps : &null_features,
2439 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2440 skb_shinfo(skb)->gso_type, skb->ip_summed);
2441}
2442
2443/*
2444 * Invalidate hardware checksum when packet is to be mangled, and
2445 * complete checksum manually on outgoing path.
2446 */
2447int skb_checksum_help(struct sk_buff *skb)
2448{
2449 __wsum csum;
2450 int ret = 0, offset;
2451
2452 if (skb->ip_summed == CHECKSUM_COMPLETE)
2453 goto out_set_summed;
2454
2455 if (unlikely(skb_shinfo(skb)->gso_size)) {
2456 skb_warn_bad_offload(skb);
2457 return -EINVAL;
2458 }
2459
2460 /* Before computing a checksum, we should make sure no frag could
2461 * be modified by an external entity : checksum could be wrong.
2462 */
2463 if (skb_has_shared_frag(skb)) {
2464 ret = __skb_linearize(skb);
2465 if (ret)
2466 goto out;
2467 }
2468
2469 offset = skb_checksum_start_offset(skb);
2470 BUG_ON(offset >= skb_headlen(skb));
2471 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2472
2473 offset += skb->csum_offset;
2474 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2475
2476 if (skb_cloned(skb) &&
2477 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2478 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2479 if (ret)
2480 goto out;
2481 }
2482
2483 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
2484out_set_summed:
2485 skb->ip_summed = CHECKSUM_NONE;
2486out:
2487 return ret;
2488}
2489EXPORT_SYMBOL(skb_checksum_help);
2490
2491/* skb_csum_offload_check - Driver helper function to determine if a device
2492 * with limited checksum offload capabilities is able to offload the checksum
2493 * for a given packet.
2494 *
2495 * Arguments:
2496 * skb - sk_buff for the packet in question
2497 * spec - contains the description of what device can offload
2498 * csum_encapped - returns true if the checksum being offloaded is
2499 * encpasulated. That is it is checksum for the transport header
2500 * in the inner headers.
2501 * checksum_help - when set indicates that helper function should
2502 * call skb_checksum_help if offload checks fail
2503 *
2504 * Returns:
2505 * true: Packet has passed the checksum checks and should be offloadable to
2506 * the device (a driver may still need to check for additional
2507 * restrictions of its device)
2508 * false: Checksum is not offloadable. If checksum_help was set then
2509 * skb_checksum_help was called to resolve checksum for non-GSO
2510 * packets and when IP protocol is not SCTP
2511 */
2512bool __skb_csum_offload_chk(struct sk_buff *skb,
2513 const struct skb_csum_offl_spec *spec,
2514 bool *csum_encapped,
2515 bool csum_help)
2516{
2517 struct iphdr *iph;
2518 struct ipv6hdr *ipv6;
2519 void *nhdr;
2520 int protocol;
2521 u8 ip_proto;
2522
2523 if (skb->protocol == htons(ETH_P_8021Q) ||
2524 skb->protocol == htons(ETH_P_8021AD)) {
2525 if (!spec->vlan_okay)
2526 goto need_help;
2527 }
2528
2529 /* We check whether the checksum refers to a transport layer checksum in
2530 * the outermost header or an encapsulated transport layer checksum that
2531 * corresponds to the inner headers of the skb. If the checksum is for
2532 * something else in the packet we need help.
2533 */
2534 if (skb_checksum_start_offset(skb) == skb_transport_offset(skb)) {
2535 /* Non-encapsulated checksum */
2536 protocol = eproto_to_ipproto(vlan_get_protocol(skb));
2537 nhdr = skb_network_header(skb);
2538 *csum_encapped = false;
2539 if (spec->no_not_encapped)
2540 goto need_help;
2541 } else if (skb->encapsulation && spec->encap_okay &&
2542 skb_checksum_start_offset(skb) ==
2543 skb_inner_transport_offset(skb)) {
2544 /* Encapsulated checksum */
2545 *csum_encapped = true;
2546 switch (skb->inner_protocol_type) {
2547 case ENCAP_TYPE_ETHER:
2548 protocol = eproto_to_ipproto(skb->inner_protocol);
2549 break;
2550 case ENCAP_TYPE_IPPROTO:
2551 protocol = skb->inner_protocol;
2552 break;
2553 }
2554 nhdr = skb_inner_network_header(skb);
2555 } else {
2556 goto need_help;
2557 }
2558
2559 switch (protocol) {
2560 case IPPROTO_IP:
2561 if (!spec->ipv4_okay)
2562 goto need_help;
2563 iph = nhdr;
2564 ip_proto = iph->protocol;
2565 if (iph->ihl != 5 && !spec->ip_options_okay)
2566 goto need_help;
2567 break;
2568 case IPPROTO_IPV6:
2569 if (!spec->ipv6_okay)
2570 goto need_help;
2571 if (spec->no_encapped_ipv6 && *csum_encapped)
2572 goto need_help;
2573 ipv6 = nhdr;
2574 nhdr += sizeof(*ipv6);
2575 ip_proto = ipv6->nexthdr;
2576 break;
2577 default:
2578 goto need_help;
2579 }
2580
2581ip_proto_again:
2582 switch (ip_proto) {
2583 case IPPROTO_TCP:
2584 if (!spec->tcp_okay ||
2585 skb->csum_offset != offsetof(struct tcphdr, check))
2586 goto need_help;
2587 break;
2588 case IPPROTO_UDP:
2589 if (!spec->udp_okay ||
2590 skb->csum_offset != offsetof(struct udphdr, check))
2591 goto need_help;
2592 break;
2593 case IPPROTO_SCTP:
2594 if (!spec->sctp_okay ||
2595 skb->csum_offset != offsetof(struct sctphdr, checksum))
2596 goto cant_help;
2597 break;
2598 case NEXTHDR_HOP:
2599 case NEXTHDR_ROUTING:
2600 case NEXTHDR_DEST: {
2601 u8 *opthdr = nhdr;
2602
2603 if (protocol != IPPROTO_IPV6 || !spec->ext_hdrs_okay)
2604 goto need_help;
2605
2606 ip_proto = opthdr[0];
2607 nhdr += (opthdr[1] + 1) << 3;
2608
2609 goto ip_proto_again;
2610 }
2611 default:
2612 goto need_help;
2613 }
2614
2615 /* Passed the tests for offloading checksum */
2616 return true;
2617
2618need_help:
2619 if (csum_help && !skb_shinfo(skb)->gso_size)
2620 skb_checksum_help(skb);
2621cant_help:
2622 return false;
2623}
2624EXPORT_SYMBOL(__skb_csum_offload_chk);
2625
2626__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2627{
2628 __be16 type = skb->protocol;
2629
2630 /* Tunnel gso handlers can set protocol to ethernet. */
2631 if (type == htons(ETH_P_TEB)) {
2632 struct ethhdr *eth;
2633
2634 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2635 return 0;
2636
2637 eth = (struct ethhdr *)skb_mac_header(skb);
2638 type = eth->h_proto;
2639 }
2640
2641 return __vlan_get_protocol(skb, type, depth);
2642}
2643
2644/**
2645 * skb_mac_gso_segment - mac layer segmentation handler.
2646 * @skb: buffer to segment
2647 * @features: features for the output path (see dev->features)
2648 */
2649struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2650 netdev_features_t features)
2651{
2652 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2653 struct packet_offload *ptype;
2654 int vlan_depth = skb->mac_len;
2655 __be16 type = skb_network_protocol(skb, &vlan_depth);
2656
2657 if (unlikely(!type))
2658 return ERR_PTR(-EINVAL);
2659
2660 __skb_pull(skb, vlan_depth);
2661
2662 rcu_read_lock();
2663 list_for_each_entry_rcu(ptype, &offload_base, list) {
2664 if (ptype->type == type && ptype->callbacks.gso_segment) {
2665 segs = ptype->callbacks.gso_segment(skb, features);
2666 break;
2667 }
2668 }
2669 rcu_read_unlock();
2670
2671 __skb_push(skb, skb->data - skb_mac_header(skb));
2672
2673 return segs;
2674}
2675EXPORT_SYMBOL(skb_mac_gso_segment);
2676
2677
2678/* openvswitch calls this on rx path, so we need a different check.
2679 */
2680static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2681{
2682 if (tx_path)
2683 return skb->ip_summed != CHECKSUM_PARTIAL;
2684 else
2685 return skb->ip_summed == CHECKSUM_NONE;
2686}
2687
2688/**
2689 * __skb_gso_segment - Perform segmentation on skb.
2690 * @skb: buffer to segment
2691 * @features: features for the output path (see dev->features)
2692 * @tx_path: whether it is called in TX path
2693 *
2694 * This function segments the given skb and returns a list of segments.
2695 *
2696 * It may return NULL if the skb requires no segmentation. This is
2697 * only possible when GSO is used for verifying header integrity.
2698 *
2699 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2700 */
2701struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2702 netdev_features_t features, bool tx_path)
2703{
2704 if (unlikely(skb_needs_check(skb, tx_path))) {
2705 int err;
2706
2707 skb_warn_bad_offload(skb);
2708
2709 err = skb_cow_head(skb, 0);
2710 if (err < 0)
2711 return ERR_PTR(err);
2712 }
2713
2714 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2715 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2716
2717 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2718 SKB_GSO_CB(skb)->encap_level = 0;
2719
2720 skb_reset_mac_header(skb);
2721 skb_reset_mac_len(skb);
2722
2723 return skb_mac_gso_segment(skb, features);
2724}
2725EXPORT_SYMBOL(__skb_gso_segment);
2726
2727/* Take action when hardware reception checksum errors are detected. */
2728#ifdef CONFIG_BUG
2729void netdev_rx_csum_fault(struct net_device *dev)
2730{
2731 if (net_ratelimit()) {
2732 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2733 dump_stack();
2734 }
2735}
2736EXPORT_SYMBOL(netdev_rx_csum_fault);
2737#endif
2738
2739/* Actually, we should eliminate this check as soon as we know, that:
2740 * 1. IOMMU is present and allows to map all the memory.
2741 * 2. No high memory really exists on this machine.
2742 */
2743
2744static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2745{
2746#ifdef CONFIG_HIGHMEM
2747 int i;
2748 if (!(dev->features & NETIF_F_HIGHDMA)) {
2749 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2750 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2751 if (PageHighMem(skb_frag_page(frag)))
2752 return 1;
2753 }
2754 }
2755
2756 if (PCI_DMA_BUS_IS_PHYS) {
2757 struct device *pdev = dev->dev.parent;
2758
2759 if (!pdev)
2760 return 0;
2761 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2762 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2763 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2764 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2765 return 1;
2766 }
2767 }
2768#endif
2769 return 0;
2770}
2771
2772/* If MPLS offload request, verify we are testing hardware MPLS features
2773 * instead of standard features for the netdev.
2774 */
2775#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2776static netdev_features_t net_mpls_features(struct sk_buff *skb,
2777 netdev_features_t features,
2778 __be16 type)
2779{
2780 if (eth_p_mpls(type))
2781 features &= skb->dev->mpls_features;
2782
2783 return features;
2784}
2785#else
2786static netdev_features_t net_mpls_features(struct sk_buff *skb,
2787 netdev_features_t features,
2788 __be16 type)
2789{
2790 return features;
2791}
2792#endif
2793
2794static netdev_features_t harmonize_features(struct sk_buff *skb,
2795 netdev_features_t features)
2796{
2797 int tmp;
2798 __be16 type;
2799
2800 type = skb_network_protocol(skb, &tmp);
2801 features = net_mpls_features(skb, features, type);
2802
2803 if (skb->ip_summed != CHECKSUM_NONE &&
2804 !can_checksum_protocol(features, type)) {
2805 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2806 } else if (illegal_highdma(skb->dev, skb)) {
2807 features &= ~NETIF_F_SG;
2808 }
2809
2810 return features;
2811}
2812
2813netdev_features_t passthru_features_check(struct sk_buff *skb,
2814 struct net_device *dev,
2815 netdev_features_t features)
2816{
2817 return features;
2818}
2819EXPORT_SYMBOL(passthru_features_check);
2820
2821static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2822 struct net_device *dev,
2823 netdev_features_t features)
2824{
2825 return vlan_features_check(skb, features);
2826}
2827
2828netdev_features_t netif_skb_features(struct sk_buff *skb)
2829{
2830 struct net_device *dev = skb->dev;
2831 netdev_features_t features = dev->features;
2832 u16 gso_segs = skb_shinfo(skb)->gso_segs;
2833
2834 if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
2835 features &= ~NETIF_F_GSO_MASK;
2836
2837 /* If encapsulation offload request, verify we are testing
2838 * hardware encapsulation features instead of standard
2839 * features for the netdev
2840 */
2841 if (skb->encapsulation)
2842 features &= dev->hw_enc_features;
2843
2844 if (skb_vlan_tagged(skb))
2845 features = netdev_intersect_features(features,
2846 dev->vlan_features |
2847 NETIF_F_HW_VLAN_CTAG_TX |
2848 NETIF_F_HW_VLAN_STAG_TX);
2849
2850 if (dev->netdev_ops->ndo_features_check)
2851 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2852 features);
2853 else
2854 features &= dflt_features_check(skb, dev, features);
2855
2856 return harmonize_features(skb, features);
2857}
2858EXPORT_SYMBOL(netif_skb_features);
2859
2860static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2861 struct netdev_queue *txq, bool more)
2862{
2863 unsigned int len;
2864 int rc;
2865
2866 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2867 dev_queue_xmit_nit(skb, dev);
2868
2869 len = skb->len;
2870 trace_net_dev_start_xmit(skb, dev);
2871 rc = netdev_start_xmit(skb, dev, txq, more);
2872 trace_net_dev_xmit(skb, rc, dev, len);
2873
2874 return rc;
2875}
2876
2877struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2878 struct netdev_queue *txq, int *ret)
2879{
2880 struct sk_buff *skb = first;
2881 int rc = NETDEV_TX_OK;
2882
2883 while (skb) {
2884 struct sk_buff *next = skb->next;
2885
2886 skb->next = NULL;
2887 rc = xmit_one(skb, dev, txq, next != NULL);
2888 if (unlikely(!dev_xmit_complete(rc))) {
2889 skb->next = next;
2890 goto out;
2891 }
2892
2893 skb = next;
2894 if (netif_xmit_stopped(txq) && skb) {
2895 rc = NETDEV_TX_BUSY;
2896 break;
2897 }
2898 }
2899
2900out:
2901 *ret = rc;
2902 return skb;
2903}
2904
2905static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2906 netdev_features_t features)
2907{
2908 if (skb_vlan_tag_present(skb) &&
2909 !vlan_hw_offload_capable(features, skb->vlan_proto))
2910 skb = __vlan_hwaccel_push_inside(skb);
2911 return skb;
2912}
2913
2914static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2915{
2916 netdev_features_t features;
2917
2918 if (skb->next)
2919 return skb;
2920
2921 features = netif_skb_features(skb);
2922 skb = validate_xmit_vlan(skb, features);
2923 if (unlikely(!skb))
2924 goto out_null;
2925
2926 if (netif_needs_gso(skb, features)) {
2927 struct sk_buff *segs;
2928
2929 segs = skb_gso_segment(skb, features);
2930 if (IS_ERR(segs)) {
2931 goto out_kfree_skb;
2932 } else if (segs) {
2933 consume_skb(skb);
2934 skb = segs;
2935 }
2936 } else {
2937 if (skb_needs_linearize(skb, features) &&
2938 __skb_linearize(skb))
2939 goto out_kfree_skb;
2940
2941 /* If packet is not checksummed and device does not
2942 * support checksumming for this protocol, complete
2943 * checksumming here.
2944 */
2945 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2946 if (skb->encapsulation)
2947 skb_set_inner_transport_header(skb,
2948 skb_checksum_start_offset(skb));
2949 else
2950 skb_set_transport_header(skb,
2951 skb_checksum_start_offset(skb));
2952 if (!(features & NETIF_F_CSUM_MASK) &&
2953 skb_checksum_help(skb))
2954 goto out_kfree_skb;
2955 }
2956 }
2957
2958 return skb;
2959
2960out_kfree_skb:
2961 kfree_skb(skb);
2962out_null:
2963 return NULL;
2964}
2965
2966struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2967{
2968 struct sk_buff *next, *head = NULL, *tail;
2969
2970 for (; skb != NULL; skb = next) {
2971 next = skb->next;
2972 skb->next = NULL;
2973
2974 /* in case skb wont be segmented, point to itself */
2975 skb->prev = skb;
2976
2977 skb = validate_xmit_skb(skb, dev);
2978 if (!skb)
2979 continue;
2980
2981 if (!head)
2982 head = skb;
2983 else
2984 tail->next = skb;
2985 /* If skb was segmented, skb->prev points to
2986 * the last segment. If not, it still contains skb.
2987 */
2988 tail = skb->prev;
2989 }
2990 return head;
2991}
2992
2993static void qdisc_pkt_len_init(struct sk_buff *skb)
2994{
2995 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2996
2997 qdisc_skb_cb(skb)->pkt_len = skb->len;
2998
2999 /* To get more precise estimation of bytes sent on wire,
3000 * we add to pkt_len the headers size of all segments
3001 */
3002 if (shinfo->gso_size) {
3003 unsigned int hdr_len;
3004 u16 gso_segs = shinfo->gso_segs;
3005
3006 /* mac layer + network layer */
3007 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3008
3009 /* + transport layer */
3010 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3011 hdr_len += tcp_hdrlen(skb);
3012 else
3013 hdr_len += sizeof(struct udphdr);
3014
3015 if (shinfo->gso_type & SKB_GSO_DODGY)
3016 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3017 shinfo->gso_size);
3018
3019 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3020 }
3021}
3022
3023static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3024 struct net_device *dev,
3025 struct netdev_queue *txq)
3026{
3027 spinlock_t *root_lock = qdisc_lock(q);
3028 bool contended;
3029 int rc;
3030
3031 qdisc_calculate_pkt_len(skb, q);
3032 /*
3033 * Heuristic to force contended enqueues to serialize on a
3034 * separate lock before trying to get qdisc main lock.
3035 * This permits __QDISC___STATE_RUNNING owner to get the lock more
3036 * often and dequeue packets faster.
3037 */
3038 contended = qdisc_is_running(q);
3039 if (unlikely(contended))
3040 spin_lock(&q->busylock);
3041
3042 spin_lock(root_lock);
3043 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3044 kfree_skb(skb);
3045 rc = NET_XMIT_DROP;
3046 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3047 qdisc_run_begin(q)) {
3048 /*
3049 * This is a work-conserving queue; there are no old skbs
3050 * waiting to be sent out; and the qdisc is not running -
3051 * xmit the skb directly.
3052 */
3053
3054 qdisc_bstats_update(q, skb);
3055
3056 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3057 if (unlikely(contended)) {
3058 spin_unlock(&q->busylock);
3059 contended = false;
3060 }
3061 __qdisc_run(q);
3062 } else
3063 qdisc_run_end(q);
3064
3065 rc = NET_XMIT_SUCCESS;
3066 } else {
3067 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
3068 if (qdisc_run_begin(q)) {
3069 if (unlikely(contended)) {
3070 spin_unlock(&q->busylock);
3071 contended = false;
3072 }
3073 __qdisc_run(q);
3074 }
3075 }
3076 spin_unlock(root_lock);
3077 if (unlikely(contended))
3078 spin_unlock(&q->busylock);
3079 return rc;
3080}
3081
3082#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3083static void skb_update_prio(struct sk_buff *skb)
3084{
3085 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
3086
3087 if (!skb->priority && skb->sk && map) {
3088 unsigned int prioidx =
3089 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
3090
3091 if (prioidx < map->priomap_len)
3092 skb->priority = map->priomap[prioidx];
3093 }
3094}
3095#else
3096#define skb_update_prio(skb)
3097#endif
3098
3099DEFINE_PER_CPU(int, xmit_recursion);
3100EXPORT_SYMBOL(xmit_recursion);
3101
3102#define RECURSION_LIMIT 10
3103
3104/**
3105 * dev_loopback_xmit - loop back @skb
3106 * @net: network namespace this loopback is happening in
3107 * @sk: sk needed to be a netfilter okfn
3108 * @skb: buffer to transmit
3109 */
3110int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3111{
3112 skb_reset_mac_header(skb);
3113 __skb_pull(skb, skb_network_offset(skb));
3114 skb->pkt_type = PACKET_LOOPBACK;
3115 skb->ip_summed = CHECKSUM_UNNECESSARY;
3116 WARN_ON(!skb_dst(skb));
3117 skb_dst_force(skb);
3118 netif_rx_ni(skb);
3119 return 0;
3120}
3121EXPORT_SYMBOL(dev_loopback_xmit);
3122
3123#ifdef CONFIG_NET_EGRESS
3124static struct sk_buff *
3125sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3126{
3127 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3128 struct tcf_result cl_res;
3129
3130 if (!cl)
3131 return skb;
3132
3133 /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set
3134 * earlier by the caller.
3135 */
3136 qdisc_bstats_cpu_update(cl->q, skb);
3137
3138 switch (tc_classify(skb, cl, &cl_res, false)) {
3139 case TC_ACT_OK:
3140 case TC_ACT_RECLASSIFY:
3141 skb->tc_index = TC_H_MIN(cl_res.classid);
3142 break;
3143 case TC_ACT_SHOT:
3144 qdisc_qstats_cpu_drop(cl->q);
3145 *ret = NET_XMIT_DROP;
3146 goto drop;
3147 case TC_ACT_STOLEN:
3148 case TC_ACT_QUEUED:
3149 *ret = NET_XMIT_SUCCESS;
3150drop:
3151 kfree_skb(skb);
3152 return NULL;
3153 case TC_ACT_REDIRECT:
3154 /* No need to push/pop skb's mac_header here on egress! */
3155 skb_do_redirect(skb);
3156 *ret = NET_XMIT_SUCCESS;
3157 return NULL;
3158 default:
3159 break;
3160 }
3161
3162 return skb;
3163}
3164#endif /* CONFIG_NET_EGRESS */
3165
3166static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3167{
3168#ifdef CONFIG_XPS
3169 struct xps_dev_maps *dev_maps;
3170 struct xps_map *map;
3171 int queue_index = -1;
3172
3173 rcu_read_lock();
3174 dev_maps = rcu_dereference(dev->xps_maps);
3175 if (dev_maps) {
3176 map = rcu_dereference(
3177 dev_maps->cpu_map[skb->sender_cpu - 1]);
3178 if (map) {
3179 if (map->len == 1)
3180 queue_index = map->queues[0];
3181 else
3182 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3183 map->len)];
3184 if (unlikely(queue_index >= dev->real_num_tx_queues))
3185 queue_index = -1;
3186 }
3187 }
3188 rcu_read_unlock();
3189
3190 return queue_index;
3191#else
3192 return -1;
3193#endif
3194}
3195
3196static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3197{
3198 struct sock *sk = skb->sk;
3199 int queue_index = sk_tx_queue_get(sk);
3200
3201 if (queue_index < 0 || skb->ooo_okay ||
3202 queue_index >= dev->real_num_tx_queues) {
3203 int new_index = get_xps_queue(dev, skb);
3204 if (new_index < 0)
3205 new_index = skb_tx_hash(dev, skb);
3206
3207 if (queue_index != new_index && sk &&
3208 sk_fullsock(sk) &&
3209 rcu_access_pointer(sk->sk_dst_cache))
3210 sk_tx_queue_set(sk, new_index);
3211
3212 queue_index = new_index;
3213 }
3214
3215 return queue_index;
3216}
3217
3218struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3219 struct sk_buff *skb,
3220 void *accel_priv)
3221{
3222 int queue_index = 0;
3223
3224#ifdef CONFIG_XPS
3225 u32 sender_cpu = skb->sender_cpu - 1;
3226
3227 if (sender_cpu >= (u32)NR_CPUS)
3228 skb->sender_cpu = raw_smp_processor_id() + 1;
3229#endif
3230
3231 if (dev->real_num_tx_queues != 1) {
3232 const struct net_device_ops *ops = dev->netdev_ops;
3233 if (ops->ndo_select_queue)
3234 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3235 __netdev_pick_tx);
3236 else
3237 queue_index = __netdev_pick_tx(dev, skb);
3238
3239 if (!accel_priv)
3240 queue_index = netdev_cap_txqueue(dev, queue_index);
3241 }
3242
3243 skb_set_queue_mapping(skb, queue_index);
3244 return netdev_get_tx_queue(dev, queue_index);
3245}
3246
3247/**
3248 * __dev_queue_xmit - transmit a buffer
3249 * @skb: buffer to transmit
3250 * @accel_priv: private data used for L2 forwarding offload
3251 *
3252 * Queue a buffer for transmission to a network device. The caller must
3253 * have set the device and priority and built the buffer before calling
3254 * this function. The function can be called from an interrupt.
3255 *
3256 * A negative errno code is returned on a failure. A success does not
3257 * guarantee the frame will be transmitted as it may be dropped due
3258 * to congestion or traffic shaping.
3259 *
3260 * -----------------------------------------------------------------------------------
3261 * I notice this method can also return errors from the queue disciplines,
3262 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3263 * be positive.
3264 *
3265 * Regardless of the return value, the skb is consumed, so it is currently
3266 * difficult to retry a send to this method. (You can bump the ref count
3267 * before sending to hold a reference for retry if you are careful.)
3268 *
3269 * When calling this method, interrupts MUST be enabled. This is because
3270 * the BH enable code must have IRQs enabled so that it will not deadlock.
3271 * --BLG
3272 */
3273static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3274{
3275 struct net_device *dev = skb->dev;
3276 struct netdev_queue *txq;
3277 struct Qdisc *q;
3278 int rc = -ENOMEM;
3279
3280 skb_reset_mac_header(skb);
3281
3282 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3283 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3284
3285 /* Disable soft irqs for various locks below. Also
3286 * stops preemption for RCU.
3287 */
3288 rcu_read_lock_bh();
3289
3290 skb_update_prio(skb);
3291
3292 qdisc_pkt_len_init(skb);
3293#ifdef CONFIG_NET_CLS_ACT
3294 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
3295# ifdef CONFIG_NET_EGRESS
3296 if (static_key_false(&egress_needed)) {
3297 skb = sch_handle_egress(skb, &rc, dev);
3298 if (!skb)
3299 goto out;
3300 }
3301# endif
3302#endif
3303 /* If device/qdisc don't need skb->dst, release it right now while
3304 * its hot in this cpu cache.
3305 */
3306 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3307 skb_dst_drop(skb);
3308 else
3309 skb_dst_force(skb);
3310
3311#ifdef CONFIG_NET_SWITCHDEV
3312 /* Don't forward if offload device already forwarded */
3313 if (skb->offload_fwd_mark &&
3314 skb->offload_fwd_mark == dev->offload_fwd_mark) {
3315 consume_skb(skb);
3316 rc = NET_XMIT_SUCCESS;
3317 goto out;
3318 }
3319#endif
3320
3321 txq = netdev_pick_tx(dev, skb, accel_priv);
3322 q = rcu_dereference_bh(txq->qdisc);
3323
3324 trace_net_dev_queue(skb);
3325 if (q->enqueue) {
3326 rc = __dev_xmit_skb(skb, q, dev, txq);
3327 goto out;
3328 }
3329
3330 /* The device has no queue. Common case for software devices:
3331 loopback, all the sorts of tunnels...
3332
3333 Really, it is unlikely that netif_tx_lock protection is necessary
3334 here. (f.e. loopback and IP tunnels are clean ignoring statistics
3335 counters.)
3336 However, it is possible, that they rely on protection
3337 made by us here.
3338
3339 Check this and shot the lock. It is not prone from deadlocks.
3340 Either shot noqueue qdisc, it is even simpler 8)
3341 */
3342 if (dev->flags & IFF_UP) {
3343 int cpu = smp_processor_id(); /* ok because BHs are off */
3344
3345 if (txq->xmit_lock_owner != cpu) {
3346
3347 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
3348 goto recursion_alert;
3349
3350 skb = validate_xmit_skb(skb, dev);
3351 if (!skb)
3352 goto drop;
3353
3354 HARD_TX_LOCK(dev, txq, cpu);
3355
3356 if (!netif_xmit_stopped(txq)) {
3357 __this_cpu_inc(xmit_recursion);
3358 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3359 __this_cpu_dec(xmit_recursion);
3360 if (dev_xmit_complete(rc)) {
3361 HARD_TX_UNLOCK(dev, txq);
3362 goto out;
3363 }
3364 }
3365 HARD_TX_UNLOCK(dev, txq);
3366 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3367 dev->name);
3368 } else {
3369 /* Recursion is detected! It is possible,
3370 * unfortunately
3371 */
3372recursion_alert:
3373 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3374 dev->name);
3375 }
3376 }
3377
3378 rc = -ENETDOWN;
3379drop:
3380 rcu_read_unlock_bh();
3381
3382 atomic_long_inc(&dev->tx_dropped);
3383 kfree_skb_list(skb);
3384 return rc;
3385out:
3386 rcu_read_unlock_bh();
3387 return rc;
3388}
3389
3390int dev_queue_xmit(struct sk_buff *skb)
3391{
3392 return __dev_queue_xmit(skb, NULL);
3393}
3394EXPORT_SYMBOL(dev_queue_xmit);
3395
3396int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3397{
3398 return __dev_queue_xmit(skb, accel_priv);
3399}
3400EXPORT_SYMBOL(dev_queue_xmit_accel);
3401
3402
3403/*=======================================================================
3404 Receiver routines
3405 =======================================================================*/
3406
3407int netdev_max_backlog __read_mostly = 1000;
3408EXPORT_SYMBOL(netdev_max_backlog);
3409
3410int netdev_tstamp_prequeue __read_mostly = 1;
3411int netdev_budget __read_mostly = 300;
3412int weight_p __read_mostly = 64; /* old backlog weight */
3413
3414/* Called with irq disabled */
3415static inline void ____napi_schedule(struct softnet_data *sd,
3416 struct napi_struct *napi)
3417{
3418 list_add_tail(&napi->poll_list, &sd->poll_list);
3419 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3420}
3421
3422#ifdef CONFIG_RPS
3423
3424/* One global table that all flow-based protocols share. */
3425struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3426EXPORT_SYMBOL(rps_sock_flow_table);
3427u32 rps_cpu_mask __read_mostly;
3428EXPORT_SYMBOL(rps_cpu_mask);
3429
3430struct static_key rps_needed __read_mostly;
3431
3432static struct rps_dev_flow *
3433set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3434 struct rps_dev_flow *rflow, u16 next_cpu)
3435{
3436 if (next_cpu < nr_cpu_ids) {
3437#ifdef CONFIG_RFS_ACCEL
3438 struct netdev_rx_queue *rxqueue;
3439 struct rps_dev_flow_table *flow_table;
3440 struct rps_dev_flow *old_rflow;
3441 u32 flow_id;
3442 u16 rxq_index;
3443 int rc;
3444
3445 /* Should we steer this flow to a different hardware queue? */
3446 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3447 !(dev->features & NETIF_F_NTUPLE))
3448 goto out;
3449 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3450 if (rxq_index == skb_get_rx_queue(skb))
3451 goto out;
3452
3453 rxqueue = dev->_rx + rxq_index;
3454 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3455 if (!flow_table)
3456 goto out;
3457 flow_id = skb_get_hash(skb) & flow_table->mask;
3458 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3459 rxq_index, flow_id);
3460 if (rc < 0)
3461 goto out;
3462 old_rflow = rflow;
3463 rflow = &flow_table->flows[flow_id];
3464 rflow->filter = rc;
3465 if (old_rflow->filter == rflow->filter)
3466 old_rflow->filter = RPS_NO_FILTER;
3467 out:
3468#endif
3469 rflow->last_qtail =
3470 per_cpu(softnet_data, next_cpu).input_queue_head;
3471 }
3472
3473 rflow->cpu = next_cpu;
3474 return rflow;
3475}
3476
3477/*
3478 * get_rps_cpu is called from netif_receive_skb and returns the target
3479 * CPU from the RPS map of the receiving queue for a given skb.
3480 * rcu_read_lock must be held on entry.
3481 */
3482static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3483 struct rps_dev_flow **rflowp)
3484{
3485 const struct rps_sock_flow_table *sock_flow_table;
3486 struct netdev_rx_queue *rxqueue = dev->_rx;
3487 struct rps_dev_flow_table *flow_table;
3488 struct rps_map *map;
3489 int cpu = -1;
3490 u32 tcpu;
3491 u32 hash;
3492
3493 if (skb_rx_queue_recorded(skb)) {
3494 u16 index = skb_get_rx_queue(skb);
3495
3496 if (unlikely(index >= dev->real_num_rx_queues)) {
3497 WARN_ONCE(dev->real_num_rx_queues > 1,
3498 "%s received packet on queue %u, but number "
3499 "of RX queues is %u\n",
3500 dev->name, index, dev->real_num_rx_queues);
3501 goto done;
3502 }
3503 rxqueue += index;
3504 }
3505
3506 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3507
3508 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3509 map = rcu_dereference(rxqueue->rps_map);
3510 if (!flow_table && !map)
3511 goto done;
3512
3513 skb_reset_network_header(skb);
3514 hash = skb_get_hash(skb);
3515 if (!hash)
3516 goto done;
3517
3518 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3519 if (flow_table && sock_flow_table) {
3520 struct rps_dev_flow *rflow;
3521 u32 next_cpu;
3522 u32 ident;
3523
3524 /* First check into global flow table if there is a match */
3525 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3526 if ((ident ^ hash) & ~rps_cpu_mask)
3527 goto try_rps;
3528
3529 next_cpu = ident & rps_cpu_mask;
3530
3531 /* OK, now we know there is a match,
3532 * we can look at the local (per receive queue) flow table
3533 */
3534 rflow = &flow_table->flows[hash & flow_table->mask];
3535 tcpu = rflow->cpu;
3536
3537 /*
3538 * If the desired CPU (where last recvmsg was done) is
3539 * different from current CPU (one in the rx-queue flow
3540 * table entry), switch if one of the following holds:
3541 * - Current CPU is unset (>= nr_cpu_ids).
3542 * - Current CPU is offline.
3543 * - The current CPU's queue tail has advanced beyond the
3544 * last packet that was enqueued using this table entry.
3545 * This guarantees that all previous packets for the flow
3546 * have been dequeued, thus preserving in order delivery.
3547 */
3548 if (unlikely(tcpu != next_cpu) &&
3549 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3550 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3551 rflow->last_qtail)) >= 0)) {
3552 tcpu = next_cpu;
3553 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3554 }
3555
3556 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3557 *rflowp = rflow;
3558 cpu = tcpu;
3559 goto done;
3560 }
3561 }
3562
3563try_rps:
3564
3565 if (map) {
3566 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3567 if (cpu_online(tcpu)) {
3568 cpu = tcpu;
3569 goto done;
3570 }
3571 }
3572
3573done:
3574 return cpu;
3575}
3576
3577#ifdef CONFIG_RFS_ACCEL
3578
3579/**
3580 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3581 * @dev: Device on which the filter was set
3582 * @rxq_index: RX queue index
3583 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3584 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3585 *
3586 * Drivers that implement ndo_rx_flow_steer() should periodically call
3587 * this function for each installed filter and remove the filters for
3588 * which it returns %true.
3589 */
3590bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3591 u32 flow_id, u16 filter_id)
3592{
3593 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3594 struct rps_dev_flow_table *flow_table;
3595 struct rps_dev_flow *rflow;
3596 bool expire = true;
3597 unsigned int cpu;
3598
3599 rcu_read_lock();
3600 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3601 if (flow_table && flow_id <= flow_table->mask) {
3602 rflow = &flow_table->flows[flow_id];
3603 cpu = ACCESS_ONCE(rflow->cpu);
3604 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3605 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3606 rflow->last_qtail) <
3607 (int)(10 * flow_table->mask)))
3608 expire = false;
3609 }
3610 rcu_read_unlock();
3611 return expire;
3612}
3613EXPORT_SYMBOL(rps_may_expire_flow);
3614
3615#endif /* CONFIG_RFS_ACCEL */
3616
3617/* Called from hardirq (IPI) context */
3618static void rps_trigger_softirq(void *data)
3619{
3620 struct softnet_data *sd = data;
3621
3622 ____napi_schedule(sd, &sd->backlog);
3623 sd->received_rps++;
3624}
3625
3626#endif /* CONFIG_RPS */
3627
3628/*
3629 * Check if this softnet_data structure is another cpu one
3630 * If yes, queue it to our IPI list and return 1
3631 * If no, return 0
3632 */
3633static int rps_ipi_queued(struct softnet_data *sd)
3634{
3635#ifdef CONFIG_RPS
3636 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3637
3638 if (sd != mysd) {
3639 sd->rps_ipi_next = mysd->rps_ipi_list;
3640 mysd->rps_ipi_list = sd;
3641
3642 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3643 return 1;
3644 }
3645#endif /* CONFIG_RPS */
3646 return 0;
3647}
3648
3649#ifdef CONFIG_NET_FLOW_LIMIT
3650int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3651#endif
3652
3653static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3654{
3655#ifdef CONFIG_NET_FLOW_LIMIT
3656 struct sd_flow_limit *fl;
3657 struct softnet_data *sd;
3658 unsigned int old_flow, new_flow;
3659
3660 if (qlen < (netdev_max_backlog >> 1))
3661 return false;
3662
3663 sd = this_cpu_ptr(&softnet_data);
3664
3665 rcu_read_lock();
3666 fl = rcu_dereference(sd->flow_limit);
3667 if (fl) {
3668 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3669 old_flow = fl->history[fl->history_head];
3670 fl->history[fl->history_head] = new_flow;
3671
3672 fl->history_head++;
3673 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3674
3675 if (likely(fl->buckets[old_flow]))
3676 fl->buckets[old_flow]--;
3677
3678 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3679 fl->count++;
3680 rcu_read_unlock();
3681 return true;
3682 }
3683 }
3684 rcu_read_unlock();
3685#endif
3686 return false;
3687}
3688
3689/*
3690 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3691 * queue (may be a remote CPU queue).
3692 */
3693static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3694 unsigned int *qtail)
3695{
3696 struct softnet_data *sd;
3697 unsigned long flags;
3698 unsigned int qlen;
3699
3700 sd = &per_cpu(softnet_data, cpu);
3701
3702 local_irq_save(flags);
3703
3704 rps_lock(sd);
3705 if (!netif_running(skb->dev))
3706 goto drop;
3707 qlen = skb_queue_len(&sd->input_pkt_queue);
3708 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3709 if (qlen) {
3710enqueue:
3711 __skb_queue_tail(&sd->input_pkt_queue, skb);
3712 input_queue_tail_incr_save(sd, qtail);
3713 rps_unlock(sd);
3714 local_irq_restore(flags);
3715 return NET_RX_SUCCESS;
3716 }
3717
3718 /* Schedule NAPI for backlog device
3719 * We can use non atomic operation since we own the queue lock
3720 */
3721 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3722 if (!rps_ipi_queued(sd))
3723 ____napi_schedule(sd, &sd->backlog);
3724 }
3725 goto enqueue;
3726 }
3727
3728drop:
3729 sd->dropped++;
3730 rps_unlock(sd);
3731
3732 local_irq_restore(flags);
3733
3734 atomic_long_inc(&skb->dev->rx_dropped);
3735 kfree_skb(skb);
3736 return NET_RX_DROP;
3737}
3738
3739static int netif_rx_internal(struct sk_buff *skb)
3740{
3741 int ret;
3742
3743 net_timestamp_check(netdev_tstamp_prequeue, skb);
3744
3745 trace_netif_rx(skb);
3746#ifdef CONFIG_RPS
3747 if (static_key_false(&rps_needed)) {
3748 struct rps_dev_flow voidflow, *rflow = &voidflow;
3749 int cpu;
3750
3751 preempt_disable();
3752 rcu_read_lock();
3753
3754 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3755 if (cpu < 0)
3756 cpu = smp_processor_id();
3757
3758 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3759
3760 rcu_read_unlock();
3761 preempt_enable();
3762 } else
3763#endif
3764 {
3765 unsigned int qtail;
3766 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3767 put_cpu();
3768 }
3769 return ret;
3770}
3771
3772/**
3773 * netif_rx - post buffer to the network code
3774 * @skb: buffer to post
3775 *
3776 * This function receives a packet from a device driver and queues it for
3777 * the upper (protocol) levels to process. It always succeeds. The buffer
3778 * may be dropped during processing for congestion control or by the
3779 * protocol layers.
3780 *
3781 * return values:
3782 * NET_RX_SUCCESS (no congestion)
3783 * NET_RX_DROP (packet was dropped)
3784 *
3785 */
3786
3787int netif_rx(struct sk_buff *skb)
3788{
3789 trace_netif_rx_entry(skb);
3790
3791 return netif_rx_internal(skb);
3792}
3793EXPORT_SYMBOL(netif_rx);
3794
3795int netif_rx_ni(struct sk_buff *skb)
3796{
3797 int err;
3798
3799 trace_netif_rx_ni_entry(skb);
3800
3801 preempt_disable();
3802 err = netif_rx_internal(skb);
3803 if (local_softirq_pending())
3804 do_softirq();
3805 preempt_enable();
3806
3807 return err;
3808}
3809EXPORT_SYMBOL(netif_rx_ni);
3810
3811static void net_tx_action(struct softirq_action *h)
3812{
3813 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3814
3815 if (sd->completion_queue) {
3816 struct sk_buff *clist;
3817
3818 local_irq_disable();
3819 clist = sd->completion_queue;
3820 sd->completion_queue = NULL;
3821 local_irq_enable();
3822
3823 while (clist) {
3824 struct sk_buff *skb = clist;
3825 clist = clist->next;
3826
3827 WARN_ON(atomic_read(&skb->users));
3828 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3829 trace_consume_skb(skb);
3830 else
3831 trace_kfree_skb(skb, net_tx_action);
3832
3833 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3834 __kfree_skb(skb);
3835 else
3836 __kfree_skb_defer(skb);
3837 }
3838
3839 __kfree_skb_flush();
3840 }
3841
3842 if (sd->output_queue) {
3843 struct Qdisc *head;
3844
3845 local_irq_disable();
3846 head = sd->output_queue;
3847 sd->output_queue = NULL;
3848 sd->output_queue_tailp = &sd->output_queue;
3849 local_irq_enable();
3850
3851 while (head) {
3852 struct Qdisc *q = head;
3853 spinlock_t *root_lock;
3854
3855 head = head->next_sched;
3856
3857 root_lock = qdisc_lock(q);
3858 if (spin_trylock(root_lock)) {
3859 smp_mb__before_atomic();
3860 clear_bit(__QDISC_STATE_SCHED,
3861 &q->state);
3862 qdisc_run(q);
3863 spin_unlock(root_lock);
3864 } else {
3865 if (!test_bit(__QDISC_STATE_DEACTIVATED,
3866 &q->state)) {
3867 __netif_reschedule(q);
3868 } else {
3869 smp_mb__before_atomic();
3870 clear_bit(__QDISC_STATE_SCHED,
3871 &q->state);
3872 }
3873 }
3874 }
3875 }
3876}
3877
3878#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3879 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3880/* This hook is defined here for ATM LANE */
3881int (*br_fdb_test_addr_hook)(struct net_device *dev,
3882 unsigned char *addr) __read_mostly;
3883EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3884#endif
3885
3886static inline struct sk_buff *
3887sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3888 struct net_device *orig_dev)
3889{
3890#ifdef CONFIG_NET_CLS_ACT
3891 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3892 struct tcf_result cl_res;
3893
3894 /* If there's at least one ingress present somewhere (so
3895 * we get here via enabled static key), remaining devices
3896 * that are not configured with an ingress qdisc will bail
3897 * out here.
3898 */
3899 if (!cl)
3900 return skb;
3901 if (*pt_prev) {
3902 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3903 *pt_prev = NULL;
3904 }
3905
3906 qdisc_skb_cb(skb)->pkt_len = skb->len;
3907 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3908 qdisc_bstats_cpu_update(cl->q, skb);
3909
3910 switch (tc_classify(skb, cl, &cl_res, false)) {
3911 case TC_ACT_OK:
3912 case TC_ACT_RECLASSIFY:
3913 skb->tc_index = TC_H_MIN(cl_res.classid);
3914 break;
3915 case TC_ACT_SHOT:
3916 qdisc_qstats_cpu_drop(cl->q);
3917 case TC_ACT_STOLEN:
3918 case TC_ACT_QUEUED:
3919 kfree_skb(skb);
3920 return NULL;
3921 case TC_ACT_REDIRECT:
3922 /* skb_mac_header check was done by cls/act_bpf, so
3923 * we can safely push the L2 header back before
3924 * redirecting to another netdev
3925 */
3926 __skb_push(skb, skb->mac_len);
3927 skb_do_redirect(skb);
3928 return NULL;
3929 default:
3930 break;
3931 }
3932#endif /* CONFIG_NET_CLS_ACT */
3933 return skb;
3934}
3935
3936/**
3937 * netdev_rx_handler_register - register receive handler
3938 * @dev: device to register a handler for
3939 * @rx_handler: receive handler to register
3940 * @rx_handler_data: data pointer that is used by rx handler
3941 *
3942 * Register a receive handler for a device. This handler will then be
3943 * called from __netif_receive_skb. A negative errno code is returned
3944 * on a failure.
3945 *
3946 * The caller must hold the rtnl_mutex.
3947 *
3948 * For a general description of rx_handler, see enum rx_handler_result.
3949 */
3950int netdev_rx_handler_register(struct net_device *dev,
3951 rx_handler_func_t *rx_handler,
3952 void *rx_handler_data)
3953{
3954 ASSERT_RTNL();
3955
3956 if (dev->rx_handler)
3957 return -EBUSY;
3958
3959 /* Note: rx_handler_data must be set before rx_handler */
3960 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3961 rcu_assign_pointer(dev->rx_handler, rx_handler);
3962
3963 return 0;
3964}
3965EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3966
3967/**
3968 * netdev_rx_handler_unregister - unregister receive handler
3969 * @dev: device to unregister a handler from
3970 *
3971 * Unregister a receive handler from a device.
3972 *
3973 * The caller must hold the rtnl_mutex.
3974 */
3975void netdev_rx_handler_unregister(struct net_device *dev)
3976{
3977
3978 ASSERT_RTNL();
3979 RCU_INIT_POINTER(dev->rx_handler, NULL);
3980 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3981 * section has a guarantee to see a non NULL rx_handler_data
3982 * as well.
3983 */
3984 synchronize_net();
3985 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
3986}
3987EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3988
3989/*
3990 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3991 * the special handling of PFMEMALLOC skbs.
3992 */
3993static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3994{
3995 switch (skb->protocol) {
3996 case htons(ETH_P_ARP):
3997 case htons(ETH_P_IP):
3998 case htons(ETH_P_IPV6):
3999 case htons(ETH_P_8021Q):
4000 case htons(ETH_P_8021AD):
4001 return true;
4002 default:
4003 return false;
4004 }
4005}
4006
4007static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4008 int *ret, struct net_device *orig_dev)
4009{
4010#ifdef CONFIG_NETFILTER_INGRESS
4011 if (nf_hook_ingress_active(skb)) {
4012 if (*pt_prev) {
4013 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4014 *pt_prev = NULL;
4015 }
4016
4017 return nf_hook_ingress(skb);
4018 }
4019#endif /* CONFIG_NETFILTER_INGRESS */
4020 return 0;
4021}
4022
4023static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4024{
4025 struct packet_type *ptype, *pt_prev;
4026 rx_handler_func_t *rx_handler;
4027 struct net_device *orig_dev;
4028 bool deliver_exact = false;
4029 int ret = NET_RX_DROP;
4030 __be16 type;
4031
4032 net_timestamp_check(!netdev_tstamp_prequeue, skb);
4033
4034 trace_netif_receive_skb(skb);
4035
4036 orig_dev = skb->dev;
4037
4038 skb_reset_network_header(skb);
4039 if (!skb_transport_header_was_set(skb))
4040 skb_reset_transport_header(skb);
4041 skb_reset_mac_len(skb);
4042
4043 pt_prev = NULL;
4044
4045another_round:
4046 skb->skb_iif = skb->dev->ifindex;
4047
4048 __this_cpu_inc(softnet_data.processed);
4049
4050 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4051 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4052 skb = skb_vlan_untag(skb);
4053 if (unlikely(!skb))
4054 goto out;
4055 }
4056
4057#ifdef CONFIG_NET_CLS_ACT
4058 if (skb->tc_verd & TC_NCLS) {
4059 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
4060 goto ncls;
4061 }
4062#endif
4063
4064 if (pfmemalloc)
4065 goto skip_taps;
4066
4067 list_for_each_entry_rcu(ptype, &ptype_all, list) {
4068 if (pt_prev)
4069 ret = deliver_skb(skb, pt_prev, orig_dev);
4070 pt_prev = ptype;
4071 }
4072
4073 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4074 if (pt_prev)
4075 ret = deliver_skb(skb, pt_prev, orig_dev);
4076 pt_prev = ptype;
4077 }
4078
4079skip_taps:
4080#ifdef CONFIG_NET_INGRESS
4081 if (static_key_false(&ingress_needed)) {
4082 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4083 if (!skb)
4084 goto out;
4085
4086 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4087 goto out;
4088 }
4089#endif
4090#ifdef CONFIG_NET_CLS_ACT
4091 skb->tc_verd = 0;
4092ncls:
4093#endif
4094 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4095 goto drop;
4096
4097 if (skb_vlan_tag_present(skb)) {
4098 if (pt_prev) {
4099 ret = deliver_skb(skb, pt_prev, orig_dev);
4100 pt_prev = NULL;
4101 }
4102 if (vlan_do_receive(&skb))
4103 goto another_round;
4104 else if (unlikely(!skb))
4105 goto out;
4106 }
4107
4108 rx_handler = rcu_dereference(skb->dev->rx_handler);
4109 if (rx_handler) {
4110 if (pt_prev) {
4111 ret = deliver_skb(skb, pt_prev, orig_dev);
4112 pt_prev = NULL;
4113 }
4114 switch (rx_handler(&skb)) {
4115 case RX_HANDLER_CONSUMED:
4116 ret = NET_RX_SUCCESS;
4117 goto out;
4118 case RX_HANDLER_ANOTHER:
4119 goto another_round;
4120 case RX_HANDLER_EXACT:
4121 deliver_exact = true;
4122 case RX_HANDLER_PASS:
4123 break;
4124 default:
4125 BUG();
4126 }
4127 }
4128
4129 if (unlikely(skb_vlan_tag_present(skb))) {
4130 if (skb_vlan_tag_get_id(skb))
4131 skb->pkt_type = PACKET_OTHERHOST;
4132 /* Note: we might in the future use prio bits
4133 * and set skb->priority like in vlan_do_receive()
4134 * For the time being, just ignore Priority Code Point
4135 */
4136 skb->vlan_tci = 0;
4137 }
4138
4139 type = skb->protocol;
4140
4141 /* deliver only exact match when indicated */
4142 if (likely(!deliver_exact)) {
4143 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4144 &ptype_base[ntohs(type) &
4145 PTYPE_HASH_MASK]);
4146 }
4147
4148 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4149 &orig_dev->ptype_specific);
4150
4151 if (unlikely(skb->dev != orig_dev)) {
4152 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4153 &skb->dev->ptype_specific);
4154 }
4155
4156 if (pt_prev) {
4157 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
4158 goto drop;
4159 else
4160 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4161 } else {
4162drop:
4163 if (!deliver_exact)
4164 atomic_long_inc(&skb->dev->rx_dropped);
4165 else
4166 atomic_long_inc(&skb->dev->rx_nohandler);
4167 kfree_skb(skb);
4168 /* Jamal, now you will not able to escape explaining
4169 * me how you were going to use this. :-)
4170 */
4171 ret = NET_RX_DROP;
4172 }
4173
4174out:
4175 return ret;
4176}
4177
4178static int __netif_receive_skb(struct sk_buff *skb)
4179{
4180 int ret;
4181
4182 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4183 unsigned long pflags = current->flags;
4184
4185 /*
4186 * PFMEMALLOC skbs are special, they should
4187 * - be delivered to SOCK_MEMALLOC sockets only
4188 * - stay away from userspace
4189 * - have bounded memory usage
4190 *
4191 * Use PF_MEMALLOC as this saves us from propagating the allocation
4192 * context down to all allocation sites.
4193 */
4194 current->flags |= PF_MEMALLOC;
4195 ret = __netif_receive_skb_core(skb, true);
4196 tsk_restore_flags(current, pflags, PF_MEMALLOC);
4197 } else
4198 ret = __netif_receive_skb_core(skb, false);
4199
4200 return ret;
4201}
4202
4203static int netif_receive_skb_internal(struct sk_buff *skb)
4204{
4205 int ret;
4206
4207 net_timestamp_check(netdev_tstamp_prequeue, skb);
4208
4209 if (skb_defer_rx_timestamp(skb))
4210 return NET_RX_SUCCESS;
4211
4212 rcu_read_lock();
4213
4214#ifdef CONFIG_RPS
4215 if (static_key_false(&rps_needed)) {
4216 struct rps_dev_flow voidflow, *rflow = &voidflow;
4217 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4218
4219 if (cpu >= 0) {
4220 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4221 rcu_read_unlock();
4222 return ret;
4223 }
4224 }
4225#endif
4226 ret = __netif_receive_skb(skb);
4227 rcu_read_unlock();
4228 return ret;
4229}
4230
4231/**
4232 * netif_receive_skb - process receive buffer from network
4233 * @skb: buffer to process
4234 *
4235 * netif_receive_skb() is the main receive data processing function.
4236 * It always succeeds. The buffer may be dropped during processing
4237 * for congestion control or by the protocol layers.
4238 *
4239 * This function may only be called from softirq context and interrupts
4240 * should be enabled.
4241 *
4242 * Return values (usually ignored):
4243 * NET_RX_SUCCESS: no congestion
4244 * NET_RX_DROP: packet was dropped
4245 */
4246int netif_receive_skb(struct sk_buff *skb)
4247{
4248 trace_netif_receive_skb_entry(skb);
4249
4250 return netif_receive_skb_internal(skb);
4251}
4252EXPORT_SYMBOL(netif_receive_skb);
4253
4254/* Network device is going away, flush any packets still pending
4255 * Called with irqs disabled.
4256 */
4257static void flush_backlog(void *arg)
4258{
4259 struct net_device *dev = arg;
4260 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4261 struct sk_buff *skb, *tmp;
4262
4263 rps_lock(sd);
4264 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4265 if (skb->dev == dev) {
4266 __skb_unlink(skb, &sd->input_pkt_queue);
4267 kfree_skb(skb);
4268 input_queue_head_incr(sd);
4269 }
4270 }
4271 rps_unlock(sd);
4272
4273 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4274 if (skb->dev == dev) {
4275 __skb_unlink(skb, &sd->process_queue);
4276 kfree_skb(skb);
4277 input_queue_head_incr(sd);
4278 }
4279 }
4280}
4281
4282static int napi_gro_complete(struct sk_buff *skb)
4283{
4284 struct packet_offload *ptype;
4285 __be16 type = skb->protocol;
4286 struct list_head *head = &offload_base;
4287 int err = -ENOENT;
4288
4289 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4290
4291 if (NAPI_GRO_CB(skb)->count == 1) {
4292 skb_shinfo(skb)->gso_size = 0;
4293 goto out;
4294 }
4295
4296 rcu_read_lock();
4297 list_for_each_entry_rcu(ptype, head, list) {
4298 if (ptype->type != type || !ptype->callbacks.gro_complete)
4299 continue;
4300
4301 err = ptype->callbacks.gro_complete(skb, 0);
4302 break;
4303 }
4304 rcu_read_unlock();
4305
4306 if (err) {
4307 WARN_ON(&ptype->list == head);
4308 kfree_skb(skb);
4309 return NET_RX_SUCCESS;
4310 }
4311
4312out:
4313 return netif_receive_skb_internal(skb);
4314}
4315
4316/* napi->gro_list contains packets ordered by age.
4317 * youngest packets at the head of it.
4318 * Complete skbs in reverse order to reduce latencies.
4319 */
4320void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4321{
4322 struct sk_buff *skb, *prev = NULL;
4323
4324 /* scan list and build reverse chain */
4325 for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4326 skb->prev = prev;
4327 prev = skb;
4328 }
4329
4330 for (skb = prev; skb; skb = prev) {
4331 skb->next = NULL;
4332
4333 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4334 return;
4335
4336 prev = skb->prev;
4337 napi_gro_complete(skb);
4338 napi->gro_count--;
4339 }
4340
4341 napi->gro_list = NULL;
4342}
4343EXPORT_SYMBOL(napi_gro_flush);
4344
4345static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4346{
4347 struct sk_buff *p;
4348 unsigned int maclen = skb->dev->hard_header_len;
4349 u32 hash = skb_get_hash_raw(skb);
4350
4351 for (p = napi->gro_list; p; p = p->next) {
4352 unsigned long diffs;
4353
4354 NAPI_GRO_CB(p)->flush = 0;
4355
4356 if (hash != skb_get_hash_raw(p)) {
4357 NAPI_GRO_CB(p)->same_flow = 0;
4358 continue;
4359 }
4360
4361 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4362 diffs |= p->vlan_tci ^ skb->vlan_tci;
4363 diffs |= skb_metadata_dst_cmp(p, skb);
4364 if (maclen == ETH_HLEN)
4365 diffs |= compare_ether_header(skb_mac_header(p),
4366 skb_mac_header(skb));
4367 else if (!diffs)
4368 diffs = memcmp(skb_mac_header(p),
4369 skb_mac_header(skb),
4370 maclen);
4371 NAPI_GRO_CB(p)->same_flow = !diffs;
4372 }
4373}
4374
4375static void skb_gro_reset_offset(struct sk_buff *skb)
4376{
4377 const struct skb_shared_info *pinfo = skb_shinfo(skb);
4378 const skb_frag_t *frag0 = &pinfo->frags[0];
4379
4380 NAPI_GRO_CB(skb)->data_offset = 0;
4381 NAPI_GRO_CB(skb)->frag0 = NULL;
4382 NAPI_GRO_CB(skb)->frag0_len = 0;
4383
4384 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4385 pinfo->nr_frags &&
4386 !PageHighMem(skb_frag_page(frag0))) {
4387 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4388 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
4389 }
4390}
4391
4392static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4393{
4394 struct skb_shared_info *pinfo = skb_shinfo(skb);
4395
4396 BUG_ON(skb->end - skb->tail < grow);
4397
4398 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4399
4400 skb->data_len -= grow;
4401 skb->tail += grow;
4402
4403 pinfo->frags[0].page_offset += grow;
4404 skb_frag_size_sub(&pinfo->frags[0], grow);
4405
4406 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4407 skb_frag_unref(skb, 0);
4408 memmove(pinfo->frags, pinfo->frags + 1,
4409 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4410 }
4411}
4412
4413static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4414{
4415 struct sk_buff **pp = NULL;
4416 struct packet_offload *ptype;
4417 __be16 type = skb->protocol;
4418 struct list_head *head = &offload_base;
4419 int same_flow;
4420 enum gro_result ret;
4421 int grow;
4422
4423 if (!(skb->dev->features & NETIF_F_GRO))
4424 goto normal;
4425
4426 if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
4427 goto normal;
4428
4429 gro_list_prepare(napi, skb);
4430
4431 rcu_read_lock();
4432 list_for_each_entry_rcu(ptype, head, list) {
4433 if (ptype->type != type || !ptype->callbacks.gro_receive)
4434 continue;
4435
4436 skb_set_network_header(skb, skb_gro_offset(skb));
4437 skb_reset_mac_len(skb);
4438 NAPI_GRO_CB(skb)->same_flow = 0;
4439 NAPI_GRO_CB(skb)->flush = 0;
4440 NAPI_GRO_CB(skb)->free = 0;
4441 NAPI_GRO_CB(skb)->encap_mark = 0;
4442 NAPI_GRO_CB(skb)->is_fou = 0;
4443 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4444
4445 /* Setup for GRO checksum validation */
4446 switch (skb->ip_summed) {
4447 case CHECKSUM_COMPLETE:
4448 NAPI_GRO_CB(skb)->csum = skb->csum;
4449 NAPI_GRO_CB(skb)->csum_valid = 1;
4450 NAPI_GRO_CB(skb)->csum_cnt = 0;
4451 break;
4452 case CHECKSUM_UNNECESSARY:
4453 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4454 NAPI_GRO_CB(skb)->csum_valid = 0;
4455 break;
4456 default:
4457 NAPI_GRO_CB(skb)->csum_cnt = 0;
4458 NAPI_GRO_CB(skb)->csum_valid = 0;
4459 }
4460
4461 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4462 break;
4463 }
4464 rcu_read_unlock();
4465
4466 if (&ptype->list == head)
4467 goto normal;
4468
4469 same_flow = NAPI_GRO_CB(skb)->same_flow;
4470 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4471
4472 if (pp) {
4473 struct sk_buff *nskb = *pp;
4474
4475 *pp = nskb->next;
4476 nskb->next = NULL;
4477 napi_gro_complete(nskb);
4478 napi->gro_count--;
4479 }
4480
4481 if (same_flow)
4482 goto ok;
4483
4484 if (NAPI_GRO_CB(skb)->flush)
4485 goto normal;
4486
4487 if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4488 struct sk_buff *nskb = napi->gro_list;
4489
4490 /* locate the end of the list to select the 'oldest' flow */
4491 while (nskb->next) {
4492 pp = &nskb->next;
4493 nskb = *pp;
4494 }
4495 *pp = NULL;
4496 nskb->next = NULL;
4497 napi_gro_complete(nskb);
4498 } else {
4499 napi->gro_count++;
4500 }
4501 NAPI_GRO_CB(skb)->count = 1;
4502 NAPI_GRO_CB(skb)->age = jiffies;
4503 NAPI_GRO_CB(skb)->last = skb;
4504 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4505 skb->next = napi->gro_list;
4506 napi->gro_list = skb;
4507 ret = GRO_HELD;
4508
4509pull:
4510 grow = skb_gro_offset(skb) - skb_headlen(skb);
4511 if (grow > 0)
4512 gro_pull_from_frag0(skb, grow);
4513ok:
4514 return ret;
4515
4516normal:
4517 ret = GRO_NORMAL;
4518 goto pull;
4519}
4520
4521struct packet_offload *gro_find_receive_by_type(__be16 type)
4522{
4523 struct list_head *offload_head = &offload_base;
4524 struct packet_offload *ptype;
4525
4526 list_for_each_entry_rcu(ptype, offload_head, list) {
4527 if (ptype->type != type || !ptype->callbacks.gro_receive)
4528 continue;
4529 return ptype;
4530 }
4531 return NULL;
4532}
4533EXPORT_SYMBOL(gro_find_receive_by_type);
4534
4535struct packet_offload *gro_find_complete_by_type(__be16 type)
4536{
4537 struct list_head *offload_head = &offload_base;
4538 struct packet_offload *ptype;
4539
4540 list_for_each_entry_rcu(ptype, offload_head, list) {
4541 if (ptype->type != type || !ptype->callbacks.gro_complete)
4542 continue;
4543 return ptype;
4544 }
4545 return NULL;
4546}
4547EXPORT_SYMBOL(gro_find_complete_by_type);
4548
4549static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4550{
4551 switch (ret) {
4552 case GRO_NORMAL:
4553 if (netif_receive_skb_internal(skb))
4554 ret = GRO_DROP;
4555 break;
4556
4557 case GRO_DROP:
4558 kfree_skb(skb);
4559 break;
4560
4561 case GRO_MERGED_FREE:
4562 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) {
4563 skb_dst_drop(skb);
4564 kmem_cache_free(skbuff_head_cache, skb);
4565 } else {
4566 __kfree_skb(skb);
4567 }
4568 break;
4569
4570 case GRO_HELD:
4571 case GRO_MERGED:
4572 break;
4573 }
4574
4575 return ret;
4576}
4577
4578gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4579{
4580 skb_mark_napi_id(skb, napi);
4581 trace_napi_gro_receive_entry(skb);
4582
4583 skb_gro_reset_offset(skb);
4584
4585 return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4586}
4587EXPORT_SYMBOL(napi_gro_receive);
4588
4589static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4590{
4591 if (unlikely(skb->pfmemalloc)) {
4592 consume_skb(skb);
4593 return;
4594 }
4595 __skb_pull(skb, skb_headlen(skb));
4596 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4597 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4598 skb->vlan_tci = 0;
4599 skb->dev = napi->dev;
4600 skb->skb_iif = 0;
4601 skb->encapsulation = 0;
4602 skb_shinfo(skb)->gso_type = 0;
4603 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4604
4605 napi->skb = skb;
4606}
4607
4608struct sk_buff *napi_get_frags(struct napi_struct *napi)
4609{
4610 struct sk_buff *skb = napi->skb;
4611
4612 if (!skb) {
4613 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
4614 if (skb) {
4615 napi->skb = skb;
4616 skb_mark_napi_id(skb, napi);
4617 }
4618 }
4619 return skb;
4620}
4621EXPORT_SYMBOL(napi_get_frags);
4622
4623static gro_result_t napi_frags_finish(struct napi_struct *napi,
4624 struct sk_buff *skb,
4625 gro_result_t ret)
4626{
4627 switch (ret) {
4628 case GRO_NORMAL:
4629 case GRO_HELD:
4630 __skb_push(skb, ETH_HLEN);
4631 skb->protocol = eth_type_trans(skb, skb->dev);
4632 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4633 ret = GRO_DROP;
4634 break;
4635
4636 case GRO_DROP:
4637 case GRO_MERGED_FREE:
4638 napi_reuse_skb(napi, skb);
4639 break;
4640
4641 case GRO_MERGED:
4642 break;
4643 }
4644
4645 return ret;
4646}
4647
4648/* Upper GRO stack assumes network header starts at gro_offset=0
4649 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4650 * We copy ethernet header into skb->data to have a common layout.
4651 */
4652static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4653{
4654 struct sk_buff *skb = napi->skb;
4655 const struct ethhdr *eth;
4656 unsigned int hlen = sizeof(*eth);
4657
4658 napi->skb = NULL;
4659
4660 skb_reset_mac_header(skb);
4661 skb_gro_reset_offset(skb);
4662
4663 eth = skb_gro_header_fast(skb, 0);
4664 if (unlikely(skb_gro_header_hard(skb, hlen))) {
4665 eth = skb_gro_header_slow(skb, hlen, 0);
4666 if (unlikely(!eth)) {
4667 napi_reuse_skb(napi, skb);
4668 return NULL;
4669 }
4670 } else {
4671 gro_pull_from_frag0(skb, hlen);
4672 NAPI_GRO_CB(skb)->frag0 += hlen;
4673 NAPI_GRO_CB(skb)->frag0_len -= hlen;
4674 }
4675 __skb_pull(skb, hlen);
4676
4677 /*
4678 * This works because the only protocols we care about don't require
4679 * special handling.
4680 * We'll fix it up properly in napi_frags_finish()
4681 */
4682 skb->protocol = eth->h_proto;
4683
4684 return skb;
4685}
4686
4687gro_result_t napi_gro_frags(struct napi_struct *napi)
4688{
4689 struct sk_buff *skb = napi_frags_skb(napi);
4690
4691 if (!skb)
4692 return GRO_DROP;
4693
4694 trace_napi_gro_frags_entry(skb);
4695
4696 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
4697}
4698EXPORT_SYMBOL(napi_gro_frags);
4699
4700/* Compute the checksum from gro_offset and return the folded value
4701 * after adding in any pseudo checksum.
4702 */
4703__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4704{
4705 __wsum wsum;
4706 __sum16 sum;
4707
4708 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4709
4710 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4711 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4712 if (likely(!sum)) {
4713 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4714 !skb->csum_complete_sw)
4715 netdev_rx_csum_fault(skb->dev);
4716 }
4717
4718 NAPI_GRO_CB(skb)->csum = wsum;
4719 NAPI_GRO_CB(skb)->csum_valid = 1;
4720
4721 return sum;
4722}
4723EXPORT_SYMBOL(__skb_gro_checksum_complete);
4724
4725/*
4726 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4727 * Note: called with local irq disabled, but exits with local irq enabled.
4728 */
4729static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4730{
4731#ifdef CONFIG_RPS
4732 struct softnet_data *remsd = sd->rps_ipi_list;
4733
4734 if (remsd) {
4735 sd->rps_ipi_list = NULL;
4736
4737 local_irq_enable();
4738
4739 /* Send pending IPI's to kick RPS processing on remote cpus. */
4740 while (remsd) {
4741 struct softnet_data *next = remsd->rps_ipi_next;
4742
4743 if (cpu_online(remsd->cpu))
4744 smp_call_function_single_async(remsd->cpu,
4745 &remsd->csd);
4746 remsd = next;
4747 }
4748 } else
4749#endif
4750 local_irq_enable();
4751}
4752
4753static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4754{
4755#ifdef CONFIG_RPS
4756 return sd->rps_ipi_list != NULL;
4757#else
4758 return false;
4759#endif
4760}
4761
4762static int process_backlog(struct napi_struct *napi, int quota)
4763{
4764 int work = 0;
4765 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4766
4767 /* Check if we have pending ipi, its better to send them now,
4768 * not waiting net_rx_action() end.
4769 */
4770 if (sd_has_rps_ipi_waiting(sd)) {
4771 local_irq_disable();
4772 net_rps_action_and_irq_enable(sd);
4773 }
4774
4775 napi->weight = weight_p;
4776 local_irq_disable();
4777 while (1) {
4778 struct sk_buff *skb;
4779
4780 while ((skb = __skb_dequeue(&sd->process_queue))) {
4781 rcu_read_lock();
4782 local_irq_enable();
4783 __netif_receive_skb(skb);
4784 rcu_read_unlock();
4785 local_irq_disable();
4786 input_queue_head_incr(sd);
4787 if (++work >= quota) {
4788 local_irq_enable();
4789 return work;
4790 }
4791 }
4792
4793 rps_lock(sd);
4794 if (skb_queue_empty(&sd->input_pkt_queue)) {
4795 /*
4796 * Inline a custom version of __napi_complete().
4797 * only current cpu owns and manipulates this napi,
4798 * and NAPI_STATE_SCHED is the only possible flag set
4799 * on backlog.
4800 * We can use a plain write instead of clear_bit(),
4801 * and we dont need an smp_mb() memory barrier.
4802 */
4803 napi->state = 0;
4804 rps_unlock(sd);
4805
4806 break;
4807 }
4808
4809 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4810 &sd->process_queue);
4811 rps_unlock(sd);
4812 }
4813 local_irq_enable();
4814
4815 return work;
4816}
4817
4818/**
4819 * __napi_schedule - schedule for receive
4820 * @n: entry to schedule
4821 *
4822 * The entry's receive function will be scheduled to run.
4823 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
4824 */
4825void __napi_schedule(struct napi_struct *n)
4826{
4827 unsigned long flags;
4828
4829 local_irq_save(flags);
4830 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4831 local_irq_restore(flags);
4832}
4833EXPORT_SYMBOL(__napi_schedule);
4834
4835/**
4836 * __napi_schedule_irqoff - schedule for receive
4837 * @n: entry to schedule
4838 *
4839 * Variant of __napi_schedule() assuming hard irqs are masked
4840 */
4841void __napi_schedule_irqoff(struct napi_struct *n)
4842{
4843 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4844}
4845EXPORT_SYMBOL(__napi_schedule_irqoff);
4846
4847void __napi_complete(struct napi_struct *n)
4848{
4849 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4850
4851 list_del_init(&n->poll_list);
4852 smp_mb__before_atomic();
4853 clear_bit(NAPI_STATE_SCHED, &n->state);
4854}
4855EXPORT_SYMBOL(__napi_complete);
4856
4857void napi_complete_done(struct napi_struct *n, int work_done)
4858{
4859 unsigned long flags;
4860
4861 /*
4862 * don't let napi dequeue from the cpu poll list
4863 * just in case its running on a different cpu
4864 */
4865 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4866 return;
4867
4868 if (n->gro_list) {
4869 unsigned long timeout = 0;
4870
4871 if (work_done)
4872 timeout = n->dev->gro_flush_timeout;
4873
4874 if (timeout)
4875 hrtimer_start(&n->timer, ns_to_ktime(timeout),
4876 HRTIMER_MODE_REL_PINNED);
4877 else
4878 napi_gro_flush(n, false);
4879 }
4880 if (likely(list_empty(&n->poll_list))) {
4881 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4882 } else {
4883 /* If n->poll_list is not empty, we need to mask irqs */
4884 local_irq_save(flags);
4885 __napi_complete(n);
4886 local_irq_restore(flags);
4887 }
4888}
4889EXPORT_SYMBOL(napi_complete_done);
4890
4891/* must be called under rcu_read_lock(), as we dont take a reference */
4892static struct napi_struct *napi_by_id(unsigned int napi_id)
4893{
4894 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4895 struct napi_struct *napi;
4896
4897 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4898 if (napi->napi_id == napi_id)
4899 return napi;
4900
4901 return NULL;
4902}
4903
4904#if defined(CONFIG_NET_RX_BUSY_POLL)
4905#define BUSY_POLL_BUDGET 8
4906bool sk_busy_loop(struct sock *sk, int nonblock)
4907{
4908 unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0;
4909 int (*busy_poll)(struct napi_struct *dev);
4910 struct napi_struct *napi;
4911 int rc = false;
4912
4913 rcu_read_lock();
4914
4915 napi = napi_by_id(sk->sk_napi_id);
4916 if (!napi)
4917 goto out;
4918
4919 /* Note: ndo_busy_poll method is optional in linux-4.5 */
4920 busy_poll = napi->dev->netdev_ops->ndo_busy_poll;
4921
4922 do {
4923 rc = 0;
4924 local_bh_disable();
4925 if (busy_poll) {
4926 rc = busy_poll(napi);
4927 } else if (napi_schedule_prep(napi)) {
4928 void *have = netpoll_poll_lock(napi);
4929
4930 if (test_bit(NAPI_STATE_SCHED, &napi->state)) {
4931 rc = napi->poll(napi, BUSY_POLL_BUDGET);
4932 trace_napi_poll(napi);
4933 if (rc == BUSY_POLL_BUDGET) {
4934 napi_complete_done(napi, rc);
4935 napi_schedule(napi);
4936 }
4937 }
4938 netpoll_poll_unlock(have);
4939 }
4940 if (rc > 0)
4941 NET_ADD_STATS_BH(sock_net(sk),
4942 LINUX_MIB_BUSYPOLLRXPACKETS, rc);
4943 local_bh_enable();
4944
4945 if (rc == LL_FLUSH_FAILED)
4946 break; /* permanent failure */
4947
4948 cpu_relax();
4949 } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) &&
4950 !need_resched() && !busy_loop_timeout(end_time));
4951
4952 rc = !skb_queue_empty(&sk->sk_receive_queue);
4953out:
4954 rcu_read_unlock();
4955 return rc;
4956}
4957EXPORT_SYMBOL(sk_busy_loop);
4958
4959#endif /* CONFIG_NET_RX_BUSY_POLL */
4960
4961void napi_hash_add(struct napi_struct *napi)
4962{
4963 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
4964 test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
4965 return;
4966
4967 spin_lock(&napi_hash_lock);
4968
4969 /* 0..NR_CPUS+1 range is reserved for sender_cpu use */
4970 do {
4971 if (unlikely(++napi_gen_id < NR_CPUS + 1))
4972 napi_gen_id = NR_CPUS + 1;
4973 } while (napi_by_id(napi_gen_id));
4974 napi->napi_id = napi_gen_id;
4975
4976 hlist_add_head_rcu(&napi->napi_hash_node,
4977 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4978
4979 spin_unlock(&napi_hash_lock);
4980}
4981EXPORT_SYMBOL_GPL(napi_hash_add);
4982
4983/* Warning : caller is responsible to make sure rcu grace period
4984 * is respected before freeing memory containing @napi
4985 */
4986bool napi_hash_del(struct napi_struct *napi)
4987{
4988 bool rcu_sync_needed = false;
4989
4990 spin_lock(&napi_hash_lock);
4991
4992 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
4993 rcu_sync_needed = true;
4994 hlist_del_rcu(&napi->napi_hash_node);
4995 }
4996 spin_unlock(&napi_hash_lock);
4997 return rcu_sync_needed;
4998}
4999EXPORT_SYMBOL_GPL(napi_hash_del);
5000
5001static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5002{
5003 struct napi_struct *napi;
5004
5005 napi = container_of(timer, struct napi_struct, timer);
5006 if (napi->gro_list)
5007 napi_schedule(napi);
5008
5009 return HRTIMER_NORESTART;
5010}
5011
5012void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5013 int (*poll)(struct napi_struct *, int), int weight)
5014{
5015 INIT_LIST_HEAD(&napi->poll_list);
5016 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5017 napi->timer.function = napi_watchdog;
5018 napi->gro_count = 0;
5019 napi->gro_list = NULL;
5020 napi->skb = NULL;
5021 napi->poll = poll;
5022 if (weight > NAPI_POLL_WEIGHT)
5023 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5024 weight, dev->name);
5025 napi->weight = weight;
5026 list_add(&napi->dev_list, &dev->napi_list);
5027 napi->dev = dev;
5028#ifdef CONFIG_NETPOLL
5029 spin_lock_init(&napi->poll_lock);
5030 napi->poll_owner = -1;
5031#endif
5032 set_bit(NAPI_STATE_SCHED, &napi->state);
5033 napi_hash_add(napi);
5034}
5035EXPORT_SYMBOL(netif_napi_add);
5036
5037void napi_disable(struct napi_struct *n)
5038{
5039 might_sleep();
5040 set_bit(NAPI_STATE_DISABLE, &n->state);
5041
5042 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5043 msleep(1);
5044 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5045 msleep(1);
5046
5047 hrtimer_cancel(&n->timer);
5048
5049 clear_bit(NAPI_STATE_DISABLE, &n->state);
5050}
5051EXPORT_SYMBOL(napi_disable);
5052
5053/* Must be called in process context */
5054void netif_napi_del(struct napi_struct *napi)
5055{
5056 might_sleep();
5057 if (napi_hash_del(napi))
5058 synchronize_net();
5059 list_del_init(&napi->dev_list);
5060 napi_free_frags(napi);
5061
5062 kfree_skb_list(napi->gro_list);
5063 napi->gro_list = NULL;
5064 napi->gro_count = 0;
5065}
5066EXPORT_SYMBOL(netif_napi_del);
5067
5068static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5069{
5070 void *have;
5071 int work, weight;
5072
5073 list_del_init(&n->poll_list);
5074
5075 have = netpoll_poll_lock(n);
5076
5077 weight = n->weight;
5078
5079 /* This NAPI_STATE_SCHED test is for avoiding a race
5080 * with netpoll's poll_napi(). Only the entity which
5081 * obtains the lock and sees NAPI_STATE_SCHED set will
5082 * actually make the ->poll() call. Therefore we avoid
5083 * accidentally calling ->poll() when NAPI is not scheduled.
5084 */
5085 work = 0;
5086 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5087 work = n->poll(n, weight);
5088 trace_napi_poll(n);
5089 }
5090
5091 WARN_ON_ONCE(work > weight);
5092
5093 if (likely(work < weight))
5094 goto out_unlock;
5095
5096 /* Drivers must not modify the NAPI state if they
5097 * consume the entire weight. In such cases this code
5098 * still "owns" the NAPI instance and therefore can
5099 * move the instance around on the list at-will.
5100 */
5101 if (unlikely(napi_disable_pending(n))) {
5102 napi_complete(n);
5103 goto out_unlock;
5104 }
5105
5106 if (n->gro_list) {
5107 /* flush too old packets
5108 * If HZ < 1000, flush all packets.
5109 */
5110 napi_gro_flush(n, HZ >= 1000);
5111 }
5112
5113 /* Some drivers may have called napi_schedule
5114 * prior to exhausting their budget.
5115 */
5116 if (unlikely(!list_empty(&n->poll_list))) {
5117 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5118 n->dev ? n->dev->name : "backlog");
5119 goto out_unlock;
5120 }
5121
5122 list_add_tail(&n->poll_list, repoll);
5123
5124out_unlock:
5125 netpoll_poll_unlock(have);
5126
5127 return work;
5128}
5129
5130static void net_rx_action(struct softirq_action *h)
5131{
5132 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5133 unsigned long time_limit = jiffies + 2;
5134 int budget = netdev_budget;
5135 LIST_HEAD(list);
5136 LIST_HEAD(repoll);
5137
5138 local_irq_disable();
5139 list_splice_init(&sd->poll_list, &list);
5140 local_irq_enable();
5141
5142 for (;;) {
5143 struct napi_struct *n;
5144
5145 if (list_empty(&list)) {
5146 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5147 return;
5148 break;
5149 }
5150
5151 n = list_first_entry(&list, struct napi_struct, poll_list);
5152 budget -= napi_poll(n, &repoll);
5153
5154 /* If softirq window is exhausted then punt.
5155 * Allow this to run for 2 jiffies since which will allow
5156 * an average latency of 1.5/HZ.
5157 */
5158 if (unlikely(budget <= 0 ||
5159 time_after_eq(jiffies, time_limit))) {
5160 sd->time_squeeze++;
5161 break;
5162 }
5163 }
5164
5165 __kfree_skb_flush();
5166 local_irq_disable();
5167
5168 list_splice_tail_init(&sd->poll_list, &list);
5169 list_splice_tail(&repoll, &list);
5170 list_splice(&list, &sd->poll_list);
5171 if (!list_empty(&sd->poll_list))
5172 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5173
5174 net_rps_action_and_irq_enable(sd);
5175}
5176
5177struct netdev_adjacent {
5178 struct net_device *dev;
5179
5180 /* upper master flag, there can only be one master device per list */
5181 bool master;
5182
5183 /* counter for the number of times this device was added to us */
5184 u16 ref_nr;
5185
5186 /* private field for the users */
5187 void *private;
5188
5189 struct list_head list;
5190 struct rcu_head rcu;
5191};
5192
5193static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5194 struct list_head *adj_list)
5195{
5196 struct netdev_adjacent *adj;
5197
5198 list_for_each_entry(adj, adj_list, list) {
5199 if (adj->dev == adj_dev)
5200 return adj;
5201 }
5202 return NULL;
5203}
5204
5205/**
5206 * netdev_has_upper_dev - Check if device is linked to an upper device
5207 * @dev: device
5208 * @upper_dev: upper device to check
5209 *
5210 * Find out if a device is linked to specified upper device and return true
5211 * in case it is. Note that this checks only immediate upper device,
5212 * not through a complete stack of devices. The caller must hold the RTNL lock.
5213 */
5214bool netdev_has_upper_dev(struct net_device *dev,
5215 struct net_device *upper_dev)
5216{
5217 ASSERT_RTNL();
5218
5219 return __netdev_find_adj(upper_dev, &dev->all_adj_list.upper);
5220}
5221EXPORT_SYMBOL(netdev_has_upper_dev);
5222
5223/**
5224 * netdev_has_any_upper_dev - Check if device is linked to some device
5225 * @dev: device
5226 *
5227 * Find out if a device is linked to an upper device and return true in case
5228 * it is. The caller must hold the RTNL lock.
5229 */
5230static bool netdev_has_any_upper_dev(struct net_device *dev)
5231{
5232 ASSERT_RTNL();
5233
5234 return !list_empty(&dev->all_adj_list.upper);
5235}
5236
5237/**
5238 * netdev_master_upper_dev_get - Get master upper device
5239 * @dev: device
5240 *
5241 * Find a master upper device and return pointer to it or NULL in case
5242 * it's not there. The caller must hold the RTNL lock.
5243 */
5244struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5245{
5246 struct netdev_adjacent *upper;
5247
5248 ASSERT_RTNL();
5249
5250 if (list_empty(&dev->adj_list.upper))
5251 return NULL;
5252
5253 upper = list_first_entry(&dev->adj_list.upper,
5254 struct netdev_adjacent, list);
5255 if (likely(upper->master))
5256 return upper->dev;
5257 return NULL;
5258}
5259EXPORT_SYMBOL(netdev_master_upper_dev_get);
5260
5261void *netdev_adjacent_get_private(struct list_head *adj_list)
5262{
5263 struct netdev_adjacent *adj;
5264
5265 adj = list_entry(adj_list, struct netdev_adjacent, list);
5266
5267 return adj->private;
5268}
5269EXPORT_SYMBOL(netdev_adjacent_get_private);
5270
5271/**
5272 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5273 * @dev: device
5274 * @iter: list_head ** of the current position
5275 *
5276 * Gets the next device from the dev's upper list, starting from iter
5277 * position. The caller must hold RCU read lock.
5278 */
5279struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5280 struct list_head **iter)
5281{
5282 struct netdev_adjacent *upper;
5283
5284 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5285
5286 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5287
5288 if (&upper->list == &dev->adj_list.upper)
5289 return NULL;
5290
5291 *iter = &upper->list;
5292
5293 return upper->dev;
5294}
5295EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5296
5297/**
5298 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
5299 * @dev: device
5300 * @iter: list_head ** of the current position
5301 *
5302 * Gets the next device from the dev's upper list, starting from iter
5303 * position. The caller must hold RCU read lock.
5304 */
5305struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
5306 struct list_head **iter)
5307{
5308 struct netdev_adjacent *upper;
5309
5310 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5311
5312 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5313
5314 if (&upper->list == &dev->all_adj_list.upper)
5315 return NULL;
5316
5317 *iter = &upper->list;
5318
5319 return upper->dev;
5320}
5321EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
5322
5323/**
5324 * netdev_lower_get_next_private - Get the next ->private from the
5325 * lower neighbour list
5326 * @dev: device
5327 * @iter: list_head ** of the current position
5328 *
5329 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5330 * list, starting from iter position. The caller must hold either hold the
5331 * RTNL lock or its own locking that guarantees that the neighbour lower
5332 * list will remain unchanged.
5333 */
5334void *netdev_lower_get_next_private(struct net_device *dev,
5335 struct list_head **iter)
5336{
5337 struct netdev_adjacent *lower;
5338
5339 lower = list_entry(*iter, struct netdev_adjacent, list);
5340
5341 if (&lower->list == &dev->adj_list.lower)
5342 return NULL;
5343
5344 *iter = lower->list.next;
5345
5346 return lower->private;
5347}
5348EXPORT_SYMBOL(netdev_lower_get_next_private);
5349
5350/**
5351 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5352 * lower neighbour list, RCU
5353 * variant
5354 * @dev: device
5355 * @iter: list_head ** of the current position
5356 *
5357 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5358 * list, starting from iter position. The caller must hold RCU read lock.
5359 */
5360void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5361 struct list_head **iter)
5362{
5363 struct netdev_adjacent *lower;
5364
5365 WARN_ON_ONCE(!rcu_read_lock_held());
5366
5367 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5368
5369 if (&lower->list == &dev->adj_list.lower)
5370 return NULL;
5371
5372 *iter = &lower->list;
5373
5374 return lower->private;
5375}
5376EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5377
5378/**
5379 * netdev_lower_get_next - Get the next device from the lower neighbour
5380 * list
5381 * @dev: device
5382 * @iter: list_head ** of the current position
5383 *
5384 * Gets the next netdev_adjacent from the dev's lower neighbour
5385 * list, starting from iter position. The caller must hold RTNL lock or
5386 * its own locking that guarantees that the neighbour lower
5387 * list will remain unchanged.
5388 */
5389void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5390{
5391 struct netdev_adjacent *lower;
5392
5393 lower = list_entry(*iter, struct netdev_adjacent, list);
5394
5395 if (&lower->list == &dev->adj_list.lower)
5396 return NULL;
5397
5398 *iter = lower->list.next;
5399
5400 return lower->dev;
5401}
5402EXPORT_SYMBOL(netdev_lower_get_next);
5403
5404/**
5405 * netdev_lower_get_first_private_rcu - Get the first ->private from the
5406 * lower neighbour list, RCU
5407 * variant
5408 * @dev: device
5409 *
5410 * Gets the first netdev_adjacent->private from the dev's lower neighbour
5411 * list. The caller must hold RCU read lock.
5412 */
5413void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5414{
5415 struct netdev_adjacent *lower;
5416
5417 lower = list_first_or_null_rcu(&dev->adj_list.lower,
5418 struct netdev_adjacent, list);
5419 if (lower)
5420 return lower->private;
5421 return NULL;
5422}
5423EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5424
5425/**
5426 * netdev_master_upper_dev_get_rcu - Get master upper device
5427 * @dev: device
5428 *
5429 * Find a master upper device and return pointer to it or NULL in case
5430 * it's not there. The caller must hold the RCU read lock.
5431 */
5432struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5433{
5434 struct netdev_adjacent *upper;
5435
5436 upper = list_first_or_null_rcu(&dev->adj_list.upper,
5437 struct netdev_adjacent, list);
5438 if (upper && likely(upper->master))
5439 return upper->dev;
5440 return NULL;
5441}
5442EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5443
5444static int netdev_adjacent_sysfs_add(struct net_device *dev,
5445 struct net_device *adj_dev,
5446 struct list_head *dev_list)
5447{
5448 char linkname[IFNAMSIZ+7];
5449 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5450 "upper_%s" : "lower_%s", adj_dev->name);
5451 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5452 linkname);
5453}
5454static void netdev_adjacent_sysfs_del(struct net_device *dev,
5455 char *name,
5456 struct list_head *dev_list)
5457{
5458 char linkname[IFNAMSIZ+7];
5459 sprintf(linkname, dev_list == &dev->adj_list.upper ?
5460 "upper_%s" : "lower_%s", name);
5461 sysfs_remove_link(&(dev->dev.kobj), linkname);
5462}
5463
5464static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5465 struct net_device *adj_dev,
5466 struct list_head *dev_list)
5467{
5468 return (dev_list == &dev->adj_list.upper ||
5469 dev_list == &dev->adj_list.lower) &&
5470 net_eq(dev_net(dev), dev_net(adj_dev));
5471}
5472
5473static int __netdev_adjacent_dev_insert(struct net_device *dev,
5474 struct net_device *adj_dev,
5475 struct list_head *dev_list,
5476 void *private, bool master)
5477{
5478 struct netdev_adjacent *adj;
5479 int ret;
5480
5481 adj = __netdev_find_adj(adj_dev, dev_list);
5482
5483 if (adj) {
5484 adj->ref_nr++;
5485 return 0;
5486 }
5487
5488 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5489 if (!adj)
5490 return -ENOMEM;
5491
5492 adj->dev = adj_dev;
5493 adj->master = master;
5494 adj->ref_nr = 1;
5495 adj->private = private;
5496 dev_hold(adj_dev);
5497
5498 pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5499 adj_dev->name, dev->name, adj_dev->name);
5500
5501 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
5502 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
5503 if (ret)
5504 goto free_adj;
5505 }
5506
5507 /* Ensure that master link is always the first item in list. */
5508 if (master) {
5509 ret = sysfs_create_link(&(dev->dev.kobj),
5510 &(adj_dev->dev.kobj), "master");
5511 if (ret)
5512 goto remove_symlinks;
5513
5514 list_add_rcu(&adj->list, dev_list);
5515 } else {
5516 list_add_tail_rcu(&adj->list, dev_list);
5517 }
5518
5519 return 0;
5520
5521remove_symlinks:
5522 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5523 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5524free_adj:
5525 kfree(adj);
5526 dev_put(adj_dev);
5527
5528 return ret;
5529}
5530
5531static void __netdev_adjacent_dev_remove(struct net_device *dev,
5532 struct net_device *adj_dev,
5533 struct list_head *dev_list)
5534{
5535 struct netdev_adjacent *adj;
5536
5537 adj = __netdev_find_adj(adj_dev, dev_list);
5538
5539 if (!adj) {
5540 pr_err("tried to remove device %s from %s\n",
5541 dev->name, adj_dev->name);
5542 BUG();
5543 }
5544
5545 if (adj->ref_nr > 1) {
5546 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
5547 adj->ref_nr-1);
5548 adj->ref_nr--;
5549 return;
5550 }
5551
5552 if (adj->master)
5553 sysfs_remove_link(&(dev->dev.kobj), "master");
5554
5555 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5556 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5557
5558 list_del_rcu(&adj->list);
5559 pr_debug("dev_put for %s, because link removed from %s to %s\n",
5560 adj_dev->name, dev->name, adj_dev->name);
5561 dev_put(adj_dev);
5562 kfree_rcu(adj, rcu);
5563}
5564
5565static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5566 struct net_device *upper_dev,
5567 struct list_head *up_list,
5568 struct list_head *down_list,
5569 void *private, bool master)
5570{
5571 int ret;
5572
5573 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
5574 master);
5575 if (ret)
5576 return ret;
5577
5578 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
5579 false);
5580 if (ret) {
5581 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5582 return ret;
5583 }
5584
5585 return 0;
5586}
5587
5588static int __netdev_adjacent_dev_link(struct net_device *dev,
5589 struct net_device *upper_dev)
5590{
5591 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5592 &dev->all_adj_list.upper,
5593 &upper_dev->all_adj_list.lower,
5594 NULL, false);
5595}
5596
5597static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5598 struct net_device *upper_dev,
5599 struct list_head *up_list,
5600 struct list_head *down_list)
5601{
5602 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5603 __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
5604}
5605
5606static void __netdev_adjacent_dev_unlink(struct net_device *dev,
5607 struct net_device *upper_dev)
5608{
5609 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5610 &dev->all_adj_list.upper,
5611 &upper_dev->all_adj_list.lower);
5612}
5613
5614static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5615 struct net_device *upper_dev,
5616 void *private, bool master)
5617{
5618 int ret = __netdev_adjacent_dev_link(dev, upper_dev);
5619
5620 if (ret)
5621 return ret;
5622
5623 ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
5624 &dev->adj_list.upper,
5625 &upper_dev->adj_list.lower,
5626 private, master);
5627 if (ret) {
5628 __netdev_adjacent_dev_unlink(dev, upper_dev);
5629 return ret;
5630 }
5631
5632 return 0;
5633}
5634
5635static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5636 struct net_device *upper_dev)
5637{
5638 __netdev_adjacent_dev_unlink(dev, upper_dev);
5639 __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5640 &dev->adj_list.upper,
5641 &upper_dev->adj_list.lower);
5642}
5643
5644static int __netdev_upper_dev_link(struct net_device *dev,
5645 struct net_device *upper_dev, bool master,
5646 void *upper_priv, void *upper_info)
5647{
5648 struct netdev_notifier_changeupper_info changeupper_info;
5649 struct netdev_adjacent *i, *j, *to_i, *to_j;
5650 int ret = 0;
5651
5652 ASSERT_RTNL();
5653
5654 if (dev == upper_dev)
5655 return -EBUSY;
5656
5657 /* To prevent loops, check if dev is not upper device to upper_dev. */
5658 if (__netdev_find_adj(dev, &upper_dev->all_adj_list.upper))
5659 return -EBUSY;
5660
5661 if (__netdev_find_adj(upper_dev, &dev->adj_list.upper))
5662 return -EEXIST;
5663
5664 if (master && netdev_master_upper_dev_get(dev))
5665 return -EBUSY;
5666
5667 changeupper_info.upper_dev = upper_dev;
5668 changeupper_info.master = master;
5669 changeupper_info.linking = true;
5670 changeupper_info.upper_info = upper_info;
5671
5672 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5673 &changeupper_info.info);
5674 ret = notifier_to_errno(ret);
5675 if (ret)
5676 return ret;
5677
5678 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
5679 master);
5680 if (ret)
5681 return ret;
5682
5683 /* Now that we linked these devs, make all the upper_dev's
5684 * all_adj_list.upper visible to every dev's all_adj_list.lower an
5685 * versa, and don't forget the devices itself. All of these
5686 * links are non-neighbours.
5687 */
5688 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5689 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5690 pr_debug("Interlinking %s with %s, non-neighbour\n",
5691 i->dev->name, j->dev->name);
5692 ret = __netdev_adjacent_dev_link(i->dev, j->dev);
5693 if (ret)
5694 goto rollback_mesh;
5695 }
5696 }
5697
5698 /* add dev to every upper_dev's upper device */
5699 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5700 pr_debug("linking %s's upper device %s with %s\n",
5701 upper_dev->name, i->dev->name, dev->name);
5702 ret = __netdev_adjacent_dev_link(dev, i->dev);
5703 if (ret)
5704 goto rollback_upper_mesh;
5705 }
5706
5707 /* add upper_dev to every dev's lower device */
5708 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5709 pr_debug("linking %s's lower device %s with %s\n", dev->name,
5710 i->dev->name, upper_dev->name);
5711 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
5712 if (ret)
5713 goto rollback_lower_mesh;
5714 }
5715
5716 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5717 &changeupper_info.info);
5718 ret = notifier_to_errno(ret);
5719 if (ret)
5720 goto rollback_lower_mesh;
5721
5722 return 0;
5723
5724rollback_lower_mesh:
5725 to_i = i;
5726 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5727 if (i == to_i)
5728 break;
5729 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5730 }
5731
5732 i = NULL;
5733
5734rollback_upper_mesh:
5735 to_i = i;
5736 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5737 if (i == to_i)
5738 break;
5739 __netdev_adjacent_dev_unlink(dev, i->dev);
5740 }
5741
5742 i = j = NULL;
5743
5744rollback_mesh:
5745 to_i = i;
5746 to_j = j;
5747 list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5748 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5749 if (i == to_i && j == to_j)
5750 break;
5751 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5752 }
5753 if (i == to_i)
5754 break;
5755 }
5756
5757 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5758
5759 return ret;
5760}
5761
5762/**
5763 * netdev_upper_dev_link - Add a link to the upper device
5764 * @dev: device
5765 * @upper_dev: new upper device
5766 *
5767 * Adds a link to device which is upper to this one. The caller must hold
5768 * the RTNL lock. On a failure a negative errno code is returned.
5769 * On success the reference counts are adjusted and the function
5770 * returns zero.
5771 */
5772int netdev_upper_dev_link(struct net_device *dev,
5773 struct net_device *upper_dev)
5774{
5775 return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
5776}
5777EXPORT_SYMBOL(netdev_upper_dev_link);
5778
5779/**
5780 * netdev_master_upper_dev_link - Add a master link to the upper device
5781 * @dev: device
5782 * @upper_dev: new upper device
5783 * @upper_priv: upper device private
5784 * @upper_info: upper info to be passed down via notifier
5785 *
5786 * Adds a link to device which is upper to this one. In this case, only
5787 * one master upper device can be linked, although other non-master devices
5788 * might be linked as well. The caller must hold the RTNL lock.
5789 * On a failure a negative errno code is returned. On success the reference
5790 * counts are adjusted and the function returns zero.
5791 */
5792int netdev_master_upper_dev_link(struct net_device *dev,
5793 struct net_device *upper_dev,
5794 void *upper_priv, void *upper_info)
5795{
5796 return __netdev_upper_dev_link(dev, upper_dev, true,
5797 upper_priv, upper_info);
5798}
5799EXPORT_SYMBOL(netdev_master_upper_dev_link);
5800
5801/**
5802 * netdev_upper_dev_unlink - Removes a link to upper device
5803 * @dev: device
5804 * @upper_dev: new upper device
5805 *
5806 * Removes a link to device which is upper to this one. The caller must hold
5807 * the RTNL lock.
5808 */
5809void netdev_upper_dev_unlink(struct net_device *dev,
5810 struct net_device *upper_dev)
5811{
5812 struct netdev_notifier_changeupper_info changeupper_info;
5813 struct netdev_adjacent *i, *j;
5814 ASSERT_RTNL();
5815
5816 changeupper_info.upper_dev = upper_dev;
5817 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
5818 changeupper_info.linking = false;
5819
5820 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5821 &changeupper_info.info);
5822
5823 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5824
5825 /* Here is the tricky part. We must remove all dev's lower
5826 * devices from all upper_dev's upper devices and vice
5827 * versa, to maintain the graph relationship.
5828 */
5829 list_for_each_entry(i, &dev->all_adj_list.lower, list)
5830 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
5831 __netdev_adjacent_dev_unlink(i->dev, j->dev);
5832
5833 /* remove also the devices itself from lower/upper device
5834 * list
5835 */
5836 list_for_each_entry(i, &dev->all_adj_list.lower, list)
5837 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5838
5839 list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
5840 __netdev_adjacent_dev_unlink(dev, i->dev);
5841
5842 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5843 &changeupper_info.info);
5844}
5845EXPORT_SYMBOL(netdev_upper_dev_unlink);
5846
5847/**
5848 * netdev_bonding_info_change - Dispatch event about slave change
5849 * @dev: device
5850 * @bonding_info: info to dispatch
5851 *
5852 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5853 * The caller must hold the RTNL lock.
5854 */
5855void netdev_bonding_info_change(struct net_device *dev,
5856 struct netdev_bonding_info *bonding_info)
5857{
5858 struct netdev_notifier_bonding_info info;
5859
5860 memcpy(&info.bonding_info, bonding_info,
5861 sizeof(struct netdev_bonding_info));
5862 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5863 &info.info);
5864}
5865EXPORT_SYMBOL(netdev_bonding_info_change);
5866
5867static void netdev_adjacent_add_links(struct net_device *dev)
5868{
5869 struct netdev_adjacent *iter;
5870
5871 struct net *net = dev_net(dev);
5872
5873 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5874 if (!net_eq(net,dev_net(iter->dev)))
5875 continue;
5876 netdev_adjacent_sysfs_add(iter->dev, dev,
5877 &iter->dev->adj_list.lower);
5878 netdev_adjacent_sysfs_add(dev, iter->dev,
5879 &dev->adj_list.upper);
5880 }
5881
5882 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5883 if (!net_eq(net,dev_net(iter->dev)))
5884 continue;
5885 netdev_adjacent_sysfs_add(iter->dev, dev,
5886 &iter->dev->adj_list.upper);
5887 netdev_adjacent_sysfs_add(dev, iter->dev,
5888 &dev->adj_list.lower);
5889 }
5890}
5891
5892static void netdev_adjacent_del_links(struct net_device *dev)
5893{
5894 struct netdev_adjacent *iter;
5895
5896 struct net *net = dev_net(dev);
5897
5898 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5899 if (!net_eq(net,dev_net(iter->dev)))
5900 continue;
5901 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5902 &iter->dev->adj_list.lower);
5903 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5904 &dev->adj_list.upper);
5905 }
5906
5907 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5908 if (!net_eq(net,dev_net(iter->dev)))
5909 continue;
5910 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5911 &iter->dev->adj_list.upper);
5912 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5913 &dev->adj_list.lower);
5914 }
5915}
5916
5917void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
5918{
5919 struct netdev_adjacent *iter;
5920
5921 struct net *net = dev_net(dev);
5922
5923 list_for_each_entry(iter, &dev->adj_list.upper, list) {
5924 if (!net_eq(net,dev_net(iter->dev)))
5925 continue;
5926 netdev_adjacent_sysfs_del(iter->dev, oldname,
5927 &iter->dev->adj_list.lower);
5928 netdev_adjacent_sysfs_add(iter->dev, dev,
5929 &iter->dev->adj_list.lower);
5930 }
5931
5932 list_for_each_entry(iter, &dev->adj_list.lower, list) {
5933 if (!net_eq(net,dev_net(iter->dev)))
5934 continue;
5935 netdev_adjacent_sysfs_del(iter->dev, oldname,
5936 &iter->dev->adj_list.upper);
5937 netdev_adjacent_sysfs_add(iter->dev, dev,
5938 &iter->dev->adj_list.upper);
5939 }
5940}
5941
5942void *netdev_lower_dev_get_private(struct net_device *dev,
5943 struct net_device *lower_dev)
5944{
5945 struct netdev_adjacent *lower;
5946
5947 if (!lower_dev)
5948 return NULL;
5949 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
5950 if (!lower)
5951 return NULL;
5952
5953 return lower->private;
5954}
5955EXPORT_SYMBOL(netdev_lower_dev_get_private);
5956
5957
5958int dev_get_nest_level(struct net_device *dev,
5959 bool (*type_check)(const struct net_device *dev))
5960{
5961 struct net_device *lower = NULL;
5962 struct list_head *iter;
5963 int max_nest = -1;
5964 int nest;
5965
5966 ASSERT_RTNL();
5967
5968 netdev_for_each_lower_dev(dev, lower, iter) {
5969 nest = dev_get_nest_level(lower, type_check);
5970 if (max_nest < nest)
5971 max_nest = nest;
5972 }
5973
5974 if (type_check(dev))
5975 max_nest++;
5976
5977 return max_nest;
5978}
5979EXPORT_SYMBOL(dev_get_nest_level);
5980
5981/**
5982 * netdev_lower_change - Dispatch event about lower device state change
5983 * @lower_dev: device
5984 * @lower_state_info: state to dispatch
5985 *
5986 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
5987 * The caller must hold the RTNL lock.
5988 */
5989void netdev_lower_state_changed(struct net_device *lower_dev,
5990 void *lower_state_info)
5991{
5992 struct netdev_notifier_changelowerstate_info changelowerstate_info;
5993
5994 ASSERT_RTNL();
5995 changelowerstate_info.lower_state_info = lower_state_info;
5996 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
5997 &changelowerstate_info.info);
5998}
5999EXPORT_SYMBOL(netdev_lower_state_changed);
6000
6001static void dev_change_rx_flags(struct net_device *dev, int flags)
6002{
6003 const struct net_device_ops *ops = dev->netdev_ops;
6004
6005 if (ops->ndo_change_rx_flags)
6006 ops->ndo_change_rx_flags(dev, flags);
6007}
6008
6009static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
6010{
6011 unsigned int old_flags = dev->flags;
6012 kuid_t uid;
6013 kgid_t gid;
6014
6015 ASSERT_RTNL();
6016
6017 dev->flags |= IFF_PROMISC;
6018 dev->promiscuity += inc;
6019 if (dev->promiscuity == 0) {
6020 /*
6021 * Avoid overflow.
6022 * If inc causes overflow, untouch promisc and return error.
6023 */
6024 if (inc < 0)
6025 dev->flags &= ~IFF_PROMISC;
6026 else {
6027 dev->promiscuity -= inc;
6028 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6029 dev->name);
6030 return -EOVERFLOW;
6031 }
6032 }
6033 if (dev->flags != old_flags) {
6034 pr_info("device %s %s promiscuous mode\n",
6035 dev->name,
6036 dev->flags & IFF_PROMISC ? "entered" : "left");
6037 if (audit_enabled) {
6038 current_uid_gid(&uid, &gid);
6039 audit_log(current->audit_context, GFP_ATOMIC,
6040 AUDIT_ANOM_PROMISCUOUS,
6041 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6042 dev->name, (dev->flags & IFF_PROMISC),
6043 (old_flags & IFF_PROMISC),
6044 from_kuid(&init_user_ns, audit_get_loginuid(current)),
6045 from_kuid(&init_user_ns, uid),
6046 from_kgid(&init_user_ns, gid),
6047 audit_get_sessionid(current));
6048 }
6049
6050 dev_change_rx_flags(dev, IFF_PROMISC);
6051 }
6052 if (notify)
6053 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
6054 return 0;
6055}
6056
6057/**
6058 * dev_set_promiscuity - update promiscuity count on a device
6059 * @dev: device
6060 * @inc: modifier
6061 *
6062 * Add or remove promiscuity from a device. While the count in the device
6063 * remains above zero the interface remains promiscuous. Once it hits zero
6064 * the device reverts back to normal filtering operation. A negative inc
6065 * value is used to drop promiscuity on the device.
6066 * Return 0 if successful or a negative errno code on error.
6067 */
6068int dev_set_promiscuity(struct net_device *dev, int inc)
6069{
6070 unsigned int old_flags = dev->flags;
6071 int err;
6072
6073 err = __dev_set_promiscuity(dev, inc, true);
6074 if (err < 0)
6075 return err;
6076 if (dev->flags != old_flags)
6077 dev_set_rx_mode(dev);
6078 return err;
6079}
6080EXPORT_SYMBOL(dev_set_promiscuity);
6081
6082static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
6083{
6084 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
6085
6086 ASSERT_RTNL();
6087
6088 dev->flags |= IFF_ALLMULTI;
6089 dev->allmulti += inc;
6090 if (dev->allmulti == 0) {
6091 /*
6092 * Avoid overflow.
6093 * If inc causes overflow, untouch allmulti and return error.
6094 */
6095 if (inc < 0)
6096 dev->flags &= ~IFF_ALLMULTI;
6097 else {
6098 dev->allmulti -= inc;
6099 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6100 dev->name);
6101 return -EOVERFLOW;
6102 }
6103 }
6104 if (dev->flags ^ old_flags) {
6105 dev_change_rx_flags(dev, IFF_ALLMULTI);
6106 dev_set_rx_mode(dev);
6107 if (notify)
6108 __dev_notify_flags(dev, old_flags,
6109 dev->gflags ^ old_gflags);
6110 }
6111 return 0;
6112}
6113
6114/**
6115 * dev_set_allmulti - update allmulti count on a device
6116 * @dev: device
6117 * @inc: modifier
6118 *
6119 * Add or remove reception of all multicast frames to a device. While the
6120 * count in the device remains above zero the interface remains listening
6121 * to all interfaces. Once it hits zero the device reverts back to normal
6122 * filtering operation. A negative @inc value is used to drop the counter
6123 * when releasing a resource needing all multicasts.
6124 * Return 0 if successful or a negative errno code on error.
6125 */
6126
6127int dev_set_allmulti(struct net_device *dev, int inc)
6128{
6129 return __dev_set_allmulti(dev, inc, true);
6130}
6131EXPORT_SYMBOL(dev_set_allmulti);
6132
6133/*
6134 * Upload unicast and multicast address lists to device and
6135 * configure RX filtering. When the device doesn't support unicast
6136 * filtering it is put in promiscuous mode while unicast addresses
6137 * are present.
6138 */
6139void __dev_set_rx_mode(struct net_device *dev)
6140{
6141 const struct net_device_ops *ops = dev->netdev_ops;
6142
6143 /* dev_open will call this function so the list will stay sane. */
6144 if (!(dev->flags&IFF_UP))
6145 return;
6146
6147 if (!netif_device_present(dev))
6148 return;
6149
6150 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
6151 /* Unicast addresses changes may only happen under the rtnl,
6152 * therefore calling __dev_set_promiscuity here is safe.
6153 */
6154 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
6155 __dev_set_promiscuity(dev, 1, false);
6156 dev->uc_promisc = true;
6157 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
6158 __dev_set_promiscuity(dev, -1, false);
6159 dev->uc_promisc = false;
6160 }
6161 }
6162
6163 if (ops->ndo_set_rx_mode)
6164 ops->ndo_set_rx_mode(dev);
6165}
6166
6167void dev_set_rx_mode(struct net_device *dev)
6168{
6169 netif_addr_lock_bh(dev);
6170 __dev_set_rx_mode(dev);
6171 netif_addr_unlock_bh(dev);
6172}
6173
6174/**
6175 * dev_get_flags - get flags reported to userspace
6176 * @dev: device
6177 *
6178 * Get the combination of flag bits exported through APIs to userspace.
6179 */
6180unsigned int dev_get_flags(const struct net_device *dev)
6181{
6182 unsigned int flags;
6183
6184 flags = (dev->flags & ~(IFF_PROMISC |
6185 IFF_ALLMULTI |
6186 IFF_RUNNING |
6187 IFF_LOWER_UP |
6188 IFF_DORMANT)) |
6189 (dev->gflags & (IFF_PROMISC |
6190 IFF_ALLMULTI));
6191
6192 if (netif_running(dev)) {
6193 if (netif_oper_up(dev))
6194 flags |= IFF_RUNNING;
6195 if (netif_carrier_ok(dev))
6196 flags |= IFF_LOWER_UP;
6197 if (netif_dormant(dev))
6198 flags |= IFF_DORMANT;
6199 }
6200
6201 return flags;
6202}
6203EXPORT_SYMBOL(dev_get_flags);
6204
6205int __dev_change_flags(struct net_device *dev, unsigned int flags)
6206{
6207 unsigned int old_flags = dev->flags;
6208 int ret;
6209
6210 ASSERT_RTNL();
6211
6212 /*
6213 * Set the flags on our device.
6214 */
6215
6216 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6217 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6218 IFF_AUTOMEDIA)) |
6219 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6220 IFF_ALLMULTI));
6221
6222 /*
6223 * Load in the correct multicast list now the flags have changed.
6224 */
6225
6226 if ((old_flags ^ flags) & IFF_MULTICAST)
6227 dev_change_rx_flags(dev, IFF_MULTICAST);
6228
6229 dev_set_rx_mode(dev);
6230
6231 /*
6232 * Have we downed the interface. We handle IFF_UP ourselves
6233 * according to user attempts to set it, rather than blindly
6234 * setting it.
6235 */
6236
6237 ret = 0;
6238 if ((old_flags ^ flags) & IFF_UP)
6239 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
6240
6241 if ((flags ^ dev->gflags) & IFF_PROMISC) {
6242 int inc = (flags & IFF_PROMISC) ? 1 : -1;
6243 unsigned int old_flags = dev->flags;
6244
6245 dev->gflags ^= IFF_PROMISC;
6246
6247 if (__dev_set_promiscuity(dev, inc, false) >= 0)
6248 if (dev->flags != old_flags)
6249 dev_set_rx_mode(dev);
6250 }
6251
6252 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
6253 is important. Some (broken) drivers set IFF_PROMISC, when
6254 IFF_ALLMULTI is requested not asking us and not reporting.
6255 */
6256 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
6257 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6258
6259 dev->gflags ^= IFF_ALLMULTI;
6260 __dev_set_allmulti(dev, inc, false);
6261 }
6262
6263 return ret;
6264}
6265
6266void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6267 unsigned int gchanges)
6268{
6269 unsigned int changes = dev->flags ^ old_flags;
6270
6271 if (gchanges)
6272 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
6273
6274 if (changes & IFF_UP) {
6275 if (dev->flags & IFF_UP)
6276 call_netdevice_notifiers(NETDEV_UP, dev);
6277 else
6278 call_netdevice_notifiers(NETDEV_DOWN, dev);
6279 }
6280
6281 if (dev->flags & IFF_UP &&
6282 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6283 struct netdev_notifier_change_info change_info;
6284
6285 change_info.flags_changed = changes;
6286 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6287 &change_info.info);
6288 }
6289}
6290
6291/**
6292 * dev_change_flags - change device settings
6293 * @dev: device
6294 * @flags: device state flags
6295 *
6296 * Change settings on device based state flags. The flags are
6297 * in the userspace exported format.
6298 */
6299int dev_change_flags(struct net_device *dev, unsigned int flags)
6300{
6301 int ret;
6302 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
6303
6304 ret = __dev_change_flags(dev, flags);
6305 if (ret < 0)
6306 return ret;
6307
6308 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
6309 __dev_notify_flags(dev, old_flags, changes);
6310 return ret;
6311}
6312EXPORT_SYMBOL(dev_change_flags);
6313
6314static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6315{
6316 const struct net_device_ops *ops = dev->netdev_ops;
6317
6318 if (ops->ndo_change_mtu)
6319 return ops->ndo_change_mtu(dev, new_mtu);
6320
6321 dev->mtu = new_mtu;
6322 return 0;
6323}
6324
6325/**
6326 * dev_set_mtu - Change maximum transfer unit
6327 * @dev: device
6328 * @new_mtu: new transfer unit
6329 *
6330 * Change the maximum transfer size of the network device.
6331 */
6332int dev_set_mtu(struct net_device *dev, int new_mtu)
6333{
6334 int err, orig_mtu;
6335
6336 if (new_mtu == dev->mtu)
6337 return 0;
6338
6339 /* MTU must be positive. */
6340 if (new_mtu < 0)
6341 return -EINVAL;
6342
6343 if (!netif_device_present(dev))
6344 return -ENODEV;
6345
6346 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6347 err = notifier_to_errno(err);
6348 if (err)
6349 return err;
6350
6351 orig_mtu = dev->mtu;
6352 err = __dev_set_mtu(dev, new_mtu);
6353
6354 if (!err) {
6355 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6356 err = notifier_to_errno(err);
6357 if (err) {
6358 /* setting mtu back and notifying everyone again,
6359 * so that they have a chance to revert changes.
6360 */
6361 __dev_set_mtu(dev, orig_mtu);
6362 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6363 }
6364 }
6365 return err;
6366}
6367EXPORT_SYMBOL(dev_set_mtu);
6368
6369/**
6370 * dev_set_group - Change group this device belongs to
6371 * @dev: device
6372 * @new_group: group this device should belong to
6373 */
6374void dev_set_group(struct net_device *dev, int new_group)
6375{
6376 dev->group = new_group;
6377}
6378EXPORT_SYMBOL(dev_set_group);
6379
6380/**
6381 * dev_set_mac_address - Change Media Access Control Address
6382 * @dev: device
6383 * @sa: new address
6384 *
6385 * Change the hardware (MAC) address of the device
6386 */
6387int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6388{
6389 const struct net_device_ops *ops = dev->netdev_ops;
6390 int err;
6391
6392 if (!ops->ndo_set_mac_address)
6393 return -EOPNOTSUPP;
6394 if (sa->sa_family != dev->type)
6395 return -EINVAL;
6396 if (!netif_device_present(dev))
6397 return -ENODEV;
6398 err = ops->ndo_set_mac_address(dev, sa);
6399 if (err)
6400 return err;
6401 dev->addr_assign_type = NET_ADDR_SET;
6402 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
6403 add_device_randomness(dev->dev_addr, dev->addr_len);
6404 return 0;
6405}
6406EXPORT_SYMBOL(dev_set_mac_address);
6407
6408/**
6409 * dev_change_carrier - Change device carrier
6410 * @dev: device
6411 * @new_carrier: new value
6412 *
6413 * Change device carrier
6414 */
6415int dev_change_carrier(struct net_device *dev, bool new_carrier)
6416{
6417 const struct net_device_ops *ops = dev->netdev_ops;
6418
6419 if (!ops->ndo_change_carrier)
6420 return -EOPNOTSUPP;
6421 if (!netif_device_present(dev))
6422 return -ENODEV;
6423 return ops->ndo_change_carrier(dev, new_carrier);
6424}
6425EXPORT_SYMBOL(dev_change_carrier);
6426
6427/**
6428 * dev_get_phys_port_id - Get device physical port ID
6429 * @dev: device
6430 * @ppid: port ID
6431 *
6432 * Get device physical port ID
6433 */
6434int dev_get_phys_port_id(struct net_device *dev,
6435 struct netdev_phys_item_id *ppid)
6436{
6437 const struct net_device_ops *ops = dev->netdev_ops;
6438
6439 if (!ops->ndo_get_phys_port_id)
6440 return -EOPNOTSUPP;
6441 return ops->ndo_get_phys_port_id(dev, ppid);
6442}
6443EXPORT_SYMBOL(dev_get_phys_port_id);
6444
6445/**
6446 * dev_get_phys_port_name - Get device physical port name
6447 * @dev: device
6448 * @name: port name
6449 * @len: limit of bytes to copy to name
6450 *
6451 * Get device physical port name
6452 */
6453int dev_get_phys_port_name(struct net_device *dev,
6454 char *name, size_t len)
6455{
6456 const struct net_device_ops *ops = dev->netdev_ops;
6457
6458 if (!ops->ndo_get_phys_port_name)
6459 return -EOPNOTSUPP;
6460 return ops->ndo_get_phys_port_name(dev, name, len);
6461}
6462EXPORT_SYMBOL(dev_get_phys_port_name);
6463
6464/**
6465 * dev_change_proto_down - update protocol port state information
6466 * @dev: device
6467 * @proto_down: new value
6468 *
6469 * This info can be used by switch drivers to set the phys state of the
6470 * port.
6471 */
6472int dev_change_proto_down(struct net_device *dev, bool proto_down)
6473{
6474 const struct net_device_ops *ops = dev->netdev_ops;
6475
6476 if (!ops->ndo_change_proto_down)
6477 return -EOPNOTSUPP;
6478 if (!netif_device_present(dev))
6479 return -ENODEV;
6480 return ops->ndo_change_proto_down(dev, proto_down);
6481}
6482EXPORT_SYMBOL(dev_change_proto_down);
6483
6484/**
6485 * dev_new_index - allocate an ifindex
6486 * @net: the applicable net namespace
6487 *
6488 * Returns a suitable unique value for a new device interface
6489 * number. The caller must hold the rtnl semaphore or the
6490 * dev_base_lock to be sure it remains unique.
6491 */
6492static int dev_new_index(struct net *net)
6493{
6494 int ifindex = net->ifindex;
6495 for (;;) {
6496 if (++ifindex <= 0)
6497 ifindex = 1;
6498 if (!__dev_get_by_index(net, ifindex))
6499 return net->ifindex = ifindex;
6500 }
6501}
6502
6503/* Delayed registration/unregisteration */
6504static LIST_HEAD(net_todo_list);
6505DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
6506
6507static void net_set_todo(struct net_device *dev)
6508{
6509 list_add_tail(&dev->todo_list, &net_todo_list);
6510 dev_net(dev)->dev_unreg_count++;
6511}
6512
6513static void rollback_registered_many(struct list_head *head)
6514{
6515 struct net_device *dev, *tmp;
6516 LIST_HEAD(close_head);
6517
6518 BUG_ON(dev_boot_phase);
6519 ASSERT_RTNL();
6520
6521 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
6522 /* Some devices call without registering
6523 * for initialization unwind. Remove those
6524 * devices and proceed with the remaining.
6525 */
6526 if (dev->reg_state == NETREG_UNINITIALIZED) {
6527 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6528 dev->name, dev);
6529
6530 WARN_ON(1);
6531 list_del(&dev->unreg_list);
6532 continue;
6533 }
6534 dev->dismantle = true;
6535 BUG_ON(dev->reg_state != NETREG_REGISTERED);
6536 }
6537
6538 /* If device is running, close it first. */
6539 list_for_each_entry(dev, head, unreg_list)
6540 list_add_tail(&dev->close_list, &close_head);
6541 dev_close_many(&close_head, true);
6542
6543 list_for_each_entry(dev, head, unreg_list) {
6544 /* And unlink it from device chain. */
6545 unlist_netdevice(dev);
6546
6547 dev->reg_state = NETREG_UNREGISTERING;
6548 on_each_cpu(flush_backlog, dev, 1);
6549 }
6550
6551 synchronize_net();
6552
6553 list_for_each_entry(dev, head, unreg_list) {
6554 struct sk_buff *skb = NULL;
6555
6556 /* Shutdown queueing discipline. */
6557 dev_shutdown(dev);
6558
6559
6560 /* Notify protocols, that we are about to destroy
6561 this device. They should clean all the things.
6562 */
6563 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6564
6565 if (!dev->rtnl_link_ops ||
6566 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6567 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6568 GFP_KERNEL);
6569
6570 /*
6571 * Flush the unicast and multicast chains
6572 */
6573 dev_uc_flush(dev);
6574 dev_mc_flush(dev);
6575
6576 if (dev->netdev_ops->ndo_uninit)
6577 dev->netdev_ops->ndo_uninit(dev);
6578
6579 if (skb)
6580 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
6581
6582 /* Notifier chain MUST detach us all upper devices. */
6583 WARN_ON(netdev_has_any_upper_dev(dev));
6584
6585 /* Remove entries from kobject tree */
6586 netdev_unregister_kobject(dev);
6587#ifdef CONFIG_XPS
6588 /* Remove XPS queueing entries */
6589 netif_reset_xps_queues_gt(dev, 0);
6590#endif
6591 }
6592
6593 synchronize_net();
6594
6595 list_for_each_entry(dev, head, unreg_list)
6596 dev_put(dev);
6597}
6598
6599static void rollback_registered(struct net_device *dev)
6600{
6601 LIST_HEAD(single);
6602
6603 list_add(&dev->unreg_list, &single);
6604 rollback_registered_many(&single);
6605 list_del(&single);
6606}
6607
6608static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
6609 struct net_device *upper, netdev_features_t features)
6610{
6611 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6612 netdev_features_t feature;
6613 int feature_bit;
6614
6615 for_each_netdev_feature(&upper_disables, feature_bit) {
6616 feature = __NETIF_F_BIT(feature_bit);
6617 if (!(upper->wanted_features & feature)
6618 && (features & feature)) {
6619 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
6620 &feature, upper->name);
6621 features &= ~feature;
6622 }
6623 }
6624
6625 return features;
6626}
6627
6628static void netdev_sync_lower_features(struct net_device *upper,
6629 struct net_device *lower, netdev_features_t features)
6630{
6631 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6632 netdev_features_t feature;
6633 int feature_bit;
6634
6635 for_each_netdev_feature(&upper_disables, feature_bit) {
6636 feature = __NETIF_F_BIT(feature_bit);
6637 if (!(features & feature) && (lower->features & feature)) {
6638 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
6639 &feature, lower->name);
6640 lower->wanted_features &= ~feature;
6641 netdev_update_features(lower);
6642
6643 if (unlikely(lower->features & feature))
6644 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
6645 &feature, lower->name);
6646 }
6647 }
6648}
6649
6650static netdev_features_t netdev_fix_features(struct net_device *dev,
6651 netdev_features_t features)
6652{
6653 /* Fix illegal checksum combinations */
6654 if ((features & NETIF_F_HW_CSUM) &&
6655 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6656 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
6657 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6658 }
6659
6660 /* TSO requires that SG is present as well. */
6661 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
6662 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
6663 features &= ~NETIF_F_ALL_TSO;
6664 }
6665
6666 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6667 !(features & NETIF_F_IP_CSUM)) {
6668 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6669 features &= ~NETIF_F_TSO;
6670 features &= ~NETIF_F_TSO_ECN;
6671 }
6672
6673 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6674 !(features & NETIF_F_IPV6_CSUM)) {
6675 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6676 features &= ~NETIF_F_TSO6;
6677 }
6678
6679 /* TSO ECN requires that TSO is present as well. */
6680 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6681 features &= ~NETIF_F_TSO_ECN;
6682
6683 /* Software GSO depends on SG. */
6684 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
6685 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
6686 features &= ~NETIF_F_GSO;
6687 }
6688
6689 /* UFO needs SG and checksumming */
6690 if (features & NETIF_F_UFO) {
6691 /* maybe split UFO into V4 and V6? */
6692 if (!(features & NETIF_F_HW_CSUM) &&
6693 ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) !=
6694 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) {
6695 netdev_dbg(dev,
6696 "Dropping NETIF_F_UFO since no checksum offload features.\n");
6697 features &= ~NETIF_F_UFO;
6698 }
6699
6700 if (!(features & NETIF_F_SG)) {
6701 netdev_dbg(dev,
6702 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
6703 features &= ~NETIF_F_UFO;
6704 }
6705 }
6706
6707#ifdef CONFIG_NET_RX_BUSY_POLL
6708 if (dev->netdev_ops->ndo_busy_poll)
6709 features |= NETIF_F_BUSY_POLL;
6710 else
6711#endif
6712 features &= ~NETIF_F_BUSY_POLL;
6713
6714 return features;
6715}
6716
6717int __netdev_update_features(struct net_device *dev)
6718{
6719 struct net_device *upper, *lower;
6720 netdev_features_t features;
6721 struct list_head *iter;
6722 int err = -1;
6723
6724 ASSERT_RTNL();
6725
6726 features = netdev_get_wanted_features(dev);
6727
6728 if (dev->netdev_ops->ndo_fix_features)
6729 features = dev->netdev_ops->ndo_fix_features(dev, features);
6730
6731 /* driver might be less strict about feature dependencies */
6732 features = netdev_fix_features(dev, features);
6733
6734 /* some features can't be enabled if they're off an an upper device */
6735 netdev_for_each_upper_dev_rcu(dev, upper, iter)
6736 features = netdev_sync_upper_features(dev, upper, features);
6737
6738 if (dev->features == features)
6739 goto sync_lower;
6740
6741 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6742 &dev->features, &features);
6743
6744 if (dev->netdev_ops->ndo_set_features)
6745 err = dev->netdev_ops->ndo_set_features(dev, features);
6746 else
6747 err = 0;
6748
6749 if (unlikely(err < 0)) {
6750 netdev_err(dev,
6751 "set_features() failed (%d); wanted %pNF, left %pNF\n",
6752 err, &features, &dev->features);
6753 /* return non-0 since some features might have changed and
6754 * it's better to fire a spurious notification than miss it
6755 */
6756 return -1;
6757 }
6758
6759sync_lower:
6760 /* some features must be disabled on lower devices when disabled
6761 * on an upper device (think: bonding master or bridge)
6762 */
6763 netdev_for_each_lower_dev(dev, lower, iter)
6764 netdev_sync_lower_features(dev, lower, features);
6765
6766 if (!err)
6767 dev->features = features;
6768
6769 return err < 0 ? 0 : 1;
6770}
6771
6772/**
6773 * netdev_update_features - recalculate device features
6774 * @dev: the device to check
6775 *
6776 * Recalculate dev->features set and send notifications if it
6777 * has changed. Should be called after driver or hardware dependent
6778 * conditions might have changed that influence the features.
6779 */
6780void netdev_update_features(struct net_device *dev)
6781{
6782 if (__netdev_update_features(dev))
6783 netdev_features_change(dev);
6784}
6785EXPORT_SYMBOL(netdev_update_features);
6786
6787/**
6788 * netdev_change_features - recalculate device features
6789 * @dev: the device to check
6790 *
6791 * Recalculate dev->features set and send notifications even
6792 * if they have not changed. Should be called instead of
6793 * netdev_update_features() if also dev->vlan_features might
6794 * have changed to allow the changes to be propagated to stacked
6795 * VLAN devices.
6796 */
6797void netdev_change_features(struct net_device *dev)
6798{
6799 __netdev_update_features(dev);
6800 netdev_features_change(dev);
6801}
6802EXPORT_SYMBOL(netdev_change_features);
6803
6804/**
6805 * netif_stacked_transfer_operstate - transfer operstate
6806 * @rootdev: the root or lower level device to transfer state from
6807 * @dev: the device to transfer operstate to
6808 *
6809 * Transfer operational state from root to device. This is normally
6810 * called when a stacking relationship exists between the root
6811 * device and the device(a leaf device).
6812 */
6813void netif_stacked_transfer_operstate(const struct net_device *rootdev,
6814 struct net_device *dev)
6815{
6816 if (rootdev->operstate == IF_OPER_DORMANT)
6817 netif_dormant_on(dev);
6818 else
6819 netif_dormant_off(dev);
6820
6821 if (netif_carrier_ok(rootdev)) {
6822 if (!netif_carrier_ok(dev))
6823 netif_carrier_on(dev);
6824 } else {
6825 if (netif_carrier_ok(dev))
6826 netif_carrier_off(dev);
6827 }
6828}
6829EXPORT_SYMBOL(netif_stacked_transfer_operstate);
6830
6831#ifdef CONFIG_SYSFS
6832static int netif_alloc_rx_queues(struct net_device *dev)
6833{
6834 unsigned int i, count = dev->num_rx_queues;
6835 struct netdev_rx_queue *rx;
6836 size_t sz = count * sizeof(*rx);
6837
6838 BUG_ON(count < 1);
6839
6840 rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6841 if (!rx) {
6842 rx = vzalloc(sz);
6843 if (!rx)
6844 return -ENOMEM;
6845 }
6846 dev->_rx = rx;
6847
6848 for (i = 0; i < count; i++)
6849 rx[i].dev = dev;
6850 return 0;
6851}
6852#endif
6853
6854static void netdev_init_one_queue(struct net_device *dev,
6855 struct netdev_queue *queue, void *_unused)
6856{
6857 /* Initialize queue lock */
6858 spin_lock_init(&queue->_xmit_lock);
6859 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
6860 queue->xmit_lock_owner = -1;
6861 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
6862 queue->dev = dev;
6863#ifdef CONFIG_BQL
6864 dql_init(&queue->dql, HZ);
6865#endif
6866}
6867
6868static void netif_free_tx_queues(struct net_device *dev)
6869{
6870 kvfree(dev->_tx);
6871}
6872
6873static int netif_alloc_netdev_queues(struct net_device *dev)
6874{
6875 unsigned int count = dev->num_tx_queues;
6876 struct netdev_queue *tx;
6877 size_t sz = count * sizeof(*tx);
6878
6879 if (count < 1 || count > 0xffff)
6880 return -EINVAL;
6881
6882 tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6883 if (!tx) {
6884 tx = vzalloc(sz);
6885 if (!tx)
6886 return -ENOMEM;
6887 }
6888 dev->_tx = tx;
6889
6890 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
6891 spin_lock_init(&dev->tx_global_lock);
6892
6893 return 0;
6894}
6895
6896void netif_tx_stop_all_queues(struct net_device *dev)
6897{
6898 unsigned int i;
6899
6900 for (i = 0; i < dev->num_tx_queues; i++) {
6901 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
6902 netif_tx_stop_queue(txq);
6903 }
6904}
6905EXPORT_SYMBOL(netif_tx_stop_all_queues);
6906
6907/**
6908 * register_netdevice - register a network device
6909 * @dev: device to register
6910 *
6911 * Take a completed network device structure and add it to the kernel
6912 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6913 * chain. 0 is returned on success. A negative errno code is returned
6914 * on a failure to set up the device, or if the name is a duplicate.
6915 *
6916 * Callers must hold the rtnl semaphore. You may want
6917 * register_netdev() instead of this.
6918 *
6919 * BUGS:
6920 * The locking appears insufficient to guarantee two parallel registers
6921 * will not get the same name.
6922 */
6923
6924int register_netdevice(struct net_device *dev)
6925{
6926 int ret;
6927 struct net *net = dev_net(dev);
6928
6929 BUG_ON(dev_boot_phase);
6930 ASSERT_RTNL();
6931
6932 might_sleep();
6933
6934 /* When net_device's are persistent, this will be fatal. */
6935 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
6936 BUG_ON(!net);
6937
6938 spin_lock_init(&dev->addr_list_lock);
6939 netdev_set_addr_lockdep_class(dev);
6940
6941 ret = dev_get_valid_name(net, dev, dev->name);
6942 if (ret < 0)
6943 goto out;
6944
6945 /* Init, if this function is available */
6946 if (dev->netdev_ops->ndo_init) {
6947 ret = dev->netdev_ops->ndo_init(dev);
6948 if (ret) {
6949 if (ret > 0)
6950 ret = -EIO;
6951 goto out;
6952 }
6953 }
6954
6955 if (((dev->hw_features | dev->features) &
6956 NETIF_F_HW_VLAN_CTAG_FILTER) &&
6957 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6958 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6959 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6960 ret = -EINVAL;
6961 goto err_uninit;
6962 }
6963
6964 ret = -EBUSY;
6965 if (!dev->ifindex)
6966 dev->ifindex = dev_new_index(net);
6967 else if (__dev_get_by_index(net, dev->ifindex))
6968 goto err_uninit;
6969
6970 /* Transfer changeable features to wanted_features and enable
6971 * software offloads (GSO and GRO).
6972 */
6973 dev->hw_features |= NETIF_F_SOFT_FEATURES;
6974 dev->features |= NETIF_F_SOFT_FEATURES;
6975 dev->wanted_features = dev->features & dev->hw_features;
6976
6977 if (!(dev->flags & IFF_LOOPBACK)) {
6978 dev->hw_features |= NETIF_F_NOCACHE_COPY;
6979 }
6980
6981 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
6982 */
6983 dev->vlan_features |= NETIF_F_HIGHDMA;
6984
6985 /* Make NETIF_F_SG inheritable to tunnel devices.
6986 */
6987 dev->hw_enc_features |= NETIF_F_SG;
6988
6989 /* Make NETIF_F_SG inheritable to MPLS.
6990 */
6991 dev->mpls_features |= NETIF_F_SG;
6992
6993 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6994 ret = notifier_to_errno(ret);
6995 if (ret)
6996 goto err_uninit;
6997
6998 ret = netdev_register_kobject(dev);
6999 if (ret)
7000 goto err_uninit;
7001 dev->reg_state = NETREG_REGISTERED;
7002
7003 __netdev_update_features(dev);
7004
7005 /*
7006 * Default initial state at registry is that the
7007 * device is present.
7008 */
7009
7010 set_bit(__LINK_STATE_PRESENT, &dev->state);
7011
7012 linkwatch_init_dev(dev);
7013
7014 dev_init_scheduler(dev);
7015 dev_hold(dev);
7016 list_netdevice(dev);
7017 add_device_randomness(dev->dev_addr, dev->addr_len);
7018
7019 /* If the device has permanent device address, driver should
7020 * set dev_addr and also addr_assign_type should be set to
7021 * NET_ADDR_PERM (default value).
7022 */
7023 if (dev->addr_assign_type == NET_ADDR_PERM)
7024 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7025
7026 /* Notify protocols, that a new device appeared. */
7027 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
7028 ret = notifier_to_errno(ret);
7029 if (ret) {
7030 rollback_registered(dev);
7031 dev->reg_state = NETREG_UNREGISTERED;
7032 }
7033 /*
7034 * Prevent userspace races by waiting until the network
7035 * device is fully setup before sending notifications.
7036 */
7037 if (!dev->rtnl_link_ops ||
7038 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7039 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7040
7041out:
7042 return ret;
7043
7044err_uninit:
7045 if (dev->netdev_ops->ndo_uninit)
7046 dev->netdev_ops->ndo_uninit(dev);
7047 goto out;
7048}
7049EXPORT_SYMBOL(register_netdevice);
7050
7051/**
7052 * init_dummy_netdev - init a dummy network device for NAPI
7053 * @dev: device to init
7054 *
7055 * This takes a network device structure and initialize the minimum
7056 * amount of fields so it can be used to schedule NAPI polls without
7057 * registering a full blown interface. This is to be used by drivers
7058 * that need to tie several hardware interfaces to a single NAPI
7059 * poll scheduler due to HW limitations.
7060 */
7061int init_dummy_netdev(struct net_device *dev)
7062{
7063 /* Clear everything. Note we don't initialize spinlocks
7064 * are they aren't supposed to be taken by any of the
7065 * NAPI code and this dummy netdev is supposed to be
7066 * only ever used for NAPI polls
7067 */
7068 memset(dev, 0, sizeof(struct net_device));
7069
7070 /* make sure we BUG if trying to hit standard
7071 * register/unregister code path
7072 */
7073 dev->reg_state = NETREG_DUMMY;
7074
7075 /* NAPI wants this */
7076 INIT_LIST_HEAD(&dev->napi_list);
7077
7078 /* a dummy interface is started by default */
7079 set_bit(__LINK_STATE_PRESENT, &dev->state);
7080 set_bit(__LINK_STATE_START, &dev->state);
7081
7082 /* Note : We dont allocate pcpu_refcnt for dummy devices,
7083 * because users of this 'device' dont need to change
7084 * its refcount.
7085 */
7086
7087 return 0;
7088}
7089EXPORT_SYMBOL_GPL(init_dummy_netdev);
7090
7091
7092/**
7093 * register_netdev - register a network device
7094 * @dev: device to register
7095 *
7096 * Take a completed network device structure and add it to the kernel
7097 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7098 * chain. 0 is returned on success. A negative errno code is returned
7099 * on a failure to set up the device, or if the name is a duplicate.
7100 *
7101 * This is a wrapper around register_netdevice that takes the rtnl semaphore
7102 * and expands the device name if you passed a format string to
7103 * alloc_netdev.
7104 */
7105int register_netdev(struct net_device *dev)
7106{
7107 int err;
7108
7109 rtnl_lock();
7110 err = register_netdevice(dev);
7111 rtnl_unlock();
7112 return err;
7113}
7114EXPORT_SYMBOL(register_netdev);
7115
7116int netdev_refcnt_read(const struct net_device *dev)
7117{
7118 int i, refcnt = 0;
7119
7120 for_each_possible_cpu(i)
7121 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7122 return refcnt;
7123}
7124EXPORT_SYMBOL(netdev_refcnt_read);
7125
7126/**
7127 * netdev_wait_allrefs - wait until all references are gone.
7128 * @dev: target net_device
7129 *
7130 * This is called when unregistering network devices.
7131 *
7132 * Any protocol or device that holds a reference should register
7133 * for netdevice notification, and cleanup and put back the
7134 * reference if they receive an UNREGISTER event.
7135 * We can get stuck here if buggy protocols don't correctly
7136 * call dev_put.
7137 */
7138static void netdev_wait_allrefs(struct net_device *dev)
7139{
7140 unsigned long rebroadcast_time, warning_time;
7141 int refcnt;
7142
7143 linkwatch_forget_dev(dev);
7144
7145 rebroadcast_time = warning_time = jiffies;
7146 refcnt = netdev_refcnt_read(dev);
7147
7148 while (refcnt != 0) {
7149 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
7150 rtnl_lock();
7151
7152 /* Rebroadcast unregister notification */
7153 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7154
7155 __rtnl_unlock();
7156 rcu_barrier();
7157 rtnl_lock();
7158
7159 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7160 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7161 &dev->state)) {
7162 /* We must not have linkwatch events
7163 * pending on unregister. If this
7164 * happens, we simply run the queue
7165 * unscheduled, resulting in a noop
7166 * for this device.
7167 */
7168 linkwatch_run_queue();
7169 }
7170
7171 __rtnl_unlock();
7172
7173 rebroadcast_time = jiffies;
7174 }
7175
7176 msleep(250);
7177
7178 refcnt = netdev_refcnt_read(dev);
7179
7180 if (time_after(jiffies, warning_time + 10 * HZ)) {
7181 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7182 dev->name, refcnt);
7183 warning_time = jiffies;
7184 }
7185 }
7186}
7187
7188/* The sequence is:
7189 *
7190 * rtnl_lock();
7191 * ...
7192 * register_netdevice(x1);
7193 * register_netdevice(x2);
7194 * ...
7195 * unregister_netdevice(y1);
7196 * unregister_netdevice(y2);
7197 * ...
7198 * rtnl_unlock();
7199 * free_netdev(y1);
7200 * free_netdev(y2);
7201 *
7202 * We are invoked by rtnl_unlock().
7203 * This allows us to deal with problems:
7204 * 1) We can delete sysfs objects which invoke hotplug
7205 * without deadlocking with linkwatch via keventd.
7206 * 2) Since we run with the RTNL semaphore not held, we can sleep
7207 * safely in order to wait for the netdev refcnt to drop to zero.
7208 *
7209 * We must not return until all unregister events added during
7210 * the interval the lock was held have been completed.
7211 */
7212void netdev_run_todo(void)
7213{
7214 struct list_head list;
7215
7216 /* Snapshot list, allow later requests */
7217 list_replace_init(&net_todo_list, &list);
7218
7219 __rtnl_unlock();
7220
7221
7222 /* Wait for rcu callbacks to finish before next phase */
7223 if (!list_empty(&list))
7224 rcu_barrier();
7225
7226 while (!list_empty(&list)) {
7227 struct net_device *dev
7228 = list_first_entry(&list, struct net_device, todo_list);
7229 list_del(&dev->todo_list);
7230
7231 rtnl_lock();
7232 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7233 __rtnl_unlock();
7234
7235 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
7236 pr_err("network todo '%s' but state %d\n",
7237 dev->name, dev->reg_state);
7238 dump_stack();
7239 continue;
7240 }
7241
7242 dev->reg_state = NETREG_UNREGISTERED;
7243
7244 netdev_wait_allrefs(dev);
7245
7246 /* paranoia */
7247 BUG_ON(netdev_refcnt_read(dev));
7248 BUG_ON(!list_empty(&dev->ptype_all));
7249 BUG_ON(!list_empty(&dev->ptype_specific));
7250 WARN_ON(rcu_access_pointer(dev->ip_ptr));
7251 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
7252 WARN_ON(dev->dn_ptr);
7253
7254 if (dev->destructor)
7255 dev->destructor(dev);
7256
7257 /* Report a network device has been unregistered */
7258 rtnl_lock();
7259 dev_net(dev)->dev_unreg_count--;
7260 __rtnl_unlock();
7261 wake_up(&netdev_unregistering_wq);
7262
7263 /* Free network device */
7264 kobject_put(&dev->dev.kobj);
7265 }
7266}
7267
7268/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7269 * all the same fields in the same order as net_device_stats, with only
7270 * the type differing, but rtnl_link_stats64 may have additional fields
7271 * at the end for newer counters.
7272 */
7273void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7274 const struct net_device_stats *netdev_stats)
7275{
7276#if BITS_PER_LONG == 64
7277 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
7278 memcpy(stats64, netdev_stats, sizeof(*stats64));
7279 /* zero out counters that only exist in rtnl_link_stats64 */
7280 memset((char *)stats64 + sizeof(*netdev_stats), 0,
7281 sizeof(*stats64) - sizeof(*netdev_stats));
7282#else
7283 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
7284 const unsigned long *src = (const unsigned long *)netdev_stats;
7285 u64 *dst = (u64 *)stats64;
7286
7287 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
7288 for (i = 0; i < n; i++)
7289 dst[i] = src[i];
7290 /* zero out counters that only exist in rtnl_link_stats64 */
7291 memset((char *)stats64 + n * sizeof(u64), 0,
7292 sizeof(*stats64) - n * sizeof(u64));
7293#endif
7294}
7295EXPORT_SYMBOL(netdev_stats_to_stats64);
7296
7297/**
7298 * dev_get_stats - get network device statistics
7299 * @dev: device to get statistics from
7300 * @storage: place to store stats
7301 *
7302 * Get network statistics from device. Return @storage.
7303 * The device driver may provide its own method by setting
7304 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7305 * otherwise the internal statistics structure is used.
7306 */
7307struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7308 struct rtnl_link_stats64 *storage)
7309{
7310 const struct net_device_ops *ops = dev->netdev_ops;
7311
7312 if (ops->ndo_get_stats64) {
7313 memset(storage, 0, sizeof(*storage));
7314 ops->ndo_get_stats64(dev, storage);
7315 } else if (ops->ndo_get_stats) {
7316 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
7317 } else {
7318 netdev_stats_to_stats64(storage, &dev->stats);
7319 }
7320 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
7321 storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
7322 storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler);
7323 return storage;
7324}
7325EXPORT_SYMBOL(dev_get_stats);
7326
7327struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
7328{
7329 struct netdev_queue *queue = dev_ingress_queue(dev);
7330
7331#ifdef CONFIG_NET_CLS_ACT
7332 if (queue)
7333 return queue;
7334 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7335 if (!queue)
7336 return NULL;
7337 netdev_init_one_queue(dev, queue, NULL);
7338 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
7339 queue->qdisc_sleeping = &noop_qdisc;
7340 rcu_assign_pointer(dev->ingress_queue, queue);
7341#endif
7342 return queue;
7343}
7344
7345static const struct ethtool_ops default_ethtool_ops;
7346
7347void netdev_set_default_ethtool_ops(struct net_device *dev,
7348 const struct ethtool_ops *ops)
7349{
7350 if (dev->ethtool_ops == &default_ethtool_ops)
7351 dev->ethtool_ops = ops;
7352}
7353EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7354
7355void netdev_freemem(struct net_device *dev)
7356{
7357 char *addr = (char *)dev - dev->padded;
7358
7359 kvfree(addr);
7360}
7361
7362/**
7363 * alloc_netdev_mqs - allocate network device
7364 * @sizeof_priv: size of private data to allocate space for
7365 * @name: device name format string
7366 * @name_assign_type: origin of device name
7367 * @setup: callback to initialize device
7368 * @txqs: the number of TX subqueues to allocate
7369 * @rxqs: the number of RX subqueues to allocate
7370 *
7371 * Allocates a struct net_device with private data area for driver use
7372 * and performs basic initialization. Also allocates subqueue structs
7373 * for each queue on the device.
7374 */
7375struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
7376 unsigned char name_assign_type,
7377 void (*setup)(struct net_device *),
7378 unsigned int txqs, unsigned int rxqs)
7379{
7380 struct net_device *dev;
7381 size_t alloc_size;
7382 struct net_device *p;
7383
7384 BUG_ON(strlen(name) >= sizeof(dev->name));
7385
7386 if (txqs < 1) {
7387 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
7388 return NULL;
7389 }
7390
7391#ifdef CONFIG_SYSFS
7392 if (rxqs < 1) {
7393 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
7394 return NULL;
7395 }
7396#endif
7397
7398 alloc_size = sizeof(struct net_device);
7399 if (sizeof_priv) {
7400 /* ensure 32-byte alignment of private area */
7401 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
7402 alloc_size += sizeof_priv;
7403 }
7404 /* ensure 32-byte alignment of whole construct */
7405 alloc_size += NETDEV_ALIGN - 1;
7406
7407 p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7408 if (!p)
7409 p = vzalloc(alloc_size);
7410 if (!p)
7411 return NULL;
7412
7413 dev = PTR_ALIGN(p, NETDEV_ALIGN);
7414 dev->padded = (char *)dev - (char *)p;
7415
7416 dev->pcpu_refcnt = alloc_percpu(int);
7417 if (!dev->pcpu_refcnt)
7418 goto free_dev;
7419
7420 if (dev_addr_init(dev))
7421 goto free_pcpu;
7422
7423 dev_mc_init(dev);
7424 dev_uc_init(dev);
7425
7426 dev_net_set(dev, &init_net);
7427
7428 dev->gso_max_size = GSO_MAX_SIZE;
7429 dev->gso_max_segs = GSO_MAX_SEGS;
7430 dev->gso_min_segs = 0;
7431
7432 INIT_LIST_HEAD(&dev->napi_list);
7433 INIT_LIST_HEAD(&dev->unreg_list);
7434 INIT_LIST_HEAD(&dev->close_list);
7435 INIT_LIST_HEAD(&dev->link_watch_list);
7436 INIT_LIST_HEAD(&dev->adj_list.upper);
7437 INIT_LIST_HEAD(&dev->adj_list.lower);
7438 INIT_LIST_HEAD(&dev->all_adj_list.upper);
7439 INIT_LIST_HEAD(&dev->all_adj_list.lower);
7440 INIT_LIST_HEAD(&dev->ptype_all);
7441 INIT_LIST_HEAD(&dev->ptype_specific);
7442 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
7443 setup(dev);
7444
7445 if (!dev->tx_queue_len) {
7446 dev->priv_flags |= IFF_NO_QUEUE;
7447 dev->tx_queue_len = 1;
7448 }
7449
7450 dev->num_tx_queues = txqs;
7451 dev->real_num_tx_queues = txqs;
7452 if (netif_alloc_netdev_queues(dev))
7453 goto free_all;
7454
7455#ifdef CONFIG_SYSFS
7456 dev->num_rx_queues = rxqs;
7457 dev->real_num_rx_queues = rxqs;
7458 if (netif_alloc_rx_queues(dev))
7459 goto free_all;
7460#endif
7461
7462 strcpy(dev->name, name);
7463 dev->name_assign_type = name_assign_type;
7464 dev->group = INIT_NETDEV_GROUP;
7465 if (!dev->ethtool_ops)
7466 dev->ethtool_ops = &default_ethtool_ops;
7467
7468 nf_hook_ingress_init(dev);
7469
7470 return dev;
7471
7472free_all:
7473 free_netdev(dev);
7474 return NULL;
7475
7476free_pcpu:
7477 free_percpu(dev->pcpu_refcnt);
7478free_dev:
7479 netdev_freemem(dev);
7480 return NULL;
7481}
7482EXPORT_SYMBOL(alloc_netdev_mqs);
7483
7484/**
7485 * free_netdev - free network device
7486 * @dev: device
7487 *
7488 * This function does the last stage of destroying an allocated device
7489 * interface. The reference to the device object is released.
7490 * If this is the last reference then it will be freed.
7491 * Must be called in process context.
7492 */
7493void free_netdev(struct net_device *dev)
7494{
7495 struct napi_struct *p, *n;
7496
7497 might_sleep();
7498 netif_free_tx_queues(dev);
7499#ifdef CONFIG_SYSFS
7500 kvfree(dev->_rx);
7501#endif
7502
7503 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
7504
7505 /* Flush device addresses */
7506 dev_addr_flush(dev);
7507
7508 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7509 netif_napi_del(p);
7510
7511 free_percpu(dev->pcpu_refcnt);
7512 dev->pcpu_refcnt = NULL;
7513
7514 /* Compatibility with error handling in drivers */
7515 if (dev->reg_state == NETREG_UNINITIALIZED) {
7516 netdev_freemem(dev);
7517 return;
7518 }
7519
7520 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7521 dev->reg_state = NETREG_RELEASED;
7522
7523 /* will free via device release */
7524 put_device(&dev->dev);
7525}
7526EXPORT_SYMBOL(free_netdev);
7527
7528/**
7529 * synchronize_net - Synchronize with packet receive processing
7530 *
7531 * Wait for packets currently being received to be done.
7532 * Does not block later packets from starting.
7533 */
7534void synchronize_net(void)
7535{
7536 might_sleep();
7537 if (rtnl_is_locked())
7538 synchronize_rcu_expedited();
7539 else
7540 synchronize_rcu();
7541}
7542EXPORT_SYMBOL(synchronize_net);
7543
7544/**
7545 * unregister_netdevice_queue - remove device from the kernel
7546 * @dev: device
7547 * @head: list
7548 *
7549 * This function shuts down a device interface and removes it
7550 * from the kernel tables.
7551 * If head not NULL, device is queued to be unregistered later.
7552 *
7553 * Callers must hold the rtnl semaphore. You may want
7554 * unregister_netdev() instead of this.
7555 */
7556
7557void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
7558{
7559 ASSERT_RTNL();
7560
7561 if (head) {
7562 list_move_tail(&dev->unreg_list, head);
7563 } else {
7564 rollback_registered(dev);
7565 /* Finish processing unregister after unlock */
7566 net_set_todo(dev);
7567 }
7568}
7569EXPORT_SYMBOL(unregister_netdevice_queue);
7570
7571/**
7572 * unregister_netdevice_many - unregister many devices
7573 * @head: list of devices
7574 *
7575 * Note: As most callers use a stack allocated list_head,
7576 * we force a list_del() to make sure stack wont be corrupted later.
7577 */
7578void unregister_netdevice_many(struct list_head *head)
7579{
7580 struct net_device *dev;
7581
7582 if (!list_empty(head)) {
7583 rollback_registered_many(head);
7584 list_for_each_entry(dev, head, unreg_list)
7585 net_set_todo(dev);
7586 list_del(head);
7587 }
7588}
7589EXPORT_SYMBOL(unregister_netdevice_many);
7590
7591/**
7592 * unregister_netdev - remove device from the kernel
7593 * @dev: device
7594 *
7595 * This function shuts down a device interface and removes it
7596 * from the kernel tables.
7597 *
7598 * This is just a wrapper for unregister_netdevice that takes
7599 * the rtnl semaphore. In general you want to use this and not
7600 * unregister_netdevice.
7601 */
7602void unregister_netdev(struct net_device *dev)
7603{
7604 rtnl_lock();
7605 unregister_netdevice(dev);
7606 rtnl_unlock();
7607}
7608EXPORT_SYMBOL(unregister_netdev);
7609
7610/**
7611 * dev_change_net_namespace - move device to different nethost namespace
7612 * @dev: device
7613 * @net: network namespace
7614 * @pat: If not NULL name pattern to try if the current device name
7615 * is already taken in the destination network namespace.
7616 *
7617 * This function shuts down a device interface and moves it
7618 * to a new network namespace. On success 0 is returned, on
7619 * a failure a netagive errno code is returned.
7620 *
7621 * Callers must hold the rtnl semaphore.
7622 */
7623
7624int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7625{
7626 int err;
7627
7628 ASSERT_RTNL();
7629
7630 /* Don't allow namespace local devices to be moved. */
7631 err = -EINVAL;
7632 if (dev->features & NETIF_F_NETNS_LOCAL)
7633 goto out;
7634
7635 /* Ensure the device has been registrered */
7636 if (dev->reg_state != NETREG_REGISTERED)
7637 goto out;
7638
7639 /* Get out if there is nothing todo */
7640 err = 0;
7641 if (net_eq(dev_net(dev), net))
7642 goto out;
7643
7644 /* Pick the destination device name, and ensure
7645 * we can use it in the destination network namespace.
7646 */
7647 err = -EEXIST;
7648 if (__dev_get_by_name(net, dev->name)) {
7649 /* We get here if we can't use the current device name */
7650 if (!pat)
7651 goto out;
7652 if (dev_get_valid_name(net, dev, pat) < 0)
7653 goto out;
7654 }
7655
7656 /*
7657 * And now a mini version of register_netdevice unregister_netdevice.
7658 */
7659
7660 /* If device is running close it first. */
7661 dev_close(dev);
7662
7663 /* And unlink it from device chain */
7664 err = -ENODEV;
7665 unlist_netdevice(dev);
7666
7667 synchronize_net();
7668
7669 /* Shutdown queueing discipline. */
7670 dev_shutdown(dev);
7671
7672 /* Notify protocols, that we are about to destroy
7673 this device. They should clean all the things.
7674
7675 Note that dev->reg_state stays at NETREG_REGISTERED.
7676 This is wanted because this way 8021q and macvlan know
7677 the device is just moving and can keep their slaves up.
7678 */
7679 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7680 rcu_barrier();
7681 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7682 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
7683
7684 /*
7685 * Flush the unicast and multicast chains
7686 */
7687 dev_uc_flush(dev);
7688 dev_mc_flush(dev);
7689
7690 /* Send a netdev-removed uevent to the old namespace */
7691 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
7692 netdev_adjacent_del_links(dev);
7693
7694 /* Actually switch the network namespace */
7695 dev_net_set(dev, net);
7696
7697 /* If there is an ifindex conflict assign a new one */
7698 if (__dev_get_by_index(net, dev->ifindex))
7699 dev->ifindex = dev_new_index(net);
7700
7701 /* Send a netdev-add uevent to the new namespace */
7702 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
7703 netdev_adjacent_add_links(dev);
7704
7705 /* Fixup kobjects */
7706 err = device_rename(&dev->dev, dev->name);
7707 WARN_ON(err);
7708
7709 /* Add the device back in the hashes */
7710 list_netdevice(dev);
7711
7712 /* Notify protocols, that a new device appeared. */
7713 call_netdevice_notifiers(NETDEV_REGISTER, dev);
7714
7715 /*
7716 * Prevent userspace races by waiting until the network
7717 * device is fully setup before sending notifications.
7718 */
7719 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7720
7721 synchronize_net();
7722 err = 0;
7723out:
7724 return err;
7725}
7726EXPORT_SYMBOL_GPL(dev_change_net_namespace);
7727
7728static int dev_cpu_callback(struct notifier_block *nfb,
7729 unsigned long action,
7730 void *ocpu)
7731{
7732 struct sk_buff **list_skb;
7733 struct sk_buff *skb;
7734 unsigned int cpu, oldcpu = (unsigned long)ocpu;
7735 struct softnet_data *sd, *oldsd;
7736
7737 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
7738 return NOTIFY_OK;
7739
7740 local_irq_disable();
7741 cpu = smp_processor_id();
7742 sd = &per_cpu(softnet_data, cpu);
7743 oldsd = &per_cpu(softnet_data, oldcpu);
7744
7745 /* Find end of our completion_queue. */
7746 list_skb = &sd->completion_queue;
7747 while (*list_skb)
7748 list_skb = &(*list_skb)->next;
7749 /* Append completion queue from offline CPU. */
7750 *list_skb = oldsd->completion_queue;
7751 oldsd->completion_queue = NULL;
7752
7753 /* Append output queue from offline CPU. */
7754 if (oldsd->output_queue) {
7755 *sd->output_queue_tailp = oldsd->output_queue;
7756 sd->output_queue_tailp = oldsd->output_queue_tailp;
7757 oldsd->output_queue = NULL;
7758 oldsd->output_queue_tailp = &oldsd->output_queue;
7759 }
7760 /* Append NAPI poll list from offline CPU, with one exception :
7761 * process_backlog() must be called by cpu owning percpu backlog.
7762 * We properly handle process_queue & input_pkt_queue later.
7763 */
7764 while (!list_empty(&oldsd->poll_list)) {
7765 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7766 struct napi_struct,
7767 poll_list);
7768
7769 list_del_init(&napi->poll_list);
7770 if (napi->poll == process_backlog)
7771 napi->state = 0;
7772 else
7773 ____napi_schedule(sd, napi);
7774 }
7775
7776 raise_softirq_irqoff(NET_TX_SOFTIRQ);
7777 local_irq_enable();
7778
7779 /* Process offline CPU's input_pkt_queue */
7780 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
7781 netif_rx_ni(skb);
7782 input_queue_head_incr(oldsd);
7783 }
7784 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
7785 netif_rx_ni(skb);
7786 input_queue_head_incr(oldsd);
7787 }
7788
7789 return NOTIFY_OK;
7790}
7791
7792
7793/**
7794 * netdev_increment_features - increment feature set by one
7795 * @all: current feature set
7796 * @one: new feature set
7797 * @mask: mask feature set
7798 *
7799 * Computes a new feature set after adding a device with feature set
7800 * @one to the master device with current feature set @all. Will not
7801 * enable anything that is off in @mask. Returns the new feature set.
7802 */
7803netdev_features_t netdev_increment_features(netdev_features_t all,
7804 netdev_features_t one, netdev_features_t mask)
7805{
7806 if (mask & NETIF_F_HW_CSUM)
7807 mask |= NETIF_F_CSUM_MASK;
7808 mask |= NETIF_F_VLAN_CHALLENGED;
7809
7810 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
7811 all &= one | ~NETIF_F_ALL_FOR_ALL;
7812
7813 /* If one device supports hw checksumming, set for all. */
7814 if (all & NETIF_F_HW_CSUM)
7815 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
7816
7817 return all;
7818}
7819EXPORT_SYMBOL(netdev_increment_features);
7820
7821static struct hlist_head * __net_init netdev_create_hash(void)
7822{
7823 int i;
7824 struct hlist_head *hash;
7825
7826 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
7827 if (hash != NULL)
7828 for (i = 0; i < NETDEV_HASHENTRIES; i++)
7829 INIT_HLIST_HEAD(&hash[i]);
7830
7831 return hash;
7832}
7833
7834/* Initialize per network namespace state */
7835static int __net_init netdev_init(struct net *net)
7836{
7837 if (net != &init_net)
7838 INIT_LIST_HEAD(&net->dev_base_head);
7839
7840 net->dev_name_head = netdev_create_hash();
7841 if (net->dev_name_head == NULL)
7842 goto err_name;
7843
7844 net->dev_index_head = netdev_create_hash();
7845 if (net->dev_index_head == NULL)
7846 goto err_idx;
7847
7848 return 0;
7849
7850err_idx:
7851 kfree(net->dev_name_head);
7852err_name:
7853 return -ENOMEM;
7854}
7855
7856/**
7857 * netdev_drivername - network driver for the device
7858 * @dev: network device
7859 *
7860 * Determine network driver for device.
7861 */
7862const char *netdev_drivername(const struct net_device *dev)
7863{
7864 const struct device_driver *driver;
7865 const struct device *parent;
7866 const char *empty = "";
7867
7868 parent = dev->dev.parent;
7869 if (!parent)
7870 return empty;
7871
7872 driver = parent->driver;
7873 if (driver && driver->name)
7874 return driver->name;
7875 return empty;
7876}
7877
7878static void __netdev_printk(const char *level, const struct net_device *dev,
7879 struct va_format *vaf)
7880{
7881 if (dev && dev->dev.parent) {
7882 dev_printk_emit(level[1] - '0',
7883 dev->dev.parent,
7884 "%s %s %s%s: %pV",
7885 dev_driver_string(dev->dev.parent),
7886 dev_name(dev->dev.parent),
7887 netdev_name(dev), netdev_reg_state(dev),
7888 vaf);
7889 } else if (dev) {
7890 printk("%s%s%s: %pV",
7891 level, netdev_name(dev), netdev_reg_state(dev), vaf);
7892 } else {
7893 printk("%s(NULL net_device): %pV", level, vaf);
7894 }
7895}
7896
7897void netdev_printk(const char *level, const struct net_device *dev,
7898 const char *format, ...)
7899{
7900 struct va_format vaf;
7901 va_list args;
7902
7903 va_start(args, format);
7904
7905 vaf.fmt = format;
7906 vaf.va = &args;
7907
7908 __netdev_printk(level, dev, &vaf);
7909
7910 va_end(args);
7911}
7912EXPORT_SYMBOL(netdev_printk);
7913
7914#define define_netdev_printk_level(func, level) \
7915void func(const struct net_device *dev, const char *fmt, ...) \
7916{ \
7917 struct va_format vaf; \
7918 va_list args; \
7919 \
7920 va_start(args, fmt); \
7921 \
7922 vaf.fmt = fmt; \
7923 vaf.va = &args; \
7924 \
7925 __netdev_printk(level, dev, &vaf); \
7926 \
7927 va_end(args); \
7928} \
7929EXPORT_SYMBOL(func);
7930
7931define_netdev_printk_level(netdev_emerg, KERN_EMERG);
7932define_netdev_printk_level(netdev_alert, KERN_ALERT);
7933define_netdev_printk_level(netdev_crit, KERN_CRIT);
7934define_netdev_printk_level(netdev_err, KERN_ERR);
7935define_netdev_printk_level(netdev_warn, KERN_WARNING);
7936define_netdev_printk_level(netdev_notice, KERN_NOTICE);
7937define_netdev_printk_level(netdev_info, KERN_INFO);
7938
7939static void __net_exit netdev_exit(struct net *net)
7940{
7941 kfree(net->dev_name_head);
7942 kfree(net->dev_index_head);
7943}
7944
7945static struct pernet_operations __net_initdata netdev_net_ops = {
7946 .init = netdev_init,
7947 .exit = netdev_exit,
7948};
7949
7950static void __net_exit default_device_exit(struct net *net)
7951{
7952 struct net_device *dev, *aux;
7953 /*
7954 * Push all migratable network devices back to the
7955 * initial network namespace
7956 */
7957 rtnl_lock();
7958 for_each_netdev_safe(net, dev, aux) {
7959 int err;
7960 char fb_name[IFNAMSIZ];
7961
7962 /* Ignore unmoveable devices (i.e. loopback) */
7963 if (dev->features & NETIF_F_NETNS_LOCAL)
7964 continue;
7965
7966 /* Leave virtual devices for the generic cleanup */
7967 if (dev->rtnl_link_ops)
7968 continue;
7969
7970 /* Push remaining network devices to init_net */
7971 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7972 err = dev_change_net_namespace(dev, &init_net, fb_name);
7973 if (err) {
7974 pr_emerg("%s: failed to move %s to init_net: %d\n",
7975 __func__, dev->name, err);
7976 BUG();
7977 }
7978 }
7979 rtnl_unlock();
7980}
7981
7982static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7983{
7984 /* Return with the rtnl_lock held when there are no network
7985 * devices unregistering in any network namespace in net_list.
7986 */
7987 struct net *net;
7988 bool unregistering;
7989 DEFINE_WAIT_FUNC(wait, woken_wake_function);
7990
7991 add_wait_queue(&netdev_unregistering_wq, &wait);
7992 for (;;) {
7993 unregistering = false;
7994 rtnl_lock();
7995 list_for_each_entry(net, net_list, exit_list) {
7996 if (net->dev_unreg_count > 0) {
7997 unregistering = true;
7998 break;
7999 }
8000 }
8001 if (!unregistering)
8002 break;
8003 __rtnl_unlock();
8004
8005 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
8006 }
8007 remove_wait_queue(&netdev_unregistering_wq, &wait);
8008}
8009
8010static void __net_exit default_device_exit_batch(struct list_head *net_list)
8011{
8012 /* At exit all network devices most be removed from a network
8013 * namespace. Do this in the reverse order of registration.
8014 * Do this across as many network namespaces as possible to
8015 * improve batching efficiency.
8016 */
8017 struct net_device *dev;
8018 struct net *net;
8019 LIST_HEAD(dev_kill_list);
8020
8021 /* To prevent network device cleanup code from dereferencing
8022 * loopback devices or network devices that have been freed
8023 * wait here for all pending unregistrations to complete,
8024 * before unregistring the loopback device and allowing the
8025 * network namespace be freed.
8026 *
8027 * The netdev todo list containing all network devices
8028 * unregistrations that happen in default_device_exit_batch
8029 * will run in the rtnl_unlock() at the end of
8030 * default_device_exit_batch.
8031 */
8032 rtnl_lock_unregistering(net_list);
8033 list_for_each_entry(net, net_list, exit_list) {
8034 for_each_netdev_reverse(net, dev) {
8035 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
8036 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8037 else
8038 unregister_netdevice_queue(dev, &dev_kill_list);
8039 }
8040 }
8041 unregister_netdevice_many(&dev_kill_list);
8042 rtnl_unlock();
8043}
8044
8045static struct pernet_operations __net_initdata default_device_ops = {
8046 .exit = default_device_exit,
8047 .exit_batch = default_device_exit_batch,
8048};
8049
8050/*
8051 * Initialize the DEV module. At boot time this walks the device list and
8052 * unhooks any devices that fail to initialise (normally hardware not
8053 * present) and leaves us with a valid list of present and active devices.
8054 *
8055 */
8056
8057/*
8058 * This is called single threaded during boot, so no need
8059 * to take the rtnl semaphore.
8060 */
8061static int __init net_dev_init(void)
8062{
8063 int i, rc = -ENOMEM;
8064
8065 BUG_ON(!dev_boot_phase);
8066
8067 if (dev_proc_init())
8068 goto out;
8069
8070 if (netdev_kobject_init())
8071 goto out;
8072
8073 INIT_LIST_HEAD(&ptype_all);
8074 for (i = 0; i < PTYPE_HASH_SIZE; i++)
8075 INIT_LIST_HEAD(&ptype_base[i]);
8076
8077 INIT_LIST_HEAD(&offload_base);
8078
8079 if (register_pernet_subsys(&netdev_net_ops))
8080 goto out;
8081
8082 /*
8083 * Initialise the packet receive queues.
8084 */
8085
8086 for_each_possible_cpu(i) {
8087 struct softnet_data *sd = &per_cpu(softnet_data, i);
8088
8089 skb_queue_head_init(&sd->input_pkt_queue);
8090 skb_queue_head_init(&sd->process_queue);
8091 INIT_LIST_HEAD(&sd->poll_list);
8092 sd->output_queue_tailp = &sd->output_queue;
8093#ifdef CONFIG_RPS
8094 sd->csd.func = rps_trigger_softirq;
8095 sd->csd.info = sd;
8096 sd->cpu = i;
8097#endif
8098
8099 sd->backlog.poll = process_backlog;
8100 sd->backlog.weight = weight_p;
8101 }
8102
8103 dev_boot_phase = 0;
8104
8105 /* The loopback device is special if any other network devices
8106 * is present in a network namespace the loopback device must
8107 * be present. Since we now dynamically allocate and free the
8108 * loopback device ensure this invariant is maintained by
8109 * keeping the loopback device as the first device on the
8110 * list of network devices. Ensuring the loopback devices
8111 * is the first device that appears and the last network device
8112 * that disappears.
8113 */
8114 if (register_pernet_device(&loopback_net_ops))
8115 goto out;
8116
8117 if (register_pernet_device(&default_device_ops))
8118 goto out;
8119
8120 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8121 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
8122
8123 hotcpu_notifier(dev_cpu_callback, 0);
8124 dst_subsys_init();
8125 rc = 0;
8126out:
8127 return rc;
8128}
8129
8130subsys_initcall(net_dev_init);