Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * NET3 Protocol independent device support routines.
4 *
5 * Derived from the non IP parts of dev.c 1.0.19
6 * Authors: Ross Biro
7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8 * Mark Evans, <evansmp@uhura.aston.ac.uk>
9 *
10 * Additional Authors:
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
12 * Alan Cox <gw4pts@gw4pts.ampr.org>
13 * David Hinds <dahinds@users.sourceforge.net>
14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15 * Adam Sulmicki <adam@cfar.umd.edu>
16 * Pekka Riikonen <priikone@poesidon.pspt.fi>
17 *
18 * Changes:
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
20 * to 2 if register_netdev gets called
21 * before net_dev_init & also removed a
22 * few lines of code in the process.
23 * Alan Cox : device private ioctl copies fields back.
24 * Alan Cox : Transmit queue code does relevant
25 * stunts to keep the queue safe.
26 * Alan Cox : Fixed double lock.
27 * Alan Cox : Fixed promisc NULL pointer trap
28 * ???????? : Support the full private ioctl range
29 * Alan Cox : Moved ioctl permission check into
30 * drivers
31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
32 * Alan Cox : 100 backlog just doesn't cut it when
33 * you start doing multicast video 8)
34 * Alan Cox : Rewrote net_bh and list manager.
35 * Alan Cox : Fix ETH_P_ALL echoback lengths.
36 * Alan Cox : Took out transmit every packet pass
37 * Saved a few bytes in the ioctl handler
38 * Alan Cox : Network driver sets packet type before
39 * calling netif_rx. Saves a function
40 * call a packet.
41 * Alan Cox : Hashed net_bh()
42 * Richard Kooijman: Timestamp fixes.
43 * Alan Cox : Wrong field in SIOCGIFDSTADDR
44 * Alan Cox : Device lock protection.
45 * Alan Cox : Fixed nasty side effect of device close
46 * changes.
47 * Rudi Cilibrasi : Pass the right thing to
48 * set_mac_address()
49 * Dave Miller : 32bit quantity for the device lock to
50 * make it work out on a Sparc.
51 * Bjorn Ekwall : Added KERNELD hack.
52 * Alan Cox : Cleaned up the backlog initialise.
53 * Craig Metz : SIOCGIFCONF fix if space for under
54 * 1 device.
55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
56 * is no device open function.
57 * Andi Kleen : Fix error reporting for SIOCGIFCONF
58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
59 * Cyrus Durgin : Cleaned for KMOD
60 * Adam Sulmicki : Bug Fix : Network Device Unload
61 * A network device unload needs to purge
62 * the backlog queue.
63 * Paul Rusty Russell : SIOCSIFNAME
64 * Pekka Riikonen : Netdev boot-time settings code
65 * Andrew Morton : Make unregister_netdevice wait
66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
68 * - netif_rx() feedback
69 */
70
71#include <linux/uaccess.h>
72#include <linux/bitmap.h>
73#include <linux/capability.h>
74#include <linux/cpu.h>
75#include <linux/types.h>
76#include <linux/kernel.h>
77#include <linux/hash.h>
78#include <linux/slab.h>
79#include <linux/sched.h>
80#include <linux/sched/isolation.h>
81#include <linux/sched/mm.h>
82#include <linux/smpboot.h>
83#include <linux/mutex.h>
84#include <linux/rwsem.h>
85#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
94#include <linux/ethtool.h>
95#include <linux/skbuff.h>
96#include <linux/kthread.h>
97#include <linux/bpf.h>
98#include <linux/bpf_trace.h>
99#include <net/net_namespace.h>
100#include <net/sock.h>
101#include <net/busy_poll.h>
102#include <linux/rtnetlink.h>
103#include <linux/stat.h>
104#include <net/dsa.h>
105#include <net/dst.h>
106#include <net/dst_metadata.h>
107#include <net/gro.h>
108#include <net/pkt_sched.h>
109#include <net/pkt_cls.h>
110#include <net/checksum.h>
111#include <net/xfrm.h>
112#include <net/tcx.h>
113#include <linux/highmem.h>
114#include <linux/init.h>
115#include <linux/module.h>
116#include <linux/netpoll.h>
117#include <linux/rcupdate.h>
118#include <linux/delay.h>
119#include <net/iw_handler.h>
120#include <asm/current.h>
121#include <linux/audit.h>
122#include <linux/dmaengine.h>
123#include <linux/err.h>
124#include <linux/ctype.h>
125#include <linux/if_arp.h>
126#include <linux/if_vlan.h>
127#include <linux/ip.h>
128#include <net/ip.h>
129#include <net/mpls.h>
130#include <linux/ipv6.h>
131#include <linux/in.h>
132#include <linux/jhash.h>
133#include <linux/random.h>
134#include <trace/events/napi.h>
135#include <trace/events/net.h>
136#include <trace/events/skb.h>
137#include <trace/events/qdisc.h>
138#include <trace/events/xdp.h>
139#include <linux/inetdevice.h>
140#include <linux/cpu_rmap.h>
141#include <linux/static_key.h>
142#include <linux/hashtable.h>
143#include <linux/vmalloc.h>
144#include <linux/if_macvlan.h>
145#include <linux/errqueue.h>
146#include <linux/hrtimer.h>
147#include <linux/netfilter_netdev.h>
148#include <linux/crash_dump.h>
149#include <linux/sctp.h>
150#include <net/udp_tunnel.h>
151#include <linux/net_namespace.h>
152#include <linux/indirect_call_wrapper.h>
153#include <net/devlink.h>
154#include <linux/pm_runtime.h>
155#include <linux/prandom.h>
156#include <linux/once_lite.h>
157#include <net/netdev_rx_queue.h>
158#include <net/page_pool/types.h>
159#include <net/page_pool/helpers.h>
160#include <net/rps.h>
161#include <linux/phy_link_topology.h>
162
163#include "dev.h"
164#include "devmem.h"
165#include "net-sysfs.h"
166
167static DEFINE_SPINLOCK(ptype_lock);
168struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
169
170static int netif_rx_internal(struct sk_buff *skb);
171static int call_netdevice_notifiers_extack(unsigned long val,
172 struct net_device *dev,
173 struct netlink_ext_ack *extack);
174
175static DEFINE_MUTEX(ifalias_mutex);
176
177/* protects napi_hash addition/deletion and napi_gen_id */
178static DEFINE_SPINLOCK(napi_hash_lock);
179
180static unsigned int napi_gen_id = NR_CPUS;
181static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
182
183static DECLARE_RWSEM(devnet_rename_sem);
184
185static inline void dev_base_seq_inc(struct net *net)
186{
187 unsigned int val = net->dev_base_seq + 1;
188
189 WRITE_ONCE(net->dev_base_seq, val ?: 1);
190}
191
192static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
193{
194 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
195
196 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
197}
198
199static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
200{
201 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
202}
203
204#ifndef CONFIG_PREEMPT_RT
205
206static DEFINE_STATIC_KEY_FALSE(use_backlog_threads_key);
207
208static int __init setup_backlog_napi_threads(char *arg)
209{
210 static_branch_enable(&use_backlog_threads_key);
211 return 0;
212}
213early_param("thread_backlog_napi", setup_backlog_napi_threads);
214
215static bool use_backlog_threads(void)
216{
217 return static_branch_unlikely(&use_backlog_threads_key);
218}
219
220#else
221
222static bool use_backlog_threads(void)
223{
224 return true;
225}
226
227#endif
228
229static inline void backlog_lock_irq_save(struct softnet_data *sd,
230 unsigned long *flags)
231{
232 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
233 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
234 else
235 local_irq_save(*flags);
236}
237
238static inline void backlog_lock_irq_disable(struct softnet_data *sd)
239{
240 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
241 spin_lock_irq(&sd->input_pkt_queue.lock);
242 else
243 local_irq_disable();
244}
245
246static inline void backlog_unlock_irq_restore(struct softnet_data *sd,
247 unsigned long *flags)
248{
249 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
250 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
251 else
252 local_irq_restore(*flags);
253}
254
255static inline void backlog_unlock_irq_enable(struct softnet_data *sd)
256{
257 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
258 spin_unlock_irq(&sd->input_pkt_queue.lock);
259 else
260 local_irq_enable();
261}
262
263static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
264 const char *name)
265{
266 struct netdev_name_node *name_node;
267
268 name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
269 if (!name_node)
270 return NULL;
271 INIT_HLIST_NODE(&name_node->hlist);
272 name_node->dev = dev;
273 name_node->name = name;
274 return name_node;
275}
276
277static struct netdev_name_node *
278netdev_name_node_head_alloc(struct net_device *dev)
279{
280 struct netdev_name_node *name_node;
281
282 name_node = netdev_name_node_alloc(dev, dev->name);
283 if (!name_node)
284 return NULL;
285 INIT_LIST_HEAD(&name_node->list);
286 return name_node;
287}
288
289static void netdev_name_node_free(struct netdev_name_node *name_node)
290{
291 kfree(name_node);
292}
293
294static void netdev_name_node_add(struct net *net,
295 struct netdev_name_node *name_node)
296{
297 hlist_add_head_rcu(&name_node->hlist,
298 dev_name_hash(net, name_node->name));
299}
300
301static void netdev_name_node_del(struct netdev_name_node *name_node)
302{
303 hlist_del_rcu(&name_node->hlist);
304}
305
306static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
307 const char *name)
308{
309 struct hlist_head *head = dev_name_hash(net, name);
310 struct netdev_name_node *name_node;
311
312 hlist_for_each_entry(name_node, head, hlist)
313 if (!strcmp(name_node->name, name))
314 return name_node;
315 return NULL;
316}
317
318static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
319 const char *name)
320{
321 struct hlist_head *head = dev_name_hash(net, name);
322 struct netdev_name_node *name_node;
323
324 hlist_for_each_entry_rcu(name_node, head, hlist)
325 if (!strcmp(name_node->name, name))
326 return name_node;
327 return NULL;
328}
329
330bool netdev_name_in_use(struct net *net, const char *name)
331{
332 return netdev_name_node_lookup(net, name);
333}
334EXPORT_SYMBOL(netdev_name_in_use);
335
336int netdev_name_node_alt_create(struct net_device *dev, const char *name)
337{
338 struct netdev_name_node *name_node;
339 struct net *net = dev_net(dev);
340
341 name_node = netdev_name_node_lookup(net, name);
342 if (name_node)
343 return -EEXIST;
344 name_node = netdev_name_node_alloc(dev, name);
345 if (!name_node)
346 return -ENOMEM;
347 netdev_name_node_add(net, name_node);
348 /* The node that holds dev->name acts as a head of per-device list. */
349 list_add_tail_rcu(&name_node->list, &dev->name_node->list);
350
351 return 0;
352}
353
354static void netdev_name_node_alt_free(struct rcu_head *head)
355{
356 struct netdev_name_node *name_node =
357 container_of(head, struct netdev_name_node, rcu);
358
359 kfree(name_node->name);
360 netdev_name_node_free(name_node);
361}
362
363static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
364{
365 netdev_name_node_del(name_node);
366 list_del(&name_node->list);
367 call_rcu(&name_node->rcu, netdev_name_node_alt_free);
368}
369
370int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
371{
372 struct netdev_name_node *name_node;
373 struct net *net = dev_net(dev);
374
375 name_node = netdev_name_node_lookup(net, name);
376 if (!name_node)
377 return -ENOENT;
378 /* lookup might have found our primary name or a name belonging
379 * to another device.
380 */
381 if (name_node == dev->name_node || name_node->dev != dev)
382 return -EINVAL;
383
384 __netdev_name_node_alt_destroy(name_node);
385 return 0;
386}
387
388static void netdev_name_node_alt_flush(struct net_device *dev)
389{
390 struct netdev_name_node *name_node, *tmp;
391
392 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) {
393 list_del(&name_node->list);
394 netdev_name_node_alt_free(&name_node->rcu);
395 }
396}
397
398/* Device list insertion */
399static void list_netdevice(struct net_device *dev)
400{
401 struct netdev_name_node *name_node;
402 struct net *net = dev_net(dev);
403
404 ASSERT_RTNL();
405
406 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
407 netdev_name_node_add(net, dev->name_node);
408 hlist_add_head_rcu(&dev->index_hlist,
409 dev_index_hash(net, dev->ifindex));
410
411 netdev_for_each_altname(dev, name_node)
412 netdev_name_node_add(net, name_node);
413
414 /* We reserved the ifindex, this can't fail */
415 WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL));
416
417 dev_base_seq_inc(net);
418}
419
420/* Device list removal
421 * caller must respect a RCU grace period before freeing/reusing dev
422 */
423static void unlist_netdevice(struct net_device *dev)
424{
425 struct netdev_name_node *name_node;
426 struct net *net = dev_net(dev);
427
428 ASSERT_RTNL();
429
430 xa_erase(&net->dev_by_index, dev->ifindex);
431
432 netdev_for_each_altname(dev, name_node)
433 netdev_name_node_del(name_node);
434
435 /* Unlink dev from the device chain */
436 list_del_rcu(&dev->dev_list);
437 netdev_name_node_del(dev->name_node);
438 hlist_del_rcu(&dev->index_hlist);
439
440 dev_base_seq_inc(dev_net(dev));
441}
442
443/*
444 * Our notifier list
445 */
446
447static RAW_NOTIFIER_HEAD(netdev_chain);
448
449/*
450 * Device drivers call our routines to queue packets here. We empty the
451 * queue in the local softnet handler.
452 */
453
454DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data) = {
455 .process_queue_bh_lock = INIT_LOCAL_LOCK(process_queue_bh_lock),
456};
457EXPORT_PER_CPU_SYMBOL(softnet_data);
458
459/* Page_pool has a lockless array/stack to alloc/recycle pages.
460 * PP consumers must pay attention to run APIs in the appropriate context
461 * (e.g. NAPI context).
462 */
463static DEFINE_PER_CPU(struct page_pool *, system_page_pool);
464
465#ifdef CONFIG_LOCKDEP
466/*
467 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
468 * according to dev->type
469 */
470static const unsigned short netdev_lock_type[] = {
471 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
472 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
473 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
474 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
475 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
476 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
477 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
478 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
479 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
480 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
481 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
482 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
483 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
484 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
485 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
486
487static const char *const netdev_lock_name[] = {
488 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
489 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
490 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
491 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
492 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
493 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
494 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
495 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
496 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
497 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
498 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
499 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
500 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
501 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
502 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
503
504static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
505static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
506
507static inline unsigned short netdev_lock_pos(unsigned short dev_type)
508{
509 int i;
510
511 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
512 if (netdev_lock_type[i] == dev_type)
513 return i;
514 /* the last key is used by default */
515 return ARRAY_SIZE(netdev_lock_type) - 1;
516}
517
518static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
519 unsigned short dev_type)
520{
521 int i;
522
523 i = netdev_lock_pos(dev_type);
524 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
525 netdev_lock_name[i]);
526}
527
528static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
529{
530 int i;
531
532 i = netdev_lock_pos(dev->type);
533 lockdep_set_class_and_name(&dev->addr_list_lock,
534 &netdev_addr_lock_key[i],
535 netdev_lock_name[i]);
536}
537#else
538static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
539 unsigned short dev_type)
540{
541}
542
543static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
544{
545}
546#endif
547
548/*******************************************************************************
549 *
550 * Protocol management and registration routines
551 *
552 *******************************************************************************/
553
554
555/*
556 * Add a protocol ID to the list. Now that the input handler is
557 * smarter we can dispense with all the messy stuff that used to be
558 * here.
559 *
560 * BEWARE!!! Protocol handlers, mangling input packets,
561 * MUST BE last in hash buckets and checking protocol handlers
562 * MUST start from promiscuous ptype_all chain in net_bh.
563 * It is true now, do not change it.
564 * Explanation follows: if protocol handler, mangling packet, will
565 * be the first on list, it is not able to sense, that packet
566 * is cloned and should be copied-on-write, so that it will
567 * change it and subsequent readers will get broken packet.
568 * --ANK (980803)
569 */
570
571static inline struct list_head *ptype_head(const struct packet_type *pt)
572{
573 if (pt->type == htons(ETH_P_ALL))
574 return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all;
575 else
576 return pt->dev ? &pt->dev->ptype_specific :
577 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
578}
579
580/**
581 * dev_add_pack - add packet handler
582 * @pt: packet type declaration
583 *
584 * Add a protocol handler to the networking stack. The passed &packet_type
585 * is linked into kernel lists and may not be freed until it has been
586 * removed from the kernel lists.
587 *
588 * This call does not sleep therefore it can not
589 * guarantee all CPU's that are in middle of receiving packets
590 * will see the new packet type (until the next received packet).
591 */
592
593void dev_add_pack(struct packet_type *pt)
594{
595 struct list_head *head = ptype_head(pt);
596
597 spin_lock(&ptype_lock);
598 list_add_rcu(&pt->list, head);
599 spin_unlock(&ptype_lock);
600}
601EXPORT_SYMBOL(dev_add_pack);
602
603/**
604 * __dev_remove_pack - remove packet handler
605 * @pt: packet type declaration
606 *
607 * Remove a protocol handler that was previously added to the kernel
608 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
609 * from the kernel lists and can be freed or reused once this function
610 * returns.
611 *
612 * The packet type might still be in use by receivers
613 * and must not be freed until after all the CPU's have gone
614 * through a quiescent state.
615 */
616void __dev_remove_pack(struct packet_type *pt)
617{
618 struct list_head *head = ptype_head(pt);
619 struct packet_type *pt1;
620
621 spin_lock(&ptype_lock);
622
623 list_for_each_entry(pt1, head, list) {
624 if (pt == pt1) {
625 list_del_rcu(&pt->list);
626 goto out;
627 }
628 }
629
630 pr_warn("dev_remove_pack: %p not found\n", pt);
631out:
632 spin_unlock(&ptype_lock);
633}
634EXPORT_SYMBOL(__dev_remove_pack);
635
636/**
637 * dev_remove_pack - remove packet handler
638 * @pt: packet type declaration
639 *
640 * Remove a protocol handler that was previously added to the kernel
641 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
642 * from the kernel lists and can be freed or reused once this function
643 * returns.
644 *
645 * This call sleeps to guarantee that no CPU is looking at the packet
646 * type after return.
647 */
648void dev_remove_pack(struct packet_type *pt)
649{
650 __dev_remove_pack(pt);
651
652 synchronize_net();
653}
654EXPORT_SYMBOL(dev_remove_pack);
655
656
657/*******************************************************************************
658 *
659 * Device Interface Subroutines
660 *
661 *******************************************************************************/
662
663/**
664 * dev_get_iflink - get 'iflink' value of a interface
665 * @dev: targeted interface
666 *
667 * Indicates the ifindex the interface is linked to.
668 * Physical interfaces have the same 'ifindex' and 'iflink' values.
669 */
670
671int dev_get_iflink(const struct net_device *dev)
672{
673 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
674 return dev->netdev_ops->ndo_get_iflink(dev);
675
676 return READ_ONCE(dev->ifindex);
677}
678EXPORT_SYMBOL(dev_get_iflink);
679
680/**
681 * dev_fill_metadata_dst - Retrieve tunnel egress information.
682 * @dev: targeted interface
683 * @skb: The packet.
684 *
685 * For better visibility of tunnel traffic OVS needs to retrieve
686 * egress tunnel information for a packet. Following API allows
687 * user to get this info.
688 */
689int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
690{
691 struct ip_tunnel_info *info;
692
693 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
694 return -EINVAL;
695
696 info = skb_tunnel_info_unclone(skb);
697 if (!info)
698 return -ENOMEM;
699 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
700 return -EINVAL;
701
702 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
703}
704EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
705
706static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
707{
708 int k = stack->num_paths++;
709
710 if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
711 return NULL;
712
713 return &stack->path[k];
714}
715
716int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
717 struct net_device_path_stack *stack)
718{
719 const struct net_device *last_dev;
720 struct net_device_path_ctx ctx = {
721 .dev = dev,
722 };
723 struct net_device_path *path;
724 int ret = 0;
725
726 memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
727 stack->num_paths = 0;
728 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
729 last_dev = ctx.dev;
730 path = dev_fwd_path(stack);
731 if (!path)
732 return -1;
733
734 memset(path, 0, sizeof(struct net_device_path));
735 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
736 if (ret < 0)
737 return -1;
738
739 if (WARN_ON_ONCE(last_dev == ctx.dev))
740 return -1;
741 }
742
743 if (!ctx.dev)
744 return ret;
745
746 path = dev_fwd_path(stack);
747 if (!path)
748 return -1;
749 path->type = DEV_PATH_ETHERNET;
750 path->dev = ctx.dev;
751
752 return ret;
753}
754EXPORT_SYMBOL_GPL(dev_fill_forward_path);
755
756/* must be called under rcu_read_lock(), as we dont take a reference */
757static struct napi_struct *napi_by_id(unsigned int napi_id)
758{
759 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
760 struct napi_struct *napi;
761
762 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
763 if (napi->napi_id == napi_id)
764 return napi;
765
766 return NULL;
767}
768
769/* must be called under rcu_read_lock(), as we dont take a reference */
770struct napi_struct *netdev_napi_by_id(struct net *net, unsigned int napi_id)
771{
772 struct napi_struct *napi;
773
774 napi = napi_by_id(napi_id);
775 if (!napi)
776 return NULL;
777
778 if (WARN_ON_ONCE(!napi->dev))
779 return NULL;
780 if (!net_eq(net, dev_net(napi->dev)))
781 return NULL;
782
783 return napi;
784}
785
786/**
787 * __dev_get_by_name - find a device by its name
788 * @net: the applicable net namespace
789 * @name: name to find
790 *
791 * Find an interface by name. Must be called under RTNL semaphore.
792 * If the name is found a pointer to the device is returned.
793 * If the name is not found then %NULL is returned. The
794 * reference counters are not incremented so the caller must be
795 * careful with locks.
796 */
797
798struct net_device *__dev_get_by_name(struct net *net, const char *name)
799{
800 struct netdev_name_node *node_name;
801
802 node_name = netdev_name_node_lookup(net, name);
803 return node_name ? node_name->dev : NULL;
804}
805EXPORT_SYMBOL(__dev_get_by_name);
806
807/**
808 * dev_get_by_name_rcu - find a device by its name
809 * @net: the applicable net namespace
810 * @name: name to find
811 *
812 * Find an interface by name.
813 * If the name is found a pointer to the device is returned.
814 * If the name is not found then %NULL is returned.
815 * The reference counters are not incremented so the caller must be
816 * careful with locks. The caller must hold RCU lock.
817 */
818
819struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
820{
821 struct netdev_name_node *node_name;
822
823 node_name = netdev_name_node_lookup_rcu(net, name);
824 return node_name ? node_name->dev : NULL;
825}
826EXPORT_SYMBOL(dev_get_by_name_rcu);
827
828/* Deprecated for new users, call netdev_get_by_name() instead */
829struct net_device *dev_get_by_name(struct net *net, const char *name)
830{
831 struct net_device *dev;
832
833 rcu_read_lock();
834 dev = dev_get_by_name_rcu(net, name);
835 dev_hold(dev);
836 rcu_read_unlock();
837 return dev;
838}
839EXPORT_SYMBOL(dev_get_by_name);
840
841/**
842 * netdev_get_by_name() - find a device by its name
843 * @net: the applicable net namespace
844 * @name: name to find
845 * @tracker: tracking object for the acquired reference
846 * @gfp: allocation flags for the tracker
847 *
848 * Find an interface by name. This can be called from any
849 * context and does its own locking. The returned handle has
850 * the usage count incremented and the caller must use netdev_put() to
851 * release it when it is no longer needed. %NULL is returned if no
852 * matching device is found.
853 */
854struct net_device *netdev_get_by_name(struct net *net, const char *name,
855 netdevice_tracker *tracker, gfp_t gfp)
856{
857 struct net_device *dev;
858
859 dev = dev_get_by_name(net, name);
860 if (dev)
861 netdev_tracker_alloc(dev, tracker, gfp);
862 return dev;
863}
864EXPORT_SYMBOL(netdev_get_by_name);
865
866/**
867 * __dev_get_by_index - find a device by its ifindex
868 * @net: the applicable net namespace
869 * @ifindex: index of device
870 *
871 * Search for an interface by index. Returns %NULL if the device
872 * is not found or a pointer to the device. The device has not
873 * had its reference counter increased so the caller must be careful
874 * about locking. The caller must hold the RTNL semaphore.
875 */
876
877struct net_device *__dev_get_by_index(struct net *net, int ifindex)
878{
879 struct net_device *dev;
880 struct hlist_head *head = dev_index_hash(net, ifindex);
881
882 hlist_for_each_entry(dev, head, index_hlist)
883 if (dev->ifindex == ifindex)
884 return dev;
885
886 return NULL;
887}
888EXPORT_SYMBOL(__dev_get_by_index);
889
890/**
891 * dev_get_by_index_rcu - find a device by its ifindex
892 * @net: the applicable net namespace
893 * @ifindex: index of device
894 *
895 * Search for an interface by index. Returns %NULL if the device
896 * is not found or a pointer to the device. The device has not
897 * had its reference counter increased so the caller must be careful
898 * about locking. The caller must hold RCU lock.
899 */
900
901struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
902{
903 struct net_device *dev;
904 struct hlist_head *head = dev_index_hash(net, ifindex);
905
906 hlist_for_each_entry_rcu(dev, head, index_hlist)
907 if (dev->ifindex == ifindex)
908 return dev;
909
910 return NULL;
911}
912EXPORT_SYMBOL(dev_get_by_index_rcu);
913
914/* Deprecated for new users, call netdev_get_by_index() instead */
915struct net_device *dev_get_by_index(struct net *net, int ifindex)
916{
917 struct net_device *dev;
918
919 rcu_read_lock();
920 dev = dev_get_by_index_rcu(net, ifindex);
921 dev_hold(dev);
922 rcu_read_unlock();
923 return dev;
924}
925EXPORT_SYMBOL(dev_get_by_index);
926
927/**
928 * netdev_get_by_index() - find a device by its ifindex
929 * @net: the applicable net namespace
930 * @ifindex: index of device
931 * @tracker: tracking object for the acquired reference
932 * @gfp: allocation flags for the tracker
933 *
934 * Search for an interface by index. Returns NULL if the device
935 * is not found or a pointer to the device. The device returned has
936 * had a reference added and the pointer is safe until the user calls
937 * netdev_put() to indicate they have finished with it.
938 */
939struct net_device *netdev_get_by_index(struct net *net, int ifindex,
940 netdevice_tracker *tracker, gfp_t gfp)
941{
942 struct net_device *dev;
943
944 dev = dev_get_by_index(net, ifindex);
945 if (dev)
946 netdev_tracker_alloc(dev, tracker, gfp);
947 return dev;
948}
949EXPORT_SYMBOL(netdev_get_by_index);
950
951/**
952 * dev_get_by_napi_id - find a device by napi_id
953 * @napi_id: ID of the NAPI struct
954 *
955 * Search for an interface by NAPI ID. Returns %NULL if the device
956 * is not found or a pointer to the device. The device has not had
957 * its reference counter increased so the caller must be careful
958 * about locking. The caller must hold RCU lock.
959 */
960
961struct net_device *dev_get_by_napi_id(unsigned int napi_id)
962{
963 struct napi_struct *napi;
964
965 WARN_ON_ONCE(!rcu_read_lock_held());
966
967 if (napi_id < MIN_NAPI_ID)
968 return NULL;
969
970 napi = napi_by_id(napi_id);
971
972 return napi ? napi->dev : NULL;
973}
974EXPORT_SYMBOL(dev_get_by_napi_id);
975
976static DEFINE_SEQLOCK(netdev_rename_lock);
977
978void netdev_copy_name(struct net_device *dev, char *name)
979{
980 unsigned int seq;
981
982 do {
983 seq = read_seqbegin(&netdev_rename_lock);
984 strscpy(name, dev->name, IFNAMSIZ);
985 } while (read_seqretry(&netdev_rename_lock, seq));
986}
987
988/**
989 * netdev_get_name - get a netdevice name, knowing its ifindex.
990 * @net: network namespace
991 * @name: a pointer to the buffer where the name will be stored.
992 * @ifindex: the ifindex of the interface to get the name from.
993 */
994int netdev_get_name(struct net *net, char *name, int ifindex)
995{
996 struct net_device *dev;
997 int ret;
998
999 rcu_read_lock();
1000
1001 dev = dev_get_by_index_rcu(net, ifindex);
1002 if (!dev) {
1003 ret = -ENODEV;
1004 goto out;
1005 }
1006
1007 netdev_copy_name(dev, name);
1008
1009 ret = 0;
1010out:
1011 rcu_read_unlock();
1012 return ret;
1013}
1014
1015static bool dev_addr_cmp(struct net_device *dev, unsigned short type,
1016 const char *ha)
1017{
1018 return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len);
1019}
1020
1021/**
1022 * dev_getbyhwaddr_rcu - find a device by its hardware address
1023 * @net: the applicable net namespace
1024 * @type: media type of device
1025 * @ha: hardware address
1026 *
1027 * Search for an interface by MAC address. Returns NULL if the device
1028 * is not found or a pointer to the device.
1029 * The caller must hold RCU.
1030 * The returned device has not had its ref count increased
1031 * and the caller must therefore be careful about locking
1032 *
1033 */
1034
1035struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
1036 const char *ha)
1037{
1038 struct net_device *dev;
1039
1040 for_each_netdev_rcu(net, dev)
1041 if (dev_addr_cmp(dev, type, ha))
1042 return dev;
1043
1044 return NULL;
1045}
1046EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
1047
1048/**
1049 * dev_getbyhwaddr() - find a device by its hardware address
1050 * @net: the applicable net namespace
1051 * @type: media type of device
1052 * @ha: hardware address
1053 *
1054 * Similar to dev_getbyhwaddr_rcu(), but the owner needs to hold
1055 * rtnl_lock.
1056 *
1057 * Context: rtnl_lock() must be held.
1058 * Return: pointer to the net_device, or NULL if not found
1059 */
1060struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type,
1061 const char *ha)
1062{
1063 struct net_device *dev;
1064
1065 ASSERT_RTNL();
1066 for_each_netdev(net, dev)
1067 if (dev_addr_cmp(dev, type, ha))
1068 return dev;
1069
1070 return NULL;
1071}
1072EXPORT_SYMBOL(dev_getbyhwaddr);
1073
1074struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
1075{
1076 struct net_device *dev, *ret = NULL;
1077
1078 rcu_read_lock();
1079 for_each_netdev_rcu(net, dev)
1080 if (dev->type == type) {
1081 dev_hold(dev);
1082 ret = dev;
1083 break;
1084 }
1085 rcu_read_unlock();
1086 return ret;
1087}
1088EXPORT_SYMBOL(dev_getfirstbyhwtype);
1089
1090/**
1091 * __dev_get_by_flags - find any device with given flags
1092 * @net: the applicable net namespace
1093 * @if_flags: IFF_* values
1094 * @mask: bitmask of bits in if_flags to check
1095 *
1096 * Search for any interface with the given flags. Returns NULL if a device
1097 * is not found or a pointer to the device. Must be called inside
1098 * rtnl_lock(), and result refcount is unchanged.
1099 */
1100
1101struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1102 unsigned short mask)
1103{
1104 struct net_device *dev, *ret;
1105
1106 ASSERT_RTNL();
1107
1108 ret = NULL;
1109 for_each_netdev(net, dev) {
1110 if (((dev->flags ^ if_flags) & mask) == 0) {
1111 ret = dev;
1112 break;
1113 }
1114 }
1115 return ret;
1116}
1117EXPORT_SYMBOL(__dev_get_by_flags);
1118
1119/**
1120 * dev_valid_name - check if name is okay for network device
1121 * @name: name string
1122 *
1123 * Network device names need to be valid file names to
1124 * allow sysfs to work. We also disallow any kind of
1125 * whitespace.
1126 */
1127bool dev_valid_name(const char *name)
1128{
1129 if (*name == '\0')
1130 return false;
1131 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1132 return false;
1133 if (!strcmp(name, ".") || !strcmp(name, ".."))
1134 return false;
1135
1136 while (*name) {
1137 if (*name == '/' || *name == ':' || isspace(*name))
1138 return false;
1139 name++;
1140 }
1141 return true;
1142}
1143EXPORT_SYMBOL(dev_valid_name);
1144
1145/**
1146 * __dev_alloc_name - allocate a name for a device
1147 * @net: network namespace to allocate the device name in
1148 * @name: name format string
1149 * @res: result name string
1150 *
1151 * Passed a format string - eg "lt%d" it will try and find a suitable
1152 * id. It scans list of devices to build up a free map, then chooses
1153 * the first empty slot. The caller must hold the dev_base or rtnl lock
1154 * while allocating the name and adding the device in order to avoid
1155 * duplicates.
1156 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1157 * Returns the number of the unit assigned or a negative errno code.
1158 */
1159
1160static int __dev_alloc_name(struct net *net, const char *name, char *res)
1161{
1162 int i = 0;
1163 const char *p;
1164 const int max_netdevices = 8*PAGE_SIZE;
1165 unsigned long *inuse;
1166 struct net_device *d;
1167 char buf[IFNAMSIZ];
1168
1169 /* Verify the string as this thing may have come from the user.
1170 * There must be one "%d" and no other "%" characters.
1171 */
1172 p = strchr(name, '%');
1173 if (!p || p[1] != 'd' || strchr(p + 2, '%'))
1174 return -EINVAL;
1175
1176 /* Use one page as a bit array of possible slots */
1177 inuse = bitmap_zalloc(max_netdevices, GFP_ATOMIC);
1178 if (!inuse)
1179 return -ENOMEM;
1180
1181 for_each_netdev(net, d) {
1182 struct netdev_name_node *name_node;
1183
1184 netdev_for_each_altname(d, name_node) {
1185 if (!sscanf(name_node->name, name, &i))
1186 continue;
1187 if (i < 0 || i >= max_netdevices)
1188 continue;
1189
1190 /* avoid cases where sscanf is not exact inverse of printf */
1191 snprintf(buf, IFNAMSIZ, name, i);
1192 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1193 __set_bit(i, inuse);
1194 }
1195 if (!sscanf(d->name, name, &i))
1196 continue;
1197 if (i < 0 || i >= max_netdevices)
1198 continue;
1199
1200 /* avoid cases where sscanf is not exact inverse of printf */
1201 snprintf(buf, IFNAMSIZ, name, i);
1202 if (!strncmp(buf, d->name, IFNAMSIZ))
1203 __set_bit(i, inuse);
1204 }
1205
1206 i = find_first_zero_bit(inuse, max_netdevices);
1207 bitmap_free(inuse);
1208 if (i == max_netdevices)
1209 return -ENFILE;
1210
1211 /* 'res' and 'name' could overlap, use 'buf' as an intermediate buffer */
1212 strscpy(buf, name, IFNAMSIZ);
1213 snprintf(res, IFNAMSIZ, buf, i);
1214 return i;
1215}
1216
1217/* Returns negative errno or allocated unit id (see __dev_alloc_name()) */
1218static int dev_prep_valid_name(struct net *net, struct net_device *dev,
1219 const char *want_name, char *out_name,
1220 int dup_errno)
1221{
1222 if (!dev_valid_name(want_name))
1223 return -EINVAL;
1224
1225 if (strchr(want_name, '%'))
1226 return __dev_alloc_name(net, want_name, out_name);
1227
1228 if (netdev_name_in_use(net, want_name))
1229 return -dup_errno;
1230 if (out_name != want_name)
1231 strscpy(out_name, want_name, IFNAMSIZ);
1232 return 0;
1233}
1234
1235/**
1236 * dev_alloc_name - allocate a name for a device
1237 * @dev: device
1238 * @name: name format string
1239 *
1240 * Passed a format string - eg "lt%d" it will try and find a suitable
1241 * id. It scans list of devices to build up a free map, then chooses
1242 * the first empty slot. The caller must hold the dev_base or rtnl lock
1243 * while allocating the name and adding the device in order to avoid
1244 * duplicates.
1245 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1246 * Returns the number of the unit assigned or a negative errno code.
1247 */
1248
1249int dev_alloc_name(struct net_device *dev, const char *name)
1250{
1251 return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE);
1252}
1253EXPORT_SYMBOL(dev_alloc_name);
1254
1255static int dev_get_valid_name(struct net *net, struct net_device *dev,
1256 const char *name)
1257{
1258 int ret;
1259
1260 ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST);
1261 return ret < 0 ? ret : 0;
1262}
1263
1264/**
1265 * dev_change_name - change name of a device
1266 * @dev: device
1267 * @newname: name (or format string) must be at least IFNAMSIZ
1268 *
1269 * Change name of a device, can pass format strings "eth%d".
1270 * for wildcarding.
1271 */
1272int dev_change_name(struct net_device *dev, const char *newname)
1273{
1274 unsigned char old_assign_type;
1275 char oldname[IFNAMSIZ];
1276 int err = 0;
1277 int ret;
1278 struct net *net;
1279
1280 ASSERT_RTNL();
1281 BUG_ON(!dev_net(dev));
1282
1283 net = dev_net(dev);
1284
1285 down_write(&devnet_rename_sem);
1286
1287 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1288 up_write(&devnet_rename_sem);
1289 return 0;
1290 }
1291
1292 memcpy(oldname, dev->name, IFNAMSIZ);
1293
1294 write_seqlock_bh(&netdev_rename_lock);
1295 err = dev_get_valid_name(net, dev, newname);
1296 write_sequnlock_bh(&netdev_rename_lock);
1297
1298 if (err < 0) {
1299 up_write(&devnet_rename_sem);
1300 return err;
1301 }
1302
1303 if (oldname[0] && !strchr(oldname, '%'))
1304 netdev_info(dev, "renamed from %s%s\n", oldname,
1305 dev->flags & IFF_UP ? " (while UP)" : "");
1306
1307 old_assign_type = dev->name_assign_type;
1308 WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED);
1309
1310rollback:
1311 ret = device_rename(&dev->dev, dev->name);
1312 if (ret) {
1313 write_seqlock_bh(&netdev_rename_lock);
1314 memcpy(dev->name, oldname, IFNAMSIZ);
1315 write_sequnlock_bh(&netdev_rename_lock);
1316 WRITE_ONCE(dev->name_assign_type, old_assign_type);
1317 up_write(&devnet_rename_sem);
1318 return ret;
1319 }
1320
1321 up_write(&devnet_rename_sem);
1322
1323 netdev_adjacent_rename_links(dev, oldname);
1324
1325 netdev_name_node_del(dev->name_node);
1326
1327 synchronize_net();
1328
1329 netdev_name_node_add(net, dev->name_node);
1330
1331 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1332 ret = notifier_to_errno(ret);
1333
1334 if (ret) {
1335 /* err >= 0 after dev_alloc_name() or stores the first errno */
1336 if (err >= 0) {
1337 err = ret;
1338 down_write(&devnet_rename_sem);
1339 write_seqlock_bh(&netdev_rename_lock);
1340 memcpy(dev->name, oldname, IFNAMSIZ);
1341 write_sequnlock_bh(&netdev_rename_lock);
1342 memcpy(oldname, newname, IFNAMSIZ);
1343 WRITE_ONCE(dev->name_assign_type, old_assign_type);
1344 old_assign_type = NET_NAME_RENAMED;
1345 goto rollback;
1346 } else {
1347 netdev_err(dev, "name change rollback failed: %d\n",
1348 ret);
1349 }
1350 }
1351
1352 return err;
1353}
1354
1355/**
1356 * dev_set_alias - change ifalias of a device
1357 * @dev: device
1358 * @alias: name up to IFALIASZ
1359 * @len: limit of bytes to copy from info
1360 *
1361 * Set ifalias for a device,
1362 */
1363int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1364{
1365 struct dev_ifalias *new_alias = NULL;
1366
1367 if (len >= IFALIASZ)
1368 return -EINVAL;
1369
1370 if (len) {
1371 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1372 if (!new_alias)
1373 return -ENOMEM;
1374
1375 memcpy(new_alias->ifalias, alias, len);
1376 new_alias->ifalias[len] = 0;
1377 }
1378
1379 mutex_lock(&ifalias_mutex);
1380 new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1381 mutex_is_locked(&ifalias_mutex));
1382 mutex_unlock(&ifalias_mutex);
1383
1384 if (new_alias)
1385 kfree_rcu(new_alias, rcuhead);
1386
1387 return len;
1388}
1389EXPORT_SYMBOL(dev_set_alias);
1390
1391/**
1392 * dev_get_alias - get ifalias of a device
1393 * @dev: device
1394 * @name: buffer to store name of ifalias
1395 * @len: size of buffer
1396 *
1397 * get ifalias for a device. Caller must make sure dev cannot go
1398 * away, e.g. rcu read lock or own a reference count to device.
1399 */
1400int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1401{
1402 const struct dev_ifalias *alias;
1403 int ret = 0;
1404
1405 rcu_read_lock();
1406 alias = rcu_dereference(dev->ifalias);
1407 if (alias)
1408 ret = snprintf(name, len, "%s", alias->ifalias);
1409 rcu_read_unlock();
1410
1411 return ret;
1412}
1413
1414/**
1415 * netdev_features_change - device changes features
1416 * @dev: device to cause notification
1417 *
1418 * Called to indicate a device has changed features.
1419 */
1420void netdev_features_change(struct net_device *dev)
1421{
1422 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1423}
1424EXPORT_SYMBOL(netdev_features_change);
1425
1426/**
1427 * netdev_state_change - device changes state
1428 * @dev: device to cause notification
1429 *
1430 * Called to indicate a device has changed state. This function calls
1431 * the notifier chains for netdev_chain and sends a NEWLINK message
1432 * to the routing socket.
1433 */
1434void netdev_state_change(struct net_device *dev)
1435{
1436 if (dev->flags & IFF_UP) {
1437 struct netdev_notifier_change_info change_info = {
1438 .info.dev = dev,
1439 };
1440
1441 call_netdevice_notifiers_info(NETDEV_CHANGE,
1442 &change_info.info);
1443 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL);
1444 }
1445}
1446EXPORT_SYMBOL(netdev_state_change);
1447
1448/**
1449 * __netdev_notify_peers - notify network peers about existence of @dev,
1450 * to be called when rtnl lock is already held.
1451 * @dev: network device
1452 *
1453 * Generate traffic such that interested network peers are aware of
1454 * @dev, such as by generating a gratuitous ARP. This may be used when
1455 * a device wants to inform the rest of the network about some sort of
1456 * reconfiguration such as a failover event or virtual machine
1457 * migration.
1458 */
1459void __netdev_notify_peers(struct net_device *dev)
1460{
1461 ASSERT_RTNL();
1462 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1463 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1464}
1465EXPORT_SYMBOL(__netdev_notify_peers);
1466
1467/**
1468 * netdev_notify_peers - notify network peers about existence of @dev
1469 * @dev: network device
1470 *
1471 * Generate traffic such that interested network peers are aware of
1472 * @dev, such as by generating a gratuitous ARP. This may be used when
1473 * a device wants to inform the rest of the network about some sort of
1474 * reconfiguration such as a failover event or virtual machine
1475 * migration.
1476 */
1477void netdev_notify_peers(struct net_device *dev)
1478{
1479 rtnl_lock();
1480 __netdev_notify_peers(dev);
1481 rtnl_unlock();
1482}
1483EXPORT_SYMBOL(netdev_notify_peers);
1484
1485static int napi_threaded_poll(void *data);
1486
1487static int napi_kthread_create(struct napi_struct *n)
1488{
1489 int err = 0;
1490
1491 /* Create and wake up the kthread once to put it in
1492 * TASK_INTERRUPTIBLE mode to avoid the blocked task
1493 * warning and work with loadavg.
1494 */
1495 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1496 n->dev->name, n->napi_id);
1497 if (IS_ERR(n->thread)) {
1498 err = PTR_ERR(n->thread);
1499 pr_err("kthread_run failed with err %d\n", err);
1500 n->thread = NULL;
1501 }
1502
1503 return err;
1504}
1505
1506static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1507{
1508 const struct net_device_ops *ops = dev->netdev_ops;
1509 int ret;
1510
1511 ASSERT_RTNL();
1512 dev_addr_check(dev);
1513
1514 if (!netif_device_present(dev)) {
1515 /* may be detached because parent is runtime-suspended */
1516 if (dev->dev.parent)
1517 pm_runtime_resume(dev->dev.parent);
1518 if (!netif_device_present(dev))
1519 return -ENODEV;
1520 }
1521
1522 /* Block netpoll from trying to do any rx path servicing.
1523 * If we don't do this there is a chance ndo_poll_controller
1524 * or ndo_poll may be running while we open the device
1525 */
1526 netpoll_poll_disable(dev);
1527
1528 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1529 ret = notifier_to_errno(ret);
1530 if (ret)
1531 return ret;
1532
1533 set_bit(__LINK_STATE_START, &dev->state);
1534
1535 if (ops->ndo_validate_addr)
1536 ret = ops->ndo_validate_addr(dev);
1537
1538 if (!ret && ops->ndo_open)
1539 ret = ops->ndo_open(dev);
1540
1541 netpoll_poll_enable(dev);
1542
1543 if (ret)
1544 clear_bit(__LINK_STATE_START, &dev->state);
1545 else {
1546 dev->flags |= IFF_UP;
1547 dev_set_rx_mode(dev);
1548 dev_activate(dev);
1549 add_device_randomness(dev->dev_addr, dev->addr_len);
1550 }
1551
1552 return ret;
1553}
1554
1555/**
1556 * dev_open - prepare an interface for use.
1557 * @dev: device to open
1558 * @extack: netlink extended ack
1559 *
1560 * Takes a device from down to up state. The device's private open
1561 * function is invoked and then the multicast lists are loaded. Finally
1562 * the device is moved into the up state and a %NETDEV_UP message is
1563 * sent to the netdev notifier chain.
1564 *
1565 * Calling this function on an active interface is a nop. On a failure
1566 * a negative errno code is returned.
1567 */
1568int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1569{
1570 int ret;
1571
1572 if (dev->flags & IFF_UP)
1573 return 0;
1574
1575 ret = __dev_open(dev, extack);
1576 if (ret < 0)
1577 return ret;
1578
1579 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1580 call_netdevice_notifiers(NETDEV_UP, dev);
1581
1582 return ret;
1583}
1584EXPORT_SYMBOL(dev_open);
1585
1586static void __dev_close_many(struct list_head *head)
1587{
1588 struct net_device *dev;
1589
1590 ASSERT_RTNL();
1591 might_sleep();
1592
1593 list_for_each_entry(dev, head, close_list) {
1594 /* Temporarily disable netpoll until the interface is down */
1595 netpoll_poll_disable(dev);
1596
1597 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1598
1599 clear_bit(__LINK_STATE_START, &dev->state);
1600
1601 /* Synchronize to scheduled poll. We cannot touch poll list, it
1602 * can be even on different cpu. So just clear netif_running().
1603 *
1604 * dev->stop() will invoke napi_disable() on all of it's
1605 * napi_struct instances on this device.
1606 */
1607 smp_mb__after_atomic(); /* Commit netif_running(). */
1608 }
1609
1610 dev_deactivate_many(head);
1611
1612 list_for_each_entry(dev, head, close_list) {
1613 const struct net_device_ops *ops = dev->netdev_ops;
1614
1615 /*
1616 * Call the device specific close. This cannot fail.
1617 * Only if device is UP
1618 *
1619 * We allow it to be called even after a DETACH hot-plug
1620 * event.
1621 */
1622 if (ops->ndo_stop)
1623 ops->ndo_stop(dev);
1624
1625 dev->flags &= ~IFF_UP;
1626 netpoll_poll_enable(dev);
1627 }
1628}
1629
1630static void __dev_close(struct net_device *dev)
1631{
1632 LIST_HEAD(single);
1633
1634 list_add(&dev->close_list, &single);
1635 __dev_close_many(&single);
1636 list_del(&single);
1637}
1638
1639void dev_close_many(struct list_head *head, bool unlink)
1640{
1641 struct net_device *dev, *tmp;
1642
1643 /* Remove the devices that don't need to be closed */
1644 list_for_each_entry_safe(dev, tmp, head, close_list)
1645 if (!(dev->flags & IFF_UP))
1646 list_del_init(&dev->close_list);
1647
1648 __dev_close_many(head);
1649
1650 list_for_each_entry_safe(dev, tmp, head, close_list) {
1651 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1652 call_netdevice_notifiers(NETDEV_DOWN, dev);
1653 if (unlink)
1654 list_del_init(&dev->close_list);
1655 }
1656}
1657EXPORT_SYMBOL(dev_close_many);
1658
1659/**
1660 * dev_close - shutdown an interface.
1661 * @dev: device to shutdown
1662 *
1663 * This function moves an active device into down state. A
1664 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1665 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1666 * chain.
1667 */
1668void dev_close(struct net_device *dev)
1669{
1670 if (dev->flags & IFF_UP) {
1671 LIST_HEAD(single);
1672
1673 list_add(&dev->close_list, &single);
1674 dev_close_many(&single, true);
1675 list_del(&single);
1676 }
1677}
1678EXPORT_SYMBOL(dev_close);
1679
1680
1681/**
1682 * dev_disable_lro - disable Large Receive Offload on a device
1683 * @dev: device
1684 *
1685 * Disable Large Receive Offload (LRO) on a net device. Must be
1686 * called under RTNL. This is needed if received packets may be
1687 * forwarded to another interface.
1688 */
1689void dev_disable_lro(struct net_device *dev)
1690{
1691 struct net_device *lower_dev;
1692 struct list_head *iter;
1693
1694 dev->wanted_features &= ~NETIF_F_LRO;
1695 netdev_update_features(dev);
1696
1697 if (unlikely(dev->features & NETIF_F_LRO))
1698 netdev_WARN(dev, "failed to disable LRO!\n");
1699
1700 netdev_for_each_lower_dev(dev, lower_dev, iter)
1701 dev_disable_lro(lower_dev);
1702}
1703EXPORT_SYMBOL(dev_disable_lro);
1704
1705/**
1706 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1707 * @dev: device
1708 *
1709 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1710 * called under RTNL. This is needed if Generic XDP is installed on
1711 * the device.
1712 */
1713static void dev_disable_gro_hw(struct net_device *dev)
1714{
1715 dev->wanted_features &= ~NETIF_F_GRO_HW;
1716 netdev_update_features(dev);
1717
1718 if (unlikely(dev->features & NETIF_F_GRO_HW))
1719 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1720}
1721
1722const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1723{
1724#define N(val) \
1725 case NETDEV_##val: \
1726 return "NETDEV_" __stringify(val);
1727 switch (cmd) {
1728 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1729 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1730 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1731 N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN)
1732 N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA)
1733 N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE)
1734 N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1735 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1736 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1737 N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1738 N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1739 N(XDP_FEAT_CHANGE)
1740 }
1741#undef N
1742 return "UNKNOWN_NETDEV_EVENT";
1743}
1744EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1745
1746static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1747 struct net_device *dev)
1748{
1749 struct netdev_notifier_info info = {
1750 .dev = dev,
1751 };
1752
1753 return nb->notifier_call(nb, val, &info);
1754}
1755
1756static int call_netdevice_register_notifiers(struct notifier_block *nb,
1757 struct net_device *dev)
1758{
1759 int err;
1760
1761 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1762 err = notifier_to_errno(err);
1763 if (err)
1764 return err;
1765
1766 if (!(dev->flags & IFF_UP))
1767 return 0;
1768
1769 call_netdevice_notifier(nb, NETDEV_UP, dev);
1770 return 0;
1771}
1772
1773static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1774 struct net_device *dev)
1775{
1776 if (dev->flags & IFF_UP) {
1777 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1778 dev);
1779 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1780 }
1781 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1782}
1783
1784static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1785 struct net *net)
1786{
1787 struct net_device *dev;
1788 int err;
1789
1790 for_each_netdev(net, dev) {
1791 err = call_netdevice_register_notifiers(nb, dev);
1792 if (err)
1793 goto rollback;
1794 }
1795 return 0;
1796
1797rollback:
1798 for_each_netdev_continue_reverse(net, dev)
1799 call_netdevice_unregister_notifiers(nb, dev);
1800 return err;
1801}
1802
1803static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1804 struct net *net)
1805{
1806 struct net_device *dev;
1807
1808 for_each_netdev(net, dev)
1809 call_netdevice_unregister_notifiers(nb, dev);
1810}
1811
1812static int dev_boot_phase = 1;
1813
1814/**
1815 * register_netdevice_notifier - register a network notifier block
1816 * @nb: notifier
1817 *
1818 * Register a notifier to be called when network device events occur.
1819 * The notifier passed is linked into the kernel structures and must
1820 * not be reused until it has been unregistered. A negative errno code
1821 * is returned on a failure.
1822 *
1823 * When registered all registration and up events are replayed
1824 * to the new notifier to allow device to have a race free
1825 * view of the network device list.
1826 */
1827
1828int register_netdevice_notifier(struct notifier_block *nb)
1829{
1830 struct net *net;
1831 int err;
1832
1833 /* Close race with setup_net() and cleanup_net() */
1834 down_write(&pernet_ops_rwsem);
1835 rtnl_lock();
1836 err = raw_notifier_chain_register(&netdev_chain, nb);
1837 if (err)
1838 goto unlock;
1839 if (dev_boot_phase)
1840 goto unlock;
1841 for_each_net(net) {
1842 err = call_netdevice_register_net_notifiers(nb, net);
1843 if (err)
1844 goto rollback;
1845 }
1846
1847unlock:
1848 rtnl_unlock();
1849 up_write(&pernet_ops_rwsem);
1850 return err;
1851
1852rollback:
1853 for_each_net_continue_reverse(net)
1854 call_netdevice_unregister_net_notifiers(nb, net);
1855
1856 raw_notifier_chain_unregister(&netdev_chain, nb);
1857 goto unlock;
1858}
1859EXPORT_SYMBOL(register_netdevice_notifier);
1860
1861/**
1862 * unregister_netdevice_notifier - unregister a network notifier block
1863 * @nb: notifier
1864 *
1865 * Unregister a notifier previously registered by
1866 * register_netdevice_notifier(). The notifier is unlinked into the
1867 * kernel structures and may then be reused. A negative errno code
1868 * is returned on a failure.
1869 *
1870 * After unregistering unregister and down device events are synthesized
1871 * for all devices on the device list to the removed notifier to remove
1872 * the need for special case cleanup code.
1873 */
1874
1875int unregister_netdevice_notifier(struct notifier_block *nb)
1876{
1877 struct net *net;
1878 int err;
1879
1880 /* Close race with setup_net() and cleanup_net() */
1881 down_write(&pernet_ops_rwsem);
1882 rtnl_lock();
1883 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1884 if (err)
1885 goto unlock;
1886
1887 for_each_net(net)
1888 call_netdevice_unregister_net_notifiers(nb, net);
1889
1890unlock:
1891 rtnl_unlock();
1892 up_write(&pernet_ops_rwsem);
1893 return err;
1894}
1895EXPORT_SYMBOL(unregister_netdevice_notifier);
1896
1897static int __register_netdevice_notifier_net(struct net *net,
1898 struct notifier_block *nb,
1899 bool ignore_call_fail)
1900{
1901 int err;
1902
1903 err = raw_notifier_chain_register(&net->netdev_chain, nb);
1904 if (err)
1905 return err;
1906 if (dev_boot_phase)
1907 return 0;
1908
1909 err = call_netdevice_register_net_notifiers(nb, net);
1910 if (err && !ignore_call_fail)
1911 goto chain_unregister;
1912
1913 return 0;
1914
1915chain_unregister:
1916 raw_notifier_chain_unregister(&net->netdev_chain, nb);
1917 return err;
1918}
1919
1920static int __unregister_netdevice_notifier_net(struct net *net,
1921 struct notifier_block *nb)
1922{
1923 int err;
1924
1925 err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1926 if (err)
1927 return err;
1928
1929 call_netdevice_unregister_net_notifiers(nb, net);
1930 return 0;
1931}
1932
1933/**
1934 * register_netdevice_notifier_net - register a per-netns network notifier block
1935 * @net: network namespace
1936 * @nb: notifier
1937 *
1938 * Register a notifier to be called when network device events occur.
1939 * The notifier passed is linked into the kernel structures and must
1940 * not be reused until it has been unregistered. A negative errno code
1941 * is returned on a failure.
1942 *
1943 * When registered all registration and up events are replayed
1944 * to the new notifier to allow device to have a race free
1945 * view of the network device list.
1946 */
1947
1948int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1949{
1950 int err;
1951
1952 rtnl_lock();
1953 err = __register_netdevice_notifier_net(net, nb, false);
1954 rtnl_unlock();
1955 return err;
1956}
1957EXPORT_SYMBOL(register_netdevice_notifier_net);
1958
1959/**
1960 * unregister_netdevice_notifier_net - unregister a per-netns
1961 * network notifier block
1962 * @net: network namespace
1963 * @nb: notifier
1964 *
1965 * Unregister a notifier previously registered by
1966 * register_netdevice_notifier_net(). The notifier is unlinked from the
1967 * kernel structures and may then be reused. A negative errno code
1968 * is returned on a failure.
1969 *
1970 * After unregistering unregister and down device events are synthesized
1971 * for all devices on the device list to the removed notifier to remove
1972 * the need for special case cleanup code.
1973 */
1974
1975int unregister_netdevice_notifier_net(struct net *net,
1976 struct notifier_block *nb)
1977{
1978 int err;
1979
1980 rtnl_lock();
1981 err = __unregister_netdevice_notifier_net(net, nb);
1982 rtnl_unlock();
1983 return err;
1984}
1985EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1986
1987static void __move_netdevice_notifier_net(struct net *src_net,
1988 struct net *dst_net,
1989 struct notifier_block *nb)
1990{
1991 __unregister_netdevice_notifier_net(src_net, nb);
1992 __register_netdevice_notifier_net(dst_net, nb, true);
1993}
1994
1995int register_netdevice_notifier_dev_net(struct net_device *dev,
1996 struct notifier_block *nb,
1997 struct netdev_net_notifier *nn)
1998{
1999 int err;
2000
2001 rtnl_lock();
2002 err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
2003 if (!err) {
2004 nn->nb = nb;
2005 list_add(&nn->list, &dev->net_notifier_list);
2006 }
2007 rtnl_unlock();
2008 return err;
2009}
2010EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
2011
2012int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2013 struct notifier_block *nb,
2014 struct netdev_net_notifier *nn)
2015{
2016 int err;
2017
2018 rtnl_lock();
2019 list_del(&nn->list);
2020 err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
2021 rtnl_unlock();
2022 return err;
2023}
2024EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
2025
2026static void move_netdevice_notifiers_dev_net(struct net_device *dev,
2027 struct net *net)
2028{
2029 struct netdev_net_notifier *nn;
2030
2031 list_for_each_entry(nn, &dev->net_notifier_list, list)
2032 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb);
2033}
2034
2035/**
2036 * call_netdevice_notifiers_info - call all network notifier blocks
2037 * @val: value passed unmodified to notifier function
2038 * @info: notifier information data
2039 *
2040 * Call all network notifier blocks. Parameters and return value
2041 * are as for raw_notifier_call_chain().
2042 */
2043
2044int call_netdevice_notifiers_info(unsigned long val,
2045 struct netdev_notifier_info *info)
2046{
2047 struct net *net = dev_net(info->dev);
2048 int ret;
2049
2050 ASSERT_RTNL();
2051
2052 /* Run per-netns notifier block chain first, then run the global one.
2053 * Hopefully, one day, the global one is going to be removed after
2054 * all notifier block registrators get converted to be per-netns.
2055 */
2056 ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
2057 if (ret & NOTIFY_STOP_MASK)
2058 return ret;
2059 return raw_notifier_call_chain(&netdev_chain, val, info);
2060}
2061
2062/**
2063 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks
2064 * for and rollback on error
2065 * @val_up: value passed unmodified to notifier function
2066 * @val_down: value passed unmodified to the notifier function when
2067 * recovering from an error on @val_up
2068 * @info: notifier information data
2069 *
2070 * Call all per-netns network notifier blocks, but not notifier blocks on
2071 * the global notifier chain. Parameters and return value are as for
2072 * raw_notifier_call_chain_robust().
2073 */
2074
2075static int
2076call_netdevice_notifiers_info_robust(unsigned long val_up,
2077 unsigned long val_down,
2078 struct netdev_notifier_info *info)
2079{
2080 struct net *net = dev_net(info->dev);
2081
2082 ASSERT_RTNL();
2083
2084 return raw_notifier_call_chain_robust(&net->netdev_chain,
2085 val_up, val_down, info);
2086}
2087
2088static int call_netdevice_notifiers_extack(unsigned long val,
2089 struct net_device *dev,
2090 struct netlink_ext_ack *extack)
2091{
2092 struct netdev_notifier_info info = {
2093 .dev = dev,
2094 .extack = extack,
2095 };
2096
2097 return call_netdevice_notifiers_info(val, &info);
2098}
2099
2100/**
2101 * call_netdevice_notifiers - call all network notifier blocks
2102 * @val: value passed unmodified to notifier function
2103 * @dev: net_device pointer passed unmodified to notifier function
2104 *
2105 * Call all network notifier blocks. Parameters and return value
2106 * are as for raw_notifier_call_chain().
2107 */
2108
2109int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
2110{
2111 return call_netdevice_notifiers_extack(val, dev, NULL);
2112}
2113EXPORT_SYMBOL(call_netdevice_notifiers);
2114
2115/**
2116 * call_netdevice_notifiers_mtu - call all network notifier blocks
2117 * @val: value passed unmodified to notifier function
2118 * @dev: net_device pointer passed unmodified to notifier function
2119 * @arg: additional u32 argument passed to the notifier function
2120 *
2121 * Call all network notifier blocks. Parameters and return value
2122 * are as for raw_notifier_call_chain().
2123 */
2124static int call_netdevice_notifiers_mtu(unsigned long val,
2125 struct net_device *dev, u32 arg)
2126{
2127 struct netdev_notifier_info_ext info = {
2128 .info.dev = dev,
2129 .ext.mtu = arg,
2130 };
2131
2132 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2133
2134 return call_netdevice_notifiers_info(val, &info.info);
2135}
2136
2137#ifdef CONFIG_NET_INGRESS
2138static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2139
2140void net_inc_ingress_queue(void)
2141{
2142 static_branch_inc(&ingress_needed_key);
2143}
2144EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2145
2146void net_dec_ingress_queue(void)
2147{
2148 static_branch_dec(&ingress_needed_key);
2149}
2150EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2151#endif
2152
2153#ifdef CONFIG_NET_EGRESS
2154static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2155
2156void net_inc_egress_queue(void)
2157{
2158 static_branch_inc(&egress_needed_key);
2159}
2160EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2161
2162void net_dec_egress_queue(void)
2163{
2164 static_branch_dec(&egress_needed_key);
2165}
2166EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2167#endif
2168
2169#ifdef CONFIG_NET_CLS_ACT
2170DEFINE_STATIC_KEY_FALSE(tcf_sw_enabled_key);
2171EXPORT_SYMBOL(tcf_sw_enabled_key);
2172#endif
2173
2174DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2175EXPORT_SYMBOL(netstamp_needed_key);
2176#ifdef CONFIG_JUMP_LABEL
2177static atomic_t netstamp_needed_deferred;
2178static atomic_t netstamp_wanted;
2179static void netstamp_clear(struct work_struct *work)
2180{
2181 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2182 int wanted;
2183
2184 wanted = atomic_add_return(deferred, &netstamp_wanted);
2185 if (wanted > 0)
2186 static_branch_enable(&netstamp_needed_key);
2187 else
2188 static_branch_disable(&netstamp_needed_key);
2189}
2190static DECLARE_WORK(netstamp_work, netstamp_clear);
2191#endif
2192
2193void net_enable_timestamp(void)
2194{
2195#ifdef CONFIG_JUMP_LABEL
2196 int wanted = atomic_read(&netstamp_wanted);
2197
2198 while (wanted > 0) {
2199 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1))
2200 return;
2201 }
2202 atomic_inc(&netstamp_needed_deferred);
2203 schedule_work(&netstamp_work);
2204#else
2205 static_branch_inc(&netstamp_needed_key);
2206#endif
2207}
2208EXPORT_SYMBOL(net_enable_timestamp);
2209
2210void net_disable_timestamp(void)
2211{
2212#ifdef CONFIG_JUMP_LABEL
2213 int wanted = atomic_read(&netstamp_wanted);
2214
2215 while (wanted > 1) {
2216 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1))
2217 return;
2218 }
2219 atomic_dec(&netstamp_needed_deferred);
2220 schedule_work(&netstamp_work);
2221#else
2222 static_branch_dec(&netstamp_needed_key);
2223#endif
2224}
2225EXPORT_SYMBOL(net_disable_timestamp);
2226
2227static inline void net_timestamp_set(struct sk_buff *skb)
2228{
2229 skb->tstamp = 0;
2230 skb->tstamp_type = SKB_CLOCK_REALTIME;
2231 if (static_branch_unlikely(&netstamp_needed_key))
2232 skb->tstamp = ktime_get_real();
2233}
2234
2235#define net_timestamp_check(COND, SKB) \
2236 if (static_branch_unlikely(&netstamp_needed_key)) { \
2237 if ((COND) && !(SKB)->tstamp) \
2238 (SKB)->tstamp = ktime_get_real(); \
2239 } \
2240
2241bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2242{
2243 return __is_skb_forwardable(dev, skb, true);
2244}
2245EXPORT_SYMBOL_GPL(is_skb_forwardable);
2246
2247static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2248 bool check_mtu)
2249{
2250 int ret = ____dev_forward_skb(dev, skb, check_mtu);
2251
2252 if (likely(!ret)) {
2253 skb->protocol = eth_type_trans(skb, dev);
2254 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2255 }
2256
2257 return ret;
2258}
2259
2260int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2261{
2262 return __dev_forward_skb2(dev, skb, true);
2263}
2264EXPORT_SYMBOL_GPL(__dev_forward_skb);
2265
2266/**
2267 * dev_forward_skb - loopback an skb to another netif
2268 *
2269 * @dev: destination network device
2270 * @skb: buffer to forward
2271 *
2272 * return values:
2273 * NET_RX_SUCCESS (no congestion)
2274 * NET_RX_DROP (packet was dropped, but freed)
2275 *
2276 * dev_forward_skb can be used for injecting an skb from the
2277 * start_xmit function of one device into the receive queue
2278 * of another device.
2279 *
2280 * The receiving device may be in another namespace, so
2281 * we have to clear all information in the skb that could
2282 * impact namespace isolation.
2283 */
2284int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2285{
2286 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2287}
2288EXPORT_SYMBOL_GPL(dev_forward_skb);
2289
2290int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2291{
2292 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2293}
2294
2295static inline int deliver_skb(struct sk_buff *skb,
2296 struct packet_type *pt_prev,
2297 struct net_device *orig_dev)
2298{
2299 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2300 return -ENOMEM;
2301 refcount_inc(&skb->users);
2302 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2303}
2304
2305static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2306 struct packet_type **pt,
2307 struct net_device *orig_dev,
2308 __be16 type,
2309 struct list_head *ptype_list)
2310{
2311 struct packet_type *ptype, *pt_prev = *pt;
2312
2313 list_for_each_entry_rcu(ptype, ptype_list, list) {
2314 if (ptype->type != type)
2315 continue;
2316 if (pt_prev)
2317 deliver_skb(skb, pt_prev, orig_dev);
2318 pt_prev = ptype;
2319 }
2320 *pt = pt_prev;
2321}
2322
2323static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2324{
2325 if (!ptype->af_packet_priv || !skb->sk)
2326 return false;
2327
2328 if (ptype->id_match)
2329 return ptype->id_match(ptype, skb->sk);
2330 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2331 return true;
2332
2333 return false;
2334}
2335
2336/**
2337 * dev_nit_active - return true if any network interface taps are in use
2338 *
2339 * @dev: network device to check for the presence of taps
2340 */
2341bool dev_nit_active(struct net_device *dev)
2342{
2343 return !list_empty(&net_hotdata.ptype_all) ||
2344 !list_empty(&dev->ptype_all);
2345}
2346EXPORT_SYMBOL_GPL(dev_nit_active);
2347
2348/*
2349 * Support routine. Sends outgoing frames to any network
2350 * taps currently in use.
2351 */
2352
2353void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2354{
2355 struct list_head *ptype_list = &net_hotdata.ptype_all;
2356 struct packet_type *ptype, *pt_prev = NULL;
2357 struct sk_buff *skb2 = NULL;
2358
2359 rcu_read_lock();
2360again:
2361 list_for_each_entry_rcu(ptype, ptype_list, list) {
2362 if (READ_ONCE(ptype->ignore_outgoing))
2363 continue;
2364
2365 /* Never send packets back to the socket
2366 * they originated from - MvS (miquels@drinkel.ow.org)
2367 */
2368 if (skb_loop_sk(ptype, skb))
2369 continue;
2370
2371 if (pt_prev) {
2372 deliver_skb(skb2, pt_prev, skb->dev);
2373 pt_prev = ptype;
2374 continue;
2375 }
2376
2377 /* need to clone skb, done only once */
2378 skb2 = skb_clone(skb, GFP_ATOMIC);
2379 if (!skb2)
2380 goto out_unlock;
2381
2382 net_timestamp_set(skb2);
2383
2384 /* skb->nh should be correctly
2385 * set by sender, so that the second statement is
2386 * just protection against buggy protocols.
2387 */
2388 skb_reset_mac_header(skb2);
2389
2390 if (skb_network_header(skb2) < skb2->data ||
2391 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2392 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2393 ntohs(skb2->protocol),
2394 dev->name);
2395 skb_reset_network_header(skb2);
2396 }
2397
2398 skb2->transport_header = skb2->network_header;
2399 skb2->pkt_type = PACKET_OUTGOING;
2400 pt_prev = ptype;
2401 }
2402
2403 if (ptype_list == &net_hotdata.ptype_all) {
2404 ptype_list = &dev->ptype_all;
2405 goto again;
2406 }
2407out_unlock:
2408 if (pt_prev) {
2409 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2410 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2411 else
2412 kfree_skb(skb2);
2413 }
2414 rcu_read_unlock();
2415}
2416EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2417
2418/**
2419 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2420 * @dev: Network device
2421 * @txq: number of queues available
2422 *
2423 * If real_num_tx_queues is changed the tc mappings may no longer be
2424 * valid. To resolve this verify the tc mapping remains valid and if
2425 * not NULL the mapping. With no priorities mapping to this
2426 * offset/count pair it will no longer be used. In the worst case TC0
2427 * is invalid nothing can be done so disable priority mappings. If is
2428 * expected that drivers will fix this mapping if they can before
2429 * calling netif_set_real_num_tx_queues.
2430 */
2431static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2432{
2433 int i;
2434 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2435
2436 /* If TC0 is invalidated disable TC mapping */
2437 if (tc->offset + tc->count > txq) {
2438 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2439 dev->num_tc = 0;
2440 return;
2441 }
2442
2443 /* Invalidated prio to tc mappings set to TC0 */
2444 for (i = 1; i < TC_BITMASK + 1; i++) {
2445 int q = netdev_get_prio_tc_map(dev, i);
2446
2447 tc = &dev->tc_to_txq[q];
2448 if (tc->offset + tc->count > txq) {
2449 netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2450 i, q);
2451 netdev_set_prio_tc_map(dev, i, 0);
2452 }
2453 }
2454}
2455
2456int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2457{
2458 if (dev->num_tc) {
2459 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2460 int i;
2461
2462 /* walk through the TCs and see if it falls into any of them */
2463 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2464 if ((txq - tc->offset) < tc->count)
2465 return i;
2466 }
2467
2468 /* didn't find it, just return -1 to indicate no match */
2469 return -1;
2470 }
2471
2472 return 0;
2473}
2474EXPORT_SYMBOL(netdev_txq_to_tc);
2475
2476#ifdef CONFIG_XPS
2477static struct static_key xps_needed __read_mostly;
2478static struct static_key xps_rxqs_needed __read_mostly;
2479static DEFINE_MUTEX(xps_map_mutex);
2480#define xmap_dereference(P) \
2481 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2482
2483static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2484 struct xps_dev_maps *old_maps, int tci, u16 index)
2485{
2486 struct xps_map *map = NULL;
2487 int pos;
2488
2489 map = xmap_dereference(dev_maps->attr_map[tci]);
2490 if (!map)
2491 return false;
2492
2493 for (pos = map->len; pos--;) {
2494 if (map->queues[pos] != index)
2495 continue;
2496
2497 if (map->len > 1) {
2498 map->queues[pos] = map->queues[--map->len];
2499 break;
2500 }
2501
2502 if (old_maps)
2503 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2504 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2505 kfree_rcu(map, rcu);
2506 return false;
2507 }
2508
2509 return true;
2510}
2511
2512static bool remove_xps_queue_cpu(struct net_device *dev,
2513 struct xps_dev_maps *dev_maps,
2514 int cpu, u16 offset, u16 count)
2515{
2516 int num_tc = dev_maps->num_tc;
2517 bool active = false;
2518 int tci;
2519
2520 for (tci = cpu * num_tc; num_tc--; tci++) {
2521 int i, j;
2522
2523 for (i = count, j = offset; i--; j++) {
2524 if (!remove_xps_queue(dev_maps, NULL, tci, j))
2525 break;
2526 }
2527
2528 active |= i < 0;
2529 }
2530
2531 return active;
2532}
2533
2534static void reset_xps_maps(struct net_device *dev,
2535 struct xps_dev_maps *dev_maps,
2536 enum xps_map_type type)
2537{
2538 static_key_slow_dec_cpuslocked(&xps_needed);
2539 if (type == XPS_RXQS)
2540 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2541
2542 RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2543
2544 kfree_rcu(dev_maps, rcu);
2545}
2546
2547static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2548 u16 offset, u16 count)
2549{
2550 struct xps_dev_maps *dev_maps;
2551 bool active = false;
2552 int i, j;
2553
2554 dev_maps = xmap_dereference(dev->xps_maps[type]);
2555 if (!dev_maps)
2556 return;
2557
2558 for (j = 0; j < dev_maps->nr_ids; j++)
2559 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2560 if (!active)
2561 reset_xps_maps(dev, dev_maps, type);
2562
2563 if (type == XPS_CPUS) {
2564 for (i = offset + (count - 1); count--; i--)
2565 netdev_queue_numa_node_write(
2566 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2567 }
2568}
2569
2570static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2571 u16 count)
2572{
2573 if (!static_key_false(&xps_needed))
2574 return;
2575
2576 cpus_read_lock();
2577 mutex_lock(&xps_map_mutex);
2578
2579 if (static_key_false(&xps_rxqs_needed))
2580 clean_xps_maps(dev, XPS_RXQS, offset, count);
2581
2582 clean_xps_maps(dev, XPS_CPUS, offset, count);
2583
2584 mutex_unlock(&xps_map_mutex);
2585 cpus_read_unlock();
2586}
2587
2588static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2589{
2590 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2591}
2592
2593static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2594 u16 index, bool is_rxqs_map)
2595{
2596 struct xps_map *new_map;
2597 int alloc_len = XPS_MIN_MAP_ALLOC;
2598 int i, pos;
2599
2600 for (pos = 0; map && pos < map->len; pos++) {
2601 if (map->queues[pos] != index)
2602 continue;
2603 return map;
2604 }
2605
2606 /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2607 if (map) {
2608 if (pos < map->alloc_len)
2609 return map;
2610
2611 alloc_len = map->alloc_len * 2;
2612 }
2613
2614 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2615 * map
2616 */
2617 if (is_rxqs_map)
2618 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2619 else
2620 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2621 cpu_to_node(attr_index));
2622 if (!new_map)
2623 return NULL;
2624
2625 for (i = 0; i < pos; i++)
2626 new_map->queues[i] = map->queues[i];
2627 new_map->alloc_len = alloc_len;
2628 new_map->len = pos;
2629
2630 return new_map;
2631}
2632
2633/* Copy xps maps at a given index */
2634static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2635 struct xps_dev_maps *new_dev_maps, int index,
2636 int tc, bool skip_tc)
2637{
2638 int i, tci = index * dev_maps->num_tc;
2639 struct xps_map *map;
2640
2641 /* copy maps belonging to foreign traffic classes */
2642 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2643 if (i == tc && skip_tc)
2644 continue;
2645
2646 /* fill in the new device map from the old device map */
2647 map = xmap_dereference(dev_maps->attr_map[tci]);
2648 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2649 }
2650}
2651
2652/* Must be called under cpus_read_lock */
2653int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2654 u16 index, enum xps_map_type type)
2655{
2656 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2657 const unsigned long *online_mask = NULL;
2658 bool active = false, copy = false;
2659 int i, j, tci, numa_node_id = -2;
2660 int maps_sz, num_tc = 1, tc = 0;
2661 struct xps_map *map, *new_map;
2662 unsigned int nr_ids;
2663
2664 WARN_ON_ONCE(index >= dev->num_tx_queues);
2665
2666 if (dev->num_tc) {
2667 /* Do not allow XPS on subordinate device directly */
2668 num_tc = dev->num_tc;
2669 if (num_tc < 0)
2670 return -EINVAL;
2671
2672 /* If queue belongs to subordinate dev use its map */
2673 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2674
2675 tc = netdev_txq_to_tc(dev, index);
2676 if (tc < 0)
2677 return -EINVAL;
2678 }
2679
2680 mutex_lock(&xps_map_mutex);
2681
2682 dev_maps = xmap_dereference(dev->xps_maps[type]);
2683 if (type == XPS_RXQS) {
2684 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2685 nr_ids = dev->num_rx_queues;
2686 } else {
2687 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2688 if (num_possible_cpus() > 1)
2689 online_mask = cpumask_bits(cpu_online_mask);
2690 nr_ids = nr_cpu_ids;
2691 }
2692
2693 if (maps_sz < L1_CACHE_BYTES)
2694 maps_sz = L1_CACHE_BYTES;
2695
2696 /* The old dev_maps could be larger or smaller than the one we're
2697 * setting up now, as dev->num_tc or nr_ids could have been updated in
2698 * between. We could try to be smart, but let's be safe instead and only
2699 * copy foreign traffic classes if the two map sizes match.
2700 */
2701 if (dev_maps &&
2702 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2703 copy = true;
2704
2705 /* allocate memory for queue storage */
2706 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2707 j < nr_ids;) {
2708 if (!new_dev_maps) {
2709 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2710 if (!new_dev_maps) {
2711 mutex_unlock(&xps_map_mutex);
2712 return -ENOMEM;
2713 }
2714
2715 new_dev_maps->nr_ids = nr_ids;
2716 new_dev_maps->num_tc = num_tc;
2717 }
2718
2719 tci = j * num_tc + tc;
2720 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2721
2722 map = expand_xps_map(map, j, index, type == XPS_RXQS);
2723 if (!map)
2724 goto error;
2725
2726 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2727 }
2728
2729 if (!new_dev_maps)
2730 goto out_no_new_maps;
2731
2732 if (!dev_maps) {
2733 /* Increment static keys at most once per type */
2734 static_key_slow_inc_cpuslocked(&xps_needed);
2735 if (type == XPS_RXQS)
2736 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2737 }
2738
2739 for (j = 0; j < nr_ids; j++) {
2740 bool skip_tc = false;
2741
2742 tci = j * num_tc + tc;
2743 if (netif_attr_test_mask(j, mask, nr_ids) &&
2744 netif_attr_test_online(j, online_mask, nr_ids)) {
2745 /* add tx-queue to CPU/rx-queue maps */
2746 int pos = 0;
2747
2748 skip_tc = true;
2749
2750 map = xmap_dereference(new_dev_maps->attr_map[tci]);
2751 while ((pos < map->len) && (map->queues[pos] != index))
2752 pos++;
2753
2754 if (pos == map->len)
2755 map->queues[map->len++] = index;
2756#ifdef CONFIG_NUMA
2757 if (type == XPS_CPUS) {
2758 if (numa_node_id == -2)
2759 numa_node_id = cpu_to_node(j);
2760 else if (numa_node_id != cpu_to_node(j))
2761 numa_node_id = -1;
2762 }
2763#endif
2764 }
2765
2766 if (copy)
2767 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2768 skip_tc);
2769 }
2770
2771 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2772
2773 /* Cleanup old maps */
2774 if (!dev_maps)
2775 goto out_no_old_maps;
2776
2777 for (j = 0; j < dev_maps->nr_ids; j++) {
2778 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2779 map = xmap_dereference(dev_maps->attr_map[tci]);
2780 if (!map)
2781 continue;
2782
2783 if (copy) {
2784 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2785 if (map == new_map)
2786 continue;
2787 }
2788
2789 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2790 kfree_rcu(map, rcu);
2791 }
2792 }
2793
2794 old_dev_maps = dev_maps;
2795
2796out_no_old_maps:
2797 dev_maps = new_dev_maps;
2798 active = true;
2799
2800out_no_new_maps:
2801 if (type == XPS_CPUS)
2802 /* update Tx queue numa node */
2803 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2804 (numa_node_id >= 0) ?
2805 numa_node_id : NUMA_NO_NODE);
2806
2807 if (!dev_maps)
2808 goto out_no_maps;
2809
2810 /* removes tx-queue from unused CPUs/rx-queues */
2811 for (j = 0; j < dev_maps->nr_ids; j++) {
2812 tci = j * dev_maps->num_tc;
2813
2814 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2815 if (i == tc &&
2816 netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2817 netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2818 continue;
2819
2820 active |= remove_xps_queue(dev_maps,
2821 copy ? old_dev_maps : NULL,
2822 tci, index);
2823 }
2824 }
2825
2826 if (old_dev_maps)
2827 kfree_rcu(old_dev_maps, rcu);
2828
2829 /* free map if not active */
2830 if (!active)
2831 reset_xps_maps(dev, dev_maps, type);
2832
2833out_no_maps:
2834 mutex_unlock(&xps_map_mutex);
2835
2836 return 0;
2837error:
2838 /* remove any maps that we added */
2839 for (j = 0; j < nr_ids; j++) {
2840 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2841 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2842 map = copy ?
2843 xmap_dereference(dev_maps->attr_map[tci]) :
2844 NULL;
2845 if (new_map && new_map != map)
2846 kfree(new_map);
2847 }
2848 }
2849
2850 mutex_unlock(&xps_map_mutex);
2851
2852 kfree(new_dev_maps);
2853 return -ENOMEM;
2854}
2855EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2856
2857int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2858 u16 index)
2859{
2860 int ret;
2861
2862 cpus_read_lock();
2863 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2864 cpus_read_unlock();
2865
2866 return ret;
2867}
2868EXPORT_SYMBOL(netif_set_xps_queue);
2869
2870#endif
2871static void netdev_unbind_all_sb_channels(struct net_device *dev)
2872{
2873 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2874
2875 /* Unbind any subordinate channels */
2876 while (txq-- != &dev->_tx[0]) {
2877 if (txq->sb_dev)
2878 netdev_unbind_sb_channel(dev, txq->sb_dev);
2879 }
2880}
2881
2882void netdev_reset_tc(struct net_device *dev)
2883{
2884#ifdef CONFIG_XPS
2885 netif_reset_xps_queues_gt(dev, 0);
2886#endif
2887 netdev_unbind_all_sb_channels(dev);
2888
2889 /* Reset TC configuration of device */
2890 dev->num_tc = 0;
2891 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2892 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2893}
2894EXPORT_SYMBOL(netdev_reset_tc);
2895
2896int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2897{
2898 if (tc >= dev->num_tc)
2899 return -EINVAL;
2900
2901#ifdef CONFIG_XPS
2902 netif_reset_xps_queues(dev, offset, count);
2903#endif
2904 dev->tc_to_txq[tc].count = count;
2905 dev->tc_to_txq[tc].offset = offset;
2906 return 0;
2907}
2908EXPORT_SYMBOL(netdev_set_tc_queue);
2909
2910int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2911{
2912 if (num_tc > TC_MAX_QUEUE)
2913 return -EINVAL;
2914
2915#ifdef CONFIG_XPS
2916 netif_reset_xps_queues_gt(dev, 0);
2917#endif
2918 netdev_unbind_all_sb_channels(dev);
2919
2920 dev->num_tc = num_tc;
2921 return 0;
2922}
2923EXPORT_SYMBOL(netdev_set_num_tc);
2924
2925void netdev_unbind_sb_channel(struct net_device *dev,
2926 struct net_device *sb_dev)
2927{
2928 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2929
2930#ifdef CONFIG_XPS
2931 netif_reset_xps_queues_gt(sb_dev, 0);
2932#endif
2933 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2934 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2935
2936 while (txq-- != &dev->_tx[0]) {
2937 if (txq->sb_dev == sb_dev)
2938 txq->sb_dev = NULL;
2939 }
2940}
2941EXPORT_SYMBOL(netdev_unbind_sb_channel);
2942
2943int netdev_bind_sb_channel_queue(struct net_device *dev,
2944 struct net_device *sb_dev,
2945 u8 tc, u16 count, u16 offset)
2946{
2947 /* Make certain the sb_dev and dev are already configured */
2948 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2949 return -EINVAL;
2950
2951 /* We cannot hand out queues we don't have */
2952 if ((offset + count) > dev->real_num_tx_queues)
2953 return -EINVAL;
2954
2955 /* Record the mapping */
2956 sb_dev->tc_to_txq[tc].count = count;
2957 sb_dev->tc_to_txq[tc].offset = offset;
2958
2959 /* Provide a way for Tx queue to find the tc_to_txq map or
2960 * XPS map for itself.
2961 */
2962 while (count--)
2963 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2964
2965 return 0;
2966}
2967EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2968
2969int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2970{
2971 /* Do not use a multiqueue device to represent a subordinate channel */
2972 if (netif_is_multiqueue(dev))
2973 return -ENODEV;
2974
2975 /* We allow channels 1 - 32767 to be used for subordinate channels.
2976 * Channel 0 is meant to be "native" mode and used only to represent
2977 * the main root device. We allow writing 0 to reset the device back
2978 * to normal mode after being used as a subordinate channel.
2979 */
2980 if (channel > S16_MAX)
2981 return -EINVAL;
2982
2983 dev->num_tc = -channel;
2984
2985 return 0;
2986}
2987EXPORT_SYMBOL(netdev_set_sb_channel);
2988
2989/*
2990 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2991 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2992 */
2993int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2994{
2995 bool disabling;
2996 int rc;
2997
2998 disabling = txq < dev->real_num_tx_queues;
2999
3000 if (txq < 1 || txq > dev->num_tx_queues)
3001 return -EINVAL;
3002
3003 if (dev->reg_state == NETREG_REGISTERED ||
3004 dev->reg_state == NETREG_UNREGISTERING) {
3005 ASSERT_RTNL();
3006
3007 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
3008 txq);
3009 if (rc)
3010 return rc;
3011
3012 if (dev->num_tc)
3013 netif_setup_tc(dev, txq);
3014
3015 net_shaper_set_real_num_tx_queues(dev, txq);
3016
3017 dev_qdisc_change_real_num_tx(dev, txq);
3018
3019 dev->real_num_tx_queues = txq;
3020
3021 if (disabling) {
3022 synchronize_net();
3023 qdisc_reset_all_tx_gt(dev, txq);
3024#ifdef CONFIG_XPS
3025 netif_reset_xps_queues_gt(dev, txq);
3026#endif
3027 }
3028 } else {
3029 dev->real_num_tx_queues = txq;
3030 }
3031
3032 return 0;
3033}
3034EXPORT_SYMBOL(netif_set_real_num_tx_queues);
3035
3036#ifdef CONFIG_SYSFS
3037/**
3038 * netif_set_real_num_rx_queues - set actual number of RX queues used
3039 * @dev: Network device
3040 * @rxq: Actual number of RX queues
3041 *
3042 * This must be called either with the rtnl_lock held or before
3043 * registration of the net device. Returns 0 on success, or a
3044 * negative error code. If called before registration, it always
3045 * succeeds.
3046 */
3047int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
3048{
3049 int rc;
3050
3051 if (rxq < 1 || rxq > dev->num_rx_queues)
3052 return -EINVAL;
3053
3054 if (dev->reg_state == NETREG_REGISTERED) {
3055 ASSERT_RTNL();
3056
3057 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
3058 rxq);
3059 if (rc)
3060 return rc;
3061 }
3062
3063 dev->real_num_rx_queues = rxq;
3064 return 0;
3065}
3066EXPORT_SYMBOL(netif_set_real_num_rx_queues);
3067#endif
3068
3069/**
3070 * netif_set_real_num_queues - set actual number of RX and TX queues used
3071 * @dev: Network device
3072 * @txq: Actual number of TX queues
3073 * @rxq: Actual number of RX queues
3074 *
3075 * Set the real number of both TX and RX queues.
3076 * Does nothing if the number of queues is already correct.
3077 */
3078int netif_set_real_num_queues(struct net_device *dev,
3079 unsigned int txq, unsigned int rxq)
3080{
3081 unsigned int old_rxq = dev->real_num_rx_queues;
3082 int err;
3083
3084 if (txq < 1 || txq > dev->num_tx_queues ||
3085 rxq < 1 || rxq > dev->num_rx_queues)
3086 return -EINVAL;
3087
3088 /* Start from increases, so the error path only does decreases -
3089 * decreases can't fail.
3090 */
3091 if (rxq > dev->real_num_rx_queues) {
3092 err = netif_set_real_num_rx_queues(dev, rxq);
3093 if (err)
3094 return err;
3095 }
3096 if (txq > dev->real_num_tx_queues) {
3097 err = netif_set_real_num_tx_queues(dev, txq);
3098 if (err)
3099 goto undo_rx;
3100 }
3101 if (rxq < dev->real_num_rx_queues)
3102 WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
3103 if (txq < dev->real_num_tx_queues)
3104 WARN_ON(netif_set_real_num_tx_queues(dev, txq));
3105
3106 return 0;
3107undo_rx:
3108 WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
3109 return err;
3110}
3111EXPORT_SYMBOL(netif_set_real_num_queues);
3112
3113/**
3114 * netif_set_tso_max_size() - set the max size of TSO frames supported
3115 * @dev: netdev to update
3116 * @size: max skb->len of a TSO frame
3117 *
3118 * Set the limit on the size of TSO super-frames the device can handle.
3119 * Unless explicitly set the stack will assume the value of
3120 * %GSO_LEGACY_MAX_SIZE.
3121 */
3122void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
3123{
3124 dev->tso_max_size = min(GSO_MAX_SIZE, size);
3125 if (size < READ_ONCE(dev->gso_max_size))
3126 netif_set_gso_max_size(dev, size);
3127 if (size < READ_ONCE(dev->gso_ipv4_max_size))
3128 netif_set_gso_ipv4_max_size(dev, size);
3129}
3130EXPORT_SYMBOL(netif_set_tso_max_size);
3131
3132/**
3133 * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3134 * @dev: netdev to update
3135 * @segs: max number of TCP segments
3136 *
3137 * Set the limit on the number of TCP segments the device can generate from
3138 * a single TSO super-frame.
3139 * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3140 */
3141void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3142{
3143 dev->tso_max_segs = segs;
3144 if (segs < READ_ONCE(dev->gso_max_segs))
3145 netif_set_gso_max_segs(dev, segs);
3146}
3147EXPORT_SYMBOL(netif_set_tso_max_segs);
3148
3149/**
3150 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3151 * @to: netdev to update
3152 * @from: netdev from which to copy the limits
3153 */
3154void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
3155{
3156 netif_set_tso_max_size(to, from->tso_max_size);
3157 netif_set_tso_max_segs(to, from->tso_max_segs);
3158}
3159EXPORT_SYMBOL(netif_inherit_tso_max);
3160
3161/**
3162 * netif_get_num_default_rss_queues - default number of RSS queues
3163 *
3164 * Default value is the number of physical cores if there are only 1 or 2, or
3165 * divided by 2 if there are more.
3166 */
3167int netif_get_num_default_rss_queues(void)
3168{
3169 cpumask_var_t cpus;
3170 int cpu, count = 0;
3171
3172 if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3173 return 1;
3174
3175 cpumask_copy(cpus, cpu_online_mask);
3176 for_each_cpu(cpu, cpus) {
3177 ++count;
3178 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3179 }
3180 free_cpumask_var(cpus);
3181
3182 return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3183}
3184EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3185
3186static void __netif_reschedule(struct Qdisc *q)
3187{
3188 struct softnet_data *sd;
3189 unsigned long flags;
3190
3191 local_irq_save(flags);
3192 sd = this_cpu_ptr(&softnet_data);
3193 q->next_sched = NULL;
3194 *sd->output_queue_tailp = q;
3195 sd->output_queue_tailp = &q->next_sched;
3196 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3197 local_irq_restore(flags);
3198}
3199
3200void __netif_schedule(struct Qdisc *q)
3201{
3202 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3203 __netif_reschedule(q);
3204}
3205EXPORT_SYMBOL(__netif_schedule);
3206
3207struct dev_kfree_skb_cb {
3208 enum skb_drop_reason reason;
3209};
3210
3211static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3212{
3213 return (struct dev_kfree_skb_cb *)skb->cb;
3214}
3215
3216void netif_schedule_queue(struct netdev_queue *txq)
3217{
3218 rcu_read_lock();
3219 if (!netif_xmit_stopped(txq)) {
3220 struct Qdisc *q = rcu_dereference(txq->qdisc);
3221
3222 __netif_schedule(q);
3223 }
3224 rcu_read_unlock();
3225}
3226EXPORT_SYMBOL(netif_schedule_queue);
3227
3228void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3229{
3230 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3231 struct Qdisc *q;
3232
3233 rcu_read_lock();
3234 q = rcu_dereference(dev_queue->qdisc);
3235 __netif_schedule(q);
3236 rcu_read_unlock();
3237 }
3238}
3239EXPORT_SYMBOL(netif_tx_wake_queue);
3240
3241void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3242{
3243 unsigned long flags;
3244
3245 if (unlikely(!skb))
3246 return;
3247
3248 if (likely(refcount_read(&skb->users) == 1)) {
3249 smp_rmb();
3250 refcount_set(&skb->users, 0);
3251 } else if (likely(!refcount_dec_and_test(&skb->users))) {
3252 return;
3253 }
3254 get_kfree_skb_cb(skb)->reason = reason;
3255 local_irq_save(flags);
3256 skb->next = __this_cpu_read(softnet_data.completion_queue);
3257 __this_cpu_write(softnet_data.completion_queue, skb);
3258 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3259 local_irq_restore(flags);
3260}
3261EXPORT_SYMBOL(dev_kfree_skb_irq_reason);
3262
3263void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3264{
3265 if (in_hardirq() || irqs_disabled())
3266 dev_kfree_skb_irq_reason(skb, reason);
3267 else
3268 kfree_skb_reason(skb, reason);
3269}
3270EXPORT_SYMBOL(dev_kfree_skb_any_reason);
3271
3272
3273/**
3274 * netif_device_detach - mark device as removed
3275 * @dev: network device
3276 *
3277 * Mark device as removed from system and therefore no longer available.
3278 */
3279void netif_device_detach(struct net_device *dev)
3280{
3281 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3282 netif_running(dev)) {
3283 netif_tx_stop_all_queues(dev);
3284 }
3285}
3286EXPORT_SYMBOL(netif_device_detach);
3287
3288/**
3289 * netif_device_attach - mark device as attached
3290 * @dev: network device
3291 *
3292 * Mark device as attached from system and restart if needed.
3293 */
3294void netif_device_attach(struct net_device *dev)
3295{
3296 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3297 netif_running(dev)) {
3298 netif_tx_wake_all_queues(dev);
3299 __netdev_watchdog_up(dev);
3300 }
3301}
3302EXPORT_SYMBOL(netif_device_attach);
3303
3304/*
3305 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3306 * to be used as a distribution range.
3307 */
3308static u16 skb_tx_hash(const struct net_device *dev,
3309 const struct net_device *sb_dev,
3310 struct sk_buff *skb)
3311{
3312 u32 hash;
3313 u16 qoffset = 0;
3314 u16 qcount = dev->real_num_tx_queues;
3315
3316 if (dev->num_tc) {
3317 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3318
3319 qoffset = sb_dev->tc_to_txq[tc].offset;
3320 qcount = sb_dev->tc_to_txq[tc].count;
3321 if (unlikely(!qcount)) {
3322 net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3323 sb_dev->name, qoffset, tc);
3324 qoffset = 0;
3325 qcount = dev->real_num_tx_queues;
3326 }
3327 }
3328
3329 if (skb_rx_queue_recorded(skb)) {
3330 DEBUG_NET_WARN_ON_ONCE(qcount == 0);
3331 hash = skb_get_rx_queue(skb);
3332 if (hash >= qoffset)
3333 hash -= qoffset;
3334 while (unlikely(hash >= qcount))
3335 hash -= qcount;
3336 return hash + qoffset;
3337 }
3338
3339 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3340}
3341
3342void skb_warn_bad_offload(const struct sk_buff *skb)
3343{
3344 static const netdev_features_t null_features;
3345 struct net_device *dev = skb->dev;
3346 const char *name = "";
3347
3348 if (!net_ratelimit())
3349 return;
3350
3351 if (dev) {
3352 if (dev->dev.parent)
3353 name = dev_driver_string(dev->dev.parent);
3354 else
3355 name = netdev_name(dev);
3356 }
3357 skb_dump(KERN_WARNING, skb, false);
3358 WARN(1, "%s: caps=(%pNF, %pNF)\n",
3359 name, dev ? &dev->features : &null_features,
3360 skb->sk ? &skb->sk->sk_route_caps : &null_features);
3361}
3362
3363/*
3364 * Invalidate hardware checksum when packet is to be mangled, and
3365 * complete checksum manually on outgoing path.
3366 */
3367int skb_checksum_help(struct sk_buff *skb)
3368{
3369 __wsum csum;
3370 int ret = 0, offset;
3371
3372 if (skb->ip_summed == CHECKSUM_COMPLETE)
3373 goto out_set_summed;
3374
3375 if (unlikely(skb_is_gso(skb))) {
3376 skb_warn_bad_offload(skb);
3377 return -EINVAL;
3378 }
3379
3380 if (!skb_frags_readable(skb)) {
3381 return -EFAULT;
3382 }
3383
3384 /* Before computing a checksum, we should make sure no frag could
3385 * be modified by an external entity : checksum could be wrong.
3386 */
3387 if (skb_has_shared_frag(skb)) {
3388 ret = __skb_linearize(skb);
3389 if (ret)
3390 goto out;
3391 }
3392
3393 offset = skb_checksum_start_offset(skb);
3394 ret = -EINVAL;
3395 if (unlikely(offset >= skb_headlen(skb))) {
3396 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3397 WARN_ONCE(true, "offset (%d) >= skb_headlen() (%u)\n",
3398 offset, skb_headlen(skb));
3399 goto out;
3400 }
3401 csum = skb_checksum(skb, offset, skb->len - offset, 0);
3402
3403 offset += skb->csum_offset;
3404 if (unlikely(offset + sizeof(__sum16) > skb_headlen(skb))) {
3405 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3406 WARN_ONCE(true, "offset+2 (%zu) > skb_headlen() (%u)\n",
3407 offset + sizeof(__sum16), skb_headlen(skb));
3408 goto out;
3409 }
3410 ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3411 if (ret)
3412 goto out;
3413
3414 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3415out_set_summed:
3416 skb->ip_summed = CHECKSUM_NONE;
3417out:
3418 return ret;
3419}
3420EXPORT_SYMBOL(skb_checksum_help);
3421
3422int skb_crc32c_csum_help(struct sk_buff *skb)
3423{
3424 __le32 crc32c_csum;
3425 int ret = 0, offset, start;
3426
3427 if (skb->ip_summed != CHECKSUM_PARTIAL)
3428 goto out;
3429
3430 if (unlikely(skb_is_gso(skb)))
3431 goto out;
3432
3433 /* Before computing a checksum, we should make sure no frag could
3434 * be modified by an external entity : checksum could be wrong.
3435 */
3436 if (unlikely(skb_has_shared_frag(skb))) {
3437 ret = __skb_linearize(skb);
3438 if (ret)
3439 goto out;
3440 }
3441 start = skb_checksum_start_offset(skb);
3442 offset = start + offsetof(struct sctphdr, checksum);
3443 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3444 ret = -EINVAL;
3445 goto out;
3446 }
3447
3448 ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3449 if (ret)
3450 goto out;
3451
3452 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3453 skb->len - start, ~(__u32)0,
3454 crc32c_csum_stub));
3455 *(__le32 *)(skb->data + offset) = crc32c_csum;
3456 skb_reset_csum_not_inet(skb);
3457out:
3458 return ret;
3459}
3460EXPORT_SYMBOL(skb_crc32c_csum_help);
3461
3462__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3463{
3464 __be16 type = skb->protocol;
3465
3466 /* Tunnel gso handlers can set protocol to ethernet. */
3467 if (type == htons(ETH_P_TEB)) {
3468 struct ethhdr *eth;
3469
3470 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3471 return 0;
3472
3473 eth = (struct ethhdr *)skb->data;
3474 type = eth->h_proto;
3475 }
3476
3477 return vlan_get_protocol_and_depth(skb, type, depth);
3478}
3479
3480
3481/* Take action when hardware reception checksum errors are detected. */
3482#ifdef CONFIG_BUG
3483static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3484{
3485 netdev_err(dev, "hw csum failure\n");
3486 skb_dump(KERN_ERR, skb, true);
3487 dump_stack();
3488}
3489
3490void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3491{
3492 DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3493}
3494EXPORT_SYMBOL(netdev_rx_csum_fault);
3495#endif
3496
3497/* XXX: check that highmem exists at all on the given machine. */
3498static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3499{
3500#ifdef CONFIG_HIGHMEM
3501 int i;
3502
3503 if (!(dev->features & NETIF_F_HIGHDMA)) {
3504 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3505 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3506 struct page *page = skb_frag_page(frag);
3507
3508 if (page && PageHighMem(page))
3509 return 1;
3510 }
3511 }
3512#endif
3513 return 0;
3514}
3515
3516/* If MPLS offload request, verify we are testing hardware MPLS features
3517 * instead of standard features for the netdev.
3518 */
3519#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3520static netdev_features_t net_mpls_features(struct sk_buff *skb,
3521 netdev_features_t features,
3522 __be16 type)
3523{
3524 if (eth_p_mpls(type))
3525 features &= skb->dev->mpls_features;
3526
3527 return features;
3528}
3529#else
3530static netdev_features_t net_mpls_features(struct sk_buff *skb,
3531 netdev_features_t features,
3532 __be16 type)
3533{
3534 return features;
3535}
3536#endif
3537
3538static netdev_features_t harmonize_features(struct sk_buff *skb,
3539 netdev_features_t features)
3540{
3541 __be16 type;
3542
3543 type = skb_network_protocol(skb, NULL);
3544 features = net_mpls_features(skb, features, type);
3545
3546 if (skb->ip_summed != CHECKSUM_NONE &&
3547 !can_checksum_protocol(features, type)) {
3548 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3549 }
3550 if (illegal_highdma(skb->dev, skb))
3551 features &= ~NETIF_F_SG;
3552
3553 return features;
3554}
3555
3556netdev_features_t passthru_features_check(struct sk_buff *skb,
3557 struct net_device *dev,
3558 netdev_features_t features)
3559{
3560 return features;
3561}
3562EXPORT_SYMBOL(passthru_features_check);
3563
3564static netdev_features_t dflt_features_check(struct sk_buff *skb,
3565 struct net_device *dev,
3566 netdev_features_t features)
3567{
3568 return vlan_features_check(skb, features);
3569}
3570
3571static netdev_features_t gso_features_check(const struct sk_buff *skb,
3572 struct net_device *dev,
3573 netdev_features_t features)
3574{
3575 u16 gso_segs = skb_shinfo(skb)->gso_segs;
3576
3577 if (gso_segs > READ_ONCE(dev->gso_max_segs))
3578 return features & ~NETIF_F_GSO_MASK;
3579
3580 if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb)))
3581 return features & ~NETIF_F_GSO_MASK;
3582
3583 if (!skb_shinfo(skb)->gso_type) {
3584 skb_warn_bad_offload(skb);
3585 return features & ~NETIF_F_GSO_MASK;
3586 }
3587
3588 /* Support for GSO partial features requires software
3589 * intervention before we can actually process the packets
3590 * so we need to strip support for any partial features now
3591 * and we can pull them back in after we have partially
3592 * segmented the frame.
3593 */
3594 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3595 features &= ~dev->gso_partial_features;
3596
3597 /* Make sure to clear the IPv4 ID mangling feature if the
3598 * IPv4 header has the potential to be fragmented.
3599 */
3600 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3601 struct iphdr *iph = skb->encapsulation ?
3602 inner_ip_hdr(skb) : ip_hdr(skb);
3603
3604 if (!(iph->frag_off & htons(IP_DF)))
3605 features &= ~NETIF_F_TSO_MANGLEID;
3606 }
3607
3608 return features;
3609}
3610
3611netdev_features_t netif_skb_features(struct sk_buff *skb)
3612{
3613 struct net_device *dev = skb->dev;
3614 netdev_features_t features = dev->features;
3615
3616 if (skb_is_gso(skb))
3617 features = gso_features_check(skb, dev, features);
3618
3619 /* If encapsulation offload request, verify we are testing
3620 * hardware encapsulation features instead of standard
3621 * features for the netdev
3622 */
3623 if (skb->encapsulation)
3624 features &= dev->hw_enc_features;
3625
3626 if (skb_vlan_tagged(skb))
3627 features = netdev_intersect_features(features,
3628 dev->vlan_features |
3629 NETIF_F_HW_VLAN_CTAG_TX |
3630 NETIF_F_HW_VLAN_STAG_TX);
3631
3632 if (dev->netdev_ops->ndo_features_check)
3633 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3634 features);
3635 else
3636 features &= dflt_features_check(skb, dev, features);
3637
3638 return harmonize_features(skb, features);
3639}
3640EXPORT_SYMBOL(netif_skb_features);
3641
3642static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3643 struct netdev_queue *txq, bool more)
3644{
3645 unsigned int len;
3646 int rc;
3647
3648 if (dev_nit_active(dev))
3649 dev_queue_xmit_nit(skb, dev);
3650
3651 len = skb->len;
3652 trace_net_dev_start_xmit(skb, dev);
3653 rc = netdev_start_xmit(skb, dev, txq, more);
3654 trace_net_dev_xmit(skb, rc, dev, len);
3655
3656 return rc;
3657}
3658
3659struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3660 struct netdev_queue *txq, int *ret)
3661{
3662 struct sk_buff *skb = first;
3663 int rc = NETDEV_TX_OK;
3664
3665 while (skb) {
3666 struct sk_buff *next = skb->next;
3667
3668 skb_mark_not_on_list(skb);
3669 rc = xmit_one(skb, dev, txq, next != NULL);
3670 if (unlikely(!dev_xmit_complete(rc))) {
3671 skb->next = next;
3672 goto out;
3673 }
3674
3675 skb = next;
3676 if (netif_tx_queue_stopped(txq) && skb) {
3677 rc = NETDEV_TX_BUSY;
3678 break;
3679 }
3680 }
3681
3682out:
3683 *ret = rc;
3684 return skb;
3685}
3686
3687static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3688 netdev_features_t features)
3689{
3690 if (skb_vlan_tag_present(skb) &&
3691 !vlan_hw_offload_capable(features, skb->vlan_proto))
3692 skb = __vlan_hwaccel_push_inside(skb);
3693 return skb;
3694}
3695
3696int skb_csum_hwoffload_help(struct sk_buff *skb,
3697 const netdev_features_t features)
3698{
3699 if (unlikely(skb_csum_is_sctp(skb)))
3700 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3701 skb_crc32c_csum_help(skb);
3702
3703 if (features & NETIF_F_HW_CSUM)
3704 return 0;
3705
3706 if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3707 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6) &&
3708 skb_network_header_len(skb) != sizeof(struct ipv6hdr) &&
3709 !ipv6_has_hopopt_jumbo(skb))
3710 goto sw_checksum;
3711
3712 switch (skb->csum_offset) {
3713 case offsetof(struct tcphdr, check):
3714 case offsetof(struct udphdr, check):
3715 return 0;
3716 }
3717 }
3718
3719sw_checksum:
3720 return skb_checksum_help(skb);
3721}
3722EXPORT_SYMBOL(skb_csum_hwoffload_help);
3723
3724static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3725{
3726 netdev_features_t features;
3727
3728 features = netif_skb_features(skb);
3729 skb = validate_xmit_vlan(skb, features);
3730 if (unlikely(!skb))
3731 goto out_null;
3732
3733 skb = sk_validate_xmit_skb(skb, dev);
3734 if (unlikely(!skb))
3735 goto out_null;
3736
3737 if (netif_needs_gso(skb, features)) {
3738 struct sk_buff *segs;
3739
3740 segs = skb_gso_segment(skb, features);
3741 if (IS_ERR(segs)) {
3742 goto out_kfree_skb;
3743 } else if (segs) {
3744 consume_skb(skb);
3745 skb = segs;
3746 }
3747 } else {
3748 if (skb_needs_linearize(skb, features) &&
3749 __skb_linearize(skb))
3750 goto out_kfree_skb;
3751
3752 /* If packet is not checksummed and device does not
3753 * support checksumming for this protocol, complete
3754 * checksumming here.
3755 */
3756 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3757 if (skb->encapsulation)
3758 skb_set_inner_transport_header(skb,
3759 skb_checksum_start_offset(skb));
3760 else
3761 skb_set_transport_header(skb,
3762 skb_checksum_start_offset(skb));
3763 if (skb_csum_hwoffload_help(skb, features))
3764 goto out_kfree_skb;
3765 }
3766 }
3767
3768 skb = validate_xmit_xfrm(skb, features, again);
3769
3770 return skb;
3771
3772out_kfree_skb:
3773 kfree_skb(skb);
3774out_null:
3775 dev_core_stats_tx_dropped_inc(dev);
3776 return NULL;
3777}
3778
3779struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3780{
3781 struct sk_buff *next, *head = NULL, *tail;
3782
3783 for (; skb != NULL; skb = next) {
3784 next = skb->next;
3785 skb_mark_not_on_list(skb);
3786
3787 /* in case skb won't be segmented, point to itself */
3788 skb->prev = skb;
3789
3790 skb = validate_xmit_skb(skb, dev, again);
3791 if (!skb)
3792 continue;
3793
3794 if (!head)
3795 head = skb;
3796 else
3797 tail->next = skb;
3798 /* If skb was segmented, skb->prev points to
3799 * the last segment. If not, it still contains skb.
3800 */
3801 tail = skb->prev;
3802 }
3803 return head;
3804}
3805EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3806
3807static void qdisc_pkt_len_init(struct sk_buff *skb)
3808{
3809 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3810
3811 qdisc_skb_cb(skb)->pkt_len = skb->len;
3812
3813 /* To get more precise estimation of bytes sent on wire,
3814 * we add to pkt_len the headers size of all segments
3815 */
3816 if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3817 u16 gso_segs = shinfo->gso_segs;
3818 unsigned int hdr_len;
3819
3820 /* mac layer + network layer */
3821 hdr_len = skb_transport_offset(skb);
3822
3823 /* + transport layer */
3824 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3825 const struct tcphdr *th;
3826 struct tcphdr _tcphdr;
3827
3828 th = skb_header_pointer(skb, hdr_len,
3829 sizeof(_tcphdr), &_tcphdr);
3830 if (likely(th))
3831 hdr_len += __tcp_hdrlen(th);
3832 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
3833 struct udphdr _udphdr;
3834
3835 if (skb_header_pointer(skb, hdr_len,
3836 sizeof(_udphdr), &_udphdr))
3837 hdr_len += sizeof(struct udphdr);
3838 }
3839
3840 if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) {
3841 int payload = skb->len - hdr_len;
3842
3843 /* Malicious packet. */
3844 if (payload <= 0)
3845 return;
3846 gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size);
3847 }
3848 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3849 }
3850}
3851
3852static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3853 struct sk_buff **to_free,
3854 struct netdev_queue *txq)
3855{
3856 int rc;
3857
3858 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3859 if (rc == NET_XMIT_SUCCESS)
3860 trace_qdisc_enqueue(q, txq, skb);
3861 return rc;
3862}
3863
3864static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3865 struct net_device *dev,
3866 struct netdev_queue *txq)
3867{
3868 spinlock_t *root_lock = qdisc_lock(q);
3869 struct sk_buff *to_free = NULL;
3870 bool contended;
3871 int rc;
3872
3873 qdisc_calculate_pkt_len(skb, q);
3874
3875 tcf_set_drop_reason(skb, SKB_DROP_REASON_QDISC_DROP);
3876
3877 if (q->flags & TCQ_F_NOLOCK) {
3878 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3879 qdisc_run_begin(q)) {
3880 /* Retest nolock_qdisc_is_empty() within the protection
3881 * of q->seqlock to protect from racing with requeuing.
3882 */
3883 if (unlikely(!nolock_qdisc_is_empty(q))) {
3884 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3885 __qdisc_run(q);
3886 qdisc_run_end(q);
3887
3888 goto no_lock_out;
3889 }
3890
3891 qdisc_bstats_cpu_update(q, skb);
3892 if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3893 !nolock_qdisc_is_empty(q))
3894 __qdisc_run(q);
3895
3896 qdisc_run_end(q);
3897 return NET_XMIT_SUCCESS;
3898 }
3899
3900 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3901 qdisc_run(q);
3902
3903no_lock_out:
3904 if (unlikely(to_free))
3905 kfree_skb_list_reason(to_free,
3906 tcf_get_drop_reason(to_free));
3907 return rc;
3908 }
3909
3910 if (unlikely(READ_ONCE(q->owner) == smp_processor_id())) {
3911 kfree_skb_reason(skb, SKB_DROP_REASON_TC_RECLASSIFY_LOOP);
3912 return NET_XMIT_DROP;
3913 }
3914 /*
3915 * Heuristic to force contended enqueues to serialize on a
3916 * separate lock before trying to get qdisc main lock.
3917 * This permits qdisc->running owner to get the lock more
3918 * often and dequeue packets faster.
3919 * On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
3920 * and then other tasks will only enqueue packets. The packets will be
3921 * sent after the qdisc owner is scheduled again. To prevent this
3922 * scenario the task always serialize on the lock.
3923 */
3924 contended = qdisc_is_running(q) || IS_ENABLED(CONFIG_PREEMPT_RT);
3925 if (unlikely(contended))
3926 spin_lock(&q->busylock);
3927
3928 spin_lock(root_lock);
3929 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3930 __qdisc_drop(skb, &to_free);
3931 rc = NET_XMIT_DROP;
3932 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3933 qdisc_run_begin(q)) {
3934 /*
3935 * This is a work-conserving queue; there are no old skbs
3936 * waiting to be sent out; and the qdisc is not running -
3937 * xmit the skb directly.
3938 */
3939
3940 qdisc_bstats_update(q, skb);
3941
3942 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3943 if (unlikely(contended)) {
3944 spin_unlock(&q->busylock);
3945 contended = false;
3946 }
3947 __qdisc_run(q);
3948 }
3949
3950 qdisc_run_end(q);
3951 rc = NET_XMIT_SUCCESS;
3952 } else {
3953 WRITE_ONCE(q->owner, smp_processor_id());
3954 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3955 WRITE_ONCE(q->owner, -1);
3956 if (qdisc_run_begin(q)) {
3957 if (unlikely(contended)) {
3958 spin_unlock(&q->busylock);
3959 contended = false;
3960 }
3961 __qdisc_run(q);
3962 qdisc_run_end(q);
3963 }
3964 }
3965 spin_unlock(root_lock);
3966 if (unlikely(to_free))
3967 kfree_skb_list_reason(to_free,
3968 tcf_get_drop_reason(to_free));
3969 if (unlikely(contended))
3970 spin_unlock(&q->busylock);
3971 return rc;
3972}
3973
3974#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3975static void skb_update_prio(struct sk_buff *skb)
3976{
3977 const struct netprio_map *map;
3978 const struct sock *sk;
3979 unsigned int prioidx;
3980
3981 if (skb->priority)
3982 return;
3983 map = rcu_dereference_bh(skb->dev->priomap);
3984 if (!map)
3985 return;
3986 sk = skb_to_full_sk(skb);
3987 if (!sk)
3988 return;
3989
3990 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3991
3992 if (prioidx < map->priomap_len)
3993 skb->priority = map->priomap[prioidx];
3994}
3995#else
3996#define skb_update_prio(skb)
3997#endif
3998
3999/**
4000 * dev_loopback_xmit - loop back @skb
4001 * @net: network namespace this loopback is happening in
4002 * @sk: sk needed to be a netfilter okfn
4003 * @skb: buffer to transmit
4004 */
4005int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
4006{
4007 skb_reset_mac_header(skb);
4008 __skb_pull(skb, skb_network_offset(skb));
4009 skb->pkt_type = PACKET_LOOPBACK;
4010 if (skb->ip_summed == CHECKSUM_NONE)
4011 skb->ip_summed = CHECKSUM_UNNECESSARY;
4012 DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
4013 skb_dst_force(skb);
4014 netif_rx(skb);
4015 return 0;
4016}
4017EXPORT_SYMBOL(dev_loopback_xmit);
4018
4019#ifdef CONFIG_NET_EGRESS
4020static struct netdev_queue *
4021netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
4022{
4023 int qm = skb_get_queue_mapping(skb);
4024
4025 return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
4026}
4027
4028#ifndef CONFIG_PREEMPT_RT
4029static bool netdev_xmit_txqueue_skipped(void)
4030{
4031 return __this_cpu_read(softnet_data.xmit.skip_txqueue);
4032}
4033
4034void netdev_xmit_skip_txqueue(bool skip)
4035{
4036 __this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
4037}
4038EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4039
4040#else
4041static bool netdev_xmit_txqueue_skipped(void)
4042{
4043 return current->net_xmit.skip_txqueue;
4044}
4045
4046void netdev_xmit_skip_txqueue(bool skip)
4047{
4048 current->net_xmit.skip_txqueue = skip;
4049}
4050EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4051#endif
4052#endif /* CONFIG_NET_EGRESS */
4053
4054#ifdef CONFIG_NET_XGRESS
4055static int tc_run(struct tcx_entry *entry, struct sk_buff *skb,
4056 enum skb_drop_reason *drop_reason)
4057{
4058 int ret = TC_ACT_UNSPEC;
4059#ifdef CONFIG_NET_CLS_ACT
4060 struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq);
4061 struct tcf_result res;
4062
4063 if (!miniq)
4064 return ret;
4065
4066 /* Global bypass */
4067 if (!static_branch_likely(&tcf_sw_enabled_key))
4068 return ret;
4069
4070 /* Block-wise bypass */
4071 if (tcf_block_bypass_sw(miniq->block))
4072 return ret;
4073
4074 tc_skb_cb(skb)->mru = 0;
4075 tc_skb_cb(skb)->post_ct = false;
4076 tcf_set_drop_reason(skb, *drop_reason);
4077
4078 mini_qdisc_bstats_cpu_update(miniq, skb);
4079 ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false);
4080 /* Only tcf related quirks below. */
4081 switch (ret) {
4082 case TC_ACT_SHOT:
4083 *drop_reason = tcf_get_drop_reason(skb);
4084 mini_qdisc_qstats_cpu_drop(miniq);
4085 break;
4086 case TC_ACT_OK:
4087 case TC_ACT_RECLASSIFY:
4088 skb->tc_index = TC_H_MIN(res.classid);
4089 break;
4090 }
4091#endif /* CONFIG_NET_CLS_ACT */
4092 return ret;
4093}
4094
4095static DEFINE_STATIC_KEY_FALSE(tcx_needed_key);
4096
4097void tcx_inc(void)
4098{
4099 static_branch_inc(&tcx_needed_key);
4100}
4101
4102void tcx_dec(void)
4103{
4104 static_branch_dec(&tcx_needed_key);
4105}
4106
4107static __always_inline enum tcx_action_base
4108tcx_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
4109 const bool needs_mac)
4110{
4111 const struct bpf_mprog_fp *fp;
4112 const struct bpf_prog *prog;
4113 int ret = TCX_NEXT;
4114
4115 if (needs_mac)
4116 __skb_push(skb, skb->mac_len);
4117 bpf_mprog_foreach_prog(entry, fp, prog) {
4118 bpf_compute_data_pointers(skb);
4119 ret = bpf_prog_run(prog, skb);
4120 if (ret != TCX_NEXT)
4121 break;
4122 }
4123 if (needs_mac)
4124 __skb_pull(skb, skb->mac_len);
4125 return tcx_action_code(skb, ret);
4126}
4127
4128static __always_inline struct sk_buff *
4129sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4130 struct net_device *orig_dev, bool *another)
4131{
4132 struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress);
4133 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_INGRESS;
4134 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
4135 int sch_ret;
4136
4137 if (!entry)
4138 return skb;
4139
4140 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
4141 if (*pt_prev) {
4142 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4143 *pt_prev = NULL;
4144 }
4145
4146 qdisc_skb_cb(skb)->pkt_len = skb->len;
4147 tcx_set_ingress(skb, true);
4148
4149 if (static_branch_unlikely(&tcx_needed_key)) {
4150 sch_ret = tcx_run(entry, skb, true);
4151 if (sch_ret != TC_ACT_UNSPEC)
4152 goto ingress_verdict;
4153 }
4154 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4155ingress_verdict:
4156 switch (sch_ret) {
4157 case TC_ACT_REDIRECT:
4158 /* skb_mac_header check was done by BPF, so we can safely
4159 * push the L2 header back before redirecting to another
4160 * netdev.
4161 */
4162 __skb_push(skb, skb->mac_len);
4163 if (skb_do_redirect(skb) == -EAGAIN) {
4164 __skb_pull(skb, skb->mac_len);
4165 *another = true;
4166 break;
4167 }
4168 *ret = NET_RX_SUCCESS;
4169 bpf_net_ctx_clear(bpf_net_ctx);
4170 return NULL;
4171 case TC_ACT_SHOT:
4172 kfree_skb_reason(skb, drop_reason);
4173 *ret = NET_RX_DROP;
4174 bpf_net_ctx_clear(bpf_net_ctx);
4175 return NULL;
4176 /* used by tc_run */
4177 case TC_ACT_STOLEN:
4178 case TC_ACT_QUEUED:
4179 case TC_ACT_TRAP:
4180 consume_skb(skb);
4181 fallthrough;
4182 case TC_ACT_CONSUMED:
4183 *ret = NET_RX_SUCCESS;
4184 bpf_net_ctx_clear(bpf_net_ctx);
4185 return NULL;
4186 }
4187 bpf_net_ctx_clear(bpf_net_ctx);
4188
4189 return skb;
4190}
4191
4192static __always_inline struct sk_buff *
4193sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4194{
4195 struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress);
4196 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_EGRESS;
4197 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
4198 int sch_ret;
4199
4200 if (!entry)
4201 return skb;
4202
4203 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
4204
4205 /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was
4206 * already set by the caller.
4207 */
4208 if (static_branch_unlikely(&tcx_needed_key)) {
4209 sch_ret = tcx_run(entry, skb, false);
4210 if (sch_ret != TC_ACT_UNSPEC)
4211 goto egress_verdict;
4212 }
4213 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4214egress_verdict:
4215 switch (sch_ret) {
4216 case TC_ACT_REDIRECT:
4217 /* No need to push/pop skb's mac_header here on egress! */
4218 skb_do_redirect(skb);
4219 *ret = NET_XMIT_SUCCESS;
4220 bpf_net_ctx_clear(bpf_net_ctx);
4221 return NULL;
4222 case TC_ACT_SHOT:
4223 kfree_skb_reason(skb, drop_reason);
4224 *ret = NET_XMIT_DROP;
4225 bpf_net_ctx_clear(bpf_net_ctx);
4226 return NULL;
4227 /* used by tc_run */
4228 case TC_ACT_STOLEN:
4229 case TC_ACT_QUEUED:
4230 case TC_ACT_TRAP:
4231 consume_skb(skb);
4232 fallthrough;
4233 case TC_ACT_CONSUMED:
4234 *ret = NET_XMIT_SUCCESS;
4235 bpf_net_ctx_clear(bpf_net_ctx);
4236 return NULL;
4237 }
4238 bpf_net_ctx_clear(bpf_net_ctx);
4239
4240 return skb;
4241}
4242#else
4243static __always_inline struct sk_buff *
4244sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4245 struct net_device *orig_dev, bool *another)
4246{
4247 return skb;
4248}
4249
4250static __always_inline struct sk_buff *
4251sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4252{
4253 return skb;
4254}
4255#endif /* CONFIG_NET_XGRESS */
4256
4257#ifdef CONFIG_XPS
4258static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4259 struct xps_dev_maps *dev_maps, unsigned int tci)
4260{
4261 int tc = netdev_get_prio_tc_map(dev, skb->priority);
4262 struct xps_map *map;
4263 int queue_index = -1;
4264
4265 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
4266 return queue_index;
4267
4268 tci *= dev_maps->num_tc;
4269 tci += tc;
4270
4271 map = rcu_dereference(dev_maps->attr_map[tci]);
4272 if (map) {
4273 if (map->len == 1)
4274 queue_index = map->queues[0];
4275 else
4276 queue_index = map->queues[reciprocal_scale(
4277 skb_get_hash(skb), map->len)];
4278 if (unlikely(queue_index >= dev->real_num_tx_queues))
4279 queue_index = -1;
4280 }
4281 return queue_index;
4282}
4283#endif
4284
4285static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4286 struct sk_buff *skb)
4287{
4288#ifdef CONFIG_XPS
4289 struct xps_dev_maps *dev_maps;
4290 struct sock *sk = skb->sk;
4291 int queue_index = -1;
4292
4293 if (!static_key_false(&xps_needed))
4294 return -1;
4295
4296 rcu_read_lock();
4297 if (!static_key_false(&xps_rxqs_needed))
4298 goto get_cpus_map;
4299
4300 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4301 if (dev_maps) {
4302 int tci = sk_rx_queue_get(sk);
4303
4304 if (tci >= 0)
4305 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4306 tci);
4307 }
4308
4309get_cpus_map:
4310 if (queue_index < 0) {
4311 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4312 if (dev_maps) {
4313 unsigned int tci = skb->sender_cpu - 1;
4314
4315 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4316 tci);
4317 }
4318 }
4319 rcu_read_unlock();
4320
4321 return queue_index;
4322#else
4323 return -1;
4324#endif
4325}
4326
4327u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4328 struct net_device *sb_dev)
4329{
4330 return 0;
4331}
4332EXPORT_SYMBOL(dev_pick_tx_zero);
4333
4334u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4335 struct net_device *sb_dev)
4336{
4337 struct sock *sk = skb->sk;
4338 int queue_index = sk_tx_queue_get(sk);
4339
4340 sb_dev = sb_dev ? : dev;
4341
4342 if (queue_index < 0 || skb->ooo_okay ||
4343 queue_index >= dev->real_num_tx_queues) {
4344 int new_index = get_xps_queue(dev, sb_dev, skb);
4345
4346 if (new_index < 0)
4347 new_index = skb_tx_hash(dev, sb_dev, skb);
4348
4349 if (queue_index != new_index && sk &&
4350 sk_fullsock(sk) &&
4351 rcu_access_pointer(sk->sk_dst_cache))
4352 sk_tx_queue_set(sk, new_index);
4353
4354 queue_index = new_index;
4355 }
4356
4357 return queue_index;
4358}
4359EXPORT_SYMBOL(netdev_pick_tx);
4360
4361struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4362 struct sk_buff *skb,
4363 struct net_device *sb_dev)
4364{
4365 int queue_index = 0;
4366
4367#ifdef CONFIG_XPS
4368 u32 sender_cpu = skb->sender_cpu - 1;
4369
4370 if (sender_cpu >= (u32)NR_CPUS)
4371 skb->sender_cpu = raw_smp_processor_id() + 1;
4372#endif
4373
4374 if (dev->real_num_tx_queues != 1) {
4375 const struct net_device_ops *ops = dev->netdev_ops;
4376
4377 if (ops->ndo_select_queue)
4378 queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4379 else
4380 queue_index = netdev_pick_tx(dev, skb, sb_dev);
4381
4382 queue_index = netdev_cap_txqueue(dev, queue_index);
4383 }
4384
4385 skb_set_queue_mapping(skb, queue_index);
4386 return netdev_get_tx_queue(dev, queue_index);
4387}
4388
4389/**
4390 * __dev_queue_xmit() - transmit a buffer
4391 * @skb: buffer to transmit
4392 * @sb_dev: suboordinate device used for L2 forwarding offload
4393 *
4394 * Queue a buffer for transmission to a network device. The caller must
4395 * have set the device and priority and built the buffer before calling
4396 * this function. The function can be called from an interrupt.
4397 *
4398 * When calling this method, interrupts MUST be enabled. This is because
4399 * the BH enable code must have IRQs enabled so that it will not deadlock.
4400 *
4401 * Regardless of the return value, the skb is consumed, so it is currently
4402 * difficult to retry a send to this method. (You can bump the ref count
4403 * before sending to hold a reference for retry if you are careful.)
4404 *
4405 * Return:
4406 * * 0 - buffer successfully transmitted
4407 * * positive qdisc return code - NET_XMIT_DROP etc.
4408 * * negative errno - other errors
4409 */
4410int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4411{
4412 struct net_device *dev = skb->dev;
4413 struct netdev_queue *txq = NULL;
4414 struct Qdisc *q;
4415 int rc = -ENOMEM;
4416 bool again = false;
4417
4418 skb_reset_mac_header(skb);
4419 skb_assert_len(skb);
4420
4421 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4422 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4423
4424 /* Disable soft irqs for various locks below. Also
4425 * stops preemption for RCU.
4426 */
4427 rcu_read_lock_bh();
4428
4429 skb_update_prio(skb);
4430
4431 qdisc_pkt_len_init(skb);
4432 tcx_set_ingress(skb, false);
4433#ifdef CONFIG_NET_EGRESS
4434 if (static_branch_unlikely(&egress_needed_key)) {
4435 if (nf_hook_egress_active()) {
4436 skb = nf_hook_egress(skb, &rc, dev);
4437 if (!skb)
4438 goto out;
4439 }
4440
4441 netdev_xmit_skip_txqueue(false);
4442
4443 nf_skip_egress(skb, true);
4444 skb = sch_handle_egress(skb, &rc, dev);
4445 if (!skb)
4446 goto out;
4447 nf_skip_egress(skb, false);
4448
4449 if (netdev_xmit_txqueue_skipped())
4450 txq = netdev_tx_queue_mapping(dev, skb);
4451 }
4452#endif
4453 /* If device/qdisc don't need skb->dst, release it right now while
4454 * its hot in this cpu cache.
4455 */
4456 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4457 skb_dst_drop(skb);
4458 else
4459 skb_dst_force(skb);
4460
4461 if (!txq)
4462 txq = netdev_core_pick_tx(dev, skb, sb_dev);
4463
4464 q = rcu_dereference_bh(txq->qdisc);
4465
4466 trace_net_dev_queue(skb);
4467 if (q->enqueue) {
4468 rc = __dev_xmit_skb(skb, q, dev, txq);
4469 goto out;
4470 }
4471
4472 /* The device has no queue. Common case for software devices:
4473 * loopback, all the sorts of tunnels...
4474
4475 * Really, it is unlikely that netif_tx_lock protection is necessary
4476 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
4477 * counters.)
4478 * However, it is possible, that they rely on protection
4479 * made by us here.
4480
4481 * Check this and shot the lock. It is not prone from deadlocks.
4482 *Either shot noqueue qdisc, it is even simpler 8)
4483 */
4484 if (dev->flags & IFF_UP) {
4485 int cpu = smp_processor_id(); /* ok because BHs are off */
4486
4487 /* Other cpus might concurrently change txq->xmit_lock_owner
4488 * to -1 or to their cpu id, but not to our id.
4489 */
4490 if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
4491 if (dev_xmit_recursion())
4492 goto recursion_alert;
4493
4494 skb = validate_xmit_skb(skb, dev, &again);
4495 if (!skb)
4496 goto out;
4497
4498 HARD_TX_LOCK(dev, txq, cpu);
4499
4500 if (!netif_xmit_stopped(txq)) {
4501 dev_xmit_recursion_inc();
4502 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4503 dev_xmit_recursion_dec();
4504 if (dev_xmit_complete(rc)) {
4505 HARD_TX_UNLOCK(dev, txq);
4506 goto out;
4507 }
4508 }
4509 HARD_TX_UNLOCK(dev, txq);
4510 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4511 dev->name);
4512 } else {
4513 /* Recursion is detected! It is possible,
4514 * unfortunately
4515 */
4516recursion_alert:
4517 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4518 dev->name);
4519 }
4520 }
4521
4522 rc = -ENETDOWN;
4523 rcu_read_unlock_bh();
4524
4525 dev_core_stats_tx_dropped_inc(dev);
4526 kfree_skb_list(skb);
4527 return rc;
4528out:
4529 rcu_read_unlock_bh();
4530 return rc;
4531}
4532EXPORT_SYMBOL(__dev_queue_xmit);
4533
4534int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4535{
4536 struct net_device *dev = skb->dev;
4537 struct sk_buff *orig_skb = skb;
4538 struct netdev_queue *txq;
4539 int ret = NETDEV_TX_BUSY;
4540 bool again = false;
4541
4542 if (unlikely(!netif_running(dev) ||
4543 !netif_carrier_ok(dev)))
4544 goto drop;
4545
4546 skb = validate_xmit_skb_list(skb, dev, &again);
4547 if (skb != orig_skb)
4548 goto drop;
4549
4550 skb_set_queue_mapping(skb, queue_id);
4551 txq = skb_get_tx_queue(dev, skb);
4552
4553 local_bh_disable();
4554
4555 dev_xmit_recursion_inc();
4556 HARD_TX_LOCK(dev, txq, smp_processor_id());
4557 if (!netif_xmit_frozen_or_drv_stopped(txq))
4558 ret = netdev_start_xmit(skb, dev, txq, false);
4559 HARD_TX_UNLOCK(dev, txq);
4560 dev_xmit_recursion_dec();
4561
4562 local_bh_enable();
4563 return ret;
4564drop:
4565 dev_core_stats_tx_dropped_inc(dev);
4566 kfree_skb_list(skb);
4567 return NET_XMIT_DROP;
4568}
4569EXPORT_SYMBOL(__dev_direct_xmit);
4570
4571/*************************************************************************
4572 * Receiver routines
4573 *************************************************************************/
4574static DEFINE_PER_CPU(struct task_struct *, backlog_napi);
4575
4576int weight_p __read_mostly = 64; /* old backlog weight */
4577int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
4578int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
4579
4580/* Called with irq disabled */
4581static inline void ____napi_schedule(struct softnet_data *sd,
4582 struct napi_struct *napi)
4583{
4584 struct task_struct *thread;
4585
4586 lockdep_assert_irqs_disabled();
4587
4588 if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4589 /* Paired with smp_mb__before_atomic() in
4590 * napi_enable()/dev_set_threaded().
4591 * Use READ_ONCE() to guarantee a complete
4592 * read on napi->thread. Only call
4593 * wake_up_process() when it's not NULL.
4594 */
4595 thread = READ_ONCE(napi->thread);
4596 if (thread) {
4597 if (use_backlog_threads() && thread == raw_cpu_read(backlog_napi))
4598 goto use_local_napi;
4599
4600 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4601 wake_up_process(thread);
4602 return;
4603 }
4604 }
4605
4606use_local_napi:
4607 list_add_tail(&napi->poll_list, &sd->poll_list);
4608 WRITE_ONCE(napi->list_owner, smp_processor_id());
4609 /* If not called from net_rx_action()
4610 * we have to raise NET_RX_SOFTIRQ.
4611 */
4612 if (!sd->in_net_rx_action)
4613 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4614}
4615
4616#ifdef CONFIG_RPS
4617
4618struct static_key_false rps_needed __read_mostly;
4619EXPORT_SYMBOL(rps_needed);
4620struct static_key_false rfs_needed __read_mostly;
4621EXPORT_SYMBOL(rfs_needed);
4622
4623static struct rps_dev_flow *
4624set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4625 struct rps_dev_flow *rflow, u16 next_cpu)
4626{
4627 if (next_cpu < nr_cpu_ids) {
4628 u32 head;
4629#ifdef CONFIG_RFS_ACCEL
4630 struct netdev_rx_queue *rxqueue;
4631 struct rps_dev_flow_table *flow_table;
4632 struct rps_dev_flow *old_rflow;
4633 u16 rxq_index;
4634 u32 flow_id;
4635 int rc;
4636
4637 /* Should we steer this flow to a different hardware queue? */
4638 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4639 !(dev->features & NETIF_F_NTUPLE))
4640 goto out;
4641 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4642 if (rxq_index == skb_get_rx_queue(skb))
4643 goto out;
4644
4645 rxqueue = dev->_rx + rxq_index;
4646 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4647 if (!flow_table)
4648 goto out;
4649 flow_id = skb_get_hash(skb) & flow_table->mask;
4650 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4651 rxq_index, flow_id);
4652 if (rc < 0)
4653 goto out;
4654 old_rflow = rflow;
4655 rflow = &flow_table->flows[flow_id];
4656 WRITE_ONCE(rflow->filter, rc);
4657 if (old_rflow->filter == rc)
4658 WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER);
4659 out:
4660#endif
4661 head = READ_ONCE(per_cpu(softnet_data, next_cpu).input_queue_head);
4662 rps_input_queue_tail_save(&rflow->last_qtail, head);
4663 }
4664
4665 WRITE_ONCE(rflow->cpu, next_cpu);
4666 return rflow;
4667}
4668
4669/*
4670 * get_rps_cpu is called from netif_receive_skb and returns the target
4671 * CPU from the RPS map of the receiving queue for a given skb.
4672 * rcu_read_lock must be held on entry.
4673 */
4674static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4675 struct rps_dev_flow **rflowp)
4676{
4677 const struct rps_sock_flow_table *sock_flow_table;
4678 struct netdev_rx_queue *rxqueue = dev->_rx;
4679 struct rps_dev_flow_table *flow_table;
4680 struct rps_map *map;
4681 int cpu = -1;
4682 u32 tcpu;
4683 u32 hash;
4684
4685 if (skb_rx_queue_recorded(skb)) {
4686 u16 index = skb_get_rx_queue(skb);
4687
4688 if (unlikely(index >= dev->real_num_rx_queues)) {
4689 WARN_ONCE(dev->real_num_rx_queues > 1,
4690 "%s received packet on queue %u, but number "
4691 "of RX queues is %u\n",
4692 dev->name, index, dev->real_num_rx_queues);
4693 goto done;
4694 }
4695 rxqueue += index;
4696 }
4697
4698 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4699
4700 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4701 map = rcu_dereference(rxqueue->rps_map);
4702 if (!flow_table && !map)
4703 goto done;
4704
4705 skb_reset_network_header(skb);
4706 hash = skb_get_hash(skb);
4707 if (!hash)
4708 goto done;
4709
4710 sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
4711 if (flow_table && sock_flow_table) {
4712 struct rps_dev_flow *rflow;
4713 u32 next_cpu;
4714 u32 ident;
4715
4716 /* First check into global flow table if there is a match.
4717 * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
4718 */
4719 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);
4720 if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask)
4721 goto try_rps;
4722
4723 next_cpu = ident & net_hotdata.rps_cpu_mask;
4724
4725 /* OK, now we know there is a match,
4726 * we can look at the local (per receive queue) flow table
4727 */
4728 rflow = &flow_table->flows[hash & flow_table->mask];
4729 tcpu = rflow->cpu;
4730
4731 /*
4732 * If the desired CPU (where last recvmsg was done) is
4733 * different from current CPU (one in the rx-queue flow
4734 * table entry), switch if one of the following holds:
4735 * - Current CPU is unset (>= nr_cpu_ids).
4736 * - Current CPU is offline.
4737 * - The current CPU's queue tail has advanced beyond the
4738 * last packet that was enqueued using this table entry.
4739 * This guarantees that all previous packets for the flow
4740 * have been dequeued, thus preserving in order delivery.
4741 */
4742 if (unlikely(tcpu != next_cpu) &&
4743 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4744 ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) -
4745 rflow->last_qtail)) >= 0)) {
4746 tcpu = next_cpu;
4747 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4748 }
4749
4750 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4751 *rflowp = rflow;
4752 cpu = tcpu;
4753 goto done;
4754 }
4755 }
4756
4757try_rps:
4758
4759 if (map) {
4760 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4761 if (cpu_online(tcpu)) {
4762 cpu = tcpu;
4763 goto done;
4764 }
4765 }
4766
4767done:
4768 return cpu;
4769}
4770
4771#ifdef CONFIG_RFS_ACCEL
4772
4773/**
4774 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4775 * @dev: Device on which the filter was set
4776 * @rxq_index: RX queue index
4777 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4778 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4779 *
4780 * Drivers that implement ndo_rx_flow_steer() should periodically call
4781 * this function for each installed filter and remove the filters for
4782 * which it returns %true.
4783 */
4784bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4785 u32 flow_id, u16 filter_id)
4786{
4787 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4788 struct rps_dev_flow_table *flow_table;
4789 struct rps_dev_flow *rflow;
4790 bool expire = true;
4791 unsigned int cpu;
4792
4793 rcu_read_lock();
4794 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4795 if (flow_table && flow_id <= flow_table->mask) {
4796 rflow = &flow_table->flows[flow_id];
4797 cpu = READ_ONCE(rflow->cpu);
4798 if (READ_ONCE(rflow->filter) == filter_id && cpu < nr_cpu_ids &&
4799 ((int)(READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head) -
4800 READ_ONCE(rflow->last_qtail)) <
4801 (int)(10 * flow_table->mask)))
4802 expire = false;
4803 }
4804 rcu_read_unlock();
4805 return expire;
4806}
4807EXPORT_SYMBOL(rps_may_expire_flow);
4808
4809#endif /* CONFIG_RFS_ACCEL */
4810
4811/* Called from hardirq (IPI) context */
4812static void rps_trigger_softirq(void *data)
4813{
4814 struct softnet_data *sd = data;
4815
4816 ____napi_schedule(sd, &sd->backlog);
4817 sd->received_rps++;
4818}
4819
4820#endif /* CONFIG_RPS */
4821
4822/* Called from hardirq (IPI) context */
4823static void trigger_rx_softirq(void *data)
4824{
4825 struct softnet_data *sd = data;
4826
4827 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4828 smp_store_release(&sd->defer_ipi_scheduled, 0);
4829}
4830
4831/*
4832 * After we queued a packet into sd->input_pkt_queue,
4833 * we need to make sure this queue is serviced soon.
4834 *
4835 * - If this is another cpu queue, link it to our rps_ipi_list,
4836 * and make sure we will process rps_ipi_list from net_rx_action().
4837 *
4838 * - If this is our own queue, NAPI schedule our backlog.
4839 * Note that this also raises NET_RX_SOFTIRQ.
4840 */
4841static void napi_schedule_rps(struct softnet_data *sd)
4842{
4843 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4844
4845#ifdef CONFIG_RPS
4846 if (sd != mysd) {
4847 if (use_backlog_threads()) {
4848 __napi_schedule_irqoff(&sd->backlog);
4849 return;
4850 }
4851
4852 sd->rps_ipi_next = mysd->rps_ipi_list;
4853 mysd->rps_ipi_list = sd;
4854
4855 /* If not called from net_rx_action() or napi_threaded_poll()
4856 * we have to raise NET_RX_SOFTIRQ.
4857 */
4858 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll)
4859 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4860 return;
4861 }
4862#endif /* CONFIG_RPS */
4863 __napi_schedule_irqoff(&mysd->backlog);
4864}
4865
4866void kick_defer_list_purge(struct softnet_data *sd, unsigned int cpu)
4867{
4868 unsigned long flags;
4869
4870 if (use_backlog_threads()) {
4871 backlog_lock_irq_save(sd, &flags);
4872
4873 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
4874 __napi_schedule_irqoff(&sd->backlog);
4875
4876 backlog_unlock_irq_restore(sd, &flags);
4877
4878 } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) {
4879 smp_call_function_single_async(cpu, &sd->defer_csd);
4880 }
4881}
4882
4883#ifdef CONFIG_NET_FLOW_LIMIT
4884int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4885#endif
4886
4887static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4888{
4889#ifdef CONFIG_NET_FLOW_LIMIT
4890 struct sd_flow_limit *fl;
4891 struct softnet_data *sd;
4892 unsigned int old_flow, new_flow;
4893
4894 if (qlen < (READ_ONCE(net_hotdata.max_backlog) >> 1))
4895 return false;
4896
4897 sd = this_cpu_ptr(&softnet_data);
4898
4899 rcu_read_lock();
4900 fl = rcu_dereference(sd->flow_limit);
4901 if (fl) {
4902 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4903 old_flow = fl->history[fl->history_head];
4904 fl->history[fl->history_head] = new_flow;
4905
4906 fl->history_head++;
4907 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4908
4909 if (likely(fl->buckets[old_flow]))
4910 fl->buckets[old_flow]--;
4911
4912 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4913 fl->count++;
4914 rcu_read_unlock();
4915 return true;
4916 }
4917 }
4918 rcu_read_unlock();
4919#endif
4920 return false;
4921}
4922
4923/*
4924 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4925 * queue (may be a remote CPU queue).
4926 */
4927static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4928 unsigned int *qtail)
4929{
4930 enum skb_drop_reason reason;
4931 struct softnet_data *sd;
4932 unsigned long flags;
4933 unsigned int qlen;
4934 int max_backlog;
4935 u32 tail;
4936
4937 reason = SKB_DROP_REASON_DEV_READY;
4938 if (!netif_running(skb->dev))
4939 goto bad_dev;
4940
4941 reason = SKB_DROP_REASON_CPU_BACKLOG;
4942 sd = &per_cpu(softnet_data, cpu);
4943
4944 qlen = skb_queue_len_lockless(&sd->input_pkt_queue);
4945 max_backlog = READ_ONCE(net_hotdata.max_backlog);
4946 if (unlikely(qlen > max_backlog))
4947 goto cpu_backlog_drop;
4948 backlog_lock_irq_save(sd, &flags);
4949 qlen = skb_queue_len(&sd->input_pkt_queue);
4950 if (qlen <= max_backlog && !skb_flow_limit(skb, qlen)) {
4951 if (!qlen) {
4952 /* Schedule NAPI for backlog device. We can use
4953 * non atomic operation as we own the queue lock.
4954 */
4955 if (!__test_and_set_bit(NAPI_STATE_SCHED,
4956 &sd->backlog.state))
4957 napi_schedule_rps(sd);
4958 }
4959 __skb_queue_tail(&sd->input_pkt_queue, skb);
4960 tail = rps_input_queue_tail_incr(sd);
4961 backlog_unlock_irq_restore(sd, &flags);
4962
4963 /* save the tail outside of the critical section */
4964 rps_input_queue_tail_save(qtail, tail);
4965 return NET_RX_SUCCESS;
4966 }
4967
4968 backlog_unlock_irq_restore(sd, &flags);
4969
4970cpu_backlog_drop:
4971 atomic_inc(&sd->dropped);
4972bad_dev:
4973 dev_core_stats_rx_dropped_inc(skb->dev);
4974 kfree_skb_reason(skb, reason);
4975 return NET_RX_DROP;
4976}
4977
4978static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4979{
4980 struct net_device *dev = skb->dev;
4981 struct netdev_rx_queue *rxqueue;
4982
4983 rxqueue = dev->_rx;
4984
4985 if (skb_rx_queue_recorded(skb)) {
4986 u16 index = skb_get_rx_queue(skb);
4987
4988 if (unlikely(index >= dev->real_num_rx_queues)) {
4989 WARN_ONCE(dev->real_num_rx_queues > 1,
4990 "%s received packet on queue %u, but number "
4991 "of RX queues is %u\n",
4992 dev->name, index, dev->real_num_rx_queues);
4993
4994 return rxqueue; /* Return first rxqueue */
4995 }
4996 rxqueue += index;
4997 }
4998 return rxqueue;
4999}
5000
5001u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
5002 struct bpf_prog *xdp_prog)
5003{
5004 void *orig_data, *orig_data_end, *hard_start;
5005 struct netdev_rx_queue *rxqueue;
5006 bool orig_bcast, orig_host;
5007 u32 mac_len, frame_sz;
5008 __be16 orig_eth_type;
5009 struct ethhdr *eth;
5010 u32 metalen, act;
5011 int off;
5012
5013 /* The XDP program wants to see the packet starting at the MAC
5014 * header.
5015 */
5016 mac_len = skb->data - skb_mac_header(skb);
5017 hard_start = skb->data - skb_headroom(skb);
5018
5019 /* SKB "head" area always have tailroom for skb_shared_info */
5020 frame_sz = (void *)skb_end_pointer(skb) - hard_start;
5021 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
5022
5023 rxqueue = netif_get_rxqueue(skb);
5024 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
5025 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
5026 skb_headlen(skb) + mac_len, true);
5027 if (skb_is_nonlinear(skb)) {
5028 skb_shinfo(skb)->xdp_frags_size = skb->data_len;
5029 xdp_buff_set_frags_flag(xdp);
5030 } else {
5031 xdp_buff_clear_frags_flag(xdp);
5032 }
5033
5034 orig_data_end = xdp->data_end;
5035 orig_data = xdp->data;
5036 eth = (struct ethhdr *)xdp->data;
5037 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
5038 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
5039 orig_eth_type = eth->h_proto;
5040
5041 act = bpf_prog_run_xdp(xdp_prog, xdp);
5042
5043 /* check if bpf_xdp_adjust_head was used */
5044 off = xdp->data - orig_data;
5045 if (off) {
5046 if (off > 0)
5047 __skb_pull(skb, off);
5048 else if (off < 0)
5049 __skb_push(skb, -off);
5050
5051 skb->mac_header += off;
5052 skb_reset_network_header(skb);
5053 }
5054
5055 /* check if bpf_xdp_adjust_tail was used */
5056 off = xdp->data_end - orig_data_end;
5057 if (off != 0) {
5058 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
5059 skb->len += off; /* positive on grow, negative on shrink */
5060 }
5061
5062 /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers
5063 * (e.g. bpf_xdp_adjust_tail), we need to update data_len here.
5064 */
5065 if (xdp_buff_has_frags(xdp))
5066 skb->data_len = skb_shinfo(skb)->xdp_frags_size;
5067 else
5068 skb->data_len = 0;
5069
5070 /* check if XDP changed eth hdr such SKB needs update */
5071 eth = (struct ethhdr *)xdp->data;
5072 if ((orig_eth_type != eth->h_proto) ||
5073 (orig_host != ether_addr_equal_64bits(eth->h_dest,
5074 skb->dev->dev_addr)) ||
5075 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
5076 __skb_push(skb, ETH_HLEN);
5077 skb->pkt_type = PACKET_HOST;
5078 skb->protocol = eth_type_trans(skb, skb->dev);
5079 }
5080
5081 /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
5082 * before calling us again on redirect path. We do not call do_redirect
5083 * as we leave that up to the caller.
5084 *
5085 * Caller is responsible for managing lifetime of skb (i.e. calling
5086 * kfree_skb in response to actions it cannot handle/XDP_DROP).
5087 */
5088 switch (act) {
5089 case XDP_REDIRECT:
5090 case XDP_TX:
5091 __skb_push(skb, mac_len);
5092 break;
5093 case XDP_PASS:
5094 metalen = xdp->data - xdp->data_meta;
5095 if (metalen)
5096 skb_metadata_set(skb, metalen);
5097 break;
5098 }
5099
5100 return act;
5101}
5102
5103static int
5104netif_skb_check_for_xdp(struct sk_buff **pskb, struct bpf_prog *prog)
5105{
5106 struct sk_buff *skb = *pskb;
5107 int err, hroom, troom;
5108
5109 if (!skb_cow_data_for_xdp(this_cpu_read(system_page_pool), pskb, prog))
5110 return 0;
5111
5112 /* In case we have to go down the path and also linearize,
5113 * then lets do the pskb_expand_head() work just once here.
5114 */
5115 hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
5116 troom = skb->tail + skb->data_len - skb->end;
5117 err = pskb_expand_head(skb,
5118 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
5119 troom > 0 ? troom + 128 : 0, GFP_ATOMIC);
5120 if (err)
5121 return err;
5122
5123 return skb_linearize(skb);
5124}
5125
5126static u32 netif_receive_generic_xdp(struct sk_buff **pskb,
5127 struct xdp_buff *xdp,
5128 struct bpf_prog *xdp_prog)
5129{
5130 struct sk_buff *skb = *pskb;
5131 u32 mac_len, act = XDP_DROP;
5132
5133 /* Reinjected packets coming from act_mirred or similar should
5134 * not get XDP generic processing.
5135 */
5136 if (skb_is_redirected(skb))
5137 return XDP_PASS;
5138
5139 /* XDP packets must have sufficient headroom of XDP_PACKET_HEADROOM
5140 * bytes. This is the guarantee that also native XDP provides,
5141 * thus we need to do it here as well.
5142 */
5143 mac_len = skb->data - skb_mac_header(skb);
5144 __skb_push(skb, mac_len);
5145
5146 if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
5147 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
5148 if (netif_skb_check_for_xdp(pskb, xdp_prog))
5149 goto do_drop;
5150 }
5151
5152 __skb_pull(*pskb, mac_len);
5153
5154 act = bpf_prog_run_generic_xdp(*pskb, xdp, xdp_prog);
5155 switch (act) {
5156 case XDP_REDIRECT:
5157 case XDP_TX:
5158 case XDP_PASS:
5159 break;
5160 default:
5161 bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act);
5162 fallthrough;
5163 case XDP_ABORTED:
5164 trace_xdp_exception((*pskb)->dev, xdp_prog, act);
5165 fallthrough;
5166 case XDP_DROP:
5167 do_drop:
5168 kfree_skb(*pskb);
5169 break;
5170 }
5171
5172 return act;
5173}
5174
5175/* When doing generic XDP we have to bypass the qdisc layer and the
5176 * network taps in order to match in-driver-XDP behavior. This also means
5177 * that XDP packets are able to starve other packets going through a qdisc,
5178 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
5179 * queues, so they do not have this starvation issue.
5180 */
5181void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
5182{
5183 struct net_device *dev = skb->dev;
5184 struct netdev_queue *txq;
5185 bool free_skb = true;
5186 int cpu, rc;
5187
5188 txq = netdev_core_pick_tx(dev, skb, NULL);
5189 cpu = smp_processor_id();
5190 HARD_TX_LOCK(dev, txq, cpu);
5191 if (!netif_xmit_frozen_or_drv_stopped(txq)) {
5192 rc = netdev_start_xmit(skb, dev, txq, 0);
5193 if (dev_xmit_complete(rc))
5194 free_skb = false;
5195 }
5196 HARD_TX_UNLOCK(dev, txq);
5197 if (free_skb) {
5198 trace_xdp_exception(dev, xdp_prog, XDP_TX);
5199 dev_core_stats_tx_dropped_inc(dev);
5200 kfree_skb(skb);
5201 }
5202}
5203
5204static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
5205
5206int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb)
5207{
5208 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
5209
5210 if (xdp_prog) {
5211 struct xdp_buff xdp;
5212 u32 act;
5213 int err;
5214
5215 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
5216 act = netif_receive_generic_xdp(pskb, &xdp, xdp_prog);
5217 if (act != XDP_PASS) {
5218 switch (act) {
5219 case XDP_REDIRECT:
5220 err = xdp_do_generic_redirect((*pskb)->dev, *pskb,
5221 &xdp, xdp_prog);
5222 if (err)
5223 goto out_redir;
5224 break;
5225 case XDP_TX:
5226 generic_xdp_tx(*pskb, xdp_prog);
5227 break;
5228 }
5229 bpf_net_ctx_clear(bpf_net_ctx);
5230 return XDP_DROP;
5231 }
5232 bpf_net_ctx_clear(bpf_net_ctx);
5233 }
5234 return XDP_PASS;
5235out_redir:
5236 bpf_net_ctx_clear(bpf_net_ctx);
5237 kfree_skb_reason(*pskb, SKB_DROP_REASON_XDP);
5238 return XDP_DROP;
5239}
5240EXPORT_SYMBOL_GPL(do_xdp_generic);
5241
5242static int netif_rx_internal(struct sk_buff *skb)
5243{
5244 int ret;
5245
5246 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5247
5248 trace_netif_rx(skb);
5249
5250#ifdef CONFIG_RPS
5251 if (static_branch_unlikely(&rps_needed)) {
5252 struct rps_dev_flow voidflow, *rflow = &voidflow;
5253 int cpu;
5254
5255 rcu_read_lock();
5256
5257 cpu = get_rps_cpu(skb->dev, skb, &rflow);
5258 if (cpu < 0)
5259 cpu = smp_processor_id();
5260
5261 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5262
5263 rcu_read_unlock();
5264 } else
5265#endif
5266 {
5267 unsigned int qtail;
5268
5269 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
5270 }
5271 return ret;
5272}
5273
5274/**
5275 * __netif_rx - Slightly optimized version of netif_rx
5276 * @skb: buffer to post
5277 *
5278 * This behaves as netif_rx except that it does not disable bottom halves.
5279 * As a result this function may only be invoked from the interrupt context
5280 * (either hard or soft interrupt).
5281 */
5282int __netif_rx(struct sk_buff *skb)
5283{
5284 int ret;
5285
5286 lockdep_assert_once(hardirq_count() | softirq_count());
5287
5288 trace_netif_rx_entry(skb);
5289 ret = netif_rx_internal(skb);
5290 trace_netif_rx_exit(ret);
5291 return ret;
5292}
5293EXPORT_SYMBOL(__netif_rx);
5294
5295/**
5296 * netif_rx - post buffer to the network code
5297 * @skb: buffer to post
5298 *
5299 * This function receives a packet from a device driver and queues it for
5300 * the upper (protocol) levels to process via the backlog NAPI device. It
5301 * always succeeds. The buffer may be dropped during processing for
5302 * congestion control or by the protocol layers.
5303 * The network buffer is passed via the backlog NAPI device. Modern NIC
5304 * driver should use NAPI and GRO.
5305 * This function can used from interrupt and from process context. The
5306 * caller from process context must not disable interrupts before invoking
5307 * this function.
5308 *
5309 * return values:
5310 * NET_RX_SUCCESS (no congestion)
5311 * NET_RX_DROP (packet was dropped)
5312 *
5313 */
5314int netif_rx(struct sk_buff *skb)
5315{
5316 bool need_bh_off = !(hardirq_count() | softirq_count());
5317 int ret;
5318
5319 if (need_bh_off)
5320 local_bh_disable();
5321 trace_netif_rx_entry(skb);
5322 ret = netif_rx_internal(skb);
5323 trace_netif_rx_exit(ret);
5324 if (need_bh_off)
5325 local_bh_enable();
5326 return ret;
5327}
5328EXPORT_SYMBOL(netif_rx);
5329
5330static __latent_entropy void net_tx_action(void)
5331{
5332 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5333
5334 if (sd->completion_queue) {
5335 struct sk_buff *clist;
5336
5337 local_irq_disable();
5338 clist = sd->completion_queue;
5339 sd->completion_queue = NULL;
5340 local_irq_enable();
5341
5342 while (clist) {
5343 struct sk_buff *skb = clist;
5344
5345 clist = clist->next;
5346
5347 WARN_ON(refcount_read(&skb->users));
5348 if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED))
5349 trace_consume_skb(skb, net_tx_action);
5350 else
5351 trace_kfree_skb(skb, net_tx_action,
5352 get_kfree_skb_cb(skb)->reason, NULL);
5353
5354 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5355 __kfree_skb(skb);
5356 else
5357 __napi_kfree_skb(skb,
5358 get_kfree_skb_cb(skb)->reason);
5359 }
5360 }
5361
5362 if (sd->output_queue) {
5363 struct Qdisc *head;
5364
5365 local_irq_disable();
5366 head = sd->output_queue;
5367 sd->output_queue = NULL;
5368 sd->output_queue_tailp = &sd->output_queue;
5369 local_irq_enable();
5370
5371 rcu_read_lock();
5372
5373 while (head) {
5374 struct Qdisc *q = head;
5375 spinlock_t *root_lock = NULL;
5376
5377 head = head->next_sched;
5378
5379 /* We need to make sure head->next_sched is read
5380 * before clearing __QDISC_STATE_SCHED
5381 */
5382 smp_mb__before_atomic();
5383
5384 if (!(q->flags & TCQ_F_NOLOCK)) {
5385 root_lock = qdisc_lock(q);
5386 spin_lock(root_lock);
5387 } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5388 &q->state))) {
5389 /* There is a synchronize_net() between
5390 * STATE_DEACTIVATED flag being set and
5391 * qdisc_reset()/some_qdisc_is_busy() in
5392 * dev_deactivate(), so we can safely bail out
5393 * early here to avoid data race between
5394 * qdisc_deactivate() and some_qdisc_is_busy()
5395 * for lockless qdisc.
5396 */
5397 clear_bit(__QDISC_STATE_SCHED, &q->state);
5398 continue;
5399 }
5400
5401 clear_bit(__QDISC_STATE_SCHED, &q->state);
5402 qdisc_run(q);
5403 if (root_lock)
5404 spin_unlock(root_lock);
5405 }
5406
5407 rcu_read_unlock();
5408 }
5409
5410 xfrm_dev_backlog(sd);
5411}
5412
5413#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
5414/* This hook is defined here for ATM LANE */
5415int (*br_fdb_test_addr_hook)(struct net_device *dev,
5416 unsigned char *addr) __read_mostly;
5417EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
5418#endif
5419
5420/**
5421 * netdev_is_rx_handler_busy - check if receive handler is registered
5422 * @dev: device to check
5423 *
5424 * Check if a receive handler is already registered for a given device.
5425 * Return true if there one.
5426 *
5427 * The caller must hold the rtnl_mutex.
5428 */
5429bool netdev_is_rx_handler_busy(struct net_device *dev)
5430{
5431 ASSERT_RTNL();
5432 return dev && rtnl_dereference(dev->rx_handler);
5433}
5434EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5435
5436/**
5437 * netdev_rx_handler_register - register receive handler
5438 * @dev: device to register a handler for
5439 * @rx_handler: receive handler to register
5440 * @rx_handler_data: data pointer that is used by rx handler
5441 *
5442 * Register a receive handler for a device. This handler will then be
5443 * called from __netif_receive_skb. A negative errno code is returned
5444 * on a failure.
5445 *
5446 * The caller must hold the rtnl_mutex.
5447 *
5448 * For a general description of rx_handler, see enum rx_handler_result.
5449 */
5450int netdev_rx_handler_register(struct net_device *dev,
5451 rx_handler_func_t *rx_handler,
5452 void *rx_handler_data)
5453{
5454 if (netdev_is_rx_handler_busy(dev))
5455 return -EBUSY;
5456
5457 if (dev->priv_flags & IFF_NO_RX_HANDLER)
5458 return -EINVAL;
5459
5460 /* Note: rx_handler_data must be set before rx_handler */
5461 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
5462 rcu_assign_pointer(dev->rx_handler, rx_handler);
5463
5464 return 0;
5465}
5466EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5467
5468/**
5469 * netdev_rx_handler_unregister - unregister receive handler
5470 * @dev: device to unregister a handler from
5471 *
5472 * Unregister a receive handler from a device.
5473 *
5474 * The caller must hold the rtnl_mutex.
5475 */
5476void netdev_rx_handler_unregister(struct net_device *dev)
5477{
5478
5479 ASSERT_RTNL();
5480 RCU_INIT_POINTER(dev->rx_handler, NULL);
5481 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5482 * section has a guarantee to see a non NULL rx_handler_data
5483 * as well.
5484 */
5485 synchronize_net();
5486 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
5487}
5488EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5489
5490/*
5491 * Limit the use of PFMEMALLOC reserves to those protocols that implement
5492 * the special handling of PFMEMALLOC skbs.
5493 */
5494static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5495{
5496 switch (skb->protocol) {
5497 case htons(ETH_P_ARP):
5498 case htons(ETH_P_IP):
5499 case htons(ETH_P_IPV6):
5500 case htons(ETH_P_8021Q):
5501 case htons(ETH_P_8021AD):
5502 return true;
5503 default:
5504 return false;
5505 }
5506}
5507
5508static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5509 int *ret, struct net_device *orig_dev)
5510{
5511 if (nf_hook_ingress_active(skb)) {
5512 int ingress_retval;
5513
5514 if (*pt_prev) {
5515 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5516 *pt_prev = NULL;
5517 }
5518
5519 rcu_read_lock();
5520 ingress_retval = nf_hook_ingress(skb);
5521 rcu_read_unlock();
5522 return ingress_retval;
5523 }
5524 return 0;
5525}
5526
5527static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
5528 struct packet_type **ppt_prev)
5529{
5530 struct packet_type *ptype, *pt_prev;
5531 rx_handler_func_t *rx_handler;
5532 struct sk_buff *skb = *pskb;
5533 struct net_device *orig_dev;
5534 bool deliver_exact = false;
5535 int ret = NET_RX_DROP;
5536 __be16 type;
5537
5538 net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5539
5540 trace_netif_receive_skb(skb);
5541
5542 orig_dev = skb->dev;
5543
5544 skb_reset_network_header(skb);
5545 if (!skb_transport_header_was_set(skb))
5546 skb_reset_transport_header(skb);
5547 skb_reset_mac_len(skb);
5548
5549 pt_prev = NULL;
5550
5551another_round:
5552 skb->skb_iif = skb->dev->ifindex;
5553
5554 __this_cpu_inc(softnet_data.processed);
5555
5556 if (static_branch_unlikely(&generic_xdp_needed_key)) {
5557 int ret2;
5558
5559 migrate_disable();
5560 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog),
5561 &skb);
5562 migrate_enable();
5563
5564 if (ret2 != XDP_PASS) {
5565 ret = NET_RX_DROP;
5566 goto out;
5567 }
5568 }
5569
5570 if (eth_type_vlan(skb->protocol)) {
5571 skb = skb_vlan_untag(skb);
5572 if (unlikely(!skb))
5573 goto out;
5574 }
5575
5576 if (skb_skip_tc_classify(skb))
5577 goto skip_classify;
5578
5579 if (pfmemalloc)
5580 goto skip_taps;
5581
5582 list_for_each_entry_rcu(ptype, &net_hotdata.ptype_all, list) {
5583 if (pt_prev)
5584 ret = deliver_skb(skb, pt_prev, orig_dev);
5585 pt_prev = ptype;
5586 }
5587
5588 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5589 if (pt_prev)
5590 ret = deliver_skb(skb, pt_prev, orig_dev);
5591 pt_prev = ptype;
5592 }
5593
5594skip_taps:
5595#ifdef CONFIG_NET_INGRESS
5596 if (static_branch_unlikely(&ingress_needed_key)) {
5597 bool another = false;
5598
5599 nf_skip_egress(skb, true);
5600 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5601 &another);
5602 if (another)
5603 goto another_round;
5604 if (!skb)
5605 goto out;
5606
5607 nf_skip_egress(skb, false);
5608 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
5609 goto out;
5610 }
5611#endif
5612 skb_reset_redirect(skb);
5613skip_classify:
5614 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
5615 goto drop;
5616
5617 if (skb_vlan_tag_present(skb)) {
5618 if (pt_prev) {
5619 ret = deliver_skb(skb, pt_prev, orig_dev);
5620 pt_prev = NULL;
5621 }
5622 if (vlan_do_receive(&skb))
5623 goto another_round;
5624 else if (unlikely(!skb))
5625 goto out;
5626 }
5627
5628 rx_handler = rcu_dereference(skb->dev->rx_handler);
5629 if (rx_handler) {
5630 if (pt_prev) {
5631 ret = deliver_skb(skb, pt_prev, orig_dev);
5632 pt_prev = NULL;
5633 }
5634 switch (rx_handler(&skb)) {
5635 case RX_HANDLER_CONSUMED:
5636 ret = NET_RX_SUCCESS;
5637 goto out;
5638 case RX_HANDLER_ANOTHER:
5639 goto another_round;
5640 case RX_HANDLER_EXACT:
5641 deliver_exact = true;
5642 break;
5643 case RX_HANDLER_PASS:
5644 break;
5645 default:
5646 BUG();
5647 }
5648 }
5649
5650 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
5651check_vlan_id:
5652 if (skb_vlan_tag_get_id(skb)) {
5653 /* Vlan id is non 0 and vlan_do_receive() above couldn't
5654 * find vlan device.
5655 */
5656 skb->pkt_type = PACKET_OTHERHOST;
5657 } else if (eth_type_vlan(skb->protocol)) {
5658 /* Outer header is 802.1P with vlan 0, inner header is
5659 * 802.1Q or 802.1AD and vlan_do_receive() above could
5660 * not find vlan dev for vlan id 0.
5661 */
5662 __vlan_hwaccel_clear_tag(skb);
5663 skb = skb_vlan_untag(skb);
5664 if (unlikely(!skb))
5665 goto out;
5666 if (vlan_do_receive(&skb))
5667 /* After stripping off 802.1P header with vlan 0
5668 * vlan dev is found for inner header.
5669 */
5670 goto another_round;
5671 else if (unlikely(!skb))
5672 goto out;
5673 else
5674 /* We have stripped outer 802.1P vlan 0 header.
5675 * But could not find vlan dev.
5676 * check again for vlan id to set OTHERHOST.
5677 */
5678 goto check_vlan_id;
5679 }
5680 /* Note: we might in the future use prio bits
5681 * and set skb->priority like in vlan_do_receive()
5682 * For the time being, just ignore Priority Code Point
5683 */
5684 __vlan_hwaccel_clear_tag(skb);
5685 }
5686
5687 type = skb->protocol;
5688
5689 /* deliver only exact match when indicated */
5690 if (likely(!deliver_exact)) {
5691 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5692 &ptype_base[ntohs(type) &
5693 PTYPE_HASH_MASK]);
5694 }
5695
5696 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5697 &orig_dev->ptype_specific);
5698
5699 if (unlikely(skb->dev != orig_dev)) {
5700 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5701 &skb->dev->ptype_specific);
5702 }
5703
5704 if (pt_prev) {
5705 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
5706 goto drop;
5707 *ppt_prev = pt_prev;
5708 } else {
5709drop:
5710 if (!deliver_exact)
5711 dev_core_stats_rx_dropped_inc(skb->dev);
5712 else
5713 dev_core_stats_rx_nohandler_inc(skb->dev);
5714 kfree_skb_reason(skb, SKB_DROP_REASON_UNHANDLED_PROTO);
5715 /* Jamal, now you will not able to escape explaining
5716 * me how you were going to use this. :-)
5717 */
5718 ret = NET_RX_DROP;
5719 }
5720
5721out:
5722 /* The invariant here is that if *ppt_prev is not NULL
5723 * then skb should also be non-NULL.
5724 *
5725 * Apparently *ppt_prev assignment above holds this invariant due to
5726 * skb dereferencing near it.
5727 */
5728 *pskb = skb;
5729 return ret;
5730}
5731
5732static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5733{
5734 struct net_device *orig_dev = skb->dev;
5735 struct packet_type *pt_prev = NULL;
5736 int ret;
5737
5738 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5739 if (pt_prev)
5740 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5741 skb->dev, pt_prev, orig_dev);
5742 return ret;
5743}
5744
5745/**
5746 * netif_receive_skb_core - special purpose version of netif_receive_skb
5747 * @skb: buffer to process
5748 *
5749 * More direct receive version of netif_receive_skb(). It should
5750 * only be used by callers that have a need to skip RPS and Generic XDP.
5751 * Caller must also take care of handling if ``(page_is_)pfmemalloc``.
5752 *
5753 * This function may only be called from softirq context and interrupts
5754 * should be enabled.
5755 *
5756 * Return values (usually ignored):
5757 * NET_RX_SUCCESS: no congestion
5758 * NET_RX_DROP: packet was dropped
5759 */
5760int netif_receive_skb_core(struct sk_buff *skb)
5761{
5762 int ret;
5763
5764 rcu_read_lock();
5765 ret = __netif_receive_skb_one_core(skb, false);
5766 rcu_read_unlock();
5767
5768 return ret;
5769}
5770EXPORT_SYMBOL(netif_receive_skb_core);
5771
5772static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5773 struct packet_type *pt_prev,
5774 struct net_device *orig_dev)
5775{
5776 struct sk_buff *skb, *next;
5777
5778 if (!pt_prev)
5779 return;
5780 if (list_empty(head))
5781 return;
5782 if (pt_prev->list_func != NULL)
5783 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5784 ip_list_rcv, head, pt_prev, orig_dev);
5785 else
5786 list_for_each_entry_safe(skb, next, head, list) {
5787 skb_list_del_init(skb);
5788 pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5789 }
5790}
5791
5792static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5793{
5794 /* Fast-path assumptions:
5795 * - There is no RX handler.
5796 * - Only one packet_type matches.
5797 * If either of these fails, we will end up doing some per-packet
5798 * processing in-line, then handling the 'last ptype' for the whole
5799 * sublist. This can't cause out-of-order delivery to any single ptype,
5800 * because the 'last ptype' must be constant across the sublist, and all
5801 * other ptypes are handled per-packet.
5802 */
5803 /* Current (common) ptype of sublist */
5804 struct packet_type *pt_curr = NULL;
5805 /* Current (common) orig_dev of sublist */
5806 struct net_device *od_curr = NULL;
5807 struct sk_buff *skb, *next;
5808 LIST_HEAD(sublist);
5809
5810 list_for_each_entry_safe(skb, next, head, list) {
5811 struct net_device *orig_dev = skb->dev;
5812 struct packet_type *pt_prev = NULL;
5813
5814 skb_list_del_init(skb);
5815 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5816 if (!pt_prev)
5817 continue;
5818 if (pt_curr != pt_prev || od_curr != orig_dev) {
5819 /* dispatch old sublist */
5820 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5821 /* start new sublist */
5822 INIT_LIST_HEAD(&sublist);
5823 pt_curr = pt_prev;
5824 od_curr = orig_dev;
5825 }
5826 list_add_tail(&skb->list, &sublist);
5827 }
5828
5829 /* dispatch final sublist */
5830 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5831}
5832
5833static int __netif_receive_skb(struct sk_buff *skb)
5834{
5835 int ret;
5836
5837 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5838 unsigned int noreclaim_flag;
5839
5840 /*
5841 * PFMEMALLOC skbs are special, they should
5842 * - be delivered to SOCK_MEMALLOC sockets only
5843 * - stay away from userspace
5844 * - have bounded memory usage
5845 *
5846 * Use PF_MEMALLOC as this saves us from propagating the allocation
5847 * context down to all allocation sites.
5848 */
5849 noreclaim_flag = memalloc_noreclaim_save();
5850 ret = __netif_receive_skb_one_core(skb, true);
5851 memalloc_noreclaim_restore(noreclaim_flag);
5852 } else
5853 ret = __netif_receive_skb_one_core(skb, false);
5854
5855 return ret;
5856}
5857
5858static void __netif_receive_skb_list(struct list_head *head)
5859{
5860 unsigned long noreclaim_flag = 0;
5861 struct sk_buff *skb, *next;
5862 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5863
5864 list_for_each_entry_safe(skb, next, head, list) {
5865 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5866 struct list_head sublist;
5867
5868 /* Handle the previous sublist */
5869 list_cut_before(&sublist, head, &skb->list);
5870 if (!list_empty(&sublist))
5871 __netif_receive_skb_list_core(&sublist, pfmemalloc);
5872 pfmemalloc = !pfmemalloc;
5873 /* See comments in __netif_receive_skb */
5874 if (pfmemalloc)
5875 noreclaim_flag = memalloc_noreclaim_save();
5876 else
5877 memalloc_noreclaim_restore(noreclaim_flag);
5878 }
5879 }
5880 /* Handle the remaining sublist */
5881 if (!list_empty(head))
5882 __netif_receive_skb_list_core(head, pfmemalloc);
5883 /* Restore pflags */
5884 if (pfmemalloc)
5885 memalloc_noreclaim_restore(noreclaim_flag);
5886}
5887
5888static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5889{
5890 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5891 struct bpf_prog *new = xdp->prog;
5892 int ret = 0;
5893
5894 switch (xdp->command) {
5895 case XDP_SETUP_PROG:
5896 rcu_assign_pointer(dev->xdp_prog, new);
5897 if (old)
5898 bpf_prog_put(old);
5899
5900 if (old && !new) {
5901 static_branch_dec(&generic_xdp_needed_key);
5902 } else if (new && !old) {
5903 static_branch_inc(&generic_xdp_needed_key);
5904 dev_disable_lro(dev);
5905 dev_disable_gro_hw(dev);
5906 }
5907 break;
5908
5909 default:
5910 ret = -EINVAL;
5911 break;
5912 }
5913
5914 return ret;
5915}
5916
5917static int netif_receive_skb_internal(struct sk_buff *skb)
5918{
5919 int ret;
5920
5921 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5922
5923 if (skb_defer_rx_timestamp(skb))
5924 return NET_RX_SUCCESS;
5925
5926 rcu_read_lock();
5927#ifdef CONFIG_RPS
5928 if (static_branch_unlikely(&rps_needed)) {
5929 struct rps_dev_flow voidflow, *rflow = &voidflow;
5930 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5931
5932 if (cpu >= 0) {
5933 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5934 rcu_read_unlock();
5935 return ret;
5936 }
5937 }
5938#endif
5939 ret = __netif_receive_skb(skb);
5940 rcu_read_unlock();
5941 return ret;
5942}
5943
5944void netif_receive_skb_list_internal(struct list_head *head)
5945{
5946 struct sk_buff *skb, *next;
5947 LIST_HEAD(sublist);
5948
5949 list_for_each_entry_safe(skb, next, head, list) {
5950 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue),
5951 skb);
5952 skb_list_del_init(skb);
5953 if (!skb_defer_rx_timestamp(skb))
5954 list_add_tail(&skb->list, &sublist);
5955 }
5956 list_splice_init(&sublist, head);
5957
5958 rcu_read_lock();
5959#ifdef CONFIG_RPS
5960 if (static_branch_unlikely(&rps_needed)) {
5961 list_for_each_entry_safe(skb, next, head, list) {
5962 struct rps_dev_flow voidflow, *rflow = &voidflow;
5963 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5964
5965 if (cpu >= 0) {
5966 /* Will be handled, remove from list */
5967 skb_list_del_init(skb);
5968 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5969 }
5970 }
5971 }
5972#endif
5973 __netif_receive_skb_list(head);
5974 rcu_read_unlock();
5975}
5976
5977/**
5978 * netif_receive_skb - process receive buffer from network
5979 * @skb: buffer to process
5980 *
5981 * netif_receive_skb() is the main receive data processing function.
5982 * It always succeeds. The buffer may be dropped during processing
5983 * for congestion control or by the protocol layers.
5984 *
5985 * This function may only be called from softirq context and interrupts
5986 * should be enabled.
5987 *
5988 * Return values (usually ignored):
5989 * NET_RX_SUCCESS: no congestion
5990 * NET_RX_DROP: packet was dropped
5991 */
5992int netif_receive_skb(struct sk_buff *skb)
5993{
5994 int ret;
5995
5996 trace_netif_receive_skb_entry(skb);
5997
5998 ret = netif_receive_skb_internal(skb);
5999 trace_netif_receive_skb_exit(ret);
6000
6001 return ret;
6002}
6003EXPORT_SYMBOL(netif_receive_skb);
6004
6005/**
6006 * netif_receive_skb_list - process many receive buffers from network
6007 * @head: list of skbs to process.
6008 *
6009 * Since return value of netif_receive_skb() is normally ignored, and
6010 * wouldn't be meaningful for a list, this function returns void.
6011 *
6012 * This function may only be called from softirq context and interrupts
6013 * should be enabled.
6014 */
6015void netif_receive_skb_list(struct list_head *head)
6016{
6017 struct sk_buff *skb;
6018
6019 if (list_empty(head))
6020 return;
6021 if (trace_netif_receive_skb_list_entry_enabled()) {
6022 list_for_each_entry(skb, head, list)
6023 trace_netif_receive_skb_list_entry(skb);
6024 }
6025 netif_receive_skb_list_internal(head);
6026 trace_netif_receive_skb_list_exit(0);
6027}
6028EXPORT_SYMBOL(netif_receive_skb_list);
6029
6030static DEFINE_PER_CPU(struct work_struct, flush_works);
6031
6032/* Network device is going away, flush any packets still pending */
6033static void flush_backlog(struct work_struct *work)
6034{
6035 struct sk_buff *skb, *tmp;
6036 struct softnet_data *sd;
6037
6038 local_bh_disable();
6039 sd = this_cpu_ptr(&softnet_data);
6040
6041 backlog_lock_irq_disable(sd);
6042 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
6043 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
6044 __skb_unlink(skb, &sd->input_pkt_queue);
6045 dev_kfree_skb_irq(skb);
6046 rps_input_queue_head_incr(sd);
6047 }
6048 }
6049 backlog_unlock_irq_enable(sd);
6050
6051 local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6052 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
6053 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
6054 __skb_unlink(skb, &sd->process_queue);
6055 kfree_skb(skb);
6056 rps_input_queue_head_incr(sd);
6057 }
6058 }
6059 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6060 local_bh_enable();
6061}
6062
6063static bool flush_required(int cpu)
6064{
6065#if IS_ENABLED(CONFIG_RPS)
6066 struct softnet_data *sd = &per_cpu(softnet_data, cpu);
6067 bool do_flush;
6068
6069 backlog_lock_irq_disable(sd);
6070
6071 /* as insertion into process_queue happens with the rps lock held,
6072 * process_queue access may race only with dequeue
6073 */
6074 do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
6075 !skb_queue_empty_lockless(&sd->process_queue);
6076 backlog_unlock_irq_enable(sd);
6077
6078 return do_flush;
6079#endif
6080 /* without RPS we can't safely check input_pkt_queue: during a
6081 * concurrent remote skb_queue_splice() we can detect as empty both
6082 * input_pkt_queue and process_queue even if the latter could end-up
6083 * containing a lot of packets.
6084 */
6085 return true;
6086}
6087
6088static void flush_all_backlogs(void)
6089{
6090 static cpumask_t flush_cpus;
6091 unsigned int cpu;
6092
6093 /* since we are under rtnl lock protection we can use static data
6094 * for the cpumask and avoid allocating on stack the possibly
6095 * large mask
6096 */
6097 ASSERT_RTNL();
6098
6099 cpus_read_lock();
6100
6101 cpumask_clear(&flush_cpus);
6102 for_each_online_cpu(cpu) {
6103 if (flush_required(cpu)) {
6104 queue_work_on(cpu, system_highpri_wq,
6105 per_cpu_ptr(&flush_works, cpu));
6106 cpumask_set_cpu(cpu, &flush_cpus);
6107 }
6108 }
6109
6110 /* we can have in flight packet[s] on the cpus we are not flushing,
6111 * synchronize_net() in unregister_netdevice_many() will take care of
6112 * them
6113 */
6114 for_each_cpu(cpu, &flush_cpus)
6115 flush_work(per_cpu_ptr(&flush_works, cpu));
6116
6117 cpus_read_unlock();
6118}
6119
6120static void net_rps_send_ipi(struct softnet_data *remsd)
6121{
6122#ifdef CONFIG_RPS
6123 while (remsd) {
6124 struct softnet_data *next = remsd->rps_ipi_next;
6125
6126 if (cpu_online(remsd->cpu))
6127 smp_call_function_single_async(remsd->cpu, &remsd->csd);
6128 remsd = next;
6129 }
6130#endif
6131}
6132
6133/*
6134 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
6135 * Note: called with local irq disabled, but exits with local irq enabled.
6136 */
6137static void net_rps_action_and_irq_enable(struct softnet_data *sd)
6138{
6139#ifdef CONFIG_RPS
6140 struct softnet_data *remsd = sd->rps_ipi_list;
6141
6142 if (!use_backlog_threads() && remsd) {
6143 sd->rps_ipi_list = NULL;
6144
6145 local_irq_enable();
6146
6147 /* Send pending IPI's to kick RPS processing on remote cpus. */
6148 net_rps_send_ipi(remsd);
6149 } else
6150#endif
6151 local_irq_enable();
6152}
6153
6154static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
6155{
6156#ifdef CONFIG_RPS
6157 return !use_backlog_threads() && sd->rps_ipi_list;
6158#else
6159 return false;
6160#endif
6161}
6162
6163static int process_backlog(struct napi_struct *napi, int quota)
6164{
6165 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
6166 bool again = true;
6167 int work = 0;
6168
6169 /* Check if we have pending ipi, its better to send them now,
6170 * not waiting net_rx_action() end.
6171 */
6172 if (sd_has_rps_ipi_waiting(sd)) {
6173 local_irq_disable();
6174 net_rps_action_and_irq_enable(sd);
6175 }
6176
6177 napi->weight = READ_ONCE(net_hotdata.dev_rx_weight);
6178 while (again) {
6179 struct sk_buff *skb;
6180
6181 local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6182 while ((skb = __skb_dequeue(&sd->process_queue))) {
6183 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6184 rcu_read_lock();
6185 __netif_receive_skb(skb);
6186 rcu_read_unlock();
6187 if (++work >= quota) {
6188 rps_input_queue_head_add(sd, work);
6189 return work;
6190 }
6191
6192 local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6193 }
6194 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6195
6196 backlog_lock_irq_disable(sd);
6197 if (skb_queue_empty(&sd->input_pkt_queue)) {
6198 /*
6199 * Inline a custom version of __napi_complete().
6200 * only current cpu owns and manipulates this napi,
6201 * and NAPI_STATE_SCHED is the only possible flag set
6202 * on backlog.
6203 * We can use a plain write instead of clear_bit(),
6204 * and we dont need an smp_mb() memory barrier.
6205 */
6206 napi->state &= NAPIF_STATE_THREADED;
6207 again = false;
6208 } else {
6209 local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6210 skb_queue_splice_tail_init(&sd->input_pkt_queue,
6211 &sd->process_queue);
6212 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6213 }
6214 backlog_unlock_irq_enable(sd);
6215 }
6216
6217 if (work)
6218 rps_input_queue_head_add(sd, work);
6219 return work;
6220}
6221
6222/**
6223 * __napi_schedule - schedule for receive
6224 * @n: entry to schedule
6225 *
6226 * The entry's receive function will be scheduled to run.
6227 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
6228 */
6229void __napi_schedule(struct napi_struct *n)
6230{
6231 unsigned long flags;
6232
6233 local_irq_save(flags);
6234 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6235 local_irq_restore(flags);
6236}
6237EXPORT_SYMBOL(__napi_schedule);
6238
6239/**
6240 * napi_schedule_prep - check if napi can be scheduled
6241 * @n: napi context
6242 *
6243 * Test if NAPI routine is already running, and if not mark
6244 * it as running. This is used as a condition variable to
6245 * insure only one NAPI poll instance runs. We also make
6246 * sure there is no pending NAPI disable.
6247 */
6248bool napi_schedule_prep(struct napi_struct *n)
6249{
6250 unsigned long new, val = READ_ONCE(n->state);
6251
6252 do {
6253 if (unlikely(val & NAPIF_STATE_DISABLE))
6254 return false;
6255 new = val | NAPIF_STATE_SCHED;
6256
6257 /* Sets STATE_MISSED bit if STATE_SCHED was already set
6258 * This was suggested by Alexander Duyck, as compiler
6259 * emits better code than :
6260 * if (val & NAPIF_STATE_SCHED)
6261 * new |= NAPIF_STATE_MISSED;
6262 */
6263 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6264 NAPIF_STATE_MISSED;
6265 } while (!try_cmpxchg(&n->state, &val, new));
6266
6267 return !(val & NAPIF_STATE_SCHED);
6268}
6269EXPORT_SYMBOL(napi_schedule_prep);
6270
6271/**
6272 * __napi_schedule_irqoff - schedule for receive
6273 * @n: entry to schedule
6274 *
6275 * Variant of __napi_schedule() assuming hard irqs are masked.
6276 *
6277 * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6278 * because the interrupt disabled assumption might not be true
6279 * due to force-threaded interrupts and spinlock substitution.
6280 */
6281void __napi_schedule_irqoff(struct napi_struct *n)
6282{
6283 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6284 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6285 else
6286 __napi_schedule(n);
6287}
6288EXPORT_SYMBOL(__napi_schedule_irqoff);
6289
6290bool napi_complete_done(struct napi_struct *n, int work_done)
6291{
6292 unsigned long flags, val, new, timeout = 0;
6293 bool ret = true;
6294
6295 /*
6296 * 1) Don't let napi dequeue from the cpu poll list
6297 * just in case its running on a different cpu.
6298 * 2) If we are busy polling, do nothing here, we have
6299 * the guarantee we will be called later.
6300 */
6301 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6302 NAPIF_STATE_IN_BUSY_POLL)))
6303 return false;
6304
6305 if (work_done) {
6306 if (n->gro_bitmask)
6307 timeout = napi_get_gro_flush_timeout(n);
6308 n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n);
6309 }
6310 if (n->defer_hard_irqs_count > 0) {
6311 n->defer_hard_irqs_count--;
6312 timeout = napi_get_gro_flush_timeout(n);
6313 if (timeout)
6314 ret = false;
6315 }
6316 if (n->gro_bitmask) {
6317 /* When the NAPI instance uses a timeout and keeps postponing
6318 * it, we need to bound somehow the time packets are kept in
6319 * the GRO layer
6320 */
6321 napi_gro_flush(n, !!timeout);
6322 }
6323
6324 gro_normal_list(n);
6325
6326 if (unlikely(!list_empty(&n->poll_list))) {
6327 /* If n->poll_list is not empty, we need to mask irqs */
6328 local_irq_save(flags);
6329 list_del_init(&n->poll_list);
6330 local_irq_restore(flags);
6331 }
6332 WRITE_ONCE(n->list_owner, -1);
6333
6334 val = READ_ONCE(n->state);
6335 do {
6336 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6337
6338 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
6339 NAPIF_STATE_SCHED_THREADED |
6340 NAPIF_STATE_PREFER_BUSY_POLL);
6341
6342 /* If STATE_MISSED was set, leave STATE_SCHED set,
6343 * because we will call napi->poll() one more time.
6344 * This C code was suggested by Alexander Duyck to help gcc.
6345 */
6346 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6347 NAPIF_STATE_SCHED;
6348 } while (!try_cmpxchg(&n->state, &val, new));
6349
6350 if (unlikely(val & NAPIF_STATE_MISSED)) {
6351 __napi_schedule(n);
6352 return false;
6353 }
6354
6355 if (timeout)
6356 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6357 HRTIMER_MODE_REL_PINNED);
6358 return ret;
6359}
6360EXPORT_SYMBOL(napi_complete_done);
6361
6362static void skb_defer_free_flush(struct softnet_data *sd)
6363{
6364 struct sk_buff *skb, *next;
6365
6366 /* Paired with WRITE_ONCE() in skb_attempt_defer_free() */
6367 if (!READ_ONCE(sd->defer_list))
6368 return;
6369
6370 spin_lock(&sd->defer_lock);
6371 skb = sd->defer_list;
6372 sd->defer_list = NULL;
6373 sd->defer_count = 0;
6374 spin_unlock(&sd->defer_lock);
6375
6376 while (skb != NULL) {
6377 next = skb->next;
6378 napi_consume_skb(skb, 1);
6379 skb = next;
6380 }
6381}
6382
6383#if defined(CONFIG_NET_RX_BUSY_POLL)
6384
6385static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
6386{
6387 if (!skip_schedule) {
6388 gro_normal_list(napi);
6389 __napi_schedule(napi);
6390 return;
6391 }
6392
6393 if (napi->gro_bitmask) {
6394 /* flush too old packets
6395 * If HZ < 1000, flush all packets.
6396 */
6397 napi_gro_flush(napi, HZ >= 1000);
6398 }
6399
6400 gro_normal_list(napi);
6401 clear_bit(NAPI_STATE_SCHED, &napi->state);
6402}
6403
6404enum {
6405 NAPI_F_PREFER_BUSY_POLL = 1,
6406 NAPI_F_END_ON_RESCHED = 2,
6407};
6408
6409static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
6410 unsigned flags, u16 budget)
6411{
6412 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
6413 bool skip_schedule = false;
6414 unsigned long timeout;
6415 int rc;
6416
6417 /* Busy polling means there is a high chance device driver hard irq
6418 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6419 * set in napi_schedule_prep().
6420 * Since we are about to call napi->poll() once more, we can safely
6421 * clear NAPI_STATE_MISSED.
6422 *
6423 * Note: x86 could use a single "lock and ..." instruction
6424 * to perform these two clear_bit()
6425 */
6426 clear_bit(NAPI_STATE_MISSED, &napi->state);
6427 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6428
6429 local_bh_disable();
6430 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
6431
6432 if (flags & NAPI_F_PREFER_BUSY_POLL) {
6433 napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi);
6434 timeout = napi_get_gro_flush_timeout(napi);
6435 if (napi->defer_hard_irqs_count && timeout) {
6436 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6437 skip_schedule = true;
6438 }
6439 }
6440
6441 /* All we really want here is to re-enable device interrupts.
6442 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6443 */
6444 rc = napi->poll(napi, budget);
6445 /* We can't gro_normal_list() here, because napi->poll() might have
6446 * rearmed the napi (napi_complete_done()) in which case it could
6447 * already be running on another CPU.
6448 */
6449 trace_napi_poll(napi, rc, budget);
6450 netpoll_poll_unlock(have_poll_lock);
6451 if (rc == budget)
6452 __busy_poll_stop(napi, skip_schedule);
6453 bpf_net_ctx_clear(bpf_net_ctx);
6454 local_bh_enable();
6455}
6456
6457static void __napi_busy_loop(unsigned int napi_id,
6458 bool (*loop_end)(void *, unsigned long),
6459 void *loop_end_arg, unsigned flags, u16 budget)
6460{
6461 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6462 int (*napi_poll)(struct napi_struct *napi, int budget);
6463 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
6464 void *have_poll_lock = NULL;
6465 struct napi_struct *napi;
6466
6467 WARN_ON_ONCE(!rcu_read_lock_held());
6468
6469restart:
6470 napi_poll = NULL;
6471
6472 napi = napi_by_id(napi_id);
6473 if (!napi)
6474 return;
6475
6476 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6477 preempt_disable();
6478 for (;;) {
6479 int work = 0;
6480
6481 local_bh_disable();
6482 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
6483 if (!napi_poll) {
6484 unsigned long val = READ_ONCE(napi->state);
6485
6486 /* If multiple threads are competing for this napi,
6487 * we avoid dirtying napi->state as much as we can.
6488 */
6489 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6490 NAPIF_STATE_IN_BUSY_POLL)) {
6491 if (flags & NAPI_F_PREFER_BUSY_POLL)
6492 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6493 goto count;
6494 }
6495 if (cmpxchg(&napi->state, val,
6496 val | NAPIF_STATE_IN_BUSY_POLL |
6497 NAPIF_STATE_SCHED) != val) {
6498 if (flags & NAPI_F_PREFER_BUSY_POLL)
6499 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6500 goto count;
6501 }
6502 have_poll_lock = netpoll_poll_lock(napi);
6503 napi_poll = napi->poll;
6504 }
6505 work = napi_poll(napi, budget);
6506 trace_napi_poll(napi, work, budget);
6507 gro_normal_list(napi);
6508count:
6509 if (work > 0)
6510 __NET_ADD_STATS(dev_net(napi->dev),
6511 LINUX_MIB_BUSYPOLLRXPACKETS, work);
6512 skb_defer_free_flush(this_cpu_ptr(&softnet_data));
6513 bpf_net_ctx_clear(bpf_net_ctx);
6514 local_bh_enable();
6515
6516 if (!loop_end || loop_end(loop_end_arg, start_time))
6517 break;
6518
6519 if (unlikely(need_resched())) {
6520 if (flags & NAPI_F_END_ON_RESCHED)
6521 break;
6522 if (napi_poll)
6523 busy_poll_stop(napi, have_poll_lock, flags, budget);
6524 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6525 preempt_enable();
6526 rcu_read_unlock();
6527 cond_resched();
6528 rcu_read_lock();
6529 if (loop_end(loop_end_arg, start_time))
6530 return;
6531 goto restart;
6532 }
6533 cpu_relax();
6534 }
6535 if (napi_poll)
6536 busy_poll_stop(napi, have_poll_lock, flags, budget);
6537 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6538 preempt_enable();
6539}
6540
6541void napi_busy_loop_rcu(unsigned int napi_id,
6542 bool (*loop_end)(void *, unsigned long),
6543 void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6544{
6545 unsigned flags = NAPI_F_END_ON_RESCHED;
6546
6547 if (prefer_busy_poll)
6548 flags |= NAPI_F_PREFER_BUSY_POLL;
6549
6550 __napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
6551}
6552
6553void napi_busy_loop(unsigned int napi_id,
6554 bool (*loop_end)(void *, unsigned long),
6555 void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6556{
6557 unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0;
6558
6559 rcu_read_lock();
6560 __napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
6561 rcu_read_unlock();
6562}
6563EXPORT_SYMBOL(napi_busy_loop);
6564
6565void napi_suspend_irqs(unsigned int napi_id)
6566{
6567 struct napi_struct *napi;
6568
6569 rcu_read_lock();
6570 napi = napi_by_id(napi_id);
6571 if (napi) {
6572 unsigned long timeout = napi_get_irq_suspend_timeout(napi);
6573
6574 if (timeout)
6575 hrtimer_start(&napi->timer, ns_to_ktime(timeout),
6576 HRTIMER_MODE_REL_PINNED);
6577 }
6578 rcu_read_unlock();
6579}
6580
6581void napi_resume_irqs(unsigned int napi_id)
6582{
6583 struct napi_struct *napi;
6584
6585 rcu_read_lock();
6586 napi = napi_by_id(napi_id);
6587 if (napi) {
6588 /* If irq_suspend_timeout is set to 0 between the call to
6589 * napi_suspend_irqs and now, the original value still
6590 * determines the safety timeout as intended and napi_watchdog
6591 * will resume irq processing.
6592 */
6593 if (napi_get_irq_suspend_timeout(napi)) {
6594 local_bh_disable();
6595 napi_schedule(napi);
6596 local_bh_enable();
6597 }
6598 }
6599 rcu_read_unlock();
6600}
6601
6602#endif /* CONFIG_NET_RX_BUSY_POLL */
6603
6604static void __napi_hash_add_with_id(struct napi_struct *napi,
6605 unsigned int napi_id)
6606{
6607 WRITE_ONCE(napi->napi_id, napi_id);
6608 hlist_add_head_rcu(&napi->napi_hash_node,
6609 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
6610}
6611
6612static void napi_hash_add_with_id(struct napi_struct *napi,
6613 unsigned int napi_id)
6614{
6615 unsigned long flags;
6616
6617 spin_lock_irqsave(&napi_hash_lock, flags);
6618 WARN_ON_ONCE(napi_by_id(napi_id));
6619 __napi_hash_add_with_id(napi, napi_id);
6620 spin_unlock_irqrestore(&napi_hash_lock, flags);
6621}
6622
6623static void napi_hash_add(struct napi_struct *napi)
6624{
6625 unsigned long flags;
6626
6627 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
6628 return;
6629
6630 spin_lock_irqsave(&napi_hash_lock, flags);
6631
6632 /* 0..NR_CPUS range is reserved for sender_cpu use */
6633 do {
6634 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6635 napi_gen_id = MIN_NAPI_ID;
6636 } while (napi_by_id(napi_gen_id));
6637
6638 __napi_hash_add_with_id(napi, napi_gen_id);
6639
6640 spin_unlock_irqrestore(&napi_hash_lock, flags);
6641}
6642
6643/* Warning : caller is responsible to make sure rcu grace period
6644 * is respected before freeing memory containing @napi
6645 */
6646static void napi_hash_del(struct napi_struct *napi)
6647{
6648 unsigned long flags;
6649
6650 spin_lock_irqsave(&napi_hash_lock, flags);
6651
6652 hlist_del_init_rcu(&napi->napi_hash_node);
6653
6654 spin_unlock_irqrestore(&napi_hash_lock, flags);
6655}
6656
6657static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6658{
6659 struct napi_struct *napi;
6660
6661 napi = container_of(timer, struct napi_struct, timer);
6662
6663 /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6664 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6665 */
6666 if (!napi_disable_pending(napi) &&
6667 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6668 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6669 __napi_schedule_irqoff(napi);
6670 }
6671
6672 return HRTIMER_NORESTART;
6673}
6674
6675static void init_gro_hash(struct napi_struct *napi)
6676{
6677 int i;
6678
6679 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6680 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6681 napi->gro_hash[i].count = 0;
6682 }
6683 napi->gro_bitmask = 0;
6684}
6685
6686int dev_set_threaded(struct net_device *dev, bool threaded)
6687{
6688 struct napi_struct *napi;
6689 int err = 0;
6690
6691 if (dev->threaded == threaded)
6692 return 0;
6693
6694 if (threaded) {
6695 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6696 if (!napi->thread) {
6697 err = napi_kthread_create(napi);
6698 if (err) {
6699 threaded = false;
6700 break;
6701 }
6702 }
6703 }
6704 }
6705
6706 WRITE_ONCE(dev->threaded, threaded);
6707
6708 /* Make sure kthread is created before THREADED bit
6709 * is set.
6710 */
6711 smp_mb__before_atomic();
6712
6713 /* Setting/unsetting threaded mode on a napi might not immediately
6714 * take effect, if the current napi instance is actively being
6715 * polled. In this case, the switch between threaded mode and
6716 * softirq mode will happen in the next round of napi_schedule().
6717 * This should not cause hiccups/stalls to the live traffic.
6718 */
6719 list_for_each_entry(napi, &dev->napi_list, dev_list)
6720 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded);
6721
6722 return err;
6723}
6724EXPORT_SYMBOL(dev_set_threaded);
6725
6726/**
6727 * netif_queue_set_napi - Associate queue with the napi
6728 * @dev: device to which NAPI and queue belong
6729 * @queue_index: Index of queue
6730 * @type: queue type as RX or TX
6731 * @napi: NAPI context, pass NULL to clear previously set NAPI
6732 *
6733 * Set queue with its corresponding napi context. This should be done after
6734 * registering the NAPI handler for the queue-vector and the queues have been
6735 * mapped to the corresponding interrupt vector.
6736 */
6737void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
6738 enum netdev_queue_type type, struct napi_struct *napi)
6739{
6740 struct netdev_rx_queue *rxq;
6741 struct netdev_queue *txq;
6742
6743 if (WARN_ON_ONCE(napi && !napi->dev))
6744 return;
6745 if (dev->reg_state >= NETREG_REGISTERED)
6746 ASSERT_RTNL();
6747
6748 switch (type) {
6749 case NETDEV_QUEUE_TYPE_RX:
6750 rxq = __netif_get_rx_queue(dev, queue_index);
6751 rxq->napi = napi;
6752 return;
6753 case NETDEV_QUEUE_TYPE_TX:
6754 txq = netdev_get_tx_queue(dev, queue_index);
6755 txq->napi = napi;
6756 return;
6757 default:
6758 return;
6759 }
6760}
6761EXPORT_SYMBOL(netif_queue_set_napi);
6762
6763static void napi_restore_config(struct napi_struct *n)
6764{
6765 n->defer_hard_irqs = n->config->defer_hard_irqs;
6766 n->gro_flush_timeout = n->config->gro_flush_timeout;
6767 n->irq_suspend_timeout = n->config->irq_suspend_timeout;
6768 /* a NAPI ID might be stored in the config, if so use it. if not, use
6769 * napi_hash_add to generate one for us. It will be saved to the config
6770 * in napi_disable.
6771 */
6772 if (n->config->napi_id)
6773 napi_hash_add_with_id(n, n->config->napi_id);
6774 else
6775 napi_hash_add(n);
6776}
6777
6778static void napi_save_config(struct napi_struct *n)
6779{
6780 n->config->defer_hard_irqs = n->defer_hard_irqs;
6781 n->config->gro_flush_timeout = n->gro_flush_timeout;
6782 n->config->irq_suspend_timeout = n->irq_suspend_timeout;
6783 n->config->napi_id = n->napi_id;
6784 napi_hash_del(n);
6785}
6786
6787void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
6788 int (*poll)(struct napi_struct *, int), int weight)
6789{
6790 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6791 return;
6792
6793 INIT_LIST_HEAD(&napi->poll_list);
6794 INIT_HLIST_NODE(&napi->napi_hash_node);
6795 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6796 napi->timer.function = napi_watchdog;
6797 init_gro_hash(napi);
6798 napi->skb = NULL;
6799 INIT_LIST_HEAD(&napi->rx_list);
6800 napi->rx_count = 0;
6801 napi->poll = poll;
6802 if (weight > NAPI_POLL_WEIGHT)
6803 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6804 weight);
6805 napi->weight = weight;
6806 napi->dev = dev;
6807#ifdef CONFIG_NETPOLL
6808 napi->poll_owner = -1;
6809#endif
6810 napi->list_owner = -1;
6811 set_bit(NAPI_STATE_SCHED, &napi->state);
6812 set_bit(NAPI_STATE_NPSVC, &napi->state);
6813 list_add_rcu(&napi->dev_list, &dev->napi_list);
6814
6815 /* default settings from sysfs are applied to all NAPIs. any per-NAPI
6816 * configuration will be loaded in napi_enable
6817 */
6818 napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs));
6819 napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout));
6820
6821 napi_get_frags_check(napi);
6822 /* Create kthread for this napi if dev->threaded is set.
6823 * Clear dev->threaded if kthread creation failed so that
6824 * threaded mode will not be enabled in napi_enable().
6825 */
6826 if (dev->threaded && napi_kthread_create(napi))
6827 dev->threaded = false;
6828 netif_napi_set_irq(napi, -1);
6829}
6830EXPORT_SYMBOL(netif_napi_add_weight);
6831
6832void napi_disable(struct napi_struct *n)
6833{
6834 unsigned long val, new;
6835
6836 might_sleep();
6837 set_bit(NAPI_STATE_DISABLE, &n->state);
6838
6839 val = READ_ONCE(n->state);
6840 do {
6841 while (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) {
6842 usleep_range(20, 200);
6843 val = READ_ONCE(n->state);
6844 }
6845
6846 new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC;
6847 new &= ~(NAPIF_STATE_THREADED | NAPIF_STATE_PREFER_BUSY_POLL);
6848 } while (!try_cmpxchg(&n->state, &val, new));
6849
6850 hrtimer_cancel(&n->timer);
6851
6852 if (n->config)
6853 napi_save_config(n);
6854 else
6855 napi_hash_del(n);
6856
6857 clear_bit(NAPI_STATE_DISABLE, &n->state);
6858}
6859EXPORT_SYMBOL(napi_disable);
6860
6861/**
6862 * napi_enable - enable NAPI scheduling
6863 * @n: NAPI context
6864 *
6865 * Resume NAPI from being scheduled on this context.
6866 * Must be paired with napi_disable.
6867 */
6868void napi_enable(struct napi_struct *n)
6869{
6870 unsigned long new, val = READ_ONCE(n->state);
6871
6872 if (n->config)
6873 napi_restore_config(n);
6874 else
6875 napi_hash_add(n);
6876
6877 do {
6878 BUG_ON(!test_bit(NAPI_STATE_SCHED, &val));
6879
6880 new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC);
6881 if (n->dev->threaded && n->thread)
6882 new |= NAPIF_STATE_THREADED;
6883 } while (!try_cmpxchg(&n->state, &val, new));
6884}
6885EXPORT_SYMBOL(napi_enable);
6886
6887static void flush_gro_hash(struct napi_struct *napi)
6888{
6889 int i;
6890
6891 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6892 struct sk_buff *skb, *n;
6893
6894 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
6895 kfree_skb(skb);
6896 napi->gro_hash[i].count = 0;
6897 }
6898}
6899
6900/* Must be called in process context */
6901void __netif_napi_del(struct napi_struct *napi)
6902{
6903 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
6904 return;
6905
6906 if (napi->config) {
6907 napi->index = -1;
6908 napi->config = NULL;
6909 }
6910
6911 list_del_rcu(&napi->dev_list);
6912 napi_free_frags(napi);
6913
6914 flush_gro_hash(napi);
6915 napi->gro_bitmask = 0;
6916
6917 if (napi->thread) {
6918 kthread_stop(napi->thread);
6919 napi->thread = NULL;
6920 }
6921}
6922EXPORT_SYMBOL(__netif_napi_del);
6923
6924static int __napi_poll(struct napi_struct *n, bool *repoll)
6925{
6926 int work, weight;
6927
6928 weight = n->weight;
6929
6930 /* This NAPI_STATE_SCHED test is for avoiding a race
6931 * with netpoll's poll_napi(). Only the entity which
6932 * obtains the lock and sees NAPI_STATE_SCHED set will
6933 * actually make the ->poll() call. Therefore we avoid
6934 * accidentally calling ->poll() when NAPI is not scheduled.
6935 */
6936 work = 0;
6937 if (napi_is_scheduled(n)) {
6938 work = n->poll(n, weight);
6939 trace_napi_poll(n, work, weight);
6940
6941 xdp_do_check_flushed(n);
6942 }
6943
6944 if (unlikely(work > weight))
6945 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
6946 n->poll, work, weight);
6947
6948 if (likely(work < weight))
6949 return work;
6950
6951 /* Drivers must not modify the NAPI state if they
6952 * consume the entire weight. In such cases this code
6953 * still "owns" the NAPI instance and therefore can
6954 * move the instance around on the list at-will.
6955 */
6956 if (unlikely(napi_disable_pending(n))) {
6957 napi_complete(n);
6958 return work;
6959 }
6960
6961 /* The NAPI context has more processing work, but busy-polling
6962 * is preferred. Exit early.
6963 */
6964 if (napi_prefer_busy_poll(n)) {
6965 if (napi_complete_done(n, work)) {
6966 /* If timeout is not set, we need to make sure
6967 * that the NAPI is re-scheduled.
6968 */
6969 napi_schedule(n);
6970 }
6971 return work;
6972 }
6973
6974 if (n->gro_bitmask) {
6975 /* flush too old packets
6976 * If HZ < 1000, flush all packets.
6977 */
6978 napi_gro_flush(n, HZ >= 1000);
6979 }
6980
6981 gro_normal_list(n);
6982
6983 /* Some drivers may have called napi_schedule
6984 * prior to exhausting their budget.
6985 */
6986 if (unlikely(!list_empty(&n->poll_list))) {
6987 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6988 n->dev ? n->dev->name : "backlog");
6989 return work;
6990 }
6991
6992 *repoll = true;
6993
6994 return work;
6995}
6996
6997static int napi_poll(struct napi_struct *n, struct list_head *repoll)
6998{
6999 bool do_repoll = false;
7000 void *have;
7001 int work;
7002
7003 list_del_init(&n->poll_list);
7004
7005 have = netpoll_poll_lock(n);
7006
7007 work = __napi_poll(n, &do_repoll);
7008
7009 if (do_repoll)
7010 list_add_tail(&n->poll_list, repoll);
7011
7012 netpoll_poll_unlock(have);
7013
7014 return work;
7015}
7016
7017static int napi_thread_wait(struct napi_struct *napi)
7018{
7019 set_current_state(TASK_INTERRUPTIBLE);
7020
7021 while (!kthread_should_stop()) {
7022 /* Testing SCHED_THREADED bit here to make sure the current
7023 * kthread owns this napi and could poll on this napi.
7024 * Testing SCHED bit is not enough because SCHED bit might be
7025 * set by some other busy poll thread or by napi_disable().
7026 */
7027 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) {
7028 WARN_ON(!list_empty(&napi->poll_list));
7029 __set_current_state(TASK_RUNNING);
7030 return 0;
7031 }
7032
7033 schedule();
7034 set_current_state(TASK_INTERRUPTIBLE);
7035 }
7036 __set_current_state(TASK_RUNNING);
7037
7038 return -1;
7039}
7040
7041static void napi_threaded_poll_loop(struct napi_struct *napi)
7042{
7043 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
7044 struct softnet_data *sd;
7045 unsigned long last_qs = jiffies;
7046
7047 for (;;) {
7048 bool repoll = false;
7049 void *have;
7050
7051 local_bh_disable();
7052 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
7053
7054 sd = this_cpu_ptr(&softnet_data);
7055 sd->in_napi_threaded_poll = true;
7056
7057 have = netpoll_poll_lock(napi);
7058 __napi_poll(napi, &repoll);
7059 netpoll_poll_unlock(have);
7060
7061 sd->in_napi_threaded_poll = false;
7062 barrier();
7063
7064 if (sd_has_rps_ipi_waiting(sd)) {
7065 local_irq_disable();
7066 net_rps_action_and_irq_enable(sd);
7067 }
7068 skb_defer_free_flush(sd);
7069 bpf_net_ctx_clear(bpf_net_ctx);
7070 local_bh_enable();
7071
7072 if (!repoll)
7073 break;
7074
7075 rcu_softirq_qs_periodic(last_qs);
7076 cond_resched();
7077 }
7078}
7079
7080static int napi_threaded_poll(void *data)
7081{
7082 struct napi_struct *napi = data;
7083
7084 while (!napi_thread_wait(napi))
7085 napi_threaded_poll_loop(napi);
7086
7087 return 0;
7088}
7089
7090static __latent_entropy void net_rx_action(void)
7091{
7092 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
7093 unsigned long time_limit = jiffies +
7094 usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs));
7095 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
7096 int budget = READ_ONCE(net_hotdata.netdev_budget);
7097 LIST_HEAD(list);
7098 LIST_HEAD(repoll);
7099
7100 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
7101start:
7102 sd->in_net_rx_action = true;
7103 local_irq_disable();
7104 list_splice_init(&sd->poll_list, &list);
7105 local_irq_enable();
7106
7107 for (;;) {
7108 struct napi_struct *n;
7109
7110 skb_defer_free_flush(sd);
7111
7112 if (list_empty(&list)) {
7113 if (list_empty(&repoll)) {
7114 sd->in_net_rx_action = false;
7115 barrier();
7116 /* We need to check if ____napi_schedule()
7117 * had refilled poll_list while
7118 * sd->in_net_rx_action was true.
7119 */
7120 if (!list_empty(&sd->poll_list))
7121 goto start;
7122 if (!sd_has_rps_ipi_waiting(sd))
7123 goto end;
7124 }
7125 break;
7126 }
7127
7128 n = list_first_entry(&list, struct napi_struct, poll_list);
7129 budget -= napi_poll(n, &repoll);
7130
7131 /* If softirq window is exhausted then punt.
7132 * Allow this to run for 2 jiffies since which will allow
7133 * an average latency of 1.5/HZ.
7134 */
7135 if (unlikely(budget <= 0 ||
7136 time_after_eq(jiffies, time_limit))) {
7137 sd->time_squeeze++;
7138 break;
7139 }
7140 }
7141
7142 local_irq_disable();
7143
7144 list_splice_tail_init(&sd->poll_list, &list);
7145 list_splice_tail(&repoll, &list);
7146 list_splice(&list, &sd->poll_list);
7147 if (!list_empty(&sd->poll_list))
7148 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
7149 else
7150 sd->in_net_rx_action = false;
7151
7152 net_rps_action_and_irq_enable(sd);
7153end:
7154 bpf_net_ctx_clear(bpf_net_ctx);
7155}
7156
7157struct netdev_adjacent {
7158 struct net_device *dev;
7159 netdevice_tracker dev_tracker;
7160
7161 /* upper master flag, there can only be one master device per list */
7162 bool master;
7163
7164 /* lookup ignore flag */
7165 bool ignore;
7166
7167 /* counter for the number of times this device was added to us */
7168 u16 ref_nr;
7169
7170 /* private field for the users */
7171 void *private;
7172
7173 struct list_head list;
7174 struct rcu_head rcu;
7175};
7176
7177static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
7178 struct list_head *adj_list)
7179{
7180 struct netdev_adjacent *adj;
7181
7182 list_for_each_entry(adj, adj_list, list) {
7183 if (adj->dev == adj_dev)
7184 return adj;
7185 }
7186 return NULL;
7187}
7188
7189static int ____netdev_has_upper_dev(struct net_device *upper_dev,
7190 struct netdev_nested_priv *priv)
7191{
7192 struct net_device *dev = (struct net_device *)priv->data;
7193
7194 return upper_dev == dev;
7195}
7196
7197/**
7198 * netdev_has_upper_dev - Check if device is linked to an upper device
7199 * @dev: device
7200 * @upper_dev: upper device to check
7201 *
7202 * Find out if a device is linked to specified upper device and return true
7203 * in case it is. Note that this checks only immediate upper device,
7204 * not through a complete stack of devices. The caller must hold the RTNL lock.
7205 */
7206bool netdev_has_upper_dev(struct net_device *dev,
7207 struct net_device *upper_dev)
7208{
7209 struct netdev_nested_priv priv = {
7210 .data = (void *)upper_dev,
7211 };
7212
7213 ASSERT_RTNL();
7214
7215 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
7216 &priv);
7217}
7218EXPORT_SYMBOL(netdev_has_upper_dev);
7219
7220/**
7221 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
7222 * @dev: device
7223 * @upper_dev: upper device to check
7224 *
7225 * Find out if a device is linked to specified upper device and return true
7226 * in case it is. Note that this checks the entire upper device chain.
7227 * The caller must hold rcu lock.
7228 */
7229
7230bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
7231 struct net_device *upper_dev)
7232{
7233 struct netdev_nested_priv priv = {
7234 .data = (void *)upper_dev,
7235 };
7236
7237 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
7238 &priv);
7239}
7240EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
7241
7242/**
7243 * netdev_has_any_upper_dev - Check if device is linked to some device
7244 * @dev: device
7245 *
7246 * Find out if a device is linked to an upper device and return true in case
7247 * it is. The caller must hold the RTNL lock.
7248 */
7249bool netdev_has_any_upper_dev(struct net_device *dev)
7250{
7251 ASSERT_RTNL();
7252
7253 return !list_empty(&dev->adj_list.upper);
7254}
7255EXPORT_SYMBOL(netdev_has_any_upper_dev);
7256
7257/**
7258 * netdev_master_upper_dev_get - Get master upper device
7259 * @dev: device
7260 *
7261 * Find a master upper device and return pointer to it or NULL in case
7262 * it's not there. The caller must hold the RTNL lock.
7263 */
7264struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
7265{
7266 struct netdev_adjacent *upper;
7267
7268 ASSERT_RTNL();
7269
7270 if (list_empty(&dev->adj_list.upper))
7271 return NULL;
7272
7273 upper = list_first_entry(&dev->adj_list.upper,
7274 struct netdev_adjacent, list);
7275 if (likely(upper->master))
7276 return upper->dev;
7277 return NULL;
7278}
7279EXPORT_SYMBOL(netdev_master_upper_dev_get);
7280
7281static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
7282{
7283 struct netdev_adjacent *upper;
7284
7285 ASSERT_RTNL();
7286
7287 if (list_empty(&dev->adj_list.upper))
7288 return NULL;
7289
7290 upper = list_first_entry(&dev->adj_list.upper,
7291 struct netdev_adjacent, list);
7292 if (likely(upper->master) && !upper->ignore)
7293 return upper->dev;
7294 return NULL;
7295}
7296
7297/**
7298 * netdev_has_any_lower_dev - Check if device is linked to some device
7299 * @dev: device
7300 *
7301 * Find out if a device is linked to a lower device and return true in case
7302 * it is. The caller must hold the RTNL lock.
7303 */
7304static bool netdev_has_any_lower_dev(struct net_device *dev)
7305{
7306 ASSERT_RTNL();
7307
7308 return !list_empty(&dev->adj_list.lower);
7309}
7310
7311void *netdev_adjacent_get_private(struct list_head *adj_list)
7312{
7313 struct netdev_adjacent *adj;
7314
7315 adj = list_entry(adj_list, struct netdev_adjacent, list);
7316
7317 return adj->private;
7318}
7319EXPORT_SYMBOL(netdev_adjacent_get_private);
7320
7321/**
7322 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
7323 * @dev: device
7324 * @iter: list_head ** of the current position
7325 *
7326 * Gets the next device from the dev's upper list, starting from iter
7327 * position. The caller must hold RCU read lock.
7328 */
7329struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
7330 struct list_head **iter)
7331{
7332 struct netdev_adjacent *upper;
7333
7334 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7335
7336 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7337
7338 if (&upper->list == &dev->adj_list.upper)
7339 return NULL;
7340
7341 *iter = &upper->list;
7342
7343 return upper->dev;
7344}
7345EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
7346
7347static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
7348 struct list_head **iter,
7349 bool *ignore)
7350{
7351 struct netdev_adjacent *upper;
7352
7353 upper = list_entry((*iter)->next, struct netdev_adjacent, list);
7354
7355 if (&upper->list == &dev->adj_list.upper)
7356 return NULL;
7357
7358 *iter = &upper->list;
7359 *ignore = upper->ignore;
7360
7361 return upper->dev;
7362}
7363
7364static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
7365 struct list_head **iter)
7366{
7367 struct netdev_adjacent *upper;
7368
7369 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7370
7371 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7372
7373 if (&upper->list == &dev->adj_list.upper)
7374 return NULL;
7375
7376 *iter = &upper->list;
7377
7378 return upper->dev;
7379}
7380
7381static int __netdev_walk_all_upper_dev(struct net_device *dev,
7382 int (*fn)(struct net_device *dev,
7383 struct netdev_nested_priv *priv),
7384 struct netdev_nested_priv *priv)
7385{
7386 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7387 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7388 int ret, cur = 0;
7389 bool ignore;
7390
7391 now = dev;
7392 iter = &dev->adj_list.upper;
7393
7394 while (1) {
7395 if (now != dev) {
7396 ret = fn(now, priv);
7397 if (ret)
7398 return ret;
7399 }
7400
7401 next = NULL;
7402 while (1) {
7403 udev = __netdev_next_upper_dev(now, &iter, &ignore);
7404 if (!udev)
7405 break;
7406 if (ignore)
7407 continue;
7408
7409 next = udev;
7410 niter = &udev->adj_list.upper;
7411 dev_stack[cur] = now;
7412 iter_stack[cur++] = iter;
7413 break;
7414 }
7415
7416 if (!next) {
7417 if (!cur)
7418 return 0;
7419 next = dev_stack[--cur];
7420 niter = iter_stack[cur];
7421 }
7422
7423 now = next;
7424 iter = niter;
7425 }
7426
7427 return 0;
7428}
7429
7430int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
7431 int (*fn)(struct net_device *dev,
7432 struct netdev_nested_priv *priv),
7433 struct netdev_nested_priv *priv)
7434{
7435 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7436 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7437 int ret, cur = 0;
7438
7439 now = dev;
7440 iter = &dev->adj_list.upper;
7441
7442 while (1) {
7443 if (now != dev) {
7444 ret = fn(now, priv);
7445 if (ret)
7446 return ret;
7447 }
7448
7449 next = NULL;
7450 while (1) {
7451 udev = netdev_next_upper_dev_rcu(now, &iter);
7452 if (!udev)
7453 break;
7454
7455 next = udev;
7456 niter = &udev->adj_list.upper;
7457 dev_stack[cur] = now;
7458 iter_stack[cur++] = iter;
7459 break;
7460 }
7461
7462 if (!next) {
7463 if (!cur)
7464 return 0;
7465 next = dev_stack[--cur];
7466 niter = iter_stack[cur];
7467 }
7468
7469 now = next;
7470 iter = niter;
7471 }
7472
7473 return 0;
7474}
7475EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
7476
7477static bool __netdev_has_upper_dev(struct net_device *dev,
7478 struct net_device *upper_dev)
7479{
7480 struct netdev_nested_priv priv = {
7481 .flags = 0,
7482 .data = (void *)upper_dev,
7483 };
7484
7485 ASSERT_RTNL();
7486
7487 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
7488 &priv);
7489}
7490
7491/**
7492 * netdev_lower_get_next_private - Get the next ->private from the
7493 * lower neighbour list
7494 * @dev: device
7495 * @iter: list_head ** of the current position
7496 *
7497 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7498 * list, starting from iter position. The caller must hold either hold the
7499 * RTNL lock or its own locking that guarantees that the neighbour lower
7500 * list will remain unchanged.
7501 */
7502void *netdev_lower_get_next_private(struct net_device *dev,
7503 struct list_head **iter)
7504{
7505 struct netdev_adjacent *lower;
7506
7507 lower = list_entry(*iter, struct netdev_adjacent, list);
7508
7509 if (&lower->list == &dev->adj_list.lower)
7510 return NULL;
7511
7512 *iter = lower->list.next;
7513
7514 return lower->private;
7515}
7516EXPORT_SYMBOL(netdev_lower_get_next_private);
7517
7518/**
7519 * netdev_lower_get_next_private_rcu - Get the next ->private from the
7520 * lower neighbour list, RCU
7521 * variant
7522 * @dev: device
7523 * @iter: list_head ** of the current position
7524 *
7525 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7526 * list, starting from iter position. The caller must hold RCU read lock.
7527 */
7528void *netdev_lower_get_next_private_rcu(struct net_device *dev,
7529 struct list_head **iter)
7530{
7531 struct netdev_adjacent *lower;
7532
7533 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
7534
7535 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7536
7537 if (&lower->list == &dev->adj_list.lower)
7538 return NULL;
7539
7540 *iter = &lower->list;
7541
7542 return lower->private;
7543}
7544EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
7545
7546/**
7547 * netdev_lower_get_next - Get the next device from the lower neighbour
7548 * list
7549 * @dev: device
7550 * @iter: list_head ** of the current position
7551 *
7552 * Gets the next netdev_adjacent from the dev's lower neighbour
7553 * list, starting from iter position. The caller must hold RTNL lock or
7554 * its own locking that guarantees that the neighbour lower
7555 * list will remain unchanged.
7556 */
7557void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
7558{
7559 struct netdev_adjacent *lower;
7560
7561 lower = list_entry(*iter, struct netdev_adjacent, list);
7562
7563 if (&lower->list == &dev->adj_list.lower)
7564 return NULL;
7565
7566 *iter = lower->list.next;
7567
7568 return lower->dev;
7569}
7570EXPORT_SYMBOL(netdev_lower_get_next);
7571
7572static struct net_device *netdev_next_lower_dev(struct net_device *dev,
7573 struct list_head **iter)
7574{
7575 struct netdev_adjacent *lower;
7576
7577 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7578
7579 if (&lower->list == &dev->adj_list.lower)
7580 return NULL;
7581
7582 *iter = &lower->list;
7583
7584 return lower->dev;
7585}
7586
7587static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
7588 struct list_head **iter,
7589 bool *ignore)
7590{
7591 struct netdev_adjacent *lower;
7592
7593 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7594
7595 if (&lower->list == &dev->adj_list.lower)
7596 return NULL;
7597
7598 *iter = &lower->list;
7599 *ignore = lower->ignore;
7600
7601 return lower->dev;
7602}
7603
7604int netdev_walk_all_lower_dev(struct net_device *dev,
7605 int (*fn)(struct net_device *dev,
7606 struct netdev_nested_priv *priv),
7607 struct netdev_nested_priv *priv)
7608{
7609 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7610 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7611 int ret, cur = 0;
7612
7613 now = dev;
7614 iter = &dev->adj_list.lower;
7615
7616 while (1) {
7617 if (now != dev) {
7618 ret = fn(now, priv);
7619 if (ret)
7620 return ret;
7621 }
7622
7623 next = NULL;
7624 while (1) {
7625 ldev = netdev_next_lower_dev(now, &iter);
7626 if (!ldev)
7627 break;
7628
7629 next = ldev;
7630 niter = &ldev->adj_list.lower;
7631 dev_stack[cur] = now;
7632 iter_stack[cur++] = iter;
7633 break;
7634 }
7635
7636 if (!next) {
7637 if (!cur)
7638 return 0;
7639 next = dev_stack[--cur];
7640 niter = iter_stack[cur];
7641 }
7642
7643 now = next;
7644 iter = niter;
7645 }
7646
7647 return 0;
7648}
7649EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
7650
7651static int __netdev_walk_all_lower_dev(struct net_device *dev,
7652 int (*fn)(struct net_device *dev,
7653 struct netdev_nested_priv *priv),
7654 struct netdev_nested_priv *priv)
7655{
7656 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7657 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7658 int ret, cur = 0;
7659 bool ignore;
7660
7661 now = dev;
7662 iter = &dev->adj_list.lower;
7663
7664 while (1) {
7665 if (now != dev) {
7666 ret = fn(now, priv);
7667 if (ret)
7668 return ret;
7669 }
7670
7671 next = NULL;
7672 while (1) {
7673 ldev = __netdev_next_lower_dev(now, &iter, &ignore);
7674 if (!ldev)
7675 break;
7676 if (ignore)
7677 continue;
7678
7679 next = ldev;
7680 niter = &ldev->adj_list.lower;
7681 dev_stack[cur] = now;
7682 iter_stack[cur++] = iter;
7683 break;
7684 }
7685
7686 if (!next) {
7687 if (!cur)
7688 return 0;
7689 next = dev_stack[--cur];
7690 niter = iter_stack[cur];
7691 }
7692
7693 now = next;
7694 iter = niter;
7695 }
7696
7697 return 0;
7698}
7699
7700struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
7701 struct list_head **iter)
7702{
7703 struct netdev_adjacent *lower;
7704
7705 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7706 if (&lower->list == &dev->adj_list.lower)
7707 return NULL;
7708
7709 *iter = &lower->list;
7710
7711 return lower->dev;
7712}
7713EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
7714
7715static u8 __netdev_upper_depth(struct net_device *dev)
7716{
7717 struct net_device *udev;
7718 struct list_head *iter;
7719 u8 max_depth = 0;
7720 bool ignore;
7721
7722 for (iter = &dev->adj_list.upper,
7723 udev = __netdev_next_upper_dev(dev, &iter, &ignore);
7724 udev;
7725 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
7726 if (ignore)
7727 continue;
7728 if (max_depth < udev->upper_level)
7729 max_depth = udev->upper_level;
7730 }
7731
7732 return max_depth;
7733}
7734
7735static u8 __netdev_lower_depth(struct net_device *dev)
7736{
7737 struct net_device *ldev;
7738 struct list_head *iter;
7739 u8 max_depth = 0;
7740 bool ignore;
7741
7742 for (iter = &dev->adj_list.lower,
7743 ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
7744 ldev;
7745 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
7746 if (ignore)
7747 continue;
7748 if (max_depth < ldev->lower_level)
7749 max_depth = ldev->lower_level;
7750 }
7751
7752 return max_depth;
7753}
7754
7755static int __netdev_update_upper_level(struct net_device *dev,
7756 struct netdev_nested_priv *__unused)
7757{
7758 dev->upper_level = __netdev_upper_depth(dev) + 1;
7759 return 0;
7760}
7761
7762#ifdef CONFIG_LOCKDEP
7763static LIST_HEAD(net_unlink_list);
7764
7765static void net_unlink_todo(struct net_device *dev)
7766{
7767 if (list_empty(&dev->unlink_list))
7768 list_add_tail(&dev->unlink_list, &net_unlink_list);
7769}
7770#endif
7771
7772static int __netdev_update_lower_level(struct net_device *dev,
7773 struct netdev_nested_priv *priv)
7774{
7775 dev->lower_level = __netdev_lower_depth(dev) + 1;
7776
7777#ifdef CONFIG_LOCKDEP
7778 if (!priv)
7779 return 0;
7780
7781 if (priv->flags & NESTED_SYNC_IMM)
7782 dev->nested_level = dev->lower_level - 1;
7783 if (priv->flags & NESTED_SYNC_TODO)
7784 net_unlink_todo(dev);
7785#endif
7786 return 0;
7787}
7788
7789int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
7790 int (*fn)(struct net_device *dev,
7791 struct netdev_nested_priv *priv),
7792 struct netdev_nested_priv *priv)
7793{
7794 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7795 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7796 int ret, cur = 0;
7797
7798 now = dev;
7799 iter = &dev->adj_list.lower;
7800
7801 while (1) {
7802 if (now != dev) {
7803 ret = fn(now, priv);
7804 if (ret)
7805 return ret;
7806 }
7807
7808 next = NULL;
7809 while (1) {
7810 ldev = netdev_next_lower_dev_rcu(now, &iter);
7811 if (!ldev)
7812 break;
7813
7814 next = ldev;
7815 niter = &ldev->adj_list.lower;
7816 dev_stack[cur] = now;
7817 iter_stack[cur++] = iter;
7818 break;
7819 }
7820
7821 if (!next) {
7822 if (!cur)
7823 return 0;
7824 next = dev_stack[--cur];
7825 niter = iter_stack[cur];
7826 }
7827
7828 now = next;
7829 iter = niter;
7830 }
7831
7832 return 0;
7833}
7834EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
7835
7836/**
7837 * netdev_lower_get_first_private_rcu - Get the first ->private from the
7838 * lower neighbour list, RCU
7839 * variant
7840 * @dev: device
7841 *
7842 * Gets the first netdev_adjacent->private from the dev's lower neighbour
7843 * list. The caller must hold RCU read lock.
7844 */
7845void *netdev_lower_get_first_private_rcu(struct net_device *dev)
7846{
7847 struct netdev_adjacent *lower;
7848
7849 lower = list_first_or_null_rcu(&dev->adj_list.lower,
7850 struct netdev_adjacent, list);
7851 if (lower)
7852 return lower->private;
7853 return NULL;
7854}
7855EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
7856
7857/**
7858 * netdev_master_upper_dev_get_rcu - Get master upper device
7859 * @dev: device
7860 *
7861 * Find a master upper device and return pointer to it or NULL in case
7862 * it's not there. The caller must hold the RCU read lock.
7863 */
7864struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
7865{
7866 struct netdev_adjacent *upper;
7867
7868 upper = list_first_or_null_rcu(&dev->adj_list.upper,
7869 struct netdev_adjacent, list);
7870 if (upper && likely(upper->master))
7871 return upper->dev;
7872 return NULL;
7873}
7874EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
7875
7876static int netdev_adjacent_sysfs_add(struct net_device *dev,
7877 struct net_device *adj_dev,
7878 struct list_head *dev_list)
7879{
7880 char linkname[IFNAMSIZ+7];
7881
7882 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7883 "upper_%s" : "lower_%s", adj_dev->name);
7884 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
7885 linkname);
7886}
7887static void netdev_adjacent_sysfs_del(struct net_device *dev,
7888 char *name,
7889 struct list_head *dev_list)
7890{
7891 char linkname[IFNAMSIZ+7];
7892
7893 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7894 "upper_%s" : "lower_%s", name);
7895 sysfs_remove_link(&(dev->dev.kobj), linkname);
7896}
7897
7898static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
7899 struct net_device *adj_dev,
7900 struct list_head *dev_list)
7901{
7902 return (dev_list == &dev->adj_list.upper ||
7903 dev_list == &dev->adj_list.lower) &&
7904 net_eq(dev_net(dev), dev_net(adj_dev));
7905}
7906
7907static int __netdev_adjacent_dev_insert(struct net_device *dev,
7908 struct net_device *adj_dev,
7909 struct list_head *dev_list,
7910 void *private, bool master)
7911{
7912 struct netdev_adjacent *adj;
7913 int ret;
7914
7915 adj = __netdev_find_adj(adj_dev, dev_list);
7916
7917 if (adj) {
7918 adj->ref_nr += 1;
7919 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
7920 dev->name, adj_dev->name, adj->ref_nr);
7921
7922 return 0;
7923 }
7924
7925 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
7926 if (!adj)
7927 return -ENOMEM;
7928
7929 adj->dev = adj_dev;
7930 adj->master = master;
7931 adj->ref_nr = 1;
7932 adj->private = private;
7933 adj->ignore = false;
7934 netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL);
7935
7936 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
7937 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
7938
7939 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
7940 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
7941 if (ret)
7942 goto free_adj;
7943 }
7944
7945 /* Ensure that master link is always the first item in list. */
7946 if (master) {
7947 ret = sysfs_create_link(&(dev->dev.kobj),
7948 &(adj_dev->dev.kobj), "master");
7949 if (ret)
7950 goto remove_symlinks;
7951
7952 list_add_rcu(&adj->list, dev_list);
7953 } else {
7954 list_add_tail_rcu(&adj->list, dev_list);
7955 }
7956
7957 return 0;
7958
7959remove_symlinks:
7960 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7961 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7962free_adj:
7963 netdev_put(adj_dev, &adj->dev_tracker);
7964 kfree(adj);
7965
7966 return ret;
7967}
7968
7969static void __netdev_adjacent_dev_remove(struct net_device *dev,
7970 struct net_device *adj_dev,
7971 u16 ref_nr,
7972 struct list_head *dev_list)
7973{
7974 struct netdev_adjacent *adj;
7975
7976 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
7977 dev->name, adj_dev->name, ref_nr);
7978
7979 adj = __netdev_find_adj(adj_dev, dev_list);
7980
7981 if (!adj) {
7982 pr_err("Adjacency does not exist for device %s from %s\n",
7983 dev->name, adj_dev->name);
7984 WARN_ON(1);
7985 return;
7986 }
7987
7988 if (adj->ref_nr > ref_nr) {
7989 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
7990 dev->name, adj_dev->name, ref_nr,
7991 adj->ref_nr - ref_nr);
7992 adj->ref_nr -= ref_nr;
7993 return;
7994 }
7995
7996 if (adj->master)
7997 sysfs_remove_link(&(dev->dev.kobj), "master");
7998
7999 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
8000 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
8001
8002 list_del_rcu(&adj->list);
8003 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
8004 adj_dev->name, dev->name, adj_dev->name);
8005 netdev_put(adj_dev, &adj->dev_tracker);
8006 kfree_rcu(adj, rcu);
8007}
8008
8009static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
8010 struct net_device *upper_dev,
8011 struct list_head *up_list,
8012 struct list_head *down_list,
8013 void *private, bool master)
8014{
8015 int ret;
8016
8017 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
8018 private, master);
8019 if (ret)
8020 return ret;
8021
8022 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
8023 private, false);
8024 if (ret) {
8025 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
8026 return ret;
8027 }
8028
8029 return 0;
8030}
8031
8032static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
8033 struct net_device *upper_dev,
8034 u16 ref_nr,
8035 struct list_head *up_list,
8036 struct list_head *down_list)
8037{
8038 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
8039 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
8040}
8041
8042static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
8043 struct net_device *upper_dev,
8044 void *private, bool master)
8045{
8046 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
8047 &dev->adj_list.upper,
8048 &upper_dev->adj_list.lower,
8049 private, master);
8050}
8051
8052static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
8053 struct net_device *upper_dev)
8054{
8055 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
8056 &dev->adj_list.upper,
8057 &upper_dev->adj_list.lower);
8058}
8059
8060static int __netdev_upper_dev_link(struct net_device *dev,
8061 struct net_device *upper_dev, bool master,
8062 void *upper_priv, void *upper_info,
8063 struct netdev_nested_priv *priv,
8064 struct netlink_ext_ack *extack)
8065{
8066 struct netdev_notifier_changeupper_info changeupper_info = {
8067 .info = {
8068 .dev = dev,
8069 .extack = extack,
8070 },
8071 .upper_dev = upper_dev,
8072 .master = master,
8073 .linking = true,
8074 .upper_info = upper_info,
8075 };
8076 struct net_device *master_dev;
8077 int ret = 0;
8078
8079 ASSERT_RTNL();
8080
8081 if (dev == upper_dev)
8082 return -EBUSY;
8083
8084 /* To prevent loops, check if dev is not upper device to upper_dev. */
8085 if (__netdev_has_upper_dev(upper_dev, dev))
8086 return -EBUSY;
8087
8088 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
8089 return -EMLINK;
8090
8091 if (!master) {
8092 if (__netdev_has_upper_dev(dev, upper_dev))
8093 return -EEXIST;
8094 } else {
8095 master_dev = __netdev_master_upper_dev_get(dev);
8096 if (master_dev)
8097 return master_dev == upper_dev ? -EEXIST : -EBUSY;
8098 }
8099
8100 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
8101 &changeupper_info.info);
8102 ret = notifier_to_errno(ret);
8103 if (ret)
8104 return ret;
8105
8106 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
8107 master);
8108 if (ret)
8109 return ret;
8110
8111 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
8112 &changeupper_info.info);
8113 ret = notifier_to_errno(ret);
8114 if (ret)
8115 goto rollback;
8116
8117 __netdev_update_upper_level(dev, NULL);
8118 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
8119
8120 __netdev_update_lower_level(upper_dev, priv);
8121 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
8122 priv);
8123
8124 return 0;
8125
8126rollback:
8127 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
8128
8129 return ret;
8130}
8131
8132/**
8133 * netdev_upper_dev_link - Add a link to the upper device
8134 * @dev: device
8135 * @upper_dev: new upper device
8136 * @extack: netlink extended ack
8137 *
8138 * Adds a link to device which is upper to this one. The caller must hold
8139 * the RTNL lock. On a failure a negative errno code is returned.
8140 * On success the reference counts are adjusted and the function
8141 * returns zero.
8142 */
8143int netdev_upper_dev_link(struct net_device *dev,
8144 struct net_device *upper_dev,
8145 struct netlink_ext_ack *extack)
8146{
8147 struct netdev_nested_priv priv = {
8148 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8149 .data = NULL,
8150 };
8151
8152 return __netdev_upper_dev_link(dev, upper_dev, false,
8153 NULL, NULL, &priv, extack);
8154}
8155EXPORT_SYMBOL(netdev_upper_dev_link);
8156
8157/**
8158 * netdev_master_upper_dev_link - Add a master link to the upper device
8159 * @dev: device
8160 * @upper_dev: new upper device
8161 * @upper_priv: upper device private
8162 * @upper_info: upper info to be passed down via notifier
8163 * @extack: netlink extended ack
8164 *
8165 * Adds a link to device which is upper to this one. In this case, only
8166 * one master upper device can be linked, although other non-master devices
8167 * might be linked as well. The caller must hold the RTNL lock.
8168 * On a failure a negative errno code is returned. On success the reference
8169 * counts are adjusted and the function returns zero.
8170 */
8171int netdev_master_upper_dev_link(struct net_device *dev,
8172 struct net_device *upper_dev,
8173 void *upper_priv, void *upper_info,
8174 struct netlink_ext_ack *extack)
8175{
8176 struct netdev_nested_priv priv = {
8177 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8178 .data = NULL,
8179 };
8180
8181 return __netdev_upper_dev_link(dev, upper_dev, true,
8182 upper_priv, upper_info, &priv, extack);
8183}
8184EXPORT_SYMBOL(netdev_master_upper_dev_link);
8185
8186static void __netdev_upper_dev_unlink(struct net_device *dev,
8187 struct net_device *upper_dev,
8188 struct netdev_nested_priv *priv)
8189{
8190 struct netdev_notifier_changeupper_info changeupper_info = {
8191 .info = {
8192 .dev = dev,
8193 },
8194 .upper_dev = upper_dev,
8195 .linking = false,
8196 };
8197
8198 ASSERT_RTNL();
8199
8200 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
8201
8202 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
8203 &changeupper_info.info);
8204
8205 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
8206
8207 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
8208 &changeupper_info.info);
8209
8210 __netdev_update_upper_level(dev, NULL);
8211 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
8212
8213 __netdev_update_lower_level(upper_dev, priv);
8214 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
8215 priv);
8216}
8217
8218/**
8219 * netdev_upper_dev_unlink - Removes a link to upper device
8220 * @dev: device
8221 * @upper_dev: new upper device
8222 *
8223 * Removes a link to device which is upper to this one. The caller must hold
8224 * the RTNL lock.
8225 */
8226void netdev_upper_dev_unlink(struct net_device *dev,
8227 struct net_device *upper_dev)
8228{
8229 struct netdev_nested_priv priv = {
8230 .flags = NESTED_SYNC_TODO,
8231 .data = NULL,
8232 };
8233
8234 __netdev_upper_dev_unlink(dev, upper_dev, &priv);
8235}
8236EXPORT_SYMBOL(netdev_upper_dev_unlink);
8237
8238static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
8239 struct net_device *lower_dev,
8240 bool val)
8241{
8242 struct netdev_adjacent *adj;
8243
8244 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
8245 if (adj)
8246 adj->ignore = val;
8247
8248 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
8249 if (adj)
8250 adj->ignore = val;
8251}
8252
8253static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
8254 struct net_device *lower_dev)
8255{
8256 __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
8257}
8258
8259static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
8260 struct net_device *lower_dev)
8261{
8262 __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
8263}
8264
8265int netdev_adjacent_change_prepare(struct net_device *old_dev,
8266 struct net_device *new_dev,
8267 struct net_device *dev,
8268 struct netlink_ext_ack *extack)
8269{
8270 struct netdev_nested_priv priv = {
8271 .flags = 0,
8272 .data = NULL,
8273 };
8274 int err;
8275
8276 if (!new_dev)
8277 return 0;
8278
8279 if (old_dev && new_dev != old_dev)
8280 netdev_adjacent_dev_disable(dev, old_dev);
8281 err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
8282 extack);
8283 if (err) {
8284 if (old_dev && new_dev != old_dev)
8285 netdev_adjacent_dev_enable(dev, old_dev);
8286 return err;
8287 }
8288
8289 return 0;
8290}
8291EXPORT_SYMBOL(netdev_adjacent_change_prepare);
8292
8293void netdev_adjacent_change_commit(struct net_device *old_dev,
8294 struct net_device *new_dev,
8295 struct net_device *dev)
8296{
8297 struct netdev_nested_priv priv = {
8298 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8299 .data = NULL,
8300 };
8301
8302 if (!new_dev || !old_dev)
8303 return;
8304
8305 if (new_dev == old_dev)
8306 return;
8307
8308 netdev_adjacent_dev_enable(dev, old_dev);
8309 __netdev_upper_dev_unlink(old_dev, dev, &priv);
8310}
8311EXPORT_SYMBOL(netdev_adjacent_change_commit);
8312
8313void netdev_adjacent_change_abort(struct net_device *old_dev,
8314 struct net_device *new_dev,
8315 struct net_device *dev)
8316{
8317 struct netdev_nested_priv priv = {
8318 .flags = 0,
8319 .data = NULL,
8320 };
8321
8322 if (!new_dev)
8323 return;
8324
8325 if (old_dev && new_dev != old_dev)
8326 netdev_adjacent_dev_enable(dev, old_dev);
8327
8328 __netdev_upper_dev_unlink(new_dev, dev, &priv);
8329}
8330EXPORT_SYMBOL(netdev_adjacent_change_abort);
8331
8332/**
8333 * netdev_bonding_info_change - Dispatch event about slave change
8334 * @dev: device
8335 * @bonding_info: info to dispatch
8336 *
8337 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
8338 * The caller must hold the RTNL lock.
8339 */
8340void netdev_bonding_info_change(struct net_device *dev,
8341 struct netdev_bonding_info *bonding_info)
8342{
8343 struct netdev_notifier_bonding_info info = {
8344 .info.dev = dev,
8345 };
8346
8347 memcpy(&info.bonding_info, bonding_info,
8348 sizeof(struct netdev_bonding_info));
8349 call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
8350 &info.info);
8351}
8352EXPORT_SYMBOL(netdev_bonding_info_change);
8353
8354static int netdev_offload_xstats_enable_l3(struct net_device *dev,
8355 struct netlink_ext_ack *extack)
8356{
8357 struct netdev_notifier_offload_xstats_info info = {
8358 .info.dev = dev,
8359 .info.extack = extack,
8360 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8361 };
8362 int err;
8363 int rc;
8364
8365 dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3),
8366 GFP_KERNEL);
8367 if (!dev->offload_xstats_l3)
8368 return -ENOMEM;
8369
8370 rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE,
8371 NETDEV_OFFLOAD_XSTATS_DISABLE,
8372 &info.info);
8373 err = notifier_to_errno(rc);
8374 if (err)
8375 goto free_stats;
8376
8377 return 0;
8378
8379free_stats:
8380 kfree(dev->offload_xstats_l3);
8381 dev->offload_xstats_l3 = NULL;
8382 return err;
8383}
8384
8385int netdev_offload_xstats_enable(struct net_device *dev,
8386 enum netdev_offload_xstats_type type,
8387 struct netlink_ext_ack *extack)
8388{
8389 ASSERT_RTNL();
8390
8391 if (netdev_offload_xstats_enabled(dev, type))
8392 return -EALREADY;
8393
8394 switch (type) {
8395 case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8396 return netdev_offload_xstats_enable_l3(dev, extack);
8397 }
8398
8399 WARN_ON(1);
8400 return -EINVAL;
8401}
8402EXPORT_SYMBOL(netdev_offload_xstats_enable);
8403
8404static void netdev_offload_xstats_disable_l3(struct net_device *dev)
8405{
8406 struct netdev_notifier_offload_xstats_info info = {
8407 .info.dev = dev,
8408 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8409 };
8410
8411 call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE,
8412 &info.info);
8413 kfree(dev->offload_xstats_l3);
8414 dev->offload_xstats_l3 = NULL;
8415}
8416
8417int netdev_offload_xstats_disable(struct net_device *dev,
8418 enum netdev_offload_xstats_type type)
8419{
8420 ASSERT_RTNL();
8421
8422 if (!netdev_offload_xstats_enabled(dev, type))
8423 return -EALREADY;
8424
8425 switch (type) {
8426 case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8427 netdev_offload_xstats_disable_l3(dev);
8428 return 0;
8429 }
8430
8431 WARN_ON(1);
8432 return -EINVAL;
8433}
8434EXPORT_SYMBOL(netdev_offload_xstats_disable);
8435
8436static void netdev_offload_xstats_disable_all(struct net_device *dev)
8437{
8438 netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
8439}
8440
8441static struct rtnl_hw_stats64 *
8442netdev_offload_xstats_get_ptr(const struct net_device *dev,
8443 enum netdev_offload_xstats_type type)
8444{
8445 switch (type) {
8446 case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8447 return dev->offload_xstats_l3;
8448 }
8449
8450 WARN_ON(1);
8451 return NULL;
8452}
8453
8454bool netdev_offload_xstats_enabled(const struct net_device *dev,
8455 enum netdev_offload_xstats_type type)
8456{
8457 ASSERT_RTNL();
8458
8459 return netdev_offload_xstats_get_ptr(dev, type);
8460}
8461EXPORT_SYMBOL(netdev_offload_xstats_enabled);
8462
8463struct netdev_notifier_offload_xstats_ru {
8464 bool used;
8465};
8466
8467struct netdev_notifier_offload_xstats_rd {
8468 struct rtnl_hw_stats64 stats;
8469 bool used;
8470};
8471
8472static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest,
8473 const struct rtnl_hw_stats64 *src)
8474{
8475 dest->rx_packets += src->rx_packets;
8476 dest->tx_packets += src->tx_packets;
8477 dest->rx_bytes += src->rx_bytes;
8478 dest->tx_bytes += src->tx_bytes;
8479 dest->rx_errors += src->rx_errors;
8480 dest->tx_errors += src->tx_errors;
8481 dest->rx_dropped += src->rx_dropped;
8482 dest->tx_dropped += src->tx_dropped;
8483 dest->multicast += src->multicast;
8484}
8485
8486static int netdev_offload_xstats_get_used(struct net_device *dev,
8487 enum netdev_offload_xstats_type type,
8488 bool *p_used,
8489 struct netlink_ext_ack *extack)
8490{
8491 struct netdev_notifier_offload_xstats_ru report_used = {};
8492 struct netdev_notifier_offload_xstats_info info = {
8493 .info.dev = dev,
8494 .info.extack = extack,
8495 .type = type,
8496 .report_used = &report_used,
8497 };
8498 int rc;
8499
8500 WARN_ON(!netdev_offload_xstats_enabled(dev, type));
8501 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED,
8502 &info.info);
8503 *p_used = report_used.used;
8504 return notifier_to_errno(rc);
8505}
8506
8507static int netdev_offload_xstats_get_stats(struct net_device *dev,
8508 enum netdev_offload_xstats_type type,
8509 struct rtnl_hw_stats64 *p_stats,
8510 bool *p_used,
8511 struct netlink_ext_ack *extack)
8512{
8513 struct netdev_notifier_offload_xstats_rd report_delta = {};
8514 struct netdev_notifier_offload_xstats_info info = {
8515 .info.dev = dev,
8516 .info.extack = extack,
8517 .type = type,
8518 .report_delta = &report_delta,
8519 };
8520 struct rtnl_hw_stats64 *stats;
8521 int rc;
8522
8523 stats = netdev_offload_xstats_get_ptr(dev, type);
8524 if (WARN_ON(!stats))
8525 return -EINVAL;
8526
8527 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
8528 &info.info);
8529
8530 /* Cache whatever we got, even if there was an error, otherwise the
8531 * successful stats retrievals would get lost.
8532 */
8533 netdev_hw_stats64_add(stats, &report_delta.stats);
8534
8535 if (p_stats)
8536 *p_stats = *stats;
8537 *p_used = report_delta.used;
8538
8539 return notifier_to_errno(rc);
8540}
8541
8542int netdev_offload_xstats_get(struct net_device *dev,
8543 enum netdev_offload_xstats_type type,
8544 struct rtnl_hw_stats64 *p_stats, bool *p_used,
8545 struct netlink_ext_ack *extack)
8546{
8547 ASSERT_RTNL();
8548
8549 if (p_stats)
8550 return netdev_offload_xstats_get_stats(dev, type, p_stats,
8551 p_used, extack);
8552 else
8553 return netdev_offload_xstats_get_used(dev, type, p_used,
8554 extack);
8555}
8556EXPORT_SYMBOL(netdev_offload_xstats_get);
8557
8558void
8559netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta,
8560 const struct rtnl_hw_stats64 *stats)
8561{
8562 report_delta->used = true;
8563 netdev_hw_stats64_add(&report_delta->stats, stats);
8564}
8565EXPORT_SYMBOL(netdev_offload_xstats_report_delta);
8566
8567void
8568netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used)
8569{
8570 report_used->used = true;
8571}
8572EXPORT_SYMBOL(netdev_offload_xstats_report_used);
8573
8574void netdev_offload_xstats_push_delta(struct net_device *dev,
8575 enum netdev_offload_xstats_type type,
8576 const struct rtnl_hw_stats64 *p_stats)
8577{
8578 struct rtnl_hw_stats64 *stats;
8579
8580 ASSERT_RTNL();
8581
8582 stats = netdev_offload_xstats_get_ptr(dev, type);
8583 if (WARN_ON(!stats))
8584 return;
8585
8586 netdev_hw_stats64_add(stats, p_stats);
8587}
8588EXPORT_SYMBOL(netdev_offload_xstats_push_delta);
8589
8590/**
8591 * netdev_get_xmit_slave - Get the xmit slave of master device
8592 * @dev: device
8593 * @skb: The packet
8594 * @all_slaves: assume all the slaves are active
8595 *
8596 * The reference counters are not incremented so the caller must be
8597 * careful with locks. The caller must hold RCU lock.
8598 * %NULL is returned if no slave is found.
8599 */
8600
8601struct net_device *netdev_get_xmit_slave(struct net_device *dev,
8602 struct sk_buff *skb,
8603 bool all_slaves)
8604{
8605 const struct net_device_ops *ops = dev->netdev_ops;
8606
8607 if (!ops->ndo_get_xmit_slave)
8608 return NULL;
8609 return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
8610}
8611EXPORT_SYMBOL(netdev_get_xmit_slave);
8612
8613static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
8614 struct sock *sk)
8615{
8616 const struct net_device_ops *ops = dev->netdev_ops;
8617
8618 if (!ops->ndo_sk_get_lower_dev)
8619 return NULL;
8620 return ops->ndo_sk_get_lower_dev(dev, sk);
8621}
8622
8623/**
8624 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8625 * @dev: device
8626 * @sk: the socket
8627 *
8628 * %NULL is returned if no lower device is found.
8629 */
8630
8631struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
8632 struct sock *sk)
8633{
8634 struct net_device *lower;
8635
8636 lower = netdev_sk_get_lower_dev(dev, sk);
8637 while (lower) {
8638 dev = lower;
8639 lower = netdev_sk_get_lower_dev(dev, sk);
8640 }
8641
8642 return dev;
8643}
8644EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
8645
8646static void netdev_adjacent_add_links(struct net_device *dev)
8647{
8648 struct netdev_adjacent *iter;
8649
8650 struct net *net = dev_net(dev);
8651
8652 list_for_each_entry(iter, &dev->adj_list.upper, list) {
8653 if (!net_eq(net, dev_net(iter->dev)))
8654 continue;
8655 netdev_adjacent_sysfs_add(iter->dev, dev,
8656 &iter->dev->adj_list.lower);
8657 netdev_adjacent_sysfs_add(dev, iter->dev,
8658 &dev->adj_list.upper);
8659 }
8660
8661 list_for_each_entry(iter, &dev->adj_list.lower, list) {
8662 if (!net_eq(net, dev_net(iter->dev)))
8663 continue;
8664 netdev_adjacent_sysfs_add(iter->dev, dev,
8665 &iter->dev->adj_list.upper);
8666 netdev_adjacent_sysfs_add(dev, iter->dev,
8667 &dev->adj_list.lower);
8668 }
8669}
8670
8671static void netdev_adjacent_del_links(struct net_device *dev)
8672{
8673 struct netdev_adjacent *iter;
8674
8675 struct net *net = dev_net(dev);
8676
8677 list_for_each_entry(iter, &dev->adj_list.upper, list) {
8678 if (!net_eq(net, dev_net(iter->dev)))
8679 continue;
8680 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8681 &iter->dev->adj_list.lower);
8682 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8683 &dev->adj_list.upper);
8684 }
8685
8686 list_for_each_entry(iter, &dev->adj_list.lower, list) {
8687 if (!net_eq(net, dev_net(iter->dev)))
8688 continue;
8689 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8690 &iter->dev->adj_list.upper);
8691 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8692 &dev->adj_list.lower);
8693 }
8694}
8695
8696void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
8697{
8698 struct netdev_adjacent *iter;
8699
8700 struct net *net = dev_net(dev);
8701
8702 list_for_each_entry(iter, &dev->adj_list.upper, list) {
8703 if (!net_eq(net, dev_net(iter->dev)))
8704 continue;
8705 netdev_adjacent_sysfs_del(iter->dev, oldname,
8706 &iter->dev->adj_list.lower);
8707 netdev_adjacent_sysfs_add(iter->dev, dev,
8708 &iter->dev->adj_list.lower);
8709 }
8710
8711 list_for_each_entry(iter, &dev->adj_list.lower, list) {
8712 if (!net_eq(net, dev_net(iter->dev)))
8713 continue;
8714 netdev_adjacent_sysfs_del(iter->dev, oldname,
8715 &iter->dev->adj_list.upper);
8716 netdev_adjacent_sysfs_add(iter->dev, dev,
8717 &iter->dev->adj_list.upper);
8718 }
8719}
8720
8721void *netdev_lower_dev_get_private(struct net_device *dev,
8722 struct net_device *lower_dev)
8723{
8724 struct netdev_adjacent *lower;
8725
8726 if (!lower_dev)
8727 return NULL;
8728 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
8729 if (!lower)
8730 return NULL;
8731
8732 return lower->private;
8733}
8734EXPORT_SYMBOL(netdev_lower_dev_get_private);
8735
8736
8737/**
8738 * netdev_lower_state_changed - Dispatch event about lower device state change
8739 * @lower_dev: device
8740 * @lower_state_info: state to dispatch
8741 *
8742 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
8743 * The caller must hold the RTNL lock.
8744 */
8745void netdev_lower_state_changed(struct net_device *lower_dev,
8746 void *lower_state_info)
8747{
8748 struct netdev_notifier_changelowerstate_info changelowerstate_info = {
8749 .info.dev = lower_dev,
8750 };
8751
8752 ASSERT_RTNL();
8753 changelowerstate_info.lower_state_info = lower_state_info;
8754 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
8755 &changelowerstate_info.info);
8756}
8757EXPORT_SYMBOL(netdev_lower_state_changed);
8758
8759static void dev_change_rx_flags(struct net_device *dev, int flags)
8760{
8761 const struct net_device_ops *ops = dev->netdev_ops;
8762
8763 if (ops->ndo_change_rx_flags)
8764 ops->ndo_change_rx_flags(dev, flags);
8765}
8766
8767static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
8768{
8769 unsigned int old_flags = dev->flags;
8770 unsigned int promiscuity, flags;
8771 kuid_t uid;
8772 kgid_t gid;
8773
8774 ASSERT_RTNL();
8775
8776 promiscuity = dev->promiscuity + inc;
8777 if (promiscuity == 0) {
8778 /*
8779 * Avoid overflow.
8780 * If inc causes overflow, untouch promisc and return error.
8781 */
8782 if (unlikely(inc > 0)) {
8783 netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n");
8784 return -EOVERFLOW;
8785 }
8786 flags = old_flags & ~IFF_PROMISC;
8787 } else {
8788 flags = old_flags | IFF_PROMISC;
8789 }
8790 WRITE_ONCE(dev->promiscuity, promiscuity);
8791 if (flags != old_flags) {
8792 WRITE_ONCE(dev->flags, flags);
8793 netdev_info(dev, "%s promiscuous mode\n",
8794 dev->flags & IFF_PROMISC ? "entered" : "left");
8795 if (audit_enabled) {
8796 current_uid_gid(&uid, &gid);
8797 audit_log(audit_context(), GFP_ATOMIC,
8798 AUDIT_ANOM_PROMISCUOUS,
8799 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
8800 dev->name, (dev->flags & IFF_PROMISC),
8801 (old_flags & IFF_PROMISC),
8802 from_kuid(&init_user_ns, audit_get_loginuid(current)),
8803 from_kuid(&init_user_ns, uid),
8804 from_kgid(&init_user_ns, gid),
8805 audit_get_sessionid(current));
8806 }
8807
8808 dev_change_rx_flags(dev, IFF_PROMISC);
8809 }
8810 if (notify)
8811 __dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL);
8812 return 0;
8813}
8814
8815/**
8816 * dev_set_promiscuity - update promiscuity count on a device
8817 * @dev: device
8818 * @inc: modifier
8819 *
8820 * Add or remove promiscuity from a device. While the count in the device
8821 * remains above zero the interface remains promiscuous. Once it hits zero
8822 * the device reverts back to normal filtering operation. A negative inc
8823 * value is used to drop promiscuity on the device.
8824 * Return 0 if successful or a negative errno code on error.
8825 */
8826int dev_set_promiscuity(struct net_device *dev, int inc)
8827{
8828 unsigned int old_flags = dev->flags;
8829 int err;
8830
8831 err = __dev_set_promiscuity(dev, inc, true);
8832 if (err < 0)
8833 return err;
8834 if (dev->flags != old_flags)
8835 dev_set_rx_mode(dev);
8836 return err;
8837}
8838EXPORT_SYMBOL(dev_set_promiscuity);
8839
8840static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
8841{
8842 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
8843 unsigned int allmulti, flags;
8844
8845 ASSERT_RTNL();
8846
8847 allmulti = dev->allmulti + inc;
8848 if (allmulti == 0) {
8849 /*
8850 * Avoid overflow.
8851 * If inc causes overflow, untouch allmulti and return error.
8852 */
8853 if (unlikely(inc > 0)) {
8854 netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n");
8855 return -EOVERFLOW;
8856 }
8857 flags = old_flags & ~IFF_ALLMULTI;
8858 } else {
8859 flags = old_flags | IFF_ALLMULTI;
8860 }
8861 WRITE_ONCE(dev->allmulti, allmulti);
8862 if (flags != old_flags) {
8863 WRITE_ONCE(dev->flags, flags);
8864 netdev_info(dev, "%s allmulticast mode\n",
8865 dev->flags & IFF_ALLMULTI ? "entered" : "left");
8866 dev_change_rx_flags(dev, IFF_ALLMULTI);
8867 dev_set_rx_mode(dev);
8868 if (notify)
8869 __dev_notify_flags(dev, old_flags,
8870 dev->gflags ^ old_gflags, 0, NULL);
8871 }
8872 return 0;
8873}
8874
8875/**
8876 * dev_set_allmulti - update allmulti count on a device
8877 * @dev: device
8878 * @inc: modifier
8879 *
8880 * Add or remove reception of all multicast frames to a device. While the
8881 * count in the device remains above zero the interface remains listening
8882 * to all interfaces. Once it hits zero the device reverts back to normal
8883 * filtering operation. A negative @inc value is used to drop the counter
8884 * when releasing a resource needing all multicasts.
8885 * Return 0 if successful or a negative errno code on error.
8886 */
8887
8888int dev_set_allmulti(struct net_device *dev, int inc)
8889{
8890 return __dev_set_allmulti(dev, inc, true);
8891}
8892EXPORT_SYMBOL(dev_set_allmulti);
8893
8894/*
8895 * Upload unicast and multicast address lists to device and
8896 * configure RX filtering. When the device doesn't support unicast
8897 * filtering it is put in promiscuous mode while unicast addresses
8898 * are present.
8899 */
8900void __dev_set_rx_mode(struct net_device *dev)
8901{
8902 const struct net_device_ops *ops = dev->netdev_ops;
8903
8904 /* dev_open will call this function so the list will stay sane. */
8905 if (!(dev->flags&IFF_UP))
8906 return;
8907
8908 if (!netif_device_present(dev))
8909 return;
8910
8911 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
8912 /* Unicast addresses changes may only happen under the rtnl,
8913 * therefore calling __dev_set_promiscuity here is safe.
8914 */
8915 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
8916 __dev_set_promiscuity(dev, 1, false);
8917 dev->uc_promisc = true;
8918 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
8919 __dev_set_promiscuity(dev, -1, false);
8920 dev->uc_promisc = false;
8921 }
8922 }
8923
8924 if (ops->ndo_set_rx_mode)
8925 ops->ndo_set_rx_mode(dev);
8926}
8927
8928void dev_set_rx_mode(struct net_device *dev)
8929{
8930 netif_addr_lock_bh(dev);
8931 __dev_set_rx_mode(dev);
8932 netif_addr_unlock_bh(dev);
8933}
8934
8935/**
8936 * dev_get_flags - get flags reported to userspace
8937 * @dev: device
8938 *
8939 * Get the combination of flag bits exported through APIs to userspace.
8940 */
8941unsigned int dev_get_flags(const struct net_device *dev)
8942{
8943 unsigned int flags;
8944
8945 flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC |
8946 IFF_ALLMULTI |
8947 IFF_RUNNING |
8948 IFF_LOWER_UP |
8949 IFF_DORMANT)) |
8950 (READ_ONCE(dev->gflags) & (IFF_PROMISC |
8951 IFF_ALLMULTI));
8952
8953 if (netif_running(dev)) {
8954 if (netif_oper_up(dev))
8955 flags |= IFF_RUNNING;
8956 if (netif_carrier_ok(dev))
8957 flags |= IFF_LOWER_UP;
8958 if (netif_dormant(dev))
8959 flags |= IFF_DORMANT;
8960 }
8961
8962 return flags;
8963}
8964EXPORT_SYMBOL(dev_get_flags);
8965
8966int __dev_change_flags(struct net_device *dev, unsigned int flags,
8967 struct netlink_ext_ack *extack)
8968{
8969 unsigned int old_flags = dev->flags;
8970 int ret;
8971
8972 ASSERT_RTNL();
8973
8974 /*
8975 * Set the flags on our device.
8976 */
8977
8978 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
8979 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
8980 IFF_AUTOMEDIA)) |
8981 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
8982 IFF_ALLMULTI));
8983
8984 /*
8985 * Load in the correct multicast list now the flags have changed.
8986 */
8987
8988 if ((old_flags ^ flags) & IFF_MULTICAST)
8989 dev_change_rx_flags(dev, IFF_MULTICAST);
8990
8991 dev_set_rx_mode(dev);
8992
8993 /*
8994 * Have we downed the interface. We handle IFF_UP ourselves
8995 * according to user attempts to set it, rather than blindly
8996 * setting it.
8997 */
8998
8999 ret = 0;
9000 if ((old_flags ^ flags) & IFF_UP) {
9001 if (old_flags & IFF_UP)
9002 __dev_close(dev);
9003 else
9004 ret = __dev_open(dev, extack);
9005 }
9006
9007 if ((flags ^ dev->gflags) & IFF_PROMISC) {
9008 int inc = (flags & IFF_PROMISC) ? 1 : -1;
9009 unsigned int old_flags = dev->flags;
9010
9011 dev->gflags ^= IFF_PROMISC;
9012
9013 if (__dev_set_promiscuity(dev, inc, false) >= 0)
9014 if (dev->flags != old_flags)
9015 dev_set_rx_mode(dev);
9016 }
9017
9018 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
9019 * is important. Some (broken) drivers set IFF_PROMISC, when
9020 * IFF_ALLMULTI is requested not asking us and not reporting.
9021 */
9022 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
9023 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
9024
9025 dev->gflags ^= IFF_ALLMULTI;
9026 __dev_set_allmulti(dev, inc, false);
9027 }
9028
9029 return ret;
9030}
9031
9032void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
9033 unsigned int gchanges, u32 portid,
9034 const struct nlmsghdr *nlh)
9035{
9036 unsigned int changes = dev->flags ^ old_flags;
9037
9038 if (gchanges)
9039 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh);
9040
9041 if (changes & IFF_UP) {
9042 if (dev->flags & IFF_UP)
9043 call_netdevice_notifiers(NETDEV_UP, dev);
9044 else
9045 call_netdevice_notifiers(NETDEV_DOWN, dev);
9046 }
9047
9048 if (dev->flags & IFF_UP &&
9049 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
9050 struct netdev_notifier_change_info change_info = {
9051 .info = {
9052 .dev = dev,
9053 },
9054 .flags_changed = changes,
9055 };
9056
9057 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
9058 }
9059}
9060
9061/**
9062 * dev_change_flags - change device settings
9063 * @dev: device
9064 * @flags: device state flags
9065 * @extack: netlink extended ack
9066 *
9067 * Change settings on device based state flags. The flags are
9068 * in the userspace exported format.
9069 */
9070int dev_change_flags(struct net_device *dev, unsigned int flags,
9071 struct netlink_ext_ack *extack)
9072{
9073 int ret;
9074 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
9075
9076 ret = __dev_change_flags(dev, flags, extack);
9077 if (ret < 0)
9078 return ret;
9079
9080 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
9081 __dev_notify_flags(dev, old_flags, changes, 0, NULL);
9082 return ret;
9083}
9084EXPORT_SYMBOL(dev_change_flags);
9085
9086int __dev_set_mtu(struct net_device *dev, int new_mtu)
9087{
9088 const struct net_device_ops *ops = dev->netdev_ops;
9089
9090 if (ops->ndo_change_mtu)
9091 return ops->ndo_change_mtu(dev, new_mtu);
9092
9093 /* Pairs with all the lockless reads of dev->mtu in the stack */
9094 WRITE_ONCE(dev->mtu, new_mtu);
9095 return 0;
9096}
9097EXPORT_SYMBOL(__dev_set_mtu);
9098
9099int dev_validate_mtu(struct net_device *dev, int new_mtu,
9100 struct netlink_ext_ack *extack)
9101{
9102 /* MTU must be positive, and in range */
9103 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
9104 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
9105 return -EINVAL;
9106 }
9107
9108 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
9109 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
9110 return -EINVAL;
9111 }
9112 return 0;
9113}
9114
9115/**
9116 * dev_set_mtu_ext - Change maximum transfer unit
9117 * @dev: device
9118 * @new_mtu: new transfer unit
9119 * @extack: netlink extended ack
9120 *
9121 * Change the maximum transfer size of the network device.
9122 */
9123int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
9124 struct netlink_ext_ack *extack)
9125{
9126 int err, orig_mtu;
9127
9128 if (new_mtu == dev->mtu)
9129 return 0;
9130
9131 err = dev_validate_mtu(dev, new_mtu, extack);
9132 if (err)
9133 return err;
9134
9135 if (!netif_device_present(dev))
9136 return -ENODEV;
9137
9138 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
9139 err = notifier_to_errno(err);
9140 if (err)
9141 return err;
9142
9143 orig_mtu = dev->mtu;
9144 err = __dev_set_mtu(dev, new_mtu);
9145
9146 if (!err) {
9147 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
9148 orig_mtu);
9149 err = notifier_to_errno(err);
9150 if (err) {
9151 /* setting mtu back and notifying everyone again,
9152 * so that they have a chance to revert changes.
9153 */
9154 __dev_set_mtu(dev, orig_mtu);
9155 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
9156 new_mtu);
9157 }
9158 }
9159 return err;
9160}
9161
9162int dev_set_mtu(struct net_device *dev, int new_mtu)
9163{
9164 struct netlink_ext_ack extack;
9165 int err;
9166
9167 memset(&extack, 0, sizeof(extack));
9168 err = dev_set_mtu_ext(dev, new_mtu, &extack);
9169 if (err && extack._msg)
9170 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
9171 return err;
9172}
9173EXPORT_SYMBOL(dev_set_mtu);
9174
9175/**
9176 * dev_change_tx_queue_len - Change TX queue length of a netdevice
9177 * @dev: device
9178 * @new_len: new tx queue length
9179 */
9180int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
9181{
9182 unsigned int orig_len = dev->tx_queue_len;
9183 int res;
9184
9185 if (new_len != (unsigned int)new_len)
9186 return -ERANGE;
9187
9188 if (new_len != orig_len) {
9189 WRITE_ONCE(dev->tx_queue_len, new_len);
9190 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
9191 res = notifier_to_errno(res);
9192 if (res)
9193 goto err_rollback;
9194 res = dev_qdisc_change_tx_queue_len(dev);
9195 if (res)
9196 goto err_rollback;
9197 }
9198
9199 return 0;
9200
9201err_rollback:
9202 netdev_err(dev, "refused to change device tx_queue_len\n");
9203 WRITE_ONCE(dev->tx_queue_len, orig_len);
9204 return res;
9205}
9206
9207/**
9208 * dev_set_group - Change group this device belongs to
9209 * @dev: device
9210 * @new_group: group this device should belong to
9211 */
9212void dev_set_group(struct net_device *dev, int new_group)
9213{
9214 dev->group = new_group;
9215}
9216
9217/**
9218 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
9219 * @dev: device
9220 * @addr: new address
9221 * @extack: netlink extended ack
9222 */
9223int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
9224 struct netlink_ext_ack *extack)
9225{
9226 struct netdev_notifier_pre_changeaddr_info info = {
9227 .info.dev = dev,
9228 .info.extack = extack,
9229 .dev_addr = addr,
9230 };
9231 int rc;
9232
9233 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
9234 return notifier_to_errno(rc);
9235}
9236EXPORT_SYMBOL(dev_pre_changeaddr_notify);
9237
9238/**
9239 * dev_set_mac_address - Change Media Access Control Address
9240 * @dev: device
9241 * @sa: new address
9242 * @extack: netlink extended ack
9243 *
9244 * Change the hardware (MAC) address of the device
9245 */
9246int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
9247 struct netlink_ext_ack *extack)
9248{
9249 const struct net_device_ops *ops = dev->netdev_ops;
9250 int err;
9251
9252 if (!ops->ndo_set_mac_address)
9253 return -EOPNOTSUPP;
9254 if (sa->sa_family != dev->type)
9255 return -EINVAL;
9256 if (!netif_device_present(dev))
9257 return -ENODEV;
9258 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
9259 if (err)
9260 return err;
9261 if (memcmp(dev->dev_addr, sa->sa_data, dev->addr_len)) {
9262 err = ops->ndo_set_mac_address(dev, sa);
9263 if (err)
9264 return err;
9265 }
9266 dev->addr_assign_type = NET_ADDR_SET;
9267 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
9268 add_device_randomness(dev->dev_addr, dev->addr_len);
9269 return 0;
9270}
9271EXPORT_SYMBOL(dev_set_mac_address);
9272
9273DECLARE_RWSEM(dev_addr_sem);
9274
9275int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
9276 struct netlink_ext_ack *extack)
9277{
9278 int ret;
9279
9280 down_write(&dev_addr_sem);
9281 ret = dev_set_mac_address(dev, sa, extack);
9282 up_write(&dev_addr_sem);
9283 return ret;
9284}
9285EXPORT_SYMBOL(dev_set_mac_address_user);
9286
9287int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
9288{
9289 size_t size = sizeof(sa->sa_data_min);
9290 struct net_device *dev;
9291 int ret = 0;
9292
9293 down_read(&dev_addr_sem);
9294 rcu_read_lock();
9295
9296 dev = dev_get_by_name_rcu(net, dev_name);
9297 if (!dev) {
9298 ret = -ENODEV;
9299 goto unlock;
9300 }
9301 if (!dev->addr_len)
9302 memset(sa->sa_data, 0, size);
9303 else
9304 memcpy(sa->sa_data, dev->dev_addr,
9305 min_t(size_t, size, dev->addr_len));
9306 sa->sa_family = dev->type;
9307
9308unlock:
9309 rcu_read_unlock();
9310 up_read(&dev_addr_sem);
9311 return ret;
9312}
9313EXPORT_SYMBOL(dev_get_mac_address);
9314
9315/**
9316 * dev_change_carrier - Change device carrier
9317 * @dev: device
9318 * @new_carrier: new value
9319 *
9320 * Change device carrier
9321 */
9322int dev_change_carrier(struct net_device *dev, bool new_carrier)
9323{
9324 const struct net_device_ops *ops = dev->netdev_ops;
9325
9326 if (!ops->ndo_change_carrier)
9327 return -EOPNOTSUPP;
9328 if (!netif_device_present(dev))
9329 return -ENODEV;
9330 return ops->ndo_change_carrier(dev, new_carrier);
9331}
9332
9333/**
9334 * dev_get_phys_port_id - Get device physical port ID
9335 * @dev: device
9336 * @ppid: port ID
9337 *
9338 * Get device physical port ID
9339 */
9340int dev_get_phys_port_id(struct net_device *dev,
9341 struct netdev_phys_item_id *ppid)
9342{
9343 const struct net_device_ops *ops = dev->netdev_ops;
9344
9345 if (!ops->ndo_get_phys_port_id)
9346 return -EOPNOTSUPP;
9347 return ops->ndo_get_phys_port_id(dev, ppid);
9348}
9349
9350/**
9351 * dev_get_phys_port_name - Get device physical port name
9352 * @dev: device
9353 * @name: port name
9354 * @len: limit of bytes to copy to name
9355 *
9356 * Get device physical port name
9357 */
9358int dev_get_phys_port_name(struct net_device *dev,
9359 char *name, size_t len)
9360{
9361 const struct net_device_ops *ops = dev->netdev_ops;
9362 int err;
9363
9364 if (ops->ndo_get_phys_port_name) {
9365 err = ops->ndo_get_phys_port_name(dev, name, len);
9366 if (err != -EOPNOTSUPP)
9367 return err;
9368 }
9369 return devlink_compat_phys_port_name_get(dev, name, len);
9370}
9371
9372/**
9373 * dev_get_port_parent_id - Get the device's port parent identifier
9374 * @dev: network device
9375 * @ppid: pointer to a storage for the port's parent identifier
9376 * @recurse: allow/disallow recursion to lower devices
9377 *
9378 * Get the devices's port parent identifier
9379 */
9380int dev_get_port_parent_id(struct net_device *dev,
9381 struct netdev_phys_item_id *ppid,
9382 bool recurse)
9383{
9384 const struct net_device_ops *ops = dev->netdev_ops;
9385 struct netdev_phys_item_id first = { };
9386 struct net_device *lower_dev;
9387 struct list_head *iter;
9388 int err;
9389
9390 if (ops->ndo_get_port_parent_id) {
9391 err = ops->ndo_get_port_parent_id(dev, ppid);
9392 if (err != -EOPNOTSUPP)
9393 return err;
9394 }
9395
9396 err = devlink_compat_switch_id_get(dev, ppid);
9397 if (!recurse || err != -EOPNOTSUPP)
9398 return err;
9399
9400 netdev_for_each_lower_dev(dev, lower_dev, iter) {
9401 err = dev_get_port_parent_id(lower_dev, ppid, true);
9402 if (err)
9403 break;
9404 if (!first.id_len)
9405 first = *ppid;
9406 else if (memcmp(&first, ppid, sizeof(*ppid)))
9407 return -EOPNOTSUPP;
9408 }
9409
9410 return err;
9411}
9412EXPORT_SYMBOL(dev_get_port_parent_id);
9413
9414/**
9415 * netdev_port_same_parent_id - Indicate if two network devices have
9416 * the same port parent identifier
9417 * @a: first network device
9418 * @b: second network device
9419 */
9420bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
9421{
9422 struct netdev_phys_item_id a_id = { };
9423 struct netdev_phys_item_id b_id = { };
9424
9425 if (dev_get_port_parent_id(a, &a_id, true) ||
9426 dev_get_port_parent_id(b, &b_id, true))
9427 return false;
9428
9429 return netdev_phys_item_id_same(&a_id, &b_id);
9430}
9431EXPORT_SYMBOL(netdev_port_same_parent_id);
9432
9433/**
9434 * dev_change_proto_down - set carrier according to proto_down.
9435 *
9436 * @dev: device
9437 * @proto_down: new value
9438 */
9439int dev_change_proto_down(struct net_device *dev, bool proto_down)
9440{
9441 if (!dev->change_proto_down)
9442 return -EOPNOTSUPP;
9443 if (!netif_device_present(dev))
9444 return -ENODEV;
9445 if (proto_down)
9446 netif_carrier_off(dev);
9447 else
9448 netif_carrier_on(dev);
9449 WRITE_ONCE(dev->proto_down, proto_down);
9450 return 0;
9451}
9452
9453/**
9454 * dev_change_proto_down_reason - proto down reason
9455 *
9456 * @dev: device
9457 * @mask: proto down mask
9458 * @value: proto down value
9459 */
9460void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
9461 u32 value)
9462{
9463 u32 proto_down_reason;
9464 int b;
9465
9466 if (!mask) {
9467 proto_down_reason = value;
9468 } else {
9469 proto_down_reason = dev->proto_down_reason;
9470 for_each_set_bit(b, &mask, 32) {
9471 if (value & (1 << b))
9472 proto_down_reason |= BIT(b);
9473 else
9474 proto_down_reason &= ~BIT(b);
9475 }
9476 }
9477 WRITE_ONCE(dev->proto_down_reason, proto_down_reason);
9478}
9479
9480struct bpf_xdp_link {
9481 struct bpf_link link;
9482 struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
9483 int flags;
9484};
9485
9486static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
9487{
9488 if (flags & XDP_FLAGS_HW_MODE)
9489 return XDP_MODE_HW;
9490 if (flags & XDP_FLAGS_DRV_MODE)
9491 return XDP_MODE_DRV;
9492 if (flags & XDP_FLAGS_SKB_MODE)
9493 return XDP_MODE_SKB;
9494 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
9495}
9496
9497static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
9498{
9499 switch (mode) {
9500 case XDP_MODE_SKB:
9501 return generic_xdp_install;
9502 case XDP_MODE_DRV:
9503 case XDP_MODE_HW:
9504 return dev->netdev_ops->ndo_bpf;
9505 default:
9506 return NULL;
9507 }
9508}
9509
9510static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
9511 enum bpf_xdp_mode mode)
9512{
9513 return dev->xdp_state[mode].link;
9514}
9515
9516static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
9517 enum bpf_xdp_mode mode)
9518{
9519 struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
9520
9521 if (link)
9522 return link->link.prog;
9523 return dev->xdp_state[mode].prog;
9524}
9525
9526u8 dev_xdp_prog_count(struct net_device *dev)
9527{
9528 u8 count = 0;
9529 int i;
9530
9531 for (i = 0; i < __MAX_XDP_MODE; i++)
9532 if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
9533 count++;
9534 return count;
9535}
9536EXPORT_SYMBOL_GPL(dev_xdp_prog_count);
9537
9538int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
9539{
9540 if (!dev->netdev_ops->ndo_bpf)
9541 return -EOPNOTSUPP;
9542
9543 if (dev_get_min_mp_channel_count(dev)) {
9544 NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider");
9545 return -EBUSY;
9546 }
9547
9548 return dev->netdev_ops->ndo_bpf(dev, bpf);
9549}
9550EXPORT_SYMBOL_GPL(dev_xdp_propagate);
9551
9552u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
9553{
9554 struct bpf_prog *prog = dev_xdp_prog(dev, mode);
9555
9556 return prog ? prog->aux->id : 0;
9557}
9558
9559static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
9560 struct bpf_xdp_link *link)
9561{
9562 dev->xdp_state[mode].link = link;
9563 dev->xdp_state[mode].prog = NULL;
9564}
9565
9566static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
9567 struct bpf_prog *prog)
9568{
9569 dev->xdp_state[mode].link = NULL;
9570 dev->xdp_state[mode].prog = prog;
9571}
9572
9573static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
9574 bpf_op_t bpf_op, struct netlink_ext_ack *extack,
9575 u32 flags, struct bpf_prog *prog)
9576{
9577 struct netdev_bpf xdp;
9578 int err;
9579
9580 if (dev_get_min_mp_channel_count(dev)) {
9581 NL_SET_ERR_MSG(extack, "unable to install XDP to device using memory provider");
9582 return -EBUSY;
9583 }
9584
9585 memset(&xdp, 0, sizeof(xdp));
9586 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
9587 xdp.extack = extack;
9588 xdp.flags = flags;
9589 xdp.prog = prog;
9590
9591 /* Drivers assume refcnt is already incremented (i.e, prog pointer is
9592 * "moved" into driver), so they don't increment it on their own, but
9593 * they do decrement refcnt when program is detached or replaced.
9594 * Given net_device also owns link/prog, we need to bump refcnt here
9595 * to prevent drivers from underflowing it.
9596 */
9597 if (prog)
9598 bpf_prog_inc(prog);
9599 err = bpf_op(dev, &xdp);
9600 if (err) {
9601 if (prog)
9602 bpf_prog_put(prog);
9603 return err;
9604 }
9605
9606 if (mode != XDP_MODE_HW)
9607 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
9608
9609 return 0;
9610}
9611
9612static void dev_xdp_uninstall(struct net_device *dev)
9613{
9614 struct bpf_xdp_link *link;
9615 struct bpf_prog *prog;
9616 enum bpf_xdp_mode mode;
9617 bpf_op_t bpf_op;
9618
9619 ASSERT_RTNL();
9620
9621 for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
9622 prog = dev_xdp_prog(dev, mode);
9623 if (!prog)
9624 continue;
9625
9626 bpf_op = dev_xdp_bpf_op(dev, mode);
9627 if (!bpf_op)
9628 continue;
9629
9630 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9631
9632 /* auto-detach link from net device */
9633 link = dev_xdp_link(dev, mode);
9634 if (link)
9635 link->dev = NULL;
9636 else
9637 bpf_prog_put(prog);
9638
9639 dev_xdp_set_link(dev, mode, NULL);
9640 }
9641}
9642
9643static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
9644 struct bpf_xdp_link *link, struct bpf_prog *new_prog,
9645 struct bpf_prog *old_prog, u32 flags)
9646{
9647 unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
9648 struct bpf_prog *cur_prog;
9649 struct net_device *upper;
9650 struct list_head *iter;
9651 enum bpf_xdp_mode mode;
9652 bpf_op_t bpf_op;
9653 int err;
9654
9655 ASSERT_RTNL();
9656
9657 /* either link or prog attachment, never both */
9658 if (link && (new_prog || old_prog))
9659 return -EINVAL;
9660 /* link supports only XDP mode flags */
9661 if (link && (flags & ~XDP_FLAGS_MODES)) {
9662 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
9663 return -EINVAL;
9664 }
9665 /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
9666 if (num_modes > 1) {
9667 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
9668 return -EINVAL;
9669 }
9670 /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
9671 if (!num_modes && dev_xdp_prog_count(dev) > 1) {
9672 NL_SET_ERR_MSG(extack,
9673 "More than one program loaded, unset mode is ambiguous");
9674 return -EINVAL;
9675 }
9676 /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
9677 if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
9678 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
9679 return -EINVAL;
9680 }
9681
9682 mode = dev_xdp_mode(dev, flags);
9683 /* can't replace attached link */
9684 if (dev_xdp_link(dev, mode)) {
9685 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
9686 return -EBUSY;
9687 }
9688
9689 /* don't allow if an upper device already has a program */
9690 netdev_for_each_upper_dev_rcu(dev, upper, iter) {
9691 if (dev_xdp_prog_count(upper) > 0) {
9692 NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program");
9693 return -EEXIST;
9694 }
9695 }
9696
9697 cur_prog = dev_xdp_prog(dev, mode);
9698 /* can't replace attached prog with link */
9699 if (link && cur_prog) {
9700 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
9701 return -EBUSY;
9702 }
9703 if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
9704 NL_SET_ERR_MSG(extack, "Active program does not match expected");
9705 return -EEXIST;
9706 }
9707
9708 /* put effective new program into new_prog */
9709 if (link)
9710 new_prog = link->link.prog;
9711
9712 if (new_prog) {
9713 bool offload = mode == XDP_MODE_HW;
9714 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
9715 ? XDP_MODE_DRV : XDP_MODE_SKB;
9716
9717 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
9718 NL_SET_ERR_MSG(extack, "XDP program already attached");
9719 return -EBUSY;
9720 }
9721 if (!offload && dev_xdp_prog(dev, other_mode)) {
9722 NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
9723 return -EEXIST;
9724 }
9725 if (!offload && bpf_prog_is_offloaded(new_prog->aux)) {
9726 NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported");
9727 return -EINVAL;
9728 }
9729 if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) {
9730 NL_SET_ERR_MSG(extack, "Program bound to different device");
9731 return -EINVAL;
9732 }
9733 if (bpf_prog_is_dev_bound(new_prog->aux) && mode == XDP_MODE_SKB) {
9734 NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode");
9735 return -EINVAL;
9736 }
9737 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
9738 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
9739 return -EINVAL;
9740 }
9741 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
9742 NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
9743 return -EINVAL;
9744 }
9745 }
9746
9747 /* don't call drivers if the effective program didn't change */
9748 if (new_prog != cur_prog) {
9749 bpf_op = dev_xdp_bpf_op(dev, mode);
9750 if (!bpf_op) {
9751 NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
9752 return -EOPNOTSUPP;
9753 }
9754
9755 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
9756 if (err)
9757 return err;
9758 }
9759
9760 if (link)
9761 dev_xdp_set_link(dev, mode, link);
9762 else
9763 dev_xdp_set_prog(dev, mode, new_prog);
9764 if (cur_prog)
9765 bpf_prog_put(cur_prog);
9766
9767 return 0;
9768}
9769
9770static int dev_xdp_attach_link(struct net_device *dev,
9771 struct netlink_ext_ack *extack,
9772 struct bpf_xdp_link *link)
9773{
9774 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
9775}
9776
9777static int dev_xdp_detach_link(struct net_device *dev,
9778 struct netlink_ext_ack *extack,
9779 struct bpf_xdp_link *link)
9780{
9781 enum bpf_xdp_mode mode;
9782 bpf_op_t bpf_op;
9783
9784 ASSERT_RTNL();
9785
9786 mode = dev_xdp_mode(dev, link->flags);
9787 if (dev_xdp_link(dev, mode) != link)
9788 return -EINVAL;
9789
9790 bpf_op = dev_xdp_bpf_op(dev, mode);
9791 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9792 dev_xdp_set_link(dev, mode, NULL);
9793 return 0;
9794}
9795
9796static void bpf_xdp_link_release(struct bpf_link *link)
9797{
9798 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9799
9800 rtnl_lock();
9801
9802 /* if racing with net_device's tear down, xdp_link->dev might be
9803 * already NULL, in which case link was already auto-detached
9804 */
9805 if (xdp_link->dev) {
9806 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
9807 xdp_link->dev = NULL;
9808 }
9809
9810 rtnl_unlock();
9811}
9812
9813static int bpf_xdp_link_detach(struct bpf_link *link)
9814{
9815 bpf_xdp_link_release(link);
9816 return 0;
9817}
9818
9819static void bpf_xdp_link_dealloc(struct bpf_link *link)
9820{
9821 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9822
9823 kfree(xdp_link);
9824}
9825
9826static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
9827 struct seq_file *seq)
9828{
9829 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9830 u32 ifindex = 0;
9831
9832 rtnl_lock();
9833 if (xdp_link->dev)
9834 ifindex = xdp_link->dev->ifindex;
9835 rtnl_unlock();
9836
9837 seq_printf(seq, "ifindex:\t%u\n", ifindex);
9838}
9839
9840static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
9841 struct bpf_link_info *info)
9842{
9843 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9844 u32 ifindex = 0;
9845
9846 rtnl_lock();
9847 if (xdp_link->dev)
9848 ifindex = xdp_link->dev->ifindex;
9849 rtnl_unlock();
9850
9851 info->xdp.ifindex = ifindex;
9852 return 0;
9853}
9854
9855static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
9856 struct bpf_prog *old_prog)
9857{
9858 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9859 enum bpf_xdp_mode mode;
9860 bpf_op_t bpf_op;
9861 int err = 0;
9862
9863 rtnl_lock();
9864
9865 /* link might have been auto-released already, so fail */
9866 if (!xdp_link->dev) {
9867 err = -ENOLINK;
9868 goto out_unlock;
9869 }
9870
9871 if (old_prog && link->prog != old_prog) {
9872 err = -EPERM;
9873 goto out_unlock;
9874 }
9875 old_prog = link->prog;
9876 if (old_prog->type != new_prog->type ||
9877 old_prog->expected_attach_type != new_prog->expected_attach_type) {
9878 err = -EINVAL;
9879 goto out_unlock;
9880 }
9881
9882 if (old_prog == new_prog) {
9883 /* no-op, don't disturb drivers */
9884 bpf_prog_put(new_prog);
9885 goto out_unlock;
9886 }
9887
9888 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
9889 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
9890 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
9891 xdp_link->flags, new_prog);
9892 if (err)
9893 goto out_unlock;
9894
9895 old_prog = xchg(&link->prog, new_prog);
9896 bpf_prog_put(old_prog);
9897
9898out_unlock:
9899 rtnl_unlock();
9900 return err;
9901}
9902
9903static const struct bpf_link_ops bpf_xdp_link_lops = {
9904 .release = bpf_xdp_link_release,
9905 .dealloc = bpf_xdp_link_dealloc,
9906 .detach = bpf_xdp_link_detach,
9907 .show_fdinfo = bpf_xdp_link_show_fdinfo,
9908 .fill_link_info = bpf_xdp_link_fill_link_info,
9909 .update_prog = bpf_xdp_link_update,
9910};
9911
9912int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
9913{
9914 struct net *net = current->nsproxy->net_ns;
9915 struct bpf_link_primer link_primer;
9916 struct netlink_ext_ack extack = {};
9917 struct bpf_xdp_link *link;
9918 struct net_device *dev;
9919 int err, fd;
9920
9921 rtnl_lock();
9922 dev = dev_get_by_index(net, attr->link_create.target_ifindex);
9923 if (!dev) {
9924 rtnl_unlock();
9925 return -EINVAL;
9926 }
9927
9928 link = kzalloc(sizeof(*link), GFP_USER);
9929 if (!link) {
9930 err = -ENOMEM;
9931 goto unlock;
9932 }
9933
9934 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
9935 link->dev = dev;
9936 link->flags = attr->link_create.flags;
9937
9938 err = bpf_link_prime(&link->link, &link_primer);
9939 if (err) {
9940 kfree(link);
9941 goto unlock;
9942 }
9943
9944 err = dev_xdp_attach_link(dev, &extack, link);
9945 rtnl_unlock();
9946
9947 if (err) {
9948 link->dev = NULL;
9949 bpf_link_cleanup(&link_primer);
9950 trace_bpf_xdp_link_attach_failed(extack._msg);
9951 goto out_put_dev;
9952 }
9953
9954 fd = bpf_link_settle(&link_primer);
9955 /* link itself doesn't hold dev's refcnt to not complicate shutdown */
9956 dev_put(dev);
9957 return fd;
9958
9959unlock:
9960 rtnl_unlock();
9961
9962out_put_dev:
9963 dev_put(dev);
9964 return err;
9965}
9966
9967/**
9968 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
9969 * @dev: device
9970 * @extack: netlink extended ack
9971 * @fd: new program fd or negative value to clear
9972 * @expected_fd: old program fd that userspace expects to replace or clear
9973 * @flags: xdp-related flags
9974 *
9975 * Set or clear a bpf program for a device
9976 */
9977int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
9978 int fd, int expected_fd, u32 flags)
9979{
9980 enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
9981 struct bpf_prog *new_prog = NULL, *old_prog = NULL;
9982 int err;
9983
9984 ASSERT_RTNL();
9985
9986 if (fd >= 0) {
9987 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
9988 mode != XDP_MODE_SKB);
9989 if (IS_ERR(new_prog))
9990 return PTR_ERR(new_prog);
9991 }
9992
9993 if (expected_fd >= 0) {
9994 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
9995 mode != XDP_MODE_SKB);
9996 if (IS_ERR(old_prog)) {
9997 err = PTR_ERR(old_prog);
9998 old_prog = NULL;
9999 goto err_out;
10000 }
10001 }
10002
10003 err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
10004
10005err_out:
10006 if (err && new_prog)
10007 bpf_prog_put(new_prog);
10008 if (old_prog)
10009 bpf_prog_put(old_prog);
10010 return err;
10011}
10012
10013u32 dev_get_min_mp_channel_count(const struct net_device *dev)
10014{
10015 int i;
10016
10017 ASSERT_RTNL();
10018
10019 for (i = dev->real_num_rx_queues - 1; i >= 0; i--)
10020 if (dev->_rx[i].mp_params.mp_priv)
10021 /* The channel count is the idx plus 1. */
10022 return i + 1;
10023
10024 return 0;
10025}
10026
10027/**
10028 * dev_index_reserve() - allocate an ifindex in a namespace
10029 * @net: the applicable net namespace
10030 * @ifindex: requested ifindex, pass %0 to get one allocated
10031 *
10032 * Allocate a ifindex for a new device. Caller must either use the ifindex
10033 * to store the device (via list_netdevice()) or call dev_index_release()
10034 * to give the index up.
10035 *
10036 * Return: a suitable unique value for a new device interface number or -errno.
10037 */
10038static int dev_index_reserve(struct net *net, u32 ifindex)
10039{
10040 int err;
10041
10042 if (ifindex > INT_MAX) {
10043 DEBUG_NET_WARN_ON_ONCE(1);
10044 return -EINVAL;
10045 }
10046
10047 if (!ifindex)
10048 err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL,
10049 xa_limit_31b, &net->ifindex, GFP_KERNEL);
10050 else
10051 err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL);
10052 if (err < 0)
10053 return err;
10054
10055 return ifindex;
10056}
10057
10058static void dev_index_release(struct net *net, int ifindex)
10059{
10060 /* Expect only unused indexes, unlist_netdevice() removes the used */
10061 WARN_ON(xa_erase(&net->dev_by_index, ifindex));
10062}
10063
10064/* Delayed registration/unregisteration */
10065LIST_HEAD(net_todo_list);
10066DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
10067atomic_t dev_unreg_count = ATOMIC_INIT(0);
10068
10069static void net_set_todo(struct net_device *dev)
10070{
10071 list_add_tail(&dev->todo_list, &net_todo_list);
10072}
10073
10074static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
10075 struct net_device *upper, netdev_features_t features)
10076{
10077 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
10078 netdev_features_t feature;
10079 int feature_bit;
10080
10081 for_each_netdev_feature(upper_disables, feature_bit) {
10082 feature = __NETIF_F_BIT(feature_bit);
10083 if (!(upper->wanted_features & feature)
10084 && (features & feature)) {
10085 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
10086 &feature, upper->name);
10087 features &= ~feature;
10088 }
10089 }
10090
10091 return features;
10092}
10093
10094static void netdev_sync_lower_features(struct net_device *upper,
10095 struct net_device *lower, netdev_features_t features)
10096{
10097 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
10098 netdev_features_t feature;
10099 int feature_bit;
10100
10101 for_each_netdev_feature(upper_disables, feature_bit) {
10102 feature = __NETIF_F_BIT(feature_bit);
10103 if (!(features & feature) && (lower->features & feature)) {
10104 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
10105 &feature, lower->name);
10106 lower->wanted_features &= ~feature;
10107 __netdev_update_features(lower);
10108
10109 if (unlikely(lower->features & feature))
10110 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
10111 &feature, lower->name);
10112 else
10113 netdev_features_change(lower);
10114 }
10115 }
10116}
10117
10118static bool netdev_has_ip_or_hw_csum(netdev_features_t features)
10119{
10120 netdev_features_t ip_csum_mask = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
10121 bool ip_csum = (features & ip_csum_mask) == ip_csum_mask;
10122 bool hw_csum = features & NETIF_F_HW_CSUM;
10123
10124 return ip_csum || hw_csum;
10125}
10126
10127static netdev_features_t netdev_fix_features(struct net_device *dev,
10128 netdev_features_t features)
10129{
10130 /* Fix illegal checksum combinations */
10131 if ((features & NETIF_F_HW_CSUM) &&
10132 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
10133 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
10134 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
10135 }
10136
10137 /* TSO requires that SG is present as well. */
10138 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
10139 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
10140 features &= ~NETIF_F_ALL_TSO;
10141 }
10142
10143 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
10144 !(features & NETIF_F_IP_CSUM)) {
10145 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
10146 features &= ~NETIF_F_TSO;
10147 features &= ~NETIF_F_TSO_ECN;
10148 }
10149
10150 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
10151 !(features & NETIF_F_IPV6_CSUM)) {
10152 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
10153 features &= ~NETIF_F_TSO6;
10154 }
10155
10156 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
10157 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
10158 features &= ~NETIF_F_TSO_MANGLEID;
10159
10160 /* TSO ECN requires that TSO is present as well. */
10161 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
10162 features &= ~NETIF_F_TSO_ECN;
10163
10164 /* Software GSO depends on SG. */
10165 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
10166 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
10167 features &= ~NETIF_F_GSO;
10168 }
10169
10170 /* GSO partial features require GSO partial be set */
10171 if ((features & dev->gso_partial_features) &&
10172 !(features & NETIF_F_GSO_PARTIAL)) {
10173 netdev_dbg(dev,
10174 "Dropping partially supported GSO features since no GSO partial.\n");
10175 features &= ~dev->gso_partial_features;
10176 }
10177
10178 if (!(features & NETIF_F_RXCSUM)) {
10179 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
10180 * successfully merged by hardware must also have the
10181 * checksum verified by hardware. If the user does not
10182 * want to enable RXCSUM, logically, we should disable GRO_HW.
10183 */
10184 if (features & NETIF_F_GRO_HW) {
10185 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
10186 features &= ~NETIF_F_GRO_HW;
10187 }
10188 }
10189
10190 /* LRO/HW-GRO features cannot be combined with RX-FCS */
10191 if (features & NETIF_F_RXFCS) {
10192 if (features & NETIF_F_LRO) {
10193 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
10194 features &= ~NETIF_F_LRO;
10195 }
10196
10197 if (features & NETIF_F_GRO_HW) {
10198 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
10199 features &= ~NETIF_F_GRO_HW;
10200 }
10201 }
10202
10203 if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) {
10204 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n");
10205 features &= ~NETIF_F_LRO;
10206 }
10207
10208 if ((features & NETIF_F_HW_TLS_TX) && !netdev_has_ip_or_hw_csum(features)) {
10209 netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
10210 features &= ~NETIF_F_HW_TLS_TX;
10211 }
10212
10213 if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
10214 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
10215 features &= ~NETIF_F_HW_TLS_RX;
10216 }
10217
10218 if ((features & NETIF_F_GSO_UDP_L4) && !netdev_has_ip_or_hw_csum(features)) {
10219 netdev_dbg(dev, "Dropping USO feature since no CSUM feature.\n");
10220 features &= ~NETIF_F_GSO_UDP_L4;
10221 }
10222
10223 return features;
10224}
10225
10226int __netdev_update_features(struct net_device *dev)
10227{
10228 struct net_device *upper, *lower;
10229 netdev_features_t features;
10230 struct list_head *iter;
10231 int err = -1;
10232
10233 ASSERT_RTNL();
10234
10235 features = netdev_get_wanted_features(dev);
10236
10237 if (dev->netdev_ops->ndo_fix_features)
10238 features = dev->netdev_ops->ndo_fix_features(dev, features);
10239
10240 /* driver might be less strict about feature dependencies */
10241 features = netdev_fix_features(dev, features);
10242
10243 /* some features can't be enabled if they're off on an upper device */
10244 netdev_for_each_upper_dev_rcu(dev, upper, iter)
10245 features = netdev_sync_upper_features(dev, upper, features);
10246
10247 if (dev->features == features)
10248 goto sync_lower;
10249
10250 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
10251 &dev->features, &features);
10252
10253 if (dev->netdev_ops->ndo_set_features)
10254 err = dev->netdev_ops->ndo_set_features(dev, features);
10255 else
10256 err = 0;
10257
10258 if (unlikely(err < 0)) {
10259 netdev_err(dev,
10260 "set_features() failed (%d); wanted %pNF, left %pNF\n",
10261 err, &features, &dev->features);
10262 /* return non-0 since some features might have changed and
10263 * it's better to fire a spurious notification than miss it
10264 */
10265 return -1;
10266 }
10267
10268sync_lower:
10269 /* some features must be disabled on lower devices when disabled
10270 * on an upper device (think: bonding master or bridge)
10271 */
10272 netdev_for_each_lower_dev(dev, lower, iter)
10273 netdev_sync_lower_features(dev, lower, features);
10274
10275 if (!err) {
10276 netdev_features_t diff = features ^ dev->features;
10277
10278 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
10279 /* udp_tunnel_{get,drop}_rx_info both need
10280 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
10281 * device, or they won't do anything.
10282 * Thus we need to update dev->features
10283 * *before* calling udp_tunnel_get_rx_info,
10284 * but *after* calling udp_tunnel_drop_rx_info.
10285 */
10286 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
10287 dev->features = features;
10288 udp_tunnel_get_rx_info(dev);
10289 } else {
10290 udp_tunnel_drop_rx_info(dev);
10291 }
10292 }
10293
10294 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
10295 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
10296 dev->features = features;
10297 err |= vlan_get_rx_ctag_filter_info(dev);
10298 } else {
10299 vlan_drop_rx_ctag_filter_info(dev);
10300 }
10301 }
10302
10303 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
10304 if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
10305 dev->features = features;
10306 err |= vlan_get_rx_stag_filter_info(dev);
10307 } else {
10308 vlan_drop_rx_stag_filter_info(dev);
10309 }
10310 }
10311
10312 dev->features = features;
10313 }
10314
10315 return err < 0 ? 0 : 1;
10316}
10317
10318/**
10319 * netdev_update_features - recalculate device features
10320 * @dev: the device to check
10321 *
10322 * Recalculate dev->features set and send notifications if it
10323 * has changed. Should be called after driver or hardware dependent
10324 * conditions might have changed that influence the features.
10325 */
10326void netdev_update_features(struct net_device *dev)
10327{
10328 if (__netdev_update_features(dev))
10329 netdev_features_change(dev);
10330}
10331EXPORT_SYMBOL(netdev_update_features);
10332
10333/**
10334 * netdev_change_features - recalculate device features
10335 * @dev: the device to check
10336 *
10337 * Recalculate dev->features set and send notifications even
10338 * if they have not changed. Should be called instead of
10339 * netdev_update_features() if also dev->vlan_features might
10340 * have changed to allow the changes to be propagated to stacked
10341 * VLAN devices.
10342 */
10343void netdev_change_features(struct net_device *dev)
10344{
10345 __netdev_update_features(dev);
10346 netdev_features_change(dev);
10347}
10348EXPORT_SYMBOL(netdev_change_features);
10349
10350/**
10351 * netif_stacked_transfer_operstate - transfer operstate
10352 * @rootdev: the root or lower level device to transfer state from
10353 * @dev: the device to transfer operstate to
10354 *
10355 * Transfer operational state from root to device. This is normally
10356 * called when a stacking relationship exists between the root
10357 * device and the device(a leaf device).
10358 */
10359void netif_stacked_transfer_operstate(const struct net_device *rootdev,
10360 struct net_device *dev)
10361{
10362 if (rootdev->operstate == IF_OPER_DORMANT)
10363 netif_dormant_on(dev);
10364 else
10365 netif_dormant_off(dev);
10366
10367 if (rootdev->operstate == IF_OPER_TESTING)
10368 netif_testing_on(dev);
10369 else
10370 netif_testing_off(dev);
10371
10372 if (netif_carrier_ok(rootdev))
10373 netif_carrier_on(dev);
10374 else
10375 netif_carrier_off(dev);
10376}
10377EXPORT_SYMBOL(netif_stacked_transfer_operstate);
10378
10379static int netif_alloc_rx_queues(struct net_device *dev)
10380{
10381 unsigned int i, count = dev->num_rx_queues;
10382 struct netdev_rx_queue *rx;
10383 size_t sz = count * sizeof(*rx);
10384 int err = 0;
10385
10386 BUG_ON(count < 1);
10387
10388 rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
10389 if (!rx)
10390 return -ENOMEM;
10391
10392 dev->_rx = rx;
10393
10394 for (i = 0; i < count; i++) {
10395 rx[i].dev = dev;
10396
10397 /* XDP RX-queue setup */
10398 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
10399 if (err < 0)
10400 goto err_rxq_info;
10401 }
10402 return 0;
10403
10404err_rxq_info:
10405 /* Rollback successful reg's and free other resources */
10406 while (i--)
10407 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
10408 kvfree(dev->_rx);
10409 dev->_rx = NULL;
10410 return err;
10411}
10412
10413static void netif_free_rx_queues(struct net_device *dev)
10414{
10415 unsigned int i, count = dev->num_rx_queues;
10416
10417 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
10418 if (!dev->_rx)
10419 return;
10420
10421 for (i = 0; i < count; i++)
10422 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
10423
10424 kvfree(dev->_rx);
10425}
10426
10427static void netdev_init_one_queue(struct net_device *dev,
10428 struct netdev_queue *queue, void *_unused)
10429{
10430 /* Initialize queue lock */
10431 spin_lock_init(&queue->_xmit_lock);
10432 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
10433 queue->xmit_lock_owner = -1;
10434 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
10435 queue->dev = dev;
10436#ifdef CONFIG_BQL
10437 dql_init(&queue->dql, HZ);
10438#endif
10439}
10440
10441static void netif_free_tx_queues(struct net_device *dev)
10442{
10443 kvfree(dev->_tx);
10444}
10445
10446static int netif_alloc_netdev_queues(struct net_device *dev)
10447{
10448 unsigned int count = dev->num_tx_queues;
10449 struct netdev_queue *tx;
10450 size_t sz = count * sizeof(*tx);
10451
10452 if (count < 1 || count > 0xffff)
10453 return -EINVAL;
10454
10455 tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
10456 if (!tx)
10457 return -ENOMEM;
10458
10459 dev->_tx = tx;
10460
10461 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
10462 spin_lock_init(&dev->tx_global_lock);
10463
10464 return 0;
10465}
10466
10467void netif_tx_stop_all_queues(struct net_device *dev)
10468{
10469 unsigned int i;
10470
10471 for (i = 0; i < dev->num_tx_queues; i++) {
10472 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
10473
10474 netif_tx_stop_queue(txq);
10475 }
10476}
10477EXPORT_SYMBOL(netif_tx_stop_all_queues);
10478
10479static int netdev_do_alloc_pcpu_stats(struct net_device *dev)
10480{
10481 void __percpu *v;
10482
10483 /* Drivers implementing ndo_get_peer_dev must support tstat
10484 * accounting, so that skb_do_redirect() can bump the dev's
10485 * RX stats upon network namespace switch.
10486 */
10487 if (dev->netdev_ops->ndo_get_peer_dev &&
10488 dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS)
10489 return -EOPNOTSUPP;
10490
10491 switch (dev->pcpu_stat_type) {
10492 case NETDEV_PCPU_STAT_NONE:
10493 return 0;
10494 case NETDEV_PCPU_STAT_LSTATS:
10495 v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
10496 break;
10497 case NETDEV_PCPU_STAT_TSTATS:
10498 v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
10499 break;
10500 case NETDEV_PCPU_STAT_DSTATS:
10501 v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats);
10502 break;
10503 default:
10504 return -EINVAL;
10505 }
10506
10507 return v ? 0 : -ENOMEM;
10508}
10509
10510static void netdev_do_free_pcpu_stats(struct net_device *dev)
10511{
10512 switch (dev->pcpu_stat_type) {
10513 case NETDEV_PCPU_STAT_NONE:
10514 return;
10515 case NETDEV_PCPU_STAT_LSTATS:
10516 free_percpu(dev->lstats);
10517 break;
10518 case NETDEV_PCPU_STAT_TSTATS:
10519 free_percpu(dev->tstats);
10520 break;
10521 case NETDEV_PCPU_STAT_DSTATS:
10522 free_percpu(dev->dstats);
10523 break;
10524 }
10525}
10526
10527static void netdev_free_phy_link_topology(struct net_device *dev)
10528{
10529 struct phy_link_topology *topo = dev->link_topo;
10530
10531 if (IS_ENABLED(CONFIG_PHYLIB) && topo) {
10532 xa_destroy(&topo->phys);
10533 kfree(topo);
10534 dev->link_topo = NULL;
10535 }
10536}
10537
10538/**
10539 * register_netdevice() - register a network device
10540 * @dev: device to register
10541 *
10542 * Take a prepared network device structure and make it externally accessible.
10543 * A %NETDEV_REGISTER message is sent to the netdev notifier chain.
10544 * Callers must hold the rtnl lock - you may want register_netdev()
10545 * instead of this.
10546 */
10547int register_netdevice(struct net_device *dev)
10548{
10549 int ret;
10550 struct net *net = dev_net(dev);
10551
10552 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
10553 NETDEV_FEATURE_COUNT);
10554 BUG_ON(dev_boot_phase);
10555 ASSERT_RTNL();
10556
10557 might_sleep();
10558
10559 /* When net_device's are persistent, this will be fatal. */
10560 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
10561 BUG_ON(!net);
10562
10563 ret = ethtool_check_ops(dev->ethtool_ops);
10564 if (ret)
10565 return ret;
10566
10567 /* rss ctx ID 0 is reserved for the default context, start from 1 */
10568 xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1);
10569 mutex_init(&dev->ethtool->rss_lock);
10570
10571 spin_lock_init(&dev->addr_list_lock);
10572 netdev_set_addr_lockdep_class(dev);
10573
10574 ret = dev_get_valid_name(net, dev, dev->name);
10575 if (ret < 0)
10576 goto out;
10577
10578 ret = -ENOMEM;
10579 dev->name_node = netdev_name_node_head_alloc(dev);
10580 if (!dev->name_node)
10581 goto out;
10582
10583 /* Init, if this function is available */
10584 if (dev->netdev_ops->ndo_init) {
10585 ret = dev->netdev_ops->ndo_init(dev);
10586 if (ret) {
10587 if (ret > 0)
10588 ret = -EIO;
10589 goto err_free_name;
10590 }
10591 }
10592
10593 if (((dev->hw_features | dev->features) &
10594 NETIF_F_HW_VLAN_CTAG_FILTER) &&
10595 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
10596 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
10597 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
10598 ret = -EINVAL;
10599 goto err_uninit;
10600 }
10601
10602 ret = netdev_do_alloc_pcpu_stats(dev);
10603 if (ret)
10604 goto err_uninit;
10605
10606 ret = dev_index_reserve(net, dev->ifindex);
10607 if (ret < 0)
10608 goto err_free_pcpu;
10609 dev->ifindex = ret;
10610
10611 /* Transfer changeable features to wanted_features and enable
10612 * software offloads (GSO and GRO).
10613 */
10614 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
10615 dev->features |= NETIF_F_SOFT_FEATURES;
10616
10617 if (dev->udp_tunnel_nic_info) {
10618 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10619 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10620 }
10621
10622 dev->wanted_features = dev->features & dev->hw_features;
10623
10624 if (!(dev->flags & IFF_LOOPBACK))
10625 dev->hw_features |= NETIF_F_NOCACHE_COPY;
10626
10627 /* If IPv4 TCP segmentation offload is supported we should also
10628 * allow the device to enable segmenting the frame with the option
10629 * of ignoring a static IP ID value. This doesn't enable the
10630 * feature itself but allows the user to enable it later.
10631 */
10632 if (dev->hw_features & NETIF_F_TSO)
10633 dev->hw_features |= NETIF_F_TSO_MANGLEID;
10634 if (dev->vlan_features & NETIF_F_TSO)
10635 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
10636 if (dev->mpls_features & NETIF_F_TSO)
10637 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
10638 if (dev->hw_enc_features & NETIF_F_TSO)
10639 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
10640
10641 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
10642 */
10643 dev->vlan_features |= NETIF_F_HIGHDMA;
10644
10645 /* Make NETIF_F_SG inheritable to tunnel devices.
10646 */
10647 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
10648
10649 /* Make NETIF_F_SG inheritable to MPLS.
10650 */
10651 dev->mpls_features |= NETIF_F_SG;
10652
10653 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
10654 ret = notifier_to_errno(ret);
10655 if (ret)
10656 goto err_ifindex_release;
10657
10658 ret = netdev_register_kobject(dev);
10659
10660 WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED);
10661
10662 if (ret)
10663 goto err_uninit_notify;
10664
10665 __netdev_update_features(dev);
10666
10667 /*
10668 * Default initial state at registry is that the
10669 * device is present.
10670 */
10671
10672 set_bit(__LINK_STATE_PRESENT, &dev->state);
10673
10674 linkwatch_init_dev(dev);
10675
10676 dev_init_scheduler(dev);
10677
10678 netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL);
10679 list_netdevice(dev);
10680
10681 add_device_randomness(dev->dev_addr, dev->addr_len);
10682
10683 /* If the device has permanent device address, driver should
10684 * set dev_addr and also addr_assign_type should be set to
10685 * NET_ADDR_PERM (default value).
10686 */
10687 if (dev->addr_assign_type == NET_ADDR_PERM)
10688 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10689
10690 /* Notify protocols, that a new device appeared. */
10691 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
10692 ret = notifier_to_errno(ret);
10693 if (ret) {
10694 /* Expect explicit free_netdev() on failure */
10695 dev->needs_free_netdev = false;
10696 unregister_netdevice_queue(dev, NULL);
10697 goto out;
10698 }
10699 /*
10700 * Prevent userspace races by waiting until the network
10701 * device is fully setup before sending notifications.
10702 */
10703 if (!dev->rtnl_link_ops ||
10704 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10705 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
10706
10707out:
10708 return ret;
10709
10710err_uninit_notify:
10711 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
10712err_ifindex_release:
10713 dev_index_release(net, dev->ifindex);
10714err_free_pcpu:
10715 netdev_do_free_pcpu_stats(dev);
10716err_uninit:
10717 if (dev->netdev_ops->ndo_uninit)
10718 dev->netdev_ops->ndo_uninit(dev);
10719 if (dev->priv_destructor)
10720 dev->priv_destructor(dev);
10721err_free_name:
10722 netdev_name_node_free(dev->name_node);
10723 goto out;
10724}
10725EXPORT_SYMBOL(register_netdevice);
10726
10727/* Initialize the core of a dummy net device.
10728 * This is useful if you are calling this function after alloc_netdev(),
10729 * since it does not memset the net_device fields.
10730 */
10731static void init_dummy_netdev_core(struct net_device *dev)
10732{
10733 /* make sure we BUG if trying to hit standard
10734 * register/unregister code path
10735 */
10736 dev->reg_state = NETREG_DUMMY;
10737
10738 /* NAPI wants this */
10739 INIT_LIST_HEAD(&dev->napi_list);
10740
10741 /* a dummy interface is started by default */
10742 set_bit(__LINK_STATE_PRESENT, &dev->state);
10743 set_bit(__LINK_STATE_START, &dev->state);
10744
10745 /* napi_busy_loop stats accounting wants this */
10746 dev_net_set(dev, &init_net);
10747
10748 /* Note : We dont allocate pcpu_refcnt for dummy devices,
10749 * because users of this 'device' dont need to change
10750 * its refcount.
10751 */
10752}
10753
10754/**
10755 * init_dummy_netdev - init a dummy network device for NAPI
10756 * @dev: device to init
10757 *
10758 * This takes a network device structure and initializes the minimum
10759 * amount of fields so it can be used to schedule NAPI polls without
10760 * registering a full blown interface. This is to be used by drivers
10761 * that need to tie several hardware interfaces to a single NAPI
10762 * poll scheduler due to HW limitations.
10763 */
10764void init_dummy_netdev(struct net_device *dev)
10765{
10766 /* Clear everything. Note we don't initialize spinlocks
10767 * as they aren't supposed to be taken by any of the
10768 * NAPI code and this dummy netdev is supposed to be
10769 * only ever used for NAPI polls
10770 */
10771 memset(dev, 0, sizeof(struct net_device));
10772 init_dummy_netdev_core(dev);
10773}
10774EXPORT_SYMBOL_GPL(init_dummy_netdev);
10775
10776/**
10777 * register_netdev - register a network device
10778 * @dev: device to register
10779 *
10780 * Take a completed network device structure and add it to the kernel
10781 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10782 * chain. 0 is returned on success. A negative errno code is returned
10783 * on a failure to set up the device, or if the name is a duplicate.
10784 *
10785 * This is a wrapper around register_netdevice that takes the rtnl semaphore
10786 * and expands the device name if you passed a format string to
10787 * alloc_netdev.
10788 */
10789int register_netdev(struct net_device *dev)
10790{
10791 int err;
10792
10793 if (rtnl_lock_killable())
10794 return -EINTR;
10795 err = register_netdevice(dev);
10796 rtnl_unlock();
10797 return err;
10798}
10799EXPORT_SYMBOL(register_netdev);
10800
10801int netdev_refcnt_read(const struct net_device *dev)
10802{
10803#ifdef CONFIG_PCPU_DEV_REFCNT
10804 int i, refcnt = 0;
10805
10806 for_each_possible_cpu(i)
10807 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10808 return refcnt;
10809#else
10810 return refcount_read(&dev->dev_refcnt);
10811#endif
10812}
10813EXPORT_SYMBOL(netdev_refcnt_read);
10814
10815int netdev_unregister_timeout_secs __read_mostly = 10;
10816
10817#define WAIT_REFS_MIN_MSECS 1
10818#define WAIT_REFS_MAX_MSECS 250
10819/**
10820 * netdev_wait_allrefs_any - wait until all references are gone.
10821 * @list: list of net_devices to wait on
10822 *
10823 * This is called when unregistering network devices.
10824 *
10825 * Any protocol or device that holds a reference should register
10826 * for netdevice notification, and cleanup and put back the
10827 * reference if they receive an UNREGISTER event.
10828 * We can get stuck here if buggy protocols don't correctly
10829 * call dev_put.
10830 */
10831static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
10832{
10833 unsigned long rebroadcast_time, warning_time;
10834 struct net_device *dev;
10835 int wait = 0;
10836
10837 rebroadcast_time = warning_time = jiffies;
10838
10839 list_for_each_entry(dev, list, todo_list)
10840 if (netdev_refcnt_read(dev) == 1)
10841 return dev;
10842
10843 while (true) {
10844 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
10845 rtnl_lock();
10846
10847 /* Rebroadcast unregister notification */
10848 list_for_each_entry(dev, list, todo_list)
10849 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10850
10851 __rtnl_unlock();
10852 rcu_barrier();
10853 rtnl_lock();
10854
10855 list_for_each_entry(dev, list, todo_list)
10856 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
10857 &dev->state)) {
10858 /* We must not have linkwatch events
10859 * pending on unregister. If this
10860 * happens, we simply run the queue
10861 * unscheduled, resulting in a noop
10862 * for this device.
10863 */
10864 linkwatch_run_queue();
10865 break;
10866 }
10867
10868 __rtnl_unlock();
10869
10870 rebroadcast_time = jiffies;
10871 }
10872
10873 rcu_barrier();
10874
10875 if (!wait) {
10876 wait = WAIT_REFS_MIN_MSECS;
10877 } else {
10878 msleep(wait);
10879 wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10880 }
10881
10882 list_for_each_entry(dev, list, todo_list)
10883 if (netdev_refcnt_read(dev) == 1)
10884 return dev;
10885
10886 if (time_after(jiffies, warning_time +
10887 READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
10888 list_for_each_entry(dev, list, todo_list) {
10889 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10890 dev->name, netdev_refcnt_read(dev));
10891 ref_tracker_dir_print(&dev->refcnt_tracker, 10);
10892 }
10893
10894 warning_time = jiffies;
10895 }
10896 }
10897}
10898
10899/* The sequence is:
10900 *
10901 * rtnl_lock();
10902 * ...
10903 * register_netdevice(x1);
10904 * register_netdevice(x2);
10905 * ...
10906 * unregister_netdevice(y1);
10907 * unregister_netdevice(y2);
10908 * ...
10909 * rtnl_unlock();
10910 * free_netdev(y1);
10911 * free_netdev(y2);
10912 *
10913 * We are invoked by rtnl_unlock().
10914 * This allows us to deal with problems:
10915 * 1) We can delete sysfs objects which invoke hotplug
10916 * without deadlocking with linkwatch via keventd.
10917 * 2) Since we run with the RTNL semaphore not held, we can sleep
10918 * safely in order to wait for the netdev refcnt to drop to zero.
10919 *
10920 * We must not return until all unregister events added during
10921 * the interval the lock was held have been completed.
10922 */
10923void netdev_run_todo(void)
10924{
10925 struct net_device *dev, *tmp;
10926 struct list_head list;
10927 int cnt;
10928#ifdef CONFIG_LOCKDEP
10929 struct list_head unlink_list;
10930
10931 list_replace_init(&net_unlink_list, &unlink_list);
10932
10933 while (!list_empty(&unlink_list)) {
10934 struct net_device *dev = list_first_entry(&unlink_list,
10935 struct net_device,
10936 unlink_list);
10937 list_del_init(&dev->unlink_list);
10938 dev->nested_level = dev->lower_level - 1;
10939 }
10940#endif
10941
10942 /* Snapshot list, allow later requests */
10943 list_replace_init(&net_todo_list, &list);
10944
10945 __rtnl_unlock();
10946
10947 /* Wait for rcu callbacks to finish before next phase */
10948 if (!list_empty(&list))
10949 rcu_barrier();
10950
10951 list_for_each_entry_safe(dev, tmp, &list, todo_list) {
10952 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
10953 netdev_WARN(dev, "run_todo but not unregistering\n");
10954 list_del(&dev->todo_list);
10955 continue;
10956 }
10957
10958 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED);
10959 linkwatch_sync_dev(dev);
10960 }
10961
10962 cnt = 0;
10963 while (!list_empty(&list)) {
10964 dev = netdev_wait_allrefs_any(&list);
10965 list_del(&dev->todo_list);
10966
10967 /* paranoia */
10968 BUG_ON(netdev_refcnt_read(dev) != 1);
10969 BUG_ON(!list_empty(&dev->ptype_all));
10970 BUG_ON(!list_empty(&dev->ptype_specific));
10971 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10972 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
10973
10974 netdev_do_free_pcpu_stats(dev);
10975 if (dev->priv_destructor)
10976 dev->priv_destructor(dev);
10977 if (dev->needs_free_netdev)
10978 free_netdev(dev);
10979
10980 cnt++;
10981
10982 /* Free network device */
10983 kobject_put(&dev->dev.kobj);
10984 }
10985 if (cnt && atomic_sub_and_test(cnt, &dev_unreg_count))
10986 wake_up(&netdev_unregistering_wq);
10987}
10988
10989/* Collate per-cpu network dstats statistics
10990 *
10991 * Read per-cpu network statistics from dev->dstats and populate the related
10992 * fields in @s.
10993 */
10994static void dev_fetch_dstats(struct rtnl_link_stats64 *s,
10995 const struct pcpu_dstats __percpu *dstats)
10996{
10997 int cpu;
10998
10999 for_each_possible_cpu(cpu) {
11000 u64 rx_packets, rx_bytes, rx_drops;
11001 u64 tx_packets, tx_bytes, tx_drops;
11002 const struct pcpu_dstats *stats;
11003 unsigned int start;
11004
11005 stats = per_cpu_ptr(dstats, cpu);
11006 do {
11007 start = u64_stats_fetch_begin(&stats->syncp);
11008 rx_packets = u64_stats_read(&stats->rx_packets);
11009 rx_bytes = u64_stats_read(&stats->rx_bytes);
11010 rx_drops = u64_stats_read(&stats->rx_drops);
11011 tx_packets = u64_stats_read(&stats->tx_packets);
11012 tx_bytes = u64_stats_read(&stats->tx_bytes);
11013 tx_drops = u64_stats_read(&stats->tx_drops);
11014 } while (u64_stats_fetch_retry(&stats->syncp, start));
11015
11016 s->rx_packets += rx_packets;
11017 s->rx_bytes += rx_bytes;
11018 s->rx_dropped += rx_drops;
11019 s->tx_packets += tx_packets;
11020 s->tx_bytes += tx_bytes;
11021 s->tx_dropped += tx_drops;
11022 }
11023}
11024
11025/* ndo_get_stats64 implementation for dtstats-based accounting.
11026 *
11027 * Populate @s from dev->stats and dev->dstats. This is used internally by the
11028 * core for NETDEV_PCPU_STAT_DSTAT-type stats collection.
11029 */
11030static void dev_get_dstats64(const struct net_device *dev,
11031 struct rtnl_link_stats64 *s)
11032{
11033 netdev_stats_to_stats64(s, &dev->stats);
11034 dev_fetch_dstats(s, dev->dstats);
11035}
11036
11037/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
11038 * all the same fields in the same order as net_device_stats, with only
11039 * the type differing, but rtnl_link_stats64 may have additional fields
11040 * at the end for newer counters.
11041 */
11042void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
11043 const struct net_device_stats *netdev_stats)
11044{
11045 size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t);
11046 const atomic_long_t *src = (atomic_long_t *)netdev_stats;
11047 u64 *dst = (u64 *)stats64;
11048
11049 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
11050 for (i = 0; i < n; i++)
11051 dst[i] = (unsigned long)atomic_long_read(&src[i]);
11052 /* zero out counters that only exist in rtnl_link_stats64 */
11053 memset((char *)stats64 + n * sizeof(u64), 0,
11054 sizeof(*stats64) - n * sizeof(u64));
11055}
11056EXPORT_SYMBOL(netdev_stats_to_stats64);
11057
11058static __cold struct net_device_core_stats __percpu *netdev_core_stats_alloc(
11059 struct net_device *dev)
11060{
11061 struct net_device_core_stats __percpu *p;
11062
11063 p = alloc_percpu_gfp(struct net_device_core_stats,
11064 GFP_ATOMIC | __GFP_NOWARN);
11065
11066 if (p && cmpxchg(&dev->core_stats, NULL, p))
11067 free_percpu(p);
11068
11069 /* This READ_ONCE() pairs with the cmpxchg() above */
11070 return READ_ONCE(dev->core_stats);
11071}
11072
11073noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset)
11074{
11075 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
11076 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats);
11077 unsigned long __percpu *field;
11078
11079 if (unlikely(!p)) {
11080 p = netdev_core_stats_alloc(dev);
11081 if (!p)
11082 return;
11083 }
11084
11085 field = (unsigned long __percpu *)((void __percpu *)p + offset);
11086 this_cpu_inc(*field);
11087}
11088EXPORT_SYMBOL_GPL(netdev_core_stats_inc);
11089
11090/**
11091 * dev_get_stats - get network device statistics
11092 * @dev: device to get statistics from
11093 * @storage: place to store stats
11094 *
11095 * Get network statistics from device. Return @storage.
11096 * The device driver may provide its own method by setting
11097 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
11098 * otherwise the internal statistics structure is used.
11099 */
11100struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
11101 struct rtnl_link_stats64 *storage)
11102{
11103 const struct net_device_ops *ops = dev->netdev_ops;
11104 const struct net_device_core_stats __percpu *p;
11105
11106 if (ops->ndo_get_stats64) {
11107 memset(storage, 0, sizeof(*storage));
11108 ops->ndo_get_stats64(dev, storage);
11109 } else if (ops->ndo_get_stats) {
11110 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
11111 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) {
11112 dev_get_tstats64(dev, storage);
11113 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) {
11114 dev_get_dstats64(dev, storage);
11115 } else {
11116 netdev_stats_to_stats64(storage, &dev->stats);
11117 }
11118
11119 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
11120 p = READ_ONCE(dev->core_stats);
11121 if (p) {
11122 const struct net_device_core_stats *core_stats;
11123 int i;
11124
11125 for_each_possible_cpu(i) {
11126 core_stats = per_cpu_ptr(p, i);
11127 storage->rx_dropped += READ_ONCE(core_stats->rx_dropped);
11128 storage->tx_dropped += READ_ONCE(core_stats->tx_dropped);
11129 storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler);
11130 storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped);
11131 }
11132 }
11133 return storage;
11134}
11135EXPORT_SYMBOL(dev_get_stats);
11136
11137/**
11138 * dev_fetch_sw_netstats - get per-cpu network device statistics
11139 * @s: place to store stats
11140 * @netstats: per-cpu network stats to read from
11141 *
11142 * Read per-cpu network statistics and populate the related fields in @s.
11143 */
11144void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
11145 const struct pcpu_sw_netstats __percpu *netstats)
11146{
11147 int cpu;
11148
11149 for_each_possible_cpu(cpu) {
11150 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
11151 const struct pcpu_sw_netstats *stats;
11152 unsigned int start;
11153
11154 stats = per_cpu_ptr(netstats, cpu);
11155 do {
11156 start = u64_stats_fetch_begin(&stats->syncp);
11157 rx_packets = u64_stats_read(&stats->rx_packets);
11158 rx_bytes = u64_stats_read(&stats->rx_bytes);
11159 tx_packets = u64_stats_read(&stats->tx_packets);
11160 tx_bytes = u64_stats_read(&stats->tx_bytes);
11161 } while (u64_stats_fetch_retry(&stats->syncp, start));
11162
11163 s->rx_packets += rx_packets;
11164 s->rx_bytes += rx_bytes;
11165 s->tx_packets += tx_packets;
11166 s->tx_bytes += tx_bytes;
11167 }
11168}
11169EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
11170
11171/**
11172 * dev_get_tstats64 - ndo_get_stats64 implementation
11173 * @dev: device to get statistics from
11174 * @s: place to store stats
11175 *
11176 * Populate @s from dev->stats and dev->tstats. Can be used as
11177 * ndo_get_stats64() callback.
11178 */
11179void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
11180{
11181 netdev_stats_to_stats64(s, &dev->stats);
11182 dev_fetch_sw_netstats(s, dev->tstats);
11183}
11184EXPORT_SYMBOL_GPL(dev_get_tstats64);
11185
11186struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
11187{
11188 struct netdev_queue *queue = dev_ingress_queue(dev);
11189
11190#ifdef CONFIG_NET_CLS_ACT
11191 if (queue)
11192 return queue;
11193 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
11194 if (!queue)
11195 return NULL;
11196 netdev_init_one_queue(dev, queue, NULL);
11197 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
11198 RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc);
11199 rcu_assign_pointer(dev->ingress_queue, queue);
11200#endif
11201 return queue;
11202}
11203
11204static const struct ethtool_ops default_ethtool_ops;
11205
11206void netdev_set_default_ethtool_ops(struct net_device *dev,
11207 const struct ethtool_ops *ops)
11208{
11209 if (dev->ethtool_ops == &default_ethtool_ops)
11210 dev->ethtool_ops = ops;
11211}
11212EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
11213
11214/**
11215 * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
11216 * @dev: netdev to enable the IRQ coalescing on
11217 *
11218 * Sets a conservative default for SW IRQ coalescing. Users can use
11219 * sysfs attributes to override the default values.
11220 */
11221void netdev_sw_irq_coalesce_default_on(struct net_device *dev)
11222{
11223 WARN_ON(dev->reg_state == NETREG_REGISTERED);
11224
11225 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
11226 netdev_set_gro_flush_timeout(dev, 20000);
11227 netdev_set_defer_hard_irqs(dev, 1);
11228 }
11229}
11230EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on);
11231
11232/**
11233 * alloc_netdev_mqs - allocate network device
11234 * @sizeof_priv: size of private data to allocate space for
11235 * @name: device name format string
11236 * @name_assign_type: origin of device name
11237 * @setup: callback to initialize device
11238 * @txqs: the number of TX subqueues to allocate
11239 * @rxqs: the number of RX subqueues to allocate
11240 *
11241 * Allocates a struct net_device with private data area for driver use
11242 * and performs basic initialization. Also allocates subqueue structs
11243 * for each queue on the device.
11244 */
11245struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
11246 unsigned char name_assign_type,
11247 void (*setup)(struct net_device *),
11248 unsigned int txqs, unsigned int rxqs)
11249{
11250 struct net_device *dev;
11251 size_t napi_config_sz;
11252 unsigned int maxqs;
11253
11254 BUG_ON(strlen(name) >= sizeof(dev->name));
11255
11256 if (txqs < 1) {
11257 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
11258 return NULL;
11259 }
11260
11261 if (rxqs < 1) {
11262 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
11263 return NULL;
11264 }
11265
11266 maxqs = max(txqs, rxqs);
11267
11268 dev = kvzalloc(struct_size(dev, priv, sizeof_priv),
11269 GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
11270 if (!dev)
11271 return NULL;
11272
11273 dev->priv_len = sizeof_priv;
11274
11275 ref_tracker_dir_init(&dev->refcnt_tracker, 128, name);
11276#ifdef CONFIG_PCPU_DEV_REFCNT
11277 dev->pcpu_refcnt = alloc_percpu(int);
11278 if (!dev->pcpu_refcnt)
11279 goto free_dev;
11280 __dev_hold(dev);
11281#else
11282 refcount_set(&dev->dev_refcnt, 1);
11283#endif
11284
11285 if (dev_addr_init(dev))
11286 goto free_pcpu;
11287
11288 dev_mc_init(dev);
11289 dev_uc_init(dev);
11290
11291 dev_net_set(dev, &init_net);
11292
11293 dev->gso_max_size = GSO_LEGACY_MAX_SIZE;
11294 dev->xdp_zc_max_segs = 1;
11295 dev->gso_max_segs = GSO_MAX_SEGS;
11296 dev->gro_max_size = GRO_LEGACY_MAX_SIZE;
11297 dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE;
11298 dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE;
11299 dev->tso_max_size = TSO_LEGACY_MAX_SIZE;
11300 dev->tso_max_segs = TSO_MAX_SEGS;
11301 dev->upper_level = 1;
11302 dev->lower_level = 1;
11303#ifdef CONFIG_LOCKDEP
11304 dev->nested_level = 0;
11305 INIT_LIST_HEAD(&dev->unlink_list);
11306#endif
11307
11308 INIT_LIST_HEAD(&dev->napi_list);
11309 INIT_LIST_HEAD(&dev->unreg_list);
11310 INIT_LIST_HEAD(&dev->close_list);
11311 INIT_LIST_HEAD(&dev->link_watch_list);
11312 INIT_LIST_HEAD(&dev->adj_list.upper);
11313 INIT_LIST_HEAD(&dev->adj_list.lower);
11314 INIT_LIST_HEAD(&dev->ptype_all);
11315 INIT_LIST_HEAD(&dev->ptype_specific);
11316 INIT_LIST_HEAD(&dev->net_notifier_list);
11317#ifdef CONFIG_NET_SCHED
11318 hash_init(dev->qdisc_hash);
11319#endif
11320
11321 mutex_init(&dev->lock);
11322
11323 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
11324 setup(dev);
11325
11326 if (!dev->tx_queue_len) {
11327 dev->priv_flags |= IFF_NO_QUEUE;
11328 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
11329 }
11330
11331 dev->num_tx_queues = txqs;
11332 dev->real_num_tx_queues = txqs;
11333 if (netif_alloc_netdev_queues(dev))
11334 goto free_all;
11335
11336 dev->num_rx_queues = rxqs;
11337 dev->real_num_rx_queues = rxqs;
11338 if (netif_alloc_rx_queues(dev))
11339 goto free_all;
11340 dev->ethtool = kzalloc(sizeof(*dev->ethtool), GFP_KERNEL_ACCOUNT);
11341 if (!dev->ethtool)
11342 goto free_all;
11343
11344 napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config));
11345 dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT);
11346 if (!dev->napi_config)
11347 goto free_all;
11348
11349 strscpy(dev->name, name);
11350 dev->name_assign_type = name_assign_type;
11351 dev->group = INIT_NETDEV_GROUP;
11352 if (!dev->ethtool_ops)
11353 dev->ethtool_ops = &default_ethtool_ops;
11354
11355 nf_hook_netdev_init(dev);
11356
11357 return dev;
11358
11359free_all:
11360 free_netdev(dev);
11361 return NULL;
11362
11363free_pcpu:
11364#ifdef CONFIG_PCPU_DEV_REFCNT
11365 free_percpu(dev->pcpu_refcnt);
11366free_dev:
11367#endif
11368 kvfree(dev);
11369 return NULL;
11370}
11371EXPORT_SYMBOL(alloc_netdev_mqs);
11372
11373/**
11374 * free_netdev - free network device
11375 * @dev: device
11376 *
11377 * This function does the last stage of destroying an allocated device
11378 * interface. The reference to the device object is released. If this
11379 * is the last reference then it will be freed.Must be called in process
11380 * context.
11381 */
11382void free_netdev(struct net_device *dev)
11383{
11384 struct napi_struct *p, *n;
11385
11386 might_sleep();
11387
11388 /* When called immediately after register_netdevice() failed the unwind
11389 * handling may still be dismantling the device. Handle that case by
11390 * deferring the free.
11391 */
11392 if (dev->reg_state == NETREG_UNREGISTERING) {
11393 ASSERT_RTNL();
11394 dev->needs_free_netdev = true;
11395 return;
11396 }
11397
11398 mutex_destroy(&dev->lock);
11399
11400 kfree(dev->ethtool);
11401 netif_free_tx_queues(dev);
11402 netif_free_rx_queues(dev);
11403
11404 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
11405
11406 /* Flush device addresses */
11407 dev_addr_flush(dev);
11408
11409 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
11410 netif_napi_del(p);
11411
11412 kvfree(dev->napi_config);
11413
11414 ref_tracker_dir_exit(&dev->refcnt_tracker);
11415#ifdef CONFIG_PCPU_DEV_REFCNT
11416 free_percpu(dev->pcpu_refcnt);
11417 dev->pcpu_refcnt = NULL;
11418#endif
11419 free_percpu(dev->core_stats);
11420 dev->core_stats = NULL;
11421 free_percpu(dev->xdp_bulkq);
11422 dev->xdp_bulkq = NULL;
11423
11424 netdev_free_phy_link_topology(dev);
11425
11426 /* Compatibility with error handling in drivers */
11427 if (dev->reg_state == NETREG_UNINITIALIZED ||
11428 dev->reg_state == NETREG_DUMMY) {
11429 kvfree(dev);
11430 return;
11431 }
11432
11433 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
11434 WRITE_ONCE(dev->reg_state, NETREG_RELEASED);
11435
11436 /* will free via device release */
11437 put_device(&dev->dev);
11438}
11439EXPORT_SYMBOL(free_netdev);
11440
11441/**
11442 * alloc_netdev_dummy - Allocate and initialize a dummy net device.
11443 * @sizeof_priv: size of private data to allocate space for
11444 *
11445 * Return: the allocated net_device on success, NULL otherwise
11446 */
11447struct net_device *alloc_netdev_dummy(int sizeof_priv)
11448{
11449 return alloc_netdev(sizeof_priv, "dummy#", NET_NAME_UNKNOWN,
11450 init_dummy_netdev_core);
11451}
11452EXPORT_SYMBOL_GPL(alloc_netdev_dummy);
11453
11454/**
11455 * synchronize_net - Synchronize with packet receive processing
11456 *
11457 * Wait for packets currently being received to be done.
11458 * Does not block later packets from starting.
11459 */
11460void synchronize_net(void)
11461{
11462 might_sleep();
11463 if (rtnl_is_locked())
11464 synchronize_rcu_expedited();
11465 else
11466 synchronize_rcu();
11467}
11468EXPORT_SYMBOL(synchronize_net);
11469
11470static void netdev_rss_contexts_free(struct net_device *dev)
11471{
11472 struct ethtool_rxfh_context *ctx;
11473 unsigned long context;
11474
11475 mutex_lock(&dev->ethtool->rss_lock);
11476 xa_for_each(&dev->ethtool->rss_ctx, context, ctx) {
11477 struct ethtool_rxfh_param rxfh;
11478
11479 rxfh.indir = ethtool_rxfh_context_indir(ctx);
11480 rxfh.key = ethtool_rxfh_context_key(ctx);
11481 rxfh.hfunc = ctx->hfunc;
11482 rxfh.input_xfrm = ctx->input_xfrm;
11483 rxfh.rss_context = context;
11484 rxfh.rss_delete = true;
11485
11486 xa_erase(&dev->ethtool->rss_ctx, context);
11487 if (dev->ethtool_ops->create_rxfh_context)
11488 dev->ethtool_ops->remove_rxfh_context(dev, ctx,
11489 context, NULL);
11490 else
11491 dev->ethtool_ops->set_rxfh(dev, &rxfh, NULL);
11492 kfree(ctx);
11493 }
11494 xa_destroy(&dev->ethtool->rss_ctx);
11495 mutex_unlock(&dev->ethtool->rss_lock);
11496}
11497
11498/**
11499 * unregister_netdevice_queue - remove device from the kernel
11500 * @dev: device
11501 * @head: list
11502 *
11503 * This function shuts down a device interface and removes it
11504 * from the kernel tables.
11505 * If head not NULL, device is queued to be unregistered later.
11506 *
11507 * Callers must hold the rtnl semaphore. You may want
11508 * unregister_netdev() instead of this.
11509 */
11510
11511void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
11512{
11513 ASSERT_RTNL();
11514
11515 if (head) {
11516 list_move_tail(&dev->unreg_list, head);
11517 } else {
11518 LIST_HEAD(single);
11519
11520 list_add(&dev->unreg_list, &single);
11521 unregister_netdevice_many(&single);
11522 }
11523}
11524EXPORT_SYMBOL(unregister_netdevice_queue);
11525
11526void unregister_netdevice_many_notify(struct list_head *head,
11527 u32 portid, const struct nlmsghdr *nlh)
11528{
11529 struct net_device *dev, *tmp;
11530 LIST_HEAD(close_head);
11531 int cnt = 0;
11532
11533 BUG_ON(dev_boot_phase);
11534 ASSERT_RTNL();
11535
11536 if (list_empty(head))
11537 return;
11538
11539 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
11540 /* Some devices call without registering
11541 * for initialization unwind. Remove those
11542 * devices and proceed with the remaining.
11543 */
11544 if (dev->reg_state == NETREG_UNINITIALIZED) {
11545 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
11546 dev->name, dev);
11547
11548 WARN_ON(1);
11549 list_del(&dev->unreg_list);
11550 continue;
11551 }
11552 dev->dismantle = true;
11553 BUG_ON(dev->reg_state != NETREG_REGISTERED);
11554 }
11555
11556 /* If device is running, close it first. */
11557 list_for_each_entry(dev, head, unreg_list)
11558 list_add_tail(&dev->close_list, &close_head);
11559 dev_close_many(&close_head, true);
11560
11561 list_for_each_entry(dev, head, unreg_list) {
11562 /* And unlink it from device chain. */
11563 unlist_netdevice(dev);
11564 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING);
11565 }
11566 flush_all_backlogs();
11567
11568 synchronize_net();
11569
11570 list_for_each_entry(dev, head, unreg_list) {
11571 struct sk_buff *skb = NULL;
11572
11573 /* Shutdown queueing discipline. */
11574 dev_shutdown(dev);
11575 dev_tcx_uninstall(dev);
11576 dev_xdp_uninstall(dev);
11577 bpf_dev_bound_netdev_unregister(dev);
11578 dev_dmabuf_uninstall(dev);
11579
11580 netdev_offload_xstats_disable_all(dev);
11581
11582 /* Notify protocols, that we are about to destroy
11583 * this device. They should clean all the things.
11584 */
11585 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11586
11587 if (!dev->rtnl_link_ops ||
11588 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
11589 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
11590 GFP_KERNEL, NULL, 0,
11591 portid, nlh);
11592
11593 /*
11594 * Flush the unicast and multicast chains
11595 */
11596 dev_uc_flush(dev);
11597 dev_mc_flush(dev);
11598
11599 netdev_name_node_alt_flush(dev);
11600 netdev_name_node_free(dev->name_node);
11601
11602 netdev_rss_contexts_free(dev);
11603
11604 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
11605
11606 if (dev->netdev_ops->ndo_uninit)
11607 dev->netdev_ops->ndo_uninit(dev);
11608
11609 mutex_destroy(&dev->ethtool->rss_lock);
11610
11611 net_shaper_flush_netdev(dev);
11612
11613 if (skb)
11614 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh);
11615
11616 /* Notifier chain MUST detach us all upper devices. */
11617 WARN_ON(netdev_has_any_upper_dev(dev));
11618 WARN_ON(netdev_has_any_lower_dev(dev));
11619
11620 /* Remove entries from kobject tree */
11621 netdev_unregister_kobject(dev);
11622#ifdef CONFIG_XPS
11623 /* Remove XPS queueing entries */
11624 netif_reset_xps_queues_gt(dev, 0);
11625#endif
11626 }
11627
11628 synchronize_net();
11629
11630 list_for_each_entry(dev, head, unreg_list) {
11631 netdev_put(dev, &dev->dev_registered_tracker);
11632 net_set_todo(dev);
11633 cnt++;
11634 }
11635 atomic_add(cnt, &dev_unreg_count);
11636
11637 list_del(head);
11638}
11639
11640/**
11641 * unregister_netdevice_many - unregister many devices
11642 * @head: list of devices
11643 *
11644 * Note: As most callers use a stack allocated list_head,
11645 * we force a list_del() to make sure stack won't be corrupted later.
11646 */
11647void unregister_netdevice_many(struct list_head *head)
11648{
11649 unregister_netdevice_many_notify(head, 0, NULL);
11650}
11651EXPORT_SYMBOL(unregister_netdevice_many);
11652
11653/**
11654 * unregister_netdev - remove device from the kernel
11655 * @dev: device
11656 *
11657 * This function shuts down a device interface and removes it
11658 * from the kernel tables.
11659 *
11660 * This is just a wrapper for unregister_netdevice that takes
11661 * the rtnl semaphore. In general you want to use this and not
11662 * unregister_netdevice.
11663 */
11664void unregister_netdev(struct net_device *dev)
11665{
11666 rtnl_lock();
11667 unregister_netdevice(dev);
11668 rtnl_unlock();
11669}
11670EXPORT_SYMBOL(unregister_netdev);
11671
11672/**
11673 * __dev_change_net_namespace - move device to different nethost namespace
11674 * @dev: device
11675 * @net: network namespace
11676 * @pat: If not NULL name pattern to try if the current device name
11677 * is already taken in the destination network namespace.
11678 * @new_ifindex: If not zero, specifies device index in the target
11679 * namespace.
11680 *
11681 * This function shuts down a device interface and moves it
11682 * to a new network namespace. On success 0 is returned, on
11683 * a failure a netagive errno code is returned.
11684 *
11685 * Callers must hold the rtnl semaphore.
11686 */
11687
11688int __dev_change_net_namespace(struct net_device *dev, struct net *net,
11689 const char *pat, int new_ifindex)
11690{
11691 struct netdev_name_node *name_node;
11692 struct net *net_old = dev_net(dev);
11693 char new_name[IFNAMSIZ] = {};
11694 int err, new_nsid;
11695
11696 ASSERT_RTNL();
11697
11698 /* Don't allow namespace local devices to be moved. */
11699 err = -EINVAL;
11700 if (dev->netns_local)
11701 goto out;
11702
11703 /* Ensure the device has been registered */
11704 if (dev->reg_state != NETREG_REGISTERED)
11705 goto out;
11706
11707 /* Get out if there is nothing todo */
11708 err = 0;
11709 if (net_eq(net_old, net))
11710 goto out;
11711
11712 /* Pick the destination device name, and ensure
11713 * we can use it in the destination network namespace.
11714 */
11715 err = -EEXIST;
11716 if (netdev_name_in_use(net, dev->name)) {
11717 /* We get here if we can't use the current device name */
11718 if (!pat)
11719 goto out;
11720 err = dev_prep_valid_name(net, dev, pat, new_name, EEXIST);
11721 if (err < 0)
11722 goto out;
11723 }
11724 /* Check that none of the altnames conflicts. */
11725 err = -EEXIST;
11726 netdev_for_each_altname(dev, name_node)
11727 if (netdev_name_in_use(net, name_node->name))
11728 goto out;
11729
11730 /* Check that new_ifindex isn't used yet. */
11731 if (new_ifindex) {
11732 err = dev_index_reserve(net, new_ifindex);
11733 if (err < 0)
11734 goto out;
11735 } else {
11736 /* If there is an ifindex conflict assign a new one */
11737 err = dev_index_reserve(net, dev->ifindex);
11738 if (err == -EBUSY)
11739 err = dev_index_reserve(net, 0);
11740 if (err < 0)
11741 goto out;
11742 new_ifindex = err;
11743 }
11744
11745 /*
11746 * And now a mini version of register_netdevice unregister_netdevice.
11747 */
11748
11749 /* If device is running close it first. */
11750 dev_close(dev);
11751
11752 /* And unlink it from device chain */
11753 unlist_netdevice(dev);
11754
11755 synchronize_net();
11756
11757 /* Shutdown queueing discipline. */
11758 dev_shutdown(dev);
11759
11760 /* Notify protocols, that we are about to destroy
11761 * this device. They should clean all the things.
11762 *
11763 * Note that dev->reg_state stays at NETREG_REGISTERED.
11764 * This is wanted because this way 8021q and macvlan know
11765 * the device is just moving and can keep their slaves up.
11766 */
11767 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11768 rcu_barrier();
11769
11770 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
11771
11772 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11773 new_ifindex);
11774
11775 /*
11776 * Flush the unicast and multicast chains
11777 */
11778 dev_uc_flush(dev);
11779 dev_mc_flush(dev);
11780
11781 /* Send a netdev-removed uevent to the old namespace */
11782 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
11783 netdev_adjacent_del_links(dev);
11784
11785 /* Move per-net netdevice notifiers that are following the netdevice */
11786 move_netdevice_notifiers_dev_net(dev, net);
11787
11788 /* Actually switch the network namespace */
11789 dev_net_set(dev, net);
11790 dev->ifindex = new_ifindex;
11791
11792 if (new_name[0]) {
11793 /* Rename the netdev to prepared name */
11794 write_seqlock_bh(&netdev_rename_lock);
11795 strscpy(dev->name, new_name, IFNAMSIZ);
11796 write_sequnlock_bh(&netdev_rename_lock);
11797 }
11798
11799 /* Fixup kobjects */
11800 dev_set_uevent_suppress(&dev->dev, 1);
11801 err = device_rename(&dev->dev, dev->name);
11802 dev_set_uevent_suppress(&dev->dev, 0);
11803 WARN_ON(err);
11804
11805 /* Send a netdev-add uevent to the new namespace */
11806 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
11807 netdev_adjacent_add_links(dev);
11808
11809 /* Adapt owner in case owning user namespace of target network
11810 * namespace is different from the original one.
11811 */
11812 err = netdev_change_owner(dev, net_old, net);
11813 WARN_ON(err);
11814
11815 /* Add the device back in the hashes */
11816 list_netdevice(dev);
11817
11818 /* Notify protocols, that a new device appeared. */
11819 call_netdevice_notifiers(NETDEV_REGISTER, dev);
11820
11821 /*
11822 * Prevent userspace races by waiting until the network
11823 * device is fully setup before sending notifications.
11824 */
11825 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
11826
11827 synchronize_net();
11828 err = 0;
11829out:
11830 return err;
11831}
11832EXPORT_SYMBOL_GPL(__dev_change_net_namespace);
11833
11834static int dev_cpu_dead(unsigned int oldcpu)
11835{
11836 struct sk_buff **list_skb;
11837 struct sk_buff *skb;
11838 unsigned int cpu;
11839 struct softnet_data *sd, *oldsd, *remsd = NULL;
11840
11841 local_irq_disable();
11842 cpu = smp_processor_id();
11843 sd = &per_cpu(softnet_data, cpu);
11844 oldsd = &per_cpu(softnet_data, oldcpu);
11845
11846 /* Find end of our completion_queue. */
11847 list_skb = &sd->completion_queue;
11848 while (*list_skb)
11849 list_skb = &(*list_skb)->next;
11850 /* Append completion queue from offline CPU. */
11851 *list_skb = oldsd->completion_queue;
11852 oldsd->completion_queue = NULL;
11853
11854 /* Append output queue from offline CPU. */
11855 if (oldsd->output_queue) {
11856 *sd->output_queue_tailp = oldsd->output_queue;
11857 sd->output_queue_tailp = oldsd->output_queue_tailp;
11858 oldsd->output_queue = NULL;
11859 oldsd->output_queue_tailp = &oldsd->output_queue;
11860 }
11861 /* Append NAPI poll list from offline CPU, with one exception :
11862 * process_backlog() must be called by cpu owning percpu backlog.
11863 * We properly handle process_queue & input_pkt_queue later.
11864 */
11865 while (!list_empty(&oldsd->poll_list)) {
11866 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11867 struct napi_struct,
11868 poll_list);
11869
11870 list_del_init(&napi->poll_list);
11871 if (napi->poll == process_backlog)
11872 napi->state &= NAPIF_STATE_THREADED;
11873 else
11874 ____napi_schedule(sd, napi);
11875 }
11876
11877 raise_softirq_irqoff(NET_TX_SOFTIRQ);
11878 local_irq_enable();
11879
11880 if (!use_backlog_threads()) {
11881#ifdef CONFIG_RPS
11882 remsd = oldsd->rps_ipi_list;
11883 oldsd->rps_ipi_list = NULL;
11884#endif
11885 /* send out pending IPI's on offline CPU */
11886 net_rps_send_ipi(remsd);
11887 }
11888
11889 /* Process offline CPU's input_pkt_queue */
11890 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
11891 netif_rx(skb);
11892 rps_input_queue_head_incr(oldsd);
11893 }
11894 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
11895 netif_rx(skb);
11896 rps_input_queue_head_incr(oldsd);
11897 }
11898
11899 return 0;
11900}
11901
11902/**
11903 * netdev_increment_features - increment feature set by one
11904 * @all: current feature set
11905 * @one: new feature set
11906 * @mask: mask feature set
11907 *
11908 * Computes a new feature set after adding a device with feature set
11909 * @one to the master device with current feature set @all. Will not
11910 * enable anything that is off in @mask. Returns the new feature set.
11911 */
11912netdev_features_t netdev_increment_features(netdev_features_t all,
11913 netdev_features_t one, netdev_features_t mask)
11914{
11915 if (mask & NETIF_F_HW_CSUM)
11916 mask |= NETIF_F_CSUM_MASK;
11917 mask |= NETIF_F_VLAN_CHALLENGED;
11918
11919 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
11920 all &= one | ~NETIF_F_ALL_FOR_ALL;
11921
11922 /* If one device supports hw checksumming, set for all. */
11923 if (all & NETIF_F_HW_CSUM)
11924 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
11925
11926 return all;
11927}
11928EXPORT_SYMBOL(netdev_increment_features);
11929
11930static struct hlist_head * __net_init netdev_create_hash(void)
11931{
11932 int i;
11933 struct hlist_head *hash;
11934
11935 hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
11936 if (hash != NULL)
11937 for (i = 0; i < NETDEV_HASHENTRIES; i++)
11938 INIT_HLIST_HEAD(&hash[i]);
11939
11940 return hash;
11941}
11942
11943/* Initialize per network namespace state */
11944static int __net_init netdev_init(struct net *net)
11945{
11946 BUILD_BUG_ON(GRO_HASH_BUCKETS >
11947 8 * sizeof_field(struct napi_struct, gro_bitmask));
11948
11949 INIT_LIST_HEAD(&net->dev_base_head);
11950
11951 net->dev_name_head = netdev_create_hash();
11952 if (net->dev_name_head == NULL)
11953 goto err_name;
11954
11955 net->dev_index_head = netdev_create_hash();
11956 if (net->dev_index_head == NULL)
11957 goto err_idx;
11958
11959 xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1);
11960
11961 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
11962
11963 return 0;
11964
11965err_idx:
11966 kfree(net->dev_name_head);
11967err_name:
11968 return -ENOMEM;
11969}
11970
11971/**
11972 * netdev_drivername - network driver for the device
11973 * @dev: network device
11974 *
11975 * Determine network driver for device.
11976 */
11977const char *netdev_drivername(const struct net_device *dev)
11978{
11979 const struct device_driver *driver;
11980 const struct device *parent;
11981 const char *empty = "";
11982
11983 parent = dev->dev.parent;
11984 if (!parent)
11985 return empty;
11986
11987 driver = parent->driver;
11988 if (driver && driver->name)
11989 return driver->name;
11990 return empty;
11991}
11992
11993static void __netdev_printk(const char *level, const struct net_device *dev,
11994 struct va_format *vaf)
11995{
11996 if (dev && dev->dev.parent) {
11997 dev_printk_emit(level[1] - '0',
11998 dev->dev.parent,
11999 "%s %s %s%s: %pV",
12000 dev_driver_string(dev->dev.parent),
12001 dev_name(dev->dev.parent),
12002 netdev_name(dev), netdev_reg_state(dev),
12003 vaf);
12004 } else if (dev) {
12005 printk("%s%s%s: %pV",
12006 level, netdev_name(dev), netdev_reg_state(dev), vaf);
12007 } else {
12008 printk("%s(NULL net_device): %pV", level, vaf);
12009 }
12010}
12011
12012void netdev_printk(const char *level, const struct net_device *dev,
12013 const char *format, ...)
12014{
12015 struct va_format vaf;
12016 va_list args;
12017
12018 va_start(args, format);
12019
12020 vaf.fmt = format;
12021 vaf.va = &args;
12022
12023 __netdev_printk(level, dev, &vaf);
12024
12025 va_end(args);
12026}
12027EXPORT_SYMBOL(netdev_printk);
12028
12029#define define_netdev_printk_level(func, level) \
12030void func(const struct net_device *dev, const char *fmt, ...) \
12031{ \
12032 struct va_format vaf; \
12033 va_list args; \
12034 \
12035 va_start(args, fmt); \
12036 \
12037 vaf.fmt = fmt; \
12038 vaf.va = &args; \
12039 \
12040 __netdev_printk(level, dev, &vaf); \
12041 \
12042 va_end(args); \
12043} \
12044EXPORT_SYMBOL(func);
12045
12046define_netdev_printk_level(netdev_emerg, KERN_EMERG);
12047define_netdev_printk_level(netdev_alert, KERN_ALERT);
12048define_netdev_printk_level(netdev_crit, KERN_CRIT);
12049define_netdev_printk_level(netdev_err, KERN_ERR);
12050define_netdev_printk_level(netdev_warn, KERN_WARNING);
12051define_netdev_printk_level(netdev_notice, KERN_NOTICE);
12052define_netdev_printk_level(netdev_info, KERN_INFO);
12053
12054static void __net_exit netdev_exit(struct net *net)
12055{
12056 kfree(net->dev_name_head);
12057 kfree(net->dev_index_head);
12058 xa_destroy(&net->dev_by_index);
12059 if (net != &init_net)
12060 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
12061}
12062
12063static struct pernet_operations __net_initdata netdev_net_ops = {
12064 .init = netdev_init,
12065 .exit = netdev_exit,
12066};
12067
12068static void __net_exit default_device_exit_net(struct net *net)
12069{
12070 struct netdev_name_node *name_node, *tmp;
12071 struct net_device *dev, *aux;
12072 /*
12073 * Push all migratable network devices back to the
12074 * initial network namespace
12075 */
12076 ASSERT_RTNL();
12077 for_each_netdev_safe(net, dev, aux) {
12078 int err;
12079 char fb_name[IFNAMSIZ];
12080
12081 /* Ignore unmoveable devices (i.e. loopback) */
12082 if (dev->netns_local)
12083 continue;
12084
12085 /* Leave virtual devices for the generic cleanup */
12086 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
12087 continue;
12088
12089 /* Push remaining network devices to init_net */
12090 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
12091 if (netdev_name_in_use(&init_net, fb_name))
12092 snprintf(fb_name, IFNAMSIZ, "dev%%d");
12093
12094 netdev_for_each_altname_safe(dev, name_node, tmp)
12095 if (netdev_name_in_use(&init_net, name_node->name))
12096 __netdev_name_node_alt_destroy(name_node);
12097
12098 err = dev_change_net_namespace(dev, &init_net, fb_name);
12099 if (err) {
12100 pr_emerg("%s: failed to move %s to init_net: %d\n",
12101 __func__, dev->name, err);
12102 BUG();
12103 }
12104 }
12105}
12106
12107static void __net_exit default_device_exit_batch(struct list_head *net_list)
12108{
12109 /* At exit all network devices most be removed from a network
12110 * namespace. Do this in the reverse order of registration.
12111 * Do this across as many network namespaces as possible to
12112 * improve batching efficiency.
12113 */
12114 struct net_device *dev;
12115 struct net *net;
12116 LIST_HEAD(dev_kill_list);
12117
12118 rtnl_lock();
12119 list_for_each_entry(net, net_list, exit_list) {
12120 default_device_exit_net(net);
12121 cond_resched();
12122 }
12123
12124 list_for_each_entry(net, net_list, exit_list) {
12125 for_each_netdev_reverse(net, dev) {
12126 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
12127 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
12128 else
12129 unregister_netdevice_queue(dev, &dev_kill_list);
12130 }
12131 }
12132 unregister_netdevice_many(&dev_kill_list);
12133 rtnl_unlock();
12134}
12135
12136static struct pernet_operations __net_initdata default_device_ops = {
12137 .exit_batch = default_device_exit_batch,
12138};
12139
12140static void __init net_dev_struct_check(void)
12141{
12142 /* TX read-mostly hotpath */
12143 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags_fast);
12144 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops);
12145 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops);
12146 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx);
12147 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, real_num_tx_queues);
12148 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size);
12149 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size);
12150 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs);
12151 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_partial_features);
12152 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc);
12153 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu);
12154 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom);
12155 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tc_to_txq);
12156#ifdef CONFIG_XPS
12157 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, xps_maps);
12158#endif
12159#ifdef CONFIG_NETFILTER_EGRESS
12160 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, nf_hooks_egress);
12161#endif
12162#ifdef CONFIG_NET_XGRESS
12163 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress);
12164#endif
12165 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 160);
12166
12167 /* TXRX read-mostly hotpath */
12168 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, lstats);
12169 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, state);
12170 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags);
12171 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len);
12172 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features);
12173 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr);
12174 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 46);
12175
12176 /* RX read-mostly hotpath */
12177 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific);
12178 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ifindex);
12179 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, real_num_rx_queues);
12180 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, _rx);
12181 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_max_size);
12182 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_ipv4_max_size);
12183 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler);
12184 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler_data);
12185 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, nd_net);
12186#ifdef CONFIG_NETPOLL
12187 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, npinfo);
12188#endif
12189#ifdef CONFIG_NET_XGRESS
12190 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress);
12191#endif
12192 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 92);
12193}
12194
12195/*
12196 * Initialize the DEV module. At boot time this walks the device list and
12197 * unhooks any devices that fail to initialise (normally hardware not
12198 * present) and leaves us with a valid list of present and active devices.
12199 *
12200 */
12201
12202/* We allocate 256 pages for each CPU if PAGE_SHIFT is 12 */
12203#define SYSTEM_PERCPU_PAGE_POOL_SIZE ((1 << 20) / PAGE_SIZE)
12204
12205static int net_page_pool_create(int cpuid)
12206{
12207#if IS_ENABLED(CONFIG_PAGE_POOL)
12208 struct page_pool_params page_pool_params = {
12209 .pool_size = SYSTEM_PERCPU_PAGE_POOL_SIZE,
12210 .flags = PP_FLAG_SYSTEM_POOL,
12211 .nid = cpu_to_mem(cpuid),
12212 };
12213 struct page_pool *pp_ptr;
12214
12215 pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid);
12216 if (IS_ERR(pp_ptr))
12217 return -ENOMEM;
12218
12219 per_cpu(system_page_pool, cpuid) = pp_ptr;
12220#endif
12221 return 0;
12222}
12223
12224static int backlog_napi_should_run(unsigned int cpu)
12225{
12226 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
12227 struct napi_struct *napi = &sd->backlog;
12228
12229 return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
12230}
12231
12232static void run_backlog_napi(unsigned int cpu)
12233{
12234 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
12235
12236 napi_threaded_poll_loop(&sd->backlog);
12237}
12238
12239static void backlog_napi_setup(unsigned int cpu)
12240{
12241 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
12242 struct napi_struct *napi = &sd->backlog;
12243
12244 napi->thread = this_cpu_read(backlog_napi);
12245 set_bit(NAPI_STATE_THREADED, &napi->state);
12246}
12247
12248static struct smp_hotplug_thread backlog_threads = {
12249 .store = &backlog_napi,
12250 .thread_should_run = backlog_napi_should_run,
12251 .thread_fn = run_backlog_napi,
12252 .thread_comm = "backlog_napi/%u",
12253 .setup = backlog_napi_setup,
12254};
12255
12256/*
12257 * This is called single threaded during boot, so no need
12258 * to take the rtnl semaphore.
12259 */
12260static int __init net_dev_init(void)
12261{
12262 int i, rc = -ENOMEM;
12263
12264 BUG_ON(!dev_boot_phase);
12265
12266 net_dev_struct_check();
12267
12268 if (dev_proc_init())
12269 goto out;
12270
12271 if (netdev_kobject_init())
12272 goto out;
12273
12274 for (i = 0; i < PTYPE_HASH_SIZE; i++)
12275 INIT_LIST_HEAD(&ptype_base[i]);
12276
12277 if (register_pernet_subsys(&netdev_net_ops))
12278 goto out;
12279
12280 /*
12281 * Initialise the packet receive queues.
12282 */
12283
12284 for_each_possible_cpu(i) {
12285 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
12286 struct softnet_data *sd = &per_cpu(softnet_data, i);
12287
12288 INIT_WORK(flush, flush_backlog);
12289
12290 skb_queue_head_init(&sd->input_pkt_queue);
12291 skb_queue_head_init(&sd->process_queue);
12292#ifdef CONFIG_XFRM_OFFLOAD
12293 skb_queue_head_init(&sd->xfrm_backlog);
12294#endif
12295 INIT_LIST_HEAD(&sd->poll_list);
12296 sd->output_queue_tailp = &sd->output_queue;
12297#ifdef CONFIG_RPS
12298 INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
12299 sd->cpu = i;
12300#endif
12301 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
12302 spin_lock_init(&sd->defer_lock);
12303
12304 init_gro_hash(&sd->backlog);
12305 sd->backlog.poll = process_backlog;
12306 sd->backlog.weight = weight_p;
12307 INIT_LIST_HEAD(&sd->backlog.poll_list);
12308
12309 if (net_page_pool_create(i))
12310 goto out;
12311 }
12312 if (use_backlog_threads())
12313 smpboot_register_percpu_thread(&backlog_threads);
12314
12315 dev_boot_phase = 0;
12316
12317 /* The loopback device is special if any other network devices
12318 * is present in a network namespace the loopback device must
12319 * be present. Since we now dynamically allocate and free the
12320 * loopback device ensure this invariant is maintained by
12321 * keeping the loopback device as the first device on the
12322 * list of network devices. Ensuring the loopback devices
12323 * is the first device that appears and the last network device
12324 * that disappears.
12325 */
12326 if (register_pernet_device(&loopback_net_ops))
12327 goto out;
12328
12329 if (register_pernet_device(&default_device_ops))
12330 goto out;
12331
12332 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
12333 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
12334
12335 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
12336 NULL, dev_cpu_dead);
12337 WARN_ON(rc < 0);
12338 rc = 0;
12339
12340 /* avoid static key IPIs to isolated CPUs */
12341 if (housekeeping_enabled(HK_TYPE_MISC))
12342 net_enable_timestamp();
12343out:
12344 if (rc < 0) {
12345 for_each_possible_cpu(i) {
12346 struct page_pool *pp_ptr;
12347
12348 pp_ptr = per_cpu(system_page_pool, i);
12349 if (!pp_ptr)
12350 continue;
12351
12352 page_pool_destroy(pp_ptr);
12353 per_cpu(system_page_pool, i) = NULL;
12354 }
12355 }
12356
12357 return rc;
12358}
12359
12360subsys_initcall(net_dev_init);
1/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <asm/uaccess.h>
76#include <asm/system.h>
77#include <linux/bitops.h>
78#include <linux/capability.h>
79#include <linux/cpu.h>
80#include <linux/types.h>
81#include <linux/kernel.h>
82#include <linux/hash.h>
83#include <linux/slab.h>
84#include <linux/sched.h>
85#include <linux/mutex.h>
86#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
95#include <linux/ethtool.h>
96#include <linux/notifier.h>
97#include <linux/skbuff.h>
98#include <net/net_namespace.h>
99#include <net/sock.h>
100#include <linux/rtnetlink.h>
101#include <linux/proc_fs.h>
102#include <linux/seq_file.h>
103#include <linux/stat.h>
104#include <net/dst.h>
105#include <net/pkt_sched.h>
106#include <net/checksum.h>
107#include <net/xfrm.h>
108#include <linux/highmem.h>
109#include <linux/init.h>
110#include <linux/kmod.h>
111#include <linux/module.h>
112#include <linux/netpoll.h>
113#include <linux/rcupdate.h>
114#include <linux/delay.h>
115#include <net/wext.h>
116#include <net/iw_handler.h>
117#include <asm/current.h>
118#include <linux/audit.h>
119#include <linux/dmaengine.h>
120#include <linux/err.h>
121#include <linux/ctype.h>
122#include <linux/if_arp.h>
123#include <linux/if_vlan.h>
124#include <linux/ip.h>
125#include <net/ip.h>
126#include <linux/ipv6.h>
127#include <linux/in.h>
128#include <linux/jhash.h>
129#include <linux/random.h>
130#include <trace/events/napi.h>
131#include <trace/events/net.h>
132#include <trace/events/skb.h>
133#include <linux/pci.h>
134#include <linux/inetdevice.h>
135#include <linux/cpu_rmap.h>
136
137#include "net-sysfs.h"
138
139/* Instead of increasing this, you should create a hash table. */
140#define MAX_GRO_SKBS 8
141
142/* This should be increased if a protocol with a bigger head is added. */
143#define GRO_MAX_HEAD (MAX_HEADER + 128)
144
145/*
146 * The list of packet types we will receive (as opposed to discard)
147 * and the routines to invoke.
148 *
149 * Why 16. Because with 16 the only overlap we get on a hash of the
150 * low nibble of the protocol value is RARP/SNAP/X.25.
151 *
152 * NOTE: That is no longer true with the addition of VLAN tags. Not
153 * sure which should go first, but I bet it won't make much
154 * difference if we are running VLANs. The good news is that
155 * this protocol won't be in the list unless compiled in, so
156 * the average user (w/out VLANs) will not be adversely affected.
157 * --BLG
158 *
159 * 0800 IP
160 * 8100 802.1Q VLAN
161 * 0001 802.3
162 * 0002 AX.25
163 * 0004 802.2
164 * 8035 RARP
165 * 0005 SNAP
166 * 0805 X.25
167 * 0806 ARP
168 * 8137 IPX
169 * 0009 Localtalk
170 * 86DD IPv6
171 */
172
173#define PTYPE_HASH_SIZE (16)
174#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
175
176static DEFINE_SPINLOCK(ptype_lock);
177static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
178static struct list_head ptype_all __read_mostly; /* Taps */
179
180/*
181 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
182 * semaphore.
183 *
184 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
185 *
186 * Writers must hold the rtnl semaphore while they loop through the
187 * dev_base_head list, and hold dev_base_lock for writing when they do the
188 * actual updates. This allows pure readers to access the list even
189 * while a writer is preparing to update it.
190 *
191 * To put it another way, dev_base_lock is held for writing only to
192 * protect against pure readers; the rtnl semaphore provides the
193 * protection against other writers.
194 *
195 * See, for example usages, register_netdevice() and
196 * unregister_netdevice(), which must be called with the rtnl
197 * semaphore held.
198 */
199DEFINE_RWLOCK(dev_base_lock);
200EXPORT_SYMBOL(dev_base_lock);
201
202static inline void dev_base_seq_inc(struct net *net)
203{
204 while (++net->dev_base_seq == 0);
205}
206
207static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
208{
209 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
210 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
211}
212
213static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
214{
215 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
216}
217
218static inline void rps_lock(struct softnet_data *sd)
219{
220#ifdef CONFIG_RPS
221 spin_lock(&sd->input_pkt_queue.lock);
222#endif
223}
224
225static inline void rps_unlock(struct softnet_data *sd)
226{
227#ifdef CONFIG_RPS
228 spin_unlock(&sd->input_pkt_queue.lock);
229#endif
230}
231
232/* Device list insertion */
233static int list_netdevice(struct net_device *dev)
234{
235 struct net *net = dev_net(dev);
236
237 ASSERT_RTNL();
238
239 write_lock_bh(&dev_base_lock);
240 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
241 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
242 hlist_add_head_rcu(&dev->index_hlist,
243 dev_index_hash(net, dev->ifindex));
244 write_unlock_bh(&dev_base_lock);
245
246 dev_base_seq_inc(net);
247
248 return 0;
249}
250
251/* Device list removal
252 * caller must respect a RCU grace period before freeing/reusing dev
253 */
254static void unlist_netdevice(struct net_device *dev)
255{
256 ASSERT_RTNL();
257
258 /* Unlink dev from the device chain */
259 write_lock_bh(&dev_base_lock);
260 list_del_rcu(&dev->dev_list);
261 hlist_del_rcu(&dev->name_hlist);
262 hlist_del_rcu(&dev->index_hlist);
263 write_unlock_bh(&dev_base_lock);
264
265 dev_base_seq_inc(dev_net(dev));
266}
267
268/*
269 * Our notifier list
270 */
271
272static RAW_NOTIFIER_HEAD(netdev_chain);
273
274/*
275 * Device drivers call our routines to queue packets here. We empty the
276 * queue in the local softnet handler.
277 */
278
279DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
280EXPORT_PER_CPU_SYMBOL(softnet_data);
281
282#ifdef CONFIG_LOCKDEP
283/*
284 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
285 * according to dev->type
286 */
287static const unsigned short netdev_lock_type[] =
288 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
289 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
290 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
291 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
292 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
293 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
294 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
295 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
296 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
297 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
298 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
299 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
300 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
301 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
302 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
303 ARPHRD_VOID, ARPHRD_NONE};
304
305static const char *const netdev_lock_name[] =
306 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
307 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
308 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
309 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
310 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
311 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
312 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
313 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
314 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
315 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
316 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
317 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
318 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
319 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
320 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
321 "_xmit_VOID", "_xmit_NONE"};
322
323static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
324static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
325
326static inline unsigned short netdev_lock_pos(unsigned short dev_type)
327{
328 int i;
329
330 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
331 if (netdev_lock_type[i] == dev_type)
332 return i;
333 /* the last key is used by default */
334 return ARRAY_SIZE(netdev_lock_type) - 1;
335}
336
337static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
338 unsigned short dev_type)
339{
340 int i;
341
342 i = netdev_lock_pos(dev_type);
343 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
344 netdev_lock_name[i]);
345}
346
347static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
348{
349 int i;
350
351 i = netdev_lock_pos(dev->type);
352 lockdep_set_class_and_name(&dev->addr_list_lock,
353 &netdev_addr_lock_key[i],
354 netdev_lock_name[i]);
355}
356#else
357static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
358 unsigned short dev_type)
359{
360}
361static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
362{
363}
364#endif
365
366/*******************************************************************************
367
368 Protocol management and registration routines
369
370*******************************************************************************/
371
372/*
373 * Add a protocol ID to the list. Now that the input handler is
374 * smarter we can dispense with all the messy stuff that used to be
375 * here.
376 *
377 * BEWARE!!! Protocol handlers, mangling input packets,
378 * MUST BE last in hash buckets and checking protocol handlers
379 * MUST start from promiscuous ptype_all chain in net_bh.
380 * It is true now, do not change it.
381 * Explanation follows: if protocol handler, mangling packet, will
382 * be the first on list, it is not able to sense, that packet
383 * is cloned and should be copied-on-write, so that it will
384 * change it and subsequent readers will get broken packet.
385 * --ANK (980803)
386 */
387
388static inline struct list_head *ptype_head(const struct packet_type *pt)
389{
390 if (pt->type == htons(ETH_P_ALL))
391 return &ptype_all;
392 else
393 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
394}
395
396/**
397 * dev_add_pack - add packet handler
398 * @pt: packet type declaration
399 *
400 * Add a protocol handler to the networking stack. The passed &packet_type
401 * is linked into kernel lists and may not be freed until it has been
402 * removed from the kernel lists.
403 *
404 * This call does not sleep therefore it can not
405 * guarantee all CPU's that are in middle of receiving packets
406 * will see the new packet type (until the next received packet).
407 */
408
409void dev_add_pack(struct packet_type *pt)
410{
411 struct list_head *head = ptype_head(pt);
412
413 spin_lock(&ptype_lock);
414 list_add_rcu(&pt->list, head);
415 spin_unlock(&ptype_lock);
416}
417EXPORT_SYMBOL(dev_add_pack);
418
419/**
420 * __dev_remove_pack - remove packet handler
421 * @pt: packet type declaration
422 *
423 * Remove a protocol handler that was previously added to the kernel
424 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
425 * from the kernel lists and can be freed or reused once this function
426 * returns.
427 *
428 * The packet type might still be in use by receivers
429 * and must not be freed until after all the CPU's have gone
430 * through a quiescent state.
431 */
432void __dev_remove_pack(struct packet_type *pt)
433{
434 struct list_head *head = ptype_head(pt);
435 struct packet_type *pt1;
436
437 spin_lock(&ptype_lock);
438
439 list_for_each_entry(pt1, head, list) {
440 if (pt == pt1) {
441 list_del_rcu(&pt->list);
442 goto out;
443 }
444 }
445
446 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
447out:
448 spin_unlock(&ptype_lock);
449}
450EXPORT_SYMBOL(__dev_remove_pack);
451
452/**
453 * dev_remove_pack - remove packet handler
454 * @pt: packet type declaration
455 *
456 * Remove a protocol handler that was previously added to the kernel
457 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
458 * from the kernel lists and can be freed or reused once this function
459 * returns.
460 *
461 * This call sleeps to guarantee that no CPU is looking at the packet
462 * type after return.
463 */
464void dev_remove_pack(struct packet_type *pt)
465{
466 __dev_remove_pack(pt);
467
468 synchronize_net();
469}
470EXPORT_SYMBOL(dev_remove_pack);
471
472/******************************************************************************
473
474 Device Boot-time Settings Routines
475
476*******************************************************************************/
477
478/* Boot time configuration table */
479static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
480
481/**
482 * netdev_boot_setup_add - add new setup entry
483 * @name: name of the device
484 * @map: configured settings for the device
485 *
486 * Adds new setup entry to the dev_boot_setup list. The function
487 * returns 0 on error and 1 on success. This is a generic routine to
488 * all netdevices.
489 */
490static int netdev_boot_setup_add(char *name, struct ifmap *map)
491{
492 struct netdev_boot_setup *s;
493 int i;
494
495 s = dev_boot_setup;
496 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
497 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
498 memset(s[i].name, 0, sizeof(s[i].name));
499 strlcpy(s[i].name, name, IFNAMSIZ);
500 memcpy(&s[i].map, map, sizeof(s[i].map));
501 break;
502 }
503 }
504
505 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
506}
507
508/**
509 * netdev_boot_setup_check - check boot time settings
510 * @dev: the netdevice
511 *
512 * Check boot time settings for the device.
513 * The found settings are set for the device to be used
514 * later in the device probing.
515 * Returns 0 if no settings found, 1 if they are.
516 */
517int netdev_boot_setup_check(struct net_device *dev)
518{
519 struct netdev_boot_setup *s = dev_boot_setup;
520 int i;
521
522 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
523 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
524 !strcmp(dev->name, s[i].name)) {
525 dev->irq = s[i].map.irq;
526 dev->base_addr = s[i].map.base_addr;
527 dev->mem_start = s[i].map.mem_start;
528 dev->mem_end = s[i].map.mem_end;
529 return 1;
530 }
531 }
532 return 0;
533}
534EXPORT_SYMBOL(netdev_boot_setup_check);
535
536
537/**
538 * netdev_boot_base - get address from boot time settings
539 * @prefix: prefix for network device
540 * @unit: id for network device
541 *
542 * Check boot time settings for the base address of device.
543 * The found settings are set for the device to be used
544 * later in the device probing.
545 * Returns 0 if no settings found.
546 */
547unsigned long netdev_boot_base(const char *prefix, int unit)
548{
549 const struct netdev_boot_setup *s = dev_boot_setup;
550 char name[IFNAMSIZ];
551 int i;
552
553 sprintf(name, "%s%d", prefix, unit);
554
555 /*
556 * If device already registered then return base of 1
557 * to indicate not to probe for this interface
558 */
559 if (__dev_get_by_name(&init_net, name))
560 return 1;
561
562 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
563 if (!strcmp(name, s[i].name))
564 return s[i].map.base_addr;
565 return 0;
566}
567
568/*
569 * Saves at boot time configured settings for any netdevice.
570 */
571int __init netdev_boot_setup(char *str)
572{
573 int ints[5];
574 struct ifmap map;
575
576 str = get_options(str, ARRAY_SIZE(ints), ints);
577 if (!str || !*str)
578 return 0;
579
580 /* Save settings */
581 memset(&map, 0, sizeof(map));
582 if (ints[0] > 0)
583 map.irq = ints[1];
584 if (ints[0] > 1)
585 map.base_addr = ints[2];
586 if (ints[0] > 2)
587 map.mem_start = ints[3];
588 if (ints[0] > 3)
589 map.mem_end = ints[4];
590
591 /* Add new entry to the list */
592 return netdev_boot_setup_add(str, &map);
593}
594
595__setup("netdev=", netdev_boot_setup);
596
597/*******************************************************************************
598
599 Device Interface Subroutines
600
601*******************************************************************************/
602
603/**
604 * __dev_get_by_name - find a device by its name
605 * @net: the applicable net namespace
606 * @name: name to find
607 *
608 * Find an interface by name. Must be called under RTNL semaphore
609 * or @dev_base_lock. If the name is found a pointer to the device
610 * is returned. If the name is not found then %NULL is returned. The
611 * reference counters are not incremented so the caller must be
612 * careful with locks.
613 */
614
615struct net_device *__dev_get_by_name(struct net *net, const char *name)
616{
617 struct hlist_node *p;
618 struct net_device *dev;
619 struct hlist_head *head = dev_name_hash(net, name);
620
621 hlist_for_each_entry(dev, p, head, name_hlist)
622 if (!strncmp(dev->name, name, IFNAMSIZ))
623 return dev;
624
625 return NULL;
626}
627EXPORT_SYMBOL(__dev_get_by_name);
628
629/**
630 * dev_get_by_name_rcu - find a device by its name
631 * @net: the applicable net namespace
632 * @name: name to find
633 *
634 * Find an interface by name.
635 * If the name is found a pointer to the device is returned.
636 * If the name is not found then %NULL is returned.
637 * The reference counters are not incremented so the caller must be
638 * careful with locks. The caller must hold RCU lock.
639 */
640
641struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
642{
643 struct hlist_node *p;
644 struct net_device *dev;
645 struct hlist_head *head = dev_name_hash(net, name);
646
647 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
648 if (!strncmp(dev->name, name, IFNAMSIZ))
649 return dev;
650
651 return NULL;
652}
653EXPORT_SYMBOL(dev_get_by_name_rcu);
654
655/**
656 * dev_get_by_name - find a device by its name
657 * @net: the applicable net namespace
658 * @name: name to find
659 *
660 * Find an interface by name. This can be called from any
661 * context and does its own locking. The returned handle has
662 * the usage count incremented and the caller must use dev_put() to
663 * release it when it is no longer needed. %NULL is returned if no
664 * matching device is found.
665 */
666
667struct net_device *dev_get_by_name(struct net *net, const char *name)
668{
669 struct net_device *dev;
670
671 rcu_read_lock();
672 dev = dev_get_by_name_rcu(net, name);
673 if (dev)
674 dev_hold(dev);
675 rcu_read_unlock();
676 return dev;
677}
678EXPORT_SYMBOL(dev_get_by_name);
679
680/**
681 * __dev_get_by_index - find a device by its ifindex
682 * @net: the applicable net namespace
683 * @ifindex: index of device
684 *
685 * Search for an interface by index. Returns %NULL if the device
686 * is not found or a pointer to the device. The device has not
687 * had its reference counter increased so the caller must be careful
688 * about locking. The caller must hold either the RTNL semaphore
689 * or @dev_base_lock.
690 */
691
692struct net_device *__dev_get_by_index(struct net *net, int ifindex)
693{
694 struct hlist_node *p;
695 struct net_device *dev;
696 struct hlist_head *head = dev_index_hash(net, ifindex);
697
698 hlist_for_each_entry(dev, p, head, index_hlist)
699 if (dev->ifindex == ifindex)
700 return dev;
701
702 return NULL;
703}
704EXPORT_SYMBOL(__dev_get_by_index);
705
706/**
707 * dev_get_by_index_rcu - find a device by its ifindex
708 * @net: the applicable net namespace
709 * @ifindex: index of device
710 *
711 * Search for an interface by index. Returns %NULL if the device
712 * is not found or a pointer to the device. The device has not
713 * had its reference counter increased so the caller must be careful
714 * about locking. The caller must hold RCU lock.
715 */
716
717struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
718{
719 struct hlist_node *p;
720 struct net_device *dev;
721 struct hlist_head *head = dev_index_hash(net, ifindex);
722
723 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
724 if (dev->ifindex == ifindex)
725 return dev;
726
727 return NULL;
728}
729EXPORT_SYMBOL(dev_get_by_index_rcu);
730
731
732/**
733 * dev_get_by_index - find a device by its ifindex
734 * @net: the applicable net namespace
735 * @ifindex: index of device
736 *
737 * Search for an interface by index. Returns NULL if the device
738 * is not found or a pointer to the device. The device returned has
739 * had a reference added and the pointer is safe until the user calls
740 * dev_put to indicate they have finished with it.
741 */
742
743struct net_device *dev_get_by_index(struct net *net, int ifindex)
744{
745 struct net_device *dev;
746
747 rcu_read_lock();
748 dev = dev_get_by_index_rcu(net, ifindex);
749 if (dev)
750 dev_hold(dev);
751 rcu_read_unlock();
752 return dev;
753}
754EXPORT_SYMBOL(dev_get_by_index);
755
756/**
757 * dev_getbyhwaddr_rcu - find a device by its hardware address
758 * @net: the applicable net namespace
759 * @type: media type of device
760 * @ha: hardware address
761 *
762 * Search for an interface by MAC address. Returns NULL if the device
763 * is not found or a pointer to the device.
764 * The caller must hold RCU or RTNL.
765 * The returned device has not had its ref count increased
766 * and the caller must therefore be careful about locking
767 *
768 */
769
770struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
771 const char *ha)
772{
773 struct net_device *dev;
774
775 for_each_netdev_rcu(net, dev)
776 if (dev->type == type &&
777 !memcmp(dev->dev_addr, ha, dev->addr_len))
778 return dev;
779
780 return NULL;
781}
782EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
783
784struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
785{
786 struct net_device *dev;
787
788 ASSERT_RTNL();
789 for_each_netdev(net, dev)
790 if (dev->type == type)
791 return dev;
792
793 return NULL;
794}
795EXPORT_SYMBOL(__dev_getfirstbyhwtype);
796
797struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
798{
799 struct net_device *dev, *ret = NULL;
800
801 rcu_read_lock();
802 for_each_netdev_rcu(net, dev)
803 if (dev->type == type) {
804 dev_hold(dev);
805 ret = dev;
806 break;
807 }
808 rcu_read_unlock();
809 return ret;
810}
811EXPORT_SYMBOL(dev_getfirstbyhwtype);
812
813/**
814 * dev_get_by_flags_rcu - find any device with given flags
815 * @net: the applicable net namespace
816 * @if_flags: IFF_* values
817 * @mask: bitmask of bits in if_flags to check
818 *
819 * Search for any interface with the given flags. Returns NULL if a device
820 * is not found or a pointer to the device. Must be called inside
821 * rcu_read_lock(), and result refcount is unchanged.
822 */
823
824struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
825 unsigned short mask)
826{
827 struct net_device *dev, *ret;
828
829 ret = NULL;
830 for_each_netdev_rcu(net, dev) {
831 if (((dev->flags ^ if_flags) & mask) == 0) {
832 ret = dev;
833 break;
834 }
835 }
836 return ret;
837}
838EXPORT_SYMBOL(dev_get_by_flags_rcu);
839
840/**
841 * dev_valid_name - check if name is okay for network device
842 * @name: name string
843 *
844 * Network device names need to be valid file names to
845 * to allow sysfs to work. We also disallow any kind of
846 * whitespace.
847 */
848int dev_valid_name(const char *name)
849{
850 if (*name == '\0')
851 return 0;
852 if (strlen(name) >= IFNAMSIZ)
853 return 0;
854 if (!strcmp(name, ".") || !strcmp(name, ".."))
855 return 0;
856
857 while (*name) {
858 if (*name == '/' || isspace(*name))
859 return 0;
860 name++;
861 }
862 return 1;
863}
864EXPORT_SYMBOL(dev_valid_name);
865
866/**
867 * __dev_alloc_name - allocate a name for a device
868 * @net: network namespace to allocate the device name in
869 * @name: name format string
870 * @buf: scratch buffer and result name string
871 *
872 * Passed a format string - eg "lt%d" it will try and find a suitable
873 * id. It scans list of devices to build up a free map, then chooses
874 * the first empty slot. The caller must hold the dev_base or rtnl lock
875 * while allocating the name and adding the device in order to avoid
876 * duplicates.
877 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
878 * Returns the number of the unit assigned or a negative errno code.
879 */
880
881static int __dev_alloc_name(struct net *net, const char *name, char *buf)
882{
883 int i = 0;
884 const char *p;
885 const int max_netdevices = 8*PAGE_SIZE;
886 unsigned long *inuse;
887 struct net_device *d;
888
889 p = strnchr(name, IFNAMSIZ-1, '%');
890 if (p) {
891 /*
892 * Verify the string as this thing may have come from
893 * the user. There must be either one "%d" and no other "%"
894 * characters.
895 */
896 if (p[1] != 'd' || strchr(p + 2, '%'))
897 return -EINVAL;
898
899 /* Use one page as a bit array of possible slots */
900 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
901 if (!inuse)
902 return -ENOMEM;
903
904 for_each_netdev(net, d) {
905 if (!sscanf(d->name, name, &i))
906 continue;
907 if (i < 0 || i >= max_netdevices)
908 continue;
909
910 /* avoid cases where sscanf is not exact inverse of printf */
911 snprintf(buf, IFNAMSIZ, name, i);
912 if (!strncmp(buf, d->name, IFNAMSIZ))
913 set_bit(i, inuse);
914 }
915
916 i = find_first_zero_bit(inuse, max_netdevices);
917 free_page((unsigned long) inuse);
918 }
919
920 if (buf != name)
921 snprintf(buf, IFNAMSIZ, name, i);
922 if (!__dev_get_by_name(net, buf))
923 return i;
924
925 /* It is possible to run out of possible slots
926 * when the name is long and there isn't enough space left
927 * for the digits, or if all bits are used.
928 */
929 return -ENFILE;
930}
931
932/**
933 * dev_alloc_name - allocate a name for a device
934 * @dev: device
935 * @name: name format string
936 *
937 * Passed a format string - eg "lt%d" it will try and find a suitable
938 * id. It scans list of devices to build up a free map, then chooses
939 * the first empty slot. The caller must hold the dev_base or rtnl lock
940 * while allocating the name and adding the device in order to avoid
941 * duplicates.
942 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
943 * Returns the number of the unit assigned or a negative errno code.
944 */
945
946int dev_alloc_name(struct net_device *dev, const char *name)
947{
948 char buf[IFNAMSIZ];
949 struct net *net;
950 int ret;
951
952 BUG_ON(!dev_net(dev));
953 net = dev_net(dev);
954 ret = __dev_alloc_name(net, name, buf);
955 if (ret >= 0)
956 strlcpy(dev->name, buf, IFNAMSIZ);
957 return ret;
958}
959EXPORT_SYMBOL(dev_alloc_name);
960
961static int dev_get_valid_name(struct net_device *dev, const char *name)
962{
963 struct net *net;
964
965 BUG_ON(!dev_net(dev));
966 net = dev_net(dev);
967
968 if (!dev_valid_name(name))
969 return -EINVAL;
970
971 if (strchr(name, '%'))
972 return dev_alloc_name(dev, name);
973 else if (__dev_get_by_name(net, name))
974 return -EEXIST;
975 else if (dev->name != name)
976 strlcpy(dev->name, name, IFNAMSIZ);
977
978 return 0;
979}
980
981/**
982 * dev_change_name - change name of a device
983 * @dev: device
984 * @newname: name (or format string) must be at least IFNAMSIZ
985 *
986 * Change name of a device, can pass format strings "eth%d".
987 * for wildcarding.
988 */
989int dev_change_name(struct net_device *dev, const char *newname)
990{
991 char oldname[IFNAMSIZ];
992 int err = 0;
993 int ret;
994 struct net *net;
995
996 ASSERT_RTNL();
997 BUG_ON(!dev_net(dev));
998
999 net = dev_net(dev);
1000 if (dev->flags & IFF_UP)
1001 return -EBUSY;
1002
1003 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
1004 return 0;
1005
1006 memcpy(oldname, dev->name, IFNAMSIZ);
1007
1008 err = dev_get_valid_name(dev, newname);
1009 if (err < 0)
1010 return err;
1011
1012rollback:
1013 ret = device_rename(&dev->dev, dev->name);
1014 if (ret) {
1015 memcpy(dev->name, oldname, IFNAMSIZ);
1016 return ret;
1017 }
1018
1019 write_lock_bh(&dev_base_lock);
1020 hlist_del_rcu(&dev->name_hlist);
1021 write_unlock_bh(&dev_base_lock);
1022
1023 synchronize_rcu();
1024
1025 write_lock_bh(&dev_base_lock);
1026 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1027 write_unlock_bh(&dev_base_lock);
1028
1029 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1030 ret = notifier_to_errno(ret);
1031
1032 if (ret) {
1033 /* err >= 0 after dev_alloc_name() or stores the first errno */
1034 if (err >= 0) {
1035 err = ret;
1036 memcpy(dev->name, oldname, IFNAMSIZ);
1037 goto rollback;
1038 } else {
1039 printk(KERN_ERR
1040 "%s: name change rollback failed: %d.\n",
1041 dev->name, ret);
1042 }
1043 }
1044
1045 return err;
1046}
1047
1048/**
1049 * dev_set_alias - change ifalias of a device
1050 * @dev: device
1051 * @alias: name up to IFALIASZ
1052 * @len: limit of bytes to copy from info
1053 *
1054 * Set ifalias for a device,
1055 */
1056int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1057{
1058 ASSERT_RTNL();
1059
1060 if (len >= IFALIASZ)
1061 return -EINVAL;
1062
1063 if (!len) {
1064 if (dev->ifalias) {
1065 kfree(dev->ifalias);
1066 dev->ifalias = NULL;
1067 }
1068 return 0;
1069 }
1070
1071 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1072 if (!dev->ifalias)
1073 return -ENOMEM;
1074
1075 strlcpy(dev->ifalias, alias, len+1);
1076 return len;
1077}
1078
1079
1080/**
1081 * netdev_features_change - device changes features
1082 * @dev: device to cause notification
1083 *
1084 * Called to indicate a device has changed features.
1085 */
1086void netdev_features_change(struct net_device *dev)
1087{
1088 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1089}
1090EXPORT_SYMBOL(netdev_features_change);
1091
1092/**
1093 * netdev_state_change - device changes state
1094 * @dev: device to cause notification
1095 *
1096 * Called to indicate a device has changed state. This function calls
1097 * the notifier chains for netdev_chain and sends a NEWLINK message
1098 * to the routing socket.
1099 */
1100void netdev_state_change(struct net_device *dev)
1101{
1102 if (dev->flags & IFF_UP) {
1103 call_netdevice_notifiers(NETDEV_CHANGE, dev);
1104 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1105 }
1106}
1107EXPORT_SYMBOL(netdev_state_change);
1108
1109int netdev_bonding_change(struct net_device *dev, unsigned long event)
1110{
1111 return call_netdevice_notifiers(event, dev);
1112}
1113EXPORT_SYMBOL(netdev_bonding_change);
1114
1115/**
1116 * dev_load - load a network module
1117 * @net: the applicable net namespace
1118 * @name: name of interface
1119 *
1120 * If a network interface is not present and the process has suitable
1121 * privileges this function loads the module. If module loading is not
1122 * available in this kernel then it becomes a nop.
1123 */
1124
1125void dev_load(struct net *net, const char *name)
1126{
1127 struct net_device *dev;
1128 int no_module;
1129
1130 rcu_read_lock();
1131 dev = dev_get_by_name_rcu(net, name);
1132 rcu_read_unlock();
1133
1134 no_module = !dev;
1135 if (no_module && capable(CAP_NET_ADMIN))
1136 no_module = request_module("netdev-%s", name);
1137 if (no_module && capable(CAP_SYS_MODULE)) {
1138 if (!request_module("%s", name))
1139 pr_err("Loading kernel module for a network device "
1140"with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%s "
1141"instead\n", name);
1142 }
1143}
1144EXPORT_SYMBOL(dev_load);
1145
1146static int __dev_open(struct net_device *dev)
1147{
1148 const struct net_device_ops *ops = dev->netdev_ops;
1149 int ret;
1150
1151 ASSERT_RTNL();
1152
1153 if (!netif_device_present(dev))
1154 return -ENODEV;
1155
1156 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1157 ret = notifier_to_errno(ret);
1158 if (ret)
1159 return ret;
1160
1161 set_bit(__LINK_STATE_START, &dev->state);
1162
1163 if (ops->ndo_validate_addr)
1164 ret = ops->ndo_validate_addr(dev);
1165
1166 if (!ret && ops->ndo_open)
1167 ret = ops->ndo_open(dev);
1168
1169 if (ret)
1170 clear_bit(__LINK_STATE_START, &dev->state);
1171 else {
1172 dev->flags |= IFF_UP;
1173 net_dmaengine_get();
1174 dev_set_rx_mode(dev);
1175 dev_activate(dev);
1176 }
1177
1178 return ret;
1179}
1180
1181/**
1182 * dev_open - prepare an interface for use.
1183 * @dev: device to open
1184 *
1185 * Takes a device from down to up state. The device's private open
1186 * function is invoked and then the multicast lists are loaded. Finally
1187 * the device is moved into the up state and a %NETDEV_UP message is
1188 * sent to the netdev notifier chain.
1189 *
1190 * Calling this function on an active interface is a nop. On a failure
1191 * a negative errno code is returned.
1192 */
1193int dev_open(struct net_device *dev)
1194{
1195 int ret;
1196
1197 if (dev->flags & IFF_UP)
1198 return 0;
1199
1200 ret = __dev_open(dev);
1201 if (ret < 0)
1202 return ret;
1203
1204 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1205 call_netdevice_notifiers(NETDEV_UP, dev);
1206
1207 return ret;
1208}
1209EXPORT_SYMBOL(dev_open);
1210
1211static int __dev_close_many(struct list_head *head)
1212{
1213 struct net_device *dev;
1214
1215 ASSERT_RTNL();
1216 might_sleep();
1217
1218 list_for_each_entry(dev, head, unreg_list) {
1219 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1220
1221 clear_bit(__LINK_STATE_START, &dev->state);
1222
1223 /* Synchronize to scheduled poll. We cannot touch poll list, it
1224 * can be even on different cpu. So just clear netif_running().
1225 *
1226 * dev->stop() will invoke napi_disable() on all of it's
1227 * napi_struct instances on this device.
1228 */
1229 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1230 }
1231
1232 dev_deactivate_many(head);
1233
1234 list_for_each_entry(dev, head, unreg_list) {
1235 const struct net_device_ops *ops = dev->netdev_ops;
1236
1237 /*
1238 * Call the device specific close. This cannot fail.
1239 * Only if device is UP
1240 *
1241 * We allow it to be called even after a DETACH hot-plug
1242 * event.
1243 */
1244 if (ops->ndo_stop)
1245 ops->ndo_stop(dev);
1246
1247 dev->flags &= ~IFF_UP;
1248 net_dmaengine_put();
1249 }
1250
1251 return 0;
1252}
1253
1254static int __dev_close(struct net_device *dev)
1255{
1256 int retval;
1257 LIST_HEAD(single);
1258
1259 list_add(&dev->unreg_list, &single);
1260 retval = __dev_close_many(&single);
1261 list_del(&single);
1262 return retval;
1263}
1264
1265static int dev_close_many(struct list_head *head)
1266{
1267 struct net_device *dev, *tmp;
1268 LIST_HEAD(tmp_list);
1269
1270 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1271 if (!(dev->flags & IFF_UP))
1272 list_move(&dev->unreg_list, &tmp_list);
1273
1274 __dev_close_many(head);
1275
1276 list_for_each_entry(dev, head, unreg_list) {
1277 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1278 call_netdevice_notifiers(NETDEV_DOWN, dev);
1279 }
1280
1281 /* rollback_registered_many needs the complete original list */
1282 list_splice(&tmp_list, head);
1283 return 0;
1284}
1285
1286/**
1287 * dev_close - shutdown an interface.
1288 * @dev: device to shutdown
1289 *
1290 * This function moves an active device into down state. A
1291 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1292 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1293 * chain.
1294 */
1295int dev_close(struct net_device *dev)
1296{
1297 if (dev->flags & IFF_UP) {
1298 LIST_HEAD(single);
1299
1300 list_add(&dev->unreg_list, &single);
1301 dev_close_many(&single);
1302 list_del(&single);
1303 }
1304 return 0;
1305}
1306EXPORT_SYMBOL(dev_close);
1307
1308
1309/**
1310 * dev_disable_lro - disable Large Receive Offload on a device
1311 * @dev: device
1312 *
1313 * Disable Large Receive Offload (LRO) on a net device. Must be
1314 * called under RTNL. This is needed if received packets may be
1315 * forwarded to another interface.
1316 */
1317void dev_disable_lro(struct net_device *dev)
1318{
1319 u32 flags;
1320
1321 /*
1322 * If we're trying to disable lro on a vlan device
1323 * use the underlying physical device instead
1324 */
1325 if (is_vlan_dev(dev))
1326 dev = vlan_dev_real_dev(dev);
1327
1328 if (dev->ethtool_ops && dev->ethtool_ops->get_flags)
1329 flags = dev->ethtool_ops->get_flags(dev);
1330 else
1331 flags = ethtool_op_get_flags(dev);
1332
1333 if (!(flags & ETH_FLAG_LRO))
1334 return;
1335
1336 __ethtool_set_flags(dev, flags & ~ETH_FLAG_LRO);
1337 if (unlikely(dev->features & NETIF_F_LRO))
1338 netdev_WARN(dev, "failed to disable LRO!\n");
1339}
1340EXPORT_SYMBOL(dev_disable_lro);
1341
1342
1343static int dev_boot_phase = 1;
1344
1345/**
1346 * register_netdevice_notifier - register a network notifier block
1347 * @nb: notifier
1348 *
1349 * Register a notifier to be called when network device events occur.
1350 * The notifier passed is linked into the kernel structures and must
1351 * not be reused until it has been unregistered. A negative errno code
1352 * is returned on a failure.
1353 *
1354 * When registered all registration and up events are replayed
1355 * to the new notifier to allow device to have a race free
1356 * view of the network device list.
1357 */
1358
1359int register_netdevice_notifier(struct notifier_block *nb)
1360{
1361 struct net_device *dev;
1362 struct net_device *last;
1363 struct net *net;
1364 int err;
1365
1366 rtnl_lock();
1367 err = raw_notifier_chain_register(&netdev_chain, nb);
1368 if (err)
1369 goto unlock;
1370 if (dev_boot_phase)
1371 goto unlock;
1372 for_each_net(net) {
1373 for_each_netdev(net, dev) {
1374 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1375 err = notifier_to_errno(err);
1376 if (err)
1377 goto rollback;
1378
1379 if (!(dev->flags & IFF_UP))
1380 continue;
1381
1382 nb->notifier_call(nb, NETDEV_UP, dev);
1383 }
1384 }
1385
1386unlock:
1387 rtnl_unlock();
1388 return err;
1389
1390rollback:
1391 last = dev;
1392 for_each_net(net) {
1393 for_each_netdev(net, dev) {
1394 if (dev == last)
1395 break;
1396
1397 if (dev->flags & IFF_UP) {
1398 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1399 nb->notifier_call(nb, NETDEV_DOWN, dev);
1400 }
1401 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1402 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
1403 }
1404 }
1405
1406 raw_notifier_chain_unregister(&netdev_chain, nb);
1407 goto unlock;
1408}
1409EXPORT_SYMBOL(register_netdevice_notifier);
1410
1411/**
1412 * unregister_netdevice_notifier - unregister a network notifier block
1413 * @nb: notifier
1414 *
1415 * Unregister a notifier previously registered by
1416 * register_netdevice_notifier(). The notifier is unlinked into the
1417 * kernel structures and may then be reused. A negative errno code
1418 * is returned on a failure.
1419 */
1420
1421int unregister_netdevice_notifier(struct notifier_block *nb)
1422{
1423 int err;
1424
1425 rtnl_lock();
1426 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1427 rtnl_unlock();
1428 return err;
1429}
1430EXPORT_SYMBOL(unregister_netdevice_notifier);
1431
1432/**
1433 * call_netdevice_notifiers - call all network notifier blocks
1434 * @val: value passed unmodified to notifier function
1435 * @dev: net_device pointer passed unmodified to notifier function
1436 *
1437 * Call all network notifier blocks. Parameters and return value
1438 * are as for raw_notifier_call_chain().
1439 */
1440
1441int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1442{
1443 ASSERT_RTNL();
1444 return raw_notifier_call_chain(&netdev_chain, val, dev);
1445}
1446EXPORT_SYMBOL(call_netdevice_notifiers);
1447
1448/* When > 0 there are consumers of rx skb time stamps */
1449static atomic_t netstamp_needed = ATOMIC_INIT(0);
1450
1451void net_enable_timestamp(void)
1452{
1453 atomic_inc(&netstamp_needed);
1454}
1455EXPORT_SYMBOL(net_enable_timestamp);
1456
1457void net_disable_timestamp(void)
1458{
1459 atomic_dec(&netstamp_needed);
1460}
1461EXPORT_SYMBOL(net_disable_timestamp);
1462
1463static inline void net_timestamp_set(struct sk_buff *skb)
1464{
1465 if (atomic_read(&netstamp_needed))
1466 __net_timestamp(skb);
1467 else
1468 skb->tstamp.tv64 = 0;
1469}
1470
1471static inline void net_timestamp_check(struct sk_buff *skb)
1472{
1473 if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))
1474 __net_timestamp(skb);
1475}
1476
1477static inline bool is_skb_forwardable(struct net_device *dev,
1478 struct sk_buff *skb)
1479{
1480 unsigned int len;
1481
1482 if (!(dev->flags & IFF_UP))
1483 return false;
1484
1485 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1486 if (skb->len <= len)
1487 return true;
1488
1489 /* if TSO is enabled, we don't care about the length as the packet
1490 * could be forwarded without being segmented before
1491 */
1492 if (skb_is_gso(skb))
1493 return true;
1494
1495 return false;
1496}
1497
1498/**
1499 * dev_forward_skb - loopback an skb to another netif
1500 *
1501 * @dev: destination network device
1502 * @skb: buffer to forward
1503 *
1504 * return values:
1505 * NET_RX_SUCCESS (no congestion)
1506 * NET_RX_DROP (packet was dropped, but freed)
1507 *
1508 * dev_forward_skb can be used for injecting an skb from the
1509 * start_xmit function of one device into the receive queue
1510 * of another device.
1511 *
1512 * The receiving device may be in another namespace, so
1513 * we have to clear all information in the skb that could
1514 * impact namespace isolation.
1515 */
1516int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1517{
1518 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
1519 if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
1520 atomic_long_inc(&dev->rx_dropped);
1521 kfree_skb(skb);
1522 return NET_RX_DROP;
1523 }
1524 }
1525
1526 skb_orphan(skb);
1527 nf_reset(skb);
1528
1529 if (unlikely(!is_skb_forwardable(dev, skb))) {
1530 atomic_long_inc(&dev->rx_dropped);
1531 kfree_skb(skb);
1532 return NET_RX_DROP;
1533 }
1534 skb_set_dev(skb, dev);
1535 skb->tstamp.tv64 = 0;
1536 skb->pkt_type = PACKET_HOST;
1537 skb->protocol = eth_type_trans(skb, dev);
1538 return netif_rx(skb);
1539}
1540EXPORT_SYMBOL_GPL(dev_forward_skb);
1541
1542static inline int deliver_skb(struct sk_buff *skb,
1543 struct packet_type *pt_prev,
1544 struct net_device *orig_dev)
1545{
1546 atomic_inc(&skb->users);
1547 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1548}
1549
1550/*
1551 * Support routine. Sends outgoing frames to any network
1552 * taps currently in use.
1553 */
1554
1555static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1556{
1557 struct packet_type *ptype;
1558 struct sk_buff *skb2 = NULL;
1559 struct packet_type *pt_prev = NULL;
1560
1561 rcu_read_lock();
1562 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1563 /* Never send packets back to the socket
1564 * they originated from - MvS (miquels@drinkel.ow.org)
1565 */
1566 if ((ptype->dev == dev || !ptype->dev) &&
1567 (ptype->af_packet_priv == NULL ||
1568 (struct sock *)ptype->af_packet_priv != skb->sk)) {
1569 if (pt_prev) {
1570 deliver_skb(skb2, pt_prev, skb->dev);
1571 pt_prev = ptype;
1572 continue;
1573 }
1574
1575 skb2 = skb_clone(skb, GFP_ATOMIC);
1576 if (!skb2)
1577 break;
1578
1579 net_timestamp_set(skb2);
1580
1581 /* skb->nh should be correctly
1582 set by sender, so that the second statement is
1583 just protection against buggy protocols.
1584 */
1585 skb_reset_mac_header(skb2);
1586
1587 if (skb_network_header(skb2) < skb2->data ||
1588 skb2->network_header > skb2->tail) {
1589 if (net_ratelimit())
1590 printk(KERN_CRIT "protocol %04x is "
1591 "buggy, dev %s\n",
1592 ntohs(skb2->protocol),
1593 dev->name);
1594 skb_reset_network_header(skb2);
1595 }
1596
1597 skb2->transport_header = skb2->network_header;
1598 skb2->pkt_type = PACKET_OUTGOING;
1599 pt_prev = ptype;
1600 }
1601 }
1602 if (pt_prev)
1603 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1604 rcu_read_unlock();
1605}
1606
1607/* netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1608 * @dev: Network device
1609 * @txq: number of queues available
1610 *
1611 * If real_num_tx_queues is changed the tc mappings may no longer be
1612 * valid. To resolve this verify the tc mapping remains valid and if
1613 * not NULL the mapping. With no priorities mapping to this
1614 * offset/count pair it will no longer be used. In the worst case TC0
1615 * is invalid nothing can be done so disable priority mappings. If is
1616 * expected that drivers will fix this mapping if they can before
1617 * calling netif_set_real_num_tx_queues.
1618 */
1619static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1620{
1621 int i;
1622 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1623
1624 /* If TC0 is invalidated disable TC mapping */
1625 if (tc->offset + tc->count > txq) {
1626 pr_warning("Number of in use tx queues changed "
1627 "invalidating tc mappings. Priority "
1628 "traffic classification disabled!\n");
1629 dev->num_tc = 0;
1630 return;
1631 }
1632
1633 /* Invalidated prio to tc mappings set to TC0 */
1634 for (i = 1; i < TC_BITMASK + 1; i++) {
1635 int q = netdev_get_prio_tc_map(dev, i);
1636
1637 tc = &dev->tc_to_txq[q];
1638 if (tc->offset + tc->count > txq) {
1639 pr_warning("Number of in use tx queues "
1640 "changed. Priority %i to tc "
1641 "mapping %i is no longer valid "
1642 "setting map to 0\n",
1643 i, q);
1644 netdev_set_prio_tc_map(dev, i, 0);
1645 }
1646 }
1647}
1648
1649/*
1650 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1651 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1652 */
1653int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
1654{
1655 int rc;
1656
1657 if (txq < 1 || txq > dev->num_tx_queues)
1658 return -EINVAL;
1659
1660 if (dev->reg_state == NETREG_REGISTERED ||
1661 dev->reg_state == NETREG_UNREGISTERING) {
1662 ASSERT_RTNL();
1663
1664 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1665 txq);
1666 if (rc)
1667 return rc;
1668
1669 if (dev->num_tc)
1670 netif_setup_tc(dev, txq);
1671
1672 if (txq < dev->real_num_tx_queues)
1673 qdisc_reset_all_tx_gt(dev, txq);
1674 }
1675
1676 dev->real_num_tx_queues = txq;
1677 return 0;
1678}
1679EXPORT_SYMBOL(netif_set_real_num_tx_queues);
1680
1681#ifdef CONFIG_RPS
1682/**
1683 * netif_set_real_num_rx_queues - set actual number of RX queues used
1684 * @dev: Network device
1685 * @rxq: Actual number of RX queues
1686 *
1687 * This must be called either with the rtnl_lock held or before
1688 * registration of the net device. Returns 0 on success, or a
1689 * negative error code. If called before registration, it always
1690 * succeeds.
1691 */
1692int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1693{
1694 int rc;
1695
1696 if (rxq < 1 || rxq > dev->num_rx_queues)
1697 return -EINVAL;
1698
1699 if (dev->reg_state == NETREG_REGISTERED) {
1700 ASSERT_RTNL();
1701
1702 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1703 rxq);
1704 if (rc)
1705 return rc;
1706 }
1707
1708 dev->real_num_rx_queues = rxq;
1709 return 0;
1710}
1711EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1712#endif
1713
1714static inline void __netif_reschedule(struct Qdisc *q)
1715{
1716 struct softnet_data *sd;
1717 unsigned long flags;
1718
1719 local_irq_save(flags);
1720 sd = &__get_cpu_var(softnet_data);
1721 q->next_sched = NULL;
1722 *sd->output_queue_tailp = q;
1723 sd->output_queue_tailp = &q->next_sched;
1724 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1725 local_irq_restore(flags);
1726}
1727
1728void __netif_schedule(struct Qdisc *q)
1729{
1730 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1731 __netif_reschedule(q);
1732}
1733EXPORT_SYMBOL(__netif_schedule);
1734
1735void dev_kfree_skb_irq(struct sk_buff *skb)
1736{
1737 if (atomic_dec_and_test(&skb->users)) {
1738 struct softnet_data *sd;
1739 unsigned long flags;
1740
1741 local_irq_save(flags);
1742 sd = &__get_cpu_var(softnet_data);
1743 skb->next = sd->completion_queue;
1744 sd->completion_queue = skb;
1745 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1746 local_irq_restore(flags);
1747 }
1748}
1749EXPORT_SYMBOL(dev_kfree_skb_irq);
1750
1751void dev_kfree_skb_any(struct sk_buff *skb)
1752{
1753 if (in_irq() || irqs_disabled())
1754 dev_kfree_skb_irq(skb);
1755 else
1756 dev_kfree_skb(skb);
1757}
1758EXPORT_SYMBOL(dev_kfree_skb_any);
1759
1760
1761/**
1762 * netif_device_detach - mark device as removed
1763 * @dev: network device
1764 *
1765 * Mark device as removed from system and therefore no longer available.
1766 */
1767void netif_device_detach(struct net_device *dev)
1768{
1769 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1770 netif_running(dev)) {
1771 netif_tx_stop_all_queues(dev);
1772 }
1773}
1774EXPORT_SYMBOL(netif_device_detach);
1775
1776/**
1777 * netif_device_attach - mark device as attached
1778 * @dev: network device
1779 *
1780 * Mark device as attached from system and restart if needed.
1781 */
1782void netif_device_attach(struct net_device *dev)
1783{
1784 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1785 netif_running(dev)) {
1786 netif_tx_wake_all_queues(dev);
1787 __netdev_watchdog_up(dev);
1788 }
1789}
1790EXPORT_SYMBOL(netif_device_attach);
1791
1792/**
1793 * skb_dev_set -- assign a new device to a buffer
1794 * @skb: buffer for the new device
1795 * @dev: network device
1796 *
1797 * If an skb is owned by a device already, we have to reset
1798 * all data private to the namespace a device belongs to
1799 * before assigning it a new device.
1800 */
1801#ifdef CONFIG_NET_NS
1802void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1803{
1804 skb_dst_drop(skb);
1805 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1806 secpath_reset(skb);
1807 nf_reset(skb);
1808 skb_init_secmark(skb);
1809 skb->mark = 0;
1810 skb->priority = 0;
1811 skb->nf_trace = 0;
1812 skb->ipvs_property = 0;
1813#ifdef CONFIG_NET_SCHED
1814 skb->tc_index = 0;
1815#endif
1816 }
1817 skb->dev = dev;
1818}
1819EXPORT_SYMBOL(skb_set_dev);
1820#endif /* CONFIG_NET_NS */
1821
1822/*
1823 * Invalidate hardware checksum when packet is to be mangled, and
1824 * complete checksum manually on outgoing path.
1825 */
1826int skb_checksum_help(struct sk_buff *skb)
1827{
1828 __wsum csum;
1829 int ret = 0, offset;
1830
1831 if (skb->ip_summed == CHECKSUM_COMPLETE)
1832 goto out_set_summed;
1833
1834 if (unlikely(skb_shinfo(skb)->gso_size)) {
1835 /* Let GSO fix up the checksum. */
1836 goto out_set_summed;
1837 }
1838
1839 offset = skb_checksum_start_offset(skb);
1840 BUG_ON(offset >= skb_headlen(skb));
1841 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1842
1843 offset += skb->csum_offset;
1844 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1845
1846 if (skb_cloned(skb) &&
1847 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
1848 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1849 if (ret)
1850 goto out;
1851 }
1852
1853 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
1854out_set_summed:
1855 skb->ip_summed = CHECKSUM_NONE;
1856out:
1857 return ret;
1858}
1859EXPORT_SYMBOL(skb_checksum_help);
1860
1861/**
1862 * skb_gso_segment - Perform segmentation on skb.
1863 * @skb: buffer to segment
1864 * @features: features for the output path (see dev->features)
1865 *
1866 * This function segments the given skb and returns a list of segments.
1867 *
1868 * It may return NULL if the skb requires no segmentation. This is
1869 * only possible when GSO is used for verifying header integrity.
1870 */
1871struct sk_buff *skb_gso_segment(struct sk_buff *skb, u32 features)
1872{
1873 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1874 struct packet_type *ptype;
1875 __be16 type = skb->protocol;
1876 int vlan_depth = ETH_HLEN;
1877 int err;
1878
1879 while (type == htons(ETH_P_8021Q)) {
1880 struct vlan_hdr *vh;
1881
1882 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
1883 return ERR_PTR(-EINVAL);
1884
1885 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
1886 type = vh->h_vlan_encapsulated_proto;
1887 vlan_depth += VLAN_HLEN;
1888 }
1889
1890 skb_reset_mac_header(skb);
1891 skb->mac_len = skb->network_header - skb->mac_header;
1892 __skb_pull(skb, skb->mac_len);
1893
1894 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1895 struct net_device *dev = skb->dev;
1896 struct ethtool_drvinfo info = {};
1897
1898 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1899 dev->ethtool_ops->get_drvinfo(dev, &info);
1900
1901 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n",
1902 info.driver, dev ? dev->features : 0L,
1903 skb->sk ? skb->sk->sk_route_caps : 0L,
1904 skb->len, skb->data_len, skb->ip_summed);
1905
1906 if (skb_header_cloned(skb) &&
1907 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1908 return ERR_PTR(err);
1909 }
1910
1911 rcu_read_lock();
1912 list_for_each_entry_rcu(ptype,
1913 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
1914 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
1915 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1916 err = ptype->gso_send_check(skb);
1917 segs = ERR_PTR(err);
1918 if (err || skb_gso_ok(skb, features))
1919 break;
1920 __skb_push(skb, (skb->data -
1921 skb_network_header(skb)));
1922 }
1923 segs = ptype->gso_segment(skb, features);
1924 break;
1925 }
1926 }
1927 rcu_read_unlock();
1928
1929 __skb_push(skb, skb->data - skb_mac_header(skb));
1930
1931 return segs;
1932}
1933EXPORT_SYMBOL(skb_gso_segment);
1934
1935/* Take action when hardware reception checksum errors are detected. */
1936#ifdef CONFIG_BUG
1937void netdev_rx_csum_fault(struct net_device *dev)
1938{
1939 if (net_ratelimit()) {
1940 printk(KERN_ERR "%s: hw csum failure.\n",
1941 dev ? dev->name : "<unknown>");
1942 dump_stack();
1943 }
1944}
1945EXPORT_SYMBOL(netdev_rx_csum_fault);
1946#endif
1947
1948/* Actually, we should eliminate this check as soon as we know, that:
1949 * 1. IOMMU is present and allows to map all the memory.
1950 * 2. No high memory really exists on this machine.
1951 */
1952
1953static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
1954{
1955#ifdef CONFIG_HIGHMEM
1956 int i;
1957 if (!(dev->features & NETIF_F_HIGHDMA)) {
1958 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1959 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1960 return 1;
1961 }
1962
1963 if (PCI_DMA_BUS_IS_PHYS) {
1964 struct device *pdev = dev->dev.parent;
1965
1966 if (!pdev)
1967 return 0;
1968 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1969 dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);
1970 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
1971 return 1;
1972 }
1973 }
1974#endif
1975 return 0;
1976}
1977
1978struct dev_gso_cb {
1979 void (*destructor)(struct sk_buff *skb);
1980};
1981
1982#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1983
1984static void dev_gso_skb_destructor(struct sk_buff *skb)
1985{
1986 struct dev_gso_cb *cb;
1987
1988 do {
1989 struct sk_buff *nskb = skb->next;
1990
1991 skb->next = nskb->next;
1992 nskb->next = NULL;
1993 kfree_skb(nskb);
1994 } while (skb->next);
1995
1996 cb = DEV_GSO_CB(skb);
1997 if (cb->destructor)
1998 cb->destructor(skb);
1999}
2000
2001/**
2002 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2003 * @skb: buffer to segment
2004 * @features: device features as applicable to this skb
2005 *
2006 * This function segments the given skb and stores the list of segments
2007 * in skb->next.
2008 */
2009static int dev_gso_segment(struct sk_buff *skb, int features)
2010{
2011 struct sk_buff *segs;
2012
2013 segs = skb_gso_segment(skb, features);
2014
2015 /* Verifying header integrity only. */
2016 if (!segs)
2017 return 0;
2018
2019 if (IS_ERR(segs))
2020 return PTR_ERR(segs);
2021
2022 skb->next = segs;
2023 DEV_GSO_CB(skb)->destructor = skb->destructor;
2024 skb->destructor = dev_gso_skb_destructor;
2025
2026 return 0;
2027}
2028
2029/*
2030 * Try to orphan skb early, right before transmission by the device.
2031 * We cannot orphan skb if tx timestamp is requested or the sk-reference
2032 * is needed on driver level for other reasons, e.g. see net/can/raw.c
2033 */
2034static inline void skb_orphan_try(struct sk_buff *skb)
2035{
2036 struct sock *sk = skb->sk;
2037
2038 if (sk && !skb_shinfo(skb)->tx_flags) {
2039 /* skb_tx_hash() wont be able to get sk.
2040 * We copy sk_hash into skb->rxhash
2041 */
2042 if (!skb->rxhash)
2043 skb->rxhash = sk->sk_hash;
2044 skb_orphan(skb);
2045 }
2046}
2047
2048static bool can_checksum_protocol(unsigned long features, __be16 protocol)
2049{
2050 return ((features & NETIF_F_GEN_CSUM) ||
2051 ((features & NETIF_F_V4_CSUM) &&
2052 protocol == htons(ETH_P_IP)) ||
2053 ((features & NETIF_F_V6_CSUM) &&
2054 protocol == htons(ETH_P_IPV6)) ||
2055 ((features & NETIF_F_FCOE_CRC) &&
2056 protocol == htons(ETH_P_FCOE)));
2057}
2058
2059static u32 harmonize_features(struct sk_buff *skb, __be16 protocol, u32 features)
2060{
2061 if (!can_checksum_protocol(features, protocol)) {
2062 features &= ~NETIF_F_ALL_CSUM;
2063 features &= ~NETIF_F_SG;
2064 } else if (illegal_highdma(skb->dev, skb)) {
2065 features &= ~NETIF_F_SG;
2066 }
2067
2068 return features;
2069}
2070
2071u32 netif_skb_features(struct sk_buff *skb)
2072{
2073 __be16 protocol = skb->protocol;
2074 u32 features = skb->dev->features;
2075
2076 if (protocol == htons(ETH_P_8021Q)) {
2077 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2078 protocol = veh->h_vlan_encapsulated_proto;
2079 } else if (!vlan_tx_tag_present(skb)) {
2080 return harmonize_features(skb, protocol, features);
2081 }
2082
2083 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
2084
2085 if (protocol != htons(ETH_P_8021Q)) {
2086 return harmonize_features(skb, protocol, features);
2087 } else {
2088 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
2089 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
2090 return harmonize_features(skb, protocol, features);
2091 }
2092}
2093EXPORT_SYMBOL(netif_skb_features);
2094
2095/*
2096 * Returns true if either:
2097 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2098 * 2. skb is fragmented and the device does not support SG, or if
2099 * at least one of fragments is in highmem and device does not
2100 * support DMA from it.
2101 */
2102static inline int skb_needs_linearize(struct sk_buff *skb,
2103 int features)
2104{
2105 return skb_is_nonlinear(skb) &&
2106 ((skb_has_frag_list(skb) &&
2107 !(features & NETIF_F_FRAGLIST)) ||
2108 (skb_shinfo(skb)->nr_frags &&
2109 !(features & NETIF_F_SG)));
2110}
2111
2112int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2113 struct netdev_queue *txq)
2114{
2115 const struct net_device_ops *ops = dev->netdev_ops;
2116 int rc = NETDEV_TX_OK;
2117 unsigned int skb_len;
2118
2119 if (likely(!skb->next)) {
2120 u32 features;
2121
2122 /*
2123 * If device doesn't need skb->dst, release it right now while
2124 * its hot in this cpu cache
2125 */
2126 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2127 skb_dst_drop(skb);
2128
2129 if (!list_empty(&ptype_all))
2130 dev_queue_xmit_nit(skb, dev);
2131
2132 skb_orphan_try(skb);
2133
2134 features = netif_skb_features(skb);
2135
2136 if (vlan_tx_tag_present(skb) &&
2137 !(features & NETIF_F_HW_VLAN_TX)) {
2138 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2139 if (unlikely(!skb))
2140 goto out;
2141
2142 skb->vlan_tci = 0;
2143 }
2144
2145 if (netif_needs_gso(skb, features)) {
2146 if (unlikely(dev_gso_segment(skb, features)))
2147 goto out_kfree_skb;
2148 if (skb->next)
2149 goto gso;
2150 } else {
2151 if (skb_needs_linearize(skb, features) &&
2152 __skb_linearize(skb))
2153 goto out_kfree_skb;
2154
2155 /* If packet is not checksummed and device does not
2156 * support checksumming for this protocol, complete
2157 * checksumming here.
2158 */
2159 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2160 skb_set_transport_header(skb,
2161 skb_checksum_start_offset(skb));
2162 if (!(features & NETIF_F_ALL_CSUM) &&
2163 skb_checksum_help(skb))
2164 goto out_kfree_skb;
2165 }
2166 }
2167
2168 skb_len = skb->len;
2169 rc = ops->ndo_start_xmit(skb, dev);
2170 trace_net_dev_xmit(skb, rc, dev, skb_len);
2171 if (rc == NETDEV_TX_OK)
2172 txq_trans_update(txq);
2173 return rc;
2174 }
2175
2176gso:
2177 do {
2178 struct sk_buff *nskb = skb->next;
2179
2180 skb->next = nskb->next;
2181 nskb->next = NULL;
2182
2183 /*
2184 * If device doesn't need nskb->dst, release it right now while
2185 * its hot in this cpu cache
2186 */
2187 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2188 skb_dst_drop(nskb);
2189
2190 skb_len = nskb->len;
2191 rc = ops->ndo_start_xmit(nskb, dev);
2192 trace_net_dev_xmit(nskb, rc, dev, skb_len);
2193 if (unlikely(rc != NETDEV_TX_OK)) {
2194 if (rc & ~NETDEV_TX_MASK)
2195 goto out_kfree_gso_skb;
2196 nskb->next = skb->next;
2197 skb->next = nskb;
2198 return rc;
2199 }
2200 txq_trans_update(txq);
2201 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
2202 return NETDEV_TX_BUSY;
2203 } while (skb->next);
2204
2205out_kfree_gso_skb:
2206 if (likely(skb->next == NULL))
2207 skb->destructor = DEV_GSO_CB(skb)->destructor;
2208out_kfree_skb:
2209 kfree_skb(skb);
2210out:
2211 return rc;
2212}
2213
2214static u32 hashrnd __read_mostly;
2215
2216/*
2217 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2218 * to be used as a distribution range.
2219 */
2220u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2221 unsigned int num_tx_queues)
2222{
2223 u32 hash;
2224 u16 qoffset = 0;
2225 u16 qcount = num_tx_queues;
2226
2227 if (skb_rx_queue_recorded(skb)) {
2228 hash = skb_get_rx_queue(skb);
2229 while (unlikely(hash >= num_tx_queues))
2230 hash -= num_tx_queues;
2231 return hash;
2232 }
2233
2234 if (dev->num_tc) {
2235 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2236 qoffset = dev->tc_to_txq[tc].offset;
2237 qcount = dev->tc_to_txq[tc].count;
2238 }
2239
2240 if (skb->sk && skb->sk->sk_hash)
2241 hash = skb->sk->sk_hash;
2242 else
2243 hash = (__force u16) skb->protocol ^ skb->rxhash;
2244 hash = jhash_1word(hash, hashrnd);
2245
2246 return (u16) (((u64) hash * qcount) >> 32) + qoffset;
2247}
2248EXPORT_SYMBOL(__skb_tx_hash);
2249
2250static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2251{
2252 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2253 if (net_ratelimit()) {
2254 pr_warning("%s selects TX queue %d, but "
2255 "real number of TX queues is %d\n",
2256 dev->name, queue_index, dev->real_num_tx_queues);
2257 }
2258 return 0;
2259 }
2260 return queue_index;
2261}
2262
2263static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2264{
2265#ifdef CONFIG_XPS
2266 struct xps_dev_maps *dev_maps;
2267 struct xps_map *map;
2268 int queue_index = -1;
2269
2270 rcu_read_lock();
2271 dev_maps = rcu_dereference(dev->xps_maps);
2272 if (dev_maps) {
2273 map = rcu_dereference(
2274 dev_maps->cpu_map[raw_smp_processor_id()]);
2275 if (map) {
2276 if (map->len == 1)
2277 queue_index = map->queues[0];
2278 else {
2279 u32 hash;
2280 if (skb->sk && skb->sk->sk_hash)
2281 hash = skb->sk->sk_hash;
2282 else
2283 hash = (__force u16) skb->protocol ^
2284 skb->rxhash;
2285 hash = jhash_1word(hash, hashrnd);
2286 queue_index = map->queues[
2287 ((u64)hash * map->len) >> 32];
2288 }
2289 if (unlikely(queue_index >= dev->real_num_tx_queues))
2290 queue_index = -1;
2291 }
2292 }
2293 rcu_read_unlock();
2294
2295 return queue_index;
2296#else
2297 return -1;
2298#endif
2299}
2300
2301static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2302 struct sk_buff *skb)
2303{
2304 int queue_index;
2305 const struct net_device_ops *ops = dev->netdev_ops;
2306
2307 if (dev->real_num_tx_queues == 1)
2308 queue_index = 0;
2309 else if (ops->ndo_select_queue) {
2310 queue_index = ops->ndo_select_queue(dev, skb);
2311 queue_index = dev_cap_txqueue(dev, queue_index);
2312 } else {
2313 struct sock *sk = skb->sk;
2314 queue_index = sk_tx_queue_get(sk);
2315
2316 if (queue_index < 0 || skb->ooo_okay ||
2317 queue_index >= dev->real_num_tx_queues) {
2318 int old_index = queue_index;
2319
2320 queue_index = get_xps_queue(dev, skb);
2321 if (queue_index < 0)
2322 queue_index = skb_tx_hash(dev, skb);
2323
2324 if (queue_index != old_index && sk) {
2325 struct dst_entry *dst =
2326 rcu_dereference_check(sk->sk_dst_cache, 1);
2327
2328 if (dst && skb_dst(skb) == dst)
2329 sk_tx_queue_set(sk, queue_index);
2330 }
2331 }
2332 }
2333
2334 skb_set_queue_mapping(skb, queue_index);
2335 return netdev_get_tx_queue(dev, queue_index);
2336}
2337
2338static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2339 struct net_device *dev,
2340 struct netdev_queue *txq)
2341{
2342 spinlock_t *root_lock = qdisc_lock(q);
2343 bool contended;
2344 int rc;
2345
2346 qdisc_skb_cb(skb)->pkt_len = skb->len;
2347 qdisc_calculate_pkt_len(skb, q);
2348 /*
2349 * Heuristic to force contended enqueues to serialize on a
2350 * separate lock before trying to get qdisc main lock.
2351 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2352 * and dequeue packets faster.
2353 */
2354 contended = qdisc_is_running(q);
2355 if (unlikely(contended))
2356 spin_lock(&q->busylock);
2357
2358 spin_lock(root_lock);
2359 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2360 kfree_skb(skb);
2361 rc = NET_XMIT_DROP;
2362 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2363 qdisc_run_begin(q)) {
2364 /*
2365 * This is a work-conserving queue; there are no old skbs
2366 * waiting to be sent out; and the qdisc is not running -
2367 * xmit the skb directly.
2368 */
2369 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2370 skb_dst_force(skb);
2371
2372 qdisc_bstats_update(q, skb);
2373
2374 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2375 if (unlikely(contended)) {
2376 spin_unlock(&q->busylock);
2377 contended = false;
2378 }
2379 __qdisc_run(q);
2380 } else
2381 qdisc_run_end(q);
2382
2383 rc = NET_XMIT_SUCCESS;
2384 } else {
2385 skb_dst_force(skb);
2386 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
2387 if (qdisc_run_begin(q)) {
2388 if (unlikely(contended)) {
2389 spin_unlock(&q->busylock);
2390 contended = false;
2391 }
2392 __qdisc_run(q);
2393 }
2394 }
2395 spin_unlock(root_lock);
2396 if (unlikely(contended))
2397 spin_unlock(&q->busylock);
2398 return rc;
2399}
2400
2401static DEFINE_PER_CPU(int, xmit_recursion);
2402#define RECURSION_LIMIT 10
2403
2404/**
2405 * dev_queue_xmit - transmit a buffer
2406 * @skb: buffer to transmit
2407 *
2408 * Queue a buffer for transmission to a network device. The caller must
2409 * have set the device and priority and built the buffer before calling
2410 * this function. The function can be called from an interrupt.
2411 *
2412 * A negative errno code is returned on a failure. A success does not
2413 * guarantee the frame will be transmitted as it may be dropped due
2414 * to congestion or traffic shaping.
2415 *
2416 * -----------------------------------------------------------------------------------
2417 * I notice this method can also return errors from the queue disciplines,
2418 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2419 * be positive.
2420 *
2421 * Regardless of the return value, the skb is consumed, so it is currently
2422 * difficult to retry a send to this method. (You can bump the ref count
2423 * before sending to hold a reference for retry if you are careful.)
2424 *
2425 * When calling this method, interrupts MUST be enabled. This is because
2426 * the BH enable code must have IRQs enabled so that it will not deadlock.
2427 * --BLG
2428 */
2429int dev_queue_xmit(struct sk_buff *skb)
2430{
2431 struct net_device *dev = skb->dev;
2432 struct netdev_queue *txq;
2433 struct Qdisc *q;
2434 int rc = -ENOMEM;
2435
2436 /* Disable soft irqs for various locks below. Also
2437 * stops preemption for RCU.
2438 */
2439 rcu_read_lock_bh();
2440
2441 txq = dev_pick_tx(dev, skb);
2442 q = rcu_dereference_bh(txq->qdisc);
2443
2444#ifdef CONFIG_NET_CLS_ACT
2445 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
2446#endif
2447 trace_net_dev_queue(skb);
2448 if (q->enqueue) {
2449 rc = __dev_xmit_skb(skb, q, dev, txq);
2450 goto out;
2451 }
2452
2453 /* The device has no queue. Common case for software devices:
2454 loopback, all the sorts of tunnels...
2455
2456 Really, it is unlikely that netif_tx_lock protection is necessary
2457 here. (f.e. loopback and IP tunnels are clean ignoring statistics
2458 counters.)
2459 However, it is possible, that they rely on protection
2460 made by us here.
2461
2462 Check this and shot the lock. It is not prone from deadlocks.
2463 Either shot noqueue qdisc, it is even simpler 8)
2464 */
2465 if (dev->flags & IFF_UP) {
2466 int cpu = smp_processor_id(); /* ok because BHs are off */
2467
2468 if (txq->xmit_lock_owner != cpu) {
2469
2470 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2471 goto recursion_alert;
2472
2473 HARD_TX_LOCK(dev, txq, cpu);
2474
2475 if (!netif_tx_queue_stopped(txq)) {
2476 __this_cpu_inc(xmit_recursion);
2477 rc = dev_hard_start_xmit(skb, dev, txq);
2478 __this_cpu_dec(xmit_recursion);
2479 if (dev_xmit_complete(rc)) {
2480 HARD_TX_UNLOCK(dev, txq);
2481 goto out;
2482 }
2483 }
2484 HARD_TX_UNLOCK(dev, txq);
2485 if (net_ratelimit())
2486 printk(KERN_CRIT "Virtual device %s asks to "
2487 "queue packet!\n", dev->name);
2488 } else {
2489 /* Recursion is detected! It is possible,
2490 * unfortunately
2491 */
2492recursion_alert:
2493 if (net_ratelimit())
2494 printk(KERN_CRIT "Dead loop on virtual device "
2495 "%s, fix it urgently!\n", dev->name);
2496 }
2497 }
2498
2499 rc = -ENETDOWN;
2500 rcu_read_unlock_bh();
2501
2502 kfree_skb(skb);
2503 return rc;
2504out:
2505 rcu_read_unlock_bh();
2506 return rc;
2507}
2508EXPORT_SYMBOL(dev_queue_xmit);
2509
2510
2511/*=======================================================================
2512 Receiver routines
2513 =======================================================================*/
2514
2515int netdev_max_backlog __read_mostly = 1000;
2516int netdev_tstamp_prequeue __read_mostly = 1;
2517int netdev_budget __read_mostly = 300;
2518int weight_p __read_mostly = 64; /* old backlog weight */
2519
2520/* Called with irq disabled */
2521static inline void ____napi_schedule(struct softnet_data *sd,
2522 struct napi_struct *napi)
2523{
2524 list_add_tail(&napi->poll_list, &sd->poll_list);
2525 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2526}
2527
2528/*
2529 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
2530 * and src/dst port numbers. Returns a non-zero hash number on success
2531 * and 0 on failure.
2532 */
2533__u32 __skb_get_rxhash(struct sk_buff *skb)
2534{
2535 int nhoff, hash = 0, poff;
2536 const struct ipv6hdr *ip6;
2537 const struct iphdr *ip;
2538 u8 ip_proto;
2539 u32 addr1, addr2, ihl;
2540 union {
2541 u32 v32;
2542 u16 v16[2];
2543 } ports;
2544
2545 nhoff = skb_network_offset(skb);
2546
2547 switch (skb->protocol) {
2548 case __constant_htons(ETH_P_IP):
2549 if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))
2550 goto done;
2551
2552 ip = (const struct iphdr *) (skb->data + nhoff);
2553 if (ip_is_fragment(ip))
2554 ip_proto = 0;
2555 else
2556 ip_proto = ip->protocol;
2557 addr1 = (__force u32) ip->saddr;
2558 addr2 = (__force u32) ip->daddr;
2559 ihl = ip->ihl;
2560 break;
2561 case __constant_htons(ETH_P_IPV6):
2562 if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))
2563 goto done;
2564
2565 ip6 = (const struct ipv6hdr *) (skb->data + nhoff);
2566 ip_proto = ip6->nexthdr;
2567 addr1 = (__force u32) ip6->saddr.s6_addr32[3];
2568 addr2 = (__force u32) ip6->daddr.s6_addr32[3];
2569 ihl = (40 >> 2);
2570 break;
2571 default:
2572 goto done;
2573 }
2574
2575 ports.v32 = 0;
2576 poff = proto_ports_offset(ip_proto);
2577 if (poff >= 0) {
2578 nhoff += ihl * 4 + poff;
2579 if (pskb_may_pull(skb, nhoff + 4)) {
2580 ports.v32 = * (__force u32 *) (skb->data + nhoff);
2581 if (ports.v16[1] < ports.v16[0])
2582 swap(ports.v16[0], ports.v16[1]);
2583 }
2584 }
2585
2586 /* get a consistent hash (same value on both flow directions) */
2587 if (addr2 < addr1)
2588 swap(addr1, addr2);
2589
2590 hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
2591 if (!hash)
2592 hash = 1;
2593
2594done:
2595 return hash;
2596}
2597EXPORT_SYMBOL(__skb_get_rxhash);
2598
2599#ifdef CONFIG_RPS
2600
2601/* One global table that all flow-based protocols share. */
2602struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
2603EXPORT_SYMBOL(rps_sock_flow_table);
2604
2605static struct rps_dev_flow *
2606set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2607 struct rps_dev_flow *rflow, u16 next_cpu)
2608{
2609 u16 tcpu;
2610
2611 tcpu = rflow->cpu = next_cpu;
2612 if (tcpu != RPS_NO_CPU) {
2613#ifdef CONFIG_RFS_ACCEL
2614 struct netdev_rx_queue *rxqueue;
2615 struct rps_dev_flow_table *flow_table;
2616 struct rps_dev_flow *old_rflow;
2617 u32 flow_id;
2618 u16 rxq_index;
2619 int rc;
2620
2621 /* Should we steer this flow to a different hardware queue? */
2622 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
2623 !(dev->features & NETIF_F_NTUPLE))
2624 goto out;
2625 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
2626 if (rxq_index == skb_get_rx_queue(skb))
2627 goto out;
2628
2629 rxqueue = dev->_rx + rxq_index;
2630 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2631 if (!flow_table)
2632 goto out;
2633 flow_id = skb->rxhash & flow_table->mask;
2634 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
2635 rxq_index, flow_id);
2636 if (rc < 0)
2637 goto out;
2638 old_rflow = rflow;
2639 rflow = &flow_table->flows[flow_id];
2640 rflow->cpu = next_cpu;
2641 rflow->filter = rc;
2642 if (old_rflow->filter == rflow->filter)
2643 old_rflow->filter = RPS_NO_FILTER;
2644 out:
2645#endif
2646 rflow->last_qtail =
2647 per_cpu(softnet_data, tcpu).input_queue_head;
2648 }
2649
2650 return rflow;
2651}
2652
2653/*
2654 * get_rps_cpu is called from netif_receive_skb and returns the target
2655 * CPU from the RPS map of the receiving queue for a given skb.
2656 * rcu_read_lock must be held on entry.
2657 */
2658static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2659 struct rps_dev_flow **rflowp)
2660{
2661 struct netdev_rx_queue *rxqueue;
2662 struct rps_map *map;
2663 struct rps_dev_flow_table *flow_table;
2664 struct rps_sock_flow_table *sock_flow_table;
2665 int cpu = -1;
2666 u16 tcpu;
2667
2668 if (skb_rx_queue_recorded(skb)) {
2669 u16 index = skb_get_rx_queue(skb);
2670 if (unlikely(index >= dev->real_num_rx_queues)) {
2671 WARN_ONCE(dev->real_num_rx_queues > 1,
2672 "%s received packet on queue %u, but number "
2673 "of RX queues is %u\n",
2674 dev->name, index, dev->real_num_rx_queues);
2675 goto done;
2676 }
2677 rxqueue = dev->_rx + index;
2678 } else
2679 rxqueue = dev->_rx;
2680
2681 map = rcu_dereference(rxqueue->rps_map);
2682 if (map) {
2683 if (map->len == 1 &&
2684 !rcu_dereference_raw(rxqueue->rps_flow_table)) {
2685 tcpu = map->cpus[0];
2686 if (cpu_online(tcpu))
2687 cpu = tcpu;
2688 goto done;
2689 }
2690 } else if (!rcu_dereference_raw(rxqueue->rps_flow_table)) {
2691 goto done;
2692 }
2693
2694 skb_reset_network_header(skb);
2695 if (!skb_get_rxhash(skb))
2696 goto done;
2697
2698 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2699 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2700 if (flow_table && sock_flow_table) {
2701 u16 next_cpu;
2702 struct rps_dev_flow *rflow;
2703
2704 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2705 tcpu = rflow->cpu;
2706
2707 next_cpu = sock_flow_table->ents[skb->rxhash &
2708 sock_flow_table->mask];
2709
2710 /*
2711 * If the desired CPU (where last recvmsg was done) is
2712 * different from current CPU (one in the rx-queue flow
2713 * table entry), switch if one of the following holds:
2714 * - Current CPU is unset (equal to RPS_NO_CPU).
2715 * - Current CPU is offline.
2716 * - The current CPU's queue tail has advanced beyond the
2717 * last packet that was enqueued using this table entry.
2718 * This guarantees that all previous packets for the flow
2719 * have been dequeued, thus preserving in order delivery.
2720 */
2721 if (unlikely(tcpu != next_cpu) &&
2722 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2723 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
2724 rflow->last_qtail)) >= 0))
2725 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
2726
2727 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2728 *rflowp = rflow;
2729 cpu = tcpu;
2730 goto done;
2731 }
2732 }
2733
2734 if (map) {
2735 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
2736
2737 if (cpu_online(tcpu)) {
2738 cpu = tcpu;
2739 goto done;
2740 }
2741 }
2742
2743done:
2744 return cpu;
2745}
2746
2747#ifdef CONFIG_RFS_ACCEL
2748
2749/**
2750 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2751 * @dev: Device on which the filter was set
2752 * @rxq_index: RX queue index
2753 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2754 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2755 *
2756 * Drivers that implement ndo_rx_flow_steer() should periodically call
2757 * this function for each installed filter and remove the filters for
2758 * which it returns %true.
2759 */
2760bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
2761 u32 flow_id, u16 filter_id)
2762{
2763 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
2764 struct rps_dev_flow_table *flow_table;
2765 struct rps_dev_flow *rflow;
2766 bool expire = true;
2767 int cpu;
2768
2769 rcu_read_lock();
2770 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2771 if (flow_table && flow_id <= flow_table->mask) {
2772 rflow = &flow_table->flows[flow_id];
2773 cpu = ACCESS_ONCE(rflow->cpu);
2774 if (rflow->filter == filter_id && cpu != RPS_NO_CPU &&
2775 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
2776 rflow->last_qtail) <
2777 (int)(10 * flow_table->mask)))
2778 expire = false;
2779 }
2780 rcu_read_unlock();
2781 return expire;
2782}
2783EXPORT_SYMBOL(rps_may_expire_flow);
2784
2785#endif /* CONFIG_RFS_ACCEL */
2786
2787/* Called from hardirq (IPI) context */
2788static void rps_trigger_softirq(void *data)
2789{
2790 struct softnet_data *sd = data;
2791
2792 ____napi_schedule(sd, &sd->backlog);
2793 sd->received_rps++;
2794}
2795
2796#endif /* CONFIG_RPS */
2797
2798/*
2799 * Check if this softnet_data structure is another cpu one
2800 * If yes, queue it to our IPI list and return 1
2801 * If no, return 0
2802 */
2803static int rps_ipi_queued(struct softnet_data *sd)
2804{
2805#ifdef CONFIG_RPS
2806 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2807
2808 if (sd != mysd) {
2809 sd->rps_ipi_next = mysd->rps_ipi_list;
2810 mysd->rps_ipi_list = sd;
2811
2812 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2813 return 1;
2814 }
2815#endif /* CONFIG_RPS */
2816 return 0;
2817}
2818
2819/*
2820 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2821 * queue (may be a remote CPU queue).
2822 */
2823static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2824 unsigned int *qtail)
2825{
2826 struct softnet_data *sd;
2827 unsigned long flags;
2828
2829 sd = &per_cpu(softnet_data, cpu);
2830
2831 local_irq_save(flags);
2832
2833 rps_lock(sd);
2834 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2835 if (skb_queue_len(&sd->input_pkt_queue)) {
2836enqueue:
2837 __skb_queue_tail(&sd->input_pkt_queue, skb);
2838 input_queue_tail_incr_save(sd, qtail);
2839 rps_unlock(sd);
2840 local_irq_restore(flags);
2841 return NET_RX_SUCCESS;
2842 }
2843
2844 /* Schedule NAPI for backlog device
2845 * We can use non atomic operation since we own the queue lock
2846 */
2847 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
2848 if (!rps_ipi_queued(sd))
2849 ____napi_schedule(sd, &sd->backlog);
2850 }
2851 goto enqueue;
2852 }
2853
2854 sd->dropped++;
2855 rps_unlock(sd);
2856
2857 local_irq_restore(flags);
2858
2859 atomic_long_inc(&skb->dev->rx_dropped);
2860 kfree_skb(skb);
2861 return NET_RX_DROP;
2862}
2863
2864/**
2865 * netif_rx - post buffer to the network code
2866 * @skb: buffer to post
2867 *
2868 * This function receives a packet from a device driver and queues it for
2869 * the upper (protocol) levels to process. It always succeeds. The buffer
2870 * may be dropped during processing for congestion control or by the
2871 * protocol layers.
2872 *
2873 * return values:
2874 * NET_RX_SUCCESS (no congestion)
2875 * NET_RX_DROP (packet was dropped)
2876 *
2877 */
2878
2879int netif_rx(struct sk_buff *skb)
2880{
2881 int ret;
2882
2883 /* if netpoll wants it, pretend we never saw it */
2884 if (netpoll_rx(skb))
2885 return NET_RX_DROP;
2886
2887 if (netdev_tstamp_prequeue)
2888 net_timestamp_check(skb);
2889
2890 trace_netif_rx(skb);
2891#ifdef CONFIG_RPS
2892 {
2893 struct rps_dev_flow voidflow, *rflow = &voidflow;
2894 int cpu;
2895
2896 preempt_disable();
2897 rcu_read_lock();
2898
2899 cpu = get_rps_cpu(skb->dev, skb, &rflow);
2900 if (cpu < 0)
2901 cpu = smp_processor_id();
2902
2903 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2904
2905 rcu_read_unlock();
2906 preempt_enable();
2907 }
2908#else
2909 {
2910 unsigned int qtail;
2911 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2912 put_cpu();
2913 }
2914#endif
2915 return ret;
2916}
2917EXPORT_SYMBOL(netif_rx);
2918
2919int netif_rx_ni(struct sk_buff *skb)
2920{
2921 int err;
2922
2923 preempt_disable();
2924 err = netif_rx(skb);
2925 if (local_softirq_pending())
2926 do_softirq();
2927 preempt_enable();
2928
2929 return err;
2930}
2931EXPORT_SYMBOL(netif_rx_ni);
2932
2933static void net_tx_action(struct softirq_action *h)
2934{
2935 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2936
2937 if (sd->completion_queue) {
2938 struct sk_buff *clist;
2939
2940 local_irq_disable();
2941 clist = sd->completion_queue;
2942 sd->completion_queue = NULL;
2943 local_irq_enable();
2944
2945 while (clist) {
2946 struct sk_buff *skb = clist;
2947 clist = clist->next;
2948
2949 WARN_ON(atomic_read(&skb->users));
2950 trace_kfree_skb(skb, net_tx_action);
2951 __kfree_skb(skb);
2952 }
2953 }
2954
2955 if (sd->output_queue) {
2956 struct Qdisc *head;
2957
2958 local_irq_disable();
2959 head = sd->output_queue;
2960 sd->output_queue = NULL;
2961 sd->output_queue_tailp = &sd->output_queue;
2962 local_irq_enable();
2963
2964 while (head) {
2965 struct Qdisc *q = head;
2966 spinlock_t *root_lock;
2967
2968 head = head->next_sched;
2969
2970 root_lock = qdisc_lock(q);
2971 if (spin_trylock(root_lock)) {
2972 smp_mb__before_clear_bit();
2973 clear_bit(__QDISC_STATE_SCHED,
2974 &q->state);
2975 qdisc_run(q);
2976 spin_unlock(root_lock);
2977 } else {
2978 if (!test_bit(__QDISC_STATE_DEACTIVATED,
2979 &q->state)) {
2980 __netif_reschedule(q);
2981 } else {
2982 smp_mb__before_clear_bit();
2983 clear_bit(__QDISC_STATE_SCHED,
2984 &q->state);
2985 }
2986 }
2987 }
2988 }
2989}
2990
2991#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
2992 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
2993/* This hook is defined here for ATM LANE */
2994int (*br_fdb_test_addr_hook)(struct net_device *dev,
2995 unsigned char *addr) __read_mostly;
2996EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
2997#endif
2998
2999#ifdef CONFIG_NET_CLS_ACT
3000/* TODO: Maybe we should just force sch_ingress to be compiled in
3001 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3002 * a compare and 2 stores extra right now if we dont have it on
3003 * but have CONFIG_NET_CLS_ACT
3004 * NOTE: This doesn't stop any functionality; if you dont have
3005 * the ingress scheduler, you just can't add policies on ingress.
3006 *
3007 */
3008static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
3009{
3010 struct net_device *dev = skb->dev;
3011 u32 ttl = G_TC_RTTL(skb->tc_verd);
3012 int result = TC_ACT_OK;
3013 struct Qdisc *q;
3014
3015 if (unlikely(MAX_RED_LOOP < ttl++)) {
3016 if (net_ratelimit())
3017 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
3018 skb->skb_iif, dev->ifindex);
3019 return TC_ACT_SHOT;
3020 }
3021
3022 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
3023 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3024
3025 q = rxq->qdisc;
3026 if (q != &noop_qdisc) {
3027 spin_lock(qdisc_lock(q));
3028 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3029 result = qdisc_enqueue_root(skb, q);
3030 spin_unlock(qdisc_lock(q));
3031 }
3032
3033 return result;
3034}
3035
3036static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3037 struct packet_type **pt_prev,
3038 int *ret, struct net_device *orig_dev)
3039{
3040 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
3041
3042 if (!rxq || rxq->qdisc == &noop_qdisc)
3043 goto out;
3044
3045 if (*pt_prev) {
3046 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3047 *pt_prev = NULL;
3048 }
3049
3050 switch (ing_filter(skb, rxq)) {
3051 case TC_ACT_SHOT:
3052 case TC_ACT_STOLEN:
3053 kfree_skb(skb);
3054 return NULL;
3055 }
3056
3057out:
3058 skb->tc_verd = 0;
3059 return skb;
3060}
3061#endif
3062
3063/**
3064 * netdev_rx_handler_register - register receive handler
3065 * @dev: device to register a handler for
3066 * @rx_handler: receive handler to register
3067 * @rx_handler_data: data pointer that is used by rx handler
3068 *
3069 * Register a receive hander for a device. This handler will then be
3070 * called from __netif_receive_skb. A negative errno code is returned
3071 * on a failure.
3072 *
3073 * The caller must hold the rtnl_mutex.
3074 *
3075 * For a general description of rx_handler, see enum rx_handler_result.
3076 */
3077int netdev_rx_handler_register(struct net_device *dev,
3078 rx_handler_func_t *rx_handler,
3079 void *rx_handler_data)
3080{
3081 ASSERT_RTNL();
3082
3083 if (dev->rx_handler)
3084 return -EBUSY;
3085
3086 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3087 rcu_assign_pointer(dev->rx_handler, rx_handler);
3088
3089 return 0;
3090}
3091EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3092
3093/**
3094 * netdev_rx_handler_unregister - unregister receive handler
3095 * @dev: device to unregister a handler from
3096 *
3097 * Unregister a receive hander from a device.
3098 *
3099 * The caller must hold the rtnl_mutex.
3100 */
3101void netdev_rx_handler_unregister(struct net_device *dev)
3102{
3103
3104 ASSERT_RTNL();
3105 rcu_assign_pointer(dev->rx_handler, NULL);
3106 rcu_assign_pointer(dev->rx_handler_data, NULL);
3107}
3108EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3109
3110static int __netif_receive_skb(struct sk_buff *skb)
3111{
3112 struct packet_type *ptype, *pt_prev;
3113 rx_handler_func_t *rx_handler;
3114 struct net_device *orig_dev;
3115 struct net_device *null_or_dev;
3116 bool deliver_exact = false;
3117 int ret = NET_RX_DROP;
3118 __be16 type;
3119
3120 if (!netdev_tstamp_prequeue)
3121 net_timestamp_check(skb);
3122
3123 trace_netif_receive_skb(skb);
3124
3125 /* if we've gotten here through NAPI, check netpoll */
3126 if (netpoll_receive_skb(skb))
3127 return NET_RX_DROP;
3128
3129 if (!skb->skb_iif)
3130 skb->skb_iif = skb->dev->ifindex;
3131 orig_dev = skb->dev;
3132
3133 skb_reset_network_header(skb);
3134 skb_reset_transport_header(skb);
3135 skb_reset_mac_len(skb);
3136
3137 pt_prev = NULL;
3138
3139 rcu_read_lock();
3140
3141another_round:
3142
3143 __this_cpu_inc(softnet_data.processed);
3144
3145 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3146 skb = vlan_untag(skb);
3147 if (unlikely(!skb))
3148 goto out;
3149 }
3150
3151#ifdef CONFIG_NET_CLS_ACT
3152 if (skb->tc_verd & TC_NCLS) {
3153 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3154 goto ncls;
3155 }
3156#endif
3157
3158 list_for_each_entry_rcu(ptype, &ptype_all, list) {
3159 if (!ptype->dev || ptype->dev == skb->dev) {
3160 if (pt_prev)
3161 ret = deliver_skb(skb, pt_prev, orig_dev);
3162 pt_prev = ptype;
3163 }
3164 }
3165
3166#ifdef CONFIG_NET_CLS_ACT
3167 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3168 if (!skb)
3169 goto out;
3170ncls:
3171#endif
3172
3173 rx_handler = rcu_dereference(skb->dev->rx_handler);
3174 if (rx_handler) {
3175 if (pt_prev) {
3176 ret = deliver_skb(skb, pt_prev, orig_dev);
3177 pt_prev = NULL;
3178 }
3179 switch (rx_handler(&skb)) {
3180 case RX_HANDLER_CONSUMED:
3181 goto out;
3182 case RX_HANDLER_ANOTHER:
3183 goto another_round;
3184 case RX_HANDLER_EXACT:
3185 deliver_exact = true;
3186 case RX_HANDLER_PASS:
3187 break;
3188 default:
3189 BUG();
3190 }
3191 }
3192
3193 if (vlan_tx_tag_present(skb)) {
3194 if (pt_prev) {
3195 ret = deliver_skb(skb, pt_prev, orig_dev);
3196 pt_prev = NULL;
3197 }
3198 if (vlan_do_receive(&skb)) {
3199 ret = __netif_receive_skb(skb);
3200 goto out;
3201 } else if (unlikely(!skb))
3202 goto out;
3203 }
3204
3205 /* deliver only exact match when indicated */
3206 null_or_dev = deliver_exact ? skb->dev : NULL;
3207
3208 type = skb->protocol;
3209 list_for_each_entry_rcu(ptype,
3210 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
3211 if (ptype->type == type &&
3212 (ptype->dev == null_or_dev || ptype->dev == skb->dev ||
3213 ptype->dev == orig_dev)) {
3214 if (pt_prev)
3215 ret = deliver_skb(skb, pt_prev, orig_dev);
3216 pt_prev = ptype;
3217 }
3218 }
3219
3220 if (pt_prev) {
3221 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
3222 } else {
3223 atomic_long_inc(&skb->dev->rx_dropped);
3224 kfree_skb(skb);
3225 /* Jamal, now you will not able to escape explaining
3226 * me how you were going to use this. :-)
3227 */
3228 ret = NET_RX_DROP;
3229 }
3230
3231out:
3232 rcu_read_unlock();
3233 return ret;
3234}
3235
3236/**
3237 * netif_receive_skb - process receive buffer from network
3238 * @skb: buffer to process
3239 *
3240 * netif_receive_skb() is the main receive data processing function.
3241 * It always succeeds. The buffer may be dropped during processing
3242 * for congestion control or by the protocol layers.
3243 *
3244 * This function may only be called from softirq context and interrupts
3245 * should be enabled.
3246 *
3247 * Return values (usually ignored):
3248 * NET_RX_SUCCESS: no congestion
3249 * NET_RX_DROP: packet was dropped
3250 */
3251int netif_receive_skb(struct sk_buff *skb)
3252{
3253 if (netdev_tstamp_prequeue)
3254 net_timestamp_check(skb);
3255
3256 if (skb_defer_rx_timestamp(skb))
3257 return NET_RX_SUCCESS;
3258
3259#ifdef CONFIG_RPS
3260 {
3261 struct rps_dev_flow voidflow, *rflow = &voidflow;
3262 int cpu, ret;
3263
3264 rcu_read_lock();
3265
3266 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3267
3268 if (cpu >= 0) {
3269 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3270 rcu_read_unlock();
3271 } else {
3272 rcu_read_unlock();
3273 ret = __netif_receive_skb(skb);
3274 }
3275
3276 return ret;
3277 }
3278#else
3279 return __netif_receive_skb(skb);
3280#endif
3281}
3282EXPORT_SYMBOL(netif_receive_skb);
3283
3284/* Network device is going away, flush any packets still pending
3285 * Called with irqs disabled.
3286 */
3287static void flush_backlog(void *arg)
3288{
3289 struct net_device *dev = arg;
3290 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3291 struct sk_buff *skb, *tmp;
3292
3293 rps_lock(sd);
3294 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3295 if (skb->dev == dev) {
3296 __skb_unlink(skb, &sd->input_pkt_queue);
3297 kfree_skb(skb);
3298 input_queue_head_incr(sd);
3299 }
3300 }
3301 rps_unlock(sd);
3302
3303 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3304 if (skb->dev == dev) {
3305 __skb_unlink(skb, &sd->process_queue);
3306 kfree_skb(skb);
3307 input_queue_head_incr(sd);
3308 }
3309 }
3310}
3311
3312static int napi_gro_complete(struct sk_buff *skb)
3313{
3314 struct packet_type *ptype;
3315 __be16 type = skb->protocol;
3316 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3317 int err = -ENOENT;
3318
3319 if (NAPI_GRO_CB(skb)->count == 1) {
3320 skb_shinfo(skb)->gso_size = 0;
3321 goto out;
3322 }
3323
3324 rcu_read_lock();
3325 list_for_each_entry_rcu(ptype, head, list) {
3326 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3327 continue;
3328
3329 err = ptype->gro_complete(skb);
3330 break;
3331 }
3332 rcu_read_unlock();
3333
3334 if (err) {
3335 WARN_ON(&ptype->list == head);
3336 kfree_skb(skb);
3337 return NET_RX_SUCCESS;
3338 }
3339
3340out:
3341 return netif_receive_skb(skb);
3342}
3343
3344inline void napi_gro_flush(struct napi_struct *napi)
3345{
3346 struct sk_buff *skb, *next;
3347
3348 for (skb = napi->gro_list; skb; skb = next) {
3349 next = skb->next;
3350 skb->next = NULL;
3351 napi_gro_complete(skb);
3352 }
3353
3354 napi->gro_count = 0;
3355 napi->gro_list = NULL;
3356}
3357EXPORT_SYMBOL(napi_gro_flush);
3358
3359enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3360{
3361 struct sk_buff **pp = NULL;
3362 struct packet_type *ptype;
3363 __be16 type = skb->protocol;
3364 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3365 int same_flow;
3366 int mac_len;
3367 enum gro_result ret;
3368
3369 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
3370 goto normal;
3371
3372 if (skb_is_gso(skb) || skb_has_frag_list(skb))
3373 goto normal;
3374
3375 rcu_read_lock();
3376 list_for_each_entry_rcu(ptype, head, list) {
3377 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3378 continue;
3379
3380 skb_set_network_header(skb, skb_gro_offset(skb));
3381 mac_len = skb->network_header - skb->mac_header;
3382 skb->mac_len = mac_len;
3383 NAPI_GRO_CB(skb)->same_flow = 0;
3384 NAPI_GRO_CB(skb)->flush = 0;
3385 NAPI_GRO_CB(skb)->free = 0;
3386
3387 pp = ptype->gro_receive(&napi->gro_list, skb);
3388 break;
3389 }
3390 rcu_read_unlock();
3391
3392 if (&ptype->list == head)
3393 goto normal;
3394
3395 same_flow = NAPI_GRO_CB(skb)->same_flow;
3396 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
3397
3398 if (pp) {
3399 struct sk_buff *nskb = *pp;
3400
3401 *pp = nskb->next;
3402 nskb->next = NULL;
3403 napi_gro_complete(nskb);
3404 napi->gro_count--;
3405 }
3406
3407 if (same_flow)
3408 goto ok;
3409
3410 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
3411 goto normal;
3412
3413 napi->gro_count++;
3414 NAPI_GRO_CB(skb)->count = 1;
3415 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
3416 skb->next = napi->gro_list;
3417 napi->gro_list = skb;
3418 ret = GRO_HELD;
3419
3420pull:
3421 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3422 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3423
3424 BUG_ON(skb->end - skb->tail < grow);
3425
3426 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3427
3428 skb->tail += grow;
3429 skb->data_len -= grow;
3430
3431 skb_shinfo(skb)->frags[0].page_offset += grow;
3432 skb_shinfo(skb)->frags[0].size -= grow;
3433
3434 if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
3435 put_page(skb_shinfo(skb)->frags[0].page);
3436 memmove(skb_shinfo(skb)->frags,
3437 skb_shinfo(skb)->frags + 1,
3438 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
3439 }
3440 }
3441
3442ok:
3443 return ret;
3444
3445normal:
3446 ret = GRO_NORMAL;
3447 goto pull;
3448}
3449EXPORT_SYMBOL(dev_gro_receive);
3450
3451static inline gro_result_t
3452__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3453{
3454 struct sk_buff *p;
3455
3456 for (p = napi->gro_list; p; p = p->next) {
3457 unsigned long diffs;
3458
3459 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3460 diffs |= p->vlan_tci ^ skb->vlan_tci;
3461 diffs |= compare_ether_header(skb_mac_header(p),
3462 skb_gro_mac_header(skb));
3463 NAPI_GRO_CB(p)->same_flow = !diffs;
3464 NAPI_GRO_CB(p)->flush = 0;
3465 }
3466
3467 return dev_gro_receive(napi, skb);
3468}
3469
3470gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
3471{
3472 switch (ret) {
3473 case GRO_NORMAL:
3474 if (netif_receive_skb(skb))
3475 ret = GRO_DROP;
3476 break;
3477
3478 case GRO_DROP:
3479 case GRO_MERGED_FREE:
3480 kfree_skb(skb);
3481 break;
3482
3483 case GRO_HELD:
3484 case GRO_MERGED:
3485 break;
3486 }
3487
3488 return ret;
3489}
3490EXPORT_SYMBOL(napi_skb_finish);
3491
3492void skb_gro_reset_offset(struct sk_buff *skb)
3493{
3494 NAPI_GRO_CB(skb)->data_offset = 0;
3495 NAPI_GRO_CB(skb)->frag0 = NULL;
3496 NAPI_GRO_CB(skb)->frag0_len = 0;
3497
3498 if (skb->mac_header == skb->tail &&
3499 !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
3500 NAPI_GRO_CB(skb)->frag0 =
3501 page_address(skb_shinfo(skb)->frags[0].page) +
3502 skb_shinfo(skb)->frags[0].page_offset;
3503 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
3504 }
3505}
3506EXPORT_SYMBOL(skb_gro_reset_offset);
3507
3508gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3509{
3510 skb_gro_reset_offset(skb);
3511
3512 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
3513}
3514EXPORT_SYMBOL(napi_gro_receive);
3515
3516static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
3517{
3518 __skb_pull(skb, skb_headlen(skb));
3519 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
3520 skb->vlan_tci = 0;
3521 skb->dev = napi->dev;
3522 skb->skb_iif = 0;
3523
3524 napi->skb = skb;
3525}
3526
3527struct sk_buff *napi_get_frags(struct napi_struct *napi)
3528{
3529 struct sk_buff *skb = napi->skb;
3530
3531 if (!skb) {
3532 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3533 if (skb)
3534 napi->skb = skb;
3535 }
3536 return skb;
3537}
3538EXPORT_SYMBOL(napi_get_frags);
3539
3540gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3541 gro_result_t ret)
3542{
3543 switch (ret) {
3544 case GRO_NORMAL:
3545 case GRO_HELD:
3546 skb->protocol = eth_type_trans(skb, skb->dev);
3547
3548 if (ret == GRO_HELD)
3549 skb_gro_pull(skb, -ETH_HLEN);
3550 else if (netif_receive_skb(skb))
3551 ret = GRO_DROP;
3552 break;
3553
3554 case GRO_DROP:
3555 case GRO_MERGED_FREE:
3556 napi_reuse_skb(napi, skb);
3557 break;
3558
3559 case GRO_MERGED:
3560 break;
3561 }
3562
3563 return ret;
3564}
3565EXPORT_SYMBOL(napi_frags_finish);
3566
3567struct sk_buff *napi_frags_skb(struct napi_struct *napi)
3568{
3569 struct sk_buff *skb = napi->skb;
3570 struct ethhdr *eth;
3571 unsigned int hlen;
3572 unsigned int off;
3573
3574 napi->skb = NULL;
3575
3576 skb_reset_mac_header(skb);
3577 skb_gro_reset_offset(skb);
3578
3579 off = skb_gro_offset(skb);
3580 hlen = off + sizeof(*eth);
3581 eth = skb_gro_header_fast(skb, off);
3582 if (skb_gro_header_hard(skb, hlen)) {
3583 eth = skb_gro_header_slow(skb, hlen, off);
3584 if (unlikely(!eth)) {
3585 napi_reuse_skb(napi, skb);
3586 skb = NULL;
3587 goto out;
3588 }
3589 }
3590
3591 skb_gro_pull(skb, sizeof(*eth));
3592
3593 /*
3594 * This works because the only protocols we care about don't require
3595 * special handling. We'll fix it up properly at the end.
3596 */
3597 skb->protocol = eth->h_proto;
3598
3599out:
3600 return skb;
3601}
3602EXPORT_SYMBOL(napi_frags_skb);
3603
3604gro_result_t napi_gro_frags(struct napi_struct *napi)
3605{
3606 struct sk_buff *skb = napi_frags_skb(napi);
3607
3608 if (!skb)
3609 return GRO_DROP;
3610
3611 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
3612}
3613EXPORT_SYMBOL(napi_gro_frags);
3614
3615/*
3616 * net_rps_action sends any pending IPI's for rps.
3617 * Note: called with local irq disabled, but exits with local irq enabled.
3618 */
3619static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3620{
3621#ifdef CONFIG_RPS
3622 struct softnet_data *remsd = sd->rps_ipi_list;
3623
3624 if (remsd) {
3625 sd->rps_ipi_list = NULL;
3626
3627 local_irq_enable();
3628
3629 /* Send pending IPI's to kick RPS processing on remote cpus. */
3630 while (remsd) {
3631 struct softnet_data *next = remsd->rps_ipi_next;
3632
3633 if (cpu_online(remsd->cpu))
3634 __smp_call_function_single(remsd->cpu,
3635 &remsd->csd, 0);
3636 remsd = next;
3637 }
3638 } else
3639#endif
3640 local_irq_enable();
3641}
3642
3643static int process_backlog(struct napi_struct *napi, int quota)
3644{
3645 int work = 0;
3646 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
3647
3648#ifdef CONFIG_RPS
3649 /* Check if we have pending ipi, its better to send them now,
3650 * not waiting net_rx_action() end.
3651 */
3652 if (sd->rps_ipi_list) {
3653 local_irq_disable();
3654 net_rps_action_and_irq_enable(sd);
3655 }
3656#endif
3657 napi->weight = weight_p;
3658 local_irq_disable();
3659 while (work < quota) {
3660 struct sk_buff *skb;
3661 unsigned int qlen;
3662
3663 while ((skb = __skb_dequeue(&sd->process_queue))) {
3664 local_irq_enable();
3665 __netif_receive_skb(skb);
3666 local_irq_disable();
3667 input_queue_head_incr(sd);
3668 if (++work >= quota) {
3669 local_irq_enable();
3670 return work;
3671 }
3672 }
3673
3674 rps_lock(sd);
3675 qlen = skb_queue_len(&sd->input_pkt_queue);
3676 if (qlen)
3677 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3678 &sd->process_queue);
3679
3680 if (qlen < quota - work) {
3681 /*
3682 * Inline a custom version of __napi_complete().
3683 * only current cpu owns and manipulates this napi,
3684 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3685 * we can use a plain write instead of clear_bit(),
3686 * and we dont need an smp_mb() memory barrier.
3687 */
3688 list_del(&napi->poll_list);
3689 napi->state = 0;
3690
3691 quota = work + qlen;
3692 }
3693 rps_unlock(sd);
3694 }
3695 local_irq_enable();
3696
3697 return work;
3698}
3699
3700/**
3701 * __napi_schedule - schedule for receive
3702 * @n: entry to schedule
3703 *
3704 * The entry's receive function will be scheduled to run
3705 */
3706void __napi_schedule(struct napi_struct *n)
3707{
3708 unsigned long flags;
3709
3710 local_irq_save(flags);
3711 ____napi_schedule(&__get_cpu_var(softnet_data), n);
3712 local_irq_restore(flags);
3713}
3714EXPORT_SYMBOL(__napi_schedule);
3715
3716void __napi_complete(struct napi_struct *n)
3717{
3718 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3719 BUG_ON(n->gro_list);
3720
3721 list_del(&n->poll_list);
3722 smp_mb__before_clear_bit();
3723 clear_bit(NAPI_STATE_SCHED, &n->state);
3724}
3725EXPORT_SYMBOL(__napi_complete);
3726
3727void napi_complete(struct napi_struct *n)
3728{
3729 unsigned long flags;
3730
3731 /*
3732 * don't let napi dequeue from the cpu poll list
3733 * just in case its running on a different cpu
3734 */
3735 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3736 return;
3737
3738 napi_gro_flush(n);
3739 local_irq_save(flags);
3740 __napi_complete(n);
3741 local_irq_restore(flags);
3742}
3743EXPORT_SYMBOL(napi_complete);
3744
3745void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3746 int (*poll)(struct napi_struct *, int), int weight)
3747{
3748 INIT_LIST_HEAD(&napi->poll_list);
3749 napi->gro_count = 0;
3750 napi->gro_list = NULL;
3751 napi->skb = NULL;
3752 napi->poll = poll;
3753 napi->weight = weight;
3754 list_add(&napi->dev_list, &dev->napi_list);
3755 napi->dev = dev;
3756#ifdef CONFIG_NETPOLL
3757 spin_lock_init(&napi->poll_lock);
3758 napi->poll_owner = -1;
3759#endif
3760 set_bit(NAPI_STATE_SCHED, &napi->state);
3761}
3762EXPORT_SYMBOL(netif_napi_add);
3763
3764void netif_napi_del(struct napi_struct *napi)
3765{
3766 struct sk_buff *skb, *next;
3767
3768 list_del_init(&napi->dev_list);
3769 napi_free_frags(napi);
3770
3771 for (skb = napi->gro_list; skb; skb = next) {
3772 next = skb->next;
3773 skb->next = NULL;
3774 kfree_skb(skb);
3775 }
3776
3777 napi->gro_list = NULL;
3778 napi->gro_count = 0;
3779}
3780EXPORT_SYMBOL(netif_napi_del);
3781
3782static void net_rx_action(struct softirq_action *h)
3783{
3784 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3785 unsigned long time_limit = jiffies + 2;
3786 int budget = netdev_budget;
3787 void *have;
3788
3789 local_irq_disable();
3790
3791 while (!list_empty(&sd->poll_list)) {
3792 struct napi_struct *n;
3793 int work, weight;
3794
3795 /* If softirq window is exhuasted then punt.
3796 * Allow this to run for 2 jiffies since which will allow
3797 * an average latency of 1.5/HZ.
3798 */
3799 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
3800 goto softnet_break;
3801
3802 local_irq_enable();
3803
3804 /* Even though interrupts have been re-enabled, this
3805 * access is safe because interrupts can only add new
3806 * entries to the tail of this list, and only ->poll()
3807 * calls can remove this head entry from the list.
3808 */
3809 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
3810
3811 have = netpoll_poll_lock(n);
3812
3813 weight = n->weight;
3814
3815 /* This NAPI_STATE_SCHED test is for avoiding a race
3816 * with netpoll's poll_napi(). Only the entity which
3817 * obtains the lock and sees NAPI_STATE_SCHED set will
3818 * actually make the ->poll() call. Therefore we avoid
3819 * accidentally calling ->poll() when NAPI is not scheduled.
3820 */
3821 work = 0;
3822 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
3823 work = n->poll(n, weight);
3824 trace_napi_poll(n);
3825 }
3826
3827 WARN_ON_ONCE(work > weight);
3828
3829 budget -= work;
3830
3831 local_irq_disable();
3832
3833 /* Drivers must not modify the NAPI state if they
3834 * consume the entire weight. In such cases this code
3835 * still "owns" the NAPI instance and therefore can
3836 * move the instance around on the list at-will.
3837 */
3838 if (unlikely(work == weight)) {
3839 if (unlikely(napi_disable_pending(n))) {
3840 local_irq_enable();
3841 napi_complete(n);
3842 local_irq_disable();
3843 } else
3844 list_move_tail(&n->poll_list, &sd->poll_list);
3845 }
3846
3847 netpoll_poll_unlock(have);
3848 }
3849out:
3850 net_rps_action_and_irq_enable(sd);
3851
3852#ifdef CONFIG_NET_DMA
3853 /*
3854 * There may not be any more sk_buffs coming right now, so push
3855 * any pending DMA copies to hardware
3856 */
3857 dma_issue_pending_all();
3858#endif
3859
3860 return;
3861
3862softnet_break:
3863 sd->time_squeeze++;
3864 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3865 goto out;
3866}
3867
3868static gifconf_func_t *gifconf_list[NPROTO];
3869
3870/**
3871 * register_gifconf - register a SIOCGIF handler
3872 * @family: Address family
3873 * @gifconf: Function handler
3874 *
3875 * Register protocol dependent address dumping routines. The handler
3876 * that is passed must not be freed or reused until it has been replaced
3877 * by another handler.
3878 */
3879int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
3880{
3881 if (family >= NPROTO)
3882 return -EINVAL;
3883 gifconf_list[family] = gifconf;
3884 return 0;
3885}
3886EXPORT_SYMBOL(register_gifconf);
3887
3888
3889/*
3890 * Map an interface index to its name (SIOCGIFNAME)
3891 */
3892
3893/*
3894 * We need this ioctl for efficient implementation of the
3895 * if_indextoname() function required by the IPv6 API. Without
3896 * it, we would have to search all the interfaces to find a
3897 * match. --pb
3898 */
3899
3900static int dev_ifname(struct net *net, struct ifreq __user *arg)
3901{
3902 struct net_device *dev;
3903 struct ifreq ifr;
3904
3905 /*
3906 * Fetch the caller's info block.
3907 */
3908
3909 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3910 return -EFAULT;
3911
3912 rcu_read_lock();
3913 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
3914 if (!dev) {
3915 rcu_read_unlock();
3916 return -ENODEV;
3917 }
3918
3919 strcpy(ifr.ifr_name, dev->name);
3920 rcu_read_unlock();
3921
3922 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3923 return -EFAULT;
3924 return 0;
3925}
3926
3927/*
3928 * Perform a SIOCGIFCONF call. This structure will change
3929 * size eventually, and there is nothing I can do about it.
3930 * Thus we will need a 'compatibility mode'.
3931 */
3932
3933static int dev_ifconf(struct net *net, char __user *arg)
3934{
3935 struct ifconf ifc;
3936 struct net_device *dev;
3937 char __user *pos;
3938 int len;
3939 int total;
3940 int i;
3941
3942 /*
3943 * Fetch the caller's info block.
3944 */
3945
3946 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
3947 return -EFAULT;
3948
3949 pos = ifc.ifc_buf;
3950 len = ifc.ifc_len;
3951
3952 /*
3953 * Loop over the interfaces, and write an info block for each.
3954 */
3955
3956 total = 0;
3957 for_each_netdev(net, dev) {
3958 for (i = 0; i < NPROTO; i++) {
3959 if (gifconf_list[i]) {
3960 int done;
3961 if (!pos)
3962 done = gifconf_list[i](dev, NULL, 0);
3963 else
3964 done = gifconf_list[i](dev, pos + total,
3965 len - total);
3966 if (done < 0)
3967 return -EFAULT;
3968 total += done;
3969 }
3970 }
3971 }
3972
3973 /*
3974 * All done. Write the updated control block back to the caller.
3975 */
3976 ifc.ifc_len = total;
3977
3978 /*
3979 * Both BSD and Solaris return 0 here, so we do too.
3980 */
3981 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
3982}
3983
3984#ifdef CONFIG_PROC_FS
3985/*
3986 * This is invoked by the /proc filesystem handler to display a device
3987 * in detail.
3988 */
3989void *dev_seq_start(struct seq_file *seq, loff_t *pos)
3990 __acquires(RCU)
3991{
3992 struct net *net = seq_file_net(seq);
3993 loff_t off;
3994 struct net_device *dev;
3995
3996 rcu_read_lock();
3997 if (!*pos)
3998 return SEQ_START_TOKEN;
3999
4000 off = 1;
4001 for_each_netdev_rcu(net, dev)
4002 if (off++ == *pos)
4003 return dev;
4004
4005 return NULL;
4006}
4007
4008void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4009{
4010 struct net_device *dev = v;
4011
4012 if (v == SEQ_START_TOKEN)
4013 dev = first_net_device_rcu(seq_file_net(seq));
4014 else
4015 dev = next_net_device_rcu(dev);
4016
4017 ++*pos;
4018 return dev;
4019}
4020
4021void dev_seq_stop(struct seq_file *seq, void *v)
4022 __releases(RCU)
4023{
4024 rcu_read_unlock();
4025}
4026
4027static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
4028{
4029 struct rtnl_link_stats64 temp;
4030 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
4031
4032 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
4033 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
4034 dev->name, stats->rx_bytes, stats->rx_packets,
4035 stats->rx_errors,
4036 stats->rx_dropped + stats->rx_missed_errors,
4037 stats->rx_fifo_errors,
4038 stats->rx_length_errors + stats->rx_over_errors +
4039 stats->rx_crc_errors + stats->rx_frame_errors,
4040 stats->rx_compressed, stats->multicast,
4041 stats->tx_bytes, stats->tx_packets,
4042 stats->tx_errors, stats->tx_dropped,
4043 stats->tx_fifo_errors, stats->collisions,
4044 stats->tx_carrier_errors +
4045 stats->tx_aborted_errors +
4046 stats->tx_window_errors +
4047 stats->tx_heartbeat_errors,
4048 stats->tx_compressed);
4049}
4050
4051/*
4052 * Called from the PROCfs module. This now uses the new arbitrary sized
4053 * /proc/net interface to create /proc/net/dev
4054 */
4055static int dev_seq_show(struct seq_file *seq, void *v)
4056{
4057 if (v == SEQ_START_TOKEN)
4058 seq_puts(seq, "Inter-| Receive "
4059 " | Transmit\n"
4060 " face |bytes packets errs drop fifo frame "
4061 "compressed multicast|bytes packets errs "
4062 "drop fifo colls carrier compressed\n");
4063 else
4064 dev_seq_printf_stats(seq, v);
4065 return 0;
4066}
4067
4068static struct softnet_data *softnet_get_online(loff_t *pos)
4069{
4070 struct softnet_data *sd = NULL;
4071
4072 while (*pos < nr_cpu_ids)
4073 if (cpu_online(*pos)) {
4074 sd = &per_cpu(softnet_data, *pos);
4075 break;
4076 } else
4077 ++*pos;
4078 return sd;
4079}
4080
4081static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4082{
4083 return softnet_get_online(pos);
4084}
4085
4086static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4087{
4088 ++*pos;
4089 return softnet_get_online(pos);
4090}
4091
4092static void softnet_seq_stop(struct seq_file *seq, void *v)
4093{
4094}
4095
4096static int softnet_seq_show(struct seq_file *seq, void *v)
4097{
4098 struct softnet_data *sd = v;
4099
4100 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
4101 sd->processed, sd->dropped, sd->time_squeeze, 0,
4102 0, 0, 0, 0, /* was fastroute */
4103 sd->cpu_collision, sd->received_rps);
4104 return 0;
4105}
4106
4107static const struct seq_operations dev_seq_ops = {
4108 .start = dev_seq_start,
4109 .next = dev_seq_next,
4110 .stop = dev_seq_stop,
4111 .show = dev_seq_show,
4112};
4113
4114static int dev_seq_open(struct inode *inode, struct file *file)
4115{
4116 return seq_open_net(inode, file, &dev_seq_ops,
4117 sizeof(struct seq_net_private));
4118}
4119
4120static const struct file_operations dev_seq_fops = {
4121 .owner = THIS_MODULE,
4122 .open = dev_seq_open,
4123 .read = seq_read,
4124 .llseek = seq_lseek,
4125 .release = seq_release_net,
4126};
4127
4128static const struct seq_operations softnet_seq_ops = {
4129 .start = softnet_seq_start,
4130 .next = softnet_seq_next,
4131 .stop = softnet_seq_stop,
4132 .show = softnet_seq_show,
4133};
4134
4135static int softnet_seq_open(struct inode *inode, struct file *file)
4136{
4137 return seq_open(file, &softnet_seq_ops);
4138}
4139
4140static const struct file_operations softnet_seq_fops = {
4141 .owner = THIS_MODULE,
4142 .open = softnet_seq_open,
4143 .read = seq_read,
4144 .llseek = seq_lseek,
4145 .release = seq_release,
4146};
4147
4148static void *ptype_get_idx(loff_t pos)
4149{
4150 struct packet_type *pt = NULL;
4151 loff_t i = 0;
4152 int t;
4153
4154 list_for_each_entry_rcu(pt, &ptype_all, list) {
4155 if (i == pos)
4156 return pt;
4157 ++i;
4158 }
4159
4160 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
4161 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4162 if (i == pos)
4163 return pt;
4164 ++i;
4165 }
4166 }
4167 return NULL;
4168}
4169
4170static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
4171 __acquires(RCU)
4172{
4173 rcu_read_lock();
4174 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4175}
4176
4177static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4178{
4179 struct packet_type *pt;
4180 struct list_head *nxt;
4181 int hash;
4182
4183 ++*pos;
4184 if (v == SEQ_START_TOKEN)
4185 return ptype_get_idx(0);
4186
4187 pt = v;
4188 nxt = pt->list.next;
4189 if (pt->type == htons(ETH_P_ALL)) {
4190 if (nxt != &ptype_all)
4191 goto found;
4192 hash = 0;
4193 nxt = ptype_base[0].next;
4194 } else
4195 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
4196
4197 while (nxt == &ptype_base[hash]) {
4198 if (++hash >= PTYPE_HASH_SIZE)
4199 return NULL;
4200 nxt = ptype_base[hash].next;
4201 }
4202found:
4203 return list_entry(nxt, struct packet_type, list);
4204}
4205
4206static void ptype_seq_stop(struct seq_file *seq, void *v)
4207 __releases(RCU)
4208{
4209 rcu_read_unlock();
4210}
4211
4212static int ptype_seq_show(struct seq_file *seq, void *v)
4213{
4214 struct packet_type *pt = v;
4215
4216 if (v == SEQ_START_TOKEN)
4217 seq_puts(seq, "Type Device Function\n");
4218 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
4219 if (pt->type == htons(ETH_P_ALL))
4220 seq_puts(seq, "ALL ");
4221 else
4222 seq_printf(seq, "%04x", ntohs(pt->type));
4223
4224 seq_printf(seq, " %-8s %pF\n",
4225 pt->dev ? pt->dev->name : "", pt->func);
4226 }
4227
4228 return 0;
4229}
4230
4231static const struct seq_operations ptype_seq_ops = {
4232 .start = ptype_seq_start,
4233 .next = ptype_seq_next,
4234 .stop = ptype_seq_stop,
4235 .show = ptype_seq_show,
4236};
4237
4238static int ptype_seq_open(struct inode *inode, struct file *file)
4239{
4240 return seq_open_net(inode, file, &ptype_seq_ops,
4241 sizeof(struct seq_net_private));
4242}
4243
4244static const struct file_operations ptype_seq_fops = {
4245 .owner = THIS_MODULE,
4246 .open = ptype_seq_open,
4247 .read = seq_read,
4248 .llseek = seq_lseek,
4249 .release = seq_release_net,
4250};
4251
4252
4253static int __net_init dev_proc_net_init(struct net *net)
4254{
4255 int rc = -ENOMEM;
4256
4257 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
4258 goto out;
4259 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
4260 goto out_dev;
4261 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
4262 goto out_softnet;
4263
4264 if (wext_proc_init(net))
4265 goto out_ptype;
4266 rc = 0;
4267out:
4268 return rc;
4269out_ptype:
4270 proc_net_remove(net, "ptype");
4271out_softnet:
4272 proc_net_remove(net, "softnet_stat");
4273out_dev:
4274 proc_net_remove(net, "dev");
4275 goto out;
4276}
4277
4278static void __net_exit dev_proc_net_exit(struct net *net)
4279{
4280 wext_proc_exit(net);
4281
4282 proc_net_remove(net, "ptype");
4283 proc_net_remove(net, "softnet_stat");
4284 proc_net_remove(net, "dev");
4285}
4286
4287static struct pernet_operations __net_initdata dev_proc_ops = {
4288 .init = dev_proc_net_init,
4289 .exit = dev_proc_net_exit,
4290};
4291
4292static int __init dev_proc_init(void)
4293{
4294 return register_pernet_subsys(&dev_proc_ops);
4295}
4296#else
4297#define dev_proc_init() 0
4298#endif /* CONFIG_PROC_FS */
4299
4300
4301/**
4302 * netdev_set_master - set up master pointer
4303 * @slave: slave device
4304 * @master: new master device
4305 *
4306 * Changes the master device of the slave. Pass %NULL to break the
4307 * bonding. The caller must hold the RTNL semaphore. On a failure
4308 * a negative errno code is returned. On success the reference counts
4309 * are adjusted and the function returns zero.
4310 */
4311int netdev_set_master(struct net_device *slave, struct net_device *master)
4312{
4313 struct net_device *old = slave->master;
4314
4315 ASSERT_RTNL();
4316
4317 if (master) {
4318 if (old)
4319 return -EBUSY;
4320 dev_hold(master);
4321 }
4322
4323 slave->master = master;
4324
4325 if (old)
4326 dev_put(old);
4327 return 0;
4328}
4329EXPORT_SYMBOL(netdev_set_master);
4330
4331/**
4332 * netdev_set_bond_master - set up bonding master/slave pair
4333 * @slave: slave device
4334 * @master: new master device
4335 *
4336 * Changes the master device of the slave. Pass %NULL to break the
4337 * bonding. The caller must hold the RTNL semaphore. On a failure
4338 * a negative errno code is returned. On success %RTM_NEWLINK is sent
4339 * to the routing socket and the function returns zero.
4340 */
4341int netdev_set_bond_master(struct net_device *slave, struct net_device *master)
4342{
4343 int err;
4344
4345 ASSERT_RTNL();
4346
4347 err = netdev_set_master(slave, master);
4348 if (err)
4349 return err;
4350 if (master)
4351 slave->flags |= IFF_SLAVE;
4352 else
4353 slave->flags &= ~IFF_SLAVE;
4354
4355 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4356 return 0;
4357}
4358EXPORT_SYMBOL(netdev_set_bond_master);
4359
4360static void dev_change_rx_flags(struct net_device *dev, int flags)
4361{
4362 const struct net_device_ops *ops = dev->netdev_ops;
4363
4364 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4365 ops->ndo_change_rx_flags(dev, flags);
4366}
4367
4368static int __dev_set_promiscuity(struct net_device *dev, int inc)
4369{
4370 unsigned short old_flags = dev->flags;
4371 uid_t uid;
4372 gid_t gid;
4373
4374 ASSERT_RTNL();
4375
4376 dev->flags |= IFF_PROMISC;
4377 dev->promiscuity += inc;
4378 if (dev->promiscuity == 0) {
4379 /*
4380 * Avoid overflow.
4381 * If inc causes overflow, untouch promisc and return error.
4382 */
4383 if (inc < 0)
4384 dev->flags &= ~IFF_PROMISC;
4385 else {
4386 dev->promiscuity -= inc;
4387 printk(KERN_WARNING "%s: promiscuity touches roof, "
4388 "set promiscuity failed, promiscuity feature "
4389 "of device might be broken.\n", dev->name);
4390 return -EOVERFLOW;
4391 }
4392 }
4393 if (dev->flags != old_flags) {
4394 printk(KERN_INFO "device %s %s promiscuous mode\n",
4395 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
4396 "left");
4397 if (audit_enabled) {
4398 current_uid_gid(&uid, &gid);
4399 audit_log(current->audit_context, GFP_ATOMIC,
4400 AUDIT_ANOM_PROMISCUOUS,
4401 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4402 dev->name, (dev->flags & IFF_PROMISC),
4403 (old_flags & IFF_PROMISC),
4404 audit_get_loginuid(current),
4405 uid, gid,
4406 audit_get_sessionid(current));
4407 }
4408
4409 dev_change_rx_flags(dev, IFF_PROMISC);
4410 }
4411 return 0;
4412}
4413
4414/**
4415 * dev_set_promiscuity - update promiscuity count on a device
4416 * @dev: device
4417 * @inc: modifier
4418 *
4419 * Add or remove promiscuity from a device. While the count in the device
4420 * remains above zero the interface remains promiscuous. Once it hits zero
4421 * the device reverts back to normal filtering operation. A negative inc
4422 * value is used to drop promiscuity on the device.
4423 * Return 0 if successful or a negative errno code on error.
4424 */
4425int dev_set_promiscuity(struct net_device *dev, int inc)
4426{
4427 unsigned short old_flags = dev->flags;
4428 int err;
4429
4430 err = __dev_set_promiscuity(dev, inc);
4431 if (err < 0)
4432 return err;
4433 if (dev->flags != old_flags)
4434 dev_set_rx_mode(dev);
4435 return err;
4436}
4437EXPORT_SYMBOL(dev_set_promiscuity);
4438
4439/**
4440 * dev_set_allmulti - update allmulti count on a device
4441 * @dev: device
4442 * @inc: modifier
4443 *
4444 * Add or remove reception of all multicast frames to a device. While the
4445 * count in the device remains above zero the interface remains listening
4446 * to all interfaces. Once it hits zero the device reverts back to normal
4447 * filtering operation. A negative @inc value is used to drop the counter
4448 * when releasing a resource needing all multicasts.
4449 * Return 0 if successful or a negative errno code on error.
4450 */
4451
4452int dev_set_allmulti(struct net_device *dev, int inc)
4453{
4454 unsigned short old_flags = dev->flags;
4455
4456 ASSERT_RTNL();
4457
4458 dev->flags |= IFF_ALLMULTI;
4459 dev->allmulti += inc;
4460 if (dev->allmulti == 0) {
4461 /*
4462 * Avoid overflow.
4463 * If inc causes overflow, untouch allmulti and return error.
4464 */
4465 if (inc < 0)
4466 dev->flags &= ~IFF_ALLMULTI;
4467 else {
4468 dev->allmulti -= inc;
4469 printk(KERN_WARNING "%s: allmulti touches roof, "
4470 "set allmulti failed, allmulti feature of "
4471 "device might be broken.\n", dev->name);
4472 return -EOVERFLOW;
4473 }
4474 }
4475 if (dev->flags ^ old_flags) {
4476 dev_change_rx_flags(dev, IFF_ALLMULTI);
4477 dev_set_rx_mode(dev);
4478 }
4479 return 0;
4480}
4481EXPORT_SYMBOL(dev_set_allmulti);
4482
4483/*
4484 * Upload unicast and multicast address lists to device and
4485 * configure RX filtering. When the device doesn't support unicast
4486 * filtering it is put in promiscuous mode while unicast addresses
4487 * are present.
4488 */
4489void __dev_set_rx_mode(struct net_device *dev)
4490{
4491 const struct net_device_ops *ops = dev->netdev_ops;
4492
4493 /* dev_open will call this function so the list will stay sane. */
4494 if (!(dev->flags&IFF_UP))
4495 return;
4496
4497 if (!netif_device_present(dev))
4498 return;
4499
4500 if (ops->ndo_set_rx_mode)
4501 ops->ndo_set_rx_mode(dev);
4502 else {
4503 /* Unicast addresses changes may only happen under the rtnl,
4504 * therefore calling __dev_set_promiscuity here is safe.
4505 */
4506 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
4507 __dev_set_promiscuity(dev, 1);
4508 dev->uc_promisc = true;
4509 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
4510 __dev_set_promiscuity(dev, -1);
4511 dev->uc_promisc = false;
4512 }
4513
4514 if (ops->ndo_set_multicast_list)
4515 ops->ndo_set_multicast_list(dev);
4516 }
4517}
4518
4519void dev_set_rx_mode(struct net_device *dev)
4520{
4521 netif_addr_lock_bh(dev);
4522 __dev_set_rx_mode(dev);
4523 netif_addr_unlock_bh(dev);
4524}
4525
4526/**
4527 * dev_ethtool_get_settings - call device's ethtool_ops::get_settings()
4528 * @dev: device
4529 * @cmd: memory area for ethtool_ops::get_settings() result
4530 *
4531 * The cmd arg is initialized properly (cleared and
4532 * ethtool_cmd::cmd field set to ETHTOOL_GSET).
4533 *
4534 * Return device's ethtool_ops::get_settings() result value or
4535 * -EOPNOTSUPP when device doesn't expose
4536 * ethtool_ops::get_settings() operation.
4537 */
4538int dev_ethtool_get_settings(struct net_device *dev,
4539 struct ethtool_cmd *cmd)
4540{
4541 if (!dev->ethtool_ops || !dev->ethtool_ops->get_settings)
4542 return -EOPNOTSUPP;
4543
4544 memset(cmd, 0, sizeof(struct ethtool_cmd));
4545 cmd->cmd = ETHTOOL_GSET;
4546 return dev->ethtool_ops->get_settings(dev, cmd);
4547}
4548EXPORT_SYMBOL(dev_ethtool_get_settings);
4549
4550/**
4551 * dev_get_flags - get flags reported to userspace
4552 * @dev: device
4553 *
4554 * Get the combination of flag bits exported through APIs to userspace.
4555 */
4556unsigned dev_get_flags(const struct net_device *dev)
4557{
4558 unsigned flags;
4559
4560 flags = (dev->flags & ~(IFF_PROMISC |
4561 IFF_ALLMULTI |
4562 IFF_RUNNING |
4563 IFF_LOWER_UP |
4564 IFF_DORMANT)) |
4565 (dev->gflags & (IFF_PROMISC |
4566 IFF_ALLMULTI));
4567
4568 if (netif_running(dev)) {
4569 if (netif_oper_up(dev))
4570 flags |= IFF_RUNNING;
4571 if (netif_carrier_ok(dev))
4572 flags |= IFF_LOWER_UP;
4573 if (netif_dormant(dev))
4574 flags |= IFF_DORMANT;
4575 }
4576
4577 return flags;
4578}
4579EXPORT_SYMBOL(dev_get_flags);
4580
4581int __dev_change_flags(struct net_device *dev, unsigned int flags)
4582{
4583 int old_flags = dev->flags;
4584 int ret;
4585
4586 ASSERT_RTNL();
4587
4588 /*
4589 * Set the flags on our device.
4590 */
4591
4592 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4593 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4594 IFF_AUTOMEDIA)) |
4595 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4596 IFF_ALLMULTI));
4597
4598 /*
4599 * Load in the correct multicast list now the flags have changed.
4600 */
4601
4602 if ((old_flags ^ flags) & IFF_MULTICAST)
4603 dev_change_rx_flags(dev, IFF_MULTICAST);
4604
4605 dev_set_rx_mode(dev);
4606
4607 /*
4608 * Have we downed the interface. We handle IFF_UP ourselves
4609 * according to user attempts to set it, rather than blindly
4610 * setting it.
4611 */
4612
4613 ret = 0;
4614 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
4615 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
4616
4617 if (!ret)
4618 dev_set_rx_mode(dev);
4619 }
4620
4621 if ((flags ^ dev->gflags) & IFF_PROMISC) {
4622 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4623
4624 dev->gflags ^= IFF_PROMISC;
4625 dev_set_promiscuity(dev, inc);
4626 }
4627
4628 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4629 is important. Some (broken) drivers set IFF_PROMISC, when
4630 IFF_ALLMULTI is requested not asking us and not reporting.
4631 */
4632 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
4633 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4634
4635 dev->gflags ^= IFF_ALLMULTI;
4636 dev_set_allmulti(dev, inc);
4637 }
4638
4639 return ret;
4640}
4641
4642void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4643{
4644 unsigned int changes = dev->flags ^ old_flags;
4645
4646 if (changes & IFF_UP) {
4647 if (dev->flags & IFF_UP)
4648 call_netdevice_notifiers(NETDEV_UP, dev);
4649 else
4650 call_netdevice_notifiers(NETDEV_DOWN, dev);
4651 }
4652
4653 if (dev->flags & IFF_UP &&
4654 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4655 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4656}
4657
4658/**
4659 * dev_change_flags - change device settings
4660 * @dev: device
4661 * @flags: device state flags
4662 *
4663 * Change settings on device based state flags. The flags are
4664 * in the userspace exported format.
4665 */
4666int dev_change_flags(struct net_device *dev, unsigned flags)
4667{
4668 int ret, changes;
4669 int old_flags = dev->flags;
4670
4671 ret = __dev_change_flags(dev, flags);
4672 if (ret < 0)
4673 return ret;
4674
4675 changes = old_flags ^ dev->flags;
4676 if (changes)
4677 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
4678
4679 __dev_notify_flags(dev, old_flags);
4680 return ret;
4681}
4682EXPORT_SYMBOL(dev_change_flags);
4683
4684/**
4685 * dev_set_mtu - Change maximum transfer unit
4686 * @dev: device
4687 * @new_mtu: new transfer unit
4688 *
4689 * Change the maximum transfer size of the network device.
4690 */
4691int dev_set_mtu(struct net_device *dev, int new_mtu)
4692{
4693 const struct net_device_ops *ops = dev->netdev_ops;
4694 int err;
4695
4696 if (new_mtu == dev->mtu)
4697 return 0;
4698
4699 /* MTU must be positive. */
4700 if (new_mtu < 0)
4701 return -EINVAL;
4702
4703 if (!netif_device_present(dev))
4704 return -ENODEV;
4705
4706 err = 0;
4707 if (ops->ndo_change_mtu)
4708 err = ops->ndo_change_mtu(dev, new_mtu);
4709 else
4710 dev->mtu = new_mtu;
4711
4712 if (!err && dev->flags & IFF_UP)
4713 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
4714 return err;
4715}
4716EXPORT_SYMBOL(dev_set_mtu);
4717
4718/**
4719 * dev_set_group - Change group this device belongs to
4720 * @dev: device
4721 * @new_group: group this device should belong to
4722 */
4723void dev_set_group(struct net_device *dev, int new_group)
4724{
4725 dev->group = new_group;
4726}
4727EXPORT_SYMBOL(dev_set_group);
4728
4729/**
4730 * dev_set_mac_address - Change Media Access Control Address
4731 * @dev: device
4732 * @sa: new address
4733 *
4734 * Change the hardware (MAC) address of the device
4735 */
4736int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4737{
4738 const struct net_device_ops *ops = dev->netdev_ops;
4739 int err;
4740
4741 if (!ops->ndo_set_mac_address)
4742 return -EOPNOTSUPP;
4743 if (sa->sa_family != dev->type)
4744 return -EINVAL;
4745 if (!netif_device_present(dev))
4746 return -ENODEV;
4747 err = ops->ndo_set_mac_address(dev, sa);
4748 if (!err)
4749 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4750 return err;
4751}
4752EXPORT_SYMBOL(dev_set_mac_address);
4753
4754/*
4755 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
4756 */
4757static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
4758{
4759 int err;
4760 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
4761
4762 if (!dev)
4763 return -ENODEV;
4764
4765 switch (cmd) {
4766 case SIOCGIFFLAGS: /* Get interface flags */
4767 ifr->ifr_flags = (short) dev_get_flags(dev);
4768 return 0;
4769
4770 case SIOCGIFMETRIC: /* Get the metric on the interface
4771 (currently unused) */
4772 ifr->ifr_metric = 0;
4773 return 0;
4774
4775 case SIOCGIFMTU: /* Get the MTU of a device */
4776 ifr->ifr_mtu = dev->mtu;
4777 return 0;
4778
4779 case SIOCGIFHWADDR:
4780 if (!dev->addr_len)
4781 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4782 else
4783 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4784 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4785 ifr->ifr_hwaddr.sa_family = dev->type;
4786 return 0;
4787
4788 case SIOCGIFSLAVE:
4789 err = -EINVAL;
4790 break;
4791
4792 case SIOCGIFMAP:
4793 ifr->ifr_map.mem_start = dev->mem_start;
4794 ifr->ifr_map.mem_end = dev->mem_end;
4795 ifr->ifr_map.base_addr = dev->base_addr;
4796 ifr->ifr_map.irq = dev->irq;
4797 ifr->ifr_map.dma = dev->dma;
4798 ifr->ifr_map.port = dev->if_port;
4799 return 0;
4800
4801 case SIOCGIFINDEX:
4802 ifr->ifr_ifindex = dev->ifindex;
4803 return 0;
4804
4805 case SIOCGIFTXQLEN:
4806 ifr->ifr_qlen = dev->tx_queue_len;
4807 return 0;
4808
4809 default:
4810 /* dev_ioctl() should ensure this case
4811 * is never reached
4812 */
4813 WARN_ON(1);
4814 err = -ENOTTY;
4815 break;
4816
4817 }
4818 return err;
4819}
4820
4821/*
4822 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4823 */
4824static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4825{
4826 int err;
4827 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
4828 const struct net_device_ops *ops;
4829
4830 if (!dev)
4831 return -ENODEV;
4832
4833 ops = dev->netdev_ops;
4834
4835 switch (cmd) {
4836 case SIOCSIFFLAGS: /* Set interface flags */
4837 return dev_change_flags(dev, ifr->ifr_flags);
4838
4839 case SIOCSIFMETRIC: /* Set the metric on the interface
4840 (currently unused) */
4841 return -EOPNOTSUPP;
4842
4843 case SIOCSIFMTU: /* Set the MTU of a device */
4844 return dev_set_mtu(dev, ifr->ifr_mtu);
4845
4846 case SIOCSIFHWADDR:
4847 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
4848
4849 case SIOCSIFHWBROADCAST:
4850 if (ifr->ifr_hwaddr.sa_family != dev->type)
4851 return -EINVAL;
4852 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4853 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4854 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4855 return 0;
4856
4857 case SIOCSIFMAP:
4858 if (ops->ndo_set_config) {
4859 if (!netif_device_present(dev))
4860 return -ENODEV;
4861 return ops->ndo_set_config(dev, &ifr->ifr_map);
4862 }
4863 return -EOPNOTSUPP;
4864
4865 case SIOCADDMULTI:
4866 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4867 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4868 return -EINVAL;
4869 if (!netif_device_present(dev))
4870 return -ENODEV;
4871 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
4872
4873 case SIOCDELMULTI:
4874 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4875 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4876 return -EINVAL;
4877 if (!netif_device_present(dev))
4878 return -ENODEV;
4879 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
4880
4881 case SIOCSIFTXQLEN:
4882 if (ifr->ifr_qlen < 0)
4883 return -EINVAL;
4884 dev->tx_queue_len = ifr->ifr_qlen;
4885 return 0;
4886
4887 case SIOCSIFNAME:
4888 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4889 return dev_change_name(dev, ifr->ifr_newname);
4890
4891 /*
4892 * Unknown or private ioctl
4893 */
4894 default:
4895 if ((cmd >= SIOCDEVPRIVATE &&
4896 cmd <= SIOCDEVPRIVATE + 15) ||
4897 cmd == SIOCBONDENSLAVE ||
4898 cmd == SIOCBONDRELEASE ||
4899 cmd == SIOCBONDSETHWADDR ||
4900 cmd == SIOCBONDSLAVEINFOQUERY ||
4901 cmd == SIOCBONDINFOQUERY ||
4902 cmd == SIOCBONDCHANGEACTIVE ||
4903 cmd == SIOCGMIIPHY ||
4904 cmd == SIOCGMIIREG ||
4905 cmd == SIOCSMIIREG ||
4906 cmd == SIOCBRADDIF ||
4907 cmd == SIOCBRDELIF ||
4908 cmd == SIOCSHWTSTAMP ||
4909 cmd == SIOCWANDEV) {
4910 err = -EOPNOTSUPP;
4911 if (ops->ndo_do_ioctl) {
4912 if (netif_device_present(dev))
4913 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4914 else
4915 err = -ENODEV;
4916 }
4917 } else
4918 err = -EINVAL;
4919
4920 }
4921 return err;
4922}
4923
4924/*
4925 * This function handles all "interface"-type I/O control requests. The actual
4926 * 'doing' part of this is dev_ifsioc above.
4927 */
4928
4929/**
4930 * dev_ioctl - network device ioctl
4931 * @net: the applicable net namespace
4932 * @cmd: command to issue
4933 * @arg: pointer to a struct ifreq in user space
4934 *
4935 * Issue ioctl functions to devices. This is normally called by the
4936 * user space syscall interfaces but can sometimes be useful for
4937 * other purposes. The return value is the return from the syscall if
4938 * positive or a negative errno code on error.
4939 */
4940
4941int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
4942{
4943 struct ifreq ifr;
4944 int ret;
4945 char *colon;
4946
4947 /* One special case: SIOCGIFCONF takes ifconf argument
4948 and requires shared lock, because it sleeps writing
4949 to user space.
4950 */
4951
4952 if (cmd == SIOCGIFCONF) {
4953 rtnl_lock();
4954 ret = dev_ifconf(net, (char __user *) arg);
4955 rtnl_unlock();
4956 return ret;
4957 }
4958 if (cmd == SIOCGIFNAME)
4959 return dev_ifname(net, (struct ifreq __user *)arg);
4960
4961 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4962 return -EFAULT;
4963
4964 ifr.ifr_name[IFNAMSIZ-1] = 0;
4965
4966 colon = strchr(ifr.ifr_name, ':');
4967 if (colon)
4968 *colon = 0;
4969
4970 /*
4971 * See which interface the caller is talking about.
4972 */
4973
4974 switch (cmd) {
4975 /*
4976 * These ioctl calls:
4977 * - can be done by all.
4978 * - atomic and do not require locking.
4979 * - return a value
4980 */
4981 case SIOCGIFFLAGS:
4982 case SIOCGIFMETRIC:
4983 case SIOCGIFMTU:
4984 case SIOCGIFHWADDR:
4985 case SIOCGIFSLAVE:
4986 case SIOCGIFMAP:
4987 case SIOCGIFINDEX:
4988 case SIOCGIFTXQLEN:
4989 dev_load(net, ifr.ifr_name);
4990 rcu_read_lock();
4991 ret = dev_ifsioc_locked(net, &ifr, cmd);
4992 rcu_read_unlock();
4993 if (!ret) {
4994 if (colon)
4995 *colon = ':';
4996 if (copy_to_user(arg, &ifr,
4997 sizeof(struct ifreq)))
4998 ret = -EFAULT;
4999 }
5000 return ret;
5001
5002 case SIOCETHTOOL:
5003 dev_load(net, ifr.ifr_name);
5004 rtnl_lock();
5005 ret = dev_ethtool(net, &ifr);
5006 rtnl_unlock();
5007 if (!ret) {
5008 if (colon)
5009 *colon = ':';
5010 if (copy_to_user(arg, &ifr,
5011 sizeof(struct ifreq)))
5012 ret = -EFAULT;
5013 }
5014 return ret;
5015
5016 /*
5017 * These ioctl calls:
5018 * - require superuser power.
5019 * - require strict serialization.
5020 * - return a value
5021 */
5022 case SIOCGMIIPHY:
5023 case SIOCGMIIREG:
5024 case SIOCSIFNAME:
5025 if (!capable(CAP_NET_ADMIN))
5026 return -EPERM;
5027 dev_load(net, ifr.ifr_name);
5028 rtnl_lock();
5029 ret = dev_ifsioc(net, &ifr, cmd);
5030 rtnl_unlock();
5031 if (!ret) {
5032 if (colon)
5033 *colon = ':';
5034 if (copy_to_user(arg, &ifr,
5035 sizeof(struct ifreq)))
5036 ret = -EFAULT;
5037 }
5038 return ret;
5039
5040 /*
5041 * These ioctl calls:
5042 * - require superuser power.
5043 * - require strict serialization.
5044 * - do not return a value
5045 */
5046 case SIOCSIFFLAGS:
5047 case SIOCSIFMETRIC:
5048 case SIOCSIFMTU:
5049 case SIOCSIFMAP:
5050 case SIOCSIFHWADDR:
5051 case SIOCSIFSLAVE:
5052 case SIOCADDMULTI:
5053 case SIOCDELMULTI:
5054 case SIOCSIFHWBROADCAST:
5055 case SIOCSIFTXQLEN:
5056 case SIOCSMIIREG:
5057 case SIOCBONDENSLAVE:
5058 case SIOCBONDRELEASE:
5059 case SIOCBONDSETHWADDR:
5060 case SIOCBONDCHANGEACTIVE:
5061 case SIOCBRADDIF:
5062 case SIOCBRDELIF:
5063 case SIOCSHWTSTAMP:
5064 if (!capable(CAP_NET_ADMIN))
5065 return -EPERM;
5066 /* fall through */
5067 case SIOCBONDSLAVEINFOQUERY:
5068 case SIOCBONDINFOQUERY:
5069 dev_load(net, ifr.ifr_name);
5070 rtnl_lock();
5071 ret = dev_ifsioc(net, &ifr, cmd);
5072 rtnl_unlock();
5073 return ret;
5074
5075 case SIOCGIFMEM:
5076 /* Get the per device memory space. We can add this but
5077 * currently do not support it */
5078 case SIOCSIFMEM:
5079 /* Set the per device memory buffer space.
5080 * Not applicable in our case */
5081 case SIOCSIFLINK:
5082 return -ENOTTY;
5083
5084 /*
5085 * Unknown or private ioctl.
5086 */
5087 default:
5088 if (cmd == SIOCWANDEV ||
5089 (cmd >= SIOCDEVPRIVATE &&
5090 cmd <= SIOCDEVPRIVATE + 15)) {
5091 dev_load(net, ifr.ifr_name);
5092 rtnl_lock();
5093 ret = dev_ifsioc(net, &ifr, cmd);
5094 rtnl_unlock();
5095 if (!ret && copy_to_user(arg, &ifr,
5096 sizeof(struct ifreq)))
5097 ret = -EFAULT;
5098 return ret;
5099 }
5100 /* Take care of Wireless Extensions */
5101 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5102 return wext_handle_ioctl(net, &ifr, cmd, arg);
5103 return -ENOTTY;
5104 }
5105}
5106
5107
5108/**
5109 * dev_new_index - allocate an ifindex
5110 * @net: the applicable net namespace
5111 *
5112 * Returns a suitable unique value for a new device interface
5113 * number. The caller must hold the rtnl semaphore or the
5114 * dev_base_lock to be sure it remains unique.
5115 */
5116static int dev_new_index(struct net *net)
5117{
5118 static int ifindex;
5119 for (;;) {
5120 if (++ifindex <= 0)
5121 ifindex = 1;
5122 if (!__dev_get_by_index(net, ifindex))
5123 return ifindex;
5124 }
5125}
5126
5127/* Delayed registration/unregisteration */
5128static LIST_HEAD(net_todo_list);
5129
5130static void net_set_todo(struct net_device *dev)
5131{
5132 list_add_tail(&dev->todo_list, &net_todo_list);
5133}
5134
5135static void rollback_registered_many(struct list_head *head)
5136{
5137 struct net_device *dev, *tmp;
5138
5139 BUG_ON(dev_boot_phase);
5140 ASSERT_RTNL();
5141
5142 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
5143 /* Some devices call without registering
5144 * for initialization unwind. Remove those
5145 * devices and proceed with the remaining.
5146 */
5147 if (dev->reg_state == NETREG_UNINITIALIZED) {
5148 pr_debug("unregister_netdevice: device %s/%p never "
5149 "was registered\n", dev->name, dev);
5150
5151 WARN_ON(1);
5152 list_del(&dev->unreg_list);
5153 continue;
5154 }
5155 dev->dismantle = true;
5156 BUG_ON(dev->reg_state != NETREG_REGISTERED);
5157 }
5158
5159 /* If device is running, close it first. */
5160 dev_close_many(head);
5161
5162 list_for_each_entry(dev, head, unreg_list) {
5163 /* And unlink it from device chain. */
5164 unlist_netdevice(dev);
5165
5166 dev->reg_state = NETREG_UNREGISTERING;
5167 }
5168
5169 synchronize_net();
5170
5171 list_for_each_entry(dev, head, unreg_list) {
5172 /* Shutdown queueing discipline. */
5173 dev_shutdown(dev);
5174
5175
5176 /* Notify protocols, that we are about to destroy
5177 this device. They should clean all the things.
5178 */
5179 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5180
5181 if (!dev->rtnl_link_ops ||
5182 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5183 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5184
5185 /*
5186 * Flush the unicast and multicast chains
5187 */
5188 dev_uc_flush(dev);
5189 dev_mc_flush(dev);
5190
5191 if (dev->netdev_ops->ndo_uninit)
5192 dev->netdev_ops->ndo_uninit(dev);
5193
5194 /* Notifier chain MUST detach us from master device. */
5195 WARN_ON(dev->master);
5196
5197 /* Remove entries from kobject tree */
5198 netdev_unregister_kobject(dev);
5199 }
5200
5201 /* Process any work delayed until the end of the batch */
5202 dev = list_first_entry(head, struct net_device, unreg_list);
5203 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5204
5205 rcu_barrier();
5206
5207 list_for_each_entry(dev, head, unreg_list)
5208 dev_put(dev);
5209}
5210
5211static void rollback_registered(struct net_device *dev)
5212{
5213 LIST_HEAD(single);
5214
5215 list_add(&dev->unreg_list, &single);
5216 rollback_registered_many(&single);
5217 list_del(&single);
5218}
5219
5220static u32 netdev_fix_features(struct net_device *dev, u32 features)
5221{
5222 /* Fix illegal checksum combinations */
5223 if ((features & NETIF_F_HW_CSUM) &&
5224 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5225 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
5226 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5227 }
5228
5229 if ((features & NETIF_F_NO_CSUM) &&
5230 (features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5231 netdev_warn(dev, "mixed no checksumming and other settings.\n");
5232 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5233 }
5234
5235 /* Fix illegal SG+CSUM combinations. */
5236 if ((features & NETIF_F_SG) &&
5237 !(features & NETIF_F_ALL_CSUM)) {
5238 netdev_dbg(dev,
5239 "Dropping NETIF_F_SG since no checksum feature.\n");
5240 features &= ~NETIF_F_SG;
5241 }
5242
5243 /* TSO requires that SG is present as well. */
5244 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
5245 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
5246 features &= ~NETIF_F_ALL_TSO;
5247 }
5248
5249 /* TSO ECN requires that TSO is present as well. */
5250 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
5251 features &= ~NETIF_F_TSO_ECN;
5252
5253 /* Software GSO depends on SG. */
5254 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
5255 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
5256 features &= ~NETIF_F_GSO;
5257 }
5258
5259 /* UFO needs SG and checksumming */
5260 if (features & NETIF_F_UFO) {
5261 /* maybe split UFO into V4 and V6? */
5262 if (!((features & NETIF_F_GEN_CSUM) ||
5263 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5264 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5265 netdev_dbg(dev,
5266 "Dropping NETIF_F_UFO since no checksum offload features.\n");
5267 features &= ~NETIF_F_UFO;
5268 }
5269
5270 if (!(features & NETIF_F_SG)) {
5271 netdev_dbg(dev,
5272 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
5273 features &= ~NETIF_F_UFO;
5274 }
5275 }
5276
5277 return features;
5278}
5279
5280int __netdev_update_features(struct net_device *dev)
5281{
5282 u32 features;
5283 int err = 0;
5284
5285 ASSERT_RTNL();
5286
5287 features = netdev_get_wanted_features(dev);
5288
5289 if (dev->netdev_ops->ndo_fix_features)
5290 features = dev->netdev_ops->ndo_fix_features(dev, features);
5291
5292 /* driver might be less strict about feature dependencies */
5293 features = netdev_fix_features(dev, features);
5294
5295 if (dev->features == features)
5296 return 0;
5297
5298 netdev_dbg(dev, "Features changed: 0x%08x -> 0x%08x\n",
5299 dev->features, features);
5300
5301 if (dev->netdev_ops->ndo_set_features)
5302 err = dev->netdev_ops->ndo_set_features(dev, features);
5303
5304 if (unlikely(err < 0)) {
5305 netdev_err(dev,
5306 "set_features() failed (%d); wanted 0x%08x, left 0x%08x\n",
5307 err, features, dev->features);
5308 return -1;
5309 }
5310
5311 if (!err)
5312 dev->features = features;
5313
5314 return 1;
5315}
5316
5317/**
5318 * netdev_update_features - recalculate device features
5319 * @dev: the device to check
5320 *
5321 * Recalculate dev->features set and send notifications if it
5322 * has changed. Should be called after driver or hardware dependent
5323 * conditions might have changed that influence the features.
5324 */
5325void netdev_update_features(struct net_device *dev)
5326{
5327 if (__netdev_update_features(dev))
5328 netdev_features_change(dev);
5329}
5330EXPORT_SYMBOL(netdev_update_features);
5331
5332/**
5333 * netdev_change_features - recalculate device features
5334 * @dev: the device to check
5335 *
5336 * Recalculate dev->features set and send notifications even
5337 * if they have not changed. Should be called instead of
5338 * netdev_update_features() if also dev->vlan_features might
5339 * have changed to allow the changes to be propagated to stacked
5340 * VLAN devices.
5341 */
5342void netdev_change_features(struct net_device *dev)
5343{
5344 __netdev_update_features(dev);
5345 netdev_features_change(dev);
5346}
5347EXPORT_SYMBOL(netdev_change_features);
5348
5349/**
5350 * netif_stacked_transfer_operstate - transfer operstate
5351 * @rootdev: the root or lower level device to transfer state from
5352 * @dev: the device to transfer operstate to
5353 *
5354 * Transfer operational state from root to device. This is normally
5355 * called when a stacking relationship exists between the root
5356 * device and the device(a leaf device).
5357 */
5358void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5359 struct net_device *dev)
5360{
5361 if (rootdev->operstate == IF_OPER_DORMANT)
5362 netif_dormant_on(dev);
5363 else
5364 netif_dormant_off(dev);
5365
5366 if (netif_carrier_ok(rootdev)) {
5367 if (!netif_carrier_ok(dev))
5368 netif_carrier_on(dev);
5369 } else {
5370 if (netif_carrier_ok(dev))
5371 netif_carrier_off(dev);
5372 }
5373}
5374EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5375
5376#ifdef CONFIG_RPS
5377static int netif_alloc_rx_queues(struct net_device *dev)
5378{
5379 unsigned int i, count = dev->num_rx_queues;
5380 struct netdev_rx_queue *rx;
5381
5382 BUG_ON(count < 1);
5383
5384 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5385 if (!rx) {
5386 pr_err("netdev: Unable to allocate %u rx queues.\n", count);
5387 return -ENOMEM;
5388 }
5389 dev->_rx = rx;
5390
5391 for (i = 0; i < count; i++)
5392 rx[i].dev = dev;
5393 return 0;
5394}
5395#endif
5396
5397static void netdev_init_one_queue(struct net_device *dev,
5398 struct netdev_queue *queue, void *_unused)
5399{
5400 /* Initialize queue lock */
5401 spin_lock_init(&queue->_xmit_lock);
5402 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5403 queue->xmit_lock_owner = -1;
5404 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
5405 queue->dev = dev;
5406}
5407
5408static int netif_alloc_netdev_queues(struct net_device *dev)
5409{
5410 unsigned int count = dev->num_tx_queues;
5411 struct netdev_queue *tx;
5412
5413 BUG_ON(count < 1);
5414
5415 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5416 if (!tx) {
5417 pr_err("netdev: Unable to allocate %u tx queues.\n",
5418 count);
5419 return -ENOMEM;
5420 }
5421 dev->_tx = tx;
5422
5423 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5424 spin_lock_init(&dev->tx_global_lock);
5425
5426 return 0;
5427}
5428
5429/**
5430 * register_netdevice - register a network device
5431 * @dev: device to register
5432 *
5433 * Take a completed network device structure and add it to the kernel
5434 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5435 * chain. 0 is returned on success. A negative errno code is returned
5436 * on a failure to set up the device, or if the name is a duplicate.
5437 *
5438 * Callers must hold the rtnl semaphore. You may want
5439 * register_netdev() instead of this.
5440 *
5441 * BUGS:
5442 * The locking appears insufficient to guarantee two parallel registers
5443 * will not get the same name.
5444 */
5445
5446int register_netdevice(struct net_device *dev)
5447{
5448 int ret;
5449 struct net *net = dev_net(dev);
5450
5451 BUG_ON(dev_boot_phase);
5452 ASSERT_RTNL();
5453
5454 might_sleep();
5455
5456 /* When net_device's are persistent, this will be fatal. */
5457 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
5458 BUG_ON(!net);
5459
5460 spin_lock_init(&dev->addr_list_lock);
5461 netdev_set_addr_lockdep_class(dev);
5462
5463 dev->iflink = -1;
5464
5465 ret = dev_get_valid_name(dev, dev->name);
5466 if (ret < 0)
5467 goto out;
5468
5469 /* Init, if this function is available */
5470 if (dev->netdev_ops->ndo_init) {
5471 ret = dev->netdev_ops->ndo_init(dev);
5472 if (ret) {
5473 if (ret > 0)
5474 ret = -EIO;
5475 goto out;
5476 }
5477 }
5478
5479 dev->ifindex = dev_new_index(net);
5480 if (dev->iflink == -1)
5481 dev->iflink = dev->ifindex;
5482
5483 /* Transfer changeable features to wanted_features and enable
5484 * software offloads (GSO and GRO).
5485 */
5486 dev->hw_features |= NETIF_F_SOFT_FEATURES;
5487 dev->features |= NETIF_F_SOFT_FEATURES;
5488 dev->wanted_features = dev->features & dev->hw_features;
5489
5490 /* Turn on no cache copy if HW is doing checksum */
5491 dev->hw_features |= NETIF_F_NOCACHE_COPY;
5492 if ((dev->features & NETIF_F_ALL_CSUM) &&
5493 !(dev->features & NETIF_F_NO_CSUM)) {
5494 dev->wanted_features |= NETIF_F_NOCACHE_COPY;
5495 dev->features |= NETIF_F_NOCACHE_COPY;
5496 }
5497
5498 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
5499 */
5500 dev->vlan_features |= NETIF_F_HIGHDMA;
5501
5502 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5503 ret = notifier_to_errno(ret);
5504 if (ret)
5505 goto err_uninit;
5506
5507 ret = netdev_register_kobject(dev);
5508 if (ret)
5509 goto err_uninit;
5510 dev->reg_state = NETREG_REGISTERED;
5511
5512 __netdev_update_features(dev);
5513
5514 /*
5515 * Default initial state at registry is that the
5516 * device is present.
5517 */
5518
5519 set_bit(__LINK_STATE_PRESENT, &dev->state);
5520
5521 dev_init_scheduler(dev);
5522 dev_hold(dev);
5523 list_netdevice(dev);
5524
5525 /* Notify protocols, that a new device appeared. */
5526 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
5527 ret = notifier_to_errno(ret);
5528 if (ret) {
5529 rollback_registered(dev);
5530 dev->reg_state = NETREG_UNREGISTERED;
5531 }
5532 /*
5533 * Prevent userspace races by waiting until the network
5534 * device is fully setup before sending notifications.
5535 */
5536 if (!dev->rtnl_link_ops ||
5537 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5538 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5539
5540out:
5541 return ret;
5542
5543err_uninit:
5544 if (dev->netdev_ops->ndo_uninit)
5545 dev->netdev_ops->ndo_uninit(dev);
5546 goto out;
5547}
5548EXPORT_SYMBOL(register_netdevice);
5549
5550/**
5551 * init_dummy_netdev - init a dummy network device for NAPI
5552 * @dev: device to init
5553 *
5554 * This takes a network device structure and initialize the minimum
5555 * amount of fields so it can be used to schedule NAPI polls without
5556 * registering a full blown interface. This is to be used by drivers
5557 * that need to tie several hardware interfaces to a single NAPI
5558 * poll scheduler due to HW limitations.
5559 */
5560int init_dummy_netdev(struct net_device *dev)
5561{
5562 /* Clear everything. Note we don't initialize spinlocks
5563 * are they aren't supposed to be taken by any of the
5564 * NAPI code and this dummy netdev is supposed to be
5565 * only ever used for NAPI polls
5566 */
5567 memset(dev, 0, sizeof(struct net_device));
5568
5569 /* make sure we BUG if trying to hit standard
5570 * register/unregister code path
5571 */
5572 dev->reg_state = NETREG_DUMMY;
5573
5574 /* NAPI wants this */
5575 INIT_LIST_HEAD(&dev->napi_list);
5576
5577 /* a dummy interface is started by default */
5578 set_bit(__LINK_STATE_PRESENT, &dev->state);
5579 set_bit(__LINK_STATE_START, &dev->state);
5580
5581 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5582 * because users of this 'device' dont need to change
5583 * its refcount.
5584 */
5585
5586 return 0;
5587}
5588EXPORT_SYMBOL_GPL(init_dummy_netdev);
5589
5590
5591/**
5592 * register_netdev - register a network device
5593 * @dev: device to register
5594 *
5595 * Take a completed network device structure and add it to the kernel
5596 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5597 * chain. 0 is returned on success. A negative errno code is returned
5598 * on a failure to set up the device, or if the name is a duplicate.
5599 *
5600 * This is a wrapper around register_netdevice that takes the rtnl semaphore
5601 * and expands the device name if you passed a format string to
5602 * alloc_netdev.
5603 */
5604int register_netdev(struct net_device *dev)
5605{
5606 int err;
5607
5608 rtnl_lock();
5609 err = register_netdevice(dev);
5610 rtnl_unlock();
5611 return err;
5612}
5613EXPORT_SYMBOL(register_netdev);
5614
5615int netdev_refcnt_read(const struct net_device *dev)
5616{
5617 int i, refcnt = 0;
5618
5619 for_each_possible_cpu(i)
5620 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5621 return refcnt;
5622}
5623EXPORT_SYMBOL(netdev_refcnt_read);
5624
5625/*
5626 * netdev_wait_allrefs - wait until all references are gone.
5627 *
5628 * This is called when unregistering network devices.
5629 *
5630 * Any protocol or device that holds a reference should register
5631 * for netdevice notification, and cleanup and put back the
5632 * reference if they receive an UNREGISTER event.
5633 * We can get stuck here if buggy protocols don't correctly
5634 * call dev_put.
5635 */
5636static void netdev_wait_allrefs(struct net_device *dev)
5637{
5638 unsigned long rebroadcast_time, warning_time;
5639 int refcnt;
5640
5641 linkwatch_forget_dev(dev);
5642
5643 rebroadcast_time = warning_time = jiffies;
5644 refcnt = netdev_refcnt_read(dev);
5645
5646 while (refcnt != 0) {
5647 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
5648 rtnl_lock();
5649
5650 /* Rebroadcast unregister notification */
5651 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5652 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
5653 * should have already handle it the first time */
5654
5655 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5656 &dev->state)) {
5657 /* We must not have linkwatch events
5658 * pending on unregister. If this
5659 * happens, we simply run the queue
5660 * unscheduled, resulting in a noop
5661 * for this device.
5662 */
5663 linkwatch_run_queue();
5664 }
5665
5666 __rtnl_unlock();
5667
5668 rebroadcast_time = jiffies;
5669 }
5670
5671 msleep(250);
5672
5673 refcnt = netdev_refcnt_read(dev);
5674
5675 if (time_after(jiffies, warning_time + 10 * HZ)) {
5676 printk(KERN_EMERG "unregister_netdevice: "
5677 "waiting for %s to become free. Usage "
5678 "count = %d\n",
5679 dev->name, refcnt);
5680 warning_time = jiffies;
5681 }
5682 }
5683}
5684
5685/* The sequence is:
5686 *
5687 * rtnl_lock();
5688 * ...
5689 * register_netdevice(x1);
5690 * register_netdevice(x2);
5691 * ...
5692 * unregister_netdevice(y1);
5693 * unregister_netdevice(y2);
5694 * ...
5695 * rtnl_unlock();
5696 * free_netdev(y1);
5697 * free_netdev(y2);
5698 *
5699 * We are invoked by rtnl_unlock().
5700 * This allows us to deal with problems:
5701 * 1) We can delete sysfs objects which invoke hotplug
5702 * without deadlocking with linkwatch via keventd.
5703 * 2) Since we run with the RTNL semaphore not held, we can sleep
5704 * safely in order to wait for the netdev refcnt to drop to zero.
5705 *
5706 * We must not return until all unregister events added during
5707 * the interval the lock was held have been completed.
5708 */
5709void netdev_run_todo(void)
5710{
5711 struct list_head list;
5712
5713 /* Snapshot list, allow later requests */
5714 list_replace_init(&net_todo_list, &list);
5715
5716 __rtnl_unlock();
5717
5718 while (!list_empty(&list)) {
5719 struct net_device *dev
5720 = list_first_entry(&list, struct net_device, todo_list);
5721 list_del(&dev->todo_list);
5722
5723 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
5724 printk(KERN_ERR "network todo '%s' but state %d\n",
5725 dev->name, dev->reg_state);
5726 dump_stack();
5727 continue;
5728 }
5729
5730 dev->reg_state = NETREG_UNREGISTERED;
5731
5732 on_each_cpu(flush_backlog, dev, 1);
5733
5734 netdev_wait_allrefs(dev);
5735
5736 /* paranoia */
5737 BUG_ON(netdev_refcnt_read(dev));
5738 WARN_ON(rcu_dereference_raw(dev->ip_ptr));
5739 WARN_ON(rcu_dereference_raw(dev->ip6_ptr));
5740 WARN_ON(dev->dn_ptr);
5741
5742 if (dev->destructor)
5743 dev->destructor(dev);
5744
5745 /* Free network device */
5746 kobject_put(&dev->dev.kobj);
5747 }
5748}
5749
5750/* Convert net_device_stats to rtnl_link_stats64. They have the same
5751 * fields in the same order, with only the type differing.
5752 */
5753static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5754 const struct net_device_stats *netdev_stats)
5755{
5756#if BITS_PER_LONG == 64
5757 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5758 memcpy(stats64, netdev_stats, sizeof(*stats64));
5759#else
5760 size_t i, n = sizeof(*stats64) / sizeof(u64);
5761 const unsigned long *src = (const unsigned long *)netdev_stats;
5762 u64 *dst = (u64 *)stats64;
5763
5764 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5765 sizeof(*stats64) / sizeof(u64));
5766 for (i = 0; i < n; i++)
5767 dst[i] = src[i];
5768#endif
5769}
5770
5771/**
5772 * dev_get_stats - get network device statistics
5773 * @dev: device to get statistics from
5774 * @storage: place to store stats
5775 *
5776 * Get network statistics from device. Return @storage.
5777 * The device driver may provide its own method by setting
5778 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5779 * otherwise the internal statistics structure is used.
5780 */
5781struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5782 struct rtnl_link_stats64 *storage)
5783{
5784 const struct net_device_ops *ops = dev->netdev_ops;
5785
5786 if (ops->ndo_get_stats64) {
5787 memset(storage, 0, sizeof(*storage));
5788 ops->ndo_get_stats64(dev, storage);
5789 } else if (ops->ndo_get_stats) {
5790 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
5791 } else {
5792 netdev_stats_to_stats64(storage, &dev->stats);
5793 }
5794 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
5795 return storage;
5796}
5797EXPORT_SYMBOL(dev_get_stats);
5798
5799struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
5800{
5801 struct netdev_queue *queue = dev_ingress_queue(dev);
5802
5803#ifdef CONFIG_NET_CLS_ACT
5804 if (queue)
5805 return queue;
5806 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5807 if (!queue)
5808 return NULL;
5809 netdev_init_one_queue(dev, queue, NULL);
5810 queue->qdisc = &noop_qdisc;
5811 queue->qdisc_sleeping = &noop_qdisc;
5812 rcu_assign_pointer(dev->ingress_queue, queue);
5813#endif
5814 return queue;
5815}
5816
5817/**
5818 * alloc_netdev_mqs - allocate network device
5819 * @sizeof_priv: size of private data to allocate space for
5820 * @name: device name format string
5821 * @setup: callback to initialize device
5822 * @txqs: the number of TX subqueues to allocate
5823 * @rxqs: the number of RX subqueues to allocate
5824 *
5825 * Allocates a struct net_device with private data area for driver use
5826 * and performs basic initialization. Also allocates subquue structs
5827 * for each queue on the device.
5828 */
5829struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5830 void (*setup)(struct net_device *),
5831 unsigned int txqs, unsigned int rxqs)
5832{
5833 struct net_device *dev;
5834 size_t alloc_size;
5835 struct net_device *p;
5836
5837 BUG_ON(strlen(name) >= sizeof(dev->name));
5838
5839 if (txqs < 1) {
5840 pr_err("alloc_netdev: Unable to allocate device "
5841 "with zero queues.\n");
5842 return NULL;
5843 }
5844
5845#ifdef CONFIG_RPS
5846 if (rxqs < 1) {
5847 pr_err("alloc_netdev: Unable to allocate device "
5848 "with zero RX queues.\n");
5849 return NULL;
5850 }
5851#endif
5852
5853 alloc_size = sizeof(struct net_device);
5854 if (sizeof_priv) {
5855 /* ensure 32-byte alignment of private area */
5856 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
5857 alloc_size += sizeof_priv;
5858 }
5859 /* ensure 32-byte alignment of whole construct */
5860 alloc_size += NETDEV_ALIGN - 1;
5861
5862 p = kzalloc(alloc_size, GFP_KERNEL);
5863 if (!p) {
5864 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
5865 return NULL;
5866 }
5867
5868 dev = PTR_ALIGN(p, NETDEV_ALIGN);
5869 dev->padded = (char *)dev - (char *)p;
5870
5871 dev->pcpu_refcnt = alloc_percpu(int);
5872 if (!dev->pcpu_refcnt)
5873 goto free_p;
5874
5875 if (dev_addr_init(dev))
5876 goto free_pcpu;
5877
5878 dev_mc_init(dev);
5879 dev_uc_init(dev);
5880
5881 dev_net_set(dev, &init_net);
5882
5883 dev->gso_max_size = GSO_MAX_SIZE;
5884
5885 INIT_LIST_HEAD(&dev->napi_list);
5886 INIT_LIST_HEAD(&dev->unreg_list);
5887 INIT_LIST_HEAD(&dev->link_watch_list);
5888 dev->priv_flags = IFF_XMIT_DST_RELEASE;
5889 setup(dev);
5890
5891 dev->num_tx_queues = txqs;
5892 dev->real_num_tx_queues = txqs;
5893 if (netif_alloc_netdev_queues(dev))
5894 goto free_all;
5895
5896#ifdef CONFIG_RPS
5897 dev->num_rx_queues = rxqs;
5898 dev->real_num_rx_queues = rxqs;
5899 if (netif_alloc_rx_queues(dev))
5900 goto free_all;
5901#endif
5902
5903 strcpy(dev->name, name);
5904 dev->group = INIT_NETDEV_GROUP;
5905 return dev;
5906
5907free_all:
5908 free_netdev(dev);
5909 return NULL;
5910
5911free_pcpu:
5912 free_percpu(dev->pcpu_refcnt);
5913 kfree(dev->_tx);
5914#ifdef CONFIG_RPS
5915 kfree(dev->_rx);
5916#endif
5917
5918free_p:
5919 kfree(p);
5920 return NULL;
5921}
5922EXPORT_SYMBOL(alloc_netdev_mqs);
5923
5924/**
5925 * free_netdev - free network device
5926 * @dev: device
5927 *
5928 * This function does the last stage of destroying an allocated device
5929 * interface. The reference to the device object is released.
5930 * If this is the last reference then it will be freed.
5931 */
5932void free_netdev(struct net_device *dev)
5933{
5934 struct napi_struct *p, *n;
5935
5936 release_net(dev_net(dev));
5937
5938 kfree(dev->_tx);
5939#ifdef CONFIG_RPS
5940 kfree(dev->_rx);
5941#endif
5942
5943 kfree(rcu_dereference_raw(dev->ingress_queue));
5944
5945 /* Flush device addresses */
5946 dev_addr_flush(dev);
5947
5948 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5949 netif_napi_del(p);
5950
5951 free_percpu(dev->pcpu_refcnt);
5952 dev->pcpu_refcnt = NULL;
5953
5954 /* Compatibility with error handling in drivers */
5955 if (dev->reg_state == NETREG_UNINITIALIZED) {
5956 kfree((char *)dev - dev->padded);
5957 return;
5958 }
5959
5960 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5961 dev->reg_state = NETREG_RELEASED;
5962
5963 /* will free via device release */
5964 put_device(&dev->dev);
5965}
5966EXPORT_SYMBOL(free_netdev);
5967
5968/**
5969 * synchronize_net - Synchronize with packet receive processing
5970 *
5971 * Wait for packets currently being received to be done.
5972 * Does not block later packets from starting.
5973 */
5974void synchronize_net(void)
5975{
5976 might_sleep();
5977 if (rtnl_is_locked())
5978 synchronize_rcu_expedited();
5979 else
5980 synchronize_rcu();
5981}
5982EXPORT_SYMBOL(synchronize_net);
5983
5984/**
5985 * unregister_netdevice_queue - remove device from the kernel
5986 * @dev: device
5987 * @head: list
5988 *
5989 * This function shuts down a device interface and removes it
5990 * from the kernel tables.
5991 * If head not NULL, device is queued to be unregistered later.
5992 *
5993 * Callers must hold the rtnl semaphore. You may want
5994 * unregister_netdev() instead of this.
5995 */
5996
5997void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
5998{
5999 ASSERT_RTNL();
6000
6001 if (head) {
6002 list_move_tail(&dev->unreg_list, head);
6003 } else {
6004 rollback_registered(dev);
6005 /* Finish processing unregister after unlock */
6006 net_set_todo(dev);
6007 }
6008}
6009EXPORT_SYMBOL(unregister_netdevice_queue);
6010
6011/**
6012 * unregister_netdevice_many - unregister many devices
6013 * @head: list of devices
6014 */
6015void unregister_netdevice_many(struct list_head *head)
6016{
6017 struct net_device *dev;
6018
6019 if (!list_empty(head)) {
6020 rollback_registered_many(head);
6021 list_for_each_entry(dev, head, unreg_list)
6022 net_set_todo(dev);
6023 }
6024}
6025EXPORT_SYMBOL(unregister_netdevice_many);
6026
6027/**
6028 * unregister_netdev - remove device from the kernel
6029 * @dev: device
6030 *
6031 * This function shuts down a device interface and removes it
6032 * from the kernel tables.
6033 *
6034 * This is just a wrapper for unregister_netdevice that takes
6035 * the rtnl semaphore. In general you want to use this and not
6036 * unregister_netdevice.
6037 */
6038void unregister_netdev(struct net_device *dev)
6039{
6040 rtnl_lock();
6041 unregister_netdevice(dev);
6042 rtnl_unlock();
6043}
6044EXPORT_SYMBOL(unregister_netdev);
6045
6046/**
6047 * dev_change_net_namespace - move device to different nethost namespace
6048 * @dev: device
6049 * @net: network namespace
6050 * @pat: If not NULL name pattern to try if the current device name
6051 * is already taken in the destination network namespace.
6052 *
6053 * This function shuts down a device interface and moves it
6054 * to a new network namespace. On success 0 is returned, on
6055 * a failure a netagive errno code is returned.
6056 *
6057 * Callers must hold the rtnl semaphore.
6058 */
6059
6060int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
6061{
6062 int err;
6063
6064 ASSERT_RTNL();
6065
6066 /* Don't allow namespace local devices to be moved. */
6067 err = -EINVAL;
6068 if (dev->features & NETIF_F_NETNS_LOCAL)
6069 goto out;
6070
6071 /* Ensure the device has been registrered */
6072 err = -EINVAL;
6073 if (dev->reg_state != NETREG_REGISTERED)
6074 goto out;
6075
6076 /* Get out if there is nothing todo */
6077 err = 0;
6078 if (net_eq(dev_net(dev), net))
6079 goto out;
6080
6081 /* Pick the destination device name, and ensure
6082 * we can use it in the destination network namespace.
6083 */
6084 err = -EEXIST;
6085 if (__dev_get_by_name(net, dev->name)) {
6086 /* We get here if we can't use the current device name */
6087 if (!pat)
6088 goto out;
6089 if (dev_get_valid_name(dev, pat) < 0)
6090 goto out;
6091 }
6092
6093 /*
6094 * And now a mini version of register_netdevice unregister_netdevice.
6095 */
6096
6097 /* If device is running close it first. */
6098 dev_close(dev);
6099
6100 /* And unlink it from device chain */
6101 err = -ENODEV;
6102 unlist_netdevice(dev);
6103
6104 synchronize_net();
6105
6106 /* Shutdown queueing discipline. */
6107 dev_shutdown(dev);
6108
6109 /* Notify protocols, that we are about to destroy
6110 this device. They should clean all the things.
6111
6112 Note that dev->reg_state stays at NETREG_REGISTERED.
6113 This is wanted because this way 8021q and macvlan know
6114 the device is just moving and can keep their slaves up.
6115 */
6116 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6117 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
6118
6119 /*
6120 * Flush the unicast and multicast chains
6121 */
6122 dev_uc_flush(dev);
6123 dev_mc_flush(dev);
6124
6125 /* Actually switch the network namespace */
6126 dev_net_set(dev, net);
6127
6128 /* If there is an ifindex conflict assign a new one */
6129 if (__dev_get_by_index(net, dev->ifindex)) {
6130 int iflink = (dev->iflink == dev->ifindex);
6131 dev->ifindex = dev_new_index(net);
6132 if (iflink)
6133 dev->iflink = dev->ifindex;
6134 }
6135
6136 /* Fixup kobjects */
6137 err = device_rename(&dev->dev, dev->name);
6138 WARN_ON(err);
6139
6140 /* Add the device back in the hashes */
6141 list_netdevice(dev);
6142
6143 /* Notify protocols, that a new device appeared. */
6144 call_netdevice_notifiers(NETDEV_REGISTER, dev);
6145
6146 /*
6147 * Prevent userspace races by waiting until the network
6148 * device is fully setup before sending notifications.
6149 */
6150 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
6151
6152 synchronize_net();
6153 err = 0;
6154out:
6155 return err;
6156}
6157EXPORT_SYMBOL_GPL(dev_change_net_namespace);
6158
6159static int dev_cpu_callback(struct notifier_block *nfb,
6160 unsigned long action,
6161 void *ocpu)
6162{
6163 struct sk_buff **list_skb;
6164 struct sk_buff *skb;
6165 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6166 struct softnet_data *sd, *oldsd;
6167
6168 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
6169 return NOTIFY_OK;
6170
6171 local_irq_disable();
6172 cpu = smp_processor_id();
6173 sd = &per_cpu(softnet_data, cpu);
6174 oldsd = &per_cpu(softnet_data, oldcpu);
6175
6176 /* Find end of our completion_queue. */
6177 list_skb = &sd->completion_queue;
6178 while (*list_skb)
6179 list_skb = &(*list_skb)->next;
6180 /* Append completion queue from offline CPU. */
6181 *list_skb = oldsd->completion_queue;
6182 oldsd->completion_queue = NULL;
6183
6184 /* Append output queue from offline CPU. */
6185 if (oldsd->output_queue) {
6186 *sd->output_queue_tailp = oldsd->output_queue;
6187 sd->output_queue_tailp = oldsd->output_queue_tailp;
6188 oldsd->output_queue = NULL;
6189 oldsd->output_queue_tailp = &oldsd->output_queue;
6190 }
6191 /* Append NAPI poll list from offline CPU. */
6192 if (!list_empty(&oldsd->poll_list)) {
6193 list_splice_init(&oldsd->poll_list, &sd->poll_list);
6194 raise_softirq_irqoff(NET_RX_SOFTIRQ);
6195 }
6196
6197 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6198 local_irq_enable();
6199
6200 /* Process offline CPU's input_pkt_queue */
6201 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6202 netif_rx(skb);
6203 input_queue_head_incr(oldsd);
6204 }
6205 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
6206 netif_rx(skb);
6207 input_queue_head_incr(oldsd);
6208 }
6209
6210 return NOTIFY_OK;
6211}
6212
6213
6214/**
6215 * netdev_increment_features - increment feature set by one
6216 * @all: current feature set
6217 * @one: new feature set
6218 * @mask: mask feature set
6219 *
6220 * Computes a new feature set after adding a device with feature set
6221 * @one to the master device with current feature set @all. Will not
6222 * enable anything that is off in @mask. Returns the new feature set.
6223 */
6224u32 netdev_increment_features(u32 all, u32 one, u32 mask)
6225{
6226 if (mask & NETIF_F_GEN_CSUM)
6227 mask |= NETIF_F_ALL_CSUM;
6228 mask |= NETIF_F_VLAN_CHALLENGED;
6229
6230 all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
6231 all &= one | ~NETIF_F_ALL_FOR_ALL;
6232
6233 /* If device needs checksumming, downgrade to it. */
6234 if (all & (NETIF_F_ALL_CSUM & ~NETIF_F_NO_CSUM))
6235 all &= ~NETIF_F_NO_CSUM;
6236
6237 /* If one device supports hw checksumming, set for all. */
6238 if (all & NETIF_F_GEN_CSUM)
6239 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
6240
6241 return all;
6242}
6243EXPORT_SYMBOL(netdev_increment_features);
6244
6245static struct hlist_head *netdev_create_hash(void)
6246{
6247 int i;
6248 struct hlist_head *hash;
6249
6250 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6251 if (hash != NULL)
6252 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6253 INIT_HLIST_HEAD(&hash[i]);
6254
6255 return hash;
6256}
6257
6258/* Initialize per network namespace state */
6259static int __net_init netdev_init(struct net *net)
6260{
6261 INIT_LIST_HEAD(&net->dev_base_head);
6262
6263 net->dev_name_head = netdev_create_hash();
6264 if (net->dev_name_head == NULL)
6265 goto err_name;
6266
6267 net->dev_index_head = netdev_create_hash();
6268 if (net->dev_index_head == NULL)
6269 goto err_idx;
6270
6271 return 0;
6272
6273err_idx:
6274 kfree(net->dev_name_head);
6275err_name:
6276 return -ENOMEM;
6277}
6278
6279/**
6280 * netdev_drivername - network driver for the device
6281 * @dev: network device
6282 *
6283 * Determine network driver for device.
6284 */
6285const char *netdev_drivername(const struct net_device *dev)
6286{
6287 const struct device_driver *driver;
6288 const struct device *parent;
6289 const char *empty = "";
6290
6291 parent = dev->dev.parent;
6292 if (!parent)
6293 return empty;
6294
6295 driver = parent->driver;
6296 if (driver && driver->name)
6297 return driver->name;
6298 return empty;
6299}
6300
6301static int __netdev_printk(const char *level, const struct net_device *dev,
6302 struct va_format *vaf)
6303{
6304 int r;
6305
6306 if (dev && dev->dev.parent)
6307 r = dev_printk(level, dev->dev.parent, "%s: %pV",
6308 netdev_name(dev), vaf);
6309 else if (dev)
6310 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6311 else
6312 r = printk("%s(NULL net_device): %pV", level, vaf);
6313
6314 return r;
6315}
6316
6317int netdev_printk(const char *level, const struct net_device *dev,
6318 const char *format, ...)
6319{
6320 struct va_format vaf;
6321 va_list args;
6322 int r;
6323
6324 va_start(args, format);
6325
6326 vaf.fmt = format;
6327 vaf.va = &args;
6328
6329 r = __netdev_printk(level, dev, &vaf);
6330 va_end(args);
6331
6332 return r;
6333}
6334EXPORT_SYMBOL(netdev_printk);
6335
6336#define define_netdev_printk_level(func, level) \
6337int func(const struct net_device *dev, const char *fmt, ...) \
6338{ \
6339 int r; \
6340 struct va_format vaf; \
6341 va_list args; \
6342 \
6343 va_start(args, fmt); \
6344 \
6345 vaf.fmt = fmt; \
6346 vaf.va = &args; \
6347 \
6348 r = __netdev_printk(level, dev, &vaf); \
6349 va_end(args); \
6350 \
6351 return r; \
6352} \
6353EXPORT_SYMBOL(func);
6354
6355define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6356define_netdev_printk_level(netdev_alert, KERN_ALERT);
6357define_netdev_printk_level(netdev_crit, KERN_CRIT);
6358define_netdev_printk_level(netdev_err, KERN_ERR);
6359define_netdev_printk_level(netdev_warn, KERN_WARNING);
6360define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6361define_netdev_printk_level(netdev_info, KERN_INFO);
6362
6363static void __net_exit netdev_exit(struct net *net)
6364{
6365 kfree(net->dev_name_head);
6366 kfree(net->dev_index_head);
6367}
6368
6369static struct pernet_operations __net_initdata netdev_net_ops = {
6370 .init = netdev_init,
6371 .exit = netdev_exit,
6372};
6373
6374static void __net_exit default_device_exit(struct net *net)
6375{
6376 struct net_device *dev, *aux;
6377 /*
6378 * Push all migratable network devices back to the
6379 * initial network namespace
6380 */
6381 rtnl_lock();
6382 for_each_netdev_safe(net, dev, aux) {
6383 int err;
6384 char fb_name[IFNAMSIZ];
6385
6386 /* Ignore unmoveable devices (i.e. loopback) */
6387 if (dev->features & NETIF_F_NETNS_LOCAL)
6388 continue;
6389
6390 /* Leave virtual devices for the generic cleanup */
6391 if (dev->rtnl_link_ops)
6392 continue;
6393
6394 /* Push remaining network devices to init_net */
6395 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6396 err = dev_change_net_namespace(dev, &init_net, fb_name);
6397 if (err) {
6398 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
6399 __func__, dev->name, err);
6400 BUG();
6401 }
6402 }
6403 rtnl_unlock();
6404}
6405
6406static void __net_exit default_device_exit_batch(struct list_head *net_list)
6407{
6408 /* At exit all network devices most be removed from a network
6409 * namespace. Do this in the reverse order of registration.
6410 * Do this across as many network namespaces as possible to
6411 * improve batching efficiency.
6412 */
6413 struct net_device *dev;
6414 struct net *net;
6415 LIST_HEAD(dev_kill_list);
6416
6417 rtnl_lock();
6418 list_for_each_entry(net, net_list, exit_list) {
6419 for_each_netdev_reverse(net, dev) {
6420 if (dev->rtnl_link_ops)
6421 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6422 else
6423 unregister_netdevice_queue(dev, &dev_kill_list);
6424 }
6425 }
6426 unregister_netdevice_many(&dev_kill_list);
6427 list_del(&dev_kill_list);
6428 rtnl_unlock();
6429}
6430
6431static struct pernet_operations __net_initdata default_device_ops = {
6432 .exit = default_device_exit,
6433 .exit_batch = default_device_exit_batch,
6434};
6435
6436/*
6437 * Initialize the DEV module. At boot time this walks the device list and
6438 * unhooks any devices that fail to initialise (normally hardware not
6439 * present) and leaves us with a valid list of present and active devices.
6440 *
6441 */
6442
6443/*
6444 * This is called single threaded during boot, so no need
6445 * to take the rtnl semaphore.
6446 */
6447static int __init net_dev_init(void)
6448{
6449 int i, rc = -ENOMEM;
6450
6451 BUG_ON(!dev_boot_phase);
6452
6453 if (dev_proc_init())
6454 goto out;
6455
6456 if (netdev_kobject_init())
6457 goto out;
6458
6459 INIT_LIST_HEAD(&ptype_all);
6460 for (i = 0; i < PTYPE_HASH_SIZE; i++)
6461 INIT_LIST_HEAD(&ptype_base[i]);
6462
6463 if (register_pernet_subsys(&netdev_net_ops))
6464 goto out;
6465
6466 /*
6467 * Initialise the packet receive queues.
6468 */
6469
6470 for_each_possible_cpu(i) {
6471 struct softnet_data *sd = &per_cpu(softnet_data, i);
6472
6473 memset(sd, 0, sizeof(*sd));
6474 skb_queue_head_init(&sd->input_pkt_queue);
6475 skb_queue_head_init(&sd->process_queue);
6476 sd->completion_queue = NULL;
6477 INIT_LIST_HEAD(&sd->poll_list);
6478 sd->output_queue = NULL;
6479 sd->output_queue_tailp = &sd->output_queue;
6480#ifdef CONFIG_RPS
6481 sd->csd.func = rps_trigger_softirq;
6482 sd->csd.info = sd;
6483 sd->csd.flags = 0;
6484 sd->cpu = i;
6485#endif
6486
6487 sd->backlog.poll = process_backlog;
6488 sd->backlog.weight = weight_p;
6489 sd->backlog.gro_list = NULL;
6490 sd->backlog.gro_count = 0;
6491 }
6492
6493 dev_boot_phase = 0;
6494
6495 /* The loopback device is special if any other network devices
6496 * is present in a network namespace the loopback device must
6497 * be present. Since we now dynamically allocate and free the
6498 * loopback device ensure this invariant is maintained by
6499 * keeping the loopback device as the first device on the
6500 * list of network devices. Ensuring the loopback devices
6501 * is the first device that appears and the last network device
6502 * that disappears.
6503 */
6504 if (register_pernet_device(&loopback_net_ops))
6505 goto out;
6506
6507 if (register_pernet_device(&default_device_ops))
6508 goto out;
6509
6510 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6511 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
6512
6513 hotcpu_notifier(dev_cpu_callback, 0);
6514 dst_init();
6515 dev_mcast_init();
6516 rc = 0;
6517out:
6518 return rc;
6519}
6520
6521subsys_initcall(net_dev_init);
6522
6523static int __init initialize_hashrnd(void)
6524{
6525 get_random_bytes(&hashrnd, sizeof(hashrnd));
6526 return 0;
6527}
6528
6529late_initcall_sync(initialize_hashrnd);
6530