1/*
   2 *  TUN - Universal TUN/TAP device driver.
   3 *  Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com>
   4 *
   5 *  This program is free software; you can redistribute it and/or modify
   6 *  it under the terms of the GNU General Public License as published by
   7 *  the Free Software Foundation; either version 2 of the License, or
   8 *  (at your option) any later version.
   9 *
  10 *  This program is distributed in the hope that it will be useful,
  11 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 *  GNU General Public License for more details.
  14 *
  15 *  $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $
  16 */
  17
  18/*
  19 *  Changes:
  20 *
  21 *  Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14
  22 *    Add TUNSETLINK ioctl to set the link encapsulation
  23 *
  24 *  Mark Smith <markzzzsmith@yahoo.com.au>
  25 *    Use eth_random_addr() for tap MAC address.
  26 *
  27 *  Harald Roelle <harald.roelle@ifi.lmu.de>  2004/04/20
  28 *    Fixes in packet dropping, queue length setting and queue wakeup.
  29 *    Increased default tx queue length.
  30 *    Added ethtool API.
  31 *    Minor cleanups
  32 *
  33 *  Daniel Podlejski <underley@underley.eu.org>
  34 *    Modifications for 2.3.99-pre5 kernel.
  35 */
  36
  37#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  38
  39#define DRV_NAME	"tun"
  40#define DRV_VERSION	"1.6"
  41#define DRV_DESCRIPTION	"Universal TUN/TAP device driver"
  42#define DRV_COPYRIGHT	"(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>"
  43
  44#include <linux/module.h>
  45#include <linux/errno.h>
  46#include <linux/kernel.h>
  47#include <linux/sched/signal.h>
  48#include <linux/major.h>
  49#include <linux/slab.h>
  50#include <linux/poll.h>
  51#include <linux/fcntl.h>
  52#include <linux/init.h>
  53#include <linux/skbuff.h>
  54#include <linux/netdevice.h>
  55#include <linux/etherdevice.h>
  56#include <linux/miscdevice.h>
  57#include <linux/ethtool.h>
  58#include <linux/rtnetlink.h>
  59#include <linux/compat.h>
  60#include <linux/if.h>
  61#include <linux/if_arp.h>
  62#include <linux/if_ether.h>
  63#include <linux/if_tun.h>
  64#include <linux/if_vlan.h>
  65#include <linux/crc32.h>
  66#include <linux/nsproxy.h>
  67#include <linux/virtio_net.h>
  68#include <linux/rcupdate.h>
  69#include <net/net_namespace.h>
  70#include <net/netns/generic.h>
  71#include <net/rtnetlink.h>
  72#include <net/sock.h>
  73#include <linux/seq_file.h>
  74#include <linux/uio.h>
  75#include <linux/skb_array.h>
  76#include <linux/bpf.h>
  77#include <linux/bpf_trace.h>
  78#include <linux/mutex.h>
  79
  80#include <linux/uaccess.h>
  81#include <linux/proc_fs.h>
  82
  83/* Uncomment to enable debugging */
  84/* #define TUN_DEBUG 1 */
  85
  86#ifdef TUN_DEBUG
  87static int debug;
  88
  89#define tun_debug(level, tun, fmt, args...)			\
  90do {								\
  91	if (tun->debug)						\
  92		netdev_printk(level, tun->dev, fmt, ##args);	\
  93} while (0)
  94#define DBG1(level, fmt, args...)				\
  95do {								\
  96	if (debug == 2)						\
  97		printk(level fmt, ##args);			\
  98} while (0)
  99#else
 100#define tun_debug(level, tun, fmt, args...)			\
 101do {								\
 102	if (0)							\
 103		netdev_printk(level, tun->dev, fmt, ##args);	\
 104} while (0)
 105#define DBG1(level, fmt, args...)				\
 106do {								\
 107	if (0)							\
 108		printk(level fmt, ##args);			\
 109} while (0)
 110#endif
 111
 112#define TUN_HEADROOM 256
 113#define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
 114
 115/* TUN device flags */
 116
 117/* IFF_ATTACH_QUEUE is never stored in device flags,
 118 * overload it to mean fasync when stored there.
 119 */
 120#define TUN_FASYNC	IFF_ATTACH_QUEUE
 121/* High bits in flags field are unused. */
 122#define TUN_VNET_LE     0x80000000
 123#define TUN_VNET_BE     0x40000000
 124
 125#define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \
 126		      IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS)
 127
 128#define GOODCOPY_LEN 128
 129
 130#define FLT_EXACT_COUNT 8
 131struct tap_filter {
 132	unsigned int    count;    /* Number of addrs. Zero means disabled */
 133	u32             mask[2];  /* Mask of the hashed addrs */
 134	unsigned char	addr[FLT_EXACT_COUNT][ETH_ALEN];
 135};
 136
 137/* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
 138 * to max number of VCPUs in guest. */
 139#define MAX_TAP_QUEUES 256
 140#define MAX_TAP_FLOWS  4096
 141
 142#define TUN_FLOW_EXPIRE (3 * HZ)
 143
 144struct tun_pcpu_stats {
 145	u64 rx_packets;
 146	u64 rx_bytes;
 147	u64 tx_packets;
 148	u64 tx_bytes;
 149	struct u64_stats_sync syncp;
 150	u32 rx_dropped;
 151	u32 tx_dropped;
 152	u32 rx_frame_errors;
 153};
 154
 155/* A tun_file connects an open character device to a tuntap netdevice. It
 156 * also contains all socket related structures (except sock_fprog and tap_filter)
 157 * to serve as one transmit queue for tuntap device. The sock_fprog and
 158 * tap_filter were kept in tun_struct since they were used for filtering for the
 159 * netdevice not for a specific queue (at least I didn't see the requirement for
 160 * this).
 161 *
 162 * RCU usage:
 163 * The tun_file and tun_struct are loosely coupled, the pointer from one to the
 164 * other can only be read while rcu_read_lock or rtnl_lock is held.
 165 */
 166struct tun_file {
 167	struct sock sk;
 168	struct socket socket;
 169	struct socket_wq wq;
 170	struct tun_struct __rcu *tun;
 171	struct fasync_struct *fasync;
 172	/* only used for fasnyc */
 173	unsigned int flags;
 174	union {
 175		u16 queue_index;
 176		unsigned int ifindex;
 177	};
 178	struct napi_struct napi;
 179	bool napi_enabled;
 180	struct mutex napi_mutex;	/* Protects access to the above napi */
 181	struct list_head next;
 182	struct tun_struct *detached;
 183	struct ptr_ring tx_ring;
 184	struct xdp_rxq_info xdp_rxq;
 185};
 186
 187struct tun_flow_entry {
 188	struct hlist_node hash_link;
 189	struct rcu_head rcu;
 190	struct tun_struct *tun;
 191
 192	u32 rxhash;
 193	u32 rps_rxhash;
 194	int queue_index;
 195	unsigned long updated;
 196};
 197
 198#define TUN_NUM_FLOW_ENTRIES 1024
 199
 200struct tun_prog {
 201	struct rcu_head rcu;
 202	struct bpf_prog *prog;
 203};
 204
 205/* Since the socket were moved to tun_file, to preserve the behavior of persist
 206 * device, socket filter, sndbuf and vnet header size were restore when the
 207 * file were attached to a persist device.
 208 */
 209struct tun_struct {
 210	struct tun_file __rcu	*tfiles[MAX_TAP_QUEUES];
 211	unsigned int            numqueues;
 212	unsigned int 		flags;
 213	kuid_t			owner;
 214	kgid_t			group;
 215
 216	struct net_device	*dev;
 217	netdev_features_t	set_features;
 218#define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \
 219			  NETIF_F_TSO6)
 
 
 
 
 
 220
 221	int			align;
 222	int			vnet_hdr_sz;
 223	int			sndbuf;
 224	struct tap_filter	txflt;
 225	struct sock_fprog	fprog;
 226	/* protected by rtnl lock */
 227	bool			filter_attached;
 228#ifdef TUN_DEBUG
 229	int debug;
 230#endif
 231	spinlock_t lock;
 232	struct hlist_head flows[TUN_NUM_FLOW_ENTRIES];
 233	struct timer_list flow_gc_timer;
 234	unsigned long ageing_time;
 235	unsigned int numdisabled;
 236	struct list_head disabled;
 237	void *security;
 238	u32 flow_count;
 239	u32 rx_batched;
 240	struct tun_pcpu_stats __percpu *pcpu_stats;
 241	struct bpf_prog __rcu *xdp_prog;
 242	struct tun_prog __rcu *steering_prog;
 243	struct tun_prog __rcu *filter_prog;
 244};
 245
 246struct veth {
 247	__be16 h_vlan_proto;
 248	__be16 h_vlan_TCI;
 249};
 250
 251bool tun_is_xdp_buff(void *ptr)
 252{
 253	return (unsigned long)ptr & TUN_XDP_FLAG;
 254}
 255EXPORT_SYMBOL(tun_is_xdp_buff);
 256
 257void *tun_xdp_to_ptr(void *ptr)
 258{
 259	return (void *)((unsigned long)ptr | TUN_XDP_FLAG);
 260}
 261EXPORT_SYMBOL(tun_xdp_to_ptr);
 262
 263void *tun_ptr_to_xdp(void *ptr)
 264{
 265	return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG);
 266}
 267EXPORT_SYMBOL(tun_ptr_to_xdp);
 268
 269static int tun_napi_receive(struct napi_struct *napi, int budget)
 270{
 271	struct tun_file *tfile = container_of(napi, struct tun_file, napi);
 272	struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
 273	struct sk_buff_head process_queue;
 274	struct sk_buff *skb;
 275	int received = 0;
 276
 277	__skb_queue_head_init(&process_queue);
 278
 279	spin_lock(&queue->lock);
 280	skb_queue_splice_tail_init(queue, &process_queue);
 281	spin_unlock(&queue->lock);
 282
 283	while (received < budget && (skb = __skb_dequeue(&process_queue))) {
 284		napi_gro_receive(napi, skb);
 285		++received;
 286	}
 287
 288	if (!skb_queue_empty(&process_queue)) {
 289		spin_lock(&queue->lock);
 290		skb_queue_splice(&process_queue, queue);
 291		spin_unlock(&queue->lock);
 292	}
 293
 294	return received;
 295}
 296
 297static int tun_napi_poll(struct napi_struct *napi, int budget)
 298{
 299	unsigned int received;
 300
 301	received = tun_napi_receive(napi, budget);
 302
 303	if (received < budget)
 304		napi_complete_done(napi, received);
 305
 306	return received;
 307}
 308
 309static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
 310			  bool napi_en)
 311{
 312	tfile->napi_enabled = napi_en;
 313	if (napi_en) {
 314		netif_napi_add(tun->dev, &tfile->napi, tun_napi_poll,
 315			       NAPI_POLL_WEIGHT);
 316		napi_enable(&tfile->napi);
 317		mutex_init(&tfile->napi_mutex);
 318	}
 319}
 320
 321static void tun_napi_disable(struct tun_struct *tun, struct tun_file *tfile)
 322{
 323	if (tfile->napi_enabled)
 324		napi_disable(&tfile->napi);
 325}
 326
 327static void tun_napi_del(struct tun_struct *tun, struct tun_file *tfile)
 328{
 329	if (tfile->napi_enabled)
 330		netif_napi_del(&tfile->napi);
 331}
 332
 333static bool tun_napi_frags_enabled(const struct tun_struct *tun)
 334{
 335	return READ_ONCE(tun->flags) & IFF_NAPI_FRAGS;
 336}
 337
 338#ifdef CONFIG_TUN_VNET_CROSS_LE
 339static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
 340{
 341	return tun->flags & TUN_VNET_BE ? false :
 342		virtio_legacy_is_little_endian();
 343}
 344
 345static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
 346{
 347	int be = !!(tun->flags & TUN_VNET_BE);
 348
 349	if (put_user(be, argp))
 350		return -EFAULT;
 351
 352	return 0;
 353}
 354
 355static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
 356{
 357	int be;
 358
 359	if (get_user(be, argp))
 360		return -EFAULT;
 361
 362	if (be)
 363		tun->flags |= TUN_VNET_BE;
 364	else
 365		tun->flags &= ~TUN_VNET_BE;
 366
 367	return 0;
 368}
 369#else
 370static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
 371{
 372	return virtio_legacy_is_little_endian();
 373}
 374
 375static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
 376{
 377	return -EINVAL;
 378}
 379
 380static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
 381{
 382	return -EINVAL;
 383}
 384#endif /* CONFIG_TUN_VNET_CROSS_LE */
 385
 386static inline bool tun_is_little_endian(struct tun_struct *tun)
 387{
 388	return tun->flags & TUN_VNET_LE ||
 389		tun_legacy_is_little_endian(tun);
 390}
 391
 392static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val)
 393{
 394	return __virtio16_to_cpu(tun_is_little_endian(tun), val);
 395}
 396
 397static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val)
 398{
 399	return __cpu_to_virtio16(tun_is_little_endian(tun), val);
 400}
 401
 402static inline u32 tun_hashfn(u32 rxhash)
 403{
 404	return rxhash & 0x3ff;
 405}
 406
 407static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash)
 408{
 409	struct tun_flow_entry *e;
 410
 411	hlist_for_each_entry_rcu(e, head, hash_link) {
 412		if (e->rxhash == rxhash)
 413			return e;
 414	}
 415	return NULL;
 416}
 417
 418static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun,
 419					      struct hlist_head *head,
 420					      u32 rxhash, u16 queue_index)
 421{
 422	struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC);
 423
 424	if (e) {
 425		tun_debug(KERN_INFO, tun, "create flow: hash %u index %u\n",
 426			  rxhash, queue_index);
 427		e->updated = jiffies;
 428		e->rxhash = rxhash;
 429		e->rps_rxhash = 0;
 430		e->queue_index = queue_index;
 431		e->tun = tun;
 432		hlist_add_head_rcu(&e->hash_link, head);
 433		++tun->flow_count;
 434	}
 435	return e;
 436}
 437
 438static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e)
 439{
 440	tun_debug(KERN_INFO, tun, "delete flow: hash %u index %u\n",
 441		  e->rxhash, e->queue_index);
 442	hlist_del_rcu(&e->hash_link);
 443	kfree_rcu(e, rcu);
 444	--tun->flow_count;
 445}
 446
 447static void tun_flow_flush(struct tun_struct *tun)
 448{
 449	int i;
 450
 451	spin_lock_bh(&tun->lock);
 452	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
 453		struct tun_flow_entry *e;
 454		struct hlist_node *n;
 455
 456		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link)
 457			tun_flow_delete(tun, e);
 458	}
 459	spin_unlock_bh(&tun->lock);
 460}
 461
 462static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index)
 463{
 464	int i;
 465
 466	spin_lock_bh(&tun->lock);
 467	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
 468		struct tun_flow_entry *e;
 469		struct hlist_node *n;
 470
 471		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
 472			if (e->queue_index == queue_index)
 473				tun_flow_delete(tun, e);
 474		}
 475	}
 476	spin_unlock_bh(&tun->lock);
 477}
 478
 479static void tun_flow_cleanup(struct timer_list *t)
 480{
 481	struct tun_struct *tun = from_timer(tun, t, flow_gc_timer);
 482	unsigned long delay = tun->ageing_time;
 483	unsigned long next_timer = jiffies + delay;
 484	unsigned long count = 0;
 485	int i;
 486
 487	tun_debug(KERN_INFO, tun, "tun_flow_cleanup\n");
 488
 489	spin_lock(&tun->lock);
 490	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
 491		struct tun_flow_entry *e;
 492		struct hlist_node *n;
 493
 494		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
 495			unsigned long this_timer;
 496
 497			this_timer = e->updated + delay;
 498			if (time_before_eq(this_timer, jiffies)) {
 499				tun_flow_delete(tun, e);
 500				continue;
 501			}
 502			count++;
 503			if (time_before(this_timer, next_timer))
 504				next_timer = this_timer;
 505		}
 506	}
 507
 508	if (count)
 509		mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer));
 510	spin_unlock(&tun->lock);
 511}
 512
 513static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
 514			    struct tun_file *tfile)
 515{
 516	struct hlist_head *head;
 517	struct tun_flow_entry *e;
 518	unsigned long delay = tun->ageing_time;
 519	u16 queue_index = tfile->queue_index;
 520
 521	if (!rxhash)
 522		return;
 523	else
 524		head = &tun->flows[tun_hashfn(rxhash)];
 525
 526	rcu_read_lock();
 527
 528	/* We may get a very small possibility of OOO during switching, not
 529	 * worth to optimize.*/
 530	if (tun->numqueues == 1 || tfile->detached)
 531		goto unlock;
 532
 533	e = tun_flow_find(head, rxhash);
 534	if (likely(e)) {
 535		/* TODO: keep queueing to old queue until it's empty? */
 536		e->queue_index = queue_index;
 537		e->updated = jiffies;
 538		sock_rps_record_flow_hash(e->rps_rxhash);
 539	} else {
 540		spin_lock_bh(&tun->lock);
 541		if (!tun_flow_find(head, rxhash) &&
 542		    tun->flow_count < MAX_TAP_FLOWS)
 543			tun_flow_create(tun, head, rxhash, queue_index);
 544
 545		if (!timer_pending(&tun->flow_gc_timer))
 546			mod_timer(&tun->flow_gc_timer,
 547				  round_jiffies_up(jiffies + delay));
 548		spin_unlock_bh(&tun->lock);
 549	}
 550
 551unlock:
 552	rcu_read_unlock();
 553}
 554
 555/**
 556 * Save the hash received in the stack receive path and update the
 557 * flow_hash table accordingly.
 558 */
 559static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash)
 560{
 561	if (unlikely(e->rps_rxhash != hash))
 562		e->rps_rxhash = hash;
 563}
 564
 565/* We try to identify a flow through its rxhash first. The reason that
 566 * we do not check rxq no. is because some cards(e.g 82599), chooses
 567 * the rxq based on the txq where the last packet of the flow comes. As
 568 * the userspace application move between processors, we may get a
 569 * different rxq no. here. If we could not get rxhash, then we would
 570 * hope the rxq no. may help here.
 571 */
 572static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)
 573{
 574	struct tun_flow_entry *e;
 575	u32 txq = 0;
 576	u32 numqueues = 0;
 577
 578	numqueues = READ_ONCE(tun->numqueues);
 579
 580	txq = __skb_get_hash_symmetric(skb);
 581	if (txq) {
 582		e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq);
 583		if (e) {
 584			tun_flow_save_rps_rxhash(e, txq);
 585			txq = e->queue_index;
 586		} else
 587			/* use multiply and shift instead of expensive divide */
 588			txq = ((u64)txq * numqueues) >> 32;
 589	} else if (likely(skb_rx_queue_recorded(skb))) {
 590		txq = skb_get_rx_queue(skb);
 591		while (unlikely(txq >= numqueues))
 592			txq -= numqueues;
 593	}
 594
 595	return txq;
 596}
 597
 598static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)
 599{
 600	struct tun_prog *prog;
 601	u16 ret = 0;
 602
 603	prog = rcu_dereference(tun->steering_prog);
 604	if (prog)
 605		ret = bpf_prog_run_clear_cb(prog->prog, skb);
 606
 607	return ret % tun->numqueues;
 608}
 609
 610static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
 611			    void *accel_priv, select_queue_fallback_t fallback)
 612{
 613	struct tun_struct *tun = netdev_priv(dev);
 614	u16 ret;
 615
 616	rcu_read_lock();
 617	if (rcu_dereference(tun->steering_prog))
 618		ret = tun_ebpf_select_queue(tun, skb);
 619	else
 620		ret = tun_automq_select_queue(tun, skb);
 621	rcu_read_unlock();
 622
 623	return ret;
 624}
 625
 626static inline bool tun_not_capable(struct tun_struct *tun)
 627{
 628	const struct cred *cred = current_cred();
 629	struct net *net = dev_net(tun->dev);
 630
 631	return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) ||
 632		  (gid_valid(tun->group) && !in_egroup_p(tun->group))) &&
 633		!ns_capable(net->user_ns, CAP_NET_ADMIN);
 634}
 635
 636static void tun_set_real_num_queues(struct tun_struct *tun)
 637{
 638	netif_set_real_num_tx_queues(tun->dev, tun->numqueues);
 639	netif_set_real_num_rx_queues(tun->dev, tun->numqueues);
 640}
 641
 642static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
 643{
 644	tfile->detached = tun;
 645	list_add_tail(&tfile->next, &tun->disabled);
 646	++tun->numdisabled;
 647}
 648
 649static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
 650{
 651	struct tun_struct *tun = tfile->detached;
 652
 653	tfile->detached = NULL;
 654	list_del_init(&tfile->next);
 655	--tun->numdisabled;
 656	return tun;
 657}
 658
 659void tun_ptr_free(void *ptr)
 660{
 661	if (!ptr)
 662		return;
 663	if (tun_is_xdp_buff(ptr)) {
 664		struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
 665
 666		put_page(virt_to_head_page(xdp->data));
 667	} else {
 668		__skb_array_destroy_skb(ptr);
 669	}
 670}
 671EXPORT_SYMBOL_GPL(tun_ptr_free);
 672
 673static void tun_queue_purge(struct tun_file *tfile)
 674{
 675	void *ptr;
 676
 677	while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
 678		tun_ptr_free(ptr);
 
 
 
 
 
 
 679
 680	skb_queue_purge(&tfile->sk.sk_write_queue);
 681	skb_queue_purge(&tfile->sk.sk_error_queue);
 
 682}
 683
 684static void __tun_detach(struct tun_file *tfile, bool clean)
 685{
 686	struct tun_file *ntfile;
 687	struct tun_struct *tun;
 688
 689	tun = rtnl_dereference(tfile->tun);
 
 
 690
 691	if (tun && clean) {
 692		tun_napi_disable(tun, tfile);
 693		tun_napi_del(tun, tfile);
 694	}
 695
 696	if (tun && !tfile->detached) {
 697		u16 index = tfile->queue_index;
 698		BUG_ON(index >= tun->numqueues);
 699
 700		rcu_assign_pointer(tun->tfiles[index],
 701				   tun->tfiles[tun->numqueues - 1]);
 702		ntfile = rtnl_dereference(tun->tfiles[index]);
 703		ntfile->queue_index = index;
 704
 705		--tun->numqueues;
 706		if (clean) {
 707			RCU_INIT_POINTER(tfile->tun, NULL);
 708			sock_put(&tfile->sk);
 709		} else
 710			tun_disable_queue(tun, tfile);
 711
 712		synchronize_net();
 713		tun_flow_delete_by_queue(tun, tun->numqueues + 1);
 714		/* Drop read queue */
 715		tun_queue_purge(tfile);
 716		tun_set_real_num_queues(tun);
 717	} else if (tfile->detached && clean) {
 718		tun = tun_enable_queue(tfile);
 719		sock_put(&tfile->sk);
 720	}
 721
 722	if (clean) {
 723		if (tun && tun->numqueues == 0 && tun->numdisabled == 0) {
 724			netif_carrier_off(tun->dev);
 725
 726			if (!(tun->flags & IFF_PERSIST) &&
 727			    tun->dev->reg_state == NETREG_REGISTERED)
 728				unregister_netdevice(tun->dev);
 729		}
 730		if (tun)
 731			xdp_rxq_info_unreg(&tfile->xdp_rxq);
 732		ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
 733		sock_put(&tfile->sk);
 734	}
 735}
 736
 737static void tun_detach(struct tun_file *tfile, bool clean)
 738{
 739	struct tun_struct *tun;
 740	struct net_device *dev;
 741
 742	rtnl_lock();
 743	tun = rtnl_dereference(tfile->tun);
 744	dev = tun ? tun->dev : NULL;
 745	__tun_detach(tfile, clean);
 746	if (dev)
 747		netdev_state_change(dev);
 748	rtnl_unlock();
 749}
 750
 751static void tun_detach_all(struct net_device *dev)
 752{
 753	struct tun_struct *tun = netdev_priv(dev);
 754	struct tun_file *tfile, *tmp;
 755	int i, n = tun->numqueues;
 756
 757	for (i = 0; i < n; i++) {
 758		tfile = rtnl_dereference(tun->tfiles[i]);
 759		BUG_ON(!tfile);
 760		tun_napi_disable(tun, tfile);
 761		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
 762		tfile->socket.sk->sk_data_ready(tfile->socket.sk);
 763		RCU_INIT_POINTER(tfile->tun, NULL);
 764		--tun->numqueues;
 765	}
 766	list_for_each_entry(tfile, &tun->disabled, next) {
 767		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
 768		tfile->socket.sk->sk_data_ready(tfile->socket.sk);
 769		RCU_INIT_POINTER(tfile->tun, NULL);
 770	}
 771	BUG_ON(tun->numqueues != 0);
 772
 773	synchronize_net();
 774	for (i = 0; i < n; i++) {
 775		tfile = rtnl_dereference(tun->tfiles[i]);
 776		tun_napi_del(tun, tfile);
 777		/* Drop read queue */
 778		tun_queue_purge(tfile);
 779		xdp_rxq_info_unreg(&tfile->xdp_rxq);
 780		sock_put(&tfile->sk);
 781	}
 782	list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
 783		tun_enable_queue(tfile);
 784		tun_queue_purge(tfile);
 785		xdp_rxq_info_unreg(&tfile->xdp_rxq);
 786		sock_put(&tfile->sk);
 787	}
 788	BUG_ON(tun->numdisabled != 0);
 789
 790	if (tun->flags & IFF_PERSIST)
 791		module_put(THIS_MODULE);
 792}
 793
 794static int tun_attach(struct tun_struct *tun, struct file *file,
 795		      bool skip_filter, bool napi)
 796{
 797	struct tun_file *tfile = file->private_data;
 798	struct net_device *dev = tun->dev;
 799	int err;
 800
 801	err = security_tun_dev_attach(tfile->socket.sk, tun->security);
 802	if (err < 0)
 803		goto out;
 804
 805	err = -EINVAL;
 806	if (rtnl_dereference(tfile->tun) && !tfile->detached)
 807		goto out;
 808
 809	err = -EBUSY;
 810	if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1)
 811		goto out;
 812
 813	err = -E2BIG;
 814	if (!tfile->detached &&
 815	    tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES)
 816		goto out;
 817
 818	err = 0;
 819
 820	/* Re-attach the filter to persist device */
 821	if (!skip_filter && (tun->filter_attached == true)) {
 822		lock_sock(tfile->socket.sk);
 823		err = sk_attach_filter(&tun->fprog, tfile->socket.sk);
 824		release_sock(tfile->socket.sk);
 825		if (!err)
 826			goto out;
 827	}
 828
 829	if (!tfile->detached &&
 830	    ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len,
 831			    GFP_KERNEL, tun_ptr_free)) {
 832		err = -ENOMEM;
 833		goto out;
 834	}
 835
 836	tfile->queue_index = tun->numqueues;
 837	tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
 838
 839	if (tfile->detached) {
 840		/* Re-attach detached tfile, updating XDP queue_index */
 841		WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));
 842
 843		if (tfile->xdp_rxq.queue_index    != tfile->queue_index)
 844			tfile->xdp_rxq.queue_index = tfile->queue_index;
 845	} else {
 846		/* Setup XDP RX-queue info, for new tfile getting attached */
 847		err = xdp_rxq_info_reg(&tfile->xdp_rxq,
 848				       tun->dev, tfile->queue_index);
 849		if (err < 0)
 850			goto out;
 851		err = 0;
 852	}
 853
 854	rcu_assign_pointer(tfile->tun, tun);
 855	rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
 856	tun->numqueues++;
 857
 858	if (tfile->detached) {
 859		tun_enable_queue(tfile);
 860	} else {
 861		sock_hold(&tfile->sk);
 862		tun_napi_init(tun, tfile, napi);
 863	}
 864
 865	tun_set_real_num_queues(tun);
 866
 867	/* device is allowed to go away first, so no need to hold extra
 868	 * refcnt.
 869	 */
 870
 871out:
 872	return err;
 873}
 874
 875static struct tun_struct *tun_get(struct tun_file *tfile)
 876{
 877	struct tun_struct *tun;
 878
 879	rcu_read_lock();
 880	tun = rcu_dereference(tfile->tun);
 881	if (tun)
 882		dev_hold(tun->dev);
 883	rcu_read_unlock();
 884
 885	return tun;
 886}
 887
 888static void tun_put(struct tun_struct *tun)
 889{
 890	dev_put(tun->dev);
 
 
 
 891}
 892
 893/* TAP filtering */
 894static void addr_hash_set(u32 *mask, const u8 *addr)
 895{
 896	int n = ether_crc(ETH_ALEN, addr) >> 26;
 897	mask[n >> 5] |= (1 << (n & 31));
 898}
 899
 900static unsigned int addr_hash_test(const u32 *mask, const u8 *addr)
 901{
 902	int n = ether_crc(ETH_ALEN, addr) >> 26;
 903	return mask[n >> 5] & (1 << (n & 31));
 904}
 905
 906static int update_filter(struct tap_filter *filter, void __user *arg)
 907{
 908	struct { u8 u[ETH_ALEN]; } *addr;
 909	struct tun_filter uf;
 910	int err, alen, n, nexact;
 911
 912	if (copy_from_user(&uf, arg, sizeof(uf)))
 913		return -EFAULT;
 914
 915	if (!uf.count) {
 916		/* Disabled */
 917		filter->count = 0;
 918		return 0;
 919	}
 920
 921	alen = ETH_ALEN * uf.count;
 922	addr = memdup_user(arg + sizeof(uf), alen);
 923	if (IS_ERR(addr))
 924		return PTR_ERR(addr);
 
 
 
 
 
 925
 926	/* The filter is updated without holding any locks. Which is
 927	 * perfectly safe. We disable it first and in the worst
 928	 * case we'll accept a few undesired packets. */
 929	filter->count = 0;
 930	wmb();
 931
 932	/* Use first set of addresses as an exact filter */
 933	for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++)
 934		memcpy(filter->addr[n], addr[n].u, ETH_ALEN);
 935
 936	nexact = n;
 937
 938	/* Remaining multicast addresses are hashed,
 939	 * unicast will leave the filter disabled. */
 940	memset(filter->mask, 0, sizeof(filter->mask));
 941	for (; n < uf.count; n++) {
 942		if (!is_multicast_ether_addr(addr[n].u)) {
 943			err = 0; /* no filter */
 944			goto free_addr;
 945		}
 946		addr_hash_set(filter->mask, addr[n].u);
 947	}
 948
 949	/* For ALLMULTI just set the mask to all ones.
 950	 * This overrides the mask populated above. */
 951	if ((uf.flags & TUN_FLT_ALLMULTI))
 952		memset(filter->mask, ~0, sizeof(filter->mask));
 953
 954	/* Now enable the filter */
 955	wmb();
 956	filter->count = nexact;
 957
 958	/* Return the number of exact filters */
 959	err = nexact;
 960free_addr:
 
 961	kfree(addr);
 962	return err;
 963}
 964
 965/* Returns: 0 - drop, !=0 - accept */
 966static int run_filter(struct tap_filter *filter, const struct sk_buff *skb)
 967{
 968	/* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect
 969	 * at this point. */
 970	struct ethhdr *eh = (struct ethhdr *) skb->data;
 971	int i;
 972
 973	/* Exact match */
 974	for (i = 0; i < filter->count; i++)
 975		if (ether_addr_equal(eh->h_dest, filter->addr[i]))
 976			return 1;
 977
 978	/* Inexact match (multicast only) */
 979	if (is_multicast_ether_addr(eh->h_dest))
 980		return addr_hash_test(filter->mask, eh->h_dest);
 981
 982	return 0;
 983}
 984
 985/*
 986 * Checks whether the packet is accepted or not.
 987 * Returns: 0 - drop, !=0 - accept
 988 */
 989static int check_filter(struct tap_filter *filter, const struct sk_buff *skb)
 990{
 991	if (!filter->count)
 992		return 1;
 993
 994	return run_filter(filter, skb);
 995}
 996
 997/* Network device part of the driver */
 998
 999static const struct ethtool_ops tun_ethtool_ops;
1000
1001/* Net device detach from fd. */
1002static void tun_net_uninit(struct net_device *dev)
1003{
1004	tun_detach_all(dev);
 
 
 
 
 
 
 
 
 
1005}
1006
1007/* Net device open. */
1008static int tun_net_open(struct net_device *dev)
1009{
1010	struct tun_struct *tun = netdev_priv(dev);
1011	int i;
1012
1013	netif_tx_start_all_queues(dev);
1014
1015	for (i = 0; i < tun->numqueues; i++) {
1016		struct tun_file *tfile;
1017
1018		tfile = rtnl_dereference(tun->tfiles[i]);
1019		tfile->socket.sk->sk_write_space(tfile->socket.sk);
1020	}
1021
 
 
 
 
1022	return 0;
1023}
1024
1025/* Net device close. */
1026static int tun_net_close(struct net_device *dev)
1027{
1028	netif_tx_stop_all_queues(dev);
1029	return 0;
1030}
1031
1032/* Net device start xmit */
1033static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb)
1034{
1035#ifdef CONFIG_RPS
1036	if (tun->numqueues == 1 && static_key_false(&rps_needed)) {
1037		/* Select queue was not called for the skbuff, so we extract the
1038		 * RPS hash and save it into the flow_table here.
1039		 */
1040		__u32 rxhash;
1041
1042		rxhash = __skb_get_hash_symmetric(skb);
1043		if (rxhash) {
1044			struct tun_flow_entry *e;
1045			e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)],
1046					rxhash);
1047			if (e)
1048				tun_flow_save_rps_rxhash(e, rxhash);
1049		}
1050	}
1051#endif
1052}
1053
1054static unsigned int run_ebpf_filter(struct tun_struct *tun,
1055				    struct sk_buff *skb,
1056				    int len)
1057{
1058	struct tun_prog *prog = rcu_dereference(tun->filter_prog);
1059
1060	if (prog)
1061		len = bpf_prog_run_clear_cb(prog->prog, skb);
1062
1063	return len;
1064}
1065
1066/* Net device start xmit */
1067static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
1068{
1069	struct tun_struct *tun = netdev_priv(dev);
1070	int txq = skb->queue_mapping;
1071	struct tun_file *tfile;
1072	int len = skb->len;
1073
1074	rcu_read_lock();
1075	tfile = rcu_dereference(tun->tfiles[txq]);
1076
1077	/* Drop packet if interface is not attached */
1078	if (txq >= tun->numqueues)
1079		goto drop;
1080
1081	if (!rcu_dereference(tun->steering_prog))
1082		tun_automq_xmit(tun, skb);
1083
1084	tun_debug(KERN_INFO, tun, "tun_net_xmit %d\n", skb->len);
1085
1086	BUG_ON(!tfile);
1087
1088	/* Drop if the filter does not like it.
1089	 * This is a noop if the filter is disabled.
1090	 * Filter can be enabled only for the TAP devices. */
1091	if (!check_filter(&tun->txflt, skb))
1092		goto drop;
1093
1094	if (tfile->socket.sk->sk_filter &&
1095	    sk_filter(tfile->socket.sk, skb))
1096		goto drop;
1097
1098	len = run_ebpf_filter(tun, skb, len);
1099	if (len == 0 || pskb_trim(skb, len))
1100		goto drop;
1101
1102	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
1103		goto drop;
1104
1105	skb_tx_timestamp(skb);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1106
1107	/* Orphan the skb - required as we might hang on to it
1108	 * for indefinite time.
1109	 */
1110	skb_orphan(skb);
1111
1112	nf_reset(skb);
1113
1114	if (ptr_ring_produce(&tfile->tx_ring, skb))
1115		goto drop;
1116
1117	/* Notify and wake up reader process */
1118	if (tfile->flags & TUN_FASYNC)
1119		kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1120	tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1121
1122	rcu_read_unlock();
1123	return NETDEV_TX_OK;
1124
1125drop:
1126	this_cpu_inc(tun->pcpu_stats->tx_dropped);
1127	skb_tx_error(skb);
1128	kfree_skb(skb);
1129	rcu_read_unlock();
1130	return NET_XMIT_DROP;
1131}
1132
1133static void tun_net_mclist(struct net_device *dev)
1134{
1135	/*
1136	 * This callback is supposed to deal with mc filter in
1137	 * _rx_ path and has nothing to do with the _tx_ path.
1138	 * In rx path we always accept everything userspace gives us.
1139	 */
1140}
1141
1142static netdev_features_t tun_net_fix_features(struct net_device *dev,
1143	netdev_features_t features)
 
 
 
 
 
 
 
 
 
 
 
1144{
1145	struct tun_struct *tun = netdev_priv(dev);
1146
1147	return (features & tun->set_features) | (features & ~TUN_USER_FEATURES);
1148}
1149#ifdef CONFIG_NET_POLL_CONTROLLER
1150static void tun_poll_controller(struct net_device *dev)
1151{
1152	/*
1153	 * Tun only receives frames when:
1154	 * 1) the char device endpoint gets data from user space
1155	 * 2) the tun socket gets a sendmsg call from user space
1156	 * If NAPI is not enabled, since both of those are synchronous
1157	 * operations, we are guaranteed never to have pending data when we poll
1158	 * for it so there is nothing to do here but return.
1159	 * We need this though so netpoll recognizes us as an interface that
1160	 * supports polling, which enables bridge devices in virt setups to
1161	 * still use netconsole
1162	 * If NAPI is enabled, however, we need to schedule polling for all
1163	 * queues unless we are using napi_gro_frags(), which we call in
1164	 * process context and not in NAPI context.
1165	 */
1166	struct tun_struct *tun = netdev_priv(dev);
1167
1168	if (tun->flags & IFF_NAPI) {
1169		struct tun_file *tfile;
1170		int i;
1171
1172		if (tun_napi_frags_enabled(tun))
1173			return;
1174
1175		rcu_read_lock();
1176		for (i = 0; i < tun->numqueues; i++) {
1177			tfile = rcu_dereference(tun->tfiles[i]);
1178			if (tfile->napi_enabled)
1179				napi_schedule(&tfile->napi);
1180		}
1181		rcu_read_unlock();
1182	}
1183	return;
1184}
1185#endif
1186
1187static void tun_set_headroom(struct net_device *dev, int new_hr)
1188{
1189	struct tun_struct *tun = netdev_priv(dev);
1190
1191	if (new_hr < NET_SKB_PAD)
1192		new_hr = NET_SKB_PAD;
1193
1194	tun->align = new_hr;
1195}
1196
1197static void
1198tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
1199{
1200	u32 rx_dropped = 0, tx_dropped = 0, rx_frame_errors = 0;
1201	struct tun_struct *tun = netdev_priv(dev);
1202	struct tun_pcpu_stats *p;
1203	int i;
1204
1205	for_each_possible_cpu(i) {
1206		u64 rxpackets, rxbytes, txpackets, txbytes;
1207		unsigned int start;
1208
1209		p = per_cpu_ptr(tun->pcpu_stats, i);
1210		do {
1211			start = u64_stats_fetch_begin(&p->syncp);
1212			rxpackets	= p->rx_packets;
1213			rxbytes		= p->rx_bytes;
1214			txpackets	= p->tx_packets;
1215			txbytes		= p->tx_bytes;
1216		} while (u64_stats_fetch_retry(&p->syncp, start));
1217
1218		stats->rx_packets	+= rxpackets;
1219		stats->rx_bytes		+= rxbytes;
1220		stats->tx_packets	+= txpackets;
1221		stats->tx_bytes		+= txbytes;
1222
1223		/* u32 counters */
1224		rx_dropped	+= p->rx_dropped;
1225		rx_frame_errors	+= p->rx_frame_errors;
1226		tx_dropped	+= p->tx_dropped;
1227	}
1228	stats->rx_dropped  = rx_dropped;
1229	stats->rx_frame_errors = rx_frame_errors;
1230	stats->tx_dropped = tx_dropped;
1231}
1232
1233static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1234		       struct netlink_ext_ack *extack)
1235{
1236	struct tun_struct *tun = netdev_priv(dev);
1237	struct bpf_prog *old_prog;
1238
1239	old_prog = rtnl_dereference(tun->xdp_prog);
1240	rcu_assign_pointer(tun->xdp_prog, prog);
1241	if (old_prog)
1242		bpf_prog_put(old_prog);
1243
1244	return 0;
1245}
1246
1247static u32 tun_xdp_query(struct net_device *dev)
1248{
1249	struct tun_struct *tun = netdev_priv(dev);
1250	const struct bpf_prog *xdp_prog;
1251
1252	xdp_prog = rtnl_dereference(tun->xdp_prog);
1253	if (xdp_prog)
1254		return xdp_prog->aux->id;
1255
1256	return 0;
1257}
1258
1259static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1260{
1261	switch (xdp->command) {
1262	case XDP_SETUP_PROG:
1263		return tun_xdp_set(dev, xdp->prog, xdp->extack);
1264	case XDP_QUERY_PROG:
1265		xdp->prog_id = tun_xdp_query(dev);
1266		xdp->prog_attached = !!xdp->prog_id;
1267		return 0;
1268	default:
1269		return -EINVAL;
1270	}
1271}
1272
1273static const struct net_device_ops tun_netdev_ops = {
1274	.ndo_uninit		= tun_net_uninit,
1275	.ndo_open		= tun_net_open,
1276	.ndo_stop		= tun_net_close,
1277	.ndo_start_xmit		= tun_net_xmit,
 
1278	.ndo_fix_features	= tun_net_fix_features,
1279	.ndo_select_queue	= tun_select_queue,
1280#ifdef CONFIG_NET_POLL_CONTROLLER
1281	.ndo_poll_controller	= tun_poll_controller,
1282#endif
1283	.ndo_set_rx_headroom	= tun_set_headroom,
1284	.ndo_get_stats64	= tun_net_get_stats64,
1285};
1286
1287static int tun_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
1288{
1289	struct tun_struct *tun = netdev_priv(dev);
1290	struct xdp_buff *buff = xdp->data_hard_start;
1291	int headroom = xdp->data - xdp->data_hard_start;
1292	struct tun_file *tfile;
1293	u32 numqueues;
1294	int ret = 0;
1295
1296	/* Assure headroom is available and buff is properly aligned */
1297	if (unlikely(headroom < sizeof(*xdp) || tun_is_xdp_buff(xdp)))
1298		return -ENOSPC;
1299
1300	*buff = *xdp;
1301
1302	rcu_read_lock();
1303
1304	numqueues = READ_ONCE(tun->numqueues);
1305	if (!numqueues) {
1306		ret = -ENOSPC;
1307		goto out;
1308	}
1309
1310	tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
1311					    numqueues]);
1312	/* Encode the XDP flag into lowest bit for consumer to differ
1313	 * XDP buffer from sk_buff.
1314	 */
1315	if (ptr_ring_produce(&tfile->tx_ring, tun_xdp_to_ptr(buff))) {
1316		this_cpu_inc(tun->pcpu_stats->tx_dropped);
1317		ret = -ENOSPC;
1318	}
1319
1320out:
1321	rcu_read_unlock();
1322	return ret;
1323}
1324
1325static void tun_xdp_flush(struct net_device *dev)
1326{
1327	struct tun_struct *tun = netdev_priv(dev);
1328	struct tun_file *tfile;
1329	u32 numqueues;
1330
1331	rcu_read_lock();
1332
1333	numqueues = READ_ONCE(tun->numqueues);
1334	if (!numqueues)
1335		goto out;
1336
1337	tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
1338					    numqueues]);
1339	/* Notify and wake up reader process */
1340	if (tfile->flags & TUN_FASYNC)
1341		kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1342	tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1343
1344out:
1345	rcu_read_unlock();
1346}
1347
1348static const struct net_device_ops tap_netdev_ops = {
1349	.ndo_uninit		= tun_net_uninit,
1350	.ndo_open		= tun_net_open,
1351	.ndo_stop		= tun_net_close,
1352	.ndo_start_xmit		= tun_net_xmit,
 
1353	.ndo_fix_features	= tun_net_fix_features,
1354	.ndo_set_rx_mode	= tun_net_mclist,
1355	.ndo_set_mac_address	= eth_mac_addr,
1356	.ndo_validate_addr	= eth_validate_addr,
1357	.ndo_select_queue	= tun_select_queue,
1358#ifdef CONFIG_NET_POLL_CONTROLLER
1359	.ndo_poll_controller	= tun_poll_controller,
1360#endif
1361	.ndo_features_check	= passthru_features_check,
1362	.ndo_set_rx_headroom	= tun_set_headroom,
1363	.ndo_get_stats64	= tun_net_get_stats64,
1364	.ndo_bpf		= tun_xdp,
1365	.ndo_xdp_xmit		= tun_xdp_xmit,
1366	.ndo_xdp_flush		= tun_xdp_flush,
1367};
1368
1369static void tun_flow_init(struct tun_struct *tun)
1370{
1371	int i;
1372
1373	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++)
1374		INIT_HLIST_HEAD(&tun->flows[i]);
1375
1376	tun->ageing_time = TUN_FLOW_EXPIRE;
1377	timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0);
1378	mod_timer(&tun->flow_gc_timer,
1379		  round_jiffies_up(jiffies + tun->ageing_time));
1380}
1381
1382static void tun_flow_uninit(struct tun_struct *tun)
1383{
1384	del_timer_sync(&tun->flow_gc_timer);
1385	tun_flow_flush(tun);
1386}
1387
1388#define MIN_MTU 68
1389#define MAX_MTU 65535
1390
1391/* Initialize net device. */
1392static void tun_net_init(struct net_device *dev)
1393{
1394	struct tun_struct *tun = netdev_priv(dev);
1395
1396	switch (tun->flags & TUN_TYPE_MASK) {
1397	case IFF_TUN:
1398		dev->netdev_ops = &tun_netdev_ops;
1399
1400		/* Point-to-Point TUN Device */
1401		dev->hard_header_len = 0;
1402		dev->addr_len = 0;
1403		dev->mtu = 1500;
1404
1405		/* Zero header length */
1406		dev->type = ARPHRD_NONE;
1407		dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
 
1408		break;
1409
1410	case IFF_TAP:
1411		dev->netdev_ops = &tap_netdev_ops;
1412		/* Ethernet TAP Device */
1413		ether_setup(dev);
1414		dev->priv_flags &= ~IFF_TX_SKB_SHARING;
1415		dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
1416
1417		eth_hw_addr_random(dev);
1418
 
1419		break;
1420	}
1421
1422	dev->min_mtu = MIN_MTU;
1423	dev->max_mtu = MAX_MTU - dev->hard_header_len;
1424}
1425
1426static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile)
1427{
1428	struct sock *sk = tfile->socket.sk;
1429
1430	return (tun->dev->flags & IFF_UP) && sock_writeable(sk);
1431}
1432
1433/* Character device part */
1434
1435/* Poll */
1436static __poll_t tun_chr_poll(struct file *file, poll_table *wait)
1437{
1438	struct tun_file *tfile = file->private_data;
1439	struct tun_struct *tun = tun_get(tfile);
1440	struct sock *sk;
1441	__poll_t mask = 0;
1442
1443	if (!tun)
1444		return EPOLLERR;
1445
1446	sk = tfile->socket.sk;
1447
1448	tun_debug(KERN_INFO, tun, "tun_chr_poll\n");
1449
1450	poll_wait(file, sk_sleep(sk), wait);
1451
1452	if (!ptr_ring_empty(&tfile->tx_ring))
1453		mask |= EPOLLIN | EPOLLRDNORM;
1454
1455	/* Make sure SOCKWQ_ASYNC_NOSPACE is set if not writable to
1456	 * guarantee EPOLLOUT to be raised by either here or
1457	 * tun_sock_write_space(). Then process could get notification
1458	 * after it writes to a down device and meets -EIO.
1459	 */
1460	if (tun_sock_writeable(tun, tfile) ||
1461	    (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) &&
1462	     tun_sock_writeable(tun, tfile)))
1463		mask |= EPOLLOUT | EPOLLWRNORM;
1464
1465	if (tun->dev->reg_state != NETREG_REGISTERED)
1466		mask = EPOLLERR;
1467
1468	tun_put(tun);
1469	return mask;
1470}
1471
1472static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
1473					    size_t len,
1474					    const struct iov_iter *it)
1475{
1476	struct sk_buff *skb;
1477	size_t linear;
1478	int err;
1479	int i;
1480
1481	if (it->nr_segs > MAX_SKB_FRAGS + 1)
1482		return ERR_PTR(-ENOMEM);
1483
1484	local_bh_disable();
1485	skb = napi_get_frags(&tfile->napi);
1486	local_bh_enable();
1487	if (!skb)
1488		return ERR_PTR(-ENOMEM);
1489
1490	linear = iov_iter_single_seg_count(it);
1491	err = __skb_grow(skb, linear);
1492	if (err)
1493		goto free;
1494
1495	skb->len = len;
1496	skb->data_len = len - linear;
1497	skb->truesize += skb->data_len;
1498
1499	for (i = 1; i < it->nr_segs; i++) {
1500		struct page_frag *pfrag = &current->task_frag;
1501		size_t fragsz = it->iov[i].iov_len;
1502
1503		if (fragsz == 0 || fragsz > PAGE_SIZE) {
1504			err = -EINVAL;
1505			goto free;
1506		}
1507
1508		if (!skb_page_frag_refill(fragsz, pfrag, GFP_KERNEL)) {
1509			err = -ENOMEM;
1510			goto free;
1511		}
1512
1513		skb_fill_page_desc(skb, i - 1, pfrag->page,
1514				   pfrag->offset, fragsz);
1515		page_ref_inc(pfrag->page);
1516		pfrag->offset += fragsz;
1517	}
1518
1519	return skb;
1520free:
1521	/* frees skb and all frags allocated with napi_alloc_frag() */
1522	napi_free_frags(&tfile->napi);
1523	return ERR_PTR(err);
1524}
1525
1526/* prepad is the amount to reserve at front.  len is length after that.
1527 * linear is a hint as to how much to copy (usually headers). */
1528static struct sk_buff *tun_alloc_skb(struct tun_file *tfile,
1529				     size_t prepad, size_t len,
1530				     size_t linear, int noblock)
1531{
1532	struct sock *sk = tfile->socket.sk;
1533	struct sk_buff *skb;
1534	int err;
1535
 
 
1536	/* Under a page?  Don't bother with paged skb. */
1537	if (prepad + len < PAGE_SIZE || !linear)
1538		linear = len;
1539
1540	skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
1541				   &err, 0);
1542	if (!skb)
1543		return ERR_PTR(err);
1544
1545	skb_reserve(skb, prepad);
1546	skb_put(skb, linear);
1547	skb->data_len = len - linear;
1548	skb->len += len - linear;
1549
1550	return skb;
1551}
1552
1553static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,
1554			   struct sk_buff *skb, int more)
1555{
1556	struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1557	struct sk_buff_head process_queue;
1558	u32 rx_batched = tun->rx_batched;
1559	bool rcv = false;
1560
1561	if (!rx_batched || (!more && skb_queue_empty(queue))) {
1562		local_bh_disable();
1563		netif_receive_skb(skb);
1564		local_bh_enable();
1565		return;
1566	}
1567
1568	spin_lock(&queue->lock);
1569	if (!more || skb_queue_len(queue) == rx_batched) {
1570		__skb_queue_head_init(&process_queue);
1571		skb_queue_splice_tail_init(queue, &process_queue);
1572		rcv = true;
1573	} else {
1574		__skb_queue_tail(queue, skb);
1575	}
1576	spin_unlock(&queue->lock);
1577
1578	if (rcv) {
1579		struct sk_buff *nskb;
1580
1581		local_bh_disable();
1582		while ((nskb = __skb_dequeue(&process_queue)))
1583			netif_receive_skb(nskb);
1584		netif_receive_skb(skb);
1585		local_bh_enable();
1586	}
1587}
1588
1589static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,
1590			      int len, int noblock, bool zerocopy)
1591{
1592	if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
1593		return false;
1594
1595	if (tfile->socket.sk->sk_sndbuf != INT_MAX)
1596		return false;
1597
1598	if (!noblock)
1599		return false;
1600
1601	if (zerocopy)
1602		return false;
1603
1604	if (SKB_DATA_ALIGN(len + TUN_RX_PAD) +
1605	    SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE)
1606		return false;
1607
1608	return true;
1609}
1610
1611static struct sk_buff *tun_build_skb(struct tun_struct *tun,
1612				     struct tun_file *tfile,
1613				     struct iov_iter *from,
1614				     struct virtio_net_hdr *hdr,
1615				     int len, int *skb_xdp)
1616{
1617	struct page_frag *alloc_frag = &current->task_frag;
1618	struct sk_buff *skb;
1619	struct bpf_prog *xdp_prog;
1620	int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1621	unsigned int delta = 0;
1622	char *buf;
1623	size_t copied;
1624	int err, pad = TUN_RX_PAD;
1625
1626	rcu_read_lock();
1627	xdp_prog = rcu_dereference(tun->xdp_prog);
1628	if (xdp_prog)
1629		pad += TUN_HEADROOM;
1630	buflen += SKB_DATA_ALIGN(len + pad);
1631	rcu_read_unlock();
1632
1633	alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
1634	if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL)))
1635		return ERR_PTR(-ENOMEM);
1636
1637	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1638	copied = copy_page_from_iter(alloc_frag->page,
1639				     alloc_frag->offset + pad,
1640				     len, from);
1641	if (copied != len)
1642		return ERR_PTR(-EFAULT);
1643
1644	/* There's a small window that XDP may be set after the check
1645	 * of xdp_prog above, this should be rare and for simplicity
1646	 * we do XDP on skb in case the headroom is not enough.
1647	 */
1648	if (hdr->gso_type || !xdp_prog)
1649		*skb_xdp = 1;
1650	else
1651		*skb_xdp = 0;
1652
1653	local_bh_disable();
1654	rcu_read_lock();
1655	xdp_prog = rcu_dereference(tun->xdp_prog);
1656	if (xdp_prog && !*skb_xdp) {
1657		struct xdp_buff xdp;
1658		void *orig_data;
1659		u32 act;
1660
1661		xdp.data_hard_start = buf;
1662		xdp.data = buf + pad;
1663		xdp_set_data_meta_invalid(&xdp);
1664		xdp.data_end = xdp.data + len;
1665		xdp.rxq = &tfile->xdp_rxq;
1666		orig_data = xdp.data;
1667		act = bpf_prog_run_xdp(xdp_prog, &xdp);
1668
1669		switch (act) {
1670		case XDP_REDIRECT:
1671			get_page(alloc_frag->page);
1672			alloc_frag->offset += buflen;
1673			err = xdp_do_redirect(tun->dev, &xdp, xdp_prog);
1674			xdp_do_flush_map();
1675			if (err)
1676				goto err_redirect;
1677			rcu_read_unlock();
1678			local_bh_enable();
1679			return NULL;
1680		case XDP_TX:
1681			get_page(alloc_frag->page);
1682			alloc_frag->offset += buflen;
1683			if (tun_xdp_xmit(tun->dev, &xdp))
1684				goto err_redirect;
1685			tun_xdp_flush(tun->dev);
1686			rcu_read_unlock();
1687			local_bh_enable();
1688			return NULL;
1689		case XDP_PASS:
1690			delta = orig_data - xdp.data;
1691			break;
1692		default:
1693			bpf_warn_invalid_xdp_action(act);
1694			/* fall through */
1695		case XDP_ABORTED:
1696			trace_xdp_exception(tun->dev, xdp_prog, act);
1697			/* fall through */
1698		case XDP_DROP:
1699			goto err_xdp;
1700		}
1701	}
1702
1703	skb = build_skb(buf, buflen);
1704	if (!skb) {
1705		rcu_read_unlock();
1706		local_bh_enable();
1707		return ERR_PTR(-ENOMEM);
1708	}
1709
1710	skb_reserve(skb, pad - delta);
1711	skb_put(skb, len + delta);
1712	get_page(alloc_frag->page);
1713	alloc_frag->offset += buflen;
1714
1715	rcu_read_unlock();
1716	local_bh_enable();
1717
1718	return skb;
1719
1720err_redirect:
1721	put_page(alloc_frag->page);
1722err_xdp:
1723	rcu_read_unlock();
1724	local_bh_enable();
1725	this_cpu_inc(tun->pcpu_stats->rx_dropped);
1726	return NULL;
1727}
1728
1729/* Get packet from user space buffer */
1730static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1731			    void *msg_control, struct iov_iter *from,
1732			    int noblock, bool more)
1733{
1734	struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) };
1735	struct sk_buff *skb;
1736	size_t total_len = iov_iter_count(from);
1737	size_t len = total_len, align = tun->align, linear;
1738	struct virtio_net_hdr gso = { 0 };
1739	struct tun_pcpu_stats *stats;
1740	int good_linear;
1741	int copylen;
1742	bool zerocopy = false;
1743	int err;
1744	u32 rxhash = 0;
1745	int skb_xdp = 1;
1746	bool frags = tun_napi_frags_enabled(tun);
1747
1748	if (!(tun->dev->flags & IFF_UP))
1749		return -EIO;
1750
1751	if (!(tun->flags & IFF_NO_PI)) {
1752		if (len < sizeof(pi))
1753			return -EINVAL;
1754		len -= sizeof(pi);
1755
1756		if (!copy_from_iter_full(&pi, sizeof(pi), from))
1757			return -EFAULT;
 
1758	}
1759
1760	if (tun->flags & IFF_VNET_HDR) {
1761		int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
1762
1763		if (len < vnet_hdr_sz)
1764			return -EINVAL;
1765		len -= vnet_hdr_sz;
1766
1767		if (!copy_from_iter_full(&gso, sizeof(gso), from))
1768			return -EFAULT;
1769
1770		if ((gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
1771		    tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso.hdr_len))
1772			gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2);
1773
1774		if (tun16_to_cpu(tun, gso.hdr_len) > len)
1775			return -EINVAL;
1776		iov_iter_advance(from, vnet_hdr_sz - sizeof(gso));
1777	}
1778
1779	if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) {
1780		align += NET_IP_ALIGN;
1781		if (unlikely(len < ETH_HLEN ||
1782			     (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN)))
1783			return -EINVAL;
1784	}
1785
1786	good_linear = SKB_MAX_HEAD(align);
1787
1788	if (msg_control) {
1789		struct iov_iter i = *from;
1790
1791		/* There are 256 bytes to be copied in skb, so there is
1792		 * enough room for skb expand head in case it is used.
1793		 * The rest of the buffer is mapped from userspace.
1794		 */
1795		copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN;
1796		if (copylen > good_linear)
1797			copylen = good_linear;
1798		linear = copylen;
1799		iov_iter_advance(&i, copylen);
1800		if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS)
1801			zerocopy = true;
1802	}
1803
1804	if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {
1805		/* For the packet that is not easy to be processed
1806		 * (e.g gso or jumbo packet), we will do it at after
1807		 * skb was created with generic XDP routine.
1808		 */
1809		skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp);
1810		if (IS_ERR(skb)) {
1811			this_cpu_inc(tun->pcpu_stats->rx_dropped);
1812			return PTR_ERR(skb);
1813		}
1814		if (!skb)
1815			return total_len;
1816	} else {
1817		if (!zerocopy) {
1818			copylen = len;
1819			if (tun16_to_cpu(tun, gso.hdr_len) > good_linear)
1820				linear = good_linear;
1821			else
1822				linear = tun16_to_cpu(tun, gso.hdr_len);
1823		}
1824
1825		if (frags) {
1826			mutex_lock(&tfile->napi_mutex);
1827			skb = tun_napi_alloc_frags(tfile, copylen, from);
1828			/* tun_napi_alloc_frags() enforces a layout for the skb.
1829			 * If zerocopy is enabled, then this layout will be
1830			 * overwritten by zerocopy_sg_from_iter().
1831			 */
1832			zerocopy = false;
1833		} else {
1834			skb = tun_alloc_skb(tfile, align, copylen, linear,
1835					    noblock);
1836		}
1837
1838		if (IS_ERR(skb)) {
1839			if (PTR_ERR(skb) != -EAGAIN)
1840				this_cpu_inc(tun->pcpu_stats->rx_dropped);
1841			if (frags)
1842				mutex_unlock(&tfile->napi_mutex);
1843			return PTR_ERR(skb);
1844		}
1845
1846		if (zerocopy)
1847			err = zerocopy_sg_from_iter(skb, from);
1848		else
1849			err = skb_copy_datagram_from_iter(skb, 0, from, len);
1850
1851		if (err) {
1852			this_cpu_inc(tun->pcpu_stats->rx_dropped);
1853			kfree_skb(skb);
1854			if (frags) {
1855				tfile->napi.skb = NULL;
1856				mutex_unlock(&tfile->napi_mutex);
1857			}
1858
1859			return -EFAULT;
1860		}
1861	}
1862
1863	if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) {
1864		this_cpu_inc(tun->pcpu_stats->rx_frame_errors);
1865		kfree_skb(skb);
1866		if (frags) {
1867			tfile->napi.skb = NULL;
1868			mutex_unlock(&tfile->napi_mutex);
1869		}
1870
1871		return -EINVAL;
 
 
 
 
 
 
1872	}
1873
1874	switch (tun->flags & TUN_TYPE_MASK) {
1875	case IFF_TUN:
1876		if (tun->flags & IFF_NO_PI) {
1877			u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0;
1878
1879			switch (ip_version) {
1880			case 4:
1881				pi.proto = htons(ETH_P_IP);
1882				break;
1883			case 6:
1884				pi.proto = htons(ETH_P_IPV6);
1885				break;
1886			default:
1887				this_cpu_inc(tun->pcpu_stats->rx_dropped);
1888				kfree_skb(skb);
1889				return -EINVAL;
1890			}
1891		}
1892
1893		skb_reset_mac_header(skb);
1894		skb->protocol = pi.proto;
1895		skb->dev = tun->dev;
1896		break;
1897	case IFF_TAP:
1898		if (!frags)
1899			skb->protocol = eth_type_trans(skb, tun->dev);
1900		break;
1901	}
1902
1903	/* copy skb_ubuf_info for callback when skb has no error */
1904	if (zerocopy) {
1905		skb_shinfo(skb)->destructor_arg = msg_control;
1906		skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
1907		skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
1908	} else if (msg_control) {
1909		struct ubuf_info *uarg = msg_control;
1910		uarg->callback(uarg, false);
1911	}
1912
1913	skb_reset_network_header(skb);
1914	skb_probe_transport_header(skb, 0);
1915
1916	if (skb_xdp) {
1917		struct bpf_prog *xdp_prog;
1918		int ret;
1919
1920		local_bh_disable();
1921		rcu_read_lock();
1922		xdp_prog = rcu_dereference(tun->xdp_prog);
1923		if (xdp_prog) {
1924			ret = do_xdp_generic(xdp_prog, skb);
1925			if (ret != XDP_PASS) {
1926				rcu_read_unlock();
1927				local_bh_enable();
1928				return total_len;
1929			}
1930		}
1931		rcu_read_unlock();
1932		local_bh_enable();
1933	}
1934
1935	rcu_read_lock();
1936	if (!rcu_dereference(tun->steering_prog))
1937		rxhash = __skb_get_hash_symmetric(skb);
1938	rcu_read_unlock();
1939
1940	if (frags) {
1941		/* Exercise flow dissector code path. */
1942		u32 headlen = eth_get_headlen(skb->data, skb_headlen(skb));
1943
1944		if (unlikely(headlen > skb_headlen(skb))) {
1945			this_cpu_inc(tun->pcpu_stats->rx_dropped);
1946			napi_free_frags(&tfile->napi);
1947			mutex_unlock(&tfile->napi_mutex);
1948			WARN_ON(1);
1949			return -ENOMEM;
1950		}
1951
1952		local_bh_disable();
1953		napi_gro_frags(&tfile->napi);
1954		local_bh_enable();
1955		mutex_unlock(&tfile->napi_mutex);
1956	} else if (tfile->napi_enabled) {
1957		struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1958		int queue_len;
1959
1960		spin_lock_bh(&queue->lock);
1961		__skb_queue_tail(queue, skb);
1962		queue_len = skb_queue_len(queue);
1963		spin_unlock(&queue->lock);
1964
1965		if (!more || queue_len > NAPI_POLL_WEIGHT)
1966			napi_schedule(&tfile->napi);
1967
1968		local_bh_enable();
1969	} else if (!IS_ENABLED(CONFIG_4KSTACKS)) {
1970		tun_rx_batched(tun, tfile, skb, more);
1971	} else {
1972		netif_rx_ni(skb);
1973	}
1974
1975	stats = get_cpu_ptr(tun->pcpu_stats);
1976	u64_stats_update_begin(&stats->syncp);
1977	stats->rx_packets++;
1978	stats->rx_bytes += len;
1979	u64_stats_update_end(&stats->syncp);
1980	put_cpu_ptr(stats);
1981
1982	if (rxhash)
1983		tun_flow_update(tun, rxhash, tfile);
1984
1985	return total_len;
 
 
 
1986}
1987
1988static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from)
 
1989{
1990	struct file *file = iocb->ki_filp;
1991	struct tun_file *tfile = file->private_data;
1992	struct tun_struct *tun = tun_get(tfile);
1993	ssize_t result;
1994
1995	if (!tun)
1996		return -EBADFD;
1997
1998	result = tun_get_user(tun, tfile, NULL, from,
1999			      file->f_flags & O_NONBLOCK, false);
 
 
2000
2001	tun_put(tun);
2002	return result;
2003}
2004
2005static ssize_t tun_put_user_xdp(struct tun_struct *tun,
2006				struct tun_file *tfile,
2007				struct xdp_buff *xdp,
2008				struct iov_iter *iter)
2009{
2010	int vnet_hdr_sz = 0;
2011	size_t size = xdp->data_end - xdp->data;
2012	struct tun_pcpu_stats *stats;
2013	size_t ret;
2014
2015	if (tun->flags & IFF_VNET_HDR) {
2016		struct virtio_net_hdr gso = { 0 };
2017
2018		vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
2019		if (unlikely(iov_iter_count(iter) < vnet_hdr_sz))
2020			return -EINVAL;
2021		if (unlikely(copy_to_iter(&gso, sizeof(gso), iter) !=
2022			     sizeof(gso)))
2023			return -EFAULT;
2024		iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
2025	}
2026
2027	ret = copy_to_iter(xdp->data, size, iter) + vnet_hdr_sz;
2028
2029	stats = get_cpu_ptr(tun->pcpu_stats);
2030	u64_stats_update_begin(&stats->syncp);
2031	stats->tx_packets++;
2032	stats->tx_bytes += ret;
2033	u64_stats_update_end(&stats->syncp);
2034	put_cpu_ptr(tun->pcpu_stats);
2035
2036	return ret;
2037}
2038
2039/* Put packet to the user space buffer */
2040static ssize_t tun_put_user(struct tun_struct *tun,
2041			    struct tun_file *tfile,
2042			    struct sk_buff *skb,
2043			    struct iov_iter *iter)
2044{
2045	struct tun_pi pi = { 0, skb->protocol };
2046	struct tun_pcpu_stats *stats;
2047	ssize_t total;
2048	int vlan_offset = 0;
2049	int vlan_hlen = 0;
2050	int vnet_hdr_sz = 0;
2051
2052	if (skb_vlan_tag_present(skb))
2053		vlan_hlen = VLAN_HLEN;
2054
2055	if (tun->flags & IFF_VNET_HDR)
2056		vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
2057
2058	total = skb->len + vlan_hlen + vnet_hdr_sz;
2059
2060	if (!(tun->flags & IFF_NO_PI)) {
2061		if (iov_iter_count(iter) < sizeof(pi))
2062			return -EINVAL;
2063
2064		total += sizeof(pi);
2065		if (iov_iter_count(iter) < total) {
2066			/* Packet will be striped */
2067			pi.flags |= TUN_PKT_STRIP;
2068		}
2069
2070		if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi))
2071			return -EFAULT;
 
2072	}
2073
2074	if (vnet_hdr_sz) {
2075		struct virtio_net_hdr gso;
2076
2077		if (iov_iter_count(iter) < vnet_hdr_sz)
2078			return -EINVAL;
2079
2080		if (virtio_net_hdr_from_skb(skb, &gso,
2081					    tun_is_little_endian(tun), true)) {
2082			struct skb_shared_info *sinfo = skb_shinfo(skb);
2083			pr_err("unexpected GSO type: "
2084			       "0x%x, gso_size %d, hdr_len %d\n",
2085			       sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),
2086			       tun16_to_cpu(tun, gso.hdr_len));
2087			print_hex_dump(KERN_ERR, "tun: ",
2088				       DUMP_PREFIX_NONE,
2089				       16, 1, skb->head,
2090				       min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);
2091			WARN_ON_ONCE(1);
2092			return -EINVAL;
2093		}
2094
2095		if (copy_to_iter(&gso, sizeof(gso), iter) != sizeof(gso))
2096			return -EFAULT;
2097
2098		iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
2099	}
2100
2101	if (vlan_hlen) {
2102		int ret;
2103		struct veth veth;
2104
2105		veth.h_vlan_proto = skb->vlan_proto;
2106		veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
2107
2108		vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2109
2110		ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset);
2111		if (ret || !iov_iter_count(iter))
2112			goto done;
 
 
 
 
2113
2114		ret = copy_to_iter(&veth, sizeof(veth), iter);
2115		if (ret != sizeof(veth) || !iov_iter_count(iter))
2116			goto done;
 
2117	}
2118
2119	skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);
2120
2121done:
2122	/* caller is in process context, */
2123	stats = get_cpu_ptr(tun->pcpu_stats);
2124	u64_stats_update_begin(&stats->syncp);
2125	stats->tx_packets++;
2126	stats->tx_bytes += skb->len + vlan_hlen;
2127	u64_stats_update_end(&stats->syncp);
2128	put_cpu_ptr(tun->pcpu_stats);
2129
2130	return total;
2131}
2132
2133static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
 
 
2134{
2135	DECLARE_WAITQUEUE(wait, current);
2136	void *ptr = NULL;
2137	int error = 0;
2138
2139	ptr = ptr_ring_consume(&tfile->tx_ring);
2140	if (ptr)
2141		goto out;
2142	if (noblock) {
2143		error = -EAGAIN;
2144		goto out;
2145	}
2146
2147	add_wait_queue(&tfile->wq.wait, &wait);
2148	current->state = TASK_INTERRUPTIBLE;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2149
2150	while (1) {
2151		ptr = ptr_ring_consume(&tfile->tx_ring);
2152		if (ptr)
2153			break;
2154		if (signal_pending(current)) {
2155			error = -ERESTARTSYS;
2156			break;
2157		}
2158		if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) {
2159			error = -EFAULT;
2160			break;
2161		}
 
2162
2163		schedule();
 
 
2164	}
2165
2166	current->state = TASK_RUNNING;
2167	remove_wait_queue(&tfile->wq.wait, &wait);
2168
2169out:
2170	*err = error;
2171	return ptr;
2172}
2173
2174static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
2175			   struct iov_iter *to,
2176			   int noblock, void *ptr)
2177{
2178	ssize_t ret;
2179	int err;
2180
2181	tun_debug(KERN_INFO, tun, "tun_do_read\n");
2182
2183	if (!iov_iter_count(to)) {
2184		tun_ptr_free(ptr);
2185		return 0;
2186	}
2187
2188	if (!ptr) {
2189		/* Read frames from ring */
2190		ptr = tun_ring_recv(tfile, noblock, &err);
2191		if (!ptr)
2192			return err;
2193	}
2194
2195	if (tun_is_xdp_buff(ptr)) {
2196		struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
2197
2198		ret = tun_put_user_xdp(tun, tfile, xdp, to);
2199		put_page(virt_to_head_page(xdp->data));
2200	} else {
2201		struct sk_buff *skb = ptr;
2202
2203		ret = tun_put_user(tun, tfile, skb, to);
2204		if (unlikely(ret < 0))
2205			kfree_skb(skb);
2206		else
2207			consume_skb(skb);
2208	}
2209
2210	return ret;
2211}
2212
2213static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
2214{
2215	struct file *file = iocb->ki_filp;
2216	struct tun_file *tfile = file->private_data;
2217	struct tun_struct *tun = tun_get(tfile);
2218	ssize_t len = iov_iter_count(to), ret;
2219
2220	if (!tun)
2221		return -EBADFD;
2222	ret = tun_do_read(tun, tfile, to, file->f_flags & O_NONBLOCK, NULL);
 
 
 
 
 
 
2223	ret = min_t(ssize_t, ret, len);
2224	if (ret > 0)
2225		iocb->ki_pos = ret;
2226	tun_put(tun);
2227	return ret;
2228}
2229
2230static void tun_prog_free(struct rcu_head *rcu)
2231{
2232	struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu);
2233
2234	bpf_prog_destroy(prog->prog);
2235	kfree(prog);
2236}
2237
2238static int __tun_set_ebpf(struct tun_struct *tun,
2239			  struct tun_prog __rcu **prog_p,
2240			  struct bpf_prog *prog)
2241{
2242	struct tun_prog *old, *new = NULL;
2243
2244	if (prog) {
2245		new = kmalloc(sizeof(*new), GFP_KERNEL);
2246		if (!new)
2247			return -ENOMEM;
2248		new->prog = prog;
2249	}
2250
2251	spin_lock_bh(&tun->lock);
2252	old = rcu_dereference_protected(*prog_p,
2253					lockdep_is_held(&tun->lock));
2254	rcu_assign_pointer(*prog_p, new);
2255	spin_unlock_bh(&tun->lock);
2256
2257	if (old)
2258		call_rcu(&old->rcu, tun_prog_free);
2259
2260	return 0;
2261}
2262
2263static void tun_free_netdev(struct net_device *dev)
2264{
2265	struct tun_struct *tun = netdev_priv(dev);
2266
2267	BUG_ON(!(list_empty(&tun->disabled)));
2268	free_percpu(tun->pcpu_stats);
2269	tun_flow_uninit(tun);
2270	security_tun_dev_free_security(tun->security);
2271	__tun_set_ebpf(tun, &tun->steering_prog, NULL);
2272	__tun_set_ebpf(tun, &tun->filter_prog, NULL);
2273}
2274
2275static void tun_setup(struct net_device *dev)
2276{
2277	struct tun_struct *tun = netdev_priv(dev);
2278
2279	tun->owner = INVALID_UID;
2280	tun->group = INVALID_GID;
2281
2282	dev->ethtool_ops = &tun_ethtool_ops;
2283	dev->needs_free_netdev = true;
2284	dev->priv_destructor = tun_free_netdev;
2285	/* We prefer our own queue length */
2286	dev->tx_queue_len = TUN_READQ_SIZE;
2287}
2288
2289/* Trivial set of netlink ops to allow deleting tun or tap
2290 * device with netlink.
2291 */
2292static int tun_validate(struct nlattr *tb[], struct nlattr *data[],
2293			struct netlink_ext_ack *extack)
2294{
2295	return -EINVAL;
2296}
2297
2298static size_t tun_get_size(const struct net_device *dev)
2299{
2300	BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t));
2301	BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t));
2302
2303	return nla_total_size(sizeof(uid_t)) + /* OWNER */
2304	       nla_total_size(sizeof(gid_t)) + /* GROUP */
2305	       nla_total_size(sizeof(u8)) + /* TYPE */
2306	       nla_total_size(sizeof(u8)) + /* PI */
2307	       nla_total_size(sizeof(u8)) + /* VNET_HDR */
2308	       nla_total_size(sizeof(u8)) + /* PERSIST */
2309	       nla_total_size(sizeof(u8)) + /* MULTI_QUEUE */
2310	       nla_total_size(sizeof(u32)) + /* NUM_QUEUES */
2311	       nla_total_size(sizeof(u32)) + /* NUM_DISABLED_QUEUES */
2312	       0;
2313}
2314
2315static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev)
2316{
2317	struct tun_struct *tun = netdev_priv(dev);
2318
2319	if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK))
2320		goto nla_put_failure;
2321	if (uid_valid(tun->owner) &&
2322	    nla_put_u32(skb, IFLA_TUN_OWNER,
2323			from_kuid_munged(current_user_ns(), tun->owner)))
2324		goto nla_put_failure;
2325	if (gid_valid(tun->group) &&
2326	    nla_put_u32(skb, IFLA_TUN_GROUP,
2327			from_kgid_munged(current_user_ns(), tun->group)))
2328		goto nla_put_failure;
2329	if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI)))
2330		goto nla_put_failure;
2331	if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR)))
2332		goto nla_put_failure;
2333	if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST)))
2334		goto nla_put_failure;
2335	if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE,
2336		       !!(tun->flags & IFF_MULTI_QUEUE)))
2337		goto nla_put_failure;
2338	if (tun->flags & IFF_MULTI_QUEUE) {
2339		if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues))
2340			goto nla_put_failure;
2341		if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES,
2342				tun->numdisabled))
2343			goto nla_put_failure;
2344	}
2345
2346	return 0;
2347
2348nla_put_failure:
2349	return -EMSGSIZE;
2350}
2351
2352static struct rtnl_link_ops tun_link_ops __read_mostly = {
2353	.kind		= DRV_NAME,
2354	.priv_size	= sizeof(struct tun_struct),
2355	.setup		= tun_setup,
2356	.validate	= tun_validate,
2357	.get_size       = tun_get_size,
2358	.fill_info      = tun_fill_info,
2359};
2360
2361static void tun_sock_write_space(struct sock *sk)
2362{
2363	struct tun_file *tfile;
2364	wait_queue_head_t *wqueue;
2365
2366	if (!sock_writeable(sk))
2367		return;
2368
2369	if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags))
2370		return;
2371
2372	wqueue = sk_sleep(sk);
2373	if (wqueue && waitqueue_active(wqueue))
2374		wake_up_interruptible_sync_poll(wqueue, EPOLLOUT |
2375						EPOLLWRNORM | EPOLLWRBAND);
2376
2377	tfile = container_of(sk, struct tun_file, sk);
2378	kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
2379}
2380
2381static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
2382{
2383	int ret;
2384	struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2385	struct tun_struct *tun = tun_get(tfile);
2386
2387	if (!tun)
2388		return -EBADFD;
2389
2390	ret = tun_get_user(tun, tfile, m->msg_control, &m->msg_iter,
2391			   m->msg_flags & MSG_DONTWAIT,
2392			   m->msg_flags & MSG_MORE);
2393	tun_put(tun);
2394	return ret;
 
2395}
2396
2397static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
 
2398		       int flags)
2399{
2400	struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2401	struct tun_struct *tun = tun_get(tfile);
2402	void *ptr = m->msg_control;
2403	int ret;
2404
2405	if (!tun) {
2406		ret = -EBADFD;
2407		goto out_free;
2408	}
2409
2410	if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) {
2411		ret = -EINVAL;
2412		goto out_put_tun;
2413	}
2414	if (flags & MSG_ERRQUEUE) {
2415		ret = sock_recv_errqueue(sock->sk, m, total_len,
2416					 SOL_PACKET, TUN_TX_TIMESTAMP);
2417		goto out;
2418	}
2419	ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);
2420	if (ret > (ssize_t)total_len) {
2421		m->msg_flags |= MSG_TRUNC;
2422		ret = flags & MSG_TRUNC ? ret : total_len;
2423	}
2424out:
2425	tun_put(tun);
2426	return ret;
2427
2428out_put_tun:
2429	tun_put(tun);
2430out_free:
2431	tun_ptr_free(ptr);
2432	return ret;
2433}
2434
2435static int tun_ptr_peek_len(void *ptr)
2436{
2437	if (likely(ptr)) {
2438		if (tun_is_xdp_buff(ptr)) {
2439			struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
2440
2441			return xdp->data_end - xdp->data;
2442		}
2443		return __skb_array_len_with_tag(ptr);
2444	} else {
2445		return 0;
2446	}
2447}
2448
2449static int tun_peek_len(struct socket *sock)
2450{
2451	struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2452	struct tun_struct *tun;
2453	int ret = 0;
2454
2455	tun = tun_get(tfile);
2456	if (!tun)
2457		return 0;
2458
2459	ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len);
2460	tun_put(tun);
2461
2462	return ret;
2463}
2464
2465/* Ops structure to mimic raw sockets with tun */
2466static const struct proto_ops tun_socket_ops = {
2467	.peek_len = tun_peek_len,
2468	.sendmsg = tun_sendmsg,
2469	.recvmsg = tun_recvmsg,
2470};
2471
2472static struct proto tun_proto = {
2473	.name		= "tun",
2474	.owner		= THIS_MODULE,
2475	.obj_size	= sizeof(struct tun_file),
2476};
2477
2478static int tun_flags(struct tun_struct *tun)
2479{
2480	return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2481}
2482
2483static ssize_t tun_show_flags(struct device *dev, struct device_attribute *attr,
2484			      char *buf)
2485{
2486	struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2487	return sprintf(buf, "0x%x\n", tun_flags(tun));
2488}
2489
2490static ssize_t tun_show_owner(struct device *dev, struct device_attribute *attr,
2491			      char *buf)
2492{
2493	struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2494	return uid_valid(tun->owner)?
2495		sprintf(buf, "%u\n",
2496			from_kuid_munged(current_user_ns(), tun->owner)):
2497		sprintf(buf, "-1\n");
2498}
2499
2500static ssize_t tun_show_group(struct device *dev, struct device_attribute *attr,
2501			      char *buf)
2502{
2503	struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2504	return gid_valid(tun->group) ?
2505		sprintf(buf, "%u\n",
2506			from_kgid_munged(current_user_ns(), tun->group)):
2507		sprintf(buf, "-1\n");
2508}
2509
2510static DEVICE_ATTR(tun_flags, 0444, tun_show_flags, NULL);
2511static DEVICE_ATTR(owner, 0444, tun_show_owner, NULL);
2512static DEVICE_ATTR(group, 0444, tun_show_group, NULL);
2513
2514static struct attribute *tun_dev_attrs[] = {
2515	&dev_attr_tun_flags.attr,
2516	&dev_attr_owner.attr,
2517	&dev_attr_group.attr,
2518	NULL
2519};
2520
2521static const struct attribute_group tun_attr_group = {
2522	.attrs = tun_dev_attrs
2523};
2524
2525static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
2526{
 
2527	struct tun_struct *tun;
2528	struct tun_file *tfile = file->private_data;
2529	struct net_device *dev;
2530	int err;
2531
2532	if (tfile->detached)
2533		return -EINVAL;
2534
2535	if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) {
2536		if (!capable(CAP_NET_ADMIN))
2537			return -EPERM;
2538
2539		if (!(ifr->ifr_flags & IFF_NAPI) ||
2540		    (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP)
2541			return -EINVAL;
2542	}
2543
2544	dev = __dev_get_by_name(net, ifr->ifr_name);
2545	if (dev) {
 
 
2546		if (ifr->ifr_flags & IFF_TUN_EXCL)
2547			return -EBUSY;
2548		if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops)
2549			tun = netdev_priv(dev);
2550		else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops)
2551			tun = netdev_priv(dev);
2552		else
2553			return -EINVAL;
2554
2555		if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) !=
2556		    !!(tun->flags & IFF_MULTI_QUEUE))
2557			return -EINVAL;
2558
2559		if (tun_not_capable(tun))
2560			return -EPERM;
2561		err = security_tun_dev_open(tun->security);
2562		if (err < 0)
2563			return err;
2564
2565		err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER,
2566				 ifr->ifr_flags & IFF_NAPI);
2567		if (err < 0)
2568			return err;
2569
2570		if (tun->flags & IFF_MULTI_QUEUE &&
2571		    (tun->numqueues + tun->numdisabled > 1)) {
2572			/* One or more queue has already been attached, no need
2573			 * to initialize the device again.
2574			 */
2575			netdev_state_change(dev);
2576			return 0;
2577		}
2578
2579		tun->flags = (tun->flags & ~TUN_FEATURES) |
2580			      (ifr->ifr_flags & TUN_FEATURES);
2581
2582		netdev_state_change(dev);
2583	} else {
2584		char *name;
2585		unsigned long flags = 0;
2586		int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ?
2587			     MAX_TAP_QUEUES : 1;
2588
2589		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
2590			return -EPERM;
2591		err = security_tun_dev_create();
2592		if (err < 0)
2593			return err;
2594
2595		/* Set dev type */
2596		if (ifr->ifr_flags & IFF_TUN) {
2597			/* TUN device */
2598			flags |= IFF_TUN;
2599			name = "tun%d";
2600		} else if (ifr->ifr_flags & IFF_TAP) {
2601			/* TAP device */
2602			flags |= IFF_TAP;
2603			name = "tap%d";
2604		} else
2605			return -EINVAL;
2606
2607		if (*ifr->ifr_name)
2608			name = ifr->ifr_name;
2609
2610		dev = alloc_netdev_mqs(sizeof(struct tun_struct), name,
2611				       NET_NAME_UNKNOWN, tun_setup, queues,
2612				       queues);
2613
2614		if (!dev)
2615			return -ENOMEM;
2616		err = dev_get_valid_name(net, dev, name);
2617		if (err < 0)
2618			goto err_free_dev;
2619
2620		dev_net_set(dev, net);
2621		dev->rtnl_link_ops = &tun_link_ops;
2622		dev->ifindex = tfile->ifindex;
2623		dev->sysfs_groups[0] = &tun_attr_group;
2624
2625		tun = netdev_priv(dev);
2626		tun->dev = dev;
2627		tun->flags = flags;
2628		tun->txflt.count = 0;
2629		tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
2630
2631		tun->align = NET_SKB_PAD;
2632		tun->filter_attached = false;
2633		tun->sndbuf = tfile->socket.sk->sk_sndbuf;
2634		tun->rx_batched = 0;
2635		RCU_INIT_POINTER(tun->steering_prog, NULL);
2636
2637		tun->pcpu_stats = netdev_alloc_pcpu_stats(struct tun_pcpu_stats);
2638		if (!tun->pcpu_stats) {
2639			err = -ENOMEM;
2640			goto err_free_dev;
2641		}
2642
2643		spin_lock_init(&tun->lock);
 
 
 
 
 
2644
2645		err = security_tun_dev_alloc_security(&tun->security);
2646		if (err < 0)
2647			goto err_free_stat;
2648
2649		tun_net_init(dev);
2650		tun_flow_init(tun);
2651
2652		dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST |
2653				   TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
2654				   NETIF_F_HW_VLAN_STAG_TX;
2655		dev->features = dev->hw_features | NETIF_F_LLTX;
2656		dev->vlan_features = dev->features &
2657				     ~(NETIF_F_HW_VLAN_CTAG_TX |
2658				       NETIF_F_HW_VLAN_STAG_TX);
2659
2660		tun->flags = (tun->flags & ~TUN_FEATURES) |
2661			      (ifr->ifr_flags & TUN_FEATURES);
2662
2663		INIT_LIST_HEAD(&tun->disabled);
2664		err = tun_attach(tun, file, false, ifr->ifr_flags & IFF_NAPI);
2665		if (err < 0)
2666			goto err_free_flow;
2667
2668		err = register_netdevice(tun->dev);
 
 
 
 
 
 
 
2669		if (err < 0)
2670			goto err_detach;
2671	}
2672
2673	netif_carrier_on(tun->dev);
2674
2675	tun_debug(KERN_INFO, tun, "tun_set_iff\n");
2676
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2677	/* Make sure persistent devices do not get stuck in
2678	 * xoff state.
2679	 */
2680	if (netif_running(tun->dev))
2681		netif_tx_wake_all_queues(tun->dev);
2682
2683	strcpy(ifr->ifr_name, tun->dev->name);
2684	return 0;
2685
2686err_detach:
2687	tun_detach_all(dev);
2688	/* register_netdevice() already called tun_free_netdev() */
2689	goto err_free_dev;
2690
2691err_free_flow:
2692	tun_flow_uninit(tun);
2693	security_tun_dev_free_security(tun->security);
2694err_free_stat:
2695	free_percpu(tun->pcpu_stats);
2696err_free_dev:
2697	free_netdev(dev);
 
2698	return err;
2699}
2700
2701static void tun_get_iff(struct net *net, struct tun_struct *tun,
2702		       struct ifreq *ifr)
2703{
2704	tun_debug(KERN_INFO, tun, "tun_get_iff\n");
2705
2706	strcpy(ifr->ifr_name, tun->dev->name);
2707
2708	ifr->ifr_flags = tun_flags(tun);
2709
 
2710}
2711
2712/* This is like a cut-down ethtool ops, except done via tun fd so no
2713 * privs required. */
2714static int set_offload(struct tun_struct *tun, unsigned long arg)
2715{
2716	netdev_features_t features = 0;
2717
2718	if (arg & TUN_F_CSUM) {
2719		features |= NETIF_F_HW_CSUM;
2720		arg &= ~TUN_F_CSUM;
2721
2722		if (arg & (TUN_F_TSO4|TUN_F_TSO6)) {
2723			if (arg & TUN_F_TSO_ECN) {
2724				features |= NETIF_F_TSO_ECN;
2725				arg &= ~TUN_F_TSO_ECN;
2726			}
2727			if (arg & TUN_F_TSO4)
2728				features |= NETIF_F_TSO;
2729			if (arg & TUN_F_TSO6)
2730				features |= NETIF_F_TSO6;
2731			arg &= ~(TUN_F_TSO4|TUN_F_TSO6);
2732		}
2733
2734		arg &= ~TUN_F_UFO;
 
 
 
2735	}
2736
2737	/* This gives the user a way to test for new features in future by
2738	 * trying to set them. */
2739	if (arg)
2740		return -EINVAL;
2741
2742	tun->set_features = features;
2743	tun->dev->wanted_features &= ~TUN_USER_FEATURES;
2744	tun->dev->wanted_features |= features;
2745	netdev_update_features(tun->dev);
2746
2747	return 0;
2748}
2749
2750static void tun_detach_filter(struct tun_struct *tun, int n)
2751{
2752	int i;
2753	struct tun_file *tfile;
2754
2755	for (i = 0; i < n; i++) {
2756		tfile = rtnl_dereference(tun->tfiles[i]);
2757		lock_sock(tfile->socket.sk);
2758		sk_detach_filter(tfile->socket.sk);
2759		release_sock(tfile->socket.sk);
2760	}
2761
2762	tun->filter_attached = false;
2763}
2764
2765static int tun_attach_filter(struct tun_struct *tun)
2766{
2767	int i, ret = 0;
2768	struct tun_file *tfile;
2769
2770	for (i = 0; i < tun->numqueues; i++) {
2771		tfile = rtnl_dereference(tun->tfiles[i]);
2772		lock_sock(tfile->socket.sk);
2773		ret = sk_attach_filter(&tun->fprog, tfile->socket.sk);
2774		release_sock(tfile->socket.sk);
2775		if (ret) {
2776			tun_detach_filter(tun, i);
2777			return ret;
2778		}
2779	}
2780
2781	tun->filter_attached = true;
2782	return ret;
2783}
2784
2785static void tun_set_sndbuf(struct tun_struct *tun)
2786{
2787	struct tun_file *tfile;
2788	int i;
2789
2790	for (i = 0; i < tun->numqueues; i++) {
2791		tfile = rtnl_dereference(tun->tfiles[i]);
2792		tfile->socket.sk->sk_sndbuf = tun->sndbuf;
2793	}
2794}
2795
2796static int tun_set_queue(struct file *file, struct ifreq *ifr)
2797{
2798	struct tun_file *tfile = file->private_data;
2799	struct tun_struct *tun;
2800	int ret = 0;
2801
2802	rtnl_lock();
2803
2804	if (ifr->ifr_flags & IFF_ATTACH_QUEUE) {
2805		tun = tfile->detached;
2806		if (!tun) {
2807			ret = -EINVAL;
2808			goto unlock;
2809		}
2810		ret = security_tun_dev_attach_queue(tun->security);
2811		if (ret < 0)
2812			goto unlock;
2813		ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI);
2814	} else if (ifr->ifr_flags & IFF_DETACH_QUEUE) {
2815		tun = rtnl_dereference(tfile->tun);
2816		if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)
2817			ret = -EINVAL;
2818		else
2819			__tun_detach(tfile, false);
2820	} else
2821		ret = -EINVAL;
2822
2823	if (ret >= 0)
2824		netdev_state_change(tun->dev);
2825
2826unlock:
2827	rtnl_unlock();
2828	return ret;
2829}
2830
2831static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog **prog_p,
2832			void __user *data)
2833{
2834	struct bpf_prog *prog;
2835	int fd;
2836
2837	if (copy_from_user(&fd, data, sizeof(fd)))
2838		return -EFAULT;
2839
2840	if (fd == -1) {
2841		prog = NULL;
2842	} else {
2843		prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
2844		if (IS_ERR(prog))
2845			return PTR_ERR(prog);
2846	}
2847
2848	return __tun_set_ebpf(tun, prog_p, prog);
2849}
2850
2851static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
2852			    unsigned long arg, int ifreq_len)
2853{
2854	struct tun_file *tfile = file->private_data;
2855	struct tun_struct *tun;
2856	void __user* argp = (void __user*)arg;
 
2857	struct ifreq ifr;
2858	struct net *net;
2859	kuid_t owner;
2860	kgid_t group;
2861	int sndbuf;
2862	int vnet_hdr_sz;
2863	unsigned int ifindex;
2864	int le;
2865	int ret;
2866	bool do_notify = false;
2867
2868	if (cmd == TUNSETIFF || cmd == TUNSETQUEUE ||
2869	    (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) {
2870		if (copy_from_user(&ifr, argp, ifreq_len))
2871			return -EFAULT;
2872	} else {
2873		memset(&ifr, 0, sizeof(ifr));
2874	}
2875	if (cmd == TUNGETFEATURES) {
2876		/* Currently this just means: "what IFF flags are valid?".
2877		 * This is needed because we never checked for invalid flags on
2878		 * TUNSETIFF.
2879		 */
2880		return put_user(IFF_TUN | IFF_TAP | TUN_FEATURES,
2881				(unsigned int __user*)argp);
2882	} else if (cmd == TUNSETQUEUE)
2883		return tun_set_queue(file, &ifr);
2884
2885	ret = 0;
2886	rtnl_lock();
2887
2888	tun = tun_get(tfile);
2889	net = sock_net(&tfile->sk);
2890	if (cmd == TUNSETIFF) {
2891		ret = -EEXIST;
2892		if (tun)
2893			goto unlock;
2894
2895		ifr.ifr_name[IFNAMSIZ-1] = '\0';
2896
2897		ret = tun_set_iff(net, file, &ifr);
2898
2899		if (ret)
2900			goto unlock;
2901
2902		if (copy_to_user(argp, &ifr, ifreq_len))
2903			ret = -EFAULT;
2904		goto unlock;
2905	}
2906	if (cmd == TUNSETIFINDEX) {
2907		ret = -EPERM;
2908		if (tun)
2909			goto unlock;
2910
2911		ret = -EFAULT;
2912		if (copy_from_user(&ifindex, argp, sizeof(ifindex)))
2913			goto unlock;
2914
2915		ret = 0;
2916		tfile->ifindex = ifindex;
2917		goto unlock;
2918	}
2919	if (cmd == SIOCGSKNS) {
2920		ret = -EPERM;
2921		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
2922			goto unlock;
2923
2924		ret = open_related_ns(&net->ns, get_net_ns);
2925		goto unlock;
2926	}
2927
2928	ret = -EBADFD;
2929	if (!tun)
2930		goto unlock;
2931
2932	tun_debug(KERN_INFO, tun, "tun_chr_ioctl cmd %u\n", cmd);
2933
2934	ret = 0;
2935	switch (cmd) {
2936	case TUNGETIFF:
2937		tun_get_iff(current->nsproxy->net_ns, tun, &ifr);
2938
2939		if (tfile->detached)
2940			ifr.ifr_flags |= IFF_DETACH_QUEUE;
2941		if (!tfile->socket.sk->sk_filter)
2942			ifr.ifr_flags |= IFF_NOFILTER;
2943
2944		if (copy_to_user(argp, &ifr, ifreq_len))
2945			ret = -EFAULT;
2946		break;
2947
2948	case TUNSETNOCSUM:
2949		/* Disable/Enable checksum */
2950
2951		/* [unimplemented] */
2952		tun_debug(KERN_INFO, tun, "ignored: set checksum %s\n",
2953			  arg ? "disabled" : "enabled");
2954		break;
2955
2956	case TUNSETPERSIST:
2957		/* Disable/Enable persist mode. Keep an extra reference to the
2958		 * module to prevent the module being unprobed.
2959		 */
2960		if (arg && !(tun->flags & IFF_PERSIST)) {
2961			tun->flags |= IFF_PERSIST;
2962			__module_get(THIS_MODULE);
2963			do_notify = true;
2964		}
2965		if (!arg && (tun->flags & IFF_PERSIST)) {
2966			tun->flags &= ~IFF_PERSIST;
2967			module_put(THIS_MODULE);
2968			do_notify = true;
2969		}
2970
2971		tun_debug(KERN_INFO, tun, "persist %s\n",
2972			  arg ? "enabled" : "disabled");
2973		break;
2974
2975	case TUNSETOWNER:
2976		/* Set owner of the device */
2977		owner = make_kuid(current_user_ns(), arg);
2978		if (!uid_valid(owner)) {
2979			ret = -EINVAL;
2980			break;
2981		}
2982		tun->owner = owner;
2983		do_notify = true;
2984		tun_debug(KERN_INFO, tun, "owner set to %u\n",
2985			  from_kuid(&init_user_ns, tun->owner));
2986		break;
2987
2988	case TUNSETGROUP:
2989		/* Set group of the device */
2990		group = make_kgid(current_user_ns(), arg);
2991		if (!gid_valid(group)) {
2992			ret = -EINVAL;
2993			break;
2994		}
2995		tun->group = group;
2996		do_notify = true;
2997		tun_debug(KERN_INFO, tun, "group set to %u\n",
2998			  from_kgid(&init_user_ns, tun->group));
2999		break;
3000
3001	case TUNSETLINK:
3002		/* Only allow setting the type when the interface is down */
3003		if (tun->dev->flags & IFF_UP) {
3004			tun_debug(KERN_INFO, tun,
3005				  "Linktype set failed because interface is up\n");
3006			ret = -EBUSY;
3007		} else {
3008			tun->dev->type = (int) arg;
3009			tun_debug(KERN_INFO, tun, "linktype set to %d\n",
3010				  tun->dev->type);
3011			ret = 0;
3012		}
3013		break;
3014
3015#ifdef TUN_DEBUG
3016	case TUNSETDEBUG:
3017		tun->debug = arg;
3018		break;
3019#endif
3020	case TUNSETOFFLOAD:
3021		ret = set_offload(tun, arg);
3022		break;
3023
3024	case TUNSETTXFILTER:
3025		/* Can be set only for TAPs */
3026		ret = -EINVAL;
3027		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3028			break;
3029		ret = update_filter(&tun->txflt, (void __user *)arg);
3030		break;
3031
3032	case SIOCGIFHWADDR:
3033		/* Get hw address */
3034		memcpy(ifr.ifr_hwaddr.sa_data, tun->dev->dev_addr, ETH_ALEN);
3035		ifr.ifr_hwaddr.sa_family = tun->dev->type;
3036		if (copy_to_user(argp, &ifr, ifreq_len))
3037			ret = -EFAULT;
3038		break;
3039
3040	case SIOCSIFHWADDR:
3041		/* Set hw address */
3042		tun_debug(KERN_DEBUG, tun, "set hw address: %pM\n",
3043			  ifr.ifr_hwaddr.sa_data);
3044
3045		ret = dev_set_mac_address(tun->dev, &ifr.ifr_hwaddr);
3046		break;
3047
3048	case TUNGETSNDBUF:
3049		sndbuf = tfile->socket.sk->sk_sndbuf;
3050		if (copy_to_user(argp, &sndbuf, sizeof(sndbuf)))
3051			ret = -EFAULT;
3052		break;
3053
3054	case TUNSETSNDBUF:
3055		if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) {
3056			ret = -EFAULT;
3057			break;
3058		}
3059		if (sndbuf <= 0) {
3060			ret = -EINVAL;
3061			break;
3062		}
3063
3064		tun->sndbuf = sndbuf;
3065		tun_set_sndbuf(tun);
3066		break;
3067
3068	case TUNGETVNETHDRSZ:
3069		vnet_hdr_sz = tun->vnet_hdr_sz;
3070		if (copy_to_user(argp, &vnet_hdr_sz, sizeof(vnet_hdr_sz)))
3071			ret = -EFAULT;
3072		break;
3073
3074	case TUNSETVNETHDRSZ:
3075		if (copy_from_user(&vnet_hdr_sz, argp, sizeof(vnet_hdr_sz))) {
3076			ret = -EFAULT;
3077			break;
3078		}
3079		if (vnet_hdr_sz < (int)sizeof(struct virtio_net_hdr)) {
3080			ret = -EINVAL;
3081			break;
3082		}
3083
3084		tun->vnet_hdr_sz = vnet_hdr_sz;
3085		break;
3086
3087	case TUNGETVNETLE:
3088		le = !!(tun->flags & TUN_VNET_LE);
3089		if (put_user(le, (int __user *)argp))
3090			ret = -EFAULT;
3091		break;
3092
3093	case TUNSETVNETLE:
3094		if (get_user(le, (int __user *)argp)) {
3095			ret = -EFAULT;
3096			break;
3097		}
3098		if (le)
3099			tun->flags |= TUN_VNET_LE;
3100		else
3101			tun->flags &= ~TUN_VNET_LE;
3102		break;
3103
3104	case TUNGETVNETBE:
3105		ret = tun_get_vnet_be(tun, argp);
3106		break;
3107
3108	case TUNSETVNETBE:
3109		ret = tun_set_vnet_be(tun, argp);
3110		break;
3111
3112	case TUNATTACHFILTER:
3113		/* Can be set only for TAPs */
3114		ret = -EINVAL;
3115		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3116			break;
3117		ret = -EFAULT;
3118		if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog)))
3119			break;
3120
3121		ret = tun_attach_filter(tun);
3122		break;
3123
3124	case TUNDETACHFILTER:
3125		/* Can be set only for TAPs */
3126		ret = -EINVAL;
3127		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3128			break;
3129		ret = 0;
3130		tun_detach_filter(tun, tun->numqueues);
3131		break;
3132
3133	case TUNGETFILTER:
3134		ret = -EINVAL;
3135		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3136			break;
3137		ret = -EFAULT;
3138		if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog)))
3139			break;
3140		ret = 0;
3141		break;
3142
3143	case TUNSETSTEERINGEBPF:
3144		ret = tun_set_ebpf(tun, &tun->steering_prog, argp);
3145		break;
3146
3147	case TUNSETFILTEREBPF:
3148		ret = tun_set_ebpf(tun, &tun->filter_prog, argp);
3149		break;
3150
3151	default:
3152		ret = -EINVAL;
3153		break;
3154	}
3155
3156	if (do_notify)
3157		netdev_state_change(tun->dev);
3158
3159unlock:
3160	rtnl_unlock();
3161	if (tun)
3162		tun_put(tun);
3163	return ret;
3164}
3165
3166static long tun_chr_ioctl(struct file *file,
3167			  unsigned int cmd, unsigned long arg)
3168{
3169	return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq));
3170}
3171
3172#ifdef CONFIG_COMPAT
3173static long tun_chr_compat_ioctl(struct file *file,
3174			 unsigned int cmd, unsigned long arg)
3175{
3176	switch (cmd) {
3177	case TUNSETIFF:
3178	case TUNGETIFF:
3179	case TUNSETTXFILTER:
3180	case TUNGETSNDBUF:
3181	case TUNSETSNDBUF:
3182	case SIOCGIFHWADDR:
3183	case SIOCSIFHWADDR:
3184		arg = (unsigned long)compat_ptr(arg);
3185		break;
3186	default:
3187		arg = (compat_ulong_t)arg;
3188		break;
3189	}
3190
3191	/*
3192	 * compat_ifreq is shorter than ifreq, so we must not access beyond
3193	 * the end of that structure. All fields that are used in this
3194	 * driver are compatible though, we don't need to convert the
3195	 * contents.
3196	 */
3197	return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq));
3198}
3199#endif /* CONFIG_COMPAT */
3200
3201static int tun_chr_fasync(int fd, struct file *file, int on)
3202{
3203	struct tun_file *tfile = file->private_data;
3204	int ret;
3205
3206	if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0)
 
 
 
 
 
3207		goto out;
3208
3209	if (on) {
3210		__f_setown(file, task_pid(current), PIDTYPE_PID, 0);
3211		tfile->flags |= TUN_FASYNC;
 
 
3212	} else
3213		tfile->flags &= ~TUN_FASYNC;
3214	ret = 0;
3215out:
 
3216	return ret;
3217}
3218
3219static int tun_chr_open(struct inode *inode, struct file * file)
3220{
3221	struct net *net = current->nsproxy->net_ns;
3222	struct tun_file *tfile;
3223
3224	DBG1(KERN_INFO, "tunX: tun_chr_open\n");
3225
3226	tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
3227					    &tun_proto, 0);
3228	if (!tfile)
3229		return -ENOMEM;
3230	if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) {
3231		sk_free(&tfile->sk);
3232		return -ENOMEM;
3233	}
3234
3235	RCU_INIT_POINTER(tfile->tun, NULL);
3236	tfile->flags = 0;
3237	tfile->ifindex = 0;
3238
3239	init_waitqueue_head(&tfile->wq.wait);
3240	RCU_INIT_POINTER(tfile->socket.wq, &tfile->wq);
3241
3242	tfile->socket.file = file;
3243	tfile->socket.ops = &tun_socket_ops;
3244
3245	sock_init_data(&tfile->socket, &tfile->sk);
3246
3247	tfile->sk.sk_write_space = tun_sock_write_space;
3248	tfile->sk.sk_sndbuf = INT_MAX;
3249
3250	file->private_data = tfile;
3251	INIT_LIST_HEAD(&tfile->next);
3252
3253	sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);
3254
3255	return 0;
3256}
3257
3258static int tun_chr_close(struct inode *inode, struct file *file)
3259{
3260	struct tun_file *tfile = file->private_data;
 
3261
3262	tun_detach(tfile, true);
 
 
3263
3264	return 0;
3265}
3266
3267#ifdef CONFIG_PROC_FS
3268static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file)
3269{
3270	struct tun_file *tfile = file->private_data;
3271	struct tun_struct *tun;
3272	struct ifreq ifr;
3273
3274	memset(&ifr, 0, sizeof(ifr));
 
 
 
 
 
 
 
3275
3276	rtnl_lock();
3277	tun = tun_get(tfile);
3278	if (tun)
3279		tun_get_iff(current->nsproxy->net_ns, tun, &ifr);
3280	rtnl_unlock();
3281
3282	if (tun)
3283		tun_put(tun);
3284
3285	seq_printf(m, "iff:\t%s\n", ifr.ifr_name);
3286}
3287#endif
3288
3289static const struct file_operations tun_fops = {
3290	.owner	= THIS_MODULE,
3291	.llseek = no_llseek,
3292	.read_iter  = tun_chr_read_iter,
3293	.write_iter = tun_chr_write_iter,
 
 
3294	.poll	= tun_chr_poll,
3295	.unlocked_ioctl	= tun_chr_ioctl,
3296#ifdef CONFIG_COMPAT
3297	.compat_ioctl = tun_chr_compat_ioctl,
3298#endif
3299	.open	= tun_chr_open,
3300	.release = tun_chr_close,
3301	.fasync = tun_chr_fasync,
3302#ifdef CONFIG_PROC_FS
3303	.show_fdinfo = tun_chr_show_fdinfo,
3304#endif
3305};
3306
3307static struct miscdevice tun_miscdev = {
3308	.minor = TUN_MINOR,
3309	.name = "tun",
3310	.nodename = "net/tun",
3311	.fops = &tun_fops,
3312};
3313
3314/* ethtool interface */
3315
3316static int tun_get_link_ksettings(struct net_device *dev,
3317				  struct ethtool_link_ksettings *cmd)
3318{
3319	ethtool_link_ksettings_zero_link_mode(cmd, supported);
3320	ethtool_link_ksettings_zero_link_mode(cmd, advertising);
3321	cmd->base.speed		= SPEED_10;
3322	cmd->base.duplex	= DUPLEX_FULL;
3323	cmd->base.port		= PORT_TP;
3324	cmd->base.phy_address	= 0;
3325	cmd->base.autoneg	= AUTONEG_DISABLE;
 
 
 
3326	return 0;
3327}
3328
3329static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
3330{
3331	struct tun_struct *tun = netdev_priv(dev);
3332
3333	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
3334	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
 
3335
3336	switch (tun->flags & TUN_TYPE_MASK) {
3337	case IFF_TUN:
3338		strlcpy(info->bus_info, "tun", sizeof(info->bus_info));
3339		break;
3340	case IFF_TAP:
3341		strlcpy(info->bus_info, "tap", sizeof(info->bus_info));
3342		break;
3343	}
3344}
3345
3346static u32 tun_get_msglevel(struct net_device *dev)
3347{
3348#ifdef TUN_DEBUG
3349	struct tun_struct *tun = netdev_priv(dev);
3350	return tun->debug;
3351#else
3352	return -EOPNOTSUPP;
3353#endif
3354}
3355
3356static void tun_set_msglevel(struct net_device *dev, u32 value)
3357{
3358#ifdef TUN_DEBUG
3359	struct tun_struct *tun = netdev_priv(dev);
3360	tun->debug = value;
3361#endif
3362}
3363
3364static int tun_get_coalesce(struct net_device *dev,
3365			    struct ethtool_coalesce *ec)
3366{
3367	struct tun_struct *tun = netdev_priv(dev);
3368
3369	ec->rx_max_coalesced_frames = tun->rx_batched;
3370
3371	return 0;
3372}
3373
3374static int tun_set_coalesce(struct net_device *dev,
3375			    struct ethtool_coalesce *ec)
3376{
3377	struct tun_struct *tun = netdev_priv(dev);
3378
3379	if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT)
3380		tun->rx_batched = NAPI_POLL_WEIGHT;
3381	else
3382		tun->rx_batched = ec->rx_max_coalesced_frames;
3383
3384	return 0;
3385}
3386
3387static const struct ethtool_ops tun_ethtool_ops = {
 
3388	.get_drvinfo	= tun_get_drvinfo,
3389	.get_msglevel	= tun_get_msglevel,
3390	.set_msglevel	= tun_set_msglevel,
3391	.get_link	= ethtool_op_get_link,
3392	.get_ts_info	= ethtool_op_get_ts_info,
3393	.get_coalesce   = tun_get_coalesce,
3394	.set_coalesce   = tun_set_coalesce,
3395	.get_link_ksettings = tun_get_link_ksettings,
3396};
3397
3398static int tun_queue_resize(struct tun_struct *tun)
3399{
3400	struct net_device *dev = tun->dev;
3401	struct tun_file *tfile;
3402	struct ptr_ring **rings;
3403	int n = tun->numqueues + tun->numdisabled;
3404	int ret, i;
3405
3406	rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL);
3407	if (!rings)
3408		return -ENOMEM;
3409
3410	for (i = 0; i < tun->numqueues; i++) {
3411		tfile = rtnl_dereference(tun->tfiles[i]);
3412		rings[i] = &tfile->tx_ring;
3413	}
3414	list_for_each_entry(tfile, &tun->disabled, next)
3415		rings[i++] = &tfile->tx_ring;
3416
3417	ret = ptr_ring_resize_multiple(rings, n,
3418				       dev->tx_queue_len, GFP_KERNEL,
3419				       tun_ptr_free);
3420
3421	kfree(rings);
3422	return ret;
3423}
3424
3425static int tun_device_event(struct notifier_block *unused,
3426			    unsigned long event, void *ptr)
3427{
3428	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3429	struct tun_struct *tun = netdev_priv(dev);
3430
3431	if (dev->rtnl_link_ops != &tun_link_ops)
3432		return NOTIFY_DONE;
3433
3434	switch (event) {
3435	case NETDEV_CHANGE_TX_QUEUE_LEN:
3436		if (tun_queue_resize(tun))
3437			return NOTIFY_BAD;
3438		break;
3439	default:
3440		break;
3441	}
3442
3443	return NOTIFY_DONE;
3444}
3445
3446static struct notifier_block tun_notifier_block __read_mostly = {
3447	.notifier_call	= tun_device_event,
3448};
3449
3450static int __init tun_init(void)
3451{
3452	int ret = 0;
3453
3454	pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION);
 
3455
3456	ret = rtnl_link_register(&tun_link_ops);
3457	if (ret) {
3458		pr_err("Can't register link_ops\n");
3459		goto err_linkops;
3460	}
3461
3462	ret = misc_register(&tun_miscdev);
3463	if (ret) {
3464		pr_err("Can't register misc device %d\n", TUN_MINOR);
3465		goto err_misc;
3466	}
3467
3468	ret = register_netdevice_notifier(&tun_notifier_block);
3469	if (ret) {
3470		pr_err("Can't register netdevice notifier\n");
3471		goto err_notifier;
3472	}
3473
3474	return  0;
3475
3476err_notifier:
3477	misc_deregister(&tun_miscdev);
3478err_misc:
3479	rtnl_link_unregister(&tun_link_ops);
3480err_linkops:
3481	return ret;
3482}
3483
3484static void tun_cleanup(void)
3485{
3486	misc_deregister(&tun_miscdev);
3487	rtnl_link_unregister(&tun_link_ops);
3488	unregister_netdevice_notifier(&tun_notifier_block);
3489}
3490
3491/* Get an underlying socket object from tun file.  Returns error unless file is
3492 * attached to a device.  The returned object works like a packet socket, it
3493 * can be used for sock_sendmsg/sock_recvmsg.  The caller is responsible for
3494 * holding a reference to the file for as long as the socket is in use. */
3495struct socket *tun_get_socket(struct file *file)
3496{
3497	struct tun_file *tfile;
3498	if (file->f_op != &tun_fops)
3499		return ERR_PTR(-EINVAL);
3500	tfile = file->private_data;
3501	if (!tfile)
3502		return ERR_PTR(-EBADFD);
3503	return &tfile->socket;
 
3504}
3505EXPORT_SYMBOL_GPL(tun_get_socket);
3506
3507struct ptr_ring *tun_get_tx_ring(struct file *file)
3508{
3509	struct tun_file *tfile;
3510
3511	if (file->f_op != &tun_fops)
3512		return ERR_PTR(-EINVAL);
3513	tfile = file->private_data;
3514	if (!tfile)
3515		return ERR_PTR(-EBADFD);
3516	return &tfile->tx_ring;
3517}
3518EXPORT_SYMBOL_GPL(tun_get_tx_ring);
3519
3520module_init(tun_init);
3521module_exit(tun_cleanup);
3522MODULE_DESCRIPTION(DRV_DESCRIPTION);
3523MODULE_AUTHOR(DRV_COPYRIGHT);
3524MODULE_LICENSE("GPL");
3525MODULE_ALIAS_MISCDEV(TUN_MINOR);
3526MODULE_ALIAS("devname:net/tun");