1/*
   2 * VXLAN: Virtual eXtensible Local Area Network
   3 *
   4 * Copyright (c) 2012-2013 Vyatta Inc.
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 */
  10
  11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13#include <linux/kernel.h>
  14#include <linux/types.h>
  15#include <linux/module.h>
  16#include <linux/errno.h>
  17#include <linux/slab.h>
  18#include <linux/skbuff.h>
  19#include <linux/rculist.h>
  20#include <linux/netdevice.h>
  21#include <linux/in.h>
  22#include <linux/ip.h>
  23#include <linux/udp.h>
  24#include <linux/igmp.h>
  25#include <linux/etherdevice.h>
  26#include <linux/if_ether.h>
  27#include <linux/if_vlan.h>
  28#include <linux/hash.h>
  29#include <linux/ethtool.h>
  30#include <net/arp.h>
  31#include <net/ndisc.h>
  32#include <net/ip.h>
  33#include <net/ip_tunnels.h>
  34#include <net/icmp.h>
  35#include <net/udp.h>
  36#include <net/rtnetlink.h>
  37#include <net/route.h>
  38#include <net/dsfield.h>
  39#include <net/inet_ecn.h>
  40#include <net/net_namespace.h>
  41#include <net/netns/generic.h>
  42#include <net/vxlan.h>
  43#include <net/protocol.h>
  44#if IS_ENABLED(CONFIG_IPV6)
  45#include <net/ipv6.h>
  46#include <net/addrconf.h>
  47#include <net/ip6_tunnel.h>
  48#include <net/ip6_checksum.h>
  49#endif
  50
  51#define VXLAN_VERSION	"0.1"
  52
  53#define PORT_HASH_BITS	8
  54#define PORT_HASH_SIZE  (1<<PORT_HASH_BITS)
  55#define VNI_HASH_BITS	10
  56#define VNI_HASH_SIZE	(1<<VNI_HASH_BITS)
  57#define FDB_HASH_BITS	8
  58#define FDB_HASH_SIZE	(1<<FDB_HASH_BITS)
  59#define FDB_AGE_DEFAULT 300 /* 5 min */
  60#define FDB_AGE_INTERVAL (10 * HZ)	/* rescan interval */
  61
  62#define VXLAN_N_VID	(1u << 24)
  63#define VXLAN_VID_MASK	(VXLAN_N_VID - 1)
  64#define VXLAN_HLEN (sizeof(struct udphdr) + sizeof(struct vxlanhdr))
  65
  66#define VXLAN_FLAGS 0x08000000	/* struct vxlanhdr.vx_flags required value. */
  67
  68/* VXLAN protocol header */
  69struct vxlanhdr {
  70	__be32 vx_flags;
  71	__be32 vx_vni;
  72};
  73
  74/* UDP port for VXLAN traffic.
  75 * The IANA assigned port is 4789, but the Linux default is 8472
  76 * for compatibility with early adopters.
  77 */
  78static unsigned short vxlan_port __read_mostly = 8472;
  79module_param_named(udp_port, vxlan_port, ushort, 0444);
  80MODULE_PARM_DESC(udp_port, "Destination UDP port");
  81
  82static bool log_ecn_error = true;
  83module_param(log_ecn_error, bool, 0644);
  84MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");
  85
  86static int vxlan_net_id;
  87
  88static const u8 all_zeros_mac[ETH_ALEN];
  89
  90/* per-network namespace private data for this module */
  91struct vxlan_net {
  92	struct list_head  vxlan_list;
  93	struct hlist_head sock_list[PORT_HASH_SIZE];
  94	spinlock_t	  sock_lock;
  95};
  96
  97union vxlan_addr {
  98	struct sockaddr_in sin;
  99	struct sockaddr_in6 sin6;
 100	struct sockaddr sa;
 101};
 102
 103struct vxlan_rdst {
 104	union vxlan_addr	 remote_ip;
 105	__be16			 remote_port;
 106	u32			 remote_vni;
 107	u32			 remote_ifindex;
 108	struct list_head	 list;
 109	struct rcu_head		 rcu;
 110};
 111
 112/* Forwarding table entry */
 113struct vxlan_fdb {
 114	struct hlist_node hlist;	/* linked list of entries */
 115	struct rcu_head	  rcu;
 116	unsigned long	  updated;	/* jiffies */
 117	unsigned long	  used;
 118	struct list_head  remotes;
 119	u16		  state;	/* see ndm_state */
 120	u8		  flags;	/* see ndm_flags */
 121	u8		  eth_addr[ETH_ALEN];
 122};
 123
 124/* Pseudo network device */
 125struct vxlan_dev {
 126	struct hlist_node hlist;	/* vni hash table */
 127	struct list_head  next;		/* vxlan's per namespace list */
 128	struct vxlan_sock *vn_sock;	/* listening socket */
 129	struct net_device *dev;
 130	struct vxlan_rdst default_dst;	/* default destination */
 131	union vxlan_addr  saddr;	/* source address */
 132	__be16		  dst_port;
 133	__u16		  port_min;	/* source port range */
 134	__u16		  port_max;
 135	__u8		  tos;		/* TOS override */
 136	__u8		  ttl;
 137	u32		  flags;	/* VXLAN_F_* below */
 138
 139	struct work_struct sock_work;
 140	struct work_struct igmp_join;
 141	struct work_struct igmp_leave;
 142
 143	unsigned long	  age_interval;
 144	struct timer_list age_timer;
 145	spinlock_t	  hash_lock;
 146	unsigned int	  addrcnt;
 147	unsigned int	  addrmax;
 148
 149	struct hlist_head fdb_head[FDB_HASH_SIZE];
 150};
 151
 152#define VXLAN_F_LEARN	0x01
 153#define VXLAN_F_PROXY	0x02
 154#define VXLAN_F_RSC	0x04
 155#define VXLAN_F_L2MISS	0x08
 156#define VXLAN_F_L3MISS	0x10
 157#define VXLAN_F_IPV6	0x20 /* internal flag */
 158
 159/* salt for hash table */
 160static u32 vxlan_salt __read_mostly;
 161static struct workqueue_struct *vxlan_wq;
 162
 163static void vxlan_sock_work(struct work_struct *work);
 164
 165#if IS_ENABLED(CONFIG_IPV6)
 166static inline
 167bool vxlan_addr_equal(const union vxlan_addr *a, const union vxlan_addr *b)
 168{
 169       if (a->sa.sa_family != b->sa.sa_family)
 170               return false;
 171       if (a->sa.sa_family == AF_INET6)
 172               return ipv6_addr_equal(&a->sin6.sin6_addr, &b->sin6.sin6_addr);
 173       else
 174               return a->sin.sin_addr.s_addr == b->sin.sin_addr.s_addr;
 175}
 176
 177static inline bool vxlan_addr_any(const union vxlan_addr *ipa)
 178{
 179       if (ipa->sa.sa_family == AF_INET6)
 180               return ipv6_addr_any(&ipa->sin6.sin6_addr);
 181       else
 182               return ipa->sin.sin_addr.s_addr == htonl(INADDR_ANY);
 183}
 184
 185static inline bool vxlan_addr_multicast(const union vxlan_addr *ipa)
 186{
 187       if (ipa->sa.sa_family == AF_INET6)
 188               return ipv6_addr_is_multicast(&ipa->sin6.sin6_addr);
 189       else
 190               return IN_MULTICAST(ntohl(ipa->sin.sin_addr.s_addr));
 191}
 192
 193static int vxlan_nla_get_addr(union vxlan_addr *ip, struct nlattr *nla)
 194{
 195       if (nla_len(nla) >= sizeof(struct in6_addr)) {
 196               nla_memcpy(&ip->sin6.sin6_addr, nla, sizeof(struct in6_addr));
 197               ip->sa.sa_family = AF_INET6;
 198               return 0;
 199       } else if (nla_len(nla) >= sizeof(__be32)) {
 200               ip->sin.sin_addr.s_addr = nla_get_be32(nla);
 201               ip->sa.sa_family = AF_INET;
 202               return 0;
 203       } else {
 204               return -EAFNOSUPPORT;
 205       }
 206}
 207
 208static int vxlan_nla_put_addr(struct sk_buff *skb, int attr,
 209                             const union vxlan_addr *ip)
 210{
 211       if (ip->sa.sa_family == AF_INET6)
 212               return nla_put(skb, attr, sizeof(struct in6_addr), &ip->sin6.sin6_addr);
 213       else
 214               return nla_put_be32(skb, attr, ip->sin.sin_addr.s_addr);
 215}
 216
 217#else /* !CONFIG_IPV6 */
 218
 219static inline
 220bool vxlan_addr_equal(const union vxlan_addr *a, const union vxlan_addr *b)
 221{
 222       return a->sin.sin_addr.s_addr == b->sin.sin_addr.s_addr;
 223}
 224
 225static inline bool vxlan_addr_any(const union vxlan_addr *ipa)
 226{
 227       return ipa->sin.sin_addr.s_addr == htonl(INADDR_ANY);
 228}
 229
 230static inline bool vxlan_addr_multicast(const union vxlan_addr *ipa)
 231{
 232       return IN_MULTICAST(ntohl(ipa->sin.sin_addr.s_addr));
 233}
 234
 235static int vxlan_nla_get_addr(union vxlan_addr *ip, struct nlattr *nla)
 236{
 237       if (nla_len(nla) >= sizeof(struct in6_addr)) {
 238               return -EAFNOSUPPORT;
 239       } else if (nla_len(nla) >= sizeof(__be32)) {
 240               ip->sin.sin_addr.s_addr = nla_get_be32(nla);
 241               ip->sa.sa_family = AF_INET;
 242               return 0;
 243       } else {
 244               return -EAFNOSUPPORT;
 245       }
 246}
 247
 248static int vxlan_nla_put_addr(struct sk_buff *skb, int attr,
 249                             const union vxlan_addr *ip)
 250{
 251       return nla_put_be32(skb, attr, ip->sin.sin_addr.s_addr);
 252}
 253#endif
 254
 255/* Virtual Network hash table head */
 256static inline struct hlist_head *vni_head(struct vxlan_sock *vs, u32 id)
 257{
 258	return &vs->vni_list[hash_32(id, VNI_HASH_BITS)];
 259}
 260
 261/* Socket hash table head */
 262static inline struct hlist_head *vs_head(struct net *net, __be16 port)
 263{
 264	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
 265
 266	return &vn->sock_list[hash_32(ntohs(port), PORT_HASH_BITS)];
 267}
 268
 269/* First remote destination for a forwarding entry.
 270 * Guaranteed to be non-NULL because remotes are never deleted.
 271 */
 272static inline struct vxlan_rdst *first_remote_rcu(struct vxlan_fdb *fdb)
 273{
 274	return list_entry_rcu(fdb->remotes.next, struct vxlan_rdst, list);
 275}
 276
 277static inline struct vxlan_rdst *first_remote_rtnl(struct vxlan_fdb *fdb)
 278{
 279	return list_first_entry(&fdb->remotes, struct vxlan_rdst, list);
 280}
 281
 282/* Find VXLAN socket based on network namespace and UDP port */
 283static struct vxlan_sock *vxlan_find_sock(struct net *net, __be16 port)
 284{
 285	struct vxlan_sock *vs;
 286
 287	hlist_for_each_entry_rcu(vs, vs_head(net, port), hlist) {
 288		if (inet_sk(vs->sock->sk)->inet_sport == port)
 289			return vs;
 290	}
 291	return NULL;
 292}
 293
 294static struct vxlan_dev *vxlan_vs_find_vni(struct vxlan_sock *vs, u32 id)
 295{
 296	struct vxlan_dev *vxlan;
 297
 298	hlist_for_each_entry_rcu(vxlan, vni_head(vs, id), hlist) {
 299		if (vxlan->default_dst.remote_vni == id)
 300			return vxlan;
 301	}
 302
 303	return NULL;
 304}
 305
 306/* Look up VNI in a per net namespace table */
 307static struct vxlan_dev *vxlan_find_vni(struct net *net, u32 id, __be16 port)
 308{
 309	struct vxlan_sock *vs;
 310
 311	vs = vxlan_find_sock(net, port);
 312	if (!vs)
 313		return NULL;
 314
 315	return vxlan_vs_find_vni(vs, id);
 316}
 317
 318/* Fill in neighbour message in skbuff. */
 319static int vxlan_fdb_info(struct sk_buff *skb, struct vxlan_dev *vxlan,
 320			  const struct vxlan_fdb *fdb,
 321			  u32 portid, u32 seq, int type, unsigned int flags,
 322			  const struct vxlan_rdst *rdst)
 323{
 324	unsigned long now = jiffies;
 325	struct nda_cacheinfo ci;
 326	struct nlmsghdr *nlh;
 327	struct ndmsg *ndm;
 328	bool send_ip, send_eth;
 329
 330	nlh = nlmsg_put(skb, portid, seq, type, sizeof(*ndm), flags);
 331	if (nlh == NULL)
 332		return -EMSGSIZE;
 333
 334	ndm = nlmsg_data(nlh);
 335	memset(ndm, 0, sizeof(*ndm));
 336
 337	send_eth = send_ip = true;
 338
 339	if (type == RTM_GETNEIGH) {
 340		ndm->ndm_family	= AF_INET;
 341		send_ip = !vxlan_addr_any(&rdst->remote_ip);
 342		send_eth = !is_zero_ether_addr(fdb->eth_addr);
 343	} else
 344		ndm->ndm_family	= AF_BRIDGE;
 345	ndm->ndm_state = fdb->state;
 346	ndm->ndm_ifindex = vxlan->dev->ifindex;
 347	ndm->ndm_flags = fdb->flags;
 348	ndm->ndm_type = NDA_DST;
 349
 350	if (send_eth && nla_put(skb, NDA_LLADDR, ETH_ALEN, &fdb->eth_addr))
 351		goto nla_put_failure;
 352
 353	if (send_ip && vxlan_nla_put_addr(skb, NDA_DST, &rdst->remote_ip))
 354		goto nla_put_failure;
 355
 356	if (rdst->remote_port && rdst->remote_port != vxlan->dst_port &&
 357	    nla_put_be16(skb, NDA_PORT, rdst->remote_port))
 358		goto nla_put_failure;
 359	if (rdst->remote_vni != vxlan->default_dst.remote_vni &&
 360	    nla_put_u32(skb, NDA_VNI, rdst->remote_vni))
 361		goto nla_put_failure;
 362	if (rdst->remote_ifindex &&
 363	    nla_put_u32(skb, NDA_IFINDEX, rdst->remote_ifindex))
 364		goto nla_put_failure;
 365
 366	ci.ndm_used	 = jiffies_to_clock_t(now - fdb->used);
 367	ci.ndm_confirmed = 0;
 368	ci.ndm_updated	 = jiffies_to_clock_t(now - fdb->updated);
 369	ci.ndm_refcnt	 = 0;
 370
 371	if (nla_put(skb, NDA_CACHEINFO, sizeof(ci), &ci))
 372		goto nla_put_failure;
 373
 374	return nlmsg_end(skb, nlh);
 375
 376nla_put_failure:
 377	nlmsg_cancel(skb, nlh);
 378	return -EMSGSIZE;
 379}
 380
 381static inline size_t vxlan_nlmsg_size(void)
 382{
 383	return NLMSG_ALIGN(sizeof(struct ndmsg))
 384		+ nla_total_size(ETH_ALEN) /* NDA_LLADDR */
 385		+ nla_total_size(sizeof(struct in6_addr)) /* NDA_DST */
 386		+ nla_total_size(sizeof(__be16)) /* NDA_PORT */
 387		+ nla_total_size(sizeof(__be32)) /* NDA_VNI */
 388		+ nla_total_size(sizeof(__u32)) /* NDA_IFINDEX */
 389		+ nla_total_size(sizeof(struct nda_cacheinfo));
 390}
 391
 392static void vxlan_fdb_notify(struct vxlan_dev *vxlan, struct vxlan_fdb *fdb,
 393			     struct vxlan_rdst *rd, int type)
 394{
 395	struct net *net = dev_net(vxlan->dev);
 396	struct sk_buff *skb;
 397	int err = -ENOBUFS;
 398
 399	skb = nlmsg_new(vxlan_nlmsg_size(), GFP_ATOMIC);
 400	if (skb == NULL)
 401		goto errout;
 402
 403	err = vxlan_fdb_info(skb, vxlan, fdb, 0, 0, type, 0, rd);
 404	if (err < 0) {
 405		/* -EMSGSIZE implies BUG in vxlan_nlmsg_size() */
 406		WARN_ON(err == -EMSGSIZE);
 407		kfree_skb(skb);
 408		goto errout;
 409	}
 410
 411	rtnl_notify(skb, net, 0, RTNLGRP_NEIGH, NULL, GFP_ATOMIC);
 412	return;
 413errout:
 414	if (err < 0)
 415		rtnl_set_sk_err(net, RTNLGRP_NEIGH, err);
 416}
 417
 418static void vxlan_ip_miss(struct net_device *dev, union vxlan_addr *ipa)
 419{
 420	struct vxlan_dev *vxlan = netdev_priv(dev);
 421	struct vxlan_fdb f = {
 422		.state = NUD_STALE,
 423	};
 424	struct vxlan_rdst remote = {
 425		.remote_ip = *ipa, /* goes to NDA_DST */
 426		.remote_vni = VXLAN_N_VID,
 427	};
 428
 429	vxlan_fdb_notify(vxlan, &f, &remote, RTM_GETNEIGH);
 430}
 431
 432static void vxlan_fdb_miss(struct vxlan_dev *vxlan, const u8 eth_addr[ETH_ALEN])
 433{
 434	struct vxlan_fdb f = {
 435		.state = NUD_STALE,
 436	};
 437	struct vxlan_rdst remote = { };
 438
 439	memcpy(f.eth_addr, eth_addr, ETH_ALEN);
 440
 441	vxlan_fdb_notify(vxlan, &f, &remote, RTM_GETNEIGH);
 442}
 443
 444/* Hash Ethernet address */
 445static u32 eth_hash(const unsigned char *addr)
 446{
 447	u64 value = get_unaligned((u64 *)addr);
 448
 449	/* only want 6 bytes */
 450#ifdef __BIG_ENDIAN
 451	value >>= 16;
 452#else
 453	value <<= 16;
 454#endif
 455	return hash_64(value, FDB_HASH_BITS);
 456}
 457
 458/* Hash chain to use given mac address */
 459static inline struct hlist_head *vxlan_fdb_head(struct vxlan_dev *vxlan,
 460						const u8 *mac)
 461{
 462	return &vxlan->fdb_head[eth_hash(mac)];
 463}
 464
 465/* Look up Ethernet address in forwarding table */
 466static struct vxlan_fdb *__vxlan_find_mac(struct vxlan_dev *vxlan,
 467					const u8 *mac)
 468{
 469	struct hlist_head *head = vxlan_fdb_head(vxlan, mac);
 470	struct vxlan_fdb *f;
 471
 472	hlist_for_each_entry_rcu(f, head, hlist) {
 473		if (ether_addr_equal(mac, f->eth_addr))
 474			return f;
 475	}
 476
 477	return NULL;
 478}
 479
 480static struct vxlan_fdb *vxlan_find_mac(struct vxlan_dev *vxlan,
 481					const u8 *mac)
 482{
 483	struct vxlan_fdb *f;
 484
 485	f = __vxlan_find_mac(vxlan, mac);
 486	if (f)
 487		f->used = jiffies;
 488
 489	return f;
 490}
 491
 492/* caller should hold vxlan->hash_lock */
 493static struct vxlan_rdst *vxlan_fdb_find_rdst(struct vxlan_fdb *f,
 494					      union vxlan_addr *ip, __be16 port,
 495					      __u32 vni, __u32 ifindex)
 496{
 497	struct vxlan_rdst *rd;
 498
 499	list_for_each_entry(rd, &f->remotes, list) {
 500		if (vxlan_addr_equal(&rd->remote_ip, ip) &&
 501		    rd->remote_port == port &&
 502		    rd->remote_vni == vni &&
 503		    rd->remote_ifindex == ifindex)
 504			return rd;
 505	}
 506
 507	return NULL;
 508}
 509
 510/* Replace destination of unicast mac */
 511static int vxlan_fdb_replace(struct vxlan_fdb *f,
 512			     union vxlan_addr *ip, __be16 port, __u32 vni, __u32 ifindex)
 513{
 514	struct vxlan_rdst *rd;
 515
 516	rd = vxlan_fdb_find_rdst(f, ip, port, vni, ifindex);
 517	if (rd)
 518		return 0;
 519
 520	rd = list_first_entry_or_null(&f->remotes, struct vxlan_rdst, list);
 521	if (!rd)
 522		return 0;
 523	rd->remote_ip = *ip;
 524	rd->remote_port = port;
 525	rd->remote_vni = vni;
 526	rd->remote_ifindex = ifindex;
 527	return 1;
 528}
 529
 530/* Add/update destinations for multicast */
 531static int vxlan_fdb_append(struct vxlan_fdb *f,
 532			    union vxlan_addr *ip, __be16 port, __u32 vni,
 533			    __u32 ifindex, struct vxlan_rdst **rdp)
 534{
 535	struct vxlan_rdst *rd;
 536
 537	rd = vxlan_fdb_find_rdst(f, ip, port, vni, ifindex);
 538	if (rd)
 539		return 0;
 540
 541	rd = kmalloc(sizeof(*rd), GFP_ATOMIC);
 542	if (rd == NULL)
 543		return -ENOBUFS;
 544	rd->remote_ip = *ip;
 545	rd->remote_port = port;
 546	rd->remote_vni = vni;
 547	rd->remote_ifindex = ifindex;
 548
 549	list_add_tail_rcu(&rd->list, &f->remotes);
 550
 551	*rdp = rd;
 552	return 1;
 553}
 554
 555static struct sk_buff **vxlan_gro_receive(struct sk_buff **head, struct sk_buff *skb)
 556{
 557	struct sk_buff *p, **pp = NULL;
 558	struct vxlanhdr *vh, *vh2;
 559	struct ethhdr *eh, *eh2;
 560	unsigned int hlen, off_vx, off_eth;
 561	const struct packet_offload *ptype;
 562	__be16 type;
 563	int flush = 1;
 564
 565	off_vx = skb_gro_offset(skb);
 566	hlen = off_vx + sizeof(*vh);
 567	vh   = skb_gro_header_fast(skb, off_vx);
 568	if (skb_gro_header_hard(skb, hlen)) {
 569		vh = skb_gro_header_slow(skb, hlen, off_vx);
 570		if (unlikely(!vh))
 571			goto out;
 572	}
 573	skb_gro_pull(skb, sizeof(struct vxlanhdr)); /* pull vxlan header */
 574
 575	off_eth = skb_gro_offset(skb);
 576	hlen = off_eth + sizeof(*eh);
 577	eh   = skb_gro_header_fast(skb, off_eth);
 578	if (skb_gro_header_hard(skb, hlen)) {
 579		eh = skb_gro_header_slow(skb, hlen, off_eth);
 580		if (unlikely(!eh))
 581			goto out;
 582	}
 583
 584	flush = 0;
 585
 586	for (p = *head; p; p = p->next) {
 587		if (!NAPI_GRO_CB(p)->same_flow)
 588			continue;
 589
 590		vh2 = (struct vxlanhdr *)(p->data + off_vx);
 591		eh2 = (struct ethhdr   *)(p->data + off_eth);
 592		if (vh->vx_vni != vh2->vx_vni || compare_ether_header(eh, eh2)) {
 593			NAPI_GRO_CB(p)->same_flow = 0;
 594			continue;
 595		}
 596	}
 597
 598	type = eh->h_proto;
 599
 600	rcu_read_lock();
 601	ptype = gro_find_receive_by_type(type);
 602	if (ptype == NULL) {
 603		flush = 1;
 604		goto out_unlock;
 605	}
 606
 607	skb_gro_pull(skb, sizeof(*eh)); /* pull inner eth header */
 608	pp = ptype->callbacks.gro_receive(head, skb);
 609
 610out_unlock:
 611	rcu_read_unlock();
 612out:
 613	NAPI_GRO_CB(skb)->flush |= flush;
 614
 615	return pp;
 616}
 617
 618static int vxlan_gro_complete(struct sk_buff *skb, int nhoff)
 619{
 620	struct ethhdr *eh;
 621	struct packet_offload *ptype;
 622	__be16 type;
 623	int vxlan_len  = sizeof(struct vxlanhdr) + sizeof(struct ethhdr);
 624	int err = -ENOSYS;
 625
 626	eh = (struct ethhdr *)(skb->data + nhoff + sizeof(struct vxlanhdr));
 627	type = eh->h_proto;
 628
 629	rcu_read_lock();
 630	ptype = gro_find_complete_by_type(type);
 631	if (ptype != NULL)
 632		err = ptype->callbacks.gro_complete(skb, nhoff + vxlan_len);
 633
 634	rcu_read_unlock();
 635	return err;
 636}
 637
 638/* Notify netdevs that UDP port started listening */
 639static void vxlan_notify_add_rx_port(struct vxlan_sock *vs)
 640{
 641	struct net_device *dev;
 642	struct sock *sk = vs->sock->sk;
 643	struct net *net = sock_net(sk);
 644	sa_family_t sa_family = sk->sk_family;
 645	__be16 port = inet_sk(sk)->inet_sport;
 646	int err;
 647
 648	if (sa_family == AF_INET) {
 649		err = udp_add_offload(&vs->udp_offloads);
 650		if (err)
 651			pr_warn("vxlan: udp_add_offload failed with status %d\n", err);
 652	}
 653
 654	rcu_read_lock();
 655	for_each_netdev_rcu(net, dev) {
 656		if (dev->netdev_ops->ndo_add_vxlan_port)
 657			dev->netdev_ops->ndo_add_vxlan_port(dev, sa_family,
 658							    port);
 659	}
 660	rcu_read_unlock();
 661}
 662
 663/* Notify netdevs that UDP port is no more listening */
 664static void vxlan_notify_del_rx_port(struct vxlan_sock *vs)
 665{
 666	struct net_device *dev;
 667	struct sock *sk = vs->sock->sk;
 668	struct net *net = sock_net(sk);
 669	sa_family_t sa_family = sk->sk_family;
 670	__be16 port = inet_sk(sk)->inet_sport;
 671
 672	rcu_read_lock();
 673	for_each_netdev_rcu(net, dev) {
 674		if (dev->netdev_ops->ndo_del_vxlan_port)
 675			dev->netdev_ops->ndo_del_vxlan_port(dev, sa_family,
 676							    port);
 677	}
 678	rcu_read_unlock();
 679
 680	if (sa_family == AF_INET)
 681		udp_del_offload(&vs->udp_offloads);
 682}
 683
 684/* Add new entry to forwarding table -- assumes lock held */
 685static int vxlan_fdb_create(struct vxlan_dev *vxlan,
 686			    const u8 *mac, union vxlan_addr *ip,
 687			    __u16 state, __u16 flags,
 688			    __be16 port, __u32 vni, __u32 ifindex,
 689			    __u8 ndm_flags)
 690{
 691	struct vxlan_rdst *rd = NULL;
 692	struct vxlan_fdb *f;
 693	int notify = 0;
 694
 695	f = __vxlan_find_mac(vxlan, mac);
 696	if (f) {
 697		if (flags & NLM_F_EXCL) {
 698			netdev_dbg(vxlan->dev,
 699				   "lost race to create %pM\n", mac);
 700			return -EEXIST;
 701		}
 702		if (f->state != state) {
 703			f->state = state;
 704			f->updated = jiffies;
 705			notify = 1;
 706		}
 707		if (f->flags != ndm_flags) {
 708			f->flags = ndm_flags;
 709			f->updated = jiffies;
 710			notify = 1;
 711		}
 712		if ((flags & NLM_F_REPLACE)) {
 713			/* Only change unicasts */
 714			if (!(is_multicast_ether_addr(f->eth_addr) ||
 715			     is_zero_ether_addr(f->eth_addr))) {
 716				int rc = vxlan_fdb_replace(f, ip, port, vni,
 717							   ifindex);
 718
 719				if (rc < 0)
 720					return rc;
 721				notify |= rc;
 722			} else
 723				return -EOPNOTSUPP;
 724		}
 725		if ((flags & NLM_F_APPEND) &&
 726		    (is_multicast_ether_addr(f->eth_addr) ||
 727		     is_zero_ether_addr(f->eth_addr))) {
 728			int rc = vxlan_fdb_append(f, ip, port, vni, ifindex,
 729						  &rd);
 730
 731			if (rc < 0)
 732				return rc;
 733			notify |= rc;
 734		}
 735	} else {
 736		if (!(flags & NLM_F_CREATE))
 737			return -ENOENT;
 738
 739		if (vxlan->addrmax && vxlan->addrcnt >= vxlan->addrmax)
 740			return -ENOSPC;
 741
 742		/* Disallow replace to add a multicast entry */
 743		if ((flags & NLM_F_REPLACE) &&
 744		    (is_multicast_ether_addr(mac) || is_zero_ether_addr(mac)))
 745			return -EOPNOTSUPP;
 746
 747		netdev_dbg(vxlan->dev, "add %pM -> %pIS\n", mac, ip);
 748		f = kmalloc(sizeof(*f), GFP_ATOMIC);
 749		if (!f)
 750			return -ENOMEM;
 751
 752		notify = 1;
 753		f->state = state;
 754		f->flags = ndm_flags;
 755		f->updated = f->used = jiffies;
 756		INIT_LIST_HEAD(&f->remotes);
 757		memcpy(f->eth_addr, mac, ETH_ALEN);
 758
 759		vxlan_fdb_append(f, ip, port, vni, ifindex, &rd);
 760
 761		++vxlan->addrcnt;
 762		hlist_add_head_rcu(&f->hlist,
 763				   vxlan_fdb_head(vxlan, mac));
 764	}
 765
 766	if (notify) {
 767		if (rd == NULL)
 768			rd = first_remote_rtnl(f);
 769		vxlan_fdb_notify(vxlan, f, rd, RTM_NEWNEIGH);
 770	}
 771
 772	return 0;
 773}
 774
 775static void vxlan_fdb_free(struct rcu_head *head)
 776{
 777	struct vxlan_fdb *f = container_of(head, struct vxlan_fdb, rcu);
 778	struct vxlan_rdst *rd, *nd;
 779
 780	list_for_each_entry_safe(rd, nd, &f->remotes, list)
 781		kfree(rd);
 782	kfree(f);
 783}
 784
 785static void vxlan_fdb_destroy(struct vxlan_dev *vxlan, struct vxlan_fdb *f)
 786{
 787	netdev_dbg(vxlan->dev,
 788		    "delete %pM\n", f->eth_addr);
 789
 790	--vxlan->addrcnt;
 791	vxlan_fdb_notify(vxlan, f, first_remote_rtnl(f), RTM_DELNEIGH);
 792
 793	hlist_del_rcu(&f->hlist);
 794	call_rcu(&f->rcu, vxlan_fdb_free);
 795}
 796
 797static int vxlan_fdb_parse(struct nlattr *tb[], struct vxlan_dev *vxlan,
 798			   union vxlan_addr *ip, __be16 *port, u32 *vni, u32 *ifindex)
 799{
 800	struct net *net = dev_net(vxlan->dev);
 801	int err;
 802
 803	if (tb[NDA_DST]) {
 804		err = vxlan_nla_get_addr(ip, tb[NDA_DST]);
 805		if (err)
 806			return err;
 807	} else {
 808		union vxlan_addr *remote = &vxlan->default_dst.remote_ip;
 809		if (remote->sa.sa_family == AF_INET) {
 810			ip->sin.sin_addr.s_addr = htonl(INADDR_ANY);
 811			ip->sa.sa_family = AF_INET;
 812#if IS_ENABLED(CONFIG_IPV6)
 813		} else {
 814			ip->sin6.sin6_addr = in6addr_any;
 815			ip->sa.sa_family = AF_INET6;
 816#endif
 817		}
 818	}
 819
 820	if (tb[NDA_PORT]) {
 821		if (nla_len(tb[NDA_PORT]) != sizeof(__be16))
 822			return -EINVAL;
 823		*port = nla_get_be16(tb[NDA_PORT]);
 824	} else {
 825		*port = vxlan->dst_port;
 826	}
 827
 828	if (tb[NDA_VNI]) {
 829		if (nla_len(tb[NDA_VNI]) != sizeof(u32))
 830			return -EINVAL;
 831		*vni = nla_get_u32(tb[NDA_VNI]);
 832	} else {
 833		*vni = vxlan->default_dst.remote_vni;
 834	}
 835
 836	if (tb[NDA_IFINDEX]) {
 837		struct net_device *tdev;
 838
 839		if (nla_len(tb[NDA_IFINDEX]) != sizeof(u32))
 840			return -EINVAL;
 841		*ifindex = nla_get_u32(tb[NDA_IFINDEX]);
 842		tdev = __dev_get_by_index(net, *ifindex);
 843		if (!tdev)
 844			return -EADDRNOTAVAIL;
 845	} else {
 846		*ifindex = 0;
 847	}
 848
 849	return 0;
 850}
 851
 852/* Add static entry (via netlink) */
 853static int vxlan_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
 854			 struct net_device *dev,
 855			 const unsigned char *addr, u16 flags)
 856{
 857	struct vxlan_dev *vxlan = netdev_priv(dev);
 858	/* struct net *net = dev_net(vxlan->dev); */
 859	union vxlan_addr ip;
 860	__be16 port;
 861	u32 vni, ifindex;
 862	int err;
 863
 864	if (!(ndm->ndm_state & (NUD_PERMANENT|NUD_REACHABLE))) {
 865		pr_info("RTM_NEWNEIGH with invalid state %#x\n",
 866			ndm->ndm_state);
 867		return -EINVAL;
 868	}
 869
 870	if (tb[NDA_DST] == NULL)
 871		return -EINVAL;
 872
 873	err = vxlan_fdb_parse(tb, vxlan, &ip, &port, &vni, &ifindex);
 874	if (err)
 875		return err;
 876
 877	if (vxlan->default_dst.remote_ip.sa.sa_family != ip.sa.sa_family)
 878		return -EAFNOSUPPORT;
 879
 880	spin_lock_bh(&vxlan->hash_lock);
 881	err = vxlan_fdb_create(vxlan, addr, &ip, ndm->ndm_state, flags,
 882			       port, vni, ifindex, ndm->ndm_flags);
 883	spin_unlock_bh(&vxlan->hash_lock);
 884
 885	return err;
 886}
 887
 888/* Delete entry (via netlink) */
 889static int vxlan_fdb_delete(struct ndmsg *ndm, struct nlattr *tb[],
 890			    struct net_device *dev,
 891			    const unsigned char *addr)
 892{
 893	struct vxlan_dev *vxlan = netdev_priv(dev);
 894	struct vxlan_fdb *f;
 895	struct vxlan_rdst *rd = NULL;
 896	union vxlan_addr ip;
 897	__be16 port;
 898	u32 vni, ifindex;
 899	int err;
 900
 901	err = vxlan_fdb_parse(tb, vxlan, &ip, &port, &vni, &ifindex);
 902	if (err)
 903		return err;
 904
 905	err = -ENOENT;
 906
 907	spin_lock_bh(&vxlan->hash_lock);
 908	f = vxlan_find_mac(vxlan, addr);
 909	if (!f)
 910		goto out;
 911
 912	if (!vxlan_addr_any(&ip)) {
 913		rd = vxlan_fdb_find_rdst(f, &ip, port, vni, ifindex);
 914		if (!rd)
 915			goto out;
 916	}
 917
 918	err = 0;
 919
 920	/* remove a destination if it's not the only one on the list,
 921	 * otherwise destroy the fdb entry
 922	 */
 923	if (rd && !list_is_singular(&f->remotes)) {
 924		list_del_rcu(&rd->list);
 925		vxlan_fdb_notify(vxlan, f, rd, RTM_DELNEIGH);
 926		kfree_rcu(rd, rcu);
 927		goto out;
 928	}
 929
 930	vxlan_fdb_destroy(vxlan, f);
 931
 932out:
 933	spin_unlock_bh(&vxlan->hash_lock);
 934
 935	return err;
 936}
 937
 938/* Dump forwarding table */
 939static int vxlan_fdb_dump(struct sk_buff *skb, struct netlink_callback *cb,
 940			  struct net_device *dev, int idx)
 941{
 942	struct vxlan_dev *vxlan = netdev_priv(dev);
 943	unsigned int h;
 944
 945	for (h = 0; h < FDB_HASH_SIZE; ++h) {
 946		struct vxlan_fdb *f;
 947		int err;
 948
 949		hlist_for_each_entry_rcu(f, &vxlan->fdb_head[h], hlist) {
 950			struct vxlan_rdst *rd;
 951
 952			if (idx < cb->args[0])
 953				goto skip;
 954
 955			list_for_each_entry_rcu(rd, &f->remotes, list) {
 956				err = vxlan_fdb_info(skb, vxlan, f,
 957						     NETLINK_CB(cb->skb).portid,
 958						     cb->nlh->nlmsg_seq,
 959						     RTM_NEWNEIGH,
 960						     NLM_F_MULTI, rd);
 961				if (err < 0)
 962					goto out;
 963			}
 964skip:
 965			++idx;
 966		}
 967	}
 968out:
 969	return idx;
 970}
 971
 972/* Watch incoming packets to learn mapping between Ethernet address
 973 * and Tunnel endpoint.
 974 * Return true if packet is bogus and should be droppped.
 975 */
 976static bool vxlan_snoop(struct net_device *dev,
 977			union vxlan_addr *src_ip, const u8 *src_mac)
 978{
 979	struct vxlan_dev *vxlan = netdev_priv(dev);
 980	struct vxlan_fdb *f;
 981
 982	f = vxlan_find_mac(vxlan, src_mac);
 983	if (likely(f)) {
 984		struct vxlan_rdst *rdst = first_remote_rcu(f);
 985
 986		if (likely(vxlan_addr_equal(&rdst->remote_ip, src_ip)))
 987			return false;
 988
 989		/* Don't migrate static entries, drop packets */
 990		if (f->state & NUD_NOARP)
 991			return true;
 992
 993		if (net_ratelimit())
 994			netdev_info(dev,
 995				    "%pM migrated from %pIS to %pIS\n",
 996				    src_mac, &rdst->remote_ip, &src_ip);
 997
 998		rdst->remote_ip = *src_ip;
 999		f->updated = jiffies;
1000		vxlan_fdb_notify(vxlan, f, rdst, RTM_NEWNEIGH);
1001	} else {
1002		/* learned new entry */
1003		spin_lock(&vxlan->hash_lock);
1004
1005		/* close off race between vxlan_flush and incoming packets */
1006		if (netif_running(dev))
1007			vxlan_fdb_create(vxlan, src_mac, src_ip,
1008					 NUD_REACHABLE,
1009					 NLM_F_EXCL|NLM_F_CREATE,
1010					 vxlan->dst_port,
1011					 vxlan->default_dst.remote_vni,
1012					 0, NTF_SELF);
1013		spin_unlock(&vxlan->hash_lock);
1014	}
1015
1016	return false;
1017}
1018
1019/* See if multicast group is already in use by other ID */
1020static bool vxlan_group_used(struct vxlan_net *vn, struct vxlan_dev *dev)
1021{
1022	struct vxlan_dev *vxlan;
1023
1024	/* The vxlan_sock is only used by dev, leaving group has
1025	 * no effect on other vxlan devices.
1026	 */
1027	if (atomic_read(&dev->vn_sock->refcnt) == 1)
1028		return false;
1029
1030	list_for_each_entry(vxlan, &vn->vxlan_list, next) {
1031		if (!netif_running(vxlan->dev) || vxlan == dev)
1032			continue;
1033
1034		if (vxlan->vn_sock != dev->vn_sock)
1035			continue;
1036
1037		if (!vxlan_addr_equal(&vxlan->default_dst.remote_ip,
1038				      &dev->default_dst.remote_ip))
1039			continue;
1040
1041		if (vxlan->default_dst.remote_ifindex !=
1042		    dev->default_dst.remote_ifindex)
1043			continue;
1044
1045		return true;
1046	}
1047
1048	return false;
1049}
1050
1051static void vxlan_sock_hold(struct vxlan_sock *vs)
1052{
1053	atomic_inc(&vs->refcnt);
1054}
1055
1056void vxlan_sock_release(struct vxlan_sock *vs)
1057{
1058	struct sock *sk = vs->sock->sk;
1059	struct net *net = sock_net(sk);
1060	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
1061
1062	if (!atomic_dec_and_test(&vs->refcnt))
1063		return;
1064
1065	spin_lock(&vn->sock_lock);
1066	hlist_del_rcu(&vs->hlist);
1067	rcu_assign_sk_user_data(vs->sock->sk, NULL);
1068	vxlan_notify_del_rx_port(vs);
1069	spin_unlock(&vn->sock_lock);
1070
1071	queue_work(vxlan_wq, &vs->del_work);
1072}
1073EXPORT_SYMBOL_GPL(vxlan_sock_release);
1074
1075/* Callback to update multicast group membership when first VNI on
1076 * multicast asddress is brought up
1077 * Done as workqueue because ip_mc_join_group acquires RTNL.
1078 */
1079static void vxlan_igmp_join(struct work_struct *work)
1080{
1081	struct vxlan_dev *vxlan = container_of(work, struct vxlan_dev, igmp_join);
1082	struct vxlan_sock *vs = vxlan->vn_sock;
1083	struct sock *sk = vs->sock->sk;
1084	union vxlan_addr *ip = &vxlan->default_dst.remote_ip;
1085	int ifindex = vxlan->default_dst.remote_ifindex;
1086
1087	lock_sock(sk);
1088	if (ip->sa.sa_family == AF_INET) {
1089		struct ip_mreqn mreq = {
1090			.imr_multiaddr.s_addr	= ip->sin.sin_addr.s_addr,
1091			.imr_ifindex		= ifindex,
1092		};
1093
1094		ip_mc_join_group(sk, &mreq);
1095#if IS_ENABLED(CONFIG_IPV6)
1096	} else {
1097		ipv6_stub->ipv6_sock_mc_join(sk, ifindex,
1098					     &ip->sin6.sin6_addr);
1099#endif
1100	}
1101	release_sock(sk);
1102
1103	vxlan_sock_release(vs);
1104	dev_put(vxlan->dev);
1105}
1106
1107/* Inverse of vxlan_igmp_join when last VNI is brought down */
1108static void vxlan_igmp_leave(struct work_struct *work)
1109{
1110	struct vxlan_dev *vxlan = container_of(work, struct vxlan_dev, igmp_leave);
1111	struct vxlan_sock *vs = vxlan->vn_sock;
1112	struct sock *sk = vs->sock->sk;
1113	union vxlan_addr *ip = &vxlan->default_dst.remote_ip;
1114	int ifindex = vxlan->default_dst.remote_ifindex;
1115
1116	lock_sock(sk);
1117	if (ip->sa.sa_family == AF_INET) {
1118		struct ip_mreqn mreq = {
1119			.imr_multiaddr.s_addr	= ip->sin.sin_addr.s_addr,
1120			.imr_ifindex		= ifindex,
1121		};
1122
1123		ip_mc_leave_group(sk, &mreq);
1124#if IS_ENABLED(CONFIG_IPV6)
1125	} else {
1126		ipv6_stub->ipv6_sock_mc_drop(sk, ifindex,
1127					     &ip->sin6.sin6_addr);
1128#endif
1129	}
1130
1131	release_sock(sk);
1132
1133	vxlan_sock_release(vs);
1134	dev_put(vxlan->dev);
1135}
1136
1137/* Callback from net/ipv4/udp.c to receive packets */
1138static int vxlan_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
1139{
1140	struct vxlan_sock *vs;
1141	struct vxlanhdr *vxh;
1142
1143	/* Need Vxlan and inner Ethernet header to be present */
1144	if (!pskb_may_pull(skb, VXLAN_HLEN))
1145		goto error;
1146
1147	/* Return packets with reserved bits set */
1148	vxh = (struct vxlanhdr *)(udp_hdr(skb) + 1);
1149	if (vxh->vx_flags != htonl(VXLAN_FLAGS) ||
1150	    (vxh->vx_vni & htonl(0xff))) {
1151		netdev_dbg(skb->dev, "invalid vxlan flags=%#x vni=%#x\n",
1152			   ntohl(vxh->vx_flags), ntohl(vxh->vx_vni));
1153		goto error;
1154	}
1155
1156	if (iptunnel_pull_header(skb, VXLAN_HLEN, htons(ETH_P_TEB)))
1157		goto drop;
1158
1159	vs = rcu_dereference_sk_user_data(sk);
1160	if (!vs)
1161		goto drop;
1162
1163	/* If the NIC driver gave us an encapsulated packet
1164	 * with the encapsulation mark, the device checksummed it
1165	 * for us. Otherwise force the upper layers to verify it.
1166	 */
1167	if ((skb->ip_summed != CHECKSUM_UNNECESSARY && skb->ip_summed != CHECKSUM_PARTIAL) ||
1168	    !skb->encapsulation)
1169		skb->ip_summed = CHECKSUM_NONE;
1170
1171	skb->encapsulation = 0;
1172
1173	vs->rcv(vs, skb, vxh->vx_vni);
1174	return 0;
1175
1176drop:
1177	/* Consume bad packet */
1178	kfree_skb(skb);
1179	return 0;
1180
1181error:
1182	/* Return non vxlan pkt */
1183	return 1;
1184}
1185
1186static void vxlan_rcv(struct vxlan_sock *vs,
1187		      struct sk_buff *skb, __be32 vx_vni)
1188{
1189	struct iphdr *oip = NULL;
1190	struct ipv6hdr *oip6 = NULL;
1191	struct vxlan_dev *vxlan;
1192	struct pcpu_sw_netstats *stats;
1193	union vxlan_addr saddr;
1194	__u32 vni;
1195	int err = 0;
1196	union vxlan_addr *remote_ip;
1197
1198	vni = ntohl(vx_vni) >> 8;
1199	/* Is this VNI defined? */
1200	vxlan = vxlan_vs_find_vni(vs, vni);
1201	if (!vxlan)
1202		goto drop;
1203
1204	remote_ip = &vxlan->default_dst.remote_ip;
1205	skb_reset_mac_header(skb);
1206	skb->protocol = eth_type_trans(skb, vxlan->dev);
1207
1208	/* Ignore packet loops (and multicast echo) */
1209	if (ether_addr_equal(eth_hdr(skb)->h_source, vxlan->dev->dev_addr))
1210		goto drop;
1211
1212	/* Re-examine inner Ethernet packet */
1213	if (remote_ip->sa.sa_family == AF_INET) {
1214		oip = ip_hdr(skb);
1215		saddr.sin.sin_addr.s_addr = oip->saddr;
1216		saddr.sa.sa_family = AF_INET;
1217#if IS_ENABLED(CONFIG_IPV6)
1218	} else {
1219		oip6 = ipv6_hdr(skb);
1220		saddr.sin6.sin6_addr = oip6->saddr;
1221		saddr.sa.sa_family = AF_INET6;
1222#endif
1223	}
1224
1225	if ((vxlan->flags & VXLAN_F_LEARN) &&
1226	    vxlan_snoop(skb->dev, &saddr, eth_hdr(skb)->h_source))
1227		goto drop;
1228
1229	skb_reset_network_header(skb);
1230
1231	if (oip6)
1232		err = IP6_ECN_decapsulate(oip6, skb);
1233	if (oip)
1234		err = IP_ECN_decapsulate(oip, skb);
1235
1236	if (unlikely(err)) {
1237		if (log_ecn_error) {
1238			if (oip6)
1239				net_info_ratelimited("non-ECT from %pI6\n",
1240						     &oip6->saddr);
1241			if (oip)
1242				net_info_ratelimited("non-ECT from %pI4 with TOS=%#x\n",
1243						     &oip->saddr, oip->tos);
1244		}
1245		if (err > 1) {
1246			++vxlan->dev->stats.rx_frame_errors;
1247			++vxlan->dev->stats.rx_errors;
1248			goto drop;
1249		}
1250	}
1251
1252	stats = this_cpu_ptr(vxlan->dev->tstats);
1253	u64_stats_update_begin(&stats->syncp);
1254	stats->rx_packets++;
1255	stats->rx_bytes += skb->len;
1256	u64_stats_update_end(&stats->syncp);
1257
1258	netif_rx(skb);
1259
1260	return;
1261drop:
1262	/* Consume bad packet */
1263	kfree_skb(skb);
1264}
1265
1266static int arp_reduce(struct net_device *dev, struct sk_buff *skb)
1267{
1268	struct vxlan_dev *vxlan = netdev_priv(dev);
1269	struct arphdr *parp;
1270	u8 *arpptr, *sha;
1271	__be32 sip, tip;
1272	struct neighbour *n;
1273
1274	if (dev->flags & IFF_NOARP)
1275		goto out;
1276
1277	if (!pskb_may_pull(skb, arp_hdr_len(dev))) {
1278		dev->stats.tx_dropped++;
1279		goto out;
1280	}
1281	parp = arp_hdr(skb);
1282
1283	if ((parp->ar_hrd != htons(ARPHRD_ETHER) &&
1284	     parp->ar_hrd != htons(ARPHRD_IEEE802)) ||
1285	    parp->ar_pro != htons(ETH_P_IP) ||
1286	    parp->ar_op != htons(ARPOP_REQUEST) ||
1287	    parp->ar_hln != dev->addr_len ||
1288	    parp->ar_pln != 4)
1289		goto out;
1290	arpptr = (u8 *)parp + sizeof(struct arphdr);
1291	sha = arpptr;
1292	arpptr += dev->addr_len;	/* sha */
1293	memcpy(&sip, arpptr, sizeof(sip));
1294	arpptr += sizeof(sip);
1295	arpptr += dev->addr_len;	/* tha */
1296	memcpy(&tip, arpptr, sizeof(tip));
1297
1298	if (ipv4_is_loopback(tip) ||
1299	    ipv4_is_multicast(tip))
1300		goto out;
1301
1302	n = neigh_lookup(&arp_tbl, &tip, dev);
1303
1304	if (n) {
1305		struct vxlan_fdb *f;
1306		struct sk_buff	*reply;
1307
1308		if (!(n->nud_state & NUD_CONNECTED)) {
1309			neigh_release(n);
1310			goto out;
1311		}
1312
1313		f = vxlan_find_mac(vxlan, n->ha);
1314		if (f && vxlan_addr_any(&(first_remote_rcu(f)->remote_ip))) {
1315			/* bridge-local neighbor */
1316			neigh_release(n);
1317			goto out;
1318		}
1319
1320		reply = arp_create(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha,
1321				n->ha, sha);
1322
1323		neigh_release(n);
1324
1325		if (reply == NULL)
1326			goto out;
1327
1328		skb_reset_mac_header(reply);
1329		__skb_pull(reply, skb_network_offset(reply));
1330		reply->ip_summed = CHECKSUM_UNNECESSARY;
1331		reply->pkt_type = PACKET_HOST;
1332
1333		if (netif_rx_ni(reply) == NET_RX_DROP)
1334			dev->stats.rx_dropped++;
1335	} else if (vxlan->flags & VXLAN_F_L3MISS) {
1336		union vxlan_addr ipa = {
1337			.sin.sin_addr.s_addr = tip,
1338			.sa.sa_family = AF_INET,
1339		};
1340
1341		vxlan_ip_miss(dev, &ipa);
1342	}
1343out:
1344	consume_skb(skb);
1345	return NETDEV_TX_OK;
1346}
1347
1348#if IS_ENABLED(CONFIG_IPV6)
1349
1350static struct sk_buff *vxlan_na_create(struct sk_buff *request,
1351	struct neighbour *n, bool isrouter)
1352{
1353	struct net_device *dev = request->dev;
1354	struct sk_buff *reply;
1355	struct nd_msg *ns, *na;
1356	struct ipv6hdr *pip6;
1357	u8 *daddr;
1358	int na_olen = 8; /* opt hdr + ETH_ALEN for target */
1359	int ns_olen;
1360	int i, len;
1361
1362	if (dev == NULL)
1363		return NULL;
1364
1365	len = LL_RESERVED_SPACE(dev) + sizeof(struct ipv6hdr) +
1366		sizeof(*na) + na_olen + dev->needed_tailroom;
1367	reply = alloc_skb(len, GFP_ATOMIC);
1368	if (reply == NULL)
1369		return NULL;
1370
1371	reply->protocol = htons(ETH_P_IPV6);
1372	reply->dev = dev;
1373	skb_reserve(reply, LL_RESERVED_SPACE(request->dev));
1374	skb_push(reply, sizeof(struct ethhdr));
1375	skb_set_mac_header(reply, 0);
1376
1377	ns = (struct nd_msg *)skb_transport_header(request);
1378
1379	daddr = eth_hdr(request)->h_source;
1380	ns_olen = request->len - skb_transport_offset(request) - sizeof(*ns);
1381	for (i = 0; i < ns_olen-1; i += (ns->opt[i+1]<<3)) {
1382		if (ns->opt[i] == ND_OPT_SOURCE_LL_ADDR) {
1383			daddr = ns->opt + i + sizeof(struct nd_opt_hdr);
1384			break;
1385		}
1386	}
1387
1388	/* Ethernet header */
1389	ether_addr_copy(eth_hdr(reply)->h_dest, daddr);
1390	ether_addr_copy(eth_hdr(reply)->h_source, n->ha);
1391	eth_hdr(reply)->h_proto = htons(ETH_P_IPV6);
1392	reply->protocol = htons(ETH_P_IPV6);
1393
1394	skb_pull(reply, sizeof(struct ethhdr));
1395	skb_set_network_header(reply, 0);
1396	skb_put(reply, sizeof(struct ipv6hdr));
1397
1398	/* IPv6 header */
1399
1400	pip6 = ipv6_hdr(reply);
1401	memset(pip6, 0, sizeof(struct ipv6hdr));
1402	pip6->version = 6;
1403	pip6->priority = ipv6_hdr(request)->priority;
1404	pip6->nexthdr = IPPROTO_ICMPV6;
1405	pip6->hop_limit = 255;
1406	pip6->daddr = ipv6_hdr(request)->saddr;
1407	pip6->saddr = *(struct in6_addr *)n->primary_key;
1408
1409	skb_pull(reply, sizeof(struct ipv6hdr));
1410	skb_set_transport_header(reply, 0);
1411
1412	na = (struct nd_msg *)skb_put(reply, sizeof(*na) + na_olen);
1413
1414	/* Neighbor Advertisement */
1415	memset(na, 0, sizeof(*na)+na_olen);
1416	na->icmph.icmp6_type = NDISC_NEIGHBOUR_ADVERTISEMENT;
1417	na->icmph.icmp6_router = isrouter;
1418	na->icmph.icmp6_override = 1;
1419	na->icmph.icmp6_solicited = 1;
1420	na->target = ns->target;
1421	ether_addr_copy(&na->opt[2], n->ha);
1422	na->opt[0] = ND_OPT_TARGET_LL_ADDR;
1423	na->opt[1] = na_olen >> 3;
1424
1425	na->icmph.icmp6_cksum = csum_ipv6_magic(&pip6->saddr,
1426		&pip6->daddr, sizeof(*na)+na_olen, IPPROTO_ICMPV6,
1427		csum_partial(na, sizeof(*na)+na_olen, 0));
1428
1429	pip6->payload_len = htons(sizeof(*na)+na_olen);
1430
1431	skb_push(reply, sizeof(struct ipv6hdr));
1432
1433	reply->ip_summed = CHECKSUM_UNNECESSARY;
1434
1435	return reply;
1436}
1437
1438static int neigh_reduce(struct net_device *dev, struct sk_buff *skb)
1439{
1440	struct vxlan_dev *vxlan = netdev_priv(dev);
1441	struct nd_msg *msg;
1442	const struct ipv6hdr *iphdr;
1443	const struct in6_addr *saddr, *daddr;
1444	struct neighbour *n;
1445	struct inet6_dev *in6_dev;
1446
1447	in6_dev = __in6_dev_get(dev);
1448	if (!in6_dev)
1449		goto out;
1450
1451	if (!pskb_may_pull(skb, skb->len))
1452		goto out;
1453
1454	iphdr = ipv6_hdr(skb);
1455	saddr = &iphdr->saddr;
1456	daddr = &iphdr->daddr;
1457
1458	msg = (struct nd_msg *)skb_transport_header(skb);
1459	if (msg->icmph.icmp6_code != 0 ||
1460	    msg->icmph.icmp6_type != NDISC_NEIGHBOUR_SOLICITATION)
1461		goto out;
1462
1463	if (ipv6_addr_loopback(daddr) ||
1464	    ipv6_addr_is_multicast(&msg->target))
1465		goto out;
1466
1467	n = neigh_lookup(ipv6_stub->nd_tbl, &msg->target, dev);
1468
1469	if (n) {
1470		struct vxlan_fdb *f;
1471		struct sk_buff *reply;
1472
1473		if (!(n->nud_state & NUD_CONNECTED)) {
1474			neigh_release(n);
1475			goto out;
1476		}
1477
1478		f = vxlan_find_mac(vxlan, n->ha);
1479		if (f && vxlan_addr_any(&(first_remote_rcu(f)->remote_ip))) {
1480			/* bridge-local neighbor */
1481			neigh_release(n);
1482			goto out;
1483		}
1484
1485		reply = vxlan_na_create(skb, n,
1486					!!(f ? f->flags & NTF_ROUTER : 0));
1487
1488		neigh_release(n);
1489
1490		if (reply == NULL)
1491			goto out;
1492
1493		if (netif_rx_ni(reply) == NET_RX_DROP)
1494			dev->stats.rx_dropped++;
1495
1496	} else if (vxlan->flags & VXLAN_F_L3MISS) {
1497		union vxlan_addr ipa = {
1498			.sin6.sin6_addr = msg->target,
1499			.sa.sa_family = AF_INET6,
1500		};
1501
1502		vxlan_ip_miss(dev, &ipa);
1503	}
1504
1505out:
1506	consume_skb(skb);
1507	return NETDEV_TX_OK;
1508}
1509#endif
1510
1511static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb)
1512{
1513	struct vxlan_dev *vxlan = netdev_priv(dev);
1514	struct neighbour *n;
1515
1516	if (is_multicast_ether_addr(eth_hdr(skb)->h_dest))
1517		return false;
1518
1519	n = NULL;
1520	switch (ntohs(eth_hdr(skb)->h_proto)) {
1521	case ETH_P_IP:
1522	{
1523		struct iphdr *pip;
1524
1525		if (!pskb_may_pull(skb, sizeof(struct iphdr)))
1526			return false;
1527		pip = ip_hdr(skb);
1528		n = neigh_lookup(&arp_tbl, &pip->daddr, dev);
1529		if (!n && (vxlan->flags & VXLAN_F_L3MISS)) {
1530			union vxlan_addr ipa = {
1531				.sin.sin_addr.s_addr = pip->daddr,
1532				.sa.sa_family = AF_INET,
1533			};
1534
1535			vxlan_ip_miss(dev, &ipa);
1536			return false;
1537		}
1538
1539		break;
1540	}
1541#if IS_ENABLED(CONFIG_IPV6)
1542	case ETH_P_IPV6:
1543	{
1544		struct ipv6hdr *pip6;
1545
1546		if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
1547			return false;
1548		pip6 = ipv6_hdr(skb);
1549		n = neigh_lookup(ipv6_stub->nd_tbl, &pip6->daddr, dev);
1550		if (!n && (vxlan->flags & VXLAN_F_L3MISS)) {
1551			union vxlan_addr ipa = {
1552				.sin6.sin6_addr = pip6->daddr,
1553				.sa.sa_family = AF_INET6,
1554			};
1555
1556			vxlan_ip_miss(dev, &ipa);
1557			return false;
1558		}
1559
1560		break;
1561	}
1562#endif
1563	default:
1564		return false;
1565	}
1566
1567	if (n) {
1568		bool diff;
1569
1570		diff = !ether_addr_equal(eth_hdr(skb)->h_dest, n->ha);
1571		if (diff) {
1572			memcpy(eth_hdr(skb)->h_source, eth_hdr(skb)->h_dest,
1573				dev->addr_len);
1574			memcpy(eth_hdr(skb)->h_dest, n->ha, dev->addr_len);
1575		}
1576		neigh_release(n);
1577		return diff;
1578	}
1579
1580	return false;
1581}
1582
1583/* Compute source port for outgoing packet
1584 *   first choice to use L4 flow hash since it will spread
1585 *     better and maybe available from hardware
1586 *   secondary choice is to use jhash on the Ethernet header
1587 */
1588__be16 vxlan_src_port(__u16 port_min, __u16 port_max, struct sk_buff *skb)
1589{
1590	unsigned int range = (port_max - port_min) + 1;
1591	u32 hash;
1592
1593	hash = skb_get_hash(skb);
1594	if (!hash)
1595		hash = jhash(skb->data, 2 * ETH_ALEN,
1596			     (__force u32) skb->protocol);
1597
1598	return htons((((u64) hash * range) >> 32) + port_min);
1599}
1600EXPORT_SYMBOL_GPL(vxlan_src_port);
1601
1602static int handle_offloads(struct sk_buff *skb)
1603{
1604	if (skb_is_gso(skb)) {
1605		int err = skb_unclone(skb, GFP_ATOMIC);
1606		if (unlikely(err))
1607			return err;
1608
1609		skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1610	} else if (skb->ip_summed != CHECKSUM_PARTIAL)
1611		skb->ip_summed = CHECKSUM_NONE;
1612
1613	return 0;
1614}
1615
1616#if IS_ENABLED(CONFIG_IPV6)
1617static int vxlan6_xmit_skb(struct vxlan_sock *vs,
1618			   struct dst_entry *dst, struct sk_buff *skb,
1619			   struct net_device *dev, struct in6_addr *saddr,
1620			   struct in6_addr *daddr, __u8 prio, __u8 ttl,
1621			   __be16 src_port, __be16 dst_port, __be32 vni)
1622{
1623	struct ipv6hdr *ip6h;
1624	struct vxlanhdr *vxh;
1625	struct udphdr *uh;
1626	int min_headroom;
1627	int err;
1628
1629	if (!skb->encapsulation) {
1630		skb_reset_inner_headers(skb);
1631		skb->encapsulation = 1;
1632	}
1633
1634	skb_scrub_packet(skb, false);
1635
1636	min_headroom = LL_RESERVED_SPACE(dst->dev) + dst->header_len
1637			+ VXLAN_HLEN + sizeof(struct ipv6hdr)
1638			+ (vlan_tx_tag_present(skb) ? VLAN_HLEN : 0);
1639
1640	/* Need space for new headers (invalidates iph ptr) */
1641	err = skb_cow_head(skb, min_headroom);
1642	if (unlikely(err))
1643		return err;
1644
1645	if (vlan_tx_tag_present(skb)) {
1646		if (WARN_ON(!__vlan_put_tag(skb,
1647					    skb->vlan_proto,
1648					    vlan_tx_tag_get(skb))))
1649			return -ENOMEM;
1650
1651		skb->vlan_tci = 0;
1652	}
1653
1654	vxh = (struct vxlanhdr *) __skb_push(skb, sizeof(*vxh));
1655	vxh->vx_flags = htonl(VXLAN_FLAGS);
1656	vxh->vx_vni = vni;
1657
1658	__skb_push(skb, sizeof(*uh));
1659	skb_reset_transport_header(skb);
1660	uh = udp_hdr(skb);
1661
1662	uh->dest = dst_port;
1663	uh->source = src_port;
1664
1665	uh->len = htons(skb->len);
1666	uh->check = 0;
1667
1668	memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
1669	IPCB(skb)->flags &= ~(IPSKB_XFRM_TUNNEL_SIZE | IPSKB_XFRM_TRANSFORMED |
1670			      IPSKB_REROUTED);
1671	skb_dst_set(skb, dst);
1672
1673	if (!skb_is_gso(skb) && !(dst->dev->features & NETIF_F_IPV6_CSUM)) {
1674		__wsum csum = skb_checksum(skb, 0, skb->len, 0);
1675		skb->ip_summed = CHECKSUM_UNNECESSARY;
1676		uh->check = csum_ipv6_magic(saddr, daddr, skb->len,
1677					    IPPROTO_UDP, csum);
1678		if (uh->check == 0)
1679			uh->check = CSUM_MANGLED_0;
1680	} else {
1681		skb->ip_summed = CHECKSUM_PARTIAL;
1682		skb->csum_start = skb_transport_header(skb) - skb->head;
1683		skb->csum_offset = offsetof(struct udphdr, check);
1684		uh->check = ~csum_ipv6_magic(saddr, daddr,
1685					     skb->len, IPPROTO_UDP, 0);
1686	}
1687
1688	__skb_push(skb, sizeof(*ip6h));
1689	skb_reset_network_header(skb);
1690	ip6h		  = ipv6_hdr(skb);
1691	ip6h->version	  = 6;
1692	ip6h->priority	  = prio;
1693	ip6h->flow_lbl[0] = 0;
1694	ip6h->flow_lbl[1] = 0;
1695	ip6h->flow_lbl[2] = 0;
1696	ip6h->payload_len = htons(skb->len);
1697	ip6h->nexthdr     = IPPROTO_UDP;
1698	ip6h->hop_limit   = ttl;
1699	ip6h->daddr	  = *daddr;
1700	ip6h->saddr	  = *saddr;
1701
1702	err = handle_offloads(skb);
1703	if (err)
1704		return err;
1705
1706	ip6tunnel_xmit(skb, dev);
1707	return 0;
1708}
1709#endif
1710
1711int vxlan_xmit_skb(struct vxlan_sock *vs,
1712		   struct rtable *rt, struct sk_buff *skb,
1713		   __be32 src, __be32 dst, __u8 tos, __u8 ttl, __be16 df,
1714		   __be16 src_port, __be16 dst_port, __be32 vni)
1715{
1716	struct vxlanhdr *vxh;
1717	struct udphdr *uh;
1718	int min_headroom;
1719	int err;
1720
1721	if (!skb->encapsulation) {
1722		skb_reset_inner_headers(skb);
1723		skb->encapsulation = 1;
1724	}
1725
1726	min_headroom = LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len
1727			+ VXLAN_HLEN + sizeof(struct iphdr)
1728			+ (vlan_tx_tag_present(skb) ? VLAN_HLEN : 0);
1729
1730	/* Need space for new headers (invalidates iph ptr) */
1731	err = skb_cow_head(skb, min_headroom);
1732	if (unlikely(err))
1733		return err;
1734
1735	if (vlan_tx_tag_present(skb)) {
1736		if (WARN_ON(!__vlan_put_tag(skb,
1737					    skb->vlan_proto,
1738					    vlan_tx_tag_get(skb))))
1739			return -ENOMEM;
1740
1741		skb->vlan_tci = 0;
1742	}
1743
1744	vxh = (struct vxlanhdr *) __skb_push(skb, sizeof(*vxh));
1745	vxh->vx_flags = htonl(VXLAN_FLAGS);
1746	vxh->vx_vni = vni;
1747
1748	__skb_push(skb, sizeof(*uh));
1749	skb_reset_transport_header(skb);
1750	uh = udp_hdr(skb);
1751
1752	uh->dest = dst_port;
1753	uh->source = src_port;
1754
1755	uh->len = htons(skb->len);
1756	uh->check = 0;
1757
1758	err = handle_offloads(skb);
1759	if (err)
1760		return err;
1761
1762	return iptunnel_xmit(vs->sock->sk, rt, skb, src, dst, IPPROTO_UDP,
1763			     tos, ttl, df, false);
1764}
1765EXPORT_SYMBOL_GPL(vxlan_xmit_skb);
1766
1767/* Bypass encapsulation if the destination is local */
1768static void vxlan_encap_bypass(struct sk_buff *skb, struct vxlan_dev *src_vxlan,
1769			       struct vxlan_dev *dst_vxlan)
1770{
1771	struct pcpu_sw_netstats *tx_stats, *rx_stats;
1772	union vxlan_addr loopback;
1773	union vxlan_addr *remote_ip = &dst_vxlan->default_dst.remote_ip;
1774
1775	tx_stats = this_cpu_ptr(src_vxlan->dev->tstats);
1776	rx_stats = this_cpu_ptr(dst_vxlan->dev->tstats);
1777	skb->pkt_type = PACKET_HOST;
1778	skb->encapsulation = 0;
1779	skb->dev = dst_vxlan->dev;
1780	__skb_pull(skb, skb_network_offset(skb));
1781
1782	if (remote_ip->sa.sa_family == AF_INET) {
1783		loopback.sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
1784		loopback.sa.sa_family =  AF_INET;
1785#if IS_ENABLED(CONFIG_IPV6)
1786	} else {
1787		loopback.sin6.sin6_addr = in6addr_loopback;
1788		loopback.sa.sa_family =  AF_INET6;
1789#endif
1790	}
1791
1792	if (dst_vxlan->flags & VXLAN_F_LEARN)
1793		vxlan_snoop(skb->dev, &loopback, eth_hdr(skb)->h_source);
1794
1795	u64_stats_update_begin(&tx_stats->syncp);
1796	tx_stats->tx_packets++;
1797	tx_stats->tx_bytes += skb->len;
1798	u64_stats_update_end(&tx_stats->syncp);
1799
1800	if (netif_rx(skb) == NET_RX_SUCCESS) {
1801		u64_stats_update_begin(&rx_stats->syncp);
1802		rx_stats->rx_packets++;
1803		rx_stats->rx_bytes += skb->len;
1804		u64_stats_update_end(&rx_stats->syncp);
1805	} else {
1806		skb->dev->stats.rx_dropped++;
1807	}
1808}
1809
1810static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
1811			   struct vxlan_rdst *rdst, bool did_rsc)
1812{
1813	struct vxlan_dev *vxlan = netdev_priv(dev);
1814	struct rtable *rt = NULL;
1815	const struct iphdr *old_iph;
1816	struct flowi4 fl4;
1817	union vxlan_addr *dst;
1818	__be16 src_port = 0, dst_port;
1819	u32 vni;
1820	__be16 df = 0;
1821	__u8 tos, ttl;
1822	int err;
1823
1824	dst_port = rdst->remote_port ? rdst->remote_port : vxlan->dst_port;
1825	vni = rdst->remote_vni;
1826	dst = &rdst->remote_ip;
1827
1828	if (vxlan_addr_any(dst)) {
1829		if (did_rsc) {
1830			/* short-circuited back to local bridge */
1831			vxlan_encap_bypass(skb, vxlan, vxlan);
1832			return;
1833		}
1834		goto drop;
1835	}
1836
1837	old_iph = ip_hdr(skb);
1838
1839	ttl = vxlan->ttl;
1840	if (!ttl && vxlan_addr_multicast(dst))
1841		ttl = 1;
1842
1843	tos = vxlan->tos;
1844	if (tos == 1)
1845		tos = ip_tunnel_get_dsfield(old_iph, skb);
1846
1847	src_port = vxlan_src_port(vxlan->port_min, vxlan->port_max, skb);
1848
1849	if (dst->sa.sa_family == AF_INET) {
1850		memset(&fl4, 0, sizeof(fl4));
1851		fl4.flowi4_oif = rdst->remote_ifindex;
1852		fl4.flowi4_tos = RT_TOS(tos);
1853		fl4.daddr = dst->sin.sin_addr.s_addr;
1854		fl4.saddr = vxlan->saddr.sin.sin_addr.s_addr;
1855
1856		rt = ip_route_output_key(dev_net(dev), &fl4);
1857		if (IS_ERR(rt)) {
1858			netdev_dbg(dev, "no route to %pI4\n",
1859				   &dst->sin.sin_addr.s_addr);
1860			dev->stats.tx_carrier_errors++;
1861			goto tx_error;
1862		}
1863
1864		if (rt->dst.dev == dev) {
1865			netdev_dbg(dev, "circular route to %pI4\n",
1866				   &dst->sin.sin_addr.s_addr);
1867			dev->stats.collisions++;
1868			goto rt_tx_error;
1869		}
1870
1871		/* Bypass encapsulation if the destination is local */
1872		if (rt->rt_flags & RTCF_LOCAL &&
1873		    !(rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))) {
1874			struct vxlan_dev *dst_vxlan;
1875
1876			ip_rt_put(rt);
1877			dst_vxlan = vxlan_find_vni(dev_net(dev), vni, dst_port);
1878			if (!dst_vxlan)
1879				goto tx_error;
1880			vxlan_encap_bypass(skb, vxlan, dst_vxlan);
1881			return;
1882		}
1883
1884		tos = ip_tunnel_ecn_encap(tos, old_iph, skb);
1885		ttl = ttl ? : ip4_dst_hoplimit(&rt->dst);
1886
1887		err = vxlan_xmit_skb(vxlan->vn_sock, rt, skb,
1888				     fl4.saddr, dst->sin.sin_addr.s_addr,
1889				     tos, ttl, df, src_port, dst_port,
1890				     htonl(vni << 8));
1891
1892		if (err < 0)
1893			goto rt_tx_error;
1894		iptunnel_xmit_stats(err, &dev->stats, dev->tstats);
1895#if IS_ENABLED(CONFIG_IPV6)
1896	} else {
1897		struct sock *sk = vxlan->vn_sock->sock->sk;
1898		struct dst_entry *ndst;
1899		struct flowi6 fl6;
1900		u32 flags;
1901
1902		memset(&fl6, 0, sizeof(fl6));
1903		fl6.flowi6_oif = rdst->remote_ifindex;
1904		fl6.daddr = dst->sin6.sin6_addr;
1905		fl6.saddr = vxlan->saddr.sin6.sin6_addr;
1906		fl6.flowi6_proto = IPPROTO_UDP;
1907
1908		if (ipv6_stub->ipv6_dst_lookup(sk, &ndst, &fl6)) {
1909			netdev_dbg(dev, "no route to %pI6\n",
1910				   &dst->sin6.sin6_addr);
1911			dev->stats.tx_carrier_errors++;
1912			goto tx_error;
1913		}
1914
1915		if (ndst->dev == dev) {
1916			netdev_dbg(dev, "circular route to %pI6\n",
1917				   &dst->sin6.sin6_addr);
1918			dst_release(ndst);
1919			dev->stats.collisions++;
1920			goto tx_error;
1921		}
1922
1923		/* Bypass encapsulation if the destination is local */
1924		flags = ((struct rt6_info *)ndst)->rt6i_flags;
1925		if (flags & RTF_LOCAL &&
1926		    !(flags & (RTCF_BROADCAST | RTCF_MULTICAST))) {
1927			struct vxlan_dev *dst_vxlan;
1928
1929			dst_release(ndst);
1930			dst_vxlan = vxlan_find_vni(dev_net(dev), vni, dst_port);
1931			if (!dst_vxlan)
1932				goto tx_error;
1933			vxlan_encap_bypass(skb, vxlan, dst_vxlan);
1934			return;
1935		}
1936
1937		ttl = ttl ? : ip6_dst_hoplimit(ndst);
1938
1939		err = vxlan6_xmit_skb(vxlan->vn_sock, ndst, skb,
1940				      dev, &fl6.saddr, &fl6.daddr, 0, ttl,
1941				      src_port, dst_port, htonl(vni << 8));
1942#endif
1943	}
1944
1945	return;
1946
1947drop:
1948	dev->stats.tx_dropped++;
1949	goto tx_free;
1950
1951rt_tx_error:
1952	ip_rt_put(rt);
1953tx_error:
1954	dev->stats.tx_errors++;
1955tx_free:
1956	dev_kfree_skb(skb);
1957}
1958
1959/* Transmit local packets over Vxlan
1960 *
1961 * Outer IP header inherits ECN and DF from inner header.
1962 * Outer UDP destination is the VXLAN assigned port.
1963 *           source port is based on hash of flow
1964 */
1965static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
1966{
1967	struct vxlan_dev *vxlan = netdev_priv(dev);
1968	struct ethhdr *eth;
1969	bool did_rsc = false;
1970	struct vxlan_rdst *rdst, *fdst = NULL;
1971	struct vxlan_fdb *f;
1972
1973	skb_reset_mac_header(skb);
1974	eth = eth_hdr(skb);
1975
1976	if ((vxlan->flags & VXLAN_F_PROXY)) {
1977		if (ntohs(eth->h_proto) == ETH_P_ARP)
1978			return arp_reduce(dev, skb);
1979#if IS_ENABLED(CONFIG_IPV6)
1980		else if (ntohs(eth->h_proto) == ETH_P_IPV6 &&
1981			 skb->len >= sizeof(struct ipv6hdr) + sizeof(struct nd_msg) &&
1982			 ipv6_hdr(skb)->nexthdr == IPPROTO_ICMPV6) {
1983				struct nd_msg *msg;
1984
1985				msg = (struct nd_msg *)skb_transport_header(skb);
1986				if (msg->icmph.icmp6_code == 0 &&
1987				    msg->icmph.icmp6_type == NDISC_NEIGHBOUR_SOLICITATION)
1988					return neigh_reduce(dev, skb);
1989		}
1990#endif
1991	}
1992
1993	f = vxlan_find_mac(vxlan, eth->h_dest);
1994	did_rsc = false;
1995
1996	if (f && (f->flags & NTF_ROUTER) && (vxlan->flags & VXLAN_F_RSC) &&
1997	    (ntohs(eth->h_proto) == ETH_P_IP ||
1998	     ntohs(eth->h_proto) == ETH_P_IPV6)) {
1999		did_rsc = route_shortcircuit(dev, skb);
2000		if (did_rsc)
2001			f = vxlan_find_mac(vxlan, eth->h_dest);
2002	}
2003
2004	if (f == NULL) {
2005		f = vxlan_find_mac(vxlan, all_zeros_mac);
2006		if (f == NULL) {
2007			if ((vxlan->flags & VXLAN_F_L2MISS) &&
2008			    !is_multicast_ether_addr(eth->h_dest))
2009				vxlan_fdb_miss(vxlan, eth->h_dest);
2010
2011			dev->stats.tx_dropped++;
2012			kfree_skb(skb);
2013			return NETDEV_TX_OK;
2014		}
2015	}
2016
2017	list_for_each_entry_rcu(rdst, &f->remotes, list) {
2018		struct sk_buff *skb1;
2019
2020		if (!fdst) {
2021			fdst = rdst;
2022			continue;
2023		}
2024		skb1 = skb_clone(skb, GFP_ATOMIC);
2025		if (skb1)
2026			vxlan_xmit_one(skb1, dev, rdst, did_rsc);
2027	}
2028
2029	if (fdst)
2030		vxlan_xmit_one(skb, dev, fdst, did_rsc);
2031	else
2032		kfree_skb(skb);
2033	return NETDEV_TX_OK;
2034}
2035
2036/* Walk the forwarding table and purge stale entries */
2037static void vxlan_cleanup(unsigned long arg)
2038{
2039	struct vxlan_dev *vxlan = (struct vxlan_dev *) arg;
2040	unsigned long next_timer = jiffies + FDB_AGE_INTERVAL;
2041	unsigned int h;
2042
2043	if (!netif_running(vxlan->dev))
2044		return;
2045
2046	spin_lock_bh(&vxlan->hash_lock);
2047	for (h = 0; h < FDB_HASH_SIZE; ++h) {
2048		struct hlist_node *p, *n;
2049		hlist_for_each_safe(p, n, &vxlan->fdb_head[h]) {
2050			struct vxlan_fdb *f
2051				= container_of(p, struct vxlan_fdb, hlist);
2052			unsigned long timeout;
2053
2054			if (f->state & NUD_PERMANENT)
2055				continue;
2056
2057			timeout = f->used + vxlan->age_interval * HZ;
2058			if (time_before_eq(timeout, jiffies)) {
2059				netdev_dbg(vxlan->dev,
2060					   "garbage collect %pM\n",
2061					   f->eth_addr);
2062				f->state = NUD_STALE;
2063				vxlan_fdb_destroy(vxlan, f);
2064			} else if (time_before(timeout, next_timer))
2065				next_timer = timeout;
2066		}
2067	}
2068	spin_unlock_bh(&vxlan->hash_lock);
2069
2070	mod_timer(&vxlan->age_timer, next_timer);
2071}
2072
2073static void vxlan_vs_add_dev(struct vxlan_sock *vs, struct vxlan_dev *vxlan)
2074{
2075	__u32 vni = vxlan->default_dst.remote_vni;
2076
2077	vxlan->vn_sock = vs;
2078	hlist_add_head_rcu(&vxlan->hlist, vni_head(vs, vni));
2079}
2080
2081/* Setup stats when device is created */
2082static int vxlan_init(struct net_device *dev)
2083{
2084	struct vxlan_dev *vxlan = netdev_priv(dev);
2085	struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id);
2086	struct vxlan_sock *vs;
2087
2088	dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
2089	if (!dev->tstats)
2090		return -ENOMEM;
2091
2092	spin_lock(&vn->sock_lock);
2093	vs = vxlan_find_sock(dev_net(dev), vxlan->dst_port);
2094	if (vs) {
2095		/* If we have a socket with same port already, reuse it */
2096		atomic_inc(&vs->refcnt);
2097		vxlan_vs_add_dev(vs, vxlan);
2098	} else {
2099		/* otherwise make new socket outside of RTNL */
2100		dev_hold(dev);
2101		queue_work(vxlan_wq, &vxlan->sock_work);
2102	}
2103	spin_unlock(&vn->sock_lock);
2104
2105	return 0;
2106}
2107
2108static void vxlan_fdb_delete_default(struct vxlan_dev *vxlan)
2109{
2110	struct vxlan_fdb *f;
2111
2112	spin_lock_bh(&vxlan->hash_lock);
2113	f = __vxlan_find_mac(vxlan, all_zeros_mac);
2114	if (f)
2115		vxlan_fdb_destroy(vxlan, f);
2116	spin_unlock_bh(&vxlan->hash_lock);
2117}
2118
2119static void vxlan_uninit(struct net_device *dev)
2120{
2121	struct vxlan_dev *vxlan = netdev_priv(dev);
2122	struct vxlan_sock *vs = vxlan->vn_sock;
2123
2124	vxlan_fdb_delete_default(vxlan);
2125
2126	if (vs)
2127		vxlan_sock_release(vs);
2128	free_percpu(dev->tstats);
2129}
2130
2131/* Start ageing timer and join group when device is brought up */
2132static int vxlan_open(struct net_device *dev)
2133{
2134	struct vxlan_dev *vxlan = netdev_priv(dev);
2135	struct vxlan_sock *vs = vxlan->vn_sock;
2136
2137	/* socket hasn't been created */
2138	if (!vs)
2139		return -ENOTCONN;
2140
2141	if (vxlan_addr_multicast(&vxlan->default_dst.remote_ip)) {
2142		vxlan_sock_hold(vs);
2143		dev_hold(dev);
2144		queue_work(vxlan_wq, &vxlan->igmp_join);
2145	}
2146
2147	if (vxlan->age_interval)
2148		mod_timer(&vxlan->age_timer, jiffies + FDB_AGE_INTERVAL);
2149
2150	return 0;
2151}
2152
2153/* Purge the forwarding table */
2154static void vxlan_flush(struct vxlan_dev *vxlan)
2155{
2156	unsigned int h;
2157
2158	spin_lock_bh(&vxlan->hash_lock);
2159	for (h = 0; h < FDB_HASH_SIZE; ++h) {
2160		struct hlist_node *p, *n;
2161		hlist_for_each_safe(p, n, &vxlan->fdb_head[h]) {
2162			struct vxlan_fdb *f
2163				= container_of(p, struct vxlan_fdb, hlist);
2164			/* the all_zeros_mac entry is deleted at vxlan_uninit */
2165			if (!is_zero_ether_addr(f->eth_addr))
2166				vxlan_fdb_destroy(vxlan, f);
2167		}
2168	}
2169	spin_unlock_bh(&vxlan->hash_lock);
2170}
2171
2172/* Cleanup timer and forwarding table on shutdown */
2173static int vxlan_stop(struct net_device *dev)
2174{
2175	struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id);
2176	struct vxlan_dev *vxlan = netdev_priv(dev);
2177	struct vxlan_sock *vs = vxlan->vn_sock;
2178
2179	if (vs && vxlan_addr_multicast(&vxlan->default_dst.remote_ip) &&
2180	    !vxlan_group_used(vn, vxlan)) {
2181		vxlan_sock_hold(vs);
2182		dev_hold(dev);
2183		queue_work(vxlan_wq, &vxlan->igmp_leave);
2184	}
2185
2186	del_timer_sync(&vxlan->age_timer);
2187
2188	vxlan_flush(vxlan);
2189
2190	return 0;
2191}
2192
2193/* Stub, nothing needs to be done. */
2194static void vxlan_set_multicast_list(struct net_device *dev)
2195{
2196}
2197
2198static int vxlan_change_mtu(struct net_device *dev, int new_mtu)
2199{
2200	struct vxlan_dev *vxlan = netdev_priv(dev);
2201	struct vxlan_rdst *dst = &vxlan->default_dst;
2202	struct net_device *lowerdev;
2203	int max_mtu;
2204
2205	lowerdev = __dev_get_by_index(dev_net(dev), dst->remote_ifindex);
2206	if (lowerdev == NULL)
2207		return eth_change_mtu(dev, new_mtu);
2208
2209	if (dst->remote_ip.sa.sa_family == AF_INET6)
2210		max_mtu = lowerdev->mtu - VXLAN6_HEADROOM;
2211	else
2212		max_mtu = lowerdev->mtu - VXLAN_HEADROOM;
2213
2214	if (new_mtu < 68 || new_mtu > max_mtu)
2215		return -EINVAL;
2216
2217	dev->mtu = new_mtu;
2218	return 0;
2219}
2220
2221static const struct net_device_ops vxlan_netdev_ops = {
2222	.ndo_init		= vxlan_init,
2223	.ndo_uninit		= vxlan_uninit,
2224	.ndo_open		= vxlan_open,
2225	.ndo_stop		= vxlan_stop,
2226	.ndo_start_xmit		= vxlan_xmit,
2227	.ndo_get_stats64	= ip_tunnel_get_stats64,
2228	.ndo_set_rx_mode	= vxlan_set_multicast_list,
2229	.ndo_change_mtu		= vxlan_change_mtu,
2230	.ndo_validate_addr	= eth_validate_addr,
2231	.ndo_set_mac_address	= eth_mac_addr,
2232	.ndo_fdb_add		= vxlan_fdb_add,
2233	.ndo_fdb_del		= vxlan_fdb_delete,
2234	.ndo_fdb_dump		= vxlan_fdb_dump,
2235};
2236
2237/* Info for udev, that this is a virtual tunnel endpoint */
2238static struct device_type vxlan_type = {
2239	.name = "vxlan",
2240};
2241
2242/* Calls the ndo_add_vxlan_port of the caller in order to
2243 * supply the listening VXLAN udp ports. Callers are expected
2244 * to implement the ndo_add_vxlan_port.
2245 */
2246void vxlan_get_rx_port(struct net_device *dev)
2247{
2248	struct vxlan_sock *vs;
2249	struct net *net = dev_net(dev);
2250	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
2251	sa_family_t sa_family;
2252	__be16 port;
2253	unsigned int i;
2254
2255	spin_lock(&vn->sock_lock);
2256	for (i = 0; i < PORT_HASH_SIZE; ++i) {
2257		hlist_for_each_entry_rcu(vs, &vn->sock_list[i], hlist) {
2258			port = inet_sk(vs->sock->sk)->inet_sport;
2259			sa_family = vs->sock->sk->sk_family;
2260			dev->netdev_ops->ndo_add_vxlan_port(dev, sa_family,
2261							    port);
2262		}
2263	}
2264	spin_unlock(&vn->sock_lock);
2265}
2266EXPORT_SYMBOL_GPL(vxlan_get_rx_port);
2267
2268/* Initialize the device structure. */
2269static void vxlan_setup(struct net_device *dev)
2270{
2271	struct vxlan_dev *vxlan = netdev_priv(dev);
2272	unsigned int h;
2273	int low, high;
2274
2275	eth_hw_addr_random(dev);
2276	ether_setup(dev);
2277	if (vxlan->default_dst.remote_ip.sa.sa_family == AF_INET6)
2278		dev->hard_header_len = ETH_HLEN + VXLAN6_HEADROOM;
2279	else
2280		dev->hard_header_len = ETH_HLEN + VXLAN_HEADROOM;
2281
2282	dev->netdev_ops = &vxlan_netdev_ops;
2283	dev->destructor = free_netdev;
2284	SET_NETDEV_DEVTYPE(dev, &vxlan_type);
2285
2286	dev->tx_queue_len = 0;
2287	dev->features	|= NETIF_F_LLTX;
2288	dev->features	|= NETIF_F_NETNS_LOCAL;
2289	dev->features	|= NETIF_F_SG | NETIF_F_HW_CSUM;
2290	dev->features   |= NETIF_F_RXCSUM;
2291	dev->features   |= NETIF_F_GSO_SOFTWARE;
2292
2293	dev->vlan_features = dev->features;
2294	dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX;
2295	dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
2296	dev->hw_features |= NETIF_F_GSO_SOFTWARE;
2297	dev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX;
2298	dev->priv_flags	&= ~IFF_XMIT_DST_RELEASE;
2299	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
2300
2301	INIT_LIST_HEAD(&vxlan->next);
2302	spin_lock_init(&vxlan->hash_lock);
2303	INIT_WORK(&vxlan->igmp_join, vxlan_igmp_join);
2304	INIT_WORK(&vxlan->igmp_leave, vxlan_igmp_leave);
2305	INIT_WORK(&vxlan->sock_work, vxlan_sock_work);
2306
2307	init_timer_deferrable(&vxlan->age_timer);
2308	vxlan->age_timer.function = vxlan_cleanup;
2309	vxlan->age_timer.data = (unsigned long) vxlan;
2310
2311	inet_get_local_port_range(dev_net(dev), &low, &high);
2312	vxlan->port_min = low;
2313	vxlan->port_max = high;
2314	vxlan->dst_port = htons(vxlan_port);
2315
2316	vxlan->dev = dev;
2317
2318	for (h = 0; h < FDB_HASH_SIZE; ++h)
2319		INIT_HLIST_HEAD(&vxlan->fdb_head[h]);
2320}
2321
2322static const struct nla_policy vxlan_policy[IFLA_VXLAN_MAX + 1] = {
2323	[IFLA_VXLAN_ID]		= { .type = NLA_U32 },
2324	[IFLA_VXLAN_GROUP]	= { .len = FIELD_SIZEOF(struct iphdr, daddr) },
2325	[IFLA_VXLAN_GROUP6]	= { .len = sizeof(struct in6_addr) },
2326	[IFLA_VXLAN_LINK]	= { .type = NLA_U32 },
2327	[IFLA_VXLAN_LOCAL]	= { .len = FIELD_SIZEOF(struct iphdr, saddr) },
2328	[IFLA_VXLAN_LOCAL6]	= { .len = sizeof(struct in6_addr) },
2329	[IFLA_VXLAN_TOS]	= { .type = NLA_U8 },
2330	[IFLA_VXLAN_TTL]	= { .type = NLA_U8 },
2331	[IFLA_VXLAN_LEARNING]	= { .type = NLA_U8 },
2332	[IFLA_VXLAN_AGEING]	= { .type = NLA_U32 },
2333	[IFLA_VXLAN_LIMIT]	= { .type = NLA_U32 },
2334	[IFLA_VXLAN_PORT_RANGE] = { .len  = sizeof(struct ifla_vxlan_port_range) },
2335	[IFLA_VXLAN_PROXY]	= { .type = NLA_U8 },
2336	[IFLA_VXLAN_RSC]	= { .type = NLA_U8 },
2337	[IFLA_VXLAN_L2MISS]	= { .type = NLA_U8 },
2338	[IFLA_VXLAN_L3MISS]	= { .type = NLA_U8 },
2339	[IFLA_VXLAN_PORT]	= { .type = NLA_U16 },
2340};
2341
2342static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[])
2343{
2344	if (tb[IFLA_ADDRESS]) {
2345		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
2346			pr_debug("invalid link address (not ethernet)\n");
2347			return -EINVAL;
2348		}
2349
2350		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
2351			pr_debug("invalid all zero ethernet address\n");
2352			return -EADDRNOTAVAIL;
2353		}
2354	}
2355
2356	if (!data)
2357		return -EINVAL;
2358
2359	if (data[IFLA_VXLAN_ID]) {
2360		__u32 id = nla_get_u32(data[IFLA_VXLAN_ID]);
2361		if (id >= VXLAN_VID_MASK)
2362			return -ERANGE;
2363	}
2364
2365	if (data[IFLA_VXLAN_PORT_RANGE]) {
2366		const struct ifla_vxlan_port_range *p
2367			= nla_data(data[IFLA_VXLAN_PORT_RANGE]);
2368
2369		if (ntohs(p->high) < ntohs(p->low)) {
2370			pr_debug("port range %u .. %u not valid\n",
2371				 ntohs(p->low), ntohs(p->high));
2372			return -EINVAL;
2373		}
2374	}
2375
2376	return 0;
2377}
2378
2379static void vxlan_get_drvinfo(struct net_device *netdev,
2380			      struct ethtool_drvinfo *drvinfo)
2381{
2382	strlcpy(drvinfo->version, VXLAN_VERSION, sizeof(drvinfo->version));
2383	strlcpy(drvinfo->driver, "vxlan", sizeof(drvinfo->driver));
2384}
2385
2386static const struct ethtool_ops vxlan_ethtool_ops = {
2387	.get_drvinfo	= vxlan_get_drvinfo,
2388	.get_link	= ethtool_op_get_link,
2389};
2390
2391static void vxlan_del_work(struct work_struct *work)
2392{
2393	struct vxlan_sock *vs = container_of(work, struct vxlan_sock, del_work);
2394
2395	sk_release_kernel(vs->sock->sk);
2396	kfree_rcu(vs, rcu);
2397}
2398
2399#if IS_ENABLED(CONFIG_IPV6)
2400/* Create UDP socket for encapsulation receive. AF_INET6 socket
2401 * could be used for both IPv4 and IPv6 communications, but
2402 * users may set bindv6only=1.
2403 */
2404static struct socket *create_v6_sock(struct net *net, __be16 port)
2405{
2406	struct sock *sk;
2407	struct socket *sock;
2408	struct sockaddr_in6 vxlan_addr = {
2409		.sin6_family = AF_INET6,
2410		.sin6_port = port,
2411	};
2412	int rc, val = 1;
2413
2414	rc = sock_create_kern(AF_INET6, SOCK_DGRAM, IPPROTO_UDP, &sock);
2415	if (rc < 0) {
2416		pr_debug("UDPv6 socket create failed\n");
2417		return ERR_PTR(rc);
2418	}
2419
2420	/* Put in proper namespace */
2421	sk = sock->sk;
2422	sk_change_net(sk, net);
2423
2424	kernel_setsockopt(sock, SOL_IPV6, IPV6_V6ONLY,
2425			  (char *)&val, sizeof(val));
2426	rc = kernel_bind(sock, (struct sockaddr *)&vxlan_addr,
2427			 sizeof(struct sockaddr_in6));
2428	if (rc < 0) {
2429		pr_debug("bind for UDPv6 socket %pI6:%u (%d)\n",
2430			 &vxlan_addr.sin6_addr, ntohs(vxlan_addr.sin6_port), rc);
2431		sk_release_kernel(sk);
2432		return ERR_PTR(rc);
2433	}
2434	/* At this point, IPv6 module should have been loaded in
2435	 * sock_create_kern().
2436	 */
2437	BUG_ON(!ipv6_stub);
2438
2439	/* Disable multicast loopback */
2440	inet_sk(sk)->mc_loop = 0;
2441	return sock;
2442}
2443
2444#else
2445
2446static struct socket *create_v6_sock(struct net *net, __be16 port)
2447{
2448		return ERR_PTR(-EPFNOSUPPORT);
2449}
2450#endif
2451
2452static struct socket *create_v4_sock(struct net *net, __be16 port)
2453{
2454	struct sock *sk;
2455	struct socket *sock;
2456	struct sockaddr_in vxlan_addr = {
2457		.sin_family = AF_INET,
2458		.sin_addr.s_addr = htonl(INADDR_ANY),
2459		.sin_port = port,
2460	};
2461	int rc;
2462
2463	/* Create UDP socket for encapsulation receive. */
2464	rc = sock_create_kern(AF_INET, SOCK_DGRAM, IPPROTO_UDP, &sock);
2465	if (rc < 0) {
2466		pr_debug("UDP socket create failed\n");
2467		return ERR_PTR(rc);
2468	}
2469
2470	/* Put in proper namespace */
2471	sk = sock->sk;
2472	sk_change_net(sk, net);
2473
2474	rc = kernel_bind(sock, (struct sockaddr *) &vxlan_addr,
2475			 sizeof(vxlan_addr));
2476	if (rc < 0) {
2477		pr_debug("bind for UDP socket %pI4:%u (%d)\n",
2478			 &vxlan_addr.sin_addr, ntohs(vxlan_addr.sin_port), rc);
2479		sk_release_kernel(sk);
2480		return ERR_PTR(rc);
2481	}
2482
2483	/* Disable multicast loopback */
2484	inet_sk(sk)->mc_loop = 0;
2485	return sock;
2486}
2487
2488/* Create new listen socket if needed */
2489static struct vxlan_sock *vxlan_socket_create(struct net *net, __be16 port,
2490					      vxlan_rcv_t *rcv, void *data, bool ipv6)
2491{
2492	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
2493	struct vxlan_sock *vs;
2494	struct socket *sock;
2495	struct sock *sk;
2496	unsigned int h;
2497
2498	vs = kzalloc(sizeof(*vs), GFP_KERNEL);
2499	if (!vs)
2500		return ERR_PTR(-ENOMEM);
2501
2502	for (h = 0; h < VNI_HASH_SIZE; ++h)
2503		INIT_HLIST_HEAD(&vs->vni_list[h]);
2504
2505	INIT_WORK(&vs->del_work, vxlan_del_work);
2506
2507	if (ipv6)
2508		sock = create_v6_sock(net, port);
2509	else
2510		sock = create_v4_sock(net, port);
2511	if (IS_ERR(sock)) {
2512		kfree(vs);
2513		return ERR_CAST(sock);
2514	}
2515
2516	vs->sock = sock;
2517	sk = sock->sk;
2518	atomic_set(&vs->refcnt, 1);
2519	vs->rcv = rcv;
2520	vs->data = data;
2521	rcu_assign_sk_user_data(vs->sock->sk, vs);
2522
2523	/* Initialize the vxlan udp offloads structure */
2524	vs->udp_offloads.port = port;
2525	vs->udp_offloads.callbacks.gro_receive  = vxlan_gro_receive;
2526	vs->udp_offloads.callbacks.gro_complete = vxlan_gro_complete;
2527
2528	spin_lock(&vn->sock_lock);
2529	hlist_add_head_rcu(&vs->hlist, vs_head(net, port));
2530	vxlan_notify_add_rx_port(vs);
2531	spin_unlock(&vn->sock_lock);
2532
2533	/* Mark socket as an encapsulation socket. */
2534	udp_sk(sk)->encap_type = 1;
2535	udp_sk(sk)->encap_rcv = vxlan_udp_encap_recv;
2536#if IS_ENABLED(CONFIG_IPV6)
2537	if (ipv6)
2538		ipv6_stub->udpv6_encap_enable();
2539	else
2540#endif
2541		udp_encap_enable();
2542
2543	return vs;
2544}
2545
2546struct vxlan_sock *vxlan_sock_add(struct net *net, __be16 port,
2547				  vxlan_rcv_t *rcv, void *data,
2548				  bool no_share, bool ipv6)
2549{
2550	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
2551	struct vxlan_sock *vs;
2552
2553	vs = vxlan_socket_create(net, port, rcv, data, ipv6);
2554	if (!IS_ERR(vs))
2555		return vs;
2556
2557	if (no_share)	/* Return error if sharing is not allowed. */
2558		return vs;
2559
2560	spin_lock(&vn->sock_lock);
2561	vs = vxlan_find_sock(net, port);
2562	if (vs) {
2563		if (vs->rcv == rcv)
2564			atomic_inc(&vs->refcnt);
2565		else
2566			vs = ERR_PTR(-EBUSY);
2567	}
2568	spin_unlock(&vn->sock_lock);
2569
2570	if (!vs)
2571		vs = ERR_PTR(-EINVAL);
2572
2573	return vs;
2574}
2575EXPORT_SYMBOL_GPL(vxlan_sock_add);
2576
2577/* Scheduled at device creation to bind to a socket */
2578static void vxlan_sock_work(struct work_struct *work)
2579{
2580	struct vxlan_dev *vxlan = container_of(work, struct vxlan_dev, sock_work);
2581	struct net *net = dev_net(vxlan->dev);
2582	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
2583	__be16 port = vxlan->dst_port;
2584	struct vxlan_sock *nvs;
2585
2586	nvs = vxlan_sock_add(net, port, vxlan_rcv, NULL, false, vxlan->flags & VXLAN_F_IPV6);
2587	spin_lock(&vn->sock_lock);
2588	if (!IS_ERR(nvs))
2589		vxlan_vs_add_dev(nvs, vxlan);
2590	spin_unlock(&vn->sock_lock);
2591
2592	dev_put(vxlan->dev);
2593}
2594
2595static int vxlan_newlink(struct net *net, struct net_device *dev,
2596			 struct nlattr *tb[], struct nlattr *data[])
2597{
2598	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
2599	struct vxlan_dev *vxlan = netdev_priv(dev);
2600	struct vxlan_rdst *dst = &vxlan->default_dst;
2601	__u32 vni;
2602	int err;
2603	bool use_ipv6 = false;
2604
2605	if (!data[IFLA_VXLAN_ID])
2606		return -EINVAL;
2607
2608	vni = nla_get_u32(data[IFLA_VXLAN_ID]);
2609	dst->remote_vni = vni;
2610
2611	/* Unless IPv6 is explicitly requested, assume IPv4 */
2612	dst->remote_ip.sa.sa_family = AF_INET;
2613	if (data[IFLA_VXLAN_GROUP]) {
2614		dst->remote_ip.sin.sin_addr.s_addr = nla_get_be32(data[IFLA_VXLAN_GROUP]);
2615	} else if (data[IFLA_VXLAN_GROUP6]) {
2616		if (!IS_ENABLED(CONFIG_IPV6))
2617			return -EPFNOSUPPORT;
2618
2619		nla_memcpy(&dst->remote_ip.sin6.sin6_addr, data[IFLA_VXLAN_GROUP6],
2620			   sizeof(struct in6_addr));
2621		dst->remote_ip.sa.sa_family = AF_INET6;
2622		use_ipv6 = true;
2623	}
2624
2625	if (data[IFLA_VXLAN_LOCAL]) {
2626		vxlan->saddr.sin.sin_addr.s_addr = nla_get_be32(data[IFLA_VXLAN_LOCAL]);
2627		vxlan->saddr.sa.sa_family = AF_INET;
2628	} else if (data[IFLA_VXLAN_LOCAL6]) {
2629		if (!IS_ENABLED(CONFIG_IPV6))
2630			return -EPFNOSUPPORT;
2631
2632		/* TODO: respect scope id */
2633		nla_memcpy(&vxlan->saddr.sin6.sin6_addr, data[IFLA_VXLAN_LOCAL6],
2634			   sizeof(struct in6_addr));
2635		vxlan->saddr.sa.sa_family = AF_INET6;
2636		use_ipv6 = true;
2637	}
2638
2639	if (data[IFLA_VXLAN_LINK] &&
2640	    (dst->remote_ifindex = nla_get_u32(data[IFLA_VXLAN_LINK]))) {
2641		struct net_device *lowerdev
2642			 = __dev_get_by_index(net, dst->remote_ifindex);
2643
2644		if (!lowerdev) {
2645			pr_info("ifindex %d does not exist\n", dst->remote_ifindex);
2646			return -ENODEV;
2647		}
2648
2649#if IS_ENABLED(CONFIG_IPV6)
2650		if (use_ipv6) {
2651			struct inet6_dev *idev = __in6_dev_get(lowerdev);
2652			if (idev && idev->cnf.disable_ipv6) {
2653				pr_info("IPv6 is disabled via sysctl\n");
2654				return -EPERM;
2655			}
2656			vxlan->flags |= VXLAN_F_IPV6;
2657		}
2658#endif
2659
2660		if (!tb[IFLA_MTU])
2661			dev->mtu = lowerdev->mtu - (use_ipv6 ? VXLAN6_HEADROOM : VXLAN_HEADROOM);
2662
2663		/* update header length based on lower device */
2664		dev->hard_header_len = lowerdev->hard_header_len +
2665				       (use_ipv6 ? VXLAN6_HEADROOM : VXLAN_HEADROOM);
2666	} else if (use_ipv6)
2667		vxlan->flags |= VXLAN_F_IPV6;
2668
2669	if (data[IFLA_VXLAN_TOS])
2670		vxlan->tos  = nla_get_u8(data[IFLA_VXLAN_TOS]);
2671
2672	if (data[IFLA_VXLAN_TTL])
2673		vxlan->ttl = nla_get_u8(data[IFLA_VXLAN_TTL]);
2674
2675	if (!data[IFLA_VXLAN_LEARNING] || nla_get_u8(data[IFLA_VXLAN_LEARNING]))
2676		vxlan->flags |= VXLAN_F_LEARN;
2677
2678	if (data[IFLA_VXLAN_AGEING])
2679		vxlan->age_interval = nla_get_u32(data[IFLA_VXLAN_AGEING]);
2680	else
2681		vxlan->age_interval = FDB_AGE_DEFAULT;
2682
2683	if (data[IFLA_VXLAN_PROXY] && nla_get_u8(data[IFLA_VXLAN_PROXY]))
2684		vxlan->flags |= VXLAN_F_PROXY;
2685
2686	if (data[IFLA_VXLAN_RSC] && nla_get_u8(data[IFLA_VXLAN_RSC]))
2687		vxlan->flags |= VXLAN_F_RSC;
2688
2689	if (data[IFLA_VXLAN_L2MISS] && nla_get_u8(data[IFLA_VXLAN_L2MISS]))
2690		vxlan->flags |= VXLAN_F_L2MISS;
2691
2692	if (data[IFLA_VXLAN_L3MISS] && nla_get_u8(data[IFLA_VXLAN_L3MISS]))
2693		vxlan->flags |= VXLAN_F_L3MISS;
2694
2695	if (data[IFLA_VXLAN_LIMIT])
2696		vxlan->addrmax = nla_get_u32(data[IFLA_VXLAN_LIMIT]);
2697
2698	if (data[IFLA_VXLAN_PORT_RANGE]) {
2699		const struct ifla_vxlan_port_range *p
2700			= nla_data(data[IFLA_VXLAN_PORT_RANGE]);
2701		vxlan->port_min = ntohs(p->low);
2702		vxlan->port_max = ntohs(p->high);
2703	}
2704
2705	if (data[IFLA_VXLAN_PORT])
2706		vxlan->dst_port = nla_get_be16(data[IFLA_VXLAN_PORT]);
2707
2708	if (vxlan_find_vni(net, vni, vxlan->dst_port)) {
2709		pr_info("duplicate VNI %u\n", vni);
2710		return -EEXIST;
2711	}
2712
2713	SET_ETHTOOL_OPS(dev, &vxlan_ethtool_ops);
2714
2715	/* create an fdb entry for a valid default destination */
2716	if (!vxlan_addr_any(&vxlan->default_dst.remote_ip)) {
2717		err = vxlan_fdb_create(vxlan, all_zeros_mac,
2718				       &vxlan->default_dst.remote_ip,
2719				       NUD_REACHABLE|NUD_PERMANENT,
2720				       NLM_F_EXCL|NLM_F_CREATE,
2721				       vxlan->dst_port,
2722				       vxlan->default_dst.remote_vni,
2723				       vxlan->default_dst.remote_ifindex,
2724				       NTF_SELF);
2725		if (err)
2726			return err;
2727	}
2728
2729	err = register_netdevice(dev);
2730	if (err) {
2731		vxlan_fdb_delete_default(vxlan);
2732		return err;
2733	}
2734
2735	list_add(&vxlan->next, &vn->vxlan_list);
2736
2737	return 0;
2738}
2739
2740static void vxlan_dellink(struct net_device *dev, struct list_head *head)
2741{
2742	struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id);
2743	struct vxlan_dev *vxlan = netdev_priv(dev);
2744
2745	spin_lock(&vn->sock_lock);
2746	if (!hlist_unhashed(&vxlan->hlist))
2747		hlist_del_rcu(&vxlan->hlist);
2748	spin_unlock(&vn->sock_lock);
2749
2750	list_del(&vxlan->next);
2751	unregister_netdevice_queue(dev, head);
2752}
2753
2754static size_t vxlan_get_size(const struct net_device *dev)
2755{
2756
2757	return nla_total_size(sizeof(__u32)) +	/* IFLA_VXLAN_ID */
2758		nla_total_size(sizeof(struct in6_addr)) + /* IFLA_VXLAN_GROUP{6} */
2759		nla_total_size(sizeof(__u32)) +	/* IFLA_VXLAN_LINK */
2760		nla_total_size(sizeof(struct in6_addr)) + /* IFLA_VXLAN_LOCAL{6} */
2761		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_TTL */
2762		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_TOS */
2763		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_LEARNING */
2764		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_PROXY */
2765		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_RSC */
2766		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_L2MISS */
2767		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_L3MISS */
2768		nla_total_size(sizeof(__u32)) +	/* IFLA_VXLAN_AGEING */
2769		nla_total_size(sizeof(__u32)) +	/* IFLA_VXLAN_LIMIT */
2770		nla_total_size(sizeof(struct ifla_vxlan_port_range)) +
2771		nla_total_size(sizeof(__be16))+ /* IFLA_VXLAN_PORT */
2772		0;
2773}
2774
2775static int vxlan_fill_info(struct sk_buff *skb, const struct net_device *dev)
2776{
2777	const struct vxlan_dev *vxlan = netdev_priv(dev);
2778	const struct vxlan_rdst *dst = &vxlan->default_dst;
2779	struct ifla_vxlan_port_range ports = {
2780		.low =  htons(vxlan->port_min),
2781		.high = htons(vxlan->port_max),
2782	};
2783
2784	if (nla_put_u32(skb, IFLA_VXLAN_ID, dst->remote_vni))
2785		goto nla_put_failure;
2786
2787	if (!vxlan_addr_any(&dst->remote_ip)) {
2788		if (dst->remote_ip.sa.sa_family == AF_INET) {
2789			if (nla_put_be32(skb, IFLA_VXLAN_GROUP,
2790					 dst->remote_ip.sin.sin_addr.s_addr))
2791				goto nla_put_failure;
2792#if IS_ENABLED(CONFIG_IPV6)
2793		} else {
2794			if (nla_put(skb, IFLA_VXLAN_GROUP6, sizeof(struct in6_addr),
2795				    &dst->remote_ip.sin6.sin6_addr))
2796				goto nla_put_failure;
2797#endif
2798		}
2799	}
2800
2801	if (dst->remote_ifindex && nla_put_u32(skb, IFLA_VXLAN_LINK, dst->remote_ifindex))
2802		goto nla_put_failure;
2803
2804	if (!vxlan_addr_any(&vxlan->saddr)) {
2805		if (vxlan->saddr.sa.sa_family == AF_INET) {
2806			if (nla_put_be32(skb, IFLA_VXLAN_LOCAL,
2807					 vxlan->saddr.sin.sin_addr.s_addr))
2808				goto nla_put_failure;
2809#if IS_ENABLED(CONFIG_IPV6)
2810		} else {
2811			if (nla_put(skb, IFLA_VXLAN_LOCAL6, sizeof(struct in6_addr),
2812				    &vxlan->saddr.sin6.sin6_addr))
2813				goto nla_put_failure;
2814#endif
2815		}
2816	}
2817
2818	if (nla_put_u8(skb, IFLA_VXLAN_TTL, vxlan->ttl) ||
2819	    nla_put_u8(skb, IFLA_VXLAN_TOS, vxlan->tos) ||
2820	    nla_put_u8(skb, IFLA_VXLAN_LEARNING,
2821			!!(vxlan->flags & VXLAN_F_LEARN)) ||
2822	    nla_put_u8(skb, IFLA_VXLAN_PROXY,
2823			!!(vxlan->flags & VXLAN_F_PROXY)) ||
2824	    nla_put_u8(skb, IFLA_VXLAN_RSC, !!(vxlan->flags & VXLAN_F_RSC)) ||
2825	    nla_put_u8(skb, IFLA_VXLAN_L2MISS,
2826			!!(vxlan->flags & VXLAN_F_L2MISS)) ||
2827	    nla_put_u8(skb, IFLA_VXLAN_L3MISS,
2828			!!(vxlan->flags & VXLAN_F_L3MISS)) ||
2829	    nla_put_u32(skb, IFLA_VXLAN_AGEING, vxlan->age_interval) ||
2830	    nla_put_u32(skb, IFLA_VXLAN_LIMIT, vxlan->addrmax) ||
2831	    nla_put_be16(skb, IFLA_VXLAN_PORT, vxlan->dst_port))
2832		goto nla_put_failure;
2833
2834	if (nla_put(skb, IFLA_VXLAN_PORT_RANGE, sizeof(ports), &ports))
2835		goto nla_put_failure;
2836
2837	return 0;
2838
2839nla_put_failure:
2840	return -EMSGSIZE;
2841}
2842
2843static struct rtnl_link_ops vxlan_link_ops __read_mostly = {
2844	.kind		= "vxlan",
2845	.maxtype	= IFLA_VXLAN_MAX,
2846	.policy		= vxlan_policy,
2847	.priv_size	= sizeof(struct vxlan_dev),
2848	.setup		= vxlan_setup,
2849	.validate	= vxlan_validate,
2850	.newlink	= vxlan_newlink,
2851	.dellink	= vxlan_dellink,
2852	.get_size	= vxlan_get_size,
2853	.fill_info	= vxlan_fill_info,
2854};
2855
2856static void vxlan_handle_lowerdev_unregister(struct vxlan_net *vn,
2857					     struct net_device *dev)
2858{
2859	struct vxlan_dev *vxlan, *next;
2860	LIST_HEAD(list_kill);
2861
2862	list_for_each_entry_safe(vxlan, next, &vn->vxlan_list, next) {
2863		struct vxlan_rdst *dst = &vxlan->default_dst;
2864
2865		/* In case we created vxlan device with carrier
2866		 * and we loose the carrier due to module unload
2867		 * we also need to remove vxlan device. In other
2868		 * cases, it's not necessary and remote_ifindex
2869		 * is 0 here, so no matches.
2870		 */
2871		if (dst->remote_ifindex == dev->ifindex)
2872			vxlan_dellink(vxlan->dev, &list_kill);
2873	}
2874
2875	unregister_netdevice_many(&list_kill);
2876}
2877
2878static int vxlan_lowerdev_event(struct notifier_block *unused,
2879				unsigned long event, void *ptr)
2880{
2881	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2882	struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id);
2883
2884	if (event == NETDEV_UNREGISTER)
2885		vxlan_handle_lowerdev_unregister(vn, dev);
2886
2887	return NOTIFY_DONE;
2888}
2889
2890static struct notifier_block vxlan_notifier_block __read_mostly = {
2891	.notifier_call = vxlan_lowerdev_event,
2892};
2893
2894static __net_init int vxlan_init_net(struct net *net)
2895{
2896	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
2897	unsigned int h;
2898
2899	INIT_LIST_HEAD(&vn->vxlan_list);
2900	spin_lock_init(&vn->sock_lock);
2901
2902	for (h = 0; h < PORT_HASH_SIZE; ++h)
2903		INIT_HLIST_HEAD(&vn->sock_list[h]);
2904
2905	return 0;
2906}
2907
2908static struct pernet_operations vxlan_net_ops = {
2909	.init = vxlan_init_net,
2910	.id   = &vxlan_net_id,
2911	.size = sizeof(struct vxlan_net),
2912};
2913
2914static int __init vxlan_init_module(void)
2915{
2916	int rc;
2917
2918	vxlan_wq = alloc_workqueue("vxlan", 0, 0);
2919	if (!vxlan_wq)
2920		return -ENOMEM;
2921
2922	get_random_bytes(&vxlan_salt, sizeof(vxlan_salt));
2923
2924	rc = register_pernet_subsys(&vxlan_net_ops);
2925	if (rc)
2926		goto out1;
2927
2928	rc = register_netdevice_notifier(&vxlan_notifier_block);
2929	if (rc)
2930		goto out2;
2931
2932	rc = rtnl_link_register(&vxlan_link_ops);
2933	if (rc)
2934		goto out3;
2935
2936	return 0;
2937out3:
2938	unregister_netdevice_notifier(&vxlan_notifier_block);
2939out2:
2940	unregister_pernet_subsys(&vxlan_net_ops);
2941out1:
2942	destroy_workqueue(vxlan_wq);
2943	return rc;
2944}
2945late_initcall(vxlan_init_module);
2946
2947static void __exit vxlan_cleanup_module(void)
2948{
2949	rtnl_link_unregister(&vxlan_link_ops);
2950	unregister_netdevice_notifier(&vxlan_notifier_block);
2951	destroy_workqueue(vxlan_wq);
2952	unregister_pernet_subsys(&vxlan_net_ops);
2953	/* rcu_barrier() is called by netns */
2954}
2955module_exit(vxlan_cleanup_module);
2956
2957MODULE_LICENSE("GPL");
2958MODULE_VERSION(VXLAN_VERSION);
2959MODULE_AUTHOR("Stephen Hemminger <stephen@networkplumber.org>");
2960MODULE_DESCRIPTION("Driver for VXLAN encapsulated traffic");
2961MODULE_ALIAS_RTNL_LINK("vxlan");