1/*
   2 * Copyright (c) 2007-2014 Nicira, Inc.
   3 *
   4 * This program is free software; you can redistribute it and/or
   5 * modify it under the terms of version 2 of the GNU General Public
   6 * License as published by the Free Software Foundation.
   7 *
   8 * This program is distributed in the hope that it will be useful, but
   9 * WITHOUT ANY WARRANTY; without even the implied warranty of
  10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11 * General Public License for more details.
  12 *
  13 * You should have received a copy of the GNU General Public License
  14 * along with this program; if not, write to the Free Software
  15 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16 * 02110-1301, USA
  17 */
  18
  19#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21#include <linux/init.h>
  22#include <linux/module.h>
  23#include <linux/if_arp.h>
  24#include <linux/if_vlan.h>
  25#include <linux/in.h>
  26#include <linux/ip.h>
  27#include <linux/jhash.h>
  28#include <linux/delay.h>
  29#include <linux/time.h>
  30#include <linux/etherdevice.h>
  31#include <linux/genetlink.h>
  32#include <linux/kernel.h>
  33#include <linux/kthread.h>
  34#include <linux/mutex.h>
  35#include <linux/percpu.h>
  36#include <linux/rcupdate.h>
  37#include <linux/tcp.h>
  38#include <linux/udp.h>
  39#include <linux/ethtool.h>
  40#include <linux/wait.h>
  41#include <asm/div64.h>
  42#include <linux/highmem.h>
  43#include <linux/netfilter_bridge.h>
  44#include <linux/netfilter_ipv4.h>
  45#include <linux/inetdevice.h>
  46#include <linux/list.h>
  47#include <linux/openvswitch.h>
  48#include <linux/rculist.h>
  49#include <linux/dmi.h>
  50#include <net/genetlink.h>
  51#include <net/net_namespace.h>
  52#include <net/netns/generic.h>
  53
  54#include "datapath.h"
  55#include "flow.h"
  56#include "flow_table.h"
  57#include "flow_netlink.h"
  58#include "meter.h"
  59#include "vport-internal_dev.h"
  60#include "vport-netdev.h"
  61
  62unsigned int ovs_net_id __read_mostly;
 
  63
  64static struct genl_family dp_packet_genl_family;
  65static struct genl_family dp_flow_genl_family;
  66static struct genl_family dp_datapath_genl_family;
  67
  68static const struct nla_policy flow_policy[];
  69
  70static const struct genl_multicast_group ovs_dp_flow_multicast_group = {
  71	.name = OVS_FLOW_MCGROUP,
  72};
  73
  74static const struct genl_multicast_group ovs_dp_datapath_multicast_group = {
  75	.name = OVS_DATAPATH_MCGROUP,
  76};
  77
  78static const struct genl_multicast_group ovs_dp_vport_multicast_group = {
  79	.name = OVS_VPORT_MCGROUP,
  80};
  81
  82/* Check if need to build a reply message.
  83 * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */
  84static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
  85			    unsigned int group)
  86{
  87	return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
  88	       genl_has_listeners(family, genl_info_net(info), group);
  89}
  90
  91static void ovs_notify(struct genl_family *family,
  92		       struct sk_buff *skb, struct genl_info *info)
  93{
  94	genl_notify(family, skb, info, 0, GFP_KERNEL);
  95}
  96
  97/**
  98 * DOC: Locking:
  99 *
 100 * All writes e.g. Writes to device state (add/remove datapath, port, set
 101 * operations on vports, etc.), Writes to other state (flow table
 102 * modifications, set miscellaneous datapath parameters, etc.) are protected
 103 * by ovs_lock.
 104 *
 105 * Reads are protected by RCU.
 106 *
 107 * There are a few special cases (mostly stats) that have their own
 108 * synchronization but they nest under all of above and don't interact with
 109 * each other.
 110 *
 111 * The RTNL lock nests inside ovs_mutex.
 112 */
 113
 114static DEFINE_MUTEX(ovs_mutex);
 115
 116void ovs_lock(void)
 117{
 118	mutex_lock(&ovs_mutex);
 119}
 120
 121void ovs_unlock(void)
 122{
 123	mutex_unlock(&ovs_mutex);
 124}
 125
 126#ifdef CONFIG_LOCKDEP
 127int lockdep_ovsl_is_held(void)
 128{
 129	if (debug_locks)
 130		return lockdep_is_held(&ovs_mutex);
 131	else
 132		return 1;
 133}
 
 134#endif
 135
 136static struct vport *new_vport(const struct vport_parms *);
 137static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
 138			     const struct sw_flow_key *,
 139			     const struct dp_upcall_info *,
 140			     uint32_t cutlen);
 141static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
 142				  const struct sw_flow_key *,
 143				  const struct dp_upcall_info *,
 144				  uint32_t cutlen);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 145
 146/* Must be called with rcu_read_lock or ovs_mutex. */
 147const char *ovs_dp_name(const struct datapath *dp)
 148{
 149	struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
 150	return ovs_vport_name(vport);
 151}
 152
 153static int get_dpifindex(const struct datapath *dp)
 154{
 155	struct vport *local;
 156	int ifindex;
 157
 158	rcu_read_lock();
 159
 160	local = ovs_vport_rcu(dp, OVSP_LOCAL);
 161	if (local)
 162		ifindex = local->dev->ifindex;
 163	else
 164		ifindex = 0;
 165
 166	rcu_read_unlock();
 167
 168	return ifindex;
 169}
 170
 171static void destroy_dp_rcu(struct rcu_head *rcu)
 172{
 173	struct datapath *dp = container_of(rcu, struct datapath, rcu);
 174
 175	ovs_flow_tbl_destroy(&dp->table);
 176	free_percpu(dp->stats_percpu);
 177	kfree(dp->ports);
 178	ovs_meters_exit(dp);
 179	kfree(dp);
 180}
 181
 182static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
 183					    u16 port_no)
 184{
 185	return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
 186}
 187
 188/* Called with ovs_mutex or RCU read lock. */
 189struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
 190{
 191	struct vport *vport;
 192	struct hlist_head *head;
 193
 194	head = vport_hash_bucket(dp, port_no);
 195	hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
 196		if (vport->port_no == port_no)
 197			return vport;
 198	}
 199	return NULL;
 200}
 201
 202/* Called with ovs_mutex. */
 203static struct vport *new_vport(const struct vport_parms *parms)
 204{
 205	struct vport *vport;
 206
 207	vport = ovs_vport_add(parms);
 208	if (!IS_ERR(vport)) {
 209		struct datapath *dp = parms->dp;
 210		struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
 211
 212		hlist_add_head_rcu(&vport->dp_hash_node, head);
 213	}
 214	return vport;
 215}
 216
 217void ovs_dp_detach_port(struct vport *p)
 218{
 219	ASSERT_OVSL();
 220
 221	/* First drop references to device. */
 222	hlist_del_rcu(&p->dp_hash_node);
 223
 224	/* Then destroy it. */
 225	ovs_vport_del(p);
 226}
 227
 228/* Must be called with rcu_read_lock. */
 229void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
 230{
 231	const struct vport *p = OVS_CB(skb)->input_vport;
 232	struct datapath *dp = p->dp;
 233	struct sw_flow *flow;
 234	struct sw_flow_actions *sf_acts;
 235	struct dp_stats_percpu *stats;
 236	u64 *stats_counter;
 237	u32 n_mask_hit;
 238
 239	stats = this_cpu_ptr(dp->stats_percpu);
 240
 241	/* Look up flow. */
 242	flow = ovs_flow_tbl_lookup_stats(&dp->table, key, &n_mask_hit);
 243	if (unlikely(!flow)) {
 244		struct dp_upcall_info upcall;
 245		int error;
 246
 247		memset(&upcall, 0, sizeof(upcall));
 248		upcall.cmd = OVS_PACKET_CMD_MISS;
 249		upcall.portid = ovs_vport_find_upcall_portid(p, skb);
 250		upcall.mru = OVS_CB(skb)->mru;
 251		error = ovs_dp_upcall(dp, skb, key, &upcall, 0);
 252		if (unlikely(error))
 253			kfree_skb(skb);
 254		else
 255			consume_skb(skb);
 256		stats_counter = &stats->n_missed;
 257		goto out;
 258	}
 259
 260	ovs_flow_stats_update(flow, key->tp.flags, skb);
 261	sf_acts = rcu_dereference(flow->sf_acts);
 262	ovs_execute_actions(dp, skb, sf_acts, key);
 263
 264	stats_counter = &stats->n_hit;
 265
 266out:
 267	/* Update datapath statistics. */
 268	u64_stats_update_begin(&stats->syncp);
 269	(*stats_counter)++;
 270	stats->n_mask_hit += n_mask_hit;
 271	u64_stats_update_end(&stats->syncp);
 272}
 273
 274int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
 275		  const struct sw_flow_key *key,
 276		  const struct dp_upcall_info *upcall_info,
 277		  uint32_t cutlen)
 278{
 279	struct dp_stats_percpu *stats;
 280	int err;
 281
 282	if (upcall_info->portid == 0) {
 283		err = -ENOTCONN;
 284		goto err;
 285	}
 286
 287	if (!skb_is_gso(skb))
 288		err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
 289	else
 290		err = queue_gso_packets(dp, skb, key, upcall_info, cutlen);
 291	if (err)
 292		goto err;
 293
 294	return 0;
 295
 296err:
 297	stats = this_cpu_ptr(dp->stats_percpu);
 298
 299	u64_stats_update_begin(&stats->syncp);
 300	stats->n_lost++;
 301	u64_stats_update_end(&stats->syncp);
 302
 303	return err;
 304}
 305
 306static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
 307			     const struct sw_flow_key *key,
 308			     const struct dp_upcall_info *upcall_info,
 309				 uint32_t cutlen)
 310{
 311	unsigned int gso_type = skb_shinfo(skb)->gso_type;
 312	struct sw_flow_key later_key;
 313	struct sk_buff *segs, *nskb;
 314	int err;
 315
 316	BUILD_BUG_ON(sizeof(*OVS_CB(skb)) > SKB_SGO_CB_OFFSET);
 317	segs = __skb_gso_segment(skb, NETIF_F_SG, false);
 318	if (IS_ERR(segs))
 319		return PTR_ERR(segs);
 320	if (segs == NULL)
 321		return -EINVAL;
 322
 323	if (gso_type & SKB_GSO_UDP) {
 324		/* The initial flow key extracted by ovs_flow_key_extract()
 325		 * in this case is for a first fragment, so we need to
 326		 * properly mark later fragments.
 327		 */
 328		later_key = *key;
 329		later_key.ip.frag = OVS_FRAG_TYPE_LATER;
 330	}
 331
 332	/* Queue all of the segments. */
 333	skb = segs;
 334	do {
 335		if (gso_type & SKB_GSO_UDP && skb != segs)
 336			key = &later_key;
 337
 338		err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
 339		if (err)
 340			break;
 341
 342	} while ((skb = skb->next));
 343
 344	/* Free all of the segments. */
 345	skb = segs;
 346	do {
 347		nskb = skb->next;
 348		if (err)
 349			kfree_skb(skb);
 350		else
 351			consume_skb(skb);
 352	} while ((skb = nskb));
 353	return err;
 354}
 355
 356static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
 357			      unsigned int hdrlen, int actions_attrlen)
 358{
 359	size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
 360		+ nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
 361		+ nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */
 362		+ nla_total_size(sizeof(unsigned int)); /* OVS_PACKET_ATTR_LEN */
 363
 364	/* OVS_PACKET_ATTR_USERDATA */
 365	if (upcall_info->userdata)
 366		size += NLA_ALIGN(upcall_info->userdata->nla_len);
 367
 368	/* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
 369	if (upcall_info->egress_tun_info)
 370		size += nla_total_size(ovs_tun_key_attr_size());
 371
 372	/* OVS_PACKET_ATTR_ACTIONS */
 373	if (upcall_info->actions_len)
 374		size += nla_total_size(actions_attrlen);
 375
 376	/* OVS_PACKET_ATTR_MRU */
 377	if (upcall_info->mru)
 378		size += nla_total_size(sizeof(upcall_info->mru));
 379
 380	return size;
 381}
 382
 383static void pad_packet(struct datapath *dp, struct sk_buff *skb)
 384{
 385	if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
 386		size_t plen = NLA_ALIGN(skb->len) - skb->len;
 387
 388		if (plen > 0)
 389			skb_put_zero(skb, plen);
 390	}
 391}
 392
 393static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
 394				  const struct sw_flow_key *key,
 395				  const struct dp_upcall_info *upcall_info,
 396				  uint32_t cutlen)
 397{
 398	struct ovs_header *upcall;
 399	struct sk_buff *nskb = NULL;
 400	struct sk_buff *user_skb = NULL; /* to be queued to userspace */
 401	struct nlattr *nla;
 402	size_t len;
 403	unsigned int hlen;
 404	int err, dp_ifindex;
 405
 406	dp_ifindex = get_dpifindex(dp);
 407	if (!dp_ifindex)
 408		return -ENODEV;
 409
 410	if (skb_vlan_tag_present(skb)) {
 411		nskb = skb_clone(skb, GFP_ATOMIC);
 412		if (!nskb)
 413			return -ENOMEM;
 414
 415		nskb = __vlan_hwaccel_push_inside(nskb);
 416		if (!nskb)
 417			return -ENOMEM;
 418
 419		skb = nskb;
 420	}
 421
 422	if (nla_attr_size(skb->len) > USHRT_MAX) {
 423		err = -EFBIG;
 424		goto out;
 425	}
 426
 427	/* Complete checksum if needed */
 428	if (skb->ip_summed == CHECKSUM_PARTIAL &&
 429	    (err = skb_csum_hwoffload_help(skb, 0)))
 430		goto out;
 431
 432	/* Older versions of OVS user space enforce alignment of the last
 433	 * Netlink attribute to NLA_ALIGNTO which would require extensive
 434	 * padding logic. Only perform zerocopy if padding is not required.
 435	 */
 436	if (dp->user_features & OVS_DP_F_UNALIGNED)
 437		hlen = skb_zerocopy_headlen(skb);
 438	else
 439		hlen = skb->len;
 440
 441	len = upcall_msg_size(upcall_info, hlen - cutlen,
 442			      OVS_CB(skb)->acts_origlen);
 443	user_skb = genlmsg_new(len, GFP_ATOMIC);
 444	if (!user_skb) {
 445		err = -ENOMEM;
 446		goto out;
 447	}
 448
 449	upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
 450			     0, upcall_info->cmd);
 451	upcall->dp_ifindex = dp_ifindex;
 452
 453	err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
 454	BUG_ON(err);
 455
 456	if (upcall_info->userdata)
 457		__nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
 458			  nla_len(upcall_info->userdata),
 459			  nla_data(upcall_info->userdata));
 460
 461	if (upcall_info->egress_tun_info) {
 462		nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_EGRESS_TUN_KEY);
 463		err = ovs_nla_put_tunnel_info(user_skb,
 464					      upcall_info->egress_tun_info);
 465		BUG_ON(err);
 466		nla_nest_end(user_skb, nla);
 467	}
 468
 469	if (upcall_info->actions_len) {
 470		nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_ACTIONS);
 471		err = ovs_nla_put_actions(upcall_info->actions,
 472					  upcall_info->actions_len,
 473					  user_skb);
 474		if (!err)
 475			nla_nest_end(user_skb, nla);
 476		else
 477			nla_nest_cancel(user_skb, nla);
 478	}
 479
 480	/* Add OVS_PACKET_ATTR_MRU */
 481	if (upcall_info->mru) {
 482		if (nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU,
 483				upcall_info->mru)) {
 484			err = -ENOBUFS;
 485			goto out;
 486		}
 487		pad_packet(dp, user_skb);
 488	}
 489
 490	/* Add OVS_PACKET_ATTR_LEN when packet is truncated */
 491	if (cutlen > 0) {
 492		if (nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN,
 493				skb->len)) {
 494			err = -ENOBUFS;
 495			goto out;
 496		}
 497		pad_packet(dp, user_skb);
 498	}
 499
 500	/* Only reserve room for attribute header, packet data is added
 501	 * in skb_zerocopy() */
 502	if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
 503		err = -ENOBUFS;
 504		goto out;
 505	}
 506	nla->nla_len = nla_attr_size(skb->len - cutlen);
 507
 508	err = skb_zerocopy(user_skb, skb, skb->len - cutlen, hlen);
 509	if (err)
 510		goto out;
 511
 512	/* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
 513	pad_packet(dp, user_skb);
 514
 515	((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
 516
 517	err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
 518	user_skb = NULL;
 519out:
 520	if (err)
 521		skb_tx_error(skb);
 522	kfree_skb(user_skb);
 523	kfree_skb(nskb);
 524	return err;
 525}
 526
 527static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 528{
 529	struct ovs_header *ovs_header = info->userhdr;
 530	struct net *net = sock_net(skb->sk);
 531	struct nlattr **a = info->attrs;
 532	struct sw_flow_actions *acts;
 533	struct sk_buff *packet;
 534	struct sw_flow *flow;
 535	struct sw_flow_actions *sf_acts;
 536	struct datapath *dp;
 
 537	struct vport *input_vport;
 538	u16 mru = 0;
 539	int len;
 540	int err;
 541	bool log = !a[OVS_PACKET_ATTR_PROBE];
 542
 543	err = -EINVAL;
 544	if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
 545	    !a[OVS_PACKET_ATTR_ACTIONS])
 546		goto err;
 547
 548	len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
 549	packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
 550	err = -ENOMEM;
 551	if (!packet)
 552		goto err;
 553	skb_reserve(packet, NET_IP_ALIGN);
 554
 555	nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
 556
 
 
 
 
 
 
 
 
 
 
 
 557	/* Set packet's mru */
 558	if (a[OVS_PACKET_ATTR_MRU]) {
 559		mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]);
 560		packet->ignore_df = 1;
 561	}
 562	OVS_CB(packet)->mru = mru;
 563
 564	/* Build an sw_flow for sending this packet. */
 565	flow = ovs_flow_alloc();
 566	err = PTR_ERR(flow);
 567	if (IS_ERR(flow))
 568		goto err_kfree_skb;
 569
 570	err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY],
 571					     packet, &flow->key, log);
 572	if (err)
 573		goto err_flow_free;
 574
 575	err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS],
 576				   &flow->key, &acts, log);
 577	if (err)
 578		goto err_flow_free;
 579
 580	rcu_assign_pointer(flow->sf_acts, acts);
 581	packet->priority = flow->key.phy.priority;
 582	packet->mark = flow->key.phy.skb_mark;
 583
 584	rcu_read_lock();
 585	dp = get_dp_rcu(net, ovs_header->dp_ifindex);
 586	err = -ENODEV;
 587	if (!dp)
 588		goto err_unlock;
 589
 590	input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
 591	if (!input_vport)
 592		input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
 593
 594	if (!input_vport)
 595		goto err_unlock;
 596
 597	packet->dev = input_vport->dev;
 598	OVS_CB(packet)->input_vport = input_vport;
 599	sf_acts = rcu_dereference(flow->sf_acts);
 600
 601	local_bh_disable();
 602	err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
 603	local_bh_enable();
 604	rcu_read_unlock();
 605
 606	ovs_flow_free(flow, false);
 607	return err;
 608
 609err_unlock:
 610	rcu_read_unlock();
 611err_flow_free:
 612	ovs_flow_free(flow, false);
 613err_kfree_skb:
 614	kfree_skb(packet);
 615err:
 616	return err;
 617}
 618
 619static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
 620	[OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
 621	[OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
 622	[OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
 623	[OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
 624	[OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 },
 625};
 626
 627static const struct genl_ops dp_packet_genl_ops[] = {
 628	{ .cmd = OVS_PACKET_CMD_EXECUTE,
 629	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
 630	  .policy = packet_policy,
 631	  .doit = ovs_packet_cmd_execute
 632	}
 633};
 634
 635static struct genl_family dp_packet_genl_family __ro_after_init = {
 
 636	.hdrsize = sizeof(struct ovs_header),
 637	.name = OVS_PACKET_FAMILY,
 638	.version = OVS_PACKET_VERSION,
 639	.maxattr = OVS_PACKET_ATTR_MAX,
 640	.netnsok = true,
 641	.parallel_ops = true,
 642	.ops = dp_packet_genl_ops,
 643	.n_ops = ARRAY_SIZE(dp_packet_genl_ops),
 644	.module = THIS_MODULE,
 645};
 646
 647static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
 648			 struct ovs_dp_megaflow_stats *mega_stats)
 649{
 650	int i;
 651
 652	memset(mega_stats, 0, sizeof(*mega_stats));
 653
 654	stats->n_flows = ovs_flow_tbl_count(&dp->table);
 655	mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
 656
 657	stats->n_hit = stats->n_missed = stats->n_lost = 0;
 658
 659	for_each_possible_cpu(i) {
 660		const struct dp_stats_percpu *percpu_stats;
 661		struct dp_stats_percpu local_stats;
 662		unsigned int start;
 663
 664		percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
 665
 666		do {
 667			start = u64_stats_fetch_begin_irq(&percpu_stats->syncp);
 668			local_stats = *percpu_stats;
 669		} while (u64_stats_fetch_retry_irq(&percpu_stats->syncp, start));
 670
 671		stats->n_hit += local_stats.n_hit;
 672		stats->n_missed += local_stats.n_missed;
 673		stats->n_lost += local_stats.n_lost;
 674		mega_stats->n_mask_hit += local_stats.n_mask_hit;
 675	}
 676}
 677
 678static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
 679{
 680	return ovs_identifier_is_ufid(sfid) &&
 681	       !(ufid_flags & OVS_UFID_F_OMIT_KEY);
 682}
 683
 684static bool should_fill_mask(uint32_t ufid_flags)
 685{
 686	return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
 687}
 688
 689static bool should_fill_actions(uint32_t ufid_flags)
 690{
 691	return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
 692}
 693
 694static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
 695				    const struct sw_flow_id *sfid,
 696				    uint32_t ufid_flags)
 697{
 698	size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
 699
 700	/* OVS_FLOW_ATTR_UFID */
 701	if (sfid && ovs_identifier_is_ufid(sfid))
 702		len += nla_total_size(sfid->ufid_len);
 703
 704	/* OVS_FLOW_ATTR_KEY */
 705	if (!sfid || should_fill_key(sfid, ufid_flags))
 706		len += nla_total_size(ovs_key_attr_size());
 707
 708	/* OVS_FLOW_ATTR_MASK */
 709	if (should_fill_mask(ufid_flags))
 710		len += nla_total_size(ovs_key_attr_size());
 711
 712	/* OVS_FLOW_ATTR_ACTIONS */
 713	if (should_fill_actions(ufid_flags))
 714		len += nla_total_size(acts->orig_len);
 715
 716	return len
 717		+ nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
 718		+ nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
 719		+ nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */
 720}
 721
 722/* Called with ovs_mutex or RCU read lock. */
 723static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
 724				   struct sk_buff *skb)
 725{
 726	struct ovs_flow_stats stats;
 727	__be16 tcp_flags;
 728	unsigned long used;
 729
 730	ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
 731
 732	if (used &&
 733	    nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used),
 734			      OVS_FLOW_ATTR_PAD))
 735		return -EMSGSIZE;
 736
 737	if (stats.n_packets &&
 738	    nla_put_64bit(skb, OVS_FLOW_ATTR_STATS,
 739			  sizeof(struct ovs_flow_stats), &stats,
 740			  OVS_FLOW_ATTR_PAD))
 741		return -EMSGSIZE;
 742
 743	if ((u8)ntohs(tcp_flags) &&
 744	     nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
 745		return -EMSGSIZE;
 746
 747	return 0;
 748}
 749
 750/* Called with ovs_mutex or RCU read lock. */
 751static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
 752				     struct sk_buff *skb, int skb_orig_len)
 753{
 754	struct nlattr *start;
 755	int err;
 756
 757	/* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
 758	 * this is the first flow to be dumped into 'skb'.  This is unusual for
 759	 * Netlink but individual action lists can be longer than
 760	 * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
 761	 * The userspace caller can always fetch the actions separately if it
 762	 * really wants them.  (Most userspace callers in fact don't care.)
 763	 *
 764	 * This can only fail for dump operations because the skb is always
 765	 * properly sized for single flows.
 766	 */
 767	start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
 768	if (start) {
 769		const struct sw_flow_actions *sf_acts;
 770
 771		sf_acts = rcu_dereference_ovsl(flow->sf_acts);
 772		err = ovs_nla_put_actions(sf_acts->actions,
 773					  sf_acts->actions_len, skb);
 774
 775		if (!err)
 776			nla_nest_end(skb, start);
 777		else {
 778			if (skb_orig_len)
 779				return err;
 780
 781			nla_nest_cancel(skb, start);
 782		}
 783	} else if (skb_orig_len) {
 784		return -EMSGSIZE;
 785	}
 786
 787	return 0;
 788}
 789
 790/* Called with ovs_mutex or RCU read lock. */
 791static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
 792				  struct sk_buff *skb, u32 portid,
 793				  u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
 794{
 795	const int skb_orig_len = skb->len;
 796	struct ovs_header *ovs_header;
 797	int err;
 798
 799	ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
 800				 flags, cmd);
 801	if (!ovs_header)
 802		return -EMSGSIZE;
 803
 804	ovs_header->dp_ifindex = dp_ifindex;
 805
 806	err = ovs_nla_put_identifier(flow, skb);
 807	if (err)
 808		goto error;
 809
 810	if (should_fill_key(&flow->id, ufid_flags)) {
 811		err = ovs_nla_put_masked_key(flow, skb);
 812		if (err)
 813			goto error;
 814	}
 815
 816	if (should_fill_mask(ufid_flags)) {
 817		err = ovs_nla_put_mask(flow, skb);
 818		if (err)
 819			goto error;
 820	}
 821
 822	err = ovs_flow_cmd_fill_stats(flow, skb);
 823	if (err)
 824		goto error;
 825
 826	if (should_fill_actions(ufid_flags)) {
 827		err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
 828		if (err)
 829			goto error;
 830	}
 831
 832	genlmsg_end(skb, ovs_header);
 833	return 0;
 834
 835error:
 836	genlmsg_cancel(skb, ovs_header);
 837	return err;
 838}
 839
 840/* May not be called with RCU read lock. */
 841static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
 842					       const struct sw_flow_id *sfid,
 843					       struct genl_info *info,
 844					       bool always,
 845					       uint32_t ufid_flags)
 846{
 847	struct sk_buff *skb;
 848	size_t len;
 849
 850	if (!always && !ovs_must_notify(&dp_flow_genl_family, info, 0))
 851		return NULL;
 852
 853	len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
 854	skb = genlmsg_new(len, GFP_KERNEL);
 855	if (!skb)
 856		return ERR_PTR(-ENOMEM);
 857
 858	return skb;
 859}
 860
 861/* Called with ovs_mutex. */
 862static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
 863					       int dp_ifindex,
 864					       struct genl_info *info, u8 cmd,
 865					       bool always, u32 ufid_flags)
 866{
 867	struct sk_buff *skb;
 868	int retval;
 869
 870	skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
 871				      &flow->id, info, always, ufid_flags);
 872	if (IS_ERR_OR_NULL(skb))
 873		return skb;
 874
 875	retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
 876					info->snd_portid, info->snd_seq, 0,
 877					cmd, ufid_flags);
 878	BUG_ON(retval < 0);
 879	return skb;
 880}
 881
 882static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
 883{
 884	struct net *net = sock_net(skb->sk);
 885	struct nlattr **a = info->attrs;
 886	struct ovs_header *ovs_header = info->userhdr;
 887	struct sw_flow *flow = NULL, *new_flow;
 888	struct sw_flow_mask mask;
 889	struct sk_buff *reply;
 890	struct datapath *dp;
 
 891	struct sw_flow_actions *acts;
 892	struct sw_flow_match match;
 893	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
 894	int error;
 895	bool log = !a[OVS_FLOW_ATTR_PROBE];
 896
 897	/* Must have key and actions. */
 898	error = -EINVAL;
 899	if (!a[OVS_FLOW_ATTR_KEY]) {
 900		OVS_NLERR(log, "Flow key attr not present in new flow.");
 901		goto error;
 902	}
 903	if (!a[OVS_FLOW_ATTR_ACTIONS]) {
 904		OVS_NLERR(log, "Flow actions attr not present in new flow.");
 905		goto error;
 906	}
 907
 908	/* Most of the time we need to allocate a new flow, do it before
 909	 * locking.
 910	 */
 911	new_flow = ovs_flow_alloc();
 912	if (IS_ERR(new_flow)) {
 913		error = PTR_ERR(new_flow);
 914		goto error;
 915	}
 916
 917	/* Extract key. */
 918	ovs_match_init(&match, &new_flow->key, false, &mask);
 919	error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
 920				  a[OVS_FLOW_ATTR_MASK], log);
 921	if (error)
 922		goto err_kfree_flow;
 923
 
 
 924	/* Extract flow identifier. */
 925	error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
 926				       &new_flow->key, log);
 927	if (error)
 928		goto err_kfree_flow;
 929
 930	/* unmasked key is needed to match when ufid is not used. */
 931	if (ovs_identifier_is_key(&new_flow->id))
 932		match.key = new_flow->id.unmasked_key;
 933
 934	ovs_flow_mask_key(&new_flow->key, &new_flow->key, true, &mask);
 935
 936	/* Validate actions. */
 937	error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS],
 938				     &new_flow->key, &acts, log);
 939	if (error) {
 940		OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
 941		goto err_kfree_flow;
 942	}
 943
 944	reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
 945					ufid_flags);
 946	if (IS_ERR(reply)) {
 947		error = PTR_ERR(reply);
 948		goto err_kfree_acts;
 949	}
 950
 951	ovs_lock();
 952	dp = get_dp(net, ovs_header->dp_ifindex);
 953	if (unlikely(!dp)) {
 954		error = -ENODEV;
 955		goto err_unlock_ovs;
 956	}
 957
 958	/* Check if this is a duplicate flow */
 959	if (ovs_identifier_is_ufid(&new_flow->id))
 960		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
 961	if (!flow)
 962		flow = ovs_flow_tbl_lookup(&dp->table, &new_flow->key);
 963	if (likely(!flow)) {
 964		rcu_assign_pointer(new_flow->sf_acts, acts);
 965
 966		/* Put flow in bucket. */
 967		error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
 968		if (unlikely(error)) {
 969			acts = NULL;
 970			goto err_unlock_ovs;
 971		}
 972
 973		if (unlikely(reply)) {
 974			error = ovs_flow_cmd_fill_info(new_flow,
 975						       ovs_header->dp_ifindex,
 976						       reply, info->snd_portid,
 977						       info->snd_seq, 0,
 978						       OVS_FLOW_CMD_NEW,
 979						       ufid_flags);
 980			BUG_ON(error < 0);
 981		}
 982		ovs_unlock();
 983	} else {
 984		struct sw_flow_actions *old_acts;
 985
 986		/* Bail out if we're not allowed to modify an existing flow.
 987		 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
 988		 * because Generic Netlink treats the latter as a dump
 989		 * request.  We also accept NLM_F_EXCL in case that bug ever
 990		 * gets fixed.
 991		 */
 992		if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
 993							 | NLM_F_EXCL))) {
 994			error = -EEXIST;
 995			goto err_unlock_ovs;
 996		}
 997		/* The flow identifier has to be the same for flow updates.
 998		 * Look for any overlapping flow.
 999		 */
1000		if (unlikely(!ovs_flow_cmp(flow, &match))) {
1001			if (ovs_identifier_is_key(&flow->id))
1002				flow = ovs_flow_tbl_lookup_exact(&dp->table,
1003								 &match);
1004			else /* UFID matches but key is different */
1005				flow = NULL;
1006			if (!flow) {
1007				error = -ENOENT;
1008				goto err_unlock_ovs;
1009			}
1010		}
1011		/* Update actions. */
1012		old_acts = ovsl_dereference(flow->sf_acts);
1013		rcu_assign_pointer(flow->sf_acts, acts);
1014
1015		if (unlikely(reply)) {
1016			error = ovs_flow_cmd_fill_info(flow,
1017						       ovs_header->dp_ifindex,
1018						       reply, info->snd_portid,
1019						       info->snd_seq, 0,
1020						       OVS_FLOW_CMD_NEW,
1021						       ufid_flags);
1022			BUG_ON(error < 0);
1023		}
1024		ovs_unlock();
1025
1026		ovs_nla_free_flow_actions_rcu(old_acts);
1027		ovs_flow_free(new_flow, false);
1028	}
1029
1030	if (reply)
1031		ovs_notify(&dp_flow_genl_family, reply, info);
1032	return 0;
1033
1034err_unlock_ovs:
1035	ovs_unlock();
1036	kfree_skb(reply);
1037err_kfree_acts:
1038	ovs_nla_free_flow_actions(acts);
1039err_kfree_flow:
1040	ovs_flow_free(new_flow, false);
1041error:
1042	return error;
1043}
1044
1045/* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
1046static struct sw_flow_actions *get_flow_actions(struct net *net,
1047						const struct nlattr *a,
1048						const struct sw_flow_key *key,
1049						const struct sw_flow_mask *mask,
1050						bool log)
1051{
1052	struct sw_flow_actions *acts;
1053	struct sw_flow_key masked_key;
1054	int error;
1055
1056	ovs_flow_mask_key(&masked_key, key, true, mask);
1057	error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log);
1058	if (error) {
1059		OVS_NLERR(log,
1060			  "Actions may not be safe on all matching packets");
1061		return ERR_PTR(error);
1062	}
1063
1064	return acts;
1065}
1066
1067/* Factor out match-init and action-copy to avoid
1068 * "Wframe-larger-than=1024" warning. Because mask is only
1069 * used to get actions, we new a function to save some
1070 * stack space.
1071 *
1072 * If there are not key and action attrs, we return 0
1073 * directly. In the case, the caller will also not use the
1074 * match as before. If there is action attr, we try to get
1075 * actions and save them to *acts. Before returning from
1076 * the function, we reset the match->mask pointer. Because
1077 * we should not to return match object with dangling reference
1078 * to mask.
1079 * */
1080static int ovs_nla_init_match_and_action(struct net *net,
1081					 struct sw_flow_match *match,
1082					 struct sw_flow_key *key,
1083					 struct nlattr **a,
1084					 struct sw_flow_actions **acts,
1085					 bool log)
1086{
1087	struct sw_flow_mask mask;
1088	int error = 0;
1089
1090	if (a[OVS_FLOW_ATTR_KEY]) {
1091		ovs_match_init(match, key, true, &mask);
1092		error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY],
1093					  a[OVS_FLOW_ATTR_MASK], log);
1094		if (error)
1095			goto error;
1096	}
1097
1098	if (a[OVS_FLOW_ATTR_ACTIONS]) {
1099		if (!a[OVS_FLOW_ATTR_KEY]) {
1100			OVS_NLERR(log,
1101				  "Flow key attribute not present in set flow.");
1102			error = -EINVAL;
1103			goto error;
1104		}
1105
1106		*acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key,
1107					 &mask, log);
1108		if (IS_ERR(*acts)) {
1109			error = PTR_ERR(*acts);
1110			goto error;
1111		}
1112	}
1113
1114	/* On success, error is 0. */
1115error:
1116	match->mask = NULL;
1117	return error;
1118}
1119
1120static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1121{
1122	struct net *net = sock_net(skb->sk);
1123	struct nlattr **a = info->attrs;
1124	struct ovs_header *ovs_header = info->userhdr;
1125	struct sw_flow_key key;
1126	struct sw_flow *flow;
 
1127	struct sk_buff *reply = NULL;
1128	struct datapath *dp;
1129	struct sw_flow_actions *old_acts = NULL, *acts = NULL;
1130	struct sw_flow_match match;
1131	struct sw_flow_id sfid;
1132	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1133	int error = 0;
1134	bool log = !a[OVS_FLOW_ATTR_PROBE];
1135	bool ufid_present;
1136
1137	ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
1138	if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) {
 
 
 
 
1139		OVS_NLERR(log,
1140			  "Flow set message rejected, Key attribute missing.");
1141		return -EINVAL;
1142	}
1143
1144	error = ovs_nla_init_match_and_action(net, &match, &key, a,
1145					      &acts, log);
1146	if (error)
1147		goto error;
1148
1149	if (acts) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1150		/* Can allocate before locking if have acts. */
1151		reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
1152						ufid_flags);
1153		if (IS_ERR(reply)) {
1154			error = PTR_ERR(reply);
1155			goto err_kfree_acts;
1156		}
1157	}
1158
1159	ovs_lock();
1160	dp = get_dp(net, ovs_header->dp_ifindex);
1161	if (unlikely(!dp)) {
1162		error = -ENODEV;
1163		goto err_unlock_ovs;
1164	}
1165	/* Check that the flow exists. */
1166	if (ufid_present)
1167		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
1168	else
1169		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1170	if (unlikely(!flow)) {
1171		error = -ENOENT;
1172		goto err_unlock_ovs;
1173	}
1174
1175	/* Update actions, if present. */
1176	if (likely(acts)) {
1177		old_acts = ovsl_dereference(flow->sf_acts);
1178		rcu_assign_pointer(flow->sf_acts, acts);
1179
1180		if (unlikely(reply)) {
1181			error = ovs_flow_cmd_fill_info(flow,
1182						       ovs_header->dp_ifindex,
1183						       reply, info->snd_portid,
1184						       info->snd_seq, 0,
1185						       OVS_FLOW_CMD_NEW,
1186						       ufid_flags);
1187			BUG_ON(error < 0);
1188		}
1189	} else {
1190		/* Could not alloc without acts before locking. */
1191		reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
1192						info, OVS_FLOW_CMD_NEW, false,
1193						ufid_flags);
1194
1195		if (IS_ERR(reply)) {
1196			error = PTR_ERR(reply);
1197			goto err_unlock_ovs;
1198		}
1199	}
1200
1201	/* Clear stats. */
1202	if (a[OVS_FLOW_ATTR_CLEAR])
1203		ovs_flow_stats_clear(flow);
1204	ovs_unlock();
1205
1206	if (reply)
1207		ovs_notify(&dp_flow_genl_family, reply, info);
1208	if (old_acts)
1209		ovs_nla_free_flow_actions_rcu(old_acts);
1210
1211	return 0;
1212
1213err_unlock_ovs:
1214	ovs_unlock();
1215	kfree_skb(reply);
1216err_kfree_acts:
1217	ovs_nla_free_flow_actions(acts);
1218error:
1219	return error;
1220}
1221
1222static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1223{
1224	struct nlattr **a = info->attrs;
1225	struct ovs_header *ovs_header = info->userhdr;
1226	struct net *net = sock_net(skb->sk);
1227	struct sw_flow_key key;
1228	struct sk_buff *reply;
1229	struct sw_flow *flow;
1230	struct datapath *dp;
1231	struct sw_flow_match match;
1232	struct sw_flow_id ufid;
1233	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1234	int err = 0;
1235	bool log = !a[OVS_FLOW_ATTR_PROBE];
1236	bool ufid_present;
1237
1238	ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1239	if (a[OVS_FLOW_ATTR_KEY]) {
1240		ovs_match_init(&match, &key, true, NULL);
1241		err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL,
1242					log);
1243	} else if (!ufid_present) {
1244		OVS_NLERR(log,
1245			  "Flow get message rejected, Key attribute missing.");
1246		err = -EINVAL;
1247	}
1248	if (err)
1249		return err;
1250
1251	ovs_lock();
1252	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1253	if (!dp) {
1254		err = -ENODEV;
1255		goto unlock;
1256	}
1257
1258	if (ufid_present)
1259		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1260	else
1261		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1262	if (!flow) {
1263		err = -ENOENT;
1264		goto unlock;
1265	}
1266
1267	reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
1268					OVS_FLOW_CMD_NEW, true, ufid_flags);
1269	if (IS_ERR(reply)) {
1270		err = PTR_ERR(reply);
1271		goto unlock;
1272	}
1273
1274	ovs_unlock();
1275	return genlmsg_reply(reply, info);
1276unlock:
1277	ovs_unlock();
1278	return err;
1279}
1280
1281static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1282{
1283	struct nlattr **a = info->attrs;
1284	struct ovs_header *ovs_header = info->userhdr;
1285	struct net *net = sock_net(skb->sk);
1286	struct sw_flow_key key;
1287	struct sk_buff *reply;
1288	struct sw_flow *flow = NULL;
1289	struct datapath *dp;
1290	struct sw_flow_match match;
1291	struct sw_flow_id ufid;
1292	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1293	int err;
1294	bool log = !a[OVS_FLOW_ATTR_PROBE];
1295	bool ufid_present;
1296
1297	ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1298	if (a[OVS_FLOW_ATTR_KEY]) {
1299		ovs_match_init(&match, &key, true, NULL);
1300		err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1301					NULL, log);
1302		if (unlikely(err))
1303			return err;
1304	}
1305
1306	ovs_lock();
1307	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1308	if (unlikely(!dp)) {
1309		err = -ENODEV;
1310		goto unlock;
1311	}
1312
1313	if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
1314		err = ovs_flow_tbl_flush(&dp->table);
1315		goto unlock;
1316	}
1317
1318	if (ufid_present)
1319		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1320	else
1321		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1322	if (unlikely(!flow)) {
1323		err = -ENOENT;
1324		goto unlock;
1325	}
1326
1327	ovs_flow_tbl_remove(&dp->table, flow);
1328	ovs_unlock();
1329
1330	reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *) flow->sf_acts,
1331					&flow->id, info, false, ufid_flags);
1332	if (likely(reply)) {
1333		if (likely(!IS_ERR(reply))) {
1334			rcu_read_lock();	/*To keep RCU checker happy. */
1335			err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
1336						     reply, info->snd_portid,
1337						     info->snd_seq, 0,
1338						     OVS_FLOW_CMD_DEL,
1339						     ufid_flags);
1340			rcu_read_unlock();
1341			BUG_ON(err < 0);
1342
1343			ovs_notify(&dp_flow_genl_family, reply, info);
1344		} else {
1345			netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0, PTR_ERR(reply));
1346		}
1347	}
1348
1349	ovs_flow_free(flow, true);
1350	return 0;
1351unlock:
1352	ovs_unlock();
1353	return err;
1354}
1355
1356static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1357{
1358	struct nlattr *a[__OVS_FLOW_ATTR_MAX];
1359	struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1360	struct table_instance *ti;
1361	struct datapath *dp;
1362	u32 ufid_flags;
1363	int err;
1364
1365	err = genlmsg_parse(cb->nlh, &dp_flow_genl_family, a,
1366			    OVS_FLOW_ATTR_MAX, flow_policy, NULL);
1367	if (err)
1368		return err;
1369	ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1370
1371	rcu_read_lock();
1372	dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
1373	if (!dp) {
1374		rcu_read_unlock();
1375		return -ENODEV;
1376	}
1377
1378	ti = rcu_dereference(dp->table.ti);
1379	for (;;) {
1380		struct sw_flow *flow;
1381		u32 bucket, obj;
1382
1383		bucket = cb->args[0];
1384		obj = cb->args[1];
1385		flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1386		if (!flow)
1387			break;
1388
1389		if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
1390					   NETLINK_CB(cb->skb).portid,
1391					   cb->nlh->nlmsg_seq, NLM_F_MULTI,
1392					   OVS_FLOW_CMD_NEW, ufid_flags) < 0)
1393			break;
1394
1395		cb->args[0] = bucket;
1396		cb->args[1] = obj;
1397	}
1398	rcu_read_unlock();
1399	return skb->len;
1400}
1401
1402static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1403	[OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1404	[OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
1405	[OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1406	[OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1407	[OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
1408	[OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
1409	[OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
1410};
1411
1412static const struct genl_ops dp_flow_genl_ops[] = {
1413	{ .cmd = OVS_FLOW_CMD_NEW,
1414	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1415	  .policy = flow_policy,
1416	  .doit = ovs_flow_cmd_new
1417	},
1418	{ .cmd = OVS_FLOW_CMD_DEL,
1419	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1420	  .policy = flow_policy,
1421	  .doit = ovs_flow_cmd_del
1422	},
1423	{ .cmd = OVS_FLOW_CMD_GET,
1424	  .flags = 0,		    /* OK for unprivileged users. */
1425	  .policy = flow_policy,
1426	  .doit = ovs_flow_cmd_get,
1427	  .dumpit = ovs_flow_cmd_dump
1428	},
1429	{ .cmd = OVS_FLOW_CMD_SET,
1430	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1431	  .policy = flow_policy,
1432	  .doit = ovs_flow_cmd_set,
1433	},
1434};
1435
1436static struct genl_family dp_flow_genl_family __ro_after_init = {
 
1437	.hdrsize = sizeof(struct ovs_header),
1438	.name = OVS_FLOW_FAMILY,
1439	.version = OVS_FLOW_VERSION,
1440	.maxattr = OVS_FLOW_ATTR_MAX,
1441	.netnsok = true,
1442	.parallel_ops = true,
1443	.ops = dp_flow_genl_ops,
1444	.n_ops = ARRAY_SIZE(dp_flow_genl_ops),
1445	.mcgrps = &ovs_dp_flow_multicast_group,
1446	.n_mcgrps = 1,
1447	.module = THIS_MODULE,
1448};
1449
1450static size_t ovs_dp_cmd_msg_size(void)
1451{
1452	size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1453
1454	msgsize += nla_total_size(IFNAMSIZ);
1455	msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats));
1456	msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
1457	msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1458
1459	return msgsize;
1460}
1461
1462/* Called with ovs_mutex. */
1463static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1464				u32 portid, u32 seq, u32 flags, u8 cmd)
1465{
1466	struct ovs_header *ovs_header;
1467	struct ovs_dp_stats dp_stats;
1468	struct ovs_dp_megaflow_stats dp_megaflow_stats;
1469	int err;
1470
1471	ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1472				   flags, cmd);
1473	if (!ovs_header)
1474		goto error;
1475
1476	ovs_header->dp_ifindex = get_dpifindex(dp);
1477
1478	err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1479	if (err)
1480		goto nla_put_failure;
1481
1482	get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1483	if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1484			  &dp_stats, OVS_DP_ATTR_PAD))
1485		goto nla_put_failure;
1486
1487	if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1488			  sizeof(struct ovs_dp_megaflow_stats),
1489			  &dp_megaflow_stats, OVS_DP_ATTR_PAD))
1490		goto nla_put_failure;
1491
1492	if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1493		goto nla_put_failure;
1494
1495	genlmsg_end(skb, ovs_header);
1496	return 0;
1497
1498nla_put_failure:
1499	genlmsg_cancel(skb, ovs_header);
1500error:
1501	return -EMSGSIZE;
1502}
1503
1504static struct sk_buff *ovs_dp_cmd_alloc_info(void)
1505{
1506	return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
1507}
1508
1509/* Called with rcu_read_lock or ovs_mutex. */
1510static struct datapath *lookup_datapath(struct net *net,
1511					const struct ovs_header *ovs_header,
1512					struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1513{
1514	struct datapath *dp;
1515
1516	if (!a[OVS_DP_ATTR_NAME])
1517		dp = get_dp(net, ovs_header->dp_ifindex);
1518	else {
1519		struct vport *vport;
1520
1521		vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1522		dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1523	}
1524	return dp ? dp : ERR_PTR(-ENODEV);
1525}
1526
1527static void ovs_dp_reset_user_features(struct sk_buff *skb, struct genl_info *info)
1528{
1529	struct datapath *dp;
1530
1531	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1532	if (IS_ERR(dp))
1533		return;
1534
1535	WARN(dp->user_features, "Dropping previously announced user features\n");
1536	dp->user_features = 0;
1537}
1538
1539static void ovs_dp_change(struct datapath *dp, struct nlattr *a[])
1540{
1541	if (a[OVS_DP_ATTR_USER_FEATURES])
1542		dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1543}
1544
1545static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1546{
1547	struct nlattr **a = info->attrs;
1548	struct vport_parms parms;
1549	struct sk_buff *reply;
1550	struct datapath *dp;
1551	struct vport *vport;
1552	struct ovs_net *ovs_net;
1553	int err, i;
1554
1555	err = -EINVAL;
1556	if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1557		goto err;
1558
1559	reply = ovs_dp_cmd_alloc_info();
1560	if (!reply)
1561		return -ENOMEM;
1562
1563	err = -ENOMEM;
1564	dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1565	if (dp == NULL)
1566		goto err_free_reply;
1567
1568	ovs_dp_set_net(dp, sock_net(skb->sk));
1569
1570	/* Allocate table. */
1571	err = ovs_flow_tbl_init(&dp->table);
1572	if (err)
1573		goto err_free_dp;
1574
1575	dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
1576	if (!dp->stats_percpu) {
1577		err = -ENOMEM;
1578		goto err_destroy_table;
1579	}
1580
1581	dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1582			    GFP_KERNEL);
1583	if (!dp->ports) {
1584		err = -ENOMEM;
1585		goto err_destroy_percpu;
1586	}
1587
1588	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1589		INIT_HLIST_HEAD(&dp->ports[i]);
1590
1591	err = ovs_meters_init(dp);
1592	if (err)
1593		goto err_destroy_ports_array;
1594
1595	/* Set up our datapath device. */
1596	parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1597	parms.type = OVS_VPORT_TYPE_INTERNAL;
1598	parms.options = NULL;
1599	parms.dp = dp;
1600	parms.port_no = OVSP_LOCAL;
1601	parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1602
1603	ovs_dp_change(dp, a);
1604
1605	/* So far only local changes have been made, now need the lock. */
1606	ovs_lock();
1607
1608	vport = new_vport(&parms);
1609	if (IS_ERR(vport)) {
1610		err = PTR_ERR(vport);
1611		if (err == -EBUSY)
1612			err = -EEXIST;
1613
1614		if (err == -EEXIST) {
1615			/* An outdated user space instance that does not understand
1616			 * the concept of user_features has attempted to create a new
1617			 * datapath and is likely to reuse it. Drop all user features.
1618			 */
1619			if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1620				ovs_dp_reset_user_features(skb, info);
1621		}
1622
1623		goto err_destroy_meters;
1624	}
1625
1626	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1627				   info->snd_seq, 0, OVS_DP_CMD_NEW);
1628	BUG_ON(err < 0);
1629
1630	ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1631	list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1632
1633	ovs_unlock();
1634
1635	ovs_notify(&dp_datapath_genl_family, reply, info);
1636	return 0;
1637
1638err_destroy_meters:
1639	ovs_unlock();
1640	ovs_meters_exit(dp);
1641err_destroy_ports_array:
 
1642	kfree(dp->ports);
1643err_destroy_percpu:
1644	free_percpu(dp->stats_percpu);
1645err_destroy_table:
1646	ovs_flow_tbl_destroy(&dp->table);
1647err_free_dp:
1648	kfree(dp);
1649err_free_reply:
1650	kfree_skb(reply);
1651err:
1652	return err;
1653}
1654
1655/* Called with ovs_mutex. */
1656static void __dp_destroy(struct datapath *dp)
1657{
1658	int i;
1659
1660	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1661		struct vport *vport;
1662		struct hlist_node *n;
1663
1664		hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1665			if (vport->port_no != OVSP_LOCAL)
1666				ovs_dp_detach_port(vport);
1667	}
1668
1669	list_del_rcu(&dp->list_node);
1670
1671	/* OVSP_LOCAL is datapath internal port. We need to make sure that
1672	 * all ports in datapath are destroyed first before freeing datapath.
1673	 */
1674	ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1675
1676	/* RCU destroy the flow table */
1677	call_rcu(&dp->rcu, destroy_dp_rcu);
1678}
1679
1680static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1681{
1682	struct sk_buff *reply;
1683	struct datapath *dp;
1684	int err;
1685
1686	reply = ovs_dp_cmd_alloc_info();
1687	if (!reply)
1688		return -ENOMEM;
1689
1690	ovs_lock();
1691	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1692	err = PTR_ERR(dp);
1693	if (IS_ERR(dp))
1694		goto err_unlock_free;
1695
1696	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1697				   info->snd_seq, 0, OVS_DP_CMD_DEL);
1698	BUG_ON(err < 0);
1699
1700	__dp_destroy(dp);
1701	ovs_unlock();
1702
1703	ovs_notify(&dp_datapath_genl_family, reply, info);
1704
1705	return 0;
1706
1707err_unlock_free:
1708	ovs_unlock();
1709	kfree_skb(reply);
1710	return err;
1711}
1712
1713static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1714{
1715	struct sk_buff *reply;
1716	struct datapath *dp;
1717	int err;
1718
1719	reply = ovs_dp_cmd_alloc_info();
1720	if (!reply)
1721		return -ENOMEM;
1722
1723	ovs_lock();
1724	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1725	err = PTR_ERR(dp);
1726	if (IS_ERR(dp))
1727		goto err_unlock_free;
1728
1729	ovs_dp_change(dp, info->attrs);
1730
1731	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1732				   info->snd_seq, 0, OVS_DP_CMD_NEW);
1733	BUG_ON(err < 0);
1734
1735	ovs_unlock();
1736	ovs_notify(&dp_datapath_genl_family, reply, info);
1737
1738	return 0;
1739
1740err_unlock_free:
1741	ovs_unlock();
1742	kfree_skb(reply);
1743	return err;
1744}
1745
1746static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1747{
1748	struct sk_buff *reply;
1749	struct datapath *dp;
1750	int err;
1751
1752	reply = ovs_dp_cmd_alloc_info();
1753	if (!reply)
1754		return -ENOMEM;
1755
1756	ovs_lock();
1757	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1758	if (IS_ERR(dp)) {
1759		err = PTR_ERR(dp);
1760		goto err_unlock_free;
1761	}
1762	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1763				   info->snd_seq, 0, OVS_DP_CMD_NEW);
1764	BUG_ON(err < 0);
1765	ovs_unlock();
1766
1767	return genlmsg_reply(reply, info);
1768
1769err_unlock_free:
1770	ovs_unlock();
1771	kfree_skb(reply);
1772	return err;
1773}
1774
1775static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1776{
1777	struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1778	struct datapath *dp;
1779	int skip = cb->args[0];
1780	int i = 0;
1781
1782	ovs_lock();
1783	list_for_each_entry(dp, &ovs_net->dps, list_node) {
1784		if (i >= skip &&
1785		    ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1786					 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1787					 OVS_DP_CMD_NEW) < 0)
1788			break;
1789		i++;
1790	}
1791	ovs_unlock();
1792
1793	cb->args[0] = i;
1794
1795	return skb->len;
1796}
1797
1798static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1799	[OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1800	[OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1801	[OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
1802};
1803
1804static const struct genl_ops dp_datapath_genl_ops[] = {
1805	{ .cmd = OVS_DP_CMD_NEW,
1806	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1807	  .policy = datapath_policy,
1808	  .doit = ovs_dp_cmd_new
1809	},
1810	{ .cmd = OVS_DP_CMD_DEL,
1811	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1812	  .policy = datapath_policy,
1813	  .doit = ovs_dp_cmd_del
1814	},
1815	{ .cmd = OVS_DP_CMD_GET,
1816	  .flags = 0,		    /* OK for unprivileged users. */
1817	  .policy = datapath_policy,
1818	  .doit = ovs_dp_cmd_get,
1819	  .dumpit = ovs_dp_cmd_dump
1820	},
1821	{ .cmd = OVS_DP_CMD_SET,
1822	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1823	  .policy = datapath_policy,
1824	  .doit = ovs_dp_cmd_set,
1825	},
1826};
1827
1828static struct genl_family dp_datapath_genl_family __ro_after_init = {
 
1829	.hdrsize = sizeof(struct ovs_header),
1830	.name = OVS_DATAPATH_FAMILY,
1831	.version = OVS_DATAPATH_VERSION,
1832	.maxattr = OVS_DP_ATTR_MAX,
1833	.netnsok = true,
1834	.parallel_ops = true,
1835	.ops = dp_datapath_genl_ops,
1836	.n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
1837	.mcgrps = &ovs_dp_datapath_multicast_group,
1838	.n_mcgrps = 1,
1839	.module = THIS_MODULE,
1840};
1841
1842/* Called with ovs_mutex or RCU read lock. */
1843static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
1844				   struct net *net, u32 portid, u32 seq,
1845				   u32 flags, u8 cmd)
1846{
1847	struct ovs_header *ovs_header;
1848	struct ovs_vport_stats vport_stats;
1849	int err;
1850
1851	ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
1852				 flags, cmd);
1853	if (!ovs_header)
1854		return -EMSGSIZE;
1855
1856	ovs_header->dp_ifindex = get_dpifindex(vport->dp);
1857
1858	if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1859	    nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
1860	    nla_put_string(skb, OVS_VPORT_ATTR_NAME,
1861			   ovs_vport_name(vport)) ||
1862	    nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex))
1863		goto nla_put_failure;
1864
1865	if (!net_eq(net, dev_net(vport->dev))) {
1866		int id = peernet2id_alloc(net, dev_net(vport->dev));
1867
1868		if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id))
1869			goto nla_put_failure;
1870	}
1871
1872	ovs_vport_get_stats(vport, &vport_stats);
1873	if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS,
1874			  sizeof(struct ovs_vport_stats), &vport_stats,
1875			  OVS_VPORT_ATTR_PAD))
1876		goto nla_put_failure;
1877
1878	if (ovs_vport_get_upcall_portids(vport, skb))
1879		goto nla_put_failure;
1880
1881	err = ovs_vport_get_options(vport, skb);
1882	if (err == -EMSGSIZE)
1883		goto error;
1884
1885	genlmsg_end(skb, ovs_header);
1886	return 0;
1887
1888nla_put_failure:
1889	err = -EMSGSIZE;
1890error:
1891	genlmsg_cancel(skb, ovs_header);
1892	return err;
1893}
1894
1895static struct sk_buff *ovs_vport_cmd_alloc_info(void)
1896{
1897	return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1898}
1899
1900/* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
1901struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net,
1902					 u32 portid, u32 seq, u8 cmd)
1903{
1904	struct sk_buff *skb;
1905	int retval;
1906
1907	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
1908	if (!skb)
1909		return ERR_PTR(-ENOMEM);
1910
1911	retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd);
1912	BUG_ON(retval < 0);
1913
1914	return skb;
1915}
1916
1917/* Called with ovs_mutex or RCU read lock. */
1918static struct vport *lookup_vport(struct net *net,
1919				  const struct ovs_header *ovs_header,
1920				  struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1921{
1922	struct datapath *dp;
1923	struct vport *vport;
1924
1925	if (a[OVS_VPORT_ATTR_IFINDEX])
1926		return ERR_PTR(-EOPNOTSUPP);
1927	if (a[OVS_VPORT_ATTR_NAME]) {
1928		vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
1929		if (!vport)
1930			return ERR_PTR(-ENODEV);
1931		if (ovs_header->dp_ifindex &&
1932		    ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1933			return ERR_PTR(-ENODEV);
1934		return vport;
1935	} else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1936		u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1937
1938		if (port_no >= DP_MAX_PORTS)
1939			return ERR_PTR(-EFBIG);
1940
1941		dp = get_dp(net, ovs_header->dp_ifindex);
1942		if (!dp)
1943			return ERR_PTR(-ENODEV);
1944
1945		vport = ovs_vport_ovsl_rcu(dp, port_no);
1946		if (!vport)
1947			return ERR_PTR(-ENODEV);
1948		return vport;
1949	} else
1950		return ERR_PTR(-EINVAL);
1951
1952}
1953
1954/* Called with ovs_mutex */
1955static void update_headroom(struct datapath *dp)
1956{
1957	unsigned dev_headroom, max_headroom = 0;
1958	struct net_device *dev;
1959	struct vport *vport;
1960	int i;
1961
1962	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1963		hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
1964			dev = vport->dev;
1965			dev_headroom = netdev_get_fwd_headroom(dev);
1966			if (dev_headroom > max_headroom)
1967				max_headroom = dev_headroom;
1968		}
1969	}
1970
1971	dp->max_headroom = max_headroom;
1972	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1973		hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node)
1974			netdev_set_rx_headroom(vport->dev, max_headroom);
1975}
1976
1977static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
1978{
1979	struct nlattr **a = info->attrs;
1980	struct ovs_header *ovs_header = info->userhdr;
1981	struct vport_parms parms;
1982	struct sk_buff *reply;
1983	struct vport *vport;
1984	struct datapath *dp;
1985	u32 port_no;
1986	int err;
1987
1988	if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
1989	    !a[OVS_VPORT_ATTR_UPCALL_PID])
1990		return -EINVAL;
1991	if (a[OVS_VPORT_ATTR_IFINDEX])
1992		return -EOPNOTSUPP;
1993
1994	port_no = a[OVS_VPORT_ATTR_PORT_NO]
1995		? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
1996	if (port_no >= DP_MAX_PORTS)
1997		return -EFBIG;
1998
1999	reply = ovs_vport_cmd_alloc_info();
2000	if (!reply)
2001		return -ENOMEM;
2002
2003	ovs_lock();
2004restart:
2005	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
2006	err = -ENODEV;
2007	if (!dp)
2008		goto exit_unlock_free;
2009
2010	if (port_no) {
2011		vport = ovs_vport_ovsl(dp, port_no);
2012		err = -EBUSY;
2013		if (vport)
2014			goto exit_unlock_free;
2015	} else {
2016		for (port_no = 1; ; port_no++) {
2017			if (port_no >= DP_MAX_PORTS) {
2018				err = -EFBIG;
2019				goto exit_unlock_free;
2020			}
2021			vport = ovs_vport_ovsl(dp, port_no);
2022			if (!vport)
2023				break;
2024		}
2025	}
2026
2027	parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
2028	parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
2029	parms.options = a[OVS_VPORT_ATTR_OPTIONS];
2030	parms.dp = dp;
2031	parms.port_no = port_no;
2032	parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
2033
2034	vport = new_vport(&parms);
2035	err = PTR_ERR(vport);
2036	if (IS_ERR(vport)) {
2037		if (err == -EAGAIN)
2038			goto restart;
2039		goto exit_unlock_free;
2040	}
2041
2042	err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2043				      info->snd_portid, info->snd_seq, 0,
2044				      OVS_VPORT_CMD_NEW);
2045
2046	if (netdev_get_fwd_headroom(vport->dev) > dp->max_headroom)
2047		update_headroom(dp);
2048	else
2049		netdev_set_rx_headroom(vport->dev, dp->max_headroom);
2050
2051	BUG_ON(err < 0);
2052	ovs_unlock();
2053
2054	ovs_notify(&dp_vport_genl_family, reply, info);
2055	return 0;
2056
2057exit_unlock_free:
2058	ovs_unlock();
2059	kfree_skb(reply);
2060	return err;
2061}
2062
2063static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
2064{
2065	struct nlattr **a = info->attrs;
2066	struct sk_buff *reply;
2067	struct vport *vport;
2068	int err;
2069
2070	reply = ovs_vport_cmd_alloc_info();
2071	if (!reply)
2072		return -ENOMEM;
2073
2074	ovs_lock();
2075	vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2076	err = PTR_ERR(vport);
2077	if (IS_ERR(vport))
2078		goto exit_unlock_free;
2079
2080	if (a[OVS_VPORT_ATTR_TYPE] &&
2081	    nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
2082		err = -EINVAL;
2083		goto exit_unlock_free;
2084	}
2085
2086	if (a[OVS_VPORT_ATTR_OPTIONS]) {
2087		err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
2088		if (err)
2089			goto exit_unlock_free;
2090	}
2091
2092
2093	if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
2094		struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
2095
2096		err = ovs_vport_set_upcall_portids(vport, ids);
2097		if (err)
2098			goto exit_unlock_free;
2099	}
2100
2101	err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2102				      info->snd_portid, info->snd_seq, 0,
2103				      OVS_VPORT_CMD_NEW);
2104	BUG_ON(err < 0);
2105
2106	ovs_unlock();
2107	ovs_notify(&dp_vport_genl_family, reply, info);
2108	return 0;
2109
2110exit_unlock_free:
2111	ovs_unlock();
2112	kfree_skb(reply);
2113	return err;
2114}
2115
2116static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
2117{
2118	bool must_update_headroom = false;
2119	struct nlattr **a = info->attrs;
2120	struct sk_buff *reply;
2121	struct datapath *dp;
2122	struct vport *vport;
2123	int err;
2124
2125	reply = ovs_vport_cmd_alloc_info();
2126	if (!reply)
2127		return -ENOMEM;
2128
2129	ovs_lock();
2130	vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2131	err = PTR_ERR(vport);
2132	if (IS_ERR(vport))
2133		goto exit_unlock_free;
2134
2135	if (vport->port_no == OVSP_LOCAL) {
2136		err = -EINVAL;
2137		goto exit_unlock_free;
2138	}
2139
2140	err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2141				      info->snd_portid, info->snd_seq, 0,
2142				      OVS_VPORT_CMD_DEL);
2143	BUG_ON(err < 0);
2144
2145	/* the vport deletion may trigger dp headroom update */
2146	dp = vport->dp;
2147	if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom)
2148		must_update_headroom = true;
2149	netdev_reset_rx_headroom(vport->dev);
2150	ovs_dp_detach_port(vport);
2151
2152	if (must_update_headroom)
2153		update_headroom(dp);
2154	ovs_unlock();
2155
2156	ovs_notify(&dp_vport_genl_family, reply, info);
2157	return 0;
2158
2159exit_unlock_free:
2160	ovs_unlock();
2161	kfree_skb(reply);
2162	return err;
2163}
2164
2165static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
2166{
2167	struct nlattr **a = info->attrs;
2168	struct ovs_header *ovs_header = info->userhdr;
2169	struct sk_buff *reply;
2170	struct vport *vport;
2171	int err;
2172
2173	reply = ovs_vport_cmd_alloc_info();
2174	if (!reply)
2175		return -ENOMEM;
2176
2177	rcu_read_lock();
2178	vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2179	err = PTR_ERR(vport);
2180	if (IS_ERR(vport))
2181		goto exit_unlock_free;
2182	err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2183				      info->snd_portid, info->snd_seq, 0,
2184				      OVS_VPORT_CMD_NEW);
2185	BUG_ON(err < 0);
2186	rcu_read_unlock();
2187
2188	return genlmsg_reply(reply, info);
2189
2190exit_unlock_free:
2191	rcu_read_unlock();
2192	kfree_skb(reply);
2193	return err;
2194}
2195
2196static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2197{
2198	struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2199	struct datapath *dp;
2200	int bucket = cb->args[0], skip = cb->args[1];
2201	int i, j = 0;
2202
2203	rcu_read_lock();
2204	dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
2205	if (!dp) {
2206		rcu_read_unlock();
2207		return -ENODEV;
2208	}
2209	for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2210		struct vport *vport;
2211
2212		j = 0;
2213		hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2214			if (j >= skip &&
2215			    ovs_vport_cmd_fill_info(vport, skb,
2216						    sock_net(skb->sk),
2217						    NETLINK_CB(cb->skb).portid,
2218						    cb->nlh->nlmsg_seq,
2219						    NLM_F_MULTI,
2220						    OVS_VPORT_CMD_NEW) < 0)
2221				goto out;
2222
2223			j++;
2224		}
2225		skip = 0;
2226	}
2227out:
2228	rcu_read_unlock();
2229
2230	cb->args[0] = i;
2231	cb->args[1] = j;
2232
2233	return skb->len;
2234}
2235
2236static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2237	[OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2238	[OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2239	[OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2240	[OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
2241	[OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
2242	[OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
2243	[OVS_VPORT_ATTR_IFINDEX] = { .type = NLA_U32 },
2244	[OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 },
2245};
2246
2247static const struct genl_ops dp_vport_genl_ops[] = {
2248	{ .cmd = OVS_VPORT_CMD_NEW,
2249	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2250	  .policy = vport_policy,
2251	  .doit = ovs_vport_cmd_new
2252	},
2253	{ .cmd = OVS_VPORT_CMD_DEL,
2254	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2255	  .policy = vport_policy,
2256	  .doit = ovs_vport_cmd_del
2257	},
2258	{ .cmd = OVS_VPORT_CMD_GET,
2259	  .flags = 0,		    /* OK for unprivileged users. */
2260	  .policy = vport_policy,
2261	  .doit = ovs_vport_cmd_get,
2262	  .dumpit = ovs_vport_cmd_dump
2263	},
2264	{ .cmd = OVS_VPORT_CMD_SET,
2265	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2266	  .policy = vport_policy,
2267	  .doit = ovs_vport_cmd_set,
2268	},
2269};
2270
2271struct genl_family dp_vport_genl_family __ro_after_init = {
 
2272	.hdrsize = sizeof(struct ovs_header),
2273	.name = OVS_VPORT_FAMILY,
2274	.version = OVS_VPORT_VERSION,
2275	.maxattr = OVS_VPORT_ATTR_MAX,
2276	.netnsok = true,
2277	.parallel_ops = true,
2278	.ops = dp_vport_genl_ops,
2279	.n_ops = ARRAY_SIZE(dp_vport_genl_ops),
2280	.mcgrps = &ovs_dp_vport_multicast_group,
2281	.n_mcgrps = 1,
2282	.module = THIS_MODULE,
2283};
2284
2285static struct genl_family * const dp_genl_families[] = {
2286	&dp_datapath_genl_family,
2287	&dp_vport_genl_family,
2288	&dp_flow_genl_family,
2289	&dp_packet_genl_family,
2290	&dp_meter_genl_family,
2291};
2292
2293static void dp_unregister_genl(int n_families)
2294{
2295	int i;
2296
2297	for (i = 0; i < n_families; i++)
2298		genl_unregister_family(dp_genl_families[i]);
2299}
2300
2301static int __init dp_register_genl(void)
2302{
2303	int err;
2304	int i;
2305
2306	for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2307
2308		err = genl_register_family(dp_genl_families[i]);
2309		if (err)
2310			goto error;
2311	}
2312
2313	return 0;
2314
2315error:
2316	dp_unregister_genl(i);
2317	return err;
2318}
2319
2320static int __net_init ovs_init_net(struct net *net)
2321{
2322	struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2323
2324	INIT_LIST_HEAD(&ovs_net->dps);
2325	INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2326	ovs_ct_init(net);
2327	return 0;
2328}
2329
2330static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
2331					    struct list_head *head)
2332{
2333	struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2334	struct datapath *dp;
2335
2336	list_for_each_entry(dp, &ovs_net->dps, list_node) {
2337		int i;
2338
2339		for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2340			struct vport *vport;
2341
2342			hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
2343				if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
2344					continue;
2345
2346				if (dev_net(vport->dev) == dnet)
2347					list_add(&vport->detach_list, head);
2348			}
2349		}
2350	}
2351}
2352
2353static void __net_exit ovs_exit_net(struct net *dnet)
2354{
2355	struct datapath *dp, *dp_next;
2356	struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
2357	struct vport *vport, *vport_next;
2358	struct net *net;
2359	LIST_HEAD(head);
2360
2361	ovs_ct_exit(dnet);
2362	ovs_lock();
2363	list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2364		__dp_destroy(dp);
2365
2366	down_read(&net_rwsem);
2367	for_each_net(net)
2368		list_vports_from_net(net, dnet, &head);
2369	up_read(&net_rwsem);
2370
2371	/* Detach all vports from given namespace. */
2372	list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
2373		list_del(&vport->detach_list);
2374		ovs_dp_detach_port(vport);
2375	}
2376
2377	ovs_unlock();
2378
2379	cancel_work_sync(&ovs_net->dp_notify_work);
2380}
2381
2382static struct pernet_operations ovs_net_ops = {
2383	.init = ovs_init_net,
2384	.exit = ovs_exit_net,
2385	.id   = &ovs_net_id,
2386	.size = sizeof(struct ovs_net),
2387};
2388
2389static int __init dp_init(void)
2390{
2391	int err;
2392
2393	BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
2394
2395	pr_info("Open vSwitch switching datapath\n");
2396
2397	err = action_fifos_init();
2398	if (err)
2399		goto error;
2400
2401	err = ovs_internal_dev_rtnl_link_register();
2402	if (err)
2403		goto error_action_fifos_exit;
2404
2405	err = ovs_flow_init();
2406	if (err)
2407		goto error_unreg_rtnl_link;
2408
2409	err = ovs_vport_init();
2410	if (err)
2411		goto error_flow_exit;
2412
2413	err = register_pernet_device(&ovs_net_ops);
2414	if (err)
2415		goto error_vport_exit;
2416
2417	err = register_netdevice_notifier(&ovs_dp_device_notifier);
2418	if (err)
2419		goto error_netns_exit;
2420
2421	err = ovs_netdev_init();
2422	if (err)
2423		goto error_unreg_notifier;
2424
2425	err = dp_register_genl();
2426	if (err < 0)
2427		goto error_unreg_netdev;
2428
2429	return 0;
2430
2431error_unreg_netdev:
2432	ovs_netdev_exit();
2433error_unreg_notifier:
2434	unregister_netdevice_notifier(&ovs_dp_device_notifier);
2435error_netns_exit:
2436	unregister_pernet_device(&ovs_net_ops);
2437error_vport_exit:
2438	ovs_vport_exit();
2439error_flow_exit:
2440	ovs_flow_exit();
2441error_unreg_rtnl_link:
2442	ovs_internal_dev_rtnl_link_unregister();
2443error_action_fifos_exit:
2444	action_fifos_exit();
2445error:
2446	return err;
2447}
2448
2449static void dp_cleanup(void)
2450{
2451	dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2452	ovs_netdev_exit();
2453	unregister_netdevice_notifier(&ovs_dp_device_notifier);
2454	unregister_pernet_device(&ovs_net_ops);
2455	rcu_barrier();
2456	ovs_vport_exit();
2457	ovs_flow_exit();
2458	ovs_internal_dev_rtnl_link_unregister();
2459	action_fifos_exit();
2460}
2461
2462module_init(dp_init);
2463module_exit(dp_cleanup);
2464
2465MODULE_DESCRIPTION("Open vSwitch switching datapath");
2466MODULE_LICENSE("GPL");
2467MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY);
2468MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY);
2469MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY);
2470MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY);
2471MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY);