1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2009, Microsoft Corporation.
   4 *
 
 
 
 
 
 
 
 
 
 
 
 
   5 * Authors:
   6 *   Haiyang Zhang <haiyangz@microsoft.com>
   7 *   Hank Janssen  <hjanssen@microsoft.com>
   8 */
   9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  10
  11#include <linux/init.h>
  12#include <linux/atomic.h>
  13#include <linux/module.h>
  14#include <linux/highmem.h>
  15#include <linux/device.h>
  16#include <linux/io.h>
  17#include <linux/delay.h>
  18#include <linux/netdevice.h>
  19#include <linux/inetdevice.h>
  20#include <linux/etherdevice.h>
  21#include <linux/pci.h>
  22#include <linux/skbuff.h>
  23#include <linux/if_vlan.h>
  24#include <linux/in.h>
  25#include <linux/slab.h>
  26#include <linux/rtnetlink.h>
  27#include <linux/netpoll.h>
 
  28
  29#include <net/arp.h>
  30#include <net/route.h>
  31#include <net/sock.h>
  32#include <net/pkt_sched.h>
  33#include <net/checksum.h>
  34#include <net/ip6_checksum.h>
  35
  36#include "hyperv_net.h"
  37
  38#define RING_SIZE_MIN	64
  39#define RETRY_US_LO	5000
  40#define RETRY_US_HI	10000
  41#define RETRY_MAX	2000	/* >10 sec */
  42
  43#define LINKCHANGE_INT (2 * HZ)
  44#define VF_TAKEOVER_INT (HZ / 10)
  45
  46static unsigned int ring_size __ro_after_init = 128;
  47module_param(ring_size, uint, 0444);
  48MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
  49unsigned int netvsc_ring_bytes __ro_after_init;
 
  50
  51static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
  52				NETIF_MSG_LINK | NETIF_MSG_IFUP |
  53				NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR |
  54				NETIF_MSG_TX_ERR;
  55
  56static int debug = -1;
  57module_param(debug, int, 0444);
  58MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  59
  60static LIST_HEAD(netvsc_dev_list);
  61
  62static void netvsc_change_rx_flags(struct net_device *net, int change)
  63{
  64	struct net_device_context *ndev_ctx = netdev_priv(net);
  65	struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
  66	int inc;
  67
  68	if (!vf_netdev)
  69		return;
  70
  71	if (change & IFF_PROMISC) {
  72		inc = (net->flags & IFF_PROMISC) ? 1 : -1;
  73		dev_set_promiscuity(vf_netdev, inc);
  74	}
  75
  76	if (change & IFF_ALLMULTI) {
  77		inc = (net->flags & IFF_ALLMULTI) ? 1 : -1;
  78		dev_set_allmulti(vf_netdev, inc);
  79	}
  80}
  81
  82static void netvsc_set_rx_mode(struct net_device *net)
  83{
  84	struct net_device_context *ndev_ctx = netdev_priv(net);
  85	struct net_device *vf_netdev;
  86	struct netvsc_device *nvdev;
  87
  88	rcu_read_lock();
  89	vf_netdev = rcu_dereference(ndev_ctx->vf_netdev);
  90	if (vf_netdev) {
  91		dev_uc_sync(vf_netdev, net);
  92		dev_mc_sync(vf_netdev, net);
  93	}
  94
  95	nvdev = rcu_dereference(ndev_ctx->nvdev);
  96	if (nvdev)
  97		rndis_filter_update(nvdev);
  98	rcu_read_unlock();
  99}
 100
 101static void netvsc_tx_enable(struct netvsc_device *nvscdev,
 102			     struct net_device *ndev)
 103{
 104	nvscdev->tx_disable = false;
 105	virt_wmb(); /* ensure queue wake up mechanism is on */
 106
 107	netif_tx_wake_all_queues(ndev);
 108}
 109
 110static int netvsc_open(struct net_device *net)
 111{
 112	struct net_device_context *ndev_ctx = netdev_priv(net);
 113	struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
 114	struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev);
 115	struct rndis_device *rdev;
 116	int ret = 0;
 117
 118	netif_carrier_off(net);
 119
 120	/* Open up the device */
 121	ret = rndis_filter_open(nvdev);
 122	if (ret != 0) {
 123		netdev_err(net, "unable to open device (ret %d).\n", ret);
 124		return ret;
 125	}
 126
 127	rdev = nvdev->extension;
 128	if (!rdev->link_state) {
 129		netif_carrier_on(net);
 130		netvsc_tx_enable(nvdev, net);
 131	}
 132
 133	if (vf_netdev) {
 134		/* Setting synthetic device up transparently sets
 135		 * slave as up. If open fails, then slave will be
 136		 * still be offline (and not used).
 137		 */
 138		ret = dev_open(vf_netdev, NULL);
 139		if (ret)
 140			netdev_warn(net,
 141				    "unable to open slave: %s: %d\n",
 142				    vf_netdev->name, ret);
 143	}
 144	return 0;
 145}
 146
 147static int netvsc_wait_until_empty(struct netvsc_device *nvdev)
 148{
 149	unsigned int retry = 0;
 150	int i;
 151
 152	/* Ensure pending bytes in ring are read */
 153	for (;;) {
 154		u32 aread = 0;
 155
 156		for (i = 0; i < nvdev->num_chn; i++) {
 157			struct vmbus_channel *chn
 158				= nvdev->chan_table[i].channel;
 159
 160			if (!chn)
 161				continue;
 162
 163			/* make sure receive not running now */
 164			napi_synchronize(&nvdev->chan_table[i].napi);
 165
 166			aread = hv_get_bytes_to_read(&chn->inbound);
 167			if (aread)
 168				break;
 169
 170			aread = hv_get_bytes_to_read(&chn->outbound);
 171			if (aread)
 172				break;
 173		}
 174
 175		if (aread == 0)
 176			return 0;
 177
 178		if (++retry > RETRY_MAX)
 179			return -ETIMEDOUT;
 180
 181		usleep_range(RETRY_US_LO, RETRY_US_HI);
 182	}
 183}
 184
 185static void netvsc_tx_disable(struct netvsc_device *nvscdev,
 186			      struct net_device *ndev)
 187{
 188	if (nvscdev) {
 189		nvscdev->tx_disable = true;
 190		virt_wmb(); /* ensure txq will not wake up after stop */
 191	}
 192
 193	netif_tx_disable(ndev);
 194}
 195
 196static int netvsc_close(struct net_device *net)
 197{
 198	struct net_device_context *net_device_ctx = netdev_priv(net);
 199	struct net_device *vf_netdev
 200		= rtnl_dereference(net_device_ctx->vf_netdev);
 201	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 202	int ret;
 203
 204	netvsc_tx_disable(nvdev, net);
 205
 206	/* No need to close rndis filter if it is removed already */
 207	if (!nvdev)
 208		return 0;
 209
 210	ret = rndis_filter_close(nvdev);
 211	if (ret != 0) {
 212		netdev_err(net, "unable to close device (ret %d).\n", ret);
 213		return ret;
 214	}
 215
 216	ret = netvsc_wait_until_empty(nvdev);
 217	if (ret)
 218		netdev_err(net, "Ring buffer not empty after closing rndis\n");
 219
 220	if (vf_netdev)
 221		dev_close(vf_netdev);
 222
 223	return ret;
 224}
 225
 226static inline void *init_ppi_data(struct rndis_message *msg,
 227				  u32 ppi_size, u32 pkt_type)
 228{
 229	struct rndis_packet *rndis_pkt = &msg->msg.pkt;
 230	struct rndis_per_packet_info *ppi;
 231
 232	rndis_pkt->data_offset += ppi_size;
 233	ppi = (void *)rndis_pkt + rndis_pkt->per_pkt_info_offset
 234		+ rndis_pkt->per_pkt_info_len;
 235
 236	ppi->size = ppi_size;
 237	ppi->type = pkt_type;
 238	ppi->internal = 0;
 239	ppi->ppi_offset = sizeof(struct rndis_per_packet_info);
 240
 241	rndis_pkt->per_pkt_info_len += ppi_size;
 242
 243	return ppi + 1;
 244}
 245
 246/* Azure hosts don't support non-TCP port numbers in hashing for fragmented
 247 * packets. We can use ethtool to change UDP hash level when necessary.
 248 */
 249static inline u32 netvsc_get_hash(
 250	struct sk_buff *skb,
 251	const struct net_device_context *ndc)
 252{
 253	struct flow_keys flow;
 254	u32 hash, pkt_proto = 0;
 255	static u32 hashrnd __read_mostly;
 256
 257	net_get_random_once(&hashrnd, sizeof(hashrnd));
 258
 259	if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
 260		return 0;
 261
 262	switch (flow.basic.ip_proto) {
 263	case IPPROTO_TCP:
 264		if (flow.basic.n_proto == htons(ETH_P_IP))
 265			pkt_proto = HV_TCP4_L4HASH;
 266		else if (flow.basic.n_proto == htons(ETH_P_IPV6))
 267			pkt_proto = HV_TCP6_L4HASH;
 268
 269		break;
 270
 271	case IPPROTO_UDP:
 272		if (flow.basic.n_proto == htons(ETH_P_IP))
 273			pkt_proto = HV_UDP4_L4HASH;
 274		else if (flow.basic.n_proto == htons(ETH_P_IPV6))
 275			pkt_proto = HV_UDP6_L4HASH;
 276
 277		break;
 278	}
 279
 280	if (pkt_proto & ndc->l4_hash) {
 281		return skb_get_hash(skb);
 282	} else {
 283		if (flow.basic.n_proto == htons(ETH_P_IP))
 284			hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd);
 285		else if (flow.basic.n_proto == htons(ETH_P_IPV6))
 286			hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd);
 287		else
 288			hash = 0;
 289
 290		skb_set_hash(skb, hash, PKT_HASH_TYPE_L3);
 291	}
 292
 293	return hash;
 294}
 295
 296static inline int netvsc_get_tx_queue(struct net_device *ndev,
 297				      struct sk_buff *skb, int old_idx)
 298{
 299	const struct net_device_context *ndc = netdev_priv(ndev);
 300	struct sock *sk = skb->sk;
 301	int q_idx;
 302
 303	q_idx = ndc->tx_table[netvsc_get_hash(skb, ndc) &
 304			      (VRSS_SEND_TAB_SIZE - 1)];
 305
 306	/* If queue index changed record the new value */
 307	if (q_idx != old_idx &&
 308	    sk && sk_fullsock(sk) && rcu_access_pointer(sk->sk_dst_cache))
 309		sk_tx_queue_set(sk, q_idx);
 310
 311	return q_idx;
 312}
 313
 314/*
 315 * Select queue for transmit.
 316 *
 317 * If a valid queue has already been assigned, then use that.
 318 * Otherwise compute tx queue based on hash and the send table.
 319 *
 320 * This is basically similar to default (netdev_pick_tx) with the added step
 321 * of using the host send_table when no other queue has been assigned.
 322 *
 323 * TODO support XPS - but get_xps_queue not exported
 324 */
 325static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb)
 326{
 327	int q_idx = sk_tx_queue_get(skb->sk);
 328
 329	if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) {
 330		/* If forwarding a packet, we use the recorded queue when
 331		 * available for better cache locality.
 332		 */
 333		if (skb_rx_queue_recorded(skb))
 334			q_idx = skb_get_rx_queue(skb);
 335		else
 336			q_idx = netvsc_get_tx_queue(ndev, skb, q_idx);
 337	}
 338
 339	return q_idx;
 340}
 341
 342static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
 343			       struct net_device *sb_dev)
 
 344{
 345	struct net_device_context *ndc = netdev_priv(ndev);
 346	struct net_device *vf_netdev;
 347	u16 txq;
 348
 349	rcu_read_lock();
 350	vf_netdev = rcu_dereference(ndc->vf_netdev);
 351	if (vf_netdev) {
 352		const struct net_device_ops *vf_ops = vf_netdev->netdev_ops;
 353
 354		if (vf_ops->ndo_select_queue)
 355			txq = vf_ops->ndo_select_queue(vf_netdev, skb, sb_dev);
 
 356		else
 357			txq = netdev_pick_tx(vf_netdev, skb, NULL);
 358
 359		/* Record the queue selected by VF so that it can be
 360		 * used for common case where VF has more queues than
 361		 * the synthetic device.
 362		 */
 363		qdisc_skb_cb(skb)->slave_dev_queue_mapping = txq;
 364	} else {
 365		txq = netvsc_pick_tx(ndev, skb);
 366	}
 367	rcu_read_unlock();
 368
 369	while (unlikely(txq >= ndev->real_num_tx_queues))
 370		txq -= ndev->real_num_tx_queues;
 371
 372	return txq;
 373}
 374
 375static u32 fill_pg_buf(struct page *page, u32 offset, u32 len,
 376		       struct hv_page_buffer *pb)
 377{
 378	int j = 0;
 379
 380	/* Deal with compound pages by ignoring unused part
 381	 * of the page.
 382	 */
 383	page += (offset >> PAGE_SHIFT);
 384	offset &= ~PAGE_MASK;
 385
 386	while (len > 0) {
 387		unsigned long bytes;
 388
 389		bytes = PAGE_SIZE - offset;
 390		if (bytes > len)
 391			bytes = len;
 392		pb[j].pfn = page_to_pfn(page);
 393		pb[j].offset = offset;
 394		pb[j].len = bytes;
 395
 396		offset += bytes;
 397		len -= bytes;
 398
 399		if (offset == PAGE_SIZE && len) {
 400			page++;
 401			offset = 0;
 402			j++;
 403		}
 404	}
 405
 406	return j + 1;
 407}
 408
 409static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
 410			   struct hv_netvsc_packet *packet,
 411			   struct hv_page_buffer *pb)
 412{
 413	u32 slots_used = 0;
 414	char *data = skb->data;
 415	int frags = skb_shinfo(skb)->nr_frags;
 416	int i;
 417
 418	/* The packet is laid out thus:
 419	 * 1. hdr: RNDIS header and PPI
 420	 * 2. skb linear data
 421	 * 3. skb fragment data
 422	 */
 423	slots_used += fill_pg_buf(virt_to_page(hdr),
 424				  offset_in_page(hdr),
 425				  len, &pb[slots_used]);
 426
 427	packet->rmsg_size = len;
 428	packet->rmsg_pgcnt = slots_used;
 429
 430	slots_used += fill_pg_buf(virt_to_page(data),
 431				offset_in_page(data),
 432				skb_headlen(skb), &pb[slots_used]);
 433
 434	for (i = 0; i < frags; i++) {
 435		skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 436
 437		slots_used += fill_pg_buf(skb_frag_page(frag),
 438					skb_frag_off(frag),
 439					skb_frag_size(frag), &pb[slots_used]);
 440	}
 441	return slots_used;
 442}
 443
 444static int count_skb_frag_slots(struct sk_buff *skb)
 445{
 446	int i, frags = skb_shinfo(skb)->nr_frags;
 447	int pages = 0;
 448
 449	for (i = 0; i < frags; i++) {
 450		skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 451		unsigned long size = skb_frag_size(frag);
 452		unsigned long offset = skb_frag_off(frag);
 453
 454		/* Skip unused frames from start of page */
 455		offset &= ~PAGE_MASK;
 456		pages += PFN_UP(offset + size);
 457	}
 458	return pages;
 459}
 460
 461static int netvsc_get_slots(struct sk_buff *skb)
 462{
 463	char *data = skb->data;
 464	unsigned int offset = offset_in_page(data);
 465	unsigned int len = skb_headlen(skb);
 466	int slots;
 467	int frag_slots;
 468
 469	slots = DIV_ROUND_UP(offset + len, PAGE_SIZE);
 470	frag_slots = count_skb_frag_slots(skb);
 471	return slots + frag_slots;
 472}
 473
 474static u32 net_checksum_info(struct sk_buff *skb)
 475{
 476	if (skb->protocol == htons(ETH_P_IP)) {
 477		struct iphdr *ip = ip_hdr(skb);
 478
 479		if (ip->protocol == IPPROTO_TCP)
 480			return TRANSPORT_INFO_IPV4_TCP;
 481		else if (ip->protocol == IPPROTO_UDP)
 482			return TRANSPORT_INFO_IPV4_UDP;
 483	} else {
 484		struct ipv6hdr *ip6 = ipv6_hdr(skb);
 485
 486		if (ip6->nexthdr == IPPROTO_TCP)
 487			return TRANSPORT_INFO_IPV6_TCP;
 488		else if (ip6->nexthdr == IPPROTO_UDP)
 489			return TRANSPORT_INFO_IPV6_UDP;
 490	}
 491
 492	return TRANSPORT_INFO_NOT_IP;
 493}
 494
 495/* Send skb on the slave VF device. */
 496static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev,
 497			  struct sk_buff *skb)
 498{
 499	struct net_device_context *ndev_ctx = netdev_priv(net);
 500	unsigned int len = skb->len;
 501	int rc;
 502
 503	skb->dev = vf_netdev;
 504	skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;
 505
 506	rc = dev_queue_xmit(skb);
 507	if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) {
 508		struct netvsc_vf_pcpu_stats *pcpu_stats
 509			= this_cpu_ptr(ndev_ctx->vf_stats);
 510
 511		u64_stats_update_begin(&pcpu_stats->syncp);
 512		pcpu_stats->tx_packets++;
 513		pcpu_stats->tx_bytes += len;
 514		u64_stats_update_end(&pcpu_stats->syncp);
 515	} else {
 516		this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
 517	}
 518
 519	return rc;
 520}
 521
 522static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 523{
 524	struct net_device_context *net_device_ctx = netdev_priv(net);
 525	struct hv_netvsc_packet *packet = NULL;
 526	int ret;
 527	unsigned int num_data_pgs;
 528	struct rndis_message *rndis_msg;
 529	struct net_device *vf_netdev;
 530	u32 rndis_msg_size;
 531	u32 hash;
 532	struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT];
 533
 534	/* if VF is present and up then redirect packets
 535	 * already called with rcu_read_lock_bh
 536	 */
 537	vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev);
 538	if (vf_netdev && netif_running(vf_netdev) &&
 539	    !netpoll_tx_running(net))
 540		return netvsc_vf_xmit(net, vf_netdev, skb);
 541
 542	/* We will atmost need two pages to describe the rndis
 543	 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
 544	 * of pages in a single packet. If skb is scattered around
 545	 * more pages we try linearizing it.
 546	 */
 547
 548	num_data_pgs = netvsc_get_slots(skb) + 2;
 549
 550	if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
 551		++net_device_ctx->eth_stats.tx_scattered;
 552
 553		if (skb_linearize(skb))
 554			goto no_memory;
 555
 556		num_data_pgs = netvsc_get_slots(skb) + 2;
 557		if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
 558			++net_device_ctx->eth_stats.tx_too_big;
 559			goto drop;
 560		}
 561	}
 562
 563	/*
 564	 * Place the rndis header in the skb head room and
 565	 * the skb->cb will be used for hv_netvsc_packet
 566	 * structure.
 567	 */
 568	ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
 569	if (ret)
 570		goto no_memory;
 571
 572	/* Use the skb control buffer for building up the packet */
 573	BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
 574			FIELD_SIZEOF(struct sk_buff, cb));
 575	packet = (struct hv_netvsc_packet *)skb->cb;
 576
 577	packet->q_idx = skb_get_queue_mapping(skb);
 578
 579	packet->total_data_buflen = skb->len;
 580	packet->total_bytes = skb->len;
 581	packet->total_packets = 1;
 582
 583	rndis_msg = (struct rndis_message *)skb->head;
 584
 585	/* Add the rndis header */
 586	rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
 587	rndis_msg->msg_len = packet->total_data_buflen;
 588
 589	rndis_msg->msg.pkt = (struct rndis_packet) {
 590		.data_offset = sizeof(struct rndis_packet),
 591		.data_len = packet->total_data_buflen,
 592		.per_pkt_info_offset = sizeof(struct rndis_packet),
 593	};
 594
 595	rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
 596
 597	hash = skb_get_hash_raw(skb);
 598	if (hash != 0 && net->real_num_tx_queues > 1) {
 599		u32 *hash_info;
 600
 601		rndis_msg_size += NDIS_HASH_PPI_SIZE;
 602		hash_info = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
 603					  NBL_HASH_VALUE);
 604		*hash_info = hash;
 605	}
 606
 607	if (skb_vlan_tag_present(skb)) {
 608		struct ndis_pkt_8021q_info *vlan;
 609
 610		rndis_msg_size += NDIS_VLAN_PPI_SIZE;
 611		vlan = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
 612				     IEEE_8021Q_INFO);
 613
 614		vlan->value = 0;
 615		vlan->vlanid = skb_vlan_tag_get_id(skb);
 616		vlan->cfi = skb_vlan_tag_get_cfi(skb);
 617		vlan->pri = skb_vlan_tag_get_prio(skb);
 618	}
 619
 620	if (skb_is_gso(skb)) {
 621		struct ndis_tcp_lso_info *lso_info;
 622
 623		rndis_msg_size += NDIS_LSO_PPI_SIZE;
 624		lso_info = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
 625					 TCP_LARGESEND_PKTINFO);
 626
 627		lso_info->value = 0;
 628		lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
 629		if (skb->protocol == htons(ETH_P_IP)) {
 630			lso_info->lso_v2_transmit.ip_version =
 631				NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
 632			ip_hdr(skb)->tot_len = 0;
 633			ip_hdr(skb)->check = 0;
 634			tcp_hdr(skb)->check =
 635				~csum_tcpudp_magic(ip_hdr(skb)->saddr,
 636						   ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
 637		} else {
 638			lso_info->lso_v2_transmit.ip_version =
 639				NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
 640			ipv6_hdr(skb)->payload_len = 0;
 641			tcp_hdr(skb)->check =
 642				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
 643						 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
 644		}
 645		lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb);
 646		lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
 647	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 648		if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) {
 649			struct ndis_tcp_ip_checksum_info *csum_info;
 650
 651			rndis_msg_size += NDIS_CSUM_PPI_SIZE;
 652			csum_info = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
 653						  TCPIP_CHKSUM_PKTINFO);
 654
 655			csum_info->value = 0;
 656			csum_info->transmit.tcp_header_offset = skb_transport_offset(skb);
 657
 658			if (skb->protocol == htons(ETH_P_IP)) {
 659				csum_info->transmit.is_ipv4 = 1;
 660
 661				if (ip_hdr(skb)->protocol == IPPROTO_TCP)
 662					csum_info->transmit.tcp_checksum = 1;
 663				else
 664					csum_info->transmit.udp_checksum = 1;
 665			} else {
 666				csum_info->transmit.is_ipv6 = 1;
 667
 668				if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
 669					csum_info->transmit.tcp_checksum = 1;
 670				else
 671					csum_info->transmit.udp_checksum = 1;
 672			}
 673		} else {
 674			/* Can't do offload of this type of checksum */
 675			if (skb_checksum_help(skb))
 676				goto drop;
 677		}
 678	}
 679
 680	/* Start filling in the page buffers with the rndis hdr */
 681	rndis_msg->msg_len += rndis_msg_size;
 682	packet->total_data_buflen = rndis_msg->msg_len;
 683	packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
 684					       skb, packet, pb);
 685
 686	/* timestamp packet in software */
 687	skb_tx_timestamp(skb);
 688
 689	ret = netvsc_send(net, packet, rndis_msg, pb, skb);
 690	if (likely(ret == 0))
 691		return NETDEV_TX_OK;
 692
 693	if (ret == -EAGAIN) {
 694		++net_device_ctx->eth_stats.tx_busy;
 695		return NETDEV_TX_BUSY;
 696	}
 697
 698	if (ret == -ENOSPC)
 699		++net_device_ctx->eth_stats.tx_no_space;
 700
 701drop:
 702	dev_kfree_skb_any(skb);
 703	net->stats.tx_dropped++;
 704
 705	return NETDEV_TX_OK;
 706
 707no_memory:
 708	++net_device_ctx->eth_stats.tx_no_memory;
 709	goto drop;
 710}
 711
 712/*
 713 * netvsc_linkstatus_callback - Link up/down notification
 714 */
 715void netvsc_linkstatus_callback(struct net_device *net,
 716				struct rndis_message *resp)
 717{
 718	struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
 719	struct net_device_context *ndev_ctx = netdev_priv(net);
 720	struct netvsc_reconfig *event;
 721	unsigned long flags;
 722
 723	/* Update the physical link speed when changing to another vSwitch */
 724	if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
 725		u32 speed;
 726
 727		speed = *(u32 *)((void *)indicate
 728				 + indicate->status_buf_offset) / 10000;
 729		ndev_ctx->speed = speed;
 730		return;
 731	}
 732
 733	/* Handle these link change statuses below */
 734	if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE &&
 735	    indicate->status != RNDIS_STATUS_MEDIA_CONNECT &&
 736	    indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT)
 737		return;
 738
 739	if (net->reg_state != NETREG_REGISTERED)
 740		return;
 741
 742	event = kzalloc(sizeof(*event), GFP_ATOMIC);
 743	if (!event)
 744		return;
 745	event->event = indicate->status;
 746
 747	spin_lock_irqsave(&ndev_ctx->lock, flags);
 748	list_add_tail(&event->list, &ndev_ctx->reconfig_events);
 749	spin_unlock_irqrestore(&ndev_ctx->lock, flags);
 750
 751	schedule_delayed_work(&ndev_ctx->dwork, 0);
 752}
 753
 754static void netvsc_comp_ipcsum(struct sk_buff *skb)
 755{
 756	struct iphdr *iph = (struct iphdr *)skb->data;
 757
 758	iph->check = 0;
 759	iph->check = ip_fast_csum(iph, iph->ihl);
 760}
 761
 762static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
 763					     struct netvsc_channel *nvchan)
 
 
 
 764{
 765	struct napi_struct *napi = &nvchan->napi;
 766	const struct ndis_pkt_8021q_info *vlan = nvchan->rsc.vlan;
 767	const struct ndis_tcp_ip_checksum_info *csum_info =
 768						nvchan->rsc.csum_info;
 769	struct sk_buff *skb;
 770	int i;
 771
 772	skb = napi_alloc_skb(napi, nvchan->rsc.pktlen);
 773	if (!skb)
 774		return skb;
 775
 776	/*
 777	 * Copy to skb. This copy is needed here since the memory pointed by
 778	 * hv_netvsc_packet cannot be deallocated
 779	 */
 780	for (i = 0; i < nvchan->rsc.cnt; i++)
 781		skb_put_data(skb, nvchan->rsc.data[i], nvchan->rsc.len[i]);
 782
 783	skb->protocol = eth_type_trans(skb, net);
 784
 785	/* skb is already created with CHECKSUM_NONE */
 786	skb_checksum_none_assert(skb);
 787
 788	/* Incoming packets may have IP header checksum verified by the host.
 789	 * They may not have IP header checksum computed after coalescing.
 790	 * We compute it here if the flags are set, because on Linux, the IP
 791	 * checksum is always checked.
 792	 */
 793	if (csum_info && csum_info->receive.ip_checksum_value_invalid &&
 794	    csum_info->receive.ip_checksum_succeeded &&
 795	    skb->protocol == htons(ETH_P_IP))
 796		netvsc_comp_ipcsum(skb);
 797
 798	/* Do L4 checksum offload if enabled and present.
 799	 */
 800	if (csum_info && (net->features & NETIF_F_RXCSUM)) {
 801		if (csum_info->receive.tcp_checksum_succeeded ||
 802		    csum_info->receive.udp_checksum_succeeded)
 803			skb->ip_summed = CHECKSUM_UNNECESSARY;
 804	}
 805
 806	if (vlan) {
 807		u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT) |
 808			(vlan->cfi ? VLAN_CFI_MASK : 0);
 809
 810		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 811				       vlan_tci);
 812	}
 813
 814	return skb;
 815}
 816
 817/*
 818 * netvsc_recv_callback -  Callback when we receive a packet from the
 819 * "wire" on the specified device.
 820 */
 821int netvsc_recv_callback(struct net_device *net,
 822			 struct netvsc_device *net_device,
 823			 struct netvsc_channel *nvchan)
 
 
 
 824{
 825	struct net_device_context *net_device_ctx = netdev_priv(net);
 826	struct vmbus_channel *channel = nvchan->channel;
 827	u16 q_idx = channel->offermsg.offer.sub_channel_index;
 
 828	struct sk_buff *skb;
 829	struct netvsc_stats *rx_stats;
 830
 831	if (net->reg_state != NETREG_REGISTERED)
 832		return NVSP_STAT_FAIL;
 833
 834	/* Allocate a skb - TODO direct I/O to pages? */
 835	skb = netvsc_alloc_recv_skb(net, nvchan);
 836
 837	if (unlikely(!skb)) {
 838		++net_device_ctx->eth_stats.rx_no_memory;
 
 839		return NVSP_STAT_FAIL;
 840	}
 841
 842	skb_record_rx_queue(skb, q_idx);
 843
 844	/*
 845	 * Even if injecting the packet, record the statistics
 846	 * on the synthetic device because modifying the VF device
 847	 * statistics will not work correctly.
 848	 */
 849	rx_stats = &nvchan->rx_stats;
 850	u64_stats_update_begin(&rx_stats->syncp);
 851	rx_stats->packets++;
 852	rx_stats->bytes += nvchan->rsc.pktlen;
 853
 854	if (skb->pkt_type == PACKET_BROADCAST)
 855		++rx_stats->broadcast;
 856	else if (skb->pkt_type == PACKET_MULTICAST)
 857		++rx_stats->multicast;
 858	u64_stats_update_end(&rx_stats->syncp);
 859
 860	napi_gro_receive(&nvchan->napi, skb);
 861	return NVSP_STAT_SUCCESS;
 862}
 863
 864static void netvsc_get_drvinfo(struct net_device *net,
 865			       struct ethtool_drvinfo *info)
 866{
 867	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
 868	strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
 869}
 870
 871static void netvsc_get_channels(struct net_device *net,
 872				struct ethtool_channels *channel)
 873{
 874	struct net_device_context *net_device_ctx = netdev_priv(net);
 875	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 876
 877	if (nvdev) {
 878		channel->max_combined	= nvdev->max_chn;
 879		channel->combined_count = nvdev->num_chn;
 880	}
 881}
 882
 883/* Alloc struct netvsc_device_info, and initialize it from either existing
 884 * struct netvsc_device, or from default values.
 885 */
 886static struct netvsc_device_info *netvsc_devinfo_get
 887			(struct netvsc_device *nvdev)
 888{
 889	struct netvsc_device_info *dev_info;
 890
 891	dev_info = kzalloc(sizeof(*dev_info), GFP_ATOMIC);
 892
 893	if (!dev_info)
 894		return NULL;
 895
 896	if (nvdev) {
 897		dev_info->num_chn = nvdev->num_chn;
 898		dev_info->send_sections = nvdev->send_section_cnt;
 899		dev_info->send_section_size = nvdev->send_section_size;
 900		dev_info->recv_sections = nvdev->recv_section_cnt;
 901		dev_info->recv_section_size = nvdev->recv_section_size;
 902
 903		memcpy(dev_info->rss_key, nvdev->extension->rss_key,
 904		       NETVSC_HASH_KEYLEN);
 905	} else {
 906		dev_info->num_chn = VRSS_CHANNEL_DEFAULT;
 907		dev_info->send_sections = NETVSC_DEFAULT_TX;
 908		dev_info->send_section_size = NETVSC_SEND_SECTION_SIZE;
 909		dev_info->recv_sections = NETVSC_DEFAULT_RX;
 910		dev_info->recv_section_size = NETVSC_RECV_SECTION_SIZE;
 911	}
 912
 913	return dev_info;
 914}
 915
 916static int netvsc_detach(struct net_device *ndev,
 917			 struct netvsc_device *nvdev)
 918{
 919	struct net_device_context *ndev_ctx = netdev_priv(ndev);
 920	struct hv_device *hdev = ndev_ctx->device_ctx;
 921	int ret;
 922
 923	/* Don't try continuing to try and setup sub channels */
 924	if (cancel_work_sync(&nvdev->subchan_work))
 925		nvdev->num_chn = 1;
 926
 927	/* If device was up (receiving) then shutdown */
 928	if (netif_running(ndev)) {
 929		netvsc_tx_disable(nvdev, ndev);
 930
 931		ret = rndis_filter_close(nvdev);
 932		if (ret) {
 933			netdev_err(ndev,
 934				   "unable to close device (ret %d).\n", ret);
 935			return ret;
 936		}
 937
 938		ret = netvsc_wait_until_empty(nvdev);
 939		if (ret) {
 940			netdev_err(ndev,
 941				   "Ring buffer not empty after closing rndis\n");
 942			return ret;
 943		}
 944	}
 945
 946	netif_device_detach(ndev);
 947
 948	rndis_filter_device_remove(hdev, nvdev);
 949
 950	return 0;
 951}
 952
 953static int netvsc_attach(struct net_device *ndev,
 954			 struct netvsc_device_info *dev_info)
 955{
 956	struct net_device_context *ndev_ctx = netdev_priv(ndev);
 957	struct hv_device *hdev = ndev_ctx->device_ctx;
 958	struct netvsc_device *nvdev;
 959	struct rndis_device *rdev;
 960	int ret;
 961
 962	nvdev = rndis_filter_device_add(hdev, dev_info);
 963	if (IS_ERR(nvdev))
 964		return PTR_ERR(nvdev);
 965
 966	if (nvdev->num_chn > 1) {
 967		ret = rndis_set_subchannel(ndev, nvdev, dev_info);
 968
 969		/* if unavailable, just proceed with one queue */
 970		if (ret) {
 971			nvdev->max_chn = 1;
 972			nvdev->num_chn = 1;
 973		}
 974	}
 975
 976	/* In any case device is now ready */
 977	netif_device_attach(ndev);
 978
 979	/* Note: enable and attach happen when sub-channels setup */
 
 980	netif_carrier_off(ndev);
 981
 982	if (netif_running(ndev)) {
 983		ret = rndis_filter_open(nvdev);
 984		if (ret)
 985			goto err;
 986
 987		rdev = nvdev->extension;
 988		if (!rdev->link_state)
 989			netif_carrier_on(ndev);
 990	}
 991
 992	return 0;
 993
 994err:
 995	netif_device_detach(ndev);
 996
 997	rndis_filter_device_remove(hdev, nvdev);
 998
 999	return ret;
1000}
1001
1002static int netvsc_set_channels(struct net_device *net,
1003			       struct ethtool_channels *channels)
1004{
1005	struct net_device_context *net_device_ctx = netdev_priv(net);
1006	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
1007	unsigned int orig, count = channels->combined_count;
1008	struct netvsc_device_info *device_info;
1009	int ret;
1010
1011	/* We do not support separate count for rx, tx, or other */
1012	if (count == 0 ||
1013	    channels->rx_count || channels->tx_count || channels->other_count)
1014		return -EINVAL;
1015
1016	if (!nvdev || nvdev->destroy)
1017		return -ENODEV;
1018
1019	if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5)
1020		return -EINVAL;
1021
1022	if (count > nvdev->max_chn)
1023		return -EINVAL;
1024
1025	orig = nvdev->num_chn;
1026
1027	device_info = netvsc_devinfo_get(nvdev);
1028
1029	if (!device_info)
1030		return -ENOMEM;
1031
1032	device_info->num_chn = count;
1033
1034	ret = netvsc_detach(net, nvdev);
1035	if (ret)
1036		goto out;
1037
1038	ret = netvsc_attach(net, device_info);
1039	if (ret) {
1040		device_info->num_chn = orig;
1041		if (netvsc_attach(net, device_info))
1042			netdev_err(net, "restoring channel setting failed\n");
1043	}
1044
1045out:
1046	kfree(device_info);
1047	return ret;
1048}
1049
1050static bool
1051netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
1052{
1053	struct ethtool_link_ksettings diff1 = *cmd;
1054	struct ethtool_link_ksettings diff2 = {};
1055
1056	diff1.base.speed = 0;
1057	diff1.base.duplex = 0;
1058	/* advertising and cmd are usually set */
1059	ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
1060	diff1.base.cmd = 0;
1061	/* We set port to PORT_OTHER */
1062	diff2.base.port = PORT_OTHER;
1063
1064	return !memcmp(&diff1, &diff2, sizeof(diff1));
1065}
1066
1067static void netvsc_init_settings(struct net_device *dev)
1068{
1069	struct net_device_context *ndc = netdev_priv(dev);
1070
1071	ndc->l4_hash = HV_DEFAULT_L4HASH;
1072
1073	ndc->speed = SPEED_UNKNOWN;
1074	ndc->duplex = DUPLEX_FULL;
1075
1076	dev->features = NETIF_F_LRO;
1077}
1078
1079static int netvsc_get_link_ksettings(struct net_device *dev,
1080				     struct ethtool_link_ksettings *cmd)
1081{
1082	struct net_device_context *ndc = netdev_priv(dev);
1083
1084	cmd->base.speed = ndc->speed;
1085	cmd->base.duplex = ndc->duplex;
1086	cmd->base.port = PORT_OTHER;
1087
1088	return 0;
1089}
1090
1091static int netvsc_set_link_ksettings(struct net_device *dev,
1092				     const struct ethtool_link_ksettings *cmd)
1093{
1094	struct net_device_context *ndc = netdev_priv(dev);
1095	u32 speed;
1096
1097	speed = cmd->base.speed;
1098	if (!ethtool_validate_speed(speed) ||
1099	    !ethtool_validate_duplex(cmd->base.duplex) ||
1100	    !netvsc_validate_ethtool_ss_cmd(cmd))
1101		return -EINVAL;
1102
1103	ndc->speed = speed;
1104	ndc->duplex = cmd->base.duplex;
1105
1106	return 0;
1107}
1108
1109static int netvsc_change_mtu(struct net_device *ndev, int mtu)
1110{
1111	struct net_device_context *ndevctx = netdev_priv(ndev);
1112	struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
1113	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1114	int orig_mtu = ndev->mtu;
1115	struct netvsc_device_info *device_info;
1116	int ret = 0;
1117
1118	if (!nvdev || nvdev->destroy)
1119		return -ENODEV;
1120
1121	device_info = netvsc_devinfo_get(nvdev);
1122
1123	if (!device_info)
1124		return -ENOMEM;
1125
1126	/* Change MTU of underlying VF netdev first. */
1127	if (vf_netdev) {
1128		ret = dev_set_mtu(vf_netdev, mtu);
1129		if (ret)
1130			goto out;
1131	}
1132
 
 
 
 
 
 
 
1133	ret = netvsc_detach(ndev, nvdev);
1134	if (ret)
1135		goto rollback_vf;
1136
1137	ndev->mtu = mtu;
1138
1139	ret = netvsc_attach(ndev, device_info);
1140	if (!ret)
1141		goto out;
 
 
1142
 
1143	/* Attempt rollback to original MTU */
1144	ndev->mtu = orig_mtu;
1145
1146	if (netvsc_attach(ndev, device_info))
1147		netdev_err(ndev, "restoring mtu failed\n");
1148rollback_vf:
1149	if (vf_netdev)
1150		dev_set_mtu(vf_netdev, orig_mtu);
1151
1152out:
1153	kfree(device_info);
1154	return ret;
1155}
1156
1157static void netvsc_get_vf_stats(struct net_device *net,
1158				struct netvsc_vf_pcpu_stats *tot)
1159{
1160	struct net_device_context *ndev_ctx = netdev_priv(net);
1161	int i;
1162
1163	memset(tot, 0, sizeof(*tot));
1164
1165	for_each_possible_cpu(i) {
1166		const struct netvsc_vf_pcpu_stats *stats
1167			= per_cpu_ptr(ndev_ctx->vf_stats, i);
1168		u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
1169		unsigned int start;
1170
1171		do {
1172			start = u64_stats_fetch_begin_irq(&stats->syncp);
1173			rx_packets = stats->rx_packets;
1174			tx_packets = stats->tx_packets;
1175			rx_bytes = stats->rx_bytes;
1176			tx_bytes = stats->tx_bytes;
1177		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1178
1179		tot->rx_packets += rx_packets;
1180		tot->tx_packets += tx_packets;
1181		tot->rx_bytes   += rx_bytes;
1182		tot->tx_bytes   += tx_bytes;
1183		tot->tx_dropped += stats->tx_dropped;
1184	}
1185}
1186
1187static void netvsc_get_pcpu_stats(struct net_device *net,
1188				  struct netvsc_ethtool_pcpu_stats *pcpu_tot)
1189{
1190	struct net_device_context *ndev_ctx = netdev_priv(net);
1191	struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev);
1192	int i;
1193
1194	/* fetch percpu stats of vf */
1195	for_each_possible_cpu(i) {
1196		const struct netvsc_vf_pcpu_stats *stats =
1197			per_cpu_ptr(ndev_ctx->vf_stats, i);
1198		struct netvsc_ethtool_pcpu_stats *this_tot = &pcpu_tot[i];
1199		unsigned int start;
1200
1201		do {
1202			start = u64_stats_fetch_begin_irq(&stats->syncp);
1203			this_tot->vf_rx_packets = stats->rx_packets;
1204			this_tot->vf_tx_packets = stats->tx_packets;
1205			this_tot->vf_rx_bytes = stats->rx_bytes;
1206			this_tot->vf_tx_bytes = stats->tx_bytes;
1207		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1208		this_tot->rx_packets = this_tot->vf_rx_packets;
1209		this_tot->tx_packets = this_tot->vf_tx_packets;
1210		this_tot->rx_bytes   = this_tot->vf_rx_bytes;
1211		this_tot->tx_bytes   = this_tot->vf_tx_bytes;
1212	}
1213
1214	/* fetch percpu stats of netvsc */
1215	for (i = 0; i < nvdev->num_chn; i++) {
1216		const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
1217		const struct netvsc_stats *stats;
1218		struct netvsc_ethtool_pcpu_stats *this_tot =
1219			&pcpu_tot[nvchan->channel->target_cpu];
1220		u64 packets, bytes;
1221		unsigned int start;
1222
1223		stats = &nvchan->tx_stats;
1224		do {
1225			start = u64_stats_fetch_begin_irq(&stats->syncp);
1226			packets = stats->packets;
1227			bytes = stats->bytes;
1228		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1229
1230		this_tot->tx_bytes	+= bytes;
1231		this_tot->tx_packets	+= packets;
1232
1233		stats = &nvchan->rx_stats;
1234		do {
1235			start = u64_stats_fetch_begin_irq(&stats->syncp);
1236			packets = stats->packets;
1237			bytes = stats->bytes;
1238		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1239
1240		this_tot->rx_bytes	+= bytes;
1241		this_tot->rx_packets	+= packets;
1242	}
1243}
1244
1245static void netvsc_get_stats64(struct net_device *net,
1246			       struct rtnl_link_stats64 *t)
1247{
1248	struct net_device_context *ndev_ctx = netdev_priv(net);
1249	struct netvsc_device *nvdev;
1250	struct netvsc_vf_pcpu_stats vf_tot;
1251	int i;
1252
1253	rcu_read_lock();
1254
1255	nvdev = rcu_dereference(ndev_ctx->nvdev);
1256	if (!nvdev)
1257		goto out;
1258
1259	netdev_stats_to_stats64(t, &net->stats);
1260
1261	netvsc_get_vf_stats(net, &vf_tot);
1262	t->rx_packets += vf_tot.rx_packets;
1263	t->tx_packets += vf_tot.tx_packets;
1264	t->rx_bytes   += vf_tot.rx_bytes;
1265	t->tx_bytes   += vf_tot.tx_bytes;
1266	t->tx_dropped += vf_tot.tx_dropped;
1267
1268	for (i = 0; i < nvdev->num_chn; i++) {
1269		const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
1270		const struct netvsc_stats *stats;
1271		u64 packets, bytes, multicast;
1272		unsigned int start;
1273
1274		stats = &nvchan->tx_stats;
1275		do {
1276			start = u64_stats_fetch_begin_irq(&stats->syncp);
1277			packets = stats->packets;
1278			bytes = stats->bytes;
1279		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1280
1281		t->tx_bytes	+= bytes;
1282		t->tx_packets	+= packets;
1283
1284		stats = &nvchan->rx_stats;
1285		do {
1286			start = u64_stats_fetch_begin_irq(&stats->syncp);
1287			packets = stats->packets;
1288			bytes = stats->bytes;
1289			multicast = stats->multicast + stats->broadcast;
1290		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1291
1292		t->rx_bytes	+= bytes;
1293		t->rx_packets	+= packets;
1294		t->multicast	+= multicast;
1295	}
1296out:
1297	rcu_read_unlock();
1298}
1299
1300static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
1301{
1302	struct net_device_context *ndc = netdev_priv(ndev);
1303	struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev);
1304	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1305	struct sockaddr *addr = p;
1306	int err;
1307
1308	err = eth_prepare_mac_addr_change(ndev, p);
1309	if (err)
1310		return err;
1311
1312	if (!nvdev)
1313		return -ENODEV;
1314
1315	if (vf_netdev) {
1316		err = dev_set_mac_address(vf_netdev, addr, NULL);
1317		if (err)
1318			return err;
1319	}
1320
1321	err = rndis_filter_set_device_mac(nvdev, addr->sa_data);
1322	if (!err) {
1323		eth_commit_mac_addr_change(ndev, p);
1324	} else if (vf_netdev) {
1325		/* rollback change on VF */
1326		memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN);
1327		dev_set_mac_address(vf_netdev, addr, NULL);
1328	}
1329
1330	return err;
1331}
1332
1333static const struct {
1334	char name[ETH_GSTRING_LEN];
1335	u16 offset;
1336} netvsc_stats[] = {
1337	{ "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) },
1338	{ "tx_no_memory", offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
1339	{ "tx_no_space",  offsetof(struct netvsc_ethtool_stats, tx_no_space) },
1340	{ "tx_too_big",	  offsetof(struct netvsc_ethtool_stats, tx_too_big) },
1341	{ "tx_busy",	  offsetof(struct netvsc_ethtool_stats, tx_busy) },
1342	{ "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) },
1343	{ "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) },
1344	{ "rx_no_memory", offsetof(struct netvsc_ethtool_stats, rx_no_memory) },
1345	{ "stop_queue", offsetof(struct netvsc_ethtool_stats, stop_queue) },
1346	{ "wake_queue", offsetof(struct netvsc_ethtool_stats, wake_queue) },
1347}, pcpu_stats[] = {
1348	{ "cpu%u_rx_packets",
1349		offsetof(struct netvsc_ethtool_pcpu_stats, rx_packets) },
1350	{ "cpu%u_rx_bytes",
1351		offsetof(struct netvsc_ethtool_pcpu_stats, rx_bytes) },
1352	{ "cpu%u_tx_packets",
1353		offsetof(struct netvsc_ethtool_pcpu_stats, tx_packets) },
1354	{ "cpu%u_tx_bytes",
1355		offsetof(struct netvsc_ethtool_pcpu_stats, tx_bytes) },
1356	{ "cpu%u_vf_rx_packets",
1357		offsetof(struct netvsc_ethtool_pcpu_stats, vf_rx_packets) },
1358	{ "cpu%u_vf_rx_bytes",
1359		offsetof(struct netvsc_ethtool_pcpu_stats, vf_rx_bytes) },
1360	{ "cpu%u_vf_tx_packets",
1361		offsetof(struct netvsc_ethtool_pcpu_stats, vf_tx_packets) },
1362	{ "cpu%u_vf_tx_bytes",
1363		offsetof(struct netvsc_ethtool_pcpu_stats, vf_tx_bytes) },
1364}, vf_stats[] = {
1365	{ "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) },
1366	{ "vf_rx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) },
1367	{ "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) },
1368	{ "vf_tx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) },
1369	{ "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) },
1370};
1371
1372#define NETVSC_GLOBAL_STATS_LEN	ARRAY_SIZE(netvsc_stats)
1373#define NETVSC_VF_STATS_LEN	ARRAY_SIZE(vf_stats)
1374
1375/* statistics per queue (rx/tx packets/bytes) */
1376#define NETVSC_PCPU_STATS_LEN (num_present_cpus() * ARRAY_SIZE(pcpu_stats))
1377
1378/* 4 statistics per queue (rx/tx packets/bytes) */
1379#define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
1380
1381static int netvsc_get_sset_count(struct net_device *dev, int string_set)
1382{
1383	struct net_device_context *ndc = netdev_priv(dev);
1384	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1385
1386	if (!nvdev)
1387		return -ENODEV;
1388
1389	switch (string_set) {
1390	case ETH_SS_STATS:
1391		return NETVSC_GLOBAL_STATS_LEN
1392			+ NETVSC_VF_STATS_LEN
1393			+ NETVSC_QUEUE_STATS_LEN(nvdev)
1394			+ NETVSC_PCPU_STATS_LEN;
1395	default:
1396		return -EINVAL;
1397	}
1398}
1399
1400static void netvsc_get_ethtool_stats(struct net_device *dev,
1401				     struct ethtool_stats *stats, u64 *data)
1402{
1403	struct net_device_context *ndc = netdev_priv(dev);
1404	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1405	const void *nds = &ndc->eth_stats;
1406	const struct netvsc_stats *qstats;
1407	struct netvsc_vf_pcpu_stats sum;
1408	struct netvsc_ethtool_pcpu_stats *pcpu_sum;
1409	unsigned int start;
1410	u64 packets, bytes;
1411	int i, j, cpu;
1412
1413	if (!nvdev)
1414		return;
1415
1416	for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
1417		data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
1418
1419	netvsc_get_vf_stats(dev, &sum);
1420	for (j = 0; j < NETVSC_VF_STATS_LEN; j++)
1421		data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset);
1422
1423	for (j = 0; j < nvdev->num_chn; j++) {
1424		qstats = &nvdev->chan_table[j].tx_stats;
1425
1426		do {
1427			start = u64_stats_fetch_begin_irq(&qstats->syncp);
1428			packets = qstats->packets;
1429			bytes = qstats->bytes;
1430		} while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1431		data[i++] = packets;
1432		data[i++] = bytes;
1433
1434		qstats = &nvdev->chan_table[j].rx_stats;
1435		do {
1436			start = u64_stats_fetch_begin_irq(&qstats->syncp);
1437			packets = qstats->packets;
1438			bytes = qstats->bytes;
1439		} while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1440		data[i++] = packets;
1441		data[i++] = bytes;
1442	}
1443
1444	pcpu_sum = kvmalloc_array(num_possible_cpus(),
1445				  sizeof(struct netvsc_ethtool_pcpu_stats),
1446				  GFP_KERNEL);
1447	netvsc_get_pcpu_stats(dev, pcpu_sum);
1448	for_each_present_cpu(cpu) {
1449		struct netvsc_ethtool_pcpu_stats *this_sum = &pcpu_sum[cpu];
1450
1451		for (j = 0; j < ARRAY_SIZE(pcpu_stats); j++)
1452			data[i++] = *(u64 *)((void *)this_sum
1453					     + pcpu_stats[j].offset);
1454	}
1455	kvfree(pcpu_sum);
1456}
1457
1458static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1459{
1460	struct net_device_context *ndc = netdev_priv(dev);
1461	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1462	u8 *p = data;
1463	int i, cpu;
1464
1465	if (!nvdev)
1466		return;
1467
1468	switch (stringset) {
1469	case ETH_SS_STATS:
1470		for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) {
1471			memcpy(p, netvsc_stats[i].name, ETH_GSTRING_LEN);
1472			p += ETH_GSTRING_LEN;
1473		}
1474
1475		for (i = 0; i < ARRAY_SIZE(vf_stats); i++) {
1476			memcpy(p, vf_stats[i].name, ETH_GSTRING_LEN);
1477			p += ETH_GSTRING_LEN;
1478		}
1479
1480		for (i = 0; i < nvdev->num_chn; i++) {
1481			sprintf(p, "tx_queue_%u_packets", i);
1482			p += ETH_GSTRING_LEN;
1483			sprintf(p, "tx_queue_%u_bytes", i);
1484			p += ETH_GSTRING_LEN;
1485			sprintf(p, "rx_queue_%u_packets", i);
1486			p += ETH_GSTRING_LEN;
1487			sprintf(p, "rx_queue_%u_bytes", i);
1488			p += ETH_GSTRING_LEN;
1489		}
1490
1491		for_each_present_cpu(cpu) {
1492			for (i = 0; i < ARRAY_SIZE(pcpu_stats); i++) {
1493				sprintf(p, pcpu_stats[i].name, cpu);
1494				p += ETH_GSTRING_LEN;
1495			}
1496		}
1497
1498		break;
1499	}
1500}
1501
1502static int
1503netvsc_get_rss_hash_opts(struct net_device_context *ndc,
1504			 struct ethtool_rxnfc *info)
1505{
1506	const u32 l4_flag = RXH_L4_B_0_1 | RXH_L4_B_2_3;
1507
1508	info->data = RXH_IP_SRC | RXH_IP_DST;
1509
1510	switch (info->flow_type) {
1511	case TCP_V4_FLOW:
1512		if (ndc->l4_hash & HV_TCP4_L4HASH)
1513			info->data |= l4_flag;
1514
1515		break;
1516
1517	case TCP_V6_FLOW:
1518		if (ndc->l4_hash & HV_TCP6_L4HASH)
1519			info->data |= l4_flag;
1520
1521		break;
1522
1523	case UDP_V4_FLOW:
1524		if (ndc->l4_hash & HV_UDP4_L4HASH)
1525			info->data |= l4_flag;
1526
1527		break;
1528
1529	case UDP_V6_FLOW:
1530		if (ndc->l4_hash & HV_UDP6_L4HASH)
1531			info->data |= l4_flag;
1532
1533		break;
1534
1535	case IPV4_FLOW:
1536	case IPV6_FLOW:
1537		break;
1538	default:
1539		info->data = 0;
1540		break;
1541	}
1542
1543	return 0;
1544}
1545
1546static int
1547netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1548		 u32 *rules)
1549{
1550	struct net_device_context *ndc = netdev_priv(dev);
1551	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1552
1553	if (!nvdev)
1554		return -ENODEV;
1555
1556	switch (info->cmd) {
1557	case ETHTOOL_GRXRINGS:
1558		info->data = nvdev->num_chn;
1559		return 0;
1560
1561	case ETHTOOL_GRXFH:
1562		return netvsc_get_rss_hash_opts(ndc, info);
1563	}
1564	return -EOPNOTSUPP;
1565}
1566
1567static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
1568				    struct ethtool_rxnfc *info)
1569{
1570	if (info->data == (RXH_IP_SRC | RXH_IP_DST |
1571			   RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
1572		switch (info->flow_type) {
1573		case TCP_V4_FLOW:
1574			ndc->l4_hash |= HV_TCP4_L4HASH;
1575			break;
1576
1577		case TCP_V6_FLOW:
1578			ndc->l4_hash |= HV_TCP6_L4HASH;
1579			break;
1580
1581		case UDP_V4_FLOW:
1582			ndc->l4_hash |= HV_UDP4_L4HASH;
1583			break;
1584
1585		case UDP_V6_FLOW:
1586			ndc->l4_hash |= HV_UDP6_L4HASH;
1587			break;
1588
1589		default:
1590			return -EOPNOTSUPP;
1591		}
1592
1593		return 0;
1594	}
1595
1596	if (info->data == (RXH_IP_SRC | RXH_IP_DST)) {
1597		switch (info->flow_type) {
1598		case TCP_V4_FLOW:
1599			ndc->l4_hash &= ~HV_TCP4_L4HASH;
1600			break;
1601
1602		case TCP_V6_FLOW:
1603			ndc->l4_hash &= ~HV_TCP6_L4HASH;
1604			break;
1605
1606		case UDP_V4_FLOW:
1607			ndc->l4_hash &= ~HV_UDP4_L4HASH;
1608			break;
1609
1610		case UDP_V6_FLOW:
1611			ndc->l4_hash &= ~HV_UDP6_L4HASH;
1612			break;
1613
1614		default:
1615			return -EOPNOTSUPP;
1616		}
1617
1618		return 0;
1619	}
1620
1621	return -EOPNOTSUPP;
1622}
1623
1624static int
1625netvsc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info)
1626{
1627	struct net_device_context *ndc = netdev_priv(ndev);
1628
1629	if (info->cmd == ETHTOOL_SRXFH)
1630		return netvsc_set_rss_hash_opts(ndc, info);
1631
1632	return -EOPNOTSUPP;
1633}
1634
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1635static u32 netvsc_get_rxfh_key_size(struct net_device *dev)
1636{
1637	return NETVSC_HASH_KEYLEN;
1638}
1639
1640static u32 netvsc_rss_indir_size(struct net_device *dev)
1641{
1642	return ITAB_NUM;
1643}
1644
1645static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
1646			   u8 *hfunc)
1647{
1648	struct net_device_context *ndc = netdev_priv(dev);
1649	struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1650	struct rndis_device *rndis_dev;
1651	int i;
1652
1653	if (!ndev)
1654		return -ENODEV;
1655
1656	if (hfunc)
1657		*hfunc = ETH_RSS_HASH_TOP;	/* Toeplitz */
1658
1659	rndis_dev = ndev->extension;
1660	if (indir) {
1661		for (i = 0; i < ITAB_NUM; i++)
1662			indir[i] = rndis_dev->rx_table[i];
1663	}
1664
1665	if (key)
1666		memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN);
1667
1668	return 0;
1669}
1670
1671static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir,
1672			   const u8 *key, const u8 hfunc)
1673{
1674	struct net_device_context *ndc = netdev_priv(dev);
1675	struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1676	struct rndis_device *rndis_dev;
1677	int i;
1678
1679	if (!ndev)
1680		return -ENODEV;
1681
1682	if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
1683		return -EOPNOTSUPP;
1684
1685	rndis_dev = ndev->extension;
1686	if (indir) {
1687		for (i = 0; i < ITAB_NUM; i++)
1688			if (indir[i] >= ndev->num_chn)
1689				return -EINVAL;
1690
1691		for (i = 0; i < ITAB_NUM; i++)
1692			rndis_dev->rx_table[i] = indir[i];
1693	}
1694
1695	if (!key) {
1696		if (!indir)
1697			return 0;
1698
1699		key = rndis_dev->rss_key;
1700	}
1701
1702	return rndis_filter_set_rss_param(rndis_dev, key);
1703}
1704
1705/* Hyper-V RNDIS protocol does not have ring in the HW sense.
1706 * It does have pre-allocated receive area which is divided into sections.
1707 */
1708static void __netvsc_get_ringparam(struct netvsc_device *nvdev,
1709				   struct ethtool_ringparam *ring)
1710{
1711	u32 max_buf_size;
1712
1713	ring->rx_pending = nvdev->recv_section_cnt;
1714	ring->tx_pending = nvdev->send_section_cnt;
1715
1716	if (nvdev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
1717		max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
1718	else
1719		max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
1720
1721	ring->rx_max_pending = max_buf_size / nvdev->recv_section_size;
1722	ring->tx_max_pending = NETVSC_SEND_BUFFER_SIZE
1723		/ nvdev->send_section_size;
1724}
1725
1726static void netvsc_get_ringparam(struct net_device *ndev,
1727				 struct ethtool_ringparam *ring)
1728{
1729	struct net_device_context *ndevctx = netdev_priv(ndev);
1730	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1731
1732	if (!nvdev)
1733		return;
1734
1735	__netvsc_get_ringparam(nvdev, ring);
1736}
1737
1738static int netvsc_set_ringparam(struct net_device *ndev,
1739				struct ethtool_ringparam *ring)
1740{
1741	struct net_device_context *ndevctx = netdev_priv(ndev);
1742	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1743	struct netvsc_device_info *device_info;
1744	struct ethtool_ringparam orig;
1745	u32 new_tx, new_rx;
1746	int ret = 0;
1747
1748	if (!nvdev || nvdev->destroy)
1749		return -ENODEV;
1750
1751	memset(&orig, 0, sizeof(orig));
1752	__netvsc_get_ringparam(nvdev, &orig);
1753
1754	new_tx = clamp_t(u32, ring->tx_pending,
1755			 NETVSC_MIN_TX_SECTIONS, orig.tx_max_pending);
1756	new_rx = clamp_t(u32, ring->rx_pending,
1757			 NETVSC_MIN_RX_SECTIONS, orig.rx_max_pending);
1758
1759	if (new_tx == orig.tx_pending &&
1760	    new_rx == orig.rx_pending)
1761		return 0;	 /* no change */
1762
1763	device_info = netvsc_devinfo_get(nvdev);
1764
1765	if (!device_info)
1766		return -ENOMEM;
1767
1768	device_info->send_sections = new_tx;
1769	device_info->recv_sections = new_rx;
1770
1771	ret = netvsc_detach(ndev, nvdev);
1772	if (ret)
1773		goto out;
1774
1775	ret = netvsc_attach(ndev, device_info);
1776	if (ret) {
1777		device_info->send_sections = orig.tx_pending;
1778		device_info->recv_sections = orig.rx_pending;
1779
1780		if (netvsc_attach(ndev, device_info))
1781			netdev_err(ndev, "restoring ringparam failed");
1782	}
1783
1784out:
1785	kfree(device_info);
1786	return ret;
1787}
1788
1789static int netvsc_set_features(struct net_device *ndev,
1790			       netdev_features_t features)
1791{
1792	netdev_features_t change = features ^ ndev->features;
1793	struct net_device_context *ndevctx = netdev_priv(ndev);
1794	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1795	struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
1796	struct ndis_offload_params offloads;
1797	int ret = 0;
1798
1799	if (!nvdev || nvdev->destroy)
1800		return -ENODEV;
1801
1802	if (!(change & NETIF_F_LRO))
1803		goto syncvf;
1804
1805	memset(&offloads, 0, sizeof(struct ndis_offload_params));
1806
1807	if (features & NETIF_F_LRO) {
1808		offloads.rsc_ip_v4 = NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED;
1809		offloads.rsc_ip_v6 = NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED;
1810	} else {
1811		offloads.rsc_ip_v4 = NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED;
1812		offloads.rsc_ip_v6 = NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED;
1813	}
1814
1815	ret = rndis_filter_set_offload_params(ndev, nvdev, &offloads);
1816
1817	if (ret) {
1818		features ^= NETIF_F_LRO;
1819		ndev->features = features;
1820	}
1821
1822syncvf:
1823	if (!vf_netdev)
1824		return ret;
1825
1826	vf_netdev->wanted_features = features;
1827	netdev_update_features(vf_netdev);
1828
1829	return ret;
1830}
1831
1832static u32 netvsc_get_msglevel(struct net_device *ndev)
1833{
1834	struct net_device_context *ndev_ctx = netdev_priv(ndev);
1835
1836	return ndev_ctx->msg_enable;
1837}
1838
1839static void netvsc_set_msglevel(struct net_device *ndev, u32 val)
1840{
1841	struct net_device_context *ndev_ctx = netdev_priv(ndev);
1842
1843	ndev_ctx->msg_enable = val;
1844}
1845
1846static const struct ethtool_ops ethtool_ops = {
1847	.get_drvinfo	= netvsc_get_drvinfo,
1848	.get_msglevel	= netvsc_get_msglevel,
1849	.set_msglevel	= netvsc_set_msglevel,
1850	.get_link	= ethtool_op_get_link,
1851	.get_ethtool_stats = netvsc_get_ethtool_stats,
1852	.get_sset_count = netvsc_get_sset_count,
1853	.get_strings	= netvsc_get_strings,
1854	.get_channels   = netvsc_get_channels,
1855	.set_channels   = netvsc_set_channels,
1856	.get_ts_info	= ethtool_op_get_ts_info,
1857	.get_rxnfc	= netvsc_get_rxnfc,
1858	.set_rxnfc	= netvsc_set_rxnfc,
1859	.get_rxfh_key_size = netvsc_get_rxfh_key_size,
1860	.get_rxfh_indir_size = netvsc_rss_indir_size,
1861	.get_rxfh	= netvsc_get_rxfh,
1862	.set_rxfh	= netvsc_set_rxfh,
1863	.get_link_ksettings = netvsc_get_link_ksettings,
1864	.set_link_ksettings = netvsc_set_link_ksettings,
1865	.get_ringparam	= netvsc_get_ringparam,
1866	.set_ringparam	= netvsc_set_ringparam,
1867};
1868
1869static const struct net_device_ops device_ops = {
1870	.ndo_open =			netvsc_open,
1871	.ndo_stop =			netvsc_close,
1872	.ndo_start_xmit =		netvsc_start_xmit,
1873	.ndo_change_rx_flags =		netvsc_change_rx_flags,
1874	.ndo_set_rx_mode =		netvsc_set_rx_mode,
1875	.ndo_set_features =		netvsc_set_features,
1876	.ndo_change_mtu =		netvsc_change_mtu,
1877	.ndo_validate_addr =		eth_validate_addr,
1878	.ndo_set_mac_address =		netvsc_set_mac_addr,
1879	.ndo_select_queue =		netvsc_select_queue,
1880	.ndo_get_stats64 =		netvsc_get_stats64,
 
 
 
1881};
1882
1883/*
1884 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
1885 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
1886 * present send GARP packet to network peers with netif_notify_peers().
1887 */
1888static void netvsc_link_change(struct work_struct *w)
1889{
1890	struct net_device_context *ndev_ctx =
1891		container_of(w, struct net_device_context, dwork.work);
1892	struct hv_device *device_obj = ndev_ctx->device_ctx;
1893	struct net_device *net = hv_get_drvdata(device_obj);
1894	struct netvsc_device *net_device;
1895	struct rndis_device *rdev;
1896	struct netvsc_reconfig *event = NULL;
1897	bool notify = false, reschedule = false;
1898	unsigned long flags, next_reconfig, delay;
1899
1900	/* if changes are happening, comeback later */
1901	if (!rtnl_trylock()) {
1902		schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1903		return;
1904	}
1905
1906	net_device = rtnl_dereference(ndev_ctx->nvdev);
1907	if (!net_device)
1908		goto out_unlock;
1909
1910	rdev = net_device->extension;
1911
1912	next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT;
1913	if (time_is_after_jiffies(next_reconfig)) {
1914		/* link_watch only sends one notification with current state
1915		 * per second, avoid doing reconfig more frequently. Handle
1916		 * wrap around.
1917		 */
1918		delay = next_reconfig - jiffies;
1919		delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT;
1920		schedule_delayed_work(&ndev_ctx->dwork, delay);
1921		goto out_unlock;
1922	}
1923	ndev_ctx->last_reconfig = jiffies;
1924
1925	spin_lock_irqsave(&ndev_ctx->lock, flags);
1926	if (!list_empty(&ndev_ctx->reconfig_events)) {
1927		event = list_first_entry(&ndev_ctx->reconfig_events,
1928					 struct netvsc_reconfig, list);
1929		list_del(&event->list);
1930		reschedule = !list_empty(&ndev_ctx->reconfig_events);
1931	}
1932	spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1933
1934	if (!event)
1935		goto out_unlock;
1936
1937	switch (event->event) {
1938		/* Only the following events are possible due to the check in
1939		 * netvsc_linkstatus_callback()
1940		 */
1941	case RNDIS_STATUS_MEDIA_CONNECT:
1942		if (rdev->link_state) {
1943			rdev->link_state = false;
1944			netif_carrier_on(net);
1945			netvsc_tx_enable(net_device, net);
1946		} else {
1947			notify = true;
1948		}
1949		kfree(event);
1950		break;
1951	case RNDIS_STATUS_MEDIA_DISCONNECT:
1952		if (!rdev->link_state) {
1953			rdev->link_state = true;
1954			netif_carrier_off(net);
1955			netvsc_tx_disable(net_device, net);
1956		}
1957		kfree(event);
1958		break;
1959	case RNDIS_STATUS_NETWORK_CHANGE:
1960		/* Only makes sense if carrier is present */
1961		if (!rdev->link_state) {
1962			rdev->link_state = true;
1963			netif_carrier_off(net);
1964			netvsc_tx_disable(net_device, net);
1965			event->event = RNDIS_STATUS_MEDIA_CONNECT;
1966			spin_lock_irqsave(&ndev_ctx->lock, flags);
1967			list_add(&event->list, &ndev_ctx->reconfig_events);
1968			spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1969			reschedule = true;
1970		}
1971		break;
1972	}
1973
1974	rtnl_unlock();
1975
1976	if (notify)
1977		netdev_notify_peers(net);
1978
1979	/* link_watch only sends one notification with current state per
1980	 * second, handle next reconfig event in 2 seconds.
1981	 */
1982	if (reschedule)
1983		schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1984
1985	return;
1986
1987out_unlock:
1988	rtnl_unlock();
1989}
1990
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1991static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
1992{
1993	struct net_device_context *net_device_ctx;
1994	struct net_device *dev;
1995
1996	dev = netdev_master_upper_dev_get(vf_netdev);
1997	if (!dev || dev->netdev_ops != &device_ops)
1998		return NULL;	/* not a netvsc device */
1999
2000	net_device_ctx = netdev_priv(dev);
2001	if (!rtnl_dereference(net_device_ctx->nvdev))
2002		return NULL;	/* device is removed */
2003
2004	return dev;
 
 
 
 
 
 
 
 
 
 
 
2005}
2006
2007/* Called when VF is injecting data into network stack.
2008 * Change the associated network device from VF to netvsc.
2009 * note: already called with rcu_read_lock
2010 */
2011static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb)
2012{
2013	struct sk_buff *skb = *pskb;
2014	struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data);
2015	struct net_device_context *ndev_ctx = netdev_priv(ndev);
2016	struct netvsc_vf_pcpu_stats *pcpu_stats
2017		 = this_cpu_ptr(ndev_ctx->vf_stats);
2018
2019	skb = skb_share_check(skb, GFP_ATOMIC);
2020	if (unlikely(!skb))
2021		return RX_HANDLER_CONSUMED;
2022
2023	*pskb = skb;
2024
2025	skb->dev = ndev;
2026
2027	u64_stats_update_begin(&pcpu_stats->syncp);
2028	pcpu_stats->rx_packets++;
2029	pcpu_stats->rx_bytes += skb->len;
2030	u64_stats_update_end(&pcpu_stats->syncp);
2031
2032	return RX_HANDLER_ANOTHER;
2033}
2034
2035static int netvsc_vf_join(struct net_device *vf_netdev,
2036			  struct net_device *ndev)
2037{
2038	struct net_device_context *ndev_ctx = netdev_priv(ndev);
2039	int ret;
2040
2041	ret = netdev_rx_handler_register(vf_netdev,
2042					 netvsc_vf_handle_frame, ndev);
2043	if (ret != 0) {
2044		netdev_err(vf_netdev,
2045			   "can not register netvsc VF receive handler (err = %d)\n",
2046			   ret);
2047		goto rx_handler_failed;
2048	}
2049
2050	ret = netdev_master_upper_dev_link(vf_netdev, ndev,
2051					   NULL, NULL, NULL);
2052	if (ret != 0) {
2053		netdev_err(vf_netdev,
2054			   "can not set master device %s (err = %d)\n",
2055			   ndev->name, ret);
2056		goto upper_link_failed;
2057	}
2058
2059	/* set slave flag before open to prevent IPv6 addrconf */
2060	vf_netdev->flags |= IFF_SLAVE;
2061
2062	schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
2063
2064	call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
2065
2066	netdev_info(vf_netdev, "joined to %s\n", ndev->name);
2067	return 0;
2068
2069upper_link_failed:
2070	netdev_rx_handler_unregister(vf_netdev);
2071rx_handler_failed:
2072	return ret;
2073}
2074
2075static void __netvsc_vf_setup(struct net_device *ndev,
2076			      struct net_device *vf_netdev)
2077{
2078	int ret;
2079
2080	/* Align MTU of VF with master */
2081	ret = dev_set_mtu(vf_netdev, ndev->mtu);
2082	if (ret)
2083		netdev_warn(vf_netdev,
2084			    "unable to change mtu to %u\n", ndev->mtu);
2085
2086	/* set multicast etc flags on VF */
2087	dev_change_flags(vf_netdev, ndev->flags | IFF_SLAVE, NULL);
2088
2089	/* sync address list from ndev to VF */
2090	netif_addr_lock_bh(ndev);
2091	dev_uc_sync(vf_netdev, ndev);
2092	dev_mc_sync(vf_netdev, ndev);
2093	netif_addr_unlock_bh(ndev);
2094
2095	if (netif_running(ndev)) {
2096		ret = dev_open(vf_netdev, NULL);
2097		if (ret)
2098			netdev_warn(vf_netdev,
2099				    "unable to open: %d\n", ret);
2100	}
2101}
2102
2103/* Setup VF as slave of the synthetic device.
2104 * Runs in workqueue to avoid recursion in netlink callbacks.
2105 */
2106static void netvsc_vf_setup(struct work_struct *w)
2107{
2108	struct net_device_context *ndev_ctx
2109		= container_of(w, struct net_device_context, vf_takeover.work);
2110	struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx);
2111	struct net_device *vf_netdev;
2112
2113	if (!rtnl_trylock()) {
2114		schedule_delayed_work(&ndev_ctx->vf_takeover, 0);
2115		return;
2116	}
2117
2118	vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
2119	if (vf_netdev)
2120		__netvsc_vf_setup(ndev, vf_netdev);
2121
2122	rtnl_unlock();
2123}
2124
2125/* Find netvsc by VF serial number.
2126 * The PCI hyperv controller records the serial number as the slot kobj name.
2127 */
2128static struct net_device *get_netvsc_byslot(const struct net_device *vf_netdev)
2129{
2130	struct device *parent = vf_netdev->dev.parent;
2131	struct net_device_context *ndev_ctx;
2132	struct pci_dev *pdev;
2133	u32 serial;
2134
2135	if (!parent || !dev_is_pci(parent))
2136		return NULL; /* not a PCI device */
2137
2138	pdev = to_pci_dev(parent);
2139	if (!pdev->slot) {
2140		netdev_notice(vf_netdev, "no PCI slot information\n");
2141		return NULL;
2142	}
2143
2144	if (kstrtou32(pci_slot_name(pdev->slot), 10, &serial)) {
2145		netdev_notice(vf_netdev, "Invalid vf serial:%s\n",
2146			      pci_slot_name(pdev->slot));
2147		return NULL;
2148	}
2149
2150	list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) {
2151		if (!ndev_ctx->vf_alloc)
2152			continue;
2153
2154		if (ndev_ctx->vf_serial == serial)
2155			return hv_get_drvdata(ndev_ctx->device_ctx);
2156	}
2157
2158	netdev_notice(vf_netdev,
2159		      "no netdev found for vf serial:%u\n", serial);
2160	return NULL;
2161}
2162
2163static int netvsc_register_vf(struct net_device *vf_netdev)
2164{
 
2165	struct net_device_context *net_device_ctx;
2166	struct netvsc_device *netvsc_dev;
2167	struct net_device *ndev;
2168	int ret;
2169
2170	if (vf_netdev->addr_len != ETH_ALEN)
2171		return NOTIFY_DONE;
2172
2173	ndev = get_netvsc_byslot(vf_netdev);
 
 
 
 
 
2174	if (!ndev)
2175		return NOTIFY_DONE;
2176
2177	net_device_ctx = netdev_priv(ndev);
2178	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
2179	if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
2180		return NOTIFY_DONE;
2181
2182	/* if synthetic interface is a different namespace,
2183	 * then move the VF to that namespace; join will be
2184	 * done again in that context.
2185	 */
2186	if (!net_eq(dev_net(ndev), dev_net(vf_netdev))) {
2187		ret = dev_change_net_namespace(vf_netdev,
2188					       dev_net(ndev), "eth%d");
2189		if (ret)
2190			netdev_err(vf_netdev,
2191				   "could not move to same namespace as %s: %d\n",
2192				   ndev->name, ret);
2193		else
2194			netdev_info(vf_netdev,
2195				    "VF moved to namespace with: %s\n",
2196				    ndev->name);
2197		return NOTIFY_DONE;
2198	}
2199
2200	netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
2201
2202	if (netvsc_vf_join(vf_netdev, ndev) != 0)
2203		return NOTIFY_DONE;
2204
2205	dev_hold(vf_netdev);
2206	rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
2207
2208	vf_netdev->wanted_features = ndev->features;
2209	netdev_update_features(vf_netdev);
2210
2211	return NOTIFY_OK;
2212}
2213
2214/* VF up/down change detected, schedule to change data path */
2215static int netvsc_vf_changed(struct net_device *vf_netdev)
2216{
2217	struct net_device_context *net_device_ctx;
2218	struct netvsc_device *netvsc_dev;
2219	struct net_device *ndev;
2220	bool vf_is_up = netif_running(vf_netdev);
2221
2222	ndev = get_netvsc_byref(vf_netdev);
2223	if (!ndev)
2224		return NOTIFY_DONE;
2225
2226	net_device_ctx = netdev_priv(ndev);
2227	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
2228	if (!netvsc_dev)
2229		return NOTIFY_DONE;
2230
2231	netvsc_switch_datapath(ndev, vf_is_up);
2232	netdev_info(ndev, "Data path switched %s VF: %s\n",
2233		    vf_is_up ? "to" : "from", vf_netdev->name);
2234
2235	return NOTIFY_OK;
2236}
2237
2238static int netvsc_unregister_vf(struct net_device *vf_netdev)
2239{
2240	struct net_device *ndev;
2241	struct net_device_context *net_device_ctx;
2242
2243	ndev = get_netvsc_byref(vf_netdev);
2244	if (!ndev)
2245		return NOTIFY_DONE;
2246
2247	net_device_ctx = netdev_priv(ndev);
2248	cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
2249
2250	netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
2251
2252	netdev_rx_handler_unregister(vf_netdev);
2253	netdev_upper_dev_unlink(vf_netdev, ndev);
2254	RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
2255	dev_put(vf_netdev);
2256
2257	return NOTIFY_OK;
2258}
2259
2260static int netvsc_probe(struct hv_device *dev,
2261			const struct hv_vmbus_device_id *dev_id)
2262{
2263	struct net_device *net = NULL;
2264	struct net_device_context *net_device_ctx;
2265	struct netvsc_device_info *device_info = NULL;
2266	struct netvsc_device *nvdev;
2267	int ret = -ENOMEM;
2268
2269	net = alloc_etherdev_mq(sizeof(struct net_device_context),
2270				VRSS_CHANNEL_MAX);
2271	if (!net)
2272		goto no_net;
2273
2274	netif_carrier_off(net);
2275
2276	netvsc_init_settings(net);
2277
2278	net_device_ctx = netdev_priv(net);
2279	net_device_ctx->device_ctx = dev;
2280	net_device_ctx->msg_enable = netif_msg_init(debug, default_msg);
2281	if (netif_msg_probe(net_device_ctx))
2282		netdev_dbg(net, "netvsc msg_enable: %d\n",
2283			   net_device_ctx->msg_enable);
2284
2285	hv_set_drvdata(dev, net);
2286
2287	INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change);
2288
2289	spin_lock_init(&net_device_ctx->lock);
2290	INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
2291	INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup);
2292
2293	net_device_ctx->vf_stats
2294		= netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats);
2295	if (!net_device_ctx->vf_stats)
2296		goto no_stats;
2297
2298	net->netdev_ops = &device_ops;
2299	net->ethtool_ops = &ethtool_ops;
2300	SET_NETDEV_DEV(net, &dev->device);
2301
2302	/* We always need headroom for rndis header */
2303	net->needed_headroom = RNDIS_AND_PPI_SIZE;
2304
2305	/* Initialize the number of queues to be 1, we may change it if more
2306	 * channels are offered later.
2307	 */
2308	netif_set_real_num_tx_queues(net, 1);
2309	netif_set_real_num_rx_queues(net, 1);
2310
2311	/* Notify the netvsc driver of the new device */
2312	device_info = netvsc_devinfo_get(NULL);
2313
2314	if (!device_info) {
2315		ret = -ENOMEM;
2316		goto devinfo_failed;
2317	}
2318
2319	nvdev = rndis_filter_device_add(dev, device_info);
2320	if (IS_ERR(nvdev)) {
2321		ret = PTR_ERR(nvdev);
2322		netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
2323		goto rndis_failed;
2324	}
2325
2326	memcpy(net->dev_addr, device_info->mac_adr, ETH_ALEN);
2327
2328	/* We must get rtnl lock before scheduling nvdev->subchan_work,
2329	 * otherwise netvsc_subchan_work() can get rtnl lock first and wait
2330	 * all subchannels to show up, but that may not happen because
2331	 * netvsc_probe() can't get rtnl lock and as a result vmbus_onoffer()
2332	 * -> ... -> device_add() -> ... -> __device_attach() can't get
2333	 * the device lock, so all the subchannels can't be processed --
2334	 * finally netvsc_subchan_work() hangs forever.
2335	 */
2336	rtnl_lock();
2337
2338	if (nvdev->num_chn > 1)
2339		schedule_work(&nvdev->subchan_work);
2340
2341	/* hw_features computed in rndis_netdev_set_hwcaps() */
2342	net->features = net->hw_features |
2343		NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX |
2344		NETIF_F_HW_VLAN_CTAG_RX;
2345	net->vlan_features = net->features;
2346
 
 
2347	/* MTU range: 68 - 1500 or 65521 */
2348	net->min_mtu = NETVSC_MTU_MIN;
2349	if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
2350		net->max_mtu = NETVSC_MTU - ETH_HLEN;
2351	else
2352		net->max_mtu = ETH_DATA_LEN;
2353
2354	ret = register_netdevice(net);
2355	if (ret != 0) {
2356		pr_err("Unable to register netdev.\n");
2357		goto register_failed;
2358	}
2359
2360	list_add(&net_device_ctx->list, &netvsc_dev_list);
2361	rtnl_unlock();
2362
2363	kfree(device_info);
2364	return 0;
2365
2366register_failed:
2367	rtnl_unlock();
2368	rndis_filter_device_remove(dev, nvdev);
2369rndis_failed:
2370	kfree(device_info);
2371devinfo_failed:
2372	free_percpu(net_device_ctx->vf_stats);
2373no_stats:
2374	hv_set_drvdata(dev, NULL);
2375	free_netdev(net);
2376no_net:
2377	return ret;
2378}
2379
2380static int netvsc_remove(struct hv_device *dev)
2381{
2382	struct net_device_context *ndev_ctx;
2383	struct net_device *vf_netdev, *net;
2384	struct netvsc_device *nvdev;
2385
2386	net = hv_get_drvdata(dev);
2387	if (net == NULL) {
2388		dev_err(&dev->device, "No net device to remove\n");
2389		return 0;
2390	}
2391
2392	ndev_ctx = netdev_priv(net);
2393
2394	cancel_delayed_work_sync(&ndev_ctx->dwork);
2395
2396	rtnl_lock();
2397	nvdev = rtnl_dereference(ndev_ctx->nvdev);
2398	if (nvdev)
 
2399		cancel_work_sync(&nvdev->subchan_work);
2400
2401	/*
2402	 * Call to the vsc driver to let it know that the device is being
2403	 * removed. Also blocks mtu and channel changes.
2404	 */
 
2405	vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
2406	if (vf_netdev)
2407		netvsc_unregister_vf(vf_netdev);
2408
2409	if (nvdev)
2410		rndis_filter_device_remove(dev, nvdev);
2411
2412	unregister_netdevice(net);
2413	list_del(&ndev_ctx->list);
2414
2415	rtnl_unlock();
 
2416
2417	hv_set_drvdata(dev, NULL);
2418
2419	free_percpu(ndev_ctx->vf_stats);
2420	free_netdev(net);
2421	return 0;
2422}
2423
2424static const struct hv_vmbus_device_id id_table[] = {
2425	/* Network guid */
2426	{ HV_NIC_GUID, },
2427	{ },
2428};
2429
2430MODULE_DEVICE_TABLE(vmbus, id_table);
2431
2432/* The one and only one */
2433static struct  hv_driver netvsc_drv = {
2434	.name = KBUILD_MODNAME,
2435	.id_table = id_table,
2436	.probe = netvsc_probe,
2437	.remove = netvsc_remove,
2438	.driver = {
2439		.probe_type = PROBE_FORCE_SYNCHRONOUS,
2440	},
2441};
2442
2443/*
2444 * On Hyper-V, every VF interface is matched with a corresponding
2445 * synthetic interface. The synthetic interface is presented first
2446 * to the guest. When the corresponding VF instance is registered,
2447 * we will take care of switching the data path.
2448 */
2449static int netvsc_netdev_event(struct notifier_block *this,
2450			       unsigned long event, void *ptr)
2451{
2452	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
2453
2454	/* Skip our own events */
2455	if (event_dev->netdev_ops == &device_ops)
2456		return NOTIFY_DONE;
2457
2458	/* Avoid non-Ethernet type devices */
2459	if (event_dev->type != ARPHRD_ETHER)
2460		return NOTIFY_DONE;
2461
2462	/* Avoid Vlan dev with same MAC registering as VF */
2463	if (is_vlan_dev(event_dev))
2464		return NOTIFY_DONE;
2465
2466	/* Avoid Bonding master dev with same MAC registering as VF */
2467	if ((event_dev->priv_flags & IFF_BONDING) &&
2468	    (event_dev->flags & IFF_MASTER))
2469		return NOTIFY_DONE;
2470
2471	switch (event) {
2472	case NETDEV_REGISTER:
2473		return netvsc_register_vf(event_dev);
2474	case NETDEV_UNREGISTER:
2475		return netvsc_unregister_vf(event_dev);
2476	case NETDEV_UP:
2477	case NETDEV_DOWN:
2478		return netvsc_vf_changed(event_dev);
2479	default:
2480		return NOTIFY_DONE;
2481	}
2482}
2483
2484static struct notifier_block netvsc_netdev_notifier = {
2485	.notifier_call = netvsc_netdev_event,
2486};
2487
2488static void __exit netvsc_drv_exit(void)
2489{
2490	unregister_netdevice_notifier(&netvsc_netdev_notifier);
2491	vmbus_driver_unregister(&netvsc_drv);
2492}
2493
2494static int __init netvsc_drv_init(void)
2495{
2496	int ret;
2497
2498	if (ring_size < RING_SIZE_MIN) {
2499		ring_size = RING_SIZE_MIN;
2500		pr_info("Increased ring_size to %u (min allowed)\n",
2501			ring_size);
2502	}
2503	netvsc_ring_bytes = ring_size * PAGE_SIZE;
 
2504
2505	ret = vmbus_driver_register(&netvsc_drv);
2506	if (ret)
2507		return ret;
2508
2509	register_netdevice_notifier(&netvsc_netdev_notifier);
2510	return 0;
2511}
2512
2513MODULE_LICENSE("GPL");
2514MODULE_DESCRIPTION("Microsoft Hyper-V network driver");
2515
2516module_init(netvsc_drv_init);
2517module_exit(netvsc_drv_exit);