1// SPDX-License-Identifier: GPL-2.0-or-later
   2/* A network driver using virtio.
   3 *
   4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
   5 */
   6//#define DEBUG
   7#include <linux/netdevice.h>
   8#include <linux/etherdevice.h>
   9#include <linux/ethtool.h>
  10#include <linux/module.h>
  11#include <linux/virtio.h>
  12#include <linux/virtio_net.h>
  13#include <linux/bpf.h>
  14#include <linux/bpf_trace.h>
  15#include <linux/scatterlist.h>
  16#include <linux/if_vlan.h>
  17#include <linux/slab.h>
  18#include <linux/cpu.h>
  19#include <linux/average.h>
  20#include <linux/filter.h>
  21#include <linux/kernel.h>
  22#include <net/route.h>
  23#include <net/xdp.h>
  24#include <net/net_failover.h>
  25
  26static int napi_weight = NAPI_POLL_WEIGHT;
  27module_param(napi_weight, int, 0444);
  28
  29static bool csum = true, gso = true, napi_tx = true;
  30module_param(csum, bool, 0444);
  31module_param(gso, bool, 0444);
  32module_param(napi_tx, bool, 0644);
  33
  34/* FIXME: MTU in config. */
  35#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
  36#define GOOD_COPY_LEN	128
  37
  38#define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
  39
  40/* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
  41#define VIRTIO_XDP_HEADROOM 256
  42
  43/* Separating two types of XDP xmit */
  44#define VIRTIO_XDP_TX		BIT(0)
  45#define VIRTIO_XDP_REDIR	BIT(1)
  46
  47#define VIRTIO_XDP_FLAG	BIT(0)
  48
  49/* RX packet size EWMA. The average packet size is used to determine the packet
  50 * buffer size when refilling RX rings. As the entire RX ring may be refilled
  51 * at once, the weight is chosen so that the EWMA will be insensitive to short-
  52 * term, transient changes in packet size.
  53 */
  54DECLARE_EWMA(pkt_len, 0, 64)
  55
  56#define VIRTNET_DRIVER_VERSION "1.0.0"
  57
  58static const unsigned long guest_offloads[] = {
  59	VIRTIO_NET_F_GUEST_TSO4,
  60	VIRTIO_NET_F_GUEST_TSO6,
  61	VIRTIO_NET_F_GUEST_ECN,
  62	VIRTIO_NET_F_GUEST_UFO,
  63	VIRTIO_NET_F_GUEST_CSUM
  64};
  65
  66#define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
  67				(1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
  68				(1ULL << VIRTIO_NET_F_GUEST_ECN)  | \
  69				(1ULL << VIRTIO_NET_F_GUEST_UFO))
  70
  71struct virtnet_stat_desc {
  72	char desc[ETH_GSTRING_LEN];
  73	size_t offset;
  74};
  75
  76struct virtnet_sq_stats {
  77	struct u64_stats_sync syncp;
  78	u64 packets;
  79	u64 bytes;
  80	u64 xdp_tx;
  81	u64 xdp_tx_drops;
  82	u64 kicks;
  83};
  84
  85struct virtnet_rq_stats {
  86	struct u64_stats_sync syncp;
  87	u64 packets;
  88	u64 bytes;
  89	u64 drops;
  90	u64 xdp_packets;
  91	u64 xdp_tx;
  92	u64 xdp_redirects;
  93	u64 xdp_drops;
  94	u64 kicks;
  95};
  96
  97#define VIRTNET_SQ_STAT(m)	offsetof(struct virtnet_sq_stats, m)
  98#define VIRTNET_RQ_STAT(m)	offsetof(struct virtnet_rq_stats, m)
  99
 100static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
 101	{ "packets",		VIRTNET_SQ_STAT(packets) },
 102	{ "bytes",		VIRTNET_SQ_STAT(bytes) },
 103	{ "xdp_tx",		VIRTNET_SQ_STAT(xdp_tx) },
 104	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
 105	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
 106};
 107
 108static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
 109	{ "packets",		VIRTNET_RQ_STAT(packets) },
 110	{ "bytes",		VIRTNET_RQ_STAT(bytes) },
 111	{ "drops",		VIRTNET_RQ_STAT(drops) },
 112	{ "xdp_packets",	VIRTNET_RQ_STAT(xdp_packets) },
 113	{ "xdp_tx",		VIRTNET_RQ_STAT(xdp_tx) },
 114	{ "xdp_redirects",	VIRTNET_RQ_STAT(xdp_redirects) },
 115	{ "xdp_drops",		VIRTNET_RQ_STAT(xdp_drops) },
 116	{ "kicks",		VIRTNET_RQ_STAT(kicks) },
 117};
 118
 119#define VIRTNET_SQ_STATS_LEN	ARRAY_SIZE(virtnet_sq_stats_desc)
 120#define VIRTNET_RQ_STATS_LEN	ARRAY_SIZE(virtnet_rq_stats_desc)
 121
 122/* Internal representation of a send virtqueue */
 123struct send_queue {
 124	/* Virtqueue associated with this send _queue */
 125	struct virtqueue *vq;
 126
 127	/* TX: fragments + linear part + virtio header */
 128	struct scatterlist sg[MAX_SKB_FRAGS + 2];
 129
 130	/* Name of the send queue: output.$index */
 131	char name[40];
 132
 133	struct virtnet_sq_stats stats;
 134
 135	struct napi_struct napi;
 136};
 137
 138/* Internal representation of a receive virtqueue */
 139struct receive_queue {
 140	/* Virtqueue associated with this receive_queue */
 141	struct virtqueue *vq;
 142
 143	struct napi_struct napi;
 144
 145	struct bpf_prog __rcu *xdp_prog;
 146
 147	struct virtnet_rq_stats stats;
 148
 149	/* Chain pages by the private ptr. */
 150	struct page *pages;
 151
 152	/* Average packet length for mergeable receive buffers. */
 153	struct ewma_pkt_len mrg_avg_pkt_len;
 154
 155	/* Page frag for packet buffer allocation. */
 156	struct page_frag alloc_frag;
 157
 158	/* RX: fragments + linear part + virtio header */
 159	struct scatterlist sg[MAX_SKB_FRAGS + 2];
 160
 161	/* Min single buffer size for mergeable buffers case. */
 162	unsigned int min_buf_len;
 163
 164	/* Name of this receive queue: input.$index */
 165	char name[40];
 166
 167	struct xdp_rxq_info xdp_rxq;
 168};
 169
 170/* Control VQ buffers: protected by the rtnl lock */
 171struct control_buf {
 172	struct virtio_net_ctrl_hdr hdr;
 173	virtio_net_ctrl_ack status;
 174	struct virtio_net_ctrl_mq mq;
 175	u8 promisc;
 176	u8 allmulti;
 177	__virtio16 vid;
 178	__virtio64 offloads;
 179};
 180
 181struct virtnet_info {
 182	struct virtio_device *vdev;
 183	struct virtqueue *cvq;
 184	struct net_device *dev;
 185	struct send_queue *sq;
 186	struct receive_queue *rq;
 187	unsigned int status;
 188
 189	/* Max # of queue pairs supported by the device */
 190	u16 max_queue_pairs;
 191
 192	/* # of queue pairs currently used by the driver */
 193	u16 curr_queue_pairs;
 194
 195	/* # of XDP queue pairs currently used by the driver */
 196	u16 xdp_queue_pairs;
 197
 198	/* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
 199	bool xdp_enabled;
 200
 201	/* I like... big packets and I cannot lie! */
 202	bool big_packets;
 203
 204	/* Host will merge rx buffers for big packets (shake it! shake it!) */
 205	bool mergeable_rx_bufs;
 206
 207	/* Has control virtqueue */
 208	bool has_cvq;
 209
 210	/* Host can handle any s/g split between our header and packet data */
 211	bool any_header_sg;
 212
 213	/* Packet virtio header size */
 214	u8 hdr_len;
 215
 216	/* Work struct for refilling if we run low on memory. */
 217	struct delayed_work refill;
 218
 219	/* Work struct for config space updates */
 220	struct work_struct config_work;
 221
 222	/* Does the affinity hint is set for virtqueues? */
 223	bool affinity_hint_set;
 224
 225	/* CPU hotplug instances for online & dead */
 226	struct hlist_node node;
 227	struct hlist_node node_dead;
 228
 229	struct control_buf *ctrl;
 230
 231	/* Ethtool settings */
 232	u8 duplex;
 233	u32 speed;
 234
 235	unsigned long guest_offloads;
 236	unsigned long guest_offloads_capable;
 237
 238	/* failover when STANDBY feature enabled */
 239	struct failover *failover;
 240};
 241
 242struct padded_vnet_hdr {
 243	struct virtio_net_hdr_mrg_rxbuf hdr;
 244	/*
 245	 * hdr is in a separate sg buffer, and data sg buffer shares same page
 246	 * with this header sg. This padding makes next sg 16 byte aligned
 247	 * after the header.
 248	 */
 249	char padding[4];
 250};
 251
 252static bool is_xdp_frame(void *ptr)
 253{
 254	return (unsigned long)ptr & VIRTIO_XDP_FLAG;
 255}
 256
 257static void *xdp_to_ptr(struct xdp_frame *ptr)
 258{
 259	return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG);
 260}
 261
 262static struct xdp_frame *ptr_to_xdp(void *ptr)
 263{
 264	return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
 265}
 266
 267/* Converting between virtqueue no. and kernel tx/rx queue no.
 268 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
 269 */
 270static int vq2txq(struct virtqueue *vq)
 271{
 272	return (vq->index - 1) / 2;
 273}
 274
 275static int txq2vq(int txq)
 276{
 277	return txq * 2 + 1;
 278}
 279
 280static int vq2rxq(struct virtqueue *vq)
 281{
 282	return vq->index / 2;
 283}
 284
 285static int rxq2vq(int rxq)
 286{
 287	return rxq * 2;
 288}
 289
 290static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
 291{
 292	return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
 293}
 294
 295/*
 296 * private is used to chain pages for big packets, put the whole
 297 * most recent used list in the beginning for reuse
 298 */
 299static void give_pages(struct receive_queue *rq, struct page *page)
 300{
 301	struct page *end;
 302
 303	/* Find end of list, sew whole thing into vi->rq.pages. */
 304	for (end = page; end->private; end = (struct page *)end->private);
 305	end->private = (unsigned long)rq->pages;
 306	rq->pages = page;
 307}
 308
 309static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
 310{
 311	struct page *p = rq->pages;
 312
 313	if (p) {
 314		rq->pages = (struct page *)p->private;
 315		/* clear private here, it is used to chain pages */
 316		p->private = 0;
 317	} else
 318		p = alloc_page(gfp_mask);
 319	return p;
 320}
 321
 322static void virtqueue_napi_schedule(struct napi_struct *napi,
 323				    struct virtqueue *vq)
 324{
 325	if (napi_schedule_prep(napi)) {
 326		virtqueue_disable_cb(vq);
 327		__napi_schedule(napi);
 328	}
 329}
 330
 331static void virtqueue_napi_complete(struct napi_struct *napi,
 332				    struct virtqueue *vq, int processed)
 333{
 334	int opaque;
 335
 336	opaque = virtqueue_enable_cb_prepare(vq);
 337	if (napi_complete_done(napi, processed)) {
 338		if (unlikely(virtqueue_poll(vq, opaque)))
 339			virtqueue_napi_schedule(napi, vq);
 340	} else {
 341		virtqueue_disable_cb(vq);
 342	}
 343}
 344
 345static void skb_xmit_done(struct virtqueue *vq)
 346{
 347	struct virtnet_info *vi = vq->vdev->priv;
 348	struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi;
 349
 350	/* Suppress further interrupts. */
 351	virtqueue_disable_cb(vq);
 352
 353	if (napi->weight)
 354		virtqueue_napi_schedule(napi, vq);
 355	else
 356		/* We were probably waiting for more output buffers. */
 357		netif_wake_subqueue(vi->dev, vq2txq(vq));
 358}
 359
 360#define MRG_CTX_HEADER_SHIFT 22
 361static void *mergeable_len_to_ctx(unsigned int truesize,
 362				  unsigned int headroom)
 363{
 364	return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
 365}
 366
 367static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
 368{
 369	return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
 370}
 371
 372static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
 373{
 374	return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
 375}
 376
 377/* Called from bottom half context */
 378static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 379				   struct receive_queue *rq,
 380				   struct page *page, unsigned int offset,
 381				   unsigned int len, unsigned int truesize,
 382				   bool hdr_valid, unsigned int metasize,
 383				   bool whole_page)
 384{
 385	struct sk_buff *skb;
 386	struct virtio_net_hdr_mrg_rxbuf *hdr;
 387	unsigned int copy, hdr_len, hdr_padded_len;
 388	struct page *page_to_free = NULL;
 389	int tailroom, shinfo_size;
 390	char *p, *hdr_p, *buf;
 391
 392	p = page_address(page) + offset;
 393	hdr_p = p;
 
 
 
 
 
 
 394
 395	hdr_len = vi->hdr_len;
 396	if (vi->mergeable_rx_bufs)
 397		hdr_padded_len = sizeof(*hdr);
 398	else
 399		hdr_padded_len = sizeof(struct padded_vnet_hdr);
 400
 401	/* If whole_page, there is an offset between the beginning of the
 402	 * data and the allocated space, otherwise the data and the allocated
 403	 * space are aligned.
 404	 *
 405	 * Buffers with headroom use PAGE_SIZE as alloc size, see
 406	 * add_recvbuf_mergeable() + get_mergeable_buf_len()
 407	 */
 408	if (whole_page) {
 409		/* Buffers with whole_page use PAGE_SIZE as alloc size,
 410		 * see add_recvbuf_mergeable() + get_mergeable_buf_len()
 411		 */
 412		truesize = PAGE_SIZE;
 413
 414		/* page maybe head page, so we should get the buf by p, not the
 415		 * page
 416		 */
 417		tailroom = truesize - len - offset_in_page(p);
 418		buf = (char *)((unsigned long)p & PAGE_MASK);
 419	} else {
 420		tailroom = truesize - len;
 421		buf = p;
 422	}
 423
 424	len -= hdr_len;
 425	offset += hdr_padded_len;
 426	p += hdr_padded_len;
 427
 428	shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 429
 430	/* copy small packet so we can reuse these pages */
 431	if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) {
 432		skb = build_skb(buf, truesize);
 433		if (unlikely(!skb))
 434			return NULL;
 435
 436		skb_reserve(skb, p - buf);
 437		skb_put(skb, len);
 438
 439		page = (struct page *)page->private;
 440		if (page)
 441			give_pages(rq, page);
 442		goto ok;
 443	}
 444
 445	/* copy small packet so we can reuse these pages for small data */
 446	skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
 447	if (unlikely(!skb))
 448		return NULL;
 449
 450	/* Copy all frame if it fits skb->head, otherwise
 451	 * we let virtio_net_hdr_to_skb() and GRO pull headers as needed.
 452	 */
 453	if (len <= skb_tailroom(skb))
 454		copy = len;
 455	else
 456		copy = ETH_HLEN + metasize;
 457	skb_put_data(skb, p, copy);
 458
 459	len -= copy;
 460	offset += copy;
 461
 462	if (vi->mergeable_rx_bufs) {
 463		if (len)
 464			skb_add_rx_frag(skb, 0, page, offset, len, truesize);
 465		else
 466			page_to_free = page;
 467		goto ok;
 468	}
 469
 470	/*
 471	 * Verify that we can indeed put this data into a skb.
 472	 * This is here to handle cases when the device erroneously
 473	 * tries to receive more than is possible. This is usually
 474	 * the case of a broken device.
 475	 */
 476	if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
 477		net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
 478		dev_kfree_skb(skb);
 479		return NULL;
 480	}
 481	BUG_ON(offset >= PAGE_SIZE);
 482	while (len) {
 483		unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
 484		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
 485				frag_size, truesize);
 486		len -= frag_size;
 487		page = (struct page *)page->private;
 488		offset = 0;
 489	}
 490
 491	if (page)
 492		give_pages(rq, page);
 493
 494ok:
 495	/* hdr_valid means no XDP, so we can copy the vnet header */
 496	if (hdr_valid) {
 497		hdr = skb_vnet_hdr(skb);
 498		memcpy(hdr, hdr_p, hdr_len);
 499	}
 500	if (page_to_free)
 501		put_page(page_to_free);
 502
 503	if (metasize) {
 504		__skb_pull(skb, metasize);
 505		skb_metadata_set(skb, metasize);
 506	}
 507
 508	return skb;
 509}
 510
 511static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
 512				   struct send_queue *sq,
 513				   struct xdp_frame *xdpf)
 514{
 515	struct virtio_net_hdr_mrg_rxbuf *hdr;
 516	int err;
 517
 518	if (unlikely(xdpf->headroom < vi->hdr_len))
 519		return -EOVERFLOW;
 520
 521	/* Make room for virtqueue hdr (also change xdpf->headroom?) */
 522	xdpf->data -= vi->hdr_len;
 523	/* Zero header and leave csum up to XDP layers */
 524	hdr = xdpf->data;
 525	memset(hdr, 0, vi->hdr_len);
 526	xdpf->len   += vi->hdr_len;
 527
 528	sg_init_one(sq->sg, xdpf->data, xdpf->len);
 529
 530	err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp_to_ptr(xdpf),
 531				   GFP_ATOMIC);
 532	if (unlikely(err))
 533		return -ENOSPC; /* Caller handle free/refcnt */
 534
 535	return 0;
 536}
 537
 538/* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on
 539 * the current cpu, so it does not need to be locked.
 540 *
 541 * Here we use marco instead of inline functions because we have to deal with
 542 * three issues at the same time: 1. the choice of sq. 2. judge and execute the
 543 * lock/unlock of txq 3. make sparse happy. It is difficult for two inline
 544 * functions to perfectly solve these three problems at the same time.
 545 */
 546#define virtnet_xdp_get_sq(vi) ({                                       \
 547	struct netdev_queue *txq;                                       \
 548	typeof(vi) v = (vi);                                            \
 549	unsigned int qp;                                                \
 550									\
 551	if (v->curr_queue_pairs > nr_cpu_ids) {                         \
 552		qp = v->curr_queue_pairs - v->xdp_queue_pairs;          \
 553		qp += smp_processor_id();                               \
 554		txq = netdev_get_tx_queue(v->dev, qp);                  \
 555		__netif_tx_acquire(txq);                                \
 556	} else {                                                        \
 557		qp = smp_processor_id() % v->curr_queue_pairs;          \
 558		txq = netdev_get_tx_queue(v->dev, qp);                  \
 559		__netif_tx_lock(txq, raw_smp_processor_id());           \
 560	}                                                               \
 561	v->sq + qp;                                                     \
 562})
 563
 564#define virtnet_xdp_put_sq(vi, q) {                                     \
 565	struct netdev_queue *txq;                                       \
 566	typeof(vi) v = (vi);                                            \
 567									\
 568	txq = netdev_get_tx_queue(v->dev, (q) - v->sq);                 \
 569	if (v->curr_queue_pairs > nr_cpu_ids)                           \
 570		__netif_tx_release(txq);                                \
 571	else                                                            \
 572		__netif_tx_unlock(txq);                                 \
 573}
 574
 575static int virtnet_xdp_xmit(struct net_device *dev,
 576			    int n, struct xdp_frame **frames, u32 flags)
 577{
 578	struct virtnet_info *vi = netdev_priv(dev);
 579	struct receive_queue *rq = vi->rq;
 580	struct bpf_prog *xdp_prog;
 581	struct send_queue *sq;
 582	unsigned int len;
 583	int packets = 0;
 584	int bytes = 0;
 585	int nxmit = 0;
 586	int kicks = 0;
 
 587	void *ptr;
 588	int ret;
 589	int i;
 590
 591	/* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
 592	 * indicate XDP resources have been successfully allocated.
 593	 */
 594	xdp_prog = rcu_access_pointer(rq->xdp_prog);
 595	if (!xdp_prog)
 596		return -ENXIO;
 597
 598	sq = virtnet_xdp_get_sq(vi);
 599
 600	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) {
 601		ret = -EINVAL;
 
 602		goto out;
 603	}
 604
 605	/* Free up any pending old buffers before queueing new ones. */
 606	while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
 607		if (likely(is_xdp_frame(ptr))) {
 608			struct xdp_frame *frame = ptr_to_xdp(ptr);
 609
 610			bytes += frame->len;
 611			xdp_return_frame(frame);
 612		} else {
 613			struct sk_buff *skb = ptr;
 614
 615			bytes += skb->len;
 616			napi_consume_skb(skb, false);
 617		}
 618		packets++;
 619	}
 620
 621	for (i = 0; i < n; i++) {
 622		struct xdp_frame *xdpf = frames[i];
 623
 624		if (__virtnet_xdp_xmit_one(vi, sq, xdpf))
 625			break;
 626		nxmit++;
 
 
 627	}
 628	ret = nxmit;
 629
 630	if (flags & XDP_XMIT_FLUSH) {
 631		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
 632			kicks = 1;
 633	}
 634out:
 635	u64_stats_update_begin(&sq->stats.syncp);
 636	sq->stats.bytes += bytes;
 637	sq->stats.packets += packets;
 638	sq->stats.xdp_tx += n;
 639	sq->stats.xdp_tx_drops += n - nxmit;
 640	sq->stats.kicks += kicks;
 641	u64_stats_update_end(&sq->stats.syncp);
 642
 643	virtnet_xdp_put_sq(vi, sq);
 644	return ret;
 645}
 646
 647static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
 648{
 649	return vi->xdp_enabled ? VIRTIO_XDP_HEADROOM : 0;
 650}
 651
 652/* We copy the packet for XDP in the following cases:
 653 *
 654 * 1) Packet is scattered across multiple rx buffers.
 655 * 2) Headroom space is insufficient.
 656 *
 657 * This is inefficient but it's a temporary condition that
 658 * we hit right after XDP is enabled and until queue is refilled
 659 * with large buffers with sufficient headroom - so it should affect
 660 * at most queue size packets.
 661 * Afterwards, the conditions to enable
 662 * XDP should preclude the underlying device from sending packets
 663 * across multiple buffers (num_buf > 1), and we make sure buffers
 664 * have enough headroom.
 665 */
 666static struct page *xdp_linearize_page(struct receive_queue *rq,
 667				       u16 *num_buf,
 668				       struct page *p,
 669				       int offset,
 670				       int page_off,
 671				       unsigned int *len)
 672{
 673	struct page *page = alloc_page(GFP_ATOMIC);
 674
 675	if (!page)
 676		return NULL;
 677
 678	memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
 679	page_off += *len;
 680
 681	while (--*num_buf) {
 682		int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 683		unsigned int buflen;
 684		void *buf;
 685		int off;
 686
 687		buf = virtqueue_get_buf(rq->vq, &buflen);
 688		if (unlikely(!buf))
 689			goto err_buf;
 690
 691		p = virt_to_head_page(buf);
 692		off = buf - page_address(p);
 693
 694		/* guard against a misconfigured or uncooperative backend that
 695		 * is sending packet larger than the MTU.
 696		 */
 697		if ((page_off + buflen + tailroom) > PAGE_SIZE) {
 698			put_page(p);
 699			goto err_buf;
 700		}
 701
 702		memcpy(page_address(page) + page_off,
 703		       page_address(p) + off, buflen);
 704		page_off += buflen;
 705		put_page(p);
 706	}
 707
 708	/* Headroom does not contribute to packet length */
 709	*len = page_off - VIRTIO_XDP_HEADROOM;
 710	return page;
 711err_buf:
 712	__free_pages(page, 0);
 713	return NULL;
 714}
 715
 716static struct sk_buff *receive_small(struct net_device *dev,
 717				     struct virtnet_info *vi,
 718				     struct receive_queue *rq,
 719				     void *buf, void *ctx,
 720				     unsigned int len,
 721				     unsigned int *xdp_xmit,
 722				     struct virtnet_rq_stats *stats)
 723{
 724	struct sk_buff *skb;
 725	struct bpf_prog *xdp_prog;
 726	unsigned int xdp_headroom = (unsigned long)ctx;
 727	unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
 728	unsigned int headroom = vi->hdr_len + header_offset;
 729	unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 730			      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 731	struct page *page = virt_to_head_page(buf);
 732	unsigned int delta = 0;
 733	struct page *xdp_page;
 734	int err;
 735	unsigned int metasize = 0;
 736
 737	len -= vi->hdr_len;
 738	stats->bytes += len;
 739
 740	if (unlikely(len > GOOD_PACKET_LEN)) {
 741		pr_debug("%s: rx error: len %u exceeds max size %d\n",
 742			 dev->name, len, GOOD_PACKET_LEN);
 743		dev->stats.rx_length_errors++;
 744		goto err_len;
 745	}
 746	rcu_read_lock();
 747	xdp_prog = rcu_dereference(rq->xdp_prog);
 748	if (xdp_prog) {
 749		struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
 750		struct xdp_frame *xdpf;
 751		struct xdp_buff xdp;
 752		void *orig_data;
 753		u32 act;
 754
 755		if (unlikely(hdr->hdr.gso_type))
 756			goto err_xdp;
 757
 758		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
 759			int offset = buf - page_address(page) + header_offset;
 760			unsigned int tlen = len + vi->hdr_len;
 761			u16 num_buf = 1;
 762
 763			xdp_headroom = virtnet_get_headroom(vi);
 764			header_offset = VIRTNET_RX_PAD + xdp_headroom;
 765			headroom = vi->hdr_len + header_offset;
 766			buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 767				 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 768			xdp_page = xdp_linearize_page(rq, &num_buf, page,
 769						      offset, header_offset,
 770						      &tlen);
 771			if (!xdp_page)
 772				goto err_xdp;
 773
 774			buf = page_address(xdp_page);
 775			put_page(page);
 776			page = xdp_page;
 777		}
 778
 779		xdp_init_buff(&xdp, buflen, &rq->xdp_rxq);
 780		xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len,
 781				 xdp_headroom, len, true);
 
 
 
 782		orig_data = xdp.data;
 783		act = bpf_prog_run_xdp(xdp_prog, &xdp);
 784		stats->xdp_packets++;
 785
 786		switch (act) {
 787		case XDP_PASS:
 788			/* Recalculate length in case bpf program changed it */
 789			delta = orig_data - xdp.data;
 790			len = xdp.data_end - xdp.data;
 791			metasize = xdp.data - xdp.data_meta;
 792			break;
 793		case XDP_TX:
 794			stats->xdp_tx++;
 795			xdpf = xdp_convert_buff_to_frame(&xdp);
 796			if (unlikely(!xdpf))
 797				goto err_xdp;
 798			err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
 799			if (unlikely(!err)) {
 800				xdp_return_frame_rx_napi(xdpf);
 801			} else if (unlikely(err < 0)) {
 802				trace_xdp_exception(vi->dev, xdp_prog, act);
 803				goto err_xdp;
 804			}
 805			*xdp_xmit |= VIRTIO_XDP_TX;
 806			rcu_read_unlock();
 807			goto xdp_xmit;
 808		case XDP_REDIRECT:
 809			stats->xdp_redirects++;
 810			err = xdp_do_redirect(dev, &xdp, xdp_prog);
 811			if (err)
 812				goto err_xdp;
 813			*xdp_xmit |= VIRTIO_XDP_REDIR;
 814			rcu_read_unlock();
 815			goto xdp_xmit;
 816		default:
 817			bpf_warn_invalid_xdp_action(act);
 818			fallthrough;
 819		case XDP_ABORTED:
 820			trace_xdp_exception(vi->dev, xdp_prog, act);
 821			goto err_xdp;
 822		case XDP_DROP:
 823			goto err_xdp;
 824		}
 825	}
 826	rcu_read_unlock();
 827
 828	skb = build_skb(buf, buflen);
 829	if (!skb) {
 830		put_page(page);
 831		goto err;
 832	}
 833	skb_reserve(skb, headroom - delta);
 834	skb_put(skb, len);
 835	if (!xdp_prog) {
 836		buf += header_offset;
 837		memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
 838	} /* keep zeroed vnet hdr since XDP is loaded */
 839
 840	if (metasize)
 841		skb_metadata_set(skb, metasize);
 842
 843err:
 844	return skb;
 845
 846err_xdp:
 847	rcu_read_unlock();
 848	stats->xdp_drops++;
 849err_len:
 850	stats->drops++;
 851	put_page(page);
 852xdp_xmit:
 853	return NULL;
 854}
 855
 856static struct sk_buff *receive_big(struct net_device *dev,
 857				   struct virtnet_info *vi,
 858				   struct receive_queue *rq,
 859				   void *buf,
 860				   unsigned int len,
 861				   struct virtnet_rq_stats *stats)
 862{
 863	struct page *page = buf;
 864	struct sk_buff *skb =
 865		page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0, 0);
 866
 867	stats->bytes += len - vi->hdr_len;
 868	if (unlikely(!skb))
 869		goto err;
 870
 871	return skb;
 872
 873err:
 874	stats->drops++;
 875	give_pages(rq, page);
 876	return NULL;
 877}
 878
 879static struct sk_buff *receive_mergeable(struct net_device *dev,
 880					 struct virtnet_info *vi,
 881					 struct receive_queue *rq,
 882					 void *buf,
 883					 void *ctx,
 884					 unsigned int len,
 885					 unsigned int *xdp_xmit,
 886					 struct virtnet_rq_stats *stats)
 887{
 888	struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
 889	u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
 890	struct page *page = virt_to_head_page(buf);
 891	int offset = buf - page_address(page);
 892	struct sk_buff *head_skb, *curr_skb;
 893	struct bpf_prog *xdp_prog;
 894	unsigned int truesize = mergeable_ctx_to_truesize(ctx);
 895	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
 896	unsigned int metasize = 0;
 897	unsigned int frame_sz;
 898	int err;
 899
 900	head_skb = NULL;
 901	stats->bytes += len - vi->hdr_len;
 902
 903	if (unlikely(len > truesize)) {
 904		pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
 905			 dev->name, len, (unsigned long)ctx);
 906		dev->stats.rx_length_errors++;
 907		goto err_skb;
 908	}
 909	rcu_read_lock();
 910	xdp_prog = rcu_dereference(rq->xdp_prog);
 911	if (xdp_prog) {
 912		struct xdp_frame *xdpf;
 913		struct page *xdp_page;
 914		struct xdp_buff xdp;
 915		void *data;
 916		u32 act;
 917
 918		/* Transient failure which in theory could occur if
 919		 * in-flight packets from before XDP was enabled reach
 920		 * the receive path after XDP is loaded.
 921		 */
 922		if (unlikely(hdr->hdr.gso_type))
 923			goto err_xdp;
 924
 925		/* Buffers with headroom use PAGE_SIZE as alloc size,
 926		 * see add_recvbuf_mergeable() + get_mergeable_buf_len()
 927		 */
 928		frame_sz = headroom ? PAGE_SIZE : truesize;
 929
 930		/* This happens when rx buffer size is underestimated
 931		 * or headroom is not enough because of the buffer
 932		 * was refilled before XDP is set. This should only
 933		 * happen for the first several packets, so we don't
 934		 * care much about its performance.
 935		 */
 936		if (unlikely(num_buf > 1 ||
 937			     headroom < virtnet_get_headroom(vi))) {
 938			/* linearize data for XDP */
 939			xdp_page = xdp_linearize_page(rq, &num_buf,
 940						      page, offset,
 941						      VIRTIO_XDP_HEADROOM,
 942						      &len);
 943			frame_sz = PAGE_SIZE;
 944
 945			if (!xdp_page)
 946				goto err_xdp;
 947			offset = VIRTIO_XDP_HEADROOM;
 948		} else {
 949			xdp_page = page;
 950		}
 951
 952		/* Allow consuming headroom but reserve enough space to push
 953		 * the descriptor on if we get an XDP_TX return code.
 954		 */
 955		data = page_address(xdp_page) + offset;
 956		xdp_init_buff(&xdp, frame_sz - vi->hdr_len, &rq->xdp_rxq);
 957		xdp_prepare_buff(&xdp, data - VIRTIO_XDP_HEADROOM + vi->hdr_len,
 958				 VIRTIO_XDP_HEADROOM, len - vi->hdr_len, true);
 
 
 
 959
 960		act = bpf_prog_run_xdp(xdp_prog, &xdp);
 961		stats->xdp_packets++;
 962
 963		switch (act) {
 964		case XDP_PASS:
 965			metasize = xdp.data - xdp.data_meta;
 966
 967			/* recalculate offset to account for any header
 968			 * adjustments and minus the metasize to copy the
 969			 * metadata in page_to_skb(). Note other cases do not
 970			 * build an skb and avoid using offset
 971			 */
 972			offset = xdp.data - page_address(xdp_page) -
 973				 vi->hdr_len - metasize;
 974
 975			/* recalculate len if xdp.data, xdp.data_end or
 976			 * xdp.data_meta were adjusted
 977			 */
 978			len = xdp.data_end - xdp.data + vi->hdr_len + metasize;
 979			/* We can only create skb based on xdp_page. */
 980			if (unlikely(xdp_page != page)) {
 981				rcu_read_unlock();
 982				put_page(page);
 983				head_skb = page_to_skb(vi, rq, xdp_page, offset,
 984						       len, PAGE_SIZE, false,
 985						       metasize, true);
 986				return head_skb;
 987			}
 988			break;
 989		case XDP_TX:
 990			stats->xdp_tx++;
 991			xdpf = xdp_convert_buff_to_frame(&xdp);
 992			if (unlikely(!xdpf))
 993				goto err_xdp;
 994			err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
 995			if (unlikely(!err)) {
 996				xdp_return_frame_rx_napi(xdpf);
 997			} else if (unlikely(err < 0)) {
 998				trace_xdp_exception(vi->dev, xdp_prog, act);
 999				if (unlikely(xdp_page != page))
1000					put_page(xdp_page);
1001				goto err_xdp;
1002			}
1003			*xdp_xmit |= VIRTIO_XDP_TX;
1004			if (unlikely(xdp_page != page))
1005				put_page(page);
1006			rcu_read_unlock();
1007			goto xdp_xmit;
1008		case XDP_REDIRECT:
1009			stats->xdp_redirects++;
1010			err = xdp_do_redirect(dev, &xdp, xdp_prog);
1011			if (err) {
1012				if (unlikely(xdp_page != page))
1013					put_page(xdp_page);
1014				goto err_xdp;
1015			}
1016			*xdp_xmit |= VIRTIO_XDP_REDIR;
1017			if (unlikely(xdp_page != page))
1018				put_page(page);
1019			rcu_read_unlock();
1020			goto xdp_xmit;
1021		default:
1022			bpf_warn_invalid_xdp_action(act);
1023			fallthrough;
1024		case XDP_ABORTED:
1025			trace_xdp_exception(vi->dev, xdp_prog, act);
1026			fallthrough;
1027		case XDP_DROP:
1028			if (unlikely(xdp_page != page))
1029				__free_pages(xdp_page, 0);
1030			goto err_xdp;
1031		}
1032	}
1033	rcu_read_unlock();
1034
 
 
 
 
 
 
 
1035	head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog,
1036			       metasize, !!headroom);
1037	curr_skb = head_skb;
1038
1039	if (unlikely(!curr_skb))
1040		goto err_skb;
1041	while (--num_buf) {
1042		int num_skb_frags;
1043
1044		buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
1045		if (unlikely(!buf)) {
1046			pr_debug("%s: rx error: %d buffers out of %d missing\n",
1047				 dev->name, num_buf,
1048				 virtio16_to_cpu(vi->vdev,
1049						 hdr->num_buffers));
1050			dev->stats.rx_length_errors++;
1051			goto err_buf;
1052		}
1053
1054		stats->bytes += len;
1055		page = virt_to_head_page(buf);
1056
1057		truesize = mergeable_ctx_to_truesize(ctx);
1058		if (unlikely(len > truesize)) {
1059			pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
1060				 dev->name, len, (unsigned long)ctx);
1061			dev->stats.rx_length_errors++;
1062			goto err_skb;
1063		}
1064
1065		num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
1066		if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
1067			struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
1068
1069			if (unlikely(!nskb))
1070				goto err_skb;
1071			if (curr_skb == head_skb)
1072				skb_shinfo(curr_skb)->frag_list = nskb;
1073			else
1074				curr_skb->next = nskb;
1075			curr_skb = nskb;
1076			head_skb->truesize += nskb->truesize;
1077			num_skb_frags = 0;
1078		}
1079		if (curr_skb != head_skb) {
1080			head_skb->data_len += len;
1081			head_skb->len += len;
1082			head_skb->truesize += truesize;
1083		}
1084		offset = buf - page_address(page);
1085		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
1086			put_page(page);
1087			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
1088					     len, truesize);
1089		} else {
1090			skb_add_rx_frag(curr_skb, num_skb_frags, page,
1091					offset, len, truesize);
1092		}
1093	}
1094
1095	ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
1096	return head_skb;
1097
1098err_xdp:
1099	rcu_read_unlock();
1100	stats->xdp_drops++;
1101err_skb:
1102	put_page(page);
1103	while (num_buf-- > 1) {
1104		buf = virtqueue_get_buf(rq->vq, &len);
1105		if (unlikely(!buf)) {
1106			pr_debug("%s: rx error: %d buffers missing\n",
1107				 dev->name, num_buf);
1108			dev->stats.rx_length_errors++;
1109			break;
1110		}
1111		stats->bytes += len;
1112		page = virt_to_head_page(buf);
1113		put_page(page);
1114	}
1115err_buf:
1116	stats->drops++;
1117	dev_kfree_skb(head_skb);
1118xdp_xmit:
1119	return NULL;
1120}
1121
1122static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
1123			void *buf, unsigned int len, void **ctx,
1124			unsigned int *xdp_xmit,
1125			struct virtnet_rq_stats *stats)
1126{
1127	struct net_device *dev = vi->dev;
1128	struct sk_buff *skb;
1129	struct virtio_net_hdr_mrg_rxbuf *hdr;
1130
1131	if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
1132		pr_debug("%s: short packet %i\n", dev->name, len);
1133		dev->stats.rx_length_errors++;
1134		if (vi->mergeable_rx_bufs) {
1135			put_page(virt_to_head_page(buf));
1136		} else if (vi->big_packets) {
1137			give_pages(rq, buf);
1138		} else {
1139			put_page(virt_to_head_page(buf));
1140		}
1141		return;
1142	}
1143
1144	if (vi->mergeable_rx_bufs)
1145		skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
1146					stats);
1147	else if (vi->big_packets)
1148		skb = receive_big(dev, vi, rq, buf, len, stats);
1149	else
1150		skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats);
1151
1152	if (unlikely(!skb))
1153		return;
1154
1155	hdr = skb_vnet_hdr(skb);
1156
1157	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
1158		skb->ip_summed = CHECKSUM_UNNECESSARY;
1159
1160	if (virtio_net_hdr_to_skb(skb, &hdr->hdr,
1161				  virtio_is_little_endian(vi->vdev))) {
1162		net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
1163				     dev->name, hdr->hdr.gso_type,
1164				     hdr->hdr.gso_size);
1165		goto frame_err;
1166	}
1167
1168	skb_record_rx_queue(skb, vq2rxq(rq->vq));
1169	skb->protocol = eth_type_trans(skb, dev);
1170	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
1171		 ntohs(skb->protocol), skb->len, skb->pkt_type);
1172
1173	napi_gro_receive(&rq->napi, skb);
1174	return;
1175
1176frame_err:
1177	dev->stats.rx_frame_errors++;
1178	dev_kfree_skb(skb);
1179}
1180
1181/* Unlike mergeable buffers, all buffers are allocated to the
1182 * same size, except for the headroom. For this reason we do
1183 * not need to use  mergeable_len_to_ctx here - it is enough
1184 * to store the headroom as the context ignoring the truesize.
1185 */
1186static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
1187			     gfp_t gfp)
1188{
1189	struct page_frag *alloc_frag = &rq->alloc_frag;
1190	char *buf;
1191	unsigned int xdp_headroom = virtnet_get_headroom(vi);
1192	void *ctx = (void *)(unsigned long)xdp_headroom;
1193	int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
1194	int err;
1195
1196	len = SKB_DATA_ALIGN(len) +
1197	      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1198	if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
1199		return -ENOMEM;
1200
1201	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1202	get_page(alloc_frag->page);
1203	alloc_frag->offset += len;
1204	sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
1205		    vi->hdr_len + GOOD_PACKET_LEN);
1206	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1207	if (err < 0)
1208		put_page(virt_to_head_page(buf));
1209	return err;
1210}
1211
1212static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
1213			   gfp_t gfp)
1214{
1215	struct page *first, *list = NULL;
1216	char *p;
1217	int i, err, offset;
1218
1219	sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
1220
1221	/* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
1222	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
1223		first = get_a_page(rq, gfp);
1224		if (!first) {
1225			if (list)
1226				give_pages(rq, list);
1227			return -ENOMEM;
1228		}
1229		sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
1230
1231		/* chain new page in list head to match sg */
1232		first->private = (unsigned long)list;
1233		list = first;
1234	}
1235
1236	first = get_a_page(rq, gfp);
1237	if (!first) {
1238		give_pages(rq, list);
1239		return -ENOMEM;
1240	}
1241	p = page_address(first);
1242
1243	/* rq->sg[0], rq->sg[1] share the same page */
1244	/* a separated rq->sg[0] for header - required in case !any_header_sg */
1245	sg_set_buf(&rq->sg[0], p, vi->hdr_len);
1246
1247	/* rq->sg[1] for data packet, from offset */
1248	offset = sizeof(struct padded_vnet_hdr);
1249	sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
1250
1251	/* chain first in list head */
1252	first->private = (unsigned long)list;
1253	err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
1254				  first, gfp);
1255	if (err < 0)
1256		give_pages(rq, first);
1257
1258	return err;
1259}
1260
1261static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
1262					  struct ewma_pkt_len *avg_pkt_len,
1263					  unsigned int room)
1264{
1265	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1266	unsigned int len;
1267
1268	if (room)
1269		return PAGE_SIZE - room;
1270
1271	len = hdr_len +	clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
1272				rq->min_buf_len, PAGE_SIZE - hdr_len);
1273
1274	return ALIGN(len, L1_CACHE_BYTES);
1275}
1276
1277static int add_recvbuf_mergeable(struct virtnet_info *vi,
1278				 struct receive_queue *rq, gfp_t gfp)
1279{
1280	struct page_frag *alloc_frag = &rq->alloc_frag;
1281	unsigned int headroom = virtnet_get_headroom(vi);
1282	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1283	unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1284	char *buf;
1285	void *ctx;
1286	int err;
1287	unsigned int len, hole;
1288
1289	/* Extra tailroom is needed to satisfy XDP's assumption. This
1290	 * means rx frags coalescing won't work, but consider we've
1291	 * disabled GSO for XDP, it won't be a big issue.
1292	 */
1293	len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
1294	if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
1295		return -ENOMEM;
1296
1297	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1298	buf += headroom; /* advance address leaving hole at front of pkt */
1299	get_page(alloc_frag->page);
1300	alloc_frag->offset += len + room;
1301	hole = alloc_frag->size - alloc_frag->offset;
1302	if (hole < len + room) {
1303		/* To avoid internal fragmentation, if there is very likely not
1304		 * enough space for another buffer, add the remaining space to
1305		 * the current buffer.
1306		 */
1307		len += hole;
1308		alloc_frag->offset += hole;
1309	}
1310
1311	sg_init_one(rq->sg, buf, len);
1312	ctx = mergeable_len_to_ctx(len, headroom);
1313	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
1314	if (err < 0)
1315		put_page(virt_to_head_page(buf));
1316
1317	return err;
1318}
1319
1320/*
1321 * Returns false if we couldn't fill entirely (OOM).
1322 *
1323 * Normally run in the receive path, but can also be run from ndo_open
1324 * before we're receiving packets, or from refill_work which is
1325 * careful to disable receiving (using napi_disable).
1326 */
1327static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
1328			  gfp_t gfp)
1329{
1330	int err;
1331	bool oom;
1332
1333	do {
1334		if (vi->mergeable_rx_bufs)
1335			err = add_recvbuf_mergeable(vi, rq, gfp);
1336		else if (vi->big_packets)
1337			err = add_recvbuf_big(vi, rq, gfp);
1338		else
1339			err = add_recvbuf_small(vi, rq, gfp);
1340
1341		oom = err == -ENOMEM;
1342		if (err)
1343			break;
1344	} while (rq->vq->num_free);
1345	if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
1346		unsigned long flags;
1347
1348		flags = u64_stats_update_begin_irqsave(&rq->stats.syncp);
1349		rq->stats.kicks++;
1350		u64_stats_update_end_irqrestore(&rq->stats.syncp, flags);
1351	}
1352
1353	return !oom;
1354}
1355
1356static void skb_recv_done(struct virtqueue *rvq)
1357{
1358	struct virtnet_info *vi = rvq->vdev->priv;
1359	struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
1360
1361	virtqueue_napi_schedule(&rq->napi, rvq);
1362}
1363
1364static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
1365{
1366	napi_enable(napi);
1367
1368	/* If all buffers were filled by other side before we napi_enabled, we
1369	 * won't get another interrupt, so process any outstanding packets now.
1370	 * Call local_bh_enable after to trigger softIRQ processing.
1371	 */
1372	local_bh_disable();
1373	virtqueue_napi_schedule(napi, vq);
1374	local_bh_enable();
1375}
1376
1377static void virtnet_napi_tx_enable(struct virtnet_info *vi,
1378				   struct virtqueue *vq,
1379				   struct napi_struct *napi)
1380{
1381	if (!napi->weight)
1382		return;
1383
1384	/* Tx napi touches cachelines on the cpu handling tx interrupts. Only
1385	 * enable the feature if this is likely affine with the transmit path.
1386	 */
1387	if (!vi->affinity_hint_set) {
1388		napi->weight = 0;
1389		return;
1390	}
1391
1392	return virtnet_napi_enable(vq, napi);
1393}
1394
1395static void virtnet_napi_tx_disable(struct napi_struct *napi)
1396{
1397	if (napi->weight)
1398		napi_disable(napi);
1399}
1400
1401static void refill_work(struct work_struct *work)
1402{
1403	struct virtnet_info *vi =
1404		container_of(work, struct virtnet_info, refill.work);
1405	bool still_empty;
1406	int i;
1407
1408	for (i = 0; i < vi->curr_queue_pairs; i++) {
1409		struct receive_queue *rq = &vi->rq[i];
1410
1411		napi_disable(&rq->napi);
1412		still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
1413		virtnet_napi_enable(rq->vq, &rq->napi);
1414
1415		/* In theory, this can happen: if we don't get any buffers in
1416		 * we will *never* try to fill again.
1417		 */
1418		if (still_empty)
1419			schedule_delayed_work(&vi->refill, HZ/2);
1420	}
1421}
1422
1423static int virtnet_receive(struct receive_queue *rq, int budget,
1424			   unsigned int *xdp_xmit)
1425{
1426	struct virtnet_info *vi = rq->vq->vdev->priv;
1427	struct virtnet_rq_stats stats = {};
1428	unsigned int len;
1429	void *buf;
1430	int i;
1431
1432	if (!vi->big_packets || vi->mergeable_rx_bufs) {
1433		void *ctx;
1434
1435		while (stats.packets < budget &&
1436		       (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) {
1437			receive_buf(vi, rq, buf, len, ctx, xdp_xmit, &stats);
1438			stats.packets++;
1439		}
1440	} else {
1441		while (stats.packets < budget &&
1442		       (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
1443			receive_buf(vi, rq, buf, len, NULL, xdp_xmit, &stats);
1444			stats.packets++;
1445		}
1446	}
1447
1448	if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) {
1449		if (!try_fill_recv(vi, rq, GFP_ATOMIC))
1450			schedule_delayed_work(&vi->refill, 0);
1451	}
1452
1453	u64_stats_update_begin(&rq->stats.syncp);
1454	for (i = 0; i < VIRTNET_RQ_STATS_LEN; i++) {
1455		size_t offset = virtnet_rq_stats_desc[i].offset;
1456		u64 *item;
1457
1458		item = (u64 *)((u8 *)&rq->stats + offset);
1459		*item += *(u64 *)((u8 *)&stats + offset);
1460	}
1461	u64_stats_update_end(&rq->stats.syncp);
1462
1463	return stats.packets;
1464}
1465
1466static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi)
1467{
1468	unsigned int len;
1469	unsigned int packets = 0;
1470	unsigned int bytes = 0;
1471	void *ptr;
1472
1473	while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
1474		if (likely(!is_xdp_frame(ptr))) {
1475			struct sk_buff *skb = ptr;
1476
1477			pr_debug("Sent skb %p\n", skb);
1478
1479			bytes += skb->len;
1480			napi_consume_skb(skb, in_napi);
1481		} else {
1482			struct xdp_frame *frame = ptr_to_xdp(ptr);
1483
1484			bytes += frame->len;
1485			xdp_return_frame(frame);
1486		}
1487		packets++;
1488	}
1489
1490	/* Avoid overhead when no packets have been processed
1491	 * happens when called speculatively from start_xmit.
1492	 */
1493	if (!packets)
1494		return;
1495
1496	u64_stats_update_begin(&sq->stats.syncp);
1497	sq->stats.bytes += bytes;
1498	sq->stats.packets += packets;
1499	u64_stats_update_end(&sq->stats.syncp);
1500}
1501
1502static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
1503{
1504	if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
1505		return false;
1506	else if (q < vi->curr_queue_pairs)
1507		return true;
1508	else
1509		return false;
1510}
1511
1512static void virtnet_poll_cleantx(struct receive_queue *rq)
1513{
1514	struct virtnet_info *vi = rq->vq->vdev->priv;
1515	unsigned int index = vq2rxq(rq->vq);
1516	struct send_queue *sq = &vi->sq[index];
1517	struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
1518
1519	if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
1520		return;
1521
1522	if (__netif_tx_trylock(txq)) {
1523		do {
1524			virtqueue_disable_cb(sq->vq);
1525			free_old_xmit_skbs(sq, true);
1526		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
1527
1528		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1529			netif_tx_wake_queue(txq);
1530
1531		__netif_tx_unlock(txq);
1532	}
 
 
 
1533}
1534
1535static int virtnet_poll(struct napi_struct *napi, int budget)
1536{
1537	struct receive_queue *rq =
1538		container_of(napi, struct receive_queue, napi);
1539	struct virtnet_info *vi = rq->vq->vdev->priv;
1540	struct send_queue *sq;
1541	unsigned int received;
1542	unsigned int xdp_xmit = 0;
1543
1544	virtnet_poll_cleantx(rq);
1545
1546	received = virtnet_receive(rq, budget, &xdp_xmit);
1547
1548	/* Out of packets? */
1549	if (received < budget)
1550		virtqueue_napi_complete(napi, rq->vq, received);
1551
1552	if (xdp_xmit & VIRTIO_XDP_REDIR)
1553		xdp_do_flush();
1554
1555	if (xdp_xmit & VIRTIO_XDP_TX) {
1556		sq = virtnet_xdp_get_sq(vi);
1557		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
1558			u64_stats_update_begin(&sq->stats.syncp);
1559			sq->stats.kicks++;
1560			u64_stats_update_end(&sq->stats.syncp);
1561		}
1562		virtnet_xdp_put_sq(vi, sq);
1563	}
1564
1565	return received;
1566}
1567
1568static int virtnet_open(struct net_device *dev)
1569{
1570	struct virtnet_info *vi = netdev_priv(dev);
1571	int i, err;
1572
1573	for (i = 0; i < vi->max_queue_pairs; i++) {
1574		if (i < vi->curr_queue_pairs)
1575			/* Make sure we have some buffers: if oom use wq. */
1576			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1577				schedule_delayed_work(&vi->refill, 0);
1578
1579		err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id);
1580		if (err < 0)
1581			return err;
1582
1583		err = xdp_rxq_info_reg_mem_model(&vi->rq[i].xdp_rxq,
1584						 MEM_TYPE_PAGE_SHARED, NULL);
1585		if (err < 0) {
1586			xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
1587			return err;
1588		}
1589
1590		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
1591		virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
1592	}
1593
1594	return 0;
1595}
1596
1597static int virtnet_poll_tx(struct napi_struct *napi, int budget)
1598{
1599	struct send_queue *sq = container_of(napi, struct send_queue, napi);
1600	struct virtnet_info *vi = sq->vq->vdev->priv;
1601	unsigned int index = vq2txq(sq->vq);
1602	struct netdev_queue *txq;
1603	int opaque;
1604	bool done;
1605
1606	if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {
1607		/* We don't need to enable cb for XDP */
1608		napi_complete_done(napi, 0);
1609		return 0;
1610	}
1611
1612	txq = netdev_get_tx_queue(vi->dev, index);
1613	__netif_tx_lock(txq, raw_smp_processor_id());
1614	virtqueue_disable_cb(sq->vq);
1615	free_old_xmit_skbs(sq, true);
 
 
 
1616
1617	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
1618		netif_tx_wake_queue(txq);
1619
1620	opaque = virtqueue_enable_cb_prepare(sq->vq);
1621
1622	done = napi_complete_done(napi, 0);
1623
1624	if (!done)
1625		virtqueue_disable_cb(sq->vq);
1626
1627	__netif_tx_unlock(txq);
1628
1629	if (done) {
1630		if (unlikely(virtqueue_poll(sq->vq, opaque))) {
1631			if (napi_schedule_prep(napi)) {
1632				__netif_tx_lock(txq, raw_smp_processor_id());
1633				virtqueue_disable_cb(sq->vq);
1634				__netif_tx_unlock(txq);
1635				__napi_schedule(napi);
1636			}
1637		}
1638	}
1639
1640	return 0;
1641}
1642
1643static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
1644{
1645	struct virtio_net_hdr_mrg_rxbuf *hdr;
1646	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
1647	struct virtnet_info *vi = sq->vq->vdev->priv;
1648	int num_sg;
1649	unsigned hdr_len = vi->hdr_len;
1650	bool can_push;
1651
1652	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
1653
1654	can_push = vi->any_header_sg &&
1655		!((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
1656		!skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
1657	/* Even if we can, don't push here yet as this would skew
1658	 * csum_start offset below. */
1659	if (can_push)
1660		hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
1661	else
1662		hdr = skb_vnet_hdr(skb);
1663
1664	if (virtio_net_hdr_from_skb(skb, &hdr->hdr,
1665				    virtio_is_little_endian(vi->vdev), false,
1666				    0))
1667		return -EPROTO;
1668
1669	if (vi->mergeable_rx_bufs)
1670		hdr->num_buffers = 0;
1671
1672	sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2));
1673	if (can_push) {
1674		__skb_push(skb, hdr_len);
1675		num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
1676		if (unlikely(num_sg < 0))
1677			return num_sg;
1678		/* Pull header back to avoid skew in tx bytes calculations. */
1679		__skb_pull(skb, hdr_len);
1680	} else {
1681		sg_set_buf(sq->sg, hdr, hdr_len);
1682		num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len);
1683		if (unlikely(num_sg < 0))
1684			return num_sg;
1685		num_sg++;
1686	}
1687	return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
1688}
1689
1690static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
1691{
1692	struct virtnet_info *vi = netdev_priv(dev);
1693	int qnum = skb_get_queue_mapping(skb);
1694	struct send_queue *sq = &vi->sq[qnum];
1695	int err;
1696	struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
1697	bool kick = !netdev_xmit_more();
1698	bool use_napi = sq->napi.weight;
1699
1700	/* Free up any pending old buffers before queueing new ones. */
1701	do {
1702		if (use_napi)
1703			virtqueue_disable_cb(sq->vq);
1704
1705		free_old_xmit_skbs(sq, false);
1706
1707	} while (use_napi && kick &&
1708	       unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
1709
1710	/* timestamp packet in software */
1711	skb_tx_timestamp(skb);
1712
1713	/* Try to transmit */
1714	err = xmit_skb(sq, skb);
1715
1716	/* This should not happen! */
1717	if (unlikely(err)) {
1718		dev->stats.tx_fifo_errors++;
1719		if (net_ratelimit())
1720			dev_warn(&dev->dev,
1721				 "Unexpected TXQ (%d) queue failure: %d\n",
1722				 qnum, err);
1723		dev->stats.tx_dropped++;
1724		dev_kfree_skb_any(skb);
1725		return NETDEV_TX_OK;
1726	}
1727
1728	/* Don't wait up for transmitted skbs to be freed. */
1729	if (!use_napi) {
1730		skb_orphan(skb);
1731		nf_reset_ct(skb);
1732	}
1733
1734	/* If running out of space, stop queue to avoid getting packets that we
1735	 * are then unable to transmit.
1736	 * An alternative would be to force queuing layer to requeue the skb by
1737	 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1738	 * returned in a normal path of operation: it means that driver is not
1739	 * maintaining the TX queue stop/start state properly, and causes
1740	 * the stack to do a non-trivial amount of useless work.
1741	 * Since most packets only take 1 or 2 ring slots, stopping the queue
1742	 * early means 16 slots are typically wasted.
1743	 */
1744	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
1745		netif_stop_subqueue(dev, qnum);
1746		if (!use_napi &&
1747		    unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
1748			/* More just got used, free them then recheck. */
1749			free_old_xmit_skbs(sq, false);
1750			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
1751				netif_start_subqueue(dev, qnum);
1752				virtqueue_disable_cb(sq->vq);
1753			}
1754		}
1755	}
1756
1757	if (kick || netif_xmit_stopped(txq)) {
1758		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
1759			u64_stats_update_begin(&sq->stats.syncp);
1760			sq->stats.kicks++;
1761			u64_stats_update_end(&sq->stats.syncp);
1762		}
1763	}
1764
1765	return NETDEV_TX_OK;
1766}
1767
1768/*
1769 * Send command via the control virtqueue and check status.  Commands
1770 * supported by the hypervisor, as indicated by feature bits, should
1771 * never fail unless improperly formatted.
1772 */
1773static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
1774				 struct scatterlist *out)
1775{
1776	struct scatterlist *sgs[4], hdr, stat;
1777	unsigned out_num = 0, tmp;
1778	int ret;
1779
1780	/* Caller should know better */
1781	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
1782
1783	vi->ctrl->status = ~0;
1784	vi->ctrl->hdr.class = class;
1785	vi->ctrl->hdr.cmd = cmd;
1786	/* Add header */
1787	sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr));
1788	sgs[out_num++] = &hdr;
1789
1790	if (out)
1791		sgs[out_num++] = out;
1792
1793	/* Add return status. */
1794	sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status));
1795	sgs[out_num] = &stat;
1796
1797	BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
1798	ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
1799	if (ret < 0) {
1800		dev_warn(&vi->vdev->dev,
1801			 "Failed to add sgs for command vq: %d\n.", ret);
1802		return false;
1803	}
1804
1805	if (unlikely(!virtqueue_kick(vi->cvq)))
1806		return vi->ctrl->status == VIRTIO_NET_OK;
1807
1808	/* Spin for a response, the kick causes an ioport write, trapping
1809	 * into the hypervisor, so the request should be handled immediately.
1810	 */
1811	while (!virtqueue_get_buf(vi->cvq, &tmp) &&
1812	       !virtqueue_is_broken(vi->cvq))
1813		cpu_relax();
1814
1815	return vi->ctrl->status == VIRTIO_NET_OK;
1816}
1817
1818static int virtnet_set_mac_address(struct net_device *dev, void *p)
1819{
1820	struct virtnet_info *vi = netdev_priv(dev);
1821	struct virtio_device *vdev = vi->vdev;
1822	int ret;
1823	struct sockaddr *addr;
1824	struct scatterlist sg;
1825
1826	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
1827		return -EOPNOTSUPP;
1828
1829	addr = kmemdup(p, sizeof(*addr), GFP_KERNEL);
1830	if (!addr)
1831		return -ENOMEM;
1832
1833	ret = eth_prepare_mac_addr_change(dev, addr);
1834	if (ret)
1835		goto out;
1836
1837	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1838		sg_init_one(&sg, addr->sa_data, dev->addr_len);
1839		if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1840					  VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
1841			dev_warn(&vdev->dev,
1842				 "Failed to set mac address by vq command.\n");
1843			ret = -EINVAL;
1844			goto out;
1845		}
1846	} else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
1847		   !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1848		unsigned int i;
1849
1850		/* Naturally, this has an atomicity problem. */
1851		for (i = 0; i < dev->addr_len; i++)
1852			virtio_cwrite8(vdev,
1853				       offsetof(struct virtio_net_config, mac) +
1854				       i, addr->sa_data[i]);
1855	}
1856
1857	eth_commit_mac_addr_change(dev, p);
1858	ret = 0;
1859
1860out:
1861	kfree(addr);
1862	return ret;
1863}
1864
1865static void virtnet_stats(struct net_device *dev,
1866			  struct rtnl_link_stats64 *tot)
1867{
1868	struct virtnet_info *vi = netdev_priv(dev);
1869	unsigned int start;
1870	int i;
1871
1872	for (i = 0; i < vi->max_queue_pairs; i++) {
1873		u64 tpackets, tbytes, rpackets, rbytes, rdrops;
1874		struct receive_queue *rq = &vi->rq[i];
1875		struct send_queue *sq = &vi->sq[i];
1876
1877		do {
1878			start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
1879			tpackets = sq->stats.packets;
1880			tbytes   = sq->stats.bytes;
1881		} while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
1882
1883		do {
1884			start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
1885			rpackets = rq->stats.packets;
1886			rbytes   = rq->stats.bytes;
1887			rdrops   = rq->stats.drops;
1888		} while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
1889
1890		tot->rx_packets += rpackets;
1891		tot->tx_packets += tpackets;
1892		tot->rx_bytes   += rbytes;
1893		tot->tx_bytes   += tbytes;
1894		tot->rx_dropped += rdrops;
1895	}
1896
1897	tot->tx_dropped = dev->stats.tx_dropped;
1898	tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
1899	tot->rx_length_errors = dev->stats.rx_length_errors;
1900	tot->rx_frame_errors = dev->stats.rx_frame_errors;
1901}
1902
1903static void virtnet_ack_link_announce(struct virtnet_info *vi)
1904{
1905	rtnl_lock();
1906	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
1907				  VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
1908		dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
1909	rtnl_unlock();
1910}
1911
1912static int _virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1913{
1914	struct scatterlist sg;
1915	struct net_device *dev = vi->dev;
1916
1917	if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
1918		return 0;
1919
1920	vi->ctrl->mq.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
1921	sg_init_one(&sg, &vi->ctrl->mq, sizeof(vi->ctrl->mq));
1922
1923	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
1924				  VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
1925		dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
1926			 queue_pairs);
1927		return -EINVAL;
1928	} else {
1929		vi->curr_queue_pairs = queue_pairs;
1930		/* virtnet_open() will refill when device is going to up. */
1931		if (dev->flags & IFF_UP)
1932			schedule_delayed_work(&vi->refill, 0);
1933	}
1934
1935	return 0;
1936}
1937
1938static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1939{
1940	int err;
1941
1942	rtnl_lock();
1943	err = _virtnet_set_queues(vi, queue_pairs);
1944	rtnl_unlock();
1945	return err;
1946}
1947
1948static int virtnet_close(struct net_device *dev)
1949{
1950	struct virtnet_info *vi = netdev_priv(dev);
1951	int i;
1952
1953	/* Make sure refill_work doesn't re-enable napi! */
1954	cancel_delayed_work_sync(&vi->refill);
1955
1956	for (i = 0; i < vi->max_queue_pairs; i++) {
1957		xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
1958		napi_disable(&vi->rq[i].napi);
1959		virtnet_napi_tx_disable(&vi->sq[i].napi);
1960	}
1961
1962	return 0;
1963}
1964
1965static void virtnet_set_rx_mode(struct net_device *dev)
1966{
1967	struct virtnet_info *vi = netdev_priv(dev);
1968	struct scatterlist sg[2];
1969	struct virtio_net_ctrl_mac *mac_data;
1970	struct netdev_hw_addr *ha;
1971	int uc_count;
1972	int mc_count;
1973	void *buf;
1974	int i;
1975
1976	/* We can't dynamically set ndo_set_rx_mode, so return gracefully */
1977	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
1978		return;
1979
1980	vi->ctrl->promisc = ((dev->flags & IFF_PROMISC) != 0);
1981	vi->ctrl->allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
1982
1983	sg_init_one(sg, &vi->ctrl->promisc, sizeof(vi->ctrl->promisc));
1984
1985	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1986				  VIRTIO_NET_CTRL_RX_PROMISC, sg))
1987		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
1988			 vi->ctrl->promisc ? "en" : "dis");
1989
1990	sg_init_one(sg, &vi->ctrl->allmulti, sizeof(vi->ctrl->allmulti));
1991
1992	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1993				  VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
1994		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
1995			 vi->ctrl->allmulti ? "en" : "dis");
1996
1997	uc_count = netdev_uc_count(dev);
1998	mc_count = netdev_mc_count(dev);
1999	/* MAC filter - use one buffer for both lists */
2000	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
2001		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
2002	mac_data = buf;
2003	if (!buf)
2004		return;
2005
2006	sg_init_table(sg, 2);
2007
2008	/* Store the unicast list and count in the front of the buffer */
2009	mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
2010	i = 0;
2011	netdev_for_each_uc_addr(ha, dev)
2012		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
2013
2014	sg_set_buf(&sg[0], mac_data,
2015		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
2016
2017	/* multicast list and count fill the end */
2018	mac_data = (void *)&mac_data->macs[uc_count][0];
2019
2020	mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
2021	i = 0;
2022	netdev_for_each_mc_addr(ha, dev)
2023		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
2024
2025	sg_set_buf(&sg[1], mac_data,
2026		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
2027
2028	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
2029				  VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
2030		dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
2031
2032	kfree(buf);
2033}
2034
2035static int virtnet_vlan_rx_add_vid(struct net_device *dev,
2036				   __be16 proto, u16 vid)
2037{
2038	struct virtnet_info *vi = netdev_priv(dev);
2039	struct scatterlist sg;
2040
2041	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
2042	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
2043
2044	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
2045				  VIRTIO_NET_CTRL_VLAN_ADD, &sg))
2046		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
2047	return 0;
2048}
2049
2050static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
2051				    __be16 proto, u16 vid)
2052{
2053	struct virtnet_info *vi = netdev_priv(dev);
2054	struct scatterlist sg;
2055
2056	vi->ctrl->vid = cpu_to_virtio16(vi->vdev, vid);
2057	sg_init_one(&sg, &vi->ctrl->vid, sizeof(vi->ctrl->vid));
2058
2059	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
2060				  VIRTIO_NET_CTRL_VLAN_DEL, &sg))
2061		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
2062	return 0;
2063}
2064
2065static void virtnet_clean_affinity(struct virtnet_info *vi)
2066{
2067	int i;
2068
2069	if (vi->affinity_hint_set) {
2070		for (i = 0; i < vi->max_queue_pairs; i++) {
2071			virtqueue_set_affinity(vi->rq[i].vq, NULL);
2072			virtqueue_set_affinity(vi->sq[i].vq, NULL);
2073		}
2074
2075		vi->affinity_hint_set = false;
2076	}
2077}
2078
2079static void virtnet_set_affinity(struct virtnet_info *vi)
2080{
2081	cpumask_var_t mask;
2082	int stragglers;
2083	int group_size;
2084	int i, j, cpu;
2085	int num_cpu;
2086	int stride;
2087
2088	if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2089		virtnet_clean_affinity(vi);
2090		return;
2091	}
2092
2093	num_cpu = num_online_cpus();
2094	stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1);
2095	stragglers = num_cpu >= vi->curr_queue_pairs ?
2096			num_cpu % vi->curr_queue_pairs :
2097			0;
2098	cpu = cpumask_next(-1, cpu_online_mask);
2099
2100	for (i = 0; i < vi->curr_queue_pairs; i++) {
2101		group_size = stride + (i < stragglers ? 1 : 0);
2102
2103		for (j = 0; j < group_size; j++) {
2104			cpumask_set_cpu(cpu, mask);
2105			cpu = cpumask_next_wrap(cpu, cpu_online_mask,
2106						nr_cpu_ids, false);
2107		}
2108		virtqueue_set_affinity(vi->rq[i].vq, mask);
2109		virtqueue_set_affinity(vi->sq[i].vq, mask);
2110		__netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS);
2111		cpumask_clear(mask);
2112	}
2113
2114	vi->affinity_hint_set = true;
2115	free_cpumask_var(mask);
2116}
2117
2118static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
2119{
2120	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
2121						   node);
2122	virtnet_set_affinity(vi);
2123	return 0;
2124}
2125
2126static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
2127{
2128	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
2129						   node_dead);
2130	virtnet_set_affinity(vi);
2131	return 0;
2132}
2133
2134static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
2135{
2136	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
2137						   node);
2138
2139	virtnet_clean_affinity(vi);
2140	return 0;
2141}
2142
2143static enum cpuhp_state virtionet_online;
2144
2145static int virtnet_cpu_notif_add(struct virtnet_info *vi)
2146{
2147	int ret;
2148
2149	ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
2150	if (ret)
2151		return ret;
2152	ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
2153					       &vi->node_dead);
2154	if (!ret)
2155		return ret;
2156	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
2157	return ret;
2158}
2159
2160static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
2161{
2162	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
2163	cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
2164					    &vi->node_dead);
2165}
2166
2167static void virtnet_get_ringparam(struct net_device *dev,
2168				struct ethtool_ringparam *ring)
2169{
2170	struct virtnet_info *vi = netdev_priv(dev);
2171
2172	ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
2173	ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
2174	ring->rx_pending = ring->rx_max_pending;
2175	ring->tx_pending = ring->tx_max_pending;
2176}
2177
2178
2179static void virtnet_get_drvinfo(struct net_device *dev,
2180				struct ethtool_drvinfo *info)
2181{
2182	struct virtnet_info *vi = netdev_priv(dev);
2183	struct virtio_device *vdev = vi->vdev;
2184
2185	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
2186	strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
2187	strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
2188
2189}
2190
2191/* TODO: Eliminate OOO packets during switching */
2192static int virtnet_set_channels(struct net_device *dev,
2193				struct ethtool_channels *channels)
2194{
2195	struct virtnet_info *vi = netdev_priv(dev);
2196	u16 queue_pairs = channels->combined_count;
2197	int err;
2198
2199	/* We don't support separate rx/tx channels.
2200	 * We don't allow setting 'other' channels.
2201	 */
2202	if (channels->rx_count || channels->tx_count || channels->other_count)
2203		return -EINVAL;
2204
2205	if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
2206		return -EINVAL;
2207
2208	/* For now we don't support modifying channels while XDP is loaded
2209	 * also when XDP is loaded all RX queues have XDP programs so we only
2210	 * need to check a single RX queue.
2211	 */
2212	if (vi->rq[0].xdp_prog)
2213		return -EINVAL;
2214
2215	get_online_cpus();
2216	err = _virtnet_set_queues(vi, queue_pairs);
2217	if (err) {
2218		put_online_cpus();
2219		goto err;
 
 
2220	}
2221	virtnet_set_affinity(vi);
2222	put_online_cpus();
2223
2224	netif_set_real_num_tx_queues(dev, queue_pairs);
2225	netif_set_real_num_rx_queues(dev, queue_pairs);
2226 err:
2227	return err;
2228}
2229
2230static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data)
2231{
2232	struct virtnet_info *vi = netdev_priv(dev);
 
2233	unsigned int i, j;
2234	u8 *p = data;
2235
2236	switch (stringset) {
2237	case ETH_SS_STATS:
2238		for (i = 0; i < vi->curr_queue_pairs; i++) {
2239			for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++)
2240				ethtool_sprintf(&p, "rx_queue_%u_%s", i,
2241						virtnet_rq_stats_desc[j].desc);
 
 
2242		}
2243
2244		for (i = 0; i < vi->curr_queue_pairs; i++) {
2245			for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++)
2246				ethtool_sprintf(&p, "tx_queue_%u_%s", i,
2247						virtnet_sq_stats_desc[j].desc);
 
 
2248		}
2249		break;
2250	}
2251}
2252
2253static int virtnet_get_sset_count(struct net_device *dev, int sset)
2254{
2255	struct virtnet_info *vi = netdev_priv(dev);
2256
2257	switch (sset) {
2258	case ETH_SS_STATS:
2259		return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +
2260					       VIRTNET_SQ_STATS_LEN);
2261	default:
2262		return -EOPNOTSUPP;
2263	}
2264}
2265
2266static void virtnet_get_ethtool_stats(struct net_device *dev,
2267				      struct ethtool_stats *stats, u64 *data)
2268{
2269	struct virtnet_info *vi = netdev_priv(dev);
2270	unsigned int idx = 0, start, i, j;
2271	const u8 *stats_base;
2272	size_t offset;
2273
2274	for (i = 0; i < vi->curr_queue_pairs; i++) {
2275		struct receive_queue *rq = &vi->rq[i];
2276
2277		stats_base = (u8 *)&rq->stats;
2278		do {
2279			start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
2280			for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
2281				offset = virtnet_rq_stats_desc[j].offset;
2282				data[idx + j] = *(u64 *)(stats_base + offset);
2283			}
2284		} while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
2285		idx += VIRTNET_RQ_STATS_LEN;
2286	}
2287
2288	for (i = 0; i < vi->curr_queue_pairs; i++) {
2289		struct send_queue *sq = &vi->sq[i];
2290
2291		stats_base = (u8 *)&sq->stats;
2292		do {
2293			start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
2294			for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
2295				offset = virtnet_sq_stats_desc[j].offset;
2296				data[idx + j] = *(u64 *)(stats_base + offset);
2297			}
2298		} while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
2299		idx += VIRTNET_SQ_STATS_LEN;
2300	}
2301}
2302
2303static void virtnet_get_channels(struct net_device *dev,
2304				 struct ethtool_channels *channels)
2305{
2306	struct virtnet_info *vi = netdev_priv(dev);
2307
2308	channels->combined_count = vi->curr_queue_pairs;
2309	channels->max_combined = vi->max_queue_pairs;
2310	channels->max_other = 0;
2311	channels->rx_count = 0;
2312	channels->tx_count = 0;
2313	channels->other_count = 0;
2314}
2315
2316static int virtnet_set_link_ksettings(struct net_device *dev,
2317				      const struct ethtool_link_ksettings *cmd)
2318{
2319	struct virtnet_info *vi = netdev_priv(dev);
2320
2321	return ethtool_virtdev_set_link_ksettings(dev, cmd,
2322						  &vi->speed, &vi->duplex);
2323}
2324
2325static int virtnet_get_link_ksettings(struct net_device *dev,
2326				      struct ethtool_link_ksettings *cmd)
2327{
2328	struct virtnet_info *vi = netdev_priv(dev);
2329
2330	cmd->base.speed = vi->speed;
2331	cmd->base.duplex = vi->duplex;
2332	cmd->base.port = PORT_OTHER;
2333
2334	return 0;
2335}
2336
2337static int virtnet_set_coalesce(struct net_device *dev,
2338				struct ethtool_coalesce *ec)
2339{
2340	struct virtnet_info *vi = netdev_priv(dev);
2341	int i, napi_weight;
2342
2343	if (ec->tx_max_coalesced_frames > 1 ||
2344	    ec->rx_max_coalesced_frames != 1)
2345		return -EINVAL;
2346
2347	napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;
2348	if (napi_weight ^ vi->sq[0].napi.weight) {
2349		if (dev->flags & IFF_UP)
2350			return -EBUSY;
2351		for (i = 0; i < vi->max_queue_pairs; i++)
2352			vi->sq[i].napi.weight = napi_weight;
2353	}
2354
2355	return 0;
2356}
2357
2358static int virtnet_get_coalesce(struct net_device *dev,
2359				struct ethtool_coalesce *ec)
2360{
2361	struct ethtool_coalesce ec_default = {
2362		.cmd = ETHTOOL_GCOALESCE,
2363		.rx_max_coalesced_frames = 1,
2364	};
2365	struct virtnet_info *vi = netdev_priv(dev);
2366
2367	memcpy(ec, &ec_default, sizeof(ec_default));
2368
2369	if (vi->sq[0].napi.weight)
2370		ec->tx_max_coalesced_frames = 1;
2371
2372	return 0;
2373}
2374
2375static void virtnet_init_settings(struct net_device *dev)
2376{
2377	struct virtnet_info *vi = netdev_priv(dev);
2378
2379	vi->speed = SPEED_UNKNOWN;
2380	vi->duplex = DUPLEX_UNKNOWN;
2381}
2382
2383static void virtnet_update_settings(struct virtnet_info *vi)
2384{
2385	u32 speed;
2386	u8 duplex;
2387
2388	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))
2389		return;
2390
2391	virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed);
2392
2393	if (ethtool_validate_speed(speed))
2394		vi->speed = speed;
2395
2396	virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex);
2397
2398	if (ethtool_validate_duplex(duplex))
2399		vi->duplex = duplex;
2400}
2401
2402static const struct ethtool_ops virtnet_ethtool_ops = {
2403	.supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES,
2404	.get_drvinfo = virtnet_get_drvinfo,
2405	.get_link = ethtool_op_get_link,
2406	.get_ringparam = virtnet_get_ringparam,
2407	.get_strings = virtnet_get_strings,
2408	.get_sset_count = virtnet_get_sset_count,
2409	.get_ethtool_stats = virtnet_get_ethtool_stats,
2410	.set_channels = virtnet_set_channels,
2411	.get_channels = virtnet_get_channels,
2412	.get_ts_info = ethtool_op_get_ts_info,
2413	.get_link_ksettings = virtnet_get_link_ksettings,
2414	.set_link_ksettings = virtnet_set_link_ksettings,
2415	.set_coalesce = virtnet_set_coalesce,
2416	.get_coalesce = virtnet_get_coalesce,
2417};
2418
2419static void virtnet_freeze_down(struct virtio_device *vdev)
2420{
2421	struct virtnet_info *vi = vdev->priv;
2422	int i;
2423
2424	/* Make sure no work handler is accessing the device */
2425	flush_work(&vi->config_work);
2426
2427	netif_tx_lock_bh(vi->dev);
2428	netif_device_detach(vi->dev);
2429	netif_tx_unlock_bh(vi->dev);
2430	cancel_delayed_work_sync(&vi->refill);
2431
2432	if (netif_running(vi->dev)) {
2433		for (i = 0; i < vi->max_queue_pairs; i++) {
2434			napi_disable(&vi->rq[i].napi);
2435			virtnet_napi_tx_disable(&vi->sq[i].napi);
2436		}
2437	}
2438}
2439
2440static int init_vqs(struct virtnet_info *vi);
2441
2442static int virtnet_restore_up(struct virtio_device *vdev)
2443{
2444	struct virtnet_info *vi = vdev->priv;
2445	int err, i;
2446
2447	err = init_vqs(vi);
2448	if (err)
2449		return err;
2450
2451	virtio_device_ready(vdev);
2452
2453	if (netif_running(vi->dev)) {
2454		for (i = 0; i < vi->curr_queue_pairs; i++)
2455			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
2456				schedule_delayed_work(&vi->refill, 0);
2457
2458		for (i = 0; i < vi->max_queue_pairs; i++) {
2459			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2460			virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2461					       &vi->sq[i].napi);
2462		}
2463	}
2464
2465	netif_tx_lock_bh(vi->dev);
2466	netif_device_attach(vi->dev);
2467	netif_tx_unlock_bh(vi->dev);
2468	return err;
2469}
2470
2471static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
2472{
2473	struct scatterlist sg;
2474	vi->ctrl->offloads = cpu_to_virtio64(vi->vdev, offloads);
2475
2476	sg_init_one(&sg, &vi->ctrl->offloads, sizeof(vi->ctrl->offloads));
2477
2478	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
2479				  VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
2480		dev_warn(&vi->dev->dev, "Fail to set guest offload.\n");
2481		return -EINVAL;
2482	}
2483
2484	return 0;
2485}
2486
2487static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
2488{
2489	u64 offloads = 0;
2490
2491	if (!vi->guest_offloads)
2492		return 0;
2493
2494	return virtnet_set_guest_offloads(vi, offloads);
2495}
2496
2497static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
2498{
2499	u64 offloads = vi->guest_offloads;
2500
2501	if (!vi->guest_offloads)
2502		return 0;
2503
2504	return virtnet_set_guest_offloads(vi, offloads);
2505}
2506
2507static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
2508			   struct netlink_ext_ack *extack)
2509{
2510	unsigned long int max_sz = PAGE_SIZE - sizeof(struct padded_vnet_hdr);
2511	struct virtnet_info *vi = netdev_priv(dev);
2512	struct bpf_prog *old_prog;
2513	u16 xdp_qp = 0, curr_qp;
2514	int i, err;
2515
2516	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
2517	    && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
2518	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
2519	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
2520		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) ||
2521		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))) {
2522		NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first");
2523		return -EOPNOTSUPP;
2524	}
2525
2526	if (vi->mergeable_rx_bufs && !vi->any_header_sg) {
2527		NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required");
2528		return -EINVAL;
2529	}
2530
2531	if (dev->mtu > max_sz) {
2532		NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
2533		netdev_warn(dev, "XDP requires MTU less than %lu\n", max_sz);
2534		return -EINVAL;
2535	}
2536
2537	curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs;
2538	if (prog)
2539		xdp_qp = nr_cpu_ids;
2540
2541	/* XDP requires extra queues for XDP_TX */
2542	if (curr_qp + xdp_qp > vi->max_queue_pairs) {
2543		netdev_warn(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n",
 
2544			    curr_qp + xdp_qp, vi->max_queue_pairs);
2545		xdp_qp = 0;
2546	}
2547
2548	old_prog = rtnl_dereference(vi->rq[0].xdp_prog);
2549	if (!prog && !old_prog)
2550		return 0;
2551
2552	if (prog)
2553		bpf_prog_add(prog, vi->max_queue_pairs - 1);
2554
2555	/* Make sure NAPI is not using any XDP TX queues for RX. */
2556	if (netif_running(dev)) {
2557		for (i = 0; i < vi->max_queue_pairs; i++) {
2558			napi_disable(&vi->rq[i].napi);
2559			virtnet_napi_tx_disable(&vi->sq[i].napi);
2560		}
2561	}
2562
2563	if (!prog) {
2564		for (i = 0; i < vi->max_queue_pairs; i++) {
2565			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
2566			if (i == 0)
2567				virtnet_restore_guest_offloads(vi);
2568		}
2569		synchronize_net();
2570	}
2571
2572	err = _virtnet_set_queues(vi, curr_qp + xdp_qp);
2573	if (err)
2574		goto err;
2575	netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
2576	vi->xdp_queue_pairs = xdp_qp;
2577
2578	if (prog) {
2579		vi->xdp_enabled = true;
2580		for (i = 0; i < vi->max_queue_pairs; i++) {
2581			rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
2582			if (i == 0 && !old_prog)
2583				virtnet_clear_guest_offloads(vi);
2584		}
2585	} else {
2586		vi->xdp_enabled = false;
2587	}
2588
2589	for (i = 0; i < vi->max_queue_pairs; i++) {
2590		if (old_prog)
2591			bpf_prog_put(old_prog);
2592		if (netif_running(dev)) {
2593			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2594			virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2595					       &vi->sq[i].napi);
2596		}
2597	}
2598
2599	return 0;
2600
2601err:
2602	if (!prog) {
2603		virtnet_clear_guest_offloads(vi);
2604		for (i = 0; i < vi->max_queue_pairs; i++)
2605			rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog);
2606	}
2607
2608	if (netif_running(dev)) {
2609		for (i = 0; i < vi->max_queue_pairs; i++) {
2610			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
2611			virtnet_napi_tx_enable(vi, vi->sq[i].vq,
2612					       &vi->sq[i].napi);
2613		}
2614	}
2615	if (prog)
2616		bpf_prog_sub(prog, vi->max_queue_pairs - 1);
2617	return err;
2618}
2619
2620static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2621{
2622	switch (xdp->command) {
2623	case XDP_SETUP_PROG:
2624		return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
2625	default:
2626		return -EINVAL;
2627	}
2628}
2629
2630static int virtnet_get_phys_port_name(struct net_device *dev, char *buf,
2631				      size_t len)
2632{
2633	struct virtnet_info *vi = netdev_priv(dev);
2634	int ret;
2635
2636	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY))
2637		return -EOPNOTSUPP;
2638
2639	ret = snprintf(buf, len, "sby");
2640	if (ret >= len)
2641		return -EOPNOTSUPP;
2642
2643	return 0;
2644}
2645
2646static int virtnet_set_features(struct net_device *dev,
2647				netdev_features_t features)
2648{
2649	struct virtnet_info *vi = netdev_priv(dev);
2650	u64 offloads;
2651	int err;
2652
2653	if ((dev->features ^ features) & NETIF_F_GRO_HW) {
2654		if (vi->xdp_enabled)
2655			return -EBUSY;
2656
2657		if (features & NETIF_F_GRO_HW)
2658			offloads = vi->guest_offloads_capable;
2659		else
2660			offloads = vi->guest_offloads_capable &
2661				   ~GUEST_OFFLOAD_GRO_HW_MASK;
2662
2663		err = virtnet_set_guest_offloads(vi, offloads);
2664		if (err)
2665			return err;
2666		vi->guest_offloads = offloads;
2667	}
2668
2669	return 0;
2670}
2671
2672static const struct net_device_ops virtnet_netdev = {
2673	.ndo_open            = virtnet_open,
2674	.ndo_stop   	     = virtnet_close,
2675	.ndo_start_xmit      = start_xmit,
2676	.ndo_validate_addr   = eth_validate_addr,
2677	.ndo_set_mac_address = virtnet_set_mac_address,
2678	.ndo_set_rx_mode     = virtnet_set_rx_mode,
2679	.ndo_get_stats64     = virtnet_stats,
2680	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
2681	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
2682	.ndo_bpf		= virtnet_xdp,
2683	.ndo_xdp_xmit		= virtnet_xdp_xmit,
2684	.ndo_features_check	= passthru_features_check,
2685	.ndo_get_phys_port_name	= virtnet_get_phys_port_name,
2686	.ndo_set_features	= virtnet_set_features,
2687};
2688
2689static void virtnet_config_changed_work(struct work_struct *work)
2690{
2691	struct virtnet_info *vi =
2692		container_of(work, struct virtnet_info, config_work);
2693	u16 v;
2694
2695	if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
2696				 struct virtio_net_config, status, &v) < 0)
2697		return;
2698
2699	if (v & VIRTIO_NET_S_ANNOUNCE) {
2700		netdev_notify_peers(vi->dev);
2701		virtnet_ack_link_announce(vi);
2702	}
2703
2704	/* Ignore unknown (future) status bits */
2705	v &= VIRTIO_NET_S_LINK_UP;
2706
2707	if (vi->status == v)
2708		return;
2709
2710	vi->status = v;
2711
2712	if (vi->status & VIRTIO_NET_S_LINK_UP) {
2713		virtnet_update_settings(vi);
2714		netif_carrier_on(vi->dev);
2715		netif_tx_wake_all_queues(vi->dev);
2716	} else {
2717		netif_carrier_off(vi->dev);
2718		netif_tx_stop_all_queues(vi->dev);
2719	}
2720}
2721
2722static void virtnet_config_changed(struct virtio_device *vdev)
2723{
2724	struct virtnet_info *vi = vdev->priv;
2725
2726	schedule_work(&vi->config_work);
2727}
2728
2729static void virtnet_free_queues(struct virtnet_info *vi)
2730{
2731	int i;
2732
2733	for (i = 0; i < vi->max_queue_pairs; i++) {
2734		__netif_napi_del(&vi->rq[i].napi);
2735		__netif_napi_del(&vi->sq[i].napi);
 
2736	}
2737
2738	/* We called __netif_napi_del(),
2739	 * we need to respect an RCU grace period before freeing vi->rq
2740	 */
2741	synchronize_net();
2742
2743	kfree(vi->rq);
2744	kfree(vi->sq);
2745	kfree(vi->ctrl);
2746}
2747
2748static void _free_receive_bufs(struct virtnet_info *vi)
2749{
2750	struct bpf_prog *old_prog;
2751	int i;
2752
2753	for (i = 0; i < vi->max_queue_pairs; i++) {
2754		while (vi->rq[i].pages)
2755			__free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
2756
2757		old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
2758		RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL);
2759		if (old_prog)
2760			bpf_prog_put(old_prog);
2761	}
2762}
2763
2764static void free_receive_bufs(struct virtnet_info *vi)
2765{
2766	rtnl_lock();
2767	_free_receive_bufs(vi);
2768	rtnl_unlock();
2769}
2770
2771static void free_receive_page_frags(struct virtnet_info *vi)
2772{
2773	int i;
2774	for (i = 0; i < vi->max_queue_pairs; i++)
2775		if (vi->rq[i].alloc_frag.page)
2776			put_page(vi->rq[i].alloc_frag.page);
2777}
2778
2779static void free_unused_bufs(struct virtnet_info *vi)
2780{
2781	void *buf;
2782	int i;
2783
2784	for (i = 0; i < vi->max_queue_pairs; i++) {
2785		struct virtqueue *vq = vi->sq[i].vq;
2786		while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2787			if (!is_xdp_frame(buf))
2788				dev_kfree_skb(buf);
2789			else
2790				xdp_return_frame(ptr_to_xdp(buf));
2791		}
2792	}
2793
2794	for (i = 0; i < vi->max_queue_pairs; i++) {
2795		struct virtqueue *vq = vi->rq[i].vq;
2796
2797		while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
2798			if (vi->mergeable_rx_bufs) {
2799				put_page(virt_to_head_page(buf));
2800			} else if (vi->big_packets) {
2801				give_pages(&vi->rq[i], buf);
2802			} else {
2803				put_page(virt_to_head_page(buf));
2804			}
2805		}
2806	}
2807}
2808
2809static void virtnet_del_vqs(struct virtnet_info *vi)
2810{
2811	struct virtio_device *vdev = vi->vdev;
2812
2813	virtnet_clean_affinity(vi);
2814
2815	vdev->config->del_vqs(vdev);
2816
2817	virtnet_free_queues(vi);
2818}
2819
2820/* How large should a single buffer be so a queue full of these can fit at
2821 * least one full packet?
2822 * Logic below assumes the mergeable buffer header is used.
2823 */
2824static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
2825{
2826	const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2827	unsigned int rq_size = virtqueue_get_vring_size(vq);
2828	unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
2829	unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
2830	unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size);
2831
2832	return max(max(min_buf_len, hdr_len) - hdr_len,
2833		   (unsigned int)GOOD_PACKET_LEN);
2834}
2835
2836static int virtnet_find_vqs(struct virtnet_info *vi)
2837{
2838	vq_callback_t **callbacks;
2839	struct virtqueue **vqs;
2840	int ret = -ENOMEM;
2841	int i, total_vqs;
2842	const char **names;
2843	bool *ctx;
2844
2845	/* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
2846	 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
2847	 * possible control vq.
2848	 */
2849	total_vqs = vi->max_queue_pairs * 2 +
2850		    virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);
2851
2852	/* Allocate space for find_vqs parameters */
2853	vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
2854	if (!vqs)
2855		goto err_vq;
2856	callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL);
2857	if (!callbacks)
2858		goto err_callback;
2859	names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL);
2860	if (!names)
2861		goto err_names;
2862	if (!vi->big_packets || vi->mergeable_rx_bufs) {
2863		ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL);
2864		if (!ctx)
2865			goto err_ctx;
2866	} else {
2867		ctx = NULL;
2868	}
2869
2870	/* Parameters for control virtqueue, if any */
2871	if (vi->has_cvq) {
2872		callbacks[total_vqs - 1] = NULL;
2873		names[total_vqs - 1] = "control";
2874	}
2875
2876	/* Allocate/initialize parameters for send/receive virtqueues */
2877	for (i = 0; i < vi->max_queue_pairs; i++) {
2878		callbacks[rxq2vq(i)] = skb_recv_done;
2879		callbacks[txq2vq(i)] = skb_xmit_done;
2880		sprintf(vi->rq[i].name, "input.%d", i);
2881		sprintf(vi->sq[i].name, "output.%d", i);
2882		names[rxq2vq(i)] = vi->rq[i].name;
2883		names[txq2vq(i)] = vi->sq[i].name;
2884		if (ctx)
2885			ctx[rxq2vq(i)] = true;
2886	}
2887
2888	ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks,
2889				  names, ctx, NULL);
2890	if (ret)
2891		goto err_find;
2892
2893	if (vi->has_cvq) {
2894		vi->cvq = vqs[total_vqs - 1];
2895		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
2896			vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2897	}
2898
2899	for (i = 0; i < vi->max_queue_pairs; i++) {
2900		vi->rq[i].vq = vqs[rxq2vq(i)];
2901		vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq);
2902		vi->sq[i].vq = vqs[txq2vq(i)];
2903	}
2904
2905	/* run here: ret == 0. */
2906
2907
2908err_find:
2909	kfree(ctx);
2910err_ctx:
2911	kfree(names);
2912err_names:
2913	kfree(callbacks);
2914err_callback:
2915	kfree(vqs);
2916err_vq:
2917	return ret;
2918}
2919
2920static int virtnet_alloc_queues(struct virtnet_info *vi)
2921{
2922	int i;
2923
2924	if (vi->has_cvq) {
2925		vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL);
2926		if (!vi->ctrl)
2927			goto err_ctrl;
2928	} else {
2929		vi->ctrl = NULL;
2930	}
2931	vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL);
2932	if (!vi->sq)
2933		goto err_sq;
2934	vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL);
2935	if (!vi->rq)
2936		goto err_rq;
2937
2938	INIT_DELAYED_WORK(&vi->refill, refill_work);
2939	for (i = 0; i < vi->max_queue_pairs; i++) {
2940		vi->rq[i].pages = NULL;
2941		netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
2942			       napi_weight);
2943		netif_tx_napi_add(vi->dev, &vi->sq[i].napi, virtnet_poll_tx,
2944				  napi_tx ? napi_weight : 0);
2945
2946		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
2947		ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
2948		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
2949
2950		u64_stats_init(&vi->rq[i].stats.syncp);
2951		u64_stats_init(&vi->sq[i].stats.syncp);
2952	}
2953
2954	return 0;
2955
2956err_rq:
2957	kfree(vi->sq);
2958err_sq:
2959	kfree(vi->ctrl);
2960err_ctrl:
2961	return -ENOMEM;
2962}
2963
2964static int init_vqs(struct virtnet_info *vi)
2965{
2966	int ret;
2967
2968	/* Allocate send & receive queues */
2969	ret = virtnet_alloc_queues(vi);
2970	if (ret)
2971		goto err;
2972
2973	ret = virtnet_find_vqs(vi);
2974	if (ret)
2975		goto err_free;
2976
2977	get_online_cpus();
2978	virtnet_set_affinity(vi);
2979	put_online_cpus();
2980
2981	return 0;
2982
2983err_free:
2984	virtnet_free_queues(vi);
2985err:
2986	return ret;
2987}
2988
2989#ifdef CONFIG_SYSFS
2990static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
2991		char *buf)
2992{
2993	struct virtnet_info *vi = netdev_priv(queue->dev);
2994	unsigned int queue_index = get_netdev_rx_queue_index(queue);
2995	unsigned int headroom = virtnet_get_headroom(vi);
2996	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
2997	struct ewma_pkt_len *avg;
2998
2999	BUG_ON(queue_index >= vi->max_queue_pairs);
3000	avg = &vi->rq[queue_index].mrg_avg_pkt_len;
3001	return sprintf(buf, "%u\n",
3002		       get_mergeable_buf_len(&vi->rq[queue_index], avg,
3003				       SKB_DATA_ALIGN(headroom + tailroom)));
3004}
3005
3006static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
3007	__ATTR_RO(mergeable_rx_buffer_size);
3008
3009static struct attribute *virtio_net_mrg_rx_attrs[] = {
3010	&mergeable_rx_buffer_size_attribute.attr,
3011	NULL
3012};
3013
3014static const struct attribute_group virtio_net_mrg_rx_group = {
3015	.name = "virtio_net",
3016	.attrs = virtio_net_mrg_rx_attrs
3017};
3018#endif
3019
3020static bool virtnet_fail_on_feature(struct virtio_device *vdev,
3021				    unsigned int fbit,
3022				    const char *fname, const char *dname)
3023{
3024	if (!virtio_has_feature(vdev, fbit))
3025		return false;
3026
3027	dev_err(&vdev->dev, "device advertises feature %s but not %s",
3028		fname, dname);
3029
3030	return true;
3031}
3032
3033#define VIRTNET_FAIL_ON(vdev, fbit, dbit)			\
3034	virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
3035
3036static bool virtnet_validate_features(struct virtio_device *vdev)
3037{
3038	if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) &&
3039	    (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX,
3040			     "VIRTIO_NET_F_CTRL_VQ") ||
3041	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN,
3042			     "VIRTIO_NET_F_CTRL_VQ") ||
3043	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE,
3044			     "VIRTIO_NET_F_CTRL_VQ") ||
3045	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
3046	     VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
3047			     "VIRTIO_NET_F_CTRL_VQ"))) {
3048		return false;
3049	}
3050
3051	return true;
3052}
3053
3054#define MIN_MTU ETH_MIN_MTU
3055#define MAX_MTU ETH_MAX_MTU
3056
3057static int virtnet_validate(struct virtio_device *vdev)
3058{
3059	if (!vdev->config->get) {
3060		dev_err(&vdev->dev, "%s failure: config access disabled\n",
3061			__func__);
3062		return -EINVAL;
3063	}
3064
3065	if (!virtnet_validate_features(vdev))
3066		return -EINVAL;
3067
3068	if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
3069		int mtu = virtio_cread16(vdev,
3070					 offsetof(struct virtio_net_config,
3071						  mtu));
3072		if (mtu < MIN_MTU)
3073			__virtio_clear_bit(vdev, VIRTIO_NET_F_MTU);
3074	}
3075
3076	return 0;
3077}
3078
3079static int virtnet_probe(struct virtio_device *vdev)
3080{
3081	int i, err = -ENOMEM;
3082	struct net_device *dev;
3083	struct virtnet_info *vi;
3084	u16 max_queue_pairs;
3085	int mtu;
3086
3087	/* Find if host supports multiqueue virtio_net device */
3088	err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
3089				   struct virtio_net_config,
3090				   max_virtqueue_pairs, &max_queue_pairs);
3091
3092	/* We need at least 2 queue's */
3093	if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
3094	    max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
3095	    !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
3096		max_queue_pairs = 1;
3097
3098	/* Allocate ourselves a network device with room for our info */
3099	dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
3100	if (!dev)
3101		return -ENOMEM;
3102
3103	/* Set up network device as normal. */
3104	dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE |
3105			   IFF_TX_SKB_NO_LINEAR;
3106	dev->netdev_ops = &virtnet_netdev;
3107	dev->features = NETIF_F_HIGHDMA;
3108
3109	dev->ethtool_ops = &virtnet_ethtool_ops;
3110	SET_NETDEV_DEV(dev, &vdev->dev);
3111
3112	/* Do we support "hardware" checksums? */
3113	if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
3114		/* This opens up the world of extra features. */
3115		dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG;
3116		if (csum)
3117			dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
3118
3119		if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
3120			dev->hw_features |= NETIF_F_TSO
3121				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
3122		}
3123		/* Individual feature bits: what can host handle? */
3124		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
3125			dev->hw_features |= NETIF_F_TSO;
3126		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
3127			dev->hw_features |= NETIF_F_TSO6;
3128		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
3129			dev->hw_features |= NETIF_F_TSO_ECN;
3130
3131		dev->features |= NETIF_F_GSO_ROBUST;
3132
3133		if (gso)
3134			dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
3135		/* (!csum && gso) case will be fixed by register_netdev() */
3136	}
3137	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
3138		dev->features |= NETIF_F_RXCSUM;
3139	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
3140	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6))
3141		dev->features |= NETIF_F_GRO_HW;
3142	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS))
3143		dev->hw_features |= NETIF_F_GRO_HW;
3144
3145	dev->vlan_features = dev->features;
3146
3147	/* MTU range: 68 - 65535 */
3148	dev->min_mtu = MIN_MTU;
3149	dev->max_mtu = MAX_MTU;
3150
3151	/* Configuration may specify what MAC to use.  Otherwise random. */
3152	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
3153		virtio_cread_bytes(vdev,
3154				   offsetof(struct virtio_net_config, mac),
3155				   dev->dev_addr, dev->addr_len);
3156	else
3157		eth_hw_addr_random(dev);
3158
3159	/* Set up our device-specific information */
3160	vi = netdev_priv(dev);
3161	vi->dev = dev;
3162	vi->vdev = vdev;
3163	vdev->priv = vi;
3164
3165	INIT_WORK(&vi->config_work, virtnet_config_changed_work);
3166
3167	/* If we can receive ANY GSO packets, we must allocate large ones. */
3168	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
3169	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
3170	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
3171	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
3172		vi->big_packets = true;
3173
3174	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
3175		vi->mergeable_rx_bufs = true;
3176
3177	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
3178	    virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3179		vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
3180	else
3181		vi->hdr_len = sizeof(struct virtio_net_hdr);
3182
3183	if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) ||
3184	    virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3185		vi->any_header_sg = true;
3186
3187	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
3188		vi->has_cvq = true;
3189
3190	if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) {
3191		mtu = virtio_cread16(vdev,
3192				     offsetof(struct virtio_net_config,
3193					      mtu));
3194		if (mtu < dev->min_mtu) {
3195			/* Should never trigger: MTU was previously validated
3196			 * in virtnet_validate.
3197			 */
3198			dev_err(&vdev->dev,
3199				"device MTU appears to have changed it is now %d < %d",
3200				mtu, dev->min_mtu);
3201			err = -EINVAL;
3202			goto free;
3203		}
3204
3205		dev->mtu = mtu;
3206		dev->max_mtu = mtu;
3207
3208		/* TODO: size buffers correctly in this case. */
3209		if (dev->mtu > ETH_DATA_LEN)
3210			vi->big_packets = true;
3211	}
3212
3213	if (vi->any_header_sg)
3214		dev->needed_headroom = vi->hdr_len;
3215
3216	/* Enable multiqueue by default */
3217	if (num_online_cpus() >= max_queue_pairs)
3218		vi->curr_queue_pairs = max_queue_pairs;
3219	else
3220		vi->curr_queue_pairs = num_online_cpus();
3221	vi->max_queue_pairs = max_queue_pairs;
3222
3223	/* Allocate/initialize the rx/tx queues, and invoke find_vqs */
3224	err = init_vqs(vi);
3225	if (err)
3226		goto free;
3227
3228#ifdef CONFIG_SYSFS
3229	if (vi->mergeable_rx_bufs)
3230		dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
3231#endif
3232	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
3233	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
3234
3235	virtnet_init_settings(dev);
3236
3237	if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
3238		vi->failover = net_failover_create(vi->dev);
3239		if (IS_ERR(vi->failover)) {
3240			err = PTR_ERR(vi->failover);
3241			goto free_vqs;
3242		}
3243	}
3244
3245	err = register_netdev(dev);
3246	if (err) {
3247		pr_debug("virtio_net: registering device failed\n");
3248		goto free_failover;
3249	}
3250
3251	virtio_device_ready(vdev);
3252
3253	err = virtnet_cpu_notif_add(vi);
3254	if (err) {
3255		pr_debug("virtio_net: registering cpu notifier failed\n");
3256		goto free_unregister_netdev;
3257	}
3258
3259	virtnet_set_queues(vi, vi->curr_queue_pairs);
3260
3261	/* Assume link up if device can't report link status,
3262	   otherwise get link status from config. */
3263	netif_carrier_off(dev);
3264	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
3265		schedule_work(&vi->config_work);
3266	} else {
3267		vi->status = VIRTIO_NET_S_LINK_UP;
3268		virtnet_update_settings(vi);
3269		netif_carrier_on(dev);
3270	}
3271
3272	for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
3273		if (virtio_has_feature(vi->vdev, guest_offloads[i]))
3274			set_bit(guest_offloads[i], &vi->guest_offloads);
3275	vi->guest_offloads_capable = vi->guest_offloads;
3276
3277	pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
3278		 dev->name, max_queue_pairs);
3279
3280	return 0;
3281
3282free_unregister_netdev:
3283	vi->vdev->config->reset(vdev);
3284
3285	unregister_netdev(dev);
3286free_failover:
3287	net_failover_destroy(vi->failover);
3288free_vqs:
3289	cancel_delayed_work_sync(&vi->refill);
3290	free_receive_page_frags(vi);
3291	virtnet_del_vqs(vi);
3292free:
3293	free_netdev(dev);
3294	return err;
3295}
3296
3297static void remove_vq_common(struct virtnet_info *vi)
3298{
3299	vi->vdev->config->reset(vi->vdev);
3300
3301	/* Free unused buffers in both send and recv, if any. */
3302	free_unused_bufs(vi);
3303
3304	free_receive_bufs(vi);
3305
3306	free_receive_page_frags(vi);
3307
3308	virtnet_del_vqs(vi);
3309}
3310
3311static void virtnet_remove(struct virtio_device *vdev)
3312{
3313	struct virtnet_info *vi = vdev->priv;
3314
3315	virtnet_cpu_notif_remove(vi);
3316
3317	/* Make sure no work handler is accessing the device. */
3318	flush_work(&vi->config_work);
3319
3320	unregister_netdev(vi->dev);
3321
3322	net_failover_destroy(vi->failover);
3323
3324	remove_vq_common(vi);
3325
3326	free_netdev(vi->dev);
3327}
3328
3329static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
3330{
3331	struct virtnet_info *vi = vdev->priv;
3332
3333	virtnet_cpu_notif_remove(vi);
3334	virtnet_freeze_down(vdev);
3335	remove_vq_common(vi);
3336
3337	return 0;
3338}
3339
3340static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
3341{
3342	struct virtnet_info *vi = vdev->priv;
3343	int err;
3344
3345	err = virtnet_restore_up(vdev);
3346	if (err)
3347		return err;
3348	virtnet_set_queues(vi, vi->curr_queue_pairs);
3349
3350	err = virtnet_cpu_notif_add(vi);
3351	if (err) {
3352		virtnet_freeze_down(vdev);
3353		remove_vq_common(vi);
3354		return err;
3355	}
3356
3357	return 0;
3358}
3359
3360static struct virtio_device_id id_table[] = {
3361	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
3362	{ 0 },
3363};
3364
3365#define VIRTNET_FEATURES \
3366	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
3367	VIRTIO_NET_F_MAC, \
3368	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
3369	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
3370	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
3371	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
3372	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
3373	VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
3374	VIRTIO_NET_F_CTRL_MAC_ADDR, \
3375	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3376	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY
3377
3378static unsigned int features[] = {
3379	VIRTNET_FEATURES,
3380};
3381
3382static unsigned int features_legacy[] = {
3383	VIRTNET_FEATURES,
3384	VIRTIO_NET_F_GSO,
3385	VIRTIO_F_ANY_LAYOUT,
3386};
3387
3388static struct virtio_driver virtio_net_driver = {
3389	.feature_table = features,
3390	.feature_table_size = ARRAY_SIZE(features),
3391	.feature_table_legacy = features_legacy,
3392	.feature_table_size_legacy = ARRAY_SIZE(features_legacy),
3393	.driver.name =	KBUILD_MODNAME,
3394	.driver.owner =	THIS_MODULE,
3395	.id_table =	id_table,
3396	.validate =	virtnet_validate,
3397	.probe =	virtnet_probe,
3398	.remove =	virtnet_remove,
3399	.config_changed = virtnet_config_changed,
3400#ifdef CONFIG_PM_SLEEP
3401	.freeze =	virtnet_freeze,
3402	.restore =	virtnet_restore,
3403#endif
3404};
3405
3406static __init int virtio_net_driver_init(void)
3407{
3408	int ret;
3409
3410	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online",
3411				      virtnet_cpu_online,
3412				      virtnet_cpu_down_prep);
3413	if (ret < 0)
3414		goto out;
3415	virtionet_online = ret;
3416	ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead",
3417				      NULL, virtnet_cpu_dead);
3418	if (ret)
3419		goto err_dead;
3420
3421        ret = register_virtio_driver(&virtio_net_driver);
3422	if (ret)
3423		goto err_virtio;
3424	return 0;
3425err_virtio:
3426	cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3427err_dead:
3428	cpuhp_remove_multi_state(virtionet_online);
3429out:
3430	return ret;
3431}
3432module_init(virtio_net_driver_init);
3433
3434static __exit void virtio_net_driver_exit(void)
3435{
3436	unregister_virtio_driver(&virtio_net_driver);
3437	cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
3438	cpuhp_remove_multi_state(virtionet_online);
3439}
3440module_exit(virtio_net_driver_exit);
3441
3442MODULE_DEVICE_TABLE(virtio, id_table);
3443MODULE_DESCRIPTION("Virtio network driver");
3444MODULE_LICENSE("GPL");