Linux Audio

Check our new training course

Loading...
v3.15
   1/*
   2 * Virtual network driver for conversing with remote driver backends.
   3 *
   4 * Copyright (c) 2002-2005, K A Fraser
   5 * Copyright (c) 2005, XenSource Ltd
   6 *
   7 * This program is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU General Public License version 2
   9 * as published by the Free Software Foundation; or, when distributed
  10 * separately from the Linux kernel or incorporated into other
  11 * software packages, subject to the following license:
  12 *
  13 * Permission is hereby granted, free of charge, to any person obtaining a copy
  14 * of this source file (the "Software"), to deal in the Software without
  15 * restriction, including without limitation the rights to use, copy, modify,
  16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  17 * and to permit persons to whom the Software is furnished to do so, subject to
  18 * the following conditions:
  19 *
  20 * The above copyright notice and this permission notice shall be included in
  21 * all copies or substantial portions of the Software.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  29 * IN THE SOFTWARE.
  30 */
  31
  32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  33
  34#include <linux/module.h>
  35#include <linux/kernel.h>
  36#include <linux/netdevice.h>
  37#include <linux/etherdevice.h>
  38#include <linux/skbuff.h>
  39#include <linux/ethtool.h>
  40#include <linux/if_ether.h>
  41#include <net/tcp.h>
  42#include <linux/udp.h>
  43#include <linux/moduleparam.h>
  44#include <linux/mm.h>
  45#include <linux/slab.h>
  46#include <net/ip.h>
 
 
 
  47
  48#include <asm/xen/page.h>
  49#include <xen/xen.h>
  50#include <xen/xenbus.h>
  51#include <xen/events.h>
  52#include <xen/page.h>
  53#include <xen/platform_pci.h>
  54#include <xen/grant_table.h>
  55
  56#include <xen/interface/io/netif.h>
  57#include <xen/interface/memory.h>
  58#include <xen/interface/grant_table.h>
  59
 
 
 
 
 
 
 
 
 
 
 
 
 
  60static const struct ethtool_ops xennet_ethtool_ops;
  61
  62struct netfront_cb {
  63	int pull_to;
  64};
  65
  66#define NETFRONT_SKB_CB(skb)	((struct netfront_cb *)((skb)->cb))
  67
  68#define RX_COPY_THRESHOLD 256
  69
  70#define GRANT_INVALID_REF	0
 
 
 
 
 
 
 
 
 
 
  71
  72#define NET_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, PAGE_SIZE)
  73#define NET_RX_RING_SIZE __CONST_RING_SIZE(xen_netif_rx, PAGE_SIZE)
  74#define TX_MAX_TARGET min_t(int, NET_TX_RING_SIZE, 256)
  75
  76struct netfront_stats {
  77	u64			rx_packets;
  78	u64			tx_packets;
  79	u64			rx_bytes;
  80	u64			tx_bytes;
  81	struct u64_stats_sync	syncp;
  82};
  83
  84struct netfront_info {
  85	struct list_head list;
  86	struct net_device *netdev;
 
 
 
 
 
  87
  88	struct napi_struct napi;
  89
  90	/* Split event channels support, tx_* == rx_* when using
  91	 * single event channel.
  92	 */
  93	unsigned int tx_evtchn, rx_evtchn;
  94	unsigned int tx_irq, rx_irq;
  95	/* Only used when split event channels support is enabled */
  96	char tx_irq_name[IFNAMSIZ+4]; /* DEVNAME-tx */
  97	char rx_irq_name[IFNAMSIZ+4]; /* DEVNAME-rx */
  98
  99	struct xenbus_device *xbdev;
 100
 101	spinlock_t   tx_lock;
 102	struct xen_netif_tx_front_ring tx;
 103	int tx_ring_ref;
 104
 105	/*
 106	 * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries
 107	 * are linked from tx_skb_freelist through skb_entry.link.
 108	 *
 109	 *  NB. Freelist index entries are always going to be less than
 110	 *  PAGE_OFFSET, whereas pointers to skbs will always be equal or
 111	 *  greater than PAGE_OFFSET: we use this property to distinguish
 112	 *  them.
 113	 */
 114	union skb_entry {
 115		struct sk_buff *skb;
 116		unsigned long link;
 117	} tx_skbs[NET_TX_RING_SIZE];
 118	grant_ref_t gref_tx_head;
 119	grant_ref_t grant_tx_ref[NET_TX_RING_SIZE];
 120	struct page *grant_tx_page[NET_TX_RING_SIZE];
 121	unsigned tx_skb_freelist;
 
 122
 123	spinlock_t   rx_lock ____cacheline_aligned_in_smp;
 124	struct xen_netif_rx_front_ring rx;
 125	int rx_ring_ref;
 126
 127	/* Receive-ring batched refills. */
 128#define RX_MIN_TARGET 8
 129#define RX_DFL_MIN_TARGET 64
 130#define RX_MAX_TARGET min_t(int, NET_RX_RING_SIZE, 256)
 131	unsigned rx_min_target, rx_max_target, rx_target;
 132	struct sk_buff_head rx_batch;
 133
 134	struct timer_list rx_refill_timer;
 135
 136	struct sk_buff *rx_skbs[NET_RX_RING_SIZE];
 137	grant_ref_t gref_rx_head;
 138	grant_ref_t grant_rx_ref[NET_RX_RING_SIZE];
 139
 140	unsigned long rx_pfn_array[NET_RX_RING_SIZE];
 141	struct multicall_entry rx_mcl[NET_RX_RING_SIZE+1];
 142	struct mmu_update rx_mmu[NET_RX_RING_SIZE];
 
 
 
 
 
 
 
 
 
 
 
 
 143
 144	/* Statistics */
 145	struct netfront_stats __percpu *stats;
 
 146
 147	unsigned long rx_gso_checksum_fixup;
 
 
 
 
 
 
 
 
 
 
 148};
 149
 150struct netfront_rx_info {
 151	struct xen_netif_rx_response rx;
 152	struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
 153};
 154
 155static void skb_entry_set_link(union skb_entry *list, unsigned short id)
 156{
 157	list->link = id;
 158}
 159
 160static int skb_entry_is_link(const union skb_entry *list)
 161{
 162	BUILD_BUG_ON(sizeof(list->skb) != sizeof(list->link));
 163	return (unsigned long)list->skb < PAGE_OFFSET;
 164}
 165
 166/*
 167 * Access macros for acquiring freeing slots in tx_skbs[].
 168 */
 169
 170static void add_id_to_freelist(unsigned *head, union skb_entry *list,
 171			       unsigned short id)
 172{
 173	skb_entry_set_link(&list[id], *head);
 174	*head = id;
 175}
 176
 177static unsigned short get_id_from_freelist(unsigned *head,
 178					   union skb_entry *list)
 179{
 180	unsigned int id = *head;
 181	*head = list[id].link;
 
 
 
 
 182	return id;
 183}
 184
 185static int xennet_rxidx(RING_IDX idx)
 186{
 187	return idx & (NET_RX_RING_SIZE - 1);
 188}
 189
 190static struct sk_buff *xennet_get_rx_skb(struct netfront_info *np,
 191					 RING_IDX ri)
 192{
 193	int i = xennet_rxidx(ri);
 194	struct sk_buff *skb = np->rx_skbs[i];
 195	np->rx_skbs[i] = NULL;
 196	return skb;
 197}
 198
 199static grant_ref_t xennet_get_rx_ref(struct netfront_info *np,
 200					    RING_IDX ri)
 201{
 202	int i = xennet_rxidx(ri);
 203	grant_ref_t ref = np->grant_rx_ref[i];
 204	np->grant_rx_ref[i] = GRANT_INVALID_REF;
 205	return ref;
 206}
 207
 208#ifdef CONFIG_SYSFS
 209static int xennet_sysfs_addif(struct net_device *netdev);
 210static void xennet_sysfs_delif(struct net_device *netdev);
 211#else /* !CONFIG_SYSFS */
 212#define xennet_sysfs_addif(dev) (0)
 213#define xennet_sysfs_delif(dev) do { } while (0)
 214#endif
 215
 216static bool xennet_can_sg(struct net_device *dev)
 217{
 218	return dev->features & NETIF_F_SG;
 219}
 220
 221
 222static void rx_refill_timeout(unsigned long data)
 223{
 224	struct net_device *dev = (struct net_device *)data;
 225	struct netfront_info *np = netdev_priv(dev);
 226	napi_schedule(&np->napi);
 227}
 228
 229static int netfront_tx_slot_available(struct netfront_info *np)
 230{
 231	return (np->tx.req_prod_pvt - np->tx.rsp_cons) <
 232		(TX_MAX_TARGET - MAX_SKB_FRAGS - 2);
 233}
 234
 235static void xennet_maybe_wake_tx(struct net_device *dev)
 236{
 237	struct netfront_info *np = netdev_priv(dev);
 
 238
 239	if (unlikely(netif_queue_stopped(dev)) &&
 240	    netfront_tx_slot_available(np) &&
 241	    likely(netif_running(dev)))
 242		netif_wake_queue(dev);
 243}
 244
 245static void xennet_alloc_rx_buffers(struct net_device *dev)
 
 246{
 247	unsigned short id;
 248	struct netfront_info *np = netdev_priv(dev);
 249	struct sk_buff *skb;
 250	struct page *page;
 251	int i, batch_target, notify;
 252	RING_IDX req_prod = np->rx.req_prod_pvt;
 253	grant_ref_t ref;
 254	unsigned long pfn;
 255	void *vaddr;
 256	struct xen_netif_rx_request *req;
 257
 258	if (unlikely(!netif_carrier_ok(dev)))
 259		return;
 
 
 
 
 
 
 
 
 
 
 
 
 260
 261	/*
 262	 * Allocate skbuffs greedily, even though we batch updates to the
 263	 * receive ring. This creates a less bursty demand on the memory
 264	 * allocator, so should reduce the chance of failed allocation requests
 265	 * both for ourself and for other kernel subsystems.
 266	 */
 267	batch_target = np->rx_target - (req_prod - np->rx.rsp_cons);
 268	for (i = skb_queue_len(&np->rx_batch); i < batch_target; i++) {
 269		skb = __netdev_alloc_skb(dev, RX_COPY_THRESHOLD + NET_IP_ALIGN,
 270					 GFP_ATOMIC | __GFP_NOWARN);
 271		if (unlikely(!skb))
 272			goto no_skb;
 273
 274		/* Align ip header to a 16 bytes boundary */
 275		skb_reserve(skb, NET_IP_ALIGN);
 276
 277		page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
 278		if (!page) {
 279			kfree_skb(skb);
 280no_skb:
 281			/* Could not allocate any skbuffs. Try again later. */
 282			mod_timer(&np->rx_refill_timer,
 283				  jiffies + (HZ/10));
 284
 285			/* Any skbuffs queued for refill? Force them out. */
 286			if (i != 0)
 287				goto refill;
 288			break;
 289		}
 290
 291		skb_add_rx_frag(skb, 0, page, 0, 0, PAGE_SIZE);
 292		__skb_queue_tail(&np->rx_batch, skb);
 293	}
 
 
 294
 295	/* Is the batch large enough to be worthwhile? */
 296	if (i < (np->rx_target/2)) {
 297		if (req_prod > np->rx.sring->req_prod)
 298			goto push;
 299		return;
 300	}
 301
 302	/* Adjust our fill target if we risked running out of buffers. */
 303	if (((req_prod - np->rx.sring->rsp_prod) < (np->rx_target / 4)) &&
 304	    ((np->rx_target *= 2) > np->rx_max_target))
 305		np->rx_target = np->rx_max_target;
 306
 307 refill:
 308	for (i = 0; ; i++) {
 309		skb = __skb_dequeue(&np->rx_batch);
 310		if (skb == NULL)
 311			break;
 312
 313		skb->dev = dev;
 314
 315		id = xennet_rxidx(req_prod + i);
 
 
 
 
 316
 317		BUG_ON(np->rx_skbs[id]);
 318		np->rx_skbs[id] = skb;
 319
 320		ref = gnttab_claim_grant_reference(&np->gref_rx_head);
 321		BUG_ON((signed short)ref < 0);
 322		np->grant_rx_ref[id] = ref;
 323
 324		pfn = page_to_pfn(skb_frag_page(&skb_shinfo(skb)->frags[0]));
 325		vaddr = page_address(skb_frag_page(&skb_shinfo(skb)->frags[0]));
 
 326
 327		req = RING_GET_REQUEST(&np->rx, req_prod + i);
 328		gnttab_grant_foreign_access_ref(ref,
 329						np->xbdev->otherend_id,
 330						pfn_to_mfn(pfn),
 331						0);
 332
 
 
 
 
 
 333		req->id = id;
 334		req->gref = ref;
 335	}
 336
 337	wmb();		/* barrier so backend seens requests */
 338
 339	/* Above is a suitable barrier to ensure backend will see requests. */
 340	np->rx.req_prod_pvt = req_prod + i;
 341 push:
 342	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&np->rx, notify);
 
 
 
 
 
 
 
 
 343	if (notify)
 344		notify_remote_via_irq(np->rx_irq);
 345}
 346
 347static int xennet_open(struct net_device *dev)
 348{
 349	struct netfront_info *np = netdev_priv(dev);
 
 
 
 350
 351	napi_enable(&np->napi);
 
 352
 353	spin_lock_bh(&np->rx_lock);
 354	if (netif_carrier_ok(dev)) {
 355		xennet_alloc_rx_buffers(dev);
 356		np->rx.sring->rsp_event = np->rx.rsp_cons + 1;
 357		if (RING_HAS_UNCONSUMED_RESPONSES(&np->rx))
 358			napi_schedule(&np->napi);
 
 
 
 
 
 
 359	}
 360	spin_unlock_bh(&np->rx_lock);
 361
 362	netif_start_queue(dev);
 363
 364	return 0;
 365}
 366
 367static void xennet_tx_buf_gc(struct net_device *dev)
 368{
 369	RING_IDX cons, prod;
 370	unsigned short id;
 371	struct netfront_info *np = netdev_priv(dev);
 372	struct sk_buff *skb;
 
 
 
 373
 374	BUG_ON(!netif_carrier_ok(dev));
 375
 376	do {
 377		prod = np->tx.sring->rsp_prod;
 
 
 
 
 
 378		rmb(); /* Ensure we see responses up to 'rp'. */
 379
 380		for (cons = np->tx.rsp_cons; cons != prod; cons++) {
 381			struct xen_netif_tx_response *txrsp;
 382
 383			txrsp = RING_GET_RESPONSE(&np->tx, cons);
 384			if (txrsp->status == XEN_NETIF_RSP_NULL)
 
 
 385				continue;
 386
 387			id  = txrsp->id;
 388			skb = np->tx_skbs[id].skb;
 389			if (unlikely(gnttab_query_foreign_access(
 390				np->grant_tx_ref[id]) != 0)) {
 391				pr_alert("%s: warning -- grant still in use by backend domain\n",
 392					 __func__);
 393				BUG();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 394			}
 395			gnttab_end_foreign_access_ref(
 396				np->grant_tx_ref[id], GNTMAP_readonly);
 397			gnttab_release_grant_reference(
 398				&np->gref_tx_head, np->grant_tx_ref[id]);
 399			np->grant_tx_ref[id] = GRANT_INVALID_REF;
 400			np->grant_tx_page[id] = NULL;
 401			add_id_to_freelist(&np->tx_skb_freelist, np->tx_skbs, id);
 402			dev_kfree_skb_irq(skb);
 403		}
 404
 405		np->tx.rsp_cons = prod;
 406
 407		/*
 408		 * Set a new event, then check for race with update of tx_cons.
 409		 * Note that it is essential to schedule a callback, no matter
 410		 * how few buffers are pending. Even if there is space in the
 411		 * transmit ring, higher layers may be blocked because too much
 412		 * data is outstanding: in such cases notification from Xen is
 413		 * likely to be the only kick that we'll get.
 414		 */
 415		np->tx.sring->rsp_event =
 416			prod + ((np->tx.sring->req_prod - prod) >> 1) + 1;
 417		mb();		/* update shared area */
 418	} while ((cons == prod) && (prod != np->tx.sring->rsp_prod));
 419
 420	xennet_maybe_wake_tx(dev);
 
 
 
 
 421}
 422
 423static void xennet_make_frags(struct sk_buff *skb, struct net_device *dev,
 424			      struct xen_netif_tx_request *tx)
 
 
 
 
 
 
 
 
 
 425{
 426	struct netfront_info *np = netdev_priv(dev);
 427	char *data = skb->data;
 428	unsigned long mfn;
 429	RING_IDX prod = np->tx.req_prod_pvt;
 430	int frags = skb_shinfo(skb)->nr_frags;
 431	unsigned int offset = offset_in_page(data);
 432	unsigned int len = skb_headlen(skb);
 433	unsigned int id;
 
 434	grant_ref_t ref;
 435	int i;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 436
 437	/* While the header overlaps a page boundary (including being
 438	   larger than a page), split it it into page-sized chunks. */
 439	while (len > PAGE_SIZE - offset) {
 440		tx->size = PAGE_SIZE - offset;
 441		tx->flags |= XEN_NETTXF_more_data;
 442		len -= tx->size;
 443		data += tx->size;
 444		offset = 0;
 445
 446		id = get_id_from_freelist(&np->tx_skb_freelist, np->tx_skbs);
 447		np->tx_skbs[id].skb = skb_get(skb);
 448		tx = RING_GET_REQUEST(&np->tx, prod++);
 449		tx->id = id;
 450		ref = gnttab_claim_grant_reference(&np->gref_tx_head);
 451		BUG_ON((signed short)ref < 0);
 452
 453		mfn = virt_to_mfn(data);
 454		gnttab_grant_foreign_access_ref(ref, np->xbdev->otherend_id,
 455						mfn, GNTMAP_readonly);
 456
 457		np->grant_tx_page[id] = virt_to_page(data);
 458		tx->gref = np->grant_tx_ref[id] = ref;
 459		tx->offset = offset;
 460		tx->size = len;
 461		tx->flags = 0;
 462	}
 463
 464	/* Grant backend access to each skb fragment page. */
 465	for (i = 0; i < frags; i++) {
 466		skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 467		struct page *page = skb_frag_page(frag);
 468
 469		len = skb_frag_size(frag);
 470		offset = frag->page_offset;
 
 
 
 471
 472		/* Data must not cross a page boundary. */
 473		BUG_ON(len + offset > PAGE_SIZE<<compound_order(page));
 474
 475		/* Skip unused frames from start of page */
 476		page += offset >> PAGE_SHIFT;
 477		offset &= ~PAGE_MASK;
 478
 479		while (len > 0) {
 480			unsigned long bytes;
 
 
 481
 482			BUG_ON(offset >= PAGE_SIZE);
 
 
 
 483
 484			bytes = PAGE_SIZE - offset;
 485			if (bytes > len)
 486				bytes = len;
 487
 488			tx->flags |= XEN_NETTXF_more_data;
 489
 490			id = get_id_from_freelist(&np->tx_skb_freelist,
 491						  np->tx_skbs);
 492			np->tx_skbs[id].skb = skb_get(skb);
 493			tx = RING_GET_REQUEST(&np->tx, prod++);
 494			tx->id = id;
 495			ref = gnttab_claim_grant_reference(&np->gref_tx_head);
 496			BUG_ON((signed short)ref < 0);
 497
 498			mfn = pfn_to_mfn(page_to_pfn(page));
 499			gnttab_grant_foreign_access_ref(ref,
 500							np->xbdev->otherend_id,
 501							mfn, GNTMAP_readonly);
 502
 503			np->grant_tx_page[id] = page;
 504			tx->gref = np->grant_tx_ref[id] = ref;
 505			tx->offset = offset;
 506			tx->size = bytes;
 507			tx->flags = 0;
 508
 509			offset += bytes;
 510			len -= bytes;
 511
 512			/* Next frame */
 513			if (offset == PAGE_SIZE && len) {
 514				BUG_ON(!PageCompound(page));
 515				page++;
 516				offset = 0;
 517			}
 518		}
 519	}
 520
 521	np->tx.req_prod_pvt = prod;
 
 
 
 522}
 523
 524/*
 525 * Count how many ring slots are required to send the frags of this
 526 * skb. Each frag might be a compound page.
 527 */
 528static int xennet_count_skb_frag_slots(struct sk_buff *skb)
 529{
 530	int i, frags = skb_shinfo(skb)->nr_frags;
 531	int pages = 0;
 
 
 
 532
 533	for (i = 0; i < frags; i++) {
 534		skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 535		unsigned long size = skb_frag_size(frag);
 536		unsigned long offset = frag->page_offset;
 537
 538		/* Skip unused frames from start of page */
 539		offset &= ~PAGE_MASK;
 540
 541		pages += PFN_UP(offset + size);
 542	}
 543
 544	return pages;
 545}
 546
 547static int xennet_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 548{
 549	unsigned short id;
 550	struct netfront_info *np = netdev_priv(dev);
 551	struct netfront_stats *stats = this_cpu_ptr(np->stats);
 552	struct xen_netif_tx_request *tx;
 553	char *data = skb->data;
 554	RING_IDX i;
 555	grant_ref_t ref;
 556	unsigned long mfn;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 557	int notify;
 558	int slots;
 559	unsigned int offset = offset_in_page(data);
 560	unsigned int len = skb_headlen(skb);
 
 561	unsigned long flags;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 562
 563	/* If skb->len is too big for wire format, drop skb and alert
 564	 * user about misconfiguration.
 565	 */
 566	if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) {
 567		net_alert_ratelimited(
 568			"xennet: skb->len = %u, too big for wire format\n",
 569			skb->len);
 570		goto drop;
 571	}
 572
 573	slots = DIV_ROUND_UP(offset + len, PAGE_SIZE) +
 574		xennet_count_skb_frag_slots(skb);
 575	if (unlikely(slots > MAX_SKB_FRAGS + 1)) {
 576		net_alert_ratelimited(
 577			"xennet: skb rides the rocket: %d slots\n", slots);
 578		goto drop;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 579	}
 580
 581	spin_lock_irqsave(&np->tx_lock, flags);
 
 
 582
 583	if (unlikely(!netif_carrier_ok(dev) ||
 584		     (slots > 1 && !xennet_can_sg(dev)) ||
 585		     netif_needs_gso(skb, netif_skb_features(skb)))) {
 586		spin_unlock_irqrestore(&np->tx_lock, flags);
 587		goto drop;
 588	}
 589
 590	i = np->tx.req_prod_pvt;
 591
 592	id = get_id_from_freelist(&np->tx_skb_freelist, np->tx_skbs);
 593	np->tx_skbs[id].skb = skb;
 594
 595	tx = RING_GET_REQUEST(&np->tx, i);
 596
 597	tx->id   = id;
 598	ref = gnttab_claim_grant_reference(&np->gref_tx_head);
 599	BUG_ON((signed short)ref < 0);
 600	mfn = virt_to_mfn(data);
 601	gnttab_grant_foreign_access_ref(
 602		ref, np->xbdev->otherend_id, mfn, GNTMAP_readonly);
 603	np->grant_tx_page[id] = virt_to_page(data);
 604	tx->gref = np->grant_tx_ref[id] = ref;
 605	tx->offset = offset;
 606	tx->size = len;
 607
 608	tx->flags = 0;
 609	if (skb->ip_summed == CHECKSUM_PARTIAL)
 610		/* local packet? */
 611		tx->flags |= XEN_NETTXF_csum_blank | XEN_NETTXF_data_validated;
 
 612	else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
 613		/* remote but checksummed. */
 614		tx->flags |= XEN_NETTXF_data_validated;
 615
 
 616	if (skb_shinfo(skb)->gso_size) {
 617		struct xen_netif_extra_info *gso;
 618
 619		gso = (struct xen_netif_extra_info *)
 620			RING_GET_REQUEST(&np->tx, ++i);
 621
 622		tx->flags |= XEN_NETTXF_extra_info;
 623
 624		gso->u.gso.size = skb_shinfo(skb)->gso_size;
 625		gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ?
 626			XEN_NETIF_GSO_TYPE_TCPV6 :
 627			XEN_NETIF_GSO_TYPE_TCPV4;
 628		gso->u.gso.pad = 0;
 629		gso->u.gso.features = 0;
 630
 631		gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
 632		gso->flags = 0;
 633	}
 634
 635	np->tx.req_prod_pvt = i + 1;
 
 
 
 
 
 
 
 
 
 
 
 
 636
 637	xennet_make_frags(skb, dev, tx);
 638	tx->size = skb->len;
 639
 640	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&np->tx, notify);
 
 
 641	if (notify)
 642		notify_remote_via_irq(np->tx_irq);
 643
 644	u64_stats_update_begin(&stats->syncp);
 645	stats->tx_bytes += skb->len;
 646	stats->tx_packets++;
 647	u64_stats_update_end(&stats->syncp);
 648
 649	/* Note: It is not safe to access skb after xennet_tx_buf_gc()! */
 650	xennet_tx_buf_gc(dev);
 651
 652	if (!netfront_tx_slot_available(np))
 653		netif_stop_queue(dev);
 654
 655	spin_unlock_irqrestore(&np->tx_lock, flags);
 656
 657	return NETDEV_TX_OK;
 658
 659 drop:
 660	dev->stats.tx_dropped++;
 661	dev_kfree_skb_any(skb);
 662	return NETDEV_TX_OK;
 663}
 664
 665static int xennet_close(struct net_device *dev)
 666{
 667	struct netfront_info *np = netdev_priv(dev);
 668	netif_stop_queue(np->netdev);
 669	napi_disable(&np->napi);
 
 
 
 
 
 
 670	return 0;
 671}
 672
 673static void xennet_move_rx_slot(struct netfront_info *np, struct sk_buff *skb,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 674				grant_ref_t ref)
 675{
 676	int new = xennet_rxidx(np->rx.req_prod_pvt);
 677
 678	BUG_ON(np->rx_skbs[new]);
 679	np->rx_skbs[new] = skb;
 680	np->grant_rx_ref[new] = ref;
 681	RING_GET_REQUEST(&np->rx, np->rx.req_prod_pvt)->id = new;
 682	RING_GET_REQUEST(&np->rx, np->rx.req_prod_pvt)->gref = ref;
 683	np->rx.req_prod_pvt++;
 684}
 685
 686static int xennet_get_extras(struct netfront_info *np,
 687			     struct xen_netif_extra_info *extras,
 688			     RING_IDX rp)
 689
 690{
 691	struct xen_netif_extra_info *extra;
 692	struct device *dev = &np->netdev->dev;
 693	RING_IDX cons = np->rx.rsp_cons;
 694	int err = 0;
 695
 696	do {
 697		struct sk_buff *skb;
 698		grant_ref_t ref;
 699
 700		if (unlikely(cons + 1 == rp)) {
 701			if (net_ratelimit())
 702				dev_warn(dev, "Missing extra info\n");
 703			err = -EBADR;
 704			break;
 705		}
 706
 707		extra = (struct xen_netif_extra_info *)
 708			RING_GET_RESPONSE(&np->rx, ++cons);
 709
 710		if (unlikely(!extra->type ||
 711			     extra->type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
 712			if (net_ratelimit())
 713				dev_warn(dev, "Invalid extra type: %d\n",
 714					extra->type);
 715			err = -EINVAL;
 716		} else {
 717			memcpy(&extras[extra->type - 1], extra,
 718			       sizeof(*extra));
 719		}
 720
 721		skb = xennet_get_rx_skb(np, cons);
 722		ref = xennet_get_rx_ref(np, cons);
 723		xennet_move_rx_slot(np, skb, ref);
 724	} while (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE);
 725
 726	np->rx.rsp_cons = cons;
 727	return err;
 728}
 729
 730static int xennet_get_responses(struct netfront_info *np,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 731				struct netfront_rx_info *rinfo, RING_IDX rp,
 732				struct sk_buff_head *list)
 
 733{
 734	struct xen_netif_rx_response *rx = &rinfo->rx;
 
 
 
 735	struct xen_netif_extra_info *extras = rinfo->extras;
 736	struct device *dev = &np->netdev->dev;
 737	RING_IDX cons = np->rx.rsp_cons;
 738	struct sk_buff *skb = xennet_get_rx_skb(np, cons);
 739	grant_ref_t ref = xennet_get_rx_ref(np, cons);
 740	int max = MAX_SKB_FRAGS + (rx->status <= RX_COPY_THRESHOLD);
 741	int slots = 1;
 742	int err = 0;
 743	unsigned long ret;
 744
 745	if (rx->flags & XEN_NETRXF_extra_info) {
 746		err = xennet_get_extras(np, extras, rp);
 747		cons = np->rx.rsp_cons;
 748	}
 
 749
 750	for (;;) {
 751		if (unlikely(rx->status < 0 ||
 752			     rx->offset + rx->status > PAGE_SIZE)) {
 753			if (net_ratelimit())
 754				dev_warn(dev, "rx->offset: %x, size: %u\n",
 755					 rx->offset, rx->status);
 756			xennet_move_rx_slot(np, skb, ref);
 757			err = -EINVAL;
 758			goto next;
 759		}
 
 
 760
 
 761		/*
 762		 * This definitely indicates a bug, either in this driver or in
 763		 * the backend driver. In future this should flag the bad
 764		 * situation to the system controller to reboot the backend.
 765		 */
 766		if (ref == GRANT_INVALID_REF) {
 767			if (net_ratelimit())
 768				dev_warn(dev, "Bad rx response id %d.\n",
 769					 rx->id);
 770			err = -EINVAL;
 771			goto next;
 772		}
 773
 774		ret = gnttab_end_foreign_access_ref(ref, 0);
 775		BUG_ON(!ret);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 776
 777		gnttab_release_grant_reference(&np->gref_rx_head, ref);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 778
 779		__skb_queue_tail(list, skb);
 780
 781next:
 782		if (!(rx->flags & XEN_NETRXF_more_data))
 783			break;
 784
 785		if (cons + slots == rp) {
 786			if (net_ratelimit())
 787				dev_warn(dev, "Need more slots\n");
 788			err = -ENOENT;
 789			break;
 790		}
 791
 792		rx = RING_GET_RESPONSE(&np->rx, cons + slots);
 793		skb = xennet_get_rx_skb(np, cons + slots);
 794		ref = xennet_get_rx_ref(np, cons + slots);
 
 795		slots++;
 796	}
 797
 798	if (unlikely(slots > max)) {
 799		if (net_ratelimit())
 800			dev_warn(dev, "Too many slots\n");
 801		err = -E2BIG;
 802	}
 803
 804	if (unlikely(err))
 805		np->rx.rsp_cons = cons + slots;
 806
 807	return err;
 808}
 809
 810static int xennet_set_skb_gso(struct sk_buff *skb,
 811			      struct xen_netif_extra_info *gso)
 812{
 813	if (!gso->u.gso.size) {
 814		if (net_ratelimit())
 815			pr_warn("GSO size must not be zero\n");
 816		return -EINVAL;
 817	}
 818
 819	if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 &&
 820	    gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) {
 821		if (net_ratelimit())
 822			pr_warn("Bad GSO type %d\n", gso->u.gso.type);
 823		return -EINVAL;
 824	}
 825
 826	skb_shinfo(skb)->gso_size = gso->u.gso.size;
 827	skb_shinfo(skb)->gso_type =
 828		(gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ?
 829		SKB_GSO_TCPV4 :
 830		SKB_GSO_TCPV6;
 831
 832	/* Header must be checked, and gso_segs computed. */
 833	skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
 834	skb_shinfo(skb)->gso_segs = 0;
 835
 836	return 0;
 837}
 838
 839static RING_IDX xennet_fill_frags(struct netfront_info *np,
 840				  struct sk_buff *skb,
 841				  struct sk_buff_head *list)
 842{
 843	struct skb_shared_info *shinfo = skb_shinfo(skb);
 844	RING_IDX cons = np->rx.rsp_cons;
 845	struct sk_buff *nskb;
 846
 847	while ((nskb = __skb_dequeue(list))) {
 848		struct xen_netif_rx_response *rx =
 849			RING_GET_RESPONSE(&np->rx, ++cons);
 850		skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0];
 851
 852		if (shinfo->nr_frags == MAX_SKB_FRAGS) {
 
 
 853			unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to;
 854
 855			BUG_ON(pull_to <= skb_headlen(skb));
 856			__pskb_pull_tail(skb, pull_to - skb_headlen(skb));
 857		}
 858		BUG_ON(shinfo->nr_frags >= MAX_SKB_FRAGS);
 
 
 
 
 
 859
 860		skb_add_rx_frag(skb, shinfo->nr_frags, skb_frag_page(nfrag),
 861				rx->offset, rx->status, PAGE_SIZE);
 
 862
 863		skb_shinfo(nskb)->nr_frags = 0;
 864		kfree_skb(nskb);
 865	}
 866
 867	return cons;
 
 
 868}
 869
 870static int checksum_setup(struct net_device *dev, struct sk_buff *skb)
 871{
 872	bool recalculate_partial_csum = false;
 873
 874	/*
 875	 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
 876	 * peers can fail to set NETRXF_csum_blank when sending a GSO
 877	 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
 878	 * recalculate the partial checksum.
 879	 */
 880	if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) {
 881		struct netfront_info *np = netdev_priv(dev);
 882		np->rx_gso_checksum_fixup++;
 883		skb->ip_summed = CHECKSUM_PARTIAL;
 884		recalculate_partial_csum = true;
 885	}
 886
 887	/* A non-CHECKSUM_PARTIAL SKB does not require setup. */
 888	if (skb->ip_summed != CHECKSUM_PARTIAL)
 889		return 0;
 890
 891	return skb_checksum_setup(skb, recalculate_partial_csum);
 892}
 893
 894static int handle_incoming_queue(struct net_device *dev,
 895				 struct sk_buff_head *rxq)
 896{
 897	struct netfront_info *np = netdev_priv(dev);
 898	struct netfront_stats *stats = this_cpu_ptr(np->stats);
 899	int packets_dropped = 0;
 900	struct sk_buff *skb;
 901
 902	while ((skb = __skb_dequeue(rxq)) != NULL) {
 903		int pull_to = NETFRONT_SKB_CB(skb)->pull_to;
 904
 905		if (pull_to > skb_headlen(skb))
 906			__pskb_pull_tail(skb, pull_to - skb_headlen(skb));
 907
 908		/* Ethernet work: Delayed to here as it peeks the header. */
 909		skb->protocol = eth_type_trans(skb, dev);
 910		skb_reset_network_header(skb);
 911
 912		if (checksum_setup(dev, skb)) {
 913			kfree_skb(skb);
 914			packets_dropped++;
 915			dev->stats.rx_errors++;
 916			continue;
 917		}
 918
 919		u64_stats_update_begin(&stats->syncp);
 920		stats->rx_packets++;
 921		stats->rx_bytes += skb->len;
 922		u64_stats_update_end(&stats->syncp);
 923
 924		/* Pass it up. */
 925		napi_gro_receive(&np->napi, skb);
 926	}
 927
 928	return packets_dropped;
 929}
 930
 931static int xennet_poll(struct napi_struct *napi, int budget)
 932{
 933	struct netfront_info *np = container_of(napi, struct netfront_info, napi);
 934	struct net_device *dev = np->netdev;
 935	struct sk_buff *skb;
 936	struct netfront_rx_info rinfo;
 937	struct xen_netif_rx_response *rx = &rinfo.rx;
 938	struct xen_netif_extra_info *extras = rinfo.extras;
 939	RING_IDX i, rp;
 940	int work_done;
 941	struct sk_buff_head rxq;
 942	struct sk_buff_head errq;
 943	struct sk_buff_head tmpq;
 944	unsigned long flags;
 945	int err;
 
 946
 947	spin_lock(&np->rx_lock);
 948
 949	skb_queue_head_init(&rxq);
 950	skb_queue_head_init(&errq);
 951	skb_queue_head_init(&tmpq);
 952
 953	rp = np->rx.sring->rsp_prod;
 
 
 
 
 
 
 
 954	rmb(); /* Ensure we see queued responses up to 'rp'. */
 955
 956	i = np->rx.rsp_cons;
 957	work_done = 0;
 958	while ((i != rp) && (work_done < budget)) {
 959		memcpy(rx, RING_GET_RESPONSE(&np->rx, i), sizeof(*rx));
 960		memset(extras, 0, sizeof(rinfo.extras));
 961
 962		err = xennet_get_responses(np, &rinfo, rp, &tmpq);
 
 963
 964		if (unlikely(err)) {
 
 
 
 
 965err:
 966			while ((skb = __skb_dequeue(&tmpq)))
 967				__skb_queue_tail(&errq, skb);
 968			dev->stats.rx_errors++;
 969			i = np->rx.rsp_cons;
 970			continue;
 971		}
 972
 973		skb = __skb_dequeue(&tmpq);
 974
 975		if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
 976			struct xen_netif_extra_info *gso;
 977			gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
 978
 979			if (unlikely(xennet_set_skb_gso(skb, gso))) {
 980				__skb_queue_head(&tmpq, skb);
 981				np->rx.rsp_cons += skb_queue_len(&tmpq);
 
 
 982				goto err;
 983			}
 984		}
 985
 986		NETFRONT_SKB_CB(skb)->pull_to = rx->status;
 987		if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD)
 988			NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD;
 989
 990		skb_shinfo(skb)->frags[0].page_offset = rx->offset;
 991		skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status);
 992		skb->data_len = rx->status;
 993		skb->len += rx->status;
 994
 995		i = xennet_fill_frags(np, skb, &tmpq);
 
 996
 997		if (rx->flags & XEN_NETRXF_csum_blank)
 998			skb->ip_summed = CHECKSUM_PARTIAL;
 999		else if (rx->flags & XEN_NETRXF_data_validated)
1000			skb->ip_summed = CHECKSUM_UNNECESSARY;
1001
1002		__skb_queue_tail(&rxq, skb);
1003
1004		np->rx.rsp_cons = ++i;
 
1005		work_done++;
1006	}
 
 
1007
1008	__skb_queue_purge(&errq);
1009
1010	work_done -= handle_incoming_queue(dev, &rxq);
1011
1012	/* If we get a callback with very few responses, reduce fill target. */
1013	/* NB. Note exponential increase, linear decrease. */
1014	if (((np->rx.req_prod_pvt - np->rx.sring->rsp_prod) >
1015	     ((3*np->rx_target) / 4)) &&
1016	    (--np->rx_target < np->rx_min_target))
1017		np->rx_target = np->rx_min_target;
1018
1019	xennet_alloc_rx_buffers(dev);
1020
1021	if (work_done < budget) {
1022		int more_to_do = 0;
1023
1024		napi_gro_flush(napi, false);
1025
1026		local_irq_save(flags);
1027
1028		RING_FINAL_CHECK_FOR_RESPONSES(&np->rx, more_to_do);
1029		if (!more_to_do)
1030			__napi_complete(napi);
1031
1032		local_irq_restore(flags);
1033	}
1034
1035	spin_unlock(&np->rx_lock);
1036
1037	return work_done;
1038}
1039
1040static int xennet_change_mtu(struct net_device *dev, int mtu)
1041{
1042	int max = xennet_can_sg(dev) ?
1043		XEN_NETIF_MAX_TX_SIZE - MAX_TCP_HEADER : ETH_DATA_LEN;
1044
1045	if (mtu > max)
1046		return -EINVAL;
1047	dev->mtu = mtu;
1048	return 0;
1049}
1050
1051static struct rtnl_link_stats64 *xennet_get_stats64(struct net_device *dev,
1052						    struct rtnl_link_stats64 *tot)
1053{
1054	struct netfront_info *np = netdev_priv(dev);
1055	int cpu;
1056
1057	for_each_possible_cpu(cpu) {
1058		struct netfront_stats *stats = per_cpu_ptr(np->stats, cpu);
 
1059		u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
1060		unsigned int start;
1061
1062		do {
1063			start = u64_stats_fetch_begin_irq(&stats->syncp);
 
 
 
1064
1065			rx_packets = stats->rx_packets;
1066			tx_packets = stats->tx_packets;
1067			rx_bytes = stats->rx_bytes;
1068			tx_bytes = stats->tx_bytes;
1069		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1070
1071		tot->rx_packets += rx_packets;
1072		tot->tx_packets += tx_packets;
1073		tot->rx_bytes   += rx_bytes;
1074		tot->tx_bytes   += tx_bytes;
1075	}
1076
1077	tot->rx_errors  = dev->stats.rx_errors;
1078	tot->tx_dropped = dev->stats.tx_dropped;
1079
1080	return tot;
1081}
1082
1083static void xennet_release_tx_bufs(struct netfront_info *np)
1084{
1085	struct sk_buff *skb;
1086	int i;
1087
1088	for (i = 0; i < NET_TX_RING_SIZE; i++) {
1089		/* Skip over entries which are actually freelist references */
1090		if (skb_entry_is_link(&np->tx_skbs[i]))
1091			continue;
1092
1093		skb = np->tx_skbs[i].skb;
1094		get_page(np->grant_tx_page[i]);
1095		gnttab_end_foreign_access(np->grant_tx_ref[i],
1096					  GNTMAP_readonly,
1097					  (unsigned long)page_address(np->grant_tx_page[i]));
1098		np->grant_tx_page[i] = NULL;
1099		np->grant_tx_ref[i] = GRANT_INVALID_REF;
1100		add_id_to_freelist(&np->tx_skb_freelist, np->tx_skbs, i);
1101		dev_kfree_skb_irq(skb);
1102	}
1103}
1104
1105static void xennet_release_rx_bufs(struct netfront_info *np)
1106{
1107	int id, ref;
1108
1109	spin_lock_bh(&np->rx_lock);
1110
1111	for (id = 0; id < NET_RX_RING_SIZE; id++) {
1112		struct sk_buff *skb;
1113		struct page *page;
1114
1115		skb = np->rx_skbs[id];
1116		if (!skb)
1117			continue;
1118
1119		ref = np->grant_rx_ref[id];
1120		if (ref == GRANT_INVALID_REF)
1121			continue;
1122
1123		page = skb_frag_page(&skb_shinfo(skb)->frags[0]);
1124
1125		/* gnttab_end_foreign_access() needs a page ref until
1126		 * foreign access is ended (which may be deferred).
1127		 */
1128		get_page(page);
1129		gnttab_end_foreign_access(ref, 0,
1130					  (unsigned long)page_address(page));
1131		np->grant_rx_ref[id] = GRANT_INVALID_REF;
1132
1133		kfree_skb(skb);
1134	}
1135
1136	spin_unlock_bh(&np->rx_lock);
1137}
1138
1139static void xennet_uninit(struct net_device *dev)
1140{
1141	struct netfront_info *np = netdev_priv(dev);
1142	xennet_release_tx_bufs(np);
1143	xennet_release_rx_bufs(np);
1144	gnttab_free_grant_references(np->gref_tx_head);
1145	gnttab_free_grant_references(np->gref_rx_head);
1146}
1147
1148static netdev_features_t xennet_fix_features(struct net_device *dev,
1149	netdev_features_t features)
1150{
1151	struct netfront_info *np = netdev_priv(dev);
1152	int val;
1153
1154	if (features & NETIF_F_SG) {
1155		if (xenbus_scanf(XBT_NIL, np->xbdev->otherend, "feature-sg",
1156				 "%d", &val) < 0)
1157			val = 0;
1158
1159		if (!val)
1160			features &= ~NETIF_F_SG;
1161	}
1162
1163	if (features & NETIF_F_IPV6_CSUM) {
1164		if (xenbus_scanf(XBT_NIL, np->xbdev->otherend,
1165				 "feature-ipv6-csum-offload", "%d", &val) < 0)
1166			val = 0;
1167
1168		if (!val)
1169			features &= ~NETIF_F_IPV6_CSUM;
1170	}
1171
1172	if (features & NETIF_F_TSO) {
1173		if (xenbus_scanf(XBT_NIL, np->xbdev->otherend,
1174				 "feature-gso-tcpv4", "%d", &val) < 0)
1175			val = 0;
1176
1177		if (!val)
1178			features &= ~NETIF_F_TSO;
1179	}
1180
1181	if (features & NETIF_F_TSO6) {
1182		if (xenbus_scanf(XBT_NIL, np->xbdev->otherend,
1183				 "feature-gso-tcpv6", "%d", &val) < 0)
1184			val = 0;
1185
1186		if (!val)
1187			features &= ~NETIF_F_TSO6;
1188	}
1189
1190	return features;
1191}
1192
1193static int xennet_set_features(struct net_device *dev,
1194	netdev_features_t features)
1195{
1196	if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) {
1197		netdev_info(dev, "Reducing MTU because no SG offload");
1198		dev->mtu = ETH_DATA_LEN;
1199	}
1200
1201	return 0;
1202}
1203
1204static irqreturn_t xennet_tx_interrupt(int irq, void *dev_id)
1205{
1206	struct netfront_info *np = dev_id;
1207	struct net_device *dev = np->netdev;
1208	unsigned long flags;
1209
1210	spin_lock_irqsave(&np->tx_lock, flags);
1211	xennet_tx_buf_gc(dev);
1212	spin_unlock_irqrestore(&np->tx_lock, flags);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1213
1214	return IRQ_HANDLED;
1215}
1216
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1217static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
1218{
1219	struct netfront_info *np = dev_id;
1220	struct net_device *dev = np->netdev;
1221
1222	if (likely(netif_carrier_ok(dev) &&
1223		   RING_HAS_UNCONSUMED_RESPONSES(&np->rx)))
1224			napi_schedule(&np->napi);
1225
1226	return IRQ_HANDLED;
1227}
1228
1229static irqreturn_t xennet_interrupt(int irq, void *dev_id)
1230{
1231	xennet_tx_interrupt(irq, dev_id);
1232	xennet_rx_interrupt(irq, dev_id);
 
 
 
 
1233	return IRQ_HANDLED;
1234}
1235
1236#ifdef CONFIG_NET_POLL_CONTROLLER
1237static void xennet_poll_controller(struct net_device *dev)
1238{
1239	xennet_interrupt(0, dev);
 
 
 
 
 
 
 
 
 
1240}
1241#endif
1242
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1243static const struct net_device_ops xennet_netdev_ops = {
1244	.ndo_open            = xennet_open,
1245	.ndo_uninit          = xennet_uninit,
 
1246	.ndo_stop            = xennet_close,
1247	.ndo_start_xmit      = xennet_start_xmit,
1248	.ndo_change_mtu	     = xennet_change_mtu,
1249	.ndo_get_stats64     = xennet_get_stats64,
1250	.ndo_set_mac_address = eth_mac_addr,
1251	.ndo_validate_addr   = eth_validate_addr,
1252	.ndo_fix_features    = xennet_fix_features,
1253	.ndo_set_features    = xennet_set_features,
 
 
 
1254#ifdef CONFIG_NET_POLL_CONTROLLER
1255	.ndo_poll_controller = xennet_poll_controller,
1256#endif
1257};
1258
 
 
 
 
 
 
 
 
 
1259static struct net_device *xennet_create_dev(struct xenbus_device *dev)
1260{
1261	int i, err;
1262	struct net_device *netdev;
1263	struct netfront_info *np;
1264
1265	netdev = alloc_etherdev(sizeof(struct netfront_info));
1266	if (!netdev)
1267		return ERR_PTR(-ENOMEM);
1268
1269	np                   = netdev_priv(netdev);
1270	np->xbdev            = dev;
1271
1272	spin_lock_init(&np->tx_lock);
1273	spin_lock_init(&np->rx_lock);
1274
1275	skb_queue_head_init(&np->rx_batch);
1276	np->rx_target     = RX_DFL_MIN_TARGET;
1277	np->rx_min_target = RX_DFL_MIN_TARGET;
1278	np->rx_max_target = RX_MAX_TARGET;
1279
1280	init_timer(&np->rx_refill_timer);
1281	np->rx_refill_timer.data = (unsigned long)netdev;
1282	np->rx_refill_timer.function = rx_refill_timeout;
1283
1284	err = -ENOMEM;
1285	np->stats = netdev_alloc_pcpu_stats(struct netfront_stats);
1286	if (np->stats == NULL)
 
 
 
1287		goto exit;
1288
1289	/* Initialise tx_skbs as a free chain containing every entry. */
1290	np->tx_skb_freelist = 0;
1291	for (i = 0; i < NET_TX_RING_SIZE; i++) {
1292		skb_entry_set_link(&np->tx_skbs[i], i+1);
1293		np->grant_tx_ref[i] = GRANT_INVALID_REF;
1294		np->grant_tx_page[i] = NULL;
1295	}
1296
1297	/* Clear out rx_skbs */
1298	for (i = 0; i < NET_RX_RING_SIZE; i++) {
1299		np->rx_skbs[i] = NULL;
1300		np->grant_rx_ref[i] = GRANT_INVALID_REF;
1301	}
1302
1303	/* A grant for every tx ring slot */
1304	if (gnttab_alloc_grant_references(TX_MAX_TARGET,
1305					  &np->gref_tx_head) < 0) {
1306		pr_alert("can't alloc tx grant refs\n");
1307		err = -ENOMEM;
1308		goto exit_free_stats;
1309	}
1310	/* A grant for every rx ring slot */
1311	if (gnttab_alloc_grant_references(RX_MAX_TARGET,
1312					  &np->gref_rx_head) < 0) {
1313		pr_alert("can't alloc rx grant refs\n");
1314		err = -ENOMEM;
1315		goto exit_free_tx;
1316	}
1317
1318	netdev->netdev_ops	= &xennet_netdev_ops;
1319
1320	netif_napi_add(netdev, &np->napi, xennet_poll, 64);
1321	netdev->features        = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
1322				  NETIF_F_GSO_ROBUST;
1323	netdev->hw_features	= NETIF_F_SG |
1324				  NETIF_F_IPV6_CSUM |
1325				  NETIF_F_TSO | NETIF_F_TSO6;
1326
1327	/*
1328         * Assume that all hw features are available for now. This set
1329         * will be adjusted by the call to netdev_update_features() in
1330         * xennet_connect() which is the earliest point where we can
1331         * negotiate with the backend regarding supported features.
1332         */
1333	netdev->features |= netdev->hw_features;
 
 
1334
1335	SET_ETHTOOL_OPS(netdev, &xennet_ethtool_ops);
 
 
1336	SET_NETDEV_DEV(netdev, &dev->dev);
1337
1338	netif_set_gso_max_size(netdev, XEN_NETIF_MAX_TX_SIZE - MAX_TCP_HEADER);
1339
1340	np->netdev = netdev;
 
1341
1342	netif_carrier_off(netdev);
1343
 
 
 
 
 
 
 
 
 
1344	return netdev;
1345
1346 exit_free_tx:
1347	gnttab_free_grant_references(np->gref_tx_head);
1348 exit_free_stats:
1349	free_percpu(np->stats);
1350 exit:
1351	free_netdev(netdev);
1352	return ERR_PTR(err);
1353}
1354
1355/**
1356 * Entry point to this code when a new device is created.  Allocate the basic
1357 * structures and the ring buffers for communication with the backend, and
1358 * inform the backend of the appropriate details for those.
1359 */
1360static int netfront_probe(struct xenbus_device *dev,
1361			  const struct xenbus_device_id *id)
1362{
1363	int err;
1364	struct net_device *netdev;
1365	struct netfront_info *info;
1366
1367	netdev = xennet_create_dev(dev);
1368	if (IS_ERR(netdev)) {
1369		err = PTR_ERR(netdev);
1370		xenbus_dev_fatal(dev, err, "creating netdev");
1371		return err;
1372	}
1373
1374	info = netdev_priv(netdev);
1375	dev_set_drvdata(&dev->dev, info);
1376
1377	err = register_netdev(info->netdev);
1378	if (err) {
1379		pr_warn("%s: register_netdev err=%d\n", __func__, err);
1380		goto fail;
1381	}
1382
1383	err = xennet_sysfs_addif(info->netdev);
1384	if (err) {
1385		unregister_netdev(info->netdev);
1386		pr_warn("%s: add sysfs failed err=%d\n", __func__, err);
1387		goto fail;
1388	}
1389
1390	return 0;
1391
1392 fail:
1393	free_netdev(netdev);
1394	dev_set_drvdata(&dev->dev, NULL);
1395	return err;
1396}
1397
1398static void xennet_end_access(int ref, void *page)
1399{
1400	/* This frees the page as a side-effect */
1401	if (ref != GRANT_INVALID_REF)
1402		gnttab_end_foreign_access(ref, 0, (unsigned long)page);
1403}
1404
1405static void xennet_disconnect_backend(struct netfront_info *info)
1406{
1407	/* Stop old i/f to prevent errors whilst we rebuild the state. */
1408	spin_lock_bh(&info->rx_lock);
1409	spin_lock_irq(&info->tx_lock);
1410	netif_carrier_off(info->netdev);
1411	spin_unlock_irq(&info->tx_lock);
1412	spin_unlock_bh(&info->rx_lock);
1413
1414	if (info->tx_irq && (info->tx_irq == info->rx_irq))
1415		unbind_from_irqhandler(info->tx_irq, info);
1416	if (info->tx_irq && (info->tx_irq != info->rx_irq)) {
1417		unbind_from_irqhandler(info->tx_irq, info);
1418		unbind_from_irqhandler(info->rx_irq, info);
1419	}
1420	info->tx_evtchn = info->rx_evtchn = 0;
1421	info->tx_irq = info->rx_irq = 0;
 
 
 
 
 
 
 
 
1422
1423	/* End access and free the pages */
1424	xennet_end_access(info->tx_ring_ref, info->tx.sring);
1425	xennet_end_access(info->rx_ring_ref, info->rx.sring);
 
 
 
 
 
 
 
 
 
 
1426
1427	info->tx_ring_ref = GRANT_INVALID_REF;
1428	info->rx_ring_ref = GRANT_INVALID_REF;
1429	info->tx.sring = NULL;
1430	info->rx.sring = NULL;
1431}
1432
1433/**
1434 * We are reconnecting to the backend, due to a suspend/resume, or a backend
1435 * driver restart.  We tear down our netif structure and recreate it, but
1436 * leave the device-layer structures intact so that this is transparent to the
1437 * rest of the kernel.
1438 */
1439static int netfront_resume(struct xenbus_device *dev)
1440{
1441	struct netfront_info *info = dev_get_drvdata(&dev->dev);
1442
1443	dev_dbg(&dev->dev, "%s\n", dev->nodename);
1444
 
 
 
 
1445	xennet_disconnect_backend(info);
 
 
 
 
 
 
1446	return 0;
1447}
1448
1449static int xen_net_read_mac(struct xenbus_device *dev, u8 mac[])
1450{
1451	char *s, *e, *macstr;
1452	int i;
1453
1454	macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL);
1455	if (IS_ERR(macstr))
1456		return PTR_ERR(macstr);
1457
1458	for (i = 0; i < ETH_ALEN; i++) {
1459		mac[i] = simple_strtoul(s, &e, 16);
1460		if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) {
1461			kfree(macstr);
1462			return -ENOENT;
1463		}
1464		s = e+1;
1465	}
1466
1467	kfree(macstr);
1468	return 0;
1469}
1470
1471static int setup_netfront_single(struct netfront_info *info)
1472{
1473	int err;
1474
1475	err = xenbus_alloc_evtchn(info->xbdev, &info->tx_evtchn);
1476	if (err < 0)
1477		goto fail;
1478
1479	err = bind_evtchn_to_irqhandler(info->tx_evtchn,
1480					xennet_interrupt,
1481					0, info->netdev->name, info);
 
1482	if (err < 0)
1483		goto bind_fail;
1484	info->rx_evtchn = info->tx_evtchn;
1485	info->rx_irq = info->tx_irq = err;
1486
1487	return 0;
1488
1489bind_fail:
1490	xenbus_free_evtchn(info->xbdev, info->tx_evtchn);
1491	info->tx_evtchn = 0;
1492fail:
1493	return err;
1494}
1495
1496static int setup_netfront_split(struct netfront_info *info)
1497{
1498	int err;
1499
1500	err = xenbus_alloc_evtchn(info->xbdev, &info->tx_evtchn);
1501	if (err < 0)
1502		goto fail;
1503	err = xenbus_alloc_evtchn(info->xbdev, &info->rx_evtchn);
1504	if (err < 0)
1505		goto alloc_rx_evtchn_fail;
1506
1507	snprintf(info->tx_irq_name, sizeof(info->tx_irq_name),
1508		 "%s-tx", info->netdev->name);
1509	err = bind_evtchn_to_irqhandler(info->tx_evtchn,
1510					xennet_tx_interrupt,
1511					0, info->tx_irq_name, info);
1512	if (err < 0)
1513		goto bind_tx_fail;
1514	info->tx_irq = err;
1515
1516	snprintf(info->rx_irq_name, sizeof(info->rx_irq_name),
1517		 "%s-rx", info->netdev->name);
1518	err = bind_evtchn_to_irqhandler(info->rx_evtchn,
1519					xennet_rx_interrupt,
1520					0, info->rx_irq_name, info);
1521	if (err < 0)
1522		goto bind_rx_fail;
1523	info->rx_irq = err;
1524
1525	return 0;
1526
1527bind_rx_fail:
1528	unbind_from_irqhandler(info->tx_irq, info);
1529	info->tx_irq = 0;
1530bind_tx_fail:
1531	xenbus_free_evtchn(info->xbdev, info->rx_evtchn);
1532	info->rx_evtchn = 0;
1533alloc_rx_evtchn_fail:
1534	xenbus_free_evtchn(info->xbdev, info->tx_evtchn);
1535	info->tx_evtchn = 0;
1536fail:
1537	return err;
1538}
1539
1540static int setup_netfront(struct xenbus_device *dev, struct netfront_info *info)
 
1541{
1542	struct xen_netif_tx_sring *txs;
1543	struct xen_netif_rx_sring *rxs;
1544	int err;
1545	struct net_device *netdev = info->netdev;
1546	unsigned int feature_split_evtchn;
1547
1548	info->tx_ring_ref = GRANT_INVALID_REF;
1549	info->rx_ring_ref = GRANT_INVALID_REF;
1550	info->rx.sring = NULL;
1551	info->tx.sring = NULL;
1552	netdev->irq = 0;
1553
1554	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
1555			   "feature-split-event-channels", "%u",
1556			   &feature_split_evtchn);
1557	if (err < 0)
1558		feature_split_evtchn = 0;
1559
1560	err = xen_net_read_mac(dev, netdev->dev_addr);
1561	if (err) {
1562		xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename);
1563		goto fail;
1564	}
1565
1566	txs = (struct xen_netif_tx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH);
1567	if (!txs) {
1568		err = -ENOMEM;
1569		xenbus_dev_fatal(dev, err, "allocating tx ring page");
1570		goto fail;
1571	}
1572	SHARED_RING_INIT(txs);
1573	FRONT_RING_INIT(&info->tx, txs, PAGE_SIZE);
1574
1575	err = xenbus_grant_ring(dev, virt_to_mfn(txs));
1576	if (err < 0)
1577		goto grant_tx_ring_fail;
1578
1579	info->tx_ring_ref = err;
1580	rxs = (struct xen_netif_rx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH);
1581	if (!rxs) {
1582		err = -ENOMEM;
1583		xenbus_dev_fatal(dev, err, "allocating rx ring page");
1584		goto alloc_rx_ring_fail;
1585	}
1586	SHARED_RING_INIT(rxs);
1587	FRONT_RING_INIT(&info->rx, rxs, PAGE_SIZE);
1588
1589	err = xenbus_grant_ring(dev, virt_to_mfn(rxs));
1590	if (err < 0)
1591		goto grant_rx_ring_fail;
1592	info->rx_ring_ref = err;
1593
1594	if (feature_split_evtchn)
1595		err = setup_netfront_split(info);
1596	/* setup single event channel if
1597	 *  a) feature-split-event-channels == 0
1598	 *  b) feature-split-event-channels == 1 but failed to setup
1599	 */
1600	if (!feature_split_evtchn || (feature_split_evtchn && err))
1601		err = setup_netfront_single(info);
1602
1603	if (err)
1604		goto alloc_evtchn_fail;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1605
1606	return 0;
1607
1608	/* If we fail to setup netfront, it is safe to just revoke access to
1609	 * granted pages because backend is not accessing it at this point.
 
 
 
 
 
 
 
 
 
 
1610	 */
1611alloc_evtchn_fail:
1612	gnttab_end_foreign_access_ref(info->rx_ring_ref, 0);
1613grant_rx_ring_fail:
1614	free_page((unsigned long)rxs);
1615alloc_rx_ring_fail:
1616	gnttab_end_foreign_access_ref(info->tx_ring_ref, 0);
1617grant_tx_ring_fail:
1618	free_page((unsigned long)txs);
1619fail:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1620	return err;
1621}
1622
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1623/* Common code used when first setting up, and when resuming. */
1624static int talk_to_netback(struct xenbus_device *dev,
1625			   struct netfront_info *info)
1626{
1627	const char *message;
1628	struct xenbus_transaction xbt;
1629	int err;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1630
1631	/* Create shared ring, alloc event channel. */
1632	err = setup_netfront(dev, info);
1633	if (err)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1634		goto out;
 
 
 
 
 
 
 
 
 
 
1635
1636again:
1637	err = xenbus_transaction_start(&xbt);
1638	if (err) {
1639		xenbus_dev_fatal(dev, err, "starting transaction");
1640		goto destroy_ring;
1641	}
1642
1643	err = xenbus_printf(xbt, dev->nodename, "tx-ring-ref", "%u",
1644			    info->tx_ring_ref);
1645	if (err) {
1646		message = "writing tx ring-ref";
1647		goto abort_transaction;
1648	}
1649	err = xenbus_printf(xbt, dev->nodename, "rx-ring-ref", "%u",
1650			    info->rx_ring_ref);
1651	if (err) {
1652		message = "writing rx ring-ref";
1653		goto abort_transaction;
1654	}
1655
1656	if (info->tx_evtchn == info->rx_evtchn) {
1657		err = xenbus_printf(xbt, dev->nodename,
1658				    "event-channel", "%u", info->tx_evtchn);
1659		if (err) {
1660			message = "writing event-channel";
1661			goto abort_transaction;
1662		}
 
 
 
 
 
 
1663	} else {
1664		err = xenbus_printf(xbt, dev->nodename,
1665				    "event-channel-tx", "%u", info->tx_evtchn);
1666		if (err) {
1667			message = "writing event-channel-tx";
1668			goto abort_transaction;
1669		}
1670		err = xenbus_printf(xbt, dev->nodename,
1671				    "event-channel-rx", "%u", info->rx_evtchn);
1672		if (err) {
1673			message = "writing event-channel-rx";
1674			goto abort_transaction;
1675		}
1676	}
1677
 
1678	err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u",
1679			    1);
1680	if (err) {
1681		message = "writing request-rx-copy";
1682		goto abort_transaction;
1683	}
1684
1685	err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1);
1686	if (err) {
1687		message = "writing feature-rx-notify";
1688		goto abort_transaction;
1689	}
1690
1691	err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1);
1692	if (err) {
1693		message = "writing feature-sg";
1694		goto abort_transaction;
1695	}
1696
1697	err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1);
1698	if (err) {
1699		message = "writing feature-gso-tcpv4";
1700		goto abort_transaction;
1701	}
1702
1703	err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1");
1704	if (err) {
1705		message = "writing feature-gso-tcpv6";
1706		goto abort_transaction;
1707	}
1708
1709	err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload",
1710			   "1");
1711	if (err) {
1712		message = "writing feature-ipv6-csum-offload";
1713		goto abort_transaction;
1714	}
1715
1716	err = xenbus_transaction_end(xbt, 0);
1717	if (err) {
1718		if (err == -EAGAIN)
1719			goto again;
1720		xenbus_dev_fatal(dev, err, "completing transaction");
1721		goto destroy_ring;
1722	}
1723
1724	return 0;
1725
1726 abort_transaction:
1727	xenbus_transaction_end(xbt, 1);
1728	xenbus_dev_fatal(dev, err, "%s", message);
 
 
1729 destroy_ring:
1730	xennet_disconnect_backend(info);
 
 
1731 out:
 
 
 
1732	return err;
1733}
1734
1735static int xennet_connect(struct net_device *dev)
1736{
1737	struct netfront_info *np = netdev_priv(dev);
1738	int i, requeue_idx, err;
1739	struct sk_buff *skb;
1740	grant_ref_t ref;
1741	struct xen_netif_rx_request *req;
1742	unsigned int feature_rx_copy;
1743
1744	err = xenbus_scanf(XBT_NIL, np->xbdev->otherend,
1745			   "feature-rx-copy", "%u", &feature_rx_copy);
1746	if (err != 1)
1747		feature_rx_copy = 0;
1748
1749	if (!feature_rx_copy) {
1750		dev_info(&dev->dev,
1751			 "backend does not support copying receive path\n");
1752		return -ENODEV;
1753	}
1754
1755	err = talk_to_netback(np->xbdev, np);
1756	if (err)
1757		return err;
 
 
 
 
 
1758
1759	rtnl_lock();
1760	netdev_update_features(dev);
1761	rtnl_unlock();
1762
1763	spin_lock_bh(&np->rx_lock);
1764	spin_lock_irq(&np->tx_lock);
1765
1766	/* Step 1: Discard all pending TX packet fragments. */
1767	xennet_release_tx_bufs(np);
1768
1769	/* Step 2: Rebuild the RX buffer freelist and the RX ring itself. */
1770	for (requeue_idx = 0, i = 0; i < NET_RX_RING_SIZE; i++) {
1771		skb_frag_t *frag;
1772		const struct page *page;
1773		if (!np->rx_skbs[i])
1774			continue;
1775
1776		skb = np->rx_skbs[requeue_idx] = xennet_get_rx_skb(np, i);
1777		ref = np->grant_rx_ref[requeue_idx] = xennet_get_rx_ref(np, i);
1778		req = RING_GET_REQUEST(&np->rx, requeue_idx);
1779
1780		frag = &skb_shinfo(skb)->frags[0];
1781		page = skb_frag_page(frag);
1782		gnttab_grant_foreign_access_ref(
1783			ref, np->xbdev->otherend_id,
1784			pfn_to_mfn(page_to_pfn(page)),
1785			0);
1786		req->gref = ref;
1787		req->id   = requeue_idx;
1788
1789		requeue_idx++;
 
 
 
 
 
 
1790	}
1791
1792	np->rx.req_prod_pvt = requeue_idx;
 
 
1793
1794	/*
1795	 * Step 3: All public and private state should now be sane.  Get
1796	 * ready to start sending and receiving packets and give the driver
1797	 * domain a kick because we've probably just requeued some
1798	 * packets.
1799	 */
 
 
 
 
1800	netif_carrier_on(np->netdev);
1801	notify_remote_via_irq(np->tx_irq);
1802	if (np->tx_irq != np->rx_irq)
1803		notify_remote_via_irq(np->rx_irq);
1804	xennet_tx_buf_gc(dev);
1805	xennet_alloc_rx_buffers(dev);
1806
1807	spin_unlock_irq(&np->tx_lock);
1808	spin_unlock_bh(&np->rx_lock);
 
 
 
 
 
 
1809
1810	return 0;
1811}
1812
1813/**
1814 * Callback received when the backend's state changes.
1815 */
1816static void netback_changed(struct xenbus_device *dev,
1817			    enum xenbus_state backend_state)
1818{
1819	struct netfront_info *np = dev_get_drvdata(&dev->dev);
1820	struct net_device *netdev = np->netdev;
1821
1822	dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state));
1823
 
 
1824	switch (backend_state) {
1825	case XenbusStateInitialising:
1826	case XenbusStateInitialised:
1827	case XenbusStateReconfiguring:
1828	case XenbusStateReconfigured:
1829	case XenbusStateUnknown:
1830		break;
1831
1832	case XenbusStateInitWait:
1833		if (dev->state != XenbusStateInitialising)
1834			break;
1835		if (xennet_connect(netdev) != 0)
1836			break;
1837		xenbus_switch_state(dev, XenbusStateConnected);
1838		break;
1839
1840	case XenbusStateConnected:
1841		netdev_notify_peers(netdev);
1842		break;
1843
1844	case XenbusStateClosed:
1845		if (dev->state == XenbusStateClosed)
1846			break;
1847		/* Missed the backend's CLOSING state -- fallthrough */
1848	case XenbusStateClosing:
1849		xenbus_frontend_closed(dev);
1850		break;
1851	}
1852}
1853
1854static const struct xennet_stat {
1855	char name[ETH_GSTRING_LEN];
1856	u16 offset;
1857} xennet_stats[] = {
1858	{
1859		"rx_gso_checksum_fixup",
1860		offsetof(struct netfront_info, rx_gso_checksum_fixup)
1861	},
1862};
1863
1864static int xennet_get_sset_count(struct net_device *dev, int string_set)
1865{
1866	switch (string_set) {
1867	case ETH_SS_STATS:
1868		return ARRAY_SIZE(xennet_stats);
1869	default:
1870		return -EINVAL;
1871	}
1872}
1873
1874static void xennet_get_ethtool_stats(struct net_device *dev,
1875				     struct ethtool_stats *stats, u64 * data)
1876{
1877	void *np = netdev_priv(dev);
1878	int i;
1879
1880	for (i = 0; i < ARRAY_SIZE(xennet_stats); i++)
1881		data[i] = *(unsigned long *)(np + xennet_stats[i].offset);
1882}
1883
1884static void xennet_get_strings(struct net_device *dev, u32 stringset, u8 * data)
1885{
1886	int i;
1887
1888	switch (stringset) {
1889	case ETH_SS_STATS:
1890		for (i = 0; i < ARRAY_SIZE(xennet_stats); i++)
1891			memcpy(data + i * ETH_GSTRING_LEN,
1892			       xennet_stats[i].name, ETH_GSTRING_LEN);
1893		break;
1894	}
1895}
1896
1897static const struct ethtool_ops xennet_ethtool_ops =
1898{
1899	.get_link = ethtool_op_get_link,
1900
1901	.get_sset_count = xennet_get_sset_count,
1902	.get_ethtool_stats = xennet_get_ethtool_stats,
1903	.get_strings = xennet_get_strings,
 
1904};
1905
1906#ifdef CONFIG_SYSFS
1907static ssize_t show_rxbuf_min(struct device *dev,
1908			      struct device_attribute *attr, char *buf)
1909{
1910	struct net_device *netdev = to_net_dev(dev);
1911	struct netfront_info *info = netdev_priv(netdev);
1912
1913	return sprintf(buf, "%u\n", info->rx_min_target);
1914}
1915
1916static ssize_t store_rxbuf_min(struct device *dev,
1917			       struct device_attribute *attr,
1918			       const char *buf, size_t len)
1919{
1920	struct net_device *netdev = to_net_dev(dev);
1921	struct netfront_info *np = netdev_priv(netdev);
1922	char *endp;
1923	unsigned long target;
1924
1925	if (!capable(CAP_NET_ADMIN))
1926		return -EPERM;
1927
1928	target = simple_strtoul(buf, &endp, 0);
1929	if (endp == buf)
1930		return -EBADMSG;
1931
1932	if (target < RX_MIN_TARGET)
1933		target = RX_MIN_TARGET;
1934	if (target > RX_MAX_TARGET)
1935		target = RX_MAX_TARGET;
1936
1937	spin_lock_bh(&np->rx_lock);
1938	if (target > np->rx_max_target)
1939		np->rx_max_target = target;
1940	np->rx_min_target = target;
1941	if (target > np->rx_target)
1942		np->rx_target = target;
1943
1944	xennet_alloc_rx_buffers(netdev);
1945
1946	spin_unlock_bh(&np->rx_lock);
1947	return len;
1948}
1949
1950static ssize_t show_rxbuf_max(struct device *dev,
1951			      struct device_attribute *attr, char *buf)
1952{
1953	struct net_device *netdev = to_net_dev(dev);
1954	struct netfront_info *info = netdev_priv(netdev);
 
 
 
 
 
1955
1956	return sprintf(buf, "%u\n", info->rx_max_target);
1957}
 
 
1958
1959static ssize_t store_rxbuf_max(struct device *dev,
1960			       struct device_attribute *attr,
1961			       const char *buf, size_t len)
1962{
1963	struct net_device *netdev = to_net_dev(dev);
1964	struct netfront_info *np = netdev_priv(netdev);
1965	char *endp;
1966	unsigned long target;
1967
1968	if (!capable(CAP_NET_ADMIN))
1969		return -EPERM;
1970
1971	target = simple_strtoul(buf, &endp, 0);
1972	if (endp == buf)
1973		return -EBADMSG;
1974
1975	if (target < RX_MIN_TARGET)
1976		target = RX_MIN_TARGET;
1977	if (target > RX_MAX_TARGET)
1978		target = RX_MAX_TARGET;
1979
1980	spin_lock_bh(&np->rx_lock);
1981	if (target < np->rx_min_target)
1982		np->rx_min_target = target;
1983	np->rx_max_target = target;
1984	if (target < np->rx_target)
1985		np->rx_target = target;
 
 
1986
1987	xennet_alloc_rx_buffers(netdev);
 
1988
1989	spin_unlock_bh(&np->rx_lock);
1990	return len;
 
 
 
 
 
 
 
1991}
1992
1993static ssize_t show_rxbuf_cur(struct device *dev,
1994			      struct device_attribute *attr, char *buf)
1995{
1996	struct net_device *netdev = to_net_dev(dev);
1997	struct netfront_info *info = netdev_priv(netdev);
1998
1999	return sprintf(buf, "%u\n", info->rx_target);
2000}
2001
2002static struct device_attribute xennet_attrs[] = {
2003	__ATTR(rxbuf_min, S_IRUGO|S_IWUSR, show_rxbuf_min, store_rxbuf_min),
2004	__ATTR(rxbuf_max, S_IRUGO|S_IWUSR, show_rxbuf_max, store_rxbuf_max),
2005	__ATTR(rxbuf_cur, S_IRUGO, show_rxbuf_cur, NULL),
2006};
2007
2008static int xennet_sysfs_addif(struct net_device *netdev)
2009{
2010	int i;
2011	int err;
2012
2013	for (i = 0; i < ARRAY_SIZE(xennet_attrs); i++) {
2014		err = device_create_file(&netdev->dev,
2015					   &xennet_attrs[i]);
2016		if (err)
2017			goto fail;
2018	}
2019	return 0;
2020
2021 fail:
2022	while (--i >= 0)
2023		device_remove_file(&netdev->dev, &xennet_attrs[i]);
2024	return err;
2025}
2026
2027static void xennet_sysfs_delif(struct net_device *netdev)
2028{
2029	int i;
2030
2031	for (i = 0; i < ARRAY_SIZE(xennet_attrs); i++)
2032		device_remove_file(&netdev->dev, &xennet_attrs[i]);
2033}
2034
2035#endif /* CONFIG_SYSFS */
2036
2037static const struct xenbus_device_id netfront_ids[] = {
2038	{ "vif" },
2039	{ "" }
2040};
2041
2042
2043static int xennet_remove(struct xenbus_device *dev)
2044{
2045	struct netfront_info *info = dev_get_drvdata(&dev->dev);
2046
2047	dev_dbg(&dev->dev, "%s\n", dev->nodename);
2048
2049	xennet_disconnect_backend(info);
2050
2051	xennet_sysfs_delif(info->netdev);
2052
2053	unregister_netdev(info->netdev);
2054
2055	del_timer_sync(&info->rx_refill_timer);
2056
2057	free_percpu(info->stats);
2058
2059	free_netdev(info->netdev);
2060
2061	return 0;
2062}
2063
2064static DEFINE_XENBUS_DRIVER(netfront, ,
2065	.probe = netfront_probe,
2066	.remove = xennet_remove,
2067	.resume = netfront_resume,
2068	.otherend_changed = netback_changed,
2069);
2070
2071static int __init netif_init(void)
2072{
2073	if (!xen_domain())
2074		return -ENODEV;
2075
2076	if (!xen_has_pv_nic_devices())
2077		return -ENODEV;
2078
2079	pr_info("Initialising Xen virtual ethernet driver\n");
 
 
 
 
 
 
 
2080
2081	return xenbus_register_frontend(&netfront_driver);
2082}
2083module_init(netif_init);
2084
2085
2086static void __exit netif_exit(void)
2087{
2088	xenbus_unregister_driver(&netfront_driver);
2089}
2090module_exit(netif_exit);
2091
2092MODULE_DESCRIPTION("Xen virtual network device frontend");
2093MODULE_LICENSE("GPL");
2094MODULE_ALIAS("xen:vif");
2095MODULE_ALIAS("xennet");
v6.9.4
   1/*
   2 * Virtual network driver for conversing with remote driver backends.
   3 *
   4 * Copyright (c) 2002-2005, K A Fraser
   5 * Copyright (c) 2005, XenSource Ltd
   6 *
   7 * This program is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU General Public License version 2
   9 * as published by the Free Software Foundation; or, when distributed
  10 * separately from the Linux kernel or incorporated into other
  11 * software packages, subject to the following license:
  12 *
  13 * Permission is hereby granted, free of charge, to any person obtaining a copy
  14 * of this source file (the "Software"), to deal in the Software without
  15 * restriction, including without limitation the rights to use, copy, modify,
  16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  17 * and to permit persons to whom the Software is furnished to do so, subject to
  18 * the following conditions:
  19 *
  20 * The above copyright notice and this permission notice shall be included in
  21 * all copies or substantial portions of the Software.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  29 * IN THE SOFTWARE.
  30 */
  31
  32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  33
  34#include <linux/module.h>
  35#include <linux/kernel.h>
  36#include <linux/netdevice.h>
  37#include <linux/etherdevice.h>
  38#include <linux/skbuff.h>
  39#include <linux/ethtool.h>
  40#include <linux/if_ether.h>
  41#include <net/tcp.h>
  42#include <linux/udp.h>
  43#include <linux/moduleparam.h>
  44#include <linux/mm.h>
  45#include <linux/slab.h>
  46#include <net/ip.h>
  47#include <linux/bpf.h>
  48#include <net/page_pool/types.h>
  49#include <linux/bpf_trace.h>
  50
 
  51#include <xen/xen.h>
  52#include <xen/xenbus.h>
  53#include <xen/events.h>
  54#include <xen/page.h>
  55#include <xen/platform_pci.h>
  56#include <xen/grant_table.h>
  57
  58#include <xen/interface/io/netif.h>
  59#include <xen/interface/memory.h>
  60#include <xen/interface/grant_table.h>
  61
  62/* Module parameters */
  63#define MAX_QUEUES_DEFAULT 8
  64static unsigned int xennet_max_queues;
  65module_param_named(max_queues, xennet_max_queues, uint, 0644);
  66MODULE_PARM_DESC(max_queues,
  67		 "Maximum number of queues per virtual interface");
  68
  69static bool __read_mostly xennet_trusted = true;
  70module_param_named(trusted, xennet_trusted, bool, 0644);
  71MODULE_PARM_DESC(trusted, "Is the backend trusted");
  72
  73#define XENNET_TIMEOUT  (5 * HZ)
  74
  75static const struct ethtool_ops xennet_ethtool_ops;
  76
  77struct netfront_cb {
  78	int pull_to;
  79};
  80
  81#define NETFRONT_SKB_CB(skb)	((struct netfront_cb *)((skb)->cb))
  82
  83#define RX_COPY_THRESHOLD 256
  84
  85#define NET_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE)
  86#define NET_RX_RING_SIZE __CONST_RING_SIZE(xen_netif_rx, XEN_PAGE_SIZE)
  87
  88/* Minimum number of Rx slots (includes slot for GSO metadata). */
  89#define NET_RX_SLOTS_MIN (XEN_NETIF_NR_SLOTS_MIN + 1)
  90
  91/* Queue name is interface name with "-qNNN" appended */
  92#define QUEUE_NAME_SIZE (IFNAMSIZ + 6)
  93
  94/* IRQ name is queue name with "-tx" or "-rx" appended */
  95#define IRQ_NAME_SIZE (QUEUE_NAME_SIZE + 3)
  96
  97static DECLARE_WAIT_QUEUE_HEAD(module_wq);
 
 
  98
  99struct netfront_stats {
 100	u64			packets;
 101	u64			bytes;
 
 
 102	struct u64_stats_sync	syncp;
 103};
 104
 105struct netfront_info;
 106
 107struct netfront_queue {
 108	unsigned int id; /* Queue ID, 0-based */
 109	char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */
 110	struct netfront_info *info;
 111
 112	struct bpf_prog __rcu *xdp_prog;
 113
 114	struct napi_struct napi;
 115
 116	/* Split event channels support, tx_* == rx_* when using
 117	 * single event channel.
 118	 */
 119	unsigned int tx_evtchn, rx_evtchn;
 120	unsigned int tx_irq, rx_irq;
 121	/* Only used when split event channels support is enabled */
 122	char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */
 123	char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */
 
 
 124
 125	spinlock_t   tx_lock;
 126	struct xen_netif_tx_front_ring tx;
 127	int tx_ring_ref;
 128
 129	/*
 130	 * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries
 131	 * are linked from tx_skb_freelist through tx_link.
 
 
 
 
 
 132	 */
 133	struct sk_buff *tx_skbs[NET_TX_RING_SIZE];
 134	unsigned short tx_link[NET_TX_RING_SIZE];
 135#define TX_LINK_NONE 0xffff
 136#define TX_PENDING   0xfffe
 137	grant_ref_t gref_tx_head;
 138	grant_ref_t grant_tx_ref[NET_TX_RING_SIZE];
 139	struct page *grant_tx_page[NET_TX_RING_SIZE];
 140	unsigned tx_skb_freelist;
 141	unsigned int tx_pend_queue;
 142
 143	spinlock_t   rx_lock ____cacheline_aligned_in_smp;
 144	struct xen_netif_rx_front_ring rx;
 145	int rx_ring_ref;
 146
 
 
 
 
 
 
 
 147	struct timer_list rx_refill_timer;
 148
 149	struct sk_buff *rx_skbs[NET_RX_RING_SIZE];
 150	grant_ref_t gref_rx_head;
 151	grant_ref_t grant_rx_ref[NET_RX_RING_SIZE];
 152
 153	unsigned int rx_rsp_unconsumed;
 154	spinlock_t rx_cons_lock;
 155
 156	struct page_pool *page_pool;
 157	struct xdp_rxq_info xdp_rxq;
 158};
 159
 160struct netfront_info {
 161	struct list_head list;
 162	struct net_device *netdev;
 163
 164	struct xenbus_device *xbdev;
 165
 166	/* Multi-queue support */
 167	struct netfront_queue *queues;
 168
 169	/* Statistics */
 170	struct netfront_stats __percpu *rx_stats;
 171	struct netfront_stats __percpu *tx_stats;
 172
 173	/* XDP state */
 174	bool netback_has_xdp_headroom;
 175	bool netfront_xdp_enabled;
 176
 177	/* Is device behaving sane? */
 178	bool broken;
 179
 180	/* Should skbs be bounced into a zeroed buffer? */
 181	bool bounce;
 182
 183	atomic_t rx_gso_checksum_fixup;
 184};
 185
 186struct netfront_rx_info {
 187	struct xen_netif_rx_response rx;
 188	struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
 189};
 190
 
 
 
 
 
 
 
 
 
 
 
 191/*
 192 * Access macros for acquiring freeing slots in tx_skbs[].
 193 */
 194
 195static void add_id_to_list(unsigned *head, unsigned short *list,
 196			   unsigned short id)
 197{
 198	list[id] = *head;
 199	*head = id;
 200}
 201
 202static unsigned short get_id_from_list(unsigned *head, unsigned short *list)
 
 203{
 204	unsigned int id = *head;
 205
 206	if (id != TX_LINK_NONE) {
 207		*head = list[id];
 208		list[id] = TX_LINK_NONE;
 209	}
 210	return id;
 211}
 212
 213static int xennet_rxidx(RING_IDX idx)
 214{
 215	return idx & (NET_RX_RING_SIZE - 1);
 216}
 217
 218static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue,
 219					 RING_IDX ri)
 220{
 221	int i = xennet_rxidx(ri);
 222	struct sk_buff *skb = queue->rx_skbs[i];
 223	queue->rx_skbs[i] = NULL;
 224	return skb;
 225}
 226
 227static grant_ref_t xennet_get_rx_ref(struct netfront_queue *queue,
 228					    RING_IDX ri)
 229{
 230	int i = xennet_rxidx(ri);
 231	grant_ref_t ref = queue->grant_rx_ref[i];
 232	queue->grant_rx_ref[i] = INVALID_GRANT_REF;
 233	return ref;
 234}
 235
 236#ifdef CONFIG_SYSFS
 237static const struct attribute_group xennet_dev_group;
 
 
 
 
 238#endif
 239
 240static bool xennet_can_sg(struct net_device *dev)
 241{
 242	return dev->features & NETIF_F_SG;
 243}
 244
 245
 246static void rx_refill_timeout(struct timer_list *t)
 247{
 248	struct netfront_queue *queue = from_timer(queue, t, rx_refill_timer);
 249	napi_schedule(&queue->napi);
 
 250}
 251
 252static int netfront_tx_slot_available(struct netfront_queue *queue)
 253{
 254	return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) <
 255		(NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1);
 256}
 257
 258static void xennet_maybe_wake_tx(struct netfront_queue *queue)
 259{
 260	struct net_device *dev = queue->info->netdev;
 261	struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id);
 262
 263	if (unlikely(netif_tx_queue_stopped(dev_queue)) &&
 264	    netfront_tx_slot_available(queue) &&
 265	    likely(netif_running(dev)))
 266		netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id));
 267}
 268
 269
 270static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue)
 271{
 
 
 272	struct sk_buff *skb;
 273	struct page *page;
 
 
 
 
 
 
 274
 275	skb = __netdev_alloc_skb(queue->info->netdev,
 276				 RX_COPY_THRESHOLD + NET_IP_ALIGN,
 277				 GFP_ATOMIC | __GFP_NOWARN);
 278	if (unlikely(!skb))
 279		return NULL;
 280
 281	page = page_pool_alloc_pages(queue->page_pool,
 282				     GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO);
 283	if (unlikely(!page)) {
 284		kfree_skb(skb);
 285		return NULL;
 286	}
 287	skb_add_rx_frag(skb, 0, page, 0, 0, PAGE_SIZE);
 288	skb_mark_for_recycle(skb);
 289
 290	/* Align ip header to a 16 bytes boundary */
 291	skb_reserve(skb, NET_IP_ALIGN);
 292	skb->dev = queue->info->netdev;
 
 
 
 
 
 
 
 
 
 293
 294	return skb;
 295}
 296
 
 
 
 
 
 
 
 
 
 
 
 
 
 297
 298static void xennet_alloc_rx_buffers(struct netfront_queue *queue)
 299{
 300	RING_IDX req_prod = queue->rx.req_prod_pvt;
 301	int notify;
 302	int err = 0;
 303
 304	if (unlikely(!netif_carrier_ok(queue->info->netdev)))
 
 
 
 305		return;
 
 306
 307	for (req_prod = queue->rx.req_prod_pvt;
 308	     req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE;
 309	     req_prod++) {
 310		struct sk_buff *skb;
 311		unsigned short id;
 312		grant_ref_t ref;
 313		struct page *page;
 314		struct xen_netif_rx_request *req;
 
 
 
 
 315
 316		skb = xennet_alloc_one_rx_buffer(queue);
 317		if (!skb) {
 318			err = -ENOMEM;
 319			break;
 320		}
 321
 322		id = xennet_rxidx(req_prod);
 
 323
 324		BUG_ON(queue->rx_skbs[id]);
 325		queue->rx_skbs[id] = skb;
 
 326
 327		ref = gnttab_claim_grant_reference(&queue->gref_rx_head);
 328		WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref));
 329		queue->grant_rx_ref[id] = ref;
 330
 331		page = skb_frag_page(&skb_shinfo(skb)->frags[0]);
 
 
 
 
 332
 333		req = RING_GET_REQUEST(&queue->rx, req_prod);
 334		gnttab_page_grant_foreign_access_ref_one(ref,
 335							 queue->info->xbdev->otherend_id,
 336							 page,
 337							 0);
 338		req->id = id;
 339		req->gref = ref;
 340	}
 341
 342	queue->rx.req_prod_pvt = req_prod;
 343
 344	/* Try again later if there are not enough requests or skb allocation
 345	 * failed.
 346	 * Enough requests is quantified as the sum of newly created slots and
 347	 * the unconsumed slots at the backend.
 348	 */
 349	if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN ||
 350	    unlikely(err)) {
 351		mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10));
 352		return;
 353	}
 354
 355	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify);
 356	if (notify)
 357		notify_remote_via_irq(queue->rx_irq);
 358}
 359
 360static int xennet_open(struct net_device *dev)
 361{
 362	struct netfront_info *np = netdev_priv(dev);
 363	unsigned int num_queues = dev->real_num_tx_queues;
 364	unsigned int i = 0;
 365	struct netfront_queue *queue = NULL;
 366
 367	if (!np->queues || np->broken)
 368		return -ENODEV;
 369
 370	for (i = 0; i < num_queues; ++i) {
 371		queue = &np->queues[i];
 372		napi_enable(&queue->napi);
 373
 374		spin_lock_bh(&queue->rx_lock);
 375		if (netif_carrier_ok(dev)) {
 376			xennet_alloc_rx_buffers(queue);
 377			queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1;
 378			if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx))
 379				napi_schedule(&queue->napi);
 380		}
 381		spin_unlock_bh(&queue->rx_lock);
 382	}
 
 383
 384	netif_tx_start_all_queues(dev);
 385
 386	return 0;
 387}
 388
 389static bool xennet_tx_buf_gc(struct netfront_queue *queue)
 390{
 391	RING_IDX cons, prod;
 392	unsigned short id;
 
 393	struct sk_buff *skb;
 394	bool more_to_do;
 395	bool work_done = false;
 396	const struct device *dev = &queue->info->netdev->dev;
 397
 398	BUG_ON(!netif_carrier_ok(queue->info->netdev));
 399
 400	do {
 401		prod = queue->tx.sring->rsp_prod;
 402		if (RING_RESPONSE_PROD_OVERFLOW(&queue->tx, prod)) {
 403			dev_alert(dev, "Illegal number of responses %u\n",
 404				  prod - queue->tx.rsp_cons);
 405			goto err;
 406		}
 407		rmb(); /* Ensure we see responses up to 'rp'. */
 408
 409		for (cons = queue->tx.rsp_cons; cons != prod; cons++) {
 410			struct xen_netif_tx_response txrsp;
 411
 412			work_done = true;
 413
 414			RING_COPY_RESPONSE(&queue->tx, cons, &txrsp);
 415			if (txrsp.status == XEN_NETIF_RSP_NULL)
 416				continue;
 417
 418			id = txrsp.id;
 419			if (id >= RING_SIZE(&queue->tx)) {
 420				dev_alert(dev,
 421					  "Response has incorrect id (%u)\n",
 422					  id);
 423				goto err;
 424			}
 425			if (queue->tx_link[id] != TX_PENDING) {
 426				dev_alert(dev,
 427					  "Response for inactive request\n");
 428				goto err;
 429			}
 430
 431			queue->tx_link[id] = TX_LINK_NONE;
 432			skb = queue->tx_skbs[id];
 433			queue->tx_skbs[id] = NULL;
 434			if (unlikely(!gnttab_end_foreign_access_ref(
 435				queue->grant_tx_ref[id]))) {
 436				dev_alert(dev,
 437					  "Grant still in use by backend domain\n");
 438				goto err;
 439			}
 
 
 440			gnttab_release_grant_reference(
 441				&queue->gref_tx_head, queue->grant_tx_ref[id]);
 442			queue->grant_tx_ref[id] = INVALID_GRANT_REF;
 443			queue->grant_tx_page[id] = NULL;
 444			add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id);
 445			dev_kfree_skb_irq(skb);
 446		}
 447
 448		queue->tx.rsp_cons = prod;
 449
 450		RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do);
 451	} while (more_to_do);
 452
 453	xennet_maybe_wake_tx(queue);
 454
 455	return work_done;
 
 
 
 
 
 
 456
 457 err:
 458	queue->info->broken = true;
 459	dev_alert(dev, "Disabled for further use\n");
 460
 461	return work_done;
 462}
 463
 464struct xennet_gnttab_make_txreq {
 465	struct netfront_queue *queue;
 466	struct sk_buff *skb;
 467	struct page *page;
 468	struct xen_netif_tx_request *tx;      /* Last request on ring page */
 469	struct xen_netif_tx_request tx_local; /* Last request local copy*/
 470	unsigned int size;
 471};
 472
 473static void xennet_tx_setup_grant(unsigned long gfn, unsigned int offset,
 474				  unsigned int len, void *data)
 475{
 476	struct xennet_gnttab_make_txreq *info = data;
 
 
 
 
 
 
 477	unsigned int id;
 478	struct xen_netif_tx_request *tx;
 479	grant_ref_t ref;
 480	/* convenient aliases */
 481	struct page *page = info->page;
 482	struct netfront_queue *queue = info->queue;
 483	struct sk_buff *skb = info->skb;
 484
 485	id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link);
 486	tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++);
 487	ref = gnttab_claim_grant_reference(&queue->gref_tx_head);
 488	WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref));
 489
 490	gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id,
 491					gfn, GNTMAP_readonly);
 492
 493	queue->tx_skbs[id] = skb;
 494	queue->grant_tx_page[id] = page;
 495	queue->grant_tx_ref[id] = ref;
 496
 497	info->tx_local.id = id;
 498	info->tx_local.gref = ref;
 499	info->tx_local.offset = offset;
 500	info->tx_local.size = len;
 501	info->tx_local.flags = 0;
 502
 503	*tx = info->tx_local;
 
 
 
 
 
 
 
 504
 505	/*
 506	 * Put the request in the pending queue, it will be set to be pending
 507	 * when the producer index is about to be raised.
 508	 */
 509	add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id);
 
 
 
 
 
 
 
 
 
 
 
 
 510
 511	info->tx = tx;
 512	info->size += info->tx_local.size;
 513}
 
 514
 515static struct xen_netif_tx_request *xennet_make_first_txreq(
 516	struct xennet_gnttab_make_txreq *info,
 517	unsigned int offset, unsigned int len)
 518{
 519	info->size = 0;
 520
 521	gnttab_for_one_grant(info->page, offset, len, xennet_tx_setup_grant, info);
 
 522
 523	return info->tx;
 524}
 
 525
 526static void xennet_make_one_txreq(unsigned long gfn, unsigned int offset,
 527				  unsigned int len, void *data)
 528{
 529	struct xennet_gnttab_make_txreq *info = data;
 530
 531	info->tx->flags |= XEN_NETTXF_more_data;
 532	skb_get(info->skb);
 533	xennet_tx_setup_grant(gfn, offset, len, data);
 534}
 535
 536static void xennet_make_txreqs(
 537	struct xennet_gnttab_make_txreq *info,
 538	struct page *page,
 539	unsigned int offset, unsigned int len)
 540{
 541	/* Skip unused frames from start of page */
 542	page += offset >> PAGE_SHIFT;
 543	offset &= ~PAGE_MASK;
 544
 545	while (len) {
 546		info->page = page;
 547		info->size = 0;
 548
 549		gnttab_foreach_grant_in_range(page, offset, len,
 550					      xennet_make_one_txreq,
 551					      info);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 552
 553		page++;
 554		offset = 0;
 555		len -= info->size;
 556	}
 557}
 558
 559/*
 560 * Count how many ring slots are required to send this skb. Each frag
 561 * might be a compound page.
 562 */
 563static int xennet_count_skb_slots(struct sk_buff *skb)
 564{
 565	int i, frags = skb_shinfo(skb)->nr_frags;
 566	int slots;
 567
 568	slots = gnttab_count_grant(offset_in_page(skb->data),
 569				   skb_headlen(skb));
 570
 571	for (i = 0; i < frags; i++) {
 572		skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 573		unsigned long size = skb_frag_size(frag);
 574		unsigned long offset = skb_frag_off(frag);
 575
 576		/* Skip unused frames from start of page */
 577		offset &= ~PAGE_MASK;
 578
 579		slots += gnttab_count_grant(offset, size);
 580	}
 581
 582	return slots;
 583}
 584
 585static u16 xennet_select_queue(struct net_device *dev, struct sk_buff *skb,
 586			       struct net_device *sb_dev)
 587{
 588	unsigned int num_queues = dev->real_num_tx_queues;
 589	u32 hash;
 590	u16 queue_idx;
 591
 592	/* First, check if there is only one queue */
 593	if (num_queues == 1) {
 594		queue_idx = 0;
 595	} else {
 596		hash = skb_get_hash(skb);
 597		queue_idx = hash % num_queues;
 598	}
 599
 600	return queue_idx;
 601}
 602
 603static void xennet_mark_tx_pending(struct netfront_queue *queue)
 604{
 605	unsigned int i;
 606
 607	while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) !=
 608	       TX_LINK_NONE)
 609		queue->tx_link[i] = TX_PENDING;
 610}
 611
 612static int xennet_xdp_xmit_one(struct net_device *dev,
 613			       struct netfront_queue *queue,
 614			       struct xdp_frame *xdpf)
 615{
 
 616	struct netfront_info *np = netdev_priv(dev);
 617	struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats);
 618	struct xennet_gnttab_make_txreq info = {
 619		.queue = queue,
 620		.skb = NULL,
 621		.page = virt_to_page(xdpf->data),
 622	};
 623	int notify;
 624
 625	xennet_make_first_txreq(&info,
 626				offset_in_page(xdpf->data),
 627				xdpf->len);
 628
 629	xennet_mark_tx_pending(queue);
 630
 631	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify);
 632	if (notify)
 633		notify_remote_via_irq(queue->tx_irq);
 634
 635	u64_stats_update_begin(&tx_stats->syncp);
 636	tx_stats->bytes += xdpf->len;
 637	tx_stats->packets++;
 638	u64_stats_update_end(&tx_stats->syncp);
 639
 640	xennet_tx_buf_gc(queue);
 641
 642	return 0;
 643}
 644
 645static int xennet_xdp_xmit(struct net_device *dev, int n,
 646			   struct xdp_frame **frames, u32 flags)
 647{
 648	unsigned int num_queues = dev->real_num_tx_queues;
 649	struct netfront_info *np = netdev_priv(dev);
 650	struct netfront_queue *queue = NULL;
 651	unsigned long irq_flags;
 652	int nxmit = 0;
 653	int i;
 654
 655	if (unlikely(np->broken))
 656		return -ENODEV;
 657	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
 658		return -EINVAL;
 659
 660	queue = &np->queues[smp_processor_id() % num_queues];
 661
 662	spin_lock_irqsave(&queue->tx_lock, irq_flags);
 663	for (i = 0; i < n; i++) {
 664		struct xdp_frame *xdpf = frames[i];
 665
 666		if (!xdpf)
 667			continue;
 668		if (xennet_xdp_xmit_one(dev, queue, xdpf))
 669			break;
 670		nxmit++;
 671	}
 672	spin_unlock_irqrestore(&queue->tx_lock, irq_flags);
 673
 674	return nxmit;
 675}
 676
 677static struct sk_buff *bounce_skb(const struct sk_buff *skb)
 678{
 679	unsigned int headerlen = skb_headroom(skb);
 680	/* Align size to allocate full pages and avoid contiguous data leaks */
 681	unsigned int size = ALIGN(skb_end_offset(skb) + skb->data_len,
 682				  XEN_PAGE_SIZE);
 683	struct sk_buff *n = alloc_skb(size, GFP_ATOMIC | __GFP_ZERO);
 684
 685	if (!n)
 686		return NULL;
 687
 688	if (!IS_ALIGNED((uintptr_t)n->head, XEN_PAGE_SIZE)) {
 689		WARN_ONCE(1, "misaligned skb allocated\n");
 690		kfree_skb(n);
 691		return NULL;
 692	}
 693
 694	/* Set the data pointer */
 695	skb_reserve(n, headerlen);
 696	/* Set the tail pointer and length */
 697	skb_put(n, skb->len);
 698
 699	BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
 700
 701	skb_copy_header(n, skb);
 702	return n;
 703}
 704
 705#define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1)
 706
 707static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev)
 708{
 709	struct netfront_info *np = netdev_priv(dev);
 710	struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats);
 711	struct xen_netif_tx_request *first_tx;
 712	unsigned int i;
 713	int notify;
 714	int slots;
 715	struct page *page;
 716	unsigned int offset;
 717	unsigned int len;
 718	unsigned long flags;
 719	struct netfront_queue *queue = NULL;
 720	struct xennet_gnttab_make_txreq info = { };
 721	unsigned int num_queues = dev->real_num_tx_queues;
 722	u16 queue_index;
 723	struct sk_buff *nskb;
 724
 725	/* Drop the packet if no queues are set up */
 726	if (num_queues < 1)
 727		goto drop;
 728	if (unlikely(np->broken))
 729		goto drop;
 730	/* Determine which queue to transmit this SKB on */
 731	queue_index = skb_get_queue_mapping(skb);
 732	queue = &np->queues[queue_index];
 733
 734	/* If skb->len is too big for wire format, drop skb and alert
 735	 * user about misconfiguration.
 736	 */
 737	if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) {
 738		net_alert_ratelimited(
 739			"xennet: skb->len = %u, too big for wire format\n",
 740			skb->len);
 741		goto drop;
 742	}
 743
 744	slots = xennet_count_skb_slots(skb);
 745	if (unlikely(slots > MAX_XEN_SKB_FRAGS + 1)) {
 746		net_dbg_ratelimited("xennet: skb rides the rocket: %d slots, %d bytes\n",
 747				    slots, skb->len);
 748		if (skb_linearize(skb))
 749			goto drop;
 750	}
 751
 752	page = virt_to_page(skb->data);
 753	offset = offset_in_page(skb->data);
 754
 755	/* The first req should be at least ETH_HLEN size or the packet will be
 756	 * dropped by netback.
 757	 *
 758	 * If the backend is not trusted bounce all data to zeroed pages to
 759	 * avoid exposing contiguous data on the granted page not belonging to
 760	 * the skb.
 761	 */
 762	if (np->bounce || unlikely(PAGE_SIZE - offset < ETH_HLEN)) {
 763		nskb = bounce_skb(skb);
 764		if (!nskb)
 765			goto drop;
 766		dev_consume_skb_any(skb);
 767		skb = nskb;
 768		page = virt_to_page(skb->data);
 769		offset = offset_in_page(skb->data);
 770	}
 771
 772	len = skb_headlen(skb);
 773
 774	spin_lock_irqsave(&queue->tx_lock, flags);
 775
 776	if (unlikely(!netif_carrier_ok(dev) ||
 777		     (slots > 1 && !xennet_can_sg(dev)) ||
 778		     netif_needs_gso(skb, netif_skb_features(skb)))) {
 779		spin_unlock_irqrestore(&queue->tx_lock, flags);
 780		goto drop;
 781	}
 782
 783	/* First request for the linear area. */
 784	info.queue = queue;
 785	info.skb = skb;
 786	info.page = page;
 787	first_tx = xennet_make_first_txreq(&info, offset, len);
 788	offset += info.tx_local.size;
 789	if (offset == PAGE_SIZE) {
 790		page++;
 791		offset = 0;
 792	}
 793	len -= info.tx_local.size;
 
 
 
 
 
 
 794
 
 795	if (skb->ip_summed == CHECKSUM_PARTIAL)
 796		/* local packet? */
 797		first_tx->flags |= XEN_NETTXF_csum_blank |
 798				   XEN_NETTXF_data_validated;
 799	else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
 800		/* remote but checksummed. */
 801		first_tx->flags |= XEN_NETTXF_data_validated;
 802
 803	/* Optional extra info after the first request. */
 804	if (skb_shinfo(skb)->gso_size) {
 805		struct xen_netif_extra_info *gso;
 806
 807		gso = (struct xen_netif_extra_info *)
 808			RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++);
 809
 810		first_tx->flags |= XEN_NETTXF_extra_info;
 811
 812		gso->u.gso.size = skb_shinfo(skb)->gso_size;
 813		gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ?
 814			XEN_NETIF_GSO_TYPE_TCPV6 :
 815			XEN_NETIF_GSO_TYPE_TCPV4;
 816		gso->u.gso.pad = 0;
 817		gso->u.gso.features = 0;
 818
 819		gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
 820		gso->flags = 0;
 821	}
 822
 823	/* Requests for the rest of the linear area. */
 824	xennet_make_txreqs(&info, page, offset, len);
 825
 826	/* Requests for all the frags. */
 827	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 828		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 829		xennet_make_txreqs(&info, skb_frag_page(frag),
 830					skb_frag_off(frag),
 831					skb_frag_size(frag));
 832	}
 833
 834	/* First request has the packet length. */
 835	first_tx->size = skb->len;
 836
 837	/* timestamp packet in software */
 838	skb_tx_timestamp(skb);
 839
 840	xennet_mark_tx_pending(queue);
 841
 842	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify);
 843	if (notify)
 844		notify_remote_via_irq(queue->tx_irq);
 845
 846	u64_stats_update_begin(&tx_stats->syncp);
 847	tx_stats->bytes += skb->len;
 848	tx_stats->packets++;
 849	u64_stats_update_end(&tx_stats->syncp);
 850
 851	/* Note: It is not safe to access skb after xennet_tx_buf_gc()! */
 852	xennet_tx_buf_gc(queue);
 853
 854	if (!netfront_tx_slot_available(queue))
 855		netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id));
 856
 857	spin_unlock_irqrestore(&queue->tx_lock, flags);
 858
 859	return NETDEV_TX_OK;
 860
 861 drop:
 862	dev->stats.tx_dropped++;
 863	dev_kfree_skb_any(skb);
 864	return NETDEV_TX_OK;
 865}
 866
 867static int xennet_close(struct net_device *dev)
 868{
 869	struct netfront_info *np = netdev_priv(dev);
 870	unsigned int num_queues = dev->real_num_tx_queues;
 871	unsigned int i;
 872	struct netfront_queue *queue;
 873	netif_tx_stop_all_queues(np->netdev);
 874	for (i = 0; i < num_queues; ++i) {
 875		queue = &np->queues[i];
 876		napi_disable(&queue->napi);
 877	}
 878	return 0;
 879}
 880
 881static void xennet_destroy_queues(struct netfront_info *info)
 882{
 883	unsigned int i;
 884
 885	for (i = 0; i < info->netdev->real_num_tx_queues; i++) {
 886		struct netfront_queue *queue = &info->queues[i];
 887
 888		if (netif_running(info->netdev))
 889			napi_disable(&queue->napi);
 890		netif_napi_del(&queue->napi);
 891	}
 892
 893	kfree(info->queues);
 894	info->queues = NULL;
 895}
 896
 897static void xennet_uninit(struct net_device *dev)
 898{
 899	struct netfront_info *np = netdev_priv(dev);
 900	xennet_destroy_queues(np);
 901}
 902
 903static void xennet_set_rx_rsp_cons(struct netfront_queue *queue, RING_IDX val)
 904{
 905	unsigned long flags;
 906
 907	spin_lock_irqsave(&queue->rx_cons_lock, flags);
 908	queue->rx.rsp_cons = val;
 909	queue->rx_rsp_unconsumed = XEN_RING_NR_UNCONSUMED_RESPONSES(&queue->rx);
 910	spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
 911}
 912
 913static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb,
 914				grant_ref_t ref)
 915{
 916	int new = xennet_rxidx(queue->rx.req_prod_pvt);
 917
 918	BUG_ON(queue->rx_skbs[new]);
 919	queue->rx_skbs[new] = skb;
 920	queue->grant_rx_ref[new] = ref;
 921	RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new;
 922	RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref;
 923	queue->rx.req_prod_pvt++;
 924}
 925
 926static int xennet_get_extras(struct netfront_queue *queue,
 927			     struct xen_netif_extra_info *extras,
 928			     RING_IDX rp)
 929
 930{
 931	struct xen_netif_extra_info extra;
 932	struct device *dev = &queue->info->netdev->dev;
 933	RING_IDX cons = queue->rx.rsp_cons;
 934	int err = 0;
 935
 936	do {
 937		struct sk_buff *skb;
 938		grant_ref_t ref;
 939
 940		if (unlikely(cons + 1 == rp)) {
 941			if (net_ratelimit())
 942				dev_warn(dev, "Missing extra info\n");
 943			err = -EBADR;
 944			break;
 945		}
 946
 947		RING_COPY_RESPONSE(&queue->rx, ++cons, &extra);
 
 948
 949		if (unlikely(!extra.type ||
 950			     extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
 951			if (net_ratelimit())
 952				dev_warn(dev, "Invalid extra type: %d\n",
 953					 extra.type);
 954			err = -EINVAL;
 955		} else {
 956			extras[extra.type - 1] = extra;
 
 957		}
 958
 959		skb = xennet_get_rx_skb(queue, cons);
 960		ref = xennet_get_rx_ref(queue, cons);
 961		xennet_move_rx_slot(queue, skb, ref);
 962	} while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);
 963
 964	xennet_set_rx_rsp_cons(queue, cons);
 965	return err;
 966}
 967
 968static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata,
 969		   struct xen_netif_rx_response *rx, struct bpf_prog *prog,
 970		   struct xdp_buff *xdp, bool *need_xdp_flush)
 971{
 972	struct xdp_frame *xdpf;
 973	u32 len = rx->status;
 974	u32 act;
 975	int err;
 976
 977	xdp_init_buff(xdp, XEN_PAGE_SIZE - XDP_PACKET_HEADROOM,
 978		      &queue->xdp_rxq);
 979	xdp_prepare_buff(xdp, page_address(pdata), XDP_PACKET_HEADROOM,
 980			 len, false);
 981
 982	act = bpf_prog_run_xdp(prog, xdp);
 983	switch (act) {
 984	case XDP_TX:
 985		get_page(pdata);
 986		xdpf = xdp_convert_buff_to_frame(xdp);
 987		err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0);
 988		if (unlikely(!err))
 989			xdp_return_frame_rx_napi(xdpf);
 990		else if (unlikely(err < 0))
 991			trace_xdp_exception(queue->info->netdev, prog, act);
 992		break;
 993	case XDP_REDIRECT:
 994		get_page(pdata);
 995		err = xdp_do_redirect(queue->info->netdev, xdp, prog);
 996		*need_xdp_flush = true;
 997		if (unlikely(err))
 998			trace_xdp_exception(queue->info->netdev, prog, act);
 999		break;
1000	case XDP_PASS:
1001	case XDP_DROP:
1002		break;
1003
1004	case XDP_ABORTED:
1005		trace_xdp_exception(queue->info->netdev, prog, act);
1006		break;
1007
1008	default:
1009		bpf_warn_invalid_xdp_action(queue->info->netdev, prog, act);
1010	}
1011
1012	return act;
1013}
1014
1015static int xennet_get_responses(struct netfront_queue *queue,
1016				struct netfront_rx_info *rinfo, RING_IDX rp,
1017				struct sk_buff_head *list,
1018				bool *need_xdp_flush)
1019{
1020	struct xen_netif_rx_response *rx = &rinfo->rx, rx_local;
1021	int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD);
1022	RING_IDX cons = queue->rx.rsp_cons;
1023	struct sk_buff *skb = xennet_get_rx_skb(queue, cons);
1024	struct xen_netif_extra_info *extras = rinfo->extras;
1025	grant_ref_t ref = xennet_get_rx_ref(queue, cons);
1026	struct device *dev = &queue->info->netdev->dev;
1027	struct bpf_prog *xdp_prog;
1028	struct xdp_buff xdp;
 
1029	int slots = 1;
1030	int err = 0;
1031	u32 verdict;
1032
1033	if (rx->flags & XEN_NETRXF_extra_info) {
1034		err = xennet_get_extras(queue, extras, rp);
1035		if (!err) {
1036			if (extras[XEN_NETIF_EXTRA_TYPE_XDP - 1].type) {
1037				struct xen_netif_extra_info *xdp;
1038
1039				xdp = &extras[XEN_NETIF_EXTRA_TYPE_XDP - 1];
1040				rx->offset = xdp->u.xdp.headroom;
1041			}
 
 
 
 
 
 
1042		}
1043		cons = queue->rx.rsp_cons;
1044	}
1045
1046	for (;;) {
1047		/*
1048		 * This definitely indicates a bug, either in this driver or in
1049		 * the backend driver. In future this should flag the bad
1050		 * situation to the system controller to reboot the backend.
1051		 */
1052		if (ref == INVALID_GRANT_REF) {
1053			if (net_ratelimit())
1054				dev_warn(dev, "Bad rx response id %d.\n",
1055					 rx->id);
1056			err = -EINVAL;
1057			goto next;
1058		}
1059
1060		if (unlikely(rx->status < 0 ||
1061			     rx->offset + rx->status > XEN_PAGE_SIZE)) {
1062			if (net_ratelimit())
1063				dev_warn(dev, "rx->offset: %u, size: %d\n",
1064					 rx->offset, rx->status);
1065			xennet_move_rx_slot(queue, skb, ref);
1066			err = -EINVAL;
1067			goto next;
1068		}
1069
1070		if (!gnttab_end_foreign_access_ref(ref)) {
1071			dev_alert(dev,
1072				  "Grant still in use by backend domain\n");
1073			queue->info->broken = true;
1074			dev_alert(dev, "Disabled for further use\n");
1075			return -EINVAL;
1076		}
1077
1078		gnttab_release_grant_reference(&queue->gref_rx_head, ref);
1079
1080		rcu_read_lock();
1081		xdp_prog = rcu_dereference(queue->xdp_prog);
1082		if (xdp_prog) {
1083			if (!(rx->flags & XEN_NETRXF_more_data)) {
1084				/* currently only a single page contains data */
1085				verdict = xennet_run_xdp(queue,
1086							 skb_frag_page(&skb_shinfo(skb)->frags[0]),
1087							 rx, xdp_prog, &xdp, need_xdp_flush);
1088				if (verdict != XDP_PASS)
1089					err = -EINVAL;
1090			} else {
1091				/* drop the frame */
1092				err = -EINVAL;
1093			}
1094		}
1095		rcu_read_unlock();
1096
1097		__skb_queue_tail(list, skb);
1098
1099next:
1100		if (!(rx->flags & XEN_NETRXF_more_data))
1101			break;
1102
1103		if (cons + slots == rp) {
1104			if (net_ratelimit())
1105				dev_warn(dev, "Need more slots\n");
1106			err = -ENOENT;
1107			break;
1108		}
1109
1110		RING_COPY_RESPONSE(&queue->rx, cons + slots, &rx_local);
1111		rx = &rx_local;
1112		skb = xennet_get_rx_skb(queue, cons + slots);
1113		ref = xennet_get_rx_ref(queue, cons + slots);
1114		slots++;
1115	}
1116
1117	if (unlikely(slots > max)) {
1118		if (net_ratelimit())
1119			dev_warn(dev, "Too many slots\n");
1120		err = -E2BIG;
1121	}
1122
1123	if (unlikely(err))
1124		xennet_set_rx_rsp_cons(queue, cons + slots);
1125
1126	return err;
1127}
1128
1129static int xennet_set_skb_gso(struct sk_buff *skb,
1130			      struct xen_netif_extra_info *gso)
1131{
1132	if (!gso->u.gso.size) {
1133		if (net_ratelimit())
1134			pr_warn("GSO size must not be zero\n");
1135		return -EINVAL;
1136	}
1137
1138	if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 &&
1139	    gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) {
1140		if (net_ratelimit())
1141			pr_warn("Bad GSO type %d\n", gso->u.gso.type);
1142		return -EINVAL;
1143	}
1144
1145	skb_shinfo(skb)->gso_size = gso->u.gso.size;
1146	skb_shinfo(skb)->gso_type =
1147		(gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ?
1148		SKB_GSO_TCPV4 :
1149		SKB_GSO_TCPV6;
1150
1151	/* Header must be checked, and gso_segs computed. */
1152	skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1153	skb_shinfo(skb)->gso_segs = 0;
1154
1155	return 0;
1156}
1157
1158static int xennet_fill_frags(struct netfront_queue *queue,
1159			     struct sk_buff *skb,
1160			     struct sk_buff_head *list)
1161{
1162	RING_IDX cons = queue->rx.rsp_cons;
 
1163	struct sk_buff *nskb;
1164
1165	while ((nskb = __skb_dequeue(list))) {
1166		struct xen_netif_rx_response rx;
 
1167		skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0];
1168
1169		RING_COPY_RESPONSE(&queue->rx, ++cons, &rx);
1170
1171		if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) {
1172			unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to;
1173
1174			BUG_ON(pull_to < skb_headlen(skb));
1175			__pskb_pull_tail(skb, pull_to - skb_headlen(skb));
1176		}
1177		if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) {
1178			xennet_set_rx_rsp_cons(queue,
1179					       ++cons + skb_queue_len(list));
1180			kfree_skb(nskb);
1181			return -ENOENT;
1182		}
1183
1184		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
1185				skb_frag_page(nfrag),
1186				rx.offset, rx.status, PAGE_SIZE);
1187
1188		skb_shinfo(nskb)->nr_frags = 0;
1189		kfree_skb(nskb);
1190	}
1191
1192	xennet_set_rx_rsp_cons(queue, cons);
1193
1194	return 0;
1195}
1196
1197static int checksum_setup(struct net_device *dev, struct sk_buff *skb)
1198{
1199	bool recalculate_partial_csum = false;
1200
1201	/*
1202	 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1203	 * peers can fail to set NETRXF_csum_blank when sending a GSO
1204	 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1205	 * recalculate the partial checksum.
1206	 */
1207	if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) {
1208		struct netfront_info *np = netdev_priv(dev);
1209		atomic_inc(&np->rx_gso_checksum_fixup);
1210		skb->ip_summed = CHECKSUM_PARTIAL;
1211		recalculate_partial_csum = true;
1212	}
1213
1214	/* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1215	if (skb->ip_summed != CHECKSUM_PARTIAL)
1216		return 0;
1217
1218	return skb_checksum_setup(skb, recalculate_partial_csum);
1219}
1220
1221static int handle_incoming_queue(struct netfront_queue *queue,
1222				 struct sk_buff_head *rxq)
1223{
1224	struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats);
 
1225	int packets_dropped = 0;
1226	struct sk_buff *skb;
1227
1228	while ((skb = __skb_dequeue(rxq)) != NULL) {
1229		int pull_to = NETFRONT_SKB_CB(skb)->pull_to;
1230
1231		if (pull_to > skb_headlen(skb))
1232			__pskb_pull_tail(skb, pull_to - skb_headlen(skb));
1233
1234		/* Ethernet work: Delayed to here as it peeks the header. */
1235		skb->protocol = eth_type_trans(skb, queue->info->netdev);
1236		skb_reset_network_header(skb);
1237
1238		if (checksum_setup(queue->info->netdev, skb)) {
1239			kfree_skb(skb);
1240			packets_dropped++;
1241			queue->info->netdev->stats.rx_errors++;
1242			continue;
1243		}
1244
1245		u64_stats_update_begin(&rx_stats->syncp);
1246		rx_stats->packets++;
1247		rx_stats->bytes += skb->len;
1248		u64_stats_update_end(&rx_stats->syncp);
1249
1250		/* Pass it up. */
1251		napi_gro_receive(&queue->napi, skb);
1252	}
1253
1254	return packets_dropped;
1255}
1256
1257static int xennet_poll(struct napi_struct *napi, int budget)
1258{
1259	struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi);
1260	struct net_device *dev = queue->info->netdev;
1261	struct sk_buff *skb;
1262	struct netfront_rx_info rinfo;
1263	struct xen_netif_rx_response *rx = &rinfo.rx;
1264	struct xen_netif_extra_info *extras = rinfo.extras;
1265	RING_IDX i, rp;
1266	int work_done;
1267	struct sk_buff_head rxq;
1268	struct sk_buff_head errq;
1269	struct sk_buff_head tmpq;
 
1270	int err;
1271	bool need_xdp_flush = false;
1272
1273	spin_lock(&queue->rx_lock);
1274
1275	skb_queue_head_init(&rxq);
1276	skb_queue_head_init(&errq);
1277	skb_queue_head_init(&tmpq);
1278
1279	rp = queue->rx.sring->rsp_prod;
1280	if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) {
1281		dev_alert(&dev->dev, "Illegal number of responses %u\n",
1282			  rp - queue->rx.rsp_cons);
1283		queue->info->broken = true;
1284		spin_unlock(&queue->rx_lock);
1285		return 0;
1286	}
1287	rmb(); /* Ensure we see queued responses up to 'rp'. */
1288
1289	i = queue->rx.rsp_cons;
1290	work_done = 0;
1291	while ((i != rp) && (work_done < budget)) {
1292		RING_COPY_RESPONSE(&queue->rx, i, rx);
1293		memset(extras, 0, sizeof(rinfo.extras));
1294
1295		err = xennet_get_responses(queue, &rinfo, rp, &tmpq,
1296					   &need_xdp_flush);
1297
1298		if (unlikely(err)) {
1299			if (queue->info->broken) {
1300				spin_unlock(&queue->rx_lock);
1301				return 0;
1302			}
1303err:
1304			while ((skb = __skb_dequeue(&tmpq)))
1305				__skb_queue_tail(&errq, skb);
1306			dev->stats.rx_errors++;
1307			i = queue->rx.rsp_cons;
1308			continue;
1309		}
1310
1311		skb = __skb_dequeue(&tmpq);
1312
1313		if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
1314			struct xen_netif_extra_info *gso;
1315			gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1316
1317			if (unlikely(xennet_set_skb_gso(skb, gso))) {
1318				__skb_queue_head(&tmpq, skb);
1319				xennet_set_rx_rsp_cons(queue,
1320						       queue->rx.rsp_cons +
1321						       skb_queue_len(&tmpq));
1322				goto err;
1323			}
1324		}
1325
1326		NETFRONT_SKB_CB(skb)->pull_to = rx->status;
1327		if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD)
1328			NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD;
1329
1330		skb_frag_off_set(&skb_shinfo(skb)->frags[0], rx->offset);
1331		skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status);
1332		skb->data_len = rx->status;
1333		skb->len += rx->status;
1334
1335		if (unlikely(xennet_fill_frags(queue, skb, &tmpq)))
1336			goto err;
1337
1338		if (rx->flags & XEN_NETRXF_csum_blank)
1339			skb->ip_summed = CHECKSUM_PARTIAL;
1340		else if (rx->flags & XEN_NETRXF_data_validated)
1341			skb->ip_summed = CHECKSUM_UNNECESSARY;
1342
1343		__skb_queue_tail(&rxq, skb);
1344
1345		i = queue->rx.rsp_cons + 1;
1346		xennet_set_rx_rsp_cons(queue, i);
1347		work_done++;
1348	}
1349	if (need_xdp_flush)
1350		xdp_do_flush();
1351
1352	__skb_queue_purge(&errq);
1353
1354	work_done -= handle_incoming_queue(queue, &rxq);
 
 
 
 
 
 
 
1355
1356	xennet_alloc_rx_buffers(queue);
1357
1358	if (work_done < budget) {
1359		int more_to_do = 0;
1360
1361		napi_complete_done(napi, work_done);
1362
1363		RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do);
1364		if (more_to_do)
1365			napi_schedule(napi);
 
 
 
 
1366	}
1367
1368	spin_unlock(&queue->rx_lock);
1369
1370	return work_done;
1371}
1372
1373static int xennet_change_mtu(struct net_device *dev, int mtu)
1374{
1375	int max = xennet_can_sg(dev) ? XEN_NETIF_MAX_TX_SIZE : ETH_DATA_LEN;
 
1376
1377	if (mtu > max)
1378		return -EINVAL;
1379	dev->mtu = mtu;
1380	return 0;
1381}
1382
1383static void xennet_get_stats64(struct net_device *dev,
1384			       struct rtnl_link_stats64 *tot)
1385{
1386	struct netfront_info *np = netdev_priv(dev);
1387	int cpu;
1388
1389	for_each_possible_cpu(cpu) {
1390		struct netfront_stats *rx_stats = per_cpu_ptr(np->rx_stats, cpu);
1391		struct netfront_stats *tx_stats = per_cpu_ptr(np->tx_stats, cpu);
1392		u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
1393		unsigned int start;
1394
1395		do {
1396			start = u64_stats_fetch_begin(&tx_stats->syncp);
1397			tx_packets = tx_stats->packets;
1398			tx_bytes = tx_stats->bytes;
1399		} while (u64_stats_fetch_retry(&tx_stats->syncp, start));
1400
1401		do {
1402			start = u64_stats_fetch_begin(&rx_stats->syncp);
1403			rx_packets = rx_stats->packets;
1404			rx_bytes = rx_stats->bytes;
1405		} while (u64_stats_fetch_retry(&rx_stats->syncp, start));
1406
1407		tot->rx_packets += rx_packets;
1408		tot->tx_packets += tx_packets;
1409		tot->rx_bytes   += rx_bytes;
1410		tot->tx_bytes   += tx_bytes;
1411	}
1412
1413	tot->rx_errors  = dev->stats.rx_errors;
1414	tot->tx_dropped = dev->stats.tx_dropped;
 
 
1415}
1416
1417static void xennet_release_tx_bufs(struct netfront_queue *queue)
1418{
1419	struct sk_buff *skb;
1420	int i;
1421
1422	for (i = 0; i < NET_TX_RING_SIZE; i++) {
1423		/* Skip over entries which are actually freelist references */
1424		if (!queue->tx_skbs[i])
1425			continue;
1426
1427		skb = queue->tx_skbs[i];
1428		queue->tx_skbs[i] = NULL;
1429		get_page(queue->grant_tx_page[i]);
1430		gnttab_end_foreign_access(queue->grant_tx_ref[i],
1431					  queue->grant_tx_page[i]);
1432		queue->grant_tx_page[i] = NULL;
1433		queue->grant_tx_ref[i] = INVALID_GRANT_REF;
1434		add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, i);
1435		dev_kfree_skb_irq(skb);
1436	}
1437}
1438
1439static void xennet_release_rx_bufs(struct netfront_queue *queue)
1440{
1441	int id, ref;
1442
1443	spin_lock_bh(&queue->rx_lock);
1444
1445	for (id = 0; id < NET_RX_RING_SIZE; id++) {
1446		struct sk_buff *skb;
1447		struct page *page;
1448
1449		skb = queue->rx_skbs[id];
1450		if (!skb)
1451			continue;
1452
1453		ref = queue->grant_rx_ref[id];
1454		if (ref == INVALID_GRANT_REF)
1455			continue;
1456
1457		page = skb_frag_page(&skb_shinfo(skb)->frags[0]);
1458
1459		/* gnttab_end_foreign_access() needs a page ref until
1460		 * foreign access is ended (which may be deferred).
1461		 */
1462		get_page(page);
1463		gnttab_end_foreign_access(ref, page);
1464		queue->grant_rx_ref[id] = INVALID_GRANT_REF;
 
1465
1466		kfree_skb(skb);
1467	}
1468
1469	spin_unlock_bh(&queue->rx_lock);
 
 
 
 
 
 
 
 
 
1470}
1471
1472static netdev_features_t xennet_fix_features(struct net_device *dev,
1473	netdev_features_t features)
1474{
1475	struct netfront_info *np = netdev_priv(dev);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1476
1477	if (features & NETIF_F_SG &&
1478	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0))
1479		features &= ~NETIF_F_SG;
1480
1481	if (features & NETIF_F_IPV6_CSUM &&
1482	    !xenbus_read_unsigned(np->xbdev->otherend,
1483				  "feature-ipv6-csum-offload", 0))
1484		features &= ~NETIF_F_IPV6_CSUM;
1485
1486	if (features & NETIF_F_TSO &&
1487	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0))
1488		features &= ~NETIF_F_TSO;
1489
1490	if (features & NETIF_F_TSO6 &&
1491	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0))
1492		features &= ~NETIF_F_TSO6;
 
1493
1494	return features;
1495}
1496
1497static int xennet_set_features(struct net_device *dev,
1498	netdev_features_t features)
1499{
1500	if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) {
1501		netdev_info(dev, "Reducing MTU because no SG offload");
1502		dev->mtu = ETH_DATA_LEN;
1503	}
1504
1505	return 0;
1506}
1507
1508static bool xennet_handle_tx(struct netfront_queue *queue, unsigned int *eoi)
1509{
 
 
1510	unsigned long flags;
1511
1512	if (unlikely(queue->info->broken))
1513		return false;
1514
1515	spin_lock_irqsave(&queue->tx_lock, flags);
1516	if (xennet_tx_buf_gc(queue))
1517		*eoi = 0;
1518	spin_unlock_irqrestore(&queue->tx_lock, flags);
1519
1520	return true;
1521}
1522
1523static irqreturn_t xennet_tx_interrupt(int irq, void *dev_id)
1524{
1525	unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
1526
1527	if (likely(xennet_handle_tx(dev_id, &eoiflag)))
1528		xen_irq_lateeoi(irq, eoiflag);
1529
1530	return IRQ_HANDLED;
1531}
1532
1533static bool xennet_handle_rx(struct netfront_queue *queue, unsigned int *eoi)
1534{
1535	unsigned int work_queued;
1536	unsigned long flags;
1537
1538	if (unlikely(queue->info->broken))
1539		return false;
1540
1541	spin_lock_irqsave(&queue->rx_cons_lock, flags);
1542	work_queued = XEN_RING_NR_UNCONSUMED_RESPONSES(&queue->rx);
1543	if (work_queued > queue->rx_rsp_unconsumed) {
1544		queue->rx_rsp_unconsumed = work_queued;
1545		*eoi = 0;
1546	} else if (unlikely(work_queued < queue->rx_rsp_unconsumed)) {
1547		const struct device *dev = &queue->info->netdev->dev;
1548
1549		spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
1550		dev_alert(dev, "RX producer index going backwards\n");
1551		dev_alert(dev, "Disabled for further use\n");
1552		queue->info->broken = true;
1553		return false;
1554	}
1555	spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
1556
1557	if (likely(netif_carrier_ok(queue->info->netdev) && work_queued))
1558		napi_schedule(&queue->napi);
1559
1560	return true;
1561}
1562
1563static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
1564{
1565	unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
 
1566
1567	if (likely(xennet_handle_rx(dev_id, &eoiflag)))
1568		xen_irq_lateeoi(irq, eoiflag);
 
1569
1570	return IRQ_HANDLED;
1571}
1572
1573static irqreturn_t xennet_interrupt(int irq, void *dev_id)
1574{
1575	unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
1576
1577	if (xennet_handle_tx(dev_id, &eoiflag) &&
1578	    xennet_handle_rx(dev_id, &eoiflag))
1579		xen_irq_lateeoi(irq, eoiflag);
1580
1581	return IRQ_HANDLED;
1582}
1583
1584#ifdef CONFIG_NET_POLL_CONTROLLER
1585static void xennet_poll_controller(struct net_device *dev)
1586{
1587	/* Poll each queue */
1588	struct netfront_info *info = netdev_priv(dev);
1589	unsigned int num_queues = dev->real_num_tx_queues;
1590	unsigned int i;
1591
1592	if (info->broken)
1593		return;
1594
1595	for (i = 0; i < num_queues; ++i)
1596		xennet_interrupt(0, &info->queues[i]);
1597}
1598#endif
1599
1600#define NETBACK_XDP_HEADROOM_DISABLE	0
1601#define NETBACK_XDP_HEADROOM_ENABLE	1
1602
1603static int talk_to_netback_xdp(struct netfront_info *np, int xdp)
1604{
1605	int err;
1606	unsigned short headroom;
1607
1608	headroom = xdp ? XDP_PACKET_HEADROOM : 0;
1609	err = xenbus_printf(XBT_NIL, np->xbdev->nodename,
1610			    "xdp-headroom", "%hu",
1611			    headroom);
1612	if (err)
1613		pr_warn("Error writing xdp-headroom\n");
1614
1615	return err;
1616}
1617
1618static int xennet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1619			  struct netlink_ext_ack *extack)
1620{
1621	unsigned long max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM;
1622	struct netfront_info *np = netdev_priv(dev);
1623	struct bpf_prog *old_prog;
1624	unsigned int i, err;
1625
1626	if (dev->mtu > max_mtu) {
1627		netdev_warn(dev, "XDP requires MTU less than %lu\n", max_mtu);
1628		return -EINVAL;
1629	}
1630
1631	if (!np->netback_has_xdp_headroom)
1632		return 0;
1633
1634	xenbus_switch_state(np->xbdev, XenbusStateReconfiguring);
1635
1636	err = talk_to_netback_xdp(np, prog ? NETBACK_XDP_HEADROOM_ENABLE :
1637				  NETBACK_XDP_HEADROOM_DISABLE);
1638	if (err)
1639		return err;
1640
1641	/* avoid the race with XDP headroom adjustment */
1642	wait_event(module_wq,
1643		   xenbus_read_driver_state(np->xbdev->otherend) ==
1644		   XenbusStateReconfigured);
1645	np->netfront_xdp_enabled = true;
1646
1647	old_prog = rtnl_dereference(np->queues[0].xdp_prog);
1648
1649	if (prog)
1650		bpf_prog_add(prog, dev->real_num_tx_queues);
1651
1652	for (i = 0; i < dev->real_num_tx_queues; ++i)
1653		rcu_assign_pointer(np->queues[i].xdp_prog, prog);
1654
1655	if (old_prog)
1656		for (i = 0; i < dev->real_num_tx_queues; ++i)
1657			bpf_prog_put(old_prog);
1658
1659	xenbus_switch_state(np->xbdev, XenbusStateConnected);
1660
1661	return 0;
1662}
1663
1664static int xennet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1665{
1666	struct netfront_info *np = netdev_priv(dev);
1667
1668	if (np->broken)
1669		return -ENODEV;
1670
1671	switch (xdp->command) {
1672	case XDP_SETUP_PROG:
1673		return xennet_xdp_set(dev, xdp->prog, xdp->extack);
1674	default:
1675		return -EINVAL;
1676	}
1677}
1678
1679static const struct net_device_ops xennet_netdev_ops = {
 
1680	.ndo_uninit          = xennet_uninit,
1681	.ndo_open            = xennet_open,
1682	.ndo_stop            = xennet_close,
1683	.ndo_start_xmit      = xennet_start_xmit,
1684	.ndo_change_mtu	     = xennet_change_mtu,
1685	.ndo_get_stats64     = xennet_get_stats64,
1686	.ndo_set_mac_address = eth_mac_addr,
1687	.ndo_validate_addr   = eth_validate_addr,
1688	.ndo_fix_features    = xennet_fix_features,
1689	.ndo_set_features    = xennet_set_features,
1690	.ndo_select_queue    = xennet_select_queue,
1691	.ndo_bpf            = xennet_xdp,
1692	.ndo_xdp_xmit	    = xennet_xdp_xmit,
1693#ifdef CONFIG_NET_POLL_CONTROLLER
1694	.ndo_poll_controller = xennet_poll_controller,
1695#endif
1696};
1697
1698static void xennet_free_netdev(struct net_device *netdev)
1699{
1700	struct netfront_info *np = netdev_priv(netdev);
1701
1702	free_percpu(np->rx_stats);
1703	free_percpu(np->tx_stats);
1704	free_netdev(netdev);
1705}
1706
1707static struct net_device *xennet_create_dev(struct xenbus_device *dev)
1708{
1709	int err;
1710	struct net_device *netdev;
1711	struct netfront_info *np;
1712
1713	netdev = alloc_etherdev_mq(sizeof(struct netfront_info), xennet_max_queues);
1714	if (!netdev)
1715		return ERR_PTR(-ENOMEM);
1716
1717	np                   = netdev_priv(netdev);
1718	np->xbdev            = dev;
1719
1720	np->queues = NULL;
 
 
 
 
 
 
 
 
 
 
1721
1722	err = -ENOMEM;
1723	np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats);
1724	if (np->rx_stats == NULL)
1725		goto exit;
1726	np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats);
1727	if (np->tx_stats == NULL)
1728		goto exit;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1729
1730	netdev->netdev_ops	= &xennet_netdev_ops;
1731
 
1732	netdev->features        = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
1733				  NETIF_F_GSO_ROBUST;
1734	netdev->hw_features	= NETIF_F_SG |
1735				  NETIF_F_IPV6_CSUM |
1736				  NETIF_F_TSO | NETIF_F_TSO6;
1737
1738	/*
1739         * Assume that all hw features are available for now. This set
1740         * will be adjusted by the call to netdev_update_features() in
1741         * xennet_connect() which is the earliest point where we can
1742         * negotiate with the backend regarding supported features.
1743         */
1744	netdev->features |= netdev->hw_features;
1745	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
1746			       NETDEV_XDP_ACT_NDO_XMIT;
1747
1748	netdev->ethtool_ops = &xennet_ethtool_ops;
1749	netdev->min_mtu = ETH_MIN_MTU;
1750	netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE;
1751	SET_NETDEV_DEV(netdev, &dev->dev);
1752
 
 
1753	np->netdev = netdev;
1754	np->netfront_xdp_enabled = false;
1755
1756	netif_carrier_off(netdev);
1757
1758	do {
1759		xenbus_switch_state(dev, XenbusStateInitialising);
1760		err = wait_event_timeout(module_wq,
1761				 xenbus_read_driver_state(dev->otherend) !=
1762				 XenbusStateClosed &&
1763				 xenbus_read_driver_state(dev->otherend) !=
1764				 XenbusStateUnknown, XENNET_TIMEOUT);
1765	} while (!err);
1766
1767	return netdev;
1768
 
 
 
 
1769 exit:
1770	xennet_free_netdev(netdev);
1771	return ERR_PTR(err);
1772}
1773
1774/*
1775 * Entry point to this code when a new device is created.  Allocate the basic
1776 * structures and the ring buffers for communication with the backend, and
1777 * inform the backend of the appropriate details for those.
1778 */
1779static int netfront_probe(struct xenbus_device *dev,
1780			  const struct xenbus_device_id *id)
1781{
1782	int err;
1783	struct net_device *netdev;
1784	struct netfront_info *info;
1785
1786	netdev = xennet_create_dev(dev);
1787	if (IS_ERR(netdev)) {
1788		err = PTR_ERR(netdev);
1789		xenbus_dev_fatal(dev, err, "creating netdev");
1790		return err;
1791	}
1792
1793	info = netdev_priv(netdev);
1794	dev_set_drvdata(&dev->dev, info);
1795#ifdef CONFIG_SYSFS
1796	info->netdev->sysfs_groups[0] = &xennet_dev_group;
1797#endif
 
 
 
 
 
 
 
 
 
 
1798
1799	return 0;
 
 
 
 
 
1800}
1801
1802static void xennet_end_access(int ref, void *page)
1803{
1804	/* This frees the page as a side-effect */
1805	if (ref != INVALID_GRANT_REF)
1806		gnttab_end_foreign_access(ref, virt_to_page(page));
1807}
1808
1809static void xennet_disconnect_backend(struct netfront_info *info)
1810{
1811	unsigned int i = 0;
1812	unsigned int num_queues = info->netdev->real_num_tx_queues;
1813
1814	netif_carrier_off(info->netdev);
 
 
1815
1816	for (i = 0; i < num_queues && info->queues; ++i) {
1817		struct netfront_queue *queue = &info->queues[i];
1818
1819		del_timer_sync(&queue->rx_refill_timer);
1820
1821		if (queue->tx_irq && (queue->tx_irq == queue->rx_irq))
1822			unbind_from_irqhandler(queue->tx_irq, queue);
1823		if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) {
1824			unbind_from_irqhandler(queue->tx_irq, queue);
1825			unbind_from_irqhandler(queue->rx_irq, queue);
1826		}
1827		queue->tx_evtchn = queue->rx_evtchn = 0;
1828		queue->tx_irq = queue->rx_irq = 0;
1829
1830		if (netif_running(info->netdev))
1831			napi_synchronize(&queue->napi);
1832
1833		xennet_release_tx_bufs(queue);
1834		xennet_release_rx_bufs(queue);
1835		gnttab_free_grant_references(queue->gref_tx_head);
1836		gnttab_free_grant_references(queue->gref_rx_head);
1837
1838		/* End access and free the pages */
1839		xennet_end_access(queue->tx_ring_ref, queue->tx.sring);
1840		xennet_end_access(queue->rx_ring_ref, queue->rx.sring);
1841
1842		queue->tx_ring_ref = INVALID_GRANT_REF;
1843		queue->rx_ring_ref = INVALID_GRANT_REF;
1844		queue->tx.sring = NULL;
1845		queue->rx.sring = NULL;
1846
1847		page_pool_destroy(queue->page_pool);
1848	}
 
 
1849}
1850
1851/*
1852 * We are reconnecting to the backend, due to a suspend/resume, or a backend
1853 * driver restart.  We tear down our netif structure and recreate it, but
1854 * leave the device-layer structures intact so that this is transparent to the
1855 * rest of the kernel.
1856 */
1857static int netfront_resume(struct xenbus_device *dev)
1858{
1859	struct netfront_info *info = dev_get_drvdata(&dev->dev);
1860
1861	dev_dbg(&dev->dev, "%s\n", dev->nodename);
1862
1863	netif_tx_lock_bh(info->netdev);
1864	netif_device_detach(info->netdev);
1865	netif_tx_unlock_bh(info->netdev);
1866
1867	xennet_disconnect_backend(info);
1868
1869	rtnl_lock();
1870	if (info->queues)
1871		xennet_destroy_queues(info);
1872	rtnl_unlock();
1873
1874	return 0;
1875}
1876
1877static int xen_net_read_mac(struct xenbus_device *dev, u8 mac[])
1878{
1879	char *s, *e, *macstr;
1880	int i;
1881
1882	macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL);
1883	if (IS_ERR(macstr))
1884		return PTR_ERR(macstr);
1885
1886	for (i = 0; i < ETH_ALEN; i++) {
1887		mac[i] = simple_strtoul(s, &e, 16);
1888		if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) {
1889			kfree(macstr);
1890			return -ENOENT;
1891		}
1892		s = e+1;
1893	}
1894
1895	kfree(macstr);
1896	return 0;
1897}
1898
1899static int setup_netfront_single(struct netfront_queue *queue)
1900{
1901	int err;
1902
1903	err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn);
1904	if (err < 0)
1905		goto fail;
1906
1907	err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn,
1908						xennet_interrupt, 0,
1909						queue->info->netdev->name,
1910						queue);
1911	if (err < 0)
1912		goto bind_fail;
1913	queue->rx_evtchn = queue->tx_evtchn;
1914	queue->rx_irq = queue->tx_irq = err;
1915
1916	return 0;
1917
1918bind_fail:
1919	xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn);
1920	queue->tx_evtchn = 0;
1921fail:
1922	return err;
1923}
1924
1925static int setup_netfront_split(struct netfront_queue *queue)
1926{
1927	int err;
1928
1929	err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn);
1930	if (err < 0)
1931		goto fail;
1932	err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn);
1933	if (err < 0)
1934		goto alloc_rx_evtchn_fail;
1935
1936	snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name),
1937		 "%s-tx", queue->name);
1938	err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn,
1939						xennet_tx_interrupt, 0,
1940						queue->tx_irq_name, queue);
1941	if (err < 0)
1942		goto bind_tx_fail;
1943	queue->tx_irq = err;
1944
1945	snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name),
1946		 "%s-rx", queue->name);
1947	err = bind_evtchn_to_irqhandler_lateeoi(queue->rx_evtchn,
1948						xennet_rx_interrupt, 0,
1949						queue->rx_irq_name, queue);
1950	if (err < 0)
1951		goto bind_rx_fail;
1952	queue->rx_irq = err;
1953
1954	return 0;
1955
1956bind_rx_fail:
1957	unbind_from_irqhandler(queue->tx_irq, queue);
1958	queue->tx_irq = 0;
1959bind_tx_fail:
1960	xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn);
1961	queue->rx_evtchn = 0;
1962alloc_rx_evtchn_fail:
1963	xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn);
1964	queue->tx_evtchn = 0;
1965fail:
1966	return err;
1967}
1968
1969static int setup_netfront(struct xenbus_device *dev,
1970			struct netfront_queue *queue, unsigned int feature_split_evtchn)
1971{
1972	struct xen_netif_tx_sring *txs;
1973	struct xen_netif_rx_sring *rxs;
1974	int err;
 
 
1975
1976	queue->tx_ring_ref = INVALID_GRANT_REF;
1977	queue->rx_ring_ref = INVALID_GRANT_REF;
1978	queue->rx.sring = NULL;
1979	queue->tx.sring = NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
1980
1981	err = xenbus_setup_ring(dev, GFP_NOIO | __GFP_HIGH, (void **)&txs,
1982				1, &queue->tx_ring_ref);
1983	if (err)
 
1984		goto fail;
 
 
 
1985
1986	XEN_FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE);
 
 
1987
1988	err = xenbus_setup_ring(dev, GFP_NOIO | __GFP_HIGH, (void **)&rxs,
1989				1, &queue->rx_ring_ref);
1990	if (err)
1991		goto fail;
 
 
 
 
 
1992
1993	XEN_FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE);
 
 
 
1994
1995	if (feature_split_evtchn)
1996		err = setup_netfront_split(queue);
1997	/* setup single event channel if
1998	 *  a) feature-split-event-channels == 0
1999	 *  b) feature-split-event-channels == 1 but failed to setup
2000	 */
2001	if (!feature_split_evtchn || err)
2002		err = setup_netfront_single(queue);
2003
2004	if (err)
2005		goto fail;
2006
2007	return 0;
2008
2009 fail:
2010	xenbus_teardown_ring((void **)&queue->rx.sring, 1, &queue->rx_ring_ref);
2011	xenbus_teardown_ring((void **)&queue->tx.sring, 1, &queue->tx_ring_ref);
2012
2013	return err;
2014}
2015
2016/* Queue-specific initialisation
2017 * This used to be done in xennet_create_dev() but must now
2018 * be run per-queue.
2019 */
2020static int xennet_init_queue(struct netfront_queue *queue)
2021{
2022	unsigned short i;
2023	int err = 0;
2024	char *devid;
2025
2026	spin_lock_init(&queue->tx_lock);
2027	spin_lock_init(&queue->rx_lock);
2028	spin_lock_init(&queue->rx_cons_lock);
2029
2030	timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0);
2031
2032	devid = strrchr(queue->info->xbdev->nodename, '/') + 1;
2033	snprintf(queue->name, sizeof(queue->name), "vif%s-q%u",
2034		 devid, queue->id);
2035
2036	/* Initialise tx_skb_freelist as a free chain containing every entry. */
2037	queue->tx_skb_freelist = 0;
2038	queue->tx_pend_queue = TX_LINK_NONE;
2039	for (i = 0; i < NET_TX_RING_SIZE; i++) {
2040		queue->tx_link[i] = i + 1;
2041		queue->grant_tx_ref[i] = INVALID_GRANT_REF;
2042		queue->grant_tx_page[i] = NULL;
2043	}
2044	queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE;
2045
2046	/* Clear out rx_skbs */
2047	for (i = 0; i < NET_RX_RING_SIZE; i++) {
2048		queue->rx_skbs[i] = NULL;
2049		queue->grant_rx_ref[i] = INVALID_GRANT_REF;
2050	}
2051
2052	/* A grant for every tx ring slot */
2053	if (gnttab_alloc_grant_references(NET_TX_RING_SIZE,
2054					  &queue->gref_tx_head) < 0) {
2055		pr_alert("can't alloc tx grant refs\n");
2056		err = -ENOMEM;
2057		goto exit;
2058	}
2059
2060	/* A grant for every rx ring slot */
2061	if (gnttab_alloc_grant_references(NET_RX_RING_SIZE,
2062					  &queue->gref_rx_head) < 0) {
2063		pr_alert("can't alloc rx grant refs\n");
2064		err = -ENOMEM;
2065		goto exit_free_tx;
2066	}
2067
2068	return 0;
2069
2070 exit_free_tx:
2071	gnttab_free_grant_references(queue->gref_tx_head);
2072 exit:
2073	return err;
2074}
2075
2076static int write_queue_xenstore_keys(struct netfront_queue *queue,
2077			   struct xenbus_transaction *xbt, int write_hierarchical)
2078{
2079	/* Write the queue-specific keys into XenStore in the traditional
2080	 * way for a single queue, or in a queue subkeys for multiple
2081	 * queues.
2082	 */
2083	struct xenbus_device *dev = queue->info->xbdev;
2084	int err;
2085	const char *message;
2086	char *path;
2087	size_t pathsize;
2088
2089	/* Choose the correct place to write the keys */
2090	if (write_hierarchical) {
2091		pathsize = strlen(dev->nodename) + 10;
2092		path = kzalloc(pathsize, GFP_KERNEL);
2093		if (!path) {
2094			err = -ENOMEM;
2095			message = "out of memory while writing ring references";
2096			goto error;
2097		}
2098		snprintf(path, pathsize, "%s/queue-%u",
2099				dev->nodename, queue->id);
2100	} else {
2101		path = (char *)dev->nodename;
2102	}
2103
2104	/* Write ring references */
2105	err = xenbus_printf(*xbt, path, "tx-ring-ref", "%u",
2106			queue->tx_ring_ref);
2107	if (err) {
2108		message = "writing tx-ring-ref";
2109		goto error;
2110	}
2111
2112	err = xenbus_printf(*xbt, path, "rx-ring-ref", "%u",
2113			queue->rx_ring_ref);
2114	if (err) {
2115		message = "writing rx-ring-ref";
2116		goto error;
2117	}
2118
2119	/* Write event channels; taking into account both shared
2120	 * and split event channel scenarios.
2121	 */
2122	if (queue->tx_evtchn == queue->rx_evtchn) {
2123		/* Shared event channel */
2124		err = xenbus_printf(*xbt, path,
2125				"event-channel", "%u", queue->tx_evtchn);
2126		if (err) {
2127			message = "writing event-channel";
2128			goto error;
2129		}
2130	} else {
2131		/* Split event channels */
2132		err = xenbus_printf(*xbt, path,
2133				"event-channel-tx", "%u", queue->tx_evtchn);
2134		if (err) {
2135			message = "writing event-channel-tx";
2136			goto error;
2137		}
2138
2139		err = xenbus_printf(*xbt, path,
2140				"event-channel-rx", "%u", queue->rx_evtchn);
2141		if (err) {
2142			message = "writing event-channel-rx";
2143			goto error;
2144		}
2145	}
2146
2147	if (write_hierarchical)
2148		kfree(path);
2149	return 0;
2150
2151error:
2152	if (write_hierarchical)
2153		kfree(path);
2154	xenbus_dev_fatal(dev, err, "%s", message);
2155	return err;
2156}
2157
2158
2159
2160static int xennet_create_page_pool(struct netfront_queue *queue)
2161{
2162	int err;
2163	struct page_pool_params pp_params = {
2164		.order = 0,
2165		.flags = 0,
2166		.pool_size = NET_RX_RING_SIZE,
2167		.nid = NUMA_NO_NODE,
2168		.dev = &queue->info->netdev->dev,
2169		.offset = XDP_PACKET_HEADROOM,
2170		.max_len = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM,
2171	};
2172
2173	queue->page_pool = page_pool_create(&pp_params);
2174	if (IS_ERR(queue->page_pool)) {
2175		err = PTR_ERR(queue->page_pool);
2176		queue->page_pool = NULL;
2177		return err;
2178	}
2179
2180	err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev,
2181			       queue->id, 0);
2182	if (err) {
2183		netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n");
2184		goto err_free_pp;
2185	}
2186
2187	err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq,
2188					 MEM_TYPE_PAGE_POOL, queue->page_pool);
2189	if (err) {
2190		netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n");
2191		goto err_unregister_rxq;
2192	}
2193	return 0;
2194
2195err_unregister_rxq:
2196	xdp_rxq_info_unreg(&queue->xdp_rxq);
2197err_free_pp:
2198	page_pool_destroy(queue->page_pool);
2199	queue->page_pool = NULL;
2200	return err;
2201}
2202
2203static int xennet_create_queues(struct netfront_info *info,
2204				unsigned int *num_queues)
2205{
2206	unsigned int i;
2207	int ret;
2208
2209	info->queues = kcalloc(*num_queues, sizeof(struct netfront_queue),
2210			       GFP_KERNEL);
2211	if (!info->queues)
2212		return -ENOMEM;
2213
2214	for (i = 0; i < *num_queues; i++) {
2215		struct netfront_queue *queue = &info->queues[i];
2216
2217		queue->id = i;
2218		queue->info = info;
2219
2220		ret = xennet_init_queue(queue);
2221		if (ret < 0) {
2222			dev_warn(&info->xbdev->dev,
2223				 "only created %d queues\n", i);
2224			*num_queues = i;
2225			break;
2226		}
2227
2228		/* use page pool recycling instead of buddy allocator */
2229		ret = xennet_create_page_pool(queue);
2230		if (ret < 0) {
2231			dev_err(&info->xbdev->dev, "can't allocate page pool\n");
2232			*num_queues = i;
2233			return ret;
2234		}
2235
2236		netif_napi_add(queue->info->netdev, &queue->napi, xennet_poll);
2237		if (netif_running(info->netdev))
2238			napi_enable(&queue->napi);
2239	}
2240
2241	netif_set_real_num_tx_queues(info->netdev, *num_queues);
2242
2243	if (*num_queues == 0) {
2244		dev_err(&info->xbdev->dev, "no queues\n");
2245		return -EINVAL;
2246	}
2247	return 0;
2248}
2249
2250/* Common code used when first setting up, and when resuming. */
2251static int talk_to_netback(struct xenbus_device *dev,
2252			   struct netfront_info *info)
2253{
2254	const char *message;
2255	struct xenbus_transaction xbt;
2256	int err;
2257	unsigned int feature_split_evtchn;
2258	unsigned int i = 0;
2259	unsigned int max_queues = 0;
2260	struct netfront_queue *queue = NULL;
2261	unsigned int num_queues = 1;
2262	u8 addr[ETH_ALEN];
2263
2264	info->netdev->irq = 0;
2265
2266	/* Check if backend is trusted. */
2267	info->bounce = !xennet_trusted ||
2268		       !xenbus_read_unsigned(dev->nodename, "trusted", 1);
2269
2270	/* Check if backend supports multiple queues */
2271	max_queues = xenbus_read_unsigned(info->xbdev->otherend,
2272					  "multi-queue-max-queues", 1);
2273	num_queues = min(max_queues, xennet_max_queues);
2274
2275	/* Check feature-split-event-channels */
2276	feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend,
2277					"feature-split-event-channels", 0);
2278
2279	/* Read mac addr. */
2280	err = xen_net_read_mac(dev, addr);
2281	if (err) {
2282		xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename);
2283		goto out_unlocked;
2284	}
2285	eth_hw_addr_set(info->netdev, addr);
2286
2287	info->netback_has_xdp_headroom = xenbus_read_unsigned(info->xbdev->otherend,
2288							      "feature-xdp-headroom", 0);
2289	if (info->netback_has_xdp_headroom) {
2290		/* set the current xen-netfront xdp state */
2291		err = talk_to_netback_xdp(info, info->netfront_xdp_enabled ?
2292					  NETBACK_XDP_HEADROOM_ENABLE :
2293					  NETBACK_XDP_HEADROOM_DISABLE);
2294		if (err)
2295			goto out_unlocked;
2296	}
2297
2298	rtnl_lock();
2299	if (info->queues)
2300		xennet_destroy_queues(info);
2301
2302	/* For the case of a reconnect reset the "broken" indicator. */
2303	info->broken = false;
2304
2305	err = xennet_create_queues(info, &num_queues);
2306	if (err < 0) {
2307		xenbus_dev_fatal(dev, err, "creating queues");
2308		kfree(info->queues);
2309		info->queues = NULL;
2310		goto out;
2311	}
2312	rtnl_unlock();
2313
2314	/* Create shared ring, alloc event channel -- for each queue */
2315	for (i = 0; i < num_queues; ++i) {
2316		queue = &info->queues[i];
2317		err = setup_netfront(dev, queue, feature_split_evtchn);
2318		if (err)
2319			goto destroy_ring;
2320	}
2321
2322again:
2323	err = xenbus_transaction_start(&xbt);
2324	if (err) {
2325		xenbus_dev_fatal(dev, err, "starting transaction");
2326		goto destroy_ring;
2327	}
2328
2329	if (xenbus_exists(XBT_NIL,
2330			  info->xbdev->otherend, "multi-queue-max-queues")) {
2331		/* Write the number of queues */
 
 
 
 
 
 
 
 
 
 
 
2332		err = xenbus_printf(xbt, dev->nodename,
2333				    "multi-queue-num-queues", "%u", num_queues);
2334		if (err) {
2335			message = "writing multi-queue-num-queues";
2336			goto abort_transaction_no_dev_fatal;
2337		}
2338	}
2339
2340	if (num_queues == 1) {
2341		err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */
2342		if (err)
2343			goto abort_transaction_no_dev_fatal;
2344	} else {
2345		/* Write the keys for each queue */
2346		for (i = 0; i < num_queues; ++i) {
2347			queue = &info->queues[i];
2348			err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */
2349			if (err)
2350				goto abort_transaction_no_dev_fatal;
 
 
 
 
 
2351		}
2352	}
2353
2354	/* The remaining keys are not queue-specific */
2355	err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u",
2356			    1);
2357	if (err) {
2358		message = "writing request-rx-copy";
2359		goto abort_transaction;
2360	}
2361
2362	err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1);
2363	if (err) {
2364		message = "writing feature-rx-notify";
2365		goto abort_transaction;
2366	}
2367
2368	err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1);
2369	if (err) {
2370		message = "writing feature-sg";
2371		goto abort_transaction;
2372	}
2373
2374	err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1);
2375	if (err) {
2376		message = "writing feature-gso-tcpv4";
2377		goto abort_transaction;
2378	}
2379
2380	err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1");
2381	if (err) {
2382		message = "writing feature-gso-tcpv6";
2383		goto abort_transaction;
2384	}
2385
2386	err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload",
2387			   "1");
2388	if (err) {
2389		message = "writing feature-ipv6-csum-offload";
2390		goto abort_transaction;
2391	}
2392
2393	err = xenbus_transaction_end(xbt, 0);
2394	if (err) {
2395		if (err == -EAGAIN)
2396			goto again;
2397		xenbus_dev_fatal(dev, err, "completing transaction");
2398		goto destroy_ring;
2399	}
2400
2401	return 0;
2402
2403 abort_transaction:
 
2404	xenbus_dev_fatal(dev, err, "%s", message);
2405abort_transaction_no_dev_fatal:
2406	xenbus_transaction_end(xbt, 1);
2407 destroy_ring:
2408	xennet_disconnect_backend(info);
2409	rtnl_lock();
2410	xennet_destroy_queues(info);
2411 out:
2412	rtnl_unlock();
2413out_unlocked:
2414	device_unregister(&dev->dev);
2415	return err;
2416}
2417
2418static int xennet_connect(struct net_device *dev)
2419{
2420	struct netfront_info *np = netdev_priv(dev);
2421	unsigned int num_queues = 0;
2422	int err;
2423	unsigned int j = 0;
2424	struct netfront_queue *queue = NULL;
 
 
 
 
 
 
2425
2426	if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) {
2427		dev_info(&dev->dev,
2428			 "backend does not support copying receive path\n");
2429		return -ENODEV;
2430	}
2431
2432	err = talk_to_netback(np->xbdev, np);
2433	if (err)
2434		return err;
2435	if (np->netback_has_xdp_headroom)
2436		pr_info("backend supports XDP headroom\n");
2437	if (np->bounce)
2438		dev_info(&np->xbdev->dev,
2439			 "bouncing transmitted data to zeroed pages\n");
2440
2441	/* talk_to_netback() sets the correct number of queues */
2442	num_queues = dev->real_num_tx_queues;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2443
2444	if (dev->reg_state == NETREG_UNINITIALIZED) {
2445		err = register_netdev(dev);
2446		if (err) {
2447			pr_warn("%s: register_netdev err=%d\n", __func__, err);
2448			device_unregister(&np->xbdev->dev);
2449			return err;
2450		}
2451	}
2452
2453	rtnl_lock();
2454	netdev_update_features(dev);
2455	rtnl_unlock();
2456
2457	/*
2458	 * All public and private state should now be sane.  Get
2459	 * ready to start sending and receiving packets and give the driver
2460	 * domain a kick because we've probably just requeued some
2461	 * packets.
2462	 */
2463	netif_tx_lock_bh(np->netdev);
2464	netif_device_attach(np->netdev);
2465	netif_tx_unlock_bh(np->netdev);
2466
2467	netif_carrier_on(np->netdev);
2468	for (j = 0; j < num_queues; ++j) {
2469		queue = &np->queues[j];
 
 
 
2470
2471		notify_remote_via_irq(queue->tx_irq);
2472		if (queue->tx_irq != queue->rx_irq)
2473			notify_remote_via_irq(queue->rx_irq);
2474
2475		spin_lock_bh(&queue->rx_lock);
2476		xennet_alloc_rx_buffers(queue);
2477		spin_unlock_bh(&queue->rx_lock);
2478	}
2479
2480	return 0;
2481}
2482
2483/*
2484 * Callback received when the backend's state changes.
2485 */
2486static void netback_changed(struct xenbus_device *dev,
2487			    enum xenbus_state backend_state)
2488{
2489	struct netfront_info *np = dev_get_drvdata(&dev->dev);
2490	struct net_device *netdev = np->netdev;
2491
2492	dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state));
2493
2494	wake_up_all(&module_wq);
2495
2496	switch (backend_state) {
2497	case XenbusStateInitialising:
2498	case XenbusStateInitialised:
2499	case XenbusStateReconfiguring:
2500	case XenbusStateReconfigured:
2501	case XenbusStateUnknown:
2502		break;
2503
2504	case XenbusStateInitWait:
2505		if (dev->state != XenbusStateInitialising)
2506			break;
2507		if (xennet_connect(netdev) != 0)
2508			break;
2509		xenbus_switch_state(dev, XenbusStateConnected);
2510		break;
2511
2512	case XenbusStateConnected:
2513		netdev_notify_peers(netdev);
2514		break;
2515
2516	case XenbusStateClosed:
2517		if (dev->state == XenbusStateClosed)
2518			break;
2519		fallthrough;	/* Missed the backend's CLOSING state */
2520	case XenbusStateClosing:
2521		xenbus_frontend_closed(dev);
2522		break;
2523	}
2524}
2525
2526static const struct xennet_stat {
2527	char name[ETH_GSTRING_LEN];
2528	u16 offset;
2529} xennet_stats[] = {
2530	{
2531		"rx_gso_checksum_fixup",
2532		offsetof(struct netfront_info, rx_gso_checksum_fixup)
2533	},
2534};
2535
2536static int xennet_get_sset_count(struct net_device *dev, int string_set)
2537{
2538	switch (string_set) {
2539	case ETH_SS_STATS:
2540		return ARRAY_SIZE(xennet_stats);
2541	default:
2542		return -EINVAL;
2543	}
2544}
2545
2546static void xennet_get_ethtool_stats(struct net_device *dev,
2547				     struct ethtool_stats *stats, u64 * data)
2548{
2549	void *np = netdev_priv(dev);
2550	int i;
2551
2552	for (i = 0; i < ARRAY_SIZE(xennet_stats); i++)
2553		data[i] = atomic_read((atomic_t *)(np + xennet_stats[i].offset));
2554}
2555
2556static void xennet_get_strings(struct net_device *dev, u32 stringset, u8 * data)
2557{
2558	int i;
2559
2560	switch (stringset) {
2561	case ETH_SS_STATS:
2562		for (i = 0; i < ARRAY_SIZE(xennet_stats); i++)
2563			memcpy(data + i * ETH_GSTRING_LEN,
2564			       xennet_stats[i].name, ETH_GSTRING_LEN);
2565		break;
2566	}
2567}
2568
2569static const struct ethtool_ops xennet_ethtool_ops =
2570{
2571	.get_link = ethtool_op_get_link,
2572
2573	.get_sset_count = xennet_get_sset_count,
2574	.get_ethtool_stats = xennet_get_ethtool_stats,
2575	.get_strings = xennet_get_strings,
2576	.get_ts_info = ethtool_op_get_ts_info,
2577};
2578
2579#ifdef CONFIG_SYSFS
2580static ssize_t show_rxbuf(struct device *dev,
2581			  struct device_attribute *attr, char *buf)
2582{
2583	return sprintf(buf, "%lu\n", NET_RX_RING_SIZE);
 
 
 
2584}
2585
2586static ssize_t store_rxbuf(struct device *dev,
2587			   struct device_attribute *attr,
2588			   const char *buf, size_t len)
2589{
 
 
2590	char *endp;
 
2591
2592	if (!capable(CAP_NET_ADMIN))
2593		return -EPERM;
2594
2595	simple_strtoul(buf, &endp, 0);
2596	if (endp == buf)
2597		return -EBADMSG;
2598
2599	/* rxbuf_min and rxbuf_max are no longer configurable. */
 
 
 
 
 
 
 
 
 
 
 
 
2600
 
2601	return len;
2602}
2603
2604static DEVICE_ATTR(rxbuf_min, 0644, show_rxbuf, store_rxbuf);
2605static DEVICE_ATTR(rxbuf_max, 0644, show_rxbuf, store_rxbuf);
2606static DEVICE_ATTR(rxbuf_cur, 0444, show_rxbuf, NULL);
2607
2608static struct attribute *xennet_dev_attrs[] = {
2609	&dev_attr_rxbuf_min.attr,
2610	&dev_attr_rxbuf_max.attr,
2611	&dev_attr_rxbuf_cur.attr,
2612	NULL
2613};
2614
2615static const struct attribute_group xennet_dev_group = {
2616	.attrs = xennet_dev_attrs
2617};
2618#endif /* CONFIG_SYSFS */
2619
2620static void xennet_bus_close(struct xenbus_device *dev)
 
 
2621{
2622	int ret;
 
 
 
 
 
 
 
 
 
 
2623
2624	if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed)
2625		return;
2626	do {
2627		xenbus_switch_state(dev, XenbusStateClosing);
2628		ret = wait_event_timeout(module_wq,
2629				   xenbus_read_driver_state(dev->otherend) ==
2630				   XenbusStateClosing ||
2631				   xenbus_read_driver_state(dev->otherend) ==
2632				   XenbusStateClosed ||
2633				   xenbus_read_driver_state(dev->otherend) ==
2634				   XenbusStateUnknown,
2635				   XENNET_TIMEOUT);
2636	} while (!ret);
2637
2638	if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed)
2639		return;
2640
2641	do {
2642		xenbus_switch_state(dev, XenbusStateClosed);
2643		ret = wait_event_timeout(module_wq,
2644				   xenbus_read_driver_state(dev->otherend) ==
2645				   XenbusStateClosed ||
2646				   xenbus_read_driver_state(dev->otherend) ==
2647				   XenbusStateUnknown,
2648				   XENNET_TIMEOUT);
2649	} while (!ret);
2650}
2651
2652static void xennet_remove(struct xenbus_device *dev)
 
2653{
2654	struct netfront_info *info = dev_get_drvdata(&dev->dev);
 
 
 
 
2655
2656	xennet_bus_close(dev);
2657	xennet_disconnect_backend(info);
 
 
 
2658
2659	if (info->netdev->reg_state == NETREG_REGISTERED)
2660		unregister_netdev(info->netdev);
 
 
2661
2662	if (info->queues) {
2663		rtnl_lock();
2664		xennet_destroy_queues(info);
2665		rtnl_unlock();
 
2666	}
2667	xennet_free_netdev(info->netdev);
 
 
 
 
 
 
 
 
 
 
 
 
 
2668}
2669
 
 
2670static const struct xenbus_device_id netfront_ids[] = {
2671	{ "vif" },
2672	{ "" }
2673};
2674
2675static struct xenbus_driver netfront_driver = {
2676	.ids = netfront_ids,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2677	.probe = netfront_probe,
2678	.remove = xennet_remove,
2679	.resume = netfront_resume,
2680	.otherend_changed = netback_changed,
2681};
2682
2683static int __init netif_init(void)
2684{
2685	if (!xen_domain())
2686		return -ENODEV;
2687
2688	if (!xen_has_pv_nic_devices())
2689		return -ENODEV;
2690
2691	pr_info("Initialising Xen virtual ethernet driver\n");
2692
2693	/* Allow as many queues as there are CPUs inut max. 8 if user has not
2694	 * specified a value.
2695	 */
2696	if (xennet_max_queues == 0)
2697		xennet_max_queues = min_t(unsigned int, MAX_QUEUES_DEFAULT,
2698					  num_online_cpus());
2699
2700	return xenbus_register_frontend(&netfront_driver);
2701}
2702module_init(netif_init);
2703
2704
2705static void __exit netif_exit(void)
2706{
2707	xenbus_unregister_driver(&netfront_driver);
2708}
2709module_exit(netif_exit);
2710
2711MODULE_DESCRIPTION("Xen virtual network device frontend");
2712MODULE_LICENSE("GPL");
2713MODULE_ALIAS("xen:vif");
2714MODULE_ALIAS("xennet");