1/*
   2 * drivers/net/ethernet/rocker/rocker.c - Rocker switch device driver
   3 * Copyright (c) 2014-2016 Jiri Pirko <jiri@mellanox.com>
   4 * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation; either version 2 of the License, or
   9 * (at your option) any later version.
  10 */
  11
  12#include <linux/kernel.h>
  13#include <linux/module.h>
  14#include <linux/pci.h>
  15#include <linux/interrupt.h>
  16#include <linux/sched.h>
  17#include <linux/wait.h>
  18#include <linux/spinlock.h>
  19#include <linux/sort.h>
  20#include <linux/random.h>
  21#include <linux/netdevice.h>
  22#include <linux/skbuff.h>
  23#include <linux/socket.h>
  24#include <linux/etherdevice.h>
  25#include <linux/ethtool.h>
  26#include <linux/if_ether.h>
  27#include <linux/if_vlan.h>
  28#include <linux/if_bridge.h>
  29#include <linux/bitops.h>
  30#include <linux/ctype.h>
  31#include <linux/workqueue.h>
  32#include <net/switchdev.h>
  33#include <net/rtnetlink.h>
  34#include <net/netevent.h>
  35#include <net/arp.h>
  36#include <net/fib_rules.h>
  37#include <net/fib_notifier.h>
  38#include <linux/io-64-nonatomic-lo-hi.h>
  39#include <generated/utsrelease.h>
  40
  41#include "rocker_hw.h"
  42#include "rocker.h"
  43#include "rocker_tlv.h"
  44
  45static const char rocker_driver_name[] = "rocker";
  46
  47static const struct pci_device_id rocker_pci_id_table[] = {
  48	{PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_ROCKER), 0},
  49	{0, }
  50};
  51
  52struct rocker_wait {
  53	wait_queue_head_t wait;
  54	bool done;
  55	bool nowait;
  56};
  57
  58static void rocker_wait_reset(struct rocker_wait *wait)
  59{
  60	wait->done = false;
  61	wait->nowait = false;
  62}
  63
  64static void rocker_wait_init(struct rocker_wait *wait)
  65{
  66	init_waitqueue_head(&wait->wait);
  67	rocker_wait_reset(wait);
  68}
  69
  70static struct rocker_wait *rocker_wait_create(void)
  71{
  72	struct rocker_wait *wait;
  73
  74	wait = kzalloc(sizeof(*wait), GFP_KERNEL);
  75	if (!wait)
  76		return NULL;
  77	return wait;
  78}
  79
  80static void rocker_wait_destroy(struct rocker_wait *wait)
  81{
  82	kfree(wait);
  83}
  84
  85static bool rocker_wait_event_timeout(struct rocker_wait *wait,
  86				      unsigned long timeout)
  87{
  88	wait_event_timeout(wait->wait, wait->done, HZ / 10);
  89	if (!wait->done)
  90		return false;
  91	return true;
  92}
  93
  94static void rocker_wait_wake_up(struct rocker_wait *wait)
  95{
  96	wait->done = true;
  97	wake_up(&wait->wait);
  98}
  99
 100static u32 rocker_msix_vector(const struct rocker *rocker, unsigned int vector)
 101{
 102	return rocker->msix_entries[vector].vector;
 103}
 104
 105static u32 rocker_msix_tx_vector(const struct rocker_port *rocker_port)
 106{
 107	return rocker_msix_vector(rocker_port->rocker,
 108				  ROCKER_MSIX_VEC_TX(rocker_port->port_number));
 109}
 110
 111static u32 rocker_msix_rx_vector(const struct rocker_port *rocker_port)
 112{
 113	return rocker_msix_vector(rocker_port->rocker,
 114				  ROCKER_MSIX_VEC_RX(rocker_port->port_number));
 115}
 116
 117#define rocker_write32(rocker, reg, val)	\
 118	writel((val), (rocker)->hw_addr + (ROCKER_ ## reg))
 119#define rocker_read32(rocker, reg)	\
 120	readl((rocker)->hw_addr + (ROCKER_ ## reg))
 121#define rocker_write64(rocker, reg, val)	\
 122	writeq((val), (rocker)->hw_addr + (ROCKER_ ## reg))
 123#define rocker_read64(rocker, reg)	\
 124	readq((rocker)->hw_addr + (ROCKER_ ## reg))
 125
 126/*****************************
 127 * HW basic testing functions
 128 *****************************/
 129
 130static int rocker_reg_test(const struct rocker *rocker)
 131{
 132	const struct pci_dev *pdev = rocker->pdev;
 133	u64 test_reg;
 134	u64 rnd;
 135
 136	rnd = prandom_u32();
 137	rnd >>= 1;
 138	rocker_write32(rocker, TEST_REG, rnd);
 139	test_reg = rocker_read32(rocker, TEST_REG);
 140	if (test_reg != rnd * 2) {
 141		dev_err(&pdev->dev, "unexpected 32bit register value %08llx, expected %08llx\n",
 142			test_reg, rnd * 2);
 143		return -EIO;
 144	}
 145
 146	rnd = prandom_u32();
 147	rnd <<= 31;
 148	rnd |= prandom_u32();
 149	rocker_write64(rocker, TEST_REG64, rnd);
 150	test_reg = rocker_read64(rocker, TEST_REG64);
 151	if (test_reg != rnd * 2) {
 152		dev_err(&pdev->dev, "unexpected 64bit register value %16llx, expected %16llx\n",
 153			test_reg, rnd * 2);
 154		return -EIO;
 155	}
 156
 157	return 0;
 158}
 159
 160static int rocker_dma_test_one(const struct rocker *rocker,
 161			       struct rocker_wait *wait, u32 test_type,
 162			       dma_addr_t dma_handle, const unsigned char *buf,
 163			       const unsigned char *expect, size_t size)
 164{
 165	const struct pci_dev *pdev = rocker->pdev;
 166	int i;
 167
 168	rocker_wait_reset(wait);
 169	rocker_write32(rocker, TEST_DMA_CTRL, test_type);
 170
 171	if (!rocker_wait_event_timeout(wait, HZ / 10)) {
 172		dev_err(&pdev->dev, "no interrupt received within a timeout\n");
 173		return -EIO;
 174	}
 175
 176	for (i = 0; i < size; i++) {
 177		if (buf[i] != expect[i]) {
 178			dev_err(&pdev->dev, "unexpected memory content %02x at byte %x\n, %02x expected",
 179				buf[i], i, expect[i]);
 180			return -EIO;
 181		}
 182	}
 183	return 0;
 184}
 185
 186#define ROCKER_TEST_DMA_BUF_SIZE (PAGE_SIZE * 4)
 187#define ROCKER_TEST_DMA_FILL_PATTERN 0x96
 188
 189static int rocker_dma_test_offset(const struct rocker *rocker,
 190				  struct rocker_wait *wait, int offset)
 191{
 192	struct pci_dev *pdev = rocker->pdev;
 193	unsigned char *alloc;
 194	unsigned char *buf;
 195	unsigned char *expect;
 196	dma_addr_t dma_handle;
 197	int i;
 198	int err;
 199
 200	alloc = kzalloc(ROCKER_TEST_DMA_BUF_SIZE * 2 + offset,
 201			GFP_KERNEL | GFP_DMA);
 202	if (!alloc)
 203		return -ENOMEM;
 204	buf = alloc + offset;
 205	expect = buf + ROCKER_TEST_DMA_BUF_SIZE;
 206
 207	dma_handle = pci_map_single(pdev, buf, ROCKER_TEST_DMA_BUF_SIZE,
 208				    PCI_DMA_BIDIRECTIONAL);
 209	if (pci_dma_mapping_error(pdev, dma_handle)) {
 210		err = -EIO;
 211		goto free_alloc;
 212	}
 213
 214	rocker_write64(rocker, TEST_DMA_ADDR, dma_handle);
 215	rocker_write32(rocker, TEST_DMA_SIZE, ROCKER_TEST_DMA_BUF_SIZE);
 216
 217	memset(expect, ROCKER_TEST_DMA_FILL_PATTERN, ROCKER_TEST_DMA_BUF_SIZE);
 218	err = rocker_dma_test_one(rocker, wait, ROCKER_TEST_DMA_CTRL_FILL,
 219				  dma_handle, buf, expect,
 220				  ROCKER_TEST_DMA_BUF_SIZE);
 221	if (err)
 222		goto unmap;
 223
 224	memset(expect, 0, ROCKER_TEST_DMA_BUF_SIZE);
 225	err = rocker_dma_test_one(rocker, wait, ROCKER_TEST_DMA_CTRL_CLEAR,
 226				  dma_handle, buf, expect,
 227				  ROCKER_TEST_DMA_BUF_SIZE);
 228	if (err)
 229		goto unmap;
 230
 231	prandom_bytes(buf, ROCKER_TEST_DMA_BUF_SIZE);
 232	for (i = 0; i < ROCKER_TEST_DMA_BUF_SIZE; i++)
 233		expect[i] = ~buf[i];
 234	err = rocker_dma_test_one(rocker, wait, ROCKER_TEST_DMA_CTRL_INVERT,
 235				  dma_handle, buf, expect,
 236				  ROCKER_TEST_DMA_BUF_SIZE);
 237	if (err)
 238		goto unmap;
 239
 240unmap:
 241	pci_unmap_single(pdev, dma_handle, ROCKER_TEST_DMA_BUF_SIZE,
 242			 PCI_DMA_BIDIRECTIONAL);
 243free_alloc:
 244	kfree(alloc);
 245
 246	return err;
 247}
 248
 249static int rocker_dma_test(const struct rocker *rocker,
 250			   struct rocker_wait *wait)
 251{
 252	int i;
 253	int err;
 254
 255	for (i = 0; i < 8; i++) {
 256		err = rocker_dma_test_offset(rocker, wait, i);
 257		if (err)
 258			return err;
 259	}
 260	return 0;
 261}
 262
 263static irqreturn_t rocker_test_irq_handler(int irq, void *dev_id)
 264{
 265	struct rocker_wait *wait = dev_id;
 266
 267	rocker_wait_wake_up(wait);
 268
 269	return IRQ_HANDLED;
 270}
 271
 272static int rocker_basic_hw_test(const struct rocker *rocker)
 273{
 274	const struct pci_dev *pdev = rocker->pdev;
 275	struct rocker_wait wait;
 276	int err;
 277
 278	err = rocker_reg_test(rocker);
 279	if (err) {
 280		dev_err(&pdev->dev, "reg test failed\n");
 281		return err;
 282	}
 283
 284	err = request_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_TEST),
 285			  rocker_test_irq_handler, 0,
 286			  rocker_driver_name, &wait);
 287	if (err) {
 288		dev_err(&pdev->dev, "cannot assign test irq\n");
 289		return err;
 290	}
 291
 292	rocker_wait_init(&wait);
 293	rocker_write32(rocker, TEST_IRQ, ROCKER_MSIX_VEC_TEST);
 294
 295	if (!rocker_wait_event_timeout(&wait, HZ / 10)) {
 296		dev_err(&pdev->dev, "no interrupt received within a timeout\n");
 297		err = -EIO;
 298		goto free_irq;
 299	}
 300
 301	err = rocker_dma_test(rocker, &wait);
 302	if (err)
 303		dev_err(&pdev->dev, "dma test failed\n");
 304
 305free_irq:
 306	free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_TEST), &wait);
 307	return err;
 308}
 309
 310/******************************************
 311 * DMA rings and descriptors manipulations
 312 ******************************************/
 313
 314static u32 __pos_inc(u32 pos, size_t limit)
 315{
 316	return ++pos == limit ? 0 : pos;
 317}
 318
 319static int rocker_desc_err(const struct rocker_desc_info *desc_info)
 320{
 321	int err = desc_info->desc->comp_err & ~ROCKER_DMA_DESC_COMP_ERR_GEN;
 322
 323	switch (err) {
 324	case ROCKER_OK:
 325		return 0;
 326	case -ROCKER_ENOENT:
 327		return -ENOENT;
 328	case -ROCKER_ENXIO:
 329		return -ENXIO;
 330	case -ROCKER_ENOMEM:
 331		return -ENOMEM;
 332	case -ROCKER_EEXIST:
 333		return -EEXIST;
 334	case -ROCKER_EINVAL:
 335		return -EINVAL;
 336	case -ROCKER_EMSGSIZE:
 337		return -EMSGSIZE;
 338	case -ROCKER_ENOTSUP:
 339		return -EOPNOTSUPP;
 340	case -ROCKER_ENOBUFS:
 341		return -ENOBUFS;
 342	}
 343
 344	return -EINVAL;
 345}
 346
 347static void rocker_desc_gen_clear(const struct rocker_desc_info *desc_info)
 348{
 349	desc_info->desc->comp_err &= ~ROCKER_DMA_DESC_COMP_ERR_GEN;
 350}
 351
 352static bool rocker_desc_gen(const struct rocker_desc_info *desc_info)
 353{
 354	u32 comp_err = desc_info->desc->comp_err;
 355
 356	return comp_err & ROCKER_DMA_DESC_COMP_ERR_GEN ? true : false;
 357}
 358
 359static void *
 360rocker_desc_cookie_ptr_get(const struct rocker_desc_info *desc_info)
 361{
 362	return (void *)(uintptr_t)desc_info->desc->cookie;
 363}
 364
 365static void rocker_desc_cookie_ptr_set(const struct rocker_desc_info *desc_info,
 366				       void *ptr)
 367{
 368	desc_info->desc->cookie = (uintptr_t) ptr;
 369}
 370
 371static struct rocker_desc_info *
 372rocker_desc_head_get(const struct rocker_dma_ring_info *info)
 373{
 374	static struct rocker_desc_info *desc_info;
 375	u32 head = __pos_inc(info->head, info->size);
 376
 377	desc_info = &info->desc_info[info->head];
 378	if (head == info->tail)
 379		return NULL; /* ring full */
 380	desc_info->tlv_size = 0;
 381	return desc_info;
 382}
 383
 384static void rocker_desc_commit(const struct rocker_desc_info *desc_info)
 385{
 386	desc_info->desc->buf_size = desc_info->data_size;
 387	desc_info->desc->tlv_size = desc_info->tlv_size;
 388}
 389
 390static void rocker_desc_head_set(const struct rocker *rocker,
 391				 struct rocker_dma_ring_info *info,
 392				 const struct rocker_desc_info *desc_info)
 393{
 394	u32 head = __pos_inc(info->head, info->size);
 395
 396	BUG_ON(head == info->tail);
 397	rocker_desc_commit(desc_info);
 398	info->head = head;
 399	rocker_write32(rocker, DMA_DESC_HEAD(info->type), head);
 400}
 401
 402static struct rocker_desc_info *
 403rocker_desc_tail_get(struct rocker_dma_ring_info *info)
 404{
 405	static struct rocker_desc_info *desc_info;
 406
 407	if (info->tail == info->head)
 408		return NULL; /* nothing to be done between head and tail */
 409	desc_info = &info->desc_info[info->tail];
 410	if (!rocker_desc_gen(desc_info))
 411		return NULL; /* gen bit not set, desc is not ready yet */
 412	info->tail = __pos_inc(info->tail, info->size);
 413	desc_info->tlv_size = desc_info->desc->tlv_size;
 414	return desc_info;
 415}
 416
 417static void rocker_dma_ring_credits_set(const struct rocker *rocker,
 418					const struct rocker_dma_ring_info *info,
 419					u32 credits)
 420{
 421	if (credits)
 422		rocker_write32(rocker, DMA_DESC_CREDITS(info->type), credits);
 423}
 424
 425static unsigned long rocker_dma_ring_size_fix(size_t size)
 426{
 427	return max(ROCKER_DMA_SIZE_MIN,
 428		   min(roundup_pow_of_two(size), ROCKER_DMA_SIZE_MAX));
 429}
 430
 431static int rocker_dma_ring_create(const struct rocker *rocker,
 432				  unsigned int type,
 433				  size_t size,
 434				  struct rocker_dma_ring_info *info)
 435{
 436	int i;
 437
 438	BUG_ON(size != rocker_dma_ring_size_fix(size));
 439	info->size = size;
 440	info->type = type;
 441	info->head = 0;
 442	info->tail = 0;
 443	info->desc_info = kcalloc(info->size, sizeof(*info->desc_info),
 444				  GFP_KERNEL);
 445	if (!info->desc_info)
 446		return -ENOMEM;
 447
 448	info->desc = pci_alloc_consistent(rocker->pdev,
 449					  info->size * sizeof(*info->desc),
 450					  &info->mapaddr);
 451	if (!info->desc) {
 452		kfree(info->desc_info);
 453		return -ENOMEM;
 454	}
 455
 456	for (i = 0; i < info->size; i++)
 457		info->desc_info[i].desc = &info->desc[i];
 458
 459	rocker_write32(rocker, DMA_DESC_CTRL(info->type),
 460		       ROCKER_DMA_DESC_CTRL_RESET);
 461	rocker_write64(rocker, DMA_DESC_ADDR(info->type), info->mapaddr);
 462	rocker_write32(rocker, DMA_DESC_SIZE(info->type), info->size);
 463
 464	return 0;
 465}
 466
 467static void rocker_dma_ring_destroy(const struct rocker *rocker,
 468				    const struct rocker_dma_ring_info *info)
 469{
 470	rocker_write64(rocker, DMA_DESC_ADDR(info->type), 0);
 471
 472	pci_free_consistent(rocker->pdev,
 473			    info->size * sizeof(struct rocker_desc),
 474			    info->desc, info->mapaddr);
 475	kfree(info->desc_info);
 476}
 477
 478static void rocker_dma_ring_pass_to_producer(const struct rocker *rocker,
 479					     struct rocker_dma_ring_info *info)
 480{
 481	int i;
 482
 483	BUG_ON(info->head || info->tail);
 484
 485	/* When ring is consumer, we need to advance head for each desc.
 486	 * That tells hw that the desc is ready to be used by it.
 487	 */
 488	for (i = 0; i < info->size - 1; i++)
 489		rocker_desc_head_set(rocker, info, &info->desc_info[i]);
 490	rocker_desc_commit(&info->desc_info[i]);
 491}
 492
 493static int rocker_dma_ring_bufs_alloc(const struct rocker *rocker,
 494				      const struct rocker_dma_ring_info *info,
 495				      int direction, size_t buf_size)
 496{
 497	struct pci_dev *pdev = rocker->pdev;
 498	int i;
 499	int err;
 500
 501	for (i = 0; i < info->size; i++) {
 502		struct rocker_desc_info *desc_info = &info->desc_info[i];
 503		struct rocker_desc *desc = &info->desc[i];
 504		dma_addr_t dma_handle;
 505		char *buf;
 506
 507		buf = kzalloc(buf_size, GFP_KERNEL | GFP_DMA);
 508		if (!buf) {
 509			err = -ENOMEM;
 510			goto rollback;
 511		}
 512
 513		dma_handle = pci_map_single(pdev, buf, buf_size, direction);
 514		if (pci_dma_mapping_error(pdev, dma_handle)) {
 515			kfree(buf);
 516			err = -EIO;
 517			goto rollback;
 518		}
 519
 520		desc_info->data = buf;
 521		desc_info->data_size = buf_size;
 522		dma_unmap_addr_set(desc_info, mapaddr, dma_handle);
 523
 524		desc->buf_addr = dma_handle;
 525		desc->buf_size = buf_size;
 526	}
 527	return 0;
 528
 529rollback:
 530	for (i--; i >= 0; i--) {
 531		const struct rocker_desc_info *desc_info = &info->desc_info[i];
 532
 533		pci_unmap_single(pdev, dma_unmap_addr(desc_info, mapaddr),
 534				 desc_info->data_size, direction);
 535		kfree(desc_info->data);
 536	}
 537	return err;
 538}
 539
 540static void rocker_dma_ring_bufs_free(const struct rocker *rocker,
 541				      const struct rocker_dma_ring_info *info,
 542				      int direction)
 543{
 544	struct pci_dev *pdev = rocker->pdev;
 545	int i;
 546
 547	for (i = 0; i < info->size; i++) {
 548		const struct rocker_desc_info *desc_info = &info->desc_info[i];
 549		struct rocker_desc *desc = &info->desc[i];
 550
 551		desc->buf_addr = 0;
 552		desc->buf_size = 0;
 553		pci_unmap_single(pdev, dma_unmap_addr(desc_info, mapaddr),
 554				 desc_info->data_size, direction);
 555		kfree(desc_info->data);
 556	}
 557}
 558
 559static int rocker_dma_cmd_ring_wait_alloc(struct rocker_desc_info *desc_info)
 560{
 561	struct rocker_wait *wait;
 562
 563	wait = rocker_wait_create();
 564	if (!wait)
 565		return -ENOMEM;
 566	rocker_desc_cookie_ptr_set(desc_info, wait);
 567	return 0;
 568}
 569
 570static void
 571rocker_dma_cmd_ring_wait_free(const struct rocker_desc_info *desc_info)
 572{
 573	struct rocker_wait *wait = rocker_desc_cookie_ptr_get(desc_info);
 574
 575	rocker_wait_destroy(wait);
 576}
 577
 578static int rocker_dma_cmd_ring_waits_alloc(const struct rocker *rocker)
 579{
 580	const struct rocker_dma_ring_info *cmd_ring = &rocker->cmd_ring;
 581	int i;
 582	int err;
 583
 584	for (i = 0; i < cmd_ring->size; i++) {
 585		err = rocker_dma_cmd_ring_wait_alloc(&cmd_ring->desc_info[i]);
 586		if (err)
 587			goto rollback;
 588	}
 589	return 0;
 590
 591rollback:
 592	for (i--; i >= 0; i--)
 593		rocker_dma_cmd_ring_wait_free(&cmd_ring->desc_info[i]);
 594	return err;
 595}
 596
 597static void rocker_dma_cmd_ring_waits_free(const struct rocker *rocker)
 598{
 599	const struct rocker_dma_ring_info *cmd_ring = &rocker->cmd_ring;
 600	int i;
 601
 602	for (i = 0; i < cmd_ring->size; i++)
 603		rocker_dma_cmd_ring_wait_free(&cmd_ring->desc_info[i]);
 604}
 605
 606static int rocker_dma_rings_init(struct rocker *rocker)
 607{
 608	const struct pci_dev *pdev = rocker->pdev;
 609	int err;
 610
 611	err = rocker_dma_ring_create(rocker, ROCKER_DMA_CMD,
 612				     ROCKER_DMA_CMD_DEFAULT_SIZE,
 613				     &rocker->cmd_ring);
 614	if (err) {
 615		dev_err(&pdev->dev, "failed to create command dma ring\n");
 616		return err;
 617	}
 618
 619	spin_lock_init(&rocker->cmd_ring_lock);
 620
 621	err = rocker_dma_ring_bufs_alloc(rocker, &rocker->cmd_ring,
 622					 PCI_DMA_BIDIRECTIONAL, PAGE_SIZE);
 623	if (err) {
 624		dev_err(&pdev->dev, "failed to alloc command dma ring buffers\n");
 625		goto err_dma_cmd_ring_bufs_alloc;
 626	}
 627
 628	err = rocker_dma_cmd_ring_waits_alloc(rocker);
 629	if (err) {
 630		dev_err(&pdev->dev, "failed to alloc command dma ring waits\n");
 631		goto err_dma_cmd_ring_waits_alloc;
 632	}
 633
 634	err = rocker_dma_ring_create(rocker, ROCKER_DMA_EVENT,
 635				     ROCKER_DMA_EVENT_DEFAULT_SIZE,
 636				     &rocker->event_ring);
 637	if (err) {
 638		dev_err(&pdev->dev, "failed to create event dma ring\n");
 639		goto err_dma_event_ring_create;
 640	}
 641
 642	err = rocker_dma_ring_bufs_alloc(rocker, &rocker->event_ring,
 643					 PCI_DMA_FROMDEVICE, PAGE_SIZE);
 644	if (err) {
 645		dev_err(&pdev->dev, "failed to alloc event dma ring buffers\n");
 646		goto err_dma_event_ring_bufs_alloc;
 647	}
 648	rocker_dma_ring_pass_to_producer(rocker, &rocker->event_ring);
 649	return 0;
 650
 651err_dma_event_ring_bufs_alloc:
 652	rocker_dma_ring_destroy(rocker, &rocker->event_ring);
 653err_dma_event_ring_create:
 654	rocker_dma_ring_bufs_free(rocker, &rocker->cmd_ring,
 655				  PCI_DMA_BIDIRECTIONAL);
 656err_dma_cmd_ring_waits_alloc:
 657	rocker_dma_cmd_ring_waits_free(rocker);
 658err_dma_cmd_ring_bufs_alloc:
 659	rocker_dma_ring_destroy(rocker, &rocker->cmd_ring);
 660	return err;
 661}
 662
 663static void rocker_dma_rings_fini(struct rocker *rocker)
 664{
 665	rocker_dma_ring_bufs_free(rocker, &rocker->event_ring,
 666				  PCI_DMA_BIDIRECTIONAL);
 667	rocker_dma_ring_destroy(rocker, &rocker->event_ring);
 668	rocker_dma_cmd_ring_waits_free(rocker);
 669	rocker_dma_ring_bufs_free(rocker, &rocker->cmd_ring,
 670				  PCI_DMA_BIDIRECTIONAL);
 671	rocker_dma_ring_destroy(rocker, &rocker->cmd_ring);
 672}
 673
 674static int rocker_dma_rx_ring_skb_map(const struct rocker_port *rocker_port,
 675				      struct rocker_desc_info *desc_info,
 676				      struct sk_buff *skb, size_t buf_len)
 677{
 678	const struct rocker *rocker = rocker_port->rocker;
 679	struct pci_dev *pdev = rocker->pdev;
 680	dma_addr_t dma_handle;
 681
 682	dma_handle = pci_map_single(pdev, skb->data, buf_len,
 683				    PCI_DMA_FROMDEVICE);
 684	if (pci_dma_mapping_error(pdev, dma_handle))
 685		return -EIO;
 686	if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_RX_FRAG_ADDR, dma_handle))
 687		goto tlv_put_failure;
 688	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_RX_FRAG_MAX_LEN, buf_len))
 689		goto tlv_put_failure;
 690	return 0;
 691
 692tlv_put_failure:
 693	pci_unmap_single(pdev, dma_handle, buf_len, PCI_DMA_FROMDEVICE);
 694	desc_info->tlv_size = 0;
 695	return -EMSGSIZE;
 696}
 697
 698static size_t rocker_port_rx_buf_len(const struct rocker_port *rocker_port)
 699{
 700	return rocker_port->dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
 701}
 702
 703static int rocker_dma_rx_ring_skb_alloc(const struct rocker_port *rocker_port,
 704					struct rocker_desc_info *desc_info)
 705{
 706	struct net_device *dev = rocker_port->dev;
 707	struct sk_buff *skb;
 708	size_t buf_len = rocker_port_rx_buf_len(rocker_port);
 709	int err;
 710
 711	/* Ensure that hw will see tlv_size zero in case of an error.
 712	 * That tells hw to use another descriptor.
 713	 */
 714	rocker_desc_cookie_ptr_set(desc_info, NULL);
 715	desc_info->tlv_size = 0;
 716
 717	skb = netdev_alloc_skb_ip_align(dev, buf_len);
 718	if (!skb)
 719		return -ENOMEM;
 720	err = rocker_dma_rx_ring_skb_map(rocker_port, desc_info, skb, buf_len);
 721	if (err) {
 722		dev_kfree_skb_any(skb);
 723		return err;
 724	}
 725	rocker_desc_cookie_ptr_set(desc_info, skb);
 726	return 0;
 727}
 728
 729static void rocker_dma_rx_ring_skb_unmap(const struct rocker *rocker,
 730					 const struct rocker_tlv **attrs)
 731{
 732	struct pci_dev *pdev = rocker->pdev;
 733	dma_addr_t dma_handle;
 734	size_t len;
 735
 736	if (!attrs[ROCKER_TLV_RX_FRAG_ADDR] ||
 737	    !attrs[ROCKER_TLV_RX_FRAG_MAX_LEN])
 738		return;
 739	dma_handle = rocker_tlv_get_u64(attrs[ROCKER_TLV_RX_FRAG_ADDR]);
 740	len = rocker_tlv_get_u16(attrs[ROCKER_TLV_RX_FRAG_MAX_LEN]);
 741	pci_unmap_single(pdev, dma_handle, len, PCI_DMA_FROMDEVICE);
 742}
 743
 744static void rocker_dma_rx_ring_skb_free(const struct rocker *rocker,
 745					const struct rocker_desc_info *desc_info)
 746{
 747	const struct rocker_tlv *attrs[ROCKER_TLV_RX_MAX + 1];
 748	struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);
 749
 750	if (!skb)
 751		return;
 752	rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);
 753	rocker_dma_rx_ring_skb_unmap(rocker, attrs);
 754	dev_kfree_skb_any(skb);
 755}
 756
 757static int rocker_dma_rx_ring_skbs_alloc(const struct rocker_port *rocker_port)
 758{
 759	const struct rocker_dma_ring_info *rx_ring = &rocker_port->rx_ring;
 760	const struct rocker *rocker = rocker_port->rocker;
 761	int i;
 762	int err;
 763
 764	for (i = 0; i < rx_ring->size; i++) {
 765		err = rocker_dma_rx_ring_skb_alloc(rocker_port,
 766						   &rx_ring->desc_info[i]);
 767		if (err)
 768			goto rollback;
 769	}
 770	return 0;
 771
 772rollback:
 773	for (i--; i >= 0; i--)
 774		rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);
 775	return err;
 776}
 777
 778static void rocker_dma_rx_ring_skbs_free(const struct rocker_port *rocker_port)
 779{
 780	const struct rocker_dma_ring_info *rx_ring = &rocker_port->rx_ring;
 781	const struct rocker *rocker = rocker_port->rocker;
 782	int i;
 783
 784	for (i = 0; i < rx_ring->size; i++)
 785		rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);
 786}
 787
 788static int rocker_port_dma_rings_init(struct rocker_port *rocker_port)
 789{
 790	struct rocker *rocker = rocker_port->rocker;
 791	int err;
 792
 793	err = rocker_dma_ring_create(rocker,
 794				     ROCKER_DMA_TX(rocker_port->port_number),
 795				     ROCKER_DMA_TX_DEFAULT_SIZE,
 796				     &rocker_port->tx_ring);
 797	if (err) {
 798		netdev_err(rocker_port->dev, "failed to create tx dma ring\n");
 799		return err;
 800	}
 801
 802	err = rocker_dma_ring_bufs_alloc(rocker, &rocker_port->tx_ring,
 803					 PCI_DMA_TODEVICE,
 804					 ROCKER_DMA_TX_DESC_SIZE);
 805	if (err) {
 806		netdev_err(rocker_port->dev, "failed to alloc tx dma ring buffers\n");
 807		goto err_dma_tx_ring_bufs_alloc;
 808	}
 809
 810	err = rocker_dma_ring_create(rocker,
 811				     ROCKER_DMA_RX(rocker_port->port_number),
 812				     ROCKER_DMA_RX_DEFAULT_SIZE,
 813				     &rocker_port->rx_ring);
 814	if (err) {
 815		netdev_err(rocker_port->dev, "failed to create rx dma ring\n");
 816		goto err_dma_rx_ring_create;
 817	}
 818
 819	err = rocker_dma_ring_bufs_alloc(rocker, &rocker_port->rx_ring,
 820					 PCI_DMA_BIDIRECTIONAL,
 821					 ROCKER_DMA_RX_DESC_SIZE);
 822	if (err) {
 823		netdev_err(rocker_port->dev, "failed to alloc rx dma ring buffers\n");
 824		goto err_dma_rx_ring_bufs_alloc;
 825	}
 826
 827	err = rocker_dma_rx_ring_skbs_alloc(rocker_port);
 828	if (err) {
 829		netdev_err(rocker_port->dev, "failed to alloc rx dma ring skbs\n");
 830		goto err_dma_rx_ring_skbs_alloc;
 831	}
 832	rocker_dma_ring_pass_to_producer(rocker, &rocker_port->rx_ring);
 833
 834	return 0;
 835
 836err_dma_rx_ring_skbs_alloc:
 837	rocker_dma_ring_bufs_free(rocker, &rocker_port->rx_ring,
 838				  PCI_DMA_BIDIRECTIONAL);
 839err_dma_rx_ring_bufs_alloc:
 840	rocker_dma_ring_destroy(rocker, &rocker_port->rx_ring);
 841err_dma_rx_ring_create:
 842	rocker_dma_ring_bufs_free(rocker, &rocker_port->tx_ring,
 843				  PCI_DMA_TODEVICE);
 844err_dma_tx_ring_bufs_alloc:
 845	rocker_dma_ring_destroy(rocker, &rocker_port->tx_ring);
 846	return err;
 847}
 848
 849static void rocker_port_dma_rings_fini(struct rocker_port *rocker_port)
 850{
 851	struct rocker *rocker = rocker_port->rocker;
 852
 853	rocker_dma_rx_ring_skbs_free(rocker_port);
 854	rocker_dma_ring_bufs_free(rocker, &rocker_port->rx_ring,
 855				  PCI_DMA_BIDIRECTIONAL);
 856	rocker_dma_ring_destroy(rocker, &rocker_port->rx_ring);
 857	rocker_dma_ring_bufs_free(rocker, &rocker_port->tx_ring,
 858				  PCI_DMA_TODEVICE);
 859	rocker_dma_ring_destroy(rocker, &rocker_port->tx_ring);
 860}
 861
 862static void rocker_port_set_enable(const struct rocker_port *rocker_port,
 863				   bool enable)
 864{
 865	u64 val = rocker_read64(rocker_port->rocker, PORT_PHYS_ENABLE);
 866
 867	if (enable)
 868		val |= 1ULL << rocker_port->pport;
 869	else
 870		val &= ~(1ULL << rocker_port->pport);
 871	rocker_write64(rocker_port->rocker, PORT_PHYS_ENABLE, val);
 872}
 873
 874/********************************
 875 * Interrupt handler and helpers
 876 ********************************/
 877
 878static irqreturn_t rocker_cmd_irq_handler(int irq, void *dev_id)
 879{
 880	struct rocker *rocker = dev_id;
 881	const struct rocker_desc_info *desc_info;
 882	struct rocker_wait *wait;
 883	u32 credits = 0;
 884
 885	spin_lock(&rocker->cmd_ring_lock);
 886	while ((desc_info = rocker_desc_tail_get(&rocker->cmd_ring))) {
 887		wait = rocker_desc_cookie_ptr_get(desc_info);
 888		if (wait->nowait) {
 889			rocker_desc_gen_clear(desc_info);
 890		} else {
 891			rocker_wait_wake_up(wait);
 892		}
 893		credits++;
 894	}
 895	spin_unlock(&rocker->cmd_ring_lock);
 896	rocker_dma_ring_credits_set(rocker, &rocker->cmd_ring, credits);
 897
 898	return IRQ_HANDLED;
 899}
 900
 901static void rocker_port_link_up(const struct rocker_port *rocker_port)
 902{
 903	netif_carrier_on(rocker_port->dev);
 904	netdev_info(rocker_port->dev, "Link is up\n");
 905}
 906
 907static void rocker_port_link_down(const struct rocker_port *rocker_port)
 908{
 909	netif_carrier_off(rocker_port->dev);
 910	netdev_info(rocker_port->dev, "Link is down\n");
 911}
 912
 913static int rocker_event_link_change(const struct rocker *rocker,
 914				    const struct rocker_tlv *info)
 915{
 916	const struct rocker_tlv *attrs[ROCKER_TLV_EVENT_LINK_CHANGED_MAX + 1];
 917	unsigned int port_number;
 918	bool link_up;
 919	struct rocker_port *rocker_port;
 920
 921	rocker_tlv_parse_nested(attrs, ROCKER_TLV_EVENT_LINK_CHANGED_MAX, info);
 922	if (!attrs[ROCKER_TLV_EVENT_LINK_CHANGED_PPORT] ||
 923	    !attrs[ROCKER_TLV_EVENT_LINK_CHANGED_LINKUP])
 924		return -EIO;
 925	port_number =
 926		rocker_tlv_get_u32(attrs[ROCKER_TLV_EVENT_LINK_CHANGED_PPORT]) - 1;
 927	link_up = rocker_tlv_get_u8(attrs[ROCKER_TLV_EVENT_LINK_CHANGED_LINKUP]);
 928
 929	if (port_number >= rocker->port_count)
 930		return -EINVAL;
 931
 932	rocker_port = rocker->ports[port_number];
 933	if (netif_carrier_ok(rocker_port->dev) != link_up) {
 934		if (link_up)
 935			rocker_port_link_up(rocker_port);
 936		else
 937			rocker_port_link_down(rocker_port);
 938	}
 939
 940	return 0;
 941}
 942
 943static int rocker_world_port_ev_mac_vlan_seen(struct rocker_port *rocker_port,
 944					      const unsigned char *addr,
 945					      __be16 vlan_id);
 946
 947static int rocker_event_mac_vlan_seen(const struct rocker *rocker,
 948				      const struct rocker_tlv *info)
 949{
 950	const struct rocker_tlv *attrs[ROCKER_TLV_EVENT_MAC_VLAN_MAX + 1];
 951	unsigned int port_number;
 952	struct rocker_port *rocker_port;
 953	const unsigned char *addr;
 954	__be16 vlan_id;
 955
 956	rocker_tlv_parse_nested(attrs, ROCKER_TLV_EVENT_MAC_VLAN_MAX, info);
 957	if (!attrs[ROCKER_TLV_EVENT_MAC_VLAN_PPORT] ||
 958	    !attrs[ROCKER_TLV_EVENT_MAC_VLAN_MAC] ||
 959	    !attrs[ROCKER_TLV_EVENT_MAC_VLAN_VLAN_ID])
 960		return -EIO;
 961	port_number =
 962		rocker_tlv_get_u32(attrs[ROCKER_TLV_EVENT_MAC_VLAN_PPORT]) - 1;
 963	addr = rocker_tlv_data(attrs[ROCKER_TLV_EVENT_MAC_VLAN_MAC]);
 964	vlan_id = rocker_tlv_get_be16(attrs[ROCKER_TLV_EVENT_MAC_VLAN_VLAN_ID]);
 965
 966	if (port_number >= rocker->port_count)
 967		return -EINVAL;
 968
 969	rocker_port = rocker->ports[port_number];
 970	return rocker_world_port_ev_mac_vlan_seen(rocker_port, addr, vlan_id);
 971}
 972
 973static int rocker_event_process(const struct rocker *rocker,
 974				const struct rocker_desc_info *desc_info)
 975{
 976	const struct rocker_tlv *attrs[ROCKER_TLV_EVENT_MAX + 1];
 977	const struct rocker_tlv *info;
 978	u16 type;
 979
 980	rocker_tlv_parse_desc(attrs, ROCKER_TLV_EVENT_MAX, desc_info);
 981	if (!attrs[ROCKER_TLV_EVENT_TYPE] ||
 982	    !attrs[ROCKER_TLV_EVENT_INFO])
 983		return -EIO;
 984
 985	type = rocker_tlv_get_u16(attrs[ROCKER_TLV_EVENT_TYPE]);
 986	info = attrs[ROCKER_TLV_EVENT_INFO];
 987
 988	switch (type) {
 989	case ROCKER_TLV_EVENT_TYPE_LINK_CHANGED:
 990		return rocker_event_link_change(rocker, info);
 991	case ROCKER_TLV_EVENT_TYPE_MAC_VLAN_SEEN:
 992		return rocker_event_mac_vlan_seen(rocker, info);
 993	}
 994
 995	return -EOPNOTSUPP;
 996}
 997
 998static irqreturn_t rocker_event_irq_handler(int irq, void *dev_id)
 999{
1000	struct rocker *rocker = dev_id;
1001	const struct pci_dev *pdev = rocker->pdev;
1002	const struct rocker_desc_info *desc_info;
1003	u32 credits = 0;
1004	int err;
1005
1006	while ((desc_info = rocker_desc_tail_get(&rocker->event_ring))) {
1007		err = rocker_desc_err(desc_info);
1008		if (err) {
1009			dev_err(&pdev->dev, "event desc received with err %d\n",
1010				err);
1011		} else {
1012			err = rocker_event_process(rocker, desc_info);
1013			if (err)
1014				dev_err(&pdev->dev, "event processing failed with err %d\n",
1015					err);
1016		}
1017		rocker_desc_gen_clear(desc_info);
1018		rocker_desc_head_set(rocker, &rocker->event_ring, desc_info);
1019		credits++;
1020	}
1021	rocker_dma_ring_credits_set(rocker, &rocker->event_ring, credits);
1022
1023	return IRQ_HANDLED;
1024}
1025
1026static irqreturn_t rocker_tx_irq_handler(int irq, void *dev_id)
1027{
1028	struct rocker_port *rocker_port = dev_id;
1029
1030	napi_schedule(&rocker_port->napi_tx);
1031	return IRQ_HANDLED;
1032}
1033
1034static irqreturn_t rocker_rx_irq_handler(int irq, void *dev_id)
1035{
1036	struct rocker_port *rocker_port = dev_id;
1037
1038	napi_schedule(&rocker_port->napi_rx);
1039	return IRQ_HANDLED;
1040}
1041
1042/********************
1043 * Command interface
1044 ********************/
1045
1046int rocker_cmd_exec(struct rocker_port *rocker_port, bool nowait,
1047		    rocker_cmd_prep_cb_t prepare, void *prepare_priv,
1048		    rocker_cmd_proc_cb_t process, void *process_priv)
1049{
1050	struct rocker *rocker = rocker_port->rocker;
1051	struct rocker_desc_info *desc_info;
1052	struct rocker_wait *wait;
1053	unsigned long lock_flags;
1054	int err;
1055
1056	spin_lock_irqsave(&rocker->cmd_ring_lock, lock_flags);
1057
1058	desc_info = rocker_desc_head_get(&rocker->cmd_ring);
1059	if (!desc_info) {
1060		spin_unlock_irqrestore(&rocker->cmd_ring_lock, lock_flags);
1061		return -EAGAIN;
1062	}
1063
1064	wait = rocker_desc_cookie_ptr_get(desc_info);
1065	rocker_wait_init(wait);
1066	wait->nowait = nowait;
1067
1068	err = prepare(rocker_port, desc_info, prepare_priv);
1069	if (err) {
1070		spin_unlock_irqrestore(&rocker->cmd_ring_lock, lock_flags);
1071		return err;
1072	}
1073
1074	rocker_desc_head_set(rocker, &rocker->cmd_ring, desc_info);
1075
1076	spin_unlock_irqrestore(&rocker->cmd_ring_lock, lock_flags);
1077
1078	if (nowait)
1079		return 0;
1080
1081	if (!rocker_wait_event_timeout(wait, HZ / 10))
1082		return -EIO;
1083
1084	err = rocker_desc_err(desc_info);
1085	if (err)
1086		return err;
1087
1088	if (process)
1089		err = process(rocker_port, desc_info, process_priv);
1090
1091	rocker_desc_gen_clear(desc_info);
1092	return err;
1093}
1094
1095static int
1096rocker_cmd_get_port_settings_prep(const struct rocker_port *rocker_port,
1097				  struct rocker_desc_info *desc_info,
1098				  void *priv)
1099{
1100	struct rocker_tlv *cmd_info;
1101
1102	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1103			       ROCKER_TLV_CMD_TYPE_GET_PORT_SETTINGS))
1104		return -EMSGSIZE;
1105	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1106	if (!cmd_info)
1107		return -EMSGSIZE;
1108	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1109			       rocker_port->pport))
1110		return -EMSGSIZE;
1111	rocker_tlv_nest_end(desc_info, cmd_info);
1112	return 0;
1113}
1114
1115static int
1116rocker_cmd_get_port_settings_ethtool_proc(const struct rocker_port *rocker_port,
1117					  const struct rocker_desc_info *desc_info,
1118					  void *priv)
1119{
1120	struct ethtool_link_ksettings *ecmd = priv;
1121	const struct rocker_tlv *attrs[ROCKER_TLV_CMD_MAX + 1];
1122	const struct rocker_tlv *info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MAX + 1];
1123	u32 speed;
1124	u8 duplex;
1125	u8 autoneg;
1126
1127	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);
1128	if (!attrs[ROCKER_TLV_CMD_INFO])
1129		return -EIO;
1130
1131	rocker_tlv_parse_nested(info_attrs, ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
1132				attrs[ROCKER_TLV_CMD_INFO]);
1133	if (!info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_SPEED] ||
1134	    !info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX] ||
1135	    !info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG])
1136		return -EIO;
1137
1138	speed = rocker_tlv_get_u32(info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_SPEED]);
1139	duplex = rocker_tlv_get_u8(info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX]);
1140	autoneg = rocker_tlv_get_u8(info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG]);
1141
1142	ethtool_link_ksettings_zero_link_mode(ecmd, supported);
1143	ethtool_link_ksettings_add_link_mode(ecmd, supported, TP);
1144
1145	ecmd->base.phy_address = 0xff;
1146	ecmd->base.port = PORT_TP;
1147	ecmd->base.speed = speed;
1148	ecmd->base.duplex = duplex ? DUPLEX_FULL : DUPLEX_HALF;
1149	ecmd->base.autoneg = autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE;
1150
1151	return 0;
1152}
1153
1154static int
1155rocker_cmd_get_port_settings_macaddr_proc(const struct rocker_port *rocker_port,
1156					  const struct rocker_desc_info *desc_info,
1157					  void *priv)
1158{
1159	unsigned char *macaddr = priv;
1160	const struct rocker_tlv *attrs[ROCKER_TLV_CMD_MAX + 1];
1161	const struct rocker_tlv *info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MAX + 1];
1162	const struct rocker_tlv *attr;
1163
1164	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);
1165	if (!attrs[ROCKER_TLV_CMD_INFO])
1166		return -EIO;
1167
1168	rocker_tlv_parse_nested(info_attrs, ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
1169				attrs[ROCKER_TLV_CMD_INFO]);
1170	attr = info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR];
1171	if (!attr)
1172		return -EIO;
1173
1174	if (rocker_tlv_len(attr) != ETH_ALEN)
1175		return -EINVAL;
1176
1177	ether_addr_copy(macaddr, rocker_tlv_data(attr));
1178	return 0;
1179}
1180
1181static int
1182rocker_cmd_get_port_settings_mode_proc(const struct rocker_port *rocker_port,
1183				       const struct rocker_desc_info *desc_info,
1184				       void *priv)
1185{
1186	u8 *p_mode = priv;
1187	const struct rocker_tlv *attrs[ROCKER_TLV_CMD_MAX + 1];
1188	const struct rocker_tlv *info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MAX + 1];
1189	const struct rocker_tlv *attr;
1190
1191	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);
1192	if (!attrs[ROCKER_TLV_CMD_INFO])
1193		return -EIO;
1194
1195	rocker_tlv_parse_nested(info_attrs, ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
1196				attrs[ROCKER_TLV_CMD_INFO]);
1197	attr = info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MODE];
1198	if (!attr)
1199		return -EIO;
1200
1201	*p_mode = rocker_tlv_get_u8(info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MODE]);
1202	return 0;
1203}
1204
1205struct port_name {
1206	char *buf;
1207	size_t len;
1208};
1209
1210static int
1211rocker_cmd_get_port_settings_phys_name_proc(const struct rocker_port *rocker_port,
1212					    const struct rocker_desc_info *desc_info,
1213					    void *priv)
1214{
1215	const struct rocker_tlv *info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MAX + 1];
1216	const struct rocker_tlv *attrs[ROCKER_TLV_CMD_MAX + 1];
1217	struct port_name *name = priv;
1218	const struct rocker_tlv *attr;
1219	size_t i, j, len;
1220	const char *str;
1221
1222	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);
1223	if (!attrs[ROCKER_TLV_CMD_INFO])
1224		return -EIO;
1225
1226	rocker_tlv_parse_nested(info_attrs, ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
1227				attrs[ROCKER_TLV_CMD_INFO]);
1228	attr = info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_PHYS_NAME];
1229	if (!attr)
1230		return -EIO;
1231
1232	len = min_t(size_t, rocker_tlv_len(attr), name->len);
1233	str = rocker_tlv_data(attr);
1234
1235	/* make sure name only contains alphanumeric characters */
1236	for (i = j = 0; i < len; ++i) {
1237		if (isalnum(str[i])) {
1238			name->buf[j] = str[i];
1239			j++;
1240		}
1241	}
1242
1243	if (j == 0)
1244		return -EIO;
1245
1246	name->buf[j] = '\0';
1247
1248	return 0;
1249}
1250
1251static int
1252rocker_cmd_set_port_settings_ethtool_prep(const struct rocker_port *rocker_port,
1253					  struct rocker_desc_info *desc_info,
1254					  void *priv)
1255{
1256	struct ethtool_link_ksettings *ecmd = priv;
1257	struct rocker_tlv *cmd_info;
1258
1259	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1260			       ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS))
1261		return -EMSGSIZE;
1262	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1263	if (!cmd_info)
1264		return -EMSGSIZE;
1265	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1266			       rocker_port->pport))
1267		return -EMSGSIZE;
1268	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_SPEED,
1269			       ecmd->base.speed))
1270		return -EMSGSIZE;
1271	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX,
1272			      ecmd->base.duplex))
1273		return -EMSGSIZE;
1274	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG,
1275			      ecmd->base.autoneg))
1276		return -EMSGSIZE;
1277	rocker_tlv_nest_end(desc_info, cmd_info);
1278	return 0;
1279}
1280
1281static int
1282rocker_cmd_set_port_settings_macaddr_prep(const struct rocker_port *rocker_port,
1283					  struct rocker_desc_info *desc_info,
1284					  void *priv)
1285{
1286	const unsigned char *macaddr = priv;
1287	struct rocker_tlv *cmd_info;
1288
1289	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1290			       ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS))
1291		return -EMSGSIZE;
1292	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1293	if (!cmd_info)
1294		return -EMSGSIZE;
1295	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1296			       rocker_port->pport))
1297		return -EMSGSIZE;
1298	if (rocker_tlv_put(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR,
1299			   ETH_ALEN, macaddr))
1300		return -EMSGSIZE;
1301	rocker_tlv_nest_end(desc_info, cmd_info);
1302	return 0;
1303}
1304
1305static int
1306rocker_cmd_set_port_settings_mtu_prep(const struct rocker_port *rocker_port,
1307				      struct rocker_desc_info *desc_info,
1308				      void *priv)
1309{
1310	int mtu = *(int *)priv;
1311	struct rocker_tlv *cmd_info;
1312
1313	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1314			       ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS))
1315		return -EMSGSIZE;
1316	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1317	if (!cmd_info)
1318		return -EMSGSIZE;
1319	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1320			       rocker_port->pport))
1321		return -EMSGSIZE;
1322	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MTU,
1323			       mtu))
1324		return -EMSGSIZE;
1325	rocker_tlv_nest_end(desc_info, cmd_info);
1326	return 0;
1327}
1328
1329static int
1330rocker_cmd_set_port_learning_prep(const struct rocker_port *rocker_port,
1331				  struct rocker_desc_info *desc_info,
1332				  void *priv)
1333{
1334	bool learning = *(bool *)priv;
1335	struct rocker_tlv *cmd_info;
1336
1337	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1338			       ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS))
1339		return -EMSGSIZE;
1340	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1341	if (!cmd_info)
1342		return -EMSGSIZE;
1343	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1344			       rocker_port->pport))
1345		return -EMSGSIZE;
1346	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING,
1347			      learning))
1348		return -EMSGSIZE;
1349	rocker_tlv_nest_end(desc_info, cmd_info);
1350	return 0;
1351}
1352
1353static int
1354rocker_cmd_get_port_settings_ethtool(struct rocker_port *rocker_port,
1355				     struct ethtool_link_ksettings *ecmd)
1356{
1357	return rocker_cmd_exec(rocker_port, false,
1358			       rocker_cmd_get_port_settings_prep, NULL,
1359			       rocker_cmd_get_port_settings_ethtool_proc,
1360			       ecmd);
1361}
1362
1363static int rocker_cmd_get_port_settings_macaddr(struct rocker_port *rocker_port,
1364						unsigned char *macaddr)
1365{
1366	return rocker_cmd_exec(rocker_port, false,
1367			       rocker_cmd_get_port_settings_prep, NULL,
1368			       rocker_cmd_get_port_settings_macaddr_proc,
1369			       macaddr);
1370}
1371
1372static int rocker_cmd_get_port_settings_mode(struct rocker_port *rocker_port,
1373					     u8 *p_mode)
1374{
1375	return rocker_cmd_exec(rocker_port, false,
1376			       rocker_cmd_get_port_settings_prep, NULL,
1377			       rocker_cmd_get_port_settings_mode_proc, p_mode);
1378}
1379
1380static int
1381rocker_cmd_set_port_settings_ethtool(struct rocker_port *rocker_port,
1382				     const struct ethtool_link_ksettings *ecmd)
1383{
1384	struct ethtool_link_ksettings copy_ecmd;
1385
1386	memcpy(&copy_ecmd, ecmd, sizeof(copy_ecmd));
1387
1388	return rocker_cmd_exec(rocker_port, false,
1389			       rocker_cmd_set_port_settings_ethtool_prep,
1390			       &copy_ecmd, NULL, NULL);
1391}
1392
1393static int rocker_cmd_set_port_settings_macaddr(struct rocker_port *rocker_port,
1394						unsigned char *macaddr)
1395{
1396	return rocker_cmd_exec(rocker_port, false,
1397			       rocker_cmd_set_port_settings_macaddr_prep,
1398			       macaddr, NULL, NULL);
1399}
1400
1401static int rocker_cmd_set_port_settings_mtu(struct rocker_port *rocker_port,
1402					    int mtu)
1403{
1404	return rocker_cmd_exec(rocker_port, false,
1405			       rocker_cmd_set_port_settings_mtu_prep,
1406			       &mtu, NULL, NULL);
1407}
1408
1409int rocker_port_set_learning(struct rocker_port *rocker_port,
1410			     bool learning)
1411{
1412	return rocker_cmd_exec(rocker_port, false,
1413			       rocker_cmd_set_port_learning_prep,
1414			       &learning, NULL, NULL);
1415}
1416
1417/**********************
1418 * Worlds manipulation
1419 **********************/
1420
1421static struct rocker_world_ops *rocker_world_ops[] = {
1422	&rocker_ofdpa_ops,
1423};
1424
1425#define ROCKER_WORLD_OPS_LEN ARRAY_SIZE(rocker_world_ops)
1426
1427static struct rocker_world_ops *rocker_world_ops_find(u8 mode)
1428{
1429	int i;
1430
1431	for (i = 0; i < ROCKER_WORLD_OPS_LEN; i++)
1432		if (rocker_world_ops[i]->mode == mode)
1433			return rocker_world_ops[i];
1434	return NULL;
1435}
1436
1437static int rocker_world_init(struct rocker *rocker, u8 mode)
1438{
1439	struct rocker_world_ops *wops;
1440	int err;
1441
1442	wops = rocker_world_ops_find(mode);
1443	if (!wops) {
1444		dev_err(&rocker->pdev->dev, "port mode \"%d\" is not supported\n",
1445			mode);
1446		return -EINVAL;
1447	}
1448	rocker->wops = wops;
1449	rocker->wpriv = kzalloc(wops->priv_size, GFP_KERNEL);
1450	if (!rocker->wpriv)
1451		return -ENOMEM;
1452	if (!wops->init)
1453		return 0;
1454	err = wops->init(rocker);
1455	if (err)
1456		kfree(rocker->wpriv);
1457	return err;
1458}
1459
1460static void rocker_world_fini(struct rocker *rocker)
1461{
1462	struct rocker_world_ops *wops = rocker->wops;
1463
1464	if (!wops || !wops->fini)
1465		return;
1466	wops->fini(rocker);
1467	kfree(rocker->wpriv);
1468}
1469
1470static int rocker_world_check_init(struct rocker_port *rocker_port)
1471{
1472	struct rocker *rocker = rocker_port->rocker;
1473	u8 mode;
1474	int err;
1475
1476	err = rocker_cmd_get_port_settings_mode(rocker_port, &mode);
1477	if (err) {
1478		dev_err(&rocker->pdev->dev, "failed to get port mode\n");
1479		return err;
1480	}
1481	if (rocker->wops) {
1482		if (rocker->wops->mode != mode) {
1483			dev_err(&rocker->pdev->dev, "hardware has ports in different worlds, which is not supported\n");
1484			return -EINVAL;
1485		}
1486		return 0;
1487	}
1488	return rocker_world_init(rocker, mode);
1489}
1490
1491static int rocker_world_port_pre_init(struct rocker_port *rocker_port)
1492{
1493	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1494	int err;
1495
1496	rocker_port->wpriv = kzalloc(wops->port_priv_size, GFP_KERNEL);
1497	if (!rocker_port->wpriv)
1498		return -ENOMEM;
1499	if (!wops->port_pre_init)
1500		return 0;
1501	err = wops->port_pre_init(rocker_port);
1502	if (err)
1503		kfree(rocker_port->wpriv);
1504	return 0;
1505}
1506
1507static int rocker_world_port_init(struct rocker_port *rocker_port)
1508{
1509	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1510
1511	if (!wops->port_init)
1512		return 0;
1513	return wops->port_init(rocker_port);
1514}
1515
1516static void rocker_world_port_fini(struct rocker_port *rocker_port)
1517{
1518	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1519
1520	if (!wops->port_fini)
1521		return;
1522	wops->port_fini(rocker_port);
1523}
1524
1525static void rocker_world_port_post_fini(struct rocker_port *rocker_port)
1526{
1527	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1528
1529	if (!wops->port_post_fini)
1530		return;
1531	wops->port_post_fini(rocker_port);
1532	kfree(rocker_port->wpriv);
1533}
1534
1535static int rocker_world_port_open(struct rocker_port *rocker_port)
1536{
1537	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1538
1539	if (!wops->port_open)
1540		return 0;
1541	return wops->port_open(rocker_port);
1542}
1543
1544static void rocker_world_port_stop(struct rocker_port *rocker_port)
1545{
1546	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1547
1548	if (!wops->port_stop)
1549		return;
1550	wops->port_stop(rocker_port);
1551}
1552
1553static int rocker_world_port_attr_stp_state_set(struct rocker_port *rocker_port,
1554						u8 state,
1555						struct switchdev_trans *trans)
1556{
1557	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1558
1559	if (!wops->port_attr_stp_state_set)
1560		return -EOPNOTSUPP;
1561
1562	if (switchdev_trans_ph_prepare(trans))
1563		return 0;
1564
1565	return wops->port_attr_stp_state_set(rocker_port, state);
1566}
1567
1568static int
1569rocker_world_port_attr_bridge_flags_set(struct rocker_port *rocker_port,
1570					unsigned long brport_flags,
1571					struct switchdev_trans *trans)
1572{
1573	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1574
1575	if (!wops->port_attr_bridge_flags_set)
1576		return -EOPNOTSUPP;
1577
1578	if (switchdev_trans_ph_prepare(trans))
1579		return 0;
1580
1581	return wops->port_attr_bridge_flags_set(rocker_port, brport_flags,
1582						trans);
1583}
1584
1585static int
1586rocker_world_port_attr_bridge_flags_get(const struct rocker_port *rocker_port,
1587					unsigned long *p_brport_flags)
1588{
1589	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1590
1591	if (!wops->port_attr_bridge_flags_get)
1592		return -EOPNOTSUPP;
1593	return wops->port_attr_bridge_flags_get(rocker_port, p_brport_flags);
1594}
1595
1596static int
1597rocker_world_port_attr_bridge_flags_support_get(const struct rocker_port *
1598						rocker_port,
1599						unsigned long *
1600						p_brport_flags_support)
1601{
1602	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1603
1604	if (!wops->port_attr_bridge_flags_support_get)
1605		return -EOPNOTSUPP;
1606	return wops->port_attr_bridge_flags_support_get(rocker_port,
1607							p_brport_flags_support);
1608}
1609
1610static int
1611rocker_world_port_attr_bridge_ageing_time_set(struct rocker_port *rocker_port,
1612					      u32 ageing_time,
1613					      struct switchdev_trans *trans)
1614
1615{
1616	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1617
1618	if (!wops->port_attr_bridge_ageing_time_set)
1619		return -EOPNOTSUPP;
1620
1621	if (switchdev_trans_ph_prepare(trans))
1622		return 0;
1623
1624	return wops->port_attr_bridge_ageing_time_set(rocker_port, ageing_time,
1625						      trans);
1626}
1627
1628static int
1629rocker_world_port_obj_vlan_add(struct rocker_port *rocker_port,
1630			       const struct switchdev_obj_port_vlan *vlan,
1631			       struct switchdev_trans *trans)
1632{
1633	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1634
1635	if (!wops->port_obj_vlan_add)
1636		return -EOPNOTSUPP;
1637
1638	if (switchdev_trans_ph_prepare(trans))
1639		return 0;
1640
1641	return wops->port_obj_vlan_add(rocker_port, vlan);
1642}
1643
1644static int
1645rocker_world_port_obj_vlan_del(struct rocker_port *rocker_port,
1646			       const struct switchdev_obj_port_vlan *vlan)
1647{
1648	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1649
1650	if (!wops->port_obj_vlan_del)
1651		return -EOPNOTSUPP;
1652	return wops->port_obj_vlan_del(rocker_port, vlan);
1653}
1654
1655static int
1656rocker_world_port_fdb_add(struct rocker_port *rocker_port,
1657			  struct switchdev_notifier_fdb_info *info)
1658{
1659	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1660
1661	if (!wops->port_obj_fdb_add)
1662		return -EOPNOTSUPP;
1663
1664	return wops->port_obj_fdb_add(rocker_port, info->vid, info->addr);
1665}
1666
1667static int
1668rocker_world_port_fdb_del(struct rocker_port *rocker_port,
1669			  struct switchdev_notifier_fdb_info *info)
1670{
1671	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1672
1673	if (!wops->port_obj_fdb_del)
1674		return -EOPNOTSUPP;
1675	return wops->port_obj_fdb_del(rocker_port, info->vid, info->addr);
1676}
1677
1678static int rocker_world_port_master_linked(struct rocker_port *rocker_port,
1679					   struct net_device *master)
1680{
1681	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1682
1683	if (!wops->port_master_linked)
1684		return -EOPNOTSUPP;
1685	return wops->port_master_linked(rocker_port, master);
1686}
1687
1688static int rocker_world_port_master_unlinked(struct rocker_port *rocker_port,
1689					     struct net_device *master)
1690{
1691	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1692
1693	if (!wops->port_master_unlinked)
1694		return -EOPNOTSUPP;
1695	return wops->port_master_unlinked(rocker_port, master);
1696}
1697
1698static int rocker_world_port_neigh_update(struct rocker_port *rocker_port,
1699					  struct neighbour *n)
1700{
1701	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1702
1703	if (!wops->port_neigh_update)
1704		return -EOPNOTSUPP;
1705	return wops->port_neigh_update(rocker_port, n);
1706}
1707
1708static int rocker_world_port_neigh_destroy(struct rocker_port *rocker_port,
1709					   struct neighbour *n)
1710{
1711	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1712
1713	if (!wops->port_neigh_destroy)
1714		return -EOPNOTSUPP;
1715	return wops->port_neigh_destroy(rocker_port, n);
1716}
1717
1718static int rocker_world_port_ev_mac_vlan_seen(struct rocker_port *rocker_port,
1719					      const unsigned char *addr,
1720					      __be16 vlan_id)
1721{
1722	struct rocker_world_ops *wops = rocker_port->rocker->wops;
1723
1724	if (!wops->port_ev_mac_vlan_seen)
1725		return -EOPNOTSUPP;
1726	return wops->port_ev_mac_vlan_seen(rocker_port, addr, vlan_id);
1727}
1728
1729static int rocker_world_fib4_add(struct rocker *rocker,
1730				 const struct fib_entry_notifier_info *fen_info)
1731{
1732	struct rocker_world_ops *wops = rocker->wops;
1733
1734	if (!wops->fib4_add)
1735		return 0;
1736	return wops->fib4_add(rocker, fen_info);
1737}
1738
1739static int rocker_world_fib4_del(struct rocker *rocker,
1740				 const struct fib_entry_notifier_info *fen_info)
1741{
1742	struct rocker_world_ops *wops = rocker->wops;
1743
1744	if (!wops->fib4_del)
1745		return 0;
1746	return wops->fib4_del(rocker, fen_info);
1747}
1748
1749static void rocker_world_fib4_abort(struct rocker *rocker)
1750{
1751	struct rocker_world_ops *wops = rocker->wops;
1752
1753	if (wops->fib4_abort)
1754		wops->fib4_abort(rocker);
1755}
1756
1757/*****************
1758 * Net device ops
1759 *****************/
1760
1761static int rocker_port_open(struct net_device *dev)
1762{
1763	struct rocker_port *rocker_port = netdev_priv(dev);
1764	int err;
1765
1766	err = rocker_port_dma_rings_init(rocker_port);
1767	if (err)
1768		return err;
1769
1770	err = request_irq(rocker_msix_tx_vector(rocker_port),
1771			  rocker_tx_irq_handler, 0,
1772			  rocker_driver_name, rocker_port);
1773	if (err) {
1774		netdev_err(rocker_port->dev, "cannot assign tx irq\n");
1775		goto err_request_tx_irq;
1776	}
1777
1778	err = request_irq(rocker_msix_rx_vector(rocker_port),
1779			  rocker_rx_irq_handler, 0,
1780			  rocker_driver_name, rocker_port);
1781	if (err) {
1782		netdev_err(rocker_port->dev, "cannot assign rx irq\n");
1783		goto err_request_rx_irq;
1784	}
1785
1786	err = rocker_world_port_open(rocker_port);
1787	if (err) {
1788		netdev_err(rocker_port->dev, "cannot open port in world\n");
1789		goto err_world_port_open;
1790	}
1791
1792	napi_enable(&rocker_port->napi_tx);
1793	napi_enable(&rocker_port->napi_rx);
1794	if (!dev->proto_down)
1795		rocker_port_set_enable(rocker_port, true);
1796	netif_start_queue(dev);
1797	return 0;
1798
1799err_world_port_open:
1800	free_irq(rocker_msix_rx_vector(rocker_port), rocker_port);
1801err_request_rx_irq:
1802	free_irq(rocker_msix_tx_vector(rocker_port), rocker_port);
1803err_request_tx_irq:
1804	rocker_port_dma_rings_fini(rocker_port);
1805	return err;
1806}
1807
1808static int rocker_port_stop(struct net_device *dev)
1809{
1810	struct rocker_port *rocker_port = netdev_priv(dev);
1811
1812	netif_stop_queue(dev);
1813	rocker_port_set_enable(rocker_port, false);
1814	napi_disable(&rocker_port->napi_rx);
1815	napi_disable(&rocker_port->napi_tx);
1816	rocker_world_port_stop(rocker_port);
1817	free_irq(rocker_msix_rx_vector(rocker_port), rocker_port);
1818	free_irq(rocker_msix_tx_vector(rocker_port), rocker_port);
1819	rocker_port_dma_rings_fini(rocker_port);
1820
1821	return 0;
1822}
1823
1824static void rocker_tx_desc_frags_unmap(const struct rocker_port *rocker_port,
1825				       const struct rocker_desc_info *desc_info)
1826{
1827	const struct rocker *rocker = rocker_port->rocker;
1828	struct pci_dev *pdev = rocker->pdev;
1829	const struct rocker_tlv *attrs[ROCKER_TLV_TX_MAX + 1];
1830	struct rocker_tlv *attr;
1831	int rem;
1832
1833	rocker_tlv_parse_desc(attrs, ROCKER_TLV_TX_MAX, desc_info);
1834	if (!attrs[ROCKER_TLV_TX_FRAGS])
1835		return;
1836	rocker_tlv_for_each_nested(attr, attrs[ROCKER_TLV_TX_FRAGS], rem) {
1837		const struct rocker_tlv *frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_MAX + 1];
1838		dma_addr_t dma_handle;
1839		size_t len;
1840
1841		if (rocker_tlv_type(attr) != ROCKER_TLV_TX_FRAG)
1842			continue;
1843		rocker_tlv_parse_nested(frag_attrs, ROCKER_TLV_TX_FRAG_ATTR_MAX,
1844					attr);
1845		if (!frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_ADDR] ||
1846		    !frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_LEN])
1847			continue;
1848		dma_handle = rocker_tlv_get_u64(frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_ADDR]);
1849		len = rocker_tlv_get_u16(frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_LEN]);
1850		pci_unmap_single(pdev, dma_handle, len, DMA_TO_DEVICE);
1851	}
1852}
1853
1854static int rocker_tx_desc_frag_map_put(const struct rocker_port *rocker_port,
1855				       struct rocker_desc_info *desc_info,
1856				       char *buf, size_t buf_len)
1857{
1858	const struct rocker *rocker = rocker_port->rocker;
1859	struct pci_dev *pdev = rocker->pdev;
1860	dma_addr_t dma_handle;
1861	struct rocker_tlv *frag;
1862
1863	dma_handle = pci_map_single(pdev, buf, buf_len, DMA_TO_DEVICE);
1864	if (unlikely(pci_dma_mapping_error(pdev, dma_handle))) {
1865		if (net_ratelimit())
1866			netdev_err(rocker_port->dev, "failed to dma map tx frag\n");
1867		return -EIO;
1868	}
1869	frag = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAG);
1870	if (!frag)
1871		goto unmap_frag;
1872	if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_TX_FRAG_ATTR_ADDR,
1873			       dma_handle))
1874		goto nest_cancel;
1875	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_TX_FRAG_ATTR_LEN,
1876			       buf_len))
1877		goto nest_cancel;
1878	rocker_tlv_nest_end(desc_info, frag);
1879	return 0;
1880
1881nest_cancel:
1882	rocker_tlv_nest_cancel(desc_info, frag);
1883unmap_frag:
1884	pci_unmap_single(pdev, dma_handle, buf_len, DMA_TO_DEVICE);
1885	return -EMSGSIZE;
1886}
1887
1888static netdev_tx_t rocker_port_xmit(struct sk_buff *skb, struct net_device *dev)
1889{
1890	struct rocker_port *rocker_port = netdev_priv(dev);
1891	struct rocker *rocker = rocker_port->rocker;
1892	struct rocker_desc_info *desc_info;
1893	struct rocker_tlv *frags;
1894	int i;
1895	int err;
1896
1897	desc_info = rocker_desc_head_get(&rocker_port->tx_ring);
1898	if (unlikely(!desc_info)) {
1899		if (net_ratelimit())
1900			netdev_err(dev, "tx ring full when queue awake\n");
1901		return NETDEV_TX_BUSY;
1902	}
1903
1904	rocker_desc_cookie_ptr_set(desc_info, skb);
1905
1906	frags = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAGS);
1907	if (!frags)
1908		goto out;
1909	err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,
1910					  skb->data, skb_headlen(skb));
1911	if (err)
1912		goto nest_cancel;
1913	if (skb_shinfo(skb)->nr_frags > ROCKER_TX_FRAGS_MAX) {
1914		err = skb_linearize(skb);
1915		if (err)
1916			goto unmap_frags;
1917	}
1918
1919	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1920		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1921
1922		err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,
1923						  skb_frag_address(frag),
1924						  skb_frag_size(frag));
1925		if (err)
1926			goto unmap_frags;
1927	}
1928	rocker_tlv_nest_end(desc_info, frags);
1929
1930	rocker_desc_gen_clear(desc_info);
1931	rocker_desc_head_set(rocker, &rocker_port->tx_ring, desc_info);
1932
1933	desc_info = rocker_desc_head_get(&rocker_port->tx_ring);
1934	if (!desc_info)
1935		netif_stop_queue(dev);
1936
1937	return NETDEV_TX_OK;
1938
1939unmap_frags:
1940	rocker_tx_desc_frags_unmap(rocker_port, desc_info);
1941nest_cancel:
1942	rocker_tlv_nest_cancel(desc_info, frags);
1943out:
1944	dev_kfree_skb(skb);
1945	dev->stats.tx_dropped++;
1946
1947	return NETDEV_TX_OK;
1948}
1949
1950static int rocker_port_set_mac_address(struct net_device *dev, void *p)
1951{
1952	struct sockaddr *addr = p;
1953	struct rocker_port *rocker_port = netdev_priv(dev);
1954	int err;
1955
1956	if (!is_valid_ether_addr(addr->sa_data))
1957		return -EADDRNOTAVAIL;
1958
1959	err = rocker_cmd_set_port_settings_macaddr(rocker_port, addr->sa_data);
1960	if (err)
1961		return err;
1962	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1963	return 0;
1964}
1965
1966static int rocker_port_change_mtu(struct net_device *dev, int new_mtu)
1967{
1968	struct rocker_port *rocker_port = netdev_priv(dev);
1969	int running = netif_running(dev);
1970	int err;
1971
1972	if (running)
1973		rocker_port_stop(dev);
1974
1975	netdev_info(dev, "MTU change from %d to %d\n", dev->mtu, new_mtu);
1976	dev->mtu = new_mtu;
1977
1978	err = rocker_cmd_set_port_settings_mtu(rocker_port, new_mtu);
1979	if (err)
1980		return err;
1981
1982	if (running)
1983		err = rocker_port_open(dev);
1984
1985	return err;
1986}
1987
1988static int rocker_port_get_phys_port_name(struct net_device *dev,
1989					  char *buf, size_t len)
1990{
1991	struct rocker_port *rocker_port = netdev_priv(dev);
1992	struct port_name name = { .buf = buf, .len = len };
1993	int err;
1994
1995	err = rocker_cmd_exec(rocker_port, false,
1996			      rocker_cmd_get_port_settings_prep, NULL,
1997			      rocker_cmd_get_port_settings_phys_name_proc,
1998			      &name);
1999
2000	return err ? -EOPNOTSUPP : 0;
2001}
2002
2003static int rocker_port_change_proto_down(struct net_device *dev,
2004					 bool proto_down)
2005{
2006	struct rocker_port *rocker_port = netdev_priv(dev);
2007
2008	if (rocker_port->dev->flags & IFF_UP)
2009		rocker_port_set_enable(rocker_port, !proto_down);
2010	rocker_port->dev->proto_down = proto_down;
2011	return 0;
2012}
2013
2014static void rocker_port_neigh_destroy(struct net_device *dev,
2015				      struct neighbour *n)
2016{
2017	struct rocker_port *rocker_port = netdev_priv(n->dev);
2018	int err;
2019
2020	err = rocker_world_port_neigh_destroy(rocker_port, n);
2021	if (err)
2022		netdev_warn(rocker_port->dev, "failed to handle neigh destroy (err %d)\n",
2023			    err);
2024}
2025
2026static const struct net_device_ops rocker_port_netdev_ops = {
2027	.ndo_open			= rocker_port_open,
2028	.ndo_stop			= rocker_port_stop,
2029	.ndo_start_xmit			= rocker_port_xmit,
2030	.ndo_set_mac_address		= rocker_port_set_mac_address,
2031	.ndo_change_mtu			= rocker_port_change_mtu,
2032	.ndo_get_phys_port_name		= rocker_port_get_phys_port_name,
2033	.ndo_change_proto_down		= rocker_port_change_proto_down,
2034	.ndo_neigh_destroy		= rocker_port_neigh_destroy,
2035};
2036
2037/********************
2038 * swdev interface
2039 ********************/
2040
2041static int rocker_port_attr_get(struct net_device *dev,
2042				struct switchdev_attr *attr)
2043{
2044	const struct rocker_port *rocker_port = netdev_priv(dev);
2045	const struct rocker *rocker = rocker_port->rocker;
2046	int err = 0;
2047
2048	switch (attr->id) {
2049	case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
2050		attr->u.ppid.id_len = sizeof(rocker->hw.id);
2051		memcpy(&attr->u.ppid.id, &rocker->hw.id, attr->u.ppid.id_len);
2052		break;
2053	case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS:
2054		err = rocker_world_port_attr_bridge_flags_get(rocker_port,
2055							      &attr->u.brport_flags);
2056		break;
2057	case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS_SUPPORT:
2058		err = rocker_world_port_attr_bridge_flags_support_get(rocker_port,
2059								      &attr->u.brport_flags_support);
2060		break;
2061	default:
2062		return -EOPNOTSUPP;
2063	}
2064
2065	return err;
2066}
2067
2068static int rocker_port_attr_set(struct net_device *dev,
2069				const struct switchdev_attr *attr,
2070				struct switchdev_trans *trans)
2071{
2072	struct rocker_port *rocker_port = netdev_priv(dev);
2073	int err = 0;
2074
2075	switch (attr->id) {
2076	case SWITCHDEV_ATTR_ID_PORT_STP_STATE:
2077		err = rocker_world_port_attr_stp_state_set(rocker_port,
2078							   attr->u.stp_state,
2079							   trans);
2080		break;
2081	case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS:
2082		err = rocker_world_port_attr_bridge_flags_set(rocker_port,
2083							      attr->u.brport_flags,
2084							      trans);
2085		break;
2086	case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME:
2087		err = rocker_world_port_attr_bridge_ageing_time_set(rocker_port,
2088								    attr->u.ageing_time,
2089								    trans);
2090		break;
2091	default:
2092		err = -EOPNOTSUPP;
2093		break;
2094	}
2095
2096	return err;
2097}
2098
2099static int rocker_port_obj_add(struct net_device *dev,
2100			       const struct switchdev_obj *obj,
2101			       struct switchdev_trans *trans)
2102{
2103	struct rocker_port *rocker_port = netdev_priv(dev);
2104	int err = 0;
2105
2106	switch (obj->id) {
2107	case SWITCHDEV_OBJ_ID_PORT_VLAN:
2108		err = rocker_world_port_obj_vlan_add(rocker_port,
2109						     SWITCHDEV_OBJ_PORT_VLAN(obj),
2110						     trans);
2111		break;
2112	default:
2113		err = -EOPNOTSUPP;
2114		break;
2115	}
2116
2117	return err;
2118}
2119
2120static int rocker_port_obj_del(struct net_device *dev,
2121			       const struct switchdev_obj *obj)
2122{
2123	struct rocker_port *rocker_port = netdev_priv(dev);
2124	int err = 0;
2125
2126	switch (obj->id) {
2127	case SWITCHDEV_OBJ_ID_PORT_VLAN:
2128		err = rocker_world_port_obj_vlan_del(rocker_port,
2129						     SWITCHDEV_OBJ_PORT_VLAN(obj));
2130		break;
2131	default:
2132		err = -EOPNOTSUPP;
2133		break;
2134	}
2135
2136	return err;
2137}
2138
2139static const struct switchdev_ops rocker_port_switchdev_ops = {
2140	.switchdev_port_attr_get	= rocker_port_attr_get,
2141	.switchdev_port_attr_set	= rocker_port_attr_set,
2142	.switchdev_port_obj_add		= rocker_port_obj_add,
2143	.switchdev_port_obj_del		= rocker_port_obj_del,
2144};
2145
2146struct rocker_fib_event_work {
2147	struct work_struct work;
2148	union {
2149		struct fib_entry_notifier_info fen_info;
2150		struct fib_rule_notifier_info fr_info;
2151	};
2152	struct rocker *rocker;
2153	unsigned long event;
2154};
2155
2156static void rocker_router_fib_event_work(struct work_struct *work)
2157{
2158	struct rocker_fib_event_work *fib_work =
2159		container_of(work, struct rocker_fib_event_work, work);
2160	struct rocker *rocker = fib_work->rocker;
2161	struct fib_rule *rule;
2162	int err;
2163
2164	/* Protect internal structures from changes */
2165	rtnl_lock();
2166	switch (fib_work->event) {
2167	case FIB_EVENT_ENTRY_ADD:
2168		err = rocker_world_fib4_add(rocker, &fib_work->fen_info);
2169		if (err)
2170			rocker_world_fib4_abort(rocker);
2171		fib_info_put(fib_work->fen_info.fi);
2172		break;
2173	case FIB_EVENT_ENTRY_DEL:
2174		rocker_world_fib4_del(rocker, &fib_work->fen_info);
2175		fib_info_put(fib_work->fen_info.fi);
2176		break;
2177	case FIB_EVENT_RULE_ADD: /* fall through */
2178	case FIB_EVENT_RULE_DEL:
2179		rule = fib_work->fr_info.rule;
2180		if (!fib4_rule_default(rule))
2181			rocker_world_fib4_abort(rocker);
2182		fib_rule_put(rule);
2183		break;
2184	}
2185	rtnl_unlock();
2186	kfree(fib_work);
2187}
2188
2189/* Called with rcu_read_lock() */
2190static int rocker_router_fib_event(struct notifier_block *nb,
2191				   unsigned long event, void *ptr)
2192{
2193	struct rocker *rocker = container_of(nb, struct rocker, fib_nb);
2194	struct rocker_fib_event_work *fib_work;
2195	struct fib_notifier_info *info = ptr;
2196
2197	if (info->family != AF_INET)
2198		return NOTIFY_DONE;
2199
2200	fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2201	if (WARN_ON(!fib_work))
2202		return NOTIFY_BAD;
2203
2204	INIT_WORK(&fib_work->work, rocker_router_fib_event_work);
2205	fib_work->rocker = rocker;
2206	fib_work->event = event;
2207
2208	switch (event) {
2209	case FIB_EVENT_ENTRY_ADD: /* fall through */
2210	case FIB_EVENT_ENTRY_DEL:
2211		memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2212		/* Take referece on fib_info to prevent it from being
2213		 * freed while work is queued. Release it afterwards.
2214		 */
2215		fib_info_hold(fib_work->fen_info.fi);
2216		break;
2217	case FIB_EVENT_RULE_ADD: /* fall through */
2218	case FIB_EVENT_RULE_DEL:
2219		memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2220		fib_rule_get(fib_work->fr_info.rule);
2221		break;
2222	}
2223
2224	queue_work(rocker->rocker_owq, &fib_work->work);
2225
2226	return NOTIFY_DONE;
2227}
2228
2229/********************
2230 * ethtool interface
2231 ********************/
2232
2233static int
2234rocker_port_get_link_ksettings(struct net_device *dev,
2235			       struct ethtool_link_ksettings *ecmd)
2236{
2237	struct rocker_port *rocker_port = netdev_priv(dev);
2238
2239	return rocker_cmd_get_port_settings_ethtool(rocker_port, ecmd);
2240}
2241
2242static int
2243rocker_port_set_link_ksettings(struct net_device *dev,
2244			       const struct ethtool_link_ksettings *ecmd)
2245{
2246	struct rocker_port *rocker_port = netdev_priv(dev);
2247
2248	return rocker_cmd_set_port_settings_ethtool(rocker_port, ecmd);
2249}
2250
2251static void rocker_port_get_drvinfo(struct net_device *dev,
2252				    struct ethtool_drvinfo *drvinfo)
2253{
2254	strlcpy(drvinfo->driver, rocker_driver_name, sizeof(drvinfo->driver));
2255	strlcpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version));
2256}
2257
2258static struct rocker_port_stats {
2259	char str[ETH_GSTRING_LEN];
2260	int type;
2261} rocker_port_stats[] = {
2262	{ "rx_packets", ROCKER_TLV_CMD_PORT_STATS_RX_PKTS,    },
2263	{ "rx_bytes",   ROCKER_TLV_CMD_PORT_STATS_RX_BYTES,   },
2264	{ "rx_dropped", ROCKER_TLV_CMD_PORT_STATS_RX_DROPPED, },
2265	{ "rx_errors",  ROCKER_TLV_CMD_PORT_STATS_RX_ERRORS,  },
2266
2267	{ "tx_packets", ROCKER_TLV_CMD_PORT_STATS_TX_PKTS,    },
2268	{ "tx_bytes",   ROCKER_TLV_CMD_PORT_STATS_TX_BYTES,   },
2269	{ "tx_dropped", ROCKER_TLV_CMD_PORT_STATS_TX_DROPPED, },
2270	{ "tx_errors",  ROCKER_TLV_CMD_PORT_STATS_TX_ERRORS,  },
2271};
2272
2273#define ROCKER_PORT_STATS_LEN  ARRAY_SIZE(rocker_port_stats)
2274
2275static void rocker_port_get_strings(struct net_device *netdev, u32 stringset,
2276				    u8 *data)
2277{
2278	u8 *p = data;
2279	int i;
2280
2281	switch (stringset) {
2282	case ETH_SS_STATS:
2283		for (i = 0; i < ARRAY_SIZE(rocker_port_stats); i++) {
2284			memcpy(p, rocker_port_stats[i].str, ETH_GSTRING_LEN);
2285			p += ETH_GSTRING_LEN;
2286		}
2287		break;
2288	}
2289}
2290
2291static int
2292rocker_cmd_get_port_stats_prep(const struct rocker_port *rocker_port,
2293			       struct rocker_desc_info *desc_info,
2294			       void *priv)
2295{
2296	struct rocker_tlv *cmd_stats;
2297
2298	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
2299			       ROCKER_TLV_CMD_TYPE_GET_PORT_STATS))
2300		return -EMSGSIZE;
2301
2302	cmd_stats = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
2303	if (!cmd_stats)
2304		return -EMSGSIZE;
2305
2306	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_STATS_PPORT,
2307			       rocker_port->pport))
2308		return -EMSGSIZE;
2309
2310	rocker_tlv_nest_end(desc_info, cmd_stats);
2311
2312	return 0;
2313}
2314
2315static int
2316rocker_cmd_get_port_stats_ethtool_proc(const struct rocker_port *rocker_port,
2317				       const struct rocker_desc_info *desc_info,
2318				       void *priv)
2319{
2320	const struct rocker_tlv *attrs[ROCKER_TLV_CMD_MAX + 1];
2321	const struct rocker_tlv *stats_attrs[ROCKER_TLV_CMD_PORT_STATS_MAX + 1];
2322	const struct rocker_tlv *pattr;
2323	u32 pport;
2324	u64 *data = priv;
2325	int i;
2326
2327	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);
2328
2329	if (!attrs[ROCKER_TLV_CMD_INFO])
2330		return -EIO;
2331
2332	rocker_tlv_parse_nested(stats_attrs, ROCKER_TLV_CMD_PORT_STATS_MAX,
2333				attrs[ROCKER_TLV_CMD_INFO]);
2334
2335	if (!stats_attrs[ROCKER_TLV_CMD_PORT_STATS_PPORT])
2336		return -EIO;
2337
2338	pport = rocker_tlv_get_u32(stats_attrs[ROCKER_TLV_CMD_PORT_STATS_PPORT]);
2339	if (pport != rocker_port->pport)
2340		return -EIO;
2341
2342	for (i = 0; i < ARRAY_SIZE(rocker_port_stats); i++) {
2343		pattr = stats_attrs[rocker_port_stats[i].type];
2344		if (!pattr)
2345			continue;
2346
2347		data[i] = rocker_tlv_get_u64(pattr);
2348	}
2349
2350	return 0;
2351}
2352
2353static int rocker_cmd_get_port_stats_ethtool(struct rocker_port *rocker_port,
2354					     void *priv)
2355{
2356	return rocker_cmd_exec(rocker_port, false,
2357			       rocker_cmd_get_port_stats_prep, NULL,
2358			       rocker_cmd_get_port_stats_ethtool_proc,
2359			       priv);
2360}
2361
2362static void rocker_port_get_stats(struct net_device *dev,
2363				  struct ethtool_stats *stats, u64 *data)
2364{
2365	struct rocker_port *rocker_port = netdev_priv(dev);
2366
2367	if (rocker_cmd_get_port_stats_ethtool(rocker_port, data) != 0) {
2368		int i;
2369
2370		for (i = 0; i < ARRAY_SIZE(rocker_port_stats); ++i)
2371			data[i] = 0;
2372	}
2373}
2374
2375static int rocker_port_get_sset_count(struct net_device *netdev, int sset)
2376{
2377	switch (sset) {
2378	case ETH_SS_STATS:
2379		return ROCKER_PORT_STATS_LEN;
2380	default:
2381		return -EOPNOTSUPP;
2382	}
2383}
2384
2385static const struct ethtool_ops rocker_port_ethtool_ops = {
2386	.get_drvinfo		= rocker_port_get_drvinfo,
2387	.get_link		= ethtool_op_get_link,
2388	.get_strings		= rocker_port_get_strings,
2389	.get_ethtool_stats	= rocker_port_get_stats,
2390	.get_sset_count		= rocker_port_get_sset_count,
2391	.get_link_ksettings	= rocker_port_get_link_ksettings,
2392	.set_link_ksettings	= rocker_port_set_link_ksettings,
2393};
2394
2395/*****************
2396 * NAPI interface
2397 *****************/
2398
2399static struct rocker_port *rocker_port_napi_tx_get(struct napi_struct *napi)
2400{
2401	return container_of(napi, struct rocker_port, napi_tx);
2402}
2403
2404static int rocker_port_poll_tx(struct napi_struct *napi, int budget)
2405{
2406	struct rocker_port *rocker_port = rocker_port_napi_tx_get(napi);
2407	const struct rocker *rocker = rocker_port->rocker;
2408	const struct rocker_desc_info *desc_info;
2409	u32 credits = 0;
2410	int err;
2411
2412	/* Cleanup tx descriptors */
2413	while ((desc_info = rocker_desc_tail_get(&rocker_port->tx_ring))) {
2414		struct sk_buff *skb;
2415
2416		err = rocker_desc_err(desc_info);
2417		if (err && net_ratelimit())
2418			netdev_err(rocker_port->dev, "tx desc received with err %d\n",
2419				   err);
2420		rocker_tx_desc_frags_unmap(rocker_port, desc_info);
2421
2422		skb = rocker_desc_cookie_ptr_get(desc_info);
2423		if (err == 0) {
2424			rocker_port->dev->stats.tx_packets++;
2425			rocker_port->dev->stats.tx_bytes += skb->len;
2426		} else {
2427			rocker_port->dev->stats.tx_errors++;
2428		}
2429
2430		dev_kfree_skb_any(skb);
2431		credits++;
2432	}
2433
2434	if (credits && netif_queue_stopped(rocker_port->dev))
2435		netif_wake_queue(rocker_port->dev);
2436
2437	napi_complete(napi);
2438	rocker_dma_ring_credits_set(rocker, &rocker_port->tx_ring, credits);
2439
2440	return 0;
2441}
2442
2443static int rocker_port_rx_proc(const struct rocker *rocker,
2444			       const struct rocker_port *rocker_port,
2445			       struct rocker_desc_info *desc_info)
2446{
2447	const struct rocker_tlv *attrs[ROCKER_TLV_RX_MAX + 1];
2448	struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);
2449	size_t rx_len;
2450	u16 rx_flags = 0;
2451
2452	if (!skb)
2453		return -ENOENT;
2454
2455	rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);
2456	if (!attrs[ROCKER_TLV_RX_FRAG_LEN])
2457		return -EINVAL;
2458	if (attrs[ROCKER_TLV_RX_FLAGS])
2459		rx_flags = rocker_tlv_get_u16(attrs[ROCKER_TLV_RX_FLAGS]);
2460
2461	rocker_dma_rx_ring_skb_unmap(rocker, attrs);
2462
2463	rx_len = rocker_tlv_get_u16(attrs[ROCKER_TLV_RX_FRAG_LEN]);
2464	skb_put(skb, rx_len);
2465	skb->protocol = eth_type_trans(skb, rocker_port->dev);
2466
2467	if (rx_flags & ROCKER_RX_FLAGS_FWD_OFFLOAD)
2468		skb->offload_fwd_mark = 1;
2469
2470	rocker_port->dev->stats.rx_packets++;
2471	rocker_port->dev->stats.rx_bytes += skb->len;
2472
2473	netif_receive_skb(skb);
2474
2475	return rocker_dma_rx_ring_skb_alloc(rocker_port, desc_info);
2476}
2477
2478static struct rocker_port *rocker_port_napi_rx_get(struct napi_struct *napi)
2479{
2480	return container_of(napi, struct rocker_port, napi_rx);
2481}
2482
2483static int rocker_port_poll_rx(struct napi_struct *napi, int budget)
2484{
2485	struct rocker_port *rocker_port = rocker_port_napi_rx_get(napi);
2486	const struct rocker *rocker = rocker_port->rocker;
2487	struct rocker_desc_info *desc_info;
2488	u32 credits = 0;
2489	int err;
2490
2491	/* Process rx descriptors */
2492	while (credits < budget &&
2493	       (desc_info = rocker_desc_tail_get(&rocker_port->rx_ring))) {
2494		err = rocker_desc_err(desc_info);
2495		if (err) {
2496			if (net_ratelimit())
2497				netdev_err(rocker_port->dev, "rx desc received with err %d\n",
2498					   err);
2499		} else {
2500			err = rocker_port_rx_proc(rocker, rocker_port,
2501						  desc_info);
2502			if (err && net_ratelimit())
2503				netdev_err(rocker_port->dev, "rx processing failed with err %d\n",
2504					   err);
2505		}
2506		if (err)
2507			rocker_port->dev->stats.rx_errors++;
2508
2509		rocker_desc_gen_clear(desc_info);
2510		rocker_desc_head_set(rocker, &rocker_port->rx_ring, desc_info);
2511		credits++;
2512	}
2513
2514	if (credits < budget)
2515		napi_complete_done(napi, credits);
2516
2517	rocker_dma_ring_credits_set(rocker, &rocker_port->rx_ring, credits);
2518
2519	return credits;
2520}
2521
2522/*****************
2523 * PCI driver ops
2524 *****************/
2525
2526static void rocker_carrier_init(const struct rocker_port *rocker_port)
2527{
2528	const struct rocker *rocker = rocker_port->rocker;
2529	u64 link_status = rocker_read64(rocker, PORT_PHYS_LINK_STATUS);
2530	bool link_up;
2531
2532	link_up = link_status & (1 << rocker_port->pport);
2533	if (link_up)
2534		netif_carrier_on(rocker_port->dev);
2535	else
2536		netif_carrier_off(rocker_port->dev);
2537}
2538
2539static void rocker_remove_ports(struct rocker *rocker)
2540{
2541	struct rocker_port *rocker_port;
2542	int i;
2543
2544	for (i = 0; i < rocker->port_count; i++) {
2545		rocker_port = rocker->ports[i];
2546		if (!rocker_port)
2547			continue;
2548		rocker_world_port_fini(rocker_port);
2549		unregister_netdev(rocker_port->dev);
2550		rocker_world_port_post_fini(rocker_port);
2551		free_netdev(rocker_port->dev);
2552	}
2553	rocker_world_fini(rocker);
2554	kfree(rocker->ports);
2555}
2556
2557static void rocker_port_dev_addr_init(struct rocker_port *rocker_port)
2558{
2559	const struct rocker *rocker = rocker_port->rocker;
2560	const struct pci_dev *pdev = rocker->pdev;
2561	int err;
2562
2563	err = rocker_cmd_get_port_settings_macaddr(rocker_port,
2564						   rocker_port->dev->dev_addr);
2565	if (err) {
2566		dev_warn(&pdev->dev, "failed to get mac address, using random\n");
2567		eth_hw_addr_random(rocker_port->dev);
2568	}
2569}
2570
2571#define ROCKER_PORT_MIN_MTU	ETH_MIN_MTU
2572#define ROCKER_PORT_MAX_MTU	9000
2573static int rocker_probe_port(struct rocker *rocker, unsigned int port_number)
2574{
2575	struct pci_dev *pdev = rocker->pdev;
2576	struct rocker_port *rocker_port;
2577	struct net_device *dev;
2578	int err;
2579
2580	dev = alloc_etherdev(sizeof(struct rocker_port));
2581	if (!dev)
2582		return -ENOMEM;
2583	SET_NETDEV_DEV(dev, &pdev->dev);
2584	rocker_port = netdev_priv(dev);
2585	rocker_port->dev = dev;
2586	rocker_port->rocker = rocker;
2587	rocker_port->port_number = port_number;
2588	rocker_port->pport = port_number + 1;
2589
2590	err = rocker_world_check_init(rocker_port);
2591	if (err) {
2592		dev_err(&pdev->dev, "world init failed\n");
2593		goto err_world_check_init;
2594	}
2595
2596	rocker_port_dev_addr_init(rocker_port);
2597	dev->netdev_ops = &rocker_port_netdev_ops;
2598	dev->ethtool_ops = &rocker_port_ethtool_ops;
2599	dev->switchdev_ops = &rocker_port_switchdev_ops;
2600	netif_tx_napi_add(dev, &rocker_port->napi_tx, rocker_port_poll_tx,
2601			  NAPI_POLL_WEIGHT);
2602	netif_napi_add(dev, &rocker_port->napi_rx, rocker_port_poll_rx,
2603		       NAPI_POLL_WEIGHT);
2604	rocker_carrier_init(rocker_port);
2605
2606	dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_SG;
2607
2608	/* MTU range: 68 - 9000 */
2609	dev->min_mtu = ROCKER_PORT_MIN_MTU;
2610	dev->max_mtu = ROCKER_PORT_MAX_MTU;
2611
2612	err = rocker_world_port_pre_init(rocker_port);
2613	if (err) {
2614		dev_err(&pdev->dev, "port world pre-init failed\n");
2615		goto err_world_port_pre_init;
2616	}
2617	err = register_netdev(dev);
2618	if (err) {
2619		dev_err(&pdev->dev, "register_netdev failed\n");
2620		goto err_register_netdev;
2621	}
2622	rocker->ports[port_number] = rocker_port;
2623
2624	err = rocker_world_port_init(rocker_port);
2625	if (err) {
2626		dev_err(&pdev->dev, "port world init failed\n");
2627		goto err_world_port_init;
2628	}
2629
2630	return 0;
2631
2632err_world_port_init:
2633	rocker->ports[port_number] = NULL;
2634	unregister_netdev(dev);
2635err_register_netdev:
2636	rocker_world_port_post_fini(rocker_port);
2637err_world_port_pre_init:
2638err_world_check_init:
2639	free_netdev(dev);
2640	return err;
2641}
2642
2643static int rocker_probe_ports(struct rocker *rocker)
2644{
2645	int i;
2646	size_t alloc_size;
2647	int err;
2648
2649	alloc_size = sizeof(struct rocker_port *) * rocker->port_count;
2650	rocker->ports = kzalloc(alloc_size, GFP_KERNEL);
2651	if (!rocker->ports)
2652		return -ENOMEM;
2653	for (i = 0; i < rocker->port_count; i++) {
2654		err = rocker_probe_port(rocker, i);
2655		if (err)
2656			goto remove_ports;
2657	}
2658	return 0;
2659
2660remove_ports:
2661	rocker_remove_ports(rocker);
2662	return err;
2663}
2664
2665static int rocker_msix_init(struct rocker *rocker)
2666{
2667	struct pci_dev *pdev = rocker->pdev;
2668	int msix_entries;
2669	int i;
2670	int err;
2671
2672	msix_entries = pci_msix_vec_count(pdev);
2673	if (msix_entries < 0)
2674		return msix_entries;
2675
2676	if (msix_entries != ROCKER_MSIX_VEC_COUNT(rocker->port_count))
2677		return -EINVAL;
2678
2679	rocker->msix_entries = kmalloc_array(msix_entries,
2680					     sizeof(struct msix_entry),
2681					     GFP_KERNEL);
2682	if (!rocker->msix_entries)
2683		return -ENOMEM;
2684
2685	for (i = 0; i < msix_entries; i++)
2686		rocker->msix_entries[i].entry = i;
2687
2688	err = pci_enable_msix_exact(pdev, rocker->msix_entries, msix_entries);
2689	if (err < 0)
2690		goto err_enable_msix;
2691
2692	return 0;
2693
2694err_enable_msix:
2695	kfree(rocker->msix_entries);
2696	return err;
2697}
2698
2699static void rocker_msix_fini(const struct rocker *rocker)
2700{
2701	pci_disable_msix(rocker->pdev);
2702	kfree(rocker->msix_entries);
2703}
2704
2705static bool rocker_port_dev_check(const struct net_device *dev)
2706{
2707	return dev->netdev_ops == &rocker_port_netdev_ops;
2708}
2709
2710struct rocker_switchdev_event_work {
2711	struct work_struct work;
2712	struct switchdev_notifier_fdb_info fdb_info;
2713	struct rocker_port *rocker_port;
2714	unsigned long event;
2715};
2716
2717static void
2718rocker_fdb_offload_notify(struct rocker_port *rocker_port,
2719			  struct switchdev_notifier_fdb_info *recv_info)
2720{
2721	struct switchdev_notifier_fdb_info info;
2722
2723	info.addr = recv_info->addr;
2724	info.vid = recv_info->vid;
2725	call_switchdev_notifiers(SWITCHDEV_FDB_OFFLOADED,
2726				 rocker_port->dev, &info.info);
2727}
2728
2729static void rocker_switchdev_event_work(struct work_struct *work)
2730{
2731	struct rocker_switchdev_event_work *switchdev_work =
2732		container_of(work, struct rocker_switchdev_event_work, work);
2733	struct rocker_port *rocker_port = switchdev_work->rocker_port;
2734	struct switchdev_notifier_fdb_info *fdb_info;
2735	int err;
2736
2737	rtnl_lock();
2738	switch (switchdev_work->event) {
2739	case SWITCHDEV_FDB_ADD_TO_DEVICE:
2740		fdb_info = &switchdev_work->fdb_info;
2741		err = rocker_world_port_fdb_add(rocker_port, fdb_info);
2742		if (err) {
2743			netdev_dbg(rocker_port->dev, "fdb add failed err=%d\n", err);
2744			break;
2745		}
2746		rocker_fdb_offload_notify(rocker_port, fdb_info);
2747		break;
2748	case SWITCHDEV_FDB_DEL_TO_DEVICE:
2749		fdb_info = &switchdev_work->fdb_info;
2750		err = rocker_world_port_fdb_del(rocker_port, fdb_info);
2751		if (err)
2752			netdev_dbg(rocker_port->dev, "fdb add failed err=%d\n", err);
2753		break;
2754	}
2755	rtnl_unlock();
2756
2757	kfree(switchdev_work->fdb_info.addr);
2758	kfree(switchdev_work);
2759	dev_put(rocker_port->dev);
2760}
2761
2762/* called under rcu_read_lock() */
2763static int rocker_switchdev_event(struct notifier_block *unused,
2764				  unsigned long event, void *ptr)
2765{
2766	struct net_device *dev = switchdev_notifier_info_to_dev(ptr);
2767	struct rocker_switchdev_event_work *switchdev_work;
2768	struct switchdev_notifier_fdb_info *fdb_info = ptr;
2769	struct rocker_port *rocker_port;
2770
2771	if (!rocker_port_dev_check(dev))
2772		return NOTIFY_DONE;
2773
2774	rocker_port = netdev_priv(dev);
2775	switchdev_work = kzalloc(sizeof(*switchdev_work), GFP_ATOMIC);
2776	if (WARN_ON(!switchdev_work))
2777		return NOTIFY_BAD;
2778
2779	INIT_WORK(&switchdev_work->work, rocker_switchdev_event_work);
2780	switchdev_work->rocker_port = rocker_port;
2781	switchdev_work->event = event;
2782
2783	switch (event) {
2784	case SWITCHDEV_FDB_ADD_TO_DEVICE: /* fall through */
2785	case SWITCHDEV_FDB_DEL_TO_DEVICE:
2786		memcpy(&switchdev_work->fdb_info, ptr,
2787		       sizeof(switchdev_work->fdb_info));
2788		switchdev_work->fdb_info.addr = kzalloc(ETH_ALEN, GFP_ATOMIC);
2789		ether_addr_copy((u8 *)switchdev_work->fdb_info.addr,
2790				fdb_info->addr);
2791		/* Take a reference on the rocker device */
2792		dev_hold(dev);
2793		break;
2794	default:
2795		kfree(switchdev_work);
2796		return NOTIFY_DONE;
2797	}
2798
2799	queue_work(rocker_port->rocker->rocker_owq,
2800		   &switchdev_work->work);
2801	return NOTIFY_DONE;
2802}
2803
2804static struct notifier_block rocker_switchdev_notifier = {
2805	.notifier_call = rocker_switchdev_event,
2806};
2807
2808static int rocker_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2809{
2810	struct rocker *rocker;
2811	int err;
2812
2813	rocker = kzalloc(sizeof(*rocker), GFP_KERNEL);
2814	if (!rocker)
2815		return -ENOMEM;
2816
2817	err = pci_enable_device(pdev);
2818	if (err) {
2819		dev_err(&pdev->dev, "pci_enable_device failed\n");
2820		goto err_pci_enable_device;
2821	}
2822
2823	err = pci_request_regions(pdev, rocker_driver_name);
2824	if (err) {
2825		dev_err(&pdev->dev, "pci_request_regions failed\n");
2826		goto err_pci_request_regions;
2827	}
2828
2829	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2830	if (!err) {
2831		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2832		if (err) {
2833			dev_err(&pdev->dev, "pci_set_consistent_dma_mask failed\n");
2834			goto err_pci_set_dma_mask;
2835		}
2836	} else {
2837		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2838		if (err) {
2839			dev_err(&pdev->dev, "pci_set_dma_mask failed\n");
2840			goto err_pci_set_dma_mask;
2841		}
2842	}
2843
2844	if (pci_resource_len(pdev, 0) < ROCKER_PCI_BAR0_SIZE) {
2845		dev_err(&pdev->dev, "invalid PCI region size\n");
2846		err = -EINVAL;
2847		goto err_pci_resource_len_check;
2848	}
2849
2850	rocker->hw_addr = ioremap(pci_resource_start(pdev, 0),
2851				  pci_resource_len(pdev, 0));
2852	if (!rocker->hw_addr) {
2853		dev_err(&pdev->dev, "ioremap failed\n");
2854		err = -EIO;
2855		goto err_ioremap;
2856	}
2857	pci_set_master(pdev);
2858
2859	rocker->pdev = pdev;
2860	pci_set_drvdata(pdev, rocker);
2861
2862	rocker->port_count = rocker_read32(rocker, PORT_PHYS_COUNT);
2863
2864	err = rocker_msix_init(rocker);
2865	if (err) {
2866		dev_err(&pdev->dev, "MSI-X init failed\n");
2867		goto err_msix_init;
2868	}
2869
2870	err = rocker_basic_hw_test(rocker);
2871	if (err) {
2872		dev_err(&pdev->dev, "basic hw test failed\n");
2873		goto err_basic_hw_test;
2874	}
2875
2876	rocker_write32(rocker, CONTROL, ROCKER_CONTROL_RESET);
2877
2878	err = rocker_dma_rings_init(rocker);
2879	if (err)
2880		goto err_dma_rings_init;
2881
2882	err = request_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_CMD),
2883			  rocker_cmd_irq_handler, 0,
2884			  rocker_driver_name, rocker);
2885	if (err) {
2886		dev_err(&pdev->dev, "cannot assign cmd irq\n");
2887		goto err_request_cmd_irq;
2888	}
2889
2890	err = request_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT),
2891			  rocker_event_irq_handler, 0,
2892			  rocker_driver_name, rocker);
2893	if (err) {
2894		dev_err(&pdev->dev, "cannot assign event irq\n");
2895		goto err_request_event_irq;
2896	}
2897
2898	rocker->rocker_owq = alloc_ordered_workqueue(rocker_driver_name,
2899						     WQ_MEM_RECLAIM);
2900	if (!rocker->rocker_owq) {
2901		err = -ENOMEM;
2902		goto err_alloc_ordered_workqueue;
2903	}
2904
2905	err = rocker_probe_ports(rocker);
2906	if (err) {
2907		dev_err(&pdev->dev, "failed to probe ports\n");
2908		goto err_probe_ports;
2909	}
2910
2911	/* Only FIBs pointing to our own netdevs are programmed into
2912	 * the device, so no need to pass a callback.
2913	 */
2914	rocker->fib_nb.notifier_call = rocker_router_fib_event;
2915	err = register_fib_notifier(&rocker->fib_nb, NULL);
2916	if (err)
2917		goto err_register_fib_notifier;
2918
2919	err = register_switchdev_notifier(&rocker_switchdev_notifier);
2920	if (err) {
2921		dev_err(&pdev->dev, "Failed to register switchdev notifier\n");
2922		goto err_register_switchdev_notifier;
2923	}
2924
2925	rocker->hw.id = rocker_read64(rocker, SWITCH_ID);
2926
2927	dev_info(&pdev->dev, "Rocker switch with id %*phN\n",
2928		 (int)sizeof(rocker->hw.id), &rocker->hw.id);
2929
2930	return 0;
2931
2932err_register_switchdev_notifier:
2933	unregister_fib_notifier(&rocker->fib_nb);
2934err_register_fib_notifier:
2935	rocker_remove_ports(rocker);
2936err_probe_ports:
2937	destroy_workqueue(rocker->rocker_owq);
2938err_alloc_ordered_workqueue:
2939	free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT), rocker);
2940err_request_event_irq:
2941	free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_CMD), rocker);
2942err_request_cmd_irq:
2943	rocker_dma_rings_fini(rocker);
2944err_dma_rings_init:
2945err_basic_hw_test:
2946	rocker_msix_fini(rocker);
2947err_msix_init:
2948	iounmap(rocker->hw_addr);
2949err_ioremap:
2950err_pci_resource_len_check:
2951err_pci_set_dma_mask:
2952	pci_release_regions(pdev);
2953err_pci_request_regions:
2954	pci_disable_device(pdev);
2955err_pci_enable_device:
2956	kfree(rocker);
2957	return err;
2958}
2959
2960static void rocker_remove(struct pci_dev *pdev)
2961{
2962	struct rocker *rocker = pci_get_drvdata(pdev);
2963
2964	unregister_switchdev_notifier(&rocker_switchdev_notifier);
2965	unregister_fib_notifier(&rocker->fib_nb);
2966	rocker_remove_ports(rocker);
2967	rocker_write32(rocker, CONTROL, ROCKER_CONTROL_RESET);
2968	destroy_workqueue(rocker->rocker_owq);
2969	free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT), rocker);
2970	free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_CMD), rocker);
2971	rocker_dma_rings_fini(rocker);
2972	rocker_msix_fini(rocker);
2973	iounmap(rocker->hw_addr);
2974	pci_release_regions(rocker->pdev);
2975	pci_disable_device(rocker->pdev);
2976	kfree(rocker);
2977}
2978
2979static struct pci_driver rocker_pci_driver = {
2980	.name		= rocker_driver_name,
2981	.id_table	= rocker_pci_id_table,
2982	.probe		= rocker_probe,
2983	.remove		= rocker_remove,
2984};
2985
2986/************************************
2987 * Net device notifier event handler
2988 ************************************/
2989
2990static bool rocker_port_dev_check_under(const struct net_device *dev,
2991					struct rocker *rocker)
2992{
2993	struct rocker_port *rocker_port;
2994
2995	if (!rocker_port_dev_check(dev))
2996		return false;
2997
2998	rocker_port = netdev_priv(dev);
2999	if (rocker_port->rocker != rocker)
3000		return false;
3001
3002	return true;
3003}
3004
3005struct rocker_walk_data {
3006	struct rocker *rocker;
3007	struct rocker_port *port;
3008};
3009
3010static int rocker_lower_dev_walk(struct net_device *lower_dev, void *_data)
3011{
3012	struct rocker_walk_data *data = _data;
3013	int ret = 0;
3014
3015	if (rocker_port_dev_check_under(lower_dev, data->rocker)) {
3016		data->port = netdev_priv(lower_dev);
3017		ret = 1;
3018	}
3019
3020	return ret;
3021}
3022
3023struct rocker_port *rocker_port_dev_lower_find(struct net_device *dev,
3024					       struct rocker *rocker)
3025{
3026	struct rocker_walk_data data;
3027
3028	if (rocker_port_dev_check_under(dev, rocker))
3029		return netdev_priv(dev);
3030
3031	data.rocker = rocker;
3032	data.port = NULL;
3033	netdev_walk_all_lower_dev(dev, rocker_lower_dev_walk, &data);
3034
3035	return data.port;
3036}
3037
3038static int rocker_netdevice_event(struct notifier_block *unused,
3039				  unsigned long event, void *ptr)
3040{
3041	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3042	struct netdev_notifier_changeupper_info *info;
3043	struct rocker_port *rocker_port;
3044	int err;
3045
3046	if (!rocker_port_dev_check(dev))
3047		return NOTIFY_DONE;
3048
3049	switch (event) {
3050	case NETDEV_CHANGEUPPER:
3051		info = ptr;
3052		if (!info->master)
3053			goto out;
3054		rocker_port = netdev_priv(dev);
3055		if (info->linking) {
3056			err = rocker_world_port_master_linked(rocker_port,
3057							      info->upper_dev);
3058			if (err)
3059				netdev_warn(dev, "failed to reflect master linked (err %d)\n",
3060					    err);
3061		} else {
3062			err = rocker_world_port_master_unlinked(rocker_port,
3063								info->upper_dev);
3064			if (err)
3065				netdev_warn(dev, "failed to reflect master unlinked (err %d)\n",
3066					    err);
3067		}
3068	}
3069out:
3070	return NOTIFY_DONE;
3071}
3072
3073static struct notifier_block rocker_netdevice_nb __read_mostly = {
3074	.notifier_call = rocker_netdevice_event,
3075};
3076
3077/************************************
3078 * Net event notifier event handler
3079 ************************************/
3080
3081static int rocker_netevent_event(struct notifier_block *unused,
3082				 unsigned long event, void *ptr)
3083{
3084	struct rocker_port *rocker_port;
3085	struct net_device *dev;
3086	struct neighbour *n = ptr;
3087	int err;
3088
3089	switch (event) {
3090	case NETEVENT_NEIGH_UPDATE:
3091		if (n->tbl != &arp_tbl)
3092			return NOTIFY_DONE;
3093		dev = n->dev;
3094		if (!rocker_port_dev_check(dev))
3095			return NOTIFY_DONE;
3096		rocker_port = netdev_priv(dev);
3097		err = rocker_world_port_neigh_update(rocker_port, n);
3098		if (err)
3099			netdev_warn(dev, "failed to handle neigh update (err %d)\n",
3100				    err);
3101		break;
3102	}
3103
3104	return NOTIFY_DONE;
3105}
3106
3107static struct notifier_block rocker_netevent_nb __read_mostly = {
3108	.notifier_call = rocker_netevent_event,
3109};
3110
3111/***********************
3112 * Module init and exit
3113 ***********************/
3114
3115static int __init rocker_module_init(void)
3116{
3117	int err;
3118
3119	register_netdevice_notifier(&rocker_netdevice_nb);
3120	register_netevent_notifier(&rocker_netevent_nb);
3121	err = pci_register_driver(&rocker_pci_driver);
3122	if (err)
3123		goto err_pci_register_driver;
3124	return 0;
3125
3126err_pci_register_driver:
3127	unregister_netevent_notifier(&rocker_netevent_nb);
3128	unregister_netdevice_notifier(&rocker_netdevice_nb);
3129	return err;
3130}
3131
3132static void __exit rocker_module_exit(void)
3133{
3134	unregister_netevent_notifier(&rocker_netevent_nb);
3135	unregister_netdevice_notifier(&rocker_netdevice_nb);
3136	pci_unregister_driver(&rocker_pci_driver);
3137}
3138
3139module_init(rocker_module_init);
3140module_exit(rocker_module_exit);
3141
3142MODULE_LICENSE("GPL v2");
3143MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
3144MODULE_AUTHOR("Scott Feldman <sfeldma@gmail.com>");
3145MODULE_DESCRIPTION("Rocker switch device driver");
3146MODULE_DEVICE_TABLE(pci, rocker_pci_id_table);