1/* SPDX-License-Identifier: GPL-2.0+ */
   2/* Copyright (C) 2018 Microchip Technology Inc. */
   3
   4#include <linux/module.h>
   5#include <linux/pci.h>
   6#include <linux/netdevice.h>
   7#include <linux/etherdevice.h>
   8#include <linux/crc32.h>
   9#include <linux/microchipphy.h>
  10#include <linux/net_tstamp.h>
  11#include <linux/of_mdio.h>
  12#include <linux/of_net.h>
  13#include <linux/phy.h>
  14#include <linux/phy_fixed.h>
  15#include <linux/rtnetlink.h>
  16#include <linux/iopoll.h>
  17#include <linux/crc16.h>
  18#include "lan743x_main.h"
  19#include "lan743x_ethtool.h"
  20
  21#define MMD_ACCESS_ADDRESS	0
  22#define MMD_ACCESS_WRITE	1
  23#define MMD_ACCESS_READ		2
  24#define MMD_ACCESS_READ_INC	3
  25#define PCS_POWER_STATE_DOWN	0x6
  26#define PCS_POWER_STATE_UP	0x4
  27
  28static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter)
  29{
  30	u32 chip_rev;
  31	u32 cfg_load;
  32	u32 hw_cfg;
  33	u32 strap;
  34	int ret;
  35
  36	/* Timeout = 100 (i.e. 1 sec (10 msce * 100)) */
  37	ret = lan743x_hs_syslock_acquire(adapter, 100);
  38	if (ret < 0) {
  39		netif_err(adapter, drv, adapter->netdev,
  40			  "Sys Lock acquire failed ret:%d\n", ret);
  41		return;
  42	}
  43
  44	cfg_load = lan743x_csr_read(adapter, ETH_SYS_CONFIG_LOAD_STARTED_REG);
  45	lan743x_hs_syslock_release(adapter);
  46	hw_cfg = lan743x_csr_read(adapter, HW_CFG);
  47
  48	if (cfg_load & GEN_SYS_LOAD_STARTED_REG_ETH_ ||
  49	    hw_cfg & HW_CFG_RST_PROTECT_) {
  50		strap = lan743x_csr_read(adapter, STRAP_READ);
  51		if (strap & STRAP_READ_SGMII_EN_)
  52			adapter->is_sgmii_en = true;
  53		else
  54			adapter->is_sgmii_en = false;
  55	} else {
  56		chip_rev = lan743x_csr_read(adapter, FPGA_REV);
  57		if (chip_rev) {
  58			if (chip_rev & FPGA_SGMII_OP)
  59				adapter->is_sgmii_en = true;
  60			else
  61				adapter->is_sgmii_en = false;
  62		} else {
  63			adapter->is_sgmii_en = false;
  64		}
  65	}
  66	netif_dbg(adapter, drv, adapter->netdev,
  67		  "SGMII I/F %sable\n", adapter->is_sgmii_en ? "En" : "Dis");
  68}
  69
  70static bool is_pci11x1x_chip(struct lan743x_adapter *adapter)
  71{
  72	struct lan743x_csr *csr = &adapter->csr;
  73	u32 id_rev = csr->id_rev;
  74
  75	if (((id_rev & 0xFFFF0000) == ID_REV_ID_A011_) ||
  76	    ((id_rev & 0xFFFF0000) == ID_REV_ID_A041_)) {
  77		return true;
  78	}
  79	return false;
  80}
  81
  82static void lan743x_pci_cleanup(struct lan743x_adapter *adapter)
  83{
  84	pci_release_selected_regions(adapter->pdev,
  85				     pci_select_bars(adapter->pdev,
  86						     IORESOURCE_MEM));
  87	pci_disable_device(adapter->pdev);
  88}
  89
  90static int lan743x_pci_init(struct lan743x_adapter *adapter,
  91			    struct pci_dev *pdev)
  92{
  93	unsigned long bars = 0;
  94	int ret;
  95
  96	adapter->pdev = pdev;
  97	ret = pci_enable_device_mem(pdev);
  98	if (ret)
  99		goto return_error;
 100
 101	netif_info(adapter, probe, adapter->netdev,
 102		   "PCI: Vendor ID = 0x%04X, Device ID = 0x%04X\n",
 103		   pdev->vendor, pdev->device);
 104	bars = pci_select_bars(pdev, IORESOURCE_MEM);
 105	if (!test_bit(0, &bars))
 106		goto disable_device;
 107
 108	ret = pci_request_selected_regions(pdev, bars, DRIVER_NAME);
 109	if (ret)
 110		goto disable_device;
 111
 112	pci_set_master(pdev);
 113	return 0;
 114
 115disable_device:
 116	pci_disable_device(adapter->pdev);
 117
 118return_error:
 119	return ret;
 120}
 121
 122u32 lan743x_csr_read(struct lan743x_adapter *adapter, int offset)
 123{
 124	return ioread32(&adapter->csr.csr_address[offset]);
 125}
 126
 127void lan743x_csr_write(struct lan743x_adapter *adapter, int offset,
 128		       u32 data)
 129{
 130	iowrite32(data, &adapter->csr.csr_address[offset]);
 131}
 132
 133#define LAN743X_CSR_READ_OP(offset)	lan743x_csr_read(adapter, offset)
 134
 135static int lan743x_csr_light_reset(struct lan743x_adapter *adapter)
 136{
 137	u32 data;
 138
 139	data = lan743x_csr_read(adapter, HW_CFG);
 140	data |= HW_CFG_LRST_;
 141	lan743x_csr_write(adapter, HW_CFG, data);
 142
 143	return readx_poll_timeout(LAN743X_CSR_READ_OP, HW_CFG, data,
 144				  !(data & HW_CFG_LRST_), 100000, 10000000);
 145}
 146
 147static int lan743x_csr_wait_for_bit(struct lan743x_adapter *adapter,
 148				    int offset, u32 bit_mask,
 149				    int target_value, int usleep_min,
 150				    int usleep_max, int count)
 151{
 152	u32 data;
 153
 154	return readx_poll_timeout(LAN743X_CSR_READ_OP, offset, data,
 155				  target_value == ((data & bit_mask) ? 1 : 0),
 156				  usleep_max, usleep_min * count);
 157}
 158
 159static int lan743x_csr_init(struct lan743x_adapter *adapter)
 160{
 161	struct lan743x_csr *csr = &adapter->csr;
 162	resource_size_t bar_start, bar_length;
 163	int result;
 164
 165	bar_start = pci_resource_start(adapter->pdev, 0);
 166	bar_length = pci_resource_len(adapter->pdev, 0);
 167	csr->csr_address = devm_ioremap(&adapter->pdev->dev,
 168					bar_start, bar_length);
 169	if (!csr->csr_address) {
 170		result = -ENOMEM;
 171		goto clean_up;
 172	}
 173
 174	csr->id_rev = lan743x_csr_read(adapter, ID_REV);
 175	csr->fpga_rev = lan743x_csr_read(adapter, FPGA_REV);
 176	netif_info(adapter, probe, adapter->netdev,
 177		   "ID_REV = 0x%08X, FPGA_REV = %d.%d\n",
 178		   csr->id_rev,	FPGA_REV_GET_MAJOR_(csr->fpga_rev),
 179		   FPGA_REV_GET_MINOR_(csr->fpga_rev));
 180	if (!ID_REV_IS_VALID_CHIP_ID_(csr->id_rev)) {
 181		result = -ENODEV;
 182		goto clean_up;
 183	}
 184
 185	csr->flags = LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR;
 186	switch (csr->id_rev & ID_REV_CHIP_REV_MASK_) {
 187	case ID_REV_CHIP_REV_A0_:
 188		csr->flags |= LAN743X_CSR_FLAG_IS_A0;
 189		csr->flags &= ~LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR;
 190		break;
 191	case ID_REV_CHIP_REV_B0_:
 192		csr->flags |= LAN743X_CSR_FLAG_IS_B0;
 193		break;
 194	}
 195
 196	result = lan743x_csr_light_reset(adapter);
 197	if (result)
 198		goto clean_up;
 199	return 0;
 200clean_up:
 201	return result;
 202}
 203
 204static void lan743x_intr_software_isr(struct lan743x_adapter *adapter)
 205{
 206	struct lan743x_intr *intr = &adapter->intr;
 207
 208	/* disable the interrupt to prevent repeated re-triggering */
 209	lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_SW_GP_);
 210	intr->software_isr_flag = true;
 211	wake_up(&intr->software_isr_wq);
 212}
 213
 214static void lan743x_tx_isr(void *context, u32 int_sts, u32 flags)
 215{
 216	struct lan743x_tx *tx = context;
 217	struct lan743x_adapter *adapter = tx->adapter;
 218	bool enable_flag = true;
 219
 220	lan743x_csr_read(adapter, INT_EN_SET);
 221	if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR) {
 222		lan743x_csr_write(adapter, INT_EN_CLR,
 223				  INT_BIT_DMA_TX_(tx->channel_number));
 224	}
 225
 226	if (int_sts & INT_BIT_DMA_TX_(tx->channel_number)) {
 227		u32 ioc_bit = DMAC_INT_BIT_TX_IOC_(tx->channel_number);
 228		u32 dmac_int_sts;
 229		u32 dmac_int_en;
 230
 231		if (flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ)
 232			dmac_int_sts = lan743x_csr_read(adapter, DMAC_INT_STS);
 233		else
 234			dmac_int_sts = ioc_bit;
 235		if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK)
 236			dmac_int_en = lan743x_csr_read(adapter,
 237						       DMAC_INT_EN_SET);
 238		else
 239			dmac_int_en = ioc_bit;
 240
 241		dmac_int_en &= ioc_bit;
 242		dmac_int_sts &= dmac_int_en;
 243		if (dmac_int_sts & ioc_bit) {
 244			napi_schedule(&tx->napi);
 245			enable_flag = false;/* poll func will enable later */
 246		}
 247	}
 248
 249	if (enable_flag)
 250		/* enable isr */
 251		lan743x_csr_write(adapter, INT_EN_SET,
 252				  INT_BIT_DMA_TX_(tx->channel_number));
 253}
 254
 255static void lan743x_rx_isr(void *context, u32 int_sts, u32 flags)
 256{
 257	struct lan743x_rx *rx = context;
 258	struct lan743x_adapter *adapter = rx->adapter;
 259	bool enable_flag = true;
 260
 261	if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR) {
 262		lan743x_csr_write(adapter, INT_EN_CLR,
 263				  INT_BIT_DMA_RX_(rx->channel_number));
 264	}
 265
 266	if (int_sts & INT_BIT_DMA_RX_(rx->channel_number)) {
 267		u32 rx_frame_bit = DMAC_INT_BIT_RXFRM_(rx->channel_number);
 268		u32 dmac_int_sts;
 269		u32 dmac_int_en;
 270
 271		if (flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ)
 272			dmac_int_sts = lan743x_csr_read(adapter, DMAC_INT_STS);
 273		else
 274			dmac_int_sts = rx_frame_bit;
 275		if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK)
 276			dmac_int_en = lan743x_csr_read(adapter,
 277						       DMAC_INT_EN_SET);
 278		else
 279			dmac_int_en = rx_frame_bit;
 280
 281		dmac_int_en &= rx_frame_bit;
 282		dmac_int_sts &= dmac_int_en;
 283		if (dmac_int_sts & rx_frame_bit) {
 284			napi_schedule(&rx->napi);
 285			enable_flag = false;/* poll funct will enable later */
 286		}
 287	}
 288
 289	if (enable_flag) {
 290		/* enable isr */
 291		lan743x_csr_write(adapter, INT_EN_SET,
 292				  INT_BIT_DMA_RX_(rx->channel_number));
 293	}
 294}
 295
 296static void lan743x_intr_shared_isr(void *context, u32 int_sts, u32 flags)
 297{
 298	struct lan743x_adapter *adapter = context;
 299	unsigned int channel;
 300
 301	if (int_sts & INT_BIT_ALL_RX_) {
 302		for (channel = 0; channel < LAN743X_USED_RX_CHANNELS;
 303			channel++) {
 304			u32 int_bit = INT_BIT_DMA_RX_(channel);
 305
 306			if (int_sts & int_bit) {
 307				lan743x_rx_isr(&adapter->rx[channel],
 308					       int_bit, flags);
 309				int_sts &= ~int_bit;
 310			}
 311		}
 312	}
 313	if (int_sts & INT_BIT_ALL_TX_) {
 314		for (channel = 0; channel < adapter->used_tx_channels;
 315			channel++) {
 316			u32 int_bit = INT_BIT_DMA_TX_(channel);
 317
 318			if (int_sts & int_bit) {
 319				lan743x_tx_isr(&adapter->tx[channel],
 320					       int_bit, flags);
 321				int_sts &= ~int_bit;
 322			}
 323		}
 324	}
 325	if (int_sts & INT_BIT_ALL_OTHER_) {
 326		if (int_sts & INT_BIT_SW_GP_) {
 327			lan743x_intr_software_isr(adapter);
 328			int_sts &= ~INT_BIT_SW_GP_;
 329		}
 330		if (int_sts & INT_BIT_1588_) {
 331			lan743x_ptp_isr(adapter);
 332			int_sts &= ~INT_BIT_1588_;
 333		}
 334	}
 335	if (int_sts)
 336		lan743x_csr_write(adapter, INT_EN_CLR, int_sts);
 337}
 338
 339static irqreturn_t lan743x_intr_entry_isr(int irq, void *ptr)
 340{
 341	struct lan743x_vector *vector = ptr;
 342	struct lan743x_adapter *adapter = vector->adapter;
 343	irqreturn_t result = IRQ_NONE;
 344	u32 int_enables;
 345	u32 int_sts;
 346
 347	if (vector->flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ) {
 348		int_sts = lan743x_csr_read(adapter, INT_STS);
 349	} else if (vector->flags &
 350		   (LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C |
 351		   LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C)) {
 352		int_sts = lan743x_csr_read(adapter, INT_STS_R2C);
 353	} else {
 354		/* use mask as implied status */
 355		int_sts = vector->int_mask | INT_BIT_MAS_;
 356	}
 357
 358	if (!(int_sts & INT_BIT_MAS_))
 359		goto irq_done;
 360
 361	if (vector->flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR)
 362		/* disable vector interrupt */
 363		lan743x_csr_write(adapter,
 364				  INT_VEC_EN_CLR,
 365				  INT_VEC_EN_(vector->vector_index));
 366
 367	if (vector->flags & LAN743X_VECTOR_FLAG_MASTER_ENABLE_CLEAR)
 368		/* disable master interrupt */
 369		lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_MAS_);
 370
 371	if (vector->flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK) {
 372		int_enables = lan743x_csr_read(adapter, INT_EN_SET);
 373	} else {
 374		/*  use vector mask as implied enable mask */
 375		int_enables = vector->int_mask;
 376	}
 377
 378	int_sts &= int_enables;
 379	int_sts &= vector->int_mask;
 380	if (int_sts) {
 381		if (vector->handler) {
 382			vector->handler(vector->context,
 383					int_sts, vector->flags);
 384		} else {
 385			/* disable interrupts on this vector */
 386			lan743x_csr_write(adapter, INT_EN_CLR,
 387					  vector->int_mask);
 388		}
 389		result = IRQ_HANDLED;
 390	}
 391
 392	if (vector->flags & LAN743X_VECTOR_FLAG_MASTER_ENABLE_SET)
 393		/* enable master interrupt */
 394		lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_MAS_);
 395
 396	if (vector->flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET)
 397		/* enable vector interrupt */
 398		lan743x_csr_write(adapter,
 399				  INT_VEC_EN_SET,
 400				  INT_VEC_EN_(vector->vector_index));
 401irq_done:
 402	return result;
 403}
 404
 405static int lan743x_intr_test_isr(struct lan743x_adapter *adapter)
 406{
 407	struct lan743x_intr *intr = &adapter->intr;
 408	int ret;
 409
 410	intr->software_isr_flag = false;
 411
 412	/* enable and activate test interrupt */
 413	lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_SW_GP_);
 414	lan743x_csr_write(adapter, INT_SET, INT_BIT_SW_GP_);
 415
 416	ret = wait_event_timeout(intr->software_isr_wq,
 417				 intr->software_isr_flag,
 418				 msecs_to_jiffies(200));
 419
 420	/* disable test interrupt */
 421	lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_SW_GP_);
 422
 423	return ret > 0 ? 0 : -ENODEV;
 424}
 425
 426static int lan743x_intr_register_isr(struct lan743x_adapter *adapter,
 427				     int vector_index, u32 flags,
 428				     u32 int_mask,
 429				     lan743x_vector_handler handler,
 430				     void *context)
 431{
 432	struct lan743x_vector *vector = &adapter->intr.vector_list
 433					[vector_index];
 434	int ret;
 435
 436	vector->adapter = adapter;
 437	vector->flags = flags;
 438	vector->vector_index = vector_index;
 439	vector->int_mask = int_mask;
 440	vector->handler = handler;
 441	vector->context = context;
 442
 443	ret = request_irq(vector->irq,
 444			  lan743x_intr_entry_isr,
 445			  (flags & LAN743X_VECTOR_FLAG_IRQ_SHARED) ?
 446			  IRQF_SHARED : 0, DRIVER_NAME, vector);
 447	if (ret) {
 448		vector->handler = NULL;
 449		vector->context = NULL;
 450		vector->int_mask = 0;
 451		vector->flags = 0;
 452	}
 453	return ret;
 454}
 455
 456static void lan743x_intr_unregister_isr(struct lan743x_adapter *adapter,
 457					int vector_index)
 458{
 459	struct lan743x_vector *vector = &adapter->intr.vector_list
 460					[vector_index];
 461
 462	free_irq(vector->irq, vector);
 463	vector->handler = NULL;
 464	vector->context = NULL;
 465	vector->int_mask = 0;
 466	vector->flags = 0;
 467}
 468
 469static u32 lan743x_intr_get_vector_flags(struct lan743x_adapter *adapter,
 470					 u32 int_mask)
 471{
 472	int index;
 473
 474	for (index = 0; index < adapter->max_vector_count; index++) {
 475		if (adapter->intr.vector_list[index].int_mask & int_mask)
 476			return adapter->intr.vector_list[index].flags;
 477	}
 478	return 0;
 479}
 480
 481static void lan743x_intr_close(struct lan743x_adapter *adapter)
 482{
 483	struct lan743x_intr *intr = &adapter->intr;
 484	int index = 0;
 485
 486	lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_MAS_);
 487	if (adapter->is_pci11x1x)
 488		lan743x_csr_write(adapter, INT_VEC_EN_CLR, 0x0000FFFF);
 489	else
 490		lan743x_csr_write(adapter, INT_VEC_EN_CLR, 0x000000FF);
 491
 492	for (index = 0; index < intr->number_of_vectors; index++) {
 493		if (intr->flags & INTR_FLAG_IRQ_REQUESTED(index)) {
 494			lan743x_intr_unregister_isr(adapter, index);
 495			intr->flags &= ~INTR_FLAG_IRQ_REQUESTED(index);
 496		}
 497	}
 498
 499	if (intr->flags & INTR_FLAG_MSI_ENABLED) {
 500		pci_disable_msi(adapter->pdev);
 501		intr->flags &= ~INTR_FLAG_MSI_ENABLED;
 502	}
 503
 504	if (intr->flags & INTR_FLAG_MSIX_ENABLED) {
 505		pci_disable_msix(adapter->pdev);
 506		intr->flags &= ~INTR_FLAG_MSIX_ENABLED;
 507	}
 508}
 509
 510static int lan743x_intr_open(struct lan743x_adapter *adapter)
 511{
 512	struct msix_entry msix_entries[PCI11X1X_MAX_VECTOR_COUNT];
 513	struct lan743x_intr *intr = &adapter->intr;
 514	unsigned int used_tx_channels;
 515	u32 int_vec_en_auto_clr = 0;
 516	u8 max_vector_count;
 517	u32 int_vec_map0 = 0;
 518	u32 int_vec_map1 = 0;
 519	int ret = -ENODEV;
 520	int index = 0;
 521	u32 flags = 0;
 522
 523	intr->number_of_vectors = 0;
 524
 525	/* Try to set up MSIX interrupts */
 526	max_vector_count = adapter->max_vector_count;
 527	memset(&msix_entries[0], 0,
 528	       sizeof(struct msix_entry) * max_vector_count);
 529	for (index = 0; index < max_vector_count; index++)
 530		msix_entries[index].entry = index;
 531	used_tx_channels = adapter->used_tx_channels;
 532	ret = pci_enable_msix_range(adapter->pdev,
 533				    msix_entries, 1,
 534				    1 + used_tx_channels +
 535				    LAN743X_USED_RX_CHANNELS);
 536
 537	if (ret > 0) {
 538		intr->flags |= INTR_FLAG_MSIX_ENABLED;
 539		intr->number_of_vectors = ret;
 540		intr->using_vectors = true;
 541		for (index = 0; index < intr->number_of_vectors; index++)
 542			intr->vector_list[index].irq = msix_entries
 543						       [index].vector;
 544		netif_info(adapter, ifup, adapter->netdev,
 545			   "using MSIX interrupts, number of vectors = %d\n",
 546			   intr->number_of_vectors);
 547	}
 548
 549	/* If MSIX failed try to setup using MSI interrupts */
 550	if (!intr->number_of_vectors) {
 551		if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
 552			if (!pci_enable_msi(adapter->pdev)) {
 553				intr->flags |= INTR_FLAG_MSI_ENABLED;
 554				intr->number_of_vectors = 1;
 555				intr->using_vectors = true;
 556				intr->vector_list[0].irq =
 557					adapter->pdev->irq;
 558				netif_info(adapter, ifup, adapter->netdev,
 559					   "using MSI interrupts, number of vectors = %d\n",
 560					   intr->number_of_vectors);
 561			}
 562		}
 563	}
 564
 565	/* If MSIX, and MSI failed, setup using legacy interrupt */
 566	if (!intr->number_of_vectors) {
 567		intr->number_of_vectors = 1;
 568		intr->using_vectors = false;
 569		intr->vector_list[0].irq = intr->irq;
 570		netif_info(adapter, ifup, adapter->netdev,
 571			   "using legacy interrupts\n");
 572	}
 573
 574	/* At this point we must have at least one irq */
 575	lan743x_csr_write(adapter, INT_VEC_EN_CLR, 0xFFFFFFFF);
 576
 577	/* map all interrupts to vector 0 */
 578	lan743x_csr_write(adapter, INT_VEC_MAP0, 0x00000000);
 579	lan743x_csr_write(adapter, INT_VEC_MAP1, 0x00000000);
 580	lan743x_csr_write(adapter, INT_VEC_MAP2, 0x00000000);
 581	flags = LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ |
 582		LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C |
 583		LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK |
 584		LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR;
 585
 586	if (intr->using_vectors) {
 587		flags |= LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR |
 588			 LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET;
 589	} else {
 590		flags |= LAN743X_VECTOR_FLAG_MASTER_ENABLE_CLEAR |
 591			 LAN743X_VECTOR_FLAG_MASTER_ENABLE_SET |
 592			 LAN743X_VECTOR_FLAG_IRQ_SHARED;
 593	}
 594
 595	if (adapter->csr.flags & LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) {
 596		flags &= ~LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ;
 597		flags &= ~LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C;
 598		flags &= ~LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR;
 599		flags &= ~LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK;
 600		flags |= LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C;
 601		flags |= LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C;
 602	}
 603
 604	init_waitqueue_head(&intr->software_isr_wq);
 605
 606	ret = lan743x_intr_register_isr(adapter, 0, flags,
 607					INT_BIT_ALL_RX_ | INT_BIT_ALL_TX_ |
 608					INT_BIT_ALL_OTHER_,
 609					lan743x_intr_shared_isr, adapter);
 610	if (ret)
 611		goto clean_up;
 612	intr->flags |= INTR_FLAG_IRQ_REQUESTED(0);
 613
 614	if (intr->using_vectors)
 615		lan743x_csr_write(adapter, INT_VEC_EN_SET,
 616				  INT_VEC_EN_(0));
 617
 618	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
 619		lan743x_csr_write(adapter, INT_MOD_CFG0, LAN743X_INT_MOD);
 620		lan743x_csr_write(adapter, INT_MOD_CFG1, LAN743X_INT_MOD);
 621		lan743x_csr_write(adapter, INT_MOD_CFG2, LAN743X_INT_MOD);
 622		lan743x_csr_write(adapter, INT_MOD_CFG3, LAN743X_INT_MOD);
 623		lan743x_csr_write(adapter, INT_MOD_CFG4, LAN743X_INT_MOD);
 624		lan743x_csr_write(adapter, INT_MOD_CFG5, LAN743X_INT_MOD);
 625		lan743x_csr_write(adapter, INT_MOD_CFG6, LAN743X_INT_MOD);
 626		lan743x_csr_write(adapter, INT_MOD_CFG7, LAN743X_INT_MOD);
 627		if (adapter->is_pci11x1x) {
 628			lan743x_csr_write(adapter, INT_MOD_CFG8, LAN743X_INT_MOD);
 629			lan743x_csr_write(adapter, INT_MOD_CFG9, LAN743X_INT_MOD);
 630			lan743x_csr_write(adapter, INT_MOD_MAP0, 0x00007654);
 631			lan743x_csr_write(adapter, INT_MOD_MAP1, 0x00003210);
 632		} else {
 633			lan743x_csr_write(adapter, INT_MOD_MAP0, 0x00005432);
 634			lan743x_csr_write(adapter, INT_MOD_MAP1, 0x00000001);
 635		}
 636		lan743x_csr_write(adapter, INT_MOD_MAP2, 0x00FFFFFF);
 637	}
 638
 639	/* enable interrupts */
 640	lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_MAS_);
 641	ret = lan743x_intr_test_isr(adapter);
 642	if (ret)
 643		goto clean_up;
 644
 645	if (intr->number_of_vectors > 1) {
 646		int number_of_tx_vectors = intr->number_of_vectors - 1;
 647
 648		if (number_of_tx_vectors > used_tx_channels)
 649			number_of_tx_vectors = used_tx_channels;
 650		flags = LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ |
 651			LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C |
 652			LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK |
 653			LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR |
 654			LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR |
 655			LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET;
 656
 657		if (adapter->csr.flags &
 658		   LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) {
 659			flags = LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET |
 660				LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET |
 661				LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR |
 662				LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR;
 663		}
 664
 665		for (index = 0; index < number_of_tx_vectors; index++) {
 666			u32 int_bit = INT_BIT_DMA_TX_(index);
 667			int vector = index + 1;
 668
 669			/* map TX interrupt to vector */
 670			int_vec_map1 |= INT_VEC_MAP1_TX_VEC_(index, vector);
 671			lan743x_csr_write(adapter, INT_VEC_MAP1, int_vec_map1);
 672
 673			/* Remove TX interrupt from shared mask */
 674			intr->vector_list[0].int_mask &= ~int_bit;
 675			ret = lan743x_intr_register_isr(adapter, vector, flags,
 676							int_bit, lan743x_tx_isr,
 677							&adapter->tx[index]);
 678			if (ret)
 679				goto clean_up;
 680			intr->flags |= INTR_FLAG_IRQ_REQUESTED(vector);
 681			if (!(flags &
 682			    LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET))
 683				lan743x_csr_write(adapter, INT_VEC_EN_SET,
 684						  INT_VEC_EN_(vector));
 685		}
 686	}
 687	if ((intr->number_of_vectors - used_tx_channels) > 1) {
 688		int number_of_rx_vectors = intr->number_of_vectors -
 689						used_tx_channels - 1;
 690
 691		if (number_of_rx_vectors > LAN743X_USED_RX_CHANNELS)
 692			number_of_rx_vectors = LAN743X_USED_RX_CHANNELS;
 693
 694		flags = LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ |
 695			LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C |
 696			LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK |
 697			LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR |
 698			LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR |
 699			LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET;
 700
 701		if (adapter->csr.flags &
 702		    LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) {
 703			flags = LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_CLEAR |
 704				LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET |
 705				LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET |
 706				LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR |
 707				LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR;
 708		}
 709		for (index = 0; index < number_of_rx_vectors; index++) {
 710			int vector = index + 1 + used_tx_channels;
 711			u32 int_bit = INT_BIT_DMA_RX_(index);
 712
 713			/* map RX interrupt to vector */
 714			int_vec_map0 |= INT_VEC_MAP0_RX_VEC_(index, vector);
 715			lan743x_csr_write(adapter, INT_VEC_MAP0, int_vec_map0);
 716			if (flags &
 717			    LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_CLEAR) {
 718				int_vec_en_auto_clr |= INT_VEC_EN_(vector);
 719				lan743x_csr_write(adapter, INT_VEC_EN_AUTO_CLR,
 720						  int_vec_en_auto_clr);
 721			}
 722
 723			/* Remove RX interrupt from shared mask */
 724			intr->vector_list[0].int_mask &= ~int_bit;
 725			ret = lan743x_intr_register_isr(adapter, vector, flags,
 726							int_bit, lan743x_rx_isr,
 727							&adapter->rx[index]);
 728			if (ret)
 729				goto clean_up;
 730			intr->flags |= INTR_FLAG_IRQ_REQUESTED(vector);
 731
 732			lan743x_csr_write(adapter, INT_VEC_EN_SET,
 733					  INT_VEC_EN_(vector));
 734		}
 735	}
 736	return 0;
 737
 738clean_up:
 739	lan743x_intr_close(adapter);
 740	return ret;
 741}
 742
 743static int lan743x_dp_write(struct lan743x_adapter *adapter,
 744			    u32 select, u32 addr, u32 length, u32 *buf)
 745{
 746	u32 dp_sel;
 747	int i;
 748
 749	if (lan743x_csr_wait_for_bit(adapter, DP_SEL, DP_SEL_DPRDY_,
 750				     1, 40, 100, 100))
 751		return -EIO;
 752	dp_sel = lan743x_csr_read(adapter, DP_SEL);
 753	dp_sel &= ~DP_SEL_MASK_;
 754	dp_sel |= select;
 755	lan743x_csr_write(adapter, DP_SEL, dp_sel);
 756
 757	for (i = 0; i < length; i++) {
 758		lan743x_csr_write(adapter, DP_ADDR, addr + i);
 759		lan743x_csr_write(adapter, DP_DATA_0, buf[i]);
 760		lan743x_csr_write(adapter, DP_CMD, DP_CMD_WRITE_);
 761		if (lan743x_csr_wait_for_bit(adapter, DP_SEL, DP_SEL_DPRDY_,
 762					     1, 40, 100, 100))
 763			return -EIO;
 764	}
 765
 766	return 0;
 767}
 768
 769static u32 lan743x_mac_mii_access(u16 id, u16 index, int read)
 770{
 771	u32 ret;
 772
 773	ret = (id << MAC_MII_ACC_PHY_ADDR_SHIFT_) &
 774		MAC_MII_ACC_PHY_ADDR_MASK_;
 775	ret |= (index << MAC_MII_ACC_MIIRINDA_SHIFT_) &
 776		MAC_MII_ACC_MIIRINDA_MASK_;
 777
 778	if (read)
 779		ret |= MAC_MII_ACC_MII_READ_;
 780	else
 781		ret |= MAC_MII_ACC_MII_WRITE_;
 782	ret |= MAC_MII_ACC_MII_BUSY_;
 783
 784	return ret;
 785}
 786
 787static int lan743x_mac_mii_wait_till_not_busy(struct lan743x_adapter *adapter)
 788{
 789	u32 data;
 790
 791	return readx_poll_timeout(LAN743X_CSR_READ_OP, MAC_MII_ACC, data,
 792				  !(data & MAC_MII_ACC_MII_BUSY_), 0, 1000000);
 793}
 794
 795static int lan743x_mdiobus_read(struct mii_bus *bus, int phy_id, int index)
 796{
 797	struct lan743x_adapter *adapter = bus->priv;
 798	u32 val, mii_access;
 799	int ret;
 800
 801	/* comfirm MII not busy */
 802	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 803	if (ret < 0)
 804		return ret;
 805
 806	/* set the address, index & direction (read from PHY) */
 807	mii_access = lan743x_mac_mii_access(phy_id, index, MAC_MII_READ);
 808	lan743x_csr_write(adapter, MAC_MII_ACC, mii_access);
 809	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 810	if (ret < 0)
 811		return ret;
 812
 813	val = lan743x_csr_read(adapter, MAC_MII_DATA);
 814	return (int)(val & 0xFFFF);
 815}
 816
 817static int lan743x_mdiobus_write(struct mii_bus *bus,
 818				 int phy_id, int index, u16 regval)
 819{
 820	struct lan743x_adapter *adapter = bus->priv;
 821	u32 val, mii_access;
 822	int ret;
 823
 824	/* confirm MII not busy */
 825	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 826	if (ret < 0)
 827		return ret;
 828	val = (u32)regval;
 829	lan743x_csr_write(adapter, MAC_MII_DATA, val);
 830
 831	/* set the address, index & direction (write to PHY) */
 832	mii_access = lan743x_mac_mii_access(phy_id, index, MAC_MII_WRITE);
 833	lan743x_csr_write(adapter, MAC_MII_ACC, mii_access);
 834	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 835	return ret;
 836}
 837
 838static u32 lan743x_mac_mmd_access(int id, int index, int op)
 839{
 840	u16 dev_addr;
 841	u32 ret;
 842
 843	dev_addr = (index >> 16) & 0x1f;
 844	ret = (id << MAC_MII_ACC_PHY_ADDR_SHIFT_) &
 845		MAC_MII_ACC_PHY_ADDR_MASK_;
 846	ret |= (dev_addr << MAC_MII_ACC_MIIMMD_SHIFT_) &
 847		MAC_MII_ACC_MIIMMD_MASK_;
 848	if (op == MMD_ACCESS_WRITE)
 849		ret |= MAC_MII_ACC_MIICMD_WRITE_;
 850	else if (op == MMD_ACCESS_READ)
 851		ret |= MAC_MII_ACC_MIICMD_READ_;
 852	else if (op == MMD_ACCESS_READ_INC)
 853		ret |= MAC_MII_ACC_MIICMD_READ_INC_;
 854	else
 855		ret |= MAC_MII_ACC_MIICMD_ADDR_;
 856	ret |= (MAC_MII_ACC_MII_BUSY_ | MAC_MII_ACC_MIICL45_);
 857
 858	return ret;
 859}
 860
 861static int lan743x_mdiobus_c45_read(struct mii_bus *bus, int phy_id, int index)
 862{
 863	struct lan743x_adapter *adapter = bus->priv;
 864	u32 mmd_access;
 865	int ret;
 866
 867	/* comfirm MII not busy */
 868	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 869	if (ret < 0)
 870		return ret;
 871	if (index & MII_ADDR_C45) {
 872		/* Load Register Address */
 873		lan743x_csr_write(adapter, MAC_MII_DATA, (u32)(index & 0xffff));
 874		mmd_access = lan743x_mac_mmd_access(phy_id, index,
 875						    MMD_ACCESS_ADDRESS);
 876		lan743x_csr_write(adapter, MAC_MII_ACC, mmd_access);
 877		ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 878		if (ret < 0)
 879			return ret;
 880		/* Read Data */
 881		mmd_access = lan743x_mac_mmd_access(phy_id, index,
 882						    MMD_ACCESS_READ);
 883		lan743x_csr_write(adapter, MAC_MII_ACC, mmd_access);
 884		ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 885		if (ret < 0)
 886			return ret;
 887		ret = lan743x_csr_read(adapter, MAC_MII_DATA);
 888		return (int)(ret & 0xFFFF);
 889	}
 890
 891	ret = lan743x_mdiobus_read(bus, phy_id, index);
 892	return ret;
 893}
 894
 895static int lan743x_mdiobus_c45_write(struct mii_bus *bus,
 896				     int phy_id, int index, u16 regval)
 897{
 898	struct lan743x_adapter *adapter = bus->priv;
 899	u32 mmd_access;
 900	int ret;
 901
 902	/* confirm MII not busy */
 903	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 904	if (ret < 0)
 905		return ret;
 906	if (index & MII_ADDR_C45) {
 907		/* Load Register Address */
 908		lan743x_csr_write(adapter, MAC_MII_DATA, (u32)(index & 0xffff));
 909		mmd_access = lan743x_mac_mmd_access(phy_id, index,
 910						    MMD_ACCESS_ADDRESS);
 911		lan743x_csr_write(adapter, MAC_MII_ACC, mmd_access);
 912		ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 913		if (ret < 0)
 914			return ret;
 915		/* Write Data */
 916		lan743x_csr_write(adapter, MAC_MII_DATA, (u32)regval);
 917		mmd_access = lan743x_mac_mmd_access(phy_id, index,
 918						    MMD_ACCESS_WRITE);
 919		lan743x_csr_write(adapter, MAC_MII_ACC, mmd_access);
 920		ret = lan743x_mac_mii_wait_till_not_busy(adapter);
 921	} else {
 922		ret = lan743x_mdiobus_write(bus, phy_id, index, regval);
 923	}
 924
 925	return ret;
 926}
 927
 928static int lan743x_sgmii_wait_till_not_busy(struct lan743x_adapter *adapter)
 929{
 930	u32 data;
 931	int ret;
 932
 933	ret = readx_poll_timeout(LAN743X_CSR_READ_OP, SGMII_ACC, data,
 934				 !(data & SGMII_ACC_SGMII_BZY_), 100, 1000000);
 935	if (ret < 0)
 936		netif_err(adapter, drv, adapter->netdev,
 937			  "%s: error %d sgmii wait timeout\n", __func__, ret);
 938
 939	return ret;
 940}
 941
 942int lan743x_sgmii_read(struct lan743x_adapter *adapter, u8 mmd, u16 addr)
 943{
 944	u32 mmd_access;
 945	int ret;
 946	u32 val;
 947
 948	if (mmd > 31) {
 949		netif_err(adapter, probe, adapter->netdev,
 950			  "%s mmd should <= 31\n", __func__);
 951		return -EINVAL;
 952	}
 953
 954	mutex_lock(&adapter->sgmii_rw_lock);
 955	/* Load Register Address */
 956	mmd_access = mmd << SGMII_ACC_SGMII_MMD_SHIFT_;
 957	mmd_access |= (addr | SGMII_ACC_SGMII_BZY_);
 958	lan743x_csr_write(adapter, SGMII_ACC, mmd_access);
 959	ret = lan743x_sgmii_wait_till_not_busy(adapter);
 960	if (ret < 0)
 961		goto sgmii_unlock;
 962
 963	val = lan743x_csr_read(adapter, SGMII_DATA);
 964	ret = (int)(val & SGMII_DATA_MASK_);
 965
 966sgmii_unlock:
 967	mutex_unlock(&adapter->sgmii_rw_lock);
 968
 969	return ret;
 970}
 971
 972static int lan743x_sgmii_write(struct lan743x_adapter *adapter,
 973			       u8 mmd, u16 addr, u16 val)
 974{
 975	u32 mmd_access;
 976	int ret;
 977
 978	if (mmd > 31) {
 979		netif_err(adapter, probe, adapter->netdev,
 980			  "%s mmd should <= 31\n", __func__);
 981		return -EINVAL;
 982	}
 983	mutex_lock(&adapter->sgmii_rw_lock);
 984	/* Load Register Data */
 985	lan743x_csr_write(adapter, SGMII_DATA, (u32)(val & SGMII_DATA_MASK_));
 986	/* Load Register Address */
 987	mmd_access = mmd << SGMII_ACC_SGMII_MMD_SHIFT_;
 988	mmd_access |= (addr | SGMII_ACC_SGMII_BZY_ | SGMII_ACC_SGMII_WR_);
 989	lan743x_csr_write(adapter, SGMII_ACC, mmd_access);
 990	ret = lan743x_sgmii_wait_till_not_busy(adapter);
 991	mutex_unlock(&adapter->sgmii_rw_lock);
 992
 993	return ret;
 994}
 995
 996static int lan743x_sgmii_mpll_set(struct lan743x_adapter *adapter,
 997				  u16 baud)
 998{
 999	int mpllctrl0;
1000	int mpllctrl1;
1001	int miscctrl1;
1002	int ret;
1003
1004	mpllctrl0 = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1005				       VR_MII_GEN2_4_MPLL_CTRL0);
1006	if (mpllctrl0 < 0)
1007		return mpllctrl0;
1008
1009	mpllctrl0 &= ~VR_MII_MPLL_CTRL0_USE_REFCLK_PAD_;
1010	if (baud == VR_MII_BAUD_RATE_1P25GBPS) {
1011		mpllctrl1 = VR_MII_MPLL_MULTIPLIER_100;
1012		/* mpll_baud_clk/4 */
1013		miscctrl1 = 0xA;
1014	} else {
1015		mpllctrl1 = VR_MII_MPLL_MULTIPLIER_125;
1016		/* mpll_baud_clk/2 */
1017		miscctrl1 = 0x5;
1018	}
1019
1020	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1021				  VR_MII_GEN2_4_MPLL_CTRL0, mpllctrl0);
1022	if (ret < 0)
1023		return ret;
1024
1025	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1026				  VR_MII_GEN2_4_MPLL_CTRL1, mpllctrl1);
1027	if (ret < 0)
1028		return ret;
1029
1030	return lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1031				  VR_MII_GEN2_4_MISC_CTRL1, miscctrl1);
1032}
1033
1034static int lan743x_sgmii_2_5G_mode_set(struct lan743x_adapter *adapter,
1035				       bool enable)
1036{
1037	if (enable)
1038		return lan743x_sgmii_mpll_set(adapter,
1039					      VR_MII_BAUD_RATE_3P125GBPS);
1040	else
1041		return lan743x_sgmii_mpll_set(adapter,
1042					      VR_MII_BAUD_RATE_1P25GBPS);
1043}
1044
1045static int lan743x_is_sgmii_2_5G_mode(struct lan743x_adapter *adapter,
1046				      bool *status)
1047{
1048	int ret;
1049
1050	ret = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1051				 VR_MII_GEN2_4_MPLL_CTRL1);
1052	if (ret < 0)
1053		return ret;
1054
1055	if (ret == VR_MII_MPLL_MULTIPLIER_125 ||
1056	    ret == VR_MII_MPLL_MULTIPLIER_50)
1057		*status = true;
1058	else
1059		*status = false;
1060
1061	return 0;
1062}
1063
1064static int lan743x_sgmii_aneg_update(struct lan743x_adapter *adapter)
1065{
1066	enum lan743x_sgmii_lsd lsd = adapter->sgmii_lsd;
1067	int mii_ctrl;
1068	int dgt_ctrl;
1069	int an_ctrl;
1070	int ret;
1071
1072	if (lsd == LINK_2500_MASTER || lsd == LINK_2500_SLAVE)
1073		/* Switch to 2.5 Gbps */
1074		ret = lan743x_sgmii_2_5G_mode_set(adapter, true);
1075	else
1076		/* Switch to 10/100/1000 Mbps clock */
1077		ret = lan743x_sgmii_2_5G_mode_set(adapter, false);
1078	if (ret < 0)
1079		return ret;
1080
1081	/* Enable SGMII Auto NEG */
1082	mii_ctrl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2, MII_BMCR);
1083	if (mii_ctrl < 0)
1084		return mii_ctrl;
1085
1086	an_ctrl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2, VR_MII_AN_CTRL);
1087	if (an_ctrl < 0)
1088		return an_ctrl;
1089
1090	dgt_ctrl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1091				      VR_MII_DIG_CTRL1);
1092	if (dgt_ctrl < 0)
1093		return dgt_ctrl;
1094
1095	if (lsd == LINK_2500_MASTER || lsd == LINK_2500_SLAVE) {
1096		mii_ctrl &= ~(BMCR_ANENABLE | BMCR_ANRESTART | BMCR_SPEED100);
1097		mii_ctrl |= BMCR_SPEED1000;
1098		dgt_ctrl |= VR_MII_DIG_CTRL1_CL37_TMR_OVR_RIDE_;
1099		dgt_ctrl &= ~VR_MII_DIG_CTRL1_MAC_AUTO_SW_;
1100		/* In order for Auto-Negotiation to operate properly at
1101		 * 2.5 Gbps the 1.6ms link timer values must be adjusted
1102		 * The VR_MII_LINK_TIMER_CTRL Register must be set to
1103		 * 16'h7A1 and The CL37_TMR_OVR_RIDE bit of the
1104		 * VR_MII_DIG_CTRL1 Register set to 1
1105		 */
1106		ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1107					  VR_MII_LINK_TIMER_CTRL, 0x7A1);
1108		if (ret < 0)
1109			return ret;
1110	} else {
1111		mii_ctrl |= (BMCR_ANENABLE | BMCR_ANRESTART);
1112		an_ctrl &= ~VR_MII_AN_CTRL_SGMII_LINK_STS_;
1113		dgt_ctrl &= ~VR_MII_DIG_CTRL1_CL37_TMR_OVR_RIDE_;
1114		dgt_ctrl |= VR_MII_DIG_CTRL1_MAC_AUTO_SW_;
1115	}
1116
1117	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2, MII_BMCR,
1118				  mii_ctrl);
1119	if (ret < 0)
1120		return ret;
1121
1122	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1123				  VR_MII_DIG_CTRL1, dgt_ctrl);
1124	if (ret < 0)
1125		return ret;
1126
1127	return lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1128				  VR_MII_AN_CTRL, an_ctrl);
1129}
1130
1131static int lan743x_pcs_seq_state(struct lan743x_adapter *adapter, u8 state)
1132{
1133	u8 wait_cnt = 0;
1134	u32 dig_sts;
1135
1136	do {
1137		dig_sts = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1138					     VR_MII_DIG_STS);
1139		if (((dig_sts & VR_MII_DIG_STS_PSEQ_STATE_MASK_) >>
1140		      VR_MII_DIG_STS_PSEQ_STATE_POS_) == state)
1141			break;
1142		usleep_range(1000, 2000);
1143	} while (wait_cnt++ < 10);
1144
1145	if (wait_cnt >= 10)
1146		return -ETIMEDOUT;
1147
1148	return 0;
1149}
1150
1151static int lan743x_sgmii_config(struct lan743x_adapter *adapter)
1152{
1153	struct net_device *netdev = adapter->netdev;
1154	struct phy_device *phydev = netdev->phydev;
1155	enum lan743x_sgmii_lsd lsd = POWER_DOWN;
1156	int mii_ctl;
1157	bool status;
1158	int ret;
1159
1160	switch (phydev->speed) {
1161	case SPEED_2500:
1162		if (phydev->master_slave_state == MASTER_SLAVE_STATE_MASTER)
1163			lsd = LINK_2500_MASTER;
1164		else
1165			lsd = LINK_2500_SLAVE;
1166		break;
1167	case SPEED_1000:
1168		if (phydev->master_slave_state == MASTER_SLAVE_STATE_MASTER)
1169			lsd = LINK_1000_MASTER;
1170		else
1171			lsd = LINK_1000_SLAVE;
1172		break;
1173	case SPEED_100:
1174		if (phydev->duplex)
1175			lsd = LINK_100FD;
1176		else
1177			lsd = LINK_100HD;
1178		break;
1179	case SPEED_10:
1180		if (phydev->duplex)
1181			lsd = LINK_10FD;
1182		else
1183			lsd = LINK_10HD;
1184		break;
1185	default:
1186		netif_err(adapter, drv, adapter->netdev,
1187			  "Invalid speed %d\n", phydev->speed);
1188		return -EINVAL;
1189	}
1190
1191	adapter->sgmii_lsd = lsd;
1192	ret = lan743x_sgmii_aneg_update(adapter);
1193	if (ret < 0) {
1194		netif_err(adapter, drv, adapter->netdev,
1195			  "error %d SGMII cfg failed\n", ret);
1196		return ret;
1197	}
1198
1199	ret = lan743x_is_sgmii_2_5G_mode(adapter, &status);
1200	if (ret < 0) {
1201		netif_err(adapter, drv, adapter->netdev,
1202			  "erro %d SGMII get mode failed\n", ret);
1203		return ret;
1204	}
1205
1206	if (status)
1207		netif_dbg(adapter, drv, adapter->netdev,
1208			  "SGMII 2.5G mode enable\n");
1209	else
1210		netif_dbg(adapter, drv, adapter->netdev,
1211			  "SGMII 1G mode enable\n");
1212
1213	/* SGMII/1000/2500BASE-X PCS power down */
1214	mii_ctl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2, MII_BMCR);
1215	if (mii_ctl < 0)
1216		return mii_ctl;
1217
1218	mii_ctl |= BMCR_PDOWN;
1219	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2, MII_BMCR, mii_ctl);
1220	if (ret < 0)
1221		return ret;
1222
1223	ret = lan743x_pcs_seq_state(adapter, PCS_POWER_STATE_DOWN);
1224	if (ret < 0)
1225		return ret;
1226
1227	/* SGMII/1000/2500BASE-X PCS power up */
1228	mii_ctl &= ~BMCR_PDOWN;
1229	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2, MII_BMCR, mii_ctl);
1230	if (ret < 0)
1231		return ret;
1232
1233	ret = lan743x_pcs_seq_state(adapter, PCS_POWER_STATE_UP);
1234	if (ret < 0)
1235		return ret;
1236
1237	return 0;
1238}
1239
1240static void lan743x_mac_set_address(struct lan743x_adapter *adapter,
1241				    u8 *addr)
1242{
1243	u32 addr_lo, addr_hi;
1244
1245	addr_lo = addr[0] |
1246		addr[1] << 8 |
1247		addr[2] << 16 |
1248		addr[3] << 24;
1249	addr_hi = addr[4] |
1250		addr[5] << 8;
1251	lan743x_csr_write(adapter, MAC_RX_ADDRL, addr_lo);
1252	lan743x_csr_write(adapter, MAC_RX_ADDRH, addr_hi);
1253
1254	ether_addr_copy(adapter->mac_address, addr);
1255	netif_info(adapter, drv, adapter->netdev,
1256		   "MAC address set to %pM\n", addr);
1257}
1258
1259static int lan743x_mac_init(struct lan743x_adapter *adapter)
1260{
1261	bool mac_address_valid = true;
1262	struct net_device *netdev;
1263	u32 mac_addr_hi = 0;
1264	u32 mac_addr_lo = 0;
1265	u32 data;
1266
1267	netdev = adapter->netdev;
1268
1269	/* disable auto duplex, and speed detection. Phylib does that */
1270	data = lan743x_csr_read(adapter, MAC_CR);
1271	data &= ~(MAC_CR_ADD_ | MAC_CR_ASD_);
1272	data |= MAC_CR_CNTR_RST_;
1273	lan743x_csr_write(adapter, MAC_CR, data);
1274
1275	if (!is_valid_ether_addr(adapter->mac_address)) {
1276		mac_addr_hi = lan743x_csr_read(adapter, MAC_RX_ADDRH);
1277		mac_addr_lo = lan743x_csr_read(adapter, MAC_RX_ADDRL);
1278		adapter->mac_address[0] = mac_addr_lo & 0xFF;
1279		adapter->mac_address[1] = (mac_addr_lo >> 8) & 0xFF;
1280		adapter->mac_address[2] = (mac_addr_lo >> 16) & 0xFF;
1281		adapter->mac_address[3] = (mac_addr_lo >> 24) & 0xFF;
1282		adapter->mac_address[4] = mac_addr_hi & 0xFF;
1283		adapter->mac_address[5] = (mac_addr_hi >> 8) & 0xFF;
1284
1285		if (((mac_addr_hi & 0x0000FFFF) == 0x0000FFFF) &&
1286		    mac_addr_lo == 0xFFFFFFFF) {
1287			mac_address_valid = false;
1288		} else if (!is_valid_ether_addr(adapter->mac_address)) {
1289			mac_address_valid = false;
1290		}
1291
1292		if (!mac_address_valid)
1293			eth_random_addr(adapter->mac_address);
1294	}
1295	lan743x_mac_set_address(adapter, adapter->mac_address);
1296	eth_hw_addr_set(netdev, adapter->mac_address);
1297
1298	return 0;
1299}
1300
1301static int lan743x_mac_open(struct lan743x_adapter *adapter)
1302{
1303	u32 temp;
1304
1305	temp = lan743x_csr_read(adapter, MAC_RX);
1306	lan743x_csr_write(adapter, MAC_RX, temp | MAC_RX_RXEN_);
1307	temp = lan743x_csr_read(adapter, MAC_TX);
1308	lan743x_csr_write(adapter, MAC_TX, temp | MAC_TX_TXEN_);
1309	return 0;
1310}
1311
1312static void lan743x_mac_close(struct lan743x_adapter *adapter)
1313{
1314	u32 temp;
1315
1316	temp = lan743x_csr_read(adapter, MAC_TX);
1317	temp &= ~MAC_TX_TXEN_;
1318	lan743x_csr_write(adapter, MAC_TX, temp);
1319	lan743x_csr_wait_for_bit(adapter, MAC_TX, MAC_TX_TXD_,
1320				 1, 1000, 20000, 100);
1321
1322	temp = lan743x_csr_read(adapter, MAC_RX);
1323	temp &= ~MAC_RX_RXEN_;
1324	lan743x_csr_write(adapter, MAC_RX, temp);
1325	lan743x_csr_wait_for_bit(adapter, MAC_RX, MAC_RX_RXD_,
1326				 1, 1000, 20000, 100);
1327}
1328
1329void lan743x_mac_flow_ctrl_set_enables(struct lan743x_adapter *adapter,
1330				       bool tx_enable, bool rx_enable)
1331{
1332	u32 flow_setting = 0;
1333
1334	/* set maximum pause time because when fifo space frees
1335	 * up a zero value pause frame will be sent to release the pause
1336	 */
1337	flow_setting = MAC_FLOW_CR_FCPT_MASK_;
1338	if (tx_enable)
1339		flow_setting |= MAC_FLOW_CR_TX_FCEN_;
1340	if (rx_enable)
1341		flow_setting |= MAC_FLOW_CR_RX_FCEN_;
1342	lan743x_csr_write(adapter, MAC_FLOW, flow_setting);
1343}
1344
1345static int lan743x_mac_set_mtu(struct lan743x_adapter *adapter, int new_mtu)
1346{
1347	int enabled = 0;
1348	u32 mac_rx = 0;
1349
1350	mac_rx = lan743x_csr_read(adapter, MAC_RX);
1351	if (mac_rx & MAC_RX_RXEN_) {
1352		enabled = 1;
1353		if (mac_rx & MAC_RX_RXD_) {
1354			lan743x_csr_write(adapter, MAC_RX, mac_rx);
1355			mac_rx &= ~MAC_RX_RXD_;
1356		}
1357		mac_rx &= ~MAC_RX_RXEN_;
1358		lan743x_csr_write(adapter, MAC_RX, mac_rx);
1359		lan743x_csr_wait_for_bit(adapter, MAC_RX, MAC_RX_RXD_,
1360					 1, 1000, 20000, 100);
1361		lan743x_csr_write(adapter, MAC_RX, mac_rx | MAC_RX_RXD_);
1362	}
1363
1364	mac_rx &= ~(MAC_RX_MAX_SIZE_MASK_);
1365	mac_rx |= (((new_mtu + ETH_HLEN + ETH_FCS_LEN)
1366		  << MAC_RX_MAX_SIZE_SHIFT_) & MAC_RX_MAX_SIZE_MASK_);
1367	lan743x_csr_write(adapter, MAC_RX, mac_rx);
1368
1369	if (enabled) {
1370		mac_rx |= MAC_RX_RXEN_;
1371		lan743x_csr_write(adapter, MAC_RX, mac_rx);
1372	}
1373	return 0;
1374}
1375
1376/* PHY */
1377static int lan743x_phy_reset(struct lan743x_adapter *adapter)
1378{
1379	u32 data;
1380
1381	/* Only called with in probe, and before mdiobus_register */
1382
1383	data = lan743x_csr_read(adapter, PMT_CTL);
1384	data |= PMT_CTL_ETH_PHY_RST_;
1385	lan743x_csr_write(adapter, PMT_CTL, data);
1386
1387	return readx_poll_timeout(LAN743X_CSR_READ_OP, PMT_CTL, data,
1388				  (!(data & PMT_CTL_ETH_PHY_RST_) &&
1389				  (data & PMT_CTL_READY_)),
1390				  50000, 1000000);
1391}
1392
1393static void lan743x_phy_update_flowcontrol(struct lan743x_adapter *adapter,
1394					   u16 local_adv, u16 remote_adv)
1395{
1396	struct lan743x_phy *phy = &adapter->phy;
1397	u8 cap;
1398
1399	if (phy->fc_autoneg)
1400		cap = mii_resolve_flowctrl_fdx(local_adv, remote_adv);
1401	else
1402		cap = phy->fc_request_control;
1403
1404	lan743x_mac_flow_ctrl_set_enables(adapter,
1405					  cap & FLOW_CTRL_TX,
1406					  cap & FLOW_CTRL_RX);
1407}
1408
1409static int lan743x_phy_init(struct lan743x_adapter *adapter)
1410{
1411	return lan743x_phy_reset(adapter);
1412}
1413
1414static void lan743x_phy_link_status_change(struct net_device *netdev)
1415{
1416	struct lan743x_adapter *adapter = netdev_priv(netdev);
1417	struct phy_device *phydev = netdev->phydev;
1418	u32 data;
1419
1420	phy_print_status(phydev);
1421	if (phydev->state == PHY_RUNNING) {
1422		int remote_advertisement = 0;
1423		int local_advertisement = 0;
1424
1425		data = lan743x_csr_read(adapter, MAC_CR);
1426
1427		/* set interface mode */
1428		if (phy_interface_is_rgmii(phydev))
1429			/* RGMII */
1430			data &= ~MAC_CR_MII_EN_;
1431		else
1432			/* GMII */
1433			data |= MAC_CR_MII_EN_;
1434
1435		/* set duplex mode */
1436		if (phydev->duplex)
1437			data |= MAC_CR_DPX_;
1438		else
1439			data &= ~MAC_CR_DPX_;
1440
1441		/* set bus speed */
1442		switch (phydev->speed) {
1443		case SPEED_10:
1444			data &= ~MAC_CR_CFG_H_;
1445			data &= ~MAC_CR_CFG_L_;
1446		break;
1447		case SPEED_100:
1448			data &= ~MAC_CR_CFG_H_;
1449			data |= MAC_CR_CFG_L_;
1450		break;
1451		case SPEED_1000:
1452			data |= MAC_CR_CFG_H_;
1453			data &= ~MAC_CR_CFG_L_;
1454		break;
1455		case SPEED_2500:
1456			data |= MAC_CR_CFG_H_;
1457			data |= MAC_CR_CFG_L_;
1458		break;
1459		}
1460		lan743x_csr_write(adapter, MAC_CR, data);
1461
1462		local_advertisement =
1463			linkmode_adv_to_mii_adv_t(phydev->advertising);
1464		remote_advertisement =
1465			linkmode_adv_to_mii_adv_t(phydev->lp_advertising);
1466
1467		lan743x_phy_update_flowcontrol(adapter, local_advertisement,
1468					       remote_advertisement);
1469		lan743x_ptp_update_latency(adapter, phydev->speed);
1470		if (phydev->interface == PHY_INTERFACE_MODE_SGMII ||
1471		    phydev->interface == PHY_INTERFACE_MODE_1000BASEX ||
1472		    phydev->interface == PHY_INTERFACE_MODE_2500BASEX)
1473			lan743x_sgmii_config(adapter);
1474	}
1475}
1476
1477static void lan743x_phy_close(struct lan743x_adapter *adapter)
1478{
1479	struct net_device *netdev = adapter->netdev;
1480
1481	phy_stop(netdev->phydev);
1482	phy_disconnect(netdev->phydev);
1483	netdev->phydev = NULL;
1484}
1485
1486static int lan743x_phy_open(struct lan743x_adapter *adapter)
1487{
1488	struct net_device *netdev = adapter->netdev;
1489	struct lan743x_phy *phy = &adapter->phy;
1490	struct phy_device *phydev;
1491	int ret = -EIO;
1492
1493	/* try devicetree phy, or fixed link */
1494	phydev = of_phy_get_and_connect(netdev, adapter->pdev->dev.of_node,
1495					lan743x_phy_link_status_change);
1496
1497	if (!phydev) {
1498		/* try internal phy */
1499		phydev = phy_find_first(adapter->mdiobus);
1500		if (!phydev)
1501			goto return_error;
1502
1503		if (adapter->is_pci11x1x)
1504			ret = phy_connect_direct(netdev, phydev,
1505						 lan743x_phy_link_status_change,
1506						 PHY_INTERFACE_MODE_RGMII);
1507		else
1508			ret = phy_connect_direct(netdev, phydev,
1509						 lan743x_phy_link_status_change,
1510						 PHY_INTERFACE_MODE_GMII);
1511		if (ret)
1512			goto return_error;
1513	}
1514
1515	/* MAC doesn't support 1000T Half */
1516	phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
1517
1518	/* support both flow controls */
1519	phy_support_asym_pause(phydev);
1520	phy->fc_request_control = (FLOW_CTRL_RX | FLOW_CTRL_TX);
1521	phy->fc_autoneg = phydev->autoneg;
1522
1523	phy_start(phydev);
1524	phy_start_aneg(phydev);
1525	phy_attached_info(phydev);
1526	return 0;
1527
1528return_error:
1529	return ret;
1530}
1531
1532static void lan743x_rfe_open(struct lan743x_adapter *adapter)
1533{
1534	lan743x_csr_write(adapter, RFE_RSS_CFG,
1535		RFE_RSS_CFG_UDP_IPV6_EX_ |
1536		RFE_RSS_CFG_TCP_IPV6_EX_ |
1537		RFE_RSS_CFG_IPV6_EX_ |
1538		RFE_RSS_CFG_UDP_IPV6_ |
1539		RFE_RSS_CFG_TCP_IPV6_ |
1540		RFE_RSS_CFG_IPV6_ |
1541		RFE_RSS_CFG_UDP_IPV4_ |
1542		RFE_RSS_CFG_TCP_IPV4_ |
1543		RFE_RSS_CFG_IPV4_ |
1544		RFE_RSS_CFG_VALID_HASH_BITS_ |
1545		RFE_RSS_CFG_RSS_QUEUE_ENABLE_ |
1546		RFE_RSS_CFG_RSS_HASH_STORE_ |
1547		RFE_RSS_CFG_RSS_ENABLE_);
1548}
1549
1550static void lan743x_rfe_update_mac_address(struct lan743x_adapter *adapter)
1551{
1552	u8 *mac_addr;
1553	u32 mac_addr_hi = 0;
1554	u32 mac_addr_lo = 0;
1555
1556	/* Add mac address to perfect Filter */
1557	mac_addr = adapter->mac_address;
1558	mac_addr_lo = ((((u32)(mac_addr[0])) << 0) |
1559		      (((u32)(mac_addr[1])) << 8) |
1560		      (((u32)(mac_addr[2])) << 16) |
1561		      (((u32)(mac_addr[3])) << 24));
1562	mac_addr_hi = ((((u32)(mac_addr[4])) << 0) |
1563		      (((u32)(mac_addr[5])) << 8));
1564
1565	lan743x_csr_write(adapter, RFE_ADDR_FILT_LO(0), mac_addr_lo);
1566	lan743x_csr_write(adapter, RFE_ADDR_FILT_HI(0),
1567			  mac_addr_hi | RFE_ADDR_FILT_HI_VALID_);
1568}
1569
1570static void lan743x_rfe_set_multicast(struct lan743x_adapter *adapter)
1571{
1572	struct net_device *netdev = adapter->netdev;
1573	u32 hash_table[DP_SEL_VHF_HASH_LEN];
1574	u32 rfctl;
1575	u32 data;
1576
1577	rfctl = lan743x_csr_read(adapter, RFE_CTL);
1578	rfctl &= ~(RFE_CTL_AU_ | RFE_CTL_AM_ |
1579		 RFE_CTL_DA_PERFECT_ | RFE_CTL_MCAST_HASH_);
1580	rfctl |= RFE_CTL_AB_;
1581	if (netdev->flags & IFF_PROMISC) {
1582		rfctl |= RFE_CTL_AM_ | RFE_CTL_AU_;
1583	} else {
1584		if (netdev->flags & IFF_ALLMULTI)
1585			rfctl |= RFE_CTL_AM_;
1586	}
1587
1588	if (netdev->features & NETIF_F_RXCSUM)
1589		rfctl |= RFE_CTL_IP_COE_ | RFE_CTL_TCP_UDP_COE_;
1590
1591	memset(hash_table, 0, DP_SEL_VHF_HASH_LEN * sizeof(u32));
1592	if (netdev_mc_count(netdev)) {
1593		struct netdev_hw_addr *ha;
1594		int i;
1595
1596		rfctl |= RFE_CTL_DA_PERFECT_;
1597		i = 1;
1598		netdev_for_each_mc_addr(ha, netdev) {
1599			/* set first 32 into Perfect Filter */
1600			if (i < 33) {
1601				lan743x_csr_write(adapter,
1602						  RFE_ADDR_FILT_HI(i), 0);
1603				data = ha->addr[3];
1604				data = ha->addr[2] | (data << 8);
1605				data = ha->addr[1] | (data << 8);
1606				data = ha->addr[0] | (data << 8);
1607				lan743x_csr_write(adapter,
1608						  RFE_ADDR_FILT_LO(i), data);
1609				data = ha->addr[5];
1610				data = ha->addr[4] | (data << 8);
1611				data |= RFE_ADDR_FILT_HI_VALID_;
1612				lan743x_csr_write(adapter,
1613						  RFE_ADDR_FILT_HI(i), data);
1614			} else {
1615				u32 bitnum = (ether_crc(ETH_ALEN, ha->addr) >>
1616					     23) & 0x1FF;
1617				hash_table[bitnum / 32] |= (1 << (bitnum % 32));
1618				rfctl |= RFE_CTL_MCAST_HASH_;
1619			}
1620			i++;
1621		}
1622	}
1623
1624	lan743x_dp_write(adapter, DP_SEL_RFE_RAM,
1625			 DP_SEL_VHF_VLAN_LEN,
1626			 DP_SEL_VHF_HASH_LEN, hash_table);
1627	lan743x_csr_write(adapter, RFE_CTL, rfctl);
1628}
1629
1630static int lan743x_dmac_init(struct lan743x_adapter *adapter)
1631{
1632	u32 data = 0;
1633
1634	lan743x_csr_write(adapter, DMAC_CMD, DMAC_CMD_SWR_);
1635	lan743x_csr_wait_for_bit(adapter, DMAC_CMD, DMAC_CMD_SWR_,
1636				 0, 1000, 20000, 100);
1637	switch (DEFAULT_DMA_DESCRIPTOR_SPACING) {
1638	case DMA_DESCRIPTOR_SPACING_16:
1639		data = DMAC_CFG_MAX_DSPACE_16_;
1640		break;
1641	case DMA_DESCRIPTOR_SPACING_32:
1642		data = DMAC_CFG_MAX_DSPACE_32_;
1643		break;
1644	case DMA_DESCRIPTOR_SPACING_64:
1645		data = DMAC_CFG_MAX_DSPACE_64_;
1646		break;
1647	case DMA_DESCRIPTOR_SPACING_128:
1648		data = DMAC_CFG_MAX_DSPACE_128_;
1649		break;
1650	default:
1651		return -EPERM;
1652	}
1653	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0))
1654		data |= DMAC_CFG_COAL_EN_;
1655	data |= DMAC_CFG_CH_ARB_SEL_RX_HIGH_;
1656	data |= DMAC_CFG_MAX_READ_REQ_SET_(6);
1657	lan743x_csr_write(adapter, DMAC_CFG, data);
1658	data = DMAC_COAL_CFG_TIMER_LIMIT_SET_(1);
1659	data |= DMAC_COAL_CFG_TIMER_TX_START_;
1660	data |= DMAC_COAL_CFG_FLUSH_INTS_;
1661	data |= DMAC_COAL_CFG_INT_EXIT_COAL_;
1662	data |= DMAC_COAL_CFG_CSR_EXIT_COAL_;
1663	data |= DMAC_COAL_CFG_TX_THRES_SET_(0x0A);
1664	data |= DMAC_COAL_CFG_RX_THRES_SET_(0x0C);
1665	lan743x_csr_write(adapter, DMAC_COAL_CFG, data);
1666	data = DMAC_OBFF_TX_THRES_SET_(0x08);
1667	data |= DMAC_OBFF_RX_THRES_SET_(0x0A);
1668	lan743x_csr_write(adapter, DMAC_OBFF_CFG, data);
1669	return 0;
1670}
1671
1672static int lan743x_dmac_tx_get_state(struct lan743x_adapter *adapter,
1673				     int tx_channel)
1674{
1675	u32 dmac_cmd = 0;
1676
1677	dmac_cmd = lan743x_csr_read(adapter, DMAC_CMD);
1678	return DMAC_CHANNEL_STATE_SET((dmac_cmd &
1679				      DMAC_CMD_START_T_(tx_channel)),
1680				      (dmac_cmd &
1681				      DMAC_CMD_STOP_T_(tx_channel)));
1682}
1683
1684static int lan743x_dmac_tx_wait_till_stopped(struct lan743x_adapter *adapter,
1685					     int tx_channel)
1686{
1687	int timeout = 100;
1688	int result = 0;
1689
1690	while (timeout &&
1691	       ((result = lan743x_dmac_tx_get_state(adapter, tx_channel)) ==
1692	       DMAC_CHANNEL_STATE_STOP_PENDING)) {
1693		usleep_range(1000, 20000);
1694		timeout--;
1695	}
1696	if (result == DMAC_CHANNEL_STATE_STOP_PENDING)
1697		result = -ENODEV;
1698	return result;
1699}
1700
1701static int lan743x_dmac_rx_get_state(struct lan743x_adapter *adapter,
1702				     int rx_channel)
1703{
1704	u32 dmac_cmd = 0;
1705
1706	dmac_cmd = lan743x_csr_read(adapter, DMAC_CMD);
1707	return DMAC_CHANNEL_STATE_SET((dmac_cmd &
1708				      DMAC_CMD_START_R_(rx_channel)),
1709				      (dmac_cmd &
1710				      DMAC_CMD_STOP_R_(rx_channel)));
1711}
1712
1713static int lan743x_dmac_rx_wait_till_stopped(struct lan743x_adapter *adapter,
1714					     int rx_channel)
1715{
1716	int timeout = 100;
1717	int result = 0;
1718
1719	while (timeout &&
1720	       ((result = lan743x_dmac_rx_get_state(adapter, rx_channel)) ==
1721	       DMAC_CHANNEL_STATE_STOP_PENDING)) {
1722		usleep_range(1000, 20000);
1723		timeout--;
1724	}
1725	if (result == DMAC_CHANNEL_STATE_STOP_PENDING)
1726		result = -ENODEV;
1727	return result;
1728}
1729
1730static void lan743x_tx_release_desc(struct lan743x_tx *tx,
1731				    int descriptor_index, bool cleanup)
1732{
1733	struct lan743x_tx_buffer_info *buffer_info = NULL;
1734	struct lan743x_tx_descriptor *descriptor = NULL;
1735	u32 descriptor_type = 0;
1736	bool ignore_sync;
1737
1738	descriptor = &tx->ring_cpu_ptr[descriptor_index];
1739	buffer_info = &tx->buffer_info[descriptor_index];
1740	if (!(buffer_info->flags & TX_BUFFER_INFO_FLAG_ACTIVE))
1741		goto done;
1742
1743	descriptor_type = le32_to_cpu(descriptor->data0) &
1744			  TX_DESC_DATA0_DTYPE_MASK_;
1745	if (descriptor_type == TX_DESC_DATA0_DTYPE_DATA_)
1746		goto clean_up_data_descriptor;
1747	else
1748		goto clear_active;
1749
1750clean_up_data_descriptor:
1751	if (buffer_info->dma_ptr) {
1752		if (buffer_info->flags &
1753		    TX_BUFFER_INFO_FLAG_SKB_FRAGMENT) {
1754			dma_unmap_page(&tx->adapter->pdev->dev,
1755				       buffer_info->dma_ptr,
1756				       buffer_info->buffer_length,
1757				       DMA_TO_DEVICE);
1758		} else {
1759			dma_unmap_single(&tx->adapter->pdev->dev,
1760					 buffer_info->dma_ptr,
1761					 buffer_info->buffer_length,
1762					 DMA_TO_DEVICE);
1763		}
1764		buffer_info->dma_ptr = 0;
1765		buffer_info->buffer_length = 0;
1766	}
1767	if (!buffer_info->skb)
1768		goto clear_active;
1769
1770	if (!(buffer_info->flags & TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED)) {
1771		dev_kfree_skb_any(buffer_info->skb);
1772		goto clear_skb;
1773	}
1774
1775	if (cleanup) {
1776		lan743x_ptp_unrequest_tx_timestamp(tx->adapter);
1777		dev_kfree_skb_any(buffer_info->skb);
1778	} else {
1779		ignore_sync = (buffer_info->flags &
1780			       TX_BUFFER_INFO_FLAG_IGNORE_SYNC) != 0;
1781		lan743x_ptp_tx_timestamp_skb(tx->adapter,
1782					     buffer_info->skb, ignore_sync);
1783	}
1784
1785clear_skb:
1786	buffer_info->skb = NULL;
1787
1788clear_active:
1789	buffer_info->flags &= ~TX_BUFFER_INFO_FLAG_ACTIVE;
1790
1791done:
1792	memset(buffer_info, 0, sizeof(*buffer_info));
1793	memset(descriptor, 0, sizeof(*descriptor));
1794}
1795
1796static int lan743x_tx_next_index(struct lan743x_tx *tx, int index)
1797{
1798	return ((++index) % tx->ring_size);
1799}
1800
1801static void lan743x_tx_release_completed_descriptors(struct lan743x_tx *tx)
1802{
1803	while (le32_to_cpu(*tx->head_cpu_ptr) != (tx->last_head)) {
1804		lan743x_tx_release_desc(tx, tx->last_head, false);
1805		tx->last_head = lan743x_tx_next_index(tx, tx->last_head);
1806	}
1807}
1808
1809static void lan743x_tx_release_all_descriptors(struct lan743x_tx *tx)
1810{
1811	u32 original_head = 0;
1812
1813	original_head = tx->last_head;
1814	do {
1815		lan743x_tx_release_desc(tx, tx->last_head, true);
1816		tx->last_head = lan743x_tx_next_index(tx, tx->last_head);
1817	} while (tx->last_head != original_head);
1818	memset(tx->ring_cpu_ptr, 0,
1819	       sizeof(*tx->ring_cpu_ptr) * (tx->ring_size));
1820	memset(tx->buffer_info, 0,
1821	       sizeof(*tx->buffer_info) * (tx->ring_size));
1822}
1823
1824static int lan743x_tx_get_desc_cnt(struct lan743x_tx *tx,
1825				   struct sk_buff *skb)
1826{
1827	int result = 1; /* 1 for the main skb buffer */
1828	int nr_frags = 0;
1829
1830	if (skb_is_gso(skb))
1831		result++; /* requires an extension descriptor */
1832	nr_frags = skb_shinfo(skb)->nr_frags;
1833	result += nr_frags; /* 1 for each fragment buffer */
1834	return result;
1835}
1836
1837static int lan743x_tx_get_avail_desc(struct lan743x_tx *tx)
1838{
1839	int last_head = tx->last_head;
1840	int last_tail = tx->last_tail;
1841
1842	if (last_tail >= last_head)
1843		return tx->ring_size - last_tail + last_head - 1;
1844	else
1845		return last_head - last_tail - 1;
1846}
1847
1848void lan743x_tx_set_timestamping_mode(struct lan743x_tx *tx,
1849				      bool enable_timestamping,
1850				      bool enable_onestep_sync)
1851{
1852	if (enable_timestamping)
1853		tx->ts_flags |= TX_TS_FLAG_TIMESTAMPING_ENABLED;
1854	else
1855		tx->ts_flags &= ~TX_TS_FLAG_TIMESTAMPING_ENABLED;
1856	if (enable_onestep_sync)
1857		tx->ts_flags |= TX_TS_FLAG_ONE_STEP_SYNC;
1858	else
1859		tx->ts_flags &= ~TX_TS_FLAG_ONE_STEP_SYNC;
1860}
1861
1862static int lan743x_tx_frame_start(struct lan743x_tx *tx,
1863				  unsigned char *first_buffer,
1864				  unsigned int first_buffer_length,
1865				  unsigned int frame_length,
1866				  bool time_stamp,
1867				  bool check_sum)
1868{
1869	/* called only from within lan743x_tx_xmit_frame.
1870	 * assuming tx->ring_lock has already been acquired.
1871	 */
1872	struct lan743x_tx_descriptor *tx_descriptor = NULL;
1873	struct lan743x_tx_buffer_info *buffer_info = NULL;
1874	struct lan743x_adapter *adapter = tx->adapter;
1875	struct device *dev = &adapter->pdev->dev;
1876	dma_addr_t dma_ptr;
1877
1878	tx->frame_flags |= TX_FRAME_FLAG_IN_PROGRESS;
1879	tx->frame_first = tx->last_tail;
1880	tx->frame_tail = tx->frame_first;
1881
1882	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1883	buffer_info = &tx->buffer_info[tx->frame_tail];
1884	dma_ptr = dma_map_single(dev, first_buffer, first_buffer_length,
1885				 DMA_TO_DEVICE);
1886	if (dma_mapping_error(dev, dma_ptr))
1887		return -ENOMEM;
1888
1889	tx_descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(dma_ptr));
1890	tx_descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(dma_ptr));
1891	tx_descriptor->data3 = cpu_to_le32((frame_length << 16) &
1892		TX_DESC_DATA3_FRAME_LENGTH_MSS_MASK_);
1893
1894	buffer_info->skb = NULL;
1895	buffer_info->dma_ptr = dma_ptr;
1896	buffer_info->buffer_length = first_buffer_length;
1897	buffer_info->flags |= TX_BUFFER_INFO_FLAG_ACTIVE;
1898
1899	tx->frame_data0 = (first_buffer_length &
1900		TX_DESC_DATA0_BUF_LENGTH_MASK_) |
1901		TX_DESC_DATA0_DTYPE_DATA_ |
1902		TX_DESC_DATA0_FS_ |
1903		TX_DESC_DATA0_FCS_;
1904	if (time_stamp)
1905		tx->frame_data0 |= TX_DESC_DATA0_TSE_;
1906
1907	if (check_sum)
1908		tx->frame_data0 |= TX_DESC_DATA0_ICE_ |
1909				   TX_DESC_DATA0_IPE_ |
1910				   TX_DESC_DATA0_TPE_;
1911
1912	/* data0 will be programmed in one of other frame assembler functions */
1913	return 0;
1914}
1915
1916static void lan743x_tx_frame_add_lso(struct lan743x_tx *tx,
1917				     unsigned int frame_length,
1918				     int nr_frags)
1919{
1920	/* called only from within lan743x_tx_xmit_frame.
1921	 * assuming tx->ring_lock has already been acquired.
1922	 */
1923	struct lan743x_tx_descriptor *tx_descriptor = NULL;
1924	struct lan743x_tx_buffer_info *buffer_info = NULL;
1925
1926	/* wrap up previous descriptor */
1927	tx->frame_data0 |= TX_DESC_DATA0_EXT_;
1928	if (nr_frags <= 0) {
1929		tx->frame_data0 |= TX_DESC_DATA0_LS_;
1930		tx->frame_data0 |= TX_DESC_DATA0_IOC_;
1931	}
1932	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1933	tx_descriptor->data0 = cpu_to_le32(tx->frame_data0);
1934
1935	/* move to next descriptor */
1936	tx->frame_tail = lan743x_tx_next_index(tx, tx->frame_tail);
1937	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1938	buffer_info = &tx->buffer_info[tx->frame_tail];
1939
1940	/* add extension descriptor */
1941	tx_descriptor->data1 = 0;
1942	tx_descriptor->data2 = 0;
1943	tx_descriptor->data3 = 0;
1944
1945	buffer_info->skb = NULL;
1946	buffer_info->dma_ptr = 0;
1947	buffer_info->buffer_length = 0;
1948	buffer_info->flags |= TX_BUFFER_INFO_FLAG_ACTIVE;
1949
1950	tx->frame_data0 = (frame_length & TX_DESC_DATA0_EXT_PAY_LENGTH_MASK_) |
1951			  TX_DESC_DATA0_DTYPE_EXT_ |
1952			  TX_DESC_DATA0_EXT_LSO_;
1953
1954	/* data0 will be programmed in one of other frame assembler functions */
1955}
1956
1957static int lan743x_tx_frame_add_fragment(struct lan743x_tx *tx,
1958					 const skb_frag_t *fragment,
1959					 unsigned int frame_length)
1960{
1961	/* called only from within lan743x_tx_xmit_frame
1962	 * assuming tx->ring_lock has already been acquired
1963	 */
1964	struct lan743x_tx_descriptor *tx_descriptor = NULL;
1965	struct lan743x_tx_buffer_info *buffer_info = NULL;
1966	struct lan743x_adapter *adapter = tx->adapter;
1967	struct device *dev = &adapter->pdev->dev;
1968	unsigned int fragment_length = 0;
1969	dma_addr_t dma_ptr;
1970
1971	fragment_length = skb_frag_size(fragment);
1972	if (!fragment_length)
1973		return 0;
1974
1975	/* wrap up previous descriptor */
1976	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1977	tx_descriptor->data0 = cpu_to_le32(tx->frame_data0);
1978
1979	/* move to next descriptor */
1980	tx->frame_tail = lan743x_tx_next_index(tx, tx->frame_tail);
1981	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1982	buffer_info = &tx->buffer_info[tx->frame_tail];
1983	dma_ptr = skb_frag_dma_map(dev, fragment,
1984				   0, fragment_length,
1985				   DMA_TO_DEVICE);
1986	if (dma_mapping_error(dev, dma_ptr)) {
1987		int desc_index;
1988
1989		/* cleanup all previously setup descriptors */
1990		desc_index = tx->frame_first;
1991		while (desc_index != tx->frame_tail) {
1992			lan743x_tx_release_desc(tx, desc_index, true);
1993			desc_index = lan743x_tx_next_index(tx, desc_index);
1994		}
1995		dma_wmb();
1996		tx->frame_flags &= ~TX_FRAME_FLAG_IN_PROGRESS;
1997		tx->frame_first = 0;
1998		tx->frame_data0 = 0;
1999		tx->frame_tail = 0;
2000		return -ENOMEM;
2001	}
2002
2003	tx_descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(dma_ptr));
2004	tx_descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(dma_ptr));
2005	tx_descriptor->data3 = cpu_to_le32((frame_length << 16) &
2006			       TX_DESC_DATA3_FRAME_LENGTH_MSS_MASK_);
2007
2008	buffer_info->skb = NULL;
2009	buffer_info->dma_ptr = dma_ptr;
2010	buffer_info->buffer_length = fragment_length;
2011	buffer_info->flags |= TX_BUFFER_INFO_FLAG_ACTIVE;
2012	buffer_info->flags |= TX_BUFFER_INFO_FLAG_SKB_FRAGMENT;
2013
2014	tx->frame_data0 = (fragment_length & TX_DESC_DATA0_BUF_LENGTH_MASK_) |
2015			  TX_DESC_DATA0_DTYPE_DATA_ |
2016			  TX_DESC_DATA0_FCS_;
2017
2018	/* data0 will be programmed in one of other frame assembler functions */
2019	return 0;
2020}
2021
2022static void lan743x_tx_frame_end(struct lan743x_tx *tx,
2023				 struct sk_buff *skb,
2024				 bool time_stamp,
2025				 bool ignore_sync)
2026{
2027	/* called only from within lan743x_tx_xmit_frame
2028	 * assuming tx->ring_lock has already been acquired
2029	 */
2030	struct lan743x_tx_descriptor *tx_descriptor = NULL;
2031	struct lan743x_tx_buffer_info *buffer_info = NULL;
2032	struct lan743x_adapter *adapter = tx->adapter;
2033	u32 tx_tail_flags = 0;
2034
2035	/* wrap up previous descriptor */
2036	if ((tx->frame_data0 & TX_DESC_DATA0_DTYPE_MASK_) ==
2037	    TX_DESC_DATA0_DTYPE_DATA_) {
2038		tx->frame_data0 |= TX_DESC_DATA0_LS_;
2039		tx->frame_data0 |= TX_DESC_DATA0_IOC_;
2040	}
2041
2042	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
2043	buffer_info = &tx->buffer_info[tx->frame_tail];
2044	buffer_info->skb = skb;
2045	if (time_stamp)
2046		buffer_info->flags |= TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED;
2047	if (ignore_sync)
2048		buffer_info->flags |= TX_BUFFER_INFO_FLAG_IGNORE_SYNC;
2049
2050	tx_descriptor->data0 = cpu_to_le32(tx->frame_data0);
2051	tx->frame_tail = lan743x_tx_next_index(tx, tx->frame_tail);
2052	tx->last_tail = tx->frame_tail;
2053
2054	dma_wmb();
2055
2056	if (tx->vector_flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET)
2057		tx_tail_flags |= TX_TAIL_SET_TOP_INT_VEC_EN_;
2058	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET)
2059		tx_tail_flags |= TX_TAIL_SET_DMAC_INT_EN_ |
2060		TX_TAIL_SET_TOP_INT_EN_;
2061
2062	lan743x_csr_write(adapter, TX_TAIL(tx->channel_number),
2063			  tx_tail_flags | tx->frame_tail);
2064	tx->frame_flags &= ~TX_FRAME_FLAG_IN_PROGRESS;
2065}
2066
2067static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx,
2068					 struct sk_buff *skb)
2069{
2070	int required_number_of_descriptors = 0;
2071	unsigned int start_frame_length = 0;
2072	netdev_tx_t retval = NETDEV_TX_OK;
2073	unsigned int frame_length = 0;
2074	unsigned int head_length = 0;
2075	unsigned long irq_flags = 0;
2076	bool do_timestamp = false;
2077	bool ignore_sync = false;
2078	struct netdev_queue *txq;
2079	int nr_frags = 0;
2080	bool gso = false;
2081	int j;
2082
2083	required_number_of_descriptors = lan743x_tx_get_desc_cnt(tx, skb);
2084
2085	spin_lock_irqsave(&tx->ring_lock, irq_flags);
2086	if (required_number_of_descriptors >
2087		lan743x_tx_get_avail_desc(tx)) {
2088		if (required_number_of_descriptors > (tx->ring_size - 1)) {
2089			dev_kfree_skb_irq(skb);
2090		} else {
2091			/* save how many descriptors we needed to restart the queue */
2092			tx->rqd_descriptors = required_number_of_descriptors;
2093			retval = NETDEV_TX_BUSY;
2094			txq = netdev_get_tx_queue(tx->adapter->netdev,
2095						  tx->channel_number);
2096			netif_tx_stop_queue(txq);
2097		}
2098		goto unlock;
2099	}
2100
2101	/* space available, transmit skb  */
2102	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2103	    (tx->ts_flags & TX_TS_FLAG_TIMESTAMPING_ENABLED) &&
2104	    (lan743x_ptp_request_tx_timestamp(tx->adapter))) {
2105		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2106		do_timestamp = true;
2107		if (tx->ts_flags & TX_TS_FLAG_ONE_STEP_SYNC)
2108			ignore_sync = true;
2109	}
2110	head_length = skb_headlen(skb);
2111	frame_length = skb_pagelen(skb);
2112	nr_frags = skb_shinfo(skb)->nr_frags;
2113	start_frame_length = frame_length;
2114	gso = skb_is_gso(skb);
2115	if (gso) {
2116		start_frame_length = max(skb_shinfo(skb)->gso_size,
2117					 (unsigned short)8);
2118	}
2119
2120	if (lan743x_tx_frame_start(tx,
2121				   skb->data, head_length,
2122				   start_frame_length,
2123				   do_timestamp,
2124				   skb->ip_summed == CHECKSUM_PARTIAL)) {
2125		dev_kfree_skb_irq(skb);
2126		goto unlock;
2127	}
2128	tx->frame_count++;
2129
2130	if (gso)
2131		lan743x_tx_frame_add_lso(tx, frame_length, nr_frags);
2132
2133	if (nr_frags <= 0)
2134		goto finish;
2135
2136	for (j = 0; j < nr_frags; j++) {
2137		const skb_frag_t *frag = &(skb_shinfo(skb)->frags[j]);
2138
2139		if (lan743x_tx_frame_add_fragment(tx, frag, frame_length)) {
2140			/* upon error no need to call
2141			 *	lan743x_tx_frame_end
2142			 * frame assembler clean up was performed inside
2143			 *	lan743x_tx_frame_add_fragment
2144			 */
2145			dev_kfree_skb_irq(skb);
2146			goto unlock;
2147		}
2148	}
2149
2150finish:
2151	lan743x_tx_frame_end(tx, skb, do_timestamp, ignore_sync);
2152
2153unlock:
2154	spin_unlock_irqrestore(&tx->ring_lock, irq_flags);
2155	return retval;
2156}
2157
2158static int lan743x_tx_napi_poll(struct napi_struct *napi, int weight)
2159{
2160	struct lan743x_tx *tx = container_of(napi, struct lan743x_tx, napi);
2161	struct lan743x_adapter *adapter = tx->adapter;
2162	unsigned long irq_flags = 0;
2163	struct netdev_queue *txq;
2164	u32 ioc_bit = 0;
2165
2166	ioc_bit = DMAC_INT_BIT_TX_IOC_(tx->channel_number);
2167	lan743x_csr_read(adapter, DMAC_INT_STS);
2168	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C)
2169		lan743x_csr_write(adapter, DMAC_INT_STS, ioc_bit);
2170	spin_lock_irqsave(&tx->ring_lock, irq_flags);
2171
2172	/* clean up tx ring */
2173	lan743x_tx_release_completed_descriptors(tx);
2174	txq = netdev_get_tx_queue(adapter->netdev, tx->channel_number);
2175	if (netif_tx_queue_stopped(txq)) {
2176		if (tx->rqd_descriptors) {
2177			if (tx->rqd_descriptors <=
2178			    lan743x_tx_get_avail_desc(tx)) {
2179				tx->rqd_descriptors = 0;
2180				netif_tx_wake_queue(txq);
2181			}
2182		} else {
2183			netif_tx_wake_queue(txq);
2184		}
2185	}
2186	spin_unlock_irqrestore(&tx->ring_lock, irq_flags);
2187
2188	if (!napi_complete(napi))
2189		goto done;
2190
2191	/* enable isr */
2192	lan743x_csr_write(adapter, INT_EN_SET,
2193			  INT_BIT_DMA_TX_(tx->channel_number));
2194	lan743x_csr_read(adapter, INT_STS);
2195
2196done:
2197	return 0;
2198}
2199
2200static void lan743x_tx_ring_cleanup(struct lan743x_tx *tx)
2201{
2202	if (tx->head_cpu_ptr) {
2203		dma_free_coherent(&tx->adapter->pdev->dev,
2204				  sizeof(*tx->head_cpu_ptr), tx->head_cpu_ptr,
2205				  tx->head_dma_ptr);
2206		tx->head_cpu_ptr = NULL;
2207		tx->head_dma_ptr = 0;
2208	}
2209	kfree(tx->buffer_info);
2210	tx->buffer_info = NULL;
2211
2212	if (tx->ring_cpu_ptr) {
2213		dma_free_coherent(&tx->adapter->pdev->dev,
2214				  tx->ring_allocation_size, tx->ring_cpu_ptr,
2215				  tx->ring_dma_ptr);
2216		tx->ring_allocation_size = 0;
2217		tx->ring_cpu_ptr = NULL;
2218		tx->ring_dma_ptr = 0;
2219	}
2220	tx->ring_size = 0;
2221}
2222
2223static int lan743x_tx_ring_init(struct lan743x_tx *tx)
2224{
2225	size_t ring_allocation_size = 0;
2226	void *cpu_ptr = NULL;
2227	dma_addr_t dma_ptr;
2228	int ret = -ENOMEM;
2229
2230	tx->ring_size = LAN743X_TX_RING_SIZE;
2231	if (tx->ring_size & ~TX_CFG_B_TX_RING_LEN_MASK_) {
2232		ret = -EINVAL;
2233		goto cleanup;
2234	}
2235	if (dma_set_mask_and_coherent(&tx->adapter->pdev->dev,
2236				      DMA_BIT_MASK(64))) {
2237		dev_warn(&tx->adapter->pdev->dev,
2238			 "lan743x_: No suitable DMA available\n");
2239		ret = -ENOMEM;
2240		goto cleanup;
2241	}
2242	ring_allocation_size = ALIGN(tx->ring_size *
2243				     sizeof(struct lan743x_tx_descriptor),
2244				     PAGE_SIZE);
2245	dma_ptr = 0;
2246	cpu_ptr = dma_alloc_coherent(&tx->adapter->pdev->dev,
2247				     ring_allocation_size, &dma_ptr, GFP_KERNEL);
2248	if (!cpu_ptr) {
2249		ret = -ENOMEM;
2250		goto cleanup;
2251	}
2252
2253	tx->ring_allocation_size = ring_allocation_size;
2254	tx->ring_cpu_ptr = (struct lan743x_tx_descriptor *)cpu_ptr;
2255	tx->ring_dma_ptr = dma_ptr;
2256
2257	cpu_ptr = kcalloc(tx->ring_size, sizeof(*tx->buffer_info), GFP_KERNEL);
2258	if (!cpu_ptr) {
2259		ret = -ENOMEM;
2260		goto cleanup;
2261	}
2262	tx->buffer_info = (struct lan743x_tx_buffer_info *)cpu_ptr;
2263	dma_ptr = 0;
2264	cpu_ptr = dma_alloc_coherent(&tx->adapter->pdev->dev,
2265				     sizeof(*tx->head_cpu_ptr), &dma_ptr,
2266				     GFP_KERNEL);
2267	if (!cpu_ptr) {
2268		ret = -ENOMEM;
2269		goto cleanup;
2270	}
2271
2272	tx->head_cpu_ptr = cpu_ptr;
2273	tx->head_dma_ptr = dma_ptr;
2274	if (tx->head_dma_ptr & 0x3) {
2275		ret = -ENOMEM;
2276		goto cleanup;
2277	}
2278
2279	return 0;
2280
2281cleanup:
2282	lan743x_tx_ring_cleanup(tx);
2283	return ret;
2284}
2285
2286static void lan743x_tx_close(struct lan743x_tx *tx)
2287{
2288	struct lan743x_adapter *adapter = tx->adapter;
2289
2290	lan743x_csr_write(adapter,
2291			  DMAC_CMD,
2292			  DMAC_CMD_STOP_T_(tx->channel_number));
2293	lan743x_dmac_tx_wait_till_stopped(adapter, tx->channel_number);
2294
2295	lan743x_csr_write(adapter,
2296			  DMAC_INT_EN_CLR,
2297			  DMAC_INT_BIT_TX_IOC_(tx->channel_number));
2298	lan743x_csr_write(adapter, INT_EN_CLR,
2299			  INT_BIT_DMA_TX_(tx->channel_number));
2300	napi_disable(&tx->napi);
2301	netif_napi_del(&tx->napi);
2302
2303	lan743x_csr_write(adapter, FCT_TX_CTL,
2304			  FCT_TX_CTL_DIS_(tx->channel_number));
2305	lan743x_csr_wait_for_bit(adapter, FCT_TX_CTL,
2306				 FCT_TX_CTL_EN_(tx->channel_number),
2307				 0, 1000, 20000, 100);
2308
2309	lan743x_tx_release_all_descriptors(tx);
2310
2311	tx->rqd_descriptors = 0;
2312
2313	lan743x_tx_ring_cleanup(tx);
2314}
2315
2316static int lan743x_tx_open(struct lan743x_tx *tx)
2317{
2318	struct lan743x_adapter *adapter = NULL;
2319	u32 data = 0;
2320	int ret;
2321
2322	adapter = tx->adapter;
2323	ret = lan743x_tx_ring_init(tx);
2324	if (ret)
2325		return ret;
2326
2327	/* initialize fifo */
2328	lan743x_csr_write(adapter, FCT_TX_CTL,
2329			  FCT_TX_CTL_RESET_(tx->channel_number));
2330	lan743x_csr_wait_for_bit(adapter, FCT_TX_CTL,
2331				 FCT_TX_CTL_RESET_(tx->channel_number),
2332				 0, 1000, 20000, 100);
2333
2334	/* enable fifo */
2335	lan743x_csr_write(adapter, FCT_TX_CTL,
2336			  FCT_TX_CTL_EN_(tx->channel_number));
2337
2338	/* reset tx channel */
2339	lan743x_csr_write(adapter, DMAC_CMD,
2340			  DMAC_CMD_TX_SWR_(tx->channel_number));
2341	lan743x_csr_wait_for_bit(adapter, DMAC_CMD,
2342				 DMAC_CMD_TX_SWR_(tx->channel_number),
2343				 0, 1000, 20000, 100);
2344
2345	/* Write TX_BASE_ADDR */
2346	lan743x_csr_write(adapter,
2347			  TX_BASE_ADDRH(tx->channel_number),
2348			  DMA_ADDR_HIGH32(tx->ring_dma_ptr));
2349	lan743x_csr_write(adapter,
2350			  TX_BASE_ADDRL(tx->channel_number),
2351			  DMA_ADDR_LOW32(tx->ring_dma_ptr));
2352
2353	/* Write TX_CFG_B */
2354	data = lan743x_csr_read(adapter, TX_CFG_B(tx->channel_number));
2355	data &= ~TX_CFG_B_TX_RING_LEN_MASK_;
2356	data |= ((tx->ring_size) & TX_CFG_B_TX_RING_LEN_MASK_);
2357	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0))
2358		data |= TX_CFG_B_TDMABL_512_;
2359	lan743x_csr_write(adapter, TX_CFG_B(tx->channel_number), data);
2360
2361	/* Write TX_CFG_A */
2362	data = TX_CFG_A_TX_TMR_HPWB_SEL_IOC_ | TX_CFG_A_TX_HP_WB_EN_;
2363	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
2364		data |= TX_CFG_A_TX_HP_WB_ON_INT_TMR_;
2365		data |= TX_CFG_A_TX_PF_THRES_SET_(0x10);
2366		data |= TX_CFG_A_TX_PF_PRI_THRES_SET_(0x04);
2367		data |= TX_CFG_A_TX_HP_WB_THRES_SET_(0x07);
2368	}
2369	lan743x_csr_write(adapter, TX_CFG_A(tx->channel_number), data);
2370
2371	/* Write TX_HEAD_WRITEBACK_ADDR */
2372	lan743x_csr_write(adapter,
2373			  TX_HEAD_WRITEBACK_ADDRH(tx->channel_number),
2374			  DMA_ADDR_HIGH32(tx->head_dma_ptr));
2375	lan743x_csr_write(adapter,
2376			  TX_HEAD_WRITEBACK_ADDRL(tx->channel_number),
2377			  DMA_ADDR_LOW32(tx->head_dma_ptr));
2378
2379	/* set last head */
2380	tx->last_head = lan743x_csr_read(adapter, TX_HEAD(tx->channel_number));
2381
2382	/* write TX_TAIL */
2383	tx->last_tail = 0;
2384	lan743x_csr_write(adapter, TX_TAIL(tx->channel_number),
2385			  (u32)(tx->last_tail));
2386	tx->vector_flags = lan743x_intr_get_vector_flags(adapter,
2387							 INT_BIT_DMA_TX_
2388							 (tx->channel_number));
2389	netif_napi_add_tx_weight(adapter->netdev,
2390				 &tx->napi, lan743x_tx_napi_poll,
2391				 NAPI_POLL_WEIGHT);
2392	napi_enable(&tx->napi);
2393
2394	data = 0;
2395	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR)
2396		data |= TX_CFG_C_TX_TOP_INT_EN_AUTO_CLR_;
2397	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR)
2398		data |= TX_CFG_C_TX_DMA_INT_STS_AUTO_CLR_;
2399	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C)
2400		data |= TX_CFG_C_TX_INT_STS_R2C_MODE_MASK_;
2401	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C)
2402		data |= TX_CFG_C_TX_INT_EN_R2C_;
2403	lan743x_csr_write(adapter, TX_CFG_C(tx->channel_number), data);
2404
2405	if (!(tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET))
2406		lan743x_csr_write(adapter, INT_EN_SET,
2407				  INT_BIT_DMA_TX_(tx->channel_number));
2408	lan743x_csr_write(adapter, DMAC_INT_EN_SET,
2409			  DMAC_INT_BIT_TX_IOC_(tx->channel_number));
2410
2411	/*  start dmac channel */
2412	lan743x_csr_write(adapter, DMAC_CMD,
2413			  DMAC_CMD_START_T_(tx->channel_number));
2414	return 0;
2415}
2416
2417static int lan743x_rx_next_index(struct lan743x_rx *rx, int index)
2418{
2419	return ((++index) % rx->ring_size);
2420}
2421
2422static void lan743x_rx_update_tail(struct lan743x_rx *rx, int index)
2423{
2424	/* update the tail once per 8 descriptors */
2425	if ((index & 7) == 7)
2426		lan743x_csr_write(rx->adapter, RX_TAIL(rx->channel_number),
2427				  index);
2428}
2429
2430static int lan743x_rx_init_ring_element(struct lan743x_rx *rx, int index,
2431					gfp_t gfp)
2432{
2433	struct net_device *netdev = rx->adapter->netdev;
2434	struct device *dev = &rx->adapter->pdev->dev;
2435	struct lan743x_rx_buffer_info *buffer_info;
2436	unsigned int buffer_length, used_length;
2437	struct lan743x_rx_descriptor *descriptor;
2438	struct sk_buff *skb;
2439	dma_addr_t dma_ptr;
2440
2441	buffer_length = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + RX_HEAD_PADDING;
2442
2443	descriptor = &rx->ring_cpu_ptr[index];
2444	buffer_info = &rx->buffer_info[index];
2445	skb = __netdev_alloc_skb(netdev, buffer_length, gfp);
2446	if (!skb)
2447		return -ENOMEM;
2448	dma_ptr = dma_map_single(dev, skb->data, buffer_length, DMA_FROM_DEVICE);
2449	if (dma_mapping_error(dev, dma_ptr)) {
2450		dev_kfree_skb_any(skb);
2451		return -ENOMEM;
2452	}
2453	if (buffer_info->dma_ptr) {
2454		/* sync used area of buffer only */
2455		if (le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_LS_)
2456			/* frame length is valid only if LS bit is set.
2457			 * it's a safe upper bound for the used area in this
2458			 * buffer.
2459			 */
2460			used_length = min(RX_DESC_DATA0_FRAME_LENGTH_GET_
2461					  (le32_to_cpu(descriptor->data0)),
2462					  buffer_info->buffer_length);
2463		else
2464			used_length = buffer_info->buffer_length;
2465		dma_sync_single_for_cpu(dev, buffer_info->dma_ptr,
2466					used_length,
2467					DMA_FROM_DEVICE);
2468		dma_unmap_single_attrs(dev, buffer_info->dma_ptr,
2469				       buffer_info->buffer_length,
2470				       DMA_FROM_DEVICE,
2471				       DMA_ATTR_SKIP_CPU_SYNC);
2472	}
2473
2474	buffer_info->skb = skb;
2475	buffer_info->dma_ptr = dma_ptr;
2476	buffer_info->buffer_length = buffer_length;
2477	descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(buffer_info->dma_ptr));
2478	descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(buffer_info->dma_ptr));
2479	descriptor->data3 = 0;
2480	descriptor->data0 = cpu_to_le32((RX_DESC_DATA0_OWN_ |
2481			    (buffer_length & RX_DESC_DATA0_BUF_LENGTH_MASK_)));
2482	lan743x_rx_update_tail(rx, index);
2483
2484	return 0;
2485}
2486
2487static void lan743x_rx_reuse_ring_element(struct lan743x_rx *rx, int index)
2488{
2489	struct lan743x_rx_buffer_info *buffer_info;
2490	struct lan743x_rx_descriptor *descriptor;
2491
2492	descriptor = &rx->ring_cpu_ptr[index];
2493	buffer_info = &rx->buffer_info[index];
2494
2495	descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(buffer_info->dma_ptr));
2496	descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(buffer_info->dma_ptr));
2497	descriptor->data3 = 0;
2498	descriptor->data0 = cpu_to_le32((RX_DESC_DATA0_OWN_ |
2499			    ((buffer_info->buffer_length) &
2500			    RX_DESC_DATA0_BUF_LENGTH_MASK_)));
2501	lan743x_rx_update_tail(rx, index);
2502}
2503
2504static void lan743x_rx_release_ring_element(struct lan743x_rx *rx, int index)
2505{
2506	struct lan743x_rx_buffer_info *buffer_info;
2507	struct lan743x_rx_descriptor *descriptor;
2508
2509	descriptor = &rx->ring_cpu_ptr[index];
2510	buffer_info = &rx->buffer_info[index];
2511
2512	memset(descriptor, 0, sizeof(*descriptor));
2513
2514	if (buffer_info->dma_ptr) {
2515		dma_unmap_single(&rx->adapter->pdev->dev,
2516				 buffer_info->dma_ptr,
2517				 buffer_info->buffer_length,
2518				 DMA_FROM_DEVICE);
2519		buffer_info->dma_ptr = 0;
2520	}
2521
2522	if (buffer_info->skb) {
2523		dev_kfree_skb(buffer_info->skb);
2524		buffer_info->skb = NULL;
2525	}
2526
2527	memset(buffer_info, 0, sizeof(*buffer_info));
2528}
2529
2530static struct sk_buff *
2531lan743x_rx_trim_skb(struct sk_buff *skb, int frame_length)
2532{
2533	if (skb_linearize(skb)) {
2534		dev_kfree_skb_irq(skb);
2535		return NULL;
2536	}
2537	frame_length = max_t(int, 0, frame_length - ETH_FCS_LEN);
2538	if (skb->len > frame_length) {
2539		skb->tail -= skb->len - frame_length;
2540		skb->len = frame_length;
2541	}
2542	return skb;
2543}
2544
2545static int lan743x_rx_process_buffer(struct lan743x_rx *rx)
2546{
2547	int current_head_index = le32_to_cpu(*rx->head_cpu_ptr);
2548	struct lan743x_rx_descriptor *descriptor, *desc_ext;
2549	struct net_device *netdev = rx->adapter->netdev;
2550	int result = RX_PROCESS_RESULT_NOTHING_TO_DO;
2551	struct lan743x_rx_buffer_info *buffer_info;
2552	int frame_length, buffer_length;
2553	bool is_ice, is_tce, is_icsm;
2554	int extension_index = -1;
2555	bool is_last, is_first;
2556	struct sk_buff *skb;
2557
2558	if (current_head_index < 0 || current_head_index >= rx->ring_size)
2559		goto done;
2560
2561	if (rx->last_head < 0 || rx->last_head >= rx->ring_size)
2562		goto done;
2563
2564	if (rx->last_head == current_head_index)
2565		goto done;
2566
2567	descriptor = &rx->ring_cpu_ptr[rx->last_head];
2568	if (le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_OWN_)
2569		goto done;
2570	buffer_info = &rx->buffer_info[rx->last_head];
2571
2572	is_last = le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_LS_;
2573	is_first = le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_FS_;
2574
2575	if (is_last && le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_EXT_) {
2576		/* extension is expected to follow */
2577		int index = lan743x_rx_next_index(rx, rx->last_head);
2578
2579		if (index == current_head_index)
2580			/* extension not yet available */
2581			goto done;
2582		desc_ext = &rx->ring_cpu_ptr[index];
2583		if (le32_to_cpu(desc_ext->data0) & RX_DESC_DATA0_OWN_)
2584			/* extension not yet available */
2585			goto done;
2586		if (!(le32_to_cpu(desc_ext->data0) & RX_DESC_DATA0_EXT_))
2587			goto move_forward;
2588		extension_index = index;
2589	}
2590
2591	/* Only the last buffer in a multi-buffer frame contains the total frame
2592	 * length. The chip occasionally sends more buffers than strictly
2593	 * required to reach the total frame length.
2594	 * Handle this by adding all buffers to the skb in their entirety.
2595	 * Once the real frame length is known, trim the skb.
2596	 */
2597	frame_length =
2598		RX_DESC_DATA0_FRAME_LENGTH_GET_(le32_to_cpu(descriptor->data0));
2599	buffer_length = buffer_info->buffer_length;
2600	is_ice = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_ICE_;
2601	is_tce = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_TCE_;
2602	is_icsm = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_ICSM_;
2603
2604	netdev_dbg(netdev, "%s%schunk: %d/%d",
2605		   is_first ? "first " : "      ",
2606		   is_last  ? "last  " : "      ",
2607		   frame_length, buffer_length);
2608
2609	/* save existing skb, allocate new skb and map to dma */
2610	skb = buffer_info->skb;
2611	if (lan743x_rx_init_ring_element(rx, rx->last_head,
2612					 GFP_ATOMIC | GFP_DMA)) {
2613		/* failed to allocate next skb.
2614		 * Memory is very low.
2615		 * Drop this packet and reuse buffer.
2616		 */
2617		lan743x_rx_reuse_ring_element(rx, rx->last_head);
2618		/* drop packet that was being assembled */
2619		dev_kfree_skb_irq(rx->skb_head);
2620		rx->skb_head = NULL;
2621		goto process_extension;
2622	}
2623
2624	/* add buffers to skb via skb->frag_list */
2625	if (is_first) {
2626		skb_reserve(skb, RX_HEAD_PADDING);
2627		skb_put(skb, buffer_length - RX_HEAD_PADDING);
2628		if (rx->skb_head)
2629			dev_kfree_skb_irq(rx->skb_head);
2630		rx->skb_head = skb;
2631	} else if (rx->skb_head) {
2632		skb_put(skb, buffer_length);
2633		if (skb_shinfo(rx->skb_head)->frag_list)
2634			rx->skb_tail->next = skb;
2635		else
2636			skb_shinfo(rx->skb_head)->frag_list = skb;
2637		rx->skb_tail = skb;
2638		rx->skb_head->len += skb->len;
2639		rx->skb_head->data_len += skb->len;
2640		rx->skb_head->truesize += skb->truesize;
2641	} else {
2642		/* packet to assemble has already been dropped because one or
2643		 * more of its buffers could not be allocated
2644		 */
2645		netdev_dbg(netdev, "drop buffer intended for dropped packet");
2646		dev_kfree_skb_irq(skb);
2647	}
2648
2649process_extension:
2650	if (extension_index >= 0) {
2651		u32 ts_sec;
2652		u32 ts_nsec;
2653
2654		ts_sec = le32_to_cpu(desc_ext->data1);
2655		ts_nsec = (le32_to_cpu(desc_ext->data2) &
2656			  RX_DESC_DATA2_TS_NS_MASK_);
2657		if (rx->skb_head)
2658			skb_hwtstamps(rx->skb_head)->hwtstamp =
2659				ktime_set(ts_sec, ts_nsec);
2660		lan743x_rx_reuse_ring_element(rx, extension_index);
2661		rx->last_head = extension_index;
2662		netdev_dbg(netdev, "process extension");
2663	}
2664
2665	if (is_last && rx->skb_head)
2666		rx->skb_head = lan743x_rx_trim_skb(rx->skb_head, frame_length);
2667
2668	if (is_last && rx->skb_head) {
2669		rx->skb_head->protocol = eth_type_trans(rx->skb_head,
2670							rx->adapter->netdev);
2671		if (rx->adapter->netdev->features & NETIF_F_RXCSUM) {
2672			if (!is_ice && !is_tce && !is_icsm)
2673				skb->ip_summed = CHECKSUM_UNNECESSARY;
2674		}
2675		netdev_dbg(netdev, "sending %d byte frame to OS",
2676			   rx->skb_head->len);
2677		napi_gro_receive(&rx->napi, rx->skb_head);
2678		rx->skb_head = NULL;
2679	}
2680
2681move_forward:
2682	/* push tail and head forward */
2683	rx->last_tail = rx->last_head;
2684	rx->last_head = lan743x_rx_next_index(rx, rx->last_head);
2685	result = RX_PROCESS_RESULT_BUFFER_RECEIVED;
2686done:
2687	return result;
2688}
2689
2690static int lan743x_rx_napi_poll(struct napi_struct *napi, int weight)
2691{
2692	struct lan743x_rx *rx = container_of(napi, struct lan743x_rx, napi);
2693	struct lan743x_adapter *adapter = rx->adapter;
2694	int result = RX_PROCESS_RESULT_NOTHING_TO_DO;
2695	u32 rx_tail_flags = 0;
2696	int count;
2697
2698	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C) {
2699		/* clear int status bit before reading packet */
2700		lan743x_csr_write(adapter, DMAC_INT_STS,
2701				  DMAC_INT_BIT_RXFRM_(rx->channel_number));
2702	}
2703	for (count = 0; count < weight; count++) {
2704		result = lan743x_rx_process_buffer(rx);
2705		if (result == RX_PROCESS_RESULT_NOTHING_TO_DO)
2706			break;
2707	}
2708	rx->frame_count += count;
2709	if (count == weight || result == RX_PROCESS_RESULT_BUFFER_RECEIVED)
2710		return weight;
2711
2712	if (!napi_complete_done(napi, count))
2713		return count;
2714
2715	/* re-arm interrupts, must write to rx tail on some chip variants */
2716	if (rx->vector_flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET)
2717		rx_tail_flags |= RX_TAIL_SET_TOP_INT_VEC_EN_;
2718	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET) {
2719		rx_tail_flags |= RX_TAIL_SET_TOP_INT_EN_;
2720	} else {
2721		lan743x_csr_write(adapter, INT_EN_SET,
2722				  INT_BIT_DMA_RX_(rx->channel_number));
2723	}
2724
2725	if (rx_tail_flags)
2726		lan743x_csr_write(adapter, RX_TAIL(rx->channel_number),
2727				  rx_tail_flags | rx->last_tail);
2728
2729	return count;
2730}
2731
2732static void lan743x_rx_ring_cleanup(struct lan743x_rx *rx)
2733{
2734	if (rx->buffer_info && rx->ring_cpu_ptr) {
2735		int index;
2736
2737		for (index = 0; index < rx->ring_size; index++)
2738			lan743x_rx_release_ring_element(rx, index);
2739	}
2740
2741	if (rx->head_cpu_ptr) {
2742		dma_free_coherent(&rx->adapter->pdev->dev,
2743				  sizeof(*rx->head_cpu_ptr), rx->head_cpu_ptr,
2744				  rx->head_dma_ptr);
2745		rx->head_cpu_ptr = NULL;
2746		rx->head_dma_ptr = 0;
2747	}
2748
2749	kfree(rx->buffer_info);
2750	rx->buffer_info = NULL;
2751
2752	if (rx->ring_cpu_ptr) {
2753		dma_free_coherent(&rx->adapter->pdev->dev,
2754				  rx->ring_allocation_size, rx->ring_cpu_ptr,
2755				  rx->ring_dma_ptr);
2756		rx->ring_allocation_size = 0;
2757		rx->ring_cpu_ptr = NULL;
2758		rx->ring_dma_ptr = 0;
2759	}
2760
2761	rx->ring_size = 0;
2762	rx->last_head = 0;
2763}
2764
2765static int lan743x_rx_ring_init(struct lan743x_rx *rx)
2766{
2767	size_t ring_allocation_size = 0;
2768	dma_addr_t dma_ptr = 0;
2769	void *cpu_ptr = NULL;
2770	int ret = -ENOMEM;
2771	int index = 0;
2772
2773	rx->ring_size = LAN743X_RX_RING_SIZE;
2774	if (rx->ring_size <= 1) {
2775		ret = -EINVAL;
2776		goto cleanup;
2777	}
2778	if (rx->ring_size & ~RX_CFG_B_RX_RING_LEN_MASK_) {
2779		ret = -EINVAL;
2780		goto cleanup;
2781	}
2782	if (dma_set_mask_and_coherent(&rx->adapter->pdev->dev,
2783				      DMA_BIT_MASK(64))) {
2784		dev_warn(&rx->adapter->pdev->dev,
2785			 "lan743x_: No suitable DMA available\n");
2786		ret = -ENOMEM;
2787		goto cleanup;
2788	}
2789	ring_allocation_size = ALIGN(rx->ring_size *
2790				     sizeof(struct lan743x_rx_descriptor),
2791				     PAGE_SIZE);
2792	dma_ptr = 0;
2793	cpu_ptr = dma_alloc_coherent(&rx->adapter->pdev->dev,
2794				     ring_allocation_size, &dma_ptr, GFP_KERNEL);
2795	if (!cpu_ptr) {
2796		ret = -ENOMEM;
2797		goto cleanup;
2798	}
2799	rx->ring_allocation_size = ring_allocation_size;
2800	rx->ring_cpu_ptr = (struct lan743x_rx_descriptor *)cpu_ptr;
2801	rx->ring_dma_ptr = dma_ptr;
2802
2803	cpu_ptr = kcalloc(rx->ring_size, sizeof(*rx->buffer_info),
2804			  GFP_KERNEL);
2805	if (!cpu_ptr) {
2806		ret = -ENOMEM;
2807		goto cleanup;
2808	}
2809	rx->buffer_info = (struct lan743x_rx_buffer_info *)cpu_ptr;
2810	dma_ptr = 0;
2811	cpu_ptr = dma_alloc_coherent(&rx->adapter->pdev->dev,
2812				     sizeof(*rx->head_cpu_ptr), &dma_ptr,
2813				     GFP_KERNEL);
2814	if (!cpu_ptr) {
2815		ret = -ENOMEM;
2816		goto cleanup;
2817	}
2818
2819	rx->head_cpu_ptr = cpu_ptr;
2820	rx->head_dma_ptr = dma_ptr;
2821	if (rx->head_dma_ptr & 0x3) {
2822		ret = -ENOMEM;
2823		goto cleanup;
2824	}
2825
2826	rx->last_head = 0;
2827	for (index = 0; index < rx->ring_size; index++) {
2828		ret = lan743x_rx_init_ring_element(rx, index, GFP_KERNEL);
2829		if (ret)
2830			goto cleanup;
2831	}
2832	return 0;
2833
2834cleanup:
2835	netif_warn(rx->adapter, ifup, rx->adapter->netdev,
2836		   "Error allocating memory for LAN743x\n");
2837
2838	lan743x_rx_ring_cleanup(rx);
2839	return ret;
2840}
2841
2842static void lan743x_rx_close(struct lan743x_rx *rx)
2843{
2844	struct lan743x_adapter *adapter = rx->adapter;
2845
2846	lan743x_csr_write(adapter, FCT_RX_CTL,
2847			  FCT_RX_CTL_DIS_(rx->channel_number));
2848	lan743x_csr_wait_for_bit(adapter, FCT_RX_CTL,
2849				 FCT_RX_CTL_EN_(rx->channel_number),
2850				 0, 1000, 20000, 100);
2851
2852	lan743x_csr_write(adapter, DMAC_CMD,
2853			  DMAC_CMD_STOP_R_(rx->channel_number));
2854	lan743x_dmac_rx_wait_till_stopped(adapter, rx->channel_number);
2855
2856	lan743x_csr_write(adapter, DMAC_INT_EN_CLR,
2857			  DMAC_INT_BIT_RXFRM_(rx->channel_number));
2858	lan743x_csr_write(adapter, INT_EN_CLR,
2859			  INT_BIT_DMA_RX_(rx->channel_number));
2860	napi_disable(&rx->napi);
2861
2862	netif_napi_del(&rx->napi);
2863
2864	lan743x_rx_ring_cleanup(rx);
2865}
2866
2867static int lan743x_rx_open(struct lan743x_rx *rx)
2868{
2869	struct lan743x_adapter *adapter = rx->adapter;
2870	u32 data = 0;
2871	int ret;
2872
2873	rx->frame_count = 0;
2874	ret = lan743x_rx_ring_init(rx);
2875	if (ret)
2876		goto return_error;
2877
2878	netif_napi_add(adapter->netdev, &rx->napi, lan743x_rx_napi_poll);
2879
2880	lan743x_csr_write(adapter, DMAC_CMD,
2881			  DMAC_CMD_RX_SWR_(rx->channel_number));
2882	lan743x_csr_wait_for_bit(adapter, DMAC_CMD,
2883				 DMAC_CMD_RX_SWR_(rx->channel_number),
2884				 0, 1000, 20000, 100);
2885
2886	/* set ring base address */
2887	lan743x_csr_write(adapter,
2888			  RX_BASE_ADDRH(rx->channel_number),
2889			  DMA_ADDR_HIGH32(rx->ring_dma_ptr));
2890	lan743x_csr_write(adapter,
2891			  RX_BASE_ADDRL(rx->channel_number),
2892			  DMA_ADDR_LOW32(rx->ring_dma_ptr));
2893
2894	/* set rx write back address */
2895	lan743x_csr_write(adapter,
2896			  RX_HEAD_WRITEBACK_ADDRH(rx->channel_number),
2897			  DMA_ADDR_HIGH32(rx->head_dma_ptr));
2898	lan743x_csr_write(adapter,
2899			  RX_HEAD_WRITEBACK_ADDRL(rx->channel_number),
2900			  DMA_ADDR_LOW32(rx->head_dma_ptr));
2901	data = RX_CFG_A_RX_HP_WB_EN_;
2902	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
2903		data |= (RX_CFG_A_RX_WB_ON_INT_TMR_ |
2904			RX_CFG_A_RX_WB_THRES_SET_(0x7) |
2905			RX_CFG_A_RX_PF_THRES_SET_(16) |
2906			RX_CFG_A_RX_PF_PRI_THRES_SET_(4));
2907	}
2908
2909	/* set RX_CFG_A */
2910	lan743x_csr_write(adapter,
2911			  RX_CFG_A(rx->channel_number), data);
2912
2913	/* set RX_CFG_B */
2914	data = lan743x_csr_read(adapter, RX_CFG_B(rx->channel_number));
2915	data &= ~RX_CFG_B_RX_PAD_MASK_;
2916	if (!RX_HEAD_PADDING)
2917		data |= RX_CFG_B_RX_PAD_0_;
2918	else
2919		data |= RX_CFG_B_RX_PAD_2_;
2920	data &= ~RX_CFG_B_RX_RING_LEN_MASK_;
2921	data |= ((rx->ring_size) & RX_CFG_B_RX_RING_LEN_MASK_);
2922	data |= RX_CFG_B_TS_ALL_RX_;
2923	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0))
2924		data |= RX_CFG_B_RDMABL_512_;
2925
2926	lan743x_csr_write(adapter, RX_CFG_B(rx->channel_number), data);
2927	rx->vector_flags = lan743x_intr_get_vector_flags(adapter,
2928							 INT_BIT_DMA_RX_
2929							 (rx->channel_number));
2930
2931	/* set RX_CFG_C */
2932	data = 0;
2933	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR)
2934		data |= RX_CFG_C_RX_TOP_INT_EN_AUTO_CLR_;
2935	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR)
2936		data |= RX_CFG_C_RX_DMA_INT_STS_AUTO_CLR_;
2937	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C)
2938		data |= RX_CFG_C_RX_INT_STS_R2C_MODE_MASK_;
2939	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C)
2940		data |= RX_CFG_C_RX_INT_EN_R2C_;
2941	lan743x_csr_write(adapter, RX_CFG_C(rx->channel_number), data);
2942
2943	rx->last_tail = ((u32)(rx->ring_size - 1));
2944	lan743x_csr_write(adapter, RX_TAIL(rx->channel_number),
2945			  rx->last_tail);
2946	rx->last_head = lan743x_csr_read(adapter, RX_HEAD(rx->channel_number));
2947	if (rx->last_head) {
2948		ret = -EIO;
2949		goto napi_delete;
2950	}
2951
2952	napi_enable(&rx->napi);
2953
2954	lan743x_csr_write(adapter, INT_EN_SET,
2955			  INT_BIT_DMA_RX_(rx->channel_number));
2956	lan743x_csr_write(adapter, DMAC_INT_STS,
2957			  DMAC_INT_BIT_RXFRM_(rx->channel_number));
2958	lan743x_csr_write(adapter, DMAC_INT_EN_SET,
2959			  DMAC_INT_BIT_RXFRM_(rx->channel_number));
2960	lan743x_csr_write(adapter, DMAC_CMD,
2961			  DMAC_CMD_START_R_(rx->channel_number));
2962
2963	/* initialize fifo */
2964	lan743x_csr_write(adapter, FCT_RX_CTL,
2965			  FCT_RX_CTL_RESET_(rx->channel_number));
2966	lan743x_csr_wait_for_bit(adapter, FCT_RX_CTL,
2967				 FCT_RX_CTL_RESET_(rx->channel_number),
2968				 0, 1000, 20000, 100);
2969	lan743x_csr_write(adapter, FCT_FLOW(rx->channel_number),
2970			  FCT_FLOW_CTL_REQ_EN_ |
2971			  FCT_FLOW_CTL_ON_THRESHOLD_SET_(0x2A) |
2972			  FCT_FLOW_CTL_OFF_THRESHOLD_SET_(0xA));
2973
2974	/* enable fifo */
2975	lan743x_csr_write(adapter, FCT_RX_CTL,
2976			  FCT_RX_CTL_EN_(rx->channel_number));
2977	return 0;
2978
2979napi_delete:
2980	netif_napi_del(&rx->napi);
2981	lan743x_rx_ring_cleanup(rx);
2982
2983return_error:
2984	return ret;
2985}
2986
2987static int lan743x_netdev_close(struct net_device *netdev)
2988{
2989	struct lan743x_adapter *adapter = netdev_priv(netdev);
2990	int index;
2991
2992	for (index = 0; index < adapter->used_tx_channels; index++)
2993		lan743x_tx_close(&adapter->tx[index]);
2994
2995	for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++)
2996		lan743x_rx_close(&adapter->rx[index]);
2997
2998	lan743x_ptp_close(adapter);
2999
3000	lan743x_phy_close(adapter);
3001
3002	lan743x_mac_close(adapter);
3003
3004	lan743x_intr_close(adapter);
3005
3006	return 0;
3007}
3008
3009static int lan743x_netdev_open(struct net_device *netdev)
3010{
3011	struct lan743x_adapter *adapter = netdev_priv(netdev);
3012	int index;
3013	int ret;
3014
3015	ret = lan743x_intr_open(adapter);
3016	if (ret)
3017		goto return_error;
3018
3019	ret = lan743x_mac_open(adapter);
3020	if (ret)
3021		goto close_intr;
3022
3023	ret = lan743x_phy_open(adapter);
3024	if (ret)
3025		goto close_mac;
3026
3027	ret = lan743x_ptp_open(adapter);
3028	if (ret)
3029		goto close_phy;
3030
3031	lan743x_rfe_open(adapter);
3032
3033	for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
3034		ret = lan743x_rx_open(&adapter->rx[index]);
3035		if (ret)
3036			goto close_rx;
3037	}
3038
3039	for (index = 0; index < adapter->used_tx_channels; index++) {
3040		ret = lan743x_tx_open(&adapter->tx[index]);
3041		if (ret)
3042			goto close_tx;
3043	}
3044	return 0;
3045
3046close_tx:
3047	for (index = 0; index < adapter->used_tx_channels; index++) {
3048		if (adapter->tx[index].ring_cpu_ptr)
3049			lan743x_tx_close(&adapter->tx[index]);
3050	}
3051
3052close_rx:
3053	for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
3054		if (adapter->rx[index].ring_cpu_ptr)
3055			lan743x_rx_close(&adapter->rx[index]);
3056	}
3057	lan743x_ptp_close(adapter);
3058
3059close_phy:
3060	lan743x_phy_close(adapter);
3061
3062close_mac:
3063	lan743x_mac_close(adapter);
3064
3065close_intr:
3066	lan743x_intr_close(adapter);
3067
3068return_error:
3069	netif_warn(adapter, ifup, adapter->netdev,
3070		   "Error opening LAN743x\n");
3071	return ret;
3072}
3073
3074static netdev_tx_t lan743x_netdev_xmit_frame(struct sk_buff *skb,
3075					     struct net_device *netdev)
3076{
3077	struct lan743x_adapter *adapter = netdev_priv(netdev);
3078	u8 ch = 0;
3079
3080	if (adapter->is_pci11x1x)
3081		ch = skb->queue_mapping % PCI11X1X_USED_TX_CHANNELS;
3082
3083	return lan743x_tx_xmit_frame(&adapter->tx[ch], skb);
3084}
3085
3086static int lan743x_netdev_ioctl(struct net_device *netdev,
3087				struct ifreq *ifr, int cmd)
3088{
3089	if (!netif_running(netdev))
3090		return -EINVAL;
3091	if (cmd == SIOCSHWTSTAMP)
3092		return lan743x_ptp_ioctl(netdev, ifr, cmd);
3093	return phy_mii_ioctl(netdev->phydev, ifr, cmd);
3094}
3095
3096static void lan743x_netdev_set_multicast(struct net_device *netdev)
3097{
3098	struct lan743x_adapter *adapter = netdev_priv(netdev);
3099
3100	lan743x_rfe_set_multicast(adapter);
3101}
3102
3103static int lan743x_netdev_change_mtu(struct net_device *netdev, int new_mtu)
3104{
3105	struct lan743x_adapter *adapter = netdev_priv(netdev);
3106	int ret = 0;
3107
3108	ret = lan743x_mac_set_mtu(adapter, new_mtu);
3109	if (!ret)
3110		netdev->mtu = new_mtu;
3111	return ret;
3112}
3113
3114static void lan743x_netdev_get_stats64(struct net_device *netdev,
3115				       struct rtnl_link_stats64 *stats)
3116{
3117	struct lan743x_adapter *adapter = netdev_priv(netdev);
3118
3119	stats->rx_packets = lan743x_csr_read(adapter, STAT_RX_TOTAL_FRAMES);
3120	stats->tx_packets = lan743x_csr_read(adapter, STAT_TX_TOTAL_FRAMES);
3121	stats->rx_bytes = lan743x_csr_read(adapter,
3122					   STAT_RX_UNICAST_BYTE_COUNT) +
3123			  lan743x_csr_read(adapter,
3124					   STAT_RX_BROADCAST_BYTE_COUNT) +
3125			  lan743x_csr_read(adapter,
3126					   STAT_RX_MULTICAST_BYTE_COUNT);
3127	stats->tx_bytes = lan743x_csr_read(adapter,
3128					   STAT_TX_UNICAST_BYTE_COUNT) +
3129			  lan743x_csr_read(adapter,
3130					   STAT_TX_BROADCAST_BYTE_COUNT) +
3131			  lan743x_csr_read(adapter,
3132					   STAT_TX_MULTICAST_BYTE_COUNT);
3133	stats->rx_errors = lan743x_csr_read(adapter, STAT_RX_FCS_ERRORS) +
3134			   lan743x_csr_read(adapter,
3135					    STAT_RX_ALIGNMENT_ERRORS) +
3136			   lan743x_csr_read(adapter, STAT_RX_JABBER_ERRORS) +
3137			   lan743x_csr_read(adapter,
3138					    STAT_RX_UNDERSIZE_FRAME_ERRORS) +
3139			   lan743x_csr_read(adapter,
3140					    STAT_RX_OVERSIZE_FRAME_ERRORS);
3141	stats->tx_errors = lan743x_csr_read(adapter, STAT_TX_FCS_ERRORS) +
3142			   lan743x_csr_read(adapter,
3143					    STAT_TX_EXCESS_DEFERRAL_ERRORS) +
3144			   lan743x_csr_read(adapter, STAT_TX_CARRIER_ERRORS);
3145	stats->rx_dropped = lan743x_csr_read(adapter,
3146					     STAT_RX_DROPPED_FRAMES);
3147	stats->tx_dropped = lan743x_csr_read(adapter,
3148					     STAT_TX_EXCESSIVE_COLLISION);
3149	stats->multicast = lan743x_csr_read(adapter,
3150					    STAT_RX_MULTICAST_FRAMES) +
3151			   lan743x_csr_read(adapter,
3152					    STAT_TX_MULTICAST_FRAMES);
3153	stats->collisions = lan743x_csr_read(adapter,
3154					     STAT_TX_SINGLE_COLLISIONS) +
3155			    lan743x_csr_read(adapter,
3156					     STAT_TX_MULTIPLE_COLLISIONS) +
3157			    lan743x_csr_read(adapter,
3158					     STAT_TX_LATE_COLLISIONS);
3159}
3160
3161static int lan743x_netdev_set_mac_address(struct net_device *netdev,
3162					  void *addr)
3163{
3164	struct lan743x_adapter *adapter = netdev_priv(netdev);
3165	struct sockaddr *sock_addr = addr;
3166	int ret;
3167
3168	ret = eth_prepare_mac_addr_change(netdev, sock_addr);
3169	if (ret)
3170		return ret;
3171	eth_hw_addr_set(netdev, sock_addr->sa_data);
3172	lan743x_mac_set_address(adapter, sock_addr->sa_data);
3173	lan743x_rfe_update_mac_address(adapter);
3174	return 0;
3175}
3176
3177static const struct net_device_ops lan743x_netdev_ops = {
3178	.ndo_open		= lan743x_netdev_open,
3179	.ndo_stop		= lan743x_netdev_close,
3180	.ndo_start_xmit		= lan743x_netdev_xmit_frame,
3181	.ndo_eth_ioctl		= lan743x_netdev_ioctl,
3182	.ndo_set_rx_mode	= lan743x_netdev_set_multicast,
3183	.ndo_change_mtu		= lan743x_netdev_change_mtu,
3184	.ndo_get_stats64	= lan743x_netdev_get_stats64,
3185	.ndo_set_mac_address	= lan743x_netdev_set_mac_address,
3186};
3187
3188static void lan743x_hardware_cleanup(struct lan743x_adapter *adapter)
3189{
3190	lan743x_csr_write(adapter, INT_EN_CLR, 0xFFFFFFFF);
3191}
3192
3193static void lan743x_mdiobus_cleanup(struct lan743x_adapter *adapter)
3194{
3195	mdiobus_unregister(adapter->mdiobus);
3196}
3197
3198static void lan743x_full_cleanup(struct lan743x_adapter *adapter)
3199{
3200	unregister_netdev(adapter->netdev);
3201
3202	lan743x_mdiobus_cleanup(adapter);
3203	lan743x_hardware_cleanup(adapter);
3204	lan743x_pci_cleanup(adapter);
3205}
3206
3207static int lan743x_hardware_init(struct lan743x_adapter *adapter,
3208				 struct pci_dev *pdev)
3209{
3210	struct lan743x_tx *tx;
3211	int index;
3212	int ret;
3213
3214	adapter->is_pci11x1x = is_pci11x1x_chip(adapter);
3215	if (adapter->is_pci11x1x) {
3216		adapter->max_tx_channels = PCI11X1X_MAX_TX_CHANNELS;
3217		adapter->used_tx_channels = PCI11X1X_USED_TX_CHANNELS;
3218		adapter->max_vector_count = PCI11X1X_MAX_VECTOR_COUNT;
3219		pci11x1x_strap_get_status(adapter);
3220		spin_lock_init(&adapter->eth_syslock_spinlock);
3221		mutex_init(&adapter->sgmii_rw_lock);
3222	} else {
3223		adapter->max_tx_channels = LAN743X_MAX_TX_CHANNELS;
3224		adapter->used_tx_channels = LAN743X_USED_TX_CHANNELS;
3225		adapter->max_vector_count = LAN743X_MAX_VECTOR_COUNT;
3226	}
3227
3228	adapter->intr.irq = adapter->pdev->irq;
3229	lan743x_csr_write(adapter, INT_EN_CLR, 0xFFFFFFFF);
3230
3231	ret = lan743x_gpio_init(adapter);
3232	if (ret)
3233		return ret;
3234
3235	ret = lan743x_mac_init(adapter);
3236	if (ret)
3237		return ret;
3238
3239	ret = lan743x_phy_init(adapter);
3240	if (ret)
3241		return ret;
3242
3243	ret = lan743x_ptp_init(adapter);
3244	if (ret)
3245		return ret;
3246
3247	lan743x_rfe_update_mac_address(adapter);
3248
3249	ret = lan743x_dmac_init(adapter);
3250	if (ret)
3251		return ret;
3252
3253	for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
3254		adapter->rx[index].adapter = adapter;
3255		adapter->rx[index].channel_number = index;
3256	}
3257
3258	for (index = 0; index < adapter->used_tx_channels; index++) {
3259		tx = &adapter->tx[index];
3260		tx->adapter = adapter;
3261		tx->channel_number = index;
3262		spin_lock_init(&tx->ring_lock);
3263	}
3264
3265	return 0;
3266}
3267
3268static int lan743x_mdiobus_init(struct lan743x_adapter *adapter)
3269{
3270	u32 sgmii_ctl;
3271	int ret;
3272
3273	adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev);
3274	if (!(adapter->mdiobus)) {
3275		ret = -ENOMEM;
3276		goto return_error;
3277	}
3278
3279	adapter->mdiobus->priv = (void *)adapter;
3280	if (adapter->is_pci11x1x) {
3281		if (adapter->is_sgmii_en) {
3282			sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL);
3283			sgmii_ctl |= SGMII_CTL_SGMII_ENABLE_;
3284			sgmii_ctl &= ~SGMII_CTL_SGMII_POWER_DN_;
3285			lan743x_csr_write(adapter, SGMII_CTL, sgmii_ctl);
3286			netif_dbg(adapter, drv, adapter->netdev,
3287				  "SGMII operation\n");
3288			adapter->mdiobus->probe_capabilities = MDIOBUS_C22_C45;
3289			adapter->mdiobus->read = lan743x_mdiobus_c45_read;
3290			adapter->mdiobus->write = lan743x_mdiobus_c45_write;
3291			adapter->mdiobus->name = "lan743x-mdiobus-c45";
3292			netif_dbg(adapter, drv, adapter->netdev,
3293				  "lan743x-mdiobus-c45\n");
3294		} else {
3295			sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL);
3296			sgmii_ctl &= ~SGMII_CTL_SGMII_ENABLE_;
3297			sgmii_ctl |= SGMII_CTL_SGMII_POWER_DN_;
3298			lan743x_csr_write(adapter, SGMII_CTL, sgmii_ctl);
3299			netif_dbg(adapter, drv, adapter->netdev,
3300				  "RGMII operation\n");
3301			// Only C22 support when RGMII I/F
3302			adapter->mdiobus->probe_capabilities = MDIOBUS_C22;
3303			adapter->mdiobus->read = lan743x_mdiobus_read;
3304			adapter->mdiobus->write = lan743x_mdiobus_write;
3305			adapter->mdiobus->name = "lan743x-mdiobus";
3306			netif_dbg(adapter, drv, adapter->netdev,
3307				  "lan743x-mdiobus\n");
3308		}
3309	} else {
3310		adapter->mdiobus->read = lan743x_mdiobus_read;
3311		adapter->mdiobus->write = lan743x_mdiobus_write;
3312		adapter->mdiobus->name = "lan743x-mdiobus";
3313		netif_dbg(adapter, drv, adapter->netdev, "lan743x-mdiobus\n");
3314	}
3315
3316	snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE,
3317		 "pci-%s", pci_name(adapter->pdev));
3318
3319	if ((adapter->csr.id_rev & ID_REV_ID_MASK_) == ID_REV_ID_LAN7430_)
3320		/* LAN7430 uses internal phy at address 1 */
3321		adapter->mdiobus->phy_mask = ~(u32)BIT(1);
3322
3323	/* register mdiobus */
3324	ret = mdiobus_register(adapter->mdiobus);
3325	if (ret < 0)
3326		goto return_error;
3327	return 0;
3328
3329return_error:
3330	return ret;
3331}
3332
3333/* lan743x_pcidev_probe - Device Initialization Routine
3334 * @pdev: PCI device information struct
3335 * @id: entry in lan743x_pci_tbl
3336 *
3337 * Returns 0 on success, negative on failure
3338 *
3339 * initializes an adapter identified by a pci_dev structure.
3340 * The OS initialization, configuring of the adapter private structure,
3341 * and a hardware reset occur.
3342 **/
3343static int lan743x_pcidev_probe(struct pci_dev *pdev,
3344				const struct pci_device_id *id)
3345{
3346	struct lan743x_adapter *adapter = NULL;
3347	struct net_device *netdev = NULL;
3348	int ret = -ENODEV;
3349
3350	if (id->device == PCI_DEVICE_ID_SMSC_A011 ||
3351	    id->device == PCI_DEVICE_ID_SMSC_A041) {
3352		netdev = devm_alloc_etherdev_mqs(&pdev->dev,
3353						 sizeof(struct lan743x_adapter),
3354						 PCI11X1X_USED_TX_CHANNELS,
3355						 LAN743X_USED_RX_CHANNELS);
3356	} else {
3357		netdev = devm_alloc_etherdev_mqs(&pdev->dev,
3358						 sizeof(struct lan743x_adapter),
3359						 LAN743X_USED_TX_CHANNELS,
3360						 LAN743X_USED_RX_CHANNELS);
3361	}
3362
3363	if (!netdev)
3364		goto return_error;
3365
3366	SET_NETDEV_DEV(netdev, &pdev->dev);
3367	pci_set_drvdata(pdev, netdev);
3368	adapter = netdev_priv(netdev);
3369	adapter->netdev = netdev;
3370	adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |
3371			      NETIF_MSG_LINK | NETIF_MSG_IFUP |
3372			      NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;
3373	netdev->max_mtu = LAN743X_MAX_FRAME_SIZE;
3374
3375	of_get_mac_address(pdev->dev.of_node, adapter->mac_address);
3376
3377	ret = lan743x_pci_init(adapter, pdev);
3378	if (ret)
3379		goto return_error;
3380
3381	ret = lan743x_csr_init(adapter);
3382	if (ret)
3383		goto cleanup_pci;
3384
3385	ret = lan743x_hardware_init(adapter, pdev);
3386	if (ret)
3387		goto cleanup_pci;
3388
3389	ret = lan743x_mdiobus_init(adapter);
3390	if (ret)
3391		goto cleanup_hardware;
3392
3393	adapter->netdev->netdev_ops = &lan743x_netdev_ops;
3394	adapter->netdev->ethtool_ops = &lan743x_ethtool_ops;
3395	adapter->netdev->features = NETIF_F_SG | NETIF_F_TSO |
3396				    NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
3397	adapter->netdev->hw_features = adapter->netdev->features;
3398
3399	/* carrier off reporting is important to ethtool even BEFORE open */
3400	netif_carrier_off(netdev);
3401
3402	ret = register_netdev(adapter->netdev);
3403	if (ret < 0)
3404		goto cleanup_mdiobus;
3405	return 0;
3406
3407cleanup_mdiobus:
3408	lan743x_mdiobus_cleanup(adapter);
3409
3410cleanup_hardware:
3411	lan743x_hardware_cleanup(adapter);
3412
3413cleanup_pci:
3414	lan743x_pci_cleanup(adapter);
3415
3416return_error:
3417	pr_warn("Initialization failed\n");
3418	return ret;
3419}
3420
3421/**
3422 * lan743x_pcidev_remove - Device Removal Routine
3423 * @pdev: PCI device information struct
3424 *
3425 * this is called by the PCI subsystem to alert the driver
3426 * that it should release a PCI device.  This could be caused by a
3427 * Hot-Plug event, or because the driver is going to be removed from
3428 * memory.
3429 **/
3430static void lan743x_pcidev_remove(struct pci_dev *pdev)
3431{
3432	struct net_device *netdev = pci_get_drvdata(pdev);
3433	struct lan743x_adapter *adapter = netdev_priv(netdev);
3434
3435	lan743x_full_cleanup(adapter);
3436}
3437
3438static void lan743x_pcidev_shutdown(struct pci_dev *pdev)
3439{
3440	struct net_device *netdev = pci_get_drvdata(pdev);
3441	struct lan743x_adapter *adapter = netdev_priv(netdev);
3442
3443	rtnl_lock();
3444	netif_device_detach(netdev);
3445
3446	/* close netdev when netdev is at running state.
3447	 * For instance, it is true when system goes to sleep by pm-suspend
3448	 * However, it is false when system goes to sleep by suspend GUI menu
3449	 */
3450	if (netif_running(netdev))
3451		lan743x_netdev_close(netdev);
3452	rtnl_unlock();
3453
3454#ifdef CONFIG_PM
3455	pci_save_state(pdev);
3456#endif
3457
3458	/* clean up lan743x portion */
3459	lan743x_hardware_cleanup(adapter);
3460}
3461
3462#ifdef CONFIG_PM_SLEEP
3463static u16 lan743x_pm_wakeframe_crc16(const u8 *buf, int len)
3464{
3465	return bitrev16(crc16(0xFFFF, buf, len));
3466}
3467
3468static void lan743x_pm_set_wol(struct lan743x_adapter *adapter)
3469{
3470	const u8 ipv4_multicast[3] = { 0x01, 0x00, 0x5E };
3471	const u8 ipv6_multicast[3] = { 0x33, 0x33 };
3472	const u8 arp_type[2] = { 0x08, 0x06 };
3473	int mask_index;
3474	u32 sopass;
3475	u32 pmtctl;
3476	u32 wucsr;
3477	u32 macrx;
3478	u16 crc;
3479
3480	for (mask_index = 0; mask_index < MAC_NUM_OF_WUF_CFG; mask_index++)
3481		lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index), 0);
3482
3483	/* clear wake settings */
3484	pmtctl = lan743x_csr_read(adapter, PMT_CTL);
3485	pmtctl |= PMT_CTL_WUPS_MASK_;
3486	pmtctl &= ~(PMT_CTL_GPIO_WAKEUP_EN_ | PMT_CTL_EEE_WAKEUP_EN_ |
3487		PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_ |
3488		PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_ | PMT_CTL_ETH_PHY_WAKE_EN_);
3489
3490	macrx = lan743x_csr_read(adapter, MAC_RX);
3491
3492	wucsr = 0;
3493	mask_index = 0;
3494
3495	pmtctl |= PMT_CTL_ETH_PHY_D3_COLD_OVR_ | PMT_CTL_ETH_PHY_D3_OVR_;
3496
3497	if (adapter->wolopts & WAKE_PHY) {
3498		pmtctl |= PMT_CTL_ETH_PHY_EDPD_PLL_CTL_;
3499		pmtctl |= PMT_CTL_ETH_PHY_WAKE_EN_;
3500	}
3501	if (adapter->wolopts & WAKE_MAGIC) {
3502		wucsr |= MAC_WUCSR_MPEN_;
3503		macrx |= MAC_RX_RXEN_;
3504		pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3505	}
3506	if (adapter->wolopts & WAKE_UCAST) {
3507		wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_PFDA_EN_;
3508		macrx |= MAC_RX_RXEN_;
3509		pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3510		pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3511	}
3512	if (adapter->wolopts & WAKE_BCAST) {
3513		wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_BCST_EN_;
3514		macrx |= MAC_RX_RXEN_;
3515		pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3516		pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3517	}
3518	if (adapter->wolopts & WAKE_MCAST) {
3519		/* IPv4 multicast */
3520		crc = lan743x_pm_wakeframe_crc16(ipv4_multicast, 3);
3521		lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index),
3522				  MAC_WUF_CFG_EN_ | MAC_WUF_CFG_TYPE_MCAST_ |
3523				  (0 << MAC_WUF_CFG_OFFSET_SHIFT_) |
3524				  (crc & MAC_WUF_CFG_CRC16_MASK_));
3525		lan743x_csr_write(adapter, MAC_WUF_MASK0(mask_index), 7);
3526		lan743x_csr_write(adapter, MAC_WUF_MASK1(mask_index), 0);
3527		lan743x_csr_write(adapter, MAC_WUF_MASK2(mask_index), 0);
3528		lan743x_csr_write(adapter, MAC_WUF_MASK3(mask_index), 0);
3529		mask_index++;
3530
3531		/* IPv6 multicast */
3532		crc = lan743x_pm_wakeframe_crc16(ipv6_multicast, 2);
3533		lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index),
3534				  MAC_WUF_CFG_EN_ | MAC_WUF_CFG_TYPE_MCAST_ |
3535				  (0 << MAC_WUF_CFG_OFFSET_SHIFT_) |
3536				  (crc & MAC_WUF_CFG_CRC16_MASK_));
3537		lan743x_csr_write(adapter, MAC_WUF_MASK0(mask_index), 3);
3538		lan743x_csr_write(adapter, MAC_WUF_MASK1(mask_index), 0);
3539		lan743x_csr_write(adapter, MAC_WUF_MASK2(mask_index), 0);
3540		lan743x_csr_write(adapter, MAC_WUF_MASK3(mask_index), 0);
3541		mask_index++;
3542
3543		wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_WAKE_EN_;
3544		macrx |= MAC_RX_RXEN_;
3545		pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3546		pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3547	}
3548	if (adapter->wolopts & WAKE_ARP) {
3549		/* set MAC_WUF_CFG & WUF_MASK
3550		 * for packettype (offset 12,13) = ARP (0x0806)
3551		 */
3552		crc = lan743x_pm_wakeframe_crc16(arp_type, 2);
3553		lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index),
3554				  MAC_WUF_CFG_EN_ | MAC_WUF_CFG_TYPE_ALL_ |
3555				  (0 << MAC_WUF_CFG_OFFSET_SHIFT_) |
3556				  (crc & MAC_WUF_CFG_CRC16_MASK_));
3557		lan743x_csr_write(adapter, MAC_WUF_MASK0(mask_index), 0x3000);
3558		lan743x_csr_write(adapter, MAC_WUF_MASK1(mask_index), 0);
3559		lan743x_csr_write(adapter, MAC_WUF_MASK2(mask_index), 0);
3560		lan743x_csr_write(adapter, MAC_WUF_MASK3(mask_index), 0);
3561		mask_index++;
3562
3563		wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_WAKE_EN_;
3564		macrx |= MAC_RX_RXEN_;
3565		pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3566		pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3567	}
3568
3569	if (adapter->wolopts & WAKE_MAGICSECURE) {
3570		sopass = *(u32 *)adapter->sopass;
3571		lan743x_csr_write(adapter, MAC_MP_SO_LO, sopass);
3572		sopass = *(u16 *)&adapter->sopass[4];
3573		lan743x_csr_write(adapter, MAC_MP_SO_HI, sopass);
3574		wucsr |= MAC_MP_SO_EN_;
3575	}
3576
3577	lan743x_csr_write(adapter, MAC_WUCSR, wucsr);
3578	lan743x_csr_write(adapter, PMT_CTL, pmtctl);
3579	lan743x_csr_write(adapter, MAC_RX, macrx);
3580}
3581
3582static int lan743x_pm_suspend(struct device *dev)
3583{
3584	struct pci_dev *pdev = to_pci_dev(dev);
3585	struct net_device *netdev = pci_get_drvdata(pdev);
3586	struct lan743x_adapter *adapter = netdev_priv(netdev);
3587	u32 data;
3588
3589	lan743x_pcidev_shutdown(pdev);
3590
3591	/* clear all wakes */
3592	lan743x_csr_write(adapter, MAC_WUCSR, 0);
3593	lan743x_csr_write(adapter, MAC_WUCSR2, 0);
3594	lan743x_csr_write(adapter, MAC_WK_SRC, 0xFFFFFFFF);
3595
3596	if (adapter->wolopts)
3597		lan743x_pm_set_wol(adapter);
3598
3599	if (adapter->is_pci11x1x) {
3600		/* Save HW_CFG to config again in PM resume */
3601		data = lan743x_csr_read(adapter, HW_CFG);
3602		adapter->hw_cfg = data;
3603		data |= (HW_CFG_RST_PROTECT_PCIE_ |
3604			 HW_CFG_D3_RESET_DIS_ |
3605			 HW_CFG_D3_VAUX_OVR_ |
3606			 HW_CFG_HOT_RESET_DIS_ |
3607			 HW_CFG_RST_PROTECT_);
3608		lan743x_csr_write(adapter, HW_CFG, data);
3609	}
3610
3611	/* Host sets PME_En, put D3hot */
3612	return pci_prepare_to_sleep(pdev);
3613}
3614
3615static int lan743x_pm_resume(struct device *dev)
3616{
3617	struct pci_dev *pdev = to_pci_dev(dev);
3618	struct net_device *netdev = pci_get_drvdata(pdev);
3619	struct lan743x_adapter *adapter = netdev_priv(netdev);
3620	int ret;
3621
3622	pci_set_power_state(pdev, PCI_D0);
3623	pci_restore_state(pdev);
3624	pci_save_state(pdev);
3625
3626	/* Restore HW_CFG that was saved during pm suspend */
3627	if (adapter->is_pci11x1x)
3628		lan743x_csr_write(adapter, HW_CFG, adapter->hw_cfg);
3629
3630	ret = lan743x_hardware_init(adapter, pdev);
3631	if (ret) {
3632		netif_err(adapter, probe, adapter->netdev,
3633			  "lan743x_hardware_init returned %d\n", ret);
3634		lan743x_pci_cleanup(adapter);
3635		return ret;
3636	}
3637
3638	/* open netdev when netdev is at running state while resume.
3639	 * For instance, it is true when system wakesup after pm-suspend
3640	 * However, it is false when system wakes up after suspend GUI menu
3641	 */
3642	if (netif_running(netdev))
3643		lan743x_netdev_open(netdev);
3644
3645	netif_device_attach(netdev);
3646	ret = lan743x_csr_read(adapter, MAC_WK_SRC);
3647	netif_info(adapter, drv, adapter->netdev,
3648		   "Wakeup source : 0x%08X\n", ret);
3649
3650	return 0;
3651}
3652
3653static const struct dev_pm_ops lan743x_pm_ops = {
3654	SET_SYSTEM_SLEEP_PM_OPS(lan743x_pm_suspend, lan743x_pm_resume)
3655};
3656#endif /* CONFIG_PM_SLEEP */
3657
3658static const struct pci_device_id lan743x_pcidev_tbl[] = {
3659	{ PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_LAN7430) },
3660	{ PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_LAN7431) },
3661	{ PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_A011) },
3662	{ PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_A041) },
3663	{ 0, }
3664};
3665
3666MODULE_DEVICE_TABLE(pci, lan743x_pcidev_tbl);
3667
3668static struct pci_driver lan743x_pcidev_driver = {
3669	.name     = DRIVER_NAME,
3670	.id_table = lan743x_pcidev_tbl,
3671	.probe    = lan743x_pcidev_probe,
3672	.remove   = lan743x_pcidev_remove,
3673#ifdef CONFIG_PM_SLEEP
3674	.driver.pm = &lan743x_pm_ops,
3675#endif
3676	.shutdown = lan743x_pcidev_shutdown,
3677};
3678
3679module_pci_driver(lan743x_pcidev_driver);
3680
3681MODULE_AUTHOR(DRIVER_AUTHOR);
3682MODULE_DESCRIPTION(DRIVER_DESC);
3683MODULE_LICENSE("GPL");