1// SPDX-License-Identifier: GPL-2.0
   2/* Ethernet device driver for Cortina Systems Gemini SoC
   3 * Also known as the StorLink SL3512 and SL3516 (SL351x) or Lepus
   4 * Net Engine and Gigabit Ethernet MAC (GMAC)
   5 * This hardware contains a TCP Offload Engine (TOE) but currently the
   6 * driver does not make use of it.
   7 *
   8 * Authors:
   9 * Linus Walleij <linus.walleij@linaro.org>
  10 * Tobias Waldvogel <tobias.waldvogel@gmail.com> (OpenWRT)
  11 * Michał Mirosław <mirq-linux@rere.qmqm.pl>
  12 * Paulius Zaleckas <paulius.zaleckas@gmail.com>
  13 * Giuseppe De Robertis <Giuseppe.DeRobertis@ba.infn.it>
  14 * Gary Chen & Ch Hsu Storlink Semiconductor
  15 */
  16#include <linux/kernel.h>
  17#include <linux/init.h>
  18#include <linux/module.h>
  19#include <linux/platform_device.h>
  20#include <linux/spinlock.h>
  21#include <linux/slab.h>
  22#include <linux/dma-mapping.h>
  23#include <linux/cache.h>
  24#include <linux/interrupt.h>
  25#include <linux/reset.h>
  26#include <linux/clk.h>
  27#include <linux/of.h>
  28#include <linux/of_mdio.h>
  29#include <linux/of_net.h>
  30#include <linux/of_platform.h>
  31#include <linux/etherdevice.h>
  32#include <linux/if_vlan.h>
  33#include <linux/skbuff.h>
  34#include <linux/phy.h>
  35#include <linux/crc32.h>
  36#include <linux/ethtool.h>
  37#include <linux/tcp.h>
  38#include <linux/u64_stats_sync.h>
  39
  40#include <linux/in.h>
  41#include <linux/ip.h>
  42#include <linux/ipv6.h>
  43
  44#include "gemini.h"
  45
  46#define DRV_NAME		"gmac-gemini"
  47
  48#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  49static int debug = -1;
  50module_param(debug, int, 0);
  51MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  52
  53#define HSIZE_8			0x00
  54#define HSIZE_16		0x01
  55#define HSIZE_32		0x02
  56
  57#define HBURST_SINGLE		0x00
  58#define HBURST_INCR		0x01
  59#define HBURST_INCR4		0x02
  60#define HBURST_INCR8		0x03
  61
  62#define HPROT_DATA_CACHE	BIT(0)
  63#define HPROT_PRIVILIGED	BIT(1)
  64#define HPROT_BUFFERABLE	BIT(2)
  65#define HPROT_CACHABLE		BIT(3)
  66
  67#define DEFAULT_RX_COALESCE_NSECS	0
  68#define DEFAULT_GMAC_RXQ_ORDER		9
  69#define DEFAULT_GMAC_TXQ_ORDER		8
  70#define DEFAULT_RX_BUF_ORDER		11
  71#define TX_MAX_FRAGS			16
  72#define TX_QUEUE_NUM			1	/* max: 6 */
  73#define RX_MAX_ALLOC_ORDER		2
  74
  75#define GMAC0_IRQ0_2 (GMAC0_TXDERR_INT_BIT | GMAC0_TXPERR_INT_BIT | \
  76		      GMAC0_RXDERR_INT_BIT | GMAC0_RXPERR_INT_BIT)
  77#define GMAC0_IRQ0_TXQ0_INTS (GMAC0_SWTQ00_EOF_INT_BIT | \
  78			      GMAC0_SWTQ00_FIN_INT_BIT)
  79#define GMAC0_IRQ4_8 (GMAC0_MIB_INT_BIT | GMAC0_RX_OVERRUN_INT_BIT)
  80
  81#define GMAC_OFFLOAD_FEATURES (NETIF_F_SG | NETIF_F_IP_CSUM | \
  82			       NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | \
  83			       NETIF_F_TSO | NETIF_F_TSO_ECN | NETIF_F_TSO6)
  84
  85/**
  86 * struct gmac_queue_page - page buffer per-page info
  87 * @page: the page struct
  88 * @mapping: the dma address handle
  89 */
  90struct gmac_queue_page {
  91	struct page *page;
  92	dma_addr_t mapping;
  93};
  94
  95struct gmac_txq {
  96	struct gmac_txdesc *ring;
  97	struct sk_buff	**skb;
  98	unsigned int	cptr;
  99	unsigned int	noirq_packets;
 100};
 101
 102struct gemini_ethernet;
 103
 104struct gemini_ethernet_port {
 105	u8 id; /* 0 or 1 */
 106
 107	struct gemini_ethernet *geth;
 108	struct net_device *netdev;
 109	struct device *dev;
 110	void __iomem *dma_base;
 111	void __iomem *gmac_base;
 112	struct clk *pclk;
 113	struct reset_control *reset;
 114	int irq;
 115	__le32 mac_addr[3];
 116
 117	void __iomem		*rxq_rwptr;
 118	struct gmac_rxdesc	*rxq_ring;
 119	unsigned int		rxq_order;
 120
 121	struct napi_struct	napi;
 122	struct hrtimer		rx_coalesce_timer;
 123	unsigned int		rx_coalesce_nsecs;
 124	unsigned int		freeq_refill;
 125	struct gmac_txq		txq[TX_QUEUE_NUM];
 126	unsigned int		txq_order;
 127	unsigned int		irq_every_tx_packets;
 128
 129	dma_addr_t		rxq_dma_base;
 130	dma_addr_t		txq_dma_base;
 131
 132	unsigned int		msg_enable;
 133	spinlock_t		config_lock; /* Locks config register */
 134
 135	struct u64_stats_sync	tx_stats_syncp;
 136	struct u64_stats_sync	rx_stats_syncp;
 137	struct u64_stats_sync	ir_stats_syncp;
 138
 139	struct rtnl_link_stats64 stats;
 140	u64			hw_stats[RX_STATS_NUM];
 141	u64			rx_stats[RX_STATUS_NUM];
 142	u64			rx_csum_stats[RX_CHKSUM_NUM];
 143	u64			rx_napi_exits;
 144	u64			tx_frag_stats[TX_MAX_FRAGS];
 145	u64			tx_frags_linearized;
 146	u64			tx_hw_csummed;
 147};
 148
 149struct gemini_ethernet {
 150	struct device *dev;
 151	void __iomem *base;
 152	struct gemini_ethernet_port *port0;
 153	struct gemini_ethernet_port *port1;
 154	bool initialized;
 155
 156	spinlock_t	irq_lock; /* Locks IRQ-related registers */
 157	unsigned int	freeq_order;
 158	unsigned int	freeq_frag_order;
 159	struct gmac_rxdesc *freeq_ring;
 160	dma_addr_t	freeq_dma_base;
 161	struct gmac_queue_page	*freeq_pages;
 162	unsigned int	num_freeq_pages;
 163	spinlock_t	freeq_lock; /* Locks queue from reentrance */
 164};
 165
 166#define GMAC_STATS_NUM	( \
 167	RX_STATS_NUM + RX_STATUS_NUM + RX_CHKSUM_NUM + 1 + \
 168	TX_MAX_FRAGS + 2)
 169
 170static const char gmac_stats_strings[GMAC_STATS_NUM][ETH_GSTRING_LEN] = {
 171	"GMAC_IN_DISCARDS",
 172	"GMAC_IN_ERRORS",
 173	"GMAC_IN_MCAST",
 174	"GMAC_IN_BCAST",
 175	"GMAC_IN_MAC1",
 176	"GMAC_IN_MAC2",
 177	"RX_STATUS_GOOD_FRAME",
 178	"RX_STATUS_TOO_LONG_GOOD_CRC",
 179	"RX_STATUS_RUNT_FRAME",
 180	"RX_STATUS_SFD_NOT_FOUND",
 181	"RX_STATUS_CRC_ERROR",
 182	"RX_STATUS_TOO_LONG_BAD_CRC",
 183	"RX_STATUS_ALIGNMENT_ERROR",
 184	"RX_STATUS_TOO_LONG_BAD_ALIGN",
 185	"RX_STATUS_RX_ERR",
 186	"RX_STATUS_DA_FILTERED",
 187	"RX_STATUS_BUFFER_FULL",
 188	"RX_STATUS_11",
 189	"RX_STATUS_12",
 190	"RX_STATUS_13",
 191	"RX_STATUS_14",
 192	"RX_STATUS_15",
 193	"RX_CHKSUM_IP_UDP_TCP_OK",
 194	"RX_CHKSUM_IP_OK_ONLY",
 195	"RX_CHKSUM_NONE",
 196	"RX_CHKSUM_3",
 197	"RX_CHKSUM_IP_ERR_UNKNOWN",
 198	"RX_CHKSUM_IP_ERR",
 199	"RX_CHKSUM_TCP_UDP_ERR",
 200	"RX_CHKSUM_7",
 201	"RX_NAPI_EXITS",
 202	"TX_FRAGS[1]",
 203	"TX_FRAGS[2]",
 204	"TX_FRAGS[3]",
 205	"TX_FRAGS[4]",
 206	"TX_FRAGS[5]",
 207	"TX_FRAGS[6]",
 208	"TX_FRAGS[7]",
 209	"TX_FRAGS[8]",
 210	"TX_FRAGS[9]",
 211	"TX_FRAGS[10]",
 212	"TX_FRAGS[11]",
 213	"TX_FRAGS[12]",
 214	"TX_FRAGS[13]",
 215	"TX_FRAGS[14]",
 216	"TX_FRAGS[15]",
 217	"TX_FRAGS[16+]",
 218	"TX_FRAGS_LINEARIZED",
 219	"TX_HW_CSUMMED",
 220};
 221
 222static void gmac_dump_dma_state(struct net_device *netdev);
 223
 224static void gmac_update_config0_reg(struct net_device *netdev,
 225				    u32 val, u32 vmask)
 226{
 227	struct gemini_ethernet_port *port = netdev_priv(netdev);
 228	unsigned long flags;
 229	u32 reg;
 230
 231	spin_lock_irqsave(&port->config_lock, flags);
 232
 233	reg = readl(port->gmac_base + GMAC_CONFIG0);
 234	reg = (reg & ~vmask) | val;
 235	writel(reg, port->gmac_base + GMAC_CONFIG0);
 236
 237	spin_unlock_irqrestore(&port->config_lock, flags);
 238}
 239
 240static void gmac_enable_tx_rx(struct net_device *netdev)
 241{
 242	struct gemini_ethernet_port *port = netdev_priv(netdev);
 243	unsigned long flags;
 244	u32 reg;
 245
 246	spin_lock_irqsave(&port->config_lock, flags);
 247
 248	reg = readl(port->gmac_base + GMAC_CONFIG0);
 249	reg &= ~CONFIG0_TX_RX_DISABLE;
 250	writel(reg, port->gmac_base + GMAC_CONFIG0);
 251
 252	spin_unlock_irqrestore(&port->config_lock, flags);
 253}
 254
 255static void gmac_disable_tx_rx(struct net_device *netdev)
 256{
 257	struct gemini_ethernet_port *port = netdev_priv(netdev);
 258	unsigned long flags;
 259	u32 val;
 260
 261	spin_lock_irqsave(&port->config_lock, flags);
 262
 263	val = readl(port->gmac_base + GMAC_CONFIG0);
 264	val |= CONFIG0_TX_RX_DISABLE;
 265	writel(val, port->gmac_base + GMAC_CONFIG0);
 266
 267	spin_unlock_irqrestore(&port->config_lock, flags);
 268
 269	mdelay(10);	/* let GMAC consume packet */
 270}
 271
 272static void gmac_set_flow_control(struct net_device *netdev, bool tx, bool rx)
 273{
 274	struct gemini_ethernet_port *port = netdev_priv(netdev);
 275	unsigned long flags;
 276	u32 val;
 277
 278	spin_lock_irqsave(&port->config_lock, flags);
 279
 280	val = readl(port->gmac_base + GMAC_CONFIG0);
 281	val &= ~CONFIG0_FLOW_CTL;
 282	if (tx)
 283		val |= CONFIG0_FLOW_TX;
 284	if (rx)
 285		val |= CONFIG0_FLOW_RX;
 286	writel(val, port->gmac_base + GMAC_CONFIG0);
 287
 288	spin_unlock_irqrestore(&port->config_lock, flags);
 289}
 290
 291static void gmac_adjust_link(struct net_device *netdev)
 292{
 293	struct gemini_ethernet_port *port = netdev_priv(netdev);
 294	struct phy_device *phydev = netdev->phydev;
 295	union gmac_status status, old_status;
 296	bool pause_tx = false;
 297	bool pause_rx = false;
 298
 299	status.bits32 = readl(port->gmac_base + GMAC_STATUS);
 300	old_status.bits32 = status.bits32;
 301	status.bits.link = phydev->link;
 302	status.bits.duplex = phydev->duplex;
 303
 304	switch (phydev->speed) {
 305	case 1000:
 306		status.bits.speed = GMAC_SPEED_1000;
 307		if (phy_interface_mode_is_rgmii(phydev->interface))
 308			status.bits.mii_rmii = GMAC_PHY_RGMII_1000;
 309		netdev_dbg(netdev, "connect %s to RGMII @ 1Gbit\n",
 310			   phydev_name(phydev));
 311		break;
 312	case 100:
 313		status.bits.speed = GMAC_SPEED_100;
 314		if (phy_interface_mode_is_rgmii(phydev->interface))
 315			status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
 316		netdev_dbg(netdev, "connect %s to RGMII @ 100 Mbit\n",
 317			   phydev_name(phydev));
 318		break;
 319	case 10:
 320		status.bits.speed = GMAC_SPEED_10;
 321		if (phy_interface_mode_is_rgmii(phydev->interface))
 322			status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
 323		netdev_dbg(netdev, "connect %s to RGMII @ 10 Mbit\n",
 324			   phydev_name(phydev));
 325		break;
 326	default:
 327		netdev_warn(netdev, "Unsupported PHY speed (%d) on %s\n",
 328			    phydev->speed, phydev_name(phydev));
 329	}
 330
 331	if (phydev->duplex == DUPLEX_FULL) {
 332		phy_get_pause(phydev, &pause_tx, &pause_rx);
 333		netdev_dbg(netdev, "set negotiated pause params pause TX = %s, pause RX = %s\n",
 334			   pause_tx ? "ON" : "OFF", pause_rx ? "ON" : "OFF");
 
 
 
 
 
 335	}
 336
 337	gmac_set_flow_control(netdev, pause_tx, pause_rx);
 338
 339	if (old_status.bits32 == status.bits32)
 340		return;
 341
 342	if (netif_msg_link(port)) {
 343		phy_print_status(phydev);
 344		netdev_info(netdev, "link flow control: %s\n",
 345			    phydev->pause
 346			    ? (phydev->asym_pause ? "tx" : "both")
 347			    : (phydev->asym_pause ? "rx" : "none")
 348		);
 349	}
 350
 351	gmac_disable_tx_rx(netdev);
 352	writel(status.bits32, port->gmac_base + GMAC_STATUS);
 353	gmac_enable_tx_rx(netdev);
 354}
 355
 356static int gmac_setup_phy(struct net_device *netdev)
 357{
 358	struct gemini_ethernet_port *port = netdev_priv(netdev);
 359	union gmac_status status = { .bits32 = 0 };
 360	struct device *dev = port->dev;
 361	struct phy_device *phy;
 362
 363	phy = of_phy_get_and_connect(netdev,
 364				     dev->of_node,
 365				     gmac_adjust_link);
 366	if (!phy)
 367		return -ENODEV;
 368	netdev->phydev = phy;
 369
 370	phy_set_max_speed(phy, SPEED_1000);
 371	phy_support_asym_pause(phy);
 372
 373	/* set PHY interface type */
 374	switch (phy->interface) {
 375	case PHY_INTERFACE_MODE_MII:
 376		netdev_dbg(netdev,
 377			   "MII: set GMAC0 to GMII mode, GMAC1 disabled\n");
 378		status.bits.mii_rmii = GMAC_PHY_MII;
 379		break;
 380	case PHY_INTERFACE_MODE_GMII:
 381		netdev_dbg(netdev,
 382			   "GMII: set GMAC0 to GMII mode, GMAC1 disabled\n");
 383		status.bits.mii_rmii = GMAC_PHY_GMII;
 384		break;
 385	case PHY_INTERFACE_MODE_RGMII:
 386	case PHY_INTERFACE_MODE_RGMII_ID:
 387	case PHY_INTERFACE_MODE_RGMII_TXID:
 388	case PHY_INTERFACE_MODE_RGMII_RXID:
 389		netdev_dbg(netdev,
 390			   "RGMII: set GMAC0 and GMAC1 to MII/RGMII mode\n");
 391		status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
 392		break;
 393	default:
 394		netdev_err(netdev, "Unsupported MII interface\n");
 395		phy_disconnect(phy);
 396		netdev->phydev = NULL;
 397		return -EINVAL;
 398	}
 399	writel(status.bits32, port->gmac_base + GMAC_STATUS);
 400
 401	if (netif_msg_link(port))
 402		phy_attached_info(phy);
 403
 404	return 0;
 405}
 406
 407/* The maximum frame length is not logically enumerated in the
 408 * hardware, so we do a table lookup to find the applicable max
 409 * frame length.
 410 */
 411struct gmac_max_framelen {
 412	unsigned int max_l3_len;
 413	u8 val;
 414};
 415
 416static const struct gmac_max_framelen gmac_maxlens[] = {
 417	{
 418		.max_l3_len = 1518,
 419		.val = CONFIG0_MAXLEN_1518,
 420	},
 421	{
 422		.max_l3_len = 1522,
 423		.val = CONFIG0_MAXLEN_1522,
 424	},
 425	{
 426		.max_l3_len = 1536,
 427		.val = CONFIG0_MAXLEN_1536,
 428	},
 429	{
 430		.max_l3_len = 1548,
 431		.val = CONFIG0_MAXLEN_1548,
 432	},
 433	{
 434		.max_l3_len = 9212,
 435		.val = CONFIG0_MAXLEN_9k,
 436	},
 437	{
 438		.max_l3_len = 10236,
 439		.val = CONFIG0_MAXLEN_10k,
 440	},
 441};
 442
 443static int gmac_pick_rx_max_len(unsigned int max_l3_len)
 444{
 445	const struct gmac_max_framelen *maxlen;
 446	int maxtot;
 447	int i;
 448
 449	maxtot = max_l3_len + ETH_HLEN + VLAN_HLEN;
 450
 451	for (i = 0; i < ARRAY_SIZE(gmac_maxlens); i++) {
 452		maxlen = &gmac_maxlens[i];
 453		if (maxtot <= maxlen->max_l3_len)
 454			return maxlen->val;
 455	}
 456
 457	return -1;
 458}
 459
 460static int gmac_init(struct net_device *netdev)
 461{
 462	struct gemini_ethernet_port *port = netdev_priv(netdev);
 463	union gmac_config0 config0 = { .bits = {
 464		.dis_tx = 1,
 465		.dis_rx = 1,
 466		.ipv4_rx_chksum = 1,
 467		.ipv6_rx_chksum = 1,
 468		.rx_err_detect = 1,
 469		.rgmm_edge = 1,
 470		.port0_chk_hwq = 1,
 471		.port1_chk_hwq = 1,
 472		.port0_chk_toeq = 1,
 473		.port1_chk_toeq = 1,
 474		.port0_chk_classq = 1,
 475		.port1_chk_classq = 1,
 476	} };
 477	union gmac_ahb_weight ahb_weight = { .bits = {
 478		.rx_weight = 1,
 479		.tx_weight = 1,
 480		.hash_weight = 1,
 481		.pre_req = 0x1f,
 482		.tq_dv_threshold = 0,
 483	} };
 484	union gmac_tx_wcr0 hw_weigh = { .bits = {
 485		.hw_tq3 = 1,
 486		.hw_tq2 = 1,
 487		.hw_tq1 = 1,
 488		.hw_tq0 = 1,
 489	} };
 490	union gmac_tx_wcr1 sw_weigh = { .bits = {
 491		.sw_tq5 = 1,
 492		.sw_tq4 = 1,
 493		.sw_tq3 = 1,
 494		.sw_tq2 = 1,
 495		.sw_tq1 = 1,
 496		.sw_tq0 = 1,
 497	} };
 498	union gmac_config1 config1 = { .bits = {
 499		.set_threshold = 16,
 500		.rel_threshold = 24,
 501	} };
 502	union gmac_config2 config2 = { .bits = {
 503		.set_threshold = 16,
 504		.rel_threshold = 32,
 505	} };
 506	union gmac_config3 config3 = { .bits = {
 507		.set_threshold = 0,
 508		.rel_threshold = 0,
 509	} };
 510	union gmac_config0 tmp;
 511
 512	config0.bits.max_len = gmac_pick_rx_max_len(netdev->mtu);
 513	tmp.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
 514	config0.bits.reserved = tmp.bits.reserved;
 515	writel(config0.bits32, port->gmac_base + GMAC_CONFIG0);
 516	writel(config1.bits32, port->gmac_base + GMAC_CONFIG1);
 517	writel(config2.bits32, port->gmac_base + GMAC_CONFIG2);
 518	writel(config3.bits32, port->gmac_base + GMAC_CONFIG3);
 519
 520	readl(port->dma_base + GMAC_AHB_WEIGHT_REG);
 521	writel(ahb_weight.bits32, port->dma_base + GMAC_AHB_WEIGHT_REG);
 522
 523	writel(hw_weigh.bits32,
 524	       port->dma_base + GMAC_TX_WEIGHTING_CTRL_0_REG);
 525	writel(sw_weigh.bits32,
 526	       port->dma_base + GMAC_TX_WEIGHTING_CTRL_1_REG);
 527
 528	port->rxq_order = DEFAULT_GMAC_RXQ_ORDER;
 529	port->txq_order = DEFAULT_GMAC_TXQ_ORDER;
 530	port->rx_coalesce_nsecs = DEFAULT_RX_COALESCE_NSECS;
 531
 532	/* Mark every quarter of the queue a packet for interrupt
 533	 * in order to be able to wake up the queue if it was stopped
 534	 */
 535	port->irq_every_tx_packets = 1 << (port->txq_order - 2);
 536
 537	return 0;
 538}
 539
 540static int gmac_setup_txqs(struct net_device *netdev)
 541{
 542	struct gemini_ethernet_port *port = netdev_priv(netdev);
 543	unsigned int n_txq = netdev->num_tx_queues;
 544	struct gemini_ethernet *geth = port->geth;
 545	size_t entries = 1 << port->txq_order;
 546	struct gmac_txq *txq = port->txq;
 547	struct gmac_txdesc *desc_ring;
 548	size_t len = n_txq * entries;
 549	struct sk_buff **skb_tab;
 550	void __iomem *rwptr_reg;
 551	unsigned int r;
 552	int i;
 553
 554	rwptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
 555
 556	skb_tab = kcalloc(len, sizeof(*skb_tab), GFP_KERNEL);
 557	if (!skb_tab)
 558		return -ENOMEM;
 559
 560	desc_ring = dma_alloc_coherent(geth->dev, len * sizeof(*desc_ring),
 561				       &port->txq_dma_base, GFP_KERNEL);
 562
 563	if (!desc_ring) {
 564		kfree(skb_tab);
 565		return -ENOMEM;
 566	}
 567
 568	if (port->txq_dma_base & ~DMA_Q_BASE_MASK) {
 569		dev_warn(geth->dev, "TX queue base is not aligned\n");
 570		dma_free_coherent(geth->dev, len * sizeof(*desc_ring),
 571				  desc_ring, port->txq_dma_base);
 572		kfree(skb_tab);
 573		return -ENOMEM;
 574	}
 575
 576	writel(port->txq_dma_base | port->txq_order,
 577	       port->dma_base + GMAC_SW_TX_QUEUE_BASE_REG);
 578
 579	for (i = 0; i < n_txq; i++) {
 580		txq->ring = desc_ring;
 581		txq->skb = skb_tab;
 582		txq->noirq_packets = 0;
 583
 584		r = readw(rwptr_reg);
 585		rwptr_reg += 2;
 586		writew(r, rwptr_reg);
 587		rwptr_reg += 2;
 588		txq->cptr = r;
 589
 590		txq++;
 591		desc_ring += entries;
 592		skb_tab += entries;
 593	}
 594
 595	return 0;
 596}
 597
 598static void gmac_clean_txq(struct net_device *netdev, struct gmac_txq *txq,
 599			   unsigned int r)
 600{
 601	struct gemini_ethernet_port *port = netdev_priv(netdev);
 602	unsigned int m = (1 << port->txq_order) - 1;
 603	struct gemini_ethernet *geth = port->geth;
 604	unsigned int c = txq->cptr;
 605	union gmac_txdesc_0 word0;
 606	union gmac_txdesc_1 word1;
 607	unsigned int hwchksum = 0;
 608	unsigned long bytes = 0;
 609	struct gmac_txdesc *txd;
 610	unsigned short nfrags;
 611	unsigned int errs = 0;
 612	unsigned int pkts = 0;
 613	unsigned int word3;
 614	dma_addr_t mapping;
 615
 616	if (c == r)
 617		return;
 618
 619	while (c != r) {
 620		txd = txq->ring + c;
 621		word0 = txd->word0;
 622		word1 = txd->word1;
 623		mapping = txd->word2.buf_adr;
 624		word3 = txd->word3.bits32;
 625
 626		dma_unmap_single(geth->dev, mapping,
 627				 word0.bits.buffer_size, DMA_TO_DEVICE);
 628
 629		if (word3 & EOF_BIT)
 630			dev_kfree_skb(txq->skb[c]);
 631
 632		c++;
 633		c &= m;
 634
 635		if (!(word3 & SOF_BIT))
 636			continue;
 637
 638		if (!word0.bits.status_tx_ok) {
 639			errs++;
 640			continue;
 641		}
 642
 643		pkts++;
 644		bytes += txd->word1.bits.byte_count;
 645
 646		if (word1.bits32 & TSS_CHECKUM_ENABLE)
 647			hwchksum++;
 648
 649		nfrags = word0.bits.desc_count - 1;
 650		if (nfrags) {
 651			if (nfrags >= TX_MAX_FRAGS)
 652				nfrags = TX_MAX_FRAGS - 1;
 653
 654			u64_stats_update_begin(&port->tx_stats_syncp);
 655			port->tx_frag_stats[nfrags]++;
 656			u64_stats_update_end(&port->tx_stats_syncp);
 657		}
 658	}
 659
 660	u64_stats_update_begin(&port->ir_stats_syncp);
 661	port->stats.tx_errors += errs;
 662	port->stats.tx_packets += pkts;
 663	port->stats.tx_bytes += bytes;
 664	port->tx_hw_csummed += hwchksum;
 665	u64_stats_update_end(&port->ir_stats_syncp);
 666
 667	txq->cptr = c;
 668}
 669
 670static void gmac_cleanup_txqs(struct net_device *netdev)
 671{
 672	struct gemini_ethernet_port *port = netdev_priv(netdev);
 673	unsigned int n_txq = netdev->num_tx_queues;
 674	struct gemini_ethernet *geth = port->geth;
 675	void __iomem *rwptr_reg;
 676	unsigned int r, i;
 677
 678	rwptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
 679
 680	for (i = 0; i < n_txq; i++) {
 681		r = readw(rwptr_reg);
 682		rwptr_reg += 2;
 683		writew(r, rwptr_reg);
 684		rwptr_reg += 2;
 685
 686		gmac_clean_txq(netdev, port->txq + i, r);
 687	}
 688	writel(0, port->dma_base + GMAC_SW_TX_QUEUE_BASE_REG);
 689
 690	kfree(port->txq->skb);
 691	dma_free_coherent(geth->dev,
 692			  n_txq * sizeof(*port->txq->ring) << port->txq_order,
 693			  port->txq->ring, port->txq_dma_base);
 694}
 695
 696static int gmac_setup_rxq(struct net_device *netdev)
 697{
 698	struct gemini_ethernet_port *port = netdev_priv(netdev);
 699	struct gemini_ethernet *geth = port->geth;
 700	struct nontoe_qhdr __iomem *qhdr;
 701
 702	qhdr = geth->base + TOE_DEFAULT_Q_HDR_BASE(netdev->dev_id);
 703	port->rxq_rwptr = &qhdr->word1;
 704
 705	/* Remap a slew of memory to use for the RX queue */
 706	port->rxq_ring = dma_alloc_coherent(geth->dev,
 707				sizeof(*port->rxq_ring) << port->rxq_order,
 708				&port->rxq_dma_base, GFP_KERNEL);
 709	if (!port->rxq_ring)
 710		return -ENOMEM;
 711	if (port->rxq_dma_base & ~NONTOE_QHDR0_BASE_MASK) {
 712		dev_warn(geth->dev, "RX queue base is not aligned\n");
 713		return -ENOMEM;
 714	}
 715
 716	writel(port->rxq_dma_base | port->rxq_order, &qhdr->word0);
 717	writel(0, port->rxq_rwptr);
 718	return 0;
 719}
 720
 721static struct gmac_queue_page *
 722gmac_get_queue_page(struct gemini_ethernet *geth,
 723		    struct gemini_ethernet_port *port,
 724		    dma_addr_t addr)
 725{
 726	struct gmac_queue_page *gpage;
 727	dma_addr_t mapping;
 728	int i;
 729
 730	/* Only look for even pages */
 731	mapping = addr & PAGE_MASK;
 732
 733	if (!geth->freeq_pages) {
 734		dev_err(geth->dev, "try to get page with no page list\n");
 735		return NULL;
 736	}
 737
 738	/* Look up a ring buffer page from virtual mapping */
 739	for (i = 0; i < geth->num_freeq_pages; i++) {
 740		gpage = &geth->freeq_pages[i];
 741		if (gpage->mapping == mapping)
 742			return gpage;
 743	}
 744
 745	return NULL;
 746}
 747
 748static void gmac_cleanup_rxq(struct net_device *netdev)
 749{
 750	struct gemini_ethernet_port *port = netdev_priv(netdev);
 751	struct gemini_ethernet *geth = port->geth;
 752	struct gmac_rxdesc *rxd = port->rxq_ring;
 753	static struct gmac_queue_page *gpage;
 754	struct nontoe_qhdr __iomem *qhdr;
 755	void __iomem *dma_reg;
 756	void __iomem *ptr_reg;
 757	dma_addr_t mapping;
 758	union dma_rwptr rw;
 759	unsigned int r, w;
 760
 761	qhdr = geth->base +
 762		TOE_DEFAULT_Q_HDR_BASE(netdev->dev_id);
 763	dma_reg = &qhdr->word0;
 764	ptr_reg = &qhdr->word1;
 765
 766	rw.bits32 = readl(ptr_reg);
 767	r = rw.bits.rptr;
 768	w = rw.bits.wptr;
 769	writew(r, ptr_reg + 2);
 770
 771	writel(0, dma_reg);
 772
 773	/* Loop from read pointer to write pointer of the RX queue
 774	 * and free up all pages by the queue.
 775	 */
 776	while (r != w) {
 777		mapping = rxd[r].word2.buf_adr;
 778		r++;
 779		r &= ((1 << port->rxq_order) - 1);
 780
 781		if (!mapping)
 782			continue;
 783
 784		/* Freeq pointers are one page off */
 785		gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
 786		if (!gpage) {
 787			dev_err(geth->dev, "could not find page\n");
 788			continue;
 789		}
 790		/* Release the RX queue reference to the page */
 791		put_page(gpage->page);
 792	}
 793
 794	dma_free_coherent(geth->dev, sizeof(*port->rxq_ring) << port->rxq_order,
 795			  port->rxq_ring, port->rxq_dma_base);
 796}
 797
 798static struct page *geth_freeq_alloc_map_page(struct gemini_ethernet *geth,
 799					      int pn)
 800{
 801	struct gmac_rxdesc *freeq_entry;
 802	struct gmac_queue_page *gpage;
 803	unsigned int fpp_order;
 804	unsigned int frag_len;
 805	dma_addr_t mapping;
 806	struct page *page;
 807	int i;
 808
 809	/* First allocate and DMA map a single page */
 810	page = alloc_page(GFP_ATOMIC);
 811	if (!page)
 812		return NULL;
 813
 814	mapping = dma_map_single(geth->dev, page_address(page),
 815				 PAGE_SIZE, DMA_FROM_DEVICE);
 816	if (dma_mapping_error(geth->dev, mapping)) {
 817		put_page(page);
 818		return NULL;
 819	}
 820
 821	/* The assign the page mapping (physical address) to the buffer address
 822	 * in the hardware queue. PAGE_SHIFT on ARM is 12 (1 page is 4096 bytes,
 823	 * 4k), and the default RX frag order is 11 (fragments are up 20 2048
 824	 * bytes, 2k) so fpp_order (fragments per page order) is default 1. Thus
 825	 * each page normally needs two entries in the queue.
 826	 */
 827	frag_len = 1 << geth->freeq_frag_order; /* Usually 2048 */
 828	fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 829	freeq_entry = geth->freeq_ring + (pn << fpp_order);
 830	dev_dbg(geth->dev, "allocate page %d fragment length %d fragments per page %d, freeq entry %p\n",
 831		 pn, frag_len, (1 << fpp_order), freeq_entry);
 832	for (i = (1 << fpp_order); i > 0; i--) {
 833		freeq_entry->word2.buf_adr = mapping;
 834		freeq_entry++;
 835		mapping += frag_len;
 836	}
 837
 838	/* If the freeq entry already has a page mapped, then unmap it. */
 839	gpage = &geth->freeq_pages[pn];
 840	if (gpage->page) {
 841		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
 842		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
 843		/* This should be the last reference to the page so it gets
 844		 * released
 845		 */
 846		put_page(gpage->page);
 847	}
 848
 849	/* Then put our new mapping into the page table */
 850	dev_dbg(geth->dev, "page %d, DMA addr: %08x, page %p\n",
 851		pn, (unsigned int)mapping, page);
 852	gpage->mapping = mapping;
 853	gpage->page = page;
 854
 855	return page;
 856}
 857
 858/**
 859 * geth_fill_freeq() - Fill the freeq with empty fragments to use
 860 * @geth: the ethernet adapter
 861 * @refill: whether to reset the queue by filling in all freeq entries or
 862 * just refill it, usually the interrupt to refill the queue happens when
 863 * the queue is half empty.
 864 */
 865static unsigned int geth_fill_freeq(struct gemini_ethernet *geth, bool refill)
 866{
 867	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 868	unsigned int count = 0;
 869	unsigned int pn, epn;
 870	unsigned long flags;
 871	union dma_rwptr rw;
 872	unsigned int m_pn;
 873
 874	/* Mask for page */
 875	m_pn = (1 << (geth->freeq_order - fpp_order)) - 1;
 876
 877	spin_lock_irqsave(&geth->freeq_lock, flags);
 878
 879	rw.bits32 = readl(geth->base + GLOBAL_SWFQ_RWPTR_REG);
 880	pn = (refill ? rw.bits.wptr : rw.bits.rptr) >> fpp_order;
 881	epn = (rw.bits.rptr >> fpp_order) - 1;
 882	epn &= m_pn;
 883
 884	/* Loop over the freeq ring buffer entries */
 885	while (pn != epn) {
 886		struct gmac_queue_page *gpage;
 887		struct page *page;
 888
 889		gpage = &geth->freeq_pages[pn];
 890		page = gpage->page;
 891
 892		dev_dbg(geth->dev, "fill entry %d page ref count %d add %d refs\n",
 893			pn, page_ref_count(page), 1 << fpp_order);
 894
 895		if (page_ref_count(page) > 1) {
 896			unsigned int fl = (pn - epn) & m_pn;
 897
 898			if (fl > 64 >> fpp_order)
 899				break;
 900
 901			page = geth_freeq_alloc_map_page(geth, pn);
 902			if (!page)
 903				break;
 904		}
 905
 906		/* Add one reference per fragment in the page */
 907		page_ref_add(page, 1 << fpp_order);
 908		count += 1 << fpp_order;
 909		pn++;
 910		pn &= m_pn;
 911	}
 912
 913	writew(pn << fpp_order, geth->base + GLOBAL_SWFQ_RWPTR_REG + 2);
 914
 915	spin_unlock_irqrestore(&geth->freeq_lock, flags);
 916
 917	return count;
 918}
 919
 920static int geth_setup_freeq(struct gemini_ethernet *geth)
 921{
 922	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 923	unsigned int frag_len = 1 << geth->freeq_frag_order;
 924	unsigned int len = 1 << geth->freeq_order;
 925	unsigned int pages = len >> fpp_order;
 926	union queue_threshold qt;
 927	union dma_skb_size skbsz;
 928	unsigned int filled;
 929	unsigned int pn;
 930
 931	geth->freeq_ring = dma_alloc_coherent(geth->dev,
 932		sizeof(*geth->freeq_ring) << geth->freeq_order,
 933		&geth->freeq_dma_base, GFP_KERNEL);
 934	if (!geth->freeq_ring)
 935		return -ENOMEM;
 936	if (geth->freeq_dma_base & ~DMA_Q_BASE_MASK) {
 937		dev_warn(geth->dev, "queue ring base is not aligned\n");
 938		goto err_freeq;
 939	}
 940
 941	/* Allocate a mapping to page look-up index */
 942	geth->freeq_pages = kcalloc(pages, sizeof(*geth->freeq_pages),
 943				    GFP_KERNEL);
 944	if (!geth->freeq_pages)
 945		goto err_freeq;
 946	geth->num_freeq_pages = pages;
 947
 948	dev_info(geth->dev, "allocate %d pages for queue\n", pages);
 949	for (pn = 0; pn < pages; pn++)
 950		if (!geth_freeq_alloc_map_page(geth, pn))
 951			goto err_freeq_alloc;
 952
 953	filled = geth_fill_freeq(geth, false);
 954	if (!filled)
 955		goto err_freeq_alloc;
 956
 957	qt.bits32 = readl(geth->base + GLOBAL_QUEUE_THRESHOLD_REG);
 958	qt.bits.swfq_empty = 32;
 959	writel(qt.bits32, geth->base + GLOBAL_QUEUE_THRESHOLD_REG);
 960
 961	skbsz.bits.sw_skb_size = 1 << geth->freeq_frag_order;
 962	writel(skbsz.bits32, geth->base + GLOBAL_DMA_SKB_SIZE_REG);
 963	writel(geth->freeq_dma_base | geth->freeq_order,
 964	       geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
 965
 966	return 0;
 967
 968err_freeq_alloc:
 969	while (pn > 0) {
 970		struct gmac_queue_page *gpage;
 971		dma_addr_t mapping;
 972
 973		--pn;
 974		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
 975		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
 976		gpage = &geth->freeq_pages[pn];
 977		put_page(gpage->page);
 978	}
 979
 980	kfree(geth->freeq_pages);
 981err_freeq:
 982	dma_free_coherent(geth->dev,
 983			  sizeof(*geth->freeq_ring) << geth->freeq_order,
 984			  geth->freeq_ring, geth->freeq_dma_base);
 985	geth->freeq_ring = NULL;
 986	return -ENOMEM;
 987}
 988
 989/**
 990 * geth_cleanup_freeq() - cleanup the DMA mappings and free the queue
 991 * @geth: the Gemini global ethernet state
 992 */
 993static void geth_cleanup_freeq(struct gemini_ethernet *geth)
 994{
 995	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 996	unsigned int frag_len = 1 << geth->freeq_frag_order;
 997	unsigned int len = 1 << geth->freeq_order;
 998	unsigned int pages = len >> fpp_order;
 999	unsigned int pn;
1000
1001	writew(readw(geth->base + GLOBAL_SWFQ_RWPTR_REG),
1002	       geth->base + GLOBAL_SWFQ_RWPTR_REG + 2);
1003	writel(0, geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
1004
1005	for (pn = 0; pn < pages; pn++) {
1006		struct gmac_queue_page *gpage;
1007		dma_addr_t mapping;
1008
1009		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
1010		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
1011
1012		gpage = &geth->freeq_pages[pn];
1013		while (page_ref_count(gpage->page) > 0)
1014			put_page(gpage->page);
1015	}
1016
1017	kfree(geth->freeq_pages);
1018
1019	dma_free_coherent(geth->dev,
1020			  sizeof(*geth->freeq_ring) << geth->freeq_order,
1021			  geth->freeq_ring, geth->freeq_dma_base);
1022}
1023
1024/**
1025 * geth_resize_freeq() - resize the software queue depth
1026 * @port: the port requesting the change
1027 *
1028 * This gets called at least once during probe() so the device queue gets
1029 * "resized" from the hardware defaults. Since both ports/net devices share
1030 * the same hardware queue, some synchronization between the ports is
1031 * needed.
1032 */
1033static int geth_resize_freeq(struct gemini_ethernet_port *port)
1034{
1035	struct gemini_ethernet *geth = port->geth;
1036	struct net_device *netdev = port->netdev;
1037	struct gemini_ethernet_port *other_port;
1038	struct net_device *other_netdev;
1039	unsigned int new_size = 0;
1040	unsigned int new_order;
1041	unsigned long flags;
1042	u32 en;
1043	int ret;
1044
1045	if (netdev->dev_id == 0)
1046		other_netdev = geth->port1->netdev;
1047	else
1048		other_netdev = geth->port0->netdev;
1049
1050	if (other_netdev && netif_running(other_netdev))
1051		return -EBUSY;
1052
1053	new_size = 1 << (port->rxq_order + 1);
1054	netdev_dbg(netdev, "port %d size: %d order %d\n",
1055		   netdev->dev_id,
1056		   new_size,
1057		   port->rxq_order);
1058	if (other_netdev) {
1059		other_port = netdev_priv(other_netdev);
1060		new_size += 1 << (other_port->rxq_order + 1);
1061		netdev_dbg(other_netdev, "port %d size: %d order %d\n",
1062			   other_netdev->dev_id,
1063			   (1 << (other_port->rxq_order + 1)),
1064			   other_port->rxq_order);
1065	}
1066
1067	new_order = min(15, ilog2(new_size - 1) + 1);
1068	dev_dbg(geth->dev, "set shared queue to size %d order %d\n",
1069		new_size, new_order);
1070	if (geth->freeq_order == new_order)
1071		return 0;
1072
1073	spin_lock_irqsave(&geth->irq_lock, flags);
1074
1075	/* Disable the software queue IRQs */
1076	en = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1077	en &= ~SWFQ_EMPTY_INT_BIT;
1078	writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1079	spin_unlock_irqrestore(&geth->irq_lock, flags);
1080
1081	/* Drop the old queue */
1082	if (geth->freeq_ring)
1083		geth_cleanup_freeq(geth);
1084
1085	/* Allocate a new queue with the desired order */
1086	geth->freeq_order = new_order;
1087	ret = geth_setup_freeq(geth);
1088
1089	/* Restart the interrupts - NOTE if this is the first resize
1090	 * after probe(), this is where the interrupts get turned on
1091	 * in the first place.
1092	 */
1093	spin_lock_irqsave(&geth->irq_lock, flags);
1094	en |= SWFQ_EMPTY_INT_BIT;
1095	writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1096	spin_unlock_irqrestore(&geth->irq_lock, flags);
1097
1098	return ret;
1099}
1100
1101static void gmac_tx_irq_enable(struct net_device *netdev,
1102			       unsigned int txq, int en)
1103{
1104	struct gemini_ethernet_port *port = netdev_priv(netdev);
1105	struct gemini_ethernet *geth = port->geth;
1106	unsigned long flags;
1107	u32 val, mask;
1108
1109	netdev_dbg(netdev, "%s device %d\n", __func__, netdev->dev_id);
1110
1111	spin_lock_irqsave(&geth->irq_lock, flags);
1112
1113	mask = GMAC0_IRQ0_TXQ0_INTS << (6 * netdev->dev_id + txq);
1114
1115	if (en)
1116		writel(mask, geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
1117
1118	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1119	val = en ? val | mask : val & ~mask;
1120	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1121
1122	spin_unlock_irqrestore(&geth->irq_lock, flags);
1123}
1124
1125static void gmac_tx_irq(struct net_device *netdev, unsigned int txq_num)
1126{
1127	struct netdev_queue *ntxq = netdev_get_tx_queue(netdev, txq_num);
1128
1129	gmac_tx_irq_enable(netdev, txq_num, 0);
1130	netif_tx_wake_queue(ntxq);
1131}
1132
1133static int gmac_map_tx_bufs(struct net_device *netdev, struct sk_buff *skb,
1134			    struct gmac_txq *txq, unsigned short *desc)
1135{
1136	struct gemini_ethernet_port *port = netdev_priv(netdev);
1137	struct skb_shared_info *skb_si =  skb_shinfo(skb);
1138	unsigned short m = (1 << port->txq_order) - 1;
1139	short frag, last_frag = skb_si->nr_frags - 1;
1140	struct gemini_ethernet *geth = port->geth;
1141	unsigned int word1, word3, buflen;
1142	unsigned short w = *desc;
1143	struct gmac_txdesc *txd;
1144	skb_frag_t *skb_frag;
1145	dma_addr_t mapping;
 
1146	void *buffer;
1147	u16 mss;
1148	int ret;
 
 
 
1149
1150	word1 = skb->len;
1151	word3 = SOF_BIT;
1152
1153	mss = skb_shinfo(skb)->gso_size;
1154	if (mss) {
1155		/* This means we are dealing with TCP and skb->len is the
1156		 * sum total of all the segments. The TSO will deal with
1157		 * chopping this up for us.
1158		 */
1159		/* The accelerator needs the full frame size here */
1160		mss += skb_tcp_all_headers(skb);
1161		netdev_dbg(netdev, "segment offloading mss = %04x len=%04x\n",
1162			   mss, skb->len);
1163		word1 |= TSS_MTU_ENABLE_BIT;
1164		word3 |= mss;
1165	} else if (skb->len >= ETH_FRAME_LEN) {
1166		/* Hardware offloaded checksumming isn't working on frames
1167		 * bigger than 1514 bytes. A hypothesis about this is that the
1168		 * checksum buffer is only 1518 bytes, so when the frames get
1169		 * bigger they get truncated, or the last few bytes get
1170		 * overwritten by the FCS.
1171		 *
1172		 * Just use software checksumming and bypass on bigger frames.
1173		 */
1174		if (skb->ip_summed == CHECKSUM_PARTIAL) {
1175			ret = skb_checksum_help(skb);
1176			if (ret)
1177				return ret;
1178		}
1179		word1 |= TSS_BYPASS_BIT;
1180	}
1181
1182	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1183		int tcp = 0;
1184
1185		/* We do not switch off the checksumming on non TCP/UDP
1186		 * frames: as is shown from tests, the checksumming engine
1187		 * is smart enough to see that a frame is not actually TCP
1188		 * or UDP and then just pass it through without any changes
1189		 * to the frame.
1190		 */
1191		if (skb->protocol == htons(ETH_P_IP)) {
1192			word1 |= TSS_IP_CHKSUM_BIT;
1193			tcp = ip_hdr(skb)->protocol == IPPROTO_TCP;
1194		} else { /* IPv6 */
1195			word1 |= TSS_IPV6_ENABLE_BIT;
1196			tcp = ipv6_hdr(skb)->nexthdr == IPPROTO_TCP;
1197		}
1198
1199		word1 |= tcp ? TSS_TCP_CHKSUM_BIT : TSS_UDP_CHKSUM_BIT;
1200	}
1201
1202	frag = -1;
1203	while (frag <= last_frag) {
1204		if (frag == -1) {
1205			buffer = skb->data;
1206			buflen = skb_headlen(skb);
1207		} else {
1208			skb_frag = skb_si->frags + frag;
1209			buffer = skb_frag_address(skb_frag);
1210			buflen = skb_frag_size(skb_frag);
1211		}
1212
1213		if (frag == last_frag) {
1214			word3 |= EOF_BIT;
1215			txq->skb[w] = skb;
1216		}
1217
1218		mapping = dma_map_single(geth->dev, buffer, buflen,
1219					 DMA_TO_DEVICE);
1220		if (dma_mapping_error(geth->dev, mapping))
1221			goto map_error;
1222
1223		txd = txq->ring + w;
1224		txd->word0.bits32 = buflen;
1225		txd->word1.bits32 = word1;
1226		txd->word2.buf_adr = mapping;
1227		txd->word3.bits32 = word3;
1228
1229		word3 &= MTU_SIZE_BIT_MASK;
1230		w++;
1231		w &= m;
1232		frag++;
1233	}
1234
1235	*desc = w;
1236	return 0;
1237
1238map_error:
1239	while (w != *desc) {
1240		w--;
1241		w &= m;
1242
1243		dma_unmap_page(geth->dev, txq->ring[w].word2.buf_adr,
1244			       txq->ring[w].word0.bits.buffer_size,
1245			       DMA_TO_DEVICE);
1246	}
1247	return -ENOMEM;
1248}
1249
1250static netdev_tx_t gmac_start_xmit(struct sk_buff *skb,
1251				   struct net_device *netdev)
1252{
1253	struct gemini_ethernet_port *port = netdev_priv(netdev);
1254	unsigned short m = (1 << port->txq_order) - 1;
1255	struct netdev_queue *ntxq;
1256	unsigned short r, w, d;
1257	void __iomem *ptr_reg;
1258	struct gmac_txq *txq;
1259	int txq_num, nfrags;
1260	union dma_rwptr rw;
1261
1262	if (skb->len >= 0x10000)
1263		goto out_drop_free;
1264
1265	txq_num = skb_get_queue_mapping(skb);
1266	ptr_reg = port->dma_base + GMAC_SW_TX_QUEUE_PTR_REG(txq_num);
1267	txq = &port->txq[txq_num];
1268	ntxq = netdev_get_tx_queue(netdev, txq_num);
1269	nfrags = skb_shinfo(skb)->nr_frags;
1270
1271	rw.bits32 = readl(ptr_reg);
1272	r = rw.bits.rptr;
1273	w = rw.bits.wptr;
1274
1275	d = txq->cptr - w - 1;
1276	d &= m;
1277
1278	if (d < nfrags + 2) {
1279		gmac_clean_txq(netdev, txq, r);
1280		d = txq->cptr - w - 1;
1281		d &= m;
1282
1283		if (d < nfrags + 2) {
1284			netif_tx_stop_queue(ntxq);
1285
1286			d = txq->cptr + nfrags + 16;
1287			d &= m;
1288			txq->ring[d].word3.bits.eofie = 1;
1289			gmac_tx_irq_enable(netdev, txq_num, 1);
1290
1291			u64_stats_update_begin(&port->tx_stats_syncp);
1292			netdev->stats.tx_fifo_errors++;
1293			u64_stats_update_end(&port->tx_stats_syncp);
1294			return NETDEV_TX_BUSY;
1295		}
1296	}
1297
1298	if (gmac_map_tx_bufs(netdev, skb, txq, &w)) {
1299		if (skb_linearize(skb))
1300			goto out_drop;
1301
1302		u64_stats_update_begin(&port->tx_stats_syncp);
1303		port->tx_frags_linearized++;
1304		u64_stats_update_end(&port->tx_stats_syncp);
1305
1306		if (gmac_map_tx_bufs(netdev, skb, txq, &w))
1307			goto out_drop_free;
1308	}
1309
1310	writew(w, ptr_reg + 2);
1311
1312	gmac_clean_txq(netdev, txq, r);
1313	return NETDEV_TX_OK;
1314
1315out_drop_free:
1316	dev_kfree_skb(skb);
1317out_drop:
1318	u64_stats_update_begin(&port->tx_stats_syncp);
1319	port->stats.tx_dropped++;
1320	u64_stats_update_end(&port->tx_stats_syncp);
1321	return NETDEV_TX_OK;
1322}
1323
1324static void gmac_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1325{
1326	netdev_err(netdev, "Tx timeout\n");
1327	gmac_dump_dma_state(netdev);
1328}
1329
1330static void gmac_enable_irq(struct net_device *netdev, int enable)
1331{
1332	struct gemini_ethernet_port *port = netdev_priv(netdev);
1333	struct gemini_ethernet *geth = port->geth;
1334	unsigned long flags;
1335	u32 val, mask;
1336
1337	netdev_dbg(netdev, "%s device %d %s\n", __func__,
1338		   netdev->dev_id, enable ? "enable" : "disable");
1339	spin_lock_irqsave(&geth->irq_lock, flags);
1340
1341	mask = GMAC0_IRQ0_2 << (netdev->dev_id * 2);
1342	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1343	val = enable ? (val | mask) : (val & ~mask);
1344	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1345
1346	mask = DEFAULT_Q0_INT_BIT << netdev->dev_id;
1347	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1348	val = enable ? (val | mask) : (val & ~mask);
1349	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1350
1351	mask = GMAC0_IRQ4_8 << (netdev->dev_id * 8);
1352	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1353	val = enable ? (val | mask) : (val & ~mask);
1354	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1355
1356	spin_unlock_irqrestore(&geth->irq_lock, flags);
1357}
1358
1359static void gmac_enable_rx_irq(struct net_device *netdev, int enable)
1360{
1361	struct gemini_ethernet_port *port = netdev_priv(netdev);
1362	struct gemini_ethernet *geth = port->geth;
1363	unsigned long flags;
1364	u32 val, mask;
1365
1366	netdev_dbg(netdev, "%s device %d %s\n", __func__, netdev->dev_id,
1367		   enable ? "enable" : "disable");
1368	spin_lock_irqsave(&geth->irq_lock, flags);
1369	mask = DEFAULT_Q0_INT_BIT << netdev->dev_id;
1370
1371	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1372	val = enable ? (val | mask) : (val & ~mask);
1373	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1374
1375	spin_unlock_irqrestore(&geth->irq_lock, flags);
1376}
1377
1378static struct sk_buff *gmac_skb_if_good_frame(struct gemini_ethernet_port *port,
1379					      union gmac_rxdesc_0 word0,
1380					      unsigned int frame_len)
1381{
1382	unsigned int rx_csum = word0.bits.chksum_status;
1383	unsigned int rx_status = word0.bits.status;
1384	struct sk_buff *skb = NULL;
1385
1386	port->rx_stats[rx_status]++;
1387	port->rx_csum_stats[rx_csum]++;
1388
1389	if (word0.bits.derr || word0.bits.perr ||
1390	    rx_status || frame_len < ETH_ZLEN ||
1391	    rx_csum >= RX_CHKSUM_IP_ERR_UNKNOWN) {
1392		port->stats.rx_errors++;
1393
1394		if (frame_len < ETH_ZLEN || RX_ERROR_LENGTH(rx_status))
1395			port->stats.rx_length_errors++;
1396		if (RX_ERROR_OVER(rx_status))
1397			port->stats.rx_over_errors++;
1398		if (RX_ERROR_CRC(rx_status))
1399			port->stats.rx_crc_errors++;
1400		if (RX_ERROR_FRAME(rx_status))
1401			port->stats.rx_frame_errors++;
1402		return NULL;
1403	}
1404
1405	skb = napi_get_frags(&port->napi);
1406	if (!skb)
1407		goto update_exit;
1408
1409	if (rx_csum == RX_CHKSUM_IP_UDP_TCP_OK)
1410		skb->ip_summed = CHECKSUM_UNNECESSARY;
1411
1412update_exit:
1413	port->stats.rx_bytes += frame_len;
1414	port->stats.rx_packets++;
1415	return skb;
1416}
1417
1418static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
1419{
1420	struct gemini_ethernet_port *port = netdev_priv(netdev);
1421	unsigned short m = (1 << port->rxq_order) - 1;
1422	struct gemini_ethernet *geth = port->geth;
1423	void __iomem *ptr_reg = port->rxq_rwptr;
1424	unsigned int frame_len, frag_len;
1425	struct gmac_rxdesc *rx = NULL;
1426	struct gmac_queue_page *gpage;
1427	static struct sk_buff *skb;
1428	union gmac_rxdesc_0 word0;
1429	union gmac_rxdesc_1 word1;
1430	union gmac_rxdesc_3 word3;
1431	struct page *page = NULL;
1432	unsigned int page_offs;
1433	unsigned long flags;
1434	unsigned short r, w;
1435	union dma_rwptr rw;
1436	dma_addr_t mapping;
1437	int frag_nr = 0;
1438
1439	spin_lock_irqsave(&geth->irq_lock, flags);
1440	rw.bits32 = readl(ptr_reg);
1441	/* Reset interrupt as all packages until here are taken into account */
1442	writel(DEFAULT_Q0_INT_BIT << netdev->dev_id,
1443	       geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
1444	spin_unlock_irqrestore(&geth->irq_lock, flags);
1445
1446	r = rw.bits.rptr;
1447	w = rw.bits.wptr;
1448
1449	while (budget && w != r) {
1450		rx = port->rxq_ring + r;
1451		word0 = rx->word0;
1452		word1 = rx->word1;
1453		mapping = rx->word2.buf_adr;
1454		word3 = rx->word3;
1455
1456		r++;
1457		r &= m;
1458
1459		frag_len = word0.bits.buffer_size;
1460		frame_len = word1.bits.byte_count;
1461		page_offs = mapping & ~PAGE_MASK;
1462
1463		if (!mapping) {
1464			netdev_err(netdev,
1465				   "rxq[%u]: HW BUG: zero DMA desc\n", r);
1466			goto err_drop;
1467		}
1468
1469		/* Freeq pointers are one page off */
1470		gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
1471		if (!gpage) {
1472			dev_err(geth->dev, "could not find mapping\n");
1473			continue;
1474		}
1475		page = gpage->page;
1476
1477		if (word3.bits32 & SOF_BIT) {
1478			if (skb) {
1479				napi_free_frags(&port->napi);
1480				port->stats.rx_dropped++;
1481			}
1482
1483			skb = gmac_skb_if_good_frame(port, word0, frame_len);
1484			if (!skb)
1485				goto err_drop;
1486
1487			page_offs += NET_IP_ALIGN;
1488			frag_len -= NET_IP_ALIGN;
1489			frag_nr = 0;
1490
1491		} else if (!skb) {
1492			put_page(page);
1493			continue;
1494		}
1495
1496		if (word3.bits32 & EOF_BIT)
1497			frag_len = frame_len - skb->len;
1498
1499		/* append page frag to skb */
1500		if (frag_nr == MAX_SKB_FRAGS)
1501			goto err_drop;
1502
1503		if (frag_len == 0)
1504			netdev_err(netdev, "Received fragment with len = 0\n");
1505
1506		skb_fill_page_desc(skb, frag_nr, page, page_offs, frag_len);
1507		skb->len += frag_len;
1508		skb->data_len += frag_len;
1509		skb->truesize += frag_len;
1510		frag_nr++;
1511
1512		if (word3.bits32 & EOF_BIT) {
1513			napi_gro_frags(&port->napi);
1514			skb = NULL;
1515			--budget;
1516		}
1517		continue;
1518
1519err_drop:
1520		if (skb) {
1521			napi_free_frags(&port->napi);
1522			skb = NULL;
1523		}
1524
1525		if (mapping)
1526			put_page(page);
1527
1528		port->stats.rx_dropped++;
1529	}
1530
1531	writew(r, ptr_reg);
1532	return budget;
1533}
1534
1535static int gmac_napi_poll(struct napi_struct *napi, int budget)
1536{
1537	struct gemini_ethernet_port *port = netdev_priv(napi->dev);
1538	struct gemini_ethernet *geth = port->geth;
1539	unsigned int freeq_threshold;
1540	unsigned int received;
1541
1542	freeq_threshold = 1 << (geth->freeq_order - 1);
1543	u64_stats_update_begin(&port->rx_stats_syncp);
1544
1545	received = gmac_rx(napi->dev, budget);
1546	if (received < budget) {
1547		napi_gro_flush(napi, false);
1548		napi_complete_done(napi, received);
1549		gmac_enable_rx_irq(napi->dev, 1);
1550		++port->rx_napi_exits;
1551	}
1552
1553	port->freeq_refill += (budget - received);
1554	if (port->freeq_refill > freeq_threshold) {
1555		port->freeq_refill -= freeq_threshold;
1556		geth_fill_freeq(geth, true);
1557	}
1558
1559	u64_stats_update_end(&port->rx_stats_syncp);
1560	return received;
1561}
1562
1563static void gmac_dump_dma_state(struct net_device *netdev)
1564{
1565	struct gemini_ethernet_port *port = netdev_priv(netdev);
1566	struct gemini_ethernet *geth = port->geth;
1567	void __iomem *ptr_reg;
1568	u32 reg[5];
1569
1570	/* Interrupt status */
1571	reg[0] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
1572	reg[1] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
1573	reg[2] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_2_REG);
1574	reg[3] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_3_REG);
1575	reg[4] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
1576	netdev_err(netdev, "IRQ status: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1577		   reg[0], reg[1], reg[2], reg[3], reg[4]);
1578
1579	/* Interrupt enable */
1580	reg[0] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1581	reg[1] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1582	reg[2] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_2_REG);
1583	reg[3] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_3_REG);
1584	reg[4] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1585	netdev_err(netdev, "IRQ enable: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1586		   reg[0], reg[1], reg[2], reg[3], reg[4]);
1587
1588	/* RX DMA status */
1589	reg[0] = readl(port->dma_base + GMAC_DMA_RX_FIRST_DESC_REG);
1590	reg[1] = readl(port->dma_base + GMAC_DMA_RX_CURR_DESC_REG);
1591	reg[2] = GET_RPTR(port->rxq_rwptr);
1592	reg[3] = GET_WPTR(port->rxq_rwptr);
1593	netdev_err(netdev, "RX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
1594		   reg[0], reg[1], reg[2], reg[3]);
1595
1596	reg[0] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD0_REG);
1597	reg[1] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD1_REG);
1598	reg[2] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD2_REG);
1599	reg[3] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD3_REG);
1600	netdev_err(netdev, "RX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1601		   reg[0], reg[1], reg[2], reg[3]);
1602
1603	/* TX DMA status */
1604	ptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
1605
1606	reg[0] = readl(port->dma_base + GMAC_DMA_TX_FIRST_DESC_REG);
1607	reg[1] = readl(port->dma_base + GMAC_DMA_TX_CURR_DESC_REG);
1608	reg[2] = GET_RPTR(ptr_reg);
1609	reg[3] = GET_WPTR(ptr_reg);
1610	netdev_err(netdev, "TX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
1611		   reg[0], reg[1], reg[2], reg[3]);
1612
1613	reg[0] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD0_REG);
1614	reg[1] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD1_REG);
1615	reg[2] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD2_REG);
1616	reg[3] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD3_REG);
1617	netdev_err(netdev, "TX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1618		   reg[0], reg[1], reg[2], reg[3]);
1619
1620	/* FREE queues status */
1621	ptr_reg = geth->base + GLOBAL_SWFQ_RWPTR_REG;
1622
1623	reg[0] = GET_RPTR(ptr_reg);
1624	reg[1] = GET_WPTR(ptr_reg);
1625
1626	ptr_reg = geth->base + GLOBAL_HWFQ_RWPTR_REG;
1627
1628	reg[2] = GET_RPTR(ptr_reg);
1629	reg[3] = GET_WPTR(ptr_reg);
1630	netdev_err(netdev, "FQ SW ptr: %u %u, HW ptr: %u %u\n",
1631		   reg[0], reg[1], reg[2], reg[3]);
1632}
1633
1634static void gmac_update_hw_stats(struct net_device *netdev)
1635{
1636	struct gemini_ethernet_port *port = netdev_priv(netdev);
1637	unsigned int rx_discards, rx_mcast, rx_bcast;
1638	struct gemini_ethernet *geth = port->geth;
1639	unsigned long flags;
1640
1641	spin_lock_irqsave(&geth->irq_lock, flags);
1642	u64_stats_update_begin(&port->ir_stats_syncp);
1643
1644	rx_discards = readl(port->gmac_base + GMAC_IN_DISCARDS);
1645	port->hw_stats[0] += rx_discards;
1646	port->hw_stats[1] += readl(port->gmac_base + GMAC_IN_ERRORS);
1647	rx_mcast = readl(port->gmac_base + GMAC_IN_MCAST);
1648	port->hw_stats[2] += rx_mcast;
1649	rx_bcast = readl(port->gmac_base + GMAC_IN_BCAST);
1650	port->hw_stats[3] += rx_bcast;
1651	port->hw_stats[4] += readl(port->gmac_base + GMAC_IN_MAC1);
1652	port->hw_stats[5] += readl(port->gmac_base + GMAC_IN_MAC2);
1653
1654	port->stats.rx_missed_errors += rx_discards;
1655	port->stats.multicast += rx_mcast;
1656	port->stats.multicast += rx_bcast;
1657
1658	writel(GMAC0_MIB_INT_BIT << (netdev->dev_id * 8),
1659	       geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
1660
1661	u64_stats_update_end(&port->ir_stats_syncp);
1662	spin_unlock_irqrestore(&geth->irq_lock, flags);
1663}
1664
1665/**
1666 * gmac_get_intr_flags() - get interrupt status flags for a port from
1667 * @netdev: the net device for the port to get flags from
1668 * @i: the interrupt status register 0..4
1669 */
1670static u32 gmac_get_intr_flags(struct net_device *netdev, int i)
1671{
1672	struct gemini_ethernet_port *port = netdev_priv(netdev);
1673	struct gemini_ethernet *geth = port->geth;
1674	void __iomem *irqif_reg, *irqen_reg;
1675	unsigned int offs, val;
1676
1677	/* Calculate the offset using the stride of the status registers */
1678	offs = i * (GLOBAL_INTERRUPT_STATUS_1_REG -
1679		    GLOBAL_INTERRUPT_STATUS_0_REG);
1680
1681	irqif_reg = geth->base + GLOBAL_INTERRUPT_STATUS_0_REG + offs;
1682	irqen_reg = geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG + offs;
1683
1684	val = readl(irqif_reg) & readl(irqen_reg);
1685	return val;
1686}
1687
1688static enum hrtimer_restart gmac_coalesce_delay_expired(struct hrtimer *timer)
1689{
1690	struct gemini_ethernet_port *port =
1691		container_of(timer, struct gemini_ethernet_port,
1692			     rx_coalesce_timer);
1693
1694	napi_schedule(&port->napi);
1695	return HRTIMER_NORESTART;
1696}
1697
1698static irqreturn_t gmac_irq(int irq, void *data)
1699{
1700	struct gemini_ethernet_port *port;
1701	struct net_device *netdev = data;
1702	struct gemini_ethernet *geth;
1703	u32 val, orr = 0;
1704
1705	port = netdev_priv(netdev);
1706	geth = port->geth;
1707
1708	val = gmac_get_intr_flags(netdev, 0);
1709	orr |= val;
1710
1711	if (val & (GMAC0_IRQ0_2 << (netdev->dev_id * 2))) {
1712		/* Oh, crap */
1713		netdev_err(netdev, "hw failure/sw bug\n");
1714		gmac_dump_dma_state(netdev);
1715
1716		/* don't know how to recover, just reduce losses */
1717		gmac_enable_irq(netdev, 0);
1718		return IRQ_HANDLED;
1719	}
1720
1721	if (val & (GMAC0_IRQ0_TXQ0_INTS << (netdev->dev_id * 6)))
1722		gmac_tx_irq(netdev, 0);
1723
1724	val = gmac_get_intr_flags(netdev, 1);
1725	orr |= val;
1726
1727	if (val & (DEFAULT_Q0_INT_BIT << netdev->dev_id)) {
1728		gmac_enable_rx_irq(netdev, 0);
1729
1730		if (!port->rx_coalesce_nsecs) {
1731			napi_schedule(&port->napi);
1732		} else {
1733			ktime_t ktime;
1734
1735			ktime = ktime_set(0, port->rx_coalesce_nsecs);
1736			hrtimer_start(&port->rx_coalesce_timer, ktime,
1737				      HRTIMER_MODE_REL);
1738		}
1739	}
1740
1741	val = gmac_get_intr_flags(netdev, 4);
1742	orr |= val;
1743
1744	if (val & (GMAC0_MIB_INT_BIT << (netdev->dev_id * 8)))
1745		gmac_update_hw_stats(netdev);
1746
1747	if (val & (GMAC0_RX_OVERRUN_INT_BIT << (netdev->dev_id * 8))) {
1748		spin_lock(&geth->irq_lock);
1749		writel(GMAC0_RXDERR_INT_BIT << (netdev->dev_id * 8),
1750		       geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
 
 
1751		u64_stats_update_begin(&port->ir_stats_syncp);
1752		++port->stats.rx_fifo_errors;
1753		u64_stats_update_end(&port->ir_stats_syncp);
1754		spin_unlock(&geth->irq_lock);
1755	}
1756
1757	return orr ? IRQ_HANDLED : IRQ_NONE;
1758}
1759
1760static void gmac_start_dma(struct gemini_ethernet_port *port)
1761{
1762	void __iomem *dma_ctrl_reg = port->dma_base + GMAC_DMA_CTRL_REG;
1763	union gmac_dma_ctrl dma_ctrl;
1764
1765	dma_ctrl.bits32 = readl(dma_ctrl_reg);
1766	dma_ctrl.bits.rd_enable = 1;
1767	dma_ctrl.bits.td_enable = 1;
1768	dma_ctrl.bits.loopback = 0;
1769	dma_ctrl.bits.drop_small_ack = 0;
1770	dma_ctrl.bits.rd_insert_bytes = NET_IP_ALIGN;
1771	dma_ctrl.bits.rd_prot = HPROT_DATA_CACHE | HPROT_PRIVILIGED;
1772	dma_ctrl.bits.rd_burst_size = HBURST_INCR8;
1773	dma_ctrl.bits.rd_bus = HSIZE_8;
1774	dma_ctrl.bits.td_prot = HPROT_DATA_CACHE;
1775	dma_ctrl.bits.td_burst_size = HBURST_INCR8;
1776	dma_ctrl.bits.td_bus = HSIZE_8;
1777
1778	writel(dma_ctrl.bits32, dma_ctrl_reg);
1779}
1780
1781static void gmac_stop_dma(struct gemini_ethernet_port *port)
1782{
1783	void __iomem *dma_ctrl_reg = port->dma_base + GMAC_DMA_CTRL_REG;
1784	union gmac_dma_ctrl dma_ctrl;
1785
1786	dma_ctrl.bits32 = readl(dma_ctrl_reg);
1787	dma_ctrl.bits.rd_enable = 0;
1788	dma_ctrl.bits.td_enable = 0;
1789	writel(dma_ctrl.bits32, dma_ctrl_reg);
1790}
1791
1792static int gmac_open(struct net_device *netdev)
1793{
1794	struct gemini_ethernet_port *port = netdev_priv(netdev);
1795	int err;
1796
1797	err = request_irq(netdev->irq, gmac_irq,
1798			  IRQF_SHARED, netdev->name, netdev);
1799	if (err) {
1800		netdev_err(netdev, "no IRQ\n");
1801		return err;
1802	}
1803
1804	netif_carrier_off(netdev);
1805	phy_start(netdev->phydev);
1806
1807	err = geth_resize_freeq(port);
1808	/* It's fine if it's just busy, the other port has set up
1809	 * the freeq in that case.
1810	 */
1811	if (err && (err != -EBUSY)) {
1812		netdev_err(netdev, "could not resize freeq\n");
1813		goto err_stop_phy;
1814	}
1815
1816	err = gmac_setup_rxq(netdev);
1817	if (err) {
1818		netdev_err(netdev, "could not setup RXQ\n");
1819		goto err_stop_phy;
1820	}
1821
1822	err = gmac_setup_txqs(netdev);
1823	if (err) {
1824		netdev_err(netdev, "could not setup TXQs\n");
1825		gmac_cleanup_rxq(netdev);
1826		goto err_stop_phy;
1827	}
1828
1829	napi_enable(&port->napi);
1830
1831	gmac_start_dma(port);
1832	gmac_enable_irq(netdev, 1);
1833	gmac_enable_tx_rx(netdev);
1834	netif_tx_start_all_queues(netdev);
1835
1836	hrtimer_init(&port->rx_coalesce_timer, CLOCK_MONOTONIC,
1837		     HRTIMER_MODE_REL);
1838	port->rx_coalesce_timer.function = &gmac_coalesce_delay_expired;
1839
1840	netdev_dbg(netdev, "opened\n");
1841
1842	return 0;
1843
1844err_stop_phy:
1845	phy_stop(netdev->phydev);
1846	free_irq(netdev->irq, netdev);
1847	return err;
1848}
1849
1850static int gmac_stop(struct net_device *netdev)
1851{
1852	struct gemini_ethernet_port *port = netdev_priv(netdev);
1853
1854	hrtimer_cancel(&port->rx_coalesce_timer);
1855	netif_tx_stop_all_queues(netdev);
1856	gmac_disable_tx_rx(netdev);
1857	gmac_stop_dma(port);
1858	napi_disable(&port->napi);
1859
1860	gmac_enable_irq(netdev, 0);
1861	gmac_cleanup_rxq(netdev);
1862	gmac_cleanup_txqs(netdev);
1863
1864	phy_stop(netdev->phydev);
1865	free_irq(netdev->irq, netdev);
1866
1867	gmac_update_hw_stats(netdev);
1868	return 0;
1869}
1870
1871static void gmac_set_rx_mode(struct net_device *netdev)
1872{
1873	struct gemini_ethernet_port *port = netdev_priv(netdev);
1874	union gmac_rx_fltr filter = { .bits = {
1875		.broadcast = 1,
1876		.multicast = 1,
1877		.unicast = 1,
1878	} };
1879	struct netdev_hw_addr *ha;
1880	unsigned int bit_nr;
1881	u32 mc_filter[2];
1882
1883	mc_filter[1] = 0;
1884	mc_filter[0] = 0;
1885
1886	if (netdev->flags & IFF_PROMISC) {
1887		filter.bits.error = 1;
1888		filter.bits.promiscuous = 1;
1889		mc_filter[1] = ~0;
1890		mc_filter[0] = ~0;
1891	} else if (netdev->flags & IFF_ALLMULTI) {
1892		mc_filter[1] = ~0;
1893		mc_filter[0] = ~0;
1894	} else {
1895		netdev_for_each_mc_addr(ha, netdev) {
1896			bit_nr = ~crc32_le(~0, ha->addr, ETH_ALEN) & 0x3f;
1897			mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 0x1f);
1898		}
1899	}
1900
1901	writel(mc_filter[0], port->gmac_base + GMAC_MCAST_FIL0);
1902	writel(mc_filter[1], port->gmac_base + GMAC_MCAST_FIL1);
1903	writel(filter.bits32, port->gmac_base + GMAC_RX_FLTR);
1904}
1905
1906static void gmac_write_mac_address(struct net_device *netdev)
1907{
1908	struct gemini_ethernet_port *port = netdev_priv(netdev);
1909	__le32 addr[3];
1910
1911	memset(addr, 0, sizeof(addr));
1912	memcpy(addr, netdev->dev_addr, ETH_ALEN);
1913
1914	writel(le32_to_cpu(addr[0]), port->gmac_base + GMAC_STA_ADD0);
1915	writel(le32_to_cpu(addr[1]), port->gmac_base + GMAC_STA_ADD1);
1916	writel(le32_to_cpu(addr[2]), port->gmac_base + GMAC_STA_ADD2);
1917}
1918
1919static int gmac_set_mac_address(struct net_device *netdev, void *addr)
1920{
1921	struct sockaddr *sa = addr;
1922
1923	eth_hw_addr_set(netdev, sa->sa_data);
1924	gmac_write_mac_address(netdev);
1925
1926	return 0;
1927}
1928
1929static void gmac_clear_hw_stats(struct net_device *netdev)
1930{
1931	struct gemini_ethernet_port *port = netdev_priv(netdev);
1932
1933	readl(port->gmac_base + GMAC_IN_DISCARDS);
1934	readl(port->gmac_base + GMAC_IN_ERRORS);
1935	readl(port->gmac_base + GMAC_IN_MCAST);
1936	readl(port->gmac_base + GMAC_IN_BCAST);
1937	readl(port->gmac_base + GMAC_IN_MAC1);
1938	readl(port->gmac_base + GMAC_IN_MAC2);
1939}
1940
1941static void gmac_get_stats64(struct net_device *netdev,
1942			     struct rtnl_link_stats64 *stats)
1943{
1944	struct gemini_ethernet_port *port = netdev_priv(netdev);
1945	unsigned int start;
1946
1947	gmac_update_hw_stats(netdev);
1948
1949	/* Racing with RX NAPI */
1950	do {
1951		start = u64_stats_fetch_begin(&port->rx_stats_syncp);
1952
1953		stats->rx_packets = port->stats.rx_packets;
1954		stats->rx_bytes = port->stats.rx_bytes;
1955		stats->rx_errors = port->stats.rx_errors;
1956		stats->rx_dropped = port->stats.rx_dropped;
1957
1958		stats->rx_length_errors = port->stats.rx_length_errors;
1959		stats->rx_over_errors = port->stats.rx_over_errors;
1960		stats->rx_crc_errors = port->stats.rx_crc_errors;
1961		stats->rx_frame_errors = port->stats.rx_frame_errors;
1962
1963	} while (u64_stats_fetch_retry(&port->rx_stats_syncp, start));
1964
1965	/* Racing with MIB and TX completion interrupts */
1966	do {
1967		start = u64_stats_fetch_begin(&port->ir_stats_syncp);
1968
1969		stats->tx_errors = port->stats.tx_errors;
1970		stats->tx_packets = port->stats.tx_packets;
1971		stats->tx_bytes = port->stats.tx_bytes;
1972
1973		stats->multicast = port->stats.multicast;
1974		stats->rx_missed_errors = port->stats.rx_missed_errors;
1975		stats->rx_fifo_errors = port->stats.rx_fifo_errors;
1976
1977	} while (u64_stats_fetch_retry(&port->ir_stats_syncp, start));
1978
1979	/* Racing with hard_start_xmit */
1980	do {
1981		start = u64_stats_fetch_begin(&port->tx_stats_syncp);
1982
1983		stats->tx_dropped = port->stats.tx_dropped;
1984
1985	} while (u64_stats_fetch_retry(&port->tx_stats_syncp, start));
1986
1987	stats->rx_dropped += stats->rx_missed_errors;
1988}
1989
1990static int gmac_change_mtu(struct net_device *netdev, int new_mtu)
1991{
1992	int max_len = gmac_pick_rx_max_len(new_mtu);
1993
1994	if (max_len < 0)
1995		return -EINVAL;
1996
1997	gmac_disable_tx_rx(netdev);
1998
1999	WRITE_ONCE(netdev->mtu, new_mtu);
2000	gmac_update_config0_reg(netdev, max_len << CONFIG0_MAXLEN_SHIFT,
2001				CONFIG0_MAXLEN_MASK);
2002
2003	netdev_update_features(netdev);
2004
2005	gmac_enable_tx_rx(netdev);
2006
2007	return 0;
2008}
2009
 
 
 
 
 
 
 
 
 
2010static int gmac_set_features(struct net_device *netdev,
2011			     netdev_features_t features)
2012{
2013	struct gemini_ethernet_port *port = netdev_priv(netdev);
2014	int enable = features & NETIF_F_RXCSUM;
2015	unsigned long flags;
2016	u32 reg;
2017
2018	spin_lock_irqsave(&port->config_lock, flags);
2019
2020	reg = readl(port->gmac_base + GMAC_CONFIG0);
2021	reg = enable ? reg | CONFIG0_RX_CHKSUM : reg & ~CONFIG0_RX_CHKSUM;
2022	writel(reg, port->gmac_base + GMAC_CONFIG0);
2023
2024	spin_unlock_irqrestore(&port->config_lock, flags);
2025	return 0;
2026}
2027
2028static int gmac_get_sset_count(struct net_device *netdev, int sset)
2029{
2030	return sset == ETH_SS_STATS ? GMAC_STATS_NUM : 0;
2031}
2032
2033static void gmac_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
2034{
2035	if (stringset != ETH_SS_STATS)
2036		return;
2037
2038	memcpy(data, gmac_stats_strings, sizeof(gmac_stats_strings));
2039}
2040
2041static void gmac_get_ethtool_stats(struct net_device *netdev,
2042				   struct ethtool_stats *estats, u64 *values)
2043{
2044	struct gemini_ethernet_port *port = netdev_priv(netdev);
2045	unsigned int start;
2046	u64 *p;
2047	int i;
2048
2049	gmac_update_hw_stats(netdev);
2050
2051	/* Racing with MIB interrupt */
2052	do {
2053		p = values;
2054		start = u64_stats_fetch_begin(&port->ir_stats_syncp);
2055
2056		for (i = 0; i < RX_STATS_NUM; i++)
2057			*p++ = port->hw_stats[i];
2058
2059	} while (u64_stats_fetch_retry(&port->ir_stats_syncp, start));
2060	values = p;
2061
2062	/* Racing with RX NAPI */
2063	do {
2064		p = values;
2065		start = u64_stats_fetch_begin(&port->rx_stats_syncp);
2066
2067		for (i = 0; i < RX_STATUS_NUM; i++)
2068			*p++ = port->rx_stats[i];
2069		for (i = 0; i < RX_CHKSUM_NUM; i++)
2070			*p++ = port->rx_csum_stats[i];
2071		*p++ = port->rx_napi_exits;
2072
2073	} while (u64_stats_fetch_retry(&port->rx_stats_syncp, start));
2074	values = p;
2075
2076	/* Racing with TX start_xmit */
2077	do {
2078		p = values;
2079		start = u64_stats_fetch_begin(&port->tx_stats_syncp);
2080
2081		for (i = 0; i < TX_MAX_FRAGS; i++) {
2082			*values++ = port->tx_frag_stats[i];
2083			port->tx_frag_stats[i] = 0;
2084		}
2085		*values++ = port->tx_frags_linearized;
2086		*values++ = port->tx_hw_csummed;
2087
2088	} while (u64_stats_fetch_retry(&port->tx_stats_syncp, start));
2089}
2090
2091static int gmac_get_ksettings(struct net_device *netdev,
2092			      struct ethtool_link_ksettings *cmd)
2093{
2094	if (!netdev->phydev)
2095		return -ENXIO;
2096	phy_ethtool_ksettings_get(netdev->phydev, cmd);
2097
2098	return 0;
2099}
2100
2101static int gmac_set_ksettings(struct net_device *netdev,
2102			      const struct ethtool_link_ksettings *cmd)
2103{
2104	if (!netdev->phydev)
2105		return -ENXIO;
2106	return phy_ethtool_ksettings_set(netdev->phydev, cmd);
2107}
2108
2109static int gmac_nway_reset(struct net_device *netdev)
2110{
2111	if (!netdev->phydev)
2112		return -ENXIO;
2113	return phy_start_aneg(netdev->phydev);
2114}
2115
2116static void gmac_get_pauseparam(struct net_device *netdev,
2117				struct ethtool_pauseparam *pparam)
2118{
2119	struct gemini_ethernet_port *port = netdev_priv(netdev);
2120	union gmac_config0 config0;
2121
2122	config0.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
2123
2124	pparam->rx_pause = config0.bits.rx_fc_en;
2125	pparam->tx_pause = config0.bits.tx_fc_en;
2126	pparam->autoneg = true;
2127}
2128
2129static int gmac_set_pauseparam(struct net_device *netdev,
2130			       struct ethtool_pauseparam *pparam)
2131{
2132	struct phy_device *phydev = netdev->phydev;
2133
2134	if (!pparam->autoneg)
2135		return -EOPNOTSUPP;
2136
2137	phy_set_asym_pause(phydev, pparam->rx_pause, pparam->tx_pause);
2138
2139	return 0;
2140}
2141
2142static void gmac_get_ringparam(struct net_device *netdev,
2143			       struct ethtool_ringparam *rp,
2144			       struct kernel_ethtool_ringparam *kernel_rp,
2145			       struct netlink_ext_ack *extack)
2146{
2147	struct gemini_ethernet_port *port = netdev_priv(netdev);
2148
2149	readl(port->gmac_base + GMAC_CONFIG0);
2150
2151	rp->rx_max_pending = 1 << 15;
2152	rp->rx_mini_max_pending = 0;
2153	rp->rx_jumbo_max_pending = 0;
2154	rp->tx_max_pending = 1 << 15;
2155
2156	rp->rx_pending = 1 << port->rxq_order;
2157	rp->rx_mini_pending = 0;
2158	rp->rx_jumbo_pending = 0;
2159	rp->tx_pending = 1 << port->txq_order;
2160}
2161
2162static int gmac_set_ringparam(struct net_device *netdev,
2163			      struct ethtool_ringparam *rp,
2164			      struct kernel_ethtool_ringparam *kernel_rp,
2165			      struct netlink_ext_ack *extack)
2166{
2167	struct gemini_ethernet_port *port = netdev_priv(netdev);
2168	int err = 0;
2169
2170	if (netif_running(netdev))
2171		return -EBUSY;
2172
2173	if (rp->rx_pending) {
2174		port->rxq_order = min(15, ilog2(rp->rx_pending - 1) + 1);
2175		err = geth_resize_freeq(port);
2176	}
2177	if (rp->tx_pending) {
2178		port->txq_order = min(15, ilog2(rp->tx_pending - 1) + 1);
2179		port->irq_every_tx_packets = 1 << (port->txq_order - 2);
2180	}
2181
2182	return err;
2183}
2184
2185static int gmac_get_coalesce(struct net_device *netdev,
2186			     struct ethtool_coalesce *ecmd,
2187			     struct kernel_ethtool_coalesce *kernel_coal,
2188			     struct netlink_ext_ack *extack)
2189{
2190	struct gemini_ethernet_port *port = netdev_priv(netdev);
2191
2192	ecmd->rx_max_coalesced_frames = 1;
2193	ecmd->tx_max_coalesced_frames = port->irq_every_tx_packets;
2194	ecmd->rx_coalesce_usecs = port->rx_coalesce_nsecs / 1000;
2195
2196	return 0;
2197}
2198
2199static int gmac_set_coalesce(struct net_device *netdev,
2200			     struct ethtool_coalesce *ecmd,
2201			     struct kernel_ethtool_coalesce *kernel_coal,
2202			     struct netlink_ext_ack *extack)
2203{
2204	struct gemini_ethernet_port *port = netdev_priv(netdev);
2205
2206	if (ecmd->tx_max_coalesced_frames < 1)
2207		return -EINVAL;
2208	if (ecmd->tx_max_coalesced_frames >= 1 << port->txq_order)
2209		return -EINVAL;
2210
2211	port->irq_every_tx_packets = ecmd->tx_max_coalesced_frames;
2212	port->rx_coalesce_nsecs = ecmd->rx_coalesce_usecs * 1000;
2213
2214	return 0;
2215}
2216
2217static u32 gmac_get_msglevel(struct net_device *netdev)
2218{
2219	struct gemini_ethernet_port *port = netdev_priv(netdev);
2220
2221	return port->msg_enable;
2222}
2223
2224static void gmac_set_msglevel(struct net_device *netdev, u32 level)
2225{
2226	struct gemini_ethernet_port *port = netdev_priv(netdev);
2227
2228	port->msg_enable = level;
2229}
2230
2231static void gmac_get_drvinfo(struct net_device *netdev,
2232			     struct ethtool_drvinfo *info)
2233{
2234	strcpy(info->driver,  DRV_NAME);
2235	strcpy(info->bus_info, netdev->dev_id ? "1" : "0");
2236}
2237
2238static const struct net_device_ops gmac_351x_ops = {
2239	.ndo_init		= gmac_init,
2240	.ndo_open		= gmac_open,
2241	.ndo_stop		= gmac_stop,
2242	.ndo_start_xmit		= gmac_start_xmit,
2243	.ndo_tx_timeout		= gmac_tx_timeout,
2244	.ndo_set_rx_mode	= gmac_set_rx_mode,
2245	.ndo_set_mac_address	= gmac_set_mac_address,
2246	.ndo_get_stats64	= gmac_get_stats64,
2247	.ndo_change_mtu		= gmac_change_mtu,
 
2248	.ndo_set_features	= gmac_set_features,
2249};
2250
2251static const struct ethtool_ops gmac_351x_ethtool_ops = {
2252	.supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS |
2253				     ETHTOOL_COALESCE_MAX_FRAMES,
2254	.get_sset_count	= gmac_get_sset_count,
2255	.get_strings	= gmac_get_strings,
2256	.get_ethtool_stats = gmac_get_ethtool_stats,
2257	.get_link	= ethtool_op_get_link,
2258	.get_link_ksettings = gmac_get_ksettings,
2259	.set_link_ksettings = gmac_set_ksettings,
2260	.nway_reset	= gmac_nway_reset,
2261	.get_pauseparam	= gmac_get_pauseparam,
2262	.set_pauseparam = gmac_set_pauseparam,
2263	.get_ringparam	= gmac_get_ringparam,
2264	.set_ringparam	= gmac_set_ringparam,
2265	.get_coalesce	= gmac_get_coalesce,
2266	.set_coalesce	= gmac_set_coalesce,
2267	.get_msglevel	= gmac_get_msglevel,
2268	.set_msglevel	= gmac_set_msglevel,
2269	.get_drvinfo	= gmac_get_drvinfo,
2270};
2271
2272static irqreturn_t gemini_port_irq_thread(int irq, void *data)
2273{
2274	unsigned long irqmask = SWFQ_EMPTY_INT_BIT;
2275	struct gemini_ethernet_port *port = data;
2276	struct gemini_ethernet *geth;
2277	unsigned long flags;
2278
2279	geth = port->geth;
2280	/* The queue is half empty so refill it */
2281	geth_fill_freeq(geth, true);
2282
2283	spin_lock_irqsave(&geth->irq_lock, flags);
2284	/* ACK queue interrupt */
2285	writel(irqmask, geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
2286	/* Enable queue interrupt again */
2287	irqmask |= readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2288	writel(irqmask, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2289	spin_unlock_irqrestore(&geth->irq_lock, flags);
2290
2291	return IRQ_HANDLED;
2292}
2293
2294static irqreturn_t gemini_port_irq(int irq, void *data)
2295{
2296	struct gemini_ethernet_port *port = data;
2297	struct gemini_ethernet *geth;
2298	irqreturn_t ret = IRQ_NONE;
2299	u32 val, en;
2300
2301	geth = port->geth;
2302	spin_lock(&geth->irq_lock);
2303
2304	val = readl(geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
2305	en = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2306
2307	if (val & en & SWFQ_EMPTY_INT_BIT) {
2308		/* Disable the queue empty interrupt while we work on
2309		 * processing the queue. Also disable overrun interrupts
2310		 * as there is not much we can do about it here.
2311		 */
2312		en &= ~(SWFQ_EMPTY_INT_BIT | GMAC0_RX_OVERRUN_INT_BIT
2313					   | GMAC1_RX_OVERRUN_INT_BIT);
2314		writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2315		ret = IRQ_WAKE_THREAD;
2316	}
2317
2318	spin_unlock(&geth->irq_lock);
2319
2320	return ret;
2321}
2322
2323static void gemini_port_remove(struct gemini_ethernet_port *port)
2324{
2325	if (port->netdev) {
2326		phy_disconnect(port->netdev->phydev);
2327		unregister_netdev(port->netdev);
2328	}
2329	clk_disable_unprepare(port->pclk);
2330	geth_cleanup_freeq(port->geth);
2331}
2332
2333static void gemini_ethernet_init(struct gemini_ethernet *geth)
2334{
2335	/* Only do this once both ports are online */
2336	if (geth->initialized)
2337		return;
2338	if (geth->port0 && geth->port1)
2339		geth->initialized = true;
2340	else
2341		return;
2342
2343	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
2344	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
2345	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_2_REG);
2346	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_3_REG);
2347	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2348
2349	/* Interrupt config:
2350	 *
2351	 *	GMAC0 intr bits ------> int0 ----> eth0
2352	 *	GMAC1 intr bits ------> int1 ----> eth1
2353	 *	TOE intr -------------> int1 ----> eth1
2354	 *	Classification Intr --> int0 ----> eth0
2355	 *	Default Q0 -----------> int0 ----> eth0
2356	 *	Default Q1 -----------> int1 ----> eth1
2357	 *	FreeQ intr -----------> int1 ----> eth1
2358	 */
2359	writel(0xCCFC0FC0, geth->base + GLOBAL_INTERRUPT_SELECT_0_REG);
2360	writel(0x00F00002, geth->base + GLOBAL_INTERRUPT_SELECT_1_REG);
2361	writel(0xFFFFFFFF, geth->base + GLOBAL_INTERRUPT_SELECT_2_REG);
2362	writel(0xFFFFFFFF, geth->base + GLOBAL_INTERRUPT_SELECT_3_REG);
2363	writel(0xFF000003, geth->base + GLOBAL_INTERRUPT_SELECT_4_REG);
2364
2365	/* edge-triggered interrupts packed to level-triggered one... */
2366	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
2367	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
2368	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_2_REG);
2369	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_3_REG);
2370	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
2371
2372	/* Set up queue */
2373	writel(0, geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
2374	writel(0, geth->base + GLOBAL_HW_FREEQ_BASE_SIZE_REG);
2375	writel(0, geth->base + GLOBAL_SWFQ_RWPTR_REG);
2376	writel(0, geth->base + GLOBAL_HWFQ_RWPTR_REG);
2377
2378	geth->freeq_frag_order = DEFAULT_RX_BUF_ORDER;
2379	/* This makes the queue resize on probe() so that we
2380	 * set up and enable the queue IRQ. FIXME: fragile.
2381	 */
2382	geth->freeq_order = 1;
2383}
2384
2385static void gemini_port_save_mac_addr(struct gemini_ethernet_port *port)
2386{
2387	port->mac_addr[0] =
2388		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD0));
2389	port->mac_addr[1] =
2390		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD1));
2391	port->mac_addr[2] =
2392		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD2));
2393}
2394
2395static int gemini_ethernet_port_probe(struct platform_device *pdev)
2396{
2397	char *port_names[2] = { "ethernet0", "ethernet1" };
2398	struct device_node *np = pdev->dev.of_node;
2399	struct gemini_ethernet_port *port;
2400	struct device *dev = &pdev->dev;
2401	struct gemini_ethernet *geth;
2402	struct net_device *netdev;
2403	struct device *parent;
2404	u8 mac[ETH_ALEN];
2405	unsigned int id;
2406	int irq;
2407	int ret;
2408
2409	parent = dev->parent;
2410	geth = dev_get_drvdata(parent);
2411
2412	if (!strcmp(dev_name(dev), "60008000.ethernet-port"))
2413		id = 0;
2414	else if (!strcmp(dev_name(dev), "6000c000.ethernet-port"))
2415		id = 1;
2416	else
2417		return -ENODEV;
2418
2419	dev_info(dev, "probe %s ID %d\n", dev_name(dev), id);
2420
2421	netdev = devm_alloc_etherdev_mqs(dev, sizeof(*port), TX_QUEUE_NUM, TX_QUEUE_NUM);
2422	if (!netdev) {
2423		dev_err(dev, "Can't allocate ethernet device #%d\n", id);
2424		return -ENOMEM;
2425	}
2426
2427	port = netdev_priv(netdev);
2428	SET_NETDEV_DEV(netdev, dev);
2429	port->netdev = netdev;
2430	port->id = id;
2431	port->geth = geth;
2432	port->dev = dev;
2433	port->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2434
2435	/* DMA memory */
2436	port->dma_base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
2437	if (IS_ERR(port->dma_base)) {
2438		dev_err(dev, "get DMA address failed\n");
2439		return PTR_ERR(port->dma_base);
2440	}
2441
2442	/* GMAC config memory */
2443	port->gmac_base = devm_platform_get_and_ioremap_resource(pdev, 1, NULL);
2444	if (IS_ERR(port->gmac_base)) {
2445		dev_err(dev, "get GMAC address failed\n");
2446		return PTR_ERR(port->gmac_base);
2447	}
2448
2449	/* Interrupt */
2450	irq = platform_get_irq(pdev, 0);
2451	if (irq < 0)
2452		return irq;
2453	port->irq = irq;
2454
2455	/* Clock the port */
2456	port->pclk = devm_clk_get(dev, "PCLK");
2457	if (IS_ERR(port->pclk)) {
2458		dev_err(dev, "no PCLK\n");
2459		return PTR_ERR(port->pclk);
2460	}
2461	ret = clk_prepare_enable(port->pclk);
2462	if (ret)
2463		return ret;
2464
2465	/* Maybe there is a nice ethernet address we should use */
2466	gemini_port_save_mac_addr(port);
2467
2468	/* Reset the port */
2469	port->reset = devm_reset_control_get_exclusive(dev, NULL);
2470	if (IS_ERR(port->reset)) {
2471		dev_err(dev, "no reset\n");
2472		ret = PTR_ERR(port->reset);
2473		goto unprepare;
2474	}
2475	reset_control_reset(port->reset);
2476	usleep_range(100, 500);
2477
2478	/* Assign pointer in the main state container */
2479	if (!id)
2480		geth->port0 = port;
2481	else
2482		geth->port1 = port;
2483
2484	/* This will just be done once both ports are up and reset */
2485	gemini_ethernet_init(geth);
2486
2487	platform_set_drvdata(pdev, port);
2488
2489	/* Set up and register the netdev */
2490	netdev->dev_id = port->id;
2491	netdev->irq = irq;
2492	netdev->netdev_ops = &gmac_351x_ops;
2493	netdev->ethtool_ops = &gmac_351x_ethtool_ops;
2494
2495	spin_lock_init(&port->config_lock);
2496	gmac_clear_hw_stats(netdev);
2497
2498	netdev->hw_features = GMAC_OFFLOAD_FEATURES;
2499	netdev->features |= GMAC_OFFLOAD_FEATURES | NETIF_F_GRO;
2500	/* We can receive jumbo frames up to 10236 bytes but only
2501	 * transmit 2047 bytes so, let's accept payloads of 2047
2502	 * bytes minus VLAN and ethernet header
2503	 */
2504	netdev->min_mtu = ETH_MIN_MTU;
2505	netdev->max_mtu = MTU_SIZE_BIT_MASK - VLAN_ETH_HLEN;
2506
2507	port->freeq_refill = 0;
2508	netif_napi_add(netdev, &port->napi, gmac_napi_poll);
2509
2510	ret = of_get_mac_address(np, mac);
2511	if (!ret) {
2512		dev_info(dev, "Setting macaddr from DT %pM\n", mac);
2513		memcpy(port->mac_addr, mac, ETH_ALEN);
2514	}
2515
2516	if (is_valid_ether_addr((void *)port->mac_addr)) {
2517		eth_hw_addr_set(netdev, (u8 *)port->mac_addr);
2518	} else {
2519		dev_dbg(dev, "ethernet address 0x%08x%08x%08x invalid\n",
2520			port->mac_addr[0], port->mac_addr[1],
2521			port->mac_addr[2]);
2522		dev_info(dev, "using a random ethernet address\n");
2523		eth_hw_addr_random(netdev);
2524	}
2525	gmac_write_mac_address(netdev);
2526
2527	ret = devm_request_threaded_irq(port->dev,
2528					port->irq,
2529					gemini_port_irq,
2530					gemini_port_irq_thread,
2531					IRQF_SHARED,
2532					port_names[port->id],
2533					port);
2534	if (ret)
2535		goto unprepare;
2536
2537	ret = gmac_setup_phy(netdev);
2538	if (ret) {
2539		netdev_err(netdev,
2540			   "PHY init failed\n");
2541		goto unprepare;
2542	}
2543
2544	ret = register_netdev(netdev);
2545	if (ret)
2546		goto unprepare;
2547
2548	return 0;
2549
2550unprepare:
2551	clk_disable_unprepare(port->pclk);
2552	return ret;
2553}
2554
2555static void gemini_ethernet_port_remove(struct platform_device *pdev)
2556{
2557	struct gemini_ethernet_port *port = platform_get_drvdata(pdev);
2558
2559	gemini_port_remove(port);
 
 
2560}
2561
2562static const struct of_device_id gemini_ethernet_port_of_match[] = {
2563	{
2564		.compatible = "cortina,gemini-ethernet-port",
2565	},
2566	{},
2567};
2568MODULE_DEVICE_TABLE(of, gemini_ethernet_port_of_match);
2569
2570static struct platform_driver gemini_ethernet_port_driver = {
2571	.driver = {
2572		.name = "gemini-ethernet-port",
2573		.of_match_table = gemini_ethernet_port_of_match,
2574	},
2575	.probe = gemini_ethernet_port_probe,
2576	.remove = gemini_ethernet_port_remove,
2577};
2578
2579static int gemini_ethernet_probe(struct platform_device *pdev)
2580{
2581	struct device *dev = &pdev->dev;
2582	struct gemini_ethernet *geth;
2583	unsigned int retry = 5;
2584	u32 val;
2585
2586	/* Global registers */
2587	geth = devm_kzalloc(dev, sizeof(*geth), GFP_KERNEL);
2588	if (!geth)
2589		return -ENOMEM;
2590	geth->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
2591	if (IS_ERR(geth->base))
2592		return PTR_ERR(geth->base);
2593	geth->dev = dev;
2594
2595	/* Wait for ports to stabilize */
2596	do {
2597		udelay(2);
2598		val = readl(geth->base + GLOBAL_TOE_VERSION_REG);
2599		barrier();
2600	} while (!val && --retry);
2601	if (!retry) {
2602		dev_err(dev, "failed to reset ethernet\n");
2603		return -EIO;
2604	}
2605	dev_info(dev, "Ethernet device ID: 0x%03x, revision 0x%01x\n",
2606		 (val >> 4) & 0xFFFU, val & 0xFU);
2607
2608	spin_lock_init(&geth->irq_lock);
2609	spin_lock_init(&geth->freeq_lock);
2610
2611	/* The children will use this */
2612	platform_set_drvdata(pdev, geth);
2613
2614	/* Spawn child devices for the two ports */
2615	return devm_of_platform_populate(dev);
2616}
2617
2618static void gemini_ethernet_remove(struct platform_device *pdev)
2619{
2620	struct gemini_ethernet *geth = platform_get_drvdata(pdev);
2621
2622	geth_cleanup_freeq(geth);
2623	geth->initialized = false;
 
 
2624}
2625
2626static const struct of_device_id gemini_ethernet_of_match[] = {
2627	{
2628		.compatible = "cortina,gemini-ethernet",
2629	},
2630	{},
2631};
2632MODULE_DEVICE_TABLE(of, gemini_ethernet_of_match);
2633
2634static struct platform_driver gemini_ethernet_driver = {
2635	.driver = {
2636		.name = DRV_NAME,
2637		.of_match_table = gemini_ethernet_of_match,
2638	},
2639	.probe = gemini_ethernet_probe,
2640	.remove = gemini_ethernet_remove,
2641};
2642
2643static int __init gemini_ethernet_module_init(void)
2644{
2645	int ret;
2646
2647	ret = platform_driver_register(&gemini_ethernet_port_driver);
2648	if (ret)
2649		return ret;
2650
2651	ret = platform_driver_register(&gemini_ethernet_driver);
2652	if (ret) {
2653		platform_driver_unregister(&gemini_ethernet_port_driver);
2654		return ret;
2655	}
2656
2657	return 0;
2658}
2659module_init(gemini_ethernet_module_init);
2660
2661static void __exit gemini_ethernet_module_exit(void)
2662{
2663	platform_driver_unregister(&gemini_ethernet_driver);
2664	platform_driver_unregister(&gemini_ethernet_port_driver);
2665}
2666module_exit(gemini_ethernet_module_exit);
2667
2668MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
2669MODULE_DESCRIPTION("StorLink SL351x (Gemini) ethernet driver");
2670MODULE_LICENSE("GPL");
2671MODULE_ALIAS("platform:" DRV_NAME);