1// SPDX-License-Identifier: GPL-2.0
   2/* Ethernet device driver for Cortina Systems Gemini SoC
   3 * Also known as the StorLink SL3512 and SL3516 (SL351x) or Lepus
   4 * Net Engine and Gigabit Ethernet MAC (GMAC)
   5 * This hardware contains a TCP Offload Engine (TOE) but currently the
   6 * driver does not make use of it.
   7 *
   8 * Authors:
   9 * Linus Walleij <linus.walleij@linaro.org>
  10 * Tobias Waldvogel <tobias.waldvogel@gmail.com> (OpenWRT)
  11 * Michał Mirosław <mirq-linux@rere.qmqm.pl>
  12 * Paulius Zaleckas <paulius.zaleckas@gmail.com>
  13 * Giuseppe De Robertis <Giuseppe.DeRobertis@ba.infn.it>
  14 * Gary Chen & Ch Hsu Storlink Semiconductor
  15 */
  16#include <linux/kernel.h>
  17#include <linux/init.h>
  18#include <linux/module.h>
  19#include <linux/platform_device.h>
  20#include <linux/spinlock.h>
  21#include <linux/slab.h>
  22#include <linux/dma-mapping.h>
  23#include <linux/cache.h>
  24#include <linux/interrupt.h>
  25#include <linux/reset.h>
  26#include <linux/clk.h>
  27#include <linux/of.h>
  28#include <linux/of_mdio.h>
  29#include <linux/of_net.h>
  30#include <linux/of_platform.h>
  31#include <linux/etherdevice.h>
  32#include <linux/if_vlan.h>
  33#include <linux/skbuff.h>
  34#include <linux/phy.h>
  35#include <linux/crc32.h>
  36#include <linux/ethtool.h>
  37#include <linux/tcp.h>
  38#include <linux/u64_stats_sync.h>
  39
  40#include <linux/in.h>
  41#include <linux/ip.h>
  42#include <linux/ipv6.h>
  43
  44#include "gemini.h"
  45
  46#define DRV_NAME		"gmac-gemini"
  47#define DRV_VERSION		"1.0"
  48
  49#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  50static int debug = -1;
  51module_param(debug, int, 0);
  52MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  53
  54#define HSIZE_8			0x00
  55#define HSIZE_16		0x01
  56#define HSIZE_32		0x02
  57
  58#define HBURST_SINGLE		0x00
  59#define HBURST_INCR		0x01
  60#define HBURST_INCR4		0x02
  61#define HBURST_INCR8		0x03
  62
  63#define HPROT_DATA_CACHE	BIT(0)
  64#define HPROT_PRIVILIGED	BIT(1)
  65#define HPROT_BUFFERABLE	BIT(2)
  66#define HPROT_CACHABLE		BIT(3)
  67
  68#define DEFAULT_RX_COALESCE_NSECS	0
  69#define DEFAULT_GMAC_RXQ_ORDER		9
  70#define DEFAULT_GMAC_TXQ_ORDER		8
  71#define DEFAULT_RX_BUF_ORDER		11
  72#define DEFAULT_NAPI_WEIGHT		64
  73#define TX_MAX_FRAGS			16
  74#define TX_QUEUE_NUM			1	/* max: 6 */
  75#define RX_MAX_ALLOC_ORDER		2
  76
  77#define GMAC0_IRQ0_2 (GMAC0_TXDERR_INT_BIT | GMAC0_TXPERR_INT_BIT | \
  78		      GMAC0_RXDERR_INT_BIT | GMAC0_RXPERR_INT_BIT)
  79#define GMAC0_IRQ0_TXQ0_INTS (GMAC0_SWTQ00_EOF_INT_BIT | \
  80			      GMAC0_SWTQ00_FIN_INT_BIT)
  81#define GMAC0_IRQ4_8 (GMAC0_MIB_INT_BIT | GMAC0_RX_OVERRUN_INT_BIT)
  82
  83#define GMAC_OFFLOAD_FEATURES (NETIF_F_SG | NETIF_F_IP_CSUM | \
  84		NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | \
  85		NETIF_F_TSO | NETIF_F_TSO_ECN | NETIF_F_TSO6)
  86
  87/**
  88 * struct gmac_queue_page - page buffer per-page info
 
 
  89 */
  90struct gmac_queue_page {
  91	struct page *page;
  92	dma_addr_t mapping;
  93};
  94
  95struct gmac_txq {
  96	struct gmac_txdesc *ring;
  97	struct sk_buff	**skb;
  98	unsigned int	cptr;
  99	unsigned int	noirq_packets;
 100};
 101
 102struct gemini_ethernet;
 103
 104struct gemini_ethernet_port {
 105	u8 id; /* 0 or 1 */
 106
 107	struct gemini_ethernet *geth;
 108	struct net_device *netdev;
 109	struct device *dev;
 110	void __iomem *dma_base;
 111	void __iomem *gmac_base;
 112	struct clk *pclk;
 113	struct reset_control *reset;
 114	int irq;
 115	__le32 mac_addr[3];
 116
 117	void __iomem		*rxq_rwptr;
 118	struct gmac_rxdesc	*rxq_ring;
 119	unsigned int		rxq_order;
 120
 121	struct napi_struct	napi;
 122	struct hrtimer		rx_coalesce_timer;
 123	unsigned int		rx_coalesce_nsecs;
 124	unsigned int		freeq_refill;
 125	struct gmac_txq		txq[TX_QUEUE_NUM];
 126	unsigned int		txq_order;
 127	unsigned int		irq_every_tx_packets;
 128
 129	dma_addr_t		rxq_dma_base;
 130	dma_addr_t		txq_dma_base;
 131
 132	unsigned int		msg_enable;
 133	spinlock_t		config_lock; /* Locks config register */
 134
 135	struct u64_stats_sync	tx_stats_syncp;
 136	struct u64_stats_sync	rx_stats_syncp;
 137	struct u64_stats_sync	ir_stats_syncp;
 138
 139	struct rtnl_link_stats64 stats;
 140	u64			hw_stats[RX_STATS_NUM];
 141	u64			rx_stats[RX_STATUS_NUM];
 142	u64			rx_csum_stats[RX_CHKSUM_NUM];
 143	u64			rx_napi_exits;
 144	u64			tx_frag_stats[TX_MAX_FRAGS];
 145	u64			tx_frags_linearized;
 146	u64			tx_hw_csummed;
 147};
 148
 149struct gemini_ethernet {
 150	struct device *dev;
 151	void __iomem *base;
 152	struct gemini_ethernet_port *port0;
 153	struct gemini_ethernet_port *port1;
 154	bool initialized;
 155
 156	spinlock_t	irq_lock; /* Locks IRQ-related registers */
 157	unsigned int	freeq_order;
 158	unsigned int	freeq_frag_order;
 159	struct gmac_rxdesc *freeq_ring;
 160	dma_addr_t	freeq_dma_base;
 161	struct gmac_queue_page	*freeq_pages;
 162	unsigned int	num_freeq_pages;
 163	spinlock_t	freeq_lock; /* Locks queue from reentrance */
 164};
 165
 166#define GMAC_STATS_NUM	( \
 167	RX_STATS_NUM + RX_STATUS_NUM + RX_CHKSUM_NUM + 1 + \
 168	TX_MAX_FRAGS + 2)
 169
 170static const char gmac_stats_strings[GMAC_STATS_NUM][ETH_GSTRING_LEN] = {
 171	"GMAC_IN_DISCARDS",
 172	"GMAC_IN_ERRORS",
 173	"GMAC_IN_MCAST",
 174	"GMAC_IN_BCAST",
 175	"GMAC_IN_MAC1",
 176	"GMAC_IN_MAC2",
 177	"RX_STATUS_GOOD_FRAME",
 178	"RX_STATUS_TOO_LONG_GOOD_CRC",
 179	"RX_STATUS_RUNT_FRAME",
 180	"RX_STATUS_SFD_NOT_FOUND",
 181	"RX_STATUS_CRC_ERROR",
 182	"RX_STATUS_TOO_LONG_BAD_CRC",
 183	"RX_STATUS_ALIGNMENT_ERROR",
 184	"RX_STATUS_TOO_LONG_BAD_ALIGN",
 185	"RX_STATUS_RX_ERR",
 186	"RX_STATUS_DA_FILTERED",
 187	"RX_STATUS_BUFFER_FULL",
 188	"RX_STATUS_11",
 189	"RX_STATUS_12",
 190	"RX_STATUS_13",
 191	"RX_STATUS_14",
 192	"RX_STATUS_15",
 193	"RX_CHKSUM_IP_UDP_TCP_OK",
 194	"RX_CHKSUM_IP_OK_ONLY",
 195	"RX_CHKSUM_NONE",
 196	"RX_CHKSUM_3",
 197	"RX_CHKSUM_IP_ERR_UNKNOWN",
 198	"RX_CHKSUM_IP_ERR",
 199	"RX_CHKSUM_TCP_UDP_ERR",
 200	"RX_CHKSUM_7",
 201	"RX_NAPI_EXITS",
 202	"TX_FRAGS[1]",
 203	"TX_FRAGS[2]",
 204	"TX_FRAGS[3]",
 205	"TX_FRAGS[4]",
 206	"TX_FRAGS[5]",
 207	"TX_FRAGS[6]",
 208	"TX_FRAGS[7]",
 209	"TX_FRAGS[8]",
 210	"TX_FRAGS[9]",
 211	"TX_FRAGS[10]",
 212	"TX_FRAGS[11]",
 213	"TX_FRAGS[12]",
 214	"TX_FRAGS[13]",
 215	"TX_FRAGS[14]",
 216	"TX_FRAGS[15]",
 217	"TX_FRAGS[16+]",
 218	"TX_FRAGS_LINEARIZED",
 219	"TX_HW_CSUMMED",
 220};
 221
 222static void gmac_dump_dma_state(struct net_device *netdev);
 223
 224static void gmac_update_config0_reg(struct net_device *netdev,
 225				    u32 val, u32 vmask)
 226{
 227	struct gemini_ethernet_port *port = netdev_priv(netdev);
 228	unsigned long flags;
 229	u32 reg;
 230
 231	spin_lock_irqsave(&port->config_lock, flags);
 232
 233	reg = readl(port->gmac_base + GMAC_CONFIG0);
 234	reg = (reg & ~vmask) | val;
 235	writel(reg, port->gmac_base + GMAC_CONFIG0);
 236
 237	spin_unlock_irqrestore(&port->config_lock, flags);
 238}
 239
 240static void gmac_enable_tx_rx(struct net_device *netdev)
 241{
 242	struct gemini_ethernet_port *port = netdev_priv(netdev);
 243	unsigned long flags;
 244	u32 reg;
 245
 246	spin_lock_irqsave(&port->config_lock, flags);
 247
 248	reg = readl(port->gmac_base + GMAC_CONFIG0);
 249	reg &= ~CONFIG0_TX_RX_DISABLE;
 250	writel(reg, port->gmac_base + GMAC_CONFIG0);
 251
 252	spin_unlock_irqrestore(&port->config_lock, flags);
 253}
 254
 255static void gmac_disable_tx_rx(struct net_device *netdev)
 256{
 257	struct gemini_ethernet_port *port = netdev_priv(netdev);
 258	unsigned long flags;
 259	u32 val;
 260
 261	spin_lock_irqsave(&port->config_lock, flags);
 262
 263	val = readl(port->gmac_base + GMAC_CONFIG0);
 264	val |= CONFIG0_TX_RX_DISABLE;
 265	writel(val, port->gmac_base + GMAC_CONFIG0);
 266
 267	spin_unlock_irqrestore(&port->config_lock, flags);
 268
 269	mdelay(10);	/* let GMAC consume packet */
 270}
 271
 272static void gmac_set_flow_control(struct net_device *netdev, bool tx, bool rx)
 273{
 274	struct gemini_ethernet_port *port = netdev_priv(netdev);
 275	unsigned long flags;
 276	u32 val;
 277
 278	spin_lock_irqsave(&port->config_lock, flags);
 279
 280	val = readl(port->gmac_base + GMAC_CONFIG0);
 281	val &= ~CONFIG0_FLOW_CTL;
 282	if (tx)
 283		val |= CONFIG0_FLOW_TX;
 284	if (rx)
 285		val |= CONFIG0_FLOW_RX;
 286	writel(val, port->gmac_base + GMAC_CONFIG0);
 287
 288	spin_unlock_irqrestore(&port->config_lock, flags);
 289}
 290
 291static void gmac_speed_set(struct net_device *netdev)
 292{
 293	struct gemini_ethernet_port *port = netdev_priv(netdev);
 294	struct phy_device *phydev = netdev->phydev;
 295	union gmac_status status, old_status;
 296	int pause_tx = 0;
 297	int pause_rx = 0;
 298
 299	status.bits32 = readl(port->gmac_base + GMAC_STATUS);
 300	old_status.bits32 = status.bits32;
 301	status.bits.link = phydev->link;
 302	status.bits.duplex = phydev->duplex;
 303
 304	switch (phydev->speed) {
 305	case 1000:
 306		status.bits.speed = GMAC_SPEED_1000;
 307		if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
 308			status.bits.mii_rmii = GMAC_PHY_RGMII_1000;
 309		netdev_dbg(netdev, "connect %s to RGMII @ 1Gbit\n",
 310			   phydev_name(phydev));
 311		break;
 312	case 100:
 313		status.bits.speed = GMAC_SPEED_100;
 314		if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
 315			status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
 316		netdev_dbg(netdev, "connect %s to RGMII @ 100 Mbit\n",
 317			   phydev_name(phydev));
 318		break;
 319	case 10:
 320		status.bits.speed = GMAC_SPEED_10;
 321		if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
 322			status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
 323		netdev_dbg(netdev, "connect %s to RGMII @ 10 Mbit\n",
 324			   phydev_name(phydev));
 325		break;
 326	default:
 327		netdev_warn(netdev, "Unsupported PHY speed (%d) on %s\n",
 328			    phydev->speed, phydev_name(phydev));
 329	}
 330
 331	if (phydev->duplex == DUPLEX_FULL) {
 332		u16 lcladv = phy_read(phydev, MII_ADVERTISE);
 333		u16 rmtadv = phy_read(phydev, MII_LPA);
 334		u8 cap = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
 335
 336		if (cap & FLOW_CTRL_RX)
 337			pause_rx = 1;
 338		if (cap & FLOW_CTRL_TX)
 339			pause_tx = 1;
 340	}
 341
 342	gmac_set_flow_control(netdev, pause_tx, pause_rx);
 343
 344	if (old_status.bits32 == status.bits32)
 345		return;
 346
 347	if (netif_msg_link(port)) {
 348		phy_print_status(phydev);
 349		netdev_info(netdev, "link flow control: %s\n",
 350			    phydev->pause
 351			    ? (phydev->asym_pause ? "tx" : "both")
 352			    : (phydev->asym_pause ? "rx" : "none")
 353		);
 354	}
 355
 356	gmac_disable_tx_rx(netdev);
 357	writel(status.bits32, port->gmac_base + GMAC_STATUS);
 358	gmac_enable_tx_rx(netdev);
 359}
 360
 361static int gmac_setup_phy(struct net_device *netdev)
 362{
 363	struct gemini_ethernet_port *port = netdev_priv(netdev);
 364	union gmac_status status = { .bits32 = 0 };
 365	struct device *dev = port->dev;
 366	struct phy_device *phy;
 367
 368	phy = of_phy_get_and_connect(netdev,
 369				     dev->of_node,
 370				     gmac_speed_set);
 371	if (!phy)
 372		return -ENODEV;
 373	netdev->phydev = phy;
 374
 375	phy_set_max_speed(phy, SPEED_1000);
 376	phy_support_asym_pause(phy);
 377
 378	/* set PHY interface type */
 379	switch (phy->interface) {
 380	case PHY_INTERFACE_MODE_MII:
 381		netdev_dbg(netdev,
 382			   "MII: set GMAC0 to GMII mode, GMAC1 disabled\n");
 383		status.bits.mii_rmii = GMAC_PHY_MII;
 384		break;
 385	case PHY_INTERFACE_MODE_GMII:
 386		netdev_dbg(netdev,
 387			   "GMII: set GMAC0 to GMII mode, GMAC1 disabled\n");
 388		status.bits.mii_rmii = GMAC_PHY_GMII;
 389		break;
 390	case PHY_INTERFACE_MODE_RGMII:
 
 
 
 391		netdev_dbg(netdev,
 392			   "RGMII: set GMAC0 and GMAC1 to MII/RGMII mode\n");
 393		status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
 394		break;
 395	default:
 396		netdev_err(netdev, "Unsupported MII interface\n");
 397		phy_disconnect(phy);
 398		netdev->phydev = NULL;
 399		return -EINVAL;
 400	}
 401	writel(status.bits32, port->gmac_base + GMAC_STATUS);
 402
 403	if (netif_msg_link(port))
 404		phy_attached_info(phy);
 405
 406	return 0;
 407}
 408
 409/* The maximum frame length is not logically enumerated in the
 410 * hardware, so we do a table lookup to find the applicable max
 411 * frame length.
 412 */
 413struct gmac_max_framelen {
 414	unsigned int max_l3_len;
 415	u8 val;
 416};
 417
 418static const struct gmac_max_framelen gmac_maxlens[] = {
 419	{
 420		.max_l3_len = 1518,
 421		.val = CONFIG0_MAXLEN_1518,
 422	},
 423	{
 424		.max_l3_len = 1522,
 425		.val = CONFIG0_MAXLEN_1522,
 426	},
 427	{
 428		.max_l3_len = 1536,
 429		.val = CONFIG0_MAXLEN_1536,
 430	},
 431	{
 432		.max_l3_len = 1542,
 433		.val = CONFIG0_MAXLEN_1542,
 434	},
 435	{
 436		.max_l3_len = 9212,
 437		.val = CONFIG0_MAXLEN_9k,
 438	},
 439	{
 440		.max_l3_len = 10236,
 441		.val = CONFIG0_MAXLEN_10k,
 442	},
 443};
 444
 445static int gmac_pick_rx_max_len(unsigned int max_l3_len)
 446{
 447	const struct gmac_max_framelen *maxlen;
 448	int maxtot;
 449	int i;
 450
 451	maxtot = max_l3_len + ETH_HLEN + VLAN_HLEN;
 452
 453	for (i = 0; i < ARRAY_SIZE(gmac_maxlens); i++) {
 454		maxlen = &gmac_maxlens[i];
 455		if (maxtot <= maxlen->max_l3_len)
 456			return maxlen->val;
 457	}
 458
 459	return -1;
 460}
 461
 462static int gmac_init(struct net_device *netdev)
 463{
 464	struct gemini_ethernet_port *port = netdev_priv(netdev);
 465	union gmac_config0 config0 = { .bits = {
 466		.dis_tx = 1,
 467		.dis_rx = 1,
 468		.ipv4_rx_chksum = 1,
 469		.ipv6_rx_chksum = 1,
 470		.rx_err_detect = 1,
 471		.rgmm_edge = 1,
 472		.port0_chk_hwq = 1,
 473		.port1_chk_hwq = 1,
 474		.port0_chk_toeq = 1,
 475		.port1_chk_toeq = 1,
 476		.port0_chk_classq = 1,
 477		.port1_chk_classq = 1,
 478	} };
 479	union gmac_ahb_weight ahb_weight = { .bits = {
 480		.rx_weight = 1,
 481		.tx_weight = 1,
 482		.hash_weight = 1,
 483		.pre_req = 0x1f,
 484		.tq_dv_threshold = 0,
 485	} };
 486	union gmac_tx_wcr0 hw_weigh = { .bits = {
 487		.hw_tq3 = 1,
 488		.hw_tq2 = 1,
 489		.hw_tq1 = 1,
 490		.hw_tq0 = 1,
 491	} };
 492	union gmac_tx_wcr1 sw_weigh = { .bits = {
 493		.sw_tq5 = 1,
 494		.sw_tq4 = 1,
 495		.sw_tq3 = 1,
 496		.sw_tq2 = 1,
 497		.sw_tq1 = 1,
 498		.sw_tq0 = 1,
 499	} };
 500	union gmac_config1 config1 = { .bits = {
 501		.set_threshold = 16,
 502		.rel_threshold = 24,
 503	} };
 504	union gmac_config2 config2 = { .bits = {
 505		.set_threshold = 16,
 506		.rel_threshold = 32,
 507	} };
 508	union gmac_config3 config3 = { .bits = {
 509		.set_threshold = 0,
 510		.rel_threshold = 0,
 511	} };
 512	union gmac_config0 tmp;
 513	u32 val;
 514
 515	config0.bits.max_len = gmac_pick_rx_max_len(netdev->mtu);
 516	tmp.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
 517	config0.bits.reserved = tmp.bits.reserved;
 518	writel(config0.bits32, port->gmac_base + GMAC_CONFIG0);
 519	writel(config1.bits32, port->gmac_base + GMAC_CONFIG1);
 520	writel(config2.bits32, port->gmac_base + GMAC_CONFIG2);
 521	writel(config3.bits32, port->gmac_base + GMAC_CONFIG3);
 522
 523	val = readl(port->dma_base + GMAC_AHB_WEIGHT_REG);
 524	writel(ahb_weight.bits32, port->dma_base + GMAC_AHB_WEIGHT_REG);
 525
 526	writel(hw_weigh.bits32,
 527	       port->dma_base + GMAC_TX_WEIGHTING_CTRL_0_REG);
 528	writel(sw_weigh.bits32,
 529	       port->dma_base + GMAC_TX_WEIGHTING_CTRL_1_REG);
 530
 531	port->rxq_order = DEFAULT_GMAC_RXQ_ORDER;
 532	port->txq_order = DEFAULT_GMAC_TXQ_ORDER;
 533	port->rx_coalesce_nsecs = DEFAULT_RX_COALESCE_NSECS;
 534
 535	/* Mark every quarter of the queue a packet for interrupt
 536	 * in order to be able to wake up the queue if it was stopped
 537	 */
 538	port->irq_every_tx_packets = 1 << (port->txq_order - 2);
 539
 540	return 0;
 541}
 542
 543static void gmac_uninit(struct net_device *netdev)
 544{
 545	if (netdev->phydev)
 546		phy_disconnect(netdev->phydev);
 547}
 548
 549static int gmac_setup_txqs(struct net_device *netdev)
 550{
 551	struct gemini_ethernet_port *port = netdev_priv(netdev);
 552	unsigned int n_txq = netdev->num_tx_queues;
 553	struct gemini_ethernet *geth = port->geth;
 554	size_t entries = 1 << port->txq_order;
 555	struct gmac_txq *txq = port->txq;
 556	struct gmac_txdesc *desc_ring;
 557	size_t len = n_txq * entries;
 558	struct sk_buff **skb_tab;
 559	void __iomem *rwptr_reg;
 560	unsigned int r;
 561	int i;
 562
 563	rwptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
 564
 565	skb_tab = kcalloc(len, sizeof(*skb_tab), GFP_KERNEL);
 566	if (!skb_tab)
 567		return -ENOMEM;
 568
 569	desc_ring = dma_alloc_coherent(geth->dev, len * sizeof(*desc_ring),
 570				       &port->txq_dma_base, GFP_KERNEL);
 571
 572	if (!desc_ring) {
 573		kfree(skb_tab);
 574		return -ENOMEM;
 575	}
 576
 577	if (port->txq_dma_base & ~DMA_Q_BASE_MASK) {
 578		dev_warn(geth->dev, "TX queue base is not aligned\n");
 
 
 579		kfree(skb_tab);
 580		return -ENOMEM;
 581	}
 582
 583	writel(port->txq_dma_base | port->txq_order,
 584	       port->dma_base + GMAC_SW_TX_QUEUE_BASE_REG);
 585
 586	for (i = 0; i < n_txq; i++) {
 587		txq->ring = desc_ring;
 588		txq->skb = skb_tab;
 589		txq->noirq_packets = 0;
 590
 591		r = readw(rwptr_reg);
 592		rwptr_reg += 2;
 593		writew(r, rwptr_reg);
 594		rwptr_reg += 2;
 595		txq->cptr = r;
 596
 597		txq++;
 598		desc_ring += entries;
 599		skb_tab += entries;
 600	}
 601
 602	return 0;
 603}
 604
 605static void gmac_clean_txq(struct net_device *netdev, struct gmac_txq *txq,
 606			   unsigned int r)
 607{
 608	struct gemini_ethernet_port *port = netdev_priv(netdev);
 609	unsigned int m = (1 << port->txq_order) - 1;
 610	struct gemini_ethernet *geth = port->geth;
 611	unsigned int c = txq->cptr;
 612	union gmac_txdesc_0 word0;
 613	union gmac_txdesc_1 word1;
 614	unsigned int hwchksum = 0;
 615	unsigned long bytes = 0;
 616	struct gmac_txdesc *txd;
 617	unsigned short nfrags;
 618	unsigned int errs = 0;
 619	unsigned int pkts = 0;
 620	unsigned int word3;
 621	dma_addr_t mapping;
 622
 623	if (c == r)
 624		return;
 625
 626	while (c != r) {
 627		txd = txq->ring + c;
 628		word0 = txd->word0;
 629		word1 = txd->word1;
 630		mapping = txd->word2.buf_adr;
 631		word3 = txd->word3.bits32;
 632
 633		dma_unmap_single(geth->dev, mapping,
 634				 word0.bits.buffer_size, DMA_TO_DEVICE);
 635
 636		if (word3 & EOF_BIT)
 637			dev_kfree_skb(txq->skb[c]);
 638
 639		c++;
 640		c &= m;
 641
 642		if (!(word3 & SOF_BIT))
 643			continue;
 644
 645		if (!word0.bits.status_tx_ok) {
 646			errs++;
 647			continue;
 648		}
 649
 650		pkts++;
 651		bytes += txd->word1.bits.byte_count;
 652
 653		if (word1.bits32 & TSS_CHECKUM_ENABLE)
 654			hwchksum++;
 655
 656		nfrags = word0.bits.desc_count - 1;
 657		if (nfrags) {
 658			if (nfrags >= TX_MAX_FRAGS)
 659				nfrags = TX_MAX_FRAGS - 1;
 660
 661			u64_stats_update_begin(&port->tx_stats_syncp);
 662			port->tx_frag_stats[nfrags]++;
 663			u64_stats_update_end(&port->tx_stats_syncp);
 664		}
 665	}
 666
 667	u64_stats_update_begin(&port->ir_stats_syncp);
 668	port->stats.tx_errors += errs;
 669	port->stats.tx_packets += pkts;
 670	port->stats.tx_bytes += bytes;
 671	port->tx_hw_csummed += hwchksum;
 672	u64_stats_update_end(&port->ir_stats_syncp);
 673
 674	txq->cptr = c;
 675}
 676
 677static void gmac_cleanup_txqs(struct net_device *netdev)
 678{
 679	struct gemini_ethernet_port *port = netdev_priv(netdev);
 680	unsigned int n_txq = netdev->num_tx_queues;
 681	struct gemini_ethernet *geth = port->geth;
 682	void __iomem *rwptr_reg;
 683	unsigned int r, i;
 684
 685	rwptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
 686
 687	for (i = 0; i < n_txq; i++) {
 688		r = readw(rwptr_reg);
 689		rwptr_reg += 2;
 690		writew(r, rwptr_reg);
 691		rwptr_reg += 2;
 692
 693		gmac_clean_txq(netdev, port->txq + i, r);
 694	}
 695	writel(0, port->dma_base + GMAC_SW_TX_QUEUE_BASE_REG);
 696
 697	kfree(port->txq->skb);
 698	dma_free_coherent(geth->dev,
 699			  n_txq * sizeof(*port->txq->ring) << port->txq_order,
 700			  port->txq->ring, port->txq_dma_base);
 701}
 702
 703static int gmac_setup_rxq(struct net_device *netdev)
 704{
 705	struct gemini_ethernet_port *port = netdev_priv(netdev);
 706	struct gemini_ethernet *geth = port->geth;
 707	struct nontoe_qhdr __iomem *qhdr;
 708
 709	qhdr = geth->base + TOE_DEFAULT_Q_HDR_BASE(netdev->dev_id);
 710	port->rxq_rwptr = &qhdr->word1;
 711
 712	/* Remap a slew of memory to use for the RX queue */
 713	port->rxq_ring = dma_alloc_coherent(geth->dev,
 714				sizeof(*port->rxq_ring) << port->rxq_order,
 715				&port->rxq_dma_base, GFP_KERNEL);
 716	if (!port->rxq_ring)
 717		return -ENOMEM;
 718	if (port->rxq_dma_base & ~NONTOE_QHDR0_BASE_MASK) {
 719		dev_warn(geth->dev, "RX queue base is not aligned\n");
 720		return -ENOMEM;
 721	}
 722
 723	writel(port->rxq_dma_base | port->rxq_order, &qhdr->word0);
 724	writel(0, port->rxq_rwptr);
 725	return 0;
 726}
 727
 728static struct gmac_queue_page *
 729gmac_get_queue_page(struct gemini_ethernet *geth,
 730		    struct gemini_ethernet_port *port,
 731		    dma_addr_t addr)
 732{
 733	struct gmac_queue_page *gpage;
 734	dma_addr_t mapping;
 735	int i;
 736
 737	/* Only look for even pages */
 738	mapping = addr & PAGE_MASK;
 739
 740	if (!geth->freeq_pages) {
 741		dev_err(geth->dev, "try to get page with no page list\n");
 742		return NULL;
 743	}
 744
 745	/* Look up a ring buffer page from virtual mapping */
 746	for (i = 0; i < geth->num_freeq_pages; i++) {
 747		gpage = &geth->freeq_pages[i];
 748		if (gpage->mapping == mapping)
 749			return gpage;
 750	}
 751
 752	return NULL;
 753}
 754
 755static void gmac_cleanup_rxq(struct net_device *netdev)
 756{
 757	struct gemini_ethernet_port *port = netdev_priv(netdev);
 758	struct gemini_ethernet *geth = port->geth;
 759	struct gmac_rxdesc *rxd = port->rxq_ring;
 760	static struct gmac_queue_page *gpage;
 761	struct nontoe_qhdr __iomem *qhdr;
 762	void __iomem *dma_reg;
 763	void __iomem *ptr_reg;
 764	dma_addr_t mapping;
 765	union dma_rwptr rw;
 766	unsigned int r, w;
 767
 768	qhdr = geth->base +
 769		TOE_DEFAULT_Q_HDR_BASE(netdev->dev_id);
 770	dma_reg = &qhdr->word0;
 771	ptr_reg = &qhdr->word1;
 772
 773	rw.bits32 = readl(ptr_reg);
 774	r = rw.bits.rptr;
 775	w = rw.bits.wptr;
 776	writew(r, ptr_reg + 2);
 777
 778	writel(0, dma_reg);
 779
 780	/* Loop from read pointer to write pointer of the RX queue
 781	 * and free up all pages by the queue.
 782	 */
 783	while (r != w) {
 784		mapping = rxd[r].word2.buf_adr;
 785		r++;
 786		r &= ((1 << port->rxq_order) - 1);
 787
 788		if (!mapping)
 789			continue;
 790
 791		/* Freeq pointers are one page off */
 792		gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
 793		if (!gpage) {
 794			dev_err(geth->dev, "could not find page\n");
 795			continue;
 796		}
 797		/* Release the RX queue reference to the page */
 798		put_page(gpage->page);
 799	}
 800
 801	dma_free_coherent(geth->dev, sizeof(*port->rxq_ring) << port->rxq_order,
 802			  port->rxq_ring, port->rxq_dma_base);
 803}
 804
 805static struct page *geth_freeq_alloc_map_page(struct gemini_ethernet *geth,
 806					      int pn)
 807{
 808	struct gmac_rxdesc *freeq_entry;
 809	struct gmac_queue_page *gpage;
 810	unsigned int fpp_order;
 811	unsigned int frag_len;
 812	dma_addr_t mapping;
 813	struct page *page;
 814	int i;
 815
 816	/* First allocate and DMA map a single page */
 817	page = alloc_page(GFP_ATOMIC);
 818	if (!page)
 819		return NULL;
 820
 821	mapping = dma_map_single(geth->dev, page_address(page),
 822				 PAGE_SIZE, DMA_FROM_DEVICE);
 823	if (dma_mapping_error(geth->dev, mapping)) {
 824		put_page(page);
 825		return NULL;
 826	}
 827
 828	/* The assign the page mapping (physical address) to the buffer address
 829	 * in the hardware queue. PAGE_SHIFT on ARM is 12 (1 page is 4096 bytes,
 830	 * 4k), and the default RX frag order is 11 (fragments are up 20 2048
 831	 * bytes, 2k) so fpp_order (fragments per page order) is default 1. Thus
 832	 * each page normally needs two entries in the queue.
 833	 */
 834	frag_len = 1 << geth->freeq_frag_order; /* Usually 2048 */
 835	fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 836	freeq_entry = geth->freeq_ring + (pn << fpp_order);
 837	dev_dbg(geth->dev, "allocate page %d fragment length %d fragments per page %d, freeq entry %p\n",
 838		 pn, frag_len, (1 << fpp_order), freeq_entry);
 839	for (i = (1 << fpp_order); i > 0; i--) {
 840		freeq_entry->word2.buf_adr = mapping;
 841		freeq_entry++;
 842		mapping += frag_len;
 843	}
 844
 845	/* If the freeq entry already has a page mapped, then unmap it. */
 846	gpage = &geth->freeq_pages[pn];
 847	if (gpage->page) {
 848		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
 849		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
 850		/* This should be the last reference to the page so it gets
 851		 * released
 852		 */
 853		put_page(gpage->page);
 854	}
 855
 856	/* Then put our new mapping into the page table */
 857	dev_dbg(geth->dev, "page %d, DMA addr: %08x, page %p\n",
 858		pn, (unsigned int)mapping, page);
 859	gpage->mapping = mapping;
 860	gpage->page = page;
 861
 862	return page;
 863}
 864
 865/**
 866 * geth_fill_freeq() - Fill the freeq with empty fragments to use
 867 * @geth: the ethernet adapter
 868 * @refill: whether to reset the queue by filling in all freeq entries or
 869 * just refill it, usually the interrupt to refill the queue happens when
 870 * the queue is half empty.
 871 */
 872static unsigned int geth_fill_freeq(struct gemini_ethernet *geth, bool refill)
 873{
 874	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 875	unsigned int count = 0;
 876	unsigned int pn, epn;
 877	unsigned long flags;
 878	union dma_rwptr rw;
 879	unsigned int m_pn;
 880
 881	/* Mask for page */
 882	m_pn = (1 << (geth->freeq_order - fpp_order)) - 1;
 883
 884	spin_lock_irqsave(&geth->freeq_lock, flags);
 885
 886	rw.bits32 = readl(geth->base + GLOBAL_SWFQ_RWPTR_REG);
 887	pn = (refill ? rw.bits.wptr : rw.bits.rptr) >> fpp_order;
 888	epn = (rw.bits.rptr >> fpp_order) - 1;
 889	epn &= m_pn;
 890
 891	/* Loop over the freeq ring buffer entries */
 892	while (pn != epn) {
 893		struct gmac_queue_page *gpage;
 894		struct page *page;
 895
 896		gpage = &geth->freeq_pages[pn];
 897		page = gpage->page;
 898
 899		dev_dbg(geth->dev, "fill entry %d page ref count %d add %d refs\n",
 900			pn, page_ref_count(page), 1 << fpp_order);
 901
 902		if (page_ref_count(page) > 1) {
 903			unsigned int fl = (pn - epn) & m_pn;
 904
 905			if (fl > 64 >> fpp_order)
 906				break;
 907
 908			page = geth_freeq_alloc_map_page(geth, pn);
 909			if (!page)
 910				break;
 911		}
 912
 913		/* Add one reference per fragment in the page */
 914		page_ref_add(page, 1 << fpp_order);
 915		count += 1 << fpp_order;
 916		pn++;
 917		pn &= m_pn;
 918	}
 919
 920	writew(pn << fpp_order, geth->base + GLOBAL_SWFQ_RWPTR_REG + 2);
 921
 922	spin_unlock_irqrestore(&geth->freeq_lock, flags);
 923
 924	return count;
 925}
 926
 927static int geth_setup_freeq(struct gemini_ethernet *geth)
 928{
 929	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 930	unsigned int frag_len = 1 << geth->freeq_frag_order;
 931	unsigned int len = 1 << geth->freeq_order;
 932	unsigned int pages = len >> fpp_order;
 933	union queue_threshold qt;
 934	union dma_skb_size skbsz;
 935	unsigned int filled;
 936	unsigned int pn;
 937
 938	geth->freeq_ring = dma_alloc_coherent(geth->dev,
 939		sizeof(*geth->freeq_ring) << geth->freeq_order,
 940		&geth->freeq_dma_base, GFP_KERNEL);
 941	if (!geth->freeq_ring)
 942		return -ENOMEM;
 943	if (geth->freeq_dma_base & ~DMA_Q_BASE_MASK) {
 944		dev_warn(geth->dev, "queue ring base is not aligned\n");
 945		goto err_freeq;
 946	}
 947
 948	/* Allocate a mapping to page look-up index */
 949	geth->freeq_pages = kcalloc(pages, sizeof(*geth->freeq_pages),
 950				    GFP_KERNEL);
 951	if (!geth->freeq_pages)
 952		goto err_freeq;
 953	geth->num_freeq_pages = pages;
 954
 955	dev_info(geth->dev, "allocate %d pages for queue\n", pages);
 956	for (pn = 0; pn < pages; pn++)
 957		if (!geth_freeq_alloc_map_page(geth, pn))
 958			goto err_freeq_alloc;
 959
 960	filled = geth_fill_freeq(geth, false);
 961	if (!filled)
 962		goto err_freeq_alloc;
 963
 964	qt.bits32 = readl(geth->base + GLOBAL_QUEUE_THRESHOLD_REG);
 965	qt.bits.swfq_empty = 32;
 966	writel(qt.bits32, geth->base + GLOBAL_QUEUE_THRESHOLD_REG);
 967
 968	skbsz.bits.sw_skb_size = 1 << geth->freeq_frag_order;
 969	writel(skbsz.bits32, geth->base + GLOBAL_DMA_SKB_SIZE_REG);
 970	writel(geth->freeq_dma_base | geth->freeq_order,
 971	       geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
 972
 973	return 0;
 974
 975err_freeq_alloc:
 976	while (pn > 0) {
 977		struct gmac_queue_page *gpage;
 978		dma_addr_t mapping;
 979
 980		--pn;
 981		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
 982		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
 983		gpage = &geth->freeq_pages[pn];
 984		put_page(gpage->page);
 985	}
 986
 987	kfree(geth->freeq_pages);
 988err_freeq:
 989	dma_free_coherent(geth->dev,
 990			  sizeof(*geth->freeq_ring) << geth->freeq_order,
 991			  geth->freeq_ring, geth->freeq_dma_base);
 992	geth->freeq_ring = NULL;
 993	return -ENOMEM;
 994}
 995
 996/**
 997 * geth_cleanup_freeq() - cleanup the DMA mappings and free the queue
 998 * @geth: the Gemini global ethernet state
 999 */
1000static void geth_cleanup_freeq(struct gemini_ethernet *geth)
1001{
1002	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
1003	unsigned int frag_len = 1 << geth->freeq_frag_order;
1004	unsigned int len = 1 << geth->freeq_order;
1005	unsigned int pages = len >> fpp_order;
1006	unsigned int pn;
1007
1008	writew(readw(geth->base + GLOBAL_SWFQ_RWPTR_REG),
1009	       geth->base + GLOBAL_SWFQ_RWPTR_REG + 2);
1010	writel(0, geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
1011
1012	for (pn = 0; pn < pages; pn++) {
1013		struct gmac_queue_page *gpage;
1014		dma_addr_t mapping;
1015
1016		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
1017		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
1018
1019		gpage = &geth->freeq_pages[pn];
1020		while (page_ref_count(gpage->page) > 0)
1021			put_page(gpage->page);
1022	}
1023
1024	kfree(geth->freeq_pages);
1025
1026	dma_free_coherent(geth->dev,
1027			  sizeof(*geth->freeq_ring) << geth->freeq_order,
1028			  geth->freeq_ring, geth->freeq_dma_base);
1029}
1030
1031/**
1032 * geth_resize_freeq() - resize the software queue depth
1033 * @port: the port requesting the change
1034 *
1035 * This gets called at least once during probe() so the device queue gets
1036 * "resized" from the hardware defaults. Since both ports/net devices share
1037 * the same hardware queue, some synchronization between the ports is
1038 * needed.
1039 */
1040static int geth_resize_freeq(struct gemini_ethernet_port *port)
1041{
1042	struct gemini_ethernet *geth = port->geth;
1043	struct net_device *netdev = port->netdev;
1044	struct gemini_ethernet_port *other_port;
1045	struct net_device *other_netdev;
1046	unsigned int new_size = 0;
1047	unsigned int new_order;
1048	unsigned long flags;
1049	u32 en;
1050	int ret;
1051
1052	if (netdev->dev_id == 0)
1053		other_netdev = geth->port1->netdev;
1054	else
1055		other_netdev = geth->port0->netdev;
1056
1057	if (other_netdev && netif_running(other_netdev))
1058		return -EBUSY;
1059
1060	new_size = 1 << (port->rxq_order + 1);
1061	netdev_dbg(netdev, "port %d size: %d order %d\n",
1062		   netdev->dev_id,
1063		   new_size,
1064		   port->rxq_order);
1065	if (other_netdev) {
1066		other_port = netdev_priv(other_netdev);
1067		new_size += 1 << (other_port->rxq_order + 1);
1068		netdev_dbg(other_netdev, "port %d size: %d order %d\n",
1069			   other_netdev->dev_id,
1070			   (1 << (other_port->rxq_order + 1)),
1071			   other_port->rxq_order);
1072	}
1073
1074	new_order = min(15, ilog2(new_size - 1) + 1);
1075	dev_dbg(geth->dev, "set shared queue to size %d order %d\n",
1076		new_size, new_order);
1077	if (geth->freeq_order == new_order)
1078		return 0;
1079
1080	spin_lock_irqsave(&geth->irq_lock, flags);
1081
1082	/* Disable the software queue IRQs */
1083	en = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1084	en &= ~SWFQ_EMPTY_INT_BIT;
1085	writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1086	spin_unlock_irqrestore(&geth->irq_lock, flags);
1087
1088	/* Drop the old queue */
1089	if (geth->freeq_ring)
1090		geth_cleanup_freeq(geth);
1091
1092	/* Allocate a new queue with the desired order */
1093	geth->freeq_order = new_order;
1094	ret = geth_setup_freeq(geth);
1095
1096	/* Restart the interrupts - NOTE if this is the first resize
1097	 * after probe(), this is where the interrupts get turned on
1098	 * in the first place.
1099	 */
1100	spin_lock_irqsave(&geth->irq_lock, flags);
1101	en |= SWFQ_EMPTY_INT_BIT;
1102	writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1103	spin_unlock_irqrestore(&geth->irq_lock, flags);
1104
1105	return ret;
1106}
1107
1108static void gmac_tx_irq_enable(struct net_device *netdev,
1109			       unsigned int txq, int en)
1110{
1111	struct gemini_ethernet_port *port = netdev_priv(netdev);
1112	struct gemini_ethernet *geth = port->geth;
1113	u32 val, mask;
1114
1115	netdev_dbg(netdev, "%s device %d\n", __func__, netdev->dev_id);
1116
1117	mask = GMAC0_IRQ0_TXQ0_INTS << (6 * netdev->dev_id + txq);
1118
1119	if (en)
1120		writel(mask, geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
1121
1122	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1123	val = en ? val | mask : val & ~mask;
1124	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1125}
1126
1127static void gmac_tx_irq(struct net_device *netdev, unsigned int txq_num)
1128{
1129	struct netdev_queue *ntxq = netdev_get_tx_queue(netdev, txq_num);
1130
1131	gmac_tx_irq_enable(netdev, txq_num, 0);
1132	netif_tx_wake_queue(ntxq);
1133}
1134
1135static int gmac_map_tx_bufs(struct net_device *netdev, struct sk_buff *skb,
1136			    struct gmac_txq *txq, unsigned short *desc)
1137{
1138	struct gemini_ethernet_port *port = netdev_priv(netdev);
1139	struct skb_shared_info *skb_si =  skb_shinfo(skb);
1140	unsigned short m = (1 << port->txq_order) - 1;
1141	short frag, last_frag = skb_si->nr_frags - 1;
1142	struct gemini_ethernet *geth = port->geth;
1143	unsigned int word1, word3, buflen;
1144	unsigned short w = *desc;
1145	struct gmac_txdesc *txd;
1146	skb_frag_t *skb_frag;
1147	dma_addr_t mapping;
1148	unsigned short mtu;
1149	void *buffer;
 
1150
1151	mtu  = ETH_HLEN;
1152	mtu += netdev->mtu;
1153	if (skb->protocol == htons(ETH_P_8021Q))
1154		mtu += VLAN_HLEN;
1155
1156	word1 = skb->len;
1157	word3 = SOF_BIT;
1158
1159	if (word1 > mtu) {
1160		word1 |= TSS_MTU_ENABLE_BIT;
1161		word3 |= mtu;
1162	}
1163
1164	if (skb->ip_summed != CHECKSUM_NONE) {
 
 
 
 
 
 
 
 
 
 
1165		int tcp = 0;
1166
 
 
 
 
 
 
1167		if (skb->protocol == htons(ETH_P_IP)) {
1168			word1 |= TSS_IP_CHKSUM_BIT;
1169			tcp = ip_hdr(skb)->protocol == IPPROTO_TCP;
1170		} else { /* IPv6 */
1171			word1 |= TSS_IPV6_ENABLE_BIT;
1172			tcp = ipv6_hdr(skb)->nexthdr == IPPROTO_TCP;
1173		}
1174
1175		word1 |= tcp ? TSS_TCP_CHKSUM_BIT : TSS_UDP_CHKSUM_BIT;
1176	}
1177
1178	frag = -1;
1179	while (frag <= last_frag) {
1180		if (frag == -1) {
1181			buffer = skb->data;
1182			buflen = skb_headlen(skb);
1183		} else {
1184			skb_frag = skb_si->frags + frag;
1185			buffer = skb_frag_address(skb_frag);
1186			buflen = skb_frag_size(skb_frag);
1187		}
1188
1189		if (frag == last_frag) {
1190			word3 |= EOF_BIT;
1191			txq->skb[w] = skb;
1192		}
1193
1194		mapping = dma_map_single(geth->dev, buffer, buflen,
1195					 DMA_TO_DEVICE);
1196		if (dma_mapping_error(geth->dev, mapping))
1197			goto map_error;
1198
1199		txd = txq->ring + w;
1200		txd->word0.bits32 = buflen;
1201		txd->word1.bits32 = word1;
1202		txd->word2.buf_adr = mapping;
1203		txd->word3.bits32 = word3;
1204
1205		word3 &= MTU_SIZE_BIT_MASK;
1206		w++;
1207		w &= m;
1208		frag++;
1209	}
1210
1211	*desc = w;
1212	return 0;
1213
1214map_error:
1215	while (w != *desc) {
1216		w--;
1217		w &= m;
1218
1219		dma_unmap_page(geth->dev, txq->ring[w].word2.buf_adr,
1220			       txq->ring[w].word0.bits.buffer_size,
1221			       DMA_TO_DEVICE);
1222	}
1223	return -ENOMEM;
1224}
1225
1226static int gmac_start_xmit(struct sk_buff *skb, struct net_device *netdev)
 
1227{
1228	struct gemini_ethernet_port *port = netdev_priv(netdev);
1229	unsigned short m = (1 << port->txq_order) - 1;
1230	struct netdev_queue *ntxq;
1231	unsigned short r, w, d;
1232	void __iomem *ptr_reg;
1233	struct gmac_txq *txq;
1234	int txq_num, nfrags;
1235	union dma_rwptr rw;
1236
1237	if (skb->len >= 0x10000)
1238		goto out_drop_free;
1239
1240	txq_num = skb_get_queue_mapping(skb);
1241	ptr_reg = port->dma_base + GMAC_SW_TX_QUEUE_PTR_REG(txq_num);
1242	txq = &port->txq[txq_num];
1243	ntxq = netdev_get_tx_queue(netdev, txq_num);
1244	nfrags = skb_shinfo(skb)->nr_frags;
1245
1246	rw.bits32 = readl(ptr_reg);
1247	r = rw.bits.rptr;
1248	w = rw.bits.wptr;
1249
1250	d = txq->cptr - w - 1;
1251	d &= m;
1252
1253	if (d < nfrags + 2) {
1254		gmac_clean_txq(netdev, txq, r);
1255		d = txq->cptr - w - 1;
1256		d &= m;
1257
1258		if (d < nfrags + 2) {
1259			netif_tx_stop_queue(ntxq);
1260
1261			d = txq->cptr + nfrags + 16;
1262			d &= m;
1263			txq->ring[d].word3.bits.eofie = 1;
1264			gmac_tx_irq_enable(netdev, txq_num, 1);
1265
1266			u64_stats_update_begin(&port->tx_stats_syncp);
1267			netdev->stats.tx_fifo_errors++;
1268			u64_stats_update_end(&port->tx_stats_syncp);
1269			return NETDEV_TX_BUSY;
1270		}
1271	}
1272
1273	if (gmac_map_tx_bufs(netdev, skb, txq, &w)) {
1274		if (skb_linearize(skb))
1275			goto out_drop;
1276
1277		u64_stats_update_begin(&port->tx_stats_syncp);
1278		port->tx_frags_linearized++;
1279		u64_stats_update_end(&port->tx_stats_syncp);
1280
1281		if (gmac_map_tx_bufs(netdev, skb, txq, &w))
1282			goto out_drop_free;
1283	}
1284
1285	writew(w, ptr_reg + 2);
1286
1287	gmac_clean_txq(netdev, txq, r);
1288	return NETDEV_TX_OK;
1289
1290out_drop_free:
1291	dev_kfree_skb(skb);
1292out_drop:
1293	u64_stats_update_begin(&port->tx_stats_syncp);
1294	port->stats.tx_dropped++;
1295	u64_stats_update_end(&port->tx_stats_syncp);
1296	return NETDEV_TX_OK;
1297}
1298
1299static void gmac_tx_timeout(struct net_device *netdev)
1300{
1301	netdev_err(netdev, "Tx timeout\n");
1302	gmac_dump_dma_state(netdev);
1303}
1304
1305static void gmac_enable_irq(struct net_device *netdev, int enable)
1306{
1307	struct gemini_ethernet_port *port = netdev_priv(netdev);
1308	struct gemini_ethernet *geth = port->geth;
1309	unsigned long flags;
1310	u32 val, mask;
1311
1312	netdev_dbg(netdev, "%s device %d %s\n", __func__,
1313		   netdev->dev_id, enable ? "enable" : "disable");
1314	spin_lock_irqsave(&geth->irq_lock, flags);
1315
1316	mask = GMAC0_IRQ0_2 << (netdev->dev_id * 2);
1317	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1318	val = enable ? (val | mask) : (val & ~mask);
1319	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1320
1321	mask = DEFAULT_Q0_INT_BIT << netdev->dev_id;
1322	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1323	val = enable ? (val | mask) : (val & ~mask);
1324	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1325
1326	mask = GMAC0_IRQ4_8 << (netdev->dev_id * 8);
1327	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1328	val = enable ? (val | mask) : (val & ~mask);
1329	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1330
1331	spin_unlock_irqrestore(&geth->irq_lock, flags);
1332}
1333
1334static void gmac_enable_rx_irq(struct net_device *netdev, int enable)
1335{
1336	struct gemini_ethernet_port *port = netdev_priv(netdev);
1337	struct gemini_ethernet *geth = port->geth;
1338	unsigned long flags;
1339	u32 val, mask;
1340
1341	netdev_dbg(netdev, "%s device %d %s\n", __func__, netdev->dev_id,
1342		   enable ? "enable" : "disable");
1343	spin_lock_irqsave(&geth->irq_lock, flags);
1344	mask = DEFAULT_Q0_INT_BIT << netdev->dev_id;
1345
1346	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1347	val = enable ? (val | mask) : (val & ~mask);
1348	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1349
1350	spin_unlock_irqrestore(&geth->irq_lock, flags);
1351}
1352
1353static struct sk_buff *gmac_skb_if_good_frame(struct gemini_ethernet_port *port,
1354					      union gmac_rxdesc_0 word0,
1355					      unsigned int frame_len)
1356{
1357	unsigned int rx_csum = word0.bits.chksum_status;
1358	unsigned int rx_status = word0.bits.status;
1359	struct sk_buff *skb = NULL;
1360
1361	port->rx_stats[rx_status]++;
1362	port->rx_csum_stats[rx_csum]++;
1363
1364	if (word0.bits.derr || word0.bits.perr ||
1365	    rx_status || frame_len < ETH_ZLEN ||
1366	    rx_csum >= RX_CHKSUM_IP_ERR_UNKNOWN) {
1367		port->stats.rx_errors++;
1368
1369		if (frame_len < ETH_ZLEN || RX_ERROR_LENGTH(rx_status))
1370			port->stats.rx_length_errors++;
1371		if (RX_ERROR_OVER(rx_status))
1372			port->stats.rx_over_errors++;
1373		if (RX_ERROR_CRC(rx_status))
1374			port->stats.rx_crc_errors++;
1375		if (RX_ERROR_FRAME(rx_status))
1376			port->stats.rx_frame_errors++;
1377		return NULL;
1378	}
1379
1380	skb = napi_get_frags(&port->napi);
1381	if (!skb)
1382		goto update_exit;
1383
1384	if (rx_csum == RX_CHKSUM_IP_UDP_TCP_OK)
1385		skb->ip_summed = CHECKSUM_UNNECESSARY;
1386
1387update_exit:
1388	port->stats.rx_bytes += frame_len;
1389	port->stats.rx_packets++;
1390	return skb;
1391}
1392
1393static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
1394{
1395	struct gemini_ethernet_port *port = netdev_priv(netdev);
1396	unsigned short m = (1 << port->rxq_order) - 1;
1397	struct gemini_ethernet *geth = port->geth;
1398	void __iomem *ptr_reg = port->rxq_rwptr;
1399	unsigned int frame_len, frag_len;
1400	struct gmac_rxdesc *rx = NULL;
1401	struct gmac_queue_page *gpage;
1402	static struct sk_buff *skb;
1403	union gmac_rxdesc_0 word0;
1404	union gmac_rxdesc_1 word1;
1405	union gmac_rxdesc_3 word3;
1406	struct page *page = NULL;
1407	unsigned int page_offs;
1408	unsigned short r, w;
1409	union dma_rwptr rw;
1410	dma_addr_t mapping;
1411	int frag_nr = 0;
1412
1413	rw.bits32 = readl(ptr_reg);
1414	/* Reset interrupt as all packages until here are taken into account */
1415	writel(DEFAULT_Q0_INT_BIT << netdev->dev_id,
1416	       geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
1417	r = rw.bits.rptr;
1418	w = rw.bits.wptr;
1419
1420	while (budget && w != r) {
1421		rx = port->rxq_ring + r;
1422		word0 = rx->word0;
1423		word1 = rx->word1;
1424		mapping = rx->word2.buf_adr;
1425		word3 = rx->word3;
1426
1427		r++;
1428		r &= m;
1429
1430		frag_len = word0.bits.buffer_size;
1431		frame_len = word1.bits.byte_count;
1432		page_offs = mapping & ~PAGE_MASK;
1433
1434		if (!mapping) {
1435			netdev_err(netdev,
1436				   "rxq[%u]: HW BUG: zero DMA desc\n", r);
1437			goto err_drop;
1438		}
1439
1440		/* Freeq pointers are one page off */
1441		gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
1442		if (!gpage) {
1443			dev_err(geth->dev, "could not find mapping\n");
1444			continue;
1445		}
1446		page = gpage->page;
1447
1448		if (word3.bits32 & SOF_BIT) {
1449			if (skb) {
1450				napi_free_frags(&port->napi);
1451				port->stats.rx_dropped++;
1452			}
1453
1454			skb = gmac_skb_if_good_frame(port, word0, frame_len);
1455			if (!skb)
1456				goto err_drop;
1457
1458			page_offs += NET_IP_ALIGN;
1459			frag_len -= NET_IP_ALIGN;
1460			frag_nr = 0;
1461
1462		} else if (!skb) {
1463			put_page(page);
1464			continue;
1465		}
1466
1467		if (word3.bits32 & EOF_BIT)
1468			frag_len = frame_len - skb->len;
1469
1470		/* append page frag to skb */
1471		if (frag_nr == MAX_SKB_FRAGS)
1472			goto err_drop;
1473
1474		if (frag_len == 0)
1475			netdev_err(netdev, "Received fragment with len = 0\n");
1476
1477		skb_fill_page_desc(skb, frag_nr, page, page_offs, frag_len);
1478		skb->len += frag_len;
1479		skb->data_len += frag_len;
1480		skb->truesize += frag_len;
1481		frag_nr++;
1482
1483		if (word3.bits32 & EOF_BIT) {
1484			napi_gro_frags(&port->napi);
1485			skb = NULL;
1486			--budget;
1487		}
1488		continue;
1489
1490err_drop:
1491		if (skb) {
1492			napi_free_frags(&port->napi);
1493			skb = NULL;
1494		}
1495
1496		if (mapping)
1497			put_page(page);
1498
1499		port->stats.rx_dropped++;
1500	}
1501
1502	writew(r, ptr_reg);
1503	return budget;
1504}
1505
1506static int gmac_napi_poll(struct napi_struct *napi, int budget)
1507{
1508	struct gemini_ethernet_port *port = netdev_priv(napi->dev);
1509	struct gemini_ethernet *geth = port->geth;
1510	unsigned int freeq_threshold;
1511	unsigned int received;
1512
1513	freeq_threshold = 1 << (geth->freeq_order - 1);
1514	u64_stats_update_begin(&port->rx_stats_syncp);
1515
1516	received = gmac_rx(napi->dev, budget);
1517	if (received < budget) {
1518		napi_gro_flush(napi, false);
1519		napi_complete_done(napi, received);
1520		gmac_enable_rx_irq(napi->dev, 1);
1521		++port->rx_napi_exits;
1522	}
1523
1524	port->freeq_refill += (budget - received);
1525	if (port->freeq_refill > freeq_threshold) {
1526		port->freeq_refill -= freeq_threshold;
1527		geth_fill_freeq(geth, true);
1528	}
1529
1530	u64_stats_update_end(&port->rx_stats_syncp);
1531	return received;
1532}
1533
1534static void gmac_dump_dma_state(struct net_device *netdev)
1535{
1536	struct gemini_ethernet_port *port = netdev_priv(netdev);
1537	struct gemini_ethernet *geth = port->geth;
1538	void __iomem *ptr_reg;
1539	u32 reg[5];
1540
1541	/* Interrupt status */
1542	reg[0] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
1543	reg[1] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
1544	reg[2] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_2_REG);
1545	reg[3] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_3_REG);
1546	reg[4] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
1547	netdev_err(netdev, "IRQ status: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1548		   reg[0], reg[1], reg[2], reg[3], reg[4]);
1549
1550	/* Interrupt enable */
1551	reg[0] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1552	reg[1] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1553	reg[2] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_2_REG);
1554	reg[3] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_3_REG);
1555	reg[4] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1556	netdev_err(netdev, "IRQ enable: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1557		   reg[0], reg[1], reg[2], reg[3], reg[4]);
1558
1559	/* RX DMA status */
1560	reg[0] = readl(port->dma_base + GMAC_DMA_RX_FIRST_DESC_REG);
1561	reg[1] = readl(port->dma_base + GMAC_DMA_RX_CURR_DESC_REG);
1562	reg[2] = GET_RPTR(port->rxq_rwptr);
1563	reg[3] = GET_WPTR(port->rxq_rwptr);
1564	netdev_err(netdev, "RX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
1565		   reg[0], reg[1], reg[2], reg[3]);
1566
1567	reg[0] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD0_REG);
1568	reg[1] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD1_REG);
1569	reg[2] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD2_REG);
1570	reg[3] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD3_REG);
1571	netdev_err(netdev, "RX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1572		   reg[0], reg[1], reg[2], reg[3]);
1573
1574	/* TX DMA status */
1575	ptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
1576
1577	reg[0] = readl(port->dma_base + GMAC_DMA_TX_FIRST_DESC_REG);
1578	reg[1] = readl(port->dma_base + GMAC_DMA_TX_CURR_DESC_REG);
1579	reg[2] = GET_RPTR(ptr_reg);
1580	reg[3] = GET_WPTR(ptr_reg);
1581	netdev_err(netdev, "TX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
1582		   reg[0], reg[1], reg[2], reg[3]);
1583
1584	reg[0] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD0_REG);
1585	reg[1] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD1_REG);
1586	reg[2] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD2_REG);
1587	reg[3] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD3_REG);
1588	netdev_err(netdev, "TX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1589		   reg[0], reg[1], reg[2], reg[3]);
1590
1591	/* FREE queues status */
1592	ptr_reg = geth->base + GLOBAL_SWFQ_RWPTR_REG;
1593
1594	reg[0] = GET_RPTR(ptr_reg);
1595	reg[1] = GET_WPTR(ptr_reg);
1596
1597	ptr_reg = geth->base + GLOBAL_HWFQ_RWPTR_REG;
1598
1599	reg[2] = GET_RPTR(ptr_reg);
1600	reg[3] = GET_WPTR(ptr_reg);
1601	netdev_err(netdev, "FQ SW ptr: %u %u, HW ptr: %u %u\n",
1602		   reg[0], reg[1], reg[2], reg[3]);
1603}
1604
1605static void gmac_update_hw_stats(struct net_device *netdev)
1606{
1607	struct gemini_ethernet_port *port = netdev_priv(netdev);
1608	unsigned int rx_discards, rx_mcast, rx_bcast;
1609	struct gemini_ethernet *geth = port->geth;
1610	unsigned long flags;
1611
1612	spin_lock_irqsave(&geth->irq_lock, flags);
1613	u64_stats_update_begin(&port->ir_stats_syncp);
1614
1615	rx_discards = readl(port->gmac_base + GMAC_IN_DISCARDS);
1616	port->hw_stats[0] += rx_discards;
1617	port->hw_stats[1] += readl(port->gmac_base + GMAC_IN_ERRORS);
1618	rx_mcast = readl(port->gmac_base + GMAC_IN_MCAST);
1619	port->hw_stats[2] += rx_mcast;
1620	rx_bcast = readl(port->gmac_base + GMAC_IN_BCAST);
1621	port->hw_stats[3] += rx_bcast;
1622	port->hw_stats[4] += readl(port->gmac_base + GMAC_IN_MAC1);
1623	port->hw_stats[5] += readl(port->gmac_base + GMAC_IN_MAC2);
1624
1625	port->stats.rx_missed_errors += rx_discards;
1626	port->stats.multicast += rx_mcast;
1627	port->stats.multicast += rx_bcast;
1628
1629	writel(GMAC0_MIB_INT_BIT << (netdev->dev_id * 8),
1630	       geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
1631
1632	u64_stats_update_end(&port->ir_stats_syncp);
1633	spin_unlock_irqrestore(&geth->irq_lock, flags);
1634}
1635
1636/**
1637 * gmac_get_intr_flags() - get interrupt status flags for a port from
1638 * @netdev: the net device for the port to get flags from
1639 * @i: the interrupt status register 0..4
1640 */
1641static u32 gmac_get_intr_flags(struct net_device *netdev, int i)
1642{
1643	struct gemini_ethernet_port *port = netdev_priv(netdev);
1644	struct gemini_ethernet *geth = port->geth;
1645	void __iomem *irqif_reg, *irqen_reg;
1646	unsigned int offs, val;
1647
1648	/* Calculate the offset using the stride of the status registers */
1649	offs = i * (GLOBAL_INTERRUPT_STATUS_1_REG -
1650		    GLOBAL_INTERRUPT_STATUS_0_REG);
1651
1652	irqif_reg = geth->base + GLOBAL_INTERRUPT_STATUS_0_REG + offs;
1653	irqen_reg = geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG + offs;
1654
1655	val = readl(irqif_reg) & readl(irqen_reg);
1656	return val;
1657}
1658
1659static enum hrtimer_restart gmac_coalesce_delay_expired(struct hrtimer *timer)
1660{
1661	struct gemini_ethernet_port *port =
1662		container_of(timer, struct gemini_ethernet_port,
1663			     rx_coalesce_timer);
1664
1665	napi_schedule(&port->napi);
1666	return HRTIMER_NORESTART;
1667}
1668
1669static irqreturn_t gmac_irq(int irq, void *data)
1670{
1671	struct gemini_ethernet_port *port;
1672	struct net_device *netdev = data;
1673	struct gemini_ethernet *geth;
1674	u32 val, orr = 0;
1675
1676	port = netdev_priv(netdev);
1677	geth = port->geth;
1678
1679	val = gmac_get_intr_flags(netdev, 0);
1680	orr |= val;
1681
1682	if (val & (GMAC0_IRQ0_2 << (netdev->dev_id * 2))) {
1683		/* Oh, crap */
1684		netdev_err(netdev, "hw failure/sw bug\n");
1685		gmac_dump_dma_state(netdev);
1686
1687		/* don't know how to recover, just reduce losses */
1688		gmac_enable_irq(netdev, 0);
1689		return IRQ_HANDLED;
1690	}
1691
1692	if (val & (GMAC0_IRQ0_TXQ0_INTS << (netdev->dev_id * 6)))
1693		gmac_tx_irq(netdev, 0);
1694
1695	val = gmac_get_intr_flags(netdev, 1);
1696	orr |= val;
1697
1698	if (val & (DEFAULT_Q0_INT_BIT << netdev->dev_id)) {
1699		gmac_enable_rx_irq(netdev, 0);
1700
1701		if (!port->rx_coalesce_nsecs) {
1702			napi_schedule(&port->napi);
1703		} else {
1704			ktime_t ktime;
1705
1706			ktime = ktime_set(0, port->rx_coalesce_nsecs);
1707			hrtimer_start(&port->rx_coalesce_timer, ktime,
1708				      HRTIMER_MODE_REL);
1709		}
1710	}
1711
1712	val = gmac_get_intr_flags(netdev, 4);
1713	orr |= val;
1714
1715	if (val & (GMAC0_MIB_INT_BIT << (netdev->dev_id * 8)))
1716		gmac_update_hw_stats(netdev);
1717
1718	if (val & (GMAC0_RX_OVERRUN_INT_BIT << (netdev->dev_id * 8))) {
1719		writel(GMAC0_RXDERR_INT_BIT << (netdev->dev_id * 8),
1720		       geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
1721
1722		spin_lock(&geth->irq_lock);
1723		u64_stats_update_begin(&port->ir_stats_syncp);
1724		++port->stats.rx_fifo_errors;
1725		u64_stats_update_end(&port->ir_stats_syncp);
1726		spin_unlock(&geth->irq_lock);
1727	}
1728
1729	return orr ? IRQ_HANDLED : IRQ_NONE;
1730}
1731
1732static void gmac_start_dma(struct gemini_ethernet_port *port)
1733{
1734	void __iomem *dma_ctrl_reg = port->dma_base + GMAC_DMA_CTRL_REG;
1735	union gmac_dma_ctrl dma_ctrl;
1736
1737	dma_ctrl.bits32 = readl(dma_ctrl_reg);
1738	dma_ctrl.bits.rd_enable = 1;
1739	dma_ctrl.bits.td_enable = 1;
1740	dma_ctrl.bits.loopback = 0;
1741	dma_ctrl.bits.drop_small_ack = 0;
1742	dma_ctrl.bits.rd_insert_bytes = NET_IP_ALIGN;
1743	dma_ctrl.bits.rd_prot = HPROT_DATA_CACHE | HPROT_PRIVILIGED;
1744	dma_ctrl.bits.rd_burst_size = HBURST_INCR8;
1745	dma_ctrl.bits.rd_bus = HSIZE_8;
1746	dma_ctrl.bits.td_prot = HPROT_DATA_CACHE;
1747	dma_ctrl.bits.td_burst_size = HBURST_INCR8;
1748	dma_ctrl.bits.td_bus = HSIZE_8;
1749
1750	writel(dma_ctrl.bits32, dma_ctrl_reg);
1751}
1752
1753static void gmac_stop_dma(struct gemini_ethernet_port *port)
1754{
1755	void __iomem *dma_ctrl_reg = port->dma_base + GMAC_DMA_CTRL_REG;
1756	union gmac_dma_ctrl dma_ctrl;
1757
1758	dma_ctrl.bits32 = readl(dma_ctrl_reg);
1759	dma_ctrl.bits.rd_enable = 0;
1760	dma_ctrl.bits.td_enable = 0;
1761	writel(dma_ctrl.bits32, dma_ctrl_reg);
1762}
1763
1764static int gmac_open(struct net_device *netdev)
1765{
1766	struct gemini_ethernet_port *port = netdev_priv(netdev);
1767	int err;
1768
1769	if (!netdev->phydev) {
1770		err = gmac_setup_phy(netdev);
1771		if (err) {
1772			netif_err(port, ifup, netdev,
1773				  "PHY init failed: %d\n", err);
1774			return err;
1775		}
1776	}
1777
1778	err = request_irq(netdev->irq, gmac_irq,
1779			  IRQF_SHARED, netdev->name, netdev);
1780	if (err) {
1781		netdev_err(netdev, "no IRQ\n");
1782		return err;
1783	}
1784
1785	netif_carrier_off(netdev);
1786	phy_start(netdev->phydev);
1787
1788	err = geth_resize_freeq(port);
1789	/* It's fine if it's just busy, the other port has set up
1790	 * the freeq in that case.
1791	 */
1792	if (err && (err != -EBUSY)) {
1793		netdev_err(netdev, "could not resize freeq\n");
1794		goto err_stop_phy;
1795	}
1796
1797	err = gmac_setup_rxq(netdev);
1798	if (err) {
1799		netdev_err(netdev, "could not setup RXQ\n");
1800		goto err_stop_phy;
1801	}
1802
1803	err = gmac_setup_txqs(netdev);
1804	if (err) {
1805		netdev_err(netdev, "could not setup TXQs\n");
1806		gmac_cleanup_rxq(netdev);
1807		goto err_stop_phy;
1808	}
1809
1810	napi_enable(&port->napi);
1811
1812	gmac_start_dma(port);
1813	gmac_enable_irq(netdev, 1);
1814	gmac_enable_tx_rx(netdev);
1815	netif_tx_start_all_queues(netdev);
1816
1817	hrtimer_init(&port->rx_coalesce_timer, CLOCK_MONOTONIC,
1818		     HRTIMER_MODE_REL);
1819	port->rx_coalesce_timer.function = &gmac_coalesce_delay_expired;
1820
1821	netdev_dbg(netdev, "opened\n");
1822
1823	return 0;
1824
1825err_stop_phy:
1826	phy_stop(netdev->phydev);
1827	free_irq(netdev->irq, netdev);
1828	return err;
1829}
1830
1831static int gmac_stop(struct net_device *netdev)
1832{
1833	struct gemini_ethernet_port *port = netdev_priv(netdev);
1834
1835	hrtimer_cancel(&port->rx_coalesce_timer);
1836	netif_tx_stop_all_queues(netdev);
1837	gmac_disable_tx_rx(netdev);
1838	gmac_stop_dma(port);
1839	napi_disable(&port->napi);
1840
1841	gmac_enable_irq(netdev, 0);
1842	gmac_cleanup_rxq(netdev);
1843	gmac_cleanup_txqs(netdev);
1844
1845	phy_stop(netdev->phydev);
1846	free_irq(netdev->irq, netdev);
1847
1848	gmac_update_hw_stats(netdev);
1849	return 0;
1850}
1851
1852static void gmac_set_rx_mode(struct net_device *netdev)
1853{
1854	struct gemini_ethernet_port *port = netdev_priv(netdev);
1855	union gmac_rx_fltr filter = { .bits = {
1856		.broadcast = 1,
1857		.multicast = 1,
1858		.unicast = 1,
1859	} };
1860	struct netdev_hw_addr *ha;
1861	unsigned int bit_nr;
1862	u32 mc_filter[2];
1863
1864	mc_filter[1] = 0;
1865	mc_filter[0] = 0;
1866
1867	if (netdev->flags & IFF_PROMISC) {
1868		filter.bits.error = 1;
1869		filter.bits.promiscuous = 1;
1870		mc_filter[1] = ~0;
1871		mc_filter[0] = ~0;
1872	} else if (netdev->flags & IFF_ALLMULTI) {
1873		mc_filter[1] = ~0;
1874		mc_filter[0] = ~0;
1875	} else {
1876		netdev_for_each_mc_addr(ha, netdev) {
1877			bit_nr = ~crc32_le(~0, ha->addr, ETH_ALEN) & 0x3f;
1878			mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 0x1f);
1879		}
1880	}
1881
1882	writel(mc_filter[0], port->gmac_base + GMAC_MCAST_FIL0);
1883	writel(mc_filter[1], port->gmac_base + GMAC_MCAST_FIL1);
1884	writel(filter.bits32, port->gmac_base + GMAC_RX_FLTR);
1885}
1886
1887static void gmac_write_mac_address(struct net_device *netdev)
1888{
1889	struct gemini_ethernet_port *port = netdev_priv(netdev);
1890	__le32 addr[3];
1891
1892	memset(addr, 0, sizeof(addr));
1893	memcpy(addr, netdev->dev_addr, ETH_ALEN);
1894
1895	writel(le32_to_cpu(addr[0]), port->gmac_base + GMAC_STA_ADD0);
1896	writel(le32_to_cpu(addr[1]), port->gmac_base + GMAC_STA_ADD1);
1897	writel(le32_to_cpu(addr[2]), port->gmac_base + GMAC_STA_ADD2);
1898}
1899
1900static int gmac_set_mac_address(struct net_device *netdev, void *addr)
1901{
1902	struct sockaddr *sa = addr;
1903
1904	memcpy(netdev->dev_addr, sa->sa_data, ETH_ALEN);
1905	gmac_write_mac_address(netdev);
1906
1907	return 0;
1908}
1909
1910static void gmac_clear_hw_stats(struct net_device *netdev)
1911{
1912	struct gemini_ethernet_port *port = netdev_priv(netdev);
1913
1914	readl(port->gmac_base + GMAC_IN_DISCARDS);
1915	readl(port->gmac_base + GMAC_IN_ERRORS);
1916	readl(port->gmac_base + GMAC_IN_MCAST);
1917	readl(port->gmac_base + GMAC_IN_BCAST);
1918	readl(port->gmac_base + GMAC_IN_MAC1);
1919	readl(port->gmac_base + GMAC_IN_MAC2);
1920}
1921
1922static void gmac_get_stats64(struct net_device *netdev,
1923			     struct rtnl_link_stats64 *stats)
1924{
1925	struct gemini_ethernet_port *port = netdev_priv(netdev);
1926	unsigned int start;
1927
1928	gmac_update_hw_stats(netdev);
1929
1930	/* Racing with RX NAPI */
1931	do {
1932		start = u64_stats_fetch_begin(&port->rx_stats_syncp);
1933
1934		stats->rx_packets = port->stats.rx_packets;
1935		stats->rx_bytes = port->stats.rx_bytes;
1936		stats->rx_errors = port->stats.rx_errors;
1937		stats->rx_dropped = port->stats.rx_dropped;
1938
1939		stats->rx_length_errors = port->stats.rx_length_errors;
1940		stats->rx_over_errors = port->stats.rx_over_errors;
1941		stats->rx_crc_errors = port->stats.rx_crc_errors;
1942		stats->rx_frame_errors = port->stats.rx_frame_errors;
1943
1944	} while (u64_stats_fetch_retry(&port->rx_stats_syncp, start));
1945
1946	/* Racing with MIB and TX completion interrupts */
1947	do {
1948		start = u64_stats_fetch_begin(&port->ir_stats_syncp);
1949
1950		stats->tx_errors = port->stats.tx_errors;
1951		stats->tx_packets = port->stats.tx_packets;
1952		stats->tx_bytes = port->stats.tx_bytes;
1953
1954		stats->multicast = port->stats.multicast;
1955		stats->rx_missed_errors = port->stats.rx_missed_errors;
1956		stats->rx_fifo_errors = port->stats.rx_fifo_errors;
1957
1958	} while (u64_stats_fetch_retry(&port->ir_stats_syncp, start));
1959
1960	/* Racing with hard_start_xmit */
1961	do {
1962		start = u64_stats_fetch_begin(&port->tx_stats_syncp);
1963
1964		stats->tx_dropped = port->stats.tx_dropped;
1965
1966	} while (u64_stats_fetch_retry(&port->tx_stats_syncp, start));
1967
1968	stats->rx_dropped += stats->rx_missed_errors;
1969}
1970
1971static int gmac_change_mtu(struct net_device *netdev, int new_mtu)
1972{
1973	int max_len = gmac_pick_rx_max_len(new_mtu);
1974
1975	if (max_len < 0)
1976		return -EINVAL;
1977
1978	gmac_disable_tx_rx(netdev);
1979
1980	netdev->mtu = new_mtu;
1981	gmac_update_config0_reg(netdev, max_len << CONFIG0_MAXLEN_SHIFT,
1982				CONFIG0_MAXLEN_MASK);
1983
1984	netdev_update_features(netdev);
1985
1986	gmac_enable_tx_rx(netdev);
1987
1988	return 0;
1989}
1990
1991static netdev_features_t gmac_fix_features(struct net_device *netdev,
1992					   netdev_features_t features)
1993{
1994	if (netdev->mtu + ETH_HLEN + VLAN_HLEN > MTU_SIZE_BIT_MASK)
1995		features &= ~GMAC_OFFLOAD_FEATURES;
1996
1997	return features;
1998}
1999
2000static int gmac_set_features(struct net_device *netdev,
2001			     netdev_features_t features)
2002{
2003	struct gemini_ethernet_port *port = netdev_priv(netdev);
2004	int enable = features & NETIF_F_RXCSUM;
2005	unsigned long flags;
2006	u32 reg;
2007
2008	spin_lock_irqsave(&port->config_lock, flags);
2009
2010	reg = readl(port->gmac_base + GMAC_CONFIG0);
2011	reg = enable ? reg | CONFIG0_RX_CHKSUM : reg & ~CONFIG0_RX_CHKSUM;
2012	writel(reg, port->gmac_base + GMAC_CONFIG0);
2013
2014	spin_unlock_irqrestore(&port->config_lock, flags);
2015	return 0;
2016}
2017
2018static int gmac_get_sset_count(struct net_device *netdev, int sset)
2019{
2020	return sset == ETH_SS_STATS ? GMAC_STATS_NUM : 0;
2021}
2022
2023static void gmac_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
2024{
2025	if (stringset != ETH_SS_STATS)
2026		return;
2027
2028	memcpy(data, gmac_stats_strings, sizeof(gmac_stats_strings));
2029}
2030
2031static void gmac_get_ethtool_stats(struct net_device *netdev,
2032				   struct ethtool_stats *estats, u64 *values)
2033{
2034	struct gemini_ethernet_port *port = netdev_priv(netdev);
2035	unsigned int start;
2036	u64 *p;
2037	int i;
2038
2039	gmac_update_hw_stats(netdev);
2040
2041	/* Racing with MIB interrupt */
2042	do {
2043		p = values;
2044		start = u64_stats_fetch_begin(&port->ir_stats_syncp);
2045
2046		for (i = 0; i < RX_STATS_NUM; i++)
2047			*p++ = port->hw_stats[i];
2048
2049	} while (u64_stats_fetch_retry(&port->ir_stats_syncp, start));
2050	values = p;
2051
2052	/* Racing with RX NAPI */
2053	do {
2054		p = values;
2055		start = u64_stats_fetch_begin(&port->rx_stats_syncp);
2056
2057		for (i = 0; i < RX_STATUS_NUM; i++)
2058			*p++ = port->rx_stats[i];
2059		for (i = 0; i < RX_CHKSUM_NUM; i++)
2060			*p++ = port->rx_csum_stats[i];
2061		*p++ = port->rx_napi_exits;
2062
2063	} while (u64_stats_fetch_retry(&port->rx_stats_syncp, start));
2064	values = p;
2065
2066	/* Racing with TX start_xmit */
2067	do {
2068		p = values;
2069		start = u64_stats_fetch_begin(&port->tx_stats_syncp);
2070
2071		for (i = 0; i < TX_MAX_FRAGS; i++) {
2072			*values++ = port->tx_frag_stats[i];
2073			port->tx_frag_stats[i] = 0;
2074		}
2075		*values++ = port->tx_frags_linearized;
2076		*values++ = port->tx_hw_csummed;
2077
2078	} while (u64_stats_fetch_retry(&port->tx_stats_syncp, start));
2079}
2080
2081static int gmac_get_ksettings(struct net_device *netdev,
2082			      struct ethtool_link_ksettings *cmd)
2083{
2084	if (!netdev->phydev)
2085		return -ENXIO;
2086	phy_ethtool_ksettings_get(netdev->phydev, cmd);
2087
2088	return 0;
2089}
2090
2091static int gmac_set_ksettings(struct net_device *netdev,
2092			      const struct ethtool_link_ksettings *cmd)
2093{
2094	if (!netdev->phydev)
2095		return -ENXIO;
2096	return phy_ethtool_ksettings_set(netdev->phydev, cmd);
2097}
2098
2099static int gmac_nway_reset(struct net_device *netdev)
2100{
2101	if (!netdev->phydev)
2102		return -ENXIO;
2103	return phy_start_aneg(netdev->phydev);
2104}
2105
2106static void gmac_get_pauseparam(struct net_device *netdev,
2107				struct ethtool_pauseparam *pparam)
2108{
2109	struct gemini_ethernet_port *port = netdev_priv(netdev);
2110	union gmac_config0 config0;
2111
2112	config0.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
2113
2114	pparam->rx_pause = config0.bits.rx_fc_en;
2115	pparam->tx_pause = config0.bits.tx_fc_en;
2116	pparam->autoneg = true;
2117}
2118
2119static void gmac_get_ringparam(struct net_device *netdev,
2120			       struct ethtool_ringparam *rp)
 
 
2121{
2122	struct gemini_ethernet_port *port = netdev_priv(netdev);
2123	union gmac_config0 config0;
2124
2125	config0.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
2126
2127	rp->rx_max_pending = 1 << 15;
2128	rp->rx_mini_max_pending = 0;
2129	rp->rx_jumbo_max_pending = 0;
2130	rp->tx_max_pending = 1 << 15;
2131
2132	rp->rx_pending = 1 << port->rxq_order;
2133	rp->rx_mini_pending = 0;
2134	rp->rx_jumbo_pending = 0;
2135	rp->tx_pending = 1 << port->txq_order;
2136}
2137
2138static int gmac_set_ringparam(struct net_device *netdev,
2139			      struct ethtool_ringparam *rp)
 
 
2140{
2141	struct gemini_ethernet_port *port = netdev_priv(netdev);
2142	int err = 0;
2143
2144	if (netif_running(netdev))
2145		return -EBUSY;
2146
2147	if (rp->rx_pending) {
2148		port->rxq_order = min(15, ilog2(rp->rx_pending - 1) + 1);
2149		err = geth_resize_freeq(port);
2150	}
2151	if (rp->tx_pending) {
2152		port->txq_order = min(15, ilog2(rp->tx_pending - 1) + 1);
2153		port->irq_every_tx_packets = 1 << (port->txq_order - 2);
2154	}
2155
2156	return err;
2157}
2158
2159static int gmac_get_coalesce(struct net_device *netdev,
2160			     struct ethtool_coalesce *ecmd)
 
 
2161{
2162	struct gemini_ethernet_port *port = netdev_priv(netdev);
2163
2164	ecmd->rx_max_coalesced_frames = 1;
2165	ecmd->tx_max_coalesced_frames = port->irq_every_tx_packets;
2166	ecmd->rx_coalesce_usecs = port->rx_coalesce_nsecs / 1000;
2167
2168	return 0;
2169}
2170
2171static int gmac_set_coalesce(struct net_device *netdev,
2172			     struct ethtool_coalesce *ecmd)
 
 
2173{
2174	struct gemini_ethernet_port *port = netdev_priv(netdev);
2175
2176	if (ecmd->tx_max_coalesced_frames < 1)
2177		return -EINVAL;
2178	if (ecmd->tx_max_coalesced_frames >= 1 << port->txq_order)
2179		return -EINVAL;
2180
2181	port->irq_every_tx_packets = ecmd->tx_max_coalesced_frames;
2182	port->rx_coalesce_nsecs = ecmd->rx_coalesce_usecs * 1000;
2183
2184	return 0;
2185}
2186
2187static u32 gmac_get_msglevel(struct net_device *netdev)
2188{
2189	struct gemini_ethernet_port *port = netdev_priv(netdev);
2190
2191	return port->msg_enable;
2192}
2193
2194static void gmac_set_msglevel(struct net_device *netdev, u32 level)
2195{
2196	struct gemini_ethernet_port *port = netdev_priv(netdev);
2197
2198	port->msg_enable = level;
2199}
2200
2201static void gmac_get_drvinfo(struct net_device *netdev,
2202			     struct ethtool_drvinfo *info)
2203{
2204	strcpy(info->driver,  DRV_NAME);
2205	strcpy(info->version, DRV_VERSION);
2206	strcpy(info->bus_info, netdev->dev_id ? "1" : "0");
2207}
2208
2209static const struct net_device_ops gmac_351x_ops = {
2210	.ndo_init		= gmac_init,
2211	.ndo_uninit		= gmac_uninit,
2212	.ndo_open		= gmac_open,
2213	.ndo_stop		= gmac_stop,
2214	.ndo_start_xmit		= gmac_start_xmit,
2215	.ndo_tx_timeout		= gmac_tx_timeout,
2216	.ndo_set_rx_mode	= gmac_set_rx_mode,
2217	.ndo_set_mac_address	= gmac_set_mac_address,
2218	.ndo_get_stats64	= gmac_get_stats64,
2219	.ndo_change_mtu		= gmac_change_mtu,
2220	.ndo_fix_features	= gmac_fix_features,
2221	.ndo_set_features	= gmac_set_features,
2222};
2223
2224static const struct ethtool_ops gmac_351x_ethtool_ops = {
 
 
2225	.get_sset_count	= gmac_get_sset_count,
2226	.get_strings	= gmac_get_strings,
2227	.get_ethtool_stats = gmac_get_ethtool_stats,
2228	.get_link	= ethtool_op_get_link,
2229	.get_link_ksettings = gmac_get_ksettings,
2230	.set_link_ksettings = gmac_set_ksettings,
2231	.nway_reset	= gmac_nway_reset,
2232	.get_pauseparam	= gmac_get_pauseparam,
2233	.get_ringparam	= gmac_get_ringparam,
2234	.set_ringparam	= gmac_set_ringparam,
2235	.get_coalesce	= gmac_get_coalesce,
2236	.set_coalesce	= gmac_set_coalesce,
2237	.get_msglevel	= gmac_get_msglevel,
2238	.set_msglevel	= gmac_set_msglevel,
2239	.get_drvinfo	= gmac_get_drvinfo,
2240};
2241
2242static irqreturn_t gemini_port_irq_thread(int irq, void *data)
2243{
2244	unsigned long irqmask = SWFQ_EMPTY_INT_BIT;
2245	struct gemini_ethernet_port *port = data;
2246	struct gemini_ethernet *geth;
2247	unsigned long flags;
2248
2249	geth = port->geth;
2250	/* The queue is half empty so refill it */
2251	geth_fill_freeq(geth, true);
2252
2253	spin_lock_irqsave(&geth->irq_lock, flags);
2254	/* ACK queue interrupt */
2255	writel(irqmask, geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
2256	/* Enable queue interrupt again */
2257	irqmask |= readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2258	writel(irqmask, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2259	spin_unlock_irqrestore(&geth->irq_lock, flags);
2260
2261	return IRQ_HANDLED;
2262}
2263
2264static irqreturn_t gemini_port_irq(int irq, void *data)
2265{
2266	struct gemini_ethernet_port *port = data;
2267	struct gemini_ethernet *geth;
2268	irqreturn_t ret = IRQ_NONE;
2269	u32 val, en;
2270
2271	geth = port->geth;
2272	spin_lock(&geth->irq_lock);
2273
2274	val = readl(geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
2275	en = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2276
2277	if (val & en & SWFQ_EMPTY_INT_BIT) {
2278		/* Disable the queue empty interrupt while we work on
2279		 * processing the queue. Also disable overrun interrupts
2280		 * as there is not much we can do about it here.
2281		 */
2282		en &= ~(SWFQ_EMPTY_INT_BIT | GMAC0_RX_OVERRUN_INT_BIT
2283					   | GMAC1_RX_OVERRUN_INT_BIT);
2284		writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2285		ret = IRQ_WAKE_THREAD;
2286	}
2287
2288	spin_unlock(&geth->irq_lock);
2289
2290	return ret;
2291}
2292
2293static void gemini_port_remove(struct gemini_ethernet_port *port)
2294{
2295	if (port->netdev)
 
2296		unregister_netdev(port->netdev);
 
2297	clk_disable_unprepare(port->pclk);
2298	geth_cleanup_freeq(port->geth);
2299}
2300
2301static void gemini_ethernet_init(struct gemini_ethernet *geth)
2302{
2303	/* Only do this once both ports are online */
2304	if (geth->initialized)
2305		return;
2306	if (geth->port0 && geth->port1)
2307		geth->initialized = true;
2308	else
2309		return;
2310
2311	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
2312	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
2313	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_2_REG);
2314	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_3_REG);
2315	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2316
2317	/* Interrupt config:
2318	 *
2319	 *	GMAC0 intr bits ------> int0 ----> eth0
2320	 *	GMAC1 intr bits ------> int1 ----> eth1
2321	 *	TOE intr -------------> int1 ----> eth1
2322	 *	Classification Intr --> int0 ----> eth0
2323	 *	Default Q0 -----------> int0 ----> eth0
2324	 *	Default Q1 -----------> int1 ----> eth1
2325	 *	FreeQ intr -----------> int1 ----> eth1
2326	 */
2327	writel(0xCCFC0FC0, geth->base + GLOBAL_INTERRUPT_SELECT_0_REG);
2328	writel(0x00F00002, geth->base + GLOBAL_INTERRUPT_SELECT_1_REG);
2329	writel(0xFFFFFFFF, geth->base + GLOBAL_INTERRUPT_SELECT_2_REG);
2330	writel(0xFFFFFFFF, geth->base + GLOBAL_INTERRUPT_SELECT_3_REG);
2331	writel(0xFF000003, geth->base + GLOBAL_INTERRUPT_SELECT_4_REG);
2332
2333	/* edge-triggered interrupts packed to level-triggered one... */
2334	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
2335	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
2336	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_2_REG);
2337	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_3_REG);
2338	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
2339
2340	/* Set up queue */
2341	writel(0, geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
2342	writel(0, geth->base + GLOBAL_HW_FREEQ_BASE_SIZE_REG);
2343	writel(0, geth->base + GLOBAL_SWFQ_RWPTR_REG);
2344	writel(0, geth->base + GLOBAL_HWFQ_RWPTR_REG);
2345
2346	geth->freeq_frag_order = DEFAULT_RX_BUF_ORDER;
2347	/* This makes the queue resize on probe() so that we
2348	 * set up and enable the queue IRQ. FIXME: fragile.
2349	 */
2350	geth->freeq_order = 1;
2351}
2352
2353static void gemini_port_save_mac_addr(struct gemini_ethernet_port *port)
2354{
2355	port->mac_addr[0] =
2356		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD0));
2357	port->mac_addr[1] =
2358		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD1));
2359	port->mac_addr[2] =
2360		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD2));
2361}
2362
2363static int gemini_ethernet_port_probe(struct platform_device *pdev)
2364{
2365	char *port_names[2] = { "ethernet0", "ethernet1" };
 
2366	struct gemini_ethernet_port *port;
2367	struct device *dev = &pdev->dev;
2368	struct gemini_ethernet *geth;
2369	struct net_device *netdev;
2370	struct resource *gmacres;
2371	struct resource *dmares;
2372	struct device *parent;
 
2373	unsigned int id;
2374	int irq;
2375	int ret;
2376
2377	parent = dev->parent;
2378	geth = dev_get_drvdata(parent);
2379
2380	if (!strcmp(dev_name(dev), "60008000.ethernet-port"))
2381		id = 0;
2382	else if (!strcmp(dev_name(dev), "6000c000.ethernet-port"))
2383		id = 1;
2384	else
2385		return -ENODEV;
2386
2387	dev_info(dev, "probe %s ID %d\n", dev_name(dev), id);
2388
2389	netdev = alloc_etherdev_mq(sizeof(*port), TX_QUEUE_NUM);
2390	if (!netdev) {
2391		dev_err(dev, "Can't allocate ethernet device #%d\n", id);
2392		return -ENOMEM;
2393	}
2394
2395	port = netdev_priv(netdev);
2396	SET_NETDEV_DEV(netdev, dev);
2397	port->netdev = netdev;
2398	port->id = id;
2399	port->geth = geth;
2400	port->dev = dev;
2401	port->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2402
2403	/* DMA memory */
2404	dmares = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2405	if (!dmares) {
2406		dev_err(dev, "no DMA resource\n");
2407		return -ENODEV;
2408	}
2409	port->dma_base = devm_ioremap_resource(dev, dmares);
2410	if (IS_ERR(port->dma_base))
2411		return PTR_ERR(port->dma_base);
 
2412
2413	/* GMAC config memory */
2414	gmacres = platform_get_resource(pdev, IORESOURCE_MEM, 1);
2415	if (!gmacres) {
2416		dev_err(dev, "no GMAC resource\n");
2417		return -ENODEV;
2418	}
2419	port->gmac_base = devm_ioremap_resource(dev, gmacres);
2420	if (IS_ERR(port->gmac_base))
2421		return PTR_ERR(port->gmac_base);
 
2422
2423	/* Interrupt */
2424	irq = platform_get_irq(pdev, 0);
2425	if (irq <= 0)
2426		return irq ? irq : -ENODEV;
2427	port->irq = irq;
2428
2429	/* Clock the port */
2430	port->pclk = devm_clk_get(dev, "PCLK");
2431	if (IS_ERR(port->pclk)) {
2432		dev_err(dev, "no PCLK\n");
2433		return PTR_ERR(port->pclk);
2434	}
2435	ret = clk_prepare_enable(port->pclk);
2436	if (ret)
2437		return ret;
2438
2439	/* Maybe there is a nice ethernet address we should use */
2440	gemini_port_save_mac_addr(port);
2441
2442	/* Reset the port */
2443	port->reset = devm_reset_control_get_exclusive(dev, NULL);
2444	if (IS_ERR(port->reset)) {
2445		dev_err(dev, "no reset\n");
2446		return PTR_ERR(port->reset);
 
2447	}
2448	reset_control_reset(port->reset);
2449	usleep_range(100, 500);
2450
2451	/* Assign pointer in the main state container */
2452	if (!id)
2453		geth->port0 = port;
2454	else
2455		geth->port1 = port;
2456
2457	/* This will just be done once both ports are up and reset */
2458	gemini_ethernet_init(geth);
2459
2460	platform_set_drvdata(pdev, port);
2461
2462	/* Set up and register the netdev */
2463	netdev->dev_id = port->id;
2464	netdev->irq = irq;
2465	netdev->netdev_ops = &gmac_351x_ops;
2466	netdev->ethtool_ops = &gmac_351x_ethtool_ops;
2467
2468	spin_lock_init(&port->config_lock);
2469	gmac_clear_hw_stats(netdev);
2470
2471	netdev->hw_features = GMAC_OFFLOAD_FEATURES;
2472	netdev->features |= GMAC_OFFLOAD_FEATURES | NETIF_F_GRO;
2473	/* We can handle jumbo frames up to 10236 bytes so, let's accept
2474	 * payloads of 10236 bytes minus VLAN and ethernet header
 
2475	 */
2476	netdev->min_mtu = ETH_MIN_MTU;
2477	netdev->max_mtu = 10236 - VLAN_ETH_HLEN;
2478
2479	port->freeq_refill = 0;
2480	netif_napi_add(netdev, &port->napi, gmac_napi_poll,
2481		       DEFAULT_NAPI_WEIGHT);
 
 
 
 
 
2482
2483	if (is_valid_ether_addr((void *)port->mac_addr)) {
2484		memcpy(netdev->dev_addr, port->mac_addr, ETH_ALEN);
2485	} else {
2486		dev_dbg(dev, "ethernet address 0x%08x%08x%08x invalid\n",
2487			port->mac_addr[0], port->mac_addr[1],
2488			port->mac_addr[2]);
2489		dev_info(dev, "using a random ethernet address\n");
2490		eth_random_addr(netdev->dev_addr);
2491	}
2492	gmac_write_mac_address(netdev);
2493
2494	ret = devm_request_threaded_irq(port->dev,
2495					port->irq,
2496					gemini_port_irq,
2497					gemini_port_irq_thread,
2498					IRQF_SHARED,
2499					port_names[port->id],
2500					port);
2501	if (ret)
2502		return ret;
2503
2504	ret = register_netdev(netdev);
2505	if (!ret) {
2506		netdev_info(netdev,
2507			    "irq %d, DMA @ 0x%pap, GMAC @ 0x%pap\n",
2508			    port->irq, &dmares->start,
2509			    &gmacres->start);
2510		ret = gmac_setup_phy(netdev);
2511		if (ret)
2512			netdev_info(netdev,
2513				    "PHY init failed, deferring to ifup time\n");
2514		return 0;
2515	}
2516
2517	port->netdev = NULL;
2518	free_netdev(netdev);
 
 
 
 
 
 
2519	return ret;
2520}
2521
2522static int gemini_ethernet_port_remove(struct platform_device *pdev)
2523{
2524	struct gemini_ethernet_port *port = platform_get_drvdata(pdev);
2525
2526	gemini_port_remove(port);
2527	free_netdev(port->netdev);
2528	return 0;
2529}
2530
2531static const struct of_device_id gemini_ethernet_port_of_match[] = {
2532	{
2533		.compatible = "cortina,gemini-ethernet-port",
2534	},
2535	{},
2536};
2537MODULE_DEVICE_TABLE(of, gemini_ethernet_port_of_match);
2538
2539static struct platform_driver gemini_ethernet_port_driver = {
2540	.driver = {
2541		.name = "gemini-ethernet-port",
2542		.of_match_table = of_match_ptr(gemini_ethernet_port_of_match),
2543	},
2544	.probe = gemini_ethernet_port_probe,
2545	.remove = gemini_ethernet_port_remove,
2546};
2547
2548static int gemini_ethernet_probe(struct platform_device *pdev)
2549{
2550	struct device *dev = &pdev->dev;
2551	struct gemini_ethernet *geth;
2552	unsigned int retry = 5;
2553	struct resource *res;
2554	u32 val;
2555
2556	/* Global registers */
2557	geth = devm_kzalloc(dev, sizeof(*geth), GFP_KERNEL);
2558	if (!geth)
2559		return -ENOMEM;
2560	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2561	if (!res)
2562		return -ENODEV;
2563	geth->base = devm_ioremap_resource(dev, res);
2564	if (IS_ERR(geth->base))
2565		return PTR_ERR(geth->base);
2566	geth->dev = dev;
2567
2568	/* Wait for ports to stabilize */
2569	do {
2570		udelay(2);
2571		val = readl(geth->base + GLOBAL_TOE_VERSION_REG);
2572		barrier();
2573	} while (!val && --retry);
2574	if (!retry) {
2575		dev_err(dev, "failed to reset ethernet\n");
2576		return -EIO;
2577	}
2578	dev_info(dev, "Ethernet device ID: 0x%03x, revision 0x%01x\n",
2579		 (val >> 4) & 0xFFFU, val & 0xFU);
2580
2581	spin_lock_init(&geth->irq_lock);
2582	spin_lock_init(&geth->freeq_lock);
2583
2584	/* The children will use this */
2585	platform_set_drvdata(pdev, geth);
2586
2587	/* Spawn child devices for the two ports */
2588	return devm_of_platform_populate(dev);
2589}
2590
2591static int gemini_ethernet_remove(struct platform_device *pdev)
2592{
2593	struct gemini_ethernet *geth = platform_get_drvdata(pdev);
2594
2595	geth_cleanup_freeq(geth);
2596	geth->initialized = false;
2597
2598	return 0;
2599}
2600
2601static const struct of_device_id gemini_ethernet_of_match[] = {
2602	{
2603		.compatible = "cortina,gemini-ethernet",
2604	},
2605	{},
2606};
2607MODULE_DEVICE_TABLE(of, gemini_ethernet_of_match);
2608
2609static struct platform_driver gemini_ethernet_driver = {
2610	.driver = {
2611		.name = DRV_NAME,
2612		.of_match_table = of_match_ptr(gemini_ethernet_of_match),
2613	},
2614	.probe = gemini_ethernet_probe,
2615	.remove = gemini_ethernet_remove,
2616};
2617
2618static int __init gemini_ethernet_module_init(void)
2619{
2620	int ret;
2621
2622	ret = platform_driver_register(&gemini_ethernet_port_driver);
2623	if (ret)
2624		return ret;
2625
2626	ret = platform_driver_register(&gemini_ethernet_driver);
2627	if (ret) {
2628		platform_driver_unregister(&gemini_ethernet_port_driver);
2629		return ret;
2630	}
2631
2632	return 0;
2633}
2634module_init(gemini_ethernet_module_init);
2635
2636static void __exit gemini_ethernet_module_exit(void)
2637{
2638	platform_driver_unregister(&gemini_ethernet_driver);
2639	platform_driver_unregister(&gemini_ethernet_port_driver);
2640}
2641module_exit(gemini_ethernet_module_exit);
2642
2643MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
2644MODULE_DESCRIPTION("StorLink SL351x (Gemini) ethernet driver");
2645MODULE_LICENSE("GPL");
2646MODULE_ALIAS("platform:" DRV_NAME);