1// SPDX-License-Identifier: GPL-2.0
   2/* Texas Instruments K3 AM65 Ethernet Switch SubSystem Driver
   3 *
   4 * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/
   5 *
   6 */
   7
   8#include <linux/clk.h>
   9#include <linux/etherdevice.h>
  10#include <linux/if_vlan.h>
  11#include <linux/interrupt.h>
  12#include <linux/irqdomain.h>
  13#include <linux/kernel.h>
  14#include <linux/kmemleak.h>
  15#include <linux/module.h>
  16#include <linux/netdevice.h>
  17#include <linux/net_tstamp.h>
  18#include <linux/of.h>
  19#include <linux/of_mdio.h>
  20#include <linux/of_net.h>
  21#include <linux/of_device.h>
  22#include <linux/of_platform.h>
  23#include <linux/phylink.h>
  24#include <linux/phy/phy.h>
  25#include <linux/platform_device.h>
  26#include <linux/pm_runtime.h>
  27#include <linux/regmap.h>
  28#include <linux/rtnetlink.h>
  29#include <linux/mfd/syscon.h>
  30#include <linux/sys_soc.h>
  31#include <linux/dma/ti-cppi5.h>
  32#include <linux/dma/k3-udma-glue.h>
  33#include <net/switchdev.h>
  34
  35#include "cpsw_ale.h"
  36#include "cpsw_sl.h"
  37#include "am65-cpsw-nuss.h"
  38#include "am65-cpsw-switchdev.h"
  39#include "k3-cppi-desc-pool.h"
  40#include "am65-cpts.h"
  41
  42#define AM65_CPSW_SS_BASE	0x0
  43#define AM65_CPSW_SGMII_BASE	0x100
  44#define AM65_CPSW_XGMII_BASE	0x2100
  45#define AM65_CPSW_CPSW_NU_BASE	0x20000
  46#define AM65_CPSW_NU_PORTS_BASE	0x1000
  47#define AM65_CPSW_NU_FRAM_BASE	0x12000
  48#define AM65_CPSW_NU_STATS_BASE	0x1a000
  49#define AM65_CPSW_NU_ALE_BASE	0x1e000
  50#define AM65_CPSW_NU_CPTS_BASE	0x1d000
  51
  52#define AM65_CPSW_NU_PORTS_OFFSET	0x1000
  53#define AM65_CPSW_NU_STATS_PORT_OFFSET	0x200
  54#define AM65_CPSW_NU_FRAM_PORT_OFFSET	0x200
  55
  56#define AM65_CPSW_MAX_PORTS	8
  57
  58#define AM65_CPSW_MIN_PACKET_SIZE	VLAN_ETH_ZLEN
  59#define AM65_CPSW_MAX_PACKET_SIZE	2024
  60
  61#define AM65_CPSW_REG_CTL		0x004
  62#define AM65_CPSW_REG_STAT_PORT_EN	0x014
  63#define AM65_CPSW_REG_PTYPE		0x018
  64
  65#define AM65_CPSW_P0_REG_CTL			0x004
  66#define AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET	0x008
  67
  68#define AM65_CPSW_PORT_REG_PRI_CTL		0x01c
  69#define AM65_CPSW_PORT_REG_RX_PRI_MAP		0x020
  70#define AM65_CPSW_PORT_REG_RX_MAXLEN		0x024
  71
  72#define AM65_CPSW_PORTN_REG_SA_L		0x308
  73#define AM65_CPSW_PORTN_REG_SA_H		0x30c
  74#define AM65_CPSW_PORTN_REG_TS_CTL              0x310
  75#define AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG	0x314
  76#define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG	0x318
  77#define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2       0x31C
  78
  79#define AM65_CPSW_SGMII_CONTROL_REG		0x010
  80#define AM65_CPSW_SGMII_MR_ADV_ABILITY_REG	0x018
  81#define AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE	BIT(0)
  82
  83#define AM65_CPSW_CTL_VLAN_AWARE		BIT(1)
  84#define AM65_CPSW_CTL_P0_ENABLE			BIT(2)
  85#define AM65_CPSW_CTL_P0_TX_CRC_REMOVE		BIT(13)
  86#define AM65_CPSW_CTL_P0_RX_PAD			BIT(14)
  87
  88/* AM65_CPSW_P0_REG_CTL */
  89#define AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN	BIT(0)
  90#define AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN	BIT(16)
  91
  92/* AM65_CPSW_PORT_REG_PRI_CTL */
  93#define AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN	BIT(8)
  94
  95/* AM65_CPSW_PN_TS_CTL register fields */
  96#define AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN		BIT(4)
  97#define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN	BIT(5)
  98#define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT2_EN	BIT(6)
  99#define AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN		BIT(7)
 100#define AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN		BIT(10)
 101#define AM65_CPSW_PN_TS_CTL_TX_HOST_TS_EN	BIT(11)
 102#define AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT	16
 103
 104/* AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG register fields */
 105#define AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT	16
 106
 107/* AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 */
 108#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107	BIT(16)
 109#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129	BIT(17)
 110#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130	BIT(18)
 111#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131	BIT(19)
 112#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132	BIT(20)
 113#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319	BIT(21)
 114#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320	BIT(22)
 115#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO BIT(23)
 116
 117/* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
 118#define AM65_CPSW_TS_EVENT_MSG_TYPE_BITS (BIT(0) | BIT(1) | BIT(2) | BIT(3))
 119
 120#define AM65_CPSW_TS_SEQ_ID_OFFSET (0x1e)
 121
 122#define AM65_CPSW_TS_TX_ANX_ALL_EN		\
 123	(AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN |	\
 124	 AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN |	\
 125	 AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN)
 126
 127#define AM65_CPSW_ALE_AGEOUT_DEFAULT	30
 128/* Number of TX/RX descriptors */
 129#define AM65_CPSW_MAX_TX_DESC	500
 130#define AM65_CPSW_MAX_RX_DESC	500
 131
 132#define AM65_CPSW_NAV_PS_DATA_SIZE 16
 133#define AM65_CPSW_NAV_SW_DATA_SIZE 16
 134
 135#define AM65_CPSW_DEBUG	(NETIF_MSG_HW | NETIF_MSG_DRV | NETIF_MSG_LINK | \
 136			 NETIF_MSG_IFUP	| NETIF_MSG_PROBE | NETIF_MSG_IFDOWN | \
 137			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 138
 139#define AM65_CPSW_DEFAULT_TX_CHNS	8
 140
 141static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave,
 142				      const u8 *dev_addr)
 143{
 144	u32 mac_hi = (dev_addr[0] << 0) | (dev_addr[1] << 8) |
 145		     (dev_addr[2] << 16) | (dev_addr[3] << 24);
 146	u32 mac_lo = (dev_addr[4] << 0) | (dev_addr[5] << 8);
 147
 148	writel(mac_hi, slave->port_base + AM65_CPSW_PORTN_REG_SA_H);
 149	writel(mac_lo, slave->port_base + AM65_CPSW_PORTN_REG_SA_L);
 150}
 151
 152static void am65_cpsw_sl_ctl_reset(struct am65_cpsw_port *port)
 153{
 154	cpsw_sl_reset(port->slave.mac_sl, 100);
 155	/* Max length register has to be restored after MAC SL reset */
 156	writel(AM65_CPSW_MAX_PACKET_SIZE,
 157	       port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
 158}
 159
 160static void am65_cpsw_nuss_get_ver(struct am65_cpsw_common *common)
 161{
 162	common->nuss_ver = readl(common->ss_base);
 163	common->cpsw_ver = readl(common->cpsw_base);
 164	dev_info(common->dev,
 165		 "initializing am65 cpsw nuss version 0x%08X, cpsw version 0x%08X Ports: %u quirks:%08x\n",
 166		common->nuss_ver,
 167		common->cpsw_ver,
 168		common->port_num + 1,
 169		common->pdata.quirks);
 170}
 171
 172static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev,
 173					    __be16 proto, u16 vid)
 174{
 175	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
 176	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
 177	u32 port_mask, unreg_mcast = 0;
 178	int ret;
 179
 180	if (!common->is_emac_mode)
 181		return 0;
 182
 183	if (!netif_running(ndev) || !vid)
 184		return 0;
 185
 186	ret = pm_runtime_resume_and_get(common->dev);
 187	if (ret < 0)
 188		return ret;
 189
 190	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
 191	if (!vid)
 192		unreg_mcast = port_mask;
 193	dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid);
 194	ret = cpsw_ale_vlan_add_modify(common->ale, vid, port_mask,
 195				       unreg_mcast, port_mask, 0);
 196
 197	pm_runtime_put(common->dev);
 198	return ret;
 199}
 200
 201static int am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device *ndev,
 202					     __be16 proto, u16 vid)
 203{
 204	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
 205	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
 206	int ret;
 207
 208	if (!common->is_emac_mode)
 209		return 0;
 210
 211	if (!netif_running(ndev) || !vid)
 212		return 0;
 213
 214	ret = pm_runtime_resume_and_get(common->dev);
 215	if (ret < 0)
 216		return ret;
 217
 218	dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid);
 219	ret = cpsw_ale_del_vlan(common->ale, vid,
 220				BIT(port->port_id) | ALE_PORT_HOST);
 221
 222	pm_runtime_put(common->dev);
 223	return ret;
 224}
 225
 226static void am65_cpsw_slave_set_promisc(struct am65_cpsw_port *port,
 227					bool promisc)
 228{
 229	struct am65_cpsw_common *common = port->common;
 230
 231	if (promisc && !common->is_emac_mode) {
 232		dev_dbg(common->dev, "promisc mode requested in switch mode");
 233		return;
 234	}
 235
 236	if (promisc) {
 237		/* Enable promiscuous mode */
 238		cpsw_ale_control_set(common->ale, port->port_id,
 239				     ALE_PORT_MACONLY_CAF, 1);
 240		dev_dbg(common->dev, "promisc enabled\n");
 241	} else {
 242		/* Disable promiscuous mode */
 243		cpsw_ale_control_set(common->ale, port->port_id,
 244				     ALE_PORT_MACONLY_CAF, 0);
 245		dev_dbg(common->dev, "promisc disabled\n");
 246	}
 247}
 248
 249static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev)
 250{
 251	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
 252	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
 253	u32 port_mask;
 254	bool promisc;
 255
 256	promisc = !!(ndev->flags & IFF_PROMISC);
 257	am65_cpsw_slave_set_promisc(port, promisc);
 258
 259	if (promisc)
 260		return;
 261
 262	/* Restore allmulti on vlans if necessary */
 263	cpsw_ale_set_allmulti(common->ale,
 264			      ndev->flags & IFF_ALLMULTI, port->port_id);
 265
 266	port_mask = ALE_PORT_HOST;
 267	/* Clear all mcast from ALE */
 268	cpsw_ale_flush_multicast(common->ale, port_mask, -1);
 269
 270	if (!netdev_mc_empty(ndev)) {
 271		struct netdev_hw_addr *ha;
 272
 273		/* program multicast address list into ALE register */
 274		netdev_for_each_mc_addr(ha, ndev) {
 275			cpsw_ale_add_mcast(common->ale, ha->addr,
 276					   port_mask, 0, 0, 0);
 277		}
 278	}
 279}
 280
 281static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev,
 282					       unsigned int txqueue)
 283{
 284	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
 285	struct am65_cpsw_tx_chn *tx_chn;
 286	struct netdev_queue *netif_txq;
 287	unsigned long trans_start;
 288
 289	netif_txq = netdev_get_tx_queue(ndev, txqueue);
 290	tx_chn = &common->tx_chns[txqueue];
 291	trans_start = READ_ONCE(netif_txq->trans_start);
 292
 293	netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n",
 294		   txqueue,
 295		   netif_tx_queue_stopped(netif_txq),
 296		   jiffies_to_msecs(jiffies - trans_start),
 297		   netdev_queue_dql_avail(netif_txq),
 298		   k3_cppi_desc_pool_avail(tx_chn->desc_pool));
 299
 300	if (netif_tx_queue_stopped(netif_txq)) {
 301		/* try recover if stopped by us */
 302		txq_trans_update(netif_txq);
 303		netif_tx_wake_queue(netif_txq);
 304	}
 305}
 306
 307static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common,
 308				  struct sk_buff *skb)
 309{
 310	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
 311	struct cppi5_host_desc_t *desc_rx;
 312	struct device *dev = common->dev;
 313	u32 pkt_len = skb_tailroom(skb);
 314	dma_addr_t desc_dma;
 315	dma_addr_t buf_dma;
 316	void *swdata;
 317
 318	desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);
 319	if (!desc_rx) {
 320		dev_err(dev, "Failed to allocate RXFDQ descriptor\n");
 321		return -ENOMEM;
 322	}
 323	desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);
 324
 325	buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len,
 326				 DMA_FROM_DEVICE);
 327	if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {
 328		k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
 329		dev_err(dev, "Failed to map rx skb buffer\n");
 330		return -EINVAL;
 331	}
 332
 333	cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
 334			 AM65_CPSW_NAV_PS_DATA_SIZE);
 335	k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);
 336	cppi5_hdesc_attach_buf(desc_rx, buf_dma, skb_tailroom(skb), buf_dma, skb_tailroom(skb));
 337	swdata = cppi5_hdesc_get_swdata(desc_rx);
 338	*((void **)swdata) = skb;
 339
 340	return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0, desc_rx, desc_dma);
 341}
 342
 343void am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common *common)
 344{
 345	struct am65_cpsw_host *host_p = am65_common_get_host(common);
 346	u32 val, pri_map;
 347
 348	/* P0 set Receive Priority Type */
 349	val = readl(host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
 350
 351	if (common->pf_p0_rx_ptype_rrobin) {
 352		val |= AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
 353		/* Enet Ports fifos works in fixed priority mode only, so
 354		 * reset P0_Rx_Pri_Map so all packet will go in Enet fifo 0
 355		 */
 356		pri_map = 0x0;
 357	} else {
 358		val &= ~AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
 359		/* restore P0_Rx_Pri_Map */
 360		pri_map = 0x76543210;
 361	}
 362
 363	writel(pri_map, host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP);
 364	writel(val, host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
 365}
 366
 367static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common);
 368static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common);
 369static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port);
 370static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port);
 371
 372static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
 373{
 374	struct am65_cpsw_rx_chn *rx_chn = data;
 375	struct cppi5_host_desc_t *desc_rx;
 376	struct sk_buff *skb;
 377	dma_addr_t buf_dma;
 378	u32 buf_dma_len;
 379	void **swdata;
 380
 381	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
 382	swdata = cppi5_hdesc_get_swdata(desc_rx);
 383	skb = *swdata;
 384	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
 385	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
 386
 387	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
 388	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
 389
 390	dev_kfree_skb_any(skb);
 391}
 392
 393static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
 394				     struct cppi5_host_desc_t *desc)
 395{
 396	struct cppi5_host_desc_t *first_desc, *next_desc;
 397	dma_addr_t buf_dma, next_desc_dma;
 398	u32 buf_dma_len;
 399
 400	first_desc = desc;
 401	next_desc = first_desc;
 402
 403	cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
 404	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
 405
 406	dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);
 407
 408	next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
 409	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
 410	while (next_desc_dma) {
 411		next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
 412						       next_desc_dma);
 413		cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
 414		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
 415
 416		dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
 417			       DMA_TO_DEVICE);
 418
 419		next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
 420		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
 421
 422		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
 423	}
 424
 425	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);
 426}
 427
 428static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
 429{
 430	struct am65_cpsw_tx_chn *tx_chn = data;
 431	struct cppi5_host_desc_t *desc_tx;
 432	struct sk_buff *skb;
 433	void **swdata;
 434
 435	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
 436	swdata = cppi5_hdesc_get_swdata(desc_tx);
 437	skb = *(swdata);
 438	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
 439
 440	dev_kfree_skb_any(skb);
 441}
 442
 443static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
 444{
 445	struct am65_cpsw_host *host_p = am65_common_get_host(common);
 446	int port_idx, i, ret, tx;
 447	struct sk_buff *skb;
 448	u32 val, port_mask;
 449
 450	if (common->usage_count)
 451		return 0;
 452
 453	/* Control register */
 454	writel(AM65_CPSW_CTL_P0_ENABLE | AM65_CPSW_CTL_P0_TX_CRC_REMOVE |
 455	       AM65_CPSW_CTL_VLAN_AWARE | AM65_CPSW_CTL_P0_RX_PAD,
 456	       common->cpsw_base + AM65_CPSW_REG_CTL);
 457	/* Max length register */
 458	writel(AM65_CPSW_MAX_PACKET_SIZE,
 459	       host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
 460	/* set base flow_id */
 461	writel(common->rx_flow_id_base,
 462	       host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET);
 463	writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN | AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN,
 464	       host_p->port_base + AM65_CPSW_P0_REG_CTL);
 465
 466	am65_cpsw_nuss_set_p0_ptype(common);
 467
 468	/* enable statistic */
 469	val = BIT(HOST_PORT_NUM);
 470	for (port_idx = 0; port_idx < common->port_num; port_idx++) {
 471		struct am65_cpsw_port *port = &common->ports[port_idx];
 472
 473		if (!port->disabled)
 474			val |=  BIT(port->port_id);
 475	}
 476	writel(val, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
 477
 478	/* disable priority elevation */
 479	writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE);
 480
 481	cpsw_ale_start(common->ale);
 482
 483	/* limit to one RX flow only */
 484	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
 485			     ALE_DEFAULT_THREAD_ID, 0);
 486	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
 487			     ALE_DEFAULT_THREAD_ENABLE, 1);
 488	/* switch to vlan unaware mode */
 489	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_VLAN_AWARE, 1);
 490	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
 491			     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
 492
 493	/* default vlan cfg: create mask based on enabled ports */
 494	port_mask = GENMASK(common->port_num, 0) &
 495		    ~common->disabled_ports_mask;
 496
 497	cpsw_ale_add_vlan(common->ale, 0, port_mask,
 498			  port_mask, port_mask,
 499			  port_mask & ~ALE_PORT_HOST);
 500
 501	if (common->is_emac_mode)
 502		am65_cpsw_init_host_port_emac(common);
 503	else
 504		am65_cpsw_init_host_port_switch(common);
 505
 506	am65_cpsw_qos_tx_p0_rate_init(common);
 507
 508	for (i = 0; i < common->rx_chns.descs_num; i++) {
 509		skb = __netdev_alloc_skb_ip_align(NULL,
 510						  AM65_CPSW_MAX_PACKET_SIZE,
 511						  GFP_KERNEL);
 512		if (!skb) {
 513			ret = -ENOMEM;
 514			dev_err(common->dev, "cannot allocate skb\n");
 515			if (i)
 516				goto fail_rx;
 517
 518			return ret;
 519		}
 520
 521		ret = am65_cpsw_nuss_rx_push(common, skb);
 522		if (ret < 0) {
 523			dev_err(common->dev,
 524				"cannot submit skb to channel rx, error %d\n",
 525				ret);
 526			kfree_skb(skb);
 527			if (i)
 528				goto fail_rx;
 529
 530			return ret;
 531		}
 532	}
 533
 534	ret = k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);
 535	if (ret) {
 536		dev_err(common->dev, "couldn't enable rx chn: %d\n", ret);
 537		goto fail_rx;
 538	}
 539
 540	for (tx = 0; tx < common->tx_ch_num; tx++) {
 541		ret = k3_udma_glue_enable_tx_chn(common->tx_chns[tx].tx_chn);
 542		if (ret) {
 543			dev_err(common->dev, "couldn't enable tx chn %d: %d\n",
 544				tx, ret);
 545			tx--;
 546			goto fail_tx;
 547		}
 548		napi_enable(&common->tx_chns[tx].napi_tx);
 549	}
 550
 551	napi_enable(&common->napi_rx);
 552	if (common->rx_irq_disabled) {
 553		common->rx_irq_disabled = false;
 554		enable_irq(common->rx_chns.irq);
 555	}
 556
 557	dev_dbg(common->dev, "cpsw_nuss started\n");
 558	return 0;
 559
 560fail_tx:
 561	while (tx >= 0) {
 562		napi_disable(&common->tx_chns[tx].napi_tx);
 563		k3_udma_glue_disable_tx_chn(common->tx_chns[tx].tx_chn);
 564		tx--;
 565	}
 566
 567	k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);
 568
 569fail_rx:
 570	k3_udma_glue_reset_rx_chn(common->rx_chns.rx_chn, 0,
 571				  &common->rx_chns,
 572				  am65_cpsw_nuss_rx_cleanup, 0);
 573	return ret;
 574}
 575
 576static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
 577{
 578	int i;
 579
 580	if (common->usage_count != 1)
 581		return 0;
 582
 583	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
 584			     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
 585
 586	/* shutdown tx channels */
 587	atomic_set(&common->tdown_cnt, common->tx_ch_num);
 588	/* ensure new tdown_cnt value is visible */
 589	smp_mb__after_atomic();
 590	reinit_completion(&common->tdown_complete);
 591
 592	for (i = 0; i < common->tx_ch_num; i++)
 593		k3_udma_glue_tdown_tx_chn(common->tx_chns[i].tx_chn, false);
 594
 595	i = wait_for_completion_timeout(&common->tdown_complete,
 596					msecs_to_jiffies(1000));
 597	if (!i)
 598		dev_err(common->dev, "tx timeout\n");
 599	for (i = 0; i < common->tx_ch_num; i++) {
 600		napi_disable(&common->tx_chns[i].napi_tx);
 601		hrtimer_cancel(&common->tx_chns[i].tx_hrtimer);
 602	}
 603
 604	for (i = 0; i < common->tx_ch_num; i++) {
 605		k3_udma_glue_reset_tx_chn(common->tx_chns[i].tx_chn,
 606					  &common->tx_chns[i],
 607					  am65_cpsw_nuss_tx_cleanup);
 608		k3_udma_glue_disable_tx_chn(common->tx_chns[i].tx_chn);
 609	}
 610
 611	reinit_completion(&common->tdown_complete);
 612	k3_udma_glue_tdown_rx_chn(common->rx_chns.rx_chn, true);
 613
 614	if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) {
 615		i = wait_for_completion_timeout(&common->tdown_complete, msecs_to_jiffies(1000));
 616		if (!i)
 617			dev_err(common->dev, "rx teardown timeout\n");
 618	}
 619
 620	napi_disable(&common->napi_rx);
 621	hrtimer_cancel(&common->rx_hrtimer);
 622
 623	for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++)
 624		k3_udma_glue_reset_rx_chn(common->rx_chns.rx_chn, i,
 625					  &common->rx_chns,
 626					  am65_cpsw_nuss_rx_cleanup, !!i);
 627
 628	k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);
 629
 630	cpsw_ale_stop(common->ale);
 631
 632	writel(0, common->cpsw_base + AM65_CPSW_REG_CTL);
 633	writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
 634
 635	dev_dbg(common->dev, "cpsw_nuss stopped\n");
 636	return 0;
 637}
 638
 639static int am65_cpsw_nuss_ndo_slave_stop(struct net_device *ndev)
 640{
 641	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
 642	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
 643	int ret;
 644
 645	phylink_stop(port->slave.phylink);
 646
 647	netif_tx_stop_all_queues(ndev);
 648
 649	phylink_disconnect_phy(port->slave.phylink);
 650
 651	ret = am65_cpsw_nuss_common_stop(common);
 652	if (ret)
 653		return ret;
 654
 655	common->usage_count--;
 656	pm_runtime_put(common->dev);
 657	return 0;
 658}
 659
 660static int cpsw_restore_vlans(struct net_device *vdev, int vid, void *arg)
 661{
 662	struct am65_cpsw_port *port = arg;
 663
 664	if (!vdev)
 665		return 0;
 666
 667	return am65_cpsw_nuss_ndo_slave_add_vid(port->ndev, 0, vid);
 668}
 669
 670static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
 671{
 672	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
 673	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
 674	int ret, i;
 675	u32 reg;
 676
 677	ret = pm_runtime_resume_and_get(common->dev);
 678	if (ret < 0)
 679		return ret;
 680
 681	/* Idle MAC port */
 682	cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
 683	cpsw_sl_wait_for_idle(port->slave.mac_sl, 100);
 684	cpsw_sl_ctl_reset(port->slave.mac_sl);
 685
 686	/* soft reset MAC */
 687	cpsw_sl_reg_write(port->slave.mac_sl, CPSW_SL_SOFT_RESET, 1);
 688	mdelay(1);
 689	reg = cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_SOFT_RESET);
 690	if (reg) {
 691		dev_err(common->dev, "soft RESET didn't complete\n");
 692		ret = -ETIMEDOUT;
 693		goto runtime_put;
 694	}
 695
 696	/* Notify the stack of the actual queue counts. */
 697	ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num);
 698	if (ret) {
 699		dev_err(common->dev, "cannot set real number of tx queues\n");
 700		goto runtime_put;
 701	}
 702
 703	ret = netif_set_real_num_rx_queues(ndev, AM65_CPSW_MAX_RX_QUEUES);
 704	if (ret) {
 705		dev_err(common->dev, "cannot set real number of rx queues\n");
 706		goto runtime_put;
 707	}
 708
 709	for (i = 0; i < common->tx_ch_num; i++) {
 710		struct netdev_queue *txq = netdev_get_tx_queue(ndev, i);
 711
 712		netdev_tx_reset_queue(txq);
 713		txq->tx_maxrate =  common->tx_chns[i].rate_mbps;
 714	}
 715
 716	ret = am65_cpsw_nuss_common_open(common);
 717	if (ret)
 718		goto runtime_put;
 719
 720	common->usage_count++;
 721
 722	am65_cpsw_port_set_sl_mac(port, ndev->dev_addr);
 723
 724	if (common->is_emac_mode)
 725		am65_cpsw_init_port_emac_ale(port);
 726	else
 727		am65_cpsw_init_port_switch_ale(port);
 728
 729	/* mac_sl should be configured via phy-link interface */
 730	am65_cpsw_sl_ctl_reset(port);
 731
 732	ret = phylink_of_phy_connect(port->slave.phylink, port->slave.phy_node, 0);
 733	if (ret)
 734		goto error_cleanup;
 735
 736	/* restore vlan configurations */
 737	vlan_for_each(ndev, cpsw_restore_vlans, port);
 738
 739	phylink_start(port->slave.phylink);
 740
 741	return 0;
 742
 743error_cleanup:
 744	am65_cpsw_nuss_ndo_slave_stop(ndev);
 745	return ret;
 746
 747runtime_put:
 748	pm_runtime_put(common->dev);
 749	return ret;
 750}
 751
 752static void am65_cpsw_nuss_rx_ts(struct sk_buff *skb, u32 *psdata)
 753{
 754	struct skb_shared_hwtstamps *ssh;
 755	u64 ns;
 756
 757	ns = ((u64)psdata[1] << 32) | psdata[0];
 758
 759	ssh = skb_hwtstamps(skb);
 760	memset(ssh, 0, sizeof(*ssh));
 761	ssh->hwtstamp = ns_to_ktime(ns);
 762}
 763
 764/* RX psdata[2] word format - checksum information */
 765#define AM65_CPSW_RX_PSD_CSUM_ADD	GENMASK(15, 0)
 766#define AM65_CPSW_RX_PSD_CSUM_ERR	BIT(16)
 767#define AM65_CPSW_RX_PSD_IS_FRAGMENT	BIT(17)
 768#define AM65_CPSW_RX_PSD_IS_TCP		BIT(18)
 769#define AM65_CPSW_RX_PSD_IPV6_VALID	BIT(19)
 770#define AM65_CPSW_RX_PSD_IPV4_VALID	BIT(20)
 771
 772static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info)
 773{
 774	/* HW can verify IPv4/IPv6 TCP/UDP packets checksum
 775	 * csum information provides in psdata[2] word:
 776	 * AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error
 777	 * AM65_CPSW_RX_PSD_IPV6_VALID and AM65_CPSW_RX_PSD_IPV4_VALID
 778	 * bits - indicates IPv4/IPv6 packet
 779	 * AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet
 780	 * AM65_CPSW_RX_PSD_CSUM_ADD has value 0xFFFF for non fragmented packets
 781	 * or csum value for fragmented packets if !AM65_CPSW_RX_PSD_CSUM_ERR
 782	 */
 783	skb_checksum_none_assert(skb);
 784
 785	if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
 786		return;
 787
 788	if ((csum_info & (AM65_CPSW_RX_PSD_IPV6_VALID |
 789			  AM65_CPSW_RX_PSD_IPV4_VALID)) &&
 790			  !(csum_info & AM65_CPSW_RX_PSD_CSUM_ERR)) {
 791		/* csum for fragmented packets is unsupported */
 792		if (!(csum_info & AM65_CPSW_RX_PSD_IS_FRAGMENT))
 793			skb->ip_summed = CHECKSUM_UNNECESSARY;
 794	}
 795}
 796
 797static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
 798				     u32 flow_idx)
 799{
 800	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
 801	u32 buf_dma_len, pkt_len, port_id = 0, csum_info;
 802	struct am65_cpsw_ndev_priv *ndev_priv;
 803	struct am65_cpsw_ndev_stats *stats;
 804	struct cppi5_host_desc_t *desc_rx;
 805	struct device *dev = common->dev;
 806	struct sk_buff *skb, *new_skb;
 807	dma_addr_t desc_dma, buf_dma;
 808	struct am65_cpsw_port *port;
 809	struct net_device *ndev;
 810	void **swdata;
 811	u32 *psdata;
 812	int ret = 0;
 813
 814	ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);
 815	if (ret) {
 816		if (ret != -ENODATA)
 817			dev_err(dev, "RX: pop chn fail %d\n", ret);
 818		return ret;
 819	}
 820
 821	if (cppi5_desc_is_tdcm(desc_dma)) {
 822		dev_dbg(dev, "%s RX tdown flow: %u\n", __func__, flow_idx);
 823		if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ)
 824			complete(&common->tdown_complete);
 825		return 0;
 826	}
 827
 828	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
 829	dev_dbg(dev, "%s flow_idx: %u desc %pad\n",
 830		__func__, flow_idx, &desc_dma);
 831
 832	swdata = cppi5_hdesc_get_swdata(desc_rx);
 833	skb = *swdata;
 834	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
 835	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
 836	pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
 837	cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
 838	dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id);
 839	port = am65_common_get_port(common, port_id);
 840	ndev = port->ndev;
 841	skb->dev = ndev;
 842
 843	psdata = cppi5_hdesc_get_psdata(desc_rx);
 844	/* add RX timestamp */
 845	if (port->rx_ts_enabled)
 846		am65_cpsw_nuss_rx_ts(skb, psdata);
 847	csum_info = psdata[2];
 848	dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info);
 849
 850	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
 851
 852	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
 853
 854	new_skb = netdev_alloc_skb_ip_align(ndev, AM65_CPSW_MAX_PACKET_SIZE);
 855	if (new_skb) {
 856		ndev_priv = netdev_priv(ndev);
 857		am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark);
 858		skb_put(skb, pkt_len);
 859		skb->protocol = eth_type_trans(skb, ndev);
 860		am65_cpsw_nuss_rx_csum(skb, csum_info);
 861		napi_gro_receive(&common->napi_rx, skb);
 862
 863		stats = this_cpu_ptr(ndev_priv->stats);
 864
 865		u64_stats_update_begin(&stats->syncp);
 866		stats->rx_packets++;
 867		stats->rx_bytes += pkt_len;
 868		u64_stats_update_end(&stats->syncp);
 869		kmemleak_not_leak(new_skb);
 870	} else {
 871		ndev->stats.rx_dropped++;
 872		new_skb = skb;
 873	}
 874
 875	if (netif_dormant(ndev)) {
 876		dev_kfree_skb_any(new_skb);
 877		ndev->stats.rx_dropped++;
 878		return 0;
 879	}
 880
 881	ret = am65_cpsw_nuss_rx_push(common, new_skb);
 882	if (WARN_ON(ret < 0)) {
 883		dev_kfree_skb_any(new_skb);
 884		ndev->stats.rx_errors++;
 885		ndev->stats.rx_dropped++;
 886	}
 887
 888	return ret;
 889}
 890
 891static enum hrtimer_restart am65_cpsw_nuss_rx_timer_callback(struct hrtimer *timer)
 892{
 893	struct am65_cpsw_common *common =
 894			container_of(timer, struct am65_cpsw_common, rx_hrtimer);
 895
 896	enable_irq(common->rx_chns.irq);
 897	return HRTIMER_NORESTART;
 898}
 899
 900static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
 901{
 902	struct am65_cpsw_common *common = am65_cpsw_napi_to_common(napi_rx);
 903	int flow = AM65_CPSW_MAX_RX_FLOWS;
 904	int cur_budget, ret;
 905	int num_rx = 0;
 906
 907	/* process every flow */
 908	while (flow--) {
 909		cur_budget = budget - num_rx;
 910
 911		while (cur_budget--) {
 912			ret = am65_cpsw_nuss_rx_packets(common, flow);
 913			if (ret)
 914				break;
 915			num_rx++;
 916		}
 917
 918		if (num_rx >= budget)
 919			break;
 920	}
 921
 922	dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);
 923
 924	if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) {
 925		if (common->rx_irq_disabled) {
 926			common->rx_irq_disabled = false;
 927			if (unlikely(common->rx_pace_timeout)) {
 928				hrtimer_start(&common->rx_hrtimer,
 929					      ns_to_ktime(common->rx_pace_timeout),
 930					      HRTIMER_MODE_REL_PINNED);
 931			} else {
 932				enable_irq(common->rx_chns.irq);
 933			}
 934		}
 935	}
 936
 937	return num_rx;
 938}
 939
 940static struct sk_buff *
 941am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn,
 942			       dma_addr_t desc_dma)
 943{
 944	struct am65_cpsw_ndev_priv *ndev_priv;
 945	struct am65_cpsw_ndev_stats *stats;
 946	struct cppi5_host_desc_t *desc_tx;
 947	struct net_device *ndev;
 948	struct sk_buff *skb;
 949	void **swdata;
 950
 951	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
 952					     desc_dma);
 953	swdata = cppi5_hdesc_get_swdata(desc_tx);
 954	skb = *(swdata);
 955	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
 956
 957	ndev = skb->dev;
 958
 959	am65_cpts_tx_timestamp(tx_chn->common->cpts, skb);
 960
 961	ndev_priv = netdev_priv(ndev);
 962	stats = this_cpu_ptr(ndev_priv->stats);
 963	u64_stats_update_begin(&stats->syncp);
 964	stats->tx_packets++;
 965	stats->tx_bytes += skb->len;
 966	u64_stats_update_end(&stats->syncp);
 967
 968	return skb;
 969}
 970
 971static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,
 972				   struct netdev_queue *netif_txq)
 973{
 974	if (netif_tx_queue_stopped(netif_txq)) {
 975		/* Check whether the queue is stopped due to stalled
 976		 * tx dma, if the queue is stopped then wake the queue
 977		 * as we have free desc for tx
 978		 */
 979		__netif_tx_lock(netif_txq, smp_processor_id());
 980		if (netif_running(ndev) &&
 981		    (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))
 982			netif_tx_wake_queue(netif_txq);
 983
 984		__netif_tx_unlock(netif_txq);
 985	}
 986}
 987
 988static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
 989					   int chn, unsigned int budget, bool *tdown)
 990{
 991	struct device *dev = common->dev;
 992	struct am65_cpsw_tx_chn *tx_chn;
 993	struct netdev_queue *netif_txq;
 994	unsigned int total_bytes = 0;
 995	struct net_device *ndev;
 996	struct sk_buff *skb;
 997	dma_addr_t desc_dma;
 998	int res, num_tx = 0;
 999
1000	tx_chn = &common->tx_chns[chn];
1001
1002	while (true) {
1003		spin_lock(&tx_chn->lock);
1004		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
1005		spin_unlock(&tx_chn->lock);
1006		if (res == -ENODATA)
1007			break;
1008
1009		if (cppi5_desc_is_tdcm(desc_dma)) {
1010			if (atomic_dec_and_test(&common->tdown_cnt))
1011				complete(&common->tdown_complete);
1012			*tdown = true;
1013			break;
1014		}
1015
1016		skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);
1017		total_bytes = skb->len;
1018		ndev = skb->dev;
1019		napi_consume_skb(skb, budget);
1020		num_tx++;
1021
1022		netif_txq = netdev_get_tx_queue(ndev, chn);
1023
1024		netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1025
1026		am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1027	}
1028
1029	dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1030
1031	return num_tx;
1032}
1033
1034static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common,
1035					      int chn, unsigned int budget, bool *tdown)
1036{
1037	struct device *dev = common->dev;
1038	struct am65_cpsw_tx_chn *tx_chn;
1039	struct netdev_queue *netif_txq;
1040	unsigned int total_bytes = 0;
1041	struct net_device *ndev;
1042	struct sk_buff *skb;
1043	dma_addr_t desc_dma;
1044	int res, num_tx = 0;
1045
1046	tx_chn = &common->tx_chns[chn];
1047
1048	while (true) {
1049		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
1050		if (res == -ENODATA)
1051			break;
1052
1053		if (cppi5_desc_is_tdcm(desc_dma)) {
1054			if (atomic_dec_and_test(&common->tdown_cnt))
1055				complete(&common->tdown_complete);
1056			*tdown = true;
1057			break;
1058		}
1059
1060		skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);
1061
1062		ndev = skb->dev;
1063		total_bytes += skb->len;
1064		napi_consume_skb(skb, budget);
1065		num_tx++;
1066	}
1067
1068	if (!num_tx)
1069		return 0;
1070
1071	netif_txq = netdev_get_tx_queue(ndev, chn);
1072
1073	netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1074
1075	am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1076
1077	dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1078
1079	return num_tx;
1080}
1081
1082static enum hrtimer_restart am65_cpsw_nuss_tx_timer_callback(struct hrtimer *timer)
1083{
1084	struct am65_cpsw_tx_chn *tx_chns =
1085			container_of(timer, struct am65_cpsw_tx_chn, tx_hrtimer);
1086
1087	enable_irq(tx_chns->irq);
1088	return HRTIMER_NORESTART;
1089}
1090
1091static int am65_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget)
1092{
1093	struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx);
1094	bool tdown = false;
1095	int num_tx;
1096
1097	if (AM65_CPSW_IS_CPSW2G(tx_chn->common))
1098		num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id,
1099							    budget, &tdown);
1100	else
1101		num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common,
1102							 tx_chn->id, budget, &tdown);
1103
1104	if (num_tx >= budget)
1105		return budget;
1106
1107	if (napi_complete_done(napi_tx, num_tx)) {
1108		if (unlikely(tx_chn->tx_pace_timeout && !tdown)) {
1109			hrtimer_start(&tx_chn->tx_hrtimer,
1110				      ns_to_ktime(tx_chn->tx_pace_timeout),
1111				      HRTIMER_MODE_REL_PINNED);
1112		} else {
1113			enable_irq(tx_chn->irq);
1114		}
1115	}
1116
1117	return 0;
1118}
1119
1120static irqreturn_t am65_cpsw_nuss_rx_irq(int irq, void *dev_id)
1121{
1122	struct am65_cpsw_common *common = dev_id;
1123
1124	common->rx_irq_disabled = true;
1125	disable_irq_nosync(irq);
1126	napi_schedule(&common->napi_rx);
1127
1128	return IRQ_HANDLED;
1129}
1130
1131static irqreturn_t am65_cpsw_nuss_tx_irq(int irq, void *dev_id)
1132{
1133	struct am65_cpsw_tx_chn *tx_chn = dev_id;
1134
1135	disable_irq_nosync(irq);
1136	napi_schedule(&tx_chn->napi_tx);
1137
1138	return IRQ_HANDLED;
1139}
1140
1141static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
1142						 struct net_device *ndev)
1143{
1144	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1145	struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
1146	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1147	struct device *dev = common->dev;
1148	struct am65_cpsw_tx_chn *tx_chn;
1149	struct netdev_queue *netif_txq;
1150	dma_addr_t desc_dma, buf_dma;
1151	int ret, q_idx, i;
1152	void **swdata;
1153	u32 *psdata;
1154	u32 pkt_len;
1155
1156	/* padding enabled in hw */
1157	pkt_len = skb_headlen(skb);
1158
1159	/* SKB TX timestamp */
1160	if (port->tx_ts_enabled)
1161		am65_cpts_prep_tx_timestamp(common->cpts, skb);
1162
1163	q_idx = skb_get_queue_mapping(skb);
1164	dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx);
1165
1166	tx_chn = &common->tx_chns[q_idx];
1167	netif_txq = netdev_get_tx_queue(ndev, q_idx);
1168
1169	/* Map the linear buffer */
1170	buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len,
1171				 DMA_TO_DEVICE);
1172	if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1173		dev_err(dev, "Failed to map tx skb buffer\n");
1174		ndev->stats.tx_errors++;
1175		goto err_free_skb;
1176	}
1177
1178	first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1179	if (!first_desc) {
1180		dev_dbg(dev, "Failed to allocate descriptor\n");
1181		dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len,
1182				 DMA_TO_DEVICE);
1183		goto busy_stop_q;
1184	}
1185
1186	cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
1187			 AM65_CPSW_NAV_PS_DATA_SIZE);
1188	cppi5_desc_set_pktids(&first_desc->hdr, 0, 0x3FFF);
1189	cppi5_hdesc_set_pkttype(first_desc, 0x7);
1190	cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id);
1191
1192	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1193	cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
1194	swdata = cppi5_hdesc_get_swdata(first_desc);
1195	*(swdata) = skb;
1196	psdata = cppi5_hdesc_get_psdata(first_desc);
1197
1198	/* HW csum offload if enabled */
1199	psdata[2] = 0;
1200	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1201		unsigned int cs_start, cs_offset;
1202
1203		cs_start = skb_transport_offset(skb);
1204		cs_offset = cs_start + skb->csum_offset;
1205		/* HW numerates bytes starting from 1 */
1206		psdata[2] = ((cs_offset + 1) << 24) |
1207			    ((cs_start + 1) << 16) | (skb->len - cs_start);
1208		dev_dbg(dev, "%s tx psdata:%#x\n", __func__, psdata[2]);
1209	}
1210
1211	if (!skb_is_nonlinear(skb))
1212		goto done_tx;
1213
1214	dev_dbg(dev, "fragmented SKB\n");
1215
1216	/* Handle the case where skb is fragmented in pages */
1217	cur_desc = first_desc;
1218	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1219		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1220		u32 frag_size = skb_frag_size(frag);
1221
1222		next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1223		if (!next_desc) {
1224			dev_err(dev, "Failed to allocate descriptor\n");
1225			goto busy_free_descs;
1226		}
1227
1228		buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,
1229					   DMA_TO_DEVICE);
1230		if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1231			dev_err(dev, "Failed to map tx skb page\n");
1232			k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
1233			ndev->stats.tx_errors++;
1234			goto err_free_descs;
1235		}
1236
1237		cppi5_hdesc_reset_hbdesc(next_desc);
1238		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1239		cppi5_hdesc_attach_buf(next_desc,
1240				       buf_dma, frag_size, buf_dma, frag_size);
1241
1242		desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,
1243						      next_desc);
1244		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);
1245		cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
1246
1247		pkt_len += frag_size;
1248		cur_desc = next_desc;
1249	}
1250	WARN_ON(pkt_len != skb->len);
1251
1252done_tx:
1253	skb_tx_timestamp(skb);
1254
1255	/* report bql before sending packet */
1256	netdev_tx_sent_queue(netif_txq, pkt_len);
1257
1258	cppi5_hdesc_set_pktlen(first_desc, pkt_len);
1259	desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);
1260	if (AM65_CPSW_IS_CPSW2G(common)) {
1261		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1262	} else {
1263		spin_lock_bh(&tx_chn->lock);
1264		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1265		spin_unlock_bh(&tx_chn->lock);
1266	}
1267	if (ret) {
1268		dev_err(dev, "can't push desc %d\n", ret);
1269		/* inform bql */
1270		netdev_tx_completed_queue(netif_txq, 1, pkt_len);
1271		ndev->stats.tx_errors++;
1272		goto err_free_descs;
1273	}
1274
1275	if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {
1276		netif_tx_stop_queue(netif_txq);
1277		/* Barrier, so that stop_queue visible to other cpus */
1278		smp_mb__after_atomic();
1279		dev_dbg(dev, "netif_tx_stop_queue %d\n", q_idx);
1280
1281		/* re-check for smp */
1282		if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=
1283		    MAX_SKB_FRAGS) {
1284			netif_tx_wake_queue(netif_txq);
1285			dev_dbg(dev, "netif_tx_wake_queue %d\n", q_idx);
1286		}
1287	}
1288
1289	return NETDEV_TX_OK;
1290
1291err_free_descs:
1292	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1293err_free_skb:
1294	ndev->stats.tx_dropped++;
1295	dev_kfree_skb_any(skb);
1296	return NETDEV_TX_OK;
1297
1298busy_free_descs:
1299	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1300busy_stop_q:
1301	netif_tx_stop_queue(netif_txq);
1302	return NETDEV_TX_BUSY;
1303}
1304
1305static int am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device *ndev,
1306						    void *addr)
1307{
1308	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1309	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1310	struct sockaddr *sockaddr = (struct sockaddr *)addr;
1311	int ret;
1312
1313	ret = eth_prepare_mac_addr_change(ndev, addr);
1314	if (ret < 0)
1315		return ret;
1316
1317	ret = pm_runtime_resume_and_get(common->dev);
1318	if (ret < 0)
1319		return ret;
1320
1321	cpsw_ale_del_ucast(common->ale, ndev->dev_addr,
1322			   HOST_PORT_NUM, 0, 0);
1323	cpsw_ale_add_ucast(common->ale, sockaddr->sa_data,
1324			   HOST_PORT_NUM, ALE_SECURE, 0);
1325
1326	am65_cpsw_port_set_sl_mac(port, addr);
1327	eth_commit_mac_addr_change(ndev, sockaddr);
1328
1329	pm_runtime_put(common->dev);
1330
1331	return 0;
1332}
1333
1334static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev,
1335				       struct ifreq *ifr)
1336{
1337	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1338	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1339	u32 ts_ctrl, seq_id, ts_ctrl_ltype2, ts_vlan_ltype;
1340	struct hwtstamp_config cfg;
1341
1342	if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1343		return -EOPNOTSUPP;
1344
1345	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1346		return -EFAULT;
1347
1348	/* TX HW timestamp */
1349	switch (cfg.tx_type) {
1350	case HWTSTAMP_TX_OFF:
1351	case HWTSTAMP_TX_ON:
1352		break;
1353	default:
1354		return -ERANGE;
1355	}
1356
1357	switch (cfg.rx_filter) {
1358	case HWTSTAMP_FILTER_NONE:
1359		port->rx_ts_enabled = false;
1360		break;
1361	case HWTSTAMP_FILTER_ALL:
1362	case HWTSTAMP_FILTER_SOME:
1363	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1364	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1365	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1366	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1367	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1368	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1369	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1370	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1371	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1372	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1373	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1374	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1375	case HWTSTAMP_FILTER_NTP_ALL:
1376		port->rx_ts_enabled = true;
1377		cfg.rx_filter = HWTSTAMP_FILTER_ALL;
1378		break;
1379	default:
1380		return -ERANGE;
1381	}
1382
1383	port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON);
1384
1385	/* cfg TX timestamp */
1386	seq_id = (AM65_CPSW_TS_SEQ_ID_OFFSET <<
1387		  AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT) | ETH_P_1588;
1388
1389	ts_vlan_ltype = ETH_P_8021Q;
1390
1391	ts_ctrl_ltype2 = ETH_P_1588 |
1392			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 |
1393			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 |
1394			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 |
1395			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 |
1396			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 |
1397			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 |
1398			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 |
1399			 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO;
1400
1401	ts_ctrl = AM65_CPSW_TS_EVENT_MSG_TYPE_BITS <<
1402		  AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT;
1403
1404	if (port->tx_ts_enabled)
1405		ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN |
1406			   AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN;
1407
1408	writel(seq_id, port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG);
1409	writel(ts_vlan_ltype, port->port_base +
1410	       AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG);
1411	writel(ts_ctrl_ltype2, port->port_base +
1412	       AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2);
1413	writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL);
1414
1415	/* en/dis RX timestamp */
1416	am65_cpts_rx_enable(common->cpts, port->rx_ts_enabled);
1417
1418	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1419}
1420
1421static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev,
1422				       struct ifreq *ifr)
1423{
1424	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1425	struct hwtstamp_config cfg;
1426
1427	if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1428		return -EOPNOTSUPP;
1429
1430	cfg.flags = 0;
1431	cfg.tx_type = port->tx_ts_enabled ?
1432		      HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1433	cfg.rx_filter = port->rx_ts_enabled ?
1434			HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1435
1436	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1437}
1438
1439static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev,
1440					  struct ifreq *req, int cmd)
1441{
1442	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1443
1444	if (!netif_running(ndev))
1445		return -EINVAL;
1446
1447	switch (cmd) {
1448	case SIOCSHWTSTAMP:
1449		return am65_cpsw_nuss_hwtstamp_set(ndev, req);
1450	case SIOCGHWTSTAMP:
1451		return am65_cpsw_nuss_hwtstamp_get(ndev, req);
1452	}
1453
1454	return phylink_mii_ioctl(port->slave.phylink, req, cmd);
1455}
1456
1457static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev,
1458					 struct rtnl_link_stats64 *stats)
1459{
1460	struct am65_cpsw_ndev_priv *ndev_priv = netdev_priv(dev);
1461	unsigned int start;
1462	int cpu;
1463
1464	for_each_possible_cpu(cpu) {
1465		struct am65_cpsw_ndev_stats *cpu_stats;
1466		u64 rx_packets;
1467		u64 rx_bytes;
1468		u64 tx_packets;
1469		u64 tx_bytes;
1470
1471		cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu);
1472		do {
1473			start = u64_stats_fetch_begin(&cpu_stats->syncp);
1474			rx_packets = cpu_stats->rx_packets;
1475			rx_bytes   = cpu_stats->rx_bytes;
1476			tx_packets = cpu_stats->tx_packets;
1477			tx_bytes   = cpu_stats->tx_bytes;
1478		} while (u64_stats_fetch_retry(&cpu_stats->syncp, start));
1479
1480		stats->rx_packets += rx_packets;
1481		stats->rx_bytes   += rx_bytes;
1482		stats->tx_packets += tx_packets;
1483		stats->tx_bytes   += tx_bytes;
1484	}
1485
1486	stats->rx_errors	= dev->stats.rx_errors;
1487	stats->rx_dropped	= dev->stats.rx_dropped;
1488	stats->tx_dropped	= dev->stats.tx_dropped;
1489}
1490
1491static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
1492	.ndo_open		= am65_cpsw_nuss_ndo_slave_open,
1493	.ndo_stop		= am65_cpsw_nuss_ndo_slave_stop,
1494	.ndo_start_xmit		= am65_cpsw_nuss_ndo_slave_xmit,
1495	.ndo_set_rx_mode	= am65_cpsw_nuss_ndo_slave_set_rx_mode,
1496	.ndo_get_stats64        = am65_cpsw_nuss_ndo_get_stats,
1497	.ndo_validate_addr	= eth_validate_addr,
1498	.ndo_set_mac_address	= am65_cpsw_nuss_ndo_slave_set_mac_address,
1499	.ndo_tx_timeout		= am65_cpsw_nuss_ndo_host_tx_timeout,
1500	.ndo_vlan_rx_add_vid	= am65_cpsw_nuss_ndo_slave_add_vid,
1501	.ndo_vlan_rx_kill_vid	= am65_cpsw_nuss_ndo_slave_kill_vid,
1502	.ndo_eth_ioctl		= am65_cpsw_nuss_ndo_slave_ioctl,
1503	.ndo_setup_tc           = am65_cpsw_qos_ndo_setup_tc,
1504	.ndo_set_tx_maxrate	= am65_cpsw_qos_ndo_tx_p0_set_maxrate,
1505};
1506
1507static void am65_cpsw_disable_phy(struct phy *phy)
1508{
1509	phy_power_off(phy);
1510	phy_exit(phy);
1511}
1512
1513static int am65_cpsw_enable_phy(struct phy *phy)
1514{
1515	int ret;
1516
1517	ret = phy_init(phy);
1518	if (ret < 0)
1519		return ret;
1520
1521	ret = phy_power_on(phy);
1522	if (ret < 0) {
1523		phy_exit(phy);
1524		return ret;
1525	}
1526
1527	return 0;
1528}
1529
1530static void am65_cpsw_disable_serdes_phy(struct am65_cpsw_common *common)
1531{
1532	struct am65_cpsw_port *port;
1533	struct phy *phy;
1534	int i;
1535
1536	for (i = 0; i < common->port_num; i++) {
1537		port = &common->ports[i];
1538		phy = port->slave.serdes_phy;
1539		if (phy)
1540			am65_cpsw_disable_phy(phy);
1541	}
1542}
1543
1544static int am65_cpsw_init_serdes_phy(struct device *dev, struct device_node *port_np,
1545				     struct am65_cpsw_port *port)
1546{
1547	const char *name = "serdes";
1548	struct phy *phy;
1549	int ret;
1550
1551	phy = devm_of_phy_optional_get(dev, port_np, name);
1552	if (IS_ERR_OR_NULL(phy))
1553		return PTR_ERR_OR_ZERO(phy);
1554
1555	/* Serdes PHY exists. Store it. */
1556	port->slave.serdes_phy = phy;
1557
1558	ret =  am65_cpsw_enable_phy(phy);
1559	if (ret < 0)
1560		goto err_phy;
1561
1562	return 0;
1563
1564err_phy:
1565	devm_phy_put(dev, phy);
1566	return ret;
1567}
1568
1569static void am65_cpsw_nuss_mac_config(struct phylink_config *config, unsigned int mode,
1570				      const struct phylink_link_state *state)
1571{
1572	struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
1573							  phylink_config);
1574	struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
1575	struct am65_cpsw_common *common = port->common;
1576
1577	if (common->pdata.extra_modes & BIT(state->interface)) {
1578		if (state->interface == PHY_INTERFACE_MODE_SGMII) {
1579			writel(ADVERTISE_SGMII,
1580			       port->sgmii_base + AM65_CPSW_SGMII_MR_ADV_ABILITY_REG);
1581			cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN);
1582		} else {
1583			cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_EXT_EN);
1584		}
1585
1586		if (state->interface == PHY_INTERFACE_MODE_USXGMII) {
1587			cpsw_sl_ctl_set(port->slave.mac_sl,
1588					CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
1589		} else {
1590			cpsw_sl_ctl_clr(port->slave.mac_sl,
1591					CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
1592		}
1593
1594		writel(AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE,
1595		       port->sgmii_base + AM65_CPSW_SGMII_CONTROL_REG);
1596	}
1597}
1598
1599static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned int mode,
1600					 phy_interface_t interface)
1601{
1602	struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
1603							  phylink_config);
1604	struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
1605	struct am65_cpsw_common *common = port->common;
1606	struct net_device *ndev = port->ndev;
1607	u32 mac_control;
1608	int tmo;
1609
1610	/* disable forwarding */
1611	cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1612
1613	cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
1614
1615	tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl, 100);
1616	dev_dbg(common->dev, "down msc_sl %08x tmo %d\n",
1617		cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS), tmo);
1618
1619	/* All the bits that am65_cpsw_nuss_mac_link_up() can possibly set */
1620	mac_control = CPSW_SL_CTL_GMII_EN | CPSW_SL_CTL_GIG | CPSW_SL_CTL_IFCTL_A |
1621		      CPSW_SL_CTL_FULLDUPLEX | CPSW_SL_CTL_RX_FLOW_EN | CPSW_SL_CTL_TX_FLOW_EN;
1622	/* If interface mode is RGMII, CPSW_SL_CTL_EXT_EN might have been set for 10 Mbps */
1623	if (phy_interface_mode_is_rgmii(interface))
1624		mac_control |= CPSW_SL_CTL_EXT_EN;
1625	/* Only clear those bits that can be set by am65_cpsw_nuss_mac_link_up() */
1626	cpsw_sl_ctl_clr(port->slave.mac_sl, mac_control);
1627
1628	am65_cpsw_qos_link_down(ndev);
1629	netif_tx_stop_all_queues(ndev);
1630}
1631
1632static void am65_cpsw_nuss_mac_link_up(struct phylink_config *config, struct phy_device *phy,
1633				       unsigned int mode, phy_interface_t interface, int speed,
1634				       int duplex, bool tx_pause, bool rx_pause)
1635{
1636	struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
1637							  phylink_config);
1638	struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
1639	struct am65_cpsw_common *common = port->common;
1640	u32 mac_control = CPSW_SL_CTL_GMII_EN;
1641	struct net_device *ndev = port->ndev;
1642
1643	/* Bring the port out of idle state */
1644	cpsw_sl_ctl_clr(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
1645
1646	if (speed == SPEED_1000)
1647		mac_control |= CPSW_SL_CTL_GIG;
1648	/* TODO: Verify whether in-band is necessary for 10 Mbps RGMII */
1649	if (speed == SPEED_10 && phy_interface_mode_is_rgmii(interface))
1650		/* Can be used with in band mode only */
1651		mac_control |= CPSW_SL_CTL_EXT_EN;
1652	if (speed == SPEED_100 && interface == PHY_INTERFACE_MODE_RMII)
1653		mac_control |= CPSW_SL_CTL_IFCTL_A;
1654	if (duplex)
1655		mac_control |= CPSW_SL_CTL_FULLDUPLEX;
1656
1657	/* rx_pause/tx_pause */
1658	if (rx_pause)
1659		mac_control |= CPSW_SL_CTL_TX_FLOW_EN;
1660
1661	if (tx_pause)
1662		mac_control |= CPSW_SL_CTL_RX_FLOW_EN;
1663
1664	cpsw_sl_ctl_set(port->slave.mac_sl, mac_control);
1665
1666	/* enable forwarding */
1667	cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
1668
1669	am65_cpsw_qos_link_up(ndev, speed);
1670	netif_tx_wake_all_queues(ndev);
1671}
1672
1673static const struct phylink_mac_ops am65_cpsw_phylink_mac_ops = {
1674	.mac_config = am65_cpsw_nuss_mac_config,
1675	.mac_link_down = am65_cpsw_nuss_mac_link_down,
1676	.mac_link_up = am65_cpsw_nuss_mac_link_up,
1677};
1678
1679static void am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port *port)
1680{
1681	struct am65_cpsw_common *common = port->common;
1682
1683	if (!port->disabled)
1684		return;
1685
1686	cpsw_ale_control_set(common->ale, port->port_id,
1687			     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1688
1689	cpsw_sl_reset(port->slave.mac_sl, 100);
1690	cpsw_sl_ctl_reset(port->slave.mac_sl);
1691}
1692
1693static void am65_cpsw_nuss_free_tx_chns(void *data)
1694{
1695	struct am65_cpsw_common *common = data;
1696	int i;
1697
1698	for (i = 0; i < common->tx_ch_num; i++) {
1699		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1700
1701		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
1702			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
1703
1704		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
1705			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
1706
1707		memset(tx_chn, 0, sizeof(*tx_chn));
1708	}
1709}
1710
1711void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common)
1712{
1713	struct device *dev = common->dev;
1714	int i;
1715
1716	devm_remove_action(dev, am65_cpsw_nuss_free_tx_chns, common);
1717
1718	common->tx_ch_rate_msk = 0;
1719	for (i = 0; i < common->tx_ch_num; i++) {
1720		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1721
1722		if (tx_chn->irq)
1723			devm_free_irq(dev, tx_chn->irq, tx_chn);
1724
1725		netif_napi_del(&tx_chn->napi_tx);
1726
1727		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
1728			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
1729
1730		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
1731			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
1732
1733		memset(tx_chn, 0, sizeof(*tx_chn));
1734	}
1735}
1736
1737static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common)
1738{
1739	struct device *dev = common->dev;
1740	int i, ret = 0;
1741
1742	for (i = 0; i < common->tx_ch_num; i++) {
1743		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1744
1745		netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx,
1746				  am65_cpsw_nuss_tx_poll);
1747		hrtimer_init(&tx_chn->tx_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
1748		tx_chn->tx_hrtimer.function = &am65_cpsw_nuss_tx_timer_callback;
1749
1750		ret = devm_request_irq(dev, tx_chn->irq,
1751				       am65_cpsw_nuss_tx_irq,
1752				       IRQF_TRIGGER_HIGH,
1753				       tx_chn->tx_chn_name, tx_chn);
1754		if (ret) {
1755			dev_err(dev, "failure requesting tx%u irq %u, %d\n",
1756				tx_chn->id, tx_chn->irq, ret);
1757			goto err;
1758		}
1759	}
1760
1761err:
1762	return ret;
1763}
1764
1765static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common)
1766{
1767	u32  max_desc_num = ALIGN(AM65_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS);
1768	struct k3_udma_glue_tx_channel_cfg tx_cfg = { 0 };
1769	struct device *dev = common->dev;
1770	struct k3_ring_cfg ring_cfg = {
1771		.elm_size = K3_RINGACC_RING_ELSIZE_8,
1772		.mode = K3_RINGACC_RING_MODE_RING,
1773		.flags = 0
1774	};
1775	u32 hdesc_size;
1776	int i, ret = 0;
1777
1778	hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
1779					   AM65_CPSW_NAV_SW_DATA_SIZE);
1780
1781	tx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
1782	tx_cfg.tx_cfg = ring_cfg;
1783	tx_cfg.txcq_cfg = ring_cfg;
1784	tx_cfg.tx_cfg.size = max_desc_num;
1785	tx_cfg.txcq_cfg.size = max_desc_num;
1786
1787	for (i = 0; i < common->tx_ch_num; i++) {
1788		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1789
1790		snprintf(tx_chn->tx_chn_name,
1791			 sizeof(tx_chn->tx_chn_name), "tx%d", i);
1792
1793		spin_lock_init(&tx_chn->lock);
1794		tx_chn->common = common;
1795		tx_chn->id = i;
1796		tx_chn->descs_num = max_desc_num;
1797
1798		tx_chn->tx_chn =
1799			k3_udma_glue_request_tx_chn(dev,
1800						    tx_chn->tx_chn_name,
1801						    &tx_cfg);
1802		if (IS_ERR(tx_chn->tx_chn)) {
1803			ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn),
1804					    "Failed to request tx dma channel\n");
1805			goto err;
1806		}
1807		tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);
1808
1809		tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev,
1810								  tx_chn->descs_num,
1811								  hdesc_size,
1812								  tx_chn->tx_chn_name);
1813		if (IS_ERR(tx_chn->desc_pool)) {
1814			ret = PTR_ERR(tx_chn->desc_pool);
1815			dev_err(dev, "Failed to create poll %d\n", ret);
1816			goto err;
1817		}
1818
1819		tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);
1820		if (tx_chn->irq < 0) {
1821			dev_err(dev, "Failed to get tx dma irq %d\n",
1822				tx_chn->irq);
1823			ret = tx_chn->irq;
1824			goto err;
1825		}
1826
1827		snprintf(tx_chn->tx_chn_name,
1828			 sizeof(tx_chn->tx_chn_name), "%s-tx%d",
1829			 dev_name(dev), tx_chn->id);
1830	}
1831
1832	ret = am65_cpsw_nuss_ndev_add_tx_napi(common);
1833	if (ret) {
1834		dev_err(dev, "Failed to add tx NAPI %d\n", ret);
1835		goto err;
1836	}
1837
1838err:
1839	i = devm_add_action(dev, am65_cpsw_nuss_free_tx_chns, common);
1840	if (i) {
1841		dev_err(dev, "Failed to add free_tx_chns action %d\n", i);
1842		return i;
1843	}
1844
1845	return ret;
1846}
1847
1848static void am65_cpsw_nuss_free_rx_chns(void *data)
1849{
1850	struct am65_cpsw_common *common = data;
1851	struct am65_cpsw_rx_chn *rx_chn;
1852
1853	rx_chn = &common->rx_chns;
1854
1855	if (!IS_ERR_OR_NULL(rx_chn->desc_pool))
1856		k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
1857
1858	if (!IS_ERR_OR_NULL(rx_chn->rx_chn))
1859		k3_udma_glue_release_rx_chn(rx_chn->rx_chn);
1860}
1861
1862static void am65_cpsw_nuss_remove_rx_chns(void *data)
1863{
1864	struct am65_cpsw_common *common = data;
1865	struct am65_cpsw_rx_chn *rx_chn;
1866	struct device *dev = common->dev;
1867
1868	rx_chn = &common->rx_chns;
1869	devm_remove_action(dev, am65_cpsw_nuss_free_rx_chns, common);
1870
1871	if (!(rx_chn->irq < 0))
1872		devm_free_irq(dev, rx_chn->irq, common);
1873
1874	netif_napi_del(&common->napi_rx);
1875
1876	if (!IS_ERR_OR_NULL(rx_chn->desc_pool))
1877		k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
1878
1879	if (!IS_ERR_OR_NULL(rx_chn->rx_chn))
1880		k3_udma_glue_release_rx_chn(rx_chn->rx_chn);
1881
1882	common->rx_flow_id_base = -1;
1883}
1884
1885static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common)
1886{
1887	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
1888	struct k3_udma_glue_rx_channel_cfg rx_cfg = { 0 };
1889	u32  max_desc_num = AM65_CPSW_MAX_RX_DESC;
1890	struct device *dev = common->dev;
1891	u32 hdesc_size;
1892	u32 fdqring_id;
1893	int i, ret = 0;
1894
1895	hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
1896					   AM65_CPSW_NAV_SW_DATA_SIZE);
1897
1898	rx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
1899	rx_cfg.flow_id_num = AM65_CPSW_MAX_RX_FLOWS;
1900	rx_cfg.flow_id_base = common->rx_flow_id_base;
1901
1902	/* init all flows */
1903	rx_chn->dev = dev;
1904	rx_chn->descs_num = max_desc_num;
1905
1906	rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg);
1907	if (IS_ERR(rx_chn->rx_chn)) {
1908		ret = dev_err_probe(dev, PTR_ERR(rx_chn->rx_chn),
1909				    "Failed to request rx dma channel\n");
1910		goto err;
1911	}
1912	rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn);
1913
1914	rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev,
1915							  rx_chn->descs_num,
1916							  hdesc_size, "rx");
1917	if (IS_ERR(rx_chn->desc_pool)) {
1918		ret = PTR_ERR(rx_chn->desc_pool);
1919		dev_err(dev, "Failed to create rx poll %d\n", ret);
1920		goto err;
1921	}
1922
1923	common->rx_flow_id_base =
1924			k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);
1925	dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base);
1926
1927	fdqring_id = K3_RINGACC_RING_ID_ANY;
1928	for (i = 0; i < rx_cfg.flow_id_num; i++) {
1929		struct k3_ring_cfg rxring_cfg = {
1930			.elm_size = K3_RINGACC_RING_ELSIZE_8,
1931			.mode = K3_RINGACC_RING_MODE_RING,
1932			.flags = 0,
1933		};
1934		struct k3_ring_cfg fdqring_cfg = {
1935			.elm_size = K3_RINGACC_RING_ELSIZE_8,
1936			.flags = K3_RINGACC_RING_SHARED,
1937		};
1938		struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = {
1939			.rx_cfg = rxring_cfg,
1940			.rxfdq_cfg = fdqring_cfg,
1941			.ring_rxq_id = K3_RINGACC_RING_ID_ANY,
1942			.src_tag_lo_sel =
1943				K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
1944		};
1945
1946		rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
1947		rx_flow_cfg.rx_cfg.size = max_desc_num;
1948		rx_flow_cfg.rxfdq_cfg.size = max_desc_num;
1949		rx_flow_cfg.rxfdq_cfg.mode = common->pdata.fdqring_mode;
1950
1951		ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,
1952						i, &rx_flow_cfg);
1953		if (ret) {
1954			dev_err(dev, "Failed to init rx flow%d %d\n", i, ret);
1955			goto err;
1956		}
1957		if (!i)
1958			fdqring_id =
1959				k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,
1960								i);
1961
1962		rx_chn->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);
1963
1964		if (rx_chn->irq <= 0) {
1965			dev_err(dev, "Failed to get rx dma irq %d\n",
1966				rx_chn->irq);
1967			ret = -ENXIO;
1968			goto err;
1969		}
1970	}
1971
1972	netif_napi_add(common->dma_ndev, &common->napi_rx,
1973		       am65_cpsw_nuss_rx_poll);
1974	hrtimer_init(&common->rx_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
1975	common->rx_hrtimer.function = &am65_cpsw_nuss_rx_timer_callback;
1976
1977	ret = devm_request_irq(dev, rx_chn->irq,
1978			       am65_cpsw_nuss_rx_irq,
1979			       IRQF_TRIGGER_HIGH, dev_name(dev), common);
1980	if (ret) {
1981		dev_err(dev, "failure requesting rx irq %u, %d\n",
1982			rx_chn->irq, ret);
1983		goto err;
1984	}
1985
1986err:
1987	i = devm_add_action(dev, am65_cpsw_nuss_free_rx_chns, common);
1988	if (i) {
1989		dev_err(dev, "Failed to add free_rx_chns action %d\n", i);
1990		return i;
1991	}
1992
1993	return ret;
1994}
1995
1996static int am65_cpsw_nuss_init_host_p(struct am65_cpsw_common *common)
1997{
1998	struct am65_cpsw_host *host_p = am65_common_get_host(common);
1999
2000	host_p->common = common;
2001	host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE;
2002	host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE;
2003
2004	return 0;
2005}
2006
2007static int am65_cpsw_am654_get_efuse_macid(struct device_node *of_node,
2008					   int slave, u8 *mac_addr)
2009{
2010	u32 mac_lo, mac_hi, offset;
2011	struct regmap *syscon;
2012	int ret;
2013
2014	syscon = syscon_regmap_lookup_by_phandle(of_node, "ti,syscon-efuse");
2015	if (IS_ERR(syscon)) {
2016		if (PTR_ERR(syscon) == -ENODEV)
2017			return 0;
2018		return PTR_ERR(syscon);
2019	}
2020
2021	ret = of_property_read_u32_index(of_node, "ti,syscon-efuse", 1,
2022					 &offset);
2023	if (ret)
2024		return ret;
2025
2026	regmap_read(syscon, offset, &mac_lo);
2027	regmap_read(syscon, offset + 4, &mac_hi);
2028
2029	mac_addr[0] = (mac_hi >> 8) & 0xff;
2030	mac_addr[1] = mac_hi & 0xff;
2031	mac_addr[2] = (mac_lo >> 24) & 0xff;
2032	mac_addr[3] = (mac_lo >> 16) & 0xff;
2033	mac_addr[4] = (mac_lo >> 8) & 0xff;
2034	mac_addr[5] = mac_lo & 0xff;
2035
2036	return 0;
2037}
2038
2039static int am65_cpsw_init_cpts(struct am65_cpsw_common *common)
2040{
2041	struct device *dev = common->dev;
2042	struct device_node *node;
2043	struct am65_cpts *cpts;
2044	void __iomem *reg_base;
2045
2046	if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
2047		return 0;
2048
2049	node = of_get_child_by_name(dev->of_node, "cpts");
2050	if (!node) {
2051		dev_err(dev, "%s cpts not found\n", __func__);
2052		return -ENOENT;
2053	}
2054
2055	reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE;
2056	cpts = am65_cpts_create(dev, reg_base, node);
2057	if (IS_ERR(cpts)) {
2058		int ret = PTR_ERR(cpts);
2059
2060		of_node_put(node);
2061		dev_err(dev, "cpts create err %d\n", ret);
2062		return ret;
2063	}
2064	common->cpts = cpts;
2065	/* Forbid PM runtime if CPTS is running.
2066	 * K3 CPSWxG modules may completely lose context during ON->OFF
2067	 * transitions depending on integration.
2068	 * AM65x/J721E MCU CPSW2G: false
2069	 * J721E MAIN_CPSW9G: true
2070	 */
2071	pm_runtime_forbid(dev);
2072
2073	return 0;
2074}
2075
2076static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common)
2077{
2078	struct device_node *node, *port_np;
2079	struct device *dev = common->dev;
2080	int ret;
2081
2082	node = of_get_child_by_name(dev->of_node, "ethernet-ports");
2083	if (!node)
2084		return -ENOENT;
2085
2086	for_each_child_of_node(node, port_np) {
2087		struct am65_cpsw_port *port;
2088		u32 port_id;
2089
2090		/* it is not a slave port node, continue */
2091		if (strcmp(port_np->name, "port"))
2092			continue;
2093
2094		ret = of_property_read_u32(port_np, "reg", &port_id);
2095		if (ret < 0) {
2096			dev_err(dev, "%pOF error reading port_id %d\n",
2097				port_np, ret);
2098			goto of_node_put;
2099		}
2100
2101		if (!port_id || port_id > common->port_num) {
2102			dev_err(dev, "%pOF has invalid port_id %u %s\n",
2103				port_np, port_id, port_np->name);
2104			ret = -EINVAL;
2105			goto of_node_put;
2106		}
2107
2108		port = am65_common_get_port(common, port_id);
2109		port->port_id = port_id;
2110		port->common = common;
2111		port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE +
2112				  AM65_CPSW_NU_PORTS_OFFSET * (port_id);
2113		if (common->pdata.extra_modes)
2114			port->sgmii_base = common->ss_base + AM65_CPSW_SGMII_BASE * (port_id);
2115		port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE +
2116				  (AM65_CPSW_NU_STATS_PORT_OFFSET * port_id);
2117		port->name = of_get_property(port_np, "label", NULL);
2118		port->fetch_ram_base =
2119				common->cpsw_base + AM65_CPSW_NU_FRAM_BASE +
2120				(AM65_CPSW_NU_FRAM_PORT_OFFSET * (port_id - 1));
2121
2122		port->slave.mac_sl = cpsw_sl_get("am65", dev, port->port_base);
2123		if (IS_ERR(port->slave.mac_sl)) {
2124			ret = PTR_ERR(port->slave.mac_sl);
2125			goto of_node_put;
2126		}
2127
2128		port->disabled = !of_device_is_available(port_np);
2129		if (port->disabled) {
2130			common->disabled_ports_mask |= BIT(port->port_id);
2131			continue;
2132		}
2133
2134		port->slave.ifphy = devm_of_phy_get(dev, port_np, NULL);
2135		if (IS_ERR(port->slave.ifphy)) {
2136			ret = PTR_ERR(port->slave.ifphy);
2137			dev_err(dev, "%pOF error retrieving port phy: %d\n",
2138				port_np, ret);
2139			goto of_node_put;
2140		}
2141
2142		/* Initialize the Serdes PHY for the port */
2143		ret = am65_cpsw_init_serdes_phy(dev, port_np, port);
2144		if (ret)
2145			goto of_node_put;
2146
2147		port->slave.mac_only =
2148				of_property_read_bool(port_np, "ti,mac-only");
2149
2150		/* get phy/link info */
2151		port->slave.phy_node = port_np;
2152		ret = of_get_phy_mode(port_np, &port->slave.phy_if);
2153		if (ret) {
2154			dev_err(dev, "%pOF read phy-mode err %d\n",
2155				port_np, ret);
2156			goto of_node_put;
2157		}
2158
2159		ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET, port->slave.phy_if);
2160		if (ret)
2161			goto of_node_put;
2162
2163		ret = of_get_mac_address(port_np, port->slave.mac_addr);
2164		if (ret) {
2165			am65_cpsw_am654_get_efuse_macid(port_np,
2166							port->port_id,
2167							port->slave.mac_addr);
2168			if (!is_valid_ether_addr(port->slave.mac_addr)) {
2169				eth_random_addr(port->slave.mac_addr);
2170				dev_err(dev, "Use random MAC address\n");
2171			}
2172		}
2173
2174		/* Reset all Queue priorities to 0 */
2175		writel(0, port->port_base + AM65_CPSW_PN_REG_TX_PRI_MAP);
2176	}
2177	of_node_put(node);
2178
2179	/* is there at least one ext.port */
2180	if (!(~common->disabled_ports_mask & GENMASK(common->port_num, 1))) {
2181		dev_err(dev, "No Ext. port are available\n");
2182		return -ENODEV;
2183	}
2184
2185	return 0;
2186
2187of_node_put:
2188	of_node_put(port_np);
2189	of_node_put(node);
2190	return ret;
2191}
2192
2193static void am65_cpsw_pcpu_stats_free(void *data)
2194{
2195	struct am65_cpsw_ndev_stats __percpu *stats = data;
2196
2197	free_percpu(stats);
2198}
2199
2200static void am65_cpsw_nuss_phylink_cleanup(struct am65_cpsw_common *common)
2201{
2202	struct am65_cpsw_port *port;
2203	int i;
2204
2205	for (i = 0; i < common->port_num; i++) {
2206		port = &common->ports[i];
2207		if (port->slave.phylink)
2208			phylink_destroy(port->slave.phylink);
2209	}
2210}
2211
2212static int
2213am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
2214{
2215	struct am65_cpsw_ndev_priv *ndev_priv;
2216	struct device *dev = common->dev;
2217	struct am65_cpsw_port *port;
2218	struct phylink *phylink;
2219	int ret;
2220
2221	port = &common->ports[port_idx];
2222
2223	if (port->disabled)
2224		return 0;
2225
2226	/* alloc netdev */
2227	port->ndev = devm_alloc_etherdev_mqs(common->dev,
2228					     sizeof(struct am65_cpsw_ndev_priv),
2229					     AM65_CPSW_MAX_TX_QUEUES,
2230					     AM65_CPSW_MAX_RX_QUEUES);
2231	if (!port->ndev) {
2232		dev_err(dev, "error allocating slave net_device %u\n",
2233			port->port_id);
2234		return -ENOMEM;
2235	}
2236
2237	ndev_priv = netdev_priv(port->ndev);
2238	ndev_priv->port = port;
2239	ndev_priv->msg_enable = AM65_CPSW_DEBUG;
2240	mutex_init(&ndev_priv->mm_lock);
2241	port->qos.link_speed = SPEED_UNKNOWN;
2242	SET_NETDEV_DEV(port->ndev, dev);
2243
2244	eth_hw_addr_set(port->ndev, port->slave.mac_addr);
2245
2246	port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE;
2247	port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE -
2248			      (VLAN_ETH_HLEN + ETH_FCS_LEN);
2249	port->ndev->hw_features = NETIF_F_SG |
2250				  NETIF_F_RXCSUM |
2251				  NETIF_F_HW_CSUM |
2252				  NETIF_F_HW_TC;
2253	port->ndev->features = port->ndev->hw_features |
2254			       NETIF_F_HW_VLAN_CTAG_FILTER;
2255	port->ndev->vlan_features |=  NETIF_F_SG;
2256	port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops;
2257	port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave;
2258
2259	/* Configuring Phylink */
2260	port->slave.phylink_config.dev = &port->ndev->dev;
2261	port->slave.phylink_config.type = PHYLINK_NETDEV;
2262	port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 |
2263						      MAC_1000FD | MAC_5000FD;
2264	port->slave.phylink_config.mac_managed_pm = true; /* MAC does PM */
2265
2266	switch (port->slave.phy_if) {
2267	case PHY_INTERFACE_MODE_RGMII:
2268	case PHY_INTERFACE_MODE_RGMII_ID:
2269	case PHY_INTERFACE_MODE_RGMII_RXID:
2270	case PHY_INTERFACE_MODE_RGMII_TXID:
2271		phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces);
2272		break;
2273
2274	case PHY_INTERFACE_MODE_RMII:
2275		__set_bit(PHY_INTERFACE_MODE_RMII,
2276			  port->slave.phylink_config.supported_interfaces);
2277		break;
2278
2279	case PHY_INTERFACE_MODE_QSGMII:
2280	case PHY_INTERFACE_MODE_SGMII:
2281	case PHY_INTERFACE_MODE_USXGMII:
2282		if (common->pdata.extra_modes & BIT(port->slave.phy_if)) {
2283			__set_bit(port->slave.phy_if,
2284				  port->slave.phylink_config.supported_interfaces);
2285		} else {
2286			dev_err(dev, "selected phy-mode is not supported\n");
2287			return -EOPNOTSUPP;
2288		}
2289		break;
2290
2291	default:
2292		dev_err(dev, "selected phy-mode is not supported\n");
2293		return -EOPNOTSUPP;
2294	}
2295
2296	phylink = phylink_create(&port->slave.phylink_config,
2297				 of_node_to_fwnode(port->slave.phy_node),
2298				 port->slave.phy_if,
2299				 &am65_cpsw_phylink_mac_ops);
2300	if (IS_ERR(phylink))
2301		return PTR_ERR(phylink);
2302
2303	port->slave.phylink = phylink;
2304
2305	/* Disable TX checksum offload by default due to HW bug */
2306	if (common->pdata.quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM)
2307		port->ndev->features &= ~NETIF_F_HW_CSUM;
2308
2309	ndev_priv->stats = netdev_alloc_pcpu_stats(struct am65_cpsw_ndev_stats);
2310	if (!ndev_priv->stats)
2311		return -ENOMEM;
2312
2313	ret = devm_add_action_or_reset(dev, am65_cpsw_pcpu_stats_free,
2314				       ndev_priv->stats);
2315	if (ret)
2316		dev_err(dev, "failed to add percpu stat free action %d\n", ret);
2317
2318	if (!common->dma_ndev)
2319		common->dma_ndev = port->ndev;
2320
2321	return ret;
2322}
2323
2324static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common)
2325{
2326	int ret;
2327	int i;
2328
2329	for (i = 0; i < common->port_num; i++) {
2330		ret = am65_cpsw_nuss_init_port_ndev(common, i);
2331		if (ret)
2332			return ret;
2333	}
2334
2335	return ret;
2336}
2337
2338static void am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common *common)
2339{
2340	struct am65_cpsw_port *port;
2341	int i;
2342
2343	for (i = 0; i < common->port_num; i++) {
2344		port = &common->ports[i];
2345		if (port->ndev && port->ndev->reg_state == NETREG_REGISTERED)
2346			unregister_netdev(port->ndev);
2347	}
2348}
2349
2350static void am65_cpsw_port_offload_fwd_mark_update(struct am65_cpsw_common *common)
2351{
2352	int set_val = 0;
2353	int i;
2354
2355	if (common->br_members == (GENMASK(common->port_num, 1) & ~common->disabled_ports_mask))
2356		set_val = 1;
2357
2358	dev_dbg(common->dev, "set offload_fwd_mark %d\n", set_val);
2359
2360	for (i = 1; i <= common->port_num; i++) {
2361		struct am65_cpsw_port *port = am65_common_get_port(common, i);
2362		struct am65_cpsw_ndev_priv *priv;
2363
2364		if (!port->ndev)
2365			continue;
2366
2367		priv = am65_ndev_to_priv(port->ndev);
2368		priv->offload_fwd_mark = set_val;
2369	}
2370}
2371
2372bool am65_cpsw_port_dev_check(const struct net_device *ndev)
2373{
2374	if (ndev->netdev_ops == &am65_cpsw_nuss_netdev_ops) {
2375		struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2376
2377		return !common->is_emac_mode;
2378	}
2379
2380	return false;
2381}
2382
2383static int am65_cpsw_netdevice_port_link(struct net_device *ndev,
2384					 struct net_device *br_ndev,
2385					 struct netlink_ext_ack *extack)
2386{
2387	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2388	struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2389	int err;
2390
2391	if (!common->br_members) {
2392		common->hw_bridge_dev = br_ndev;
2393	} else {
2394		/* This is adding the port to a second bridge, this is
2395		 * unsupported
2396		 */
2397		if (common->hw_bridge_dev != br_ndev)
2398			return -EOPNOTSUPP;
2399	}
2400
2401	err = switchdev_bridge_port_offload(ndev, ndev, NULL, NULL, NULL,
2402					    false, extack);
2403	if (err)
2404		return err;
2405
2406	common->br_members |= BIT(priv->port->port_id);
2407
2408	am65_cpsw_port_offload_fwd_mark_update(common);
2409
2410	return NOTIFY_DONE;
2411}
2412
2413static void am65_cpsw_netdevice_port_unlink(struct net_device *ndev)
2414{
2415	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2416	struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2417
2418	switchdev_bridge_port_unoffload(ndev, NULL, NULL, NULL);
2419
2420	common->br_members &= ~BIT(priv->port->port_id);
2421
2422	am65_cpsw_port_offload_fwd_mark_update(common);
2423
2424	if (!common->br_members)
2425		common->hw_bridge_dev = NULL;
2426}
2427
2428/* netdev notifier */
2429static int am65_cpsw_netdevice_event(struct notifier_block *unused,
2430				     unsigned long event, void *ptr)
2431{
2432	struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr);
2433	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
2434	struct netdev_notifier_changeupper_info *info;
2435	int ret = NOTIFY_DONE;
2436
2437	if (!am65_cpsw_port_dev_check(ndev))
2438		return NOTIFY_DONE;
2439
2440	switch (event) {
2441	case NETDEV_CHANGEUPPER:
2442		info = ptr;
2443
2444		if (netif_is_bridge_master(info->upper_dev)) {
2445			if (info->linking)
2446				ret = am65_cpsw_netdevice_port_link(ndev,
2447								    info->upper_dev,
2448								    extack);
2449			else
2450				am65_cpsw_netdevice_port_unlink(ndev);
2451		}
2452		break;
2453	default:
2454		return NOTIFY_DONE;
2455	}
2456
2457	return notifier_from_errno(ret);
2458}
2459
2460static int am65_cpsw_register_notifiers(struct am65_cpsw_common *cpsw)
2461{
2462	int ret = 0;
2463
2464	if (AM65_CPSW_IS_CPSW2G(cpsw) ||
2465	    !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2466		return 0;
2467
2468	cpsw->am65_cpsw_netdevice_nb.notifier_call = &am65_cpsw_netdevice_event;
2469	ret = register_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
2470	if (ret) {
2471		dev_err(cpsw->dev, "can't register netdevice notifier\n");
2472		return ret;
2473	}
2474
2475	ret = am65_cpsw_switchdev_register_notifiers(cpsw);
2476	if (ret)
2477		unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
2478
2479	return ret;
2480}
2481
2482static void am65_cpsw_unregister_notifiers(struct am65_cpsw_common *cpsw)
2483{
2484	if (AM65_CPSW_IS_CPSW2G(cpsw) ||
2485	    !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2486		return;
2487
2488	am65_cpsw_switchdev_unregister_notifiers(cpsw);
2489	unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
2490}
2491
2492static const struct devlink_ops am65_cpsw_devlink_ops = {};
2493
2494static void am65_cpsw_init_stp_ale_entry(struct am65_cpsw_common *cpsw)
2495{
2496	cpsw_ale_add_mcast(cpsw->ale, eth_stp_addr, ALE_PORT_HOST, ALE_SUPER, 0,
2497			   ALE_MCAST_BLOCK_LEARN_FWD);
2498}
2499
2500static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common)
2501{
2502	struct am65_cpsw_host *host = am65_common_get_host(common);
2503
2504	writel(common->default_vlan, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2505
2506	am65_cpsw_init_stp_ale_entry(common);
2507
2508	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 1);
2509	dev_dbg(common->dev, "Set P0_UNI_FLOOD\n");
2510	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 0);
2511}
2512
2513static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common)
2514{
2515	struct am65_cpsw_host *host = am65_common_get_host(common);
2516
2517	writel(0, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2518
2519	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 0);
2520	dev_dbg(common->dev, "unset P0_UNI_FLOOD\n");
2521
2522	/* learning make no sense in multi-mac mode */
2523	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 1);
2524}
2525
2526static int am65_cpsw_dl_switch_mode_get(struct devlink *dl, u32 id,
2527					struct devlink_param_gset_ctx *ctx)
2528{
2529	struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
2530	struct am65_cpsw_common *common = dl_priv->common;
2531
2532	dev_dbg(common->dev, "%s id:%u\n", __func__, id);
2533
2534	if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
2535		return -EOPNOTSUPP;
2536
2537	ctx->val.vbool = !common->is_emac_mode;
2538
2539	return 0;
2540}
2541
2542static void am65_cpsw_init_port_emac_ale(struct  am65_cpsw_port *port)
2543{
2544	struct am65_cpsw_slave_data *slave = &port->slave;
2545	struct am65_cpsw_common *common = port->common;
2546	u32 port_mask;
2547
2548	writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2549
2550	if (slave->mac_only)
2551		/* enable mac-only mode on port */
2552		cpsw_ale_control_set(common->ale, port->port_id,
2553				     ALE_PORT_MACONLY, 1);
2554
2555	cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_NOLEARN, 1);
2556
2557	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
2558
2559	cpsw_ale_add_ucast(common->ale, port->ndev->dev_addr,
2560			   HOST_PORT_NUM, ALE_SECURE, slave->port_vlan);
2561	cpsw_ale_add_mcast(common->ale, port->ndev->broadcast,
2562			   port_mask, ALE_VLAN, slave->port_vlan, ALE_MCAST_FWD_2);
2563}
2564
2565static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port)
2566{
2567	struct am65_cpsw_slave_data *slave = &port->slave;
2568	struct am65_cpsw_common *cpsw = port->common;
2569	u32 port_mask;
2570
2571	cpsw_ale_control_set(cpsw->ale, port->port_id,
2572			     ALE_PORT_NOLEARN, 0);
2573
2574	cpsw_ale_add_ucast(cpsw->ale, port->ndev->dev_addr,
2575			   HOST_PORT_NUM, ALE_SECURE | ALE_BLOCKED | ALE_VLAN,
2576			   slave->port_vlan);
2577
2578	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
2579
2580	cpsw_ale_add_mcast(cpsw->ale, port->ndev->broadcast,
2581			   port_mask, ALE_VLAN, slave->port_vlan,
2582			   ALE_MCAST_FWD_2);
2583
2584	writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2585
2586	cpsw_ale_control_set(cpsw->ale, port->port_id,
2587			     ALE_PORT_MACONLY, 0);
2588}
2589
2590static int am65_cpsw_dl_switch_mode_set(struct devlink *dl, u32 id,
2591					struct devlink_param_gset_ctx *ctx)
2592{
2593	struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
2594	struct am65_cpsw_common *cpsw = dl_priv->common;
2595	bool switch_en = ctx->val.vbool;
2596	bool if_running = false;
2597	int i;
2598
2599	dev_dbg(cpsw->dev, "%s id:%u\n", __func__, id);
2600
2601	if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
2602		return -EOPNOTSUPP;
2603
2604	if (switch_en == !cpsw->is_emac_mode)
2605		return 0;
2606
2607	if (!switch_en && cpsw->br_members) {
2608		dev_err(cpsw->dev, "Remove ports from bridge before disabling switch mode\n");
2609		return -EINVAL;
2610	}
2611
2612	rtnl_lock();
2613
2614	cpsw->is_emac_mode = !switch_en;
2615
2616	for (i = 0; i < cpsw->port_num; i++) {
2617		struct net_device *sl_ndev = cpsw->ports[i].ndev;
2618
2619		if (!sl_ndev || !netif_running(sl_ndev))
2620			continue;
2621
2622		if_running = true;
2623	}
2624
2625	if (!if_running) {
2626		/* all ndevs are down */
2627		for (i = 0; i < cpsw->port_num; i++) {
2628			struct net_device *sl_ndev = cpsw->ports[i].ndev;
2629			struct am65_cpsw_slave_data *slave;
2630
2631			if (!sl_ndev)
2632				continue;
2633
2634			slave = am65_ndev_to_slave(sl_ndev);
2635			if (switch_en)
2636				slave->port_vlan = cpsw->default_vlan;
2637			else
2638				slave->port_vlan = 0;
2639		}
2640
2641		goto exit;
2642	}
2643
2644	cpsw_ale_control_set(cpsw->ale, 0, ALE_BYPASS, 1);
2645	/* clean up ALE table */
2646	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_CLEAR, 1);
2647	cpsw_ale_control_get(cpsw->ale, HOST_PORT_NUM, ALE_AGEOUT);
2648
2649	if (switch_en) {
2650		dev_info(cpsw->dev, "Enable switch mode\n");
2651
2652		am65_cpsw_init_host_port_switch(cpsw);
2653
2654		for (i = 0; i < cpsw->port_num; i++) {
2655			struct net_device *sl_ndev = cpsw->ports[i].ndev;
2656			struct am65_cpsw_slave_data *slave;
2657			struct am65_cpsw_port *port;
2658
2659			if (!sl_ndev)
2660				continue;
2661
2662			port = am65_ndev_to_port(sl_ndev);
2663			slave = am65_ndev_to_slave(sl_ndev);
2664			slave->port_vlan = cpsw->default_vlan;
2665
2666			if (netif_running(sl_ndev))
2667				am65_cpsw_init_port_switch_ale(port);
2668		}
2669
2670	} else {
2671		dev_info(cpsw->dev, "Disable switch mode\n");
2672
2673		am65_cpsw_init_host_port_emac(cpsw);
2674
2675		for (i = 0; i < cpsw->port_num; i++) {
2676			struct net_device *sl_ndev = cpsw->ports[i].ndev;
2677			struct am65_cpsw_port *port;
2678
2679			if (!sl_ndev)
2680				continue;
2681
2682			port = am65_ndev_to_port(sl_ndev);
2683			port->slave.port_vlan = 0;
2684			if (netif_running(sl_ndev))
2685				am65_cpsw_init_port_emac_ale(port);
2686		}
2687	}
2688	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_BYPASS, 0);
2689exit:
2690	rtnl_unlock();
2691
2692	return 0;
2693}
2694
2695static const struct devlink_param am65_cpsw_devlink_params[] = {
2696	DEVLINK_PARAM_DRIVER(AM65_CPSW_DL_PARAM_SWITCH_MODE, "switch_mode",
2697			     DEVLINK_PARAM_TYPE_BOOL,
2698			     BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2699			     am65_cpsw_dl_switch_mode_get,
2700			     am65_cpsw_dl_switch_mode_set, NULL),
2701};
2702
2703static int am65_cpsw_nuss_register_devlink(struct am65_cpsw_common *common)
2704{
2705	struct devlink_port_attrs attrs = {};
2706	struct am65_cpsw_devlink *dl_priv;
2707	struct device *dev = common->dev;
2708	struct devlink_port *dl_port;
2709	struct am65_cpsw_port *port;
2710	int ret = 0;
2711	int i;
2712
2713	common->devlink =
2714		devlink_alloc(&am65_cpsw_devlink_ops, sizeof(*dl_priv), dev);
2715	if (!common->devlink)
2716		return -ENOMEM;
2717
2718	dl_priv = devlink_priv(common->devlink);
2719	dl_priv->common = common;
2720
2721	/* Provide devlink hook to switch mode when multiple external ports
2722	 * are present NUSS switchdev driver is enabled.
2723	 */
2724	if (!AM65_CPSW_IS_CPSW2G(common) &&
2725	    IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) {
2726		ret = devlink_params_register(common->devlink,
2727					      am65_cpsw_devlink_params,
2728					      ARRAY_SIZE(am65_cpsw_devlink_params));
2729		if (ret) {
2730			dev_err(dev, "devlink params reg fail ret:%d\n", ret);
2731			goto dl_unreg;
2732		}
2733	}
2734
2735	for (i = 1; i <= common->port_num; i++) {
2736		port = am65_common_get_port(common, i);
2737		dl_port = &port->devlink_port;
2738
2739		if (port->ndev)
2740			attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
2741		else
2742			attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED;
2743		attrs.phys.port_number = port->port_id;
2744		attrs.switch_id.id_len = sizeof(resource_size_t);
2745		memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len);
2746		devlink_port_attrs_set(dl_port, &attrs);
2747
2748		ret = devlink_port_register(common->devlink, dl_port, port->port_id);
2749		if (ret) {
2750			dev_err(dev, "devlink_port reg fail for port %d, ret:%d\n",
2751				port->port_id, ret);
2752			goto dl_port_unreg;
2753		}
2754	}
2755	devlink_register(common->devlink);
2756	return ret;
2757
2758dl_port_unreg:
2759	for (i = i - 1; i >= 1; i--) {
2760		port = am65_common_get_port(common, i);
2761		dl_port = &port->devlink_port;
2762
2763		devlink_port_unregister(dl_port);
2764	}
2765dl_unreg:
2766	devlink_free(common->devlink);
2767	return ret;
2768}
2769
2770static void am65_cpsw_unregister_devlink(struct am65_cpsw_common *common)
2771{
2772	struct devlink_port *dl_port;
2773	struct am65_cpsw_port *port;
2774	int i;
2775
2776	devlink_unregister(common->devlink);
2777
2778	for (i = 1; i <= common->port_num; i++) {
2779		port = am65_common_get_port(common, i);
2780		dl_port = &port->devlink_port;
2781
2782		devlink_port_unregister(dl_port);
2783	}
2784
2785	if (!AM65_CPSW_IS_CPSW2G(common) &&
2786	    IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2787		devlink_params_unregister(common->devlink,
2788					  am65_cpsw_devlink_params,
2789					  ARRAY_SIZE(am65_cpsw_devlink_params));
2790
2791	devlink_free(common->devlink);
2792}
2793
2794static int am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common *common)
2795{
2796	struct device *dev = common->dev;
2797	struct am65_cpsw_port *port;
2798	int ret = 0, i;
2799
2800	/* init tx channels */
2801	ret = am65_cpsw_nuss_init_tx_chns(common);
2802	if (ret)
2803		return ret;
2804	ret = am65_cpsw_nuss_init_rx_chns(common);
2805	if (ret)
2806		return ret;
2807
2808	ret = am65_cpsw_nuss_register_devlink(common);
2809	if (ret)
2810		return ret;
2811
2812	for (i = 0; i < common->port_num; i++) {
2813		port = &common->ports[i];
2814
2815		if (!port->ndev)
2816			continue;
2817
2818		SET_NETDEV_DEVLINK_PORT(port->ndev, &port->devlink_port);
2819
2820		ret = register_netdev(port->ndev);
2821		if (ret) {
2822			dev_err(dev, "error registering slave net device%i %d\n",
2823				i, ret);
2824			goto err_cleanup_ndev;
2825		}
2826	}
2827
2828	ret = am65_cpsw_register_notifiers(common);
2829	if (ret)
2830		goto err_cleanup_ndev;
2831
2832	/* can't auto unregister ndev using devm_add_action() due to
2833	 * devres release sequence in DD core for DMA
2834	 */
2835
2836	return 0;
2837
2838err_cleanup_ndev:
2839	am65_cpsw_nuss_cleanup_ndev(common);
2840	am65_cpsw_unregister_devlink(common);
2841
2842	return ret;
2843}
2844
2845int am65_cpsw_nuss_update_tx_chns(struct am65_cpsw_common *common, int num_tx)
2846{
2847	int ret;
2848
2849	common->tx_ch_num = num_tx;
2850	ret = am65_cpsw_nuss_init_tx_chns(common);
2851
2852	return ret;
2853}
2854
2855struct am65_cpsw_soc_pdata {
2856	u32	quirks_dis;
2857};
2858
2859static const struct am65_cpsw_soc_pdata am65x_soc_sr2_0 = {
2860	.quirks_dis = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
2861};
2862
2863static const struct soc_device_attribute am65_cpsw_socinfo[] = {
2864	{ .family = "AM65X",
2865	  .revision = "SR2.0",
2866	  .data = &am65x_soc_sr2_0
2867	},
2868	{/* sentinel */}
2869};
2870
2871static const struct am65_cpsw_pdata am65x_sr1_0 = {
2872	.quirks = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
2873	.ale_dev_id = "am65x-cpsw2g",
2874	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2875};
2876
2877static const struct am65_cpsw_pdata j721e_pdata = {
2878	.quirks = 0,
2879	.ale_dev_id = "am65x-cpsw2g",
2880	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2881};
2882
2883static const struct am65_cpsw_pdata am64x_cpswxg_pdata = {
2884	.quirks = AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ,
2885	.ale_dev_id = "am64-cpswxg",
2886	.fdqring_mode = K3_RINGACC_RING_MODE_RING,
2887};
2888
2889static const struct am65_cpsw_pdata j7200_cpswxg_pdata = {
2890	.quirks = 0,
2891	.ale_dev_id = "am64-cpswxg",
2892	.fdqring_mode = K3_RINGACC_RING_MODE_RING,
2893	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII),
2894};
2895
2896static const struct am65_cpsw_pdata j721e_cpswxg_pdata = {
2897	.quirks = 0,
2898	.ale_dev_id = "am64-cpswxg",
2899	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2900	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII),
2901};
2902
2903static const struct am65_cpsw_pdata j784s4_cpswxg_pdata = {
2904	.quirks = 0,
2905	.ale_dev_id = "am64-cpswxg",
2906	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2907	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_USXGMII),
2908};
2909
2910static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
2911	{ .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0},
2912	{ .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata},
2913	{ .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata},
2914	{ .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata},
2915	{ .compatible = "ti,j721e-cpswxg-nuss", .data = &j721e_cpswxg_pdata},
2916	{ .compatible = "ti,j784s4-cpswxg-nuss", .data = &j784s4_cpswxg_pdata},
2917	{ /* sentinel */ },
2918};
2919MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
2920
2921static void am65_cpsw_nuss_apply_socinfo(struct am65_cpsw_common *common)
2922{
2923	const struct soc_device_attribute *soc;
2924
2925	soc = soc_device_match(am65_cpsw_socinfo);
2926	if (soc && soc->data) {
2927		const struct am65_cpsw_soc_pdata *socdata = soc->data;
2928
2929		/* disable quirks */
2930		common->pdata.quirks &= ~socdata->quirks_dis;
2931	}
2932}
2933
2934static int am65_cpsw_nuss_probe(struct platform_device *pdev)
2935{
2936	struct cpsw_ale_params ale_params = { 0 };
2937	const struct of_device_id *of_id;
2938	struct device *dev = &pdev->dev;
2939	struct am65_cpsw_common *common;
2940	struct device_node *node;
2941	struct resource *res;
2942	struct clk *clk;
2943	u64 id_temp;
2944	int ret, i;
2945	int ale_entries;
2946
2947	common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL);
2948	if (!common)
2949		return -ENOMEM;
2950	common->dev = dev;
2951
2952	of_id = of_match_device(am65_cpsw_nuss_of_mtable, dev);
2953	if (!of_id)
2954		return -EINVAL;
2955	common->pdata = *(const struct am65_cpsw_pdata *)of_id->data;
2956
2957	am65_cpsw_nuss_apply_socinfo(common);
2958
2959	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpsw_nuss");
2960	common->ss_base = devm_ioremap_resource(&pdev->dev, res);
2961	if (IS_ERR(common->ss_base))
2962		return PTR_ERR(common->ss_base);
2963	common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE;
2964	/* Use device's physical base address as switch id */
2965	id_temp = cpu_to_be64(res->start);
2966	memcpy(common->switch_id, &id_temp, sizeof(res->start));
2967
2968	node = of_get_child_by_name(dev->of_node, "ethernet-ports");
2969	if (!node)
2970		return -ENOENT;
2971	common->port_num = of_get_child_count(node);
2972	of_node_put(node);
2973	if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS)
2974		return -ENOENT;
2975
2976	common->rx_flow_id_base = -1;
2977	init_completion(&common->tdown_complete);
2978	common->tx_ch_num = AM65_CPSW_DEFAULT_TX_CHNS;
2979	common->pf_p0_rx_ptype_rrobin = false;
2980	common->default_vlan = 1;
2981
2982	common->ports = devm_kcalloc(dev, common->port_num,
2983				     sizeof(*common->ports),
2984				     GFP_KERNEL);
2985	if (!common->ports)
2986		return -ENOMEM;
2987
2988	clk = devm_clk_get(dev, "fck");
2989	if (IS_ERR(clk))
2990		return dev_err_probe(dev, PTR_ERR(clk), "getting fck clock\n");
2991	common->bus_freq = clk_get_rate(clk);
2992
2993	pm_runtime_enable(dev);
2994	ret = pm_runtime_resume_and_get(dev);
2995	if (ret < 0) {
2996		pm_runtime_disable(dev);
2997		return ret;
2998	}
2999
3000	node = of_get_child_by_name(dev->of_node, "mdio");
3001	if (!node) {
3002		dev_warn(dev, "MDIO node not found\n");
3003	} else if (of_device_is_available(node)) {
3004		struct platform_device *mdio_pdev;
3005
3006		mdio_pdev = of_platform_device_create(node, NULL, dev);
3007		if (!mdio_pdev) {
3008			ret = -ENODEV;
3009			goto err_pm_clear;
3010		}
3011
3012		common->mdio_dev =  &mdio_pdev->dev;
3013	}
3014	of_node_put(node);
3015
3016	am65_cpsw_nuss_get_ver(common);
3017
3018	ret = am65_cpsw_nuss_init_host_p(common);
3019	if (ret)
3020		goto err_of_clear;
3021
3022	ret = am65_cpsw_nuss_init_slave_ports(common);
3023	if (ret)
3024		goto err_of_clear;
3025
3026	/* init common data */
3027	ale_params.dev = dev;
3028	ale_params.ale_ageout = AM65_CPSW_ALE_AGEOUT_DEFAULT;
3029	ale_params.ale_ports = common->port_num + 1;
3030	ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE;
3031	ale_params.dev_id = common->pdata.ale_dev_id;
3032	ale_params.bus_freq = common->bus_freq;
3033
3034	common->ale = cpsw_ale_create(&ale_params);
3035	if (IS_ERR(common->ale)) {
3036		dev_err(dev, "error initializing ale engine\n");
3037		ret = PTR_ERR(common->ale);
3038		goto err_of_clear;
3039	}
3040
3041	ale_entries = common->ale->params.ale_entries;
3042	common->ale_context = devm_kzalloc(dev,
3043					   ale_entries * ALE_ENTRY_WORDS * sizeof(u32),
3044					   GFP_KERNEL);
3045	ret = am65_cpsw_init_cpts(common);
3046	if (ret)
3047		goto err_of_clear;
3048
3049	/* init ports */
3050	for (i = 0; i < common->port_num; i++)
3051		am65_cpsw_nuss_slave_disable_unused(&common->ports[i]);
3052
3053	dev_set_drvdata(dev, common);
3054
3055	common->is_emac_mode = true;
3056
3057	ret = am65_cpsw_nuss_init_ndevs(common);
3058	if (ret)
3059		goto err_free_phylink;
3060
3061	ret = am65_cpsw_nuss_register_ndevs(common);
3062	if (ret)
3063		goto err_free_phylink;
3064
3065	pm_runtime_put(dev);
3066	return 0;
3067
3068err_free_phylink:
3069	am65_cpsw_nuss_phylink_cleanup(common);
3070	am65_cpts_release(common->cpts);
3071err_of_clear:
3072	if (common->mdio_dev)
3073		of_platform_device_destroy(common->mdio_dev, NULL);
3074err_pm_clear:
3075	pm_runtime_put_sync(dev);
3076	pm_runtime_disable(dev);
3077	return ret;
3078}
3079
3080static void am65_cpsw_nuss_remove(struct platform_device *pdev)
3081{
3082	struct device *dev = &pdev->dev;
3083	struct am65_cpsw_common *common;
3084	int ret;
3085
3086	common = dev_get_drvdata(dev);
3087
3088	ret = pm_runtime_resume_and_get(&pdev->dev);
3089	if (ret < 0) {
3090		/* Note, if this error path is taken, we're leaking some
3091		 * resources.
3092		 */
3093		dev_err(&pdev->dev, "Failed to resume device (%pe)\n",
3094			ERR_PTR(ret));
3095		return;
3096	}
3097
3098	am65_cpsw_unregister_devlink(common);
3099	am65_cpsw_unregister_notifiers(common);
3100
3101	/* must unregister ndevs here because DD release_driver routine calls
3102	 * dma_deconfigure(dev) before devres_release_all(dev)
3103	 */
3104	am65_cpsw_nuss_cleanup_ndev(common);
3105	am65_cpsw_nuss_phylink_cleanup(common);
3106	am65_cpts_release(common->cpts);
3107	am65_cpsw_disable_serdes_phy(common);
3108
3109	if (common->mdio_dev)
3110		of_platform_device_destroy(common->mdio_dev, NULL);
3111
3112	pm_runtime_put_sync(&pdev->dev);
3113	pm_runtime_disable(&pdev->dev);
3114}
3115
3116static int am65_cpsw_nuss_suspend(struct device *dev)
3117{
3118	struct am65_cpsw_common *common = dev_get_drvdata(dev);
3119	struct am65_cpsw_host *host_p = am65_common_get_host(common);
3120	struct am65_cpsw_port *port;
3121	struct net_device *ndev;
3122	int i, ret;
3123
3124	cpsw_ale_dump(common->ale, common->ale_context);
3125	host_p->vid_context = readl(host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3126	for (i = 0; i < common->port_num; i++) {
3127		port = &common->ports[i];
3128		ndev = port->ndev;
3129
3130		if (!ndev)
3131			continue;
3132
3133		port->vid_context = readl(port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3134		netif_device_detach(ndev);
3135		if (netif_running(ndev)) {
3136			rtnl_lock();
3137			ret = am65_cpsw_nuss_ndo_slave_stop(ndev);
3138			rtnl_unlock();
3139			if (ret < 0) {
3140				netdev_err(ndev, "failed to stop: %d", ret);
3141				return ret;
3142			}
3143		}
3144	}
3145
3146	am65_cpts_suspend(common->cpts);
3147
3148	am65_cpsw_nuss_remove_rx_chns(common);
3149	am65_cpsw_nuss_remove_tx_chns(common);
3150
3151	return 0;
3152}
3153
3154static int am65_cpsw_nuss_resume(struct device *dev)
3155{
3156	struct am65_cpsw_common *common = dev_get_drvdata(dev);
3157	struct am65_cpsw_port *port;
3158	struct net_device *ndev;
3159	int i, ret;
3160	struct am65_cpsw_host *host_p = am65_common_get_host(common);
3161
3162	ret = am65_cpsw_nuss_init_tx_chns(common);
3163	if (ret)
3164		return ret;
3165	ret = am65_cpsw_nuss_init_rx_chns(common);
3166	if (ret)
3167		return ret;
3168
3169	/* If RX IRQ was disabled before suspend, keep it disabled */
3170	if (common->rx_irq_disabled)
3171		disable_irq(common->rx_chns.irq);
3172
3173	am65_cpts_resume(common->cpts);
3174
3175	for (i = 0; i < common->port_num; i++) {
3176		port = &common->ports[i];
3177		ndev = port->ndev;
3178
3179		if (!ndev)
3180			continue;
3181
3182		if (netif_running(ndev)) {
3183			rtnl_lock();
3184			ret = am65_cpsw_nuss_ndo_slave_open(ndev);
3185			rtnl_unlock();
3186			if (ret < 0) {
3187				netdev_err(ndev, "failed to start: %d", ret);
3188				return ret;
3189			}
3190		}
3191
3192		netif_device_attach(ndev);
3193		writel(port->vid_context, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3194	}
3195
3196	writel(host_p->vid_context, host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3197	cpsw_ale_restore(common->ale, common->ale_context);
3198
3199	return 0;
3200}
3201
3202static const struct dev_pm_ops am65_cpsw_nuss_dev_pm_ops = {
3203	SYSTEM_SLEEP_PM_OPS(am65_cpsw_nuss_suspend, am65_cpsw_nuss_resume)
3204};
3205
3206static struct platform_driver am65_cpsw_nuss_driver = {
3207	.driver = {
3208		.name	 = AM65_CPSW_DRV_NAME,
3209		.of_match_table = am65_cpsw_nuss_of_mtable,
3210		.pm = &am65_cpsw_nuss_dev_pm_ops,
3211	},
3212	.probe = am65_cpsw_nuss_probe,
3213	.remove_new = am65_cpsw_nuss_remove,
3214};
3215
3216module_platform_driver(am65_cpsw_nuss_driver);
3217
3218MODULE_LICENSE("GPL v2");
3219MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
3220MODULE_DESCRIPTION("TI AM65 CPSW Ethernet driver");