1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2/* Copyright 2017-2019 NXP */
   3
   4#include "enetc.h"
   5#include <linux/bpf_trace.h>
   6#include <linux/tcp.h>
   7#include <linux/udp.h>
 
   8#include <linux/vmalloc.h>
   9#include <linux/ptp_classify.h>
  10#include <net/ip6_checksum.h>
  11#include <net/pkt_sched.h>
  12#include <net/tso.h>
  13
  14u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
  15{
  16	return enetc_port_rd(&si->hw, reg);
  17}
  18EXPORT_SYMBOL_GPL(enetc_port_mac_rd);
  19
  20void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
  21{
  22	enetc_port_wr(&si->hw, reg, val);
  23	if (si->hw_features & ENETC_SI_F_QBU)
  24		enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val);
  25}
  26EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
  27
  28static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
  29					 u8 preemptible_tcs)
  30{
  31	priv->preemptible_tcs = preemptible_tcs;
  32	enetc_mm_commit_preemptible_tcs(priv);
  33}
  34
  35static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
  36{
  37	int num_tx_rings = priv->num_tx_rings;
  38
  39	if (priv->xdp_prog)
  40		return num_tx_rings - num_possible_cpus();
 
  41
  42	return num_tx_rings;
  43}
 
 
 
  44
  45static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
  46							struct enetc_bdr *tx_ring)
  47{
  48	int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
  49
  50	return priv->rx_ring[index];
  51}
  52
  53static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
  54{
  55	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
  56		return NULL;
  57
  58	return tx_swbd->skb;
 
 
  59}
  60
  61static struct xdp_frame *
  62enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
  63{
  64	if (tx_swbd->is_xdp_redirect)
  65		return tx_swbd->xdp_frame;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  66
  67	return NULL;
 
 
 
 
 
 
 
 
 
 
 
  68}
  69
  70static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
  71				struct enetc_tx_swbd *tx_swbd)
  72{
  73	/* For XDP_TX, pages come from RX, whereas for the other contexts where
  74	 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
  75	 * to match the DMA mapping length, so we need to differentiate those.
  76	 */
  77	if (tx_swbd->is_dma_page)
  78		dma_unmap_page(tx_ring->dev, tx_swbd->dma,
  79			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
  80			       tx_swbd->dir);
  81	else
  82		dma_unmap_single(tx_ring->dev, tx_swbd->dma,
  83				 tx_swbd->len, tx_swbd->dir);
  84	tx_swbd->dma = 0;
  85}
  86
  87static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
  88				struct enetc_tx_swbd *tx_swbd)
  89{
  90	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
  91	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
  92
  93	if (tx_swbd->dma)
  94		enetc_unmap_tx_buff(tx_ring, tx_swbd);
  95
  96	if (xdp_frame) {
  97		xdp_return_frame(tx_swbd->xdp_frame);
  98		tx_swbd->xdp_frame = NULL;
  99	} else if (skb) {
 100		dev_kfree_skb_any(skb);
 101		tx_swbd->skb = NULL;
 102	}
 103}
 104
 105/* Let H/W know BD ring has been updated */
 106static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
 107{
 108	/* includes wmb() */
 109	enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
 110}
 111
 112static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
 113			   u8 *msgtype, u8 *twostep,
 114			   u16 *correction_offset, u16 *body_offset)
 115{
 116	unsigned int ptp_class;
 117	struct ptp_header *hdr;
 118	unsigned int type;
 119	u8 *base;
 120
 121	ptp_class = ptp_classify_raw(skb);
 122	if (ptp_class == PTP_CLASS_NONE)
 123		return -EINVAL;
 124
 125	hdr = ptp_parse_header(skb, ptp_class);
 126	if (!hdr)
 127		return -EINVAL;
 128
 129	type = ptp_class & PTP_CLASS_PMASK;
 130	if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
 131		*udp = 1;
 132	else
 133		*udp = 0;
 134
 135	*msgtype = ptp_get_msgtype(hdr, ptp_class);
 136	*twostep = hdr->flag_field[0] & 0x2;
 137
 138	base = skb_mac_header(skb);
 139	*correction_offset = (u8 *)&hdr->correction - base;
 140	*body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
 141
 142	return 0;
 143}
 144
 145static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
 146{
 147	bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
 148	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
 149	struct enetc_hw *hw = &priv->si->hw;
 150	struct enetc_tx_swbd *tx_swbd;
 
 151	int len = skb_headlen(skb);
 152	union enetc_tx_bd temp_bd;
 153	u8 msgtype, twostep, udp;
 154	union enetc_tx_bd *txbd;
 155	u16 offset1, offset2;
 156	int i, count = 0;
 157	skb_frag_t *frag;
 158	unsigned int f;
 159	dma_addr_t dma;
 160	u8 flags = 0;
 161
 162	i = tx_ring->next_to_use;
 163	txbd = ENETC_TXBD(*tx_ring, i);
 164	prefetchw(txbd);
 165
 166	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
 167	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
 168		goto dma_err;
 169
 170	temp_bd.addr = cpu_to_le64(dma);
 171	temp_bd.buf_len = cpu_to_le16(len);
 172	temp_bd.lstatus = 0;
 173
 174	tx_swbd = &tx_ring->tx_swbd[i];
 175	tx_swbd->dma = dma;
 176	tx_swbd->len = len;
 177	tx_swbd->is_dma_page = 0;
 178	tx_swbd->dir = DMA_TO_DEVICE;
 179	count++;
 180
 181	do_vlan = skb_vlan_tag_present(skb);
 182	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 183		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
 184				    &offset2) ||
 185		    msgtype != PTP_MSGTYPE_SYNC || twostep)
 186			WARN_ONCE(1, "Bad packet for one-step timestamping\n");
 187		else
 188			do_onestep_tstamp = true;
 189	} else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
 190		do_twostep_tstamp = true;
 191	}
 192
 193	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
 194	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
 195	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
 196
 197	if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
 198		flags |= ENETC_TXBD_FLAGS_EX;
 199
 200	if (tx_ring->tsd_enable)
 
 
 201		flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
 202
 203	/* first BD needs frm_len and offload flags set */
 204	temp_bd.frm_len = cpu_to_le16(skb->len);
 205	temp_bd.flags = flags;
 206
 207	if (flags & ENETC_TXBD_FLAGS_TSE)
 208		temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
 209							  flags);
 
 
 
 
 210
 211	if (flags & ENETC_TXBD_FLAGS_EX) {
 212		u8 e_flags = 0;
 213		*txbd = temp_bd;
 214		enetc_clear_tx_bd(&temp_bd);
 215
 216		/* add extension BD for VLAN and/or timestamping */
 217		flags = 0;
 218		tx_swbd++;
 219		txbd++;
 220		i++;
 221		if (unlikely(i == tx_ring->bd_count)) {
 222			i = 0;
 223			tx_swbd = tx_ring->tx_swbd;
 224			txbd = ENETC_TXBD(*tx_ring, 0);
 225		}
 226		prefetchw(txbd);
 227
 228		if (do_vlan) {
 229			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
 230			temp_bd.ext.tpid = 0; /* < C-TAG */
 231			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
 232		}
 233
 234		if (do_onestep_tstamp) {
 235			u32 lo, hi, val;
 236			u64 sec, nsec;
 237			u8 *data;
 238
 239			lo = enetc_rd_hot(hw, ENETC_SICTR0);
 240			hi = enetc_rd_hot(hw, ENETC_SICTR1);
 241			sec = (u64)hi << 32 | lo;
 242			nsec = do_div(sec, 1000000000);
 243
 244			/* Configure extension BD */
 245			temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
 246			e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
 247
 248			/* Update originTimestamp field of Sync packet
 249			 * - 48 bits seconds field
 250			 * - 32 bits nanseconds field
 251			 */
 252			data = skb_mac_header(skb);
 253			*(__be16 *)(data + offset2) =
 254				htons((sec >> 32) & 0xffff);
 255			*(__be32 *)(data + offset2 + 2) =
 256				htonl(sec & 0xffffffff);
 257			*(__be32 *)(data + offset2 + 6) = htonl(nsec);
 258
 259			/* Configure single-step register */
 260			val = ENETC_PM0_SINGLE_STEP_EN;
 261			val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
 262			if (udp)
 263				val |= ENETC_PM0_SINGLE_STEP_CH;
 264
 265			enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP,
 266					  val);
 267		} else if (do_twostep_tstamp) {
 268			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 269			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
 270		}
 271
 272		temp_bd.ext.e_flags = e_flags;
 273		count++;
 274	}
 275
 276	frag = &skb_shinfo(skb)->frags[0];
 277	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
 278		len = skb_frag_size(frag);
 279		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
 280				       DMA_TO_DEVICE);
 281		if (dma_mapping_error(tx_ring->dev, dma))
 282			goto dma_err;
 283
 284		*txbd = temp_bd;
 285		enetc_clear_tx_bd(&temp_bd);
 286
 287		flags = 0;
 288		tx_swbd++;
 289		txbd++;
 290		i++;
 291		if (unlikely(i == tx_ring->bd_count)) {
 292			i = 0;
 293			tx_swbd = tx_ring->tx_swbd;
 294			txbd = ENETC_TXBD(*tx_ring, 0);
 295		}
 296		prefetchw(txbd);
 297
 298		temp_bd.addr = cpu_to_le64(dma);
 299		temp_bd.buf_len = cpu_to_le16(len);
 300
 301		tx_swbd->dma = dma;
 302		tx_swbd->len = len;
 303		tx_swbd->is_dma_page = 1;
 304		tx_swbd->dir = DMA_TO_DEVICE;
 305		count++;
 306	}
 307
 308	/* last BD needs 'F' bit set */
 309	flags |= ENETC_TXBD_FLAGS_F;
 310	temp_bd.flags = flags;
 311	*txbd = temp_bd;
 312
 313	tx_ring->tx_swbd[i].is_eof = true;
 314	tx_ring->tx_swbd[i].skb = skb;
 315
 316	enetc_bdr_idx_inc(tx_ring, &i);
 317	tx_ring->next_to_use = i;
 318
 319	skb_tx_timestamp(skb);
 320
 321	enetc_update_tx_ring_tail(tx_ring);
 
 322
 323	return count;
 324
 325dma_err:
 326	dev_err(tx_ring->dev, "DMA map error");
 327
 328	do {
 329		tx_swbd = &tx_ring->tx_swbd[i];
 330		enetc_free_tx_frame(tx_ring, tx_swbd);
 331		if (i == 0)
 332			i = tx_ring->bd_count;
 333		i--;
 334	} while (count--);
 335
 336	return 0;
 337}
 338
 339static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
 340				 struct enetc_tx_swbd *tx_swbd,
 341				 union enetc_tx_bd *txbd, int *i, int hdr_len,
 342				 int data_len)
 343{
 344	union enetc_tx_bd txbd_tmp;
 345	u8 flags = 0, e_flags = 0;
 346	dma_addr_t addr;
 347
 348	enetc_clear_tx_bd(&txbd_tmp);
 349	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
 350
 351	if (skb_vlan_tag_present(skb))
 352		flags |= ENETC_TXBD_FLAGS_EX;
 353
 354	txbd_tmp.addr = cpu_to_le64(addr);
 355	txbd_tmp.buf_len = cpu_to_le16(hdr_len);
 356
 357	/* first BD needs frm_len and offload flags set */
 358	txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
 359	txbd_tmp.flags = flags;
 360
 361	/* For the TSO header we do not set the dma address since we do not
 362	 * want it unmapped when we do cleanup. We still set len so that we
 363	 * count the bytes sent.
 364	 */
 365	tx_swbd->len = hdr_len;
 366	tx_swbd->do_twostep_tstamp = false;
 367	tx_swbd->check_wb = false;
 368
 369	/* Actually write the header in the BD */
 370	*txbd = txbd_tmp;
 371
 372	/* Add extension BD for VLAN */
 373	if (flags & ENETC_TXBD_FLAGS_EX) {
 374		/* Get the next BD */
 375		enetc_bdr_idx_inc(tx_ring, i);
 376		txbd = ENETC_TXBD(*tx_ring, *i);
 377		tx_swbd = &tx_ring->tx_swbd[*i];
 378		prefetchw(txbd);
 379
 380		/* Setup the VLAN fields */
 381		enetc_clear_tx_bd(&txbd_tmp);
 382		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
 383		txbd_tmp.ext.tpid = 0; /* < C-TAG */
 384		e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
 385
 386		/* Write the BD */
 387		txbd_tmp.ext.e_flags = e_flags;
 388		*txbd = txbd_tmp;
 389	}
 390}
 391
 392static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
 393				 struct enetc_tx_swbd *tx_swbd,
 394				 union enetc_tx_bd *txbd, char *data,
 395				 int size, bool last_bd)
 396{
 397	union enetc_tx_bd txbd_tmp;
 398	dma_addr_t addr;
 399	u8 flags = 0;
 400
 401	enetc_clear_tx_bd(&txbd_tmp);
 402
 403	addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
 404	if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
 405		netdev_err(tx_ring->ndev, "DMA map error\n");
 406		return -ENOMEM;
 407	}
 408
 409	if (last_bd) {
 410		flags |= ENETC_TXBD_FLAGS_F;
 411		tx_swbd->is_eof = 1;
 412	}
 413
 414	txbd_tmp.addr = cpu_to_le64(addr);
 415	txbd_tmp.buf_len = cpu_to_le16(size);
 416	txbd_tmp.flags = flags;
 417
 418	tx_swbd->dma = addr;
 419	tx_swbd->len = size;
 420	tx_swbd->dir = DMA_TO_DEVICE;
 421
 422	*txbd = txbd_tmp;
 423
 424	return 0;
 425}
 426
 427static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
 428				 char *hdr, int hdr_len, int *l4_hdr_len)
 429{
 430	char *l4_hdr = hdr + skb_transport_offset(skb);
 431	int mac_hdr_len = skb_network_offset(skb);
 432
 433	if (tso->tlen != sizeof(struct udphdr)) {
 434		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
 435
 436		tcph->check = 0;
 437	} else {
 438		struct udphdr *udph = (struct udphdr *)(l4_hdr);
 439
 440		udph->check = 0;
 441	}
 442
 443	/* Compute the IP checksum. This is necessary since tso_build_hdr()
 444	 * already incremented the IP ID field.
 445	 */
 446	if (!tso->ipv6) {
 447		struct iphdr *iph = (void *)(hdr + mac_hdr_len);
 448
 449		iph->check = 0;
 450		iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
 451	}
 452
 453	/* Compute the checksum over the L4 header. */
 454	*l4_hdr_len = hdr_len - skb_transport_offset(skb);
 455	return csum_partial(l4_hdr, *l4_hdr_len, 0);
 456}
 457
 458static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
 459				    struct sk_buff *skb, char *hdr, int len,
 460				    __wsum sum)
 461{
 462	char *l4_hdr = hdr + skb_transport_offset(skb);
 463	__sum16 csum_final;
 464
 465	/* Complete the L4 checksum by appending the pseudo-header to the
 466	 * already computed checksum.
 467	 */
 468	if (!tso->ipv6)
 469		csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
 470					       ip_hdr(skb)->daddr,
 471					       len, ip_hdr(skb)->protocol, sum);
 472	else
 473		csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
 474					     &ipv6_hdr(skb)->daddr,
 475					     len, ipv6_hdr(skb)->nexthdr, sum);
 476
 477	if (tso->tlen != sizeof(struct udphdr)) {
 478		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
 479
 480		tcph->check = csum_final;
 481	} else {
 482		struct udphdr *udph = (struct udphdr *)(l4_hdr);
 483
 484		udph->check = csum_final;
 485	}
 486}
 487
 488static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
 489{
 490	int hdr_len, total_len, data_len;
 491	struct enetc_tx_swbd *tx_swbd;
 492	union enetc_tx_bd *txbd;
 493	struct tso_t tso;
 494	__wsum csum, csum2;
 495	int count = 0, pos;
 496	int err, i, bd_data_num;
 497
 498	/* Initialize the TSO handler, and prepare the first payload */
 499	hdr_len = tso_start(skb, &tso);
 500	total_len = skb->len - hdr_len;
 501	i = tx_ring->next_to_use;
 502
 503	while (total_len > 0) {
 504		char *hdr;
 505
 506		/* Get the BD */
 507		txbd = ENETC_TXBD(*tx_ring, i);
 508		tx_swbd = &tx_ring->tx_swbd[i];
 509		prefetchw(txbd);
 510
 511		/* Determine the length of this packet */
 512		data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
 513		total_len -= data_len;
 514
 515		/* prepare packet headers: MAC + IP + TCP */
 516		hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
 517		tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
 518
 519		/* compute the csum over the L4 header */
 520		csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
 521		enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);
 522		bd_data_num = 0;
 523		count++;
 524
 525		while (data_len > 0) {
 526			int size;
 527
 528			size = min_t(int, tso.size, data_len);
 529
 530			/* Advance the index in the BDR */
 531			enetc_bdr_idx_inc(tx_ring, &i);
 532			txbd = ENETC_TXBD(*tx_ring, i);
 533			tx_swbd = &tx_ring->tx_swbd[i];
 534			prefetchw(txbd);
 535
 536			/* Compute the checksum over this segment of data and
 537			 * add it to the csum already computed (over the L4
 538			 * header and possible other data segments).
 539			 */
 540			csum2 = csum_partial(tso.data, size, 0);
 541			csum = csum_block_add(csum, csum2, pos);
 542			pos += size;
 543
 544			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
 545						    tso.data, size,
 546						    size == data_len);
 547			if (err)
 548				goto err_map_data;
 549
 550			data_len -= size;
 551			count++;
 552			bd_data_num++;
 553			tso_build_data(skb, &tso, size);
 554
 555			if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))
 556				goto err_chained_bd;
 557		}
 558
 559		enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
 560
 561		if (total_len == 0)
 562			tx_swbd->skb = skb;
 563
 564		/* Go to the next BD */
 565		enetc_bdr_idx_inc(tx_ring, &i);
 566	}
 567
 568	tx_ring->next_to_use = i;
 569	enetc_update_tx_ring_tail(tx_ring);
 570
 571	return count;
 572
 573err_map_data:
 574	dev_err(tx_ring->dev, "DMA map error");
 575
 576err_chained_bd:
 577	do {
 578		tx_swbd = &tx_ring->tx_swbd[i];
 579		enetc_free_tx_frame(tx_ring, tx_swbd);
 580		if (i == 0)
 581			i = tx_ring->bd_count;
 582		i--;
 583	} while (count--);
 584
 585	return 0;
 586}
 587
 588static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
 589				    struct net_device *ndev)
 590{
 591	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 592	struct enetc_bdr *tx_ring;
 593	int count, err;
 594
 595	/* Queue one-step Sync packet if already locked */
 596	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 597		if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
 598					  &priv->flags)) {
 599			skb_queue_tail(&priv->tx_skbs, skb);
 600			return NETDEV_TX_OK;
 601		}
 602	}
 603
 604	tx_ring = priv->tx_ring[skb->queue_mapping];
 605
 606	if (skb_is_gso(skb)) {
 607		if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
 608			netif_stop_subqueue(ndev, tx_ring->index);
 609			return NETDEV_TX_BUSY;
 610		}
 611
 612		enetc_lock_mdio();
 613		count = enetc_map_tx_tso_buffs(tx_ring, skb);
 614		enetc_unlock_mdio();
 615	} else {
 616		if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
 617			if (unlikely(skb_linearize(skb)))
 618				goto drop_packet_err;
 619
 620		count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
 621		if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
 622			netif_stop_subqueue(ndev, tx_ring->index);
 623			return NETDEV_TX_BUSY;
 624		}
 625
 626		if (skb->ip_summed == CHECKSUM_PARTIAL) {
 627			err = skb_checksum_help(skb);
 628			if (err)
 629				goto drop_packet_err;
 630		}
 631		enetc_lock_mdio();
 632		count = enetc_map_tx_buffs(tx_ring, skb);
 633		enetc_unlock_mdio();
 634	}
 635
 636	if (unlikely(!count))
 637		goto drop_packet_err;
 638
 639	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
 640		netif_stop_subqueue(ndev, tx_ring->index);
 641
 642	return NETDEV_TX_OK;
 643
 644drop_packet_err:
 645	dev_kfree_skb_any(skb);
 646	return NETDEV_TX_OK;
 647}
 648
 649netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
 650{
 651	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 652	u8 udp, msgtype, twostep;
 653	u16 offset1, offset2;
 654
 655	/* Mark tx timestamp type on skb->cb[0] if requires */
 656	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
 657	    (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
 658		skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
 659	} else {
 660		skb->cb[0] = 0;
 661	}
 662
 663	/* Fall back to two-step timestamp if not one-step Sync packet */
 664	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 665		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
 666				    &offset1, &offset2) ||
 667		    msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
 668			skb->cb[0] = ENETC_F_TX_TSTAMP;
 669	}
 670
 671	return enetc_start_xmit(skb, ndev);
 672}
 673EXPORT_SYMBOL_GPL(enetc_xmit);
 674
 675static irqreturn_t enetc_msix(int irq, void *data)
 676{
 677	struct enetc_int_vector	*v = data;
 678	int i;
 679
 680	enetc_lock_mdio();
 681
 682	/* disable interrupts */
 683	enetc_wr_reg_hot(v->rbier, 0);
 684	enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
 685
 686	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
 687		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
 688
 689	enetc_unlock_mdio();
 690
 691	napi_schedule(&v->napi);
 692
 693	return IRQ_HANDLED;
 694}
 695
 
 
 
 
 696static void enetc_rx_dim_work(struct work_struct *w)
 697{
 698	struct dim *dim = container_of(w, struct dim, work);
 699	struct dim_cq_moder moder =
 700		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
 701	struct enetc_int_vector	*v =
 702		container_of(dim, struct enetc_int_vector, rx_dim);
 703
 704	v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
 705	dim->state = DIM_START_MEASURE;
 706}
 707
 708static void enetc_rx_net_dim(struct enetc_int_vector *v)
 709{
 710	struct dim_sample dim_sample = {};
 711
 712	v->comp_cnt++;
 713
 714	if (!v->rx_napi_work)
 715		return;
 716
 717	dim_update_sample(v->comp_cnt,
 718			  v->rx_ring.stats.packets,
 719			  v->rx_ring.stats.bytes,
 720			  &dim_sample);
 721	net_dim(&v->rx_dim, dim_sample);
 722}
 723
 724static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
 725{
 726	int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
 
 
 
 
 727
 728	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
 729}
 
 730
 731static bool enetc_page_reusable(struct page *page)
 732{
 733	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
 734}
 
 735
 736static void enetc_reuse_page(struct enetc_bdr *rx_ring,
 737			     struct enetc_rx_swbd *old)
 738{
 739	struct enetc_rx_swbd *new;
 740
 741	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
 742
 743	/* next buf that may reuse a page */
 744	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
 745
 746	/* copy page reference */
 747	*new = *old;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 748}
 749
 750static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
 751				u64 *tstamp)
 752{
 753	u32 lo, hi, tstamp_lo;
 754
 755	lo = enetc_rd_hot(hw, ENETC_SICTR0);
 756	hi = enetc_rd_hot(hw, ENETC_SICTR1);
 757	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
 758	if (lo <= tstamp_lo)
 759		hi -= 1;
 760	*tstamp = (u64)hi << 32 | tstamp_lo;
 761}
 762
 763static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
 764{
 765	struct skb_shared_hwtstamps shhwtstamps;
 766
 767	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
 768		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
 769		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
 770		skb_txtime_consumed(skb);
 771		skb_tstamp_tx(skb, &shhwtstamps);
 772	}
 773}
 774
 775static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
 776				      struct enetc_tx_swbd *tx_swbd)
 777{
 778	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
 779	struct enetc_rx_swbd rx_swbd = {
 780		.dma = tx_swbd->dma,
 781		.page = tx_swbd->page,
 782		.page_offset = tx_swbd->page_offset,
 783		.dir = tx_swbd->dir,
 784		.len = tx_swbd->len,
 785	};
 786	struct enetc_bdr *rx_ring;
 787
 788	rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
 789
 790	if (likely(enetc_swbd_unused(rx_ring))) {
 791		enetc_reuse_page(rx_ring, &rx_swbd);
 792
 793		/* sync for use by the device */
 794		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
 795						 rx_swbd.page_offset,
 796						 ENETC_RXB_DMA_SIZE_XDP,
 797						 rx_swbd.dir);
 798
 799		rx_ring->stats.recycles++;
 800	} else {
 801		/* RX ring is already full, we need to unmap and free the
 802		 * page, since there's nothing useful we can do with it.
 803		 */
 804		rx_ring->stats.recycle_failures++;
 805
 806		dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
 807			       rx_swbd.dir);
 808		__free_page(rx_swbd.page);
 809	}
 810
 811	rx_ring->xdp.xdp_tx_in_flight--;
 812}
 813
 814static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
 815{
 816	int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
 817	struct net_device *ndev = tx_ring->ndev;
 818	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 819	struct enetc_tx_swbd *tx_swbd;
 820	int i, bds_to_clean;
 821	bool do_twostep_tstamp;
 822	u64 tstamp = 0;
 823
 824	i = tx_ring->next_to_clean;
 825	tx_swbd = &tx_ring->tx_swbd[i];
 826
 827	bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 828
 829	do_twostep_tstamp = false;
 830
 831	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
 832		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
 833		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
 834		bool is_eof = tx_swbd->is_eof;
 835
 836		if (unlikely(tx_swbd->check_wb)) {
 837			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
 
 
 
 838
 839			if (txbd->flags & ENETC_TXBD_FLAGS_W &&
 840			    tx_swbd->do_twostep_tstamp) {
 841				enetc_get_tx_tstamp(&priv->si->hw, txbd,
 842						    &tstamp);
 843				do_twostep_tstamp = true;
 844			}
 845
 846			if (tx_swbd->qbv_en &&
 847			    txbd->wb.status & ENETC_TXBD_STATS_WIN)
 848				tx_win_drop++;
 849		}
 850
 851		if (tx_swbd->is_xdp_tx)
 852			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
 853		else if (likely(tx_swbd->dma))
 854			enetc_unmap_tx_buff(tx_ring, tx_swbd);
 855
 856		if (xdp_frame) {
 857			xdp_return_frame(xdp_frame);
 858		} else if (skb) {
 859			if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
 860				/* Start work to release lock for next one-step
 861				 * timestamping packet. And send one skb in
 862				 * tx_skbs queue if has.
 863				 */
 864				schedule_work(&priv->tx_onestep_tstamp);
 865			} else if (unlikely(do_twostep_tstamp)) {
 866				enetc_tstamp_tx(skb, tstamp);
 867				do_twostep_tstamp = false;
 868			}
 869			napi_consume_skb(skb, napi_budget);
 
 870		}
 871
 872		tx_byte_cnt += tx_swbd->len;
 873		/* Scrub the swbd here so we don't have to do that
 874		 * when we reuse it during xmit
 875		 */
 876		memset(tx_swbd, 0, sizeof(*tx_swbd));
 877
 878		bds_to_clean--;
 879		tx_swbd++;
 880		i++;
 881		if (unlikely(i == tx_ring->bd_count)) {
 882			i = 0;
 883			tx_swbd = tx_ring->tx_swbd;
 884		}
 885
 886		/* BD iteration loop end */
 887		if (is_eof) {
 888			tx_frm_cnt++;
 889			/* re-arm interrupt source */
 890			enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
 891					 BIT(16 + tx_ring->index));
 892		}
 893
 894		if (unlikely(!bds_to_clean))
 895			bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 896	}
 897
 898	tx_ring->next_to_clean = i;
 899	tx_ring->stats.packets += tx_frm_cnt;
 900	tx_ring->stats.bytes += tx_byte_cnt;
 901	tx_ring->stats.win_drop += tx_win_drop;
 902
 903	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
 904		     __netif_subqueue_stopped(ndev, tx_ring->index) &&
 905		     (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
 906		netif_wake_subqueue(ndev, tx_ring->index);
 907	}
 908
 909	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
 910}
 911
 912static bool enetc_new_page(struct enetc_bdr *rx_ring,
 913			   struct enetc_rx_swbd *rx_swbd)
 914{
 915	bool xdp = !!(rx_ring->xdp.prog);
 916	struct page *page;
 917	dma_addr_t addr;
 918
 919	page = dev_alloc_page();
 920	if (unlikely(!page))
 921		return false;
 922
 923	/* For XDP_TX, we forgo dma_unmap -> dma_map */
 924	rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 925
 926	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
 927	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
 928		__free_page(page);
 929
 930		return false;
 931	}
 932
 933	rx_swbd->dma = addr;
 934	rx_swbd->page = page;
 935	rx_swbd->page_offset = rx_ring->buffer_offset;
 936
 937	return true;
 938}
 939
 940static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
 941{
 942	struct enetc_rx_swbd *rx_swbd;
 943	union enetc_rx_bd *rxbd;
 944	int i, j;
 945
 946	i = rx_ring->next_to_use;
 947	rx_swbd = &rx_ring->rx_swbd[i];
 948	rxbd = enetc_rxbd(rx_ring, i);
 949
 950	for (j = 0; j < buff_cnt; j++) {
 951		/* try reuse page */
 952		if (unlikely(!rx_swbd->page)) {
 953			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
 954				rx_ring->stats.rx_alloc_errs++;
 955				break;
 956			}
 957		}
 958
 959		/* update RxBD */
 960		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
 961					   rx_swbd->page_offset);
 962		/* clear 'R" as well */
 963		rxbd->r.lstatus = 0;
 964
 965		enetc_rxbd_next(rx_ring, &rxbd, &i);
 966		rx_swbd = &rx_ring->rx_swbd[i];
 
 
 
 
 
 967	}
 968
 969	if (likely(j)) {
 970		rx_ring->next_to_alloc = i; /* keep track from page reuse */
 971		rx_ring->next_to_use = i;
 972
 973		/* update ENETC's consumer index */
 974		enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
 975	}
 976
 977	return j;
 978}
 979
 980#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 981static void enetc_get_rx_tstamp(struct net_device *ndev,
 982				union enetc_rx_bd *rxbd,
 983				struct sk_buff *skb)
 984{
 985	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
 986	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 987	struct enetc_hw *hw = &priv->si->hw;
 988	u32 lo, hi, tstamp_lo;
 989	u64 tstamp;
 990
 991	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
 992		lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
 993		hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
 994		rxbd = enetc_rxbd_ext(rxbd);
 995		tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
 996		if (lo <= tstamp_lo)
 997			hi -= 1;
 998
 999		tstamp = (u64)hi << 32 | tstamp_lo;
1000		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
1001		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
1002	}
1003}
1004#endif
1005
1006static void enetc_get_offloads(struct enetc_bdr *rx_ring,
1007			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
1008{
 
1009	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1010
1011	/* TODO: hashing */
1012	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
1013		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
1014
1015		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
1016		skb->ip_summed = CHECKSUM_COMPLETE;
1017	}
1018
1019	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1020		__be16 tpid = 0;
1021
1022		switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1023		case 0:
1024			tpid = htons(ETH_P_8021Q);
1025			break;
1026		case 1:
1027			tpid = htons(ETH_P_8021AD);
1028			break;
1029		case 2:
1030			tpid = htons(enetc_port_rd(&priv->si->hw,
1031						   ENETC_PCVLANR1));
1032			break;
1033		case 3:
1034			tpid = htons(enetc_port_rd(&priv->si->hw,
1035						   ENETC_PCVLANR2));
1036			break;
1037		default:
1038			break;
1039		}
1040
1041		__vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1042	}
1043
1044#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1045	if (priv->active_offloads & ENETC_F_RX_TSTAMP)
1046		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1047#endif
1048}
1049
1050/* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1051 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1052 * mapped buffers.
1053 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1054static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1055					       int i, u16 size)
1056{
1057	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1058
1059	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1060				      rx_swbd->page_offset,
1061				      size, rx_swbd->dir);
1062	return rx_swbd;
1063}
1064
1065/* Reuse the current page without performing half-page buffer flipping */
1066static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1067			      struct enetc_rx_swbd *rx_swbd)
1068{
1069	size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1070
1071	enetc_reuse_page(rx_ring, rx_swbd);
1072
1073	dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1074					 rx_swbd->page_offset,
1075					 buffer_size, rx_swbd->dir);
1076
1077	rx_swbd->page = NULL;
1078}
1079
1080/* Reuse the current page by performing half-page buffer flipping */
1081static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1082			       struct enetc_rx_swbd *rx_swbd)
1083{
1084	if (likely(enetc_page_reusable(rx_swbd->page))) {
1085		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1086		page_ref_inc(rx_swbd->page);
1087
1088		enetc_put_rx_buff(rx_ring, rx_swbd);
 
 
 
 
 
 
1089	} else {
1090		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1091			       rx_swbd->dir);
1092		rx_swbd->page = NULL;
1093	}
 
 
1094}
1095
1096static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1097						int i, u16 size)
1098{
1099	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1100	struct sk_buff *skb;
1101	void *ba;
1102
1103	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1104	skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1105	if (unlikely(!skb)) {
1106		rx_ring->stats.rx_alloc_errs++;
1107		return NULL;
1108	}
1109
1110	skb_reserve(skb, rx_ring->buffer_offset);
1111	__skb_put(skb, size);
1112
1113	enetc_flip_rx_buff(rx_ring, rx_swbd);
1114
1115	return skb;
1116}
1117
1118static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1119				     u16 size, struct sk_buff *skb)
1120{
1121	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1122
1123	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1124			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1125
1126	enetc_flip_rx_buff(rx_ring, rx_swbd);
1127}
1128
1129static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1130					      u32 bd_status,
1131					      union enetc_rx_bd **rxbd, int *i)
1132{
1133	if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1134		return false;
1135
1136	enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1137	enetc_rxbd_next(rx_ring, rxbd, i);
1138
1139	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1140		dma_rmb();
1141		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1142
1143		enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1144		enetc_rxbd_next(rx_ring, rxbd, i);
1145	}
1146
1147	rx_ring->ndev->stats.rx_dropped++;
1148	rx_ring->ndev->stats.rx_errors++;
1149
1150	return true;
1151}
1152
1153static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1154				       u32 bd_status, union enetc_rx_bd **rxbd,
1155				       int *i, int *cleaned_cnt, int buffer_size)
1156{
1157	struct sk_buff *skb;
1158	u16 size;
1159
1160	size = le16_to_cpu((*rxbd)->r.buf_len);
1161	skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1162	if (!skb)
1163		return NULL;
1164
1165	enetc_get_offloads(rx_ring, *rxbd, skb);
1166
1167	(*cleaned_cnt)++;
1168
1169	enetc_rxbd_next(rx_ring, rxbd, i);
1170
1171	/* not last BD in frame? */
1172	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1173		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1174		size = buffer_size;
1175
1176		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1177			dma_rmb();
1178			size = le16_to_cpu((*rxbd)->r.buf_len);
1179		}
1180
1181		enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1182
1183		(*cleaned_cnt)++;
1184
1185		enetc_rxbd_next(rx_ring, rxbd, i);
1186	}
1187
1188	skb_record_rx_queue(skb, rx_ring->index);
1189	skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1190
1191	return skb;
1192}
1193
1194#define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1195
1196static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1197			       struct napi_struct *napi, int work_limit)
1198{
1199	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1200	int cleaned_cnt, i;
1201
1202	cleaned_cnt = enetc_bd_unused(rx_ring);
1203	/* next descriptor to process */
1204	i = rx_ring->next_to_clean;
1205
1206	while (likely(rx_frm_cnt < work_limit)) {
1207		union enetc_rx_bd *rxbd;
1208		struct sk_buff *skb;
1209		u32 bd_status;
 
 
 
 
1210
1211		if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1212			cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1213							    cleaned_cnt);
1214
1215		rxbd = enetc_rxbd(rx_ring, i);
1216		bd_status = le32_to_cpu(rxbd->r.lstatus);
1217		if (!bd_status)
1218			break;
1219
1220		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1221		dma_rmb(); /* for reading other rxbd fields */
1222
1223		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1224						      &rxbd, &i))
1225			break;
1226
1227		skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1228				      &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1229		if (!skb)
1230			break;
1231
1232		/* When set, the outer VLAN header is extracted and reported
1233		 * in the receive buffer descriptor. So rx_byte_cnt should
1234		 * add the length of the extracted VLAN header.
1235		 */
1236		if (bd_status & ENETC_RXBD_FLAG_VLAN)
1237			rx_byte_cnt += VLAN_HLEN;
1238		rx_byte_cnt += skb->len + ETH_HLEN;
1239		rx_frm_cnt++;
1240
1241		napi_gro_receive(napi, skb);
1242	}
1243
1244	rx_ring->next_to_clean = i;
1245
1246	rx_ring->stats.packets += rx_frm_cnt;
1247	rx_ring->stats.bytes += rx_byte_cnt;
1248
1249	return rx_frm_cnt;
1250}
1251
1252static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1253				  struct enetc_tx_swbd *tx_swbd,
1254				  int frm_len)
1255{
1256	union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1257
1258	prefetchw(txbd);
1259
1260	enetc_clear_tx_bd(txbd);
1261	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1262	txbd->buf_len = cpu_to_le16(tx_swbd->len);
1263	txbd->frm_len = cpu_to_le16(frm_len);
1264
1265	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1266}
1267
1268/* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1269 * descriptors.
1270 */
1271static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1272			 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1273{
1274	struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1275	int i, k, frm_len = tmp_tx_swbd->len;
1276
1277	if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1278		return false;
1279
1280	while (unlikely(!tmp_tx_swbd->is_eof)) {
1281		tmp_tx_swbd++;
1282		frm_len += tmp_tx_swbd->len;
1283	}
1284
1285	i = tx_ring->next_to_use;
1286
1287	for (k = 0; k < num_tx_swbd; k++) {
1288		struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1289
1290		enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1291
1292		/* last BD needs 'F' bit set */
1293		if (xdp_tx_swbd->is_eof) {
1294			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1295
1296			txbd->flags = ENETC_TXBD_FLAGS_F;
1297		}
1298
1299		enetc_bdr_idx_inc(tx_ring, &i);
1300	}
1301
1302	tx_ring->next_to_use = i;
1303
1304	return true;
1305}
1306
1307static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1308					  struct enetc_tx_swbd *xdp_tx_arr,
1309					  struct xdp_frame *xdp_frame)
1310{
1311	struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1312	struct skb_shared_info *shinfo;
1313	void *data = xdp_frame->data;
1314	int len = xdp_frame->len;
1315	skb_frag_t *frag;
1316	dma_addr_t dma;
1317	unsigned int f;
1318	int n = 0;
1319
1320	dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1321	if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1322		netdev_err(tx_ring->ndev, "DMA map error\n");
1323		return -1;
1324	}
1325
1326	xdp_tx_swbd->dma = dma;
1327	xdp_tx_swbd->dir = DMA_TO_DEVICE;
1328	xdp_tx_swbd->len = len;
1329	xdp_tx_swbd->is_xdp_redirect = true;
1330	xdp_tx_swbd->is_eof = false;
1331	xdp_tx_swbd->xdp_frame = NULL;
1332
1333	n++;
1334
1335	if (!xdp_frame_has_frags(xdp_frame))
1336		goto out;
1337
1338	xdp_tx_swbd = &xdp_tx_arr[n];
1339
1340	shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1341
1342	for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1343	     f++, frag++) {
1344		data = skb_frag_address(frag);
1345		len = skb_frag_size(frag);
1346
1347		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1348		if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1349			/* Undo the DMA mapping for all fragments */
1350			while (--n >= 0)
1351				enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1352
1353			netdev_err(tx_ring->ndev, "DMA map error\n");
1354			return -1;
1355		}
1356
1357		xdp_tx_swbd->dma = dma;
1358		xdp_tx_swbd->dir = DMA_TO_DEVICE;
1359		xdp_tx_swbd->len = len;
1360		xdp_tx_swbd->is_xdp_redirect = true;
1361		xdp_tx_swbd->is_eof = false;
1362		xdp_tx_swbd->xdp_frame = NULL;
1363
1364		n++;
1365		xdp_tx_swbd = &xdp_tx_arr[n];
1366	}
1367out:
1368	xdp_tx_arr[n - 1].is_eof = true;
1369	xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1370
1371	return n;
1372}
1373
1374int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1375		   struct xdp_frame **frames, u32 flags)
1376{
1377	struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1378	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1379	struct enetc_bdr *tx_ring;
1380	int xdp_tx_bd_cnt, i, k;
1381	int xdp_tx_frm_cnt = 0;
1382
1383	enetc_lock_mdio();
1384
1385	tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1386
1387	prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
 
 
 
 
 
 
 
 
 
 
1388
1389	for (k = 0; k < num_frames; k++) {
1390		xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1391							       xdp_redirect_arr,
1392							       frames[k]);
1393		if (unlikely(xdp_tx_bd_cnt < 0))
1394			break;
1395
1396		if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1397					   xdp_tx_bd_cnt))) {
1398			for (i = 0; i < xdp_tx_bd_cnt; i++)
1399				enetc_unmap_tx_buff(tx_ring,
1400						    &xdp_redirect_arr[i]);
1401			tx_ring->stats.xdp_tx_drops++;
1402			break;
1403		}
1404
1405		xdp_tx_frm_cnt++;
1406	}
1407
1408	if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1409		enetc_update_tx_ring_tail(tx_ring);
1410
1411	tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1412
1413	enetc_unlock_mdio();
1414
1415	return xdp_tx_frm_cnt;
1416}
1417EXPORT_SYMBOL_GPL(enetc_xdp_xmit);
1418
1419static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1420				     struct xdp_buff *xdp_buff, u16 size)
1421{
1422	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1423	void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1424
1425	/* To be used for XDP_TX */
1426	rx_swbd->len = size;
1427
1428	xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1429			 rx_ring->buffer_offset, size, false);
1430}
1431
1432static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1433				     u16 size, struct xdp_buff *xdp_buff)
1434{
1435	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1436	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1437	skb_frag_t *frag;
1438
1439	/* To be used for XDP_TX */
1440	rx_swbd->len = size;
1441
1442	if (!xdp_buff_has_frags(xdp_buff)) {
1443		xdp_buff_set_frags_flag(xdp_buff);
1444		shinfo->xdp_frags_size = size;
1445		shinfo->nr_frags = 0;
1446	} else {
1447		shinfo->xdp_frags_size += size;
1448	}
1449
1450	if (page_is_pfmemalloc(rx_swbd->page))
1451		xdp_buff_set_frag_pfmemalloc(xdp_buff);
1452
1453	frag = &shinfo->frags[shinfo->nr_frags];
1454	skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,
1455				size);
1456
1457	shinfo->nr_frags++;
1458}
1459
1460static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1461				 union enetc_rx_bd **rxbd, int *i,
1462				 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1463{
1464	u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1465
1466	xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1467
1468	enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1469	(*cleaned_cnt)++;
1470	enetc_rxbd_next(rx_ring, rxbd, i);
1471
1472	/* not last BD in frame? */
1473	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1474		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1475		size = ENETC_RXB_DMA_SIZE_XDP;
1476
1477		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1478			dma_rmb();
1479			size = le16_to_cpu((*rxbd)->r.buf_len);
1480		}
1481
1482		enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1483		(*cleaned_cnt)++;
1484		enetc_rxbd_next(rx_ring, rxbd, i);
1485	}
1486}
1487
1488/* Convert RX buffer descriptors to TX buffer descriptors. These will be
1489 * recycled back into the RX ring in enetc_clean_tx_ring.
1490 */
1491static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1492					struct enetc_bdr *rx_ring,
1493					int rx_ring_first, int rx_ring_last)
1494{
1495	int n = 0;
1496
1497	for (; rx_ring_first != rx_ring_last;
1498	     n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1499		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1500		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1501
1502		/* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1503		tx_swbd->dma = rx_swbd->dma;
1504		tx_swbd->dir = rx_swbd->dir;
1505		tx_swbd->page = rx_swbd->page;
1506		tx_swbd->page_offset = rx_swbd->page_offset;
1507		tx_swbd->len = rx_swbd->len;
1508		tx_swbd->is_dma_page = true;
1509		tx_swbd->is_xdp_tx = true;
1510		tx_swbd->is_eof = false;
1511	}
1512
1513	/* We rely on caller providing an rx_ring_last > rx_ring_first */
1514	xdp_tx_arr[n - 1].is_eof = true;
1515
1516	return n;
1517}
1518
1519static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1520			   int rx_ring_last)
1521{
1522	while (rx_ring_first != rx_ring_last) {
1523		enetc_put_rx_buff(rx_ring,
1524				  &rx_ring->rx_swbd[rx_ring_first]);
1525		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1526	}
1527	rx_ring->stats.xdp_drops++;
1528}
1529
1530static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1531				   struct napi_struct *napi, int work_limit,
1532				   struct bpf_prog *prog)
1533{
1534	int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1535	struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1536	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1537	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1538	struct enetc_bdr *tx_ring;
1539	int cleaned_cnt, i;
1540	u32 xdp_act;
1541
1542	cleaned_cnt = enetc_bd_unused(rx_ring);
1543	/* next descriptor to process */
1544	i = rx_ring->next_to_clean;
1545
1546	while (likely(rx_frm_cnt < work_limit)) {
1547		union enetc_rx_bd *rxbd, *orig_rxbd;
1548		int orig_i, orig_cleaned_cnt;
1549		struct xdp_buff xdp_buff;
1550		struct sk_buff *skb;
1551		u32 bd_status;
1552		int err;
1553
1554		rxbd = enetc_rxbd(rx_ring, i);
1555		bd_status = le32_to_cpu(rxbd->r.lstatus);
1556		if (!bd_status)
1557			break;
1558
1559		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1560		dma_rmb(); /* for reading other rxbd fields */
1561
1562		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1563						      &rxbd, &i))
1564			break;
1565
1566		orig_rxbd = rxbd;
1567		orig_cleaned_cnt = cleaned_cnt;
1568		orig_i = i;
1569
1570		enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1571				     &cleaned_cnt, &xdp_buff);
1572
1573		/* When set, the outer VLAN header is extracted and reported
1574		 * in the receive buffer descriptor. So rx_byte_cnt should
1575		 * add the length of the extracted VLAN header.
1576		 */
1577		if (bd_status & ENETC_RXBD_FLAG_VLAN)
1578			rx_byte_cnt += VLAN_HLEN;
1579		rx_byte_cnt += xdp_get_buff_len(&xdp_buff);
1580
1581		xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1582
1583		switch (xdp_act) {
1584		default:
1585			bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1586			fallthrough;
1587		case XDP_ABORTED:
1588			trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1589			fallthrough;
1590		case XDP_DROP:
1591			enetc_xdp_drop(rx_ring, orig_i, i);
1592			break;
1593		case XDP_PASS:
1594			rxbd = orig_rxbd;
1595			cleaned_cnt = orig_cleaned_cnt;
1596			i = orig_i;
1597
1598			skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1599					      &i, &cleaned_cnt,
1600					      ENETC_RXB_DMA_SIZE_XDP);
1601			if (unlikely(!skb))
1602				goto out;
1603
1604			napi_gro_receive(napi, skb);
1605			break;
1606		case XDP_TX:
1607			tx_ring = priv->xdp_tx_ring[rx_ring->index];
1608			xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1609								     rx_ring,
1610								     orig_i, i);
1611
1612			if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1613				enetc_xdp_drop(rx_ring, orig_i, i);
1614				tx_ring->stats.xdp_tx_drops++;
1615			} else {
1616				tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
1617				rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1618				xdp_tx_frm_cnt++;
1619				/* The XDP_TX enqueue was successful, so we
1620				 * need to scrub the RX software BDs because
1621				 * the ownership of the buffers no longer
1622				 * belongs to the RX ring, and we must prevent
1623				 * enetc_refill_rx_ring() from reusing
1624				 * rx_swbd->page.
1625				 */
1626				while (orig_i != i) {
1627					rx_ring->rx_swbd[orig_i].page = NULL;
1628					enetc_bdr_idx_inc(rx_ring, &orig_i);
1629				}
1630			}
1631			break;
1632		case XDP_REDIRECT:
1633			err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1634			if (unlikely(err)) {
1635				enetc_xdp_drop(rx_ring, orig_i, i);
1636				rx_ring->stats.xdp_redirect_failures++;
1637			} else {
1638				while (orig_i != i) {
1639					enetc_flip_rx_buff(rx_ring,
1640							   &rx_ring->rx_swbd[orig_i]);
1641					enetc_bdr_idx_inc(rx_ring, &orig_i);
1642				}
1643				xdp_redirect_frm_cnt++;
1644				rx_ring->stats.xdp_redirect++;
1645			}
1646		}
1647
1648		rx_frm_cnt++;
1649	}
1650
1651out:
1652	rx_ring->next_to_clean = i;
1653
1654	rx_ring->stats.packets += rx_frm_cnt;
1655	rx_ring->stats.bytes += rx_byte_cnt;
1656
1657	if (xdp_redirect_frm_cnt)
1658		xdp_do_flush();
1659
1660	if (xdp_tx_frm_cnt)
1661		enetc_update_tx_ring_tail(tx_ring);
1662
1663	if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
1664		enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
1665				     rx_ring->xdp.xdp_tx_in_flight);
1666
1667	return rx_frm_cnt;
1668}
1669
1670static int enetc_poll(struct napi_struct *napi, int budget)
1671{
1672	struct enetc_int_vector
1673		*v = container_of(napi, struct enetc_int_vector, napi);
1674	struct enetc_bdr *rx_ring = &v->rx_ring;
1675	struct bpf_prog *prog;
1676	bool complete = true;
1677	int work_done;
1678	int i;
1679
1680	enetc_lock_mdio();
1681
1682	for (i = 0; i < v->count_tx_rings; i++)
1683		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
1684			complete = false;
1685
1686	prog = rx_ring->xdp.prog;
1687	if (prog)
1688		work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1689	else
1690		work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
1691	if (work_done == budget)
1692		complete = false;
1693	if (work_done)
1694		v->rx_napi_work = true;
1695
1696	if (!complete) {
1697		enetc_unlock_mdio();
1698		return budget;
1699	}
1700
1701	napi_complete_done(napi, work_done);
1702
1703	if (likely(v->rx_dim_en))
1704		enetc_rx_net_dim(v);
1705
1706	v->rx_napi_work = false;
1707
1708	/* enable interrupts */
1709	enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
1710
1711	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1712		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
1713				 ENETC_TBIER_TXTIE);
1714
1715	enetc_unlock_mdio();
1716
1717	return work_done;
1718}
1719
1720/* Probing and Init */
1721#define ENETC_MAX_RFS_SIZE 64
1722void enetc_get_si_caps(struct enetc_si *si)
1723{
1724	struct enetc_hw *hw = &si->hw;
1725	u32 val;
1726
1727	/* find out how many of various resources we have to work with */
1728	val = enetc_rd(hw, ENETC_SICAPR0);
1729	si->num_rx_rings = (val >> 16) & 0xff;
1730	si->num_tx_rings = val & 0xff;
1731
1732	val = enetc_rd(hw, ENETC_SIRFSCAPR);
1733	si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1734	si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1735
1736	si->num_rss = 0;
1737	val = enetc_rd(hw, ENETC_SIPCAPR0);
1738	if (val & ENETC_SIPCAPR0_RSS) {
1739		u32 rss;
1740
1741		rss = enetc_rd(hw, ENETC_SIRSSCAPR);
1742		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1743	}
1744
1745	if (val & ENETC_SIPCAPR0_QBV)
1746		si->hw_features |= ENETC_SI_F_QBV;
1747
1748	if (val & ENETC_SIPCAPR0_QBU)
1749		si->hw_features |= ENETC_SI_F_QBU;
1750
1751	if (val & ENETC_SIPCAPR0_PSFP)
1752		si->hw_features |= ENETC_SI_F_PSFP;
1753}
1754EXPORT_SYMBOL_GPL(enetc_get_si_caps);
1755
1756static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
1757{
1758	size_t bd_base_size = res->bd_count * res->bd_size;
1759
1760	res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,
1761					  &res->bd_dma_base, GFP_KERNEL);
1762	if (!res->bd_base)
1763		return -ENOMEM;
1764
1765	/* h/w requires 128B alignment */
1766	if (!IS_ALIGNED(res->bd_dma_base, 128)) {
1767		dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1768				  res->bd_dma_base);
1769		return -EINVAL;
1770	}
1771
1772	return 0;
1773}
1774
1775static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
1776{
1777	size_t bd_base_size = res->bd_count * res->bd_size;
1778
1779	dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1780			  res->bd_dma_base);
1781}
1782
1783static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
1784				   struct device *dev, size_t bd_count)
1785{
1786	int err;
1787
1788	res->dev = dev;
1789	res->bd_count = bd_count;
1790	res->bd_size = sizeof(union enetc_tx_bd);
1791
1792	res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));
1793	if (!res->tx_swbd)
1794		return -ENOMEM;
1795
1796	err = enetc_dma_alloc_bdr(res);
1797	if (err)
1798		goto err_alloc_bdr;
1799
1800	res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,
1801					      &res->tso_headers_dma,
1802					      GFP_KERNEL);
1803	if (!res->tso_headers) {
1804		err = -ENOMEM;
1805		goto err_alloc_tso;
1806	}
1807
1808	return 0;
1809
1810err_alloc_tso:
1811	enetc_dma_free_bdr(res);
1812err_alloc_bdr:
1813	vfree(res->tx_swbd);
1814	res->tx_swbd = NULL;
1815
1816	return err;
1817}
1818
1819static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
1820{
1821	dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,
1822			  res->tso_headers, res->tso_headers_dma);
1823	enetc_dma_free_bdr(res);
1824	vfree(res->tx_swbd);
 
 
 
 
 
 
 
 
1825}
1826
1827static struct enetc_bdr_resource *
1828enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1829{
1830	struct enetc_bdr_resource *tx_res;
1831	int i, err;
1832
1833	tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL);
1834	if (!tx_res)
1835		return ERR_PTR(-ENOMEM);
1836
1837	for (i = 0; i < priv->num_tx_rings; i++) {
1838		struct enetc_bdr *tx_ring = priv->tx_ring[i];
1839
1840		err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,
1841					      tx_ring->bd_count);
1842		if (err)
1843			goto fail;
1844	}
1845
1846	return tx_res;
1847
1848fail:
1849	while (i-- > 0)
1850		enetc_free_tx_resource(&tx_res[i]);
1851
1852	kfree(tx_res);
1853
1854	return ERR_PTR(err);
1855}
1856
1857static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
1858				    size_t num_resources)
1859{
1860	size_t i;
1861
1862	for (i = 0; i < num_resources; i++)
1863		enetc_free_tx_resource(&tx_res[i]);
1864
1865	kfree(tx_res);
 
1866}
1867
1868static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
1869				   struct device *dev, size_t bd_count,
1870				   bool extended)
1871{
 
1872	int err;
1873
1874	res->dev = dev;
1875	res->bd_count = bd_count;
1876	res->bd_size = sizeof(union enetc_rx_bd);
1877	if (extended)
1878		res->bd_size *= 2;
1879
1880	res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));
1881	if (!res->rx_swbd)
1882		return -ENOMEM;
1883
1884	err = enetc_dma_alloc_bdr(res);
 
 
 
1885	if (err) {
1886		vfree(res->rx_swbd);
1887		return err;
1888	}
1889
 
 
 
 
 
1890	return 0;
1891}
1892
1893static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
1894{
1895	enetc_dma_free_bdr(res);
1896	vfree(res->rx_swbd);
 
 
 
 
 
 
 
1897}
1898
1899static struct enetc_bdr_resource *
1900enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
1901{
1902	struct enetc_bdr_resource *rx_res;
1903	int i, err;
1904
1905	rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL);
1906	if (!rx_res)
1907		return ERR_PTR(-ENOMEM);
1908
1909	for (i = 0; i < priv->num_rx_rings; i++) {
1910		struct enetc_bdr *rx_ring = priv->rx_ring[i];
1911
1912		err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,
1913					      rx_ring->bd_count, extended);
1914		if (err)
1915			goto fail;
1916	}
1917
1918	return rx_res;
1919
1920fail:
1921	while (i-- > 0)
1922		enetc_free_rx_resource(&rx_res[i]);
1923
1924	kfree(rx_res);
1925
1926	return ERR_PTR(err);
1927}
1928
1929static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
1930				    size_t num_resources)
1931{
1932	size_t i;
1933
1934	for (i = 0; i < num_resources; i++)
1935		enetc_free_rx_resource(&rx_res[i]);
1936
1937	kfree(rx_res);
1938}
1939
1940static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
1941				     const struct enetc_bdr_resource *res)
1942{
1943	tx_ring->bd_base = res ? res->bd_base : NULL;
1944	tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1945	tx_ring->tx_swbd = res ? res->tx_swbd : NULL;
1946	tx_ring->tso_headers = res ? res->tso_headers : NULL;
1947	tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;
1948}
1949
1950static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
1951				     const struct enetc_bdr_resource *res)
1952{
1953	rx_ring->bd_base = res ? res->bd_base : NULL;
1954	rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1955	rx_ring->rx_swbd = res ? res->rx_swbd : NULL;
1956}
1957
1958static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
1959				      const struct enetc_bdr_resource *res)
1960{
1961	int i;
1962
1963	if (priv->tx_res)
1964		enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);
1965
1966	for (i = 0; i < priv->num_tx_rings; i++) {
1967		enetc_assign_tx_resource(priv->tx_ring[i],
1968					 res ? &res[i] : NULL);
1969	}
1970
1971	priv->tx_res = res;
1972}
1973
1974static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
1975				      const struct enetc_bdr_resource *res)
1976{
1977	int i;
1978
1979	if (priv->rx_res)
1980		enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);
1981
1982	for (i = 0; i < priv->num_rx_rings; i++) {
1983		enetc_assign_rx_resource(priv->rx_ring[i],
1984					 res ? &res[i] : NULL);
1985	}
1986
1987	priv->rx_res = res;
1988}
1989
1990static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
1991{
1992	int i;
1993
 
 
 
1994	for (i = 0; i < tx_ring->bd_count; i++) {
1995		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
1996
1997		enetc_free_tx_frame(tx_ring, tx_swbd);
1998	}
 
 
 
1999}
2000
2001static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
2002{
2003	int i;
2004
 
 
 
2005	for (i = 0; i < rx_ring->bd_count; i++) {
2006		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
2007
2008		if (!rx_swbd->page)
2009			continue;
2010
2011		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
2012			       rx_swbd->dir);
2013		__free_page(rx_swbd->page);
2014		rx_swbd->page = NULL;
2015	}
 
 
 
 
2016}
2017
2018static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
2019{
2020	int i;
2021
2022	for (i = 0; i < priv->num_rx_rings; i++)
2023		enetc_free_rx_ring(priv->rx_ring[i]);
2024
2025	for (i = 0; i < priv->num_tx_rings; i++)
2026		enetc_free_tx_ring(priv->tx_ring[i]);
2027}
2028
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2029static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
2030{
2031	int *rss_table;
2032	int i;
2033
2034	rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
2035	if (!rss_table)
2036		return -ENOMEM;
2037
2038	/* Set up RSS table defaults */
2039	for (i = 0; i < si->num_rss; i++)
2040		rss_table[i] = i % num_groups;
2041
2042	enetc_set_rss_table(si, rss_table, si->num_rss);
2043
2044	kfree(rss_table);
2045
2046	return 0;
2047}
2048
2049int enetc_configure_si(struct enetc_ndev_priv *priv)
2050{
2051	struct enetc_si *si = priv->si;
2052	struct enetc_hw *hw = &si->hw;
2053	int err;
2054
 
2055	/* set SI cache attributes */
2056	enetc_wr(hw, ENETC_SICAR0,
2057		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
2058	enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
2059	/* enable SI */
2060	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
2061
2062	if (si->num_rss) {
2063		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
2064		if (err)
2065			return err;
2066	}
2067
2068	return 0;
2069}
2070EXPORT_SYMBOL_GPL(enetc_configure_si);
2071
2072void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2073{
2074	struct enetc_si *si = priv->si;
2075	int cpus = num_online_cpus();
2076
2077	priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
2078	priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2079
2080	/* Enable all available TX rings in order to configure as many
2081	 * priorities as possible, when needed.
2082	 * TODO: Make # of TX rings run-time configurable
2083	 */
2084	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2085	priv->num_tx_rings = si->num_tx_rings;
2086	priv->bdr_int_num = cpus;
2087	priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2088	priv->tx_ictt = ENETC_TXIC_TIMETHR;
 
 
 
2089}
2090EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);
2091
2092int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2093{
2094	struct enetc_si *si = priv->si;
 
 
 
 
 
2095
2096	priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2097				  GFP_KERNEL);
2098	if (!priv->cls_rules)
2099		return -ENOMEM;
 
 
 
 
 
 
2100
2101	return 0;
 
 
 
 
 
 
 
 
2102}
2103EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);
2104
2105void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2106{
 
 
 
 
 
2107	kfree(priv->cls_rules);
2108}
2109EXPORT_SYMBOL_GPL(enetc_free_si_resources);
2110
2111static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2112{
2113	int idx = tx_ring->index;
2114	u32 tbmr;
2115
2116	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2117		       lower_32_bits(tx_ring->bd_dma_base));
2118
2119	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2120		       upper_32_bits(tx_ring->bd_dma_base));
2121
2122	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2123	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2124		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
2125
2126	/* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2127	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2128	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2129
2130	/* enable Tx ints by setting pkt thr to 1 */
2131	enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2132
2133	tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);
2134	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2135		tbmr |= ENETC_TBMR_VIH;
2136
2137	/* enable ring */
2138	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2139
2140	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2141	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2142	tx_ring->idr = hw->reg + ENETC_SITXIDR;
2143}
2144
2145static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
2146			      bool extended)
2147{
2148	int idx = rx_ring->index;
2149	u32 rbmr = 0;
2150
2151	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2152		       lower_32_bits(rx_ring->bd_dma_base));
2153
2154	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2155		       upper_32_bits(rx_ring->bd_dma_base));
2156
2157	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2158	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2159		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
2160
2161	if (rx_ring->xdp.prog)
2162		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2163	else
2164		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2165
2166	/* Also prepare the consumer index in case page allocation never
2167	 * succeeds. In that case, hardware will never advance producer index
2168	 * to match consumer index, and will drop all frames.
2169	 */
2170	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2171	enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2172
2173	/* enable Rx ints by setting pkt thr to 1 */
2174	enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2175
2176	rx_ring->ext_en = extended;
 
2177	if (rx_ring->ext_en)
2178		rbmr |= ENETC_RBMR_BDS;
2179
2180	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2181		rbmr |= ENETC_RBMR_VTE;
2182
2183	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2184	rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2185
2186	rx_ring->next_to_clean = 0;
2187	rx_ring->next_to_use = 0;
2188	rx_ring->next_to_alloc = 0;
2189
2190	enetc_lock_mdio();
2191	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2192	enetc_unlock_mdio();
2193
 
2194	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2195}
2196
2197static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
2198{
2199	struct enetc_hw *hw = &priv->si->hw;
2200	int i;
2201
2202	for (i = 0; i < priv->num_tx_rings; i++)
2203		enetc_setup_txbdr(hw, priv->tx_ring[i]);
2204
2205	for (i = 0; i < priv->num_rx_rings; i++)
2206		enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);
2207}
2208
2209static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2210{
2211	int idx = tx_ring->index;
2212	u32 tbmr;
2213
2214	tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);
2215	tbmr |= ENETC_TBMR_EN;
2216	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2217}
2218
2219static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2220{
2221	int idx = rx_ring->index;
2222	u32 rbmr;
2223
2224	rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR);
2225	rbmr |= ENETC_RBMR_EN;
2226	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2227}
2228
2229static void enetc_enable_bdrs(struct enetc_ndev_priv *priv)
2230{
2231	struct enetc_hw *hw = &priv->si->hw;
2232	int i;
2233
2234	for (i = 0; i < priv->num_tx_rings; i++)
2235		enetc_enable_txbdr(hw, priv->tx_ring[i]);
2236
2237	for (i = 0; i < priv->num_rx_rings; i++)
2238		enetc_enable_rxbdr(hw, priv->rx_ring[i]);
2239}
2240
2241static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2242{
2243	int idx = rx_ring->index;
2244
2245	/* disable EN bit on ring */
2246	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2247}
2248
2249static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2250{
2251	int idx = rx_ring->index;
 
2252
2253	/* disable EN bit on ring */
2254	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2255}
2256
2257static void enetc_disable_bdrs(struct enetc_ndev_priv *priv)
2258{
2259	struct enetc_hw *hw = &priv->si->hw;
2260	int i;
2261
2262	for (i = 0; i < priv->num_tx_rings; i++)
2263		enetc_disable_txbdr(hw, priv->tx_ring[i]);
2264
2265	for (i = 0; i < priv->num_rx_rings; i++)
2266		enetc_disable_rxbdr(hw, priv->rx_ring[i]);
2267}
2268
2269static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2270{
2271	int delay = 8, timeout = 100;
2272	int idx = tx_ring->index;
2273
2274	/* wait for busy to clear */
2275	while (delay < timeout &&
2276	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2277		msleep(delay);
2278		delay *= 2;
2279	}
2280
2281	if (delay >= timeout)
2282		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2283			    idx);
2284}
2285
2286static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
2287{
2288	struct enetc_hw *hw = &priv->si->hw;
2289	int i;
2290
2291	for (i = 0; i < priv->num_tx_rings; i++)
2292		enetc_wait_txbdr(hw, priv->tx_ring[i]);
 
 
 
 
 
2293}
2294
2295static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2296{
2297	struct pci_dev *pdev = priv->si->pdev;
2298	struct enetc_hw *hw = &priv->si->hw;
2299	int i, j, err;
2300
2301	for (i = 0; i < priv->bdr_int_num; i++) {
2302		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2303		struct enetc_int_vector *v = priv->int_vector[i];
2304		int entry = ENETC_BDR_INT_BASE_IDX + i;
 
2305
2306		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2307			 priv->ndev->name, i);
2308		err = request_irq(irq, enetc_msix, 0, v->name, v);
2309		if (err) {
2310			dev_err(priv->dev, "request_irq() failed!\n");
2311			goto irq_err;
2312		}
2313		disable_irq(irq);
2314
2315		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2316		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2317		v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2318
2319		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2320
2321		for (j = 0; j < v->count_tx_rings; j++) {
2322			int idx = v->tx_ring[j].index;
2323
2324			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2325		}
2326		irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
 
 
2327	}
2328
2329	return 0;
2330
2331irq_err:
2332	while (i--) {
2333		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2334
2335		irq_set_affinity_hint(irq, NULL);
2336		free_irq(irq, priv->int_vector[i]);
2337	}
2338
2339	return err;
2340}
2341
2342static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2343{
2344	struct pci_dev *pdev = priv->si->pdev;
2345	int i;
2346
2347	for (i = 0; i < priv->bdr_int_num; i++) {
2348		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2349
2350		irq_set_affinity_hint(irq, NULL);
2351		free_irq(irq, priv->int_vector[i]);
2352	}
2353}
2354
2355static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2356{
2357	struct enetc_hw *hw = &priv->si->hw;
2358	u32 icpt, ictt;
2359	int i;
2360
2361	/* enable Tx & Rx event indication */
2362	if (priv->ic_mode &
2363	    (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2364		icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2365		/* init to non-0 minimum, will be adjusted later */
2366		ictt = 0x1;
2367	} else {
2368		icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2369		ictt = 0;
2370	}
2371
2372	for (i = 0; i < priv->num_rx_rings; i++) {
2373		enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2374		enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2375		enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2376	}
2377
2378	if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2379		icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2380	else
2381		icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2382
2383	for (i = 0; i < priv->num_tx_rings; i++) {
2384		enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2385		enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2386		enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2387	}
2388}
2389
2390static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2391{
2392	struct enetc_hw *hw = &priv->si->hw;
2393	int i;
2394
2395	for (i = 0; i < priv->num_tx_rings; i++)
2396		enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2397
2398	for (i = 0; i < priv->num_rx_rings; i++)
2399		enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2400}
2401
2402static int enetc_phylink_connect(struct net_device *ndev)
2403{
2404	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2405	struct ethtool_eee edata;
2406	int err;
2407
2408	if (!priv->phylink) {
2409		/* phy-less mode */
2410		netif_carrier_on(ndev);
2411		return 0;
2412	}
2413
2414	err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2415	if (err) {
2416		dev_err(&ndev->dev, "could not attach to PHY\n");
2417		return err;
2418	}
2419
2420	/* disable EEE autoneg, until ENETC driver supports it */
2421	memset(&edata, 0, sizeof(struct ethtool_eee));
2422	phylink_ethtool_set_eee(priv->phylink, &edata);
2423
2424	phylink_start(priv->phylink);
2425
2426	return 0;
2427}
2428
2429static void enetc_tx_onestep_tstamp(struct work_struct *work)
2430{
2431	struct enetc_ndev_priv *priv;
2432	struct sk_buff *skb;
 
2433
2434	priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
 
2435
2436	netif_tx_lock_bh(priv->ndev);
 
 
 
 
 
2437
2438	clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2439	skb = skb_dequeue(&priv->tx_skbs);
2440	if (skb)
2441		enetc_start_xmit(skb, priv->ndev);
2442
2443	netif_tx_unlock_bh(priv->ndev);
2444}
 
2445
2446static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2447{
2448	INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2449	skb_queue_head_init(&priv->tx_skbs);
2450}
2451
2452void enetc_start(struct net_device *ndev)
2453{
2454	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2455	int i;
2456
2457	enetc_setup_interrupts(priv);
2458
2459	for (i = 0; i < priv->bdr_int_num; i++) {
2460		int irq = pci_irq_vector(priv->si->pdev,
2461					 ENETC_BDR_INT_BASE_IDX + i);
2462
2463		napi_enable(&priv->int_vector[i]->napi);
2464		enable_irq(irq);
2465	}
2466
2467	enetc_enable_bdrs(priv);
 
 
 
2468
2469	netif_tx_start_all_queues(ndev);
2470}
2471EXPORT_SYMBOL_GPL(enetc_start);
2472
2473int enetc_open(struct net_device *ndev)
2474{
2475	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2476	struct enetc_bdr_resource *tx_res, *rx_res;
2477	bool extended;
2478	int err;
2479
2480	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2481
2482	err = enetc_setup_irqs(priv);
2483	if (err)
2484		return err;
2485
2486	err = enetc_phylink_connect(ndev);
2487	if (err)
2488		goto err_phy_connect;
2489
2490	tx_res = enetc_alloc_tx_resources(priv);
2491	if (IS_ERR(tx_res)) {
2492		err = PTR_ERR(tx_res);
2493		goto err_alloc_tx;
2494	}
2495
2496	rx_res = enetc_alloc_rx_resources(priv, extended);
2497	if (IS_ERR(rx_res)) {
2498		err = PTR_ERR(rx_res);
2499		goto err_alloc_rx;
2500	}
2501
2502	enetc_tx_onestep_tstamp_init(priv);
2503	enetc_assign_tx_resources(priv, tx_res);
2504	enetc_assign_rx_resources(priv, rx_res);
2505	enetc_setup_bdrs(priv, extended);
 
 
 
 
 
2506	enetc_start(ndev);
2507
2508	return 0;
2509
 
 
2510err_alloc_rx:
2511	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2512err_alloc_tx:
2513	if (priv->phylink)
2514		phylink_disconnect_phy(priv->phylink);
2515err_phy_connect:
2516	enetc_free_irqs(priv);
2517
2518	return err;
2519}
2520EXPORT_SYMBOL_GPL(enetc_open);
2521
2522void enetc_stop(struct net_device *ndev)
2523{
2524	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2525	int i;
2526
2527	netif_tx_stop_all_queues(ndev);
2528
2529	enetc_disable_bdrs(priv);
2530
2531	for (i = 0; i < priv->bdr_int_num; i++) {
2532		int irq = pci_irq_vector(priv->si->pdev,
2533					 ENETC_BDR_INT_BASE_IDX + i);
2534
2535		disable_irq(irq);
2536		napi_synchronize(&priv->int_vector[i]->napi);
2537		napi_disable(&priv->int_vector[i]->napi);
2538	}
2539
2540	enetc_wait_bdrs(priv);
 
 
 
2541
2542	enetc_clear_interrupts(priv);
2543}
2544EXPORT_SYMBOL_GPL(enetc_stop);
2545
2546int enetc_close(struct net_device *ndev)
2547{
2548	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2549
2550	enetc_stop(ndev);
 
2551
2552	if (priv->phylink) {
2553		phylink_stop(priv->phylink);
2554		phylink_disconnect_phy(priv->phylink);
2555	} else {
2556		netif_carrier_off(ndev);
2557	}
2558
2559	enetc_free_rxtx_rings(priv);
2560
2561	/* Avoids dangling pointers and also frees old resources */
2562	enetc_assign_rx_resources(priv, NULL);
2563	enetc_assign_tx_resources(priv, NULL);
2564
2565	enetc_free_irqs(priv);
2566
2567	return 0;
2568}
2569EXPORT_SYMBOL_GPL(enetc_close);
2570
2571static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
2572			     int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
2573			     void *ctx)
2574{
2575	struct enetc_bdr_resource *tx_res, *rx_res;
2576	int err;
2577
2578	ASSERT_RTNL();
2579
2580	/* If the interface is down, run the callback right away,
2581	 * without reconfiguration.
2582	 */
2583	if (!netif_running(priv->ndev)) {
2584		if (cb) {
2585			err = cb(priv, ctx);
2586			if (err)
2587				return err;
2588		}
2589
2590		return 0;
2591	}
2592
2593	tx_res = enetc_alloc_tx_resources(priv);
2594	if (IS_ERR(tx_res)) {
2595		err = PTR_ERR(tx_res);
2596		goto out;
2597	}
2598
2599	rx_res = enetc_alloc_rx_resources(priv, extended);
2600	if (IS_ERR(rx_res)) {
2601		err = PTR_ERR(rx_res);
2602		goto out_free_tx_res;
2603	}
2604
2605	enetc_stop(priv->ndev);
2606	enetc_free_rxtx_rings(priv);
2607
2608	/* Interface is down, run optional callback now */
2609	if (cb) {
2610		err = cb(priv, ctx);
2611		if (err)
2612			goto out_restart;
2613	}
2614
2615	enetc_assign_tx_resources(priv, tx_res);
2616	enetc_assign_rx_resources(priv, rx_res);
2617	enetc_setup_bdrs(priv, extended);
2618	enetc_start(priv->ndev);
2619
2620	return 0;
2621
2622out_restart:
2623	enetc_setup_bdrs(priv, extended);
2624	enetc_start(priv->ndev);
2625	enetc_free_rx_resources(rx_res, priv->num_rx_rings);
2626out_free_tx_res:
2627	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2628out:
2629	return err;
2630}
2631
2632static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
2633{
2634	int i;
2635
2636	for (i = 0; i < priv->num_tx_rings; i++)
2637		netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,
2638			   priv->tx_ring[i]->prio);
2639}
2640
2641void enetc_reset_tc_mqprio(struct net_device *ndev)
2642{
2643	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2644	struct enetc_hw *hw = &priv->si->hw;
2645	struct enetc_bdr *tx_ring;
2646	int num_stack_tx_queues;
2647	int i;
2648
2649	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2650
2651	netdev_reset_tc(ndev);
2652	netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2653	priv->min_num_stack_tx_queues = num_possible_cpus();
2654
2655	/* Reset all ring priorities to 0 */
2656	for (i = 0; i < priv->num_tx_rings; i++) {
2657		tx_ring = priv->tx_ring[i];
2658		tx_ring->prio = 0;
2659		enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2660	}
2661
2662	enetc_debug_tx_ring_prios(priv);
2663
2664	enetc_change_preemptible_tcs(priv, 0);
2665}
2666EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);
2667
2668int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
2669{
2670	struct tc_mqprio_qopt_offload *mqprio = type_data;
2671	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2672	struct tc_mqprio_qopt *qopt = &mqprio->qopt;
2673	struct enetc_hw *hw = &priv->si->hw;
2674	int num_stack_tx_queues = 0;
2675	struct enetc_bdr *tx_ring;
2676	u8 num_tc = qopt->num_tc;
2677	int offset, count;
2678	int err, tc, q;
2679
2680	if (!num_tc) {
2681		enetc_reset_tc_mqprio(ndev);
2682		return 0;
2683	}
2684
2685	err = netdev_set_num_tc(ndev, num_tc);
2686	if (err)
2687		return err;
2688
2689	for (tc = 0; tc < num_tc; tc++) {
2690		offset = qopt->offset[tc];
2691		count = qopt->count[tc];
2692		num_stack_tx_queues += count;
2693
2694		err = netdev_set_tc_queue(ndev, tc, count, offset);
2695		if (err)
2696			goto err_reset_tc;
2697
2698		for (q = offset; q < offset + count; q++) {
2699			tx_ring = priv->tx_ring[q];
2700			/* The prio_tc_map is skb_tx_hash()'s way of selecting
2701			 * between TX queues based on skb->priority. As such,
2702			 * there's nothing to offload based on it.
2703			 * Make the mqprio "traffic class" be the priority of
2704			 * this ring group, and leave the Tx IPV to traffic
2705			 * class mapping as its default mapping value of 1:1.
2706			 */
2707			tx_ring->prio = tc;
2708			enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2709		}
2710	}
2711
2712	err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2713	if (err)
2714		goto err_reset_tc;
2715
2716	priv->min_num_stack_tx_queues = num_stack_tx_queues;
2717
2718	enetc_debug_tx_ring_prios(priv);
2719
2720	enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
2721
2722	return 0;
2723
2724err_reset_tc:
2725	enetc_reset_tc_mqprio(ndev);
2726	return err;
2727}
2728EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);
2729
2730static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
2731{
2732	struct bpf_prog *old_prog, *prog = ctx;
2733	int num_stack_tx_queues;
2734	int err, i;
2735
2736	old_prog = xchg(&priv->xdp_prog, prog);
2737
2738	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2739	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
2740	if (err) {
2741		xchg(&priv->xdp_prog, old_prog);
2742		return err;
2743	}
2744
2745	if (old_prog)
2746		bpf_prog_put(old_prog);
2747
2748	for (i = 0; i < priv->num_rx_rings; i++) {
2749		struct enetc_bdr *rx_ring = priv->rx_ring[i];
2750
2751		rx_ring->xdp.prog = prog;
2752
2753		if (prog)
2754			rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2755		else
2756			rx_ring->buffer_offset = ENETC_RXB_PAD;
2757	}
2758
2759	return 0;
2760}
2761
2762static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
2763				struct netlink_ext_ack *extack)
2764{
2765	int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;
2766	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2767	bool extended;
2768
2769	if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >
2770	    priv->num_tx_rings) {
2771		NL_SET_ERR_MSG_FMT_MOD(extack,
2772				       "Reserving %d XDP TXQs does not leave a minimum of %d for stack (total %d)",
2773				       num_xdp_tx_queues,
2774				       priv->min_num_stack_tx_queues,
2775				       priv->num_tx_rings);
2776		return -EBUSY;
2777	}
2778
2779	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2780
2781	/* The buffer layout is changing, so we need to drain the old
2782	 * RX buffers and seed new ones.
2783	 */
2784	return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);
2785}
2786
2787int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
2788{
2789	switch (bpf->command) {
2790	case XDP_SETUP_PROG:
2791		return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);
2792	default:
2793		return -EINVAL;
2794	}
2795
2796	return 0;
2797}
2798EXPORT_SYMBOL_GPL(enetc_setup_bpf);
2799
2800struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2801{
2802	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2803	struct net_device_stats *stats = &ndev->stats;
2804	unsigned long packets = 0, bytes = 0;
2805	unsigned long tx_dropped = 0;
2806	int i;
2807
2808	for (i = 0; i < priv->num_rx_rings; i++) {
2809		packets += priv->rx_ring[i]->stats.packets;
2810		bytes	+= priv->rx_ring[i]->stats.bytes;
2811	}
2812
2813	stats->rx_packets = packets;
2814	stats->rx_bytes = bytes;
2815	bytes = 0;
2816	packets = 0;
2817
2818	for (i = 0; i < priv->num_tx_rings; i++) {
2819		packets += priv->tx_ring[i]->stats.packets;
2820		bytes	+= priv->tx_ring[i]->stats.bytes;
2821		tx_dropped += priv->tx_ring[i]->stats.win_drop;
2822	}
2823
2824	stats->tx_packets = packets;
2825	stats->tx_bytes = bytes;
2826	stats->tx_dropped = tx_dropped;
2827
2828	return stats;
2829}
2830EXPORT_SYMBOL_GPL(enetc_get_stats);
2831
2832static int enetc_set_rss(struct net_device *ndev, int en)
2833{
2834	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2835	struct enetc_hw *hw = &priv->si->hw;
2836	u32 reg;
2837
2838	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2839
2840	reg = enetc_rd(hw, ENETC_SIMR);
2841	reg &= ~ENETC_SIMR_RSSE;
2842	reg |= (en) ? ENETC_SIMR_RSSE : 0;
2843	enetc_wr(hw, ENETC_SIMR, reg);
2844
2845	return 0;
2846}
2847
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2848static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
2849{
2850	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2851	struct enetc_hw *hw = &priv->si->hw;
2852	int i;
2853
2854	for (i = 0; i < priv->num_rx_rings; i++)
2855		enetc_bdr_enable_rxvlan(hw, i, en);
2856}
2857
2858static void enetc_enable_txvlan(struct net_device *ndev, bool en)
2859{
2860	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2861	struct enetc_hw *hw = &priv->si->hw;
2862	int i;
2863
2864	for (i = 0; i < priv->num_tx_rings; i++)
2865		enetc_bdr_enable_txvlan(hw, i, en);
2866}
2867
2868void enetc_set_features(struct net_device *ndev, netdev_features_t features)
 
2869{
2870	netdev_features_t changed = ndev->features ^ features;
 
2871
2872	if (changed & NETIF_F_RXHASH)
2873		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2874
2875	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2876		enetc_enable_rxvlan(ndev,
2877				    !!(features & NETIF_F_HW_VLAN_CTAG_RX));
2878
2879	if (changed & NETIF_F_HW_VLAN_CTAG_TX)
2880		enetc_enable_txvlan(ndev,
2881				    !!(features & NETIF_F_HW_VLAN_CTAG_TX));
 
 
 
 
 
2882}
2883EXPORT_SYMBOL_GPL(enetc_set_features);
2884
2885#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2886static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2887{
2888	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2889	int err, new_offloads = priv->active_offloads;
2890	struct hwtstamp_config config;
 
2891
2892	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2893		return -EFAULT;
2894
2895	switch (config.tx_type) {
2896	case HWTSTAMP_TX_OFF:
2897		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2898		break;
2899	case HWTSTAMP_TX_ON:
2900		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2901		new_offloads |= ENETC_F_TX_TSTAMP;
2902		break;
2903	case HWTSTAMP_TX_ONESTEP_SYNC:
2904		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2905		new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
2906		break;
2907	default:
2908		return -ERANGE;
2909	}
2910
 
2911	switch (config.rx_filter) {
2912	case HWTSTAMP_FILTER_NONE:
2913		new_offloads &= ~ENETC_F_RX_TSTAMP;
2914		break;
2915	default:
2916		new_offloads |= ENETC_F_RX_TSTAMP;
2917		config.rx_filter = HWTSTAMP_FILTER_ALL;
2918	}
2919
2920	if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {
2921		bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);
2922
2923		err = enetc_reconfigure(priv, extended, NULL, NULL);
2924		if (err)
2925			return err;
2926	}
2927
2928	priv->active_offloads = new_offloads;
2929
2930	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2931	       -EFAULT : 0;
2932}
2933
2934static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2935{
2936	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2937	struct hwtstamp_config config;
2938
2939	config.flags = 0;
2940
2941	if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
2942		config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
2943	else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2944		config.tx_type = HWTSTAMP_TX_ON;
2945	else
2946		config.tx_type = HWTSTAMP_TX_OFF;
2947
2948	config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2949			    HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2950
2951	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2952	       -EFAULT : 0;
2953}
2954#endif
2955
2956int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2957{
2958	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2959#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2960	if (cmd == SIOCSHWTSTAMP)
2961		return enetc_hwtstamp_set(ndev, rq);
2962	if (cmd == SIOCGHWTSTAMP)
2963		return enetc_hwtstamp_get(ndev, rq);
2964#endif
2965
2966	if (!priv->phylink)
2967		return -EOPNOTSUPP;
2968
2969	return phylink_mii_ioctl(priv->phylink, rq, cmd);
2970}
2971EXPORT_SYMBOL_GPL(enetc_ioctl);
2972
2973int enetc_alloc_msix(struct enetc_ndev_priv *priv)
2974{
2975	struct pci_dev *pdev = priv->si->pdev;
2976	int num_stack_tx_queues;
2977	int first_xdp_tx_ring;
2978	int i, n, err, nvec;
2979	int v_tx_rings;
 
2980
2981	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
2982	/* allocate MSIX for both messaging and Rx/Tx interrupts */
2983	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
2984
2985	if (n < 0)
2986		return n;
2987
2988	if (n != nvec)
2989		return -EPERM;
2990
2991	/* # of tx rings per int vector */
2992	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
2993
2994	for (i = 0; i < priv->bdr_int_num; i++) {
2995		struct enetc_int_vector *v;
2996		struct enetc_bdr *bdr;
2997		int j;
2998
2999		v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
3000		if (!v) {
3001			err = -ENOMEM;
3002			goto fail;
3003		}
3004
3005		priv->int_vector[i] = v;
3006
3007		bdr = &v->rx_ring;
3008		bdr->index = i;
3009		bdr->ndev = priv->ndev;
3010		bdr->dev = priv->dev;
3011		bdr->bd_count = priv->rx_bd_count;
3012		bdr->buffer_offset = ENETC_RXB_PAD;
3013		priv->rx_ring[i] = bdr;
3014
3015		err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
3016		if (err) {
3017			kfree(v);
3018			goto fail;
3019		}
3020
3021		err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
3022						 MEM_TYPE_PAGE_SHARED, NULL);
3023		if (err) {
3024			xdp_rxq_info_unreg(&bdr->xdp.rxq);
3025			kfree(v);
3026			goto fail;
3027		}
3028
3029		/* init defaults for adaptive IC */
3030		if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
3031			v->rx_ictt = 0x1;
3032			v->rx_dim_en = true;
3033		}
3034		INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
3035		netif_napi_add(priv->ndev, &v->napi, enetc_poll);
 
3036		v->count_tx_rings = v_tx_rings;
3037
3038		for (j = 0; j < v_tx_rings; j++) {
3039			int idx;
3040
3041			/* default tx ring mapping policy */
3042			idx = priv->bdr_int_num * j + i;
 
 
 
 
3043			__set_bit(idx, &v->tx_rings_map);
3044			bdr = &v->tx_ring[j];
3045			bdr->index = idx;
3046			bdr->ndev = priv->ndev;
3047			bdr->dev = priv->dev;
3048			bdr->bd_count = priv->tx_bd_count;
3049			priv->tx_ring[idx] = bdr;
3050		}
3051	}
3052
3053	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3054
3055	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3056	if (err)
3057		goto fail;
3058
3059	err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);
3060	if (err)
3061		goto fail;
3062
3063	priv->min_num_stack_tx_queues = num_possible_cpus();
3064	first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
3065	priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
3066
3067	return 0;
3068
3069fail:
3070	while (i--) {
3071		struct enetc_int_vector *v = priv->int_vector[i];
3072		struct enetc_bdr *rx_ring = &v->rx_ring;
3073
3074		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3075		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3076		netif_napi_del(&v->napi);
3077		cancel_work_sync(&v->rx_dim.work);
3078		kfree(v);
3079	}
3080
3081	pci_free_irq_vectors(pdev);
3082
3083	return err;
3084}
3085EXPORT_SYMBOL_GPL(enetc_alloc_msix);
3086
3087void enetc_free_msix(struct enetc_ndev_priv *priv)
3088{
3089	int i;
3090
3091	for (i = 0; i < priv->bdr_int_num; i++) {
3092		struct enetc_int_vector *v = priv->int_vector[i];
3093		struct enetc_bdr *rx_ring = &v->rx_ring;
3094
3095		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3096		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3097		netif_napi_del(&v->napi);
3098		cancel_work_sync(&v->rx_dim.work);
3099	}
3100
3101	for (i = 0; i < priv->num_rx_rings; i++)
3102		priv->rx_ring[i] = NULL;
3103
3104	for (i = 0; i < priv->num_tx_rings; i++)
3105		priv->tx_ring[i] = NULL;
3106
3107	for (i = 0; i < priv->bdr_int_num; i++) {
3108		kfree(priv->int_vector[i]);
3109		priv->int_vector[i] = NULL;
3110	}
3111
3112	/* disable all MSIX for this device */
3113	pci_free_irq_vectors(priv->si->pdev);
3114}
3115EXPORT_SYMBOL_GPL(enetc_free_msix);
3116
3117static void enetc_kfree_si(struct enetc_si *si)
3118{
3119	char *p = (char *)si - si->pad;
3120
3121	kfree(p);
3122}
3123
3124static void enetc_detect_errata(struct enetc_si *si)
3125{
3126	if (si->pdev->revision == ENETC_REV1)
3127		si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
 
3128}
3129
3130int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
3131{
3132	struct enetc_si *si, *p;
3133	struct enetc_hw *hw;
3134	size_t alloc_size;
3135	int err, len;
3136
3137	pcie_flr(pdev);
3138	err = pci_enable_device_mem(pdev);
3139	if (err)
3140		return dev_err_probe(&pdev->dev, err, "device enable failed\n");
 
 
3141
3142	/* set up for high or low dma */
3143	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3144	if (err) {
3145		dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3146		goto err_dma;
 
 
 
 
3147	}
3148
3149	err = pci_request_mem_regions(pdev, name);
3150	if (err) {
3151		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
3152		goto err_pci_mem_reg;
3153	}
3154
3155	pci_set_master(pdev);
3156
3157	alloc_size = sizeof(struct enetc_si);
3158	if (sizeof_priv) {
3159		/* align priv to 32B */
3160		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
3161		alloc_size += sizeof_priv;
3162	}
3163	/* force 32B alignment for enetc_si */
3164	alloc_size += ENETC_SI_ALIGN - 1;
3165
3166	p = kzalloc(alloc_size, GFP_KERNEL);
3167	if (!p) {
3168		err = -ENOMEM;
3169		goto err_alloc_si;
3170	}
3171
3172	si = PTR_ALIGN(p, ENETC_SI_ALIGN);
3173	si->pad = (char *)si - (char *)p;
3174
3175	pci_set_drvdata(pdev, si);
3176	si->pdev = pdev;
3177	hw = &si->hw;
3178
3179	len = pci_resource_len(pdev, ENETC_BAR_REGS);
3180	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
3181	if (!hw->reg) {
3182		err = -ENXIO;
3183		dev_err(&pdev->dev, "ioremap() failed\n");
3184		goto err_ioremap;
3185	}
3186	if (len > ENETC_PORT_BASE)
3187		hw->port = hw->reg + ENETC_PORT_BASE;
3188	if (len > ENETC_GLOBAL_BASE)
3189		hw->global = hw->reg + ENETC_GLOBAL_BASE;
3190
3191	enetc_detect_errata(si);
3192
3193	return 0;
3194
3195err_ioremap:
3196	enetc_kfree_si(si);
3197err_alloc_si:
3198	pci_release_mem_regions(pdev);
3199err_pci_mem_reg:
3200err_dma:
3201	pci_disable_device(pdev);
3202
3203	return err;
3204}
3205EXPORT_SYMBOL_GPL(enetc_pci_probe);
3206
3207void enetc_pci_remove(struct pci_dev *pdev)
3208{
3209	struct enetc_si *si = pci_get_drvdata(pdev);
3210	struct enetc_hw *hw = &si->hw;
3211
3212	iounmap(hw->reg);
3213	enetc_kfree_si(si);
3214	pci_release_mem_regions(pdev);
3215	pci_disable_device(pdev);
3216}
3217EXPORT_SYMBOL_GPL(enetc_pci_remove);
3218
3219MODULE_DESCRIPTION("NXP ENETC Ethernet driver");
3220MODULE_LICENSE("Dual BSD/GPL");