Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */
3
4/* TSN endpoint Ethernet MAC driver
5 *
6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time
7 * communication. It is designed for endpoints within TSN (Time Sensitive
8 * Networking) networks; e.g., for PLCs in the industrial automation case.
9 *
10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used
11 * by the driver.
12 *
13 * More information can be found here:
14 * - www.embedded-experts.at/tsn
15 * - www.engleder-embedded.com
16 */
17
18#include "tsnep.h"
19#include "tsnep_hw.h"
20
21#include <linux/module.h>
22#include <linux/of.h>
23#include <linux/of_net.h>
24#include <linux/of_mdio.h>
25#include <linux/interrupt.h>
26#include <linux/etherdevice.h>
27#include <linux/phy.h>
28#include <linux/iopoll.h>
29#include <linux/bpf.h>
30#include <linux/bpf_trace.h>
31#include <net/page_pool/helpers.h>
32#include <net/xdp_sock_drv.h>
33
34#define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
35#define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4)
36#define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \
37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
38/* XSK buffer shall store at least Q-in-Q frame */
39#define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \
40 ETH_FRAME_LEN + ETH_FCS_LEN + \
41 VLAN_HLEN * 2, 4))
42
43#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
44#define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF))
45#else
46#define DMA_ADDR_HIGH(dma_addr) ((u32)(0))
47#endif
48#define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF))
49
50#define TSNEP_COALESCE_USECS_DEFAULT 64
51#define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \
52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1)
53
54/* mapping type */
55#define TSNEP_TX_TYPE_MAP BIT(0)
56#define TSNEP_TX_TYPE_MAP_PAGE BIT(1)
57#define TSNEP_TX_TYPE_INLINE BIT(2)
58/* buffer type */
59#define TSNEP_TX_TYPE_SKB BIT(8)
60#define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP)
61#define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE)
62#define TSNEP_TX_TYPE_SKB_FRAG BIT(9)
63#define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE)
64#define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE)
65#define TSNEP_TX_TYPE_XDP_TX BIT(10)
66#define TSNEP_TX_TYPE_XDP_NDO BIT(11)
67#define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE)
68#define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO)
69#define TSNEP_TX_TYPE_XSK BIT(12)
70
71#define TSNEP_XDP_TX BIT(0)
72#define TSNEP_XDP_REDIRECT BIT(1)
73
74static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask)
75{
76 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
77}
78
79static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask)
80{
81 mask |= ECM_INT_DISABLE;
82 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
83}
84
85static irqreturn_t tsnep_irq(int irq, void *arg)
86{
87 struct tsnep_adapter *adapter = arg;
88 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE);
89
90 /* acknowledge interrupt */
91 if (active != 0)
92 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE);
93
94 /* handle link interrupt */
95 if ((active & ECM_INT_LINK) != 0)
96 phy_mac_interrupt(adapter->netdev->phydev);
97
98 /* handle TX/RX queue 0 interrupt */
99 if ((active & adapter->queue[0].irq_mask) != 0) {
100 if (napi_schedule_prep(&adapter->queue[0].napi)) {
101 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask);
102 /* schedule after masking to avoid races */
103 __napi_schedule(&adapter->queue[0].napi);
104 }
105 }
106
107 return IRQ_HANDLED;
108}
109
110static irqreturn_t tsnep_irq_txrx(int irq, void *arg)
111{
112 struct tsnep_queue *queue = arg;
113
114 /* handle TX/RX queue interrupt */
115 if (napi_schedule_prep(&queue->napi)) {
116 tsnep_disable_irq(queue->adapter, queue->irq_mask);
117 /* schedule after masking to avoid races */
118 __napi_schedule(&queue->napi);
119 }
120
121 return IRQ_HANDLED;
122}
123
124int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs)
125{
126 if (usecs > TSNEP_COALESCE_USECS_MAX)
127 return -ERANGE;
128
129 usecs /= ECM_INT_DELAY_BASE_US;
130 usecs <<= ECM_INT_DELAY_SHIFT;
131 usecs &= ECM_INT_DELAY_MASK;
132
133 queue->irq_delay &= ~ECM_INT_DELAY_MASK;
134 queue->irq_delay |= usecs;
135 iowrite8(queue->irq_delay, queue->irq_delay_addr);
136
137 return 0;
138}
139
140u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue)
141{
142 u32 usecs;
143
144 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK);
145 usecs >>= ECM_INT_DELAY_SHIFT;
146 usecs *= ECM_INT_DELAY_BASE_US;
147
148 return usecs;
149}
150
151static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum)
152{
153 struct tsnep_adapter *adapter = bus->priv;
154 u32 md;
155 int retval;
156
157 md = ECM_MD_READ;
158 if (!adapter->suppress_preamble)
159 md |= ECM_MD_PREAMBLE;
160 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
161 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
162 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
163 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
164 !(md & ECM_MD_BUSY), 16, 1000);
165 if (retval != 0)
166 return retval;
167
168 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT;
169}
170
171static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum,
172 u16 val)
173{
174 struct tsnep_adapter *adapter = bus->priv;
175 u32 md;
176 int retval;
177
178 md = ECM_MD_WRITE;
179 if (!adapter->suppress_preamble)
180 md |= ECM_MD_PREAMBLE;
181 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
182 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
183 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK;
184 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
185 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
186 !(md & ECM_MD_BUSY), 16, 1000);
187 if (retval != 0)
188 return retval;
189
190 return 0;
191}
192
193static void tsnep_set_link_mode(struct tsnep_adapter *adapter)
194{
195 u32 mode;
196
197 switch (adapter->phydev->speed) {
198 case SPEED_100:
199 mode = ECM_LINK_MODE_100;
200 break;
201 case SPEED_1000:
202 mode = ECM_LINK_MODE_1000;
203 break;
204 default:
205 mode = ECM_LINK_MODE_OFF;
206 break;
207 }
208 iowrite32(mode, adapter->addr + ECM_STATUS);
209}
210
211static void tsnep_phy_link_status_change(struct net_device *netdev)
212{
213 struct tsnep_adapter *adapter = netdev_priv(netdev);
214 struct phy_device *phydev = netdev->phydev;
215
216 if (phydev->link)
217 tsnep_set_link_mode(adapter);
218
219 phy_print_status(netdev->phydev);
220}
221
222static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable)
223{
224 int retval;
225
226 retval = phy_loopback(adapter->phydev, enable);
227
228 /* PHY link state change is not signaled if loopback is enabled, it
229 * would delay a working loopback anyway, let's ensure that loopback
230 * is working immediately by setting link mode directly
231 */
232 if (!retval && enable) {
233 netif_carrier_on(adapter->netdev);
234 tsnep_set_link_mode(adapter);
235 }
236
237 return retval;
238}
239
240static int tsnep_phy_open(struct tsnep_adapter *adapter)
241{
242 struct phy_device *phydev;
243 struct ethtool_keee ethtool_keee;
244 int retval;
245
246 retval = phy_connect_direct(adapter->netdev, adapter->phydev,
247 tsnep_phy_link_status_change,
248 adapter->phy_mode);
249 if (retval)
250 return retval;
251 phydev = adapter->netdev->phydev;
252
253 /* MAC supports only 100Mbps|1000Mbps full duplex
254 * SPE (Single Pair Ethernet) is also an option but not implemented yet
255 */
256 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT);
257 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT);
258 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT);
259 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
260
261 /* disable EEE autoneg, EEE not supported by TSNEP */
262 memset(ðtool_keee, 0, sizeof(ethtool_keee));
263 phy_ethtool_set_eee(adapter->phydev, ðtool_keee);
264
265 adapter->phydev->irq = PHY_MAC_INTERRUPT;
266 phy_start(adapter->phydev);
267
268 return 0;
269}
270
271static void tsnep_phy_close(struct tsnep_adapter *adapter)
272{
273 phy_stop(adapter->netdev->phydev);
274 phy_disconnect(adapter->netdev->phydev);
275}
276
277static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx)
278{
279 struct device *dmadev = tx->adapter->dmadev;
280 int i;
281
282 memset(tx->entry, 0, sizeof(tx->entry));
283
284 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
285 if (tx->page[i]) {
286 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i],
287 tx->page_dma[i]);
288 tx->page[i] = NULL;
289 tx->page_dma[i] = 0;
290 }
291 }
292}
293
294static int tsnep_tx_ring_create(struct tsnep_tx *tx)
295{
296 struct device *dmadev = tx->adapter->dmadev;
297 struct tsnep_tx_entry *entry;
298 struct tsnep_tx_entry *next_entry;
299 int i, j;
300 int retval;
301
302 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
303 tx->page[i] =
304 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i],
305 GFP_KERNEL);
306 if (!tx->page[i]) {
307 retval = -ENOMEM;
308 goto alloc_failed;
309 }
310 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
311 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
312 entry->desc_wb = (struct tsnep_tx_desc_wb *)
313 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j);
314 entry->desc = (struct tsnep_tx_desc *)
315 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
316 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j;
317 entry->owner_user_flag = false;
318 }
319 }
320 for (i = 0; i < TSNEP_RING_SIZE; i++) {
321 entry = &tx->entry[i];
322 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK];
323 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
324 }
325
326 return 0;
327
328alloc_failed:
329 tsnep_tx_ring_cleanup(tx);
330 return retval;
331}
332
333static void tsnep_tx_init(struct tsnep_tx *tx)
334{
335 dma_addr_t dma;
336
337 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
338 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW);
339 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH);
340 tx->write = 0;
341 tx->read = 0;
342 tx->owner_counter = 1;
343 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
344}
345
346static void tsnep_tx_enable(struct tsnep_tx *tx)
347{
348 struct netdev_queue *nq;
349
350 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
351
352 __netif_tx_lock_bh(nq);
353 netif_tx_wake_queue(nq);
354 __netif_tx_unlock_bh(nq);
355}
356
357static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi)
358{
359 struct netdev_queue *nq;
360 u32 val;
361
362 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
363
364 __netif_tx_lock_bh(nq);
365 netif_tx_stop_queue(nq);
366 __netif_tx_unlock_bh(nq);
367
368 /* wait until TX is done in hardware */
369 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
370 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
371 1000000);
372
373 /* wait until TX is also done in software */
374 while (READ_ONCE(tx->read) != tx->write) {
375 napi_schedule(napi);
376 napi_synchronize(napi);
377 }
378}
379
380static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,
381 bool last)
382{
383 struct tsnep_tx_entry *entry = &tx->entry[index];
384
385 entry->properties = 0;
386 /* xdpf and zc are union with skb */
387 if (entry->skb) {
388 entry->properties = length & TSNEP_DESC_LENGTH_MASK;
389 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
390 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
391 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS))
392 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG;
393
394 /* toggle user flag to prevent false acknowledge
395 *
396 * Only the first fragment is acknowledged. For all other
397 * fragments no acknowledge is done and the last written owner
398 * counter stays in the writeback descriptor. Therefore, it is
399 * possible that the last written owner counter is identical to
400 * the new incremented owner counter and a false acknowledge is
401 * detected before the real acknowledge has been done by
402 * hardware.
403 *
404 * The user flag is used to prevent this situation. The user
405 * flag is copied to the writeback descriptor by the hardware
406 * and is used as additional acknowledge data. By toggeling the
407 * user flag only for the first fragment (which is
408 * acknowledged), it is guaranteed that the last acknowledge
409 * done for this descriptor has used a different user flag and
410 * cannot be detected as false acknowledge.
411 */
412 entry->owner_user_flag = !entry->owner_user_flag;
413 }
414 if (last)
415 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG;
416 if (index == tx->increment_owner_counter) {
417 tx->owner_counter++;
418 if (tx->owner_counter == 4)
419 tx->owner_counter = 1;
420 tx->increment_owner_counter--;
421 if (tx->increment_owner_counter < 0)
422 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
423 }
424 entry->properties |=
425 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
426 TSNEP_DESC_OWNER_COUNTER_MASK;
427 if (entry->owner_user_flag)
428 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG;
429 entry->desc->more_properties =
430 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK);
431 if (entry->type & TSNEP_TX_TYPE_INLINE)
432 entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG;
433
434 /* descriptor properties shall be written last, because valid data is
435 * signaled there
436 */
437 dma_wmb();
438
439 entry->desc->properties = __cpu_to_le32(entry->properties);
440}
441
442static int tsnep_tx_desc_available(struct tsnep_tx *tx)
443{
444 if (tx->read <= tx->write)
445 return TSNEP_RING_SIZE - tx->write + tx->read - 1;
446 else
447 return tx->read - tx->write - 1;
448}
449
450static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry,
451 struct device *dmadev, dma_addr_t *dma)
452{
453 unsigned int len;
454 int mapped;
455
456 len = skb_frag_size(frag);
457 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
458 *dma = skb_frag_dma_map(dmadev, frag, 0, len, DMA_TO_DEVICE);
459 if (dma_mapping_error(dmadev, *dma))
460 return -ENOMEM;
461 entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE;
462 mapped = 1;
463 } else {
464 void *fragdata = skb_frag_address_safe(frag);
465
466 if (likely(fragdata)) {
467 memcpy(&entry->desc->tx, fragdata, len);
468 } else {
469 struct page *page = skb_frag_page(frag);
470
471 fragdata = kmap_local_page(page);
472 memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag),
473 len);
474 kunmap_local(fragdata);
475 }
476 entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE;
477 mapped = 0;
478 }
479
480 return mapped;
481}
482
483static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count)
484{
485 struct device *dmadev = tx->adapter->dmadev;
486 struct tsnep_tx_entry *entry;
487 unsigned int len;
488 int map_len = 0;
489 dma_addr_t dma;
490 int i, mapped;
491
492 for (i = 0; i < count; i++) {
493 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
494
495 if (!i) {
496 len = skb_headlen(skb);
497 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
498 dma = dma_map_single(dmadev, skb->data, len,
499 DMA_TO_DEVICE);
500 if (dma_mapping_error(dmadev, dma))
501 return -ENOMEM;
502 entry->type = TSNEP_TX_TYPE_SKB_MAP;
503 mapped = 1;
504 } else {
505 memcpy(&entry->desc->tx, skb->data, len);
506 entry->type = TSNEP_TX_TYPE_SKB_INLINE;
507 mapped = 0;
508 }
509 } else {
510 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
511
512 len = skb_frag_size(frag);
513 mapped = tsnep_tx_map_frag(frag, entry, dmadev, &dma);
514 if (mapped < 0)
515 return mapped;
516 }
517
518 entry->len = len;
519 if (likely(mapped)) {
520 dma_unmap_addr_set(entry, dma, dma);
521 entry->desc->tx = __cpu_to_le64(dma);
522 }
523
524 map_len += len;
525 }
526
527 return map_len;
528}
529
530static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
531{
532 struct device *dmadev = tx->adapter->dmadev;
533 struct tsnep_tx_entry *entry;
534 int map_len = 0;
535 int i;
536
537 for (i = 0; i < count; i++) {
538 entry = &tx->entry[(index + i) & TSNEP_RING_MASK];
539
540 if (entry->len) {
541 if (entry->type & TSNEP_TX_TYPE_MAP)
542 dma_unmap_single(dmadev,
543 dma_unmap_addr(entry, dma),
544 dma_unmap_len(entry, len),
545 DMA_TO_DEVICE);
546 else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE)
547 dma_unmap_page(dmadev,
548 dma_unmap_addr(entry, dma),
549 dma_unmap_len(entry, len),
550 DMA_TO_DEVICE);
551 map_len += entry->len;
552 entry->len = 0;
553 }
554 }
555
556 return map_len;
557}
558
559static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
560 struct tsnep_tx *tx)
561{
562 int count = 1;
563 struct tsnep_tx_entry *entry;
564 int length;
565 int i;
566 int retval;
567
568 if (skb_shinfo(skb)->nr_frags > 0)
569 count += skb_shinfo(skb)->nr_frags;
570
571 if (tsnep_tx_desc_available(tx) < count) {
572 /* ring full, shall not happen because queue is stopped if full
573 * below
574 */
575 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
576
577 return NETDEV_TX_BUSY;
578 }
579
580 entry = &tx->entry[tx->write];
581 entry->skb = skb;
582
583 retval = tsnep_tx_map(skb, tx, count);
584 if (retval < 0) {
585 tsnep_tx_unmap(tx, tx->write, count);
586 dev_kfree_skb_any(entry->skb);
587 entry->skb = NULL;
588
589 tx->dropped++;
590
591 return NETDEV_TX_OK;
592 }
593 length = retval;
594
595 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
596 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
597
598 for (i = 0; i < count; i++)
599 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
600 i == count - 1);
601 tx->write = (tx->write + count) & TSNEP_RING_MASK;
602
603 skb_tx_timestamp(skb);
604
605 /* descriptor properties shall be valid before hardware is notified */
606 dma_wmb();
607
608 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
609
610 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) {
611 /* ring can get full with next frame */
612 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
613 }
614
615 return NETDEV_TX_OK;
616}
617
618static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx,
619 struct skb_shared_info *shinfo, int count, u32 type)
620{
621 struct device *dmadev = tx->adapter->dmadev;
622 struct tsnep_tx_entry *entry;
623 struct page *page;
624 skb_frag_t *frag;
625 unsigned int len;
626 int map_len = 0;
627 dma_addr_t dma;
628 void *data;
629 int i;
630
631 frag = NULL;
632 len = xdpf->len;
633 for (i = 0; i < count; i++) {
634 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
635 if (type & TSNEP_TX_TYPE_XDP_NDO) {
636 data = unlikely(frag) ? skb_frag_address(frag) :
637 xdpf->data;
638 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE);
639 if (dma_mapping_error(dmadev, dma))
640 return -ENOMEM;
641
642 entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE;
643 } else {
644 page = unlikely(frag) ? skb_frag_page(frag) :
645 virt_to_page(xdpf->data);
646 dma = page_pool_get_dma_addr(page);
647 if (unlikely(frag))
648 dma += skb_frag_off(frag);
649 else
650 dma += sizeof(*xdpf) + xdpf->headroom;
651 dma_sync_single_for_device(dmadev, dma, len,
652 DMA_BIDIRECTIONAL);
653
654 entry->type = TSNEP_TX_TYPE_XDP_TX;
655 }
656
657 entry->len = len;
658 dma_unmap_addr_set(entry, dma, dma);
659
660 entry->desc->tx = __cpu_to_le64(dma);
661
662 map_len += len;
663
664 if (i + 1 < count) {
665 frag = &shinfo->frags[i];
666 len = skb_frag_size(frag);
667 }
668 }
669
670 return map_len;
671}
672
673/* This function requires __netif_tx_lock is held by the caller. */
674static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf,
675 struct tsnep_tx *tx, u32 type)
676{
677 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf);
678 struct tsnep_tx_entry *entry;
679 int count, length, retval, i;
680
681 count = 1;
682 if (unlikely(xdp_frame_has_frags(xdpf)))
683 count += shinfo->nr_frags;
684
685 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
686 * will be available for normal TX path and queue is stopped there if
687 * necessary
688 */
689 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count))
690 return false;
691
692 entry = &tx->entry[tx->write];
693 entry->xdpf = xdpf;
694
695 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type);
696 if (retval < 0) {
697 tsnep_tx_unmap(tx, tx->write, count);
698 entry->xdpf = NULL;
699
700 tx->dropped++;
701
702 return false;
703 }
704 length = retval;
705
706 for (i = 0; i < count; i++)
707 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
708 i == count - 1);
709 tx->write = (tx->write + count) & TSNEP_RING_MASK;
710
711 /* descriptor properties shall be valid before hardware is notified */
712 dma_wmb();
713
714 return true;
715}
716
717static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx)
718{
719 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
720}
721
722static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,
723 struct xdp_buff *xdp,
724 struct netdev_queue *tx_nq, struct tsnep_tx *tx,
725 bool zc)
726{
727 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
728 bool xmit;
729 u32 type;
730
731 if (unlikely(!xdpf))
732 return false;
733
734 /* no page pool for zero copy */
735 if (zc)
736 type = TSNEP_TX_TYPE_XDP_NDO;
737 else
738 type = TSNEP_TX_TYPE_XDP_TX;
739
740 __netif_tx_lock(tx_nq, smp_processor_id());
741
742 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, type);
743
744 /* Avoid transmit queue timeout since we share it with the slow path */
745 if (xmit)
746 txq_trans_cond_update(tx_nq);
747
748 __netif_tx_unlock(tx_nq);
749
750 return xmit;
751}
752
753static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx)
754{
755 struct tsnep_tx_entry *entry;
756 dma_addr_t dma;
757
758 entry = &tx->entry[tx->write];
759 entry->zc = true;
760
761 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr);
762 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len);
763
764 entry->type = TSNEP_TX_TYPE_XSK;
765 entry->len = xdpd->len;
766
767 entry->desc->tx = __cpu_to_le64(dma);
768
769 return xdpd->len;
770}
771
772static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd,
773 struct tsnep_tx *tx)
774{
775 int length;
776
777 length = tsnep_xdp_tx_map_zc(xdpd, tx);
778
779 tsnep_tx_activate(tx, tx->write, length, true);
780 tx->write = (tx->write + 1) & TSNEP_RING_MASK;
781}
782
783static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx)
784{
785 int desc_available = tsnep_tx_desc_available(tx);
786 struct xdp_desc *descs = tx->xsk_pool->tx_descs;
787 int batch, i;
788
789 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
790 * will be available for normal TX path and queue is stopped there if
791 * necessary
792 */
793 if (desc_available <= (MAX_SKB_FRAGS + 1))
794 return;
795 desc_available -= MAX_SKB_FRAGS + 1;
796
797 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available);
798 for (i = 0; i < batch; i++)
799 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx);
800
801 if (batch) {
802 /* descriptor properties shall be valid before hardware is
803 * notified
804 */
805 dma_wmb();
806
807 tsnep_xdp_xmit_flush(tx);
808 }
809}
810
811static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
812{
813 struct tsnep_tx_entry *entry;
814 struct netdev_queue *nq;
815 int xsk_frames = 0;
816 int budget = 128;
817 int length;
818 int count;
819
820 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
821 __netif_tx_lock(nq, smp_processor_id());
822
823 do {
824 if (tx->read == tx->write)
825 break;
826
827 entry = &tx->entry[tx->read];
828 if ((__le32_to_cpu(entry->desc_wb->properties) &
829 TSNEP_TX_DESC_OWNER_MASK) !=
830 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
831 break;
832
833 /* descriptor properties shall be read first, because valid data
834 * is signaled there
835 */
836 dma_rmb();
837
838 count = 1;
839 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
840 skb_shinfo(entry->skb)->nr_frags > 0)
841 count += skb_shinfo(entry->skb)->nr_frags;
842 else if ((entry->type & TSNEP_TX_TYPE_XDP) &&
843 xdp_frame_has_frags(entry->xdpf))
844 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags;
845
846 length = tsnep_tx_unmap(tx, tx->read, count);
847
848 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
849 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) &&
850 (__le32_to_cpu(entry->desc_wb->properties) &
851 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) {
852 struct skb_shared_hwtstamps hwtstamps;
853 u64 timestamp;
854
855 if (skb_shinfo(entry->skb)->tx_flags &
856 SKBTX_HW_TSTAMP_USE_CYCLES)
857 timestamp =
858 __le64_to_cpu(entry->desc_wb->counter);
859 else
860 timestamp =
861 __le64_to_cpu(entry->desc_wb->timestamp);
862
863 memset(&hwtstamps, 0, sizeof(hwtstamps));
864 hwtstamps.hwtstamp = ns_to_ktime(timestamp);
865
866 skb_tstamp_tx(entry->skb, &hwtstamps);
867 }
868
869 if (entry->type & TSNEP_TX_TYPE_SKB)
870 napi_consume_skb(entry->skb, napi_budget);
871 else if (entry->type & TSNEP_TX_TYPE_XDP)
872 xdp_return_frame_rx_napi(entry->xdpf);
873 else
874 xsk_frames++;
875 /* xdpf and zc are union with skb */
876 entry->skb = NULL;
877
878 tx->read = (tx->read + count) & TSNEP_RING_MASK;
879
880 tx->packets++;
881 tx->bytes += length + ETH_FCS_LEN;
882
883 budget--;
884 } while (likely(budget));
885
886 if (tx->xsk_pool) {
887 if (xsk_frames)
888 xsk_tx_completed(tx->xsk_pool, xsk_frames);
889 if (xsk_uses_need_wakeup(tx->xsk_pool))
890 xsk_set_tx_need_wakeup(tx->xsk_pool);
891 tsnep_xdp_xmit_zc(tx);
892 }
893
894 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) &&
895 netif_tx_queue_stopped(nq)) {
896 netif_tx_wake_queue(nq);
897 }
898
899 __netif_tx_unlock(nq);
900
901 return budget != 0;
902}
903
904static bool tsnep_tx_pending(struct tsnep_tx *tx)
905{
906 struct tsnep_tx_entry *entry;
907 struct netdev_queue *nq;
908 bool pending = false;
909
910 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
911 __netif_tx_lock(nq, smp_processor_id());
912
913 if (tx->read != tx->write) {
914 entry = &tx->entry[tx->read];
915 if ((__le32_to_cpu(entry->desc_wb->properties) &
916 TSNEP_TX_DESC_OWNER_MASK) ==
917 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
918 pending = true;
919 }
920
921 __netif_tx_unlock(nq);
922
923 return pending;
924}
925
926static int tsnep_tx_open(struct tsnep_tx *tx)
927{
928 int retval;
929
930 retval = tsnep_tx_ring_create(tx);
931 if (retval)
932 return retval;
933
934 tsnep_tx_init(tx);
935
936 return 0;
937}
938
939static void tsnep_tx_close(struct tsnep_tx *tx)
940{
941 tsnep_tx_ring_cleanup(tx);
942}
943
944static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
945{
946 struct device *dmadev = rx->adapter->dmadev;
947 struct tsnep_rx_entry *entry;
948 int i;
949
950 for (i = 0; i < TSNEP_RING_SIZE; i++) {
951 entry = &rx->entry[i];
952 if (!rx->xsk_pool && entry->page)
953 page_pool_put_full_page(rx->page_pool, entry->page,
954 false);
955 if (rx->xsk_pool && entry->xdp)
956 xsk_buff_free(entry->xdp);
957 /* xdp is union with page */
958 entry->page = NULL;
959 }
960
961 if (rx->page_pool)
962 page_pool_destroy(rx->page_pool);
963
964 memset(rx->entry, 0, sizeof(rx->entry));
965
966 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
967 if (rx->page[i]) {
968 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i],
969 rx->page_dma[i]);
970 rx->page[i] = NULL;
971 rx->page_dma[i] = 0;
972 }
973 }
974}
975
976static int tsnep_rx_ring_create(struct tsnep_rx *rx)
977{
978 struct device *dmadev = rx->adapter->dmadev;
979 struct tsnep_rx_entry *entry;
980 struct page_pool_params pp_params = { 0 };
981 struct tsnep_rx_entry *next_entry;
982 int i, j;
983 int retval;
984
985 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
986 rx->page[i] =
987 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i],
988 GFP_KERNEL);
989 if (!rx->page[i]) {
990 retval = -ENOMEM;
991 goto failed;
992 }
993 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
994 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
995 entry->desc_wb = (struct tsnep_rx_desc_wb *)
996 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j);
997 entry->desc = (struct tsnep_rx_desc *)
998 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
999 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;
1000 }
1001 }
1002
1003 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
1004 pp_params.order = 0;
1005 pp_params.pool_size = TSNEP_RING_SIZE;
1006 pp_params.nid = dev_to_node(dmadev);
1007 pp_params.dev = dmadev;
1008 pp_params.dma_dir = DMA_BIDIRECTIONAL;
1009 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE;
1010 pp_params.offset = TSNEP_RX_OFFSET;
1011 rx->page_pool = page_pool_create(&pp_params);
1012 if (IS_ERR(rx->page_pool)) {
1013 retval = PTR_ERR(rx->page_pool);
1014 rx->page_pool = NULL;
1015 goto failed;
1016 }
1017
1018 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1019 entry = &rx->entry[i];
1020 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK];
1021 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
1022 }
1023
1024 return 0;
1025
1026failed:
1027 tsnep_rx_ring_cleanup(rx);
1028 return retval;
1029}
1030
1031static void tsnep_rx_init(struct tsnep_rx *rx)
1032{
1033 dma_addr_t dma;
1034
1035 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
1036 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);
1037 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);
1038 rx->write = 0;
1039 rx->read = 0;
1040 rx->owner_counter = 1;
1041 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1042}
1043
1044static void tsnep_rx_enable(struct tsnep_rx *rx)
1045{
1046 /* descriptor properties shall be valid before hardware is notified */
1047 dma_wmb();
1048
1049 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
1050}
1051
1052static void tsnep_rx_disable(struct tsnep_rx *rx)
1053{
1054 u32 val;
1055
1056 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);
1057 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,
1058 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000,
1059 1000000);
1060}
1061
1062static int tsnep_rx_desc_available(struct tsnep_rx *rx)
1063{
1064 if (rx->read <= rx->write)
1065 return TSNEP_RING_SIZE - rx->write + rx->read - 1;
1066 else
1067 return rx->read - rx->write - 1;
1068}
1069
1070static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx)
1071{
1072 struct page **page;
1073
1074 /* last entry of page_buffer is always zero, because ring cannot be
1075 * filled completely
1076 */
1077 page = rx->page_buffer;
1078 while (*page) {
1079 page_pool_put_full_page(rx->page_pool, *page, false);
1080 *page = NULL;
1081 page++;
1082 }
1083}
1084
1085static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx)
1086{
1087 int i;
1088
1089 /* alloc for all ring entries except the last one, because ring cannot
1090 * be filled completely
1091 */
1092 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) {
1093 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool);
1094 if (!rx->page_buffer[i]) {
1095 tsnep_rx_free_page_buffer(rx);
1096
1097 return -ENOMEM;
1098 }
1099 }
1100
1101 return 0;
1102}
1103
1104static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1105 struct page *page)
1106{
1107 entry->page = page;
1108 entry->len = TSNEP_MAX_RX_BUF_SIZE;
1109 entry->dma = page_pool_get_dma_addr(entry->page);
1110 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET);
1111}
1112
1113static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index)
1114{
1115 struct tsnep_rx_entry *entry = &rx->entry[index];
1116 struct page *page;
1117
1118 page = page_pool_dev_alloc_pages(rx->page_pool);
1119 if (unlikely(!page))
1120 return -ENOMEM;
1121 tsnep_rx_set_page(rx, entry, page);
1122
1123 return 0;
1124}
1125
1126static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index)
1127{
1128 struct tsnep_rx_entry *entry = &rx->entry[index];
1129 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1130
1131 tsnep_rx_set_page(rx, entry, read->page);
1132 read->page = NULL;
1133}
1134
1135static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
1136{
1137 struct tsnep_rx_entry *entry = &rx->entry[index];
1138
1139 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */
1140 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK;
1141 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
1142 if (index == rx->increment_owner_counter) {
1143 rx->owner_counter++;
1144 if (rx->owner_counter == 4)
1145 rx->owner_counter = 1;
1146 rx->increment_owner_counter--;
1147 if (rx->increment_owner_counter < 0)
1148 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1149 }
1150 entry->properties |=
1151 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
1152 TSNEP_DESC_OWNER_COUNTER_MASK;
1153
1154 /* descriptor properties shall be written last, because valid data is
1155 * signaled there
1156 */
1157 dma_wmb();
1158
1159 entry->desc->properties = __cpu_to_le32(entry->properties);
1160}
1161
1162static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse)
1163{
1164 bool alloc_failed = false;
1165 int i, index;
1166
1167 for (i = 0; i < count && !alloc_failed; i++) {
1168 index = (rx->write + i) & TSNEP_RING_MASK;
1169
1170 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) {
1171 rx->alloc_failed++;
1172 alloc_failed = true;
1173
1174 /* reuse only if no other allocation was successful */
1175 if (i == 0 && reuse)
1176 tsnep_rx_reuse_buffer(rx, index);
1177 else
1178 break;
1179 }
1180
1181 tsnep_rx_activate(rx, index);
1182 }
1183
1184 if (i)
1185 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1186
1187 return i;
1188}
1189
1190static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
1191{
1192 int desc_refilled;
1193
1194 desc_refilled = tsnep_rx_alloc(rx, count, reuse);
1195 if (desc_refilled)
1196 tsnep_rx_enable(rx);
1197
1198 return desc_refilled;
1199}
1200
1201static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1202 struct xdp_buff *xdp)
1203{
1204 entry->xdp = xdp;
1205 entry->len = TSNEP_XSK_RX_BUF_SIZE;
1206 entry->dma = xsk_buff_xdp_get_dma(entry->xdp);
1207 entry->desc->rx = __cpu_to_le64(entry->dma);
1208}
1209
1210static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index)
1211{
1212 struct tsnep_rx_entry *entry = &rx->entry[index];
1213 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1214
1215 tsnep_rx_set_xdp(rx, entry, read->xdp);
1216 read->xdp = NULL;
1217}
1218
1219static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse)
1220{
1221 u32 allocated;
1222 int i;
1223
1224 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count);
1225 for (i = 0; i < allocated; i++) {
1226 int index = (rx->write + i) & TSNEP_RING_MASK;
1227 struct tsnep_rx_entry *entry = &rx->entry[index];
1228
1229 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]);
1230 tsnep_rx_activate(rx, index);
1231 }
1232 if (i == 0) {
1233 rx->alloc_failed++;
1234
1235 if (reuse) {
1236 tsnep_rx_reuse_buffer_zc(rx, rx->write);
1237 tsnep_rx_activate(rx, rx->write);
1238 }
1239 }
1240
1241 if (i)
1242 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1243
1244 return i;
1245}
1246
1247static void tsnep_rx_free_zc(struct tsnep_rx *rx)
1248{
1249 int i;
1250
1251 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1252 struct tsnep_rx_entry *entry = &rx->entry[i];
1253
1254 if (entry->xdp)
1255 xsk_buff_free(entry->xdp);
1256 entry->xdp = NULL;
1257 }
1258}
1259
1260static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse)
1261{
1262 int desc_refilled;
1263
1264 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse);
1265 if (desc_refilled)
1266 tsnep_rx_enable(rx);
1267
1268 return desc_refilled;
1269}
1270
1271static void tsnep_xsk_rx_need_wakeup(struct tsnep_rx *rx, int desc_available)
1272{
1273 if (desc_available)
1274 xsk_set_rx_need_wakeup(rx->xsk_pool);
1275 else
1276 xsk_clear_rx_need_wakeup(rx->xsk_pool);
1277}
1278
1279static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,
1280 struct xdp_buff *xdp, int *status,
1281 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1282{
1283 unsigned int length;
1284 unsigned int sync;
1285 u32 act;
1286
1287 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;
1288
1289 act = bpf_prog_run_xdp(prog, xdp);
1290 switch (act) {
1291 case XDP_PASS:
1292 return false;
1293 case XDP_TX:
1294 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, false))
1295 goto out_failure;
1296 *status |= TSNEP_XDP_TX;
1297 return true;
1298 case XDP_REDIRECT:
1299 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1300 goto out_failure;
1301 *status |= TSNEP_XDP_REDIRECT;
1302 return true;
1303 default:
1304 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1305 fallthrough;
1306 case XDP_ABORTED:
1307out_failure:
1308 trace_xdp_exception(rx->adapter->netdev, prog, act);
1309 fallthrough;
1310 case XDP_DROP:
1311 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU
1312 * touch
1313 */
1314 sync = xdp->data_end - xdp->data_hard_start -
1315 XDP_PACKET_HEADROOM;
1316 sync = max(sync, length);
1317 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data),
1318 sync, true);
1319 return true;
1320 }
1321}
1322
1323static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog,
1324 struct xdp_buff *xdp, int *status,
1325 struct netdev_queue *tx_nq,
1326 struct tsnep_tx *tx)
1327{
1328 u32 act;
1329
1330 act = bpf_prog_run_xdp(prog, xdp);
1331
1332 /* XDP_REDIRECT is the main action for zero-copy */
1333 if (likely(act == XDP_REDIRECT)) {
1334 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1335 goto out_failure;
1336 *status |= TSNEP_XDP_REDIRECT;
1337 return true;
1338 }
1339
1340 switch (act) {
1341 case XDP_PASS:
1342 return false;
1343 case XDP_TX:
1344 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, true))
1345 goto out_failure;
1346 *status |= TSNEP_XDP_TX;
1347 return true;
1348 default:
1349 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1350 fallthrough;
1351 case XDP_ABORTED:
1352out_failure:
1353 trace_xdp_exception(rx->adapter->netdev, prog, act);
1354 fallthrough;
1355 case XDP_DROP:
1356 xsk_buff_free(xdp);
1357 return true;
1358 }
1359}
1360
1361static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status,
1362 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1363{
1364 if (status & TSNEP_XDP_TX) {
1365 __netif_tx_lock(tx_nq, smp_processor_id());
1366 tsnep_xdp_xmit_flush(tx);
1367 __netif_tx_unlock(tx_nq);
1368 }
1369
1370 if (status & TSNEP_XDP_REDIRECT)
1371 xdp_do_flush();
1372}
1373
1374static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,
1375 int length)
1376{
1377 struct sk_buff *skb;
1378
1379 skb = napi_build_skb(page_address(page), PAGE_SIZE);
1380 if (unlikely(!skb))
1381 return NULL;
1382
1383 /* update pointers within the skb to store the data */
1384 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE);
1385 __skb_put(skb, length - ETH_FCS_LEN);
1386
1387 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) {
1388 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
1389 struct tsnep_rx_inline *rx_inline =
1390 (struct tsnep_rx_inline *)(page_address(page) +
1391 TSNEP_RX_OFFSET);
1392
1393 skb_shinfo(skb)->tx_flags |=
1394 SKBTX_HW_TSTAMP_NETDEV;
1395 memset(hwtstamps, 0, sizeof(*hwtstamps));
1396 hwtstamps->netdev_data = rx_inline;
1397 }
1398
1399 skb_record_rx_queue(skb, rx->queue_index);
1400 skb->protocol = eth_type_trans(skb, rx->adapter->netdev);
1401
1402 return skb;
1403}
1404
1405static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi,
1406 struct page *page, int length)
1407{
1408 struct sk_buff *skb;
1409
1410 skb = tsnep_build_skb(rx, page, length);
1411 if (skb) {
1412 skb_mark_for_recycle(skb);
1413
1414 rx->packets++;
1415 rx->bytes += length;
1416 if (skb->pkt_type == PACKET_MULTICAST)
1417 rx->multicast++;
1418
1419 napi_gro_receive(napi, skb);
1420 } else {
1421 page_pool_recycle_direct(rx->page_pool, page);
1422
1423 rx->dropped++;
1424 }
1425}
1426
1427static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
1428 int budget)
1429{
1430 struct device *dmadev = rx->adapter->dmadev;
1431 enum dma_data_direction dma_dir;
1432 struct tsnep_rx_entry *entry;
1433 struct netdev_queue *tx_nq;
1434 struct bpf_prog *prog;
1435 struct xdp_buff xdp;
1436 struct tsnep_tx *tx;
1437 int desc_available;
1438 int xdp_status = 0;
1439 int done = 0;
1440 int length;
1441
1442 desc_available = tsnep_rx_desc_available(rx);
1443 dma_dir = page_pool_get_dma_dir(rx->page_pool);
1444 prog = READ_ONCE(rx->adapter->xdp_prog);
1445 if (prog) {
1446 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1447 rx->tx_queue_index);
1448 tx = &rx->adapter->tx[rx->tx_queue_index];
1449
1450 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq);
1451 }
1452
1453 while (likely(done < budget) && (rx->read != rx->write)) {
1454 entry = &rx->entry[rx->read];
1455 if ((__le32_to_cpu(entry->desc_wb->properties) &
1456 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1457 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1458 break;
1459 done++;
1460
1461 if (desc_available >= TSNEP_RING_RX_REFILL) {
1462 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1463
1464 desc_available -= tsnep_rx_refill(rx, desc_available,
1465 reuse);
1466 if (!entry->page) {
1467 /* buffer has been reused for refill to prevent
1468 * empty RX ring, thus buffer cannot be used for
1469 * RX processing
1470 */
1471 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1472 desc_available++;
1473
1474 rx->dropped++;
1475
1476 continue;
1477 }
1478 }
1479
1480 /* descriptor properties shall be read first, because valid data
1481 * is signaled there
1482 */
1483 dma_rmb();
1484
1485 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET);
1486 length = __le32_to_cpu(entry->desc_wb->properties) &
1487 TSNEP_DESC_LENGTH_MASK;
1488 dma_sync_single_range_for_cpu(dmadev, entry->dma,
1489 TSNEP_RX_OFFSET, length, dma_dir);
1490
1491 /* RX metadata with timestamps is in front of actual data,
1492 * subtract metadata size to get length of actual data and
1493 * consider metadata size as offset of actual data during RX
1494 * processing
1495 */
1496 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1497
1498 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1499 desc_available++;
1500
1501 if (prog) {
1502 bool consume;
1503
1504 xdp_prepare_buff(&xdp, page_address(entry->page),
1505 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE,
1506 length - ETH_FCS_LEN, false);
1507
1508 consume = tsnep_xdp_run_prog(rx, prog, &xdp,
1509 &xdp_status, tx_nq, tx);
1510 if (consume) {
1511 rx->packets++;
1512 rx->bytes += length;
1513
1514 entry->page = NULL;
1515
1516 continue;
1517 }
1518 }
1519
1520 tsnep_rx_page(rx, napi, entry->page, length);
1521 entry->page = NULL;
1522 }
1523
1524 if (xdp_status)
1525 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1526
1527 if (desc_available)
1528 tsnep_rx_refill(rx, desc_available, false);
1529
1530 return done;
1531}
1532
1533static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi,
1534 int budget)
1535{
1536 struct tsnep_rx_entry *entry;
1537 struct netdev_queue *tx_nq;
1538 struct bpf_prog *prog;
1539 struct tsnep_tx *tx;
1540 int desc_available;
1541 int xdp_status = 0;
1542 struct page *page;
1543 int done = 0;
1544 int length;
1545
1546 desc_available = tsnep_rx_desc_available(rx);
1547 prog = READ_ONCE(rx->adapter->xdp_prog);
1548 if (prog) {
1549 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1550 rx->tx_queue_index);
1551 tx = &rx->adapter->tx[rx->tx_queue_index];
1552 }
1553
1554 while (likely(done < budget) && (rx->read != rx->write)) {
1555 entry = &rx->entry[rx->read];
1556 if ((__le32_to_cpu(entry->desc_wb->properties) &
1557 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1558 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1559 break;
1560 done++;
1561
1562 if (desc_available >= TSNEP_RING_RX_REFILL) {
1563 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1564
1565 desc_available -= tsnep_rx_refill_zc(rx, desc_available,
1566 reuse);
1567 if (!entry->xdp) {
1568 /* buffer has been reused for refill to prevent
1569 * empty RX ring, thus buffer cannot be used for
1570 * RX processing
1571 */
1572 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1573 desc_available++;
1574
1575 rx->dropped++;
1576
1577 continue;
1578 }
1579 }
1580
1581 /* descriptor properties shall be read first, because valid data
1582 * is signaled there
1583 */
1584 dma_rmb();
1585
1586 prefetch(entry->xdp->data);
1587 length = __le32_to_cpu(entry->desc_wb->properties) &
1588 TSNEP_DESC_LENGTH_MASK;
1589 xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN);
1590 xsk_buff_dma_sync_for_cpu(entry->xdp);
1591
1592 /* RX metadata with timestamps is in front of actual data,
1593 * subtract metadata size to get length of actual data and
1594 * consider metadata size as offset of actual data during RX
1595 * processing
1596 */
1597 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1598
1599 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1600 desc_available++;
1601
1602 if (prog) {
1603 bool consume;
1604
1605 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE;
1606 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE;
1607
1608 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp,
1609 &xdp_status, tx_nq, tx);
1610 if (consume) {
1611 rx->packets++;
1612 rx->bytes += length;
1613
1614 entry->xdp = NULL;
1615
1616 continue;
1617 }
1618 }
1619
1620 page = page_pool_dev_alloc_pages(rx->page_pool);
1621 if (page) {
1622 memcpy(page_address(page) + TSNEP_RX_OFFSET,
1623 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE,
1624 length + TSNEP_RX_INLINE_METADATA_SIZE);
1625 tsnep_rx_page(rx, napi, page, length);
1626 } else {
1627 rx->dropped++;
1628 }
1629 xsk_buff_free(entry->xdp);
1630 entry->xdp = NULL;
1631 }
1632
1633 if (xdp_status)
1634 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1635
1636 if (desc_available)
1637 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false);
1638
1639 if (xsk_uses_need_wakeup(rx->xsk_pool)) {
1640 tsnep_xsk_rx_need_wakeup(rx, desc_available);
1641
1642 return done;
1643 }
1644
1645 return desc_available ? budget : done;
1646}
1647
1648static bool tsnep_rx_pending(struct tsnep_rx *rx)
1649{
1650 struct tsnep_rx_entry *entry;
1651
1652 if (rx->read != rx->write) {
1653 entry = &rx->entry[rx->read];
1654 if ((__le32_to_cpu(entry->desc_wb->properties) &
1655 TSNEP_DESC_OWNER_COUNTER_MASK) ==
1656 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1657 return true;
1658 }
1659
1660 return false;
1661}
1662
1663static int tsnep_rx_open(struct tsnep_rx *rx)
1664{
1665 int desc_available;
1666 int retval;
1667
1668 retval = tsnep_rx_ring_create(rx);
1669 if (retval)
1670 return retval;
1671
1672 tsnep_rx_init(rx);
1673
1674 desc_available = tsnep_rx_desc_available(rx);
1675 if (rx->xsk_pool)
1676 retval = tsnep_rx_alloc_zc(rx, desc_available, false);
1677 else
1678 retval = tsnep_rx_alloc(rx, desc_available, false);
1679 if (retval != desc_available) {
1680 retval = -ENOMEM;
1681
1682 goto alloc_failed;
1683 }
1684
1685 /* prealloc pages to prevent allocation failures when XSK pool is
1686 * disabled at runtime
1687 */
1688 if (rx->xsk_pool) {
1689 retval = tsnep_rx_alloc_page_buffer(rx);
1690 if (retval)
1691 goto alloc_failed;
1692 }
1693
1694 return 0;
1695
1696alloc_failed:
1697 tsnep_rx_ring_cleanup(rx);
1698 return retval;
1699}
1700
1701static void tsnep_rx_close(struct tsnep_rx *rx)
1702{
1703 if (rx->xsk_pool)
1704 tsnep_rx_free_page_buffer(rx);
1705
1706 tsnep_rx_ring_cleanup(rx);
1707}
1708
1709static void tsnep_rx_reopen(struct tsnep_rx *rx)
1710{
1711 struct page **page = rx->page_buffer;
1712 int i;
1713
1714 tsnep_rx_init(rx);
1715
1716 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1717 struct tsnep_rx_entry *entry = &rx->entry[i];
1718
1719 /* defined initial values for properties are required for
1720 * correct owner counter checking
1721 */
1722 entry->desc->properties = 0;
1723 entry->desc_wb->properties = 0;
1724
1725 /* prevent allocation failures by reusing kept pages */
1726 if (*page) {
1727 tsnep_rx_set_page(rx, entry, *page);
1728 tsnep_rx_activate(rx, rx->write);
1729 rx->write++;
1730
1731 *page = NULL;
1732 page++;
1733 }
1734 }
1735}
1736
1737static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx)
1738{
1739 struct page **page = rx->page_buffer;
1740 u32 allocated;
1741 int i;
1742
1743 tsnep_rx_init(rx);
1744
1745 /* alloc all ring entries except the last one, because ring cannot be
1746 * filled completely, as many buffers as possible is enough as wakeup is
1747 * done if new buffers are available
1748 */
1749 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch,
1750 TSNEP_RING_SIZE - 1);
1751
1752 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1753 struct tsnep_rx_entry *entry = &rx->entry[i];
1754
1755 /* keep pages to prevent allocation failures when xsk is
1756 * disabled
1757 */
1758 if (entry->page) {
1759 *page = entry->page;
1760 entry->page = NULL;
1761
1762 page++;
1763 }
1764
1765 /* defined initial values for properties are required for
1766 * correct owner counter checking
1767 */
1768 entry->desc->properties = 0;
1769 entry->desc_wb->properties = 0;
1770
1771 if (allocated) {
1772 tsnep_rx_set_xdp(rx, entry,
1773 rx->xdp_batch[allocated - 1]);
1774 tsnep_rx_activate(rx, rx->write);
1775 rx->write++;
1776
1777 allocated--;
1778 }
1779 }
1780
1781 /* set need wakeup flag immediately if ring is not filled completely,
1782 * first polling would be too late as need wakeup signalisation would
1783 * be delayed for an indefinite time
1784 */
1785 if (xsk_uses_need_wakeup(rx->xsk_pool))
1786 tsnep_xsk_rx_need_wakeup(rx, tsnep_rx_desc_available(rx));
1787}
1788
1789static bool tsnep_pending(struct tsnep_queue *queue)
1790{
1791 if (queue->tx && tsnep_tx_pending(queue->tx))
1792 return true;
1793
1794 if (queue->rx && tsnep_rx_pending(queue->rx))
1795 return true;
1796
1797 return false;
1798}
1799
1800static int tsnep_poll(struct napi_struct *napi, int budget)
1801{
1802 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue,
1803 napi);
1804 bool complete = true;
1805 int done = 0;
1806
1807 if (queue->tx)
1808 complete = tsnep_tx_poll(queue->tx, budget);
1809
1810 /* handle case where we are called by netpoll with a budget of 0 */
1811 if (unlikely(budget <= 0))
1812 return budget;
1813
1814 if (queue->rx) {
1815 done = queue->rx->xsk_pool ?
1816 tsnep_rx_poll_zc(queue->rx, napi, budget) :
1817 tsnep_rx_poll(queue->rx, napi, budget);
1818 if (done >= budget)
1819 complete = false;
1820 }
1821
1822 /* if all work not completed, return budget and keep polling */
1823 if (!complete)
1824 return budget;
1825
1826 if (likely(napi_complete_done(napi, done))) {
1827 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1828
1829 /* reschedule if work is already pending, prevent rotten packets
1830 * which are transmitted or received after polling but before
1831 * interrupt enable
1832 */
1833 if (tsnep_pending(queue)) {
1834 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1835 napi_schedule(napi);
1836 }
1837 }
1838
1839 return min(done, budget - 1);
1840}
1841
1842static int tsnep_request_irq(struct tsnep_queue *queue, bool first)
1843{
1844 const char *name = netdev_name(queue->adapter->netdev);
1845 irq_handler_t handler;
1846 void *dev;
1847 int retval;
1848
1849 if (first) {
1850 sprintf(queue->name, "%s-mac", name);
1851 handler = tsnep_irq;
1852 dev = queue->adapter;
1853 } else {
1854 if (queue->tx && queue->rx)
1855 snprintf(queue->name, sizeof(queue->name), "%s-txrx-%d",
1856 name, queue->rx->queue_index);
1857 else if (queue->tx)
1858 snprintf(queue->name, sizeof(queue->name), "%s-tx-%d",
1859 name, queue->tx->queue_index);
1860 else
1861 snprintf(queue->name, sizeof(queue->name), "%s-rx-%d",
1862 name, queue->rx->queue_index);
1863 handler = tsnep_irq_txrx;
1864 dev = queue;
1865 }
1866
1867 retval = request_irq(queue->irq, handler, 0, queue->name, dev);
1868 if (retval) {
1869 /* if name is empty, then interrupt won't be freed */
1870 memset(queue->name, 0, sizeof(queue->name));
1871 }
1872
1873 return retval;
1874}
1875
1876static void tsnep_free_irq(struct tsnep_queue *queue, bool first)
1877{
1878 void *dev;
1879
1880 if (!strlen(queue->name))
1881 return;
1882
1883 if (first)
1884 dev = queue->adapter;
1885 else
1886 dev = queue;
1887
1888 free_irq(queue->irq, dev);
1889 memset(queue->name, 0, sizeof(queue->name));
1890}
1891
1892static void tsnep_queue_close(struct tsnep_queue *queue, bool first)
1893{
1894 struct tsnep_rx *rx = queue->rx;
1895
1896 tsnep_free_irq(queue, first);
1897
1898 if (rx) {
1899 if (xdp_rxq_info_is_reg(&rx->xdp_rxq))
1900 xdp_rxq_info_unreg(&rx->xdp_rxq);
1901 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc))
1902 xdp_rxq_info_unreg(&rx->xdp_rxq_zc);
1903 }
1904
1905 netif_napi_del(&queue->napi);
1906}
1907
1908static int tsnep_queue_open(struct tsnep_adapter *adapter,
1909 struct tsnep_queue *queue, bool first)
1910{
1911 struct tsnep_rx *rx = queue->rx;
1912 struct tsnep_tx *tx = queue->tx;
1913 int retval;
1914
1915 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll);
1916
1917 if (rx) {
1918 /* choose TX queue for XDP_TX */
1919 if (tx)
1920 rx->tx_queue_index = tx->queue_index;
1921 else if (rx->queue_index < adapter->num_tx_queues)
1922 rx->tx_queue_index = rx->queue_index;
1923 else
1924 rx->tx_queue_index = 0;
1925
1926 /* prepare both memory models to eliminate possible registration
1927 * errors when memory model is switched between page pool and
1928 * XSK pool during runtime
1929 */
1930 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev,
1931 rx->queue_index, queue->napi.napi_id);
1932 if (retval)
1933 goto failed;
1934 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq,
1935 MEM_TYPE_PAGE_POOL,
1936 rx->page_pool);
1937 if (retval)
1938 goto failed;
1939 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev,
1940 rx->queue_index, queue->napi.napi_id);
1941 if (retval)
1942 goto failed;
1943 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc,
1944 MEM_TYPE_XSK_BUFF_POOL,
1945 NULL);
1946 if (retval)
1947 goto failed;
1948 if (rx->xsk_pool)
1949 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc);
1950 }
1951
1952 retval = tsnep_request_irq(queue, first);
1953 if (retval) {
1954 netif_err(adapter, drv, adapter->netdev,
1955 "can't get assigned irq %d.\n", queue->irq);
1956 goto failed;
1957 }
1958
1959 return 0;
1960
1961failed:
1962 tsnep_queue_close(queue, first);
1963
1964 return retval;
1965}
1966
1967static void tsnep_queue_enable(struct tsnep_queue *queue)
1968{
1969 napi_enable(&queue->napi);
1970 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1971
1972 if (queue->tx)
1973 tsnep_tx_enable(queue->tx);
1974
1975 if (queue->rx)
1976 tsnep_rx_enable(queue->rx);
1977}
1978
1979static void tsnep_queue_disable(struct tsnep_queue *queue)
1980{
1981 if (queue->tx)
1982 tsnep_tx_disable(queue->tx, &queue->napi);
1983
1984 napi_disable(&queue->napi);
1985 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1986
1987 /* disable RX after NAPI polling has been disabled, because RX can be
1988 * enabled during NAPI polling
1989 */
1990 if (queue->rx)
1991 tsnep_rx_disable(queue->rx);
1992}
1993
1994static int tsnep_netdev_open(struct net_device *netdev)
1995{
1996 struct tsnep_adapter *adapter = netdev_priv(netdev);
1997 int i, retval;
1998
1999 for (i = 0; i < adapter->num_queues; i++) {
2000 if (adapter->queue[i].tx) {
2001 retval = tsnep_tx_open(adapter->queue[i].tx);
2002 if (retval)
2003 goto failed;
2004 }
2005 if (adapter->queue[i].rx) {
2006 retval = tsnep_rx_open(adapter->queue[i].rx);
2007 if (retval)
2008 goto failed;
2009 }
2010
2011 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0);
2012 if (retval)
2013 goto failed;
2014 }
2015
2016 retval = netif_set_real_num_tx_queues(adapter->netdev,
2017 adapter->num_tx_queues);
2018 if (retval)
2019 goto failed;
2020 retval = netif_set_real_num_rx_queues(adapter->netdev,
2021 adapter->num_rx_queues);
2022 if (retval)
2023 goto failed;
2024
2025 tsnep_enable_irq(adapter, ECM_INT_LINK);
2026 retval = tsnep_phy_open(adapter);
2027 if (retval)
2028 goto phy_failed;
2029
2030 for (i = 0; i < adapter->num_queues; i++)
2031 tsnep_queue_enable(&adapter->queue[i]);
2032
2033 return 0;
2034
2035phy_failed:
2036 tsnep_disable_irq(adapter, ECM_INT_LINK);
2037failed:
2038 for (i = 0; i < adapter->num_queues; i++) {
2039 tsnep_queue_close(&adapter->queue[i], i == 0);
2040
2041 if (adapter->queue[i].rx)
2042 tsnep_rx_close(adapter->queue[i].rx);
2043 if (adapter->queue[i].tx)
2044 tsnep_tx_close(adapter->queue[i].tx);
2045 }
2046 return retval;
2047}
2048
2049static int tsnep_netdev_close(struct net_device *netdev)
2050{
2051 struct tsnep_adapter *adapter = netdev_priv(netdev);
2052 int i;
2053
2054 tsnep_disable_irq(adapter, ECM_INT_LINK);
2055 tsnep_phy_close(adapter);
2056
2057 for (i = 0; i < adapter->num_queues; i++) {
2058 tsnep_queue_disable(&adapter->queue[i]);
2059
2060 tsnep_queue_close(&adapter->queue[i], i == 0);
2061
2062 if (adapter->queue[i].rx)
2063 tsnep_rx_close(adapter->queue[i].rx);
2064 if (adapter->queue[i].tx)
2065 tsnep_tx_close(adapter->queue[i].tx);
2066 }
2067
2068 return 0;
2069}
2070
2071int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool)
2072{
2073 bool running = netif_running(queue->adapter->netdev);
2074 u32 frame_size;
2075
2076 frame_size = xsk_pool_get_rx_frame_size(pool);
2077 if (frame_size < TSNEP_XSK_RX_BUF_SIZE)
2078 return -EOPNOTSUPP;
2079
2080 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE,
2081 sizeof(*queue->rx->page_buffer),
2082 GFP_KERNEL);
2083 if (!queue->rx->page_buffer)
2084 return -ENOMEM;
2085 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE,
2086 sizeof(*queue->rx->xdp_batch),
2087 GFP_KERNEL);
2088 if (!queue->rx->xdp_batch) {
2089 kfree(queue->rx->page_buffer);
2090 queue->rx->page_buffer = NULL;
2091
2092 return -ENOMEM;
2093 }
2094
2095 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc);
2096
2097 if (running)
2098 tsnep_queue_disable(queue);
2099
2100 queue->tx->xsk_pool = pool;
2101 queue->rx->xsk_pool = pool;
2102
2103 if (running) {
2104 tsnep_rx_reopen_xsk(queue->rx);
2105 tsnep_queue_enable(queue);
2106 }
2107
2108 return 0;
2109}
2110
2111void tsnep_disable_xsk(struct tsnep_queue *queue)
2112{
2113 bool running = netif_running(queue->adapter->netdev);
2114
2115 if (running)
2116 tsnep_queue_disable(queue);
2117
2118 tsnep_rx_free_zc(queue->rx);
2119
2120 queue->rx->xsk_pool = NULL;
2121 queue->tx->xsk_pool = NULL;
2122
2123 if (running) {
2124 tsnep_rx_reopen(queue->rx);
2125 tsnep_queue_enable(queue);
2126 }
2127
2128 kfree(queue->rx->xdp_batch);
2129 queue->rx->xdp_batch = NULL;
2130 kfree(queue->rx->page_buffer);
2131 queue->rx->page_buffer = NULL;
2132}
2133
2134static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb,
2135 struct net_device *netdev)
2136{
2137 struct tsnep_adapter *adapter = netdev_priv(netdev);
2138 u16 queue_mapping = skb_get_queue_mapping(skb);
2139
2140 if (queue_mapping >= adapter->num_tx_queues)
2141 queue_mapping = 0;
2142
2143 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]);
2144}
2145
2146static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr,
2147 int cmd)
2148{
2149 if (!netif_running(netdev))
2150 return -EINVAL;
2151 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP)
2152 return tsnep_ptp_ioctl(netdev, ifr, cmd);
2153 return phy_mii_ioctl(netdev->phydev, ifr, cmd);
2154}
2155
2156static void tsnep_netdev_set_multicast(struct net_device *netdev)
2157{
2158 struct tsnep_adapter *adapter = netdev_priv(netdev);
2159
2160 u16 rx_filter = 0;
2161
2162 /* configured MAC address and broadcasts are never filtered */
2163 if (netdev->flags & IFF_PROMISC) {
2164 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2165 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS;
2166 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) {
2167 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2168 }
2169 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER);
2170}
2171
2172static void tsnep_netdev_get_stats64(struct net_device *netdev,
2173 struct rtnl_link_stats64 *stats)
2174{
2175 struct tsnep_adapter *adapter = netdev_priv(netdev);
2176 u32 reg;
2177 u32 val;
2178 int i;
2179
2180 for (i = 0; i < adapter->num_tx_queues; i++) {
2181 stats->tx_packets += adapter->tx[i].packets;
2182 stats->tx_bytes += adapter->tx[i].bytes;
2183 stats->tx_dropped += adapter->tx[i].dropped;
2184 }
2185 for (i = 0; i < adapter->num_rx_queues; i++) {
2186 stats->rx_packets += adapter->rx[i].packets;
2187 stats->rx_bytes += adapter->rx[i].bytes;
2188 stats->rx_dropped += adapter->rx[i].dropped;
2189 stats->multicast += adapter->rx[i].multicast;
2190
2191 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) +
2192 TSNEP_RX_STATISTIC);
2193 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >>
2194 TSNEP_RX_STATISTIC_NO_DESC_SHIFT;
2195 stats->rx_dropped += val;
2196 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >>
2197 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT;
2198 stats->rx_dropped += val;
2199 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >>
2200 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT;
2201 stats->rx_errors += val;
2202 stats->rx_fifo_errors += val;
2203 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >>
2204 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT;
2205 stats->rx_errors += val;
2206 stats->rx_frame_errors += val;
2207 }
2208
2209 reg = ioread32(adapter->addr + ECM_STAT);
2210 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT;
2211 stats->rx_errors += val;
2212 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT;
2213 stats->rx_errors += val;
2214 stats->rx_crc_errors += val;
2215 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT;
2216 stats->rx_errors += val;
2217}
2218
2219static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr)
2220{
2221 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2222 iowrite16(*(u16 *)(addr + sizeof(u32)),
2223 adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2224
2225 ether_addr_copy(adapter->mac_address, addr);
2226 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n",
2227 addr);
2228}
2229
2230static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr)
2231{
2232 struct tsnep_adapter *adapter = netdev_priv(netdev);
2233 struct sockaddr *sock_addr = addr;
2234 int retval;
2235
2236 retval = eth_prepare_mac_addr_change(netdev, sock_addr);
2237 if (retval)
2238 return retval;
2239 eth_hw_addr_set(netdev, sock_addr->sa_data);
2240 tsnep_mac_set_address(adapter, sock_addr->sa_data);
2241
2242 return 0;
2243}
2244
2245static int tsnep_netdev_set_features(struct net_device *netdev,
2246 netdev_features_t features)
2247{
2248 struct tsnep_adapter *adapter = netdev_priv(netdev);
2249 netdev_features_t changed = netdev->features ^ features;
2250 bool enable;
2251 int retval = 0;
2252
2253 if (changed & NETIF_F_LOOPBACK) {
2254 enable = !!(features & NETIF_F_LOOPBACK);
2255 retval = tsnep_phy_loopback(adapter, enable);
2256 }
2257
2258 return retval;
2259}
2260
2261static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev,
2262 const struct skb_shared_hwtstamps *hwtstamps,
2263 bool cycles)
2264{
2265 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data;
2266 u64 timestamp;
2267
2268 if (cycles)
2269 timestamp = __le64_to_cpu(rx_inline->counter);
2270 else
2271 timestamp = __le64_to_cpu(rx_inline->timestamp);
2272
2273 return ns_to_ktime(timestamp);
2274}
2275
2276static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf)
2277{
2278 struct tsnep_adapter *adapter = netdev_priv(dev);
2279
2280 switch (bpf->command) {
2281 case XDP_SETUP_PROG:
2282 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack);
2283 case XDP_SETUP_XSK_POOL:
2284 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool,
2285 bpf->xsk.queue_id);
2286 default:
2287 return -EOPNOTSUPP;
2288 }
2289}
2290
2291static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu)
2292{
2293 if (cpu >= TSNEP_MAX_QUEUES)
2294 cpu &= TSNEP_MAX_QUEUES - 1;
2295
2296 while (cpu >= adapter->num_tx_queues)
2297 cpu -= adapter->num_tx_queues;
2298
2299 return &adapter->tx[cpu];
2300}
2301
2302static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n,
2303 struct xdp_frame **xdp, u32 flags)
2304{
2305 struct tsnep_adapter *adapter = netdev_priv(dev);
2306 u32 cpu = smp_processor_id();
2307 struct netdev_queue *nq;
2308 struct tsnep_tx *tx;
2309 int nxmit;
2310 bool xmit;
2311
2312 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2313 return -EINVAL;
2314
2315 tx = tsnep_xdp_get_tx(adapter, cpu);
2316 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index);
2317
2318 __netif_tx_lock(nq, cpu);
2319
2320 for (nxmit = 0; nxmit < n; nxmit++) {
2321 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx,
2322 TSNEP_TX_TYPE_XDP_NDO);
2323 if (!xmit)
2324 break;
2325
2326 /* avoid transmit queue timeout since we share it with the slow
2327 * path
2328 */
2329 txq_trans_cond_update(nq);
2330 }
2331
2332 if (flags & XDP_XMIT_FLUSH)
2333 tsnep_xdp_xmit_flush(tx);
2334
2335 __netif_tx_unlock(nq);
2336
2337 return nxmit;
2338}
2339
2340static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id,
2341 u32 flags)
2342{
2343 struct tsnep_adapter *adapter = netdev_priv(dev);
2344 struct tsnep_queue *queue;
2345
2346 if (queue_id >= adapter->num_rx_queues ||
2347 queue_id >= adapter->num_tx_queues)
2348 return -EINVAL;
2349
2350 queue = &adapter->queue[queue_id];
2351
2352 if (!napi_if_scheduled_mark_missed(&queue->napi))
2353 napi_schedule(&queue->napi);
2354
2355 return 0;
2356}
2357
2358static const struct net_device_ops tsnep_netdev_ops = {
2359 .ndo_open = tsnep_netdev_open,
2360 .ndo_stop = tsnep_netdev_close,
2361 .ndo_start_xmit = tsnep_netdev_xmit_frame,
2362 .ndo_eth_ioctl = tsnep_netdev_ioctl,
2363 .ndo_set_rx_mode = tsnep_netdev_set_multicast,
2364 .ndo_get_stats64 = tsnep_netdev_get_stats64,
2365 .ndo_set_mac_address = tsnep_netdev_set_mac_address,
2366 .ndo_set_features = tsnep_netdev_set_features,
2367 .ndo_get_tstamp = tsnep_netdev_get_tstamp,
2368 .ndo_setup_tc = tsnep_tc_setup,
2369 .ndo_bpf = tsnep_netdev_bpf,
2370 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit,
2371 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup,
2372};
2373
2374static int tsnep_mac_init(struct tsnep_adapter *adapter)
2375{
2376 int retval;
2377
2378 /* initialize RX filtering, at least configured MAC address and
2379 * broadcast are not filtered
2380 */
2381 iowrite16(0, adapter->addr + TSNEP_RX_FILTER);
2382
2383 /* try to get MAC address in the following order:
2384 * - device tree
2385 * - valid MAC address already set
2386 * - MAC address register if valid
2387 * - random MAC address
2388 */
2389 retval = of_get_mac_address(adapter->pdev->dev.of_node,
2390 adapter->mac_address);
2391 if (retval == -EPROBE_DEFER)
2392 return retval;
2393 if (retval && !is_valid_ether_addr(adapter->mac_address)) {
2394 *(u32 *)adapter->mac_address =
2395 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2396 *(u16 *)(adapter->mac_address + sizeof(u32)) =
2397 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2398 if (!is_valid_ether_addr(adapter->mac_address))
2399 eth_random_addr(adapter->mac_address);
2400 }
2401
2402 tsnep_mac_set_address(adapter, adapter->mac_address);
2403 eth_hw_addr_set(adapter->netdev, adapter->mac_address);
2404
2405 return 0;
2406}
2407
2408static int tsnep_mdio_init(struct tsnep_adapter *adapter)
2409{
2410 struct device_node *np = adapter->pdev->dev.of_node;
2411 int retval;
2412
2413 if (np) {
2414 np = of_get_child_by_name(np, "mdio");
2415 if (!np)
2416 return 0;
2417
2418 adapter->suppress_preamble =
2419 of_property_read_bool(np, "suppress-preamble");
2420 }
2421
2422 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev);
2423 if (!adapter->mdiobus) {
2424 retval = -ENOMEM;
2425
2426 goto out;
2427 }
2428
2429 adapter->mdiobus->priv = (void *)adapter;
2430 adapter->mdiobus->parent = &adapter->pdev->dev;
2431 adapter->mdiobus->read = tsnep_mdiobus_read;
2432 adapter->mdiobus->write = tsnep_mdiobus_write;
2433 adapter->mdiobus->name = TSNEP "-mdiobus";
2434 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s",
2435 adapter->pdev->name);
2436
2437 /* do not scan broadcast address */
2438 adapter->mdiobus->phy_mask = 0x0000001;
2439
2440 retval = of_mdiobus_register(adapter->mdiobus, np);
2441
2442out:
2443 of_node_put(np);
2444
2445 return retval;
2446}
2447
2448static int tsnep_phy_init(struct tsnep_adapter *adapter)
2449{
2450 struct device_node *phy_node;
2451 int retval;
2452
2453 retval = of_get_phy_mode(adapter->pdev->dev.of_node,
2454 &adapter->phy_mode);
2455 if (retval)
2456 adapter->phy_mode = PHY_INTERFACE_MODE_GMII;
2457
2458 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle",
2459 0);
2460 adapter->phydev = of_phy_find_device(phy_node);
2461 of_node_put(phy_node);
2462 if (!adapter->phydev && adapter->mdiobus)
2463 adapter->phydev = phy_find_first(adapter->mdiobus);
2464 if (!adapter->phydev)
2465 return -EIO;
2466
2467 return 0;
2468}
2469
2470static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)
2471{
2472 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0;
2473 char name[8];
2474 int i;
2475 int retval;
2476
2477 /* one TX/RX queue pair for netdev is mandatory */
2478 if (platform_irq_count(adapter->pdev) == 1)
2479 retval = platform_get_irq(adapter->pdev, 0);
2480 else
2481 retval = platform_get_irq_byname(adapter->pdev, "mac");
2482 if (retval < 0)
2483 return retval;
2484 adapter->num_tx_queues = 1;
2485 adapter->num_rx_queues = 1;
2486 adapter->num_queues = 1;
2487 adapter->queue[0].adapter = adapter;
2488 adapter->queue[0].irq = retval;
2489 adapter->queue[0].tx = &adapter->tx[0];
2490 adapter->queue[0].tx->adapter = adapter;
2491 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0);
2492 adapter->queue[0].tx->queue_index = 0;
2493 adapter->queue[0].rx = &adapter->rx[0];
2494 adapter->queue[0].rx->adapter = adapter;
2495 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0);
2496 adapter->queue[0].rx->queue_index = 0;
2497 adapter->queue[0].irq_mask = irq_mask;
2498 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY;
2499 retval = tsnep_set_irq_coalesce(&adapter->queue[0],
2500 TSNEP_COALESCE_USECS_DEFAULT);
2501 if (retval < 0)
2502 return retval;
2503
2504 adapter->netdev->irq = adapter->queue[0].irq;
2505
2506 /* add additional TX/RX queue pairs only if dedicated interrupt is
2507 * available
2508 */
2509 for (i = 1; i < queue_count; i++) {
2510 sprintf(name, "txrx-%d", i);
2511 retval = platform_get_irq_byname_optional(adapter->pdev, name);
2512 if (retval < 0)
2513 break;
2514
2515 adapter->num_tx_queues++;
2516 adapter->num_rx_queues++;
2517 adapter->num_queues++;
2518 adapter->queue[i].adapter = adapter;
2519 adapter->queue[i].irq = retval;
2520 adapter->queue[i].tx = &adapter->tx[i];
2521 adapter->queue[i].tx->adapter = adapter;
2522 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i);
2523 adapter->queue[i].tx->queue_index = i;
2524 adapter->queue[i].rx = &adapter->rx[i];
2525 adapter->queue[i].rx->adapter = adapter;
2526 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i);
2527 adapter->queue[i].rx->queue_index = i;
2528 adapter->queue[i].irq_mask =
2529 irq_mask << (ECM_INT_TXRX_SHIFT * i);
2530 adapter->queue[i].irq_delay_addr =
2531 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i;
2532 retval = tsnep_set_irq_coalesce(&adapter->queue[i],
2533 TSNEP_COALESCE_USECS_DEFAULT);
2534 if (retval < 0)
2535 return retval;
2536 }
2537
2538 return 0;
2539}
2540
2541static int tsnep_probe(struct platform_device *pdev)
2542{
2543 struct tsnep_adapter *adapter;
2544 struct net_device *netdev;
2545 struct resource *io;
2546 u32 type;
2547 int revision;
2548 int version;
2549 int queue_count;
2550 int retval;
2551
2552 netdev = devm_alloc_etherdev_mqs(&pdev->dev,
2553 sizeof(struct tsnep_adapter),
2554 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES);
2555 if (!netdev)
2556 return -ENODEV;
2557 SET_NETDEV_DEV(netdev, &pdev->dev);
2558 adapter = netdev_priv(netdev);
2559 platform_set_drvdata(pdev, adapter);
2560 adapter->pdev = pdev;
2561 adapter->dmadev = &pdev->dev;
2562 adapter->netdev = netdev;
2563 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |
2564 NETIF_MSG_LINK | NETIF_MSG_IFUP |
2565 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;
2566
2567 netdev->min_mtu = ETH_MIN_MTU;
2568 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE;
2569
2570 mutex_init(&adapter->gate_control_lock);
2571 mutex_init(&adapter->rxnfc_lock);
2572 INIT_LIST_HEAD(&adapter->rxnfc_rules);
2573
2574 adapter->addr = devm_platform_get_and_ioremap_resource(pdev, 0, &io);
2575 if (IS_ERR(adapter->addr))
2576 return PTR_ERR(adapter->addr);
2577 netdev->mem_start = io->start;
2578 netdev->mem_end = io->end;
2579
2580 type = ioread32(adapter->addr + ECM_TYPE);
2581 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT;
2582 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT;
2583 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT;
2584 adapter->gate_control = type & ECM_GATE_CONTROL;
2585 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT;
2586
2587 tsnep_disable_irq(adapter, ECM_INT_ALL);
2588
2589 retval = tsnep_queue_init(adapter, queue_count);
2590 if (retval)
2591 return retval;
2592
2593 retval = dma_set_mask_and_coherent(&adapter->pdev->dev,
2594 DMA_BIT_MASK(64));
2595 if (retval) {
2596 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n");
2597 return retval;
2598 }
2599
2600 retval = tsnep_mac_init(adapter);
2601 if (retval)
2602 return retval;
2603
2604 retval = tsnep_mdio_init(adapter);
2605 if (retval)
2606 goto mdio_init_failed;
2607
2608 retval = tsnep_phy_init(adapter);
2609 if (retval)
2610 goto phy_init_failed;
2611
2612 retval = tsnep_ptp_init(adapter);
2613 if (retval)
2614 goto ptp_init_failed;
2615
2616 retval = tsnep_tc_init(adapter);
2617 if (retval)
2618 goto tc_init_failed;
2619
2620 retval = tsnep_rxnfc_init(adapter);
2621 if (retval)
2622 goto rxnfc_init_failed;
2623
2624 netdev->netdev_ops = &tsnep_netdev_ops;
2625 netdev->ethtool_ops = &tsnep_ethtool_ops;
2626 netdev->features = NETIF_F_SG;
2627 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK;
2628
2629 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
2630 NETDEV_XDP_ACT_NDO_XMIT |
2631 NETDEV_XDP_ACT_NDO_XMIT_SG |
2632 NETDEV_XDP_ACT_XSK_ZEROCOPY;
2633
2634 /* carrier off reporting is important to ethtool even BEFORE open */
2635 netif_carrier_off(netdev);
2636
2637 retval = register_netdev(netdev);
2638 if (retval)
2639 goto register_failed;
2640
2641 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version,
2642 revision);
2643 if (adapter->gate_control)
2644 dev_info(&adapter->pdev->dev, "gate control detected\n");
2645
2646 return 0;
2647
2648register_failed:
2649 tsnep_rxnfc_cleanup(adapter);
2650rxnfc_init_failed:
2651 tsnep_tc_cleanup(adapter);
2652tc_init_failed:
2653 tsnep_ptp_cleanup(adapter);
2654ptp_init_failed:
2655phy_init_failed:
2656 if (adapter->mdiobus)
2657 mdiobus_unregister(adapter->mdiobus);
2658mdio_init_failed:
2659 return retval;
2660}
2661
2662static void tsnep_remove(struct platform_device *pdev)
2663{
2664 struct tsnep_adapter *adapter = platform_get_drvdata(pdev);
2665
2666 unregister_netdev(adapter->netdev);
2667
2668 tsnep_rxnfc_cleanup(adapter);
2669
2670 tsnep_tc_cleanup(adapter);
2671
2672 tsnep_ptp_cleanup(adapter);
2673
2674 if (adapter->mdiobus)
2675 mdiobus_unregister(adapter->mdiobus);
2676
2677 tsnep_disable_irq(adapter, ECM_INT_ALL);
2678}
2679
2680static const struct of_device_id tsnep_of_match[] = {
2681 { .compatible = "engleder,tsnep", },
2682{ },
2683};
2684MODULE_DEVICE_TABLE(of, tsnep_of_match);
2685
2686static struct platform_driver tsnep_driver = {
2687 .driver = {
2688 .name = TSNEP,
2689 .of_match_table = tsnep_of_match,
2690 },
2691 .probe = tsnep_probe,
2692 .remove = tsnep_remove,
2693};
2694module_platform_driver(tsnep_driver);
2695
2696MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>");
2697MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver");
2698MODULE_LICENSE("GPL");