Loading...
Note: File does not exist in v6.8.
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2/*
3 * rtase is the Linux device driver released for Realtek Automotive Switch
4 * controllers with PCI-Express interface.
5 *
6 * Copyright(c) 2024 Realtek Semiconductor Corp.
7 *
8 * Below is a simplified block diagram of the chip and its relevant interfaces.
9 *
10 * *************************
11 * * *
12 * * CPU network device *
13 * * *
14 * * +-------------+ *
15 * * | PCIE Host | *
16 * ***********++************
17 * ||
18 * PCIE
19 * ||
20 * ********************++**********************
21 * * | PCIE Endpoint | *
22 * * +---------------+ *
23 * * | GMAC | *
24 * * +--++--+ Realtek *
25 * * || RTL90xx Series *
26 * * || *
27 * * +-------------++----------------+ *
28 * * | | MAC | | *
29 * * | +-----+ | *
30 * * | | *
31 * * | Ethernet Switch Core | *
32 * * | | *
33 * * | +-----+ +-----+ | *
34 * * | | MAC |...........| MAC | | *
35 * * +---+-----+-----------+-----+---+ *
36 * * | PHY |...........| PHY | *
37 * * +--++-+ +--++-+ *
38 * *************||****************||***********
39 *
40 * The block of the Realtek RTL90xx series is our entire chip architecture,
41 * the GMAC is connected to the switch core, and there is no PHY in between.
42 * In addition, this driver is mainly used to control GMAC, but does not
43 * control the switch core, so it is not the same as DSA. Linux only plays
44 * the role of a normal leaf node in this model.
45 */
46
47#include <linux/crc32.h>
48#include <linux/dma-mapping.h>
49#include <linux/etherdevice.h>
50#include <linux/if_vlan.h>
51#include <linux/in.h>
52#include <linux/init.h>
53#include <linux/interrupt.h>
54#include <linux/io.h>
55#include <linux/iopoll.h>
56#include <linux/ip.h>
57#include <linux/ipv6.h>
58#include <linux/mdio.h>
59#include <linux/module.h>
60#include <linux/netdevice.h>
61#include <linux/pci.h>
62#include <linux/pm_runtime.h>
63#include <linux/prefetch.h>
64#include <linux/rtnetlink.h>
65#include <linux/tcp.h>
66#include <asm/irq.h>
67#include <net/ip6_checksum.h>
68#include <net/netdev_queues.h>
69#include <net/page_pool/helpers.h>
70#include <net/pkt_cls.h>
71
72#include "rtase.h"
73
74#define RTK_OPTS1_DEBUG_VALUE 0x0BADBEEF
75#define RTK_MAGIC_NUMBER 0x0BADBADBADBADBAD
76
77static const struct pci_device_id rtase_pci_tbl[] = {
78 {PCI_VDEVICE(REALTEK, 0x906A)},
79 {}
80};
81
82MODULE_DEVICE_TABLE(pci, rtase_pci_tbl);
83
84MODULE_AUTHOR("Realtek ARD Software Team");
85MODULE_DESCRIPTION("Network Driver for the PCIe interface of Realtek Automotive Ethernet Switch");
86MODULE_LICENSE("Dual BSD/GPL");
87
88struct rtase_counters {
89 __le64 tx_packets;
90 __le64 rx_packets;
91 __le64 tx_errors;
92 __le32 rx_errors;
93 __le16 rx_missed;
94 __le16 align_errors;
95 __le32 tx_one_collision;
96 __le32 tx_multi_collision;
97 __le64 rx_unicast;
98 __le64 rx_broadcast;
99 __le32 rx_multicast;
100 __le16 tx_aborted;
101 __le16 tx_underrun;
102} __packed;
103
104static void rtase_w8(const struct rtase_private *tp, u16 reg, u8 val8)
105{
106 writeb(val8, tp->mmio_addr + reg);
107}
108
109static void rtase_w16(const struct rtase_private *tp, u16 reg, u16 val16)
110{
111 writew(val16, tp->mmio_addr + reg);
112}
113
114static void rtase_w32(const struct rtase_private *tp, u16 reg, u32 val32)
115{
116 writel(val32, tp->mmio_addr + reg);
117}
118
119static u8 rtase_r8(const struct rtase_private *tp, u16 reg)
120{
121 return readb(tp->mmio_addr + reg);
122}
123
124static u16 rtase_r16(const struct rtase_private *tp, u16 reg)
125{
126 return readw(tp->mmio_addr + reg);
127}
128
129static u32 rtase_r32(const struct rtase_private *tp, u16 reg)
130{
131 return readl(tp->mmio_addr + reg);
132}
133
134static void rtase_free_desc(struct rtase_private *tp)
135{
136 struct pci_dev *pdev = tp->pdev;
137 u32 i;
138
139 for (i = 0; i < tp->func_tx_queue_num; i++) {
140 if (!tp->tx_ring[i].desc)
141 continue;
142
143 dma_free_coherent(&pdev->dev, RTASE_TX_RING_DESC_SIZE,
144 tp->tx_ring[i].desc,
145 tp->tx_ring[i].phy_addr);
146 tp->tx_ring[i].desc = NULL;
147 }
148
149 for (i = 0; i < tp->func_rx_queue_num; i++) {
150 if (!tp->rx_ring[i].desc)
151 continue;
152
153 dma_free_coherent(&pdev->dev, RTASE_RX_RING_DESC_SIZE,
154 tp->rx_ring[i].desc,
155 tp->rx_ring[i].phy_addr);
156 tp->rx_ring[i].desc = NULL;
157 }
158}
159
160static int rtase_alloc_desc(struct rtase_private *tp)
161{
162 struct pci_dev *pdev = tp->pdev;
163 u32 i;
164
165 /* rx and tx descriptors needs 256 bytes alignment.
166 * dma_alloc_coherent provides more.
167 */
168 for (i = 0; i < tp->func_tx_queue_num; i++) {
169 tp->tx_ring[i].desc =
170 dma_alloc_coherent(&pdev->dev,
171 RTASE_TX_RING_DESC_SIZE,
172 &tp->tx_ring[i].phy_addr,
173 GFP_KERNEL);
174 if (!tp->tx_ring[i].desc)
175 goto err_out;
176 }
177
178 for (i = 0; i < tp->func_rx_queue_num; i++) {
179 tp->rx_ring[i].desc =
180 dma_alloc_coherent(&pdev->dev,
181 RTASE_RX_RING_DESC_SIZE,
182 &tp->rx_ring[i].phy_addr,
183 GFP_KERNEL);
184 if (!tp->rx_ring[i].desc)
185 goto err_out;
186 }
187
188 return 0;
189
190err_out:
191 rtase_free_desc(tp);
192 return -ENOMEM;
193}
194
195static void rtase_unmap_tx_skb(struct pci_dev *pdev, u32 len,
196 struct rtase_tx_desc *desc)
197{
198 dma_unmap_single(&pdev->dev, le64_to_cpu(desc->addr), len,
199 DMA_TO_DEVICE);
200 desc->opts1 = cpu_to_le32(RTK_OPTS1_DEBUG_VALUE);
201 desc->opts2 = 0x00;
202 desc->addr = cpu_to_le64(RTK_MAGIC_NUMBER);
203}
204
205static void rtase_tx_clear_range(struct rtase_ring *ring, u32 start, u32 n)
206{
207 struct rtase_tx_desc *desc_base = ring->desc;
208 struct rtase_private *tp = ring->ivec->tp;
209 u32 i;
210
211 for (i = 0; i < n; i++) {
212 u32 entry = (start + i) % RTASE_NUM_DESC;
213 struct rtase_tx_desc *desc = desc_base + entry;
214 u32 len = ring->mis.len[entry];
215 struct sk_buff *skb;
216
217 if (len == 0)
218 continue;
219
220 rtase_unmap_tx_skb(tp->pdev, len, desc);
221 ring->mis.len[entry] = 0;
222 skb = ring->skbuff[entry];
223 if (!skb)
224 continue;
225
226 tp->stats.tx_dropped++;
227 dev_kfree_skb_any(skb);
228 ring->skbuff[entry] = NULL;
229 }
230}
231
232static void rtase_tx_clear(struct rtase_private *tp)
233{
234 struct rtase_ring *ring;
235 u16 i;
236
237 for (i = 0; i < tp->func_tx_queue_num; i++) {
238 ring = &tp->tx_ring[i];
239 rtase_tx_clear_range(ring, ring->dirty_idx, RTASE_NUM_DESC);
240 ring->cur_idx = 0;
241 ring->dirty_idx = 0;
242 }
243}
244
245static void rtase_mark_to_asic(union rtase_rx_desc *desc, u32 rx_buf_sz)
246{
247 u32 eor = le32_to_cpu(desc->desc_cmd.opts1) & RTASE_RING_END;
248
249 desc->desc_status.opts2 = 0;
250 /* force memory writes to complete before releasing descriptor */
251 dma_wmb();
252 WRITE_ONCE(desc->desc_cmd.opts1,
253 cpu_to_le32(RTASE_DESC_OWN | eor | rx_buf_sz));
254}
255
256static u32 rtase_tx_avail(struct rtase_ring *ring)
257{
258 return READ_ONCE(ring->dirty_idx) + RTASE_NUM_DESC -
259 READ_ONCE(ring->cur_idx);
260}
261
262static int tx_handler(struct rtase_ring *ring, int budget)
263{
264 const struct rtase_private *tp = ring->ivec->tp;
265 struct net_device *dev = tp->dev;
266 u32 dirty_tx, tx_left;
267 u32 bytes_compl = 0;
268 u32 pkts_compl = 0;
269 int workdone = 0;
270
271 dirty_tx = ring->dirty_idx;
272 tx_left = READ_ONCE(ring->cur_idx) - dirty_tx;
273
274 while (tx_left > 0) {
275 u32 entry = dirty_tx % RTASE_NUM_DESC;
276 struct rtase_tx_desc *desc = ring->desc +
277 sizeof(struct rtase_tx_desc) * entry;
278 u32 status;
279
280 status = le32_to_cpu(desc->opts1);
281
282 if (status & RTASE_DESC_OWN)
283 break;
284
285 rtase_unmap_tx_skb(tp->pdev, ring->mis.len[entry], desc);
286 ring->mis.len[entry] = 0;
287 if (ring->skbuff[entry]) {
288 pkts_compl++;
289 bytes_compl += ring->skbuff[entry]->len;
290 napi_consume_skb(ring->skbuff[entry], budget);
291 ring->skbuff[entry] = NULL;
292 }
293
294 dirty_tx++;
295 tx_left--;
296 workdone++;
297
298 if (workdone == RTASE_TX_BUDGET_DEFAULT)
299 break;
300 }
301
302 if (ring->dirty_idx != dirty_tx) {
303 dev_sw_netstats_tx_add(dev, pkts_compl, bytes_compl);
304 WRITE_ONCE(ring->dirty_idx, dirty_tx);
305
306 netif_subqueue_completed_wake(dev, ring->index, pkts_compl,
307 bytes_compl,
308 rtase_tx_avail(ring),
309 RTASE_TX_START_THRS);
310
311 if (ring->cur_idx != dirty_tx)
312 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index));
313 }
314
315 return 0;
316}
317
318static void rtase_tx_desc_init(struct rtase_private *tp, u16 idx)
319{
320 struct rtase_ring *ring = &tp->tx_ring[idx];
321 struct rtase_tx_desc *desc;
322 u32 i;
323
324 memset(ring->desc, 0x0, RTASE_TX_RING_DESC_SIZE);
325 memset(ring->skbuff, 0x0, sizeof(ring->skbuff));
326 ring->cur_idx = 0;
327 ring->dirty_idx = 0;
328 ring->index = idx;
329 ring->alloc_fail = 0;
330
331 for (i = 0; i < RTASE_NUM_DESC; i++) {
332 ring->mis.len[i] = 0;
333 if ((RTASE_NUM_DESC - 1) == i) {
334 desc = ring->desc + sizeof(struct rtase_tx_desc) * i;
335 desc->opts1 = cpu_to_le32(RTASE_RING_END);
336 }
337 }
338
339 ring->ring_handler = tx_handler;
340 if (idx < 4) {
341 ring->ivec = &tp->int_vector[idx];
342 list_add_tail(&ring->ring_entry,
343 &tp->int_vector[idx].ring_list);
344 } else {
345 ring->ivec = &tp->int_vector[0];
346 list_add_tail(&ring->ring_entry, &tp->int_vector[0].ring_list);
347 }
348}
349
350static void rtase_map_to_asic(union rtase_rx_desc *desc, dma_addr_t mapping,
351 u32 rx_buf_sz)
352{
353 desc->desc_cmd.addr = cpu_to_le64(mapping);
354
355 rtase_mark_to_asic(desc, rx_buf_sz);
356}
357
358static void rtase_make_unusable_by_asic(union rtase_rx_desc *desc)
359{
360 desc->desc_cmd.addr = cpu_to_le64(RTK_MAGIC_NUMBER);
361 desc->desc_cmd.opts1 &= ~cpu_to_le32(RTASE_DESC_OWN | RSVD_MASK);
362}
363
364static int rtase_alloc_rx_data_buf(struct rtase_ring *ring,
365 void **p_data_buf,
366 union rtase_rx_desc *desc,
367 dma_addr_t *rx_phy_addr)
368{
369 struct rtase_int_vector *ivec = ring->ivec;
370 const struct rtase_private *tp = ivec->tp;
371 dma_addr_t mapping;
372 struct page *page;
373
374 page = page_pool_dev_alloc_pages(tp->page_pool);
375 if (!page) {
376 ring->alloc_fail++;
377 goto err_out;
378 }
379
380 *p_data_buf = page_address(page);
381 mapping = page_pool_get_dma_addr(page);
382 *rx_phy_addr = mapping;
383 rtase_map_to_asic(desc, mapping, tp->rx_buf_sz);
384
385 return 0;
386
387err_out:
388 rtase_make_unusable_by_asic(desc);
389
390 return -ENOMEM;
391}
392
393static u32 rtase_rx_ring_fill(struct rtase_ring *ring, u32 ring_start,
394 u32 ring_end)
395{
396 union rtase_rx_desc *desc_base = ring->desc;
397 u32 cur;
398
399 for (cur = ring_start; ring_end - cur > 0; cur++) {
400 u32 i = cur % RTASE_NUM_DESC;
401 union rtase_rx_desc *desc = desc_base + i;
402 int ret;
403
404 if (ring->data_buf[i])
405 continue;
406
407 ret = rtase_alloc_rx_data_buf(ring, &ring->data_buf[i], desc,
408 &ring->mis.data_phy_addr[i]);
409 if (ret)
410 break;
411 }
412
413 return cur - ring_start;
414}
415
416static void rtase_mark_as_last_descriptor(union rtase_rx_desc *desc)
417{
418 desc->desc_cmd.opts1 |= cpu_to_le32(RTASE_RING_END);
419}
420
421static void rtase_rx_ring_clear(struct page_pool *page_pool,
422 struct rtase_ring *ring)
423{
424 union rtase_rx_desc *desc;
425 struct page *page;
426 u32 i;
427
428 for (i = 0; i < RTASE_NUM_DESC; i++) {
429 desc = ring->desc + sizeof(union rtase_rx_desc) * i;
430 page = virt_to_head_page(ring->data_buf[i]);
431
432 if (ring->data_buf[i])
433 page_pool_put_full_page(page_pool, page, true);
434
435 rtase_make_unusable_by_asic(desc);
436 }
437}
438
439static int rtase_fragmented_frame(u32 status)
440{
441 return (status & (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG)) !=
442 (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG);
443}
444
445static void rtase_rx_csum(const struct rtase_private *tp, struct sk_buff *skb,
446 const union rtase_rx_desc *desc)
447{
448 u32 opts2 = le32_to_cpu(desc->desc_status.opts2);
449
450 /* rx csum offload */
451 if (((opts2 & RTASE_RX_V4F) && !(opts2 & RTASE_RX_IPF)) ||
452 (opts2 & RTASE_RX_V6F)) {
453 if (((opts2 & RTASE_RX_TCPT) && !(opts2 & RTASE_RX_TCPF)) ||
454 ((opts2 & RTASE_RX_UDPT) && !(opts2 & RTASE_RX_UDPF)))
455 skb->ip_summed = CHECKSUM_UNNECESSARY;
456 else
457 skb->ip_summed = CHECKSUM_NONE;
458 } else {
459 skb->ip_summed = CHECKSUM_NONE;
460 }
461}
462
463static void rtase_rx_vlan_skb(union rtase_rx_desc *desc, struct sk_buff *skb)
464{
465 u32 opts2 = le32_to_cpu(desc->desc_status.opts2);
466
467 if (!(opts2 & RTASE_RX_VLAN_TAG))
468 return;
469
470 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
471 swab16(opts2 & RTASE_VLAN_TAG_MASK));
472}
473
474static void rtase_rx_skb(const struct rtase_ring *ring, struct sk_buff *skb)
475{
476 struct rtase_int_vector *ivec = ring->ivec;
477
478 napi_gro_receive(&ivec->napi, skb);
479}
480
481static int rx_handler(struct rtase_ring *ring, int budget)
482{
483 union rtase_rx_desc *desc_base = ring->desc;
484 u32 pkt_size, cur_rx, delta, entry, status;
485 struct rtase_private *tp = ring->ivec->tp;
486 struct net_device *dev = tp->dev;
487 union rtase_rx_desc *desc;
488 struct sk_buff *skb;
489 int workdone = 0;
490
491 cur_rx = ring->cur_idx;
492 entry = cur_rx % RTASE_NUM_DESC;
493 desc = &desc_base[entry];
494
495 while (workdone < budget) {
496 status = le32_to_cpu(desc->desc_status.opts1);
497
498 if (status & RTASE_DESC_OWN)
499 break;
500
501 /* This barrier is needed to keep us from reading
502 * any other fields out of the rx descriptor until
503 * we know the status of RTASE_DESC_OWN
504 */
505 dma_rmb();
506
507 if (unlikely(status & RTASE_RX_RES)) {
508 if (net_ratelimit())
509 netdev_warn(dev, "Rx ERROR. status = %08x\n",
510 status);
511
512 tp->stats.rx_errors++;
513
514 if (status & (RTASE_RX_RWT | RTASE_RX_RUNT))
515 tp->stats.rx_length_errors++;
516
517 if (status & RTASE_RX_CRC)
518 tp->stats.rx_crc_errors++;
519
520 if (dev->features & NETIF_F_RXALL)
521 goto process_pkt;
522
523 rtase_mark_to_asic(desc, tp->rx_buf_sz);
524 goto skip_process_pkt;
525 }
526
527process_pkt:
528 pkt_size = status & RTASE_RX_PKT_SIZE_MASK;
529 if (likely(!(dev->features & NETIF_F_RXFCS)))
530 pkt_size -= ETH_FCS_LEN;
531
532 /* The driver does not support incoming fragmented frames.
533 * They are seen as a symptom of over-mtu sized frames.
534 */
535 if (unlikely(rtase_fragmented_frame(status))) {
536 tp->stats.rx_dropped++;
537 tp->stats.rx_length_errors++;
538 rtase_mark_to_asic(desc, tp->rx_buf_sz);
539 goto skip_process_pkt;
540 }
541
542 dma_sync_single_for_cpu(&tp->pdev->dev,
543 ring->mis.data_phy_addr[entry],
544 tp->rx_buf_sz, DMA_FROM_DEVICE);
545
546 skb = build_skb(ring->data_buf[entry], PAGE_SIZE);
547 if (!skb) {
548 tp->stats.rx_dropped++;
549 rtase_mark_to_asic(desc, tp->rx_buf_sz);
550 goto skip_process_pkt;
551 }
552 ring->data_buf[entry] = NULL;
553
554 if (dev->features & NETIF_F_RXCSUM)
555 rtase_rx_csum(tp, skb, desc);
556
557 skb_put(skb, pkt_size);
558 skb_mark_for_recycle(skb);
559 skb->protocol = eth_type_trans(skb, dev);
560
561 if (skb->pkt_type == PACKET_MULTICAST)
562 tp->stats.multicast++;
563
564 rtase_rx_vlan_skb(desc, skb);
565 rtase_rx_skb(ring, skb);
566
567 dev_sw_netstats_rx_add(dev, pkt_size);
568
569skip_process_pkt:
570 workdone++;
571 cur_rx++;
572 entry = cur_rx % RTASE_NUM_DESC;
573 desc = ring->desc + sizeof(union rtase_rx_desc) * entry;
574 }
575
576 ring->cur_idx = cur_rx;
577 delta = rtase_rx_ring_fill(ring, ring->dirty_idx, ring->cur_idx);
578 ring->dirty_idx += delta;
579
580 return workdone;
581}
582
583static void rtase_rx_desc_init(struct rtase_private *tp, u16 idx)
584{
585 struct rtase_ring *ring = &tp->rx_ring[idx];
586 u16 i;
587
588 memset(ring->desc, 0x0, RTASE_RX_RING_DESC_SIZE);
589 memset(ring->data_buf, 0x0, sizeof(ring->data_buf));
590 ring->cur_idx = 0;
591 ring->dirty_idx = 0;
592 ring->index = idx;
593 ring->alloc_fail = 0;
594
595 for (i = 0; i < RTASE_NUM_DESC; i++)
596 ring->mis.data_phy_addr[i] = 0;
597
598 ring->ring_handler = rx_handler;
599 ring->ivec = &tp->int_vector[idx];
600 list_add_tail(&ring->ring_entry, &tp->int_vector[idx].ring_list);
601}
602
603static void rtase_rx_clear(struct rtase_private *tp)
604{
605 u32 i;
606
607 for (i = 0; i < tp->func_rx_queue_num; i++)
608 rtase_rx_ring_clear(tp->page_pool, &tp->rx_ring[i]);
609
610 page_pool_destroy(tp->page_pool);
611 tp->page_pool = NULL;
612}
613
614static int rtase_init_ring(const struct net_device *dev)
615{
616 struct rtase_private *tp = netdev_priv(dev);
617 struct page_pool_params pp_params = { 0 };
618 struct page_pool *page_pool;
619 u32 num;
620 u16 i;
621
622 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
623 pp_params.order = 0;
624 pp_params.pool_size = RTASE_NUM_DESC * tp->func_rx_queue_num;
625 pp_params.nid = dev_to_node(&tp->pdev->dev);
626 pp_params.dev = &tp->pdev->dev;
627 pp_params.dma_dir = DMA_FROM_DEVICE;
628 pp_params.max_len = PAGE_SIZE;
629 pp_params.offset = 0;
630
631 page_pool = page_pool_create(&pp_params);
632 if (IS_ERR(page_pool)) {
633 netdev_err(tp->dev, "failed to create page pool\n");
634 return -ENOMEM;
635 }
636
637 tp->page_pool = page_pool;
638
639 for (i = 0; i < tp->func_tx_queue_num; i++)
640 rtase_tx_desc_init(tp, i);
641
642 for (i = 0; i < tp->func_rx_queue_num; i++) {
643 rtase_rx_desc_init(tp, i);
644
645 num = rtase_rx_ring_fill(&tp->rx_ring[i], 0, RTASE_NUM_DESC);
646 if (num != RTASE_NUM_DESC)
647 goto err_out;
648
649 rtase_mark_as_last_descriptor(tp->rx_ring[i].desc +
650 sizeof(union rtase_rx_desc) *
651 (RTASE_NUM_DESC - 1));
652 }
653
654 return 0;
655
656err_out:
657 rtase_rx_clear(tp);
658 return -ENOMEM;
659}
660
661static void rtase_interrupt_mitigation(const struct rtase_private *tp)
662{
663 u32 i;
664
665 for (i = 0; i < tp->func_tx_queue_num; i++)
666 rtase_w16(tp, RTASE_INT_MITI_TX + i * 2, tp->tx_int_mit);
667
668 for (i = 0; i < tp->func_rx_queue_num; i++)
669 rtase_w16(tp, RTASE_INT_MITI_RX + i * 2, tp->rx_int_mit);
670}
671
672static void rtase_tally_counter_addr_fill(const struct rtase_private *tp)
673{
674 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr));
675 rtase_w32(tp, RTASE_DTCCR0, lower_32_bits(tp->tally_paddr));
676}
677
678static void rtase_tally_counter_clear(const struct rtase_private *tp)
679{
680 u32 cmd = lower_32_bits(tp->tally_paddr);
681
682 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr));
683 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_RESET);
684}
685
686static void rtase_desc_addr_fill(const struct rtase_private *tp)
687{
688 const struct rtase_ring *ring;
689 u16 i, cmd, val;
690 int err;
691
692 for (i = 0; i < tp->func_tx_queue_num; i++) {
693 ring = &tp->tx_ring[i];
694
695 rtase_w32(tp, RTASE_TX_DESC_ADDR0,
696 lower_32_bits(ring->phy_addr));
697 rtase_w32(tp, RTASE_TX_DESC_ADDR4,
698 upper_32_bits(ring->phy_addr));
699
700 cmd = i | RTASE_TX_DESC_CMD_WE | RTASE_TX_DESC_CMD_CS;
701 rtase_w16(tp, RTASE_TX_DESC_COMMAND, cmd);
702
703 err = read_poll_timeout(rtase_r16, val,
704 !(val & RTASE_TX_DESC_CMD_CS), 10,
705 1000, false, tp,
706 RTASE_TX_DESC_COMMAND);
707
708 if (err == -ETIMEDOUT)
709 netdev_err(tp->dev,
710 "error occurred in fill tx descriptor\n");
711 }
712
713 for (i = 0; i < tp->func_rx_queue_num; i++) {
714 ring = &tp->rx_ring[i];
715
716 if (i == 0) {
717 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR0,
718 lower_32_bits(ring->phy_addr));
719 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR4,
720 upper_32_bits(ring->phy_addr));
721 } else {
722 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR0 + ((i - 1) * 8)),
723 lower_32_bits(ring->phy_addr));
724 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR4 + ((i - 1) * 8)),
725 upper_32_bits(ring->phy_addr));
726 }
727 }
728}
729
730static void rtase_hw_set_features(const struct net_device *dev,
731 netdev_features_t features)
732{
733 const struct rtase_private *tp = netdev_priv(dev);
734 u16 rx_config, val;
735
736 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0);
737 if (features & NETIF_F_RXALL)
738 rx_config |= (RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT);
739 else
740 rx_config &= ~(RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT);
741
742 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config);
743
744 val = rtase_r16(tp, RTASE_CPLUS_CMD);
745 if (features & NETIF_F_RXCSUM)
746 rtase_w16(tp, RTASE_CPLUS_CMD, val | RTASE_RX_CHKSUM);
747 else
748 rtase_w16(tp, RTASE_CPLUS_CMD, val & ~RTASE_RX_CHKSUM);
749
750 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_1);
751 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
752 rx_config |= (RTASE_INNER_VLAN_DETAG_EN |
753 RTASE_OUTER_VLAN_DETAG_EN);
754 else
755 rx_config &= ~(RTASE_INNER_VLAN_DETAG_EN |
756 RTASE_OUTER_VLAN_DETAG_EN);
757
758 rtase_w16(tp, RTASE_RX_CONFIG_1, rx_config);
759}
760
761static void rtase_hw_set_rx_packet_filter(struct net_device *dev)
762{
763 u32 mc_filter[2] = { 0xFFFFFFFF, 0xFFFFFFFF };
764 struct rtase_private *tp = netdev_priv(dev);
765 u16 rx_mode;
766
767 rx_mode = rtase_r16(tp, RTASE_RX_CONFIG_0) & ~RTASE_ACCEPT_MASK;
768 rx_mode |= RTASE_ACCEPT_BROADCAST | RTASE_ACCEPT_MYPHYS;
769
770 if (dev->flags & IFF_PROMISC) {
771 rx_mode |= RTASE_ACCEPT_MULTICAST | RTASE_ACCEPT_ALLPHYS;
772 } else if (dev->flags & IFF_ALLMULTI) {
773 rx_mode |= RTASE_ACCEPT_MULTICAST;
774 } else {
775 struct netdev_hw_addr *hw_addr;
776
777 mc_filter[0] = 0;
778 mc_filter[1] = 0;
779
780 netdev_for_each_mc_addr(hw_addr, dev) {
781 u32 bit_nr = eth_hw_addr_crc(hw_addr);
782 u32 idx = u32_get_bits(bit_nr, BIT(31));
783 u32 bit = u32_get_bits(bit_nr,
784 RTASE_MULTICAST_FILTER_MASK);
785
786 mc_filter[idx] |= BIT(bit);
787 rx_mode |= RTASE_ACCEPT_MULTICAST;
788 }
789 }
790
791 if (dev->features & NETIF_F_RXALL)
792 rx_mode |= RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT;
793
794 rtase_w32(tp, RTASE_MAR0, swab32(mc_filter[1]));
795 rtase_w32(tp, RTASE_MAR1, swab32(mc_filter[0]));
796 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_mode);
797}
798
799static void rtase_irq_dis_and_clear(const struct rtase_private *tp)
800{
801 const struct rtase_int_vector *ivec = &tp->int_vector[0];
802 u32 val1;
803 u16 val2;
804 u8 i;
805
806 rtase_w32(tp, ivec->imr_addr, 0);
807 val1 = rtase_r32(tp, ivec->isr_addr);
808 rtase_w32(tp, ivec->isr_addr, val1);
809
810 for (i = 1; i < tp->int_nums; i++) {
811 ivec = &tp->int_vector[i];
812 rtase_w16(tp, ivec->imr_addr, 0);
813 val2 = rtase_r16(tp, ivec->isr_addr);
814 rtase_w16(tp, ivec->isr_addr, val2);
815 }
816}
817
818static void rtase_poll_timeout(const struct rtase_private *tp, u32 cond,
819 u32 sleep_us, u64 timeout_us, u16 reg)
820{
821 int err;
822 u8 val;
823
824 err = read_poll_timeout(rtase_r8, val, val & cond, sleep_us,
825 timeout_us, false, tp, reg);
826
827 if (err == -ETIMEDOUT)
828 netdev_err(tp->dev, "poll reg 0x00%x timeout\n", reg);
829}
830
831static void rtase_nic_reset(const struct net_device *dev)
832{
833 const struct rtase_private *tp = netdev_priv(dev);
834 u16 rx_config;
835 u8 val;
836
837 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0);
838 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config & ~RTASE_ACCEPT_MASK);
839
840 val = rtase_r8(tp, RTASE_MISC);
841 rtase_w8(tp, RTASE_MISC, val | RTASE_RX_DV_GATE_EN);
842
843 val = rtase_r8(tp, RTASE_CHIP_CMD);
844 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_STOP_REQ);
845 mdelay(2);
846
847 rtase_poll_timeout(tp, RTASE_STOP_REQ_DONE, 100, 150000,
848 RTASE_CHIP_CMD);
849
850 rtase_poll_timeout(tp, RTASE_TX_FIFO_EMPTY, 100, 100000,
851 RTASE_FIFOR);
852
853 rtase_poll_timeout(tp, RTASE_RX_FIFO_EMPTY, 100, 100000,
854 RTASE_FIFOR);
855
856 val = rtase_r8(tp, RTASE_CHIP_CMD);
857 rtase_w8(tp, RTASE_CHIP_CMD, val & ~(RTASE_TE | RTASE_RE));
858 val = rtase_r8(tp, RTASE_CHIP_CMD);
859 rtase_w8(tp, RTASE_CHIP_CMD, val & ~RTASE_STOP_REQ);
860
861 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config);
862}
863
864static void rtase_hw_reset(const struct net_device *dev)
865{
866 const struct rtase_private *tp = netdev_priv(dev);
867
868 rtase_irq_dis_and_clear(tp);
869
870 rtase_nic_reset(dev);
871}
872
873static void rtase_set_rx_queue(const struct rtase_private *tp)
874{
875 u16 reg_data;
876
877 reg_data = rtase_r16(tp, RTASE_FCR);
878 switch (tp->func_rx_queue_num) {
879 case 1:
880 u16p_replace_bits(®_data, 0x1, RTASE_FCR_RXQ_MASK);
881 break;
882 case 2:
883 u16p_replace_bits(®_data, 0x2, RTASE_FCR_RXQ_MASK);
884 break;
885 case 4:
886 u16p_replace_bits(®_data, 0x3, RTASE_FCR_RXQ_MASK);
887 break;
888 }
889 rtase_w16(tp, RTASE_FCR, reg_data);
890}
891
892static void rtase_set_tx_queue(const struct rtase_private *tp)
893{
894 u16 reg_data;
895
896 reg_data = rtase_r16(tp, RTASE_TX_CONFIG_1);
897 switch (tp->tx_queue_ctrl) {
898 case 1:
899 u16p_replace_bits(®_data, 0x0, RTASE_TC_MODE_MASK);
900 break;
901 case 2:
902 u16p_replace_bits(®_data, 0x1, RTASE_TC_MODE_MASK);
903 break;
904 case 3:
905 case 4:
906 u16p_replace_bits(®_data, 0x2, RTASE_TC_MODE_MASK);
907 break;
908 default:
909 u16p_replace_bits(®_data, 0x3, RTASE_TC_MODE_MASK);
910 break;
911 }
912 rtase_w16(tp, RTASE_TX_CONFIG_1, reg_data);
913}
914
915static void rtase_hw_config(struct net_device *dev)
916{
917 const struct rtase_private *tp = netdev_priv(dev);
918 u32 reg_data32;
919 u16 reg_data16;
920
921 rtase_hw_reset(dev);
922
923 /* set rx dma burst */
924 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_0);
925 reg_data16 &= ~(RTASE_RX_SINGLE_TAG | RTASE_RX_SINGLE_FETCH);
926 u16p_replace_bits(®_data16, RTASE_RX_DMA_BURST_256,
927 RTASE_RX_MX_DMA_MASK);
928 rtase_w16(tp, RTASE_RX_CONFIG_0, reg_data16);
929
930 /* new rx descritpor */
931 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_1);
932 reg_data16 |= RTASE_RX_NEW_DESC_FORMAT_EN | RTASE_PCIE_NEW_FLOW;
933 u16p_replace_bits(®_data16, 0xF, RTASE_RX_MAX_FETCH_DESC_MASK);
934 rtase_w16(tp, RTASE_RX_CONFIG_1, reg_data16);
935
936 rtase_set_rx_queue(tp);
937
938 rtase_interrupt_mitigation(tp);
939
940 /* set tx dma burst size and interframe gap time */
941 reg_data32 = rtase_r32(tp, RTASE_TX_CONFIG_0);
942 u32p_replace_bits(®_data32, RTASE_TX_DMA_BURST_UNLIMITED,
943 RTASE_TX_DMA_MASK);
944 u32p_replace_bits(®_data32, RTASE_INTERFRAMEGAP,
945 RTASE_TX_INTER_FRAME_GAP_MASK);
946 rtase_w32(tp, RTASE_TX_CONFIG_0, reg_data32);
947
948 /* new tx descriptor */
949 reg_data16 = rtase_r16(tp, RTASE_TFUN_CTRL);
950 rtase_w16(tp, RTASE_TFUN_CTRL, reg_data16 |
951 RTASE_TX_NEW_DESC_FORMAT_EN);
952
953 /* tx fetch desc number */
954 rtase_w8(tp, RTASE_TDFNR, 0x10);
955
956 /* tag num select */
957 reg_data16 = rtase_r16(tp, RTASE_MTPS);
958 u16p_replace_bits(®_data16, 0x4, RTASE_TAG_NUM_SEL_MASK);
959 rtase_w16(tp, RTASE_MTPS, reg_data16);
960
961 rtase_set_tx_queue(tp);
962
963 rtase_w16(tp, RTASE_TOKSEL, 0x5555);
964
965 rtase_tally_counter_addr_fill(tp);
966 rtase_desc_addr_fill(tp);
967 rtase_hw_set_features(dev, dev->features);
968
969 /* enable flow control */
970 reg_data16 = rtase_r16(tp, RTASE_CPLUS_CMD);
971 reg_data16 |= (RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN);
972 rtase_w16(tp, RTASE_CPLUS_CMD, reg_data16);
973 /* set near fifo threshold - rx missed issue. */
974 rtase_w16(tp, RTASE_RFIFONFULL, 0x190);
975
976 rtase_w16(tp, RTASE_RMS, tp->rx_buf_sz);
977
978 rtase_hw_set_rx_packet_filter(dev);
979}
980
981static void rtase_nic_enable(const struct net_device *dev)
982{
983 const struct rtase_private *tp = netdev_priv(dev);
984 u16 rcr = rtase_r16(tp, RTASE_RX_CONFIG_1);
985 u8 val;
986
987 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr & ~RTASE_PCIE_RELOAD_EN);
988 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr | RTASE_PCIE_RELOAD_EN);
989
990 val = rtase_r8(tp, RTASE_CHIP_CMD);
991 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_TE | RTASE_RE);
992
993 val = rtase_r8(tp, RTASE_MISC);
994 rtase_w8(tp, RTASE_MISC, val & ~RTASE_RX_DV_GATE_EN);
995}
996
997static void rtase_enable_hw_interrupt(const struct rtase_private *tp)
998{
999 const struct rtase_int_vector *ivec = &tp->int_vector[0];
1000 u32 i;
1001
1002 rtase_w32(tp, ivec->imr_addr, ivec->imr);
1003
1004 for (i = 1; i < tp->int_nums; i++) {
1005 ivec = &tp->int_vector[i];
1006 rtase_w16(tp, ivec->imr_addr, ivec->imr);
1007 }
1008}
1009
1010static void rtase_hw_start(const struct net_device *dev)
1011{
1012 const struct rtase_private *tp = netdev_priv(dev);
1013
1014 rtase_nic_enable(dev);
1015 rtase_enable_hw_interrupt(tp);
1016}
1017
1018/* the interrupt handler does RXQ0 and TXQ0, TXQ4~7 interrutp status
1019 */
1020static irqreturn_t rtase_interrupt(int irq, void *dev_instance)
1021{
1022 const struct rtase_private *tp;
1023 struct rtase_int_vector *ivec;
1024 u32 status;
1025
1026 ivec = dev_instance;
1027 tp = ivec->tp;
1028 status = rtase_r32(tp, ivec->isr_addr);
1029
1030 rtase_w32(tp, ivec->imr_addr, 0x0);
1031 rtase_w32(tp, ivec->isr_addr, status & ~RTASE_FOVW);
1032
1033 if (napi_schedule_prep(&ivec->napi))
1034 __napi_schedule(&ivec->napi);
1035
1036 return IRQ_HANDLED;
1037}
1038
1039/* the interrupt handler does RXQ1&TXQ1 or RXQ2&TXQ2 or RXQ3&TXQ3 interrupt
1040 * status according to interrupt vector
1041 */
1042static irqreturn_t rtase_q_interrupt(int irq, void *dev_instance)
1043{
1044 const struct rtase_private *tp;
1045 struct rtase_int_vector *ivec;
1046 u16 status;
1047
1048 ivec = dev_instance;
1049 tp = ivec->tp;
1050 status = rtase_r16(tp, ivec->isr_addr);
1051
1052 rtase_w16(tp, ivec->imr_addr, 0x0);
1053 rtase_w16(tp, ivec->isr_addr, status);
1054
1055 if (napi_schedule_prep(&ivec->napi))
1056 __napi_schedule(&ivec->napi);
1057
1058 return IRQ_HANDLED;
1059}
1060
1061static int rtase_poll(struct napi_struct *napi, int budget)
1062{
1063 const struct rtase_int_vector *ivec;
1064 const struct rtase_private *tp;
1065 struct rtase_ring *ring;
1066 int total_workdone = 0;
1067
1068 ivec = container_of(napi, struct rtase_int_vector, napi);
1069 tp = ivec->tp;
1070
1071 list_for_each_entry(ring, &ivec->ring_list, ring_entry)
1072 total_workdone += ring->ring_handler(ring, budget);
1073
1074 if (total_workdone >= budget)
1075 return budget;
1076
1077 if (napi_complete_done(napi, total_workdone)) {
1078 if (!ivec->index)
1079 rtase_w32(tp, ivec->imr_addr, ivec->imr);
1080 else
1081 rtase_w16(tp, ivec->imr_addr, ivec->imr);
1082 }
1083
1084 return total_workdone;
1085}
1086
1087static int rtase_open(struct net_device *dev)
1088{
1089 struct rtase_private *tp = netdev_priv(dev);
1090 const struct pci_dev *pdev = tp->pdev;
1091 struct rtase_int_vector *ivec;
1092 u16 i = 0, j;
1093 int ret;
1094
1095 ivec = &tp->int_vector[0];
1096 tp->rx_buf_sz = RTASE_RX_BUF_SIZE;
1097
1098 ret = rtase_alloc_desc(tp);
1099 if (ret)
1100 return ret;
1101
1102 ret = rtase_init_ring(dev);
1103 if (ret)
1104 goto err_free_all_allocated_mem;
1105
1106 rtase_hw_config(dev);
1107
1108 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) {
1109 ret = request_irq(ivec->irq, rtase_interrupt, 0,
1110 dev->name, ivec);
1111 if (ret)
1112 goto err_free_all_allocated_irq;
1113
1114 /* request other interrupts to handle multiqueue */
1115 for (i = 1; i < tp->int_nums; i++) {
1116 ivec = &tp->int_vector[i];
1117 snprintf(ivec->name, sizeof(ivec->name), "%s_int%i",
1118 tp->dev->name, i);
1119 ret = request_irq(ivec->irq, rtase_q_interrupt, 0,
1120 ivec->name, ivec);
1121 if (ret)
1122 goto err_free_all_allocated_irq;
1123 }
1124 } else {
1125 ret = request_irq(pdev->irq, rtase_interrupt, 0, dev->name,
1126 ivec);
1127 if (ret)
1128 goto err_free_all_allocated_mem;
1129 }
1130
1131 rtase_hw_start(dev);
1132
1133 for (i = 0; i < tp->int_nums; i++) {
1134 ivec = &tp->int_vector[i];
1135 napi_enable(&ivec->napi);
1136 }
1137
1138 netif_carrier_on(dev);
1139 netif_wake_queue(dev);
1140
1141 return 0;
1142
1143err_free_all_allocated_irq:
1144 for (j = 0; j < i; j++)
1145 free_irq(tp->int_vector[j].irq, &tp->int_vector[j]);
1146
1147err_free_all_allocated_mem:
1148 rtase_free_desc(tp);
1149
1150 return ret;
1151}
1152
1153static void rtase_down(struct net_device *dev)
1154{
1155 struct rtase_private *tp = netdev_priv(dev);
1156 struct rtase_int_vector *ivec;
1157 struct rtase_ring *ring, *tmp;
1158 u32 i;
1159
1160 for (i = 0; i < tp->int_nums; i++) {
1161 ivec = &tp->int_vector[i];
1162 napi_disable(&ivec->napi);
1163 list_for_each_entry_safe(ring, tmp, &ivec->ring_list,
1164 ring_entry)
1165 list_del(&ring->ring_entry);
1166 }
1167
1168 netif_tx_disable(dev);
1169
1170 netif_carrier_off(dev);
1171
1172 rtase_hw_reset(dev);
1173
1174 rtase_tx_clear(tp);
1175
1176 rtase_rx_clear(tp);
1177}
1178
1179static int rtase_close(struct net_device *dev)
1180{
1181 struct rtase_private *tp = netdev_priv(dev);
1182 const struct pci_dev *pdev = tp->pdev;
1183 u32 i;
1184
1185 rtase_down(dev);
1186
1187 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) {
1188 for (i = 0; i < tp->int_nums; i++)
1189 free_irq(tp->int_vector[i].irq, &tp->int_vector[i]);
1190
1191 } else {
1192 free_irq(pdev->irq, &tp->int_vector[0]);
1193 }
1194
1195 rtase_free_desc(tp);
1196
1197 return 0;
1198}
1199
1200static u32 rtase_tx_vlan_tag(const struct rtase_private *tp,
1201 const struct sk_buff *skb)
1202{
1203 return (skb_vlan_tag_present(skb)) ?
1204 (RTASE_TX_VLAN_TAG | swab16(skb_vlan_tag_get(skb))) : 0x00;
1205}
1206
1207static u32 rtase_tx_csum(struct sk_buff *skb, const struct net_device *dev)
1208{
1209 u32 csum_cmd = 0;
1210 u8 ip_protocol;
1211
1212 switch (vlan_get_protocol(skb)) {
1213 case htons(ETH_P_IP):
1214 csum_cmd = RTASE_TX_IPCS_C;
1215 ip_protocol = ip_hdr(skb)->protocol;
1216 break;
1217
1218 case htons(ETH_P_IPV6):
1219 csum_cmd = RTASE_TX_IPV6F_C;
1220 ip_protocol = ipv6_hdr(skb)->nexthdr;
1221 break;
1222
1223 default:
1224 ip_protocol = IPPROTO_RAW;
1225 break;
1226 }
1227
1228 if (ip_protocol == IPPROTO_TCP)
1229 csum_cmd |= RTASE_TX_TCPCS_C;
1230 else if (ip_protocol == IPPROTO_UDP)
1231 csum_cmd |= RTASE_TX_UDPCS_C;
1232
1233 csum_cmd |= u32_encode_bits(skb_transport_offset(skb),
1234 RTASE_TCPHO_MASK);
1235
1236 return csum_cmd;
1237}
1238
1239static int rtase_xmit_frags(struct rtase_ring *ring, struct sk_buff *skb,
1240 u32 opts1, u32 opts2)
1241{
1242 const struct skb_shared_info *info = skb_shinfo(skb);
1243 const struct rtase_private *tp = ring->ivec->tp;
1244 const u8 nr_frags = info->nr_frags;
1245 struct rtase_tx_desc *txd = NULL;
1246 u32 cur_frag, entry;
1247
1248 entry = ring->cur_idx;
1249 for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) {
1250 const skb_frag_t *frag = &info->frags[cur_frag];
1251 dma_addr_t mapping;
1252 u32 status, len;
1253 void *addr;
1254
1255 entry = (entry + 1) % RTASE_NUM_DESC;
1256
1257 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry;
1258 len = skb_frag_size(frag);
1259 addr = skb_frag_address(frag);
1260 mapping = dma_map_single(&tp->pdev->dev, addr, len,
1261 DMA_TO_DEVICE);
1262
1263 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) {
1264 if (unlikely(net_ratelimit()))
1265 netdev_err(tp->dev,
1266 "Failed to map TX fragments DMA!\n");
1267
1268 goto err_out;
1269 }
1270
1271 if (((entry + 1) % RTASE_NUM_DESC) == 0)
1272 status = (opts1 | len | RTASE_RING_END);
1273 else
1274 status = opts1 | len;
1275
1276 if (cur_frag == (nr_frags - 1)) {
1277 ring->skbuff[entry] = skb;
1278 status |= RTASE_TX_LAST_FRAG;
1279 }
1280
1281 ring->mis.len[entry] = len;
1282 txd->addr = cpu_to_le64(mapping);
1283 txd->opts2 = cpu_to_le32(opts2);
1284
1285 /* make sure the operating fields have been updated */
1286 dma_wmb();
1287 txd->opts1 = cpu_to_le32(status);
1288 }
1289
1290 return cur_frag;
1291
1292err_out:
1293 rtase_tx_clear_range(ring, ring->cur_idx + 1, cur_frag);
1294 return -EIO;
1295}
1296
1297static netdev_tx_t rtase_start_xmit(struct sk_buff *skb,
1298 struct net_device *dev)
1299{
1300 struct skb_shared_info *shinfo = skb_shinfo(skb);
1301 struct rtase_private *tp = netdev_priv(dev);
1302 u32 q_idx, entry, len, opts1, opts2;
1303 struct netdev_queue *tx_queue;
1304 bool stop_queue, door_bell;
1305 u32 mss = shinfo->gso_size;
1306 struct rtase_tx_desc *txd;
1307 struct rtase_ring *ring;
1308 dma_addr_t mapping;
1309 int frags;
1310
1311 /* multiqueues */
1312 q_idx = skb_get_queue_mapping(skb);
1313 ring = &tp->tx_ring[q_idx];
1314 tx_queue = netdev_get_tx_queue(dev, q_idx);
1315
1316 if (unlikely(!rtase_tx_avail(ring))) {
1317 if (net_ratelimit())
1318 netdev_err(dev,
1319 "BUG! Tx Ring full when queue awake!\n");
1320
1321 netif_stop_queue(dev);
1322 return NETDEV_TX_BUSY;
1323 }
1324
1325 entry = ring->cur_idx % RTASE_NUM_DESC;
1326 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry;
1327
1328 opts1 = RTASE_DESC_OWN;
1329 opts2 = rtase_tx_vlan_tag(tp, skb);
1330
1331 /* tcp segmentation offload (or tcp large send) */
1332 if (mss) {
1333 if (shinfo->gso_type & SKB_GSO_TCPV4) {
1334 opts1 |= RTASE_GIANT_SEND_V4;
1335 } else if (shinfo->gso_type & SKB_GSO_TCPV6) {
1336 if (skb_cow_head(skb, 0))
1337 goto err_dma_0;
1338
1339 tcp_v6_gso_csum_prep(skb);
1340 opts1 |= RTASE_GIANT_SEND_V6;
1341 } else {
1342 WARN_ON_ONCE(1);
1343 }
1344
1345 opts1 |= u32_encode_bits(skb_transport_offset(skb),
1346 RTASE_TCPHO_MASK);
1347 opts2 |= u32_encode_bits(mss, RTASE_MSS_MASK);
1348 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
1349 opts2 |= rtase_tx_csum(skb, dev);
1350 }
1351
1352 frags = rtase_xmit_frags(ring, skb, opts1, opts2);
1353 if (unlikely(frags < 0))
1354 goto err_dma_0;
1355
1356 if (frags) {
1357 len = skb_headlen(skb);
1358 opts1 |= RTASE_TX_FIRST_FRAG;
1359 } else {
1360 len = skb->len;
1361 ring->skbuff[entry] = skb;
1362 opts1 |= RTASE_TX_FIRST_FRAG | RTASE_TX_LAST_FRAG;
1363 }
1364
1365 if (((entry + 1) % RTASE_NUM_DESC) == 0)
1366 opts1 |= (len | RTASE_RING_END);
1367 else
1368 opts1 |= len;
1369
1370 mapping = dma_map_single(&tp->pdev->dev, skb->data, len,
1371 DMA_TO_DEVICE);
1372
1373 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) {
1374 if (unlikely(net_ratelimit()))
1375 netdev_err(dev, "Failed to map TX DMA!\n");
1376
1377 goto err_dma_1;
1378 }
1379
1380 ring->mis.len[entry] = len;
1381 txd->addr = cpu_to_le64(mapping);
1382 txd->opts2 = cpu_to_le32(opts2);
1383 txd->opts1 = cpu_to_le32(opts1 & ~RTASE_DESC_OWN);
1384
1385 /* make sure the operating fields have been updated */
1386 dma_wmb();
1387
1388 door_bell = __netdev_tx_sent_queue(tx_queue, skb->len,
1389 netdev_xmit_more());
1390
1391 txd->opts1 = cpu_to_le32(opts1);
1392
1393 skb_tx_timestamp(skb);
1394
1395 /* tx needs to see descriptor changes before updated cur_idx */
1396 smp_wmb();
1397
1398 WRITE_ONCE(ring->cur_idx, ring->cur_idx + frags + 1);
1399
1400 stop_queue = !netif_subqueue_maybe_stop(dev, ring->index,
1401 rtase_tx_avail(ring),
1402 RTASE_TX_STOP_THRS,
1403 RTASE_TX_START_THRS);
1404
1405 if (door_bell || stop_queue)
1406 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index));
1407
1408 return NETDEV_TX_OK;
1409
1410err_dma_1:
1411 ring->skbuff[entry] = NULL;
1412 rtase_tx_clear_range(ring, ring->cur_idx + 1, frags);
1413
1414err_dma_0:
1415 tp->stats.tx_dropped++;
1416 dev_kfree_skb_any(skb);
1417 return NETDEV_TX_OK;
1418}
1419
1420static void rtase_set_rx_mode(struct net_device *dev)
1421{
1422 rtase_hw_set_rx_packet_filter(dev);
1423}
1424
1425static void rtase_enable_eem_write(const struct rtase_private *tp)
1426{
1427 u8 val;
1428
1429 val = rtase_r8(tp, RTASE_EEM);
1430 rtase_w8(tp, RTASE_EEM, val | RTASE_EEM_UNLOCK);
1431}
1432
1433static void rtase_disable_eem_write(const struct rtase_private *tp)
1434{
1435 u8 val;
1436
1437 val = rtase_r8(tp, RTASE_EEM);
1438 rtase_w8(tp, RTASE_EEM, val & ~RTASE_EEM_UNLOCK);
1439}
1440
1441static void rtase_rar_set(const struct rtase_private *tp, const u8 *addr)
1442{
1443 u32 rar_low, rar_high;
1444
1445 rar_low = (u32)addr[0] | ((u32)addr[1] << 8) |
1446 ((u32)addr[2] << 16) | ((u32)addr[3] << 24);
1447
1448 rar_high = (u32)addr[4] | ((u32)addr[5] << 8);
1449
1450 rtase_enable_eem_write(tp);
1451 rtase_w32(tp, RTASE_MAC0, rar_low);
1452 rtase_w32(tp, RTASE_MAC4, rar_high);
1453 rtase_disable_eem_write(tp);
1454 rtase_w16(tp, RTASE_LBK_CTRL, RTASE_LBK_ATLD | RTASE_LBK_CLR);
1455}
1456
1457static int rtase_set_mac_address(struct net_device *dev, void *p)
1458{
1459 struct rtase_private *tp = netdev_priv(dev);
1460 int ret;
1461
1462 ret = eth_mac_addr(dev, p);
1463 if (ret)
1464 return ret;
1465
1466 rtase_rar_set(tp, dev->dev_addr);
1467
1468 return 0;
1469}
1470
1471static int rtase_change_mtu(struct net_device *dev, int new_mtu)
1472{
1473 dev->mtu = new_mtu;
1474
1475 netdev_update_features(dev);
1476
1477 return 0;
1478}
1479
1480static void rtase_wait_for_quiescence(const struct net_device *dev)
1481{
1482 struct rtase_private *tp = netdev_priv(dev);
1483 struct rtase_int_vector *ivec;
1484 u32 i;
1485
1486 for (i = 0; i < tp->int_nums; i++) {
1487 ivec = &tp->int_vector[i];
1488 synchronize_irq(ivec->irq);
1489 /* wait for any pending NAPI task to complete */
1490 napi_disable(&ivec->napi);
1491 }
1492
1493 rtase_irq_dis_and_clear(tp);
1494
1495 for (i = 0; i < tp->int_nums; i++) {
1496 ivec = &tp->int_vector[i];
1497 napi_enable(&ivec->napi);
1498 }
1499}
1500
1501static void rtase_sw_reset(struct net_device *dev)
1502{
1503 struct rtase_private *tp = netdev_priv(dev);
1504 int ret;
1505
1506 netif_stop_queue(dev);
1507 netif_carrier_off(dev);
1508 rtase_hw_reset(dev);
1509
1510 /* let's wait a bit while any (async) irq lands on */
1511 rtase_wait_for_quiescence(dev);
1512 rtase_tx_clear(tp);
1513 rtase_rx_clear(tp);
1514
1515 ret = rtase_init_ring(dev);
1516 if (ret) {
1517 netdev_err(dev, "unable to init ring\n");
1518 rtase_free_desc(tp);
1519 return;
1520 }
1521
1522 rtase_hw_config(dev);
1523 /* always link, so start to transmit & receive */
1524 rtase_hw_start(dev);
1525
1526 netif_carrier_on(dev);
1527 netif_wake_queue(dev);
1528}
1529
1530static void rtase_dump_tally_counter(const struct rtase_private *tp)
1531{
1532 dma_addr_t paddr = tp->tally_paddr;
1533 u32 cmd = lower_32_bits(paddr);
1534 u32 val;
1535 int err;
1536
1537 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(paddr));
1538 rtase_w32(tp, RTASE_DTCCR0, cmd);
1539 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_DUMP);
1540
1541 err = read_poll_timeout(rtase_r32, val, !(val & RTASE_COUNTER_DUMP),
1542 10, 250, false, tp, RTASE_DTCCR0);
1543
1544 if (err == -ETIMEDOUT)
1545 netdev_err(tp->dev, "error occurred in dump tally counter\n");
1546}
1547
1548static void rtase_dump_state(const struct net_device *dev)
1549{
1550 const struct rtase_private *tp = netdev_priv(dev);
1551 int max_reg_size = RTASE_PCI_REGS_SIZE;
1552 const struct rtase_counters *counters;
1553 const struct rtase_ring *ring;
1554 u32 dword_rd;
1555 int n = 0;
1556
1557 ring = &tp->tx_ring[0];
1558 netdev_err(dev, "Tx descriptor info:\n");
1559 netdev_err(dev, "Tx curIdx = 0x%x\n", ring->cur_idx);
1560 netdev_err(dev, "Tx dirtyIdx = 0x%x\n", ring->dirty_idx);
1561 netdev_err(dev, "Tx phyAddr = %pad\n", &ring->phy_addr);
1562
1563 ring = &tp->rx_ring[0];
1564 netdev_err(dev, "Rx descriptor info:\n");
1565 netdev_err(dev, "Rx curIdx = 0x%x\n", ring->cur_idx);
1566 netdev_err(dev, "Rx dirtyIdx = 0x%x\n", ring->dirty_idx);
1567 netdev_err(dev, "Rx phyAddr = %pad\n", &ring->phy_addr);
1568
1569 netdev_err(dev, "Device Registers:\n");
1570 netdev_err(dev, "Chip Command = 0x%02x\n",
1571 rtase_r8(tp, RTASE_CHIP_CMD));
1572 netdev_err(dev, "IMR = %08x\n", rtase_r32(tp, RTASE_IMR0));
1573 netdev_err(dev, "ISR = %08x\n", rtase_r32(tp, RTASE_ISR0));
1574 netdev_err(dev, "Boot Ctrl Reg(0xE004) = %04x\n",
1575 rtase_r16(tp, RTASE_BOOT_CTL));
1576 netdev_err(dev, "EPHY ISR(0xE014) = %04x\n",
1577 rtase_r16(tp, RTASE_EPHY_ISR));
1578 netdev_err(dev, "EPHY IMR(0xE016) = %04x\n",
1579 rtase_r16(tp, RTASE_EPHY_IMR));
1580 netdev_err(dev, "CLKSW SET REG(0xE018) = %04x\n",
1581 rtase_r16(tp, RTASE_CLKSW_SET));
1582
1583 netdev_err(dev, "Dump PCI Registers:\n");
1584
1585 while (n < max_reg_size) {
1586 if ((n % RTASE_DWORD_MOD) == 0)
1587 netdev_err(tp->dev, "0x%03x:\n", n);
1588
1589 pci_read_config_dword(tp->pdev, n, &dword_rd);
1590 netdev_err(tp->dev, "%08x\n", dword_rd);
1591 n += 4;
1592 }
1593
1594 netdev_err(dev, "Dump tally counter:\n");
1595 counters = tp->tally_vaddr;
1596 rtase_dump_tally_counter(tp);
1597
1598 netdev_err(dev, "tx_packets %lld\n",
1599 le64_to_cpu(counters->tx_packets));
1600 netdev_err(dev, "rx_packets %lld\n",
1601 le64_to_cpu(counters->rx_packets));
1602 netdev_err(dev, "tx_errors %lld\n",
1603 le64_to_cpu(counters->tx_errors));
1604 netdev_err(dev, "rx_errors %d\n",
1605 le32_to_cpu(counters->rx_errors));
1606 netdev_err(dev, "rx_missed %d\n",
1607 le16_to_cpu(counters->rx_missed));
1608 netdev_err(dev, "align_errors %d\n",
1609 le16_to_cpu(counters->align_errors));
1610 netdev_err(dev, "tx_one_collision %d\n",
1611 le32_to_cpu(counters->tx_one_collision));
1612 netdev_err(dev, "tx_multi_collision %d\n",
1613 le32_to_cpu(counters->tx_multi_collision));
1614 netdev_err(dev, "rx_unicast %lld\n",
1615 le64_to_cpu(counters->rx_unicast));
1616 netdev_err(dev, "rx_broadcast %lld\n",
1617 le64_to_cpu(counters->rx_broadcast));
1618 netdev_err(dev, "rx_multicast %d\n",
1619 le32_to_cpu(counters->rx_multicast));
1620 netdev_err(dev, "tx_aborted %d\n",
1621 le16_to_cpu(counters->tx_aborted));
1622 netdev_err(dev, "tx_underrun %d\n",
1623 le16_to_cpu(counters->tx_underrun));
1624}
1625
1626static void rtase_tx_timeout(struct net_device *dev, unsigned int txqueue)
1627{
1628 rtase_dump_state(dev);
1629 rtase_sw_reset(dev);
1630}
1631
1632static void rtase_get_stats64(struct net_device *dev,
1633 struct rtnl_link_stats64 *stats)
1634{
1635 const struct rtase_private *tp = netdev_priv(dev);
1636 const struct rtase_counters *counters;
1637
1638 counters = tp->tally_vaddr;
1639
1640 dev_fetch_sw_netstats(stats, dev->tstats);
1641
1642 /* fetch additional counter values missing in stats collected by driver
1643 * from tally counter
1644 */
1645 rtase_dump_tally_counter(tp);
1646 stats->rx_errors = tp->stats.rx_errors;
1647 stats->tx_errors = le64_to_cpu(counters->tx_errors);
1648 stats->rx_dropped = tp->stats.rx_dropped;
1649 stats->tx_dropped = tp->stats.tx_dropped;
1650 stats->multicast = tp->stats.multicast;
1651 stats->rx_length_errors = tp->stats.rx_length_errors;
1652}
1653
1654static netdev_features_t rtase_fix_features(struct net_device *dev,
1655 netdev_features_t features)
1656{
1657 netdev_features_t features_fix = features;
1658
1659 /* not support TSO for jumbo frames */
1660 if (dev->mtu > ETH_DATA_LEN)
1661 features_fix &= ~NETIF_F_ALL_TSO;
1662
1663 return features_fix;
1664}
1665
1666static int rtase_set_features(struct net_device *dev,
1667 netdev_features_t features)
1668{
1669 netdev_features_t features_set = features;
1670
1671 features_set &= NETIF_F_RXALL | NETIF_F_RXCSUM |
1672 NETIF_F_HW_VLAN_CTAG_RX;
1673
1674 if (features_set ^ dev->features)
1675 rtase_hw_set_features(dev, features_set);
1676
1677 return 0;
1678}
1679
1680static const struct net_device_ops rtase_netdev_ops = {
1681 .ndo_open = rtase_open,
1682 .ndo_stop = rtase_close,
1683 .ndo_start_xmit = rtase_start_xmit,
1684 .ndo_set_rx_mode = rtase_set_rx_mode,
1685 .ndo_set_mac_address = rtase_set_mac_address,
1686 .ndo_change_mtu = rtase_change_mtu,
1687 .ndo_tx_timeout = rtase_tx_timeout,
1688 .ndo_get_stats64 = rtase_get_stats64,
1689 .ndo_fix_features = rtase_fix_features,
1690 .ndo_set_features = rtase_set_features,
1691};
1692
1693static void rtase_get_mac_address(struct net_device *dev)
1694{
1695 struct rtase_private *tp = netdev_priv(dev);
1696 u8 mac_addr[ETH_ALEN] __aligned(2) = {};
1697 u32 i;
1698
1699 for (i = 0; i < ETH_ALEN; i++)
1700 mac_addr[i] = rtase_r8(tp, RTASE_MAC0 + i);
1701
1702 if (!is_valid_ether_addr(mac_addr)) {
1703 eth_hw_addr_random(dev);
1704 netdev_warn(dev, "Random ether addr %pM\n", dev->dev_addr);
1705 } else {
1706 eth_hw_addr_set(dev, mac_addr);
1707 ether_addr_copy(dev->perm_addr, dev->dev_addr);
1708 }
1709
1710 rtase_rar_set(tp, dev->dev_addr);
1711}
1712
1713static int rtase_get_settings(struct net_device *dev,
1714 struct ethtool_link_ksettings *cmd)
1715{
1716 u32 supported = SUPPORTED_MII | SUPPORTED_Pause | SUPPORTED_Asym_Pause;
1717 const struct rtase_private *tp = netdev_priv(dev);
1718
1719 ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
1720 supported);
1721
1722 switch (tp->hw_ver) {
1723 case RTASE_HW_VER_906X_7XA:
1724 case RTASE_HW_VER_906X_7XC:
1725 cmd->base.speed = SPEED_5000;
1726 break;
1727 case RTASE_HW_VER_907XD_V1:
1728 cmd->base.speed = SPEED_10000;
1729 break;
1730 }
1731
1732 cmd->base.duplex = DUPLEX_FULL;
1733 cmd->base.port = PORT_MII;
1734 cmd->base.autoneg = AUTONEG_DISABLE;
1735
1736 return 0;
1737}
1738
1739static void rtase_get_pauseparam(struct net_device *dev,
1740 struct ethtool_pauseparam *pause)
1741{
1742 const struct rtase_private *tp = netdev_priv(dev);
1743 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD);
1744
1745 pause->autoneg = AUTONEG_DISABLE;
1746 pause->tx_pause = !!(value & RTASE_FORCE_TXFLOW_EN);
1747 pause->rx_pause = !!(value & RTASE_FORCE_RXFLOW_EN);
1748}
1749
1750static int rtase_set_pauseparam(struct net_device *dev,
1751 struct ethtool_pauseparam *pause)
1752{
1753 const struct rtase_private *tp = netdev_priv(dev);
1754 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD);
1755
1756 if (pause->autoneg)
1757 return -EOPNOTSUPP;
1758
1759 value &= ~(RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN);
1760
1761 if (pause->tx_pause)
1762 value |= RTASE_FORCE_TXFLOW_EN;
1763
1764 if (pause->rx_pause)
1765 value |= RTASE_FORCE_RXFLOW_EN;
1766
1767 rtase_w16(tp, RTASE_CPLUS_CMD, value);
1768 return 0;
1769}
1770
1771static void rtase_get_eth_mac_stats(struct net_device *dev,
1772 struct ethtool_eth_mac_stats *stats)
1773{
1774 struct rtase_private *tp = netdev_priv(dev);
1775 const struct rtase_counters *counters;
1776
1777 counters = tp->tally_vaddr;
1778
1779 rtase_dump_tally_counter(tp);
1780
1781 stats->FramesTransmittedOK = le64_to_cpu(counters->tx_packets);
1782 stats->FramesReceivedOK = le64_to_cpu(counters->rx_packets);
1783 stats->FramesLostDueToIntMACXmitError =
1784 le64_to_cpu(counters->tx_errors);
1785 stats->BroadcastFramesReceivedOK = le64_to_cpu(counters->rx_broadcast);
1786}
1787
1788static const struct ethtool_ops rtase_ethtool_ops = {
1789 .get_link = ethtool_op_get_link,
1790 .get_link_ksettings = rtase_get_settings,
1791 .get_pauseparam = rtase_get_pauseparam,
1792 .set_pauseparam = rtase_set_pauseparam,
1793 .get_eth_mac_stats = rtase_get_eth_mac_stats,
1794 .get_ts_info = ethtool_op_get_ts_info,
1795};
1796
1797static void rtase_init_netdev_ops(struct net_device *dev)
1798{
1799 dev->netdev_ops = &rtase_netdev_ops;
1800 dev->ethtool_ops = &rtase_ethtool_ops;
1801}
1802
1803static void rtase_reset_interrupt(struct pci_dev *pdev,
1804 const struct rtase_private *tp)
1805{
1806 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED)
1807 pci_disable_msix(pdev);
1808 else
1809 pci_disable_msi(pdev);
1810}
1811
1812static int rtase_alloc_msix(struct pci_dev *pdev, struct rtase_private *tp)
1813{
1814 int ret, irq;
1815 u16 i;
1816
1817 memset(tp->msix_entry, 0x0, RTASE_NUM_MSIX *
1818 sizeof(struct msix_entry));
1819
1820 for (i = 0; i < RTASE_NUM_MSIX; i++)
1821 tp->msix_entry[i].entry = i;
1822
1823 ret = pci_enable_msix_exact(pdev, tp->msix_entry, tp->int_nums);
1824
1825 if (ret)
1826 return ret;
1827
1828 for (i = 0; i < tp->int_nums; i++) {
1829 irq = pci_irq_vector(pdev, i);
1830 if (irq < 0) {
1831 pci_disable_msix(pdev);
1832 return irq;
1833 }
1834
1835 tp->int_vector[i].irq = irq;
1836 }
1837
1838 return 0;
1839}
1840
1841static int rtase_alloc_interrupt(struct pci_dev *pdev,
1842 struct rtase_private *tp)
1843{
1844 int ret;
1845
1846 ret = rtase_alloc_msix(pdev, tp);
1847 if (ret) {
1848 ret = pci_enable_msi(pdev);
1849 if (ret) {
1850 dev_err(&pdev->dev,
1851 "unable to alloc interrupt.(MSI)\n");
1852 return ret;
1853 }
1854
1855 tp->sw_flag |= RTASE_SWF_MSI_ENABLED;
1856 } else {
1857 tp->sw_flag |= RTASE_SWF_MSIX_ENABLED;
1858 }
1859
1860 return 0;
1861}
1862
1863static void rtase_init_hardware(const struct rtase_private *tp)
1864{
1865 u16 i;
1866
1867 for (i = 0; i < RTASE_VLAN_FILTER_ENTRY_NUM; i++)
1868 rtase_w32(tp, RTASE_VLAN_ENTRY_0 + i * 4, 0);
1869}
1870
1871static void rtase_init_int_vector(struct rtase_private *tp)
1872{
1873 u16 i;
1874
1875 /* interrupt vector 0 */
1876 tp->int_vector[0].tp = tp;
1877 tp->int_vector[0].index = 0;
1878 tp->int_vector[0].imr_addr = RTASE_IMR0;
1879 tp->int_vector[0].isr_addr = RTASE_ISR0;
1880 tp->int_vector[0].imr = RTASE_ROK | RTASE_RDU | RTASE_TOK |
1881 RTASE_TOK4 | RTASE_TOK5 | RTASE_TOK6 |
1882 RTASE_TOK7;
1883 tp->int_vector[0].poll = rtase_poll;
1884
1885 memset(tp->int_vector[0].name, 0x0, sizeof(tp->int_vector[0].name));
1886 INIT_LIST_HEAD(&tp->int_vector[0].ring_list);
1887
1888 netif_napi_add(tp->dev, &tp->int_vector[0].napi,
1889 tp->int_vector[0].poll);
1890
1891 /* interrupt vector 1 ~ 3 */
1892 for (i = 1; i < tp->int_nums; i++) {
1893 tp->int_vector[i].tp = tp;
1894 tp->int_vector[i].index = i;
1895 tp->int_vector[i].imr_addr = RTASE_IMR1 + (i - 1) * 4;
1896 tp->int_vector[i].isr_addr = RTASE_ISR1 + (i - 1) * 4;
1897 tp->int_vector[i].imr = RTASE_Q_ROK | RTASE_Q_RDU |
1898 RTASE_Q_TOK;
1899 tp->int_vector[i].poll = rtase_poll;
1900
1901 memset(tp->int_vector[i].name, 0x0,
1902 sizeof(tp->int_vector[0].name));
1903 INIT_LIST_HEAD(&tp->int_vector[i].ring_list);
1904
1905 netif_napi_add(tp->dev, &tp->int_vector[i].napi,
1906 tp->int_vector[i].poll);
1907 }
1908}
1909
1910static u16 rtase_calc_time_mitigation(u32 time_us)
1911{
1912 u8 msb, time_count, time_unit;
1913 u16 int_miti;
1914
1915 time_us = min_t(int, time_us, RTASE_MITI_MAX_TIME);
1916
1917 msb = fls(time_us);
1918 if (msb >= RTASE_MITI_COUNT_BIT_NUM) {
1919 time_unit = msb - RTASE_MITI_COUNT_BIT_NUM;
1920 time_count = time_us >> (msb - RTASE_MITI_COUNT_BIT_NUM);
1921 } else {
1922 time_unit = 0;
1923 time_count = time_us;
1924 }
1925
1926 int_miti = u16_encode_bits(time_count, RTASE_MITI_TIME_COUNT_MASK) |
1927 u16_encode_bits(time_unit, RTASE_MITI_TIME_UNIT_MASK);
1928
1929 return int_miti;
1930}
1931
1932static u16 rtase_calc_packet_num_mitigation(u16 pkt_num)
1933{
1934 u8 msb, pkt_num_count, pkt_num_unit;
1935 u16 int_miti;
1936
1937 pkt_num = min_t(int, pkt_num, RTASE_MITI_MAX_PKT_NUM);
1938
1939 if (pkt_num > 60) {
1940 pkt_num_unit = RTASE_MITI_MAX_PKT_NUM_IDX;
1941 pkt_num_count = pkt_num / RTASE_MITI_MAX_PKT_NUM_UNIT;
1942 } else {
1943 msb = fls(pkt_num);
1944 if (msb >= RTASE_MITI_COUNT_BIT_NUM) {
1945 pkt_num_unit = msb - RTASE_MITI_COUNT_BIT_NUM;
1946 pkt_num_count = pkt_num >> (msb -
1947 RTASE_MITI_COUNT_BIT_NUM);
1948 } else {
1949 pkt_num_unit = 0;
1950 pkt_num_count = pkt_num;
1951 }
1952 }
1953
1954 int_miti = u16_encode_bits(pkt_num_count,
1955 RTASE_MITI_PKT_NUM_COUNT_MASK) |
1956 u16_encode_bits(pkt_num_unit,
1957 RTASE_MITI_PKT_NUM_UNIT_MASK);
1958
1959 return int_miti;
1960}
1961
1962static void rtase_init_software_variable(struct pci_dev *pdev,
1963 struct rtase_private *tp)
1964{
1965 u16 int_miti;
1966
1967 tp->tx_queue_ctrl = RTASE_TXQ_CTRL;
1968 tp->func_tx_queue_num = RTASE_FUNC_TXQ_NUM;
1969 tp->func_rx_queue_num = RTASE_FUNC_RXQ_NUM;
1970 tp->int_nums = RTASE_INTERRUPT_NUM;
1971
1972 int_miti = rtase_calc_time_mitigation(RTASE_MITI_DEFAULT_TIME) |
1973 rtase_calc_packet_num_mitigation(RTASE_MITI_DEFAULT_PKT_NUM);
1974 tp->tx_int_mit = int_miti;
1975 tp->rx_int_mit = int_miti;
1976
1977 tp->sw_flag = 0;
1978
1979 rtase_init_int_vector(tp);
1980
1981 /* MTU range: 60 - hw-specific max */
1982 tp->dev->min_mtu = ETH_ZLEN;
1983 tp->dev->max_mtu = RTASE_MAX_JUMBO_SIZE;
1984}
1985
1986static int rtase_check_mac_version_valid(struct rtase_private *tp)
1987{
1988 int ret = -ENODEV;
1989
1990 tp->hw_ver = rtase_r32(tp, RTASE_TX_CONFIG_0) & RTASE_HW_VER_MASK;
1991
1992 switch (tp->hw_ver) {
1993 case RTASE_HW_VER_906X_7XA:
1994 case RTASE_HW_VER_906X_7XC:
1995 case RTASE_HW_VER_907XD_V1:
1996 ret = 0;
1997 break;
1998 }
1999
2000 return ret;
2001}
2002
2003static int rtase_init_board(struct pci_dev *pdev, struct net_device **dev_out,
2004 void __iomem **ioaddr_out)
2005{
2006 struct net_device *dev;
2007 void __iomem *ioaddr;
2008 int ret = -ENOMEM;
2009
2010 /* dev zeroed in alloc_etherdev */
2011 dev = alloc_etherdev_mq(sizeof(struct rtase_private),
2012 RTASE_FUNC_TXQ_NUM);
2013 if (!dev)
2014 goto err_out;
2015
2016 SET_NETDEV_DEV(dev, &pdev->dev);
2017
2018 ret = pci_enable_device(pdev);
2019 if (ret < 0)
2020 goto err_out_free_dev;
2021
2022 /* make sure PCI base addr 1 is MMIO */
2023 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
2024 ret = -ENODEV;
2025 goto err_out_disable;
2026 }
2027
2028 /* check for weird/broken PCI region reporting */
2029 if (pci_resource_len(pdev, 2) < RTASE_REGS_SIZE) {
2030 ret = -ENODEV;
2031 goto err_out_disable;
2032 }
2033
2034 ret = pci_request_regions(pdev, KBUILD_MODNAME);
2035 if (ret < 0)
2036 goto err_out_disable;
2037
2038 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2039 if (ret) {
2040 dev_err(&pdev->dev, "no usable dma addressing method\n");
2041 goto err_out_free_res;
2042 }
2043
2044 pci_set_master(pdev);
2045
2046 /* ioremap MMIO region */
2047 ioaddr = ioremap(pci_resource_start(pdev, 2),
2048 pci_resource_len(pdev, 2));
2049 if (!ioaddr) {
2050 ret = -EIO;
2051 goto err_out_free_res;
2052 }
2053
2054 *ioaddr_out = ioaddr;
2055 *dev_out = dev;
2056
2057 return ret;
2058
2059err_out_free_res:
2060 pci_release_regions(pdev);
2061
2062err_out_disable:
2063 pci_disable_device(pdev);
2064
2065err_out_free_dev:
2066 free_netdev(dev);
2067
2068err_out:
2069 *ioaddr_out = NULL;
2070 *dev_out = NULL;
2071
2072 return ret;
2073}
2074
2075static void rtase_release_board(struct pci_dev *pdev, struct net_device *dev,
2076 void __iomem *ioaddr)
2077{
2078 const struct rtase_private *tp = netdev_priv(dev);
2079
2080 rtase_rar_set(tp, tp->dev->perm_addr);
2081 iounmap(ioaddr);
2082
2083 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED)
2084 pci_disable_msix(pdev);
2085 else
2086 pci_disable_msi(pdev);
2087
2088 pci_release_regions(pdev);
2089 pci_disable_device(pdev);
2090 free_netdev(dev);
2091}
2092
2093static int rtase_init_one(struct pci_dev *pdev,
2094 const struct pci_device_id *ent)
2095{
2096 struct net_device *dev = NULL;
2097 struct rtase_int_vector *ivec;
2098 void __iomem *ioaddr = NULL;
2099 struct rtase_private *tp;
2100 int ret, i;
2101
2102 if (!pdev->is_physfn && pdev->is_virtfn) {
2103 dev_err(&pdev->dev,
2104 "This module does not support a virtual function.");
2105 return -EINVAL;
2106 }
2107
2108 dev_dbg(&pdev->dev, "Automotive Switch Ethernet driver loaded\n");
2109
2110 ret = rtase_init_board(pdev, &dev, &ioaddr);
2111 if (ret != 0)
2112 return ret;
2113
2114 tp = netdev_priv(dev);
2115 tp->mmio_addr = ioaddr;
2116 tp->dev = dev;
2117 tp->pdev = pdev;
2118
2119 /* identify chip attached to board */
2120 ret = rtase_check_mac_version_valid(tp);
2121 if (ret != 0) {
2122 dev_err(&pdev->dev,
2123 "unknown chip version: 0x%08x, contact rtase maintainers (see MAINTAINERS file)\n",
2124 tp->hw_ver);
2125 goto err_out_release_board;
2126 }
2127
2128 rtase_init_software_variable(pdev, tp);
2129 rtase_init_hardware(tp);
2130
2131 ret = rtase_alloc_interrupt(pdev, tp);
2132 if (ret < 0) {
2133 dev_err(&pdev->dev, "unable to alloc MSIX/MSI\n");
2134 goto err_out_del_napi;
2135 }
2136
2137 rtase_init_netdev_ops(dev);
2138
2139 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
2140
2141 dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2142 NETIF_F_IP_CSUM | NETIF_F_HIGHDMA |
2143 NETIF_F_RXCSUM | NETIF_F_SG |
2144 NETIF_F_TSO | NETIF_F_IPV6_CSUM |
2145 NETIF_F_TSO6;
2146
2147 dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
2148 NETIF_F_TSO | NETIF_F_RXCSUM |
2149 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2150 NETIF_F_RXALL | NETIF_F_RXFCS |
2151 NETIF_F_IPV6_CSUM | NETIF_F_TSO6;
2152
2153 dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO |
2154 NETIF_F_HIGHDMA;
2155 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
2156 netif_set_tso_max_size(dev, RTASE_LSO_64K);
2157 netif_set_tso_max_segs(dev, RTASE_NIC_MAX_PHYS_BUF_COUNT_LSO2);
2158
2159 rtase_get_mac_address(dev);
2160
2161 tp->tally_vaddr = dma_alloc_coherent(&pdev->dev,
2162 sizeof(*tp->tally_vaddr),
2163 &tp->tally_paddr,
2164 GFP_KERNEL);
2165 if (!tp->tally_vaddr) {
2166 ret = -ENOMEM;
2167 goto err_out_free_dma;
2168 }
2169
2170 rtase_tally_counter_clear(tp);
2171
2172 pci_set_drvdata(pdev, dev);
2173
2174 netif_carrier_off(dev);
2175
2176 ret = register_netdev(dev);
2177 if (ret != 0)
2178 goto err_out_free_dma;
2179
2180 netdev_dbg(dev, "%pM, IRQ %d\n", dev->dev_addr, dev->irq);
2181
2182 return 0;
2183
2184err_out_free_dma:
2185 if (tp->tally_vaddr) {
2186 dma_free_coherent(&pdev->dev,
2187 sizeof(*tp->tally_vaddr),
2188 tp->tally_vaddr,
2189 tp->tally_paddr);
2190
2191 tp->tally_vaddr = NULL;
2192 }
2193
2194err_out_del_napi:
2195 for (i = 0; i < tp->int_nums; i++) {
2196 ivec = &tp->int_vector[i];
2197 netif_napi_del(&ivec->napi);
2198 }
2199
2200err_out_release_board:
2201 rtase_release_board(pdev, dev, ioaddr);
2202
2203 return ret;
2204}
2205
2206static void rtase_remove_one(struct pci_dev *pdev)
2207{
2208 struct net_device *dev = pci_get_drvdata(pdev);
2209 struct rtase_private *tp = netdev_priv(dev);
2210 struct rtase_int_vector *ivec;
2211 u32 i;
2212
2213 unregister_netdev(dev);
2214
2215 for (i = 0; i < tp->int_nums; i++) {
2216 ivec = &tp->int_vector[i];
2217 netif_napi_del(&ivec->napi);
2218 }
2219
2220 rtase_reset_interrupt(pdev, tp);
2221 if (tp->tally_vaddr) {
2222 dma_free_coherent(&pdev->dev,
2223 sizeof(*tp->tally_vaddr),
2224 tp->tally_vaddr,
2225 tp->tally_paddr);
2226 tp->tally_vaddr = NULL;
2227 }
2228
2229 rtase_release_board(pdev, dev, tp->mmio_addr);
2230 pci_set_drvdata(pdev, NULL);
2231}
2232
2233static void rtase_shutdown(struct pci_dev *pdev)
2234{
2235 struct net_device *dev = pci_get_drvdata(pdev);
2236 const struct rtase_private *tp;
2237
2238 tp = netdev_priv(dev);
2239
2240 if (netif_running(dev))
2241 rtase_close(dev);
2242
2243 rtase_reset_interrupt(pdev, tp);
2244}
2245
2246static int rtase_suspend(struct device *device)
2247{
2248 struct net_device *dev = dev_get_drvdata(device);
2249
2250 if (netif_running(dev)) {
2251 netif_device_detach(dev);
2252 rtase_hw_reset(dev);
2253 }
2254
2255 return 0;
2256}
2257
2258static int rtase_resume(struct device *device)
2259{
2260 struct net_device *dev = dev_get_drvdata(device);
2261 struct rtase_private *tp = netdev_priv(dev);
2262 int ret;
2263
2264 /* restore last modified mac address */
2265 rtase_rar_set(tp, dev->dev_addr);
2266
2267 if (!netif_running(dev))
2268 goto out;
2269
2270 rtase_wait_for_quiescence(dev);
2271
2272 rtase_tx_clear(tp);
2273 rtase_rx_clear(tp);
2274
2275 ret = rtase_init_ring(dev);
2276 if (ret) {
2277 netdev_err(dev, "unable to init ring\n");
2278 rtase_free_desc(tp);
2279 return -ENOMEM;
2280 }
2281
2282 rtase_hw_config(dev);
2283 /* always link, so start to transmit & receive */
2284 rtase_hw_start(dev);
2285
2286 netif_device_attach(dev);
2287out:
2288
2289 return 0;
2290}
2291
2292static const struct dev_pm_ops rtase_pm_ops = {
2293 SYSTEM_SLEEP_PM_OPS(rtase_suspend, rtase_resume)
2294};
2295
2296static struct pci_driver rtase_pci_driver = {
2297 .name = KBUILD_MODNAME,
2298 .id_table = rtase_pci_tbl,
2299 .probe = rtase_init_one,
2300 .remove = rtase_remove_one,
2301 .shutdown = rtase_shutdown,
2302 .driver.pm = pm_ptr(&rtase_pm_ops),
2303};
2304
2305module_pci_driver(rtase_pci_driver);