Loading...
Note: File does not exist in v6.13.7.
1/* Synopsys DWC Ethernet Quality-of-Service v4.10a linux driver
2 *
3 * This is a driver for the Synopsys DWC Ethernet QoS IP version 4.10a (GMAC).
4 * This version introduced a lot of changes which breaks backwards
5 * compatibility the non-QoS IP from Synopsys (used in the ST Micro drivers).
6 * Some fields differ between version 4.00a and 4.10a, mainly the interrupt
7 * bit fields. The driver could be made compatible with 4.00, if all relevant
8 * HW erratas are handled.
9 *
10 * The GMAC is highly configurable at synthesis time. This driver has been
11 * developed for a subset of the total available feature set. Currently
12 * it supports:
13 * - TSO
14 * - Checksum offload for RX and TX.
15 * - Energy efficient ethernet.
16 * - GMII phy interface.
17 * - The statistics module.
18 * - Single RX and TX queue.
19 *
20 * Copyright (C) 2015 Axis Communications AB.
21 *
22 * This program is free software; you can redistribute it and/or modify it
23 * under the terms and conditions of the GNU General Public License,
24 * version 2, as published by the Free Software Foundation.
25 */
26
27#include <linux/clk.h>
28#include <linux/module.h>
29#include <linux/kernel.h>
30#include <linux/init.h>
31#include <linux/io.h>
32#include <linux/ethtool.h>
33#include <linux/stat.h>
34#include <linux/types.h>
35
36#include <linux/types.h>
37#include <linux/slab.h>
38#include <linux/delay.h>
39#include <linux/mm.h>
40#include <linux/netdevice.h>
41#include <linux/etherdevice.h>
42#include <linux/platform_device.h>
43
44#include <linux/phy.h>
45#include <linux/mii.h>
46#include <linux/delay.h>
47#include <linux/dma-mapping.h>
48#include <linux/vmalloc.h>
49#include <linux/version.h>
50
51#include <linux/device.h>
52#include <linux/bitrev.h>
53#include <linux/crc32.h>
54
55#include <linux/of.h>
56#include <linux/interrupt.h>
57#include <linux/clocksource.h>
58#include <linux/net_tstamp.h>
59#include <linux/pm_runtime.h>
60#include <linux/of_net.h>
61#include <linux/of_address.h>
62#include <linux/of_mdio.h>
63#include <linux/timer.h>
64#include <linux/tcp.h>
65
66#define DRIVER_NAME "dwceqos"
67#define DRIVER_DESCRIPTION "Synopsys DWC Ethernet QoS driver"
68#define DRIVER_VERSION "0.9"
69
70#define DWCEQOS_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
71 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP)
72
73#define DWCEQOS_TX_TIMEOUT 5 /* Seconds */
74
75#define DWCEQOS_LPI_TIMER_MIN 8
76#define DWCEQOS_LPI_TIMER_MAX ((1 << 20) - 1)
77
78#define DWCEQOS_RX_BUF_SIZE 2048
79
80#define DWCEQOS_RX_DCNT 256
81#define DWCEQOS_TX_DCNT 256
82
83#define DWCEQOS_HASH_TABLE_SIZE 64
84
85/* The size field in the DMA descriptor is 14 bits */
86#define BYTES_PER_DMA_DESC 16376
87
88/* Hardware registers */
89#define START_MAC_REG_OFFSET 0x0000
90#define MAX_MAC_REG_OFFSET 0x0bd0
91#define START_MTL_REG_OFFSET 0x0c00
92#define MAX_MTL_REG_OFFSET 0x0d7c
93#define START_DMA_REG_OFFSET 0x1000
94#define MAX_DMA_REG_OFFSET 0x117C
95
96#define REG_SPACE_SIZE 0x1800
97
98/* DMA */
99#define REG_DWCEQOS_DMA_MODE 0x1000
100#define REG_DWCEQOS_DMA_SYSBUS_MODE 0x1004
101#define REG_DWCEQOS_DMA_IS 0x1008
102#define REG_DWCEQOS_DMA_DEBUG_ST0 0x100c
103
104/* DMA channel registers */
105#define REG_DWCEQOS_DMA_CH0_CTRL 0x1100
106#define REG_DWCEQOS_DMA_CH0_TX_CTRL 0x1104
107#define REG_DWCEQOS_DMA_CH0_RX_CTRL 0x1108
108#define REG_DWCEQOS_DMA_CH0_TXDESC_LIST 0x1114
109#define REG_DWCEQOS_DMA_CH0_RXDESC_LIST 0x111c
110#define REG_DWCEQOS_DMA_CH0_TXDESC_TAIL 0x1120
111#define REG_DWCEQOS_DMA_CH0_RXDESC_TAIL 0x1128
112#define REG_DWCEQOS_DMA_CH0_TXDESC_LEN 0x112c
113#define REG_DWCEQOS_DMA_CH0_RXDESC_LEN 0x1130
114#define REG_DWCEQOS_DMA_CH0_IE 0x1134
115#define REG_DWCEQOS_DMA_CH0_CUR_TXDESC 0x1144
116#define REG_DWCEQOS_DMA_CH0_CUR_RXDESC 0x114c
117#define REG_DWCEQOS_DMA_CH0_CUR_TXBUF 0x1154
118#define REG_DWCEQOS_DMA_CH0_CUR_RXBUG 0x115c
119#define REG_DWCEQOS_DMA_CH0_STA 0x1160
120
121#define DWCEQOS_DMA_MODE_TXPR BIT(11)
122#define DWCEQOS_DMA_MODE_DA BIT(1)
123
124#define DWCEQOS_DMA_SYSBUS_MODE_EN_LPI BIT(31)
125#define DWCEQOS_DMA_SYSBUS_MODE_FB BIT(0)
126#define DWCEQOS_DMA_SYSBUS_MODE_AAL BIT(12)
127
128#define DWCEQOS_DMA_SYSBUS_MODE_RD_OSR_LIMIT(x) \
129 (((x) << 16) & 0x000F0000)
130#define DWCEQOS_DMA_SYSBUS_MODE_RD_OSR_LIMIT_DEFAULT 3
131#define DWCEQOS_DMA_SYSBUS_MODE_RD_OSR_LIMIT_MASK GENMASK(19, 16)
132
133#define DWCEQOS_DMA_SYSBUS_MODE_WR_OSR_LIMIT(x) \
134 (((x) << 24) & 0x0F000000)
135#define DWCEQOS_DMA_SYSBUS_MODE_WR_OSR_LIMIT_DEFAULT 3
136#define DWCEQOS_DMA_SYSBUS_MODE_WR_OSR_LIMIT_MASK GENMASK(27, 24)
137
138#define DWCEQOS_DMA_SYSBUS_MODE_BURST_MASK GENMASK(7, 1)
139#define DWCEQOS_DMA_SYSBUS_MODE_BURST(x) \
140 (((x) << 1) & DWCEQOS_DMA_SYSBUS_MODE_BURST_MASK)
141#define DWCEQOS_DMA_SYSBUS_MODE_BURST_DEFAULT GENMASK(3, 1)
142
143#define DWCEQOS_DMA_CH_CTRL_PBLX8 BIT(16)
144#define DWCEQOS_DMA_CH_CTRL_DSL(x) ((x) << 18)
145
146#define DWCEQOS_DMA_CH_CTRL_PBL(x) ((x) << 16)
147#define DWCEQOS_DMA_CH_CTRL_START BIT(0)
148#define DWCEQOS_DMA_CH_RX_CTRL_BUFSIZE(x) ((x) << 1)
149#define DWCEQOS_DMA_CH_TX_OSP BIT(4)
150#define DWCEQOS_DMA_CH_TX_TSE BIT(12)
151
152#define DWCEQOS_DMA_CH0_IE_NIE BIT(15)
153#define DWCEQOS_DMA_CH0_IE_AIE BIT(14)
154#define DWCEQOS_DMA_CH0_IE_RIE BIT(6)
155#define DWCEQOS_DMA_CH0_IE_TIE BIT(0)
156#define DWCEQOS_DMA_CH0_IE_FBEE BIT(12)
157#define DWCEQOS_DMA_CH0_IE_RBUE BIT(7)
158
159#define DWCEQOS_DMA_IS_DC0IS BIT(0)
160#define DWCEQOS_DMA_IS_MTLIS BIT(16)
161#define DWCEQOS_DMA_IS_MACIS BIT(17)
162
163#define DWCEQOS_DMA_CH0_IS_TI BIT(0)
164#define DWCEQOS_DMA_CH0_IS_RI BIT(6)
165#define DWCEQOS_DMA_CH0_IS_RBU BIT(7)
166#define DWCEQOS_DMA_CH0_IS_FBE BIT(12)
167#define DWCEQOS_DMA_CH0_IS_CDE BIT(13)
168#define DWCEQOS_DMA_CH0_IS_AIS BIT(14)
169
170#define DWCEQOS_DMA_CH0_IS_TEB GENMASK(18, 16)
171#define DWCEQOS_DMA_CH0_IS_TX_ERR_READ BIT(16)
172#define DWCEQOS_DMA_CH0_IS_TX_ERR_DESCR BIT(17)
173
174#define DWCEQOS_DMA_CH0_IS_REB GENMASK(21, 19)
175#define DWCEQOS_DMA_CH0_IS_RX_ERR_READ BIT(19)
176#define DWCEQOS_DMA_CH0_IS_RX_ERR_DESCR BIT(20)
177
178/* DMA descriptor bits for RX normal descriptor (read format) */
179#define DWCEQOS_DMA_RDES3_OWN BIT(31)
180#define DWCEQOS_DMA_RDES3_INTE BIT(30)
181#define DWCEQOS_DMA_RDES3_BUF2V BIT(25)
182#define DWCEQOS_DMA_RDES3_BUF1V BIT(24)
183
184/* DMA descriptor bits for RX normal descriptor (write back format) */
185#define DWCEQOS_DMA_RDES1_IPCE BIT(7)
186#define DWCEQOS_DMA_RDES3_ES BIT(15)
187#define DWCEQOS_DMA_RDES3_E_JT BIT(14)
188#define DWCEQOS_DMA_RDES3_PL(x) ((x) & 0x7fff)
189#define DWCEQOS_DMA_RDES1_PT 0x00000007
190#define DWCEQOS_DMA_RDES1_PT_UDP BIT(0)
191#define DWCEQOS_DMA_RDES1_PT_TCP BIT(1)
192#define DWCEQOS_DMA_RDES1_PT_ICMP 0x00000003
193
194/* DMA descriptor bits for TX normal descriptor (read format) */
195#define DWCEQOS_DMA_TDES2_IOC BIT(31)
196#define DWCEQOS_DMA_TDES3_OWN BIT(31)
197#define DWCEQOS_DMA_TDES3_CTXT BIT(30)
198#define DWCEQOS_DMA_TDES3_FD BIT(29)
199#define DWCEQOS_DMA_TDES3_LD BIT(28)
200#define DWCEQOS_DMA_TDES3_CIPH BIT(16)
201#define DWCEQOS_DMA_TDES3_CIPP BIT(17)
202#define DWCEQOS_DMA_TDES3_CA 0x00030000
203#define DWCEQOS_DMA_TDES3_TSE BIT(18)
204#define DWCEQOS_DMA_DES3_THL(x) ((x) << 19)
205#define DWCEQOS_DMA_DES2_B2L(x) ((x) << 16)
206
207#define DWCEQOS_DMA_TDES3_TCMSSV BIT(26)
208
209/* DMA channel states */
210#define DMA_TX_CH_STOPPED 0
211#define DMA_TX_CH_SUSPENDED 6
212
213#define DMA_GET_TX_STATE_CH0(status0) ((status0 & 0xF000) >> 12)
214
215/* MTL */
216#define REG_DWCEQOS_MTL_OPER 0x0c00
217#define REG_DWCEQOS_MTL_DEBUG_ST 0x0c0c
218#define REG_DWCEQOS_MTL_TXQ0_DEBUG_ST 0x0d08
219#define REG_DWCEQOS_MTL_RXQ0_DEBUG_ST 0x0d38
220
221#define REG_DWCEQOS_MTL_IS 0x0c20
222#define REG_DWCEQOS_MTL_TXQ0_OPER 0x0d00
223#define REG_DWCEQOS_MTL_RXQ0_OPER 0x0d30
224#define REG_DWCEQOS_MTL_RXQ0_MIS_CNT 0x0d34
225#define REG_DWCEQOS_MTL_RXQ0_CTRL 0x0d3c
226
227#define REG_DWCEQOS_MTL_Q0_ISCTRL 0x0d2c
228
229#define DWCEQOS_MTL_SCHALG_STRICT 0x00000060
230
231#define DWCEQOS_MTL_TXQ_TXQEN BIT(3)
232#define DWCEQOS_MTL_TXQ_TSF BIT(1)
233#define DWCEQOS_MTL_TXQ_FTQ BIT(0)
234#define DWCEQOS_MTL_TXQ_TTC512 0x00000070
235
236#define DWCEQOS_MTL_TXQ_SIZE(x) ((((x) - 256) & 0xff00) << 8)
237
238#define DWCEQOS_MTL_RXQ_SIZE(x) ((((x) - 256) & 0xff00) << 12)
239#define DWCEQOS_MTL_RXQ_EHFC BIT(7)
240#define DWCEQOS_MTL_RXQ_DIS_TCP_EF BIT(6)
241#define DWCEQOS_MTL_RXQ_FEP BIT(4)
242#define DWCEQOS_MTL_RXQ_FUP BIT(3)
243#define DWCEQOS_MTL_RXQ_RSF BIT(5)
244#define DWCEQOS_MTL_RXQ_RTC32 BIT(0)
245
246/* MAC */
247#define REG_DWCEQOS_MAC_CFG 0x0000
248#define REG_DWCEQOS_MAC_EXT_CFG 0x0004
249#define REG_DWCEQOS_MAC_PKT_FILT 0x0008
250#define REG_DWCEQOS_MAC_WD_TO 0x000c
251#define REG_DWCEQOS_HASTABLE_LO 0x0010
252#define REG_DWCEQOS_HASTABLE_HI 0x0014
253#define REG_DWCEQOS_MAC_IS 0x00b0
254#define REG_DWCEQOS_MAC_IE 0x00b4
255#define REG_DWCEQOS_MAC_STAT 0x00b8
256#define REG_DWCEQOS_MAC_MDIO_ADDR 0x0200
257#define REG_DWCEQOS_MAC_MDIO_DATA 0x0204
258#define REG_DWCEQOS_MAC_MAC_ADDR0_HI 0x0300
259#define REG_DWCEQOS_MAC_MAC_ADDR0_LO 0x0304
260#define REG_DWCEQOS_MAC_RXQ0_CTRL0 0x00a0
261#define REG_DWCEQOS_MAC_HW_FEATURE0 0x011c
262#define REG_DWCEQOS_MAC_HW_FEATURE1 0x0120
263#define REG_DWCEQOS_MAC_HW_FEATURE2 0x0124
264#define REG_DWCEQOS_MAC_HASHTABLE_LO 0x0010
265#define REG_DWCEQOS_MAC_HASHTABLE_HI 0x0014
266#define REG_DWCEQOS_MAC_LPI_CTRL_STATUS 0x00d0
267#define REG_DWCEQOS_MAC_LPI_TIMERS_CTRL 0x00d4
268#define REG_DWCEQOS_MAC_LPI_ENTRY_TIMER 0x00d8
269#define REG_DWCEQOS_MAC_1US_TIC_COUNTER 0x00dc
270#define REG_DWCEQOS_MAC_RX_FLOW_CTRL 0x0090
271#define REG_DWCEQOS_MAC_Q0_TX_FLOW 0x0070
272
273#define DWCEQOS_MAC_CFG_ACS BIT(20)
274#define DWCEQOS_MAC_CFG_JD BIT(17)
275#define DWCEQOS_MAC_CFG_JE BIT(16)
276#define DWCEQOS_MAC_CFG_PS BIT(15)
277#define DWCEQOS_MAC_CFG_FES BIT(14)
278#define DWCEQOS_MAC_CFG_DM BIT(13)
279#define DWCEQOS_MAC_CFG_DO BIT(10)
280#define DWCEQOS_MAC_CFG_TE BIT(1)
281#define DWCEQOS_MAC_CFG_IPC BIT(27)
282#define DWCEQOS_MAC_CFG_RE BIT(0)
283
284#define DWCEQOS_ADDR_HIGH(reg) (0x00000300 + (reg * 8))
285#define DWCEQOS_ADDR_LOW(reg) (0x00000304 + (reg * 8))
286
287#define DWCEQOS_MAC_IS_LPI_INT BIT(5)
288#define DWCEQOS_MAC_IS_MMC_INT BIT(8)
289
290#define DWCEQOS_MAC_RXQ_EN BIT(1)
291#define DWCEQOS_MAC_MAC_ADDR_HI_EN BIT(31)
292#define DWCEQOS_MAC_PKT_FILT_RA BIT(31)
293#define DWCEQOS_MAC_PKT_FILT_HPF BIT(10)
294#define DWCEQOS_MAC_PKT_FILT_SAF BIT(9)
295#define DWCEQOS_MAC_PKT_FILT_SAIF BIT(8)
296#define DWCEQOS_MAC_PKT_FILT_DBF BIT(5)
297#define DWCEQOS_MAC_PKT_FILT_PM BIT(4)
298#define DWCEQOS_MAC_PKT_FILT_DAIF BIT(3)
299#define DWCEQOS_MAC_PKT_FILT_HMC BIT(2)
300#define DWCEQOS_MAC_PKT_FILT_HUC BIT(1)
301#define DWCEQOS_MAC_PKT_FILT_PR BIT(0)
302
303#define DWCEQOS_MAC_MDIO_ADDR_CR(x) (((x & 15)) << 8)
304#define DWCEQOS_MAC_MDIO_ADDR_CR_20 2
305#define DWCEQOS_MAC_MDIO_ADDR_CR_35 3
306#define DWCEQOS_MAC_MDIO_ADDR_CR_60 0
307#define DWCEQOS_MAC_MDIO_ADDR_CR_100 1
308#define DWCEQOS_MAC_MDIO_ADDR_CR_150 4
309#define DWCEQOS_MAC_MDIO_ADDR_CR_250 5
310#define DWCEQOS_MAC_MDIO_ADDR_GOC_READ 0x0000000c
311#define DWCEQOS_MAC_MDIO_ADDR_GOC_WRITE BIT(2)
312#define DWCEQOS_MAC_MDIO_ADDR_GB BIT(0)
313
314#define DWCEQOS_MAC_LPI_CTRL_STATUS_TLPIEN BIT(0)
315#define DWCEQOS_MAC_LPI_CTRL_STATUS_TLPIEX BIT(1)
316#define DWCEQOS_MAC_LPI_CTRL_STATUS_RLPIEN BIT(2)
317#define DWCEQOS_MAC_LPI_CTRL_STATUS_RLPIEX BIT(3)
318#define DWCEQOS_MAC_LPI_CTRL_STATUS_TLPIST BIT(8)
319#define DWCEQOS_MAC_LPI_CTRL_STATUS_RLPIST BIT(9)
320#define DWCEQOS_MAC_LPI_CTRL_STATUS_LPIEN BIT(16)
321#define DWCEQOS_MAC_LPI_CTRL_STATUS_PLS BIT(17)
322#define DWCEQOS_MAC_LPI_CTRL_STATUS_PLSEN BIT(18)
323#define DWCEQOS_MAC_LPI_CTRL_STATUS_LIPTXA BIT(19)
324#define DWCEQOS_MAC_LPI_CTRL_STATUS_LPITE BIT(20)
325#define DWCEQOS_MAC_LPI_CTRL_STATUS_LPITCSE BIT(21)
326
327#define DWCEQOS_MAC_1US_TIC_COUNTER_VAL(x) ((x) & GENMASK(11, 0))
328
329#define DWCEQOS_LPI_CTRL_ENABLE_EEE (DWCEQOS_MAC_LPI_CTRL_STATUS_LPITE | \
330 DWCEQOS_MAC_LPI_CTRL_STATUS_LIPTXA | \
331 DWCEQOS_MAC_LPI_CTRL_STATUS_LPIEN)
332
333#define DWCEQOS_MAC_RX_FLOW_CTRL_RFE BIT(0)
334
335#define DWCEQOS_MAC_Q0_TX_FLOW_TFE BIT(1)
336#define DWCEQOS_MAC_Q0_TX_FLOW_PT(time) ((time) << 16)
337#define DWCEQOS_MAC_Q0_TX_FLOW_PLT_4_SLOTS (0 << 4)
338
339/* Features */
340#define DWCEQOS_MAC_HW_FEATURE0_RXCOESEL BIT(16)
341#define DWCEQOS_MAC_HW_FEATURE0_TXCOESEL BIT(14)
342#define DWCEQOS_MAC_HW_FEATURE0_HDSEL BIT(2)
343#define DWCEQOS_MAC_HW_FEATURE0_EEESEL BIT(13)
344#define DWCEQOS_MAC_HW_FEATURE0_GMIISEL BIT(1)
345#define DWCEQOS_MAC_HW_FEATURE0_MIISEL BIT(0)
346
347#define DWCEQOS_MAC_HW_FEATURE1_TSOEN BIT(18)
348#define DWCEQOS_MAC_HW_FEATURE1_TXFIFOSIZE(x) ((128 << ((x) & 0x7c0)) >> 6)
349#define DWCEQOS_MAC_HW_FEATURE1_RXFIFOSIZE(x) (128 << ((x) & 0x1f))
350
351#define DWCEQOS_MAX_PERFECT_ADDRESSES(feature1) \
352 (1 + (((feature1) & 0x1fc0000) >> 18))
353
354#define DWCEQOS_MDIO_PHYADDR(x) (((x) & 0x1f) << 21)
355#define DWCEQOS_MDIO_PHYREG(x) (((x) & 0x1f) << 16)
356
357#define DWCEQOS_DMA_MODE_SWR BIT(0)
358
359#define DWCEQOS_DWCEQOS_RX_BUF_SIZE 2048
360
361/* Mac Management Counters */
362#define REG_DWCEQOS_MMC_CTRL 0x0700
363#define REG_DWCEQOS_MMC_RXIRQ 0x0704
364#define REG_DWCEQOS_MMC_TXIRQ 0x0708
365#define REG_DWCEQOS_MMC_RXIRQMASK 0x070c
366#define REG_DWCEQOS_MMC_TXIRQMASK 0x0710
367
368#define DWCEQOS_MMC_CTRL_CNTRST BIT(0)
369#define DWCEQOS_MMC_CTRL_RSTONRD BIT(2)
370
371#define DWC_MMC_TXLPITRANSCNTR 0x07F0
372#define DWC_MMC_TXLPIUSCNTR 0x07EC
373#define DWC_MMC_TXOVERSIZE_G 0x0778
374#define DWC_MMC_TXVLANPACKETS_G 0x0774
375#define DWC_MMC_TXPAUSEPACKETS 0x0770
376#define DWC_MMC_TXEXCESSDEF 0x076C
377#define DWC_MMC_TXPACKETCOUNT_G 0x0768
378#define DWC_MMC_TXOCTETCOUNT_G 0x0764
379#define DWC_MMC_TXCARRIERERROR 0x0760
380#define DWC_MMC_TXEXCESSCOL 0x075C
381#define DWC_MMC_TXLATECOL 0x0758
382#define DWC_MMC_TXDEFERRED 0x0754
383#define DWC_MMC_TXMULTICOL_G 0x0750
384#define DWC_MMC_TXSINGLECOL_G 0x074C
385#define DWC_MMC_TXUNDERFLOWERROR 0x0748
386#define DWC_MMC_TXBROADCASTPACKETS_GB 0x0744
387#define DWC_MMC_TXMULTICASTPACKETS_GB 0x0740
388#define DWC_MMC_TXUNICASTPACKETS_GB 0x073C
389#define DWC_MMC_TX1024TOMAXOCTETS_GB 0x0738
390#define DWC_MMC_TX512TO1023OCTETS_GB 0x0734
391#define DWC_MMC_TX256TO511OCTETS_GB 0x0730
392#define DWC_MMC_TX128TO255OCTETS_GB 0x072C
393#define DWC_MMC_TX65TO127OCTETS_GB 0x0728
394#define DWC_MMC_TX64OCTETS_GB 0x0724
395#define DWC_MMC_TXMULTICASTPACKETS_G 0x0720
396#define DWC_MMC_TXBROADCASTPACKETS_G 0x071C
397#define DWC_MMC_TXPACKETCOUNT_GB 0x0718
398#define DWC_MMC_TXOCTETCOUNT_GB 0x0714
399
400#define DWC_MMC_RXLPITRANSCNTR 0x07F8
401#define DWC_MMC_RXLPIUSCNTR 0x07F4
402#define DWC_MMC_RXCTRLPACKETS_G 0x07E4
403#define DWC_MMC_RXRCVERROR 0x07E0
404#define DWC_MMC_RXWATCHDOG 0x07DC
405#define DWC_MMC_RXVLANPACKETS_GB 0x07D8
406#define DWC_MMC_RXFIFOOVERFLOW 0x07D4
407#define DWC_MMC_RXPAUSEPACKETS 0x07D0
408#define DWC_MMC_RXOUTOFRANGETYPE 0x07CC
409#define DWC_MMC_RXLENGTHERROR 0x07C8
410#define DWC_MMC_RXUNICASTPACKETS_G 0x07C4
411#define DWC_MMC_RX1024TOMAXOCTETS_GB 0x07C0
412#define DWC_MMC_RX512TO1023OCTETS_GB 0x07BC
413#define DWC_MMC_RX256TO511OCTETS_GB 0x07B8
414#define DWC_MMC_RX128TO255OCTETS_GB 0x07B4
415#define DWC_MMC_RX65TO127OCTETS_GB 0x07B0
416#define DWC_MMC_RX64OCTETS_GB 0x07AC
417#define DWC_MMC_RXOVERSIZE_G 0x07A8
418#define DWC_MMC_RXUNDERSIZE_G 0x07A4
419#define DWC_MMC_RXJABBERERROR 0x07A0
420#define DWC_MMC_RXRUNTERROR 0x079C
421#define DWC_MMC_RXALIGNMENTERROR 0x0798
422#define DWC_MMC_RXCRCERROR 0x0794
423#define DWC_MMC_RXMULTICASTPACKETS_G 0x0790
424#define DWC_MMC_RXBROADCASTPACKETS_G 0x078C
425#define DWC_MMC_RXOCTETCOUNT_G 0x0788
426#define DWC_MMC_RXOCTETCOUNT_GB 0x0784
427#define DWC_MMC_RXPACKETCOUNT_GB 0x0780
428
429static int debug = -1;
430module_param(debug, int, 0);
431MODULE_PARM_DESC(debug, "DWC_eth_qos debug level (0=none,...,16=all)");
432
433/* DMA ring descriptor. These are used as support descriptors for the HW DMA */
434struct ring_desc {
435 struct sk_buff *skb;
436 dma_addr_t mapping;
437 size_t len;
438};
439
440/* DMA hardware descriptor */
441struct dwceqos_dma_desc {
442 u32 des0;
443 u32 des1;
444 u32 des2;
445 u32 des3;
446} ____cacheline_aligned;
447
448struct dwceqos_mmc_counters {
449 __u64 txlpitranscntr;
450 __u64 txpiuscntr;
451 __u64 txoversize_g;
452 __u64 txvlanpackets_g;
453 __u64 txpausepackets;
454 __u64 txexcessdef;
455 __u64 txpacketcount_g;
456 __u64 txoctetcount_g;
457 __u64 txcarriererror;
458 __u64 txexcesscol;
459 __u64 txlatecol;
460 __u64 txdeferred;
461 __u64 txmulticol_g;
462 __u64 txsinglecol_g;
463 __u64 txunderflowerror;
464 __u64 txbroadcastpackets_gb;
465 __u64 txmulticastpackets_gb;
466 __u64 txunicastpackets_gb;
467 __u64 tx1024tomaxoctets_gb;
468 __u64 tx512to1023octets_gb;
469 __u64 tx256to511octets_gb;
470 __u64 tx128to255octets_gb;
471 __u64 tx65to127octets_gb;
472 __u64 tx64octets_gb;
473 __u64 txmulticastpackets_g;
474 __u64 txbroadcastpackets_g;
475 __u64 txpacketcount_gb;
476 __u64 txoctetcount_gb;
477
478 __u64 rxlpitranscntr;
479 __u64 rxlpiuscntr;
480 __u64 rxctrlpackets_g;
481 __u64 rxrcverror;
482 __u64 rxwatchdog;
483 __u64 rxvlanpackets_gb;
484 __u64 rxfifooverflow;
485 __u64 rxpausepackets;
486 __u64 rxoutofrangetype;
487 __u64 rxlengtherror;
488 __u64 rxunicastpackets_g;
489 __u64 rx1024tomaxoctets_gb;
490 __u64 rx512to1023octets_gb;
491 __u64 rx256to511octets_gb;
492 __u64 rx128to255octets_gb;
493 __u64 rx65to127octets_gb;
494 __u64 rx64octets_gb;
495 __u64 rxoversize_g;
496 __u64 rxundersize_g;
497 __u64 rxjabbererror;
498 __u64 rxrunterror;
499 __u64 rxalignmenterror;
500 __u64 rxcrcerror;
501 __u64 rxmulticastpackets_g;
502 __u64 rxbroadcastpackets_g;
503 __u64 rxoctetcount_g;
504 __u64 rxoctetcount_gb;
505 __u64 rxpacketcount_gb;
506};
507
508/* Ethtool statistics */
509
510struct dwceqos_stat {
511 const char stat_name[ETH_GSTRING_LEN];
512 int offset;
513};
514
515#define STAT_ITEM(name, var) \
516 {\
517 name,\
518 offsetof(struct dwceqos_mmc_counters, var),\
519 }
520
521static const struct dwceqos_stat dwceqos_ethtool_stats[] = {
522 STAT_ITEM("tx_bytes", txoctetcount_gb),
523 STAT_ITEM("tx_packets", txpacketcount_gb),
524 STAT_ITEM("tx_unicst_packets", txunicastpackets_gb),
525 STAT_ITEM("tx_broadcast_packets", txbroadcastpackets_gb),
526 STAT_ITEM("tx_multicast_packets", txmulticastpackets_gb),
527 STAT_ITEM("tx_pause_packets", txpausepackets),
528 STAT_ITEM("tx_up_to_64_byte_packets", tx64octets_gb),
529 STAT_ITEM("tx_65_to_127_byte_packets", tx65to127octets_gb),
530 STAT_ITEM("tx_128_to_255_byte_packets", tx128to255octets_gb),
531 STAT_ITEM("tx_256_to_511_byte_packets", tx256to511octets_gb),
532 STAT_ITEM("tx_512_to_1023_byte_packets", tx512to1023octets_gb),
533 STAT_ITEM("tx_1024_to_maxsize_packets", tx1024tomaxoctets_gb),
534 STAT_ITEM("tx_underflow_errors", txunderflowerror),
535 STAT_ITEM("tx_lpi_count", txlpitranscntr),
536
537 STAT_ITEM("rx_bytes", rxoctetcount_gb),
538 STAT_ITEM("rx_packets", rxpacketcount_gb),
539 STAT_ITEM("rx_unicast_packets", rxunicastpackets_g),
540 STAT_ITEM("rx_broadcast_packets", rxbroadcastpackets_g),
541 STAT_ITEM("rx_multicast_packets", rxmulticastpackets_g),
542 STAT_ITEM("rx_vlan_packets", rxvlanpackets_gb),
543 STAT_ITEM("rx_pause_packets", rxpausepackets),
544 STAT_ITEM("rx_up_to_64_byte_packets", rx64octets_gb),
545 STAT_ITEM("rx_65_to_127_byte_packets", rx65to127octets_gb),
546 STAT_ITEM("rx_128_to_255_byte_packets", rx128to255octets_gb),
547 STAT_ITEM("rx_256_to_511_byte_packets", rx256to511octets_gb),
548 STAT_ITEM("rx_512_to_1023_byte_packets", rx512to1023octets_gb),
549 STAT_ITEM("rx_1024_to_maxsize_packets", rx1024tomaxoctets_gb),
550 STAT_ITEM("rx_fifo_overflow_errors", rxfifooverflow),
551 STAT_ITEM("rx_oversize_packets", rxoversize_g),
552 STAT_ITEM("rx_undersize_packets", rxundersize_g),
553 STAT_ITEM("rx_jabbers", rxjabbererror),
554 STAT_ITEM("rx_align_errors", rxalignmenterror),
555 STAT_ITEM("rx_crc_errors", rxcrcerror),
556 STAT_ITEM("rx_lpi_count", rxlpitranscntr),
557};
558
559/* Configuration of AXI bus parameters.
560 * These values depend on the parameters set on the MAC core as well
561 * as the AXI interconnect.
562 */
563struct dwceqos_bus_cfg {
564 /* Enable AXI low-power interface. */
565 bool en_lpi;
566 /* Limit on number of outstanding AXI write requests. */
567 u32 write_requests;
568 /* Limit on number of outstanding AXI read requests. */
569 u32 read_requests;
570 /* Bitmap of allowed AXI burst lengths, 4-256 beats. */
571 u32 burst_map;
572 /* DMA Programmable burst length*/
573 u32 tx_pbl;
574 u32 rx_pbl;
575};
576
577struct dwceqos_flowcontrol {
578 int autoneg;
579 int rx;
580 int rx_current;
581 int tx;
582 int tx_current;
583};
584
585struct net_local {
586 void __iomem *baseaddr;
587 struct clk *phy_ref_clk;
588 struct clk *apb_pclk;
589
590 struct device_node *phy_node;
591 struct net_device *ndev;
592 struct platform_device *pdev;
593
594 u32 msg_enable;
595
596 struct tasklet_struct tx_bdreclaim_tasklet;
597 struct workqueue_struct *txtimeout_handler_wq;
598 struct work_struct txtimeout_reinit;
599
600 phy_interface_t phy_interface;
601 struct phy_device *phy_dev;
602 struct mii_bus *mii_bus;
603
604 unsigned int link;
605 unsigned int speed;
606 unsigned int duplex;
607
608 struct napi_struct napi;
609
610 /* DMA Descriptor Areas */
611 struct ring_desc *rx_skb;
612 struct ring_desc *tx_skb;
613
614 struct dwceqos_dma_desc *tx_descs;
615 struct dwceqos_dma_desc *rx_descs;
616
617 /* DMA Mapped Descriptor areas*/
618 dma_addr_t tx_descs_addr;
619 dma_addr_t rx_descs_addr;
620 dma_addr_t tx_descs_tail_addr;
621 dma_addr_t rx_descs_tail_addr;
622
623 size_t tx_free;
624 size_t tx_next;
625 size_t rx_cur;
626 size_t tx_cur;
627
628 /* Spinlocks for accessing DMA Descriptors */
629 spinlock_t tx_lock;
630
631 /* Spinlock for register read-modify-writes. */
632 spinlock_t hw_lock;
633
634 u32 feature0;
635 u32 feature1;
636 u32 feature2;
637
638 struct dwceqos_bus_cfg bus_cfg;
639 bool en_tx_lpi_clockgating;
640
641 int eee_enabled;
642 int eee_active;
643 int csr_val;
644 u32 gso_size;
645
646 struct dwceqos_mmc_counters mmc_counters;
647 /* Protect the mmc_counter updates. */
648 spinlock_t stats_lock;
649 u32 mmc_rx_counters_mask;
650 u32 mmc_tx_counters_mask;
651
652 struct dwceqos_flowcontrol flowcontrol;
653
654 /* Tracks the intermediate state of phy started but hardware
655 * init not finished yet.
656 */
657 bool phy_defer;
658};
659
660static void dwceqos_read_mmc_counters(struct net_local *lp, u32 rx_mask,
661 u32 tx_mask);
662
663static void dwceqos_set_umac_addr(struct net_local *lp, unsigned char *addr,
664 unsigned int reg_n);
665static int dwceqos_stop(struct net_device *ndev);
666static int dwceqos_open(struct net_device *ndev);
667static void dwceqos_tx_poll_demand(struct net_local *lp);
668
669static void dwceqos_set_rx_flowcontrol(struct net_local *lp, bool enable);
670static void dwceqos_set_tx_flowcontrol(struct net_local *lp, bool enable);
671
672static void dwceqos_reset_state(struct net_local *lp);
673
674#define dwceqos_read(lp, reg) \
675 readl_relaxed(((void __iomem *)((lp)->baseaddr)) + (reg))
676#define dwceqos_write(lp, reg, val) \
677 writel_relaxed((val), ((void __iomem *)((lp)->baseaddr)) + (reg))
678
679static void dwceqos_reset_state(struct net_local *lp)
680{
681 lp->link = 0;
682 lp->speed = 0;
683 lp->duplex = DUPLEX_UNKNOWN;
684 lp->flowcontrol.rx_current = 0;
685 lp->flowcontrol.tx_current = 0;
686 lp->eee_active = 0;
687 lp->eee_enabled = 0;
688}
689
690static void print_descriptor(struct net_local *lp, int index, int tx)
691{
692 struct dwceqos_dma_desc *dd;
693
694 if (tx)
695 dd = (struct dwceqos_dma_desc *)&lp->tx_descs[index];
696 else
697 dd = (struct dwceqos_dma_desc *)&lp->rx_descs[index];
698
699 pr_info("%s DMA Descriptor #%d@%p Contents:\n", tx ? "TX" : "RX",
700 index, dd);
701 pr_info("0x%08x 0x%08x 0x%08x 0x%08x\n", dd->des0, dd->des1, dd->des2,
702 dd->des3);
703}
704
705static void print_status(struct net_local *lp)
706{
707 size_t desci, i;
708
709 pr_info("tx_free %zu, tx_cur %zu, tx_next %zu\n", lp->tx_free,
710 lp->tx_cur, lp->tx_next);
711
712 print_descriptor(lp, lp->rx_cur, 0);
713
714 for (desci = (lp->tx_cur - 10) % DWCEQOS_TX_DCNT, i = 0;
715 i < DWCEQOS_TX_DCNT;
716 ++i) {
717 print_descriptor(lp, desci, 1);
718 desci = (desci + 1) % DWCEQOS_TX_DCNT;
719 }
720
721 pr_info("DMA_Debug_Status0: 0x%08x\n",
722 dwceqos_read(lp, REG_DWCEQOS_DMA_DEBUG_ST0));
723 pr_info("DMA_CH0_Status: 0x%08x\n",
724 dwceqos_read(lp, REG_DWCEQOS_DMA_IS));
725 pr_info("DMA_CH0_Current_App_TxDesc: 0x%08x\n",
726 dwceqos_read(lp, 0x1144));
727 pr_info("DMA_CH0_Current_App_TxBuff: 0x%08x\n",
728 dwceqos_read(lp, 0x1154));
729 pr_info("MTL_Debug_Status: 0x%08x\n",
730 dwceqos_read(lp, REG_DWCEQOS_MTL_DEBUG_ST));
731 pr_info("MTL_TXQ0_Debug_Status: 0x%08x\n",
732 dwceqos_read(lp, REG_DWCEQOS_MTL_TXQ0_DEBUG_ST));
733 pr_info("MTL_RXQ0_Debug_Status: 0x%08x\n",
734 dwceqos_read(lp, REG_DWCEQOS_MTL_RXQ0_DEBUG_ST));
735 pr_info("Current TX DMA: 0x%08x, RX DMA: 0x%08x\n",
736 dwceqos_read(lp, REG_DWCEQOS_DMA_CH0_CUR_TXDESC),
737 dwceqos_read(lp, REG_DWCEQOS_DMA_CH0_CUR_RXDESC));
738}
739
740static void dwceqos_mdio_set_csr(struct net_local *lp)
741{
742 int rate = clk_get_rate(lp->apb_pclk);
743
744 if (rate <= 20000000)
745 lp->csr_val = DWCEQOS_MAC_MDIO_ADDR_CR_20;
746 else if (rate <= 35000000)
747 lp->csr_val = DWCEQOS_MAC_MDIO_ADDR_CR_35;
748 else if (rate <= 60000000)
749 lp->csr_val = DWCEQOS_MAC_MDIO_ADDR_CR_60;
750 else if (rate <= 100000000)
751 lp->csr_val = DWCEQOS_MAC_MDIO_ADDR_CR_100;
752 else if (rate <= 150000000)
753 lp->csr_val = DWCEQOS_MAC_MDIO_ADDR_CR_150;
754 else if (rate <= 250000000)
755 lp->csr_val = DWCEQOS_MAC_MDIO_ADDR_CR_250;
756}
757
758/* Simple MDIO functions implementing mii_bus */
759static int dwceqos_mdio_read(struct mii_bus *bus, int mii_id, int phyreg)
760{
761 struct net_local *lp = bus->priv;
762 u32 regval;
763 int i;
764 int data;
765
766 regval = DWCEQOS_MDIO_PHYADDR(mii_id) |
767 DWCEQOS_MDIO_PHYREG(phyreg) |
768 DWCEQOS_MAC_MDIO_ADDR_CR(lp->csr_val) |
769 DWCEQOS_MAC_MDIO_ADDR_GB |
770 DWCEQOS_MAC_MDIO_ADDR_GOC_READ;
771 dwceqos_write(lp, REG_DWCEQOS_MAC_MDIO_ADDR, regval);
772
773 for (i = 0; i < 5; ++i) {
774 usleep_range(64, 128);
775 if (!(dwceqos_read(lp, REG_DWCEQOS_MAC_MDIO_ADDR) &
776 DWCEQOS_MAC_MDIO_ADDR_GB))
777 break;
778 }
779
780 data = dwceqos_read(lp, REG_DWCEQOS_MAC_MDIO_DATA);
781 if (i == 5) {
782 netdev_warn(lp->ndev, "MDIO read timed out\n");
783 data = 0xffff;
784 }
785
786 return data & 0xffff;
787}
788
789static int dwceqos_mdio_write(struct mii_bus *bus, int mii_id, int phyreg,
790 u16 value)
791{
792 struct net_local *lp = bus->priv;
793 u32 regval;
794 int i;
795
796 dwceqos_write(lp, REG_DWCEQOS_MAC_MDIO_DATA, value);
797
798 regval = DWCEQOS_MDIO_PHYADDR(mii_id) |
799 DWCEQOS_MDIO_PHYREG(phyreg) |
800 DWCEQOS_MAC_MDIO_ADDR_CR(lp->csr_val) |
801 DWCEQOS_MAC_MDIO_ADDR_GB |
802 DWCEQOS_MAC_MDIO_ADDR_GOC_WRITE;
803 dwceqos_write(lp, REG_DWCEQOS_MAC_MDIO_ADDR, regval);
804
805 for (i = 0; i < 5; ++i) {
806 usleep_range(64, 128);
807 if (!(dwceqos_read(lp, REG_DWCEQOS_MAC_MDIO_ADDR) &
808 DWCEQOS_MAC_MDIO_ADDR_GB))
809 break;
810 }
811 if (i == 5)
812 netdev_warn(lp->ndev, "MDIO write timed out\n");
813 return 0;
814}
815
816static int dwceqos_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
817{
818 struct net_local *lp = netdev_priv(ndev);
819 struct phy_device *phydev = lp->phy_dev;
820
821 if (!netif_running(ndev))
822 return -EINVAL;
823
824 if (!phydev)
825 return -ENODEV;
826
827 switch (cmd) {
828 case SIOCGMIIPHY:
829 case SIOCGMIIREG:
830 case SIOCSMIIREG:
831 return phy_mii_ioctl(phydev, rq, cmd);
832 default:
833 dev_info(&lp->pdev->dev, "ioctl %X not implemented.\n", cmd);
834 return -EOPNOTSUPP;
835 }
836}
837
838static void dwceqos_link_down(struct net_local *lp)
839{
840 u32 regval;
841 unsigned long flags;
842
843 /* Indicate link down to the LPI state machine */
844 spin_lock_irqsave(&lp->hw_lock, flags);
845 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS);
846 regval &= ~DWCEQOS_MAC_LPI_CTRL_STATUS_PLS;
847 dwceqos_write(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS, regval);
848 spin_unlock_irqrestore(&lp->hw_lock, flags);
849}
850
851static void dwceqos_link_up(struct net_local *lp)
852{
853 u32 regval;
854 unsigned long flags;
855
856 /* Indicate link up to the LPI state machine */
857 spin_lock_irqsave(&lp->hw_lock, flags);
858 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS);
859 regval |= DWCEQOS_MAC_LPI_CTRL_STATUS_PLS;
860 dwceqos_write(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS, regval);
861 spin_unlock_irqrestore(&lp->hw_lock, flags);
862
863 lp->eee_active = !phy_init_eee(lp->phy_dev, 0);
864
865 /* Check for changed EEE capability */
866 if (!lp->eee_active && lp->eee_enabled) {
867 lp->eee_enabled = 0;
868
869 spin_lock_irqsave(&lp->hw_lock, flags);
870 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS);
871 regval &= ~DWCEQOS_LPI_CTRL_ENABLE_EEE;
872 dwceqos_write(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS, regval);
873 spin_unlock_irqrestore(&lp->hw_lock, flags);
874 }
875}
876
877static void dwceqos_set_speed(struct net_local *lp)
878{
879 struct phy_device *phydev = lp->phy_dev;
880 u32 regval;
881
882 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_CFG);
883 regval &= ~(DWCEQOS_MAC_CFG_PS | DWCEQOS_MAC_CFG_FES |
884 DWCEQOS_MAC_CFG_DM);
885
886 if (phydev->duplex)
887 regval |= DWCEQOS_MAC_CFG_DM;
888 if (phydev->speed == SPEED_10) {
889 regval |= DWCEQOS_MAC_CFG_PS;
890 } else if (phydev->speed == SPEED_100) {
891 regval |= DWCEQOS_MAC_CFG_PS |
892 DWCEQOS_MAC_CFG_FES;
893 } else if (phydev->speed != SPEED_1000) {
894 netdev_err(lp->ndev,
895 "unknown PHY speed %d\n",
896 phydev->speed);
897 return;
898 }
899
900 dwceqos_write(lp, REG_DWCEQOS_MAC_CFG, regval);
901}
902
903static void dwceqos_adjust_link(struct net_device *ndev)
904{
905 struct net_local *lp = netdev_priv(ndev);
906 struct phy_device *phydev = lp->phy_dev;
907 int status_change = 0;
908
909 if (lp->phy_defer)
910 return;
911
912 if (phydev->link) {
913 if ((lp->speed != phydev->speed) ||
914 (lp->duplex != phydev->duplex)) {
915 dwceqos_set_speed(lp);
916
917 lp->speed = phydev->speed;
918 lp->duplex = phydev->duplex;
919 status_change = 1;
920 }
921
922 if (lp->flowcontrol.autoneg) {
923 lp->flowcontrol.rx = phydev->pause ||
924 phydev->asym_pause;
925 lp->flowcontrol.tx = phydev->pause ||
926 phydev->asym_pause;
927 }
928
929 if (lp->flowcontrol.rx != lp->flowcontrol.rx_current) {
930 if (netif_msg_link(lp))
931 netdev_dbg(ndev, "set rx flow to %d\n",
932 lp->flowcontrol.rx);
933 dwceqos_set_rx_flowcontrol(lp, lp->flowcontrol.rx);
934 lp->flowcontrol.rx_current = lp->flowcontrol.rx;
935 }
936 if (lp->flowcontrol.tx != lp->flowcontrol.tx_current) {
937 if (netif_msg_link(lp))
938 netdev_dbg(ndev, "set tx flow to %d\n",
939 lp->flowcontrol.tx);
940 dwceqos_set_tx_flowcontrol(lp, lp->flowcontrol.tx);
941 lp->flowcontrol.tx_current = lp->flowcontrol.tx;
942 }
943 }
944
945 if (phydev->link != lp->link) {
946 lp->link = phydev->link;
947 status_change = 1;
948 }
949
950 if (status_change) {
951 if (phydev->link) {
952 lp->ndev->trans_start = jiffies;
953 dwceqos_link_up(lp);
954 } else {
955 dwceqos_link_down(lp);
956 }
957 phy_print_status(phydev);
958 }
959}
960
961static int dwceqos_mii_probe(struct net_device *ndev)
962{
963 struct net_local *lp = netdev_priv(ndev);
964 struct phy_device *phydev = NULL;
965
966 if (lp->phy_node) {
967 phydev = of_phy_connect(lp->ndev,
968 lp->phy_node,
969 &dwceqos_adjust_link,
970 0,
971 lp->phy_interface);
972
973 if (!phydev) {
974 netdev_err(ndev, "no PHY found\n");
975 return -1;
976 }
977 } else {
978 netdev_err(ndev, "no PHY configured\n");
979 return -ENODEV;
980 }
981
982 if (netif_msg_probe(lp))
983 phy_attached_info(phydev);
984
985 phydev->supported &= PHY_GBIT_FEATURES;
986
987 lp->link = 0;
988 lp->speed = 0;
989 lp->duplex = DUPLEX_UNKNOWN;
990 lp->phy_dev = phydev;
991
992 return 0;
993}
994
995static void dwceqos_alloc_rxring_desc(struct net_local *lp, int index)
996{
997 struct sk_buff *new_skb;
998 dma_addr_t new_skb_baddr = 0;
999
1000 new_skb = netdev_alloc_skb(lp->ndev, DWCEQOS_RX_BUF_SIZE);
1001 if (!new_skb) {
1002 netdev_err(lp->ndev, "alloc_skb error for desc %d\n", index);
1003 goto err_out;
1004 }
1005
1006 new_skb_baddr = dma_map_single(lp->ndev->dev.parent,
1007 new_skb->data, DWCEQOS_RX_BUF_SIZE,
1008 DMA_FROM_DEVICE);
1009 if (dma_mapping_error(lp->ndev->dev.parent, new_skb_baddr)) {
1010 netdev_err(lp->ndev, "DMA map error\n");
1011 dev_kfree_skb(new_skb);
1012 new_skb = NULL;
1013 goto err_out;
1014 }
1015
1016 lp->rx_descs[index].des0 = new_skb_baddr;
1017 lp->rx_descs[index].des1 = 0;
1018 lp->rx_descs[index].des2 = 0;
1019 lp->rx_descs[index].des3 = DWCEQOS_DMA_RDES3_INTE |
1020 DWCEQOS_DMA_RDES3_BUF1V |
1021 DWCEQOS_DMA_RDES3_OWN;
1022
1023 lp->rx_skb[index].mapping = new_skb_baddr;
1024 lp->rx_skb[index].len = DWCEQOS_RX_BUF_SIZE;
1025
1026err_out:
1027 lp->rx_skb[index].skb = new_skb;
1028}
1029
1030static void dwceqos_clean_rings(struct net_local *lp)
1031{
1032 int i;
1033
1034 if (lp->rx_skb) {
1035 for (i = 0; i < DWCEQOS_RX_DCNT; i++) {
1036 if (lp->rx_skb[i].skb) {
1037 dma_unmap_single(lp->ndev->dev.parent,
1038 lp->rx_skb[i].mapping,
1039 lp->rx_skb[i].len,
1040 DMA_FROM_DEVICE);
1041
1042 dev_kfree_skb(lp->rx_skb[i].skb);
1043 lp->rx_skb[i].skb = NULL;
1044 lp->rx_skb[i].mapping = 0;
1045 }
1046 }
1047 }
1048
1049 if (lp->tx_skb) {
1050 for (i = 0; i < DWCEQOS_TX_DCNT; i++) {
1051 if (lp->tx_skb[i].skb) {
1052 dev_kfree_skb(lp->tx_skb[i].skb);
1053 lp->tx_skb[i].skb = NULL;
1054 }
1055 if (lp->tx_skb[i].mapping) {
1056 dma_unmap_single(lp->ndev->dev.parent,
1057 lp->tx_skb[i].mapping,
1058 lp->tx_skb[i].len,
1059 DMA_TO_DEVICE);
1060 lp->tx_skb[i].mapping = 0;
1061 }
1062 }
1063 }
1064}
1065
1066static void dwceqos_descriptor_free(struct net_local *lp)
1067{
1068 int size;
1069
1070 dwceqos_clean_rings(lp);
1071
1072 kfree(lp->tx_skb);
1073 lp->tx_skb = NULL;
1074 kfree(lp->rx_skb);
1075 lp->rx_skb = NULL;
1076
1077 size = DWCEQOS_RX_DCNT * sizeof(struct dwceqos_dma_desc);
1078 if (lp->rx_descs) {
1079 dma_free_coherent(lp->ndev->dev.parent, size,
1080 (void *)(lp->rx_descs), lp->rx_descs_addr);
1081 lp->rx_descs = NULL;
1082 }
1083
1084 size = DWCEQOS_TX_DCNT * sizeof(struct dwceqos_dma_desc);
1085 if (lp->tx_descs) {
1086 dma_free_coherent(lp->ndev->dev.parent, size,
1087 (void *)(lp->tx_descs), lp->tx_descs_addr);
1088 lp->tx_descs = NULL;
1089 }
1090}
1091
1092static int dwceqos_descriptor_init(struct net_local *lp)
1093{
1094 int size;
1095 u32 i;
1096
1097 lp->gso_size = 0;
1098
1099 lp->tx_skb = NULL;
1100 lp->rx_skb = NULL;
1101 lp->rx_descs = NULL;
1102 lp->tx_descs = NULL;
1103
1104 /* Reset the DMA indexes */
1105 lp->rx_cur = 0;
1106 lp->tx_cur = 0;
1107 lp->tx_next = 0;
1108 lp->tx_free = DWCEQOS_TX_DCNT;
1109
1110 /* Allocate Ring descriptors */
1111 size = DWCEQOS_RX_DCNT * sizeof(struct ring_desc);
1112 lp->rx_skb = kzalloc(size, GFP_KERNEL);
1113 if (!lp->rx_skb)
1114 goto err_out;
1115
1116 size = DWCEQOS_TX_DCNT * sizeof(struct ring_desc);
1117 lp->tx_skb = kzalloc(size, GFP_KERNEL);
1118 if (!lp->tx_skb)
1119 goto err_out;
1120
1121 /* Allocate DMA descriptors */
1122 size = DWCEQOS_RX_DCNT * sizeof(struct dwceqos_dma_desc);
1123 lp->rx_descs = dma_alloc_coherent(lp->ndev->dev.parent, size,
1124 &lp->rx_descs_addr, GFP_KERNEL);
1125 if (!lp->rx_descs)
1126 goto err_out;
1127 lp->rx_descs_tail_addr = lp->rx_descs_addr +
1128 sizeof(struct dwceqos_dma_desc) * DWCEQOS_RX_DCNT;
1129
1130 size = DWCEQOS_TX_DCNT * sizeof(struct dwceqos_dma_desc);
1131 lp->tx_descs = dma_alloc_coherent(lp->ndev->dev.parent, size,
1132 &lp->tx_descs_addr, GFP_KERNEL);
1133 if (!lp->tx_descs)
1134 goto err_out;
1135 lp->tx_descs_tail_addr = lp->tx_descs_addr +
1136 sizeof(struct dwceqos_dma_desc) * DWCEQOS_TX_DCNT;
1137
1138 /* Initialize RX Ring Descriptors and buffers */
1139 for (i = 0; i < DWCEQOS_RX_DCNT; ++i) {
1140 dwceqos_alloc_rxring_desc(lp, i);
1141 if (!(lp->rx_skb[lp->rx_cur].skb))
1142 goto err_out;
1143 }
1144
1145 /* Initialize TX Descriptors */
1146 for (i = 0; i < DWCEQOS_TX_DCNT; ++i) {
1147 lp->tx_descs[i].des0 = 0;
1148 lp->tx_descs[i].des1 = 0;
1149 lp->tx_descs[i].des2 = 0;
1150 lp->tx_descs[i].des3 = 0;
1151 }
1152
1153 /* Make descriptor writes visible to the DMA. */
1154 wmb();
1155
1156 return 0;
1157
1158err_out:
1159 dwceqos_descriptor_free(lp);
1160 return -ENOMEM;
1161}
1162
1163static int dwceqos_packet_avail(struct net_local *lp)
1164{
1165 return !(lp->rx_descs[lp->rx_cur].des3 & DWCEQOS_DMA_RDES3_OWN);
1166}
1167
1168static void dwceqos_get_hwfeatures(struct net_local *lp)
1169{
1170 lp->feature0 = dwceqos_read(lp, REG_DWCEQOS_MAC_HW_FEATURE0);
1171 lp->feature1 = dwceqos_read(lp, REG_DWCEQOS_MAC_HW_FEATURE1);
1172 lp->feature2 = dwceqos_read(lp, REG_DWCEQOS_MAC_HW_FEATURE2);
1173}
1174
1175static void dwceqos_dma_enable_txirq(struct net_local *lp)
1176{
1177 u32 regval;
1178 unsigned long flags;
1179
1180 spin_lock_irqsave(&lp->hw_lock, flags);
1181 regval = dwceqos_read(lp, REG_DWCEQOS_DMA_CH0_IE);
1182 regval |= DWCEQOS_DMA_CH0_IE_TIE;
1183 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_IE, regval);
1184 spin_unlock_irqrestore(&lp->hw_lock, flags);
1185}
1186
1187static void dwceqos_dma_disable_txirq(struct net_local *lp)
1188{
1189 u32 regval;
1190 unsigned long flags;
1191
1192 spin_lock_irqsave(&lp->hw_lock, flags);
1193 regval = dwceqos_read(lp, REG_DWCEQOS_DMA_CH0_IE);
1194 regval &= ~DWCEQOS_DMA_CH0_IE_TIE;
1195 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_IE, regval);
1196 spin_unlock_irqrestore(&lp->hw_lock, flags);
1197}
1198
1199static void dwceqos_dma_enable_rxirq(struct net_local *lp)
1200{
1201 u32 regval;
1202 unsigned long flags;
1203
1204 spin_lock_irqsave(&lp->hw_lock, flags);
1205 regval = dwceqos_read(lp, REG_DWCEQOS_DMA_CH0_IE);
1206 regval |= DWCEQOS_DMA_CH0_IE_RIE;
1207 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_IE, regval);
1208 spin_unlock_irqrestore(&lp->hw_lock, flags);
1209}
1210
1211static void dwceqos_dma_disable_rxirq(struct net_local *lp)
1212{
1213 u32 regval;
1214 unsigned long flags;
1215
1216 spin_lock_irqsave(&lp->hw_lock, flags);
1217 regval = dwceqos_read(lp, REG_DWCEQOS_DMA_CH0_IE);
1218 regval &= ~DWCEQOS_DMA_CH0_IE_RIE;
1219 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_IE, regval);
1220 spin_unlock_irqrestore(&lp->hw_lock, flags);
1221}
1222
1223static void dwceqos_enable_mmc_interrupt(struct net_local *lp)
1224{
1225 dwceqos_write(lp, REG_DWCEQOS_MMC_RXIRQMASK, 0);
1226 dwceqos_write(lp, REG_DWCEQOS_MMC_TXIRQMASK, 0);
1227}
1228
1229static int dwceqos_mii_init(struct net_local *lp)
1230{
1231 int ret = -ENXIO;
1232 struct resource res;
1233 struct device_node *mdionode;
1234
1235 mdionode = of_get_child_by_name(lp->pdev->dev.of_node, "mdio");
1236
1237 if (!mdionode)
1238 return 0;
1239
1240 lp->mii_bus = mdiobus_alloc();
1241 if (!lp->mii_bus) {
1242 ret = -ENOMEM;
1243 goto err_out;
1244 }
1245
1246 lp->mii_bus->name = "DWCEQOS MII bus";
1247 lp->mii_bus->read = &dwceqos_mdio_read;
1248 lp->mii_bus->write = &dwceqos_mdio_write;
1249 lp->mii_bus->priv = lp;
1250 lp->mii_bus->parent = &lp->ndev->dev;
1251
1252 of_address_to_resource(lp->pdev->dev.of_node, 0, &res);
1253 snprintf(lp->mii_bus->id, MII_BUS_ID_SIZE, "%.8llx",
1254 (unsigned long long)res.start);
1255 if (of_mdiobus_register(lp->mii_bus, mdionode))
1256 goto err_out_free_mdiobus;
1257
1258 return 0;
1259
1260err_out_free_mdiobus:
1261 mdiobus_free(lp->mii_bus);
1262err_out:
1263 of_node_put(mdionode);
1264 return ret;
1265}
1266
1267/* DMA reset. When issued also resets all MTL and MAC registers as well */
1268static void dwceqos_reset_hw(struct net_local *lp)
1269{
1270 /* Wait (at most) 0.5 seconds for DMA reset*/
1271 int i = 5000;
1272 u32 reg;
1273
1274 /* Force gigabit to guarantee a TX clock for GMII. */
1275 reg = dwceqos_read(lp, REG_DWCEQOS_MAC_CFG);
1276 reg &= ~(DWCEQOS_MAC_CFG_PS | DWCEQOS_MAC_CFG_FES);
1277 reg |= DWCEQOS_MAC_CFG_DM;
1278 dwceqos_write(lp, REG_DWCEQOS_MAC_CFG, reg);
1279
1280 dwceqos_write(lp, REG_DWCEQOS_DMA_MODE, DWCEQOS_DMA_MODE_SWR);
1281
1282 do {
1283 udelay(100);
1284 i--;
1285 reg = dwceqos_read(lp, REG_DWCEQOS_DMA_MODE);
1286 } while ((reg & DWCEQOS_DMA_MODE_SWR) && i);
1287 /* We might experience a timeout if the chip clock mux is broken */
1288 if (!i)
1289 netdev_err(lp->ndev, "DMA reset timed out!\n");
1290}
1291
1292static void dwceqos_fatal_bus_error(struct net_local *lp, u32 dma_status)
1293{
1294 if (dma_status & DWCEQOS_DMA_CH0_IS_TEB) {
1295 netdev_err(lp->ndev, "txdma bus error %s %s (status=%08x)\n",
1296 dma_status & DWCEQOS_DMA_CH0_IS_TX_ERR_READ ?
1297 "read" : "write",
1298 dma_status & DWCEQOS_DMA_CH0_IS_TX_ERR_DESCR ?
1299 "descr" : "data",
1300 dma_status);
1301
1302 print_status(lp);
1303 }
1304 if (dma_status & DWCEQOS_DMA_CH0_IS_REB) {
1305 netdev_err(lp->ndev, "rxdma bus error %s %s (status=%08x)\n",
1306 dma_status & DWCEQOS_DMA_CH0_IS_RX_ERR_READ ?
1307 "read" : "write",
1308 dma_status & DWCEQOS_DMA_CH0_IS_RX_ERR_DESCR ?
1309 "descr" : "data",
1310 dma_status);
1311
1312 print_status(lp);
1313 }
1314}
1315
1316static void dwceqos_mmc_interrupt(struct net_local *lp)
1317{
1318 unsigned long flags;
1319
1320 spin_lock_irqsave(&lp->stats_lock, flags);
1321
1322 /* A latched mmc interrupt can not be masked, we must read
1323 * all the counters with an interrupt pending.
1324 */
1325 dwceqos_read_mmc_counters(lp,
1326 dwceqos_read(lp, REG_DWCEQOS_MMC_RXIRQ),
1327 dwceqos_read(lp, REG_DWCEQOS_MMC_TXIRQ));
1328
1329 spin_unlock_irqrestore(&lp->stats_lock, flags);
1330}
1331
1332static void dwceqos_mac_interrupt(struct net_local *lp)
1333{
1334 u32 cause;
1335
1336 cause = dwceqos_read(lp, REG_DWCEQOS_MAC_IS);
1337
1338 if (cause & DWCEQOS_MAC_IS_MMC_INT)
1339 dwceqos_mmc_interrupt(lp);
1340}
1341
1342static irqreturn_t dwceqos_interrupt(int irq, void *dev_id)
1343{
1344 struct net_device *ndev = dev_id;
1345 struct net_local *lp = netdev_priv(ndev);
1346
1347 u32 cause;
1348 u32 dma_status;
1349 irqreturn_t ret = IRQ_NONE;
1350
1351 cause = dwceqos_read(lp, REG_DWCEQOS_DMA_IS);
1352 /* DMA Channel 0 Interrupt */
1353 if (cause & DWCEQOS_DMA_IS_DC0IS) {
1354 dma_status = dwceqos_read(lp, REG_DWCEQOS_DMA_CH0_STA);
1355
1356 /* Transmit Interrupt */
1357 if (dma_status & DWCEQOS_DMA_CH0_IS_TI) {
1358 tasklet_schedule(&lp->tx_bdreclaim_tasklet);
1359 dwceqos_dma_disable_txirq(lp);
1360 }
1361
1362 /* Receive Interrupt */
1363 if (dma_status & DWCEQOS_DMA_CH0_IS_RI) {
1364 /* Disable RX IRQs */
1365 dwceqos_dma_disable_rxirq(lp);
1366 napi_schedule(&lp->napi);
1367 }
1368
1369 /* Fatal Bus Error interrupt */
1370 if (unlikely(dma_status & DWCEQOS_DMA_CH0_IS_FBE)) {
1371 dwceqos_fatal_bus_error(lp, dma_status);
1372
1373 /* errata 9000831707 */
1374 dma_status |= DWCEQOS_DMA_CH0_IS_TEB |
1375 DWCEQOS_DMA_CH0_IS_REB;
1376 }
1377
1378 /* Ack all DMA Channel 0 IRQs */
1379 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_STA, dma_status);
1380 ret = IRQ_HANDLED;
1381 }
1382
1383 if (cause & DWCEQOS_DMA_IS_MTLIS) {
1384 u32 val = dwceqos_read(lp, REG_DWCEQOS_MTL_Q0_ISCTRL);
1385
1386 dwceqos_write(lp, REG_DWCEQOS_MTL_Q0_ISCTRL, val);
1387 ret = IRQ_HANDLED;
1388 }
1389
1390 if (cause & DWCEQOS_DMA_IS_MACIS) {
1391 dwceqos_mac_interrupt(lp);
1392 ret = IRQ_HANDLED;
1393 }
1394 return ret;
1395}
1396
1397static void dwceqos_set_rx_flowcontrol(struct net_local *lp, bool enable)
1398{
1399 u32 regval;
1400 unsigned long flags;
1401
1402 spin_lock_irqsave(&lp->hw_lock, flags);
1403
1404 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_RX_FLOW_CTRL);
1405 if (enable)
1406 regval |= DWCEQOS_MAC_RX_FLOW_CTRL_RFE;
1407 else
1408 regval &= ~DWCEQOS_MAC_RX_FLOW_CTRL_RFE;
1409 dwceqos_write(lp, REG_DWCEQOS_MAC_RX_FLOW_CTRL, regval);
1410
1411 spin_unlock_irqrestore(&lp->hw_lock, flags);
1412}
1413
1414static void dwceqos_set_tx_flowcontrol(struct net_local *lp, bool enable)
1415{
1416 u32 regval;
1417 unsigned long flags;
1418
1419 spin_lock_irqsave(&lp->hw_lock, flags);
1420
1421 /* MTL flow control */
1422 regval = dwceqos_read(lp, REG_DWCEQOS_MTL_RXQ0_OPER);
1423 if (enable)
1424 regval |= DWCEQOS_MTL_RXQ_EHFC;
1425 else
1426 regval &= ~DWCEQOS_MTL_RXQ_EHFC;
1427
1428 dwceqos_write(lp, REG_DWCEQOS_MTL_RXQ0_OPER, regval);
1429
1430 /* MAC flow control */
1431 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_Q0_TX_FLOW);
1432 if (enable)
1433 regval |= DWCEQOS_MAC_Q0_TX_FLOW_TFE;
1434 else
1435 regval &= ~DWCEQOS_MAC_Q0_TX_FLOW_TFE;
1436 dwceqos_write(lp, REG_DWCEQOS_MAC_Q0_TX_FLOW, regval);
1437
1438 spin_unlock_irqrestore(&lp->hw_lock, flags);
1439}
1440
1441static void dwceqos_configure_flow_control(struct net_local *lp)
1442{
1443 u32 regval;
1444 unsigned long flags;
1445 int RQS, RFD, RFA;
1446
1447 spin_lock_irqsave(&lp->hw_lock, flags);
1448
1449 regval = dwceqos_read(lp, REG_DWCEQOS_MTL_RXQ0_OPER);
1450
1451 /* The queue size is in units of 256 bytes. We want 512 bytes units for
1452 * the threshold fields.
1453 */
1454 RQS = ((regval >> 20) & 0x3FF) + 1;
1455 RQS /= 2;
1456
1457 /* The thresholds are relative to a full queue, with a bias
1458 * of 1 KiByte below full.
1459 */
1460 RFD = RQS / 2 - 2;
1461 RFA = RQS / 8 - 2;
1462
1463 regval = (regval & 0xFFF000FF) | (RFD << 14) | (RFA << 8);
1464
1465 if (RFD >= 0 && RFA >= 0) {
1466 dwceqos_write(lp, REG_DWCEQOS_MTL_RXQ0_OPER, regval);
1467 } else {
1468 netdev_warn(lp->ndev,
1469 "FIFO too small for flow control.");
1470 }
1471
1472 regval = DWCEQOS_MAC_Q0_TX_FLOW_PT(256) |
1473 DWCEQOS_MAC_Q0_TX_FLOW_PLT_4_SLOTS;
1474
1475 dwceqos_write(lp, REG_DWCEQOS_MAC_Q0_TX_FLOW, regval);
1476
1477 spin_unlock_irqrestore(&lp->hw_lock, flags);
1478}
1479
1480static void dwceqos_configure_clock(struct net_local *lp)
1481{
1482 unsigned long rate_mhz = clk_get_rate(lp->apb_pclk) / 1000000;
1483
1484 BUG_ON(!rate_mhz);
1485
1486 dwceqos_write(lp,
1487 REG_DWCEQOS_MAC_1US_TIC_COUNTER,
1488 DWCEQOS_MAC_1US_TIC_COUNTER_VAL(rate_mhz - 1));
1489}
1490
1491static void dwceqos_configure_bus(struct net_local *lp)
1492{
1493 u32 sysbus_reg;
1494
1495 /* N.B. We do not support the Fixed Burst mode because it
1496 * opens a race window by making HW access to DMA descriptors
1497 * non-atomic.
1498 */
1499
1500 sysbus_reg = DWCEQOS_DMA_SYSBUS_MODE_AAL;
1501
1502 if (lp->bus_cfg.en_lpi)
1503 sysbus_reg |= DWCEQOS_DMA_SYSBUS_MODE_EN_LPI;
1504
1505 if (lp->bus_cfg.burst_map)
1506 sysbus_reg |= DWCEQOS_DMA_SYSBUS_MODE_BURST(
1507 lp->bus_cfg.burst_map);
1508 else
1509 sysbus_reg |= DWCEQOS_DMA_SYSBUS_MODE_BURST(
1510 DWCEQOS_DMA_SYSBUS_MODE_BURST_DEFAULT);
1511
1512 if (lp->bus_cfg.read_requests)
1513 sysbus_reg |= DWCEQOS_DMA_SYSBUS_MODE_RD_OSR_LIMIT(
1514 lp->bus_cfg.read_requests - 1);
1515 else
1516 sysbus_reg |= DWCEQOS_DMA_SYSBUS_MODE_RD_OSR_LIMIT(
1517 DWCEQOS_DMA_SYSBUS_MODE_RD_OSR_LIMIT_DEFAULT);
1518
1519 if (lp->bus_cfg.write_requests)
1520 sysbus_reg |= DWCEQOS_DMA_SYSBUS_MODE_WR_OSR_LIMIT(
1521 lp->bus_cfg.write_requests - 1);
1522 else
1523 sysbus_reg |= DWCEQOS_DMA_SYSBUS_MODE_WR_OSR_LIMIT(
1524 DWCEQOS_DMA_SYSBUS_MODE_WR_OSR_LIMIT_DEFAULT);
1525
1526 if (netif_msg_hw(lp))
1527 netdev_dbg(lp->ndev, "SysbusMode %#X\n", sysbus_reg);
1528
1529 dwceqos_write(lp, REG_DWCEQOS_DMA_SYSBUS_MODE, sysbus_reg);
1530}
1531
1532static void dwceqos_init_hw(struct net_local *lp)
1533{
1534 u32 regval;
1535 u32 buswidth;
1536 u32 dma_skip;
1537
1538 /* Software reset */
1539 dwceqos_reset_hw(lp);
1540
1541 dwceqos_configure_bus(lp);
1542
1543 /* Probe data bus width, 32/64/128 bits. */
1544 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_TXDESC_TAIL, 0xF);
1545 regval = dwceqos_read(lp, REG_DWCEQOS_DMA_CH0_TXDESC_TAIL);
1546 buswidth = (regval ^ 0xF) + 1;
1547
1548 /* Cache-align dma descriptors. */
1549 dma_skip = (sizeof(struct dwceqos_dma_desc) - 16) / buswidth;
1550 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_CTRL,
1551 DWCEQOS_DMA_CH_CTRL_DSL(dma_skip) |
1552 DWCEQOS_DMA_CH_CTRL_PBLX8);
1553
1554 /* Initialize DMA Channel 0 */
1555 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_TXDESC_LEN, DWCEQOS_TX_DCNT - 1);
1556 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_RXDESC_LEN, DWCEQOS_RX_DCNT - 1);
1557 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_TXDESC_LIST,
1558 (u32)lp->tx_descs_addr);
1559 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_RXDESC_LIST,
1560 (u32)lp->rx_descs_addr);
1561
1562 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_TXDESC_TAIL,
1563 lp->tx_descs_tail_addr);
1564 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_RXDESC_TAIL,
1565 lp->rx_descs_tail_addr);
1566
1567 if (lp->bus_cfg.tx_pbl)
1568 regval = DWCEQOS_DMA_CH_CTRL_PBL(lp->bus_cfg.tx_pbl);
1569 else
1570 regval = DWCEQOS_DMA_CH_CTRL_PBL(2);
1571
1572 /* Enable TSO if the HW support it */
1573 if (lp->feature1 & DWCEQOS_MAC_HW_FEATURE1_TSOEN)
1574 regval |= DWCEQOS_DMA_CH_TX_TSE;
1575
1576 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_TX_CTRL, regval);
1577
1578 if (lp->bus_cfg.rx_pbl)
1579 regval = DWCEQOS_DMA_CH_CTRL_PBL(lp->bus_cfg.rx_pbl);
1580 else
1581 regval = DWCEQOS_DMA_CH_CTRL_PBL(2);
1582
1583 regval |= DWCEQOS_DMA_CH_RX_CTRL_BUFSIZE(DWCEQOS_DWCEQOS_RX_BUF_SIZE);
1584 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_RX_CTRL, regval);
1585
1586 regval |= DWCEQOS_DMA_CH_CTRL_START;
1587 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_RX_CTRL, regval);
1588
1589 /* Initialize MTL Queues */
1590 regval = DWCEQOS_MTL_SCHALG_STRICT;
1591 dwceqos_write(lp, REG_DWCEQOS_MTL_OPER, regval);
1592
1593 regval = DWCEQOS_MTL_TXQ_SIZE(
1594 DWCEQOS_MAC_HW_FEATURE1_TXFIFOSIZE(lp->feature1)) |
1595 DWCEQOS_MTL_TXQ_TXQEN | DWCEQOS_MTL_TXQ_TSF |
1596 DWCEQOS_MTL_TXQ_TTC512;
1597 dwceqos_write(lp, REG_DWCEQOS_MTL_TXQ0_OPER, regval);
1598
1599 regval = DWCEQOS_MTL_RXQ_SIZE(
1600 DWCEQOS_MAC_HW_FEATURE1_RXFIFOSIZE(lp->feature1)) |
1601 DWCEQOS_MTL_RXQ_FUP | DWCEQOS_MTL_RXQ_FEP | DWCEQOS_MTL_RXQ_RSF;
1602 dwceqos_write(lp, REG_DWCEQOS_MTL_RXQ0_OPER, regval);
1603
1604 dwceqos_configure_flow_control(lp);
1605
1606 /* Initialize MAC */
1607 dwceqos_set_umac_addr(lp, lp->ndev->dev_addr, 0);
1608
1609 lp->eee_enabled = 0;
1610
1611 dwceqos_configure_clock(lp);
1612
1613 /* MMC counters */
1614
1615 /* probe implemented counters */
1616 dwceqos_write(lp, REG_DWCEQOS_MMC_RXIRQMASK, ~0u);
1617 dwceqos_write(lp, REG_DWCEQOS_MMC_TXIRQMASK, ~0u);
1618 lp->mmc_rx_counters_mask = dwceqos_read(lp, REG_DWCEQOS_MMC_RXIRQMASK);
1619 lp->mmc_tx_counters_mask = dwceqos_read(lp, REG_DWCEQOS_MMC_TXIRQMASK);
1620
1621 dwceqos_write(lp, REG_DWCEQOS_MMC_CTRL, DWCEQOS_MMC_CTRL_CNTRST |
1622 DWCEQOS_MMC_CTRL_RSTONRD);
1623 dwceqos_enable_mmc_interrupt(lp);
1624
1625 /* Enable Interrupts */
1626 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_IE,
1627 DWCEQOS_DMA_CH0_IE_NIE |
1628 DWCEQOS_DMA_CH0_IE_RIE | DWCEQOS_DMA_CH0_IE_TIE |
1629 DWCEQOS_DMA_CH0_IE_AIE |
1630 DWCEQOS_DMA_CH0_IE_FBEE);
1631
1632 dwceqos_write(lp, REG_DWCEQOS_MAC_IE, 0);
1633
1634 dwceqos_write(lp, REG_DWCEQOS_MAC_CFG, DWCEQOS_MAC_CFG_IPC |
1635 DWCEQOS_MAC_CFG_DM | DWCEQOS_MAC_CFG_TE | DWCEQOS_MAC_CFG_RE);
1636
1637 /* Start TX DMA */
1638 regval = dwceqos_read(lp, REG_DWCEQOS_DMA_CH0_TX_CTRL);
1639 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_TX_CTRL,
1640 regval | DWCEQOS_DMA_CH_CTRL_START);
1641
1642 /* Enable MAC TX/RX */
1643 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_CFG);
1644 dwceqos_write(lp, REG_DWCEQOS_MAC_CFG,
1645 regval | DWCEQOS_MAC_CFG_TE | DWCEQOS_MAC_CFG_RE);
1646
1647 lp->phy_defer = false;
1648 mutex_lock(&lp->phy_dev->lock);
1649 phy_read_status(lp->phy_dev);
1650 dwceqos_adjust_link(lp->ndev);
1651 mutex_unlock(&lp->phy_dev->lock);
1652}
1653
1654static void dwceqos_tx_reclaim(unsigned long data)
1655{
1656 struct net_device *ndev = (struct net_device *)data;
1657 struct net_local *lp = netdev_priv(ndev);
1658 unsigned int tx_bytes = 0;
1659 unsigned int tx_packets = 0;
1660
1661 spin_lock(&lp->tx_lock);
1662
1663 while (lp->tx_free < DWCEQOS_TX_DCNT) {
1664 struct dwceqos_dma_desc *dd = &lp->tx_descs[lp->tx_cur];
1665 struct ring_desc *rd = &lp->tx_skb[lp->tx_cur];
1666
1667 /* Descriptor still being held by DMA ? */
1668 if (dd->des3 & DWCEQOS_DMA_TDES3_OWN)
1669 break;
1670
1671 if (rd->mapping)
1672 dma_unmap_single(ndev->dev.parent, rd->mapping, rd->len,
1673 DMA_TO_DEVICE);
1674
1675 if (unlikely(rd->skb)) {
1676 ++tx_packets;
1677 tx_bytes += rd->skb->len;
1678 dev_consume_skb_any(rd->skb);
1679 }
1680
1681 rd->skb = NULL;
1682 rd->mapping = 0;
1683 lp->tx_free++;
1684 lp->tx_cur = (lp->tx_cur + 1) % DWCEQOS_TX_DCNT;
1685
1686 if ((dd->des3 & DWCEQOS_DMA_TDES3_LD) &&
1687 (dd->des3 & DWCEQOS_DMA_RDES3_ES)) {
1688 if (netif_msg_tx_err(lp))
1689 netdev_err(ndev, "TX Error, TDES3 = 0x%x\n",
1690 dd->des3);
1691 if (netif_msg_hw(lp))
1692 print_status(lp);
1693 }
1694 }
1695 spin_unlock(&lp->tx_lock);
1696
1697 netdev_completed_queue(ndev, tx_packets, tx_bytes);
1698
1699 dwceqos_dma_enable_txirq(lp);
1700 netif_wake_queue(ndev);
1701}
1702
1703static int dwceqos_rx(struct net_local *lp, int budget)
1704{
1705 struct sk_buff *skb;
1706 u32 tot_size = 0;
1707 unsigned int n_packets = 0;
1708 unsigned int n_descs = 0;
1709 u32 len;
1710
1711 struct dwceqos_dma_desc *dd;
1712 struct sk_buff *new_skb;
1713 dma_addr_t new_skb_baddr = 0;
1714
1715 while (n_descs < budget) {
1716 if (!dwceqos_packet_avail(lp))
1717 break;
1718
1719 new_skb = netdev_alloc_skb(lp->ndev, DWCEQOS_RX_BUF_SIZE);
1720 if (!new_skb) {
1721 netdev_err(lp->ndev, "no memory for new sk_buff\n");
1722 break;
1723 }
1724
1725 /* Get dma handle of skb->data */
1726 new_skb_baddr = (u32)dma_map_single(lp->ndev->dev.parent,
1727 new_skb->data,
1728 DWCEQOS_RX_BUF_SIZE,
1729 DMA_FROM_DEVICE);
1730 if (dma_mapping_error(lp->ndev->dev.parent, new_skb_baddr)) {
1731 netdev_err(lp->ndev, "DMA map error\n");
1732 dev_kfree_skb(new_skb);
1733 break;
1734 }
1735
1736 /* Read descriptor data after reading owner bit. */
1737 dma_rmb();
1738
1739 dd = &lp->rx_descs[lp->rx_cur];
1740 len = DWCEQOS_DMA_RDES3_PL(dd->des3);
1741 skb = lp->rx_skb[lp->rx_cur].skb;
1742
1743 /* Unmap old buffer */
1744 dma_unmap_single(lp->ndev->dev.parent,
1745 lp->rx_skb[lp->rx_cur].mapping,
1746 lp->rx_skb[lp->rx_cur].len, DMA_FROM_DEVICE);
1747
1748 /* Discard packet on reception error or bad checksum */
1749 if ((dd->des3 & DWCEQOS_DMA_RDES3_ES) ||
1750 (dd->des1 & DWCEQOS_DMA_RDES1_IPCE)) {
1751 dev_kfree_skb(skb);
1752 skb = NULL;
1753 } else {
1754 skb_put(skb, len);
1755 skb->protocol = eth_type_trans(skb, lp->ndev);
1756 switch (dd->des1 & DWCEQOS_DMA_RDES1_PT) {
1757 case DWCEQOS_DMA_RDES1_PT_UDP:
1758 case DWCEQOS_DMA_RDES1_PT_TCP:
1759 case DWCEQOS_DMA_RDES1_PT_ICMP:
1760 skb->ip_summed = CHECKSUM_UNNECESSARY;
1761 break;
1762 default:
1763 skb->ip_summed = CHECKSUM_NONE;
1764 break;
1765 }
1766 }
1767
1768 if (unlikely(!skb)) {
1769 if (netif_msg_rx_err(lp))
1770 netdev_dbg(lp->ndev, "rx error: des3=%X\n",
1771 lp->rx_descs[lp->rx_cur].des3);
1772 } else {
1773 tot_size += skb->len;
1774 n_packets++;
1775
1776 netif_receive_skb(skb);
1777 }
1778
1779 lp->rx_descs[lp->rx_cur].des0 = new_skb_baddr;
1780 lp->rx_descs[lp->rx_cur].des1 = 0;
1781 lp->rx_descs[lp->rx_cur].des2 = 0;
1782 /* The DMA must observe des0/1/2 written before des3. */
1783 wmb();
1784 lp->rx_descs[lp->rx_cur].des3 = DWCEQOS_DMA_RDES3_INTE |
1785 DWCEQOS_DMA_RDES3_OWN |
1786 DWCEQOS_DMA_RDES3_BUF1V;
1787
1788 lp->rx_skb[lp->rx_cur].mapping = new_skb_baddr;
1789 lp->rx_skb[lp->rx_cur].len = DWCEQOS_RX_BUF_SIZE;
1790 lp->rx_skb[lp->rx_cur].skb = new_skb;
1791
1792 n_descs++;
1793 lp->rx_cur = (lp->rx_cur + 1) % DWCEQOS_RX_DCNT;
1794 }
1795
1796 /* Make sure any ownership update is written to the descriptors before
1797 * DMA wakeup.
1798 */
1799 wmb();
1800
1801 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_STA, DWCEQOS_DMA_CH0_IS_RI);
1802 /* Wake up RX by writing tail pointer */
1803 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_RXDESC_TAIL,
1804 lp->rx_descs_tail_addr);
1805
1806 return n_descs;
1807}
1808
1809static int dwceqos_rx_poll(struct napi_struct *napi, int budget)
1810{
1811 struct net_local *lp = container_of(napi, struct net_local, napi);
1812 int work_done = 0;
1813
1814 work_done = dwceqos_rx(lp, budget - work_done);
1815
1816 if (!dwceqos_packet_avail(lp) && work_done < budget) {
1817 napi_complete(napi);
1818 dwceqos_dma_enable_rxirq(lp);
1819 } else {
1820 work_done = budget;
1821 }
1822
1823 return work_done;
1824}
1825
1826/* Reinitialize function if a TX timed out */
1827static void dwceqos_reinit_for_txtimeout(struct work_struct *data)
1828{
1829 struct net_local *lp = container_of(data, struct net_local,
1830 txtimeout_reinit);
1831
1832 netdev_err(lp->ndev, "transmit timeout %d s, resetting...\n",
1833 DWCEQOS_TX_TIMEOUT);
1834
1835 if (netif_msg_hw(lp))
1836 print_status(lp);
1837
1838 rtnl_lock();
1839 dwceqos_stop(lp->ndev);
1840 dwceqos_open(lp->ndev);
1841 rtnl_unlock();
1842}
1843
1844/* DT Probing function called by main probe */
1845static inline int dwceqos_probe_config_dt(struct platform_device *pdev)
1846{
1847 struct net_device *ndev;
1848 struct net_local *lp;
1849 const void *mac_address;
1850 struct dwceqos_bus_cfg *bus_cfg;
1851 struct device_node *np = pdev->dev.of_node;
1852
1853 ndev = platform_get_drvdata(pdev);
1854 lp = netdev_priv(ndev);
1855 bus_cfg = &lp->bus_cfg;
1856
1857 /* Set the MAC address. */
1858 mac_address = of_get_mac_address(pdev->dev.of_node);
1859 if (mac_address)
1860 ether_addr_copy(ndev->dev_addr, mac_address);
1861
1862 /* These are all optional parameters */
1863 lp->en_tx_lpi_clockgating = of_property_read_bool(np,
1864 "snps,en-tx-lpi-clockgating");
1865 bus_cfg->en_lpi = of_property_read_bool(np, "snps,en-lpi");
1866 of_property_read_u32(np, "snps,write-requests",
1867 &bus_cfg->write_requests);
1868 of_property_read_u32(np, "snps,read-requests", &bus_cfg->read_requests);
1869 of_property_read_u32(np, "snps,burst-map", &bus_cfg->burst_map);
1870 of_property_read_u32(np, "snps,txpbl", &bus_cfg->tx_pbl);
1871 of_property_read_u32(np, "snps,rxpbl", &bus_cfg->rx_pbl);
1872
1873 netdev_dbg(ndev, "BusCfg: lpi:%u wr:%u rr:%u bm:%X rxpbl:%u txpbl:%d\n",
1874 bus_cfg->en_lpi,
1875 bus_cfg->write_requests,
1876 bus_cfg->read_requests,
1877 bus_cfg->burst_map,
1878 bus_cfg->rx_pbl,
1879 bus_cfg->tx_pbl);
1880
1881 return 0;
1882}
1883
1884static int dwceqos_open(struct net_device *ndev)
1885{
1886 struct net_local *lp = netdev_priv(ndev);
1887 int res;
1888
1889 dwceqos_reset_state(lp);
1890 res = dwceqos_descriptor_init(lp);
1891 if (res) {
1892 netdev_err(ndev, "Unable to allocate DMA memory, rc %d\n", res);
1893 return res;
1894 }
1895 netdev_reset_queue(ndev);
1896
1897 /* The dwceqos reset state machine requires all phy clocks to complete,
1898 * hence the unusual init order with phy_start first.
1899 */
1900 lp->phy_defer = true;
1901 phy_start(lp->phy_dev);
1902 dwceqos_init_hw(lp);
1903 napi_enable(&lp->napi);
1904
1905 netif_start_queue(ndev);
1906 tasklet_enable(&lp->tx_bdreclaim_tasklet);
1907
1908 return 0;
1909}
1910
1911static bool dweqos_is_tx_dma_suspended(struct net_local *lp)
1912{
1913 u32 reg;
1914
1915 reg = dwceqos_read(lp, REG_DWCEQOS_DMA_DEBUG_ST0);
1916 reg = DMA_GET_TX_STATE_CH0(reg);
1917
1918 return reg == DMA_TX_CH_SUSPENDED;
1919}
1920
1921static void dwceqos_drain_dma(struct net_local *lp)
1922{
1923 /* Wait for all pending TX buffers to be sent. Upper limit based
1924 * on max frame size on a 10 Mbit link.
1925 */
1926 size_t limit = (DWCEQOS_TX_DCNT * 1250) / 100;
1927
1928 while (!dweqos_is_tx_dma_suspended(lp) && limit--)
1929 usleep_range(100, 200);
1930}
1931
1932static int dwceqos_stop(struct net_device *ndev)
1933{
1934 struct net_local *lp = netdev_priv(ndev);
1935
1936 tasklet_disable(&lp->tx_bdreclaim_tasklet);
1937 napi_disable(&lp->napi);
1938
1939 /* Stop all tx before we drain the tx dma. */
1940 netif_tx_lock_bh(lp->ndev);
1941 netif_stop_queue(ndev);
1942 netif_tx_unlock_bh(lp->ndev);
1943
1944 dwceqos_drain_dma(lp);
1945 dwceqos_reset_hw(lp);
1946 phy_stop(lp->phy_dev);
1947
1948 dwceqos_descriptor_free(lp);
1949
1950 return 0;
1951}
1952
1953static void dwceqos_dmadesc_set_ctx(struct net_local *lp,
1954 unsigned short gso_size)
1955{
1956 struct dwceqos_dma_desc *dd = &lp->tx_descs[lp->tx_next];
1957
1958 dd->des0 = 0;
1959 dd->des1 = 0;
1960 dd->des2 = gso_size;
1961 dd->des3 = DWCEQOS_DMA_TDES3_CTXT | DWCEQOS_DMA_TDES3_TCMSSV;
1962
1963 lp->tx_next = (lp->tx_next + 1) % DWCEQOS_TX_DCNT;
1964}
1965
1966static void dwceqos_tx_poll_demand(struct net_local *lp)
1967{
1968 dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_TXDESC_TAIL,
1969 lp->tx_descs_tail_addr);
1970}
1971
1972struct dwceqos_tx {
1973 size_t nr_descriptors;
1974 size_t initial_descriptor;
1975 size_t last_descriptor;
1976 size_t prev_gso_size;
1977 size_t network_header_len;
1978};
1979
1980static void dwceqos_tx_prepare(struct sk_buff *skb, struct net_local *lp,
1981 struct dwceqos_tx *tx)
1982{
1983 size_t n = 1;
1984 size_t i;
1985
1986 if (skb_is_gso(skb) && skb_shinfo(skb)->gso_size != lp->gso_size)
1987 ++n;
1988
1989 for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) {
1990 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1991
1992 n += (skb_frag_size(frag) + BYTES_PER_DMA_DESC - 1) /
1993 BYTES_PER_DMA_DESC;
1994 }
1995
1996 tx->nr_descriptors = n;
1997 tx->initial_descriptor = lp->tx_next;
1998 tx->last_descriptor = lp->tx_next;
1999 tx->prev_gso_size = lp->gso_size;
2000
2001 tx->network_header_len = skb_transport_offset(skb);
2002 if (skb_is_gso(skb))
2003 tx->network_header_len += tcp_hdrlen(skb);
2004}
2005
2006static int dwceqos_tx_linear(struct sk_buff *skb, struct net_local *lp,
2007 struct dwceqos_tx *tx)
2008{
2009 struct ring_desc *rd;
2010 struct dwceqos_dma_desc *dd;
2011 size_t payload_len;
2012 dma_addr_t dma_handle;
2013
2014 if (skb_is_gso(skb) && skb_shinfo(skb)->gso_size != lp->gso_size) {
2015 dwceqos_dmadesc_set_ctx(lp, skb_shinfo(skb)->gso_size);
2016 lp->gso_size = skb_shinfo(skb)->gso_size;
2017 }
2018
2019 dma_handle = dma_map_single(lp->ndev->dev.parent, skb->data,
2020 skb_headlen(skb), DMA_TO_DEVICE);
2021
2022 if (dma_mapping_error(lp->ndev->dev.parent, dma_handle)) {
2023 netdev_err(lp->ndev, "TX DMA Mapping error\n");
2024 return -ENOMEM;
2025 }
2026
2027 rd = &lp->tx_skb[lp->tx_next];
2028 dd = &lp->tx_descs[lp->tx_next];
2029
2030 rd->skb = NULL;
2031 rd->len = skb_headlen(skb);
2032 rd->mapping = dma_handle;
2033
2034 /* Set up DMA Descriptor */
2035 dd->des0 = dma_handle;
2036
2037 if (skb_is_gso(skb)) {
2038 payload_len = skb_headlen(skb) - tx->network_header_len;
2039
2040 if (payload_len)
2041 dd->des1 = dma_handle + tx->network_header_len;
2042 dd->des2 = tx->network_header_len |
2043 DWCEQOS_DMA_DES2_B2L(payload_len);
2044 dd->des3 = DWCEQOS_DMA_TDES3_TSE |
2045 DWCEQOS_DMA_DES3_THL((tcp_hdrlen(skb) / 4)) |
2046 (skb->len - tx->network_header_len);
2047 } else {
2048 dd->des1 = 0;
2049 dd->des2 = skb_headlen(skb);
2050 dd->des3 = skb->len;
2051
2052 switch (skb->ip_summed) {
2053 case CHECKSUM_PARTIAL:
2054 dd->des3 |= DWCEQOS_DMA_TDES3_CA;
2055 case CHECKSUM_NONE:
2056 case CHECKSUM_UNNECESSARY:
2057 case CHECKSUM_COMPLETE:
2058 default:
2059 break;
2060 }
2061 }
2062
2063 dd->des3 |= DWCEQOS_DMA_TDES3_FD;
2064 if (lp->tx_next != tx->initial_descriptor)
2065 dd->des3 |= DWCEQOS_DMA_TDES3_OWN;
2066
2067 tx->last_descriptor = lp->tx_next;
2068 lp->tx_next = (lp->tx_next + 1) % DWCEQOS_TX_DCNT;
2069
2070 return 0;
2071}
2072
2073static int dwceqos_tx_frags(struct sk_buff *skb, struct net_local *lp,
2074 struct dwceqos_tx *tx)
2075{
2076 struct ring_desc *rd = NULL;
2077 struct dwceqos_dma_desc *dd;
2078 dma_addr_t dma_handle;
2079 size_t i;
2080
2081 /* Setup more ring and DMA descriptor if the packet is fragmented */
2082 for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) {
2083 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2084 size_t frag_size;
2085 size_t consumed_size;
2086
2087 /* Map DMA Area */
2088 dma_handle = skb_frag_dma_map(lp->ndev->dev.parent, frag, 0,
2089 skb_frag_size(frag),
2090 DMA_TO_DEVICE);
2091 if (dma_mapping_error(lp->ndev->dev.parent, dma_handle)) {
2092 netdev_err(lp->ndev, "DMA Mapping error\n");
2093 return -ENOMEM;
2094 }
2095
2096 /* order-3 fragments span more than one descriptor. */
2097 frag_size = skb_frag_size(frag);
2098 consumed_size = 0;
2099 while (consumed_size < frag_size) {
2100 size_t dma_size = min_t(size_t, 16376,
2101 frag_size - consumed_size);
2102
2103 rd = &lp->tx_skb[lp->tx_next];
2104 memset(rd, 0, sizeof(*rd));
2105
2106 dd = &lp->tx_descs[lp->tx_next];
2107
2108 /* Set DMA Descriptor fields */
2109 dd->des0 = dma_handle + consumed_size;
2110 dd->des1 = 0;
2111 dd->des2 = dma_size;
2112
2113 if (skb_is_gso(skb))
2114 dd->des3 = (skb->len - tx->network_header_len);
2115 else
2116 dd->des3 = skb->len;
2117
2118 dd->des3 |= DWCEQOS_DMA_TDES3_OWN;
2119
2120 tx->last_descriptor = lp->tx_next;
2121 lp->tx_next = (lp->tx_next + 1) % DWCEQOS_TX_DCNT;
2122 consumed_size += dma_size;
2123 }
2124
2125 rd->len = skb_frag_size(frag);
2126 rd->mapping = dma_handle;
2127 }
2128
2129 return 0;
2130}
2131
2132static void dwceqos_tx_finalize(struct sk_buff *skb, struct net_local *lp,
2133 struct dwceqos_tx *tx)
2134{
2135 lp->tx_descs[tx->last_descriptor].des3 |= DWCEQOS_DMA_TDES3_LD;
2136 lp->tx_descs[tx->last_descriptor].des2 |= DWCEQOS_DMA_TDES2_IOC;
2137
2138 lp->tx_skb[tx->last_descriptor].skb = skb;
2139
2140 /* Make all descriptor updates visible to the DMA before setting the
2141 * owner bit.
2142 */
2143 wmb();
2144
2145 lp->tx_descs[tx->initial_descriptor].des3 |= DWCEQOS_DMA_TDES3_OWN;
2146
2147 /* Make the owner bit visible before TX wakeup. */
2148 wmb();
2149
2150 dwceqos_tx_poll_demand(lp);
2151}
2152
2153static void dwceqos_tx_rollback(struct net_local *lp, struct dwceqos_tx *tx)
2154{
2155 size_t i = tx->initial_descriptor;
2156
2157 while (i != lp->tx_next) {
2158 if (lp->tx_skb[i].mapping)
2159 dma_unmap_single(lp->ndev->dev.parent,
2160 lp->tx_skb[i].mapping,
2161 lp->tx_skb[i].len,
2162 DMA_TO_DEVICE);
2163
2164 lp->tx_skb[i].mapping = 0;
2165 lp->tx_skb[i].skb = NULL;
2166
2167 memset(&lp->tx_descs[i], 0, sizeof(lp->tx_descs[i]));
2168
2169 i = (i + 1) % DWCEQOS_TX_DCNT;
2170 }
2171
2172 lp->tx_next = tx->initial_descriptor;
2173 lp->gso_size = tx->prev_gso_size;
2174}
2175
2176static int dwceqos_start_xmit(struct sk_buff *skb, struct net_device *ndev)
2177{
2178 struct net_local *lp = netdev_priv(ndev);
2179 struct dwceqos_tx trans;
2180 int err;
2181
2182 dwceqos_tx_prepare(skb, lp, &trans);
2183 if (lp->tx_free < trans.nr_descriptors) {
2184 netif_stop_queue(ndev);
2185 return NETDEV_TX_BUSY;
2186 }
2187
2188 err = dwceqos_tx_linear(skb, lp, &trans);
2189 if (err)
2190 goto tx_error;
2191
2192 err = dwceqos_tx_frags(skb, lp, &trans);
2193 if (err)
2194 goto tx_error;
2195
2196 WARN_ON(lp->tx_next !=
2197 ((trans.initial_descriptor + trans.nr_descriptors) %
2198 DWCEQOS_TX_DCNT));
2199
2200 spin_lock_bh(&lp->tx_lock);
2201 lp->tx_free -= trans.nr_descriptors;
2202 dwceqos_tx_finalize(skb, lp, &trans);
2203 netdev_sent_queue(ndev, skb->len);
2204 spin_unlock_bh(&lp->tx_lock);
2205
2206 ndev->trans_start = jiffies;
2207 return 0;
2208
2209tx_error:
2210 dwceqos_tx_rollback(lp, &trans);
2211 dev_kfree_skb(skb);
2212 return 0;
2213}
2214
2215/* Set MAC address and then update HW accordingly */
2216static int dwceqos_set_mac_address(struct net_device *ndev, void *addr)
2217{
2218 struct net_local *lp = netdev_priv(ndev);
2219 struct sockaddr *hwaddr = (struct sockaddr *)addr;
2220
2221 if (netif_running(ndev))
2222 return -EBUSY;
2223
2224 if (!is_valid_ether_addr(hwaddr->sa_data))
2225 return -EADDRNOTAVAIL;
2226
2227 memcpy(ndev->dev_addr, hwaddr->sa_data, ndev->addr_len);
2228
2229 dwceqos_set_umac_addr(lp, lp->ndev->dev_addr, 0);
2230 return 0;
2231}
2232
2233static void dwceqos_tx_timeout(struct net_device *ndev)
2234{
2235 struct net_local *lp = netdev_priv(ndev);
2236
2237 queue_work(lp->txtimeout_handler_wq, &lp->txtimeout_reinit);
2238}
2239
2240static void dwceqos_set_umac_addr(struct net_local *lp, unsigned char *addr,
2241 unsigned int reg_n)
2242{
2243 unsigned long data;
2244
2245 data = (addr[5] << 8) | addr[4];
2246 dwceqos_write(lp, DWCEQOS_ADDR_HIGH(reg_n),
2247 data | DWCEQOS_MAC_MAC_ADDR_HI_EN);
2248 data = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0];
2249 dwceqos_write(lp, DWCEQOS_ADDR_LOW(reg_n), data);
2250}
2251
2252static void dwceqos_disable_umac_addr(struct net_local *lp, unsigned int reg_n)
2253{
2254 /* Do not disable MAC address 0 */
2255 if (reg_n != 0)
2256 dwceqos_write(lp, DWCEQOS_ADDR_HIGH(reg_n), 0);
2257}
2258
2259static void dwceqos_set_rx_mode(struct net_device *ndev)
2260{
2261 struct net_local *lp = netdev_priv(ndev);
2262 u32 regval = 0;
2263 u32 mc_filter[2];
2264 int reg = 1;
2265 struct netdev_hw_addr *ha;
2266 unsigned int max_mac_addr;
2267
2268 max_mac_addr = DWCEQOS_MAX_PERFECT_ADDRESSES(lp->feature1);
2269
2270 if (ndev->flags & IFF_PROMISC) {
2271 regval = DWCEQOS_MAC_PKT_FILT_PR;
2272 } else if (((netdev_mc_count(ndev) > DWCEQOS_HASH_TABLE_SIZE) ||
2273 (ndev->flags & IFF_ALLMULTI))) {
2274 regval = DWCEQOS_MAC_PKT_FILT_PM;
2275 dwceqos_write(lp, REG_DWCEQOS_HASTABLE_LO, 0xffffffff);
2276 dwceqos_write(lp, REG_DWCEQOS_HASTABLE_HI, 0xffffffff);
2277 } else if (!netdev_mc_empty(ndev)) {
2278 regval = DWCEQOS_MAC_PKT_FILT_HMC;
2279 memset(mc_filter, 0, sizeof(mc_filter));
2280 netdev_for_each_mc_addr(ha, ndev) {
2281 /* The upper 6 bits of the calculated CRC are used to
2282 * index the contens of the hash table
2283 */
2284 int bit_nr = bitrev32(~crc32_le(~0, ha->addr, 6)) >> 26;
2285 /* The most significant bit determines the register
2286 * to use (H/L) while the other 5 bits determine
2287 * the bit within the register.
2288 */
2289 mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31);
2290 }
2291 dwceqos_write(lp, REG_DWCEQOS_HASTABLE_LO, mc_filter[0]);
2292 dwceqos_write(lp, REG_DWCEQOS_HASTABLE_HI, mc_filter[1]);
2293 }
2294 if (netdev_uc_count(ndev) > max_mac_addr) {
2295 regval |= DWCEQOS_MAC_PKT_FILT_PR;
2296 } else {
2297 netdev_for_each_uc_addr(ha, ndev) {
2298 dwceqos_set_umac_addr(lp, ha->addr, reg);
2299 reg++;
2300 }
2301 for (; reg < DWCEQOS_MAX_PERFECT_ADDRESSES(lp->feature1); reg++)
2302 dwceqos_disable_umac_addr(lp, reg);
2303 }
2304 dwceqos_write(lp, REG_DWCEQOS_MAC_PKT_FILT, regval);
2305}
2306
2307#ifdef CONFIG_NET_POLL_CONTROLLER
2308static void dwceqos_poll_controller(struct net_device *ndev)
2309{
2310 disable_irq(ndev->irq);
2311 dwceqos_interrupt(ndev->irq, ndev);
2312 enable_irq(ndev->irq);
2313}
2314#endif
2315
2316static void dwceqos_read_mmc_counters(struct net_local *lp, u32 rx_mask,
2317 u32 tx_mask)
2318{
2319 if (tx_mask & BIT(27))
2320 lp->mmc_counters.txlpitranscntr +=
2321 dwceqos_read(lp, DWC_MMC_TXLPITRANSCNTR);
2322 if (tx_mask & BIT(26))
2323 lp->mmc_counters.txpiuscntr +=
2324 dwceqos_read(lp, DWC_MMC_TXLPIUSCNTR);
2325 if (tx_mask & BIT(25))
2326 lp->mmc_counters.txoversize_g +=
2327 dwceqos_read(lp, DWC_MMC_TXOVERSIZE_G);
2328 if (tx_mask & BIT(24))
2329 lp->mmc_counters.txvlanpackets_g +=
2330 dwceqos_read(lp, DWC_MMC_TXVLANPACKETS_G);
2331 if (tx_mask & BIT(23))
2332 lp->mmc_counters.txpausepackets +=
2333 dwceqos_read(lp, DWC_MMC_TXPAUSEPACKETS);
2334 if (tx_mask & BIT(22))
2335 lp->mmc_counters.txexcessdef +=
2336 dwceqos_read(lp, DWC_MMC_TXEXCESSDEF);
2337 if (tx_mask & BIT(21))
2338 lp->mmc_counters.txpacketcount_g +=
2339 dwceqos_read(lp, DWC_MMC_TXPACKETCOUNT_G);
2340 if (tx_mask & BIT(20))
2341 lp->mmc_counters.txoctetcount_g +=
2342 dwceqos_read(lp, DWC_MMC_TXOCTETCOUNT_G);
2343 if (tx_mask & BIT(19))
2344 lp->mmc_counters.txcarriererror +=
2345 dwceqos_read(lp, DWC_MMC_TXCARRIERERROR);
2346 if (tx_mask & BIT(18))
2347 lp->mmc_counters.txexcesscol +=
2348 dwceqos_read(lp, DWC_MMC_TXEXCESSCOL);
2349 if (tx_mask & BIT(17))
2350 lp->mmc_counters.txlatecol +=
2351 dwceqos_read(lp, DWC_MMC_TXLATECOL);
2352 if (tx_mask & BIT(16))
2353 lp->mmc_counters.txdeferred +=
2354 dwceqos_read(lp, DWC_MMC_TXDEFERRED);
2355 if (tx_mask & BIT(15))
2356 lp->mmc_counters.txmulticol_g +=
2357 dwceqos_read(lp, DWC_MMC_TXMULTICOL_G);
2358 if (tx_mask & BIT(14))
2359 lp->mmc_counters.txsinglecol_g +=
2360 dwceqos_read(lp, DWC_MMC_TXSINGLECOL_G);
2361 if (tx_mask & BIT(13))
2362 lp->mmc_counters.txunderflowerror +=
2363 dwceqos_read(lp, DWC_MMC_TXUNDERFLOWERROR);
2364 if (tx_mask & BIT(12))
2365 lp->mmc_counters.txbroadcastpackets_gb +=
2366 dwceqos_read(lp, DWC_MMC_TXBROADCASTPACKETS_GB);
2367 if (tx_mask & BIT(11))
2368 lp->mmc_counters.txmulticastpackets_gb +=
2369 dwceqos_read(lp, DWC_MMC_TXMULTICASTPACKETS_GB);
2370 if (tx_mask & BIT(10))
2371 lp->mmc_counters.txunicastpackets_gb +=
2372 dwceqos_read(lp, DWC_MMC_TXUNICASTPACKETS_GB);
2373 if (tx_mask & BIT(9))
2374 lp->mmc_counters.tx1024tomaxoctets_gb +=
2375 dwceqos_read(lp, DWC_MMC_TX1024TOMAXOCTETS_GB);
2376 if (tx_mask & BIT(8))
2377 lp->mmc_counters.tx512to1023octets_gb +=
2378 dwceqos_read(lp, DWC_MMC_TX512TO1023OCTETS_GB);
2379 if (tx_mask & BIT(7))
2380 lp->mmc_counters.tx256to511octets_gb +=
2381 dwceqos_read(lp, DWC_MMC_TX256TO511OCTETS_GB);
2382 if (tx_mask & BIT(6))
2383 lp->mmc_counters.tx128to255octets_gb +=
2384 dwceqos_read(lp, DWC_MMC_TX128TO255OCTETS_GB);
2385 if (tx_mask & BIT(5))
2386 lp->mmc_counters.tx65to127octets_gb +=
2387 dwceqos_read(lp, DWC_MMC_TX65TO127OCTETS_GB);
2388 if (tx_mask & BIT(4))
2389 lp->mmc_counters.tx64octets_gb +=
2390 dwceqos_read(lp, DWC_MMC_TX64OCTETS_GB);
2391 if (tx_mask & BIT(3))
2392 lp->mmc_counters.txmulticastpackets_g +=
2393 dwceqos_read(lp, DWC_MMC_TXMULTICASTPACKETS_G);
2394 if (tx_mask & BIT(2))
2395 lp->mmc_counters.txbroadcastpackets_g +=
2396 dwceqos_read(lp, DWC_MMC_TXBROADCASTPACKETS_G);
2397 if (tx_mask & BIT(1))
2398 lp->mmc_counters.txpacketcount_gb +=
2399 dwceqos_read(lp, DWC_MMC_TXPACKETCOUNT_GB);
2400 if (tx_mask & BIT(0))
2401 lp->mmc_counters.txoctetcount_gb +=
2402 dwceqos_read(lp, DWC_MMC_TXOCTETCOUNT_GB);
2403
2404 if (rx_mask & BIT(27))
2405 lp->mmc_counters.rxlpitranscntr +=
2406 dwceqos_read(lp, DWC_MMC_RXLPITRANSCNTR);
2407 if (rx_mask & BIT(26))
2408 lp->mmc_counters.rxlpiuscntr +=
2409 dwceqos_read(lp, DWC_MMC_RXLPIUSCNTR);
2410 if (rx_mask & BIT(25))
2411 lp->mmc_counters.rxctrlpackets_g +=
2412 dwceqos_read(lp, DWC_MMC_RXCTRLPACKETS_G);
2413 if (rx_mask & BIT(24))
2414 lp->mmc_counters.rxrcverror +=
2415 dwceqos_read(lp, DWC_MMC_RXRCVERROR);
2416 if (rx_mask & BIT(23))
2417 lp->mmc_counters.rxwatchdog +=
2418 dwceqos_read(lp, DWC_MMC_RXWATCHDOG);
2419 if (rx_mask & BIT(22))
2420 lp->mmc_counters.rxvlanpackets_gb +=
2421 dwceqos_read(lp, DWC_MMC_RXVLANPACKETS_GB);
2422 if (rx_mask & BIT(21))
2423 lp->mmc_counters.rxfifooverflow +=
2424 dwceqos_read(lp, DWC_MMC_RXFIFOOVERFLOW);
2425 if (rx_mask & BIT(20))
2426 lp->mmc_counters.rxpausepackets +=
2427 dwceqos_read(lp, DWC_MMC_RXPAUSEPACKETS);
2428 if (rx_mask & BIT(19))
2429 lp->mmc_counters.rxoutofrangetype +=
2430 dwceqos_read(lp, DWC_MMC_RXOUTOFRANGETYPE);
2431 if (rx_mask & BIT(18))
2432 lp->mmc_counters.rxlengtherror +=
2433 dwceqos_read(lp, DWC_MMC_RXLENGTHERROR);
2434 if (rx_mask & BIT(17))
2435 lp->mmc_counters.rxunicastpackets_g +=
2436 dwceqos_read(lp, DWC_MMC_RXUNICASTPACKETS_G);
2437 if (rx_mask & BIT(16))
2438 lp->mmc_counters.rx1024tomaxoctets_gb +=
2439 dwceqos_read(lp, DWC_MMC_RX1024TOMAXOCTETS_GB);
2440 if (rx_mask & BIT(15))
2441 lp->mmc_counters.rx512to1023octets_gb +=
2442 dwceqos_read(lp, DWC_MMC_RX512TO1023OCTETS_GB);
2443 if (rx_mask & BIT(14))
2444 lp->mmc_counters.rx256to511octets_gb +=
2445 dwceqos_read(lp, DWC_MMC_RX256TO511OCTETS_GB);
2446 if (rx_mask & BIT(13))
2447 lp->mmc_counters.rx128to255octets_gb +=
2448 dwceqos_read(lp, DWC_MMC_RX128TO255OCTETS_GB);
2449 if (rx_mask & BIT(12))
2450 lp->mmc_counters.rx65to127octets_gb +=
2451 dwceqos_read(lp, DWC_MMC_RX65TO127OCTETS_GB);
2452 if (rx_mask & BIT(11))
2453 lp->mmc_counters.rx64octets_gb +=
2454 dwceqos_read(lp, DWC_MMC_RX64OCTETS_GB);
2455 if (rx_mask & BIT(10))
2456 lp->mmc_counters.rxoversize_g +=
2457 dwceqos_read(lp, DWC_MMC_RXOVERSIZE_G);
2458 if (rx_mask & BIT(9))
2459 lp->mmc_counters.rxundersize_g +=
2460 dwceqos_read(lp, DWC_MMC_RXUNDERSIZE_G);
2461 if (rx_mask & BIT(8))
2462 lp->mmc_counters.rxjabbererror +=
2463 dwceqos_read(lp, DWC_MMC_RXJABBERERROR);
2464 if (rx_mask & BIT(7))
2465 lp->mmc_counters.rxrunterror +=
2466 dwceqos_read(lp, DWC_MMC_RXRUNTERROR);
2467 if (rx_mask & BIT(6))
2468 lp->mmc_counters.rxalignmenterror +=
2469 dwceqos_read(lp, DWC_MMC_RXALIGNMENTERROR);
2470 if (rx_mask & BIT(5))
2471 lp->mmc_counters.rxcrcerror +=
2472 dwceqos_read(lp, DWC_MMC_RXCRCERROR);
2473 if (rx_mask & BIT(4))
2474 lp->mmc_counters.rxmulticastpackets_g +=
2475 dwceqos_read(lp, DWC_MMC_RXMULTICASTPACKETS_G);
2476 if (rx_mask & BIT(3))
2477 lp->mmc_counters.rxbroadcastpackets_g +=
2478 dwceqos_read(lp, DWC_MMC_RXBROADCASTPACKETS_G);
2479 if (rx_mask & BIT(2))
2480 lp->mmc_counters.rxoctetcount_g +=
2481 dwceqos_read(lp, DWC_MMC_RXOCTETCOUNT_G);
2482 if (rx_mask & BIT(1))
2483 lp->mmc_counters.rxoctetcount_gb +=
2484 dwceqos_read(lp, DWC_MMC_RXOCTETCOUNT_GB);
2485 if (rx_mask & BIT(0))
2486 lp->mmc_counters.rxpacketcount_gb +=
2487 dwceqos_read(lp, DWC_MMC_RXPACKETCOUNT_GB);
2488}
2489
2490static struct rtnl_link_stats64*
2491dwceqos_get_stats64(struct net_device *ndev, struct rtnl_link_stats64 *s)
2492{
2493 unsigned long flags;
2494 struct net_local *lp = netdev_priv(ndev);
2495 struct dwceqos_mmc_counters *hwstats = &lp->mmc_counters;
2496
2497 spin_lock_irqsave(&lp->stats_lock, flags);
2498 dwceqos_read_mmc_counters(lp, lp->mmc_rx_counters_mask,
2499 lp->mmc_tx_counters_mask);
2500 spin_unlock_irqrestore(&lp->stats_lock, flags);
2501
2502 s->rx_packets = hwstats->rxpacketcount_gb;
2503 s->rx_bytes = hwstats->rxoctetcount_gb;
2504 s->rx_errors = hwstats->rxpacketcount_gb -
2505 hwstats->rxbroadcastpackets_g -
2506 hwstats->rxmulticastpackets_g -
2507 hwstats->rxunicastpackets_g;
2508 s->multicast = hwstats->rxmulticastpackets_g;
2509 s->rx_length_errors = hwstats->rxlengtherror;
2510 s->rx_crc_errors = hwstats->rxcrcerror;
2511 s->rx_fifo_errors = hwstats->rxfifooverflow;
2512
2513 s->tx_packets = hwstats->txpacketcount_gb;
2514 s->tx_bytes = hwstats->txoctetcount_gb;
2515
2516 if (lp->mmc_tx_counters_mask & BIT(21))
2517 s->tx_errors = hwstats->txpacketcount_gb -
2518 hwstats->txpacketcount_g;
2519 else
2520 s->tx_errors = hwstats->txunderflowerror +
2521 hwstats->txcarriererror;
2522
2523 return s;
2524}
2525
2526static int
2527dwceqos_get_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
2528{
2529 struct net_local *lp = netdev_priv(ndev);
2530 struct phy_device *phydev = lp->phy_dev;
2531
2532 if (!phydev)
2533 return -ENODEV;
2534
2535 return phy_ethtool_gset(phydev, ecmd);
2536}
2537
2538static int
2539dwceqos_set_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
2540{
2541 struct net_local *lp = netdev_priv(ndev);
2542 struct phy_device *phydev = lp->phy_dev;
2543
2544 if (!phydev)
2545 return -ENODEV;
2546
2547 return phy_ethtool_sset(phydev, ecmd);
2548}
2549
2550static void
2551dwceqos_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *ed)
2552{
2553 const struct net_local *lp = netdev_priv(ndev);
2554
2555 strcpy(ed->driver, lp->pdev->dev.driver->name);
2556 strcpy(ed->version, DRIVER_VERSION);
2557}
2558
2559static void dwceqos_get_pauseparam(struct net_device *ndev,
2560 struct ethtool_pauseparam *pp)
2561{
2562 const struct net_local *lp = netdev_priv(ndev);
2563
2564 pp->autoneg = lp->flowcontrol.autoneg;
2565 pp->tx_pause = lp->flowcontrol.tx;
2566 pp->rx_pause = lp->flowcontrol.rx;
2567}
2568
2569static int dwceqos_set_pauseparam(struct net_device *ndev,
2570 struct ethtool_pauseparam *pp)
2571{
2572 struct net_local *lp = netdev_priv(ndev);
2573 int ret = 0;
2574
2575 lp->flowcontrol.autoneg = pp->autoneg;
2576 if (pp->autoneg) {
2577 lp->phy_dev->advertising |= ADVERTISED_Pause;
2578 lp->phy_dev->advertising |= ADVERTISED_Asym_Pause;
2579 } else {
2580 lp->phy_dev->advertising &= ~ADVERTISED_Pause;
2581 lp->phy_dev->advertising &= ~ADVERTISED_Asym_Pause;
2582 lp->flowcontrol.rx = pp->rx_pause;
2583 lp->flowcontrol.tx = pp->tx_pause;
2584 }
2585
2586 if (netif_running(ndev))
2587 ret = phy_start_aneg(lp->phy_dev);
2588
2589 return ret;
2590}
2591
2592static void dwceqos_get_strings(struct net_device *ndev, u32 stringset,
2593 u8 *data)
2594{
2595 size_t i;
2596
2597 if (stringset != ETH_SS_STATS)
2598 return;
2599
2600 for (i = 0; i < ARRAY_SIZE(dwceqos_ethtool_stats); ++i) {
2601 memcpy(data, dwceqos_ethtool_stats[i].stat_name,
2602 ETH_GSTRING_LEN);
2603 data += ETH_GSTRING_LEN;
2604 }
2605}
2606
2607static void dwceqos_get_ethtool_stats(struct net_device *ndev,
2608 struct ethtool_stats *stats, u64 *data)
2609{
2610 struct net_local *lp = netdev_priv(ndev);
2611 unsigned long flags;
2612 size_t i;
2613 u8 *mmcstat = (u8 *)&lp->mmc_counters;
2614
2615 spin_lock_irqsave(&lp->stats_lock, flags);
2616 dwceqos_read_mmc_counters(lp, lp->mmc_rx_counters_mask,
2617 lp->mmc_tx_counters_mask);
2618 spin_unlock_irqrestore(&lp->stats_lock, flags);
2619
2620 for (i = 0; i < ARRAY_SIZE(dwceqos_ethtool_stats); ++i) {
2621 memcpy(data,
2622 mmcstat + dwceqos_ethtool_stats[i].offset,
2623 sizeof(u64));
2624 data++;
2625 }
2626}
2627
2628static int dwceqos_get_sset_count(struct net_device *ndev, int sset)
2629{
2630 if (sset == ETH_SS_STATS)
2631 return ARRAY_SIZE(dwceqos_ethtool_stats);
2632
2633 return -EOPNOTSUPP;
2634}
2635
2636static void dwceqos_get_regs(struct net_device *dev, struct ethtool_regs *regs,
2637 void *space)
2638{
2639 const struct net_local *lp = netdev_priv(dev);
2640 u32 *reg_space = (u32 *)space;
2641 int reg_offset;
2642 int reg_ix = 0;
2643
2644 /* MAC registers */
2645 for (reg_offset = START_MAC_REG_OFFSET;
2646 reg_offset <= MAX_DMA_REG_OFFSET; reg_offset += 4) {
2647 reg_space[reg_ix] = dwceqos_read(lp, reg_offset);
2648 reg_ix++;
2649 }
2650 /* MTL registers */
2651 for (reg_offset = START_MTL_REG_OFFSET;
2652 reg_offset <= MAX_MTL_REG_OFFSET; reg_offset += 4) {
2653 reg_space[reg_ix] = dwceqos_read(lp, reg_offset);
2654 reg_ix++;
2655 }
2656
2657 /* DMA registers */
2658 for (reg_offset = START_DMA_REG_OFFSET;
2659 reg_offset <= MAX_DMA_REG_OFFSET; reg_offset += 4) {
2660 reg_space[reg_ix] = dwceqos_read(lp, reg_offset);
2661 reg_ix++;
2662 }
2663
2664 BUG_ON(4 * reg_ix > REG_SPACE_SIZE);
2665}
2666
2667static int dwceqos_get_regs_len(struct net_device *dev)
2668{
2669 return REG_SPACE_SIZE;
2670}
2671
2672static inline const char *dwceqos_get_rx_lpi_state(u32 lpi_ctrl)
2673{
2674 return (lpi_ctrl & DWCEQOS_MAC_LPI_CTRL_STATUS_RLPIST) ? "on" : "off";
2675}
2676
2677static inline const char *dwceqos_get_tx_lpi_state(u32 lpi_ctrl)
2678{
2679 return (lpi_ctrl & DWCEQOS_MAC_LPI_CTRL_STATUS_TLPIST) ? "on" : "off";
2680}
2681
2682static int dwceqos_get_eee(struct net_device *ndev, struct ethtool_eee *edata)
2683{
2684 struct net_local *lp = netdev_priv(ndev);
2685 u32 lpi_status;
2686 u32 lpi_enabled;
2687
2688 if (!(lp->feature0 & DWCEQOS_MAC_HW_FEATURE0_EEESEL))
2689 return -EOPNOTSUPP;
2690
2691 edata->eee_active = lp->eee_active;
2692 edata->eee_enabled = lp->eee_enabled;
2693 edata->tx_lpi_timer = dwceqos_read(lp, REG_DWCEQOS_MAC_LPI_ENTRY_TIMER);
2694 lpi_status = dwceqos_read(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS);
2695 lpi_enabled = !!(lpi_status & DWCEQOS_MAC_LPI_CTRL_STATUS_LIPTXA);
2696 edata->tx_lpi_enabled = lpi_enabled;
2697
2698 if (netif_msg_hw(lp)) {
2699 u32 regval;
2700
2701 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS);
2702
2703 netdev_info(lp->ndev, "MAC LPI State: RX:%s TX:%s\n",
2704 dwceqos_get_rx_lpi_state(regval),
2705 dwceqos_get_tx_lpi_state(regval));
2706 }
2707
2708 return phy_ethtool_get_eee(lp->phy_dev, edata);
2709}
2710
2711static int dwceqos_set_eee(struct net_device *ndev, struct ethtool_eee *edata)
2712{
2713 struct net_local *lp = netdev_priv(ndev);
2714 u32 regval;
2715 unsigned long flags;
2716
2717 if (!(lp->feature0 & DWCEQOS_MAC_HW_FEATURE0_EEESEL))
2718 return -EOPNOTSUPP;
2719
2720 if (edata->eee_enabled && !lp->eee_active)
2721 return -EOPNOTSUPP;
2722
2723 if (edata->tx_lpi_enabled) {
2724 if (edata->tx_lpi_timer < DWCEQOS_LPI_TIMER_MIN ||
2725 edata->tx_lpi_timer > DWCEQOS_LPI_TIMER_MAX)
2726 return -EINVAL;
2727 }
2728
2729 lp->eee_enabled = edata->eee_enabled;
2730
2731 if (edata->eee_enabled && edata->tx_lpi_enabled) {
2732 dwceqos_write(lp, REG_DWCEQOS_MAC_LPI_ENTRY_TIMER,
2733 edata->tx_lpi_timer);
2734
2735 spin_lock_irqsave(&lp->hw_lock, flags);
2736 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS);
2737 regval |= DWCEQOS_LPI_CTRL_ENABLE_EEE;
2738 if (lp->en_tx_lpi_clockgating)
2739 regval |= DWCEQOS_MAC_LPI_CTRL_STATUS_LPITCSE;
2740 dwceqos_write(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS, regval);
2741 spin_unlock_irqrestore(&lp->hw_lock, flags);
2742 } else {
2743 spin_lock_irqsave(&lp->hw_lock, flags);
2744 regval = dwceqos_read(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS);
2745 regval &= ~DWCEQOS_LPI_CTRL_ENABLE_EEE;
2746 dwceqos_write(lp, REG_DWCEQOS_MAC_LPI_CTRL_STATUS, regval);
2747 spin_unlock_irqrestore(&lp->hw_lock, flags);
2748 }
2749
2750 return phy_ethtool_set_eee(lp->phy_dev, edata);
2751}
2752
2753static u32 dwceqos_get_msglevel(struct net_device *ndev)
2754{
2755 const struct net_local *lp = netdev_priv(ndev);
2756
2757 return lp->msg_enable;
2758}
2759
2760static void dwceqos_set_msglevel(struct net_device *ndev, u32 msglevel)
2761{
2762 struct net_local *lp = netdev_priv(ndev);
2763
2764 lp->msg_enable = msglevel;
2765}
2766
2767static struct ethtool_ops dwceqos_ethtool_ops = {
2768 .get_settings = dwceqos_get_settings,
2769 .set_settings = dwceqos_set_settings,
2770 .get_drvinfo = dwceqos_get_drvinfo,
2771 .get_link = ethtool_op_get_link,
2772 .get_pauseparam = dwceqos_get_pauseparam,
2773 .set_pauseparam = dwceqos_set_pauseparam,
2774 .get_strings = dwceqos_get_strings,
2775 .get_ethtool_stats = dwceqos_get_ethtool_stats,
2776 .get_sset_count = dwceqos_get_sset_count,
2777 .get_regs = dwceqos_get_regs,
2778 .get_regs_len = dwceqos_get_regs_len,
2779 .get_eee = dwceqos_get_eee,
2780 .set_eee = dwceqos_set_eee,
2781 .get_msglevel = dwceqos_get_msglevel,
2782 .set_msglevel = dwceqos_set_msglevel,
2783};
2784
2785static struct net_device_ops netdev_ops = {
2786 .ndo_open = dwceqos_open,
2787 .ndo_stop = dwceqos_stop,
2788 .ndo_start_xmit = dwceqos_start_xmit,
2789 .ndo_set_rx_mode = dwceqos_set_rx_mode,
2790 .ndo_set_mac_address = dwceqos_set_mac_address,
2791#ifdef CONFIG_NET_POLL_CONTROLLER
2792 .ndo_poll_controller = dwceqos_poll_controller,
2793#endif
2794 .ndo_do_ioctl = dwceqos_ioctl,
2795 .ndo_tx_timeout = dwceqos_tx_timeout,
2796 .ndo_get_stats64 = dwceqos_get_stats64,
2797};
2798
2799static const struct of_device_id dwceq_of_match[] = {
2800 { .compatible = "snps,dwc-qos-ethernet-4.10", },
2801 {}
2802};
2803MODULE_DEVICE_TABLE(of, dwceq_of_match);
2804
2805static int dwceqos_probe(struct platform_device *pdev)
2806{
2807 struct resource *r_mem = NULL;
2808 struct net_device *ndev;
2809 struct net_local *lp;
2810 int ret = -ENXIO;
2811
2812 r_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2813 if (!r_mem) {
2814 dev_err(&pdev->dev, "no IO resource defined.\n");
2815 return -ENXIO;
2816 }
2817
2818 ndev = alloc_etherdev(sizeof(*lp));
2819 if (!ndev) {
2820 dev_err(&pdev->dev, "etherdev allocation failed.\n");
2821 return -ENOMEM;
2822 }
2823
2824 SET_NETDEV_DEV(ndev, &pdev->dev);
2825
2826 lp = netdev_priv(ndev);
2827 lp->ndev = ndev;
2828 lp->pdev = pdev;
2829 lp->msg_enable = netif_msg_init(debug, DWCEQOS_MSG_DEFAULT);
2830
2831 spin_lock_init(&lp->tx_lock);
2832 spin_lock_init(&lp->hw_lock);
2833 spin_lock_init(&lp->stats_lock);
2834
2835 lp->apb_pclk = devm_clk_get(&pdev->dev, "apb_pclk");
2836 if (IS_ERR(lp->apb_pclk)) {
2837 dev_err(&pdev->dev, "apb_pclk clock not found.\n");
2838 ret = PTR_ERR(lp->apb_pclk);
2839 goto err_out_free_netdev;
2840 }
2841
2842 ret = clk_prepare_enable(lp->apb_pclk);
2843 if (ret) {
2844 dev_err(&pdev->dev, "Unable to enable APER clock.\n");
2845 goto err_out_free_netdev;
2846 }
2847
2848 lp->baseaddr = devm_ioremap_resource(&pdev->dev, r_mem);
2849 if (IS_ERR(lp->baseaddr)) {
2850 dev_err(&pdev->dev, "failed to map baseaddress.\n");
2851 ret = PTR_ERR(lp->baseaddr);
2852 goto err_out_clk_dis_aper;
2853 }
2854
2855 ndev->irq = platform_get_irq(pdev, 0);
2856 ndev->watchdog_timeo = DWCEQOS_TX_TIMEOUT * HZ;
2857 ndev->netdev_ops = &netdev_ops;
2858 ndev->ethtool_ops = &dwceqos_ethtool_ops;
2859 ndev->base_addr = r_mem->start;
2860
2861 dwceqos_get_hwfeatures(lp);
2862 dwceqos_mdio_set_csr(lp);
2863
2864 ndev->hw_features = NETIF_F_SG;
2865
2866 if (lp->feature1 & DWCEQOS_MAC_HW_FEATURE1_TSOEN)
2867 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
2868
2869 if (lp->feature0 & DWCEQOS_MAC_HW_FEATURE0_TXCOESEL)
2870 ndev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2871
2872 if (lp->feature0 & DWCEQOS_MAC_HW_FEATURE0_RXCOESEL)
2873 ndev->hw_features |= NETIF_F_RXCSUM;
2874
2875 ndev->features = ndev->hw_features;
2876
2877 netif_napi_add(ndev, &lp->napi, dwceqos_rx_poll, NAPI_POLL_WEIGHT);
2878
2879 ret = register_netdev(ndev);
2880 if (ret) {
2881 dev_err(&pdev->dev, "Cannot register net device, aborting.\n");
2882 goto err_out_clk_dis_aper;
2883 }
2884
2885 lp->phy_ref_clk = devm_clk_get(&pdev->dev, "phy_ref_clk");
2886 if (IS_ERR(lp->phy_ref_clk)) {
2887 dev_err(&pdev->dev, "phy_ref_clk clock not found.\n");
2888 ret = PTR_ERR(lp->phy_ref_clk);
2889 goto err_out_unregister_netdev;
2890 }
2891
2892 ret = clk_prepare_enable(lp->phy_ref_clk);
2893 if (ret) {
2894 dev_err(&pdev->dev, "Unable to enable device clock.\n");
2895 goto err_out_unregister_netdev;
2896 }
2897
2898 lp->phy_node = of_parse_phandle(lp->pdev->dev.of_node,
2899 "phy-handle", 0);
2900 if (!lp->phy_node && of_phy_is_fixed_link(lp->pdev->dev.of_node)) {
2901 ret = of_phy_register_fixed_link(lp->pdev->dev.of_node);
2902 if (ret < 0) {
2903 dev_err(&pdev->dev, "invalid fixed-link");
2904 goto err_out_unregister_netdev;
2905 }
2906
2907 lp->phy_node = of_node_get(lp->pdev->dev.of_node);
2908 }
2909
2910 ret = of_get_phy_mode(lp->pdev->dev.of_node);
2911 if (ret < 0) {
2912 dev_err(&lp->pdev->dev, "error in getting phy i/f\n");
2913 goto err_out_unregister_clk_notifier;
2914 }
2915
2916 lp->phy_interface = ret;
2917
2918 ret = dwceqos_mii_init(lp);
2919 if (ret) {
2920 dev_err(&lp->pdev->dev, "error in dwceqos_mii_init\n");
2921 goto err_out_unregister_clk_notifier;
2922 }
2923
2924 ret = dwceqos_mii_probe(ndev);
2925 if (ret != 0) {
2926 netdev_err(ndev, "mii_probe fail.\n");
2927 ret = -ENXIO;
2928 goto err_out_unregister_clk_notifier;
2929 }
2930
2931 dwceqos_set_umac_addr(lp, lp->ndev->dev_addr, 0);
2932
2933 tasklet_init(&lp->tx_bdreclaim_tasklet, dwceqos_tx_reclaim,
2934 (unsigned long)ndev);
2935 tasklet_disable(&lp->tx_bdreclaim_tasklet);
2936
2937 lp->txtimeout_handler_wq = create_singlethread_workqueue(DRIVER_NAME);
2938 INIT_WORK(&lp->txtimeout_reinit, dwceqos_reinit_for_txtimeout);
2939
2940 platform_set_drvdata(pdev, ndev);
2941 ret = dwceqos_probe_config_dt(pdev);
2942 if (ret) {
2943 dev_err(&lp->pdev->dev, "Unable to retrieve DT, error %d\n",
2944 ret);
2945 goto err_out_unregister_clk_notifier;
2946 }
2947 dev_info(&lp->pdev->dev, "pdev->id %d, baseaddr 0x%08lx, irq %d\n",
2948 pdev->id, ndev->base_addr, ndev->irq);
2949
2950 ret = devm_request_irq(&pdev->dev, ndev->irq, &dwceqos_interrupt, 0,
2951 ndev->name, ndev);
2952 if (ret) {
2953 dev_err(&lp->pdev->dev, "Unable to request IRQ %d, error %d\n",
2954 ndev->irq, ret);
2955 goto err_out_unregister_clk_notifier;
2956 }
2957
2958 if (netif_msg_probe(lp))
2959 netdev_dbg(ndev, "net_local@%p\n", lp);
2960
2961 return 0;
2962
2963err_out_unregister_clk_notifier:
2964 clk_disable_unprepare(lp->phy_ref_clk);
2965err_out_unregister_netdev:
2966 unregister_netdev(ndev);
2967err_out_clk_dis_aper:
2968 clk_disable_unprepare(lp->apb_pclk);
2969err_out_free_netdev:
2970 of_node_put(lp->phy_node);
2971 free_netdev(ndev);
2972 platform_set_drvdata(pdev, NULL);
2973 return ret;
2974}
2975
2976static int dwceqos_remove(struct platform_device *pdev)
2977{
2978 struct net_device *ndev = platform_get_drvdata(pdev);
2979 struct net_local *lp;
2980
2981 if (ndev) {
2982 lp = netdev_priv(ndev);
2983
2984 if (lp->phy_dev)
2985 phy_disconnect(lp->phy_dev);
2986 mdiobus_unregister(lp->mii_bus);
2987 mdiobus_free(lp->mii_bus);
2988
2989 unregister_netdev(ndev);
2990
2991 clk_disable_unprepare(lp->phy_ref_clk);
2992 clk_disable_unprepare(lp->apb_pclk);
2993
2994 free_netdev(ndev);
2995 }
2996
2997 return 0;
2998}
2999
3000static struct platform_driver dwceqos_driver = {
3001 .probe = dwceqos_probe,
3002 .remove = dwceqos_remove,
3003 .driver = {
3004 .name = DRIVER_NAME,
3005 .of_match_table = dwceq_of_match,
3006 },
3007};
3008
3009module_platform_driver(dwceqos_driver);
3010
3011MODULE_DESCRIPTION("DWC Ethernet QoS v4.10a driver");
3012MODULE_LICENSE("GPL v2");
3013MODULE_AUTHOR("Andreas Irestaal <andreas.irestal@axis.com>");
3014MODULE_AUTHOR("Lars Persson <lars.persson@axis.com>");