1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2021 Rafał Miłecki <rafal@milecki.pl>
  4 */
  5
  6#include <linux/delay.h>
  7#include <linux/etherdevice.h>
  8#include <linux/if_vlan.h>
  9#include <linux/interrupt.h>
 10#include <linux/module.h>
 11#include <linux/of.h>
 12#include <linux/of_net.h>
 13#include <linux/platform_device.h>
 14#include <linux/slab.h>
 15#include <linux/string.h>
 16
 17#include "bcm4908_enet.h"
 18#include "unimac.h"
 19
 20#define ENET_DMA_CH_RX_CFG			ENET_DMA_CH0_CFG
 21#define ENET_DMA_CH_TX_CFG			ENET_DMA_CH1_CFG
 22#define ENET_DMA_CH_RX_STATE_RAM		ENET_DMA_CH0_STATE_RAM
 23#define ENET_DMA_CH_TX_STATE_RAM		ENET_DMA_CH1_STATE_RAM
 24
 25#define ENET_TX_BDS_NUM				200
 26#define ENET_RX_BDS_NUM				200
 27#define ENET_RX_BDS_NUM_MAX			8192
 28
 29#define ENET_DMA_INT_DEFAULTS			(ENET_DMA_CH_CFG_INT_DONE | \
 30						 ENET_DMA_CH_CFG_INT_NO_DESC | \
 31						 ENET_DMA_CH_CFG_INT_BUFF_DONE)
 32#define ENET_DMA_MAX_BURST_LEN			8 /* in 64 bit words */
 33
 34#define ENET_MTU_MAX				ETH_DATA_LEN /* Is it possible to support 2044? */
 35#define BRCM_MAX_TAG_LEN			6
 36#define ENET_MAX_ETH_OVERHEAD			(ETH_HLEN + BRCM_MAX_TAG_LEN + VLAN_HLEN + \
 37						 ETH_FCS_LEN + 4) /* 32 */
 38
 39#define ENET_RX_SKB_BUF_SIZE			(NET_SKB_PAD + NET_IP_ALIGN + \
 40						 ETH_HLEN + BRCM_MAX_TAG_LEN + VLAN_HLEN + \
 41						 ENET_MTU_MAX + ETH_FCS_LEN + 4)
 42#define ENET_RX_SKB_BUF_ALLOC_SIZE		(SKB_DATA_ALIGN(ENET_RX_SKB_BUF_SIZE) + \
 43						 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
 44#define ENET_RX_BUF_DMA_OFFSET			(NET_SKB_PAD + NET_IP_ALIGN)
 45#define ENET_RX_BUF_DMA_SIZE			(ENET_RX_SKB_BUF_SIZE - ENET_RX_BUF_DMA_OFFSET)
 46
 47struct bcm4908_enet_dma_ring_bd {
 48	__le32 ctl;
 49	__le32 addr;
 50} __packed;
 51
 52struct bcm4908_enet_dma_ring_slot {
 53	union {
 54		void *buf;			/* RX */
 55		struct sk_buff *skb;		/* TX */
 56	};
 57	unsigned int len;
 58	dma_addr_t dma_addr;
 59};
 60
 61struct bcm4908_enet_dma_ring {
 62	int is_tx;
 63	int read_idx;
 64	int write_idx;
 65	int length;
 66	u16 cfg_block;
 67	u16 st_ram_block;
 68	struct napi_struct napi;
 69
 70	union {
 71		void *cpu_addr;
 72		struct bcm4908_enet_dma_ring_bd *buf_desc;
 73	};
 74	dma_addr_t dma_addr;
 75
 76	struct bcm4908_enet_dma_ring_slot *slots;
 77};
 78
 79struct bcm4908_enet {
 80	struct device *dev;
 81	struct net_device *netdev;
 82	void __iomem *base;
 83	int irq_tx;
 84
 85	struct bcm4908_enet_dma_ring tx_ring;
 86	struct bcm4908_enet_dma_ring rx_ring;
 87};
 88
 89/***
 90 * R/W ops
 91 */
 92
 93static u32 enet_read(struct bcm4908_enet *enet, u16 offset)
 94{
 95	return readl(enet->base + offset);
 96}
 97
 98static void enet_write(struct bcm4908_enet *enet, u16 offset, u32 value)
 99{
100	writel(value, enet->base + offset);
101}
102
103static void enet_maskset(struct bcm4908_enet *enet, u16 offset, u32 mask, u32 set)
104{
105	u32 val;
106
107	WARN_ON(set & ~mask);
108
109	val = enet_read(enet, offset);
110	val = (val & ~mask) | (set & mask);
111	enet_write(enet, offset, val);
112}
113
114static void enet_set(struct bcm4908_enet *enet, u16 offset, u32 set)
115{
116	enet_maskset(enet, offset, set, set);
117}
118
119static u32 enet_umac_read(struct bcm4908_enet *enet, u16 offset)
120{
121	return enet_read(enet, ENET_UNIMAC + offset);
122}
123
124static void enet_umac_write(struct bcm4908_enet *enet, u16 offset, u32 value)
125{
126	enet_write(enet, ENET_UNIMAC + offset, value);
127}
128
129static void enet_umac_set(struct bcm4908_enet *enet, u16 offset, u32 set)
130{
131	enet_set(enet, ENET_UNIMAC + offset, set);
132}
133
134/***
135 * Helpers
136 */
137
138static void bcm4908_enet_set_mtu(struct bcm4908_enet *enet, int mtu)
139{
140	enet_umac_write(enet, UMAC_MAX_FRAME_LEN, mtu + ENET_MAX_ETH_OVERHEAD);
141}
142
143/***
144 * DMA ring ops
145 */
146
147static void bcm4908_enet_dma_ring_intrs_on(struct bcm4908_enet *enet,
148					   struct bcm4908_enet_dma_ring *ring)
149{
150	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_INT_MASK, ENET_DMA_INT_DEFAULTS);
151}
152
153static void bcm4908_enet_dma_ring_intrs_off(struct bcm4908_enet *enet,
154					    struct bcm4908_enet_dma_ring *ring)
155{
156	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_INT_MASK, 0);
157}
158
159static void bcm4908_enet_dma_ring_intrs_ack(struct bcm4908_enet *enet,
160					    struct bcm4908_enet_dma_ring *ring)
161{
162	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_INT_STAT, ENET_DMA_INT_DEFAULTS);
163}
164
165/***
166 * DMA
167 */
168
169static int bcm4908_dma_alloc_buf_descs(struct bcm4908_enet *enet,
170				       struct bcm4908_enet_dma_ring *ring)
171{
172	int size = ring->length * sizeof(struct bcm4908_enet_dma_ring_bd);
173	struct device *dev = enet->dev;
174
175	ring->cpu_addr = dma_alloc_coherent(dev, size, &ring->dma_addr, GFP_KERNEL);
176	if (!ring->cpu_addr)
177		return -ENOMEM;
178
179	if (((uintptr_t)ring->cpu_addr) & (0x40 - 1)) {
180		dev_err(dev, "Invalid DMA ring alignment\n");
181		goto err_free_buf_descs;
182	}
183
184	ring->slots = kcalloc(ring->length, sizeof(*ring->slots), GFP_KERNEL);
185	if (!ring->slots)
186		goto err_free_buf_descs;
187
188	return 0;
189
190err_free_buf_descs:
191	dma_free_coherent(dev, size, ring->cpu_addr, ring->dma_addr);
192	ring->cpu_addr = NULL;
193	return -ENOMEM;
194}
195
196static void bcm4908_enet_dma_free(struct bcm4908_enet *enet)
197{
198	struct bcm4908_enet_dma_ring *tx_ring = &enet->tx_ring;
199	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
200	struct device *dev = enet->dev;
201	int size;
202
203	size = rx_ring->length * sizeof(struct bcm4908_enet_dma_ring_bd);
204	if (rx_ring->cpu_addr)
205		dma_free_coherent(dev, size, rx_ring->cpu_addr, rx_ring->dma_addr);
206	kfree(rx_ring->slots);
207
208	size = tx_ring->length * sizeof(struct bcm4908_enet_dma_ring_bd);
209	if (tx_ring->cpu_addr)
210		dma_free_coherent(dev, size, tx_ring->cpu_addr, tx_ring->dma_addr);
211	kfree(tx_ring->slots);
212}
213
214static int bcm4908_enet_dma_alloc(struct bcm4908_enet *enet)
215{
216	struct bcm4908_enet_dma_ring *tx_ring = &enet->tx_ring;
217	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
218	struct device *dev = enet->dev;
219	int err;
220
221	tx_ring->length = ENET_TX_BDS_NUM;
222	tx_ring->is_tx = 1;
223	tx_ring->cfg_block = ENET_DMA_CH_TX_CFG;
224	tx_ring->st_ram_block = ENET_DMA_CH_TX_STATE_RAM;
225	err = bcm4908_dma_alloc_buf_descs(enet, tx_ring);
226	if (err) {
227		dev_err(dev, "Failed to alloc TX buf descriptors: %d\n", err);
228		return err;
229	}
230
231	rx_ring->length = ENET_RX_BDS_NUM;
232	rx_ring->is_tx = 0;
233	rx_ring->cfg_block = ENET_DMA_CH_RX_CFG;
234	rx_ring->st_ram_block = ENET_DMA_CH_RX_STATE_RAM;
235	err = bcm4908_dma_alloc_buf_descs(enet, rx_ring);
236	if (err) {
237		dev_err(dev, "Failed to alloc RX buf descriptors: %d\n", err);
238		bcm4908_enet_dma_free(enet);
239		return err;
240	}
241
242	return 0;
243}
244
245static void bcm4908_enet_dma_reset(struct bcm4908_enet *enet)
246{
247	struct bcm4908_enet_dma_ring *rings[] = { &enet->rx_ring, &enet->tx_ring };
248	int i;
249
250	/* Disable the DMA controller and channel */
251	for (i = 0; i < ARRAY_SIZE(rings); i++)
252		enet_write(enet, rings[i]->cfg_block + ENET_DMA_CH_CFG, 0);
253	enet_maskset(enet, ENET_DMA_CONTROLLER_CFG, ENET_DMA_CTRL_CFG_MASTER_EN, 0);
254
255	/* Reset channels state */
256	for (i = 0; i < ARRAY_SIZE(rings); i++) {
257		struct bcm4908_enet_dma_ring *ring = rings[i];
258
259		enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_BASE_DESC_PTR, 0);
260		enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_STATE_DATA, 0);
261		enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_DESC_LEN_STATUS, 0);
262		enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_DESC_BASE_BUFPTR, 0);
263	}
264}
265
266static int bcm4908_enet_dma_alloc_rx_buf(struct bcm4908_enet *enet, unsigned int idx)
267{
268	struct bcm4908_enet_dma_ring_bd *buf_desc = &enet->rx_ring.buf_desc[idx];
269	struct bcm4908_enet_dma_ring_slot *slot = &enet->rx_ring.slots[idx];
270	struct device *dev = enet->dev;
271	u32 tmp;
272	int err;
273
274	slot->buf = napi_alloc_frag(ENET_RX_SKB_BUF_ALLOC_SIZE);
275	if (!slot->buf)
276		return -ENOMEM;
277
278	slot->dma_addr = dma_map_single(dev, slot->buf + ENET_RX_BUF_DMA_OFFSET,
279					ENET_RX_BUF_DMA_SIZE, DMA_FROM_DEVICE);
280	err = dma_mapping_error(dev, slot->dma_addr);
281	if (err) {
282		dev_err(dev, "Failed to map DMA buffer: %d\n", err);
283		skb_free_frag(slot->buf);
284		slot->buf = NULL;
285		return err;
286	}
287
288	tmp = ENET_RX_BUF_DMA_SIZE << DMA_CTL_LEN_DESC_BUFLENGTH_SHIFT;
289	tmp |= DMA_CTL_STATUS_OWN;
290	if (idx == enet->rx_ring.length - 1)
291		tmp |= DMA_CTL_STATUS_WRAP;
292	buf_desc->ctl = cpu_to_le32(tmp);
293	buf_desc->addr = cpu_to_le32(slot->dma_addr);
294
295	return 0;
296}
297
298static void bcm4908_enet_dma_ring_init(struct bcm4908_enet *enet,
299				       struct bcm4908_enet_dma_ring *ring)
300{
301	int reset_channel = 0; /* We support only 1 main channel (with TX and RX) */
302	int reset_subch = ring->is_tx ? 1 : 0;
303
304	/* Reset the DMA channel */
305	enet_write(enet, ENET_DMA_CTRL_CHANNEL_RESET, BIT(reset_channel * 2 + reset_subch));
306	enet_write(enet, ENET_DMA_CTRL_CHANNEL_RESET, 0);
307
308	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG, 0);
309	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_MAX_BURST, ENET_DMA_MAX_BURST_LEN);
310	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG_INT_MASK, 0);
311
312	enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_BASE_DESC_PTR,
313		   (uint32_t)ring->dma_addr);
314
315	ring->read_idx = 0;
316	ring->write_idx = 0;
317}
318
319static void bcm4908_enet_dma_uninit(struct bcm4908_enet *enet)
320{
321	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
322	struct bcm4908_enet_dma_ring_slot *slot;
323	struct device *dev = enet->dev;
324	int i;
325
326	for (i = rx_ring->length - 1; i >= 0; i--) {
327		slot = &rx_ring->slots[i];
328		if (!slot->buf)
329			continue;
330		dma_unmap_single(dev, slot->dma_addr, slot->len, DMA_FROM_DEVICE);
331		skb_free_frag(slot->buf);
332		slot->buf = NULL;
333	}
334}
335
336static int bcm4908_enet_dma_init(struct bcm4908_enet *enet)
337{
338	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
339	struct device *dev = enet->dev;
340	int err;
341	int i;
342
343	for (i = 0; i < rx_ring->length; i++) {
344		err = bcm4908_enet_dma_alloc_rx_buf(enet, i);
345		if (err) {
346			dev_err(dev, "Failed to alloc RX buffer: %d\n", err);
347			bcm4908_enet_dma_uninit(enet);
348			return err;
349		}
350	}
351
352	bcm4908_enet_dma_ring_init(enet, &enet->tx_ring);
353	bcm4908_enet_dma_ring_init(enet, &enet->rx_ring);
354
355	return 0;
356}
357
358static void bcm4908_enet_dma_tx_ring_enable(struct bcm4908_enet *enet,
359					    struct bcm4908_enet_dma_ring *ring)
360{
361	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG, ENET_DMA_CH_CFG_ENABLE);
362}
363
364static void bcm4908_enet_dma_tx_ring_disable(struct bcm4908_enet *enet,
365					     struct bcm4908_enet_dma_ring *ring)
366{
367	enet_write(enet, ring->cfg_block + ENET_DMA_CH_CFG, 0);
368}
369
370static void bcm4908_enet_dma_rx_ring_enable(struct bcm4908_enet *enet,
371					    struct bcm4908_enet_dma_ring *ring)
372{
373	enet_set(enet, ring->cfg_block + ENET_DMA_CH_CFG, ENET_DMA_CH_CFG_ENABLE);
374}
375
376static void bcm4908_enet_dma_rx_ring_disable(struct bcm4908_enet *enet,
377					     struct bcm4908_enet_dma_ring *ring)
378{
379	unsigned long deadline;
380	u32 tmp;
381
382	enet_maskset(enet, ring->cfg_block + ENET_DMA_CH_CFG, ENET_DMA_CH_CFG_ENABLE, 0);
383
384	deadline = jiffies + usecs_to_jiffies(2000);
385	do {
386		tmp = enet_read(enet, ring->cfg_block + ENET_DMA_CH_CFG);
387		if (!(tmp & ENET_DMA_CH_CFG_ENABLE))
388			return;
389		enet_maskset(enet, ring->cfg_block + ENET_DMA_CH_CFG, ENET_DMA_CH_CFG_ENABLE, 0);
390		usleep_range(10, 30);
391	} while (!time_after_eq(jiffies, deadline));
392
393	dev_warn(enet->dev, "Timeout waiting for DMA TX stop\n");
394}
395
396/***
397 * Ethernet driver
398 */
399
400static void bcm4908_enet_gmac_init(struct bcm4908_enet *enet)
401{
402	u32 cmd;
403
404	bcm4908_enet_set_mtu(enet, enet->netdev->mtu);
405
406	cmd = enet_umac_read(enet, UMAC_CMD);
407	enet_umac_write(enet, UMAC_CMD, cmd | CMD_SW_RESET);
408	enet_umac_write(enet, UMAC_CMD, cmd & ~CMD_SW_RESET);
409
410	enet_set(enet, ENET_FLUSH, ENET_FLUSH_RXFIFO_FLUSH | ENET_FLUSH_TXFIFO_FLUSH);
411	enet_maskset(enet, ENET_FLUSH, ENET_FLUSH_RXFIFO_FLUSH | ENET_FLUSH_TXFIFO_FLUSH, 0);
412
413	enet_set(enet, ENET_MIB_CTRL, ENET_MIB_CTRL_CLR_MIB);
414	enet_maskset(enet, ENET_MIB_CTRL, ENET_MIB_CTRL_CLR_MIB, 0);
415
416	cmd = enet_umac_read(enet, UMAC_CMD);
417	cmd &= ~(CMD_SPEED_MASK << CMD_SPEED_SHIFT);
418	cmd &= ~CMD_TX_EN;
419	cmd &= ~CMD_RX_EN;
420	cmd |= CMD_SPEED_1000 << CMD_SPEED_SHIFT;
421	enet_umac_write(enet, UMAC_CMD, cmd);
422
423	enet_maskset(enet, ENET_GMAC_STATUS,
424		     ENET_GMAC_STATUS_ETH_SPEED_MASK |
425		     ENET_GMAC_STATUS_HD |
426		     ENET_GMAC_STATUS_AUTO_CFG_EN |
427		     ENET_GMAC_STATUS_LINK_UP,
428		     ENET_GMAC_STATUS_ETH_SPEED_1000 |
429		     ENET_GMAC_STATUS_AUTO_CFG_EN |
430		     ENET_GMAC_STATUS_LINK_UP);
431}
432
433static irqreturn_t bcm4908_enet_irq_handler(int irq, void *dev_id)
434{
435	struct bcm4908_enet *enet = dev_id;
436	struct bcm4908_enet_dma_ring *ring;
437
438	ring = (irq == enet->irq_tx) ? &enet->tx_ring : &enet->rx_ring;
439
440	bcm4908_enet_dma_ring_intrs_off(enet, ring);
441	bcm4908_enet_dma_ring_intrs_ack(enet, ring);
442
443	napi_schedule(&ring->napi);
444
445	return IRQ_HANDLED;
446}
447
448static int bcm4908_enet_open(struct net_device *netdev)
449{
450	struct bcm4908_enet *enet = netdev_priv(netdev);
451	struct bcm4908_enet_dma_ring *tx_ring = &enet->tx_ring;
452	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
453	struct device *dev = enet->dev;
454	int err;
455
456	err = request_irq(netdev->irq, bcm4908_enet_irq_handler, 0, "enet", enet);
457	if (err) {
458		dev_err(dev, "Failed to request IRQ %d: %d\n", netdev->irq, err);
459		return err;
460	}
461
462	if (enet->irq_tx > 0) {
463		err = request_irq(enet->irq_tx, bcm4908_enet_irq_handler, 0,
464				  "tx", enet);
465		if (err) {
466			dev_err(dev, "Failed to request IRQ %d: %d\n",
467				enet->irq_tx, err);
468			free_irq(netdev->irq, enet);
469			return err;
470		}
471	}
472
473	bcm4908_enet_gmac_init(enet);
474	bcm4908_enet_dma_reset(enet);
475	bcm4908_enet_dma_init(enet);
476
477	enet_umac_set(enet, UMAC_CMD, CMD_TX_EN | CMD_RX_EN);
478
479	enet_set(enet, ENET_DMA_CONTROLLER_CFG, ENET_DMA_CTRL_CFG_MASTER_EN);
480	enet_maskset(enet, ENET_DMA_CONTROLLER_CFG, ENET_DMA_CTRL_CFG_FLOWC_CH1_EN, 0);
481
482	if (enet->irq_tx > 0) {
483		napi_enable(&tx_ring->napi);
484		bcm4908_enet_dma_ring_intrs_ack(enet, tx_ring);
485		bcm4908_enet_dma_ring_intrs_on(enet, tx_ring);
486	}
487
488	bcm4908_enet_dma_rx_ring_enable(enet, rx_ring);
489	napi_enable(&rx_ring->napi);
490	netif_carrier_on(netdev);
491	netif_start_queue(netdev);
492	bcm4908_enet_dma_ring_intrs_ack(enet, rx_ring);
493	bcm4908_enet_dma_ring_intrs_on(enet, rx_ring);
494
495	return 0;
496}
497
498static int bcm4908_enet_stop(struct net_device *netdev)
499{
500	struct bcm4908_enet *enet = netdev_priv(netdev);
501	struct bcm4908_enet_dma_ring *tx_ring = &enet->tx_ring;
502	struct bcm4908_enet_dma_ring *rx_ring = &enet->rx_ring;
503
504	netif_stop_queue(netdev);
505	netif_carrier_off(netdev);
506	napi_disable(&rx_ring->napi);
507	napi_disable(&tx_ring->napi);
508	netdev_reset_queue(netdev);
509
510	bcm4908_enet_dma_rx_ring_disable(enet, &enet->rx_ring);
511	bcm4908_enet_dma_tx_ring_disable(enet, &enet->tx_ring);
512
513	bcm4908_enet_dma_uninit(enet);
514
515	free_irq(enet->irq_tx, enet);
516	free_irq(enet->netdev->irq, enet);
517
518	return 0;
519}
520
521static netdev_tx_t bcm4908_enet_start_xmit(struct sk_buff *skb, struct net_device *netdev)
522{
523	struct bcm4908_enet *enet = netdev_priv(netdev);
524	struct bcm4908_enet_dma_ring *ring = &enet->tx_ring;
525	struct bcm4908_enet_dma_ring_slot *slot;
526	struct device *dev = enet->dev;
527	struct bcm4908_enet_dma_ring_bd *buf_desc;
528	int free_buf_descs;
529	u32 tmp;
530
531	/* Free transmitted skbs */
532	if (enet->irq_tx < 0 &&
533	    !(le32_to_cpu(ring->buf_desc[ring->read_idx].ctl) & DMA_CTL_STATUS_OWN))
534		napi_schedule(&enet->tx_ring.napi);
535
536	/* Don't use the last empty buf descriptor */
537	if (ring->read_idx <= ring->write_idx)
538		free_buf_descs = ring->read_idx - ring->write_idx + ring->length;
539	else
540		free_buf_descs = ring->read_idx - ring->write_idx;
541	if (free_buf_descs < 2) {
542		netif_stop_queue(netdev);
543		return NETDEV_TX_BUSY;
544	}
545
546	/* Hardware removes OWN bit after sending data */
547	buf_desc = &ring->buf_desc[ring->write_idx];
548	if (unlikely(le32_to_cpu(buf_desc->ctl) & DMA_CTL_STATUS_OWN)) {
549		netif_stop_queue(netdev);
550		return NETDEV_TX_BUSY;
551	}
552
553	slot = &ring->slots[ring->write_idx];
554	slot->skb = skb;
555	slot->len = skb->len;
556	slot->dma_addr = dma_map_single(dev, skb->data, skb->len, DMA_TO_DEVICE);
557	if (unlikely(dma_mapping_error(dev, slot->dma_addr)))
558		return NETDEV_TX_BUSY;
559
560	tmp = skb->len << DMA_CTL_LEN_DESC_BUFLENGTH_SHIFT;
561	tmp |= DMA_CTL_STATUS_OWN;
562	tmp |= DMA_CTL_STATUS_SOP;
563	tmp |= DMA_CTL_STATUS_EOP;
564	tmp |= DMA_CTL_STATUS_APPEND_CRC;
565	if (ring->write_idx + 1 == ring->length - 1)
566		tmp |= DMA_CTL_STATUS_WRAP;
567
568	netdev_sent_queue(enet->netdev, skb->len);
569
570	buf_desc->addr = cpu_to_le32((uint32_t)slot->dma_addr);
571	buf_desc->ctl = cpu_to_le32(tmp);
572
573	bcm4908_enet_dma_tx_ring_enable(enet, &enet->tx_ring);
574
575	if (++ring->write_idx == ring->length - 1)
576		ring->write_idx = 0;
577
578	return NETDEV_TX_OK;
579}
580
581static int bcm4908_enet_poll_rx(struct napi_struct *napi, int weight)
582{
583	struct bcm4908_enet_dma_ring *rx_ring = container_of(napi, struct bcm4908_enet_dma_ring, napi);
584	struct bcm4908_enet *enet = container_of(rx_ring, struct bcm4908_enet, rx_ring);
585	struct device *dev = enet->dev;
586	int handled = 0;
587
588	while (handled < weight) {
589		struct bcm4908_enet_dma_ring_bd *buf_desc;
590		struct bcm4908_enet_dma_ring_slot slot;
591		struct sk_buff *skb;
592		u32 ctl;
593		int len;
594		int err;
595
596		buf_desc = &enet->rx_ring.buf_desc[enet->rx_ring.read_idx];
597		ctl = le32_to_cpu(buf_desc->ctl);
598		if (ctl & DMA_CTL_STATUS_OWN)
599			break;
600
601		slot = enet->rx_ring.slots[enet->rx_ring.read_idx];
602
603		/* Provide new buffer before unpinning the old one */
604		err = bcm4908_enet_dma_alloc_rx_buf(enet, enet->rx_ring.read_idx);
605		if (err)
606			break;
607
608		if (++enet->rx_ring.read_idx == enet->rx_ring.length)
609			enet->rx_ring.read_idx = 0;
610
611		len = (ctl & DMA_CTL_LEN_DESC_BUFLENGTH) >> DMA_CTL_LEN_DESC_BUFLENGTH_SHIFT;
612
613		if (len < ETH_ZLEN ||
614		    (ctl & (DMA_CTL_STATUS_SOP | DMA_CTL_STATUS_EOP)) != (DMA_CTL_STATUS_SOP | DMA_CTL_STATUS_EOP)) {
615			skb_free_frag(slot.buf);
616			enet->netdev->stats.rx_dropped++;
617			break;
618		}
619
620		dma_unmap_single(dev, slot.dma_addr, ENET_RX_BUF_DMA_SIZE, DMA_FROM_DEVICE);
621
622		skb = build_skb(slot.buf, ENET_RX_SKB_BUF_ALLOC_SIZE);
623		if (unlikely(!skb)) {
624			skb_free_frag(slot.buf);
625			enet->netdev->stats.rx_dropped++;
626			break;
627		}
628		skb_reserve(skb, ENET_RX_BUF_DMA_OFFSET);
629		skb_put(skb, len - ETH_FCS_LEN);
630		skb->protocol = eth_type_trans(skb, enet->netdev);
631
632		netif_receive_skb(skb);
633
634		enet->netdev->stats.rx_packets++;
635		enet->netdev->stats.rx_bytes += len;
636
637		handled++;
638	}
639
640	if (handled < weight) {
641		napi_complete_done(napi, handled);
642		bcm4908_enet_dma_ring_intrs_on(enet, rx_ring);
643	}
644
645	/* Hardware could disable ring if it run out of descriptors */
646	bcm4908_enet_dma_rx_ring_enable(enet, &enet->rx_ring);
647
648	return handled;
649}
650
651static int bcm4908_enet_poll_tx(struct napi_struct *napi, int weight)
652{
653	struct bcm4908_enet_dma_ring *tx_ring = container_of(napi, struct bcm4908_enet_dma_ring, napi);
654	struct bcm4908_enet *enet = container_of(tx_ring, struct bcm4908_enet, tx_ring);
655	struct bcm4908_enet_dma_ring_bd *buf_desc;
656	struct bcm4908_enet_dma_ring_slot *slot;
657	struct device *dev = enet->dev;
658	unsigned int bytes = 0;
659	int handled = 0;
660
661	while (handled < weight && tx_ring->read_idx != tx_ring->write_idx) {
662		buf_desc = &tx_ring->buf_desc[tx_ring->read_idx];
663		if (le32_to_cpu(buf_desc->ctl) & DMA_CTL_STATUS_OWN)
664			break;
665		slot = &tx_ring->slots[tx_ring->read_idx];
666
667		dma_unmap_single(dev, slot->dma_addr, slot->len, DMA_TO_DEVICE);
668		dev_kfree_skb(slot->skb);
669
670		handled++;
671		bytes += slot->len;
672
673		if (++tx_ring->read_idx == tx_ring->length)
674			tx_ring->read_idx = 0;
675	}
676
677	netdev_completed_queue(enet->netdev, handled, bytes);
678	enet->netdev->stats.tx_packets += handled;
679	enet->netdev->stats.tx_bytes += bytes;
680
681	if (handled < weight) {
682		napi_complete_done(napi, handled);
683		bcm4908_enet_dma_ring_intrs_on(enet, tx_ring);
684	}
685
686	if (netif_queue_stopped(enet->netdev))
687		netif_wake_queue(enet->netdev);
688
689	return handled;
690}
691
692static int bcm4908_enet_change_mtu(struct net_device *netdev, int new_mtu)
693{
694	struct bcm4908_enet *enet = netdev_priv(netdev);
695
696	bcm4908_enet_set_mtu(enet, new_mtu);
697
698	return 0;
699}
700
701static const struct net_device_ops bcm4908_enet_netdev_ops = {
702	.ndo_open = bcm4908_enet_open,
703	.ndo_stop = bcm4908_enet_stop,
704	.ndo_start_xmit = bcm4908_enet_start_xmit,
705	.ndo_set_mac_address = eth_mac_addr,
706	.ndo_change_mtu = bcm4908_enet_change_mtu,
707};
708
709static int bcm4908_enet_probe(struct platform_device *pdev)
710{
711	struct device *dev = &pdev->dev;
712	struct net_device *netdev;
713	struct bcm4908_enet *enet;
714	int err;
715
716	netdev = devm_alloc_etherdev(dev, sizeof(*enet));
717	if (!netdev)
718		return -ENOMEM;
719
720	enet = netdev_priv(netdev);
721	enet->dev = dev;
722	enet->netdev = netdev;
723
724	enet->base = devm_platform_ioremap_resource(pdev, 0);
725	if (IS_ERR(enet->base)) {
726		dev_err(dev, "Failed to map registers: %ld\n", PTR_ERR(enet->base));
727		return PTR_ERR(enet->base);
728	}
729
730	netdev->irq = platform_get_irq_byname(pdev, "rx");
731	if (netdev->irq < 0)
732		return netdev->irq;
733
734	enet->irq_tx = platform_get_irq_byname(pdev, "tx");
735
736	err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
737	if (err)
738		return err;
739
740	err = bcm4908_enet_dma_alloc(enet);
741	if (err)
742		return err;
743
744	SET_NETDEV_DEV(netdev, &pdev->dev);
745	err = of_get_ethdev_address(dev->of_node, netdev);
746	if (err == -EPROBE_DEFER)
747		goto err_dma_free;
748	if (err)
749		eth_hw_addr_random(netdev);
750	netdev->netdev_ops = &bcm4908_enet_netdev_ops;
751	netdev->min_mtu = ETH_ZLEN;
752	netdev->mtu = ETH_DATA_LEN;
753	netdev->max_mtu = ENET_MTU_MAX;
754	netif_napi_add_tx(netdev, &enet->tx_ring.napi, bcm4908_enet_poll_tx);
755	netif_napi_add(netdev, &enet->rx_ring.napi, bcm4908_enet_poll_rx);
756
757	err = register_netdev(netdev);
758	if (err)
759		goto err_dma_free;
760
761	platform_set_drvdata(pdev, enet);
762
763	return 0;
764
765err_dma_free:
766	bcm4908_enet_dma_free(enet);
767
768	return err;
769}
770
771static void bcm4908_enet_remove(struct platform_device *pdev)
772{
773	struct bcm4908_enet *enet = platform_get_drvdata(pdev);
774
775	unregister_netdev(enet->netdev);
776	netif_napi_del(&enet->rx_ring.napi);
777	netif_napi_del(&enet->tx_ring.napi);
778	bcm4908_enet_dma_free(enet);
 
 
779}
780
781static const struct of_device_id bcm4908_enet_of_match[] = {
782	{ .compatible = "brcm,bcm4908-enet"},
783	{},
784};
785
786static struct platform_driver bcm4908_enet_driver = {
787	.driver = {
788		.name = "bcm4908_enet",
789		.of_match_table = bcm4908_enet_of_match,
790	},
791	.probe	= bcm4908_enet_probe,
792	.remove_new = bcm4908_enet_remove,
793};
794module_platform_driver(bcm4908_enet_driver);
795
796MODULE_DESCRIPTION("Broadcom BCM4908 Gigabit Ethernet driver");
797MODULE_LICENSE("GPL v2");
798MODULE_DEVICE_TABLE(of, bcm4908_enet_of_match);