Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Mellanox BlueField SoC TmFifo driver
4 *
5 * Copyright (C) 2019 Mellanox Technologies
6 */
7
8#include <linux/acpi.h>
9#include <linux/bitfield.h>
10#include <linux/circ_buf.h>
11#include <linux/efi.h>
12#include <linux/irq.h>
13#include <linux/module.h>
14#include <linux/mutex.h>
15#include <linux/platform_device.h>
16#include <linux/types.h>
17
18#include <linux/virtio_config.h>
19#include <linux/virtio_console.h>
20#include <linux/virtio_ids.h>
21#include <linux/virtio_net.h>
22#include <linux/virtio_ring.h>
23
24#include "mlxbf-tmfifo-regs.h"
25
26/* Vring size. */
27#define MLXBF_TMFIFO_VRING_SIZE SZ_1K
28
29/* Console Tx buffer size. */
30#define MLXBF_TMFIFO_CON_TX_BUF_SIZE SZ_32K
31
32/* Console Tx buffer reserved space. */
33#define MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE 8
34
35/* House-keeping timer interval. */
36#define MLXBF_TMFIFO_TIMER_INTERVAL (HZ / 10)
37
38/* Virtual devices sharing the TM FIFO. */
39#define MLXBF_TMFIFO_VDEV_MAX (VIRTIO_ID_CONSOLE + 1)
40
41/*
42 * Reserve 1/16 of TmFifo space, so console messages are not starved by
43 * the networking traffic.
44 */
45#define MLXBF_TMFIFO_RESERVE_RATIO 16
46
47/* Message with data needs at least two words (for header & data). */
48#define MLXBF_TMFIFO_DATA_MIN_WORDS 2
49
50/* ACPI UID for BlueField-3. */
51#define TMFIFO_BF3_UID 1
52
53struct mlxbf_tmfifo;
54
55/**
56 * mlxbf_tmfifo_vring - Structure of the TmFifo virtual ring
57 * @va: virtual address of the ring
58 * @dma: dma address of the ring
59 * @vq: pointer to the virtio virtqueue
60 * @desc: current descriptor of the pending packet
61 * @desc_head: head descriptor of the pending packet
62 * @cur_len: processed length of the current descriptor
63 * @rem_len: remaining length of the pending packet
64 * @pkt_len: total length of the pending packet
65 * @next_avail: next avail descriptor id
66 * @num: vring size (number of descriptors)
67 * @align: vring alignment size
68 * @index: vring index
69 * @vdev_id: vring virtio id (VIRTIO_ID_xxx)
70 * @fifo: pointer to the tmfifo structure
71 */
72struct mlxbf_tmfifo_vring {
73 void *va;
74 dma_addr_t dma;
75 struct virtqueue *vq;
76 struct vring_desc *desc;
77 struct vring_desc *desc_head;
78 int cur_len;
79 int rem_len;
80 u32 pkt_len;
81 u16 next_avail;
82 int num;
83 int align;
84 int index;
85 int vdev_id;
86 struct mlxbf_tmfifo *fifo;
87};
88
89/* Interrupt types. */
90enum {
91 MLXBF_TM_RX_LWM_IRQ,
92 MLXBF_TM_RX_HWM_IRQ,
93 MLXBF_TM_TX_LWM_IRQ,
94 MLXBF_TM_TX_HWM_IRQ,
95 MLXBF_TM_MAX_IRQ
96};
97
98/* Ring types (Rx & Tx). */
99enum {
100 MLXBF_TMFIFO_VRING_RX,
101 MLXBF_TMFIFO_VRING_TX,
102 MLXBF_TMFIFO_VRING_MAX
103};
104
105/**
106 * mlxbf_tmfifo_vdev - Structure of the TmFifo virtual device
107 * @vdev: virtio device, in which the vdev.id.device field has the
108 * VIRTIO_ID_xxx id to distinguish the virtual device.
109 * @status: status of the device
110 * @features: supported features of the device
111 * @vrings: array of tmfifo vrings of this device
112 * @config.cons: virtual console config -
113 * select if vdev.id.device is VIRTIO_ID_CONSOLE
114 * @config.net: virtual network config -
115 * select if vdev.id.device is VIRTIO_ID_NET
116 * @tx_buf: tx buffer used to buffer data before writing into the FIFO
117 */
118struct mlxbf_tmfifo_vdev {
119 struct virtio_device vdev;
120 u8 status;
121 u64 features;
122 struct mlxbf_tmfifo_vring vrings[MLXBF_TMFIFO_VRING_MAX];
123 union {
124 struct virtio_console_config cons;
125 struct virtio_net_config net;
126 } config;
127 struct circ_buf tx_buf;
128};
129
130/**
131 * mlxbf_tmfifo_irq_info - Structure of the interrupt information
132 * @fifo: pointer to the tmfifo structure
133 * @irq: interrupt number
134 * @index: index into the interrupt array
135 */
136struct mlxbf_tmfifo_irq_info {
137 struct mlxbf_tmfifo *fifo;
138 int irq;
139 int index;
140};
141
142/**
143 * mlxbf_tmfifo_io - Structure of the TmFifo IO resource (for both rx & tx)
144 * @ctl: control register offset (TMFIFO_RX_CTL / TMFIFO_TX_CTL)
145 * @sts: status register offset (TMFIFO_RX_STS / TMFIFO_TX_STS)
146 * @data: data register offset (TMFIFO_RX_DATA / TMFIFO_TX_DATA)
147 */
148struct mlxbf_tmfifo_io {
149 void __iomem *ctl;
150 void __iomem *sts;
151 void __iomem *data;
152};
153
154/**
155 * mlxbf_tmfifo - Structure of the TmFifo
156 * @vdev: array of the virtual devices running over the TmFifo
157 * @lock: lock to protect the TmFifo access
158 * @res0: mapped resource block 0
159 * @res1: mapped resource block 1
160 * @rx: rx io resource
161 * @tx: tx io resource
162 * @rx_fifo_size: number of entries of the Rx FIFO
163 * @tx_fifo_size: number of entries of the Tx FIFO
164 * @pend_events: pending bits for deferred events
165 * @irq_info: interrupt information
166 * @work: work struct for deferred process
167 * @timer: background timer
168 * @vring: Tx/Rx ring
169 * @spin_lock: Tx/Rx spin lock
170 * @is_ready: ready flag
171 */
172struct mlxbf_tmfifo {
173 struct mlxbf_tmfifo_vdev *vdev[MLXBF_TMFIFO_VDEV_MAX];
174 struct mutex lock; /* TmFifo lock */
175 void __iomem *res0;
176 void __iomem *res1;
177 struct mlxbf_tmfifo_io rx;
178 struct mlxbf_tmfifo_io tx;
179 int rx_fifo_size;
180 int tx_fifo_size;
181 unsigned long pend_events;
182 struct mlxbf_tmfifo_irq_info irq_info[MLXBF_TM_MAX_IRQ];
183 struct work_struct work;
184 struct timer_list timer;
185 struct mlxbf_tmfifo_vring *vring[2];
186 spinlock_t spin_lock[2]; /* spin lock */
187 bool is_ready;
188};
189
190/**
191 * mlxbf_tmfifo_msg_hdr - Structure of the TmFifo message header
192 * @type: message type
193 * @len: payload length in network byte order. Messages sent into the FIFO
194 * will be read by the other side as data stream in the same byte order.
195 * The length needs to be encoded into network order so both sides
196 * could understand it.
197 */
198struct mlxbf_tmfifo_msg_hdr {
199 u8 type;
200 __be16 len;
201 u8 unused[5];
202} __packed __aligned(sizeof(u64));
203
204/*
205 * Default MAC.
206 * This MAC address will be read from EFI persistent variable if configured.
207 * It can also be reconfigured with standard Linux tools.
208 */
209static u8 mlxbf_tmfifo_net_default_mac[ETH_ALEN] = {
210 0x00, 0x1A, 0xCA, 0xFF, 0xFF, 0x01
211};
212
213/* EFI variable name of the MAC address. */
214static efi_char16_t mlxbf_tmfifo_efi_name[] = L"RshimMacAddr";
215
216/* Maximum L2 header length. */
217#define MLXBF_TMFIFO_NET_L2_OVERHEAD 36
218
219/* Supported virtio-net features. */
220#define MLXBF_TMFIFO_NET_FEATURES \
221 (BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_STATUS) | \
222 BIT_ULL(VIRTIO_NET_F_MAC))
223
224#define mlxbf_vdev_to_tmfifo(d) container_of(d, struct mlxbf_tmfifo_vdev, vdev)
225
226/* Free vrings of the FIFO device. */
227static void mlxbf_tmfifo_free_vrings(struct mlxbf_tmfifo *fifo,
228 struct mlxbf_tmfifo_vdev *tm_vdev)
229{
230 struct mlxbf_tmfifo_vring *vring;
231 int i, size;
232
233 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
234 vring = &tm_vdev->vrings[i];
235 if (vring->va) {
236 size = vring_size(vring->num, vring->align);
237 dma_free_coherent(tm_vdev->vdev.dev.parent, size,
238 vring->va, vring->dma);
239 vring->va = NULL;
240 if (vring->vq) {
241 vring_del_virtqueue(vring->vq);
242 vring->vq = NULL;
243 }
244 }
245 }
246}
247
248/* Allocate vrings for the FIFO. */
249static int mlxbf_tmfifo_alloc_vrings(struct mlxbf_tmfifo *fifo,
250 struct mlxbf_tmfifo_vdev *tm_vdev)
251{
252 struct mlxbf_tmfifo_vring *vring;
253 struct device *dev;
254 dma_addr_t dma;
255 int i, size;
256 void *va;
257
258 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
259 vring = &tm_vdev->vrings[i];
260 vring->fifo = fifo;
261 vring->num = MLXBF_TMFIFO_VRING_SIZE;
262 vring->align = SMP_CACHE_BYTES;
263 vring->index = i;
264 vring->vdev_id = tm_vdev->vdev.id.device;
265 dev = &tm_vdev->vdev.dev;
266
267 size = vring_size(vring->num, vring->align);
268 va = dma_alloc_coherent(dev->parent, size, &dma, GFP_KERNEL);
269 if (!va) {
270 mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
271 dev_err(dev->parent, "dma_alloc_coherent failed\n");
272 return -ENOMEM;
273 }
274
275 vring->va = va;
276 vring->dma = dma;
277 }
278
279 return 0;
280}
281
282/* Disable interrupts of the FIFO device. */
283static void mlxbf_tmfifo_disable_irqs(struct mlxbf_tmfifo *fifo)
284{
285 int i, irq;
286
287 for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) {
288 irq = fifo->irq_info[i].irq;
289 fifo->irq_info[i].irq = 0;
290 disable_irq(irq);
291 }
292}
293
294/* Interrupt handler. */
295static irqreturn_t mlxbf_tmfifo_irq_handler(int irq, void *arg)
296{
297 struct mlxbf_tmfifo_irq_info *irq_info = arg;
298
299 if (!test_and_set_bit(irq_info->index, &irq_info->fifo->pend_events))
300 schedule_work(&irq_info->fifo->work);
301
302 return IRQ_HANDLED;
303}
304
305/* Get the next packet descriptor from the vring. */
306static struct vring_desc *
307mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring)
308{
309 const struct vring *vr = virtqueue_get_vring(vring->vq);
310 struct virtio_device *vdev = vring->vq->vdev;
311 unsigned int idx, head;
312
313 if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx))
314 return NULL;
315
316 /* Make sure 'avail->idx' is visible already. */
317 virtio_rmb(false);
318
319 idx = vring->next_avail % vr->num;
320 head = virtio16_to_cpu(vdev, vr->avail->ring[idx]);
321 if (WARN_ON(head >= vr->num))
322 return NULL;
323
324 vring->next_avail++;
325
326 return &vr->desc[head];
327}
328
329/* Release virtio descriptor. */
330static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring,
331 struct vring_desc *desc, u32 len)
332{
333 const struct vring *vr = virtqueue_get_vring(vring->vq);
334 struct virtio_device *vdev = vring->vq->vdev;
335 u16 idx, vr_idx;
336
337 vr_idx = virtio16_to_cpu(vdev, vr->used->idx);
338 idx = vr_idx % vr->num;
339 vr->used->ring[idx].id = cpu_to_virtio32(vdev, desc - vr->desc);
340 vr->used->ring[idx].len = cpu_to_virtio32(vdev, len);
341
342 /*
343 * Virtio could poll and check the 'idx' to decide whether the desc is
344 * done or not. Add a memory barrier here to make sure the update above
345 * completes before updating the idx.
346 */
347 virtio_mb(false);
348 vr->used->idx = cpu_to_virtio16(vdev, vr_idx + 1);
349}
350
351/* Get the total length of the descriptor chain. */
352static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring,
353 struct vring_desc *desc)
354{
355 const struct vring *vr = virtqueue_get_vring(vring->vq);
356 struct virtio_device *vdev = vring->vq->vdev;
357 u32 len = 0, idx;
358
359 while (desc) {
360 len += virtio32_to_cpu(vdev, desc->len);
361 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT))
362 break;
363 idx = virtio16_to_cpu(vdev, desc->next);
364 desc = &vr->desc[idx];
365 }
366
367 return len;
368}
369
370static void mlxbf_tmfifo_release_pending_pkt(struct mlxbf_tmfifo_vring *vring)
371{
372 struct vring_desc *desc_head;
373 u32 len = 0;
374
375 if (vring->desc_head) {
376 desc_head = vring->desc_head;
377 len = vring->pkt_len;
378 } else {
379 desc_head = mlxbf_tmfifo_get_next_desc(vring);
380 len = mlxbf_tmfifo_get_pkt_len(vring, desc_head);
381 }
382
383 if (desc_head)
384 mlxbf_tmfifo_release_desc(vring, desc_head, len);
385
386 vring->pkt_len = 0;
387 vring->desc = NULL;
388 vring->desc_head = NULL;
389}
390
391static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring,
392 struct vring_desc *desc, bool is_rx)
393{
394 struct virtio_device *vdev = vring->vq->vdev;
395 struct virtio_net_hdr *net_hdr;
396
397 net_hdr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
398 memset(net_hdr, 0, sizeof(*net_hdr));
399}
400
401/* Get and initialize the next packet. */
402static struct vring_desc *
403mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx)
404{
405 struct vring_desc *desc;
406
407 desc = mlxbf_tmfifo_get_next_desc(vring);
408 if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET)
409 mlxbf_tmfifo_init_net_desc(vring, desc, is_rx);
410
411 vring->desc_head = desc;
412 vring->desc = desc;
413
414 return desc;
415}
416
417/* House-keeping timer. */
418static void mlxbf_tmfifo_timer(struct timer_list *t)
419{
420 struct mlxbf_tmfifo *fifo = container_of(t, struct mlxbf_tmfifo, timer);
421 int rx, tx;
422
423 rx = !test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events);
424 tx = !test_and_set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events);
425
426 if (rx || tx)
427 schedule_work(&fifo->work);
428
429 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL);
430}
431
432/* Copy one console packet into the output buffer. */
433static void mlxbf_tmfifo_console_output_one(struct mlxbf_tmfifo_vdev *cons,
434 struct mlxbf_tmfifo_vring *vring,
435 struct vring_desc *desc)
436{
437 const struct vring *vr = virtqueue_get_vring(vring->vq);
438 struct virtio_device *vdev = &cons->vdev;
439 u32 len, idx, seg;
440 void *addr;
441
442 while (desc) {
443 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
444 len = virtio32_to_cpu(vdev, desc->len);
445
446 seg = CIRC_SPACE_TO_END(cons->tx_buf.head, cons->tx_buf.tail,
447 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
448 if (len <= seg) {
449 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, len);
450 } else {
451 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, seg);
452 addr += seg;
453 memcpy(cons->tx_buf.buf, addr, len - seg);
454 }
455 cons->tx_buf.head = (cons->tx_buf.head + len) %
456 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
457
458 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT))
459 break;
460 idx = virtio16_to_cpu(vdev, desc->next);
461 desc = &vr->desc[idx];
462 }
463}
464
465/* Copy console data into the output buffer. */
466static void mlxbf_tmfifo_console_output(struct mlxbf_tmfifo_vdev *cons,
467 struct mlxbf_tmfifo_vring *vring)
468{
469 struct vring_desc *desc;
470 u32 len, avail;
471
472 desc = mlxbf_tmfifo_get_next_desc(vring);
473 while (desc) {
474 /* Release the packet if not enough space. */
475 len = mlxbf_tmfifo_get_pkt_len(vring, desc);
476 avail = CIRC_SPACE(cons->tx_buf.head, cons->tx_buf.tail,
477 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
478 if (len + MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE > avail) {
479 mlxbf_tmfifo_release_desc(vring, desc, len);
480 break;
481 }
482
483 mlxbf_tmfifo_console_output_one(cons, vring, desc);
484 mlxbf_tmfifo_release_desc(vring, desc, len);
485 desc = mlxbf_tmfifo_get_next_desc(vring);
486 }
487}
488
489/* Get the number of available words in Rx FIFO for receiving. */
490static int mlxbf_tmfifo_get_rx_avail(struct mlxbf_tmfifo *fifo)
491{
492 u64 sts;
493
494 sts = readq(fifo->rx.sts);
495 return FIELD_GET(MLXBF_TMFIFO_RX_STS__COUNT_MASK, sts);
496}
497
498/* Get the number of available words in the TmFifo for sending. */
499static int mlxbf_tmfifo_get_tx_avail(struct mlxbf_tmfifo *fifo, int vdev_id)
500{
501 int tx_reserve;
502 u32 count;
503 u64 sts;
504
505 /* Reserve some room in FIFO for console messages. */
506 if (vdev_id == VIRTIO_ID_NET)
507 tx_reserve = fifo->tx_fifo_size / MLXBF_TMFIFO_RESERVE_RATIO;
508 else
509 tx_reserve = 1;
510
511 sts = readq(fifo->tx.sts);
512 count = FIELD_GET(MLXBF_TMFIFO_TX_STS__COUNT_MASK, sts);
513 return fifo->tx_fifo_size - tx_reserve - count;
514}
515
516/* Console Tx (move data from the output buffer into the TmFifo). */
517static void mlxbf_tmfifo_console_tx(struct mlxbf_tmfifo *fifo, int avail)
518{
519 struct mlxbf_tmfifo_msg_hdr hdr;
520 struct mlxbf_tmfifo_vdev *cons;
521 unsigned long flags;
522 int size, seg;
523 void *addr;
524 u64 data;
525
526 /* Return if not enough space available. */
527 if (avail < MLXBF_TMFIFO_DATA_MIN_WORDS)
528 return;
529
530 cons = fifo->vdev[VIRTIO_ID_CONSOLE];
531 if (!cons || !cons->tx_buf.buf)
532 return;
533
534 /* Return if no data to send. */
535 size = CIRC_CNT(cons->tx_buf.head, cons->tx_buf.tail,
536 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
537 if (size == 0)
538 return;
539
540 /* Adjust the size to available space. */
541 if (size + sizeof(hdr) > avail * sizeof(u64))
542 size = avail * sizeof(u64) - sizeof(hdr);
543
544 /* Write header. */
545 hdr.type = VIRTIO_ID_CONSOLE;
546 hdr.len = htons(size);
547 writeq(*(u64 *)&hdr, fifo->tx.data);
548
549 /* Use spin-lock to protect the 'cons->tx_buf'. */
550 spin_lock_irqsave(&fifo->spin_lock[0], flags);
551
552 while (size > 0) {
553 addr = cons->tx_buf.buf + cons->tx_buf.tail;
554
555 seg = CIRC_CNT_TO_END(cons->tx_buf.head, cons->tx_buf.tail,
556 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
557 if (seg >= sizeof(u64)) {
558 memcpy(&data, addr, sizeof(u64));
559 } else {
560 memcpy(&data, addr, seg);
561 memcpy((u8 *)&data + seg, cons->tx_buf.buf,
562 sizeof(u64) - seg);
563 }
564 writeq(data, fifo->tx.data);
565
566 if (size >= sizeof(u64)) {
567 cons->tx_buf.tail = (cons->tx_buf.tail + sizeof(u64)) %
568 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
569 size -= sizeof(u64);
570 } else {
571 cons->tx_buf.tail = (cons->tx_buf.tail + size) %
572 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
573 size = 0;
574 }
575 }
576
577 spin_unlock_irqrestore(&fifo->spin_lock[0], flags);
578}
579
580/* Rx/Tx one word in the descriptor buffer. */
581static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring,
582 struct vring_desc *desc,
583 bool is_rx, int len)
584{
585 struct virtio_device *vdev = vring->vq->vdev;
586 struct mlxbf_tmfifo *fifo = vring->fifo;
587 void *addr;
588 u64 data;
589
590 /* Get the buffer address of this desc. */
591 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
592
593 /* Read a word from FIFO for Rx. */
594 if (is_rx)
595 data = readq(fifo->rx.data);
596
597 if (vring->cur_len + sizeof(u64) <= len) {
598 /* The whole word. */
599 if (is_rx)
600 memcpy(addr + vring->cur_len, &data, sizeof(u64));
601 else
602 memcpy(&data, addr + vring->cur_len, sizeof(u64));
603 vring->cur_len += sizeof(u64);
604 } else {
605 /* Leftover bytes. */
606 if (is_rx)
607 memcpy(addr + vring->cur_len, &data,
608 len - vring->cur_len);
609 else
610 memcpy(&data, addr + vring->cur_len,
611 len - vring->cur_len);
612 vring->cur_len = len;
613 }
614
615 /* Write the word into FIFO for Tx. */
616 if (!is_rx)
617 writeq(data, fifo->tx.data);
618}
619
620/*
621 * Rx/Tx packet header.
622 *
623 * In Rx case, the packet might be found to belong to a different vring since
624 * the TmFifo is shared by different services. In such case, the 'vring_change'
625 * flag is set.
626 */
627static void mlxbf_tmfifo_rxtx_header(struct mlxbf_tmfifo_vring *vring,
628 struct vring_desc *desc,
629 bool is_rx, bool *vring_change)
630{
631 struct mlxbf_tmfifo *fifo = vring->fifo;
632 struct virtio_net_config *config;
633 struct mlxbf_tmfifo_msg_hdr hdr;
634 int vdev_id, hdr_len;
635
636 /* Read/Write packet header. */
637 if (is_rx) {
638 /* Drain one word from the FIFO. */
639 *(u64 *)&hdr = readq(fifo->rx.data);
640
641 /* Skip the length 0 packets (keepalive). */
642 if (hdr.len == 0)
643 return;
644
645 /* Check packet type. */
646 if (hdr.type == VIRTIO_ID_NET) {
647 vdev_id = VIRTIO_ID_NET;
648 hdr_len = sizeof(struct virtio_net_hdr);
649 config = &fifo->vdev[vdev_id]->config.net;
650 /* A legacy-only interface for now. */
651 if (ntohs(hdr.len) >
652 __virtio16_to_cpu(virtio_legacy_is_little_endian(),
653 config->mtu) +
654 MLXBF_TMFIFO_NET_L2_OVERHEAD)
655 return;
656 } else {
657 vdev_id = VIRTIO_ID_CONSOLE;
658 hdr_len = 0;
659 }
660
661 /*
662 * Check whether the new packet still belongs to this vring.
663 * If not, update the pkt_len of the new vring.
664 */
665 if (vdev_id != vring->vdev_id) {
666 struct mlxbf_tmfifo_vdev *tm_dev2 = fifo->vdev[vdev_id];
667
668 if (!tm_dev2)
669 return;
670 vring->desc = desc;
671 vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX];
672 *vring_change = true;
673 }
674 vring->pkt_len = ntohs(hdr.len) + hdr_len;
675 } else {
676 /* Network virtio has an extra header. */
677 hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ?
678 sizeof(struct virtio_net_hdr) : 0;
679 vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, desc);
680 hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ?
681 VIRTIO_ID_NET : VIRTIO_ID_CONSOLE;
682 hdr.len = htons(vring->pkt_len - hdr_len);
683 writeq(*(u64 *)&hdr, fifo->tx.data);
684 }
685
686 vring->cur_len = hdr_len;
687 vring->rem_len = vring->pkt_len;
688 fifo->vring[is_rx] = vring;
689}
690
691/*
692 * Rx/Tx one descriptor.
693 *
694 * Return true to indicate more data available.
695 */
696static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring,
697 bool is_rx, int *avail)
698{
699 const struct vring *vr = virtqueue_get_vring(vring->vq);
700 struct mlxbf_tmfifo *fifo = vring->fifo;
701 struct virtio_device *vdev;
702 bool vring_change = false;
703 struct vring_desc *desc;
704 unsigned long flags;
705 u32 len, idx;
706
707 vdev = &fifo->vdev[vring->vdev_id]->vdev;
708
709 /* Get the descriptor of the next packet. */
710 if (!vring->desc) {
711 desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx);
712 if (!desc)
713 return false;
714 } else {
715 desc = vring->desc;
716 }
717
718 /* Beginning of a packet. Start to Rx/Tx packet header. */
719 if (vring->pkt_len == 0) {
720 mlxbf_tmfifo_rxtx_header(vring, desc, is_rx, &vring_change);
721 (*avail)--;
722
723 /* Return if new packet is for another ring. */
724 if (vring_change)
725 return false;
726 goto mlxbf_tmfifo_desc_done;
727 }
728
729 /* Get the length of this desc. */
730 len = virtio32_to_cpu(vdev, desc->len);
731 if (len > vring->rem_len)
732 len = vring->rem_len;
733
734 /* Rx/Tx one word (8 bytes) if not done. */
735 if (vring->cur_len < len) {
736 mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len);
737 (*avail)--;
738 }
739
740 /* Check again whether it's done. */
741 if (vring->cur_len == len) {
742 vring->cur_len = 0;
743 vring->rem_len -= len;
744
745 /* Get the next desc on the chain. */
746 if (vring->rem_len > 0 &&
747 (virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) {
748 idx = virtio16_to_cpu(vdev, desc->next);
749 desc = &vr->desc[idx];
750 goto mlxbf_tmfifo_desc_done;
751 }
752
753 /* Done and release the pending packet. */
754 mlxbf_tmfifo_release_pending_pkt(vring);
755 desc = NULL;
756 fifo->vring[is_rx] = NULL;
757
758 /*
759 * Make sure the load/store are in order before
760 * returning back to virtio.
761 */
762 virtio_mb(false);
763
764 /* Notify upper layer that packet is done. */
765 spin_lock_irqsave(&fifo->spin_lock[is_rx], flags);
766 vring_interrupt(0, vring->vq);
767 spin_unlock_irqrestore(&fifo->spin_lock[is_rx], flags);
768 }
769
770mlxbf_tmfifo_desc_done:
771 /* Save the current desc. */
772 vring->desc = desc;
773
774 return true;
775}
776
777/* Rx & Tx processing of a queue. */
778static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx)
779{
780 int avail = 0, devid = vring->vdev_id;
781 struct mlxbf_tmfifo *fifo;
782 bool more;
783
784 fifo = vring->fifo;
785
786 /* Return if vdev is not ready. */
787 if (!fifo->vdev[devid])
788 return;
789
790 /* Return if another vring is running. */
791 if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring)
792 return;
793
794 /* Only handle console and network for now. */
795 if (WARN_ON(devid != VIRTIO_ID_NET && devid != VIRTIO_ID_CONSOLE))
796 return;
797
798 do {
799 /* Get available FIFO space. */
800 if (avail == 0) {
801 if (is_rx)
802 avail = mlxbf_tmfifo_get_rx_avail(fifo);
803 else
804 avail = mlxbf_tmfifo_get_tx_avail(fifo, devid);
805 if (avail <= 0)
806 break;
807 }
808
809 /* Console output always comes from the Tx buffer. */
810 if (!is_rx && devid == VIRTIO_ID_CONSOLE) {
811 mlxbf_tmfifo_console_tx(fifo, avail);
812 break;
813 }
814
815 /* Handle one descriptor. */
816 more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, &avail);
817 } while (more);
818}
819
820/* Handle Rx or Tx queues. */
821static void mlxbf_tmfifo_work_rxtx(struct mlxbf_tmfifo *fifo, int queue_id,
822 int irq_id, bool is_rx)
823{
824 struct mlxbf_tmfifo_vdev *tm_vdev;
825 struct mlxbf_tmfifo_vring *vring;
826 int i;
827
828 if (!test_and_clear_bit(irq_id, &fifo->pend_events) ||
829 !fifo->irq_info[irq_id].irq)
830 return;
831
832 for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++) {
833 tm_vdev = fifo->vdev[i];
834 if (tm_vdev) {
835 vring = &tm_vdev->vrings[queue_id];
836 if (vring->vq)
837 mlxbf_tmfifo_rxtx(vring, is_rx);
838 }
839 }
840}
841
842/* Work handler for Rx and Tx case. */
843static void mlxbf_tmfifo_work_handler(struct work_struct *work)
844{
845 struct mlxbf_tmfifo *fifo;
846
847 fifo = container_of(work, struct mlxbf_tmfifo, work);
848 if (!fifo->is_ready)
849 return;
850
851 mutex_lock(&fifo->lock);
852
853 /* Tx (Send data to the TmFifo). */
854 mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_TX,
855 MLXBF_TM_TX_LWM_IRQ, false);
856
857 /* Rx (Receive data from the TmFifo). */
858 mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_RX,
859 MLXBF_TM_RX_HWM_IRQ, true);
860
861 mutex_unlock(&fifo->lock);
862}
863
864/* The notify function is called when new buffers are posted. */
865static bool mlxbf_tmfifo_virtio_notify(struct virtqueue *vq)
866{
867 struct mlxbf_tmfifo_vring *vring = vq->priv;
868 struct mlxbf_tmfifo_vdev *tm_vdev;
869 struct mlxbf_tmfifo *fifo;
870 unsigned long flags;
871
872 fifo = vring->fifo;
873
874 /*
875 * Virtio maintains vrings in pairs, even number ring for Rx
876 * and odd number ring for Tx.
877 */
878 if (vring->index & BIT(0)) {
879 /*
880 * Console could make blocking call with interrupts disabled.
881 * In such case, the vring needs to be served right away. For
882 * other cases, just set the TX LWM bit to start Tx in the
883 * worker handler.
884 */
885 if (vring->vdev_id == VIRTIO_ID_CONSOLE) {
886 spin_lock_irqsave(&fifo->spin_lock[0], flags);
887 tm_vdev = fifo->vdev[VIRTIO_ID_CONSOLE];
888 mlxbf_tmfifo_console_output(tm_vdev, vring);
889 spin_unlock_irqrestore(&fifo->spin_lock[0], flags);
890 } else if (test_and_set_bit(MLXBF_TM_TX_LWM_IRQ,
891 &fifo->pend_events)) {
892 return true;
893 }
894 } else {
895 if (test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events))
896 return true;
897 }
898
899 schedule_work(&fifo->work);
900
901 return true;
902}
903
904/* Get the array of feature bits for this device. */
905static u64 mlxbf_tmfifo_virtio_get_features(struct virtio_device *vdev)
906{
907 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
908
909 return tm_vdev->features;
910}
911
912/* Confirm device features to use. */
913static int mlxbf_tmfifo_virtio_finalize_features(struct virtio_device *vdev)
914{
915 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
916
917 tm_vdev->features = vdev->features;
918
919 return 0;
920}
921
922/* Free virtqueues found by find_vqs(). */
923static void mlxbf_tmfifo_virtio_del_vqs(struct virtio_device *vdev)
924{
925 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
926 struct mlxbf_tmfifo_vring *vring;
927 struct virtqueue *vq;
928 int i;
929
930 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
931 vring = &tm_vdev->vrings[i];
932
933 /* Release the pending packet. */
934 if (vring->desc)
935 mlxbf_tmfifo_release_pending_pkt(vring);
936 vq = vring->vq;
937 if (vq) {
938 vring->vq = NULL;
939 vring_del_virtqueue(vq);
940 }
941 }
942}
943
944/* Create and initialize the virtual queues. */
945static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev,
946 unsigned int nvqs,
947 struct virtqueue *vqs[],
948 vq_callback_t *callbacks[],
949 const char * const names[],
950 const bool *ctx,
951 struct irq_affinity *desc)
952{
953 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
954 struct mlxbf_tmfifo_vring *vring;
955 struct virtqueue *vq;
956 int i, ret, size;
957
958 if (nvqs > ARRAY_SIZE(tm_vdev->vrings))
959 return -EINVAL;
960
961 for (i = 0; i < nvqs; ++i) {
962 if (!names[i]) {
963 ret = -EINVAL;
964 goto error;
965 }
966 vring = &tm_vdev->vrings[i];
967
968 /* zero vring */
969 size = vring_size(vring->num, vring->align);
970 memset(vring->va, 0, size);
971 vq = vring_new_virtqueue(i, vring->num, vring->align, vdev,
972 false, false, vring->va,
973 mlxbf_tmfifo_virtio_notify,
974 callbacks[i], names[i]);
975 if (!vq) {
976 dev_err(&vdev->dev, "vring_new_virtqueue failed\n");
977 ret = -ENOMEM;
978 goto error;
979 }
980
981 vq->num_max = vring->num;
982
983 vqs[i] = vq;
984 vring->vq = vq;
985 vq->priv = vring;
986 }
987
988 return 0;
989
990error:
991 mlxbf_tmfifo_virtio_del_vqs(vdev);
992 return ret;
993}
994
995/* Read the status byte. */
996static u8 mlxbf_tmfifo_virtio_get_status(struct virtio_device *vdev)
997{
998 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
999
1000 return tm_vdev->status;
1001}
1002
1003/* Write the status byte. */
1004static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
1005 u8 status)
1006{
1007 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1008
1009 tm_vdev->status = status;
1010}
1011
1012/* Reset the device. Not much here for now. */
1013static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
1014{
1015 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1016
1017 tm_vdev->status = 0;
1018}
1019
1020/* Read the value of a configuration field. */
1021static void mlxbf_tmfifo_virtio_get(struct virtio_device *vdev,
1022 unsigned int offset,
1023 void *buf,
1024 unsigned int len)
1025{
1026 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1027
1028 if ((u64)offset + len > sizeof(tm_vdev->config))
1029 return;
1030
1031 memcpy(buf, (u8 *)&tm_vdev->config + offset, len);
1032}
1033
1034/* Write the value of a configuration field. */
1035static void mlxbf_tmfifo_virtio_set(struct virtio_device *vdev,
1036 unsigned int offset,
1037 const void *buf,
1038 unsigned int len)
1039{
1040 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1041
1042 if ((u64)offset + len > sizeof(tm_vdev->config))
1043 return;
1044
1045 memcpy((u8 *)&tm_vdev->config + offset, buf, len);
1046}
1047
1048static void tmfifo_virtio_dev_release(struct device *device)
1049{
1050 struct virtio_device *vdev =
1051 container_of(device, struct virtio_device, dev);
1052 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1053
1054 kfree(tm_vdev);
1055}
1056
1057/* Virtio config operations. */
1058static const struct virtio_config_ops mlxbf_tmfifo_virtio_config_ops = {
1059 .get_features = mlxbf_tmfifo_virtio_get_features,
1060 .finalize_features = mlxbf_tmfifo_virtio_finalize_features,
1061 .find_vqs = mlxbf_tmfifo_virtio_find_vqs,
1062 .del_vqs = mlxbf_tmfifo_virtio_del_vqs,
1063 .reset = mlxbf_tmfifo_virtio_reset,
1064 .set_status = mlxbf_tmfifo_virtio_set_status,
1065 .get_status = mlxbf_tmfifo_virtio_get_status,
1066 .get = mlxbf_tmfifo_virtio_get,
1067 .set = mlxbf_tmfifo_virtio_set,
1068};
1069
1070/* Create vdev for the FIFO. */
1071static int mlxbf_tmfifo_create_vdev(struct device *dev,
1072 struct mlxbf_tmfifo *fifo,
1073 int vdev_id, u64 features,
1074 void *config, u32 size)
1075{
1076 struct mlxbf_tmfifo_vdev *tm_vdev, *reg_dev = NULL;
1077 int ret;
1078
1079 mutex_lock(&fifo->lock);
1080
1081 tm_vdev = fifo->vdev[vdev_id];
1082 if (tm_vdev) {
1083 dev_err(dev, "vdev %d already exists\n", vdev_id);
1084 ret = -EEXIST;
1085 goto fail;
1086 }
1087
1088 tm_vdev = kzalloc(sizeof(*tm_vdev), GFP_KERNEL);
1089 if (!tm_vdev) {
1090 ret = -ENOMEM;
1091 goto fail;
1092 }
1093
1094 tm_vdev->vdev.id.device = vdev_id;
1095 tm_vdev->vdev.config = &mlxbf_tmfifo_virtio_config_ops;
1096 tm_vdev->vdev.dev.parent = dev;
1097 tm_vdev->vdev.dev.release = tmfifo_virtio_dev_release;
1098 tm_vdev->features = features;
1099 if (config)
1100 memcpy(&tm_vdev->config, config, size);
1101
1102 if (mlxbf_tmfifo_alloc_vrings(fifo, tm_vdev)) {
1103 dev_err(dev, "unable to allocate vring\n");
1104 ret = -ENOMEM;
1105 goto vdev_fail;
1106 }
1107
1108 /* Allocate an output buffer for the console device. */
1109 if (vdev_id == VIRTIO_ID_CONSOLE)
1110 tm_vdev->tx_buf.buf = devm_kmalloc(dev,
1111 MLXBF_TMFIFO_CON_TX_BUF_SIZE,
1112 GFP_KERNEL);
1113 fifo->vdev[vdev_id] = tm_vdev;
1114
1115 /* Register the virtio device. */
1116 ret = register_virtio_device(&tm_vdev->vdev);
1117 reg_dev = tm_vdev;
1118 if (ret) {
1119 dev_err(dev, "register_virtio_device failed\n");
1120 goto vdev_fail;
1121 }
1122
1123 mutex_unlock(&fifo->lock);
1124 return 0;
1125
1126vdev_fail:
1127 mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
1128 fifo->vdev[vdev_id] = NULL;
1129 if (reg_dev)
1130 put_device(&tm_vdev->vdev.dev);
1131 else
1132 kfree(tm_vdev);
1133fail:
1134 mutex_unlock(&fifo->lock);
1135 return ret;
1136}
1137
1138/* Delete vdev for the FIFO. */
1139static int mlxbf_tmfifo_delete_vdev(struct mlxbf_tmfifo *fifo, int vdev_id)
1140{
1141 struct mlxbf_tmfifo_vdev *tm_vdev;
1142
1143 mutex_lock(&fifo->lock);
1144
1145 /* Unregister vdev. */
1146 tm_vdev = fifo->vdev[vdev_id];
1147 if (tm_vdev) {
1148 unregister_virtio_device(&tm_vdev->vdev);
1149 mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
1150 fifo->vdev[vdev_id] = NULL;
1151 }
1152
1153 mutex_unlock(&fifo->lock);
1154
1155 return 0;
1156}
1157
1158/* Read the configured network MAC address from efi variable. */
1159static void mlxbf_tmfifo_get_cfg_mac(u8 *mac)
1160{
1161 efi_guid_t guid = EFI_GLOBAL_VARIABLE_GUID;
1162 unsigned long size = ETH_ALEN;
1163 u8 buf[ETH_ALEN];
1164 efi_status_t rc;
1165
1166 rc = efi.get_variable(mlxbf_tmfifo_efi_name, &guid, NULL, &size, buf);
1167 if (rc == EFI_SUCCESS && size == ETH_ALEN)
1168 ether_addr_copy(mac, buf);
1169 else
1170 ether_addr_copy(mac, mlxbf_tmfifo_net_default_mac);
1171}
1172
1173/* Set TmFifo thresolds which is used to trigger interrupts. */
1174static void mlxbf_tmfifo_set_threshold(struct mlxbf_tmfifo *fifo)
1175{
1176 u64 ctl;
1177
1178 /* Get Tx FIFO size and set the low/high watermark. */
1179 ctl = readq(fifo->tx.ctl);
1180 fifo->tx_fifo_size =
1181 FIELD_GET(MLXBF_TMFIFO_TX_CTL__MAX_ENTRIES_MASK, ctl);
1182 ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__LWM_MASK) |
1183 FIELD_PREP(MLXBF_TMFIFO_TX_CTL__LWM_MASK,
1184 fifo->tx_fifo_size / 2);
1185 ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__HWM_MASK) |
1186 FIELD_PREP(MLXBF_TMFIFO_TX_CTL__HWM_MASK,
1187 fifo->tx_fifo_size - 1);
1188 writeq(ctl, fifo->tx.ctl);
1189
1190 /* Get Rx FIFO size and set the low/high watermark. */
1191 ctl = readq(fifo->rx.ctl);
1192 fifo->rx_fifo_size =
1193 FIELD_GET(MLXBF_TMFIFO_RX_CTL__MAX_ENTRIES_MASK, ctl);
1194 ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__LWM_MASK) |
1195 FIELD_PREP(MLXBF_TMFIFO_RX_CTL__LWM_MASK, 0);
1196 ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__HWM_MASK) |
1197 FIELD_PREP(MLXBF_TMFIFO_RX_CTL__HWM_MASK, 1);
1198 writeq(ctl, fifo->rx.ctl);
1199}
1200
1201static void mlxbf_tmfifo_cleanup(struct mlxbf_tmfifo *fifo)
1202{
1203 int i;
1204
1205 fifo->is_ready = false;
1206 del_timer_sync(&fifo->timer);
1207 mlxbf_tmfifo_disable_irqs(fifo);
1208 cancel_work_sync(&fifo->work);
1209 for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++)
1210 mlxbf_tmfifo_delete_vdev(fifo, i);
1211}
1212
1213/* Probe the TMFIFO. */
1214static int mlxbf_tmfifo_probe(struct platform_device *pdev)
1215{
1216 struct virtio_net_config net_config;
1217 struct device *dev = &pdev->dev;
1218 struct mlxbf_tmfifo *fifo;
1219 u64 dev_id;
1220 int i, rc;
1221
1222 rc = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &dev_id);
1223 if (rc) {
1224 dev_err(dev, "Cannot retrieve UID\n");
1225 return rc;
1226 }
1227
1228 fifo = devm_kzalloc(dev, sizeof(*fifo), GFP_KERNEL);
1229 if (!fifo)
1230 return -ENOMEM;
1231
1232 spin_lock_init(&fifo->spin_lock[0]);
1233 spin_lock_init(&fifo->spin_lock[1]);
1234 INIT_WORK(&fifo->work, mlxbf_tmfifo_work_handler);
1235 mutex_init(&fifo->lock);
1236
1237 /* Get the resource of the Rx FIFO. */
1238 fifo->res0 = devm_platform_ioremap_resource(pdev, 0);
1239 if (IS_ERR(fifo->res0))
1240 return PTR_ERR(fifo->res0);
1241
1242 /* Get the resource of the Tx FIFO. */
1243 fifo->res1 = devm_platform_ioremap_resource(pdev, 1);
1244 if (IS_ERR(fifo->res1))
1245 return PTR_ERR(fifo->res1);
1246
1247 if (dev_id == TMFIFO_BF3_UID) {
1248 fifo->rx.ctl = fifo->res1 + MLXBF_TMFIFO_RX_CTL_BF3;
1249 fifo->rx.sts = fifo->res1 + MLXBF_TMFIFO_RX_STS_BF3;
1250 fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA_BF3;
1251 fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL_BF3;
1252 fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS_BF3;
1253 fifo->tx.data = fifo->res0 + MLXBF_TMFIFO_TX_DATA_BF3;
1254 } else {
1255 fifo->rx.ctl = fifo->res0 + MLXBF_TMFIFO_RX_CTL;
1256 fifo->rx.sts = fifo->res0 + MLXBF_TMFIFO_RX_STS;
1257 fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA;
1258 fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL;
1259 fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS;
1260 fifo->tx.data = fifo->res1 + MLXBF_TMFIFO_TX_DATA;
1261 }
1262
1263 platform_set_drvdata(pdev, fifo);
1264
1265 timer_setup(&fifo->timer, mlxbf_tmfifo_timer, 0);
1266
1267 for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) {
1268 fifo->irq_info[i].index = i;
1269 fifo->irq_info[i].fifo = fifo;
1270 fifo->irq_info[i].irq = platform_get_irq(pdev, i);
1271 rc = devm_request_irq(dev, fifo->irq_info[i].irq,
1272 mlxbf_tmfifo_irq_handler, 0,
1273 "tmfifo", &fifo->irq_info[i]);
1274 if (rc) {
1275 dev_err(dev, "devm_request_irq failed\n");
1276 fifo->irq_info[i].irq = 0;
1277 return rc;
1278 }
1279 }
1280
1281 mlxbf_tmfifo_set_threshold(fifo);
1282
1283 /* Create the console vdev. */
1284 rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_CONSOLE, 0, NULL, 0);
1285 if (rc)
1286 goto fail;
1287
1288 /* Create the network vdev. */
1289 memset(&net_config, 0, sizeof(net_config));
1290
1291 /* A legacy-only interface for now. */
1292 net_config.mtu = __cpu_to_virtio16(virtio_legacy_is_little_endian(),
1293 ETH_DATA_LEN);
1294 net_config.status = __cpu_to_virtio16(virtio_legacy_is_little_endian(),
1295 VIRTIO_NET_S_LINK_UP);
1296 mlxbf_tmfifo_get_cfg_mac(net_config.mac);
1297 rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_NET,
1298 MLXBF_TMFIFO_NET_FEATURES, &net_config,
1299 sizeof(net_config));
1300 if (rc)
1301 goto fail;
1302
1303 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL);
1304
1305 fifo->is_ready = true;
1306 return 0;
1307
1308fail:
1309 mlxbf_tmfifo_cleanup(fifo);
1310 return rc;
1311}
1312
1313/* Device remove function. */
1314static int mlxbf_tmfifo_remove(struct platform_device *pdev)
1315{
1316 struct mlxbf_tmfifo *fifo = platform_get_drvdata(pdev);
1317
1318 mlxbf_tmfifo_cleanup(fifo);
1319
1320 return 0;
1321}
1322
1323static const struct acpi_device_id mlxbf_tmfifo_acpi_match[] = {
1324 { "MLNXBF01", 0 },
1325 {}
1326};
1327MODULE_DEVICE_TABLE(acpi, mlxbf_tmfifo_acpi_match);
1328
1329static struct platform_driver mlxbf_tmfifo_driver = {
1330 .probe = mlxbf_tmfifo_probe,
1331 .remove = mlxbf_tmfifo_remove,
1332 .driver = {
1333 .name = "bf-tmfifo",
1334 .acpi_match_table = mlxbf_tmfifo_acpi_match,
1335 },
1336};
1337
1338module_platform_driver(mlxbf_tmfifo_driver);
1339
1340MODULE_DESCRIPTION("Mellanox BlueField SoC TmFifo Driver");
1341MODULE_LICENSE("GPL v2");
1342MODULE_AUTHOR("Mellanox Technologies");