Loading...
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Mellanox BlueField SoC TmFifo driver
4 *
5 * Copyright (C) 2019 Mellanox Technologies
6 */
7
8#include <linux/acpi.h>
9#include <linux/bitfield.h>
10#include <linux/circ_buf.h>
11#include <linux/efi.h>
12#include <linux/irq.h>
13#include <linux/module.h>
14#include <linux/mutex.h>
15#include <linux/platform_device.h>
16#include <linux/types.h>
17
18#include <linux/virtio_config.h>
19#include <linux/virtio_console.h>
20#include <linux/virtio_ids.h>
21#include <linux/virtio_net.h>
22#include <linux/virtio_ring.h>
23
24#include "mlxbf-tmfifo-regs.h"
25
26/* Vring size. */
27#define MLXBF_TMFIFO_VRING_SIZE SZ_1K
28
29/* Console Tx buffer size. */
30#define MLXBF_TMFIFO_CON_TX_BUF_SIZE SZ_32K
31
32/* Console Tx buffer reserved space. */
33#define MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE 8
34
35/* House-keeping timer interval. */
36#define MLXBF_TMFIFO_TIMER_INTERVAL (HZ / 10)
37
38/* Virtual devices sharing the TM FIFO. */
39#define MLXBF_TMFIFO_VDEV_MAX (VIRTIO_ID_CONSOLE + 1)
40
41/*
42 * Reserve 1/16 of TmFifo space, so console messages are not starved by
43 * the networking traffic.
44 */
45#define MLXBF_TMFIFO_RESERVE_RATIO 16
46
47/* Message with data needs at least two words (for header & data). */
48#define MLXBF_TMFIFO_DATA_MIN_WORDS 2
49
50/* Tx timeout in milliseconds. */
51#define TMFIFO_TX_TIMEOUT 2000
52
53/* ACPI UID for BlueField-3. */
54#define TMFIFO_BF3_UID 1
55
56struct mlxbf_tmfifo;
57
58/**
59 * struct mlxbf_tmfifo_vring - Structure of the TmFifo virtual ring
60 * @va: virtual address of the ring
61 * @dma: dma address of the ring
62 * @vq: pointer to the virtio virtqueue
63 * @desc: current descriptor of the pending packet
64 * @desc_head: head descriptor of the pending packet
65 * @drop_desc: dummy desc for packet dropping
66 * @cur_len: processed length of the current descriptor
67 * @rem_len: remaining length of the pending packet
68 * @rem_padding: remaining bytes to send as paddings
69 * @pkt_len: total length of the pending packet
70 * @next_avail: next avail descriptor id
71 * @num: vring size (number of descriptors)
72 * @align: vring alignment size
73 * @index: vring index
74 * @vdev_id: vring virtio id (VIRTIO_ID_xxx)
75 * @tx_timeout: expire time of last tx packet
76 * @fifo: pointer to the tmfifo structure
77 */
78struct mlxbf_tmfifo_vring {
79 void *va;
80 dma_addr_t dma;
81 struct virtqueue *vq;
82 struct vring_desc *desc;
83 struct vring_desc *desc_head;
84 struct vring_desc drop_desc;
85 int cur_len;
86 int rem_len;
87 int rem_padding;
88 u32 pkt_len;
89 u16 next_avail;
90 int num;
91 int align;
92 int index;
93 int vdev_id;
94 unsigned long tx_timeout;
95 struct mlxbf_tmfifo *fifo;
96};
97
98/* Check whether vring is in drop mode. */
99#define IS_VRING_DROP(_r) ({ \
100 typeof(_r) (r) = (_r); \
101 r->desc_head == &r->drop_desc; })
102
103/* A stub length to drop maximum length packet. */
104#define VRING_DROP_DESC_MAX_LEN GENMASK(15, 0)
105
106/* Interrupt types. */
107enum {
108 MLXBF_TM_RX_LWM_IRQ,
109 MLXBF_TM_RX_HWM_IRQ,
110 MLXBF_TM_TX_LWM_IRQ,
111 MLXBF_TM_TX_HWM_IRQ,
112 MLXBF_TM_MAX_IRQ
113};
114
115/* Ring types (Rx & Tx). */
116enum {
117 MLXBF_TMFIFO_VRING_RX,
118 MLXBF_TMFIFO_VRING_TX,
119 MLXBF_TMFIFO_VRING_MAX
120};
121
122/**
123 * struct mlxbf_tmfifo_vdev - Structure of the TmFifo virtual device
124 * @vdev: virtio device, in which the vdev.id.device field has the
125 * VIRTIO_ID_xxx id to distinguish the virtual device.
126 * @status: status of the device
127 * @features: supported features of the device
128 * @vrings: array of tmfifo vrings of this device
129 * @config: non-anonymous union for cons and net
130 * @config.cons: virtual console config -
131 * select if vdev.id.device is VIRTIO_ID_CONSOLE
132 * @config.net: virtual network config -
133 * select if vdev.id.device is VIRTIO_ID_NET
134 * @tx_buf: tx buffer used to buffer data before writing into the FIFO
135 */
136struct mlxbf_tmfifo_vdev {
137 struct virtio_device vdev;
138 u8 status;
139 u64 features;
140 struct mlxbf_tmfifo_vring vrings[MLXBF_TMFIFO_VRING_MAX];
141 union {
142 struct virtio_console_config cons;
143 struct virtio_net_config net;
144 } config;
145 struct circ_buf tx_buf;
146};
147
148/**
149 * struct mlxbf_tmfifo_irq_info - Structure of the interrupt information
150 * @fifo: pointer to the tmfifo structure
151 * @irq: interrupt number
152 * @index: index into the interrupt array
153 */
154struct mlxbf_tmfifo_irq_info {
155 struct mlxbf_tmfifo *fifo;
156 int irq;
157 int index;
158};
159
160/**
161 * struct mlxbf_tmfifo_io - Structure of the TmFifo IO resource (for both rx & tx)
162 * @ctl: control register offset (TMFIFO_RX_CTL / TMFIFO_TX_CTL)
163 * @sts: status register offset (TMFIFO_RX_STS / TMFIFO_TX_STS)
164 * @data: data register offset (TMFIFO_RX_DATA / TMFIFO_TX_DATA)
165 */
166struct mlxbf_tmfifo_io {
167 void __iomem *ctl;
168 void __iomem *sts;
169 void __iomem *data;
170};
171
172/**
173 * struct mlxbf_tmfifo - Structure of the TmFifo
174 * @vdev: array of the virtual devices running over the TmFifo
175 * @lock: lock to protect the TmFifo access
176 * @res0: mapped resource block 0
177 * @res1: mapped resource block 1
178 * @rx: rx io resource
179 * @tx: tx io resource
180 * @rx_fifo_size: number of entries of the Rx FIFO
181 * @tx_fifo_size: number of entries of the Tx FIFO
182 * @pend_events: pending bits for deferred events
183 * @irq_info: interrupt information
184 * @work: work struct for deferred process
185 * @timer: background timer
186 * @vring: Tx/Rx ring
187 * @spin_lock: Tx/Rx spin lock
188 * @is_ready: ready flag
189 */
190struct mlxbf_tmfifo {
191 struct mlxbf_tmfifo_vdev *vdev[MLXBF_TMFIFO_VDEV_MAX];
192 struct mutex lock; /* TmFifo lock */
193 void __iomem *res0;
194 void __iomem *res1;
195 struct mlxbf_tmfifo_io rx;
196 struct mlxbf_tmfifo_io tx;
197 int rx_fifo_size;
198 int tx_fifo_size;
199 unsigned long pend_events;
200 struct mlxbf_tmfifo_irq_info irq_info[MLXBF_TM_MAX_IRQ];
201 struct work_struct work;
202 struct timer_list timer;
203 struct mlxbf_tmfifo_vring *vring[2];
204 spinlock_t spin_lock[2]; /* spin lock */
205 bool is_ready;
206};
207
208/**
209 * struct mlxbf_tmfifo_msg_hdr - Structure of the TmFifo message header
210 * @type: message type
211 * @len: payload length in network byte order. Messages sent into the FIFO
212 * will be read by the other side as data stream in the same byte order.
213 * The length needs to be encoded into network order so both sides
214 * could understand it.
215 */
216struct mlxbf_tmfifo_msg_hdr {
217 u8 type;
218 __be16 len;
219 /* private: */
220 u8 unused[5];
221} __packed __aligned(sizeof(u64));
222
223/*
224 * Default MAC.
225 * This MAC address will be read from EFI persistent variable if configured.
226 * It can also be reconfigured with standard Linux tools.
227 */
228static u8 mlxbf_tmfifo_net_default_mac[ETH_ALEN] = {
229 0x00, 0x1A, 0xCA, 0xFF, 0xFF, 0x01
230};
231
232/* EFI variable name of the MAC address. */
233static efi_char16_t mlxbf_tmfifo_efi_name[] = L"RshimMacAddr";
234
235/* Maximum L2 header length. */
236#define MLXBF_TMFIFO_NET_L2_OVERHEAD (ETH_HLEN + VLAN_HLEN)
237
238/* Supported virtio-net features. */
239#define MLXBF_TMFIFO_NET_FEATURES \
240 (BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_STATUS) | \
241 BIT_ULL(VIRTIO_NET_F_MAC))
242
243#define mlxbf_vdev_to_tmfifo(d) container_of(d, struct mlxbf_tmfifo_vdev, vdev)
244
245/* Free vrings of the FIFO device. */
246static void mlxbf_tmfifo_free_vrings(struct mlxbf_tmfifo *fifo,
247 struct mlxbf_tmfifo_vdev *tm_vdev)
248{
249 struct mlxbf_tmfifo_vring *vring;
250 int i, size;
251
252 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
253 vring = &tm_vdev->vrings[i];
254 if (vring->va) {
255 size = vring_size(vring->num, vring->align);
256 dma_free_coherent(tm_vdev->vdev.dev.parent, size,
257 vring->va, vring->dma);
258 vring->va = NULL;
259 if (vring->vq) {
260 vring_del_virtqueue(vring->vq);
261 vring->vq = NULL;
262 }
263 }
264 }
265}
266
267/* Allocate vrings for the FIFO. */
268static int mlxbf_tmfifo_alloc_vrings(struct mlxbf_tmfifo *fifo,
269 struct mlxbf_tmfifo_vdev *tm_vdev)
270{
271 struct mlxbf_tmfifo_vring *vring;
272 struct device *dev;
273 dma_addr_t dma;
274 int i, size;
275 void *va;
276
277 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
278 vring = &tm_vdev->vrings[i];
279 vring->fifo = fifo;
280 vring->num = MLXBF_TMFIFO_VRING_SIZE;
281 vring->align = SMP_CACHE_BYTES;
282 vring->index = i;
283 vring->vdev_id = tm_vdev->vdev.id.device;
284 vring->drop_desc.len = VRING_DROP_DESC_MAX_LEN;
285 dev = &tm_vdev->vdev.dev;
286
287 size = vring_size(vring->num, vring->align);
288 va = dma_alloc_coherent(dev->parent, size, &dma, GFP_KERNEL);
289 if (!va) {
290 mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
291 dev_err(dev->parent, "dma_alloc_coherent failed\n");
292 return -ENOMEM;
293 }
294
295 vring->va = va;
296 vring->dma = dma;
297 }
298
299 return 0;
300}
301
302/* Disable interrupts of the FIFO device. */
303static void mlxbf_tmfifo_disable_irqs(struct mlxbf_tmfifo *fifo)
304{
305 int i, irq;
306
307 for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) {
308 irq = fifo->irq_info[i].irq;
309 fifo->irq_info[i].irq = 0;
310 disable_irq(irq);
311 }
312}
313
314/* Interrupt handler. */
315static irqreturn_t mlxbf_tmfifo_irq_handler(int irq, void *arg)
316{
317 struct mlxbf_tmfifo_irq_info *irq_info = arg;
318
319 if (!test_and_set_bit(irq_info->index, &irq_info->fifo->pend_events))
320 schedule_work(&irq_info->fifo->work);
321
322 return IRQ_HANDLED;
323}
324
325/* Get the next packet descriptor from the vring. */
326static struct vring_desc *
327mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring)
328{
329 const struct vring *vr = virtqueue_get_vring(vring->vq);
330 struct virtio_device *vdev = vring->vq->vdev;
331 unsigned int idx, head;
332
333 if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx))
334 return NULL;
335
336 /* Make sure 'avail->idx' is visible already. */
337 virtio_rmb(false);
338
339 idx = vring->next_avail % vr->num;
340 head = virtio16_to_cpu(vdev, vr->avail->ring[idx]);
341 if (WARN_ON(head >= vr->num))
342 return NULL;
343
344 vring->next_avail++;
345
346 return &vr->desc[head];
347}
348
349/* Release virtio descriptor. */
350static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring,
351 struct vring_desc *desc, u32 len)
352{
353 const struct vring *vr = virtqueue_get_vring(vring->vq);
354 struct virtio_device *vdev = vring->vq->vdev;
355 u16 idx, vr_idx;
356
357 vr_idx = virtio16_to_cpu(vdev, vr->used->idx);
358 idx = vr_idx % vr->num;
359 vr->used->ring[idx].id = cpu_to_virtio32(vdev, desc - vr->desc);
360 vr->used->ring[idx].len = cpu_to_virtio32(vdev, len);
361
362 /*
363 * Virtio could poll and check the 'idx' to decide whether the desc is
364 * done or not. Add a memory barrier here to make sure the update above
365 * completes before updating the idx.
366 */
367 virtio_mb(false);
368 vr->used->idx = cpu_to_virtio16(vdev, vr_idx + 1);
369}
370
371/* Get the total length of the descriptor chain. */
372static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring,
373 struct vring_desc *desc)
374{
375 const struct vring *vr = virtqueue_get_vring(vring->vq);
376 struct virtio_device *vdev = vring->vq->vdev;
377 u32 len = 0, idx;
378
379 while (desc) {
380 len += virtio32_to_cpu(vdev, desc->len);
381 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT))
382 break;
383 idx = virtio16_to_cpu(vdev, desc->next);
384 desc = &vr->desc[idx];
385 }
386
387 return len;
388}
389
390static void mlxbf_tmfifo_release_pkt(struct mlxbf_tmfifo_vring *vring)
391{
392 struct vring_desc *desc_head;
393 u32 len = 0;
394
395 if (vring->desc_head) {
396 desc_head = vring->desc_head;
397 len = vring->pkt_len;
398 } else {
399 desc_head = mlxbf_tmfifo_get_next_desc(vring);
400 len = mlxbf_tmfifo_get_pkt_len(vring, desc_head);
401 }
402
403 if (desc_head)
404 mlxbf_tmfifo_release_desc(vring, desc_head, len);
405
406 vring->pkt_len = 0;
407 vring->desc = NULL;
408 vring->desc_head = NULL;
409}
410
411static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring,
412 struct vring_desc *desc, bool is_rx)
413{
414 struct virtio_device *vdev = vring->vq->vdev;
415 struct virtio_net_hdr *net_hdr;
416
417 net_hdr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
418 memset(net_hdr, 0, sizeof(*net_hdr));
419}
420
421/* Get and initialize the next packet. */
422static struct vring_desc *
423mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx)
424{
425 struct vring_desc *desc;
426
427 desc = mlxbf_tmfifo_get_next_desc(vring);
428 if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET)
429 mlxbf_tmfifo_init_net_desc(vring, desc, is_rx);
430
431 vring->desc_head = desc;
432 vring->desc = desc;
433
434 return desc;
435}
436
437/* House-keeping timer. */
438static void mlxbf_tmfifo_timer(struct timer_list *t)
439{
440 struct mlxbf_tmfifo *fifo = container_of(t, struct mlxbf_tmfifo, timer);
441 int rx, tx;
442
443 rx = !test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events);
444 tx = !test_and_set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events);
445
446 if (rx || tx)
447 schedule_work(&fifo->work);
448
449 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL);
450}
451
452/* Copy one console packet into the output buffer. */
453static void mlxbf_tmfifo_console_output_one(struct mlxbf_tmfifo_vdev *cons,
454 struct mlxbf_tmfifo_vring *vring,
455 struct vring_desc *desc)
456{
457 const struct vring *vr = virtqueue_get_vring(vring->vq);
458 struct virtio_device *vdev = &cons->vdev;
459 u32 len, idx, seg;
460 void *addr;
461
462 while (desc) {
463 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
464 len = virtio32_to_cpu(vdev, desc->len);
465
466 seg = CIRC_SPACE_TO_END(cons->tx_buf.head, cons->tx_buf.tail,
467 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
468 if (len <= seg) {
469 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, len);
470 } else {
471 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, seg);
472 addr += seg;
473 memcpy(cons->tx_buf.buf, addr, len - seg);
474 }
475 cons->tx_buf.head = (cons->tx_buf.head + len) %
476 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
477
478 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT))
479 break;
480 idx = virtio16_to_cpu(vdev, desc->next);
481 desc = &vr->desc[idx];
482 }
483}
484
485/* Copy console data into the output buffer. */
486static void mlxbf_tmfifo_console_output(struct mlxbf_tmfifo_vdev *cons,
487 struct mlxbf_tmfifo_vring *vring)
488{
489 struct vring_desc *desc;
490 u32 len, avail;
491
492 desc = mlxbf_tmfifo_get_next_desc(vring);
493 while (desc) {
494 /* Release the packet if not enough space. */
495 len = mlxbf_tmfifo_get_pkt_len(vring, desc);
496 avail = CIRC_SPACE(cons->tx_buf.head, cons->tx_buf.tail,
497 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
498 if (len + MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE > avail) {
499 mlxbf_tmfifo_release_desc(vring, desc, len);
500 break;
501 }
502
503 mlxbf_tmfifo_console_output_one(cons, vring, desc);
504 mlxbf_tmfifo_release_desc(vring, desc, len);
505 desc = mlxbf_tmfifo_get_next_desc(vring);
506 }
507}
508
509/* Get the number of available words in Rx FIFO for receiving. */
510static int mlxbf_tmfifo_get_rx_avail(struct mlxbf_tmfifo *fifo)
511{
512 u64 sts;
513
514 sts = readq(fifo->rx.sts);
515 return FIELD_GET(MLXBF_TMFIFO_RX_STS__COUNT_MASK, sts);
516}
517
518/* Get the number of available words in the TmFifo for sending. */
519static int mlxbf_tmfifo_get_tx_avail(struct mlxbf_tmfifo *fifo, int vdev_id)
520{
521 int tx_reserve;
522 u32 count;
523 u64 sts;
524
525 /* Reserve some room in FIFO for console messages. */
526 if (vdev_id == VIRTIO_ID_NET)
527 tx_reserve = fifo->tx_fifo_size / MLXBF_TMFIFO_RESERVE_RATIO;
528 else
529 tx_reserve = 1;
530
531 sts = readq(fifo->tx.sts);
532 count = FIELD_GET(MLXBF_TMFIFO_TX_STS__COUNT_MASK, sts);
533 return fifo->tx_fifo_size - tx_reserve - count;
534}
535
536/* Console Tx (move data from the output buffer into the TmFifo). */
537static void mlxbf_tmfifo_console_tx(struct mlxbf_tmfifo *fifo, int avail)
538{
539 struct mlxbf_tmfifo_msg_hdr hdr;
540 struct mlxbf_tmfifo_vdev *cons;
541 unsigned long flags;
542 int size, seg;
543 void *addr;
544 u64 data;
545
546 /* Return if not enough space available. */
547 if (avail < MLXBF_TMFIFO_DATA_MIN_WORDS)
548 return;
549
550 cons = fifo->vdev[VIRTIO_ID_CONSOLE];
551 if (!cons || !cons->tx_buf.buf)
552 return;
553
554 /* Return if no data to send. */
555 size = CIRC_CNT(cons->tx_buf.head, cons->tx_buf.tail,
556 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
557 if (size == 0)
558 return;
559
560 /* Adjust the size to available space. */
561 if (size + sizeof(hdr) > avail * sizeof(u64))
562 size = avail * sizeof(u64) - sizeof(hdr);
563
564 /* Write header. */
565 hdr.type = VIRTIO_ID_CONSOLE;
566 hdr.len = htons(size);
567 writeq(*(u64 *)&hdr, fifo->tx.data);
568
569 /* Use spin-lock to protect the 'cons->tx_buf'. */
570 spin_lock_irqsave(&fifo->spin_lock[0], flags);
571
572 while (size > 0) {
573 addr = cons->tx_buf.buf + cons->tx_buf.tail;
574
575 seg = CIRC_CNT_TO_END(cons->tx_buf.head, cons->tx_buf.tail,
576 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
577 if (seg >= sizeof(u64)) {
578 memcpy(&data, addr, sizeof(u64));
579 } else {
580 memcpy(&data, addr, seg);
581 memcpy((u8 *)&data + seg, cons->tx_buf.buf,
582 sizeof(u64) - seg);
583 }
584 writeq(data, fifo->tx.data);
585
586 if (size >= sizeof(u64)) {
587 cons->tx_buf.tail = (cons->tx_buf.tail + sizeof(u64)) %
588 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
589 size -= sizeof(u64);
590 } else {
591 cons->tx_buf.tail = (cons->tx_buf.tail + size) %
592 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
593 size = 0;
594 }
595 }
596
597 spin_unlock_irqrestore(&fifo->spin_lock[0], flags);
598}
599
600/* Rx/Tx one word in the descriptor buffer. */
601static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring,
602 struct vring_desc *desc,
603 bool is_rx, int len)
604{
605 struct virtio_device *vdev = vring->vq->vdev;
606 struct mlxbf_tmfifo *fifo = vring->fifo;
607 void *addr;
608 u64 data;
609
610 /* Get the buffer address of this desc. */
611 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
612
613 /* Read a word from FIFO for Rx. */
614 if (is_rx)
615 data = readq(fifo->rx.data);
616
617 if (vring->cur_len + sizeof(u64) <= len) {
618 /* The whole word. */
619 if (is_rx) {
620 if (!IS_VRING_DROP(vring))
621 memcpy(addr + vring->cur_len, &data,
622 sizeof(u64));
623 } else {
624 memcpy(&data, addr + vring->cur_len,
625 sizeof(u64));
626 }
627 vring->cur_len += sizeof(u64);
628 } else {
629 /* Leftover bytes. */
630 if (is_rx) {
631 if (!IS_VRING_DROP(vring))
632 memcpy(addr + vring->cur_len, &data,
633 len - vring->cur_len);
634 } else {
635 data = 0;
636 memcpy(&data, addr + vring->cur_len,
637 len - vring->cur_len);
638 }
639 vring->cur_len = len;
640 }
641
642 /* Write the word into FIFO for Tx. */
643 if (!is_rx)
644 writeq(data, fifo->tx.data);
645}
646
647/*
648 * Rx/Tx packet header.
649 *
650 * In Rx case, the packet might be found to belong to a different vring since
651 * the TmFifo is shared by different services. In such case, the 'vring_change'
652 * flag is set.
653 */
654static void mlxbf_tmfifo_rxtx_header(struct mlxbf_tmfifo_vring *vring,
655 struct vring_desc **desc,
656 bool is_rx, bool *vring_change)
657{
658 struct mlxbf_tmfifo *fifo = vring->fifo;
659 struct virtio_net_config *config;
660 struct mlxbf_tmfifo_msg_hdr hdr;
661 int vdev_id, hdr_len;
662 bool drop_rx = false;
663
664 /* Read/Write packet header. */
665 if (is_rx) {
666 /* Drain one word from the FIFO. */
667 *(u64 *)&hdr = readq(fifo->rx.data);
668
669 /* Skip the length 0 packets (keepalive). */
670 if (hdr.len == 0)
671 return;
672
673 /* Check packet type. */
674 if (hdr.type == VIRTIO_ID_NET) {
675 vdev_id = VIRTIO_ID_NET;
676 hdr_len = sizeof(struct virtio_net_hdr);
677 config = &fifo->vdev[vdev_id]->config.net;
678 /* A legacy-only interface for now. */
679 if (ntohs(hdr.len) >
680 __virtio16_to_cpu(virtio_legacy_is_little_endian(),
681 config->mtu) +
682 MLXBF_TMFIFO_NET_L2_OVERHEAD)
683 drop_rx = true;
684 } else {
685 vdev_id = VIRTIO_ID_CONSOLE;
686 hdr_len = 0;
687 }
688
689 /*
690 * Check whether the new packet still belongs to this vring.
691 * If not, update the pkt_len of the new vring.
692 */
693 if (vdev_id != vring->vdev_id) {
694 struct mlxbf_tmfifo_vdev *tm_dev2 = fifo->vdev[vdev_id];
695
696 if (!tm_dev2)
697 return;
698 vring->desc = *desc;
699 vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX];
700 *vring_change = true;
701 }
702
703 if (drop_rx && !IS_VRING_DROP(vring)) {
704 if (vring->desc_head)
705 mlxbf_tmfifo_release_pkt(vring);
706 *desc = &vring->drop_desc;
707 vring->desc_head = *desc;
708 vring->desc = *desc;
709 }
710
711 vring->pkt_len = ntohs(hdr.len) + hdr_len;
712 } else {
713 /* Network virtio has an extra header. */
714 hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ?
715 sizeof(struct virtio_net_hdr) : 0;
716 vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, *desc);
717 hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ?
718 VIRTIO_ID_NET : VIRTIO_ID_CONSOLE;
719 hdr.len = htons(vring->pkt_len - hdr_len);
720 writeq(*(u64 *)&hdr, fifo->tx.data);
721 }
722
723 vring->cur_len = hdr_len;
724 vring->rem_len = vring->pkt_len;
725 fifo->vring[is_rx] = vring;
726}
727
728/*
729 * Rx/Tx one descriptor.
730 *
731 * Return true to indicate more data available.
732 */
733static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring,
734 bool is_rx, int *avail)
735{
736 const struct vring *vr = virtqueue_get_vring(vring->vq);
737 struct mlxbf_tmfifo *fifo = vring->fifo;
738 struct virtio_device *vdev;
739 bool vring_change = false;
740 struct vring_desc *desc;
741 unsigned long flags;
742 u32 len, idx;
743
744 vdev = &fifo->vdev[vring->vdev_id]->vdev;
745
746 /* Get the descriptor of the next packet. */
747 if (!vring->desc) {
748 desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx);
749 if (!desc) {
750 /* Drop next Rx packet to avoid stuck. */
751 if (is_rx) {
752 desc = &vring->drop_desc;
753 vring->desc_head = desc;
754 vring->desc = desc;
755 } else {
756 return false;
757 }
758 }
759 } else {
760 desc = vring->desc;
761 }
762
763 /* Beginning of a packet. Start to Rx/Tx packet header. */
764 if (vring->pkt_len == 0) {
765 mlxbf_tmfifo_rxtx_header(vring, &desc, is_rx, &vring_change);
766 (*avail)--;
767
768 /* Return if new packet is for another ring. */
769 if (vring_change)
770 return false;
771 goto mlxbf_tmfifo_desc_done;
772 }
773
774 /* Get the length of this desc. */
775 len = virtio32_to_cpu(vdev, desc->len);
776 if (len > vring->rem_len)
777 len = vring->rem_len;
778
779 /* Rx/Tx one word (8 bytes) if not done. */
780 if (vring->cur_len < len) {
781 mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len);
782 (*avail)--;
783 }
784
785 /* Check again whether it's done. */
786 if (vring->cur_len == len) {
787 vring->cur_len = 0;
788 vring->rem_len -= len;
789
790 /* Get the next desc on the chain. */
791 if (!IS_VRING_DROP(vring) && vring->rem_len > 0 &&
792 (virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) {
793 idx = virtio16_to_cpu(vdev, desc->next);
794 desc = &vr->desc[idx];
795 goto mlxbf_tmfifo_desc_done;
796 }
797
798 /* Done and release the packet. */
799 desc = NULL;
800 fifo->vring[is_rx] = NULL;
801 if (!IS_VRING_DROP(vring)) {
802 mlxbf_tmfifo_release_pkt(vring);
803 } else {
804 vring->pkt_len = 0;
805 vring->desc_head = NULL;
806 vring->desc = NULL;
807 return false;
808 }
809
810 /*
811 * Make sure the load/store are in order before
812 * returning back to virtio.
813 */
814 virtio_mb(false);
815
816 /* Notify upper layer that packet is done. */
817 spin_lock_irqsave(&fifo->spin_lock[is_rx], flags);
818 vring_interrupt(0, vring->vq);
819 spin_unlock_irqrestore(&fifo->spin_lock[is_rx], flags);
820 }
821
822mlxbf_tmfifo_desc_done:
823 /* Save the current desc. */
824 vring->desc = desc;
825
826 return true;
827}
828
829static void mlxbf_tmfifo_check_tx_timeout(struct mlxbf_tmfifo_vring *vring)
830{
831 unsigned long flags;
832
833 /* Only handle Tx timeout for network vdev. */
834 if (vring->vdev_id != VIRTIO_ID_NET)
835 return;
836
837 /* Initialize the timeout or return if not expired. */
838 if (!vring->tx_timeout) {
839 /* Initialize the timeout. */
840 vring->tx_timeout = jiffies +
841 msecs_to_jiffies(TMFIFO_TX_TIMEOUT);
842 return;
843 } else if (time_before(jiffies, vring->tx_timeout)) {
844 /* Return if not timeout yet. */
845 return;
846 }
847
848 /*
849 * Drop the packet after timeout. The outstanding packet is
850 * released and the remaining bytes will be sent with padding byte 0x00
851 * as a recovery. On the peer(host) side, the padding bytes 0x00 will be
852 * either dropped directly, or appended into existing outstanding packet
853 * thus dropped as corrupted network packet.
854 */
855 vring->rem_padding = round_up(vring->rem_len, sizeof(u64));
856 mlxbf_tmfifo_release_pkt(vring);
857 vring->cur_len = 0;
858 vring->rem_len = 0;
859 vring->fifo->vring[0] = NULL;
860
861 /*
862 * Make sure the load/store are in order before
863 * returning back to virtio.
864 */
865 virtio_mb(false);
866
867 /* Notify upper layer. */
868 spin_lock_irqsave(&vring->fifo->spin_lock[0], flags);
869 vring_interrupt(0, vring->vq);
870 spin_unlock_irqrestore(&vring->fifo->spin_lock[0], flags);
871}
872
873/* Rx & Tx processing of a queue. */
874static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx)
875{
876 int avail = 0, devid = vring->vdev_id;
877 struct mlxbf_tmfifo *fifo;
878 bool more;
879
880 fifo = vring->fifo;
881
882 /* Return if vdev is not ready. */
883 if (!fifo || !fifo->vdev[devid])
884 return;
885
886 /* Return if another vring is running. */
887 if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring)
888 return;
889
890 /* Only handle console and network for now. */
891 if (WARN_ON(devid != VIRTIO_ID_NET && devid != VIRTIO_ID_CONSOLE))
892 return;
893
894 do {
895retry:
896 /* Get available FIFO space. */
897 if (avail == 0) {
898 if (is_rx)
899 avail = mlxbf_tmfifo_get_rx_avail(fifo);
900 else
901 avail = mlxbf_tmfifo_get_tx_avail(fifo, devid);
902 if (avail <= 0)
903 break;
904 }
905
906 /* Insert paddings for discarded Tx packet. */
907 if (!is_rx) {
908 vring->tx_timeout = 0;
909 while (vring->rem_padding >= sizeof(u64)) {
910 writeq(0, vring->fifo->tx.data);
911 vring->rem_padding -= sizeof(u64);
912 if (--avail == 0)
913 goto retry;
914 }
915 }
916
917 /* Console output always comes from the Tx buffer. */
918 if (!is_rx && devid == VIRTIO_ID_CONSOLE) {
919 mlxbf_tmfifo_console_tx(fifo, avail);
920 break;
921 }
922
923 /* Handle one descriptor. */
924 more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, &avail);
925 } while (more);
926
927 /* Check Tx timeout. */
928 if (avail <= 0 && !is_rx)
929 mlxbf_tmfifo_check_tx_timeout(vring);
930}
931
932/* Handle Rx or Tx queues. */
933static void mlxbf_tmfifo_work_rxtx(struct mlxbf_tmfifo *fifo, int queue_id,
934 int irq_id, bool is_rx)
935{
936 struct mlxbf_tmfifo_vdev *tm_vdev;
937 struct mlxbf_tmfifo_vring *vring;
938 int i;
939
940 if (!test_and_clear_bit(irq_id, &fifo->pend_events) ||
941 !fifo->irq_info[irq_id].irq)
942 return;
943
944 for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++) {
945 tm_vdev = fifo->vdev[i];
946 if (tm_vdev) {
947 vring = &tm_vdev->vrings[queue_id];
948 if (vring->vq)
949 mlxbf_tmfifo_rxtx(vring, is_rx);
950 }
951 }
952}
953
954/* Work handler for Rx and Tx case. */
955static void mlxbf_tmfifo_work_handler(struct work_struct *work)
956{
957 struct mlxbf_tmfifo *fifo;
958
959 fifo = container_of(work, struct mlxbf_tmfifo, work);
960 if (!fifo->is_ready)
961 return;
962
963 mutex_lock(&fifo->lock);
964
965 /* Tx (Send data to the TmFifo). */
966 mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_TX,
967 MLXBF_TM_TX_LWM_IRQ, false);
968
969 /* Rx (Receive data from the TmFifo). */
970 mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_RX,
971 MLXBF_TM_RX_HWM_IRQ, true);
972
973 mutex_unlock(&fifo->lock);
974}
975
976/* The notify function is called when new buffers are posted. */
977static bool mlxbf_tmfifo_virtio_notify(struct virtqueue *vq)
978{
979 struct mlxbf_tmfifo_vring *vring = vq->priv;
980 struct mlxbf_tmfifo_vdev *tm_vdev;
981 struct mlxbf_tmfifo *fifo;
982 unsigned long flags;
983
984 fifo = vring->fifo;
985
986 /*
987 * Virtio maintains vrings in pairs, even number ring for Rx
988 * and odd number ring for Tx.
989 */
990 if (vring->index & BIT(0)) {
991 /*
992 * Console could make blocking call with interrupts disabled.
993 * In such case, the vring needs to be served right away. For
994 * other cases, just set the TX LWM bit to start Tx in the
995 * worker handler.
996 */
997 if (vring->vdev_id == VIRTIO_ID_CONSOLE) {
998 spin_lock_irqsave(&fifo->spin_lock[0], flags);
999 tm_vdev = fifo->vdev[VIRTIO_ID_CONSOLE];
1000 mlxbf_tmfifo_console_output(tm_vdev, vring);
1001 spin_unlock_irqrestore(&fifo->spin_lock[0], flags);
1002 set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events);
1003 } else if (test_and_set_bit(MLXBF_TM_TX_LWM_IRQ,
1004 &fifo->pend_events)) {
1005 return true;
1006 }
1007 } else {
1008 if (test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events))
1009 return true;
1010 }
1011
1012 schedule_work(&fifo->work);
1013
1014 return true;
1015}
1016
1017/* Get the array of feature bits for this device. */
1018static u64 mlxbf_tmfifo_virtio_get_features(struct virtio_device *vdev)
1019{
1020 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1021
1022 return tm_vdev->features;
1023}
1024
1025/* Confirm device features to use. */
1026static int mlxbf_tmfifo_virtio_finalize_features(struct virtio_device *vdev)
1027{
1028 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1029
1030 tm_vdev->features = vdev->features;
1031
1032 return 0;
1033}
1034
1035/* Free virtqueues found by find_vqs(). */
1036static void mlxbf_tmfifo_virtio_del_vqs(struct virtio_device *vdev)
1037{
1038 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1039 struct mlxbf_tmfifo_vring *vring;
1040 struct virtqueue *vq;
1041 int i;
1042
1043 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
1044 vring = &tm_vdev->vrings[i];
1045
1046 /* Release the pending packet. */
1047 if (vring->desc)
1048 mlxbf_tmfifo_release_pkt(vring);
1049 vq = vring->vq;
1050 if (vq) {
1051 vring->vq = NULL;
1052 vring_del_virtqueue(vq);
1053 }
1054 }
1055}
1056
1057/* Create and initialize the virtual queues. */
1058static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev,
1059 unsigned int nvqs,
1060 struct virtqueue *vqs[],
1061 vq_callback_t *callbacks[],
1062 const char * const names[],
1063 const bool *ctx,
1064 struct irq_affinity *desc)
1065{
1066 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1067 struct mlxbf_tmfifo_vring *vring;
1068 struct virtqueue *vq;
1069 int i, ret, size;
1070
1071 if (nvqs > ARRAY_SIZE(tm_vdev->vrings))
1072 return -EINVAL;
1073
1074 for (i = 0; i < nvqs; ++i) {
1075 if (!names[i]) {
1076 ret = -EINVAL;
1077 goto error;
1078 }
1079 vring = &tm_vdev->vrings[i];
1080
1081 /* zero vring */
1082 size = vring_size(vring->num, vring->align);
1083 memset(vring->va, 0, size);
1084 vq = vring_new_virtqueue(i, vring->num, vring->align, vdev,
1085 false, false, vring->va,
1086 mlxbf_tmfifo_virtio_notify,
1087 callbacks[i], names[i]);
1088 if (!vq) {
1089 dev_err(&vdev->dev, "vring_new_virtqueue failed\n");
1090 ret = -ENOMEM;
1091 goto error;
1092 }
1093
1094 vq->num_max = vring->num;
1095
1096 vq->priv = vring;
1097
1098 /* Make vq update visible before using it. */
1099 virtio_mb(false);
1100
1101 vqs[i] = vq;
1102 vring->vq = vq;
1103 }
1104
1105 return 0;
1106
1107error:
1108 mlxbf_tmfifo_virtio_del_vqs(vdev);
1109 return ret;
1110}
1111
1112/* Read the status byte. */
1113static u8 mlxbf_tmfifo_virtio_get_status(struct virtio_device *vdev)
1114{
1115 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1116
1117 return tm_vdev->status;
1118}
1119
1120/* Write the status byte. */
1121static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
1122 u8 status)
1123{
1124 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1125
1126 tm_vdev->status = status;
1127}
1128
1129/* Reset the device. Not much here for now. */
1130static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
1131{
1132 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1133
1134 tm_vdev->status = 0;
1135}
1136
1137/* Read the value of a configuration field. */
1138static void mlxbf_tmfifo_virtio_get(struct virtio_device *vdev,
1139 unsigned int offset,
1140 void *buf,
1141 unsigned int len)
1142{
1143 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1144
1145 if ((u64)offset + len > sizeof(tm_vdev->config))
1146 return;
1147
1148 memcpy(buf, (u8 *)&tm_vdev->config + offset, len);
1149}
1150
1151/* Write the value of a configuration field. */
1152static void mlxbf_tmfifo_virtio_set(struct virtio_device *vdev,
1153 unsigned int offset,
1154 const void *buf,
1155 unsigned int len)
1156{
1157 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1158
1159 if ((u64)offset + len > sizeof(tm_vdev->config))
1160 return;
1161
1162 memcpy((u8 *)&tm_vdev->config + offset, buf, len);
1163}
1164
1165static void tmfifo_virtio_dev_release(struct device *device)
1166{
1167 struct virtio_device *vdev =
1168 container_of(device, struct virtio_device, dev);
1169 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1170
1171 kfree(tm_vdev);
1172}
1173
1174/* Virtio config operations. */
1175static const struct virtio_config_ops mlxbf_tmfifo_virtio_config_ops = {
1176 .get_features = mlxbf_tmfifo_virtio_get_features,
1177 .finalize_features = mlxbf_tmfifo_virtio_finalize_features,
1178 .find_vqs = mlxbf_tmfifo_virtio_find_vqs,
1179 .del_vqs = mlxbf_tmfifo_virtio_del_vqs,
1180 .reset = mlxbf_tmfifo_virtio_reset,
1181 .set_status = mlxbf_tmfifo_virtio_set_status,
1182 .get_status = mlxbf_tmfifo_virtio_get_status,
1183 .get = mlxbf_tmfifo_virtio_get,
1184 .set = mlxbf_tmfifo_virtio_set,
1185};
1186
1187/* Create vdev for the FIFO. */
1188static int mlxbf_tmfifo_create_vdev(struct device *dev,
1189 struct mlxbf_tmfifo *fifo,
1190 int vdev_id, u64 features,
1191 void *config, u32 size)
1192{
1193 struct mlxbf_tmfifo_vdev *tm_vdev, *reg_dev = NULL;
1194 int ret;
1195
1196 mutex_lock(&fifo->lock);
1197
1198 tm_vdev = fifo->vdev[vdev_id];
1199 if (tm_vdev) {
1200 dev_err(dev, "vdev %d already exists\n", vdev_id);
1201 ret = -EEXIST;
1202 goto fail;
1203 }
1204
1205 tm_vdev = kzalloc(sizeof(*tm_vdev), GFP_KERNEL);
1206 if (!tm_vdev) {
1207 ret = -ENOMEM;
1208 goto fail;
1209 }
1210
1211 tm_vdev->vdev.id.device = vdev_id;
1212 tm_vdev->vdev.config = &mlxbf_tmfifo_virtio_config_ops;
1213 tm_vdev->vdev.dev.parent = dev;
1214 tm_vdev->vdev.dev.release = tmfifo_virtio_dev_release;
1215 tm_vdev->features = features;
1216 if (config)
1217 memcpy(&tm_vdev->config, config, size);
1218
1219 if (mlxbf_tmfifo_alloc_vrings(fifo, tm_vdev)) {
1220 dev_err(dev, "unable to allocate vring\n");
1221 ret = -ENOMEM;
1222 goto vdev_fail;
1223 }
1224
1225 /* Allocate an output buffer for the console device. */
1226 if (vdev_id == VIRTIO_ID_CONSOLE)
1227 tm_vdev->tx_buf.buf = devm_kmalloc(dev,
1228 MLXBF_TMFIFO_CON_TX_BUF_SIZE,
1229 GFP_KERNEL);
1230 fifo->vdev[vdev_id] = tm_vdev;
1231
1232 /* Register the virtio device. */
1233 ret = register_virtio_device(&tm_vdev->vdev);
1234 reg_dev = tm_vdev;
1235 if (ret) {
1236 dev_err(dev, "register_virtio_device failed\n");
1237 goto vdev_fail;
1238 }
1239
1240 mutex_unlock(&fifo->lock);
1241 return 0;
1242
1243vdev_fail:
1244 mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
1245 fifo->vdev[vdev_id] = NULL;
1246 if (reg_dev)
1247 put_device(&tm_vdev->vdev.dev);
1248 else
1249 kfree(tm_vdev);
1250fail:
1251 mutex_unlock(&fifo->lock);
1252 return ret;
1253}
1254
1255/* Delete vdev for the FIFO. */
1256static int mlxbf_tmfifo_delete_vdev(struct mlxbf_tmfifo *fifo, int vdev_id)
1257{
1258 struct mlxbf_tmfifo_vdev *tm_vdev;
1259
1260 mutex_lock(&fifo->lock);
1261
1262 /* Unregister vdev. */
1263 tm_vdev = fifo->vdev[vdev_id];
1264 if (tm_vdev) {
1265 unregister_virtio_device(&tm_vdev->vdev);
1266 mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
1267 fifo->vdev[vdev_id] = NULL;
1268 }
1269
1270 mutex_unlock(&fifo->lock);
1271
1272 return 0;
1273}
1274
1275/* Read the configured network MAC address from efi variable. */
1276static void mlxbf_tmfifo_get_cfg_mac(u8 *mac)
1277{
1278 efi_guid_t guid = EFI_GLOBAL_VARIABLE_GUID;
1279 unsigned long size = ETH_ALEN;
1280 u8 buf[ETH_ALEN];
1281 efi_status_t rc;
1282
1283 rc = efi.get_variable(mlxbf_tmfifo_efi_name, &guid, NULL, &size, buf);
1284 if (rc == EFI_SUCCESS && size == ETH_ALEN)
1285 ether_addr_copy(mac, buf);
1286 else
1287 ether_addr_copy(mac, mlxbf_tmfifo_net_default_mac);
1288}
1289
1290/* Set TmFifo thresolds which is used to trigger interrupts. */
1291static void mlxbf_tmfifo_set_threshold(struct mlxbf_tmfifo *fifo)
1292{
1293 u64 ctl;
1294
1295 /* Get Tx FIFO size and set the low/high watermark. */
1296 ctl = readq(fifo->tx.ctl);
1297 fifo->tx_fifo_size =
1298 FIELD_GET(MLXBF_TMFIFO_TX_CTL__MAX_ENTRIES_MASK, ctl);
1299 ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__LWM_MASK) |
1300 FIELD_PREP(MLXBF_TMFIFO_TX_CTL__LWM_MASK,
1301 fifo->tx_fifo_size / 2);
1302 ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__HWM_MASK) |
1303 FIELD_PREP(MLXBF_TMFIFO_TX_CTL__HWM_MASK,
1304 fifo->tx_fifo_size - 1);
1305 writeq(ctl, fifo->tx.ctl);
1306
1307 /* Get Rx FIFO size and set the low/high watermark. */
1308 ctl = readq(fifo->rx.ctl);
1309 fifo->rx_fifo_size =
1310 FIELD_GET(MLXBF_TMFIFO_RX_CTL__MAX_ENTRIES_MASK, ctl);
1311 ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__LWM_MASK) |
1312 FIELD_PREP(MLXBF_TMFIFO_RX_CTL__LWM_MASK, 0);
1313 ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__HWM_MASK) |
1314 FIELD_PREP(MLXBF_TMFIFO_RX_CTL__HWM_MASK, 1);
1315 writeq(ctl, fifo->rx.ctl);
1316}
1317
1318static void mlxbf_tmfifo_cleanup(struct mlxbf_tmfifo *fifo)
1319{
1320 int i;
1321
1322 fifo->is_ready = false;
1323 del_timer_sync(&fifo->timer);
1324 mlxbf_tmfifo_disable_irqs(fifo);
1325 cancel_work_sync(&fifo->work);
1326 for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++)
1327 mlxbf_tmfifo_delete_vdev(fifo, i);
1328}
1329
1330/* Probe the TMFIFO. */
1331static int mlxbf_tmfifo_probe(struct platform_device *pdev)
1332{
1333 struct virtio_net_config net_config;
1334 struct device *dev = &pdev->dev;
1335 struct mlxbf_tmfifo *fifo;
1336 u64 dev_id;
1337 int i, rc;
1338
1339 rc = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &dev_id);
1340 if (rc) {
1341 dev_err(dev, "Cannot retrieve UID\n");
1342 return rc;
1343 }
1344
1345 fifo = devm_kzalloc(dev, sizeof(*fifo), GFP_KERNEL);
1346 if (!fifo)
1347 return -ENOMEM;
1348
1349 spin_lock_init(&fifo->spin_lock[0]);
1350 spin_lock_init(&fifo->spin_lock[1]);
1351 INIT_WORK(&fifo->work, mlxbf_tmfifo_work_handler);
1352 mutex_init(&fifo->lock);
1353
1354 /* Get the resource of the Rx FIFO. */
1355 fifo->res0 = devm_platform_ioremap_resource(pdev, 0);
1356 if (IS_ERR(fifo->res0))
1357 return PTR_ERR(fifo->res0);
1358
1359 /* Get the resource of the Tx FIFO. */
1360 fifo->res1 = devm_platform_ioremap_resource(pdev, 1);
1361 if (IS_ERR(fifo->res1))
1362 return PTR_ERR(fifo->res1);
1363
1364 if (dev_id == TMFIFO_BF3_UID) {
1365 fifo->rx.ctl = fifo->res1 + MLXBF_TMFIFO_RX_CTL_BF3;
1366 fifo->rx.sts = fifo->res1 + MLXBF_TMFIFO_RX_STS_BF3;
1367 fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA_BF3;
1368 fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL_BF3;
1369 fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS_BF3;
1370 fifo->tx.data = fifo->res0 + MLXBF_TMFIFO_TX_DATA_BF3;
1371 } else {
1372 fifo->rx.ctl = fifo->res0 + MLXBF_TMFIFO_RX_CTL;
1373 fifo->rx.sts = fifo->res0 + MLXBF_TMFIFO_RX_STS;
1374 fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA;
1375 fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL;
1376 fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS;
1377 fifo->tx.data = fifo->res1 + MLXBF_TMFIFO_TX_DATA;
1378 }
1379
1380 platform_set_drvdata(pdev, fifo);
1381
1382 timer_setup(&fifo->timer, mlxbf_tmfifo_timer, 0);
1383
1384 for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) {
1385 fifo->irq_info[i].index = i;
1386 fifo->irq_info[i].fifo = fifo;
1387 fifo->irq_info[i].irq = platform_get_irq(pdev, i);
1388 rc = devm_request_irq(dev, fifo->irq_info[i].irq,
1389 mlxbf_tmfifo_irq_handler, 0,
1390 "tmfifo", &fifo->irq_info[i]);
1391 if (rc) {
1392 dev_err(dev, "devm_request_irq failed\n");
1393 fifo->irq_info[i].irq = 0;
1394 return rc;
1395 }
1396 }
1397
1398 mlxbf_tmfifo_set_threshold(fifo);
1399
1400 /* Create the console vdev. */
1401 rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_CONSOLE, 0, NULL, 0);
1402 if (rc)
1403 goto fail;
1404
1405 /* Create the network vdev. */
1406 memset(&net_config, 0, sizeof(net_config));
1407
1408 /* A legacy-only interface for now. */
1409 net_config.mtu = __cpu_to_virtio16(virtio_legacy_is_little_endian(),
1410 ETH_DATA_LEN);
1411 net_config.status = __cpu_to_virtio16(virtio_legacy_is_little_endian(),
1412 VIRTIO_NET_S_LINK_UP);
1413 mlxbf_tmfifo_get_cfg_mac(net_config.mac);
1414 rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_NET,
1415 MLXBF_TMFIFO_NET_FEATURES, &net_config,
1416 sizeof(net_config));
1417 if (rc)
1418 goto fail;
1419
1420 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL);
1421
1422 /* Make all updates visible before setting the 'is_ready' flag. */
1423 virtio_mb(false);
1424
1425 fifo->is_ready = true;
1426 return 0;
1427
1428fail:
1429 mlxbf_tmfifo_cleanup(fifo);
1430 return rc;
1431}
1432
1433/* Device remove function. */
1434static void mlxbf_tmfifo_remove(struct platform_device *pdev)
1435{
1436 struct mlxbf_tmfifo *fifo = platform_get_drvdata(pdev);
1437
1438 mlxbf_tmfifo_cleanup(fifo);
1439}
1440
1441static const struct acpi_device_id mlxbf_tmfifo_acpi_match[] = {
1442 { "MLNXBF01", 0 },
1443 {}
1444};
1445MODULE_DEVICE_TABLE(acpi, mlxbf_tmfifo_acpi_match);
1446
1447static struct platform_driver mlxbf_tmfifo_driver = {
1448 .probe = mlxbf_tmfifo_probe,
1449 .remove_new = mlxbf_tmfifo_remove,
1450 .driver = {
1451 .name = "bf-tmfifo",
1452 .acpi_match_table = mlxbf_tmfifo_acpi_match,
1453 },
1454};
1455
1456module_platform_driver(mlxbf_tmfifo_driver);
1457
1458MODULE_DESCRIPTION("Mellanox BlueField SoC TmFifo Driver");
1459MODULE_LICENSE("GPL v2");
1460MODULE_AUTHOR("Mellanox Technologies");
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Mellanox BlueField SoC TmFifo driver
4 *
5 * Copyright (C) 2019 Mellanox Technologies
6 */
7
8#include <linux/acpi.h>
9#include <linux/bitfield.h>
10#include <linux/circ_buf.h>
11#include <linux/efi.h>
12#include <linux/irq.h>
13#include <linux/module.h>
14#include <linux/mutex.h>
15#include <linux/platform_device.h>
16#include <linux/types.h>
17
18#include <linux/virtio_config.h>
19#include <linux/virtio_console.h>
20#include <linux/virtio_ids.h>
21#include <linux/virtio_net.h>
22#include <linux/virtio_ring.h>
23
24#include "mlxbf-tmfifo-regs.h"
25
26/* Vring size. */
27#define MLXBF_TMFIFO_VRING_SIZE SZ_1K
28
29/* Console Tx buffer size. */
30#define MLXBF_TMFIFO_CON_TX_BUF_SIZE SZ_32K
31
32/* Console Tx buffer reserved space. */
33#define MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE 8
34
35/* House-keeping timer interval. */
36#define MLXBF_TMFIFO_TIMER_INTERVAL (HZ / 10)
37
38/* Virtual devices sharing the TM FIFO. */
39#define MLXBF_TMFIFO_VDEV_MAX (VIRTIO_ID_CONSOLE + 1)
40
41/*
42 * Reserve 1/16 of TmFifo space, so console messages are not starved by
43 * the networking traffic.
44 */
45#define MLXBF_TMFIFO_RESERVE_RATIO 16
46
47/* Message with data needs at least two words (for header & data). */
48#define MLXBF_TMFIFO_DATA_MIN_WORDS 2
49
50struct mlxbf_tmfifo;
51
52/**
53 * mlxbf_tmfifo_vring - Structure of the TmFifo virtual ring
54 * @va: virtual address of the ring
55 * @dma: dma address of the ring
56 * @vq: pointer to the virtio virtqueue
57 * @desc: current descriptor of the pending packet
58 * @desc_head: head descriptor of the pending packet
59 * @cur_len: processed length of the current descriptor
60 * @rem_len: remaining length of the pending packet
61 * @pkt_len: total length of the pending packet
62 * @next_avail: next avail descriptor id
63 * @num: vring size (number of descriptors)
64 * @align: vring alignment size
65 * @index: vring index
66 * @vdev_id: vring virtio id (VIRTIO_ID_xxx)
67 * @fifo: pointer to the tmfifo structure
68 */
69struct mlxbf_tmfifo_vring {
70 void *va;
71 dma_addr_t dma;
72 struct virtqueue *vq;
73 struct vring_desc *desc;
74 struct vring_desc *desc_head;
75 int cur_len;
76 int rem_len;
77 u32 pkt_len;
78 u16 next_avail;
79 int num;
80 int align;
81 int index;
82 int vdev_id;
83 struct mlxbf_tmfifo *fifo;
84};
85
86/* Interrupt types. */
87enum {
88 MLXBF_TM_RX_LWM_IRQ,
89 MLXBF_TM_RX_HWM_IRQ,
90 MLXBF_TM_TX_LWM_IRQ,
91 MLXBF_TM_TX_HWM_IRQ,
92 MLXBF_TM_MAX_IRQ
93};
94
95/* Ring types (Rx & Tx). */
96enum {
97 MLXBF_TMFIFO_VRING_RX,
98 MLXBF_TMFIFO_VRING_TX,
99 MLXBF_TMFIFO_VRING_MAX
100};
101
102/**
103 * mlxbf_tmfifo_vdev - Structure of the TmFifo virtual device
104 * @vdev: virtio device, in which the vdev.id.device field has the
105 * VIRTIO_ID_xxx id to distinguish the virtual device.
106 * @status: status of the device
107 * @features: supported features of the device
108 * @vrings: array of tmfifo vrings of this device
109 * @config.cons: virtual console config -
110 * select if vdev.id.device is VIRTIO_ID_CONSOLE
111 * @config.net: virtual network config -
112 * select if vdev.id.device is VIRTIO_ID_NET
113 * @tx_buf: tx buffer used to buffer data before writing into the FIFO
114 */
115struct mlxbf_tmfifo_vdev {
116 struct virtio_device vdev;
117 u8 status;
118 u64 features;
119 struct mlxbf_tmfifo_vring vrings[MLXBF_TMFIFO_VRING_MAX];
120 union {
121 struct virtio_console_config cons;
122 struct virtio_net_config net;
123 } config;
124 struct circ_buf tx_buf;
125};
126
127/**
128 * mlxbf_tmfifo_irq_info - Structure of the interrupt information
129 * @fifo: pointer to the tmfifo structure
130 * @irq: interrupt number
131 * @index: index into the interrupt array
132 */
133struct mlxbf_tmfifo_irq_info {
134 struct mlxbf_tmfifo *fifo;
135 int irq;
136 int index;
137};
138
139/**
140 * mlxbf_tmfifo - Structure of the TmFifo
141 * @vdev: array of the virtual devices running over the TmFifo
142 * @lock: lock to protect the TmFifo access
143 * @rx_base: mapped register base address for the Rx FIFO
144 * @tx_base: mapped register base address for the Tx FIFO
145 * @rx_fifo_size: number of entries of the Rx FIFO
146 * @tx_fifo_size: number of entries of the Tx FIFO
147 * @pend_events: pending bits for deferred events
148 * @irq_info: interrupt information
149 * @work: work struct for deferred process
150 * @timer: background timer
151 * @vring: Tx/Rx ring
152 * @spin_lock: Tx/Rx spin lock
153 * @is_ready: ready flag
154 */
155struct mlxbf_tmfifo {
156 struct mlxbf_tmfifo_vdev *vdev[MLXBF_TMFIFO_VDEV_MAX];
157 struct mutex lock; /* TmFifo lock */
158 void __iomem *rx_base;
159 void __iomem *tx_base;
160 int rx_fifo_size;
161 int tx_fifo_size;
162 unsigned long pend_events;
163 struct mlxbf_tmfifo_irq_info irq_info[MLXBF_TM_MAX_IRQ];
164 struct work_struct work;
165 struct timer_list timer;
166 struct mlxbf_tmfifo_vring *vring[2];
167 spinlock_t spin_lock[2]; /* spin lock */
168 bool is_ready;
169};
170
171/**
172 * mlxbf_tmfifo_msg_hdr - Structure of the TmFifo message header
173 * @type: message type
174 * @len: payload length in network byte order. Messages sent into the FIFO
175 * will be read by the other side as data stream in the same byte order.
176 * The length needs to be encoded into network order so both sides
177 * could understand it.
178 */
179struct mlxbf_tmfifo_msg_hdr {
180 u8 type;
181 __be16 len;
182 u8 unused[5];
183} __packed __aligned(sizeof(u64));
184
185/*
186 * Default MAC.
187 * This MAC address will be read from EFI persistent variable if configured.
188 * It can also be reconfigured with standard Linux tools.
189 */
190static u8 mlxbf_tmfifo_net_default_mac[ETH_ALEN] = {
191 0x00, 0x1A, 0xCA, 0xFF, 0xFF, 0x01
192};
193
194/* EFI variable name of the MAC address. */
195static efi_char16_t mlxbf_tmfifo_efi_name[] = L"RshimMacAddr";
196
197/* Maximum L2 header length. */
198#define MLXBF_TMFIFO_NET_L2_OVERHEAD 36
199
200/* Supported virtio-net features. */
201#define MLXBF_TMFIFO_NET_FEATURES \
202 (BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_STATUS) | \
203 BIT_ULL(VIRTIO_NET_F_MAC))
204
205#define mlxbf_vdev_to_tmfifo(d) container_of(d, struct mlxbf_tmfifo_vdev, vdev)
206
207/* Free vrings of the FIFO device. */
208static void mlxbf_tmfifo_free_vrings(struct mlxbf_tmfifo *fifo,
209 struct mlxbf_tmfifo_vdev *tm_vdev)
210{
211 struct mlxbf_tmfifo_vring *vring;
212 int i, size;
213
214 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
215 vring = &tm_vdev->vrings[i];
216 if (vring->va) {
217 size = vring_size(vring->num, vring->align);
218 dma_free_coherent(tm_vdev->vdev.dev.parent, size,
219 vring->va, vring->dma);
220 vring->va = NULL;
221 if (vring->vq) {
222 vring_del_virtqueue(vring->vq);
223 vring->vq = NULL;
224 }
225 }
226 }
227}
228
229/* Allocate vrings for the FIFO. */
230static int mlxbf_tmfifo_alloc_vrings(struct mlxbf_tmfifo *fifo,
231 struct mlxbf_tmfifo_vdev *tm_vdev)
232{
233 struct mlxbf_tmfifo_vring *vring;
234 struct device *dev;
235 dma_addr_t dma;
236 int i, size;
237 void *va;
238
239 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
240 vring = &tm_vdev->vrings[i];
241 vring->fifo = fifo;
242 vring->num = MLXBF_TMFIFO_VRING_SIZE;
243 vring->align = SMP_CACHE_BYTES;
244 vring->index = i;
245 vring->vdev_id = tm_vdev->vdev.id.device;
246 dev = &tm_vdev->vdev.dev;
247
248 size = vring_size(vring->num, vring->align);
249 va = dma_alloc_coherent(dev->parent, size, &dma, GFP_KERNEL);
250 if (!va) {
251 mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
252 dev_err(dev->parent, "dma_alloc_coherent failed\n");
253 return -ENOMEM;
254 }
255
256 vring->va = va;
257 vring->dma = dma;
258 }
259
260 return 0;
261}
262
263/* Disable interrupts of the FIFO device. */
264static void mlxbf_tmfifo_disable_irqs(struct mlxbf_tmfifo *fifo)
265{
266 int i, irq;
267
268 for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) {
269 irq = fifo->irq_info[i].irq;
270 fifo->irq_info[i].irq = 0;
271 disable_irq(irq);
272 }
273}
274
275/* Interrupt handler. */
276static irqreturn_t mlxbf_tmfifo_irq_handler(int irq, void *arg)
277{
278 struct mlxbf_tmfifo_irq_info *irq_info = arg;
279
280 if (!test_and_set_bit(irq_info->index, &irq_info->fifo->pend_events))
281 schedule_work(&irq_info->fifo->work);
282
283 return IRQ_HANDLED;
284}
285
286/* Get the next packet descriptor from the vring. */
287static struct vring_desc *
288mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring)
289{
290 const struct vring *vr = virtqueue_get_vring(vring->vq);
291 struct virtio_device *vdev = vring->vq->vdev;
292 unsigned int idx, head;
293
294 if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx))
295 return NULL;
296
297 /* Make sure 'avail->idx' is visible already. */
298 virtio_rmb(false);
299
300 idx = vring->next_avail % vr->num;
301 head = virtio16_to_cpu(vdev, vr->avail->ring[idx]);
302 if (WARN_ON(head >= vr->num))
303 return NULL;
304
305 vring->next_avail++;
306
307 return &vr->desc[head];
308}
309
310/* Release virtio descriptor. */
311static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring,
312 struct vring_desc *desc, u32 len)
313{
314 const struct vring *vr = virtqueue_get_vring(vring->vq);
315 struct virtio_device *vdev = vring->vq->vdev;
316 u16 idx, vr_idx;
317
318 vr_idx = virtio16_to_cpu(vdev, vr->used->idx);
319 idx = vr_idx % vr->num;
320 vr->used->ring[idx].id = cpu_to_virtio32(vdev, desc - vr->desc);
321 vr->used->ring[idx].len = cpu_to_virtio32(vdev, len);
322
323 /*
324 * Virtio could poll and check the 'idx' to decide whether the desc is
325 * done or not. Add a memory barrier here to make sure the update above
326 * completes before updating the idx.
327 */
328 virtio_mb(false);
329 vr->used->idx = cpu_to_virtio16(vdev, vr_idx + 1);
330}
331
332/* Get the total length of the descriptor chain. */
333static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring,
334 struct vring_desc *desc)
335{
336 const struct vring *vr = virtqueue_get_vring(vring->vq);
337 struct virtio_device *vdev = vring->vq->vdev;
338 u32 len = 0, idx;
339
340 while (desc) {
341 len += virtio32_to_cpu(vdev, desc->len);
342 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT))
343 break;
344 idx = virtio16_to_cpu(vdev, desc->next);
345 desc = &vr->desc[idx];
346 }
347
348 return len;
349}
350
351static void mlxbf_tmfifo_release_pending_pkt(struct mlxbf_tmfifo_vring *vring)
352{
353 struct vring_desc *desc_head;
354 u32 len = 0;
355
356 if (vring->desc_head) {
357 desc_head = vring->desc_head;
358 len = vring->pkt_len;
359 } else {
360 desc_head = mlxbf_tmfifo_get_next_desc(vring);
361 len = mlxbf_tmfifo_get_pkt_len(vring, desc_head);
362 }
363
364 if (desc_head)
365 mlxbf_tmfifo_release_desc(vring, desc_head, len);
366
367 vring->pkt_len = 0;
368 vring->desc = NULL;
369 vring->desc_head = NULL;
370}
371
372static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring,
373 struct vring_desc *desc, bool is_rx)
374{
375 struct virtio_device *vdev = vring->vq->vdev;
376 struct virtio_net_hdr *net_hdr;
377
378 net_hdr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
379 memset(net_hdr, 0, sizeof(*net_hdr));
380}
381
382/* Get and initialize the next packet. */
383static struct vring_desc *
384mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx)
385{
386 struct vring_desc *desc;
387
388 desc = mlxbf_tmfifo_get_next_desc(vring);
389 if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET)
390 mlxbf_tmfifo_init_net_desc(vring, desc, is_rx);
391
392 vring->desc_head = desc;
393 vring->desc = desc;
394
395 return desc;
396}
397
398/* House-keeping timer. */
399static void mlxbf_tmfifo_timer(struct timer_list *t)
400{
401 struct mlxbf_tmfifo *fifo = container_of(t, struct mlxbf_tmfifo, timer);
402 int rx, tx;
403
404 rx = !test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events);
405 tx = !test_and_set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events);
406
407 if (rx || tx)
408 schedule_work(&fifo->work);
409
410 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL);
411}
412
413/* Copy one console packet into the output buffer. */
414static void mlxbf_tmfifo_console_output_one(struct mlxbf_tmfifo_vdev *cons,
415 struct mlxbf_tmfifo_vring *vring,
416 struct vring_desc *desc)
417{
418 const struct vring *vr = virtqueue_get_vring(vring->vq);
419 struct virtio_device *vdev = &cons->vdev;
420 u32 len, idx, seg;
421 void *addr;
422
423 while (desc) {
424 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
425 len = virtio32_to_cpu(vdev, desc->len);
426
427 seg = CIRC_SPACE_TO_END(cons->tx_buf.head, cons->tx_buf.tail,
428 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
429 if (len <= seg) {
430 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, len);
431 } else {
432 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, seg);
433 addr += seg;
434 memcpy(cons->tx_buf.buf, addr, len - seg);
435 }
436 cons->tx_buf.head = (cons->tx_buf.head + len) %
437 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
438
439 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT))
440 break;
441 idx = virtio16_to_cpu(vdev, desc->next);
442 desc = &vr->desc[idx];
443 }
444}
445
446/* Copy console data into the output buffer. */
447static void mlxbf_tmfifo_console_output(struct mlxbf_tmfifo_vdev *cons,
448 struct mlxbf_tmfifo_vring *vring)
449{
450 struct vring_desc *desc;
451 u32 len, avail;
452
453 desc = mlxbf_tmfifo_get_next_desc(vring);
454 while (desc) {
455 /* Release the packet if not enough space. */
456 len = mlxbf_tmfifo_get_pkt_len(vring, desc);
457 avail = CIRC_SPACE(cons->tx_buf.head, cons->tx_buf.tail,
458 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
459 if (len + MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE > avail) {
460 mlxbf_tmfifo_release_desc(vring, desc, len);
461 break;
462 }
463
464 mlxbf_tmfifo_console_output_one(cons, vring, desc);
465 mlxbf_tmfifo_release_desc(vring, desc, len);
466 desc = mlxbf_tmfifo_get_next_desc(vring);
467 }
468}
469
470/* Get the number of available words in Rx FIFO for receiving. */
471static int mlxbf_tmfifo_get_rx_avail(struct mlxbf_tmfifo *fifo)
472{
473 u64 sts;
474
475 sts = readq(fifo->rx_base + MLXBF_TMFIFO_RX_STS);
476 return FIELD_GET(MLXBF_TMFIFO_RX_STS__COUNT_MASK, sts);
477}
478
479/* Get the number of available words in the TmFifo for sending. */
480static int mlxbf_tmfifo_get_tx_avail(struct mlxbf_tmfifo *fifo, int vdev_id)
481{
482 int tx_reserve;
483 u32 count;
484 u64 sts;
485
486 /* Reserve some room in FIFO for console messages. */
487 if (vdev_id == VIRTIO_ID_NET)
488 tx_reserve = fifo->tx_fifo_size / MLXBF_TMFIFO_RESERVE_RATIO;
489 else
490 tx_reserve = 1;
491
492 sts = readq(fifo->tx_base + MLXBF_TMFIFO_TX_STS);
493 count = FIELD_GET(MLXBF_TMFIFO_TX_STS__COUNT_MASK, sts);
494 return fifo->tx_fifo_size - tx_reserve - count;
495}
496
497/* Console Tx (move data from the output buffer into the TmFifo). */
498static void mlxbf_tmfifo_console_tx(struct mlxbf_tmfifo *fifo, int avail)
499{
500 struct mlxbf_tmfifo_msg_hdr hdr;
501 struct mlxbf_tmfifo_vdev *cons;
502 unsigned long flags;
503 int size, seg;
504 void *addr;
505 u64 data;
506
507 /* Return if not enough space available. */
508 if (avail < MLXBF_TMFIFO_DATA_MIN_WORDS)
509 return;
510
511 cons = fifo->vdev[VIRTIO_ID_CONSOLE];
512 if (!cons || !cons->tx_buf.buf)
513 return;
514
515 /* Return if no data to send. */
516 size = CIRC_CNT(cons->tx_buf.head, cons->tx_buf.tail,
517 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
518 if (size == 0)
519 return;
520
521 /* Adjust the size to available space. */
522 if (size + sizeof(hdr) > avail * sizeof(u64))
523 size = avail * sizeof(u64) - sizeof(hdr);
524
525 /* Write header. */
526 hdr.type = VIRTIO_ID_CONSOLE;
527 hdr.len = htons(size);
528 writeq(*(u64 *)&hdr, fifo->tx_base + MLXBF_TMFIFO_TX_DATA);
529
530 /* Use spin-lock to protect the 'cons->tx_buf'. */
531 spin_lock_irqsave(&fifo->spin_lock[0], flags);
532
533 while (size > 0) {
534 addr = cons->tx_buf.buf + cons->tx_buf.tail;
535
536 seg = CIRC_CNT_TO_END(cons->tx_buf.head, cons->tx_buf.tail,
537 MLXBF_TMFIFO_CON_TX_BUF_SIZE);
538 if (seg >= sizeof(u64)) {
539 memcpy(&data, addr, sizeof(u64));
540 } else {
541 memcpy(&data, addr, seg);
542 memcpy((u8 *)&data + seg, cons->tx_buf.buf,
543 sizeof(u64) - seg);
544 }
545 writeq(data, fifo->tx_base + MLXBF_TMFIFO_TX_DATA);
546
547 if (size >= sizeof(u64)) {
548 cons->tx_buf.tail = (cons->tx_buf.tail + sizeof(u64)) %
549 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
550 size -= sizeof(u64);
551 } else {
552 cons->tx_buf.tail = (cons->tx_buf.tail + size) %
553 MLXBF_TMFIFO_CON_TX_BUF_SIZE;
554 size = 0;
555 }
556 }
557
558 spin_unlock_irqrestore(&fifo->spin_lock[0], flags);
559}
560
561/* Rx/Tx one word in the descriptor buffer. */
562static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring,
563 struct vring_desc *desc,
564 bool is_rx, int len)
565{
566 struct virtio_device *vdev = vring->vq->vdev;
567 struct mlxbf_tmfifo *fifo = vring->fifo;
568 void *addr;
569 u64 data;
570
571 /* Get the buffer address of this desc. */
572 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr));
573
574 /* Read a word from FIFO for Rx. */
575 if (is_rx)
576 data = readq(fifo->rx_base + MLXBF_TMFIFO_RX_DATA);
577
578 if (vring->cur_len + sizeof(u64) <= len) {
579 /* The whole word. */
580 if (is_rx)
581 memcpy(addr + vring->cur_len, &data, sizeof(u64));
582 else
583 memcpy(&data, addr + vring->cur_len, sizeof(u64));
584 vring->cur_len += sizeof(u64);
585 } else {
586 /* Leftover bytes. */
587 if (is_rx)
588 memcpy(addr + vring->cur_len, &data,
589 len - vring->cur_len);
590 else
591 memcpy(&data, addr + vring->cur_len,
592 len - vring->cur_len);
593 vring->cur_len = len;
594 }
595
596 /* Write the word into FIFO for Tx. */
597 if (!is_rx)
598 writeq(data, fifo->tx_base + MLXBF_TMFIFO_TX_DATA);
599}
600
601/*
602 * Rx/Tx packet header.
603 *
604 * In Rx case, the packet might be found to belong to a different vring since
605 * the TmFifo is shared by different services. In such case, the 'vring_change'
606 * flag is set.
607 */
608static void mlxbf_tmfifo_rxtx_header(struct mlxbf_tmfifo_vring *vring,
609 struct vring_desc *desc,
610 bool is_rx, bool *vring_change)
611{
612 struct mlxbf_tmfifo *fifo = vring->fifo;
613 struct virtio_net_config *config;
614 struct mlxbf_tmfifo_msg_hdr hdr;
615 int vdev_id, hdr_len;
616
617 /* Read/Write packet header. */
618 if (is_rx) {
619 /* Drain one word from the FIFO. */
620 *(u64 *)&hdr = readq(fifo->rx_base + MLXBF_TMFIFO_RX_DATA);
621
622 /* Skip the length 0 packets (keepalive). */
623 if (hdr.len == 0)
624 return;
625
626 /* Check packet type. */
627 if (hdr.type == VIRTIO_ID_NET) {
628 vdev_id = VIRTIO_ID_NET;
629 hdr_len = sizeof(struct virtio_net_hdr);
630 config = &fifo->vdev[vdev_id]->config.net;
631 /* A legacy-only interface for now. */
632 if (ntohs(hdr.len) >
633 __virtio16_to_cpu(virtio_legacy_is_little_endian(),
634 config->mtu) +
635 MLXBF_TMFIFO_NET_L2_OVERHEAD)
636 return;
637 } else {
638 vdev_id = VIRTIO_ID_CONSOLE;
639 hdr_len = 0;
640 }
641
642 /*
643 * Check whether the new packet still belongs to this vring.
644 * If not, update the pkt_len of the new vring.
645 */
646 if (vdev_id != vring->vdev_id) {
647 struct mlxbf_tmfifo_vdev *tm_dev2 = fifo->vdev[vdev_id];
648
649 if (!tm_dev2)
650 return;
651 vring->desc = desc;
652 vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX];
653 *vring_change = true;
654 }
655 vring->pkt_len = ntohs(hdr.len) + hdr_len;
656 } else {
657 /* Network virtio has an extra header. */
658 hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ?
659 sizeof(struct virtio_net_hdr) : 0;
660 vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, desc);
661 hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ?
662 VIRTIO_ID_NET : VIRTIO_ID_CONSOLE;
663 hdr.len = htons(vring->pkt_len - hdr_len);
664 writeq(*(u64 *)&hdr, fifo->tx_base + MLXBF_TMFIFO_TX_DATA);
665 }
666
667 vring->cur_len = hdr_len;
668 vring->rem_len = vring->pkt_len;
669 fifo->vring[is_rx] = vring;
670}
671
672/*
673 * Rx/Tx one descriptor.
674 *
675 * Return true to indicate more data available.
676 */
677static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring,
678 bool is_rx, int *avail)
679{
680 const struct vring *vr = virtqueue_get_vring(vring->vq);
681 struct mlxbf_tmfifo *fifo = vring->fifo;
682 struct virtio_device *vdev;
683 bool vring_change = false;
684 struct vring_desc *desc;
685 unsigned long flags;
686 u32 len, idx;
687
688 vdev = &fifo->vdev[vring->vdev_id]->vdev;
689
690 /* Get the descriptor of the next packet. */
691 if (!vring->desc) {
692 desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx);
693 if (!desc)
694 return false;
695 } else {
696 desc = vring->desc;
697 }
698
699 /* Beginning of a packet. Start to Rx/Tx packet header. */
700 if (vring->pkt_len == 0) {
701 mlxbf_tmfifo_rxtx_header(vring, desc, is_rx, &vring_change);
702 (*avail)--;
703
704 /* Return if new packet is for another ring. */
705 if (vring_change)
706 return false;
707 goto mlxbf_tmfifo_desc_done;
708 }
709
710 /* Get the length of this desc. */
711 len = virtio32_to_cpu(vdev, desc->len);
712 if (len > vring->rem_len)
713 len = vring->rem_len;
714
715 /* Rx/Tx one word (8 bytes) if not done. */
716 if (vring->cur_len < len) {
717 mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len);
718 (*avail)--;
719 }
720
721 /* Check again whether it's done. */
722 if (vring->cur_len == len) {
723 vring->cur_len = 0;
724 vring->rem_len -= len;
725
726 /* Get the next desc on the chain. */
727 if (vring->rem_len > 0 &&
728 (virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) {
729 idx = virtio16_to_cpu(vdev, desc->next);
730 desc = &vr->desc[idx];
731 goto mlxbf_tmfifo_desc_done;
732 }
733
734 /* Done and release the pending packet. */
735 mlxbf_tmfifo_release_pending_pkt(vring);
736 desc = NULL;
737 fifo->vring[is_rx] = NULL;
738
739 /*
740 * Make sure the load/store are in order before
741 * returning back to virtio.
742 */
743 virtio_mb(false);
744
745 /* Notify upper layer that packet is done. */
746 spin_lock_irqsave(&fifo->spin_lock[is_rx], flags);
747 vring_interrupt(0, vring->vq);
748 spin_unlock_irqrestore(&fifo->spin_lock[is_rx], flags);
749 }
750
751mlxbf_tmfifo_desc_done:
752 /* Save the current desc. */
753 vring->desc = desc;
754
755 return true;
756}
757
758/* Rx & Tx processing of a queue. */
759static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx)
760{
761 int avail = 0, devid = vring->vdev_id;
762 struct mlxbf_tmfifo *fifo;
763 bool more;
764
765 fifo = vring->fifo;
766
767 /* Return if vdev is not ready. */
768 if (!fifo->vdev[devid])
769 return;
770
771 /* Return if another vring is running. */
772 if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring)
773 return;
774
775 /* Only handle console and network for now. */
776 if (WARN_ON(devid != VIRTIO_ID_NET && devid != VIRTIO_ID_CONSOLE))
777 return;
778
779 do {
780 /* Get available FIFO space. */
781 if (avail == 0) {
782 if (is_rx)
783 avail = mlxbf_tmfifo_get_rx_avail(fifo);
784 else
785 avail = mlxbf_tmfifo_get_tx_avail(fifo, devid);
786 if (avail <= 0)
787 break;
788 }
789
790 /* Console output always comes from the Tx buffer. */
791 if (!is_rx && devid == VIRTIO_ID_CONSOLE) {
792 mlxbf_tmfifo_console_tx(fifo, avail);
793 break;
794 }
795
796 /* Handle one descriptor. */
797 more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, &avail);
798 } while (more);
799}
800
801/* Handle Rx or Tx queues. */
802static void mlxbf_tmfifo_work_rxtx(struct mlxbf_tmfifo *fifo, int queue_id,
803 int irq_id, bool is_rx)
804{
805 struct mlxbf_tmfifo_vdev *tm_vdev;
806 struct mlxbf_tmfifo_vring *vring;
807 int i;
808
809 if (!test_and_clear_bit(irq_id, &fifo->pend_events) ||
810 !fifo->irq_info[irq_id].irq)
811 return;
812
813 for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++) {
814 tm_vdev = fifo->vdev[i];
815 if (tm_vdev) {
816 vring = &tm_vdev->vrings[queue_id];
817 if (vring->vq)
818 mlxbf_tmfifo_rxtx(vring, is_rx);
819 }
820 }
821}
822
823/* Work handler for Rx and Tx case. */
824static void mlxbf_tmfifo_work_handler(struct work_struct *work)
825{
826 struct mlxbf_tmfifo *fifo;
827
828 fifo = container_of(work, struct mlxbf_tmfifo, work);
829 if (!fifo->is_ready)
830 return;
831
832 mutex_lock(&fifo->lock);
833
834 /* Tx (Send data to the TmFifo). */
835 mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_TX,
836 MLXBF_TM_TX_LWM_IRQ, false);
837
838 /* Rx (Receive data from the TmFifo). */
839 mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_RX,
840 MLXBF_TM_RX_HWM_IRQ, true);
841
842 mutex_unlock(&fifo->lock);
843}
844
845/* The notify function is called when new buffers are posted. */
846static bool mlxbf_tmfifo_virtio_notify(struct virtqueue *vq)
847{
848 struct mlxbf_tmfifo_vring *vring = vq->priv;
849 struct mlxbf_tmfifo_vdev *tm_vdev;
850 struct mlxbf_tmfifo *fifo;
851 unsigned long flags;
852
853 fifo = vring->fifo;
854
855 /*
856 * Virtio maintains vrings in pairs, even number ring for Rx
857 * and odd number ring for Tx.
858 */
859 if (vring->index & BIT(0)) {
860 /*
861 * Console could make blocking call with interrupts disabled.
862 * In such case, the vring needs to be served right away. For
863 * other cases, just set the TX LWM bit to start Tx in the
864 * worker handler.
865 */
866 if (vring->vdev_id == VIRTIO_ID_CONSOLE) {
867 spin_lock_irqsave(&fifo->spin_lock[0], flags);
868 tm_vdev = fifo->vdev[VIRTIO_ID_CONSOLE];
869 mlxbf_tmfifo_console_output(tm_vdev, vring);
870 spin_unlock_irqrestore(&fifo->spin_lock[0], flags);
871 } else if (test_and_set_bit(MLXBF_TM_TX_LWM_IRQ,
872 &fifo->pend_events)) {
873 return true;
874 }
875 } else {
876 if (test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events))
877 return true;
878 }
879
880 schedule_work(&fifo->work);
881
882 return true;
883}
884
885/* Get the array of feature bits for this device. */
886static u64 mlxbf_tmfifo_virtio_get_features(struct virtio_device *vdev)
887{
888 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
889
890 return tm_vdev->features;
891}
892
893/* Confirm device features to use. */
894static int mlxbf_tmfifo_virtio_finalize_features(struct virtio_device *vdev)
895{
896 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
897
898 tm_vdev->features = vdev->features;
899
900 return 0;
901}
902
903/* Free virtqueues found by find_vqs(). */
904static void mlxbf_tmfifo_virtio_del_vqs(struct virtio_device *vdev)
905{
906 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
907 struct mlxbf_tmfifo_vring *vring;
908 struct virtqueue *vq;
909 int i;
910
911 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) {
912 vring = &tm_vdev->vrings[i];
913
914 /* Release the pending packet. */
915 if (vring->desc)
916 mlxbf_tmfifo_release_pending_pkt(vring);
917 vq = vring->vq;
918 if (vq) {
919 vring->vq = NULL;
920 vring_del_virtqueue(vq);
921 }
922 }
923}
924
925/* Create and initialize the virtual queues. */
926static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev,
927 unsigned int nvqs,
928 struct virtqueue *vqs[],
929 vq_callback_t *callbacks[],
930 const char * const names[],
931 const bool *ctx,
932 struct irq_affinity *desc)
933{
934 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
935 struct mlxbf_tmfifo_vring *vring;
936 struct virtqueue *vq;
937 int i, ret, size;
938
939 if (nvqs > ARRAY_SIZE(tm_vdev->vrings))
940 return -EINVAL;
941
942 for (i = 0; i < nvqs; ++i) {
943 if (!names[i]) {
944 ret = -EINVAL;
945 goto error;
946 }
947 vring = &tm_vdev->vrings[i];
948
949 /* zero vring */
950 size = vring_size(vring->num, vring->align);
951 memset(vring->va, 0, size);
952 vq = vring_new_virtqueue(i, vring->num, vring->align, vdev,
953 false, false, vring->va,
954 mlxbf_tmfifo_virtio_notify,
955 callbacks[i], names[i]);
956 if (!vq) {
957 dev_err(&vdev->dev, "vring_new_virtqueue failed\n");
958 ret = -ENOMEM;
959 goto error;
960 }
961
962 vqs[i] = vq;
963 vring->vq = vq;
964 vq->priv = vring;
965 }
966
967 return 0;
968
969error:
970 mlxbf_tmfifo_virtio_del_vqs(vdev);
971 return ret;
972}
973
974/* Read the status byte. */
975static u8 mlxbf_tmfifo_virtio_get_status(struct virtio_device *vdev)
976{
977 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
978
979 return tm_vdev->status;
980}
981
982/* Write the status byte. */
983static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
984 u8 status)
985{
986 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
987
988 tm_vdev->status = status;
989}
990
991/* Reset the device. Not much here for now. */
992static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
993{
994 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
995
996 tm_vdev->status = 0;
997}
998
999/* Read the value of a configuration field. */
1000static void mlxbf_tmfifo_virtio_get(struct virtio_device *vdev,
1001 unsigned int offset,
1002 void *buf,
1003 unsigned int len)
1004{
1005 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1006
1007 if ((u64)offset + len > sizeof(tm_vdev->config))
1008 return;
1009
1010 memcpy(buf, (u8 *)&tm_vdev->config + offset, len);
1011}
1012
1013/* Write the value of a configuration field. */
1014static void mlxbf_tmfifo_virtio_set(struct virtio_device *vdev,
1015 unsigned int offset,
1016 const void *buf,
1017 unsigned int len)
1018{
1019 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1020
1021 if ((u64)offset + len > sizeof(tm_vdev->config))
1022 return;
1023
1024 memcpy((u8 *)&tm_vdev->config + offset, buf, len);
1025}
1026
1027static void tmfifo_virtio_dev_release(struct device *device)
1028{
1029 struct virtio_device *vdev =
1030 container_of(device, struct virtio_device, dev);
1031 struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
1032
1033 kfree(tm_vdev);
1034}
1035
1036/* Virtio config operations. */
1037static const struct virtio_config_ops mlxbf_tmfifo_virtio_config_ops = {
1038 .get_features = mlxbf_tmfifo_virtio_get_features,
1039 .finalize_features = mlxbf_tmfifo_virtio_finalize_features,
1040 .find_vqs = mlxbf_tmfifo_virtio_find_vqs,
1041 .del_vqs = mlxbf_tmfifo_virtio_del_vqs,
1042 .reset = mlxbf_tmfifo_virtio_reset,
1043 .set_status = mlxbf_tmfifo_virtio_set_status,
1044 .get_status = mlxbf_tmfifo_virtio_get_status,
1045 .get = mlxbf_tmfifo_virtio_get,
1046 .set = mlxbf_tmfifo_virtio_set,
1047};
1048
1049/* Create vdev for the FIFO. */
1050static int mlxbf_tmfifo_create_vdev(struct device *dev,
1051 struct mlxbf_tmfifo *fifo,
1052 int vdev_id, u64 features,
1053 void *config, u32 size)
1054{
1055 struct mlxbf_tmfifo_vdev *tm_vdev, *reg_dev = NULL;
1056 int ret;
1057
1058 mutex_lock(&fifo->lock);
1059
1060 tm_vdev = fifo->vdev[vdev_id];
1061 if (tm_vdev) {
1062 dev_err(dev, "vdev %d already exists\n", vdev_id);
1063 ret = -EEXIST;
1064 goto fail;
1065 }
1066
1067 tm_vdev = kzalloc(sizeof(*tm_vdev), GFP_KERNEL);
1068 if (!tm_vdev) {
1069 ret = -ENOMEM;
1070 goto fail;
1071 }
1072
1073 tm_vdev->vdev.id.device = vdev_id;
1074 tm_vdev->vdev.config = &mlxbf_tmfifo_virtio_config_ops;
1075 tm_vdev->vdev.dev.parent = dev;
1076 tm_vdev->vdev.dev.release = tmfifo_virtio_dev_release;
1077 tm_vdev->features = features;
1078 if (config)
1079 memcpy(&tm_vdev->config, config, size);
1080
1081 if (mlxbf_tmfifo_alloc_vrings(fifo, tm_vdev)) {
1082 dev_err(dev, "unable to allocate vring\n");
1083 ret = -ENOMEM;
1084 goto vdev_fail;
1085 }
1086
1087 /* Allocate an output buffer for the console device. */
1088 if (vdev_id == VIRTIO_ID_CONSOLE)
1089 tm_vdev->tx_buf.buf = devm_kmalloc(dev,
1090 MLXBF_TMFIFO_CON_TX_BUF_SIZE,
1091 GFP_KERNEL);
1092 fifo->vdev[vdev_id] = tm_vdev;
1093
1094 /* Register the virtio device. */
1095 ret = register_virtio_device(&tm_vdev->vdev);
1096 reg_dev = tm_vdev;
1097 if (ret) {
1098 dev_err(dev, "register_virtio_device failed\n");
1099 goto vdev_fail;
1100 }
1101
1102 mutex_unlock(&fifo->lock);
1103 return 0;
1104
1105vdev_fail:
1106 mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
1107 fifo->vdev[vdev_id] = NULL;
1108 if (reg_dev)
1109 put_device(&tm_vdev->vdev.dev);
1110 else
1111 kfree(tm_vdev);
1112fail:
1113 mutex_unlock(&fifo->lock);
1114 return ret;
1115}
1116
1117/* Delete vdev for the FIFO. */
1118static int mlxbf_tmfifo_delete_vdev(struct mlxbf_tmfifo *fifo, int vdev_id)
1119{
1120 struct mlxbf_tmfifo_vdev *tm_vdev;
1121
1122 mutex_lock(&fifo->lock);
1123
1124 /* Unregister vdev. */
1125 tm_vdev = fifo->vdev[vdev_id];
1126 if (tm_vdev) {
1127 unregister_virtio_device(&tm_vdev->vdev);
1128 mlxbf_tmfifo_free_vrings(fifo, tm_vdev);
1129 fifo->vdev[vdev_id] = NULL;
1130 }
1131
1132 mutex_unlock(&fifo->lock);
1133
1134 return 0;
1135}
1136
1137/* Read the configured network MAC address from efi variable. */
1138static void mlxbf_tmfifo_get_cfg_mac(u8 *mac)
1139{
1140 efi_guid_t guid = EFI_GLOBAL_VARIABLE_GUID;
1141 unsigned long size = ETH_ALEN;
1142 u8 buf[ETH_ALEN];
1143 efi_status_t rc;
1144
1145 rc = efi.get_variable(mlxbf_tmfifo_efi_name, &guid, NULL, &size, buf);
1146 if (rc == EFI_SUCCESS && size == ETH_ALEN)
1147 ether_addr_copy(mac, buf);
1148 else
1149 ether_addr_copy(mac, mlxbf_tmfifo_net_default_mac);
1150}
1151
1152/* Set TmFifo thresolds which is used to trigger interrupts. */
1153static void mlxbf_tmfifo_set_threshold(struct mlxbf_tmfifo *fifo)
1154{
1155 u64 ctl;
1156
1157 /* Get Tx FIFO size and set the low/high watermark. */
1158 ctl = readq(fifo->tx_base + MLXBF_TMFIFO_TX_CTL);
1159 fifo->tx_fifo_size =
1160 FIELD_GET(MLXBF_TMFIFO_TX_CTL__MAX_ENTRIES_MASK, ctl);
1161 ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__LWM_MASK) |
1162 FIELD_PREP(MLXBF_TMFIFO_TX_CTL__LWM_MASK,
1163 fifo->tx_fifo_size / 2);
1164 ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__HWM_MASK) |
1165 FIELD_PREP(MLXBF_TMFIFO_TX_CTL__HWM_MASK,
1166 fifo->tx_fifo_size - 1);
1167 writeq(ctl, fifo->tx_base + MLXBF_TMFIFO_TX_CTL);
1168
1169 /* Get Rx FIFO size and set the low/high watermark. */
1170 ctl = readq(fifo->rx_base + MLXBF_TMFIFO_RX_CTL);
1171 fifo->rx_fifo_size =
1172 FIELD_GET(MLXBF_TMFIFO_RX_CTL__MAX_ENTRIES_MASK, ctl);
1173 ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__LWM_MASK) |
1174 FIELD_PREP(MLXBF_TMFIFO_RX_CTL__LWM_MASK, 0);
1175 ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__HWM_MASK) |
1176 FIELD_PREP(MLXBF_TMFIFO_RX_CTL__HWM_MASK, 1);
1177 writeq(ctl, fifo->rx_base + MLXBF_TMFIFO_RX_CTL);
1178}
1179
1180static void mlxbf_tmfifo_cleanup(struct mlxbf_tmfifo *fifo)
1181{
1182 int i;
1183
1184 fifo->is_ready = false;
1185 del_timer_sync(&fifo->timer);
1186 mlxbf_tmfifo_disable_irqs(fifo);
1187 cancel_work_sync(&fifo->work);
1188 for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++)
1189 mlxbf_tmfifo_delete_vdev(fifo, i);
1190}
1191
1192/* Probe the TMFIFO. */
1193static int mlxbf_tmfifo_probe(struct platform_device *pdev)
1194{
1195 struct virtio_net_config net_config;
1196 struct device *dev = &pdev->dev;
1197 struct mlxbf_tmfifo *fifo;
1198 int i, rc;
1199
1200 fifo = devm_kzalloc(dev, sizeof(*fifo), GFP_KERNEL);
1201 if (!fifo)
1202 return -ENOMEM;
1203
1204 spin_lock_init(&fifo->spin_lock[0]);
1205 spin_lock_init(&fifo->spin_lock[1]);
1206 INIT_WORK(&fifo->work, mlxbf_tmfifo_work_handler);
1207 mutex_init(&fifo->lock);
1208
1209 /* Get the resource of the Rx FIFO. */
1210 fifo->rx_base = devm_platform_ioremap_resource(pdev, 0);
1211 if (IS_ERR(fifo->rx_base))
1212 return PTR_ERR(fifo->rx_base);
1213
1214 /* Get the resource of the Tx FIFO. */
1215 fifo->tx_base = devm_platform_ioremap_resource(pdev, 1);
1216 if (IS_ERR(fifo->tx_base))
1217 return PTR_ERR(fifo->tx_base);
1218
1219 platform_set_drvdata(pdev, fifo);
1220
1221 timer_setup(&fifo->timer, mlxbf_tmfifo_timer, 0);
1222
1223 for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) {
1224 fifo->irq_info[i].index = i;
1225 fifo->irq_info[i].fifo = fifo;
1226 fifo->irq_info[i].irq = platform_get_irq(pdev, i);
1227 rc = devm_request_irq(dev, fifo->irq_info[i].irq,
1228 mlxbf_tmfifo_irq_handler, 0,
1229 "tmfifo", &fifo->irq_info[i]);
1230 if (rc) {
1231 dev_err(dev, "devm_request_irq failed\n");
1232 fifo->irq_info[i].irq = 0;
1233 return rc;
1234 }
1235 }
1236
1237 mlxbf_tmfifo_set_threshold(fifo);
1238
1239 /* Create the console vdev. */
1240 rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_CONSOLE, 0, NULL, 0);
1241 if (rc)
1242 goto fail;
1243
1244 /* Create the network vdev. */
1245 memset(&net_config, 0, sizeof(net_config));
1246
1247 /* A legacy-only interface for now. */
1248 net_config.mtu = __cpu_to_virtio16(virtio_legacy_is_little_endian(),
1249 ETH_DATA_LEN);
1250 net_config.status = __cpu_to_virtio16(virtio_legacy_is_little_endian(),
1251 VIRTIO_NET_S_LINK_UP);
1252 mlxbf_tmfifo_get_cfg_mac(net_config.mac);
1253 rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_NET,
1254 MLXBF_TMFIFO_NET_FEATURES, &net_config,
1255 sizeof(net_config));
1256 if (rc)
1257 goto fail;
1258
1259 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL);
1260
1261 fifo->is_ready = true;
1262 return 0;
1263
1264fail:
1265 mlxbf_tmfifo_cleanup(fifo);
1266 return rc;
1267}
1268
1269/* Device remove function. */
1270static int mlxbf_tmfifo_remove(struct platform_device *pdev)
1271{
1272 struct mlxbf_tmfifo *fifo = platform_get_drvdata(pdev);
1273
1274 mlxbf_tmfifo_cleanup(fifo);
1275
1276 return 0;
1277}
1278
1279static const struct acpi_device_id mlxbf_tmfifo_acpi_match[] = {
1280 { "MLNXBF01", 0 },
1281 {}
1282};
1283MODULE_DEVICE_TABLE(acpi, mlxbf_tmfifo_acpi_match);
1284
1285static struct platform_driver mlxbf_tmfifo_driver = {
1286 .probe = mlxbf_tmfifo_probe,
1287 .remove = mlxbf_tmfifo_remove,
1288 .driver = {
1289 .name = "bf-tmfifo",
1290 .acpi_match_table = mlxbf_tmfifo_acpi_match,
1291 },
1292};
1293
1294module_platform_driver(mlxbf_tmfifo_driver);
1295
1296MODULE_DESCRIPTION("Mellanox BlueField SoC TmFifo Driver");
1297MODULE_LICENSE("GPL v2");
1298MODULE_AUTHOR("Mellanox Technologies");