Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * SolidRun DPU driver for control plane
4 *
5 * Copyright (C) 2022-2023 SolidRun
6 *
7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
8 *
9 */
10#include <linux/iopoll.h>
11
12#include "snet_vdpa.h"
13
14/* SNET DPU device ID */
15#define SNET_DEVICE_ID 0x1000
16/* SNET signature */
17#define SNET_SIGNATURE 0xD0D06363
18/* Max. config version that we can work with */
19#define SNET_CFG_VERSION 0x2
20/* Queue align */
21#define SNET_QUEUE_ALIGNMENT PAGE_SIZE
22/* Kick value to notify that new data is available */
23#define SNET_KICK_VAL 0x1
24#define SNET_CONFIG_OFF 0x0
25/* How long we are willing to wait for a SNET device */
26#define SNET_DETECT_TIMEOUT 5000000
27/* How long should we wait for the DPU to read our config */
28#define SNET_READ_CFG_TIMEOUT 3000000
29/* Size of configs written to the DPU */
30#define SNET_GENERAL_CFG_LEN 36
31#define SNET_GENERAL_CFG_VQ_LEN 40
32
33static struct snet *vdpa_to_snet(struct vdpa_device *vdpa)
34{
35 return container_of(vdpa, struct snet, vdpa);
36}
37
38static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data)
39{
40 struct snet *snet = data;
41 /* Call callback if any */
42 if (likely(snet->cb.callback))
43 return snet->cb.callback(snet->cb.private);
44
45 return IRQ_HANDLED;
46}
47
48static irqreturn_t snet_vq_irq_hndlr(int irq, void *data)
49{
50 struct snet_vq *vq = data;
51 /* Call callback if any */
52 if (likely(vq->cb.callback))
53 return vq->cb.callback(vq->cb.private);
54
55 return IRQ_HANDLED;
56}
57
58static void snet_free_irqs(struct snet *snet)
59{
60 struct psnet *psnet = snet->psnet;
61 struct pci_dev *pdev;
62 u32 i;
63
64 /* Which Device allcoated the IRQs? */
65 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))
66 pdev = snet->pdev->physfn;
67 else
68 pdev = snet->pdev;
69
70 /* Free config's IRQ */
71 if (snet->cfg_irq != -1) {
72 devm_free_irq(&pdev->dev, snet->cfg_irq, snet);
73 snet->cfg_irq = -1;
74 }
75 /* Free VQ IRQs */
76 for (i = 0; i < snet->cfg->vq_num; i++) {
77 if (snet->vqs[i] && snet->vqs[i]->irq != -1) {
78 devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]);
79 snet->vqs[i]->irq = -1;
80 }
81 }
82
83 /* IRQ vectors are freed when the pci remove callback is called */
84}
85
86static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area,
87 u64 driver_area, u64 device_area)
88{
89 struct snet *snet = vdpa_to_snet(vdev);
90 /* save received parameters in vqueue sturct */
91 snet->vqs[idx]->desc_area = desc_area;
92 snet->vqs[idx]->driver_area = driver_area;
93 snet->vqs[idx]->device_area = device_area;
94
95 return 0;
96}
97
98static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num)
99{
100 struct snet *snet = vdpa_to_snet(vdev);
101 /* save num in vqueue */
102 snet->vqs[idx]->num = num;
103}
104
105static void snet_kick_vq(struct vdpa_device *vdev, u16 idx)
106{
107 struct snet *snet = vdpa_to_snet(vdev);
108 /* not ready - ignore */
109 if (unlikely(!snet->vqs[idx]->ready))
110 return;
111
112 iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
113}
114
115static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data)
116{
117 struct snet *snet = vdpa_to_snet(vdev);
118 u16 idx = data & 0xFFFF;
119
120 /* not ready - ignore */
121 if (unlikely(!snet->vqs[idx]->ready))
122 return;
123
124 iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
125}
126
127static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb)
128{
129 struct snet *snet = vdpa_to_snet(vdev);
130
131 snet->vqs[idx]->cb.callback = cb->callback;
132 snet->vqs[idx]->cb.private = cb->private;
133}
134
135static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready)
136{
137 struct snet *snet = vdpa_to_snet(vdev);
138
139 snet->vqs[idx]->ready = ready;
140}
141
142static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx)
143{
144 struct snet *snet = vdpa_to_snet(vdev);
145
146 return snet->vqs[idx]->ready;
147}
148
149static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state)
150{
151 if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) {
152 const struct vdpa_vq_state_packed *p = &state->packed;
153
154 if (p->last_avail_counter == 1 && p->last_used_counter == 1 &&
155 p->last_avail_idx == 0 && p->last_used_idx == 0)
156 return true;
157 } else {
158 const struct vdpa_vq_state_split *s = &state->split;
159
160 if (s->avail_index == 0)
161 return true;
162 }
163
164 return false;
165}
166
167static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state)
168{
169 struct snet *snet = vdpa_to_snet(vdev);
170
171 /* We can set any state for config version 2+ */
172 if (SNET_CFG_VER(snet, 2)) {
173 memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state));
174 return 0;
175 }
176
177 /* Older config - we can't set the VQ state.
178 * Return 0 only if this is the initial state we use in the DPU.
179 */
180 if (snet_vq_state_is_initial(snet, state))
181 return 0;
182
183 return -EOPNOTSUPP;
184}
185
186static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)
187{
188 struct snet *snet = vdpa_to_snet(vdev);
189
190 return snet_read_vq_state(snet, idx, state);
191}
192
193static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx)
194{
195 struct snet *snet = vdpa_to_snet(vdev);
196
197 return snet->vqs[idx]->irq;
198}
199
200static u32 snet_get_vq_align(struct vdpa_device *vdev)
201{
202 return (u32)SNET_QUEUE_ALIGNMENT;
203}
204
205static int snet_reset_dev(struct snet *snet)
206{
207 struct pci_dev *pdev = snet->pdev;
208 int ret = 0;
209 u32 i;
210
211 /* If status is 0, nothing to do */
212 if (!snet->status)
213 return 0;
214
215 /* If DPU started, destroy it */
216 if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK)
217 ret = snet_destroy_dev(snet);
218
219 /* Clear VQs */
220 for (i = 0; i < snet->cfg->vq_num; i++) {
221 if (!snet->vqs[i])
222 continue;
223 snet->vqs[i]->cb.callback = NULL;
224 snet->vqs[i]->cb.private = NULL;
225 snet->vqs[i]->desc_area = 0;
226 snet->vqs[i]->device_area = 0;
227 snet->vqs[i]->driver_area = 0;
228 snet->vqs[i]->ready = false;
229 }
230
231 /* Clear config callback */
232 snet->cb.callback = NULL;
233 snet->cb.private = NULL;
234 /* Free IRQs */
235 snet_free_irqs(snet);
236 /* Reset status */
237 snet->status = 0;
238 snet->dpu_ready = false;
239
240 if (ret)
241 SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret);
242 else
243 SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid);
244
245 return 0;
246}
247
248static int snet_reset(struct vdpa_device *vdev)
249{
250 struct snet *snet = vdpa_to_snet(vdev);
251
252 return snet_reset_dev(snet);
253}
254
255static size_t snet_get_config_size(struct vdpa_device *vdev)
256{
257 struct snet *snet = vdpa_to_snet(vdev);
258
259 return (size_t)snet->cfg->cfg_size;
260}
261
262static u64 snet_get_features(struct vdpa_device *vdev)
263{
264 struct snet *snet = vdpa_to_snet(vdev);
265
266 return snet->cfg->features;
267}
268
269static int snet_set_drv_features(struct vdpa_device *vdev, u64 features)
270{
271 struct snet *snet = vdpa_to_snet(vdev);
272
273 snet->negotiated_features = snet->cfg->features & features;
274 return 0;
275}
276
277static u64 snet_get_drv_features(struct vdpa_device *vdev)
278{
279 struct snet *snet = vdpa_to_snet(vdev);
280
281 return snet->negotiated_features;
282}
283
284static u16 snet_get_vq_num_max(struct vdpa_device *vdev)
285{
286 struct snet *snet = vdpa_to_snet(vdev);
287
288 return (u16)snet->cfg->vq_size;
289}
290
291static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb)
292{
293 struct snet *snet = vdpa_to_snet(vdev);
294
295 snet->cb.callback = cb->callback;
296 snet->cb.private = cb->private;
297}
298
299static u32 snet_get_device_id(struct vdpa_device *vdev)
300{
301 struct snet *snet = vdpa_to_snet(vdev);
302
303 return snet->cfg->virtio_id;
304}
305
306static u32 snet_get_vendor_id(struct vdpa_device *vdev)
307{
308 return (u32)PCI_VENDOR_ID_SOLIDRUN;
309}
310
311static u8 snet_get_status(struct vdpa_device *vdev)
312{
313 struct snet *snet = vdpa_to_snet(vdev);
314
315 return snet->status;
316}
317
318static int snet_write_conf(struct snet *snet)
319{
320 u32 off, i, tmp;
321 int ret;
322
323 /* No need to write the config twice */
324 if (snet->dpu_ready)
325 return true;
326
327 /* Snet data :
328 *
329 * General data: SNET_GENERAL_CFG_LEN bytes long
330 * 0 0x4 0x8 0xC 0x10 0x14 0x1C 0x24
331 * | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES | RSVD |
332 *
333 * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long
334 * 0 0x4 0x8
335 * | VQ SID AND QUEUE SIZE | IRQ Index |
336 * | DESC AREA |
337 * | DEVICE AREA |
338 * | DRIVER AREA |
339 * | VQ STATE (CFG 2+) | RSVD |
340 *
341 * Magic number should be written last, this is the DPU indication that the data is ready
342 */
343
344 /* Init offset */
345 off = snet->psnet->cfg.host_cfg_off;
346
347 /* Ignore magic number for now */
348 off += 4;
349 snet_write32(snet, off, snet->psnet->negotiated_cfg_ver);
350 off += 4;
351 snet_write32(snet, off, snet->sid);
352 off += 4;
353 snet_write32(snet, off, snet->cfg->vq_num);
354 off += 4;
355 snet_write32(snet, off, snet->cfg_irq_idx);
356 off += 4;
357 snet_write64(snet, off, snet->negotiated_features);
358 off += 8;
359 /* Ignore reserved */
360 off += 8;
361 /* Write VQs */
362 for (i = 0 ; i < snet->cfg->vq_num ; i++) {
363 tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF);
364 snet_write32(snet, off, tmp);
365 off += 4;
366 snet_write32(snet, off, snet->vqs[i]->irq_idx);
367 off += 4;
368 snet_write64(snet, off, snet->vqs[i]->desc_area);
369 off += 8;
370 snet_write64(snet, off, snet->vqs[i]->device_area);
371 off += 8;
372 snet_write64(snet, off, snet->vqs[i]->driver_area);
373 off += 8;
374 /* Write VQ state if config version is 2+ */
375 if (SNET_CFG_VER(snet, 2))
376 snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state);
377 off += 4;
378
379 /* Ignore reserved */
380 off += 4;
381 }
382
383 /* Write magic number - data is ready */
384 snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE);
385
386 /* The DPU will ACK the config by clearing the signature */
387 ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off,
388 tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT);
389 if (ret) {
390 SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n");
391 return false;
392 }
393
394 /* set DPU flag */
395 snet->dpu_ready = true;
396
397 return true;
398}
399
400static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet)
401{
402 int ret, i, irq;
403
404 /* Request config IRQ */
405 irq = pci_irq_vector(pdev, snet->cfg_irq_idx);
406 ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0,
407 snet->cfg_irq_name, snet);
408 if (ret) {
409 SNET_ERR(pdev, "Failed to request IRQ\n");
410 return ret;
411 }
412 snet->cfg_irq = irq;
413
414 /* Request IRQ for every VQ */
415 for (i = 0; i < snet->cfg->vq_num; i++) {
416 irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx);
417 ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0,
418 snet->vqs[i]->irq_name, snet->vqs[i]);
419 if (ret) {
420 SNET_ERR(pdev, "Failed to request IRQ\n");
421 return ret;
422 }
423 snet->vqs[i]->irq = irq;
424 }
425 return 0;
426}
427
428static void snet_set_status(struct vdpa_device *vdev, u8 status)
429{
430 struct snet *snet = vdpa_to_snet(vdev);
431 struct psnet *psnet = snet->psnet;
432 struct pci_dev *pdev = snet->pdev;
433 int ret;
434 bool pf_irqs;
435
436 if (status == snet->status)
437 return;
438
439 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
440 !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
441 /* Request IRQs */
442 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);
443 ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet);
444 if (ret)
445 goto set_err;
446
447 /* Write config to the DPU */
448 if (snet_write_conf(snet)) {
449 SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid);
450 } else {
451 snet_free_irqs(snet);
452 goto set_err;
453 }
454 }
455
456 /* Save the new status */
457 snet->status = status;
458 return;
459
460set_err:
461 snet->status |= VIRTIO_CONFIG_S_FAILED;
462}
463
464static void snet_get_config(struct vdpa_device *vdev, unsigned int offset,
465 void *buf, unsigned int len)
466{
467 struct snet *snet = vdpa_to_snet(vdev);
468 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset;
469 u8 *buf_ptr = buf;
470 u32 i;
471
472 /* check for offset error */
473 if (offset + len > snet->cfg->cfg_size)
474 return;
475
476 /* Write into buffer */
477 for (i = 0; i < len; i++)
478 *buf_ptr++ = ioread8(cfg_ptr + i);
479}
480
481static void snet_set_config(struct vdpa_device *vdev, unsigned int offset,
482 const void *buf, unsigned int len)
483{
484 struct snet *snet = vdpa_to_snet(vdev);
485 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset;
486 const u8 *buf_ptr = buf;
487 u32 i;
488
489 /* check for offset error */
490 if (offset + len > snet->cfg->cfg_size)
491 return;
492
493 /* Write into PCI BAR */
494 for (i = 0; i < len; i++)
495 iowrite8(*buf_ptr++, cfg_ptr + i);
496}
497
498static int snet_suspend(struct vdpa_device *vdev)
499{
500 struct snet *snet = vdpa_to_snet(vdev);
501 int ret;
502
503 ret = snet_suspend_dev(snet);
504 if (ret)
505 SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret);
506 else
507 SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid);
508
509 return ret;
510}
511
512static int snet_resume(struct vdpa_device *vdev)
513{
514 struct snet *snet = vdpa_to_snet(vdev);
515 int ret;
516
517 ret = snet_resume_dev(snet);
518 if (ret)
519 SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret);
520 else
521 SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid);
522
523 return ret;
524}
525
526static const struct vdpa_config_ops snet_config_ops = {
527 .set_vq_address = snet_set_vq_address,
528 .set_vq_num = snet_set_vq_num,
529 .kick_vq = snet_kick_vq,
530 .kick_vq_with_data = snet_kick_vq_with_data,
531 .set_vq_cb = snet_set_vq_cb,
532 .set_vq_ready = snet_set_vq_ready,
533 .get_vq_ready = snet_get_vq_ready,
534 .set_vq_state = snet_set_vq_state,
535 .get_vq_state = snet_get_vq_state,
536 .get_vq_irq = snet_get_vq_irq,
537 .get_vq_align = snet_get_vq_align,
538 .reset = snet_reset,
539 .get_config_size = snet_get_config_size,
540 .get_device_features = snet_get_features,
541 .set_driver_features = snet_set_drv_features,
542 .get_driver_features = snet_get_drv_features,
543 .get_vq_num_min = snet_get_vq_num_max,
544 .get_vq_num_max = snet_get_vq_num_max,
545 .set_config_cb = snet_set_config_cb,
546 .get_device_id = snet_get_device_id,
547 .get_vendor_id = snet_get_vendor_id,
548 .get_status = snet_get_status,
549 .set_status = snet_set_status,
550 .get_config = snet_get_config,
551 .set_config = snet_set_config,
552 .suspend = snet_suspend,
553 .resume = snet_resume,
554};
555
556static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet)
557{
558 char *name;
559 int ret, i, mask = 0;
560 /* We don't know which BAR will be used to communicate..
561 * We will map every bar with len > 0.
562 *
563 * Later, we will discover the BAR and unmap all other BARs.
564 */
565 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
566 if (pci_resource_len(pdev, i))
567 mask |= (1 << i);
568 }
569
570 /* No BAR can be used.. */
571 if (!mask) {
572 SNET_ERR(pdev, "Failed to find a PCI BAR\n");
573 return -ENODEV;
574 }
575
576 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "psnet[%s]-bars", pci_name(pdev));
577 if (!name)
578 return -ENOMEM;
579
580 ret = pcim_iomap_regions(pdev, mask, name);
581 if (ret) {
582 SNET_ERR(pdev, "Failed to request and map PCI BARs\n");
583 return ret;
584 }
585
586 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
587 if (mask & (1 << i))
588 psnet->bars[i] = pcim_iomap_table(pdev)[i];
589 }
590
591 return 0;
592}
593
594static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet)
595{
596 char *name;
597 int ret;
598
599 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "snet[%s]-bars", pci_name(pdev));
600 if (!name)
601 return -ENOMEM;
602
603 /* Request and map BAR */
604 ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name);
605 if (ret) {
606 SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n");
607 return ret;
608 }
609
610 snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar];
611
612 return 0;
613}
614
615static void snet_free_cfg(struct snet_cfg *cfg)
616{
617 u32 i;
618
619 if (!cfg->devs)
620 return;
621
622 /* Free devices */
623 for (i = 0; i < cfg->devices_num; i++) {
624 if (!cfg->devs[i])
625 break;
626
627 kfree(cfg->devs[i]);
628 }
629 /* Free pointers to devices */
630 kfree(cfg->devs);
631}
632
633/* Detect which BAR is used for communication with the device. */
634static int psnet_detect_bar(struct psnet *psnet, u32 off)
635{
636 unsigned long exit_time;
637 int i;
638
639 exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT);
640
641 /* SNET DPU will write SNET's signature when the config is ready. */
642 while (time_before(jiffies, exit_time)) {
643 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
644 /* Is this BAR mapped? */
645 if (!psnet->bars[i])
646 continue;
647
648 if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE)
649 return i;
650 }
651 usleep_range(1000, 10000);
652 }
653
654 return -ENODEV;
655}
656
657static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet)
658{
659 int i, mask = 0;
660
661 for (i = 0; i < PCI_STD_NUM_BARS; i++) {
662 if (psnet->bars[i] && i != psnet->barno)
663 mask |= (1 << i);
664 }
665
666 if (mask)
667 pcim_iounmap_regions(pdev, mask);
668}
669
670/* Read SNET config from PCI BAR */
671static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet)
672{
673 struct snet_cfg *cfg = &psnet->cfg;
674 u32 i, off;
675 int barno;
676
677 /* Move to where the config starts */
678 off = SNET_CONFIG_OFF;
679
680 /* Find BAR used for communication */
681 barno = psnet_detect_bar(psnet, off);
682 if (barno < 0) {
683 SNET_ERR(pdev, "SNET config is not ready.\n");
684 return barno;
685 }
686
687 /* Save used BAR number and unmap all other BARs */
688 psnet->barno = barno;
689 SNET_DBG(pdev, "Using BAR number %d\n", barno);
690
691 psnet_unmap_unused_bars(pdev, psnet);
692
693 /* load config from BAR */
694 cfg->key = psnet_read32(psnet, off);
695 off += 4;
696 cfg->cfg_size = psnet_read32(psnet, off);
697 off += 4;
698 cfg->cfg_ver = psnet_read32(psnet, off);
699 off += 4;
700 /* The negotiated config version is the lower one between this driver's config
701 * and the DPU's.
702 */
703 psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION);
704 SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver);
705
706 cfg->vf_num = psnet_read32(psnet, off);
707 off += 4;
708 cfg->vf_bar = psnet_read32(psnet, off);
709 off += 4;
710 cfg->host_cfg_off = psnet_read32(psnet, off);
711 off += 4;
712 cfg->max_size_host_cfg = psnet_read32(psnet, off);
713 off += 4;
714 cfg->virtio_cfg_off = psnet_read32(psnet, off);
715 off += 4;
716 cfg->kick_off = psnet_read32(psnet, off);
717 off += 4;
718 cfg->hwmon_off = psnet_read32(psnet, off);
719 off += 4;
720 cfg->ctrl_off = psnet_read32(psnet, off);
721 off += 4;
722 cfg->flags = psnet_read32(psnet, off);
723 off += 4;
724 /* Ignore Reserved */
725 off += sizeof(cfg->rsvd);
726
727 cfg->devices_num = psnet_read32(psnet, off);
728 off += 4;
729 /* Allocate memory to hold pointer to the devices */
730 cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL);
731 if (!cfg->devs)
732 return -ENOMEM;
733
734 /* Load device configuration from BAR */
735 for (i = 0; i < cfg->devices_num; i++) {
736 cfg->devs[i] = kzalloc(sizeof(*cfg->devs[i]), GFP_KERNEL);
737 if (!cfg->devs[i]) {
738 snet_free_cfg(cfg);
739 return -ENOMEM;
740 }
741 /* Read device config */
742 cfg->devs[i]->virtio_id = psnet_read32(psnet, off);
743 off += 4;
744 cfg->devs[i]->vq_num = psnet_read32(psnet, off);
745 off += 4;
746 cfg->devs[i]->vq_size = psnet_read32(psnet, off);
747 off += 4;
748 cfg->devs[i]->vfid = psnet_read32(psnet, off);
749 off += 4;
750 cfg->devs[i]->features = psnet_read64(psnet, off);
751 off += 8;
752 /* Ignore Reserved */
753 off += sizeof(cfg->devs[i]->rsvd);
754
755 cfg->devs[i]->cfg_size = psnet_read32(psnet, off);
756 off += 4;
757
758 /* Is the config witten to the DPU going to be too big? */
759 if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num >
760 cfg->max_size_host_cfg) {
761 SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n");
762 snet_free_cfg(cfg);
763 return -EINVAL;
764 }
765 }
766 return 0;
767}
768
769static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet)
770{
771 int ret = 0;
772 u32 i, irq_num = 0;
773
774 /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */
775 for (i = 0; i < psnet->cfg.devices_num; i++)
776 irq_num += psnet->cfg.devs[i]->vq_num + 1;
777
778 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX);
779 if (ret != irq_num) {
780 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n");
781 return ret;
782 }
783 SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num);
784
785 return 0;
786}
787
788static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg)
789{
790 int ret = 0;
791 u32 irq_num;
792
793 /* We want 1 IRQ for every VQ + 1 for config change events */
794 irq_num = snet_cfg->vq_num + 1;
795
796 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX);
797 if (ret <= 0) {
798 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n");
799 return ret;
800 }
801
802 return 0;
803}
804
805static void snet_free_vqs(struct snet *snet)
806{
807 u32 i;
808
809 if (!snet->vqs)
810 return;
811
812 for (i = 0 ; i < snet->cfg->vq_num ; i++) {
813 if (!snet->vqs[i])
814 break;
815
816 kfree(snet->vqs[i]);
817 }
818 kfree(snet->vqs);
819}
820
821static int snet_build_vqs(struct snet *snet)
822{
823 u32 i;
824 /* Allocate the VQ pointers array */
825 snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL);
826 if (!snet->vqs)
827 return -ENOMEM;
828
829 /* Allocate the VQs */
830 for (i = 0; i < snet->cfg->vq_num; i++) {
831 snet->vqs[i] = kzalloc(sizeof(*snet->vqs[i]), GFP_KERNEL);
832 if (!snet->vqs[i]) {
833 snet_free_vqs(snet);
834 return -ENOMEM;
835 }
836 /* Reset IRQ num */
837 snet->vqs[i]->irq = -1;
838 /* VQ serial ID */
839 snet->vqs[i]->sid = i;
840 /* Kick address - every VQ gets 4B */
841 snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off +
842 snet->vqs[i]->sid * 4;
843 /* Clear kick address for this VQ */
844 iowrite32(0, snet->vqs[i]->kick_ptr);
845 }
846 return 0;
847}
848
849static int psnet_get_next_irq_num(struct psnet *psnet)
850{
851 int irq;
852
853 spin_lock(&psnet->lock);
854 irq = psnet->next_irq++;
855 spin_unlock(&psnet->lock);
856
857 return irq;
858}
859
860static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet)
861{
862 struct psnet *psnet = snet->psnet;
863 int i;
864
865 /* one IRQ for every VQ, and one for config changes */
866 snet->cfg_irq_idx = psnet_get_next_irq_num(psnet);
867 snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]",
868 pci_name(pdev), snet->cfg_irq_idx);
869
870 for (i = 0; i < snet->cfg->vq_num; i++) {
871 /* Get next free IRQ ID */
872 snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet);
873 /* Write IRQ name */
874 snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]",
875 pci_name(pdev), snet->vqs[i]->irq_idx);
876 }
877}
878
879/* Find a device config based on virtual function id */
880static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid)
881{
882 u32 i;
883
884 for (i = 0; i < cfg->devices_num; i++) {
885 if (cfg->devs[i]->vfid == vfid)
886 return cfg->devs[i];
887 }
888 /* Oppss.. no config found.. */
889 return NULL;
890}
891
892/* Probe function for a physical PCI function */
893static int snet_vdpa_probe_pf(struct pci_dev *pdev)
894{
895 struct psnet *psnet;
896 int ret = 0;
897 bool pf_irqs = false;
898
899 ret = pcim_enable_device(pdev);
900 if (ret) {
901 SNET_ERR(pdev, "Failed to enable PCI device\n");
902 return ret;
903 }
904
905 /* Allocate a PCI physical function device */
906 psnet = kzalloc(sizeof(*psnet), GFP_KERNEL);
907 if (!psnet)
908 return -ENOMEM;
909
910 /* Init PSNET spinlock */
911 spin_lock_init(&psnet->lock);
912
913 pci_set_master(pdev);
914 pci_set_drvdata(pdev, psnet);
915
916 /* Open SNET MAIN BAR */
917 ret = psnet_open_pf_bar(pdev, psnet);
918 if (ret)
919 goto free_psnet;
920
921 /* Try to read SNET's config from PCI BAR */
922 ret = psnet_read_cfg(pdev, psnet);
923 if (ret)
924 goto free_psnet;
925
926 /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use
927 * PF MSI-X vectors
928 */
929 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);
930
931 if (pf_irqs) {
932 ret = psnet_alloc_irq_vector(pdev, psnet);
933 if (ret)
934 goto free_cfg;
935 }
936
937 SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num);
938 ret = pci_enable_sriov(pdev, psnet->cfg.vf_num);
939 if (ret) {
940 SNET_ERR(pdev, "Failed to enable SR-IOV\n");
941 goto free_irq;
942 }
943
944 /* Create HW monitor device */
945 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) {
946#if IS_ENABLED(CONFIG_HWMON)
947 psnet_create_hwmon(pdev);
948#else
949 SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n");
950#endif
951 }
952
953 return 0;
954
955free_irq:
956 if (pf_irqs)
957 pci_free_irq_vectors(pdev);
958free_cfg:
959 snet_free_cfg(&psnet->cfg);
960free_psnet:
961 kfree(psnet);
962 return ret;
963}
964
965/* Probe function for a virtual PCI function */
966static int snet_vdpa_probe_vf(struct pci_dev *pdev)
967{
968 struct pci_dev *pdev_pf = pdev->physfn;
969 struct psnet *psnet = pci_get_drvdata(pdev_pf);
970 struct snet_dev_cfg *dev_cfg;
971 struct snet *snet;
972 u32 vfid;
973 int ret;
974 bool pf_irqs = false;
975
976 /* Get virtual function id.
977 * (the DPU counts the VFs from 1)
978 */
979 ret = pci_iov_vf_id(pdev);
980 if (ret < 0) {
981 SNET_ERR(pdev, "Failed to find a VF id\n");
982 return ret;
983 }
984 vfid = ret + 1;
985
986 /* Find the snet_dev_cfg based on vfid */
987 dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid);
988 if (!dev_cfg) {
989 SNET_WARN(pdev, "Failed to find a VF config..\n");
990 return -ENODEV;
991 }
992
993 /* Which PCI device should allocate the IRQs?
994 * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs
995 */
996 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);
997
998 ret = pcim_enable_device(pdev);
999 if (ret) {
1000 SNET_ERR(pdev, "Failed to enable PCI VF device\n");
1001 return ret;
1002 }
1003
1004 /* Request for MSI-X IRQs */
1005 if (!pf_irqs) {
1006 ret = snet_alloc_irq_vector(pdev, dev_cfg);
1007 if (ret)
1008 return ret;
1009 }
1010
1011 /* Allocate vdpa device */
1012 snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL,
1013 false);
1014 if (!snet) {
1015 SNET_ERR(pdev, "Failed to allocate a vdpa device\n");
1016 ret = -ENOMEM;
1017 goto free_irqs;
1018 }
1019
1020 /* Init control mutex and spinlock */
1021 mutex_init(&snet->ctrl_lock);
1022 spin_lock_init(&snet->ctrl_spinlock);
1023
1024 /* Save pci device pointer */
1025 snet->pdev = pdev;
1026 snet->psnet = psnet;
1027 snet->cfg = dev_cfg;
1028 snet->dpu_ready = false;
1029 snet->sid = vfid;
1030 /* Reset IRQ value */
1031 snet->cfg_irq = -1;
1032
1033 ret = snet_open_vf_bar(pdev, snet);
1034 if (ret)
1035 goto put_device;
1036
1037 /* Create a VirtIO config pointer */
1038 snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off;
1039
1040 /* Clear control registers */
1041 snet_ctrl_clear(snet);
1042
1043 pci_set_master(pdev);
1044 pci_set_drvdata(pdev, snet);
1045
1046 ret = snet_build_vqs(snet);
1047 if (ret)
1048 goto put_device;
1049
1050 /* Reserve IRQ indexes,
1051 * The IRQs may be requested and freed multiple times,
1052 * but the indexes won't change.
1053 */
1054 snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet);
1055
1056 /*set DMA device*/
1057 snet->vdpa.dma_dev = &pdev->dev;
1058
1059 /* Register VDPA device */
1060 ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num);
1061 if (ret) {
1062 SNET_ERR(pdev, "Failed to register vdpa device\n");
1063 goto free_vqs;
1064 }
1065
1066 return 0;
1067
1068free_vqs:
1069 snet_free_vqs(snet);
1070put_device:
1071 put_device(&snet->vdpa.dev);
1072free_irqs:
1073 if (!pf_irqs)
1074 pci_free_irq_vectors(pdev);
1075 return ret;
1076}
1077
1078static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1079{
1080 if (pdev->is_virtfn)
1081 return snet_vdpa_probe_vf(pdev);
1082 else
1083 return snet_vdpa_probe_pf(pdev);
1084}
1085
1086static void snet_vdpa_remove_pf(struct pci_dev *pdev)
1087{
1088 struct psnet *psnet = pci_get_drvdata(pdev);
1089
1090 pci_disable_sriov(pdev);
1091 /* If IRQs are allocated from the PF, we should free the IRQs */
1092 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))
1093 pci_free_irq_vectors(pdev);
1094
1095 snet_free_cfg(&psnet->cfg);
1096 kfree(psnet);
1097}
1098
1099static void snet_vdpa_remove_vf(struct pci_dev *pdev)
1100{
1101 struct snet *snet = pci_get_drvdata(pdev);
1102 struct psnet *psnet = snet->psnet;
1103
1104 vdpa_unregister_device(&snet->vdpa);
1105 snet_free_vqs(snet);
1106 /* If IRQs are allocated from the VF, we should free the IRQs */
1107 if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))
1108 pci_free_irq_vectors(pdev);
1109}
1110
1111static void snet_vdpa_remove(struct pci_dev *pdev)
1112{
1113 if (pdev->is_virtfn)
1114 snet_vdpa_remove_vf(pdev);
1115 else
1116 snet_vdpa_remove_pf(pdev);
1117}
1118
1119static struct pci_device_id snet_driver_pci_ids[] = {
1120 { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID,
1121 PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) },
1122 { 0 },
1123};
1124
1125MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids);
1126
1127static struct pci_driver snet_vdpa_driver = {
1128 .name = "snet-vdpa-driver",
1129 .id_table = snet_driver_pci_ids,
1130 .probe = snet_vdpa_probe,
1131 .remove = snet_vdpa_remove,
1132};
1133
1134module_pci_driver(snet_vdpa_driver);
1135
1136MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>");
1137MODULE_DESCRIPTION("SolidRun vDPA driver");
1138MODULE_LICENSE("GPL v2");