Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VIRTIO based driver for vDPA device
4 *
5 * Copyright (c) 2020, Red Hat. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/device.h>
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/uuid.h>
16#include <linux/virtio.h>
17#include <linux/vdpa.h>
18#include <linux/virtio_config.h>
19#include <linux/virtio_ring.h>
20
21#define MOD_VERSION "0.1"
22#define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
23#define MOD_DESC "vDPA bus driver for virtio devices"
24#define MOD_LICENSE "GPL v2"
25
26struct virtio_vdpa_device {
27 struct virtio_device vdev;
28 struct vdpa_device *vdpa;
29 u64 features;
30
31 /* The lock to protect virtqueue list */
32 spinlock_t lock;
33 /* List of virtio_vdpa_vq_info */
34 struct list_head virtqueues;
35};
36
37struct virtio_vdpa_vq_info {
38 /* the actual virtqueue */
39 struct virtqueue *vq;
40
41 /* the list node for the virtqueues list */
42 struct list_head node;
43};
44
45static inline struct virtio_vdpa_device *
46to_virtio_vdpa_device(struct virtio_device *dev)
47{
48 return container_of(dev, struct virtio_vdpa_device, vdev);
49}
50
51static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
52{
53 return to_virtio_vdpa_device(vdev)->vdpa;
54}
55
56static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset,
57 void *buf, unsigned int len)
58{
59 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
60
61 vdpa_get_config(vdpa, offset, buf, len);
62}
63
64static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset,
65 const void *buf, unsigned int len)
66{
67 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
68
69 vdpa_set_config(vdpa, offset, buf, len);
70}
71
72static u32 virtio_vdpa_generation(struct virtio_device *vdev)
73{
74 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
75 const struct vdpa_config_ops *ops = vdpa->config;
76
77 if (ops->get_generation)
78 return ops->get_generation(vdpa);
79
80 return 0;
81}
82
83static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
84{
85 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
86 const struct vdpa_config_ops *ops = vdpa->config;
87
88 return ops->get_status(vdpa);
89}
90
91static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
92{
93 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
94
95 return vdpa_set_status(vdpa, status);
96}
97
98static void virtio_vdpa_reset(struct virtio_device *vdev)
99{
100 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
101
102 vdpa_reset(vdpa);
103}
104
105static bool virtio_vdpa_notify(struct virtqueue *vq)
106{
107 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
108 const struct vdpa_config_ops *ops = vdpa->config;
109
110 ops->kick_vq(vdpa, vq->index);
111
112 return true;
113}
114
115static irqreturn_t virtio_vdpa_config_cb(void *private)
116{
117 struct virtio_vdpa_device *vd_dev = private;
118
119 virtio_config_changed(&vd_dev->vdev);
120
121 return IRQ_HANDLED;
122}
123
124static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
125{
126 struct virtio_vdpa_vq_info *info = private;
127
128 return vring_interrupt(0, info->vq);
129}
130
131static struct virtqueue *
132virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
133 void (*callback)(struct virtqueue *vq),
134 const char *name, bool ctx)
135{
136 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
137 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
138 const struct vdpa_config_ops *ops = vdpa->config;
139 struct virtio_vdpa_vq_info *info;
140 struct vdpa_callback cb;
141 struct virtqueue *vq;
142 u64 desc_addr, driver_addr, device_addr;
143 /* Assume split virtqueue, switch to packed if necessary */
144 struct vdpa_vq_state state = {0};
145 unsigned long flags;
146 u32 align, max_num, min_num = 1;
147 bool may_reduce_num = true;
148 int err;
149
150 if (!name)
151 return NULL;
152
153 if (index >= vdpa->nvqs)
154 return ERR_PTR(-ENOENT);
155
156 /* Queue shouldn't already be set up. */
157 if (ops->get_vq_ready(vdpa, index))
158 return ERR_PTR(-ENOENT);
159
160 /* Allocate and fill out our active queue description */
161 info = kmalloc(sizeof(*info), GFP_KERNEL);
162 if (!info)
163 return ERR_PTR(-ENOMEM);
164
165 max_num = ops->get_vq_num_max(vdpa);
166 if (max_num == 0) {
167 err = -ENOENT;
168 goto error_new_virtqueue;
169 }
170
171 if (ops->get_vq_num_min)
172 min_num = ops->get_vq_num_min(vdpa);
173
174 may_reduce_num = (max_num == min_num) ? false : true;
175
176 /* Create the vring */
177 align = ops->get_vq_align(vdpa);
178 vq = vring_create_virtqueue(index, max_num, align, vdev,
179 true, may_reduce_num, ctx,
180 virtio_vdpa_notify, callback, name);
181 if (!vq) {
182 err = -ENOMEM;
183 goto error_new_virtqueue;
184 }
185
186 vq->num_max = max_num;
187
188 /* Setup virtqueue callback */
189 cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
190 cb.private = info;
191 ops->set_vq_cb(vdpa, index, &cb);
192 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
193
194 desc_addr = virtqueue_get_desc_addr(vq);
195 driver_addr = virtqueue_get_avail_addr(vq);
196 device_addr = virtqueue_get_used_addr(vq);
197
198 if (ops->set_vq_address(vdpa, index,
199 desc_addr, driver_addr,
200 device_addr)) {
201 err = -EINVAL;
202 goto err_vq;
203 }
204
205 /* reset virtqueue state index */
206 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
207 struct vdpa_vq_state_packed *s = &state.packed;
208
209 s->last_avail_counter = 1;
210 s->last_avail_idx = 0;
211 s->last_used_counter = 1;
212 s->last_used_idx = 0;
213 }
214 err = ops->set_vq_state(vdpa, index, &state);
215 if (err)
216 goto err_vq;
217
218 ops->set_vq_ready(vdpa, index, 1);
219
220 vq->priv = info;
221 info->vq = vq;
222
223 spin_lock_irqsave(&vd_dev->lock, flags);
224 list_add(&info->node, &vd_dev->virtqueues);
225 spin_unlock_irqrestore(&vd_dev->lock, flags);
226
227 return vq;
228
229err_vq:
230 vring_del_virtqueue(vq);
231error_new_virtqueue:
232 ops->set_vq_ready(vdpa, index, 0);
233 /* VDPA driver should make sure vq is stopeed here */
234 WARN_ON(ops->get_vq_ready(vdpa, index));
235 kfree(info);
236 return ERR_PTR(err);
237}
238
239static void virtio_vdpa_del_vq(struct virtqueue *vq)
240{
241 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
242 struct vdpa_device *vdpa = vd_dev->vdpa;
243 const struct vdpa_config_ops *ops = vdpa->config;
244 struct virtio_vdpa_vq_info *info = vq->priv;
245 unsigned int index = vq->index;
246 unsigned long flags;
247
248 spin_lock_irqsave(&vd_dev->lock, flags);
249 list_del(&info->node);
250 spin_unlock_irqrestore(&vd_dev->lock, flags);
251
252 /* Select and deactivate the queue (best effort) */
253 ops->set_vq_ready(vdpa, index, 0);
254
255 vring_del_virtqueue(vq);
256
257 kfree(info);
258}
259
260static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
261{
262 struct virtqueue *vq, *n;
263
264 list_for_each_entry_safe(vq, n, &vdev->vqs, list)
265 virtio_vdpa_del_vq(vq);
266}
267
268static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
269 struct virtqueue *vqs[],
270 vq_callback_t *callbacks[],
271 const char * const names[],
272 const bool *ctx,
273 struct irq_affinity *desc)
274{
275 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
276 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
277 const struct vdpa_config_ops *ops = vdpa->config;
278 struct vdpa_callback cb;
279 int i, err, queue_idx = 0;
280
281 for (i = 0; i < nvqs; ++i) {
282 if (!names[i]) {
283 vqs[i] = NULL;
284 continue;
285 }
286
287 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
288 callbacks[i], names[i], ctx ?
289 ctx[i] : false);
290 if (IS_ERR(vqs[i])) {
291 err = PTR_ERR(vqs[i]);
292 goto err_setup_vq;
293 }
294 }
295
296 cb.callback = virtio_vdpa_config_cb;
297 cb.private = vd_dev;
298 ops->set_config_cb(vdpa, &cb);
299
300 return 0;
301
302err_setup_vq:
303 virtio_vdpa_del_vqs(vdev);
304 return err;
305}
306
307static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
308{
309 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
310 const struct vdpa_config_ops *ops = vdpa->config;
311
312 return ops->get_device_features(vdpa);
313}
314
315static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
316{
317 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
318
319 /* Give virtio_ring a chance to accept features. */
320 vring_transport_features(vdev);
321
322 return vdpa_set_features(vdpa, vdev->features);
323}
324
325static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
326{
327 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
328 struct vdpa_device *vdpa = vd_dev->vdpa;
329
330 return dev_name(&vdpa->dev);
331}
332
333static const struct virtio_config_ops virtio_vdpa_config_ops = {
334 .get = virtio_vdpa_get,
335 .set = virtio_vdpa_set,
336 .generation = virtio_vdpa_generation,
337 .get_status = virtio_vdpa_get_status,
338 .set_status = virtio_vdpa_set_status,
339 .reset = virtio_vdpa_reset,
340 .find_vqs = virtio_vdpa_find_vqs,
341 .del_vqs = virtio_vdpa_del_vqs,
342 .get_features = virtio_vdpa_get_features,
343 .finalize_features = virtio_vdpa_finalize_features,
344 .bus_name = virtio_vdpa_bus_name,
345};
346
347static void virtio_vdpa_release_dev(struct device *_d)
348{
349 struct virtio_device *vdev =
350 container_of(_d, struct virtio_device, dev);
351 struct virtio_vdpa_device *vd_dev =
352 container_of(vdev, struct virtio_vdpa_device, vdev);
353
354 kfree(vd_dev);
355}
356
357static int virtio_vdpa_probe(struct vdpa_device *vdpa)
358{
359 const struct vdpa_config_ops *ops = vdpa->config;
360 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
361 int ret = -EINVAL;
362
363 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
364 if (!vd_dev)
365 return -ENOMEM;
366
367 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa);
368 vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
369 vd_dev->vdev.config = &virtio_vdpa_config_ops;
370 vd_dev->vdpa = vdpa;
371 INIT_LIST_HEAD(&vd_dev->virtqueues);
372 spin_lock_init(&vd_dev->lock);
373
374 vd_dev->vdev.id.device = ops->get_device_id(vdpa);
375 if (vd_dev->vdev.id.device == 0)
376 goto err;
377
378 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
379 ret = register_virtio_device(&vd_dev->vdev);
380 reg_dev = vd_dev;
381 if (ret)
382 goto err;
383
384 vdpa_set_drvdata(vdpa, vd_dev);
385
386 return 0;
387
388err:
389 if (reg_dev)
390 put_device(&vd_dev->vdev.dev);
391 else
392 kfree(vd_dev);
393 return ret;
394}
395
396static void virtio_vdpa_remove(struct vdpa_device *vdpa)
397{
398 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);
399
400 unregister_virtio_device(&vd_dev->vdev);
401}
402
403static struct vdpa_driver virtio_vdpa_driver = {
404 .driver = {
405 .name = "virtio_vdpa",
406 },
407 .probe = virtio_vdpa_probe,
408 .remove = virtio_vdpa_remove,
409};
410
411module_vdpa_driver(virtio_vdpa_driver);
412
413MODULE_VERSION(MOD_VERSION);
414MODULE_LICENSE(MOD_LICENSE);
415MODULE_AUTHOR(MOD_AUTHOR);
416MODULE_DESCRIPTION(MOD_DESC);