Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VIRTIO based driver for vDPA device
4 *
5 * Copyright (c) 2020, Red Hat. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/device.h>
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/uuid.h>
16#include <linux/virtio.h>
17#include <linux/vdpa.h>
18#include <linux/virtio_config.h>
19#include <linux/virtio_ring.h>
20
21#define MOD_VERSION "0.1"
22#define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
23#define MOD_DESC "vDPA bus driver for virtio devices"
24#define MOD_LICENSE "GPL v2"
25
26struct virtio_vdpa_device {
27 struct virtio_device vdev;
28 struct vdpa_device *vdpa;
29 u64 features;
30
31 /* The lock to protect virtqueue list */
32 spinlock_t lock;
33 /* List of virtio_vdpa_vq_info */
34 struct list_head virtqueues;
35};
36
37struct virtio_vdpa_vq_info {
38 /* the actual virtqueue */
39 struct virtqueue *vq;
40
41 /* the list node for the virtqueues list */
42 struct list_head node;
43};
44
45static inline struct virtio_vdpa_device *
46to_virtio_vdpa_device(struct virtio_device *dev)
47{
48 return container_of(dev, struct virtio_vdpa_device, vdev);
49}
50
51static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
52{
53 return to_virtio_vdpa_device(vdev)->vdpa;
54}
55
56static void virtio_vdpa_get(struct virtio_device *vdev, unsigned offset,
57 void *buf, unsigned len)
58{
59 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
60
61 vdpa_get_config(vdpa, offset, buf, len);
62}
63
64static void virtio_vdpa_set(struct virtio_device *vdev, unsigned offset,
65 const void *buf, unsigned len)
66{
67 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
68 const struct vdpa_config_ops *ops = vdpa->config;
69
70 ops->set_config(vdpa, offset, buf, len);
71}
72
73static u32 virtio_vdpa_generation(struct virtio_device *vdev)
74{
75 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
76 const struct vdpa_config_ops *ops = vdpa->config;
77
78 if (ops->get_generation)
79 return ops->get_generation(vdpa);
80
81 return 0;
82}
83
84static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
85{
86 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
87 const struct vdpa_config_ops *ops = vdpa->config;
88
89 return ops->get_status(vdpa);
90}
91
92static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
93{
94 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
95 const struct vdpa_config_ops *ops = vdpa->config;
96
97 return ops->set_status(vdpa, status);
98}
99
100static void virtio_vdpa_reset(struct virtio_device *vdev)
101{
102 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
103
104 vdpa_reset(vdpa);
105}
106
107static bool virtio_vdpa_notify(struct virtqueue *vq)
108{
109 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
110 const struct vdpa_config_ops *ops = vdpa->config;
111
112 ops->kick_vq(vdpa, vq->index);
113
114 return true;
115}
116
117static irqreturn_t virtio_vdpa_config_cb(void *private)
118{
119 struct virtio_vdpa_device *vd_dev = private;
120
121 virtio_config_changed(&vd_dev->vdev);
122
123 return IRQ_HANDLED;
124}
125
126static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
127{
128 struct virtio_vdpa_vq_info *info = private;
129
130 return vring_interrupt(0, info->vq);
131}
132
133static struct virtqueue *
134virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
135 void (*callback)(struct virtqueue *vq),
136 const char *name, bool ctx)
137{
138 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
139 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
140 const struct vdpa_config_ops *ops = vdpa->config;
141 struct virtio_vdpa_vq_info *info;
142 struct vdpa_callback cb;
143 struct virtqueue *vq;
144 u64 desc_addr, driver_addr, device_addr;
145 unsigned long flags;
146 u32 align, num;
147 int err;
148
149 if (!name)
150 return NULL;
151
152 /* Queue shouldn't already be set up. */
153 if (ops->get_vq_ready(vdpa, index))
154 return ERR_PTR(-ENOENT);
155
156 /* Allocate and fill out our active queue description */
157 info = kmalloc(sizeof(*info), GFP_KERNEL);
158 if (!info)
159 return ERR_PTR(-ENOMEM);
160
161 num = ops->get_vq_num_max(vdpa);
162 if (num == 0) {
163 err = -ENOENT;
164 goto error_new_virtqueue;
165 }
166
167 /* Create the vring */
168 align = ops->get_vq_align(vdpa);
169 vq = vring_create_virtqueue(index, num, align, vdev,
170 true, true, ctx,
171 virtio_vdpa_notify, callback, name);
172 if (!vq) {
173 err = -ENOMEM;
174 goto error_new_virtqueue;
175 }
176
177 /* Setup virtqueue callback */
178 cb.callback = virtio_vdpa_virtqueue_cb;
179 cb.private = info;
180 ops->set_vq_cb(vdpa, index, &cb);
181 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
182
183 desc_addr = virtqueue_get_desc_addr(vq);
184 driver_addr = virtqueue_get_avail_addr(vq);
185 device_addr = virtqueue_get_used_addr(vq);
186
187 if (ops->set_vq_address(vdpa, index,
188 desc_addr, driver_addr,
189 device_addr)) {
190 err = -EINVAL;
191 goto err_vq;
192 }
193
194 ops->set_vq_ready(vdpa, index, 1);
195
196 vq->priv = info;
197 info->vq = vq;
198
199 spin_lock_irqsave(&vd_dev->lock, flags);
200 list_add(&info->node, &vd_dev->virtqueues);
201 spin_unlock_irqrestore(&vd_dev->lock, flags);
202
203 return vq;
204
205err_vq:
206 vring_del_virtqueue(vq);
207error_new_virtqueue:
208 ops->set_vq_ready(vdpa, index, 0);
209 /* VDPA driver should make sure vq is stopeed here */
210 WARN_ON(ops->get_vq_ready(vdpa, index));
211 kfree(info);
212 return ERR_PTR(err);
213}
214
215static void virtio_vdpa_del_vq(struct virtqueue *vq)
216{
217 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
218 struct vdpa_device *vdpa = vd_dev->vdpa;
219 const struct vdpa_config_ops *ops = vdpa->config;
220 struct virtio_vdpa_vq_info *info = vq->priv;
221 unsigned int index = vq->index;
222 unsigned long flags;
223
224 spin_lock_irqsave(&vd_dev->lock, flags);
225 list_del(&info->node);
226 spin_unlock_irqrestore(&vd_dev->lock, flags);
227
228 /* Select and deactivate the queue */
229 ops->set_vq_ready(vdpa, index, 0);
230 WARN_ON(ops->get_vq_ready(vdpa, index));
231
232 vring_del_virtqueue(vq);
233
234 kfree(info);
235}
236
237static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
238{
239 struct virtqueue *vq, *n;
240
241 list_for_each_entry_safe(vq, n, &vdev->vqs, list)
242 virtio_vdpa_del_vq(vq);
243}
244
245static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned nvqs,
246 struct virtqueue *vqs[],
247 vq_callback_t *callbacks[],
248 const char * const names[],
249 const bool *ctx,
250 struct irq_affinity *desc)
251{
252 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
253 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
254 const struct vdpa_config_ops *ops = vdpa->config;
255 struct vdpa_callback cb;
256 int i, err, queue_idx = 0;
257
258 for (i = 0; i < nvqs; ++i) {
259 if (!names[i]) {
260 vqs[i] = NULL;
261 continue;
262 }
263
264 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
265 callbacks[i], names[i], ctx ?
266 ctx[i] : false);
267 if (IS_ERR(vqs[i])) {
268 err = PTR_ERR(vqs[i]);
269 goto err_setup_vq;
270 }
271 }
272
273 cb.callback = virtio_vdpa_config_cb;
274 cb.private = vd_dev;
275 ops->set_config_cb(vdpa, &cb);
276
277 return 0;
278
279err_setup_vq:
280 virtio_vdpa_del_vqs(vdev);
281 return err;
282}
283
284static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
285{
286 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
287 const struct vdpa_config_ops *ops = vdpa->config;
288
289 return ops->get_features(vdpa);
290}
291
292static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
293{
294 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
295
296 /* Give virtio_ring a chance to accept features. */
297 vring_transport_features(vdev);
298
299 return vdpa_set_features(vdpa, vdev->features);
300}
301
302static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
303{
304 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
305 struct vdpa_device *vdpa = vd_dev->vdpa;
306
307 return dev_name(&vdpa->dev);
308}
309
310static const struct virtio_config_ops virtio_vdpa_config_ops = {
311 .get = virtio_vdpa_get,
312 .set = virtio_vdpa_set,
313 .generation = virtio_vdpa_generation,
314 .get_status = virtio_vdpa_get_status,
315 .set_status = virtio_vdpa_set_status,
316 .reset = virtio_vdpa_reset,
317 .find_vqs = virtio_vdpa_find_vqs,
318 .del_vqs = virtio_vdpa_del_vqs,
319 .get_features = virtio_vdpa_get_features,
320 .finalize_features = virtio_vdpa_finalize_features,
321 .bus_name = virtio_vdpa_bus_name,
322};
323
324static void virtio_vdpa_release_dev(struct device *_d)
325{
326 struct virtio_device *vdev =
327 container_of(_d, struct virtio_device, dev);
328 struct virtio_vdpa_device *vd_dev =
329 container_of(vdev, struct virtio_vdpa_device, vdev);
330
331 kfree(vd_dev);
332}
333
334static int virtio_vdpa_probe(struct vdpa_device *vdpa)
335{
336 const struct vdpa_config_ops *ops = vdpa->config;
337 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
338 int ret = -EINVAL;
339
340 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
341 if (!vd_dev)
342 return -ENOMEM;
343
344 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa);
345 vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
346 vd_dev->vdev.config = &virtio_vdpa_config_ops;
347 vd_dev->vdpa = vdpa;
348 INIT_LIST_HEAD(&vd_dev->virtqueues);
349 spin_lock_init(&vd_dev->lock);
350
351 vd_dev->vdev.id.device = ops->get_device_id(vdpa);
352 if (vd_dev->vdev.id.device == 0)
353 goto err;
354
355 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
356 ret = register_virtio_device(&vd_dev->vdev);
357 reg_dev = vd_dev;
358 if (ret)
359 goto err;
360
361 vdpa_set_drvdata(vdpa, vd_dev);
362
363 return 0;
364
365err:
366 if (reg_dev)
367 put_device(&vd_dev->vdev.dev);
368 else
369 kfree(vd_dev);
370 return ret;
371}
372
373static void virtio_vdpa_remove(struct vdpa_device *vdpa)
374{
375 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);
376
377 unregister_virtio_device(&vd_dev->vdev);
378}
379
380static struct vdpa_driver virtio_vdpa_driver = {
381 .driver = {
382 .name = "virtio_vdpa",
383 },
384 .probe = virtio_vdpa_probe,
385 .remove = virtio_vdpa_remove,
386};
387
388module_vdpa_driver(virtio_vdpa_driver);
389
390MODULE_VERSION(MOD_VERSION);
391MODULE_LICENSE(MOD_LICENSE);
392MODULE_AUTHOR(MOD_AUTHOR);
393MODULE_DESCRIPTION(MOD_DESC);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VIRTIO based driver for vDPA device
4 *
5 * Copyright (c) 2020, Red Hat. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/device.h>
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/uuid.h>
16#include <linux/group_cpus.h>
17#include <linux/virtio.h>
18#include <linux/vdpa.h>
19#include <linux/virtio_config.h>
20#include <linux/virtio_ring.h>
21
22#define MOD_VERSION "0.1"
23#define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
24#define MOD_DESC "vDPA bus driver for virtio devices"
25#define MOD_LICENSE "GPL v2"
26
27struct virtio_vdpa_device {
28 struct virtio_device vdev;
29 struct vdpa_device *vdpa;
30 u64 features;
31
32 /* The lock to protect virtqueue list */
33 spinlock_t lock;
34 /* List of virtio_vdpa_vq_info */
35 struct list_head virtqueues;
36};
37
38struct virtio_vdpa_vq_info {
39 /* the actual virtqueue */
40 struct virtqueue *vq;
41
42 /* the list node for the virtqueues list */
43 struct list_head node;
44};
45
46static inline struct virtio_vdpa_device *
47to_virtio_vdpa_device(struct virtio_device *dev)
48{
49 return container_of(dev, struct virtio_vdpa_device, vdev);
50}
51
52static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
53{
54 return to_virtio_vdpa_device(vdev)->vdpa;
55}
56
57static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset,
58 void *buf, unsigned int len)
59{
60 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
61
62 vdpa_get_config(vdpa, offset, buf, len);
63}
64
65static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset,
66 const void *buf, unsigned int len)
67{
68 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
69
70 vdpa_set_config(vdpa, offset, buf, len);
71}
72
73static u32 virtio_vdpa_generation(struct virtio_device *vdev)
74{
75 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
76 const struct vdpa_config_ops *ops = vdpa->config;
77
78 if (ops->get_generation)
79 return ops->get_generation(vdpa);
80
81 return 0;
82}
83
84static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
85{
86 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
87 const struct vdpa_config_ops *ops = vdpa->config;
88
89 return ops->get_status(vdpa);
90}
91
92static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
93{
94 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
95
96 return vdpa_set_status(vdpa, status);
97}
98
99static void virtio_vdpa_reset(struct virtio_device *vdev)
100{
101 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
102
103 vdpa_reset(vdpa, 0);
104}
105
106static bool virtio_vdpa_notify(struct virtqueue *vq)
107{
108 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
109 const struct vdpa_config_ops *ops = vdpa->config;
110
111 ops->kick_vq(vdpa, vq->index);
112
113 return true;
114}
115
116static bool virtio_vdpa_notify_with_data(struct virtqueue *vq)
117{
118 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
119 const struct vdpa_config_ops *ops = vdpa->config;
120 u32 data = vring_notification_data(vq);
121
122 ops->kick_vq_with_data(vdpa, data);
123
124 return true;
125}
126
127static irqreturn_t virtio_vdpa_config_cb(void *private)
128{
129 struct virtio_vdpa_device *vd_dev = private;
130
131 virtio_config_changed(&vd_dev->vdev);
132
133 return IRQ_HANDLED;
134}
135
136static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
137{
138 struct virtio_vdpa_vq_info *info = private;
139
140 return vring_interrupt(0, info->vq);
141}
142
143static struct virtqueue *
144virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
145 void (*callback)(struct virtqueue *vq),
146 const char *name, bool ctx)
147{
148 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
149 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
150 struct device *dma_dev;
151 const struct vdpa_config_ops *ops = vdpa->config;
152 struct virtio_vdpa_vq_info *info;
153 bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify;
154 struct vdpa_callback cb;
155 struct virtqueue *vq;
156 u64 desc_addr, driver_addr, device_addr;
157 /* Assume split virtqueue, switch to packed if necessary */
158 struct vdpa_vq_state state = {0};
159 unsigned long flags;
160 u32 align, max_num, min_num = 1;
161 bool may_reduce_num = true;
162 int err;
163
164 if (!name)
165 return NULL;
166
167 if (index >= vdpa->nvqs)
168 return ERR_PTR(-ENOENT);
169
170 /* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */
171 if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) {
172 if (ops->kick_vq_with_data)
173 notify = virtio_vdpa_notify_with_data;
174 else
175 __virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA);
176 }
177
178 /* Queue shouldn't already be set up. */
179 if (ops->get_vq_ready(vdpa, index))
180 return ERR_PTR(-ENOENT);
181
182 /* Allocate and fill out our active queue description */
183 info = kmalloc(sizeof(*info), GFP_KERNEL);
184 if (!info)
185 return ERR_PTR(-ENOMEM);
186 if (ops->get_vq_size)
187 max_num = ops->get_vq_size(vdpa, index);
188 else
189 max_num = ops->get_vq_num_max(vdpa);
190
191 if (max_num == 0) {
192 err = -ENOENT;
193 goto error_new_virtqueue;
194 }
195
196 if (ops->get_vq_num_min)
197 min_num = ops->get_vq_num_min(vdpa);
198
199 may_reduce_num = (max_num == min_num) ? false : true;
200
201 /* Create the vring */
202 align = ops->get_vq_align(vdpa);
203
204 if (ops->get_vq_dma_dev)
205 dma_dev = ops->get_vq_dma_dev(vdpa, index);
206 else
207 dma_dev = vdpa_get_dma_dev(vdpa);
208 vq = vring_create_virtqueue_dma(index, max_num, align, vdev,
209 true, may_reduce_num, ctx,
210 notify, callback, name, dma_dev);
211 if (!vq) {
212 err = -ENOMEM;
213 goto error_new_virtqueue;
214 }
215
216 vq->num_max = max_num;
217
218 /* Setup virtqueue callback */
219 cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
220 cb.private = info;
221 cb.trigger = NULL;
222 ops->set_vq_cb(vdpa, index, &cb);
223 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
224
225 desc_addr = virtqueue_get_desc_addr(vq);
226 driver_addr = virtqueue_get_avail_addr(vq);
227 device_addr = virtqueue_get_used_addr(vq);
228
229 if (ops->set_vq_address(vdpa, index,
230 desc_addr, driver_addr,
231 device_addr)) {
232 err = -EINVAL;
233 goto err_vq;
234 }
235
236 /* reset virtqueue state index */
237 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
238 struct vdpa_vq_state_packed *s = &state.packed;
239
240 s->last_avail_counter = 1;
241 s->last_avail_idx = 0;
242 s->last_used_counter = 1;
243 s->last_used_idx = 0;
244 }
245 err = ops->set_vq_state(vdpa, index, &state);
246 if (err)
247 goto err_vq;
248
249 ops->set_vq_ready(vdpa, index, 1);
250
251 vq->priv = info;
252 info->vq = vq;
253
254 spin_lock_irqsave(&vd_dev->lock, flags);
255 list_add(&info->node, &vd_dev->virtqueues);
256 spin_unlock_irqrestore(&vd_dev->lock, flags);
257
258 return vq;
259
260err_vq:
261 vring_del_virtqueue(vq);
262error_new_virtqueue:
263 ops->set_vq_ready(vdpa, index, 0);
264 /* VDPA driver should make sure vq is stopeed here */
265 WARN_ON(ops->get_vq_ready(vdpa, index));
266 kfree(info);
267 return ERR_PTR(err);
268}
269
270static void virtio_vdpa_del_vq(struct virtqueue *vq)
271{
272 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
273 struct vdpa_device *vdpa = vd_dev->vdpa;
274 const struct vdpa_config_ops *ops = vdpa->config;
275 struct virtio_vdpa_vq_info *info = vq->priv;
276 unsigned int index = vq->index;
277 unsigned long flags;
278
279 spin_lock_irqsave(&vd_dev->lock, flags);
280 list_del(&info->node);
281 spin_unlock_irqrestore(&vd_dev->lock, flags);
282
283 /* Select and deactivate the queue (best effort) */
284 ops->set_vq_ready(vdpa, index, 0);
285
286 vring_del_virtqueue(vq);
287
288 kfree(info);
289}
290
291static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
292{
293 struct virtqueue *vq, *n;
294
295 list_for_each_entry_safe(vq, n, &vdev->vqs, list)
296 virtio_vdpa_del_vq(vq);
297}
298
299static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)
300{
301 affd->nr_sets = 1;
302 affd->set_size[0] = affvecs;
303}
304
305static struct cpumask *
306create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
307{
308 unsigned int affvecs = 0, curvec, usedvecs, i;
309 struct cpumask *masks = NULL;
310
311 if (nvecs > affd->pre_vectors + affd->post_vectors)
312 affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
313
314 if (!affd->calc_sets)
315 affd->calc_sets = default_calc_sets;
316
317 affd->calc_sets(affd, affvecs);
318
319 if (!affvecs)
320 return NULL;
321
322 masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
323 if (!masks)
324 return NULL;
325
326 /* Fill out vectors at the beginning that don't need affinity */
327 for (curvec = 0; curvec < affd->pre_vectors; curvec++)
328 cpumask_setall(&masks[curvec]);
329
330 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {
331 unsigned int this_vecs = affd->set_size[i];
332 int j;
333 struct cpumask *result = group_cpus_evenly(this_vecs);
334
335 if (!result) {
336 kfree(masks);
337 return NULL;
338 }
339
340 for (j = 0; j < this_vecs; j++)
341 cpumask_copy(&masks[curvec + j], &result[j]);
342 kfree(result);
343
344 curvec += this_vecs;
345 usedvecs += this_vecs;
346 }
347
348 /* Fill out vectors at the end that don't need affinity */
349 if (usedvecs >= affvecs)
350 curvec = affd->pre_vectors + affvecs;
351 else
352 curvec = affd->pre_vectors + usedvecs;
353 for (; curvec < nvecs; curvec++)
354 cpumask_setall(&masks[curvec]);
355
356 return masks;
357}
358
359static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
360 struct virtqueue *vqs[],
361 struct virtqueue_info vqs_info[],
362 struct irq_affinity *desc)
363{
364 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
365 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
366 const struct vdpa_config_ops *ops = vdpa->config;
367 struct cpumask *masks;
368 struct vdpa_callback cb;
369 bool has_affinity = desc && ops->set_vq_affinity;
370 int i, err, queue_idx = 0;
371
372 if (has_affinity) {
373 masks = create_affinity_masks(nvqs, desc);
374 if (!masks)
375 return -ENOMEM;
376 }
377
378 for (i = 0; i < nvqs; ++i) {
379 struct virtqueue_info *vqi = &vqs_info[i];
380
381 if (!vqi->name) {
382 vqs[i] = NULL;
383 continue;
384 }
385
386 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, vqi->callback,
387 vqi->name, vqi->ctx);
388 if (IS_ERR(vqs[i])) {
389 err = PTR_ERR(vqs[i]);
390 goto err_setup_vq;
391 }
392
393 if (has_affinity)
394 ops->set_vq_affinity(vdpa, i, &masks[i]);
395 }
396
397 cb.callback = virtio_vdpa_config_cb;
398 cb.private = vd_dev;
399 ops->set_config_cb(vdpa, &cb);
400 if (has_affinity)
401 kfree(masks);
402
403 return 0;
404
405err_setup_vq:
406 virtio_vdpa_del_vqs(vdev);
407 if (has_affinity)
408 kfree(masks);
409 return err;
410}
411
412static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
413{
414 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
415 const struct vdpa_config_ops *ops = vdpa->config;
416
417 return ops->get_device_features(vdpa);
418}
419
420static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
421{
422 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
423
424 /* Give virtio_ring a chance to accept features. */
425 vring_transport_features(vdev);
426
427 return vdpa_set_features(vdpa, vdev->features);
428}
429
430static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
431{
432 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
433 struct vdpa_device *vdpa = vd_dev->vdpa;
434
435 return dev_name(&vdpa->dev);
436}
437
438static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq,
439 const struct cpumask *cpu_mask)
440{
441 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
442 struct vdpa_device *vdpa = vd_dev->vdpa;
443 const struct vdpa_config_ops *ops = vdpa->config;
444 unsigned int index = vq->index;
445
446 if (ops->set_vq_affinity)
447 return ops->set_vq_affinity(vdpa, index, cpu_mask);
448
449 return 0;
450}
451
452static const struct cpumask *
453virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index)
454{
455 struct vdpa_device *vdpa = vd_get_vdpa(vdev);
456 const struct vdpa_config_ops *ops = vdpa->config;
457
458 if (ops->get_vq_affinity)
459 return ops->get_vq_affinity(vdpa, index);
460
461 return NULL;
462}
463
464static const struct virtio_config_ops virtio_vdpa_config_ops = {
465 .get = virtio_vdpa_get,
466 .set = virtio_vdpa_set,
467 .generation = virtio_vdpa_generation,
468 .get_status = virtio_vdpa_get_status,
469 .set_status = virtio_vdpa_set_status,
470 .reset = virtio_vdpa_reset,
471 .find_vqs = virtio_vdpa_find_vqs,
472 .del_vqs = virtio_vdpa_del_vqs,
473 .get_features = virtio_vdpa_get_features,
474 .finalize_features = virtio_vdpa_finalize_features,
475 .bus_name = virtio_vdpa_bus_name,
476 .set_vq_affinity = virtio_vdpa_set_vq_affinity,
477 .get_vq_affinity = virtio_vdpa_get_vq_affinity,
478};
479
480static void virtio_vdpa_release_dev(struct device *_d)
481{
482 struct virtio_device *vdev =
483 container_of(_d, struct virtio_device, dev);
484 struct virtio_vdpa_device *vd_dev =
485 container_of(vdev, struct virtio_vdpa_device, vdev);
486
487 kfree(vd_dev);
488}
489
490static int virtio_vdpa_probe(struct vdpa_device *vdpa)
491{
492 const struct vdpa_config_ops *ops = vdpa->config;
493 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
494 int ret = -EINVAL;
495
496 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
497 if (!vd_dev)
498 return -ENOMEM;
499
500 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa);
501 vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
502 vd_dev->vdev.config = &virtio_vdpa_config_ops;
503 vd_dev->vdpa = vdpa;
504 INIT_LIST_HEAD(&vd_dev->virtqueues);
505 spin_lock_init(&vd_dev->lock);
506
507 vd_dev->vdev.id.device = ops->get_device_id(vdpa);
508 if (vd_dev->vdev.id.device == 0)
509 goto err;
510
511 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
512 ret = register_virtio_device(&vd_dev->vdev);
513 reg_dev = vd_dev;
514 if (ret)
515 goto err;
516
517 vdpa_set_drvdata(vdpa, vd_dev);
518
519 return 0;
520
521err:
522 if (reg_dev)
523 put_device(&vd_dev->vdev.dev);
524 else
525 kfree(vd_dev);
526 return ret;
527}
528
529static void virtio_vdpa_remove(struct vdpa_device *vdpa)
530{
531 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);
532
533 unregister_virtio_device(&vd_dev->vdev);
534}
535
536static struct vdpa_driver virtio_vdpa_driver = {
537 .driver = {
538 .name = "virtio_vdpa",
539 },
540 .probe = virtio_vdpa_probe,
541 .remove = virtio_vdpa_remove,
542};
543
544module_vdpa_driver(virtio_vdpa_driver);
545
546MODULE_VERSION(MOD_VERSION);
547MODULE_LICENSE(MOD_LICENSE);
548MODULE_AUTHOR(MOD_AUTHOR);
549MODULE_DESCRIPTION(MOD_DESC);