Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * vDPA bus.
4 *
5 * Copyright (c) 2020, Red Hat. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/module.h>
11#include <linux/idr.h>
12#include <linux/slab.h>
13#include <linux/vdpa.h>
14#include <uapi/linux/vdpa.h>
15#include <net/genetlink.h>
16#include <linux/mod_devicetable.h>
17#include <linux/virtio_ids.h>
18
19static LIST_HEAD(mdev_head);
20/* A global mutex that protects vdpa management device and device level operations. */
21static DECLARE_RWSEM(vdpa_dev_lock);
22static DEFINE_IDA(vdpa_index_ida);
23
24void vdpa_set_status(struct vdpa_device *vdev, u8 status)
25{
26 down_write(&vdev->cf_lock);
27 vdev->config->set_status(vdev, status);
28 up_write(&vdev->cf_lock);
29}
30EXPORT_SYMBOL(vdpa_set_status);
31
32static struct genl_family vdpa_nl_family;
33
34static int vdpa_dev_probe(struct device *d)
35{
36 struct vdpa_device *vdev = dev_to_vdpa(d);
37 struct vdpa_driver *drv = drv_to_vdpa(vdev->dev.driver);
38 const struct vdpa_config_ops *ops = vdev->config;
39 u32 max_num, min_num = 1;
40 int ret = 0;
41
42 d->dma_mask = &d->coherent_dma_mask;
43 ret = dma_set_mask_and_coherent(d, DMA_BIT_MASK(64));
44 if (ret)
45 return ret;
46
47 max_num = ops->get_vq_num_max(vdev);
48 if (ops->get_vq_num_min)
49 min_num = ops->get_vq_num_min(vdev);
50 if (max_num < min_num)
51 return -EINVAL;
52
53 if (drv && drv->probe)
54 ret = drv->probe(vdev);
55
56 return ret;
57}
58
59static void vdpa_dev_remove(struct device *d)
60{
61 struct vdpa_device *vdev = dev_to_vdpa(d);
62 struct vdpa_driver *drv = drv_to_vdpa(vdev->dev.driver);
63
64 if (drv && drv->remove)
65 drv->remove(vdev);
66}
67
68static int vdpa_dev_match(struct device *dev, struct device_driver *drv)
69{
70 struct vdpa_device *vdev = dev_to_vdpa(dev);
71
72 /* Check override first, and if set, only use the named driver */
73 if (vdev->driver_override)
74 return strcmp(vdev->driver_override, drv->name) == 0;
75
76 /* Currently devices must be supported by all vDPA bus drivers */
77 return 1;
78}
79
80static ssize_t driver_override_store(struct device *dev,
81 struct device_attribute *attr,
82 const char *buf, size_t count)
83{
84 struct vdpa_device *vdev = dev_to_vdpa(dev);
85 int ret;
86
87 ret = driver_set_override(dev, &vdev->driver_override, buf, count);
88 if (ret)
89 return ret;
90
91 return count;
92}
93
94static ssize_t driver_override_show(struct device *dev,
95 struct device_attribute *attr, char *buf)
96{
97 struct vdpa_device *vdev = dev_to_vdpa(dev);
98 ssize_t len;
99
100 device_lock(dev);
101 len = snprintf(buf, PAGE_SIZE, "%s\n", vdev->driver_override);
102 device_unlock(dev);
103
104 return len;
105}
106static DEVICE_ATTR_RW(driver_override);
107
108static struct attribute *vdpa_dev_attrs[] = {
109 &dev_attr_driver_override.attr,
110 NULL,
111};
112
113static const struct attribute_group vdpa_dev_group = {
114 .attrs = vdpa_dev_attrs,
115};
116__ATTRIBUTE_GROUPS(vdpa_dev);
117
118static struct bus_type vdpa_bus = {
119 .name = "vdpa",
120 .dev_groups = vdpa_dev_groups,
121 .match = vdpa_dev_match,
122 .probe = vdpa_dev_probe,
123 .remove = vdpa_dev_remove,
124};
125
126static void vdpa_release_dev(struct device *d)
127{
128 struct vdpa_device *vdev = dev_to_vdpa(d);
129 const struct vdpa_config_ops *ops = vdev->config;
130
131 if (ops->free)
132 ops->free(vdev);
133
134 ida_free(&vdpa_index_ida, vdev->index);
135 kfree(vdev->driver_override);
136 kfree(vdev);
137}
138
139/**
140 * __vdpa_alloc_device - allocate and initilaize a vDPA device
141 * This allows driver to some prepartion after device is
142 * initialized but before registered.
143 * @parent: the parent device
144 * @config: the bus operations that is supported by this device
145 * @ngroups: number of groups supported by this device
146 * @nas: number of address spaces supported by this device
147 * @size: size of the parent structure that contains private data
148 * @name: name of the vdpa device; optional.
149 * @use_va: indicate whether virtual address must be used by this device
150 *
151 * Driver should use vdpa_alloc_device() wrapper macro instead of
152 * using this directly.
153 *
154 * Return: Returns an error when parent/config/dma_dev is not set or fail to get
155 * ida.
156 */
157struct vdpa_device *__vdpa_alloc_device(struct device *parent,
158 const struct vdpa_config_ops *config,
159 unsigned int ngroups, unsigned int nas,
160 size_t size, const char *name,
161 bool use_va)
162{
163 struct vdpa_device *vdev;
164 int err = -EINVAL;
165
166 if (!config)
167 goto err;
168
169 if (!!config->dma_map != !!config->dma_unmap)
170 goto err;
171
172 /* It should only work for the device that use on-chip IOMMU */
173 if (use_va && !(config->dma_map || config->set_map))
174 goto err;
175
176 err = -ENOMEM;
177 vdev = kzalloc(size, GFP_KERNEL);
178 if (!vdev)
179 goto err;
180
181 err = ida_alloc(&vdpa_index_ida, GFP_KERNEL);
182 if (err < 0)
183 goto err_ida;
184
185 vdev->dev.bus = &vdpa_bus;
186 vdev->dev.parent = parent;
187 vdev->dev.release = vdpa_release_dev;
188 vdev->index = err;
189 vdev->config = config;
190 vdev->features_valid = false;
191 vdev->use_va = use_va;
192 vdev->ngroups = ngroups;
193 vdev->nas = nas;
194
195 if (name)
196 err = dev_set_name(&vdev->dev, "%s", name);
197 else
198 err = dev_set_name(&vdev->dev, "vdpa%u", vdev->index);
199 if (err)
200 goto err_name;
201
202 init_rwsem(&vdev->cf_lock);
203 device_initialize(&vdev->dev);
204
205 return vdev;
206
207err_name:
208 ida_free(&vdpa_index_ida, vdev->index);
209err_ida:
210 kfree(vdev);
211err:
212 return ERR_PTR(err);
213}
214EXPORT_SYMBOL_GPL(__vdpa_alloc_device);
215
216static int vdpa_name_match(struct device *dev, const void *data)
217{
218 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
219
220 return (strcmp(dev_name(&vdev->dev), data) == 0);
221}
222
223static int __vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
224{
225 struct device *dev;
226
227 vdev->nvqs = nvqs;
228
229 lockdep_assert_held(&vdpa_dev_lock);
230 dev = bus_find_device(&vdpa_bus, NULL, dev_name(&vdev->dev), vdpa_name_match);
231 if (dev) {
232 put_device(dev);
233 return -EEXIST;
234 }
235 return device_add(&vdev->dev);
236}
237
238/**
239 * _vdpa_register_device - register a vDPA device with vdpa lock held
240 * Caller must have a succeed call of vdpa_alloc_device() before.
241 * Caller must invoke this routine in the management device dev_add()
242 * callback after setting up valid mgmtdev for this vdpa device.
243 * @vdev: the vdpa device to be registered to vDPA bus
244 * @nvqs: number of virtqueues supported by this device
245 *
246 * Return: Returns an error when fail to add device to vDPA bus
247 */
248int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
249{
250 if (!vdev->mdev)
251 return -EINVAL;
252
253 return __vdpa_register_device(vdev, nvqs);
254}
255EXPORT_SYMBOL_GPL(_vdpa_register_device);
256
257/**
258 * vdpa_register_device - register a vDPA device
259 * Callers must have a succeed call of vdpa_alloc_device() before.
260 * @vdev: the vdpa device to be registered to vDPA bus
261 * @nvqs: number of virtqueues supported by this device
262 *
263 * Return: Returns an error when fail to add to vDPA bus
264 */
265int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
266{
267 int err;
268
269 down_write(&vdpa_dev_lock);
270 err = __vdpa_register_device(vdev, nvqs);
271 up_write(&vdpa_dev_lock);
272 return err;
273}
274EXPORT_SYMBOL_GPL(vdpa_register_device);
275
276/**
277 * _vdpa_unregister_device - unregister a vDPA device
278 * Caller must invoke this routine as part of management device dev_del()
279 * callback.
280 * @vdev: the vdpa device to be unregisted from vDPA bus
281 */
282void _vdpa_unregister_device(struct vdpa_device *vdev)
283{
284 lockdep_assert_held(&vdpa_dev_lock);
285 WARN_ON(!vdev->mdev);
286 device_unregister(&vdev->dev);
287}
288EXPORT_SYMBOL_GPL(_vdpa_unregister_device);
289
290/**
291 * vdpa_unregister_device - unregister a vDPA device
292 * @vdev: the vdpa device to be unregisted from vDPA bus
293 */
294void vdpa_unregister_device(struct vdpa_device *vdev)
295{
296 down_write(&vdpa_dev_lock);
297 device_unregister(&vdev->dev);
298 up_write(&vdpa_dev_lock);
299}
300EXPORT_SYMBOL_GPL(vdpa_unregister_device);
301
302/**
303 * __vdpa_register_driver - register a vDPA device driver
304 * @drv: the vdpa device driver to be registered
305 * @owner: module owner of the driver
306 *
307 * Return: Returns an err when fail to do the registration
308 */
309int __vdpa_register_driver(struct vdpa_driver *drv, struct module *owner)
310{
311 drv->driver.bus = &vdpa_bus;
312 drv->driver.owner = owner;
313
314 return driver_register(&drv->driver);
315}
316EXPORT_SYMBOL_GPL(__vdpa_register_driver);
317
318/**
319 * vdpa_unregister_driver - unregister a vDPA device driver
320 * @drv: the vdpa device driver to be unregistered
321 */
322void vdpa_unregister_driver(struct vdpa_driver *drv)
323{
324 driver_unregister(&drv->driver);
325}
326EXPORT_SYMBOL_GPL(vdpa_unregister_driver);
327
328/**
329 * vdpa_mgmtdev_register - register a vdpa management device
330 *
331 * @mdev: Pointer to vdpa management device
332 * vdpa_mgmtdev_register() register a vdpa management device which supports
333 * vdpa device management.
334 * Return: Returns 0 on success or failure when required callback ops are not
335 * initialized.
336 */
337int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev)
338{
339 if (!mdev->device || !mdev->ops || !mdev->ops->dev_add || !mdev->ops->dev_del)
340 return -EINVAL;
341
342 INIT_LIST_HEAD(&mdev->list);
343 down_write(&vdpa_dev_lock);
344 list_add_tail(&mdev->list, &mdev_head);
345 up_write(&vdpa_dev_lock);
346 return 0;
347}
348EXPORT_SYMBOL_GPL(vdpa_mgmtdev_register);
349
350static int vdpa_match_remove(struct device *dev, void *data)
351{
352 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
353 struct vdpa_mgmt_dev *mdev = vdev->mdev;
354
355 if (mdev == data)
356 mdev->ops->dev_del(mdev, vdev);
357 return 0;
358}
359
360void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev)
361{
362 down_write(&vdpa_dev_lock);
363
364 list_del(&mdev->list);
365
366 /* Filter out all the entries belong to this management device and delete it. */
367 bus_for_each_dev(&vdpa_bus, NULL, mdev, vdpa_match_remove);
368
369 up_write(&vdpa_dev_lock);
370}
371EXPORT_SYMBOL_GPL(vdpa_mgmtdev_unregister);
372
373static void vdpa_get_config_unlocked(struct vdpa_device *vdev,
374 unsigned int offset,
375 void *buf, unsigned int len)
376{
377 const struct vdpa_config_ops *ops = vdev->config;
378
379 /*
380 * Config accesses aren't supposed to trigger before features are set.
381 * If it does happen we assume a legacy guest.
382 */
383 if (!vdev->features_valid)
384 vdpa_set_features_unlocked(vdev, 0);
385 ops->get_config(vdev, offset, buf, len);
386}
387
388/**
389 * vdpa_get_config - Get one or more device configuration fields.
390 * @vdev: vdpa device to operate on
391 * @offset: starting byte offset of the field
392 * @buf: buffer pointer to read to
393 * @len: length of the configuration fields in bytes
394 */
395void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset,
396 void *buf, unsigned int len)
397{
398 down_read(&vdev->cf_lock);
399 vdpa_get_config_unlocked(vdev, offset, buf, len);
400 up_read(&vdev->cf_lock);
401}
402EXPORT_SYMBOL_GPL(vdpa_get_config);
403
404/**
405 * vdpa_set_config - Set one or more device configuration fields.
406 * @vdev: vdpa device to operate on
407 * @offset: starting byte offset of the field
408 * @buf: buffer pointer to read from
409 * @length: length of the configuration fields in bytes
410 */
411void vdpa_set_config(struct vdpa_device *vdev, unsigned int offset,
412 const void *buf, unsigned int length)
413{
414 down_write(&vdev->cf_lock);
415 vdev->config->set_config(vdev, offset, buf, length);
416 up_write(&vdev->cf_lock);
417}
418EXPORT_SYMBOL_GPL(vdpa_set_config);
419
420static bool mgmtdev_handle_match(const struct vdpa_mgmt_dev *mdev,
421 const char *busname, const char *devname)
422{
423 /* Bus name is optional for simulated management device, so ignore the
424 * device with bus if bus attribute is provided.
425 */
426 if ((busname && !mdev->device->bus) || (!busname && mdev->device->bus))
427 return false;
428
429 if (!busname && strcmp(dev_name(mdev->device), devname) == 0)
430 return true;
431
432 if (busname && (strcmp(mdev->device->bus->name, busname) == 0) &&
433 (strcmp(dev_name(mdev->device), devname) == 0))
434 return true;
435
436 return false;
437}
438
439static struct vdpa_mgmt_dev *vdpa_mgmtdev_get_from_attr(struct nlattr **attrs)
440{
441 struct vdpa_mgmt_dev *mdev;
442 const char *busname = NULL;
443 const char *devname;
444
445 if (!attrs[VDPA_ATTR_MGMTDEV_DEV_NAME])
446 return ERR_PTR(-EINVAL);
447 devname = nla_data(attrs[VDPA_ATTR_MGMTDEV_DEV_NAME]);
448 if (attrs[VDPA_ATTR_MGMTDEV_BUS_NAME])
449 busname = nla_data(attrs[VDPA_ATTR_MGMTDEV_BUS_NAME]);
450
451 list_for_each_entry(mdev, &mdev_head, list) {
452 if (mgmtdev_handle_match(mdev, busname, devname))
453 return mdev;
454 }
455 return ERR_PTR(-ENODEV);
456}
457
458static int vdpa_nl_mgmtdev_handle_fill(struct sk_buff *msg, const struct vdpa_mgmt_dev *mdev)
459{
460 if (mdev->device->bus &&
461 nla_put_string(msg, VDPA_ATTR_MGMTDEV_BUS_NAME, mdev->device->bus->name))
462 return -EMSGSIZE;
463 if (nla_put_string(msg, VDPA_ATTR_MGMTDEV_DEV_NAME, dev_name(mdev->device)))
464 return -EMSGSIZE;
465 return 0;
466}
467
468static u64 vdpa_mgmtdev_get_classes(const struct vdpa_mgmt_dev *mdev,
469 unsigned int *nclasses)
470{
471 u64 supported_classes = 0;
472 unsigned int n = 0;
473
474 for (int i = 0; mdev->id_table[i].device; i++) {
475 if (mdev->id_table[i].device > 63)
476 continue;
477 supported_classes |= BIT_ULL(mdev->id_table[i].device);
478 n++;
479 }
480 if (nclasses)
481 *nclasses = n;
482
483 return supported_classes;
484}
485
486static int vdpa_mgmtdev_fill(const struct vdpa_mgmt_dev *mdev, struct sk_buff *msg,
487 u32 portid, u32 seq, int flags)
488{
489 void *hdr;
490 int err;
491
492 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, VDPA_CMD_MGMTDEV_NEW);
493 if (!hdr)
494 return -EMSGSIZE;
495 err = vdpa_nl_mgmtdev_handle_fill(msg, mdev);
496 if (err)
497 goto msg_err;
498
499 if (nla_put_u64_64bit(msg, VDPA_ATTR_MGMTDEV_SUPPORTED_CLASSES,
500 vdpa_mgmtdev_get_classes(mdev, NULL),
501 VDPA_ATTR_UNSPEC)) {
502 err = -EMSGSIZE;
503 goto msg_err;
504 }
505 if (nla_put_u32(msg, VDPA_ATTR_DEV_MGMTDEV_MAX_VQS,
506 mdev->max_supported_vqs)) {
507 err = -EMSGSIZE;
508 goto msg_err;
509 }
510 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_SUPPORTED_FEATURES,
511 mdev->supported_features, VDPA_ATTR_PAD)) {
512 err = -EMSGSIZE;
513 goto msg_err;
514 }
515
516 genlmsg_end(msg, hdr);
517 return 0;
518
519msg_err:
520 genlmsg_cancel(msg, hdr);
521 return err;
522}
523
524static int vdpa_nl_cmd_mgmtdev_get_doit(struct sk_buff *skb, struct genl_info *info)
525{
526 struct vdpa_mgmt_dev *mdev;
527 struct sk_buff *msg;
528 int err;
529
530 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
531 if (!msg)
532 return -ENOMEM;
533
534 down_read(&vdpa_dev_lock);
535 mdev = vdpa_mgmtdev_get_from_attr(info->attrs);
536 if (IS_ERR(mdev)) {
537 up_read(&vdpa_dev_lock);
538 NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified mgmt device");
539 err = PTR_ERR(mdev);
540 goto out;
541 }
542
543 err = vdpa_mgmtdev_fill(mdev, msg, info->snd_portid, info->snd_seq, 0);
544 up_read(&vdpa_dev_lock);
545 if (err)
546 goto out;
547 err = genlmsg_reply(msg, info);
548 return err;
549
550out:
551 nlmsg_free(msg);
552 return err;
553}
554
555static int
556vdpa_nl_cmd_mgmtdev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
557{
558 struct vdpa_mgmt_dev *mdev;
559 int start = cb->args[0];
560 int idx = 0;
561 int err;
562
563 down_read(&vdpa_dev_lock);
564 list_for_each_entry(mdev, &mdev_head, list) {
565 if (idx < start) {
566 idx++;
567 continue;
568 }
569 err = vdpa_mgmtdev_fill(mdev, msg, NETLINK_CB(cb->skb).portid,
570 cb->nlh->nlmsg_seq, NLM_F_MULTI);
571 if (err)
572 goto out;
573 idx++;
574 }
575out:
576 up_read(&vdpa_dev_lock);
577 cb->args[0] = idx;
578 return msg->len;
579}
580
581#define VDPA_DEV_NET_ATTRS_MASK (BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | \
582 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU) | \
583 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP))
584
585/*
586 * Bitmask for all per-device features: feature bits VIRTIO_TRANSPORT_F_START
587 * through VIRTIO_TRANSPORT_F_END are unset, i.e. 0xfffffc000fffffff for
588 * all 64bit features. If the features are extended beyond 64 bits, or new
589 * "holes" are reserved for other type of features than per-device, this
590 * macro would have to be updated.
591 */
592#define VIRTIO_DEVICE_F_MASK (~0ULL << (VIRTIO_TRANSPORT_F_END + 1) | \
593 ((1ULL << VIRTIO_TRANSPORT_F_START) - 1))
594
595static int vdpa_nl_cmd_dev_add_set_doit(struct sk_buff *skb, struct genl_info *info)
596{
597 struct vdpa_dev_set_config config = {};
598 struct nlattr **nl_attrs = info->attrs;
599 struct vdpa_mgmt_dev *mdev;
600 unsigned int ncls = 0;
601 const u8 *macaddr;
602 const char *name;
603 u64 classes;
604 int err = 0;
605
606 if (!info->attrs[VDPA_ATTR_DEV_NAME])
607 return -EINVAL;
608
609 name = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
610
611 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]) {
612 macaddr = nla_data(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]);
613 memcpy(config.net.mac, macaddr, sizeof(config.net.mac));
614 config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
615 }
616 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU]) {
617 config.net.mtu =
618 nla_get_u16(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU]);
619 config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU);
620 }
621 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]) {
622 config.net.max_vq_pairs =
623 nla_get_u16(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]);
624 if (!config.net.max_vq_pairs) {
625 NL_SET_ERR_MSG_MOD(info->extack,
626 "At least one pair of VQs is required");
627 return -EINVAL;
628 }
629 config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
630 }
631 if (nl_attrs[VDPA_ATTR_DEV_FEATURES]) {
632 u64 missing = 0x0ULL;
633
634 config.device_features =
635 nla_get_u64(nl_attrs[VDPA_ATTR_DEV_FEATURES]);
636 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR] &&
637 !(config.device_features & BIT_ULL(VIRTIO_NET_F_MAC)))
638 missing |= BIT_ULL(VIRTIO_NET_F_MAC);
639 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU] &&
640 !(config.device_features & BIT_ULL(VIRTIO_NET_F_MTU)))
641 missing |= BIT_ULL(VIRTIO_NET_F_MTU);
642 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP] &&
643 config.net.max_vq_pairs > 1 &&
644 !(config.device_features & BIT_ULL(VIRTIO_NET_F_MQ)))
645 missing |= BIT_ULL(VIRTIO_NET_F_MQ);
646 if (missing) {
647 NL_SET_ERR_MSG_FMT_MOD(info->extack,
648 "Missing features 0x%llx for provided attributes",
649 missing);
650 return -EINVAL;
651 }
652 config.mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);
653 }
654
655 /* Skip checking capability if user didn't prefer to configure any
656 * device networking attributes. It is likely that user might have used
657 * a device specific method to configure such attributes or using device
658 * default attributes.
659 */
660 if ((config.mask & VDPA_DEV_NET_ATTRS_MASK) &&
661 !netlink_capable(skb, CAP_NET_ADMIN))
662 return -EPERM;
663
664 down_write(&vdpa_dev_lock);
665 mdev = vdpa_mgmtdev_get_from_attr(info->attrs);
666 if (IS_ERR(mdev)) {
667 NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified management device");
668 err = PTR_ERR(mdev);
669 goto err;
670 }
671
672 if ((config.mask & mdev->config_attr_mask) != config.mask) {
673 NL_SET_ERR_MSG_FMT_MOD(info->extack,
674 "Some provided attributes are not supported: 0x%llx",
675 config.mask & ~mdev->config_attr_mask);
676 err = -EOPNOTSUPP;
677 goto err;
678 }
679
680 classes = vdpa_mgmtdev_get_classes(mdev, &ncls);
681 if (config.mask & VDPA_DEV_NET_ATTRS_MASK &&
682 !(classes & BIT_ULL(VIRTIO_ID_NET))) {
683 NL_SET_ERR_MSG_MOD(info->extack,
684 "Network class attributes provided on unsupported management device");
685 err = -EINVAL;
686 goto err;
687 }
688 if (!(config.mask & VDPA_DEV_NET_ATTRS_MASK) &&
689 config.mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES) &&
690 classes & BIT_ULL(VIRTIO_ID_NET) && ncls > 1 &&
691 config.device_features & VIRTIO_DEVICE_F_MASK) {
692 NL_SET_ERR_MSG_MOD(info->extack,
693 "Management device supports multi-class while device features specified are ambiguous");
694 err = -EINVAL;
695 goto err;
696 }
697
698 err = mdev->ops->dev_add(mdev, name, &config);
699err:
700 up_write(&vdpa_dev_lock);
701 return err;
702}
703
704static int vdpa_nl_cmd_dev_del_set_doit(struct sk_buff *skb, struct genl_info *info)
705{
706 struct vdpa_mgmt_dev *mdev;
707 struct vdpa_device *vdev;
708 struct device *dev;
709 const char *name;
710 int err = 0;
711
712 if (!info->attrs[VDPA_ATTR_DEV_NAME])
713 return -EINVAL;
714 name = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
715
716 down_write(&vdpa_dev_lock);
717 dev = bus_find_device(&vdpa_bus, NULL, name, vdpa_name_match);
718 if (!dev) {
719 NL_SET_ERR_MSG_MOD(info->extack, "device not found");
720 err = -ENODEV;
721 goto dev_err;
722 }
723 vdev = container_of(dev, struct vdpa_device, dev);
724 if (!vdev->mdev) {
725 NL_SET_ERR_MSG_MOD(info->extack, "Only user created device can be deleted by user");
726 err = -EINVAL;
727 goto mdev_err;
728 }
729 mdev = vdev->mdev;
730 mdev->ops->dev_del(mdev, vdev);
731mdev_err:
732 put_device(dev);
733dev_err:
734 up_write(&vdpa_dev_lock);
735 return err;
736}
737
738static int
739vdpa_dev_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq,
740 int flags, struct netlink_ext_ack *extack)
741{
742 u16 max_vq_size;
743 u16 min_vq_size = 1;
744 u32 device_id;
745 u32 vendor_id;
746 void *hdr;
747 int err;
748
749 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, VDPA_CMD_DEV_NEW);
750 if (!hdr)
751 return -EMSGSIZE;
752
753 err = vdpa_nl_mgmtdev_handle_fill(msg, vdev->mdev);
754 if (err)
755 goto msg_err;
756
757 device_id = vdev->config->get_device_id(vdev);
758 vendor_id = vdev->config->get_vendor_id(vdev);
759 max_vq_size = vdev->config->get_vq_num_max(vdev);
760 if (vdev->config->get_vq_num_min)
761 min_vq_size = vdev->config->get_vq_num_min(vdev);
762
763 err = -EMSGSIZE;
764 if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev)))
765 goto msg_err;
766 if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id))
767 goto msg_err;
768 if (nla_put_u32(msg, VDPA_ATTR_DEV_VENDOR_ID, vendor_id))
769 goto msg_err;
770 if (nla_put_u32(msg, VDPA_ATTR_DEV_MAX_VQS, vdev->nvqs))
771 goto msg_err;
772 if (nla_put_u16(msg, VDPA_ATTR_DEV_MAX_VQ_SIZE, max_vq_size))
773 goto msg_err;
774 if (nla_put_u16(msg, VDPA_ATTR_DEV_MIN_VQ_SIZE, min_vq_size))
775 goto msg_err;
776
777 genlmsg_end(msg, hdr);
778 return 0;
779
780msg_err:
781 genlmsg_cancel(msg, hdr);
782 return err;
783}
784
785static int vdpa_nl_cmd_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
786{
787 struct vdpa_device *vdev;
788 struct sk_buff *msg;
789 const char *devname;
790 struct device *dev;
791 int err;
792
793 if (!info->attrs[VDPA_ATTR_DEV_NAME])
794 return -EINVAL;
795 devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
796 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
797 if (!msg)
798 return -ENOMEM;
799
800 down_read(&vdpa_dev_lock);
801 dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match);
802 if (!dev) {
803 NL_SET_ERR_MSG_MOD(info->extack, "device not found");
804 err = -ENODEV;
805 goto err;
806 }
807 vdev = container_of(dev, struct vdpa_device, dev);
808 if (!vdev->mdev) {
809 err = -EINVAL;
810 goto mdev_err;
811 }
812 err = vdpa_dev_fill(vdev, msg, info->snd_portid, info->snd_seq, 0, info->extack);
813 if (err)
814 goto mdev_err;
815
816 err = genlmsg_reply(msg, info);
817 put_device(dev);
818 up_read(&vdpa_dev_lock);
819 return err;
820
821mdev_err:
822 put_device(dev);
823err:
824 up_read(&vdpa_dev_lock);
825 nlmsg_free(msg);
826 return err;
827}
828
829struct vdpa_dev_dump_info {
830 struct sk_buff *msg;
831 struct netlink_callback *cb;
832 int start_idx;
833 int idx;
834};
835
836static int vdpa_dev_dump(struct device *dev, void *data)
837{
838 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
839 struct vdpa_dev_dump_info *info = data;
840 int err;
841
842 if (!vdev->mdev)
843 return 0;
844 if (info->idx < info->start_idx) {
845 info->idx++;
846 return 0;
847 }
848 err = vdpa_dev_fill(vdev, info->msg, NETLINK_CB(info->cb->skb).portid,
849 info->cb->nlh->nlmsg_seq, NLM_F_MULTI, info->cb->extack);
850 if (err)
851 return err;
852
853 info->idx++;
854 return 0;
855}
856
857static int vdpa_nl_cmd_dev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
858{
859 struct vdpa_dev_dump_info info;
860
861 info.msg = msg;
862 info.cb = cb;
863 info.start_idx = cb->args[0];
864 info.idx = 0;
865
866 down_read(&vdpa_dev_lock);
867 bus_for_each_dev(&vdpa_bus, NULL, &info, vdpa_dev_dump);
868 up_read(&vdpa_dev_lock);
869 cb->args[0] = info.idx;
870 return msg->len;
871}
872
873static int vdpa_dev_net_mq_config_fill(struct sk_buff *msg, u64 features,
874 const struct virtio_net_config *config)
875{
876 u16 val_u16;
877
878 if ((features & BIT_ULL(VIRTIO_NET_F_MQ)) == 0 &&
879 (features & BIT_ULL(VIRTIO_NET_F_RSS)) == 0)
880 return 0;
881
882 val_u16 = __virtio16_to_cpu(true, config->max_virtqueue_pairs);
883
884 return nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MAX_VQP, val_u16);
885}
886
887static int vdpa_dev_net_mtu_config_fill(struct sk_buff *msg, u64 features,
888 const struct virtio_net_config *config)
889{
890 u16 val_u16;
891
892 if ((features & BIT_ULL(VIRTIO_NET_F_MTU)) == 0)
893 return 0;
894
895 val_u16 = __virtio16_to_cpu(true, config->mtu);
896
897 return nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16);
898}
899
900static int vdpa_dev_net_mac_config_fill(struct sk_buff *msg, u64 features,
901 const struct virtio_net_config *config)
902{
903 if ((features & BIT_ULL(VIRTIO_NET_F_MAC)) == 0)
904 return 0;
905
906 return nla_put(msg, VDPA_ATTR_DEV_NET_CFG_MACADDR,
907 sizeof(config->mac), config->mac);
908}
909
910static int vdpa_dev_net_status_config_fill(struct sk_buff *msg, u64 features,
911 const struct virtio_net_config *config)
912{
913 u16 val_u16;
914
915 if ((features & BIT_ULL(VIRTIO_NET_F_STATUS)) == 0)
916 return 0;
917
918 val_u16 = __virtio16_to_cpu(true, config->status);
919 return nla_put_u16(msg, VDPA_ATTR_DEV_NET_STATUS, val_u16);
920}
921
922static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *msg)
923{
924 struct virtio_net_config config = {};
925 u64 features_device;
926
927 vdev->config->get_config(vdev, 0, &config, sizeof(config));
928
929 features_device = vdev->config->get_device_features(vdev);
930
931 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_FEATURES, features_device,
932 VDPA_ATTR_PAD))
933 return -EMSGSIZE;
934
935 if (vdpa_dev_net_mtu_config_fill(msg, features_device, &config))
936 return -EMSGSIZE;
937
938 if (vdpa_dev_net_mac_config_fill(msg, features_device, &config))
939 return -EMSGSIZE;
940
941 if (vdpa_dev_net_status_config_fill(msg, features_device, &config))
942 return -EMSGSIZE;
943
944 return vdpa_dev_net_mq_config_fill(msg, features_device, &config);
945}
946
947static int
948vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq,
949 int flags, struct netlink_ext_ack *extack)
950{
951 u64 features_driver;
952 u8 status = 0;
953 u32 device_id;
954 void *hdr;
955 int err;
956
957 down_read(&vdev->cf_lock);
958 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags,
959 VDPA_CMD_DEV_CONFIG_GET);
960 if (!hdr) {
961 err = -EMSGSIZE;
962 goto out;
963 }
964
965 if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev))) {
966 err = -EMSGSIZE;
967 goto msg_err;
968 }
969
970 device_id = vdev->config->get_device_id(vdev);
971 if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id)) {
972 err = -EMSGSIZE;
973 goto msg_err;
974 }
975
976 /* only read driver features after the feature negotiation is done */
977 status = vdev->config->get_status(vdev);
978 if (status & VIRTIO_CONFIG_S_FEATURES_OK) {
979 features_driver = vdev->config->get_driver_features(vdev);
980 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES, features_driver,
981 VDPA_ATTR_PAD)) {
982 err = -EMSGSIZE;
983 goto msg_err;
984 }
985 }
986
987 switch (device_id) {
988 case VIRTIO_ID_NET:
989 err = vdpa_dev_net_config_fill(vdev, msg);
990 break;
991 default:
992 err = -EOPNOTSUPP;
993 break;
994 }
995 if (err)
996 goto msg_err;
997
998 up_read(&vdev->cf_lock);
999 genlmsg_end(msg, hdr);
1000 return 0;
1001
1002msg_err:
1003 genlmsg_cancel(msg, hdr);
1004out:
1005 up_read(&vdev->cf_lock);
1006 return err;
1007}
1008
1009static int vdpa_fill_stats_rec(struct vdpa_device *vdev, struct sk_buff *msg,
1010 struct genl_info *info, u32 index)
1011{
1012 struct virtio_net_config config = {};
1013 u64 features;
1014 u8 status;
1015 int err;
1016
1017 status = vdev->config->get_status(vdev);
1018 if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
1019 NL_SET_ERR_MSG_MOD(info->extack, "feature negotiation not complete");
1020 return -EAGAIN;
1021 }
1022 vdpa_get_config_unlocked(vdev, 0, &config, sizeof(config));
1023
1024 features = vdev->config->get_driver_features(vdev);
1025 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES,
1026 features, VDPA_ATTR_PAD))
1027 return -EMSGSIZE;
1028
1029 err = vdpa_dev_net_mq_config_fill(msg, features, &config);
1030 if (err)
1031 return err;
1032
1033 if (nla_put_u32(msg, VDPA_ATTR_DEV_QUEUE_INDEX, index))
1034 return -EMSGSIZE;
1035
1036 err = vdev->config->get_vendor_vq_stats(vdev, index, msg, info->extack);
1037 if (err)
1038 return err;
1039
1040 return 0;
1041}
1042
1043static int vendor_stats_fill(struct vdpa_device *vdev, struct sk_buff *msg,
1044 struct genl_info *info, u32 index)
1045{
1046 int err;
1047
1048 down_read(&vdev->cf_lock);
1049 if (!vdev->config->get_vendor_vq_stats) {
1050 err = -EOPNOTSUPP;
1051 goto out;
1052 }
1053
1054 err = vdpa_fill_stats_rec(vdev, msg, info, index);
1055out:
1056 up_read(&vdev->cf_lock);
1057 return err;
1058}
1059
1060static int vdpa_dev_vendor_stats_fill(struct vdpa_device *vdev,
1061 struct sk_buff *msg,
1062 struct genl_info *info, u32 index)
1063{
1064 u32 device_id;
1065 void *hdr;
1066 int err;
1067 u32 portid = info->snd_portid;
1068 u32 seq = info->snd_seq;
1069 u32 flags = 0;
1070
1071 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags,
1072 VDPA_CMD_DEV_VSTATS_GET);
1073 if (!hdr)
1074 return -EMSGSIZE;
1075
1076 if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev))) {
1077 err = -EMSGSIZE;
1078 goto undo_msg;
1079 }
1080
1081 device_id = vdev->config->get_device_id(vdev);
1082 if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id)) {
1083 err = -EMSGSIZE;
1084 goto undo_msg;
1085 }
1086
1087 switch (device_id) {
1088 case VIRTIO_ID_NET:
1089 if (index > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) {
1090 NL_SET_ERR_MSG_MOD(info->extack, "queue index exceeds max value");
1091 err = -ERANGE;
1092 break;
1093 }
1094
1095 err = vendor_stats_fill(vdev, msg, info, index);
1096 break;
1097 default:
1098 err = -EOPNOTSUPP;
1099 break;
1100 }
1101 genlmsg_end(msg, hdr);
1102
1103 return err;
1104
1105undo_msg:
1106 genlmsg_cancel(msg, hdr);
1107 return err;
1108}
1109
1110static int vdpa_nl_cmd_dev_config_get_doit(struct sk_buff *skb, struct genl_info *info)
1111{
1112 struct vdpa_device *vdev;
1113 struct sk_buff *msg;
1114 const char *devname;
1115 struct device *dev;
1116 int err;
1117
1118 if (!info->attrs[VDPA_ATTR_DEV_NAME])
1119 return -EINVAL;
1120 devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
1121 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1122 if (!msg)
1123 return -ENOMEM;
1124
1125 down_read(&vdpa_dev_lock);
1126 dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match);
1127 if (!dev) {
1128 NL_SET_ERR_MSG_MOD(info->extack, "device not found");
1129 err = -ENODEV;
1130 goto dev_err;
1131 }
1132 vdev = container_of(dev, struct vdpa_device, dev);
1133 if (!vdev->mdev) {
1134 NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device");
1135 err = -EINVAL;
1136 goto mdev_err;
1137 }
1138 err = vdpa_dev_config_fill(vdev, msg, info->snd_portid, info->snd_seq,
1139 0, info->extack);
1140 if (!err)
1141 err = genlmsg_reply(msg, info);
1142
1143mdev_err:
1144 put_device(dev);
1145dev_err:
1146 up_read(&vdpa_dev_lock);
1147 if (err)
1148 nlmsg_free(msg);
1149 return err;
1150}
1151
1152static int vdpa_dev_config_dump(struct device *dev, void *data)
1153{
1154 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
1155 struct vdpa_dev_dump_info *info = data;
1156 int err;
1157
1158 if (!vdev->mdev)
1159 return 0;
1160 if (info->idx < info->start_idx) {
1161 info->idx++;
1162 return 0;
1163 }
1164 err = vdpa_dev_config_fill(vdev, info->msg, NETLINK_CB(info->cb->skb).portid,
1165 info->cb->nlh->nlmsg_seq, NLM_F_MULTI,
1166 info->cb->extack);
1167 if (err)
1168 return err;
1169
1170 info->idx++;
1171 return 0;
1172}
1173
1174static int
1175vdpa_nl_cmd_dev_config_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
1176{
1177 struct vdpa_dev_dump_info info;
1178
1179 info.msg = msg;
1180 info.cb = cb;
1181 info.start_idx = cb->args[0];
1182 info.idx = 0;
1183
1184 down_read(&vdpa_dev_lock);
1185 bus_for_each_dev(&vdpa_bus, NULL, &info, vdpa_dev_config_dump);
1186 up_read(&vdpa_dev_lock);
1187 cb->args[0] = info.idx;
1188 return msg->len;
1189}
1190
1191static int vdpa_nl_cmd_dev_stats_get_doit(struct sk_buff *skb,
1192 struct genl_info *info)
1193{
1194 struct vdpa_device *vdev;
1195 struct sk_buff *msg;
1196 const char *devname;
1197 struct device *dev;
1198 u32 index;
1199 int err;
1200
1201 if (!info->attrs[VDPA_ATTR_DEV_NAME])
1202 return -EINVAL;
1203
1204 if (!info->attrs[VDPA_ATTR_DEV_QUEUE_INDEX])
1205 return -EINVAL;
1206
1207 devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
1208 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1209 if (!msg)
1210 return -ENOMEM;
1211
1212 index = nla_get_u32(info->attrs[VDPA_ATTR_DEV_QUEUE_INDEX]);
1213 down_read(&vdpa_dev_lock);
1214 dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match);
1215 if (!dev) {
1216 NL_SET_ERR_MSG_MOD(info->extack, "device not found");
1217 err = -ENODEV;
1218 goto dev_err;
1219 }
1220 vdev = container_of(dev, struct vdpa_device, dev);
1221 if (!vdev->mdev) {
1222 NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device");
1223 err = -EINVAL;
1224 goto mdev_err;
1225 }
1226 err = vdpa_dev_vendor_stats_fill(vdev, msg, info, index);
1227 if (err)
1228 goto mdev_err;
1229
1230 err = genlmsg_reply(msg, info);
1231
1232 put_device(dev);
1233 up_read(&vdpa_dev_lock);
1234
1235 return err;
1236
1237mdev_err:
1238 put_device(dev);
1239dev_err:
1240 nlmsg_free(msg);
1241 up_read(&vdpa_dev_lock);
1242 return err;
1243}
1244
1245static const struct nla_policy vdpa_nl_policy[VDPA_ATTR_MAX + 1] = {
1246 [VDPA_ATTR_MGMTDEV_BUS_NAME] = { .type = NLA_NUL_STRING },
1247 [VDPA_ATTR_MGMTDEV_DEV_NAME] = { .type = NLA_STRING },
1248 [VDPA_ATTR_DEV_NAME] = { .type = NLA_STRING },
1249 [VDPA_ATTR_DEV_NET_CFG_MACADDR] = NLA_POLICY_ETH_ADDR,
1250 [VDPA_ATTR_DEV_NET_CFG_MAX_VQP] = { .type = NLA_U16 },
1251 /* virtio spec 1.1 section 5.1.4.1 for valid MTU range */
1252 [VDPA_ATTR_DEV_NET_CFG_MTU] = NLA_POLICY_MIN(NLA_U16, 68),
1253 [VDPA_ATTR_DEV_QUEUE_INDEX] = { .type = NLA_U32 },
1254 [VDPA_ATTR_DEV_FEATURES] = { .type = NLA_U64 },
1255};
1256
1257static const struct genl_ops vdpa_nl_ops[] = {
1258 {
1259 .cmd = VDPA_CMD_MGMTDEV_GET,
1260 .doit = vdpa_nl_cmd_mgmtdev_get_doit,
1261 .dumpit = vdpa_nl_cmd_mgmtdev_get_dumpit,
1262 },
1263 {
1264 .cmd = VDPA_CMD_DEV_NEW,
1265 .doit = vdpa_nl_cmd_dev_add_set_doit,
1266 .flags = GENL_ADMIN_PERM,
1267 },
1268 {
1269 .cmd = VDPA_CMD_DEV_DEL,
1270 .doit = vdpa_nl_cmd_dev_del_set_doit,
1271 .flags = GENL_ADMIN_PERM,
1272 },
1273 {
1274 .cmd = VDPA_CMD_DEV_GET,
1275 .doit = vdpa_nl_cmd_dev_get_doit,
1276 .dumpit = vdpa_nl_cmd_dev_get_dumpit,
1277 },
1278 {
1279 .cmd = VDPA_CMD_DEV_CONFIG_GET,
1280 .doit = vdpa_nl_cmd_dev_config_get_doit,
1281 .dumpit = vdpa_nl_cmd_dev_config_get_dumpit,
1282 },
1283 {
1284 .cmd = VDPA_CMD_DEV_VSTATS_GET,
1285 .doit = vdpa_nl_cmd_dev_stats_get_doit,
1286 .flags = GENL_ADMIN_PERM,
1287 },
1288};
1289
1290static struct genl_family vdpa_nl_family __ro_after_init = {
1291 .name = VDPA_GENL_NAME,
1292 .version = VDPA_GENL_VERSION,
1293 .maxattr = VDPA_ATTR_MAX,
1294 .policy = vdpa_nl_policy,
1295 .netnsok = false,
1296 .module = THIS_MODULE,
1297 .ops = vdpa_nl_ops,
1298 .n_ops = ARRAY_SIZE(vdpa_nl_ops),
1299 .resv_start_op = VDPA_CMD_DEV_VSTATS_GET + 1,
1300};
1301
1302static int vdpa_init(void)
1303{
1304 int err;
1305
1306 err = bus_register(&vdpa_bus);
1307 if (err)
1308 return err;
1309 err = genl_register_family(&vdpa_nl_family);
1310 if (err)
1311 goto err;
1312 return 0;
1313
1314err:
1315 bus_unregister(&vdpa_bus);
1316 return err;
1317}
1318
1319static void __exit vdpa_exit(void)
1320{
1321 genl_unregister_family(&vdpa_nl_family);
1322 bus_unregister(&vdpa_bus);
1323 ida_destroy(&vdpa_index_ida);
1324}
1325core_initcall(vdpa_init);
1326module_exit(vdpa_exit);
1327
1328MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>");
1329MODULE_LICENSE("GPL v2");
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * vDPA bus.
4 *
5 * Copyright (c) 2020, Red Hat. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/module.h>
11#include <linux/idr.h>
12#include <linux/slab.h>
13#include <linux/vdpa.h>
14#include <uapi/linux/vdpa.h>
15#include <net/genetlink.h>
16#include <linux/mod_devicetable.h>
17#include <linux/virtio_ids.h>
18
19static LIST_HEAD(mdev_head);
20/* A global mutex that protects vdpa management device and device level operations. */
21static DECLARE_RWSEM(vdpa_dev_lock);
22static DEFINE_IDA(vdpa_index_ida);
23
24void vdpa_set_status(struct vdpa_device *vdev, u8 status)
25{
26 down_write(&vdev->cf_lock);
27 vdev->config->set_status(vdev, status);
28 up_write(&vdev->cf_lock);
29}
30EXPORT_SYMBOL(vdpa_set_status);
31
32static struct genl_family vdpa_nl_family;
33
34static int vdpa_dev_probe(struct device *d)
35{
36 struct vdpa_device *vdev = dev_to_vdpa(d);
37 struct vdpa_driver *drv = drv_to_vdpa(vdev->dev.driver);
38 const struct vdpa_config_ops *ops = vdev->config;
39 u32 max_num, min_num = 1;
40 int ret = 0;
41
42 max_num = ops->get_vq_num_max(vdev);
43 if (ops->get_vq_num_min)
44 min_num = ops->get_vq_num_min(vdev);
45 if (max_num < min_num)
46 return -EINVAL;
47
48 if (drv && drv->probe)
49 ret = drv->probe(vdev);
50
51 return ret;
52}
53
54static void vdpa_dev_remove(struct device *d)
55{
56 struct vdpa_device *vdev = dev_to_vdpa(d);
57 struct vdpa_driver *drv = drv_to_vdpa(vdev->dev.driver);
58
59 if (drv && drv->remove)
60 drv->remove(vdev);
61}
62
63static int vdpa_dev_match(struct device *dev, struct device_driver *drv)
64{
65 struct vdpa_device *vdev = dev_to_vdpa(dev);
66
67 /* Check override first, and if set, only use the named driver */
68 if (vdev->driver_override)
69 return strcmp(vdev->driver_override, drv->name) == 0;
70
71 /* Currently devices must be supported by all vDPA bus drivers */
72 return 1;
73}
74
75static ssize_t driver_override_store(struct device *dev,
76 struct device_attribute *attr,
77 const char *buf, size_t count)
78{
79 struct vdpa_device *vdev = dev_to_vdpa(dev);
80 int ret;
81
82 ret = driver_set_override(dev, &vdev->driver_override, buf, count);
83 if (ret)
84 return ret;
85
86 return count;
87}
88
89static ssize_t driver_override_show(struct device *dev,
90 struct device_attribute *attr, char *buf)
91{
92 struct vdpa_device *vdev = dev_to_vdpa(dev);
93 ssize_t len;
94
95 device_lock(dev);
96 len = snprintf(buf, PAGE_SIZE, "%s\n", vdev->driver_override);
97 device_unlock(dev);
98
99 return len;
100}
101static DEVICE_ATTR_RW(driver_override);
102
103static struct attribute *vdpa_dev_attrs[] = {
104 &dev_attr_driver_override.attr,
105 NULL,
106};
107
108static const struct attribute_group vdpa_dev_group = {
109 .attrs = vdpa_dev_attrs,
110};
111__ATTRIBUTE_GROUPS(vdpa_dev);
112
113static struct bus_type vdpa_bus = {
114 .name = "vdpa",
115 .dev_groups = vdpa_dev_groups,
116 .match = vdpa_dev_match,
117 .probe = vdpa_dev_probe,
118 .remove = vdpa_dev_remove,
119};
120
121static void vdpa_release_dev(struct device *d)
122{
123 struct vdpa_device *vdev = dev_to_vdpa(d);
124 const struct vdpa_config_ops *ops = vdev->config;
125
126 if (ops->free)
127 ops->free(vdev);
128
129 ida_simple_remove(&vdpa_index_ida, vdev->index);
130 kfree(vdev->driver_override);
131 kfree(vdev);
132}
133
134/**
135 * __vdpa_alloc_device - allocate and initilaize a vDPA device
136 * This allows driver to some prepartion after device is
137 * initialized but before registered.
138 * @parent: the parent device
139 * @config: the bus operations that is supported by this device
140 * @ngroups: number of groups supported by this device
141 * @nas: number of address spaces supported by this device
142 * @size: size of the parent structure that contains private data
143 * @name: name of the vdpa device; optional.
144 * @use_va: indicate whether virtual address must be used by this device
145 *
146 * Driver should use vdpa_alloc_device() wrapper macro instead of
147 * using this directly.
148 *
149 * Return: Returns an error when parent/config/dma_dev is not set or fail to get
150 * ida.
151 */
152struct vdpa_device *__vdpa_alloc_device(struct device *parent,
153 const struct vdpa_config_ops *config,
154 unsigned int ngroups, unsigned int nas,
155 size_t size, const char *name,
156 bool use_va)
157{
158 struct vdpa_device *vdev;
159 int err = -EINVAL;
160
161 if (!config)
162 goto err;
163
164 if (!!config->dma_map != !!config->dma_unmap)
165 goto err;
166
167 /* It should only work for the device that use on-chip IOMMU */
168 if (use_va && !(config->dma_map || config->set_map))
169 goto err;
170
171 err = -ENOMEM;
172 vdev = kzalloc(size, GFP_KERNEL);
173 if (!vdev)
174 goto err;
175
176 err = ida_alloc(&vdpa_index_ida, GFP_KERNEL);
177 if (err < 0)
178 goto err_ida;
179
180 vdev->dev.bus = &vdpa_bus;
181 vdev->dev.parent = parent;
182 vdev->dev.release = vdpa_release_dev;
183 vdev->index = err;
184 vdev->config = config;
185 vdev->features_valid = false;
186 vdev->use_va = use_va;
187 vdev->ngroups = ngroups;
188 vdev->nas = nas;
189
190 if (name)
191 err = dev_set_name(&vdev->dev, "%s", name);
192 else
193 err = dev_set_name(&vdev->dev, "vdpa%u", vdev->index);
194 if (err)
195 goto err_name;
196
197 init_rwsem(&vdev->cf_lock);
198 device_initialize(&vdev->dev);
199
200 return vdev;
201
202err_name:
203 ida_simple_remove(&vdpa_index_ida, vdev->index);
204err_ida:
205 kfree(vdev);
206err:
207 return ERR_PTR(err);
208}
209EXPORT_SYMBOL_GPL(__vdpa_alloc_device);
210
211static int vdpa_name_match(struct device *dev, const void *data)
212{
213 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
214
215 return (strcmp(dev_name(&vdev->dev), data) == 0);
216}
217
218static int __vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
219{
220 struct device *dev;
221
222 vdev->nvqs = nvqs;
223
224 lockdep_assert_held(&vdpa_dev_lock);
225 dev = bus_find_device(&vdpa_bus, NULL, dev_name(&vdev->dev), vdpa_name_match);
226 if (dev) {
227 put_device(dev);
228 return -EEXIST;
229 }
230 return device_add(&vdev->dev);
231}
232
233/**
234 * _vdpa_register_device - register a vDPA device with vdpa lock held
235 * Caller must have a succeed call of vdpa_alloc_device() before.
236 * Caller must invoke this routine in the management device dev_add()
237 * callback after setting up valid mgmtdev for this vdpa device.
238 * @vdev: the vdpa device to be registered to vDPA bus
239 * @nvqs: number of virtqueues supported by this device
240 *
241 * Return: Returns an error when fail to add device to vDPA bus
242 */
243int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
244{
245 if (!vdev->mdev)
246 return -EINVAL;
247
248 return __vdpa_register_device(vdev, nvqs);
249}
250EXPORT_SYMBOL_GPL(_vdpa_register_device);
251
252/**
253 * vdpa_register_device - register a vDPA device
254 * Callers must have a succeed call of vdpa_alloc_device() before.
255 * @vdev: the vdpa device to be registered to vDPA bus
256 * @nvqs: number of virtqueues supported by this device
257 *
258 * Return: Returns an error when fail to add to vDPA bus
259 */
260int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
261{
262 int err;
263
264 down_write(&vdpa_dev_lock);
265 err = __vdpa_register_device(vdev, nvqs);
266 up_write(&vdpa_dev_lock);
267 return err;
268}
269EXPORT_SYMBOL_GPL(vdpa_register_device);
270
271/**
272 * _vdpa_unregister_device - unregister a vDPA device
273 * Caller must invoke this routine as part of management device dev_del()
274 * callback.
275 * @vdev: the vdpa device to be unregisted from vDPA bus
276 */
277void _vdpa_unregister_device(struct vdpa_device *vdev)
278{
279 lockdep_assert_held(&vdpa_dev_lock);
280 WARN_ON(!vdev->mdev);
281 device_unregister(&vdev->dev);
282}
283EXPORT_SYMBOL_GPL(_vdpa_unregister_device);
284
285/**
286 * vdpa_unregister_device - unregister a vDPA device
287 * @vdev: the vdpa device to be unregisted from vDPA bus
288 */
289void vdpa_unregister_device(struct vdpa_device *vdev)
290{
291 down_write(&vdpa_dev_lock);
292 device_unregister(&vdev->dev);
293 up_write(&vdpa_dev_lock);
294}
295EXPORT_SYMBOL_GPL(vdpa_unregister_device);
296
297/**
298 * __vdpa_register_driver - register a vDPA device driver
299 * @drv: the vdpa device driver to be registered
300 * @owner: module owner of the driver
301 *
302 * Return: Returns an err when fail to do the registration
303 */
304int __vdpa_register_driver(struct vdpa_driver *drv, struct module *owner)
305{
306 drv->driver.bus = &vdpa_bus;
307 drv->driver.owner = owner;
308
309 return driver_register(&drv->driver);
310}
311EXPORT_SYMBOL_GPL(__vdpa_register_driver);
312
313/**
314 * vdpa_unregister_driver - unregister a vDPA device driver
315 * @drv: the vdpa device driver to be unregistered
316 */
317void vdpa_unregister_driver(struct vdpa_driver *drv)
318{
319 driver_unregister(&drv->driver);
320}
321EXPORT_SYMBOL_GPL(vdpa_unregister_driver);
322
323/**
324 * vdpa_mgmtdev_register - register a vdpa management device
325 *
326 * @mdev: Pointer to vdpa management device
327 * vdpa_mgmtdev_register() register a vdpa management device which supports
328 * vdpa device management.
329 * Return: Returns 0 on success or failure when required callback ops are not
330 * initialized.
331 */
332int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev)
333{
334 if (!mdev->device || !mdev->ops || !mdev->ops->dev_add || !mdev->ops->dev_del)
335 return -EINVAL;
336
337 INIT_LIST_HEAD(&mdev->list);
338 down_write(&vdpa_dev_lock);
339 list_add_tail(&mdev->list, &mdev_head);
340 up_write(&vdpa_dev_lock);
341 return 0;
342}
343EXPORT_SYMBOL_GPL(vdpa_mgmtdev_register);
344
345static int vdpa_match_remove(struct device *dev, void *data)
346{
347 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
348 struct vdpa_mgmt_dev *mdev = vdev->mdev;
349
350 if (mdev == data)
351 mdev->ops->dev_del(mdev, vdev);
352 return 0;
353}
354
355void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev)
356{
357 down_write(&vdpa_dev_lock);
358
359 list_del(&mdev->list);
360
361 /* Filter out all the entries belong to this management device and delete it. */
362 bus_for_each_dev(&vdpa_bus, NULL, mdev, vdpa_match_remove);
363
364 up_write(&vdpa_dev_lock);
365}
366EXPORT_SYMBOL_GPL(vdpa_mgmtdev_unregister);
367
368static void vdpa_get_config_unlocked(struct vdpa_device *vdev,
369 unsigned int offset,
370 void *buf, unsigned int len)
371{
372 const struct vdpa_config_ops *ops = vdev->config;
373
374 /*
375 * Config accesses aren't supposed to trigger before features are set.
376 * If it does happen we assume a legacy guest.
377 */
378 if (!vdev->features_valid)
379 vdpa_set_features_unlocked(vdev, 0);
380 ops->get_config(vdev, offset, buf, len);
381}
382
383/**
384 * vdpa_get_config - Get one or more device configuration fields.
385 * @vdev: vdpa device to operate on
386 * @offset: starting byte offset of the field
387 * @buf: buffer pointer to read to
388 * @len: length of the configuration fields in bytes
389 */
390void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset,
391 void *buf, unsigned int len)
392{
393 down_read(&vdev->cf_lock);
394 vdpa_get_config_unlocked(vdev, offset, buf, len);
395 up_read(&vdev->cf_lock);
396}
397EXPORT_SYMBOL_GPL(vdpa_get_config);
398
399/**
400 * vdpa_set_config - Set one or more device configuration fields.
401 * @vdev: vdpa device to operate on
402 * @offset: starting byte offset of the field
403 * @buf: buffer pointer to read from
404 * @length: length of the configuration fields in bytes
405 */
406void vdpa_set_config(struct vdpa_device *vdev, unsigned int offset,
407 const void *buf, unsigned int length)
408{
409 down_write(&vdev->cf_lock);
410 vdev->config->set_config(vdev, offset, buf, length);
411 up_write(&vdev->cf_lock);
412}
413EXPORT_SYMBOL_GPL(vdpa_set_config);
414
415static bool mgmtdev_handle_match(const struct vdpa_mgmt_dev *mdev,
416 const char *busname, const char *devname)
417{
418 /* Bus name is optional for simulated management device, so ignore the
419 * device with bus if bus attribute is provided.
420 */
421 if ((busname && !mdev->device->bus) || (!busname && mdev->device->bus))
422 return false;
423
424 if (!busname && strcmp(dev_name(mdev->device), devname) == 0)
425 return true;
426
427 if (busname && (strcmp(mdev->device->bus->name, busname) == 0) &&
428 (strcmp(dev_name(mdev->device), devname) == 0))
429 return true;
430
431 return false;
432}
433
434static struct vdpa_mgmt_dev *vdpa_mgmtdev_get_from_attr(struct nlattr **attrs)
435{
436 struct vdpa_mgmt_dev *mdev;
437 const char *busname = NULL;
438 const char *devname;
439
440 if (!attrs[VDPA_ATTR_MGMTDEV_DEV_NAME])
441 return ERR_PTR(-EINVAL);
442 devname = nla_data(attrs[VDPA_ATTR_MGMTDEV_DEV_NAME]);
443 if (attrs[VDPA_ATTR_MGMTDEV_BUS_NAME])
444 busname = nla_data(attrs[VDPA_ATTR_MGMTDEV_BUS_NAME]);
445
446 list_for_each_entry(mdev, &mdev_head, list) {
447 if (mgmtdev_handle_match(mdev, busname, devname))
448 return mdev;
449 }
450 return ERR_PTR(-ENODEV);
451}
452
453static int vdpa_nl_mgmtdev_handle_fill(struct sk_buff *msg, const struct vdpa_mgmt_dev *mdev)
454{
455 if (mdev->device->bus &&
456 nla_put_string(msg, VDPA_ATTR_MGMTDEV_BUS_NAME, mdev->device->bus->name))
457 return -EMSGSIZE;
458 if (nla_put_string(msg, VDPA_ATTR_MGMTDEV_DEV_NAME, dev_name(mdev->device)))
459 return -EMSGSIZE;
460 return 0;
461}
462
463static int vdpa_mgmtdev_fill(const struct vdpa_mgmt_dev *mdev, struct sk_buff *msg,
464 u32 portid, u32 seq, int flags)
465{
466 u64 supported_classes = 0;
467 void *hdr;
468 int i = 0;
469 int err;
470
471 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, VDPA_CMD_MGMTDEV_NEW);
472 if (!hdr)
473 return -EMSGSIZE;
474 err = vdpa_nl_mgmtdev_handle_fill(msg, mdev);
475 if (err)
476 goto msg_err;
477
478 while (mdev->id_table[i].device) {
479 if (mdev->id_table[i].device <= 63)
480 supported_classes |= BIT_ULL(mdev->id_table[i].device);
481 i++;
482 }
483
484 if (nla_put_u64_64bit(msg, VDPA_ATTR_MGMTDEV_SUPPORTED_CLASSES,
485 supported_classes, VDPA_ATTR_UNSPEC)) {
486 err = -EMSGSIZE;
487 goto msg_err;
488 }
489 if (nla_put_u32(msg, VDPA_ATTR_DEV_MGMTDEV_MAX_VQS,
490 mdev->max_supported_vqs)) {
491 err = -EMSGSIZE;
492 goto msg_err;
493 }
494 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_SUPPORTED_FEATURES,
495 mdev->supported_features, VDPA_ATTR_PAD)) {
496 err = -EMSGSIZE;
497 goto msg_err;
498 }
499
500 genlmsg_end(msg, hdr);
501 return 0;
502
503msg_err:
504 genlmsg_cancel(msg, hdr);
505 return err;
506}
507
508static int vdpa_nl_cmd_mgmtdev_get_doit(struct sk_buff *skb, struct genl_info *info)
509{
510 struct vdpa_mgmt_dev *mdev;
511 struct sk_buff *msg;
512 int err;
513
514 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
515 if (!msg)
516 return -ENOMEM;
517
518 down_read(&vdpa_dev_lock);
519 mdev = vdpa_mgmtdev_get_from_attr(info->attrs);
520 if (IS_ERR(mdev)) {
521 up_read(&vdpa_dev_lock);
522 NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified mgmt device");
523 err = PTR_ERR(mdev);
524 goto out;
525 }
526
527 err = vdpa_mgmtdev_fill(mdev, msg, info->snd_portid, info->snd_seq, 0);
528 up_read(&vdpa_dev_lock);
529 if (err)
530 goto out;
531 err = genlmsg_reply(msg, info);
532 return err;
533
534out:
535 nlmsg_free(msg);
536 return err;
537}
538
539static int
540vdpa_nl_cmd_mgmtdev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
541{
542 struct vdpa_mgmt_dev *mdev;
543 int start = cb->args[0];
544 int idx = 0;
545 int err;
546
547 down_read(&vdpa_dev_lock);
548 list_for_each_entry(mdev, &mdev_head, list) {
549 if (idx < start) {
550 idx++;
551 continue;
552 }
553 err = vdpa_mgmtdev_fill(mdev, msg, NETLINK_CB(cb->skb).portid,
554 cb->nlh->nlmsg_seq, NLM_F_MULTI);
555 if (err)
556 goto out;
557 idx++;
558 }
559out:
560 up_read(&vdpa_dev_lock);
561 cb->args[0] = idx;
562 return msg->len;
563}
564
565#define VDPA_DEV_NET_ATTRS_MASK (BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | \
566 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU) | \
567 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP))
568
569static int vdpa_nl_cmd_dev_add_set_doit(struct sk_buff *skb, struct genl_info *info)
570{
571 struct vdpa_dev_set_config config = {};
572 struct nlattr **nl_attrs = info->attrs;
573 struct vdpa_mgmt_dev *mdev;
574 const u8 *macaddr;
575 const char *name;
576 int err = 0;
577
578 if (!info->attrs[VDPA_ATTR_DEV_NAME])
579 return -EINVAL;
580
581 name = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
582
583 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]) {
584 macaddr = nla_data(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]);
585 memcpy(config.net.mac, macaddr, sizeof(config.net.mac));
586 config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
587 }
588 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU]) {
589 config.net.mtu =
590 nla_get_u16(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU]);
591 config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU);
592 }
593 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]) {
594 config.net.max_vq_pairs =
595 nla_get_u16(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]);
596 if (!config.net.max_vq_pairs) {
597 NL_SET_ERR_MSG_MOD(info->extack,
598 "At least one pair of VQs is required");
599 return -EINVAL;
600 }
601 config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
602 }
603 if (nl_attrs[VDPA_ATTR_DEV_FEATURES]) {
604 config.device_features =
605 nla_get_u64(nl_attrs[VDPA_ATTR_DEV_FEATURES]);
606 config.mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);
607 }
608
609 /* Skip checking capability if user didn't prefer to configure any
610 * device networking attributes. It is likely that user might have used
611 * a device specific method to configure such attributes or using device
612 * default attributes.
613 */
614 if ((config.mask & VDPA_DEV_NET_ATTRS_MASK) &&
615 !netlink_capable(skb, CAP_NET_ADMIN))
616 return -EPERM;
617
618 down_write(&vdpa_dev_lock);
619 mdev = vdpa_mgmtdev_get_from_attr(info->attrs);
620 if (IS_ERR(mdev)) {
621 NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified management device");
622 err = PTR_ERR(mdev);
623 goto err;
624 }
625 if ((config.mask & mdev->config_attr_mask) != config.mask) {
626 NL_SET_ERR_MSG_MOD(info->extack,
627 "All provided attributes are not supported");
628 err = -EOPNOTSUPP;
629 goto err;
630 }
631
632 err = mdev->ops->dev_add(mdev, name, &config);
633err:
634 up_write(&vdpa_dev_lock);
635 return err;
636}
637
638static int vdpa_nl_cmd_dev_del_set_doit(struct sk_buff *skb, struct genl_info *info)
639{
640 struct vdpa_mgmt_dev *mdev;
641 struct vdpa_device *vdev;
642 struct device *dev;
643 const char *name;
644 int err = 0;
645
646 if (!info->attrs[VDPA_ATTR_DEV_NAME])
647 return -EINVAL;
648 name = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
649
650 down_write(&vdpa_dev_lock);
651 dev = bus_find_device(&vdpa_bus, NULL, name, vdpa_name_match);
652 if (!dev) {
653 NL_SET_ERR_MSG_MOD(info->extack, "device not found");
654 err = -ENODEV;
655 goto dev_err;
656 }
657 vdev = container_of(dev, struct vdpa_device, dev);
658 if (!vdev->mdev) {
659 NL_SET_ERR_MSG_MOD(info->extack, "Only user created device can be deleted by user");
660 err = -EINVAL;
661 goto mdev_err;
662 }
663 mdev = vdev->mdev;
664 mdev->ops->dev_del(mdev, vdev);
665mdev_err:
666 put_device(dev);
667dev_err:
668 up_write(&vdpa_dev_lock);
669 return err;
670}
671
672static int
673vdpa_dev_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq,
674 int flags, struct netlink_ext_ack *extack)
675{
676 u16 max_vq_size;
677 u16 min_vq_size = 1;
678 u32 device_id;
679 u32 vendor_id;
680 void *hdr;
681 int err;
682
683 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, VDPA_CMD_DEV_NEW);
684 if (!hdr)
685 return -EMSGSIZE;
686
687 err = vdpa_nl_mgmtdev_handle_fill(msg, vdev->mdev);
688 if (err)
689 goto msg_err;
690
691 device_id = vdev->config->get_device_id(vdev);
692 vendor_id = vdev->config->get_vendor_id(vdev);
693 max_vq_size = vdev->config->get_vq_num_max(vdev);
694 if (vdev->config->get_vq_num_min)
695 min_vq_size = vdev->config->get_vq_num_min(vdev);
696
697 err = -EMSGSIZE;
698 if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev)))
699 goto msg_err;
700 if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id))
701 goto msg_err;
702 if (nla_put_u32(msg, VDPA_ATTR_DEV_VENDOR_ID, vendor_id))
703 goto msg_err;
704 if (nla_put_u32(msg, VDPA_ATTR_DEV_MAX_VQS, vdev->nvqs))
705 goto msg_err;
706 if (nla_put_u16(msg, VDPA_ATTR_DEV_MAX_VQ_SIZE, max_vq_size))
707 goto msg_err;
708 if (nla_put_u16(msg, VDPA_ATTR_DEV_MIN_VQ_SIZE, min_vq_size))
709 goto msg_err;
710
711 genlmsg_end(msg, hdr);
712 return 0;
713
714msg_err:
715 genlmsg_cancel(msg, hdr);
716 return err;
717}
718
719static int vdpa_nl_cmd_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
720{
721 struct vdpa_device *vdev;
722 struct sk_buff *msg;
723 const char *devname;
724 struct device *dev;
725 int err;
726
727 if (!info->attrs[VDPA_ATTR_DEV_NAME])
728 return -EINVAL;
729 devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
730 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
731 if (!msg)
732 return -ENOMEM;
733
734 down_read(&vdpa_dev_lock);
735 dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match);
736 if (!dev) {
737 NL_SET_ERR_MSG_MOD(info->extack, "device not found");
738 err = -ENODEV;
739 goto err;
740 }
741 vdev = container_of(dev, struct vdpa_device, dev);
742 if (!vdev->mdev) {
743 err = -EINVAL;
744 goto mdev_err;
745 }
746 err = vdpa_dev_fill(vdev, msg, info->snd_portid, info->snd_seq, 0, info->extack);
747 if (err)
748 goto mdev_err;
749
750 err = genlmsg_reply(msg, info);
751 put_device(dev);
752 up_read(&vdpa_dev_lock);
753 return err;
754
755mdev_err:
756 put_device(dev);
757err:
758 up_read(&vdpa_dev_lock);
759 nlmsg_free(msg);
760 return err;
761}
762
763struct vdpa_dev_dump_info {
764 struct sk_buff *msg;
765 struct netlink_callback *cb;
766 int start_idx;
767 int idx;
768};
769
770static int vdpa_dev_dump(struct device *dev, void *data)
771{
772 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
773 struct vdpa_dev_dump_info *info = data;
774 int err;
775
776 if (!vdev->mdev)
777 return 0;
778 if (info->idx < info->start_idx) {
779 info->idx++;
780 return 0;
781 }
782 err = vdpa_dev_fill(vdev, info->msg, NETLINK_CB(info->cb->skb).portid,
783 info->cb->nlh->nlmsg_seq, NLM_F_MULTI, info->cb->extack);
784 if (err)
785 return err;
786
787 info->idx++;
788 return 0;
789}
790
791static int vdpa_nl_cmd_dev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
792{
793 struct vdpa_dev_dump_info info;
794
795 info.msg = msg;
796 info.cb = cb;
797 info.start_idx = cb->args[0];
798 info.idx = 0;
799
800 down_read(&vdpa_dev_lock);
801 bus_for_each_dev(&vdpa_bus, NULL, &info, vdpa_dev_dump);
802 up_read(&vdpa_dev_lock);
803 cb->args[0] = info.idx;
804 return msg->len;
805}
806
807static int vdpa_dev_net_mq_config_fill(struct sk_buff *msg, u64 features,
808 const struct virtio_net_config *config)
809{
810 u16 val_u16;
811
812 if ((features & BIT_ULL(VIRTIO_NET_F_MQ)) == 0 &&
813 (features & BIT_ULL(VIRTIO_NET_F_RSS)) == 0)
814 return 0;
815
816 val_u16 = __virtio16_to_cpu(true, config->max_virtqueue_pairs);
817
818 return nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MAX_VQP, val_u16);
819}
820
821static int vdpa_dev_net_mtu_config_fill(struct sk_buff *msg, u64 features,
822 const struct virtio_net_config *config)
823{
824 u16 val_u16;
825
826 if ((features & BIT_ULL(VIRTIO_NET_F_MTU)) == 0)
827 return 0;
828
829 val_u16 = __virtio16_to_cpu(true, config->mtu);
830
831 return nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16);
832}
833
834static int vdpa_dev_net_mac_config_fill(struct sk_buff *msg, u64 features,
835 const struct virtio_net_config *config)
836{
837 if ((features & BIT_ULL(VIRTIO_NET_F_MAC)) == 0)
838 return 0;
839
840 return nla_put(msg, VDPA_ATTR_DEV_NET_CFG_MACADDR,
841 sizeof(config->mac), config->mac);
842}
843
844static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *msg)
845{
846 struct virtio_net_config config = {};
847 u64 features_device;
848 u16 val_u16;
849
850 vdev->config->get_config(vdev, 0, &config, sizeof(config));
851
852 val_u16 = __virtio16_to_cpu(true, config.status);
853 if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_STATUS, val_u16))
854 return -EMSGSIZE;
855
856 features_device = vdev->config->get_device_features(vdev);
857
858 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_FEATURES, features_device,
859 VDPA_ATTR_PAD))
860 return -EMSGSIZE;
861
862 if (vdpa_dev_net_mtu_config_fill(msg, features_device, &config))
863 return -EMSGSIZE;
864
865 if (vdpa_dev_net_mac_config_fill(msg, features_device, &config))
866 return -EMSGSIZE;
867
868 return vdpa_dev_net_mq_config_fill(msg, features_device, &config);
869}
870
871static int
872vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq,
873 int flags, struct netlink_ext_ack *extack)
874{
875 u64 features_driver;
876 u8 status = 0;
877 u32 device_id;
878 void *hdr;
879 int err;
880
881 down_read(&vdev->cf_lock);
882 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags,
883 VDPA_CMD_DEV_CONFIG_GET);
884 if (!hdr) {
885 err = -EMSGSIZE;
886 goto out;
887 }
888
889 if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev))) {
890 err = -EMSGSIZE;
891 goto msg_err;
892 }
893
894 device_id = vdev->config->get_device_id(vdev);
895 if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id)) {
896 err = -EMSGSIZE;
897 goto msg_err;
898 }
899
900 /* only read driver features after the feature negotiation is done */
901 status = vdev->config->get_status(vdev);
902 if (status & VIRTIO_CONFIG_S_FEATURES_OK) {
903 features_driver = vdev->config->get_driver_features(vdev);
904 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES, features_driver,
905 VDPA_ATTR_PAD)) {
906 err = -EMSGSIZE;
907 goto msg_err;
908 }
909 }
910
911 switch (device_id) {
912 case VIRTIO_ID_NET:
913 err = vdpa_dev_net_config_fill(vdev, msg);
914 break;
915 default:
916 err = -EOPNOTSUPP;
917 break;
918 }
919 if (err)
920 goto msg_err;
921
922 up_read(&vdev->cf_lock);
923 genlmsg_end(msg, hdr);
924 return 0;
925
926msg_err:
927 genlmsg_cancel(msg, hdr);
928out:
929 up_read(&vdev->cf_lock);
930 return err;
931}
932
933static int vdpa_fill_stats_rec(struct vdpa_device *vdev, struct sk_buff *msg,
934 struct genl_info *info, u32 index)
935{
936 struct virtio_net_config config = {};
937 u64 features;
938 u8 status;
939 int err;
940
941 status = vdev->config->get_status(vdev);
942 if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
943 NL_SET_ERR_MSG_MOD(info->extack, "feature negotiation not complete");
944 return -EAGAIN;
945 }
946 vdpa_get_config_unlocked(vdev, 0, &config, sizeof(config));
947
948 features = vdev->config->get_driver_features(vdev);
949 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES,
950 features, VDPA_ATTR_PAD))
951 return -EMSGSIZE;
952
953 err = vdpa_dev_net_mq_config_fill(msg, features, &config);
954 if (err)
955 return err;
956
957 if (nla_put_u32(msg, VDPA_ATTR_DEV_QUEUE_INDEX, index))
958 return -EMSGSIZE;
959
960 err = vdev->config->get_vendor_vq_stats(vdev, index, msg, info->extack);
961 if (err)
962 return err;
963
964 return 0;
965}
966
967static int vendor_stats_fill(struct vdpa_device *vdev, struct sk_buff *msg,
968 struct genl_info *info, u32 index)
969{
970 int err;
971
972 down_read(&vdev->cf_lock);
973 if (!vdev->config->get_vendor_vq_stats) {
974 err = -EOPNOTSUPP;
975 goto out;
976 }
977
978 err = vdpa_fill_stats_rec(vdev, msg, info, index);
979out:
980 up_read(&vdev->cf_lock);
981 return err;
982}
983
984static int vdpa_dev_vendor_stats_fill(struct vdpa_device *vdev,
985 struct sk_buff *msg,
986 struct genl_info *info, u32 index)
987{
988 u32 device_id;
989 void *hdr;
990 int err;
991 u32 portid = info->snd_portid;
992 u32 seq = info->snd_seq;
993 u32 flags = 0;
994
995 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags,
996 VDPA_CMD_DEV_VSTATS_GET);
997 if (!hdr)
998 return -EMSGSIZE;
999
1000 if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev))) {
1001 err = -EMSGSIZE;
1002 goto undo_msg;
1003 }
1004
1005 device_id = vdev->config->get_device_id(vdev);
1006 if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id)) {
1007 err = -EMSGSIZE;
1008 goto undo_msg;
1009 }
1010
1011 switch (device_id) {
1012 case VIRTIO_ID_NET:
1013 if (index > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) {
1014 NL_SET_ERR_MSG_MOD(info->extack, "queue index excceeds max value");
1015 err = -ERANGE;
1016 break;
1017 }
1018
1019 err = vendor_stats_fill(vdev, msg, info, index);
1020 break;
1021 default:
1022 err = -EOPNOTSUPP;
1023 break;
1024 }
1025 genlmsg_end(msg, hdr);
1026
1027 return err;
1028
1029undo_msg:
1030 genlmsg_cancel(msg, hdr);
1031 return err;
1032}
1033
1034static int vdpa_nl_cmd_dev_config_get_doit(struct sk_buff *skb, struct genl_info *info)
1035{
1036 struct vdpa_device *vdev;
1037 struct sk_buff *msg;
1038 const char *devname;
1039 struct device *dev;
1040 int err;
1041
1042 if (!info->attrs[VDPA_ATTR_DEV_NAME])
1043 return -EINVAL;
1044 devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
1045 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1046 if (!msg)
1047 return -ENOMEM;
1048
1049 down_read(&vdpa_dev_lock);
1050 dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match);
1051 if (!dev) {
1052 NL_SET_ERR_MSG_MOD(info->extack, "device not found");
1053 err = -ENODEV;
1054 goto dev_err;
1055 }
1056 vdev = container_of(dev, struct vdpa_device, dev);
1057 if (!vdev->mdev) {
1058 NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device");
1059 err = -EINVAL;
1060 goto mdev_err;
1061 }
1062 err = vdpa_dev_config_fill(vdev, msg, info->snd_portid, info->snd_seq,
1063 0, info->extack);
1064 if (!err)
1065 err = genlmsg_reply(msg, info);
1066
1067mdev_err:
1068 put_device(dev);
1069dev_err:
1070 up_read(&vdpa_dev_lock);
1071 if (err)
1072 nlmsg_free(msg);
1073 return err;
1074}
1075
1076static int vdpa_dev_config_dump(struct device *dev, void *data)
1077{
1078 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
1079 struct vdpa_dev_dump_info *info = data;
1080 int err;
1081
1082 if (!vdev->mdev)
1083 return 0;
1084 if (info->idx < info->start_idx) {
1085 info->idx++;
1086 return 0;
1087 }
1088 err = vdpa_dev_config_fill(vdev, info->msg, NETLINK_CB(info->cb->skb).portid,
1089 info->cb->nlh->nlmsg_seq, NLM_F_MULTI,
1090 info->cb->extack);
1091 if (err)
1092 return err;
1093
1094 info->idx++;
1095 return 0;
1096}
1097
1098static int
1099vdpa_nl_cmd_dev_config_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
1100{
1101 struct vdpa_dev_dump_info info;
1102
1103 info.msg = msg;
1104 info.cb = cb;
1105 info.start_idx = cb->args[0];
1106 info.idx = 0;
1107
1108 down_read(&vdpa_dev_lock);
1109 bus_for_each_dev(&vdpa_bus, NULL, &info, vdpa_dev_config_dump);
1110 up_read(&vdpa_dev_lock);
1111 cb->args[0] = info.idx;
1112 return msg->len;
1113}
1114
1115static int vdpa_nl_cmd_dev_stats_get_doit(struct sk_buff *skb,
1116 struct genl_info *info)
1117{
1118 struct vdpa_device *vdev;
1119 struct sk_buff *msg;
1120 const char *devname;
1121 struct device *dev;
1122 u32 index;
1123 int err;
1124
1125 if (!info->attrs[VDPA_ATTR_DEV_NAME])
1126 return -EINVAL;
1127
1128 if (!info->attrs[VDPA_ATTR_DEV_QUEUE_INDEX])
1129 return -EINVAL;
1130
1131 devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
1132 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1133 if (!msg)
1134 return -ENOMEM;
1135
1136 index = nla_get_u32(info->attrs[VDPA_ATTR_DEV_QUEUE_INDEX]);
1137 down_read(&vdpa_dev_lock);
1138 dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match);
1139 if (!dev) {
1140 NL_SET_ERR_MSG_MOD(info->extack, "device not found");
1141 err = -ENODEV;
1142 goto dev_err;
1143 }
1144 vdev = container_of(dev, struct vdpa_device, dev);
1145 if (!vdev->mdev) {
1146 NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device");
1147 err = -EINVAL;
1148 goto mdev_err;
1149 }
1150 err = vdpa_dev_vendor_stats_fill(vdev, msg, info, index);
1151 if (err)
1152 goto mdev_err;
1153
1154 err = genlmsg_reply(msg, info);
1155
1156 put_device(dev);
1157 up_read(&vdpa_dev_lock);
1158
1159 return err;
1160
1161mdev_err:
1162 put_device(dev);
1163dev_err:
1164 nlmsg_free(msg);
1165 up_read(&vdpa_dev_lock);
1166 return err;
1167}
1168
1169static const struct nla_policy vdpa_nl_policy[VDPA_ATTR_MAX + 1] = {
1170 [VDPA_ATTR_MGMTDEV_BUS_NAME] = { .type = NLA_NUL_STRING },
1171 [VDPA_ATTR_MGMTDEV_DEV_NAME] = { .type = NLA_STRING },
1172 [VDPA_ATTR_DEV_NAME] = { .type = NLA_STRING },
1173 [VDPA_ATTR_DEV_NET_CFG_MACADDR] = NLA_POLICY_ETH_ADDR,
1174 /* virtio spec 1.1 section 5.1.4.1 for valid MTU range */
1175 [VDPA_ATTR_DEV_NET_CFG_MTU] = NLA_POLICY_MIN(NLA_U16, 68),
1176};
1177
1178static const struct genl_ops vdpa_nl_ops[] = {
1179 {
1180 .cmd = VDPA_CMD_MGMTDEV_GET,
1181 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1182 .doit = vdpa_nl_cmd_mgmtdev_get_doit,
1183 .dumpit = vdpa_nl_cmd_mgmtdev_get_dumpit,
1184 },
1185 {
1186 .cmd = VDPA_CMD_DEV_NEW,
1187 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1188 .doit = vdpa_nl_cmd_dev_add_set_doit,
1189 .flags = GENL_ADMIN_PERM,
1190 },
1191 {
1192 .cmd = VDPA_CMD_DEV_DEL,
1193 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1194 .doit = vdpa_nl_cmd_dev_del_set_doit,
1195 .flags = GENL_ADMIN_PERM,
1196 },
1197 {
1198 .cmd = VDPA_CMD_DEV_GET,
1199 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1200 .doit = vdpa_nl_cmd_dev_get_doit,
1201 .dumpit = vdpa_nl_cmd_dev_get_dumpit,
1202 },
1203 {
1204 .cmd = VDPA_CMD_DEV_CONFIG_GET,
1205 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1206 .doit = vdpa_nl_cmd_dev_config_get_doit,
1207 .dumpit = vdpa_nl_cmd_dev_config_get_dumpit,
1208 },
1209 {
1210 .cmd = VDPA_CMD_DEV_VSTATS_GET,
1211 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1212 .doit = vdpa_nl_cmd_dev_stats_get_doit,
1213 .flags = GENL_ADMIN_PERM,
1214 },
1215};
1216
1217static struct genl_family vdpa_nl_family __ro_after_init = {
1218 .name = VDPA_GENL_NAME,
1219 .version = VDPA_GENL_VERSION,
1220 .maxattr = VDPA_ATTR_MAX,
1221 .policy = vdpa_nl_policy,
1222 .netnsok = false,
1223 .module = THIS_MODULE,
1224 .ops = vdpa_nl_ops,
1225 .n_ops = ARRAY_SIZE(vdpa_nl_ops),
1226 .resv_start_op = VDPA_CMD_DEV_VSTATS_GET + 1,
1227};
1228
1229static int vdpa_init(void)
1230{
1231 int err;
1232
1233 err = bus_register(&vdpa_bus);
1234 if (err)
1235 return err;
1236 err = genl_register_family(&vdpa_nl_family);
1237 if (err)
1238 goto err;
1239 return 0;
1240
1241err:
1242 bus_unregister(&vdpa_bus);
1243 return err;
1244}
1245
1246static void __exit vdpa_exit(void)
1247{
1248 genl_unregister_family(&vdpa_nl_family);
1249 bus_unregister(&vdpa_bus);
1250 ida_destroy(&vdpa_index_ida);
1251}
1252core_initcall(vdpa_init);
1253module_exit(vdpa_exit);
1254
1255MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>");
1256MODULE_LICENSE("GPL v2");