1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * vDPA bus. 4 * 5 * Copyright (c) 2020, Red Hat. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/module.h> 11 #include <linux/idr.h> 12 #include <linux/slab.h> 13 #include <linux/vdpa.h> 14 #include <uapi/linux/vdpa.h> 15 #include <net/genetlink.h> 16 #include <linux/mod_devicetable.h> 17 #include <linux/virtio_ids.h> 18 19 static LIST_HEAD(mdev_head); 20 /* A global mutex that protects vdpa management device and device level operations. */ 21 static DECLARE_RWSEM(vdpa_dev_lock); 22 static DEFINE_IDA(vdpa_index_ida); 23 24 void vdpa_set_status(struct vdpa_device *vdev, u8 status) 25 { 26 down_write(&vdev->cf_lock); 27 vdev->config->set_status(vdev, status); 28 up_write(&vdev->cf_lock); 29 } 30 EXPORT_SYMBOL(vdpa_set_status); 31 32 static struct genl_family vdpa_nl_family; 33 34 static int vdpa_dev_probe(struct device *d) 35 { 36 struct vdpa_device *vdev = dev_to_vdpa(d); 37 struct vdpa_driver *drv = drv_to_vdpa(vdev->dev.driver); 38 const struct vdpa_config_ops *ops = vdev->config; 39 u32 max_num, min_num = 1; 40 int ret = 0; 41 42 max_num = ops->get_vq_num_max(vdev); 43 if (ops->get_vq_num_min) 44 min_num = ops->get_vq_num_min(vdev); 45 if (max_num < min_num) 46 return -EINVAL; 47 48 if (drv && drv->probe) 49 ret = drv->probe(vdev); 50 51 return ret; 52 } 53 54 static void vdpa_dev_remove(struct device *d) 55 { 56 struct vdpa_device *vdev = dev_to_vdpa(d); 57 struct vdpa_driver *drv = drv_to_vdpa(vdev->dev.driver); 58 59 if (drv && drv->remove) 60 drv->remove(vdev); 61 } 62 63 static int vdpa_dev_match(struct device *dev, struct device_driver *drv) 64 { 65 struct vdpa_device *vdev = dev_to_vdpa(dev); 66 67 /* Check override first, and if set, only use the named driver */ 68 if (vdev->driver_override) 69 return strcmp(vdev->driver_override, drv->name) == 0; 70 71 /* Currently devices must be supported by all vDPA bus drivers */ 72 return 1; 73 } 74 75 static ssize_t driver_override_store(struct device *dev, 76 struct device_attribute *attr, 77 const char *buf, size_t count) 78 { 79 struct vdpa_device *vdev = dev_to_vdpa(dev); 80 int ret; 81 82 ret = driver_set_override(dev, &vdev->driver_override, buf, count); 83 if (ret) 84 return ret; 85 86 return count; 87 } 88 89 static ssize_t driver_override_show(struct device *dev, 90 struct device_attribute *attr, char *buf) 91 { 92 struct vdpa_device *vdev = dev_to_vdpa(dev); 93 ssize_t len; 94 95 device_lock(dev); 96 len = snprintf(buf, PAGE_SIZE, "%s\n", vdev->driver_override); 97 device_unlock(dev); 98 99 return len; 100 } 101 static DEVICE_ATTR_RW(driver_override); 102 103 static struct attribute *vdpa_dev_attrs[] = { 104 &dev_attr_driver_override.attr, 105 NULL, 106 }; 107 108 static const struct attribute_group vdpa_dev_group = { 109 .attrs = vdpa_dev_attrs, 110 }; 111 __ATTRIBUTE_GROUPS(vdpa_dev); 112 113 static struct bus_type vdpa_bus = { 114 .name = "vdpa", 115 .dev_groups = vdpa_dev_groups, 116 .match = vdpa_dev_match, 117 .probe = vdpa_dev_probe, 118 .remove = vdpa_dev_remove, 119 }; 120 121 static void vdpa_release_dev(struct device *d) 122 { 123 struct vdpa_device *vdev = dev_to_vdpa(d); 124 const struct vdpa_config_ops *ops = vdev->config; 125 126 if (ops->free) 127 ops->free(vdev); 128 129 ida_simple_remove(&vdpa_index_ida, vdev->index); 130 kfree(vdev->driver_override); 131 kfree(vdev); 132 } 133 134 /** 135 * __vdpa_alloc_device - allocate and initilaize a vDPA device 136 * This allows driver to some prepartion after device is 137 * initialized but before registered. 138 * @parent: the parent device 139 * @config: the bus operations that is supported by this device 140 * @ngroups: number of groups supported by this device 141 * @nas: number of address spaces supported by this device 142 * @size: size of the parent structure that contains private data 143 * @name: name of the vdpa device; optional. 144 * @use_va: indicate whether virtual address must be used by this device 145 * 146 * Driver should use vdpa_alloc_device() wrapper macro instead of 147 * using this directly. 148 * 149 * Return: Returns an error when parent/config/dma_dev is not set or fail to get 150 * ida. 151 */ 152 struct vdpa_device *__vdpa_alloc_device(struct device *parent, 153 const struct vdpa_config_ops *config, 154 unsigned int ngroups, unsigned int nas, 155 size_t size, const char *name, 156 bool use_va) 157 { 158 struct vdpa_device *vdev; 159 int err = -EINVAL; 160 161 if (!config) 162 goto err; 163 164 if (!!config->dma_map != !!config->dma_unmap) 165 goto err; 166 167 /* It should only work for the device that use on-chip IOMMU */ 168 if (use_va && !(config->dma_map || config->set_map)) 169 goto err; 170 171 err = -ENOMEM; 172 vdev = kzalloc(size, GFP_KERNEL); 173 if (!vdev) 174 goto err; 175 176 err = ida_alloc(&vdpa_index_ida, GFP_KERNEL); 177 if (err < 0) 178 goto err_ida; 179 180 vdev->dev.bus = &vdpa_bus; 181 vdev->dev.parent = parent; 182 vdev->dev.release = vdpa_release_dev; 183 vdev->index = err; 184 vdev->config = config; 185 vdev->features_valid = false; 186 vdev->use_va = use_va; 187 vdev->ngroups = ngroups; 188 vdev->nas = nas; 189 190 if (name) 191 err = dev_set_name(&vdev->dev, "%s", name); 192 else 193 err = dev_set_name(&vdev->dev, "vdpa%u", vdev->index); 194 if (err) 195 goto err_name; 196 197 init_rwsem(&vdev->cf_lock); 198 device_initialize(&vdev->dev); 199 200 return vdev; 201 202 err_name: 203 ida_simple_remove(&vdpa_index_ida, vdev->index); 204 err_ida: 205 kfree(vdev); 206 err: 207 return ERR_PTR(err); 208 } 209 EXPORT_SYMBOL_GPL(__vdpa_alloc_device); 210 211 static int vdpa_name_match(struct device *dev, const void *data) 212 { 213 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev); 214 215 return (strcmp(dev_name(&vdev->dev), data) == 0); 216 } 217 218 static int __vdpa_register_device(struct vdpa_device *vdev, u32 nvqs) 219 { 220 struct device *dev; 221 222 vdev->nvqs = nvqs; 223 224 lockdep_assert_held(&vdpa_dev_lock); 225 dev = bus_find_device(&vdpa_bus, NULL, dev_name(&vdev->dev), vdpa_name_match); 226 if (dev) { 227 put_device(dev); 228 return -EEXIST; 229 } 230 return device_add(&vdev->dev); 231 } 232 233 /** 234 * _vdpa_register_device - register a vDPA device with vdpa lock held 235 * Caller must have a succeed call of vdpa_alloc_device() before. 236 * Caller must invoke this routine in the management device dev_add() 237 * callback after setting up valid mgmtdev for this vdpa device. 238 * @vdev: the vdpa device to be registered to vDPA bus 239 * @nvqs: number of virtqueues supported by this device 240 * 241 * Return: Returns an error when fail to add device to vDPA bus 242 */ 243 int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs) 244 { 245 if (!vdev->mdev) 246 return -EINVAL; 247 248 return __vdpa_register_device(vdev, nvqs); 249 } 250 EXPORT_SYMBOL_GPL(_vdpa_register_device); 251 252 /** 253 * vdpa_register_device - register a vDPA device 254 * Callers must have a succeed call of vdpa_alloc_device() before. 255 * @vdev: the vdpa device to be registered to vDPA bus 256 * @nvqs: number of virtqueues supported by this device 257 * 258 * Return: Returns an error when fail to add to vDPA bus 259 */ 260 int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs) 261 { 262 int err; 263 264 down_write(&vdpa_dev_lock); 265 err = __vdpa_register_device(vdev, nvqs); 266 up_write(&vdpa_dev_lock); 267 return err; 268 } 269 EXPORT_SYMBOL_GPL(vdpa_register_device); 270 271 /** 272 * _vdpa_unregister_device - unregister a vDPA device 273 * Caller must invoke this routine as part of management device dev_del() 274 * callback. 275 * @vdev: the vdpa device to be unregisted from vDPA bus 276 */ 277 void _vdpa_unregister_device(struct vdpa_device *vdev) 278 { 279 lockdep_assert_held(&vdpa_dev_lock); 280 WARN_ON(!vdev->mdev); 281 device_unregister(&vdev->dev); 282 } 283 EXPORT_SYMBOL_GPL(_vdpa_unregister_device); 284 285 /** 286 * vdpa_unregister_device - unregister a vDPA device 287 * @vdev: the vdpa device to be unregisted from vDPA bus 288 */ 289 void vdpa_unregister_device(struct vdpa_device *vdev) 290 { 291 down_write(&vdpa_dev_lock); 292 device_unregister(&vdev->dev); 293 up_write(&vdpa_dev_lock); 294 } 295 EXPORT_SYMBOL_GPL(vdpa_unregister_device); 296 297 /** 298 * __vdpa_register_driver - register a vDPA device driver 299 * @drv: the vdpa device driver to be registered 300 * @owner: module owner of the driver 301 * 302 * Return: Returns an err when fail to do the registration 303 */ 304 int __vdpa_register_driver(struct vdpa_driver *drv, struct module *owner) 305 { 306 drv->driver.bus = &vdpa_bus; 307 drv->driver.owner = owner; 308 309 return driver_register(&drv->driver); 310 } 311 EXPORT_SYMBOL_GPL(__vdpa_register_driver); 312 313 /** 314 * vdpa_unregister_driver - unregister a vDPA device driver 315 * @drv: the vdpa device driver to be unregistered 316 */ 317 void vdpa_unregister_driver(struct vdpa_driver *drv) 318 { 319 driver_unregister(&drv->driver); 320 } 321 EXPORT_SYMBOL_GPL(vdpa_unregister_driver); 322 323 /** 324 * vdpa_mgmtdev_register - register a vdpa management device 325 * 326 * @mdev: Pointer to vdpa management device 327 * vdpa_mgmtdev_register() register a vdpa management device which supports 328 * vdpa device management. 329 * Return: Returns 0 on success or failure when required callback ops are not 330 * initialized. 331 */ 332 int vdpa_mgmtdev_register(struct vdpa_mgmt_dev *mdev) 333 { 334 if (!mdev->device || !mdev->ops || !mdev->ops->dev_add || !mdev->ops->dev_del) 335 return -EINVAL; 336 337 INIT_LIST_HEAD(&mdev->list); 338 down_write(&vdpa_dev_lock); 339 list_add_tail(&mdev->list, &mdev_head); 340 up_write(&vdpa_dev_lock); 341 return 0; 342 } 343 EXPORT_SYMBOL_GPL(vdpa_mgmtdev_register); 344 345 static int vdpa_match_remove(struct device *dev, void *data) 346 { 347 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev); 348 struct vdpa_mgmt_dev *mdev = vdev->mdev; 349 350 if (mdev == data) 351 mdev->ops->dev_del(mdev, vdev); 352 return 0; 353 } 354 355 void vdpa_mgmtdev_unregister(struct vdpa_mgmt_dev *mdev) 356 { 357 down_write(&vdpa_dev_lock); 358 359 list_del(&mdev->list); 360 361 /* Filter out all the entries belong to this management device and delete it. */ 362 bus_for_each_dev(&vdpa_bus, NULL, mdev, vdpa_match_remove); 363 364 up_write(&vdpa_dev_lock); 365 } 366 EXPORT_SYMBOL_GPL(vdpa_mgmtdev_unregister); 367 368 static void vdpa_get_config_unlocked(struct vdpa_device *vdev, 369 unsigned int offset, 370 void *buf, unsigned int len) 371 { 372 const struct vdpa_config_ops *ops = vdev->config; 373 374 /* 375 * Config accesses aren't supposed to trigger before features are set. 376 * If it does happen we assume a legacy guest. 377 */ 378 if (!vdev->features_valid) 379 vdpa_set_features_unlocked(vdev, 0); 380 ops->get_config(vdev, offset, buf, len); 381 } 382 383 /** 384 * vdpa_get_config - Get one or more device configuration fields. 385 * @vdev: vdpa device to operate on 386 * @offset: starting byte offset of the field 387 * @buf: buffer pointer to read to 388 * @len: length of the configuration fields in bytes 389 */ 390 void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, 391 void *buf, unsigned int len) 392 { 393 down_read(&vdev->cf_lock); 394 vdpa_get_config_unlocked(vdev, offset, buf, len); 395 up_read(&vdev->cf_lock); 396 } 397 EXPORT_SYMBOL_GPL(vdpa_get_config); 398 399 /** 400 * vdpa_set_config - Set one or more device configuration fields. 401 * @vdev: vdpa device to operate on 402 * @offset: starting byte offset of the field 403 * @buf: buffer pointer to read from 404 * @length: length of the configuration fields in bytes 405 */ 406 void vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, 407 const void *buf, unsigned int length) 408 { 409 down_write(&vdev->cf_lock); 410 vdev->config->set_config(vdev, offset, buf, length); 411 up_write(&vdev->cf_lock); 412 } 413 EXPORT_SYMBOL_GPL(vdpa_set_config); 414 415 static bool mgmtdev_handle_match(const struct vdpa_mgmt_dev *mdev, 416 const char *busname, const char *devname) 417 { 418 /* Bus name is optional for simulated management device, so ignore the 419 * device with bus if bus attribute is provided. 420 */ 421 if ((busname && !mdev->device->bus) || (!busname && mdev->device->bus)) 422 return false; 423 424 if (!busname && strcmp(dev_name(mdev->device), devname) == 0) 425 return true; 426 427 if (busname && (strcmp(mdev->device->bus->name, busname) == 0) && 428 (strcmp(dev_name(mdev->device), devname) == 0)) 429 return true; 430 431 return false; 432 } 433 434 static struct vdpa_mgmt_dev *vdpa_mgmtdev_get_from_attr(struct nlattr **attrs) 435 { 436 struct vdpa_mgmt_dev *mdev; 437 const char *busname = NULL; 438 const char *devname; 439 440 if (!attrs[VDPA_ATTR_MGMTDEV_DEV_NAME]) 441 return ERR_PTR(-EINVAL); 442 devname = nla_data(attrs[VDPA_ATTR_MGMTDEV_DEV_NAME]); 443 if (attrs[VDPA_ATTR_MGMTDEV_BUS_NAME]) 444 busname = nla_data(attrs[VDPA_ATTR_MGMTDEV_BUS_NAME]); 445 446 list_for_each_entry(mdev, &mdev_head, list) { 447 if (mgmtdev_handle_match(mdev, busname, devname)) 448 return mdev; 449 } 450 return ERR_PTR(-ENODEV); 451 } 452 453 static int vdpa_nl_mgmtdev_handle_fill(struct sk_buff *msg, const struct vdpa_mgmt_dev *mdev) 454 { 455 if (mdev->device->bus && 456 nla_put_string(msg, VDPA_ATTR_MGMTDEV_BUS_NAME, mdev->device->bus->name)) 457 return -EMSGSIZE; 458 if (nla_put_string(msg, VDPA_ATTR_MGMTDEV_DEV_NAME, dev_name(mdev->device))) 459 return -EMSGSIZE; 460 return 0; 461 } 462 463 static int vdpa_mgmtdev_fill(const struct vdpa_mgmt_dev *mdev, struct sk_buff *msg, 464 u32 portid, u32 seq, int flags) 465 { 466 u64 supported_classes = 0; 467 void *hdr; 468 int i = 0; 469 int err; 470 471 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, VDPA_CMD_MGMTDEV_NEW); 472 if (!hdr) 473 return -EMSGSIZE; 474 err = vdpa_nl_mgmtdev_handle_fill(msg, mdev); 475 if (err) 476 goto msg_err; 477 478 while (mdev->id_table[i].device) { 479 if (mdev->id_table[i].device <= 63) 480 supported_classes |= BIT_ULL(mdev->id_table[i].device); 481 i++; 482 } 483 484 if (nla_put_u64_64bit(msg, VDPA_ATTR_MGMTDEV_SUPPORTED_CLASSES, 485 supported_classes, VDPA_ATTR_UNSPEC)) { 486 err = -EMSGSIZE; 487 goto msg_err; 488 } 489 if (nla_put_u32(msg, VDPA_ATTR_DEV_MGMTDEV_MAX_VQS, 490 mdev->max_supported_vqs)) { 491 err = -EMSGSIZE; 492 goto msg_err; 493 } 494 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_SUPPORTED_FEATURES, 495 mdev->supported_features, VDPA_ATTR_PAD)) { 496 err = -EMSGSIZE; 497 goto msg_err; 498 } 499 500 genlmsg_end(msg, hdr); 501 return 0; 502 503 msg_err: 504 genlmsg_cancel(msg, hdr); 505 return err; 506 } 507 508 static int vdpa_nl_cmd_mgmtdev_get_doit(struct sk_buff *skb, struct genl_info *info) 509 { 510 struct vdpa_mgmt_dev *mdev; 511 struct sk_buff *msg; 512 int err; 513 514 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 515 if (!msg) 516 return -ENOMEM; 517 518 down_read(&vdpa_dev_lock); 519 mdev = vdpa_mgmtdev_get_from_attr(info->attrs); 520 if (IS_ERR(mdev)) { 521 up_read(&vdpa_dev_lock); 522 NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified mgmt device"); 523 err = PTR_ERR(mdev); 524 goto out; 525 } 526 527 err = vdpa_mgmtdev_fill(mdev, msg, info->snd_portid, info->snd_seq, 0); 528 up_read(&vdpa_dev_lock); 529 if (err) 530 goto out; 531 err = genlmsg_reply(msg, info); 532 return err; 533 534 out: 535 nlmsg_free(msg); 536 return err; 537 } 538 539 static int 540 vdpa_nl_cmd_mgmtdev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb) 541 { 542 struct vdpa_mgmt_dev *mdev; 543 int start = cb->args[0]; 544 int idx = 0; 545 int err; 546 547 down_read(&vdpa_dev_lock); 548 list_for_each_entry(mdev, &mdev_head, list) { 549 if (idx < start) { 550 idx++; 551 continue; 552 } 553 err = vdpa_mgmtdev_fill(mdev, msg, NETLINK_CB(cb->skb).portid, 554 cb->nlh->nlmsg_seq, NLM_F_MULTI); 555 if (err) 556 goto out; 557 idx++; 558 } 559 out: 560 up_read(&vdpa_dev_lock); 561 cb->args[0] = idx; 562 return msg->len; 563 } 564 565 #define VDPA_DEV_NET_ATTRS_MASK (BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | \ 566 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU) | \ 567 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) 568 569 static int vdpa_nl_cmd_dev_add_set_doit(struct sk_buff *skb, struct genl_info *info) 570 { 571 struct vdpa_dev_set_config config = {}; 572 struct nlattr **nl_attrs = info->attrs; 573 struct vdpa_mgmt_dev *mdev; 574 const u8 *macaddr; 575 const char *name; 576 int err = 0; 577 578 if (!info->attrs[VDPA_ATTR_DEV_NAME]) 579 return -EINVAL; 580 581 name = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]); 582 583 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]) { 584 macaddr = nla_data(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]); 585 memcpy(config.net.mac, macaddr, sizeof(config.net.mac)); 586 config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR); 587 } 588 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU]) { 589 config.net.mtu = 590 nla_get_u16(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MTU]); 591 config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU); 592 } 593 if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]) { 594 config.net.max_vq_pairs = 595 nla_get_u16(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MAX_VQP]); 596 if (!config.net.max_vq_pairs) { 597 NL_SET_ERR_MSG_MOD(info->extack, 598 "At least one pair of VQs is required"); 599 return -EINVAL; 600 } 601 config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP); 602 } 603 604 /* Skip checking capability if user didn't prefer to configure any 605 * device networking attributes. It is likely that user might have used 606 * a device specific method to configure such attributes or using device 607 * default attributes. 608 */ 609 if ((config.mask & VDPA_DEV_NET_ATTRS_MASK) && 610 !netlink_capable(skb, CAP_NET_ADMIN)) 611 return -EPERM; 612 613 down_write(&vdpa_dev_lock); 614 mdev = vdpa_mgmtdev_get_from_attr(info->attrs); 615 if (IS_ERR(mdev)) { 616 NL_SET_ERR_MSG_MOD(info->extack, "Fail to find the specified management device"); 617 err = PTR_ERR(mdev); 618 goto err; 619 } 620 if ((config.mask & mdev->config_attr_mask) != config.mask) { 621 NL_SET_ERR_MSG_MOD(info->extack, 622 "All provided attributes are not supported"); 623 err = -EOPNOTSUPP; 624 goto err; 625 } 626 627 err = mdev->ops->dev_add(mdev, name, &config); 628 err: 629 up_write(&vdpa_dev_lock); 630 return err; 631 } 632 633 static int vdpa_nl_cmd_dev_del_set_doit(struct sk_buff *skb, struct genl_info *info) 634 { 635 struct vdpa_mgmt_dev *mdev; 636 struct vdpa_device *vdev; 637 struct device *dev; 638 const char *name; 639 int err = 0; 640 641 if (!info->attrs[VDPA_ATTR_DEV_NAME]) 642 return -EINVAL; 643 name = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]); 644 645 down_write(&vdpa_dev_lock); 646 dev = bus_find_device(&vdpa_bus, NULL, name, vdpa_name_match); 647 if (!dev) { 648 NL_SET_ERR_MSG_MOD(info->extack, "device not found"); 649 err = -ENODEV; 650 goto dev_err; 651 } 652 vdev = container_of(dev, struct vdpa_device, dev); 653 if (!vdev->mdev) { 654 NL_SET_ERR_MSG_MOD(info->extack, "Only user created device can be deleted by user"); 655 err = -EINVAL; 656 goto mdev_err; 657 } 658 mdev = vdev->mdev; 659 mdev->ops->dev_del(mdev, vdev); 660 mdev_err: 661 put_device(dev); 662 dev_err: 663 up_write(&vdpa_dev_lock); 664 return err; 665 } 666 667 static int 668 vdpa_dev_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq, 669 int flags, struct netlink_ext_ack *extack) 670 { 671 u16 max_vq_size; 672 u16 min_vq_size = 1; 673 u32 device_id; 674 u32 vendor_id; 675 void *hdr; 676 int err; 677 678 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, VDPA_CMD_DEV_NEW); 679 if (!hdr) 680 return -EMSGSIZE; 681 682 err = vdpa_nl_mgmtdev_handle_fill(msg, vdev->mdev); 683 if (err) 684 goto msg_err; 685 686 device_id = vdev->config->get_device_id(vdev); 687 vendor_id = vdev->config->get_vendor_id(vdev); 688 max_vq_size = vdev->config->get_vq_num_max(vdev); 689 if (vdev->config->get_vq_num_min) 690 min_vq_size = vdev->config->get_vq_num_min(vdev); 691 692 err = -EMSGSIZE; 693 if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev))) 694 goto msg_err; 695 if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id)) 696 goto msg_err; 697 if (nla_put_u32(msg, VDPA_ATTR_DEV_VENDOR_ID, vendor_id)) 698 goto msg_err; 699 if (nla_put_u32(msg, VDPA_ATTR_DEV_MAX_VQS, vdev->nvqs)) 700 goto msg_err; 701 if (nla_put_u16(msg, VDPA_ATTR_DEV_MAX_VQ_SIZE, max_vq_size)) 702 goto msg_err; 703 if (nla_put_u16(msg, VDPA_ATTR_DEV_MIN_VQ_SIZE, min_vq_size)) 704 goto msg_err; 705 706 genlmsg_end(msg, hdr); 707 return 0; 708 709 msg_err: 710 genlmsg_cancel(msg, hdr); 711 return err; 712 } 713 714 static int vdpa_nl_cmd_dev_get_doit(struct sk_buff *skb, struct genl_info *info) 715 { 716 struct vdpa_device *vdev; 717 struct sk_buff *msg; 718 const char *devname; 719 struct device *dev; 720 int err; 721 722 if (!info->attrs[VDPA_ATTR_DEV_NAME]) 723 return -EINVAL; 724 devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]); 725 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 726 if (!msg) 727 return -ENOMEM; 728 729 down_read(&vdpa_dev_lock); 730 dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match); 731 if (!dev) { 732 NL_SET_ERR_MSG_MOD(info->extack, "device not found"); 733 err = -ENODEV; 734 goto err; 735 } 736 vdev = container_of(dev, struct vdpa_device, dev); 737 if (!vdev->mdev) { 738 err = -EINVAL; 739 goto mdev_err; 740 } 741 err = vdpa_dev_fill(vdev, msg, info->snd_portid, info->snd_seq, 0, info->extack); 742 if (err) 743 goto mdev_err; 744 745 err = genlmsg_reply(msg, info); 746 put_device(dev); 747 up_read(&vdpa_dev_lock); 748 return err; 749 750 mdev_err: 751 put_device(dev); 752 err: 753 up_read(&vdpa_dev_lock); 754 nlmsg_free(msg); 755 return err; 756 } 757 758 struct vdpa_dev_dump_info { 759 struct sk_buff *msg; 760 struct netlink_callback *cb; 761 int start_idx; 762 int idx; 763 }; 764 765 static int vdpa_dev_dump(struct device *dev, void *data) 766 { 767 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev); 768 struct vdpa_dev_dump_info *info = data; 769 int err; 770 771 if (!vdev->mdev) 772 return 0; 773 if (info->idx < info->start_idx) { 774 info->idx++; 775 return 0; 776 } 777 err = vdpa_dev_fill(vdev, info->msg, NETLINK_CB(info->cb->skb).portid, 778 info->cb->nlh->nlmsg_seq, NLM_F_MULTI, info->cb->extack); 779 if (err) 780 return err; 781 782 info->idx++; 783 return 0; 784 } 785 786 static int vdpa_nl_cmd_dev_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb) 787 { 788 struct vdpa_dev_dump_info info; 789 790 info.msg = msg; 791 info.cb = cb; 792 info.start_idx = cb->args[0]; 793 info.idx = 0; 794 795 down_read(&vdpa_dev_lock); 796 bus_for_each_dev(&vdpa_bus, NULL, &info, vdpa_dev_dump); 797 up_read(&vdpa_dev_lock); 798 cb->args[0] = info.idx; 799 return msg->len; 800 } 801 802 static int vdpa_dev_net_mq_config_fill(struct vdpa_device *vdev, 803 struct sk_buff *msg, u64 features, 804 const struct virtio_net_config *config) 805 { 806 u16 val_u16; 807 808 if ((features & BIT_ULL(VIRTIO_NET_F_MQ)) == 0) 809 return 0; 810 811 val_u16 = le16_to_cpu(config->max_virtqueue_pairs); 812 return nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MAX_VQP, val_u16); 813 } 814 815 static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) 816 { 817 struct virtio_net_config config = {}; 818 u64 features; 819 u16 val_u16; 820 821 vdpa_get_config_unlocked(vdev, 0, &config, sizeof(config)); 822 823 if (nla_put(msg, VDPA_ATTR_DEV_NET_CFG_MACADDR, sizeof(config.mac), 824 config.mac)) 825 return -EMSGSIZE; 826 827 val_u16 = __virtio16_to_cpu(true, config.status); 828 if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_STATUS, val_u16)) 829 return -EMSGSIZE; 830 831 val_u16 = __virtio16_to_cpu(true, config.mtu); 832 if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16)) 833 return -EMSGSIZE; 834 835 features = vdev->config->get_driver_features(vdev); 836 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES, features, 837 VDPA_ATTR_PAD)) 838 return -EMSGSIZE; 839 840 return vdpa_dev_net_mq_config_fill(vdev, msg, features, &config); 841 } 842 843 static int 844 vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq, 845 int flags, struct netlink_ext_ack *extack) 846 { 847 u32 device_id; 848 void *hdr; 849 int err; 850 851 down_read(&vdev->cf_lock); 852 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, 853 VDPA_CMD_DEV_CONFIG_GET); 854 if (!hdr) { 855 err = -EMSGSIZE; 856 goto out; 857 } 858 859 if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev))) { 860 err = -EMSGSIZE; 861 goto msg_err; 862 } 863 864 device_id = vdev->config->get_device_id(vdev); 865 if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id)) { 866 err = -EMSGSIZE; 867 goto msg_err; 868 } 869 870 switch (device_id) { 871 case VIRTIO_ID_NET: 872 err = vdpa_dev_net_config_fill(vdev, msg); 873 break; 874 default: 875 err = -EOPNOTSUPP; 876 break; 877 } 878 if (err) 879 goto msg_err; 880 881 up_read(&vdev->cf_lock); 882 genlmsg_end(msg, hdr); 883 return 0; 884 885 msg_err: 886 genlmsg_cancel(msg, hdr); 887 out: 888 up_read(&vdev->cf_lock); 889 return err; 890 } 891 892 static int vdpa_fill_stats_rec(struct vdpa_device *vdev, struct sk_buff *msg, 893 struct genl_info *info, u32 index) 894 { 895 struct virtio_net_config config = {}; 896 u64 features; 897 u16 max_vqp; 898 u8 status; 899 int err; 900 901 status = vdev->config->get_status(vdev); 902 if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) { 903 NL_SET_ERR_MSG_MOD(info->extack, "feature negotiation not complete"); 904 return -EAGAIN; 905 } 906 vdpa_get_config_unlocked(vdev, 0, &config, sizeof(config)); 907 908 max_vqp = __virtio16_to_cpu(true, config.max_virtqueue_pairs); 909 if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MAX_VQP, max_vqp)) 910 return -EMSGSIZE; 911 912 features = vdev->config->get_driver_features(vdev); 913 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES, 914 features, VDPA_ATTR_PAD)) 915 return -EMSGSIZE; 916 917 if (nla_put_u32(msg, VDPA_ATTR_DEV_QUEUE_INDEX, index)) 918 return -EMSGSIZE; 919 920 err = vdev->config->get_vendor_vq_stats(vdev, index, msg, info->extack); 921 if (err) 922 return err; 923 924 return 0; 925 } 926 927 static int vendor_stats_fill(struct vdpa_device *vdev, struct sk_buff *msg, 928 struct genl_info *info, u32 index) 929 { 930 int err; 931 932 down_read(&vdev->cf_lock); 933 if (!vdev->config->get_vendor_vq_stats) { 934 err = -EOPNOTSUPP; 935 goto out; 936 } 937 938 err = vdpa_fill_stats_rec(vdev, msg, info, index); 939 out: 940 up_read(&vdev->cf_lock); 941 return err; 942 } 943 944 static int vdpa_dev_vendor_stats_fill(struct vdpa_device *vdev, 945 struct sk_buff *msg, 946 struct genl_info *info, u32 index) 947 { 948 u32 device_id; 949 void *hdr; 950 int err; 951 u32 portid = info->snd_portid; 952 u32 seq = info->snd_seq; 953 u32 flags = 0; 954 955 hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, 956 VDPA_CMD_DEV_VSTATS_GET); 957 if (!hdr) 958 return -EMSGSIZE; 959 960 if (nla_put_string(msg, VDPA_ATTR_DEV_NAME, dev_name(&vdev->dev))) { 961 err = -EMSGSIZE; 962 goto undo_msg; 963 } 964 965 device_id = vdev->config->get_device_id(vdev); 966 if (nla_put_u32(msg, VDPA_ATTR_DEV_ID, device_id)) { 967 err = -EMSGSIZE; 968 goto undo_msg; 969 } 970 971 switch (device_id) { 972 case VIRTIO_ID_NET: 973 if (index > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) { 974 NL_SET_ERR_MSG_MOD(info->extack, "queue index excceeds max value"); 975 err = -ERANGE; 976 break; 977 } 978 979 err = vendor_stats_fill(vdev, msg, info, index); 980 break; 981 default: 982 err = -EOPNOTSUPP; 983 break; 984 } 985 genlmsg_end(msg, hdr); 986 987 return err; 988 989 undo_msg: 990 genlmsg_cancel(msg, hdr); 991 return err; 992 } 993 994 static int vdpa_nl_cmd_dev_config_get_doit(struct sk_buff *skb, struct genl_info *info) 995 { 996 struct vdpa_device *vdev; 997 struct sk_buff *msg; 998 const char *devname; 999 struct device *dev; 1000 int err; 1001 1002 if (!info->attrs[VDPA_ATTR_DEV_NAME]) 1003 return -EINVAL; 1004 devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]); 1005 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 1006 if (!msg) 1007 return -ENOMEM; 1008 1009 down_read(&vdpa_dev_lock); 1010 dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match); 1011 if (!dev) { 1012 NL_SET_ERR_MSG_MOD(info->extack, "device not found"); 1013 err = -ENODEV; 1014 goto dev_err; 1015 } 1016 vdev = container_of(dev, struct vdpa_device, dev); 1017 if (!vdev->mdev) { 1018 NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device"); 1019 err = -EINVAL; 1020 goto mdev_err; 1021 } 1022 err = vdpa_dev_config_fill(vdev, msg, info->snd_portid, info->snd_seq, 1023 0, info->extack); 1024 if (!err) 1025 err = genlmsg_reply(msg, info); 1026 1027 mdev_err: 1028 put_device(dev); 1029 dev_err: 1030 up_read(&vdpa_dev_lock); 1031 if (err) 1032 nlmsg_free(msg); 1033 return err; 1034 } 1035 1036 static int vdpa_dev_config_dump(struct device *dev, void *data) 1037 { 1038 struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev); 1039 struct vdpa_dev_dump_info *info = data; 1040 int err; 1041 1042 if (!vdev->mdev) 1043 return 0; 1044 if (info->idx < info->start_idx) { 1045 info->idx++; 1046 return 0; 1047 } 1048 err = vdpa_dev_config_fill(vdev, info->msg, NETLINK_CB(info->cb->skb).portid, 1049 info->cb->nlh->nlmsg_seq, NLM_F_MULTI, 1050 info->cb->extack); 1051 if (err) 1052 return err; 1053 1054 info->idx++; 1055 return 0; 1056 } 1057 1058 static int 1059 vdpa_nl_cmd_dev_config_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb) 1060 { 1061 struct vdpa_dev_dump_info info; 1062 1063 info.msg = msg; 1064 info.cb = cb; 1065 info.start_idx = cb->args[0]; 1066 info.idx = 0; 1067 1068 down_read(&vdpa_dev_lock); 1069 bus_for_each_dev(&vdpa_bus, NULL, &info, vdpa_dev_config_dump); 1070 up_read(&vdpa_dev_lock); 1071 cb->args[0] = info.idx; 1072 return msg->len; 1073 } 1074 1075 static int vdpa_nl_cmd_dev_stats_get_doit(struct sk_buff *skb, 1076 struct genl_info *info) 1077 { 1078 struct vdpa_device *vdev; 1079 struct sk_buff *msg; 1080 const char *devname; 1081 struct device *dev; 1082 u32 index; 1083 int err; 1084 1085 if (!info->attrs[VDPA_ATTR_DEV_NAME]) 1086 return -EINVAL; 1087 1088 if (!info->attrs[VDPA_ATTR_DEV_QUEUE_INDEX]) 1089 return -EINVAL; 1090 1091 devname = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]); 1092 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 1093 if (!msg) 1094 return -ENOMEM; 1095 1096 index = nla_get_u32(info->attrs[VDPA_ATTR_DEV_QUEUE_INDEX]); 1097 down_read(&vdpa_dev_lock); 1098 dev = bus_find_device(&vdpa_bus, NULL, devname, vdpa_name_match); 1099 if (!dev) { 1100 NL_SET_ERR_MSG_MOD(info->extack, "device not found"); 1101 err = -ENODEV; 1102 goto dev_err; 1103 } 1104 vdev = container_of(dev, struct vdpa_device, dev); 1105 if (!vdev->mdev) { 1106 NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device"); 1107 err = -EINVAL; 1108 goto mdev_err; 1109 } 1110 err = vdpa_dev_vendor_stats_fill(vdev, msg, info, index); 1111 if (err) 1112 goto mdev_err; 1113 1114 err = genlmsg_reply(msg, info); 1115 1116 put_device(dev); 1117 up_read(&vdpa_dev_lock); 1118 1119 return err; 1120 1121 mdev_err: 1122 put_device(dev); 1123 dev_err: 1124 nlmsg_free(msg); 1125 up_read(&vdpa_dev_lock); 1126 return err; 1127 } 1128 1129 static const struct nla_policy vdpa_nl_policy[VDPA_ATTR_MAX + 1] = { 1130 [VDPA_ATTR_MGMTDEV_BUS_NAME] = { .type = NLA_NUL_STRING }, 1131 [VDPA_ATTR_MGMTDEV_DEV_NAME] = { .type = NLA_STRING }, 1132 [VDPA_ATTR_DEV_NAME] = { .type = NLA_STRING }, 1133 [VDPA_ATTR_DEV_NET_CFG_MACADDR] = NLA_POLICY_ETH_ADDR, 1134 /* virtio spec 1.1 section 5.1.4.1 for valid MTU range */ 1135 [VDPA_ATTR_DEV_NET_CFG_MTU] = NLA_POLICY_MIN(NLA_U16, 68), 1136 }; 1137 1138 static const struct genl_ops vdpa_nl_ops[] = { 1139 { 1140 .cmd = VDPA_CMD_MGMTDEV_GET, 1141 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1142 .doit = vdpa_nl_cmd_mgmtdev_get_doit, 1143 .dumpit = vdpa_nl_cmd_mgmtdev_get_dumpit, 1144 }, 1145 { 1146 .cmd = VDPA_CMD_DEV_NEW, 1147 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1148 .doit = vdpa_nl_cmd_dev_add_set_doit, 1149 .flags = GENL_ADMIN_PERM, 1150 }, 1151 { 1152 .cmd = VDPA_CMD_DEV_DEL, 1153 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1154 .doit = vdpa_nl_cmd_dev_del_set_doit, 1155 .flags = GENL_ADMIN_PERM, 1156 }, 1157 { 1158 .cmd = VDPA_CMD_DEV_GET, 1159 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1160 .doit = vdpa_nl_cmd_dev_get_doit, 1161 .dumpit = vdpa_nl_cmd_dev_get_dumpit, 1162 }, 1163 { 1164 .cmd = VDPA_CMD_DEV_CONFIG_GET, 1165 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1166 .doit = vdpa_nl_cmd_dev_config_get_doit, 1167 .dumpit = vdpa_nl_cmd_dev_config_get_dumpit, 1168 }, 1169 { 1170 .cmd = VDPA_CMD_DEV_VSTATS_GET, 1171 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1172 .doit = vdpa_nl_cmd_dev_stats_get_doit, 1173 .flags = GENL_ADMIN_PERM, 1174 }, 1175 }; 1176 1177 static struct genl_family vdpa_nl_family __ro_after_init = { 1178 .name = VDPA_GENL_NAME, 1179 .version = VDPA_GENL_VERSION, 1180 .maxattr = VDPA_ATTR_MAX, 1181 .policy = vdpa_nl_policy, 1182 .netnsok = false, 1183 .module = THIS_MODULE, 1184 .ops = vdpa_nl_ops, 1185 .n_ops = ARRAY_SIZE(vdpa_nl_ops), 1186 }; 1187 1188 static int vdpa_init(void) 1189 { 1190 int err; 1191 1192 err = bus_register(&vdpa_bus); 1193 if (err) 1194 return err; 1195 err = genl_register_family(&vdpa_nl_family); 1196 if (err) 1197 goto err; 1198 return 0; 1199 1200 err: 1201 bus_unregister(&vdpa_bus); 1202 return err; 1203 } 1204 1205 static void __exit vdpa_exit(void) 1206 { 1207 genl_unregister_family(&vdpa_nl_family); 1208 bus_unregister(&vdpa_bus); 1209 ida_destroy(&vdpa_index_ida); 1210 } 1211 core_initcall(vdpa_init); 1212 module_exit(vdpa_exit); 1213 1214 MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>"); 1215 MODULE_LICENSE("GPL v2"); 1216