1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */ 3 #include <linux/memremap.h> 4 #include <linux/device.h> 5 #include <linux/mutex.h> 6 #include <linux/list.h> 7 #include <linux/slab.h> 8 #include <linux/dax.h> 9 #include "dax-private.h" 10 #include "bus.h" 11 12 static struct class *dax_class; 13 14 static DEFINE_MUTEX(dax_bus_lock); 15 16 #define DAX_NAME_LEN 30 17 struct dax_id { 18 struct list_head list; 19 char dev_name[DAX_NAME_LEN]; 20 }; 21 22 static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env) 23 { 24 /* 25 * We only ever expect to handle device-dax instances, i.e. the 26 * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero 27 */ 28 return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0); 29 } 30 31 static struct dax_device_driver *to_dax_drv(struct device_driver *drv) 32 { 33 return container_of(drv, struct dax_device_driver, drv); 34 } 35 36 static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv, 37 const char *dev_name) 38 { 39 struct dax_id *dax_id; 40 41 lockdep_assert_held(&dax_bus_lock); 42 43 list_for_each_entry(dax_id, &dax_drv->ids, list) 44 if (sysfs_streq(dax_id->dev_name, dev_name)) 45 return dax_id; 46 return NULL; 47 } 48 49 static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev) 50 { 51 int match; 52 53 mutex_lock(&dax_bus_lock); 54 match = !!__dax_match_id(dax_drv, dev_name(dev)); 55 mutex_unlock(&dax_bus_lock); 56 57 return match; 58 } 59 60 enum id_action { 61 ID_REMOVE, 62 ID_ADD, 63 }; 64 65 static ssize_t do_id_store(struct device_driver *drv, const char *buf, 66 size_t count, enum id_action action) 67 { 68 struct dax_device_driver *dax_drv = to_dax_drv(drv); 69 unsigned int region_id, id; 70 char devname[DAX_NAME_LEN]; 71 struct dax_id *dax_id; 72 ssize_t rc = count; 73 int fields; 74 75 fields = sscanf(buf, "dax%d.%d", ®ion_id, &id); 76 if (fields != 2) 77 return -EINVAL; 78 sprintf(devname, "dax%d.%d", region_id, id); 79 if (!sysfs_streq(buf, devname)) 80 return -EINVAL; 81 82 mutex_lock(&dax_bus_lock); 83 dax_id = __dax_match_id(dax_drv, buf); 84 if (!dax_id) { 85 if (action == ID_ADD) { 86 dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL); 87 if (dax_id) { 88 strncpy(dax_id->dev_name, buf, DAX_NAME_LEN); 89 list_add(&dax_id->list, &dax_drv->ids); 90 } else 91 rc = -ENOMEM; 92 } else 93 /* nothing to remove */; 94 } else if (action == ID_REMOVE) { 95 list_del(&dax_id->list); 96 kfree(dax_id); 97 } else 98 /* dax_id already added */; 99 mutex_unlock(&dax_bus_lock); 100 101 if (rc < 0) 102 return rc; 103 if (action == ID_ADD) 104 rc = driver_attach(drv); 105 if (rc) 106 return rc; 107 return count; 108 } 109 110 static ssize_t new_id_store(struct device_driver *drv, const char *buf, 111 size_t count) 112 { 113 return do_id_store(drv, buf, count, ID_ADD); 114 } 115 static DRIVER_ATTR_WO(new_id); 116 117 static ssize_t remove_id_store(struct device_driver *drv, const char *buf, 118 size_t count) 119 { 120 return do_id_store(drv, buf, count, ID_REMOVE); 121 } 122 static DRIVER_ATTR_WO(remove_id); 123 124 static struct attribute *dax_drv_attrs[] = { 125 &driver_attr_new_id.attr, 126 &driver_attr_remove_id.attr, 127 NULL, 128 }; 129 ATTRIBUTE_GROUPS(dax_drv); 130 131 static int dax_bus_match(struct device *dev, struct device_driver *drv); 132 133 static struct bus_type dax_bus_type = { 134 .name = "dax", 135 .uevent = dax_bus_uevent, 136 .match = dax_bus_match, 137 .drv_groups = dax_drv_groups, 138 }; 139 140 static int dax_bus_match(struct device *dev, struct device_driver *drv) 141 { 142 struct dax_device_driver *dax_drv = to_dax_drv(drv); 143 144 /* 145 * All but the 'device-dax' driver, which has 'match_always' 146 * set, requires an exact id match. 147 */ 148 if (dax_drv->match_always) 149 return 1; 150 151 return dax_match_id(dax_drv, dev); 152 } 153 154 /* 155 * Rely on the fact that drvdata is set before the attributes are 156 * registered, and that the attributes are unregistered before drvdata 157 * is cleared to assume that drvdata is always valid. 158 */ 159 static ssize_t id_show(struct device *dev, 160 struct device_attribute *attr, char *buf) 161 { 162 struct dax_region *dax_region = dev_get_drvdata(dev); 163 164 return sprintf(buf, "%d\n", dax_region->id); 165 } 166 static DEVICE_ATTR_RO(id); 167 168 static ssize_t region_size_show(struct device *dev, 169 struct device_attribute *attr, char *buf) 170 { 171 struct dax_region *dax_region = dev_get_drvdata(dev); 172 173 return sprintf(buf, "%llu\n", (unsigned long long) 174 resource_size(&dax_region->res)); 175 } 176 static struct device_attribute dev_attr_region_size = __ATTR(size, 0444, 177 region_size_show, NULL); 178 179 static ssize_t align_show(struct device *dev, 180 struct device_attribute *attr, char *buf) 181 { 182 struct dax_region *dax_region = dev_get_drvdata(dev); 183 184 return sprintf(buf, "%u\n", dax_region->align); 185 } 186 static DEVICE_ATTR_RO(align); 187 188 static struct attribute *dax_region_attributes[] = { 189 &dev_attr_region_size.attr, 190 &dev_attr_align.attr, 191 &dev_attr_id.attr, 192 NULL, 193 }; 194 195 static const struct attribute_group dax_region_attribute_group = { 196 .name = "dax_region", 197 .attrs = dax_region_attributes, 198 }; 199 200 static const struct attribute_group *dax_region_attribute_groups[] = { 201 &dax_region_attribute_group, 202 NULL, 203 }; 204 205 static void dax_region_free(struct kref *kref) 206 { 207 struct dax_region *dax_region; 208 209 dax_region = container_of(kref, struct dax_region, kref); 210 kfree(dax_region); 211 } 212 213 void dax_region_put(struct dax_region *dax_region) 214 { 215 kref_put(&dax_region->kref, dax_region_free); 216 } 217 EXPORT_SYMBOL_GPL(dax_region_put); 218 219 static void dax_region_unregister(void *region) 220 { 221 struct dax_region *dax_region = region; 222 223 sysfs_remove_groups(&dax_region->dev->kobj, 224 dax_region_attribute_groups); 225 dax_region_put(dax_region); 226 } 227 228 struct dax_region *alloc_dax_region(struct device *parent, int region_id, 229 struct resource *res, int target_node, unsigned int align, 230 unsigned long pfn_flags) 231 { 232 struct dax_region *dax_region; 233 234 /* 235 * The DAX core assumes that it can store its private data in 236 * parent->driver_data. This WARN is a reminder / safeguard for 237 * developers of device-dax drivers. 238 */ 239 if (dev_get_drvdata(parent)) { 240 dev_WARN(parent, "dax core failed to setup private data\n"); 241 return NULL; 242 } 243 244 if (!IS_ALIGNED(res->start, align) 245 || !IS_ALIGNED(resource_size(res), align)) 246 return NULL; 247 248 dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL); 249 if (!dax_region) 250 return NULL; 251 252 dev_set_drvdata(parent, dax_region); 253 memcpy(&dax_region->res, res, sizeof(*res)); 254 dax_region->pfn_flags = pfn_flags; 255 kref_init(&dax_region->kref); 256 dax_region->id = region_id; 257 dax_region->align = align; 258 dax_region->dev = parent; 259 dax_region->target_node = target_node; 260 if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) { 261 kfree(dax_region); 262 return NULL; 263 } 264 265 kref_get(&dax_region->kref); 266 if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region)) 267 return NULL; 268 return dax_region; 269 } 270 EXPORT_SYMBOL_GPL(alloc_dax_region); 271 272 static ssize_t size_show(struct device *dev, 273 struct device_attribute *attr, char *buf) 274 { 275 struct dev_dax *dev_dax = to_dev_dax(dev); 276 unsigned long long size = resource_size(&dev_dax->region->res); 277 278 return sprintf(buf, "%llu\n", size); 279 } 280 static DEVICE_ATTR_RO(size); 281 282 static int dev_dax_target_node(struct dev_dax *dev_dax) 283 { 284 struct dax_region *dax_region = dev_dax->region; 285 286 return dax_region->target_node; 287 } 288 289 static ssize_t target_node_show(struct device *dev, 290 struct device_attribute *attr, char *buf) 291 { 292 struct dev_dax *dev_dax = to_dev_dax(dev); 293 294 return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax)); 295 } 296 static DEVICE_ATTR_RO(target_node); 297 298 static unsigned long long dev_dax_resource(struct dev_dax *dev_dax) 299 { 300 struct dax_region *dax_region = dev_dax->region; 301 302 return dax_region->res.start; 303 } 304 305 static ssize_t resource_show(struct device *dev, 306 struct device_attribute *attr, char *buf) 307 { 308 struct dev_dax *dev_dax = to_dev_dax(dev); 309 310 return sprintf(buf, "%#llx\n", dev_dax_resource(dev_dax)); 311 } 312 static DEVICE_ATTR_RO(resource); 313 314 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, 315 char *buf) 316 { 317 /* 318 * We only ever expect to handle device-dax instances, i.e. the 319 * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero 320 */ 321 return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0); 322 } 323 static DEVICE_ATTR_RO(modalias); 324 325 static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n) 326 { 327 struct device *dev = container_of(kobj, struct device, kobj); 328 struct dev_dax *dev_dax = to_dev_dax(dev); 329 330 if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0) 331 return 0; 332 if (a == &dev_attr_resource.attr) 333 return 0400; 334 return a->mode; 335 } 336 337 static struct attribute *dev_dax_attributes[] = { 338 &dev_attr_modalias.attr, 339 &dev_attr_size.attr, 340 &dev_attr_target_node.attr, 341 &dev_attr_resource.attr, 342 NULL, 343 }; 344 345 static const struct attribute_group dev_dax_attribute_group = { 346 .attrs = dev_dax_attributes, 347 .is_visible = dev_dax_visible, 348 }; 349 350 static const struct attribute_group *dax_attribute_groups[] = { 351 &dev_dax_attribute_group, 352 NULL, 353 }; 354 355 void kill_dev_dax(struct dev_dax *dev_dax) 356 { 357 struct dax_device *dax_dev = dev_dax->dax_dev; 358 struct inode *inode = dax_inode(dax_dev); 359 360 kill_dax(dax_dev); 361 unmap_mapping_range(inode->i_mapping, 0, 0, 1); 362 } 363 EXPORT_SYMBOL_GPL(kill_dev_dax); 364 365 static void dev_dax_release(struct device *dev) 366 { 367 struct dev_dax *dev_dax = to_dev_dax(dev); 368 struct dax_region *dax_region = dev_dax->region; 369 struct dax_device *dax_dev = dev_dax->dax_dev; 370 371 dax_region_put(dax_region); 372 put_dax(dax_dev); 373 kfree(dev_dax); 374 } 375 376 static void unregister_dev_dax(void *dev) 377 { 378 struct dev_dax *dev_dax = to_dev_dax(dev); 379 380 dev_dbg(dev, "%s\n", __func__); 381 382 kill_dev_dax(dev_dax); 383 device_del(dev); 384 put_device(dev); 385 } 386 387 struct dev_dax *__devm_create_dev_dax(struct dax_region *dax_region, int id, 388 struct dev_pagemap *pgmap, enum dev_dax_subsys subsys) 389 { 390 struct device *parent = dax_region->dev; 391 struct dax_device *dax_dev; 392 struct dev_dax *dev_dax; 393 struct inode *inode; 394 struct device *dev; 395 int rc = -ENOMEM; 396 397 if (id < 0) 398 return ERR_PTR(-EINVAL); 399 400 dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL); 401 if (!dev_dax) 402 return ERR_PTR(-ENOMEM); 403 404 memcpy(&dev_dax->pgmap, pgmap, sizeof(*pgmap)); 405 406 /* 407 * No 'host' or dax_operations since there is no access to this 408 * device outside of mmap of the resulting character device. 409 */ 410 dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC); 411 if (!dax_dev) 412 goto err; 413 414 /* a device_dax instance is dead while the driver is not attached */ 415 kill_dax(dax_dev); 416 417 /* from here on we're committed to teardown via dax_dev_release() */ 418 dev = &dev_dax->dev; 419 device_initialize(dev); 420 421 dev_dax->dax_dev = dax_dev; 422 dev_dax->region = dax_region; 423 dev_dax->target_node = dax_region->target_node; 424 kref_get(&dax_region->kref); 425 426 inode = dax_inode(dax_dev); 427 dev->devt = inode->i_rdev; 428 if (subsys == DEV_DAX_BUS) 429 dev->bus = &dax_bus_type; 430 else 431 dev->class = dax_class; 432 dev->parent = parent; 433 dev->groups = dax_attribute_groups; 434 dev->release = dev_dax_release; 435 dev_set_name(dev, "dax%d.%d", dax_region->id, id); 436 437 rc = device_add(dev); 438 if (rc) { 439 kill_dev_dax(dev_dax); 440 put_device(dev); 441 return ERR_PTR(rc); 442 } 443 444 rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev); 445 if (rc) 446 return ERR_PTR(rc); 447 448 return dev_dax; 449 450 err: 451 kfree(dev_dax); 452 453 return ERR_PTR(rc); 454 } 455 EXPORT_SYMBOL_GPL(__devm_create_dev_dax); 456 457 static int match_always_count; 458 459 int __dax_driver_register(struct dax_device_driver *dax_drv, 460 struct module *module, const char *mod_name) 461 { 462 struct device_driver *drv = &dax_drv->drv; 463 int rc = 0; 464 465 INIT_LIST_HEAD(&dax_drv->ids); 466 drv->owner = module; 467 drv->name = mod_name; 468 drv->mod_name = mod_name; 469 drv->bus = &dax_bus_type; 470 471 /* there can only be one default driver */ 472 mutex_lock(&dax_bus_lock); 473 match_always_count += dax_drv->match_always; 474 if (match_always_count > 1) { 475 match_always_count--; 476 WARN_ON(1); 477 rc = -EINVAL; 478 } 479 mutex_unlock(&dax_bus_lock); 480 if (rc) 481 return rc; 482 return driver_register(drv); 483 } 484 EXPORT_SYMBOL_GPL(__dax_driver_register); 485 486 void dax_driver_unregister(struct dax_device_driver *dax_drv) 487 { 488 struct device_driver *drv = &dax_drv->drv; 489 struct dax_id *dax_id, *_id; 490 491 mutex_lock(&dax_bus_lock); 492 match_always_count -= dax_drv->match_always; 493 list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) { 494 list_del(&dax_id->list); 495 kfree(dax_id); 496 } 497 mutex_unlock(&dax_bus_lock); 498 driver_unregister(drv); 499 } 500 EXPORT_SYMBOL_GPL(dax_driver_unregister); 501 502 int __init dax_bus_init(void) 503 { 504 int rc; 505 506 if (IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT)) { 507 dax_class = class_create(THIS_MODULE, "dax"); 508 if (IS_ERR(dax_class)) 509 return PTR_ERR(dax_class); 510 } 511 512 rc = bus_register(&dax_bus_type); 513 if (rc) 514 class_destroy(dax_class); 515 return rc; 516 } 517 518 void __exit dax_bus_exit(void) 519 { 520 bus_unregister(&dax_bus_type); 521 class_destroy(dax_class); 522 } 523