xref: /openbmc/linux/drivers/nvdimm/bus.c (revision 0edbfea5)
1 /*
2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of version 2 of the GNU General Public License as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful, but
9  * WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11  * General Public License for more details.
12  */
13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 #include <linux/vmalloc.h>
15 #include <linux/uaccess.h>
16 #include <linux/module.h>
17 #include <linux/blkdev.h>
18 #include <linux/fcntl.h>
19 #include <linux/async.h>
20 #include <linux/genhd.h>
21 #include <linux/ndctl.h>
22 #include <linux/sched.h>
23 #include <linux/slab.h>
24 #include <linux/fs.h>
25 #include <linux/io.h>
26 #include <linux/mm.h>
27 #include <linux/nd.h>
28 #include "nd-core.h"
29 #include "nd.h"
30 
31 int nvdimm_major;
32 static int nvdimm_bus_major;
33 static struct class *nd_class;
34 
35 static int to_nd_device_type(struct device *dev)
36 {
37 	if (is_nvdimm(dev))
38 		return ND_DEVICE_DIMM;
39 	else if (is_nd_pmem(dev))
40 		return ND_DEVICE_REGION_PMEM;
41 	else if (is_nd_blk(dev))
42 		return ND_DEVICE_REGION_BLK;
43 	else if (is_nd_dax(dev))
44 		return ND_DEVICE_DAX_PMEM;
45 	else if (is_nd_pmem(dev->parent) || is_nd_blk(dev->parent))
46 		return nd_region_to_nstype(to_nd_region(dev->parent));
47 
48 	return 0;
49 }
50 
51 static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
52 {
53 	/*
54 	 * Ensure that region devices always have their numa node set as
55 	 * early as possible.
56 	 */
57 	if (is_nd_pmem(dev) || is_nd_blk(dev))
58 		set_dev_node(dev, to_nd_region(dev)->numa_node);
59 	return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
60 			to_nd_device_type(dev));
61 }
62 
63 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
64 {
65 	struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
66 
67 	return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
68 }
69 
70 static struct module *to_bus_provider(struct device *dev)
71 {
72 	/* pin bus providers while regions are enabled */
73 	if (is_nd_pmem(dev) || is_nd_blk(dev)) {
74 		struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
75 
76 		return nvdimm_bus->module;
77 	}
78 	return NULL;
79 }
80 
81 static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
82 {
83 	nvdimm_bus_lock(&nvdimm_bus->dev);
84 	nvdimm_bus->probe_active++;
85 	nvdimm_bus_unlock(&nvdimm_bus->dev);
86 }
87 
88 static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
89 {
90 	nvdimm_bus_lock(&nvdimm_bus->dev);
91 	if (--nvdimm_bus->probe_active == 0)
92 		wake_up(&nvdimm_bus->probe_wait);
93 	nvdimm_bus_unlock(&nvdimm_bus->dev);
94 }
95 
96 static int nvdimm_bus_probe(struct device *dev)
97 {
98 	struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
99 	struct module *provider = to_bus_provider(dev);
100 	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
101 	int rc;
102 
103 	if (!try_module_get(provider))
104 		return -ENXIO;
105 
106 	nvdimm_bus_probe_start(nvdimm_bus);
107 	rc = nd_drv->probe(dev);
108 	if (rc == 0)
109 		nd_region_probe_success(nvdimm_bus, dev);
110 	else
111 		nd_region_disable(nvdimm_bus, dev);
112 	nvdimm_bus_probe_end(nvdimm_bus);
113 
114 	dev_dbg(&nvdimm_bus->dev, "%s.probe(%s) = %d\n", dev->driver->name,
115 			dev_name(dev), rc);
116 
117 	if (rc != 0)
118 		module_put(provider);
119 	return rc;
120 }
121 
122 static int nvdimm_bus_remove(struct device *dev)
123 {
124 	struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
125 	struct module *provider = to_bus_provider(dev);
126 	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
127 	int rc = 0;
128 
129 	if (nd_drv->remove)
130 		rc = nd_drv->remove(dev);
131 	nd_region_disable(nvdimm_bus, dev);
132 
133 	dev_dbg(&nvdimm_bus->dev, "%s.remove(%s) = %d\n", dev->driver->name,
134 			dev_name(dev), rc);
135 	module_put(provider);
136 	return rc;
137 }
138 
139 void nd_device_notify(struct device *dev, enum nvdimm_event event)
140 {
141 	device_lock(dev);
142 	if (dev->driver) {
143 		struct nd_device_driver *nd_drv;
144 
145 		nd_drv = to_nd_device_driver(dev->driver);
146 		if (nd_drv->notify)
147 			nd_drv->notify(dev, event);
148 	}
149 	device_unlock(dev);
150 }
151 EXPORT_SYMBOL(nd_device_notify);
152 
153 void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
154 {
155 	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
156 
157 	if (!nvdimm_bus)
158 		return;
159 
160 	/* caller is responsible for holding a reference on the device */
161 	nd_device_notify(&nd_region->dev, event);
162 }
163 EXPORT_SYMBOL_GPL(nvdimm_region_notify);
164 
165 long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
166 		unsigned int len)
167 {
168 	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
169 	struct nvdimm_bus_descriptor *nd_desc;
170 	struct nd_cmd_clear_error clear_err;
171 	struct nd_cmd_ars_cap ars_cap;
172 	u32 clear_err_unit, mask;
173 	int cmd_rc, rc;
174 
175 	if (!nvdimm_bus)
176 		return -ENXIO;
177 
178 	nd_desc = nvdimm_bus->nd_desc;
179 	if (!nd_desc->ndctl)
180 		return -ENXIO;
181 
182 	memset(&ars_cap, 0, sizeof(ars_cap));
183 	ars_cap.address = phys;
184 	ars_cap.length = len;
185 	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
186 			sizeof(ars_cap), &cmd_rc);
187 	if (rc < 0)
188 		return rc;
189 	if (cmd_rc < 0)
190 		return cmd_rc;
191 	clear_err_unit = ars_cap.clear_err_unit;
192 	if (!clear_err_unit || !is_power_of_2(clear_err_unit))
193 		return -ENXIO;
194 
195 	mask = clear_err_unit - 1;
196 	if ((phys | len) & mask)
197 		return -ENXIO;
198 	memset(&clear_err, 0, sizeof(clear_err));
199 	clear_err.address = phys;
200 	clear_err.length = len;
201 	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
202 			sizeof(clear_err), &cmd_rc);
203 	if (rc < 0)
204 		return rc;
205 	if (cmd_rc < 0)
206 		return cmd_rc;
207 	return clear_err.cleared;
208 }
209 EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
210 
211 static struct bus_type nvdimm_bus_type = {
212 	.name = "nd",
213 	.uevent = nvdimm_bus_uevent,
214 	.match = nvdimm_bus_match,
215 	.probe = nvdimm_bus_probe,
216 	.remove = nvdimm_bus_remove,
217 };
218 
219 static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
220 
221 void nd_synchronize(void)
222 {
223 	async_synchronize_full_domain(&nd_async_domain);
224 }
225 EXPORT_SYMBOL_GPL(nd_synchronize);
226 
227 static void nd_async_device_register(void *d, async_cookie_t cookie)
228 {
229 	struct device *dev = d;
230 
231 	if (device_add(dev) != 0) {
232 		dev_err(dev, "%s: failed\n", __func__);
233 		put_device(dev);
234 	}
235 	put_device(dev);
236 }
237 
238 static void nd_async_device_unregister(void *d, async_cookie_t cookie)
239 {
240 	struct device *dev = d;
241 
242 	/* flush bus operations before delete */
243 	nvdimm_bus_lock(dev);
244 	nvdimm_bus_unlock(dev);
245 
246 	device_unregister(dev);
247 	put_device(dev);
248 }
249 
250 void __nd_device_register(struct device *dev)
251 {
252 	if (!dev)
253 		return;
254 	dev->bus = &nvdimm_bus_type;
255 	get_device(dev);
256 	async_schedule_domain(nd_async_device_register, dev,
257 			&nd_async_domain);
258 }
259 
260 void nd_device_register(struct device *dev)
261 {
262 	device_initialize(dev);
263 	__nd_device_register(dev);
264 }
265 EXPORT_SYMBOL(nd_device_register);
266 
267 void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
268 {
269 	switch (mode) {
270 	case ND_ASYNC:
271 		get_device(dev);
272 		async_schedule_domain(nd_async_device_unregister, dev,
273 				&nd_async_domain);
274 		break;
275 	case ND_SYNC:
276 		nd_synchronize();
277 		device_unregister(dev);
278 		break;
279 	}
280 }
281 EXPORT_SYMBOL(nd_device_unregister);
282 
283 /**
284  * __nd_driver_register() - register a region or a namespace driver
285  * @nd_drv: driver to register
286  * @owner: automatically set by nd_driver_register() macro
287  * @mod_name: automatically set by nd_driver_register() macro
288  */
289 int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
290 		const char *mod_name)
291 {
292 	struct device_driver *drv = &nd_drv->drv;
293 
294 	if (!nd_drv->type) {
295 		pr_debug("driver type bitmask not set (%pf)\n",
296 				__builtin_return_address(0));
297 		return -EINVAL;
298 	}
299 
300 	if (!nd_drv->probe) {
301 		pr_debug("%s ->probe() must be specified\n", mod_name);
302 		return -EINVAL;
303 	}
304 
305 	drv->bus = &nvdimm_bus_type;
306 	drv->owner = owner;
307 	drv->mod_name = mod_name;
308 
309 	return driver_register(drv);
310 }
311 EXPORT_SYMBOL(__nd_driver_register);
312 
313 int nvdimm_revalidate_disk(struct gendisk *disk)
314 {
315 	struct device *dev = disk->driverfs_dev;
316 	struct nd_region *nd_region = to_nd_region(dev->parent);
317 	const char *pol = nd_region->ro ? "only" : "write";
318 
319 	if (nd_region->ro == get_disk_ro(disk))
320 		return 0;
321 
322 	dev_info(dev, "%s read-%s, marking %s read-%s\n",
323 			dev_name(&nd_region->dev), pol, disk->disk_name, pol);
324 	set_disk_ro(disk, nd_region->ro);
325 
326 	return 0;
327 
328 }
329 EXPORT_SYMBOL(nvdimm_revalidate_disk);
330 
331 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
332 		char *buf)
333 {
334 	return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
335 			to_nd_device_type(dev));
336 }
337 static DEVICE_ATTR_RO(modalias);
338 
339 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
340 		char *buf)
341 {
342 	return sprintf(buf, "%s\n", dev->type->name);
343 }
344 static DEVICE_ATTR_RO(devtype);
345 
346 static struct attribute *nd_device_attributes[] = {
347 	&dev_attr_modalias.attr,
348 	&dev_attr_devtype.attr,
349 	NULL,
350 };
351 
352 /**
353  * nd_device_attribute_group - generic attributes for all devices on an nd bus
354  */
355 struct attribute_group nd_device_attribute_group = {
356 	.attrs = nd_device_attributes,
357 };
358 EXPORT_SYMBOL_GPL(nd_device_attribute_group);
359 
360 static ssize_t numa_node_show(struct device *dev,
361 		struct device_attribute *attr, char *buf)
362 {
363 	return sprintf(buf, "%d\n", dev_to_node(dev));
364 }
365 static DEVICE_ATTR_RO(numa_node);
366 
367 static struct attribute *nd_numa_attributes[] = {
368 	&dev_attr_numa_node.attr,
369 	NULL,
370 };
371 
372 static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
373 		int n)
374 {
375 	if (!IS_ENABLED(CONFIG_NUMA))
376 		return 0;
377 
378 	return a->mode;
379 }
380 
381 /**
382  * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
383  */
384 struct attribute_group nd_numa_attribute_group = {
385 	.attrs = nd_numa_attributes,
386 	.is_visible = nd_numa_attr_visible,
387 };
388 EXPORT_SYMBOL_GPL(nd_numa_attribute_group);
389 
390 int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
391 {
392 	dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
393 	struct device *dev;
394 
395 	dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus,
396 			"ndctl%d", nvdimm_bus->id);
397 
398 	if (IS_ERR(dev)) {
399 		dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n",
400 				nvdimm_bus->id, PTR_ERR(dev));
401 		return PTR_ERR(dev);
402 	}
403 	return 0;
404 }
405 
406 void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
407 {
408 	device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
409 }
410 
411 static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
412 	[ND_CMD_IMPLEMENTED] = { },
413 	[ND_CMD_SMART] = {
414 		.out_num = 2,
415 		.out_sizes = { 4, 128, },
416 	},
417 	[ND_CMD_SMART_THRESHOLD] = {
418 		.out_num = 2,
419 		.out_sizes = { 4, 8, },
420 	},
421 	[ND_CMD_DIMM_FLAGS] = {
422 		.out_num = 2,
423 		.out_sizes = { 4, 4 },
424 	},
425 	[ND_CMD_GET_CONFIG_SIZE] = {
426 		.out_num = 3,
427 		.out_sizes = { 4, 4, 4, },
428 	},
429 	[ND_CMD_GET_CONFIG_DATA] = {
430 		.in_num = 2,
431 		.in_sizes = { 4, 4, },
432 		.out_num = 2,
433 		.out_sizes = { 4, UINT_MAX, },
434 	},
435 	[ND_CMD_SET_CONFIG_DATA] = {
436 		.in_num = 3,
437 		.in_sizes = { 4, 4, UINT_MAX, },
438 		.out_num = 1,
439 		.out_sizes = { 4, },
440 	},
441 	[ND_CMD_VENDOR] = {
442 		.in_num = 3,
443 		.in_sizes = { 4, 4, UINT_MAX, },
444 		.out_num = 3,
445 		.out_sizes = { 4, 4, UINT_MAX, },
446 	},
447 	[ND_CMD_CALL] = {
448 		.in_num = 2,
449 		.in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
450 		.out_num = 1,
451 		.out_sizes = { UINT_MAX, },
452 	},
453 };
454 
455 const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
456 {
457 	if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
458 		return &__nd_cmd_dimm_descs[cmd];
459 	return NULL;
460 }
461 EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
462 
463 static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
464 	[ND_CMD_IMPLEMENTED] = { },
465 	[ND_CMD_ARS_CAP] = {
466 		.in_num = 2,
467 		.in_sizes = { 8, 8, },
468 		.out_num = 4,
469 		.out_sizes = { 4, 4, 4, 4, },
470 	},
471 	[ND_CMD_ARS_START] = {
472 		.in_num = 5,
473 		.in_sizes = { 8, 8, 2, 1, 5, },
474 		.out_num = 2,
475 		.out_sizes = { 4, 4, },
476 	},
477 	[ND_CMD_ARS_STATUS] = {
478 		.out_num = 3,
479 		.out_sizes = { 4, 4, UINT_MAX, },
480 	},
481 	[ND_CMD_CLEAR_ERROR] = {
482 		.in_num = 2,
483 		.in_sizes = { 8, 8, },
484 		.out_num = 3,
485 		.out_sizes = { 4, 4, 8, },
486 	},
487 	[ND_CMD_CALL] = {
488 		.in_num = 2,
489 		.in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
490 		.out_num = 1,
491 		.out_sizes = { UINT_MAX, },
492 	},
493 };
494 
495 const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
496 {
497 	if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
498 		return &__nd_cmd_bus_descs[cmd];
499 	return NULL;
500 }
501 EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
502 
503 u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
504 		const struct nd_cmd_desc *desc, int idx, void *buf)
505 {
506 	if (idx >= desc->in_num)
507 		return UINT_MAX;
508 
509 	if (desc->in_sizes[idx] < UINT_MAX)
510 		return desc->in_sizes[idx];
511 
512 	if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
513 		struct nd_cmd_set_config_hdr *hdr = buf;
514 
515 		return hdr->in_length;
516 	} else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
517 		struct nd_cmd_vendor_hdr *hdr = buf;
518 
519 		return hdr->in_length;
520 	} else if (cmd == ND_CMD_CALL) {
521 		struct nd_cmd_pkg *pkg = buf;
522 
523 		return pkg->nd_size_in;
524 	}
525 
526 	return UINT_MAX;
527 }
528 EXPORT_SYMBOL_GPL(nd_cmd_in_size);
529 
530 u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
531 		const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
532 		const u32 *out_field)
533 {
534 	if (idx >= desc->out_num)
535 		return UINT_MAX;
536 
537 	if (desc->out_sizes[idx] < UINT_MAX)
538 		return desc->out_sizes[idx];
539 
540 	if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
541 		return in_field[1];
542 	else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
543 		return out_field[1];
544 	else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2)
545 		return out_field[1] - 8;
546 	else if (cmd == ND_CMD_CALL) {
547 		struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
548 
549 		return pkg->nd_size_out;
550 	}
551 
552 
553 	return UINT_MAX;
554 }
555 EXPORT_SYMBOL_GPL(nd_cmd_out_size);
556 
557 void wait_nvdimm_bus_probe_idle(struct device *dev)
558 {
559 	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
560 
561 	do {
562 		if (nvdimm_bus->probe_active == 0)
563 			break;
564 		nvdimm_bus_unlock(&nvdimm_bus->dev);
565 		wait_event(nvdimm_bus->probe_wait,
566 				nvdimm_bus->probe_active == 0);
567 		nvdimm_bus_lock(&nvdimm_bus->dev);
568 	} while (true);
569 }
570 
571 static int pmem_active(struct device *dev, void *data)
572 {
573 	if (is_nd_pmem(dev) && dev->driver)
574 		return -EBUSY;
575 	return 0;
576 }
577 
578 /* set_config requires an idle interleave set */
579 static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
580 		struct nvdimm *nvdimm, unsigned int cmd)
581 {
582 	struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
583 
584 	/* ask the bus provider if it would like to block this request */
585 	if (nd_desc->clear_to_send) {
586 		int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd);
587 
588 		if (rc)
589 			return rc;
590 	}
591 
592 	/* require clear error to go through the pmem driver */
593 	if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
594 		return device_for_each_child(&nvdimm_bus->dev, NULL,
595 				pmem_active);
596 
597 	if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
598 		return 0;
599 
600 	/* prevent label manipulation while the kernel owns label updates */
601 	wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
602 	if (atomic_read(&nvdimm->busy))
603 		return -EBUSY;
604 	return 0;
605 }
606 
607 static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
608 		int read_only, unsigned int ioctl_cmd, unsigned long arg)
609 {
610 	struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
611 	size_t buf_len = 0, in_len = 0, out_len = 0;
612 	static char out_env[ND_CMD_MAX_ENVELOPE];
613 	static char in_env[ND_CMD_MAX_ENVELOPE];
614 	const struct nd_cmd_desc *desc = NULL;
615 	unsigned int cmd = _IOC_NR(ioctl_cmd);
616 	void __user *p = (void __user *) arg;
617 	struct device *dev = &nvdimm_bus->dev;
618 	struct nd_cmd_pkg pkg;
619 	const char *cmd_name, *dimm_name;
620 	unsigned long cmd_mask;
621 	void *buf;
622 	int rc, i;
623 
624 	if (nvdimm) {
625 		desc = nd_cmd_dimm_desc(cmd);
626 		cmd_name = nvdimm_cmd_name(cmd);
627 		cmd_mask = nvdimm->cmd_mask;
628 		dimm_name = dev_name(&nvdimm->dev);
629 	} else {
630 		desc = nd_cmd_bus_desc(cmd);
631 		cmd_name = nvdimm_bus_cmd_name(cmd);
632 		cmd_mask = nd_desc->cmd_mask;
633 		dimm_name = "bus";
634 	}
635 
636 	if (cmd == ND_CMD_CALL) {
637 		if (copy_from_user(&pkg, p, sizeof(pkg)))
638 			return -EFAULT;
639 	}
640 
641 	if (!desc || (desc->out_num + desc->in_num == 0) ||
642 			!test_bit(cmd, &cmd_mask))
643 		return -ENOTTY;
644 
645 	/* fail write commands (when read-only) */
646 	if (read_only)
647 		switch (cmd) {
648 		case ND_CMD_VENDOR:
649 		case ND_CMD_SET_CONFIG_DATA:
650 		case ND_CMD_ARS_START:
651 		case ND_CMD_CLEAR_ERROR:
652 		case ND_CMD_CALL:
653 			dev_dbg(&nvdimm_bus->dev, "'%s' command while read-only.\n",
654 					nvdimm ? nvdimm_cmd_name(cmd)
655 					: nvdimm_bus_cmd_name(cmd));
656 			return -EPERM;
657 		default:
658 			break;
659 		}
660 
661 	/* process an input envelope */
662 	for (i = 0; i < desc->in_num; i++) {
663 		u32 in_size, copy;
664 
665 		in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
666 		if (in_size == UINT_MAX) {
667 			dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
668 					__func__, dimm_name, cmd_name, i);
669 			return -ENXIO;
670 		}
671 		if (in_len < sizeof(in_env))
672 			copy = min_t(u32, sizeof(in_env) - in_len, in_size);
673 		else
674 			copy = 0;
675 		if (copy && copy_from_user(&in_env[in_len], p + in_len, copy))
676 			return -EFAULT;
677 		in_len += in_size;
678 	}
679 
680 	if (cmd == ND_CMD_CALL) {
681 		dev_dbg(dev, "%s:%s, idx: %llu, in: %zu, out: %zu, len %zu\n",
682 				__func__, dimm_name, pkg.nd_command,
683 				in_len, out_len, buf_len);
684 
685 		for (i = 0; i < ARRAY_SIZE(pkg.nd_reserved2); i++)
686 			if (pkg.nd_reserved2[i])
687 				return -EINVAL;
688 	}
689 
690 	/* process an output envelope */
691 	for (i = 0; i < desc->out_num; i++) {
692 		u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
693 				(u32 *) in_env, (u32 *) out_env);
694 		u32 copy;
695 
696 		if (out_size == UINT_MAX) {
697 			dev_dbg(dev, "%s:%s unknown output size cmd: %s field: %d\n",
698 					__func__, dimm_name, cmd_name, i);
699 			return -EFAULT;
700 		}
701 		if (out_len < sizeof(out_env))
702 			copy = min_t(u32, sizeof(out_env) - out_len, out_size);
703 		else
704 			copy = 0;
705 		if (copy && copy_from_user(&out_env[out_len],
706 					p + in_len + out_len, copy))
707 			return -EFAULT;
708 		out_len += out_size;
709 	}
710 
711 	buf_len = out_len + in_len;
712 	if (buf_len > ND_IOCTL_MAX_BUFLEN) {
713 		dev_dbg(dev, "%s:%s cmd: %s buf_len: %zu > %d\n", __func__,
714 				dimm_name, cmd_name, buf_len,
715 				ND_IOCTL_MAX_BUFLEN);
716 		return -EINVAL;
717 	}
718 
719 	buf = vmalloc(buf_len);
720 	if (!buf)
721 		return -ENOMEM;
722 
723 	if (copy_from_user(buf, p, buf_len)) {
724 		rc = -EFAULT;
725 		goto out;
726 	}
727 
728 	nvdimm_bus_lock(&nvdimm_bus->dev);
729 	rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, cmd);
730 	if (rc)
731 		goto out_unlock;
732 
733 	rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, NULL);
734 	if (rc < 0)
735 		goto out_unlock;
736 	if (copy_to_user(p, buf, buf_len))
737 		rc = -EFAULT;
738  out_unlock:
739 	nvdimm_bus_unlock(&nvdimm_bus->dev);
740  out:
741 	vfree(buf);
742 	return rc;
743 }
744 
745 static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
746 {
747 	long id = (long) file->private_data;
748 	int rc = -ENXIO, ro;
749 	struct nvdimm_bus *nvdimm_bus;
750 
751 	ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
752 	mutex_lock(&nvdimm_bus_list_mutex);
753 	list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
754 		if (nvdimm_bus->id == id) {
755 			rc = __nd_ioctl(nvdimm_bus, NULL, ro, cmd, arg);
756 			break;
757 		}
758 	}
759 	mutex_unlock(&nvdimm_bus_list_mutex);
760 
761 	return rc;
762 }
763 
764 static int match_dimm(struct device *dev, void *data)
765 {
766 	long id = (long) data;
767 
768 	if (is_nvdimm(dev)) {
769 		struct nvdimm *nvdimm = to_nvdimm(dev);
770 
771 		return nvdimm->id == id;
772 	}
773 
774 	return 0;
775 }
776 
777 static long nvdimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
778 {
779 	int rc = -ENXIO, ro;
780 	struct nvdimm_bus *nvdimm_bus;
781 
782 	ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
783 	mutex_lock(&nvdimm_bus_list_mutex);
784 	list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
785 		struct device *dev = device_find_child(&nvdimm_bus->dev,
786 				file->private_data, match_dimm);
787 		struct nvdimm *nvdimm;
788 
789 		if (!dev)
790 			continue;
791 
792 		nvdimm = to_nvdimm(dev);
793 		rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
794 		put_device(dev);
795 		break;
796 	}
797 	mutex_unlock(&nvdimm_bus_list_mutex);
798 
799 	return rc;
800 }
801 
802 static int nd_open(struct inode *inode, struct file *file)
803 {
804 	long minor = iminor(inode);
805 
806 	file->private_data = (void *) minor;
807 	return 0;
808 }
809 
810 static const struct file_operations nvdimm_bus_fops = {
811 	.owner = THIS_MODULE,
812 	.open = nd_open,
813 	.unlocked_ioctl = nd_ioctl,
814 	.compat_ioctl = nd_ioctl,
815 	.llseek = noop_llseek,
816 };
817 
818 static const struct file_operations nvdimm_fops = {
819 	.owner = THIS_MODULE,
820 	.open = nd_open,
821 	.unlocked_ioctl = nvdimm_ioctl,
822 	.compat_ioctl = nvdimm_ioctl,
823 	.llseek = noop_llseek,
824 };
825 
826 int __init nvdimm_bus_init(void)
827 {
828 	int rc;
829 
830 	BUILD_BUG_ON(sizeof(struct nd_smart_payload) != 128);
831 	BUILD_BUG_ON(sizeof(struct nd_smart_threshold_payload) != 8);
832 
833 	rc = bus_register(&nvdimm_bus_type);
834 	if (rc)
835 		return rc;
836 
837 	rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
838 	if (rc < 0)
839 		goto err_bus_chrdev;
840 	nvdimm_bus_major = rc;
841 
842 	rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
843 	if (rc < 0)
844 		goto err_dimm_chrdev;
845 	nvdimm_major = rc;
846 
847 	nd_class = class_create(THIS_MODULE, "nd");
848 	if (IS_ERR(nd_class)) {
849 		rc = PTR_ERR(nd_class);
850 		goto err_class;
851 	}
852 
853 	return 0;
854 
855  err_class:
856 	unregister_chrdev(nvdimm_major, "dimmctl");
857  err_dimm_chrdev:
858 	unregister_chrdev(nvdimm_bus_major, "ndctl");
859  err_bus_chrdev:
860 	bus_unregister(&nvdimm_bus_type);
861 
862 	return rc;
863 }
864 
865 void nvdimm_bus_exit(void)
866 {
867 	class_destroy(nd_class);
868 	unregister_chrdev(nvdimm_bus_major, "ndctl");
869 	unregister_chrdev(nvdimm_major, "dimmctl");
870 	bus_unregister(&nvdimm_bus_type);
871 }
872