1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * RDMA Network Block Driver
4  *
5  * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
6  * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
7  * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
8  */
9 
10 #undef pr_fmt
11 #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
12 
13 #include <linux/types.h>
14 #include <linux/ctype.h>
15 #include <linux/parser.h>
16 #include <linux/module.h>
17 #include <linux/in6.h>
18 #include <linux/fs.h>
19 #include <linux/uaccess.h>
20 #include <linux/device.h>
21 #include <rdma/ib.h>
22 #include <rdma/rdma_cm.h>
23 
24 #include "rnbd-clt.h"
25 
26 static struct device *rnbd_dev;
27 static struct class *rnbd_dev_class;
28 static struct kobject *rnbd_devs_kobj;
29 
30 enum {
31 	RNBD_OPT_ERR		= 0,
32 	RNBD_OPT_DEST_PORT	= 1 << 0,
33 	RNBD_OPT_PATH		= 1 << 1,
34 	RNBD_OPT_DEV_PATH	= 1 << 2,
35 	RNBD_OPT_ACCESS_MODE	= 1 << 3,
36 	RNBD_OPT_SESSNAME	= 1 << 6,
37 	RNBD_OPT_NR_POLL_QUEUES	= 1 << 7,
38 };
39 
40 static const unsigned int rnbd_opt_mandatory[] = {
41 	RNBD_OPT_DEV_PATH,
42 	RNBD_OPT_SESSNAME,
43 };
44 
45 static const match_table_t rnbd_opt_tokens = {
46 	{RNBD_OPT_PATH,			"path=%s"		},
47 	{RNBD_OPT_DEV_PATH,		"device_path=%s"	},
48 	{RNBD_OPT_DEST_PORT,		"dest_port=%d"		},
49 	{RNBD_OPT_ACCESS_MODE,		"access_mode=%s"	},
50 	{RNBD_OPT_SESSNAME,		"sessname=%s"		},
51 	{RNBD_OPT_NR_POLL_QUEUES,	"nr_poll_queues=%d"	},
52 	{RNBD_OPT_ERR,			NULL			},
53 };
54 
55 struct rnbd_map_options {
56 	char *sessname;
57 	struct rtrs_addr *paths;
58 	size_t *path_cnt;
59 	char *pathname;
60 	u16 *dest_port;
61 	enum rnbd_access_mode *access_mode;
62 	u32 *nr_poll_queues;
63 };
64 
65 static int rnbd_clt_parse_map_options(const char *buf, size_t max_path_cnt,
66 				       struct rnbd_map_options *opt)
67 {
68 	char *options, *sep_opt;
69 	char *p;
70 	substring_t args[MAX_OPT_ARGS];
71 	int opt_mask = 0;
72 	int token;
73 	int ret = -EINVAL;
74 	int i, dest_port, nr_poll_queues;
75 	int p_cnt = 0;
76 
77 	options = kstrdup(buf, GFP_KERNEL);
78 	if (!options)
79 		return -ENOMEM;
80 
81 	sep_opt = strstrip(options);
82 	while ((p = strsep(&sep_opt, " ")) != NULL) {
83 		if (!*p)
84 			continue;
85 
86 		token = match_token(p, rnbd_opt_tokens, args);
87 		opt_mask |= token;
88 
89 		switch (token) {
90 		case RNBD_OPT_SESSNAME:
91 			p = match_strdup(args);
92 			if (!p) {
93 				ret = -ENOMEM;
94 				goto out;
95 			}
96 			if (strlen(p) > NAME_MAX) {
97 				pr_err("map_device: sessname too long\n");
98 				ret = -EINVAL;
99 				kfree(p);
100 				goto out;
101 			}
102 			strscpy(opt->sessname, p, NAME_MAX);
103 			kfree(p);
104 			break;
105 
106 		case RNBD_OPT_PATH:
107 			if (p_cnt >= max_path_cnt) {
108 				pr_err("map_device: too many (> %zu) paths provided\n",
109 				       max_path_cnt);
110 				ret = -ENOMEM;
111 				goto out;
112 			}
113 			p = match_strdup(args);
114 			if (!p) {
115 				ret = -ENOMEM;
116 				goto out;
117 			}
118 
119 			ret = rtrs_addr_to_sockaddr(p, strlen(p),
120 						    *opt->dest_port,
121 						    &opt->paths[p_cnt]);
122 			if (ret) {
123 				pr_err("Can't parse path %s: %d\n", p, ret);
124 				kfree(p);
125 				goto out;
126 			}
127 
128 			p_cnt++;
129 
130 			kfree(p);
131 			break;
132 
133 		case RNBD_OPT_DEV_PATH:
134 			p = match_strdup(args);
135 			if (!p) {
136 				ret = -ENOMEM;
137 				goto out;
138 			}
139 			if (strlen(p) > NAME_MAX) {
140 				pr_err("map_device: Device path too long\n");
141 				ret = -EINVAL;
142 				kfree(p);
143 				goto out;
144 			}
145 			strscpy(opt->pathname, p, NAME_MAX);
146 			kfree(p);
147 			break;
148 
149 		case RNBD_OPT_DEST_PORT:
150 			if (match_int(args, &dest_port) || dest_port < 0 ||
151 			    dest_port > 65535) {
152 				pr_err("bad destination port number parameter '%d'\n",
153 				       dest_port);
154 				ret = -EINVAL;
155 				goto out;
156 			}
157 			*opt->dest_port = dest_port;
158 			break;
159 
160 		case RNBD_OPT_ACCESS_MODE:
161 			p = match_strdup(args);
162 			if (!p) {
163 				ret = -ENOMEM;
164 				goto out;
165 			}
166 
167 			if (!strcmp(p, "ro")) {
168 				*opt->access_mode = RNBD_ACCESS_RO;
169 			} else if (!strcmp(p, "rw")) {
170 				*opt->access_mode = RNBD_ACCESS_RW;
171 			} else if (!strcmp(p, "migration")) {
172 				*opt->access_mode = RNBD_ACCESS_MIGRATION;
173 			} else {
174 				pr_err("map_device: Invalid access_mode: '%s'\n",
175 				       p);
176 				ret = -EINVAL;
177 				kfree(p);
178 				goto out;
179 			}
180 
181 			kfree(p);
182 			break;
183 
184 		case RNBD_OPT_NR_POLL_QUEUES:
185 			if (match_int(args, &nr_poll_queues) || nr_poll_queues < -1 ||
186 			    nr_poll_queues > (int)nr_cpu_ids) {
187 				pr_err("bad nr_poll_queues parameter '%d'\n",
188 				       nr_poll_queues);
189 				ret = -EINVAL;
190 				goto out;
191 			}
192 			if (nr_poll_queues == -1)
193 				nr_poll_queues = nr_cpu_ids;
194 			*opt->nr_poll_queues = nr_poll_queues;
195 			break;
196 
197 		default:
198 			pr_err("map_device: Unknown parameter or missing value '%s'\n",
199 			       p);
200 			ret = -EINVAL;
201 			goto out;
202 		}
203 	}
204 
205 	for (i = 0; i < ARRAY_SIZE(rnbd_opt_mandatory); i++) {
206 		if ((opt_mask & rnbd_opt_mandatory[i])) {
207 			ret = 0;
208 		} else {
209 			pr_err("map_device: Parameters missing\n");
210 			ret = -EINVAL;
211 			break;
212 		}
213 	}
214 
215 out:
216 	*opt->path_cnt = p_cnt;
217 	kfree(options);
218 	return ret;
219 }
220 
221 static ssize_t state_show(struct kobject *kobj,
222 			  struct kobj_attribute *attr, char *page)
223 {
224 	struct rnbd_clt_dev *dev;
225 
226 	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
227 
228 	switch (dev->dev_state) {
229 	case DEV_STATE_INIT:
230 		return snprintf(page, PAGE_SIZE, "init\n");
231 	case DEV_STATE_MAPPED:
232 		/* TODO fix cli tool before changing to proper state */
233 		return snprintf(page, PAGE_SIZE, "open\n");
234 	case DEV_STATE_MAPPED_DISCONNECTED:
235 		/* TODO fix cli tool before changing to proper state */
236 		return snprintf(page, PAGE_SIZE, "closed\n");
237 	case DEV_STATE_UNMAPPED:
238 		return snprintf(page, PAGE_SIZE, "unmapped\n");
239 	default:
240 		return snprintf(page, PAGE_SIZE, "unknown\n");
241 	}
242 }
243 
244 static struct kobj_attribute rnbd_clt_state_attr = __ATTR_RO(state);
245 
246 static ssize_t nr_poll_queues_show(struct kobject *kobj,
247 				   struct kobj_attribute *attr, char *page)
248 {
249 	struct rnbd_clt_dev *dev;
250 
251 	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
252 
253 	return sysfs_emit(page, "%d\n", dev->nr_poll_queues);
254 }
255 
256 static struct kobj_attribute rnbd_clt_nr_poll_queues =
257 	__ATTR_RO(nr_poll_queues);
258 
259 static ssize_t mapping_path_show(struct kobject *kobj,
260 				 struct kobj_attribute *attr, char *page)
261 {
262 	struct rnbd_clt_dev *dev;
263 
264 	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
265 
266 	return scnprintf(page, PAGE_SIZE, "%s\n", dev->pathname);
267 }
268 
269 static struct kobj_attribute rnbd_clt_mapping_path_attr =
270 	__ATTR_RO(mapping_path);
271 
272 static ssize_t access_mode_show(struct kobject *kobj,
273 				struct kobj_attribute *attr, char *page)
274 {
275 	struct rnbd_clt_dev *dev;
276 
277 	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
278 
279 	return snprintf(page, PAGE_SIZE, "%s\n",
280 			rnbd_access_mode_str(dev->access_mode));
281 }
282 
283 static struct kobj_attribute rnbd_clt_access_mode =
284 	__ATTR_RO(access_mode);
285 
286 static ssize_t rnbd_clt_unmap_dev_show(struct kobject *kobj,
287 					struct kobj_attribute *attr, char *page)
288 {
289 	return scnprintf(page, PAGE_SIZE, "Usage: echo <normal|force> > %s\n",
290 			 attr->attr.name);
291 }
292 
293 static ssize_t rnbd_clt_unmap_dev_store(struct kobject *kobj,
294 					 struct kobj_attribute *attr,
295 					 const char *buf, size_t count)
296 {
297 	struct rnbd_clt_dev *dev;
298 	char *opt, *options;
299 	bool force;
300 	int err;
301 
302 	opt = kstrdup(buf, GFP_KERNEL);
303 	if (!opt)
304 		return -ENOMEM;
305 
306 	options = strstrip(opt);
307 	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
308 	if (sysfs_streq(options, "normal")) {
309 		force = false;
310 	} else if (sysfs_streq(options, "force")) {
311 		force = true;
312 	} else {
313 		rnbd_clt_err(dev,
314 			      "unmap_device: Invalid value: %s\n",
315 			      options);
316 		err = -EINVAL;
317 		goto out;
318 	}
319 
320 	rnbd_clt_info(dev, "Unmapping device, option: %s.\n",
321 		       force ? "force" : "normal");
322 
323 	/*
324 	 * We take explicit module reference only for one reason: do not
325 	 * race with lockless rnbd_destroy_sessions().
326 	 */
327 	if (!try_module_get(THIS_MODULE)) {
328 		err = -ENODEV;
329 		goto out;
330 	}
331 	err = rnbd_clt_unmap_device(dev, force, &attr->attr);
332 	if (err) {
333 		if (err != -EALREADY)
334 			rnbd_clt_err(dev, "unmap_device: %d\n",  err);
335 		goto module_put;
336 	}
337 
338 	/*
339 	 * Here device can be vanished!
340 	 */
341 
342 	err = count;
343 
344 module_put:
345 	module_put(THIS_MODULE);
346 out:
347 	kfree(opt);
348 
349 	return err;
350 }
351 
352 static struct kobj_attribute rnbd_clt_unmap_device_attr =
353 	__ATTR(unmap_device, 0644, rnbd_clt_unmap_dev_show,
354 	       rnbd_clt_unmap_dev_store);
355 
356 static ssize_t rnbd_clt_resize_dev_show(struct kobject *kobj,
357 					 struct kobj_attribute *attr,
358 					 char *page)
359 {
360 	return scnprintf(page, PAGE_SIZE,
361 			 "Usage: echo <new size in sectors> > %s\n",
362 			 attr->attr.name);
363 }
364 
365 static ssize_t rnbd_clt_resize_dev_store(struct kobject *kobj,
366 					  struct kobj_attribute *attr,
367 					  const char *buf, size_t count)
368 {
369 	int ret;
370 	unsigned long sectors;
371 	struct rnbd_clt_dev *dev;
372 
373 	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
374 
375 	ret = kstrtoul(buf, 0, &sectors);
376 	if (ret)
377 		return ret;
378 
379 	ret = rnbd_clt_resize_disk(dev, (size_t)sectors);
380 	if (ret)
381 		return ret;
382 
383 	return count;
384 }
385 
386 static struct kobj_attribute rnbd_clt_resize_dev_attr =
387 	__ATTR(resize, 0644, rnbd_clt_resize_dev_show,
388 	       rnbd_clt_resize_dev_store);
389 
390 static ssize_t rnbd_clt_remap_dev_show(struct kobject *kobj,
391 					struct kobj_attribute *attr, char *page)
392 {
393 	return scnprintf(page, PAGE_SIZE, "Usage: echo <1> > %s\n",
394 			 attr->attr.name);
395 }
396 
397 static ssize_t rnbd_clt_remap_dev_store(struct kobject *kobj,
398 					 struct kobj_attribute *attr,
399 					 const char *buf, size_t count)
400 {
401 	struct rnbd_clt_dev *dev;
402 	char *opt, *options;
403 	int err;
404 
405 	opt = kstrdup(buf, GFP_KERNEL);
406 	if (!opt)
407 		return -ENOMEM;
408 
409 	options = strstrip(opt);
410 	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
411 	if (!sysfs_streq(options, "1")) {
412 		rnbd_clt_err(dev,
413 			      "remap_device: Invalid value: %s\n",
414 			      options);
415 		err = -EINVAL;
416 		goto out;
417 	}
418 	err = rnbd_clt_remap_device(dev);
419 	if (likely(!err))
420 		err = count;
421 
422 out:
423 	kfree(opt);
424 
425 	return err;
426 }
427 
428 static struct kobj_attribute rnbd_clt_remap_device_attr =
429 	__ATTR(remap_device, 0644, rnbd_clt_remap_dev_show,
430 	       rnbd_clt_remap_dev_store);
431 
432 static ssize_t session_show(struct kobject *kobj, struct kobj_attribute *attr,
433 			    char *page)
434 {
435 	struct rnbd_clt_dev *dev;
436 
437 	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
438 
439 	return scnprintf(page, PAGE_SIZE, "%s\n", dev->sess->sessname);
440 }
441 
442 static struct kobj_attribute rnbd_clt_session_attr =
443 	__ATTR_RO(session);
444 
445 static struct attribute *rnbd_dev_attrs[] = {
446 	&rnbd_clt_unmap_device_attr.attr,
447 	&rnbd_clt_resize_dev_attr.attr,
448 	&rnbd_clt_remap_device_attr.attr,
449 	&rnbd_clt_mapping_path_attr.attr,
450 	&rnbd_clt_state_attr.attr,
451 	&rnbd_clt_session_attr.attr,
452 	&rnbd_clt_access_mode.attr,
453 	&rnbd_clt_nr_poll_queues.attr,
454 	NULL,
455 };
456 
457 void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev)
458 {
459 	/*
460 	 * The module unload rnbd_client_exit path is racing with unmapping of
461 	 * the last single device from the sysfs manually
462 	 * i.e. rnbd_clt_unmap_dev_store() leading to a sysfs warning because
463 	 * of sysfs link already was removed already.
464 	 */
465 	if (dev->blk_symlink_name) {
466 		if (try_module_get(THIS_MODULE)) {
467 			sysfs_remove_link(rnbd_devs_kobj, dev->blk_symlink_name);
468 			module_put(THIS_MODULE);
469 		}
470 		/* It should be freed always. */
471 		kfree(dev->blk_symlink_name);
472 		dev->blk_symlink_name = NULL;
473 	}
474 }
475 
476 static struct kobj_type rnbd_dev_ktype = {
477 	.sysfs_ops      = &kobj_sysfs_ops,
478 	.default_attrs  = rnbd_dev_attrs,
479 };
480 
481 static int rnbd_clt_add_dev_kobj(struct rnbd_clt_dev *dev)
482 {
483 	int ret;
484 	struct kobject *gd_kobj = &disk_to_dev(dev->gd)->kobj;
485 
486 	ret = kobject_init_and_add(&dev->kobj, &rnbd_dev_ktype, gd_kobj, "%s",
487 				   "rnbd");
488 	if (ret) {
489 		rnbd_clt_err(dev, "Failed to create device sysfs dir, err: %d\n",
490 			      ret);
491 		kobject_put(&dev->kobj);
492 	}
493 	kobject_uevent(gd_kobj, KOBJ_ONLINE);
494 
495 	return ret;
496 }
497 
498 static ssize_t rnbd_clt_map_device_show(struct kobject *kobj,
499 					 struct kobj_attribute *attr,
500 					 char *page)
501 {
502 	return scnprintf(page, PAGE_SIZE,
503 			 "Usage: echo \"[dest_port=server port number] sessname=<name of the rtrs session> path=<[srcaddr@]dstaddr> [path=<[srcaddr@]dstaddr>] device_path=<full path on remote side> [access_mode=<ro|rw|migration>] [nr_poll_queues=<number of queues>]\" > %s\n\naddr ::= [ ip:<ipv4> | ip:<ipv6> | gid:<gid> ]\n",
504 			 attr->attr.name);
505 }
506 
507 static int rnbd_clt_get_path_name(struct rnbd_clt_dev *dev, char *buf,
508 				   size_t len)
509 {
510 	int ret;
511 	char pathname[NAME_MAX], *s;
512 
513 	strscpy(pathname, dev->pathname, sizeof(pathname));
514 	while ((s = strchr(pathname, '/')))
515 		s[0] = '!';
516 
517 	ret = snprintf(buf, len, "%s@%s", pathname, dev->sess->sessname);
518 	if (ret >= len)
519 		return -ENAMETOOLONG;
520 
521 	return 0;
522 }
523 
524 static int rnbd_clt_add_dev_symlink(struct rnbd_clt_dev *dev)
525 {
526 	struct kobject *gd_kobj = &disk_to_dev(dev->gd)->kobj;
527 	int ret, len;
528 
529 	len = strlen(dev->pathname) + strlen(dev->sess->sessname) + 2;
530 	dev->blk_symlink_name = kzalloc(len, GFP_KERNEL);
531 	if (!dev->blk_symlink_name) {
532 		rnbd_clt_err(dev, "Failed to allocate memory for blk_symlink_name\n");
533 		return -ENOMEM;
534 	}
535 
536 	ret = rnbd_clt_get_path_name(dev, dev->blk_symlink_name,
537 				      len);
538 	if (ret) {
539 		rnbd_clt_err(dev, "Failed to get /sys/block symlink path, err: %d\n",
540 			      ret);
541 		goto out_err;
542 	}
543 
544 	ret = sysfs_create_link(rnbd_devs_kobj, gd_kobj,
545 				dev->blk_symlink_name);
546 	if (ret) {
547 		rnbd_clt_err(dev, "Creating /sys/block symlink failed, err: %d\n",
548 			      ret);
549 		goto out_err;
550 	}
551 
552 	return 0;
553 
554 out_err:
555 	kfree(dev->blk_symlink_name);
556 	dev->blk_symlink_name = NULL ;
557 	return ret;
558 }
559 
560 static ssize_t rnbd_clt_map_device_store(struct kobject *kobj,
561 					  struct kobj_attribute *attr,
562 					  const char *buf, size_t count)
563 {
564 	struct rnbd_clt_dev *dev;
565 	struct rnbd_map_options opt;
566 	int ret;
567 	char pathname[NAME_MAX];
568 	char sessname[NAME_MAX];
569 	enum rnbd_access_mode access_mode = RNBD_ACCESS_RW;
570 	u16 port_nr = RTRS_PORT;
571 	u32 nr_poll_queues = 0;
572 
573 	struct sockaddr_storage *addrs;
574 	struct rtrs_addr paths[6];
575 	size_t path_cnt;
576 
577 	opt.sessname = sessname;
578 	opt.paths = paths;
579 	opt.path_cnt = &path_cnt;
580 	opt.pathname = pathname;
581 	opt.dest_port = &port_nr;
582 	opt.access_mode = &access_mode;
583 	opt.nr_poll_queues = &nr_poll_queues;
584 	addrs = kcalloc(ARRAY_SIZE(paths) * 2, sizeof(*addrs), GFP_KERNEL);
585 	if (!addrs)
586 		return -ENOMEM;
587 
588 	for (path_cnt = 0; path_cnt < ARRAY_SIZE(paths); path_cnt++) {
589 		paths[path_cnt].src = &addrs[path_cnt * 2];
590 		paths[path_cnt].dst = &addrs[path_cnt * 2 + 1];
591 	}
592 
593 	ret = rnbd_clt_parse_map_options(buf, ARRAY_SIZE(paths), &opt);
594 	if (ret)
595 		goto out;
596 
597 	pr_info("Mapping device %s on session %s, (access_mode: %s, nr_poll_queues: %d)\n",
598 		pathname, sessname,
599 		rnbd_access_mode_str(access_mode),
600 		nr_poll_queues);
601 
602 	dev = rnbd_clt_map_device(sessname, paths, path_cnt, port_nr, pathname,
603 				  access_mode, nr_poll_queues);
604 	if (IS_ERR(dev)) {
605 		ret = PTR_ERR(dev);
606 		goto out;
607 	}
608 
609 	ret = rnbd_clt_add_dev_kobj(dev);
610 	if (ret)
611 		goto unmap_dev;
612 
613 	ret = rnbd_clt_add_dev_symlink(dev);
614 	if (ret)
615 		goto unmap_dev;
616 
617 	kfree(addrs);
618 	return count;
619 
620 unmap_dev:
621 	rnbd_clt_unmap_device(dev, true, NULL);
622 out:
623 	kfree(addrs);
624 	return ret;
625 }
626 
627 static struct kobj_attribute rnbd_clt_map_device_attr =
628 	__ATTR(map_device, 0644,
629 	       rnbd_clt_map_device_show, rnbd_clt_map_device_store);
630 
631 static struct attribute *default_attrs[] = {
632 	&rnbd_clt_map_device_attr.attr,
633 	NULL,
634 };
635 
636 static struct attribute_group default_attr_group = {
637 	.attrs = default_attrs,
638 };
639 
640 static const struct attribute_group *default_attr_groups[] = {
641 	&default_attr_group,
642 	NULL,
643 };
644 
645 int rnbd_clt_create_sysfs_files(void)
646 {
647 	int err;
648 
649 	rnbd_dev_class = class_create(THIS_MODULE, "rnbd-client");
650 	if (IS_ERR(rnbd_dev_class))
651 		return PTR_ERR(rnbd_dev_class);
652 
653 	rnbd_dev = device_create_with_groups(rnbd_dev_class, NULL,
654 					      MKDEV(0, 0), NULL,
655 					      default_attr_groups, "ctl");
656 	if (IS_ERR(rnbd_dev)) {
657 		err = PTR_ERR(rnbd_dev);
658 		goto cls_destroy;
659 	}
660 	rnbd_devs_kobj = kobject_create_and_add("devices", &rnbd_dev->kobj);
661 	if (!rnbd_devs_kobj) {
662 		err = -ENOMEM;
663 		goto dev_destroy;
664 	}
665 
666 	return 0;
667 
668 dev_destroy:
669 	device_destroy(rnbd_dev_class, MKDEV(0, 0));
670 cls_destroy:
671 	class_destroy(rnbd_dev_class);
672 
673 	return err;
674 }
675 
676 void rnbd_clt_destroy_sysfs_files(void)
677 {
678 	sysfs_remove_group(&rnbd_dev->kobj, &default_attr_group);
679 	kobject_del(rnbd_devs_kobj);
680 	kobject_put(rnbd_devs_kobj);
681 	device_destroy(rnbd_dev_class, MKDEV(0, 0));
682 	class_destroy(rnbd_dev_class);
683 }
684