xref: /openbmc/linux/fs/btrfs/sysfs.c (revision 1d1997db)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2007 Oracle.  All rights reserved.
4  */
5 
6 #include <linux/sched.h>
7 #include <linux/sched/mm.h>
8 #include <linux/slab.h>
9 #include <linux/spinlock.h>
10 #include <linux/completion.h>
11 #include <linux/bug.h>
12 #include <crypto/hash.h>
13 
14 #include "ctree.h"
15 #include "disk-io.h"
16 #include "transaction.h"
17 #include "sysfs.h"
18 #include "volumes.h"
19 #include "space-info.h"
20 #include "block-group.h"
21 
22 struct btrfs_feature_attr {
23 	struct kobj_attribute kobj_attr;
24 	enum btrfs_feature_set feature_set;
25 	u64 feature_bit;
26 };
27 
28 /* For raid type sysfs entries */
29 struct raid_kobject {
30 	u64 flags;
31 	struct kobject kobj;
32 };
33 
34 #define __INIT_KOBJ_ATTR(_name, _mode, _show, _store)			\
35 {									\
36 	.attr	= { .name = __stringify(_name), .mode = _mode },	\
37 	.show	= _show,						\
38 	.store	= _store,						\
39 }
40 
41 #define BTRFS_ATTR_RW(_prefix, _name, _show, _store)			\
42 	static struct kobj_attribute btrfs_attr_##_prefix##_##_name =	\
43 			__INIT_KOBJ_ATTR(_name, 0644, _show, _store)
44 
45 #define BTRFS_ATTR(_prefix, _name, _show)				\
46 	static struct kobj_attribute btrfs_attr_##_prefix##_##_name =	\
47 			__INIT_KOBJ_ATTR(_name, 0444, _show, NULL)
48 
49 #define BTRFS_ATTR_PTR(_prefix, _name)					\
50 	(&btrfs_attr_##_prefix##_##_name.attr)
51 
52 #define BTRFS_FEAT_ATTR(_name, _feature_set, _feature_prefix, _feature_bit)  \
53 static struct btrfs_feature_attr btrfs_attr_features_##_name = {	     \
54 	.kobj_attr = __INIT_KOBJ_ATTR(_name, S_IRUGO,			     \
55 				      btrfs_feature_attr_show,		     \
56 				      btrfs_feature_attr_store),	     \
57 	.feature_set	= _feature_set,					     \
58 	.feature_bit	= _feature_prefix ##_## _feature_bit,		     \
59 }
60 #define BTRFS_FEAT_ATTR_PTR(_name)					     \
61 	(&btrfs_attr_features_##_name.kobj_attr.attr)
62 
63 #define BTRFS_FEAT_ATTR_COMPAT(name, feature) \
64 	BTRFS_FEAT_ATTR(name, FEAT_COMPAT, BTRFS_FEATURE_COMPAT, feature)
65 #define BTRFS_FEAT_ATTR_COMPAT_RO(name, feature) \
66 	BTRFS_FEAT_ATTR(name, FEAT_COMPAT_RO, BTRFS_FEATURE_COMPAT_RO, feature)
67 #define BTRFS_FEAT_ATTR_INCOMPAT(name, feature) \
68 	BTRFS_FEAT_ATTR(name, FEAT_INCOMPAT, BTRFS_FEATURE_INCOMPAT, feature)
69 
70 static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj);
71 static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj);
72 
73 static struct btrfs_feature_attr *to_btrfs_feature_attr(struct kobj_attribute *a)
74 {
75 	return container_of(a, struct btrfs_feature_attr, kobj_attr);
76 }
77 
78 static struct kobj_attribute *attr_to_btrfs_attr(struct attribute *attr)
79 {
80 	return container_of(attr, struct kobj_attribute, attr);
81 }
82 
83 static struct btrfs_feature_attr *attr_to_btrfs_feature_attr(
84 		struct attribute *attr)
85 {
86 	return to_btrfs_feature_attr(attr_to_btrfs_attr(attr));
87 }
88 
89 static u64 get_features(struct btrfs_fs_info *fs_info,
90 			enum btrfs_feature_set set)
91 {
92 	struct btrfs_super_block *disk_super = fs_info->super_copy;
93 	if (set == FEAT_COMPAT)
94 		return btrfs_super_compat_flags(disk_super);
95 	else if (set == FEAT_COMPAT_RO)
96 		return btrfs_super_compat_ro_flags(disk_super);
97 	else
98 		return btrfs_super_incompat_flags(disk_super);
99 }
100 
101 static void set_features(struct btrfs_fs_info *fs_info,
102 			 enum btrfs_feature_set set, u64 features)
103 {
104 	struct btrfs_super_block *disk_super = fs_info->super_copy;
105 	if (set == FEAT_COMPAT)
106 		btrfs_set_super_compat_flags(disk_super, features);
107 	else if (set == FEAT_COMPAT_RO)
108 		btrfs_set_super_compat_ro_flags(disk_super, features);
109 	else
110 		btrfs_set_super_incompat_flags(disk_super, features);
111 }
112 
113 static int can_modify_feature(struct btrfs_feature_attr *fa)
114 {
115 	int val = 0;
116 	u64 set, clear;
117 	switch (fa->feature_set) {
118 	case FEAT_COMPAT:
119 		set = BTRFS_FEATURE_COMPAT_SAFE_SET;
120 		clear = BTRFS_FEATURE_COMPAT_SAFE_CLEAR;
121 		break;
122 	case FEAT_COMPAT_RO:
123 		set = BTRFS_FEATURE_COMPAT_RO_SAFE_SET;
124 		clear = BTRFS_FEATURE_COMPAT_RO_SAFE_CLEAR;
125 		break;
126 	case FEAT_INCOMPAT:
127 		set = BTRFS_FEATURE_INCOMPAT_SAFE_SET;
128 		clear = BTRFS_FEATURE_INCOMPAT_SAFE_CLEAR;
129 		break;
130 	default:
131 		pr_warn("btrfs: sysfs: unknown feature set %d\n",
132 				fa->feature_set);
133 		return 0;
134 	}
135 
136 	if (set & fa->feature_bit)
137 		val |= 1;
138 	if (clear & fa->feature_bit)
139 		val |= 2;
140 
141 	return val;
142 }
143 
144 static ssize_t btrfs_feature_attr_show(struct kobject *kobj,
145 				       struct kobj_attribute *a, char *buf)
146 {
147 	int val = 0;
148 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
149 	struct btrfs_feature_attr *fa = to_btrfs_feature_attr(a);
150 	if (fs_info) {
151 		u64 features = get_features(fs_info, fa->feature_set);
152 		if (features & fa->feature_bit)
153 			val = 1;
154 	} else
155 		val = can_modify_feature(fa);
156 
157 	return snprintf(buf, PAGE_SIZE, "%d\n", val);
158 }
159 
160 static ssize_t btrfs_feature_attr_store(struct kobject *kobj,
161 					struct kobj_attribute *a,
162 					const char *buf, size_t count)
163 {
164 	struct btrfs_fs_info *fs_info;
165 	struct btrfs_feature_attr *fa = to_btrfs_feature_attr(a);
166 	u64 features, set, clear;
167 	unsigned long val;
168 	int ret;
169 
170 	fs_info = to_fs_info(kobj);
171 	if (!fs_info)
172 		return -EPERM;
173 
174 	if (sb_rdonly(fs_info->sb))
175 		return -EROFS;
176 
177 	ret = kstrtoul(skip_spaces(buf), 0, &val);
178 	if (ret)
179 		return ret;
180 
181 	if (fa->feature_set == FEAT_COMPAT) {
182 		set = BTRFS_FEATURE_COMPAT_SAFE_SET;
183 		clear = BTRFS_FEATURE_COMPAT_SAFE_CLEAR;
184 	} else if (fa->feature_set == FEAT_COMPAT_RO) {
185 		set = BTRFS_FEATURE_COMPAT_RO_SAFE_SET;
186 		clear = BTRFS_FEATURE_COMPAT_RO_SAFE_CLEAR;
187 	} else {
188 		set = BTRFS_FEATURE_INCOMPAT_SAFE_SET;
189 		clear = BTRFS_FEATURE_INCOMPAT_SAFE_CLEAR;
190 	}
191 
192 	features = get_features(fs_info, fa->feature_set);
193 
194 	/* Nothing to do */
195 	if ((val && (features & fa->feature_bit)) ||
196 	    (!val && !(features & fa->feature_bit)))
197 		return count;
198 
199 	if ((val && !(set & fa->feature_bit)) ||
200 	    (!val && !(clear & fa->feature_bit))) {
201 		btrfs_info(fs_info,
202 			"%sabling feature %s on mounted fs is not supported.",
203 			val ? "En" : "Dis", fa->kobj_attr.attr.name);
204 		return -EPERM;
205 	}
206 
207 	btrfs_info(fs_info, "%s %s feature flag",
208 		   val ? "Setting" : "Clearing", fa->kobj_attr.attr.name);
209 
210 	spin_lock(&fs_info->super_lock);
211 	features = get_features(fs_info, fa->feature_set);
212 	if (val)
213 		features |= fa->feature_bit;
214 	else
215 		features &= ~fa->feature_bit;
216 	set_features(fs_info, fa->feature_set, features);
217 	spin_unlock(&fs_info->super_lock);
218 
219 	/*
220 	 * We don't want to do full transaction commit from inside sysfs
221 	 */
222 	btrfs_set_pending(fs_info, COMMIT);
223 	wake_up_process(fs_info->transaction_kthread);
224 
225 	return count;
226 }
227 
228 static umode_t btrfs_feature_visible(struct kobject *kobj,
229 				     struct attribute *attr, int unused)
230 {
231 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
232 	umode_t mode = attr->mode;
233 
234 	if (fs_info) {
235 		struct btrfs_feature_attr *fa;
236 		u64 features;
237 
238 		fa = attr_to_btrfs_feature_attr(attr);
239 		features = get_features(fs_info, fa->feature_set);
240 
241 		if (can_modify_feature(fa))
242 			mode |= S_IWUSR;
243 		else if (!(features & fa->feature_bit))
244 			mode = 0;
245 	}
246 
247 	return mode;
248 }
249 
250 BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF);
251 BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL);
252 BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS);
253 BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO);
254 BTRFS_FEAT_ATTR_INCOMPAT(compress_zstd, COMPRESS_ZSTD);
255 BTRFS_FEAT_ATTR_INCOMPAT(big_metadata, BIG_METADATA);
256 BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF);
257 BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56);
258 BTRFS_FEAT_ATTR_INCOMPAT(skinny_metadata, SKINNY_METADATA);
259 BTRFS_FEAT_ATTR_INCOMPAT(no_holes, NO_HOLES);
260 BTRFS_FEAT_ATTR_INCOMPAT(metadata_uuid, METADATA_UUID);
261 BTRFS_FEAT_ATTR_COMPAT_RO(free_space_tree, FREE_SPACE_TREE);
262 BTRFS_FEAT_ATTR_INCOMPAT(raid1c34, RAID1C34);
263 
264 static struct attribute *btrfs_supported_feature_attrs[] = {
265 	BTRFS_FEAT_ATTR_PTR(mixed_backref),
266 	BTRFS_FEAT_ATTR_PTR(default_subvol),
267 	BTRFS_FEAT_ATTR_PTR(mixed_groups),
268 	BTRFS_FEAT_ATTR_PTR(compress_lzo),
269 	BTRFS_FEAT_ATTR_PTR(compress_zstd),
270 	BTRFS_FEAT_ATTR_PTR(big_metadata),
271 	BTRFS_FEAT_ATTR_PTR(extended_iref),
272 	BTRFS_FEAT_ATTR_PTR(raid56),
273 	BTRFS_FEAT_ATTR_PTR(skinny_metadata),
274 	BTRFS_FEAT_ATTR_PTR(no_holes),
275 	BTRFS_FEAT_ATTR_PTR(metadata_uuid),
276 	BTRFS_FEAT_ATTR_PTR(free_space_tree),
277 	BTRFS_FEAT_ATTR_PTR(raid1c34),
278 	NULL
279 };
280 
281 /*
282  * Features which depend on feature bits and may differ between each fs.
283  *
284  * /sys/fs/btrfs/features lists all available features of this kernel while
285  * /sys/fs/btrfs/UUID/features shows features of the fs which are enabled or
286  * can be changed online.
287  */
288 static const struct attribute_group btrfs_feature_attr_group = {
289 	.name = "features",
290 	.is_visible = btrfs_feature_visible,
291 	.attrs = btrfs_supported_feature_attrs,
292 };
293 
294 static ssize_t rmdir_subvol_show(struct kobject *kobj,
295 				 struct kobj_attribute *ka, char *buf)
296 {
297 	return snprintf(buf, PAGE_SIZE, "0\n");
298 }
299 BTRFS_ATTR(static_feature, rmdir_subvol, rmdir_subvol_show);
300 
301 static ssize_t supported_checksums_show(struct kobject *kobj,
302 					struct kobj_attribute *a, char *buf)
303 {
304 	ssize_t ret = 0;
305 	int i;
306 
307 	for (i = 0; i < btrfs_get_num_csums(); i++) {
308 		/*
309 		 * This "trick" only works as long as 'enum btrfs_csum_type' has
310 		 * no holes in it
311 		 */
312 		ret += snprintf(buf + ret, PAGE_SIZE - ret, "%s%s",
313 				(i == 0 ? "" : " "), btrfs_super_csum_name(i));
314 
315 	}
316 
317 	ret += snprintf(buf + ret, PAGE_SIZE - ret, "\n");
318 	return ret;
319 }
320 BTRFS_ATTR(static_feature, supported_checksums, supported_checksums_show);
321 
322 static struct attribute *btrfs_supported_static_feature_attrs[] = {
323 	BTRFS_ATTR_PTR(static_feature, rmdir_subvol),
324 	BTRFS_ATTR_PTR(static_feature, supported_checksums),
325 	NULL
326 };
327 
328 /*
329  * Features which only depend on kernel version.
330  *
331  * These are listed in /sys/fs/btrfs/features along with
332  * btrfs_feature_attr_group
333  */
334 static const struct attribute_group btrfs_static_feature_attr_group = {
335 	.name = "features",
336 	.attrs = btrfs_supported_static_feature_attrs,
337 };
338 
339 #ifdef CONFIG_BTRFS_DEBUG
340 
341 /*
342  * Runtime debugging exported via sysfs
343  *
344  * /sys/fs/btrfs/debug - applies to module or all filesystems
345  * /sys/fs/btrfs/UUID  - applies only to the given filesystem
346  */
347 static struct attribute *btrfs_debug_feature_attrs[] = {
348 	NULL
349 };
350 
351 static const struct attribute_group btrfs_debug_feature_attr_group = {
352 	.name = "debug",
353 	.attrs = btrfs_debug_feature_attrs,
354 };
355 
356 #endif
357 
358 static ssize_t btrfs_show_u64(u64 *value_ptr, spinlock_t *lock, char *buf)
359 {
360 	u64 val;
361 	if (lock)
362 		spin_lock(lock);
363 	val = *value_ptr;
364 	if (lock)
365 		spin_unlock(lock);
366 	return snprintf(buf, PAGE_SIZE, "%llu\n", val);
367 }
368 
369 static ssize_t global_rsv_size_show(struct kobject *kobj,
370 				    struct kobj_attribute *ka, char *buf)
371 {
372 	struct btrfs_fs_info *fs_info = to_fs_info(kobj->parent);
373 	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
374 	return btrfs_show_u64(&block_rsv->size, &block_rsv->lock, buf);
375 }
376 BTRFS_ATTR(allocation, global_rsv_size, global_rsv_size_show);
377 
378 static ssize_t global_rsv_reserved_show(struct kobject *kobj,
379 					struct kobj_attribute *a, char *buf)
380 {
381 	struct btrfs_fs_info *fs_info = to_fs_info(kobj->parent);
382 	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
383 	return btrfs_show_u64(&block_rsv->reserved, &block_rsv->lock, buf);
384 }
385 BTRFS_ATTR(allocation, global_rsv_reserved, global_rsv_reserved_show);
386 
387 #define to_space_info(_kobj) container_of(_kobj, struct btrfs_space_info, kobj)
388 #define to_raid_kobj(_kobj) container_of(_kobj, struct raid_kobject, kobj)
389 
390 static ssize_t raid_bytes_show(struct kobject *kobj,
391 			       struct kobj_attribute *attr, char *buf);
392 BTRFS_ATTR(raid, total_bytes, raid_bytes_show);
393 BTRFS_ATTR(raid, used_bytes, raid_bytes_show);
394 
395 static ssize_t raid_bytes_show(struct kobject *kobj,
396 			       struct kobj_attribute *attr, char *buf)
397 
398 {
399 	struct btrfs_space_info *sinfo = to_space_info(kobj->parent);
400 	struct btrfs_block_group *block_group;
401 	int index = btrfs_bg_flags_to_raid_index(to_raid_kobj(kobj)->flags);
402 	u64 val = 0;
403 
404 	down_read(&sinfo->groups_sem);
405 	list_for_each_entry(block_group, &sinfo->block_groups[index], list) {
406 		if (&attr->attr == BTRFS_ATTR_PTR(raid, total_bytes))
407 			val += block_group->length;
408 		else
409 			val += block_group->used;
410 	}
411 	up_read(&sinfo->groups_sem);
412 	return snprintf(buf, PAGE_SIZE, "%llu\n", val);
413 }
414 
415 static struct attribute *raid_attrs[] = {
416 	BTRFS_ATTR_PTR(raid, total_bytes),
417 	BTRFS_ATTR_PTR(raid, used_bytes),
418 	NULL
419 };
420 ATTRIBUTE_GROUPS(raid);
421 
422 static void release_raid_kobj(struct kobject *kobj)
423 {
424 	kfree(to_raid_kobj(kobj));
425 }
426 
427 static struct kobj_type btrfs_raid_ktype = {
428 	.sysfs_ops = &kobj_sysfs_ops,
429 	.release = release_raid_kobj,
430 	.default_groups = raid_groups,
431 };
432 
433 #define SPACE_INFO_ATTR(field)						\
434 static ssize_t btrfs_space_info_show_##field(struct kobject *kobj,	\
435 					     struct kobj_attribute *a,	\
436 					     char *buf)			\
437 {									\
438 	struct btrfs_space_info *sinfo = to_space_info(kobj);		\
439 	return btrfs_show_u64(&sinfo->field, &sinfo->lock, buf);	\
440 }									\
441 BTRFS_ATTR(space_info, field, btrfs_space_info_show_##field)
442 
443 static ssize_t btrfs_space_info_show_total_bytes_pinned(struct kobject *kobj,
444 						       struct kobj_attribute *a,
445 						       char *buf)
446 {
447 	struct btrfs_space_info *sinfo = to_space_info(kobj);
448 	s64 val = percpu_counter_sum(&sinfo->total_bytes_pinned);
449 	return snprintf(buf, PAGE_SIZE, "%lld\n", val);
450 }
451 
452 SPACE_INFO_ATTR(flags);
453 SPACE_INFO_ATTR(total_bytes);
454 SPACE_INFO_ATTR(bytes_used);
455 SPACE_INFO_ATTR(bytes_pinned);
456 SPACE_INFO_ATTR(bytes_reserved);
457 SPACE_INFO_ATTR(bytes_may_use);
458 SPACE_INFO_ATTR(bytes_readonly);
459 SPACE_INFO_ATTR(disk_used);
460 SPACE_INFO_ATTR(disk_total);
461 BTRFS_ATTR(space_info, total_bytes_pinned,
462 	   btrfs_space_info_show_total_bytes_pinned);
463 
464 static struct attribute *space_info_attrs[] = {
465 	BTRFS_ATTR_PTR(space_info, flags),
466 	BTRFS_ATTR_PTR(space_info, total_bytes),
467 	BTRFS_ATTR_PTR(space_info, bytes_used),
468 	BTRFS_ATTR_PTR(space_info, bytes_pinned),
469 	BTRFS_ATTR_PTR(space_info, bytes_reserved),
470 	BTRFS_ATTR_PTR(space_info, bytes_may_use),
471 	BTRFS_ATTR_PTR(space_info, bytes_readonly),
472 	BTRFS_ATTR_PTR(space_info, disk_used),
473 	BTRFS_ATTR_PTR(space_info, disk_total),
474 	BTRFS_ATTR_PTR(space_info, total_bytes_pinned),
475 	NULL,
476 };
477 ATTRIBUTE_GROUPS(space_info);
478 
479 static void space_info_release(struct kobject *kobj)
480 {
481 	struct btrfs_space_info *sinfo = to_space_info(kobj);
482 	percpu_counter_destroy(&sinfo->total_bytes_pinned);
483 	kfree(sinfo);
484 }
485 
486 static struct kobj_type space_info_ktype = {
487 	.sysfs_ops = &kobj_sysfs_ops,
488 	.release = space_info_release,
489 	.default_groups = space_info_groups,
490 };
491 
492 static const struct attribute *allocation_attrs[] = {
493 	BTRFS_ATTR_PTR(allocation, global_rsv_reserved),
494 	BTRFS_ATTR_PTR(allocation, global_rsv_size),
495 	NULL,
496 };
497 
498 static ssize_t btrfs_label_show(struct kobject *kobj,
499 				struct kobj_attribute *a, char *buf)
500 {
501 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
502 	char *label = fs_info->super_copy->label;
503 	ssize_t ret;
504 
505 	spin_lock(&fs_info->super_lock);
506 	ret = snprintf(buf, PAGE_SIZE, label[0] ? "%s\n" : "%s", label);
507 	spin_unlock(&fs_info->super_lock);
508 
509 	return ret;
510 }
511 
512 static ssize_t btrfs_label_store(struct kobject *kobj,
513 				 struct kobj_attribute *a,
514 				 const char *buf, size_t len)
515 {
516 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
517 	size_t p_len;
518 
519 	if (!fs_info)
520 		return -EPERM;
521 
522 	if (sb_rdonly(fs_info->sb))
523 		return -EROFS;
524 
525 	/*
526 	 * p_len is the len until the first occurrence of either
527 	 * '\n' or '\0'
528 	 */
529 	p_len = strcspn(buf, "\n");
530 
531 	if (p_len >= BTRFS_LABEL_SIZE)
532 		return -EINVAL;
533 
534 	spin_lock(&fs_info->super_lock);
535 	memset(fs_info->super_copy->label, 0, BTRFS_LABEL_SIZE);
536 	memcpy(fs_info->super_copy->label, buf, p_len);
537 	spin_unlock(&fs_info->super_lock);
538 
539 	/*
540 	 * We don't want to do full transaction commit from inside sysfs
541 	 */
542 	btrfs_set_pending(fs_info, COMMIT);
543 	wake_up_process(fs_info->transaction_kthread);
544 
545 	return len;
546 }
547 BTRFS_ATTR_RW(, label, btrfs_label_show, btrfs_label_store);
548 
549 static ssize_t btrfs_nodesize_show(struct kobject *kobj,
550 				struct kobj_attribute *a, char *buf)
551 {
552 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
553 
554 	return snprintf(buf, PAGE_SIZE, "%u\n", fs_info->super_copy->nodesize);
555 }
556 
557 BTRFS_ATTR(, nodesize, btrfs_nodesize_show);
558 
559 static ssize_t btrfs_sectorsize_show(struct kobject *kobj,
560 				struct kobj_attribute *a, char *buf)
561 {
562 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
563 
564 	return snprintf(buf, PAGE_SIZE, "%u\n",
565 			fs_info->super_copy->sectorsize);
566 }
567 
568 BTRFS_ATTR(, sectorsize, btrfs_sectorsize_show);
569 
570 static ssize_t btrfs_clone_alignment_show(struct kobject *kobj,
571 				struct kobj_attribute *a, char *buf)
572 {
573 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
574 
575 	return snprintf(buf, PAGE_SIZE, "%u\n",
576 			fs_info->super_copy->sectorsize);
577 }
578 
579 BTRFS_ATTR(, clone_alignment, btrfs_clone_alignment_show);
580 
581 static ssize_t quota_override_show(struct kobject *kobj,
582 				   struct kobj_attribute *a, char *buf)
583 {
584 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
585 	int quota_override;
586 
587 	quota_override = test_bit(BTRFS_FS_QUOTA_OVERRIDE, &fs_info->flags);
588 	return snprintf(buf, PAGE_SIZE, "%d\n", quota_override);
589 }
590 
591 static ssize_t quota_override_store(struct kobject *kobj,
592 				    struct kobj_attribute *a,
593 				    const char *buf, size_t len)
594 {
595 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
596 	unsigned long knob;
597 	int err;
598 
599 	if (!fs_info)
600 		return -EPERM;
601 
602 	if (!capable(CAP_SYS_RESOURCE))
603 		return -EPERM;
604 
605 	err = kstrtoul(buf, 10, &knob);
606 	if (err)
607 		return err;
608 	if (knob > 1)
609 		return -EINVAL;
610 
611 	if (knob)
612 		set_bit(BTRFS_FS_QUOTA_OVERRIDE, &fs_info->flags);
613 	else
614 		clear_bit(BTRFS_FS_QUOTA_OVERRIDE, &fs_info->flags);
615 
616 	return len;
617 }
618 
619 BTRFS_ATTR_RW(, quota_override, quota_override_show, quota_override_store);
620 
621 static ssize_t btrfs_metadata_uuid_show(struct kobject *kobj,
622 				struct kobj_attribute *a, char *buf)
623 {
624 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
625 
626 	return snprintf(buf, PAGE_SIZE, "%pU\n",
627 			fs_info->fs_devices->metadata_uuid);
628 }
629 
630 BTRFS_ATTR(, metadata_uuid, btrfs_metadata_uuid_show);
631 
632 static ssize_t btrfs_checksum_show(struct kobject *kobj,
633 				   struct kobj_attribute *a, char *buf)
634 {
635 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
636 	u16 csum_type = btrfs_super_csum_type(fs_info->super_copy);
637 
638 	return snprintf(buf, PAGE_SIZE, "%s (%s)\n",
639 			btrfs_super_csum_name(csum_type),
640 			crypto_shash_driver_name(fs_info->csum_shash));
641 }
642 
643 BTRFS_ATTR(, checksum, btrfs_checksum_show);
644 
645 static const struct attribute *btrfs_attrs[] = {
646 	BTRFS_ATTR_PTR(, label),
647 	BTRFS_ATTR_PTR(, nodesize),
648 	BTRFS_ATTR_PTR(, sectorsize),
649 	BTRFS_ATTR_PTR(, clone_alignment),
650 	BTRFS_ATTR_PTR(, quota_override),
651 	BTRFS_ATTR_PTR(, metadata_uuid),
652 	BTRFS_ATTR_PTR(, checksum),
653 	NULL,
654 };
655 
656 static void btrfs_release_fsid_kobj(struct kobject *kobj)
657 {
658 	struct btrfs_fs_devices *fs_devs = to_fs_devs(kobj);
659 
660 	memset(&fs_devs->fsid_kobj, 0, sizeof(struct kobject));
661 	complete(&fs_devs->kobj_unregister);
662 }
663 
664 static struct kobj_type btrfs_ktype = {
665 	.sysfs_ops	= &kobj_sysfs_ops,
666 	.release	= btrfs_release_fsid_kobj,
667 };
668 
669 static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj)
670 {
671 	if (kobj->ktype != &btrfs_ktype)
672 		return NULL;
673 	return container_of(kobj, struct btrfs_fs_devices, fsid_kobj);
674 }
675 
676 static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj)
677 {
678 	if (kobj->ktype != &btrfs_ktype)
679 		return NULL;
680 	return to_fs_devs(kobj)->fs_info;
681 }
682 
683 #define NUM_FEATURE_BITS 64
684 #define BTRFS_FEATURE_NAME_MAX 13
685 static char btrfs_unknown_feature_names[FEAT_MAX][NUM_FEATURE_BITS][BTRFS_FEATURE_NAME_MAX];
686 static struct btrfs_feature_attr btrfs_feature_attrs[FEAT_MAX][NUM_FEATURE_BITS];
687 
688 static const u64 supported_feature_masks[FEAT_MAX] = {
689 	[FEAT_COMPAT]    = BTRFS_FEATURE_COMPAT_SUPP,
690 	[FEAT_COMPAT_RO] = BTRFS_FEATURE_COMPAT_RO_SUPP,
691 	[FEAT_INCOMPAT]  = BTRFS_FEATURE_INCOMPAT_SUPP,
692 };
693 
694 static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
695 {
696 	int set;
697 
698 	for (set = 0; set < FEAT_MAX; set++) {
699 		int i;
700 		struct attribute *attrs[2];
701 		struct attribute_group agroup = {
702 			.name = "features",
703 			.attrs = attrs,
704 		};
705 		u64 features = get_features(fs_info, set);
706 		features &= ~supported_feature_masks[set];
707 
708 		if (!features)
709 			continue;
710 
711 		attrs[1] = NULL;
712 		for (i = 0; i < NUM_FEATURE_BITS; i++) {
713 			struct btrfs_feature_attr *fa;
714 
715 			if (!(features & (1ULL << i)))
716 				continue;
717 
718 			fa = &btrfs_feature_attrs[set][i];
719 			attrs[0] = &fa->kobj_attr.attr;
720 			if (add) {
721 				int ret;
722 				ret = sysfs_merge_group(&fs_info->fs_devices->fsid_kobj,
723 							&agroup);
724 				if (ret)
725 					return ret;
726 			} else
727 				sysfs_unmerge_group(&fs_info->fs_devices->fsid_kobj,
728 						    &agroup);
729 		}
730 
731 	}
732 	return 0;
733 }
734 
735 static void __btrfs_sysfs_remove_fsid(struct btrfs_fs_devices *fs_devs)
736 {
737 	if (fs_devs->device_dir_kobj) {
738 		kobject_del(fs_devs->device_dir_kobj);
739 		kobject_put(fs_devs->device_dir_kobj);
740 		fs_devs->device_dir_kobj = NULL;
741 	}
742 
743 	if (fs_devs->fsid_kobj.state_initialized) {
744 		kobject_del(&fs_devs->fsid_kobj);
745 		kobject_put(&fs_devs->fsid_kobj);
746 		wait_for_completion(&fs_devs->kobj_unregister);
747 	}
748 }
749 
750 /* when fs_devs is NULL it will remove all fsid kobject */
751 void btrfs_sysfs_remove_fsid(struct btrfs_fs_devices *fs_devs)
752 {
753 	struct list_head *fs_uuids = btrfs_get_fs_uuids();
754 
755 	if (fs_devs) {
756 		__btrfs_sysfs_remove_fsid(fs_devs);
757 		return;
758 	}
759 
760 	list_for_each_entry(fs_devs, fs_uuids, fs_list) {
761 		__btrfs_sysfs_remove_fsid(fs_devs);
762 	}
763 }
764 
765 void btrfs_sysfs_remove_mounted(struct btrfs_fs_info *fs_info)
766 {
767 	btrfs_reset_fs_info_ptr(fs_info);
768 
769 	if (fs_info->space_info_kobj) {
770 		sysfs_remove_files(fs_info->space_info_kobj, allocation_attrs);
771 		kobject_del(fs_info->space_info_kobj);
772 		kobject_put(fs_info->space_info_kobj);
773 	}
774 	addrm_unknown_feature_attrs(fs_info, false);
775 	sysfs_remove_group(&fs_info->fs_devices->fsid_kobj, &btrfs_feature_attr_group);
776 	sysfs_remove_files(&fs_info->fs_devices->fsid_kobj, btrfs_attrs);
777 	btrfs_sysfs_rm_device_link(fs_info->fs_devices, NULL);
778 }
779 
780 static const char * const btrfs_feature_set_names[FEAT_MAX] = {
781 	[FEAT_COMPAT]	 = "compat",
782 	[FEAT_COMPAT_RO] = "compat_ro",
783 	[FEAT_INCOMPAT]	 = "incompat",
784 };
785 
786 const char * const btrfs_feature_set_name(enum btrfs_feature_set set)
787 {
788 	return btrfs_feature_set_names[set];
789 }
790 
791 char *btrfs_printable_features(enum btrfs_feature_set set, u64 flags)
792 {
793 	size_t bufsize = 4096; /* safe max, 64 names * 64 bytes */
794 	int len = 0;
795 	int i;
796 	char *str;
797 
798 	str = kmalloc(bufsize, GFP_KERNEL);
799 	if (!str)
800 		return str;
801 
802 	for (i = 0; i < ARRAY_SIZE(btrfs_feature_attrs[set]); i++) {
803 		const char *name;
804 
805 		if (!(flags & (1ULL << i)))
806 			continue;
807 
808 		name = btrfs_feature_attrs[set][i].kobj_attr.attr.name;
809 		len += snprintf(str + len, bufsize - len, "%s%s",
810 				len ? "," : "", name);
811 	}
812 
813 	return str;
814 }
815 
816 static void init_feature_attrs(void)
817 {
818 	struct btrfs_feature_attr *fa;
819 	int set, i;
820 
821 	BUILD_BUG_ON(ARRAY_SIZE(btrfs_unknown_feature_names) !=
822 		     ARRAY_SIZE(btrfs_feature_attrs));
823 	BUILD_BUG_ON(ARRAY_SIZE(btrfs_unknown_feature_names[0]) !=
824 		     ARRAY_SIZE(btrfs_feature_attrs[0]));
825 
826 	memset(btrfs_feature_attrs, 0, sizeof(btrfs_feature_attrs));
827 	memset(btrfs_unknown_feature_names, 0,
828 	       sizeof(btrfs_unknown_feature_names));
829 
830 	for (i = 0; btrfs_supported_feature_attrs[i]; i++) {
831 		struct btrfs_feature_attr *sfa;
832 		struct attribute *a = btrfs_supported_feature_attrs[i];
833 		int bit;
834 		sfa = attr_to_btrfs_feature_attr(a);
835 		bit = ilog2(sfa->feature_bit);
836 		fa = &btrfs_feature_attrs[sfa->feature_set][bit];
837 
838 		fa->kobj_attr.attr.name = sfa->kobj_attr.attr.name;
839 	}
840 
841 	for (set = 0; set < FEAT_MAX; set++) {
842 		for (i = 0; i < ARRAY_SIZE(btrfs_feature_attrs[set]); i++) {
843 			char *name = btrfs_unknown_feature_names[set][i];
844 			fa = &btrfs_feature_attrs[set][i];
845 
846 			if (fa->kobj_attr.attr.name)
847 				continue;
848 
849 			snprintf(name, BTRFS_FEATURE_NAME_MAX, "%s:%u",
850 				 btrfs_feature_set_names[set], i);
851 
852 			fa->kobj_attr.attr.name = name;
853 			fa->kobj_attr.attr.mode = S_IRUGO;
854 			fa->feature_set = set;
855 			fa->feature_bit = 1ULL << i;
856 		}
857 	}
858 }
859 
860 /*
861  * Create a sysfs entry for a given block group type at path
862  * /sys/fs/btrfs/UUID/allocation/data/TYPE
863  */
864 void btrfs_sysfs_add_block_group_type(struct btrfs_block_group *cache)
865 {
866 	struct btrfs_fs_info *fs_info = cache->fs_info;
867 	struct btrfs_space_info *space_info = cache->space_info;
868 	struct raid_kobject *rkobj;
869 	const int index = btrfs_bg_flags_to_raid_index(cache->flags);
870 	unsigned int nofs_flag;
871 	int ret;
872 
873 	/*
874 	 * Setup a NOFS context because kobject_add(), deep in its call chain,
875 	 * does GFP_KERNEL allocations, and we are often called in a context
876 	 * where if reclaim is triggered we can deadlock (we are either holding
877 	 * a transaction handle or some lock required for a transaction
878 	 * commit).
879 	 */
880 	nofs_flag = memalloc_nofs_save();
881 
882 	rkobj = kzalloc(sizeof(*rkobj), GFP_NOFS);
883 	if (!rkobj) {
884 		memalloc_nofs_restore(nofs_flag);
885 		btrfs_warn(cache->fs_info,
886 				"couldn't alloc memory for raid level kobject");
887 		return;
888 	}
889 
890 	rkobj->flags = cache->flags;
891 	kobject_init(&rkobj->kobj, &btrfs_raid_ktype);
892 	ret = kobject_add(&rkobj->kobj, &space_info->kobj, "%s",
893 			  btrfs_bg_type_to_raid_name(rkobj->flags));
894 	memalloc_nofs_restore(nofs_flag);
895 	if (ret) {
896 		kobject_put(&rkobj->kobj);
897 		btrfs_warn(fs_info,
898 			"failed to add kobject for block cache, ignoring");
899 		return;
900 	}
901 
902 	space_info->block_group_kobjs[index] = &rkobj->kobj;
903 }
904 
905 /*
906  * Remove sysfs directories for all block group types of a given space info and
907  * the space info as well
908  */
909 void btrfs_sysfs_remove_space_info(struct btrfs_space_info *space_info)
910 {
911 	int i;
912 
913 	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
914 		struct kobject *kobj;
915 
916 		kobj = space_info->block_group_kobjs[i];
917 		space_info->block_group_kobjs[i] = NULL;
918 		if (kobj) {
919 			kobject_del(kobj);
920 			kobject_put(kobj);
921 		}
922 	}
923 	kobject_del(&space_info->kobj);
924 	kobject_put(&space_info->kobj);
925 }
926 
927 static const char *alloc_name(u64 flags)
928 {
929 	switch (flags) {
930 	case BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA:
931 		return "mixed";
932 	case BTRFS_BLOCK_GROUP_METADATA:
933 		return "metadata";
934 	case BTRFS_BLOCK_GROUP_DATA:
935 		return "data";
936 	case BTRFS_BLOCK_GROUP_SYSTEM:
937 		return "system";
938 	default:
939 		WARN_ON(1);
940 		return "invalid-combination";
941 	};
942 }
943 
944 /*
945  * Create a sysfs entry for a space info type at path
946  * /sys/fs/btrfs/UUID/allocation/TYPE
947  */
948 int btrfs_sysfs_add_space_info_type(struct btrfs_fs_info *fs_info,
949 				    struct btrfs_space_info *space_info)
950 {
951 	int ret;
952 
953 	ret = kobject_init_and_add(&space_info->kobj, &space_info_ktype,
954 				   fs_info->space_info_kobj, "%s",
955 				   alloc_name(space_info->flags));
956 	if (ret) {
957 		kobject_put(&space_info->kobj);
958 		return ret;
959 	}
960 
961 	return 0;
962 }
963 
964 /* when one_device is NULL, it removes all device links */
965 
966 int btrfs_sysfs_rm_device_link(struct btrfs_fs_devices *fs_devices,
967 		struct btrfs_device *one_device)
968 {
969 	struct hd_struct *disk;
970 	struct kobject *disk_kobj;
971 
972 	if (!fs_devices->device_dir_kobj)
973 		return -EINVAL;
974 
975 	if (one_device && one_device->bdev) {
976 		disk = one_device->bdev->bd_part;
977 		disk_kobj = &part_to_dev(disk)->kobj;
978 
979 		sysfs_remove_link(fs_devices->device_dir_kobj,
980 						disk_kobj->name);
981 	}
982 
983 	if (one_device)
984 		return 0;
985 
986 	list_for_each_entry(one_device,
987 			&fs_devices->devices, dev_list) {
988 		if (!one_device->bdev)
989 			continue;
990 		disk = one_device->bdev->bd_part;
991 		disk_kobj = &part_to_dev(disk)->kobj;
992 
993 		sysfs_remove_link(fs_devices->device_dir_kobj,
994 						disk_kobj->name);
995 	}
996 
997 	return 0;
998 }
999 
1000 int btrfs_sysfs_add_device(struct btrfs_fs_devices *fs_devs)
1001 {
1002 	if (!fs_devs->device_dir_kobj)
1003 		fs_devs->device_dir_kobj = kobject_create_and_add("devices",
1004 						&fs_devs->fsid_kobj);
1005 
1006 	if (!fs_devs->device_dir_kobj)
1007 		return -ENOMEM;
1008 
1009 	return 0;
1010 }
1011 
1012 int btrfs_sysfs_add_device_link(struct btrfs_fs_devices *fs_devices,
1013 				struct btrfs_device *one_device)
1014 {
1015 	int error = 0;
1016 	struct btrfs_device *dev;
1017 
1018 	list_for_each_entry(dev, &fs_devices->devices, dev_list) {
1019 		struct hd_struct *disk;
1020 		struct kobject *disk_kobj;
1021 
1022 		if (!dev->bdev)
1023 			continue;
1024 
1025 		if (one_device && one_device != dev)
1026 			continue;
1027 
1028 		disk = dev->bdev->bd_part;
1029 		disk_kobj = &part_to_dev(disk)->kobj;
1030 
1031 		error = sysfs_create_link(fs_devices->device_dir_kobj,
1032 					  disk_kobj, disk_kobj->name);
1033 		if (error)
1034 			break;
1035 	}
1036 
1037 	return error;
1038 }
1039 
1040 void btrfs_kobject_uevent(struct block_device *bdev, enum kobject_action action)
1041 {
1042 	int ret;
1043 
1044 	ret = kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, action);
1045 	if (ret)
1046 		pr_warn("BTRFS: Sending event '%d' to kobject: '%s' (%p): failed\n",
1047 			action, kobject_name(&disk_to_dev(bdev->bd_disk)->kobj),
1048 			&disk_to_dev(bdev->bd_disk)->kobj);
1049 }
1050 
1051 void btrfs_sysfs_update_sprout_fsid(struct btrfs_fs_devices *fs_devices,
1052 				    const u8 *fsid)
1053 {
1054 	char fsid_buf[BTRFS_UUID_UNPARSED_SIZE];
1055 
1056 	/*
1057 	 * Sprouting changes fsid of the mounted filesystem, rename the fsid
1058 	 * directory
1059 	 */
1060 	snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU", fsid);
1061 	if (kobject_rename(&fs_devices->fsid_kobj, fsid_buf))
1062 		btrfs_warn(fs_devices->fs_info,
1063 				"sysfs: failed to create fsid for sprout");
1064 }
1065 
1066 /* /sys/fs/btrfs/ entry */
1067 static struct kset *btrfs_kset;
1068 
1069 /*
1070  * Can be called by the device discovery thread.
1071  * And parent can be specified for seed device
1072  */
1073 int btrfs_sysfs_add_fsid(struct btrfs_fs_devices *fs_devs,
1074 				struct kobject *parent)
1075 {
1076 	int error;
1077 
1078 	init_completion(&fs_devs->kobj_unregister);
1079 	fs_devs->fsid_kobj.kset = btrfs_kset;
1080 	error = kobject_init_and_add(&fs_devs->fsid_kobj,
1081 				&btrfs_ktype, parent, "%pU", fs_devs->fsid);
1082 	if (error) {
1083 		kobject_put(&fs_devs->fsid_kobj);
1084 		return error;
1085 	}
1086 
1087 	return 0;
1088 }
1089 
1090 int btrfs_sysfs_add_mounted(struct btrfs_fs_info *fs_info)
1091 {
1092 	int error;
1093 	struct btrfs_fs_devices *fs_devs = fs_info->fs_devices;
1094 	struct kobject *fsid_kobj = &fs_devs->fsid_kobj;
1095 
1096 	btrfs_set_fs_info_ptr(fs_info);
1097 
1098 	error = btrfs_sysfs_add_device_link(fs_devs, NULL);
1099 	if (error)
1100 		return error;
1101 
1102 	error = sysfs_create_files(fsid_kobj, btrfs_attrs);
1103 	if (error) {
1104 		btrfs_sysfs_rm_device_link(fs_devs, NULL);
1105 		return error;
1106 	}
1107 
1108 	error = sysfs_create_group(fsid_kobj,
1109 				   &btrfs_feature_attr_group);
1110 	if (error)
1111 		goto failure;
1112 
1113 #ifdef CONFIG_BTRFS_DEBUG
1114 	error = sysfs_create_group(fsid_kobj,
1115 				   &btrfs_debug_feature_attr_group);
1116 	if (error)
1117 		goto failure;
1118 #endif
1119 
1120 	error = addrm_unknown_feature_attrs(fs_info, true);
1121 	if (error)
1122 		goto failure;
1123 
1124 	fs_info->space_info_kobj = kobject_create_and_add("allocation",
1125 						  fsid_kobj);
1126 	if (!fs_info->space_info_kobj) {
1127 		error = -ENOMEM;
1128 		goto failure;
1129 	}
1130 
1131 	error = sysfs_create_files(fs_info->space_info_kobj, allocation_attrs);
1132 	if (error)
1133 		goto failure;
1134 
1135 	return 0;
1136 failure:
1137 	btrfs_sysfs_remove_mounted(fs_info);
1138 	return error;
1139 }
1140 
1141 
1142 /*
1143  * Change per-fs features in /sys/fs/btrfs/UUID/features to match current
1144  * values in superblock. Call after any changes to incompat/compat_ro flags
1145  */
1146 void btrfs_sysfs_feature_update(struct btrfs_fs_info *fs_info,
1147 		u64 bit, enum btrfs_feature_set set)
1148 {
1149 	struct btrfs_fs_devices *fs_devs;
1150 	struct kobject *fsid_kobj;
1151 	u64 features;
1152 	int ret;
1153 
1154 	if (!fs_info)
1155 		return;
1156 
1157 	features = get_features(fs_info, set);
1158 	ASSERT(bit & supported_feature_masks[set]);
1159 
1160 	fs_devs = fs_info->fs_devices;
1161 	fsid_kobj = &fs_devs->fsid_kobj;
1162 
1163 	if (!fsid_kobj->state_initialized)
1164 		return;
1165 
1166 	/*
1167 	 * FIXME: this is too heavy to update just one value, ideally we'd like
1168 	 * to use sysfs_update_group but some refactoring is needed first.
1169 	 */
1170 	sysfs_remove_group(fsid_kobj, &btrfs_feature_attr_group);
1171 	ret = sysfs_create_group(fsid_kobj, &btrfs_feature_attr_group);
1172 }
1173 
1174 int __init btrfs_init_sysfs(void)
1175 {
1176 	int ret;
1177 
1178 	btrfs_kset = kset_create_and_add("btrfs", NULL, fs_kobj);
1179 	if (!btrfs_kset)
1180 		return -ENOMEM;
1181 
1182 	init_feature_attrs();
1183 	ret = sysfs_create_group(&btrfs_kset->kobj, &btrfs_feature_attr_group);
1184 	if (ret)
1185 		goto out2;
1186 	ret = sysfs_merge_group(&btrfs_kset->kobj,
1187 				&btrfs_static_feature_attr_group);
1188 	if (ret)
1189 		goto out_remove_group;
1190 
1191 #ifdef CONFIG_BTRFS_DEBUG
1192 	ret = sysfs_create_group(&btrfs_kset->kobj, &btrfs_debug_feature_attr_group);
1193 	if (ret)
1194 		goto out2;
1195 #endif
1196 
1197 	return 0;
1198 
1199 out_remove_group:
1200 	sysfs_remove_group(&btrfs_kset->kobj, &btrfs_feature_attr_group);
1201 out2:
1202 	kset_unregister(btrfs_kset);
1203 
1204 	return ret;
1205 }
1206 
1207 void __cold btrfs_exit_sysfs(void)
1208 {
1209 	sysfs_unmerge_group(&btrfs_kset->kobj,
1210 			    &btrfs_static_feature_attr_group);
1211 	sysfs_remove_group(&btrfs_kset->kobj, &btrfs_feature_attr_group);
1212 	kset_unregister(btrfs_kset);
1213 }
1214 
1215