xref: /openbmc/linux/fs/notify/mark.c (revision fc1b1e13)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Copyright (C) 2008 Red Hat, Inc., Eric Paris <eparis@redhat.com>
4  */
5 
6 /*
7  * fsnotify inode mark locking/lifetime/and refcnting
8  *
9  * REFCNT:
10  * The group->recnt and mark->refcnt tell how many "things" in the kernel
11  * currently are referencing the objects. Both kind of objects typically will
12  * live inside the kernel with a refcnt of 2, one for its creation and one for
13  * the reference a group and a mark hold to each other.
14  * If you are holding the appropriate locks, you can take a reference and the
15  * object itself is guaranteed to survive until the reference is dropped.
16  *
17  * LOCKING:
18  * There are 3 locks involved with fsnotify inode marks and they MUST be taken
19  * in order as follows:
20  *
21  * group->mark_mutex
22  * mark->lock
23  * mark->connector->lock
24  *
25  * group->mark_mutex protects the marks_list anchored inside a given group and
26  * each mark is hooked via the g_list.  It also protects the groups private
27  * data (i.e group limits).
28 
29  * mark->lock protects the marks attributes like its masks and flags.
30  * Furthermore it protects the access to a reference of the group that the mark
31  * is assigned to as well as the access to a reference of the inode/vfsmount
32  * that is being watched by the mark.
33  *
34  * mark->connector->lock protects the list of marks anchored inside an
35  * inode / vfsmount and each mark is hooked via the i_list.
36  *
37  * A list of notification marks relating to inode / mnt is contained in
38  * fsnotify_mark_connector. That structure is alive as long as there are any
39  * marks in the list and is also protected by fsnotify_mark_srcu. A mark gets
40  * detached from fsnotify_mark_connector when last reference to the mark is
41  * dropped.  Thus having mark reference is enough to protect mark->connector
42  * pointer and to make sure fsnotify_mark_connector cannot disappear. Also
43  * because we remove mark from g_list before dropping mark reference associated
44  * with that, any mark found through g_list is guaranteed to have
45  * mark->connector set until we drop group->mark_mutex.
46  *
47  * LIFETIME:
48  * Inode marks survive between when they are added to an inode and when their
49  * refcnt==0. Marks are also protected by fsnotify_mark_srcu.
50  *
51  * The inode mark can be cleared for a number of different reasons including:
52  * - The inode is unlinked for the last time.  (fsnotify_inode_remove)
53  * - The inode is being evicted from cache. (fsnotify_inode_delete)
54  * - The fs the inode is on is unmounted.  (fsnotify_inode_delete/fsnotify_unmount_inodes)
55  * - Something explicitly requests that it be removed.  (fsnotify_destroy_mark)
56  * - The fsnotify_group associated with the mark is going away and all such marks
57  *   need to be cleaned up. (fsnotify_clear_marks_by_group)
58  *
59  * This has the very interesting property of being able to run concurrently with
60  * any (or all) other directions.
61  */
62 
63 #include <linux/fs.h>
64 #include <linux/init.h>
65 #include <linux/kernel.h>
66 #include <linux/kthread.h>
67 #include <linux/module.h>
68 #include <linux/mutex.h>
69 #include <linux/slab.h>
70 #include <linux/spinlock.h>
71 #include <linux/srcu.h>
72 #include <linux/ratelimit.h>
73 
74 #include <linux/atomic.h>
75 
76 #include <linux/fsnotify_backend.h>
77 #include "fsnotify.h"
78 
79 #define FSNOTIFY_REAPER_DELAY	(1)	/* 1 jiffy */
80 
81 struct srcu_struct fsnotify_mark_srcu;
82 struct kmem_cache *fsnotify_mark_connector_cachep;
83 
84 static DEFINE_SPINLOCK(destroy_lock);
85 static LIST_HEAD(destroy_list);
86 static struct fsnotify_mark_connector *connector_destroy_list;
87 
88 static void fsnotify_mark_destroy_workfn(struct work_struct *work);
89 static DECLARE_DELAYED_WORK(reaper_work, fsnotify_mark_destroy_workfn);
90 
91 static void fsnotify_connector_destroy_workfn(struct work_struct *work);
92 static DECLARE_WORK(connector_reaper_work, fsnotify_connector_destroy_workfn);
93 
fsnotify_get_mark(struct fsnotify_mark * mark)94 void fsnotify_get_mark(struct fsnotify_mark *mark)
95 {
96 	WARN_ON_ONCE(!refcount_read(&mark->refcnt));
97 	refcount_inc(&mark->refcnt);
98 }
99 
fsnotify_conn_mask_p(struct fsnotify_mark_connector * conn)100 static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn)
101 {
102 	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)
103 		return &fsnotify_conn_inode(conn)->i_fsnotify_mask;
104 	else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT)
105 		return &fsnotify_conn_mount(conn)->mnt_fsnotify_mask;
106 	else if (conn->type == FSNOTIFY_OBJ_TYPE_SB)
107 		return &fsnotify_conn_sb(conn)->s_fsnotify_mask;
108 	return NULL;
109 }
110 
fsnotify_conn_mask(struct fsnotify_mark_connector * conn)111 __u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn)
112 {
113 	if (WARN_ON(!fsnotify_valid_obj_type(conn->type)))
114 		return 0;
115 
116 	return *fsnotify_conn_mask_p(conn);
117 }
118 
fsnotify_get_inode_ref(struct inode * inode)119 static void fsnotify_get_inode_ref(struct inode *inode)
120 {
121 	ihold(inode);
122 	atomic_long_inc(&inode->i_sb->s_fsnotify_connectors);
123 }
124 
125 /*
126  * Grab or drop inode reference for the connector if needed.
127  *
128  * When it's time to drop the reference, we only clear the HAS_IREF flag and
129  * return the inode object. fsnotify_drop_object() will be resonsible for doing
130  * iput() outside of spinlocks. This happens when last mark that wanted iref is
131  * detached.
132  */
fsnotify_update_iref(struct fsnotify_mark_connector * conn,bool want_iref)133 static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn,
134 					  bool want_iref)
135 {
136 	bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;
137 	struct inode *inode = NULL;
138 
139 	if (conn->type != FSNOTIFY_OBJ_TYPE_INODE ||
140 	    want_iref == has_iref)
141 		return NULL;
142 
143 	if (want_iref) {
144 		/* Pin inode if any mark wants inode refcount held */
145 		fsnotify_get_inode_ref(fsnotify_conn_inode(conn));
146 		conn->flags |= FSNOTIFY_CONN_FLAG_HAS_IREF;
147 	} else {
148 		/* Unpin inode after detach of last mark that wanted iref */
149 		inode = fsnotify_conn_inode(conn);
150 		conn->flags &= ~FSNOTIFY_CONN_FLAG_HAS_IREF;
151 	}
152 
153 	return inode;
154 }
155 
__fsnotify_recalc_mask(struct fsnotify_mark_connector * conn)156 static void *__fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
157 {
158 	u32 new_mask = 0;
159 	bool want_iref = false;
160 	struct fsnotify_mark *mark;
161 
162 	assert_spin_locked(&conn->lock);
163 	/* We can get detached connector here when inode is getting unlinked. */
164 	if (!fsnotify_valid_obj_type(conn->type))
165 		return NULL;
166 	hlist_for_each_entry(mark, &conn->list, obj_list) {
167 		if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED))
168 			continue;
169 		new_mask |= fsnotify_calc_mask(mark);
170 		if (conn->type == FSNOTIFY_OBJ_TYPE_INODE &&
171 		    !(mark->flags & FSNOTIFY_MARK_FLAG_NO_IREF))
172 			want_iref = true;
173 	}
174 	*fsnotify_conn_mask_p(conn) = new_mask;
175 
176 	return fsnotify_update_iref(conn, want_iref);
177 }
178 
fsnotify_conn_watches_children(struct fsnotify_mark_connector * conn)179 static bool fsnotify_conn_watches_children(
180 					struct fsnotify_mark_connector *conn)
181 {
182 	if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
183 		return false;
184 
185 	return fsnotify_inode_watches_children(fsnotify_conn_inode(conn));
186 }
187 
fsnotify_conn_set_children_dentry_flags(struct fsnotify_mark_connector * conn)188 static void fsnotify_conn_set_children_dentry_flags(
189 					struct fsnotify_mark_connector *conn)
190 {
191 	if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
192 		return;
193 
194 	fsnotify_set_children_dentry_flags(fsnotify_conn_inode(conn));
195 }
196 
197 /*
198  * Calculate mask of events for a list of marks. The caller must make sure
199  * connector and connector->obj cannot disappear under us.  Callers achieve
200  * this by holding a mark->lock or mark->group->mark_mutex for a mark on this
201  * list.
202  */
fsnotify_recalc_mask(struct fsnotify_mark_connector * conn)203 void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
204 {
205 	bool update_children;
206 
207 	if (!conn)
208 		return;
209 
210 	spin_lock(&conn->lock);
211 	update_children = !fsnotify_conn_watches_children(conn);
212 	__fsnotify_recalc_mask(conn);
213 	update_children &= fsnotify_conn_watches_children(conn);
214 	spin_unlock(&conn->lock);
215 	/*
216 	 * Set children's PARENT_WATCHED flags only if parent started watching.
217 	 * When parent stops watching, we clear false positive PARENT_WATCHED
218 	 * flags lazily in __fsnotify_parent().
219 	 */
220 	if (update_children)
221 		fsnotify_conn_set_children_dentry_flags(conn);
222 }
223 
224 /* Free all connectors queued for freeing once SRCU period ends */
fsnotify_connector_destroy_workfn(struct work_struct * work)225 static void fsnotify_connector_destroy_workfn(struct work_struct *work)
226 {
227 	struct fsnotify_mark_connector *conn, *free;
228 
229 	spin_lock(&destroy_lock);
230 	conn = connector_destroy_list;
231 	connector_destroy_list = NULL;
232 	spin_unlock(&destroy_lock);
233 
234 	synchronize_srcu(&fsnotify_mark_srcu);
235 	while (conn) {
236 		free = conn;
237 		conn = conn->destroy_next;
238 		kmem_cache_free(fsnotify_mark_connector_cachep, free);
239 	}
240 }
241 
fsnotify_put_inode_ref(struct inode * inode)242 static void fsnotify_put_inode_ref(struct inode *inode)
243 {
244 	struct super_block *sb = inode->i_sb;
245 
246 	iput(inode);
247 	if (atomic_long_dec_and_test(&sb->s_fsnotify_connectors))
248 		wake_up_var(&sb->s_fsnotify_connectors);
249 }
250 
fsnotify_get_sb_connectors(struct fsnotify_mark_connector * conn)251 static void fsnotify_get_sb_connectors(struct fsnotify_mark_connector *conn)
252 {
253 	struct super_block *sb = fsnotify_connector_sb(conn);
254 
255 	if (sb)
256 		atomic_long_inc(&sb->s_fsnotify_connectors);
257 }
258 
fsnotify_put_sb_connectors(struct fsnotify_mark_connector * conn)259 static void fsnotify_put_sb_connectors(struct fsnotify_mark_connector *conn)
260 {
261 	struct super_block *sb = fsnotify_connector_sb(conn);
262 
263 	if (sb && atomic_long_dec_and_test(&sb->s_fsnotify_connectors))
264 		wake_up_var(&sb->s_fsnotify_connectors);
265 }
266 
fsnotify_detach_connector_from_object(struct fsnotify_mark_connector * conn,unsigned int * type)267 static void *fsnotify_detach_connector_from_object(
268 					struct fsnotify_mark_connector *conn,
269 					unsigned int *type)
270 {
271 	struct inode *inode = NULL;
272 
273 	*type = conn->type;
274 	if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED)
275 		return NULL;
276 
277 	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {
278 		inode = fsnotify_conn_inode(conn);
279 		inode->i_fsnotify_mask = 0;
280 
281 		/* Unpin inode when detaching from connector */
282 		if (!(conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF))
283 			inode = NULL;
284 	} else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
285 		fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0;
286 	} else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) {
287 		fsnotify_conn_sb(conn)->s_fsnotify_mask = 0;
288 	}
289 
290 	fsnotify_put_sb_connectors(conn);
291 	rcu_assign_pointer(*(conn->obj), NULL);
292 	conn->obj = NULL;
293 	conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;
294 
295 	return inode;
296 }
297 
fsnotify_final_mark_destroy(struct fsnotify_mark * mark)298 static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)
299 {
300 	struct fsnotify_group *group = mark->group;
301 
302 	if (WARN_ON_ONCE(!group))
303 		return;
304 	group->ops->free_mark(mark);
305 	fsnotify_put_group(group);
306 }
307 
308 /* Drop object reference originally held by a connector */
fsnotify_drop_object(unsigned int type,void * objp)309 static void fsnotify_drop_object(unsigned int type, void *objp)
310 {
311 	if (!objp)
312 		return;
313 	/* Currently only inode references are passed to be dropped */
314 	if (WARN_ON_ONCE(type != FSNOTIFY_OBJ_TYPE_INODE))
315 		return;
316 	fsnotify_put_inode_ref(objp);
317 }
318 
fsnotify_put_mark(struct fsnotify_mark * mark)319 void fsnotify_put_mark(struct fsnotify_mark *mark)
320 {
321 	struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector);
322 	void *objp = NULL;
323 	unsigned int type = FSNOTIFY_OBJ_TYPE_DETACHED;
324 	bool free_conn = false;
325 
326 	/* Catch marks that were actually never attached to object */
327 	if (!conn) {
328 		if (refcount_dec_and_test(&mark->refcnt))
329 			fsnotify_final_mark_destroy(mark);
330 		return;
331 	}
332 
333 	/*
334 	 * We have to be careful so that traversals of obj_list under lock can
335 	 * safely grab mark reference.
336 	 */
337 	if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock))
338 		return;
339 
340 	hlist_del_init_rcu(&mark->obj_list);
341 	if (hlist_empty(&conn->list)) {
342 		objp = fsnotify_detach_connector_from_object(conn, &type);
343 		free_conn = true;
344 	} else {
345 		objp = __fsnotify_recalc_mask(conn);
346 		type = conn->type;
347 	}
348 	WRITE_ONCE(mark->connector, NULL);
349 	spin_unlock(&conn->lock);
350 
351 	fsnotify_drop_object(type, objp);
352 
353 	if (free_conn) {
354 		spin_lock(&destroy_lock);
355 		conn->destroy_next = connector_destroy_list;
356 		connector_destroy_list = conn;
357 		spin_unlock(&destroy_lock);
358 		queue_work(system_unbound_wq, &connector_reaper_work);
359 	}
360 	/*
361 	 * Note that we didn't update flags telling whether inode cares about
362 	 * what's happening with children. We update these flags from
363 	 * __fsnotify_parent() lazily when next event happens on one of our
364 	 * children.
365 	 */
366 	spin_lock(&destroy_lock);
367 	list_add(&mark->g_list, &destroy_list);
368 	spin_unlock(&destroy_lock);
369 	queue_delayed_work(system_unbound_wq, &reaper_work,
370 			   FSNOTIFY_REAPER_DELAY);
371 }
372 EXPORT_SYMBOL_GPL(fsnotify_put_mark);
373 
374 /*
375  * Get mark reference when we found the mark via lockless traversal of object
376  * list. Mark can be already removed from the list by now and on its way to be
377  * destroyed once SRCU period ends.
378  *
379  * Also pin the group so it doesn't disappear under us.
380  */
fsnotify_get_mark_safe(struct fsnotify_mark * mark)381 static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark)
382 {
383 	if (!mark)
384 		return true;
385 
386 	if (refcount_inc_not_zero(&mark->refcnt)) {
387 		spin_lock(&mark->lock);
388 		if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) {
389 			/* mark is attached, group is still alive then */
390 			atomic_inc(&mark->group->user_waits);
391 			spin_unlock(&mark->lock);
392 			return true;
393 		}
394 		spin_unlock(&mark->lock);
395 		fsnotify_put_mark(mark);
396 	}
397 	return false;
398 }
399 
400 /*
401  * Puts marks and wakes up group destruction if necessary.
402  *
403  * Pairs with fsnotify_get_mark_safe()
404  */
fsnotify_put_mark_wake(struct fsnotify_mark * mark)405 static void fsnotify_put_mark_wake(struct fsnotify_mark *mark)
406 {
407 	if (mark) {
408 		struct fsnotify_group *group = mark->group;
409 
410 		fsnotify_put_mark(mark);
411 		/*
412 		 * We abuse notification_waitq on group shutdown for waiting for
413 		 * all marks pinned when waiting for userspace.
414 		 */
415 		if (atomic_dec_and_test(&group->user_waits) && group->shutdown)
416 			wake_up(&group->notification_waitq);
417 	}
418 }
419 
fsnotify_prepare_user_wait(struct fsnotify_iter_info * iter_info)420 bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info)
421 	__releases(&fsnotify_mark_srcu)
422 {
423 	int type;
424 
425 	fsnotify_foreach_iter_type(type) {
426 		/* This can fail if mark is being removed */
427 		if (!fsnotify_get_mark_safe(iter_info->marks[type])) {
428 			__release(&fsnotify_mark_srcu);
429 			goto fail;
430 		}
431 	}
432 
433 	/*
434 	 * Now that both marks are pinned by refcount in the inode / vfsmount
435 	 * lists, we can drop SRCU lock, and safely resume the list iteration
436 	 * once userspace returns.
437 	 */
438 	srcu_read_unlock(&fsnotify_mark_srcu, iter_info->srcu_idx);
439 
440 	return true;
441 
442 fail:
443 	for (type--; type >= 0; type--)
444 		fsnotify_put_mark_wake(iter_info->marks[type]);
445 	return false;
446 }
447 
fsnotify_finish_user_wait(struct fsnotify_iter_info * iter_info)448 void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info)
449 	__acquires(&fsnotify_mark_srcu)
450 {
451 	int type;
452 
453 	iter_info->srcu_idx = srcu_read_lock(&fsnotify_mark_srcu);
454 	fsnotify_foreach_iter_type(type)
455 		fsnotify_put_mark_wake(iter_info->marks[type]);
456 }
457 
458 /*
459  * Mark mark as detached, remove it from group list. Mark still stays in object
460  * list until its last reference is dropped. Note that we rely on mark being
461  * removed from group list before corresponding reference to it is dropped. In
462  * particular we rely on mark->connector being valid while we hold
463  * group->mark_mutex if we found the mark through g_list.
464  *
465  * Must be called with group->mark_mutex held. The caller must either hold
466  * reference to the mark or be protected by fsnotify_mark_srcu.
467  */
fsnotify_detach_mark(struct fsnotify_mark * mark)468 void fsnotify_detach_mark(struct fsnotify_mark *mark)
469 {
470 	fsnotify_group_assert_locked(mark->group);
471 	WARN_ON_ONCE(!srcu_read_lock_held(&fsnotify_mark_srcu) &&
472 		     refcount_read(&mark->refcnt) < 1 +
473 			!!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED));
474 
475 	spin_lock(&mark->lock);
476 	/* something else already called this function on this mark */
477 	if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
478 		spin_unlock(&mark->lock);
479 		return;
480 	}
481 	mark->flags &= ~FSNOTIFY_MARK_FLAG_ATTACHED;
482 	list_del_init(&mark->g_list);
483 	spin_unlock(&mark->lock);
484 
485 	/* Drop mark reference acquired in fsnotify_add_mark_locked() */
486 	fsnotify_put_mark(mark);
487 }
488 
489 /*
490  * Free fsnotify mark. The mark is actually only marked as being freed.  The
491  * freeing is actually happening only once last reference to the mark is
492  * dropped from a workqueue which first waits for srcu period end.
493  *
494  * Caller must have a reference to the mark or be protected by
495  * fsnotify_mark_srcu.
496  */
fsnotify_free_mark(struct fsnotify_mark * mark)497 void fsnotify_free_mark(struct fsnotify_mark *mark)
498 {
499 	struct fsnotify_group *group = mark->group;
500 
501 	spin_lock(&mark->lock);
502 	/* something else already called this function on this mark */
503 	if (!(mark->flags & FSNOTIFY_MARK_FLAG_ALIVE)) {
504 		spin_unlock(&mark->lock);
505 		return;
506 	}
507 	mark->flags &= ~FSNOTIFY_MARK_FLAG_ALIVE;
508 	spin_unlock(&mark->lock);
509 
510 	/*
511 	 * Some groups like to know that marks are being freed.  This is a
512 	 * callback to the group function to let it know that this mark
513 	 * is being freed.
514 	 */
515 	if (group->ops->freeing_mark)
516 		group->ops->freeing_mark(mark, group);
517 }
518 
fsnotify_destroy_mark(struct fsnotify_mark * mark,struct fsnotify_group * group)519 void fsnotify_destroy_mark(struct fsnotify_mark *mark,
520 			   struct fsnotify_group *group)
521 {
522 	fsnotify_group_lock(group);
523 	fsnotify_detach_mark(mark);
524 	fsnotify_group_unlock(group);
525 	fsnotify_free_mark(mark);
526 }
527 EXPORT_SYMBOL_GPL(fsnotify_destroy_mark);
528 
529 /*
530  * Sorting function for lists of fsnotify marks.
531  *
532  * Fanotify supports different notification classes (reflected as priority of
533  * notification group). Events shall be passed to notification groups in
534  * decreasing priority order. To achieve this marks in notification lists for
535  * inodes and vfsmounts are sorted so that priorities of corresponding groups
536  * are descending.
537  *
538  * Furthermore correct handling of the ignore mask requires processing inode
539  * and vfsmount marks of each group together. Using the group address as
540  * further sort criterion provides a unique sorting order and thus we can
541  * merge inode and vfsmount lists of marks in linear time and find groups
542  * present in both lists.
543  *
544  * A return value of 1 signifies that b has priority over a.
545  * A return value of 0 signifies that the two marks have to be handled together.
546  * A return value of -1 signifies that a has priority over b.
547  */
fsnotify_compare_groups(struct fsnotify_group * a,struct fsnotify_group * b)548 int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b)
549 {
550 	if (a == b)
551 		return 0;
552 	if (!a)
553 		return 1;
554 	if (!b)
555 		return -1;
556 	if (a->priority < b->priority)
557 		return 1;
558 	if (a->priority > b->priority)
559 		return -1;
560 	if (a < b)
561 		return 1;
562 	return -1;
563 }
564 
fsnotify_attach_connector_to_object(fsnotify_connp_t * connp,unsigned int obj_type,__kernel_fsid_t * fsid)565 static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
566 					       unsigned int obj_type,
567 					       __kernel_fsid_t *fsid)
568 {
569 	struct fsnotify_mark_connector *conn;
570 
571 	conn = kmem_cache_alloc(fsnotify_mark_connector_cachep, GFP_KERNEL);
572 	if (!conn)
573 		return -ENOMEM;
574 	spin_lock_init(&conn->lock);
575 	INIT_HLIST_HEAD(&conn->list);
576 	conn->flags = 0;
577 	conn->type = obj_type;
578 	conn->obj = connp;
579 	/* Cache fsid of filesystem containing the object */
580 	if (fsid) {
581 		conn->fsid = *fsid;
582 		conn->flags = FSNOTIFY_CONN_FLAG_HAS_FSID;
583 	} else {
584 		conn->fsid.val[0] = conn->fsid.val[1] = 0;
585 		conn->flags = 0;
586 	}
587 	fsnotify_get_sb_connectors(conn);
588 
589 	/*
590 	 * cmpxchg() provides the barrier so that readers of *connp can see
591 	 * only initialized structure
592 	 */
593 	if (cmpxchg(connp, NULL, conn)) {
594 		/* Someone else created list structure for us */
595 		fsnotify_put_sb_connectors(conn);
596 		kmem_cache_free(fsnotify_mark_connector_cachep, conn);
597 	}
598 
599 	return 0;
600 }
601 
602 /*
603  * Get mark connector, make sure it is alive and return with its lock held.
604  * This is for users that get connector pointer from inode or mount. Users that
605  * hold reference to a mark on the list may directly lock connector->lock as
606  * they are sure list cannot go away under them.
607  */
fsnotify_grab_connector(fsnotify_connp_t * connp)608 static struct fsnotify_mark_connector *fsnotify_grab_connector(
609 						fsnotify_connp_t *connp)
610 {
611 	struct fsnotify_mark_connector *conn;
612 	int idx;
613 
614 	idx = srcu_read_lock(&fsnotify_mark_srcu);
615 	conn = srcu_dereference(*connp, &fsnotify_mark_srcu);
616 	if (!conn)
617 		goto out;
618 	spin_lock(&conn->lock);
619 	if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) {
620 		spin_unlock(&conn->lock);
621 		srcu_read_unlock(&fsnotify_mark_srcu, idx);
622 		return NULL;
623 	}
624 out:
625 	srcu_read_unlock(&fsnotify_mark_srcu, idx);
626 	return conn;
627 }
628 
629 /*
630  * Add mark into proper place in given list of marks. These marks may be used
631  * for the fsnotify backend to determine which event types should be delivered
632  * to which group and for which inodes. These marks are ordered according to
633  * priority, highest number first, and then by the group's location in memory.
634  */
fsnotify_add_mark_list(struct fsnotify_mark * mark,fsnotify_connp_t * connp,unsigned int obj_type,int add_flags,__kernel_fsid_t * fsid)635 static int fsnotify_add_mark_list(struct fsnotify_mark *mark,
636 				  fsnotify_connp_t *connp,
637 				  unsigned int obj_type,
638 				  int add_flags, __kernel_fsid_t *fsid)
639 {
640 	struct fsnotify_mark *lmark, *last = NULL;
641 	struct fsnotify_mark_connector *conn;
642 	int cmp;
643 	int err = 0;
644 
645 	if (WARN_ON(!fsnotify_valid_obj_type(obj_type)))
646 		return -EINVAL;
647 
648 	/* Backend is expected to check for zero fsid (e.g. tmpfs) */
649 	if (fsid && WARN_ON_ONCE(!fsid->val[0] && !fsid->val[1]))
650 		return -ENODEV;
651 
652 restart:
653 	spin_lock(&mark->lock);
654 	conn = fsnotify_grab_connector(connp);
655 	if (!conn) {
656 		spin_unlock(&mark->lock);
657 		err = fsnotify_attach_connector_to_object(connp, obj_type,
658 							  fsid);
659 		if (err)
660 			return err;
661 		goto restart;
662 	} else if (fsid && !(conn->flags & FSNOTIFY_CONN_FLAG_HAS_FSID)) {
663 		conn->fsid = *fsid;
664 		/* Pairs with smp_rmb() in fanotify_get_fsid() */
665 		smp_wmb();
666 		conn->flags |= FSNOTIFY_CONN_FLAG_HAS_FSID;
667 	} else if (fsid && (conn->flags & FSNOTIFY_CONN_FLAG_HAS_FSID) &&
668 		   (fsid->val[0] != conn->fsid.val[0] ||
669 		    fsid->val[1] != conn->fsid.val[1])) {
670 		/*
671 		 * Backend is expected to check for non uniform fsid
672 		 * (e.g. btrfs), but maybe we missed something?
673 		 * Only allow setting conn->fsid once to non zero fsid.
674 		 * inotify and non-fid fanotify groups do not set nor test
675 		 * conn->fsid.
676 		 */
677 		pr_warn_ratelimited("%s: fsid mismatch on object of type %u: "
678 				    "%x.%x != %x.%x\n", __func__, conn->type,
679 				    fsid->val[0], fsid->val[1],
680 				    conn->fsid.val[0], conn->fsid.val[1]);
681 		err = -EXDEV;
682 		goto out_err;
683 	}
684 
685 	/* is mark the first mark? */
686 	if (hlist_empty(&conn->list)) {
687 		hlist_add_head_rcu(&mark->obj_list, &conn->list);
688 		goto added;
689 	}
690 
691 	/* should mark be in the middle of the current list? */
692 	hlist_for_each_entry(lmark, &conn->list, obj_list) {
693 		last = lmark;
694 
695 		if ((lmark->group == mark->group) &&
696 		    (lmark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) &&
697 		    !(mark->group->flags & FSNOTIFY_GROUP_DUPS)) {
698 			err = -EEXIST;
699 			goto out_err;
700 		}
701 
702 		cmp = fsnotify_compare_groups(lmark->group, mark->group);
703 		if (cmp >= 0) {
704 			hlist_add_before_rcu(&mark->obj_list, &lmark->obj_list);
705 			goto added;
706 		}
707 	}
708 
709 	BUG_ON(last == NULL);
710 	/* mark should be the last entry.  last is the current last entry */
711 	hlist_add_behind_rcu(&mark->obj_list, &last->obj_list);
712 added:
713 	/*
714 	 * Since connector is attached to object using cmpxchg() we are
715 	 * guaranteed that connector initialization is fully visible by anyone
716 	 * seeing mark->connector set.
717 	 */
718 	WRITE_ONCE(mark->connector, conn);
719 out_err:
720 	spin_unlock(&conn->lock);
721 	spin_unlock(&mark->lock);
722 	return err;
723 }
724 
725 /*
726  * Attach an initialized mark to a given group and fs object.
727  * These marks may be used for the fsnotify backend to determine which
728  * event types should be delivered to which group.
729  */
fsnotify_add_mark_locked(struct fsnotify_mark * mark,fsnotify_connp_t * connp,unsigned int obj_type,int add_flags,__kernel_fsid_t * fsid)730 int fsnotify_add_mark_locked(struct fsnotify_mark *mark,
731 			     fsnotify_connp_t *connp, unsigned int obj_type,
732 			     int add_flags, __kernel_fsid_t *fsid)
733 {
734 	struct fsnotify_group *group = mark->group;
735 	int ret = 0;
736 
737 	fsnotify_group_assert_locked(group);
738 
739 	/*
740 	 * LOCKING ORDER!!!!
741 	 * group->mark_mutex
742 	 * mark->lock
743 	 * mark->connector->lock
744 	 */
745 	spin_lock(&mark->lock);
746 	mark->flags |= FSNOTIFY_MARK_FLAG_ALIVE | FSNOTIFY_MARK_FLAG_ATTACHED;
747 
748 	list_add(&mark->g_list, &group->marks_list);
749 	fsnotify_get_mark(mark); /* for g_list */
750 	spin_unlock(&mark->lock);
751 
752 	ret = fsnotify_add_mark_list(mark, connp, obj_type, add_flags, fsid);
753 	if (ret)
754 		goto err;
755 
756 	fsnotify_recalc_mask(mark->connector);
757 
758 	return ret;
759 err:
760 	spin_lock(&mark->lock);
761 	mark->flags &= ~(FSNOTIFY_MARK_FLAG_ALIVE |
762 			 FSNOTIFY_MARK_FLAG_ATTACHED);
763 	list_del_init(&mark->g_list);
764 	spin_unlock(&mark->lock);
765 
766 	fsnotify_put_mark(mark);
767 	return ret;
768 }
769 
fsnotify_add_mark(struct fsnotify_mark * mark,fsnotify_connp_t * connp,unsigned int obj_type,int add_flags,__kernel_fsid_t * fsid)770 int fsnotify_add_mark(struct fsnotify_mark *mark, fsnotify_connp_t *connp,
771 		      unsigned int obj_type, int add_flags,
772 		      __kernel_fsid_t *fsid)
773 {
774 	int ret;
775 	struct fsnotify_group *group = mark->group;
776 
777 	fsnotify_group_lock(group);
778 	ret = fsnotify_add_mark_locked(mark, connp, obj_type, add_flags, fsid);
779 	fsnotify_group_unlock(group);
780 	return ret;
781 }
782 EXPORT_SYMBOL_GPL(fsnotify_add_mark);
783 
784 /*
785  * Given a list of marks, find the mark associated with given group. If found
786  * take a reference to that mark and return it, else return NULL.
787  */
fsnotify_find_mark(fsnotify_connp_t * connp,struct fsnotify_group * group)788 struct fsnotify_mark *fsnotify_find_mark(fsnotify_connp_t *connp,
789 					 struct fsnotify_group *group)
790 {
791 	struct fsnotify_mark_connector *conn;
792 	struct fsnotify_mark *mark;
793 
794 	conn = fsnotify_grab_connector(connp);
795 	if (!conn)
796 		return NULL;
797 
798 	hlist_for_each_entry(mark, &conn->list, obj_list) {
799 		if (mark->group == group &&
800 		    (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
801 			fsnotify_get_mark(mark);
802 			spin_unlock(&conn->lock);
803 			return mark;
804 		}
805 	}
806 	spin_unlock(&conn->lock);
807 	return NULL;
808 }
809 EXPORT_SYMBOL_GPL(fsnotify_find_mark);
810 
811 /* Clear any marks in a group with given type mask */
fsnotify_clear_marks_by_group(struct fsnotify_group * group,unsigned int obj_type)812 void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
813 				   unsigned int obj_type)
814 {
815 	struct fsnotify_mark *lmark, *mark;
816 	LIST_HEAD(to_free);
817 	struct list_head *head = &to_free;
818 
819 	/* Skip selection step if we want to clear all marks. */
820 	if (obj_type == FSNOTIFY_OBJ_TYPE_ANY) {
821 		head = &group->marks_list;
822 		goto clear;
823 	}
824 	/*
825 	 * We have to be really careful here. Anytime we drop mark_mutex, e.g.
826 	 * fsnotify_clear_marks_by_inode() can come and free marks. Even in our
827 	 * to_free list so we have to use mark_mutex even when accessing that
828 	 * list. And freeing mark requires us to drop mark_mutex. So we can
829 	 * reliably free only the first mark in the list. That's why we first
830 	 * move marks to free to to_free list in one go and then free marks in
831 	 * to_free list one by one.
832 	 */
833 	fsnotify_group_lock(group);
834 	list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) {
835 		if (mark->connector->type == obj_type)
836 			list_move(&mark->g_list, &to_free);
837 	}
838 	fsnotify_group_unlock(group);
839 
840 clear:
841 	while (1) {
842 		fsnotify_group_lock(group);
843 		if (list_empty(head)) {
844 			fsnotify_group_unlock(group);
845 			break;
846 		}
847 		mark = list_first_entry(head, struct fsnotify_mark, g_list);
848 		fsnotify_get_mark(mark);
849 		fsnotify_detach_mark(mark);
850 		fsnotify_group_unlock(group);
851 		fsnotify_free_mark(mark);
852 		fsnotify_put_mark(mark);
853 	}
854 }
855 
856 /* Destroy all marks attached to an object via connector */
fsnotify_destroy_marks(fsnotify_connp_t * connp)857 void fsnotify_destroy_marks(fsnotify_connp_t *connp)
858 {
859 	struct fsnotify_mark_connector *conn;
860 	struct fsnotify_mark *mark, *old_mark = NULL;
861 	void *objp;
862 	unsigned int type;
863 
864 	conn = fsnotify_grab_connector(connp);
865 	if (!conn)
866 		return;
867 	/*
868 	 * We have to be careful since we can race with e.g.
869 	 * fsnotify_clear_marks_by_group() and once we drop the conn->lock, the
870 	 * list can get modified. However we are holding mark reference and
871 	 * thus our mark cannot be removed from obj_list so we can continue
872 	 * iteration after regaining conn->lock.
873 	 */
874 	hlist_for_each_entry(mark, &conn->list, obj_list) {
875 		fsnotify_get_mark(mark);
876 		spin_unlock(&conn->lock);
877 		if (old_mark)
878 			fsnotify_put_mark(old_mark);
879 		old_mark = mark;
880 		fsnotify_destroy_mark(mark, mark->group);
881 		spin_lock(&conn->lock);
882 	}
883 	/*
884 	 * Detach list from object now so that we don't pin inode until all
885 	 * mark references get dropped. It would lead to strange results such
886 	 * as delaying inode deletion or blocking unmount.
887 	 */
888 	objp = fsnotify_detach_connector_from_object(conn, &type);
889 	spin_unlock(&conn->lock);
890 	if (old_mark)
891 		fsnotify_put_mark(old_mark);
892 	fsnotify_drop_object(type, objp);
893 }
894 
895 /*
896  * Nothing fancy, just initialize lists and locks and counters.
897  */
fsnotify_init_mark(struct fsnotify_mark * mark,struct fsnotify_group * group)898 void fsnotify_init_mark(struct fsnotify_mark *mark,
899 			struct fsnotify_group *group)
900 {
901 	memset(mark, 0, sizeof(*mark));
902 	spin_lock_init(&mark->lock);
903 	refcount_set(&mark->refcnt, 1);
904 	fsnotify_get_group(group);
905 	mark->group = group;
906 	WRITE_ONCE(mark->connector, NULL);
907 }
908 EXPORT_SYMBOL_GPL(fsnotify_init_mark);
909 
910 /*
911  * Destroy all marks in destroy_list, waits for SRCU period to finish before
912  * actually freeing marks.
913  */
fsnotify_mark_destroy_workfn(struct work_struct * work)914 static void fsnotify_mark_destroy_workfn(struct work_struct *work)
915 {
916 	struct fsnotify_mark *mark, *next;
917 	struct list_head private_destroy_list;
918 
919 	spin_lock(&destroy_lock);
920 	/* exchange the list head */
921 	list_replace_init(&destroy_list, &private_destroy_list);
922 	spin_unlock(&destroy_lock);
923 
924 	synchronize_srcu(&fsnotify_mark_srcu);
925 
926 	list_for_each_entry_safe(mark, next, &private_destroy_list, g_list) {
927 		list_del_init(&mark->g_list);
928 		fsnotify_final_mark_destroy(mark);
929 	}
930 }
931 
932 /* Wait for all marks queued for destruction to be actually destroyed */
fsnotify_wait_marks_destroyed(void)933 void fsnotify_wait_marks_destroyed(void)
934 {
935 	flush_delayed_work(&reaper_work);
936 }
937 EXPORT_SYMBOL_GPL(fsnotify_wait_marks_destroyed);
938