1c73be61cSDavid Howells // SPDX-License-Identifier: GPL-2.0
2c73be61cSDavid Howells /* Watch queue and general notification mechanism, built on pipes
3c73be61cSDavid Howells *
4c73be61cSDavid Howells * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved.
5c73be61cSDavid Howells * Written by David Howells (dhowells@redhat.com)
6c73be61cSDavid Howells *
7c02b872aSMauro Carvalho Chehab * See Documentation/core-api/watch_queue.rst
8c73be61cSDavid Howells */
9c73be61cSDavid Howells
10c73be61cSDavid Howells #define pr_fmt(fmt) "watchq: " fmt
11c73be61cSDavid Howells #include <linux/module.h>
12c73be61cSDavid Howells #include <linux/init.h>
13c73be61cSDavid Howells #include <linux/sched.h>
14c73be61cSDavid Howells #include <linux/slab.h>
15c73be61cSDavid Howells #include <linux/printk.h>
16c73be61cSDavid Howells #include <linux/miscdevice.h>
17c73be61cSDavid Howells #include <linux/fs.h>
18c73be61cSDavid Howells #include <linux/mm.h>
19c73be61cSDavid Howells #include <linux/pagemap.h>
20c73be61cSDavid Howells #include <linux/poll.h>
21c73be61cSDavid Howells #include <linux/uaccess.h>
22c73be61cSDavid Howells #include <linux/vmalloc.h>
23c73be61cSDavid Howells #include <linux/file.h>
24c73be61cSDavid Howells #include <linux/security.h>
25c73be61cSDavid Howells #include <linux/cred.h>
26c73be61cSDavid Howells #include <linux/sched/signal.h>
27c73be61cSDavid Howells #include <linux/watch_queue.h>
28c73be61cSDavid Howells #include <linux/pipe_fs_i.h>
29c73be61cSDavid Howells
30c73be61cSDavid Howells MODULE_DESCRIPTION("Watch queue");
31c73be61cSDavid Howells MODULE_AUTHOR("Red Hat, Inc.");
32c73be61cSDavid Howells
33c73be61cSDavid Howells #define WATCH_QUEUE_NOTE_SIZE 128
34c73be61cSDavid Howells #define WATCH_QUEUE_NOTES_PER_PAGE (PAGE_SIZE / WATCH_QUEUE_NOTE_SIZE)
35c73be61cSDavid Howells
36353f7988SLinus Torvalds /*
37353f7988SLinus Torvalds * This must be called under the RCU read-lock, which makes
38353f7988SLinus Torvalds * sure that the wqueue still exists. It can then take the lock,
39353f7988SLinus Torvalds * and check that the wqueue hasn't been destroyed, which in
40353f7988SLinus Torvalds * turn makes sure that the notification pipe still exists.
41353f7988SLinus Torvalds */
lock_wqueue(struct watch_queue * wqueue)42353f7988SLinus Torvalds static inline bool lock_wqueue(struct watch_queue *wqueue)
43353f7988SLinus Torvalds {
44353f7988SLinus Torvalds spin_lock_bh(&wqueue->lock);
45943211c8SSiddh Raman Pant if (unlikely(!wqueue->pipe)) {
46353f7988SLinus Torvalds spin_unlock_bh(&wqueue->lock);
47353f7988SLinus Torvalds return false;
48353f7988SLinus Torvalds }
49353f7988SLinus Torvalds return true;
50353f7988SLinus Torvalds }
51353f7988SLinus Torvalds
unlock_wqueue(struct watch_queue * wqueue)52353f7988SLinus Torvalds static inline void unlock_wqueue(struct watch_queue *wqueue)
53353f7988SLinus Torvalds {
54353f7988SLinus Torvalds spin_unlock_bh(&wqueue->lock);
55353f7988SLinus Torvalds }
56353f7988SLinus Torvalds
watch_queue_pipe_buf_release(struct pipe_inode_info * pipe,struct pipe_buffer * buf)57c73be61cSDavid Howells static void watch_queue_pipe_buf_release(struct pipe_inode_info *pipe,
58c73be61cSDavid Howells struct pipe_buffer *buf)
59c73be61cSDavid Howells {
60c73be61cSDavid Howells struct watch_queue *wqueue = (struct watch_queue *)buf->private;
61c73be61cSDavid Howells struct page *page;
62c73be61cSDavid Howells unsigned int bit;
63c73be61cSDavid Howells
64c73be61cSDavid Howells /* We need to work out which note within the page this refers to, but
65c73be61cSDavid Howells * the note might have been maximum size, so merely ANDing the offset
66c73be61cSDavid Howells * off doesn't work. OTOH, the note must've been more than zero size.
67c73be61cSDavid Howells */
68c73be61cSDavid Howells bit = buf->offset + buf->len;
69c73be61cSDavid Howells if ((bit & (WATCH_QUEUE_NOTE_SIZE - 1)) == 0)
70c73be61cSDavid Howells bit -= WATCH_QUEUE_NOTE_SIZE;
71c73be61cSDavid Howells bit /= WATCH_QUEUE_NOTE_SIZE;
72c73be61cSDavid Howells
73c73be61cSDavid Howells page = buf->page;
74c73be61cSDavid Howells bit += page->index;
75c73be61cSDavid Howells
76c73be61cSDavid Howells set_bit(bit, wqueue->notes_bitmap);
77c1853fbaSDavid Howells generic_pipe_buf_release(pipe, buf);
78c73be61cSDavid Howells }
79c73be61cSDavid Howells
806c329784SLinus Torvalds // No try_steal function => no stealing
816c329784SLinus Torvalds #define watch_queue_pipe_buf_try_steal NULL
82c73be61cSDavid Howells
83c73be61cSDavid Howells /* New data written to a pipe may be appended to a buffer with this type. */
84c73be61cSDavid Howells static const struct pipe_buf_operations watch_queue_pipe_buf_ops = {
85c73be61cSDavid Howells .release = watch_queue_pipe_buf_release,
866c329784SLinus Torvalds .try_steal = watch_queue_pipe_buf_try_steal,
87c73be61cSDavid Howells .get = generic_pipe_buf_get,
88c73be61cSDavid Howells };
89c73be61cSDavid Howells
90c73be61cSDavid Howells /*
91c73be61cSDavid Howells * Post a notification to a watch queue.
92353f7988SLinus Torvalds *
93353f7988SLinus Torvalds * Must be called with the RCU lock for reading, and the
94353f7988SLinus Torvalds * watch_queue lock held, which guarantees that the pipe
95353f7988SLinus Torvalds * hasn't been released.
96c73be61cSDavid Howells */
post_one_notification(struct watch_queue * wqueue,struct watch_notification * n)97c73be61cSDavid Howells static bool post_one_notification(struct watch_queue *wqueue,
98c73be61cSDavid Howells struct watch_notification *n)
99c73be61cSDavid Howells {
100c73be61cSDavid Howells void *p;
101c73be61cSDavid Howells struct pipe_inode_info *pipe = wqueue->pipe;
102c73be61cSDavid Howells struct pipe_buffer *buf;
103c73be61cSDavid Howells struct page *page;
104c73be61cSDavid Howells unsigned int head, tail, mask, note, offset, len;
105c73be61cSDavid Howells bool done = false;
106c73be61cSDavid Howells
107c73be61cSDavid Howells spin_lock_irq(&pipe->rd_wait.lock);
108c73be61cSDavid Howells
109c73be61cSDavid Howells mask = pipe->ring_size - 1;
110c73be61cSDavid Howells head = pipe->head;
111c73be61cSDavid Howells tail = pipe->tail;
112c73be61cSDavid Howells if (pipe_full(head, tail, pipe->ring_size))
113c73be61cSDavid Howells goto lost;
114c73be61cSDavid Howells
115c73be61cSDavid Howells note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes);
116c73be61cSDavid Howells if (note >= wqueue->nr_notes)
117c73be61cSDavid Howells goto lost;
118c73be61cSDavid Howells
119c73be61cSDavid Howells page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE];
120c73be61cSDavid Howells offset = note % WATCH_QUEUE_NOTES_PER_PAGE * WATCH_QUEUE_NOTE_SIZE;
121c73be61cSDavid Howells get_page(page);
122c73be61cSDavid Howells len = n->info & WATCH_INFO_LENGTH;
123c73be61cSDavid Howells p = kmap_atomic(page);
124c73be61cSDavid Howells memcpy(p + offset, n, len);
125c73be61cSDavid Howells kunmap_atomic(p);
126c73be61cSDavid Howells
127c73be61cSDavid Howells buf = &pipe->bufs[head & mask];
128c73be61cSDavid Howells buf->page = page;
129c73be61cSDavid Howells buf->private = (unsigned long)wqueue;
130c73be61cSDavid Howells buf->ops = &watch_queue_pipe_buf_ops;
131c73be61cSDavid Howells buf->offset = offset;
132c73be61cSDavid Howells buf->len = len;
1338cfba763SDavid Howells buf->flags = PIPE_BUF_FLAG_WHOLE;
1342ed147f0SDavid Howells smp_store_release(&pipe->head, head + 1); /* vs pipe_read() */
135c73be61cSDavid Howells
136c73be61cSDavid Howells if (!test_and_clear_bit(note, wqueue->notes_bitmap)) {
137c73be61cSDavid Howells spin_unlock_irq(&pipe->rd_wait.lock);
138c73be61cSDavid Howells BUG();
139c73be61cSDavid Howells }
140c73be61cSDavid Howells wake_up_interruptible_sync_poll_locked(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
141c73be61cSDavid Howells done = true;
142c73be61cSDavid Howells
143c73be61cSDavid Howells out:
144c73be61cSDavid Howells spin_unlock_irq(&pipe->rd_wait.lock);
145c73be61cSDavid Howells if (done)
146c73be61cSDavid Howells kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
147c73be61cSDavid Howells return done;
148c73be61cSDavid Howells
149c73be61cSDavid Howells lost:
150e7d553d6SDavid Howells buf = &pipe->bufs[(head - 1) & mask];
151e7d553d6SDavid Howells buf->flags |= PIPE_BUF_FLAG_LOSS;
152c73be61cSDavid Howells goto out;
153c73be61cSDavid Howells }
154c73be61cSDavid Howells
155c73be61cSDavid Howells /*
156c73be61cSDavid Howells * Apply filter rules to a notification.
157c73be61cSDavid Howells */
filter_watch_notification(const struct watch_filter * wf,const struct watch_notification * n)158c73be61cSDavid Howells static bool filter_watch_notification(const struct watch_filter *wf,
159c73be61cSDavid Howells const struct watch_notification *n)
160c73be61cSDavid Howells {
161c73be61cSDavid Howells const struct watch_type_filter *wt;
162c73be61cSDavid Howells unsigned int st_bits = sizeof(wt->subtype_filter[0]) * 8;
163c73be61cSDavid Howells unsigned int st_index = n->subtype / st_bits;
164c73be61cSDavid Howells unsigned int st_bit = 1U << (n->subtype % st_bits);
165c73be61cSDavid Howells int i;
166c73be61cSDavid Howells
167c73be61cSDavid Howells if (!test_bit(n->type, wf->type_filter))
168c73be61cSDavid Howells return false;
169c73be61cSDavid Howells
170c73be61cSDavid Howells for (i = 0; i < wf->nr_filters; i++) {
171c73be61cSDavid Howells wt = &wf->filters[i];
172c73be61cSDavid Howells if (n->type == wt->type &&
173c73be61cSDavid Howells (wt->subtype_filter[st_index] & st_bit) &&
174c73be61cSDavid Howells (n->info & wt->info_mask) == wt->info_filter)
175c73be61cSDavid Howells return true;
176c73be61cSDavid Howells }
177c73be61cSDavid Howells
178c73be61cSDavid Howells return false; /* If there is a filter, the default is to reject. */
179c73be61cSDavid Howells }
180c73be61cSDavid Howells
181c73be61cSDavid Howells /**
182c73be61cSDavid Howells * __post_watch_notification - Post an event notification
183c73be61cSDavid Howells * @wlist: The watch list to post the event to.
184c73be61cSDavid Howells * @n: The notification record to post.
185c73be61cSDavid Howells * @cred: The creds of the process that triggered the notification.
186c73be61cSDavid Howells * @id: The ID to match on the watch.
187c73be61cSDavid Howells *
188c73be61cSDavid Howells * Post a notification of an event into a set of watch queues and let the users
189c73be61cSDavid Howells * know.
190c73be61cSDavid Howells *
191c73be61cSDavid Howells * The size of the notification should be set in n->info & WATCH_INFO_LENGTH and
192c73be61cSDavid Howells * should be in units of sizeof(*n).
193c73be61cSDavid Howells */
__post_watch_notification(struct watch_list * wlist,struct watch_notification * n,const struct cred * cred,u64 id)194c73be61cSDavid Howells void __post_watch_notification(struct watch_list *wlist,
195c73be61cSDavid Howells struct watch_notification *n,
196c73be61cSDavid Howells const struct cred *cred,
197c73be61cSDavid Howells u64 id)
198c73be61cSDavid Howells {
199c73be61cSDavid Howells const struct watch_filter *wf;
200c73be61cSDavid Howells struct watch_queue *wqueue;
201c73be61cSDavid Howells struct watch *watch;
202c73be61cSDavid Howells
203c73be61cSDavid Howells if (((n->info & WATCH_INFO_LENGTH) >> WATCH_INFO_LENGTH__SHIFT) == 0) {
204c73be61cSDavid Howells WARN_ON(1);
205c73be61cSDavid Howells return;
206c73be61cSDavid Howells }
207c73be61cSDavid Howells
208c73be61cSDavid Howells rcu_read_lock();
209c73be61cSDavid Howells
210c73be61cSDavid Howells hlist_for_each_entry_rcu(watch, &wlist->watchers, list_node) {
211c73be61cSDavid Howells if (watch->id != id)
212c73be61cSDavid Howells continue;
213c73be61cSDavid Howells n->info &= ~WATCH_INFO_ID;
214c73be61cSDavid Howells n->info |= watch->info_id;
215c73be61cSDavid Howells
216c73be61cSDavid Howells wqueue = rcu_dereference(watch->queue);
217c73be61cSDavid Howells wf = rcu_dereference(wqueue->filter);
218c73be61cSDavid Howells if (wf && !filter_watch_notification(wf, n))
219c73be61cSDavid Howells continue;
220c73be61cSDavid Howells
221c73be61cSDavid Howells if (security_post_notification(watch->cred, cred, n) < 0)
222c73be61cSDavid Howells continue;
223c73be61cSDavid Howells
224353f7988SLinus Torvalds if (lock_wqueue(wqueue)) {
225c73be61cSDavid Howells post_one_notification(wqueue, n);
22644e29e64SLinus Torvalds unlock_wqueue(wqueue);
227353f7988SLinus Torvalds }
228c73be61cSDavid Howells }
229c73be61cSDavid Howells
230c73be61cSDavid Howells rcu_read_unlock();
231c73be61cSDavid Howells }
232c73be61cSDavid Howells EXPORT_SYMBOL(__post_watch_notification);
233c73be61cSDavid Howells
234c73be61cSDavid Howells /*
235c73be61cSDavid Howells * Allocate sufficient pages to preallocation for the requested number of
236c73be61cSDavid Howells * notifications.
237c73be61cSDavid Howells */
watch_queue_set_size(struct pipe_inode_info * pipe,unsigned int nr_notes)238c73be61cSDavid Howells long watch_queue_set_size(struct pipe_inode_info *pipe, unsigned int nr_notes)
239c73be61cSDavid Howells {
240c73be61cSDavid Howells struct watch_queue *wqueue = pipe->watch_queue;
241c73be61cSDavid Howells struct page **pages;
242c73be61cSDavid Howells unsigned long *bitmap;
243c73be61cSDavid Howells unsigned long user_bufs;
244c73be61cSDavid Howells int ret, i, nr_pages;
245c73be61cSDavid Howells
246c73be61cSDavid Howells if (!wqueue)
247c73be61cSDavid Howells return -ENODEV;
248c73be61cSDavid Howells if (wqueue->notes)
249c73be61cSDavid Howells return -EBUSY;
250c73be61cSDavid Howells
251c73be61cSDavid Howells if (nr_notes < 1 ||
252c73be61cSDavid Howells nr_notes > 512) /* TODO: choose a better hard limit */
253c73be61cSDavid Howells return -EINVAL;
254c73be61cSDavid Howells
255c73be61cSDavid Howells nr_pages = (nr_notes + WATCH_QUEUE_NOTES_PER_PAGE - 1);
256c73be61cSDavid Howells nr_pages /= WATCH_QUEUE_NOTES_PER_PAGE;
257c73be61cSDavid Howells user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_pages);
258c73be61cSDavid Howells
259c73be61cSDavid Howells if (nr_pages > pipe->max_usage &&
260c73be61cSDavid Howells (too_many_pipe_buffers_hard(user_bufs) ||
261c73be61cSDavid Howells too_many_pipe_buffers_soft(user_bufs)) &&
262c73be61cSDavid Howells pipe_is_unprivileged_user()) {
263c73be61cSDavid Howells ret = -EPERM;
264c73be61cSDavid Howells goto error;
265c73be61cSDavid Howells }
266c73be61cSDavid Howells
2673b4c0371SDavid Howells nr_notes = nr_pages * WATCH_QUEUE_NOTES_PER_PAGE;
26896a4d891SDavid Howells ret = pipe_resize_ring(pipe, roundup_pow_of_two(nr_notes));
269c73be61cSDavid Howells if (ret < 0)
270c73be61cSDavid Howells goto error;
271c73be61cSDavid Howells
27203e1d60eSDavid Disseldorp ret = -ENOMEM;
273c73be61cSDavid Howells pages = kcalloc(sizeof(struct page *), nr_pages, GFP_KERNEL);
274c73be61cSDavid Howells if (!pages)
275c73be61cSDavid Howells goto error;
276c73be61cSDavid Howells
277c73be61cSDavid Howells for (i = 0; i < nr_pages; i++) {
278c73be61cSDavid Howells pages[i] = alloc_page(GFP_KERNEL);
279c73be61cSDavid Howells if (!pages[i])
280c73be61cSDavid Howells goto error_p;
281c73be61cSDavid Howells pages[i]->index = i * WATCH_QUEUE_NOTES_PER_PAGE;
282c73be61cSDavid Howells }
283c73be61cSDavid Howells
284a66bd757SChristophe JAILLET bitmap = bitmap_alloc(nr_notes, GFP_KERNEL);
285c73be61cSDavid Howells if (!bitmap)
286c73be61cSDavid Howells goto error_p;
287c73be61cSDavid Howells
288a66bd757SChristophe JAILLET bitmap_fill(bitmap, nr_notes);
289c73be61cSDavid Howells wqueue->notes = pages;
290c73be61cSDavid Howells wqueue->notes_bitmap = bitmap;
291c73be61cSDavid Howells wqueue->nr_pages = nr_pages;
2923b4c0371SDavid Howells wqueue->nr_notes = nr_notes;
293c73be61cSDavid Howells return 0;
294c73be61cSDavid Howells
295c73be61cSDavid Howells error_p:
296a635415aSDavid Howells while (--i >= 0)
297c73be61cSDavid Howells __free_page(pages[i]);
298c73be61cSDavid Howells kfree(pages);
299c73be61cSDavid Howells error:
300c73be61cSDavid Howells (void) account_pipe_buffers(pipe->user, nr_pages, pipe->nr_accounted);
301c73be61cSDavid Howells return ret;
302c73be61cSDavid Howells }
303c73be61cSDavid Howells
304c73be61cSDavid Howells /*
305c73be61cSDavid Howells * Set the filter on a watch queue.
306c73be61cSDavid Howells */
watch_queue_set_filter(struct pipe_inode_info * pipe,struct watch_notification_filter __user * _filter)307c73be61cSDavid Howells long watch_queue_set_filter(struct pipe_inode_info *pipe,
308c73be61cSDavid Howells struct watch_notification_filter __user *_filter)
309c73be61cSDavid Howells {
310c73be61cSDavid Howells struct watch_notification_type_filter *tf;
311c73be61cSDavid Howells struct watch_notification_filter filter;
312c73be61cSDavid Howells struct watch_type_filter *q;
313c73be61cSDavid Howells struct watch_filter *wfilter;
314c73be61cSDavid Howells struct watch_queue *wqueue = pipe->watch_queue;
315c73be61cSDavid Howells int ret, nr_filter = 0, i;
316c73be61cSDavid Howells
317c73be61cSDavid Howells if (!wqueue)
318c73be61cSDavid Howells return -ENODEV;
319c73be61cSDavid Howells
320c73be61cSDavid Howells if (!_filter) {
321c73be61cSDavid Howells /* Remove the old filter */
322c73be61cSDavid Howells wfilter = NULL;
323c73be61cSDavid Howells goto set;
324c73be61cSDavid Howells }
325c73be61cSDavid Howells
326c73be61cSDavid Howells /* Grab the user's filter specification */
327c73be61cSDavid Howells if (copy_from_user(&filter, _filter, sizeof(filter)) != 0)
328c73be61cSDavid Howells return -EFAULT;
329c73be61cSDavid Howells if (filter.nr_filters == 0 ||
330c73be61cSDavid Howells filter.nr_filters > 16 ||
331c73be61cSDavid Howells filter.__reserved != 0)
332c73be61cSDavid Howells return -EINVAL;
333c73be61cSDavid Howells
334*0f403ebaSPhilipp Stanner tf = memdup_array_user(_filter->filters, filter.nr_filters, sizeof(*tf));
335c73be61cSDavid Howells if (IS_ERR(tf))
336c73be61cSDavid Howells return PTR_ERR(tf);
337c73be61cSDavid Howells
338c73be61cSDavid Howells ret = -EINVAL;
339c73be61cSDavid Howells for (i = 0; i < filter.nr_filters; i++) {
340c73be61cSDavid Howells if ((tf[i].info_filter & ~tf[i].info_mask) ||
341c73be61cSDavid Howells tf[i].info_mask & WATCH_INFO_LENGTH)
342c73be61cSDavid Howells goto err_filter;
343c73be61cSDavid Howells /* Ignore any unknown types */
344c993ee0fSDavid Howells if (tf[i].type >= WATCH_TYPE__NR)
345c73be61cSDavid Howells continue;
346c73be61cSDavid Howells nr_filter++;
347c73be61cSDavid Howells }
348c73be61cSDavid Howells
349c73be61cSDavid Howells /* Now we need to build the internal filter from only the relevant
350c73be61cSDavid Howells * user-specified filters.
351c73be61cSDavid Howells */
352c73be61cSDavid Howells ret = -ENOMEM;
353c73be61cSDavid Howells wfilter = kzalloc(struct_size(wfilter, filters, nr_filter), GFP_KERNEL);
354c73be61cSDavid Howells if (!wfilter)
355c73be61cSDavid Howells goto err_filter;
356c73be61cSDavid Howells wfilter->nr_filters = nr_filter;
357c73be61cSDavid Howells
358c73be61cSDavid Howells q = wfilter->filters;
359c73be61cSDavid Howells for (i = 0; i < filter.nr_filters; i++) {
360c993ee0fSDavid Howells if (tf[i].type >= WATCH_TYPE__NR)
361c73be61cSDavid Howells continue;
362c73be61cSDavid Howells
363c73be61cSDavid Howells q->type = tf[i].type;
364c73be61cSDavid Howells q->info_filter = tf[i].info_filter;
365c73be61cSDavid Howells q->info_mask = tf[i].info_mask;
366c73be61cSDavid Howells q->subtype_filter[0] = tf[i].subtype_filter[0];
367c73be61cSDavid Howells __set_bit(q->type, wfilter->type_filter);
368c73be61cSDavid Howells q++;
369c73be61cSDavid Howells }
370c73be61cSDavid Howells
371c73be61cSDavid Howells kfree(tf);
372c73be61cSDavid Howells set:
373c73be61cSDavid Howells pipe_lock(pipe);
374c73be61cSDavid Howells wfilter = rcu_replace_pointer(wqueue->filter, wfilter,
375c73be61cSDavid Howells lockdep_is_held(&pipe->mutex));
376c73be61cSDavid Howells pipe_unlock(pipe);
377c73be61cSDavid Howells if (wfilter)
378c73be61cSDavid Howells kfree_rcu(wfilter, rcu);
379c73be61cSDavid Howells return 0;
380c73be61cSDavid Howells
381c73be61cSDavid Howells err_filter:
382c73be61cSDavid Howells kfree(tf);
383c73be61cSDavid Howells return ret;
384c73be61cSDavid Howells }
385c73be61cSDavid Howells
__put_watch_queue(struct kref * kref)386c73be61cSDavid Howells static void __put_watch_queue(struct kref *kref)
387c73be61cSDavid Howells {
388c73be61cSDavid Howells struct watch_queue *wqueue =
389c73be61cSDavid Howells container_of(kref, struct watch_queue, usage);
390c73be61cSDavid Howells struct watch_filter *wfilter;
391c73be61cSDavid Howells int i;
392c73be61cSDavid Howells
393c73be61cSDavid Howells for (i = 0; i < wqueue->nr_pages; i++)
394c73be61cSDavid Howells __free_page(wqueue->notes[i]);
395b4902070SEric Dumazet kfree(wqueue->notes);
3967ea1a012SDavid Howells bitmap_free(wqueue->notes_bitmap);
397c73be61cSDavid Howells
398c73be61cSDavid Howells wfilter = rcu_access_pointer(wqueue->filter);
399c73be61cSDavid Howells if (wfilter)
400c73be61cSDavid Howells kfree_rcu(wfilter, rcu);
401c73be61cSDavid Howells kfree_rcu(wqueue, rcu);
402c73be61cSDavid Howells }
403c73be61cSDavid Howells
404c73be61cSDavid Howells /**
405c73be61cSDavid Howells * put_watch_queue - Dispose of a ref on a watchqueue.
406c73be61cSDavid Howells * @wqueue: The watch queue to unref.
407c73be61cSDavid Howells */
put_watch_queue(struct watch_queue * wqueue)408c73be61cSDavid Howells void put_watch_queue(struct watch_queue *wqueue)
409c73be61cSDavid Howells {
410c73be61cSDavid Howells kref_put(&wqueue->usage, __put_watch_queue);
411c73be61cSDavid Howells }
412c73be61cSDavid Howells EXPORT_SYMBOL(put_watch_queue);
413c73be61cSDavid Howells
free_watch(struct rcu_head * rcu)414c73be61cSDavid Howells static void free_watch(struct rcu_head *rcu)
415c73be61cSDavid Howells {
416c73be61cSDavid Howells struct watch *watch = container_of(rcu, struct watch, rcu);
417c73be61cSDavid Howells
418c73be61cSDavid Howells put_watch_queue(rcu_access_pointer(watch->queue));
41929e44f45SDavid Howells atomic_dec(&watch->cred->user->nr_watches);
420c73be61cSDavid Howells put_cred(watch->cred);
4213d8dcf27SDavid Howells kfree(watch);
422c73be61cSDavid Howells }
423c73be61cSDavid Howells
__put_watch(struct kref * kref)424c73be61cSDavid Howells static void __put_watch(struct kref *kref)
425c73be61cSDavid Howells {
426c73be61cSDavid Howells struct watch *watch = container_of(kref, struct watch, usage);
427c73be61cSDavid Howells
428c73be61cSDavid Howells call_rcu(&watch->rcu, free_watch);
429c73be61cSDavid Howells }
430c73be61cSDavid Howells
431c73be61cSDavid Howells /*
432c73be61cSDavid Howells * Discard a watch.
433c73be61cSDavid Howells */
put_watch(struct watch * watch)434c73be61cSDavid Howells static void put_watch(struct watch *watch)
435c73be61cSDavid Howells {
436c73be61cSDavid Howells kref_put(&watch->usage, __put_watch);
437c73be61cSDavid Howells }
438c73be61cSDavid Howells
439c73be61cSDavid Howells /**
4408f0bfc25SLukas Bulwahn * init_watch - Initialise a watch
441c73be61cSDavid Howells * @watch: The watch to initialise.
442c73be61cSDavid Howells * @wqueue: The queue to assign.
443c73be61cSDavid Howells *
444c73be61cSDavid Howells * Initialise a watch and set the watch queue.
445c73be61cSDavid Howells */
init_watch(struct watch * watch,struct watch_queue * wqueue)446c73be61cSDavid Howells void init_watch(struct watch *watch, struct watch_queue *wqueue)
447c73be61cSDavid Howells {
448c73be61cSDavid Howells kref_init(&watch->usage);
449c73be61cSDavid Howells INIT_HLIST_NODE(&watch->list_node);
450c73be61cSDavid Howells INIT_HLIST_NODE(&watch->queue_node);
451c73be61cSDavid Howells rcu_assign_pointer(watch->queue, wqueue);
452c73be61cSDavid Howells }
453c73be61cSDavid Howells
add_one_watch(struct watch * watch,struct watch_list * wlist,struct watch_queue * wqueue)454e64ab2dbSLinus Torvalds static int add_one_watch(struct watch *watch, struct watch_list *wlist, struct watch_queue *wqueue)
455e64ab2dbSLinus Torvalds {
456e64ab2dbSLinus Torvalds const struct cred *cred;
457e64ab2dbSLinus Torvalds struct watch *w;
458e64ab2dbSLinus Torvalds
459e64ab2dbSLinus Torvalds hlist_for_each_entry(w, &wlist->watchers, list_node) {
460e64ab2dbSLinus Torvalds struct watch_queue *wq = rcu_access_pointer(w->queue);
461e64ab2dbSLinus Torvalds if (wqueue == wq && watch->id == w->id)
462e64ab2dbSLinus Torvalds return -EBUSY;
463e64ab2dbSLinus Torvalds }
464e64ab2dbSLinus Torvalds
465e64ab2dbSLinus Torvalds cred = current_cred();
466e64ab2dbSLinus Torvalds if (atomic_inc_return(&cred->user->nr_watches) > task_rlimit(current, RLIMIT_NOFILE)) {
467e64ab2dbSLinus Torvalds atomic_dec(&cred->user->nr_watches);
468e64ab2dbSLinus Torvalds return -EAGAIN;
469e64ab2dbSLinus Torvalds }
470e64ab2dbSLinus Torvalds
471e64ab2dbSLinus Torvalds watch->cred = get_cred(cred);
472e64ab2dbSLinus Torvalds rcu_assign_pointer(watch->watch_list, wlist);
473e64ab2dbSLinus Torvalds
474e64ab2dbSLinus Torvalds kref_get(&wqueue->usage);
475e64ab2dbSLinus Torvalds kref_get(&watch->usage);
476e64ab2dbSLinus Torvalds hlist_add_head(&watch->queue_node, &wqueue->watches);
477e64ab2dbSLinus Torvalds hlist_add_head_rcu(&watch->list_node, &wlist->watchers);
478e64ab2dbSLinus Torvalds return 0;
479e64ab2dbSLinus Torvalds }
480e64ab2dbSLinus Torvalds
481c73be61cSDavid Howells /**
482c73be61cSDavid Howells * add_watch_to_object - Add a watch on an object to a watch list
483c73be61cSDavid Howells * @watch: The watch to add
484c73be61cSDavid Howells * @wlist: The watch list to add to
485c73be61cSDavid Howells *
486c73be61cSDavid Howells * @watch->queue must have been set to point to the queue to post notifications
487c73be61cSDavid Howells * to and the watch list of the object to be watched. @watch->cred must also
488c73be61cSDavid Howells * have been set to the appropriate credentials and a ref taken on them.
489c73be61cSDavid Howells *
490c73be61cSDavid Howells * The caller must pin the queue and the list both and must hold the list
491c73be61cSDavid Howells * locked against racing watch additions/removals.
492c73be61cSDavid Howells */
add_watch_to_object(struct watch * watch,struct watch_list * wlist)493c73be61cSDavid Howells int add_watch_to_object(struct watch *watch, struct watch_list *wlist)
494c73be61cSDavid Howells {
495e64ab2dbSLinus Torvalds struct watch_queue *wqueue;
496e64ab2dbSLinus Torvalds int ret = -ENOENT;
497c73be61cSDavid Howells
498e64ab2dbSLinus Torvalds rcu_read_lock();
499c73be61cSDavid Howells
500e64ab2dbSLinus Torvalds wqueue = rcu_access_pointer(watch->queue);
501353f7988SLinus Torvalds if (lock_wqueue(wqueue)) {
502e64ab2dbSLinus Torvalds spin_lock(&wlist->lock);
503e64ab2dbSLinus Torvalds ret = add_one_watch(watch, wlist, wqueue);
504e64ab2dbSLinus Torvalds spin_unlock(&wlist->lock);
505353f7988SLinus Torvalds unlock_wqueue(wqueue);
506353f7988SLinus Torvalds }
507c73be61cSDavid Howells
508e64ab2dbSLinus Torvalds rcu_read_unlock();
509e64ab2dbSLinus Torvalds return ret;
510c73be61cSDavid Howells }
511c73be61cSDavid Howells EXPORT_SYMBOL(add_watch_to_object);
512c73be61cSDavid Howells
513c73be61cSDavid Howells /**
514c73be61cSDavid Howells * remove_watch_from_object - Remove a watch or all watches from an object.
515c73be61cSDavid Howells * @wlist: The watch list to remove from
516c73be61cSDavid Howells * @wq: The watch queue of interest (ignored if @all is true)
517c73be61cSDavid Howells * @id: The ID of the watch to remove (ignored if @all is true)
518c73be61cSDavid Howells * @all: True to remove all objects
519c73be61cSDavid Howells *
520c73be61cSDavid Howells * Remove a specific watch or all watches from an object. A notification is
521c73be61cSDavid Howells * sent to the watcher to tell them that this happened.
522c73be61cSDavid Howells */
remove_watch_from_object(struct watch_list * wlist,struct watch_queue * wq,u64 id,bool all)523c73be61cSDavid Howells int remove_watch_from_object(struct watch_list *wlist, struct watch_queue *wq,
524c73be61cSDavid Howells u64 id, bool all)
525c73be61cSDavid Howells {
526c73be61cSDavid Howells struct watch_notification_removal n;
527c73be61cSDavid Howells struct watch_queue *wqueue;
528c73be61cSDavid Howells struct watch *watch;
529c73be61cSDavid Howells int ret = -EBADSLT;
530c73be61cSDavid Howells
531c73be61cSDavid Howells rcu_read_lock();
532c73be61cSDavid Howells
533c73be61cSDavid Howells again:
534c73be61cSDavid Howells spin_lock(&wlist->lock);
535c73be61cSDavid Howells hlist_for_each_entry(watch, &wlist->watchers, list_node) {
536c73be61cSDavid Howells if (all ||
537c73be61cSDavid Howells (watch->id == id && rcu_access_pointer(watch->queue) == wq))
538c73be61cSDavid Howells goto found;
539c73be61cSDavid Howells }
540c73be61cSDavid Howells spin_unlock(&wlist->lock);
541c73be61cSDavid Howells goto out;
542c73be61cSDavid Howells
543c73be61cSDavid Howells found:
544c73be61cSDavid Howells ret = 0;
545c73be61cSDavid Howells hlist_del_init_rcu(&watch->list_node);
546c73be61cSDavid Howells rcu_assign_pointer(watch->watch_list, NULL);
547c73be61cSDavid Howells spin_unlock(&wlist->lock);
548c73be61cSDavid Howells
549c73be61cSDavid Howells /* We now own the reference on watch that used to belong to wlist. */
550c73be61cSDavid Howells
551c73be61cSDavid Howells n.watch.type = WATCH_TYPE_META;
552c73be61cSDavid Howells n.watch.subtype = WATCH_META_REMOVAL_NOTIFICATION;
553c73be61cSDavid Howells n.watch.info = watch->info_id | watch_sizeof(n.watch);
554c73be61cSDavid Howells n.id = id;
555c73be61cSDavid Howells if (id != 0)
556c73be61cSDavid Howells n.watch.info = watch->info_id | watch_sizeof(n);
557c73be61cSDavid Howells
558c73be61cSDavid Howells wqueue = rcu_dereference(watch->queue);
559c73be61cSDavid Howells
560353f7988SLinus Torvalds if (lock_wqueue(wqueue)) {
561c73be61cSDavid Howells post_one_notification(wqueue, &n.watch);
562c73be61cSDavid Howells
563c73be61cSDavid Howells if (!hlist_unhashed(&watch->queue_node)) {
564c73be61cSDavid Howells hlist_del_init_rcu(&watch->queue_node);
565c73be61cSDavid Howells put_watch(watch);
566c73be61cSDavid Howells }
567c73be61cSDavid Howells
568353f7988SLinus Torvalds unlock_wqueue(wqueue);
569c73be61cSDavid Howells }
570c73be61cSDavid Howells
571c73be61cSDavid Howells if (wlist->release_watch) {
572c73be61cSDavid Howells void (*release_watch)(struct watch *);
573c73be61cSDavid Howells
574c73be61cSDavid Howells release_watch = wlist->release_watch;
575c73be61cSDavid Howells rcu_read_unlock();
576c73be61cSDavid Howells (*release_watch)(watch);
577c73be61cSDavid Howells rcu_read_lock();
578c73be61cSDavid Howells }
579c73be61cSDavid Howells put_watch(watch);
580c73be61cSDavid Howells
581c73be61cSDavid Howells if (all && !hlist_empty(&wlist->watchers))
582c73be61cSDavid Howells goto again;
583c73be61cSDavid Howells out:
584c73be61cSDavid Howells rcu_read_unlock();
585c73be61cSDavid Howells return ret;
586c73be61cSDavid Howells }
587c73be61cSDavid Howells EXPORT_SYMBOL(remove_watch_from_object);
588c73be61cSDavid Howells
589c73be61cSDavid Howells /*
590c73be61cSDavid Howells * Remove all the watches that are contributory to a queue. This has the
591c73be61cSDavid Howells * potential to race with removal of the watches by the destruction of the
592c73be61cSDavid Howells * objects being watched or with the distribution of notifications.
593c73be61cSDavid Howells */
watch_queue_clear(struct watch_queue * wqueue)594c73be61cSDavid Howells void watch_queue_clear(struct watch_queue *wqueue)
595c73be61cSDavid Howells {
596c73be61cSDavid Howells struct watch_list *wlist;
597c73be61cSDavid Howells struct watch *watch;
598c73be61cSDavid Howells bool release;
599c73be61cSDavid Howells
600c73be61cSDavid Howells rcu_read_lock();
601c73be61cSDavid Howells spin_lock_bh(&wqueue->lock);
602c73be61cSDavid Howells
603943211c8SSiddh Raman Pant /*
604943211c8SSiddh Raman Pant * This pipe can be freed by callers like free_pipe_info().
605943211c8SSiddh Raman Pant * Removing this reference also prevents new notifications.
606943211c8SSiddh Raman Pant */
607943211c8SSiddh Raman Pant wqueue->pipe = NULL;
608c73be61cSDavid Howells
609c73be61cSDavid Howells while (!hlist_empty(&wqueue->watches)) {
610c73be61cSDavid Howells watch = hlist_entry(wqueue->watches.first, struct watch, queue_node);
611c73be61cSDavid Howells hlist_del_init_rcu(&watch->queue_node);
612c73be61cSDavid Howells /* We now own a ref on the watch. */
613c73be61cSDavid Howells spin_unlock_bh(&wqueue->lock);
614c73be61cSDavid Howells
615c73be61cSDavid Howells /* We can't do the next bit under the queue lock as we need to
616c73be61cSDavid Howells * get the list lock - which would cause a deadlock if someone
617c73be61cSDavid Howells * was removing from the opposite direction at the same time or
618c73be61cSDavid Howells * posting a notification.
619c73be61cSDavid Howells */
620c73be61cSDavid Howells wlist = rcu_dereference(watch->watch_list);
621c73be61cSDavid Howells if (wlist) {
622c73be61cSDavid Howells void (*release_watch)(struct watch *);
623c73be61cSDavid Howells
624c73be61cSDavid Howells spin_lock(&wlist->lock);
625c73be61cSDavid Howells
626c73be61cSDavid Howells release = !hlist_unhashed(&watch->list_node);
627c73be61cSDavid Howells if (release) {
628c73be61cSDavid Howells hlist_del_init_rcu(&watch->list_node);
629c73be61cSDavid Howells rcu_assign_pointer(watch->watch_list, NULL);
630c73be61cSDavid Howells
631c73be61cSDavid Howells /* We now own a second ref on the watch. */
632c73be61cSDavid Howells }
633c73be61cSDavid Howells
634c73be61cSDavid Howells release_watch = wlist->release_watch;
635c73be61cSDavid Howells spin_unlock(&wlist->lock);
636c73be61cSDavid Howells
637c73be61cSDavid Howells if (release) {
638c73be61cSDavid Howells if (release_watch) {
639c73be61cSDavid Howells rcu_read_unlock();
640c73be61cSDavid Howells /* This might need to call dput(), so
641c73be61cSDavid Howells * we have to drop all the locks.
642c73be61cSDavid Howells */
643c73be61cSDavid Howells (*release_watch)(watch);
644c73be61cSDavid Howells rcu_read_lock();
645c73be61cSDavid Howells }
646c73be61cSDavid Howells put_watch(watch);
647c73be61cSDavid Howells }
648c73be61cSDavid Howells }
649c73be61cSDavid Howells
650c73be61cSDavid Howells put_watch(watch);
651c73be61cSDavid Howells spin_lock_bh(&wqueue->lock);
652c73be61cSDavid Howells }
653c73be61cSDavid Howells
654c73be61cSDavid Howells spin_unlock_bh(&wqueue->lock);
655c73be61cSDavid Howells rcu_read_unlock();
656c73be61cSDavid Howells }
657c73be61cSDavid Howells
658c73be61cSDavid Howells /**
659c73be61cSDavid Howells * get_watch_queue - Get a watch queue from its file descriptor.
660c73be61cSDavid Howells * @fd: The fd to query.
661c73be61cSDavid Howells */
get_watch_queue(int fd)662c73be61cSDavid Howells struct watch_queue *get_watch_queue(int fd)
663c73be61cSDavid Howells {
664c73be61cSDavid Howells struct pipe_inode_info *pipe;
665c73be61cSDavid Howells struct watch_queue *wqueue = ERR_PTR(-EINVAL);
666c73be61cSDavid Howells struct fd f;
667c73be61cSDavid Howells
668c73be61cSDavid Howells f = fdget(fd);
669c73be61cSDavid Howells if (f.file) {
670c73be61cSDavid Howells pipe = get_pipe_info(f.file, false);
671c73be61cSDavid Howells if (pipe && pipe->watch_queue) {
672c73be61cSDavid Howells wqueue = pipe->watch_queue;
673c73be61cSDavid Howells kref_get(&wqueue->usage);
674c73be61cSDavid Howells }
675c73be61cSDavid Howells fdput(f);
676c73be61cSDavid Howells }
677c73be61cSDavid Howells
678c73be61cSDavid Howells return wqueue;
679c73be61cSDavid Howells }
680c73be61cSDavid Howells EXPORT_SYMBOL(get_watch_queue);
681c73be61cSDavid Howells
682c73be61cSDavid Howells /*
683c73be61cSDavid Howells * Initialise a watch queue
684c73be61cSDavid Howells */
watch_queue_init(struct pipe_inode_info * pipe)685c73be61cSDavid Howells int watch_queue_init(struct pipe_inode_info *pipe)
686c73be61cSDavid Howells {
687c73be61cSDavid Howells struct watch_queue *wqueue;
688c73be61cSDavid Howells
689c73be61cSDavid Howells wqueue = kzalloc(sizeof(*wqueue), GFP_KERNEL);
690c73be61cSDavid Howells if (!wqueue)
691c73be61cSDavid Howells return -ENOMEM;
692c73be61cSDavid Howells
693c73be61cSDavid Howells wqueue->pipe = pipe;
694c73be61cSDavid Howells kref_init(&wqueue->usage);
695c73be61cSDavid Howells spin_lock_init(&wqueue->lock);
696c73be61cSDavid Howells INIT_HLIST_HEAD(&wqueue->watches);
697c73be61cSDavid Howells
698c73be61cSDavid Howells pipe->watch_queue = wqueue;
699c73be61cSDavid Howells return 0;
700c73be61cSDavid Howells }
701