1d64f4554SDavid Howells // SPDX-License-Identifier: GPL-2.0-or-later
2d64f4554SDavid Howells /* Cache data I/O routines
3d64f4554SDavid Howells *
4d64f4554SDavid Howells * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5d64f4554SDavid Howells * Written by David Howells (dhowells@redhat.com)
6d64f4554SDavid Howells */
7d64f4554SDavid Howells #define FSCACHE_DEBUG_LEVEL OPERATION
8d64f4554SDavid Howells #include <linux/fscache-cache.h>
9d64f4554SDavid Howells #include <linux/uio.h>
10d64f4554SDavid Howells #include <linux/bvec.h>
11d64f4554SDavid Howells #include <linux/slab.h>
12d64f4554SDavid Howells #include <linux/uio.h>
13d64f4554SDavid Howells #include "internal.h"
14d64f4554SDavid Howells
15d64f4554SDavid Howells /**
16d64f4554SDavid Howells * fscache_wait_for_operation - Wait for an object become accessible
17d64f4554SDavid Howells * @cres: The cache resources for the operation being performed
18d64f4554SDavid Howells * @want_state: The minimum state the object must be at
19d64f4554SDavid Howells *
20d64f4554SDavid Howells * See if the target cache object is at the specified minimum state of
21d64f4554SDavid Howells * accessibility yet, and if not, wait for it.
22d64f4554SDavid Howells */
fscache_wait_for_operation(struct netfs_cache_resources * cres,enum fscache_want_state want_state)23d64f4554SDavid Howells bool fscache_wait_for_operation(struct netfs_cache_resources *cres,
24d64f4554SDavid Howells enum fscache_want_state want_state)
25d64f4554SDavid Howells {
26d64f4554SDavid Howells struct fscache_cookie *cookie = fscache_cres_cookie(cres);
27d64f4554SDavid Howells enum fscache_cookie_state state;
28d64f4554SDavid Howells
29d64f4554SDavid Howells again:
30d64f4554SDavid Howells if (!fscache_cache_is_live(cookie->volume->cache)) {
31d64f4554SDavid Howells _leave(" [broken]");
32d64f4554SDavid Howells return false;
33d64f4554SDavid Howells }
34d64f4554SDavid Howells
35d64f4554SDavid Howells state = fscache_cookie_state(cookie);
36d64f4554SDavid Howells _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state);
37d64f4554SDavid Howells
38d64f4554SDavid Howells switch (state) {
39d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_CREATING:
40d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_INVALIDATING:
41d64f4554SDavid Howells if (want_state == FSCACHE_WANT_PARAMS)
42d64f4554SDavid Howells goto ready; /* There can be no content */
43d64f4554SDavid Howells fallthrough;
44d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_LOOKING_UP:
45d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
46d64f4554SDavid Howells wait_var_event(&cookie->state,
47d64f4554SDavid Howells fscache_cookie_state(cookie) != state);
48d64f4554SDavid Howells goto again;
49d64f4554SDavid Howells
50d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_ACTIVE:
51d64f4554SDavid Howells goto ready;
52d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_DROPPED:
53d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_RELINQUISHING:
54d64f4554SDavid Howells default:
55d64f4554SDavid Howells _leave(" [not live]");
56d64f4554SDavid Howells return false;
57d64f4554SDavid Howells }
58d64f4554SDavid Howells
59d64f4554SDavid Howells ready:
60d64f4554SDavid Howells if (!cres->cache_priv2)
61d64f4554SDavid Howells return cookie->volume->cache->ops->begin_operation(cres, want_state);
62d64f4554SDavid Howells return true;
63d64f4554SDavid Howells }
64d64f4554SDavid Howells EXPORT_SYMBOL(fscache_wait_for_operation);
65d64f4554SDavid Howells
66d64f4554SDavid Howells /*
67d64f4554SDavid Howells * Begin an I/O operation on the cache, waiting till we reach the right state.
68d64f4554SDavid Howells *
69d64f4554SDavid Howells * Attaches the resources required to the operation resources record.
70d64f4554SDavid Howells */
fscache_begin_operation(struct netfs_cache_resources * cres,struct fscache_cookie * cookie,enum fscache_want_state want_state,enum fscache_access_trace why)71d64f4554SDavid Howells static int fscache_begin_operation(struct netfs_cache_resources *cres,
72d64f4554SDavid Howells struct fscache_cookie *cookie,
73d64f4554SDavid Howells enum fscache_want_state want_state,
74d64f4554SDavid Howells enum fscache_access_trace why)
75d64f4554SDavid Howells {
76d64f4554SDavid Howells enum fscache_cookie_state state;
77d64f4554SDavid Howells long timeo;
78d64f4554SDavid Howells bool once_only = false;
79d64f4554SDavid Howells
80d64f4554SDavid Howells cres->ops = NULL;
81d64f4554SDavid Howells cres->cache_priv = cookie;
82d64f4554SDavid Howells cres->cache_priv2 = NULL;
83d64f4554SDavid Howells cres->debug_id = cookie->debug_id;
84d64f4554SDavid Howells cres->inval_counter = cookie->inval_counter;
85d64f4554SDavid Howells
86d64f4554SDavid Howells if (!fscache_begin_cookie_access(cookie, why))
87d64f4554SDavid Howells return -ENOBUFS;
88d64f4554SDavid Howells
89d64f4554SDavid Howells again:
90d64f4554SDavid Howells spin_lock(&cookie->lock);
91d64f4554SDavid Howells
92d64f4554SDavid Howells state = fscache_cookie_state(cookie);
93d64f4554SDavid Howells _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state);
94d64f4554SDavid Howells
95d64f4554SDavid Howells switch (state) {
96d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_LOOKING_UP:
97d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
98d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_INVALIDATING:
99d64f4554SDavid Howells goto wait_for_file_wrangling;
100d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_CREATING:
101d64f4554SDavid Howells if (want_state == FSCACHE_WANT_PARAMS)
102d64f4554SDavid Howells goto ready; /* There can be no content */
103d64f4554SDavid Howells goto wait_for_file_wrangling;
104d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_ACTIVE:
105d64f4554SDavid Howells goto ready;
106d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_DROPPED:
107d64f4554SDavid Howells case FSCACHE_COOKIE_STATE_RELINQUISHING:
108d64f4554SDavid Howells WARN(1, "Can't use cookie in state %u\n", cookie->state);
109d64f4554SDavid Howells goto not_live;
110d64f4554SDavid Howells default:
111d64f4554SDavid Howells goto not_live;
112d64f4554SDavid Howells }
113d64f4554SDavid Howells
114d64f4554SDavid Howells ready:
115d64f4554SDavid Howells spin_unlock(&cookie->lock);
116d64f4554SDavid Howells if (!cookie->volume->cache->ops->begin_operation(cres, want_state))
117d64f4554SDavid Howells goto failed;
118d64f4554SDavid Howells return 0;
119d64f4554SDavid Howells
120d64f4554SDavid Howells wait_for_file_wrangling:
121d64f4554SDavid Howells spin_unlock(&cookie->lock);
122d64f4554SDavid Howells trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
123d64f4554SDavid Howells atomic_read(&cookie->n_accesses),
124d64f4554SDavid Howells fscache_access_io_wait);
125d64f4554SDavid Howells timeo = wait_var_event_timeout(&cookie->state,
126d64f4554SDavid Howells fscache_cookie_state(cookie) != state, 20 * HZ);
127d64f4554SDavid Howells if (timeo <= 1 && !once_only) {
128d64f4554SDavid Howells pr_warn("%s: cookie state change wait timed out: cookie->state=%u state=%u",
129d64f4554SDavid Howells __func__, fscache_cookie_state(cookie), state);
130d64f4554SDavid Howells fscache_print_cookie(cookie, 'O');
131d64f4554SDavid Howells once_only = true;
132d64f4554SDavid Howells }
133d64f4554SDavid Howells goto again;
134d64f4554SDavid Howells
135d64f4554SDavid Howells not_live:
136d64f4554SDavid Howells spin_unlock(&cookie->lock);
137d64f4554SDavid Howells failed:
138d64f4554SDavid Howells cres->cache_priv = NULL;
139d64f4554SDavid Howells cres->ops = NULL;
140d64f4554SDavid Howells fscache_end_cookie_access(cookie, fscache_access_io_not_live);
141d64f4554SDavid Howells _leave(" = -ENOBUFS");
142d64f4554SDavid Howells return -ENOBUFS;
143d64f4554SDavid Howells }
144d64f4554SDavid Howells
__fscache_begin_read_operation(struct netfs_cache_resources * cres,struct fscache_cookie * cookie)145d64f4554SDavid Howells int __fscache_begin_read_operation(struct netfs_cache_resources *cres,
146d64f4554SDavid Howells struct fscache_cookie *cookie)
147d64f4554SDavid Howells {
148d64f4554SDavid Howells return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS,
149d64f4554SDavid Howells fscache_access_io_read);
150d64f4554SDavid Howells }
151d64f4554SDavid Howells EXPORT_SYMBOL(__fscache_begin_read_operation);
152b6e16652SDavid Howells
__fscache_begin_write_operation(struct netfs_cache_resources * cres,struct fscache_cookie * cookie)15316f2f4e6SDavid Howells int __fscache_begin_write_operation(struct netfs_cache_resources *cres,
15416f2f4e6SDavid Howells struct fscache_cookie *cookie)
15516f2f4e6SDavid Howells {
15616f2f4e6SDavid Howells return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS,
15716f2f4e6SDavid Howells fscache_access_io_write);
15816f2f4e6SDavid Howells }
15916f2f4e6SDavid Howells EXPORT_SYMBOL(__fscache_begin_write_operation);
16016f2f4e6SDavid Howells
16108276bdaSDavid Howells /**
1628fb72b4aSMatthew Wilcox (Oracle) * fscache_dirty_folio - Mark folio dirty and pin a cache object for writeback
1638fb72b4aSMatthew Wilcox (Oracle) * @mapping: The mapping the folio belongs to.
1648fb72b4aSMatthew Wilcox (Oracle) * @folio: The folio being dirtied.
16508276bdaSDavid Howells * @cookie: The cookie referring to the cache object
16608276bdaSDavid Howells *
1678fb72b4aSMatthew Wilcox (Oracle) * Set the dirty flag on a folio and pin an in-use cache object in memory
1688fb72b4aSMatthew Wilcox (Oracle) * so that writeback can later write to it. This is intended
1698fb72b4aSMatthew Wilcox (Oracle) * to be called from the filesystem's ->dirty_folio() method.
17008276bdaSDavid Howells *
1718fb72b4aSMatthew Wilcox (Oracle) * Return: true if the dirty flag was set on the folio, false otherwise.
17208276bdaSDavid Howells */
fscache_dirty_folio(struct address_space * mapping,struct folio * folio,struct fscache_cookie * cookie)1738fb72b4aSMatthew Wilcox (Oracle) bool fscache_dirty_folio(struct address_space *mapping, struct folio *folio,
1748fb72b4aSMatthew Wilcox (Oracle) struct fscache_cookie *cookie)
17508276bdaSDavid Howells {
1768fb72b4aSMatthew Wilcox (Oracle) struct inode *inode = mapping->host;
17708276bdaSDavid Howells bool need_use = false;
17808276bdaSDavid Howells
17908276bdaSDavid Howells _enter("");
18008276bdaSDavid Howells
1818fb72b4aSMatthew Wilcox (Oracle) if (!filemap_dirty_folio(mapping, folio))
1828fb72b4aSMatthew Wilcox (Oracle) return false;
18308276bdaSDavid Howells if (!fscache_cookie_valid(cookie))
1848fb72b4aSMatthew Wilcox (Oracle) return true;
18508276bdaSDavid Howells
18608276bdaSDavid Howells if (!(inode->i_state & I_PINNING_FSCACHE_WB)) {
18708276bdaSDavid Howells spin_lock(&inode->i_lock);
18808276bdaSDavid Howells if (!(inode->i_state & I_PINNING_FSCACHE_WB)) {
18908276bdaSDavid Howells inode->i_state |= I_PINNING_FSCACHE_WB;
19008276bdaSDavid Howells need_use = true;
19108276bdaSDavid Howells }
19208276bdaSDavid Howells spin_unlock(&inode->i_lock);
19308276bdaSDavid Howells
19408276bdaSDavid Howells if (need_use)
19508276bdaSDavid Howells fscache_use_cookie(cookie, true);
19608276bdaSDavid Howells }
1978fb72b4aSMatthew Wilcox (Oracle) return true;
19808276bdaSDavid Howells }
1998fb72b4aSMatthew Wilcox (Oracle) EXPORT_SYMBOL(fscache_dirty_folio);
20008276bdaSDavid Howells
201b6e16652SDavid Howells struct fscache_write_request {
202b6e16652SDavid Howells struct netfs_cache_resources cache_resources;
203b6e16652SDavid Howells struct address_space *mapping;
204b6e16652SDavid Howells loff_t start;
205b6e16652SDavid Howells size_t len;
206b6e16652SDavid Howells bool set_bits;
207b6e16652SDavid Howells netfs_io_terminated_t term_func;
208b6e16652SDavid Howells void *term_func_priv;
209b6e16652SDavid Howells };
210b6e16652SDavid Howells
__fscache_clear_page_bits(struct address_space * mapping,loff_t start,size_t len)211b6e16652SDavid Howells void __fscache_clear_page_bits(struct address_space *mapping,
212b6e16652SDavid Howells loff_t start, size_t len)
213b6e16652SDavid Howells {
214b6e16652SDavid Howells pgoff_t first = start / PAGE_SIZE;
215b6e16652SDavid Howells pgoff_t last = (start + len - 1) / PAGE_SIZE;
216b6e16652SDavid Howells struct page *page;
217b6e16652SDavid Howells
218b6e16652SDavid Howells if (len) {
219b6e16652SDavid Howells XA_STATE(xas, &mapping->i_pages, first);
220b6e16652SDavid Howells
221b6e16652SDavid Howells rcu_read_lock();
222b6e16652SDavid Howells xas_for_each(&xas, page, last) {
223b6e16652SDavid Howells end_page_fscache(page);
224b6e16652SDavid Howells }
225b6e16652SDavid Howells rcu_read_unlock();
226b6e16652SDavid Howells }
227b6e16652SDavid Howells }
228b6e16652SDavid Howells EXPORT_SYMBOL(__fscache_clear_page_bits);
229b6e16652SDavid Howells
230b6e16652SDavid Howells /*
231b6e16652SDavid Howells * Deal with the completion of writing the data to the cache.
232b6e16652SDavid Howells */
fscache_wreq_done(void * priv,ssize_t transferred_or_error,bool was_async)233b6e16652SDavid Howells static void fscache_wreq_done(void *priv, ssize_t transferred_or_error,
234b6e16652SDavid Howells bool was_async)
235b6e16652SDavid Howells {
236b6e16652SDavid Howells struct fscache_write_request *wreq = priv;
237b6e16652SDavid Howells
2382c547f29SYue Hu fscache_clear_page_bits(wreq->mapping, wreq->start, wreq->len,
239b6e16652SDavid Howells wreq->set_bits);
240b6e16652SDavid Howells
241b6e16652SDavid Howells if (wreq->term_func)
242b6e16652SDavid Howells wreq->term_func(wreq->term_func_priv, transferred_or_error,
243b6e16652SDavid Howells was_async);
244b6e16652SDavid Howells fscache_end_operation(&wreq->cache_resources);
245b6e16652SDavid Howells kfree(wreq);
246b6e16652SDavid Howells }
247b6e16652SDavid Howells
__fscache_write_to_cache(struct fscache_cookie * cookie,struct address_space * mapping,loff_t start,size_t len,loff_t i_size,netfs_io_terminated_t term_func,void * term_func_priv,bool cond)248b6e16652SDavid Howells void __fscache_write_to_cache(struct fscache_cookie *cookie,
249b6e16652SDavid Howells struct address_space *mapping,
250b6e16652SDavid Howells loff_t start, size_t len, loff_t i_size,
251b6e16652SDavid Howells netfs_io_terminated_t term_func,
252b6e16652SDavid Howells void *term_func_priv,
253b6e16652SDavid Howells bool cond)
254b6e16652SDavid Howells {
255b6e16652SDavid Howells struct fscache_write_request *wreq;
256b6e16652SDavid Howells struct netfs_cache_resources *cres;
257b6e16652SDavid Howells struct iov_iter iter;
258b6e16652SDavid Howells int ret = -ENOBUFS;
259b6e16652SDavid Howells
260b6e16652SDavid Howells if (len == 0)
261b6e16652SDavid Howells goto abandon;
262b6e16652SDavid Howells
263b6e16652SDavid Howells _enter("%llx,%zx", start, len);
264b6e16652SDavid Howells
265b6e16652SDavid Howells wreq = kzalloc(sizeof(struct fscache_write_request), GFP_NOFS);
266b6e16652SDavid Howells if (!wreq)
267b6e16652SDavid Howells goto abandon;
268b6e16652SDavid Howells wreq->mapping = mapping;
269b6e16652SDavid Howells wreq->start = start;
270b6e16652SDavid Howells wreq->len = len;
271b6e16652SDavid Howells wreq->set_bits = cond;
272b6e16652SDavid Howells wreq->term_func = term_func;
273b6e16652SDavid Howells wreq->term_func_priv = term_func_priv;
274b6e16652SDavid Howells
275b6e16652SDavid Howells cres = &wreq->cache_resources;
276b6e16652SDavid Howells if (fscache_begin_operation(cres, cookie, FSCACHE_WANT_WRITE,
277b6e16652SDavid Howells fscache_access_io_write) < 0)
278b6e16652SDavid Howells goto abandon_free;
279b6e16652SDavid Howells
280b6e16652SDavid Howells ret = cres->ops->prepare_write(cres, &start, &len, i_size, false);
281b6e16652SDavid Howells if (ret < 0)
282b6e16652SDavid Howells goto abandon_end;
283b6e16652SDavid Howells
284b6e16652SDavid Howells /* TODO: Consider clearing page bits now for space the write isn't
285b6e16652SDavid Howells * covering. This is more complicated than it appears when THPs are
286b6e16652SDavid Howells * taken into account.
287b6e16652SDavid Howells */
288b6e16652SDavid Howells
289*de4eda9dSAl Viro iov_iter_xarray(&iter, ITER_SOURCE, &mapping->i_pages, start, len);
290b6e16652SDavid Howells fscache_write(cres, start, &iter, fscache_wreq_done, wreq);
291b6e16652SDavid Howells return;
292b6e16652SDavid Howells
293b6e16652SDavid Howells abandon_end:
294b6e16652SDavid Howells return fscache_wreq_done(wreq, ret, false);
295b6e16652SDavid Howells abandon_free:
296b6e16652SDavid Howells kfree(wreq);
297b6e16652SDavid Howells abandon:
2982c547f29SYue Hu fscache_clear_page_bits(mapping, start, len, cond);
299b6e16652SDavid Howells if (term_func)
300b6e16652SDavid Howells term_func(term_func_priv, ret, false);
301b6e16652SDavid Howells }
302b6e16652SDavid Howells EXPORT_SYMBOL(__fscache_write_to_cache);
30316a96bdfSDavid Howells
30416a96bdfSDavid Howells /*
30516a96bdfSDavid Howells * Change the size of a backing object.
30616a96bdfSDavid Howells */
__fscache_resize_cookie(struct fscache_cookie * cookie,loff_t new_size)30716a96bdfSDavid Howells void __fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size)
30816a96bdfSDavid Howells {
30916a96bdfSDavid Howells struct netfs_cache_resources cres;
31016a96bdfSDavid Howells
31116a96bdfSDavid Howells trace_fscache_resize(cookie, new_size);
31216a96bdfSDavid Howells if (fscache_begin_operation(&cres, cookie, FSCACHE_WANT_WRITE,
31316a96bdfSDavid Howells fscache_access_io_resize) == 0) {
31416a96bdfSDavid Howells fscache_stat(&fscache_n_resizes);
31516a96bdfSDavid Howells set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags);
31616a96bdfSDavid Howells
31716a96bdfSDavid Howells /* We cannot defer a resize as we need to do it inside the
31816a96bdfSDavid Howells * netfs's inode lock so that we're serialised with respect to
31916a96bdfSDavid Howells * writes.
32016a96bdfSDavid Howells */
32116a96bdfSDavid Howells cookie->volume->cache->ops->resize_cookie(&cres, new_size);
32216a96bdfSDavid Howells fscache_end_operation(&cres);
32316a96bdfSDavid Howells } else {
32416a96bdfSDavid Howells fscache_stat(&fscache_n_resizes_null);
32516a96bdfSDavid Howells }
32616a96bdfSDavid Howells }
32716a96bdfSDavid Howells EXPORT_SYMBOL(__fscache_resize_cookie);
328