xref: /openbmc/linux/fs/nfs/pnfs.c (revision d8a7055f)
1 /*
2  *  pNFS functions to call and manage layout drivers.
3  *
4  *  Copyright (c) 2002 [year of first publication]
5  *  The Regents of the University of Michigan
6  *  All Rights Reserved
7  *
8  *  Dean Hildebrand <dhildebz@umich.edu>
9  *
10  *  Permission is granted to use, copy, create derivative works, and
11  *  redistribute this software and such derivative works for any purpose,
12  *  so long as the name of the University of Michigan is not used in
13  *  any advertising or publicity pertaining to the use or distribution
14  *  of this software without specific, written prior authorization. If
15  *  the above copyright notice or any other identification of the
16  *  University of Michigan is included in any copy of any portion of
17  *  this software, then the disclaimer below must also be included.
18  *
19  *  This software is provided as is, without representation or warranty
20  *  of any kind either express or implied, including without limitation
21  *  the implied warranties of merchantability, fitness for a particular
22  *  purpose, or noninfringement.  The Regents of the University of
23  *  Michigan shall not be liable for any damages, including special,
24  *  indirect, incidental, or consequential damages, with respect to any
25  *  claim arising out of or in connection with the use of the software,
26  *  even if it has been or is hereafter advised of the possibility of
27  *  such damages.
28  */
29 
30 #include <linux/nfs_fs.h>
31 #include <linux/nfs_page.h>
32 #include <linux/module.h>
33 #include <linux/sort.h>
34 #include "internal.h"
35 #include "pnfs.h"
36 #include "iostat.h"
37 #include "nfs4trace.h"
38 #include "delegation.h"
39 #include "nfs42.h"
40 #include "nfs4_fs.h"
41 
42 #define NFSDBG_FACILITY		NFSDBG_PNFS
43 #define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
44 
45 /* Locking:
46  *
47  * pnfs_spinlock:
48  *      protects pnfs_modules_tbl.
49  */
50 static DEFINE_SPINLOCK(pnfs_spinlock);
51 
52 /*
53  * pnfs_modules_tbl holds all pnfs modules
54  */
55 static LIST_HEAD(pnfs_modules_tbl);
56 
57 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo);
58 static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
59 		struct list_head *free_me,
60 		const struct pnfs_layout_range *range,
61 		u32 seq);
62 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
63 		                struct list_head *tmp_list);
64 
65 /* Return the registered pnfs layout driver module matching given id */
66 static struct pnfs_layoutdriver_type *
find_pnfs_driver_locked(u32 id)67 find_pnfs_driver_locked(u32 id)
68 {
69 	struct pnfs_layoutdriver_type *local;
70 
71 	list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
72 		if (local->id == id)
73 			goto out;
74 	local = NULL;
75 out:
76 	dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
77 	return local;
78 }
79 
80 static struct pnfs_layoutdriver_type *
find_pnfs_driver(u32 id)81 find_pnfs_driver(u32 id)
82 {
83 	struct pnfs_layoutdriver_type *local;
84 
85 	spin_lock(&pnfs_spinlock);
86 	local = find_pnfs_driver_locked(id);
87 	if (local != NULL && !try_module_get(local->owner)) {
88 		dprintk("%s: Could not grab reference on module\n", __func__);
89 		local = NULL;
90 	}
91 	spin_unlock(&pnfs_spinlock);
92 	return local;
93 }
94 
pnfs_find_layoutdriver(u32 id)95 const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id)
96 {
97 	return find_pnfs_driver(id);
98 }
99 
pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type * ld)100 void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld)
101 {
102 	if (ld)
103 		module_put(ld->owner);
104 }
105 
106 void
unset_pnfs_layoutdriver(struct nfs_server * nfss)107 unset_pnfs_layoutdriver(struct nfs_server *nfss)
108 {
109 	if (nfss->pnfs_curr_ld) {
110 		if (nfss->pnfs_curr_ld->clear_layoutdriver)
111 			nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
112 		/* Decrement the MDS count. Purge the deviceid cache if zero */
113 		if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
114 			nfs4_deviceid_purge_client(nfss->nfs_client);
115 		module_put(nfss->pnfs_curr_ld->owner);
116 	}
117 	nfss->pnfs_curr_ld = NULL;
118 }
119 
120 /*
121  * When the server sends a list of layout types, we choose one in the order
122  * given in the list below.
123  *
124  * FIXME: should this list be configurable in some fashion? module param?
125  * 	  mount option? something else?
126  */
127 static const u32 ld_prefs[] = {
128 	LAYOUT_SCSI,
129 	LAYOUT_BLOCK_VOLUME,
130 	LAYOUT_OSD2_OBJECTS,
131 	LAYOUT_FLEX_FILES,
132 	LAYOUT_NFSV4_1_FILES,
133 	0
134 };
135 
136 static int
ld_cmp(const void * e1,const void * e2)137 ld_cmp(const void *e1, const void *e2)
138 {
139 	u32 ld1 = *((u32 *)e1);
140 	u32 ld2 = *((u32 *)e2);
141 	int i;
142 
143 	for (i = 0; ld_prefs[i] != 0; i++) {
144 		if (ld1 == ld_prefs[i])
145 			return -1;
146 
147 		if (ld2 == ld_prefs[i])
148 			return 1;
149 	}
150 	return 0;
151 }
152 
153 /*
154  * Try to set the server's pnfs module to the pnfs layout type specified by id.
155  * Currently only one pNFS layout driver per filesystem is supported.
156  *
157  * @ids array of layout types supported by MDS.
158  */
159 void
set_pnfs_layoutdriver(struct nfs_server * server,const struct nfs_fh * mntfh,struct nfs_fsinfo * fsinfo)160 set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
161 		      struct nfs_fsinfo *fsinfo)
162 {
163 	struct pnfs_layoutdriver_type *ld_type = NULL;
164 	u32 id;
165 	int i;
166 
167 	if (fsinfo->nlayouttypes == 0)
168 		goto out_no_driver;
169 	if (!(server->nfs_client->cl_exchange_flags &
170 		 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
171 		printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n",
172 			__func__, server->nfs_client->cl_exchange_flags);
173 		goto out_no_driver;
174 	}
175 
176 	sort(fsinfo->layouttype, fsinfo->nlayouttypes,
177 		sizeof(*fsinfo->layouttype), ld_cmp, NULL);
178 
179 	for (i = 0; i < fsinfo->nlayouttypes; i++) {
180 		id = fsinfo->layouttype[i];
181 		ld_type = find_pnfs_driver(id);
182 		if (!ld_type) {
183 			request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX,
184 					id);
185 			ld_type = find_pnfs_driver(id);
186 		}
187 		if (ld_type)
188 			break;
189 	}
190 
191 	if (!ld_type) {
192 		dprintk("%s: No pNFS module found!\n", __func__);
193 		goto out_no_driver;
194 	}
195 
196 	server->pnfs_curr_ld = ld_type;
197 	if (ld_type->set_layoutdriver
198 	    && ld_type->set_layoutdriver(server, mntfh)) {
199 		printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
200 			"driver %u.\n", __func__, id);
201 		module_put(ld_type->owner);
202 		goto out_no_driver;
203 	}
204 	/* Bump the MDS count */
205 	atomic_inc(&server->nfs_client->cl_mds_count);
206 
207 	dprintk("%s: pNFS module for %u set\n", __func__, id);
208 	return;
209 
210 out_no_driver:
211 	dprintk("%s: Using NFSv4 I/O\n", __func__);
212 	server->pnfs_curr_ld = NULL;
213 }
214 
215 int
pnfs_register_layoutdriver(struct pnfs_layoutdriver_type * ld_type)216 pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
217 {
218 	int status = -EINVAL;
219 	struct pnfs_layoutdriver_type *tmp;
220 
221 	if (ld_type->id == 0) {
222 		printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
223 		return status;
224 	}
225 	if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
226 		printk(KERN_ERR "NFS: %s Layout driver must provide "
227 		       "alloc_lseg and free_lseg.\n", __func__);
228 		return status;
229 	}
230 
231 	spin_lock(&pnfs_spinlock);
232 	tmp = find_pnfs_driver_locked(ld_type->id);
233 	if (!tmp) {
234 		list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
235 		status = 0;
236 		dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
237 			ld_type->name);
238 	} else {
239 		printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
240 			__func__, ld_type->id);
241 	}
242 	spin_unlock(&pnfs_spinlock);
243 
244 	return status;
245 }
246 EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);
247 
248 void
pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type * ld_type)249 pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
250 {
251 	dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
252 	spin_lock(&pnfs_spinlock);
253 	list_del(&ld_type->pnfs_tblid);
254 	spin_unlock(&pnfs_spinlock);
255 }
256 EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
257 
258 /*
259  * pNFS client layout cache
260  */
261 
262 /* Need to hold i_lock if caller does not already hold reference */
263 void
pnfs_get_layout_hdr(struct pnfs_layout_hdr * lo)264 pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
265 {
266 	refcount_inc(&lo->plh_refcount);
267 }
268 
269 static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode * ino,gfp_t gfp_flags)270 pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
271 {
272 	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
273 	return ld->alloc_layout_hdr(ino, gfp_flags);
274 }
275 
276 static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr * lo)277 pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
278 {
279 	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
280 	struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
281 
282 	if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) {
283 		struct nfs_client *clp = server->nfs_client;
284 
285 		spin_lock(&clp->cl_lock);
286 		list_del_rcu(&lo->plh_layouts);
287 		spin_unlock(&clp->cl_lock);
288 	}
289 	put_cred(lo->plh_lc_cred);
290 	return ld->free_layout_hdr(lo);
291 }
292 
293 static void
pnfs_detach_layout_hdr(struct pnfs_layout_hdr * lo)294 pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
295 {
296 	struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
297 	dprintk("%s: freeing layout cache %p\n", __func__, lo);
298 	nfsi->layout = NULL;
299 	/* Reset MDS Threshold I/O counters */
300 	nfsi->write_io = 0;
301 	nfsi->read_io = 0;
302 }
303 
304 void
pnfs_put_layout_hdr(struct pnfs_layout_hdr * lo)305 pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
306 {
307 	struct inode *inode;
308 	unsigned long i_state;
309 
310 	if (!lo)
311 		return;
312 	inode = lo->plh_inode;
313 	pnfs_layoutreturn_before_put_layout_hdr(lo);
314 
315 	if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
316 		if (!list_empty(&lo->plh_segs))
317 			WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
318 		pnfs_detach_layout_hdr(lo);
319 		i_state = inode->i_state;
320 		spin_unlock(&inode->i_lock);
321 		pnfs_free_layout_hdr(lo);
322 		/* Notify pnfs_destroy_layout_final() that we're done */
323 		if (i_state & (I_FREEING | I_CLEAR))
324 			wake_up_var(lo);
325 	}
326 }
327 
328 static struct inode *
pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr * lo)329 pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo)
330 {
331 	struct inode *inode = igrab(lo->plh_inode);
332 	if (inode)
333 		return inode;
334 	set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags);
335 	return NULL;
336 }
337 
338 /*
339  * Compare 2 layout stateid sequence ids, to see which is newer,
340  * taking into account wraparound issues.
341  */
pnfs_seqid_is_newer(u32 s1,u32 s2)342 static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
343 {
344 	return (s32)(s1 - s2) > 0;
345 }
346 
pnfs_barrier_update(struct pnfs_layout_hdr * lo,u32 newseq)347 static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq)
348 {
349 	if (pnfs_seqid_is_newer(newseq, lo->plh_barrier) || !lo->plh_barrier)
350 		lo->plh_barrier = newseq;
351 }
352 
353 static void
pnfs_set_plh_return_info(struct pnfs_layout_hdr * lo,enum pnfs_iomode iomode,u32 seq)354 pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode,
355 			 u32 seq)
356 {
357 	if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode)
358 		iomode = IOMODE_ANY;
359 	lo->plh_return_iomode = iomode;
360 	set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
361 	/*
362 	 * We must set lo->plh_return_seq to avoid livelocks with
363 	 * pnfs_layout_need_return()
364 	 */
365 	if (seq == 0)
366 		seq = be32_to_cpu(lo->plh_stateid.seqid);
367 	if (!lo->plh_return_seq || pnfs_seqid_is_newer(seq, lo->plh_return_seq))
368 		lo->plh_return_seq = seq;
369 	pnfs_barrier_update(lo, seq);
370 }
371 
372 static void
pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr * lo)373 pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo)
374 {
375 	struct pnfs_layout_segment *lseg;
376 	lo->plh_return_iomode = 0;
377 	lo->plh_return_seq = 0;
378 	clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
379 	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
380 		if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
381 			continue;
382 		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
383 	}
384 }
385 
pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr * lo)386 static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
387 {
388 	clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
389 	clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags);
390 	smp_mb__after_atomic();
391 	wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
392 	rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
393 }
394 
395 static void
pnfs_clear_lseg_state(struct pnfs_layout_segment * lseg,struct list_head * free_me)396 pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg,
397 		struct list_head *free_me)
398 {
399 	clear_bit(NFS_LSEG_ROC, &lseg->pls_flags);
400 	clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
401 	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags))
402 		pnfs_lseg_dec_and_remove_zero(lseg, free_me);
403 	if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
404 		pnfs_lseg_dec_and_remove_zero(lseg, free_me);
405 }
406 
407 /*
408  * Update the seqid of a layout stateid after receiving
409  * NFS4ERR_OLD_STATEID
410  */
nfs4_layout_refresh_old_stateid(nfs4_stateid * dst,struct pnfs_layout_range * dst_range,struct inode * inode)411 bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst,
412 		struct pnfs_layout_range *dst_range,
413 		struct inode *inode)
414 {
415 	struct pnfs_layout_hdr *lo;
416 	struct pnfs_layout_range range = {
417 		.iomode = IOMODE_ANY,
418 		.offset = 0,
419 		.length = NFS4_MAX_UINT64,
420 	};
421 	bool ret = false;
422 	LIST_HEAD(head);
423 	int err;
424 
425 	spin_lock(&inode->i_lock);
426 	lo = NFS_I(inode)->layout;
427 	if (lo &&  pnfs_layout_is_valid(lo) &&
428 	    nfs4_stateid_match_other(dst, &lo->plh_stateid)) {
429 		/* Is our call using the most recent seqid? If so, bump it */
430 		if (!nfs4_stateid_is_newer(&lo->plh_stateid, dst)) {
431 			nfs4_stateid_seqid_inc(dst);
432 			ret = true;
433 			goto out;
434 		}
435 		/* Try to update the seqid to the most recent */
436 		err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0);
437 		if (err != -EBUSY) {
438 			dst->seqid = lo->plh_stateid.seqid;
439 			*dst_range = range;
440 			ret = true;
441 		}
442 	}
443 out:
444 	spin_unlock(&inode->i_lock);
445 	pnfs_free_lseg_list(&head);
446 	return ret;
447 }
448 
449 /*
450  * Mark a pnfs_layout_hdr and all associated layout segments as invalid
451  *
452  * In order to continue using the pnfs_layout_hdr, a full recovery
453  * is required.
454  * Note that caller must hold inode->i_lock.
455  */
456 int
pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr * lo,struct list_head * lseg_list)457 pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo,
458 		struct list_head *lseg_list)
459 {
460 	struct pnfs_layout_range range = {
461 		.iomode = IOMODE_ANY,
462 		.offset = 0,
463 		.length = NFS4_MAX_UINT64,
464 	};
465 	struct pnfs_layout_segment *lseg, *next;
466 
467 	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
468 	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
469 		pnfs_clear_lseg_state(lseg, lseg_list);
470 	pnfs_clear_layoutreturn_info(lo);
471 	pnfs_free_returned_lsegs(lo, lseg_list, &range, 0);
472 	set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags);
473 	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) &&
474 	    !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
475 		pnfs_clear_layoutreturn_waitbit(lo);
476 	return !list_empty(&lo->plh_segs);
477 }
478 
479 static int
pnfs_iomode_to_fail_bit(u32 iomode)480 pnfs_iomode_to_fail_bit(u32 iomode)
481 {
482 	return iomode == IOMODE_RW ?
483 		NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
484 }
485 
486 static void
pnfs_layout_set_fail_bit(struct pnfs_layout_hdr * lo,int fail_bit)487 pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
488 {
489 	lo->plh_retry_timestamp = jiffies;
490 	if (!test_and_set_bit(fail_bit, &lo->plh_flags))
491 		refcount_inc(&lo->plh_refcount);
492 }
493 
494 static void
pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr * lo,int fail_bit)495 pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
496 {
497 	if (test_and_clear_bit(fail_bit, &lo->plh_flags))
498 		refcount_dec(&lo->plh_refcount);
499 }
500 
501 static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr * lo,u32 iomode)502 pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
503 {
504 	struct inode *inode = lo->plh_inode;
505 	struct pnfs_layout_range range = {
506 		.iomode = iomode,
507 		.offset = 0,
508 		.length = NFS4_MAX_UINT64,
509 	};
510 	LIST_HEAD(head);
511 
512 	spin_lock(&inode->i_lock);
513 	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
514 	pnfs_mark_matching_lsegs_return(lo, &head, &range, 0);
515 	spin_unlock(&inode->i_lock);
516 	pnfs_free_lseg_list(&head);
517 	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
518 			iomode == IOMODE_RW ?  "RW" : "READ");
519 }
520 
521 static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr * lo,u32 iomode)522 pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
523 {
524 	unsigned long start, end;
525 	int fail_bit = pnfs_iomode_to_fail_bit(iomode);
526 
527 	if (test_bit(fail_bit, &lo->plh_flags) == 0)
528 		return false;
529 	end = jiffies;
530 	start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
531 	if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
532 		/* It is time to retry the failed layoutgets */
533 		pnfs_layout_clear_fail_bit(lo, fail_bit);
534 		return false;
535 	}
536 	return true;
537 }
538 
539 static void
pnfs_init_lseg(struct pnfs_layout_hdr * lo,struct pnfs_layout_segment * lseg,const struct pnfs_layout_range * range,const nfs4_stateid * stateid)540 pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg,
541 		const struct pnfs_layout_range *range,
542 		const nfs4_stateid *stateid)
543 {
544 	INIT_LIST_HEAD(&lseg->pls_list);
545 	INIT_LIST_HEAD(&lseg->pls_lc_list);
546 	INIT_LIST_HEAD(&lseg->pls_commits);
547 	refcount_set(&lseg->pls_refcount, 1);
548 	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
549 	lseg->pls_layout = lo;
550 	lseg->pls_range = *range;
551 	lseg->pls_seq = be32_to_cpu(stateid->seqid);
552 }
553 
pnfs_free_lseg(struct pnfs_layout_segment * lseg)554 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
555 {
556 	if (lseg != NULL) {
557 		struct inode *inode = lseg->pls_layout->plh_inode;
558 		NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg);
559 	}
560 }
561 
562 static void
pnfs_layout_remove_lseg(struct pnfs_layout_hdr * lo,struct pnfs_layout_segment * lseg)563 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
564 		struct pnfs_layout_segment *lseg)
565 {
566 	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
567 	list_del_init(&lseg->pls_list);
568 	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
569 	refcount_dec(&lo->plh_refcount);
570 	if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
571 		return;
572 	if (list_empty(&lo->plh_segs) &&
573 	    !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) &&
574 	    !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
575 		if (atomic_read(&lo->plh_outstanding) == 0)
576 			set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
577 		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
578 	}
579 }
580 
581 static bool
pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr * lo,struct pnfs_layout_segment * lseg)582 pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo,
583 		struct pnfs_layout_segment *lseg)
584 {
585 	if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
586 	    pnfs_layout_is_valid(lo)) {
587 		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
588 		list_move_tail(&lseg->pls_list, &lo->plh_return_segs);
589 		return true;
590 	}
591 	return false;
592 }
593 
594 void
pnfs_put_lseg(struct pnfs_layout_segment * lseg)595 pnfs_put_lseg(struct pnfs_layout_segment *lseg)
596 {
597 	struct pnfs_layout_hdr *lo;
598 	struct inode *inode;
599 
600 	if (!lseg)
601 		return;
602 
603 	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
604 		refcount_read(&lseg->pls_refcount),
605 		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
606 
607 	lo = lseg->pls_layout;
608 	inode = lo->plh_inode;
609 
610 	if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
611 		pnfs_get_layout_hdr(lo);
612 		pnfs_layout_remove_lseg(lo, lseg);
613 		if (pnfs_cache_lseg_for_layoutreturn(lo, lseg))
614 			lseg = NULL;
615 		spin_unlock(&inode->i_lock);
616 		pnfs_free_lseg(lseg);
617 		pnfs_put_layout_hdr(lo);
618 	}
619 }
620 EXPORT_SYMBOL_GPL(pnfs_put_lseg);
621 
622 /*
623  * is l2 fully contained in l1?
624  *   start1                             end1
625  *   [----------------------------------)
626  *           start2           end2
627  *           [----------------)
628  */
629 static bool
pnfs_lseg_range_contained(const struct pnfs_layout_range * l1,const struct pnfs_layout_range * l2)630 pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
631 		 const struct pnfs_layout_range *l2)
632 {
633 	u64 start1 = l1->offset;
634 	u64 end1 = pnfs_end_offset(start1, l1->length);
635 	u64 start2 = l2->offset;
636 	u64 end2 = pnfs_end_offset(start2, l2->length);
637 
638 	return (start1 <= start2) && (end1 >= end2);
639 }
640 
pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment * lseg,struct list_head * tmp_list)641 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
642 		struct list_head *tmp_list)
643 {
644 	if (!refcount_dec_and_test(&lseg->pls_refcount))
645 		return false;
646 	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
647 	list_add(&lseg->pls_list, tmp_list);
648 	return true;
649 }
650 
651 /* Returns 1 if lseg is removed from list, 0 otherwise */
mark_lseg_invalid(struct pnfs_layout_segment * lseg,struct list_head * tmp_list)652 static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
653 			     struct list_head *tmp_list)
654 {
655 	int rv = 0;
656 
657 	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
658 		/* Remove the reference keeping the lseg in the
659 		 * list.  It will now be removed when all
660 		 * outstanding io is finished.
661 		 */
662 		dprintk("%s: lseg %p ref %d\n", __func__, lseg,
663 			refcount_read(&lseg->pls_refcount));
664 		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
665 			rv = 1;
666 	}
667 	return rv;
668 }
669 
670 static bool
pnfs_should_free_range(const struct pnfs_layout_range * lseg_range,const struct pnfs_layout_range * recall_range)671 pnfs_should_free_range(const struct pnfs_layout_range *lseg_range,
672 		 const struct pnfs_layout_range *recall_range)
673 {
674 	return (recall_range->iomode == IOMODE_ANY ||
675 		lseg_range->iomode == recall_range->iomode) &&
676 	       pnfs_lseg_range_intersecting(lseg_range, recall_range);
677 }
678 
679 static bool
pnfs_match_lseg_recall(const struct pnfs_layout_segment * lseg,const struct pnfs_layout_range * recall_range,u32 seq)680 pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg,
681 		const struct pnfs_layout_range *recall_range,
682 		u32 seq)
683 {
684 	if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq))
685 		return false;
686 	if (recall_range == NULL)
687 		return true;
688 	return pnfs_should_free_range(&lseg->pls_range, recall_range);
689 }
690 
691 /**
692  * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later
693  * @lo: layout header containing the lsegs
694  * @tmp_list: list head where doomed lsegs should go
695  * @recall_range: optional recall range argument to match (may be NULL)
696  * @seq: only invalidate lsegs obtained prior to this sequence (may be 0)
697  *
698  * Walk the list of lsegs in the layout header, and tear down any that should
699  * be destroyed. If "recall_range" is specified then the segment must match
700  * that range. If "seq" is non-zero, then only match segments that were handed
701  * out at or before that sequence.
702  *
703  * Returns number of matching invalid lsegs remaining in list after scanning
704  * it and purging them.
705  */
706 int
pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr * lo,struct list_head * tmp_list,const struct pnfs_layout_range * recall_range,u32 seq)707 pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
708 			    struct list_head *tmp_list,
709 			    const struct pnfs_layout_range *recall_range,
710 			    u32 seq)
711 {
712 	struct pnfs_layout_segment *lseg, *next;
713 	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
714 	int remaining = 0;
715 
716 	dprintk("%s:Begin lo %p\n", __func__, lo);
717 
718 	if (list_empty(&lo->plh_segs))
719 		return 0;
720 	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
721 		if (pnfs_match_lseg_recall(lseg, recall_range, seq)) {
722 			dprintk("%s: freeing lseg %p iomode %d seq %u "
723 				"offset %llu length %llu\n", __func__,
724 				lseg, lseg->pls_range.iomode, lseg->pls_seq,
725 				lseg->pls_range.offset, lseg->pls_range.length);
726 			if (mark_lseg_invalid(lseg, tmp_list))
727 				continue;
728 			remaining++;
729 			pnfs_lseg_cancel_io(server, lseg);
730 		}
731 	dprintk("%s:Return %i\n", __func__, remaining);
732 	return remaining;
733 }
734 
735 static void
pnfs_free_returned_lsegs(struct pnfs_layout_hdr * lo,struct list_head * free_me,const struct pnfs_layout_range * range,u32 seq)736 pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
737 		struct list_head *free_me,
738 		const struct pnfs_layout_range *range,
739 		u32 seq)
740 {
741 	struct pnfs_layout_segment *lseg, *next;
742 
743 	list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) {
744 		if (pnfs_match_lseg_recall(lseg, range, seq))
745 			list_move_tail(&lseg->pls_list, free_me);
746 	}
747 }
748 
749 /* note free_me must contain lsegs from a single layout_hdr */
750 void
pnfs_free_lseg_list(struct list_head * free_me)751 pnfs_free_lseg_list(struct list_head *free_me)
752 {
753 	struct pnfs_layout_segment *lseg, *tmp;
754 
755 	if (list_empty(free_me))
756 		return;
757 
758 	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
759 		list_del(&lseg->pls_list);
760 		pnfs_free_lseg(lseg);
761 	}
762 }
763 
__pnfs_destroy_layout(struct nfs_inode * nfsi)764 static struct pnfs_layout_hdr *__pnfs_destroy_layout(struct nfs_inode *nfsi)
765 {
766 	struct pnfs_layout_hdr *lo;
767 	LIST_HEAD(tmp_list);
768 
769 	spin_lock(&nfsi->vfs_inode.i_lock);
770 	lo = nfsi->layout;
771 	if (lo) {
772 		pnfs_get_layout_hdr(lo);
773 		pnfs_mark_layout_stateid_invalid(lo, &tmp_list);
774 		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
775 		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
776 		spin_unlock(&nfsi->vfs_inode.i_lock);
777 		pnfs_free_lseg_list(&tmp_list);
778 		nfs_commit_inode(&nfsi->vfs_inode, 0);
779 		pnfs_put_layout_hdr(lo);
780 	} else
781 		spin_unlock(&nfsi->vfs_inode.i_lock);
782 	return lo;
783 }
784 
pnfs_destroy_layout(struct nfs_inode * nfsi)785 void pnfs_destroy_layout(struct nfs_inode *nfsi)
786 {
787 	__pnfs_destroy_layout(nfsi);
788 }
789 EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
790 
pnfs_layout_removed(struct nfs_inode * nfsi,struct pnfs_layout_hdr * lo)791 static bool pnfs_layout_removed(struct nfs_inode *nfsi,
792 				struct pnfs_layout_hdr *lo)
793 {
794 	bool ret;
795 
796 	spin_lock(&nfsi->vfs_inode.i_lock);
797 	ret = nfsi->layout != lo;
798 	spin_unlock(&nfsi->vfs_inode.i_lock);
799 	return ret;
800 }
801 
pnfs_destroy_layout_final(struct nfs_inode * nfsi)802 void pnfs_destroy_layout_final(struct nfs_inode *nfsi)
803 {
804 	struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi);
805 
806 	if (lo)
807 		wait_var_event(lo, pnfs_layout_removed(nfsi, lo));
808 }
809 
810 static bool
pnfs_layout_add_bulk_destroy_list(struct inode * inode,struct list_head * layout_list)811 pnfs_layout_add_bulk_destroy_list(struct inode *inode,
812 		struct list_head *layout_list)
813 {
814 	struct pnfs_layout_hdr *lo;
815 	bool ret = false;
816 
817 	spin_lock(&inode->i_lock);
818 	lo = NFS_I(inode)->layout;
819 	if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
820 		pnfs_get_layout_hdr(lo);
821 		list_add(&lo->plh_bulk_destroy, layout_list);
822 		ret = true;
823 	}
824 	spin_unlock(&inode->i_lock);
825 	return ret;
826 }
827 
828 /* Caller must hold rcu_read_lock and clp->cl_lock */
829 static int
pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client * clp,struct nfs_server * server,struct list_head * layout_list)830 pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
831 		struct nfs_server *server,
832 		struct list_head *layout_list)
833 	__must_hold(&clp->cl_lock)
834 	__must_hold(RCU)
835 {
836 	struct pnfs_layout_hdr *lo, *next;
837 	struct inode *inode;
838 
839 	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
840 		if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) ||
841 		    test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) ||
842 		    !list_empty(&lo->plh_bulk_destroy))
843 			continue;
844 		/* If the sb is being destroyed, just bail */
845 		if (!nfs_sb_active(server->super))
846 			break;
847 		inode = pnfs_grab_inode_layout_hdr(lo);
848 		if (inode != NULL) {
849 			if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags))
850 				list_del_rcu(&lo->plh_layouts);
851 			if (pnfs_layout_add_bulk_destroy_list(inode,
852 						layout_list))
853 				continue;
854 			rcu_read_unlock();
855 			spin_unlock(&clp->cl_lock);
856 			iput(inode);
857 		} else {
858 			rcu_read_unlock();
859 			spin_unlock(&clp->cl_lock);
860 		}
861 		nfs_sb_deactive(server->super);
862 		spin_lock(&clp->cl_lock);
863 		rcu_read_lock();
864 		return -EAGAIN;
865 	}
866 	return 0;
867 }
868 
869 static int
pnfs_layout_free_bulk_destroy_list(struct list_head * layout_list,bool is_bulk_recall)870 pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
871 		bool is_bulk_recall)
872 {
873 	struct pnfs_layout_hdr *lo;
874 	struct inode *inode;
875 	LIST_HEAD(lseg_list);
876 	int ret = 0;
877 
878 	while (!list_empty(layout_list)) {
879 		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
880 				plh_bulk_destroy);
881 		dprintk("%s freeing layout for inode %lu\n", __func__,
882 			lo->plh_inode->i_ino);
883 		inode = lo->plh_inode;
884 
885 		pnfs_layoutcommit_inode(inode, false);
886 
887 		spin_lock(&inode->i_lock);
888 		list_del_init(&lo->plh_bulk_destroy);
889 		if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) {
890 			if (is_bulk_recall)
891 				set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
892 			ret = -EAGAIN;
893 		}
894 		spin_unlock(&inode->i_lock);
895 		pnfs_free_lseg_list(&lseg_list);
896 		/* Free all lsegs that are attached to commit buckets */
897 		nfs_commit_inode(inode, 0);
898 		pnfs_put_layout_hdr(lo);
899 		nfs_iput_and_deactive(inode);
900 	}
901 	return ret;
902 }
903 
904 int
pnfs_destroy_layouts_byfsid(struct nfs_client * clp,struct nfs_fsid * fsid,bool is_recall)905 pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
906 		struct nfs_fsid *fsid,
907 		bool is_recall)
908 {
909 	struct nfs_server *server;
910 	LIST_HEAD(layout_list);
911 
912 	spin_lock(&clp->cl_lock);
913 	rcu_read_lock();
914 restart:
915 	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
916 		if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
917 			continue;
918 		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
919 				server,
920 				&layout_list) != 0)
921 			goto restart;
922 	}
923 	rcu_read_unlock();
924 	spin_unlock(&clp->cl_lock);
925 
926 	if (list_empty(&layout_list))
927 		return 0;
928 	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
929 }
930 
931 int
pnfs_destroy_layouts_byclid(struct nfs_client * clp,bool is_recall)932 pnfs_destroy_layouts_byclid(struct nfs_client *clp,
933 		bool is_recall)
934 {
935 	struct nfs_server *server;
936 	LIST_HEAD(layout_list);
937 
938 	spin_lock(&clp->cl_lock);
939 	rcu_read_lock();
940 restart:
941 	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
942 		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
943 					server,
944 					&layout_list) != 0)
945 			goto restart;
946 	}
947 	rcu_read_unlock();
948 	spin_unlock(&clp->cl_lock);
949 
950 	if (list_empty(&layout_list))
951 		return 0;
952 	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
953 }
954 
955 /*
956  * Called by the state manager to remove all layouts established under an
957  * expired lease.
958  */
959 void
pnfs_destroy_all_layouts(struct nfs_client * clp)960 pnfs_destroy_all_layouts(struct nfs_client *clp)
961 {
962 	nfs4_deviceid_mark_client_invalid(clp);
963 	nfs4_deviceid_purge_client(clp);
964 
965 	pnfs_destroy_layouts_byclid(clp, false);
966 }
967 
968 static void
pnfs_set_layout_cred(struct pnfs_layout_hdr * lo,const struct cred * cred)969 pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred)
970 {
971 	const struct cred *old;
972 
973 	if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) {
974 		old = xchg(&lo->plh_lc_cred, get_cred(cred));
975 		put_cred(old);
976 	}
977 }
978 
979 /* update lo->plh_stateid with new if is more recent */
980 void
pnfs_set_layout_stateid(struct pnfs_layout_hdr * lo,const nfs4_stateid * new,const struct cred * cred,bool update_barrier)981 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
982 			const struct cred *cred, bool update_barrier)
983 {
984 	u32 oldseq = be32_to_cpu(lo->plh_stateid.seqid);
985 	u32 newseq = be32_to_cpu(new->seqid);
986 
987 	if (!pnfs_layout_is_valid(lo)) {
988 		pnfs_set_layout_cred(lo, cred);
989 		nfs4_stateid_copy(&lo->plh_stateid, new);
990 		lo->plh_barrier = newseq;
991 		pnfs_clear_layoutreturn_info(lo);
992 		clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
993 		return;
994 	}
995 
996 	if (pnfs_seqid_is_newer(newseq, oldseq))
997 		nfs4_stateid_copy(&lo->plh_stateid, new);
998 
999 	if (update_barrier) {
1000 		pnfs_barrier_update(lo, newseq);
1001 		return;
1002 	}
1003 	/*
1004 	 * Because of wraparound, we want to keep the barrier
1005 	 * "close" to the current seqids. We really only want to
1006 	 * get here from a layoutget call.
1007 	 */
1008 	if (atomic_read(&lo->plh_outstanding) == 1)
1009 		 pnfs_barrier_update(lo, be32_to_cpu(lo->plh_stateid.seqid));
1010 }
1011 
1012 static bool
pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr * lo,const nfs4_stateid * stateid)1013 pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
1014 		const nfs4_stateid *stateid)
1015 {
1016 	u32 seqid = be32_to_cpu(stateid->seqid);
1017 
1018 	return lo->plh_barrier && pnfs_seqid_is_newer(lo->plh_barrier, seqid);
1019 }
1020 
1021 /* lget is set to 1 if called from inside send_layoutget call chain */
1022 static bool
pnfs_layoutgets_blocked(const struct pnfs_layout_hdr * lo)1023 pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo)
1024 {
1025 	return lo->plh_block_lgets ||
1026 		test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
1027 }
1028 
1029 static struct nfs_server *
pnfs_find_server(struct inode * inode,struct nfs_open_context * ctx)1030 pnfs_find_server(struct inode *inode, struct nfs_open_context *ctx)
1031 {
1032 	struct nfs_server *server;
1033 
1034 	if (inode) {
1035 		server = NFS_SERVER(inode);
1036 	} else {
1037 		struct dentry *parent_dir = dget_parent(ctx->dentry);
1038 		server = NFS_SERVER(parent_dir->d_inode);
1039 		dput(parent_dir);
1040 	}
1041 	return server;
1042 }
1043 
nfs4_free_pages(struct page ** pages,size_t size)1044 static void nfs4_free_pages(struct page **pages, size_t size)
1045 {
1046 	int i;
1047 
1048 	if (!pages)
1049 		return;
1050 
1051 	for (i = 0; i < size; i++) {
1052 		if (!pages[i])
1053 			break;
1054 		__free_page(pages[i]);
1055 	}
1056 	kfree(pages);
1057 }
1058 
nfs4_alloc_pages(size_t size,gfp_t gfp_flags)1059 static struct page **nfs4_alloc_pages(size_t size, gfp_t gfp_flags)
1060 {
1061 	struct page **pages;
1062 	int i;
1063 
1064 	pages = kmalloc_array(size, sizeof(struct page *), gfp_flags);
1065 	if (!pages) {
1066 		dprintk("%s: can't alloc array of %zu pages\n", __func__, size);
1067 		return NULL;
1068 	}
1069 
1070 	for (i = 0; i < size; i++) {
1071 		pages[i] = alloc_page(gfp_flags);
1072 		if (!pages[i]) {
1073 			dprintk("%s: failed to allocate page\n", __func__);
1074 			nfs4_free_pages(pages, i);
1075 			return NULL;
1076 		}
1077 	}
1078 
1079 	return pages;
1080 }
1081 
1082 static struct nfs4_layoutget *
pnfs_alloc_init_layoutget_args(struct inode * ino,struct nfs_open_context * ctx,const nfs4_stateid * stateid,const struct pnfs_layout_range * range,gfp_t gfp_flags)1083 pnfs_alloc_init_layoutget_args(struct inode *ino,
1084 	   struct nfs_open_context *ctx,
1085 	   const nfs4_stateid *stateid,
1086 	   const struct pnfs_layout_range *range,
1087 	   gfp_t gfp_flags)
1088 {
1089 	struct nfs_server *server = pnfs_find_server(ino, ctx);
1090 	size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response;
1091 	size_t max_pages = max_response_pages(server);
1092 	struct nfs4_layoutget *lgp;
1093 
1094 	dprintk("--> %s\n", __func__);
1095 
1096 	lgp = kzalloc(sizeof(*lgp), gfp_flags);
1097 	if (lgp == NULL)
1098 		return NULL;
1099 
1100 	if (max_reply_sz) {
1101 		size_t npages = (max_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT;
1102 		if (npages < max_pages)
1103 			max_pages = npages;
1104 	}
1105 
1106 	lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags);
1107 	if (!lgp->args.layout.pages) {
1108 		kfree(lgp);
1109 		return NULL;
1110 	}
1111 	lgp->args.layout.pglen = max_pages * PAGE_SIZE;
1112 	lgp->res.layoutp = &lgp->args.layout;
1113 
1114 	/* Don't confuse uninitialised result and success */
1115 	lgp->res.status = -NFS4ERR_DELAY;
1116 
1117 	lgp->args.minlength = PAGE_SIZE;
1118 	if (lgp->args.minlength > range->length)
1119 		lgp->args.minlength = range->length;
1120 	if (ino) {
1121 		loff_t i_size = i_size_read(ino);
1122 
1123 		if (range->iomode == IOMODE_READ) {
1124 			if (range->offset >= i_size)
1125 				lgp->args.minlength = 0;
1126 			else if (i_size - range->offset < lgp->args.minlength)
1127 				lgp->args.minlength = i_size - range->offset;
1128 		}
1129 	}
1130 	lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
1131 	pnfs_copy_range(&lgp->args.range, range);
1132 	lgp->args.type = server->pnfs_curr_ld->id;
1133 	lgp->args.inode = ino;
1134 	lgp->args.ctx = get_nfs_open_context(ctx);
1135 	nfs4_stateid_copy(&lgp->args.stateid, stateid);
1136 	lgp->gfp_flags = gfp_flags;
1137 	lgp->cred = ctx->cred;
1138 	return lgp;
1139 }
1140 
pnfs_layoutget_free(struct nfs4_layoutget * lgp)1141 void pnfs_layoutget_free(struct nfs4_layoutget *lgp)
1142 {
1143 	size_t max_pages = lgp->args.layout.pglen / PAGE_SIZE;
1144 
1145 	nfs4_free_pages(lgp->args.layout.pages, max_pages);
1146 	pnfs_put_layout_hdr(lgp->lo);
1147 	put_nfs_open_context(lgp->args.ctx);
1148 	kfree(lgp);
1149 }
1150 
pnfs_clear_layoutcommit(struct inode * inode,struct list_head * head)1151 static void pnfs_clear_layoutcommit(struct inode *inode,
1152 		struct list_head *head)
1153 {
1154 	struct nfs_inode *nfsi = NFS_I(inode);
1155 	struct pnfs_layout_segment *lseg, *tmp;
1156 
1157 	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
1158 		return;
1159 	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
1160 		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
1161 			continue;
1162 		pnfs_lseg_dec_and_remove_zero(lseg, head);
1163 	}
1164 }
1165 
pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr * lo,const nfs4_stateid * arg_stateid,const struct pnfs_layout_range * range,const nfs4_stateid * stateid)1166 void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo,
1167 		const nfs4_stateid *arg_stateid,
1168 		const struct pnfs_layout_range *range,
1169 		const nfs4_stateid *stateid)
1170 {
1171 	struct inode *inode = lo->plh_inode;
1172 	LIST_HEAD(freeme);
1173 
1174 	spin_lock(&inode->i_lock);
1175 	if (!nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid))
1176 		goto out_unlock;
1177 	if (stateid && pnfs_layout_is_valid(lo)) {
1178 		u32 seq = be32_to_cpu(arg_stateid->seqid);
1179 
1180 		pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq);
1181 		pnfs_free_returned_lsegs(lo, &freeme, range, seq);
1182 		pnfs_set_layout_stateid(lo, stateid, NULL, true);
1183 	} else
1184 		pnfs_mark_layout_stateid_invalid(lo, &freeme);
1185 out_unlock:
1186 	pnfs_clear_layoutreturn_waitbit(lo);
1187 	spin_unlock(&inode->i_lock);
1188 	pnfs_free_lseg_list(&freeme);
1189 
1190 }
1191 
1192 static bool
pnfs_prepare_layoutreturn(struct pnfs_layout_hdr * lo,nfs4_stateid * stateid,const struct cred ** cred,enum pnfs_iomode * iomode)1193 pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo,
1194 		nfs4_stateid *stateid,
1195 		const struct cred **cred,
1196 		enum pnfs_iomode *iomode)
1197 {
1198 	/* Serialise LAYOUTGET/LAYOUTRETURN */
1199 	if (atomic_read(&lo->plh_outstanding) != 0)
1200 		return false;
1201 	if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
1202 		return false;
1203 	set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
1204 	pnfs_get_layout_hdr(lo);
1205 	nfs4_stateid_copy(stateid, &lo->plh_stateid);
1206 	*cred = get_cred(lo->plh_lc_cred);
1207 	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
1208 		if (lo->plh_return_seq != 0)
1209 			stateid->seqid = cpu_to_be32(lo->plh_return_seq);
1210 		if (iomode != NULL)
1211 			*iomode = lo->plh_return_iomode;
1212 		pnfs_clear_layoutreturn_info(lo);
1213 	} else if (iomode != NULL)
1214 		*iomode = IOMODE_ANY;
1215 	pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid));
1216 	return true;
1217 }
1218 
1219 static void
pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args * args,struct pnfs_layout_hdr * lo,const nfs4_stateid * stateid,enum pnfs_iomode iomode)1220 pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args,
1221 		struct pnfs_layout_hdr *lo,
1222 		const nfs4_stateid *stateid,
1223 		enum pnfs_iomode iomode)
1224 {
1225 	struct inode *inode = lo->plh_inode;
1226 
1227 	args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id;
1228 	args->inode = inode;
1229 	args->range.iomode = iomode;
1230 	args->range.offset = 0;
1231 	args->range.length = NFS4_MAX_UINT64;
1232 	args->layout = lo;
1233 	nfs4_stateid_copy(&args->stateid, stateid);
1234 }
1235 
1236 static int
pnfs_send_layoutreturn(struct pnfs_layout_hdr * lo,const nfs4_stateid * stateid,const struct cred ** pcred,enum pnfs_iomode iomode,bool sync)1237 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo,
1238 		       const nfs4_stateid *stateid,
1239 		       const struct cred **pcred,
1240 		       enum pnfs_iomode iomode,
1241 		       bool sync)
1242 {
1243 	struct inode *ino = lo->plh_inode;
1244 	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
1245 	struct nfs4_layoutreturn *lrp;
1246 	const struct cred *cred = *pcred;
1247 	int status = 0;
1248 
1249 	*pcred = NULL;
1250 	lrp = kzalloc(sizeof(*lrp), nfs_io_gfp_mask());
1251 	if (unlikely(lrp == NULL)) {
1252 		status = -ENOMEM;
1253 		spin_lock(&ino->i_lock);
1254 		pnfs_clear_layoutreturn_waitbit(lo);
1255 		spin_unlock(&ino->i_lock);
1256 		put_cred(cred);
1257 		pnfs_put_layout_hdr(lo);
1258 		goto out;
1259 	}
1260 
1261 	pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode);
1262 	lrp->args.ld_private = &lrp->ld_private;
1263 	lrp->clp = NFS_SERVER(ino)->nfs_client;
1264 	lrp->cred = cred;
1265 	if (ld->prepare_layoutreturn)
1266 		ld->prepare_layoutreturn(&lrp->args);
1267 
1268 	status = nfs4_proc_layoutreturn(lrp, sync);
1269 out:
1270 	dprintk("<-- %s status: %d\n", __func__, status);
1271 	return status;
1272 }
1273 
1274 static bool
pnfs_layout_segments_returnable(struct pnfs_layout_hdr * lo,enum pnfs_iomode iomode,u32 seq)1275 pnfs_layout_segments_returnable(struct pnfs_layout_hdr *lo,
1276 				enum pnfs_iomode iomode,
1277 				u32 seq)
1278 {
1279 	struct pnfs_layout_range recall_range = {
1280 		.length = NFS4_MAX_UINT64,
1281 		.iomode = iomode,
1282 	};
1283 	return pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs,
1284 					       &recall_range, seq) != -EBUSY;
1285 }
1286 
1287 /* Return true if layoutreturn is needed */
1288 static bool
pnfs_layout_need_return(struct pnfs_layout_hdr * lo)1289 pnfs_layout_need_return(struct pnfs_layout_hdr *lo)
1290 {
1291 	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1292 		return false;
1293 	return pnfs_layout_segments_returnable(lo, lo->plh_return_iomode,
1294 					       lo->plh_return_seq);
1295 }
1296 
pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr * lo)1297 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo)
1298 {
1299 	struct inode *inode= lo->plh_inode;
1300 
1301 	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1302 		return;
1303 	spin_lock(&inode->i_lock);
1304 	if (pnfs_layout_need_return(lo)) {
1305 		const struct cred *cred;
1306 		nfs4_stateid stateid;
1307 		enum pnfs_iomode iomode;
1308 		bool send;
1309 
1310 		send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode);
1311 		spin_unlock(&inode->i_lock);
1312 		if (send) {
1313 			/* Send an async layoutreturn so we dont deadlock */
1314 			pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, false);
1315 		}
1316 	} else
1317 		spin_unlock(&inode->i_lock);
1318 }
1319 
1320 /*
1321  * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
1322  * when the layout segment list is empty.
1323  *
1324  * Note that a pnfs_layout_hdr can exist with an empty layout segment
1325  * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
1326  * deviceid is marked invalid.
1327  */
1328 int
_pnfs_return_layout(struct inode * ino)1329 _pnfs_return_layout(struct inode *ino)
1330 {
1331 	struct pnfs_layout_hdr *lo = NULL;
1332 	struct nfs_inode *nfsi = NFS_I(ino);
1333 	struct pnfs_layout_range range = {
1334 		.iomode		= IOMODE_ANY,
1335 		.offset		= 0,
1336 		.length		= NFS4_MAX_UINT64,
1337 	};
1338 	LIST_HEAD(tmp_list);
1339 	const struct cred *cred;
1340 	nfs4_stateid stateid;
1341 	int status = 0;
1342 	bool send, valid_layout;
1343 
1344 	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
1345 
1346 	spin_lock(&ino->i_lock);
1347 	lo = nfsi->layout;
1348 	if (!lo) {
1349 		spin_unlock(&ino->i_lock);
1350 		dprintk("NFS: %s no layout to return\n", __func__);
1351 		goto out;
1352 	}
1353 	/* Reference matched in nfs4_layoutreturn_release */
1354 	pnfs_get_layout_hdr(lo);
1355 	/* Is there an outstanding layoutreturn ? */
1356 	if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) {
1357 		spin_unlock(&ino->i_lock);
1358 		if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN,
1359 					TASK_UNINTERRUPTIBLE))
1360 			goto out_put_layout_hdr;
1361 		spin_lock(&ino->i_lock);
1362 	}
1363 	valid_layout = pnfs_layout_is_valid(lo);
1364 	pnfs_clear_layoutcommit(ino, &tmp_list);
1365 	pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0);
1366 
1367 	if (NFS_SERVER(ino)->pnfs_curr_ld->return_range)
1368 		NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
1369 
1370 	/* Don't send a LAYOUTRETURN if list was initially empty */
1371 	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) ||
1372 			!valid_layout) {
1373 		spin_unlock(&ino->i_lock);
1374 		dprintk("NFS: %s no layout segments to return\n", __func__);
1375 		goto out_wait_layoutreturn;
1376 	}
1377 
1378 	send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL);
1379 	spin_unlock(&ino->i_lock);
1380 	if (send)
1381 		status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, true);
1382 out_wait_layoutreturn:
1383 	wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, TASK_UNINTERRUPTIBLE);
1384 out_put_layout_hdr:
1385 	pnfs_free_lseg_list(&tmp_list);
1386 	pnfs_put_layout_hdr(lo);
1387 out:
1388 	dprintk("<-- %s status: %d\n", __func__, status);
1389 	return status;
1390 }
1391 
1392 int
pnfs_commit_and_return_layout(struct inode * inode)1393 pnfs_commit_and_return_layout(struct inode *inode)
1394 {
1395 	struct pnfs_layout_hdr *lo;
1396 	int ret;
1397 
1398 	spin_lock(&inode->i_lock);
1399 	lo = NFS_I(inode)->layout;
1400 	if (lo == NULL) {
1401 		spin_unlock(&inode->i_lock);
1402 		return 0;
1403 	}
1404 	pnfs_get_layout_hdr(lo);
1405 	/* Block new layoutgets and read/write to ds */
1406 	lo->plh_block_lgets++;
1407 	spin_unlock(&inode->i_lock);
1408 	filemap_fdatawait(inode->i_mapping);
1409 	ret = pnfs_layoutcommit_inode(inode, true);
1410 	if (ret == 0)
1411 		ret = _pnfs_return_layout(inode);
1412 	spin_lock(&inode->i_lock);
1413 	lo->plh_block_lgets--;
1414 	spin_unlock(&inode->i_lock);
1415 	pnfs_put_layout_hdr(lo);
1416 	return ret;
1417 }
1418 
pnfs_roc(struct inode * ino,struct nfs4_layoutreturn_args * args,struct nfs4_layoutreturn_res * res,const struct cred * cred)1419 bool pnfs_roc(struct inode *ino,
1420 		struct nfs4_layoutreturn_args *args,
1421 		struct nfs4_layoutreturn_res *res,
1422 		const struct cred *cred)
1423 {
1424 	struct nfs_inode *nfsi = NFS_I(ino);
1425 	struct nfs_open_context *ctx;
1426 	struct nfs4_state *state;
1427 	struct pnfs_layout_hdr *lo;
1428 	struct pnfs_layout_segment *lseg, *next;
1429 	const struct cred *lc_cred;
1430 	nfs4_stateid stateid;
1431 	enum pnfs_iomode iomode = 0;
1432 	bool layoutreturn = false, roc = false;
1433 	bool skip_read = false;
1434 
1435 	if (!nfs_have_layout(ino))
1436 		return false;
1437 retry:
1438 	rcu_read_lock();
1439 	spin_lock(&ino->i_lock);
1440 	lo = nfsi->layout;
1441 	if (!lo || !pnfs_layout_is_valid(lo) ||
1442 	    test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
1443 		lo = NULL;
1444 		goto out_noroc;
1445 	}
1446 	pnfs_get_layout_hdr(lo);
1447 	if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) {
1448 		spin_unlock(&ino->i_lock);
1449 		rcu_read_unlock();
1450 		wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN,
1451 				TASK_UNINTERRUPTIBLE);
1452 		pnfs_put_layout_hdr(lo);
1453 		goto retry;
1454 	}
1455 
1456 	/* no roc if we hold a delegation */
1457 	if (nfs4_check_delegation(ino, FMODE_READ)) {
1458 		if (nfs4_check_delegation(ino, FMODE_WRITE))
1459 			goto out_noroc;
1460 		skip_read = true;
1461 	}
1462 
1463 	list_for_each_entry_rcu(ctx, &nfsi->open_files, list) {
1464 		state = ctx->state;
1465 		if (state == NULL)
1466 			continue;
1467 		/* Don't return layout if there is open file state */
1468 		if (state->state & FMODE_WRITE)
1469 			goto out_noroc;
1470 		if (state->state & FMODE_READ)
1471 			skip_read = true;
1472 	}
1473 
1474 
1475 	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) {
1476 		if (skip_read && lseg->pls_range.iomode == IOMODE_READ)
1477 			continue;
1478 		/* If we are sending layoutreturn, invalidate all valid lsegs */
1479 		if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags))
1480 			continue;
1481 		/*
1482 		 * Note: mark lseg for return so pnfs_layout_remove_lseg
1483 		 * doesn't invalidate the layout for us.
1484 		 */
1485 		set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
1486 		if (!mark_lseg_invalid(lseg, &lo->plh_return_segs))
1487 			continue;
1488 		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
1489 	}
1490 
1491 	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1492 		goto out_noroc;
1493 
1494 	/* ROC in two conditions:
1495 	 * 1. there are ROC lsegs
1496 	 * 2. we don't send layoutreturn
1497 	 */
1498 	/* lo ref dropped in pnfs_roc_release() */
1499 	layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode);
1500 	/* If the creds don't match, we can't compound the layoutreturn */
1501 	if (!layoutreturn || cred_fscmp(cred, lc_cred) != 0)
1502 		goto out_noroc;
1503 
1504 	roc = layoutreturn;
1505 	pnfs_init_layoutreturn_args(args, lo, &stateid, iomode);
1506 	res->lrs_present = 0;
1507 	layoutreturn = false;
1508 	put_cred(lc_cred);
1509 
1510 out_noroc:
1511 	spin_unlock(&ino->i_lock);
1512 	rcu_read_unlock();
1513 	pnfs_layoutcommit_inode(ino, true);
1514 	if (roc) {
1515 		struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
1516 		if (ld->prepare_layoutreturn)
1517 			ld->prepare_layoutreturn(args);
1518 		pnfs_put_layout_hdr(lo);
1519 		return true;
1520 	}
1521 	if (layoutreturn)
1522 		pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, true);
1523 	pnfs_put_layout_hdr(lo);
1524 	return false;
1525 }
1526 
pnfs_roc_done(struct rpc_task * task,struct nfs4_layoutreturn_args ** argpp,struct nfs4_layoutreturn_res ** respp,int * ret)1527 int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp,
1528 		  struct nfs4_layoutreturn_res **respp, int *ret)
1529 {
1530 	struct nfs4_layoutreturn_args *arg = *argpp;
1531 	int retval = -EAGAIN;
1532 
1533 	if (!arg)
1534 		return 0;
1535 	/* Handle Layoutreturn errors */
1536 	switch (*ret) {
1537 	case 0:
1538 		retval = 0;
1539 		break;
1540 	case -NFS4ERR_NOMATCHING_LAYOUT:
1541 		/* Was there an RPC level error? If not, retry */
1542 		if (task->tk_rpc_status == 0)
1543 			break;
1544 		/* If the call was not sent, let caller handle it */
1545 		if (!RPC_WAS_SENT(task))
1546 			return 0;
1547 		/*
1548 		 * Otherwise, assume the call succeeded and
1549 		 * that we need to release the layout
1550 		 */
1551 		*ret = 0;
1552 		(*respp)->lrs_present = 0;
1553 		retval = 0;
1554 		break;
1555 	case -NFS4ERR_DELAY:
1556 		/* Let the caller handle the retry */
1557 		*ret = -NFS4ERR_NOMATCHING_LAYOUT;
1558 		return 0;
1559 	case -NFS4ERR_OLD_STATEID:
1560 		if (!nfs4_layout_refresh_old_stateid(&arg->stateid,
1561 						     &arg->range, arg->inode))
1562 			break;
1563 		*ret = -NFS4ERR_NOMATCHING_LAYOUT;
1564 		return -EAGAIN;
1565 	}
1566 	*argpp = NULL;
1567 	*respp = NULL;
1568 	return retval;
1569 }
1570 
pnfs_roc_release(struct nfs4_layoutreturn_args * args,struct nfs4_layoutreturn_res * res,int ret)1571 void pnfs_roc_release(struct nfs4_layoutreturn_args *args,
1572 		struct nfs4_layoutreturn_res *res,
1573 		int ret)
1574 {
1575 	struct pnfs_layout_hdr *lo = args->layout;
1576 	struct inode *inode = args->inode;
1577 	const nfs4_stateid *res_stateid = NULL;
1578 	struct nfs4_xdr_opaque_data *ld_private = args->ld_private;
1579 
1580 	switch (ret) {
1581 	case -NFS4ERR_NOMATCHING_LAYOUT:
1582 		spin_lock(&inode->i_lock);
1583 		if (pnfs_layout_is_valid(lo) &&
1584 		    nfs4_stateid_match_other(&args->stateid, &lo->plh_stateid))
1585 			pnfs_set_plh_return_info(lo, args->range.iomode, 0);
1586 		pnfs_clear_layoutreturn_waitbit(lo);
1587 		spin_unlock(&inode->i_lock);
1588 		break;
1589 	case 0:
1590 		if (res->lrs_present)
1591 			res_stateid = &res->stateid;
1592 		fallthrough;
1593 	default:
1594 		pnfs_layoutreturn_free_lsegs(lo, &args->stateid, &args->range,
1595 					     res_stateid);
1596 	}
1597 	trace_nfs4_layoutreturn_on_close(args->inode, &args->stateid, ret);
1598 	if (ld_private && ld_private->ops && ld_private->ops->free)
1599 		ld_private->ops->free(ld_private);
1600 	pnfs_put_layout_hdr(lo);
1601 }
1602 
pnfs_wait_on_layoutreturn(struct inode * ino,struct rpc_task * task)1603 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
1604 {
1605 	struct nfs_inode *nfsi = NFS_I(ino);
1606         struct pnfs_layout_hdr *lo;
1607         bool sleep = false;
1608 
1609 	/* we might not have grabbed lo reference. so need to check under
1610 	 * i_lock */
1611         spin_lock(&ino->i_lock);
1612         lo = nfsi->layout;
1613         if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
1614                 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
1615                 sleep = true;
1616 	}
1617         spin_unlock(&ino->i_lock);
1618         return sleep;
1619 }
1620 
1621 /*
1622  * Compare two layout segments for sorting into layout cache.
1623  * We want to preferentially return RW over RO layouts, so ensure those
1624  * are seen first.
1625  */
1626 static s64
pnfs_lseg_range_cmp(const struct pnfs_layout_range * l1,const struct pnfs_layout_range * l2)1627 pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
1628 	   const struct pnfs_layout_range *l2)
1629 {
1630 	s64 d;
1631 
1632 	/* high offset > low offset */
1633 	d = l1->offset - l2->offset;
1634 	if (d)
1635 		return d;
1636 
1637 	/* short length > long length */
1638 	d = l2->length - l1->length;
1639 	if (d)
1640 		return d;
1641 
1642 	/* read > read/write */
1643 	return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
1644 }
1645 
1646 static bool
pnfs_lseg_range_is_after(const struct pnfs_layout_range * l1,const struct pnfs_layout_range * l2)1647 pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1,
1648 		const struct pnfs_layout_range *l2)
1649 {
1650 	return pnfs_lseg_range_cmp(l1, l2) > 0;
1651 }
1652 
1653 static bool
pnfs_lseg_no_merge(struct pnfs_layout_segment * lseg,struct pnfs_layout_segment * old)1654 pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,
1655 		struct pnfs_layout_segment *old)
1656 {
1657 	return false;
1658 }
1659 
1660 void
pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr * lo,struct pnfs_layout_segment * lseg,bool (* is_after)(const struct pnfs_layout_range *,const struct pnfs_layout_range *),bool (* do_merge)(struct pnfs_layout_segment *,struct pnfs_layout_segment *),struct list_head * free_me)1661 pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
1662 		   struct pnfs_layout_segment *lseg,
1663 		   bool (*is_after)(const struct pnfs_layout_range *,
1664 			   const struct pnfs_layout_range *),
1665 		   bool (*do_merge)(struct pnfs_layout_segment *,
1666 			   struct pnfs_layout_segment *),
1667 		   struct list_head *free_me)
1668 {
1669 	struct pnfs_layout_segment *lp, *tmp;
1670 
1671 	dprintk("%s:Begin\n", __func__);
1672 
1673 	list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) {
1674 		if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0)
1675 			continue;
1676 		if (do_merge(lseg, lp)) {
1677 			mark_lseg_invalid(lp, free_me);
1678 			continue;
1679 		}
1680 		if (is_after(&lseg->pls_range, &lp->pls_range))
1681 			continue;
1682 		list_add_tail(&lseg->pls_list, &lp->pls_list);
1683 		dprintk("%s: inserted lseg %p "
1684 			"iomode %d offset %llu length %llu before "
1685 			"lp %p iomode %d offset %llu length %llu\n",
1686 			__func__, lseg, lseg->pls_range.iomode,
1687 			lseg->pls_range.offset, lseg->pls_range.length,
1688 			lp, lp->pls_range.iomode, lp->pls_range.offset,
1689 			lp->pls_range.length);
1690 		goto out;
1691 	}
1692 	list_add_tail(&lseg->pls_list, &lo->plh_segs);
1693 	dprintk("%s: inserted lseg %p "
1694 		"iomode %d offset %llu length %llu at tail\n",
1695 		__func__, lseg, lseg->pls_range.iomode,
1696 		lseg->pls_range.offset, lseg->pls_range.length);
1697 out:
1698 	pnfs_get_layout_hdr(lo);
1699 
1700 	dprintk("%s:Return\n", __func__);
1701 }
1702 EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg);
1703 
1704 static void
pnfs_layout_insert_lseg(struct pnfs_layout_hdr * lo,struct pnfs_layout_segment * lseg,struct list_head * free_me)1705 pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
1706 		   struct pnfs_layout_segment *lseg,
1707 		   struct list_head *free_me)
1708 {
1709 	struct inode *inode = lo->plh_inode;
1710 	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
1711 
1712 	if (ld->add_lseg != NULL)
1713 		ld->add_lseg(lo, lseg, free_me);
1714 	else
1715 		pnfs_generic_layout_insert_lseg(lo, lseg,
1716 				pnfs_lseg_range_is_after,
1717 				pnfs_lseg_no_merge,
1718 				free_me);
1719 }
1720 
1721 static struct pnfs_layout_hdr *
alloc_init_layout_hdr(struct inode * ino,struct nfs_open_context * ctx,gfp_t gfp_flags)1722 alloc_init_layout_hdr(struct inode *ino,
1723 		      struct nfs_open_context *ctx,
1724 		      gfp_t gfp_flags)
1725 {
1726 	struct pnfs_layout_hdr *lo;
1727 
1728 	lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
1729 	if (!lo)
1730 		return NULL;
1731 	refcount_set(&lo->plh_refcount, 1);
1732 	INIT_LIST_HEAD(&lo->plh_layouts);
1733 	INIT_LIST_HEAD(&lo->plh_segs);
1734 	INIT_LIST_HEAD(&lo->plh_return_segs);
1735 	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
1736 	lo->plh_inode = ino;
1737 	lo->plh_lc_cred = get_cred(ctx->cred);
1738 	lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID;
1739 	return lo;
1740 }
1741 
1742 static struct pnfs_layout_hdr *
pnfs_find_alloc_layout(struct inode * ino,struct nfs_open_context * ctx,gfp_t gfp_flags)1743 pnfs_find_alloc_layout(struct inode *ino,
1744 		       struct nfs_open_context *ctx,
1745 		       gfp_t gfp_flags)
1746 	__releases(&ino->i_lock)
1747 	__acquires(&ino->i_lock)
1748 {
1749 	struct nfs_inode *nfsi = NFS_I(ino);
1750 	struct pnfs_layout_hdr *new = NULL;
1751 
1752 	dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);
1753 
1754 	if (nfsi->layout != NULL)
1755 		goto out_existing;
1756 	spin_unlock(&ino->i_lock);
1757 	new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
1758 	spin_lock(&ino->i_lock);
1759 
1760 	if (likely(nfsi->layout == NULL)) {	/* Won the race? */
1761 		nfsi->layout = new;
1762 		return new;
1763 	} else if (new != NULL)
1764 		pnfs_free_layout_hdr(new);
1765 out_existing:
1766 	pnfs_get_layout_hdr(nfsi->layout);
1767 	return nfsi->layout;
1768 }
1769 
1770 /*
1771  * iomode matching rules:
1772  * iomode	lseg	strict match
1773  *                      iomode
1774  * -----	-----	------ -----
1775  * ANY		READ	N/A    true
1776  * ANY		RW	N/A    true
1777  * RW		READ	N/A    false
1778  * RW		RW	N/A    true
1779  * READ		READ	N/A    true
1780  * READ		RW	true   false
1781  * READ		RW	false  true
1782  */
1783 static bool
pnfs_lseg_range_match(const struct pnfs_layout_range * ls_range,const struct pnfs_layout_range * range,bool strict_iomode)1784 pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
1785 		 const struct pnfs_layout_range *range,
1786 		 bool strict_iomode)
1787 {
1788 	struct pnfs_layout_range range1;
1789 
1790 	if ((range->iomode == IOMODE_RW &&
1791 	     ls_range->iomode != IOMODE_RW) ||
1792 	    (range->iomode != ls_range->iomode &&
1793 	     strict_iomode) ||
1794 	    !pnfs_lseg_range_intersecting(ls_range, range))
1795 		return false;
1796 
1797 	/* range1 covers only the first byte in the range */
1798 	range1 = *range;
1799 	range1.length = 1;
1800 	return pnfs_lseg_range_contained(ls_range, &range1);
1801 }
1802 
1803 /*
1804  * lookup range in layout
1805  */
1806 static struct pnfs_layout_segment *
pnfs_find_lseg(struct pnfs_layout_hdr * lo,struct pnfs_layout_range * range,bool strict_iomode)1807 pnfs_find_lseg(struct pnfs_layout_hdr *lo,
1808 		struct pnfs_layout_range *range,
1809 		bool strict_iomode)
1810 {
1811 	struct pnfs_layout_segment *lseg, *ret = NULL;
1812 
1813 	dprintk("%s:Begin\n", __func__);
1814 
1815 	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
1816 		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
1817 		    pnfs_lseg_range_match(&lseg->pls_range, range,
1818 					  strict_iomode)) {
1819 			ret = pnfs_get_lseg(lseg);
1820 			break;
1821 		}
1822 	}
1823 
1824 	dprintk("%s:Return lseg %p ref %d\n",
1825 		__func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0);
1826 	return ret;
1827 }
1828 
1829 /*
1830  * Use mdsthreshold hints set at each OPEN to determine if I/O should go
1831  * to the MDS or over pNFS
1832  *
1833  * The nfs_inode read_io and write_io fields are cumulative counters reset
1834  * when there are no layout segments. Note that in pnfs_update_layout iomode
1835  * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
1836  * WRITE request.
1837  *
1838  * A return of true means use MDS I/O.
1839  *
1840  * From rfc 5661:
1841  * If a file's size is smaller than the file size threshold, data accesses
1842  * SHOULD be sent to the metadata server.  If an I/O request has a length that
1843  * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
1844  * server.  If both file size and I/O size are provided, the client SHOULD
1845  * reach or exceed  both thresholds before sending its read or write
1846  * requests to the data server.
1847  */
pnfs_within_mdsthreshold(struct nfs_open_context * ctx,struct inode * ino,int iomode)1848 static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
1849 				     struct inode *ino, int iomode)
1850 {
1851 	struct nfs4_threshold *t = ctx->mdsthreshold;
1852 	struct nfs_inode *nfsi = NFS_I(ino);
1853 	loff_t fsize = i_size_read(ino);
1854 	bool size = false, size_set = false, io = false, io_set = false, ret = false;
1855 
1856 	if (t == NULL)
1857 		return ret;
1858 
1859 	dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
1860 		__func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);
1861 
1862 	switch (iomode) {
1863 	case IOMODE_READ:
1864 		if (t->bm & THRESHOLD_RD) {
1865 			dprintk("%s fsize %llu\n", __func__, fsize);
1866 			size_set = true;
1867 			if (fsize < t->rd_sz)
1868 				size = true;
1869 		}
1870 		if (t->bm & THRESHOLD_RD_IO) {
1871 			dprintk("%s nfsi->read_io %llu\n", __func__,
1872 				nfsi->read_io);
1873 			io_set = true;
1874 			if (nfsi->read_io < t->rd_io_sz)
1875 				io = true;
1876 		}
1877 		break;
1878 	case IOMODE_RW:
1879 		if (t->bm & THRESHOLD_WR) {
1880 			dprintk("%s fsize %llu\n", __func__, fsize);
1881 			size_set = true;
1882 			if (fsize < t->wr_sz)
1883 				size = true;
1884 		}
1885 		if (t->bm & THRESHOLD_WR_IO) {
1886 			dprintk("%s nfsi->write_io %llu\n", __func__,
1887 				nfsi->write_io);
1888 			io_set = true;
1889 			if (nfsi->write_io < t->wr_io_sz)
1890 				io = true;
1891 		}
1892 		break;
1893 	}
1894 	if (size_set && io_set) {
1895 		if (size && io)
1896 			ret = true;
1897 	} else if (size || io)
1898 		ret = true;
1899 
1900 	dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
1901 	return ret;
1902 }
1903 
pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr * lo)1904 static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
1905 {
1906 	/*
1907 	 * send layoutcommit as it can hold up layoutreturn due to lseg
1908 	 * reference
1909 	 */
1910 	pnfs_layoutcommit_inode(lo->plh_inode, false);
1911 	return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
1912 				   nfs_wait_bit_killable,
1913 				   TASK_KILLABLE|TASK_FREEZABLE_UNSAFE);
1914 }
1915 
nfs_layoutget_begin(struct pnfs_layout_hdr * lo)1916 static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo)
1917 {
1918 	atomic_inc(&lo->plh_outstanding);
1919 }
1920 
nfs_layoutget_end(struct pnfs_layout_hdr * lo)1921 static void nfs_layoutget_end(struct pnfs_layout_hdr *lo)
1922 {
1923 	if (atomic_dec_and_test(&lo->plh_outstanding) &&
1924 	    test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags))
1925 		wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN);
1926 }
1927 
pnfs_is_first_layoutget(struct pnfs_layout_hdr * lo)1928 static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo)
1929 {
1930 	return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags);
1931 }
1932 
pnfs_clear_first_layoutget(struct pnfs_layout_hdr * lo)1933 static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
1934 {
1935 	unsigned long *bitlock = &lo->plh_flags;
1936 
1937 	clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
1938 	smp_mb__after_atomic();
1939 	wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
1940 }
1941 
_add_to_server_list(struct pnfs_layout_hdr * lo,struct nfs_server * server)1942 static void _add_to_server_list(struct pnfs_layout_hdr *lo,
1943 				struct nfs_server *server)
1944 {
1945 	if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) {
1946 		struct nfs_client *clp = server->nfs_client;
1947 
1948 		/* The lo must be on the clp list if there is any
1949 		 * chance of a CB_LAYOUTRECALL(FILE) coming in.
1950 		 */
1951 		spin_lock(&clp->cl_lock);
1952 		list_add_tail_rcu(&lo->plh_layouts, &server->layouts);
1953 		spin_unlock(&clp->cl_lock);
1954 	}
1955 }
1956 
1957 /*
1958  * Layout segment is retreived from the server if not cached.
1959  * The appropriate layout segment is referenced and returned to the caller.
1960  */
1961 struct pnfs_layout_segment *
pnfs_update_layout(struct inode * ino,struct nfs_open_context * ctx,loff_t pos,u64 count,enum pnfs_iomode iomode,bool strict_iomode,gfp_t gfp_flags)1962 pnfs_update_layout(struct inode *ino,
1963 		   struct nfs_open_context *ctx,
1964 		   loff_t pos,
1965 		   u64 count,
1966 		   enum pnfs_iomode iomode,
1967 		   bool strict_iomode,
1968 		   gfp_t gfp_flags)
1969 {
1970 	struct pnfs_layout_range arg = {
1971 		.iomode = iomode,
1972 		.offset = pos,
1973 		.length = count,
1974 	};
1975 	unsigned pg_offset;
1976 	struct nfs_server *server = NFS_SERVER(ino);
1977 	struct nfs_client *clp = server->nfs_client;
1978 	struct pnfs_layout_hdr *lo = NULL;
1979 	struct pnfs_layout_segment *lseg = NULL;
1980 	struct nfs4_layoutget *lgp;
1981 	nfs4_stateid stateid;
1982 	long timeout = 0;
1983 	unsigned long giveup = jiffies + (clp->cl_lease_time << 1);
1984 	bool first;
1985 
1986 	if (!pnfs_enabled_sb(NFS_SERVER(ino))) {
1987 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1988 				 PNFS_UPDATE_LAYOUT_NO_PNFS);
1989 		goto out;
1990 	}
1991 
1992 	if (pnfs_within_mdsthreshold(ctx, ino, iomode)) {
1993 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1994 				 PNFS_UPDATE_LAYOUT_MDSTHRESH);
1995 		goto out;
1996 	}
1997 
1998 lookup_again:
1999 	if (!nfs4_valid_open_stateid(ctx->state)) {
2000 		trace_pnfs_update_layout(ino, pos, count,
2001 					 iomode, lo, lseg,
2002 					 PNFS_UPDATE_LAYOUT_INVALID_OPEN);
2003 		lseg = ERR_PTR(-EIO);
2004 		goto out;
2005 	}
2006 
2007 	lseg = ERR_PTR(nfs4_client_recover_expired_lease(clp));
2008 	if (IS_ERR(lseg))
2009 		goto out;
2010 	first = false;
2011 	spin_lock(&ino->i_lock);
2012 	lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
2013 	if (lo == NULL) {
2014 		spin_unlock(&ino->i_lock);
2015 		lseg = ERR_PTR(-ENOMEM);
2016 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
2017 				 PNFS_UPDATE_LAYOUT_NOMEM);
2018 		goto out;
2019 	}
2020 
2021 	/* Do we even need to bother with this? */
2022 	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
2023 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
2024 				 PNFS_UPDATE_LAYOUT_BULK_RECALL);
2025 		dprintk("%s matches recall, use MDS\n", __func__);
2026 		goto out_unlock;
2027 	}
2028 
2029 	/* if LAYOUTGET already failed once we don't try again */
2030 	if (pnfs_layout_io_test_failed(lo, iomode)) {
2031 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
2032 				 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL);
2033 		goto out_unlock;
2034 	}
2035 
2036 	/*
2037 	 * If the layout segment list is empty, but there are outstanding
2038 	 * layoutget calls, then they might be subject to a layoutrecall.
2039 	 */
2040 	if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) &&
2041 	    atomic_read(&lo->plh_outstanding) != 0) {
2042 		spin_unlock(&ino->i_lock);
2043 		lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN,
2044 					   TASK_KILLABLE));
2045 		if (IS_ERR(lseg))
2046 			goto out_put_layout_hdr;
2047 		pnfs_put_layout_hdr(lo);
2048 		goto lookup_again;
2049 	}
2050 
2051 	/*
2052 	 * Because we free lsegs when sending LAYOUTRETURN, we need to wait
2053 	 * for LAYOUTRETURN.
2054 	 */
2055 	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
2056 		spin_unlock(&ino->i_lock);
2057 		dprintk("%s wait for layoutreturn\n", __func__);
2058 		lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo));
2059 		if (!IS_ERR(lseg)) {
2060 			pnfs_put_layout_hdr(lo);
2061 			dprintk("%s retrying\n", __func__);
2062 			trace_pnfs_update_layout(ino, pos, count, iomode, lo,
2063 						 lseg,
2064 						 PNFS_UPDATE_LAYOUT_RETRY);
2065 			goto lookup_again;
2066 		}
2067 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
2068 					 PNFS_UPDATE_LAYOUT_RETURN);
2069 		goto out_put_layout_hdr;
2070 	}
2071 
2072 	lseg = pnfs_find_lseg(lo, &arg, strict_iomode);
2073 	if (lseg) {
2074 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
2075 				PNFS_UPDATE_LAYOUT_FOUND_CACHED);
2076 		goto out_unlock;
2077 	}
2078 
2079 	/*
2080 	 * Choose a stateid for the LAYOUTGET. If we don't have a layout
2081 	 * stateid, or it has been invalidated, then we must use the open
2082 	 * stateid.
2083 	 */
2084 	if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
2085 		int status;
2086 
2087 		/*
2088 		 * The first layoutget for the file. Need to serialize per
2089 		 * RFC 5661 Errata 3208.
2090 		 */
2091 		if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
2092 				     &lo->plh_flags)) {
2093 			spin_unlock(&ino->i_lock);
2094 			lseg = ERR_PTR(wait_on_bit(&lo->plh_flags,
2095 						NFS_LAYOUT_FIRST_LAYOUTGET,
2096 						TASK_KILLABLE));
2097 			if (IS_ERR(lseg))
2098 				goto out_put_layout_hdr;
2099 			pnfs_put_layout_hdr(lo);
2100 			dprintk("%s retrying\n", __func__);
2101 			goto lookup_again;
2102 		}
2103 
2104 		spin_unlock(&ino->i_lock);
2105 		first = true;
2106 		status = nfs4_select_rw_stateid(ctx->state,
2107 					iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ,
2108 					NULL, &stateid, NULL);
2109 		if (status != 0) {
2110 			lseg = ERR_PTR(status);
2111 			trace_pnfs_update_layout(ino, pos, count,
2112 					iomode, lo, lseg,
2113 					PNFS_UPDATE_LAYOUT_INVALID_OPEN);
2114 			nfs4_schedule_stateid_recovery(server, ctx->state);
2115 			pnfs_clear_first_layoutget(lo);
2116 			pnfs_put_layout_hdr(lo);
2117 			goto lookup_again;
2118 		}
2119 		spin_lock(&ino->i_lock);
2120 	} else {
2121 		nfs4_stateid_copy(&stateid, &lo->plh_stateid);
2122 	}
2123 
2124 	if (pnfs_layoutgets_blocked(lo)) {
2125 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
2126 				PNFS_UPDATE_LAYOUT_BLOCKED);
2127 		goto out_unlock;
2128 	}
2129 	nfs_layoutget_begin(lo);
2130 	spin_unlock(&ino->i_lock);
2131 
2132 	_add_to_server_list(lo, server);
2133 
2134 	pg_offset = arg.offset & ~PAGE_MASK;
2135 	if (pg_offset) {
2136 		arg.offset -= pg_offset;
2137 		arg.length += pg_offset;
2138 	}
2139 	if (arg.length != NFS4_MAX_UINT64)
2140 		arg.length = PAGE_ALIGN(arg.length);
2141 
2142 	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags);
2143 	if (!lgp) {
2144 		lseg = ERR_PTR(-ENOMEM);
2145 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL,
2146 					 PNFS_UPDATE_LAYOUT_NOMEM);
2147 		nfs_layoutget_end(lo);
2148 		goto out_put_layout_hdr;
2149 	}
2150 
2151 	lgp->lo = lo;
2152 	pnfs_get_layout_hdr(lo);
2153 
2154 	lseg = nfs4_proc_layoutget(lgp, &timeout);
2155 	trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
2156 				 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
2157 	nfs_layoutget_end(lo);
2158 	if (IS_ERR(lseg)) {
2159 		switch(PTR_ERR(lseg)) {
2160 		case -EBUSY:
2161 			if (time_after(jiffies, giveup))
2162 				lseg = NULL;
2163 			break;
2164 		case -ERECALLCONFLICT:
2165 		case -EAGAIN:
2166 			break;
2167 		case -ENODATA:
2168 			/* The server returned NFS4ERR_LAYOUTUNAVAILABLE */
2169 			pnfs_layout_set_fail_bit(
2170 				lo, pnfs_iomode_to_fail_bit(iomode));
2171 			lseg = NULL;
2172 			goto out_put_layout_hdr;
2173 		default:
2174 			if (!nfs_error_is_fatal(PTR_ERR(lseg))) {
2175 				pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
2176 				lseg = NULL;
2177 			}
2178 			goto out_put_layout_hdr;
2179 		}
2180 		if (lseg) {
2181 			if (first)
2182 				pnfs_clear_first_layoutget(lo);
2183 			trace_pnfs_update_layout(ino, pos, count,
2184 				iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY);
2185 			pnfs_put_layout_hdr(lo);
2186 			goto lookup_again;
2187 		}
2188 	} else {
2189 		pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
2190 	}
2191 
2192 out_put_layout_hdr:
2193 	if (first)
2194 		pnfs_clear_first_layoutget(lo);
2195 	trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
2196 				 PNFS_UPDATE_LAYOUT_EXIT);
2197 	pnfs_put_layout_hdr(lo);
2198 out:
2199 	dprintk("%s: inode %s/%llu pNFS layout segment %s for "
2200 			"(%s, offset: %llu, length: %llu)\n",
2201 			__func__, ino->i_sb->s_id,
2202 			(unsigned long long)NFS_FILEID(ino),
2203 			IS_ERR_OR_NULL(lseg) ? "not found" : "found",
2204 			iomode==IOMODE_RW ?  "read/write" : "read-only",
2205 			(unsigned long long)pos,
2206 			(unsigned long long)count);
2207 	return lseg;
2208 out_unlock:
2209 	spin_unlock(&ino->i_lock);
2210 	goto out_put_layout_hdr;
2211 }
2212 EXPORT_SYMBOL_GPL(pnfs_update_layout);
2213 
2214 static bool
pnfs_sanity_check_layout_range(struct pnfs_layout_range * range)2215 pnfs_sanity_check_layout_range(struct pnfs_layout_range *range)
2216 {
2217 	switch (range->iomode) {
2218 	case IOMODE_READ:
2219 	case IOMODE_RW:
2220 		break;
2221 	default:
2222 		return false;
2223 	}
2224 	if (range->offset == NFS4_MAX_UINT64)
2225 		return false;
2226 	if (range->length == 0)
2227 		return false;
2228 	if (range->length != NFS4_MAX_UINT64 &&
2229 	    range->length > NFS4_MAX_UINT64 - range->offset)
2230 		return false;
2231 	return true;
2232 }
2233 
2234 static struct pnfs_layout_hdr *
_pnfs_grab_empty_layout(struct inode * ino,struct nfs_open_context * ctx)2235 _pnfs_grab_empty_layout(struct inode *ino, struct nfs_open_context *ctx)
2236 {
2237 	struct pnfs_layout_hdr *lo;
2238 
2239 	spin_lock(&ino->i_lock);
2240 	lo = pnfs_find_alloc_layout(ino, ctx, nfs_io_gfp_mask());
2241 	if (!lo)
2242 		goto out_unlock;
2243 	if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags))
2244 		goto out_unlock;
2245 	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
2246 		goto out_unlock;
2247 	if (pnfs_layoutgets_blocked(lo))
2248 		goto out_unlock;
2249 	if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags))
2250 		goto out_unlock;
2251 	nfs_layoutget_begin(lo);
2252 	spin_unlock(&ino->i_lock);
2253 	_add_to_server_list(lo, NFS_SERVER(ino));
2254 	return lo;
2255 
2256 out_unlock:
2257 	spin_unlock(&ino->i_lock);
2258 	pnfs_put_layout_hdr(lo);
2259 	return NULL;
2260 }
2261 
_lgopen_prepare_attached(struct nfs4_opendata * data,struct nfs_open_context * ctx)2262 static void _lgopen_prepare_attached(struct nfs4_opendata *data,
2263 				     struct nfs_open_context *ctx)
2264 {
2265 	struct inode *ino = data->dentry->d_inode;
2266 	struct pnfs_layout_range rng = {
2267 		.iomode = (data->o_arg.fmode & FMODE_WRITE) ?
2268 			  IOMODE_RW: IOMODE_READ,
2269 		.offset = 0,
2270 		.length = NFS4_MAX_UINT64,
2271 	};
2272 	struct nfs4_layoutget *lgp;
2273 	struct pnfs_layout_hdr *lo;
2274 
2275 	/* Heuristic: don't send layoutget if we have cached data */
2276 	if (rng.iomode == IOMODE_READ &&
2277 	   (i_size_read(ino) == 0 || ino->i_mapping->nrpages != 0))
2278 		return;
2279 
2280 	lo = _pnfs_grab_empty_layout(ino, ctx);
2281 	if (!lo)
2282 		return;
2283 	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &current_stateid, &rng,
2284 					     nfs_io_gfp_mask());
2285 	if (!lgp) {
2286 		pnfs_clear_first_layoutget(lo);
2287 		nfs_layoutget_end(lo);
2288 		pnfs_put_layout_hdr(lo);
2289 		return;
2290 	}
2291 	lgp->lo = lo;
2292 	data->lgp = lgp;
2293 	data->o_arg.lg_args = &lgp->args;
2294 	data->o_res.lg_res = &lgp->res;
2295 }
2296 
_lgopen_prepare_floating(struct nfs4_opendata * data,struct nfs_open_context * ctx)2297 static void _lgopen_prepare_floating(struct nfs4_opendata *data,
2298 				     struct nfs_open_context *ctx)
2299 {
2300 	struct inode *ino = data->dentry->d_inode;
2301 	struct pnfs_layout_range rng = {
2302 		.iomode = (data->o_arg.fmode & FMODE_WRITE) ?
2303 			  IOMODE_RW: IOMODE_READ,
2304 		.offset = 0,
2305 		.length = NFS4_MAX_UINT64,
2306 	};
2307 	struct nfs4_layoutget *lgp;
2308 
2309 	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &current_stateid, &rng,
2310 					     nfs_io_gfp_mask());
2311 	if (!lgp)
2312 		return;
2313 	data->lgp = lgp;
2314 	data->o_arg.lg_args = &lgp->args;
2315 	data->o_res.lg_res = &lgp->res;
2316 }
2317 
pnfs_lgopen_prepare(struct nfs4_opendata * data,struct nfs_open_context * ctx)2318 void pnfs_lgopen_prepare(struct nfs4_opendata *data,
2319 			 struct nfs_open_context *ctx)
2320 {
2321 	struct nfs_server *server = NFS_SERVER(data->dir->d_inode);
2322 
2323 	if (!(pnfs_enabled_sb(server) &&
2324 	      server->pnfs_curr_ld->flags & PNFS_LAYOUTGET_ON_OPEN))
2325 		return;
2326 	/* Could check on max_ops, but currently hardcoded high enough */
2327 	if (!nfs_server_capable(data->dir->d_inode, NFS_CAP_LGOPEN))
2328 		return;
2329 	if (data->lgp)
2330 		return;
2331 	if (data->state)
2332 		_lgopen_prepare_attached(data, ctx);
2333 	else
2334 		_lgopen_prepare_floating(data, ctx);
2335 }
2336 
pnfs_parse_lgopen(struct inode * ino,struct nfs4_layoutget * lgp,struct nfs_open_context * ctx)2337 void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp,
2338 		       struct nfs_open_context *ctx)
2339 {
2340 	struct pnfs_layout_hdr *lo;
2341 	struct pnfs_layout_segment *lseg;
2342 	struct nfs_server *srv = NFS_SERVER(ino);
2343 	u32 iomode;
2344 
2345 	if (!lgp)
2346 		return;
2347 	dprintk("%s: entered with status %i\n", __func__, lgp->res.status);
2348 	if (lgp->res.status) {
2349 		switch (lgp->res.status) {
2350 		default:
2351 			break;
2352 		/*
2353 		 * Halt lgopen attempts if the server doesn't recognise
2354 		 * the "current stateid" value, the layout type, or the
2355 		 * layoutget operation as being valid.
2356 		 * Also if it complains about too many ops in the compound
2357 		 * or of the request/reply being too big.
2358 		 */
2359 		case -NFS4ERR_BAD_STATEID:
2360 		case -NFS4ERR_NOTSUPP:
2361 		case -NFS4ERR_REP_TOO_BIG:
2362 		case -NFS4ERR_REP_TOO_BIG_TO_CACHE:
2363 		case -NFS4ERR_REQ_TOO_BIG:
2364 		case -NFS4ERR_TOO_MANY_OPS:
2365 		case -NFS4ERR_UNKNOWN_LAYOUTTYPE:
2366 			srv->caps &= ~NFS_CAP_LGOPEN;
2367 		}
2368 		return;
2369 	}
2370 	if (!lgp->lo) {
2371 		lo = _pnfs_grab_empty_layout(ino, ctx);
2372 		if (!lo)
2373 			return;
2374 		lgp->lo = lo;
2375 	} else
2376 		lo = lgp->lo;
2377 
2378 	lseg = pnfs_layout_process(lgp);
2379 	if (!IS_ERR(lseg)) {
2380 		iomode = lgp->args.range.iomode;
2381 		pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
2382 		pnfs_put_lseg(lseg);
2383 	}
2384 }
2385 
nfs4_lgopen_release(struct nfs4_layoutget * lgp)2386 void nfs4_lgopen_release(struct nfs4_layoutget *lgp)
2387 {
2388 	if (lgp != NULL) {
2389 		if (lgp->lo) {
2390 			pnfs_clear_first_layoutget(lgp->lo);
2391 			nfs_layoutget_end(lgp->lo);
2392 		}
2393 		pnfs_layoutget_free(lgp);
2394 	}
2395 }
2396 
2397 struct pnfs_layout_segment *
pnfs_layout_process(struct nfs4_layoutget * lgp)2398 pnfs_layout_process(struct nfs4_layoutget *lgp)
2399 {
2400 	struct pnfs_layout_hdr *lo = lgp->lo;
2401 	struct nfs4_layoutget_res *res = &lgp->res;
2402 	struct pnfs_layout_segment *lseg;
2403 	struct inode *ino = lo->plh_inode;
2404 	LIST_HEAD(free_me);
2405 
2406 	if (!pnfs_sanity_check_layout_range(&res->range))
2407 		return ERR_PTR(-EINVAL);
2408 
2409 	/* Inject layout blob into I/O device driver */
2410 	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
2411 	if (IS_ERR_OR_NULL(lseg)) {
2412 		if (!lseg)
2413 			lseg = ERR_PTR(-ENOMEM);
2414 
2415 		dprintk("%s: Could not allocate layout: error %ld\n",
2416 		       __func__, PTR_ERR(lseg));
2417 		return lseg;
2418 	}
2419 
2420 	pnfs_init_lseg(lo, lseg, &res->range, &res->stateid);
2421 
2422 	spin_lock(&ino->i_lock);
2423 	if (pnfs_layoutgets_blocked(lo)) {
2424 		dprintk("%s forget reply due to state\n", __func__);
2425 		goto out_forget;
2426 	}
2427 
2428 	if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) &&
2429 	    !pnfs_is_first_layoutget(lo))
2430 		goto out_forget;
2431 
2432 	if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
2433 		/* existing state ID, make sure the sequence number matches. */
2434 		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
2435 			if (!pnfs_layout_is_valid(lo))
2436 				lo->plh_barrier = 0;
2437 			dprintk("%s forget reply due to sequence\n", __func__);
2438 			goto out_forget;
2439 		}
2440 		pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false);
2441 	} else if (pnfs_layout_is_valid(lo)) {
2442 		/*
2443 		 * We got an entirely new state ID.  Mark all segments for the
2444 		 * inode invalid, and retry the layoutget
2445 		 */
2446 		struct pnfs_layout_range range = {
2447 			.iomode = IOMODE_ANY,
2448 			.length = NFS4_MAX_UINT64,
2449 		};
2450 		pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0);
2451 		goto out_forget;
2452 	} else {
2453 		/* We have a completely new layout */
2454 		pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true);
2455 	}
2456 
2457 	pnfs_get_lseg(lseg);
2458 	pnfs_layout_insert_lseg(lo, lseg, &free_me);
2459 
2460 
2461 	if (res->return_on_close)
2462 		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);
2463 
2464 	spin_unlock(&ino->i_lock);
2465 	pnfs_free_lseg_list(&free_me);
2466 	return lseg;
2467 
2468 out_forget:
2469 	spin_unlock(&ino->i_lock);
2470 	lseg->pls_layout = lo;
2471 	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
2472 	return ERR_PTR(-EAGAIN);
2473 }
2474 
2475 /**
2476  * pnfs_mark_matching_lsegs_return - Free or return matching layout segments
2477  * @lo: pointer to layout header
2478  * @tmp_list: list header to be used with pnfs_free_lseg_list()
2479  * @return_range: describe layout segment ranges to be returned
2480  * @seq: stateid seqid to match
2481  *
2482  * This function is mainly intended for use by layoutrecall. It attempts
2483  * to free the layout segment immediately, or else to mark it for return
2484  * as soon as its reference count drops to zero.
2485  *
2486  * Returns
2487  * - 0: a layoutreturn needs to be scheduled.
2488  * - EBUSY: there are layout segment that are still in use.
2489  * - ENOENT: there are no layout segments that need to be returned.
2490  */
2491 int
pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr * lo,struct list_head * tmp_list,const struct pnfs_layout_range * return_range,u32 seq)2492 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
2493 				struct list_head *tmp_list,
2494 				const struct pnfs_layout_range *return_range,
2495 				u32 seq)
2496 {
2497 	struct pnfs_layout_segment *lseg, *next;
2498 	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
2499 	int remaining = 0;
2500 
2501 	dprintk("%s:Begin lo %p\n", __func__, lo);
2502 
2503 	assert_spin_locked(&lo->plh_inode->i_lock);
2504 
2505 	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
2506 		tmp_list = &lo->plh_return_segs;
2507 
2508 	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
2509 		if (pnfs_match_lseg_recall(lseg, return_range, seq)) {
2510 			dprintk("%s: marking lseg %p iomode %d "
2511 				"offset %llu length %llu\n", __func__,
2512 				lseg, lseg->pls_range.iomode,
2513 				lseg->pls_range.offset,
2514 				lseg->pls_range.length);
2515 			if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
2516 				tmp_list = &lo->plh_return_segs;
2517 			if (mark_lseg_invalid(lseg, tmp_list))
2518 				continue;
2519 			remaining++;
2520 			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
2521 			pnfs_lseg_cancel_io(server, lseg);
2522 		}
2523 
2524 	if (remaining) {
2525 		pnfs_set_plh_return_info(lo, return_range->iomode, seq);
2526 		return -EBUSY;
2527 	}
2528 
2529 	if (!list_empty(&lo->plh_return_segs)) {
2530 		pnfs_set_plh_return_info(lo, return_range->iomode, seq);
2531 		return 0;
2532 	}
2533 
2534 	return -ENOENT;
2535 }
2536 
2537 static void
pnfs_mark_layout_for_return(struct inode * inode,const struct pnfs_layout_range * range)2538 pnfs_mark_layout_for_return(struct inode *inode,
2539 			    const struct pnfs_layout_range *range)
2540 {
2541 	struct pnfs_layout_hdr *lo;
2542 	bool return_now = false;
2543 
2544 	spin_lock(&inode->i_lock);
2545 	lo = NFS_I(inode)->layout;
2546 	if (!pnfs_layout_is_valid(lo)) {
2547 		spin_unlock(&inode->i_lock);
2548 		return;
2549 	}
2550 	pnfs_set_plh_return_info(lo, range->iomode, 0);
2551 	/*
2552 	 * mark all matching lsegs so that we are sure to have no live
2553 	 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
2554 	 * for how it works.
2555 	 */
2556 	if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) {
2557 		const struct cred *cred;
2558 		nfs4_stateid stateid;
2559 		enum pnfs_iomode iomode;
2560 
2561 		return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode);
2562 		spin_unlock(&inode->i_lock);
2563 		if (return_now)
2564 			pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, false);
2565 	} else {
2566 		spin_unlock(&inode->i_lock);
2567 		nfs_commit_inode(inode, 0);
2568 	}
2569 }
2570 
pnfs_error_mark_layout_for_return(struct inode * inode,struct pnfs_layout_segment * lseg)2571 void pnfs_error_mark_layout_for_return(struct inode *inode,
2572 				       struct pnfs_layout_segment *lseg)
2573 {
2574 	struct pnfs_layout_range range = {
2575 		.iomode = lseg->pls_range.iomode,
2576 		.offset = 0,
2577 		.length = NFS4_MAX_UINT64,
2578 	};
2579 
2580 	pnfs_mark_layout_for_return(inode, &range);
2581 }
2582 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);
2583 
2584 static bool
pnfs_layout_can_be_returned(struct pnfs_layout_hdr * lo)2585 pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo)
2586 {
2587 	return pnfs_layout_is_valid(lo) &&
2588 		!test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) &&
2589 		!test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
2590 }
2591 
2592 static struct pnfs_layout_segment *
pnfs_find_first_lseg(struct pnfs_layout_hdr * lo,const struct pnfs_layout_range * range,enum pnfs_iomode iomode)2593 pnfs_find_first_lseg(struct pnfs_layout_hdr *lo,
2594 		     const struct pnfs_layout_range *range,
2595 		     enum pnfs_iomode iomode)
2596 {
2597 	struct pnfs_layout_segment *lseg;
2598 
2599 	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
2600 		if (!test_bit(NFS_LSEG_VALID, &lseg->pls_flags))
2601 			continue;
2602 		if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
2603 			continue;
2604 		if (lseg->pls_range.iomode != iomode && iomode != IOMODE_ANY)
2605 			continue;
2606 		if (pnfs_lseg_range_intersecting(&lseg->pls_range, range))
2607 			return lseg;
2608 	}
2609 	return NULL;
2610 }
2611 
2612 /* Find open file states whose mode matches that of the range */
2613 static bool
pnfs_should_return_unused_layout(struct pnfs_layout_hdr * lo,const struct pnfs_layout_range * range)2614 pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo,
2615 				 const struct pnfs_layout_range *range)
2616 {
2617 	struct list_head *head;
2618 	struct nfs_open_context *ctx;
2619 	fmode_t mode = 0;
2620 
2621 	if (!pnfs_layout_can_be_returned(lo) ||
2622 	    !pnfs_find_first_lseg(lo, range, range->iomode))
2623 		return false;
2624 
2625 	head = &NFS_I(lo->plh_inode)->open_files;
2626 	list_for_each_entry_rcu(ctx, head, list) {
2627 		if (ctx->state)
2628 			mode |= ctx->state->state & (FMODE_READ|FMODE_WRITE);
2629 	}
2630 
2631 	switch (range->iomode) {
2632 	default:
2633 		break;
2634 	case IOMODE_READ:
2635 		mode &= ~FMODE_WRITE;
2636 		break;
2637 	case IOMODE_RW:
2638 		if (pnfs_find_first_lseg(lo, range, IOMODE_READ))
2639 			mode &= ~FMODE_READ;
2640 	}
2641 	return mode == 0;
2642 }
2643 
pnfs_layout_return_unused_byserver(struct nfs_server * server,void * data)2644 static int pnfs_layout_return_unused_byserver(struct nfs_server *server,
2645 					      void *data)
2646 {
2647 	const struct pnfs_layout_range *range = data;
2648 	const struct cred *cred;
2649 	struct pnfs_layout_hdr *lo;
2650 	struct inode *inode;
2651 	nfs4_stateid stateid;
2652 	enum pnfs_iomode iomode;
2653 
2654 restart:
2655 	rcu_read_lock();
2656 	list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) {
2657 		inode = lo->plh_inode;
2658 		if (!inode || !pnfs_layout_can_be_returned(lo) ||
2659 		    test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
2660 			continue;
2661 		spin_lock(&inode->i_lock);
2662 		if (!lo->plh_inode ||
2663 		    !pnfs_should_return_unused_layout(lo, range)) {
2664 			spin_unlock(&inode->i_lock);
2665 			continue;
2666 		}
2667 		pnfs_get_layout_hdr(lo);
2668 		pnfs_set_plh_return_info(lo, range->iomode, 0);
2669 		if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs,
2670 						    range, 0) != 0 ||
2671 		    !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) {
2672 			spin_unlock(&inode->i_lock);
2673 			rcu_read_unlock();
2674 			pnfs_put_layout_hdr(lo);
2675 			cond_resched();
2676 			goto restart;
2677 		}
2678 		spin_unlock(&inode->i_lock);
2679 		rcu_read_unlock();
2680 		pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, false);
2681 		pnfs_put_layout_hdr(lo);
2682 		cond_resched();
2683 		goto restart;
2684 	}
2685 	rcu_read_unlock();
2686 	return 0;
2687 }
2688 
2689 void
pnfs_layout_return_unused_byclid(struct nfs_client * clp,enum pnfs_iomode iomode)2690 pnfs_layout_return_unused_byclid(struct nfs_client *clp,
2691 				 enum pnfs_iomode iomode)
2692 {
2693 	struct pnfs_layout_range range = {
2694 		.iomode = iomode,
2695 		.offset = 0,
2696 		.length = NFS4_MAX_UINT64,
2697 	};
2698 
2699 	nfs_client_for_each_server(clp, pnfs_layout_return_unused_byserver,
2700 			&range);
2701 }
2702 
2703 void
pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor * pgio)2704 pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio)
2705 {
2706 	if (pgio->pg_lseg == NULL ||
2707 	    test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags))
2708 		return;
2709 	pnfs_put_lseg(pgio->pg_lseg);
2710 	pgio->pg_lseg = NULL;
2711 }
2712 EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout);
2713 
2714 /*
2715  * Check for any intersection between the request and the pgio->pg_lseg,
2716  * and if none, put this pgio->pg_lseg away.
2717  */
2718 void
pnfs_generic_pg_check_range(struct nfs_pageio_descriptor * pgio,struct nfs_page * req)2719 pnfs_generic_pg_check_range(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
2720 {
2721 	if (pgio->pg_lseg && !pnfs_lseg_request_intersecting(pgio->pg_lseg, req)) {
2722 		pnfs_put_lseg(pgio->pg_lseg);
2723 		pgio->pg_lseg = NULL;
2724 	}
2725 }
2726 EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_range);
2727 
2728 void
pnfs_generic_pg_init_read(struct nfs_pageio_descriptor * pgio,struct nfs_page * req)2729 pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
2730 {
2731 	u64 rd_size;
2732 
2733 	pnfs_generic_pg_check_layout(pgio);
2734 	pnfs_generic_pg_check_range(pgio, req);
2735 	if (pgio->pg_lseg == NULL) {
2736 		if (pgio->pg_dreq == NULL)
2737 			rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
2738 		else
2739 			rd_size = nfs_dreq_bytes_left(pgio->pg_dreq,
2740 						      req_offset(req));
2741 
2742 		pgio->pg_lseg =
2743 			pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req),
2744 					   req_offset(req), rd_size,
2745 					   IOMODE_READ, false,
2746 					   nfs_io_gfp_mask());
2747 		if (IS_ERR(pgio->pg_lseg)) {
2748 			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
2749 			pgio->pg_lseg = NULL;
2750 			return;
2751 		}
2752 	}
2753 	/* If no lseg, fall back to read through mds */
2754 	if (pgio->pg_lseg == NULL)
2755 		nfs_pageio_reset_read_mds(pgio);
2756 
2757 }
2758 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);
2759 
2760 void
pnfs_generic_pg_init_write(struct nfs_pageio_descriptor * pgio,struct nfs_page * req,u64 wb_size)2761 pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
2762 			   struct nfs_page *req, u64 wb_size)
2763 {
2764 	pnfs_generic_pg_check_layout(pgio);
2765 	pnfs_generic_pg_check_range(pgio, req);
2766 	if (pgio->pg_lseg == NULL) {
2767 		pgio->pg_lseg =
2768 			pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req),
2769 					   req_offset(req), wb_size, IOMODE_RW,
2770 					   false, nfs_io_gfp_mask());
2771 		if (IS_ERR(pgio->pg_lseg)) {
2772 			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
2773 			pgio->pg_lseg = NULL;
2774 			return;
2775 		}
2776 	}
2777 	/* If no lseg, fall back to write through mds */
2778 	if (pgio->pg_lseg == NULL)
2779 		nfs_pageio_reset_write_mds(pgio);
2780 }
2781 EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);
2782 
2783 void
pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor * desc)2784 pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
2785 {
2786 	if (desc->pg_lseg) {
2787 		pnfs_put_lseg(desc->pg_lseg);
2788 		desc->pg_lseg = NULL;
2789 	}
2790 }
2791 EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);
2792 
2793 /*
2794  * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
2795  * of bytes (maximum @req->wb_bytes) that can be coalesced.
2796  */
2797 size_t
pnfs_generic_pg_test(struct nfs_pageio_descriptor * pgio,struct nfs_page * prev,struct nfs_page * req)2798 pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
2799 		     struct nfs_page *prev, struct nfs_page *req)
2800 {
2801 	unsigned int size;
2802 	u64 seg_end, req_start, seg_left;
2803 
2804 	size = nfs_generic_pg_test(pgio, prev, req);
2805 	if (!size)
2806 		return 0;
2807 
2808 	/*
2809 	 * 'size' contains the number of bytes left in the current page (up
2810 	 * to the original size asked for in @req->wb_bytes).
2811 	 *
2812 	 * Calculate how many bytes are left in the layout segment
2813 	 * and if there are less bytes than 'size', return that instead.
2814 	 *
2815 	 * Please also note that 'end_offset' is actually the offset of the
2816 	 * first byte that lies outside the pnfs_layout_range. FIXME?
2817 	 *
2818 	 */
2819 	if (pgio->pg_lseg) {
2820 		seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset,
2821 				     pgio->pg_lseg->pls_range.length);
2822 		req_start = req_offset(req);
2823 
2824 		/* start of request is past the last byte of this segment */
2825 		if (req_start >= seg_end)
2826 			return 0;
2827 
2828 		/* adjust 'size' iff there are fewer bytes left in the
2829 		 * segment than what nfs_generic_pg_test returned */
2830 		seg_left = seg_end - req_start;
2831 		if (seg_left < size)
2832 			size = (unsigned int)seg_left;
2833 	}
2834 
2835 	return size;
2836 }
2837 EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
2838 
pnfs_write_done_resend_to_mds(struct nfs_pgio_header * hdr)2839 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
2840 {
2841 	struct nfs_pageio_descriptor pgio;
2842 
2843 	/* Resend all requests through the MDS */
2844 	nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
2845 			      hdr->completion_ops);
2846 	return nfs_pageio_resend(&pgio, hdr);
2847 }
2848 EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
2849 
pnfs_ld_handle_write_error(struct nfs_pgio_header * hdr)2850 static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
2851 {
2852 
2853 	dprintk("pnfs write error = %d\n", hdr->pnfs_error);
2854 	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2855 	    PNFS_LAYOUTRET_ON_ERROR) {
2856 		pnfs_return_layout(hdr->inode);
2857 	}
2858 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2859 		hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
2860 }
2861 
2862 /*
2863  * Called by non rpc-based layout drivers
2864  */
pnfs_ld_write_done(struct nfs_pgio_header * hdr)2865 void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
2866 {
2867 	if (likely(!hdr->pnfs_error)) {
2868 		pnfs_set_layoutcommit(hdr->inode, hdr->lseg,
2869 				hdr->mds_offset + hdr->res.count);
2870 		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2871 	}
2872 	trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
2873 	if (unlikely(hdr->pnfs_error))
2874 		pnfs_ld_handle_write_error(hdr);
2875 	hdr->mds_ops->rpc_release(hdr);
2876 }
2877 EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
2878 
2879 static void
pnfs_write_through_mds(struct nfs_pageio_descriptor * desc,struct nfs_pgio_header * hdr)2880 pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
2881 		struct nfs_pgio_header *hdr)
2882 {
2883 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2884 
2885 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2886 		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2887 		nfs_pageio_reset_write_mds(desc);
2888 		mirror->pg_recoalesce = 1;
2889 	}
2890 	hdr->completion_ops->completion(hdr);
2891 }
2892 
2893 static enum pnfs_try_status
pnfs_try_to_write_data(struct nfs_pgio_header * hdr,const struct rpc_call_ops * call_ops,struct pnfs_layout_segment * lseg,int how)2894 pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
2895 			const struct rpc_call_ops *call_ops,
2896 			struct pnfs_layout_segment *lseg,
2897 			int how)
2898 {
2899 	struct inode *inode = hdr->inode;
2900 	enum pnfs_try_status trypnfs;
2901 	struct nfs_server *nfss = NFS_SERVER(inode);
2902 
2903 	hdr->mds_ops = call_ops;
2904 
2905 	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
2906 		inode->i_ino, hdr->args.count, hdr->args.offset, how);
2907 	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
2908 	if (trypnfs != PNFS_NOT_ATTEMPTED)
2909 		nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
2910 	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
2911 	return trypnfs;
2912 }
2913 
2914 static void
pnfs_do_write(struct nfs_pageio_descriptor * desc,struct nfs_pgio_header * hdr,int how)2915 pnfs_do_write(struct nfs_pageio_descriptor *desc,
2916 	      struct nfs_pgio_header *hdr, int how)
2917 {
2918 	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
2919 	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2920 	enum pnfs_try_status trypnfs;
2921 
2922 	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
2923 	switch (trypnfs) {
2924 	case PNFS_NOT_ATTEMPTED:
2925 		pnfs_write_through_mds(desc, hdr);
2926 		break;
2927 	case PNFS_ATTEMPTED:
2928 		break;
2929 	case PNFS_TRY_AGAIN:
2930 		/* cleanup hdr and prepare to redo pnfs */
2931 		if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2932 			struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2933 			list_splice_init(&hdr->pages, &mirror->pg_list);
2934 			mirror->pg_recoalesce = 1;
2935 		}
2936 		hdr->mds_ops->rpc_release(hdr);
2937 	}
2938 }
2939 
pnfs_writehdr_free(struct nfs_pgio_header * hdr)2940 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
2941 {
2942 	pnfs_put_lseg(hdr->lseg);
2943 	nfs_pgio_header_free(hdr);
2944 }
2945 
2946 int
pnfs_generic_pg_writepages(struct nfs_pageio_descriptor * desc)2947 pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
2948 {
2949 	struct nfs_pgio_header *hdr;
2950 	int ret;
2951 
2952 	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
2953 	if (!hdr) {
2954 		desc->pg_error = -ENOMEM;
2955 		return desc->pg_error;
2956 	}
2957 	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
2958 
2959 	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2960 	ret = nfs_generic_pgio(desc, hdr);
2961 	if (!ret)
2962 		pnfs_do_write(desc, hdr, desc->pg_ioflags);
2963 
2964 	return ret;
2965 }
2966 EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
2967 
pnfs_read_done_resend_to_mds(struct nfs_pgio_header * hdr)2968 int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
2969 {
2970 	struct nfs_pageio_descriptor pgio;
2971 
2972 	/* Resend all requests through the MDS */
2973 	nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
2974 	return nfs_pageio_resend(&pgio, hdr);
2975 }
2976 EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
2977 
pnfs_ld_handle_read_error(struct nfs_pgio_header * hdr)2978 static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
2979 {
2980 	dprintk("pnfs read error = %d\n", hdr->pnfs_error);
2981 	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2982 	    PNFS_LAYOUTRET_ON_ERROR) {
2983 		pnfs_return_layout(hdr->inode);
2984 	}
2985 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2986 		hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
2987 }
2988 
2989 /*
2990  * Called by non rpc-based layout drivers
2991  */
pnfs_ld_read_done(struct nfs_pgio_header * hdr)2992 void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
2993 {
2994 	if (likely(!hdr->pnfs_error))
2995 		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2996 	trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
2997 	if (unlikely(hdr->pnfs_error))
2998 		pnfs_ld_handle_read_error(hdr);
2999 	hdr->mds_ops->rpc_release(hdr);
3000 }
3001 EXPORT_SYMBOL_GPL(pnfs_ld_read_done);
3002 
3003 static void
pnfs_read_through_mds(struct nfs_pageio_descriptor * desc,struct nfs_pgio_header * hdr)3004 pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
3005 		struct nfs_pgio_header *hdr)
3006 {
3007 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
3008 
3009 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
3010 		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
3011 		nfs_pageio_reset_read_mds(desc);
3012 		mirror->pg_recoalesce = 1;
3013 	}
3014 	hdr->completion_ops->completion(hdr);
3015 }
3016 
3017 /*
3018  * Call the appropriate parallel I/O subsystem read function.
3019  */
3020 static enum pnfs_try_status
pnfs_try_to_read_data(struct nfs_pgio_header * hdr,const struct rpc_call_ops * call_ops,struct pnfs_layout_segment * lseg)3021 pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
3022 		       const struct rpc_call_ops *call_ops,
3023 		       struct pnfs_layout_segment *lseg)
3024 {
3025 	struct inode *inode = hdr->inode;
3026 	struct nfs_server *nfss = NFS_SERVER(inode);
3027 	enum pnfs_try_status trypnfs;
3028 
3029 	hdr->mds_ops = call_ops;
3030 
3031 	dprintk("%s: Reading ino:%lu %u@%llu\n",
3032 		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
3033 
3034 	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
3035 	if (trypnfs != PNFS_NOT_ATTEMPTED)
3036 		nfs_inc_stats(inode, NFSIOS_PNFS_READ);
3037 	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
3038 	return trypnfs;
3039 }
3040 
3041 /* Resend all requests through pnfs. */
pnfs_read_resend_pnfs(struct nfs_pgio_header * hdr,unsigned int mirror_idx)3042 void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr,
3043 			   unsigned int mirror_idx)
3044 {
3045 	struct nfs_pageio_descriptor pgio;
3046 
3047 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
3048 		/* Prevent deadlocks with layoutreturn! */
3049 		pnfs_put_lseg(hdr->lseg);
3050 		hdr->lseg = NULL;
3051 
3052 		nfs_pageio_init_read(&pgio, hdr->inode, false,
3053 					hdr->completion_ops);
3054 		pgio.pg_mirror_idx = mirror_idx;
3055 		hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr);
3056 	}
3057 }
3058 EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);
3059 
3060 static void
pnfs_do_read(struct nfs_pageio_descriptor * desc,struct nfs_pgio_header * hdr)3061 pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
3062 {
3063 	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
3064 	struct pnfs_layout_segment *lseg = desc->pg_lseg;
3065 	enum pnfs_try_status trypnfs;
3066 
3067 	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
3068 	switch (trypnfs) {
3069 	case PNFS_NOT_ATTEMPTED:
3070 		pnfs_read_through_mds(desc, hdr);
3071 		break;
3072 	case PNFS_ATTEMPTED:
3073 		break;
3074 	case PNFS_TRY_AGAIN:
3075 		/* cleanup hdr and prepare to redo pnfs */
3076 		if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
3077 			struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
3078 			list_splice_init(&hdr->pages, &mirror->pg_list);
3079 			mirror->pg_recoalesce = 1;
3080 		}
3081 		hdr->mds_ops->rpc_release(hdr);
3082 	}
3083 }
3084 
pnfs_readhdr_free(struct nfs_pgio_header * hdr)3085 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
3086 {
3087 	pnfs_put_lseg(hdr->lseg);
3088 	nfs_pgio_header_free(hdr);
3089 }
3090 
3091 int
pnfs_generic_pg_readpages(struct nfs_pageio_descriptor * desc)3092 pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
3093 {
3094 	struct nfs_pgio_header *hdr;
3095 	int ret;
3096 
3097 	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
3098 	if (!hdr) {
3099 		desc->pg_error = -ENOMEM;
3100 		return desc->pg_error;
3101 	}
3102 	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
3103 	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
3104 	ret = nfs_generic_pgio(desc, hdr);
3105 	if (!ret)
3106 		pnfs_do_read(desc, hdr);
3107 	return ret;
3108 }
3109 EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);
3110 
pnfs_clear_layoutcommitting(struct inode * inode)3111 static void pnfs_clear_layoutcommitting(struct inode *inode)
3112 {
3113 	unsigned long *bitlock = &NFS_I(inode)->flags;
3114 
3115 	clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
3116 	smp_mb__after_atomic();
3117 	wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
3118 }
3119 
3120 /*
3121  * There can be multiple RW segments.
3122  */
pnfs_list_write_lseg(struct inode * inode,struct list_head * listp)3123 static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
3124 {
3125 	struct pnfs_layout_segment *lseg;
3126 
3127 	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
3128 		if (lseg->pls_range.iomode == IOMODE_RW &&
3129 		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
3130 			list_add(&lseg->pls_lc_list, listp);
3131 	}
3132 }
3133 
pnfs_list_write_lseg_done(struct inode * inode,struct list_head * listp)3134 static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
3135 {
3136 	struct pnfs_layout_segment *lseg, *tmp;
3137 
3138 	/* Matched by references in pnfs_set_layoutcommit */
3139 	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
3140 		list_del_init(&lseg->pls_lc_list);
3141 		pnfs_put_lseg(lseg);
3142 	}
3143 
3144 	pnfs_clear_layoutcommitting(inode);
3145 }
3146 
pnfs_set_lo_fail(struct pnfs_layout_segment * lseg)3147 void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
3148 {
3149 	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
3150 }
3151 EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);
3152 
3153 void
pnfs_set_layoutcommit(struct inode * inode,struct pnfs_layout_segment * lseg,loff_t end_pos)3154 pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
3155 		loff_t end_pos)
3156 {
3157 	struct nfs_inode *nfsi = NFS_I(inode);
3158 	bool mark_as_dirty = false;
3159 
3160 	spin_lock(&inode->i_lock);
3161 	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
3162 		nfsi->layout->plh_lwb = end_pos;
3163 		mark_as_dirty = true;
3164 		dprintk("%s: Set layoutcommit for inode %lu ",
3165 			__func__, inode->i_ino);
3166 	} else if (end_pos > nfsi->layout->plh_lwb)
3167 		nfsi->layout->plh_lwb = end_pos;
3168 	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {
3169 		/* references matched in nfs4_layoutcommit_release */
3170 		pnfs_get_lseg(lseg);
3171 	}
3172 	spin_unlock(&inode->i_lock);
3173 	dprintk("%s: lseg %p end_pos %llu\n",
3174 		__func__, lseg, nfsi->layout->plh_lwb);
3175 
3176 	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
3177 	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
3178 	if (mark_as_dirty)
3179 		mark_inode_dirty_sync(inode);
3180 }
3181 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
3182 
pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data * data)3183 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
3184 {
3185 	struct nfs_server *nfss = NFS_SERVER(data->args.inode);
3186 
3187 	if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
3188 		nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
3189 	pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
3190 }
3191 
3192 /*
3193  * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
3194  * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
3195  * data to disk to allow the server to recover the data if it crashes.
3196  * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
3197  * is off, and a COMMIT is sent to a data server, or
3198  * if WRITEs to a data server return NFS_DATA_SYNC.
3199  */
3200 int
pnfs_layoutcommit_inode(struct inode * inode,bool sync)3201 pnfs_layoutcommit_inode(struct inode *inode, bool sync)
3202 {
3203 	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
3204 	struct nfs4_layoutcommit_data *data;
3205 	struct nfs_inode *nfsi = NFS_I(inode);
3206 	loff_t end_pos;
3207 	int status;
3208 
3209 	if (!pnfs_layoutcommit_outstanding(inode))
3210 		return 0;
3211 
3212 	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
3213 
3214 	status = -EAGAIN;
3215 	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
3216 		if (!sync)
3217 			goto out;
3218 		status = wait_on_bit_lock_action(&nfsi->flags,
3219 				NFS_INO_LAYOUTCOMMITTING,
3220 				nfs_wait_bit_killable,
3221 				TASK_KILLABLE|TASK_FREEZABLE_UNSAFE);
3222 		if (status)
3223 			goto out;
3224 	}
3225 
3226 	status = -ENOMEM;
3227 	/* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
3228 	data = kzalloc(sizeof(*data), nfs_io_gfp_mask());
3229 	if (!data)
3230 		goto clear_layoutcommitting;
3231 
3232 	status = 0;
3233 	spin_lock(&inode->i_lock);
3234 	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
3235 		goto out_unlock;
3236 
3237 	INIT_LIST_HEAD(&data->lseg_list);
3238 	pnfs_list_write_lseg(inode, &data->lseg_list);
3239 
3240 	end_pos = nfsi->layout->plh_lwb;
3241 
3242 	nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
3243 	data->cred = get_cred(nfsi->layout->plh_lc_cred);
3244 	spin_unlock(&inode->i_lock);
3245 
3246 	data->args.inode = inode;
3247 	nfs_fattr_init(&data->fattr);
3248 	data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
3249 	data->res.fattr = &data->fattr;
3250 	if (end_pos != 0)
3251 		data->args.lastbytewritten = end_pos - 1;
3252 	else
3253 		data->args.lastbytewritten = U64_MAX;
3254 	data->res.server = NFS_SERVER(inode);
3255 
3256 	if (ld->prepare_layoutcommit) {
3257 		status = ld->prepare_layoutcommit(&data->args);
3258 		if (status) {
3259 			put_cred(data->cred);
3260 			spin_lock(&inode->i_lock);
3261 			set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
3262 			if (end_pos > nfsi->layout->plh_lwb)
3263 				nfsi->layout->plh_lwb = end_pos;
3264 			goto out_unlock;
3265 		}
3266 	}
3267 
3268 
3269 	status = nfs4_proc_layoutcommit(data, sync);
3270 out:
3271 	if (status)
3272 		mark_inode_dirty_sync(inode);
3273 	dprintk("<-- %s status %d\n", __func__, status);
3274 	return status;
3275 out_unlock:
3276 	spin_unlock(&inode->i_lock);
3277 	kfree(data);
3278 clear_layoutcommitting:
3279 	pnfs_clear_layoutcommitting(inode);
3280 	goto out;
3281 }
3282 EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
3283 
3284 int
pnfs_generic_sync(struct inode * inode,bool datasync)3285 pnfs_generic_sync(struct inode *inode, bool datasync)
3286 {
3287 	return pnfs_layoutcommit_inode(inode, true);
3288 }
3289 EXPORT_SYMBOL_GPL(pnfs_generic_sync);
3290 
pnfs_mdsthreshold_alloc(void)3291 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
3292 {
3293 	struct nfs4_threshold *thp;
3294 
3295 	thp = kzalloc(sizeof(*thp), nfs_io_gfp_mask());
3296 	if (!thp) {
3297 		dprintk("%s mdsthreshold allocation failed\n", __func__);
3298 		return NULL;
3299 	}
3300 	return thp;
3301 }
3302 
3303 #if IS_ENABLED(CONFIG_NFS_V4_2)
3304 int
pnfs_report_layoutstat(struct inode * inode,gfp_t gfp_flags)3305 pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags)
3306 {
3307 	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
3308 	struct nfs_server *server = NFS_SERVER(inode);
3309 	struct nfs_inode *nfsi = NFS_I(inode);
3310 	struct nfs42_layoutstat_data *data;
3311 	struct pnfs_layout_hdr *hdr;
3312 	int status = 0;
3313 
3314 	if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats)
3315 		goto out;
3316 
3317 	if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS))
3318 		goto out;
3319 
3320 	if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags))
3321 		goto out;
3322 
3323 	spin_lock(&inode->i_lock);
3324 	if (!NFS_I(inode)->layout) {
3325 		spin_unlock(&inode->i_lock);
3326 		goto out_clear_layoutstats;
3327 	}
3328 	hdr = NFS_I(inode)->layout;
3329 	pnfs_get_layout_hdr(hdr);
3330 	spin_unlock(&inode->i_lock);
3331 
3332 	data = kzalloc(sizeof(*data), gfp_flags);
3333 	if (!data) {
3334 		status = -ENOMEM;
3335 		goto out_put;
3336 	}
3337 
3338 	data->args.fh = NFS_FH(inode);
3339 	data->args.inode = inode;
3340 	status = ld->prepare_layoutstats(&data->args);
3341 	if (status)
3342 		goto out_free;
3343 
3344 	status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data);
3345 
3346 out:
3347 	dprintk("%s returns %d\n", __func__, status);
3348 	return status;
3349 
3350 out_free:
3351 	kfree(data);
3352 out_put:
3353 	pnfs_put_layout_hdr(hdr);
3354 out_clear_layoutstats:
3355 	smp_mb__before_atomic();
3356 	clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags);
3357 	smp_mb__after_atomic();
3358 	goto out;
3359 }
3360 EXPORT_SYMBOL_GPL(pnfs_report_layoutstat);
3361 #endif
3362 
3363 unsigned int layoutstats_timer;
3364 module_param(layoutstats_timer, uint, 0644);
3365 EXPORT_SYMBOL_GPL(layoutstats_timer);
3366