xref: /openbmc/linux/fs/nfs/direct.c (revision bc5e98e0)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * linux/fs/nfs/direct.c
4  *
5  * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
6  *
7  * High-performance uncached I/O for the Linux NFS client
8  *
9  * There are important applications whose performance or correctness
10  * depends on uncached access to file data.  Database clusters
11  * (multiple copies of the same instance running on separate hosts)
12  * implement their own cache coherency protocol that subsumes file
13  * system cache protocols.  Applications that process datasets
14  * considerably larger than the client's memory do not always benefit
15  * from a local cache.  A streaming video server, for instance, has no
16  * need to cache the contents of a file.
17  *
18  * When an application requests uncached I/O, all read and write requests
19  * are made directly to the server; data stored or fetched via these
20  * requests is not cached in the Linux page cache.  The client does not
21  * correct unaligned requests from applications.  All requested bytes are
22  * held on permanent storage before a direct write system call returns to
23  * an application.
24  *
25  * Solaris implements an uncached I/O facility called directio() that
26  * is used for backups and sequential I/O to very large files.  Solaris
27  * also supports uncaching whole NFS partitions with "-o forcedirectio,"
28  * an undocumented mount option.
29  *
30  * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
31  * help from Andrew Morton.
32  *
33  * 18 Dec 2001	Initial implementation for 2.4  --cel
34  * 08 Jul 2002	Version for 2.4.19, with bug fixes --trondmy
35  * 08 Jun 2003	Port to 2.5 APIs  --cel
36  * 31 Mar 2004	Handle direct I/O without VFS support  --cel
37  * 15 Sep 2004	Parallel async reads  --cel
38  * 04 May 2005	support O_DIRECT with aio  --cel
39  *
40  */
41 
42 #include <linux/errno.h>
43 #include <linux/sched.h>
44 #include <linux/kernel.h>
45 #include <linux/file.h>
46 #include <linux/pagemap.h>
47 #include <linux/kref.h>
48 #include <linux/slab.h>
49 #include <linux/task_io_accounting_ops.h>
50 #include <linux/module.h>
51 
52 #include <linux/nfs_fs.h>
53 #include <linux/nfs_page.h>
54 #include <linux/sunrpc/clnt.h>
55 
56 #include <linux/uaccess.h>
57 #include <linux/atomic.h>
58 
59 #include "internal.h"
60 #include "iostat.h"
61 #include "pnfs.h"
62 #include "fscache.h"
63 #include "nfstrace.h"
64 
65 #define NFSDBG_FACILITY		NFSDBG_VFS
66 
67 static struct kmem_cache *nfs_direct_cachep;
68 
69 static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
70 static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
71 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
72 static void nfs_direct_write_schedule_work(struct work_struct *work);
73 
get_dreq(struct nfs_direct_req * dreq)74 static inline void get_dreq(struct nfs_direct_req *dreq)
75 {
76 	atomic_inc(&dreq->io_count);
77 }
78 
put_dreq(struct nfs_direct_req * dreq)79 static inline int put_dreq(struct nfs_direct_req *dreq)
80 {
81 	return atomic_dec_and_test(&dreq->io_count);
82 }
83 
84 static void
nfs_direct_handle_truncated(struct nfs_direct_req * dreq,const struct nfs_pgio_header * hdr,ssize_t dreq_len)85 nfs_direct_handle_truncated(struct nfs_direct_req *dreq,
86 			    const struct nfs_pgio_header *hdr,
87 			    ssize_t dreq_len)
88 {
89 	if (!(test_bit(NFS_IOHDR_ERROR, &hdr->flags) ||
90 	      test_bit(NFS_IOHDR_EOF, &hdr->flags)))
91 		return;
92 	if (dreq->max_count >= dreq_len) {
93 		dreq->max_count = dreq_len;
94 		if (dreq->count > dreq_len)
95 			dreq->count = dreq_len;
96 	}
97 
98 	if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && !dreq->error)
99 		dreq->error = hdr->error;
100 }
101 
102 static void
nfs_direct_count_bytes(struct nfs_direct_req * dreq,const struct nfs_pgio_header * hdr)103 nfs_direct_count_bytes(struct nfs_direct_req *dreq,
104 		       const struct nfs_pgio_header *hdr)
105 {
106 	loff_t hdr_end = hdr->io_start + hdr->good_bytes;
107 	ssize_t dreq_len = 0;
108 
109 	if (hdr_end > dreq->io_start)
110 		dreq_len = hdr_end - dreq->io_start;
111 
112 	nfs_direct_handle_truncated(dreq, hdr, dreq_len);
113 
114 	if (dreq_len > dreq->max_count)
115 		dreq_len = dreq->max_count;
116 
117 	if (dreq->count < dreq_len)
118 		dreq->count = dreq_len;
119 }
120 
nfs_direct_truncate_request(struct nfs_direct_req * dreq,struct nfs_page * req)121 static void nfs_direct_truncate_request(struct nfs_direct_req *dreq,
122 					struct nfs_page *req)
123 {
124 	loff_t offs = req_offset(req);
125 	size_t req_start = (size_t)(offs - dreq->io_start);
126 
127 	if (req_start < dreq->max_count)
128 		dreq->max_count = req_start;
129 	if (req_start < dreq->count)
130 		dreq->count = req_start;
131 }
132 
133 /**
134  * nfs_swap_rw - NFS address space operation for swap I/O
135  * @iocb: target I/O control block
136  * @iter: I/O buffer
137  *
138  * Perform IO to the swap-file.  This is much like direct IO.
139  */
nfs_swap_rw(struct kiocb * iocb,struct iov_iter * iter)140 int nfs_swap_rw(struct kiocb *iocb, struct iov_iter *iter)
141 {
142 	ssize_t ret;
143 
144 	if (iov_iter_rw(iter) == READ)
145 		ret = nfs_file_direct_read(iocb, iter, true);
146 	else
147 		ret = nfs_file_direct_write(iocb, iter, true);
148 	if (ret < 0)
149 		return ret;
150 	return 0;
151 }
152 
nfs_direct_release_pages(struct page ** pages,unsigned int npages)153 static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
154 {
155 	unsigned int i;
156 	for (i = 0; i < npages; i++)
157 		put_page(pages[i]);
158 }
159 
nfs_init_cinfo_from_dreq(struct nfs_commit_info * cinfo,struct nfs_direct_req * dreq)160 void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
161 			      struct nfs_direct_req *dreq)
162 {
163 	cinfo->inode = dreq->inode;
164 	cinfo->mds = &dreq->mds_cinfo;
165 	cinfo->ds = &dreq->ds_cinfo;
166 	cinfo->dreq = dreq;
167 	cinfo->completion_ops = &nfs_direct_commit_completion_ops;
168 }
169 
nfs_direct_req_alloc(void)170 static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
171 {
172 	struct nfs_direct_req *dreq;
173 
174 	dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
175 	if (!dreq)
176 		return NULL;
177 
178 	kref_init(&dreq->kref);
179 	kref_get(&dreq->kref);
180 	init_completion(&dreq->completion);
181 	INIT_LIST_HEAD(&dreq->mds_cinfo.list);
182 	pnfs_init_ds_commit_info(&dreq->ds_cinfo);
183 	INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
184 	spin_lock_init(&dreq->lock);
185 
186 	return dreq;
187 }
188 
nfs_direct_req_free(struct kref * kref)189 static void nfs_direct_req_free(struct kref *kref)
190 {
191 	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
192 
193 	pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode);
194 	if (dreq->l_ctx != NULL)
195 		nfs_put_lock_context(dreq->l_ctx);
196 	if (dreq->ctx != NULL)
197 		put_nfs_open_context(dreq->ctx);
198 	kmem_cache_free(nfs_direct_cachep, dreq);
199 }
200 
nfs_direct_req_release(struct nfs_direct_req * dreq)201 static void nfs_direct_req_release(struct nfs_direct_req *dreq)
202 {
203 	kref_put(&dreq->kref, nfs_direct_req_free);
204 }
205 
nfs_dreq_bytes_left(struct nfs_direct_req * dreq,loff_t offset)206 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq, loff_t offset)
207 {
208 	loff_t start = offset - dreq->io_start;
209 	return dreq->max_count - start;
210 }
211 EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);
212 
213 /*
214  * Collects and returns the final error value/byte-count.
215  */
nfs_direct_wait(struct nfs_direct_req * dreq)216 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
217 {
218 	ssize_t result = -EIOCBQUEUED;
219 
220 	/* Async requests don't wait here */
221 	if (dreq->iocb)
222 		goto out;
223 
224 	result = wait_for_completion_killable(&dreq->completion);
225 
226 	if (!result) {
227 		result = dreq->count;
228 		WARN_ON_ONCE(dreq->count < 0);
229 	}
230 	if (!result)
231 		result = dreq->error;
232 
233 out:
234 	return (ssize_t) result;
235 }
236 
237 /*
238  * Synchronous I/O uses a stack-allocated iocb.  Thus we can't trust
239  * the iocb is still valid here if this is a synchronous request.
240  */
nfs_direct_complete(struct nfs_direct_req * dreq)241 static void nfs_direct_complete(struct nfs_direct_req *dreq)
242 {
243 	struct inode *inode = dreq->inode;
244 
245 	inode_dio_end(inode);
246 
247 	if (dreq->iocb) {
248 		long res = (long) dreq->error;
249 		if (dreq->count != 0) {
250 			res = (long) dreq->count;
251 			WARN_ON_ONCE(dreq->count < 0);
252 		}
253 		dreq->iocb->ki_complete(dreq->iocb, res);
254 	}
255 
256 	complete(&dreq->completion);
257 
258 	nfs_direct_req_release(dreq);
259 }
260 
nfs_direct_read_completion(struct nfs_pgio_header * hdr)261 static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
262 {
263 	unsigned long bytes = 0;
264 	struct nfs_direct_req *dreq = hdr->dreq;
265 
266 	spin_lock(&dreq->lock);
267 	if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) {
268 		spin_unlock(&dreq->lock);
269 		goto out_put;
270 	}
271 
272 	nfs_direct_count_bytes(dreq, hdr);
273 	spin_unlock(&dreq->lock);
274 
275 	while (!list_empty(&hdr->pages)) {
276 		struct nfs_page *req = nfs_list_entry(hdr->pages.next);
277 		struct page *page = req->wb_page;
278 
279 		if (!PageCompound(page) && bytes < hdr->good_bytes &&
280 		    (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY))
281 			set_page_dirty(page);
282 		bytes += req->wb_bytes;
283 		nfs_list_remove_request(req);
284 		nfs_release_request(req);
285 	}
286 out_put:
287 	if (put_dreq(dreq))
288 		nfs_direct_complete(dreq);
289 	hdr->release(hdr);
290 }
291 
nfs_read_sync_pgio_error(struct list_head * head,int error)292 static void nfs_read_sync_pgio_error(struct list_head *head, int error)
293 {
294 	struct nfs_page *req;
295 
296 	while (!list_empty(head)) {
297 		req = nfs_list_entry(head->next);
298 		nfs_list_remove_request(req);
299 		nfs_release_request(req);
300 	}
301 }
302 
nfs_direct_pgio_init(struct nfs_pgio_header * hdr)303 static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
304 {
305 	get_dreq(hdr->dreq);
306 }
307 
308 static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
309 	.error_cleanup = nfs_read_sync_pgio_error,
310 	.init_hdr = nfs_direct_pgio_init,
311 	.completion = nfs_direct_read_completion,
312 };
313 
314 /*
315  * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
316  * operation.  If nfs_readdata_alloc() or get_user_pages() fails,
317  * bail and stop sending more reads.  Read length accounting is
318  * handled automatically by nfs_direct_read_result().  Otherwise, if
319  * no requests have been sent, just return an error.
320  */
321 
nfs_direct_read_schedule_iovec(struct nfs_direct_req * dreq,struct iov_iter * iter,loff_t pos)322 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
323 					      struct iov_iter *iter,
324 					      loff_t pos)
325 {
326 	struct nfs_pageio_descriptor desc;
327 	struct inode *inode = dreq->inode;
328 	ssize_t result = -EINVAL;
329 	size_t requested_bytes = 0;
330 	size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE);
331 
332 	nfs_pageio_init_read(&desc, dreq->inode, false,
333 			     &nfs_direct_read_completion_ops);
334 	get_dreq(dreq);
335 	desc.pg_dreq = dreq;
336 	inode_dio_begin(inode);
337 
338 	while (iov_iter_count(iter)) {
339 		struct page **pagevec;
340 		size_t bytes;
341 		size_t pgbase;
342 		unsigned npages, i;
343 
344 		result = iov_iter_get_pages_alloc2(iter, &pagevec,
345 						  rsize, &pgbase);
346 		if (result < 0)
347 			break;
348 
349 		bytes = result;
350 		npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
351 		for (i = 0; i < npages; i++) {
352 			struct nfs_page *req;
353 			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
354 			/* XXX do we need to do the eof zeroing found in async_filler? */
355 			req = nfs_page_create_from_page(dreq->ctx, pagevec[i],
356 							pgbase, pos, req_len);
357 			if (IS_ERR(req)) {
358 				result = PTR_ERR(req);
359 				break;
360 			}
361 			if (!nfs_pageio_add_request(&desc, req)) {
362 				result = desc.pg_error;
363 				nfs_release_request(req);
364 				break;
365 			}
366 			pgbase = 0;
367 			bytes -= req_len;
368 			requested_bytes += req_len;
369 			pos += req_len;
370 			dreq->bytes_left -= req_len;
371 		}
372 		nfs_direct_release_pages(pagevec, npages);
373 		kvfree(pagevec);
374 		if (result < 0)
375 			break;
376 	}
377 
378 	nfs_pageio_complete(&desc);
379 
380 	/*
381 	 * If no bytes were started, return the error, and let the
382 	 * generic layer handle the completion.
383 	 */
384 	if (requested_bytes == 0) {
385 		inode_dio_end(inode);
386 		nfs_direct_req_release(dreq);
387 		return result < 0 ? result : -EIO;
388 	}
389 
390 	if (put_dreq(dreq))
391 		nfs_direct_complete(dreq);
392 	return requested_bytes;
393 }
394 
395 /**
396  * nfs_file_direct_read - file direct read operation for NFS files
397  * @iocb: target I/O control block
398  * @iter: vector of user buffers into which to read data
399  * @swap: flag indicating this is swap IO, not O_DIRECT IO
400  *
401  * We use this function for direct reads instead of calling
402  * generic_file_aio_read() in order to avoid gfar's check to see if
403  * the request starts before the end of the file.  For that check
404  * to work, we must generate a GETATTR before each direct read, and
405  * even then there is a window between the GETATTR and the subsequent
406  * READ where the file size could change.  Our preference is simply
407  * to do all reads the application wants, and the server will take
408  * care of managing the end of file boundary.
409  *
410  * This function also eliminates unnecessarily updating the file's
411  * atime locally, as the NFS server sets the file's atime, and this
412  * client must read the updated atime from the server back into its
413  * cache.
414  */
nfs_file_direct_read(struct kiocb * iocb,struct iov_iter * iter,bool swap)415 ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter,
416 			     bool swap)
417 {
418 	struct file *file = iocb->ki_filp;
419 	struct address_space *mapping = file->f_mapping;
420 	struct inode *inode = mapping->host;
421 	struct nfs_direct_req *dreq;
422 	struct nfs_lock_context *l_ctx;
423 	ssize_t result, requested;
424 	size_t count = iov_iter_count(iter);
425 	nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
426 
427 	dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n",
428 		file, count, (long long) iocb->ki_pos);
429 
430 	result = 0;
431 	if (!count)
432 		goto out;
433 
434 	task_io_account_read(count);
435 
436 	result = -ENOMEM;
437 	dreq = nfs_direct_req_alloc();
438 	if (dreq == NULL)
439 		goto out;
440 
441 	dreq->inode = inode;
442 	dreq->bytes_left = dreq->max_count = count;
443 	dreq->io_start = iocb->ki_pos;
444 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
445 	l_ctx = nfs_get_lock_context(dreq->ctx);
446 	if (IS_ERR(l_ctx)) {
447 		result = PTR_ERR(l_ctx);
448 		nfs_direct_req_release(dreq);
449 		goto out_release;
450 	}
451 	dreq->l_ctx = l_ctx;
452 	if (!is_sync_kiocb(iocb))
453 		dreq->iocb = iocb;
454 
455 	if (user_backed_iter(iter))
456 		dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;
457 
458 	if (!swap)
459 		nfs_start_io_direct(inode);
460 
461 	NFS_I(inode)->read_io += count;
462 	requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);
463 
464 	if (!swap)
465 		nfs_end_io_direct(inode);
466 
467 	if (requested > 0) {
468 		result = nfs_direct_wait(dreq);
469 		if (result > 0) {
470 			requested -= result;
471 			iocb->ki_pos += result;
472 		}
473 		iov_iter_revert(iter, requested);
474 	} else {
475 		result = requested;
476 	}
477 
478 out_release:
479 	nfs_direct_req_release(dreq);
480 out:
481 	return result;
482 }
483 
nfs_direct_add_page_head(struct list_head * list,struct nfs_page * req)484 static void nfs_direct_add_page_head(struct list_head *list,
485 				     struct nfs_page *req)
486 {
487 	struct nfs_page *head = req->wb_head;
488 
489 	if (!list_empty(&head->wb_list) || !nfs_lock_request(head))
490 		return;
491 	if (!list_empty(&head->wb_list)) {
492 		nfs_unlock_request(head);
493 		return;
494 	}
495 	list_add(&head->wb_list, list);
496 	kref_get(&head->wb_kref);
497 	kref_get(&head->wb_kref);
498 }
499 
nfs_direct_join_group(struct list_head * list,struct nfs_commit_info * cinfo,struct inode * inode)500 static void nfs_direct_join_group(struct list_head *list,
501 				  struct nfs_commit_info *cinfo,
502 				  struct inode *inode)
503 {
504 	struct nfs_page *req, *subreq;
505 
506 	list_for_each_entry(req, list, wb_list) {
507 		if (req->wb_head != req) {
508 			nfs_direct_add_page_head(&req->wb_list, req);
509 			continue;
510 		}
511 		subreq = req->wb_this_page;
512 		if (subreq == req)
513 			continue;
514 		do {
515 			/*
516 			 * Remove subrequests from this list before freeing
517 			 * them in the call to nfs_join_page_group().
518 			 */
519 			if (!list_empty(&subreq->wb_list)) {
520 				nfs_list_remove_request(subreq);
521 				nfs_release_request(subreq);
522 			}
523 		} while ((subreq = subreq->wb_this_page) != req);
524 		nfs_join_page_group(req, cinfo, inode);
525 	}
526 }
527 
528 static void
nfs_direct_write_scan_commit_list(struct inode * inode,struct list_head * list,struct nfs_commit_info * cinfo)529 nfs_direct_write_scan_commit_list(struct inode *inode,
530 				  struct list_head *list,
531 				  struct nfs_commit_info *cinfo)
532 {
533 	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
534 	pnfs_recover_commit_reqs(list, cinfo);
535 	nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
536 	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
537 }
538 
nfs_direct_write_reschedule(struct nfs_direct_req * dreq)539 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
540 {
541 	struct nfs_pageio_descriptor desc;
542 	struct nfs_page *req;
543 	LIST_HEAD(reqs);
544 	struct nfs_commit_info cinfo;
545 
546 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
547 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
548 
549 	nfs_direct_join_group(&reqs, &cinfo, dreq->inode);
550 
551 	nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);
552 	get_dreq(dreq);
553 
554 	nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
555 			      &nfs_direct_write_completion_ops);
556 	desc.pg_dreq = dreq;
557 
558 	while (!list_empty(&reqs)) {
559 		req = nfs_list_entry(reqs.next);
560 		/* Bump the transmission count */
561 		req->wb_nio++;
562 		if (!nfs_pageio_add_request(&desc, req)) {
563 			spin_lock(&dreq->lock);
564 			if (dreq->error < 0) {
565 				desc.pg_error = dreq->error;
566 			} else if (desc.pg_error != -EAGAIN) {
567 				dreq->flags = 0;
568 				if (!desc.pg_error)
569 					desc.pg_error = -EIO;
570 				dreq->error = desc.pg_error;
571 			} else
572 				dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
573 			spin_unlock(&dreq->lock);
574 			break;
575 		}
576 		nfs_release_request(req);
577 	}
578 	nfs_pageio_complete(&desc);
579 
580 	while (!list_empty(&reqs)) {
581 		req = nfs_list_entry(reqs.next);
582 		nfs_list_remove_request(req);
583 		nfs_unlock_and_release_request(req);
584 		if (desc.pg_error == -EAGAIN) {
585 			nfs_mark_request_commit(req, NULL, &cinfo, 0);
586 		} else {
587 			spin_lock(&dreq->lock);
588 			nfs_direct_truncate_request(dreq, req);
589 			spin_unlock(&dreq->lock);
590 			nfs_release_request(req);
591 		}
592 	}
593 
594 	if (put_dreq(dreq))
595 		nfs_direct_write_complete(dreq);
596 }
597 
nfs_direct_commit_complete(struct nfs_commit_data * data)598 static void nfs_direct_commit_complete(struct nfs_commit_data *data)
599 {
600 	const struct nfs_writeverf *verf = data->res.verf;
601 	struct nfs_direct_req *dreq = data->dreq;
602 	struct nfs_commit_info cinfo;
603 	struct nfs_page *req;
604 	int status = data->task.tk_status;
605 
606 	trace_nfs_direct_commit_complete(dreq);
607 
608 	if (status < 0) {
609 		/* Errors in commit are fatal */
610 		dreq->error = status;
611 		dreq->flags = NFS_ODIRECT_DONE;
612 	} else {
613 		status = dreq->error;
614 	}
615 
616 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
617 
618 	while (!list_empty(&data->pages)) {
619 		req = nfs_list_entry(data->pages.next);
620 		nfs_list_remove_request(req);
621 		if (status < 0) {
622 			spin_lock(&dreq->lock);
623 			nfs_direct_truncate_request(dreq, req);
624 			spin_unlock(&dreq->lock);
625 			nfs_release_request(req);
626 		} else if (!nfs_write_match_verf(verf, req)) {
627 			dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
628 			/*
629 			 * Despite the reboot, the write was successful,
630 			 * so reset wb_nio.
631 			 */
632 			req->wb_nio = 0;
633 			nfs_mark_request_commit(req, NULL, &cinfo, 0);
634 		} else
635 			nfs_release_request(req);
636 		nfs_unlock_and_release_request(req);
637 	}
638 
639 	if (nfs_commit_end(cinfo.mds))
640 		nfs_direct_write_complete(dreq);
641 }
642 
nfs_direct_resched_write(struct nfs_commit_info * cinfo,struct nfs_page * req)643 static void nfs_direct_resched_write(struct nfs_commit_info *cinfo,
644 		struct nfs_page *req)
645 {
646 	struct nfs_direct_req *dreq = cinfo->dreq;
647 
648 	trace_nfs_direct_resched_write(dreq);
649 
650 	spin_lock(&dreq->lock);
651 	if (dreq->flags != NFS_ODIRECT_DONE)
652 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
653 	spin_unlock(&dreq->lock);
654 	nfs_mark_request_commit(req, NULL, cinfo, 0);
655 }
656 
657 static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
658 	.completion = nfs_direct_commit_complete,
659 	.resched_write = nfs_direct_resched_write,
660 };
661 
nfs_direct_commit_schedule(struct nfs_direct_req * dreq)662 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
663 {
664 	int res;
665 	struct nfs_commit_info cinfo;
666 	LIST_HEAD(mds_list);
667 
668 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
669 	nfs_commit_begin(cinfo.mds);
670 	nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
671 	res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
672 	if (res < 0) { /* res == -ENOMEM */
673 		spin_lock(&dreq->lock);
674 		if (dreq->flags == 0)
675 			dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
676 		spin_unlock(&dreq->lock);
677 	}
678 	if (nfs_commit_end(cinfo.mds))
679 		nfs_direct_write_complete(dreq);
680 }
681 
nfs_direct_write_clear_reqs(struct nfs_direct_req * dreq)682 static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq)
683 {
684 	struct nfs_commit_info cinfo;
685 	struct nfs_page *req;
686 	LIST_HEAD(reqs);
687 
688 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
689 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
690 
691 	while (!list_empty(&reqs)) {
692 		req = nfs_list_entry(reqs.next);
693 		nfs_list_remove_request(req);
694 		nfs_direct_truncate_request(dreq, req);
695 		nfs_release_request(req);
696 		nfs_unlock_and_release_request(req);
697 	}
698 }
699 
nfs_direct_write_schedule_work(struct work_struct * work)700 static void nfs_direct_write_schedule_work(struct work_struct *work)
701 {
702 	struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
703 	int flags = dreq->flags;
704 
705 	dreq->flags = 0;
706 	switch (flags) {
707 		case NFS_ODIRECT_DO_COMMIT:
708 			nfs_direct_commit_schedule(dreq);
709 			break;
710 		case NFS_ODIRECT_RESCHED_WRITES:
711 			nfs_direct_write_reschedule(dreq);
712 			break;
713 		default:
714 			nfs_direct_write_clear_reqs(dreq);
715 			nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);
716 			nfs_direct_complete(dreq);
717 	}
718 }
719 
nfs_direct_write_complete(struct nfs_direct_req * dreq)720 static void nfs_direct_write_complete(struct nfs_direct_req *dreq)
721 {
722 	trace_nfs_direct_write_complete(dreq);
723 	queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */
724 }
725 
nfs_direct_write_completion(struct nfs_pgio_header * hdr)726 static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
727 {
728 	struct nfs_direct_req *dreq = hdr->dreq;
729 	struct nfs_commit_info cinfo;
730 	struct nfs_page *req = nfs_list_entry(hdr->pages.next);
731 	int flags = NFS_ODIRECT_DONE;
732 
733 	trace_nfs_direct_write_completion(dreq);
734 
735 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
736 
737 	spin_lock(&dreq->lock);
738 	if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) {
739 		spin_unlock(&dreq->lock);
740 		goto out_put;
741 	}
742 
743 	nfs_direct_count_bytes(dreq, hdr);
744 	if (test_bit(NFS_IOHDR_UNSTABLE_WRITES, &hdr->flags) &&
745 	    !test_bit(NFS_IOHDR_ERROR, &hdr->flags)) {
746 		if (!dreq->flags)
747 			dreq->flags = NFS_ODIRECT_DO_COMMIT;
748 		flags = dreq->flags;
749 	}
750 	spin_unlock(&dreq->lock);
751 
752 	while (!list_empty(&hdr->pages)) {
753 
754 		req = nfs_list_entry(hdr->pages.next);
755 		nfs_list_remove_request(req);
756 		if (flags == NFS_ODIRECT_DO_COMMIT) {
757 			kref_get(&req->wb_kref);
758 			memcpy(&req->wb_verf, &hdr->verf.verifier,
759 			       sizeof(req->wb_verf));
760 			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
761 				hdr->ds_commit_idx);
762 		} else if (flags == NFS_ODIRECT_RESCHED_WRITES) {
763 			kref_get(&req->wb_kref);
764 			nfs_mark_request_commit(req, NULL, &cinfo, 0);
765 		}
766 		nfs_unlock_and_release_request(req);
767 	}
768 
769 out_put:
770 	if (put_dreq(dreq))
771 		nfs_direct_write_complete(dreq);
772 	hdr->release(hdr);
773 }
774 
nfs_write_sync_pgio_error(struct list_head * head,int error)775 static void nfs_write_sync_pgio_error(struct list_head *head, int error)
776 {
777 	struct nfs_page *req;
778 
779 	while (!list_empty(head)) {
780 		req = nfs_list_entry(head->next);
781 		nfs_list_remove_request(req);
782 		nfs_unlock_and_release_request(req);
783 	}
784 }
785 
nfs_direct_write_reschedule_io(struct nfs_pgio_header * hdr)786 static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr)
787 {
788 	struct nfs_direct_req *dreq = hdr->dreq;
789 	struct nfs_page *req;
790 	struct nfs_commit_info cinfo;
791 
792 	trace_nfs_direct_write_reschedule_io(dreq);
793 
794 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
795 	spin_lock(&dreq->lock);
796 	if (dreq->error == 0)
797 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
798 	set_bit(NFS_IOHDR_REDO, &hdr->flags);
799 	spin_unlock(&dreq->lock);
800 	while (!list_empty(&hdr->pages)) {
801 		req = nfs_list_entry(hdr->pages.next);
802 		nfs_list_remove_request(req);
803 		nfs_unlock_request(req);
804 		nfs_mark_request_commit(req, NULL, &cinfo, 0);
805 	}
806 }
807 
808 static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
809 	.error_cleanup = nfs_write_sync_pgio_error,
810 	.init_hdr = nfs_direct_pgio_init,
811 	.completion = nfs_direct_write_completion,
812 	.reschedule_io = nfs_direct_write_reschedule_io,
813 };
814 
815 
816 /*
817  * NB: Return the value of the first error return code.  Subsequent
818  *     errors after the first one are ignored.
819  */
820 /*
821  * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
822  * operation.  If nfs_writedata_alloc() or get_user_pages() fails,
823  * bail and stop sending more writes.  Write length accounting is
824  * handled automatically by nfs_direct_write_result().  Otherwise, if
825  * no requests have been sent, just return an error.
826  */
nfs_direct_write_schedule_iovec(struct nfs_direct_req * dreq,struct iov_iter * iter,loff_t pos,int ioflags)827 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
828 					       struct iov_iter *iter,
829 					       loff_t pos, int ioflags)
830 {
831 	struct nfs_pageio_descriptor desc;
832 	struct inode *inode = dreq->inode;
833 	struct nfs_commit_info cinfo;
834 	ssize_t result = 0;
835 	size_t requested_bytes = 0;
836 	size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
837 	bool defer = false;
838 
839 	trace_nfs_direct_write_schedule_iovec(dreq);
840 
841 	nfs_pageio_init_write(&desc, inode, ioflags, false,
842 			      &nfs_direct_write_completion_ops);
843 	desc.pg_dreq = dreq;
844 	get_dreq(dreq);
845 	inode_dio_begin(inode);
846 
847 	NFS_I(inode)->write_io += iov_iter_count(iter);
848 	while (iov_iter_count(iter)) {
849 		struct page **pagevec;
850 		size_t bytes;
851 		size_t pgbase;
852 		unsigned npages, i;
853 
854 		result = iov_iter_get_pages_alloc2(iter, &pagevec,
855 						  wsize, &pgbase);
856 		if (result < 0)
857 			break;
858 
859 		bytes = result;
860 		npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
861 		for (i = 0; i < npages; i++) {
862 			struct nfs_page *req;
863 			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
864 
865 			req = nfs_page_create_from_page(dreq->ctx, pagevec[i],
866 							pgbase, pos, req_len);
867 			if (IS_ERR(req)) {
868 				result = PTR_ERR(req);
869 				break;
870 			}
871 
872 			if (desc.pg_error < 0) {
873 				nfs_free_request(req);
874 				result = desc.pg_error;
875 				break;
876 			}
877 
878 			pgbase = 0;
879 			bytes -= req_len;
880 			requested_bytes += req_len;
881 			pos += req_len;
882 			dreq->bytes_left -= req_len;
883 
884 			if (defer) {
885 				nfs_mark_request_commit(req, NULL, &cinfo, 0);
886 				continue;
887 			}
888 
889 			nfs_lock_request(req);
890 			if (nfs_pageio_add_request(&desc, req))
891 				continue;
892 
893 			/* Exit on hard errors */
894 			if (desc.pg_error < 0 && desc.pg_error != -EAGAIN) {
895 				result = desc.pg_error;
896 				nfs_unlock_and_release_request(req);
897 				break;
898 			}
899 
900 			/* If the error is soft, defer remaining requests */
901 			nfs_init_cinfo_from_dreq(&cinfo, dreq);
902 			spin_lock(&dreq->lock);
903 			dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
904 			spin_unlock(&dreq->lock);
905 			nfs_unlock_request(req);
906 			nfs_mark_request_commit(req, NULL, &cinfo, 0);
907 			desc.pg_error = 0;
908 			defer = true;
909 		}
910 		nfs_direct_release_pages(pagevec, npages);
911 		kvfree(pagevec);
912 		if (result < 0)
913 			break;
914 	}
915 	nfs_pageio_complete(&desc);
916 
917 	/*
918 	 * If no bytes were started, return the error, and let the
919 	 * generic layer handle the completion.
920 	 */
921 	if (requested_bytes == 0) {
922 		inode_dio_end(inode);
923 		nfs_direct_req_release(dreq);
924 		return result < 0 ? result : -EIO;
925 	}
926 
927 	if (put_dreq(dreq))
928 		nfs_direct_write_complete(dreq);
929 	return requested_bytes;
930 }
931 
932 /**
933  * nfs_file_direct_write - file direct write operation for NFS files
934  * @iocb: target I/O control block
935  * @iter: vector of user buffers from which to write data
936  * @swap: flag indicating this is swap IO, not O_DIRECT IO
937  *
938  * We use this function for direct writes instead of calling
939  * generic_file_aio_write() in order to avoid taking the inode
940  * semaphore and updating the i_size.  The NFS server will set
941  * the new i_size and this client must read the updated size
942  * back into its cache.  We let the server do generic write
943  * parameter checking and report problems.
944  *
945  * We eliminate local atime updates, see direct read above.
946  *
947  * We avoid unnecessary page cache invalidations for normal cached
948  * readers of this file.
949  *
950  * Note that O_APPEND is not supported for NFS direct writes, as there
951  * is no atomic O_APPEND write facility in the NFS protocol.
952  */
nfs_file_direct_write(struct kiocb * iocb,struct iov_iter * iter,bool swap)953 ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter,
954 			      bool swap)
955 {
956 	ssize_t result, requested;
957 	size_t count;
958 	struct file *file = iocb->ki_filp;
959 	struct address_space *mapping = file->f_mapping;
960 	struct inode *inode = mapping->host;
961 	struct nfs_direct_req *dreq;
962 	struct nfs_lock_context *l_ctx;
963 	loff_t pos, end;
964 
965 	dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
966 		file, iov_iter_count(iter), (long long) iocb->ki_pos);
967 
968 	if (swap)
969 		/* bypass generic checks */
970 		result =  iov_iter_count(iter);
971 	else
972 		result = generic_write_checks(iocb, iter);
973 	if (result <= 0)
974 		return result;
975 	count = result;
976 	nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
977 
978 	pos = iocb->ki_pos;
979 	end = (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT;
980 
981 	task_io_account_write(count);
982 
983 	result = -ENOMEM;
984 	dreq = nfs_direct_req_alloc();
985 	if (!dreq)
986 		goto out;
987 
988 	dreq->inode = inode;
989 	dreq->bytes_left = dreq->max_count = count;
990 	dreq->io_start = pos;
991 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
992 	l_ctx = nfs_get_lock_context(dreq->ctx);
993 	if (IS_ERR(l_ctx)) {
994 		result = PTR_ERR(l_ctx);
995 		nfs_direct_req_release(dreq);
996 		goto out_release;
997 	}
998 	dreq->l_ctx = l_ctx;
999 	if (!is_sync_kiocb(iocb))
1000 		dreq->iocb = iocb;
1001 	pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode);
1002 
1003 	if (swap) {
1004 		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
1005 							    FLUSH_STABLE);
1006 	} else {
1007 		nfs_start_io_direct(inode);
1008 
1009 		requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
1010 							    FLUSH_COND_STABLE);
1011 
1012 		if (mapping->nrpages) {
1013 			invalidate_inode_pages2_range(mapping,
1014 						      pos >> PAGE_SHIFT, end);
1015 		}
1016 
1017 		nfs_end_io_direct(inode);
1018 	}
1019 
1020 	if (requested > 0) {
1021 		result = nfs_direct_wait(dreq);
1022 		if (result > 0) {
1023 			requested -= result;
1024 			iocb->ki_pos = pos + result;
1025 			/* XXX: should check the generic_write_sync retval */
1026 			generic_write_sync(iocb, result);
1027 		}
1028 		iov_iter_revert(iter, requested);
1029 	} else {
1030 		result = requested;
1031 	}
1032 	nfs_fscache_invalidate(inode, FSCACHE_INVAL_DIO_WRITE);
1033 out_release:
1034 	nfs_direct_req_release(dreq);
1035 out:
1036 	return result;
1037 }
1038 
1039 /**
1040  * nfs_init_directcache - create a slab cache for nfs_direct_req structures
1041  *
1042  */
nfs_init_directcache(void)1043 int __init nfs_init_directcache(void)
1044 {
1045 	nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
1046 						sizeof(struct nfs_direct_req),
1047 						0, (SLAB_RECLAIM_ACCOUNT|
1048 							SLAB_MEM_SPREAD),
1049 						NULL);
1050 	if (nfs_direct_cachep == NULL)
1051 		return -ENOMEM;
1052 
1053 	return 0;
1054 }
1055 
1056 /**
1057  * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
1058  *
1059  */
nfs_destroy_directcache(void)1060 void nfs_destroy_directcache(void)
1061 {
1062 	kmem_cache_destroy(nfs_direct_cachep);
1063 }
1064