27 static blk_opf_t dio_bio_write_op(struct kiocb *iocb)  in dio_bio_write_op()  argument
32 	if (iocb_is_dsync(iocb))  in dio_bio_write_op()
46 static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,  in __blkdev_direct_IO_simple()  argument
49 	struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host);  in __blkdev_direct_IO_simple()
51 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO_simple()
73 		bio_init(&bio, bdev, vecs, nr_pages, dio_bio_write_op(iocb));  in __blkdev_direct_IO_simple()
76 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
86 	if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO_simple()
111 		struct kiocb		*iocb;  member
132 			struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io()  local
135 			WRITE_ONCE(iocb->private, NULL);  in blkdev_bio_end_io()
139 				iocb->ki_pos += ret;  in blkdev_bio_end_io()
144 			dio->iocb->ki_complete(iocb, ret);  in blkdev_bio_end_io()
162 static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,  in __blkdev_direct_IO()  argument
165 	struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host);  in __blkdev_direct_IO()
170 	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);  in __blkdev_direct_IO()
171 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO()
177 	if (iocb->ki_flags & IOCB_ALLOC_CACHE)  in __blkdev_direct_IO()
189 	is_sync = is_sync_kiocb(iocb);  in __blkdev_direct_IO()
195 		dio->iocb = iocb;  in __blkdev_direct_IO()
208 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
216 		if (iocb->ki_flags & IOCB_NOWAIT) {  in __blkdev_direct_IO()
279 	struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io_async()  local
282 	WRITE_ONCE(iocb->private, NULL);  in blkdev_bio_end_io_async()
286 		iocb->ki_pos += ret;  in blkdev_bio_end_io_async()
291 	iocb->ki_complete(iocb, ret);  in blkdev_bio_end_io_async()
301 static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,  in __blkdev_direct_IO_async()  argument
305 	struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host);  in __blkdev_direct_IO_async()
307 	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);  in __blkdev_direct_IO_async()
310 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO_async()
316 	if (iocb->ki_flags & IOCB_ALLOC_CACHE)  in __blkdev_direct_IO_async()
322 	dio->iocb = iocb;  in __blkdev_direct_IO_async()
325 	bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_async()
353 	if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO_async()
356 	if (iocb->ki_flags & IOCB_HIPRI) {  in __blkdev_direct_IO_async()
359 		WRITE_ONCE(iocb->private, bio);  in __blkdev_direct_IO_async()
366 static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)  in blkdev_direct_IO()  argument
375 		if (is_sync_kiocb(iocb))  in blkdev_direct_IO()
376 			return __blkdev_direct_IO_simple(iocb, iter, nr_pages);  in blkdev_direct_IO()
377 		return __blkdev_direct_IO_async(iocb, iter, nr_pages);  in blkdev_direct_IO()
379 	return __blkdev_direct_IO(iocb, iter, bio_max_segs(nr_pages));  in blkdev_direct_IO()
609 blkdev_direct_write(struct kiocb *iocb, struct iov_iter *from)  in blkdev_direct_write()  argument
614 	written = kiocb_invalidate_pages(iocb, count);  in blkdev_direct_write()
621 	written = blkdev_direct_IO(iocb, from);  in blkdev_direct_write()
623 		kiocb_invalidate_post_direct_write(iocb, count);  in blkdev_direct_write()
624 		iocb->ki_pos += written;  in blkdev_direct_write()
632 static ssize_t blkdev_buffered_write(struct kiocb *iocb, struct iov_iter *from)  in blkdev_buffered_write()  argument
634 	return iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops);  in blkdev_buffered_write()
644 static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)  in blkdev_write_iter()  argument
646 	struct file *file = iocb->ki_filp;  in blkdev_write_iter()
662 	if (iocb->ki_pos >= size)  in blkdev_write_iter()
665 	if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)  in blkdev_write_iter()
668 	size -= iocb->ki_pos;  in blkdev_write_iter()
678 	if (iocb->ki_flags & IOCB_DIRECT) {  in blkdev_write_iter()
679 		ret = blkdev_direct_write(iocb, from);  in blkdev_write_iter()
681 			ret = direct_write_fallback(iocb, from, ret,  in blkdev_write_iter()
682 					blkdev_buffered_write(iocb, from));  in blkdev_write_iter()
684 		ret = blkdev_buffered_write(iocb, from);  in blkdev_write_iter()
688 		ret = generic_write_sync(iocb, ret);  in blkdev_write_iter()
693 static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)  in blkdev_read_iter()  argument
695 	struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host);  in blkdev_read_iter()
697 	loff_t pos = iocb->ki_pos;  in blkdev_read_iter()
714 	if (iocb->ki_flags & IOCB_DIRECT) {  in blkdev_read_iter()
715 		ret = kiocb_write_and_wait(iocb, count);  in blkdev_read_iter()
718 		file_accessed(iocb->ki_filp);  in blkdev_read_iter()
720 		ret = blkdev_direct_IO(iocb, to);  in blkdev_read_iter()
722 			iocb->ki_pos += ret;  in blkdev_read_iter()
730 	ret = filemap_read(iocb, to, ret);  in blkdev_read_iter()