55 static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter)  in ext4_should_use_dio()  argument
57 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_should_use_dio()
66 	return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align);  in ext4_should_use_dio()
69 static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dio_read_iter()  argument
72 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_read_iter()
74 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_read_iter()
81 	if (!ext4_should_use_dio(iocb, to)) {  in ext4_dio_read_iter()
90 		iocb->ki_flags &= ~IOCB_DIRECT;  in ext4_dio_read_iter()
91 		return generic_file_read_iter(iocb, to);  in ext4_dio_read_iter()
94 	ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0);  in ext4_dio_read_iter()
97 	file_accessed(iocb->ki_filp);  in ext4_dio_read_iter()
102 static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dax_read_iter()  argument
104 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_read_iter()
107 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_read_iter()
120 		return generic_file_read_iter(iocb, to);  in ext4_dax_read_iter()
122 	ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);  in ext4_dax_read_iter()
125 	file_accessed(iocb->ki_filp);  in ext4_dax_read_iter()
130 static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_file_read_iter()  argument
132 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_read_iter()
142 		return ext4_dax_read_iter(iocb, to);  in ext4_file_read_iter()
144 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_read_iter()
145 		return ext4_dio_read_iter(iocb, to);  in ext4_file_read_iter()
147 	return generic_file_read_iter(iocb, to);  in ext4_file_read_iter()
243 static ssize_t ext4_generic_write_checks(struct kiocb *iocb,  in ext4_generic_write_checks()  argument
246 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_generic_write_checks()
252 	ret = generic_write_checks(iocb, from);  in ext4_generic_write_checks()
263 		if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)  in ext4_generic_write_checks()
265 		iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);  in ext4_generic_write_checks()
271 static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)  in ext4_write_checks()  argument
275 	count = ext4_generic_write_checks(iocb, from);  in ext4_write_checks()
279 	ret = file_modified(iocb->ki_filp);  in ext4_write_checks()
285 static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,  in ext4_buffered_write_iter()  argument
289 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_buffered_write_iter()
291 	if (iocb->ki_flags & IOCB_NOWAIT)  in ext4_buffered_write_iter()
295 	ret = ext4_write_checks(iocb, from);  in ext4_buffered_write_iter()
299 	ret = generic_perform_write(iocb, from);  in ext4_buffered_write_iter()
305 	return generic_write_sync(iocb, ret);  in ext4_buffered_write_iter()
374 static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,  in ext4_dio_write_end_io()  argument
377 	loff_t pos = iocb->ki_pos;  in ext4_dio_write_end_io()
378 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_end_io()
421 static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,  in ext4_dio_write_checks()  argument
425 	struct file *file = iocb->ki_filp;  in ext4_dio_write_checks()
433 	ret = ext4_generic_write_checks(iocb, from);  in ext4_dio_write_checks()
437 	offset = iocb->ki_pos;  in ext4_dio_write_checks()
458 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
476 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
498 static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dio_write_iter()  argument
502 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_iter()
503 	loff_t offset = iocb->ki_pos;  in ext4_dio_write_iter()
518 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_iter()
534 	if (!ext4_should_use_dio(iocb, from)) {  in ext4_dio_write_iter()
539 		return ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
551 	ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend,  in ext4_dio_write_iter()
556 	offset = iocb->ki_pos;  in ext4_dio_write_iter()
577 	ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,  in ext4_dio_write_iter()
602 		offset = iocb->ki_pos;  in ext4_dio_write_iter()
603 		err = ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
616 		err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
619 			invalidate_mapping_pages(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
629 ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dax_write_iter()  argument
636 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_write_iter()
638 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_write_iter()
645 	ret = ext4_write_checks(iocb, from);  in ext4_dax_write_iter()
649 	offset = iocb->ki_pos;  in ext4_dax_write_iter()
669 	ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);  in ext4_dax_write_iter()
678 		ret = generic_write_sync(iocb, ret);  in ext4_dax_write_iter()
684 ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_file_write_iter()  argument
686 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_write_iter()
693 		return ext4_dax_write_iter(iocb, from);  in ext4_file_write_iter()
695 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_write_iter()
696 		return ext4_dio_write_iter(iocb, from);  in ext4_file_write_iter()
698 		return ext4_buffered_write_iter(iocb, from);  in ext4_file_write_iter()