31 	struct kiocb		*iocb;  member
67 	struct kiocb *iocb = dio->iocb;  in iomap_dio_submit_bio()  local
72 	if ((iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(iocb)) {  in iomap_dio_submit_bio()
73 		bio_set_polled(bio, iocb);  in iomap_dio_submit_bio()
74 		WRITE_ONCE(iocb->private, bio);  in iomap_dio_submit_bio()
86 	struct kiocb *iocb = dio->iocb;  in iomap_dio_complete()  local
87 	loff_t offset = iocb->ki_pos;  in iomap_dio_complete()
91 		ret = dops->end_io(iocb, dio->size, ret, dio->flags);  in iomap_dio_complete()
114 		kiocb_invalidate_post_direct_write(iocb, dio->size);  in iomap_dio_complete()
116 	inode_dio_end(file_inode(iocb->ki_filp));  in iomap_dio_complete()
119 		iocb->ki_pos += ret;  in iomap_dio_complete()
126 			ret = generic_write_sync(iocb, ret);  in iomap_dio_complete()
130 	trace_iomap_dio_complete(iocb, dio->error, ret);  in iomap_dio_complete()
144 	struct kiocb *iocb = dio->iocb;  in iomap_dio_complete_work()  local
146 	iocb->ki_complete(iocb, iomap_dio_complete(dio));  in iomap_dio_complete_work()
163 	struct kiocb *iocb = dio->iocb;  in iomap_dio_bio_end_io()  local
186 		WRITE_ONCE(iocb->private, NULL);  in iomap_dio_bio_end_io()
197 		iocb->private = dio;  in iomap_dio_bio_end_io()
198 		iocb->dio_complete = iomap_dio_deferred_complete;  in iomap_dio_bio_end_io()
212 		iocb->ki_complete(iocb, 0);  in iomap_dio_bio_end_io()
223 	queue_work(file_inode(iocb->ki_filp)->i_sb->s_dio_done_wq,  in iomap_dio_bio_end_io()
238 	struct inode *inode = file_inode(dio->iocb->ki_filp);  in iomap_dio_zero()
354 		dio->iocb->ki_flags &= ~IOCB_HIPRI;  in iomap_dio_bio_iter()
383 		bio->bi_ioprio = dio->iocb->ki_ioprio;  in iomap_dio_bio_iter()
416 			dio->iocb->ki_flags &= ~IOCB_HIPRI;  in iomap_dio_bio_iter()
511 				    dio->iocb->ki_filp, current->comm);  in iomap_dio_iter()
540 __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,  in __iomap_dio_rw()  argument
544 	struct inode *inode = file_inode(iocb->ki_filp);  in __iomap_dio_rw()
547 		.pos		= iocb->ki_pos,  in __iomap_dio_rw()
553 		is_sync_kiocb(iocb) || (dio_flags & IOMAP_DIO_FORCE_WAIT);  in __iomap_dio_rw()
558 	trace_iomap_dio_rw_begin(iocb, iter, dio_flags, done_before);  in __iomap_dio_rw()
567 	dio->iocb = iocb;  in __iomap_dio_rw()
579 	if (iocb->ki_flags & IOCB_NOWAIT)  in __iomap_dio_rw()
592 		ret = kiocb_write_and_wait(iocb, iomi.len);  in __iomap_dio_rw()
605 		if (iocb->ki_flags & IOCB_DIO_CALLER_COMP)  in __iomap_dio_rw()
617 		if (iocb_is_dsync(iocb)) {  in __iomap_dio_rw()
629 			if (!(iocb->ki_flags & IOCB_SYNC))  in __iomap_dio_rw()
638 		ret = kiocb_invalidate_pages(iocb, iomi.len);  in __iomap_dio_rw()
664 		iocb->ki_flags &= ~IOCB_HIPRI;  in __iomap_dio_rw()
678 		if (!(iocb->ki_flags & IOCB_NOWAIT))  in __iomap_dio_rw()
742 iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,  in iomap_dio_rw()  argument
748 	dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private,  in iomap_dio_rw()