13b77495aSJens Axboe // SPDX-License-Identifier: GPL-2.0 23b77495aSJens Axboe #ifndef IOU_KBUF_H 33b77495aSJens Axboe #define IOU_KBUF_H 43b77495aSJens Axboe 53b77495aSJens Axboe #include <uapi/linux/io_uring.h> 63b77495aSJens Axboe 73b77495aSJens Axboe struct io_buffer_list { 83b77495aSJens Axboe /* 93b77495aSJens Axboe * If ->buf_nr_pages is set, then buf_pages/buf_ring are used. If not, 103b77495aSJens Axboe * then these are classic provided buffers and ->buf_list is used. 113b77495aSJens Axboe */ 123b77495aSJens Axboe union { 133b77495aSJens Axboe struct list_head buf_list; 143b77495aSJens Axboe struct { 153b77495aSJens Axboe struct page **buf_pages; 163b77495aSJens Axboe struct io_uring_buf_ring *buf_ring; 173b77495aSJens Axboe }; 183b77495aSJens Axboe }; 193b77495aSJens Axboe __u16 bgid; 203b77495aSJens Axboe 213b77495aSJens Axboe /* below is for ring provided buffers */ 223b77495aSJens Axboe __u16 buf_nr_pages; 233b77495aSJens Axboe __u16 nr_entries; 243b77495aSJens Axboe __u16 head; 253b77495aSJens Axboe __u16 mask; 263b77495aSJens Axboe }; 273b77495aSJens Axboe 283b77495aSJens Axboe struct io_buffer { 293b77495aSJens Axboe struct list_head list; 303b77495aSJens Axboe __u64 addr; 313b77495aSJens Axboe __u32 len; 323b77495aSJens Axboe __u16 bid; 333b77495aSJens Axboe __u16 bgid; 343b77495aSJens Axboe }; 353b77495aSJens Axboe 363b77495aSJens Axboe void __user *io_buffer_select(struct io_kiocb *req, size_t *len, 373b77495aSJens Axboe unsigned int issue_flags); 383b77495aSJens Axboe void io_destroy_buffers(struct io_ring_ctx *ctx); 393b77495aSJens Axboe 403b77495aSJens Axboe int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 413b77495aSJens Axboe int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags); 423b77495aSJens Axboe 433b77495aSJens Axboe int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 443b77495aSJens Axboe int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags); 453b77495aSJens Axboe 463b77495aSJens Axboe int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg); 473b77495aSJens Axboe int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg); 483b77495aSJens Axboe 4953ccf69bSPavel Begunkov unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags); 5053ccf69bSPavel Begunkov 51024b8fdeSHao Xu void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags); 52795bbbc8SHao Xu 53795bbbc8SHao Xu static inline void io_kbuf_recycle_ring(struct io_kiocb *req) 54795bbbc8SHao Xu { 55795bbbc8SHao Xu /* 56795bbbc8SHao Xu * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear 57795bbbc8SHao Xu * the flag and hence ensure that bl->head doesn't get incremented. 58795bbbc8SHao Xu * If the tail has already been incremented, hang on to it. 59795bbbc8SHao Xu * The exception is partial io, that case we should increment bl->head 60795bbbc8SHao Xu * to monopolize the buffer. 61795bbbc8SHao Xu */ 62795bbbc8SHao Xu if (req->buf_list) { 63795bbbc8SHao Xu if (req->flags & REQ_F_PARTIAL_IO) { 64795bbbc8SHao Xu /* 65795bbbc8SHao Xu * If we end up here, then the io_uring_lock has 66795bbbc8SHao Xu * been kept held since we retrieved the buffer. 67795bbbc8SHao Xu * For the io-wq case, we already cleared 68795bbbc8SHao Xu * req->buf_list when the buffer was retrieved, 69795bbbc8SHao Xu * hence it cannot be set here for that case. 70795bbbc8SHao Xu */ 71795bbbc8SHao Xu req->buf_list->head++; 72795bbbc8SHao Xu req->buf_list = NULL; 73795bbbc8SHao Xu } else { 74795bbbc8SHao Xu req->buf_index = req->buf_list->bgid; 75795bbbc8SHao Xu req->flags &= ~REQ_F_BUFFER_RING; 76795bbbc8SHao Xu } 77795bbbc8SHao Xu } 78795bbbc8SHao Xu } 79024b8fdeSHao Xu 803b77495aSJens Axboe static inline bool io_do_buffer_select(struct io_kiocb *req) 813b77495aSJens Axboe { 823b77495aSJens Axboe if (!(req->flags & REQ_F_BUFFER_SELECT)) 833b77495aSJens Axboe return false; 843b77495aSJens Axboe return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)); 853b77495aSJens Axboe } 863b77495aSJens Axboe 873b77495aSJens Axboe static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) 883b77495aSJens Axboe { 893b77495aSJens Axboe /* 903b77495aSJens Axboe * READV uses fields in `struct io_rw` (len/addr) to stash the selected 913b77495aSJens Axboe * buffer data. However if that buffer is recycled the original request 923b77495aSJens Axboe * data stored in addr is lost. Therefore forbid recycling for now. 933b77495aSJens Axboe */ 943b77495aSJens Axboe if (req->opcode == IORING_OP_READV) 953b77495aSJens Axboe return; 963b77495aSJens Axboe 97024b8fdeSHao Xu if (req->flags & REQ_F_BUFFER_SELECTED) 98024b8fdeSHao Xu io_kbuf_recycle_legacy(req, issue_flags); 99024b8fdeSHao Xu if (req->flags & REQ_F_BUFFER_RING) 100024b8fdeSHao Xu io_kbuf_recycle_ring(req); 1013b77495aSJens Axboe } 1023b77495aSJens Axboe 10353ccf69bSPavel Begunkov static inline unsigned int __io_put_kbuf_list(struct io_kiocb *req, 10453ccf69bSPavel Begunkov struct list_head *list) 1053b77495aSJens Axboe { 106*32f3c434SDylan Yudaken unsigned int ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT); 107*32f3c434SDylan Yudaken 1083b77495aSJens Axboe if (req->flags & REQ_F_BUFFER_RING) { 109*32f3c434SDylan Yudaken if (req->buf_list) { 110*32f3c434SDylan Yudaken req->buf_index = req->buf_list->bgid; 1113b77495aSJens Axboe req->buf_list->head++; 112*32f3c434SDylan Yudaken } 1133b77495aSJens Axboe req->flags &= ~REQ_F_BUFFER_RING; 1143b77495aSJens Axboe } else { 115*32f3c434SDylan Yudaken req->buf_index = req->kbuf->bgid; 1163b77495aSJens Axboe list_add(&req->kbuf->list, list); 1173b77495aSJens Axboe req->flags &= ~REQ_F_BUFFER_SELECTED; 1183b77495aSJens Axboe } 1193b77495aSJens Axboe 120*32f3c434SDylan Yudaken return ret; 1213b77495aSJens Axboe } 1223b77495aSJens Axboe 1233b77495aSJens Axboe static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req) 1243b77495aSJens Axboe { 1253b77495aSJens Axboe lockdep_assert_held(&req->ctx->completion_lock); 1263b77495aSJens Axboe 1273b77495aSJens Axboe if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))) 1283b77495aSJens Axboe return 0; 12953ccf69bSPavel Begunkov return __io_put_kbuf_list(req, &req->ctx->io_buffers_comp); 1303b77495aSJens Axboe } 1313b77495aSJens Axboe 1323b77495aSJens Axboe static inline unsigned int io_put_kbuf(struct io_kiocb *req, 1333b77495aSJens Axboe unsigned issue_flags) 1343b77495aSJens Axboe { 1353b77495aSJens Axboe 1363b77495aSJens Axboe if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))) 1373b77495aSJens Axboe return 0; 13853ccf69bSPavel Begunkov return __io_put_kbuf(req, issue_flags); 1393b77495aSJens Axboe } 1403b77495aSJens Axboe #endif 141