13b77495aSJens Axboe // SPDX-License-Identifier: GPL-2.0
23b77495aSJens Axboe #ifndef IOU_KBUF_H
33b77495aSJens Axboe #define IOU_KBUF_H
43b77495aSJens Axboe
53b77495aSJens Axboe #include <uapi/linux/io_uring.h>
63b77495aSJens Axboe
73b77495aSJens Axboe struct io_buffer_list {
83b77495aSJens Axboe /*
93b77495aSJens Axboe * If ->buf_nr_pages is set, then buf_pages/buf_ring are used. If not,
103b77495aSJens Axboe * then these are classic provided buffers and ->buf_list is used.
113b77495aSJens Axboe */
123b77495aSJens Axboe union {
133b77495aSJens Axboe struct list_head buf_list;
143b77495aSJens Axboe struct {
153b77495aSJens Axboe struct page **buf_pages;
163b77495aSJens Axboe struct io_uring_buf_ring *buf_ring;
173b77495aSJens Axboe };
1809f75200SJens Axboe struct rcu_head rcu;
193b77495aSJens Axboe };
203b77495aSJens Axboe __u16 bgid;
213b77495aSJens Axboe
223b77495aSJens Axboe /* below is for ring provided buffers */
233b77495aSJens Axboe __u16 buf_nr_pages;
243b77495aSJens Axboe __u16 nr_entries;
253b77495aSJens Axboe __u16 head;
263b77495aSJens Axboe __u16 mask;
2725a2c188SJens Axboe
28b392402dSJens Axboe atomic_t refs;
29b392402dSJens Axboe
3025a2c188SJens Axboe /* ring mapped provided buffers */
3125a2c188SJens Axboe __u8 is_mapped;
32c56e022cSJens Axboe /* ring mapped provided buffers, but mmap'ed by application */
33c56e022cSJens Axboe __u8 is_mmap;
343b77495aSJens Axboe };
353b77495aSJens Axboe
363b77495aSJens Axboe struct io_buffer {
373b77495aSJens Axboe struct list_head list;
383b77495aSJens Axboe __u64 addr;
393b77495aSJens Axboe __u32 len;
403b77495aSJens Axboe __u16 bid;
413b77495aSJens Axboe __u16 bgid;
423b77495aSJens Axboe };
433b77495aSJens Axboe
443b77495aSJens Axboe void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
453b77495aSJens Axboe unsigned int issue_flags);
463b77495aSJens Axboe void io_destroy_buffers(struct io_ring_ctx *ctx);
473b77495aSJens Axboe
483b77495aSJens Axboe int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
493b77495aSJens Axboe int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags);
503b77495aSJens Axboe
513b77495aSJens Axboe int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
523b77495aSJens Axboe int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags);
533b77495aSJens Axboe
543b77495aSJens Axboe int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
553b77495aSJens Axboe int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
563b77495aSJens Axboe
577138ebbeSJens Axboe void io_kbuf_mmap_list_free(struct io_ring_ctx *ctx);
587138ebbeSJens Axboe
5953ccf69bSPavel Begunkov unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags);
6053ccf69bSPavel Begunkov
61024b8fdeSHao Xu void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
62795bbbc8SHao Xu
63*65938e81SJens Axboe void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl);
64*65938e81SJens Axboe struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx,
65*65938e81SJens Axboe unsigned long bgid);
66c56e022cSJens Axboe
io_kbuf_recycle_ring(struct io_kiocb * req)67795bbbc8SHao Xu static inline void io_kbuf_recycle_ring(struct io_kiocb *req)
68795bbbc8SHao Xu {
69795bbbc8SHao Xu /*
70795bbbc8SHao Xu * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
71795bbbc8SHao Xu * the flag and hence ensure that bl->head doesn't get incremented.
72795bbbc8SHao Xu * If the tail has already been incremented, hang on to it.
73795bbbc8SHao Xu * The exception is partial io, that case we should increment bl->head
74795bbbc8SHao Xu * to monopolize the buffer.
75795bbbc8SHao Xu */
76795bbbc8SHao Xu if (req->buf_list) {
77795bbbc8SHao Xu if (req->flags & REQ_F_PARTIAL_IO) {
78795bbbc8SHao Xu /*
79795bbbc8SHao Xu * If we end up here, then the io_uring_lock has
80795bbbc8SHao Xu * been kept held since we retrieved the buffer.
81795bbbc8SHao Xu * For the io-wq case, we already cleared
82795bbbc8SHao Xu * req->buf_list when the buffer was retrieved,
83795bbbc8SHao Xu * hence it cannot be set here for that case.
84795bbbc8SHao Xu */
85795bbbc8SHao Xu req->buf_list->head++;
86795bbbc8SHao Xu req->buf_list = NULL;
87795bbbc8SHao Xu } else {
88795bbbc8SHao Xu req->buf_index = req->buf_list->bgid;
89795bbbc8SHao Xu req->flags &= ~REQ_F_BUFFER_RING;
90795bbbc8SHao Xu }
91795bbbc8SHao Xu }
92795bbbc8SHao Xu }
93024b8fdeSHao Xu
io_do_buffer_select(struct io_kiocb * req)943b77495aSJens Axboe static inline bool io_do_buffer_select(struct io_kiocb *req)
953b77495aSJens Axboe {
963b77495aSJens Axboe if (!(req->flags & REQ_F_BUFFER_SELECT))
973b77495aSJens Axboe return false;
983b77495aSJens Axboe return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));
993b77495aSJens Axboe }
1003b77495aSJens Axboe
io_kbuf_recycle(struct io_kiocb * req,unsigned issue_flags)1013b77495aSJens Axboe static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
1023b77495aSJens Axboe {
103024b8fdeSHao Xu if (req->flags & REQ_F_BUFFER_SELECTED)
104024b8fdeSHao Xu io_kbuf_recycle_legacy(req, issue_flags);
105024b8fdeSHao Xu if (req->flags & REQ_F_BUFFER_RING)
106024b8fdeSHao Xu io_kbuf_recycle_ring(req);
1073b77495aSJens Axboe }
1083b77495aSJens Axboe
__io_put_kbuf_list(struct io_kiocb * req,struct list_head * list)10953ccf69bSPavel Begunkov static inline unsigned int __io_put_kbuf_list(struct io_kiocb *req,
11053ccf69bSPavel Begunkov struct list_head *list)
1113b77495aSJens Axboe {
11232f3c434SDylan Yudaken unsigned int ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
11332f3c434SDylan Yudaken
1143b77495aSJens Axboe if (req->flags & REQ_F_BUFFER_RING) {
11532f3c434SDylan Yudaken if (req->buf_list) {
11632f3c434SDylan Yudaken req->buf_index = req->buf_list->bgid;
1173b77495aSJens Axboe req->buf_list->head++;
11832f3c434SDylan Yudaken }
1193b77495aSJens Axboe req->flags &= ~REQ_F_BUFFER_RING;
1203b77495aSJens Axboe } else {
12132f3c434SDylan Yudaken req->buf_index = req->kbuf->bgid;
1223b77495aSJens Axboe list_add(&req->kbuf->list, list);
1233b77495aSJens Axboe req->flags &= ~REQ_F_BUFFER_SELECTED;
1243b77495aSJens Axboe }
1253b77495aSJens Axboe
12632f3c434SDylan Yudaken return ret;
1273b77495aSJens Axboe }
1283b77495aSJens Axboe
io_put_kbuf_comp(struct io_kiocb * req)1293b77495aSJens Axboe static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)
1303b77495aSJens Axboe {
1313b77495aSJens Axboe lockdep_assert_held(&req->ctx->completion_lock);
1323b77495aSJens Axboe
1333b77495aSJens Axboe if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
1343b77495aSJens Axboe return 0;
13553ccf69bSPavel Begunkov return __io_put_kbuf_list(req, &req->ctx->io_buffers_comp);
1363b77495aSJens Axboe }
1373b77495aSJens Axboe
io_put_kbuf(struct io_kiocb * req,unsigned issue_flags)1383b77495aSJens Axboe static inline unsigned int io_put_kbuf(struct io_kiocb *req,
1393b77495aSJens Axboe unsigned issue_flags)
1403b77495aSJens Axboe {
1413b77495aSJens Axboe
1423b77495aSJens Axboe if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
1433b77495aSJens Axboe return 0;
14453ccf69bSPavel Begunkov return __io_put_kbuf(req, issue_flags);
1453b77495aSJens Axboe }
1463b77495aSJens Axboe #endif
147