173572984SJens Axboe // SPDX-License-Identifier: GPL-2.0
273572984SJens Axboe #ifndef IOU_RSRC_H
373572984SJens Axboe #define IOU_RSRC_H
473572984SJens Axboe
573572984SJens Axboe #include <net/af_unix.h>
673572984SJens Axboe
79eae8655SPavel Begunkov #include "alloc_cache.h"
89eae8655SPavel Begunkov
969bbc6adSPavel Begunkov #define IO_NODE_ALLOC_CACHE_MAX 32
1069bbc6adSPavel Begunkov
1173572984SJens Axboe #define IO_RSRC_TAG_TABLE_SHIFT (PAGE_SHIFT - 3)
1273572984SJens Axboe #define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT)
1373572984SJens Axboe #define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1)
1473572984SJens Axboe
1573572984SJens Axboe enum {
1673572984SJens Axboe IORING_RSRC_FILE = 0,
1773572984SJens Axboe IORING_RSRC_BUFFER = 1,
1873572984SJens Axboe };
1973572984SJens Axboe
2073572984SJens Axboe struct io_rsrc_put {
2173572984SJens Axboe u64 tag;
2273572984SJens Axboe union {
2373572984SJens Axboe void *rsrc;
2473572984SJens Axboe struct file *file;
2573572984SJens Axboe struct io_mapped_ubuf *buf;
2673572984SJens Axboe };
2773572984SJens Axboe };
2873572984SJens Axboe
2973572984SJens Axboe typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
3073572984SJens Axboe
3173572984SJens Axboe struct io_rsrc_data {
3273572984SJens Axboe struct io_ring_ctx *ctx;
3373572984SJens Axboe
3473572984SJens Axboe u64 **tags;
3573572984SJens Axboe unsigned int nr;
36fc7f3a8dSPavel Begunkov u16 rsrc_type;
3773572984SJens Axboe bool quiesce;
3873572984SJens Axboe };
3973572984SJens Axboe
4073572984SJens Axboe struct io_rsrc_node {
419eae8655SPavel Begunkov union {
429eae8655SPavel Begunkov struct io_cache_entry cache;
432236b390SPavel Begunkov struct io_ring_ctx *ctx;
449eae8655SPavel Begunkov };
45ef8ae64fSPavel Begunkov int refs;
4626147da3SPavel Begunkov bool empty;
472236b390SPavel Begunkov u16 type;
48c376644fSPavel Begunkov struct list_head node;
49c376644fSPavel Begunkov struct io_rsrc_put item;
5073572984SJens Axboe };
5173572984SJens Axboe
52ad163a7eSJens Axboe struct io_mapped_ubuf {
53ad163a7eSJens Axboe u64 ubuf;
54ad163a7eSJens Axboe u64 ubuf_end;
55ad163a7eSJens Axboe unsigned int nr_bvecs;
56ad163a7eSJens Axboe unsigned long acct_pages;
57*04d9244cSKees Cook struct bio_vec bvec[] __counted_by(nr_bvecs);
58ad163a7eSJens Axboe };
59ad163a7eSJens Axboe
60b8fb5b4fSPavel Begunkov void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
619eae8655SPavel Begunkov void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
622933ae6eSPavel Begunkov struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
6363fea890SPavel Begunkov int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc);
6473572984SJens Axboe
65c059f785SPavel Begunkov int io_import_fixed(int ddir, struct iov_iter *iter,
66c059f785SPavel Begunkov struct io_mapped_ubuf *imu,
67c059f785SPavel Begunkov u64 buf_addr, size_t len);
6873572984SJens Axboe
6973572984SJens Axboe void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
7073572984SJens Axboe int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
7173572984SJens Axboe int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
7273572984SJens Axboe unsigned int nr_args, u64 __user *tags);
7373572984SJens Axboe void __io_sqe_files_unregister(struct io_ring_ctx *ctx);
7473572984SJens Axboe int io_sqe_files_unregister(struct io_ring_ctx *ctx);
7573572984SJens Axboe int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
7673572984SJens Axboe unsigned nr_args, u64 __user *tags);
7773572984SJens Axboe
7873572984SJens Axboe int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
7973572984SJens Axboe unsigned nr_args);
8073572984SJens Axboe int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
8173572984SJens Axboe unsigned size, unsigned type);
8273572984SJens Axboe int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
8373572984SJens Axboe unsigned int size, unsigned int type);
8473572984SJens Axboe
io_put_rsrc_node(struct io_ring_ctx * ctx,struct io_rsrc_node * node)851f2c8f61SPavel Begunkov static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
8673572984SJens Axboe {
871f2c8f61SPavel Begunkov lockdep_assert_held(&ctx->uring_lock);
881f2c8f61SPavel Begunkov
89ef8ae64fSPavel Begunkov if (node && !--node->refs)
90ef8ae64fSPavel Begunkov io_rsrc_node_ref_zero(node);
9173572984SJens Axboe }
9273572984SJens Axboe
io_req_put_rsrc_locked(struct io_kiocb * req,struct io_ring_ctx * ctx)9373572984SJens Axboe static inline void io_req_put_rsrc_locked(struct io_kiocb *req,
9473572984SJens Axboe struct io_ring_ctx *ctx)
9573572984SJens Axboe {
961f2c8f61SPavel Begunkov io_put_rsrc_node(ctx, req->rsrc_node);
9773572984SJens Axboe }
9873572984SJens Axboe
io_charge_rsrc_node(struct io_ring_ctx * ctx,struct io_rsrc_node * node)998e15c0e7SPavel Begunkov static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx,
1008e15c0e7SPavel Begunkov struct io_rsrc_node *node)
10168ef5578SPavel Begunkov {
102ef8ae64fSPavel Begunkov node->refs++;
10368ef5578SPavel Begunkov }
10468ef5578SPavel Begunkov
io_req_set_rsrc_node(struct io_kiocb * req,struct io_ring_ctx * ctx,unsigned int issue_flags)10573572984SJens Axboe static inline void io_req_set_rsrc_node(struct io_kiocb *req,
10673572984SJens Axboe struct io_ring_ctx *ctx,
10773572984SJens Axboe unsigned int issue_flags)
10873572984SJens Axboe {
10973572984SJens Axboe if (!req->rsrc_node) {
1104ff0b50dSPavel Begunkov io_ring_submit_lock(ctx, issue_flags);
11173572984SJens Axboe
11273572984SJens Axboe lockdep_assert_held(&ctx->uring_lock);
11368ef5578SPavel Begunkov
1144ff0b50dSPavel Begunkov req->rsrc_node = ctx->rsrc_node;
1158e15c0e7SPavel Begunkov io_charge_rsrc_node(ctx, ctx->rsrc_node);
1164ff0b50dSPavel Begunkov io_ring_submit_unlock(ctx, issue_flags);
11773572984SJens Axboe }
11873572984SJens Axboe }
11973572984SJens Axboe
io_get_tag_slot(struct io_rsrc_data * data,unsigned int idx)12073572984SJens Axboe static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
12173572984SJens Axboe {
12273572984SJens Axboe unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK;
12373572984SJens Axboe unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT;
12473572984SJens Axboe
12573572984SJens Axboe return &data->tags[table_idx][off];
12673572984SJens Axboe }
12773572984SJens Axboe
io_rsrc_init(struct io_ring_ctx * ctx)1282933ae6eSPavel Begunkov static inline int io_rsrc_init(struct io_ring_ctx *ctx)
1292933ae6eSPavel Begunkov {
1302933ae6eSPavel Begunkov ctx->rsrc_node = io_rsrc_node_alloc(ctx);
1312933ae6eSPavel Begunkov return ctx->rsrc_node ? 0 : -ENOMEM;
1322933ae6eSPavel Begunkov }
1332933ae6eSPavel Begunkov
134d9808cebSPavel Begunkov int io_files_update(struct io_kiocb *req, unsigned int issue_flags);
135d9808cebSPavel Begunkov int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
1366a9ce66fSPavel Begunkov
1376a9ce66fSPavel Begunkov int __io_account_mem(struct user_struct *user, unsigned long nr_pages);
1386a9ce66fSPavel Begunkov
__io_unaccount_mem(struct user_struct * user,unsigned long nr_pages)1396a9ce66fSPavel Begunkov static inline void __io_unaccount_mem(struct user_struct *user,
1406a9ce66fSPavel Begunkov unsigned long nr_pages)
1416a9ce66fSPavel Begunkov {
1426a9ce66fSPavel Begunkov atomic_long_sub(nr_pages, &user->locked_vm);
1436a9ce66fSPavel Begunkov }
1446a9ce66fSPavel Begunkov
14573572984SJens Axboe #endif
146