173572984SJens Axboe // SPDX-License-Identifier: GPL-2.0 273572984SJens Axboe #ifndef IOU_RSRC_H 373572984SJens Axboe #define IOU_RSRC_H 473572984SJens Axboe 573572984SJens Axboe #include <net/af_unix.h> 673572984SJens Axboe 79eae8655SPavel Begunkov #include "alloc_cache.h" 89eae8655SPavel Begunkov 969bbc6adSPavel Begunkov #define IO_NODE_ALLOC_CACHE_MAX 32 1069bbc6adSPavel Begunkov 1173572984SJens Axboe #define IO_RSRC_TAG_TABLE_SHIFT (PAGE_SHIFT - 3) 1273572984SJens Axboe #define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT) 1373572984SJens Axboe #define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1) 1473572984SJens Axboe 1573572984SJens Axboe enum { 1673572984SJens Axboe IORING_RSRC_FILE = 0, 1773572984SJens Axboe IORING_RSRC_BUFFER = 1, 1873572984SJens Axboe }; 1973572984SJens Axboe 2073572984SJens Axboe struct io_rsrc_put { 2173572984SJens Axboe u64 tag; 2273572984SJens Axboe union { 2373572984SJens Axboe void *rsrc; 2473572984SJens Axboe struct file *file; 2573572984SJens Axboe struct io_mapped_ubuf *buf; 2673572984SJens Axboe }; 2773572984SJens Axboe }; 2873572984SJens Axboe 2973572984SJens Axboe typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc); 3073572984SJens Axboe 3173572984SJens Axboe struct io_rsrc_data { 3273572984SJens Axboe struct io_ring_ctx *ctx; 3373572984SJens Axboe 3473572984SJens Axboe u64 **tags; 3573572984SJens Axboe unsigned int nr; 3673572984SJens Axboe rsrc_put_fn *do_put; 3773572984SJens Axboe bool quiesce; 3873572984SJens Axboe }; 3973572984SJens Axboe 4073572984SJens Axboe struct io_rsrc_node { 419eae8655SPavel Begunkov union { 429eae8655SPavel Begunkov struct io_cache_entry cache; 4373572984SJens Axboe struct io_rsrc_data *rsrc_data; 449eae8655SPavel Begunkov }; 45ef8ae64fSPavel Begunkov int refs; 46*26147da3SPavel Begunkov bool empty; 47c376644fSPavel Begunkov struct list_head node; 48c376644fSPavel Begunkov struct io_rsrc_put item; 4973572984SJens Axboe }; 5073572984SJens Axboe 51ad163a7eSJens Axboe struct io_mapped_ubuf { 52ad163a7eSJens Axboe u64 ubuf; 53ad163a7eSJens Axboe u64 ubuf_end; 54ad163a7eSJens Axboe unsigned int nr_bvecs; 55ad163a7eSJens Axboe unsigned long acct_pages; 56ad163a7eSJens Axboe struct bio_vec bvec[]; 57ad163a7eSJens Axboe }; 58ad163a7eSJens Axboe 59d34b1b0bSPavel Begunkov void io_rsrc_put_tw(struct callback_head *cb); 60b8fb5b4fSPavel Begunkov void io_rsrc_node_ref_zero(struct io_rsrc_node *node); 6173572984SJens Axboe void io_rsrc_put_work(struct work_struct *work); 629eae8655SPavel Begunkov void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node); 632933ae6eSPavel Begunkov struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx); 6463fea890SPavel Begunkov int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc); 6573572984SJens Axboe 66c059f785SPavel Begunkov int io_import_fixed(int ddir, struct iov_iter *iter, 67c059f785SPavel Begunkov struct io_mapped_ubuf *imu, 68c059f785SPavel Begunkov u64 buf_addr, size_t len); 6973572984SJens Axboe 7073572984SJens Axboe void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx); 7173572984SJens Axboe int io_sqe_buffers_unregister(struct io_ring_ctx *ctx); 7273572984SJens Axboe int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, 7373572984SJens Axboe unsigned int nr_args, u64 __user *tags); 7473572984SJens Axboe void __io_sqe_files_unregister(struct io_ring_ctx *ctx); 7573572984SJens Axboe int io_sqe_files_unregister(struct io_ring_ctx *ctx); 7673572984SJens Axboe int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, 7773572984SJens Axboe unsigned nr_args, u64 __user *tags); 7873572984SJens Axboe 7973572984SJens Axboe int __io_scm_file_account(struct io_ring_ctx *ctx, struct file *file); 8073572984SJens Axboe 8173572984SJens Axboe #if defined(CONFIG_UNIX) 8273572984SJens Axboe static inline bool io_file_need_scm(struct file *filp) 8373572984SJens Axboe { 8473572984SJens Axboe return !!unix_get_socket(filp); 8573572984SJens Axboe } 8673572984SJens Axboe #else 8773572984SJens Axboe static inline bool io_file_need_scm(struct file *filp) 8873572984SJens Axboe { 8973572984SJens Axboe return false; 9073572984SJens Axboe } 9173572984SJens Axboe #endif 9273572984SJens Axboe 9373572984SJens Axboe static inline int io_scm_file_account(struct io_ring_ctx *ctx, 9473572984SJens Axboe struct file *file) 9573572984SJens Axboe { 9673572984SJens Axboe if (likely(!io_file_need_scm(file))) 9773572984SJens Axboe return 0; 9873572984SJens Axboe return __io_scm_file_account(ctx, file); 9973572984SJens Axboe } 10073572984SJens Axboe 10173572984SJens Axboe int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg, 10273572984SJens Axboe unsigned nr_args); 10373572984SJens Axboe int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, 10473572984SJens Axboe unsigned size, unsigned type); 10573572984SJens Axboe int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, 10673572984SJens Axboe unsigned int size, unsigned int type); 10773572984SJens Axboe 1081f2c8f61SPavel Begunkov static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node) 10973572984SJens Axboe { 1101f2c8f61SPavel Begunkov lockdep_assert_held(&ctx->uring_lock); 1111f2c8f61SPavel Begunkov 112ef8ae64fSPavel Begunkov if (node && !--node->refs) 113ef8ae64fSPavel Begunkov io_rsrc_node_ref_zero(node); 11473572984SJens Axboe } 11573572984SJens Axboe 11673572984SJens Axboe static inline void io_req_put_rsrc_locked(struct io_kiocb *req, 11773572984SJens Axboe struct io_ring_ctx *ctx) 11873572984SJens Axboe { 1191f2c8f61SPavel Begunkov io_put_rsrc_node(ctx, req->rsrc_node); 12073572984SJens Axboe } 12173572984SJens Axboe 1228e15c0e7SPavel Begunkov static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx, 1238e15c0e7SPavel Begunkov struct io_rsrc_node *node) 12468ef5578SPavel Begunkov { 125ef8ae64fSPavel Begunkov node->refs++; 12668ef5578SPavel Begunkov } 12768ef5578SPavel Begunkov 12873572984SJens Axboe static inline void io_req_set_rsrc_node(struct io_kiocb *req, 12973572984SJens Axboe struct io_ring_ctx *ctx, 13073572984SJens Axboe unsigned int issue_flags) 13173572984SJens Axboe { 13273572984SJens Axboe if (!req->rsrc_node) { 1334ff0b50dSPavel Begunkov io_ring_submit_lock(ctx, issue_flags); 13473572984SJens Axboe 13573572984SJens Axboe lockdep_assert_held(&ctx->uring_lock); 13668ef5578SPavel Begunkov 1374ff0b50dSPavel Begunkov req->rsrc_node = ctx->rsrc_node; 1388e15c0e7SPavel Begunkov io_charge_rsrc_node(ctx, ctx->rsrc_node); 1394ff0b50dSPavel Begunkov io_ring_submit_unlock(ctx, issue_flags); 14073572984SJens Axboe } 14173572984SJens Axboe } 14273572984SJens Axboe 14373572984SJens Axboe static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx) 14473572984SJens Axboe { 14573572984SJens Axboe unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK; 14673572984SJens Axboe unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT; 14773572984SJens Axboe 14873572984SJens Axboe return &data->tags[table_idx][off]; 14973572984SJens Axboe } 15073572984SJens Axboe 1512933ae6eSPavel Begunkov static inline int io_rsrc_init(struct io_ring_ctx *ctx) 1522933ae6eSPavel Begunkov { 1532933ae6eSPavel Begunkov ctx->rsrc_node = io_rsrc_node_alloc(ctx); 1542933ae6eSPavel Begunkov return ctx->rsrc_node ? 0 : -ENOMEM; 1552933ae6eSPavel Begunkov } 1562933ae6eSPavel Begunkov 157d9808cebSPavel Begunkov int io_files_update(struct io_kiocb *req, unsigned int issue_flags); 158d9808cebSPavel Begunkov int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 1596a9ce66fSPavel Begunkov 1606a9ce66fSPavel Begunkov int __io_account_mem(struct user_struct *user, unsigned long nr_pages); 1616a9ce66fSPavel Begunkov 1626a9ce66fSPavel Begunkov static inline void __io_unaccount_mem(struct user_struct *user, 1636a9ce66fSPavel Begunkov unsigned long nr_pages) 1646a9ce66fSPavel Begunkov { 1656a9ce66fSPavel Begunkov atomic_long_sub(nr_pages, &user->locked_vm); 1666a9ce66fSPavel Begunkov } 1676a9ce66fSPavel Begunkov 16873572984SJens Axboe #endif 169