1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/mm.h> 7 #include <linux/slab.h> 8 #include <linux/namei.h> 9 #include <linux/nospec.h> 10 #include <linux/io_uring.h> 11 12 #include <uapi/linux/io_uring.h> 13 14 #include "io_uring.h" 15 #include "tctx.h" 16 #include "poll.h" 17 #include "timeout.h" 18 #include "cancel.h" 19 20 struct io_cancel { 21 struct file *file; 22 u64 addr; 23 u32 flags; 24 s32 fd; 25 u8 opcode; 26 }; 27 28 #define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \ 29 IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED | \ 30 IORING_ASYNC_CANCEL_USERDATA | IORING_ASYNC_CANCEL_OP) 31 32 /* 33 * Returns true if the request matches the criteria outlined by 'cd'. 34 */ 35 bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd) 36 { 37 bool match_user_data = cd->flags & IORING_ASYNC_CANCEL_USERDATA; 38 39 if (req->ctx != cd->ctx) 40 return false; 41 42 if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP))) 43 match_user_data = true; 44 45 if (cd->flags & IORING_ASYNC_CANCEL_ANY) 46 goto check_seq; 47 if (cd->flags & IORING_ASYNC_CANCEL_FD) { 48 if (req->file != cd->file) 49 return false; 50 } 51 if (cd->flags & IORING_ASYNC_CANCEL_OP) { 52 if (req->opcode != cd->opcode) 53 return false; 54 } 55 if (match_user_data && req->cqe.user_data != cd->data) 56 return false; 57 if (cd->flags & IORING_ASYNC_CANCEL_ALL) { 58 check_seq: 59 if (cd->seq == req->work.cancel_seq) 60 return false; 61 req->work.cancel_seq = cd->seq; 62 } 63 64 return true; 65 } 66 67 static bool io_cancel_cb(struct io_wq_work *work, void *data) 68 { 69 struct io_kiocb *req = container_of(work, struct io_kiocb, work); 70 struct io_cancel_data *cd = data; 71 72 return io_cancel_req_match(req, cd); 73 } 74 75 static int io_async_cancel_one(struct io_uring_task *tctx, 76 struct io_cancel_data *cd) 77 { 78 enum io_wq_cancel cancel_ret; 79 int ret = 0; 80 bool all; 81 82 if (!tctx || !tctx->io_wq) 83 return -ENOENT; 84 85 all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY); 86 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, cd, all); 87 switch (cancel_ret) { 88 case IO_WQ_CANCEL_OK: 89 ret = 0; 90 break; 91 case IO_WQ_CANCEL_RUNNING: 92 ret = -EALREADY; 93 break; 94 case IO_WQ_CANCEL_NOTFOUND: 95 ret = -ENOENT; 96 break; 97 } 98 99 return ret; 100 } 101 102 int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, 103 unsigned issue_flags) 104 { 105 struct io_ring_ctx *ctx = cd->ctx; 106 int ret; 107 108 WARN_ON_ONCE(!io_wq_current_is_worker() && tctx != current->io_uring); 109 110 ret = io_async_cancel_one(tctx, cd); 111 /* 112 * Fall-through even for -EALREADY, as we may have poll armed 113 * that need unarming. 114 */ 115 if (!ret) 116 return 0; 117 118 ret = io_poll_cancel(ctx, cd, issue_flags); 119 if (ret != -ENOENT) 120 return ret; 121 122 spin_lock(&ctx->completion_lock); 123 if (!(cd->flags & IORING_ASYNC_CANCEL_FD)) 124 ret = io_timeout_cancel(ctx, cd); 125 spin_unlock(&ctx->completion_lock); 126 return ret; 127 } 128 129 int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 130 { 131 struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel); 132 133 if (unlikely(req->flags & REQ_F_BUFFER_SELECT)) 134 return -EINVAL; 135 if (sqe->off || sqe->splice_fd_in) 136 return -EINVAL; 137 138 cancel->addr = READ_ONCE(sqe->addr); 139 cancel->flags = READ_ONCE(sqe->cancel_flags); 140 if (cancel->flags & ~CANCEL_FLAGS) 141 return -EINVAL; 142 if (cancel->flags & IORING_ASYNC_CANCEL_FD) { 143 if (cancel->flags & IORING_ASYNC_CANCEL_ANY) 144 return -EINVAL; 145 cancel->fd = READ_ONCE(sqe->fd); 146 } 147 if (cancel->flags & IORING_ASYNC_CANCEL_OP) { 148 if (cancel->flags & IORING_ASYNC_CANCEL_ANY) 149 return -EINVAL; 150 cancel->opcode = READ_ONCE(sqe->len); 151 } 152 153 return 0; 154 } 155 156 static int __io_async_cancel(struct io_cancel_data *cd, 157 struct io_uring_task *tctx, 158 unsigned int issue_flags) 159 { 160 bool all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY); 161 struct io_ring_ctx *ctx = cd->ctx; 162 struct io_tctx_node *node; 163 int ret, nr = 0; 164 165 do { 166 ret = io_try_cancel(tctx, cd, issue_flags); 167 if (ret == -ENOENT) 168 break; 169 if (!all) 170 return ret; 171 nr++; 172 } while (1); 173 174 /* slow path, try all io-wq's */ 175 io_ring_submit_lock(ctx, issue_flags); 176 ret = -ENOENT; 177 list_for_each_entry(node, &ctx->tctx_list, ctx_node) { 178 struct io_uring_task *tctx = node->task->io_uring; 179 180 ret = io_async_cancel_one(tctx, cd); 181 if (ret != -ENOENT) { 182 if (!all) 183 break; 184 nr++; 185 } 186 } 187 io_ring_submit_unlock(ctx, issue_flags); 188 return all ? nr : ret; 189 } 190 191 int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) 192 { 193 struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel); 194 struct io_cancel_data cd = { 195 .ctx = req->ctx, 196 .data = cancel->addr, 197 .flags = cancel->flags, 198 .opcode = cancel->opcode, 199 .seq = atomic_inc_return(&req->ctx->cancel_seq), 200 }; 201 struct io_uring_task *tctx = req->task->io_uring; 202 int ret; 203 204 if (cd.flags & IORING_ASYNC_CANCEL_FD) { 205 if (req->flags & REQ_F_FIXED_FILE || 206 cd.flags & IORING_ASYNC_CANCEL_FD_FIXED) { 207 req->flags |= REQ_F_FIXED_FILE; 208 req->file = io_file_get_fixed(req, cancel->fd, 209 issue_flags); 210 } else { 211 req->file = io_file_get_normal(req, cancel->fd); 212 } 213 if (!req->file) { 214 ret = -EBADF; 215 goto done; 216 } 217 cd.file = req->file; 218 } 219 220 ret = __io_async_cancel(&cd, tctx, issue_flags); 221 done: 222 if (ret < 0) 223 req_set_fail(req); 224 io_req_set_res(req, ret, 0); 225 return IOU_OK; 226 } 227 228 void init_hash_table(struct io_hash_table *table, unsigned size) 229 { 230 unsigned int i; 231 232 for (i = 0; i < size; i++) { 233 spin_lock_init(&table->hbs[i].lock); 234 INIT_HLIST_HEAD(&table->hbs[i].list); 235 } 236 } 237 238 static int __io_sync_cancel(struct io_uring_task *tctx, 239 struct io_cancel_data *cd, int fd) 240 { 241 struct io_ring_ctx *ctx = cd->ctx; 242 243 /* fixed must be grabbed every time since we drop the uring_lock */ 244 if ((cd->flags & IORING_ASYNC_CANCEL_FD) && 245 (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) { 246 if (unlikely(fd >= ctx->nr_user_files)) 247 return -EBADF; 248 fd = array_index_nospec(fd, ctx->nr_user_files); 249 cd->file = io_file_from_index(&ctx->file_table, fd); 250 if (!cd->file) 251 return -EBADF; 252 } 253 254 return __io_async_cancel(cd, tctx, 0); 255 } 256 257 int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg) 258 __must_hold(&ctx->uring_lock) 259 { 260 struct io_cancel_data cd = { 261 .ctx = ctx, 262 .seq = atomic_inc_return(&ctx->cancel_seq), 263 }; 264 ktime_t timeout = KTIME_MAX; 265 struct io_uring_sync_cancel_reg sc; 266 struct fd f = { }; 267 DEFINE_WAIT(wait); 268 int ret, i; 269 270 if (copy_from_user(&sc, arg, sizeof(sc))) 271 return -EFAULT; 272 if (sc.flags & ~CANCEL_FLAGS) 273 return -EINVAL; 274 for (i = 0; i < ARRAY_SIZE(sc.pad); i++) 275 if (sc.pad[i]) 276 return -EINVAL; 277 for (i = 0; i < ARRAY_SIZE(sc.pad2); i++) 278 if (sc.pad2[i]) 279 return -EINVAL; 280 281 cd.data = sc.addr; 282 cd.flags = sc.flags; 283 cd.opcode = sc.opcode; 284 285 /* we can grab a normal file descriptor upfront */ 286 if ((cd.flags & IORING_ASYNC_CANCEL_FD) && 287 !(cd.flags & IORING_ASYNC_CANCEL_FD_FIXED)) { 288 f = fdget(sc.fd); 289 if (!f.file) 290 return -EBADF; 291 cd.file = f.file; 292 } 293 294 ret = __io_sync_cancel(current->io_uring, &cd, sc.fd); 295 296 /* found something, done! */ 297 if (ret != -EALREADY) 298 goto out; 299 300 if (sc.timeout.tv_sec != -1UL || sc.timeout.tv_nsec != -1UL) { 301 struct timespec64 ts = { 302 .tv_sec = sc.timeout.tv_sec, 303 .tv_nsec = sc.timeout.tv_nsec 304 }; 305 306 timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns()); 307 } 308 309 /* 310 * Keep looking until we get -ENOENT. we'll get woken everytime 311 * every time a request completes and will retry the cancelation. 312 */ 313 do { 314 cd.seq = atomic_inc_return(&ctx->cancel_seq); 315 316 prepare_to_wait(&ctx->cq_wait, &wait, TASK_INTERRUPTIBLE); 317 318 ret = __io_sync_cancel(current->io_uring, &cd, sc.fd); 319 320 mutex_unlock(&ctx->uring_lock); 321 if (ret != -EALREADY) 322 break; 323 324 ret = io_run_task_work_sig(ctx); 325 if (ret < 0) 326 break; 327 ret = schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS); 328 if (!ret) { 329 ret = -ETIME; 330 break; 331 } 332 mutex_lock(&ctx->uring_lock); 333 } while (1); 334 335 finish_wait(&ctx->cq_wait, &wait); 336 mutex_lock(&ctx->uring_lock); 337 338 if (ret == -ENOENT || ret > 0) 339 ret = 0; 340 out: 341 fdput(f); 342 return ret; 343 } 344