1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/io_uring.h> 6 7 #include <trace/events/io_uring.h> 8 9 #include <uapi/linux/io_uring.h> 10 11 #include "io_uring.h" 12 #include "refs.h" 13 #include "cancel.h" 14 #include "timeout.h" 15 16 struct io_timeout { 17 struct file *file; 18 u32 off; 19 u32 target_seq; 20 u32 repeats; 21 struct list_head list; 22 /* head of the link, used by linked timeouts only */ 23 struct io_kiocb *head; 24 /* for linked completions */ 25 struct io_kiocb *prev; 26 }; 27 28 struct io_timeout_rem { 29 struct file *file; 30 u64 addr; 31 32 /* timeout update */ 33 struct timespec64 ts; 34 u32 flags; 35 bool ltimeout; 36 }; 37 38 static inline bool io_is_timeout_noseq(struct io_kiocb *req) 39 { 40 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 41 struct io_timeout_data *data = req->async_data; 42 43 return !timeout->off || data->flags & IORING_TIMEOUT_MULTISHOT; 44 } 45 46 static inline void io_put_req(struct io_kiocb *req) 47 { 48 if (req_ref_put_and_test(req)) { 49 io_queue_next(req); 50 io_free_req(req); 51 } 52 } 53 54 static inline bool io_timeout_finish(struct io_timeout *timeout, 55 struct io_timeout_data *data) 56 { 57 if (!(data->flags & IORING_TIMEOUT_MULTISHOT)) 58 return true; 59 60 if (!timeout->off || (timeout->repeats && --timeout->repeats)) 61 return false; 62 63 return true; 64 } 65 66 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer); 67 68 static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts) 69 { 70 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 71 struct io_timeout_data *data = req->async_data; 72 struct io_ring_ctx *ctx = req->ctx; 73 74 if (!io_timeout_finish(timeout, data)) { 75 bool filled; 76 filled = io_fill_cqe_req_aux(req, ts->locked, -ETIME, 77 IORING_CQE_F_MORE); 78 if (filled) { 79 /* re-arm timer */ 80 spin_lock_irq(&ctx->timeout_lock); 81 list_add(&timeout->list, ctx->timeout_list.prev); 82 data->timer.function = io_timeout_fn; 83 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); 84 spin_unlock_irq(&ctx->timeout_lock); 85 return; 86 } 87 } 88 89 io_req_task_complete(req, ts); 90 } 91 92 static bool io_kill_timeout(struct io_kiocb *req, int status) 93 __must_hold(&req->ctx->timeout_lock) 94 { 95 struct io_timeout_data *io = req->async_data; 96 97 if (hrtimer_try_to_cancel(&io->timer) != -1) { 98 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 99 100 if (status) 101 req_set_fail(req); 102 atomic_set(&req->ctx->cq_timeouts, 103 atomic_read(&req->ctx->cq_timeouts) + 1); 104 list_del_init(&timeout->list); 105 io_req_queue_tw_complete(req, status); 106 return true; 107 } 108 return false; 109 } 110 111 __cold void io_flush_timeouts(struct io_ring_ctx *ctx) 112 { 113 u32 seq; 114 struct io_timeout *timeout, *tmp; 115 116 spin_lock_irq(&ctx->timeout_lock); 117 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); 118 119 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 120 struct io_kiocb *req = cmd_to_io_kiocb(timeout); 121 u32 events_needed, events_got; 122 123 if (io_is_timeout_noseq(req)) 124 break; 125 126 /* 127 * Since seq can easily wrap around over time, subtract 128 * the last seq at which timeouts were flushed before comparing. 129 * Assuming not more than 2^31-1 events have happened since, 130 * these subtractions won't have wrapped, so we can check if 131 * target is in [last_seq, current_seq] by comparing the two. 132 */ 133 events_needed = timeout->target_seq - ctx->cq_last_tm_flush; 134 events_got = seq - ctx->cq_last_tm_flush; 135 if (events_got < events_needed) 136 break; 137 138 io_kill_timeout(req, 0); 139 } 140 ctx->cq_last_tm_flush = seq; 141 spin_unlock_irq(&ctx->timeout_lock); 142 } 143 144 static void io_req_tw_fail_links(struct io_kiocb *link, struct io_tw_state *ts) 145 { 146 io_tw_lock(link->ctx, ts); 147 while (link) { 148 struct io_kiocb *nxt = link->link; 149 long res = -ECANCELED; 150 151 if (link->flags & REQ_F_FAIL) 152 res = link->cqe.res; 153 link->link = NULL; 154 io_req_set_res(link, res, 0); 155 io_req_task_complete(link, ts); 156 link = nxt; 157 } 158 } 159 160 static void io_fail_links(struct io_kiocb *req) 161 __must_hold(&req->ctx->completion_lock) 162 { 163 struct io_kiocb *link = req->link; 164 bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES; 165 166 if (!link) 167 return; 168 169 while (link) { 170 if (ignore_cqes) 171 link->flags |= REQ_F_CQE_SKIP; 172 else 173 link->flags &= ~REQ_F_CQE_SKIP; 174 trace_io_uring_fail_link(req, link); 175 link = link->link; 176 } 177 178 link = req->link; 179 link->io_task_work.func = io_req_tw_fail_links; 180 io_req_task_work_add(link); 181 req->link = NULL; 182 } 183 184 static inline void io_remove_next_linked(struct io_kiocb *req) 185 { 186 struct io_kiocb *nxt = req->link; 187 188 req->link = nxt->link; 189 nxt->link = NULL; 190 } 191 192 void io_disarm_next(struct io_kiocb *req) 193 __must_hold(&req->ctx->completion_lock) 194 { 195 struct io_kiocb *link = NULL; 196 197 if (req->flags & REQ_F_ARM_LTIMEOUT) { 198 link = req->link; 199 req->flags &= ~REQ_F_ARM_LTIMEOUT; 200 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { 201 io_remove_next_linked(req); 202 io_req_queue_tw_complete(link, -ECANCELED); 203 } 204 } else if (req->flags & REQ_F_LINK_TIMEOUT) { 205 struct io_ring_ctx *ctx = req->ctx; 206 207 spin_lock_irq(&ctx->timeout_lock); 208 link = io_disarm_linked_timeout(req); 209 spin_unlock_irq(&ctx->timeout_lock); 210 if (link) 211 io_req_queue_tw_complete(link, -ECANCELED); 212 } 213 if (unlikely((req->flags & REQ_F_FAIL) && 214 !(req->flags & REQ_F_HARDLINK))) 215 io_fail_links(req); 216 } 217 218 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, 219 struct io_kiocb *link) 220 __must_hold(&req->ctx->completion_lock) 221 __must_hold(&req->ctx->timeout_lock) 222 { 223 struct io_timeout_data *io = link->async_data; 224 struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout); 225 226 io_remove_next_linked(req); 227 timeout->head = NULL; 228 if (hrtimer_try_to_cancel(&io->timer) != -1) { 229 list_del(&timeout->list); 230 return link; 231 } 232 233 return NULL; 234 } 235 236 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) 237 { 238 struct io_timeout_data *data = container_of(timer, 239 struct io_timeout_data, timer); 240 struct io_kiocb *req = data->req; 241 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 242 struct io_ring_ctx *ctx = req->ctx; 243 unsigned long flags; 244 245 spin_lock_irqsave(&ctx->timeout_lock, flags); 246 list_del_init(&timeout->list); 247 atomic_set(&req->ctx->cq_timeouts, 248 atomic_read(&req->ctx->cq_timeouts) + 1); 249 spin_unlock_irqrestore(&ctx->timeout_lock, flags); 250 251 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS)) 252 req_set_fail(req); 253 254 io_req_set_res(req, -ETIME, 0); 255 req->io_task_work.func = io_timeout_complete; 256 io_req_task_work_add(req); 257 return HRTIMER_NORESTART; 258 } 259 260 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, 261 struct io_cancel_data *cd) 262 __must_hold(&ctx->timeout_lock) 263 { 264 struct io_timeout *timeout; 265 struct io_timeout_data *io; 266 struct io_kiocb *req = NULL; 267 268 list_for_each_entry(timeout, &ctx->timeout_list, list) { 269 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 270 271 if (io_cancel_req_match(tmp, cd)) { 272 req = tmp; 273 break; 274 } 275 } 276 if (!req) 277 return ERR_PTR(-ENOENT); 278 279 io = req->async_data; 280 if (hrtimer_try_to_cancel(&io->timer) == -1) 281 return ERR_PTR(-EALREADY); 282 timeout = io_kiocb_to_cmd(req, struct io_timeout); 283 list_del_init(&timeout->list); 284 return req; 285 } 286 287 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) 288 __must_hold(&ctx->completion_lock) 289 { 290 struct io_kiocb *req; 291 292 spin_lock_irq(&ctx->timeout_lock); 293 req = io_timeout_extract(ctx, cd); 294 spin_unlock_irq(&ctx->timeout_lock); 295 296 if (IS_ERR(req)) 297 return PTR_ERR(req); 298 io_req_task_queue_fail(req, -ECANCELED); 299 return 0; 300 } 301 302 static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts) 303 { 304 unsigned issue_flags = ts->locked ? 0 : IO_URING_F_UNLOCKED; 305 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 306 struct io_kiocb *prev = timeout->prev; 307 int ret = -ENOENT; 308 309 if (prev) { 310 if (!(req->task->flags & PF_EXITING)) { 311 struct io_cancel_data cd = { 312 .ctx = req->ctx, 313 .data = prev->cqe.user_data, 314 }; 315 316 ret = io_try_cancel(req->task->io_uring, &cd, issue_flags); 317 } 318 io_req_set_res(req, ret ?: -ETIME, 0); 319 io_req_task_complete(req, ts); 320 io_put_req(prev); 321 } else { 322 io_req_set_res(req, -ETIME, 0); 323 io_req_task_complete(req, ts); 324 } 325 } 326 327 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) 328 { 329 struct io_timeout_data *data = container_of(timer, 330 struct io_timeout_data, timer); 331 struct io_kiocb *prev, *req = data->req; 332 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 333 struct io_ring_ctx *ctx = req->ctx; 334 unsigned long flags; 335 336 spin_lock_irqsave(&ctx->timeout_lock, flags); 337 prev = timeout->head; 338 timeout->head = NULL; 339 340 /* 341 * We don't expect the list to be empty, that will only happen if we 342 * race with the completion of the linked work. 343 */ 344 if (prev) { 345 io_remove_next_linked(prev); 346 if (!req_ref_inc_not_zero(prev)) 347 prev = NULL; 348 } 349 list_del(&timeout->list); 350 timeout->prev = prev; 351 spin_unlock_irqrestore(&ctx->timeout_lock, flags); 352 353 req->io_task_work.func = io_req_task_link_timeout; 354 io_req_task_work_add(req); 355 return HRTIMER_NORESTART; 356 } 357 358 static clockid_t io_timeout_get_clock(struct io_timeout_data *data) 359 { 360 switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) { 361 case IORING_TIMEOUT_BOOTTIME: 362 return CLOCK_BOOTTIME; 363 case IORING_TIMEOUT_REALTIME: 364 return CLOCK_REALTIME; 365 default: 366 /* can't happen, vetted at prep time */ 367 WARN_ON_ONCE(1); 368 fallthrough; 369 case 0: 370 return CLOCK_MONOTONIC; 371 } 372 } 373 374 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 375 struct timespec64 *ts, enum hrtimer_mode mode) 376 __must_hold(&ctx->timeout_lock) 377 { 378 struct io_timeout_data *io; 379 struct io_timeout *timeout; 380 struct io_kiocb *req = NULL; 381 382 list_for_each_entry(timeout, &ctx->ltimeout_list, list) { 383 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); 384 385 if (user_data == tmp->cqe.user_data) { 386 req = tmp; 387 break; 388 } 389 } 390 if (!req) 391 return -ENOENT; 392 393 io = req->async_data; 394 if (hrtimer_try_to_cancel(&io->timer) == -1) 395 return -EALREADY; 396 hrtimer_init(&io->timer, io_timeout_get_clock(io), mode); 397 io->timer.function = io_link_timeout_fn; 398 hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode); 399 return 0; 400 } 401 402 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, 403 struct timespec64 *ts, enum hrtimer_mode mode) 404 __must_hold(&ctx->timeout_lock) 405 { 406 struct io_cancel_data cd = { .ctx = ctx, .data = user_data, }; 407 struct io_kiocb *req = io_timeout_extract(ctx, &cd); 408 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 409 struct io_timeout_data *data; 410 411 if (IS_ERR(req)) 412 return PTR_ERR(req); 413 414 timeout->off = 0; /* noseq */ 415 data = req->async_data; 416 list_add_tail(&timeout->list, &ctx->timeout_list); 417 hrtimer_init(&data->timer, io_timeout_get_clock(data), mode); 418 data->timer.function = io_timeout_fn; 419 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode); 420 return 0; 421 } 422 423 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 424 { 425 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); 426 427 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) 428 return -EINVAL; 429 if (sqe->buf_index || sqe->len || sqe->splice_fd_in) 430 return -EINVAL; 431 432 tr->ltimeout = false; 433 tr->addr = READ_ONCE(sqe->addr); 434 tr->flags = READ_ONCE(sqe->timeout_flags); 435 if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) { 436 if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 437 return -EINVAL; 438 if (tr->flags & IORING_LINK_TIMEOUT_UPDATE) 439 tr->ltimeout = true; 440 if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS)) 441 return -EINVAL; 442 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2))) 443 return -EFAULT; 444 if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0) 445 return -EINVAL; 446 } else if (tr->flags) { 447 /* timeout removal doesn't support flags */ 448 return -EINVAL; 449 } 450 451 return 0; 452 } 453 454 static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) 455 { 456 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS 457 : HRTIMER_MODE_REL; 458 } 459 460 /* 461 * Remove or update an existing timeout command 462 */ 463 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) 464 { 465 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); 466 struct io_ring_ctx *ctx = req->ctx; 467 int ret; 468 469 if (!(tr->flags & IORING_TIMEOUT_UPDATE)) { 470 struct io_cancel_data cd = { .ctx = ctx, .data = tr->addr, }; 471 472 spin_lock(&ctx->completion_lock); 473 ret = io_timeout_cancel(ctx, &cd); 474 spin_unlock(&ctx->completion_lock); 475 } else { 476 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags); 477 478 spin_lock_irq(&ctx->timeout_lock); 479 if (tr->ltimeout) 480 ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode); 481 else 482 ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode); 483 spin_unlock_irq(&ctx->timeout_lock); 484 } 485 486 if (ret < 0) 487 req_set_fail(req); 488 io_req_set_res(req, ret, 0); 489 return IOU_OK; 490 } 491 492 static int __io_timeout_prep(struct io_kiocb *req, 493 const struct io_uring_sqe *sqe, 494 bool is_timeout_link) 495 { 496 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 497 struct io_timeout_data *data; 498 unsigned flags; 499 u32 off = READ_ONCE(sqe->off); 500 501 if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in) 502 return -EINVAL; 503 if (off && is_timeout_link) 504 return -EINVAL; 505 flags = READ_ONCE(sqe->timeout_flags); 506 if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK | 507 IORING_TIMEOUT_ETIME_SUCCESS | 508 IORING_TIMEOUT_MULTISHOT)) 509 return -EINVAL; 510 /* more than one clock specified is invalid, obviously */ 511 if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1) 512 return -EINVAL; 513 /* multishot requests only make sense with rel values */ 514 if (!(~flags & (IORING_TIMEOUT_MULTISHOT | IORING_TIMEOUT_ABS))) 515 return -EINVAL; 516 517 INIT_LIST_HEAD(&timeout->list); 518 timeout->off = off; 519 if (unlikely(off && !req->ctx->off_timeout_used)) 520 req->ctx->off_timeout_used = true; 521 /* 522 * for multishot reqs w/ fixed nr of repeats, repeats tracks the 523 * remaining nr 524 */ 525 timeout->repeats = 0; 526 if ((flags & IORING_TIMEOUT_MULTISHOT) && off > 0) 527 timeout->repeats = off; 528 529 if (WARN_ON_ONCE(req_has_async_data(req))) 530 return -EFAULT; 531 if (io_alloc_async_data(req)) 532 return -ENOMEM; 533 534 data = req->async_data; 535 data->req = req; 536 data->flags = flags; 537 538 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr))) 539 return -EFAULT; 540 541 if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0) 542 return -EINVAL; 543 544 INIT_LIST_HEAD(&timeout->list); 545 data->mode = io_translate_timeout_mode(flags); 546 hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode); 547 548 if (is_timeout_link) { 549 struct io_submit_link *link = &req->ctx->submit_state.link; 550 551 if (!link->head) 552 return -EINVAL; 553 if (link->last->opcode == IORING_OP_LINK_TIMEOUT) 554 return -EINVAL; 555 timeout->head = link->last; 556 link->last->flags |= REQ_F_ARM_LTIMEOUT; 557 } 558 return 0; 559 } 560 561 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 562 { 563 return __io_timeout_prep(req, sqe, false); 564 } 565 566 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 567 { 568 return __io_timeout_prep(req, sqe, true); 569 } 570 571 int io_timeout(struct io_kiocb *req, unsigned int issue_flags) 572 { 573 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 574 struct io_ring_ctx *ctx = req->ctx; 575 struct io_timeout_data *data = req->async_data; 576 struct list_head *entry; 577 u32 tail, off = timeout->off; 578 579 spin_lock_irq(&ctx->timeout_lock); 580 581 /* 582 * sqe->off holds how many events that need to occur for this 583 * timeout event to be satisfied. If it isn't set, then this is 584 * a pure timeout request, sequence isn't used. 585 */ 586 if (io_is_timeout_noseq(req)) { 587 entry = ctx->timeout_list.prev; 588 goto add; 589 } 590 591 tail = data_race(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts); 592 timeout->target_seq = tail + off; 593 594 /* Update the last seq here in case io_flush_timeouts() hasn't. 595 * This is safe because ->completion_lock is held, and submissions 596 * and completions are never mixed in the same ->completion_lock section. 597 */ 598 ctx->cq_last_tm_flush = tail; 599 600 /* 601 * Insertion sort, ensuring the first entry in the list is always 602 * the one we need first. 603 */ 604 list_for_each_prev(entry, &ctx->timeout_list) { 605 struct io_timeout *nextt = list_entry(entry, struct io_timeout, list); 606 struct io_kiocb *nxt = cmd_to_io_kiocb(nextt); 607 608 if (io_is_timeout_noseq(nxt)) 609 continue; 610 /* nxt.seq is behind @tail, otherwise would've been completed */ 611 if (off >= nextt->target_seq - tail) 612 break; 613 } 614 add: 615 list_add(&timeout->list, entry); 616 data->timer.function = io_timeout_fn; 617 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode); 618 spin_unlock_irq(&ctx->timeout_lock); 619 return IOU_ISSUE_SKIP_COMPLETE; 620 } 621 622 void io_queue_linked_timeout(struct io_kiocb *req) 623 { 624 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); 625 struct io_ring_ctx *ctx = req->ctx; 626 627 spin_lock_irq(&ctx->timeout_lock); 628 /* 629 * If the back reference is NULL, then our linked request finished 630 * before we got a chance to setup the timer 631 */ 632 if (timeout->head) { 633 struct io_timeout_data *data = req->async_data; 634 635 data->timer.function = io_link_timeout_fn; 636 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), 637 data->mode); 638 list_add_tail(&timeout->list, &ctx->ltimeout_list); 639 } 640 spin_unlock_irq(&ctx->timeout_lock); 641 /* drop submission reference */ 642 io_put_req(req); 643 } 644 645 static bool io_match_task(struct io_kiocb *head, struct task_struct *task, 646 bool cancel_all) 647 __must_hold(&req->ctx->timeout_lock) 648 { 649 struct io_kiocb *req; 650 651 if (task && head->task != task) 652 return false; 653 if (cancel_all) 654 return true; 655 656 io_for_each_link(req, head) { 657 if (req->flags & REQ_F_INFLIGHT) 658 return true; 659 } 660 return false; 661 } 662 663 /* Returns true if we found and killed one or more timeouts */ 664 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, 665 bool cancel_all) 666 { 667 struct io_timeout *timeout, *tmp; 668 int canceled = 0; 669 670 /* 671 * completion_lock is needed for io_match_task(). Take it before 672 * timeout_lockfirst to keep locking ordering. 673 */ 674 spin_lock(&ctx->completion_lock); 675 spin_lock_irq(&ctx->timeout_lock); 676 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { 677 struct io_kiocb *req = cmd_to_io_kiocb(timeout); 678 679 if (io_match_task(req, tsk, cancel_all) && 680 io_kill_timeout(req, -ECANCELED)) 681 canceled++; 682 } 683 spin_unlock_irq(&ctx->timeout_lock); 684 spin_unlock(&ctx->completion_lock); 685 return canceled != 0; 686 } 687