1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * io_uring opcode handling table 4 */ 5 #include <linux/kernel.h> 6 #include <linux/errno.h> 7 #include <linux/fs.h> 8 #include <linux/file.h> 9 #include <linux/io_uring.h> 10 11 #include "io_uring.h" 12 #include "opdef.h" 13 #include "refs.h" 14 #include "tctx.h" 15 #include "sqpoll.h" 16 #include "fdinfo.h" 17 #include "kbuf.h" 18 #include "rsrc.h" 19 20 #include "xattr.h" 21 #include "nop.h" 22 #include "fs.h" 23 #include "splice.h" 24 #include "sync.h" 25 #include "advise.h" 26 #include "openclose.h" 27 #include "uring_cmd.h" 28 #include "epoll.h" 29 #include "statx.h" 30 #include "net.h" 31 #include "msg_ring.h" 32 #include "timeout.h" 33 #include "poll.h" 34 #include "cancel.h" 35 #include "rw.h" 36 37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags) 38 { 39 WARN_ON_ONCE(1); 40 return -ECANCELED; 41 } 42 43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb, 44 const struct io_uring_sqe *sqe) 45 { 46 return -EOPNOTSUPP; 47 } 48 49 const struct io_issue_def io_issue_defs[] = { 50 [IORING_OP_NOP] = { 51 .audit_skip = 1, 52 .iopoll = 1, 53 .prep = io_nop_prep, 54 .issue = io_nop, 55 }, 56 [IORING_OP_READV] = { 57 .needs_file = 1, 58 .unbound_nonreg_file = 1, 59 .pollin = 1, 60 .buffer_select = 1, 61 .plug = 1, 62 .audit_skip = 1, 63 .ioprio = 1, 64 .iopoll = 1, 65 .iopoll_queue = 1, 66 .prep = io_prep_rw, 67 .issue = io_read, 68 }, 69 [IORING_OP_WRITEV] = { 70 .needs_file = 1, 71 .hash_reg_file = 1, 72 .unbound_nonreg_file = 1, 73 .pollout = 1, 74 .plug = 1, 75 .audit_skip = 1, 76 .ioprio = 1, 77 .iopoll = 1, 78 .iopoll_queue = 1, 79 .prep = io_prep_rw, 80 .issue = io_write, 81 }, 82 [IORING_OP_FSYNC] = { 83 .needs_file = 1, 84 .audit_skip = 1, 85 .prep = io_fsync_prep, 86 .issue = io_fsync, 87 }, 88 [IORING_OP_READ_FIXED] = { 89 .needs_file = 1, 90 .unbound_nonreg_file = 1, 91 .pollin = 1, 92 .plug = 1, 93 .audit_skip = 1, 94 .ioprio = 1, 95 .iopoll = 1, 96 .iopoll_queue = 1, 97 .prep = io_prep_rw, 98 .issue = io_read, 99 }, 100 [IORING_OP_WRITE_FIXED] = { 101 .needs_file = 1, 102 .hash_reg_file = 1, 103 .unbound_nonreg_file = 1, 104 .pollout = 1, 105 .plug = 1, 106 .audit_skip = 1, 107 .ioprio = 1, 108 .iopoll = 1, 109 .iopoll_queue = 1, 110 .prep = io_prep_rw, 111 .issue = io_write, 112 }, 113 [IORING_OP_POLL_ADD] = { 114 .needs_file = 1, 115 .unbound_nonreg_file = 1, 116 .audit_skip = 1, 117 .prep = io_poll_add_prep, 118 .issue = io_poll_add, 119 }, 120 [IORING_OP_POLL_REMOVE] = { 121 .audit_skip = 1, 122 .prep = io_poll_remove_prep, 123 .issue = io_poll_remove, 124 }, 125 [IORING_OP_SYNC_FILE_RANGE] = { 126 .needs_file = 1, 127 .audit_skip = 1, 128 .prep = io_sfr_prep, 129 .issue = io_sync_file_range, 130 }, 131 [IORING_OP_SENDMSG] = { 132 .needs_file = 1, 133 .unbound_nonreg_file = 1, 134 .pollout = 1, 135 .ioprio = 1, 136 .manual_alloc = 1, 137 #if defined(CONFIG_NET) 138 .prep = io_sendmsg_prep, 139 .issue = io_sendmsg, 140 #else 141 .prep = io_eopnotsupp_prep, 142 #endif 143 }, 144 [IORING_OP_RECVMSG] = { 145 .needs_file = 1, 146 .unbound_nonreg_file = 1, 147 .pollin = 1, 148 .buffer_select = 1, 149 .ioprio = 1, 150 .manual_alloc = 1, 151 #if defined(CONFIG_NET) 152 .prep = io_recvmsg_prep, 153 .issue = io_recvmsg, 154 #else 155 .prep = io_eopnotsupp_prep, 156 #endif 157 }, 158 [IORING_OP_TIMEOUT] = { 159 .audit_skip = 1, 160 .prep = io_timeout_prep, 161 .issue = io_timeout, 162 }, 163 [IORING_OP_TIMEOUT_REMOVE] = { 164 /* used by timeout updates' prep() */ 165 .audit_skip = 1, 166 .prep = io_timeout_remove_prep, 167 .issue = io_timeout_remove, 168 }, 169 [IORING_OP_ACCEPT] = { 170 .needs_file = 1, 171 .unbound_nonreg_file = 1, 172 .pollin = 1, 173 .poll_exclusive = 1, 174 .ioprio = 1, /* used for flags */ 175 #if defined(CONFIG_NET) 176 .prep = io_accept_prep, 177 .issue = io_accept, 178 #else 179 .prep = io_eopnotsupp_prep, 180 #endif 181 }, 182 [IORING_OP_ASYNC_CANCEL] = { 183 .audit_skip = 1, 184 .prep = io_async_cancel_prep, 185 .issue = io_async_cancel, 186 }, 187 [IORING_OP_LINK_TIMEOUT] = { 188 .audit_skip = 1, 189 .prep = io_link_timeout_prep, 190 .issue = io_no_issue, 191 }, 192 [IORING_OP_CONNECT] = { 193 .needs_file = 1, 194 .unbound_nonreg_file = 1, 195 .pollout = 1, 196 #if defined(CONFIG_NET) 197 .prep = io_connect_prep, 198 .issue = io_connect, 199 #else 200 .prep = io_eopnotsupp_prep, 201 #endif 202 }, 203 [IORING_OP_FALLOCATE] = { 204 .needs_file = 1, 205 .prep = io_fallocate_prep, 206 .issue = io_fallocate, 207 }, 208 [IORING_OP_OPENAT] = { 209 .prep = io_openat_prep, 210 .issue = io_openat, 211 }, 212 [IORING_OP_CLOSE] = { 213 .prep = io_close_prep, 214 .issue = io_close, 215 }, 216 [IORING_OP_FILES_UPDATE] = { 217 .audit_skip = 1, 218 .iopoll = 1, 219 .prep = io_files_update_prep, 220 .issue = io_files_update, 221 }, 222 [IORING_OP_STATX] = { 223 .audit_skip = 1, 224 .prep = io_statx_prep, 225 .issue = io_statx, 226 }, 227 [IORING_OP_READ] = { 228 .needs_file = 1, 229 .unbound_nonreg_file = 1, 230 .pollin = 1, 231 .buffer_select = 1, 232 .plug = 1, 233 .audit_skip = 1, 234 .ioprio = 1, 235 .iopoll = 1, 236 .iopoll_queue = 1, 237 .prep = io_prep_rw, 238 .issue = io_read, 239 }, 240 [IORING_OP_WRITE] = { 241 .needs_file = 1, 242 .hash_reg_file = 1, 243 .unbound_nonreg_file = 1, 244 .pollout = 1, 245 .plug = 1, 246 .audit_skip = 1, 247 .ioprio = 1, 248 .iopoll = 1, 249 .iopoll_queue = 1, 250 .prep = io_prep_rw, 251 .issue = io_write, 252 }, 253 [IORING_OP_FADVISE] = { 254 .needs_file = 1, 255 .audit_skip = 1, 256 .prep = io_fadvise_prep, 257 .issue = io_fadvise, 258 }, 259 [IORING_OP_MADVISE] = { 260 .audit_skip = 1, 261 .prep = io_madvise_prep, 262 .issue = io_madvise, 263 }, 264 [IORING_OP_SEND] = { 265 .needs_file = 1, 266 .unbound_nonreg_file = 1, 267 .pollout = 1, 268 .audit_skip = 1, 269 .ioprio = 1, 270 .manual_alloc = 1, 271 #if defined(CONFIG_NET) 272 .prep = io_sendmsg_prep, 273 .issue = io_send, 274 #else 275 .prep = io_eopnotsupp_prep, 276 #endif 277 }, 278 [IORING_OP_RECV] = { 279 .needs_file = 1, 280 .unbound_nonreg_file = 1, 281 .pollin = 1, 282 .buffer_select = 1, 283 .audit_skip = 1, 284 .ioprio = 1, 285 #if defined(CONFIG_NET) 286 .prep = io_recvmsg_prep, 287 .issue = io_recv, 288 #else 289 .prep = io_eopnotsupp_prep, 290 #endif 291 }, 292 [IORING_OP_OPENAT2] = { 293 .prep = io_openat2_prep, 294 .issue = io_openat2, 295 }, 296 [IORING_OP_EPOLL_CTL] = { 297 .unbound_nonreg_file = 1, 298 .audit_skip = 1, 299 #if defined(CONFIG_EPOLL) 300 .prep = io_epoll_ctl_prep, 301 .issue = io_epoll_ctl, 302 #else 303 .prep = io_eopnotsupp_prep, 304 #endif 305 }, 306 [IORING_OP_SPLICE] = { 307 .needs_file = 1, 308 .hash_reg_file = 1, 309 .unbound_nonreg_file = 1, 310 .audit_skip = 1, 311 .prep = io_splice_prep, 312 .issue = io_splice, 313 }, 314 [IORING_OP_PROVIDE_BUFFERS] = { 315 .audit_skip = 1, 316 .iopoll = 1, 317 .prep = io_provide_buffers_prep, 318 .issue = io_provide_buffers, 319 }, 320 [IORING_OP_REMOVE_BUFFERS] = { 321 .audit_skip = 1, 322 .iopoll = 1, 323 .prep = io_remove_buffers_prep, 324 .issue = io_remove_buffers, 325 }, 326 [IORING_OP_TEE] = { 327 .needs_file = 1, 328 .hash_reg_file = 1, 329 .unbound_nonreg_file = 1, 330 .audit_skip = 1, 331 .prep = io_tee_prep, 332 .issue = io_tee, 333 }, 334 [IORING_OP_SHUTDOWN] = { 335 .needs_file = 1, 336 #if defined(CONFIG_NET) 337 .prep = io_shutdown_prep, 338 .issue = io_shutdown, 339 #else 340 .prep = io_eopnotsupp_prep, 341 #endif 342 }, 343 [IORING_OP_RENAMEAT] = { 344 .prep = io_renameat_prep, 345 .issue = io_renameat, 346 }, 347 [IORING_OP_UNLINKAT] = { 348 .prep = io_unlinkat_prep, 349 .issue = io_unlinkat, 350 }, 351 [IORING_OP_MKDIRAT] = { 352 .prep = io_mkdirat_prep, 353 .issue = io_mkdirat, 354 }, 355 [IORING_OP_SYMLINKAT] = { 356 .prep = io_symlinkat_prep, 357 .issue = io_symlinkat, 358 }, 359 [IORING_OP_LINKAT] = { 360 .prep = io_linkat_prep, 361 .issue = io_linkat, 362 }, 363 [IORING_OP_MSG_RING] = { 364 .needs_file = 1, 365 .iopoll = 1, 366 .prep = io_msg_ring_prep, 367 .issue = io_msg_ring, 368 }, 369 [IORING_OP_FSETXATTR] = { 370 .needs_file = 1, 371 .prep = io_fsetxattr_prep, 372 .issue = io_fsetxattr, 373 }, 374 [IORING_OP_SETXATTR] = { 375 .prep = io_setxattr_prep, 376 .issue = io_setxattr, 377 }, 378 [IORING_OP_FGETXATTR] = { 379 .needs_file = 1, 380 .prep = io_fgetxattr_prep, 381 .issue = io_fgetxattr, 382 }, 383 [IORING_OP_GETXATTR] = { 384 .prep = io_getxattr_prep, 385 .issue = io_getxattr, 386 }, 387 [IORING_OP_SOCKET] = { 388 .audit_skip = 1, 389 #if defined(CONFIG_NET) 390 .prep = io_socket_prep, 391 .issue = io_socket, 392 #else 393 .prep = io_eopnotsupp_prep, 394 #endif 395 }, 396 [IORING_OP_URING_CMD] = { 397 .needs_file = 1, 398 .plug = 1, 399 .iopoll = 1, 400 .iopoll_queue = 1, 401 .prep = io_uring_cmd_prep, 402 .issue = io_uring_cmd, 403 }, 404 [IORING_OP_SEND_ZC] = { 405 .needs_file = 1, 406 .unbound_nonreg_file = 1, 407 .pollout = 1, 408 .audit_skip = 1, 409 .ioprio = 1, 410 .manual_alloc = 1, 411 #if defined(CONFIG_NET) 412 .prep = io_send_zc_prep, 413 .issue = io_send_zc, 414 #else 415 .prep = io_eopnotsupp_prep, 416 #endif 417 }, 418 [IORING_OP_SENDMSG_ZC] = { 419 .needs_file = 1, 420 .unbound_nonreg_file = 1, 421 .pollout = 1, 422 .ioprio = 1, 423 .manual_alloc = 1, 424 #if defined(CONFIG_NET) 425 .prep = io_send_zc_prep, 426 .issue = io_sendmsg_zc, 427 #else 428 .prep = io_eopnotsupp_prep, 429 #endif 430 }, 431 }; 432 433 434 const struct io_cold_def io_cold_defs[] = { 435 [IORING_OP_NOP] = { 436 .name = "NOP", 437 }, 438 [IORING_OP_READV] = { 439 .async_size = sizeof(struct io_async_rw), 440 .name = "READV", 441 .prep_async = io_readv_prep_async, 442 .cleanup = io_readv_writev_cleanup, 443 .fail = io_rw_fail, 444 }, 445 [IORING_OP_WRITEV] = { 446 .async_size = sizeof(struct io_async_rw), 447 .name = "WRITEV", 448 .prep_async = io_writev_prep_async, 449 .cleanup = io_readv_writev_cleanup, 450 .fail = io_rw_fail, 451 }, 452 [IORING_OP_FSYNC] = { 453 .name = "FSYNC", 454 }, 455 [IORING_OP_READ_FIXED] = { 456 .async_size = sizeof(struct io_async_rw), 457 .name = "READ_FIXED", 458 .fail = io_rw_fail, 459 }, 460 [IORING_OP_WRITE_FIXED] = { 461 .async_size = sizeof(struct io_async_rw), 462 .name = "WRITE_FIXED", 463 .fail = io_rw_fail, 464 }, 465 [IORING_OP_POLL_ADD] = { 466 .name = "POLL_ADD", 467 }, 468 [IORING_OP_POLL_REMOVE] = { 469 .name = "POLL_REMOVE", 470 }, 471 [IORING_OP_SYNC_FILE_RANGE] = { 472 .name = "SYNC_FILE_RANGE", 473 }, 474 [IORING_OP_SENDMSG] = { 475 .name = "SENDMSG", 476 #if defined(CONFIG_NET) 477 .async_size = sizeof(struct io_async_msghdr), 478 .prep_async = io_sendmsg_prep_async, 479 .cleanup = io_sendmsg_recvmsg_cleanup, 480 .fail = io_sendrecv_fail, 481 #endif 482 }, 483 [IORING_OP_RECVMSG] = { 484 .name = "RECVMSG", 485 #if defined(CONFIG_NET) 486 .async_size = sizeof(struct io_async_msghdr), 487 .prep_async = io_recvmsg_prep_async, 488 .cleanup = io_sendmsg_recvmsg_cleanup, 489 .fail = io_sendrecv_fail, 490 #endif 491 }, 492 [IORING_OP_TIMEOUT] = { 493 .async_size = sizeof(struct io_timeout_data), 494 .name = "TIMEOUT", 495 }, 496 [IORING_OP_TIMEOUT_REMOVE] = { 497 .name = "TIMEOUT_REMOVE", 498 }, 499 [IORING_OP_ACCEPT] = { 500 .name = "ACCEPT", 501 }, 502 [IORING_OP_ASYNC_CANCEL] = { 503 .name = "ASYNC_CANCEL", 504 }, 505 [IORING_OP_LINK_TIMEOUT] = { 506 .async_size = sizeof(struct io_timeout_data), 507 .name = "LINK_TIMEOUT", 508 }, 509 [IORING_OP_CONNECT] = { 510 .name = "CONNECT", 511 #if defined(CONFIG_NET) 512 .async_size = sizeof(struct io_async_connect), 513 .prep_async = io_connect_prep_async, 514 #endif 515 }, 516 [IORING_OP_FALLOCATE] = { 517 .name = "FALLOCATE", 518 }, 519 [IORING_OP_OPENAT] = { 520 .name = "OPENAT", 521 .cleanup = io_open_cleanup, 522 }, 523 [IORING_OP_CLOSE] = { 524 .name = "CLOSE", 525 }, 526 [IORING_OP_FILES_UPDATE] = { 527 .name = "FILES_UPDATE", 528 }, 529 [IORING_OP_STATX] = { 530 .name = "STATX", 531 .cleanup = io_statx_cleanup, 532 }, 533 [IORING_OP_READ] = { 534 .async_size = sizeof(struct io_async_rw), 535 .name = "READ", 536 .fail = io_rw_fail, 537 }, 538 [IORING_OP_WRITE] = { 539 .async_size = sizeof(struct io_async_rw), 540 .name = "WRITE", 541 .fail = io_rw_fail, 542 }, 543 [IORING_OP_FADVISE] = { 544 .name = "FADVISE", 545 }, 546 [IORING_OP_MADVISE] = { 547 .name = "MADVISE", 548 }, 549 [IORING_OP_SEND] = { 550 .name = "SEND", 551 #if defined(CONFIG_NET) 552 .async_size = sizeof(struct io_async_msghdr), 553 .fail = io_sendrecv_fail, 554 .prep_async = io_send_prep_async, 555 #endif 556 }, 557 [IORING_OP_RECV] = { 558 .name = "RECV", 559 #if defined(CONFIG_NET) 560 .fail = io_sendrecv_fail, 561 #endif 562 }, 563 [IORING_OP_OPENAT2] = { 564 .name = "OPENAT2", 565 .cleanup = io_open_cleanup, 566 }, 567 [IORING_OP_EPOLL_CTL] = { 568 .name = "EPOLL", 569 }, 570 [IORING_OP_SPLICE] = { 571 .name = "SPLICE", 572 }, 573 [IORING_OP_PROVIDE_BUFFERS] = { 574 .name = "PROVIDE_BUFFERS", 575 }, 576 [IORING_OP_REMOVE_BUFFERS] = { 577 .name = "REMOVE_BUFFERS", 578 }, 579 [IORING_OP_TEE] = { 580 .name = "TEE", 581 }, 582 [IORING_OP_SHUTDOWN] = { 583 .name = "SHUTDOWN", 584 }, 585 [IORING_OP_RENAMEAT] = { 586 .name = "RENAMEAT", 587 .cleanup = io_renameat_cleanup, 588 }, 589 [IORING_OP_UNLINKAT] = { 590 .name = "UNLINKAT", 591 .cleanup = io_unlinkat_cleanup, 592 }, 593 [IORING_OP_MKDIRAT] = { 594 .name = "MKDIRAT", 595 .cleanup = io_mkdirat_cleanup, 596 }, 597 [IORING_OP_SYMLINKAT] = { 598 .name = "SYMLINKAT", 599 .cleanup = io_link_cleanup, 600 }, 601 [IORING_OP_LINKAT] = { 602 .name = "LINKAT", 603 .cleanup = io_link_cleanup, 604 }, 605 [IORING_OP_MSG_RING] = { 606 .name = "MSG_RING", 607 .cleanup = io_msg_ring_cleanup, 608 }, 609 [IORING_OP_FSETXATTR] = { 610 .name = "FSETXATTR", 611 .cleanup = io_xattr_cleanup, 612 }, 613 [IORING_OP_SETXATTR] = { 614 .name = "SETXATTR", 615 .cleanup = io_xattr_cleanup, 616 }, 617 [IORING_OP_FGETXATTR] = { 618 .name = "FGETXATTR", 619 .cleanup = io_xattr_cleanup, 620 }, 621 [IORING_OP_GETXATTR] = { 622 .name = "GETXATTR", 623 .cleanup = io_xattr_cleanup, 624 }, 625 [IORING_OP_SOCKET] = { 626 .name = "SOCKET", 627 }, 628 [IORING_OP_URING_CMD] = { 629 .name = "URING_CMD", 630 .async_size = 2 * sizeof(struct io_uring_sqe), 631 .prep_async = io_uring_cmd_prep_async, 632 }, 633 [IORING_OP_SEND_ZC] = { 634 .name = "SEND_ZC", 635 #if defined(CONFIG_NET) 636 .async_size = sizeof(struct io_async_msghdr), 637 .prep_async = io_send_prep_async, 638 .cleanup = io_send_zc_cleanup, 639 .fail = io_sendrecv_fail, 640 #endif 641 }, 642 [IORING_OP_SENDMSG_ZC] = { 643 .name = "SENDMSG_ZC", 644 #if defined(CONFIG_NET) 645 .async_size = sizeof(struct io_async_msghdr), 646 .prep_async = io_sendmsg_prep_async, 647 .cleanup = io_send_zc_cleanup, 648 .fail = io_sendrecv_fail, 649 #endif 650 }, 651 }; 652 653 const char *io_uring_get_opcode(u8 opcode) 654 { 655 if (opcode < IORING_OP_LAST) 656 return io_cold_defs[opcode].name; 657 return "INVALID"; 658 } 659 660 void __init io_uring_optable_init(void) 661 { 662 int i; 663 664 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST); 665 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST); 666 667 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) { 668 BUG_ON(!io_issue_defs[i].prep); 669 if (io_issue_defs[i].prep != io_eopnotsupp_prep) 670 BUG_ON(!io_issue_defs[i].issue); 671 WARN_ON_ONCE(!io_cold_defs[i].name); 672 } 673 } 674