1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * io_uring opcode handling table 4 */ 5 #include <linux/kernel.h> 6 #include <linux/errno.h> 7 #include <linux/fs.h> 8 #include <linux/file.h> 9 #include <linux/io_uring.h> 10 11 #include "io_uring.h" 12 #include "opdef.h" 13 #include "refs.h" 14 #include "tctx.h" 15 #include "sqpoll.h" 16 #include "fdinfo.h" 17 #include "kbuf.h" 18 #include "rsrc.h" 19 20 #include "xattr.h" 21 #include "nop.h" 22 #include "fs.h" 23 #include "splice.h" 24 #include "sync.h" 25 #include "advise.h" 26 #include "openclose.h" 27 #include "uring_cmd.h" 28 #include "epoll.h" 29 #include "statx.h" 30 #include "net.h" 31 #include "msg_ring.h" 32 #include "timeout.h" 33 #include "poll.h" 34 #include "cancel.h" 35 #include "rw.h" 36 37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags) 38 { 39 WARN_ON_ONCE(1); 40 return -ECANCELED; 41 } 42 43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb, 44 const struct io_uring_sqe *sqe) 45 { 46 return -EOPNOTSUPP; 47 } 48 49 const struct io_op_def io_op_defs[] = { 50 [IORING_OP_NOP] = { 51 .audit_skip = 1, 52 .iopoll = 1, 53 .name = "NOP", 54 .prep = io_nop_prep, 55 .issue = io_nop, 56 }, 57 [IORING_OP_READV] = { 58 .needs_file = 1, 59 .unbound_nonreg_file = 1, 60 .pollin = 1, 61 .buffer_select = 1, 62 .plug = 1, 63 .audit_skip = 1, 64 .ioprio = 1, 65 .iopoll = 1, 66 .iopoll_queue = 1, 67 .async_size = sizeof(struct io_async_rw), 68 .name = "READV", 69 .prep = io_prep_rw, 70 .issue = io_read, 71 .prep_async = io_readv_prep_async, 72 .cleanup = io_readv_writev_cleanup, 73 .fail = io_rw_fail, 74 }, 75 [IORING_OP_WRITEV] = { 76 .needs_file = 1, 77 .hash_reg_file = 1, 78 .unbound_nonreg_file = 1, 79 .pollout = 1, 80 .plug = 1, 81 .audit_skip = 1, 82 .ioprio = 1, 83 .iopoll = 1, 84 .iopoll_queue = 1, 85 .async_size = sizeof(struct io_async_rw), 86 .name = "WRITEV", 87 .prep = io_prep_rw, 88 .issue = io_write, 89 .prep_async = io_writev_prep_async, 90 .cleanup = io_readv_writev_cleanup, 91 .fail = io_rw_fail, 92 }, 93 [IORING_OP_FSYNC] = { 94 .needs_file = 1, 95 .audit_skip = 1, 96 .name = "FSYNC", 97 .prep = io_fsync_prep, 98 .issue = io_fsync, 99 }, 100 [IORING_OP_READ_FIXED] = { 101 .needs_file = 1, 102 .unbound_nonreg_file = 1, 103 .pollin = 1, 104 .plug = 1, 105 .audit_skip = 1, 106 .ioprio = 1, 107 .iopoll = 1, 108 .iopoll_queue = 1, 109 .async_size = sizeof(struct io_async_rw), 110 .name = "READ_FIXED", 111 .prep = io_prep_rw, 112 .issue = io_read, 113 .fail = io_rw_fail, 114 }, 115 [IORING_OP_WRITE_FIXED] = { 116 .needs_file = 1, 117 .hash_reg_file = 1, 118 .unbound_nonreg_file = 1, 119 .pollout = 1, 120 .plug = 1, 121 .audit_skip = 1, 122 .ioprio = 1, 123 .iopoll = 1, 124 .iopoll_queue = 1, 125 .async_size = sizeof(struct io_async_rw), 126 .name = "WRITE_FIXED", 127 .prep = io_prep_rw, 128 .issue = io_write, 129 .fail = io_rw_fail, 130 }, 131 [IORING_OP_POLL_ADD] = { 132 .needs_file = 1, 133 .unbound_nonreg_file = 1, 134 .audit_skip = 1, 135 .name = "POLL_ADD", 136 .prep = io_poll_add_prep, 137 .issue = io_poll_add, 138 }, 139 [IORING_OP_POLL_REMOVE] = { 140 .audit_skip = 1, 141 .name = "POLL_REMOVE", 142 .prep = io_poll_remove_prep, 143 .issue = io_poll_remove, 144 }, 145 [IORING_OP_SYNC_FILE_RANGE] = { 146 .needs_file = 1, 147 .audit_skip = 1, 148 .name = "SYNC_FILE_RANGE", 149 .prep = io_sfr_prep, 150 .issue = io_sync_file_range, 151 }, 152 [IORING_OP_SENDMSG] = { 153 .needs_file = 1, 154 .unbound_nonreg_file = 1, 155 .pollout = 1, 156 .ioprio = 1, 157 .manual_alloc = 1, 158 .name = "SENDMSG", 159 #if defined(CONFIG_NET) 160 .async_size = sizeof(struct io_async_msghdr), 161 .prep = io_sendmsg_prep, 162 .issue = io_sendmsg, 163 .prep_async = io_sendmsg_prep_async, 164 .cleanup = io_sendmsg_recvmsg_cleanup, 165 .fail = io_sendrecv_fail, 166 #else 167 .prep = io_eopnotsupp_prep, 168 #endif 169 }, 170 [IORING_OP_RECVMSG] = { 171 .needs_file = 1, 172 .unbound_nonreg_file = 1, 173 .pollin = 1, 174 .buffer_select = 1, 175 .ioprio = 1, 176 .manual_alloc = 1, 177 .name = "RECVMSG", 178 #if defined(CONFIG_NET) 179 .async_size = sizeof(struct io_async_msghdr), 180 .prep = io_recvmsg_prep, 181 .issue = io_recvmsg, 182 .prep_async = io_recvmsg_prep_async, 183 .cleanup = io_sendmsg_recvmsg_cleanup, 184 .fail = io_sendrecv_fail, 185 #else 186 .prep = io_eopnotsupp_prep, 187 #endif 188 }, 189 [IORING_OP_TIMEOUT] = { 190 .audit_skip = 1, 191 .async_size = sizeof(struct io_timeout_data), 192 .name = "TIMEOUT", 193 .prep = io_timeout_prep, 194 .issue = io_timeout, 195 }, 196 [IORING_OP_TIMEOUT_REMOVE] = { 197 /* used by timeout updates' prep() */ 198 .audit_skip = 1, 199 .name = "TIMEOUT_REMOVE", 200 .prep = io_timeout_remove_prep, 201 .issue = io_timeout_remove, 202 }, 203 [IORING_OP_ACCEPT] = { 204 .needs_file = 1, 205 .unbound_nonreg_file = 1, 206 .pollin = 1, 207 .poll_exclusive = 1, 208 .ioprio = 1, /* used for flags */ 209 .name = "ACCEPT", 210 #if defined(CONFIG_NET) 211 .prep = io_accept_prep, 212 .issue = io_accept, 213 #else 214 .prep = io_eopnotsupp_prep, 215 #endif 216 }, 217 [IORING_OP_ASYNC_CANCEL] = { 218 .audit_skip = 1, 219 .name = "ASYNC_CANCEL", 220 .prep = io_async_cancel_prep, 221 .issue = io_async_cancel, 222 }, 223 [IORING_OP_LINK_TIMEOUT] = { 224 .audit_skip = 1, 225 .async_size = sizeof(struct io_timeout_data), 226 .name = "LINK_TIMEOUT", 227 .prep = io_link_timeout_prep, 228 .issue = io_no_issue, 229 }, 230 [IORING_OP_CONNECT] = { 231 .needs_file = 1, 232 .unbound_nonreg_file = 1, 233 .pollout = 1, 234 .name = "CONNECT", 235 #if defined(CONFIG_NET) 236 .async_size = sizeof(struct io_async_connect), 237 .prep = io_connect_prep, 238 .issue = io_connect, 239 .prep_async = io_connect_prep_async, 240 #else 241 .prep = io_eopnotsupp_prep, 242 #endif 243 }, 244 [IORING_OP_FALLOCATE] = { 245 .needs_file = 1, 246 .name = "FALLOCATE", 247 .prep = io_fallocate_prep, 248 .issue = io_fallocate, 249 }, 250 [IORING_OP_OPENAT] = { 251 .name = "OPENAT", 252 .prep = io_openat_prep, 253 .issue = io_openat, 254 .cleanup = io_open_cleanup, 255 }, 256 [IORING_OP_CLOSE] = { 257 .name = "CLOSE", 258 .prep = io_close_prep, 259 .issue = io_close, 260 }, 261 [IORING_OP_FILES_UPDATE] = { 262 .audit_skip = 1, 263 .iopoll = 1, 264 .name = "FILES_UPDATE", 265 .prep = io_files_update_prep, 266 .issue = io_files_update, 267 }, 268 [IORING_OP_STATX] = { 269 .audit_skip = 1, 270 .name = "STATX", 271 .prep = io_statx_prep, 272 .issue = io_statx, 273 .cleanup = io_statx_cleanup, 274 }, 275 [IORING_OP_READ] = { 276 .needs_file = 1, 277 .unbound_nonreg_file = 1, 278 .pollin = 1, 279 .buffer_select = 1, 280 .plug = 1, 281 .audit_skip = 1, 282 .ioprio = 1, 283 .iopoll = 1, 284 .iopoll_queue = 1, 285 .async_size = sizeof(struct io_async_rw), 286 .name = "READ", 287 .prep = io_prep_rw, 288 .issue = io_read, 289 .fail = io_rw_fail, 290 }, 291 [IORING_OP_WRITE] = { 292 .needs_file = 1, 293 .hash_reg_file = 1, 294 .unbound_nonreg_file = 1, 295 .pollout = 1, 296 .plug = 1, 297 .audit_skip = 1, 298 .ioprio = 1, 299 .iopoll = 1, 300 .iopoll_queue = 1, 301 .async_size = sizeof(struct io_async_rw), 302 .name = "WRITE", 303 .prep = io_prep_rw, 304 .issue = io_write, 305 .fail = io_rw_fail, 306 }, 307 [IORING_OP_FADVISE] = { 308 .needs_file = 1, 309 .audit_skip = 1, 310 .name = "FADVISE", 311 .prep = io_fadvise_prep, 312 .issue = io_fadvise, 313 }, 314 [IORING_OP_MADVISE] = { 315 .name = "MADVISE", 316 .prep = io_madvise_prep, 317 .issue = io_madvise, 318 }, 319 [IORING_OP_SEND] = { 320 .needs_file = 1, 321 .unbound_nonreg_file = 1, 322 .pollout = 1, 323 .audit_skip = 1, 324 .ioprio = 1, 325 .manual_alloc = 1, 326 .name = "SEND", 327 #if defined(CONFIG_NET) 328 .async_size = sizeof(struct io_async_msghdr), 329 .prep = io_sendmsg_prep, 330 .issue = io_send, 331 .fail = io_sendrecv_fail, 332 .prep_async = io_send_prep_async, 333 #else 334 .prep = io_eopnotsupp_prep, 335 #endif 336 }, 337 [IORING_OP_RECV] = { 338 .needs_file = 1, 339 .unbound_nonreg_file = 1, 340 .pollin = 1, 341 .buffer_select = 1, 342 .audit_skip = 1, 343 .ioprio = 1, 344 .name = "RECV", 345 #if defined(CONFIG_NET) 346 .prep = io_recvmsg_prep, 347 .issue = io_recv, 348 .fail = io_sendrecv_fail, 349 #else 350 .prep = io_eopnotsupp_prep, 351 #endif 352 }, 353 [IORING_OP_OPENAT2] = { 354 .name = "OPENAT2", 355 .prep = io_openat2_prep, 356 .issue = io_openat2, 357 .cleanup = io_open_cleanup, 358 }, 359 [IORING_OP_EPOLL_CTL] = { 360 .unbound_nonreg_file = 1, 361 .audit_skip = 1, 362 .name = "EPOLL", 363 #if defined(CONFIG_EPOLL) 364 .prep = io_epoll_ctl_prep, 365 .issue = io_epoll_ctl, 366 #else 367 .prep = io_eopnotsupp_prep, 368 #endif 369 }, 370 [IORING_OP_SPLICE] = { 371 .needs_file = 1, 372 .hash_reg_file = 1, 373 .unbound_nonreg_file = 1, 374 .audit_skip = 1, 375 .name = "SPLICE", 376 .prep = io_splice_prep, 377 .issue = io_splice, 378 }, 379 [IORING_OP_PROVIDE_BUFFERS] = { 380 .audit_skip = 1, 381 .iopoll = 1, 382 .name = "PROVIDE_BUFFERS", 383 .prep = io_provide_buffers_prep, 384 .issue = io_provide_buffers, 385 }, 386 [IORING_OP_REMOVE_BUFFERS] = { 387 .audit_skip = 1, 388 .iopoll = 1, 389 .name = "REMOVE_BUFFERS", 390 .prep = io_remove_buffers_prep, 391 .issue = io_remove_buffers, 392 }, 393 [IORING_OP_TEE] = { 394 .needs_file = 1, 395 .hash_reg_file = 1, 396 .unbound_nonreg_file = 1, 397 .audit_skip = 1, 398 .name = "TEE", 399 .prep = io_tee_prep, 400 .issue = io_tee, 401 }, 402 [IORING_OP_SHUTDOWN] = { 403 .needs_file = 1, 404 .name = "SHUTDOWN", 405 #if defined(CONFIG_NET) 406 .prep = io_shutdown_prep, 407 .issue = io_shutdown, 408 #else 409 .prep = io_eopnotsupp_prep, 410 #endif 411 }, 412 [IORING_OP_RENAMEAT] = { 413 .name = "RENAMEAT", 414 .prep = io_renameat_prep, 415 .issue = io_renameat, 416 .cleanup = io_renameat_cleanup, 417 }, 418 [IORING_OP_UNLINKAT] = { 419 .name = "UNLINKAT", 420 .prep = io_unlinkat_prep, 421 .issue = io_unlinkat, 422 .cleanup = io_unlinkat_cleanup, 423 }, 424 [IORING_OP_MKDIRAT] = { 425 .name = "MKDIRAT", 426 .prep = io_mkdirat_prep, 427 .issue = io_mkdirat, 428 .cleanup = io_mkdirat_cleanup, 429 }, 430 [IORING_OP_SYMLINKAT] = { 431 .name = "SYMLINKAT", 432 .prep = io_symlinkat_prep, 433 .issue = io_symlinkat, 434 .cleanup = io_link_cleanup, 435 }, 436 [IORING_OP_LINKAT] = { 437 .name = "LINKAT", 438 .prep = io_linkat_prep, 439 .issue = io_linkat, 440 .cleanup = io_link_cleanup, 441 }, 442 [IORING_OP_MSG_RING] = { 443 .needs_file = 1, 444 .iopoll = 1, 445 .name = "MSG_RING", 446 .prep = io_msg_ring_prep, 447 .issue = io_msg_ring, 448 .cleanup = io_msg_ring_cleanup, 449 }, 450 [IORING_OP_FSETXATTR] = { 451 .needs_file = 1, 452 .name = "FSETXATTR", 453 .prep = io_fsetxattr_prep, 454 .issue = io_fsetxattr, 455 .cleanup = io_xattr_cleanup, 456 }, 457 [IORING_OP_SETXATTR] = { 458 .name = "SETXATTR", 459 .prep = io_setxattr_prep, 460 .issue = io_setxattr, 461 .cleanup = io_xattr_cleanup, 462 }, 463 [IORING_OP_FGETXATTR] = { 464 .needs_file = 1, 465 .name = "FGETXATTR", 466 .prep = io_fgetxattr_prep, 467 .issue = io_fgetxattr, 468 .cleanup = io_xattr_cleanup, 469 }, 470 [IORING_OP_GETXATTR] = { 471 .name = "GETXATTR", 472 .prep = io_getxattr_prep, 473 .issue = io_getxattr, 474 .cleanup = io_xattr_cleanup, 475 }, 476 [IORING_OP_SOCKET] = { 477 .audit_skip = 1, 478 .name = "SOCKET", 479 #if defined(CONFIG_NET) 480 .prep = io_socket_prep, 481 .issue = io_socket, 482 #else 483 .prep = io_eopnotsupp_prep, 484 #endif 485 }, 486 [IORING_OP_URING_CMD] = { 487 .needs_file = 1, 488 .plug = 1, 489 .name = "URING_CMD", 490 .iopoll = 1, 491 .iopoll_queue = 1, 492 .async_size = uring_cmd_pdu_size(1), 493 .prep = io_uring_cmd_prep, 494 .issue = io_uring_cmd, 495 .prep_async = io_uring_cmd_prep_async, 496 }, 497 [IORING_OP_SEND_ZC] = { 498 .name = "SEND_ZC", 499 .needs_file = 1, 500 .unbound_nonreg_file = 1, 501 .pollout = 1, 502 .audit_skip = 1, 503 .ioprio = 1, 504 .manual_alloc = 1, 505 #if defined(CONFIG_NET) 506 .async_size = sizeof(struct io_async_msghdr), 507 .prep = io_send_zc_prep, 508 .issue = io_send_zc, 509 .prep_async = io_send_prep_async, 510 .cleanup = io_send_zc_cleanup, 511 .fail = io_sendrecv_fail, 512 #else 513 .prep = io_eopnotsupp_prep, 514 #endif 515 }, 516 [IORING_OP_SENDMSG_ZC] = { 517 .name = "SENDMSG_ZC", 518 .needs_file = 1, 519 .unbound_nonreg_file = 1, 520 .pollout = 1, 521 .ioprio = 1, 522 .manual_alloc = 1, 523 #if defined(CONFIG_NET) 524 .async_size = sizeof(struct io_async_msghdr), 525 .prep = io_send_zc_prep, 526 .issue = io_sendmsg_zc, 527 .prep_async = io_sendmsg_prep_async, 528 .cleanup = io_send_zc_cleanup, 529 .fail = io_sendrecv_fail, 530 #else 531 .prep = io_eopnotsupp_prep, 532 #endif 533 }, 534 }; 535 536 const char *io_uring_get_opcode(u8 opcode) 537 { 538 if (opcode < IORING_OP_LAST) 539 return io_op_defs[opcode].name; 540 return "INVALID"; 541 } 542 543 void __init io_uring_optable_init(void) 544 { 545 int i; 546 547 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); 548 549 for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) { 550 BUG_ON(!io_op_defs[i].prep); 551 if (io_op_defs[i].prep != io_eopnotsupp_prep) 552 BUG_ON(!io_op_defs[i].issue); 553 WARN_ON_ONCE(!io_op_defs[i].name); 554 } 555 } 556