1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * io_uring opcode handling table 4 */ 5 #include <linux/kernel.h> 6 #include <linux/errno.h> 7 #include <linux/fs.h> 8 #include <linux/file.h> 9 #include <linux/io_uring.h> 10 11 #include "io_uring.h" 12 #include "opdef.h" 13 #include "refs.h" 14 #include "tctx.h" 15 #include "sqpoll.h" 16 #include "fdinfo.h" 17 #include "kbuf.h" 18 #include "rsrc.h" 19 20 #include "xattr.h" 21 #include "nop.h" 22 #include "fs.h" 23 #include "splice.h" 24 #include "sync.h" 25 #include "advise.h" 26 #include "openclose.h" 27 #include "uring_cmd.h" 28 #include "epoll.h" 29 #include "statx.h" 30 #include "net.h" 31 #include "msg_ring.h" 32 #include "timeout.h" 33 #include "poll.h" 34 #include "cancel.h" 35 #include "rw.h" 36 37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags) 38 { 39 WARN_ON_ONCE(1); 40 return -ECANCELED; 41 } 42 43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb, 44 const struct io_uring_sqe *sqe) 45 { 46 return -EOPNOTSUPP; 47 } 48 49 const struct io_op_def io_op_defs[] = { 50 [IORING_OP_NOP] = { 51 .audit_skip = 1, 52 .iopoll = 1, 53 .name = "NOP", 54 .prep = io_nop_prep, 55 .issue = io_nop, 56 }, 57 [IORING_OP_READV] = { 58 .needs_file = 1, 59 .unbound_nonreg_file = 1, 60 .pollin = 1, 61 .buffer_select = 1, 62 .plug = 1, 63 .audit_skip = 1, 64 .ioprio = 1, 65 .iopoll = 1, 66 .async_size = sizeof(struct io_async_rw), 67 .name = "READV", 68 .prep = io_prep_rw, 69 .issue = io_read, 70 .prep_async = io_readv_prep_async, 71 .cleanup = io_readv_writev_cleanup, 72 .fail = io_rw_fail, 73 }, 74 [IORING_OP_WRITEV] = { 75 .needs_file = 1, 76 .hash_reg_file = 1, 77 .unbound_nonreg_file = 1, 78 .pollout = 1, 79 .plug = 1, 80 .audit_skip = 1, 81 .ioprio = 1, 82 .iopoll = 1, 83 .async_size = sizeof(struct io_async_rw), 84 .name = "WRITEV", 85 .prep = io_prep_rw, 86 .issue = io_write, 87 .prep_async = io_writev_prep_async, 88 .cleanup = io_readv_writev_cleanup, 89 .fail = io_rw_fail, 90 }, 91 [IORING_OP_FSYNC] = { 92 .needs_file = 1, 93 .audit_skip = 1, 94 .name = "FSYNC", 95 .prep = io_fsync_prep, 96 .issue = io_fsync, 97 }, 98 [IORING_OP_READ_FIXED] = { 99 .needs_file = 1, 100 .unbound_nonreg_file = 1, 101 .pollin = 1, 102 .plug = 1, 103 .audit_skip = 1, 104 .ioprio = 1, 105 .iopoll = 1, 106 .async_size = sizeof(struct io_async_rw), 107 .name = "READ_FIXED", 108 .prep = io_prep_rw, 109 .issue = io_read, 110 .fail = io_rw_fail, 111 }, 112 [IORING_OP_WRITE_FIXED] = { 113 .needs_file = 1, 114 .hash_reg_file = 1, 115 .unbound_nonreg_file = 1, 116 .pollout = 1, 117 .plug = 1, 118 .audit_skip = 1, 119 .ioprio = 1, 120 .iopoll = 1, 121 .async_size = sizeof(struct io_async_rw), 122 .name = "WRITE_FIXED", 123 .prep = io_prep_rw, 124 .issue = io_write, 125 .fail = io_rw_fail, 126 }, 127 [IORING_OP_POLL_ADD] = { 128 .needs_file = 1, 129 .unbound_nonreg_file = 1, 130 .audit_skip = 1, 131 .name = "POLL_ADD", 132 .prep = io_poll_add_prep, 133 .issue = io_poll_add, 134 }, 135 [IORING_OP_POLL_REMOVE] = { 136 .audit_skip = 1, 137 .name = "POLL_REMOVE", 138 .prep = io_poll_remove_prep, 139 .issue = io_poll_remove, 140 }, 141 [IORING_OP_SYNC_FILE_RANGE] = { 142 .needs_file = 1, 143 .audit_skip = 1, 144 .name = "SYNC_FILE_RANGE", 145 .prep = io_sfr_prep, 146 .issue = io_sync_file_range, 147 }, 148 [IORING_OP_SENDMSG] = { 149 .needs_file = 1, 150 .unbound_nonreg_file = 1, 151 .pollout = 1, 152 .ioprio = 1, 153 .manual_alloc = 1, 154 .name = "SENDMSG", 155 #if defined(CONFIG_NET) 156 .async_size = sizeof(struct io_async_msghdr), 157 .prep = io_sendmsg_prep, 158 .issue = io_sendmsg, 159 .prep_async = io_sendmsg_prep_async, 160 .cleanup = io_sendmsg_recvmsg_cleanup, 161 .fail = io_sendrecv_fail, 162 #else 163 .prep = io_eopnotsupp_prep, 164 #endif 165 }, 166 [IORING_OP_RECVMSG] = { 167 .needs_file = 1, 168 .unbound_nonreg_file = 1, 169 .pollin = 1, 170 .buffer_select = 1, 171 .ioprio = 1, 172 .manual_alloc = 1, 173 .name = "RECVMSG", 174 #if defined(CONFIG_NET) 175 .async_size = sizeof(struct io_async_msghdr), 176 .prep = io_recvmsg_prep, 177 .issue = io_recvmsg, 178 .prep_async = io_recvmsg_prep_async, 179 .cleanup = io_sendmsg_recvmsg_cleanup, 180 .fail = io_sendrecv_fail, 181 #else 182 .prep = io_eopnotsupp_prep, 183 #endif 184 }, 185 [IORING_OP_TIMEOUT] = { 186 .audit_skip = 1, 187 .async_size = sizeof(struct io_timeout_data), 188 .name = "TIMEOUT", 189 .prep = io_timeout_prep, 190 .issue = io_timeout, 191 }, 192 [IORING_OP_TIMEOUT_REMOVE] = { 193 /* used by timeout updates' prep() */ 194 .audit_skip = 1, 195 .name = "TIMEOUT_REMOVE", 196 .prep = io_timeout_remove_prep, 197 .issue = io_timeout_remove, 198 }, 199 [IORING_OP_ACCEPT] = { 200 .needs_file = 1, 201 .unbound_nonreg_file = 1, 202 .pollin = 1, 203 .poll_exclusive = 1, 204 .ioprio = 1, /* used for flags */ 205 .name = "ACCEPT", 206 #if defined(CONFIG_NET) 207 .prep = io_accept_prep, 208 .issue = io_accept, 209 #else 210 .prep = io_eopnotsupp_prep, 211 #endif 212 }, 213 [IORING_OP_ASYNC_CANCEL] = { 214 .audit_skip = 1, 215 .name = "ASYNC_CANCEL", 216 .prep = io_async_cancel_prep, 217 .issue = io_async_cancel, 218 }, 219 [IORING_OP_LINK_TIMEOUT] = { 220 .audit_skip = 1, 221 .async_size = sizeof(struct io_timeout_data), 222 .name = "LINK_TIMEOUT", 223 .prep = io_link_timeout_prep, 224 .issue = io_no_issue, 225 }, 226 [IORING_OP_CONNECT] = { 227 .needs_file = 1, 228 .unbound_nonreg_file = 1, 229 .pollout = 1, 230 .name = "CONNECT", 231 #if defined(CONFIG_NET) 232 .async_size = sizeof(struct io_async_connect), 233 .prep = io_connect_prep, 234 .issue = io_connect, 235 .prep_async = io_connect_prep_async, 236 #else 237 .prep = io_eopnotsupp_prep, 238 #endif 239 }, 240 [IORING_OP_FALLOCATE] = { 241 .needs_file = 1, 242 .name = "FALLOCATE", 243 .prep = io_fallocate_prep, 244 .issue = io_fallocate, 245 }, 246 [IORING_OP_OPENAT] = { 247 .name = "OPENAT", 248 .prep = io_openat_prep, 249 .issue = io_openat, 250 .cleanup = io_open_cleanup, 251 }, 252 [IORING_OP_CLOSE] = { 253 .name = "CLOSE", 254 .prep = io_close_prep, 255 .issue = io_close, 256 }, 257 [IORING_OP_FILES_UPDATE] = { 258 .audit_skip = 1, 259 .iopoll = 1, 260 .name = "FILES_UPDATE", 261 .prep = io_files_update_prep, 262 .issue = io_files_update, 263 }, 264 [IORING_OP_STATX] = { 265 .audit_skip = 1, 266 .name = "STATX", 267 .prep = io_statx_prep, 268 .issue = io_statx, 269 .cleanup = io_statx_cleanup, 270 }, 271 [IORING_OP_READ] = { 272 .needs_file = 1, 273 .unbound_nonreg_file = 1, 274 .pollin = 1, 275 .buffer_select = 1, 276 .plug = 1, 277 .audit_skip = 1, 278 .ioprio = 1, 279 .iopoll = 1, 280 .async_size = sizeof(struct io_async_rw), 281 .name = "READ", 282 .prep = io_prep_rw, 283 .issue = io_read, 284 .fail = io_rw_fail, 285 }, 286 [IORING_OP_WRITE] = { 287 .needs_file = 1, 288 .hash_reg_file = 1, 289 .unbound_nonreg_file = 1, 290 .pollout = 1, 291 .plug = 1, 292 .audit_skip = 1, 293 .ioprio = 1, 294 .iopoll = 1, 295 .async_size = sizeof(struct io_async_rw), 296 .name = "WRITE", 297 .prep = io_prep_rw, 298 .issue = io_write, 299 .fail = io_rw_fail, 300 }, 301 [IORING_OP_FADVISE] = { 302 .needs_file = 1, 303 .audit_skip = 1, 304 .name = "FADVISE", 305 .prep = io_fadvise_prep, 306 .issue = io_fadvise, 307 }, 308 [IORING_OP_MADVISE] = { 309 .name = "MADVISE", 310 .prep = io_madvise_prep, 311 .issue = io_madvise, 312 }, 313 [IORING_OP_SEND] = { 314 .needs_file = 1, 315 .unbound_nonreg_file = 1, 316 .pollout = 1, 317 .audit_skip = 1, 318 .ioprio = 1, 319 .manual_alloc = 1, 320 .name = "SEND", 321 #if defined(CONFIG_NET) 322 .async_size = sizeof(struct io_async_msghdr), 323 .prep = io_sendmsg_prep, 324 .issue = io_send, 325 .fail = io_sendrecv_fail, 326 .prep_async = io_send_prep_async, 327 #else 328 .prep = io_eopnotsupp_prep, 329 #endif 330 }, 331 [IORING_OP_RECV] = { 332 .needs_file = 1, 333 .unbound_nonreg_file = 1, 334 .pollin = 1, 335 .buffer_select = 1, 336 .audit_skip = 1, 337 .ioprio = 1, 338 .name = "RECV", 339 #if defined(CONFIG_NET) 340 .prep = io_recvmsg_prep, 341 .issue = io_recv, 342 .fail = io_sendrecv_fail, 343 #else 344 .prep = io_eopnotsupp_prep, 345 #endif 346 }, 347 [IORING_OP_OPENAT2] = { 348 .name = "OPENAT2", 349 .prep = io_openat2_prep, 350 .issue = io_openat2, 351 .cleanup = io_open_cleanup, 352 }, 353 [IORING_OP_EPOLL_CTL] = { 354 .unbound_nonreg_file = 1, 355 .audit_skip = 1, 356 .name = "EPOLL", 357 #if defined(CONFIG_EPOLL) 358 .prep = io_epoll_ctl_prep, 359 .issue = io_epoll_ctl, 360 #else 361 .prep = io_eopnotsupp_prep, 362 #endif 363 }, 364 [IORING_OP_SPLICE] = { 365 .needs_file = 1, 366 .hash_reg_file = 1, 367 .unbound_nonreg_file = 1, 368 .audit_skip = 1, 369 .name = "SPLICE", 370 .prep = io_splice_prep, 371 .issue = io_splice, 372 }, 373 [IORING_OP_PROVIDE_BUFFERS] = { 374 .audit_skip = 1, 375 .iopoll = 1, 376 .name = "PROVIDE_BUFFERS", 377 .prep = io_provide_buffers_prep, 378 .issue = io_provide_buffers, 379 }, 380 [IORING_OP_REMOVE_BUFFERS] = { 381 .audit_skip = 1, 382 .iopoll = 1, 383 .name = "REMOVE_BUFFERS", 384 .prep = io_remove_buffers_prep, 385 .issue = io_remove_buffers, 386 }, 387 [IORING_OP_TEE] = { 388 .needs_file = 1, 389 .hash_reg_file = 1, 390 .unbound_nonreg_file = 1, 391 .audit_skip = 1, 392 .name = "TEE", 393 .prep = io_tee_prep, 394 .issue = io_tee, 395 }, 396 [IORING_OP_SHUTDOWN] = { 397 .needs_file = 1, 398 .name = "SHUTDOWN", 399 #if defined(CONFIG_NET) 400 .prep = io_shutdown_prep, 401 .issue = io_shutdown, 402 #else 403 .prep = io_eopnotsupp_prep, 404 #endif 405 }, 406 [IORING_OP_RENAMEAT] = { 407 .name = "RENAMEAT", 408 .prep = io_renameat_prep, 409 .issue = io_renameat, 410 .cleanup = io_renameat_cleanup, 411 }, 412 [IORING_OP_UNLINKAT] = { 413 .name = "UNLINKAT", 414 .prep = io_unlinkat_prep, 415 .issue = io_unlinkat, 416 .cleanup = io_unlinkat_cleanup, 417 }, 418 [IORING_OP_MKDIRAT] = { 419 .name = "MKDIRAT", 420 .prep = io_mkdirat_prep, 421 .issue = io_mkdirat, 422 .cleanup = io_mkdirat_cleanup, 423 }, 424 [IORING_OP_SYMLINKAT] = { 425 .name = "SYMLINKAT", 426 .prep = io_symlinkat_prep, 427 .issue = io_symlinkat, 428 .cleanup = io_link_cleanup, 429 }, 430 [IORING_OP_LINKAT] = { 431 .name = "LINKAT", 432 .prep = io_linkat_prep, 433 .issue = io_linkat, 434 .cleanup = io_link_cleanup, 435 }, 436 [IORING_OP_MSG_RING] = { 437 .needs_file = 1, 438 .iopoll = 1, 439 .name = "MSG_RING", 440 .prep = io_msg_ring_prep, 441 .issue = io_msg_ring, 442 }, 443 [IORING_OP_FSETXATTR] = { 444 .needs_file = 1, 445 .name = "FSETXATTR", 446 .prep = io_fsetxattr_prep, 447 .issue = io_fsetxattr, 448 .cleanup = io_xattr_cleanup, 449 }, 450 [IORING_OP_SETXATTR] = { 451 .name = "SETXATTR", 452 .prep = io_setxattr_prep, 453 .issue = io_setxattr, 454 .cleanup = io_xattr_cleanup, 455 }, 456 [IORING_OP_FGETXATTR] = { 457 .needs_file = 1, 458 .name = "FGETXATTR", 459 .prep = io_fgetxattr_prep, 460 .issue = io_fgetxattr, 461 .cleanup = io_xattr_cleanup, 462 }, 463 [IORING_OP_GETXATTR] = { 464 .name = "GETXATTR", 465 .prep = io_getxattr_prep, 466 .issue = io_getxattr, 467 .cleanup = io_xattr_cleanup, 468 }, 469 [IORING_OP_SOCKET] = { 470 .audit_skip = 1, 471 .name = "SOCKET", 472 #if defined(CONFIG_NET) 473 .prep = io_socket_prep, 474 .issue = io_socket, 475 #else 476 .prep = io_eopnotsupp_prep, 477 #endif 478 }, 479 [IORING_OP_URING_CMD] = { 480 .needs_file = 1, 481 .plug = 1, 482 .name = "URING_CMD", 483 .iopoll = 1, 484 .async_size = uring_cmd_pdu_size(1), 485 .prep = io_uring_cmd_prep, 486 .issue = io_uring_cmd, 487 .prep_async = io_uring_cmd_prep_async, 488 }, 489 [IORING_OP_SEND_ZC] = { 490 .name = "SEND_ZC", 491 .needs_file = 1, 492 .unbound_nonreg_file = 1, 493 .pollout = 1, 494 .audit_skip = 1, 495 .ioprio = 1, 496 .manual_alloc = 1, 497 #if defined(CONFIG_NET) 498 .async_size = sizeof(struct io_async_msghdr), 499 .prep = io_send_zc_prep, 500 .issue = io_send_zc, 501 .prep_async = io_send_prep_async, 502 .cleanup = io_send_zc_cleanup, 503 .fail = io_sendrecv_fail, 504 #else 505 .prep = io_eopnotsupp_prep, 506 #endif 507 }, 508 [IORING_OP_SENDMSG_ZC] = { 509 .name = "SENDMSG_ZC", 510 .needs_file = 1, 511 .unbound_nonreg_file = 1, 512 .pollout = 1, 513 .ioprio = 1, 514 .manual_alloc = 1, 515 #if defined(CONFIG_NET) 516 .async_size = sizeof(struct io_async_msghdr), 517 .prep = io_send_zc_prep, 518 .issue = io_sendmsg_zc, 519 .prep_async = io_sendmsg_prep_async, 520 .cleanup = io_send_zc_cleanup, 521 .fail = io_sendrecv_fail, 522 #else 523 .prep = io_eopnotsupp_prep, 524 #endif 525 }, 526 }; 527 528 const char *io_uring_get_opcode(u8 opcode) 529 { 530 if (opcode < IORING_OP_LAST) 531 return io_op_defs[opcode].name; 532 return "INVALID"; 533 } 534 535 void __init io_uring_optable_init(void) 536 { 537 int i; 538 539 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); 540 541 for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) { 542 BUG_ON(!io_op_defs[i].prep); 543 if (io_op_defs[i].prep != io_eopnotsupp_prep) 544 BUG_ON(!io_op_defs[i].issue); 545 WARN_ON_ONCE(!io_op_defs[i].name); 546 } 547 } 548