1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * io_uring opcode handling table 4 */ 5 #include <linux/kernel.h> 6 #include <linux/errno.h> 7 #include <linux/fs.h> 8 #include <linux/file.h> 9 #include <linux/io_uring.h> 10 11 #include "io_uring.h" 12 #include "opdef.h" 13 #include "refs.h" 14 #include "tctx.h" 15 #include "sqpoll.h" 16 #include "fdinfo.h" 17 #include "kbuf.h" 18 #include "rsrc.h" 19 20 #include "xattr.h" 21 #include "nop.h" 22 #include "fs.h" 23 #include "splice.h" 24 #include "sync.h" 25 #include "advise.h" 26 #include "openclose.h" 27 #include "uring_cmd.h" 28 #include "epoll.h" 29 #include "statx.h" 30 #include "net.h" 31 #include "msg_ring.h" 32 #include "timeout.h" 33 #include "poll.h" 34 #include "cancel.h" 35 #include "rw.h" 36 37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags) 38 { 39 WARN_ON_ONCE(1); 40 return -ECANCELED; 41 } 42 43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb, 44 const struct io_uring_sqe *sqe) 45 { 46 return -EOPNOTSUPP; 47 } 48 49 const struct io_op_def io_op_defs[] = { 50 [IORING_OP_NOP] = { 51 .audit_skip = 1, 52 .iopoll = 1, 53 .name = "NOP", 54 .prep = io_nop_prep, 55 .issue = io_nop, 56 }, 57 [IORING_OP_READV] = { 58 .needs_file = 1, 59 .unbound_nonreg_file = 1, 60 .pollin = 1, 61 .buffer_select = 1, 62 .plug = 1, 63 .audit_skip = 1, 64 .ioprio = 1, 65 .iopoll = 1, 66 .async_size = sizeof(struct io_async_rw), 67 .name = "READV", 68 .prep = io_prep_rw, 69 .issue = io_read, 70 .prep_async = io_readv_prep_async, 71 .cleanup = io_readv_writev_cleanup, 72 }, 73 [IORING_OP_WRITEV] = { 74 .needs_file = 1, 75 .hash_reg_file = 1, 76 .unbound_nonreg_file = 1, 77 .pollout = 1, 78 .plug = 1, 79 .audit_skip = 1, 80 .ioprio = 1, 81 .iopoll = 1, 82 .async_size = sizeof(struct io_async_rw), 83 .name = "WRITEV", 84 .prep = io_prep_rw, 85 .issue = io_write, 86 .prep_async = io_writev_prep_async, 87 .cleanup = io_readv_writev_cleanup, 88 }, 89 [IORING_OP_FSYNC] = { 90 .needs_file = 1, 91 .audit_skip = 1, 92 .name = "FSYNC", 93 .prep = io_fsync_prep, 94 .issue = io_fsync, 95 }, 96 [IORING_OP_READ_FIXED] = { 97 .needs_file = 1, 98 .unbound_nonreg_file = 1, 99 .pollin = 1, 100 .plug = 1, 101 .audit_skip = 1, 102 .ioprio = 1, 103 .iopoll = 1, 104 .async_size = sizeof(struct io_async_rw), 105 .name = "READ_FIXED", 106 .prep = io_prep_rw, 107 .issue = io_read, 108 }, 109 [IORING_OP_WRITE_FIXED] = { 110 .needs_file = 1, 111 .hash_reg_file = 1, 112 .unbound_nonreg_file = 1, 113 .pollout = 1, 114 .plug = 1, 115 .audit_skip = 1, 116 .ioprio = 1, 117 .iopoll = 1, 118 .async_size = sizeof(struct io_async_rw), 119 .name = "WRITE_FIXED", 120 .prep = io_prep_rw, 121 .issue = io_write, 122 }, 123 [IORING_OP_POLL_ADD] = { 124 .needs_file = 1, 125 .unbound_nonreg_file = 1, 126 .audit_skip = 1, 127 .name = "POLL_ADD", 128 .prep = io_poll_add_prep, 129 .issue = io_poll_add, 130 }, 131 [IORING_OP_POLL_REMOVE] = { 132 .audit_skip = 1, 133 .name = "POLL_REMOVE", 134 .prep = io_poll_remove_prep, 135 .issue = io_poll_remove, 136 }, 137 [IORING_OP_SYNC_FILE_RANGE] = { 138 .needs_file = 1, 139 .audit_skip = 1, 140 .name = "SYNC_FILE_RANGE", 141 .prep = io_sfr_prep, 142 .issue = io_sync_file_range, 143 }, 144 [IORING_OP_SENDMSG] = { 145 .needs_file = 1, 146 .unbound_nonreg_file = 1, 147 .pollout = 1, 148 .ioprio = 1, 149 .name = "SENDMSG", 150 #if defined(CONFIG_NET) 151 .async_size = sizeof(struct io_async_msghdr), 152 .prep = io_sendmsg_prep, 153 .issue = io_sendmsg, 154 .prep_async = io_sendmsg_prep_async, 155 .cleanup = io_sendmsg_recvmsg_cleanup, 156 #else 157 .prep = io_eopnotsupp_prep, 158 #endif 159 }, 160 [IORING_OP_RECVMSG] = { 161 .needs_file = 1, 162 .unbound_nonreg_file = 1, 163 .pollin = 1, 164 .buffer_select = 1, 165 .ioprio = 1, 166 .name = "RECVMSG", 167 #if defined(CONFIG_NET) 168 .async_size = sizeof(struct io_async_msghdr), 169 .prep = io_recvmsg_prep, 170 .issue = io_recvmsg, 171 .prep_async = io_recvmsg_prep_async, 172 .cleanup = io_sendmsg_recvmsg_cleanup, 173 #else 174 .prep = io_eopnotsupp_prep, 175 #endif 176 }, 177 [IORING_OP_TIMEOUT] = { 178 .audit_skip = 1, 179 .async_size = sizeof(struct io_timeout_data), 180 .name = "TIMEOUT", 181 .prep = io_timeout_prep, 182 .issue = io_timeout, 183 }, 184 [IORING_OP_TIMEOUT_REMOVE] = { 185 /* used by timeout updates' prep() */ 186 .audit_skip = 1, 187 .name = "TIMEOUT_REMOVE", 188 .prep = io_timeout_remove_prep, 189 .issue = io_timeout_remove, 190 }, 191 [IORING_OP_ACCEPT] = { 192 .needs_file = 1, 193 .unbound_nonreg_file = 1, 194 .pollin = 1, 195 .poll_exclusive = 1, 196 .ioprio = 1, /* used for flags */ 197 .name = "ACCEPT", 198 #if defined(CONFIG_NET) 199 .prep = io_accept_prep, 200 .issue = io_accept, 201 #else 202 .prep = io_eopnotsupp_prep, 203 #endif 204 }, 205 [IORING_OP_ASYNC_CANCEL] = { 206 .audit_skip = 1, 207 .name = "ASYNC_CANCEL", 208 .prep = io_async_cancel_prep, 209 .issue = io_async_cancel, 210 }, 211 [IORING_OP_LINK_TIMEOUT] = { 212 .audit_skip = 1, 213 .async_size = sizeof(struct io_timeout_data), 214 .name = "LINK_TIMEOUT", 215 .prep = io_link_timeout_prep, 216 .issue = io_no_issue, 217 }, 218 [IORING_OP_CONNECT] = { 219 .needs_file = 1, 220 .unbound_nonreg_file = 1, 221 .pollout = 1, 222 .name = "CONNECT", 223 #if defined(CONFIG_NET) 224 .async_size = sizeof(struct io_async_connect), 225 .prep = io_connect_prep, 226 .issue = io_connect, 227 .prep_async = io_connect_prep_async, 228 #else 229 .prep = io_eopnotsupp_prep, 230 #endif 231 }, 232 [IORING_OP_FALLOCATE] = { 233 .needs_file = 1, 234 .name = "FALLOCATE", 235 .prep = io_fallocate_prep, 236 .issue = io_fallocate, 237 }, 238 [IORING_OP_OPENAT] = { 239 .name = "OPENAT", 240 .prep = io_openat_prep, 241 .issue = io_openat, 242 .cleanup = io_open_cleanup, 243 }, 244 [IORING_OP_CLOSE] = { 245 .name = "CLOSE", 246 .prep = io_close_prep, 247 .issue = io_close, 248 }, 249 [IORING_OP_RSRC_UPDATE] = { 250 .audit_skip = 1, 251 .iopoll = 1, 252 .name = "RSRC_UPDATE", 253 .prep = io_rsrc_update_prep, 254 .issue = io_rsrc_update, 255 .ioprio = 1, 256 }, 257 [IORING_OP_STATX] = { 258 .audit_skip = 1, 259 .name = "STATX", 260 .prep = io_statx_prep, 261 .issue = io_statx, 262 .cleanup = io_statx_cleanup, 263 }, 264 [IORING_OP_READ] = { 265 .needs_file = 1, 266 .unbound_nonreg_file = 1, 267 .pollin = 1, 268 .buffer_select = 1, 269 .plug = 1, 270 .audit_skip = 1, 271 .ioprio = 1, 272 .iopoll = 1, 273 .async_size = sizeof(struct io_async_rw), 274 .name = "READ", 275 .prep = io_prep_rw, 276 .issue = io_read, 277 }, 278 [IORING_OP_WRITE] = { 279 .needs_file = 1, 280 .hash_reg_file = 1, 281 .unbound_nonreg_file = 1, 282 .pollout = 1, 283 .plug = 1, 284 .audit_skip = 1, 285 .ioprio = 1, 286 .iopoll = 1, 287 .async_size = sizeof(struct io_async_rw), 288 .name = "WRITE", 289 .prep = io_prep_rw, 290 .issue = io_write, 291 }, 292 [IORING_OP_FADVISE] = { 293 .needs_file = 1, 294 .audit_skip = 1, 295 .name = "FADVISE", 296 .prep = io_fadvise_prep, 297 .issue = io_fadvise, 298 }, 299 [IORING_OP_MADVISE] = { 300 .name = "MADVISE", 301 .prep = io_madvise_prep, 302 .issue = io_madvise, 303 }, 304 [IORING_OP_SEND] = { 305 .needs_file = 1, 306 .unbound_nonreg_file = 1, 307 .pollout = 1, 308 .audit_skip = 1, 309 .ioprio = 1, 310 .name = "SEND", 311 #if defined(CONFIG_NET) 312 .prep = io_sendmsg_prep, 313 .issue = io_send, 314 #else 315 .prep = io_eopnotsupp_prep, 316 #endif 317 }, 318 [IORING_OP_RECV] = { 319 .needs_file = 1, 320 .unbound_nonreg_file = 1, 321 .pollin = 1, 322 .buffer_select = 1, 323 .audit_skip = 1, 324 .ioprio = 1, 325 .name = "RECV", 326 #if defined(CONFIG_NET) 327 .prep = io_recvmsg_prep, 328 .issue = io_recv, 329 #else 330 .prep = io_eopnotsupp_prep, 331 #endif 332 }, 333 [IORING_OP_OPENAT2] = { 334 .name = "OPENAT2", 335 .prep = io_openat2_prep, 336 .issue = io_openat2, 337 .cleanup = io_open_cleanup, 338 }, 339 [IORING_OP_EPOLL_CTL] = { 340 .unbound_nonreg_file = 1, 341 .audit_skip = 1, 342 .name = "EPOLL", 343 #if defined(CONFIG_EPOLL) 344 .prep = io_epoll_ctl_prep, 345 .issue = io_epoll_ctl, 346 #else 347 .prep = io_eopnotsupp_prep, 348 #endif 349 }, 350 [IORING_OP_SPLICE] = { 351 .needs_file = 1, 352 .hash_reg_file = 1, 353 .unbound_nonreg_file = 1, 354 .audit_skip = 1, 355 .name = "SPLICE", 356 .prep = io_splice_prep, 357 .issue = io_splice, 358 }, 359 [IORING_OP_PROVIDE_BUFFERS] = { 360 .audit_skip = 1, 361 .iopoll = 1, 362 .name = "PROVIDE_BUFFERS", 363 .prep = io_provide_buffers_prep, 364 .issue = io_provide_buffers, 365 }, 366 [IORING_OP_REMOVE_BUFFERS] = { 367 .audit_skip = 1, 368 .iopoll = 1, 369 .name = "REMOVE_BUFFERS", 370 .prep = io_remove_buffers_prep, 371 .issue = io_remove_buffers, 372 }, 373 [IORING_OP_TEE] = { 374 .needs_file = 1, 375 .hash_reg_file = 1, 376 .unbound_nonreg_file = 1, 377 .audit_skip = 1, 378 .name = "TEE", 379 .prep = io_tee_prep, 380 .issue = io_tee, 381 }, 382 [IORING_OP_SHUTDOWN] = { 383 .needs_file = 1, 384 .name = "SHUTDOWN", 385 #if defined(CONFIG_NET) 386 .prep = io_shutdown_prep, 387 .issue = io_shutdown, 388 #else 389 .prep = io_eopnotsupp_prep, 390 #endif 391 }, 392 [IORING_OP_RENAMEAT] = { 393 .name = "RENAMEAT", 394 .prep = io_renameat_prep, 395 .issue = io_renameat, 396 .cleanup = io_renameat_cleanup, 397 }, 398 [IORING_OP_UNLINKAT] = { 399 .name = "UNLINKAT", 400 .prep = io_unlinkat_prep, 401 .issue = io_unlinkat, 402 .cleanup = io_unlinkat_cleanup, 403 }, 404 [IORING_OP_MKDIRAT] = { 405 .name = "MKDIRAT", 406 .prep = io_mkdirat_prep, 407 .issue = io_mkdirat, 408 .cleanup = io_mkdirat_cleanup, 409 }, 410 [IORING_OP_SYMLINKAT] = { 411 .name = "SYMLINKAT", 412 .prep = io_symlinkat_prep, 413 .issue = io_symlinkat, 414 .cleanup = io_link_cleanup, 415 }, 416 [IORING_OP_LINKAT] = { 417 .name = "LINKAT", 418 .prep = io_linkat_prep, 419 .issue = io_linkat, 420 .cleanup = io_link_cleanup, 421 }, 422 [IORING_OP_MSG_RING] = { 423 .needs_file = 1, 424 .iopoll = 1, 425 .name = "MSG_RING", 426 .prep = io_msg_ring_prep, 427 .issue = io_msg_ring, 428 }, 429 [IORING_OP_FSETXATTR] = { 430 .needs_file = 1, 431 .name = "FSETXATTR", 432 .prep = io_fsetxattr_prep, 433 .issue = io_fsetxattr, 434 .cleanup = io_xattr_cleanup, 435 }, 436 [IORING_OP_SETXATTR] = { 437 .name = "SETXATTR", 438 .prep = io_setxattr_prep, 439 .issue = io_setxattr, 440 .cleanup = io_xattr_cleanup, 441 }, 442 [IORING_OP_FGETXATTR] = { 443 .needs_file = 1, 444 .name = "FGETXATTR", 445 .prep = io_fgetxattr_prep, 446 .issue = io_fgetxattr, 447 .cleanup = io_xattr_cleanup, 448 }, 449 [IORING_OP_GETXATTR] = { 450 .name = "GETXATTR", 451 .prep = io_getxattr_prep, 452 .issue = io_getxattr, 453 .cleanup = io_xattr_cleanup, 454 }, 455 [IORING_OP_SOCKET] = { 456 .audit_skip = 1, 457 .name = "SOCKET", 458 #if defined(CONFIG_NET) 459 .prep = io_socket_prep, 460 .issue = io_socket, 461 #else 462 .prep = io_eopnotsupp_prep, 463 #endif 464 }, 465 [IORING_OP_URING_CMD] = { 466 .needs_file = 1, 467 .plug = 1, 468 .name = "URING_CMD", 469 .async_size = uring_cmd_pdu_size(1), 470 .prep = io_uring_cmd_prep, 471 .issue = io_uring_cmd, 472 .prep_async = io_uring_cmd_prep_async, 473 }, 474 [IORING_OP_SENDZC_NOTIF] = { 475 .name = "SENDZC_NOTIF", 476 .needs_file = 1, 477 .unbound_nonreg_file = 1, 478 .pollout = 1, 479 .audit_skip = 1, 480 .ioprio = 1, 481 .manual_alloc = 1, 482 #if defined(CONFIG_NET) 483 .async_size = sizeof(struct io_async_msghdr), 484 .prep = io_sendzc_prep, 485 .issue = io_sendzc, 486 .prep_async = io_sendzc_prep_async, 487 #else 488 .prep = io_eopnotsupp_prep, 489 #endif 490 }, 491 }; 492 493 const char *io_uring_get_opcode(u8 opcode) 494 { 495 if (opcode < IORING_OP_LAST) 496 return io_op_defs[opcode].name; 497 return "INVALID"; 498 } 499 500 void __init io_uring_optable_init(void) 501 { 502 int i; 503 504 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); 505 506 for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) { 507 BUG_ON(!io_op_defs[i].prep); 508 if (io_op_defs[i].prep != io_eopnotsupp_prep) 509 BUG_ON(!io_op_defs[i].issue); 510 WARN_ON_ONCE(!io_op_defs[i].name); 511 } 512 } 513