1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * io_uring opcode handling table
4 */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "opdef.h"
13 #include "refs.h"
14 #include "tctx.h"
15 #include "sqpoll.h"
16 #include "fdinfo.h"
17 #include "kbuf.h"
18 #include "rsrc.h"
19
20 #include "xattr.h"
21 #include "nop.h"
22 #include "fs.h"
23 #include "splice.h"
24 #include "sync.h"
25 #include "advise.h"
26 #include "openclose.h"
27 #include "uring_cmd.h"
28 #include "epoll.h"
29 #include "statx.h"
30 #include "net.h"
31 #include "msg_ring.h"
32 #include "timeout.h"
33 #include "poll.h"
34 #include "cancel.h"
35 #include "rw.h"
36
io_no_issue(struct io_kiocb * req,unsigned int issue_flags)37 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
38 {
39 WARN_ON_ONCE(1);
40 return -ECANCELED;
41 }
42
io_eopnotsupp_prep(struct io_kiocb * kiocb,const struct io_uring_sqe * sqe)43 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
44 const struct io_uring_sqe *sqe)
45 {
46 return -EOPNOTSUPP;
47 }
48
49 const struct io_issue_def io_issue_defs[] = {
50 [IORING_OP_NOP] = {
51 .audit_skip = 1,
52 .iopoll = 1,
53 .prep = io_nop_prep,
54 .issue = io_nop,
55 },
56 [IORING_OP_READV] = {
57 .needs_file = 1,
58 .unbound_nonreg_file = 1,
59 .pollin = 1,
60 .buffer_select = 1,
61 .plug = 1,
62 .audit_skip = 1,
63 .ioprio = 1,
64 .iopoll = 1,
65 .iopoll_queue = 1,
66 .prep = io_prep_rw,
67 .issue = io_read,
68 },
69 [IORING_OP_WRITEV] = {
70 .needs_file = 1,
71 .hash_reg_file = 1,
72 .unbound_nonreg_file = 1,
73 .pollout = 1,
74 .plug = 1,
75 .audit_skip = 1,
76 .ioprio = 1,
77 .iopoll = 1,
78 .iopoll_queue = 1,
79 .prep = io_prep_rw,
80 .issue = io_write,
81 },
82 [IORING_OP_FSYNC] = {
83 .needs_file = 1,
84 .audit_skip = 1,
85 .prep = io_fsync_prep,
86 .issue = io_fsync,
87 },
88 [IORING_OP_READ_FIXED] = {
89 .needs_file = 1,
90 .unbound_nonreg_file = 1,
91 .pollin = 1,
92 .plug = 1,
93 .audit_skip = 1,
94 .ioprio = 1,
95 .iopoll = 1,
96 .iopoll_queue = 1,
97 .prep = io_prep_rw,
98 .issue = io_read,
99 },
100 [IORING_OP_WRITE_FIXED] = {
101 .needs_file = 1,
102 .hash_reg_file = 1,
103 .unbound_nonreg_file = 1,
104 .pollout = 1,
105 .plug = 1,
106 .audit_skip = 1,
107 .ioprio = 1,
108 .iopoll = 1,
109 .iopoll_queue = 1,
110 .prep = io_prep_rw,
111 .issue = io_write,
112 },
113 [IORING_OP_POLL_ADD] = {
114 .needs_file = 1,
115 .unbound_nonreg_file = 1,
116 .audit_skip = 1,
117 .prep = io_poll_add_prep,
118 .issue = io_poll_add,
119 },
120 [IORING_OP_POLL_REMOVE] = {
121 .audit_skip = 1,
122 .prep = io_poll_remove_prep,
123 .issue = io_poll_remove,
124 },
125 [IORING_OP_SYNC_FILE_RANGE] = {
126 .needs_file = 1,
127 .audit_skip = 1,
128 .prep = io_sfr_prep,
129 .issue = io_sync_file_range,
130 },
131 [IORING_OP_SENDMSG] = {
132 .needs_file = 1,
133 .unbound_nonreg_file = 1,
134 .pollout = 1,
135 .ioprio = 1,
136 .manual_alloc = 1,
137 #if defined(CONFIG_NET)
138 .prep = io_sendmsg_prep,
139 .issue = io_sendmsg,
140 #else
141 .prep = io_eopnotsupp_prep,
142 #endif
143 },
144 [IORING_OP_RECVMSG] = {
145 .needs_file = 1,
146 .unbound_nonreg_file = 1,
147 .pollin = 1,
148 .buffer_select = 1,
149 .ioprio = 1,
150 .manual_alloc = 1,
151 #if defined(CONFIG_NET)
152 .prep = io_recvmsg_prep,
153 .issue = io_recvmsg,
154 #else
155 .prep = io_eopnotsupp_prep,
156 #endif
157 },
158 [IORING_OP_TIMEOUT] = {
159 .audit_skip = 1,
160 .prep = io_timeout_prep,
161 .issue = io_timeout,
162 },
163 [IORING_OP_TIMEOUT_REMOVE] = {
164 /* used by timeout updates' prep() */
165 .audit_skip = 1,
166 .prep = io_timeout_remove_prep,
167 .issue = io_timeout_remove,
168 },
169 [IORING_OP_ACCEPT] = {
170 .needs_file = 1,
171 .unbound_nonreg_file = 1,
172 .pollin = 1,
173 .poll_exclusive = 1,
174 .ioprio = 1, /* used for flags */
175 #if defined(CONFIG_NET)
176 .prep = io_accept_prep,
177 .issue = io_accept,
178 #else
179 .prep = io_eopnotsupp_prep,
180 #endif
181 },
182 [IORING_OP_ASYNC_CANCEL] = {
183 .audit_skip = 1,
184 .prep = io_async_cancel_prep,
185 .issue = io_async_cancel,
186 },
187 [IORING_OP_LINK_TIMEOUT] = {
188 .audit_skip = 1,
189 .prep = io_link_timeout_prep,
190 .issue = io_no_issue,
191 },
192 [IORING_OP_CONNECT] = {
193 .needs_file = 1,
194 .unbound_nonreg_file = 1,
195 .pollout = 1,
196 #if defined(CONFIG_NET)
197 .prep = io_connect_prep,
198 .issue = io_connect,
199 #else
200 .prep = io_eopnotsupp_prep,
201 #endif
202 },
203 [IORING_OP_FALLOCATE] = {
204 .needs_file = 1,
205 .prep = io_fallocate_prep,
206 .issue = io_fallocate,
207 },
208 [IORING_OP_OPENAT] = {
209 .prep = io_openat_prep,
210 .issue = io_openat,
211 },
212 [IORING_OP_CLOSE] = {
213 .prep = io_close_prep,
214 .issue = io_close,
215 },
216 [IORING_OP_FILES_UPDATE] = {
217 .audit_skip = 1,
218 .iopoll = 1,
219 .prep = io_files_update_prep,
220 .issue = io_files_update,
221 },
222 [IORING_OP_STATX] = {
223 .audit_skip = 1,
224 .prep = io_statx_prep,
225 .issue = io_statx,
226 },
227 [IORING_OP_READ] = {
228 .needs_file = 1,
229 .unbound_nonreg_file = 1,
230 .pollin = 1,
231 .buffer_select = 1,
232 .plug = 1,
233 .audit_skip = 1,
234 .ioprio = 1,
235 .iopoll = 1,
236 .iopoll_queue = 1,
237 .prep = io_prep_rw,
238 .issue = io_read,
239 },
240 [IORING_OP_WRITE] = {
241 .needs_file = 1,
242 .hash_reg_file = 1,
243 .unbound_nonreg_file = 1,
244 .pollout = 1,
245 .plug = 1,
246 .audit_skip = 1,
247 .ioprio = 1,
248 .iopoll = 1,
249 .iopoll_queue = 1,
250 .prep = io_prep_rw,
251 .issue = io_write,
252 },
253 [IORING_OP_FADVISE] = {
254 .needs_file = 1,
255 .audit_skip = 1,
256 .prep = io_fadvise_prep,
257 .issue = io_fadvise,
258 },
259 [IORING_OP_MADVISE] = {
260 .audit_skip = 1,
261 .prep = io_madvise_prep,
262 .issue = io_madvise,
263 },
264 [IORING_OP_SEND] = {
265 .needs_file = 1,
266 .unbound_nonreg_file = 1,
267 .pollout = 1,
268 .audit_skip = 1,
269 .ioprio = 1,
270 .manual_alloc = 1,
271 #if defined(CONFIG_NET)
272 .prep = io_sendmsg_prep,
273 .issue = io_send,
274 #else
275 .prep = io_eopnotsupp_prep,
276 #endif
277 },
278 [IORING_OP_RECV] = {
279 .needs_file = 1,
280 .unbound_nonreg_file = 1,
281 .pollin = 1,
282 .buffer_select = 1,
283 .audit_skip = 1,
284 .ioprio = 1,
285 #if defined(CONFIG_NET)
286 .prep = io_recvmsg_prep,
287 .issue = io_recv,
288 #else
289 .prep = io_eopnotsupp_prep,
290 #endif
291 },
292 [IORING_OP_OPENAT2] = {
293 .prep = io_openat2_prep,
294 .issue = io_openat2,
295 },
296 [IORING_OP_EPOLL_CTL] = {
297 .unbound_nonreg_file = 1,
298 .audit_skip = 1,
299 #if defined(CONFIG_EPOLL)
300 .prep = io_epoll_ctl_prep,
301 .issue = io_epoll_ctl,
302 #else
303 .prep = io_eopnotsupp_prep,
304 #endif
305 },
306 [IORING_OP_SPLICE] = {
307 .needs_file = 1,
308 .hash_reg_file = 1,
309 .unbound_nonreg_file = 1,
310 .audit_skip = 1,
311 .prep = io_splice_prep,
312 .issue = io_splice,
313 },
314 [IORING_OP_PROVIDE_BUFFERS] = {
315 .audit_skip = 1,
316 .iopoll = 1,
317 .prep = io_provide_buffers_prep,
318 .issue = io_provide_buffers,
319 },
320 [IORING_OP_REMOVE_BUFFERS] = {
321 .audit_skip = 1,
322 .iopoll = 1,
323 .prep = io_remove_buffers_prep,
324 .issue = io_remove_buffers,
325 },
326 [IORING_OP_TEE] = {
327 .needs_file = 1,
328 .hash_reg_file = 1,
329 .unbound_nonreg_file = 1,
330 .audit_skip = 1,
331 .prep = io_tee_prep,
332 .issue = io_tee,
333 },
334 [IORING_OP_SHUTDOWN] = {
335 .needs_file = 1,
336 #if defined(CONFIG_NET)
337 .prep = io_shutdown_prep,
338 .issue = io_shutdown,
339 #else
340 .prep = io_eopnotsupp_prep,
341 #endif
342 },
343 [IORING_OP_RENAMEAT] = {
344 .prep = io_renameat_prep,
345 .issue = io_renameat,
346 },
347 [IORING_OP_UNLINKAT] = {
348 .prep = io_unlinkat_prep,
349 .issue = io_unlinkat,
350 },
351 [IORING_OP_MKDIRAT] = {
352 .prep = io_mkdirat_prep,
353 .issue = io_mkdirat,
354 },
355 [IORING_OP_SYMLINKAT] = {
356 .prep = io_symlinkat_prep,
357 .issue = io_symlinkat,
358 },
359 [IORING_OP_LINKAT] = {
360 .prep = io_linkat_prep,
361 .issue = io_linkat,
362 },
363 [IORING_OP_MSG_RING] = {
364 .needs_file = 1,
365 .iopoll = 1,
366 .prep = io_msg_ring_prep,
367 .issue = io_msg_ring,
368 },
369 [IORING_OP_FSETXATTR] = {
370 .needs_file = 1,
371 .prep = io_fsetxattr_prep,
372 .issue = io_fsetxattr,
373 },
374 [IORING_OP_SETXATTR] = {
375 .prep = io_setxattr_prep,
376 .issue = io_setxattr,
377 },
378 [IORING_OP_FGETXATTR] = {
379 .needs_file = 1,
380 .prep = io_fgetxattr_prep,
381 .issue = io_fgetxattr,
382 },
383 [IORING_OP_GETXATTR] = {
384 .prep = io_getxattr_prep,
385 .issue = io_getxattr,
386 },
387 [IORING_OP_SOCKET] = {
388 .audit_skip = 1,
389 #if defined(CONFIG_NET)
390 .prep = io_socket_prep,
391 .issue = io_socket,
392 #else
393 .prep = io_eopnotsupp_prep,
394 #endif
395 },
396 [IORING_OP_URING_CMD] = {
397 .needs_file = 1,
398 .plug = 1,
399 .iopoll = 1,
400 .iopoll_queue = 1,
401 .prep = io_uring_cmd_prep,
402 .issue = io_uring_cmd,
403 },
404 [IORING_OP_SEND_ZC] = {
405 .needs_file = 1,
406 .unbound_nonreg_file = 1,
407 .pollout = 1,
408 .audit_skip = 1,
409 .ioprio = 1,
410 .manual_alloc = 1,
411 #if defined(CONFIG_NET)
412 .prep = io_send_zc_prep,
413 .issue = io_send_zc,
414 #else
415 .prep = io_eopnotsupp_prep,
416 #endif
417 },
418 [IORING_OP_SENDMSG_ZC] = {
419 .needs_file = 1,
420 .unbound_nonreg_file = 1,
421 .pollout = 1,
422 .ioprio = 1,
423 .manual_alloc = 1,
424 #if defined(CONFIG_NET)
425 .prep = io_send_zc_prep,
426 .issue = io_sendmsg_zc,
427 #else
428 .prep = io_eopnotsupp_prep,
429 #endif
430 },
431 };
432
433
434 const struct io_cold_def io_cold_defs[] = {
435 [IORING_OP_NOP] = {
436 .name = "NOP",
437 },
438 [IORING_OP_READV] = {
439 .async_size = sizeof(struct io_async_rw),
440 .name = "READV",
441 .prep_async = io_readv_prep_async,
442 .cleanup = io_readv_writev_cleanup,
443 .fail = io_rw_fail,
444 },
445 [IORING_OP_WRITEV] = {
446 .async_size = sizeof(struct io_async_rw),
447 .name = "WRITEV",
448 .prep_async = io_writev_prep_async,
449 .cleanup = io_readv_writev_cleanup,
450 .fail = io_rw_fail,
451 },
452 [IORING_OP_FSYNC] = {
453 .name = "FSYNC",
454 },
455 [IORING_OP_READ_FIXED] = {
456 .async_size = sizeof(struct io_async_rw),
457 .name = "READ_FIXED",
458 .fail = io_rw_fail,
459 },
460 [IORING_OP_WRITE_FIXED] = {
461 .async_size = sizeof(struct io_async_rw),
462 .name = "WRITE_FIXED",
463 .fail = io_rw_fail,
464 },
465 [IORING_OP_POLL_ADD] = {
466 .name = "POLL_ADD",
467 },
468 [IORING_OP_POLL_REMOVE] = {
469 .name = "POLL_REMOVE",
470 },
471 [IORING_OP_SYNC_FILE_RANGE] = {
472 .name = "SYNC_FILE_RANGE",
473 },
474 [IORING_OP_SENDMSG] = {
475 .name = "SENDMSG",
476 #if defined(CONFIG_NET)
477 .async_size = sizeof(struct io_async_msghdr),
478 .prep_async = io_sendmsg_prep_async,
479 .cleanup = io_sendmsg_recvmsg_cleanup,
480 .fail = io_sendrecv_fail,
481 #endif
482 },
483 [IORING_OP_RECVMSG] = {
484 .name = "RECVMSG",
485 #if defined(CONFIG_NET)
486 .async_size = sizeof(struct io_async_msghdr),
487 .prep_async = io_recvmsg_prep_async,
488 .cleanup = io_sendmsg_recvmsg_cleanup,
489 .fail = io_sendrecv_fail,
490 #endif
491 },
492 [IORING_OP_TIMEOUT] = {
493 .async_size = sizeof(struct io_timeout_data),
494 .name = "TIMEOUT",
495 },
496 [IORING_OP_TIMEOUT_REMOVE] = {
497 .name = "TIMEOUT_REMOVE",
498 },
499 [IORING_OP_ACCEPT] = {
500 .name = "ACCEPT",
501 },
502 [IORING_OP_ASYNC_CANCEL] = {
503 .name = "ASYNC_CANCEL",
504 },
505 [IORING_OP_LINK_TIMEOUT] = {
506 .async_size = sizeof(struct io_timeout_data),
507 .name = "LINK_TIMEOUT",
508 },
509 [IORING_OP_CONNECT] = {
510 .name = "CONNECT",
511 #if defined(CONFIG_NET)
512 .async_size = sizeof(struct io_async_connect),
513 .prep_async = io_connect_prep_async,
514 #endif
515 },
516 [IORING_OP_FALLOCATE] = {
517 .name = "FALLOCATE",
518 },
519 [IORING_OP_OPENAT] = {
520 .name = "OPENAT",
521 .cleanup = io_open_cleanup,
522 },
523 [IORING_OP_CLOSE] = {
524 .name = "CLOSE",
525 },
526 [IORING_OP_FILES_UPDATE] = {
527 .name = "FILES_UPDATE",
528 },
529 [IORING_OP_STATX] = {
530 .name = "STATX",
531 .cleanup = io_statx_cleanup,
532 },
533 [IORING_OP_READ] = {
534 .async_size = sizeof(struct io_async_rw),
535 .name = "READ",
536 .fail = io_rw_fail,
537 },
538 [IORING_OP_WRITE] = {
539 .async_size = sizeof(struct io_async_rw),
540 .name = "WRITE",
541 .fail = io_rw_fail,
542 },
543 [IORING_OP_FADVISE] = {
544 .name = "FADVISE",
545 },
546 [IORING_OP_MADVISE] = {
547 .name = "MADVISE",
548 },
549 [IORING_OP_SEND] = {
550 .name = "SEND",
551 #if defined(CONFIG_NET)
552 .async_size = sizeof(struct io_async_msghdr),
553 .fail = io_sendrecv_fail,
554 .prep_async = io_send_prep_async,
555 #endif
556 },
557 [IORING_OP_RECV] = {
558 .name = "RECV",
559 #if defined(CONFIG_NET)
560 .fail = io_sendrecv_fail,
561 #endif
562 },
563 [IORING_OP_OPENAT2] = {
564 .name = "OPENAT2",
565 .cleanup = io_open_cleanup,
566 },
567 [IORING_OP_EPOLL_CTL] = {
568 .name = "EPOLL",
569 },
570 [IORING_OP_SPLICE] = {
571 .name = "SPLICE",
572 },
573 [IORING_OP_PROVIDE_BUFFERS] = {
574 .name = "PROVIDE_BUFFERS",
575 },
576 [IORING_OP_REMOVE_BUFFERS] = {
577 .name = "REMOVE_BUFFERS",
578 },
579 [IORING_OP_TEE] = {
580 .name = "TEE",
581 },
582 [IORING_OP_SHUTDOWN] = {
583 .name = "SHUTDOWN",
584 },
585 [IORING_OP_RENAMEAT] = {
586 .name = "RENAMEAT",
587 .cleanup = io_renameat_cleanup,
588 },
589 [IORING_OP_UNLINKAT] = {
590 .name = "UNLINKAT",
591 .cleanup = io_unlinkat_cleanup,
592 },
593 [IORING_OP_MKDIRAT] = {
594 .name = "MKDIRAT",
595 .cleanup = io_mkdirat_cleanup,
596 },
597 [IORING_OP_SYMLINKAT] = {
598 .name = "SYMLINKAT",
599 .cleanup = io_link_cleanup,
600 },
601 [IORING_OP_LINKAT] = {
602 .name = "LINKAT",
603 .cleanup = io_link_cleanup,
604 },
605 [IORING_OP_MSG_RING] = {
606 .name = "MSG_RING",
607 .cleanup = io_msg_ring_cleanup,
608 },
609 [IORING_OP_FSETXATTR] = {
610 .name = "FSETXATTR",
611 .cleanup = io_xattr_cleanup,
612 },
613 [IORING_OP_SETXATTR] = {
614 .name = "SETXATTR",
615 .cleanup = io_xattr_cleanup,
616 },
617 [IORING_OP_FGETXATTR] = {
618 .name = "FGETXATTR",
619 .cleanup = io_xattr_cleanup,
620 },
621 [IORING_OP_GETXATTR] = {
622 .name = "GETXATTR",
623 .cleanup = io_xattr_cleanup,
624 },
625 [IORING_OP_SOCKET] = {
626 .name = "SOCKET",
627 },
628 [IORING_OP_URING_CMD] = {
629 .name = "URING_CMD",
630 .async_size = 2 * sizeof(struct io_uring_sqe),
631 .prep_async = io_uring_cmd_prep_async,
632 },
633 [IORING_OP_SEND_ZC] = {
634 .name = "SEND_ZC",
635 #if defined(CONFIG_NET)
636 .async_size = sizeof(struct io_async_msghdr),
637 .prep_async = io_send_prep_async,
638 .cleanup = io_send_zc_cleanup,
639 .fail = io_sendrecv_fail,
640 #endif
641 },
642 [IORING_OP_SENDMSG_ZC] = {
643 .name = "SENDMSG_ZC",
644 #if defined(CONFIG_NET)
645 .async_size = sizeof(struct io_async_msghdr),
646 .prep_async = io_sendmsg_prep_async,
647 .cleanup = io_send_zc_cleanup,
648 .fail = io_sendrecv_fail,
649 #endif
650 },
651 };
652
io_uring_get_opcode(u8 opcode)653 const char *io_uring_get_opcode(u8 opcode)
654 {
655 if (opcode < IORING_OP_LAST)
656 return io_cold_defs[opcode].name;
657 return "INVALID";
658 }
659
io_uring_optable_init(void)660 void __init io_uring_optable_init(void)
661 {
662 int i;
663
664 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
665 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
666
667 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
668 BUG_ON(!io_issue_defs[i].prep);
669 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
670 BUG_ON(!io_issue_defs[i].issue);
671 WARN_ON_ONCE(!io_cold_defs[i].name);
672 }
673 }
674