Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * io_uring opcode handling table
4 */
5#include <linux/kernel.h>
6#include <linux/errno.h>
7#include <linux/fs.h>
8#include <linux/file.h>
9#include <linux/io_uring.h>
10
11#include "io_uring.h"
12#include "opdef.h"
13#include "refs.h"
14#include "tctx.h"
15#include "sqpoll.h"
16#include "fdinfo.h"
17#include "kbuf.h"
18#include "rsrc.h"
19
20#include "xattr.h"
21#include "nop.h"
22#include "fs.h"
23#include "splice.h"
24#include "sync.h"
25#include "advise.h"
26#include "openclose.h"
27#include "uring_cmd.h"
28#include "epoll.h"
29#include "statx.h"
30#include "net.h"
31#include "msg_ring.h"
32#include "timeout.h"
33#include "poll.h"
34#include "cancel.h"
35#include "rw.h"
36#include "waitid.h"
37#include "futex.h"
38
39static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
40{
41 WARN_ON_ONCE(1);
42 return -ECANCELED;
43}
44
45static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
46 const struct io_uring_sqe *sqe)
47{
48 return -EOPNOTSUPP;
49}
50
51const struct io_issue_def io_issue_defs[] = {
52 [IORING_OP_NOP] = {
53 .audit_skip = 1,
54 .iopoll = 1,
55 .prep = io_nop_prep,
56 .issue = io_nop,
57 },
58 [IORING_OP_READV] = {
59 .needs_file = 1,
60 .unbound_nonreg_file = 1,
61 .pollin = 1,
62 .buffer_select = 1,
63 .plug = 1,
64 .audit_skip = 1,
65 .ioprio = 1,
66 .iopoll = 1,
67 .iopoll_queue = 1,
68 .vectored = 1,
69 .prep = io_prep_rwv,
70 .issue = io_read,
71 },
72 [IORING_OP_WRITEV] = {
73 .needs_file = 1,
74 .hash_reg_file = 1,
75 .unbound_nonreg_file = 1,
76 .pollout = 1,
77 .plug = 1,
78 .audit_skip = 1,
79 .ioprio = 1,
80 .iopoll = 1,
81 .iopoll_queue = 1,
82 .vectored = 1,
83 .prep = io_prep_rwv,
84 .issue = io_write,
85 },
86 [IORING_OP_FSYNC] = {
87 .needs_file = 1,
88 .audit_skip = 1,
89 .prep = io_fsync_prep,
90 .issue = io_fsync,
91 },
92 [IORING_OP_READ_FIXED] = {
93 .needs_file = 1,
94 .unbound_nonreg_file = 1,
95 .pollin = 1,
96 .plug = 1,
97 .audit_skip = 1,
98 .ioprio = 1,
99 .iopoll = 1,
100 .iopoll_queue = 1,
101 .prep = io_prep_rw_fixed,
102 .issue = io_read,
103 },
104 [IORING_OP_WRITE_FIXED] = {
105 .needs_file = 1,
106 .hash_reg_file = 1,
107 .unbound_nonreg_file = 1,
108 .pollout = 1,
109 .plug = 1,
110 .audit_skip = 1,
111 .ioprio = 1,
112 .iopoll = 1,
113 .iopoll_queue = 1,
114 .prep = io_prep_rw_fixed,
115 .issue = io_write,
116 },
117 [IORING_OP_POLL_ADD] = {
118 .needs_file = 1,
119 .unbound_nonreg_file = 1,
120 .audit_skip = 1,
121 .prep = io_poll_add_prep,
122 .issue = io_poll_add,
123 },
124 [IORING_OP_POLL_REMOVE] = {
125 .audit_skip = 1,
126 .prep = io_poll_remove_prep,
127 .issue = io_poll_remove,
128 },
129 [IORING_OP_SYNC_FILE_RANGE] = {
130 .needs_file = 1,
131 .audit_skip = 1,
132 .prep = io_sfr_prep,
133 .issue = io_sync_file_range,
134 },
135 [IORING_OP_SENDMSG] = {
136 .needs_file = 1,
137 .unbound_nonreg_file = 1,
138 .pollout = 1,
139 .ioprio = 1,
140 .manual_alloc = 1,
141#if defined(CONFIG_NET)
142 .prep = io_sendmsg_prep,
143 .issue = io_sendmsg,
144#else
145 .prep = io_eopnotsupp_prep,
146#endif
147 },
148 [IORING_OP_RECVMSG] = {
149 .needs_file = 1,
150 .unbound_nonreg_file = 1,
151 .pollin = 1,
152 .buffer_select = 1,
153 .ioprio = 1,
154 .manual_alloc = 1,
155#if defined(CONFIG_NET)
156 .prep = io_recvmsg_prep,
157 .issue = io_recvmsg,
158#else
159 .prep = io_eopnotsupp_prep,
160#endif
161 },
162 [IORING_OP_TIMEOUT] = {
163 .audit_skip = 1,
164 .prep = io_timeout_prep,
165 .issue = io_timeout,
166 },
167 [IORING_OP_TIMEOUT_REMOVE] = {
168 /* used by timeout updates' prep() */
169 .audit_skip = 1,
170 .prep = io_timeout_remove_prep,
171 .issue = io_timeout_remove,
172 },
173 [IORING_OP_ACCEPT] = {
174 .needs_file = 1,
175 .unbound_nonreg_file = 1,
176 .pollin = 1,
177 .poll_exclusive = 1,
178 .ioprio = 1, /* used for flags */
179#if defined(CONFIG_NET)
180 .prep = io_accept_prep,
181 .issue = io_accept,
182#else
183 .prep = io_eopnotsupp_prep,
184#endif
185 },
186 [IORING_OP_ASYNC_CANCEL] = {
187 .audit_skip = 1,
188 .prep = io_async_cancel_prep,
189 .issue = io_async_cancel,
190 },
191 [IORING_OP_LINK_TIMEOUT] = {
192 .audit_skip = 1,
193 .prep = io_link_timeout_prep,
194 .issue = io_no_issue,
195 },
196 [IORING_OP_CONNECT] = {
197 .needs_file = 1,
198 .unbound_nonreg_file = 1,
199 .pollout = 1,
200#if defined(CONFIG_NET)
201 .prep = io_connect_prep,
202 .issue = io_connect,
203#else
204 .prep = io_eopnotsupp_prep,
205#endif
206 },
207 [IORING_OP_FALLOCATE] = {
208 .needs_file = 1,
209 .prep = io_fallocate_prep,
210 .issue = io_fallocate,
211 },
212 [IORING_OP_OPENAT] = {
213 .prep = io_openat_prep,
214 .issue = io_openat,
215 },
216 [IORING_OP_CLOSE] = {
217 .prep = io_close_prep,
218 .issue = io_close,
219 },
220 [IORING_OP_FILES_UPDATE] = {
221 .audit_skip = 1,
222 .iopoll = 1,
223 .prep = io_files_update_prep,
224 .issue = io_files_update,
225 },
226 [IORING_OP_STATX] = {
227 .audit_skip = 1,
228 .prep = io_statx_prep,
229 .issue = io_statx,
230 },
231 [IORING_OP_READ] = {
232 .needs_file = 1,
233 .unbound_nonreg_file = 1,
234 .pollin = 1,
235 .buffer_select = 1,
236 .plug = 1,
237 .audit_skip = 1,
238 .ioprio = 1,
239 .iopoll = 1,
240 .iopoll_queue = 1,
241 .prep = io_prep_rw,
242 .issue = io_read,
243 },
244 [IORING_OP_WRITE] = {
245 .needs_file = 1,
246 .hash_reg_file = 1,
247 .unbound_nonreg_file = 1,
248 .pollout = 1,
249 .plug = 1,
250 .audit_skip = 1,
251 .ioprio = 1,
252 .iopoll = 1,
253 .iopoll_queue = 1,
254 .prep = io_prep_rw,
255 .issue = io_write,
256 },
257 [IORING_OP_FADVISE] = {
258 .needs_file = 1,
259 .audit_skip = 1,
260 .prep = io_fadvise_prep,
261 .issue = io_fadvise,
262 },
263 [IORING_OP_MADVISE] = {
264 .audit_skip = 1,
265 .prep = io_madvise_prep,
266 .issue = io_madvise,
267 },
268 [IORING_OP_SEND] = {
269 .needs_file = 1,
270 .unbound_nonreg_file = 1,
271 .pollout = 1,
272 .audit_skip = 1,
273 .ioprio = 1,
274 .manual_alloc = 1,
275#if defined(CONFIG_NET)
276 .prep = io_sendmsg_prep,
277 .issue = io_send,
278#else
279 .prep = io_eopnotsupp_prep,
280#endif
281 },
282 [IORING_OP_RECV] = {
283 .needs_file = 1,
284 .unbound_nonreg_file = 1,
285 .pollin = 1,
286 .buffer_select = 1,
287 .audit_skip = 1,
288 .ioprio = 1,
289#if defined(CONFIG_NET)
290 .prep = io_recvmsg_prep,
291 .issue = io_recv,
292#else
293 .prep = io_eopnotsupp_prep,
294#endif
295 },
296 [IORING_OP_OPENAT2] = {
297 .prep = io_openat2_prep,
298 .issue = io_openat2,
299 },
300 [IORING_OP_EPOLL_CTL] = {
301 .unbound_nonreg_file = 1,
302 .audit_skip = 1,
303#if defined(CONFIG_EPOLL)
304 .prep = io_epoll_ctl_prep,
305 .issue = io_epoll_ctl,
306#else
307 .prep = io_eopnotsupp_prep,
308#endif
309 },
310 [IORING_OP_SPLICE] = {
311 .needs_file = 1,
312 .hash_reg_file = 1,
313 .unbound_nonreg_file = 1,
314 .audit_skip = 1,
315 .prep = io_splice_prep,
316 .issue = io_splice,
317 },
318 [IORING_OP_PROVIDE_BUFFERS] = {
319 .audit_skip = 1,
320 .iopoll = 1,
321 .prep = io_provide_buffers_prep,
322 .issue = io_provide_buffers,
323 },
324 [IORING_OP_REMOVE_BUFFERS] = {
325 .audit_skip = 1,
326 .iopoll = 1,
327 .prep = io_remove_buffers_prep,
328 .issue = io_remove_buffers,
329 },
330 [IORING_OP_TEE] = {
331 .needs_file = 1,
332 .hash_reg_file = 1,
333 .unbound_nonreg_file = 1,
334 .audit_skip = 1,
335 .prep = io_tee_prep,
336 .issue = io_tee,
337 },
338 [IORING_OP_SHUTDOWN] = {
339 .needs_file = 1,
340#if defined(CONFIG_NET)
341 .prep = io_shutdown_prep,
342 .issue = io_shutdown,
343#else
344 .prep = io_eopnotsupp_prep,
345#endif
346 },
347 [IORING_OP_RENAMEAT] = {
348 .prep = io_renameat_prep,
349 .issue = io_renameat,
350 },
351 [IORING_OP_UNLINKAT] = {
352 .prep = io_unlinkat_prep,
353 .issue = io_unlinkat,
354 },
355 [IORING_OP_MKDIRAT] = {
356 .prep = io_mkdirat_prep,
357 .issue = io_mkdirat,
358 },
359 [IORING_OP_SYMLINKAT] = {
360 .prep = io_symlinkat_prep,
361 .issue = io_symlinkat,
362 },
363 [IORING_OP_LINKAT] = {
364 .prep = io_linkat_prep,
365 .issue = io_linkat,
366 },
367 [IORING_OP_MSG_RING] = {
368 .needs_file = 1,
369 .iopoll = 1,
370 .prep = io_msg_ring_prep,
371 .issue = io_msg_ring,
372 },
373 [IORING_OP_FSETXATTR] = {
374 .needs_file = 1,
375 .prep = io_fsetxattr_prep,
376 .issue = io_fsetxattr,
377 },
378 [IORING_OP_SETXATTR] = {
379 .prep = io_setxattr_prep,
380 .issue = io_setxattr,
381 },
382 [IORING_OP_FGETXATTR] = {
383 .needs_file = 1,
384 .prep = io_fgetxattr_prep,
385 .issue = io_fgetxattr,
386 },
387 [IORING_OP_GETXATTR] = {
388 .prep = io_getxattr_prep,
389 .issue = io_getxattr,
390 },
391 [IORING_OP_SOCKET] = {
392 .audit_skip = 1,
393#if defined(CONFIG_NET)
394 .prep = io_socket_prep,
395 .issue = io_socket,
396#else
397 .prep = io_eopnotsupp_prep,
398#endif
399 },
400 [IORING_OP_URING_CMD] = {
401 .needs_file = 1,
402 .plug = 1,
403 .iopoll = 1,
404 .iopoll_queue = 1,
405 .prep = io_uring_cmd_prep,
406 .issue = io_uring_cmd,
407 },
408 [IORING_OP_SEND_ZC] = {
409 .needs_file = 1,
410 .unbound_nonreg_file = 1,
411 .pollout = 1,
412 .audit_skip = 1,
413 .ioprio = 1,
414 .manual_alloc = 1,
415#if defined(CONFIG_NET)
416 .prep = io_send_zc_prep,
417 .issue = io_send_zc,
418#else
419 .prep = io_eopnotsupp_prep,
420#endif
421 },
422 [IORING_OP_SENDMSG_ZC] = {
423 .needs_file = 1,
424 .unbound_nonreg_file = 1,
425 .pollout = 1,
426 .ioprio = 1,
427 .manual_alloc = 1,
428#if defined(CONFIG_NET)
429 .prep = io_send_zc_prep,
430 .issue = io_sendmsg_zc,
431#else
432 .prep = io_eopnotsupp_prep,
433#endif
434 },
435 [IORING_OP_READ_MULTISHOT] = {
436 .needs_file = 1,
437 .unbound_nonreg_file = 1,
438 .pollin = 1,
439 .buffer_select = 1,
440 .audit_skip = 1,
441 .prep = io_read_mshot_prep,
442 .issue = io_read_mshot,
443 },
444 [IORING_OP_WAITID] = {
445 .prep = io_waitid_prep,
446 .issue = io_waitid,
447 },
448 [IORING_OP_FUTEX_WAIT] = {
449#if defined(CONFIG_FUTEX)
450 .prep = io_futex_prep,
451 .issue = io_futex_wait,
452#else
453 .prep = io_eopnotsupp_prep,
454#endif
455 },
456 [IORING_OP_FUTEX_WAKE] = {
457#if defined(CONFIG_FUTEX)
458 .prep = io_futex_prep,
459 .issue = io_futex_wake,
460#else
461 .prep = io_eopnotsupp_prep,
462#endif
463 },
464 [IORING_OP_FUTEX_WAITV] = {
465#if defined(CONFIG_FUTEX)
466 .prep = io_futexv_prep,
467 .issue = io_futexv_wait,
468#else
469 .prep = io_eopnotsupp_prep,
470#endif
471 },
472 [IORING_OP_FIXED_FD_INSTALL] = {
473 .needs_file = 1,
474 .prep = io_install_fixed_fd_prep,
475 .issue = io_install_fixed_fd,
476 },
477};
478
479const struct io_cold_def io_cold_defs[] = {
480 [IORING_OP_NOP] = {
481 .name = "NOP",
482 },
483 [IORING_OP_READV] = {
484 .async_size = sizeof(struct io_async_rw),
485 .name = "READV",
486 .prep_async = io_readv_prep_async,
487 .cleanup = io_readv_writev_cleanup,
488 .fail = io_rw_fail,
489 },
490 [IORING_OP_WRITEV] = {
491 .async_size = sizeof(struct io_async_rw),
492 .name = "WRITEV",
493 .prep_async = io_writev_prep_async,
494 .cleanup = io_readv_writev_cleanup,
495 .fail = io_rw_fail,
496 },
497 [IORING_OP_FSYNC] = {
498 .name = "FSYNC",
499 },
500 [IORING_OP_READ_FIXED] = {
501 .async_size = sizeof(struct io_async_rw),
502 .name = "READ_FIXED",
503 .fail = io_rw_fail,
504 },
505 [IORING_OP_WRITE_FIXED] = {
506 .async_size = sizeof(struct io_async_rw),
507 .name = "WRITE_FIXED",
508 .fail = io_rw_fail,
509 },
510 [IORING_OP_POLL_ADD] = {
511 .name = "POLL_ADD",
512 },
513 [IORING_OP_POLL_REMOVE] = {
514 .name = "POLL_REMOVE",
515 },
516 [IORING_OP_SYNC_FILE_RANGE] = {
517 .name = "SYNC_FILE_RANGE",
518 },
519 [IORING_OP_SENDMSG] = {
520 .name = "SENDMSG",
521#if defined(CONFIG_NET)
522 .async_size = sizeof(struct io_async_msghdr),
523 .prep_async = io_sendmsg_prep_async,
524 .cleanup = io_sendmsg_recvmsg_cleanup,
525 .fail = io_sendrecv_fail,
526#endif
527 },
528 [IORING_OP_RECVMSG] = {
529 .name = "RECVMSG",
530#if defined(CONFIG_NET)
531 .async_size = sizeof(struct io_async_msghdr),
532 .prep_async = io_recvmsg_prep_async,
533 .cleanup = io_sendmsg_recvmsg_cleanup,
534 .fail = io_sendrecv_fail,
535#endif
536 },
537 [IORING_OP_TIMEOUT] = {
538 .async_size = sizeof(struct io_timeout_data),
539 .name = "TIMEOUT",
540 },
541 [IORING_OP_TIMEOUT_REMOVE] = {
542 .name = "TIMEOUT_REMOVE",
543 },
544 [IORING_OP_ACCEPT] = {
545 .name = "ACCEPT",
546 },
547 [IORING_OP_ASYNC_CANCEL] = {
548 .name = "ASYNC_CANCEL",
549 },
550 [IORING_OP_LINK_TIMEOUT] = {
551 .async_size = sizeof(struct io_timeout_data),
552 .name = "LINK_TIMEOUT",
553 },
554 [IORING_OP_CONNECT] = {
555 .name = "CONNECT",
556#if defined(CONFIG_NET)
557 .async_size = sizeof(struct io_async_connect),
558 .prep_async = io_connect_prep_async,
559#endif
560 },
561 [IORING_OP_FALLOCATE] = {
562 .name = "FALLOCATE",
563 },
564 [IORING_OP_OPENAT] = {
565 .name = "OPENAT",
566 .cleanup = io_open_cleanup,
567 },
568 [IORING_OP_CLOSE] = {
569 .name = "CLOSE",
570 },
571 [IORING_OP_FILES_UPDATE] = {
572 .name = "FILES_UPDATE",
573 },
574 [IORING_OP_STATX] = {
575 .name = "STATX",
576 .cleanup = io_statx_cleanup,
577 },
578 [IORING_OP_READ] = {
579 .async_size = sizeof(struct io_async_rw),
580 .name = "READ",
581 .fail = io_rw_fail,
582 },
583 [IORING_OP_WRITE] = {
584 .async_size = sizeof(struct io_async_rw),
585 .name = "WRITE",
586 .fail = io_rw_fail,
587 },
588 [IORING_OP_FADVISE] = {
589 .name = "FADVISE",
590 },
591 [IORING_OP_MADVISE] = {
592 .name = "MADVISE",
593 },
594 [IORING_OP_SEND] = {
595 .name = "SEND",
596#if defined(CONFIG_NET)
597 .async_size = sizeof(struct io_async_msghdr),
598 .fail = io_sendrecv_fail,
599 .prep_async = io_send_prep_async,
600#endif
601 },
602 [IORING_OP_RECV] = {
603 .name = "RECV",
604#if defined(CONFIG_NET)
605 .fail = io_sendrecv_fail,
606#endif
607 },
608 [IORING_OP_OPENAT2] = {
609 .name = "OPENAT2",
610 .cleanup = io_open_cleanup,
611 },
612 [IORING_OP_EPOLL_CTL] = {
613 .name = "EPOLL",
614 },
615 [IORING_OP_SPLICE] = {
616 .name = "SPLICE",
617 },
618 [IORING_OP_PROVIDE_BUFFERS] = {
619 .name = "PROVIDE_BUFFERS",
620 },
621 [IORING_OP_REMOVE_BUFFERS] = {
622 .name = "REMOVE_BUFFERS",
623 },
624 [IORING_OP_TEE] = {
625 .name = "TEE",
626 },
627 [IORING_OP_SHUTDOWN] = {
628 .name = "SHUTDOWN",
629 },
630 [IORING_OP_RENAMEAT] = {
631 .name = "RENAMEAT",
632 .cleanup = io_renameat_cleanup,
633 },
634 [IORING_OP_UNLINKAT] = {
635 .name = "UNLINKAT",
636 .cleanup = io_unlinkat_cleanup,
637 },
638 [IORING_OP_MKDIRAT] = {
639 .name = "MKDIRAT",
640 .cleanup = io_mkdirat_cleanup,
641 },
642 [IORING_OP_SYMLINKAT] = {
643 .name = "SYMLINKAT",
644 .cleanup = io_link_cleanup,
645 },
646 [IORING_OP_LINKAT] = {
647 .name = "LINKAT",
648 .cleanup = io_link_cleanup,
649 },
650 [IORING_OP_MSG_RING] = {
651 .name = "MSG_RING",
652 .cleanup = io_msg_ring_cleanup,
653 },
654 [IORING_OP_FSETXATTR] = {
655 .name = "FSETXATTR",
656 .cleanup = io_xattr_cleanup,
657 },
658 [IORING_OP_SETXATTR] = {
659 .name = "SETXATTR",
660 .cleanup = io_xattr_cleanup,
661 },
662 [IORING_OP_FGETXATTR] = {
663 .name = "FGETXATTR",
664 .cleanup = io_xattr_cleanup,
665 },
666 [IORING_OP_GETXATTR] = {
667 .name = "GETXATTR",
668 .cleanup = io_xattr_cleanup,
669 },
670 [IORING_OP_SOCKET] = {
671 .name = "SOCKET",
672 },
673 [IORING_OP_URING_CMD] = {
674 .name = "URING_CMD",
675 .async_size = 2 * sizeof(struct io_uring_sqe),
676 .prep_async = io_uring_cmd_prep_async,
677 },
678 [IORING_OP_SEND_ZC] = {
679 .name = "SEND_ZC",
680#if defined(CONFIG_NET)
681 .async_size = sizeof(struct io_async_msghdr),
682 .prep_async = io_send_prep_async,
683 .cleanup = io_send_zc_cleanup,
684 .fail = io_sendrecv_fail,
685#endif
686 },
687 [IORING_OP_SENDMSG_ZC] = {
688 .name = "SENDMSG_ZC",
689#if defined(CONFIG_NET)
690 .async_size = sizeof(struct io_async_msghdr),
691 .prep_async = io_sendmsg_prep_async,
692 .cleanup = io_send_zc_cleanup,
693 .fail = io_sendrecv_fail,
694#endif
695 },
696 [IORING_OP_READ_MULTISHOT] = {
697 .name = "READ_MULTISHOT",
698 },
699 [IORING_OP_WAITID] = {
700 .name = "WAITID",
701 .async_size = sizeof(struct io_waitid_async),
702 },
703 [IORING_OP_FUTEX_WAIT] = {
704 .name = "FUTEX_WAIT",
705 },
706 [IORING_OP_FUTEX_WAKE] = {
707 .name = "FUTEX_WAKE",
708 },
709 [IORING_OP_FUTEX_WAITV] = {
710 .name = "FUTEX_WAITV",
711 },
712 [IORING_OP_FIXED_FD_INSTALL] = {
713 .name = "FIXED_FD_INSTALL",
714 },
715};
716
717const char *io_uring_get_opcode(u8 opcode)
718{
719 if (opcode < IORING_OP_LAST)
720 return io_cold_defs[opcode].name;
721 return "INVALID";
722}
723
724void __init io_uring_optable_init(void)
725{
726 int i;
727
728 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
729 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
730
731 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
732 BUG_ON(!io_issue_defs[i].prep);
733 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
734 BUG_ON(!io_issue_defs[i].issue);
735 WARN_ON_ONCE(!io_cold_defs[i].name);
736 }
737}