1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/kernel.h>
  3#include <linux/errno.h>
  4#include <linux/fs.h>
  5#include <linux/file.h>
  6#include <linux/mm.h>
  7#include <linux/slab.h>
  8#include <linux/namei.h>
  9#include <linux/io_uring.h>
 10#include <linux/splice.h>
 11
 12#include <uapi/linux/io_uring.h>
 13
 14#include "io_uring.h"
 15#include "splice.h"
 16
 17struct io_splice {
 18	struct file			*file_out;
 19	loff_t				off_out;
 20	loff_t				off_in;
 21	u64				len;
 22	int				splice_fd_in;
 23	unsigned int			flags;
 24	struct io_rsrc_node		*rsrc_node;
 25};
 26
 27static int __io_splice_prep(struct io_kiocb *req,
 28			    const struct io_uring_sqe *sqe)
 29{
 30	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 31	unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
 32
 33	sp->len = READ_ONCE(sqe->len);
 34	sp->flags = READ_ONCE(sqe->splice_flags);
 35	if (unlikely(sp->flags & ~valid_flags))
 36		return -EINVAL;
 37	sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
 38	sp->rsrc_node = NULL;
 39	req->flags |= REQ_F_FORCE_ASYNC;
 40	return 0;
 41}
 42
 43int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 44{
 45	if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
 46		return -EINVAL;
 47	return __io_splice_prep(req, sqe);
 48}
 49
 50void io_splice_cleanup(struct io_kiocb *req)
 51{
 52	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 53
 54	io_put_rsrc_node(req->ctx, sp->rsrc_node);
 55}
 56
 57static struct file *io_splice_get_file(struct io_kiocb *req,
 58				       unsigned int issue_flags)
 59{
 60	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 61	struct io_ring_ctx *ctx = req->ctx;
 62	struct io_rsrc_node *node;
 63	struct file *file = NULL;
 64
 65	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
 66		return io_file_get_normal(req, sp->splice_fd_in);
 67
 68	io_ring_submit_lock(ctx, issue_flags);
 69	node = io_rsrc_node_lookup(&ctx->file_table.data, sp->splice_fd_in);
 70	if (node) {
 71		node->refs++;
 72		sp->rsrc_node = node;
 73		file = io_slot_file(node);
 74		req->flags |= REQ_F_NEED_CLEANUP;
 75	}
 76	io_ring_submit_unlock(ctx, issue_flags);
 77	return file;
 78}
 79
 80int io_tee(struct io_kiocb *req, unsigned int issue_flags)
 81{
 82	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 83	struct file *out = sp->file_out;
 84	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
 85	struct file *in;
 86	ssize_t ret = 0;
 87
 88	WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
 89
 90	in = io_splice_get_file(req, issue_flags);
 91	if (!in) {
 92		ret = -EBADF;
 93		goto done;
 94	}
 95
 96	if (sp->len)
 97		ret = do_tee(in, out, sp->len, flags);
 98
 99	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
100		fput(in);
101done:
102	if (ret != sp->len)
103		req_set_fail(req);
104	io_req_set_res(req, ret, 0);
105	return IOU_OK;
106}
107
108int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
109{
110	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
111
112	sp->off_in = READ_ONCE(sqe->splice_off_in);
113	sp->off_out = READ_ONCE(sqe->off);
114	return __io_splice_prep(req, sqe);
115}
116
117int io_splice(struct io_kiocb *req, unsigned int issue_flags)
118{
119	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
120	struct file *out = sp->file_out;
121	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
122	loff_t *poff_in, *poff_out;
123	struct file *in;
124	ssize_t ret = 0;
125
126	WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
127
128	in = io_splice_get_file(req, issue_flags);
129	if (!in) {
130		ret = -EBADF;
131		goto done;
132	}
133
134	poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
135	poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
136
137	if (sp->len)
138		ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
139
140	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
141		fput(in);
142done:
143	if (ret != sp->len)
144		req_set_fail(req);
145	io_req_set_res(req, ret, 0);
146	return IOU_OK;
147}