1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/kernel.h>
  3#include <linux/errno.h>
  4#include <linux/fs.h>
  5#include <linux/file.h>
  6#include <linux/mm.h>
  7#include <linux/slab.h>
  8#include <linux/namei.h>
  9#include <linux/io_uring.h>
 10#include <linux/splice.h>
 11
 12#include <uapi/linux/io_uring.h>
 13
 14#include "io_uring.h"
 15#include "splice.h"
 16
 17struct io_splice {
 18	struct file			*file_out;
 19	loff_t				off_out;
 20	loff_t				off_in;
 21	u64				len;
 22	int				splice_fd_in;
 23	unsigned int			flags;
 24};
 25
 26static int __io_splice_prep(struct io_kiocb *req,
 27			    const struct io_uring_sqe *sqe)
 28{
 29	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 30	unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
 31
 32	sp->len = READ_ONCE(sqe->len);
 33	sp->flags = READ_ONCE(sqe->splice_flags);
 34	if (unlikely(sp->flags & ~valid_flags))
 35		return -EINVAL;
 36	sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
 37	req->flags |= REQ_F_FORCE_ASYNC;
 38	return 0;
 39}
 40
 41int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 42{
 43	if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
 44		return -EINVAL;
 45	return __io_splice_prep(req, sqe);
 46}
 47
 48int io_tee(struct io_kiocb *req, unsigned int issue_flags)
 49{
 50	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 51	struct file *out = sp->file_out;
 52	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
 53	struct file *in;
 54	ssize_t ret = 0;
 55
 56	WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
 57
 58	if (sp->flags & SPLICE_F_FD_IN_FIXED)
 59		in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags);
 60	else
 61		in = io_file_get_normal(req, sp->splice_fd_in);
 62	if (!in) {
 63		ret = -EBADF;
 64		goto done;
 65	}
 66
 67	if (sp->len)
 68		ret = do_tee(in, out, sp->len, flags);
 69
 70	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
 71		fput(in);
 72done:
 73	if (ret != sp->len)
 74		req_set_fail(req);
 75	io_req_set_res(req, ret, 0);
 76	return IOU_OK;
 77}
 78
 79int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 80{
 81	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 82
 83	sp->off_in = READ_ONCE(sqe->splice_off_in);
 84	sp->off_out = READ_ONCE(sqe->off);
 85	return __io_splice_prep(req, sqe);
 86}
 87
 88int io_splice(struct io_kiocb *req, unsigned int issue_flags)
 89{
 90	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 91	struct file *out = sp->file_out;
 92	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
 93	loff_t *poff_in, *poff_out;
 94	struct file *in;
 95	ssize_t ret = 0;
 96
 97	WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
 98
 99	if (sp->flags & SPLICE_F_FD_IN_FIXED)
100		in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags);
101	else
102		in = io_file_get_normal(req, sp->splice_fd_in);
103	if (!in) {
104		ret = -EBADF;
105		goto done;
106	}
107
108	poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
109	poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
110
111	if (sp->len)
112		ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
113
114	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
115		fput(in);
116done:
117	if (ret != sp->len)
118		req_set_fail(req);
119	io_req_set_res(req, ret, 0);
120	return IOU_OK;
121}