1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/kernel.h>
  3#include <linux/errno.h>
  4#include <linux/file.h>
  5#include <linux/io_uring.h>
  6#include <linux/security.h>
  7#include <linux/nospec.h>
  8
  9#include <uapi/linux/io_uring.h>
 10
 11#include "io_uring.h"
 12#include "rsrc.h"
 13#include "uring_cmd.h"
 14
 15static void io_uring_cmd_work(struct io_kiocb *req, bool *locked)
 16{
 17	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
 18
 19	ioucmd->task_work_cb(ioucmd);
 20}
 21
 22void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
 23			void (*task_work_cb)(struct io_uring_cmd *))
 24{
 25	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
 26
 27	ioucmd->task_work_cb = task_work_cb;
 28	req->io_task_work.func = io_uring_cmd_work;
 29	io_req_task_work_add(req);
 30}
 31EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task);
 32
 33static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
 34					  u64 extra1, u64 extra2)
 35{
 36	req->extra1 = extra1;
 37	req->extra2 = extra2;
 38	req->flags |= REQ_F_CQE32_INIT;
 39}
 40
 41/*
 42 * Called by consumers of io_uring_cmd, if they originally returned
 43 * -EIOCBQUEUED upon receiving the command.
 44 */
 45void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)
 46{
 47	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
 48
 49	if (ret < 0)
 50		req_set_fail(req);
 51
 52	io_req_set_res(req, ret, 0);
 53	if (req->ctx->flags & IORING_SETUP_CQE32)
 54		io_req_set_cqe32_extra(req, res2, 0);
 55	if (req->ctx->flags & IORING_SETUP_IOPOLL)
 56		/* order with io_iopoll_req_issued() checking ->iopoll_complete */
 57		smp_store_release(&req->iopoll_completed, 1);
 58	else
 59		io_req_complete_post(req, 0);
 60}
 61EXPORT_SYMBOL_GPL(io_uring_cmd_done);
 62
 63int io_uring_cmd_prep_async(struct io_kiocb *req)
 64{
 65	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
 66	size_t cmd_size;
 67
 68	BUILD_BUG_ON(uring_cmd_pdu_size(0) != 16);
 69	BUILD_BUG_ON(uring_cmd_pdu_size(1) != 80);
 70
 71	cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);
 72
 73	memcpy(req->async_data, ioucmd->cmd, cmd_size);
 74	return 0;
 75}
 76
 77int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 78{
 79	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
 80
 81	if (sqe->__pad1)
 82		return -EINVAL;
 83
 84	ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags);
 85	if (ioucmd->flags & ~IORING_URING_CMD_FIXED)
 86		return -EINVAL;
 87
 88	if (ioucmd->flags & IORING_URING_CMD_FIXED) {
 89		struct io_ring_ctx *ctx = req->ctx;
 90		u16 index;
 91
 92		req->buf_index = READ_ONCE(sqe->buf_index);
 93		if (unlikely(req->buf_index >= ctx->nr_user_bufs))
 94			return -EFAULT;
 95		index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
 96		req->imu = ctx->user_bufs[index];
 97		io_req_set_rsrc_node(req, ctx, 0);
 98	}
 99	ioucmd->cmd = sqe->cmd;
100	ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
101	return 0;
102}
103
104int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
105{
106	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
107	struct io_ring_ctx *ctx = req->ctx;
108	struct file *file = req->file;
109	int ret;
110
111	if (!req->file->f_op->uring_cmd)
112		return -EOPNOTSUPP;
113
114	ret = security_uring_cmd(ioucmd);
115	if (ret)
116		return ret;
117
118	if (ctx->flags & IORING_SETUP_SQE128)
119		issue_flags |= IO_URING_F_SQE128;
120	if (ctx->flags & IORING_SETUP_CQE32)
121		issue_flags |= IO_URING_F_CQE32;
122	if (ctx->flags & IORING_SETUP_IOPOLL) {
123		issue_flags |= IO_URING_F_IOPOLL;
124		req->iopoll_completed = 0;
125		WRITE_ONCE(ioucmd->cookie, NULL);
126	}
127
128	if (req_has_async_data(req))
129		ioucmd->cmd = req->async_data;
130
131	ret = file->f_op->uring_cmd(ioucmd, issue_flags);
132	if (ret == -EAGAIN) {
133		if (!req_has_async_data(req)) {
134			if (io_alloc_async_data(req))
135				return -ENOMEM;
136			io_uring_cmd_prep_async(req);
137		}
138		return -EAGAIN;
139	}
140
141	if (ret != -EIOCBQUEUED) {
142		if (ret < 0)
143			req_set_fail(req);
144		io_req_set_res(req, ret, 0);
145		return ret;
146	}
147
148	return IOU_ISSUE_SKIP_COMPLETE;
149}
150
151int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
152			      struct iov_iter *iter, void *ioucmd)
153{
154	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
155
156	return io_import_fixed(rw, iter, req->imu, ubuf, len);
157}
158EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed);