1// SPDX-License-Identifier: GPL-2.0
  2#ifndef IOU_RSRC_H
  3#define IOU_RSRC_H
  4
  5#include <net/af_unix.h>
  6
  7#define IO_RSRC_TAG_TABLE_SHIFT	(PAGE_SHIFT - 3)
  8#define IO_RSRC_TAG_TABLE_MAX	(1U << IO_RSRC_TAG_TABLE_SHIFT)
  9#define IO_RSRC_TAG_TABLE_MASK	(IO_RSRC_TAG_TABLE_MAX - 1)
 10
 11enum {
 12	IORING_RSRC_FILE		= 0,
 13	IORING_RSRC_BUFFER		= 1,
 14};
 15
 16struct io_rsrc_put {
 17	struct list_head list;
 18	u64 tag;
 19	union {
 20		void *rsrc;
 21		struct file *file;
 22		struct io_mapped_ubuf *buf;
 23	};
 24};
 25
 26typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
 27
 28struct io_rsrc_data {
 29	struct io_ring_ctx		*ctx;
 30
 31	u64				**tags;
 32	unsigned int			nr;
 33	rsrc_put_fn			*do_put;
 34	atomic_t			refs;
 35	struct completion		done;
 36	bool				quiesce;
 37};
 38
 39struct io_rsrc_node {
 40	struct percpu_ref		refs;
 41	struct list_head		node;
 42	struct list_head		rsrc_list;
 43	struct io_rsrc_data		*rsrc_data;
 44	struct llist_node		llist;
 45	bool				done;
 46};
 47
 48struct io_mapped_ubuf {
 49	u64		ubuf;
 50	u64		ubuf_end;
 51	unsigned int	nr_bvecs;
 52	unsigned long	acct_pages;
 53	struct bio_vec	bvec[];
 54};
 55
 56void io_rsrc_put_tw(struct callback_head *cb);
 57void io_rsrc_put_work(struct work_struct *work);
 58void io_rsrc_refs_refill(struct io_ring_ctx *ctx);
 59void io_wait_rsrc_data(struct io_rsrc_data *data);
 60void io_rsrc_node_destroy(struct io_rsrc_node *ref_node);
 61void io_rsrc_refs_drop(struct io_ring_ctx *ctx);
 62int io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
 63int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
 64			  struct io_rsrc_node *node, void *rsrc);
 65void io_rsrc_node_switch(struct io_ring_ctx *ctx,
 66			 struct io_rsrc_data *data_to_kill);
 67
 68int io_import_fixed(int ddir, struct iov_iter *iter,
 69			   struct io_mapped_ubuf *imu,
 70			   u64 buf_addr, size_t len);
 71
 72void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
 73int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
 74int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
 75			    unsigned int nr_args, u64 __user *tags);
 76void __io_sqe_files_unregister(struct io_ring_ctx *ctx);
 77int io_sqe_files_unregister(struct io_ring_ctx *ctx);
 78int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
 79			  unsigned nr_args, u64 __user *tags);
 80
 81int __io_scm_file_account(struct io_ring_ctx *ctx, struct file *file);
 82
 83#if defined(CONFIG_UNIX)
 84static inline bool io_file_need_scm(struct file *filp)
 85{
 86	return !!unix_get_socket(filp);
 87}
 88#else
 89static inline bool io_file_need_scm(struct file *filp)
 90{
 91	return false;
 92}
 93#endif
 94
 95static inline int io_scm_file_account(struct io_ring_ctx *ctx,
 96				      struct file *file)
 97{
 98	if (likely(!io_file_need_scm(file)))
 99		return 0;
100	return __io_scm_file_account(ctx, file);
101}
102
103int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
104			     unsigned nr_args);
105int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
106			    unsigned size, unsigned type);
107int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
108			unsigned int size, unsigned int type);
109
110static inline void io_rsrc_put_node(struct io_rsrc_node *node, int nr)
111{
112	percpu_ref_put_many(&node->refs, nr);
113}
114
115static inline void io_req_put_rsrc(struct io_kiocb *req)
116{
117	if (req->rsrc_node)
118		io_rsrc_put_node(req->rsrc_node, 1);
119}
120
121static inline void io_req_put_rsrc_locked(struct io_kiocb *req,
122					  struct io_ring_ctx *ctx)
123	__must_hold(&ctx->uring_lock)
124{
125	struct io_rsrc_node *node = req->rsrc_node;
126
127	if (node) {
128		if (node == ctx->rsrc_node)
129			ctx->rsrc_cached_refs++;
130		else
131			io_rsrc_put_node(node, 1);
132	}
133}
134
135static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx)
136{
137	ctx->rsrc_cached_refs--;
138	if (unlikely(ctx->rsrc_cached_refs < 0))
139		io_rsrc_refs_refill(ctx);
140}
141
142static inline void io_req_set_rsrc_node(struct io_kiocb *req,
143					struct io_ring_ctx *ctx,
144					unsigned int issue_flags)
145{
146	if (!req->rsrc_node) {
147		req->rsrc_node = ctx->rsrc_node;
148
149		if (!(issue_flags & IO_URING_F_UNLOCKED)) {
150			lockdep_assert_held(&ctx->uring_lock);
151
152			io_charge_rsrc_node(ctx);
153		} else {
154			percpu_ref_get(&req->rsrc_node->refs);
155		}
156	}
157}
158
159static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
160{
161	unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK;
162	unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT;
163
164	return &data->tags[table_idx][off];
165}
166
167int io_files_update(struct io_kiocb *req, unsigned int issue_flags);
168int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
169
170int __io_account_mem(struct user_struct *user, unsigned long nr_pages);
171
172static inline void __io_unaccount_mem(struct user_struct *user,
173				      unsigned long nr_pages)
174{
175	atomic_long_sub(nr_pages, &user->locked_vm);
176}
177
178#endif