1#ifndef INTERNAL_IO_WQ_H
  2#define INTERNAL_IO_WQ_H
  3
  4struct io_wq;
  5
  6enum {
  7	IO_WQ_WORK_CANCEL	= 1,
  8	IO_WQ_WORK_HASHED	= 2,
  9	IO_WQ_WORK_UNBOUND	= 4,
 10	IO_WQ_WORK_NO_CANCEL	= 8,
 11	IO_WQ_WORK_CONCURRENT	= 16,
 12
 13	IO_WQ_HASH_SHIFT	= 24,	/* upper 8 bits are used for hash key */
 14};
 15
 16enum io_wq_cancel {
 17	IO_WQ_CANCEL_OK,	/* cancelled before started */
 18	IO_WQ_CANCEL_RUNNING,	/* found, running, and attempted cancelled */
 19	IO_WQ_CANCEL_NOTFOUND,	/* work not found */
 20};
 21
 22struct io_wq_work_node {
 23	struct io_wq_work_node *next;
 24};
 25
 26struct io_wq_work_list {
 27	struct io_wq_work_node *first;
 28	struct io_wq_work_node *last;
 29};
 30
 31static inline void wq_list_add_after(struct io_wq_work_node *node,
 32				     struct io_wq_work_node *pos,
 33				     struct io_wq_work_list *list)
 34{
 35	struct io_wq_work_node *next = pos->next;
 36
 37	pos->next = node;
 38	node->next = next;
 39	if (!next)
 40		list->last = node;
 41}
 42
 43static inline void wq_list_add_tail(struct io_wq_work_node *node,
 44				    struct io_wq_work_list *list)
 45{
 46	if (!list->first) {
 47		list->last = node;
 48		WRITE_ONCE(list->first, node);
 49	} else {
 50		list->last->next = node;
 51		list->last = node;
 52	}
 53}
 54
 55static inline void wq_list_cut(struct io_wq_work_list *list,
 56			       struct io_wq_work_node *last,
 57			       struct io_wq_work_node *prev)
 58{
 59	/* first in the list, if prev==NULL */
 60	if (!prev)
 61		WRITE_ONCE(list->first, last->next);
 62	else
 63		prev->next = last->next;
 64
 65	if (last == list->last)
 66		list->last = prev;
 67	last->next = NULL;
 68}
 69
 70static inline void wq_list_del(struct io_wq_work_list *list,
 71			       struct io_wq_work_node *node,
 72			       struct io_wq_work_node *prev)
 73{
 74	wq_list_cut(list, node, prev);
 75}
 76
 77#define wq_list_for_each(pos, prv, head)			\
 78	for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next)
 79
 80#define wq_list_empty(list)	(READ_ONCE((list)->first) == NULL)
 81#define INIT_WQ_LIST(list)	do {				\
 82	(list)->first = NULL;					\
 83	(list)->last = NULL;					\
 84} while (0)
 85
 86struct io_wq_work {
 87	struct io_wq_work_node list;
 88	struct files_struct *files;
 89	struct mm_struct *mm;
 90	const struct cred *creds;
 91	struct fs_struct *fs;
 92	unsigned long fsize;
 93	unsigned flags;
 94};
 95
 96static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
 97{
 98	if (!work->list.next)
 99		return NULL;
100
101	return container_of(work->list.next, struct io_wq_work, list);
102}
103
104typedef void (free_work_fn)(struct io_wq_work *);
105typedef struct io_wq_work *(io_wq_work_fn)(struct io_wq_work *);
106
107struct io_wq_data {
108	struct user_struct *user;
109
110	io_wq_work_fn *do_work;
111	free_work_fn *free_work;
112};
113
114struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);
115bool io_wq_get(struct io_wq *wq, struct io_wq_data *data);
116void io_wq_destroy(struct io_wq *wq);
117
118void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);
119void io_wq_hash_work(struct io_wq_work *work, void *val);
120
121static inline bool io_wq_is_hashed(struct io_wq_work *work)
122{
123	return work->flags & IO_WQ_WORK_HASHED;
124}
125
126void io_wq_cancel_all(struct io_wq *wq);
127enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork);
128
129typedef bool (work_cancel_fn)(struct io_wq_work *, void *);
130
131enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
132					void *data, bool cancel_all);
133
134struct task_struct *io_wq_get_task(struct io_wq *wq);
135
136#if defined(CONFIG_IO_WQ)
137extern void io_wq_worker_sleeping(struct task_struct *);
138extern void io_wq_worker_running(struct task_struct *);
139#else
140static inline void io_wq_worker_sleeping(struct task_struct *tsk)
141{
142}
143static inline void io_wq_worker_running(struct task_struct *tsk)
144{
145}
146#endif
147
148static inline bool io_wq_current_is_worker(void)
149{
150	return in_task() && (current->flags & PF_IO_WORKER);
151}
152#endif