Loading...
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7#include "rxe.h"
8
9static struct workqueue_struct *rxe_wq;
10
11int rxe_alloc_wq(void)
12{
13 rxe_wq = alloc_workqueue("rxe_wq", WQ_UNBOUND, WQ_MAX_ACTIVE);
14 if (!rxe_wq)
15 return -ENOMEM;
16
17 return 0;
18}
19
20void rxe_destroy_wq(void)
21{
22 destroy_workqueue(rxe_wq);
23}
24
25/* Check if task is idle i.e. not running, not scheduled in
26 * work queue and not draining. If so move to busy to
27 * reserve a slot in do_task() by setting to busy and taking
28 * a qp reference to cover the gap from now until the task finishes.
29 * state will move out of busy if task returns a non zero value
30 * in do_task(). If state is already busy it is raised to armed
31 * to indicate to do_task that additional pass should be made
32 * over the task.
33 * Context: caller should hold task->lock.
34 * Returns: true if state transitioned from idle to busy else false.
35 */
36static bool __reserve_if_idle(struct rxe_task *task)
37{
38 WARN_ON(rxe_read(task->qp) <= 0);
39
40 if (task->state == TASK_STATE_IDLE) {
41 rxe_get(task->qp);
42 task->state = TASK_STATE_BUSY;
43 task->num_sched++;
44 return true;
45 }
46
47 if (task->state == TASK_STATE_BUSY)
48 task->state = TASK_STATE_ARMED;
49
50 return false;
51}
52
53/* check if task is idle or drained and not currently
54 * scheduled in the work queue. This routine is
55 * called by rxe_cleanup_task or rxe_disable_task to
56 * see if the queue is empty.
57 * Context: caller should hold task->lock.
58 * Returns true if done else false.
59 */
60static bool __is_done(struct rxe_task *task)
61{
62 if (work_pending(&task->work))
63 return false;
64
65 if (task->state == TASK_STATE_IDLE ||
66 task->state == TASK_STATE_DRAINED) {
67 return true;
68 }
69
70 return false;
71}
72
73/* a locked version of __is_done */
74static bool is_done(struct rxe_task *task)
75{
76 unsigned long flags;
77 int done;
78
79 spin_lock_irqsave(&task->lock, flags);
80 done = __is_done(task);
81 spin_unlock_irqrestore(&task->lock, flags);
82
83 return done;
84}
85
86/* do_task is a wrapper for the three tasks (requester,
87 * completer, responder) and calls them in a loop until
88 * they return a non-zero value. It is called either
89 * directly by rxe_run_task or indirectly if rxe_sched_task
90 * schedules the task. They must call __reserve_if_idle to
91 * move the task to busy before calling or scheduling.
92 * The task can also be moved to drained or invalid
93 * by calls to rxe_cleanup_task or rxe_disable_task.
94 * In that case tasks which get here are not executed but
95 * just flushed. The tasks are designed to look to see if
96 * there is work to do and then do part of it before returning
97 * here with a return value of zero until all the work
98 * has been consumed then it returns a non-zero value.
99 * The number of times the task can be run is limited by
100 * max iterations so one task cannot hold the cpu forever.
101 * If the limit is hit and work remains the task is rescheduled.
102 */
103static void do_task(struct rxe_task *task)
104{
105 unsigned int iterations;
106 unsigned long flags;
107 int resched = 0;
108 int cont;
109 int ret;
110
111 WARN_ON(rxe_read(task->qp) <= 0);
112
113 spin_lock_irqsave(&task->lock, flags);
114 if (task->state >= TASK_STATE_DRAINED) {
115 rxe_put(task->qp);
116 task->num_done++;
117 spin_unlock_irqrestore(&task->lock, flags);
118 return;
119 }
120 spin_unlock_irqrestore(&task->lock, flags);
121
122 do {
123 iterations = RXE_MAX_ITERATIONS;
124 cont = 0;
125
126 do {
127 ret = task->func(task->qp);
128 } while (ret == 0 && iterations-- > 0);
129
130 spin_lock_irqsave(&task->lock, flags);
131 /* we're not done yet but we ran out of iterations.
132 * yield the cpu and reschedule the task
133 */
134 if (!ret) {
135 task->state = TASK_STATE_IDLE;
136 resched = 1;
137 goto exit;
138 }
139
140 switch (task->state) {
141 case TASK_STATE_BUSY:
142 task->state = TASK_STATE_IDLE;
143 break;
144
145 /* someone tried to schedule the task while we
146 * were running, keep going
147 */
148 case TASK_STATE_ARMED:
149 task->state = TASK_STATE_BUSY;
150 cont = 1;
151 break;
152
153 case TASK_STATE_DRAINING:
154 task->state = TASK_STATE_DRAINED;
155 break;
156
157 default:
158 WARN_ON(1);
159 rxe_dbg_qp(task->qp, "unexpected task state = %d",
160 task->state);
161 task->state = TASK_STATE_IDLE;
162 }
163
164exit:
165 if (!cont) {
166 task->num_done++;
167 if (WARN_ON(task->num_done != task->num_sched))
168 rxe_dbg_qp(
169 task->qp,
170 "%ld tasks scheduled, %ld tasks done",
171 task->num_sched, task->num_done);
172 }
173 spin_unlock_irqrestore(&task->lock, flags);
174 } while (cont);
175
176 task->ret = ret;
177
178 if (resched)
179 rxe_sched_task(task);
180
181 rxe_put(task->qp);
182}
183
184/* wrapper around do_task to fix argument for work queue */
185static void do_work(struct work_struct *work)
186{
187 do_task(container_of(work, struct rxe_task, work));
188}
189
190int rxe_init_task(struct rxe_task *task, struct rxe_qp *qp,
191 int (*func)(struct rxe_qp *))
192{
193 WARN_ON(rxe_read(qp) <= 0);
194
195 task->qp = qp;
196 task->func = func;
197 task->state = TASK_STATE_IDLE;
198 spin_lock_init(&task->lock);
199 INIT_WORK(&task->work, do_work);
200
201 return 0;
202}
203
204/* rxe_cleanup_task is only called from rxe_do_qp_cleanup in
205 * process context. The qp is already completed with no
206 * remaining references. Once the queue is drained the
207 * task is moved to invalid and returns. The qp cleanup
208 * code then calls the task functions directly without
209 * using the task struct to drain any late arriving packets
210 * or work requests.
211 */
212void rxe_cleanup_task(struct rxe_task *task)
213{
214 unsigned long flags;
215
216 spin_lock_irqsave(&task->lock, flags);
217 if (!__is_done(task) && task->state < TASK_STATE_DRAINED) {
218 task->state = TASK_STATE_DRAINING;
219 } else {
220 task->state = TASK_STATE_INVALID;
221 spin_unlock_irqrestore(&task->lock, flags);
222 return;
223 }
224 spin_unlock_irqrestore(&task->lock, flags);
225
226 /* now the task cannot be scheduled or run just wait
227 * for the previously scheduled tasks to finish.
228 */
229 while (!is_done(task))
230 cond_resched();
231
232 spin_lock_irqsave(&task->lock, flags);
233 task->state = TASK_STATE_INVALID;
234 spin_unlock_irqrestore(&task->lock, flags);
235}
236
237/* run the task inline if it is currently idle
238 * cannot call do_task holding the lock
239 */
240void rxe_run_task(struct rxe_task *task)
241{
242 unsigned long flags;
243 bool run;
244
245 WARN_ON(rxe_read(task->qp) <= 0);
246
247 spin_lock_irqsave(&task->lock, flags);
248 run = __reserve_if_idle(task);
249 spin_unlock_irqrestore(&task->lock, flags);
250
251 if (run)
252 do_task(task);
253}
254
255/* schedule the task to run later as a work queue entry.
256 * the queue_work call can be called holding
257 * the lock.
258 */
259void rxe_sched_task(struct rxe_task *task)
260{
261 unsigned long flags;
262
263 WARN_ON(rxe_read(task->qp) <= 0);
264
265 spin_lock_irqsave(&task->lock, flags);
266 if (__reserve_if_idle(task))
267 queue_work(rxe_wq, &task->work);
268 spin_unlock_irqrestore(&task->lock, flags);
269}
270
271/* rxe_disable/enable_task are only called from
272 * rxe_modify_qp in process context. Task is moved
273 * to the drained state by do_task.
274 */
275void rxe_disable_task(struct rxe_task *task)
276{
277 unsigned long flags;
278
279 WARN_ON(rxe_read(task->qp) <= 0);
280
281 spin_lock_irqsave(&task->lock, flags);
282 if (!__is_done(task) && task->state < TASK_STATE_DRAINED) {
283 task->state = TASK_STATE_DRAINING;
284 } else {
285 task->state = TASK_STATE_DRAINED;
286 spin_unlock_irqrestore(&task->lock, flags);
287 return;
288 }
289 spin_unlock_irqrestore(&task->lock, flags);
290
291 while (!is_done(task))
292 cond_resched();
293
294 spin_lock_irqsave(&task->lock, flags);
295 task->state = TASK_STATE_DRAINED;
296 spin_unlock_irqrestore(&task->lock, flags);
297}
298
299void rxe_enable_task(struct rxe_task *task)
300{
301 unsigned long flags;
302
303 WARN_ON(rxe_read(task->qp) <= 0);
304
305 spin_lock_irqsave(&task->lock, flags);
306 if (task->state == TASK_STATE_INVALID) {
307 spin_unlock_irqrestore(&task->lock, flags);
308 return;
309 }
310
311 task->state = TASK_STATE_IDLE;
312 spin_unlock_irqrestore(&task->lock, flags);
313}
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7#include "rxe.h"
8
9static struct workqueue_struct *rxe_wq;
10
11int rxe_alloc_wq(void)
12{
13 rxe_wq = alloc_workqueue("rxe_wq", WQ_UNBOUND, WQ_MAX_ACTIVE);
14 if (!rxe_wq)
15 return -ENOMEM;
16
17 return 0;
18}
19
20void rxe_destroy_wq(void)
21{
22 destroy_workqueue(rxe_wq);
23}
24
25/* Check if task is idle i.e. not running, not scheduled in
26 * work queue and not draining. If so move to busy to
27 * reserve a slot in do_task() by setting to busy and taking
28 * a qp reference to cover the gap from now until the task finishes.
29 * state will move out of busy if task returns a non zero value
30 * in do_task(). If state is already busy it is raised to armed
31 * to indicate to do_task that additional pass should be made
32 * over the task.
33 * Context: caller should hold task->lock.
34 * Returns: true if state transitioned from idle to busy else false.
35 */
36static bool __reserve_if_idle(struct rxe_task *task)
37{
38 WARN_ON(rxe_read(task->qp) <= 0);
39
40 if (task->state == TASK_STATE_IDLE) {
41 rxe_get(task->qp);
42 task->state = TASK_STATE_BUSY;
43 task->num_sched++;
44 return true;
45 }
46
47 if (task->state == TASK_STATE_BUSY)
48 task->state = TASK_STATE_ARMED;
49
50 return false;
51}
52
53/* check if task is idle or drained and not currently
54 * scheduled in the work queue. This routine is
55 * called by rxe_cleanup_task or rxe_disable_task to
56 * see if the queue is empty.
57 * Context: caller should hold task->lock.
58 * Returns true if done else false.
59 */
60static bool __is_done(struct rxe_task *task)
61{
62 if (work_pending(&task->work))
63 return false;
64
65 if (task->state == TASK_STATE_IDLE ||
66 task->state == TASK_STATE_DRAINED) {
67 return true;
68 }
69
70 return false;
71}
72
73/* a locked version of __is_done */
74static bool is_done(struct rxe_task *task)
75{
76 unsigned long flags;
77 int done;
78
79 spin_lock_irqsave(&task->lock, flags);
80 done = __is_done(task);
81 spin_unlock_irqrestore(&task->lock, flags);
82
83 return done;
84}
85
86/* do_task is a wrapper for the three tasks (requester,
87 * completer, responder) and calls them in a loop until
88 * they return a non-zero value. It is called either
89 * directly by rxe_run_task or indirectly if rxe_sched_task
90 * schedules the task. They must call __reserve_if_idle to
91 * move the task to busy before calling or scheduling.
92 * The task can also be moved to drained or invalid
93 * by calls to rxe_cleanup_task or rxe_disable_task.
94 * In that case tasks which get here are not executed but
95 * just flushed. The tasks are designed to look to see if
96 * there is work to do and then do part of it before returning
97 * here with a return value of zero until all the work
98 * has been consumed then it returns a non-zero value.
99 * The number of times the task can be run is limited by
100 * max iterations so one task cannot hold the cpu forever.
101 * If the limit is hit and work remains the task is rescheduled.
102 */
103static void do_task(struct rxe_task *task)
104{
105 unsigned int iterations;
106 unsigned long flags;
107 int resched = 0;
108 int cont;
109 int ret;
110
111 WARN_ON(rxe_read(task->qp) <= 0);
112
113 spin_lock_irqsave(&task->lock, flags);
114 if (task->state >= TASK_STATE_DRAINED) {
115 rxe_put(task->qp);
116 task->num_done++;
117 spin_unlock_irqrestore(&task->lock, flags);
118 return;
119 }
120 spin_unlock_irqrestore(&task->lock, flags);
121
122 do {
123 iterations = RXE_MAX_ITERATIONS;
124 cont = 0;
125
126 do {
127 ret = task->func(task->qp);
128 } while (ret == 0 && iterations-- > 0);
129
130 spin_lock_irqsave(&task->lock, flags);
131 /* we're not done yet but we ran out of iterations.
132 * yield the cpu and reschedule the task
133 */
134 if (!ret) {
135 task->state = TASK_STATE_IDLE;
136 resched = 1;
137 goto exit;
138 }
139
140 switch (task->state) {
141 case TASK_STATE_BUSY:
142 task->state = TASK_STATE_IDLE;
143 break;
144
145 /* someone tried to schedule the task while we
146 * were running, keep going
147 */
148 case TASK_STATE_ARMED:
149 task->state = TASK_STATE_BUSY;
150 cont = 1;
151 break;
152
153 case TASK_STATE_DRAINING:
154 task->state = TASK_STATE_DRAINED;
155 break;
156
157 default:
158 WARN_ON(1);
159 rxe_dbg_qp(task->qp, "unexpected task state = %d\n",
160 task->state);
161 task->state = TASK_STATE_IDLE;
162 }
163
164exit:
165 if (!cont) {
166 task->num_done++;
167 if (WARN_ON(task->num_done != task->num_sched))
168 rxe_dbg_qp(
169 task->qp,
170 "%ld tasks scheduled, %ld tasks done\n",
171 task->num_sched, task->num_done);
172 }
173 spin_unlock_irqrestore(&task->lock, flags);
174 } while (cont);
175
176 task->ret = ret;
177
178 if (resched)
179 rxe_sched_task(task);
180
181 rxe_put(task->qp);
182}
183
184/* wrapper around do_task to fix argument for work queue */
185static void do_work(struct work_struct *work)
186{
187 do_task(container_of(work, struct rxe_task, work));
188}
189
190int rxe_init_task(struct rxe_task *task, struct rxe_qp *qp,
191 int (*func)(struct rxe_qp *))
192{
193 WARN_ON(rxe_read(qp) <= 0);
194
195 task->qp = qp;
196 task->func = func;
197 task->state = TASK_STATE_IDLE;
198 spin_lock_init(&task->lock);
199 INIT_WORK(&task->work, do_work);
200
201 return 0;
202}
203
204/* rxe_cleanup_task is only called from rxe_do_qp_cleanup in
205 * process context. The qp is already completed with no
206 * remaining references. Once the queue is drained the
207 * task is moved to invalid and returns. The qp cleanup
208 * code then calls the task functions directly without
209 * using the task struct to drain any late arriving packets
210 * or work requests.
211 */
212void rxe_cleanup_task(struct rxe_task *task)
213{
214 unsigned long flags;
215
216 spin_lock_irqsave(&task->lock, flags);
217 if (!__is_done(task) && task->state < TASK_STATE_DRAINED) {
218 task->state = TASK_STATE_DRAINING;
219 } else {
220 task->state = TASK_STATE_INVALID;
221 spin_unlock_irqrestore(&task->lock, flags);
222 return;
223 }
224 spin_unlock_irqrestore(&task->lock, flags);
225
226 /* now the task cannot be scheduled or run just wait
227 * for the previously scheduled tasks to finish.
228 */
229 while (!is_done(task))
230 cond_resched();
231
232 spin_lock_irqsave(&task->lock, flags);
233 task->state = TASK_STATE_INVALID;
234 spin_unlock_irqrestore(&task->lock, flags);
235}
236
237/* run the task inline if it is currently idle
238 * cannot call do_task holding the lock
239 */
240void rxe_run_task(struct rxe_task *task)
241{
242 unsigned long flags;
243 bool run;
244
245 WARN_ON(rxe_read(task->qp) <= 0);
246
247 spin_lock_irqsave(&task->lock, flags);
248 run = __reserve_if_idle(task);
249 spin_unlock_irqrestore(&task->lock, flags);
250
251 if (run)
252 do_task(task);
253}
254
255/* schedule the task to run later as a work queue entry.
256 * the queue_work call can be called holding
257 * the lock.
258 */
259void rxe_sched_task(struct rxe_task *task)
260{
261 unsigned long flags;
262
263 WARN_ON(rxe_read(task->qp) <= 0);
264
265 spin_lock_irqsave(&task->lock, flags);
266 if (__reserve_if_idle(task))
267 queue_work(rxe_wq, &task->work);
268 spin_unlock_irqrestore(&task->lock, flags);
269}
270
271/* rxe_disable/enable_task are only called from
272 * rxe_modify_qp in process context. Task is moved
273 * to the drained state by do_task.
274 */
275void rxe_disable_task(struct rxe_task *task)
276{
277 unsigned long flags;
278
279 WARN_ON(rxe_read(task->qp) <= 0);
280
281 spin_lock_irqsave(&task->lock, flags);
282 if (!__is_done(task) && task->state < TASK_STATE_DRAINED) {
283 task->state = TASK_STATE_DRAINING;
284 } else {
285 task->state = TASK_STATE_DRAINED;
286 spin_unlock_irqrestore(&task->lock, flags);
287 return;
288 }
289 spin_unlock_irqrestore(&task->lock, flags);
290
291 while (!is_done(task))
292 cond_resched();
293
294 spin_lock_irqsave(&task->lock, flags);
295 task->state = TASK_STATE_DRAINED;
296 spin_unlock_irqrestore(&task->lock, flags);
297}
298
299void rxe_enable_task(struct rxe_task *task)
300{
301 unsigned long flags;
302
303 WARN_ON(rxe_read(task->qp) <= 0);
304
305 spin_lock_irqsave(&task->lock, flags);
306 if (task->state == TASK_STATE_INVALID) {
307 spin_unlock_irqrestore(&task->lock, flags);
308 return;
309 }
310
311 task->state = TASK_STATE_IDLE;
312 spin_unlock_irqrestore(&task->lock, flags);
313}