Loading...
1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2/*
3 * Copyright(c) 2015 - 2018 Intel Corporation.
4 */
5
6#ifndef _HFI1_IOWAIT_H
7#define _HFI1_IOWAIT_H
8
9#include <linux/list.h>
10#include <linux/workqueue.h>
11#include <linux/wait.h>
12#include <linux/sched.h>
13
14#include "sdma_txreq.h"
15
16/*
17 * typedef (*restart_t)() - restart callback
18 * @work: pointer to work structure
19 */
20typedef void (*restart_t)(struct work_struct *work);
21
22#define IOWAIT_PENDING_IB 0x0
23#define IOWAIT_PENDING_TID 0x1
24
25/*
26 * A QP can have multiple Send Engines (SEs).
27 *
28 * The current use case is for supporting a TID RDMA
29 * packet build/xmit mechanism independent from verbs.
30 */
31#define IOWAIT_SES 2
32#define IOWAIT_IB_SE 0
33#define IOWAIT_TID_SE 1
34
35struct sdma_txreq;
36struct sdma_engine;
37/**
38 * @iowork: the work struct
39 * @tx_head: list of prebuilt packets
40 * @iow: the parent iowait structure
41 *
42 * This structure is the work item (process) specific
43 * details associated with the each of the two SEs of the
44 * QP.
45 *
46 * The workstruct and the queued TXs are unique to each
47 * SE.
48 */
49struct iowait;
50struct iowait_work {
51 struct work_struct iowork;
52 struct list_head tx_head;
53 struct iowait *iow;
54};
55
56/**
57 * @list: used to add/insert into QP/PQ wait lists
58 * @tx_head: overflow list of sdma_txreq's
59 * @sleep: no space callback
60 * @wakeup: space callback wakeup
61 * @sdma_drained: sdma count drained
62 * @init_priority: callback to manipulate priority
63 * @lock: lock protected head of wait queue
64 * @iowork: workqueue overhead
65 * @wait_dma: wait for sdma_busy == 0
66 * @wait_pio: wait for pio_busy == 0
67 * @sdma_busy: # of packets in flight
68 * @count: total number of descriptors in tx_head'ed list
69 * @tx_limit: limit for overflow queuing
70 * @tx_count: number of tx entry's in tx_head'ed list
71 * @flags: wait flags (one per QP)
72 * @wait: SE array for multiple legs
73 *
74 * This is to be embedded in user's state structure
75 * (QP or PQ).
76 *
77 * The sleep and wakeup members are a
78 * bit misnamed. They do not strictly
79 * speaking sleep or wake up, but they
80 * are callbacks for the ULP to implement
81 * what ever queuing/dequeuing of
82 * the embedded iowait and its containing struct
83 * when a resource shortage like SDMA ring space
84 * or PIO credit space is seen.
85 *
86 * Both potentially have locks help
87 * so sleeping is not allowed and it is not
88 * supported to submit txreqs from the wakeup
89 * call directly because of lock conflicts.
90 *
91 * The wait_dma member along with the iow
92 *
93 * The lock field is used by waiters to record
94 * the seqlock_t that guards the list head.
95 * Waiters explicity know that, but the destroy
96 * code that unwaits QPs does not.
97 */
98struct iowait {
99 struct list_head list;
100 int (*sleep)(
101 struct sdma_engine *sde,
102 struct iowait_work *wait,
103 struct sdma_txreq *tx,
104 uint seq,
105 bool pkts_sent
106 );
107 void (*wakeup)(struct iowait *wait, int reason);
108 void (*sdma_drained)(struct iowait *wait);
109 void (*init_priority)(struct iowait *wait);
110 seqlock_t *lock;
111 wait_queue_head_t wait_dma;
112 wait_queue_head_t wait_pio;
113 atomic_t sdma_busy;
114 atomic_t pio_busy;
115 u32 count;
116 u32 tx_limit;
117 u32 tx_count;
118 u8 starved_cnt;
119 u8 priority;
120 unsigned long flags;
121 struct iowait_work wait[IOWAIT_SES];
122};
123
124#define SDMA_AVAIL_REASON 0
125
126void iowait_set_flag(struct iowait *wait, u32 flag);
127bool iowait_flag_set(struct iowait *wait, u32 flag);
128void iowait_clear_flag(struct iowait *wait, u32 flag);
129
130void iowait_init(struct iowait *wait, u32 tx_limit,
131 void (*func)(struct work_struct *work),
132 void (*tidfunc)(struct work_struct *work),
133 int (*sleep)(struct sdma_engine *sde,
134 struct iowait_work *wait,
135 struct sdma_txreq *tx,
136 uint seq,
137 bool pkts_sent),
138 void (*wakeup)(struct iowait *wait, int reason),
139 void (*sdma_drained)(struct iowait *wait),
140 void (*init_priority)(struct iowait *wait));
141
142/**
143 * iowait_schedule() - schedule the default send engine work
144 * @wait: wait struct to schedule
145 * @wq: workqueue for schedule
146 * @cpu: cpu
147 */
148static inline bool iowait_schedule(struct iowait *wait,
149 struct workqueue_struct *wq, int cpu)
150{
151 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
152}
153
154/**
155 * iowait_tid_schedule - schedule the tid SE
156 * @wait: the iowait structure
157 * @wq: the work queue
158 * @cpu: the cpu
159 */
160static inline bool iowait_tid_schedule(struct iowait *wait,
161 struct workqueue_struct *wq, int cpu)
162{
163 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
164}
165
166/**
167 * iowait_sdma_drain() - wait for DMAs to drain
168 *
169 * @wait: iowait structure
170 *
171 * This will delay until the iowait sdmas have
172 * completed.
173 */
174static inline void iowait_sdma_drain(struct iowait *wait)
175{
176 wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
177}
178
179/**
180 * iowait_sdma_pending() - return sdma pending count
181 *
182 * @wait: iowait structure
183 *
184 */
185static inline int iowait_sdma_pending(struct iowait *wait)
186{
187 return atomic_read(&wait->sdma_busy);
188}
189
190/**
191 * iowait_sdma_inc - note sdma io pending
192 * @wait: iowait structure
193 */
194static inline void iowait_sdma_inc(struct iowait *wait)
195{
196 atomic_inc(&wait->sdma_busy);
197}
198
199/**
200 * iowait_sdma_add - add count to pending
201 * @wait: iowait structure
202 */
203static inline void iowait_sdma_add(struct iowait *wait, int count)
204{
205 atomic_add(count, &wait->sdma_busy);
206}
207
208/**
209 * iowait_sdma_dec - note sdma complete
210 * @wait: iowait structure
211 */
212static inline int iowait_sdma_dec(struct iowait *wait)
213{
214 if (!wait)
215 return 0;
216 return atomic_dec_and_test(&wait->sdma_busy);
217}
218
219/**
220 * iowait_pio_drain() - wait for pios to drain
221 *
222 * @wait: iowait structure
223 *
224 * This will delay until the iowait pios have
225 * completed.
226 */
227static inline void iowait_pio_drain(struct iowait *wait)
228{
229 wait_event_timeout(wait->wait_pio,
230 !atomic_read(&wait->pio_busy),
231 HZ);
232}
233
234/**
235 * iowait_pio_pending() - return pio pending count
236 *
237 * @wait: iowait structure
238 *
239 */
240static inline int iowait_pio_pending(struct iowait *wait)
241{
242 return atomic_read(&wait->pio_busy);
243}
244
245/**
246 * iowait_pio_inc - note pio pending
247 * @wait: iowait structure
248 */
249static inline void iowait_pio_inc(struct iowait *wait)
250{
251 atomic_inc(&wait->pio_busy);
252}
253
254/**
255 * iowait_pio_dec - note pio complete
256 * @wait: iowait structure
257 */
258static inline int iowait_pio_dec(struct iowait *wait)
259{
260 if (!wait)
261 return 0;
262 return atomic_dec_and_test(&wait->pio_busy);
263}
264
265/**
266 * iowait_drain_wakeup() - trigger iowait_drain() waiter
267 *
268 * @wait: iowait structure
269 *
270 * This will trigger any waiters.
271 */
272static inline void iowait_drain_wakeup(struct iowait *wait)
273{
274 wake_up(&wait->wait_dma);
275 wake_up(&wait->wait_pio);
276 if (wait->sdma_drained)
277 wait->sdma_drained(wait);
278}
279
280/**
281 * iowait_get_txhead() - get packet off of iowait list
282 *
283 * @wait: iowait_work structure
284 */
285static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
286{
287 struct sdma_txreq *tx = NULL;
288
289 if (!list_empty(&wait->tx_head)) {
290 tx = list_first_entry(
291 &wait->tx_head,
292 struct sdma_txreq,
293 list);
294 list_del_init(&tx->list);
295 }
296 return tx;
297}
298
299static inline u16 iowait_get_desc(struct iowait_work *w)
300{
301 u16 num_desc = 0;
302 struct sdma_txreq *tx = NULL;
303
304 if (!list_empty(&w->tx_head)) {
305 tx = list_first_entry(&w->tx_head, struct sdma_txreq,
306 list);
307 num_desc = tx->num_desc;
308 if (tx->flags & SDMA_TXREQ_F_VIP)
309 w->iow->priority++;
310 }
311 return num_desc;
312}
313
314static inline u32 iowait_get_all_desc(struct iowait *w)
315{
316 u32 num_desc = 0;
317
318 num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
319 num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
320 return num_desc;
321}
322
323static inline void iowait_update_priority(struct iowait_work *w)
324{
325 struct sdma_txreq *tx = NULL;
326
327 if (!list_empty(&w->tx_head)) {
328 tx = list_first_entry(&w->tx_head, struct sdma_txreq,
329 list);
330 if (tx->flags & SDMA_TXREQ_F_VIP)
331 w->iow->priority++;
332 }
333}
334
335static inline void iowait_update_all_priority(struct iowait *w)
336{
337 iowait_update_priority(&w->wait[IOWAIT_IB_SE]);
338 iowait_update_priority(&w->wait[IOWAIT_TID_SE]);
339}
340
341static inline void iowait_init_priority(struct iowait *w)
342{
343 w->priority = 0;
344 if (w->init_priority)
345 w->init_priority(w);
346}
347
348static inline void iowait_get_priority(struct iowait *w)
349{
350 iowait_init_priority(w);
351 iowait_update_all_priority(w);
352}
353
354/**
355 * iowait_queue - Put the iowait on a wait queue
356 * @pkts_sent: have some packets been sent before queuing?
357 * @w: the iowait struct
358 * @wait_head: the wait queue
359 *
360 * This function is called to insert an iowait struct into a
361 * wait queue after a resource (eg, sdma descriptor or pio
362 * buffer) is run out.
363 */
364static inline void iowait_queue(bool pkts_sent, struct iowait *w,
365 struct list_head *wait_head)
366{
367 /*
368 * To play fair, insert the iowait at the tail of the wait queue if it
369 * has already sent some packets; Otherwise, put it at the head.
370 * However, if it has priority packets to send, also put it at the
371 * head.
372 */
373 if (pkts_sent)
374 w->starved_cnt = 0;
375 else
376 w->starved_cnt++;
377
378 if (w->priority > 0 || !pkts_sent)
379 list_add(&w->list, wait_head);
380 else
381 list_add_tail(&w->list, wait_head);
382}
383
384/**
385 * iowait_starve_clear - clear the wait queue's starve count
386 * @pkts_sent: have some packets been sent?
387 * @w: the iowait struct
388 *
389 * This function is called to clear the starve count. If no
390 * packets have been sent, the starve count will not be cleared.
391 */
392static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
393{
394 if (pkts_sent)
395 w->starved_cnt = 0;
396}
397
398/* Update the top priority index */
399uint iowait_priority_update_top(struct iowait *w,
400 struct iowait *top,
401 uint idx, uint top_idx);
402
403/**
404 * iowait_packet_queued() - determine if a packet is queued
405 * @wait: the iowait_work structure
406 */
407static inline bool iowait_packet_queued(struct iowait_work *wait)
408{
409 return !list_empty(&wait->tx_head);
410}
411
412/**
413 * inc_wait_count - increment wait counts
414 * @w: the log work struct
415 * @n: the count
416 */
417static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
418{
419 if (!w)
420 return;
421 w->iow->tx_count++;
422 w->iow->count += n;
423}
424
425/**
426 * iowait_get_tid_work - return iowait_work for tid SE
427 * @w: the iowait struct
428 */
429static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
430{
431 return &w->wait[IOWAIT_TID_SE];
432}
433
434/**
435 * iowait_get_ib_work - return iowait_work for ib SE
436 * @w: the iowait struct
437 */
438static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
439{
440 return &w->wait[IOWAIT_IB_SE];
441}
442
443/**
444 * iowait_ioww_to_iow - return iowait given iowait_work
445 * @w: the iowait_work struct
446 */
447static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
448{
449 if (likely(w))
450 return w->iow;
451 return NULL;
452}
453
454void iowait_cancel_work(struct iowait *w);
455int iowait_set_work_flag(struct iowait_work *w);
456
457#endif
1#ifndef _HFI1_IOWAIT_H
2#define _HFI1_IOWAIT_H
3/*
4 * Copyright(c) 2015 - 2018 Intel Corporation.
5 *
6 * This file is provided under a dual BSD/GPLv2 license. When using or
7 * redistributing this file, you may do so under either license.
8 *
9 * GPL LICENSE SUMMARY
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * BSD LICENSE
21 *
22 * Redistribution and use in source and binary forms, with or without
23 * modification, are permitted provided that the following conditions
24 * are met:
25 *
26 * - Redistributions of source code must retain the above copyright
27 * notice, this list of conditions and the following disclaimer.
28 * - Redistributions in binary form must reproduce the above copyright
29 * notice, this list of conditions and the following disclaimer in
30 * the documentation and/or other materials provided with the
31 * distribution.
32 * - Neither the name of Intel Corporation nor the names of its
33 * contributors may be used to endorse or promote products derived
34 * from this software without specific prior written permission.
35 *
36 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
37 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
38 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
39 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
40 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
41 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
42 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
43 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
44 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
45 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
46 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47 *
48 */
49
50#include <linux/list.h>
51#include <linux/workqueue.h>
52#include <linux/wait.h>
53#include <linux/sched.h>
54
55#include "sdma_txreq.h"
56
57/*
58 * typedef (*restart_t)() - restart callback
59 * @work: pointer to work structure
60 */
61typedef void (*restart_t)(struct work_struct *work);
62
63#define IOWAIT_PENDING_IB 0x0
64#define IOWAIT_PENDING_TID 0x1
65
66/*
67 * A QP can have multiple Send Engines (SEs).
68 *
69 * The current use case is for supporting a TID RDMA
70 * packet build/xmit mechanism independent from verbs.
71 */
72#define IOWAIT_SES 2
73#define IOWAIT_IB_SE 0
74#define IOWAIT_TID_SE 1
75
76struct sdma_txreq;
77struct sdma_engine;
78/**
79 * @iowork: the work struct
80 * @tx_head: list of prebuilt packets
81 * @iow: the parent iowait structure
82 *
83 * This structure is the work item (process) specific
84 * details associated with the each of the two SEs of the
85 * QP.
86 *
87 * The workstruct and the queued TXs are unique to each
88 * SE.
89 */
90struct iowait;
91struct iowait_work {
92 struct work_struct iowork;
93 struct list_head tx_head;
94 struct iowait *iow;
95};
96
97/**
98 * @list: used to add/insert into QP/PQ wait lists
99 * @tx_head: overflow list of sdma_txreq's
100 * @sleep: no space callback
101 * @wakeup: space callback wakeup
102 * @sdma_drained: sdma count drained
103 * @init_priority: callback to manipulate priority
104 * @lock: lock protected head of wait queue
105 * @iowork: workqueue overhead
106 * @wait_dma: wait for sdma_busy == 0
107 * @wait_pio: wait for pio_busy == 0
108 * @sdma_busy: # of packets in flight
109 * @count: total number of descriptors in tx_head'ed list
110 * @tx_limit: limit for overflow queuing
111 * @tx_count: number of tx entry's in tx_head'ed list
112 * @flags: wait flags (one per QP)
113 * @wait: SE array for multiple legs
114 *
115 * This is to be embedded in user's state structure
116 * (QP or PQ).
117 *
118 * The sleep and wakeup members are a
119 * bit misnamed. They do not strictly
120 * speaking sleep or wake up, but they
121 * are callbacks for the ULP to implement
122 * what ever queuing/dequeuing of
123 * the embedded iowait and its containing struct
124 * when a resource shortage like SDMA ring space
125 * or PIO credit space is seen.
126 *
127 * Both potentially have locks help
128 * so sleeping is not allowed and it is not
129 * supported to submit txreqs from the wakeup
130 * call directly because of lock conflicts.
131 *
132 * The wait_dma member along with the iow
133 *
134 * The lock field is used by waiters to record
135 * the seqlock_t that guards the list head.
136 * Waiters explicity know that, but the destroy
137 * code that unwaits QPs does not.
138 */
139struct iowait {
140 struct list_head list;
141 int (*sleep)(
142 struct sdma_engine *sde,
143 struct iowait_work *wait,
144 struct sdma_txreq *tx,
145 uint seq,
146 bool pkts_sent
147 );
148 void (*wakeup)(struct iowait *wait, int reason);
149 void (*sdma_drained)(struct iowait *wait);
150 void (*init_priority)(struct iowait *wait);
151 seqlock_t *lock;
152 wait_queue_head_t wait_dma;
153 wait_queue_head_t wait_pio;
154 atomic_t sdma_busy;
155 atomic_t pio_busy;
156 u32 count;
157 u32 tx_limit;
158 u32 tx_count;
159 u8 starved_cnt;
160 u8 priority;
161 unsigned long flags;
162 struct iowait_work wait[IOWAIT_SES];
163};
164
165#define SDMA_AVAIL_REASON 0
166
167void iowait_set_flag(struct iowait *wait, u32 flag);
168bool iowait_flag_set(struct iowait *wait, u32 flag);
169void iowait_clear_flag(struct iowait *wait, u32 flag);
170
171void iowait_init(struct iowait *wait, u32 tx_limit,
172 void (*func)(struct work_struct *work),
173 void (*tidfunc)(struct work_struct *work),
174 int (*sleep)(struct sdma_engine *sde,
175 struct iowait_work *wait,
176 struct sdma_txreq *tx,
177 uint seq,
178 bool pkts_sent),
179 void (*wakeup)(struct iowait *wait, int reason),
180 void (*sdma_drained)(struct iowait *wait),
181 void (*init_priority)(struct iowait *wait));
182
183/**
184 * iowait_schedule() - schedule the default send engine work
185 * @wait: wait struct to schedule
186 * @wq: workqueue for schedule
187 * @cpu: cpu
188 */
189static inline bool iowait_schedule(struct iowait *wait,
190 struct workqueue_struct *wq, int cpu)
191{
192 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
193}
194
195/**
196 * iowait_tid_schedule - schedule the tid SE
197 * @wait: the iowait structure
198 * @wq: the work queue
199 * @cpu: the cpu
200 */
201static inline bool iowait_tid_schedule(struct iowait *wait,
202 struct workqueue_struct *wq, int cpu)
203{
204 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
205}
206
207/**
208 * iowait_sdma_drain() - wait for DMAs to drain
209 *
210 * @wait: iowait structure
211 *
212 * This will delay until the iowait sdmas have
213 * completed.
214 */
215static inline void iowait_sdma_drain(struct iowait *wait)
216{
217 wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
218}
219
220/**
221 * iowait_sdma_pending() - return sdma pending count
222 *
223 * @wait: iowait structure
224 *
225 */
226static inline int iowait_sdma_pending(struct iowait *wait)
227{
228 return atomic_read(&wait->sdma_busy);
229}
230
231/**
232 * iowait_sdma_inc - note sdma io pending
233 * @wait: iowait structure
234 */
235static inline void iowait_sdma_inc(struct iowait *wait)
236{
237 atomic_inc(&wait->sdma_busy);
238}
239
240/**
241 * iowait_sdma_add - add count to pending
242 * @wait: iowait structure
243 */
244static inline void iowait_sdma_add(struct iowait *wait, int count)
245{
246 atomic_add(count, &wait->sdma_busy);
247}
248
249/**
250 * iowait_sdma_dec - note sdma complete
251 * @wait: iowait structure
252 */
253static inline int iowait_sdma_dec(struct iowait *wait)
254{
255 if (!wait)
256 return 0;
257 return atomic_dec_and_test(&wait->sdma_busy);
258}
259
260/**
261 * iowait_pio_drain() - wait for pios to drain
262 *
263 * @wait: iowait structure
264 *
265 * This will delay until the iowait pios have
266 * completed.
267 */
268static inline void iowait_pio_drain(struct iowait *wait)
269{
270 wait_event_timeout(wait->wait_pio,
271 !atomic_read(&wait->pio_busy),
272 HZ);
273}
274
275/**
276 * iowait_pio_pending() - return pio pending count
277 *
278 * @wait: iowait structure
279 *
280 */
281static inline int iowait_pio_pending(struct iowait *wait)
282{
283 return atomic_read(&wait->pio_busy);
284}
285
286/**
287 * iowait_pio_inc - note pio pending
288 * @wait: iowait structure
289 */
290static inline void iowait_pio_inc(struct iowait *wait)
291{
292 atomic_inc(&wait->pio_busy);
293}
294
295/**
296 * iowait_pio_dec - note pio complete
297 * @wait: iowait structure
298 */
299static inline int iowait_pio_dec(struct iowait *wait)
300{
301 if (!wait)
302 return 0;
303 return atomic_dec_and_test(&wait->pio_busy);
304}
305
306/**
307 * iowait_drain_wakeup() - trigger iowait_drain() waiter
308 *
309 * @wait: iowait structure
310 *
311 * This will trigger any waiters.
312 */
313static inline void iowait_drain_wakeup(struct iowait *wait)
314{
315 wake_up(&wait->wait_dma);
316 wake_up(&wait->wait_pio);
317 if (wait->sdma_drained)
318 wait->sdma_drained(wait);
319}
320
321/**
322 * iowait_get_txhead() - get packet off of iowait list
323 *
324 * @wait iowait_work struture
325 */
326static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
327{
328 struct sdma_txreq *tx = NULL;
329
330 if (!list_empty(&wait->tx_head)) {
331 tx = list_first_entry(
332 &wait->tx_head,
333 struct sdma_txreq,
334 list);
335 list_del_init(&tx->list);
336 }
337 return tx;
338}
339
340static inline u16 iowait_get_desc(struct iowait_work *w)
341{
342 u16 num_desc = 0;
343 struct sdma_txreq *tx = NULL;
344
345 if (!list_empty(&w->tx_head)) {
346 tx = list_first_entry(&w->tx_head, struct sdma_txreq,
347 list);
348 num_desc = tx->num_desc;
349 if (tx->flags & SDMA_TXREQ_F_VIP)
350 w->iow->priority++;
351 }
352 return num_desc;
353}
354
355static inline u32 iowait_get_all_desc(struct iowait *w)
356{
357 u32 num_desc = 0;
358
359 num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
360 num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
361 return num_desc;
362}
363
364static inline void iowait_update_priority(struct iowait_work *w)
365{
366 struct sdma_txreq *tx = NULL;
367
368 if (!list_empty(&w->tx_head)) {
369 tx = list_first_entry(&w->tx_head, struct sdma_txreq,
370 list);
371 if (tx->flags & SDMA_TXREQ_F_VIP)
372 w->iow->priority++;
373 }
374}
375
376static inline void iowait_update_all_priority(struct iowait *w)
377{
378 iowait_update_priority(&w->wait[IOWAIT_IB_SE]);
379 iowait_update_priority(&w->wait[IOWAIT_TID_SE]);
380}
381
382static inline void iowait_init_priority(struct iowait *w)
383{
384 w->priority = 0;
385 if (w->init_priority)
386 w->init_priority(w);
387}
388
389static inline void iowait_get_priority(struct iowait *w)
390{
391 iowait_init_priority(w);
392 iowait_update_all_priority(w);
393}
394
395/**
396 * iowait_queue - Put the iowait on a wait queue
397 * @pkts_sent: have some packets been sent before queuing?
398 * @w: the iowait struct
399 * @wait_head: the wait queue
400 *
401 * This function is called to insert an iowait struct into a
402 * wait queue after a resource (eg, sdma descriptor or pio
403 * buffer) is run out.
404 */
405static inline void iowait_queue(bool pkts_sent, struct iowait *w,
406 struct list_head *wait_head)
407{
408 /*
409 * To play fair, insert the iowait at the tail of the wait queue if it
410 * has already sent some packets; Otherwise, put it at the head.
411 * However, if it has priority packets to send, also put it at the
412 * head.
413 */
414 if (pkts_sent)
415 w->starved_cnt = 0;
416 else
417 w->starved_cnt++;
418
419 if (w->priority > 0 || !pkts_sent)
420 list_add(&w->list, wait_head);
421 else
422 list_add_tail(&w->list, wait_head);
423}
424
425/**
426 * iowait_starve_clear - clear the wait queue's starve count
427 * @pkts_sent: have some packets been sent?
428 * @w: the iowait struct
429 *
430 * This function is called to clear the starve count. If no
431 * packets have been sent, the starve count will not be cleared.
432 */
433static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
434{
435 if (pkts_sent)
436 w->starved_cnt = 0;
437}
438
439/* Update the top priority index */
440uint iowait_priority_update_top(struct iowait *w,
441 struct iowait *top,
442 uint idx, uint top_idx);
443
444/**
445 * iowait_packet_queued() - determine if a packet is queued
446 * @wait: the iowait_work structure
447 */
448static inline bool iowait_packet_queued(struct iowait_work *wait)
449{
450 return !list_empty(&wait->tx_head);
451}
452
453/**
454 * inc_wait_count - increment wait counts
455 * @w: the log work struct
456 * @n: the count
457 */
458static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
459{
460 if (!w)
461 return;
462 w->iow->tx_count++;
463 w->iow->count += n;
464}
465
466/**
467 * iowait_get_tid_work - return iowait_work for tid SE
468 * @w: the iowait struct
469 */
470static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
471{
472 return &w->wait[IOWAIT_TID_SE];
473}
474
475/**
476 * iowait_get_ib_work - return iowait_work for ib SE
477 * @w: the iowait struct
478 */
479static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
480{
481 return &w->wait[IOWAIT_IB_SE];
482}
483
484/**
485 * iowait_ioww_to_iow - return iowait given iowait_work
486 * @w: the iowait_work struct
487 */
488static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
489{
490 if (likely(w))
491 return w->iow;
492 return NULL;
493}
494
495void iowait_cancel_work(struct iowait *w);
496int iowait_set_work_flag(struct iowait_work *w);
497
498#endif