1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
  2/*
  3 * Copyright(c) 2018 Intel Corporation.
  4 *
  5 */
  6#include "iowait.h"
  7#include "trace_iowait.h"
  8
  9/* 1 priority == 16 starve_cnt */
 10#define IOWAIT_PRIORITY_STARVE_SHIFT 4
 11
 12void iowait_set_flag(struct iowait *wait, u32 flag)
 13{
 14	trace_hfi1_iowait_set(wait, flag);
 15	set_bit(flag, &wait->flags);
 16}
 17
 18bool iowait_flag_set(struct iowait *wait, u32 flag)
 19{
 20	return test_bit(flag, &wait->flags);
 21}
 22
 23inline void iowait_clear_flag(struct iowait *wait, u32 flag)
 24{
 25	trace_hfi1_iowait_clear(wait, flag);
 26	clear_bit(flag, &wait->flags);
 27}
 28
 29/*
 30 * iowait_init() - initialize wait structure
 31 * @wait: wait struct to initialize
 32 * @tx_limit: limit for overflow queuing
 33 * @func: restart function for workqueue
 34 * @sleep: sleep function for no space
 35 * @resume: wakeup function for no space
 36 *
 37 * This function initializes the iowait
 38 * structure embedded in the QP or PQ.
 39 *
 40 */
 41void iowait_init(struct iowait *wait, u32 tx_limit,
 42		 void (*func)(struct work_struct *work),
 43		 void (*tidfunc)(struct work_struct *work),
 44		 int (*sleep)(struct sdma_engine *sde,
 45			      struct iowait_work *wait,
 46			      struct sdma_txreq *tx,
 47			      uint seq,
 48			      bool pkts_sent),
 49		 void (*wakeup)(struct iowait *wait, int reason),
 50		 void (*sdma_drained)(struct iowait *wait),
 51		 void (*init_priority)(struct iowait *wait))
 52{
 53	int i;
 54
 55	wait->count = 0;
 56	INIT_LIST_HEAD(&wait->list);
 57	init_waitqueue_head(&wait->wait_dma);
 58	init_waitqueue_head(&wait->wait_pio);
 59	atomic_set(&wait->sdma_busy, 0);
 60	atomic_set(&wait->pio_busy, 0);
 61	wait->tx_limit = tx_limit;
 62	wait->sleep = sleep;
 63	wait->wakeup = wakeup;
 64	wait->sdma_drained = sdma_drained;
 65	wait->init_priority = init_priority;
 66	wait->flags = 0;
 67	for (i = 0; i < IOWAIT_SES; i++) {
 68		wait->wait[i].iow = wait;
 69		INIT_LIST_HEAD(&wait->wait[i].tx_head);
 70		if (i == IOWAIT_IB_SE)
 71			INIT_WORK(&wait->wait[i].iowork, func);
 72		else
 73			INIT_WORK(&wait->wait[i].iowork, tidfunc);
 74	}
 75}
 76
 77/**
 78 * iowait_cancel_work - cancel all work in iowait
 79 * @w: the iowait struct
 80 */
 81void iowait_cancel_work(struct iowait *w)
 82{
 83	cancel_work_sync(&iowait_get_ib_work(w)->iowork);
 84	/* Make sure that the iowork for TID RDMA is used */
 85	if (iowait_get_tid_work(w)->iowork.func)
 86		cancel_work_sync(&iowait_get_tid_work(w)->iowork);
 87}
 88
 89/**
 90 * iowait_set_work_flag - set work flag based on leg
 91 * @w: the iowait work struct
 92 */
 93int iowait_set_work_flag(struct iowait_work *w)
 94{
 95	if (w == &w->iow->wait[IOWAIT_IB_SE]) {
 96		iowait_set_flag(w->iow, IOWAIT_PENDING_IB);
 97		return IOWAIT_IB_SE;
 98	}
 99	iowait_set_flag(w->iow, IOWAIT_PENDING_TID);
100	return IOWAIT_TID_SE;
101}
102
103/**
104 * iowait_priority_update_top - update the top priority entry
105 * @w: the iowait struct
106 * @top: a pointer to the top priority entry
107 * @idx: the index of the current iowait in an array
108 * @top_idx: the array index for the iowait entry that has the top priority
109 *
110 * This function is called to compare the priority of a given
111 * iowait with the given top priority entry. The top index will
112 * be returned.
113 */
114uint iowait_priority_update_top(struct iowait *w,
115				struct iowait *top,
116				uint idx, uint top_idx)
117{
118	u8 cnt, tcnt;
119
120	/* Convert priority into starve_cnt and compare the total.*/
121	cnt = (w->priority << IOWAIT_PRIORITY_STARVE_SHIFT) + w->starved_cnt;
122	tcnt = (top->priority << IOWAIT_PRIORITY_STARVE_SHIFT) +
123		top->starved_cnt;
124	if (cnt > tcnt)
125		return idx;
126	else
127		return top_idx;
128}