1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2/*
  3 * Copyright(c) 2016 Intel Corporation.
  4 */
  5
  6#include <rdma/rdmavt_qp.h>
  7#include <rdma/ib_hdrs.h>
  8
  9/*
 10 * Convert the AETH credit code into the number of credits.
 11 */
 12static const u16 credit_table[31] = {
 13	0,                      /* 0 */
 14	1,                      /* 1 */
 15	2,                      /* 2 */
 16	3,                      /* 3 */
 17	4,                      /* 4 */
 18	6,                      /* 5 */
 19	8,                      /* 6 */
 20	12,                     /* 7 */
 21	16,                     /* 8 */
 22	24,                     /* 9 */
 23	32,                     /* A */
 24	48,                     /* B */
 25	64,                     /* C */
 26	96,                     /* D */
 27	128,                    /* E */
 28	192,                    /* F */
 29	256,                    /* 10 */
 30	384,                    /* 11 */
 31	512,                    /* 12 */
 32	768,                    /* 13 */
 33	1024,                   /* 14 */
 34	1536,                   /* 15 */
 35	2048,                   /* 16 */
 36	3072,                   /* 17 */
 37	4096,                   /* 18 */
 38	6144,                   /* 19 */
 39	8192,                   /* 1A */
 40	12288,                  /* 1B */
 41	16384,                  /* 1C */
 42	24576,                  /* 1D */
 43	32768                   /* 1E */
 44};
 45
 46/**
 47 * rvt_compute_aeth - compute the AETH (syndrome + MSN)
 48 * @qp: the queue pair to compute the AETH for
 49 *
 50 * Returns the AETH.
 51 */
 52__be32 rvt_compute_aeth(struct rvt_qp *qp)
 53{
 54	u32 aeth = qp->r_msn & IB_MSN_MASK;
 55
 56	if (qp->ibqp.srq) {
 57		/*
 58		 * Shared receive queues don't generate credits.
 59		 * Set the credit field to the invalid value.
 60		 */
 61		aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
 62	} else {
 63		u32 min, max, x;
 64		u32 credits;
 65		u32 head;
 66		u32 tail;
 67
 68		credits = READ_ONCE(qp->r_rq.kwq->count);
 69		if (credits == 0) {
 70			/* sanity check pointers before trusting them */
 71			if (qp->ip) {
 72				head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
 73				tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
 74			} else {
 75				head = READ_ONCE(qp->r_rq.kwq->head);
 76				tail = READ_ONCE(qp->r_rq.kwq->tail);
 77			}
 78			if (head >= qp->r_rq.size)
 79				head = 0;
 80			if (tail >= qp->r_rq.size)
 81				tail = 0;
 82			/*
 83			 * Compute the number of credits available (RWQEs).
 84			 * There is a small chance that the pair of reads are
 85			 * not atomic, which is OK, since the fuzziness is
 86			 * resolved as further ACKs go out.
 87			 */
 88			credits = rvt_get_rq_count(&qp->r_rq, head, tail);
 89		}
 90		/*
 91		 * Binary search the credit table to find the code to
 92		 * use.
 93		 */
 94		min = 0;
 95		max = 31;
 96		for (;;) {
 97			x = (min + max) / 2;
 98			if (credit_table[x] == credits)
 99				break;
100			if (credit_table[x] > credits) {
101				max = x;
102			} else {
103				if (min == x)
104					break;
105				min = x;
106			}
107		}
108		aeth |= x << IB_AETH_CREDIT_SHIFT;
109	}
110	return cpu_to_be32(aeth);
111}
112EXPORT_SYMBOL(rvt_compute_aeth);
113
114/**
115 * rvt_get_credit - flush the send work queue of a QP
116 * @qp: the qp who's send work queue to flush
117 * @aeth: the Acknowledge Extended Transport Header
118 *
119 * The QP s_lock should be held.
120 */
121void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
122{
123	struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
124	u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
125
126	lockdep_assert_held(&qp->s_lock);
127	/*
128	 * If the credit is invalid, we can send
129	 * as many packets as we like.  Otherwise, we have to
130	 * honor the credit field.
131	 */
132	if (credit == IB_AETH_CREDIT_INVAL) {
133		if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
134			qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
135			if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
136				qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
137				rdi->driver_f.schedule_send(qp);
138			}
139		}
140	} else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
141		/* Compute new LSN (i.e., MSN + credit) */
142		credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
143		if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
144			qp->s_lsn = credit;
145			if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
146				qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
147				rdi->driver_f.schedule_send(qp);
148			}
149		}
150	}
151}
152EXPORT_SYMBOL(rvt_get_credit);
153
154/**
155 * rvt_restart_sge - rewind the sge state for a wqe
156 * @ss: the sge state pointer
157 * @wqe: the wqe to rewind
158 * @len: the data length from the start of the wqe in bytes
159 *
160 * Returns the remaining data length.
161 */
162u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
163{
164	ss->sge = wqe->sg_list[0];
165	ss->sg_list = wqe->sg_list + 1;
166	ss->num_sge = wqe->wr.num_sge;
167	ss->total_len = wqe->length;
168	rvt_skip_sge(ss, len, false);
169	return wqe->length - len;
170}
171EXPORT_SYMBOL(rvt_restart_sge);
172