1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * SolidRun DPU driver for control plane
  4 *
  5 * Copyright (C) 2022-2023 SolidRun
  6 *
  7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
  8 *
  9 */
 10
 11#include <linux/iopoll.h>
 12
 13#include "snet_vdpa.h"
 14
 15enum snet_ctrl_opcodes {
 16	SNET_CTRL_OP_DESTROY = 1,
 17	SNET_CTRL_OP_READ_VQ_STATE,
 18	SNET_CTRL_OP_SUSPEND,
 19	SNET_CTRL_OP_RESUME,
 20};
 21
 22#define SNET_CTRL_TIMEOUT	        2000000
 23
 24#define SNET_CTRL_DATA_SIZE_MASK	0x0000FFFF
 25#define SNET_CTRL_IN_PROCESS_MASK	0x00010000
 26#define SNET_CTRL_CHUNK_RDY_MASK	0x00020000
 27#define SNET_CTRL_ERROR_MASK		0x0FFC0000
 28
 29#define SNET_VAL_TO_ERR(val)		(-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
 30#define SNET_EMPTY_CTRL(val)		(((val) & SNET_CTRL_ERROR_MASK) || \
 31						!((val) & SNET_CTRL_IN_PROCESS_MASK))
 32#define SNET_DATA_READY(val)		((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))
 33
 34/* Control register used to read data from the DPU */
 35struct snet_ctrl_reg_ctrl {
 36	/* Chunk size in 4B words */
 37	u16 data_size;
 38	/* We are in the middle of a command */
 39	u16 in_process:1;
 40	/* A data chunk is ready and can be consumed */
 41	u16 chunk_ready:1;
 42	/* Error code */
 43	u16 error:10;
 44	/* Saved for future usage */
 45	u16 rsvd:4;
 46};
 47
 48/* Opcode register */
 49struct snet_ctrl_reg_op {
 50	u16 opcode;
 51	/* Only if VQ index is relevant for the command */
 52	u16 vq_idx;
 53};
 54
 55struct snet_ctrl_regs {
 56	struct snet_ctrl_reg_op op;
 57	struct snet_ctrl_reg_ctrl ctrl;
 58	u32 rsvd;
 59	u32 data[];
 60};
 61
 62static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)
 63{
 64	return snet->bar + snet->psnet->cfg.ctrl_off;
 65}
 66
 67static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)
 68{
 69	u32 val;
 70
 71	return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_EMPTY_CTRL(val), 10,
 72				  SNET_CTRL_TIMEOUT);
 73}
 74
 75static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)
 76{
 77	u32 val;
 78
 79	return readx_poll_timeout(ioread32, &regs->op, val, !val, 10, SNET_CTRL_TIMEOUT);
 80}
 81
 82static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)
 83{
 84	u32 val;
 85
 86	return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_DATA_READY(val), 10,
 87				  SNET_CTRL_TIMEOUT);
 88}
 89
 90static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)
 91{
 92	return ioread32(&ctrl_regs->data[word_idx]);
 93}
 94
 95static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)
 96{
 97	return ioread32(&ctrl_regs->ctrl);
 98}
 99
100static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
101{
102	iowrite32(val, &ctrl_regs->ctrl);
103}
104
105static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
106{
107	iowrite32(val, &ctrl_regs->op);
108}
109
110static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)
111{
112	/* Wait until the DPU finishes completely.
113	 * It will clear the opcode register.
114	 */
115	return snet_wait_for_empty_op(ctrl_regs);
116}
117
118/* Reading ctrl from the DPU:
119 * buf_size must be 4B aligned
120 *
121 * Steps:
122 *
123 * (1) Verify that the DPU is not in the middle of another operation by
124 *     reading the in_process and error bits in the control register.
125 * (2) Write the request opcode and the VQ idx in the opcode register
126 *     and write the buffer size in the control register.
127 * (3) Start readind chunks of data, chunk_ready bit indicates that a
128 *     data chunk is available, we signal that we read the data by clearing the bit.
129 * (4) Detect that the transfer is completed when the in_process bit
130 *     in the control register is cleared or when the an error appears.
131 */
132static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,
133				   u32 buf_size)
134{
135	struct pci_dev *pdev = snet->pdev;
136	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
137	u32 *bfr_ptr = (u32 *)buffer;
138	u32 val;
139	u16 buf_words;
140	int ret;
141	u16 words, i, tot_words = 0;
142
143	/* Supported for config 2+ */
144	if (!SNET_CFG_VER(snet, 2))
145		return -EOPNOTSUPP;
146
147	if (!IS_ALIGNED(buf_size, 4))
148		return -EINVAL;
149
150	mutex_lock(&snet->ctrl_lock);
151
152	buf_words = buf_size / 4;
153
154	/* Make sure control register is empty */
155	ret = snet_wait_for_empty_ctrl(regs);
156	if (ret) {
157		SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
158		goto exit;
159	}
160
161	/* We need to write the buffer size in the control register, and the opcode + vq index in
162	 * the opcode register.
163	 * We use a spinlock to serialize the writes.
164	 */
165	spin_lock(&snet->ctrl_spinlock);
166
167	snet_write_ctrl(regs, buf_words);
168	snet_write_op(regs, opcode | (vq_idx << 16));
169
170	spin_unlock(&snet->ctrl_spinlock);
171
172	while (buf_words != tot_words) {
173		ret = snet_wait_for_data(regs);
174		if (ret) {
175			SNET_WARN(pdev, "Timeout waiting for control data\n");
176			goto exit;
177		}
178
179		val = snet_read_ctrl(regs);
180
181		/* Error? */
182		if (val & SNET_CTRL_ERROR_MASK) {
183			ret = SNET_VAL_TO_ERR(val);
184			SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);
185			goto exit;
186		}
187
188		words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);
189
190		for (i = 0; i < words; i++) {
191			*bfr_ptr = snet_read32_word(regs, i);
192			bfr_ptr++;
193		}
194
195		tot_words += words;
196
197		/* Is the job completed? */
198		if (!(val & SNET_CTRL_IN_PROCESS_MASK))
199			break;
200
201		/* Clear the chunk ready bit and continue */
202		val &= ~SNET_CTRL_CHUNK_RDY_MASK;
203		snet_write_ctrl(regs, val);
204	}
205
206	ret = snet_wait_for_dpu_completion(regs);
207	if (ret)
208		SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");
209
210exit:
211	mutex_unlock(&snet->ctrl_lock);
212	return ret;
213}
214
215/* Send a control message to the DPU using the old mechanism
216 * used with config version 1.
217 */
218static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)
219{
220	struct pci_dev *pdev = snet->pdev;
221	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
222	int ret;
223
224	mutex_lock(&snet->ctrl_lock);
225
226	/* Old mechanism uses just 1 register, the opcode register.
227	 * Make sure that the opcode register is empty, and that the DPU isn't
228	 * processing an old message.
229	 */
230	ret = snet_wait_for_empty_op(regs);
231	if (ret) {
232		SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");
233		goto exit;
234	}
235
236	/* Write the message */
237	snet_write_op(regs, opcode);
238
239	/* DPU ACKs the message by clearing the opcode register */
240	ret = snet_wait_for_empty_op(regs);
241	if (ret)
242		SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");
243
244exit:
245	mutex_unlock(&snet->ctrl_lock);
246	return ret;
247}
248
249/* Send a control message to the DPU.
250 * A control message is a message without payload.
251 */
252static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)
253{
254	struct pci_dev *pdev = snet->pdev;
255	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
256	u32 val;
257	int ret;
258
259	/* If config version is not 2+, use the old mechanism */
260	if (!SNET_CFG_VER(snet, 2))
261		return snet_send_ctrl_msg_old(snet, opcode);
262
263	mutex_lock(&snet->ctrl_lock);
264
265	/* Make sure control register is empty */
266	ret = snet_wait_for_empty_ctrl(regs);
267	if (ret) {
268		SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
269		goto exit;
270	}
271
272	/* We need to clear the control register and write the opcode + vq index in the opcode
273	 * register.
274	 * We use a spinlock to serialize the writes.
275	 */
276	spin_lock(&snet->ctrl_spinlock);
277
278	snet_write_ctrl(regs, 0);
279	snet_write_op(regs, opcode | (vq_idx << 16));
280
281	spin_unlock(&snet->ctrl_spinlock);
282
283	/* The DPU ACKs control messages by setting the chunk ready bit
284	 * without data.
285	 */
286	ret = snet_wait_for_data(regs);
287	if (ret) {
288		SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");
289		goto exit;
290	}
291
292	/* Check for errors */
293	val = snet_read_ctrl(regs);
294	ret = SNET_VAL_TO_ERR(val);
295
296	/* Clear the chunk ready bit */
297	val &= ~SNET_CTRL_CHUNK_RDY_MASK;
298	snet_write_ctrl(regs, val);
299
300	ret = snet_wait_for_dpu_completion(regs);
301	if (ret)
302		SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",
303			  ret);
304
305exit:
306	mutex_unlock(&snet->ctrl_lock);
307	return ret;
308}
309
310void snet_ctrl_clear(struct snet *snet)
311{
312	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
313
314	snet_write_op(regs, 0);
315}
316
317int snet_destroy_dev(struct snet *snet)
318{
319	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);
320}
321
322int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)
323{
324	return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,
325				       sizeof(*state));
326}
327
328int snet_suspend_dev(struct snet *snet)
329{
330	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);
331}
332
333int snet_resume_dev(struct snet *snet)
334{
335	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_RESUME, 0);
336}