1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Data Object Exchange
  4 *	PCIe r6.0, sec 6.30 DOE
  5 *
  6 * Copyright (C) 2021 Huawei
  7 *	Jonathan Cameron <Jonathan.Cameron@huawei.com>
  8 *
  9 * Copyright (C) 2022 Intel Corporation
 10 *	Ira Weiny <ira.weiny@intel.com>
 11 */
 12
 13#define dev_fmt(fmt) "DOE: " fmt
 14
 15#include <linux/bitfield.h>
 16#include <linux/delay.h>
 17#include <linux/jiffies.h>
 18#include <linux/mutex.h>
 19#include <linux/pci.h>
 20#include <linux/pci-doe.h>
 21#include <linux/workqueue.h>
 22
 23#define PCI_DOE_PROTOCOL_DISCOVERY 0
 24
 25/* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
 26#define PCI_DOE_TIMEOUT HZ
 27#define PCI_DOE_POLL_INTERVAL	(PCI_DOE_TIMEOUT / 128)
 28
 29#define PCI_DOE_FLAG_CANCEL	0
 30#define PCI_DOE_FLAG_DEAD	1
 31
 32/* Max data object length is 2^18 dwords */
 33#define PCI_DOE_MAX_LENGTH	(1 << 18)
 34
 35/**
 36 * struct pci_doe_mb - State for a single DOE mailbox
 37 *
 38 * This state is used to manage a single DOE mailbox capability.  All fields
 39 * should be considered opaque to the consumers and the structure passed into
 40 * the helpers below after being created by devm_pci_doe_create()
 41 *
 42 * @pdev: PCI device this mailbox belongs to
 43 * @cap_offset: Capability offset
 44 * @prots: Array of protocols supported (encoded as long values)
 45 * @wq: Wait queue for work item
 46 * @work_queue: Queue of pci_doe_work items
 47 * @flags: Bit array of PCI_DOE_FLAG_* flags
 48 */
 49struct pci_doe_mb {
 50	struct pci_dev *pdev;
 51	u16 cap_offset;
 52	struct xarray prots;
 53
 54	wait_queue_head_t wq;
 55	struct workqueue_struct *work_queue;
 56	unsigned long flags;
 57};
 58
 59static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
 60{
 61	if (wait_event_timeout(doe_mb->wq,
 62			       test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
 63			       timeout))
 64		return -EIO;
 65	return 0;
 66}
 67
 68static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
 69{
 70	struct pci_dev *pdev = doe_mb->pdev;
 71	int offset = doe_mb->cap_offset;
 72
 73	pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val);
 74}
 75
 76static int pci_doe_abort(struct pci_doe_mb *doe_mb)
 77{
 78	struct pci_dev *pdev = doe_mb->pdev;
 79	int offset = doe_mb->cap_offset;
 80	unsigned long timeout_jiffies;
 81
 82	pci_dbg(pdev, "[%x] Issuing Abort\n", offset);
 83
 84	timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
 85	pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
 86
 87	do {
 88		int rc;
 89		u32 val;
 90
 91		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
 92		if (rc)
 93			return rc;
 94		pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
 95
 96		/* Abort success! */
 97		if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) &&
 98		    !FIELD_GET(PCI_DOE_STATUS_BUSY, val))
 99			return 0;
100
101	} while (!time_after(jiffies, timeout_jiffies));
102
103	/* Abort has timed out and the MB is dead */
104	pci_err(pdev, "[%x] ABORT timed out\n", offset);
105	return -EIO;
106}
107
108static int pci_doe_send_req(struct pci_doe_mb *doe_mb,
109			    struct pci_doe_task *task)
110{
111	struct pci_dev *pdev = doe_mb->pdev;
112	int offset = doe_mb->cap_offset;
113	size_t length;
114	u32 val;
115	int i;
116
117	/*
118	 * Check the DOE busy bit is not set. If it is set, this could indicate
119	 * someone other than Linux (e.g. firmware) is using the mailbox. Note
120	 * it is expected that firmware and OS will negotiate access rights via
121	 * an, as yet to be defined, method.
122	 */
123	pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
124	if (FIELD_GET(PCI_DOE_STATUS_BUSY, val))
125		return -EBUSY;
126
127	if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
128		return -EIO;
129
130	/* Length is 2 DW of header + length of payload in DW */
131	length = 2 + task->request_pl_sz / sizeof(u32);
132	if (length > PCI_DOE_MAX_LENGTH)
133		return -EIO;
134	if (length == PCI_DOE_MAX_LENGTH)
135		length = 0;
136
137	/* Write DOE Header */
138	val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->prot.vid) |
139		FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->prot.type);
140	pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
141	pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
142			       FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH,
143					  length));
144	for (i = 0; i < task->request_pl_sz / sizeof(u32); i++)
145		pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
146				       task->request_pl[i]);
147
148	pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO);
149
150	return 0;
151}
152
153static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb)
154{
155	struct pci_dev *pdev = doe_mb->pdev;
156	int offset = doe_mb->cap_offset;
157	u32 val;
158
159	pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
160	if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val))
161		return true;
162	return false;
163}
164
165static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
166{
167	struct pci_dev *pdev = doe_mb->pdev;
168	int offset = doe_mb->cap_offset;
169	size_t length, payload_length;
170	u32 val;
171	int i;
172
173	/* Read the first dword to get the protocol */
174	pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
175	if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->prot.vid) ||
176	    (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->prot.type)) {
177		dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Protocol] = [%04x, %02x], got [%04x, %02x]\n",
178				    doe_mb->cap_offset, task->prot.vid, task->prot.type,
179				    FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val),
180				    FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val));
181		return -EIO;
182	}
183
184	pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
185	/* Read the second dword to get the length */
186	pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
187	pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
188
189	length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val);
190	/* A value of 0x0 indicates max data object length */
191	if (!length)
192		length = PCI_DOE_MAX_LENGTH;
193	if (length < 2)
194		return -EIO;
195
196	/* First 2 dwords have already been read */
197	length -= 2;
198	payload_length = min(length, task->response_pl_sz / sizeof(u32));
199	/* Read the rest of the response payload */
200	for (i = 0; i < payload_length; i++) {
201		pci_read_config_dword(pdev, offset + PCI_DOE_READ,
202				      &task->response_pl[i]);
203		/* Prior to the last ack, ensure Data Object Ready */
204		if (i == (payload_length - 1) && !pci_doe_data_obj_ready(doe_mb))
205			return -EIO;
206		pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
207	}
208
209	/* Flush excess length */
210	for (; i < length; i++) {
211		pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
212		pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
213	}
214
215	/* Final error check to pick up on any since Data Object Ready */
216	pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
217	if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
218		return -EIO;
219
220	return min(length, task->response_pl_sz / sizeof(u32)) * sizeof(u32);
221}
222
223static void signal_task_complete(struct pci_doe_task *task, int rv)
224{
225	task->rv = rv;
226	task->complete(task);
227}
228
229static void signal_task_abort(struct pci_doe_task *task, int rv)
230{
231	struct pci_doe_mb *doe_mb = task->doe_mb;
232	struct pci_dev *pdev = doe_mb->pdev;
233
234	if (pci_doe_abort(doe_mb)) {
235		/*
236		 * If the device can't process an abort; set the mailbox dead
237		 *	- no more submissions
238		 */
239		pci_err(pdev, "[%x] Abort failed marking mailbox dead\n",
240			doe_mb->cap_offset);
241		set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
242	}
243	signal_task_complete(task, rv);
244}
245
246static void doe_statemachine_work(struct work_struct *work)
247{
248	struct pci_doe_task *task = container_of(work, struct pci_doe_task,
249						 work);
250	struct pci_doe_mb *doe_mb = task->doe_mb;
251	struct pci_dev *pdev = doe_mb->pdev;
252	int offset = doe_mb->cap_offset;
253	unsigned long timeout_jiffies;
254	u32 val;
255	int rc;
256
257	if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) {
258		signal_task_complete(task, -EIO);
259		return;
260	}
261
262	/* Send request */
263	rc = pci_doe_send_req(doe_mb, task);
264	if (rc) {
265		/*
266		 * The specification does not provide any guidance on how to
267		 * resolve conflicting requests from other entities.
268		 * Furthermore, it is likely that busy will not be detected
269		 * most of the time.  Flag any detection of status busy with an
270		 * error.
271		 */
272		if (rc == -EBUSY)
273			dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n",
274					    offset);
275		signal_task_abort(task, rc);
276		return;
277	}
278
279	timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
280	/* Poll for response */
281retry_resp:
282	pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
283	if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) {
284		signal_task_abort(task, -EIO);
285		return;
286	}
287
288	if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) {
289		if (time_after(jiffies, timeout_jiffies)) {
290			signal_task_abort(task, -EIO);
291			return;
292		}
293		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
294		if (rc) {
295			signal_task_abort(task, rc);
296			return;
297		}
298		goto retry_resp;
299	}
300
301	rc  = pci_doe_recv_resp(doe_mb, task);
302	if (rc < 0) {
303		signal_task_abort(task, rc);
304		return;
305	}
306
307	signal_task_complete(task, rc);
308}
309
310static void pci_doe_task_complete(struct pci_doe_task *task)
311{
312	complete(task->private);
313}
314
315static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid,
316			     u8 *protocol)
317{
318	u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX,
319				    *index);
320	u32 response_pl;
321	DECLARE_COMPLETION_ONSTACK(c);
322	struct pci_doe_task task = {
323		.prot.vid = PCI_VENDOR_ID_PCI_SIG,
324		.prot.type = PCI_DOE_PROTOCOL_DISCOVERY,
325		.request_pl = &request_pl,
326		.request_pl_sz = sizeof(request_pl),
327		.response_pl = &response_pl,
328		.response_pl_sz = sizeof(response_pl),
329		.complete = pci_doe_task_complete,
330		.private = &c,
331	};
332	int rc;
333
334	rc = pci_doe_submit_task(doe_mb, &task);
335	if (rc < 0)
336		return rc;
337
338	wait_for_completion(&c);
339
340	if (task.rv != sizeof(response_pl))
341		return -EIO;
342
343	*vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl);
344	*protocol = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL,
345			      response_pl);
346	*index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX,
347			   response_pl);
348
349	return 0;
350}
351
352static void *pci_doe_xa_prot_entry(u16 vid, u8 prot)
353{
354	return xa_mk_value((vid << 8) | prot);
355}
356
357static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb)
358{
359	u8 index = 0;
360	u8 xa_idx = 0;
361
362	do {
363		int rc;
364		u16 vid;
365		u8 prot;
366
367		rc = pci_doe_discovery(doe_mb, &index, &vid, &prot);
368		if (rc)
369			return rc;
370
371		pci_dbg(doe_mb->pdev,
372			"[%x] Found protocol %d vid: %x prot: %x\n",
373			doe_mb->cap_offset, xa_idx, vid, prot);
374
375		rc = xa_insert(&doe_mb->prots, xa_idx++,
376			       pci_doe_xa_prot_entry(vid, prot), GFP_KERNEL);
377		if (rc)
378			return rc;
379	} while (index);
380
381	return 0;
382}
383
384static void pci_doe_xa_destroy(void *mb)
385{
386	struct pci_doe_mb *doe_mb = mb;
387
388	xa_destroy(&doe_mb->prots);
389}
390
391static void pci_doe_destroy_workqueue(void *mb)
392{
393	struct pci_doe_mb *doe_mb = mb;
394
395	destroy_workqueue(doe_mb->work_queue);
396}
397
398static void pci_doe_flush_mb(void *mb)
399{
400	struct pci_doe_mb *doe_mb = mb;
401
402	/* Stop all pending work items from starting */
403	set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
404
405	/* Cancel an in progress work item, if necessary */
406	set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
407	wake_up(&doe_mb->wq);
408
409	/* Flush all work items */
410	flush_workqueue(doe_mb->work_queue);
411}
412
413/**
414 * pcim_doe_create_mb() - Create a DOE mailbox object
415 *
416 * @pdev: PCI device to create the DOE mailbox for
417 * @cap_offset: Offset of the DOE mailbox
418 *
419 * Create a single mailbox object to manage the mailbox protocol at the
420 * cap_offset specified.
421 *
422 * RETURNS: created mailbox object on success
423 *	    ERR_PTR(-errno) on failure
424 */
425struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset)
426{
427	struct pci_doe_mb *doe_mb;
428	struct device *dev = &pdev->dev;
429	int rc;
430
431	doe_mb = devm_kzalloc(dev, sizeof(*doe_mb), GFP_KERNEL);
432	if (!doe_mb)
433		return ERR_PTR(-ENOMEM);
434
435	doe_mb->pdev = pdev;
436	doe_mb->cap_offset = cap_offset;
437	init_waitqueue_head(&doe_mb->wq);
438
439	xa_init(&doe_mb->prots);
440	rc = devm_add_action(dev, pci_doe_xa_destroy, doe_mb);
441	if (rc)
442		return ERR_PTR(rc);
443
444	doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0,
445						dev_driver_string(&pdev->dev),
446						pci_name(pdev),
447						doe_mb->cap_offset);
448	if (!doe_mb->work_queue) {
449		pci_err(pdev, "[%x] failed to allocate work queue\n",
450			doe_mb->cap_offset);
451		return ERR_PTR(-ENOMEM);
452	}
453	rc = devm_add_action_or_reset(dev, pci_doe_destroy_workqueue, doe_mb);
454	if (rc)
455		return ERR_PTR(rc);
456
457	/* Reset the mailbox by issuing an abort */
458	rc = pci_doe_abort(doe_mb);
459	if (rc) {
460		pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n",
461			doe_mb->cap_offset, rc);
462		return ERR_PTR(rc);
463	}
464
465	/*
466	 * The state machine and the mailbox should be in sync now;
467	 * Set up mailbox flush prior to using the mailbox to query protocols.
468	 */
469	rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb);
470	if (rc)
471		return ERR_PTR(rc);
472
473	rc = pci_doe_cache_protocols(doe_mb);
474	if (rc) {
475		pci_err(pdev, "[%x] failed to cache protocols : %d\n",
476			doe_mb->cap_offset, rc);
477		return ERR_PTR(rc);
478	}
479
480	return doe_mb;
481}
482EXPORT_SYMBOL_GPL(pcim_doe_create_mb);
483
484/**
485 * pci_doe_supports_prot() - Return if the DOE instance supports the given
486 *			     protocol
487 * @doe_mb: DOE mailbox capability to query
488 * @vid: Protocol Vendor ID
489 * @type: Protocol type
490 *
491 * RETURNS: True if the DOE mailbox supports the protocol specified
492 */
493bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type)
494{
495	unsigned long index;
496	void *entry;
497
498	/* The discovery protocol must always be supported */
499	if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_PROTOCOL_DISCOVERY)
500		return true;
501
502	xa_for_each(&doe_mb->prots, index, entry)
503		if (entry == pci_doe_xa_prot_entry(vid, type))
504			return true;
505
506	return false;
507}
508EXPORT_SYMBOL_GPL(pci_doe_supports_prot);
509
510/**
511 * pci_doe_submit_task() - Submit a task to be processed by the state machine
512 *
513 * @doe_mb: DOE mailbox capability to submit to
514 * @task: task to be queued
515 *
516 * Submit a DOE task (request/response) to the DOE mailbox to be processed.
517 * Returns upon queueing the task object.  If the queue is full this function
518 * will sleep until there is room in the queue.
519 *
520 * task->complete will be called when the state machine is done processing this
521 * task.
522 *
523 * Excess data will be discarded.
524 *
525 * RETURNS: 0 when task has been successfully queued, -ERRNO on error
526 */
527int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
528{
529	if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type))
530		return -EINVAL;
531
532	/*
533	 * DOE requests must be a whole number of DW and the response needs to
534	 * be big enough for at least 1 DW
535	 */
536	if (task->request_pl_sz % sizeof(u32) ||
537	    task->response_pl_sz < sizeof(u32))
538		return -EINVAL;
539
540	if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags))
541		return -EIO;
542
543	task->doe_mb = doe_mb;
544	INIT_WORK(&task->work, doe_statemachine_work);
545	queue_work(doe_mb->work_queue, &task->work);
546	return 0;
547}
548EXPORT_SYMBOL_GPL(pci_doe_submit_task);