Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Data Object Exchange
4 * PCIe r6.0, sec 6.30 DOE
5 *
6 * Copyright (C) 2021 Huawei
7 * Jonathan Cameron <Jonathan.Cameron@huawei.com>
8 *
9 * Copyright (C) 2022 Intel Corporation
10 * Ira Weiny <ira.weiny@intel.com>
11 */
12
13#define dev_fmt(fmt) "DOE: " fmt
14
15#include <linux/bitfield.h>
16#include <linux/delay.h>
17#include <linux/jiffies.h>
18#include <linux/mutex.h>
19#include <linux/pci.h>
20#include <linux/pci-doe.h>
21#include <linux/workqueue.h>
22
23#include "pci.h"
24
25#define PCI_DOE_PROTOCOL_DISCOVERY 0
26
27/* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
28#define PCI_DOE_TIMEOUT HZ
29#define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128)
30
31#define PCI_DOE_FLAG_CANCEL 0
32#define PCI_DOE_FLAG_DEAD 1
33
34/* Max data object length is 2^18 dwords */
35#define PCI_DOE_MAX_LENGTH (1 << 18)
36
37/**
38 * struct pci_doe_mb - State for a single DOE mailbox
39 *
40 * This state is used to manage a single DOE mailbox capability. All fields
41 * should be considered opaque to the consumers and the structure passed into
42 * the helpers below after being created by pci_doe_create_mb().
43 *
44 * @pdev: PCI device this mailbox belongs to
45 * @cap_offset: Capability offset
46 * @prots: Array of protocols supported (encoded as long values)
47 * @wq: Wait queue for work item
48 * @work_queue: Queue of pci_doe_work items
49 * @flags: Bit array of PCI_DOE_FLAG_* flags
50 */
51struct pci_doe_mb {
52 struct pci_dev *pdev;
53 u16 cap_offset;
54 struct xarray prots;
55
56 wait_queue_head_t wq;
57 struct workqueue_struct *work_queue;
58 unsigned long flags;
59};
60
61struct pci_doe_protocol {
62 u16 vid;
63 u8 type;
64};
65
66/**
67 * struct pci_doe_task - represents a single query/response
68 *
69 * @prot: DOE Protocol
70 * @request_pl: The request payload
71 * @request_pl_sz: Size of the request payload (bytes)
72 * @response_pl: The response payload
73 * @response_pl_sz: Size of the response payload (bytes)
74 * @rv: Return value. Length of received response or error (bytes)
75 * @complete: Called when task is complete
76 * @private: Private data for the consumer
77 * @work: Used internally by the mailbox
78 * @doe_mb: Used internally by the mailbox
79 */
80struct pci_doe_task {
81 struct pci_doe_protocol prot;
82 const __le32 *request_pl;
83 size_t request_pl_sz;
84 __le32 *response_pl;
85 size_t response_pl_sz;
86 int rv;
87 void (*complete)(struct pci_doe_task *task);
88 void *private;
89
90 /* initialized by pci_doe_submit_task() */
91 struct work_struct work;
92 struct pci_doe_mb *doe_mb;
93};
94
95static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
96{
97 if (wait_event_timeout(doe_mb->wq,
98 test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
99 timeout))
100 return -EIO;
101 return 0;
102}
103
104static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
105{
106 struct pci_dev *pdev = doe_mb->pdev;
107 int offset = doe_mb->cap_offset;
108
109 pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val);
110}
111
112static int pci_doe_abort(struct pci_doe_mb *doe_mb)
113{
114 struct pci_dev *pdev = doe_mb->pdev;
115 int offset = doe_mb->cap_offset;
116 unsigned long timeout_jiffies;
117
118 pci_dbg(pdev, "[%x] Issuing Abort\n", offset);
119
120 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
121 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
122
123 do {
124 int rc;
125 u32 val;
126
127 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
128 if (rc)
129 return rc;
130 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
131
132 /* Abort success! */
133 if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) &&
134 !FIELD_GET(PCI_DOE_STATUS_BUSY, val))
135 return 0;
136
137 } while (!time_after(jiffies, timeout_jiffies));
138
139 /* Abort has timed out and the MB is dead */
140 pci_err(pdev, "[%x] ABORT timed out\n", offset);
141 return -EIO;
142}
143
144static int pci_doe_send_req(struct pci_doe_mb *doe_mb,
145 struct pci_doe_task *task)
146{
147 struct pci_dev *pdev = doe_mb->pdev;
148 int offset = doe_mb->cap_offset;
149 size_t length, remainder;
150 u32 val;
151 int i;
152
153 /*
154 * Check the DOE busy bit is not set. If it is set, this could indicate
155 * someone other than Linux (e.g. firmware) is using the mailbox. Note
156 * it is expected that firmware and OS will negotiate access rights via
157 * an, as yet to be defined, method.
158 */
159 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
160 if (FIELD_GET(PCI_DOE_STATUS_BUSY, val))
161 return -EBUSY;
162
163 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
164 return -EIO;
165
166 /* Length is 2 DW of header + length of payload in DW */
167 length = 2 + DIV_ROUND_UP(task->request_pl_sz, sizeof(__le32));
168 if (length > PCI_DOE_MAX_LENGTH)
169 return -EIO;
170 if (length == PCI_DOE_MAX_LENGTH)
171 length = 0;
172
173 /* Write DOE Header */
174 val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->prot.vid) |
175 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->prot.type);
176 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
177 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
178 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH,
179 length));
180
181 /* Write payload */
182 for (i = 0; i < task->request_pl_sz / sizeof(__le32); i++)
183 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
184 le32_to_cpu(task->request_pl[i]));
185
186 /* Write last payload dword */
187 remainder = task->request_pl_sz % sizeof(__le32);
188 if (remainder) {
189 val = 0;
190 memcpy(&val, &task->request_pl[i], remainder);
191 le32_to_cpus(&val);
192 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
193 }
194
195 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO);
196
197 return 0;
198}
199
200static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb)
201{
202 struct pci_dev *pdev = doe_mb->pdev;
203 int offset = doe_mb->cap_offset;
204 u32 val;
205
206 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
207 if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val))
208 return true;
209 return false;
210}
211
212static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
213{
214 size_t length, payload_length, remainder, received;
215 struct pci_dev *pdev = doe_mb->pdev;
216 int offset = doe_mb->cap_offset;
217 int i = 0;
218 u32 val;
219
220 /* Read the first dword to get the protocol */
221 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
222 if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->prot.vid) ||
223 (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->prot.type)) {
224 dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Protocol] = [%04x, %02x], got [%04x, %02x]\n",
225 doe_mb->cap_offset, task->prot.vid, task->prot.type,
226 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val),
227 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val));
228 return -EIO;
229 }
230
231 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
232 /* Read the second dword to get the length */
233 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
234 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
235
236 length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val);
237 /* A value of 0x0 indicates max data object length */
238 if (!length)
239 length = PCI_DOE_MAX_LENGTH;
240 if (length < 2)
241 return -EIO;
242
243 /* First 2 dwords have already been read */
244 length -= 2;
245 received = task->response_pl_sz;
246 payload_length = DIV_ROUND_UP(task->response_pl_sz, sizeof(__le32));
247 remainder = task->response_pl_sz % sizeof(__le32);
248
249 /* remainder signifies number of data bytes in last payload dword */
250 if (!remainder)
251 remainder = sizeof(__le32);
252
253 if (length < payload_length) {
254 received = length * sizeof(__le32);
255 payload_length = length;
256 remainder = sizeof(__le32);
257 }
258
259 if (payload_length) {
260 /* Read all payload dwords except the last */
261 for (; i < payload_length - 1; i++) {
262 pci_read_config_dword(pdev, offset + PCI_DOE_READ,
263 &val);
264 task->response_pl[i] = cpu_to_le32(val);
265 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
266 }
267
268 /* Read last payload dword */
269 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
270 cpu_to_le32s(&val);
271 memcpy(&task->response_pl[i], &val, remainder);
272 /* Prior to the last ack, ensure Data Object Ready */
273 if (!pci_doe_data_obj_ready(doe_mb))
274 return -EIO;
275 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
276 i++;
277 }
278
279 /* Flush excess length */
280 for (; i < length; i++) {
281 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
282 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
283 }
284
285 /* Final error check to pick up on any since Data Object Ready */
286 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
287 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
288 return -EIO;
289
290 return received;
291}
292
293static void signal_task_complete(struct pci_doe_task *task, int rv)
294{
295 task->rv = rv;
296 destroy_work_on_stack(&task->work);
297 task->complete(task);
298}
299
300static void signal_task_abort(struct pci_doe_task *task, int rv)
301{
302 struct pci_doe_mb *doe_mb = task->doe_mb;
303 struct pci_dev *pdev = doe_mb->pdev;
304
305 if (pci_doe_abort(doe_mb)) {
306 /*
307 * If the device can't process an abort; set the mailbox dead
308 * - no more submissions
309 */
310 pci_err(pdev, "[%x] Abort failed marking mailbox dead\n",
311 doe_mb->cap_offset);
312 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
313 }
314 signal_task_complete(task, rv);
315}
316
317static void doe_statemachine_work(struct work_struct *work)
318{
319 struct pci_doe_task *task = container_of(work, struct pci_doe_task,
320 work);
321 struct pci_doe_mb *doe_mb = task->doe_mb;
322 struct pci_dev *pdev = doe_mb->pdev;
323 int offset = doe_mb->cap_offset;
324 unsigned long timeout_jiffies;
325 u32 val;
326 int rc;
327
328 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) {
329 signal_task_complete(task, -EIO);
330 return;
331 }
332
333 /* Send request */
334 rc = pci_doe_send_req(doe_mb, task);
335 if (rc) {
336 /*
337 * The specification does not provide any guidance on how to
338 * resolve conflicting requests from other entities.
339 * Furthermore, it is likely that busy will not be detected
340 * most of the time. Flag any detection of status busy with an
341 * error.
342 */
343 if (rc == -EBUSY)
344 dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n",
345 offset);
346 signal_task_abort(task, rc);
347 return;
348 }
349
350 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
351 /* Poll for response */
352retry_resp:
353 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
354 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) {
355 signal_task_abort(task, -EIO);
356 return;
357 }
358
359 if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) {
360 if (time_after(jiffies, timeout_jiffies)) {
361 signal_task_abort(task, -EIO);
362 return;
363 }
364 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
365 if (rc) {
366 signal_task_abort(task, rc);
367 return;
368 }
369 goto retry_resp;
370 }
371
372 rc = pci_doe_recv_resp(doe_mb, task);
373 if (rc < 0) {
374 signal_task_abort(task, rc);
375 return;
376 }
377
378 signal_task_complete(task, rc);
379}
380
381static void pci_doe_task_complete(struct pci_doe_task *task)
382{
383 complete(task->private);
384}
385
386static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid,
387 u8 *protocol)
388{
389 u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX,
390 *index);
391 __le32 request_pl_le = cpu_to_le32(request_pl);
392 __le32 response_pl_le;
393 u32 response_pl;
394 int rc;
395
396 rc = pci_doe(doe_mb, PCI_VENDOR_ID_PCI_SIG, PCI_DOE_PROTOCOL_DISCOVERY,
397 &request_pl_le, sizeof(request_pl_le),
398 &response_pl_le, sizeof(response_pl_le));
399 if (rc < 0)
400 return rc;
401
402 if (rc != sizeof(response_pl_le))
403 return -EIO;
404
405 response_pl = le32_to_cpu(response_pl_le);
406 *vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl);
407 *protocol = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL,
408 response_pl);
409 *index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX,
410 response_pl);
411
412 return 0;
413}
414
415static void *pci_doe_xa_prot_entry(u16 vid, u8 prot)
416{
417 return xa_mk_value((vid << 8) | prot);
418}
419
420static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb)
421{
422 u8 index = 0;
423 u8 xa_idx = 0;
424
425 do {
426 int rc;
427 u16 vid;
428 u8 prot;
429
430 rc = pci_doe_discovery(doe_mb, &index, &vid, &prot);
431 if (rc)
432 return rc;
433
434 pci_dbg(doe_mb->pdev,
435 "[%x] Found protocol %d vid: %x prot: %x\n",
436 doe_mb->cap_offset, xa_idx, vid, prot);
437
438 rc = xa_insert(&doe_mb->prots, xa_idx++,
439 pci_doe_xa_prot_entry(vid, prot), GFP_KERNEL);
440 if (rc)
441 return rc;
442 } while (index);
443
444 return 0;
445}
446
447static void pci_doe_cancel_tasks(struct pci_doe_mb *doe_mb)
448{
449 /* Stop all pending work items from starting */
450 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
451
452 /* Cancel an in progress work item, if necessary */
453 set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
454 wake_up(&doe_mb->wq);
455}
456
457/**
458 * pci_doe_create_mb() - Create a DOE mailbox object
459 *
460 * @pdev: PCI device to create the DOE mailbox for
461 * @cap_offset: Offset of the DOE mailbox
462 *
463 * Create a single mailbox object to manage the mailbox protocol at the
464 * cap_offset specified.
465 *
466 * RETURNS: created mailbox object on success
467 * ERR_PTR(-errno) on failure
468 */
469static struct pci_doe_mb *pci_doe_create_mb(struct pci_dev *pdev,
470 u16 cap_offset)
471{
472 struct pci_doe_mb *doe_mb;
473 int rc;
474
475 doe_mb = kzalloc(sizeof(*doe_mb), GFP_KERNEL);
476 if (!doe_mb)
477 return ERR_PTR(-ENOMEM);
478
479 doe_mb->pdev = pdev;
480 doe_mb->cap_offset = cap_offset;
481 init_waitqueue_head(&doe_mb->wq);
482 xa_init(&doe_mb->prots);
483
484 doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0,
485 dev_bus_name(&pdev->dev),
486 pci_name(pdev),
487 doe_mb->cap_offset);
488 if (!doe_mb->work_queue) {
489 pci_err(pdev, "[%x] failed to allocate work queue\n",
490 doe_mb->cap_offset);
491 rc = -ENOMEM;
492 goto err_free;
493 }
494
495 /* Reset the mailbox by issuing an abort */
496 rc = pci_doe_abort(doe_mb);
497 if (rc) {
498 pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n",
499 doe_mb->cap_offset, rc);
500 goto err_destroy_wq;
501 }
502
503 /*
504 * The state machine and the mailbox should be in sync now;
505 * Use the mailbox to query protocols.
506 */
507 rc = pci_doe_cache_protocols(doe_mb);
508 if (rc) {
509 pci_err(pdev, "[%x] failed to cache protocols : %d\n",
510 doe_mb->cap_offset, rc);
511 goto err_cancel;
512 }
513
514 return doe_mb;
515
516err_cancel:
517 pci_doe_cancel_tasks(doe_mb);
518 xa_destroy(&doe_mb->prots);
519err_destroy_wq:
520 destroy_workqueue(doe_mb->work_queue);
521err_free:
522 kfree(doe_mb);
523 return ERR_PTR(rc);
524}
525
526/**
527 * pci_doe_destroy_mb() - Destroy a DOE mailbox object
528 *
529 * @doe_mb: DOE mailbox
530 *
531 * Destroy all internal data structures created for the DOE mailbox.
532 */
533static void pci_doe_destroy_mb(struct pci_doe_mb *doe_mb)
534{
535 pci_doe_cancel_tasks(doe_mb);
536 xa_destroy(&doe_mb->prots);
537 destroy_workqueue(doe_mb->work_queue);
538 kfree(doe_mb);
539}
540
541/**
542 * pci_doe_supports_prot() - Return if the DOE instance supports the given
543 * protocol
544 * @doe_mb: DOE mailbox capability to query
545 * @vid: Protocol Vendor ID
546 * @type: Protocol type
547 *
548 * RETURNS: True if the DOE mailbox supports the protocol specified
549 */
550static bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type)
551{
552 unsigned long index;
553 void *entry;
554
555 /* The discovery protocol must always be supported */
556 if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_PROTOCOL_DISCOVERY)
557 return true;
558
559 xa_for_each(&doe_mb->prots, index, entry)
560 if (entry == pci_doe_xa_prot_entry(vid, type))
561 return true;
562
563 return false;
564}
565
566/**
567 * pci_doe_submit_task() - Submit a task to be processed by the state machine
568 *
569 * @doe_mb: DOE mailbox capability to submit to
570 * @task: task to be queued
571 *
572 * Submit a DOE task (request/response) to the DOE mailbox to be processed.
573 * Returns upon queueing the task object. If the queue is full this function
574 * will sleep until there is room in the queue.
575 *
576 * task->complete will be called when the state machine is done processing this
577 * task.
578 *
579 * @task must be allocated on the stack.
580 *
581 * Excess data will be discarded.
582 *
583 * RETURNS: 0 when task has been successfully queued, -ERRNO on error
584 */
585static int pci_doe_submit_task(struct pci_doe_mb *doe_mb,
586 struct pci_doe_task *task)
587{
588 if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type))
589 return -EINVAL;
590
591 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags))
592 return -EIO;
593
594 task->doe_mb = doe_mb;
595 INIT_WORK_ONSTACK(&task->work, doe_statemachine_work);
596 queue_work(doe_mb->work_queue, &task->work);
597 return 0;
598}
599
600/**
601 * pci_doe() - Perform Data Object Exchange
602 *
603 * @doe_mb: DOE Mailbox
604 * @vendor: Vendor ID
605 * @type: Data Object Type
606 * @request: Request payload
607 * @request_sz: Size of request payload (bytes)
608 * @response: Response payload
609 * @response_sz: Size of response payload (bytes)
610 *
611 * Submit @request to @doe_mb and store the @response.
612 * The DOE exchange is performed synchronously and may therefore sleep.
613 *
614 * Payloads are treated as opaque byte streams which are transmitted verbatim,
615 * without byte-swapping. If payloads contain little-endian register values,
616 * the caller is responsible for conversion with cpu_to_le32() / le32_to_cpu().
617 *
618 * For convenience, arbitrary payload sizes are allowed even though PCIe r6.0
619 * sec 6.30.1 specifies the Data Object Header 2 "Length" in dwords. The last
620 * (partial) dword is copied with byte granularity and padded with zeroes if
621 * necessary. Callers are thus relieved of using dword-sized bounce buffers.
622 *
623 * RETURNS: Length of received response or negative errno.
624 * Received data in excess of @response_sz is discarded.
625 * The length may be smaller than @response_sz and the caller
626 * is responsible for checking that.
627 */
628int pci_doe(struct pci_doe_mb *doe_mb, u16 vendor, u8 type,
629 const void *request, size_t request_sz,
630 void *response, size_t response_sz)
631{
632 DECLARE_COMPLETION_ONSTACK(c);
633 struct pci_doe_task task = {
634 .prot.vid = vendor,
635 .prot.type = type,
636 .request_pl = request,
637 .request_pl_sz = request_sz,
638 .response_pl = response,
639 .response_pl_sz = response_sz,
640 .complete = pci_doe_task_complete,
641 .private = &c,
642 };
643 int rc;
644
645 rc = pci_doe_submit_task(doe_mb, &task);
646 if (rc)
647 return rc;
648
649 wait_for_completion(&c);
650
651 return task.rv;
652}
653EXPORT_SYMBOL_GPL(pci_doe);
654
655/**
656 * pci_find_doe_mailbox() - Find Data Object Exchange mailbox
657 *
658 * @pdev: PCI device
659 * @vendor: Vendor ID
660 * @type: Data Object Type
661 *
662 * Find first DOE mailbox of a PCI device which supports the given protocol.
663 *
664 * RETURNS: Pointer to the DOE mailbox or NULL if none was found.
665 */
666struct pci_doe_mb *pci_find_doe_mailbox(struct pci_dev *pdev, u16 vendor,
667 u8 type)
668{
669 struct pci_doe_mb *doe_mb;
670 unsigned long index;
671
672 xa_for_each(&pdev->doe_mbs, index, doe_mb)
673 if (pci_doe_supports_prot(doe_mb, vendor, type))
674 return doe_mb;
675
676 return NULL;
677}
678EXPORT_SYMBOL_GPL(pci_find_doe_mailbox);
679
680void pci_doe_init(struct pci_dev *pdev)
681{
682 struct pci_doe_mb *doe_mb;
683 u16 offset = 0;
684 int rc;
685
686 xa_init(&pdev->doe_mbs);
687
688 while ((offset = pci_find_next_ext_capability(pdev, offset,
689 PCI_EXT_CAP_ID_DOE))) {
690 doe_mb = pci_doe_create_mb(pdev, offset);
691 if (IS_ERR(doe_mb)) {
692 pci_err(pdev, "[%x] failed to create mailbox: %ld\n",
693 offset, PTR_ERR(doe_mb));
694 continue;
695 }
696
697 rc = xa_insert(&pdev->doe_mbs, offset, doe_mb, GFP_KERNEL);
698 if (rc) {
699 pci_err(pdev, "[%x] failed to insert mailbox: %d\n",
700 offset, rc);
701 pci_doe_destroy_mb(doe_mb);
702 }
703 }
704}
705
706void pci_doe_destroy(struct pci_dev *pdev)
707{
708 struct pci_doe_mb *doe_mb;
709 unsigned long index;
710
711 xa_for_each(&pdev->doe_mbs, index, doe_mb)
712 pci_doe_destroy_mb(doe_mb);
713
714 xa_destroy(&pdev->doe_mbs);
715}
716
717void pci_doe_disconnected(struct pci_dev *pdev)
718{
719 struct pci_doe_mb *doe_mb;
720 unsigned long index;
721
722 xa_for_each(&pdev->doe_mbs, index, doe_mb)
723 pci_doe_cancel_tasks(doe_mb);
724}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Data Object Exchange
4 * PCIe r6.0, sec 6.30 DOE
5 *
6 * Copyright (C) 2021 Huawei
7 * Jonathan Cameron <Jonathan.Cameron@huawei.com>
8 *
9 * Copyright (C) 2022 Intel Corporation
10 * Ira Weiny <ira.weiny@intel.com>
11 */
12
13#define dev_fmt(fmt) "DOE: " fmt
14
15#include <linux/bitfield.h>
16#include <linux/delay.h>
17#include <linux/jiffies.h>
18#include <linux/mutex.h>
19#include <linux/pci.h>
20#include <linux/pci-doe.h>
21#include <linux/workqueue.h>
22
23#define PCI_DOE_PROTOCOL_DISCOVERY 0
24
25/* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
26#define PCI_DOE_TIMEOUT HZ
27#define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128)
28
29#define PCI_DOE_FLAG_CANCEL 0
30#define PCI_DOE_FLAG_DEAD 1
31
32/* Max data object length is 2^18 dwords */
33#define PCI_DOE_MAX_LENGTH (1 << 18)
34
35/**
36 * struct pci_doe_mb - State for a single DOE mailbox
37 *
38 * This state is used to manage a single DOE mailbox capability. All fields
39 * should be considered opaque to the consumers and the structure passed into
40 * the helpers below after being created by devm_pci_doe_create()
41 *
42 * @pdev: PCI device this mailbox belongs to
43 * @cap_offset: Capability offset
44 * @prots: Array of protocols supported (encoded as long values)
45 * @wq: Wait queue for work item
46 * @work_queue: Queue of pci_doe_work items
47 * @flags: Bit array of PCI_DOE_FLAG_* flags
48 */
49struct pci_doe_mb {
50 struct pci_dev *pdev;
51 u16 cap_offset;
52 struct xarray prots;
53
54 wait_queue_head_t wq;
55 struct workqueue_struct *work_queue;
56 unsigned long flags;
57};
58
59static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
60{
61 if (wait_event_timeout(doe_mb->wq,
62 test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
63 timeout))
64 return -EIO;
65 return 0;
66}
67
68static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
69{
70 struct pci_dev *pdev = doe_mb->pdev;
71 int offset = doe_mb->cap_offset;
72
73 pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val);
74}
75
76static int pci_doe_abort(struct pci_doe_mb *doe_mb)
77{
78 struct pci_dev *pdev = doe_mb->pdev;
79 int offset = doe_mb->cap_offset;
80 unsigned long timeout_jiffies;
81
82 pci_dbg(pdev, "[%x] Issuing Abort\n", offset);
83
84 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
85 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
86
87 do {
88 int rc;
89 u32 val;
90
91 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
92 if (rc)
93 return rc;
94 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
95
96 /* Abort success! */
97 if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) &&
98 !FIELD_GET(PCI_DOE_STATUS_BUSY, val))
99 return 0;
100
101 } while (!time_after(jiffies, timeout_jiffies));
102
103 /* Abort has timed out and the MB is dead */
104 pci_err(pdev, "[%x] ABORT timed out\n", offset);
105 return -EIO;
106}
107
108static int pci_doe_send_req(struct pci_doe_mb *doe_mb,
109 struct pci_doe_task *task)
110{
111 struct pci_dev *pdev = doe_mb->pdev;
112 int offset = doe_mb->cap_offset;
113 size_t length;
114 u32 val;
115 int i;
116
117 /*
118 * Check the DOE busy bit is not set. If it is set, this could indicate
119 * someone other than Linux (e.g. firmware) is using the mailbox. Note
120 * it is expected that firmware and OS will negotiate access rights via
121 * an, as yet to be defined, method.
122 */
123 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
124 if (FIELD_GET(PCI_DOE_STATUS_BUSY, val))
125 return -EBUSY;
126
127 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
128 return -EIO;
129
130 /* Length is 2 DW of header + length of payload in DW */
131 length = 2 + task->request_pl_sz / sizeof(u32);
132 if (length > PCI_DOE_MAX_LENGTH)
133 return -EIO;
134 if (length == PCI_DOE_MAX_LENGTH)
135 length = 0;
136
137 /* Write DOE Header */
138 val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->prot.vid) |
139 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->prot.type);
140 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
141 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
142 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH,
143 length));
144 for (i = 0; i < task->request_pl_sz / sizeof(u32); i++)
145 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
146 task->request_pl[i]);
147
148 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO);
149
150 return 0;
151}
152
153static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb)
154{
155 struct pci_dev *pdev = doe_mb->pdev;
156 int offset = doe_mb->cap_offset;
157 u32 val;
158
159 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
160 if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val))
161 return true;
162 return false;
163}
164
165static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
166{
167 struct pci_dev *pdev = doe_mb->pdev;
168 int offset = doe_mb->cap_offset;
169 size_t length, payload_length;
170 u32 val;
171 int i;
172
173 /* Read the first dword to get the protocol */
174 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
175 if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->prot.vid) ||
176 (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->prot.type)) {
177 dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Protocol] = [%04x, %02x], got [%04x, %02x]\n",
178 doe_mb->cap_offset, task->prot.vid, task->prot.type,
179 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val),
180 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val));
181 return -EIO;
182 }
183
184 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
185 /* Read the second dword to get the length */
186 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
187 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
188
189 length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val);
190 /* A value of 0x0 indicates max data object length */
191 if (!length)
192 length = PCI_DOE_MAX_LENGTH;
193 if (length < 2)
194 return -EIO;
195
196 /* First 2 dwords have already been read */
197 length -= 2;
198 payload_length = min(length, task->response_pl_sz / sizeof(u32));
199 /* Read the rest of the response payload */
200 for (i = 0; i < payload_length; i++) {
201 pci_read_config_dword(pdev, offset + PCI_DOE_READ,
202 &task->response_pl[i]);
203 /* Prior to the last ack, ensure Data Object Ready */
204 if (i == (payload_length - 1) && !pci_doe_data_obj_ready(doe_mb))
205 return -EIO;
206 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
207 }
208
209 /* Flush excess length */
210 for (; i < length; i++) {
211 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
212 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
213 }
214
215 /* Final error check to pick up on any since Data Object Ready */
216 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
217 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
218 return -EIO;
219
220 return min(length, task->response_pl_sz / sizeof(u32)) * sizeof(u32);
221}
222
223static void signal_task_complete(struct pci_doe_task *task, int rv)
224{
225 task->rv = rv;
226 task->complete(task);
227}
228
229static void signal_task_abort(struct pci_doe_task *task, int rv)
230{
231 struct pci_doe_mb *doe_mb = task->doe_mb;
232 struct pci_dev *pdev = doe_mb->pdev;
233
234 if (pci_doe_abort(doe_mb)) {
235 /*
236 * If the device can't process an abort; set the mailbox dead
237 * - no more submissions
238 */
239 pci_err(pdev, "[%x] Abort failed marking mailbox dead\n",
240 doe_mb->cap_offset);
241 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
242 }
243 signal_task_complete(task, rv);
244}
245
246static void doe_statemachine_work(struct work_struct *work)
247{
248 struct pci_doe_task *task = container_of(work, struct pci_doe_task,
249 work);
250 struct pci_doe_mb *doe_mb = task->doe_mb;
251 struct pci_dev *pdev = doe_mb->pdev;
252 int offset = doe_mb->cap_offset;
253 unsigned long timeout_jiffies;
254 u32 val;
255 int rc;
256
257 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) {
258 signal_task_complete(task, -EIO);
259 return;
260 }
261
262 /* Send request */
263 rc = pci_doe_send_req(doe_mb, task);
264 if (rc) {
265 /*
266 * The specification does not provide any guidance on how to
267 * resolve conflicting requests from other entities.
268 * Furthermore, it is likely that busy will not be detected
269 * most of the time. Flag any detection of status busy with an
270 * error.
271 */
272 if (rc == -EBUSY)
273 dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n",
274 offset);
275 signal_task_abort(task, rc);
276 return;
277 }
278
279 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
280 /* Poll for response */
281retry_resp:
282 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
283 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) {
284 signal_task_abort(task, -EIO);
285 return;
286 }
287
288 if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) {
289 if (time_after(jiffies, timeout_jiffies)) {
290 signal_task_abort(task, -EIO);
291 return;
292 }
293 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
294 if (rc) {
295 signal_task_abort(task, rc);
296 return;
297 }
298 goto retry_resp;
299 }
300
301 rc = pci_doe_recv_resp(doe_mb, task);
302 if (rc < 0) {
303 signal_task_abort(task, rc);
304 return;
305 }
306
307 signal_task_complete(task, rc);
308}
309
310static void pci_doe_task_complete(struct pci_doe_task *task)
311{
312 complete(task->private);
313}
314
315static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid,
316 u8 *protocol)
317{
318 u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX,
319 *index);
320 u32 response_pl;
321 DECLARE_COMPLETION_ONSTACK(c);
322 struct pci_doe_task task = {
323 .prot.vid = PCI_VENDOR_ID_PCI_SIG,
324 .prot.type = PCI_DOE_PROTOCOL_DISCOVERY,
325 .request_pl = &request_pl,
326 .request_pl_sz = sizeof(request_pl),
327 .response_pl = &response_pl,
328 .response_pl_sz = sizeof(response_pl),
329 .complete = pci_doe_task_complete,
330 .private = &c,
331 };
332 int rc;
333
334 rc = pci_doe_submit_task(doe_mb, &task);
335 if (rc < 0)
336 return rc;
337
338 wait_for_completion(&c);
339
340 if (task.rv != sizeof(response_pl))
341 return -EIO;
342
343 *vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl);
344 *protocol = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL,
345 response_pl);
346 *index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX,
347 response_pl);
348
349 return 0;
350}
351
352static void *pci_doe_xa_prot_entry(u16 vid, u8 prot)
353{
354 return xa_mk_value((vid << 8) | prot);
355}
356
357static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb)
358{
359 u8 index = 0;
360 u8 xa_idx = 0;
361
362 do {
363 int rc;
364 u16 vid;
365 u8 prot;
366
367 rc = pci_doe_discovery(doe_mb, &index, &vid, &prot);
368 if (rc)
369 return rc;
370
371 pci_dbg(doe_mb->pdev,
372 "[%x] Found protocol %d vid: %x prot: %x\n",
373 doe_mb->cap_offset, xa_idx, vid, prot);
374
375 rc = xa_insert(&doe_mb->prots, xa_idx++,
376 pci_doe_xa_prot_entry(vid, prot), GFP_KERNEL);
377 if (rc)
378 return rc;
379 } while (index);
380
381 return 0;
382}
383
384static void pci_doe_xa_destroy(void *mb)
385{
386 struct pci_doe_mb *doe_mb = mb;
387
388 xa_destroy(&doe_mb->prots);
389}
390
391static void pci_doe_destroy_workqueue(void *mb)
392{
393 struct pci_doe_mb *doe_mb = mb;
394
395 destroy_workqueue(doe_mb->work_queue);
396}
397
398static void pci_doe_flush_mb(void *mb)
399{
400 struct pci_doe_mb *doe_mb = mb;
401
402 /* Stop all pending work items from starting */
403 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
404
405 /* Cancel an in progress work item, if necessary */
406 set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
407 wake_up(&doe_mb->wq);
408
409 /* Flush all work items */
410 flush_workqueue(doe_mb->work_queue);
411}
412
413/**
414 * pcim_doe_create_mb() - Create a DOE mailbox object
415 *
416 * @pdev: PCI device to create the DOE mailbox for
417 * @cap_offset: Offset of the DOE mailbox
418 *
419 * Create a single mailbox object to manage the mailbox protocol at the
420 * cap_offset specified.
421 *
422 * RETURNS: created mailbox object on success
423 * ERR_PTR(-errno) on failure
424 */
425struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset)
426{
427 struct pci_doe_mb *doe_mb;
428 struct device *dev = &pdev->dev;
429 int rc;
430
431 doe_mb = devm_kzalloc(dev, sizeof(*doe_mb), GFP_KERNEL);
432 if (!doe_mb)
433 return ERR_PTR(-ENOMEM);
434
435 doe_mb->pdev = pdev;
436 doe_mb->cap_offset = cap_offset;
437 init_waitqueue_head(&doe_mb->wq);
438
439 xa_init(&doe_mb->prots);
440 rc = devm_add_action(dev, pci_doe_xa_destroy, doe_mb);
441 if (rc)
442 return ERR_PTR(rc);
443
444 doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0,
445 dev_driver_string(&pdev->dev),
446 pci_name(pdev),
447 doe_mb->cap_offset);
448 if (!doe_mb->work_queue) {
449 pci_err(pdev, "[%x] failed to allocate work queue\n",
450 doe_mb->cap_offset);
451 return ERR_PTR(-ENOMEM);
452 }
453 rc = devm_add_action_or_reset(dev, pci_doe_destroy_workqueue, doe_mb);
454 if (rc)
455 return ERR_PTR(rc);
456
457 /* Reset the mailbox by issuing an abort */
458 rc = pci_doe_abort(doe_mb);
459 if (rc) {
460 pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n",
461 doe_mb->cap_offset, rc);
462 return ERR_PTR(rc);
463 }
464
465 /*
466 * The state machine and the mailbox should be in sync now;
467 * Set up mailbox flush prior to using the mailbox to query protocols.
468 */
469 rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb);
470 if (rc)
471 return ERR_PTR(rc);
472
473 rc = pci_doe_cache_protocols(doe_mb);
474 if (rc) {
475 pci_err(pdev, "[%x] failed to cache protocols : %d\n",
476 doe_mb->cap_offset, rc);
477 return ERR_PTR(rc);
478 }
479
480 return doe_mb;
481}
482EXPORT_SYMBOL_GPL(pcim_doe_create_mb);
483
484/**
485 * pci_doe_supports_prot() - Return if the DOE instance supports the given
486 * protocol
487 * @doe_mb: DOE mailbox capability to query
488 * @vid: Protocol Vendor ID
489 * @type: Protocol type
490 *
491 * RETURNS: True if the DOE mailbox supports the protocol specified
492 */
493bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type)
494{
495 unsigned long index;
496 void *entry;
497
498 /* The discovery protocol must always be supported */
499 if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_PROTOCOL_DISCOVERY)
500 return true;
501
502 xa_for_each(&doe_mb->prots, index, entry)
503 if (entry == pci_doe_xa_prot_entry(vid, type))
504 return true;
505
506 return false;
507}
508EXPORT_SYMBOL_GPL(pci_doe_supports_prot);
509
510/**
511 * pci_doe_submit_task() - Submit a task to be processed by the state machine
512 *
513 * @doe_mb: DOE mailbox capability to submit to
514 * @task: task to be queued
515 *
516 * Submit a DOE task (request/response) to the DOE mailbox to be processed.
517 * Returns upon queueing the task object. If the queue is full this function
518 * will sleep until there is room in the queue.
519 *
520 * task->complete will be called when the state machine is done processing this
521 * task.
522 *
523 * Excess data will be discarded.
524 *
525 * RETURNS: 0 when task has been successfully queued, -ERRNO on error
526 */
527int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
528{
529 if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type))
530 return -EINVAL;
531
532 /*
533 * DOE requests must be a whole number of DW and the response needs to
534 * be big enough for at least 1 DW
535 */
536 if (task->request_pl_sz % sizeof(u32) ||
537 task->response_pl_sz < sizeof(u32))
538 return -EINVAL;
539
540 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags))
541 return -EIO;
542
543 task->doe_mb = doe_mb;
544 INIT_WORK(&task->work, doe_statemachine_work);
545 queue_work(doe_mb->work_queue, &task->work);
546 return 0;
547}
548EXPORT_SYMBOL_GPL(pci_doe_submit_task);