Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * SSH request transport layer.
4 *
5 * Copyright (C) 2019-2022 Maximilian Luz <luzmaximilian@gmail.com>
6 */
7
8#include <asm/unaligned.h>
9#include <linux/atomic.h>
10#include <linux/completion.h>
11#include <linux/error-injection.h>
12#include <linux/ktime.h>
13#include <linux/limits.h>
14#include <linux/list.h>
15#include <linux/slab.h>
16#include <linux/spinlock.h>
17#include <linux/types.h>
18#include <linux/workqueue.h>
19
20#include <linux/surface_aggregator/serial_hub.h>
21#include <linux/surface_aggregator/controller.h>
22
23#include "ssh_packet_layer.h"
24#include "ssh_request_layer.h"
25
26#include "trace.h"
27
28/*
29 * SSH_RTL_REQUEST_TIMEOUT - Request timeout.
30 *
31 * Timeout as ktime_t delta for request responses. If we have not received a
32 * response in this time-frame after finishing the underlying packet
33 * transmission, the request will be completed with %-ETIMEDOUT as status
34 * code.
35 */
36#define SSH_RTL_REQUEST_TIMEOUT ms_to_ktime(3000)
37
38/*
39 * SSH_RTL_REQUEST_TIMEOUT_RESOLUTION - Request timeout granularity.
40 *
41 * Time-resolution for timeouts. Should be larger than one jiffy to avoid
42 * direct re-scheduling of reaper work_struct.
43 */
44#define SSH_RTL_REQUEST_TIMEOUT_RESOLUTION ms_to_ktime(max(2000 / HZ, 50))
45
46/*
47 * SSH_RTL_MAX_PENDING - Maximum number of pending requests.
48 *
49 * Maximum number of requests concurrently waiting to be completed (i.e.
50 * waiting for the corresponding packet transmission to finish if they don't
51 * have a response or waiting for a response if they have one).
52 */
53#define SSH_RTL_MAX_PENDING 3
54
55/*
56 * SSH_RTL_TX_BATCH - Maximum number of requests processed per work execution.
57 * Used to prevent livelocking of the workqueue. Value chosen via educated
58 * guess, may be adjusted.
59 */
60#define SSH_RTL_TX_BATCH 10
61
62#ifdef CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION
63
64/**
65 * ssh_rtl_should_drop_response() - Error injection hook to drop request
66 * responses.
67 *
68 * Useful to cause request transmission timeouts in the driver by dropping the
69 * response to a request.
70 */
71static noinline bool ssh_rtl_should_drop_response(void)
72{
73 return false;
74}
75ALLOW_ERROR_INJECTION(ssh_rtl_should_drop_response, TRUE);
76
77#else
78
79static inline bool ssh_rtl_should_drop_response(void)
80{
81 return false;
82}
83
84#endif
85
86static u16 ssh_request_get_rqid(struct ssh_request *rqst)
87{
88 return get_unaligned_le16(rqst->packet.data.ptr
89 + SSH_MSGOFFSET_COMMAND(rqid));
90}
91
92static u32 ssh_request_get_rqid_safe(struct ssh_request *rqst)
93{
94 if (!rqst->packet.data.ptr)
95 return U32_MAX;
96
97 return ssh_request_get_rqid(rqst);
98}
99
100static void ssh_rtl_queue_remove(struct ssh_request *rqst)
101{
102 struct ssh_rtl *rtl = ssh_request_rtl(rqst);
103
104 spin_lock(&rtl->queue.lock);
105
106 if (!test_and_clear_bit(SSH_REQUEST_SF_QUEUED_BIT, &rqst->state)) {
107 spin_unlock(&rtl->queue.lock);
108 return;
109 }
110
111 list_del(&rqst->node);
112
113 spin_unlock(&rtl->queue.lock);
114 ssh_request_put(rqst);
115}
116
117static bool ssh_rtl_queue_empty(struct ssh_rtl *rtl)
118{
119 bool empty;
120
121 spin_lock(&rtl->queue.lock);
122 empty = list_empty(&rtl->queue.head);
123 spin_unlock(&rtl->queue.lock);
124
125 return empty;
126}
127
128static void ssh_rtl_pending_remove(struct ssh_request *rqst)
129{
130 struct ssh_rtl *rtl = ssh_request_rtl(rqst);
131
132 spin_lock(&rtl->pending.lock);
133
134 if (!test_and_clear_bit(SSH_REQUEST_SF_PENDING_BIT, &rqst->state)) {
135 spin_unlock(&rtl->pending.lock);
136 return;
137 }
138
139 atomic_dec(&rtl->pending.count);
140 list_del(&rqst->node);
141
142 spin_unlock(&rtl->pending.lock);
143
144 ssh_request_put(rqst);
145}
146
147static int ssh_rtl_tx_pending_push(struct ssh_request *rqst)
148{
149 struct ssh_rtl *rtl = ssh_request_rtl(rqst);
150
151 spin_lock(&rtl->pending.lock);
152
153 if (test_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state)) {
154 spin_unlock(&rtl->pending.lock);
155 return -EINVAL;
156 }
157
158 if (test_and_set_bit(SSH_REQUEST_SF_PENDING_BIT, &rqst->state)) {
159 spin_unlock(&rtl->pending.lock);
160 return -EALREADY;
161 }
162
163 atomic_inc(&rtl->pending.count);
164 list_add_tail(&ssh_request_get(rqst)->node, &rtl->pending.head);
165
166 spin_unlock(&rtl->pending.lock);
167 return 0;
168}
169
170static void ssh_rtl_complete_with_status(struct ssh_request *rqst, int status)
171{
172 struct ssh_rtl *rtl = ssh_request_rtl(rqst);
173
174 trace_ssam_request_complete(rqst, status);
175
176 /* rtl/ptl may not be set if we're canceling before submitting. */
177 rtl_dbg_cond(rtl, "rtl: completing request (rqid: %#06x, status: %d)\n",
178 ssh_request_get_rqid_safe(rqst), status);
179
180 rqst->ops->complete(rqst, NULL, NULL, status);
181}
182
183static void ssh_rtl_complete_with_rsp(struct ssh_request *rqst,
184 const struct ssh_command *cmd,
185 const struct ssam_span *data)
186{
187 struct ssh_rtl *rtl = ssh_request_rtl(rqst);
188
189 trace_ssam_request_complete(rqst, 0);
190
191 rtl_dbg(rtl, "rtl: completing request with response (rqid: %#06x)\n",
192 ssh_request_get_rqid(rqst));
193
194 rqst->ops->complete(rqst, cmd, data, 0);
195}
196
197static bool ssh_rtl_tx_can_process(struct ssh_request *rqst)
198{
199 struct ssh_rtl *rtl = ssh_request_rtl(rqst);
200
201 if (test_bit(SSH_REQUEST_TY_FLUSH_BIT, &rqst->state))
202 return !atomic_read(&rtl->pending.count);
203
204 return atomic_read(&rtl->pending.count) < SSH_RTL_MAX_PENDING;
205}
206
207static struct ssh_request *ssh_rtl_tx_next(struct ssh_rtl *rtl)
208{
209 struct ssh_request *rqst = ERR_PTR(-ENOENT);
210 struct ssh_request *p, *n;
211
212 spin_lock(&rtl->queue.lock);
213
214 /* Find first non-locked request and remove it. */
215 list_for_each_entry_safe(p, n, &rtl->queue.head, node) {
216 if (unlikely(test_bit(SSH_REQUEST_SF_LOCKED_BIT, &p->state)))
217 continue;
218
219 if (!ssh_rtl_tx_can_process(p)) {
220 rqst = ERR_PTR(-EBUSY);
221 break;
222 }
223
224 /* Remove from queue and mark as transmitting. */
225 set_bit(SSH_REQUEST_SF_TRANSMITTING_BIT, &p->state);
226 /* Ensure state never gets zero. */
227 smp_mb__before_atomic();
228 clear_bit(SSH_REQUEST_SF_QUEUED_BIT, &p->state);
229
230 list_del(&p->node);
231
232 rqst = p;
233 break;
234 }
235
236 spin_unlock(&rtl->queue.lock);
237 return rqst;
238}
239
240static int ssh_rtl_tx_try_process_one(struct ssh_rtl *rtl)
241{
242 struct ssh_request *rqst;
243 int status;
244
245 /* Get and prepare next request for transmit. */
246 rqst = ssh_rtl_tx_next(rtl);
247 if (IS_ERR(rqst))
248 return PTR_ERR(rqst);
249
250 /* Add it to/mark it as pending. */
251 status = ssh_rtl_tx_pending_push(rqst);
252 if (status) {
253 ssh_request_put(rqst);
254 return -EAGAIN;
255 }
256
257 /* Submit packet. */
258 status = ssh_ptl_submit(&rtl->ptl, &rqst->packet);
259 if (status == -ESHUTDOWN) {
260 /*
261 * Packet has been refused due to the packet layer shutting
262 * down. Complete it here.
263 */
264 set_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state);
265 /*
266 * Note: A barrier is not required here, as there are only two
267 * references in the system at this point: The one that we have,
268 * and the other one that belongs to the pending set. Due to the
269 * request being marked as "transmitting", our process is the
270 * only one allowed to remove the pending node and change the
271 * state. Normally, the task would fall to the packet callback,
272 * but as this is a path where submission failed, this callback
273 * will never be executed.
274 */
275
276 ssh_rtl_pending_remove(rqst);
277 ssh_rtl_complete_with_status(rqst, -ESHUTDOWN);
278
279 ssh_request_put(rqst);
280 return -ESHUTDOWN;
281
282 } else if (status) {
283 /*
284 * If submitting the packet failed and the packet layer isn't
285 * shutting down, the packet has either been submitted/queued
286 * before (-EALREADY, which cannot happen as we have
287 * guaranteed that requests cannot be re-submitted), or the
288 * packet was marked as locked (-EINVAL). To mark the packet
289 * locked at this stage, the request, and thus the packets
290 * itself, had to have been canceled. Simply drop the
291 * reference. Cancellation itself will remove it from the set
292 * of pending requests.
293 */
294
295 WARN_ON(status != -EINVAL);
296
297 ssh_request_put(rqst);
298 return -EAGAIN;
299 }
300
301 ssh_request_put(rqst);
302 return 0;
303}
304
305static bool ssh_rtl_tx_schedule(struct ssh_rtl *rtl)
306{
307 if (atomic_read(&rtl->pending.count) >= SSH_RTL_MAX_PENDING)
308 return false;
309
310 if (ssh_rtl_queue_empty(rtl))
311 return false;
312
313 return schedule_work(&rtl->tx.work);
314}
315
316static void ssh_rtl_tx_work_fn(struct work_struct *work)
317{
318 struct ssh_rtl *rtl = to_ssh_rtl(work, tx.work);
319 unsigned int iterations = SSH_RTL_TX_BATCH;
320 int status;
321
322 /*
323 * Try to be nice and not block/live-lock the workqueue: Run a maximum
324 * of 10 tries, then re-submit if necessary. This should not be
325 * necessary for normal execution, but guarantee it anyway.
326 */
327 do {
328 status = ssh_rtl_tx_try_process_one(rtl);
329 if (status == -ENOENT || status == -EBUSY)
330 return; /* No more requests to process. */
331
332 if (status == -ESHUTDOWN) {
333 /*
334 * Packet system shutting down. No new packets can be
335 * transmitted. Return silently, the party initiating
336 * the shutdown should handle the rest.
337 */
338 return;
339 }
340
341 WARN_ON(status != 0 && status != -EAGAIN);
342 } while (--iterations);
343
344 /* Out of tries, reschedule. */
345 ssh_rtl_tx_schedule(rtl);
346}
347
348/**
349 * ssh_rtl_submit() - Submit a request to the transport layer.
350 * @rtl: The request transport layer.
351 * @rqst: The request to submit.
352 *
353 * Submits a request to the transport layer. A single request may not be
354 * submitted multiple times without reinitializing it.
355 *
356 * Return: Returns zero on success, %-EINVAL if the request type is invalid or
357 * the request has been canceled prior to submission, %-EALREADY if the
358 * request has already been submitted, or %-ESHUTDOWN in case the request
359 * transport layer has been shut down.
360 */
361int ssh_rtl_submit(struct ssh_rtl *rtl, struct ssh_request *rqst)
362{
363 trace_ssam_request_submit(rqst);
364
365 /*
366 * Ensure that requests expecting a response are sequenced. If this
367 * invariant ever changes, see the comment in ssh_rtl_complete() on what
368 * is required to be changed in the code.
369 */
370 if (test_bit(SSH_REQUEST_TY_HAS_RESPONSE_BIT, &rqst->state))
371 if (!test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &rqst->packet.state))
372 return -EINVAL;
373
374 spin_lock(&rtl->queue.lock);
375
376 /*
377 * Try to set ptl and check if this request has already been submitted.
378 *
379 * Must be inside lock as we might run into a lost update problem
380 * otherwise: If this were outside of the lock, cancellation in
381 * ssh_rtl_cancel_nonpending() may run after we've set the ptl
382 * reference but before we enter the lock. In that case, we'd detect
383 * that the request is being added to the queue and would try to remove
384 * it from that, but removal might fail because it hasn't actually been
385 * added yet. By putting this cmpxchg in the critical section, we
386 * ensure that the queuing detection only triggers when we are already
387 * in the critical section and the remove process will wait until the
388 * push operation has been completed (via lock) due to that. Only then,
389 * we can safely try to remove it.
390 */
391 if (cmpxchg(&rqst->packet.ptl, NULL, &rtl->ptl)) {
392 spin_unlock(&rtl->queue.lock);
393 return -EALREADY;
394 }
395
396 /*
397 * Ensure that we set ptl reference before we continue modifying state.
398 * This is required for non-pending cancellation. This barrier is paired
399 * with the one in ssh_rtl_cancel_nonpending().
400 *
401 * By setting the ptl reference before we test for "locked", we can
402 * check if the "locked" test may have already run. See comments in
403 * ssh_rtl_cancel_nonpending() for more detail.
404 */
405 smp_mb__after_atomic();
406
407 if (test_bit(SSH_RTL_SF_SHUTDOWN_BIT, &rtl->state)) {
408 spin_unlock(&rtl->queue.lock);
409 return -ESHUTDOWN;
410 }
411
412 if (test_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state)) {
413 spin_unlock(&rtl->queue.lock);
414 return -EINVAL;
415 }
416
417 set_bit(SSH_REQUEST_SF_QUEUED_BIT, &rqst->state);
418 list_add_tail(&ssh_request_get(rqst)->node, &rtl->queue.head);
419
420 spin_unlock(&rtl->queue.lock);
421
422 ssh_rtl_tx_schedule(rtl);
423 return 0;
424}
425
426static void ssh_rtl_timeout_reaper_mod(struct ssh_rtl *rtl, ktime_t now,
427 ktime_t expires)
428{
429 unsigned long delta = msecs_to_jiffies(ktime_ms_delta(expires, now));
430 ktime_t aexp = ktime_add(expires, SSH_RTL_REQUEST_TIMEOUT_RESOLUTION);
431
432 spin_lock(&rtl->rtx_timeout.lock);
433
434 /* Re-adjust / schedule reaper only if it is above resolution delta. */
435 if (ktime_before(aexp, rtl->rtx_timeout.expires)) {
436 rtl->rtx_timeout.expires = expires;
437 mod_delayed_work(system_wq, &rtl->rtx_timeout.reaper, delta);
438 }
439
440 spin_unlock(&rtl->rtx_timeout.lock);
441}
442
443static void ssh_rtl_timeout_start(struct ssh_request *rqst)
444{
445 struct ssh_rtl *rtl = ssh_request_rtl(rqst);
446 ktime_t timestamp = ktime_get_coarse_boottime();
447 ktime_t timeout = rtl->rtx_timeout.timeout;
448
449 if (test_bit(SSH_REQUEST_SF_LOCKED_BIT, &rqst->state))
450 return;
451
452 /*
453 * Note: The timestamp gets set only once. This happens on the packet
454 * callback. All other access to it is read-only.
455 */
456 WRITE_ONCE(rqst->timestamp, timestamp);
457 /*
458 * Ensure timestamp is set before starting the reaper. Paired with
459 * implicit barrier following check on ssh_request_get_expiration() in
460 * ssh_rtl_timeout_reap.
461 */
462 smp_mb__after_atomic();
463
464 ssh_rtl_timeout_reaper_mod(rtl, timestamp, timestamp + timeout);
465}
466
467static void ssh_rtl_complete(struct ssh_rtl *rtl,
468 const struct ssh_command *command,
469 const struct ssam_span *command_data)
470{
471 struct ssh_request *r = NULL;
472 struct ssh_request *p, *n;
473 u16 rqid = get_unaligned_le16(&command->rqid);
474
475 trace_ssam_rx_response_received(command, command_data->len);
476
477 /*
478 * Get request from pending based on request ID and mark it as response
479 * received and locked.
480 */
481 spin_lock(&rtl->pending.lock);
482 list_for_each_entry_safe(p, n, &rtl->pending.head, node) {
483 /* We generally expect requests to be processed in order. */
484 if (unlikely(ssh_request_get_rqid(p) != rqid))
485 continue;
486
487 /* Simulate response timeout. */
488 if (ssh_rtl_should_drop_response()) {
489 spin_unlock(&rtl->pending.lock);
490
491 trace_ssam_ei_rx_drop_response(p);
492 rtl_info(rtl, "request error injection: dropping response for request %p\n",
493 &p->packet);
494 return;
495 }
496
497 /*
498 * Mark as "response received" and "locked" as we're going to
499 * complete it.
500 */
501 set_bit(SSH_REQUEST_SF_LOCKED_BIT, &p->state);
502 set_bit(SSH_REQUEST_SF_RSPRCVD_BIT, &p->state);
503 /* Ensure state never gets zero. */
504 smp_mb__before_atomic();
505 clear_bit(SSH_REQUEST_SF_PENDING_BIT, &p->state);
506
507 atomic_dec(&rtl->pending.count);
508 list_del(&p->node);
509
510 r = p;
511 break;
512 }
513 spin_unlock(&rtl->pending.lock);
514
515 if (!r) {
516 rtl_warn(rtl, "rtl: dropping unexpected command message (rqid = %#06x)\n",
517 rqid);
518 return;
519 }
520
521 /* If the request hasn't been completed yet, we will do this now. */
522 if (test_and_set_bit(SSH_REQUEST_SF_COMPLETED_BIT, &r->state)) {
523 ssh_request_put(r);
524 ssh_rtl_tx_schedule(rtl);
525 return;
526 }
527
528 /*
529 * Make sure the request has been transmitted. In case of a sequenced
530 * request, we are guaranteed that the completion callback will run on
531 * the receiver thread directly when the ACK for the packet has been
532 * received. Similarly, this function is guaranteed to run on the
533 * receiver thread. Thus we are guaranteed that if the packet has been
534 * successfully transmitted and received an ACK, the transmitted flag
535 * has been set and is visible here.
536 *
537 * We are currently not handling unsequenced packets here, as those
538 * should never expect a response as ensured in ssh_rtl_submit. If this
539 * ever changes, one would have to test for
540 *
541 * (r->state & (transmitting | transmitted))
542 *
543 * on unsequenced packets to determine if they could have been
544 * transmitted. There are no synchronization guarantees as in the
545 * sequenced case, since, in this case, the callback function will not
546 * run on the same thread. Thus an exact determination is impossible.
547 */
548 if (!test_bit(SSH_REQUEST_SF_TRANSMITTED_BIT, &r->state)) {
549 rtl_err(rtl, "rtl: received response before ACK for request (rqid = %#06x)\n",
550 rqid);
551
552 /*
553 * NB: Timeout has already been canceled, request already been
554 * removed from pending and marked as locked and completed. As
555 * we receive a "false" response, the packet might still be
556 * queued though.
557 */
558 ssh_rtl_queue_remove(r);
559
560 ssh_rtl_complete_with_status(r, -EREMOTEIO);
561 ssh_request_put(r);
562
563 ssh_rtl_tx_schedule(rtl);
564 return;
565 }
566
567 /*
568 * NB: Timeout has already been canceled, request already been
569 * removed from pending and marked as locked and completed. The request
570 * can also not be queued any more, as it has been marked as
571 * transmitting and later transmitted. Thus no need to remove it from
572 * anywhere.
573 */
574
575 ssh_rtl_complete_with_rsp(r, command, command_data);
576 ssh_request_put(r);
577
578 ssh_rtl_tx_schedule(rtl);
579}
580
581static bool ssh_rtl_cancel_nonpending(struct ssh_request *r)
582{
583 struct ssh_rtl *rtl;
584 unsigned long flags, fixed;
585 bool remove;
586
587 /*
588 * Handle unsubmitted request: Try to mark the packet as locked,
589 * expecting the state to be zero (i.e. unsubmitted). Note that, if
590 * setting the state worked, we might still be adding the packet to the
591 * queue in a currently executing submit call. In that case, however,
592 * ptl reference must have been set previously, as locked is checked
593 * after setting ptl. Furthermore, when the ptl reference is set, the
594 * submission process is guaranteed to have entered the critical
595 * section. Thus only if we successfully locked this request and ptl is
596 * NULL, we have successfully removed the request, i.e. we are
597 * guaranteed that, due to the "locked" check in ssh_rtl_submit(), the
598 * packet will never be added. Otherwise, we need to try and grab it
599 * from the queue, where we are now guaranteed that the packet is or has
600 * been due to the critical section.
601 *
602 * Note that if the cmpxchg() fails, we are guaranteed that ptl has
603 * been set and is non-NULL, as states can only be nonzero after this
604 * has been set. Also note that we need to fetch the static (type)
605 * flags to ensure that they don't cause the cmpxchg() to fail.
606 */
607 fixed = READ_ONCE(r->state) & SSH_REQUEST_FLAGS_TY_MASK;
608 flags = cmpxchg(&r->state, fixed, SSH_REQUEST_SF_LOCKED_BIT);
609
610 /*
611 * Force correct ordering with regards to state and ptl reference access
612 * to safe-guard cancellation to concurrent submission against a
613 * lost-update problem. First try to exchange state, then also check
614 * ptl if that worked. This barrier is paired with the
615 * one in ssh_rtl_submit().
616 */
617 smp_mb__after_atomic();
618
619 if (flags == fixed && !READ_ONCE(r->packet.ptl)) {
620 if (test_and_set_bit(SSH_REQUEST_SF_COMPLETED_BIT, &r->state))
621 return true;
622
623 ssh_rtl_complete_with_status(r, -ECANCELED);
624 return true;
625 }
626
627 rtl = ssh_request_rtl(r);
628 spin_lock(&rtl->queue.lock);
629
630 /*
631 * Note: 1) Requests cannot be re-submitted. 2) If a request is
632 * queued, it cannot be "transmitting"/"pending" yet. Thus, if we
633 * successfully remove the request here, we have removed all its
634 * occurrences in the system.
635 */
636
637 remove = test_and_clear_bit(SSH_REQUEST_SF_QUEUED_BIT, &r->state);
638 if (!remove) {
639 spin_unlock(&rtl->queue.lock);
640 return false;
641 }
642
643 set_bit(SSH_REQUEST_SF_LOCKED_BIT, &r->state);
644 list_del(&r->node);
645
646 spin_unlock(&rtl->queue.lock);
647
648 ssh_request_put(r); /* Drop reference obtained from queue. */
649
650 if (test_and_set_bit(SSH_REQUEST_SF_COMPLETED_BIT, &r->state))
651 return true;
652
653 ssh_rtl_complete_with_status(r, -ECANCELED);
654 return true;
655}
656
657static bool ssh_rtl_cancel_pending(struct ssh_request *r)
658{
659 /* If the packet is already locked, it's going to be removed shortly. */
660 if (test_and_set_bit(SSH_REQUEST_SF_LOCKED_BIT, &r->state))
661 return true;
662
663 /*
664 * Now that we have locked the packet, we have guaranteed that it can't
665 * be added to the system any more. If ptl is NULL, the locked
666 * check in ssh_rtl_submit() has not been run and any submission,
667 * currently in progress or called later, won't add the packet. Thus we
668 * can directly complete it.
669 *
670 * The implicit memory barrier of test_and_set_bit() should be enough
671 * to ensure that the correct order (first lock, then check ptl) is
672 * ensured. This is paired with the barrier in ssh_rtl_submit().
673 */
674 if (!READ_ONCE(r->packet.ptl)) {
675 if (test_and_set_bit(SSH_REQUEST_SF_COMPLETED_BIT, &r->state))
676 return true;
677
678 ssh_rtl_complete_with_status(r, -ECANCELED);
679 return true;
680 }
681
682 /*
683 * Try to cancel the packet. If the packet has not been completed yet,
684 * this will subsequently (and synchronously) call the completion
685 * callback of the packet, which will complete the request.
686 */
687 ssh_ptl_cancel(&r->packet);
688
689 /*
690 * If the packet has been completed with success, i.e. has not been
691 * canceled by the above call, the request may not have been completed
692 * yet (may be waiting for a response). Check if we need to do this
693 * here.
694 */
695 if (test_and_set_bit(SSH_REQUEST_SF_COMPLETED_BIT, &r->state))
696 return true;
697
698 ssh_rtl_queue_remove(r);
699 ssh_rtl_pending_remove(r);
700 ssh_rtl_complete_with_status(r, -ECANCELED);
701
702 return true;
703}
704
705/**
706 * ssh_rtl_cancel() - Cancel request.
707 * @rqst: The request to cancel.
708 * @pending: Whether to also cancel pending requests.
709 *
710 * Cancels the given request. If @pending is %false, this will not cancel
711 * pending requests, i.e. requests that have already been submitted to the
712 * packet layer but not been completed yet. If @pending is %true, this will
713 * cancel the given request regardless of the state it is in.
714 *
715 * If the request has been canceled by calling this function, both completion
716 * and release callbacks of the request will be executed in a reasonable
717 * time-frame. This may happen during execution of this function, however,
718 * there is no guarantee for this. For example, a request currently
719 * transmitting will be canceled/completed only after transmission has
720 * completed, and the respective callbacks will be executed on the transmitter
721 * thread, which may happen during, but also some time after execution of the
722 * cancel function.
723 *
724 * Return: Returns %true if the given request has been canceled or completed,
725 * either by this function or prior to calling this function, %false
726 * otherwise. If @pending is %true, this function will always return %true.
727 */
728bool ssh_rtl_cancel(struct ssh_request *rqst, bool pending)
729{
730 struct ssh_rtl *rtl;
731 bool canceled;
732
733 if (test_and_set_bit(SSH_REQUEST_SF_CANCELED_BIT, &rqst->state))
734 return true;
735
736 trace_ssam_request_cancel(rqst);
737
738 if (pending)
739 canceled = ssh_rtl_cancel_pending(rqst);
740 else
741 canceled = ssh_rtl_cancel_nonpending(rqst);
742
743 /* Note: rtl may be NULL if request has not been submitted yet. */
744 rtl = ssh_request_rtl(rqst);
745 if (canceled && rtl)
746 ssh_rtl_tx_schedule(rtl);
747
748 return canceled;
749}
750
751static void ssh_rtl_packet_callback(struct ssh_packet *p, int status)
752{
753 struct ssh_request *r = to_ssh_request(p);
754
755 if (unlikely(status)) {
756 set_bit(SSH_REQUEST_SF_LOCKED_BIT, &r->state);
757
758 if (test_and_set_bit(SSH_REQUEST_SF_COMPLETED_BIT, &r->state))
759 return;
760
761 /*
762 * The packet may get canceled even though it has not been
763 * submitted yet. The request may still be queued. Check the
764 * queue and remove it if necessary. As the timeout would have
765 * been started in this function on success, there's no need
766 * to cancel it here.
767 */
768 ssh_rtl_queue_remove(r);
769 ssh_rtl_pending_remove(r);
770 ssh_rtl_complete_with_status(r, status);
771
772 ssh_rtl_tx_schedule(ssh_request_rtl(r));
773 return;
774 }
775
776 /* Update state: Mark as transmitted and clear transmitting. */
777 set_bit(SSH_REQUEST_SF_TRANSMITTED_BIT, &r->state);
778 /* Ensure state never gets zero. */
779 smp_mb__before_atomic();
780 clear_bit(SSH_REQUEST_SF_TRANSMITTING_BIT, &r->state);
781
782 /* If we expect a response, we just need to start the timeout. */
783 if (test_bit(SSH_REQUEST_TY_HAS_RESPONSE_BIT, &r->state)) {
784 /*
785 * Note: This is the only place where the timestamp gets set,
786 * all other access to it is read-only.
787 */
788 ssh_rtl_timeout_start(r);
789 return;
790 }
791
792 /*
793 * If we don't expect a response, lock, remove, and complete the
794 * request. Note that, at this point, the request is guaranteed to have
795 * left the queue and no timeout has been started. Thus we only need to
796 * remove it from pending. If the request has already been completed (it
797 * may have been canceled) return.
798 */
799
800 set_bit(SSH_REQUEST_SF_LOCKED_BIT, &r->state);
801 if (test_and_set_bit(SSH_REQUEST_SF_COMPLETED_BIT, &r->state))
802 return;
803
804 ssh_rtl_pending_remove(r);
805 ssh_rtl_complete_with_status(r, 0);
806
807 ssh_rtl_tx_schedule(ssh_request_rtl(r));
808}
809
810static ktime_t ssh_request_get_expiration(struct ssh_request *r, ktime_t timeout)
811{
812 ktime_t timestamp = READ_ONCE(r->timestamp);
813
814 if (timestamp != KTIME_MAX)
815 return ktime_add(timestamp, timeout);
816 else
817 return KTIME_MAX;
818}
819
820static void ssh_rtl_timeout_reap(struct work_struct *work)
821{
822 struct ssh_rtl *rtl = to_ssh_rtl(work, rtx_timeout.reaper.work);
823 struct ssh_request *r, *n;
824 LIST_HEAD(claimed);
825 ktime_t now = ktime_get_coarse_boottime();
826 ktime_t timeout = rtl->rtx_timeout.timeout;
827 ktime_t next = KTIME_MAX;
828
829 trace_ssam_rtl_timeout_reap(atomic_read(&rtl->pending.count));
830
831 /*
832 * Mark reaper as "not pending". This is done before checking any
833 * requests to avoid lost-update type problems.
834 */
835 spin_lock(&rtl->rtx_timeout.lock);
836 rtl->rtx_timeout.expires = KTIME_MAX;
837 spin_unlock(&rtl->rtx_timeout.lock);
838
839 spin_lock(&rtl->pending.lock);
840 list_for_each_entry_safe(r, n, &rtl->pending.head, node) {
841 ktime_t expires = ssh_request_get_expiration(r, timeout);
842
843 /*
844 * Check if the timeout hasn't expired yet. Find out next
845 * expiration date to be handled after this run.
846 */
847 if (ktime_after(expires, now)) {
848 next = ktime_before(expires, next) ? expires : next;
849 continue;
850 }
851
852 /* Avoid further transitions if locked. */
853 if (test_and_set_bit(SSH_REQUEST_SF_LOCKED_BIT, &r->state))
854 continue;
855
856 /*
857 * We have now marked the packet as locked. Thus it cannot be
858 * added to the pending or queued lists again after we've
859 * removed it here. We can therefore re-use the node of this
860 * packet temporarily.
861 */
862
863 clear_bit(SSH_REQUEST_SF_PENDING_BIT, &r->state);
864
865 atomic_dec(&rtl->pending.count);
866 list_move_tail(&r->node, &claimed);
867 }
868 spin_unlock(&rtl->pending.lock);
869
870 /* Cancel and complete the request. */
871 list_for_each_entry_safe(r, n, &claimed, node) {
872 trace_ssam_request_timeout(r);
873
874 /*
875 * At this point we've removed the packet from pending. This
876 * means that we've obtained the last (only) reference of the
877 * system to it. Thus we can just complete it.
878 */
879 if (!test_and_set_bit(SSH_REQUEST_SF_COMPLETED_BIT, &r->state))
880 ssh_rtl_complete_with_status(r, -ETIMEDOUT);
881
882 /*
883 * Drop the reference we've obtained by removing it from the
884 * pending set.
885 */
886 list_del(&r->node);
887 ssh_request_put(r);
888 }
889
890 /* Ensure that the reaper doesn't run again immediately. */
891 next = max(next, ktime_add(now, SSH_RTL_REQUEST_TIMEOUT_RESOLUTION));
892 if (next != KTIME_MAX)
893 ssh_rtl_timeout_reaper_mod(rtl, now, next);
894
895 ssh_rtl_tx_schedule(rtl);
896}
897
898static void ssh_rtl_rx_event(struct ssh_rtl *rtl, const struct ssh_command *cmd,
899 const struct ssam_span *data)
900{
901 trace_ssam_rx_event_received(cmd, data->len);
902
903 rtl_dbg(rtl, "rtl: handling event (rqid: %#06x)\n",
904 get_unaligned_le16(&cmd->rqid));
905
906 rtl->ops.handle_event(rtl, cmd, data);
907}
908
909static void ssh_rtl_rx_command(struct ssh_ptl *p, const struct ssam_span *data)
910{
911 struct ssh_rtl *rtl = to_ssh_rtl(p, ptl);
912 struct device *dev = &p->serdev->dev;
913 struct ssh_command *command;
914 struct ssam_span command_data;
915
916 if (sshp_parse_command(dev, data, &command, &command_data))
917 return;
918
919 /*
920 * Check if the message was intended for us. If not, drop it.
921 *
922 * Note: We will need to change this to handle debug messages. On newer
923 * generation devices, these seem to be sent to tid_out=0x03. We as
924 * host can still receive them as they can be forwarded via an override
925 * option on SAM, but doing so does not change tid_out=0x00.
926 */
927 if (command->tid_out != 0x00) {
928 rtl_warn(rtl, "rtl: dropping message not intended for us (tid = %#04x)\n",
929 command->tid_out);
930 return;
931 }
932
933 if (ssh_rqid_is_event(get_unaligned_le16(&command->rqid)))
934 ssh_rtl_rx_event(rtl, command, &command_data);
935 else
936 ssh_rtl_complete(rtl, command, &command_data);
937}
938
939static void ssh_rtl_rx_data(struct ssh_ptl *p, const struct ssam_span *data)
940{
941 if (!data->len) {
942 ptl_err(p, "rtl: rx: no data frame payload\n");
943 return;
944 }
945
946 switch (data->ptr[0]) {
947 case SSH_PLD_TYPE_CMD:
948 ssh_rtl_rx_command(p, data);
949 break;
950
951 default:
952 ptl_err(p, "rtl: rx: unknown frame payload type (type: %#04x)\n",
953 data->ptr[0]);
954 break;
955 }
956}
957
958static void ssh_rtl_packet_release(struct ssh_packet *p)
959{
960 struct ssh_request *rqst;
961
962 rqst = to_ssh_request(p);
963 rqst->ops->release(rqst);
964}
965
966static const struct ssh_packet_ops ssh_rtl_packet_ops = {
967 .complete = ssh_rtl_packet_callback,
968 .release = ssh_rtl_packet_release,
969};
970
971/**
972 * ssh_request_init() - Initialize SSH request.
973 * @rqst: The request to initialize.
974 * @flags: Request flags, determining the type of the request.
975 * @ops: Request operations.
976 *
977 * Initializes the given SSH request and underlying packet. Sets the message
978 * buffer pointer to %NULL and the message buffer length to zero. This buffer
979 * has to be set separately via ssh_request_set_data() before submission and
980 * must contain a valid SSH request message.
981 *
982 * Return: Returns zero on success or %-EINVAL if the given flags are invalid.
983 */
984int ssh_request_init(struct ssh_request *rqst, enum ssam_request_flags flags,
985 const struct ssh_request_ops *ops)
986{
987 unsigned long type = BIT(SSH_PACKET_TY_BLOCKING_BIT);
988
989 /* Unsequenced requests cannot have a response. */
990 if (flags & SSAM_REQUEST_UNSEQUENCED && flags & SSAM_REQUEST_HAS_RESPONSE)
991 return -EINVAL;
992
993 if (!(flags & SSAM_REQUEST_UNSEQUENCED))
994 type |= BIT(SSH_PACKET_TY_SEQUENCED_BIT);
995
996 ssh_packet_init(&rqst->packet, type, SSH_PACKET_PRIORITY(DATA, 0),
997 &ssh_rtl_packet_ops);
998
999 INIT_LIST_HEAD(&rqst->node);
1000
1001 rqst->state = 0;
1002 if (flags & SSAM_REQUEST_HAS_RESPONSE)
1003 rqst->state |= BIT(SSH_REQUEST_TY_HAS_RESPONSE_BIT);
1004
1005 rqst->timestamp = KTIME_MAX;
1006 rqst->ops = ops;
1007
1008 return 0;
1009}
1010
1011/**
1012 * ssh_rtl_init() - Initialize request transport layer.
1013 * @rtl: The request transport layer to initialize.
1014 * @serdev: The underlying serial device, i.e. the lower-level transport.
1015 * @ops: Request transport layer operations.
1016 *
1017 * Initializes the given request transport layer and associated packet
1018 * transport layer. Transmitter and receiver threads must be started
1019 * separately via ssh_rtl_start(), after the request-layer has been
1020 * initialized and the lower-level serial device layer has been set up.
1021 *
1022 * Return: Returns zero on success and a nonzero error code on failure.
1023 */
1024int ssh_rtl_init(struct ssh_rtl *rtl, struct serdev_device *serdev,
1025 const struct ssh_rtl_ops *ops)
1026{
1027 struct ssh_ptl_ops ptl_ops;
1028 int status;
1029
1030 ptl_ops.data_received = ssh_rtl_rx_data;
1031
1032 status = ssh_ptl_init(&rtl->ptl, serdev, &ptl_ops);
1033 if (status)
1034 return status;
1035
1036 spin_lock_init(&rtl->queue.lock);
1037 INIT_LIST_HEAD(&rtl->queue.head);
1038
1039 spin_lock_init(&rtl->pending.lock);
1040 INIT_LIST_HEAD(&rtl->pending.head);
1041 atomic_set_release(&rtl->pending.count, 0);
1042
1043 INIT_WORK(&rtl->tx.work, ssh_rtl_tx_work_fn);
1044
1045 spin_lock_init(&rtl->rtx_timeout.lock);
1046 rtl->rtx_timeout.timeout = SSH_RTL_REQUEST_TIMEOUT;
1047 rtl->rtx_timeout.expires = KTIME_MAX;
1048 INIT_DELAYED_WORK(&rtl->rtx_timeout.reaper, ssh_rtl_timeout_reap);
1049
1050 rtl->ops = *ops;
1051
1052 return 0;
1053}
1054
1055/**
1056 * ssh_rtl_destroy() - Deinitialize request transport layer.
1057 * @rtl: The request transport layer to deinitialize.
1058 *
1059 * Deinitializes the given request transport layer and frees resources
1060 * associated with it. If receiver and/or transmitter threads have been
1061 * started, the layer must first be shut down via ssh_rtl_shutdown() before
1062 * this function can be called.
1063 */
1064void ssh_rtl_destroy(struct ssh_rtl *rtl)
1065{
1066 ssh_ptl_destroy(&rtl->ptl);
1067}
1068
1069/**
1070 * ssh_rtl_start() - Start request transmitter and receiver.
1071 * @rtl: The request transport layer.
1072 *
1073 * Return: Returns zero on success, a negative error code on failure.
1074 */
1075int ssh_rtl_start(struct ssh_rtl *rtl)
1076{
1077 int status;
1078
1079 status = ssh_ptl_tx_start(&rtl->ptl);
1080 if (status)
1081 return status;
1082
1083 ssh_rtl_tx_schedule(rtl);
1084
1085 status = ssh_ptl_rx_start(&rtl->ptl);
1086 if (status) {
1087 ssh_rtl_flush(rtl, msecs_to_jiffies(5000));
1088 ssh_ptl_tx_stop(&rtl->ptl);
1089 return status;
1090 }
1091
1092 return 0;
1093}
1094
1095struct ssh_flush_request {
1096 struct ssh_request base;
1097 struct completion completion;
1098 int status;
1099};
1100
1101static void ssh_rtl_flush_request_complete(struct ssh_request *r,
1102 const struct ssh_command *cmd,
1103 const struct ssam_span *data,
1104 int status)
1105{
1106 struct ssh_flush_request *rqst;
1107
1108 rqst = container_of(r, struct ssh_flush_request, base);
1109 rqst->status = status;
1110}
1111
1112static void ssh_rtl_flush_request_release(struct ssh_request *r)
1113{
1114 struct ssh_flush_request *rqst;
1115
1116 rqst = container_of(r, struct ssh_flush_request, base);
1117 complete_all(&rqst->completion);
1118}
1119
1120static const struct ssh_request_ops ssh_rtl_flush_request_ops = {
1121 .complete = ssh_rtl_flush_request_complete,
1122 .release = ssh_rtl_flush_request_release,
1123};
1124
1125/**
1126 * ssh_rtl_flush() - Flush the request transport layer.
1127 * @rtl: request transport layer
1128 * @timeout: timeout for the flush operation in jiffies
1129 *
1130 * Queue a special flush request and wait for its completion. This request
1131 * will be completed after all other currently queued and pending requests
1132 * have been completed. Instead of a normal data packet, this request submits
1133 * a special flush packet, meaning that upon completion, also the underlying
1134 * packet transport layer has been flushed.
1135 *
1136 * Flushing the request layer guarantees that all previously submitted
1137 * requests have been fully completed before this call returns. Additionally,
1138 * flushing blocks execution of all later submitted requests until the flush
1139 * has been completed.
1140 *
1141 * If the caller ensures that no new requests are submitted after a call to
1142 * this function, the request transport layer is guaranteed to have no
1143 * remaining requests when this call returns. The same guarantee does not hold
1144 * for the packet layer, on which control packets may still be queued after
1145 * this call.
1146 *
1147 * Return: Returns zero on success, %-ETIMEDOUT if the flush timed out and has
1148 * been canceled as a result of the timeout, or %-ESHUTDOWN if the packet
1149 * and/or request transport layer has been shut down before this call. May
1150 * also return %-EINTR if the underlying packet transmission has been
1151 * interrupted.
1152 */
1153int ssh_rtl_flush(struct ssh_rtl *rtl, unsigned long timeout)
1154{
1155 const unsigned int init_flags = SSAM_REQUEST_UNSEQUENCED;
1156 struct ssh_flush_request rqst;
1157 int status;
1158
1159 ssh_request_init(&rqst.base, init_flags, &ssh_rtl_flush_request_ops);
1160 rqst.base.packet.state |= BIT(SSH_PACKET_TY_FLUSH_BIT);
1161 rqst.base.packet.priority = SSH_PACKET_PRIORITY(FLUSH, 0);
1162 rqst.base.state |= BIT(SSH_REQUEST_TY_FLUSH_BIT);
1163
1164 init_completion(&rqst.completion);
1165
1166 status = ssh_rtl_submit(rtl, &rqst.base);
1167 if (status)
1168 return status;
1169
1170 ssh_request_put(&rqst.base);
1171
1172 if (!wait_for_completion_timeout(&rqst.completion, timeout)) {
1173 ssh_rtl_cancel(&rqst.base, true);
1174 wait_for_completion(&rqst.completion);
1175 }
1176
1177 WARN_ON(rqst.status != 0 && rqst.status != -ECANCELED &&
1178 rqst.status != -ESHUTDOWN && rqst.status != -EINTR);
1179
1180 return rqst.status == -ECANCELED ? -ETIMEDOUT : rqst.status;
1181}
1182
1183/**
1184 * ssh_rtl_shutdown() - Shut down request transport layer.
1185 * @rtl: The request transport layer.
1186 *
1187 * Shuts down the request transport layer, removing and canceling all queued
1188 * and pending requests. Requests canceled by this operation will be completed
1189 * with %-ESHUTDOWN as status. Receiver and transmitter threads will be
1190 * stopped, the lower-level packet layer will be shutdown.
1191 *
1192 * As a result of this function, the transport layer will be marked as shut
1193 * down. Submission of requests after the transport layer has been shut down
1194 * will fail with %-ESHUTDOWN.
1195 */
1196void ssh_rtl_shutdown(struct ssh_rtl *rtl)
1197{
1198 struct ssh_request *r, *n;
1199 LIST_HEAD(claimed);
1200 int pending;
1201
1202 set_bit(SSH_RTL_SF_SHUTDOWN_BIT, &rtl->state);
1203 /*
1204 * Ensure that the layer gets marked as shut-down before actually
1205 * stopping it. In combination with the check in ssh_rtl_submit(),
1206 * this guarantees that no new requests can be added and all already
1207 * queued requests are properly canceled.
1208 */
1209 smp_mb__after_atomic();
1210
1211 /* Remove requests from queue. */
1212 spin_lock(&rtl->queue.lock);
1213 list_for_each_entry_safe(r, n, &rtl->queue.head, node) {
1214 set_bit(SSH_REQUEST_SF_LOCKED_BIT, &r->state);
1215 /* Ensure state never gets zero. */
1216 smp_mb__before_atomic();
1217 clear_bit(SSH_REQUEST_SF_QUEUED_BIT, &r->state);
1218
1219 list_move_tail(&r->node, &claimed);
1220 }
1221 spin_unlock(&rtl->queue.lock);
1222
1223 /*
1224 * We have now guaranteed that the queue is empty and no more new
1225 * requests can be submitted (i.e. it will stay empty). This means that
1226 * calling ssh_rtl_tx_schedule() will not schedule tx.work any more. So
1227 * we can simply call cancel_work_sync() on tx.work here and when that
1228 * returns, we've locked it down. This also means that after this call,
1229 * we don't submit any more packets to the underlying packet layer, so
1230 * we can also shut that down.
1231 */
1232
1233 cancel_work_sync(&rtl->tx.work);
1234 ssh_ptl_shutdown(&rtl->ptl);
1235 cancel_delayed_work_sync(&rtl->rtx_timeout.reaper);
1236
1237 /*
1238 * Shutting down the packet layer should also have canceled all
1239 * requests. Thus the pending set should be empty. Attempt to handle
1240 * this gracefully anyways, even though this should be dead code.
1241 */
1242
1243 pending = atomic_read(&rtl->pending.count);
1244 if (WARN_ON(pending)) {
1245 spin_lock(&rtl->pending.lock);
1246 list_for_each_entry_safe(r, n, &rtl->pending.head, node) {
1247 set_bit(SSH_REQUEST_SF_LOCKED_BIT, &r->state);
1248 /* Ensure state never gets zero. */
1249 smp_mb__before_atomic();
1250 clear_bit(SSH_REQUEST_SF_PENDING_BIT, &r->state);
1251
1252 list_move_tail(&r->node, &claimed);
1253 }
1254 spin_unlock(&rtl->pending.lock);
1255 }
1256
1257 /* Finally, cancel and complete the requests we claimed before. */
1258 list_for_each_entry_safe(r, n, &claimed, node) {
1259 /*
1260 * We need test_and_set() because we still might compete with
1261 * cancellation.
1262 */
1263 if (!test_and_set_bit(SSH_REQUEST_SF_COMPLETED_BIT, &r->state))
1264 ssh_rtl_complete_with_status(r, -ESHUTDOWN);
1265
1266 /*
1267 * Drop the reference we've obtained by removing it from the
1268 * lists.
1269 */
1270 list_del(&r->node);
1271 ssh_request_put(r);
1272 }
1273}