Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * SSH packet transport layer.
4 *
5 * Copyright (C) 2019-2022 Maximilian Luz <luzmaximilian@gmail.com>
6 */
7
8#include <asm/unaligned.h>
9#include <linux/atomic.h>
10#include <linux/error-injection.h>
11#include <linux/jiffies.h>
12#include <linux/kfifo.h>
13#include <linux/kref.h>
14#include <linux/kthread.h>
15#include <linux/ktime.h>
16#include <linux/limits.h>
17#include <linux/list.h>
18#include <linux/lockdep.h>
19#include <linux/serdev.h>
20#include <linux/slab.h>
21#include <linux/spinlock.h>
22#include <linux/workqueue.h>
23
24#include <linux/surface_aggregator/serial_hub.h>
25
26#include "ssh_msgb.h"
27#include "ssh_packet_layer.h"
28#include "ssh_parser.h"
29
30#include "trace.h"
31
32/*
33 * To simplify reasoning about the code below, we define a few concepts. The
34 * system below is similar to a state-machine for packets, however, there are
35 * too many states to explicitly write them down. To (somewhat) manage the
36 * states and packets we rely on flags, reference counting, and some simple
37 * concepts. State transitions are triggered by actions.
38 *
39 * >> Actions <<
40 *
41 * - submit
42 * - transmission start (process next item in queue)
43 * - transmission finished (guaranteed to never be parallel to transmission
44 * start)
45 * - ACK received
46 * - NAK received (this is equivalent to issuing re-submit for all pending
47 * packets)
48 * - timeout (this is equivalent to re-issuing a submit or canceling)
49 * - cancel (non-pending and pending)
50 *
51 * >> Data Structures, Packet Ownership, General Overview <<
52 *
53 * The code below employs two main data structures: The packet queue,
54 * containing all packets scheduled for transmission, and the set of pending
55 * packets, containing all packets awaiting an ACK.
56 *
57 * Shared ownership of a packet is controlled via reference counting. Inside
58 * the transport system are a total of five packet owners:
59 *
60 * - the packet queue,
61 * - the pending set,
62 * - the transmitter thread,
63 * - the receiver thread (via ACKing), and
64 * - the timeout work item.
65 *
66 * Normal operation is as follows: The initial reference of the packet is
67 * obtained by submitting the packet and queuing it. The receiver thread takes
68 * packets from the queue. By doing this, it does not increment the refcount
69 * but takes over the reference (removing it from the queue). If the packet is
70 * sequenced (i.e. needs to be ACKed by the client), the transmitter thread
71 * sets-up the timeout and adds the packet to the pending set before starting
72 * to transmit it. As the timeout is handled by a reaper task, no additional
73 * reference for it is needed. After the transmit is done, the reference held
74 * by the transmitter thread is dropped. If the packet is unsequenced (i.e.
75 * does not need an ACK), the packet is completed by the transmitter thread
76 * before dropping that reference.
77 *
78 * On receival of an ACK, the receiver thread removes and obtains the
79 * reference to the packet from the pending set. The receiver thread will then
80 * complete the packet and drop its reference.
81 *
82 * On receival of a NAK, the receiver thread re-submits all currently pending
83 * packets.
84 *
85 * Packet timeouts are detected by the timeout reaper. This is a task,
86 * scheduled depending on the earliest packet timeout expiration date,
87 * checking all currently pending packets if their timeout has expired. If the
88 * timeout of a packet has expired, it is re-submitted and the number of tries
89 * of this packet is incremented. If this number reaches its limit, the packet
90 * will be completed with a failure.
91 *
92 * On transmission failure (such as repeated packet timeouts), the completion
93 * callback is immediately run by on thread on which the error was detected.
94 *
95 * To ensure that a packet eventually leaves the system it is marked as
96 * "locked" directly before it is going to be completed or when it is
97 * canceled. Marking a packet as "locked" has the effect that passing and
98 * creating new references of the packet is disallowed. This means that the
99 * packet cannot be added to the queue, the pending set, and the timeout, or
100 * be picked up by the transmitter thread or receiver thread. To remove a
101 * packet from the system it has to be marked as locked and subsequently all
102 * references from the data structures (queue, pending) have to be removed.
103 * References held by threads will eventually be dropped automatically as
104 * their execution progresses.
105 *
106 * Note that the packet completion callback is, in case of success and for a
107 * sequenced packet, guaranteed to run on the receiver thread, thus providing
108 * a way to reliably identify responses to the packet. The packet completion
109 * callback is only run once and it does not indicate that the packet has
110 * fully left the system (for this, one should rely on the release method,
111 * triggered when the reference count of the packet reaches zero). In case of
112 * re-submission (and with somewhat unlikely timing), it may be possible that
113 * the packet is being re-transmitted while the completion callback runs.
114 * Completion will occur both on success and internal error, as well as when
115 * the packet is canceled.
116 *
117 * >> Flags <<
118 *
119 * Flags are used to indicate the state and progression of a packet. Some flags
120 * have stricter guarantees than other:
121 *
122 * - locked
123 * Indicates if the packet is locked. If the packet is locked, passing and/or
124 * creating additional references to the packet is forbidden. The packet thus
125 * may not be queued, dequeued, or removed or added to the pending set. Note
126 * that the packet state flags may still change (e.g. it may be marked as
127 * ACKed, transmitted, ...).
128 *
129 * - completed
130 * Indicates if the packet completion callback has been executed or is about
131 * to be executed. This flag is used to ensure that the packet completion
132 * callback is only run once.
133 *
134 * - queued
135 * Indicates if a packet is present in the submission queue or not. This flag
136 * must only be modified with the queue lock held, and must be coherent to the
137 * presence of the packet in the queue.
138 *
139 * - pending
140 * Indicates if a packet is present in the set of pending packets or not.
141 * This flag must only be modified with the pending lock held, and must be
142 * coherent to the presence of the packet in the pending set.
143 *
144 * - transmitting
145 * Indicates if the packet is currently transmitting. In case of
146 * re-transmissions, it is only safe to wait on the "transmitted" completion
147 * after this flag has been set. The completion will be set both in success
148 * and error case.
149 *
150 * - transmitted
151 * Indicates if the packet has been transmitted. This flag is not cleared by
152 * the system, thus it indicates the first transmission only.
153 *
154 * - acked
155 * Indicates if the packet has been acknowledged by the client. There are no
156 * other guarantees given. For example, the packet may still be canceled
157 * and/or the completion may be triggered an error even though this bit is
158 * set. Rely on the status provided to the completion callback instead.
159 *
160 * - canceled
161 * Indicates if the packet has been canceled from the outside. There are no
162 * other guarantees given. Specifically, the packet may be completed by
163 * another part of the system before the cancellation attempts to complete it.
164 *
165 * >> General Notes <<
166 *
167 * - To avoid deadlocks, if both queue and pending locks are required, the
168 * pending lock must be acquired before the queue lock.
169 *
170 * - The packet priority must be accessed only while holding the queue lock.
171 *
172 * - The packet timestamp must be accessed only while holding the pending
173 * lock.
174 */
175
176/*
177 * SSH_PTL_MAX_PACKET_TRIES - Maximum transmission attempts for packet.
178 *
179 * Maximum number of transmission attempts per sequenced packet in case of
180 * time-outs. Must be smaller than 16. If the packet times out after this
181 * amount of tries, the packet will be completed with %-ETIMEDOUT as status
182 * code.
183 */
184#define SSH_PTL_MAX_PACKET_TRIES 3
185
186/*
187 * SSH_PTL_TX_TIMEOUT - Packet transmission timeout.
188 *
189 * Timeout in jiffies for packet transmission via the underlying serial
190 * device. If transmitting the packet takes longer than this timeout, the
191 * packet will be completed with -ETIMEDOUT. It will not be re-submitted.
192 */
193#define SSH_PTL_TX_TIMEOUT HZ
194
195/*
196 * SSH_PTL_PACKET_TIMEOUT - Packet response timeout.
197 *
198 * Timeout as ktime_t delta for ACKs. If we have not received an ACK in this
199 * time-frame after starting transmission, the packet will be re-submitted.
200 */
201#define SSH_PTL_PACKET_TIMEOUT ms_to_ktime(1000)
202
203/*
204 * SSH_PTL_PACKET_TIMEOUT_RESOLUTION - Packet timeout granularity.
205 *
206 * Time-resolution for timeouts. Should be larger than one jiffy to avoid
207 * direct re-scheduling of reaper work_struct.
208 */
209#define SSH_PTL_PACKET_TIMEOUT_RESOLUTION ms_to_ktime(max(2000 / HZ, 50))
210
211/*
212 * SSH_PTL_MAX_PENDING - Maximum number of pending packets.
213 *
214 * Maximum number of sequenced packets concurrently waiting for an ACK.
215 * Packets marked as blocking will not be transmitted while this limit is
216 * reached.
217 */
218#define SSH_PTL_MAX_PENDING 1
219
220/*
221 * SSH_PTL_RX_BUF_LEN - Evaluation-buffer size in bytes.
222 */
223#define SSH_PTL_RX_BUF_LEN 4096
224
225/*
226 * SSH_PTL_RX_FIFO_LEN - Fifo input-buffer size in bytes.
227 */
228#define SSH_PTL_RX_FIFO_LEN 4096
229
230#ifdef CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION
231
232/**
233 * ssh_ptl_should_drop_ack_packet() - Error injection hook to drop ACK packets.
234 *
235 * Useful to test detection and handling of automated re-transmits by the EC.
236 * Specifically of packets that the EC considers not-ACKed but the driver
237 * already considers ACKed (due to dropped ACK). In this case, the EC
238 * re-transmits the packet-to-be-ACKed and the driver should detect it as
239 * duplicate/already handled. Note that the driver should still send an ACK
240 * for the re-transmitted packet.
241 */
242static noinline bool ssh_ptl_should_drop_ack_packet(void)
243{
244 return false;
245}
246ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_ack_packet, TRUE);
247
248/**
249 * ssh_ptl_should_drop_nak_packet() - Error injection hook to drop NAK packets.
250 *
251 * Useful to test/force automated (timeout-based) re-transmit by the EC.
252 * Specifically, packets that have not reached the driver completely/with valid
253 * checksums. Only useful in combination with receival of (injected) bad data.
254 */
255static noinline bool ssh_ptl_should_drop_nak_packet(void)
256{
257 return false;
258}
259ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_nak_packet, TRUE);
260
261/**
262 * ssh_ptl_should_drop_dsq_packet() - Error injection hook to drop sequenced
263 * data packet.
264 *
265 * Useful to test re-transmit timeout of the driver. If the data packet has not
266 * been ACKed after a certain time, the driver should re-transmit the packet up
267 * to limited number of times defined in SSH_PTL_MAX_PACKET_TRIES.
268 */
269static noinline bool ssh_ptl_should_drop_dsq_packet(void)
270{
271 return false;
272}
273ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_dsq_packet, TRUE);
274
275/**
276 * ssh_ptl_should_fail_write() - Error injection hook to make
277 * serdev_device_write() fail.
278 *
279 * Hook to simulate errors in serdev_device_write when transmitting packets.
280 */
281static noinline int ssh_ptl_should_fail_write(void)
282{
283 return 0;
284}
285ALLOW_ERROR_INJECTION(ssh_ptl_should_fail_write, ERRNO);
286
287/**
288 * ssh_ptl_should_corrupt_tx_data() - Error injection hook to simulate invalid
289 * data being sent to the EC.
290 *
291 * Hook to simulate corrupt/invalid data being sent from host (driver) to EC.
292 * Causes the packet data to be actively corrupted by overwriting it with
293 * pre-defined values, such that it becomes invalid, causing the EC to respond
294 * with a NAK packet. Useful to test handling of NAK packets received by the
295 * driver.
296 */
297static noinline bool ssh_ptl_should_corrupt_tx_data(void)
298{
299 return false;
300}
301ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_tx_data, TRUE);
302
303/**
304 * ssh_ptl_should_corrupt_rx_syn() - Error injection hook to simulate invalid
305 * data being sent by the EC.
306 *
307 * Hook to simulate invalid SYN bytes, i.e. an invalid start of messages and
308 * test handling thereof in the driver.
309 */
310static noinline bool ssh_ptl_should_corrupt_rx_syn(void)
311{
312 return false;
313}
314ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_rx_syn, TRUE);
315
316/**
317 * ssh_ptl_should_corrupt_rx_data() - Error injection hook to simulate invalid
318 * data being sent by the EC.
319 *
320 * Hook to simulate invalid data/checksum of the message frame and test handling
321 * thereof in the driver.
322 */
323static noinline bool ssh_ptl_should_corrupt_rx_data(void)
324{
325 return false;
326}
327ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_rx_data, TRUE);
328
329static bool __ssh_ptl_should_drop_ack_packet(struct ssh_packet *packet)
330{
331 if (likely(!ssh_ptl_should_drop_ack_packet()))
332 return false;
333
334 trace_ssam_ei_tx_drop_ack_packet(packet);
335 ptl_info(packet->ptl, "packet error injection: dropping ACK packet %p\n",
336 packet);
337
338 return true;
339}
340
341static bool __ssh_ptl_should_drop_nak_packet(struct ssh_packet *packet)
342{
343 if (likely(!ssh_ptl_should_drop_nak_packet()))
344 return false;
345
346 trace_ssam_ei_tx_drop_nak_packet(packet);
347 ptl_info(packet->ptl, "packet error injection: dropping NAK packet %p\n",
348 packet);
349
350 return true;
351}
352
353static bool __ssh_ptl_should_drop_dsq_packet(struct ssh_packet *packet)
354{
355 if (likely(!ssh_ptl_should_drop_dsq_packet()))
356 return false;
357
358 trace_ssam_ei_tx_drop_dsq_packet(packet);
359 ptl_info(packet->ptl,
360 "packet error injection: dropping sequenced data packet %p\n",
361 packet);
362
363 return true;
364}
365
366static bool ssh_ptl_should_drop_packet(struct ssh_packet *packet)
367{
368 /* Ignore packets that don't carry any data (i.e. flush). */
369 if (!packet->data.ptr || !packet->data.len)
370 return false;
371
372 switch (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)]) {
373 case SSH_FRAME_TYPE_ACK:
374 return __ssh_ptl_should_drop_ack_packet(packet);
375
376 case SSH_FRAME_TYPE_NAK:
377 return __ssh_ptl_should_drop_nak_packet(packet);
378
379 case SSH_FRAME_TYPE_DATA_SEQ:
380 return __ssh_ptl_should_drop_dsq_packet(packet);
381
382 default:
383 return false;
384 }
385}
386
387static int ssh_ptl_write_buf(struct ssh_ptl *ptl, struct ssh_packet *packet,
388 const unsigned char *buf, size_t count)
389{
390 int status;
391
392 status = ssh_ptl_should_fail_write();
393 if (unlikely(status)) {
394 trace_ssam_ei_tx_fail_write(packet, status);
395 ptl_info(packet->ptl,
396 "packet error injection: simulating transmit error %d, packet %p\n",
397 status, packet);
398
399 return status;
400 }
401
402 return serdev_device_write_buf(ptl->serdev, buf, count);
403}
404
405static void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet)
406{
407 /* Ignore packets that don't carry any data (i.e. flush). */
408 if (!packet->data.ptr || !packet->data.len)
409 return;
410
411 /* Only allow sequenced data packets to be modified. */
412 if (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)] != SSH_FRAME_TYPE_DATA_SEQ)
413 return;
414
415 if (likely(!ssh_ptl_should_corrupt_tx_data()))
416 return;
417
418 trace_ssam_ei_tx_corrupt_data(packet);
419 ptl_info(packet->ptl,
420 "packet error injection: simulating invalid transmit data on packet %p\n",
421 packet);
422
423 /*
424 * NB: The value 0xb3 has been chosen more or less randomly so that it
425 * doesn't have any (major) overlap with the SYN bytes (aa 55) and is
426 * non-trivial (i.e. non-zero, non-0xff).
427 */
428 memset(packet->data.ptr, 0xb3, packet->data.len);
429}
430
431static void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl,
432 struct ssam_span *data)
433{
434 struct ssam_span frame;
435
436 /* Check if there actually is something to corrupt. */
437 if (!sshp_find_syn(data, &frame))
438 return;
439
440 if (likely(!ssh_ptl_should_corrupt_rx_syn()))
441 return;
442
443 trace_ssam_ei_rx_corrupt_syn(data->len);
444
445 data->ptr[1] = 0xb3; /* Set second byte of SYN to "random" value. */
446}
447
448static void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl,
449 struct ssam_span *frame)
450{
451 size_t payload_len, message_len;
452 struct ssh_frame *sshf;
453
454 /* Ignore incomplete messages, will get handled once it's complete. */
455 if (frame->len < SSH_MESSAGE_LENGTH(0))
456 return;
457
458 /* Ignore incomplete messages, part 2. */
459 payload_len = get_unaligned_le16(&frame->ptr[SSH_MSGOFFSET_FRAME(len)]);
460 message_len = SSH_MESSAGE_LENGTH(payload_len);
461 if (frame->len < message_len)
462 return;
463
464 if (likely(!ssh_ptl_should_corrupt_rx_data()))
465 return;
466
467 sshf = (struct ssh_frame *)&frame->ptr[SSH_MSGOFFSET_FRAME(type)];
468 trace_ssam_ei_rx_corrupt_data(sshf);
469
470 /*
471 * Flip bits in first byte of payload checksum. This is basically
472 * equivalent to a payload/frame data error without us having to worry
473 * about (the, arguably pretty small, probability of) accidental
474 * checksum collisions.
475 */
476 frame->ptr[frame->len - 2] = ~frame->ptr[frame->len - 2];
477}
478
479#else /* CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION */
480
481static inline bool ssh_ptl_should_drop_packet(struct ssh_packet *packet)
482{
483 return false;
484}
485
486static inline int ssh_ptl_write_buf(struct ssh_ptl *ptl,
487 struct ssh_packet *packet,
488 const unsigned char *buf,
489 size_t count)
490{
491 return serdev_device_write_buf(ptl->serdev, buf, count);
492}
493
494static inline void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet)
495{
496}
497
498static inline void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl,
499 struct ssam_span *data)
500{
501}
502
503static inline void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl,
504 struct ssam_span *frame)
505{
506}
507
508#endif /* CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION */
509
510static void __ssh_ptl_packet_release(struct kref *kref)
511{
512 struct ssh_packet *p = container_of(kref, struct ssh_packet, refcnt);
513
514 trace_ssam_packet_release(p);
515
516 ptl_dbg_cond(p->ptl, "ptl: releasing packet %p\n", p);
517 p->ops->release(p);
518}
519
520/**
521 * ssh_packet_get() - Increment reference count of packet.
522 * @packet: The packet to increment the reference count of.
523 *
524 * Increments the reference count of the given packet. See ssh_packet_put()
525 * for the counter-part of this function.
526 *
527 * Return: Returns the packet provided as input.
528 */
529struct ssh_packet *ssh_packet_get(struct ssh_packet *packet)
530{
531 if (packet)
532 kref_get(&packet->refcnt);
533 return packet;
534}
535EXPORT_SYMBOL_GPL(ssh_packet_get);
536
537/**
538 * ssh_packet_put() - Decrement reference count of packet.
539 * @packet: The packet to decrement the reference count of.
540 *
541 * If the reference count reaches zero, the ``release`` callback specified in
542 * the packet's &struct ssh_packet_ops, i.e. ``packet->ops->release``, will be
543 * called.
544 *
545 * See ssh_packet_get() for the counter-part of this function.
546 */
547void ssh_packet_put(struct ssh_packet *packet)
548{
549 if (packet)
550 kref_put(&packet->refcnt, __ssh_ptl_packet_release);
551}
552EXPORT_SYMBOL_GPL(ssh_packet_put);
553
554static u8 ssh_packet_get_seq(struct ssh_packet *packet)
555{
556 return packet->data.ptr[SSH_MSGOFFSET_FRAME(seq)];
557}
558
559/**
560 * ssh_packet_init() - Initialize SSH packet.
561 * @packet: The packet to initialize.
562 * @type: Type-flags of the packet.
563 * @priority: Priority of the packet. See SSH_PACKET_PRIORITY() for details.
564 * @ops: Packet operations.
565 *
566 * Initializes the given SSH packet. Sets the transmission buffer pointer to
567 * %NULL and the transmission buffer length to zero. For data-type packets,
568 * this buffer has to be set separately via ssh_packet_set_data() before
569 * submission, and must contain a valid SSH message, i.e. frame with optional
570 * payload of any type.
571 */
572void ssh_packet_init(struct ssh_packet *packet, unsigned long type,
573 u8 priority, const struct ssh_packet_ops *ops)
574{
575 kref_init(&packet->refcnt);
576
577 packet->ptl = NULL;
578 INIT_LIST_HEAD(&packet->queue_node);
579 INIT_LIST_HEAD(&packet->pending_node);
580
581 packet->state = type & SSH_PACKET_FLAGS_TY_MASK;
582 packet->priority = priority;
583 packet->timestamp = KTIME_MAX;
584
585 packet->data.ptr = NULL;
586 packet->data.len = 0;
587
588 packet->ops = ops;
589}
590
591static struct kmem_cache *ssh_ctrl_packet_cache;
592
593/**
594 * ssh_ctrl_packet_cache_init() - Initialize the control packet cache.
595 */
596int ssh_ctrl_packet_cache_init(void)
597{
598 const unsigned int size = sizeof(struct ssh_packet) + SSH_MSG_LEN_CTRL;
599 const unsigned int align = __alignof__(struct ssh_packet);
600 struct kmem_cache *cache;
601
602 cache = kmem_cache_create("ssam_ctrl_packet", size, align, 0, NULL);
603 if (!cache)
604 return -ENOMEM;
605
606 ssh_ctrl_packet_cache = cache;
607 return 0;
608}
609
610/**
611 * ssh_ctrl_packet_cache_destroy() - Deinitialize the control packet cache.
612 */
613void ssh_ctrl_packet_cache_destroy(void)
614{
615 kmem_cache_destroy(ssh_ctrl_packet_cache);
616 ssh_ctrl_packet_cache = NULL;
617}
618
619/**
620 * ssh_ctrl_packet_alloc() - Allocate packet from control packet cache.
621 * @packet: Where the pointer to the newly allocated packet should be stored.
622 * @buffer: The buffer corresponding to this packet.
623 * @flags: Flags used for allocation.
624 *
625 * Allocates a packet and corresponding transport buffer from the control
626 * packet cache. Sets the packet's buffer reference to the allocated buffer.
627 * The packet must be freed via ssh_ctrl_packet_free(), which will also free
628 * the corresponding buffer. The corresponding buffer must not be freed
629 * separately. Intended to be used with %ssh_ptl_ctrl_packet_ops as packet
630 * operations.
631 *
632 * Return: Returns zero on success, %-ENOMEM if the allocation failed.
633 */
634static int ssh_ctrl_packet_alloc(struct ssh_packet **packet,
635 struct ssam_span *buffer, gfp_t flags)
636{
637 *packet = kmem_cache_alloc(ssh_ctrl_packet_cache, flags);
638 if (!*packet)
639 return -ENOMEM;
640
641 buffer->ptr = (u8 *)(*packet + 1);
642 buffer->len = SSH_MSG_LEN_CTRL;
643
644 trace_ssam_ctrl_packet_alloc(*packet, buffer->len);
645 return 0;
646}
647
648/**
649 * ssh_ctrl_packet_free() - Free packet allocated from control packet cache.
650 * @p: The packet to free.
651 */
652static void ssh_ctrl_packet_free(struct ssh_packet *p)
653{
654 trace_ssam_ctrl_packet_free(p);
655 kmem_cache_free(ssh_ctrl_packet_cache, p);
656}
657
658static const struct ssh_packet_ops ssh_ptl_ctrl_packet_ops = {
659 .complete = NULL,
660 .release = ssh_ctrl_packet_free,
661};
662
663static void ssh_ptl_timeout_reaper_mod(struct ssh_ptl *ptl, ktime_t now,
664 ktime_t expires)
665{
666 unsigned long delta = msecs_to_jiffies(ktime_ms_delta(expires, now));
667 ktime_t aexp = ktime_add(expires, SSH_PTL_PACKET_TIMEOUT_RESOLUTION);
668
669 spin_lock(&ptl->rtx_timeout.lock);
670
671 /* Re-adjust / schedule reaper only if it is above resolution delta. */
672 if (ktime_before(aexp, ptl->rtx_timeout.expires)) {
673 ptl->rtx_timeout.expires = expires;
674 mod_delayed_work(system_wq, &ptl->rtx_timeout.reaper, delta);
675 }
676
677 spin_unlock(&ptl->rtx_timeout.lock);
678}
679
680/* Must be called with queue lock held. */
681static void ssh_packet_next_try(struct ssh_packet *p)
682{
683 u8 base = ssh_packet_priority_get_base(p->priority);
684 u8 try = ssh_packet_priority_get_try(p->priority);
685
686 lockdep_assert_held(&p->ptl->queue.lock);
687
688 /*
689 * Ensure that we write the priority in one go via WRITE_ONCE() so we
690 * can access it via READ_ONCE() for tracing. Note that other access
691 * is guarded by the queue lock, so no need to use READ_ONCE() there.
692 */
693 WRITE_ONCE(p->priority, __SSH_PACKET_PRIORITY(base, try + 1));
694}
695
696/* Must be called with queue lock held. */
697static struct list_head *__ssh_ptl_queue_find_entrypoint(struct ssh_packet *p)
698{
699 struct list_head *head;
700 struct ssh_packet *q;
701
702 lockdep_assert_held(&p->ptl->queue.lock);
703
704 /*
705 * We generally assume that there are less control (ACK/NAK) packets
706 * and re-submitted data packets as there are normal data packets (at
707 * least in situations in which many packets are queued; if there
708 * aren't many packets queued the decision on how to iterate should be
709 * basically irrelevant; the number of control/data packets is more or
710 * less limited via the maximum number of pending packets). Thus, when
711 * inserting a control or re-submitted data packet, (determined by
712 * their priority), we search from front to back. Normal data packets
713 * are, usually queued directly at the tail of the queue, so for those
714 * search from back to front.
715 */
716
717 if (p->priority > SSH_PACKET_PRIORITY(DATA, 0)) {
718 list_for_each(head, &p->ptl->queue.head) {
719 q = list_entry(head, struct ssh_packet, queue_node);
720
721 if (q->priority < p->priority)
722 break;
723 }
724 } else {
725 list_for_each_prev(head, &p->ptl->queue.head) {
726 q = list_entry(head, struct ssh_packet, queue_node);
727
728 if (q->priority >= p->priority) {
729 head = head->next;
730 break;
731 }
732 }
733 }
734
735 return head;
736}
737
738/* Must be called with queue lock held. */
739static int __ssh_ptl_queue_push(struct ssh_packet *packet)
740{
741 struct ssh_ptl *ptl = packet->ptl;
742 struct list_head *head;
743
744 lockdep_assert_held(&ptl->queue.lock);
745
746 if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))
747 return -ESHUTDOWN;
748
749 /* Avoid further transitions when canceling/completing. */
750 if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state))
751 return -EINVAL;
752
753 /* If this packet has already been queued, do not add it. */
754 if (test_and_set_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state))
755 return -EALREADY;
756
757 head = __ssh_ptl_queue_find_entrypoint(packet);
758
759 list_add_tail(&ssh_packet_get(packet)->queue_node, head);
760 return 0;
761}
762
763static int ssh_ptl_queue_push(struct ssh_packet *packet)
764{
765 int status;
766
767 spin_lock(&packet->ptl->queue.lock);
768 status = __ssh_ptl_queue_push(packet);
769 spin_unlock(&packet->ptl->queue.lock);
770
771 return status;
772}
773
774static void ssh_ptl_queue_remove(struct ssh_packet *packet)
775{
776 struct ssh_ptl *ptl = packet->ptl;
777
778 spin_lock(&ptl->queue.lock);
779
780 if (!test_and_clear_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state)) {
781 spin_unlock(&ptl->queue.lock);
782 return;
783 }
784
785 list_del(&packet->queue_node);
786
787 spin_unlock(&ptl->queue.lock);
788 ssh_packet_put(packet);
789}
790
791static void ssh_ptl_pending_push(struct ssh_packet *p)
792{
793 struct ssh_ptl *ptl = p->ptl;
794 const ktime_t timestamp = ktime_get_coarse_boottime();
795 const ktime_t timeout = ptl->rtx_timeout.timeout;
796
797 /*
798 * Note: We can get the time for the timestamp before acquiring the
799 * lock as this is the only place we're setting it and this function
800 * is called only from the transmitter thread. Thus it is not possible
801 * to overwrite the timestamp with an outdated value below.
802 */
803
804 spin_lock(&ptl->pending.lock);
805
806 /* If we are canceling/completing this packet, do not add it. */
807 if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state)) {
808 spin_unlock(&ptl->pending.lock);
809 return;
810 }
811
812 /*
813 * On re-submission, the packet has already been added the pending
814 * set. We still need to update the timestamp as the packet timeout is
815 * reset for each (re-)submission.
816 */
817 p->timestamp = timestamp;
818
819 /* In case it is already pending (e.g. re-submission), do not add it. */
820 if (!test_and_set_bit(SSH_PACKET_SF_PENDING_BIT, &p->state)) {
821 atomic_inc(&ptl->pending.count);
822 list_add_tail(&ssh_packet_get(p)->pending_node, &ptl->pending.head);
823 }
824
825 spin_unlock(&ptl->pending.lock);
826
827 /* Arm/update timeout reaper. */
828 ssh_ptl_timeout_reaper_mod(ptl, timestamp, timestamp + timeout);
829}
830
831static void ssh_ptl_pending_remove(struct ssh_packet *packet)
832{
833 struct ssh_ptl *ptl = packet->ptl;
834
835 spin_lock(&ptl->pending.lock);
836
837 if (!test_and_clear_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state)) {
838 spin_unlock(&ptl->pending.lock);
839 return;
840 }
841
842 list_del(&packet->pending_node);
843 atomic_dec(&ptl->pending.count);
844
845 spin_unlock(&ptl->pending.lock);
846
847 ssh_packet_put(packet);
848}
849
850/* Warning: Does not check/set "completed" bit. */
851static void __ssh_ptl_complete(struct ssh_packet *p, int status)
852{
853 struct ssh_ptl *ptl = READ_ONCE(p->ptl);
854
855 trace_ssam_packet_complete(p, status);
856 ptl_dbg_cond(ptl, "ptl: completing packet %p (status: %d)\n", p, status);
857
858 if (p->ops->complete)
859 p->ops->complete(p, status);
860}
861
862static void ssh_ptl_remove_and_complete(struct ssh_packet *p, int status)
863{
864 /*
865 * A call to this function should in general be preceded by
866 * set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->flags) to avoid re-adding the
867 * packet to the structures it's going to be removed from.
868 *
869 * The set_bit call does not need explicit memory barriers as the
870 * implicit barrier of the test_and_set_bit() call below ensure that the
871 * flag is visible before we actually attempt to remove the packet.
872 */
873
874 if (test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state))
875 return;
876
877 ssh_ptl_queue_remove(p);
878 ssh_ptl_pending_remove(p);
879
880 __ssh_ptl_complete(p, status);
881}
882
883static bool ssh_ptl_tx_can_process(struct ssh_packet *packet)
884{
885 struct ssh_ptl *ptl = packet->ptl;
886
887 if (test_bit(SSH_PACKET_TY_FLUSH_BIT, &packet->state))
888 return !atomic_read(&ptl->pending.count);
889
890 /* We can always process non-blocking packets. */
891 if (!test_bit(SSH_PACKET_TY_BLOCKING_BIT, &packet->state))
892 return true;
893
894 /* If we are already waiting for this packet, send it again. */
895 if (test_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state))
896 return true;
897
898 /* Otherwise: Check if we have the capacity to send. */
899 return atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING;
900}
901
902static struct ssh_packet *ssh_ptl_tx_pop(struct ssh_ptl *ptl)
903{
904 struct ssh_packet *packet = ERR_PTR(-ENOENT);
905 struct ssh_packet *p, *n;
906
907 spin_lock(&ptl->queue.lock);
908 list_for_each_entry_safe(p, n, &ptl->queue.head, queue_node) {
909 /*
910 * If we are canceling or completing this packet, ignore it.
911 * It's going to be removed from this queue shortly.
912 */
913 if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))
914 continue;
915
916 /*
917 * Packets should be ordered non-blocking/to-be-resent first.
918 * If we cannot process this packet, assume that we can't
919 * process any following packet either and abort.
920 */
921 if (!ssh_ptl_tx_can_process(p)) {
922 packet = ERR_PTR(-EBUSY);
923 break;
924 }
925
926 /*
927 * We are allowed to change the state now. Remove it from the
928 * queue and mark it as being transmitted.
929 */
930
931 list_del(&p->queue_node);
932
933 set_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &p->state);
934 /* Ensure that state never gets zero. */
935 smp_mb__before_atomic();
936 clear_bit(SSH_PACKET_SF_QUEUED_BIT, &p->state);
937
938 /*
939 * Update number of tries. This directly influences the
940 * priority in case the packet is re-submitted (e.g. via
941 * timeout/NAK). Note that all reads and writes to the
942 * priority after the first submission are guarded by the
943 * queue lock.
944 */
945 ssh_packet_next_try(p);
946
947 packet = p;
948 break;
949 }
950 spin_unlock(&ptl->queue.lock);
951
952 return packet;
953}
954
955static struct ssh_packet *ssh_ptl_tx_next(struct ssh_ptl *ptl)
956{
957 struct ssh_packet *p;
958
959 p = ssh_ptl_tx_pop(ptl);
960 if (IS_ERR(p))
961 return p;
962
963 if (test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &p->state)) {
964 ptl_dbg(ptl, "ptl: transmitting sequenced packet %p\n", p);
965 ssh_ptl_pending_push(p);
966 } else {
967 ptl_dbg(ptl, "ptl: transmitting non-sequenced packet %p\n", p);
968 }
969
970 return p;
971}
972
973static void ssh_ptl_tx_compl_success(struct ssh_packet *packet)
974{
975 struct ssh_ptl *ptl = packet->ptl;
976
977 ptl_dbg(ptl, "ptl: successfully transmitted packet %p\n", packet);
978
979 /* Transition state to "transmitted". */
980 set_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &packet->state);
981 /* Ensure that state never gets zero. */
982 smp_mb__before_atomic();
983 clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state);
984
985 /* If the packet is unsequenced, we're done: Lock and complete. */
986 if (!test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &packet->state)) {
987 set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state);
988 ssh_ptl_remove_and_complete(packet, 0);
989 }
990
991 /*
992 * Notify that a packet transmission has finished. In general we're only
993 * waiting for one packet (if any), so wake_up_all should be fine.
994 */
995 wake_up_all(&ptl->tx.packet_wq);
996}
997
998static void ssh_ptl_tx_compl_error(struct ssh_packet *packet, int status)
999{
1000 /* Transmission failure: Lock the packet and try to complete it. */
1001 set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state);
1002 /* Ensure that state never gets zero. */
1003 smp_mb__before_atomic();
1004 clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state);
1005
1006 ptl_err(packet->ptl, "ptl: transmission error: %d\n", status);
1007 ptl_dbg(packet->ptl, "ptl: failed to transmit packet: %p\n", packet);
1008
1009 ssh_ptl_remove_and_complete(packet, status);
1010
1011 /*
1012 * Notify that a packet transmission has finished. In general we're only
1013 * waiting for one packet (if any), so wake_up_all should be fine.
1014 */
1015 wake_up_all(&packet->ptl->tx.packet_wq);
1016}
1017
1018static long ssh_ptl_tx_wait_packet(struct ssh_ptl *ptl)
1019{
1020 int status;
1021
1022 status = wait_for_completion_interruptible(&ptl->tx.thread_cplt_pkt);
1023 reinit_completion(&ptl->tx.thread_cplt_pkt);
1024
1025 /*
1026 * Ensure completion is cleared before continuing to avoid lost update
1027 * problems.
1028 */
1029 smp_mb__after_atomic();
1030
1031 return status;
1032}
1033
1034static long ssh_ptl_tx_wait_transfer(struct ssh_ptl *ptl, long timeout)
1035{
1036 long status;
1037
1038 status = wait_for_completion_interruptible_timeout(&ptl->tx.thread_cplt_tx,
1039 timeout);
1040 reinit_completion(&ptl->tx.thread_cplt_tx);
1041
1042 /*
1043 * Ensure completion is cleared before continuing to avoid lost update
1044 * problems.
1045 */
1046 smp_mb__after_atomic();
1047
1048 return status;
1049}
1050
1051static int ssh_ptl_tx_packet(struct ssh_ptl *ptl, struct ssh_packet *packet)
1052{
1053 long timeout = SSH_PTL_TX_TIMEOUT;
1054 size_t offset = 0;
1055
1056 /* Note: Flush-packets don't have any data. */
1057 if (unlikely(!packet->data.ptr))
1058 return 0;
1059
1060 /* Error injection: drop packet to simulate transmission problem. */
1061 if (ssh_ptl_should_drop_packet(packet))
1062 return 0;
1063
1064 /* Error injection: simulate invalid packet data. */
1065 ssh_ptl_tx_inject_invalid_data(packet);
1066
1067 ptl_dbg(ptl, "tx: sending data (length: %zu)\n", packet->data.len);
1068 print_hex_dump_debug("tx: ", DUMP_PREFIX_OFFSET, 16, 1,
1069 packet->data.ptr, packet->data.len, false);
1070
1071 do {
1072 ssize_t status, len;
1073 u8 *buf;
1074
1075 buf = packet->data.ptr + offset;
1076 len = packet->data.len - offset;
1077
1078 status = ssh_ptl_write_buf(ptl, packet, buf, len);
1079 if (status < 0)
1080 return status;
1081
1082 if (status == len)
1083 return 0;
1084
1085 offset += status;
1086
1087 timeout = ssh_ptl_tx_wait_transfer(ptl, timeout);
1088 if (kthread_should_stop() || !atomic_read(&ptl->tx.running))
1089 return -ESHUTDOWN;
1090
1091 if (timeout < 0)
1092 return -EINTR;
1093
1094 if (timeout == 0)
1095 return -ETIMEDOUT;
1096 } while (true);
1097}
1098
1099static int ssh_ptl_tx_threadfn(void *data)
1100{
1101 struct ssh_ptl *ptl = data;
1102
1103 while (!kthread_should_stop() && atomic_read(&ptl->tx.running)) {
1104 struct ssh_packet *packet;
1105 int status;
1106
1107 /* Try to get the next packet. */
1108 packet = ssh_ptl_tx_next(ptl);
1109
1110 /* If no packet can be processed, we are done. */
1111 if (IS_ERR(packet)) {
1112 ssh_ptl_tx_wait_packet(ptl);
1113 continue;
1114 }
1115
1116 /* Transfer and complete packet. */
1117 status = ssh_ptl_tx_packet(ptl, packet);
1118 if (status)
1119 ssh_ptl_tx_compl_error(packet, status);
1120 else
1121 ssh_ptl_tx_compl_success(packet);
1122
1123 ssh_packet_put(packet);
1124 }
1125
1126 return 0;
1127}
1128
1129/**
1130 * ssh_ptl_tx_wakeup_packet() - Wake up packet transmitter thread for new
1131 * packet.
1132 * @ptl: The packet transport layer.
1133 *
1134 * Wakes up the packet transmitter thread, notifying it that a new packet has
1135 * arrived and is ready for transfer. If the packet transport layer has been
1136 * shut down, calls to this function will be ignored.
1137 */
1138static void ssh_ptl_tx_wakeup_packet(struct ssh_ptl *ptl)
1139{
1140 if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))
1141 return;
1142
1143 complete(&ptl->tx.thread_cplt_pkt);
1144}
1145
1146/**
1147 * ssh_ptl_tx_start() - Start packet transmitter thread.
1148 * @ptl: The packet transport layer.
1149 *
1150 * Return: Returns zero on success, a negative error code on failure.
1151 */
1152int ssh_ptl_tx_start(struct ssh_ptl *ptl)
1153{
1154 atomic_set_release(&ptl->tx.running, 1);
1155
1156 ptl->tx.thread = kthread_run(ssh_ptl_tx_threadfn, ptl, "ssam_serial_hub-tx");
1157 if (IS_ERR(ptl->tx.thread))
1158 return PTR_ERR(ptl->tx.thread);
1159
1160 return 0;
1161}
1162
1163/**
1164 * ssh_ptl_tx_stop() - Stop packet transmitter thread.
1165 * @ptl: The packet transport layer.
1166 *
1167 * Return: Returns zero on success, a negative error code on failure.
1168 */
1169int ssh_ptl_tx_stop(struct ssh_ptl *ptl)
1170{
1171 int status = 0;
1172
1173 if (!IS_ERR_OR_NULL(ptl->tx.thread)) {
1174 /* Tell thread to stop. */
1175 atomic_set_release(&ptl->tx.running, 0);
1176
1177 /*
1178 * Wake up thread in case it is paused. Do not use wakeup
1179 * helpers as this may be called when the shutdown bit has
1180 * already been set.
1181 */
1182 complete(&ptl->tx.thread_cplt_pkt);
1183 complete(&ptl->tx.thread_cplt_tx);
1184
1185 /* Finally, wait for thread to stop. */
1186 status = kthread_stop(ptl->tx.thread);
1187 ptl->tx.thread = NULL;
1188 }
1189
1190 return status;
1191}
1192
1193static struct ssh_packet *ssh_ptl_ack_pop(struct ssh_ptl *ptl, u8 seq_id)
1194{
1195 struct ssh_packet *packet = ERR_PTR(-ENOENT);
1196 struct ssh_packet *p, *n;
1197
1198 spin_lock(&ptl->pending.lock);
1199 list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {
1200 /*
1201 * We generally expect packets to be in order, so first packet
1202 * to be added to pending is first to be sent, is first to be
1203 * ACKed.
1204 */
1205 if (unlikely(ssh_packet_get_seq(p) != seq_id))
1206 continue;
1207
1208 /*
1209 * In case we receive an ACK while handling a transmission
1210 * error completion. The packet will be removed shortly.
1211 */
1212 if (unlikely(test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))) {
1213 packet = ERR_PTR(-EPERM);
1214 break;
1215 }
1216
1217 /*
1218 * Mark the packet as ACKed and remove it from pending by
1219 * removing its node and decrementing the pending counter.
1220 */
1221 set_bit(SSH_PACKET_SF_ACKED_BIT, &p->state);
1222 /* Ensure that state never gets zero. */
1223 smp_mb__before_atomic();
1224 clear_bit(SSH_PACKET_SF_PENDING_BIT, &p->state);
1225
1226 atomic_dec(&ptl->pending.count);
1227 list_del(&p->pending_node);
1228 packet = p;
1229
1230 break;
1231 }
1232 spin_unlock(&ptl->pending.lock);
1233
1234 return packet;
1235}
1236
1237static void ssh_ptl_wait_until_transmitted(struct ssh_packet *packet)
1238{
1239 wait_event(packet->ptl->tx.packet_wq,
1240 test_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &packet->state) ||
1241 test_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state));
1242}
1243
1244static void ssh_ptl_acknowledge(struct ssh_ptl *ptl, u8 seq)
1245{
1246 struct ssh_packet *p;
1247
1248 p = ssh_ptl_ack_pop(ptl, seq);
1249 if (IS_ERR(p)) {
1250 if (PTR_ERR(p) == -ENOENT) {
1251 /*
1252 * The packet has not been found in the set of pending
1253 * packets.
1254 */
1255 ptl_warn(ptl, "ptl: received ACK for non-pending packet\n");
1256 } else {
1257 /*
1258 * The packet is pending, but we are not allowed to take
1259 * it because it has been locked.
1260 */
1261 WARN_ON(PTR_ERR(p) != -EPERM);
1262 }
1263 return;
1264 }
1265
1266 ptl_dbg(ptl, "ptl: received ACK for packet %p\n", p);
1267
1268 /*
1269 * It is possible that the packet has been transmitted, but the state
1270 * has not been updated from "transmitting" to "transmitted" yet.
1271 * In that case, we need to wait for this transition to occur in order
1272 * to determine between success or failure.
1273 *
1274 * On transmission failure, the packet will be locked after this call.
1275 * On success, the transmitted bit will be set.
1276 */
1277 ssh_ptl_wait_until_transmitted(p);
1278
1279 /*
1280 * The packet will already be locked in case of a transmission error or
1281 * cancellation. Let the transmitter or cancellation issuer complete the
1282 * packet.
1283 */
1284 if (unlikely(test_and_set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))) {
1285 if (unlikely(!test_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &p->state)))
1286 ptl_err(ptl, "ptl: received ACK before packet had been fully transmitted\n");
1287
1288 ssh_packet_put(p);
1289 return;
1290 }
1291
1292 ssh_ptl_remove_and_complete(p, 0);
1293 ssh_packet_put(p);
1294
1295 if (atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING)
1296 ssh_ptl_tx_wakeup_packet(ptl);
1297}
1298
1299/**
1300 * ssh_ptl_submit() - Submit a packet to the transport layer.
1301 * @ptl: The packet transport layer to submit the packet to.
1302 * @p: The packet to submit.
1303 *
1304 * Submits a new packet to the transport layer, queuing it to be sent. This
1305 * function should not be used for re-submission.
1306 *
1307 * Return: Returns zero on success, %-EINVAL if a packet field is invalid or
1308 * the packet has been canceled prior to submission, %-EALREADY if the packet
1309 * has already been submitted, or %-ESHUTDOWN if the packet transport layer
1310 * has been shut down.
1311 */
1312int ssh_ptl_submit(struct ssh_ptl *ptl, struct ssh_packet *p)
1313{
1314 struct ssh_ptl *ptl_old;
1315 int status;
1316
1317 trace_ssam_packet_submit(p);
1318
1319 /* Validate packet fields. */
1320 if (test_bit(SSH_PACKET_TY_FLUSH_BIT, &p->state)) {
1321 if (p->data.ptr || test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &p->state))
1322 return -EINVAL;
1323 } else if (!p->data.ptr) {
1324 return -EINVAL;
1325 }
1326
1327 /*
1328 * The ptl reference only gets set on or before the first submission.
1329 * After the first submission, it has to be read-only.
1330 *
1331 * Note that ptl may already be set from upper-layer request
1332 * submission, thus we cannot expect it to be NULL.
1333 */
1334 ptl_old = READ_ONCE(p->ptl);
1335 if (!ptl_old)
1336 WRITE_ONCE(p->ptl, ptl);
1337 else if (WARN_ON(ptl_old != ptl))
1338 return -EALREADY; /* Submitted on different PTL. */
1339
1340 status = ssh_ptl_queue_push(p);
1341 if (status)
1342 return status;
1343
1344 if (!test_bit(SSH_PACKET_TY_BLOCKING_BIT, &p->state) ||
1345 (atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING))
1346 ssh_ptl_tx_wakeup_packet(ptl);
1347
1348 return 0;
1349}
1350
1351/*
1352 * __ssh_ptl_resubmit() - Re-submit a packet to the transport layer.
1353 * @packet: The packet to re-submit.
1354 *
1355 * Re-submits the given packet: Checks if it can be re-submitted and queues it
1356 * if it can, resetting the packet timestamp in the process. Must be called
1357 * with the pending lock held.
1358 *
1359 * Return: Returns %-ECANCELED if the packet has exceeded its number of tries,
1360 * %-EINVAL if the packet has been locked, %-EALREADY if the packet is already
1361 * on the queue, and %-ESHUTDOWN if the transmission layer has been shut down.
1362 */
1363static int __ssh_ptl_resubmit(struct ssh_packet *packet)
1364{
1365 int status;
1366 u8 try;
1367
1368 lockdep_assert_held(&packet->ptl->pending.lock);
1369
1370 trace_ssam_packet_resubmit(packet);
1371
1372 spin_lock(&packet->ptl->queue.lock);
1373
1374 /* Check if the packet is out of tries. */
1375 try = ssh_packet_priority_get_try(packet->priority);
1376 if (try >= SSH_PTL_MAX_PACKET_TRIES) {
1377 spin_unlock(&packet->ptl->queue.lock);
1378 return -ECANCELED;
1379 }
1380
1381 status = __ssh_ptl_queue_push(packet);
1382 if (status) {
1383 /*
1384 * An error here indicates that the packet has either already
1385 * been queued, been locked, or the transport layer is being
1386 * shut down. In all cases: Ignore the error.
1387 */
1388 spin_unlock(&packet->ptl->queue.lock);
1389 return status;
1390 }
1391
1392 packet->timestamp = KTIME_MAX;
1393
1394 spin_unlock(&packet->ptl->queue.lock);
1395 return 0;
1396}
1397
1398static void ssh_ptl_resubmit_pending(struct ssh_ptl *ptl)
1399{
1400 struct ssh_packet *p;
1401 bool resub = false;
1402
1403 /*
1404 * Note: We deliberately do not remove/attempt to cancel and complete
1405 * packets that are out of tires in this function. The packet will be
1406 * eventually canceled and completed by the timeout. Removing the packet
1407 * here could lead to overly eager cancellation if the packet has not
1408 * been re-transmitted yet but the tries-counter already updated (i.e
1409 * ssh_ptl_tx_next() removed the packet from the queue and updated the
1410 * counter, but re-transmission for the last try has not actually
1411 * started yet).
1412 */
1413
1414 spin_lock(&ptl->pending.lock);
1415
1416 /* Re-queue all pending packets. */
1417 list_for_each_entry(p, &ptl->pending.head, pending_node) {
1418 /*
1419 * Re-submission fails if the packet is out of tries, has been
1420 * locked, is already queued, or the layer is being shut down.
1421 * No need to re-schedule tx-thread in those cases.
1422 */
1423 if (!__ssh_ptl_resubmit(p))
1424 resub = true;
1425 }
1426
1427 spin_unlock(&ptl->pending.lock);
1428
1429 if (resub)
1430 ssh_ptl_tx_wakeup_packet(ptl);
1431}
1432
1433/**
1434 * ssh_ptl_cancel() - Cancel a packet.
1435 * @p: The packet to cancel.
1436 *
1437 * Cancels a packet. There are no guarantees on when completion and release
1438 * callbacks will be called. This may occur during execution of this function
1439 * or may occur at any point later.
1440 *
1441 * Note that it is not guaranteed that the packet will actually be canceled if
1442 * the packet is concurrently completed by another process. The only guarantee
1443 * of this function is that the packet will be completed (with success,
1444 * failure, or cancellation) and released from the transport layer in a
1445 * reasonable time-frame.
1446 *
1447 * May be called before the packet has been submitted, in which case any later
1448 * packet submission fails.
1449 */
1450void ssh_ptl_cancel(struct ssh_packet *p)
1451{
1452 if (test_and_set_bit(SSH_PACKET_SF_CANCELED_BIT, &p->state))
1453 return;
1454
1455 trace_ssam_packet_cancel(p);
1456
1457 /*
1458 * Lock packet and commit with memory barrier. If this packet has
1459 * already been locked, it's going to be removed and completed by
1460 * another party, which should have precedence.
1461 */
1462 if (test_and_set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))
1463 return;
1464
1465 /*
1466 * By marking the packet as locked and employing the implicit memory
1467 * barrier of test_and_set_bit, we have guaranteed that, at this point,
1468 * the packet cannot be added to the queue any more.
1469 *
1470 * In case the packet has never been submitted, packet->ptl is NULL. If
1471 * the packet is currently being submitted, packet->ptl may be NULL or
1472 * non-NULL. Due marking the packet as locked above and committing with
1473 * the memory barrier, we have guaranteed that, if packet->ptl is NULL,
1474 * the packet will never be added to the queue. If packet->ptl is
1475 * non-NULL, we don't have any guarantees.
1476 */
1477
1478 if (READ_ONCE(p->ptl)) {
1479 ssh_ptl_remove_and_complete(p, -ECANCELED);
1480
1481 if (atomic_read(&p->ptl->pending.count) < SSH_PTL_MAX_PENDING)
1482 ssh_ptl_tx_wakeup_packet(p->ptl);
1483
1484 } else if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state)) {
1485 __ssh_ptl_complete(p, -ECANCELED);
1486 }
1487}
1488
1489/* Must be called with pending lock held */
1490static ktime_t ssh_packet_get_expiration(struct ssh_packet *p, ktime_t timeout)
1491{
1492 lockdep_assert_held(&p->ptl->pending.lock);
1493
1494 if (p->timestamp != KTIME_MAX)
1495 return ktime_add(p->timestamp, timeout);
1496 else
1497 return KTIME_MAX;
1498}
1499
1500static void ssh_ptl_timeout_reap(struct work_struct *work)
1501{
1502 struct ssh_ptl *ptl = to_ssh_ptl(work, rtx_timeout.reaper.work);
1503 struct ssh_packet *p, *n;
1504 LIST_HEAD(claimed);
1505 ktime_t now = ktime_get_coarse_boottime();
1506 ktime_t timeout = ptl->rtx_timeout.timeout;
1507 ktime_t next = KTIME_MAX;
1508 bool resub = false;
1509 int status;
1510
1511 trace_ssam_ptl_timeout_reap(atomic_read(&ptl->pending.count));
1512
1513 /*
1514 * Mark reaper as "not pending". This is done before checking any
1515 * packets to avoid lost-update type problems.
1516 */
1517 spin_lock(&ptl->rtx_timeout.lock);
1518 ptl->rtx_timeout.expires = KTIME_MAX;
1519 spin_unlock(&ptl->rtx_timeout.lock);
1520
1521 spin_lock(&ptl->pending.lock);
1522
1523 list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {
1524 ktime_t expires = ssh_packet_get_expiration(p, timeout);
1525
1526 /*
1527 * Check if the timeout hasn't expired yet. Find out next
1528 * expiration date to be handled after this run.
1529 */
1530 if (ktime_after(expires, now)) {
1531 next = ktime_before(expires, next) ? expires : next;
1532 continue;
1533 }
1534
1535 trace_ssam_packet_timeout(p);
1536
1537 status = __ssh_ptl_resubmit(p);
1538
1539 /*
1540 * Re-submission fails if the packet is out of tries, has been
1541 * locked, is already queued, or the layer is being shut down.
1542 * No need to re-schedule tx-thread in those cases.
1543 */
1544 if (!status)
1545 resub = true;
1546
1547 /* Go to next packet if this packet is not out of tries. */
1548 if (status != -ECANCELED)
1549 continue;
1550
1551 /* No more tries left: Cancel the packet. */
1552
1553 /*
1554 * If someone else has locked the packet already, don't use it
1555 * and let the other party complete it.
1556 */
1557 if (test_and_set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))
1558 continue;
1559
1560 /*
1561 * We have now marked the packet as locked. Thus it cannot be
1562 * added to the pending list again after we've removed it here.
1563 * We can therefore re-use the pending_node of this packet
1564 * temporarily.
1565 */
1566
1567 clear_bit(SSH_PACKET_SF_PENDING_BIT, &p->state);
1568
1569 atomic_dec(&ptl->pending.count);
1570 list_move_tail(&p->pending_node, &claimed);
1571 }
1572
1573 spin_unlock(&ptl->pending.lock);
1574
1575 /* Cancel and complete the packet. */
1576 list_for_each_entry_safe(p, n, &claimed, pending_node) {
1577 if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state)) {
1578 ssh_ptl_queue_remove(p);
1579 __ssh_ptl_complete(p, -ETIMEDOUT);
1580 }
1581
1582 /*
1583 * Drop the reference we've obtained by removing it from
1584 * the pending set.
1585 */
1586 list_del(&p->pending_node);
1587 ssh_packet_put(p);
1588 }
1589
1590 /* Ensure that reaper doesn't run again immediately. */
1591 next = max(next, ktime_add(now, SSH_PTL_PACKET_TIMEOUT_RESOLUTION));
1592 if (next != KTIME_MAX)
1593 ssh_ptl_timeout_reaper_mod(ptl, now, next);
1594
1595 if (resub)
1596 ssh_ptl_tx_wakeup_packet(ptl);
1597}
1598
1599static bool ssh_ptl_rx_retransmit_check(struct ssh_ptl *ptl, const struct ssh_frame *frame)
1600{
1601 int i;
1602
1603 /*
1604 * Ignore unsequenced packets. On some devices (notably Surface Pro 9),
1605 * unsequenced events will always be sent with SEQ=0x00. Attempting to
1606 * detect retransmission would thus just block all events.
1607 *
1608 * While sequence numbers would also allow detection of retransmitted
1609 * packets in unsequenced communication, they have only ever been used
1610 * to cover edge-cases in sequenced transmission. In particular, the
1611 * only instance of packets being retransmitted (that we are aware of)
1612 * is due to an ACK timeout. As this does not happen in unsequenced
1613 * communication, skip the retransmission check for those packets
1614 * entirely.
1615 */
1616 if (frame->type == SSH_FRAME_TYPE_DATA_NSQ)
1617 return false;
1618
1619 /*
1620 * Check if SEQ has been seen recently (i.e. packet was
1621 * re-transmitted and we should ignore it).
1622 */
1623 for (i = 0; i < ARRAY_SIZE(ptl->rx.blocked.seqs); i++) {
1624 if (likely(ptl->rx.blocked.seqs[i] != frame->seq))
1625 continue;
1626
1627 ptl_dbg(ptl, "ptl: ignoring repeated data packet\n");
1628 return true;
1629 }
1630
1631 /* Update list of blocked sequence IDs. */
1632 ptl->rx.blocked.seqs[ptl->rx.blocked.offset] = frame->seq;
1633 ptl->rx.blocked.offset = (ptl->rx.blocked.offset + 1)
1634 % ARRAY_SIZE(ptl->rx.blocked.seqs);
1635
1636 return false;
1637}
1638
1639static void ssh_ptl_rx_dataframe(struct ssh_ptl *ptl,
1640 const struct ssh_frame *frame,
1641 const struct ssam_span *payload)
1642{
1643 if (ssh_ptl_rx_retransmit_check(ptl, frame))
1644 return;
1645
1646 ptl->ops.data_received(ptl, payload);
1647}
1648
1649static void ssh_ptl_send_ack(struct ssh_ptl *ptl, u8 seq)
1650{
1651 struct ssh_packet *packet;
1652 struct ssam_span buf;
1653 struct msgbuf msgb;
1654 int status;
1655
1656 status = ssh_ctrl_packet_alloc(&packet, &buf, GFP_KERNEL);
1657 if (status) {
1658 ptl_err(ptl, "ptl: failed to allocate ACK packet\n");
1659 return;
1660 }
1661
1662 ssh_packet_init(packet, 0, SSH_PACKET_PRIORITY(ACK, 0),
1663 &ssh_ptl_ctrl_packet_ops);
1664
1665 msgb_init(&msgb, buf.ptr, buf.len);
1666 msgb_push_ack(&msgb, seq);
1667 ssh_packet_set_data(packet, msgb.begin, msgb_bytes_used(&msgb));
1668
1669 ssh_ptl_submit(ptl, packet);
1670 ssh_packet_put(packet);
1671}
1672
1673static void ssh_ptl_send_nak(struct ssh_ptl *ptl)
1674{
1675 struct ssh_packet *packet;
1676 struct ssam_span buf;
1677 struct msgbuf msgb;
1678 int status;
1679
1680 status = ssh_ctrl_packet_alloc(&packet, &buf, GFP_KERNEL);
1681 if (status) {
1682 ptl_err(ptl, "ptl: failed to allocate NAK packet\n");
1683 return;
1684 }
1685
1686 ssh_packet_init(packet, 0, SSH_PACKET_PRIORITY(NAK, 0),
1687 &ssh_ptl_ctrl_packet_ops);
1688
1689 msgb_init(&msgb, buf.ptr, buf.len);
1690 msgb_push_nak(&msgb);
1691 ssh_packet_set_data(packet, msgb.begin, msgb_bytes_used(&msgb));
1692
1693 ssh_ptl_submit(ptl, packet);
1694 ssh_packet_put(packet);
1695}
1696
1697static size_t ssh_ptl_rx_eval(struct ssh_ptl *ptl, struct ssam_span *source)
1698{
1699 struct ssh_frame *frame;
1700 struct ssam_span payload;
1701 struct ssam_span aligned;
1702 bool syn_found;
1703 int status;
1704
1705 /* Error injection: Modify data to simulate corrupt SYN bytes. */
1706 ssh_ptl_rx_inject_invalid_syn(ptl, source);
1707
1708 /* Find SYN. */
1709 syn_found = sshp_find_syn(source, &aligned);
1710
1711 if (unlikely(aligned.ptr != source->ptr)) {
1712 /*
1713 * We expect aligned.ptr == source->ptr. If this is not the
1714 * case, then aligned.ptr > source->ptr and we've encountered
1715 * some unexpected data where we'd expect the start of a new
1716 * message (i.e. the SYN sequence).
1717 *
1718 * This can happen when a CRC check for the previous message
1719 * failed and we start actively searching for the next one
1720 * (via the call to sshp_find_syn() above), or the first bytes
1721 * of a message got dropped or corrupted.
1722 *
1723 * In any case, we issue a warning, send a NAK to the EC to
1724 * request re-transmission of any data we haven't acknowledged
1725 * yet, and finally, skip everything up to the next SYN
1726 * sequence.
1727 */
1728
1729 ptl_warn(ptl, "rx: parser: invalid start of frame, skipping\n");
1730
1731 /*
1732 * Notes:
1733 * - This might send multiple NAKs in case the communication
1734 * starts with an invalid SYN and is broken down into multiple
1735 * pieces. This should generally be handled fine, we just
1736 * might receive duplicate data in this case, which is
1737 * detected when handling data frames.
1738 * - This path will also be executed on invalid CRCs: When an
1739 * invalid CRC is encountered, the code below will skip data
1740 * until directly after the SYN. This causes the search for
1741 * the next SYN, which is generally not placed directly after
1742 * the last one.
1743 *
1744 * Open question: Should we send this in case of invalid
1745 * payload CRCs if the frame-type is non-sequential (current
1746 * implementation) or should we drop that frame without
1747 * telling the EC?
1748 */
1749 ssh_ptl_send_nak(ptl);
1750 }
1751
1752 if (unlikely(!syn_found))
1753 return aligned.ptr - source->ptr;
1754
1755 /* Error injection: Modify data to simulate corruption. */
1756 ssh_ptl_rx_inject_invalid_data(ptl, &aligned);
1757
1758 /* Parse and validate frame. */
1759 status = sshp_parse_frame(&ptl->serdev->dev, &aligned, &frame, &payload,
1760 SSH_PTL_RX_BUF_LEN);
1761 if (status) /* Invalid frame: skip to next SYN. */
1762 return aligned.ptr - source->ptr + sizeof(u16);
1763 if (!frame) /* Not enough data. */
1764 return aligned.ptr - source->ptr;
1765
1766 trace_ssam_rx_frame_received(frame);
1767
1768 switch (frame->type) {
1769 case SSH_FRAME_TYPE_ACK:
1770 ssh_ptl_acknowledge(ptl, frame->seq);
1771 break;
1772
1773 case SSH_FRAME_TYPE_NAK:
1774 ssh_ptl_resubmit_pending(ptl);
1775 break;
1776
1777 case SSH_FRAME_TYPE_DATA_SEQ:
1778 ssh_ptl_send_ack(ptl, frame->seq);
1779 fallthrough;
1780
1781 case SSH_FRAME_TYPE_DATA_NSQ:
1782 ssh_ptl_rx_dataframe(ptl, frame, &payload);
1783 break;
1784
1785 default:
1786 ptl_warn(ptl, "ptl: received frame with unknown type %#04x\n",
1787 frame->type);
1788 break;
1789 }
1790
1791 return aligned.ptr - source->ptr + SSH_MESSAGE_LENGTH(payload.len);
1792}
1793
1794static int ssh_ptl_rx_threadfn(void *data)
1795{
1796 struct ssh_ptl *ptl = data;
1797
1798 while (true) {
1799 struct ssam_span span;
1800 size_t offs = 0;
1801 size_t n;
1802
1803 wait_event_interruptible(ptl->rx.wq,
1804 !kfifo_is_empty(&ptl->rx.fifo) ||
1805 kthread_should_stop());
1806 if (kthread_should_stop())
1807 break;
1808
1809 /* Copy from fifo to evaluation buffer. */
1810 n = sshp_buf_read_from_fifo(&ptl->rx.buf, &ptl->rx.fifo);
1811
1812 ptl_dbg(ptl, "rx: received data (size: %zu)\n", n);
1813 print_hex_dump_debug("rx: ", DUMP_PREFIX_OFFSET, 16, 1,
1814 ptl->rx.buf.ptr + ptl->rx.buf.len - n,
1815 n, false);
1816
1817 /* Parse until we need more bytes or buffer is empty. */
1818 while (offs < ptl->rx.buf.len) {
1819 sshp_buf_span_from(&ptl->rx.buf, offs, &span);
1820 n = ssh_ptl_rx_eval(ptl, &span);
1821 if (n == 0)
1822 break; /* Need more bytes. */
1823
1824 offs += n;
1825 }
1826
1827 /* Throw away the evaluated parts. */
1828 sshp_buf_drop(&ptl->rx.buf, offs);
1829 }
1830
1831 return 0;
1832}
1833
1834static void ssh_ptl_rx_wakeup(struct ssh_ptl *ptl)
1835{
1836 wake_up(&ptl->rx.wq);
1837}
1838
1839/**
1840 * ssh_ptl_rx_start() - Start packet transport layer receiver thread.
1841 * @ptl: The packet transport layer.
1842 *
1843 * Return: Returns zero on success, a negative error code on failure.
1844 */
1845int ssh_ptl_rx_start(struct ssh_ptl *ptl)
1846{
1847 if (ptl->rx.thread)
1848 return 0;
1849
1850 ptl->rx.thread = kthread_run(ssh_ptl_rx_threadfn, ptl,
1851 "ssam_serial_hub-rx");
1852 if (IS_ERR(ptl->rx.thread))
1853 return PTR_ERR(ptl->rx.thread);
1854
1855 return 0;
1856}
1857
1858/**
1859 * ssh_ptl_rx_stop() - Stop packet transport layer receiver thread.
1860 * @ptl: The packet transport layer.
1861 *
1862 * Return: Returns zero on success, a negative error code on failure.
1863 */
1864int ssh_ptl_rx_stop(struct ssh_ptl *ptl)
1865{
1866 int status = 0;
1867
1868 if (ptl->rx.thread) {
1869 status = kthread_stop(ptl->rx.thread);
1870 ptl->rx.thread = NULL;
1871 }
1872
1873 return status;
1874}
1875
1876/**
1877 * ssh_ptl_rx_rcvbuf() - Push data from lower-layer transport to the packet
1878 * layer.
1879 * @ptl: The packet transport layer.
1880 * @buf: Pointer to the data to push to the layer.
1881 * @n: Size of the data to push to the layer, in bytes.
1882 *
1883 * Pushes data from a lower-layer transport to the receiver fifo buffer of the
1884 * packet layer and notifies the receiver thread. Calls to this function are
1885 * ignored once the packet layer has been shut down.
1886 *
1887 * Return: Returns the number of bytes transferred (positive or zero) on
1888 * success. Returns %-ESHUTDOWN if the packet layer has been shut down.
1889 */
1890ssize_t ssh_ptl_rx_rcvbuf(struct ssh_ptl *ptl, const u8 *buf, size_t n)
1891{
1892 size_t used;
1893
1894 if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))
1895 return -ESHUTDOWN;
1896
1897 used = kfifo_in(&ptl->rx.fifo, buf, n);
1898 if (used)
1899 ssh_ptl_rx_wakeup(ptl);
1900
1901 return used;
1902}
1903
1904/**
1905 * ssh_ptl_shutdown() - Shut down the packet transport layer.
1906 * @ptl: The packet transport layer.
1907 *
1908 * Shuts down the packet transport layer, removing and canceling all queued
1909 * and pending packets. Packets canceled by this operation will be completed
1910 * with %-ESHUTDOWN as status. Receiver and transmitter threads will be
1911 * stopped.
1912 *
1913 * As a result of this function, the transport layer will be marked as shut
1914 * down. Submission of packets after the transport layer has been shut down
1915 * will fail with %-ESHUTDOWN.
1916 */
1917void ssh_ptl_shutdown(struct ssh_ptl *ptl)
1918{
1919 LIST_HEAD(complete_q);
1920 LIST_HEAD(complete_p);
1921 struct ssh_packet *p, *n;
1922 int status;
1923
1924 /* Ensure that no new packets (including ACK/NAK) can be submitted. */
1925 set_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state);
1926 /*
1927 * Ensure that the layer gets marked as shut-down before actually
1928 * stopping it. In combination with the check in ssh_ptl_queue_push(),
1929 * this guarantees that no new packets can be added and all already
1930 * queued packets are properly canceled. In combination with the check
1931 * in ssh_ptl_rx_rcvbuf(), this guarantees that received data is
1932 * properly cut off.
1933 */
1934 smp_mb__after_atomic();
1935
1936 status = ssh_ptl_rx_stop(ptl);
1937 if (status)
1938 ptl_err(ptl, "ptl: failed to stop receiver thread\n");
1939
1940 status = ssh_ptl_tx_stop(ptl);
1941 if (status)
1942 ptl_err(ptl, "ptl: failed to stop transmitter thread\n");
1943
1944 cancel_delayed_work_sync(&ptl->rtx_timeout.reaper);
1945
1946 /*
1947 * At this point, all threads have been stopped. This means that the
1948 * only references to packets from inside the system are in the queue
1949 * and pending set.
1950 *
1951 * Note: We still need locks here because someone could still be
1952 * canceling packets.
1953 *
1954 * Note 2: We can re-use queue_node (or pending_node) if we mark the
1955 * packet as locked an then remove it from the queue (or pending set
1956 * respectively). Marking the packet as locked avoids re-queuing
1957 * (which should already be prevented by having stopped the treads...)
1958 * and not setting QUEUED_BIT (or PENDING_BIT) prevents removal from a
1959 * new list via other threads (e.g. cancellation).
1960 *
1961 * Note 3: There may be overlap between complete_p and complete_q.
1962 * This is handled via test_and_set_bit() on the "completed" flag
1963 * (also handles cancellation).
1964 */
1965
1966 /* Mark queued packets as locked and move them to complete_q. */
1967 spin_lock(&ptl->queue.lock);
1968 list_for_each_entry_safe(p, n, &ptl->queue.head, queue_node) {
1969 set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state);
1970 /* Ensure that state does not get zero. */
1971 smp_mb__before_atomic();
1972 clear_bit(SSH_PACKET_SF_QUEUED_BIT, &p->state);
1973
1974 list_move_tail(&p->queue_node, &complete_q);
1975 }
1976 spin_unlock(&ptl->queue.lock);
1977
1978 /* Mark pending packets as locked and move them to complete_p. */
1979 spin_lock(&ptl->pending.lock);
1980 list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {
1981 set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state);
1982 /* Ensure that state does not get zero. */
1983 smp_mb__before_atomic();
1984 clear_bit(SSH_PACKET_SF_PENDING_BIT, &p->state);
1985
1986 list_move_tail(&p->pending_node, &complete_q);
1987 }
1988 atomic_set(&ptl->pending.count, 0);
1989 spin_unlock(&ptl->pending.lock);
1990
1991 /* Complete and drop packets on complete_q. */
1992 list_for_each_entry(p, &complete_q, queue_node) {
1993 if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state))
1994 __ssh_ptl_complete(p, -ESHUTDOWN);
1995
1996 ssh_packet_put(p);
1997 }
1998
1999 /* Complete and drop packets on complete_p. */
2000 list_for_each_entry(p, &complete_p, pending_node) {
2001 if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state))
2002 __ssh_ptl_complete(p, -ESHUTDOWN);
2003
2004 ssh_packet_put(p);
2005 }
2006
2007 /*
2008 * At this point we have guaranteed that the system doesn't reference
2009 * any packets any more.
2010 */
2011}
2012
2013/**
2014 * ssh_ptl_init() - Initialize packet transport layer.
2015 * @ptl: The packet transport layer to initialize.
2016 * @serdev: The underlying serial device, i.e. the lower-level transport.
2017 * @ops: Packet layer operations.
2018 *
2019 * Initializes the given packet transport layer. Transmitter and receiver
2020 * threads must be started separately via ssh_ptl_tx_start() and
2021 * ssh_ptl_rx_start(), after the packet-layer has been initialized and the
2022 * lower-level transport layer has been set up.
2023 *
2024 * Return: Returns zero on success and a nonzero error code on failure.
2025 */
2026int ssh_ptl_init(struct ssh_ptl *ptl, struct serdev_device *serdev,
2027 struct ssh_ptl_ops *ops)
2028{
2029 int i, status;
2030
2031 ptl->serdev = serdev;
2032 ptl->state = 0;
2033
2034 spin_lock_init(&ptl->queue.lock);
2035 INIT_LIST_HEAD(&ptl->queue.head);
2036
2037 spin_lock_init(&ptl->pending.lock);
2038 INIT_LIST_HEAD(&ptl->pending.head);
2039 atomic_set_release(&ptl->pending.count, 0);
2040
2041 ptl->tx.thread = NULL;
2042 atomic_set(&ptl->tx.running, 0);
2043 init_completion(&ptl->tx.thread_cplt_pkt);
2044 init_completion(&ptl->tx.thread_cplt_tx);
2045 init_waitqueue_head(&ptl->tx.packet_wq);
2046
2047 ptl->rx.thread = NULL;
2048 init_waitqueue_head(&ptl->rx.wq);
2049
2050 spin_lock_init(&ptl->rtx_timeout.lock);
2051 ptl->rtx_timeout.timeout = SSH_PTL_PACKET_TIMEOUT;
2052 ptl->rtx_timeout.expires = KTIME_MAX;
2053 INIT_DELAYED_WORK(&ptl->rtx_timeout.reaper, ssh_ptl_timeout_reap);
2054
2055 ptl->ops = *ops;
2056
2057 /* Initialize list of recent/blocked SEQs with invalid sequence IDs. */
2058 for (i = 0; i < ARRAY_SIZE(ptl->rx.blocked.seqs); i++)
2059 ptl->rx.blocked.seqs[i] = U16_MAX;
2060 ptl->rx.blocked.offset = 0;
2061
2062 status = kfifo_alloc(&ptl->rx.fifo, SSH_PTL_RX_FIFO_LEN, GFP_KERNEL);
2063 if (status)
2064 return status;
2065
2066 status = sshp_buf_alloc(&ptl->rx.buf, SSH_PTL_RX_BUF_LEN, GFP_KERNEL);
2067 if (status)
2068 kfifo_free(&ptl->rx.fifo);
2069
2070 return status;
2071}
2072
2073/**
2074 * ssh_ptl_destroy() - Deinitialize packet transport layer.
2075 * @ptl: The packet transport layer to deinitialize.
2076 *
2077 * Deinitializes the given packet transport layer and frees resources
2078 * associated with it. If receiver and/or transmitter threads have been
2079 * started, the layer must first be shut down via ssh_ptl_shutdown() before
2080 * this function can be called.
2081 */
2082void ssh_ptl_destroy(struct ssh_ptl *ptl)
2083{
2084 kfifo_free(&ptl->rx.fifo);
2085 sshp_buf_free(&ptl->rx.buf);
2086}