Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* RxRPC recvmsg() implementation
3 *
4 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/net.h>
11#include <linux/skbuff.h>
12#include <linux/export.h>
13#include <linux/sched/signal.h>
14
15#include <net/sock.h>
16#include <net/af_rxrpc.h>
17#include "ar-internal.h"
18
19/*
20 * Post a call for attention by the socket or kernel service. Further
21 * notifications are suppressed by putting recvmsg_link on a dummy queue.
22 */
23void rxrpc_notify_socket(struct rxrpc_call *call)
24{
25 struct rxrpc_sock *rx;
26 struct sock *sk;
27
28 _enter("%d", call->debug_id);
29
30 if (!list_empty(&call->recvmsg_link))
31 return;
32
33 rcu_read_lock();
34
35 rx = rcu_dereference(call->socket);
36 sk = &rx->sk;
37 if (rx && sk->sk_state < RXRPC_CLOSE) {
38 if (call->notify_rx) {
39 spin_lock_bh(&call->notify_lock);
40 call->notify_rx(sk, call, call->user_call_ID);
41 spin_unlock_bh(&call->notify_lock);
42 } else {
43 write_lock_bh(&rx->recvmsg_lock);
44 if (list_empty(&call->recvmsg_link)) {
45 rxrpc_get_call(call, rxrpc_call_got);
46 list_add_tail(&call->recvmsg_link, &rx->recvmsg_q);
47 }
48 write_unlock_bh(&rx->recvmsg_lock);
49
50 if (!sock_flag(sk, SOCK_DEAD)) {
51 _debug("call %ps", sk->sk_data_ready);
52 sk->sk_data_ready(sk);
53 }
54 }
55 }
56
57 rcu_read_unlock();
58 _leave("");
59}
60
61/*
62 * Transition a call to the complete state.
63 */
64bool __rxrpc_set_call_completion(struct rxrpc_call *call,
65 enum rxrpc_call_completion compl,
66 u32 abort_code,
67 int error)
68{
69 if (call->state < RXRPC_CALL_COMPLETE) {
70 call->abort_code = abort_code;
71 call->error = error;
72 call->completion = compl,
73 call->state = RXRPC_CALL_COMPLETE;
74 trace_rxrpc_call_complete(call);
75 wake_up(&call->waitq);
76 rxrpc_notify_socket(call);
77 return true;
78 }
79 return false;
80}
81
82bool rxrpc_set_call_completion(struct rxrpc_call *call,
83 enum rxrpc_call_completion compl,
84 u32 abort_code,
85 int error)
86{
87 bool ret = false;
88
89 if (call->state < RXRPC_CALL_COMPLETE) {
90 write_lock_bh(&call->state_lock);
91 ret = __rxrpc_set_call_completion(call, compl, abort_code, error);
92 write_unlock_bh(&call->state_lock);
93 }
94 return ret;
95}
96
97/*
98 * Record that a call successfully completed.
99 */
100bool __rxrpc_call_completed(struct rxrpc_call *call)
101{
102 return __rxrpc_set_call_completion(call, RXRPC_CALL_SUCCEEDED, 0, 0);
103}
104
105bool rxrpc_call_completed(struct rxrpc_call *call)
106{
107 bool ret = false;
108
109 if (call->state < RXRPC_CALL_COMPLETE) {
110 write_lock_bh(&call->state_lock);
111 ret = __rxrpc_call_completed(call);
112 write_unlock_bh(&call->state_lock);
113 }
114 return ret;
115}
116
117/*
118 * Record that a call is locally aborted.
119 */
120bool __rxrpc_abort_call(const char *why, struct rxrpc_call *call,
121 rxrpc_seq_t seq, u32 abort_code, int error)
122{
123 trace_rxrpc_abort(call->debug_id, why, call->cid, call->call_id, seq,
124 abort_code, error);
125 return __rxrpc_set_call_completion(call, RXRPC_CALL_LOCALLY_ABORTED,
126 abort_code, error);
127}
128
129bool rxrpc_abort_call(const char *why, struct rxrpc_call *call,
130 rxrpc_seq_t seq, u32 abort_code, int error)
131{
132 bool ret;
133
134 write_lock_bh(&call->state_lock);
135 ret = __rxrpc_abort_call(why, call, seq, abort_code, error);
136 write_unlock_bh(&call->state_lock);
137 return ret;
138}
139
140/*
141 * Pass a call terminating message to userspace.
142 */
143static int rxrpc_recvmsg_term(struct rxrpc_call *call, struct msghdr *msg)
144{
145 u32 tmp = 0;
146 int ret;
147
148 switch (call->completion) {
149 case RXRPC_CALL_SUCCEEDED:
150 ret = 0;
151 if (rxrpc_is_service_call(call))
152 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ACK, 0, &tmp);
153 break;
154 case RXRPC_CALL_REMOTELY_ABORTED:
155 tmp = call->abort_code;
156 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ABORT, 4, &tmp);
157 break;
158 case RXRPC_CALL_LOCALLY_ABORTED:
159 tmp = call->abort_code;
160 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ABORT, 4, &tmp);
161 break;
162 case RXRPC_CALL_NETWORK_ERROR:
163 tmp = -call->error;
164 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_NET_ERROR, 4, &tmp);
165 break;
166 case RXRPC_CALL_LOCAL_ERROR:
167 tmp = -call->error;
168 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_LOCAL_ERROR, 4, &tmp);
169 break;
170 default:
171 pr_err("Invalid terminal call state %u\n", call->state);
172 BUG();
173 break;
174 }
175
176 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_terminal, call->rx_hard_ack,
177 call->rx_pkt_offset, call->rx_pkt_len, ret);
178 return ret;
179}
180
181/*
182 * End the packet reception phase.
183 */
184static void rxrpc_end_rx_phase(struct rxrpc_call *call, rxrpc_serial_t serial)
185{
186 _enter("%d,%s", call->debug_id, rxrpc_call_states[call->state]);
187
188 trace_rxrpc_receive(call, rxrpc_receive_end, 0, call->rx_top);
189 ASSERTCMP(call->rx_hard_ack, ==, call->rx_top);
190
191 if (call->state == RXRPC_CALL_CLIENT_RECV_REPLY) {
192 rxrpc_propose_ACK(call, RXRPC_ACK_IDLE, serial, false, true,
193 rxrpc_propose_ack_terminal_ack);
194 //rxrpc_send_ack_packet(call, false, NULL);
195 }
196
197 write_lock_bh(&call->state_lock);
198
199 switch (call->state) {
200 case RXRPC_CALL_CLIENT_RECV_REPLY:
201 __rxrpc_call_completed(call);
202 write_unlock_bh(&call->state_lock);
203 break;
204
205 case RXRPC_CALL_SERVER_RECV_REQUEST:
206 call->tx_phase = true;
207 call->state = RXRPC_CALL_SERVER_ACK_REQUEST;
208 call->expect_req_by = jiffies + MAX_JIFFY_OFFSET;
209 write_unlock_bh(&call->state_lock);
210 rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, serial, false, true,
211 rxrpc_propose_ack_processing_op);
212 break;
213 default:
214 write_unlock_bh(&call->state_lock);
215 break;
216 }
217}
218
219/*
220 * Discard a packet we've used up and advance the Rx window by one.
221 */
222static void rxrpc_rotate_rx_window(struct rxrpc_call *call)
223{
224 struct rxrpc_skb_priv *sp;
225 struct sk_buff *skb;
226 rxrpc_serial_t serial;
227 rxrpc_seq_t hard_ack, top;
228 bool last = false;
229 u8 subpacket;
230 int ix;
231
232 _enter("%d", call->debug_id);
233
234 hard_ack = call->rx_hard_ack;
235 top = smp_load_acquire(&call->rx_top);
236 ASSERT(before(hard_ack, top));
237
238 hard_ack++;
239 ix = hard_ack & RXRPC_RXTX_BUFF_MASK;
240 skb = call->rxtx_buffer[ix];
241 rxrpc_see_skb(skb, rxrpc_skb_rotated);
242 sp = rxrpc_skb(skb);
243
244 subpacket = call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;
245 serial = sp->hdr.serial + subpacket;
246
247 if (subpacket == sp->nr_subpackets - 1 &&
248 sp->rx_flags & RXRPC_SKB_INCL_LAST)
249 last = true;
250
251 call->rxtx_buffer[ix] = NULL;
252 call->rxtx_annotations[ix] = 0;
253 /* Barrier against rxrpc_input_data(). */
254 smp_store_release(&call->rx_hard_ack, hard_ack);
255
256 rxrpc_free_skb(skb, rxrpc_skb_freed);
257
258 trace_rxrpc_receive(call, rxrpc_receive_rotate, serial, hard_ack);
259 if (last) {
260 rxrpc_end_rx_phase(call, serial);
261 } else {
262 /* Check to see if there's an ACK that needs sending. */
263 if (after_eq(hard_ack, call->ackr_consumed + 2) ||
264 after_eq(top, call->ackr_seen + 2) ||
265 (hard_ack == top && after(hard_ack, call->ackr_consumed)))
266 rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, serial,
267 true, true,
268 rxrpc_propose_ack_rotate_rx);
269 if (call->ackr_reason && call->ackr_reason != RXRPC_ACK_DELAY)
270 rxrpc_send_ack_packet(call, false, NULL);
271 }
272}
273
274/*
275 * Decrypt and verify a (sub)packet. The packet's length may be changed due to
276 * padding, but if this is the case, the packet length will be resident in the
277 * socket buffer. Note that we can't modify the master skb info as the skb may
278 * be the home to multiple subpackets.
279 */
280static int rxrpc_verify_packet(struct rxrpc_call *call, struct sk_buff *skb,
281 u8 annotation,
282 unsigned int offset, unsigned int len)
283{
284 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
285 rxrpc_seq_t seq = sp->hdr.seq;
286 u16 cksum = sp->hdr.cksum;
287 u8 subpacket = annotation & RXRPC_RX_ANNO_SUBPACKET;
288
289 _enter("");
290
291 /* For all but the head jumbo subpacket, the security checksum is in a
292 * jumbo header immediately prior to the data.
293 */
294 if (subpacket > 0) {
295 __be16 tmp;
296 if (skb_copy_bits(skb, offset - 2, &tmp, 2) < 0)
297 BUG();
298 cksum = ntohs(tmp);
299 seq += subpacket;
300 }
301
302 return call->security->verify_packet(call, skb, offset, len,
303 seq, cksum);
304}
305
306/*
307 * Locate the data within a packet. This is complicated by:
308 *
309 * (1) An skb may contain a jumbo packet - so we have to find the appropriate
310 * subpacket.
311 *
312 * (2) The (sub)packets may be encrypted and, if so, the encrypted portion
313 * contains an extra header which includes the true length of the data,
314 * excluding any encrypted padding.
315 */
316static int rxrpc_locate_data(struct rxrpc_call *call, struct sk_buff *skb,
317 u8 *_annotation,
318 unsigned int *_offset, unsigned int *_len,
319 bool *_last)
320{
321 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
322 unsigned int offset = sizeof(struct rxrpc_wire_header);
323 unsigned int len;
324 bool last = false;
325 int ret;
326 u8 annotation = *_annotation;
327 u8 subpacket = annotation & RXRPC_RX_ANNO_SUBPACKET;
328
329 /* Locate the subpacket */
330 offset += subpacket * RXRPC_JUMBO_SUBPKTLEN;
331 len = skb->len - offset;
332 if (subpacket < sp->nr_subpackets - 1)
333 len = RXRPC_JUMBO_DATALEN;
334 else if (sp->rx_flags & RXRPC_SKB_INCL_LAST)
335 last = true;
336
337 if (!(annotation & RXRPC_RX_ANNO_VERIFIED)) {
338 ret = rxrpc_verify_packet(call, skb, annotation, offset, len);
339 if (ret < 0)
340 return ret;
341 *_annotation |= RXRPC_RX_ANNO_VERIFIED;
342 }
343
344 *_offset = offset;
345 *_len = len;
346 *_last = last;
347 call->security->locate_data(call, skb, _offset, _len);
348 return 0;
349}
350
351/*
352 * Deliver messages to a call. This keeps processing packets until the buffer
353 * is filled and we find either more DATA (returns 0) or the end of the DATA
354 * (returns 1). If more packets are required, it returns -EAGAIN.
355 */
356static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
357 struct msghdr *msg, struct iov_iter *iter,
358 size_t len, int flags, size_t *_offset)
359{
360 struct rxrpc_skb_priv *sp;
361 struct sk_buff *skb;
362 rxrpc_serial_t serial;
363 rxrpc_seq_t hard_ack, top, seq;
364 size_t remain;
365 bool rx_pkt_last;
366 unsigned int rx_pkt_offset, rx_pkt_len;
367 int ix, copy, ret = -EAGAIN, ret2;
368
369 if (test_and_clear_bit(RXRPC_CALL_RX_UNDERRUN, &call->flags) &&
370 call->ackr_reason)
371 rxrpc_send_ack_packet(call, false, NULL);
372
373 rx_pkt_offset = call->rx_pkt_offset;
374 rx_pkt_len = call->rx_pkt_len;
375 rx_pkt_last = call->rx_pkt_last;
376
377 if (call->state >= RXRPC_CALL_SERVER_ACK_REQUEST) {
378 seq = call->rx_hard_ack;
379 ret = 1;
380 goto done;
381 }
382
383 /* Barriers against rxrpc_input_data(). */
384 hard_ack = call->rx_hard_ack;
385 seq = hard_ack + 1;
386
387 while (top = smp_load_acquire(&call->rx_top),
388 before_eq(seq, top)
389 ) {
390 ix = seq & RXRPC_RXTX_BUFF_MASK;
391 skb = call->rxtx_buffer[ix];
392 if (!skb) {
393 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_hole, seq,
394 rx_pkt_offset, rx_pkt_len, 0);
395 break;
396 }
397 smp_rmb();
398 rxrpc_see_skb(skb, rxrpc_skb_seen);
399 sp = rxrpc_skb(skb);
400
401 if (!(flags & MSG_PEEK)) {
402 serial = sp->hdr.serial;
403 serial += call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;
404 trace_rxrpc_receive(call, rxrpc_receive_front,
405 serial, seq);
406 }
407
408 if (msg)
409 sock_recv_timestamp(msg, sock->sk, skb);
410
411 if (rx_pkt_offset == 0) {
412 ret2 = rxrpc_locate_data(call, skb,
413 &call->rxtx_annotations[ix],
414 &rx_pkt_offset, &rx_pkt_len,
415 &rx_pkt_last);
416 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_next, seq,
417 rx_pkt_offset, rx_pkt_len, ret2);
418 if (ret2 < 0) {
419 ret = ret2;
420 goto out;
421 }
422 } else {
423 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_cont, seq,
424 rx_pkt_offset, rx_pkt_len, 0);
425 }
426
427 /* We have to handle short, empty and used-up DATA packets. */
428 remain = len - *_offset;
429 copy = rx_pkt_len;
430 if (copy > remain)
431 copy = remain;
432 if (copy > 0) {
433 ret2 = skb_copy_datagram_iter(skb, rx_pkt_offset, iter,
434 copy);
435 if (ret2 < 0) {
436 ret = ret2;
437 goto out;
438 }
439
440 /* handle piecemeal consumption of data packets */
441 rx_pkt_offset += copy;
442 rx_pkt_len -= copy;
443 *_offset += copy;
444 }
445
446 if (rx_pkt_len > 0) {
447 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_full, seq,
448 rx_pkt_offset, rx_pkt_len, 0);
449 ASSERTCMP(*_offset, ==, len);
450 ret = 0;
451 break;
452 }
453
454 /* The whole packet has been transferred. */
455 if (!(flags & MSG_PEEK))
456 rxrpc_rotate_rx_window(call);
457 rx_pkt_offset = 0;
458 rx_pkt_len = 0;
459
460 if (rx_pkt_last) {
461 ASSERTCMP(seq, ==, READ_ONCE(call->rx_top));
462 ret = 1;
463 goto out;
464 }
465
466 seq++;
467 }
468
469out:
470 if (!(flags & MSG_PEEK)) {
471 call->rx_pkt_offset = rx_pkt_offset;
472 call->rx_pkt_len = rx_pkt_len;
473 call->rx_pkt_last = rx_pkt_last;
474 }
475done:
476 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_data_return, seq,
477 rx_pkt_offset, rx_pkt_len, ret);
478 if (ret == -EAGAIN)
479 set_bit(RXRPC_CALL_RX_UNDERRUN, &call->flags);
480 return ret;
481}
482
483/*
484 * Receive a message from an RxRPC socket
485 * - we need to be careful about two or more threads calling recvmsg
486 * simultaneously
487 */
488int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
489 int flags)
490{
491 struct rxrpc_call *call;
492 struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
493 struct list_head *l;
494 size_t copied = 0;
495 long timeo;
496 int ret;
497
498 DEFINE_WAIT(wait);
499
500 trace_rxrpc_recvmsg(NULL, rxrpc_recvmsg_enter, 0, 0, 0, 0);
501
502 if (flags & (MSG_OOB | MSG_TRUNC))
503 return -EOPNOTSUPP;
504
505 timeo = sock_rcvtimeo(&rx->sk, flags & MSG_DONTWAIT);
506
507try_again:
508 lock_sock(&rx->sk);
509
510 /* Return immediately if a client socket has no outstanding calls */
511 if (RB_EMPTY_ROOT(&rx->calls) &&
512 list_empty(&rx->recvmsg_q) &&
513 rx->sk.sk_state != RXRPC_SERVER_LISTENING) {
514 release_sock(&rx->sk);
515 return -EAGAIN;
516 }
517
518 if (list_empty(&rx->recvmsg_q)) {
519 ret = -EWOULDBLOCK;
520 if (timeo == 0) {
521 call = NULL;
522 goto error_no_call;
523 }
524
525 release_sock(&rx->sk);
526
527 /* Wait for something to happen */
528 prepare_to_wait_exclusive(sk_sleep(&rx->sk), &wait,
529 TASK_INTERRUPTIBLE);
530 ret = sock_error(&rx->sk);
531 if (ret)
532 goto wait_error;
533
534 if (list_empty(&rx->recvmsg_q)) {
535 if (signal_pending(current))
536 goto wait_interrupted;
537 trace_rxrpc_recvmsg(NULL, rxrpc_recvmsg_wait,
538 0, 0, 0, 0);
539 timeo = schedule_timeout(timeo);
540 }
541 finish_wait(sk_sleep(&rx->sk), &wait);
542 goto try_again;
543 }
544
545 /* Find the next call and dequeue it if we're not just peeking. If we
546 * do dequeue it, that comes with a ref that we will need to release.
547 */
548 write_lock_bh(&rx->recvmsg_lock);
549 l = rx->recvmsg_q.next;
550 call = list_entry(l, struct rxrpc_call, recvmsg_link);
551 if (!(flags & MSG_PEEK))
552 list_del_init(&call->recvmsg_link);
553 else
554 rxrpc_get_call(call, rxrpc_call_got);
555 write_unlock_bh(&rx->recvmsg_lock);
556
557 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_dequeue, 0, 0, 0, 0);
558
559 /* We're going to drop the socket lock, so we need to lock the call
560 * against interference by sendmsg.
561 */
562 if (!mutex_trylock(&call->user_mutex)) {
563 ret = -EWOULDBLOCK;
564 if (flags & MSG_DONTWAIT)
565 goto error_requeue_call;
566 ret = -ERESTARTSYS;
567 if (mutex_lock_interruptible(&call->user_mutex) < 0)
568 goto error_requeue_call;
569 }
570
571 release_sock(&rx->sk);
572
573 if (test_bit(RXRPC_CALL_RELEASED, &call->flags))
574 BUG();
575
576 if (test_bit(RXRPC_CALL_HAS_USERID, &call->flags)) {
577 if (flags & MSG_CMSG_COMPAT) {
578 unsigned int id32 = call->user_call_ID;
579
580 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
581 sizeof(unsigned int), &id32);
582 } else {
583 unsigned long idl = call->user_call_ID;
584
585 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
586 sizeof(unsigned long), &idl);
587 }
588 if (ret < 0)
589 goto error_unlock_call;
590 }
591
592 if (msg->msg_name && call->peer) {
593 struct sockaddr_rxrpc *srx = msg->msg_name;
594 size_t len = sizeof(call->peer->srx);
595
596 memcpy(msg->msg_name, &call->peer->srx, len);
597 srx->srx_service = call->service_id;
598 msg->msg_namelen = len;
599 }
600
601 switch (READ_ONCE(call->state)) {
602 case RXRPC_CALL_CLIENT_RECV_REPLY:
603 case RXRPC_CALL_SERVER_RECV_REQUEST:
604 case RXRPC_CALL_SERVER_ACK_REQUEST:
605 ret = rxrpc_recvmsg_data(sock, call, msg, &msg->msg_iter, len,
606 flags, &copied);
607 if (ret == -EAGAIN)
608 ret = 0;
609
610 if (after(call->rx_top, call->rx_hard_ack) &&
611 call->rxtx_buffer[(call->rx_hard_ack + 1) & RXRPC_RXTX_BUFF_MASK])
612 rxrpc_notify_socket(call);
613 break;
614 default:
615 ret = 0;
616 break;
617 }
618
619 if (ret < 0)
620 goto error_unlock_call;
621
622 if (call->state == RXRPC_CALL_COMPLETE) {
623 ret = rxrpc_recvmsg_term(call, msg);
624 if (ret < 0)
625 goto error_unlock_call;
626 if (!(flags & MSG_PEEK))
627 rxrpc_release_call(rx, call);
628 msg->msg_flags |= MSG_EOR;
629 ret = 1;
630 }
631
632 if (ret == 0)
633 msg->msg_flags |= MSG_MORE;
634 else
635 msg->msg_flags &= ~MSG_MORE;
636 ret = copied;
637
638error_unlock_call:
639 mutex_unlock(&call->user_mutex);
640 rxrpc_put_call(call, rxrpc_call_put);
641 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);
642 return ret;
643
644error_requeue_call:
645 if (!(flags & MSG_PEEK)) {
646 write_lock_bh(&rx->recvmsg_lock);
647 list_add(&call->recvmsg_link, &rx->recvmsg_q);
648 write_unlock_bh(&rx->recvmsg_lock);
649 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_requeue, 0, 0, 0, 0);
650 } else {
651 rxrpc_put_call(call, rxrpc_call_put);
652 }
653error_no_call:
654 release_sock(&rx->sk);
655error_trace:
656 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);
657 return ret;
658
659wait_interrupted:
660 ret = sock_intr_errno(timeo);
661wait_error:
662 finish_wait(sk_sleep(&rx->sk), &wait);
663 call = NULL;
664 goto error_trace;
665}
666
667/**
668 * rxrpc_kernel_recv_data - Allow a kernel service to receive data/info
669 * @sock: The socket that the call exists on
670 * @call: The call to send data through
671 * @iter: The buffer to receive into
672 * @want_more: True if more data is expected to be read
673 * @_abort: Where the abort code is stored if -ECONNABORTED is returned
674 * @_service: Where to store the actual service ID (may be upgraded)
675 *
676 * Allow a kernel service to receive data and pick up information about the
677 * state of a call. Returns 0 if got what was asked for and there's more
678 * available, 1 if we got what was asked for and we're at the end of the data
679 * and -EAGAIN if we need more data.
680 *
681 * Note that we may return -EAGAIN to drain empty packets at the end of the
682 * data, even if we've already copied over the requested data.
683 *
684 * *_abort should also be initialised to 0.
685 */
686int rxrpc_kernel_recv_data(struct socket *sock, struct rxrpc_call *call,
687 struct iov_iter *iter,
688 bool want_more, u32 *_abort, u16 *_service)
689{
690 size_t offset = 0;
691 int ret;
692
693 _enter("{%d,%s},%zu,%d",
694 call->debug_id, rxrpc_call_states[call->state],
695 iov_iter_count(iter), want_more);
696
697 ASSERTCMP(call->state, !=, RXRPC_CALL_SERVER_SECURING);
698
699 mutex_lock(&call->user_mutex);
700
701 switch (READ_ONCE(call->state)) {
702 case RXRPC_CALL_CLIENT_RECV_REPLY:
703 case RXRPC_CALL_SERVER_RECV_REQUEST:
704 case RXRPC_CALL_SERVER_ACK_REQUEST:
705 ret = rxrpc_recvmsg_data(sock, call, NULL, iter,
706 iov_iter_count(iter), 0,
707 &offset);
708 if (ret < 0)
709 goto out;
710
711 /* We can only reach here with a partially full buffer if we
712 * have reached the end of the data. We must otherwise have a
713 * full buffer or have been given -EAGAIN.
714 */
715 if (ret == 1) {
716 if (iov_iter_count(iter) > 0)
717 goto short_data;
718 if (!want_more)
719 goto read_phase_complete;
720 ret = 0;
721 goto out;
722 }
723
724 if (!want_more)
725 goto excess_data;
726 goto out;
727
728 case RXRPC_CALL_COMPLETE:
729 goto call_complete;
730
731 default:
732 ret = -EINPROGRESS;
733 goto out;
734 }
735
736read_phase_complete:
737 ret = 1;
738out:
739 switch (call->ackr_reason) {
740 case RXRPC_ACK_IDLE:
741 break;
742 case RXRPC_ACK_DELAY:
743 if (ret != -EAGAIN)
744 break;
745 fallthrough;
746 default:
747 rxrpc_send_ack_packet(call, false, NULL);
748 }
749
750 if (_service)
751 *_service = call->service_id;
752 mutex_unlock(&call->user_mutex);
753 _leave(" = %d [%zu,%d]", ret, iov_iter_count(iter), *_abort);
754 return ret;
755
756short_data:
757 trace_rxrpc_rx_eproto(call, 0, tracepoint_string("short_data"));
758 ret = -EBADMSG;
759 goto out;
760excess_data:
761 trace_rxrpc_rx_eproto(call, 0, tracepoint_string("excess_data"));
762 ret = -EMSGSIZE;
763 goto out;
764call_complete:
765 *_abort = call->abort_code;
766 ret = call->error;
767 if (call->completion == RXRPC_CALL_SUCCEEDED) {
768 ret = 1;
769 if (iov_iter_count(iter) > 0)
770 ret = -ECONNRESET;
771 }
772 goto out;
773}
774EXPORT_SYMBOL(rxrpc_kernel_recv_data);
775
776/**
777 * rxrpc_kernel_get_reply_time - Get timestamp on first reply packet
778 * @sock: The socket that the call exists on
779 * @call: The call to query
780 * @_ts: Where to put the timestamp
781 *
782 * Retrieve the timestamp from the first DATA packet of the reply if it is
783 * in the ring. Returns true if successful, false if not.
784 */
785bool rxrpc_kernel_get_reply_time(struct socket *sock, struct rxrpc_call *call,
786 ktime_t *_ts)
787{
788 struct sk_buff *skb;
789 rxrpc_seq_t hard_ack, top, seq;
790 bool success = false;
791
792 mutex_lock(&call->user_mutex);
793
794 if (READ_ONCE(call->state) != RXRPC_CALL_CLIENT_RECV_REPLY)
795 goto out;
796
797 hard_ack = call->rx_hard_ack;
798 if (hard_ack != 0)
799 goto out;
800
801 seq = hard_ack + 1;
802 top = smp_load_acquire(&call->rx_top);
803 if (after(seq, top))
804 goto out;
805
806 skb = call->rxtx_buffer[seq & RXRPC_RXTX_BUFF_MASK];
807 if (!skb)
808 goto out;
809
810 *_ts = skb_get_ktime(skb);
811 success = true;
812
813out:
814 mutex_unlock(&call->user_mutex);
815 return success;
816}
817EXPORT_SYMBOL(rxrpc_kernel_get_reply_time);
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* RxRPC recvmsg() implementation
3 *
4 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/net.h>
11#include <linux/skbuff.h>
12#include <linux/export.h>
13#include <linux/sched/signal.h>
14
15#include <net/sock.h>
16#include <net/af_rxrpc.h>
17#include "ar-internal.h"
18
19/*
20 * Post a call for attention by the socket or kernel service. Further
21 * notifications are suppressed by putting recvmsg_link on a dummy queue.
22 */
23void rxrpc_notify_socket(struct rxrpc_call *call)
24{
25 struct rxrpc_sock *rx;
26 struct sock *sk;
27
28 _enter("%d", call->debug_id);
29
30 if (!list_empty(&call->recvmsg_link))
31 return;
32
33 rcu_read_lock();
34
35 rx = rcu_dereference(call->socket);
36 sk = &rx->sk;
37 if (rx && sk->sk_state < RXRPC_CLOSE) {
38 if (call->notify_rx) {
39 spin_lock(&call->notify_lock);
40 call->notify_rx(sk, call, call->user_call_ID);
41 spin_unlock(&call->notify_lock);
42 } else {
43 write_lock(&rx->recvmsg_lock);
44 if (list_empty(&call->recvmsg_link)) {
45 rxrpc_get_call(call, rxrpc_call_get_notify_socket);
46 list_add_tail(&call->recvmsg_link, &rx->recvmsg_q);
47 }
48 write_unlock(&rx->recvmsg_lock);
49
50 if (!sock_flag(sk, SOCK_DEAD)) {
51 _debug("call %ps", sk->sk_data_ready);
52 sk->sk_data_ready(sk);
53 }
54 }
55 }
56
57 rcu_read_unlock();
58 _leave("");
59}
60
61/*
62 * Pass a call terminating message to userspace.
63 */
64static int rxrpc_recvmsg_term(struct rxrpc_call *call, struct msghdr *msg)
65{
66 u32 tmp = 0;
67 int ret;
68
69 switch (call->completion) {
70 case RXRPC_CALL_SUCCEEDED:
71 ret = 0;
72 if (rxrpc_is_service_call(call))
73 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ACK, 0, &tmp);
74 break;
75 case RXRPC_CALL_REMOTELY_ABORTED:
76 tmp = call->abort_code;
77 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ABORT, 4, &tmp);
78 break;
79 case RXRPC_CALL_LOCALLY_ABORTED:
80 tmp = call->abort_code;
81 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ABORT, 4, &tmp);
82 break;
83 case RXRPC_CALL_NETWORK_ERROR:
84 tmp = -call->error;
85 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_NET_ERROR, 4, &tmp);
86 break;
87 case RXRPC_CALL_LOCAL_ERROR:
88 tmp = -call->error;
89 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_LOCAL_ERROR, 4, &tmp);
90 break;
91 default:
92 pr_err("Invalid terminal call state %u\n", call->completion);
93 BUG();
94 break;
95 }
96
97 trace_rxrpc_recvdata(call, rxrpc_recvmsg_terminal,
98 lower_32_bits(atomic64_read(&call->ackr_window)) - 1,
99 call->rx_pkt_offset, call->rx_pkt_len, ret);
100 return ret;
101}
102
103/*
104 * Discard a packet we've used up and advance the Rx window by one.
105 */
106static void rxrpc_rotate_rx_window(struct rxrpc_call *call)
107{
108 struct rxrpc_skb_priv *sp;
109 struct sk_buff *skb;
110 rxrpc_serial_t serial;
111 rxrpc_seq_t old_consumed = call->rx_consumed, tseq;
112 bool last;
113 int acked;
114
115 _enter("%d", call->debug_id);
116
117 skb = skb_dequeue(&call->recvmsg_queue);
118 rxrpc_see_skb(skb, rxrpc_skb_see_rotate);
119
120 sp = rxrpc_skb(skb);
121 tseq = sp->hdr.seq;
122 serial = sp->hdr.serial;
123 last = sp->hdr.flags & RXRPC_LAST_PACKET;
124
125 /* Barrier against rxrpc_input_data(). */
126 if (after(tseq, call->rx_consumed))
127 smp_store_release(&call->rx_consumed, tseq);
128
129 rxrpc_free_skb(skb, rxrpc_skb_put_rotate);
130
131 trace_rxrpc_receive(call, last ? rxrpc_receive_rotate_last : rxrpc_receive_rotate,
132 serial, call->rx_consumed);
133
134 if (last)
135 set_bit(RXRPC_CALL_RECVMSG_READ_ALL, &call->flags);
136
137 /* Check to see if there's an ACK that needs sending. */
138 acked = atomic_add_return(call->rx_consumed - old_consumed,
139 &call->ackr_nr_consumed);
140 if (acked > 2 &&
141 !test_and_set_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags))
142 rxrpc_poke_call(call, rxrpc_call_poke_idle);
143}
144
145/*
146 * Decrypt and verify a DATA packet.
147 */
148static int rxrpc_verify_data(struct rxrpc_call *call, struct sk_buff *skb)
149{
150 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
151
152 if (sp->flags & RXRPC_RX_VERIFIED)
153 return 0;
154 return call->security->verify_packet(call, skb);
155}
156
157/*
158 * Deliver messages to a call. This keeps processing packets until the buffer
159 * is filled and we find either more DATA (returns 0) or the end of the DATA
160 * (returns 1). If more packets are required, it returns -EAGAIN and if the
161 * call has failed it returns -EIO.
162 */
163static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
164 struct msghdr *msg, struct iov_iter *iter,
165 size_t len, int flags, size_t *_offset)
166{
167 struct rxrpc_skb_priv *sp;
168 struct sk_buff *skb;
169 rxrpc_seq_t seq = 0;
170 size_t remain;
171 unsigned int rx_pkt_offset, rx_pkt_len;
172 int copy, ret = -EAGAIN, ret2;
173
174 rx_pkt_offset = call->rx_pkt_offset;
175 rx_pkt_len = call->rx_pkt_len;
176
177 if (rxrpc_call_has_failed(call)) {
178 seq = lower_32_bits(atomic64_read(&call->ackr_window)) - 1;
179 ret = -EIO;
180 goto done;
181 }
182
183 if (test_bit(RXRPC_CALL_RECVMSG_READ_ALL, &call->flags)) {
184 seq = lower_32_bits(atomic64_read(&call->ackr_window)) - 1;
185 ret = 1;
186 goto done;
187 }
188
189 /* No one else can be removing stuff from the queue, so we shouldn't
190 * need the Rx lock to walk it.
191 */
192 skb = skb_peek(&call->recvmsg_queue);
193 while (skb) {
194 rxrpc_see_skb(skb, rxrpc_skb_see_recvmsg);
195 sp = rxrpc_skb(skb);
196 seq = sp->hdr.seq;
197
198 if (!(flags & MSG_PEEK))
199 trace_rxrpc_receive(call, rxrpc_receive_front,
200 sp->hdr.serial, seq);
201
202 if (msg)
203 sock_recv_timestamp(msg, sock->sk, skb);
204
205 if (rx_pkt_offset == 0) {
206 ret2 = rxrpc_verify_data(call, skb);
207 trace_rxrpc_recvdata(call, rxrpc_recvmsg_next, seq,
208 sp->offset, sp->len, ret2);
209 if (ret2 < 0) {
210 kdebug("verify = %d", ret2);
211 ret = ret2;
212 goto out;
213 }
214 rx_pkt_offset = sp->offset;
215 rx_pkt_len = sp->len;
216 } else {
217 trace_rxrpc_recvdata(call, rxrpc_recvmsg_cont, seq,
218 rx_pkt_offset, rx_pkt_len, 0);
219 }
220
221 /* We have to handle short, empty and used-up DATA packets. */
222 remain = len - *_offset;
223 copy = rx_pkt_len;
224 if (copy > remain)
225 copy = remain;
226 if (copy > 0) {
227 ret2 = skb_copy_datagram_iter(skb, rx_pkt_offset, iter,
228 copy);
229 if (ret2 < 0) {
230 ret = ret2;
231 goto out;
232 }
233
234 /* handle piecemeal consumption of data packets */
235 rx_pkt_offset += copy;
236 rx_pkt_len -= copy;
237 *_offset += copy;
238 }
239
240 if (rx_pkt_len > 0) {
241 trace_rxrpc_recvdata(call, rxrpc_recvmsg_full, seq,
242 rx_pkt_offset, rx_pkt_len, 0);
243 ASSERTCMP(*_offset, ==, len);
244 ret = 0;
245 break;
246 }
247
248 /* The whole packet has been transferred. */
249 if (sp->hdr.flags & RXRPC_LAST_PACKET)
250 ret = 1;
251 rx_pkt_offset = 0;
252 rx_pkt_len = 0;
253
254 skb = skb_peek_next(skb, &call->recvmsg_queue);
255
256 if (!(flags & MSG_PEEK))
257 rxrpc_rotate_rx_window(call);
258 }
259
260out:
261 if (!(flags & MSG_PEEK)) {
262 call->rx_pkt_offset = rx_pkt_offset;
263 call->rx_pkt_len = rx_pkt_len;
264 }
265done:
266 trace_rxrpc_recvdata(call, rxrpc_recvmsg_data_return, seq,
267 rx_pkt_offset, rx_pkt_len, ret);
268 if (ret == -EAGAIN)
269 set_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags);
270 return ret;
271}
272
273/*
274 * Receive a message from an RxRPC socket
275 * - we need to be careful about two or more threads calling recvmsg
276 * simultaneously
277 */
278int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
279 int flags)
280{
281 struct rxrpc_call *call;
282 struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
283 struct list_head *l;
284 unsigned int call_debug_id = 0;
285 size_t copied = 0;
286 long timeo;
287 int ret;
288
289 DEFINE_WAIT(wait);
290
291 trace_rxrpc_recvmsg(0, rxrpc_recvmsg_enter, 0);
292
293 if (flags & (MSG_OOB | MSG_TRUNC))
294 return -EOPNOTSUPP;
295
296 timeo = sock_rcvtimeo(&rx->sk, flags & MSG_DONTWAIT);
297
298try_again:
299 lock_sock(&rx->sk);
300
301 /* Return immediately if a client socket has no outstanding calls */
302 if (RB_EMPTY_ROOT(&rx->calls) &&
303 list_empty(&rx->recvmsg_q) &&
304 rx->sk.sk_state != RXRPC_SERVER_LISTENING) {
305 release_sock(&rx->sk);
306 return -EAGAIN;
307 }
308
309 if (list_empty(&rx->recvmsg_q)) {
310 ret = -EWOULDBLOCK;
311 if (timeo == 0) {
312 call = NULL;
313 goto error_no_call;
314 }
315
316 release_sock(&rx->sk);
317
318 /* Wait for something to happen */
319 prepare_to_wait_exclusive(sk_sleep(&rx->sk), &wait,
320 TASK_INTERRUPTIBLE);
321 ret = sock_error(&rx->sk);
322 if (ret)
323 goto wait_error;
324
325 if (list_empty(&rx->recvmsg_q)) {
326 if (signal_pending(current))
327 goto wait_interrupted;
328 trace_rxrpc_recvmsg(0, rxrpc_recvmsg_wait, 0);
329 timeo = schedule_timeout(timeo);
330 }
331 finish_wait(sk_sleep(&rx->sk), &wait);
332 goto try_again;
333 }
334
335 /* Find the next call and dequeue it if we're not just peeking. If we
336 * do dequeue it, that comes with a ref that we will need to release.
337 */
338 write_lock(&rx->recvmsg_lock);
339 l = rx->recvmsg_q.next;
340 call = list_entry(l, struct rxrpc_call, recvmsg_link);
341 if (!(flags & MSG_PEEK))
342 list_del_init(&call->recvmsg_link);
343 else
344 rxrpc_get_call(call, rxrpc_call_get_recvmsg);
345 write_unlock(&rx->recvmsg_lock);
346
347 call_debug_id = call->debug_id;
348 trace_rxrpc_recvmsg(call_debug_id, rxrpc_recvmsg_dequeue, 0);
349
350 /* We're going to drop the socket lock, so we need to lock the call
351 * against interference by sendmsg.
352 */
353 if (!mutex_trylock(&call->user_mutex)) {
354 ret = -EWOULDBLOCK;
355 if (flags & MSG_DONTWAIT)
356 goto error_requeue_call;
357 ret = -ERESTARTSYS;
358 if (mutex_lock_interruptible(&call->user_mutex) < 0)
359 goto error_requeue_call;
360 }
361
362 release_sock(&rx->sk);
363
364 if (test_bit(RXRPC_CALL_RELEASED, &call->flags))
365 BUG();
366
367 if (test_bit(RXRPC_CALL_HAS_USERID, &call->flags)) {
368 if (flags & MSG_CMSG_COMPAT) {
369 unsigned int id32 = call->user_call_ID;
370
371 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
372 sizeof(unsigned int), &id32);
373 } else {
374 unsigned long idl = call->user_call_ID;
375
376 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
377 sizeof(unsigned long), &idl);
378 }
379 if (ret < 0)
380 goto error_unlock_call;
381 }
382
383 if (msg->msg_name && call->peer) {
384 size_t len = sizeof(call->dest_srx);
385
386 memcpy(msg->msg_name, &call->dest_srx, len);
387 msg->msg_namelen = len;
388 }
389
390 ret = rxrpc_recvmsg_data(sock, call, msg, &msg->msg_iter, len,
391 flags, &copied);
392 if (ret == -EAGAIN)
393 ret = 0;
394 if (ret == -EIO)
395 goto call_failed;
396 if (ret < 0)
397 goto error_unlock_call;
398
399 if (rxrpc_call_is_complete(call) &&
400 skb_queue_empty(&call->recvmsg_queue))
401 goto call_complete;
402 if (rxrpc_call_has_failed(call))
403 goto call_failed;
404
405 rxrpc_notify_socket(call);
406 goto not_yet_complete;
407
408call_failed:
409 rxrpc_purge_queue(&call->recvmsg_queue);
410call_complete:
411 ret = rxrpc_recvmsg_term(call, msg);
412 if (ret < 0)
413 goto error_unlock_call;
414 if (!(flags & MSG_PEEK))
415 rxrpc_release_call(rx, call);
416 msg->msg_flags |= MSG_EOR;
417 ret = 1;
418
419not_yet_complete:
420 if (ret == 0)
421 msg->msg_flags |= MSG_MORE;
422 else
423 msg->msg_flags &= ~MSG_MORE;
424 ret = copied;
425
426error_unlock_call:
427 mutex_unlock(&call->user_mutex);
428 rxrpc_put_call(call, rxrpc_call_put_recvmsg);
429 trace_rxrpc_recvmsg(call_debug_id, rxrpc_recvmsg_return, ret);
430 return ret;
431
432error_requeue_call:
433 if (!(flags & MSG_PEEK)) {
434 write_lock(&rx->recvmsg_lock);
435 list_add(&call->recvmsg_link, &rx->recvmsg_q);
436 write_unlock(&rx->recvmsg_lock);
437 trace_rxrpc_recvmsg(call_debug_id, rxrpc_recvmsg_requeue, 0);
438 } else {
439 rxrpc_put_call(call, rxrpc_call_put_recvmsg);
440 }
441error_no_call:
442 release_sock(&rx->sk);
443error_trace:
444 trace_rxrpc_recvmsg(call_debug_id, rxrpc_recvmsg_return, ret);
445 return ret;
446
447wait_interrupted:
448 ret = sock_intr_errno(timeo);
449wait_error:
450 finish_wait(sk_sleep(&rx->sk), &wait);
451 call = NULL;
452 goto error_trace;
453}
454
455/**
456 * rxrpc_kernel_recv_data - Allow a kernel service to receive data/info
457 * @sock: The socket that the call exists on
458 * @call: The call to send data through
459 * @iter: The buffer to receive into
460 * @_len: The amount of data we want to receive (decreased on return)
461 * @want_more: True if more data is expected to be read
462 * @_abort: Where the abort code is stored if -ECONNABORTED is returned
463 * @_service: Where to store the actual service ID (may be upgraded)
464 *
465 * Allow a kernel service to receive data and pick up information about the
466 * state of a call. Returns 0 if got what was asked for and there's more
467 * available, 1 if we got what was asked for and we're at the end of the data
468 * and -EAGAIN if we need more data.
469 *
470 * Note that we may return -EAGAIN to drain empty packets at the end of the
471 * data, even if we've already copied over the requested data.
472 *
473 * *_abort should also be initialised to 0.
474 */
475int rxrpc_kernel_recv_data(struct socket *sock, struct rxrpc_call *call,
476 struct iov_iter *iter, size_t *_len,
477 bool want_more, u32 *_abort, u16 *_service)
478{
479 size_t offset = 0;
480 int ret;
481
482 _enter("{%d},%zu,%d", call->debug_id, *_len, want_more);
483
484 mutex_lock(&call->user_mutex);
485
486 ret = rxrpc_recvmsg_data(sock, call, NULL, iter, *_len, 0, &offset);
487 *_len -= offset;
488 if (ret == -EIO)
489 goto call_failed;
490 if (ret < 0)
491 goto out;
492
493 /* We can only reach here with a partially full buffer if we have
494 * reached the end of the data. We must otherwise have a full buffer
495 * or have been given -EAGAIN.
496 */
497 if (ret == 1) {
498 if (iov_iter_count(iter) > 0)
499 goto short_data;
500 if (!want_more)
501 goto read_phase_complete;
502 ret = 0;
503 goto out;
504 }
505
506 if (!want_more)
507 goto excess_data;
508 goto out;
509
510read_phase_complete:
511 ret = 1;
512out:
513 if (_service)
514 *_service = call->dest_srx.srx_service;
515 mutex_unlock(&call->user_mutex);
516 _leave(" = %d [%zu,%d]", ret, iov_iter_count(iter), *_abort);
517 return ret;
518
519short_data:
520 trace_rxrpc_abort(call->debug_id, rxrpc_recvmsg_short_data,
521 call->cid, call->call_id, call->rx_consumed,
522 0, -EBADMSG);
523 ret = -EBADMSG;
524 goto out;
525excess_data:
526 trace_rxrpc_abort(call->debug_id, rxrpc_recvmsg_excess_data,
527 call->cid, call->call_id, call->rx_consumed,
528 0, -EMSGSIZE);
529 ret = -EMSGSIZE;
530 goto out;
531call_failed:
532 *_abort = call->abort_code;
533 ret = call->error;
534 if (call->completion == RXRPC_CALL_SUCCEEDED) {
535 ret = 1;
536 if (iov_iter_count(iter) > 0)
537 ret = -ECONNRESET;
538 }
539 goto out;
540}
541EXPORT_SYMBOL(rxrpc_kernel_recv_data);