Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* RxRPC recvmsg() implementation
3 *
4 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/net.h>
11#include <linux/skbuff.h>
12#include <linux/export.h>
13#include <linux/sched/signal.h>
14
15#include <net/sock.h>
16#include <net/af_rxrpc.h>
17#include "ar-internal.h"
18
19/*
20 * Post a call for attention by the socket or kernel service. Further
21 * notifications are suppressed by putting recvmsg_link on a dummy queue.
22 */
23void rxrpc_notify_socket(struct rxrpc_call *call)
24{
25 struct rxrpc_sock *rx;
26 struct sock *sk;
27
28 _enter("%d", call->debug_id);
29
30 if (!list_empty(&call->recvmsg_link))
31 return;
32
33 rcu_read_lock();
34
35 rx = rcu_dereference(call->socket);
36 sk = &rx->sk;
37 if (rx && sk->sk_state < RXRPC_CLOSE) {
38 if (call->notify_rx) {
39 spin_lock(&call->notify_lock);
40 call->notify_rx(sk, call, call->user_call_ID);
41 spin_unlock(&call->notify_lock);
42 } else {
43 spin_lock(&rx->recvmsg_lock);
44 if (list_empty(&call->recvmsg_link)) {
45 rxrpc_get_call(call, rxrpc_call_get_notify_socket);
46 list_add_tail(&call->recvmsg_link, &rx->recvmsg_q);
47 }
48 spin_unlock(&rx->recvmsg_lock);
49
50 if (!sock_flag(sk, SOCK_DEAD)) {
51 _debug("call %ps", sk->sk_data_ready);
52 sk->sk_data_ready(sk);
53 }
54 }
55 }
56
57 rcu_read_unlock();
58 _leave("");
59}
60
61/*
62 * Pass a call terminating message to userspace.
63 */
64static int rxrpc_recvmsg_term(struct rxrpc_call *call, struct msghdr *msg)
65{
66 u32 tmp = 0;
67 int ret;
68
69 switch (call->completion) {
70 case RXRPC_CALL_SUCCEEDED:
71 ret = 0;
72 if (rxrpc_is_service_call(call))
73 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ACK, 0, &tmp);
74 break;
75 case RXRPC_CALL_REMOTELY_ABORTED:
76 tmp = call->abort_code;
77 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ABORT, 4, &tmp);
78 break;
79 case RXRPC_CALL_LOCALLY_ABORTED:
80 tmp = call->abort_code;
81 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ABORT, 4, &tmp);
82 break;
83 case RXRPC_CALL_NETWORK_ERROR:
84 tmp = -call->error;
85 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_NET_ERROR, 4, &tmp);
86 break;
87 case RXRPC_CALL_LOCAL_ERROR:
88 tmp = -call->error;
89 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_LOCAL_ERROR, 4, &tmp);
90 break;
91 default:
92 pr_err("Invalid terminal call state %u\n", call->completion);
93 BUG();
94 break;
95 }
96
97 trace_rxrpc_recvdata(call, rxrpc_recvmsg_terminal,
98 call->ackr_window - 1,
99 call->rx_pkt_offset, call->rx_pkt_len, ret);
100 return ret;
101}
102
103/*
104 * Discard a packet we've used up and advance the Rx window by one.
105 */
106static void rxrpc_rotate_rx_window(struct rxrpc_call *call)
107{
108 struct rxrpc_skb_priv *sp;
109 struct sk_buff *skb;
110 rxrpc_serial_t serial;
111 rxrpc_seq_t old_consumed = call->rx_consumed, tseq;
112 bool last;
113 int acked;
114
115 _enter("%d", call->debug_id);
116
117 skb = skb_dequeue(&call->recvmsg_queue);
118 rxrpc_see_skb(skb, rxrpc_skb_see_rotate);
119
120 sp = rxrpc_skb(skb);
121 tseq = sp->hdr.seq;
122 serial = sp->hdr.serial;
123 last = sp->hdr.flags & RXRPC_LAST_PACKET;
124
125 /* Barrier against rxrpc_input_data(). */
126 if (after(tseq, call->rx_consumed))
127 smp_store_release(&call->rx_consumed, tseq);
128
129 rxrpc_free_skb(skb, rxrpc_skb_put_rotate);
130
131 trace_rxrpc_receive(call, last ? rxrpc_receive_rotate_last : rxrpc_receive_rotate,
132 serial, call->rx_consumed);
133
134 if (last)
135 set_bit(RXRPC_CALL_RECVMSG_READ_ALL, &call->flags);
136
137 /* Check to see if there's an ACK that needs sending. */
138 acked = atomic_add_return(call->rx_consumed - old_consumed,
139 &call->ackr_nr_consumed);
140 if (acked > 8 &&
141 !test_and_set_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags))
142 rxrpc_poke_call(call, rxrpc_call_poke_idle);
143}
144
145/*
146 * Decrypt and verify a DATA packet.
147 */
148static int rxrpc_verify_data(struct rxrpc_call *call, struct sk_buff *skb)
149{
150 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
151
152 if (sp->flags & RXRPC_RX_VERIFIED)
153 return 0;
154 return call->security->verify_packet(call, skb);
155}
156
157/*
158 * Deliver messages to a call. This keeps processing packets until the buffer
159 * is filled and we find either more DATA (returns 0) or the end of the DATA
160 * (returns 1). If more packets are required, it returns -EAGAIN and if the
161 * call has failed it returns -EIO.
162 */
163static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
164 struct msghdr *msg, struct iov_iter *iter,
165 size_t len, int flags, size_t *_offset)
166{
167 struct rxrpc_skb_priv *sp;
168 struct sk_buff *skb;
169 rxrpc_seq_t seq = 0;
170 size_t remain;
171 unsigned int rx_pkt_offset, rx_pkt_len;
172 int copy, ret = -EAGAIN, ret2;
173
174 rx_pkt_offset = call->rx_pkt_offset;
175 rx_pkt_len = call->rx_pkt_len;
176
177 if (rxrpc_call_has_failed(call)) {
178 seq = call->ackr_window - 1;
179 ret = -EIO;
180 goto done;
181 }
182
183 if (test_bit(RXRPC_CALL_RECVMSG_READ_ALL, &call->flags)) {
184 seq = call->ackr_window - 1;
185 ret = 1;
186 goto done;
187 }
188
189 /* No one else can be removing stuff from the queue, so we shouldn't
190 * need the Rx lock to walk it.
191 */
192 skb = skb_peek(&call->recvmsg_queue);
193 while (skb) {
194 rxrpc_see_skb(skb, rxrpc_skb_see_recvmsg);
195 sp = rxrpc_skb(skb);
196 seq = sp->hdr.seq;
197
198 if (!(flags & MSG_PEEK))
199 trace_rxrpc_receive(call, rxrpc_receive_front,
200 sp->hdr.serial, seq);
201
202 if (msg)
203 sock_recv_timestamp(msg, sock->sk, skb);
204
205 if (rx_pkt_offset == 0) {
206 ret2 = rxrpc_verify_data(call, skb);
207 trace_rxrpc_recvdata(call, rxrpc_recvmsg_next, seq,
208 sp->offset, sp->len, ret2);
209 if (ret2 < 0) {
210 kdebug("verify = %d", ret2);
211 ret = ret2;
212 goto out;
213 }
214 rx_pkt_offset = sp->offset;
215 rx_pkt_len = sp->len;
216 } else {
217 trace_rxrpc_recvdata(call, rxrpc_recvmsg_cont, seq,
218 rx_pkt_offset, rx_pkt_len, 0);
219 }
220
221 /* We have to handle short, empty and used-up DATA packets. */
222 remain = len - *_offset;
223 copy = rx_pkt_len;
224 if (copy > remain)
225 copy = remain;
226 if (copy > 0) {
227 ret2 = skb_copy_datagram_iter(skb, rx_pkt_offset, iter,
228 copy);
229 if (ret2 < 0) {
230 ret = ret2;
231 goto out;
232 }
233
234 /* handle piecemeal consumption of data packets */
235 rx_pkt_offset += copy;
236 rx_pkt_len -= copy;
237 *_offset += copy;
238 }
239
240 if (rx_pkt_len > 0) {
241 trace_rxrpc_recvdata(call, rxrpc_recvmsg_full, seq,
242 rx_pkt_offset, rx_pkt_len, 0);
243 ASSERTCMP(*_offset, ==, len);
244 ret = 0;
245 break;
246 }
247
248 /* The whole packet has been transferred. */
249 if (sp->hdr.flags & RXRPC_LAST_PACKET)
250 ret = 1;
251 rx_pkt_offset = 0;
252 rx_pkt_len = 0;
253
254 skb = skb_peek_next(skb, &call->recvmsg_queue);
255
256 if (!(flags & MSG_PEEK))
257 rxrpc_rotate_rx_window(call);
258 }
259
260out:
261 if (!(flags & MSG_PEEK)) {
262 call->rx_pkt_offset = rx_pkt_offset;
263 call->rx_pkt_len = rx_pkt_len;
264 }
265done:
266 trace_rxrpc_recvdata(call, rxrpc_recvmsg_data_return, seq,
267 rx_pkt_offset, rx_pkt_len, ret);
268 if (ret == -EAGAIN)
269 set_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags);
270 return ret;
271}
272
273/*
274 * Receive a message from an RxRPC socket
275 * - we need to be careful about two or more threads calling recvmsg
276 * simultaneously
277 */
278int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
279 int flags)
280{
281 struct rxrpc_call *call;
282 struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
283 struct list_head *l;
284 unsigned int call_debug_id = 0;
285 size_t copied = 0;
286 long timeo;
287 int ret;
288
289 DEFINE_WAIT(wait);
290
291 trace_rxrpc_recvmsg(0, rxrpc_recvmsg_enter, 0);
292
293 if (flags & (MSG_OOB | MSG_TRUNC))
294 return -EOPNOTSUPP;
295
296 timeo = sock_rcvtimeo(&rx->sk, flags & MSG_DONTWAIT);
297
298try_again:
299 lock_sock(&rx->sk);
300
301 /* Return immediately if a client socket has no outstanding calls */
302 if (RB_EMPTY_ROOT(&rx->calls) &&
303 list_empty(&rx->recvmsg_q) &&
304 rx->sk.sk_state != RXRPC_SERVER_LISTENING) {
305 release_sock(&rx->sk);
306 return -EAGAIN;
307 }
308
309 if (list_empty(&rx->recvmsg_q)) {
310 ret = -EWOULDBLOCK;
311 if (timeo == 0) {
312 call = NULL;
313 goto error_no_call;
314 }
315
316 release_sock(&rx->sk);
317
318 /* Wait for something to happen */
319 prepare_to_wait_exclusive(sk_sleep(&rx->sk), &wait,
320 TASK_INTERRUPTIBLE);
321 ret = sock_error(&rx->sk);
322 if (ret)
323 goto wait_error;
324
325 if (list_empty(&rx->recvmsg_q)) {
326 if (signal_pending(current))
327 goto wait_interrupted;
328 trace_rxrpc_recvmsg(0, rxrpc_recvmsg_wait, 0);
329 timeo = schedule_timeout(timeo);
330 }
331 finish_wait(sk_sleep(&rx->sk), &wait);
332 goto try_again;
333 }
334
335 /* Find the next call and dequeue it if we're not just peeking. If we
336 * do dequeue it, that comes with a ref that we will need to release.
337 * We also want to weed out calls that got requeued whilst we were
338 * shovelling data out.
339 */
340 spin_lock(&rx->recvmsg_lock);
341 l = rx->recvmsg_q.next;
342 call = list_entry(l, struct rxrpc_call, recvmsg_link);
343
344 if (!rxrpc_call_is_complete(call) &&
345 skb_queue_empty(&call->recvmsg_queue)) {
346 list_del_init(&call->recvmsg_link);
347 spin_unlock(&rx->recvmsg_lock);
348 release_sock(&rx->sk);
349 trace_rxrpc_recvmsg(call->debug_id, rxrpc_recvmsg_unqueue, 0);
350 rxrpc_put_call(call, rxrpc_call_put_recvmsg);
351 goto try_again;
352 }
353
354 if (!(flags & MSG_PEEK))
355 list_del_init(&call->recvmsg_link);
356 else
357 rxrpc_get_call(call, rxrpc_call_get_recvmsg);
358 spin_unlock(&rx->recvmsg_lock);
359
360 call_debug_id = call->debug_id;
361 trace_rxrpc_recvmsg(call_debug_id, rxrpc_recvmsg_dequeue, 0);
362
363 /* We're going to drop the socket lock, so we need to lock the call
364 * against interference by sendmsg.
365 */
366 if (!mutex_trylock(&call->user_mutex)) {
367 ret = -EWOULDBLOCK;
368 if (flags & MSG_DONTWAIT)
369 goto error_requeue_call;
370 ret = -ERESTARTSYS;
371 if (mutex_lock_interruptible(&call->user_mutex) < 0)
372 goto error_requeue_call;
373 }
374
375 release_sock(&rx->sk);
376
377 if (test_bit(RXRPC_CALL_RELEASED, &call->flags))
378 BUG();
379
380 if (test_bit(RXRPC_CALL_HAS_USERID, &call->flags)) {
381 if (flags & MSG_CMSG_COMPAT) {
382 unsigned int id32 = call->user_call_ID;
383
384 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
385 sizeof(unsigned int), &id32);
386 } else {
387 unsigned long idl = call->user_call_ID;
388
389 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
390 sizeof(unsigned long), &idl);
391 }
392 if (ret < 0)
393 goto error_unlock_call;
394 }
395
396 if (msg->msg_name && call->peer) {
397 size_t len = sizeof(call->dest_srx);
398
399 memcpy(msg->msg_name, &call->dest_srx, len);
400 msg->msg_namelen = len;
401 }
402
403 ret = rxrpc_recvmsg_data(sock, call, msg, &msg->msg_iter, len,
404 flags, &copied);
405 if (ret == -EAGAIN)
406 ret = 0;
407 if (ret == -EIO)
408 goto call_failed;
409 if (ret < 0)
410 goto error_unlock_call;
411
412 if (rxrpc_call_is_complete(call) &&
413 skb_queue_empty(&call->recvmsg_queue))
414 goto call_complete;
415 if (rxrpc_call_has_failed(call))
416 goto call_failed;
417
418 if (!skb_queue_empty(&call->recvmsg_queue))
419 rxrpc_notify_socket(call);
420 goto not_yet_complete;
421
422call_failed:
423 rxrpc_purge_queue(&call->recvmsg_queue);
424call_complete:
425 ret = rxrpc_recvmsg_term(call, msg);
426 if (ret < 0)
427 goto error_unlock_call;
428 if (!(flags & MSG_PEEK))
429 rxrpc_release_call(rx, call);
430 msg->msg_flags |= MSG_EOR;
431 ret = 1;
432
433not_yet_complete:
434 if (ret == 0)
435 msg->msg_flags |= MSG_MORE;
436 else
437 msg->msg_flags &= ~MSG_MORE;
438 ret = copied;
439
440error_unlock_call:
441 mutex_unlock(&call->user_mutex);
442 rxrpc_put_call(call, rxrpc_call_put_recvmsg);
443 trace_rxrpc_recvmsg(call_debug_id, rxrpc_recvmsg_return, ret);
444 return ret;
445
446error_requeue_call:
447 if (!(flags & MSG_PEEK)) {
448 spin_lock(&rx->recvmsg_lock);
449 list_add(&call->recvmsg_link, &rx->recvmsg_q);
450 spin_unlock(&rx->recvmsg_lock);
451 trace_rxrpc_recvmsg(call_debug_id, rxrpc_recvmsg_requeue, 0);
452 } else {
453 rxrpc_put_call(call, rxrpc_call_put_recvmsg);
454 }
455error_no_call:
456 release_sock(&rx->sk);
457error_trace:
458 trace_rxrpc_recvmsg(call_debug_id, rxrpc_recvmsg_return, ret);
459 return ret;
460
461wait_interrupted:
462 ret = sock_intr_errno(timeo);
463wait_error:
464 finish_wait(sk_sleep(&rx->sk), &wait);
465 call = NULL;
466 goto error_trace;
467}
468
469/**
470 * rxrpc_kernel_recv_data - Allow a kernel service to receive data/info
471 * @sock: The socket that the call exists on
472 * @call: The call to send data through
473 * @iter: The buffer to receive into
474 * @_len: The amount of data we want to receive (decreased on return)
475 * @want_more: True if more data is expected to be read
476 * @_abort: Where the abort code is stored if -ECONNABORTED is returned
477 * @_service: Where to store the actual service ID (may be upgraded)
478 *
479 * Allow a kernel service to receive data and pick up information about the
480 * state of a call. Returns 0 if got what was asked for and there's more
481 * available, 1 if we got what was asked for and we're at the end of the data
482 * and -EAGAIN if we need more data.
483 *
484 * Note that we may return -EAGAIN to drain empty packets at the end of the
485 * data, even if we've already copied over the requested data.
486 *
487 * *_abort should also be initialised to 0.
488 */
489int rxrpc_kernel_recv_data(struct socket *sock, struct rxrpc_call *call,
490 struct iov_iter *iter, size_t *_len,
491 bool want_more, u32 *_abort, u16 *_service)
492{
493 size_t offset = 0;
494 int ret;
495
496 _enter("{%d},%zu,%d", call->debug_id, *_len, want_more);
497
498 mutex_lock(&call->user_mutex);
499
500 ret = rxrpc_recvmsg_data(sock, call, NULL, iter, *_len, 0, &offset);
501 *_len -= offset;
502 if (ret == -EIO)
503 goto call_failed;
504 if (ret < 0)
505 goto out;
506
507 /* We can only reach here with a partially full buffer if we have
508 * reached the end of the data. We must otherwise have a full buffer
509 * or have been given -EAGAIN.
510 */
511 if (ret == 1) {
512 if (iov_iter_count(iter) > 0)
513 goto short_data;
514 if (!want_more)
515 goto read_phase_complete;
516 ret = 0;
517 goto out;
518 }
519
520 if (!want_more)
521 goto excess_data;
522 goto out;
523
524read_phase_complete:
525 ret = 1;
526out:
527 if (_service)
528 *_service = call->dest_srx.srx_service;
529 mutex_unlock(&call->user_mutex);
530 _leave(" = %d [%zu,%d]", ret, iov_iter_count(iter), *_abort);
531 return ret;
532
533short_data:
534 trace_rxrpc_abort(call->debug_id, rxrpc_recvmsg_short_data,
535 call->cid, call->call_id, call->rx_consumed,
536 0, -EBADMSG);
537 ret = -EBADMSG;
538 goto out;
539excess_data:
540 trace_rxrpc_abort(call->debug_id, rxrpc_recvmsg_excess_data,
541 call->cid, call->call_id, call->rx_consumed,
542 0, -EMSGSIZE);
543 ret = -EMSGSIZE;
544 goto out;
545call_failed:
546 *_abort = call->abort_code;
547 ret = call->error;
548 if (call->completion == RXRPC_CALL_SUCCEEDED) {
549 ret = 1;
550 if (iov_iter_count(iter) > 0)
551 ret = -ECONNRESET;
552 }
553 goto out;
554}
555EXPORT_SYMBOL(rxrpc_kernel_recv_data);
1/* RxRPC recvmsg() implementation
2 *
3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
12#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13
14#include <linux/net.h>
15#include <linux/skbuff.h>
16#include <linux/export.h>
17#include <linux/sched/signal.h>
18
19#include <net/sock.h>
20#include <net/af_rxrpc.h>
21#include "ar-internal.h"
22
23/*
24 * Post a call for attention by the socket or kernel service. Further
25 * notifications are suppressed by putting recvmsg_link on a dummy queue.
26 */
27void rxrpc_notify_socket(struct rxrpc_call *call)
28{
29 struct rxrpc_sock *rx;
30 struct sock *sk;
31
32 _enter("%d", call->debug_id);
33
34 if (!list_empty(&call->recvmsg_link))
35 return;
36
37 rcu_read_lock();
38
39 rx = rcu_dereference(call->socket);
40 sk = &rx->sk;
41 if (rx && sk->sk_state < RXRPC_CLOSE) {
42 if (call->notify_rx) {
43 spin_lock_bh(&call->notify_lock);
44 call->notify_rx(sk, call, call->user_call_ID);
45 spin_unlock_bh(&call->notify_lock);
46 } else {
47 write_lock_bh(&rx->recvmsg_lock);
48 if (list_empty(&call->recvmsg_link)) {
49 rxrpc_get_call(call, rxrpc_call_got);
50 list_add_tail(&call->recvmsg_link, &rx->recvmsg_q);
51 }
52 write_unlock_bh(&rx->recvmsg_lock);
53
54 if (!sock_flag(sk, SOCK_DEAD)) {
55 _debug("call %ps", sk->sk_data_ready);
56 sk->sk_data_ready(sk);
57 }
58 }
59 }
60
61 rcu_read_unlock();
62 _leave("");
63}
64
65/*
66 * Pass a call terminating message to userspace.
67 */
68static int rxrpc_recvmsg_term(struct rxrpc_call *call, struct msghdr *msg)
69{
70 u32 tmp = 0;
71 int ret;
72
73 switch (call->completion) {
74 case RXRPC_CALL_SUCCEEDED:
75 ret = 0;
76 if (rxrpc_is_service_call(call))
77 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ACK, 0, &tmp);
78 break;
79 case RXRPC_CALL_REMOTELY_ABORTED:
80 tmp = call->abort_code;
81 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ABORT, 4, &tmp);
82 break;
83 case RXRPC_CALL_LOCALLY_ABORTED:
84 tmp = call->abort_code;
85 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_ABORT, 4, &tmp);
86 break;
87 case RXRPC_CALL_NETWORK_ERROR:
88 tmp = -call->error;
89 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_NET_ERROR, 4, &tmp);
90 break;
91 case RXRPC_CALL_LOCAL_ERROR:
92 tmp = -call->error;
93 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_LOCAL_ERROR, 4, &tmp);
94 break;
95 default:
96 pr_err("Invalid terminal call state %u\n", call->state);
97 BUG();
98 break;
99 }
100
101 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_terminal, call->rx_hard_ack,
102 call->rx_pkt_offset, call->rx_pkt_len, ret);
103 return ret;
104}
105
106/*
107 * Pass back notification of a new call. The call is added to the
108 * to-be-accepted list. This means that the next call to be accepted might not
109 * be the last call seen awaiting acceptance, but unless we leave this on the
110 * front of the queue and block all other messages until someone gives us a
111 * user_ID for it, there's not a lot we can do.
112 */
113static int rxrpc_recvmsg_new_call(struct rxrpc_sock *rx,
114 struct rxrpc_call *call,
115 struct msghdr *msg, int flags)
116{
117 int tmp = 0, ret;
118
119 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_NEW_CALL, 0, &tmp);
120
121 if (ret == 0 && !(flags & MSG_PEEK)) {
122 _debug("to be accepted");
123 write_lock_bh(&rx->recvmsg_lock);
124 list_del_init(&call->recvmsg_link);
125 write_unlock_bh(&rx->recvmsg_lock);
126
127 rxrpc_get_call(call, rxrpc_call_got);
128 write_lock(&rx->call_lock);
129 list_add_tail(&call->accept_link, &rx->to_be_accepted);
130 write_unlock(&rx->call_lock);
131 }
132
133 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_to_be_accepted, 1, 0, 0, ret);
134 return ret;
135}
136
137/*
138 * End the packet reception phase.
139 */
140static void rxrpc_end_rx_phase(struct rxrpc_call *call, rxrpc_serial_t serial)
141{
142 _enter("%d,%s", call->debug_id, rxrpc_call_states[call->state]);
143
144 trace_rxrpc_receive(call, rxrpc_receive_end, 0, call->rx_top);
145 ASSERTCMP(call->rx_hard_ack, ==, call->rx_top);
146
147#if 0 // TODO: May want to transmit final ACK under some circumstances anyway
148 if (call->state == RXRPC_CALL_CLIENT_RECV_REPLY) {
149 rxrpc_propose_ACK(call, RXRPC_ACK_IDLE, 0, serial, true, false,
150 rxrpc_propose_ack_terminal_ack);
151 rxrpc_send_ack_packet(call, false, NULL);
152 }
153#endif
154
155 write_lock_bh(&call->state_lock);
156
157 switch (call->state) {
158 case RXRPC_CALL_CLIENT_RECV_REPLY:
159 __rxrpc_call_completed(call);
160 write_unlock_bh(&call->state_lock);
161 break;
162
163 case RXRPC_CALL_SERVER_RECV_REQUEST:
164 call->tx_phase = true;
165 call->state = RXRPC_CALL_SERVER_ACK_REQUEST;
166 call->expect_req_by = jiffies + MAX_JIFFY_OFFSET;
167 write_unlock_bh(&call->state_lock);
168 rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, 0, serial, false, true,
169 rxrpc_propose_ack_processing_op);
170 break;
171 default:
172 write_unlock_bh(&call->state_lock);
173 break;
174 }
175}
176
177/*
178 * Discard a packet we've used up and advance the Rx window by one.
179 */
180static void rxrpc_rotate_rx_window(struct rxrpc_call *call)
181{
182 struct rxrpc_skb_priv *sp;
183 struct sk_buff *skb;
184 rxrpc_serial_t serial;
185 rxrpc_seq_t hard_ack, top;
186 u8 flags;
187 int ix;
188
189 _enter("%d", call->debug_id);
190
191 hard_ack = call->rx_hard_ack;
192 top = smp_load_acquire(&call->rx_top);
193 ASSERT(before(hard_ack, top));
194
195 hard_ack++;
196 ix = hard_ack & RXRPC_RXTX_BUFF_MASK;
197 skb = call->rxtx_buffer[ix];
198 rxrpc_see_skb(skb, rxrpc_skb_rx_rotated);
199 sp = rxrpc_skb(skb);
200 flags = sp->hdr.flags;
201 serial = sp->hdr.serial;
202 if (call->rxtx_annotations[ix] & RXRPC_RX_ANNO_JUMBO)
203 serial += (call->rxtx_annotations[ix] & RXRPC_RX_ANNO_JUMBO) - 1;
204
205 call->rxtx_buffer[ix] = NULL;
206 call->rxtx_annotations[ix] = 0;
207 /* Barrier against rxrpc_input_data(). */
208 smp_store_release(&call->rx_hard_ack, hard_ack);
209
210 rxrpc_free_skb(skb, rxrpc_skb_rx_freed);
211
212 _debug("%u,%u,%02x", hard_ack, top, flags);
213 trace_rxrpc_receive(call, rxrpc_receive_rotate, serial, hard_ack);
214 if (flags & RXRPC_LAST_PACKET) {
215 rxrpc_end_rx_phase(call, serial);
216 } else {
217 /* Check to see if there's an ACK that needs sending. */
218 if (after_eq(hard_ack, call->ackr_consumed + 2) ||
219 after_eq(top, call->ackr_seen + 2) ||
220 (hard_ack == top && after(hard_ack, call->ackr_consumed)))
221 rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, 0, serial,
222 true, true,
223 rxrpc_propose_ack_rotate_rx);
224 if (call->ackr_reason && call->ackr_reason != RXRPC_ACK_DELAY)
225 rxrpc_send_ack_packet(call, false, NULL);
226 }
227}
228
229/*
230 * Decrypt and verify a (sub)packet. The packet's length may be changed due to
231 * padding, but if this is the case, the packet length will be resident in the
232 * socket buffer. Note that we can't modify the master skb info as the skb may
233 * be the home to multiple subpackets.
234 */
235static int rxrpc_verify_packet(struct rxrpc_call *call, struct sk_buff *skb,
236 u8 annotation,
237 unsigned int offset, unsigned int len)
238{
239 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
240 rxrpc_seq_t seq = sp->hdr.seq;
241 u16 cksum = sp->hdr.cksum;
242
243 _enter("");
244
245 /* For all but the head jumbo subpacket, the security checksum is in a
246 * jumbo header immediately prior to the data.
247 */
248 if ((annotation & RXRPC_RX_ANNO_JUMBO) > 1) {
249 __be16 tmp;
250 if (skb_copy_bits(skb, offset - 2, &tmp, 2) < 0)
251 BUG();
252 cksum = ntohs(tmp);
253 seq += (annotation & RXRPC_RX_ANNO_JUMBO) - 1;
254 }
255
256 return call->conn->security->verify_packet(call, skb, offset, len,
257 seq, cksum);
258}
259
260/*
261 * Locate the data within a packet. This is complicated by:
262 *
263 * (1) An skb may contain a jumbo packet - so we have to find the appropriate
264 * subpacket.
265 *
266 * (2) The (sub)packets may be encrypted and, if so, the encrypted portion
267 * contains an extra header which includes the true length of the data,
268 * excluding any encrypted padding.
269 */
270static int rxrpc_locate_data(struct rxrpc_call *call, struct sk_buff *skb,
271 u8 *_annotation,
272 unsigned int *_offset, unsigned int *_len)
273{
274 unsigned int offset = sizeof(struct rxrpc_wire_header);
275 unsigned int len;
276 int ret;
277 u8 annotation = *_annotation;
278
279 /* Locate the subpacket */
280 len = skb->len - offset;
281 if ((annotation & RXRPC_RX_ANNO_JUMBO) > 0) {
282 offset += (((annotation & RXRPC_RX_ANNO_JUMBO) - 1) *
283 RXRPC_JUMBO_SUBPKTLEN);
284 len = (annotation & RXRPC_RX_ANNO_JLAST) ?
285 skb->len - offset : RXRPC_JUMBO_SUBPKTLEN;
286 }
287
288 if (!(annotation & RXRPC_RX_ANNO_VERIFIED)) {
289 ret = rxrpc_verify_packet(call, skb, annotation, offset, len);
290 if (ret < 0)
291 return ret;
292 *_annotation |= RXRPC_RX_ANNO_VERIFIED;
293 }
294
295 *_offset = offset;
296 *_len = len;
297 call->conn->security->locate_data(call, skb, _offset, _len);
298 return 0;
299}
300
301/*
302 * Deliver messages to a call. This keeps processing packets until the buffer
303 * is filled and we find either more DATA (returns 0) or the end of the DATA
304 * (returns 1). If more packets are required, it returns -EAGAIN.
305 */
306static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
307 struct msghdr *msg, struct iov_iter *iter,
308 size_t len, int flags, size_t *_offset)
309{
310 struct rxrpc_skb_priv *sp;
311 struct sk_buff *skb;
312 rxrpc_seq_t hard_ack, top, seq;
313 size_t remain;
314 bool last;
315 unsigned int rx_pkt_offset, rx_pkt_len;
316 int ix, copy, ret = -EAGAIN, ret2;
317
318 rx_pkt_offset = call->rx_pkt_offset;
319 rx_pkt_len = call->rx_pkt_len;
320
321 if (call->state >= RXRPC_CALL_SERVER_ACK_REQUEST) {
322 seq = call->rx_hard_ack;
323 ret = 1;
324 goto done;
325 }
326
327 /* Barriers against rxrpc_input_data(). */
328 hard_ack = call->rx_hard_ack;
329 seq = hard_ack + 1;
330 while (top = smp_load_acquire(&call->rx_top),
331 before_eq(seq, top)
332 ) {
333 ix = seq & RXRPC_RXTX_BUFF_MASK;
334 skb = call->rxtx_buffer[ix];
335 if (!skb) {
336 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_hole, seq,
337 rx_pkt_offset, rx_pkt_len, 0);
338 break;
339 }
340 smp_rmb();
341 rxrpc_see_skb(skb, rxrpc_skb_rx_seen);
342 sp = rxrpc_skb(skb);
343
344 if (!(flags & MSG_PEEK))
345 trace_rxrpc_receive(call, rxrpc_receive_front,
346 sp->hdr.serial, seq);
347
348 if (msg)
349 sock_recv_timestamp(msg, sock->sk, skb);
350
351 if (rx_pkt_offset == 0) {
352 ret2 = rxrpc_locate_data(call, skb,
353 &call->rxtx_annotations[ix],
354 &rx_pkt_offset, &rx_pkt_len);
355 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_next, seq,
356 rx_pkt_offset, rx_pkt_len, ret2);
357 if (ret2 < 0) {
358 ret = ret2;
359 goto out;
360 }
361 } else {
362 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_cont, seq,
363 rx_pkt_offset, rx_pkt_len, 0);
364 }
365
366 /* We have to handle short, empty and used-up DATA packets. */
367 remain = len - *_offset;
368 copy = rx_pkt_len;
369 if (copy > remain)
370 copy = remain;
371 if (copy > 0) {
372 ret2 = skb_copy_datagram_iter(skb, rx_pkt_offset, iter,
373 copy);
374 if (ret2 < 0) {
375 ret = ret2;
376 goto out;
377 }
378
379 /* handle piecemeal consumption of data packets */
380 rx_pkt_offset += copy;
381 rx_pkt_len -= copy;
382 *_offset += copy;
383 }
384
385 if (rx_pkt_len > 0) {
386 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_full, seq,
387 rx_pkt_offset, rx_pkt_len, 0);
388 ASSERTCMP(*_offset, ==, len);
389 ret = 0;
390 break;
391 }
392
393 /* The whole packet has been transferred. */
394 last = sp->hdr.flags & RXRPC_LAST_PACKET;
395 if (!(flags & MSG_PEEK))
396 rxrpc_rotate_rx_window(call);
397 rx_pkt_offset = 0;
398 rx_pkt_len = 0;
399
400 if (last) {
401 ASSERTCMP(seq, ==, READ_ONCE(call->rx_top));
402 ret = 1;
403 goto out;
404 }
405
406 seq++;
407 }
408
409out:
410 if (!(flags & MSG_PEEK)) {
411 call->rx_pkt_offset = rx_pkt_offset;
412 call->rx_pkt_len = rx_pkt_len;
413 }
414done:
415 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_data_return, seq,
416 rx_pkt_offset, rx_pkt_len, ret);
417 return ret;
418}
419
420/*
421 * Receive a message from an RxRPC socket
422 * - we need to be careful about two or more threads calling recvmsg
423 * simultaneously
424 */
425int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
426 int flags)
427{
428 struct rxrpc_call *call;
429 struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
430 struct list_head *l;
431 size_t copied = 0;
432 long timeo;
433 int ret;
434
435 DEFINE_WAIT(wait);
436
437 trace_rxrpc_recvmsg(NULL, rxrpc_recvmsg_enter, 0, 0, 0, 0);
438
439 if (flags & (MSG_OOB | MSG_TRUNC))
440 return -EOPNOTSUPP;
441
442 timeo = sock_rcvtimeo(&rx->sk, flags & MSG_DONTWAIT);
443
444try_again:
445 lock_sock(&rx->sk);
446
447 /* Return immediately if a client socket has no outstanding calls */
448 if (RB_EMPTY_ROOT(&rx->calls) &&
449 list_empty(&rx->recvmsg_q) &&
450 rx->sk.sk_state != RXRPC_SERVER_LISTENING) {
451 release_sock(&rx->sk);
452 return -ENODATA;
453 }
454
455 if (list_empty(&rx->recvmsg_q)) {
456 ret = -EWOULDBLOCK;
457 if (timeo == 0) {
458 call = NULL;
459 goto error_no_call;
460 }
461
462 release_sock(&rx->sk);
463
464 /* Wait for something to happen */
465 prepare_to_wait_exclusive(sk_sleep(&rx->sk), &wait,
466 TASK_INTERRUPTIBLE);
467 ret = sock_error(&rx->sk);
468 if (ret)
469 goto wait_error;
470
471 if (list_empty(&rx->recvmsg_q)) {
472 if (signal_pending(current))
473 goto wait_interrupted;
474 trace_rxrpc_recvmsg(NULL, rxrpc_recvmsg_wait,
475 0, 0, 0, 0);
476 timeo = schedule_timeout(timeo);
477 }
478 finish_wait(sk_sleep(&rx->sk), &wait);
479 goto try_again;
480 }
481
482 /* Find the next call and dequeue it if we're not just peeking. If we
483 * do dequeue it, that comes with a ref that we will need to release.
484 */
485 write_lock_bh(&rx->recvmsg_lock);
486 l = rx->recvmsg_q.next;
487 call = list_entry(l, struct rxrpc_call, recvmsg_link);
488 if (!(flags & MSG_PEEK))
489 list_del_init(&call->recvmsg_link);
490 else
491 rxrpc_get_call(call, rxrpc_call_got);
492 write_unlock_bh(&rx->recvmsg_lock);
493
494 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_dequeue, 0, 0, 0, 0);
495
496 /* We're going to drop the socket lock, so we need to lock the call
497 * against interference by sendmsg.
498 */
499 if (!mutex_trylock(&call->user_mutex)) {
500 ret = -EWOULDBLOCK;
501 if (flags & MSG_DONTWAIT)
502 goto error_requeue_call;
503 ret = -ERESTARTSYS;
504 if (mutex_lock_interruptible(&call->user_mutex) < 0)
505 goto error_requeue_call;
506 }
507
508 release_sock(&rx->sk);
509
510 if (test_bit(RXRPC_CALL_RELEASED, &call->flags))
511 BUG();
512
513 if (test_bit(RXRPC_CALL_HAS_USERID, &call->flags)) {
514 if (flags & MSG_CMSG_COMPAT) {
515 unsigned int id32 = call->user_call_ID;
516
517 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
518 sizeof(unsigned int), &id32);
519 } else {
520 unsigned long idl = call->user_call_ID;
521
522 ret = put_cmsg(msg, SOL_RXRPC, RXRPC_USER_CALL_ID,
523 sizeof(unsigned long), &idl);
524 }
525 if (ret < 0)
526 goto error_unlock_call;
527 }
528
529 if (msg->msg_name) {
530 struct sockaddr_rxrpc *srx = msg->msg_name;
531 size_t len = sizeof(call->peer->srx);
532
533 memcpy(msg->msg_name, &call->peer->srx, len);
534 srx->srx_service = call->service_id;
535 msg->msg_namelen = len;
536 }
537
538 switch (READ_ONCE(call->state)) {
539 case RXRPC_CALL_SERVER_ACCEPTING:
540 ret = rxrpc_recvmsg_new_call(rx, call, msg, flags);
541 break;
542 case RXRPC_CALL_CLIENT_RECV_REPLY:
543 case RXRPC_CALL_SERVER_RECV_REQUEST:
544 case RXRPC_CALL_SERVER_ACK_REQUEST:
545 ret = rxrpc_recvmsg_data(sock, call, msg, &msg->msg_iter, len,
546 flags, &copied);
547 if (ret == -EAGAIN)
548 ret = 0;
549
550 if (after(call->rx_top, call->rx_hard_ack) &&
551 call->rxtx_buffer[(call->rx_hard_ack + 1) & RXRPC_RXTX_BUFF_MASK])
552 rxrpc_notify_socket(call);
553 break;
554 default:
555 ret = 0;
556 break;
557 }
558
559 if (ret < 0)
560 goto error_unlock_call;
561
562 if (call->state == RXRPC_CALL_COMPLETE) {
563 ret = rxrpc_recvmsg_term(call, msg);
564 if (ret < 0)
565 goto error_unlock_call;
566 if (!(flags & MSG_PEEK))
567 rxrpc_release_call(rx, call);
568 msg->msg_flags |= MSG_EOR;
569 ret = 1;
570 }
571
572 if (ret == 0)
573 msg->msg_flags |= MSG_MORE;
574 else
575 msg->msg_flags &= ~MSG_MORE;
576 ret = copied;
577
578error_unlock_call:
579 mutex_unlock(&call->user_mutex);
580 rxrpc_put_call(call, rxrpc_call_put);
581 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);
582 return ret;
583
584error_requeue_call:
585 if (!(flags & MSG_PEEK)) {
586 write_lock_bh(&rx->recvmsg_lock);
587 list_add(&call->recvmsg_link, &rx->recvmsg_q);
588 write_unlock_bh(&rx->recvmsg_lock);
589 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_requeue, 0, 0, 0, 0);
590 } else {
591 rxrpc_put_call(call, rxrpc_call_put);
592 }
593error_no_call:
594 release_sock(&rx->sk);
595 trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);
596 return ret;
597
598wait_interrupted:
599 ret = sock_intr_errno(timeo);
600wait_error:
601 finish_wait(sk_sleep(&rx->sk), &wait);
602 call = NULL;
603 goto error_no_call;
604}
605
606/**
607 * rxrpc_kernel_recv_data - Allow a kernel service to receive data/info
608 * @sock: The socket that the call exists on
609 * @call: The call to send data through
610 * @buf: The buffer to receive into
611 * @size: The size of the buffer, including data already read
612 * @_offset: The running offset into the buffer.
613 * @want_more: True if more data is expected to be read
614 * @_abort: Where the abort code is stored if -ECONNABORTED is returned
615 * @_service: Where to store the actual service ID (may be upgraded)
616 *
617 * Allow a kernel service to receive data and pick up information about the
618 * state of a call. Returns 0 if got what was asked for and there's more
619 * available, 1 if we got what was asked for and we're at the end of the data
620 * and -EAGAIN if we need more data.
621 *
622 * Note that we may return -EAGAIN to drain empty packets at the end of the
623 * data, even if we've already copied over the requested data.
624 *
625 * This function adds the amount it transfers to *_offset, so this should be
626 * precleared as appropriate. Note that the amount remaining in the buffer is
627 * taken to be size - *_offset.
628 *
629 * *_abort should also be initialised to 0.
630 */
631int rxrpc_kernel_recv_data(struct socket *sock, struct rxrpc_call *call,
632 void *buf, size_t size, size_t *_offset,
633 bool want_more, u32 *_abort, u16 *_service)
634{
635 struct iov_iter iter;
636 struct kvec iov;
637 int ret;
638
639 _enter("{%d,%s},%zu/%zu,%d",
640 call->debug_id, rxrpc_call_states[call->state],
641 *_offset, size, want_more);
642
643 ASSERTCMP(*_offset, <=, size);
644 ASSERTCMP(call->state, !=, RXRPC_CALL_SERVER_ACCEPTING);
645
646 iov.iov_base = buf + *_offset;
647 iov.iov_len = size - *_offset;
648 iov_iter_kvec(&iter, ITER_KVEC | READ, &iov, 1, size - *_offset);
649
650 mutex_lock(&call->user_mutex);
651
652 switch (READ_ONCE(call->state)) {
653 case RXRPC_CALL_CLIENT_RECV_REPLY:
654 case RXRPC_CALL_SERVER_RECV_REQUEST:
655 case RXRPC_CALL_SERVER_ACK_REQUEST:
656 ret = rxrpc_recvmsg_data(sock, call, NULL, &iter, size, 0,
657 _offset);
658 if (ret < 0)
659 goto out;
660
661 /* We can only reach here with a partially full buffer if we
662 * have reached the end of the data. We must otherwise have a
663 * full buffer or have been given -EAGAIN.
664 */
665 if (ret == 1) {
666 if (*_offset < size)
667 goto short_data;
668 if (!want_more)
669 goto read_phase_complete;
670 ret = 0;
671 goto out;
672 }
673
674 if (!want_more)
675 goto excess_data;
676 goto out;
677
678 case RXRPC_CALL_COMPLETE:
679 goto call_complete;
680
681 default:
682 ret = -EINPROGRESS;
683 goto out;
684 }
685
686read_phase_complete:
687 ret = 1;
688out:
689 if (_service)
690 *_service = call->service_id;
691 mutex_unlock(&call->user_mutex);
692 _leave(" = %d [%zu,%d]", ret, *_offset, *_abort);
693 return ret;
694
695short_data:
696 trace_rxrpc_rx_eproto(call, 0, tracepoint_string("short_data"));
697 ret = -EBADMSG;
698 goto out;
699excess_data:
700 trace_rxrpc_rx_eproto(call, 0, tracepoint_string("excess_data"));
701 ret = -EMSGSIZE;
702 goto out;
703call_complete:
704 *_abort = call->abort_code;
705 ret = call->error;
706 if (call->completion == RXRPC_CALL_SUCCEEDED) {
707 ret = 1;
708 if (size > 0)
709 ret = -ECONNRESET;
710 }
711 goto out;
712}
713EXPORT_SYMBOL(rxrpc_kernel_recv_data);