Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0
2/* Multipath TCP
3 *
4 * Copyright (c) 2017 - 2019, Intel Corporation.
5 */
6
7#define pr_fmt(fmt) "MPTCP: " fmt
8
9#include <linux/kernel.h>
10#include <linux/module.h>
11#include <linux/netdevice.h>
12#include <crypto/algapi.h>
13#include <crypto/sha.h>
14#include <net/sock.h>
15#include <net/inet_common.h>
16#include <net/inet_hashtables.h>
17#include <net/protocol.h>
18#include <net/tcp.h>
19#if IS_ENABLED(CONFIG_MPTCP_IPV6)
20#include <net/ip6_route.h>
21#endif
22#include <net/mptcp.h>
23#include "protocol.h"
24#include "mib.h"
25
26static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,
27 enum linux_mptcp_mib_field field)
28{
29 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field);
30}
31
32static void subflow_req_destructor(struct request_sock *req)
33{
34 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
35
36 pr_debug("subflow_req=%p", subflow_req);
37
38 if (subflow_req->msk)
39 sock_put((struct sock *)subflow_req->msk);
40
41 mptcp_token_destroy_request(req);
42 tcp_request_sock_ops.destructor(req);
43}
44
45static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2,
46 void *hmac)
47{
48 u8 msg[8];
49
50 put_unaligned_be32(nonce1, &msg[0]);
51 put_unaligned_be32(nonce2, &msg[4]);
52
53 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac);
54}
55
56static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk)
57{
58 return mptcp_is_fully_established((void *)msk) &&
59 READ_ONCE(msk->pm.accept_subflow);
60}
61
62/* validate received token and create truncated hmac and nonce for SYN-ACK */
63static struct mptcp_sock *subflow_token_join_request(struct request_sock *req,
64 const struct sk_buff *skb)
65{
66 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
67 u8 hmac[SHA256_DIGEST_SIZE];
68 struct mptcp_sock *msk;
69 int local_id;
70
71 msk = mptcp_token_get_sock(subflow_req->token);
72 if (!msk) {
73 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN);
74 return NULL;
75 }
76
77 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req);
78 if (local_id < 0) {
79 sock_put((struct sock *)msk);
80 return NULL;
81 }
82 subflow_req->local_id = local_id;
83
84 get_random_bytes(&subflow_req->local_nonce, sizeof(u32));
85
86 subflow_generate_hmac(msk->local_key, msk->remote_key,
87 subflow_req->local_nonce,
88 subflow_req->remote_nonce, hmac);
89
90 subflow_req->thmac = get_unaligned_be64(hmac);
91 return msk;
92}
93
94static int __subflow_init_req(struct request_sock *req, const struct sock *sk_listener)
95{
96 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
97
98 subflow_req->mp_capable = 0;
99 subflow_req->mp_join = 0;
100 subflow_req->msk = NULL;
101 mptcp_token_init_request(req);
102
103#ifdef CONFIG_TCP_MD5SIG
104 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of
105 * TCP option space.
106 */
107 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info))
108 return -EINVAL;
109#endif
110
111 return 0;
112}
113
114static void subflow_init_req(struct request_sock *req,
115 const struct sock *sk_listener,
116 struct sk_buff *skb)
117{
118 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
119 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
120 struct mptcp_options_received mp_opt;
121 int ret;
122
123 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener);
124
125 ret = __subflow_init_req(req, sk_listener);
126 if (ret)
127 return;
128
129 mptcp_get_options(skb, &mp_opt);
130
131 if (mp_opt.mp_capable) {
132 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE);
133
134 if (mp_opt.mp_join)
135 return;
136 } else if (mp_opt.mp_join) {
137 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX);
138 }
139
140 if (mp_opt.mp_capable && listener->request_mptcp) {
141 int err, retries = 4;
142
143 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
144again:
145 do {
146 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key));
147 } while (subflow_req->local_key == 0);
148
149 if (unlikely(req->syncookie)) {
150 mptcp_crypto_key_sha(subflow_req->local_key,
151 &subflow_req->token,
152 &subflow_req->idsn);
153 if (mptcp_token_exists(subflow_req->token)) {
154 if (retries-- > 0)
155 goto again;
156 } else {
157 subflow_req->mp_capable = 1;
158 }
159 return;
160 }
161
162 err = mptcp_token_new_request(req);
163 if (err == 0)
164 subflow_req->mp_capable = 1;
165 else if (retries-- > 0)
166 goto again;
167
168 } else if (mp_opt.mp_join && listener->request_mptcp) {
169 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
170 subflow_req->mp_join = 1;
171 subflow_req->backup = mp_opt.backup;
172 subflow_req->remote_id = mp_opt.join_id;
173 subflow_req->token = mp_opt.token;
174 subflow_req->remote_nonce = mp_opt.nonce;
175 subflow_req->msk = subflow_token_join_request(req, skb);
176
177 if (unlikely(req->syncookie) && subflow_req->msk) {
178 if (mptcp_can_accept_new_subflow(subflow_req->msk))
179 subflow_init_req_cookie_join_save(subflow_req, skb);
180 }
181
182 pr_debug("token=%u, remote_nonce=%u msk=%p", subflow_req->token,
183 subflow_req->remote_nonce, subflow_req->msk);
184 }
185}
186
187int mptcp_subflow_init_cookie_req(struct request_sock *req,
188 const struct sock *sk_listener,
189 struct sk_buff *skb)
190{
191 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
192 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
193 struct mptcp_options_received mp_opt;
194 int err;
195
196 err = __subflow_init_req(req, sk_listener);
197 if (err)
198 return err;
199
200 mptcp_get_options(skb, &mp_opt);
201
202 if (mp_opt.mp_capable && mp_opt.mp_join)
203 return -EINVAL;
204
205 if (mp_opt.mp_capable && listener->request_mptcp) {
206 if (mp_opt.sndr_key == 0)
207 return -EINVAL;
208
209 subflow_req->local_key = mp_opt.rcvr_key;
210 err = mptcp_token_new_request(req);
211 if (err)
212 return err;
213
214 subflow_req->mp_capable = 1;
215 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
216 } else if (mp_opt.mp_join && listener->request_mptcp) {
217 if (!mptcp_token_join_cookie_init_state(subflow_req, skb))
218 return -EINVAL;
219
220 if (mptcp_can_accept_new_subflow(subflow_req->msk))
221 subflow_req->mp_join = 1;
222
223 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
224 }
225
226 return 0;
227}
228EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req);
229
230static void subflow_v4_init_req(struct request_sock *req,
231 const struct sock *sk_listener,
232 struct sk_buff *skb)
233{
234 tcp_rsk(req)->is_mptcp = 1;
235
236 tcp_request_sock_ipv4_ops.init_req(req, sk_listener, skb);
237
238 subflow_init_req(req, sk_listener, skb);
239}
240
241#if IS_ENABLED(CONFIG_MPTCP_IPV6)
242static void subflow_v6_init_req(struct request_sock *req,
243 const struct sock *sk_listener,
244 struct sk_buff *skb)
245{
246 tcp_rsk(req)->is_mptcp = 1;
247
248 tcp_request_sock_ipv6_ops.init_req(req, sk_listener, skb);
249
250 subflow_init_req(req, sk_listener, skb);
251}
252#endif
253
254/* validate received truncated hmac and create hmac for third ACK */
255static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow)
256{
257 u8 hmac[SHA256_DIGEST_SIZE];
258 u64 thmac;
259
260 subflow_generate_hmac(subflow->remote_key, subflow->local_key,
261 subflow->remote_nonce, subflow->local_nonce,
262 hmac);
263
264 thmac = get_unaligned_be64(hmac);
265 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n",
266 subflow, subflow->token,
267 (unsigned long long)thmac,
268 (unsigned long long)subflow->thmac);
269
270 return thmac == subflow->thmac;
271}
272
273static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
274{
275 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
276 struct mptcp_options_received mp_opt;
277 struct sock *parent = subflow->conn;
278
279 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb);
280
281 if (inet_sk_state_load(parent) == TCP_SYN_SENT) {
282 inet_sk_state_store(parent, TCP_ESTABLISHED);
283 parent->sk_state_change(parent);
284 }
285
286 /* be sure no special action on any packet other than syn-ack */
287 if (subflow->conn_finished)
288 return;
289
290 subflow->rel_write_seq = 1;
291 subflow->conn_finished = 1;
292 subflow->ssn_offset = TCP_SKB_CB(skb)->seq;
293 pr_debug("subflow=%p synack seq=%x", subflow, subflow->ssn_offset);
294
295 mptcp_get_options(skb, &mp_opt);
296 if (subflow->request_mptcp) {
297 if (!mp_opt.mp_capable) {
298 MPTCP_INC_STATS(sock_net(sk),
299 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK);
300 mptcp_do_fallback(sk);
301 pr_fallback(mptcp_sk(subflow->conn));
302 goto fallback;
303 }
304
305 subflow->mp_capable = 1;
306 subflow->can_ack = 1;
307 subflow->remote_key = mp_opt.sndr_key;
308 pr_debug("subflow=%p, remote_key=%llu", subflow,
309 subflow->remote_key);
310 mptcp_finish_connect(sk);
311 } else if (subflow->request_join) {
312 u8 hmac[SHA256_DIGEST_SIZE];
313
314 if (!mp_opt.mp_join)
315 goto do_reset;
316
317 subflow->thmac = mp_opt.thmac;
318 subflow->remote_nonce = mp_opt.nonce;
319 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow,
320 subflow->thmac, subflow->remote_nonce);
321
322 if (!subflow_thmac_valid(subflow)) {
323 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC);
324 goto do_reset;
325 }
326
327 subflow_generate_hmac(subflow->local_key, subflow->remote_key,
328 subflow->local_nonce,
329 subflow->remote_nonce,
330 hmac);
331 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN);
332
333 if (!mptcp_finish_join(sk))
334 goto do_reset;
335
336 subflow->mp_join = 1;
337 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX);
338 } else if (mptcp_check_fallback(sk)) {
339fallback:
340 mptcp_rcv_space_init(mptcp_sk(parent), sk);
341 }
342 return;
343
344do_reset:
345 tcp_send_active_reset(sk, GFP_ATOMIC);
346 tcp_done(sk);
347}
348
349struct request_sock_ops mptcp_subflow_request_sock_ops;
350EXPORT_SYMBOL_GPL(mptcp_subflow_request_sock_ops);
351static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops;
352
353static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb)
354{
355 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
356
357 pr_debug("subflow=%p", subflow);
358
359 /* Never answer to SYNs sent to broadcast or multicast */
360 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
361 goto drop;
362
363 return tcp_conn_request(&mptcp_subflow_request_sock_ops,
364 &subflow_request_sock_ipv4_ops,
365 sk, skb);
366drop:
367 tcp_listendrop(sk);
368 return 0;
369}
370
371#if IS_ENABLED(CONFIG_MPTCP_IPV6)
372static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops;
373static struct inet_connection_sock_af_ops subflow_v6_specific;
374static struct inet_connection_sock_af_ops subflow_v6m_specific;
375
376static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb)
377{
378 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
379
380 pr_debug("subflow=%p", subflow);
381
382 if (skb->protocol == htons(ETH_P_IP))
383 return subflow_v4_conn_request(sk, skb);
384
385 if (!ipv6_unicast_destination(skb))
386 goto drop;
387
388 return tcp_conn_request(&mptcp_subflow_request_sock_ops,
389 &subflow_request_sock_ipv6_ops, sk, skb);
390
391drop:
392 tcp_listendrop(sk);
393 return 0; /* don't send reset */
394}
395#endif
396
397/* validate hmac received in third ACK */
398static bool subflow_hmac_valid(const struct request_sock *req,
399 const struct mptcp_options_received *mp_opt)
400{
401 const struct mptcp_subflow_request_sock *subflow_req;
402 u8 hmac[SHA256_DIGEST_SIZE];
403 struct mptcp_sock *msk;
404
405 subflow_req = mptcp_subflow_rsk(req);
406 msk = subflow_req->msk;
407 if (!msk)
408 return false;
409
410 subflow_generate_hmac(msk->remote_key, msk->local_key,
411 subflow_req->remote_nonce,
412 subflow_req->local_nonce, hmac);
413
414 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN);
415}
416
417static void mptcp_sock_destruct(struct sock *sk)
418{
419 /* if new mptcp socket isn't accepted, it is free'd
420 * from the tcp listener sockets request queue, linked
421 * from req->sk. The tcp socket is released.
422 * This calls the ULP release function which will
423 * also remove the mptcp socket, via
424 * sock_put(ctx->conn).
425 *
426 * Problem is that the mptcp socket will be in
427 * ESTABLISHED state and will not have the SOCK_DEAD flag.
428 * Both result in warnings from inet_sock_destruct.
429 */
430
431 if (sk->sk_state == TCP_ESTABLISHED) {
432 sk->sk_state = TCP_CLOSE;
433 WARN_ON_ONCE(sk->sk_socket);
434 sock_orphan(sk);
435 }
436
437 mptcp_token_destroy(mptcp_sk(sk));
438 inet_sock_destruct(sk);
439}
440
441static void mptcp_force_close(struct sock *sk)
442{
443 inet_sk_state_store(sk, TCP_CLOSE);
444 sk_common_release(sk);
445}
446
447static void subflow_ulp_fallback(struct sock *sk,
448 struct mptcp_subflow_context *old_ctx)
449{
450 struct inet_connection_sock *icsk = inet_csk(sk);
451
452 mptcp_subflow_tcp_fallback(sk, old_ctx);
453 icsk->icsk_ulp_ops = NULL;
454 rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
455 tcp_sk(sk)->is_mptcp = 0;
456}
457
458static void subflow_drop_ctx(struct sock *ssk)
459{
460 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk);
461
462 if (!ctx)
463 return;
464
465 subflow_ulp_fallback(ssk, ctx);
466 if (ctx->conn)
467 sock_put(ctx->conn);
468
469 kfree_rcu(ctx, rcu);
470}
471
472void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow,
473 struct mptcp_options_received *mp_opt)
474{
475 struct mptcp_sock *msk = mptcp_sk(subflow->conn);
476
477 subflow->remote_key = mp_opt->sndr_key;
478 subflow->fully_established = 1;
479 subflow->can_ack = 1;
480 WRITE_ONCE(msk->fully_established, true);
481}
482
483static struct sock *subflow_syn_recv_sock(const struct sock *sk,
484 struct sk_buff *skb,
485 struct request_sock *req,
486 struct dst_entry *dst,
487 struct request_sock *req_unhash,
488 bool *own_req)
489{
490 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk);
491 struct mptcp_subflow_request_sock *subflow_req;
492 struct mptcp_options_received mp_opt;
493 bool fallback, fallback_is_fatal;
494 struct sock *new_msk = NULL;
495 struct sock *child;
496
497 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn);
498
499 /* After child creation we must look for 'mp_capable' even when options
500 * are not parsed
501 */
502 mp_opt.mp_capable = 0;
503
504 /* hopefully temporary handling for MP_JOIN+syncookie */
505 subflow_req = mptcp_subflow_rsk(req);
506 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join;
507 fallback = !tcp_rsk(req)->is_mptcp;
508 if (fallback)
509 goto create_child;
510
511 /* if the sk is MP_CAPABLE, we try to fetch the client key */
512 if (subflow_req->mp_capable) {
513 if (TCP_SKB_CB(skb)->seq != subflow_req->ssn_offset + 1) {
514 /* here we can receive and accept an in-window,
515 * out-of-order pkt, which will not carry the MP_CAPABLE
516 * opt even on mptcp enabled paths
517 */
518 goto create_msk;
519 }
520
521 mptcp_get_options(skb, &mp_opt);
522 if (!mp_opt.mp_capable) {
523 fallback = true;
524 goto create_child;
525 }
526
527create_msk:
528 new_msk = mptcp_sk_clone(listener->conn, &mp_opt, req);
529 if (!new_msk)
530 fallback = true;
531 } else if (subflow_req->mp_join) {
532 mptcp_get_options(skb, &mp_opt);
533 if (!mp_opt.mp_join ||
534 !mptcp_can_accept_new_subflow(subflow_req->msk) ||
535 !subflow_hmac_valid(req, &mp_opt)) {
536 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);
537 fallback = true;
538 }
539 }
540
541create_child:
542 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst,
543 req_unhash, own_req);
544
545 if (child && *own_req) {
546 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child);
547
548 tcp_rsk(req)->drop_req = false;
549
550 /* we need to fallback on ctx allocation failure and on pre-reqs
551 * checking above. In the latter scenario we additionally need
552 * to reset the context to non MPTCP status.
553 */
554 if (!ctx || fallback) {
555 if (fallback_is_fatal)
556 goto dispose_child;
557
558 subflow_drop_ctx(child);
559 goto out;
560 }
561
562 if (ctx->mp_capable) {
563 /* this can't race with mptcp_close(), as the msk is
564 * not yet exposted to user-space
565 */
566 inet_sk_state_store((void *)new_msk, TCP_ESTABLISHED);
567
568 /* new mpc subflow takes ownership of the newly
569 * created mptcp socket
570 */
571 new_msk->sk_destruct = mptcp_sock_destruct;
572 mptcp_pm_new_connection(mptcp_sk(new_msk), 1);
573 mptcp_token_accept(subflow_req, mptcp_sk(new_msk));
574 ctx->conn = new_msk;
575 new_msk = NULL;
576
577 /* with OoO packets we can reach here without ingress
578 * mpc option
579 */
580 if (mp_opt.mp_capable)
581 mptcp_subflow_fully_established(ctx, &mp_opt);
582 } else if (ctx->mp_join) {
583 struct mptcp_sock *owner;
584
585 owner = subflow_req->msk;
586 if (!owner)
587 goto dispose_child;
588
589 /* move the msk reference ownership to the subflow */
590 subflow_req->msk = NULL;
591 ctx->conn = (struct sock *)owner;
592 if (!mptcp_finish_join(child))
593 goto dispose_child;
594
595 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX);
596 tcp_rsk(req)->drop_req = true;
597 }
598 }
599
600out:
601 /* dispose of the left over mptcp master, if any */
602 if (unlikely(new_msk))
603 mptcp_force_close(new_msk);
604
605 /* check for expected invariant - should never trigger, just help
606 * catching eariler subtle bugs
607 */
608 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp &&
609 (!mptcp_subflow_ctx(child) ||
610 !mptcp_subflow_ctx(child)->conn));
611 return child;
612
613dispose_child:
614 subflow_drop_ctx(child);
615 tcp_rsk(req)->drop_req = true;
616 inet_csk_prepare_for_destroy_sock(child);
617 tcp_done(child);
618 req->rsk_ops->send_reset(sk, skb);
619
620 /* The last child reference will be released by the caller */
621 return child;
622}
623
624static struct inet_connection_sock_af_ops subflow_specific;
625
626enum mapping_status {
627 MAPPING_OK,
628 MAPPING_INVALID,
629 MAPPING_EMPTY,
630 MAPPING_DATA_FIN,
631 MAPPING_DUMMY
632};
633
634static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq)
635{
636 if ((u32)seq == (u32)old_seq)
637 return old_seq;
638
639 /* Assume map covers data not mapped yet. */
640 return seq | ((old_seq + old_data_len + 1) & GENMASK_ULL(63, 32));
641}
642
643static void warn_bad_map(struct mptcp_subflow_context *subflow, u32 ssn)
644{
645 WARN_ONCE(1, "Bad mapping: ssn=%d map_seq=%d map_data_len=%d",
646 ssn, subflow->map_subflow_seq, subflow->map_data_len);
647}
648
649static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb)
650{
651 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
652 unsigned int skb_consumed;
653
654 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq;
655 if (WARN_ON_ONCE(skb_consumed >= skb->len))
656 return true;
657
658 return skb->len - skb_consumed <= subflow->map_data_len -
659 mptcp_subflow_get_map_offset(subflow);
660}
661
662static bool validate_mapping(struct sock *ssk, struct sk_buff *skb)
663{
664 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
665 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
666
667 if (unlikely(before(ssn, subflow->map_subflow_seq))) {
668 /* Mapping covers data later in the subflow stream,
669 * currently unsupported.
670 */
671 warn_bad_map(subflow, ssn);
672 return false;
673 }
674 if (unlikely(!before(ssn, subflow->map_subflow_seq +
675 subflow->map_data_len))) {
676 /* Mapping does covers past subflow data, invalid */
677 warn_bad_map(subflow, ssn + skb->len);
678 return false;
679 }
680 return true;
681}
682
683static enum mapping_status get_mapping_status(struct sock *ssk,
684 struct mptcp_sock *msk)
685{
686 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
687 struct mptcp_ext *mpext;
688 struct sk_buff *skb;
689 u16 data_len;
690 u64 map_seq;
691
692 skb = skb_peek(&ssk->sk_receive_queue);
693 if (!skb)
694 return MAPPING_EMPTY;
695
696 if (mptcp_check_fallback(ssk))
697 return MAPPING_DUMMY;
698
699 mpext = mptcp_get_ext(skb);
700 if (!mpext || !mpext->use_map) {
701 if (!subflow->map_valid && !skb->len) {
702 /* the TCP stack deliver 0 len FIN pkt to the receive
703 * queue, that is the only 0len pkts ever expected here,
704 * and we can admit no mapping only for 0 len pkts
705 */
706 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
707 WARN_ONCE(1, "0len seq %d:%d flags %x",
708 TCP_SKB_CB(skb)->seq,
709 TCP_SKB_CB(skb)->end_seq,
710 TCP_SKB_CB(skb)->tcp_flags);
711 sk_eat_skb(ssk, skb);
712 return MAPPING_EMPTY;
713 }
714
715 if (!subflow->map_valid)
716 return MAPPING_INVALID;
717
718 goto validate_seq;
719 }
720
721 pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d",
722 mpext->data_seq, mpext->dsn64, mpext->subflow_seq,
723 mpext->data_len, mpext->data_fin);
724
725 data_len = mpext->data_len;
726 if (data_len == 0) {
727 pr_err("Infinite mapping not handled");
728 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX);
729 return MAPPING_INVALID;
730 }
731
732 if (mpext->data_fin == 1) {
733 if (data_len == 1) {
734 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq,
735 mpext->dsn64);
736 pr_debug("DATA_FIN with no payload seq=%llu", mpext->data_seq);
737 if (subflow->map_valid) {
738 /* A DATA_FIN might arrive in a DSS
739 * option before the previous mapping
740 * has been fully consumed. Continue
741 * handling the existing mapping.
742 */
743 skb_ext_del(skb, SKB_EXT_MPTCP);
744 return MAPPING_OK;
745 } else {
746 if (updated && schedule_work(&msk->work))
747 sock_hold((struct sock *)msk);
748
749 return MAPPING_DATA_FIN;
750 }
751 } else {
752 u64 data_fin_seq = mpext->data_seq + data_len - 1;
753
754 /* If mpext->data_seq is a 32-bit value, data_fin_seq
755 * must also be limited to 32 bits.
756 */
757 if (!mpext->dsn64)
758 data_fin_seq &= GENMASK_ULL(31, 0);
759
760 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64);
761 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d",
762 data_fin_seq, mpext->dsn64);
763 }
764
765 /* Adjust for DATA_FIN using 1 byte of sequence space */
766 data_len--;
767 }
768
769 if (!mpext->dsn64) {
770 map_seq = expand_seq(subflow->map_seq, subflow->map_data_len,
771 mpext->data_seq);
772 subflow->use_64bit_ack = 0;
773 pr_debug("expanded seq=%llu", subflow->map_seq);
774 } else {
775 map_seq = mpext->data_seq;
776 subflow->use_64bit_ack = 1;
777 }
778
779 if (subflow->map_valid) {
780 /* Allow replacing only with an identical map */
781 if (subflow->map_seq == map_seq &&
782 subflow->map_subflow_seq == mpext->subflow_seq &&
783 subflow->map_data_len == data_len) {
784 skb_ext_del(skb, SKB_EXT_MPTCP);
785 return MAPPING_OK;
786 }
787
788 /* If this skb data are fully covered by the current mapping,
789 * the new map would need caching, which is not supported
790 */
791 if (skb_is_fully_mapped(ssk, skb)) {
792 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH);
793 return MAPPING_INVALID;
794 }
795
796 /* will validate the next map after consuming the current one */
797 return MAPPING_OK;
798 }
799
800 subflow->map_seq = map_seq;
801 subflow->map_subflow_seq = mpext->subflow_seq;
802 subflow->map_data_len = data_len;
803 subflow->map_valid = 1;
804 subflow->mpc_map = mpext->mpc_map;
805 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u",
806 subflow->map_seq, subflow->map_subflow_seq,
807 subflow->map_data_len);
808
809validate_seq:
810 /* we revalidate valid mapping on new skb, because we must ensure
811 * the current skb is completely covered by the available mapping
812 */
813 if (!validate_mapping(ssk, skb))
814 return MAPPING_INVALID;
815
816 skb_ext_del(skb, SKB_EXT_MPTCP);
817 return MAPPING_OK;
818}
819
820static int subflow_read_actor(read_descriptor_t *desc,
821 struct sk_buff *skb,
822 unsigned int offset, size_t len)
823{
824 size_t copy_len = min(desc->count, len);
825
826 desc->count -= copy_len;
827
828 pr_debug("flushed %zu bytes, %zu left", copy_len, desc->count);
829 return copy_len;
830}
831
832static bool subflow_check_data_avail(struct sock *ssk)
833{
834 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
835 enum mapping_status status;
836 struct mptcp_sock *msk;
837 struct sk_buff *skb;
838
839 pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow->conn, ssk,
840 subflow->data_avail, skb_peek(&ssk->sk_receive_queue));
841 if (subflow->data_avail)
842 return true;
843
844 msk = mptcp_sk(subflow->conn);
845 for (;;) {
846 u32 map_remaining;
847 size_t delta;
848 u64 ack_seq;
849 u64 old_ack;
850
851 status = get_mapping_status(ssk, msk);
852 pr_debug("msk=%p ssk=%p status=%d", msk, ssk, status);
853 if (status == MAPPING_INVALID) {
854 ssk->sk_err = EBADMSG;
855 goto fatal;
856 }
857 if (status == MAPPING_DUMMY) {
858 __mptcp_do_fallback(msk);
859 skb = skb_peek(&ssk->sk_receive_queue);
860 subflow->map_valid = 1;
861 subflow->map_seq = READ_ONCE(msk->ack_seq);
862 subflow->map_data_len = skb->len;
863 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq -
864 subflow->ssn_offset;
865 return true;
866 }
867
868 if (status != MAPPING_OK)
869 return false;
870
871 skb = skb_peek(&ssk->sk_receive_queue);
872 if (WARN_ON_ONCE(!skb))
873 return false;
874
875 /* if msk lacks the remote key, this subflow must provide an
876 * MP_CAPABLE-based mapping
877 */
878 if (unlikely(!READ_ONCE(msk->can_ack))) {
879 if (!subflow->mpc_map) {
880 ssk->sk_err = EBADMSG;
881 goto fatal;
882 }
883 WRITE_ONCE(msk->remote_key, subflow->remote_key);
884 WRITE_ONCE(msk->ack_seq, subflow->map_seq);
885 WRITE_ONCE(msk->can_ack, true);
886 }
887
888 old_ack = READ_ONCE(msk->ack_seq);
889 ack_seq = mptcp_subflow_get_mapped_dsn(subflow);
890 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack,
891 ack_seq);
892 if (ack_seq == old_ack)
893 break;
894
895 /* only accept in-sequence mapping. Old values are spurious
896 * retransmission; we can hit "future" values on active backup
897 * subflow switch, we relay on retransmissions to get
898 * in-sequence data.
899 * Cuncurrent subflows support will require subflow data
900 * reordering
901 */
902 map_remaining = subflow->map_data_len -
903 mptcp_subflow_get_map_offset(subflow);
904 if (before64(ack_seq, old_ack))
905 delta = min_t(size_t, old_ack - ack_seq, map_remaining);
906 else
907 delta = min_t(size_t, ack_seq - old_ack, map_remaining);
908
909 /* discard mapped data */
910 pr_debug("discarding %zu bytes, current map len=%d", delta,
911 map_remaining);
912 if (delta) {
913 read_descriptor_t desc = {
914 .count = delta,
915 };
916 int ret;
917
918 ret = tcp_read_sock(ssk, &desc, subflow_read_actor);
919 if (ret < 0) {
920 ssk->sk_err = -ret;
921 goto fatal;
922 }
923 if (ret < delta)
924 return false;
925 if (delta == map_remaining)
926 subflow->map_valid = 0;
927 }
928 }
929 return true;
930
931fatal:
932 /* fatal protocol error, close the socket */
933 /* This barrier is coupled with smp_rmb() in tcp_poll() */
934 smp_wmb();
935 ssk->sk_error_report(ssk);
936 tcp_set_state(ssk, TCP_CLOSE);
937 tcp_send_active_reset(ssk, GFP_ATOMIC);
938 return false;
939}
940
941bool mptcp_subflow_data_available(struct sock *sk)
942{
943 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
944 struct sk_buff *skb;
945
946 /* check if current mapping is still valid */
947 if (subflow->map_valid &&
948 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) {
949 subflow->map_valid = 0;
950 subflow->data_avail = 0;
951
952 pr_debug("Done with mapping: seq=%u data_len=%u",
953 subflow->map_subflow_seq,
954 subflow->map_data_len);
955 }
956
957 if (!subflow_check_data_avail(sk)) {
958 subflow->data_avail = 0;
959 return false;
960 }
961
962 skb = skb_peek(&sk->sk_receive_queue);
963 subflow->data_avail = skb &&
964 before(tcp_sk(sk)->copied_seq, TCP_SKB_CB(skb)->end_seq);
965 return subflow->data_avail;
966}
967
968/* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy,
969 * not the ssk one.
970 *
971 * In mptcp, rwin is about the mptcp-level connection data.
972 *
973 * Data that is still on the ssk rx queue can thus be ignored,
974 * as far as mptcp peer is concerened that data is still inflight.
975 * DSS ACK is updated when skb is moved to the mptcp rx queue.
976 */
977void mptcp_space(const struct sock *ssk, int *space, int *full_space)
978{
979 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
980 const struct sock *sk = subflow->conn;
981
982 *space = tcp_space(sk);
983 *full_space = tcp_full_space(sk);
984}
985
986static void subflow_data_ready(struct sock *sk)
987{
988 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
989 u16 state = 1 << inet_sk_state_load(sk);
990 struct sock *parent = subflow->conn;
991 struct mptcp_sock *msk;
992
993 msk = mptcp_sk(parent);
994 if (state & TCPF_LISTEN) {
995 set_bit(MPTCP_DATA_READY, &msk->flags);
996 parent->sk_data_ready(parent);
997 return;
998 }
999
1000 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable &&
1001 !subflow->mp_join && !(state & TCPF_CLOSE));
1002
1003 if (mptcp_subflow_data_available(sk))
1004 mptcp_data_ready(parent, sk);
1005}
1006
1007static void subflow_write_space(struct sock *sk)
1008{
1009 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1010 struct sock *parent = subflow->conn;
1011
1012 sk_stream_write_space(sk);
1013 if (sk_stream_is_writeable(sk)) {
1014 set_bit(MPTCP_SEND_SPACE, &mptcp_sk(parent)->flags);
1015 smp_mb__after_atomic();
1016 /* set SEND_SPACE before sk_stream_write_space clears NOSPACE */
1017 sk_stream_write_space(parent);
1018 }
1019}
1020
1021static struct inet_connection_sock_af_ops *
1022subflow_default_af_ops(struct sock *sk)
1023{
1024#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1025 if (sk->sk_family == AF_INET6)
1026 return &subflow_v6_specific;
1027#endif
1028 return &subflow_specific;
1029}
1030
1031#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1032void mptcpv6_handle_mapped(struct sock *sk, bool mapped)
1033{
1034 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1035 struct inet_connection_sock *icsk = inet_csk(sk);
1036 struct inet_connection_sock_af_ops *target;
1037
1038 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk);
1039
1040 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d",
1041 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped);
1042
1043 if (likely(icsk->icsk_af_ops == target))
1044 return;
1045
1046 subflow->icsk_af_ops = icsk->icsk_af_ops;
1047 icsk->icsk_af_ops = target;
1048}
1049#endif
1050
1051static void mptcp_info2sockaddr(const struct mptcp_addr_info *info,
1052 struct sockaddr_storage *addr)
1053{
1054 memset(addr, 0, sizeof(*addr));
1055 addr->ss_family = info->family;
1056 if (addr->ss_family == AF_INET) {
1057 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr;
1058
1059 in_addr->sin_addr = info->addr;
1060 in_addr->sin_port = info->port;
1061 }
1062#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1063 else if (addr->ss_family == AF_INET6) {
1064 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr;
1065
1066 in6_addr->sin6_addr = info->addr6;
1067 in6_addr->sin6_port = info->port;
1068 }
1069#endif
1070}
1071
1072int __mptcp_subflow_connect(struct sock *sk, int ifindex,
1073 const struct mptcp_addr_info *loc,
1074 const struct mptcp_addr_info *remote)
1075{
1076 struct mptcp_sock *msk = mptcp_sk(sk);
1077 struct mptcp_subflow_context *subflow;
1078 struct sockaddr_storage addr;
1079 int remote_id = remote->id;
1080 int local_id = loc->id;
1081 struct socket *sf;
1082 struct sock *ssk;
1083 u32 remote_token;
1084 int addrlen;
1085 int err;
1086
1087 if (!mptcp_is_fully_established(sk))
1088 return -ENOTCONN;
1089
1090 err = mptcp_subflow_create_socket(sk, &sf);
1091 if (err)
1092 return err;
1093
1094 ssk = sf->sk;
1095 subflow = mptcp_subflow_ctx(ssk);
1096 do {
1097 get_random_bytes(&subflow->local_nonce, sizeof(u32));
1098 } while (!subflow->local_nonce);
1099
1100 if (!local_id) {
1101 err = mptcp_pm_get_local_id(msk, (struct sock_common *)ssk);
1102 if (err < 0)
1103 goto failed;
1104
1105 local_id = err;
1106 }
1107
1108 subflow->remote_key = msk->remote_key;
1109 subflow->local_key = msk->local_key;
1110 subflow->token = msk->token;
1111 mptcp_info2sockaddr(loc, &addr);
1112
1113 addrlen = sizeof(struct sockaddr_in);
1114#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1115 if (loc->family == AF_INET6)
1116 addrlen = sizeof(struct sockaddr_in6);
1117#endif
1118 ssk->sk_bound_dev_if = ifindex;
1119 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen);
1120 if (err)
1121 goto failed;
1122
1123 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL);
1124 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d", msk,
1125 remote_token, local_id, remote_id);
1126 subflow->remote_token = remote_token;
1127 subflow->local_id = local_id;
1128 subflow->remote_id = remote_id;
1129 subflow->request_join = 1;
1130 subflow->request_bkup = 1;
1131 mptcp_info2sockaddr(remote, &addr);
1132
1133 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK);
1134 if (err && err != -EINPROGRESS)
1135 goto failed;
1136
1137 spin_lock_bh(&msk->join_list_lock);
1138 list_add_tail(&subflow->node, &msk->join_list);
1139 spin_unlock_bh(&msk->join_list_lock);
1140
1141 return err;
1142
1143failed:
1144 sock_release(sf);
1145 return err;
1146}
1147
1148int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock)
1149{
1150 struct mptcp_subflow_context *subflow;
1151 struct net *net = sock_net(sk);
1152 struct socket *sf;
1153 int err;
1154
1155 /* un-accepted server sockets can reach here - on bad configuration
1156 * bail early to avoid greater trouble later
1157 */
1158 if (unlikely(!sk->sk_socket))
1159 return -EINVAL;
1160
1161 err = sock_create_kern(net, sk->sk_family, SOCK_STREAM, IPPROTO_TCP,
1162 &sf);
1163 if (err)
1164 return err;
1165
1166 lock_sock(sf->sk);
1167
1168 /* kernel sockets do not by default acquire net ref, but TCP timer
1169 * needs it.
1170 */
1171 sf->sk->sk_net_refcnt = 1;
1172 get_net(net);
1173#ifdef CONFIG_PROC_FS
1174 this_cpu_add(*net->core.sock_inuse, 1);
1175#endif
1176 err = tcp_set_ulp(sf->sk, "mptcp");
1177 release_sock(sf->sk);
1178
1179 if (err) {
1180 sock_release(sf);
1181 return err;
1182 }
1183
1184 /* the newly created socket really belongs to the owning MPTCP master
1185 * socket, even if for additional subflows the allocation is performed
1186 * by a kernel workqueue. Adjust inode references, so that the
1187 * procfs/diag interaces really show this one belonging to the correct
1188 * user.
1189 */
1190 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino;
1191 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid;
1192 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid;
1193
1194 subflow = mptcp_subflow_ctx(sf->sk);
1195 pr_debug("subflow=%p", subflow);
1196
1197 *new_sock = sf;
1198 sock_hold(sk);
1199 subflow->conn = sk;
1200
1201 return 0;
1202}
1203
1204static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk,
1205 gfp_t priority)
1206{
1207 struct inet_connection_sock *icsk = inet_csk(sk);
1208 struct mptcp_subflow_context *ctx;
1209
1210 ctx = kzalloc(sizeof(*ctx), priority);
1211 if (!ctx)
1212 return NULL;
1213
1214 rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
1215 INIT_LIST_HEAD(&ctx->node);
1216
1217 pr_debug("subflow=%p", ctx);
1218
1219 ctx->tcp_sock = sk;
1220
1221 return ctx;
1222}
1223
1224static void __subflow_state_change(struct sock *sk)
1225{
1226 struct socket_wq *wq;
1227
1228 rcu_read_lock();
1229 wq = rcu_dereference(sk->sk_wq);
1230 if (skwq_has_sleeper(wq))
1231 wake_up_interruptible_all(&wq->wait);
1232 rcu_read_unlock();
1233}
1234
1235static bool subflow_is_done(const struct sock *sk)
1236{
1237 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE;
1238}
1239
1240static void subflow_state_change(struct sock *sk)
1241{
1242 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1243 struct sock *parent = subflow->conn;
1244
1245 __subflow_state_change(sk);
1246
1247 if (subflow_simultaneous_connect(sk)) {
1248 mptcp_do_fallback(sk);
1249 mptcp_rcv_space_init(mptcp_sk(parent), sk);
1250 pr_fallback(mptcp_sk(parent));
1251 subflow->conn_finished = 1;
1252 if (inet_sk_state_load(parent) == TCP_SYN_SENT) {
1253 inet_sk_state_store(parent, TCP_ESTABLISHED);
1254 parent->sk_state_change(parent);
1255 }
1256 }
1257
1258 /* as recvmsg() does not acquire the subflow socket for ssk selection
1259 * a fin packet carrying a DSS can be unnoticed if we don't trigger
1260 * the data available machinery here.
1261 */
1262 if (mptcp_subflow_data_available(sk))
1263 mptcp_data_ready(parent, sk);
1264
1265 if (__mptcp_check_fallback(mptcp_sk(parent)) &&
1266 !(parent->sk_shutdown & RCV_SHUTDOWN) &&
1267 !subflow->rx_eof && subflow_is_done(sk)) {
1268 subflow->rx_eof = 1;
1269 mptcp_subflow_eof(parent);
1270 }
1271}
1272
1273static int subflow_ulp_init(struct sock *sk)
1274{
1275 struct inet_connection_sock *icsk = inet_csk(sk);
1276 struct mptcp_subflow_context *ctx;
1277 struct tcp_sock *tp = tcp_sk(sk);
1278 int err = 0;
1279
1280 /* disallow attaching ULP to a socket unless it has been
1281 * created with sock_create_kern()
1282 */
1283 if (!sk->sk_kern_sock) {
1284 err = -EOPNOTSUPP;
1285 goto out;
1286 }
1287
1288 ctx = subflow_create_ctx(sk, GFP_KERNEL);
1289 if (!ctx) {
1290 err = -ENOMEM;
1291 goto out;
1292 }
1293
1294 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family);
1295
1296 tp->is_mptcp = 1;
1297 ctx->icsk_af_ops = icsk->icsk_af_ops;
1298 icsk->icsk_af_ops = subflow_default_af_ops(sk);
1299 ctx->tcp_data_ready = sk->sk_data_ready;
1300 ctx->tcp_state_change = sk->sk_state_change;
1301 ctx->tcp_write_space = sk->sk_write_space;
1302 sk->sk_data_ready = subflow_data_ready;
1303 sk->sk_write_space = subflow_write_space;
1304 sk->sk_state_change = subflow_state_change;
1305out:
1306 return err;
1307}
1308
1309static void subflow_ulp_release(struct sock *sk)
1310{
1311 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(sk);
1312
1313 if (!ctx)
1314 return;
1315
1316 if (ctx->conn)
1317 sock_put(ctx->conn);
1318
1319 kfree_rcu(ctx, rcu);
1320}
1321
1322static void subflow_ulp_clone(const struct request_sock *req,
1323 struct sock *newsk,
1324 const gfp_t priority)
1325{
1326 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
1327 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk);
1328 struct mptcp_subflow_context *new_ctx;
1329
1330 if (!tcp_rsk(req)->is_mptcp ||
1331 (!subflow_req->mp_capable && !subflow_req->mp_join)) {
1332 subflow_ulp_fallback(newsk, old_ctx);
1333 return;
1334 }
1335
1336 new_ctx = subflow_create_ctx(newsk, priority);
1337 if (!new_ctx) {
1338 subflow_ulp_fallback(newsk, old_ctx);
1339 return;
1340 }
1341
1342 new_ctx->conn_finished = 1;
1343 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops;
1344 new_ctx->tcp_data_ready = old_ctx->tcp_data_ready;
1345 new_ctx->tcp_state_change = old_ctx->tcp_state_change;
1346 new_ctx->tcp_write_space = old_ctx->tcp_write_space;
1347 new_ctx->rel_write_seq = 1;
1348 new_ctx->tcp_sock = newsk;
1349
1350 if (subflow_req->mp_capable) {
1351 /* see comments in subflow_syn_recv_sock(), MPTCP connection
1352 * is fully established only after we receive the remote key
1353 */
1354 new_ctx->mp_capable = 1;
1355 new_ctx->local_key = subflow_req->local_key;
1356 new_ctx->token = subflow_req->token;
1357 new_ctx->ssn_offset = subflow_req->ssn_offset;
1358 new_ctx->idsn = subflow_req->idsn;
1359 } else if (subflow_req->mp_join) {
1360 new_ctx->ssn_offset = subflow_req->ssn_offset;
1361 new_ctx->mp_join = 1;
1362 new_ctx->fully_established = 1;
1363 new_ctx->backup = subflow_req->backup;
1364 new_ctx->local_id = subflow_req->local_id;
1365 new_ctx->remote_id = subflow_req->remote_id;
1366 new_ctx->token = subflow_req->token;
1367 new_ctx->thmac = subflow_req->thmac;
1368 }
1369}
1370
1371static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = {
1372 .name = "mptcp",
1373 .owner = THIS_MODULE,
1374 .init = subflow_ulp_init,
1375 .release = subflow_ulp_release,
1376 .clone = subflow_ulp_clone,
1377};
1378
1379static int subflow_ops_init(struct request_sock_ops *subflow_ops)
1380{
1381 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock);
1382 subflow_ops->slab_name = "request_sock_subflow";
1383
1384 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name,
1385 subflow_ops->obj_size, 0,
1386 SLAB_ACCOUNT |
1387 SLAB_TYPESAFE_BY_RCU,
1388 NULL);
1389 if (!subflow_ops->slab)
1390 return -ENOMEM;
1391
1392 subflow_ops->destructor = subflow_req_destructor;
1393
1394 return 0;
1395}
1396
1397void __init mptcp_subflow_init(void)
1398{
1399 mptcp_subflow_request_sock_ops = tcp_request_sock_ops;
1400 if (subflow_ops_init(&mptcp_subflow_request_sock_ops) != 0)
1401 panic("MPTCP: failed to init subflow request sock ops\n");
1402
1403 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops;
1404 subflow_request_sock_ipv4_ops.init_req = subflow_v4_init_req;
1405
1406 subflow_specific = ipv4_specific;
1407 subflow_specific.conn_request = subflow_v4_conn_request;
1408 subflow_specific.syn_recv_sock = subflow_syn_recv_sock;
1409 subflow_specific.sk_rx_dst_set = subflow_finish_connect;
1410
1411#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1412 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops;
1413 subflow_request_sock_ipv6_ops.init_req = subflow_v6_init_req;
1414
1415 subflow_v6_specific = ipv6_specific;
1416 subflow_v6_specific.conn_request = subflow_v6_conn_request;
1417 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock;
1418 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect;
1419
1420 subflow_v6m_specific = subflow_v6_specific;
1421 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit;
1422 subflow_v6m_specific.send_check = ipv4_specific.send_check;
1423 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len;
1424 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced;
1425 subflow_v6m_specific.net_frag_header_len = 0;
1426#endif
1427
1428 mptcp_diag_subflow_init(&subflow_ulp_ops);
1429
1430 if (tcp_register_ulp(&subflow_ulp_ops) != 0)
1431 panic("MPTCP: failed to register subflows to ULP\n");
1432}