1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 *	TCP over IPv6
   4 *	Linux INET6 implementation
   5 *
   6 *	Authors:
   7 *	Pedro Roque		<roque@di.fc.ul.pt>
   8 *
   9 *	Based on:
  10 *	linux/net/ipv4/tcp.c
  11 *	linux/net/ipv4/tcp_input.c
  12 *	linux/net/ipv4/tcp_output.c
  13 *
  14 *	Fixes:
  15 *	Hideaki YOSHIFUJI	:	sin6_scope_id support
  16 *	YOSHIFUJI Hideaki @USAGI and:	Support IPV6_V6ONLY socket option, which
  17 *	Alexey Kuznetsov		allow both IPv4 and IPv6 sockets to bind
  18 *					a single port at the same time.
  19 *	YOSHIFUJI Hideaki @USAGI:	convert /proc/net/tcp6 to seq_file.
  20 */
  21
  22#include <linux/bottom_half.h>
  23#include <linux/module.h>
  24#include <linux/errno.h>
  25#include <linux/types.h>
  26#include <linux/socket.h>
  27#include <linux/sockios.h>
  28#include <linux/net.h>
  29#include <linux/jiffies.h>
  30#include <linux/in.h>
  31#include <linux/in6.h>
  32#include <linux/netdevice.h>
  33#include <linux/init.h>
  34#include <linux/jhash.h>
  35#include <linux/ipsec.h>
  36#include <linux/times.h>
  37#include <linux/slab.h>
  38#include <linux/uaccess.h>
  39#include <linux/ipv6.h>
  40#include <linux/icmpv6.h>
  41#include <linux/random.h>
  42#include <linux/indirect_call_wrapper.h>
  43
  44#include <net/tcp.h>
  45#include <net/ndisc.h>
  46#include <net/inet6_hashtables.h>
  47#include <net/inet6_connection_sock.h>
  48#include <net/ipv6.h>
  49#include <net/transp_v6.h>
  50#include <net/addrconf.h>
  51#include <net/ip6_route.h>
  52#include <net/ip6_checksum.h>
  53#include <net/inet_ecn.h>
  54#include <net/protocol.h>
  55#include <net/xfrm.h>
  56#include <net/snmp.h>
  57#include <net/dsfield.h>
  58#include <net/timewait_sock.h>
  59#include <net/inet_common.h>
  60#include <net/secure_seq.h>
  61#include <net/hotdata.h>
  62#include <net/busy_poll.h>
  63#include <net/rstreason.h>
  64
  65#include <linux/proc_fs.h>
  66#include <linux/seq_file.h>
  67
  68#include <crypto/hash.h>
  69#include <linux/scatterlist.h>
  70
  71#include <trace/events/tcp.h>
  72
  73static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
  74			      enum sk_rst_reason reason);
  75static void	tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
  76				      struct request_sock *req);
  77
  78INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
  79
  80static const struct inet_connection_sock_af_ops ipv6_mapped;
  81const struct inet_connection_sock_af_ops ipv6_specific;
  82#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
  83static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
  84static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
  85#endif
  86
  87/* Helper returning the inet6 address from a given tcp socket.
  88 * It can be used in TCP stack instead of inet6_sk(sk).
  89 * This avoids a dereference and allow compiler optimizations.
  90 * It is a specialized version of inet6_sk_generic().
  91 */
  92#define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
  93					      struct tcp6_sock, tcp)->inet6)
  94
  95static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
  96{
  97	struct dst_entry *dst = skb_dst(skb);
  98
  99	if (dst && dst_hold_safe(dst)) {
 100		rcu_assign_pointer(sk->sk_rx_dst, dst);
 101		sk->sk_rx_dst_ifindex = skb->skb_iif;
 102		sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst));
 103	}
 104}
 105
 106static u32 tcp_v6_init_seq(const struct sk_buff *skb)
 107{
 108	return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32,
 109				ipv6_hdr(skb)->saddr.s6_addr32,
 110				tcp_hdr(skb)->dest,
 111				tcp_hdr(skb)->source);
 112}
 113
 114static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb)
 115{
 116	return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32,
 117				   ipv6_hdr(skb)->saddr.s6_addr32);
 118}
 119
 120static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
 121			      int addr_len)
 122{
 123	/* This check is replicated from tcp_v6_connect() and intended to
 124	 * prevent BPF program called below from accessing bytes that are out
 125	 * of the bound specified by user in addr_len.
 126	 */
 127	if (addr_len < SIN6_LEN_RFC2133)
 128		return -EINVAL;
 129
 130	sock_owned_by_me(sk);
 131
 132	return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
 133}
 134
 135static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 136			  int addr_len)
 137{
 138	struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
 139	struct inet_connection_sock *icsk = inet_csk(sk);
 140	struct in6_addr *saddr = NULL, *final_p, final;
 141	struct inet_timewait_death_row *tcp_death_row;
 142	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 143	struct inet_sock *inet = inet_sk(sk);
 144	struct tcp_sock *tp = tcp_sk(sk);
 145	struct net *net = sock_net(sk);
 146	struct ipv6_txoptions *opt;
 147	struct dst_entry *dst;
 148	struct flowi6 fl6;
 149	int addr_type;
 150	int err;
 151
 152	if (addr_len < SIN6_LEN_RFC2133)
 153		return -EINVAL;
 154
 155	if (usin->sin6_family != AF_INET6)
 156		return -EAFNOSUPPORT;
 157
 158	memset(&fl6, 0, sizeof(fl6));
 159
 160	if (inet6_test_bit(SNDFLOW, sk)) {
 161		fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
 162		IP6_ECN_flow_init(fl6.flowlabel);
 163		if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
 164			struct ip6_flowlabel *flowlabel;
 165			flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
 166			if (IS_ERR(flowlabel))
 167				return -EINVAL;
 168			fl6_sock_release(flowlabel);
 169		}
 170	}
 171
 172	/*
 173	 *	connect() to INADDR_ANY means loopback (BSD'ism).
 174	 */
 175
 176	if (ipv6_addr_any(&usin->sin6_addr)) {
 177		if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
 178			ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
 179					       &usin->sin6_addr);
 180		else
 181			usin->sin6_addr = in6addr_loopback;
 182	}
 183
 184	addr_type = ipv6_addr_type(&usin->sin6_addr);
 185
 186	if (addr_type & IPV6_ADDR_MULTICAST)
 187		return -ENETUNREACH;
 188
 189	if (addr_type&IPV6_ADDR_LINKLOCAL) {
 190		if (addr_len >= sizeof(struct sockaddr_in6) &&
 191		    usin->sin6_scope_id) {
 192			/* If interface is set while binding, indices
 193			 * must coincide.
 194			 */
 195			if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
 196				return -EINVAL;
 197
 198			sk->sk_bound_dev_if = usin->sin6_scope_id;
 199		}
 200
 201		/* Connect to link-local address requires an interface */
 202		if (!sk->sk_bound_dev_if)
 203			return -EINVAL;
 204	}
 205
 206	if (tp->rx_opt.ts_recent_stamp &&
 207	    !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
 208		tp->rx_opt.ts_recent = 0;
 209		tp->rx_opt.ts_recent_stamp = 0;
 210		WRITE_ONCE(tp->write_seq, 0);
 211	}
 212
 213	sk->sk_v6_daddr = usin->sin6_addr;
 214	np->flow_label = fl6.flowlabel;
 215
 216	/*
 217	 *	TCP over IPv4
 218	 */
 219
 220	if (addr_type & IPV6_ADDR_MAPPED) {
 221		u32 exthdrlen = icsk->icsk_ext_hdr_len;
 222		struct sockaddr_in sin;
 223
 224		if (ipv6_only_sock(sk))
 225			return -ENETUNREACH;
 226
 227		sin.sin_family = AF_INET;
 228		sin.sin_port = usin->sin6_port;
 229		sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
 230
 231		/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 232		WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
 233		if (sk_is_mptcp(sk))
 234			mptcpv6_handle_mapped(sk, true);
 235		sk->sk_backlog_rcv = tcp_v4_do_rcv;
 236#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 237		tp->af_specific = &tcp_sock_ipv6_mapped_specific;
 238#endif
 239
 240		err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin));
 241
 242		if (err) {
 243			icsk->icsk_ext_hdr_len = exthdrlen;
 244			/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 245			WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
 246			if (sk_is_mptcp(sk))
 247				mptcpv6_handle_mapped(sk, false);
 248			sk->sk_backlog_rcv = tcp_v6_do_rcv;
 249#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 250			tp->af_specific = &tcp_sock_ipv6_specific;
 251#endif
 252			goto failure;
 253		}
 254		np->saddr = sk->sk_v6_rcv_saddr;
 255
 256		return err;
 257	}
 258
 259	if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
 260		saddr = &sk->sk_v6_rcv_saddr;
 261
 262	fl6.flowi6_proto = IPPROTO_TCP;
 263	fl6.daddr = sk->sk_v6_daddr;
 264	fl6.saddr = saddr ? *saddr : np->saddr;
 265	fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
 266	fl6.flowi6_oif = sk->sk_bound_dev_if;
 267	fl6.flowi6_mark = sk->sk_mark;
 268	fl6.fl6_dport = usin->sin6_port;
 269	fl6.fl6_sport = inet->inet_sport;
 270	fl6.flowi6_uid = sk->sk_uid;
 271
 272	opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
 273	final_p = fl6_update_dst(&fl6, opt, &final);
 274
 275	security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
 276
 277	dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p);
 278	if (IS_ERR(dst)) {
 279		err = PTR_ERR(dst);
 280		goto failure;
 281	}
 282
 283	tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
 284	tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
 285
 286	if (!saddr) {
 287		saddr = &fl6.saddr;
 288
 289		err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
 290		if (err)
 291			goto failure;
 292	}
 293
 294	/* set the source address */
 295	np->saddr = *saddr;
 296	inet->inet_rcv_saddr = LOOPBACK4_IPV6;
 297
 298	sk->sk_gso_type = SKB_GSO_TCPV6;
 299	ip6_dst_store(sk, dst, NULL, NULL);
 300
 301	icsk->icsk_ext_hdr_len = 0;
 302	if (opt)
 303		icsk->icsk_ext_hdr_len = opt->opt_flen +
 304					 opt->opt_nflen;
 305
 306	tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
 307
 308	inet->inet_dport = usin->sin6_port;
 309
 310	tcp_set_state(sk, TCP_SYN_SENT);
 311	err = inet6_hash_connect(tcp_death_row, sk);
 312	if (err)
 313		goto late_failure;
 314
 315	sk_set_txhash(sk);
 316
 317	if (likely(!tp->repair)) {
 318		if (!tp->write_seq)
 319			WRITE_ONCE(tp->write_seq,
 320				   secure_tcpv6_seq(np->saddr.s6_addr32,
 321						    sk->sk_v6_daddr.s6_addr32,
 322						    inet->inet_sport,
 323						    inet->inet_dport));
 324		tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32,
 325						   sk->sk_v6_daddr.s6_addr32);
 326	}
 327
 328	if (tcp_fastopen_defer_connect(sk, &err))
 329		return err;
 330	if (err)
 331		goto late_failure;
 332
 333	err = tcp_connect(sk);
 334	if (err)
 335		goto late_failure;
 336
 337	return 0;
 338
 339late_failure:
 340	tcp_set_state(sk, TCP_CLOSE);
 341	inet_bhash2_reset_saddr(sk);
 342failure:
 343	inet->inet_dport = 0;
 344	sk->sk_route_caps = 0;
 345	return err;
 346}
 347
 348static void tcp_v6_mtu_reduced(struct sock *sk)
 349{
 350	struct dst_entry *dst;
 351	u32 mtu;
 352
 353	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
 354		return;
 355
 356	mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
 357
 358	/* Drop requests trying to increase our current mss.
 359	 * Check done in __ip6_rt_update_pmtu() is too late.
 360	 */
 361	if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache)
 362		return;
 363
 364	dst = inet6_csk_update_pmtu(sk, mtu);
 365	if (!dst)
 366		return;
 367
 368	if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
 369		tcp_sync_mss(sk, dst_mtu(dst));
 370		tcp_simple_retransmit(sk);
 371	}
 372}
 373
 374static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 375		u8 type, u8 code, int offset, __be32 info)
 376{
 377	const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
 378	const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
 379	struct net *net = dev_net(skb->dev);
 380	struct request_sock *fastopen;
 381	struct ipv6_pinfo *np;
 382	struct tcp_sock *tp;
 383	__u32 seq, snd_una;
 384	struct sock *sk;
 385	bool fatal;
 386	int err;
 387
 388	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 389					&hdr->daddr, th->dest,
 390					&hdr->saddr, ntohs(th->source),
 391					skb->dev->ifindex, inet6_sdif(skb));
 392
 393	if (!sk) {
 394		__ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
 395				  ICMP6_MIB_INERRORS);
 396		return -ENOENT;
 397	}
 398
 399	if (sk->sk_state == TCP_TIME_WAIT) {
 400		/* To increase the counter of ignored icmps for TCP-AO */
 401		tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
 402		inet_twsk_put(inet_twsk(sk));
 403		return 0;
 404	}
 405	seq = ntohl(th->seq);
 406	fatal = icmpv6_err_convert(type, code, &err);
 407	if (sk->sk_state == TCP_NEW_SYN_RECV) {
 408		tcp_req_err(sk, seq, fatal);
 409		return 0;
 410	}
 411
 412	if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
 413		sock_put(sk);
 414		return 0;
 415	}
 416
 417	bh_lock_sock(sk);
 418	if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
 419		__NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
 420
 421	if (sk->sk_state == TCP_CLOSE)
 422		goto out;
 423
 424	if (static_branch_unlikely(&ip6_min_hopcount)) {
 425		/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
 426		if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
 427			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
 428			goto out;
 429		}
 430	}
 431
 432	tp = tcp_sk(sk);
 433	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
 434	fastopen = rcu_dereference(tp->fastopen_rsk);
 435	snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
 436	if (sk->sk_state != TCP_LISTEN &&
 437	    !between(seq, snd_una, tp->snd_nxt)) {
 438		__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 439		goto out;
 440	}
 441
 442	np = tcp_inet6_sk(sk);
 443
 444	if (type == NDISC_REDIRECT) {
 445		if (!sock_owned_by_user(sk)) {
 446			struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);
 447
 448			if (dst)
 449				dst->ops->redirect(dst, sk, skb);
 450		}
 451		goto out;
 452	}
 453
 454	if (type == ICMPV6_PKT_TOOBIG) {
 455		u32 mtu = ntohl(info);
 456
 457		/* We are not interested in TCP_LISTEN and open_requests
 458		 * (SYN-ACKs send out by Linux are always <576bytes so
 459		 * they should go through unfragmented).
 460		 */
 461		if (sk->sk_state == TCP_LISTEN)
 462			goto out;
 463
 464		if (!ip6_sk_accept_pmtu(sk))
 465			goto out;
 466
 467		if (mtu < IPV6_MIN_MTU)
 468			goto out;
 469
 470		WRITE_ONCE(tp->mtu_info, mtu);
 471
 472		if (!sock_owned_by_user(sk))
 473			tcp_v6_mtu_reduced(sk);
 474		else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
 475					   &sk->sk_tsq_flags))
 476			sock_hold(sk);
 477		goto out;
 478	}
 479
 480
 481	/* Might be for an request_sock */
 482	switch (sk->sk_state) {
 483	case TCP_SYN_SENT:
 484	case TCP_SYN_RECV:
 485		/* Only in fast or simultaneous open. If a fast open socket is
 486		 * already accepted it is treated as a connected one below.
 487		 */
 488		if (fastopen && !fastopen->sk)
 489			break;
 490
 491		ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
 492
 493		if (!sock_owned_by_user(sk))
 494			tcp_done_with_error(sk, err);
 495		else
 
 
 
 496			WRITE_ONCE(sk->sk_err_soft, err);
 
 497		goto out;
 498	case TCP_LISTEN:
 499		break;
 500	default:
 501		/* check if this ICMP message allows revert of backoff.
 502		 * (see RFC 6069)
 503		 */
 504		if (!fastopen && type == ICMPV6_DEST_UNREACH &&
 505		    code == ICMPV6_NOROUTE)
 506			tcp_ld_RTO_revert(sk, seq);
 507	}
 508
 509	if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
 510		WRITE_ONCE(sk->sk_err, err);
 511		sk_error_report(sk);
 512	} else {
 513		WRITE_ONCE(sk->sk_err_soft, err);
 514	}
 515out:
 516	bh_unlock_sock(sk);
 517	sock_put(sk);
 518	return 0;
 519}
 520
 521
 522static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
 523			      struct flowi *fl,
 524			      struct request_sock *req,
 525			      struct tcp_fastopen_cookie *foc,
 526			      enum tcp_synack_type synack_type,
 527			      struct sk_buff *syn_skb)
 528{
 529	struct inet_request_sock *ireq = inet_rsk(req);
 530	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 531	struct ipv6_txoptions *opt;
 532	struct flowi6 *fl6 = &fl->u.ip6;
 533	struct sk_buff *skb;
 534	int err = -ENOMEM;
 535	u8 tclass;
 536
 537	/* First, grab a route. */
 538	if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
 539					       IPPROTO_TCP)) == NULL)
 540		goto done;
 541
 542	skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
 543
 544	if (skb) {
 545		__tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
 546				    &ireq->ir_v6_rmt_addr);
 547
 548		fl6->daddr = ireq->ir_v6_rmt_addr;
 549		if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
 550			fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));
 551
 552		tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
 553				(tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
 554				(np->tclass & INET_ECN_MASK) :
 555				np->tclass;
 556
 557		if (!INET_ECN_is_capable(tclass) &&
 558		    tcp_bpf_ca_needs_ecn((struct sock *)req))
 559			tclass |= INET_ECN_ECT_0;
 560
 561		rcu_read_lock();
 562		opt = ireq->ipv6_opt;
 563		if (!opt)
 564			opt = rcu_dereference(np->opt);
 565		err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark),
 566			       opt, tclass, READ_ONCE(sk->sk_priority));
 567		rcu_read_unlock();
 568		err = net_xmit_eval(err);
 569	}
 570
 571done:
 572	return err;
 573}
 574
 575
 576static void tcp_v6_reqsk_destructor(struct request_sock *req)
 577{
 578	kfree(inet_rsk(req)->ipv6_opt);
 579	consume_skb(inet_rsk(req)->pktopts);
 580}
 581
 582#ifdef CONFIG_TCP_MD5SIG
 583static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
 584						   const struct in6_addr *addr,
 585						   int l3index)
 586{
 587	return tcp_md5_do_lookup(sk, l3index,
 588				 (union tcp_md5_addr *)addr, AF_INET6);
 589}
 590
 591static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
 592						const struct sock *addr_sk)
 593{
 594	int l3index;
 595
 596	l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 597						 addr_sk->sk_bound_dev_if);
 598	return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
 599				    l3index);
 600}
 601
 602static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
 603				 sockptr_t optval, int optlen)
 604{
 605	struct tcp_md5sig cmd;
 606	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
 607	union tcp_ao_addr *addr;
 608	int l3index = 0;
 609	u8 prefixlen;
 610	bool l3flag;
 611	u8 flags;
 612
 613	if (optlen < sizeof(cmd))
 614		return -EINVAL;
 615
 616	if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
 617		return -EFAULT;
 618
 619	if (sin6->sin6_family != AF_INET6)
 620		return -EINVAL;
 621
 622	flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 623	l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 624
 625	if (optname == TCP_MD5SIG_EXT &&
 626	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
 627		prefixlen = cmd.tcpm_prefixlen;
 628		if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
 629					prefixlen > 32))
 630			return -EINVAL;
 631	} else {
 632		prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
 633	}
 634
 635	if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
 636	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
 637		struct net_device *dev;
 638
 639		rcu_read_lock();
 640		dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
 641		if (dev && netif_is_l3_master(dev))
 642			l3index = dev->ifindex;
 643		rcu_read_unlock();
 644
 645		/* ok to reference set/not set outside of rcu;
 646		 * right now device MUST be an L3 master
 647		 */
 648		if (!dev || !l3index)
 649			return -EINVAL;
 650	}
 651
 652	if (!cmd.tcpm_keylen) {
 653		if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 654			return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 655					      AF_INET, prefixlen,
 656					      l3index, flags);
 657		return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 658				      AF_INET6, prefixlen, l3index, flags);
 659	}
 660
 661	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
 662		return -EINVAL;
 663
 664	if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
 665		addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];
 666
 667		/* Don't allow keys for peers that have a matching TCP-AO key.
 668		 * See the comment in tcp_ao_add_cmd()
 669		 */
 670		if (tcp_ao_required(sk, addr, AF_INET,
 671				    l3flag ? l3index : -1, false))
 672			return -EKEYREJECTED;
 673		return tcp_md5_do_add(sk, addr,
 674				      AF_INET, prefixlen, l3index, flags,
 675				      cmd.tcpm_key, cmd.tcpm_keylen);
 676	}
 677
 678	addr = (union tcp_md5_addr *)&sin6->sin6_addr;
 679
 680	/* Don't allow keys for peers that have a matching TCP-AO key.
 681	 * See the comment in tcp_ao_add_cmd()
 682	 */
 683	if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false))
 684		return -EKEYREJECTED;
 685
 686	return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
 687			      cmd.tcpm_key, cmd.tcpm_keylen);
 688}
 689
 690static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp,
 691				   const struct in6_addr *daddr,
 692				   const struct in6_addr *saddr,
 693				   const struct tcphdr *th, int nbytes)
 694{
 695	struct tcp6_pseudohdr *bp;
 696	struct scatterlist sg;
 697	struct tcphdr *_th;
 698
 699	bp = hp->scratch;
 700	/* 1. TCP pseudo-header (RFC2460) */
 701	bp->saddr = *saddr;
 702	bp->daddr = *daddr;
 703	bp->protocol = cpu_to_be32(IPPROTO_TCP);
 704	bp->len = cpu_to_be32(nbytes);
 705
 706	_th = (struct tcphdr *)(bp + 1);
 707	memcpy(_th, th, sizeof(*th));
 708	_th->check = 0;
 709
 710	sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
 711	ahash_request_set_crypt(hp->req, &sg, NULL,
 712				sizeof(*bp) + sizeof(*th));
 713	return crypto_ahash_update(hp->req);
 714}
 715
 716static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
 717			       const struct in6_addr *daddr, struct in6_addr *saddr,
 718			       const struct tcphdr *th)
 719{
 720	struct tcp_sigpool hp;
 721
 722	if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 723		goto clear_hash_nostart;
 724
 725	if (crypto_ahash_init(hp.req))
 726		goto clear_hash;
 727	if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2))
 728		goto clear_hash;
 729	if (tcp_md5_hash_key(&hp, key))
 730		goto clear_hash;
 731	ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 732	if (crypto_ahash_final(hp.req))
 733		goto clear_hash;
 734
 735	tcp_sigpool_end(&hp);
 736	return 0;
 737
 738clear_hash:
 739	tcp_sigpool_end(&hp);
 740clear_hash_nostart:
 741	memset(md5_hash, 0, 16);
 742	return 1;
 743}
 744
 745static int tcp_v6_md5_hash_skb(char *md5_hash,
 746			       const struct tcp_md5sig_key *key,
 747			       const struct sock *sk,
 748			       const struct sk_buff *skb)
 749{
 750	const struct tcphdr *th = tcp_hdr(skb);
 751	const struct in6_addr *saddr, *daddr;
 752	struct tcp_sigpool hp;
 753
 754	if (sk) { /* valid for establish/request sockets */
 755		saddr = &sk->sk_v6_rcv_saddr;
 756		daddr = &sk->sk_v6_daddr;
 757	} else {
 758		const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 759		saddr = &ip6h->saddr;
 760		daddr = &ip6h->daddr;
 761	}
 762
 763	if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 764		goto clear_hash_nostart;
 765
 766	if (crypto_ahash_init(hp.req))
 767		goto clear_hash;
 768
 769	if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len))
 770		goto clear_hash;
 771	if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
 772		goto clear_hash;
 773	if (tcp_md5_hash_key(&hp, key))
 774		goto clear_hash;
 775	ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 776	if (crypto_ahash_final(hp.req))
 777		goto clear_hash;
 778
 779	tcp_sigpool_end(&hp);
 780	return 0;
 781
 782clear_hash:
 783	tcp_sigpool_end(&hp);
 784clear_hash_nostart:
 785	memset(md5_hash, 0, 16);
 786	return 1;
 787}
 788#endif
 789
 790static void tcp_v6_init_req(struct request_sock *req,
 791			    const struct sock *sk_listener,
 792			    struct sk_buff *skb,
 793			    u32 tw_isn)
 794{
 795	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
 796	struct inet_request_sock *ireq = inet_rsk(req);
 797	const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
 798
 799	ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
 800	ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
 801
 802	/* So that link locals have meaning */
 803	if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
 804	    ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
 805		ireq->ir_iif = tcp_v6_iif(skb);
 806
 807	if (!tw_isn &&
 808	    (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
 809	     np->rxopt.bits.rxinfo ||
 810	     np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
 811	     np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
 812		refcount_inc(&skb->users);
 813		ireq->pktopts = skb;
 814	}
 815}
 816
 817static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
 818					  struct sk_buff *skb,
 819					  struct flowi *fl,
 820					  struct request_sock *req,
 821					  u32 tw_isn)
 822{
 823	tcp_v6_init_req(req, sk, skb, tw_isn);
 824
 825	if (security_inet_conn_request(sk, skb, req))
 826		return NULL;
 827
 828	return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP);
 829}
 830
 831struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
 832	.family		=	AF_INET6,
 833	.obj_size	=	sizeof(struct tcp6_request_sock),
 834	.rtx_syn_ack	=	tcp_rtx_synack,
 835	.send_ack	=	tcp_v6_reqsk_send_ack,
 836	.destructor	=	tcp_v6_reqsk_destructor,
 837	.send_reset	=	tcp_v6_send_reset,
 838	.syn_ack_timeout =	tcp_syn_ack_timeout,
 839};
 840
 841const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
 842	.mss_clamp	=	IPV6_MIN_MTU - sizeof(struct tcphdr) -
 843				sizeof(struct ipv6hdr),
 844#ifdef CONFIG_TCP_MD5SIG
 845	.req_md5_lookup	=	tcp_v6_md5_lookup,
 846	.calc_md5_hash	=	tcp_v6_md5_hash_skb,
 847#endif
 848#ifdef CONFIG_TCP_AO
 849	.ao_lookup	=	tcp_v6_ao_lookup_rsk,
 850	.ao_calc_key	=	tcp_v6_ao_calc_key_rsk,
 851	.ao_synack_hash =	tcp_v6_ao_synack_hash,
 852#endif
 853#ifdef CONFIG_SYN_COOKIES
 854	.cookie_init_seq =	cookie_v6_init_sequence,
 855#endif
 856	.route_req	=	tcp_v6_route_req,
 857	.init_seq	=	tcp_v6_init_seq,
 858	.init_ts_off	=	tcp_v6_init_ts_off,
 859	.send_synack	=	tcp_v6_send_synack,
 860};
 861
 862static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
 863				 u32 ack, u32 win, u32 tsval, u32 tsecr,
 864				 int oif, int rst, u8 tclass, __be32 label,
 865				 u32 priority, u32 txhash, struct tcp_key *key)
 866{
 867	const struct tcphdr *th = tcp_hdr(skb);
 868	struct tcphdr *t1;
 869	struct sk_buff *buff;
 870	struct flowi6 fl6;
 871	struct net *net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
 872	struct sock *ctl_sk = net->ipv6.tcp_sk;
 873	unsigned int tot_len = sizeof(struct tcphdr);
 874	__be32 mrst = 0, *topt;
 875	struct dst_entry *dst;
 876	__u32 mark = 0;
 877
 878	if (tsecr)
 879		tot_len += TCPOLEN_TSTAMP_ALIGNED;
 880	if (tcp_key_is_md5(key))
 881		tot_len += TCPOLEN_MD5SIG_ALIGNED;
 882	if (tcp_key_is_ao(key))
 883		tot_len += tcp_ao_len_aligned(key->ao_key);
 884
 885#ifdef CONFIG_MPTCP
 886	if (rst && !tcp_key_is_md5(key)) {
 887		mrst = mptcp_reset_option(skb);
 888
 889		if (mrst)
 890			tot_len += sizeof(__be32);
 891	}
 892#endif
 893
 894	buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
 895	if (!buff)
 896		return;
 897
 898	skb_reserve(buff, MAX_TCP_HEADER);
 899
 900	t1 = skb_push(buff, tot_len);
 901	skb_reset_transport_header(buff);
 902
 903	/* Swap the send and the receive. */
 904	memset(t1, 0, sizeof(*t1));
 905	t1->dest = th->source;
 906	t1->source = th->dest;
 907	t1->doff = tot_len / 4;
 908	t1->seq = htonl(seq);
 909	t1->ack_seq = htonl(ack);
 910	t1->ack = !rst || !th->ack;
 911	t1->rst = rst;
 912	t1->window = htons(win);
 913
 914	topt = (__be32 *)(t1 + 1);
 915
 916	if (tsecr) {
 917		*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 918				(TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
 919		*topt++ = htonl(tsval);
 920		*topt++ = htonl(tsecr);
 921	}
 922
 923	if (mrst)
 924		*topt++ = mrst;
 925
 926#ifdef CONFIG_TCP_MD5SIG
 927	if (tcp_key_is_md5(key)) {
 928		*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 929				(TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
 930		tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key,
 931				    &ipv6_hdr(skb)->saddr,
 932				    &ipv6_hdr(skb)->daddr, t1);
 933	}
 934#endif
 935#ifdef CONFIG_TCP_AO
 936	if (tcp_key_is_ao(key)) {
 937		*topt++ = htonl((TCPOPT_AO << 24) |
 938				(tcp_ao_len(key->ao_key) << 16) |
 939				(key->ao_key->sndid << 8) |
 940				(key->rcv_next));
 941
 942		tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key,
 943				key->traffic_key,
 944				(union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
 945				(union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
 946				t1, key->sne);
 947	}
 948#endif
 949
 950	memset(&fl6, 0, sizeof(fl6));
 951	fl6.daddr = ipv6_hdr(skb)->saddr;
 952	fl6.saddr = ipv6_hdr(skb)->daddr;
 953	fl6.flowlabel = label;
 954
 955	buff->ip_summed = CHECKSUM_PARTIAL;
 956
 957	__tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);
 958
 959	fl6.flowi6_proto = IPPROTO_TCP;
 960	if (rt6_need_strict(&fl6.daddr) && !oif)
 961		fl6.flowi6_oif = tcp_v6_iif(skb);
 962	else {
 963		if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
 964			oif = skb->skb_iif;
 965
 966		fl6.flowi6_oif = oif;
 967	}
 968
 969	if (sk) {
 970		/* unconstify the socket only to attach it to buff with care. */
 971		skb_set_owner_edemux(buff, (struct sock *)sk);
 972
 973		if (sk->sk_state == TCP_TIME_WAIT)
 974			mark = inet_twsk(sk)->tw_mark;
 975		else
 976			mark = READ_ONCE(sk->sk_mark);
 977		skb_set_delivery_time(buff, tcp_transmit_time(sk), SKB_CLOCK_MONOTONIC);
 978	}
 979	if (txhash) {
 980		/* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
 981		skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4);
 982	}
 983	fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
 984	fl6.fl6_dport = t1->dest;
 985	fl6.fl6_sport = t1->source;
 986	fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
 987	security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6));
 988
 989	/* Pass a socket to ip6_dst_lookup either it is for RST
 990	 * Underlying function will use this to retrieve the network
 991	 * namespace
 992	 */
 993	if (sk && sk->sk_state != TCP_TIME_WAIT)
 994		dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/
 995	else
 996		dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL);
 997	if (!IS_ERR(dst)) {
 998		skb_dst_set(buff, dst);
 999		ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL,
1000			 tclass & ~INET_ECN_MASK, priority);
1001		TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
1002		if (rst)
1003			TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
1004		return;
1005	}
1006
1007	kfree_skb(buff);
1008}
1009
1010static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
1011			      enum sk_rst_reason reason)
1012{
1013	const struct tcphdr *th = tcp_hdr(skb);
1014	struct ipv6hdr *ipv6h = ipv6_hdr(skb);
1015	const __u8 *md5_hash_location = NULL;
1016#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1017	bool allocated_traffic_key = false;
1018#endif
1019	const struct tcp_ao_hdr *aoh;
1020	struct tcp_key key = {};
1021	u32 seq = 0, ack_seq = 0;
1022	__be32 label = 0;
1023	u32 priority = 0;
1024	struct net *net;
1025	u32 txhash = 0;
1026	int oif = 0;
1027#ifdef CONFIG_TCP_MD5SIG
1028	unsigned char newhash[16];
1029	int genhash;
1030	struct sock *sk1 = NULL;
1031#endif
1032
1033	if (th->rst)
1034		return;
1035
1036	/* If sk not NULL, it means we did a successful lookup and incoming
1037	 * route had to be correct. prequeue might have dropped our dst.
1038	 */
1039	if (!sk && !ipv6_unicast_destination(skb))
1040		return;
1041
1042	net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
1043	/* Invalid TCP option size or twice included auth */
1044	if (tcp_parse_auth_options(th, &md5_hash_location, &aoh))
1045		return;
1046#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1047	rcu_read_lock();
1048#endif
1049#ifdef CONFIG_TCP_MD5SIG
1050	if (sk && sk_fullsock(sk)) {
1051		int l3index;
1052
1053		/* sdif set, means packet ingressed via a device
1054		 * in an L3 domain and inet_iif is set to it.
1055		 */
1056		l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1057		key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
1058		if (key.md5_key)
1059			key.type = TCP_KEY_MD5;
1060	} else if (md5_hash_location) {
1061		int dif = tcp_v6_iif_l3_slave(skb);
1062		int sdif = tcp_v6_sdif(skb);
1063		int l3index;
1064
1065		/*
1066		 * active side is lost. Try to find listening socket through
1067		 * source port, and then find md5 key through listening socket.
1068		 * we are not loose security here:
1069		 * Incoming packet is checked with md5 hash with finding key,
1070		 * no RST generated if md5 hash doesn't match.
1071		 */
1072		sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1073					    NULL, 0, &ipv6h->saddr, th->source,
1074					    &ipv6h->daddr, ntohs(th->source),
1075					    dif, sdif);
1076		if (!sk1)
1077			goto out;
1078
1079		/* sdif set, means packet ingressed via a device
1080		 * in an L3 domain and dif is set to it.
1081		 */
1082		l3index = tcp_v6_sdif(skb) ? dif : 0;
1083
1084		key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
1085		if (!key.md5_key)
1086			goto out;
1087		key.type = TCP_KEY_MD5;
1088
1089		genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb);
1090		if (genhash || memcmp(md5_hash_location, newhash, 16) != 0)
1091			goto out;
1092	}
1093#endif
1094
1095	if (th->ack)
1096		seq = ntohl(th->ack_seq);
1097	else
1098		ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1099			  (th->doff << 2);
1100
1101#ifdef CONFIG_TCP_AO
1102	if (aoh) {
1103		int l3index;
1104
1105		l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1106		if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
1107					 &key.ao_key, &key.traffic_key,
1108					 &allocated_traffic_key,
1109					 &key.rcv_next, &key.sne))
1110			goto out;
1111		key.type = TCP_KEY_AO;
1112	}
1113#endif
1114
1115	if (sk) {
1116		oif = sk->sk_bound_dev_if;
1117		if (sk_fullsock(sk)) {
 
1118			if (inet6_test_bit(REPFLOW, sk))
1119				label = ip6_flowlabel(ipv6h);
1120			priority = READ_ONCE(sk->sk_priority);
1121			txhash = sk->sk_txhash;
1122		}
1123		if (sk->sk_state == TCP_TIME_WAIT) {
1124			label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1125			priority = inet_twsk(sk)->tw_priority;
1126			txhash = inet_twsk(sk)->tw_txhash;
1127		}
1128	} else {
1129		if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
1130			label = ip6_flowlabel(ipv6h);
1131	}
1132
1133	trace_tcp_send_reset(sk, skb, reason);
1134
1135	tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1,
1136			     ipv6_get_dsfield(ipv6h), label, priority, txhash,
1137			     &key);
1138
1139#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1140out:
1141	if (allocated_traffic_key)
1142		kfree(key.traffic_key);
1143	rcu_read_unlock();
1144#endif
1145}
1146
1147static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1148			    u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1149			    struct tcp_key *key, u8 tclass,
1150			    __be32 label, u32 priority, u32 txhash)
1151{
1152	tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0,
1153			     tclass, label, priority, txhash, key);
1154}
1155
1156static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
1157{
1158	struct inet_timewait_sock *tw = inet_twsk(sk);
1159	struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1160	struct tcp_key key = {};
1161#ifdef CONFIG_TCP_AO
1162	struct tcp_ao_info *ao_info;
1163
1164	if (static_branch_unlikely(&tcp_ao_needed.key)) {
1165
1166		/* FIXME: the segment to-be-acked is not verified yet */
1167		ao_info = rcu_dereference(tcptw->ao_info);
1168		if (ao_info) {
1169			const struct tcp_ao_hdr *aoh;
1170
1171			/* Invalid TCP option size or twice included auth */
1172			if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1173				goto out;
1174			if (aoh)
1175				key.ao_key = tcp_ao_established_key(sk, ao_info,
1176								    aoh->rnext_keyid, -1);
1177		}
1178	}
1179	if (key.ao_key) {
1180		struct tcp_ao_key *rnext_key;
1181
1182		key.traffic_key = snd_other_key(key.ao_key);
1183		/* rcv_next switches to our rcv_next */
1184		rnext_key = READ_ONCE(ao_info->rnext_key);
1185		key.rcv_next = rnext_key->rcvid;
1186		key.sne = READ_ONCE(ao_info->snd_sne);
1187		key.type = TCP_KEY_AO;
1188#else
1189	if (0) {
1190#endif
1191#ifdef CONFIG_TCP_MD5SIG
1192	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1193		key.md5_key = tcp_twsk_md5_key(tcptw);
1194		if (key.md5_key)
1195			key.type = TCP_KEY_MD5;
1196#endif
1197	}
1198
1199	tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt,
1200			READ_ONCE(tcptw->tw_rcv_nxt),
1201			tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1202			tcp_tw_tsval(tcptw),
1203			READ_ONCE(tcptw->tw_ts_recent), tw->tw_bound_dev_if,
1204			&key, tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel),
1205			tw->tw_priority, tw->tw_txhash);
1206
1207#ifdef CONFIG_TCP_AO
1208out:
1209#endif
1210	inet_twsk_put(tw);
1211}
1212
1213static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1214				  struct request_sock *req)
1215{
1216	struct tcp_key key = {};
1217
1218#ifdef CONFIG_TCP_AO
1219	if (static_branch_unlikely(&tcp_ao_needed.key) &&
1220	    tcp_rsk_used_ao(req)) {
1221		const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
1222		const struct tcp_ao_hdr *aoh;
1223		int l3index;
1224
1225		l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1226		/* Invalid TCP option size or twice included auth */
1227		if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1228			return;
1229		if (!aoh)
1230			return;
1231		key.ao_key = tcp_ao_do_lookup(sk, l3index,
1232					      (union tcp_ao_addr *)addr,
1233					      AF_INET6, aoh->rnext_keyid, -1);
1234		if (unlikely(!key.ao_key)) {
1235			/* Send ACK with any matching MKT for the peer */
1236			key.ao_key = tcp_ao_do_lookup(sk, l3index,
1237						      (union tcp_ao_addr *)addr,
1238						      AF_INET6, -1, -1);
1239			/* Matching key disappeared (user removed the key?)
1240			 * let the handshake timeout.
1241			 */
1242			if (!key.ao_key) {
1243				net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
1244						     addr,
1245						     ntohs(tcp_hdr(skb)->source),
1246						     &ipv6_hdr(skb)->daddr,
1247						     ntohs(tcp_hdr(skb)->dest));
1248				return;
1249			}
1250		}
1251		key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1252		if (!key.traffic_key)
1253			return;
1254
1255		key.type = TCP_KEY_AO;
1256		key.rcv_next = aoh->keyid;
1257		tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
1258#else
1259	if (0) {
1260#endif
1261#ifdef CONFIG_TCP_MD5SIG
1262	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1263		int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1264
1265		key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr,
1266						   l3index);
1267		if (key.md5_key)
1268			key.type = TCP_KEY_MD5;
1269#endif
1270	}
1271
1272	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1273	 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1274	 */
 
 
 
 
 
1275	tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
1276			tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1277			tcp_rsk(req)->rcv_nxt,
1278			tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
1279			tcp_rsk_tsval(tcp_rsk(req)),
1280			READ_ONCE(req->ts_recent), sk->sk_bound_dev_if,
1281			&key, ipv6_get_dsfield(ipv6_hdr(skb)), 0,
1282			READ_ONCE(sk->sk_priority),
1283			READ_ONCE(tcp_rsk(req)->txhash));
1284	if (tcp_key_is_ao(&key))
1285		kfree(key.traffic_key);
1286}
1287
1288
1289static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1290{
1291#ifdef CONFIG_SYN_COOKIES
1292	const struct tcphdr *th = tcp_hdr(skb);
1293
1294	if (!th->syn)
1295		sk = cookie_v6_check(sk, skb);
1296#endif
1297	return sk;
1298}
1299
1300u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1301			 struct tcphdr *th, u32 *cookie)
1302{
1303	u16 mss = 0;
1304#ifdef CONFIG_SYN_COOKIES
1305	mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
1306				    &tcp_request_sock_ipv6_ops, sk, th);
1307	if (mss) {
1308		*cookie = __cookie_v6_init_sequence(iph, th, &mss);
1309		tcp_synq_overflow(sk);
1310	}
1311#endif
1312	return mss;
1313}
1314
1315static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1316{
1317	if (skb->protocol == htons(ETH_P_IP))
1318		return tcp_v4_conn_request(sk, skb);
1319
1320	if (!ipv6_unicast_destination(skb))
1321		goto drop;
1322
1323	if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
1324		__IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1325		return 0;
1326	}
1327
1328	return tcp_conn_request(&tcp6_request_sock_ops,
1329				&tcp_request_sock_ipv6_ops, sk, skb);
1330
1331drop:
1332	tcp_listendrop(sk);
1333	return 0; /* don't send reset */
1334}
1335
1336static void tcp_v6_restore_cb(struct sk_buff *skb)
1337{
1338	/* We need to move header back to the beginning if xfrm6_policy_check()
1339	 * and tcp_v6_fill_cb() are going to be called again.
1340	 * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1341	 */
1342	memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1343		sizeof(struct inet6_skb_parm));
1344}
1345
1346static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1347					 struct request_sock *req,
1348					 struct dst_entry *dst,
1349					 struct request_sock *req_unhash,
1350					 bool *own_req)
1351{
1352	struct inet_request_sock *ireq;
1353	struct ipv6_pinfo *newnp;
1354	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1355	struct ipv6_txoptions *opt;
1356	struct inet_sock *newinet;
1357	bool found_dup_sk = false;
1358	struct tcp_sock *newtp;
1359	struct sock *newsk;
1360#ifdef CONFIG_TCP_MD5SIG
1361	struct tcp_md5sig_key *key;
1362	int l3index;
1363#endif
1364	struct flowi6 fl6;
1365
1366	if (skb->protocol == htons(ETH_P_IP)) {
1367		/*
1368		 *	v6 mapped
1369		 */
1370
1371		newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst,
1372					     req_unhash, own_req);
1373
1374		if (!newsk)
1375			return NULL;
1376
1377		inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1378
1379		newnp = tcp_inet6_sk(newsk);
1380		newtp = tcp_sk(newsk);
1381
1382		memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1383
1384		newnp->saddr = newsk->sk_v6_rcv_saddr;
1385
1386		inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1387		if (sk_is_mptcp(newsk))
1388			mptcpv6_handle_mapped(newsk, true);
1389		newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1390#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1391		newtp->af_specific = &tcp_sock_ipv6_mapped_specific;
1392#endif
1393
1394		newnp->ipv6_mc_list = NULL;
1395		newnp->ipv6_ac_list = NULL;
1396		newnp->ipv6_fl_list = NULL;
1397		newnp->pktoptions  = NULL;
1398		newnp->opt	   = NULL;
1399		newnp->mcast_oif   = inet_iif(skb);
1400		newnp->mcast_hops  = ip_hdr(skb)->ttl;
1401		newnp->rcv_flowinfo = 0;
1402		if (inet6_test_bit(REPFLOW, sk))
1403			newnp->flow_label = 0;
1404
1405		/*
1406		 * No need to charge this sock to the relevant IPv6 refcnt debug socks count
1407		 * here, tcp_create_openreq_child now does this for us, see the comment in
1408		 * that function for the gory details. -acme
1409		 */
1410
1411		/* It is tricky place. Until this moment IPv4 tcp
1412		   worked with IPv6 icsk.icsk_af_ops.
1413		   Sync it now.
1414		 */
1415		tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie);
1416
1417		return newsk;
1418	}
1419
1420	ireq = inet_rsk(req);
1421
1422	if (sk_acceptq_is_full(sk))
1423		goto out_overflow;
1424
1425	if (!dst) {
1426		dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP);
1427		if (!dst)
1428			goto out;
1429	}
1430
1431	newsk = tcp_create_openreq_child(sk, req, skb);
1432	if (!newsk)
1433		goto out_nonewsk;
1434
1435	/*
1436	 * No need to charge this sock to the relevant IPv6 refcnt debug socks
1437	 * count here, tcp_create_openreq_child now does this for us, see the
1438	 * comment in that function for the gory details. -acme
1439	 */
1440
1441	newsk->sk_gso_type = SKB_GSO_TCPV6;
 
1442	inet6_sk_rx_dst_set(newsk, skb);
1443
1444	inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1445
1446	newtp = tcp_sk(newsk);
1447	newinet = inet_sk(newsk);
1448	newnp = tcp_inet6_sk(newsk);
1449
1450	memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1451
1452	ip6_dst_store(newsk, dst, NULL, NULL);
1453
1454	newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr;
1455	newnp->saddr = ireq->ir_v6_loc_addr;
1456	newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr;
1457	newsk->sk_bound_dev_if = ireq->ir_iif;
1458
1459	/* Now IPv6 options...
1460
1461	   First: no IPv4 options.
1462	 */
1463	newinet->inet_opt = NULL;
1464	newnp->ipv6_mc_list = NULL;
1465	newnp->ipv6_ac_list = NULL;
1466	newnp->ipv6_fl_list = NULL;
1467
1468	/* Clone RX bits */
1469	newnp->rxopt.all = np->rxopt.all;
1470
1471	newnp->pktoptions = NULL;
1472	newnp->opt	  = NULL;
1473	newnp->mcast_oif  = tcp_v6_iif(skb);
1474	newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1475	newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
1476	if (inet6_test_bit(REPFLOW, sk))
1477		newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));
1478
1479	/* Set ToS of the new socket based upon the value of incoming SYN.
1480	 * ECT bits are set later in tcp_init_transfer().
1481	 */
1482	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1483		newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1484
1485	/* Clone native IPv6 options from listening socket (if any)
1486
1487	   Yes, keeping reference count would be much more clever,
1488	   but we make one more one thing there: reattach optmem
1489	   to newsk.
1490	 */
1491	opt = ireq->ipv6_opt;
1492	if (!opt)
1493		opt = rcu_dereference(np->opt);
1494	if (opt) {
1495		opt = ipv6_dup_options(newsk, opt);
1496		RCU_INIT_POINTER(newnp->opt, opt);
1497	}
1498	inet_csk(newsk)->icsk_ext_hdr_len = 0;
1499	if (opt)
1500		inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1501						    opt->opt_flen;
1502
1503	tcp_ca_openreq_child(newsk, dst);
1504
1505	tcp_sync_mss(newsk, dst_mtu(dst));
1506	newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1507
1508	tcp_initialize_rcv_mss(newsk);
1509
1510	newinet->inet_daddr = newinet->inet_saddr = LOOPBACK4_IPV6;
1511	newinet->inet_rcv_saddr = LOOPBACK4_IPV6;
1512
1513#ifdef CONFIG_TCP_MD5SIG
1514	l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1515
1516	if (!tcp_rsk_used_ao(req)) {
1517		/* Copy over the MD5 key from the original socket */
1518		key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
1519		if (key) {
1520			const union tcp_md5_addr *addr;
1521
1522			addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
1523			if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) {
1524				inet_csk_prepare_forced_close(newsk);
1525				tcp_done(newsk);
1526				goto out;
1527			}
1528		}
1529	}
1530#endif
1531#ifdef CONFIG_TCP_AO
1532	/* Copy over tcp_ao_info if any */
1533	if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
1534		goto out; /* OOM */
1535#endif
1536
1537	if (__inet_inherit_port(sk, newsk) < 0) {
1538		inet_csk_prepare_forced_close(newsk);
1539		tcp_done(newsk);
1540		goto out;
1541	}
1542	*own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1543				       &found_dup_sk);
1544	if (*own_req) {
1545		tcp_move_syn(newtp, req);
1546
1547		/* Clone pktoptions received with SYN, if we own the req */
1548		if (ireq->pktopts) {
1549			newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk);
1550			consume_skb(ireq->pktopts);
1551			ireq->pktopts = NULL;
1552			if (newnp->pktoptions)
1553				tcp_v6_restore_cb(newnp->pktoptions);
1554		}
1555	} else {
1556		if (!req_unhash && found_dup_sk) {
1557			/* This code path should only be executed in the
1558			 * syncookie case only
1559			 */
1560			bh_unlock_sock(newsk);
1561			sock_put(newsk);
1562			newsk = NULL;
1563		}
1564	}
1565
1566	return newsk;
1567
1568out_overflow:
1569	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1570out_nonewsk:
1571	dst_release(dst);
1572out:
1573	tcp_listendrop(sk);
1574	return NULL;
1575}
1576
1577INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1578							   u32));
1579/* The socket must have it's spinlock held when we get
1580 * here, unless it is a TCP_LISTEN socket.
1581 *
1582 * We have a potential double-lock case here, so even when
1583 * doing backlog processing we use the BH locking scheme.
1584 * This is because we cannot sleep with the original spinlock
1585 * held.
1586 */
1587INDIRECT_CALLABLE_SCOPE
1588int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1589{
1590	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1591	struct sk_buff *opt_skb = NULL;
1592	enum skb_drop_reason reason;
1593	struct tcp_sock *tp;
1594
1595	/* Imagine: socket is IPv6. IPv4 packet arrives,
1596	   goes to IPv4 receive handler and backlogged.
1597	   From backlog it always goes here. Kerboom...
1598	   Fortunately, tcp_rcv_established and rcv_established
1599	   handle them correctly, but it is not case with
1600	   tcp_v6_hnd_req and tcp_v6_send_reset().   --ANK
1601	 */
1602
1603	if (skb->protocol == htons(ETH_P_IP))
1604		return tcp_v4_do_rcv(sk, skb);
1605
1606	/*
1607	 *	socket locking is here for SMP purposes as backlog rcv
1608	 *	is currently called with bh processing disabled.
1609	 */
1610
1611	/* Do Stevens' IPV6_PKTOPTIONS.
1612
1613	   Yes, guys, it is the only place in our code, where we
1614	   may make it not affecting IPv4.
1615	   The rest of code is protocol independent,
1616	   and I do not like idea to uglify IPv4.
1617
1618	   Actually, all the idea behind IPV6_PKTOPTIONS
1619	   looks not very well thought. For now we latch
1620	   options, received in the last packet, enqueued
1621	   by tcp. Feel free to propose better solution.
1622					       --ANK (980728)
1623	 */
1624	if (np->rxopt.all && sk->sk_state != TCP_LISTEN)
1625		opt_skb = skb_clone_and_charge_r(skb, sk);
1626
1627	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1628		struct dst_entry *dst;
1629
1630		dst = rcu_dereference_protected(sk->sk_rx_dst,
1631						lockdep_sock_is_held(sk));
1632
1633		sock_rps_save_rxhash(sk, skb);
1634		sk_mark_napi_id(sk, skb);
1635		if (dst) {
1636			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1637			    INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1638					    dst, sk->sk_rx_dst_cookie) == NULL) {
1639				RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1640				dst_release(dst);
1641			}
1642		}
1643
1644		tcp_rcv_established(sk, skb);
1645		if (opt_skb)
1646			goto ipv6_pktoptions;
1647		return 0;
1648	}
1649
1650	if (tcp_checksum_complete(skb))
1651		goto csum_err;
1652
1653	if (sk->sk_state == TCP_LISTEN) {
1654		struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1655
1656		if (nsk != sk) {
1657			if (nsk) {
1658				reason = tcp_child_process(sk, nsk, skb);
1659				if (reason)
1660					goto reset;
1661			}
 
 
1662			return 0;
1663		}
1664	} else
1665		sock_rps_save_rxhash(sk, skb);
1666
1667	reason = tcp_rcv_state_process(sk, skb);
1668	if (reason)
1669		goto reset;
1670	if (opt_skb)
1671		goto ipv6_pktoptions;
1672	return 0;
1673
1674reset:
1675	tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason));
1676discard:
1677	if (opt_skb)
1678		__kfree_skb(opt_skb);
1679	sk_skb_reason_drop(sk, skb, reason);
1680	return 0;
1681csum_err:
1682	reason = SKB_DROP_REASON_TCP_CSUM;
1683	trace_tcp_bad_csum(skb);
1684	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1685	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1686	goto discard;
1687
1688
1689ipv6_pktoptions:
1690	/* Do you ask, what is it?
1691
1692	   1. skb was enqueued by tcp.
1693	   2. skb is added to tail of read queue, rather than out of order.
1694	   3. socket is not in passive state.
1695	   4. Finally, it really contains options, which user wants to receive.
1696	 */
1697	tp = tcp_sk(sk);
1698	if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1699	    !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1700		if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1701			WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb));
1702		if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1703			WRITE_ONCE(np->mcast_hops,
1704				   ipv6_hdr(opt_skb)->hop_limit);
1705		if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1706			np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
1707		if (inet6_test_bit(REPFLOW, sk))
1708			np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
1709		if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
1710			tcp_v6_restore_cb(opt_skb);
1711			opt_skb = xchg(&np->pktoptions, opt_skb);
1712		} else {
1713			__kfree_skb(opt_skb);
1714			opt_skb = xchg(&np->pktoptions, NULL);
1715		}
1716	}
1717
1718	consume_skb(opt_skb);
1719	return 0;
1720}
1721
1722static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1723			   const struct tcphdr *th)
1724{
1725	/* This is tricky: we move IP6CB at its correct location into
1726	 * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1727	 * _decode_session6() uses IP6CB().
1728	 * barrier() makes sure compiler won't play aliasing games.
1729	 */
1730	memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1731		sizeof(struct inet6_skb_parm));
1732	barrier();
1733
1734	TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1735	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1736				    skb->len - th->doff*4);
1737	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1738	TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
 
1739	TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
1740	TCP_SKB_CB(skb)->sacked = 0;
1741	TCP_SKB_CB(skb)->has_rxtstamp =
1742			skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1743}
1744
1745INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1746{
1747	enum skb_drop_reason drop_reason;
1748	int sdif = inet6_sdif(skb);
1749	int dif = inet6_iif(skb);
1750	const struct tcphdr *th;
1751	const struct ipv6hdr *hdr;
1752	struct sock *sk = NULL;
1753	bool refcounted;
 
1754	int ret;
1755	u32 isn;
1756	struct net *net = dev_net(skb->dev);
1757
1758	drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1759	if (skb->pkt_type != PACKET_HOST)
1760		goto discard_it;
1761
1762	/*
1763	 *	Count it even if it's bad.
1764	 */
1765	__TCP_INC_STATS(net, TCP_MIB_INSEGS);
1766
1767	if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1768		goto discard_it;
1769
1770	th = (const struct tcphdr *)skb->data;
1771
1772	if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1773		drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1774		goto bad_packet;
1775	}
1776	if (!pskb_may_pull(skb, th->doff*4))
1777		goto discard_it;
1778
1779	if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1780		goto csum_error;
1781
1782	th = (const struct tcphdr *)skb->data;
1783	hdr = ipv6_hdr(skb);
1784
1785lookup:
1786	sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th),
1787				th->source, th->dest, inet6_iif(skb), sdif,
1788				&refcounted);
1789	if (!sk)
1790		goto no_tcp_socket;
1791
 
1792	if (sk->sk_state == TCP_TIME_WAIT)
1793		goto do_time_wait;
1794
1795	if (sk->sk_state == TCP_NEW_SYN_RECV) {
1796		struct request_sock *req = inet_reqsk(sk);
1797		bool req_stolen = false;
1798		struct sock *nsk;
1799
1800		sk = req->rsk_listener;
1801		if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
1802			drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1803		else
1804			drop_reason = tcp_inbound_hash(sk, req, skb,
1805						       &hdr->saddr, &hdr->daddr,
1806						       AF_INET6, dif, sdif);
1807		if (drop_reason) {
1808			sk_drops_add(sk, skb);
1809			reqsk_put(req);
1810			goto discard_it;
1811		}
1812		if (tcp_checksum_complete(skb)) {
1813			reqsk_put(req);
1814			goto csum_error;
1815		}
1816		if (unlikely(sk->sk_state != TCP_LISTEN)) {
1817			nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
1818			if (!nsk) {
1819				inet_csk_reqsk_queue_drop_and_put(sk, req);
1820				goto lookup;
1821			}
1822			sk = nsk;
1823			/* reuseport_migrate_sock() has already held one sk_refcnt
1824			 * before returning.
1825			 */
1826		} else {
1827			sock_hold(sk);
1828		}
1829		refcounted = true;
1830		nsk = NULL;
1831		if (!tcp_filter(sk, skb)) {
1832			th = (const struct tcphdr *)skb->data;
1833			hdr = ipv6_hdr(skb);
1834			tcp_v6_fill_cb(skb, hdr, th);
1835			nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1836		} else {
1837			drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1838		}
1839		if (!nsk) {
1840			reqsk_put(req);
1841			if (req_stolen) {
1842				/* Another cpu got exclusive access to req
1843				 * and created a full blown socket.
1844				 * Try to feed this packet to this socket
1845				 * instead of discarding it.
1846				 */
1847				tcp_v6_restore_cb(skb);
1848				sock_put(sk);
1849				goto lookup;
1850			}
1851			goto discard_and_relse;
1852		}
1853		nf_reset_ct(skb);
1854		if (nsk == sk) {
1855			reqsk_put(req);
1856			tcp_v6_restore_cb(skb);
1857		} else {
1858			drop_reason = tcp_child_process(sk, nsk, skb);
1859			if (drop_reason) {
1860				enum sk_rst_reason rst_reason;
1861
1862				rst_reason = sk_rst_convert_drop_reason(drop_reason);
1863				tcp_v6_send_reset(nsk, skb, rst_reason);
1864				goto discard_and_relse;
1865			}
1866			sock_put(sk);
1867			return 0;
1868		}
1869	}
1870
1871process:
1872	if (static_branch_unlikely(&ip6_min_hopcount)) {
1873		/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
1874		if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
1875			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1876			drop_reason = SKB_DROP_REASON_TCP_MINTTL;
1877			goto discard_and_relse;
1878		}
1879	}
1880
1881	if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
1882		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1883		goto discard_and_relse;
1884	}
1885
1886	drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr,
1887				       AF_INET6, dif, sdif);
1888	if (drop_reason)
1889		goto discard_and_relse;
1890
1891	nf_reset_ct(skb);
1892
1893	if (tcp_filter(sk, skb)) {
1894		drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1895		goto discard_and_relse;
1896	}
1897	th = (const struct tcphdr *)skb->data;
1898	hdr = ipv6_hdr(skb);
1899	tcp_v6_fill_cb(skb, hdr, th);
1900
1901	skb->dev = NULL;
1902
1903	if (sk->sk_state == TCP_LISTEN) {
1904		ret = tcp_v6_do_rcv(sk, skb);
1905		goto put_and_return;
1906	}
1907
1908	sk_incoming_cpu_update(sk);
1909
1910	bh_lock_sock_nested(sk);
1911	tcp_segs_in(tcp_sk(sk), skb);
1912	ret = 0;
1913	if (!sock_owned_by_user(sk)) {
1914		ret = tcp_v6_do_rcv(sk, skb);
1915	} else {
1916		if (tcp_add_backlog(sk, skb, &drop_reason))
1917			goto discard_and_relse;
1918	}
1919	bh_unlock_sock(sk);
1920put_and_return:
1921	if (refcounted)
1922		sock_put(sk);
1923	return ret ? -1 : 0;
1924
1925no_tcp_socket:
1926	drop_reason = SKB_DROP_REASON_NO_SOCKET;
1927	if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1928		goto discard_it;
1929
1930	tcp_v6_fill_cb(skb, hdr, th);
1931
1932	if (tcp_checksum_complete(skb)) {
1933csum_error:
1934		drop_reason = SKB_DROP_REASON_TCP_CSUM;
1935		trace_tcp_bad_csum(skb);
1936		__TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1937bad_packet:
1938		__TCP_INC_STATS(net, TCP_MIB_INERRS);
1939	} else {
1940		tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
1941	}
1942
1943discard_it:
1944	SKB_DR_OR(drop_reason, NOT_SPECIFIED);
1945	sk_skb_reason_drop(sk, skb, drop_reason);
1946	return 0;
1947
1948discard_and_relse:
1949	sk_drops_add(sk, skb);
1950	if (refcounted)
1951		sock_put(sk);
1952	goto discard_it;
1953
1954do_time_wait:
1955	if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1956		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1957		inet_twsk_put(inet_twsk(sk));
1958		goto discard_it;
1959	}
1960
1961	tcp_v6_fill_cb(skb, hdr, th);
1962
1963	if (tcp_checksum_complete(skb)) {
1964		inet_twsk_put(inet_twsk(sk));
1965		goto csum_error;
1966	}
1967
1968	switch (tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn)) {
1969	case TCP_TW_SYN:
1970	{
1971		struct sock *sk2;
1972
1973		sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1974					    skb, __tcp_hdrlen(th),
1975					    &ipv6_hdr(skb)->saddr, th->source,
1976					    &ipv6_hdr(skb)->daddr,
1977					    ntohs(th->dest),
1978					    tcp_v6_iif_l3_slave(skb),
1979					    sdif);
1980		if (sk2) {
1981			struct inet_timewait_sock *tw = inet_twsk(sk);
1982			inet_twsk_deschedule_put(tw);
1983			sk = sk2;
1984			tcp_v6_restore_cb(skb);
1985			refcounted = false;
1986			__this_cpu_write(tcp_tw_isn, isn);
1987			goto process;
1988		}
1989	}
1990		/* to ACK */
1991		fallthrough;
1992	case TCP_TW_ACK:
1993		tcp_v6_timewait_ack(sk, skb);
1994		break;
1995	case TCP_TW_RST:
1996		tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
1997		inet_twsk_deschedule_put(inet_twsk(sk));
1998		goto discard_it;
1999	case TCP_TW_SUCCESS:
2000		;
2001	}
2002	goto discard_it;
2003}
2004
2005void tcp_v6_early_demux(struct sk_buff *skb)
2006{
2007	struct net *net = dev_net(skb->dev);
2008	const struct ipv6hdr *hdr;
2009	const struct tcphdr *th;
2010	struct sock *sk;
2011
2012	if (skb->pkt_type != PACKET_HOST)
2013		return;
2014
2015	if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
2016		return;
2017
2018	hdr = ipv6_hdr(skb);
2019	th = tcp_hdr(skb);
2020
2021	if (th->doff < sizeof(struct tcphdr) / 4)
2022		return;
2023
2024	/* Note : We use inet6_iif() here, not tcp_v6_iif() */
2025	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
2026					&hdr->saddr, th->source,
2027					&hdr->daddr, ntohs(th->dest),
2028					inet6_iif(skb), inet6_sdif(skb));
2029	if (sk) {
2030		skb->sk = sk;
2031		skb->destructor = sock_edemux;
2032		if (sk_fullsock(sk)) {
2033			struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
2034
2035			if (dst)
2036				dst = dst_check(dst, sk->sk_rx_dst_cookie);
2037			if (dst &&
2038			    sk->sk_rx_dst_ifindex == skb->skb_iif)
2039				skb_dst_set_noref(skb, dst);
2040		}
2041	}
2042}
2043
2044static struct timewait_sock_ops tcp6_timewait_sock_ops = {
2045	.twsk_obj_size	= sizeof(struct tcp6_timewait_sock),
 
2046	.twsk_destructor = tcp_twsk_destructor,
2047};
2048
2049INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
2050{
2051	__tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr);
2052}
2053
2054const struct inet_connection_sock_af_ops ipv6_specific = {
2055	.queue_xmit	   = inet6_csk_xmit,
2056	.send_check	   = tcp_v6_send_check,
2057	.rebuild_header	   = inet6_sk_rebuild_header,
2058	.sk_rx_dst_set	   = inet6_sk_rx_dst_set,
2059	.conn_request	   = tcp_v6_conn_request,
2060	.syn_recv_sock	   = tcp_v6_syn_recv_sock,
2061	.net_header_len	   = sizeof(struct ipv6hdr),
2062	.setsockopt	   = ipv6_setsockopt,
2063	.getsockopt	   = ipv6_getsockopt,
2064	.addr2sockaddr	   = inet6_csk_addr2sockaddr,
2065	.sockaddr_len	   = sizeof(struct sockaddr_in6),
2066	.mtu_reduced	   = tcp_v6_mtu_reduced,
2067};
2068
2069#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2070static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
2071#ifdef CONFIG_TCP_MD5SIG
2072	.md5_lookup	=	tcp_v6_md5_lookup,
2073	.calc_md5_hash	=	tcp_v6_md5_hash_skb,
2074	.md5_parse	=	tcp_v6_parse_md5_keys,
2075#endif
2076#ifdef CONFIG_TCP_AO
2077	.ao_lookup	=	tcp_v6_ao_lookup,
2078	.calc_ao_hash	=	tcp_v6_ao_hash_skb,
2079	.ao_parse	=	tcp_v6_parse_ao,
2080	.ao_calc_key_sk	=	tcp_v6_ao_calc_key_sk,
2081#endif
2082};
2083#endif
2084
2085/*
2086 *	TCP over IPv4 via INET6 API
2087 */
2088static const struct inet_connection_sock_af_ops ipv6_mapped = {
2089	.queue_xmit	   = ip_queue_xmit,
2090	.send_check	   = tcp_v4_send_check,
2091	.rebuild_header	   = inet_sk_rebuild_header,
2092	.sk_rx_dst_set	   = inet_sk_rx_dst_set,
2093	.conn_request	   = tcp_v6_conn_request,
2094	.syn_recv_sock	   = tcp_v6_syn_recv_sock,
2095	.net_header_len	   = sizeof(struct iphdr),
2096	.setsockopt	   = ipv6_setsockopt,
2097	.getsockopt	   = ipv6_getsockopt,
2098	.addr2sockaddr	   = inet6_csk_addr2sockaddr,
2099	.sockaddr_len	   = sizeof(struct sockaddr_in6),
2100	.mtu_reduced	   = tcp_v4_mtu_reduced,
2101};
2102
2103#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2104static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
2105#ifdef CONFIG_TCP_MD5SIG
2106	.md5_lookup	=	tcp_v4_md5_lookup,
2107	.calc_md5_hash	=	tcp_v4_md5_hash_skb,
2108	.md5_parse	=	tcp_v6_parse_md5_keys,
2109#endif
2110#ifdef CONFIG_TCP_AO
2111	.ao_lookup	=	tcp_v6_ao_lookup,
2112	.calc_ao_hash	=	tcp_v4_ao_hash_skb,
2113	.ao_parse	=	tcp_v6_parse_ao,
2114	.ao_calc_key_sk	=	tcp_v4_ao_calc_key_sk,
2115#endif
2116};
2117#endif
2118
2119/* NOTE: A lot of things set to zero explicitly by call to
2120 *       sk_alloc() so need not be done here.
2121 */
2122static int tcp_v6_init_sock(struct sock *sk)
2123{
2124	struct inet_connection_sock *icsk = inet_csk(sk);
2125
2126	tcp_init_sock(sk);
2127
2128	icsk->icsk_af_ops = &ipv6_specific;
2129
2130#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2131	tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
2132#endif
2133
2134	return 0;
2135}
2136
2137#ifdef CONFIG_PROC_FS
2138/* Proc filesystem TCPv6 sock list dumping. */
2139static void get_openreq6(struct seq_file *seq,
2140			 const struct request_sock *req, int i)
2141{
2142	long ttd = req->rsk_timer.expires - jiffies;
2143	const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
2144	const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;
2145
2146	if (ttd < 0)
2147		ttd = 0;
2148
2149	seq_printf(seq,
2150		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2151		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
2152		   i,
2153		   src->s6_addr32[0], src->s6_addr32[1],
2154		   src->s6_addr32[2], src->s6_addr32[3],
2155		   inet_rsk(req)->ir_num,
2156		   dest->s6_addr32[0], dest->s6_addr32[1],
2157		   dest->s6_addr32[2], dest->s6_addr32[3],
2158		   ntohs(inet_rsk(req)->ir_rmt_port),
2159		   TCP_SYN_RECV,
2160		   0, 0, /* could print option size, but that is af dependent. */
2161		   1,   /* timers active (only the expire timer) */
2162		   jiffies_to_clock_t(ttd),
2163		   req->num_timeout,
2164		   from_kuid_munged(seq_user_ns(seq),
2165				    sock_i_uid(req->rsk_listener)),
2166		   0,  /* non standard timer */
2167		   0, /* open_requests have no inode */
2168		   0, req);
2169}
2170
2171static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
2172{
2173	const struct in6_addr *dest, *src;
2174	__u16 destp, srcp;
2175	int timer_active;
2176	unsigned long timer_expires;
2177	const struct inet_sock *inet = inet_sk(sp);
2178	const struct tcp_sock *tp = tcp_sk(sp);
2179	const struct inet_connection_sock *icsk = inet_csk(sp);
2180	const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2181	u8 icsk_pending;
2182	int rx_queue;
2183	int state;
2184
2185	dest  = &sp->sk_v6_daddr;
2186	src   = &sp->sk_v6_rcv_saddr;
2187	destp = ntohs(inet->inet_dport);
2188	srcp  = ntohs(inet->inet_sport);
2189
2190	icsk_pending = smp_load_acquire(&icsk->icsk_pending);
2191	if (icsk_pending == ICSK_TIME_RETRANS ||
2192	    icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2193	    icsk_pending == ICSK_TIME_LOSS_PROBE) {
2194		timer_active	= 1;
2195		timer_expires	= icsk->icsk_timeout;
2196	} else if (icsk_pending == ICSK_TIME_PROBE0) {
2197		timer_active	= 4;
2198		timer_expires	= icsk->icsk_timeout;
2199	} else if (timer_pending(&sp->sk_timer)) {
2200		timer_active	= 2;
2201		timer_expires	= sp->sk_timer.expires;
2202	} else {
2203		timer_active	= 0;
2204		timer_expires = jiffies;
2205	}
2206
2207	state = inet_sk_state_load(sp);
2208	if (state == TCP_LISTEN)
2209		rx_queue = READ_ONCE(sp->sk_ack_backlog);
2210	else
2211		/* Because we don't lock the socket,
2212		 * we might find a transient negative value.
2213		 */
2214		rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2215				      READ_ONCE(tp->copied_seq), 0);
2216
2217	seq_printf(seq,
2218		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2219		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
2220		   i,
2221		   src->s6_addr32[0], src->s6_addr32[1],
2222		   src->s6_addr32[2], src->s6_addr32[3], srcp,
2223		   dest->s6_addr32[0], dest->s6_addr32[1],
2224		   dest->s6_addr32[2], dest->s6_addr32[3], destp,
2225		   state,
2226		   READ_ONCE(tp->write_seq) - tp->snd_una,
2227		   rx_queue,
2228		   timer_active,
2229		   jiffies_delta_to_clock_t(timer_expires - jiffies),
2230		   icsk->icsk_retransmits,
2231		   from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
2232		   icsk->icsk_probes_out,
2233		   sock_i_ino(sp),
2234		   refcount_read(&sp->sk_refcnt), sp,
2235		   jiffies_to_clock_t(icsk->icsk_rto),
2236		   jiffies_to_clock_t(icsk->icsk_ack.ato),
2237		   (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
2238		   tcp_snd_cwnd(tp),
2239		   state == TCP_LISTEN ?
2240			fastopenq->max_qlen :
2241			(tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2242		   );
2243}
2244
2245static void get_timewait6_sock(struct seq_file *seq,
2246			       struct inet_timewait_sock *tw, int i)
2247{
2248	long delta = tw->tw_timer.expires - jiffies;
2249	const struct in6_addr *dest, *src;
2250	__u16 destp, srcp;
2251
2252	dest = &tw->tw_v6_daddr;
2253	src  = &tw->tw_v6_rcv_saddr;
2254	destp = ntohs(tw->tw_dport);
2255	srcp  = ntohs(tw->tw_sport);
2256
2257	seq_printf(seq,
2258		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2259		   "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2260		   i,
2261		   src->s6_addr32[0], src->s6_addr32[1],
2262		   src->s6_addr32[2], src->s6_addr32[3], srcp,
2263		   dest->s6_addr32[0], dest->s6_addr32[1],
2264		   dest->s6_addr32[2], dest->s6_addr32[3], destp,
2265		   READ_ONCE(tw->tw_substate), 0, 0,
2266		   3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2267		   refcount_read(&tw->tw_refcnt), tw);
2268}
2269
2270static int tcp6_seq_show(struct seq_file *seq, void *v)
2271{
2272	struct tcp_iter_state *st;
2273	struct sock *sk = v;
2274
2275	if (v == SEQ_START_TOKEN) {
2276		seq_puts(seq,
2277			 "  sl  "
2278			 "local_address                         "
2279			 "remote_address                        "
2280			 "st tx_queue rx_queue tr tm->when retrnsmt"
2281			 "   uid  timeout inode\n");
2282		goto out;
2283	}
2284	st = seq->private;
2285
2286	if (sk->sk_state == TCP_TIME_WAIT)
2287		get_timewait6_sock(seq, v, st->num);
2288	else if (sk->sk_state == TCP_NEW_SYN_RECV)
2289		get_openreq6(seq, v, st->num);
2290	else
2291		get_tcp6_sock(seq, v, st->num);
2292out:
2293	return 0;
2294}
2295
2296static const struct seq_operations tcp6_seq_ops = {
2297	.show		= tcp6_seq_show,
2298	.start		= tcp_seq_start,
2299	.next		= tcp_seq_next,
2300	.stop		= tcp_seq_stop,
2301};
2302
2303static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2304	.family		= AF_INET6,
2305};
2306
2307int __net_init tcp6_proc_init(struct net *net)
2308{
2309	if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
2310			sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
2311		return -ENOMEM;
2312	return 0;
2313}
2314
2315void tcp6_proc_exit(struct net *net)
2316{
2317	remove_proc_entry("tcp6", net->proc_net);
2318}
2319#endif
2320
2321struct proto tcpv6_prot = {
2322	.name			= "TCPv6",
2323	.owner			= THIS_MODULE,
2324	.close			= tcp_close,
2325	.pre_connect		= tcp_v6_pre_connect,
2326	.connect		= tcp_v6_connect,
2327	.disconnect		= tcp_disconnect,
2328	.accept			= inet_csk_accept,
2329	.ioctl			= tcp_ioctl,
2330	.init			= tcp_v6_init_sock,
2331	.destroy		= tcp_v4_destroy_sock,
2332	.shutdown		= tcp_shutdown,
2333	.setsockopt		= tcp_setsockopt,
2334	.getsockopt		= tcp_getsockopt,
2335	.bpf_bypass_getsockopt	= tcp_bpf_bypass_getsockopt,
2336	.keepalive		= tcp_set_keepalive,
2337	.recvmsg		= tcp_recvmsg,
2338	.sendmsg		= tcp_sendmsg,
2339	.splice_eof		= tcp_splice_eof,
2340	.backlog_rcv		= tcp_v6_do_rcv,
2341	.release_cb		= tcp_release_cb,
2342	.hash			= inet6_hash,
2343	.unhash			= inet_unhash,
2344	.get_port		= inet_csk_get_port,
2345	.put_port		= inet_put_port,
2346#ifdef CONFIG_BPF_SYSCALL
2347	.psock_update_sk_prot	= tcp_bpf_update_proto,
2348#endif
2349	.enter_memory_pressure	= tcp_enter_memory_pressure,
2350	.leave_memory_pressure	= tcp_leave_memory_pressure,
2351	.stream_memory_free	= tcp_stream_memory_free,
2352	.sockets_allocated	= &tcp_sockets_allocated,
2353
2354	.memory_allocated	= &tcp_memory_allocated,
2355	.per_cpu_fw_alloc	= &tcp_memory_per_cpu_fw_alloc,
2356
2357	.memory_pressure	= &tcp_memory_pressure,
2358	.orphan_count		= &tcp_orphan_count,
2359	.sysctl_mem		= sysctl_tcp_mem,
2360	.sysctl_wmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_wmem),
2361	.sysctl_rmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_rmem),
2362	.max_header		= MAX_TCP_HEADER,
2363	.obj_size		= sizeof(struct tcp6_sock),
2364	.ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
2365	.slab_flags		= SLAB_TYPESAFE_BY_RCU,
2366	.twsk_prot		= &tcp6_timewait_sock_ops,
2367	.rsk_prot		= &tcp6_request_sock_ops,
2368	.h.hashinfo		= NULL,
2369	.no_autobind		= true,
2370	.diag_destroy		= tcp_abort,
2371};
2372EXPORT_SYMBOL_GPL(tcpv6_prot);
2373
2374
2375static struct inet_protosw tcpv6_protosw = {
2376	.type		=	SOCK_STREAM,
2377	.protocol	=	IPPROTO_TCP,
2378	.prot		=	&tcpv6_prot,
2379	.ops		=	&inet6_stream_ops,
2380	.flags		=	INET_PROTOSW_PERMANENT |
2381				INET_PROTOSW_ICSK,
2382};
2383
2384static int __net_init tcpv6_net_init(struct net *net)
2385{
2386	int res;
2387
2388	res = inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
2389				   SOCK_RAW, IPPROTO_TCP, net);
2390	if (!res)
2391		net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC;
2392
2393	return res;
2394}
2395
2396static void __net_exit tcpv6_net_exit(struct net *net)
2397{
2398	inet_ctl_sock_destroy(net->ipv6.tcp_sk);
2399}
2400
 
 
 
 
 
2401static struct pernet_operations tcpv6_net_ops = {
2402	.init	    = tcpv6_net_init,
2403	.exit	    = tcpv6_net_exit,
 
2404};
2405
2406int __init tcpv6_init(void)
2407{
2408	int ret;
2409
2410	net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
2411		.handler     = tcp_v6_rcv,
2412		.err_handler = tcp_v6_err,
2413		.flags	     = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
2414	};
2415	ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2416	if (ret)
2417		goto out;
2418
2419	/* register inet6 protocol */
2420	ret = inet6_register_protosw(&tcpv6_protosw);
2421	if (ret)
2422		goto out_tcpv6_protocol;
2423
2424	ret = register_pernet_subsys(&tcpv6_net_ops);
2425	if (ret)
2426		goto out_tcpv6_protosw;
2427
2428	ret = mptcpv6_init();
2429	if (ret)
2430		goto out_tcpv6_pernet_subsys;
2431
2432out:
2433	return ret;
2434
2435out_tcpv6_pernet_subsys:
2436	unregister_pernet_subsys(&tcpv6_net_ops);
2437out_tcpv6_protosw:
2438	inet6_unregister_protosw(&tcpv6_protosw);
2439out_tcpv6_protocol:
2440	inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2441	goto out;
2442}
2443
2444void tcpv6_exit(void)
2445{
2446	unregister_pernet_subsys(&tcpv6_net_ops);
2447	inet6_unregister_protosw(&tcpv6_protosw);
2448	inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2449}