1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 *	TCP over IPv6
   4 *	Linux INET6 implementation
   5 *
   6 *	Authors:
   7 *	Pedro Roque		<roque@di.fc.ul.pt>
   8 *
   9 *	Based on:
  10 *	linux/net/ipv4/tcp.c
  11 *	linux/net/ipv4/tcp_input.c
  12 *	linux/net/ipv4/tcp_output.c
  13 *
  14 *	Fixes:
  15 *	Hideaki YOSHIFUJI	:	sin6_scope_id support
  16 *	YOSHIFUJI Hideaki @USAGI and:	Support IPV6_V6ONLY socket option, which
  17 *	Alexey Kuznetsov		allow both IPv4 and IPv6 sockets to bind
  18 *					a single port at the same time.
  19 *	YOSHIFUJI Hideaki @USAGI:	convert /proc/net/tcp6 to seq_file.
  20 */
  21
  22#include <linux/bottom_half.h>
  23#include <linux/module.h>
  24#include <linux/errno.h>
  25#include <linux/types.h>
  26#include <linux/socket.h>
  27#include <linux/sockios.h>
  28#include <linux/net.h>
  29#include <linux/jiffies.h>
  30#include <linux/in.h>
  31#include <linux/in6.h>
  32#include <linux/netdevice.h>
  33#include <linux/init.h>
  34#include <linux/jhash.h>
  35#include <linux/ipsec.h>
  36#include <linux/times.h>
  37#include <linux/slab.h>
  38#include <linux/uaccess.h>
  39#include <linux/ipv6.h>
  40#include <linux/icmpv6.h>
  41#include <linux/random.h>
  42#include <linux/indirect_call_wrapper.h>
  43
  44#include <net/tcp.h>
  45#include <net/ndisc.h>
  46#include <net/inet6_hashtables.h>
  47#include <net/inet6_connection_sock.h>
  48#include <net/ipv6.h>
  49#include <net/transp_v6.h>
  50#include <net/addrconf.h>
  51#include <net/ip6_route.h>
  52#include <net/ip6_checksum.h>
  53#include <net/inet_ecn.h>
  54#include <net/protocol.h>
  55#include <net/xfrm.h>
  56#include <net/snmp.h>
  57#include <net/dsfield.h>
  58#include <net/timewait_sock.h>
  59#include <net/inet_common.h>
  60#include <net/secure_seq.h>
  61#include <net/busy_poll.h>
  62
  63#include <linux/proc_fs.h>
  64#include <linux/seq_file.h>
  65
  66#include <crypto/hash.h>
  67#include <linux/scatterlist.h>
  68
  69#include <trace/events/tcp.h>
  70
  71static void	tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb);
  72static void	tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
  73				      struct request_sock *req);
  74
  75INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
  76
  77static const struct inet_connection_sock_af_ops ipv6_mapped;
  78const struct inet_connection_sock_af_ops ipv6_specific;
  79#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
  80static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
  81static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
  82#endif
  83
  84/* Helper returning the inet6 address from a given tcp socket.
  85 * It can be used in TCP stack instead of inet6_sk(sk).
  86 * This avoids a dereference and allow compiler optimizations.
  87 * It is a specialized version of inet6_sk_generic().
  88 */
  89#define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
  90					      struct tcp6_sock, tcp)->inet6)
  91
  92static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
  93{
  94	struct dst_entry *dst = skb_dst(skb);
  95
  96	if (dst && dst_hold_safe(dst)) {
  97		const struct rt6_info *rt = (const struct rt6_info *)dst;
  98
  99		rcu_assign_pointer(sk->sk_rx_dst, dst);
 100		sk->sk_rx_dst_ifindex = skb->skb_iif;
 101		sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
 102	}
 103}
 104
 105static u32 tcp_v6_init_seq(const struct sk_buff *skb)
 106{
 107	return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32,
 108				ipv6_hdr(skb)->saddr.s6_addr32,
 109				tcp_hdr(skb)->dest,
 110				tcp_hdr(skb)->source);
 111}
 112
 113static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb)
 114{
 115	return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32,
 116				   ipv6_hdr(skb)->saddr.s6_addr32);
 117}
 118
 119static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
 120			      int addr_len)
 121{
 122	/* This check is replicated from tcp_v6_connect() and intended to
 123	 * prevent BPF program called below from accessing bytes that are out
 124	 * of the bound specified by user in addr_len.
 125	 */
 126	if (addr_len < SIN6_LEN_RFC2133)
 127		return -EINVAL;
 128
 129	sock_owned_by_me(sk);
 130
 131	return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
 132}
 133
 134static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 135			  int addr_len)
 136{
 137	struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
 138	struct inet_connection_sock *icsk = inet_csk(sk);
 139	struct in6_addr *saddr = NULL, *final_p, final;
 140	struct inet_timewait_death_row *tcp_death_row;
 141	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 142	struct inet_sock *inet = inet_sk(sk);
 143	struct tcp_sock *tp = tcp_sk(sk);
 144	struct net *net = sock_net(sk);
 145	struct ipv6_txoptions *opt;
 146	struct dst_entry *dst;
 147	struct flowi6 fl6;
 148	int addr_type;
 149	int err;
 150
 151	if (addr_len < SIN6_LEN_RFC2133)
 152		return -EINVAL;
 153
 154	if (usin->sin6_family != AF_INET6)
 155		return -EAFNOSUPPORT;
 156
 157	memset(&fl6, 0, sizeof(fl6));
 158
 159	if (inet6_test_bit(SNDFLOW, sk)) {
 160		fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
 161		IP6_ECN_flow_init(fl6.flowlabel);
 162		if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
 163			struct ip6_flowlabel *flowlabel;
 164			flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
 165			if (IS_ERR(flowlabel))
 166				return -EINVAL;
 167			fl6_sock_release(flowlabel);
 168		}
 169	}
 170
 171	/*
 172	 *	connect() to INADDR_ANY means loopback (BSD'ism).
 173	 */
 174
 175	if (ipv6_addr_any(&usin->sin6_addr)) {
 176		if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
 177			ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
 178					       &usin->sin6_addr);
 179		else
 180			usin->sin6_addr = in6addr_loopback;
 181	}
 182
 183	addr_type = ipv6_addr_type(&usin->sin6_addr);
 184
 185	if (addr_type & IPV6_ADDR_MULTICAST)
 186		return -ENETUNREACH;
 187
 188	if (addr_type&IPV6_ADDR_LINKLOCAL) {
 189		if (addr_len >= sizeof(struct sockaddr_in6) &&
 190		    usin->sin6_scope_id) {
 191			/* If interface is set while binding, indices
 192			 * must coincide.
 193			 */
 194			if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
 195				return -EINVAL;
 196
 197			sk->sk_bound_dev_if = usin->sin6_scope_id;
 198		}
 199
 200		/* Connect to link-local address requires an interface */
 201		if (!sk->sk_bound_dev_if)
 202			return -EINVAL;
 203	}
 204
 205	if (tp->rx_opt.ts_recent_stamp &&
 206	    !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
 207		tp->rx_opt.ts_recent = 0;
 208		tp->rx_opt.ts_recent_stamp = 0;
 209		WRITE_ONCE(tp->write_seq, 0);
 210	}
 211
 212	sk->sk_v6_daddr = usin->sin6_addr;
 213	np->flow_label = fl6.flowlabel;
 214
 215	/*
 216	 *	TCP over IPv4
 217	 */
 218
 219	if (addr_type & IPV6_ADDR_MAPPED) {
 220		u32 exthdrlen = icsk->icsk_ext_hdr_len;
 221		struct sockaddr_in sin;
 222
 223		if (ipv6_only_sock(sk))
 224			return -ENETUNREACH;
 225
 226		sin.sin_family = AF_INET;
 227		sin.sin_port = usin->sin6_port;
 228		sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
 229
 230		/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 231		WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
 232		if (sk_is_mptcp(sk))
 233			mptcpv6_handle_mapped(sk, true);
 234		sk->sk_backlog_rcv = tcp_v4_do_rcv;
 235#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 236		tp->af_specific = &tcp_sock_ipv6_mapped_specific;
 237#endif
 238
 239		err = tcp_v4_connect(sk, (struct sockaddr *)&sin, sizeof(sin));
 240
 241		if (err) {
 242			icsk->icsk_ext_hdr_len = exthdrlen;
 243			/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 244			WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
 245			if (sk_is_mptcp(sk))
 246				mptcpv6_handle_mapped(sk, false);
 247			sk->sk_backlog_rcv = tcp_v6_do_rcv;
 248#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 249			tp->af_specific = &tcp_sock_ipv6_specific;
 250#endif
 251			goto failure;
 252		}
 253		np->saddr = sk->sk_v6_rcv_saddr;
 254
 255		return err;
 256	}
 257
 258	if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
 259		saddr = &sk->sk_v6_rcv_saddr;
 260
 261	fl6.flowi6_proto = IPPROTO_TCP;
 262	fl6.daddr = sk->sk_v6_daddr;
 263	fl6.saddr = saddr ? *saddr : np->saddr;
 264	fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
 265	fl6.flowi6_oif = sk->sk_bound_dev_if;
 266	fl6.flowi6_mark = sk->sk_mark;
 267	fl6.fl6_dport = usin->sin6_port;
 268	fl6.fl6_sport = inet->inet_sport;
 269	fl6.flowi6_uid = sk->sk_uid;
 270
 271	opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
 272	final_p = fl6_update_dst(&fl6, opt, &final);
 273
 274	security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
 275
 276	dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p);
 277	if (IS_ERR(dst)) {
 278		err = PTR_ERR(dst);
 279		goto failure;
 280	}
 281
 282	tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
 283	tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
 284
 285	if (!saddr) {
 286		saddr = &fl6.saddr;
 287
 288		err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
 289		if (err)
 290			goto failure;
 291	}
 292
 293	/* set the source address */
 294	np->saddr = *saddr;
 295	inet->inet_rcv_saddr = LOOPBACK4_IPV6;
 296
 297	sk->sk_gso_type = SKB_GSO_TCPV6;
 298	ip6_dst_store(sk, dst, NULL, NULL);
 299
 300	icsk->icsk_ext_hdr_len = 0;
 301	if (opt)
 302		icsk->icsk_ext_hdr_len = opt->opt_flen +
 303					 opt->opt_nflen;
 304
 305	tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
 306
 307	inet->inet_dport = usin->sin6_port;
 308
 309	tcp_set_state(sk, TCP_SYN_SENT);
 310	err = inet6_hash_connect(tcp_death_row, sk);
 311	if (err)
 312		goto late_failure;
 313
 314	sk_set_txhash(sk);
 315
 316	if (likely(!tp->repair)) {
 317		if (!tp->write_seq)
 318			WRITE_ONCE(tp->write_seq,
 319				   secure_tcpv6_seq(np->saddr.s6_addr32,
 320						    sk->sk_v6_daddr.s6_addr32,
 321						    inet->inet_sport,
 322						    inet->inet_dport));
 323		tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32,
 324						   sk->sk_v6_daddr.s6_addr32);
 325	}
 326
 327	if (tcp_fastopen_defer_connect(sk, &err))
 328		return err;
 329	if (err)
 330		goto late_failure;
 331
 332	err = tcp_connect(sk);
 333	if (err)
 334		goto late_failure;
 335
 336	return 0;
 337
 338late_failure:
 339	tcp_set_state(sk, TCP_CLOSE);
 340	inet_bhash2_reset_saddr(sk);
 341failure:
 342	inet->inet_dport = 0;
 343	sk->sk_route_caps = 0;
 344	return err;
 345}
 346
 347static void tcp_v6_mtu_reduced(struct sock *sk)
 348{
 349	struct dst_entry *dst;
 350	u32 mtu;
 351
 352	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
 353		return;
 354
 355	mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
 356
 357	/* Drop requests trying to increase our current mss.
 358	 * Check done in __ip6_rt_update_pmtu() is too late.
 359	 */
 360	if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache)
 361		return;
 362
 363	dst = inet6_csk_update_pmtu(sk, mtu);
 364	if (!dst)
 365		return;
 366
 367	if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
 368		tcp_sync_mss(sk, dst_mtu(dst));
 369		tcp_simple_retransmit(sk);
 370	}
 371}
 372
 373static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 374		u8 type, u8 code, int offset, __be32 info)
 375{
 376	const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
 377	const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
 378	struct net *net = dev_net(skb->dev);
 379	struct request_sock *fastopen;
 380	struct ipv6_pinfo *np;
 381	struct tcp_sock *tp;
 382	__u32 seq, snd_una;
 383	struct sock *sk;
 384	bool fatal;
 385	int err;
 386
 387	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 388					&hdr->daddr, th->dest,
 389					&hdr->saddr, ntohs(th->source),
 390					skb->dev->ifindex, inet6_sdif(skb));
 391
 392	if (!sk) {
 393		__ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
 394				  ICMP6_MIB_INERRORS);
 395		return -ENOENT;
 396	}
 397
 398	if (sk->sk_state == TCP_TIME_WAIT) {
 399		/* To increase the counter of ignored icmps for TCP-AO */
 400		tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
 401		inet_twsk_put(inet_twsk(sk));
 402		return 0;
 403	}
 404	seq = ntohl(th->seq);
 405	fatal = icmpv6_err_convert(type, code, &err);
 406	if (sk->sk_state == TCP_NEW_SYN_RECV) {
 407		tcp_req_err(sk, seq, fatal);
 408		return 0;
 409	}
 410
 411	if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
 412		sock_put(sk);
 413		return 0;
 414	}
 415
 416	bh_lock_sock(sk);
 417	if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
 418		__NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
 419
 420	if (sk->sk_state == TCP_CLOSE)
 421		goto out;
 422
 423	if (static_branch_unlikely(&ip6_min_hopcount)) {
 424		/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
 425		if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
 426			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
 427			goto out;
 428		}
 429	}
 430
 431	tp = tcp_sk(sk);
 432	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
 433	fastopen = rcu_dereference(tp->fastopen_rsk);
 434	snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
 435	if (sk->sk_state != TCP_LISTEN &&
 436	    !between(seq, snd_una, tp->snd_nxt)) {
 437		__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 438		goto out;
 439	}
 440
 441	np = tcp_inet6_sk(sk);
 442
 443	if (type == NDISC_REDIRECT) {
 444		if (!sock_owned_by_user(sk)) {
 445			struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);
 446
 447			if (dst)
 448				dst->ops->redirect(dst, sk, skb);
 449		}
 450		goto out;
 451	}
 452
 453	if (type == ICMPV6_PKT_TOOBIG) {
 454		u32 mtu = ntohl(info);
 455
 456		/* We are not interested in TCP_LISTEN and open_requests
 457		 * (SYN-ACKs send out by Linux are always <576bytes so
 458		 * they should go through unfragmented).
 459		 */
 460		if (sk->sk_state == TCP_LISTEN)
 461			goto out;
 462
 463		if (!ip6_sk_accept_pmtu(sk))
 464			goto out;
 465
 466		if (mtu < IPV6_MIN_MTU)
 467			goto out;
 468
 469		WRITE_ONCE(tp->mtu_info, mtu);
 470
 471		if (!sock_owned_by_user(sk))
 472			tcp_v6_mtu_reduced(sk);
 473		else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
 474					   &sk->sk_tsq_flags))
 475			sock_hold(sk);
 476		goto out;
 477	}
 478
 479
 480	/* Might be for an request_sock */
 481	switch (sk->sk_state) {
 482	case TCP_SYN_SENT:
 483	case TCP_SYN_RECV:
 484		/* Only in fast or simultaneous open. If a fast open socket is
 485		 * already accepted it is treated as a connected one below.
 486		 */
 487		if (fastopen && !fastopen->sk)
 488			break;
 489
 490		ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
 491
 492		if (!sock_owned_by_user(sk)) {
 493			WRITE_ONCE(sk->sk_err, err);
 494			sk_error_report(sk);		/* Wake people up to see the error (see connect in sock.c) */
 495
 496			tcp_done(sk);
 497		} else {
 498			WRITE_ONCE(sk->sk_err_soft, err);
 499		}
 500		goto out;
 501	case TCP_LISTEN:
 502		break;
 503	default:
 504		/* check if this ICMP message allows revert of backoff.
 505		 * (see RFC 6069)
 506		 */
 507		if (!fastopen && type == ICMPV6_DEST_UNREACH &&
 508		    code == ICMPV6_NOROUTE)
 509			tcp_ld_RTO_revert(sk, seq);
 510	}
 511
 512	if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
 513		WRITE_ONCE(sk->sk_err, err);
 514		sk_error_report(sk);
 515	} else {
 516		WRITE_ONCE(sk->sk_err_soft, err);
 517	}
 518out:
 519	bh_unlock_sock(sk);
 520	sock_put(sk);
 521	return 0;
 522}
 523
 524
 525static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
 526			      struct flowi *fl,
 527			      struct request_sock *req,
 528			      struct tcp_fastopen_cookie *foc,
 529			      enum tcp_synack_type synack_type,
 530			      struct sk_buff *syn_skb)
 531{
 532	struct inet_request_sock *ireq = inet_rsk(req);
 533	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 534	struct ipv6_txoptions *opt;
 535	struct flowi6 *fl6 = &fl->u.ip6;
 536	struct sk_buff *skb;
 537	int err = -ENOMEM;
 538	u8 tclass;
 539
 540	/* First, grab a route. */
 541	if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
 542					       IPPROTO_TCP)) == NULL)
 543		goto done;
 544
 545	skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
 546
 547	if (skb) {
 548		__tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
 549				    &ireq->ir_v6_rmt_addr);
 550
 551		fl6->daddr = ireq->ir_v6_rmt_addr;
 552		if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
 553			fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));
 554
 555		tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
 556				(tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
 557				(np->tclass & INET_ECN_MASK) :
 558				np->tclass;
 559
 560		if (!INET_ECN_is_capable(tclass) &&
 561		    tcp_bpf_ca_needs_ecn((struct sock *)req))
 562			tclass |= INET_ECN_ECT_0;
 563
 564		rcu_read_lock();
 565		opt = ireq->ipv6_opt;
 566		if (!opt)
 567			opt = rcu_dereference(np->opt);
 568		err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark),
 569			       opt, tclass, READ_ONCE(sk->sk_priority));
 570		rcu_read_unlock();
 571		err = net_xmit_eval(err);
 572	}
 573
 574done:
 575	return err;
 576}
 577
 578
 579static void tcp_v6_reqsk_destructor(struct request_sock *req)
 580{
 581	kfree(inet_rsk(req)->ipv6_opt);
 582	consume_skb(inet_rsk(req)->pktopts);
 583}
 584
 585#ifdef CONFIG_TCP_MD5SIG
 586static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
 587						   const struct in6_addr *addr,
 588						   int l3index)
 589{
 590	return tcp_md5_do_lookup(sk, l3index,
 591				 (union tcp_md5_addr *)addr, AF_INET6);
 592}
 593
 594static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
 595						const struct sock *addr_sk)
 596{
 597	int l3index;
 598
 599	l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 600						 addr_sk->sk_bound_dev_if);
 601	return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
 602				    l3index);
 603}
 604
 605static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
 606				 sockptr_t optval, int optlen)
 607{
 608	struct tcp_md5sig cmd;
 609	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
 610	union tcp_ao_addr *addr;
 611	int l3index = 0;
 612	u8 prefixlen;
 613	bool l3flag;
 614	u8 flags;
 615
 616	if (optlen < sizeof(cmd))
 617		return -EINVAL;
 618
 619	if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
 620		return -EFAULT;
 621
 622	if (sin6->sin6_family != AF_INET6)
 623		return -EINVAL;
 624
 625	flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 626	l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 627
 628	if (optname == TCP_MD5SIG_EXT &&
 629	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
 630		prefixlen = cmd.tcpm_prefixlen;
 631		if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
 632					prefixlen > 32))
 633			return -EINVAL;
 634	} else {
 635		prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
 636	}
 637
 638	if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
 639	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
 640		struct net_device *dev;
 641
 642		rcu_read_lock();
 643		dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
 644		if (dev && netif_is_l3_master(dev))
 645			l3index = dev->ifindex;
 646		rcu_read_unlock();
 647
 648		/* ok to reference set/not set outside of rcu;
 649		 * right now device MUST be an L3 master
 650		 */
 651		if (!dev || !l3index)
 652			return -EINVAL;
 653	}
 654
 655	if (!cmd.tcpm_keylen) {
 656		if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 657			return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 658					      AF_INET, prefixlen,
 659					      l3index, flags);
 660		return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 661				      AF_INET6, prefixlen, l3index, flags);
 662	}
 663
 664	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
 665		return -EINVAL;
 666
 667	if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
 668		addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];
 669
 670		/* Don't allow keys for peers that have a matching TCP-AO key.
 671		 * See the comment in tcp_ao_add_cmd()
 672		 */
 673		if (tcp_ao_required(sk, addr, AF_INET,
 674				    l3flag ? l3index : -1, false))
 675			return -EKEYREJECTED;
 676		return tcp_md5_do_add(sk, addr,
 677				      AF_INET, prefixlen, l3index, flags,
 678				      cmd.tcpm_key, cmd.tcpm_keylen);
 679	}
 680
 681	addr = (union tcp_md5_addr *)&sin6->sin6_addr;
 682
 683	/* Don't allow keys for peers that have a matching TCP-AO key.
 684	 * See the comment in tcp_ao_add_cmd()
 685	 */
 686	if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false))
 687		return -EKEYREJECTED;
 688
 689	return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
 690			      cmd.tcpm_key, cmd.tcpm_keylen);
 691}
 692
 693static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp,
 694				   const struct in6_addr *daddr,
 695				   const struct in6_addr *saddr,
 696				   const struct tcphdr *th, int nbytes)
 697{
 698	struct tcp6_pseudohdr *bp;
 699	struct scatterlist sg;
 700	struct tcphdr *_th;
 701
 702	bp = hp->scratch;
 703	/* 1. TCP pseudo-header (RFC2460) */
 704	bp->saddr = *saddr;
 705	bp->daddr = *daddr;
 706	bp->protocol = cpu_to_be32(IPPROTO_TCP);
 707	bp->len = cpu_to_be32(nbytes);
 708
 709	_th = (struct tcphdr *)(bp + 1);
 710	memcpy(_th, th, sizeof(*th));
 711	_th->check = 0;
 712
 713	sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
 714	ahash_request_set_crypt(hp->req, &sg, NULL,
 715				sizeof(*bp) + sizeof(*th));
 716	return crypto_ahash_update(hp->req);
 717}
 718
 719static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
 720			       const struct in6_addr *daddr, struct in6_addr *saddr,
 721			       const struct tcphdr *th)
 722{
 723	struct tcp_sigpool hp;
 724
 725	if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 726		goto clear_hash_nostart;
 727
 728	if (crypto_ahash_init(hp.req))
 729		goto clear_hash;
 730	if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2))
 731		goto clear_hash;
 732	if (tcp_md5_hash_key(&hp, key))
 733		goto clear_hash;
 734	ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 735	if (crypto_ahash_final(hp.req))
 736		goto clear_hash;
 737
 738	tcp_sigpool_end(&hp);
 739	return 0;
 740
 741clear_hash:
 742	tcp_sigpool_end(&hp);
 743clear_hash_nostart:
 744	memset(md5_hash, 0, 16);
 745	return 1;
 746}
 747
 748static int tcp_v6_md5_hash_skb(char *md5_hash,
 749			       const struct tcp_md5sig_key *key,
 750			       const struct sock *sk,
 751			       const struct sk_buff *skb)
 752{
 753	const struct tcphdr *th = tcp_hdr(skb);
 754	const struct in6_addr *saddr, *daddr;
 755	struct tcp_sigpool hp;
 756
 757	if (sk) { /* valid for establish/request sockets */
 758		saddr = &sk->sk_v6_rcv_saddr;
 759		daddr = &sk->sk_v6_daddr;
 760	} else {
 761		const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 762		saddr = &ip6h->saddr;
 763		daddr = &ip6h->daddr;
 764	}
 765
 766	if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
 767		goto clear_hash_nostart;
 768
 769	if (crypto_ahash_init(hp.req))
 770		goto clear_hash;
 771
 772	if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len))
 773		goto clear_hash;
 774	if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
 775		goto clear_hash;
 776	if (tcp_md5_hash_key(&hp, key))
 777		goto clear_hash;
 778	ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
 779	if (crypto_ahash_final(hp.req))
 780		goto clear_hash;
 781
 782	tcp_sigpool_end(&hp);
 783	return 0;
 784
 785clear_hash:
 786	tcp_sigpool_end(&hp);
 787clear_hash_nostart:
 788	memset(md5_hash, 0, 16);
 789	return 1;
 790}
 791#endif
 792
 793static void tcp_v6_init_req(struct request_sock *req,
 794			    const struct sock *sk_listener,
 795			    struct sk_buff *skb)
 796{
 797	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
 798	struct inet_request_sock *ireq = inet_rsk(req);
 799	const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
 800
 801	ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
 802	ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
 803
 804	/* So that link locals have meaning */
 805	if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
 806	    ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
 807		ireq->ir_iif = tcp_v6_iif(skb);
 808
 809	if (!TCP_SKB_CB(skb)->tcp_tw_isn &&
 810	    (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
 811	     np->rxopt.bits.rxinfo ||
 812	     np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
 813	     np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
 814		refcount_inc(&skb->users);
 815		ireq->pktopts = skb;
 816	}
 817}
 818
 819static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
 820					  struct sk_buff *skb,
 821					  struct flowi *fl,
 822					  struct request_sock *req)
 823{
 824	tcp_v6_init_req(req, sk, skb);
 825
 826	if (security_inet_conn_request(sk, skb, req))
 827		return NULL;
 828
 829	return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP);
 830}
 831
 832struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
 833	.family		=	AF_INET6,
 834	.obj_size	=	sizeof(struct tcp6_request_sock),
 835	.rtx_syn_ack	=	tcp_rtx_synack,
 836	.send_ack	=	tcp_v6_reqsk_send_ack,
 837	.destructor	=	tcp_v6_reqsk_destructor,
 838	.send_reset	=	tcp_v6_send_reset,
 839	.syn_ack_timeout =	tcp_syn_ack_timeout,
 840};
 841
 842const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
 843	.mss_clamp	=	IPV6_MIN_MTU - sizeof(struct tcphdr) -
 844				sizeof(struct ipv6hdr),
 845#ifdef CONFIG_TCP_MD5SIG
 846	.req_md5_lookup	=	tcp_v6_md5_lookup,
 847	.calc_md5_hash	=	tcp_v6_md5_hash_skb,
 848#endif
 849#ifdef CONFIG_TCP_AO
 850	.ao_lookup	=	tcp_v6_ao_lookup_rsk,
 851	.ao_calc_key	=	tcp_v6_ao_calc_key_rsk,
 852	.ao_synack_hash =	tcp_v6_ao_synack_hash,
 853#endif
 854#ifdef CONFIG_SYN_COOKIES
 855	.cookie_init_seq =	cookie_v6_init_sequence,
 856#endif
 857	.route_req	=	tcp_v6_route_req,
 858	.init_seq	=	tcp_v6_init_seq,
 859	.init_ts_off	=	tcp_v6_init_ts_off,
 860	.send_synack	=	tcp_v6_send_synack,
 861};
 862
 863static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
 864				 u32 ack, u32 win, u32 tsval, u32 tsecr,
 865				 int oif, int rst, u8 tclass, __be32 label,
 866				 u32 priority, u32 txhash, struct tcp_key *key)
 867{
 868	const struct tcphdr *th = tcp_hdr(skb);
 869	struct tcphdr *t1;
 870	struct sk_buff *buff;
 871	struct flowi6 fl6;
 872	struct net *net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
 873	struct sock *ctl_sk = net->ipv6.tcp_sk;
 874	unsigned int tot_len = sizeof(struct tcphdr);
 875	__be32 mrst = 0, *topt;
 876	struct dst_entry *dst;
 877	__u32 mark = 0;
 878
 879	if (tsecr)
 880		tot_len += TCPOLEN_TSTAMP_ALIGNED;
 881	if (tcp_key_is_md5(key))
 882		tot_len += TCPOLEN_MD5SIG_ALIGNED;
 883	if (tcp_key_is_ao(key))
 884		tot_len += tcp_ao_len_aligned(key->ao_key);
 885
 886#ifdef CONFIG_MPTCP
 887	if (rst && !tcp_key_is_md5(key)) {
 888		mrst = mptcp_reset_option(skb);
 889
 890		if (mrst)
 891			tot_len += sizeof(__be32);
 892	}
 893#endif
 894
 895	buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
 896	if (!buff)
 897		return;
 898
 899	skb_reserve(buff, MAX_TCP_HEADER);
 900
 901	t1 = skb_push(buff, tot_len);
 902	skb_reset_transport_header(buff);
 903
 904	/* Swap the send and the receive. */
 905	memset(t1, 0, sizeof(*t1));
 906	t1->dest = th->source;
 907	t1->source = th->dest;
 908	t1->doff = tot_len / 4;
 909	t1->seq = htonl(seq);
 910	t1->ack_seq = htonl(ack);
 911	t1->ack = !rst || !th->ack;
 912	t1->rst = rst;
 913	t1->window = htons(win);
 914
 915	topt = (__be32 *)(t1 + 1);
 916
 917	if (tsecr) {
 918		*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 919				(TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
 920		*topt++ = htonl(tsval);
 921		*topt++ = htonl(tsecr);
 922	}
 923
 924	if (mrst)
 925		*topt++ = mrst;
 926
 927#ifdef CONFIG_TCP_MD5SIG
 928	if (tcp_key_is_md5(key)) {
 929		*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 930				(TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
 931		tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key,
 932				    &ipv6_hdr(skb)->saddr,
 933				    &ipv6_hdr(skb)->daddr, t1);
 934	}
 935#endif
 936#ifdef CONFIG_TCP_AO
 937	if (tcp_key_is_ao(key)) {
 938		*topt++ = htonl((TCPOPT_AO << 24) |
 939				(tcp_ao_len(key->ao_key) << 16) |
 940				(key->ao_key->sndid << 8) |
 941				(key->rcv_next));
 942
 943		tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key,
 944				key->traffic_key,
 945				(union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
 946				(union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
 947				t1, key->sne);
 948	}
 949#endif
 950
 951	memset(&fl6, 0, sizeof(fl6));
 952	fl6.daddr = ipv6_hdr(skb)->saddr;
 953	fl6.saddr = ipv6_hdr(skb)->daddr;
 954	fl6.flowlabel = label;
 955
 956	buff->ip_summed = CHECKSUM_PARTIAL;
 957
 958	__tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);
 959
 960	fl6.flowi6_proto = IPPROTO_TCP;
 961	if (rt6_need_strict(&fl6.daddr) && !oif)
 962		fl6.flowi6_oif = tcp_v6_iif(skb);
 963	else {
 964		if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
 965			oif = skb->skb_iif;
 966
 967		fl6.flowi6_oif = oif;
 968	}
 969
 970	if (sk) {
 971		if (sk->sk_state == TCP_TIME_WAIT)
 972			mark = inet_twsk(sk)->tw_mark;
 973		else
 974			mark = READ_ONCE(sk->sk_mark);
 975		skb_set_delivery_time(buff, tcp_transmit_time(sk), true);
 976	}
 977	if (txhash) {
 978		/* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
 979		skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4);
 980	}
 981	fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
 982	fl6.fl6_dport = t1->dest;
 983	fl6.fl6_sport = t1->source;
 984	fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
 985	security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6));
 986
 987	/* Pass a socket to ip6_dst_lookup either it is for RST
 988	 * Underlying function will use this to retrieve the network
 989	 * namespace
 990	 */
 991	if (sk && sk->sk_state != TCP_TIME_WAIT)
 992		dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/
 993	else
 994		dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL);
 995	if (!IS_ERR(dst)) {
 996		skb_dst_set(buff, dst);
 997		ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL,
 998			 tclass & ~INET_ECN_MASK, priority);
 999		TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
1000		if (rst)
1001			TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
1002		return;
1003	}
1004
1005	kfree_skb(buff);
1006}
1007
1008static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
1009{
1010	const struct tcphdr *th = tcp_hdr(skb);
1011	struct ipv6hdr *ipv6h = ipv6_hdr(skb);
1012	const __u8 *md5_hash_location = NULL;
1013#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1014	bool allocated_traffic_key = false;
1015#endif
1016	const struct tcp_ao_hdr *aoh;
1017	struct tcp_key key = {};
1018	u32 seq = 0, ack_seq = 0;
1019	__be32 label = 0;
1020	u32 priority = 0;
1021	struct net *net;
1022	u32 txhash = 0;
1023	int oif = 0;
1024#ifdef CONFIG_TCP_MD5SIG
1025	unsigned char newhash[16];
1026	int genhash;
1027	struct sock *sk1 = NULL;
1028#endif
1029
1030	if (th->rst)
1031		return;
1032
1033	/* If sk not NULL, it means we did a successful lookup and incoming
1034	 * route had to be correct. prequeue might have dropped our dst.
1035	 */
1036	if (!sk && !ipv6_unicast_destination(skb))
1037		return;
1038
1039	net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
1040	/* Invalid TCP option size or twice included auth */
1041	if (tcp_parse_auth_options(th, &md5_hash_location, &aoh))
1042		return;
1043#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1044	rcu_read_lock();
1045#endif
1046#ifdef CONFIG_TCP_MD5SIG
1047	if (sk && sk_fullsock(sk)) {
1048		int l3index;
1049
1050		/* sdif set, means packet ingressed via a device
1051		 * in an L3 domain and inet_iif is set to it.
1052		 */
1053		l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1054		key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
1055		if (key.md5_key)
1056			key.type = TCP_KEY_MD5;
1057	} else if (md5_hash_location) {
1058		int dif = tcp_v6_iif_l3_slave(skb);
1059		int sdif = tcp_v6_sdif(skb);
1060		int l3index;
1061
1062		/*
1063		 * active side is lost. Try to find listening socket through
1064		 * source port, and then find md5 key through listening socket.
1065		 * we are not loose security here:
1066		 * Incoming packet is checked with md5 hash with finding key,
1067		 * no RST generated if md5 hash doesn't match.
1068		 */
1069		sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1070					    NULL, 0, &ipv6h->saddr, th->source,
1071					    &ipv6h->daddr, ntohs(th->source),
1072					    dif, sdif);
1073		if (!sk1)
1074			goto out;
1075
1076		/* sdif set, means packet ingressed via a device
1077		 * in an L3 domain and dif is set to it.
1078		 */
1079		l3index = tcp_v6_sdif(skb) ? dif : 0;
1080
1081		key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
1082		if (!key.md5_key)
1083			goto out;
1084		key.type = TCP_KEY_MD5;
1085
1086		genhash = tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb);
1087		if (genhash || memcmp(md5_hash_location, newhash, 16) != 0)
1088			goto out;
1089	}
1090#endif
1091
1092	if (th->ack)
1093		seq = ntohl(th->ack_seq);
1094	else
1095		ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1096			  (th->doff << 2);
1097
1098#ifdef CONFIG_TCP_AO
1099	if (aoh) {
1100		int l3index;
1101
1102		l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1103		if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
1104					 &key.ao_key, &key.traffic_key,
1105					 &allocated_traffic_key,
1106					 &key.rcv_next, &key.sne))
1107			goto out;
1108		key.type = TCP_KEY_AO;
1109	}
1110#endif
1111
1112	if (sk) {
1113		oif = sk->sk_bound_dev_if;
1114		if (sk_fullsock(sk)) {
1115			trace_tcp_send_reset(sk, skb);
1116			if (inet6_test_bit(REPFLOW, sk))
1117				label = ip6_flowlabel(ipv6h);
1118			priority = READ_ONCE(sk->sk_priority);
1119			txhash = sk->sk_txhash;
1120		}
1121		if (sk->sk_state == TCP_TIME_WAIT) {
1122			label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1123			priority = inet_twsk(sk)->tw_priority;
1124			txhash = inet_twsk(sk)->tw_txhash;
1125		}
1126	} else {
1127		if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
1128			label = ip6_flowlabel(ipv6h);
1129	}
1130
1131	tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1,
1132			     ipv6_get_dsfield(ipv6h), label, priority, txhash,
1133			     &key);
1134
1135#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1136out:
1137	if (allocated_traffic_key)
1138		kfree(key.traffic_key);
1139	rcu_read_unlock();
1140#endif
1141}
1142
1143static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1144			    u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1145			    struct tcp_key *key, u8 tclass,
1146			    __be32 label, u32 priority, u32 txhash)
1147{
1148	tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0,
1149			     tclass, label, priority, txhash, key);
1150}
1151
1152static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
1153{
1154	struct inet_timewait_sock *tw = inet_twsk(sk);
1155	struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1156	struct tcp_key key = {};
1157#ifdef CONFIG_TCP_AO
1158	struct tcp_ao_info *ao_info;
1159
1160	if (static_branch_unlikely(&tcp_ao_needed.key)) {
1161
1162		/* FIXME: the segment to-be-acked is not verified yet */
1163		ao_info = rcu_dereference(tcptw->ao_info);
1164		if (ao_info) {
1165			const struct tcp_ao_hdr *aoh;
1166
1167			/* Invalid TCP option size or twice included auth */
1168			if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1169				goto out;
1170			if (aoh)
1171				key.ao_key = tcp_ao_established_key(ao_info,
1172						aoh->rnext_keyid, -1);
1173		}
1174	}
1175	if (key.ao_key) {
1176		struct tcp_ao_key *rnext_key;
1177
1178		key.traffic_key = snd_other_key(key.ao_key);
1179		/* rcv_next switches to our rcv_next */
1180		rnext_key = READ_ONCE(ao_info->rnext_key);
1181		key.rcv_next = rnext_key->rcvid;
1182		key.sne = READ_ONCE(ao_info->snd_sne);
1183		key.type = TCP_KEY_AO;
1184#else
1185	if (0) {
1186#endif
1187#ifdef CONFIG_TCP_MD5SIG
1188	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1189		key.md5_key = tcp_twsk_md5_key(tcptw);
1190		if (key.md5_key)
1191			key.type = TCP_KEY_MD5;
1192#endif
1193	}
1194
1195	tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
1196			tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1197			tcp_tw_tsval(tcptw),
1198			tcptw->tw_ts_recent, tw->tw_bound_dev_if, &key,
1199			tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), tw->tw_priority,
1200			tw->tw_txhash);
1201
1202#ifdef CONFIG_TCP_AO
1203out:
1204#endif
1205	inet_twsk_put(tw);
1206}
1207
1208static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1209				  struct request_sock *req)
1210{
1211	struct tcp_key key = {};
1212
1213#ifdef CONFIG_TCP_AO
1214	if (static_branch_unlikely(&tcp_ao_needed.key) &&
1215	    tcp_rsk_used_ao(req)) {
1216		const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
1217		const struct tcp_ao_hdr *aoh;
1218		int l3index;
1219
1220		l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1221		/* Invalid TCP option size or twice included auth */
1222		if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1223			return;
1224		if (!aoh)
1225			return;
1226		key.ao_key = tcp_ao_do_lookup(sk, l3index,
1227					      (union tcp_ao_addr *)addr,
1228					      AF_INET6, aoh->rnext_keyid, -1);
1229		if (unlikely(!key.ao_key)) {
1230			/* Send ACK with any matching MKT for the peer */
1231			key.ao_key = tcp_ao_do_lookup(sk, l3index,
1232						      (union tcp_ao_addr *)addr,
1233						      AF_INET6, -1, -1);
1234			/* Matching key disappeared (user removed the key?)
1235			 * let the handshake timeout.
1236			 */
1237			if (!key.ao_key) {
1238				net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
1239						     addr,
1240						     ntohs(tcp_hdr(skb)->source),
1241						     &ipv6_hdr(skb)->daddr,
1242						     ntohs(tcp_hdr(skb)->dest));
1243				return;
1244			}
1245		}
1246		key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1247		if (!key.traffic_key)
1248			return;
1249
1250		key.type = TCP_KEY_AO;
1251		key.rcv_next = aoh->keyid;
1252		tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
1253#else
1254	if (0) {
1255#endif
1256#ifdef CONFIG_TCP_MD5SIG
1257	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1258		int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1259
1260		key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr,
1261						   l3index);
1262		if (key.md5_key)
1263			key.type = TCP_KEY_MD5;
1264#endif
1265	}
1266
1267	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1268	 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1269	 */
1270	/* RFC 7323 2.3
1271	 * The window field (SEG.WND) of every outgoing segment, with the
1272	 * exception of <SYN> segments, MUST be right-shifted by
1273	 * Rcv.Wind.Shift bits:
1274	 */
1275	tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
1276			tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1277			tcp_rsk(req)->rcv_nxt,
1278			req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
1279			tcp_rsk_tsval(tcp_rsk(req)),
1280			READ_ONCE(req->ts_recent), sk->sk_bound_dev_if,
1281			&key, ipv6_get_dsfield(ipv6_hdr(skb)), 0,
1282			READ_ONCE(sk->sk_priority),
1283			READ_ONCE(tcp_rsk(req)->txhash));
1284	if (tcp_key_is_ao(&key))
1285		kfree(key.traffic_key);
1286}
1287
1288
1289static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1290{
1291#ifdef CONFIG_SYN_COOKIES
1292	const struct tcphdr *th = tcp_hdr(skb);
1293
1294	if (!th->syn)
1295		sk = cookie_v6_check(sk, skb);
1296#endif
1297	return sk;
1298}
1299
1300u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1301			 struct tcphdr *th, u32 *cookie)
1302{
1303	u16 mss = 0;
1304#ifdef CONFIG_SYN_COOKIES
1305	mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
1306				    &tcp_request_sock_ipv6_ops, sk, th);
1307	if (mss) {
1308		*cookie = __cookie_v6_init_sequence(iph, th, &mss);
1309		tcp_synq_overflow(sk);
1310	}
1311#endif
1312	return mss;
1313}
1314
1315static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1316{
1317	if (skb->protocol == htons(ETH_P_IP))
1318		return tcp_v4_conn_request(sk, skb);
1319
1320	if (!ipv6_unicast_destination(skb))
1321		goto drop;
1322
1323	if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
1324		__IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1325		return 0;
1326	}
1327
1328	return tcp_conn_request(&tcp6_request_sock_ops,
1329				&tcp_request_sock_ipv6_ops, sk, skb);
1330
1331drop:
1332	tcp_listendrop(sk);
1333	return 0; /* don't send reset */
1334}
1335
1336static void tcp_v6_restore_cb(struct sk_buff *skb)
1337{
1338	/* We need to move header back to the beginning if xfrm6_policy_check()
1339	 * and tcp_v6_fill_cb() are going to be called again.
1340	 * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1341	 */
1342	memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1343		sizeof(struct inet6_skb_parm));
1344}
1345
1346static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1347					 struct request_sock *req,
1348					 struct dst_entry *dst,
1349					 struct request_sock *req_unhash,
1350					 bool *own_req)
1351{
1352	struct inet_request_sock *ireq;
1353	struct ipv6_pinfo *newnp;
1354	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1355	struct ipv6_txoptions *opt;
1356	struct inet_sock *newinet;
1357	bool found_dup_sk = false;
1358	struct tcp_sock *newtp;
1359	struct sock *newsk;
1360#ifdef CONFIG_TCP_MD5SIG
1361	struct tcp_md5sig_key *key;
1362	int l3index;
1363#endif
1364	struct flowi6 fl6;
1365
1366	if (skb->protocol == htons(ETH_P_IP)) {
1367		/*
1368		 *	v6 mapped
1369		 */
1370
1371		newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst,
1372					     req_unhash, own_req);
1373
1374		if (!newsk)
1375			return NULL;
1376
1377		inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1378
1379		newnp = tcp_inet6_sk(newsk);
1380		newtp = tcp_sk(newsk);
1381
1382		memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1383
1384		newnp->saddr = newsk->sk_v6_rcv_saddr;
1385
1386		inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1387		if (sk_is_mptcp(newsk))
1388			mptcpv6_handle_mapped(newsk, true);
1389		newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1390#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1391		newtp->af_specific = &tcp_sock_ipv6_mapped_specific;
1392#endif
1393
1394		newnp->ipv6_mc_list = NULL;
1395		newnp->ipv6_ac_list = NULL;
1396		newnp->ipv6_fl_list = NULL;
1397		newnp->pktoptions  = NULL;
1398		newnp->opt	   = NULL;
1399		newnp->mcast_oif   = inet_iif(skb);
1400		newnp->mcast_hops  = ip_hdr(skb)->ttl;
1401		newnp->rcv_flowinfo = 0;
1402		if (inet6_test_bit(REPFLOW, sk))
1403			newnp->flow_label = 0;
1404
1405		/*
1406		 * No need to charge this sock to the relevant IPv6 refcnt debug socks count
1407		 * here, tcp_create_openreq_child now does this for us, see the comment in
1408		 * that function for the gory details. -acme
1409		 */
1410
1411		/* It is tricky place. Until this moment IPv4 tcp
1412		   worked with IPv6 icsk.icsk_af_ops.
1413		   Sync it now.
1414		 */
1415		tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie);
1416
1417		return newsk;
1418	}
1419
1420	ireq = inet_rsk(req);
1421
1422	if (sk_acceptq_is_full(sk))
1423		goto out_overflow;
1424
1425	if (!dst) {
1426		dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP);
1427		if (!dst)
1428			goto out;
1429	}
1430
1431	newsk = tcp_create_openreq_child(sk, req, skb);
1432	if (!newsk)
1433		goto out_nonewsk;
1434
1435	/*
1436	 * No need to charge this sock to the relevant IPv6 refcnt debug socks
1437	 * count here, tcp_create_openreq_child now does this for us, see the
1438	 * comment in that function for the gory details. -acme
1439	 */
1440
1441	newsk->sk_gso_type = SKB_GSO_TCPV6;
1442	ip6_dst_store(newsk, dst, NULL, NULL);
1443	inet6_sk_rx_dst_set(newsk, skb);
1444
1445	inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1446
1447	newtp = tcp_sk(newsk);
1448	newinet = inet_sk(newsk);
1449	newnp = tcp_inet6_sk(newsk);
1450
1451	memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1452
1453	newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr;
1454	newnp->saddr = ireq->ir_v6_loc_addr;
1455	newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr;
1456	newsk->sk_bound_dev_if = ireq->ir_iif;
1457
1458	/* Now IPv6 options...
1459
1460	   First: no IPv4 options.
1461	 */
1462	newinet->inet_opt = NULL;
1463	newnp->ipv6_mc_list = NULL;
1464	newnp->ipv6_ac_list = NULL;
1465	newnp->ipv6_fl_list = NULL;
1466
1467	/* Clone RX bits */
1468	newnp->rxopt.all = np->rxopt.all;
1469
1470	newnp->pktoptions = NULL;
1471	newnp->opt	  = NULL;
1472	newnp->mcast_oif  = tcp_v6_iif(skb);
1473	newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1474	newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
1475	if (inet6_test_bit(REPFLOW, sk))
1476		newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));
1477
1478	/* Set ToS of the new socket based upon the value of incoming SYN.
1479	 * ECT bits are set later in tcp_init_transfer().
1480	 */
1481	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1482		newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1483
1484	/* Clone native IPv6 options from listening socket (if any)
1485
1486	   Yes, keeping reference count would be much more clever,
1487	   but we make one more one thing there: reattach optmem
1488	   to newsk.
1489	 */
1490	opt = ireq->ipv6_opt;
1491	if (!opt)
1492		opt = rcu_dereference(np->opt);
1493	if (opt) {
1494		opt = ipv6_dup_options(newsk, opt);
1495		RCU_INIT_POINTER(newnp->opt, opt);
1496	}
1497	inet_csk(newsk)->icsk_ext_hdr_len = 0;
1498	if (opt)
1499		inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1500						    opt->opt_flen;
1501
1502	tcp_ca_openreq_child(newsk, dst);
1503
1504	tcp_sync_mss(newsk, dst_mtu(dst));
1505	newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1506
1507	tcp_initialize_rcv_mss(newsk);
1508
1509	newinet->inet_daddr = newinet->inet_saddr = LOOPBACK4_IPV6;
1510	newinet->inet_rcv_saddr = LOOPBACK4_IPV6;
1511
1512#ifdef CONFIG_TCP_MD5SIG
1513	l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1514
1515	if (!tcp_rsk_used_ao(req)) {
1516		/* Copy over the MD5 key from the original socket */
1517		key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
1518		if (key) {
1519			const union tcp_md5_addr *addr;
1520
1521			addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
1522			if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key)) {
1523				inet_csk_prepare_forced_close(newsk);
1524				tcp_done(newsk);
1525				goto out;
1526			}
1527		}
1528	}
1529#endif
1530#ifdef CONFIG_TCP_AO
1531	/* Copy over tcp_ao_info if any */
1532	if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
1533		goto out; /* OOM */
1534#endif
1535
1536	if (__inet_inherit_port(sk, newsk) < 0) {
1537		inet_csk_prepare_forced_close(newsk);
1538		tcp_done(newsk);
1539		goto out;
1540	}
1541	*own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1542				       &found_dup_sk);
1543	if (*own_req) {
1544		tcp_move_syn(newtp, req);
1545
1546		/* Clone pktoptions received with SYN, if we own the req */
1547		if (ireq->pktopts) {
1548			newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk);
1549			consume_skb(ireq->pktopts);
1550			ireq->pktopts = NULL;
1551			if (newnp->pktoptions)
1552				tcp_v6_restore_cb(newnp->pktoptions);
1553		}
1554	} else {
1555		if (!req_unhash && found_dup_sk) {
1556			/* This code path should only be executed in the
1557			 * syncookie case only
1558			 */
1559			bh_unlock_sock(newsk);
1560			sock_put(newsk);
1561			newsk = NULL;
1562		}
1563	}
1564
1565	return newsk;
1566
1567out_overflow:
1568	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1569out_nonewsk:
1570	dst_release(dst);
1571out:
1572	tcp_listendrop(sk);
1573	return NULL;
1574}
1575
1576INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1577							   u32));
1578/* The socket must have it's spinlock held when we get
1579 * here, unless it is a TCP_LISTEN socket.
1580 *
1581 * We have a potential double-lock case here, so even when
1582 * doing backlog processing we use the BH locking scheme.
1583 * This is because we cannot sleep with the original spinlock
1584 * held.
1585 */
1586INDIRECT_CALLABLE_SCOPE
1587int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1588{
1589	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1590	struct sk_buff *opt_skb = NULL;
1591	enum skb_drop_reason reason;
1592	struct tcp_sock *tp;
1593
1594	/* Imagine: socket is IPv6. IPv4 packet arrives,
1595	   goes to IPv4 receive handler and backlogged.
1596	   From backlog it always goes here. Kerboom...
1597	   Fortunately, tcp_rcv_established and rcv_established
1598	   handle them correctly, but it is not case with
1599	   tcp_v6_hnd_req and tcp_v6_send_reset().   --ANK
1600	 */
1601
1602	if (skb->protocol == htons(ETH_P_IP))
1603		return tcp_v4_do_rcv(sk, skb);
1604
1605	/*
1606	 *	socket locking is here for SMP purposes as backlog rcv
1607	 *	is currently called with bh processing disabled.
1608	 */
1609
1610	/* Do Stevens' IPV6_PKTOPTIONS.
1611
1612	   Yes, guys, it is the only place in our code, where we
1613	   may make it not affecting IPv4.
1614	   The rest of code is protocol independent,
1615	   and I do not like idea to uglify IPv4.
1616
1617	   Actually, all the idea behind IPV6_PKTOPTIONS
1618	   looks not very well thought. For now we latch
1619	   options, received in the last packet, enqueued
1620	   by tcp. Feel free to propose better solution.
1621					       --ANK (980728)
1622	 */
1623	if (np->rxopt.all)
1624		opt_skb = skb_clone_and_charge_r(skb, sk);
1625
1626	reason = SKB_DROP_REASON_NOT_SPECIFIED;
1627	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1628		struct dst_entry *dst;
1629
1630		dst = rcu_dereference_protected(sk->sk_rx_dst,
1631						lockdep_sock_is_held(sk));
1632
1633		sock_rps_save_rxhash(sk, skb);
1634		sk_mark_napi_id(sk, skb);
1635		if (dst) {
1636			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1637			    INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1638					    dst, sk->sk_rx_dst_cookie) == NULL) {
1639				RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1640				dst_release(dst);
1641			}
1642		}
1643
1644		tcp_rcv_established(sk, skb);
1645		if (opt_skb)
1646			goto ipv6_pktoptions;
1647		return 0;
1648	}
1649
1650	if (tcp_checksum_complete(skb))
1651		goto csum_err;
1652
1653	if (sk->sk_state == TCP_LISTEN) {
1654		struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1655
1656		if (!nsk)
1657			goto discard;
1658
1659		if (nsk != sk) {
1660			if (tcp_child_process(sk, nsk, skb))
1661				goto reset;
1662			if (opt_skb)
1663				__kfree_skb(opt_skb);
1664			return 0;
1665		}
1666	} else
1667		sock_rps_save_rxhash(sk, skb);
1668
1669	if (tcp_rcv_state_process(sk, skb))
1670		goto reset;
1671	if (opt_skb)
1672		goto ipv6_pktoptions;
1673	return 0;
1674
1675reset:
1676	tcp_v6_send_reset(sk, skb);
1677discard:
1678	if (opt_skb)
1679		__kfree_skb(opt_skb);
1680	kfree_skb_reason(skb, reason);
1681	return 0;
1682csum_err:
1683	reason = SKB_DROP_REASON_TCP_CSUM;
1684	trace_tcp_bad_csum(skb);
1685	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1686	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1687	goto discard;
1688
1689
1690ipv6_pktoptions:
1691	/* Do you ask, what is it?
1692
1693	   1. skb was enqueued by tcp.
1694	   2. skb is added to tail of read queue, rather than out of order.
1695	   3. socket is not in passive state.
1696	   4. Finally, it really contains options, which user wants to receive.
1697	 */
1698	tp = tcp_sk(sk);
1699	if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1700	    !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1701		if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1702			WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb));
1703		if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1704			WRITE_ONCE(np->mcast_hops,
1705				   ipv6_hdr(opt_skb)->hop_limit);
1706		if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1707			np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
1708		if (inet6_test_bit(REPFLOW, sk))
1709			np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
1710		if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
1711			tcp_v6_restore_cb(opt_skb);
1712			opt_skb = xchg(&np->pktoptions, opt_skb);
1713		} else {
1714			__kfree_skb(opt_skb);
1715			opt_skb = xchg(&np->pktoptions, NULL);
1716		}
1717	}
1718
1719	consume_skb(opt_skb);
1720	return 0;
1721}
1722
1723static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1724			   const struct tcphdr *th)
1725{
1726	/* This is tricky: we move IP6CB at its correct location into
1727	 * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1728	 * _decode_session6() uses IP6CB().
1729	 * barrier() makes sure compiler won't play aliasing games.
1730	 */
1731	memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1732		sizeof(struct inet6_skb_parm));
1733	barrier();
1734
1735	TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1736	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1737				    skb->len - th->doff*4);
1738	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1739	TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1740	TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1741	TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
1742	TCP_SKB_CB(skb)->sacked = 0;
1743	TCP_SKB_CB(skb)->has_rxtstamp =
1744			skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1745}
1746
1747INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1748{
1749	enum skb_drop_reason drop_reason;
1750	int sdif = inet6_sdif(skb);
1751	int dif = inet6_iif(skb);
1752	const struct tcphdr *th;
1753	const struct ipv6hdr *hdr;
1754	bool refcounted;
1755	struct sock *sk;
1756	int ret;
1757	struct net *net = dev_net(skb->dev);
1758
1759	drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1760	if (skb->pkt_type != PACKET_HOST)
1761		goto discard_it;
1762
1763	/*
1764	 *	Count it even if it's bad.
1765	 */
1766	__TCP_INC_STATS(net, TCP_MIB_INSEGS);
1767
1768	if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1769		goto discard_it;
1770
1771	th = (const struct tcphdr *)skb->data;
1772
1773	if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1774		drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1775		goto bad_packet;
1776	}
1777	if (!pskb_may_pull(skb, th->doff*4))
1778		goto discard_it;
1779
1780	if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1781		goto csum_error;
1782
1783	th = (const struct tcphdr *)skb->data;
1784	hdr = ipv6_hdr(skb);
1785
1786lookup:
1787	sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th),
1788				th->source, th->dest, inet6_iif(skb), sdif,
1789				&refcounted);
1790	if (!sk)
1791		goto no_tcp_socket;
1792
1793process:
1794	if (sk->sk_state == TCP_TIME_WAIT)
1795		goto do_time_wait;
1796
1797	if (sk->sk_state == TCP_NEW_SYN_RECV) {
1798		struct request_sock *req = inet_reqsk(sk);
1799		bool req_stolen = false;
1800		struct sock *nsk;
1801
1802		sk = req->rsk_listener;
1803		if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
1804			drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1805		else
1806			drop_reason = tcp_inbound_hash(sk, req, skb,
1807						       &hdr->saddr, &hdr->daddr,
1808						       AF_INET6, dif, sdif);
1809		if (drop_reason) {
1810			sk_drops_add(sk, skb);
1811			reqsk_put(req);
1812			goto discard_it;
1813		}
1814		if (tcp_checksum_complete(skb)) {
1815			reqsk_put(req);
1816			goto csum_error;
1817		}
1818		if (unlikely(sk->sk_state != TCP_LISTEN)) {
1819			nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
1820			if (!nsk) {
1821				inet_csk_reqsk_queue_drop_and_put(sk, req);
1822				goto lookup;
1823			}
1824			sk = nsk;
1825			/* reuseport_migrate_sock() has already held one sk_refcnt
1826			 * before returning.
1827			 */
1828		} else {
1829			sock_hold(sk);
1830		}
1831		refcounted = true;
1832		nsk = NULL;
1833		if (!tcp_filter(sk, skb)) {
1834			th = (const struct tcphdr *)skb->data;
1835			hdr = ipv6_hdr(skb);
1836			tcp_v6_fill_cb(skb, hdr, th);
1837			nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
1838		} else {
1839			drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1840		}
1841		if (!nsk) {
1842			reqsk_put(req);
1843			if (req_stolen) {
1844				/* Another cpu got exclusive access to req
1845				 * and created a full blown socket.
1846				 * Try to feed this packet to this socket
1847				 * instead of discarding it.
1848				 */
1849				tcp_v6_restore_cb(skb);
1850				sock_put(sk);
1851				goto lookup;
1852			}
1853			goto discard_and_relse;
1854		}
1855		nf_reset_ct(skb);
1856		if (nsk == sk) {
1857			reqsk_put(req);
1858			tcp_v6_restore_cb(skb);
1859		} else if (tcp_child_process(sk, nsk, skb)) {
1860			tcp_v6_send_reset(nsk, skb);
1861			goto discard_and_relse;
1862		} else {
1863			sock_put(sk);
1864			return 0;
1865		}
1866	}
1867
1868	if (static_branch_unlikely(&ip6_min_hopcount)) {
1869		/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
1870		if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
1871			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1872			drop_reason = SKB_DROP_REASON_TCP_MINTTL;
1873			goto discard_and_relse;
1874		}
1875	}
1876
1877	if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
1878		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1879		goto discard_and_relse;
1880	}
1881
1882	drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr,
1883				       AF_INET6, dif, sdif);
1884	if (drop_reason)
1885		goto discard_and_relse;
1886
1887	nf_reset_ct(skb);
1888
1889	if (tcp_filter(sk, skb)) {
1890		drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1891		goto discard_and_relse;
1892	}
1893	th = (const struct tcphdr *)skb->data;
1894	hdr = ipv6_hdr(skb);
1895	tcp_v6_fill_cb(skb, hdr, th);
1896
1897	skb->dev = NULL;
1898
1899	if (sk->sk_state == TCP_LISTEN) {
1900		ret = tcp_v6_do_rcv(sk, skb);
1901		goto put_and_return;
1902	}
1903
1904	sk_incoming_cpu_update(sk);
1905
1906	bh_lock_sock_nested(sk);
1907	tcp_segs_in(tcp_sk(sk), skb);
1908	ret = 0;
1909	if (!sock_owned_by_user(sk)) {
1910		ret = tcp_v6_do_rcv(sk, skb);
1911	} else {
1912		if (tcp_add_backlog(sk, skb, &drop_reason))
1913			goto discard_and_relse;
1914	}
1915	bh_unlock_sock(sk);
1916put_and_return:
1917	if (refcounted)
1918		sock_put(sk);
1919	return ret ? -1 : 0;
1920
1921no_tcp_socket:
1922	drop_reason = SKB_DROP_REASON_NO_SOCKET;
1923	if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1924		goto discard_it;
1925
1926	tcp_v6_fill_cb(skb, hdr, th);
1927
1928	if (tcp_checksum_complete(skb)) {
1929csum_error:
1930		drop_reason = SKB_DROP_REASON_TCP_CSUM;
1931		trace_tcp_bad_csum(skb);
1932		__TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1933bad_packet:
1934		__TCP_INC_STATS(net, TCP_MIB_INERRS);
1935	} else {
1936		tcp_v6_send_reset(NULL, skb);
1937	}
1938
1939discard_it:
1940	SKB_DR_OR(drop_reason, NOT_SPECIFIED);
1941	kfree_skb_reason(skb, drop_reason);
1942	return 0;
1943
1944discard_and_relse:
1945	sk_drops_add(sk, skb);
1946	if (refcounted)
1947		sock_put(sk);
1948	goto discard_it;
1949
1950do_time_wait:
1951	if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1952		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1953		inet_twsk_put(inet_twsk(sk));
1954		goto discard_it;
1955	}
1956
1957	tcp_v6_fill_cb(skb, hdr, th);
1958
1959	if (tcp_checksum_complete(skb)) {
1960		inet_twsk_put(inet_twsk(sk));
1961		goto csum_error;
1962	}
1963
1964	switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
1965	case TCP_TW_SYN:
1966	{
1967		struct sock *sk2;
1968
1969		sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
1970					    skb, __tcp_hdrlen(th),
1971					    &ipv6_hdr(skb)->saddr, th->source,
1972					    &ipv6_hdr(skb)->daddr,
1973					    ntohs(th->dest),
1974					    tcp_v6_iif_l3_slave(skb),
1975					    sdif);
1976		if (sk2) {
1977			struct inet_timewait_sock *tw = inet_twsk(sk);
1978			inet_twsk_deschedule_put(tw);
1979			sk = sk2;
1980			tcp_v6_restore_cb(skb);
1981			refcounted = false;
1982			goto process;
1983		}
1984	}
1985		/* to ACK */
1986		fallthrough;
1987	case TCP_TW_ACK:
1988		tcp_v6_timewait_ack(sk, skb);
1989		break;
1990	case TCP_TW_RST:
1991		tcp_v6_send_reset(sk, skb);
1992		inet_twsk_deschedule_put(inet_twsk(sk));
1993		goto discard_it;
1994	case TCP_TW_SUCCESS:
1995		;
1996	}
1997	goto discard_it;
1998}
1999
2000void tcp_v6_early_demux(struct sk_buff *skb)
2001{
2002	struct net *net = dev_net(skb->dev);
2003	const struct ipv6hdr *hdr;
2004	const struct tcphdr *th;
2005	struct sock *sk;
2006
2007	if (skb->pkt_type != PACKET_HOST)
2008		return;
2009
2010	if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
2011		return;
2012
2013	hdr = ipv6_hdr(skb);
2014	th = tcp_hdr(skb);
2015
2016	if (th->doff < sizeof(struct tcphdr) / 4)
2017		return;
2018
2019	/* Note : We use inet6_iif() here, not tcp_v6_iif() */
2020	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
2021					&hdr->saddr, th->source,
2022					&hdr->daddr, ntohs(th->dest),
2023					inet6_iif(skb), inet6_sdif(skb));
2024	if (sk) {
2025		skb->sk = sk;
2026		skb->destructor = sock_edemux;
2027		if (sk_fullsock(sk)) {
2028			struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
2029
2030			if (dst)
2031				dst = dst_check(dst, sk->sk_rx_dst_cookie);
2032			if (dst &&
2033			    sk->sk_rx_dst_ifindex == skb->skb_iif)
2034				skb_dst_set_noref(skb, dst);
2035		}
2036	}
2037}
2038
2039static struct timewait_sock_ops tcp6_timewait_sock_ops = {
2040	.twsk_obj_size	= sizeof(struct tcp6_timewait_sock),
2041	.twsk_unique	= tcp_twsk_unique,
2042	.twsk_destructor = tcp_twsk_destructor,
2043};
2044
2045INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
2046{
2047	__tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr);
2048}
2049
2050const struct inet_connection_sock_af_ops ipv6_specific = {
2051	.queue_xmit	   = inet6_csk_xmit,
2052	.send_check	   = tcp_v6_send_check,
2053	.rebuild_header	   = inet6_sk_rebuild_header,
2054	.sk_rx_dst_set	   = inet6_sk_rx_dst_set,
2055	.conn_request	   = tcp_v6_conn_request,
2056	.syn_recv_sock	   = tcp_v6_syn_recv_sock,
2057	.net_header_len	   = sizeof(struct ipv6hdr),
2058	.setsockopt	   = ipv6_setsockopt,
2059	.getsockopt	   = ipv6_getsockopt,
2060	.addr2sockaddr	   = inet6_csk_addr2sockaddr,
2061	.sockaddr_len	   = sizeof(struct sockaddr_in6),
2062	.mtu_reduced	   = tcp_v6_mtu_reduced,
2063};
2064
2065#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2066static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
2067#ifdef CONFIG_TCP_MD5SIG
2068	.md5_lookup	=	tcp_v6_md5_lookup,
2069	.calc_md5_hash	=	tcp_v6_md5_hash_skb,
2070	.md5_parse	=	tcp_v6_parse_md5_keys,
2071#endif
2072#ifdef CONFIG_TCP_AO
2073	.ao_lookup	=	tcp_v6_ao_lookup,
2074	.calc_ao_hash	=	tcp_v6_ao_hash_skb,
2075	.ao_parse	=	tcp_v6_parse_ao,
2076	.ao_calc_key_sk	=	tcp_v6_ao_calc_key_sk,
2077#endif
2078};
2079#endif
2080
2081/*
2082 *	TCP over IPv4 via INET6 API
2083 */
2084static const struct inet_connection_sock_af_ops ipv6_mapped = {
2085	.queue_xmit	   = ip_queue_xmit,
2086	.send_check	   = tcp_v4_send_check,
2087	.rebuild_header	   = inet_sk_rebuild_header,
2088	.sk_rx_dst_set	   = inet_sk_rx_dst_set,
2089	.conn_request	   = tcp_v6_conn_request,
2090	.syn_recv_sock	   = tcp_v6_syn_recv_sock,
2091	.net_header_len	   = sizeof(struct iphdr),
2092	.setsockopt	   = ipv6_setsockopt,
2093	.getsockopt	   = ipv6_getsockopt,
2094	.addr2sockaddr	   = inet6_csk_addr2sockaddr,
2095	.sockaddr_len	   = sizeof(struct sockaddr_in6),
2096	.mtu_reduced	   = tcp_v4_mtu_reduced,
2097};
2098
2099#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2100static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
2101#ifdef CONFIG_TCP_MD5SIG
2102	.md5_lookup	=	tcp_v4_md5_lookup,
2103	.calc_md5_hash	=	tcp_v4_md5_hash_skb,
2104	.md5_parse	=	tcp_v6_parse_md5_keys,
2105#endif
2106#ifdef CONFIG_TCP_AO
2107	.ao_lookup	=	tcp_v6_ao_lookup,
2108	.calc_ao_hash	=	tcp_v4_ao_hash_skb,
2109	.ao_parse	=	tcp_v6_parse_ao,
2110	.ao_calc_key_sk	=	tcp_v4_ao_calc_key_sk,
2111#endif
2112};
2113#endif
2114
2115/* NOTE: A lot of things set to zero explicitly by call to
2116 *       sk_alloc() so need not be done here.
2117 */
2118static int tcp_v6_init_sock(struct sock *sk)
2119{
2120	struct inet_connection_sock *icsk = inet_csk(sk);
2121
2122	tcp_init_sock(sk);
2123
2124	icsk->icsk_af_ops = &ipv6_specific;
2125
2126#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2127	tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
2128#endif
2129
2130	return 0;
2131}
2132
2133#ifdef CONFIG_PROC_FS
2134/* Proc filesystem TCPv6 sock list dumping. */
2135static void get_openreq6(struct seq_file *seq,
2136			 const struct request_sock *req, int i)
2137{
2138	long ttd = req->rsk_timer.expires - jiffies;
2139	const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
2140	const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;
2141
2142	if (ttd < 0)
2143		ttd = 0;
2144
2145	seq_printf(seq,
2146		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2147		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
2148		   i,
2149		   src->s6_addr32[0], src->s6_addr32[1],
2150		   src->s6_addr32[2], src->s6_addr32[3],
2151		   inet_rsk(req)->ir_num,
2152		   dest->s6_addr32[0], dest->s6_addr32[1],
2153		   dest->s6_addr32[2], dest->s6_addr32[3],
2154		   ntohs(inet_rsk(req)->ir_rmt_port),
2155		   TCP_SYN_RECV,
2156		   0, 0, /* could print option size, but that is af dependent. */
2157		   1,   /* timers active (only the expire timer) */
2158		   jiffies_to_clock_t(ttd),
2159		   req->num_timeout,
2160		   from_kuid_munged(seq_user_ns(seq),
2161				    sock_i_uid(req->rsk_listener)),
2162		   0,  /* non standard timer */
2163		   0, /* open_requests have no inode */
2164		   0, req);
2165}
2166
2167static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
2168{
2169	const struct in6_addr *dest, *src;
2170	__u16 destp, srcp;
2171	int timer_active;
2172	unsigned long timer_expires;
2173	const struct inet_sock *inet = inet_sk(sp);
2174	const struct tcp_sock *tp = tcp_sk(sp);
2175	const struct inet_connection_sock *icsk = inet_csk(sp);
2176	const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2177	int rx_queue;
2178	int state;
2179
2180	dest  = &sp->sk_v6_daddr;
2181	src   = &sp->sk_v6_rcv_saddr;
2182	destp = ntohs(inet->inet_dport);
2183	srcp  = ntohs(inet->inet_sport);
2184
2185	if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2186	    icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2187	    icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2188		timer_active	= 1;
2189		timer_expires	= icsk->icsk_timeout;
2190	} else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2191		timer_active	= 4;
2192		timer_expires	= icsk->icsk_timeout;
2193	} else if (timer_pending(&sp->sk_timer)) {
2194		timer_active	= 2;
2195		timer_expires	= sp->sk_timer.expires;
2196	} else {
2197		timer_active	= 0;
2198		timer_expires = jiffies;
2199	}
2200
2201	state = inet_sk_state_load(sp);
2202	if (state == TCP_LISTEN)
2203		rx_queue = READ_ONCE(sp->sk_ack_backlog);
2204	else
2205		/* Because we don't lock the socket,
2206		 * we might find a transient negative value.
2207		 */
2208		rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2209				      READ_ONCE(tp->copied_seq), 0);
2210
2211	seq_printf(seq,
2212		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2213		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
2214		   i,
2215		   src->s6_addr32[0], src->s6_addr32[1],
2216		   src->s6_addr32[2], src->s6_addr32[3], srcp,
2217		   dest->s6_addr32[0], dest->s6_addr32[1],
2218		   dest->s6_addr32[2], dest->s6_addr32[3], destp,
2219		   state,
2220		   READ_ONCE(tp->write_seq) - tp->snd_una,
2221		   rx_queue,
2222		   timer_active,
2223		   jiffies_delta_to_clock_t(timer_expires - jiffies),
2224		   icsk->icsk_retransmits,
2225		   from_kuid_munged(seq_user_ns(seq), sock_i_uid(sp)),
2226		   icsk->icsk_probes_out,
2227		   sock_i_ino(sp),
2228		   refcount_read(&sp->sk_refcnt), sp,
2229		   jiffies_to_clock_t(icsk->icsk_rto),
2230		   jiffies_to_clock_t(icsk->icsk_ack.ato),
2231		   (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
2232		   tcp_snd_cwnd(tp),
2233		   state == TCP_LISTEN ?
2234			fastopenq->max_qlen :
2235			(tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2236		   );
2237}
2238
2239static void get_timewait6_sock(struct seq_file *seq,
2240			       struct inet_timewait_sock *tw, int i)
2241{
2242	long delta = tw->tw_timer.expires - jiffies;
2243	const struct in6_addr *dest, *src;
2244	__u16 destp, srcp;
2245
2246	dest = &tw->tw_v6_daddr;
2247	src  = &tw->tw_v6_rcv_saddr;
2248	destp = ntohs(tw->tw_dport);
2249	srcp  = ntohs(tw->tw_sport);
2250
2251	seq_printf(seq,
2252		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2253		   "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2254		   i,
2255		   src->s6_addr32[0], src->s6_addr32[1],
2256		   src->s6_addr32[2], src->s6_addr32[3], srcp,
2257		   dest->s6_addr32[0], dest->s6_addr32[1],
2258		   dest->s6_addr32[2], dest->s6_addr32[3], destp,
2259		   tw->tw_substate, 0, 0,
2260		   3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2261		   refcount_read(&tw->tw_refcnt), tw);
2262}
2263
2264static int tcp6_seq_show(struct seq_file *seq, void *v)
2265{
2266	struct tcp_iter_state *st;
2267	struct sock *sk = v;
2268
2269	if (v == SEQ_START_TOKEN) {
2270		seq_puts(seq,
2271			 "  sl  "
2272			 "local_address                         "
2273			 "remote_address                        "
2274			 "st tx_queue rx_queue tr tm->when retrnsmt"
2275			 "   uid  timeout inode\n");
2276		goto out;
2277	}
2278	st = seq->private;
2279
2280	if (sk->sk_state == TCP_TIME_WAIT)
2281		get_timewait6_sock(seq, v, st->num);
2282	else if (sk->sk_state == TCP_NEW_SYN_RECV)
2283		get_openreq6(seq, v, st->num);
2284	else
2285		get_tcp6_sock(seq, v, st->num);
2286out:
2287	return 0;
2288}
2289
2290static const struct seq_operations tcp6_seq_ops = {
2291	.show		= tcp6_seq_show,
2292	.start		= tcp_seq_start,
2293	.next		= tcp_seq_next,
2294	.stop		= tcp_seq_stop,
2295};
2296
2297static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2298	.family		= AF_INET6,
2299};
2300
2301int __net_init tcp6_proc_init(struct net *net)
2302{
2303	if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
2304			sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
2305		return -ENOMEM;
2306	return 0;
2307}
2308
2309void tcp6_proc_exit(struct net *net)
2310{
2311	remove_proc_entry("tcp6", net->proc_net);
2312}
2313#endif
2314
2315struct proto tcpv6_prot = {
2316	.name			= "TCPv6",
2317	.owner			= THIS_MODULE,
2318	.close			= tcp_close,
2319	.pre_connect		= tcp_v6_pre_connect,
2320	.connect		= tcp_v6_connect,
2321	.disconnect		= tcp_disconnect,
2322	.accept			= inet_csk_accept,
2323	.ioctl			= tcp_ioctl,
2324	.init			= tcp_v6_init_sock,
2325	.destroy		= tcp_v4_destroy_sock,
2326	.shutdown		= tcp_shutdown,
2327	.setsockopt		= tcp_setsockopt,
2328	.getsockopt		= tcp_getsockopt,
2329	.bpf_bypass_getsockopt	= tcp_bpf_bypass_getsockopt,
2330	.keepalive		= tcp_set_keepalive,
2331	.recvmsg		= tcp_recvmsg,
2332	.sendmsg		= tcp_sendmsg,
2333	.splice_eof		= tcp_splice_eof,
2334	.backlog_rcv		= tcp_v6_do_rcv,
2335	.release_cb		= tcp_release_cb,
2336	.hash			= inet6_hash,
2337	.unhash			= inet_unhash,
2338	.get_port		= inet_csk_get_port,
2339	.put_port		= inet_put_port,
2340#ifdef CONFIG_BPF_SYSCALL
2341	.psock_update_sk_prot	= tcp_bpf_update_proto,
2342#endif
2343	.enter_memory_pressure	= tcp_enter_memory_pressure,
2344	.leave_memory_pressure	= tcp_leave_memory_pressure,
2345	.stream_memory_free	= tcp_stream_memory_free,
2346	.sockets_allocated	= &tcp_sockets_allocated,
2347
2348	.memory_allocated	= &tcp_memory_allocated,
2349	.per_cpu_fw_alloc	= &tcp_memory_per_cpu_fw_alloc,
2350
2351	.memory_pressure	= &tcp_memory_pressure,
2352	.orphan_count		= &tcp_orphan_count,
2353	.sysctl_mem		= sysctl_tcp_mem,
2354	.sysctl_wmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_wmem),
2355	.sysctl_rmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_rmem),
2356	.max_header		= MAX_TCP_HEADER,
2357	.obj_size		= sizeof(struct tcp6_sock),
2358	.ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
2359	.slab_flags		= SLAB_TYPESAFE_BY_RCU,
2360	.twsk_prot		= &tcp6_timewait_sock_ops,
2361	.rsk_prot		= &tcp6_request_sock_ops,
2362	.h.hashinfo		= NULL,
2363	.no_autobind		= true,
2364	.diag_destroy		= tcp_abort,
2365};
2366EXPORT_SYMBOL_GPL(tcpv6_prot);
2367
2368static const struct inet6_protocol tcpv6_protocol = {
2369	.handler	=	tcp_v6_rcv,
2370	.err_handler	=	tcp_v6_err,
2371	.flags		=	INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
2372};
2373
2374static struct inet_protosw tcpv6_protosw = {
2375	.type		=	SOCK_STREAM,
2376	.protocol	=	IPPROTO_TCP,
2377	.prot		=	&tcpv6_prot,
2378	.ops		=	&inet6_stream_ops,
2379	.flags		=	INET_PROTOSW_PERMANENT |
2380				INET_PROTOSW_ICSK,
2381};
2382
2383static int __net_init tcpv6_net_init(struct net *net)
2384{
2385	return inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
2386				    SOCK_RAW, IPPROTO_TCP, net);
2387}
2388
2389static void __net_exit tcpv6_net_exit(struct net *net)
2390{
2391	inet_ctl_sock_destroy(net->ipv6.tcp_sk);
2392}
2393
2394static void __net_exit tcpv6_net_exit_batch(struct list_head *net_exit_list)
2395{
2396	tcp_twsk_purge(net_exit_list, AF_INET6);
2397}
2398
2399static struct pernet_operations tcpv6_net_ops = {
2400	.init	    = tcpv6_net_init,
2401	.exit	    = tcpv6_net_exit,
2402	.exit_batch = tcpv6_net_exit_batch,
2403};
2404
2405int __init tcpv6_init(void)
2406{
2407	int ret;
2408
2409	ret = inet6_add_protocol(&tcpv6_protocol, IPPROTO_TCP);
2410	if (ret)
2411		goto out;
2412
2413	/* register inet6 protocol */
2414	ret = inet6_register_protosw(&tcpv6_protosw);
2415	if (ret)
2416		goto out_tcpv6_protocol;
2417
2418	ret = register_pernet_subsys(&tcpv6_net_ops);
2419	if (ret)
2420		goto out_tcpv6_protosw;
2421
2422	ret = mptcpv6_init();
2423	if (ret)
2424		goto out_tcpv6_pernet_subsys;
2425
2426out:
2427	return ret;
2428
2429out_tcpv6_pernet_subsys:
2430	unregister_pernet_subsys(&tcpv6_net_ops);
2431out_tcpv6_protosw:
2432	inet6_unregister_protosw(&tcpv6_protosw);
2433out_tcpv6_protocol:
2434	inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2435	goto out;
2436}
2437
2438void tcpv6_exit(void)
2439{
2440	unregister_pernet_subsys(&tcpv6_net_ops);
2441	inet6_unregister_protosw(&tcpv6_protosw);
2442	inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2443}