1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4 *
   5 *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6 *  applies to SOCK_STREAM sockets only
   7 *  offers an alternative communication option for TCP-protocol sockets
   8 *  applicable with RoCE-cards only
   9 *
  10 *  Initial restrictions:
  11 *    - support for alternate links postponed
  12 *
  13 *  Copyright IBM Corp. 2016, 2018
  14 *
  15 *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16 *              based on prototype from Frank Blaschka
  17 */
  18
  19#define KMSG_COMPONENT "smc"
  20#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22#include <linux/module.h>
  23#include <linux/socket.h>
  24#include <linux/workqueue.h>
  25#include <linux/in.h>
  26#include <linux/sched/signal.h>
  27#include <linux/if_vlan.h>
  28#include <linux/rcupdate_wait.h>
  29#include <linux/ctype.h>
  30
  31#include <net/sock.h>
  32#include <net/tcp.h>
  33#include <net/smc.h>
  34#include <asm/ioctls.h>
  35
  36#include <net/net_namespace.h>
  37#include <net/netns/generic.h>
  38#include "smc_netns.h"
  39
  40#include "smc.h"
  41#include "smc_clc.h"
  42#include "smc_llc.h"
  43#include "smc_cdc.h"
  44#include "smc_core.h"
  45#include "smc_ib.h"
  46#include "smc_ism.h"
  47#include "smc_pnet.h"
  48#include "smc_netlink.h"
  49#include "smc_tx.h"
  50#include "smc_rx.h"
  51#include "smc_close.h"
  52#include "smc_stats.h"
  53#include "smc_tracepoint.h"
  54#include "smc_sysctl.h"
  55
  56static DEFINE_MUTEX(smc_server_lgr_pending);	/* serialize link group
  57						 * creation on server
  58						 */
  59static DEFINE_MUTEX(smc_client_lgr_pending);	/* serialize link group
  60						 * creation on client
  61						 */
  62
  63static struct workqueue_struct	*smc_tcp_ls_wq;	/* wq for tcp listen work */
  64struct workqueue_struct	*smc_hs_wq;	/* wq for handshake work */
  65struct workqueue_struct	*smc_close_wq;	/* wq for close work */
  66
  67static void smc_tcp_listen_work(struct work_struct *);
  68static void smc_connect_work(struct work_struct *);
  69
  70int smc_nl_dump_hs_limitation(struct sk_buff *skb, struct netlink_callback *cb)
  71{
  72	struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb);
  73	void *hdr;
  74
  75	if (cb_ctx->pos[0])
  76		goto out;
  77
  78	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
  79			  &smc_gen_nl_family, NLM_F_MULTI,
  80			  SMC_NETLINK_DUMP_HS_LIMITATION);
  81	if (!hdr)
  82		return -ENOMEM;
  83
  84	if (nla_put_u8(skb, SMC_NLA_HS_LIMITATION_ENABLED,
  85		       sock_net(skb->sk)->smc.limit_smc_hs))
  86		goto err;
  87
  88	genlmsg_end(skb, hdr);
  89	cb_ctx->pos[0] = 1;
  90out:
  91	return skb->len;
  92err:
  93	genlmsg_cancel(skb, hdr);
  94	return -EMSGSIZE;
  95}
  96
  97int smc_nl_enable_hs_limitation(struct sk_buff *skb, struct genl_info *info)
  98{
  99	sock_net(skb->sk)->smc.limit_smc_hs = true;
 100	return 0;
 101}
 102
 103int smc_nl_disable_hs_limitation(struct sk_buff *skb, struct genl_info *info)
 104{
 105	sock_net(skb->sk)->smc.limit_smc_hs = false;
 106	return 0;
 107}
 108
 109static void smc_set_keepalive(struct sock *sk, int val)
 110{
 111	struct smc_sock *smc = smc_sk(sk);
 112
 113	smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
 114}
 115
 116static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk,
 117					  struct sk_buff *skb,
 118					  struct request_sock *req,
 119					  struct dst_entry *dst,
 120					  struct request_sock *req_unhash,
 121					  bool *own_req)
 122{
 123	struct smc_sock *smc;
 124	struct sock *child;
 125
 126	smc = smc_clcsock_user_data(sk);
 127
 128	if (READ_ONCE(sk->sk_ack_backlog) + atomic_read(&smc->queued_smc_hs) >
 129				sk->sk_max_ack_backlog)
 130		goto drop;
 131
 132	if (sk_acceptq_is_full(&smc->sk)) {
 133		NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
 134		goto drop;
 135	}
 136
 137	/* passthrough to original syn recv sock fct */
 138	child = smc->ori_af_ops->syn_recv_sock(sk, skb, req, dst, req_unhash,
 139					       own_req);
 140	/* child must not inherit smc or its ops */
 141	if (child) {
 142		rcu_assign_sk_user_data(child, NULL);
 143
 144		/* v4-mapped sockets don't inherit parent ops. Don't restore. */
 145		if (inet_csk(child)->icsk_af_ops == inet_csk(sk)->icsk_af_ops)
 146			inet_csk(child)->icsk_af_ops = smc->ori_af_ops;
 147	}
 148	return child;
 149
 150drop:
 151	dst_release(dst);
 152	tcp_listendrop(sk);
 153	return NULL;
 154}
 155
 156static bool smc_hs_congested(const struct sock *sk)
 157{
 158	const struct smc_sock *smc;
 159
 160	smc = smc_clcsock_user_data(sk);
 161
 162	if (!smc)
 163		return true;
 164
 165	if (workqueue_congested(WORK_CPU_UNBOUND, smc_hs_wq))
 166		return true;
 167
 168	return false;
 169}
 170
 171static struct smc_hashinfo smc_v4_hashinfo = {
 172	.lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
 173};
 174
 175static struct smc_hashinfo smc_v6_hashinfo = {
 176	.lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
 177};
 178
 179int smc_hash_sk(struct sock *sk)
 180{
 181	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
 182	struct hlist_head *head;
 183
 184	head = &h->ht;
 185
 186	write_lock_bh(&h->lock);
 187	sk_add_node(sk, head);
 188	write_unlock_bh(&h->lock);
 189	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
 190
 191	return 0;
 192}
 193EXPORT_SYMBOL_GPL(smc_hash_sk);
 194
 195void smc_unhash_sk(struct sock *sk)
 196{
 197	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
 198
 199	write_lock_bh(&h->lock);
 200	if (sk_del_node_init(sk))
 201		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 202	write_unlock_bh(&h->lock);
 203}
 204EXPORT_SYMBOL_GPL(smc_unhash_sk);
 205
 206/* This will be called before user really release sock_lock. So do the
 207 * work which we didn't do because of user hold the sock_lock in the
 208 * BH context
 209 */
 210static void smc_release_cb(struct sock *sk)
 211{
 212	struct smc_sock *smc = smc_sk(sk);
 213
 214	if (smc->conn.tx_in_release_sock) {
 215		smc_tx_pending(&smc->conn);
 216		smc->conn.tx_in_release_sock = false;
 217	}
 218}
 219
 220struct proto smc_proto = {
 221	.name		= "SMC",
 222	.owner		= THIS_MODULE,
 223	.keepalive	= smc_set_keepalive,
 224	.hash		= smc_hash_sk,
 225	.unhash		= smc_unhash_sk,
 226	.release_cb	= smc_release_cb,
 227	.obj_size	= sizeof(struct smc_sock),
 228	.h.smc_hash	= &smc_v4_hashinfo,
 229	.slab_flags	= SLAB_TYPESAFE_BY_RCU,
 230};
 231EXPORT_SYMBOL_GPL(smc_proto);
 232
 233struct proto smc_proto6 = {
 234	.name		= "SMC6",
 235	.owner		= THIS_MODULE,
 236	.keepalive	= smc_set_keepalive,
 237	.hash		= smc_hash_sk,
 238	.unhash		= smc_unhash_sk,
 239	.release_cb	= smc_release_cb,
 240	.obj_size	= sizeof(struct smc_sock),
 241	.h.smc_hash	= &smc_v6_hashinfo,
 242	.slab_flags	= SLAB_TYPESAFE_BY_RCU,
 243};
 244EXPORT_SYMBOL_GPL(smc_proto6);
 245
 246static void smc_fback_restore_callbacks(struct smc_sock *smc)
 247{
 248	struct sock *clcsk = smc->clcsock->sk;
 249
 250	write_lock_bh(&clcsk->sk_callback_lock);
 251	clcsk->sk_user_data = NULL;
 252
 253	smc_clcsock_restore_cb(&clcsk->sk_state_change, &smc->clcsk_state_change);
 254	smc_clcsock_restore_cb(&clcsk->sk_data_ready, &smc->clcsk_data_ready);
 255	smc_clcsock_restore_cb(&clcsk->sk_write_space, &smc->clcsk_write_space);
 256	smc_clcsock_restore_cb(&clcsk->sk_error_report, &smc->clcsk_error_report);
 257
 258	write_unlock_bh(&clcsk->sk_callback_lock);
 259}
 260
 261static void smc_restore_fallback_changes(struct smc_sock *smc)
 262{
 263	if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
 264		smc->clcsock->file->private_data = smc->sk.sk_socket;
 265		smc->clcsock->file = NULL;
 266		smc_fback_restore_callbacks(smc);
 267	}
 268}
 269
 270static int __smc_release(struct smc_sock *smc)
 271{
 272	struct sock *sk = &smc->sk;
 273	int rc = 0;
 274
 275	if (!smc->use_fallback) {
 276		rc = smc_close_active(smc);
 277		sock_set_flag(sk, SOCK_DEAD);
 278		sk->sk_shutdown |= SHUTDOWN_MASK;
 279	} else {
 280		if (sk->sk_state != SMC_CLOSED) {
 281			if (sk->sk_state != SMC_LISTEN &&
 282			    sk->sk_state != SMC_INIT)
 283				sock_put(sk); /* passive closing */
 284			if (sk->sk_state == SMC_LISTEN) {
 285				/* wake up clcsock accept */
 286				rc = kernel_sock_shutdown(smc->clcsock,
 287							  SHUT_RDWR);
 288			}
 289			sk->sk_state = SMC_CLOSED;
 290			sk->sk_state_change(sk);
 291		}
 292		smc_restore_fallback_changes(smc);
 293	}
 294
 295	sk->sk_prot->unhash(sk);
 296
 297	if (sk->sk_state == SMC_CLOSED) {
 298		if (smc->clcsock) {
 299			release_sock(sk);
 300			smc_clcsock_release(smc);
 301			lock_sock(sk);
 302		}
 303		if (!smc->use_fallback)
 304			smc_conn_free(&smc->conn);
 305	}
 306
 307	return rc;
 308}
 309
 310static int smc_release(struct socket *sock)
 311{
 312	struct sock *sk = sock->sk;
 313	struct smc_sock *smc;
 314	int old_state, rc = 0;
 315
 316	if (!sk)
 317		goto out;
 318
 319	sock_hold(sk); /* sock_put below */
 320	smc = smc_sk(sk);
 321
 322	old_state = sk->sk_state;
 323
 324	/* cleanup for a dangling non-blocking connect */
 325	if (smc->connect_nonblock && old_state == SMC_INIT)
 326		tcp_abort(smc->clcsock->sk, ECONNABORTED);
 327
 328	if (cancel_work_sync(&smc->connect_work))
 329		sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */
 330
 331	if (sk->sk_state == SMC_LISTEN)
 332		/* smc_close_non_accepted() is called and acquires
 333		 * sock lock for child sockets again
 334		 */
 335		lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 336	else
 337		lock_sock(sk);
 338
 339	if (old_state == SMC_INIT && sk->sk_state == SMC_ACTIVE &&
 340	    !smc->use_fallback)
 341		smc_close_active_abort(smc);
 342
 343	rc = __smc_release(smc);
 344
 345	/* detach socket */
 346	sock_orphan(sk);
 347	sock->sk = NULL;
 348	release_sock(sk);
 349
 350	sock_put(sk); /* sock_hold above */
 351	sock_put(sk); /* final sock_put */
 352out:
 353	return rc;
 354}
 355
 356static void smc_destruct(struct sock *sk)
 357{
 358	if (sk->sk_state != SMC_CLOSED)
 359		return;
 360	if (!sock_flag(sk, SOCK_DEAD))
 361		return;
 362
 363	sk_refcnt_debug_dec(sk);
 364}
 365
 366static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 367				   int protocol)
 368{
 369	struct smc_sock *smc;
 370	struct proto *prot;
 371	struct sock *sk;
 372
 373	prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 374	sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 375	if (!sk)
 376		return NULL;
 377
 378	sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 379	sk->sk_state = SMC_INIT;
 380	sk->sk_destruct = smc_destruct;
 381	sk->sk_protocol = protocol;
 382	WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(net->smc.sysctl_wmem));
 383	WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(net->smc.sysctl_rmem));
 384	smc = smc_sk(sk);
 385	INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 386	INIT_WORK(&smc->connect_work, smc_connect_work);
 387	INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 388	INIT_LIST_HEAD(&smc->accept_q);
 389	spin_lock_init(&smc->accept_q_lock);
 390	spin_lock_init(&smc->conn.send_lock);
 391	sk->sk_prot->hash(sk);
 392	sk_refcnt_debug_inc(sk);
 393	mutex_init(&smc->clcsock_release_lock);
 394	smc_init_saved_callbacks(smc);
 395
 396	return sk;
 397}
 398
 399static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 400		    int addr_len)
 401{
 402	struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 403	struct sock *sk = sock->sk;
 404	struct smc_sock *smc;
 405	int rc;
 406
 407	smc = smc_sk(sk);
 408
 409	/* replicate tests from inet_bind(), to be safe wrt. future changes */
 410	rc = -EINVAL;
 411	if (addr_len < sizeof(struct sockaddr_in))
 412		goto out;
 413
 414	rc = -EAFNOSUPPORT;
 415	if (addr->sin_family != AF_INET &&
 416	    addr->sin_family != AF_INET6 &&
 417	    addr->sin_family != AF_UNSPEC)
 418		goto out;
 419	/* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 420	if (addr->sin_family == AF_UNSPEC &&
 421	    addr->sin_addr.s_addr != htonl(INADDR_ANY))
 422		goto out;
 423
 424	lock_sock(sk);
 425
 426	/* Check if socket is already active */
 427	rc = -EINVAL;
 428	if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 429		goto out_rel;
 430
 431	smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 432	smc->clcsock->sk->sk_reuseport = sk->sk_reuseport;
 433	rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 434
 435out_rel:
 436	release_sock(sk);
 437out:
 438	return rc;
 439}
 440
 441static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 442				   unsigned long mask)
 443{
 444	/* options we don't get control via setsockopt for */
 445	nsk->sk_type = osk->sk_type;
 446	nsk->sk_sndbuf = osk->sk_sndbuf;
 447	nsk->sk_rcvbuf = osk->sk_rcvbuf;
 448	nsk->sk_sndtimeo = osk->sk_sndtimeo;
 449	nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 450	nsk->sk_mark = osk->sk_mark;
 451	nsk->sk_priority = osk->sk_priority;
 452	nsk->sk_rcvlowat = osk->sk_rcvlowat;
 453	nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 454	nsk->sk_err = osk->sk_err;
 455
 456	nsk->sk_flags &= ~mask;
 457	nsk->sk_flags |= osk->sk_flags & mask;
 458}
 459
 460#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 461			     (1UL << SOCK_KEEPOPEN) | \
 462			     (1UL << SOCK_LINGER) | \
 463			     (1UL << SOCK_BROADCAST) | \
 464			     (1UL << SOCK_TIMESTAMP) | \
 465			     (1UL << SOCK_DBG) | \
 466			     (1UL << SOCK_RCVTSTAMP) | \
 467			     (1UL << SOCK_RCVTSTAMPNS) | \
 468			     (1UL << SOCK_LOCALROUTE) | \
 469			     (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 470			     (1UL << SOCK_RXQ_OVFL) | \
 471			     (1UL << SOCK_WIFI_STATUS) | \
 472			     (1UL << SOCK_NOFCS) | \
 473			     (1UL << SOCK_FILTER_LOCKED) | \
 474			     (1UL << SOCK_TSTAMP_NEW))
 475/* copy only relevant settings and flags of SOL_SOCKET level from smc to
 476 * clc socket (since smc is not called for these options from net/core)
 477 */
 478static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 479{
 480	smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 481}
 482
 483#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 484			     (1UL << SOCK_KEEPOPEN) | \
 485			     (1UL << SOCK_LINGER) | \
 486			     (1UL << SOCK_DBG))
 487/* copy only settings and flags relevant for smc from clc to smc socket */
 488static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 489{
 490	smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 491}
 492
 493/* register the new vzalloced sndbuf on all links */
 494static int smcr_lgr_reg_sndbufs(struct smc_link *link,
 495				struct smc_buf_desc *snd_desc)
 496{
 497	struct smc_link_group *lgr = link->lgr;
 498	int i, rc = 0;
 499
 500	if (!snd_desc->is_vm)
 501		return -EINVAL;
 502
 503	/* protect against parallel smcr_link_reg_buf() */
 504	mutex_lock(&lgr->llc_conf_mutex);
 505	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 506		if (!smc_link_active(&lgr->lnk[i]))
 507			continue;
 508		rc = smcr_link_reg_buf(&lgr->lnk[i], snd_desc);
 509		if (rc)
 510			break;
 511	}
 512	mutex_unlock(&lgr->llc_conf_mutex);
 513	return rc;
 514}
 515
 516/* register the new rmb on all links */
 517static int smcr_lgr_reg_rmbs(struct smc_link *link,
 518			     struct smc_buf_desc *rmb_desc)
 519{
 520	struct smc_link_group *lgr = link->lgr;
 521	int i, rc = 0;
 522
 523	rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
 524	if (rc)
 525		return rc;
 526	/* protect against parallel smc_llc_cli_rkey_exchange() and
 527	 * parallel smcr_link_reg_buf()
 528	 */
 529	mutex_lock(&lgr->llc_conf_mutex);
 530	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 531		if (!smc_link_active(&lgr->lnk[i]))
 532			continue;
 533		rc = smcr_link_reg_buf(&lgr->lnk[i], rmb_desc);
 534		if (rc)
 535			goto out;
 536	}
 537
 538	/* exchange confirm_rkey msg with peer */
 539	rc = smc_llc_do_confirm_rkey(link, rmb_desc);
 540	if (rc) {
 541		rc = -EFAULT;
 542		goto out;
 543	}
 544	rmb_desc->is_conf_rkey = true;
 545out:
 546	mutex_unlock(&lgr->llc_conf_mutex);
 547	smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
 548	return rc;
 549}
 550
 551static int smcr_clnt_conf_first_link(struct smc_sock *smc)
 552{
 553	struct smc_link *link = smc->conn.lnk;
 554	struct smc_llc_qentry *qentry;
 555	int rc;
 556
 557	/* receive CONFIRM LINK request from server over RoCE fabric */
 558	qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 559			      SMC_LLC_CONFIRM_LINK);
 560	if (!qentry) {
 561		struct smc_clc_msg_decline dclc;
 562
 563		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 564				      SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 565		return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 566	}
 567	smc_llc_save_peer_uid(qentry);
 568	rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
 569	smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
 570	if (rc)
 571		return SMC_CLC_DECL_RMBE_EC;
 572
 573	rc = smc_ib_modify_qp_rts(link);
 574	if (rc)
 575		return SMC_CLC_DECL_ERR_RDYLNK;
 576
 577	smc_wr_remember_qp_attr(link);
 578
 579	/* reg the sndbuf if it was vzalloced */
 580	if (smc->conn.sndbuf_desc->is_vm) {
 581		if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc))
 582			return SMC_CLC_DECL_ERR_REGBUF;
 583	}
 584
 585	/* reg the rmb */
 586	if (smcr_link_reg_buf(link, smc->conn.rmb_desc))
 587		return SMC_CLC_DECL_ERR_REGBUF;
 588
 589	/* confirm_rkey is implicit on 1st contact */
 590	smc->conn.rmb_desc->is_conf_rkey = true;
 591
 592	/* send CONFIRM LINK response over RoCE fabric */
 593	rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 594	if (rc < 0)
 595		return SMC_CLC_DECL_TIMEOUT_CL;
 596
 597	smc_llc_link_active(link);
 598	smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
 599
 600	/* optional 2nd link, receive ADD LINK request from server */
 601	qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 602			      SMC_LLC_ADD_LINK);
 603	if (!qentry) {
 604		struct smc_clc_msg_decline dclc;
 605
 606		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 607				      SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 608		if (rc == -EAGAIN)
 609			rc = 0; /* no DECLINE received, go with one link */
 610		return rc;
 611	}
 612	smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
 613	smc_llc_cli_add_link(link, qentry);
 614	return 0;
 615}
 616
 617static bool smc_isascii(char *hostname)
 618{
 619	int i;
 620
 621	for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
 622		if (!isascii(hostname[i]))
 623			return false;
 624	return true;
 625}
 626
 627static void smc_conn_save_peer_info_fce(struct smc_sock *smc,
 628					struct smc_clc_msg_accept_confirm *clc)
 629{
 630	struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 631		(struct smc_clc_msg_accept_confirm_v2 *)clc;
 632	struct smc_clc_first_contact_ext *fce;
 633	int clc_v2_len;
 634
 635	if (clc->hdr.version == SMC_V1 ||
 636	    !(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK))
 637		return;
 638
 639	if (smc->conn.lgr->is_smcd) {
 640		memcpy(smc->conn.lgr->negotiated_eid, clc_v2->d1.eid,
 641		       SMC_MAX_EID_LEN);
 642		clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
 643					 d1);
 644	} else {
 645		memcpy(smc->conn.lgr->negotiated_eid, clc_v2->r1.eid,
 646		       SMC_MAX_EID_LEN);
 647		clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
 648					 r1);
 649	}
 650	fce = (struct smc_clc_first_contact_ext *)(((u8 *)clc_v2) + clc_v2_len);
 651	smc->conn.lgr->peer_os = fce->os_type;
 652	smc->conn.lgr->peer_smc_release = fce->release;
 653	if (smc_isascii(fce->hostname))
 654		memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
 655		       SMC_MAX_HOSTNAME_LEN);
 656}
 657
 658static void smcr_conn_save_peer_info(struct smc_sock *smc,
 659				     struct smc_clc_msg_accept_confirm *clc)
 660{
 661	int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
 662
 663	smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
 664	smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
 665	smc->conn.peer_rmbe_size = bufsize;
 666	atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 667	smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 668}
 669
 670static void smcd_conn_save_peer_info(struct smc_sock *smc,
 671				     struct smc_clc_msg_accept_confirm *clc)
 672{
 673	int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
 674
 675	smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
 676	smc->conn.peer_token = clc->d0.token;
 677	/* msg header takes up space in the buffer */
 678	smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 679	atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 680	smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 681}
 682
 683static void smc_conn_save_peer_info(struct smc_sock *smc,
 684				    struct smc_clc_msg_accept_confirm *clc)
 685{
 686	if (smc->conn.lgr->is_smcd)
 687		smcd_conn_save_peer_info(smc, clc);
 688	else
 689		smcr_conn_save_peer_info(smc, clc);
 690	smc_conn_save_peer_info_fce(smc, clc);
 691}
 692
 693static void smc_link_save_peer_info(struct smc_link *link,
 694				    struct smc_clc_msg_accept_confirm *clc,
 695				    struct smc_init_info *ini)
 696{
 697	link->peer_qpn = ntoh24(clc->r0.qpn);
 698	memcpy(link->peer_gid, ini->peer_gid, SMC_GID_SIZE);
 699	memcpy(link->peer_mac, ini->peer_mac, sizeof(link->peer_mac));
 700	link->peer_psn = ntoh24(clc->r0.psn);
 701	link->peer_mtu = clc->r0.qp_mtu;
 702}
 703
 704static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
 705				       struct smc_stats_fback *fback_arr)
 706{
 707	int cnt;
 708
 709	for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) {
 710		if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
 711			fback_arr[cnt].count++;
 712			break;
 713		}
 714		if (!fback_arr[cnt].fback_code) {
 715			fback_arr[cnt].fback_code = smc->fallback_rsn;
 716			fback_arr[cnt].count++;
 717			break;
 718		}
 719	}
 720}
 721
 722static void smc_stat_fallback(struct smc_sock *smc)
 723{
 724	struct net *net = sock_net(&smc->sk);
 725
 726	mutex_lock(&net->smc.mutex_fback_rsn);
 727	if (smc->listen_smc) {
 728		smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv);
 729		net->smc.fback_rsn->srv_fback_cnt++;
 730	} else {
 731		smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt);
 732		net->smc.fback_rsn->clnt_fback_cnt++;
 733	}
 734	mutex_unlock(&net->smc.mutex_fback_rsn);
 735}
 736
 737/* must be called under rcu read lock */
 738static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key)
 739{
 740	struct socket_wq *wq;
 741	__poll_t flags;
 742
 743	wq = rcu_dereference(smc->sk.sk_wq);
 744	if (!skwq_has_sleeper(wq))
 745		return;
 746
 747	/* wake up smc sk->sk_wq */
 748	if (!key) {
 749		/* sk_state_change */
 750		wake_up_interruptible_all(&wq->wait);
 751	} else {
 752		flags = key_to_poll(key);
 753		if (flags & (EPOLLIN | EPOLLOUT))
 754			/* sk_data_ready or sk_write_space */
 755			wake_up_interruptible_sync_poll(&wq->wait, flags);
 756		else if (flags & EPOLLERR)
 757			/* sk_error_report */
 758			wake_up_interruptible_poll(&wq->wait, flags);
 759	}
 760}
 761
 762static int smc_fback_mark_woken(wait_queue_entry_t *wait,
 763				unsigned int mode, int sync, void *key)
 764{
 765	struct smc_mark_woken *mark =
 766		container_of(wait, struct smc_mark_woken, wait_entry);
 767
 768	mark->woken = true;
 769	mark->key = key;
 770	return 0;
 771}
 772
 773static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk,
 774				     void (*clcsock_callback)(struct sock *sk))
 775{
 776	struct smc_mark_woken mark = { .woken = false };
 777	struct socket_wq *wq;
 778
 779	init_waitqueue_func_entry(&mark.wait_entry,
 780				  smc_fback_mark_woken);
 781	rcu_read_lock();
 782	wq = rcu_dereference(clcsk->sk_wq);
 783	if (!wq)
 784		goto out;
 785	add_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
 786	clcsock_callback(clcsk);
 787	remove_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
 788
 789	if (mark.woken)
 790		smc_fback_wakeup_waitqueue(smc, mark.key);
 791out:
 792	rcu_read_unlock();
 793}
 794
 795static void smc_fback_state_change(struct sock *clcsk)
 796{
 797	struct smc_sock *smc;
 798
 799	read_lock_bh(&clcsk->sk_callback_lock);
 800	smc = smc_clcsock_user_data(clcsk);
 801	if (smc)
 802		smc_fback_forward_wakeup(smc, clcsk,
 803					 smc->clcsk_state_change);
 804	read_unlock_bh(&clcsk->sk_callback_lock);
 805}
 806
 807static void smc_fback_data_ready(struct sock *clcsk)
 808{
 809	struct smc_sock *smc;
 810
 811	read_lock_bh(&clcsk->sk_callback_lock);
 812	smc = smc_clcsock_user_data(clcsk);
 813	if (smc)
 814		smc_fback_forward_wakeup(smc, clcsk,
 815					 smc->clcsk_data_ready);
 816	read_unlock_bh(&clcsk->sk_callback_lock);
 817}
 818
 819static void smc_fback_write_space(struct sock *clcsk)
 820{
 821	struct smc_sock *smc;
 822
 823	read_lock_bh(&clcsk->sk_callback_lock);
 824	smc = smc_clcsock_user_data(clcsk);
 825	if (smc)
 826		smc_fback_forward_wakeup(smc, clcsk,
 827					 smc->clcsk_write_space);
 828	read_unlock_bh(&clcsk->sk_callback_lock);
 829}
 830
 831static void smc_fback_error_report(struct sock *clcsk)
 832{
 833	struct smc_sock *smc;
 834
 835	read_lock_bh(&clcsk->sk_callback_lock);
 836	smc = smc_clcsock_user_data(clcsk);
 837	if (smc)
 838		smc_fback_forward_wakeup(smc, clcsk,
 839					 smc->clcsk_error_report);
 840	read_unlock_bh(&clcsk->sk_callback_lock);
 841}
 842
 843static void smc_fback_replace_callbacks(struct smc_sock *smc)
 844{
 845	struct sock *clcsk = smc->clcsock->sk;
 846
 847	write_lock_bh(&clcsk->sk_callback_lock);
 848	clcsk->sk_user_data = (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
 849
 850	smc_clcsock_replace_cb(&clcsk->sk_state_change, smc_fback_state_change,
 851			       &smc->clcsk_state_change);
 852	smc_clcsock_replace_cb(&clcsk->sk_data_ready, smc_fback_data_ready,
 853			       &smc->clcsk_data_ready);
 854	smc_clcsock_replace_cb(&clcsk->sk_write_space, smc_fback_write_space,
 855			       &smc->clcsk_write_space);
 856	smc_clcsock_replace_cb(&clcsk->sk_error_report, smc_fback_error_report,
 857			       &smc->clcsk_error_report);
 858
 859	write_unlock_bh(&clcsk->sk_callback_lock);
 860}
 861
 862static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
 863{
 864	int rc = 0;
 865
 866	mutex_lock(&smc->clcsock_release_lock);
 867	if (!smc->clcsock) {
 868		rc = -EBADF;
 869		goto out;
 870	}
 871
 872	smc->use_fallback = true;
 873	smc->fallback_rsn = reason_code;
 874	smc_stat_fallback(smc);
 875	trace_smc_switch_to_fallback(smc, reason_code);
 876	if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 877		smc->clcsock->file = smc->sk.sk_socket->file;
 878		smc->clcsock->file->private_data = smc->clcsock;
 879		smc->clcsock->wq.fasync_list =
 880			smc->sk.sk_socket->wq.fasync_list;
 881
 882		/* There might be some wait entries remaining
 883		 * in smc sk->sk_wq and they should be woken up
 884		 * as clcsock's wait queue is woken up.
 885		 */
 886		smc_fback_replace_callbacks(smc);
 887	}
 888out:
 889	mutex_unlock(&smc->clcsock_release_lock);
 890	return rc;
 891}
 892
 893/* fall back during connect */
 894static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 895{
 896	struct net *net = sock_net(&smc->sk);
 897	int rc = 0;
 898
 899	rc = smc_switch_to_fallback(smc, reason_code);
 900	if (rc) { /* fallback fails */
 901		this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 902		if (smc->sk.sk_state == SMC_INIT)
 903			sock_put(&smc->sk); /* passive closing */
 904		return rc;
 905	}
 906	smc_copy_sock_settings_to_clc(smc);
 907	smc->connect_nonblock = 0;
 908	if (smc->sk.sk_state == SMC_INIT)
 909		smc->sk.sk_state = SMC_ACTIVE;
 910	return 0;
 911}
 912
 913/* decline and fall back during connect */
 914static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
 915					u8 version)
 916{
 917	struct net *net = sock_net(&smc->sk);
 918	int rc;
 919
 920	if (reason_code < 0) { /* error, fallback is not possible */
 921		this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 922		if (smc->sk.sk_state == SMC_INIT)
 923			sock_put(&smc->sk); /* passive closing */
 924		return reason_code;
 925	}
 926	if (reason_code != SMC_CLC_DECL_PEERDECL) {
 927		rc = smc_clc_send_decline(smc, reason_code, version);
 928		if (rc < 0) {
 929			this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
 930			if (smc->sk.sk_state == SMC_INIT)
 931				sock_put(&smc->sk); /* passive closing */
 932			return rc;
 933		}
 934	}
 935	return smc_connect_fallback(smc, reason_code);
 936}
 937
 938static void smc_conn_abort(struct smc_sock *smc, int local_first)
 939{
 940	struct smc_connection *conn = &smc->conn;
 941	struct smc_link_group *lgr = conn->lgr;
 942	bool lgr_valid = false;
 943
 944	if (smc_conn_lgr_valid(conn))
 945		lgr_valid = true;
 946
 947	smc_conn_free(conn);
 948	if (local_first && lgr_valid)
 949		smc_lgr_cleanup_early(lgr);
 950}
 951
 952/* check if there is a rdma device available for this connection. */
 953/* called for connect and listen */
 954static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 955{
 956	/* PNET table look up: search active ib_device and port
 957	 * within same PNETID that also contains the ethernet device
 958	 * used for the internal TCP socket
 959	 */
 960	smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 961	if (!ini->check_smcrv2 && !ini->ib_dev)
 962		return SMC_CLC_DECL_NOSMCRDEV;
 963	if (ini->check_smcrv2 && !ini->smcrv2.ib_dev_v2)
 964		return SMC_CLC_DECL_NOSMCRDEV;
 965	return 0;
 966}
 967
 968/* check if there is an ISM device available for this connection. */
 969/* called for connect and listen */
 970static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 971{
 972	/* Find ISM device with same PNETID as connecting interface  */
 973	smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 974	if (!ini->ism_dev[0])
 975		return SMC_CLC_DECL_NOSMCDDEV;
 976	else
 977		ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
 978	return 0;
 979}
 980
 981/* is chid unique for the ism devices that are already determined? */
 982static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
 983					   int cnt)
 984{
 985	int i = (!ini->ism_dev[0]) ? 1 : 0;
 986
 987	for (; i < cnt; i++)
 988		if (ini->ism_chid[i] == chid)
 989			return false;
 990	return true;
 991}
 992
 993/* determine possible V2 ISM devices (either without PNETID or with PNETID plus
 994 * PNETID matching net_device)
 995 */
 996static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
 997				       struct smc_init_info *ini)
 998{
 999	int rc = SMC_CLC_DECL_NOSMCDDEV;
1000	struct smcd_dev *smcd;
1001	int i = 1;
1002	u16 chid;
1003
1004	if (smcd_indicated(ini->smc_type_v1))
1005		rc = 0;		/* already initialized for V1 */
1006	mutex_lock(&smcd_dev_list.mutex);
1007	list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1008		if (smcd->going_away || smcd == ini->ism_dev[0])
1009			continue;
1010		chid = smc_ism_get_chid(smcd);
1011		if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
1012			continue;
1013		if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
1014		    smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
1015			ini->ism_dev[i] = smcd;
1016			ini->ism_chid[i] = chid;
1017			ini->is_smcd = true;
1018			rc = 0;
1019			i++;
1020			if (i > SMC_MAX_ISM_DEVS)
1021				break;
1022		}
1023	}
1024	mutex_unlock(&smcd_dev_list.mutex);
1025	ini->ism_offered_cnt = i - 1;
1026	if (!ini->ism_dev[0] && !ini->ism_dev[1])
1027		ini->smcd_version = 0;
1028
1029	return rc;
1030}
1031
1032/* Check for VLAN ID and register it on ISM device just for CLC handshake */
1033static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
1034				      struct smc_init_info *ini)
1035{
1036	if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
1037		return SMC_CLC_DECL_ISMVLANERR;
1038	return 0;
1039}
1040
1041static int smc_find_proposal_devices(struct smc_sock *smc,
1042				     struct smc_init_info *ini)
1043{
1044	int rc = 0;
1045
1046	/* check if there is an ism device available */
1047	if (!(ini->smcd_version & SMC_V1) ||
1048	    smc_find_ism_device(smc, ini) ||
1049	    smc_connect_ism_vlan_setup(smc, ini))
1050		ini->smcd_version &= ~SMC_V1;
1051	/* else ISM V1 is supported for this connection */
1052
1053	/* check if there is an rdma device available */
1054	if (!(ini->smcr_version & SMC_V1) ||
1055	    smc_find_rdma_device(smc, ini))
1056		ini->smcr_version &= ~SMC_V1;
1057	/* else RDMA is supported for this connection */
1058
1059	ini->smc_type_v1 = smc_indicated_type(ini->smcd_version & SMC_V1,
1060					      ini->smcr_version & SMC_V1);
1061
1062	/* check if there is an ism v2 device available */
1063	if (!(ini->smcd_version & SMC_V2) ||
1064	    !smc_ism_is_v2_capable() ||
1065	    smc_find_ism_v2_device_clnt(smc, ini))
1066		ini->smcd_version &= ~SMC_V2;
1067
1068	/* check if there is an rdma v2 device available */
1069	ini->check_smcrv2 = true;
1070	ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr;
1071	if (!(ini->smcr_version & SMC_V2) ||
1072	    smc->clcsock->sk->sk_family != AF_INET ||
1073	    !smc_clc_ueid_count() ||
1074	    smc_find_rdma_device(smc, ini))
1075		ini->smcr_version &= ~SMC_V2;
1076	ini->check_smcrv2 = false;
1077
1078	ini->smc_type_v2 = smc_indicated_type(ini->smcd_version & SMC_V2,
1079					      ini->smcr_version & SMC_V2);
1080
1081	/* if neither ISM nor RDMA are supported, fallback */
1082	if (ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
1083		rc = SMC_CLC_DECL_NOSMCDEV;
1084
1085	return rc;
1086}
1087
1088/* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
1089 * used, the VLAN ID will be registered again during the connection setup.
1090 */
1091static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
1092					struct smc_init_info *ini)
1093{
1094	if (!smcd_indicated(ini->smc_type_v1))
1095		return 0;
1096	if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
1097		return SMC_CLC_DECL_CNFERR;
1098	return 0;
1099}
1100
1101#define SMC_CLC_MAX_ACCEPT_LEN \
1102	(sizeof(struct smc_clc_msg_accept_confirm_v2) + \
1103	 sizeof(struct smc_clc_first_contact_ext) + \
1104	 sizeof(struct smc_clc_msg_trail))
1105
1106/* CLC handshake during connect */
1107static int smc_connect_clc(struct smc_sock *smc,
1108			   struct smc_clc_msg_accept_confirm_v2 *aclc2,
1109			   struct smc_init_info *ini)
1110{
1111	int rc = 0;
1112
1113	/* do inband token exchange */
1114	rc = smc_clc_send_proposal(smc, ini);
1115	if (rc)
1116		return rc;
1117	/* receive SMC Accept CLC message */
1118	return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
1119				SMC_CLC_ACCEPT, CLC_WAIT_TIME);
1120}
1121
1122void smc_fill_gid_list(struct smc_link_group *lgr,
1123		       struct smc_gidlist *gidlist,
1124		       struct smc_ib_device *known_dev, u8 *known_gid)
1125{
1126	struct smc_init_info *alt_ini = NULL;
1127
1128	memset(gidlist, 0, sizeof(*gidlist));
1129	memcpy(gidlist->list[gidlist->len++], known_gid, SMC_GID_SIZE);
1130
1131	alt_ini = kzalloc(sizeof(*alt_ini), GFP_KERNEL);
1132	if (!alt_ini)
1133		goto out;
1134
1135	alt_ini->vlan_id = lgr->vlan_id;
1136	alt_ini->check_smcrv2 = true;
1137	alt_ini->smcrv2.saddr = lgr->saddr;
1138	smc_pnet_find_alt_roce(lgr, alt_ini, known_dev);
1139
1140	if (!alt_ini->smcrv2.ib_dev_v2)
1141		goto out;
1142
1143	memcpy(gidlist->list[gidlist->len++], alt_ini->smcrv2.ib_gid_v2,
1144	       SMC_GID_SIZE);
1145
1146out:
1147	kfree(alt_ini);
1148}
1149
1150static int smc_connect_rdma_v2_prepare(struct smc_sock *smc,
1151				       struct smc_clc_msg_accept_confirm *aclc,
1152				       struct smc_init_info *ini)
1153{
1154	struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1155		(struct smc_clc_msg_accept_confirm_v2 *)aclc;
1156	struct smc_clc_first_contact_ext *fce =
1157		(struct smc_clc_first_contact_ext *)
1158			(((u8 *)clc_v2) + sizeof(*clc_v2));
1159
1160	if (!ini->first_contact_peer || aclc->hdr.version == SMC_V1)
1161		return 0;
1162
1163	if (fce->v2_direct) {
1164		memcpy(ini->smcrv2.nexthop_mac, &aclc->r0.lcl.mac, ETH_ALEN);
1165		ini->smcrv2.uses_gateway = false;
1166	} else {
1167		if (smc_ib_find_route(smc->clcsock->sk->sk_rcv_saddr,
1168				      smc_ib_gid_to_ipv4(aclc->r0.lcl.gid),
1169				      ini->smcrv2.nexthop_mac,
1170				      &ini->smcrv2.uses_gateway))
1171			return SMC_CLC_DECL_NOROUTE;
1172		if (!ini->smcrv2.uses_gateway) {
1173			/* mismatch: peer claims indirect, but its direct */
1174			return SMC_CLC_DECL_NOINDIRECT;
1175		}
1176	}
1177	return 0;
1178}
1179
1180/* setup for RDMA connection of client */
1181static int smc_connect_rdma(struct smc_sock *smc,
1182			    struct smc_clc_msg_accept_confirm *aclc,
1183			    struct smc_init_info *ini)
1184{
1185	int i, reason_code = 0;
1186	struct smc_link *link;
1187	u8 *eid = NULL;
1188
1189	ini->is_smcd = false;
1190	ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
1191	ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1192	memcpy(ini->peer_systemid, aclc->r0.lcl.id_for_peer, SMC_SYSTEMID_LEN);
1193	memcpy(ini->peer_gid, aclc->r0.lcl.gid, SMC_GID_SIZE);
1194	memcpy(ini->peer_mac, aclc->r0.lcl.mac, ETH_ALEN);
1195
1196	reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini);
1197	if (reason_code)
1198		return reason_code;
1199
1200	mutex_lock(&smc_client_lgr_pending);
1201	reason_code = smc_conn_create(smc, ini);
1202	if (reason_code) {
1203		mutex_unlock(&smc_client_lgr_pending);
1204		return reason_code;
1205	}
1206
1207	smc_conn_save_peer_info(smc, aclc);
1208
1209	if (ini->first_contact_local) {
1210		link = smc->conn.lnk;
1211	} else {
1212		/* set link that was assigned by server */
1213		link = NULL;
1214		for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1215			struct smc_link *l = &smc->conn.lgr->lnk[i];
1216
1217			if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
1218			    !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
1219				    SMC_GID_SIZE) &&
1220			    (aclc->hdr.version > SMC_V1 ||
1221			     !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
1222				     sizeof(l->peer_mac)))) {
1223				link = l;
1224				break;
1225			}
1226		}
1227		if (!link) {
1228			reason_code = SMC_CLC_DECL_NOSRVLINK;
1229			goto connect_abort;
1230		}
1231		smc_switch_link_and_count(&smc->conn, link);
1232	}
1233
1234	/* create send buffer and rmb */
1235	if (smc_buf_create(smc, false)) {
1236		reason_code = SMC_CLC_DECL_MEM;
1237		goto connect_abort;
1238	}
1239
1240	if (ini->first_contact_local)
1241		smc_link_save_peer_info(link, aclc, ini);
1242
1243	if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
1244		reason_code = SMC_CLC_DECL_ERR_RTOK;
1245		goto connect_abort;
1246	}
1247
1248	smc_close_init(smc);
1249	smc_rx_init(smc);
1250
1251	if (ini->first_contact_local) {
1252		if (smc_ib_ready_link(link)) {
1253			reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1254			goto connect_abort;
1255		}
1256	} else {
1257		/* reg sendbufs if they were vzalloced */
1258		if (smc->conn.sndbuf_desc->is_vm) {
1259			if (smcr_lgr_reg_sndbufs(link, smc->conn.sndbuf_desc)) {
1260				reason_code = SMC_CLC_DECL_ERR_REGBUF;
1261				goto connect_abort;
1262			}
1263		}
1264		if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
1265			reason_code = SMC_CLC_DECL_ERR_REGBUF;
1266			goto connect_abort;
1267		}
1268	}
1269
1270	if (aclc->hdr.version > SMC_V1) {
1271		struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1272			(struct smc_clc_msg_accept_confirm_v2 *)aclc;
1273
1274		eid = clc_v2->r1.eid;
1275		if (ini->first_contact_local)
1276			smc_fill_gid_list(link->lgr, &ini->smcrv2.gidlist,
1277					  link->smcibdev, link->gid);
1278	}
1279
1280	reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
1281					   aclc->hdr.version, eid, ini);
1282	if (reason_code)
1283		goto connect_abort;
1284
1285	smc_tx_init(smc);
1286
1287	if (ini->first_contact_local) {
1288		/* QP confirmation over RoCE fabric */
1289		smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
1290		reason_code = smcr_clnt_conf_first_link(smc);
1291		smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
1292		if (reason_code)
1293			goto connect_abort;
1294	}
1295	mutex_unlock(&smc_client_lgr_pending);
1296
1297	smc_copy_sock_settings_to_clc(smc);
1298	smc->connect_nonblock = 0;
1299	if (smc->sk.sk_state == SMC_INIT)
1300		smc->sk.sk_state = SMC_ACTIVE;
1301
1302	return 0;
1303connect_abort:
1304	smc_conn_abort(smc, ini->first_contact_local);
1305	mutex_unlock(&smc_client_lgr_pending);
1306	smc->connect_nonblock = 0;
1307
1308	return reason_code;
1309}
1310
1311/* The server has chosen one of the proposed ISM devices for the communication.
1312 * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
1313 */
1314static int
1315smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
1316			       struct smc_init_info *ini)
1317{
1318	int i;
1319
1320	for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
1321		if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) {
1322			ini->ism_selected = i;
1323			return 0;
1324		}
1325	}
1326
1327	return -EPROTO;
1328}
1329
1330/* setup for ISM connection of client */
1331static int smc_connect_ism(struct smc_sock *smc,
1332			   struct smc_clc_msg_accept_confirm *aclc,
1333			   struct smc_init_info *ini)
1334{
1335	u8 *eid = NULL;
1336	int rc = 0;
1337
1338	ini->is_smcd = true;
1339	ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
1340
1341	if (aclc->hdr.version == SMC_V2) {
1342		struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
1343			(struct smc_clc_msg_accept_confirm_v2 *)aclc;
1344
1345		rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
1346		if (rc)
1347			return rc;
1348	}
1349	ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
1350
1351	/* there is only one lgr role for SMC-D; use server lock */
1352	mutex_lock(&smc_server_lgr_pending);
1353	rc = smc_conn_create(smc, ini);
1354	if (rc) {
1355		mutex_unlock(&smc_server_lgr_pending);
1356		return rc;
1357	}
1358
1359	/* Create send and receive buffers */
1360	rc = smc_buf_create(smc, true);
1361	if (rc) {
1362		rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
1363		goto connect_abort;
1364	}
1365
1366	smc_conn_save_peer_info(smc, aclc);
1367	smc_close_init(smc);
1368	smc_rx_init(smc);
1369	smc_tx_init(smc);
1370
1371	if (aclc->hdr.version > SMC_V1) {
1372		struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1373			(struct smc_clc_msg_accept_confirm_v2 *)aclc;
1374
1375		eid = clc_v2->d1.eid;
1376	}
1377
1378	rc = smc_clc_send_confirm(smc, ini->first_contact_local,
1379				  aclc->hdr.version, eid, NULL);
1380	if (rc)
1381		goto connect_abort;
1382	mutex_unlock(&smc_server_lgr_pending);
1383
1384	smc_copy_sock_settings_to_clc(smc);
1385	smc->connect_nonblock = 0;
1386	if (smc->sk.sk_state == SMC_INIT)
1387		smc->sk.sk_state = SMC_ACTIVE;
1388
1389	return 0;
1390connect_abort:
1391	smc_conn_abort(smc, ini->first_contact_local);
1392	mutex_unlock(&smc_server_lgr_pending);
1393	smc->connect_nonblock = 0;
1394
1395	return rc;
1396}
1397
1398/* check if received accept type and version matches a proposed one */
1399static int smc_connect_check_aclc(struct smc_init_info *ini,
1400				  struct smc_clc_msg_accept_confirm *aclc)
1401{
1402	if (aclc->hdr.typev1 != SMC_TYPE_R &&
1403	    aclc->hdr.typev1 != SMC_TYPE_D)
1404		return SMC_CLC_DECL_MODEUNSUPP;
1405
1406	if (aclc->hdr.version >= SMC_V2) {
1407		if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1408		     !smcr_indicated(ini->smc_type_v2)) ||
1409		    (aclc->hdr.typev1 == SMC_TYPE_D &&
1410		     !smcd_indicated(ini->smc_type_v2)))
1411			return SMC_CLC_DECL_MODEUNSUPP;
1412	} else {
1413		if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1414		     !smcr_indicated(ini->smc_type_v1)) ||
1415		    (aclc->hdr.typev1 == SMC_TYPE_D &&
1416		     !smcd_indicated(ini->smc_type_v1)))
1417			return SMC_CLC_DECL_MODEUNSUPP;
1418	}
1419
1420	return 0;
1421}
1422
1423/* perform steps before actually connecting */
1424static int __smc_connect(struct smc_sock *smc)
1425{
1426	u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
1427	struct smc_clc_msg_accept_confirm_v2 *aclc2;
1428	struct smc_clc_msg_accept_confirm *aclc;
1429	struct smc_init_info *ini = NULL;
1430	u8 *buf = NULL;
1431	int rc = 0;
1432
1433	if (smc->use_fallback)
1434		return smc_connect_fallback(smc, smc->fallback_rsn);
1435
1436	/* if peer has not signalled SMC-capability, fall back */
1437	if (!tcp_sk(smc->clcsock->sk)->syn_smc)
1438		return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
1439
1440	/* IPSec connections opt out of SMC optimizations */
1441	if (using_ipsec(smc))
1442		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
1443						    version);
1444
1445	ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1446	if (!ini)
1447		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
1448						    version);
1449
1450	ini->smcd_version = SMC_V1 | SMC_V2;
1451	ini->smcr_version = SMC_V1 | SMC_V2;
1452	ini->smc_type_v1 = SMC_TYPE_B;
1453	ini->smc_type_v2 = SMC_TYPE_B;
1454
1455	/* get vlan id from IP device */
1456	if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
1457		ini->smcd_version &= ~SMC_V1;
1458		ini->smcr_version = 0;
1459		ini->smc_type_v1 = SMC_TYPE_N;
1460		if (!ini->smcd_version) {
1461			rc = SMC_CLC_DECL_GETVLANERR;
1462			goto fallback;
1463		}
1464	}
1465
1466	rc = smc_find_proposal_devices(smc, ini);
1467	if (rc)
1468		goto fallback;
1469
1470	buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
1471	if (!buf) {
1472		rc = SMC_CLC_DECL_MEM;
1473		goto fallback;
1474	}
1475	aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1476	aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1477
1478	/* perform CLC handshake */
1479	rc = smc_connect_clc(smc, aclc2, ini);
1480	if (rc) {
1481		/* -EAGAIN on timeout, see tcp_recvmsg() */
1482		if (rc == -EAGAIN) {
1483			rc = -ETIMEDOUT;
1484			smc->sk.sk_err = ETIMEDOUT;
1485		}
1486		goto vlan_cleanup;
1487	}
1488
1489	/* check if smc modes and versions of CLC proposal and accept match */
1490	rc = smc_connect_check_aclc(ini, aclc);
1491	version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1492	if (rc)
1493		goto vlan_cleanup;
1494
1495	/* depending on previous steps, connect using rdma or ism */
1496	if (aclc->hdr.typev1 == SMC_TYPE_R) {
1497		ini->smcr_version = version;
1498		rc = smc_connect_rdma(smc, aclc, ini);
1499	} else if (aclc->hdr.typev1 == SMC_TYPE_D) {
1500		ini->smcd_version = version;
1501		rc = smc_connect_ism(smc, aclc, ini);
1502	}
1503	if (rc)
1504		goto vlan_cleanup;
1505
1506	SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
1507	smc_connect_ism_vlan_cleanup(smc, ini);
1508	kfree(buf);
1509	kfree(ini);
1510	return 0;
1511
1512vlan_cleanup:
1513	smc_connect_ism_vlan_cleanup(smc, ini);
1514	kfree(buf);
1515fallback:
1516	kfree(ini);
1517	return smc_connect_decline_fallback(smc, rc, version);
1518}
1519
1520static void smc_connect_work(struct work_struct *work)
1521{
1522	struct smc_sock *smc = container_of(work, struct smc_sock,
1523					    connect_work);
1524	long timeo = smc->sk.sk_sndtimeo;
1525	int rc = 0;
1526
1527	if (!timeo)
1528		timeo = MAX_SCHEDULE_TIMEOUT;
1529	lock_sock(smc->clcsock->sk);
1530	if (smc->clcsock->sk->sk_err) {
1531		smc->sk.sk_err = smc->clcsock->sk->sk_err;
1532	} else if ((1 << smc->clcsock->sk->sk_state) &
1533					(TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1534		rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1535		if ((rc == -EPIPE) &&
1536		    ((1 << smc->clcsock->sk->sk_state) &
1537					(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1538			rc = 0;
1539	}
1540	release_sock(smc->clcsock->sk);
1541	lock_sock(&smc->sk);
1542	if (rc != 0 || smc->sk.sk_err) {
1543		smc->sk.sk_state = SMC_CLOSED;
1544		if (rc == -EPIPE || rc == -EAGAIN)
1545			smc->sk.sk_err = EPIPE;
1546		else if (rc == -ECONNREFUSED)
1547			smc->sk.sk_err = ECONNREFUSED;
1548		else if (signal_pending(current))
1549			smc->sk.sk_err = -sock_intr_errno(timeo);
1550		sock_put(&smc->sk); /* passive closing */
1551		goto out;
1552	}
1553
1554	rc = __smc_connect(smc);
1555	if (rc < 0)
1556		smc->sk.sk_err = -rc;
1557
1558out:
1559	if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1560		if (smc->sk.sk_err) {
1561			smc->sk.sk_state_change(&smc->sk);
1562		} else { /* allow polling before and after fallback decision */
1563			smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1564			smc->sk.sk_write_space(&smc->sk);
1565		}
1566	}
1567	release_sock(&smc->sk);
1568}
1569
1570static int smc_connect(struct socket *sock, struct sockaddr *addr,
1571		       int alen, int flags)
1572{
1573	struct sock *sk = sock->sk;
1574	struct smc_sock *smc;
1575	int rc = -EINVAL;
1576
1577	smc = smc_sk(sk);
1578
1579	/* separate smc parameter checking to be safe */
1580	if (alen < sizeof(addr->sa_family))
1581		goto out_err;
1582	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
1583		goto out_err;
1584
1585	lock_sock(sk);
1586	switch (sock->state) {
1587	default:
1588		rc = -EINVAL;
1589		goto out;
1590	case SS_CONNECTED:
1591		rc = sk->sk_state == SMC_ACTIVE ? -EISCONN : -EINVAL;
1592		goto out;
1593	case SS_CONNECTING:
1594		if (sk->sk_state == SMC_ACTIVE)
1595			goto connected;
1596		break;
1597	case SS_UNCONNECTED:
1598		sock->state = SS_CONNECTING;
1599		break;
1600	}
1601
1602	switch (sk->sk_state) {
1603	default:
1604		goto out;
1605	case SMC_CLOSED:
1606		rc = sock_error(sk) ? : -ECONNABORTED;
1607		sock->state = SS_UNCONNECTED;
1608		goto out;
1609	case SMC_ACTIVE:
1610		rc = -EISCONN;
1611		goto out;
1612	case SMC_INIT:
1613		break;
1614	}
1615
1616	smc_copy_sock_settings_to_clc(smc);
1617	tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1618	if (smc->connect_nonblock) {
1619		rc = -EALREADY;
1620		goto out;
1621	}
1622	rc = kernel_connect(smc->clcsock, addr, alen, flags);
1623	if (rc && rc != -EINPROGRESS)
1624		goto out;
1625
1626	if (smc->use_fallback) {
1627		sock->state = rc ? SS_CONNECTING : SS_CONNECTED;
1628		goto out;
1629	}
1630	sock_hold(&smc->sk); /* sock put in passive closing */
1631	if (flags & O_NONBLOCK) {
1632		if (queue_work(smc_hs_wq, &smc->connect_work))
1633			smc->connect_nonblock = 1;
1634		rc = -EINPROGRESS;
1635		goto out;
1636	} else {
1637		rc = __smc_connect(smc);
1638		if (rc < 0)
1639			goto out;
1640	}
1641
1642connected:
1643	rc = 0;
1644	sock->state = SS_CONNECTED;
1645out:
1646	release_sock(sk);
1647out_err:
1648	return rc;
1649}
1650
1651static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1652{
1653	struct socket *new_clcsock = NULL;
1654	struct sock *lsk = &lsmc->sk;
1655	struct sock *new_sk;
1656	int rc = -EINVAL;
1657
1658	release_sock(lsk);
1659	new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
1660	if (!new_sk) {
1661		rc = -ENOMEM;
1662		lsk->sk_err = ENOMEM;
1663		*new_smc = NULL;
1664		lock_sock(lsk);
1665		goto out;
1666	}
1667	*new_smc = smc_sk(new_sk);
1668
1669	mutex_lock(&lsmc->clcsock_release_lock);
1670	if (lsmc->clcsock)
1671		rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
1672	mutex_unlock(&lsmc->clcsock_release_lock);
1673	lock_sock(lsk);
1674	if  (rc < 0 && rc != -EAGAIN)
1675		lsk->sk_err = -rc;
1676	if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1677		new_sk->sk_prot->unhash(new_sk);
1678		if (new_clcsock)
1679			sock_release(new_clcsock);
1680		new_sk->sk_state = SMC_CLOSED;
1681		sock_set_flag(new_sk, SOCK_DEAD);
1682		sock_put(new_sk); /* final */
1683		*new_smc = NULL;
1684		goto out;
1685	}
1686
1687	/* new clcsock has inherited the smc listen-specific sk_data_ready
1688	 * function; switch it back to the original sk_data_ready function
1689	 */
1690	new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
1691
1692	/* if new clcsock has also inherited the fallback-specific callback
1693	 * functions, switch them back to the original ones.
1694	 */
1695	if (lsmc->use_fallback) {
1696		if (lsmc->clcsk_state_change)
1697			new_clcsock->sk->sk_state_change = lsmc->clcsk_state_change;
1698		if (lsmc->clcsk_write_space)
1699			new_clcsock->sk->sk_write_space = lsmc->clcsk_write_space;
1700		if (lsmc->clcsk_error_report)
1701			new_clcsock->sk->sk_error_report = lsmc->clcsk_error_report;
1702	}
1703
1704	(*new_smc)->clcsock = new_clcsock;
1705out:
1706	return rc;
1707}
1708
1709/* add a just created sock to the accept queue of the listen sock as
1710 * candidate for a following socket accept call from user space
1711 */
1712static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1713{
1714	struct smc_sock *par = smc_sk(parent);
1715
1716	sock_hold(sk); /* sock_put in smc_accept_unlink () */
1717	spin_lock(&par->accept_q_lock);
1718	list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1719	spin_unlock(&par->accept_q_lock);
1720	sk_acceptq_added(parent);
1721}
1722
1723/* remove a socket from the accept queue of its parental listening socket */
1724static void smc_accept_unlink(struct sock *sk)
1725{
1726	struct smc_sock *par = smc_sk(sk)->listen_smc;
1727
1728	spin_lock(&par->accept_q_lock);
1729	list_del_init(&smc_sk(sk)->accept_q);
1730	spin_unlock(&par->accept_q_lock);
1731	sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
1732	sock_put(sk); /* sock_hold in smc_accept_enqueue */
1733}
1734
1735/* remove a sock from the accept queue to bind it to a new socket created
1736 * for a socket accept call from user space
1737 */
1738struct sock *smc_accept_dequeue(struct sock *parent,
1739				struct socket *new_sock)
1740{
1741	struct smc_sock *isk, *n;
1742	struct sock *new_sk;
1743
1744	list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1745		new_sk = (struct sock *)isk;
1746
1747		smc_accept_unlink(new_sk);
1748		if (new_sk->sk_state == SMC_CLOSED) {
1749			new_sk->sk_prot->unhash(new_sk);
1750			if (isk->clcsock) {
1751				sock_release(isk->clcsock);
1752				isk->clcsock = NULL;
1753			}
1754			sock_put(new_sk); /* final */
1755			continue;
1756		}
1757		if (new_sock) {
1758			sock_graft(new_sk, new_sock);
1759			new_sock->state = SS_CONNECTED;
1760			if (isk->use_fallback) {
1761				smc_sk(new_sk)->clcsock->file = new_sock->file;
1762				isk->clcsock->file->private_data = isk->clcsock;
1763			}
1764		}
1765		return new_sk;
1766	}
1767	return NULL;
1768}
1769
1770/* clean up for a created but never accepted sock */
1771void smc_close_non_accepted(struct sock *sk)
1772{
1773	struct smc_sock *smc = smc_sk(sk);
1774
1775	sock_hold(sk); /* sock_put below */
1776	lock_sock(sk);
1777	if (!sk->sk_lingertime)
1778		/* wait for peer closing */
1779		sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1780	__smc_release(smc);
1781	release_sock(sk);
1782	sock_put(sk); /* sock_hold above */
1783	sock_put(sk); /* final sock_put */
1784}
1785
1786static int smcr_serv_conf_first_link(struct smc_sock *smc)
1787{
1788	struct smc_link *link = smc->conn.lnk;
1789	struct smc_llc_qentry *qentry;
1790	int rc;
1791
1792	/* reg the sndbuf if it was vzalloced*/
1793	if (smc->conn.sndbuf_desc->is_vm) {
1794		if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc))
1795			return SMC_CLC_DECL_ERR_REGBUF;
1796	}
1797
1798	/* reg the rmb */
1799	if (smcr_link_reg_buf(link, smc->conn.rmb_desc))
1800		return SMC_CLC_DECL_ERR_REGBUF;
1801
1802	/* send CONFIRM LINK request to client over the RoCE fabric */
1803	rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1804	if (rc < 0)
1805		return SMC_CLC_DECL_TIMEOUT_CL;
1806
1807	/* receive CONFIRM LINK response from client over the RoCE fabric */
1808	qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1809			      SMC_LLC_CONFIRM_LINK);
1810	if (!qentry) {
1811		struct smc_clc_msg_decline dclc;
1812
1813		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1814				      SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1815		return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1816	}
1817	smc_llc_save_peer_uid(qentry);
1818	rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
1819	smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1820	if (rc)
1821		return SMC_CLC_DECL_RMBE_EC;
1822
1823	/* confirm_rkey is implicit on 1st contact */
1824	smc->conn.rmb_desc->is_conf_rkey = true;
1825
1826	smc_llc_link_active(link);
1827	smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
1828
1829	/* initial contact - try to establish second link */
1830	smc_llc_srv_add_link(link, NULL);
1831	return 0;
1832}
1833
1834/* listen worker: finish */
1835static void smc_listen_out(struct smc_sock *new_smc)
1836{
1837	struct smc_sock *lsmc = new_smc->listen_smc;
1838	struct sock *newsmcsk = &new_smc->sk;
1839
1840	if (tcp_sk(new_smc->clcsock->sk)->syn_smc)
1841		atomic_dec(&lsmc->queued_smc_hs);
1842
1843	if (lsmc->sk.sk_state == SMC_LISTEN) {
1844		lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1845		smc_accept_enqueue(&lsmc->sk, newsmcsk);
1846		release_sock(&lsmc->sk);
1847	} else { /* no longer listening */
1848		smc_close_non_accepted(newsmcsk);
1849	}
1850
1851	/* Wake up accept */
1852	lsmc->sk.sk_data_ready(&lsmc->sk);
1853	sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1854}
1855
1856/* listen worker: finish in state connected */
1857static void smc_listen_out_connected(struct smc_sock *new_smc)
1858{
1859	struct sock *newsmcsk = &new_smc->sk;
1860
1861	if (newsmcsk->sk_state == SMC_INIT)
1862		newsmcsk->sk_state = SMC_ACTIVE;
1863
1864	smc_listen_out(new_smc);
1865}
1866
1867/* listen worker: finish in error state */
1868static void smc_listen_out_err(struct smc_sock *new_smc)
1869{
1870	struct sock *newsmcsk = &new_smc->sk;
1871	struct net *net = sock_net(newsmcsk);
1872
1873	this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
1874	if (newsmcsk->sk_state == SMC_INIT)
1875		sock_put(&new_smc->sk); /* passive closing */
1876	newsmcsk->sk_state = SMC_CLOSED;
1877
1878	smc_listen_out(new_smc);
1879}
1880
1881/* listen worker: decline and fall back if possible */
1882static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1883			       int local_first, u8 version)
1884{
1885	/* RDMA setup failed, switch back to TCP */
1886	smc_conn_abort(new_smc, local_first);
1887	if (reason_code < 0 ||
1888	    smc_switch_to_fallback(new_smc, reason_code)) {
1889		/* error, no fallback possible */
1890		smc_listen_out_err(new_smc);
1891		return;
1892	}
1893	if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1894		if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
1895			smc_listen_out_err(new_smc);
1896			return;
1897		}
1898	}
1899	smc_listen_out_connected(new_smc);
1900}
1901
1902/* listen worker: version checking */
1903static int smc_listen_v2_check(struct smc_sock *new_smc,
1904			       struct smc_clc_msg_proposal *pclc,
1905			       struct smc_init_info *ini)
1906{
1907	struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1908	struct smc_clc_v2_extension *pclc_v2_ext;
1909	int rc = SMC_CLC_DECL_PEERNOSMC;
1910
1911	ini->smc_type_v1 = pclc->hdr.typev1;
1912	ini->smc_type_v2 = pclc->hdr.typev2;
1913	ini->smcd_version = smcd_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1914	ini->smcr_version = smcr_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1915	if (pclc->hdr.version > SMC_V1) {
1916		if (smcd_indicated(ini->smc_type_v2))
1917			ini->smcd_version |= SMC_V2;
1918		if (smcr_indicated(ini->smc_type_v2))
1919			ini->smcr_version |= SMC_V2;
1920	}
1921	if (!(ini->smcd_version & SMC_V2) && !(ini->smcr_version & SMC_V2)) {
1922		rc = SMC_CLC_DECL_PEERNOSMC;
1923		goto out;
1924	}
1925	pclc_v2_ext = smc_get_clc_v2_ext(pclc);
1926	if (!pclc_v2_ext) {
1927		ini->smcd_version &= ~SMC_V2;
1928		ini->smcr_version &= ~SMC_V2;
1929		rc = SMC_CLC_DECL_NOV2EXT;
1930		goto out;
1931	}
1932	pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
1933	if (ini->smcd_version & SMC_V2) {
1934		if (!smc_ism_is_v2_capable()) {
1935			ini->smcd_version &= ~SMC_V2;
1936			rc = SMC_CLC_DECL_NOISM2SUPP;
1937		} else if (!pclc_smcd_v2_ext) {
1938			ini->smcd_version &= ~SMC_V2;
1939			rc = SMC_CLC_DECL_NOV2DEXT;
1940		} else if (!pclc_v2_ext->hdr.eid_cnt &&
1941			   !pclc_v2_ext->hdr.flag.seid) {
1942			ini->smcd_version &= ~SMC_V2;
1943			rc = SMC_CLC_DECL_NOUEID;
1944		}
1945	}
1946	if (ini->smcr_version & SMC_V2) {
1947		if (!pclc_v2_ext->hdr.eid_cnt) {
1948			ini->smcr_version &= ~SMC_V2;
1949			rc = SMC_CLC_DECL_NOUEID;
1950		}
1951	}
1952
1953out:
1954	if (!ini->smcd_version && !ini->smcr_version)
1955		return rc;
1956
1957	return 0;
1958}
1959
1960/* listen worker: check prefixes */
1961static int smc_listen_prfx_check(struct smc_sock *new_smc,
1962				 struct smc_clc_msg_proposal *pclc)
1963{
1964	struct smc_clc_msg_proposal_prefix *pclc_prfx;
1965	struct socket *newclcsock = new_smc->clcsock;
1966
1967	if (pclc->hdr.typev1 == SMC_TYPE_N)
1968		return 0;
1969	pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1970	if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1971		return SMC_CLC_DECL_DIFFPREFIX;
1972
1973	return 0;
1974}
1975
1976/* listen worker: initialize connection and buffers */
1977static int smc_listen_rdma_init(struct smc_sock *new_smc,
1978				struct smc_init_info *ini)
1979{
1980	int rc;
1981
1982	/* allocate connection / link group */
1983	rc = smc_conn_create(new_smc, ini);
1984	if (rc)
1985		return rc;
1986
1987	/* create send buffer and rmb */
1988	if (smc_buf_create(new_smc, false))
1989		return SMC_CLC_DECL_MEM;
1990
1991	return 0;
1992}
1993
1994/* listen worker: initialize connection and buffers for SMC-D */
1995static int smc_listen_ism_init(struct smc_sock *new_smc,
1996			       struct smc_init_info *ini)
1997{
1998	int rc;
1999
2000	rc = smc_conn_create(new_smc, ini);
2001	if (rc)
2002		return rc;
2003
2004	/* Create send and receive buffers */
2005	rc = smc_buf_create(new_smc, true);
2006	if (rc) {
2007		smc_conn_abort(new_smc, ini->first_contact_local);
2008		return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
2009					 SMC_CLC_DECL_MEM;
2010	}
2011
2012	return 0;
2013}
2014
2015static bool smc_is_already_selected(struct smcd_dev *smcd,
2016				    struct smc_init_info *ini,
2017				    int matches)
2018{
2019	int i;
2020
2021	for (i = 0; i < matches; i++)
2022		if (smcd == ini->ism_dev[i])
2023			return true;
2024
2025	return false;
2026}
2027
2028/* check for ISM devices matching proposed ISM devices */
2029static void smc_check_ism_v2_match(struct smc_init_info *ini,
2030				   u16 proposed_chid, u64 proposed_gid,
2031				   unsigned int *matches)
2032{
2033	struct smcd_dev *smcd;
2034
2035	list_for_each_entry(smcd, &smcd_dev_list.list, list) {
2036		if (smcd->going_away)
2037			continue;
2038		if (smc_is_already_selected(smcd, ini, *matches))
2039			continue;
2040		if (smc_ism_get_chid(smcd) == proposed_chid &&
2041		    !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
2042			ini->ism_peer_gid[*matches] = proposed_gid;
2043			ini->ism_dev[*matches] = smcd;
2044			(*matches)++;
2045			break;
2046		}
2047	}
2048}
2049
2050static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
2051{
2052	if (!ini->rc)
2053		ini->rc = rc;
2054}
2055
2056static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
2057					struct smc_clc_msg_proposal *pclc,
2058					struct smc_init_info *ini)
2059{
2060	struct smc_clc_smcd_v2_extension *smcd_v2_ext;
2061	struct smc_clc_v2_extension *smc_v2_ext;
2062	struct smc_clc_msg_smcd *pclc_smcd;
2063	unsigned int matches = 0;
2064	u8 smcd_version;
2065	u8 *eid = NULL;
2066	int i, rc;
2067
2068	if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
2069		goto not_found;
2070
2071	pclc_smcd = smc_get_clc_msg_smcd(pclc);
2072	smc_v2_ext = smc_get_clc_v2_ext(pclc);
2073	smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
2074
2075	mutex_lock(&smcd_dev_list.mutex);
2076	if (pclc_smcd->ism.chid)
2077		/* check for ISM device matching proposed native ISM device */
2078		smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
2079				       ntohll(pclc_smcd->ism.gid), &matches);
2080	for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
2081		/* check for ISM devices matching proposed non-native ISM
2082		 * devices
2083		 */
2084		smc_check_ism_v2_match(ini,
2085				       ntohs(smcd_v2_ext->gidchid[i - 1].chid),
2086				       ntohll(smcd_v2_ext->gidchid[i - 1].gid),
2087				       &matches);
2088	}
2089	mutex_unlock(&smcd_dev_list.mutex);
2090
2091	if (!ini->ism_dev[0]) {
2092		smc_find_ism_store_rc(SMC_CLC_DECL_NOSMCD2DEV, ini);
2093		goto not_found;
2094	}
2095
2096	smc_ism_get_system_eid(&eid);
2097	if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext,
2098			       smcd_v2_ext->system_eid, eid))
2099		goto not_found;
2100
2101	/* separate - outside the smcd_dev_list.lock */
2102	smcd_version = ini->smcd_version;
2103	for (i = 0; i < matches; i++) {
2104		ini->smcd_version = SMC_V2;
2105		ini->is_smcd = true;
2106		ini->ism_selected = i;
2107		rc = smc_listen_ism_init(new_smc, ini);
2108		if (rc) {
2109			smc_find_ism_store_rc(rc, ini);
2110			/* try next active ISM device */
2111			continue;
2112		}
2113		return; /* matching and usable V2 ISM device found */
2114	}
2115	/* no V2 ISM device could be initialized */
2116	ini->smcd_version = smcd_version;	/* restore original value */
2117	ini->negotiated_eid[0] = 0;
2118
2119not_found:
2120	ini->smcd_version &= ~SMC_V2;
2121	ini->ism_dev[0] = NULL;
2122	ini->is_smcd = false;
2123}
2124
2125static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
2126					struct smc_clc_msg_proposal *pclc,
2127					struct smc_init_info *ini)
2128{
2129	struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);
2130	int rc = 0;
2131
2132	/* check if ISM V1 is available */
2133	if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
2134		goto not_found;
2135	ini->is_smcd = true; /* prepare ISM check */
2136	ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
2137	rc = smc_find_ism_device(new_smc, ini);
2138	if (rc)
2139		goto not_found;
2140	ini->ism_selected = 0;
2141	rc = smc_listen_ism_init(new_smc, ini);
2142	if (!rc)
2143		return;		/* V1 ISM device found */
2144
2145not_found:
2146	smc_find_ism_store_rc(rc, ini);
2147	ini->smcd_version &= ~SMC_V1;
2148	ini->ism_dev[0] = NULL;
2149	ini->is_smcd = false;
2150}
2151
2152/* listen worker: register buffers */
2153static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
2154{
2155	struct smc_connection *conn = &new_smc->conn;
2156
2157	if (!local_first) {
2158		/* reg sendbufs if they were vzalloced */
2159		if (conn->sndbuf_desc->is_vm) {
2160			if (smcr_lgr_reg_sndbufs(conn->lnk,
2161						 conn->sndbuf_desc))
2162				return SMC_CLC_DECL_ERR_REGBUF;
2163		}
2164		if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
2165			return SMC_CLC_DECL_ERR_REGBUF;
2166	}
2167
2168	return 0;
2169}
2170
2171static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc,
2172					 struct smc_clc_msg_proposal *pclc,
2173					 struct smc_init_info *ini)
2174{
2175	struct smc_clc_v2_extension *smc_v2_ext;
2176	u8 smcr_version;
2177	int rc;
2178
2179	if (!(ini->smcr_version & SMC_V2) || !smcr_indicated(ini->smc_type_v2))
2180		goto not_found;
2181
2182	smc_v2_ext = smc_get_clc_v2_ext(pclc);
2183	if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, NULL, NULL))
2184		goto not_found;
2185
2186	/* prepare RDMA check */
2187	memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
2188	memcpy(ini->peer_gid, smc_v2_ext->roce, SMC_GID_SIZE);
2189	memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
2190	ini->check_smcrv2 = true;
2191	ini->smcrv2.clc_sk = new_smc->clcsock->sk;
2192	ini->smcrv2.saddr = new_smc->clcsock->sk->sk_rcv_saddr;
2193	ini->smcrv2.daddr = smc_ib_gid_to_ipv4(smc_v2_ext->roce);
2194	rc = smc_find_rdma_device(new_smc, ini);
2195	if (rc) {
2196		smc_find_ism_store_rc(rc, ini);
2197		goto not_found;
2198	}
2199	if (!ini->smcrv2.uses_gateway)
2200		memcpy(ini->smcrv2.nexthop_mac, pclc->lcl.mac, ETH_ALEN);
2201
2202	smcr_version = ini->smcr_version;
2203	ini->smcr_version = SMC_V2;
2204	rc = smc_listen_rdma_init(new_smc, ini);
2205	if (!rc)
2206		rc = smc_listen_rdma_reg(new_smc, ini->first_contact_local);
2207	if (!rc)
2208		return;
2209	ini->smcr_version = smcr_version;
2210	smc_find_ism_store_rc(rc, ini);
2211
2212not_found:
2213	ini->smcr_version &= ~SMC_V2;
2214	ini->smcrv2.ib_dev_v2 = NULL;
2215	ini->check_smcrv2 = false;
2216}
2217
2218static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
2219					struct smc_clc_msg_proposal *pclc,
2220					struct smc_init_info *ini)
2221{
2222	int rc;
2223
2224	if (!(ini->smcr_version & SMC_V1) || !smcr_indicated(ini->smc_type_v1))
2225		return SMC_CLC_DECL_NOSMCDEV;
2226
2227	/* prepare RDMA check */
2228	memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
2229	memcpy(ini->peer_gid, pclc->lcl.gid, SMC_GID_SIZE);
2230	memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
2231	rc = smc_find_rdma_device(new_smc, ini);
2232	if (rc) {
2233		/* no RDMA device found */
2234		return SMC_CLC_DECL_NOSMCDEV;
2235	}
2236	rc = smc_listen_rdma_init(new_smc, ini);
2237	if (rc)
2238		return rc;
2239	return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
2240}
2241
2242/* determine the local device matching to proposal */
2243static int smc_listen_find_device(struct smc_sock *new_smc,
2244				  struct smc_clc_msg_proposal *pclc,
2245				  struct smc_init_info *ini)
2246{
2247	int prfx_rc;
2248
2249	/* check for ISM device matching V2 proposed device */
2250	smc_find_ism_v2_device_serv(new_smc, pclc, ini);
2251	if (ini->ism_dev[0])
2252		return 0;
2253
2254	/* check for matching IP prefix and subnet length (V1) */
2255	prfx_rc = smc_listen_prfx_check(new_smc, pclc);
2256	if (prfx_rc)
2257		smc_find_ism_store_rc(prfx_rc, ini);
2258
2259	/* get vlan id from IP device */
2260	if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
2261		return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
2262
2263	/* check for ISM device matching V1 proposed device */
2264	if (!prfx_rc)
2265		smc_find_ism_v1_device_serv(new_smc, pclc, ini);
2266	if (ini->ism_dev[0])
2267		return 0;
2268
2269	if (!smcr_indicated(pclc->hdr.typev1) &&
2270	    !smcr_indicated(pclc->hdr.typev2))
2271		/* skip RDMA and decline */
2272		return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
2273
2274	/* check if RDMA V2 is available */
2275	smc_find_rdma_v2_device_serv(new_smc, pclc, ini);
2276	if (ini->smcrv2.ib_dev_v2)
2277		return 0;
2278
2279	/* check if RDMA V1 is available */
2280	if (!prfx_rc) {
2281		int rc;
2282
2283		rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
2284		smc_find_ism_store_rc(rc, ini);
2285		return (!rc) ? 0 : ini->rc;
2286	}
2287	return SMC_CLC_DECL_NOSMCDEV;
2288}
2289
2290/* listen worker: finish RDMA setup */
2291static int smc_listen_rdma_finish(struct smc_sock *new_smc,
2292				  struct smc_clc_msg_accept_confirm *cclc,
2293				  bool local_first,
2294				  struct smc_init_info *ini)
2295{
2296	struct smc_link *link = new_smc->conn.lnk;
2297	int reason_code = 0;
2298
2299	if (local_first)
2300		smc_link_save_peer_info(link, cclc, ini);
2301
2302	if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
2303		return SMC_CLC_DECL_ERR_RTOK;
2304
2305	if (local_first) {
2306		if (smc_ib_ready_link(link))
2307			return SMC_CLC_DECL_ERR_RDYLNK;
2308		/* QP confirmation over RoCE fabric */
2309		smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
2310		reason_code = smcr_serv_conf_first_link(new_smc);
2311		smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
2312	}
2313	return reason_code;
2314}
2315
2316/* setup for connection of server */
2317static void smc_listen_work(struct work_struct *work)
2318{
2319	struct smc_sock *new_smc = container_of(work, struct smc_sock,
2320						smc_listen_work);
2321	struct socket *newclcsock = new_smc->clcsock;
2322	struct smc_clc_msg_accept_confirm *cclc;
2323	struct smc_clc_msg_proposal_area *buf;
2324	struct smc_clc_msg_proposal *pclc;
2325	struct smc_init_info *ini = NULL;
2326	u8 proposal_version = SMC_V1;
2327	u8 accept_version;
2328	int rc = 0;
2329
2330	if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
2331		return smc_listen_out_err(new_smc);
2332
2333	if (new_smc->use_fallback) {
2334		smc_listen_out_connected(new_smc);
2335		return;
2336	}
2337
2338	/* check if peer is smc capable */
2339	if (!tcp_sk(newclcsock->sk)->syn_smc) {
2340		rc = smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
2341		if (rc)
2342			smc_listen_out_err(new_smc);
2343		else
2344			smc_listen_out_connected(new_smc);
2345		return;
2346	}
2347
2348	/* do inband token exchange -
2349	 * wait for and receive SMC Proposal CLC message
2350	 */
2351	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
2352	if (!buf) {
2353		rc = SMC_CLC_DECL_MEM;
2354		goto out_decl;
2355	}
2356	pclc = (struct smc_clc_msg_proposal *)buf;
2357	rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
2358			      SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
2359	if (rc)
2360		goto out_decl;
2361
2362	if (pclc->hdr.version > SMC_V1)
2363		proposal_version = SMC_V2;
2364
2365	/* IPSec connections opt out of SMC optimizations */
2366	if (using_ipsec(new_smc)) {
2367		rc = SMC_CLC_DECL_IPSEC;
2368		goto out_decl;
2369	}
2370
2371	ini = kzalloc(sizeof(*ini), GFP_KERNEL);
2372	if (!ini) {
2373		rc = SMC_CLC_DECL_MEM;
2374		goto out_decl;
2375	}
2376
2377	/* initial version checking */
2378	rc = smc_listen_v2_check(new_smc, pclc, ini);
2379	if (rc)
2380		goto out_decl;
2381
2382	mutex_lock(&smc_server_lgr_pending);
2383	smc_close_init(new_smc);
2384	smc_rx_init(new_smc);
2385	smc_tx_init(new_smc);
2386
2387	/* determine ISM or RoCE device used for connection */
2388	rc = smc_listen_find_device(new_smc, pclc, ini);
2389	if (rc)
2390		goto out_unlock;
2391
2392	/* send SMC Accept CLC message */
2393	accept_version = ini->is_smcd ? ini->smcd_version : ini->smcr_version;
2394	rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
2395				 accept_version, ini->negotiated_eid);
2396	if (rc)
2397		goto out_unlock;
2398
2399	/* SMC-D does not need this lock any more */
2400	if (ini->is_smcd)
2401		mutex_unlock(&smc_server_lgr_pending);
2402
2403	/* receive SMC Confirm CLC message */
2404	memset(buf, 0, sizeof(*buf));
2405	cclc = (struct smc_clc_msg_accept_confirm *)buf;
2406	rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
2407			      SMC_CLC_CONFIRM, CLC_WAIT_TIME);
2408	if (rc) {
2409		if (!ini->is_smcd)
2410			goto out_unlock;
2411		goto out_decl;
2412	}
2413
2414	/* finish worker */
2415	if (!ini->is_smcd) {
2416		rc = smc_listen_rdma_finish(new_smc, cclc,
2417					    ini->first_contact_local, ini);
2418		if (rc)
2419			goto out_unlock;
2420		mutex_unlock(&smc_server_lgr_pending);
2421	}
2422	smc_conn_save_peer_info(new_smc, cclc);
2423	smc_listen_out_connected(new_smc);
2424	SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
2425	goto out_free;
2426
2427out_unlock:
2428	mutex_unlock(&smc_server_lgr_pending);
2429out_decl:
2430	smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
2431			   proposal_version);
2432out_free:
2433	kfree(ini);
2434	kfree(buf);
2435}
2436
2437static void smc_tcp_listen_work(struct work_struct *work)
2438{
2439	struct smc_sock *lsmc = container_of(work, struct smc_sock,
2440					     tcp_listen_work);
2441	struct sock *lsk = &lsmc->sk;
2442	struct smc_sock *new_smc;
2443	int rc = 0;
2444
2445	lock_sock(lsk);
2446	while (lsk->sk_state == SMC_LISTEN) {
2447		rc = smc_clcsock_accept(lsmc, &new_smc);
2448		if (rc) /* clcsock accept queue empty or error */
2449			goto out;
2450		if (!new_smc)
2451			continue;
2452
2453		if (tcp_sk(new_smc->clcsock->sk)->syn_smc)
2454			atomic_inc(&lsmc->queued_smc_hs);
2455
2456		new_smc->listen_smc = lsmc;
2457		new_smc->use_fallback = lsmc->use_fallback;
2458		new_smc->fallback_rsn = lsmc->fallback_rsn;
2459		sock_hold(lsk); /* sock_put in smc_listen_work */
2460		INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
2461		smc_copy_sock_settings_to_smc(new_smc);
2462		new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
2463		new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
2464		sock_hold(&new_smc->sk); /* sock_put in passive closing */
2465		if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
2466			sock_put(&new_smc->sk);
2467	}
2468
2469out:
2470	release_sock(lsk);
2471	sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
2472}
2473
2474static void smc_clcsock_data_ready(struct sock *listen_clcsock)
2475{
2476	struct smc_sock *lsmc;
2477
2478	read_lock_bh(&listen_clcsock->sk_callback_lock);
2479	lsmc = smc_clcsock_user_data(listen_clcsock);
2480	if (!lsmc)
2481		goto out;
2482	lsmc->clcsk_data_ready(listen_clcsock);
2483	if (lsmc->sk.sk_state == SMC_LISTEN) {
2484		sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
2485		if (!queue_work(smc_tcp_ls_wq, &lsmc->tcp_listen_work))
2486			sock_put(&lsmc->sk);
2487	}
2488out:
2489	read_unlock_bh(&listen_clcsock->sk_callback_lock);
2490}
2491
2492static int smc_listen(struct socket *sock, int backlog)
2493{
2494	struct sock *sk = sock->sk;
2495	struct smc_sock *smc;
2496	int rc;
2497
2498	smc = smc_sk(sk);
2499	lock_sock(sk);
2500
2501	rc = -EINVAL;
2502	if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
2503	    smc->connect_nonblock || sock->state != SS_UNCONNECTED)
2504		goto out;
2505
2506	rc = 0;
2507	if (sk->sk_state == SMC_LISTEN) {
2508		sk->sk_max_ack_backlog = backlog;
2509		goto out;
2510	}
2511	/* some socket options are handled in core, so we could not apply
2512	 * them to the clc socket -- copy smc socket options to clc socket
2513	 */
2514	smc_copy_sock_settings_to_clc(smc);
2515	if (!smc->use_fallback)
2516		tcp_sk(smc->clcsock->sk)->syn_smc = 1;
2517
2518	/* save original sk_data_ready function and establish
2519	 * smc-specific sk_data_ready function
2520	 */
2521	write_lock_bh(&smc->clcsock->sk->sk_callback_lock);
2522	smc->clcsock->sk->sk_user_data =
2523		(void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
2524	smc_clcsock_replace_cb(&smc->clcsock->sk->sk_data_ready,
2525			       smc_clcsock_data_ready, &smc->clcsk_data_ready);
2526	write_unlock_bh(&smc->clcsock->sk->sk_callback_lock);
2527
2528	/* save original ops */
2529	smc->ori_af_ops = inet_csk(smc->clcsock->sk)->icsk_af_ops;
2530
2531	smc->af_ops = *smc->ori_af_ops;
2532	smc->af_ops.syn_recv_sock = smc_tcp_syn_recv_sock;
2533
2534	inet_csk(smc->clcsock->sk)->icsk_af_ops = &smc->af_ops;
2535
2536	if (smc->limit_smc_hs)
2537		tcp_sk(smc->clcsock->sk)->smc_hs_congested = smc_hs_congested;
2538
2539	rc = kernel_listen(smc->clcsock, backlog);
2540	if (rc) {
2541		write_lock_bh(&smc->clcsock->sk->sk_callback_lock);
2542		smc_clcsock_restore_cb(&smc->clcsock->sk->sk_data_ready,
2543				       &smc->clcsk_data_ready);
2544		smc->clcsock->sk->sk_user_data = NULL;
2545		write_unlock_bh(&smc->clcsock->sk->sk_callback_lock);
2546		goto out;
2547	}
2548	sk->sk_max_ack_backlog = backlog;
2549	sk->sk_ack_backlog = 0;
2550	sk->sk_state = SMC_LISTEN;
2551
2552out:
2553	release_sock(sk);
2554	return rc;
2555}
2556
2557static int smc_accept(struct socket *sock, struct socket *new_sock,
2558		      int flags, bool kern)
2559{
2560	struct sock *sk = sock->sk, *nsk;
2561	DECLARE_WAITQUEUE(wait, current);
2562	struct smc_sock *lsmc;
2563	long timeo;
2564	int rc = 0;
2565
2566	lsmc = smc_sk(sk);
2567	sock_hold(sk); /* sock_put below */
2568	lock_sock(sk);
2569
2570	if (lsmc->sk.sk_state != SMC_LISTEN) {
2571		rc = -EINVAL;
2572		release_sock(sk);
2573		goto out;
2574	}
2575
2576	/* Wait for an incoming connection */
2577	timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
2578	add_wait_queue_exclusive(sk_sleep(sk), &wait);
2579	while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
2580		set_current_state(TASK_INTERRUPTIBLE);
2581		if (!timeo) {
2582			rc = -EAGAIN;
2583			break;
2584		}
2585		release_sock(sk);
2586		timeo = schedule_timeout(timeo);
2587		/* wakeup by sk_data_ready in smc_listen_work() */
2588		sched_annotate_sleep();
2589		lock_sock(sk);
2590		if (signal_pending(current)) {
2591			rc = sock_intr_errno(timeo);
2592			break;
2593		}
2594	}
2595	set_current_state(TASK_RUNNING);
2596	remove_wait_queue(sk_sleep(sk), &wait);
2597
2598	if (!rc)
2599		rc = sock_error(nsk);
2600	release_sock(sk);
2601	if (rc)
2602		goto out;
2603
2604	if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
2605		/* wait till data arrives on the socket */
2606		timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
2607								MSEC_PER_SEC);
2608		if (smc_sk(nsk)->use_fallback) {
2609			struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
2610
2611			lock_sock(clcsk);
2612			if (skb_queue_empty(&clcsk->sk_receive_queue))
2613				sk_wait_data(clcsk, &timeo, NULL);
2614			release_sock(clcsk);
2615		} else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
2616			lock_sock(nsk);
2617			smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
2618			release_sock(nsk);
2619		}
2620	}
2621
2622out:
2623	sock_put(sk); /* sock_hold above */
2624	return rc;
2625}
2626
2627static int smc_getname(struct socket *sock, struct sockaddr *addr,
2628		       int peer)
2629{
2630	struct smc_sock *smc;
2631
2632	if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2633	    (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
2634		return -ENOTCONN;
2635
2636	smc = smc_sk(sock->sk);
2637
2638	return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
2639}
2640
2641static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2642{
2643	struct sock *sk = sock->sk;
2644	struct smc_sock *smc;
2645	int rc = -EPIPE;
2646
2647	smc = smc_sk(sk);
2648	lock_sock(sk);
2649	if ((sk->sk_state != SMC_ACTIVE) &&
2650	    (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2651	    (sk->sk_state != SMC_INIT))
2652		goto out;
2653
2654	if (msg->msg_flags & MSG_FASTOPEN) {
2655		if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2656			rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2657			if (rc)
2658				goto out;
2659		} else {
2660			rc = -EINVAL;
2661			goto out;
2662		}
2663	}
2664
2665	if (smc->use_fallback) {
2666		rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
2667	} else {
2668		rc = smc_tx_sendmsg(smc, msg, len);
2669		SMC_STAT_TX_PAYLOAD(smc, len, rc);
2670	}
2671out:
2672	release_sock(sk);
2673	return rc;
2674}
2675
2676static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2677		       int flags)
2678{
2679	struct sock *sk = sock->sk;
2680	struct smc_sock *smc;
2681	int rc = -ENOTCONN;
2682
2683	smc = smc_sk(sk);
2684	lock_sock(sk);
2685	if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2686		/* socket was connected before, no more data to read */
2687		rc = 0;
2688		goto out;
2689	}
2690	if ((sk->sk_state == SMC_INIT) ||
2691	    (sk->sk_state == SMC_LISTEN) ||
2692	    (sk->sk_state == SMC_CLOSED))
2693		goto out;
2694
2695	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2696		rc = 0;
2697		goto out;
2698	}
2699
2700	if (smc->use_fallback) {
2701		rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
2702	} else {
2703		msg->msg_namelen = 0;
2704		rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
2705		SMC_STAT_RX_PAYLOAD(smc, rc, rc);
2706	}
2707
2708out:
2709	release_sock(sk);
2710	return rc;
2711}
2712
2713static __poll_t smc_accept_poll(struct sock *parent)
2714{
2715	struct smc_sock *isk = smc_sk(parent);
2716	__poll_t mask = 0;
2717
2718	spin_lock(&isk->accept_q_lock);
2719	if (!list_empty(&isk->accept_q))
2720		mask = EPOLLIN | EPOLLRDNORM;
2721	spin_unlock(&isk->accept_q_lock);
2722
2723	return mask;
2724}
2725
2726static __poll_t smc_poll(struct file *file, struct socket *sock,
2727			     poll_table *wait)
2728{
2729	struct sock *sk = sock->sk;
2730	struct smc_sock *smc;
2731	__poll_t mask = 0;
2732
2733	if (!sk)
2734		return EPOLLNVAL;
2735
2736	smc = smc_sk(sock->sk);
2737	if (smc->use_fallback) {
2738		/* delegate to CLC child sock */
2739		mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2740		sk->sk_err = smc->clcsock->sk->sk_err;
2741	} else {
2742		if (sk->sk_state != SMC_CLOSED)
2743			sock_poll_wait(file, sock, wait);
2744		if (sk->sk_err)
2745			mask |= EPOLLERR;
2746		if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2747		    (sk->sk_state == SMC_CLOSED))
2748			mask |= EPOLLHUP;
2749		if (sk->sk_state == SMC_LISTEN) {
2750			/* woken up by sk_data_ready in smc_listen_work() */
2751			mask |= smc_accept_poll(sk);
2752		} else if (smc->use_fallback) { /* as result of connect_work()*/
2753			mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2754							   wait);
2755			sk->sk_err = smc->clcsock->sk->sk_err;
2756		} else {
2757			if ((sk->sk_state != SMC_INIT &&
2758			     atomic_read(&smc->conn.sndbuf_space)) ||
2759			    sk->sk_shutdown & SEND_SHUTDOWN) {
2760				mask |= EPOLLOUT | EPOLLWRNORM;
2761			} else {
2762				sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2763				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2764			}
2765			if (atomic_read(&smc->conn.bytes_to_rcv))
2766				mask |= EPOLLIN | EPOLLRDNORM;
2767			if (sk->sk_shutdown & RCV_SHUTDOWN)
2768				mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2769			if (sk->sk_state == SMC_APPCLOSEWAIT1)
2770				mask |= EPOLLIN;
2771			if (smc->conn.urg_state == SMC_URG_VALID)
2772				mask |= EPOLLPRI;
2773		}
2774	}
2775
2776	return mask;
2777}
2778
2779static int smc_shutdown(struct socket *sock, int how)
2780{
2781	struct sock *sk = sock->sk;
2782	bool do_shutdown = true;
2783	struct smc_sock *smc;
2784	int rc = -EINVAL;
2785	int old_state;
2786	int rc1 = 0;
2787
2788	smc = smc_sk(sk);
2789
2790	if ((how < SHUT_RD) || (how > SHUT_RDWR))
2791		return rc;
2792
2793	lock_sock(sk);
2794
2795	if (sock->state == SS_CONNECTING) {
2796		if (sk->sk_state == SMC_ACTIVE)
2797			sock->state = SS_CONNECTED;
2798		else if (sk->sk_state == SMC_PEERCLOSEWAIT1 ||
2799			 sk->sk_state == SMC_PEERCLOSEWAIT2 ||
2800			 sk->sk_state == SMC_APPCLOSEWAIT1 ||
2801			 sk->sk_state == SMC_APPCLOSEWAIT2 ||
2802			 sk->sk_state == SMC_APPFINCLOSEWAIT)
2803			sock->state = SS_DISCONNECTING;
2804	}
2805
2806	rc = -ENOTCONN;
2807	if ((sk->sk_state != SMC_ACTIVE) &&
2808	    (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2809	    (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2810	    (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2811	    (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2812	    (sk->sk_state != SMC_APPFINCLOSEWAIT))
2813		goto out;
2814	if (smc->use_fallback) {
2815		rc = kernel_sock_shutdown(smc->clcsock, how);
2816		sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2817		if (sk->sk_shutdown == SHUTDOWN_MASK) {
2818			sk->sk_state = SMC_CLOSED;
2819			sk->sk_socket->state = SS_UNCONNECTED;
2820			sock_put(sk);
2821		}
2822		goto out;
2823	}
2824	switch (how) {
2825	case SHUT_RDWR:		/* shutdown in both directions */
2826		old_state = sk->sk_state;
2827		rc = smc_close_active(smc);
2828		if (old_state == SMC_ACTIVE &&
2829		    sk->sk_state == SMC_PEERCLOSEWAIT1)
2830			do_shutdown = false;
2831		break;
2832	case SHUT_WR:
2833		rc = smc_close_shutdown_write(smc);
2834		break;
2835	case SHUT_RD:
2836		rc = 0;
2837		/* nothing more to do because peer is not involved */
2838		break;
2839	}
2840	if (do_shutdown && smc->clcsock)
2841		rc1 = kernel_sock_shutdown(smc->clcsock, how);
2842	/* map sock_shutdown_cmd constants to sk_shutdown value range */
2843	sk->sk_shutdown |= how + 1;
2844
2845	if (sk->sk_state == SMC_CLOSED)
2846		sock->state = SS_UNCONNECTED;
2847	else
2848		sock->state = SS_DISCONNECTING;
2849out:
2850	release_sock(sk);
2851	return rc ? rc : rc1;
2852}
2853
2854static int __smc_getsockopt(struct socket *sock, int level, int optname,
2855			    char __user *optval, int __user *optlen)
2856{
2857	struct smc_sock *smc;
2858	int val, len;
2859
2860	smc = smc_sk(sock->sk);
2861
2862	if (get_user(len, optlen))
2863		return -EFAULT;
2864
2865	len = min_t(int, len, sizeof(int));
2866
2867	if (len < 0)
2868		return -EINVAL;
2869
2870	switch (optname) {
2871	case SMC_LIMIT_HS:
2872		val = smc->limit_smc_hs;
2873		break;
2874	default:
2875		return -EOPNOTSUPP;
2876	}
2877
2878	if (put_user(len, optlen))
2879		return -EFAULT;
2880	if (copy_to_user(optval, &val, len))
2881		return -EFAULT;
2882
2883	return 0;
2884}
2885
2886static int __smc_setsockopt(struct socket *sock, int level, int optname,
2887			    sockptr_t optval, unsigned int optlen)
2888{
2889	struct sock *sk = sock->sk;
2890	struct smc_sock *smc;
2891	int val, rc;
2892
2893	smc = smc_sk(sk);
2894
2895	lock_sock(sk);
2896	switch (optname) {
2897	case SMC_LIMIT_HS:
2898		if (optlen < sizeof(int)) {
2899			rc = -EINVAL;
2900			break;
2901		}
2902		if (copy_from_sockptr(&val, optval, sizeof(int))) {
2903			rc = -EFAULT;
2904			break;
2905		}
2906
2907		smc->limit_smc_hs = !!val;
2908		rc = 0;
2909		break;
2910	default:
2911		rc = -EOPNOTSUPP;
2912		break;
2913	}
2914	release_sock(sk);
2915
2916	return rc;
2917}
2918
2919static int smc_setsockopt(struct socket *sock, int level, int optname,
2920			  sockptr_t optval, unsigned int optlen)
2921{
2922	struct sock *sk = sock->sk;
2923	struct smc_sock *smc;
2924	int val, rc;
2925
2926	if (level == SOL_TCP && optname == TCP_ULP)
2927		return -EOPNOTSUPP;
2928	else if (level == SOL_SMC)
2929		return __smc_setsockopt(sock, level, optname, optval, optlen);
2930
2931	smc = smc_sk(sk);
2932
2933	/* generic setsockopts reaching us here always apply to the
2934	 * CLC socket
2935	 */
2936	mutex_lock(&smc->clcsock_release_lock);
2937	if (!smc->clcsock) {
2938		mutex_unlock(&smc->clcsock_release_lock);
2939		return -EBADF;
2940	}
2941	if (unlikely(!smc->clcsock->ops->setsockopt))
2942		rc = -EOPNOTSUPP;
2943	else
2944		rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
2945						   optval, optlen);
2946	if (smc->clcsock->sk->sk_err) {
2947		sk->sk_err = smc->clcsock->sk->sk_err;
2948		sk_error_report(sk);
2949	}
2950	mutex_unlock(&smc->clcsock_release_lock);
2951
2952	if (optlen < sizeof(int))
2953		return -EINVAL;
2954	if (copy_from_sockptr(&val, optval, sizeof(int)))
2955		return -EFAULT;
2956
2957	lock_sock(sk);
2958	if (rc || smc->use_fallback)
2959		goto out;
2960	switch (optname) {
2961	case TCP_FASTOPEN:
2962	case TCP_FASTOPEN_CONNECT:
2963	case TCP_FASTOPEN_KEY:
2964	case TCP_FASTOPEN_NO_COOKIE:
2965		/* option not supported by SMC */
2966		if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2967			rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2968		} else {
2969			rc = -EINVAL;
2970		}
2971		break;
2972	case TCP_NODELAY:
2973		if (sk->sk_state != SMC_INIT &&
2974		    sk->sk_state != SMC_LISTEN &&
2975		    sk->sk_state != SMC_CLOSED) {
2976			if (val) {
2977				SMC_STAT_INC(smc, ndly_cnt);
2978				smc_tx_pending(&smc->conn);
2979				cancel_delayed_work(&smc->conn.tx_work);
2980			}
2981		}
2982		break;
2983	case TCP_CORK:
2984		if (sk->sk_state != SMC_INIT &&
2985		    sk->sk_state != SMC_LISTEN &&
2986		    sk->sk_state != SMC_CLOSED) {
2987			if (!val) {
2988				SMC_STAT_INC(smc, cork_cnt);
2989				smc_tx_pending(&smc->conn);
2990				cancel_delayed_work(&smc->conn.tx_work);
2991			}
2992		}
2993		break;
2994	case TCP_DEFER_ACCEPT:
2995		smc->sockopt_defer_accept = val;
2996		break;
2997	default:
2998		break;
2999	}
3000out:
3001	release_sock(sk);
3002
3003	return rc;
3004}
3005
3006static int smc_getsockopt(struct socket *sock, int level, int optname,
3007			  char __user *optval, int __user *optlen)
3008{
3009	struct smc_sock *smc;
3010	int rc;
3011
3012	if (level == SOL_SMC)
3013		return __smc_getsockopt(sock, level, optname, optval, optlen);
3014
3015	smc = smc_sk(sock->sk);
3016	mutex_lock(&smc->clcsock_release_lock);
3017	if (!smc->clcsock) {
3018		mutex_unlock(&smc->clcsock_release_lock);
3019		return -EBADF;
3020	}
3021	/* socket options apply to the CLC socket */
3022	if (unlikely(!smc->clcsock->ops->getsockopt)) {
3023		mutex_unlock(&smc->clcsock_release_lock);
3024		return -EOPNOTSUPP;
3025	}
3026	rc = smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
3027					   optval, optlen);
3028	mutex_unlock(&smc->clcsock_release_lock);
3029	return rc;
3030}
3031
3032static int smc_ioctl(struct socket *sock, unsigned int cmd,
3033		     unsigned long arg)
3034{
3035	union smc_host_cursor cons, urg;
3036	struct smc_connection *conn;
3037	struct smc_sock *smc;
3038	int answ;
3039
3040	smc = smc_sk(sock->sk);
3041	conn = &smc->conn;
3042	lock_sock(&smc->sk);
3043	if (smc->use_fallback) {
3044		if (!smc->clcsock) {
3045			release_sock(&smc->sk);
3046			return -EBADF;
3047		}
3048		answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
3049		release_sock(&smc->sk);
3050		return answ;
3051	}
3052	switch (cmd) {
3053	case SIOCINQ: /* same as FIONREAD */
3054		if (smc->sk.sk_state == SMC_LISTEN) {
3055			release_sock(&smc->sk);
3056			return -EINVAL;
3057		}
3058		if (smc->sk.sk_state == SMC_INIT ||
3059		    smc->sk.sk_state == SMC_CLOSED)
3060			answ = 0;
3061		else
3062			answ = atomic_read(&smc->conn.bytes_to_rcv);
3063		break;
3064	case SIOCOUTQ:
3065		/* output queue size (not send + not acked) */
3066		if (smc->sk.sk_state == SMC_LISTEN) {
3067			release_sock(&smc->sk);
3068			return -EINVAL;
3069		}
3070		if (smc->sk.sk_state == SMC_INIT ||
3071		    smc->sk.sk_state == SMC_CLOSED)
3072			answ = 0;
3073		else
3074			answ = smc->conn.sndbuf_desc->len -
3075					atomic_read(&smc->conn.sndbuf_space);
3076		break;
3077	case SIOCOUTQNSD:
3078		/* output queue size (not send only) */
3079		if (smc->sk.sk_state == SMC_LISTEN) {
3080			release_sock(&smc->sk);
3081			return -EINVAL;
3082		}
3083		if (smc->sk.sk_state == SMC_INIT ||
3084		    smc->sk.sk_state == SMC_CLOSED)
3085			answ = 0;
3086		else
3087			answ = smc_tx_prepared_sends(&smc->conn);
3088		break;
3089	case SIOCATMARK:
3090		if (smc->sk.sk_state == SMC_LISTEN) {
3091			release_sock(&smc->sk);
3092			return -EINVAL;
3093		}
3094		if (smc->sk.sk_state == SMC_INIT ||
3095		    smc->sk.sk_state == SMC_CLOSED) {
3096			answ = 0;
3097		} else {
3098			smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
3099			smc_curs_copy(&urg, &conn->urg_curs, conn);
3100			answ = smc_curs_diff(conn->rmb_desc->len,
3101					     &cons, &urg) == 1;
3102		}
3103		break;
3104	default:
3105		release_sock(&smc->sk);
3106		return -ENOIOCTLCMD;
3107	}
3108	release_sock(&smc->sk);
3109
3110	return put_user(answ, (int __user *)arg);
3111}
3112
3113static ssize_t smc_sendpage(struct socket *sock, struct page *page,
3114			    int offset, size_t size, int flags)
3115{
3116	struct sock *sk = sock->sk;
3117	struct smc_sock *smc;
3118	int rc = -EPIPE;
3119
3120	smc = smc_sk(sk);
3121	lock_sock(sk);
3122	if (sk->sk_state != SMC_ACTIVE) {
3123		release_sock(sk);
3124		goto out;
3125	}
3126	release_sock(sk);
3127	if (smc->use_fallback) {
3128		rc = kernel_sendpage(smc->clcsock, page, offset,
3129				     size, flags);
3130	} else {
3131		lock_sock(sk);
3132		rc = smc_tx_sendpage(smc, page, offset, size, flags);
3133		release_sock(sk);
3134		SMC_STAT_INC(smc, sendpage_cnt);
3135	}
3136
3137out:
3138	return rc;
3139}
3140
3141/* Map the affected portions of the rmbe into an spd, note the number of bytes
3142 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
3143 * updates till whenever a respective page has been fully processed.
3144 * Note that subsequent recv() calls have to wait till all splice() processing
3145 * completed.
3146 */
3147static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
3148			       struct pipe_inode_info *pipe, size_t len,
3149			       unsigned int flags)
3150{
3151	struct sock *sk = sock->sk;
3152	struct smc_sock *smc;
3153	int rc = -ENOTCONN;
3154
3155	smc = smc_sk(sk);
3156	lock_sock(sk);
3157	if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
3158		/* socket was connected before, no more data to read */
3159		rc = 0;
3160		goto out;
3161	}
3162	if (sk->sk_state == SMC_INIT ||
3163	    sk->sk_state == SMC_LISTEN ||
3164	    sk->sk_state == SMC_CLOSED)
3165		goto out;
3166
3167	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
3168		rc = 0;
3169		goto out;
3170	}
3171
3172	if (smc->use_fallback) {
3173		rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
3174						    pipe, len, flags);
3175	} else {
3176		if (*ppos) {
3177			rc = -ESPIPE;
3178			goto out;
3179		}
3180		if (flags & SPLICE_F_NONBLOCK)
3181			flags = MSG_DONTWAIT;
3182		else
3183			flags = 0;
3184		SMC_STAT_INC(smc, splice_cnt);
3185		rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
3186	}
3187out:
3188	release_sock(sk);
3189
3190	return rc;
3191}
3192
3193/* must look like tcp */
3194static const struct proto_ops smc_sock_ops = {
3195	.family		= PF_SMC,
3196	.owner		= THIS_MODULE,
3197	.release	= smc_release,
3198	.bind		= smc_bind,
3199	.connect	= smc_connect,
3200	.socketpair	= sock_no_socketpair,
3201	.accept		= smc_accept,
3202	.getname	= smc_getname,
3203	.poll		= smc_poll,
3204	.ioctl		= smc_ioctl,
3205	.listen		= smc_listen,
3206	.shutdown	= smc_shutdown,
3207	.setsockopt	= smc_setsockopt,
3208	.getsockopt	= smc_getsockopt,
3209	.sendmsg	= smc_sendmsg,
3210	.recvmsg	= smc_recvmsg,
3211	.mmap		= sock_no_mmap,
3212	.sendpage	= smc_sendpage,
3213	.splice_read	= smc_splice_read,
3214};
3215
3216static int __smc_create(struct net *net, struct socket *sock, int protocol,
3217			int kern, struct socket *clcsock)
3218{
3219	int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
3220	struct smc_sock *smc;
3221	struct sock *sk;
3222	int rc;
3223
3224	rc = -ESOCKTNOSUPPORT;
3225	if (sock->type != SOCK_STREAM)
3226		goto out;
3227
3228	rc = -EPROTONOSUPPORT;
3229	if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
3230		goto out;
3231
3232	rc = -ENOBUFS;
3233	sock->ops = &smc_sock_ops;
3234	sock->state = SS_UNCONNECTED;
3235	sk = smc_sock_alloc(net, sock, protocol);
3236	if (!sk)
3237		goto out;
3238
3239	/* create internal TCP socket for CLC handshake and fallback */
3240	smc = smc_sk(sk);
3241	smc->use_fallback = false; /* assume rdma capability first */
3242	smc->fallback_rsn = 0;
3243
3244	/* default behavior from limit_smc_hs in every net namespace */
3245	smc->limit_smc_hs = net->smc.limit_smc_hs;
3246
3247	rc = 0;
3248	if (!clcsock) {
3249		rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
3250				      &smc->clcsock);
3251		if (rc) {
3252			sk_common_release(sk);
3253			goto out;
3254		}
3255	} else {
3256		smc->clcsock = clcsock;
3257	}
3258
3259out:
3260	return rc;
3261}
3262
3263static int smc_create(struct net *net, struct socket *sock, int protocol,
3264		      int kern)
3265{
3266	return __smc_create(net, sock, protocol, kern, NULL);
3267}
3268
3269static const struct net_proto_family smc_sock_family_ops = {
3270	.family	= PF_SMC,
3271	.owner	= THIS_MODULE,
3272	.create	= smc_create,
3273};
3274
3275static int smc_ulp_init(struct sock *sk)
3276{
3277	struct socket *tcp = sk->sk_socket;
3278	struct net *net = sock_net(sk);
3279	struct socket *smcsock;
3280	int protocol, ret;
3281
3282	/* only TCP can be replaced */
3283	if (tcp->type != SOCK_STREAM || sk->sk_protocol != IPPROTO_TCP ||
3284	    (sk->sk_family != AF_INET && sk->sk_family != AF_INET6))
3285		return -ESOCKTNOSUPPORT;
3286	/* don't handle wq now */
3287	if (tcp->state != SS_UNCONNECTED || !tcp->file || tcp->wq.fasync_list)
3288		return -ENOTCONN;
3289
3290	if (sk->sk_family == AF_INET)
3291		protocol = SMCPROTO_SMC;
3292	else
3293		protocol = SMCPROTO_SMC6;
3294
3295	smcsock = sock_alloc();
3296	if (!smcsock)
3297		return -ENFILE;
3298
3299	smcsock->type = SOCK_STREAM;
3300	__module_get(THIS_MODULE); /* tried in __tcp_ulp_find_autoload */
3301	ret = __smc_create(net, smcsock, protocol, 1, tcp);
3302	if (ret) {
3303		sock_release(smcsock); /* module_put() which ops won't be NULL */
3304		return ret;
3305	}
3306
3307	/* replace tcp socket to smc */
3308	smcsock->file = tcp->file;
3309	smcsock->file->private_data = smcsock;
3310	smcsock->file->f_inode = SOCK_INODE(smcsock); /* replace inode when sock_close */
3311	smcsock->file->f_path.dentry->d_inode = SOCK_INODE(smcsock); /* dput() in __fput */
3312	tcp->file = NULL;
3313
3314	return ret;
3315}
3316
3317static void smc_ulp_clone(const struct request_sock *req, struct sock *newsk,
3318			  const gfp_t priority)
3319{
3320	struct inet_connection_sock *icsk = inet_csk(newsk);
3321
3322	/* don't inherit ulp ops to child when listen */
3323	icsk->icsk_ulp_ops = NULL;
3324}
3325
3326static struct tcp_ulp_ops smc_ulp_ops __read_mostly = {
3327	.name		= "smc",
3328	.owner		= THIS_MODULE,
3329	.init		= smc_ulp_init,
3330	.clone		= smc_ulp_clone,
3331};
3332
3333unsigned int smc_net_id;
3334
3335static __net_init int smc_net_init(struct net *net)
3336{
3337	int rc;
3338
3339	rc = smc_sysctl_net_init(net);
3340	if (rc)
3341		return rc;
3342	return smc_pnet_net_init(net);
3343}
3344
3345static void __net_exit smc_net_exit(struct net *net)
3346{
3347	smc_sysctl_net_exit(net);
3348	smc_pnet_net_exit(net);
3349}
3350
3351static __net_init int smc_net_stat_init(struct net *net)
3352{
3353	return smc_stats_init(net);
3354}
3355
3356static void __net_exit smc_net_stat_exit(struct net *net)
3357{
3358	smc_stats_exit(net);
3359}
3360
3361static struct pernet_operations smc_net_ops = {
3362	.init = smc_net_init,
3363	.exit = smc_net_exit,
3364	.id   = &smc_net_id,
3365	.size = sizeof(struct smc_net),
3366};
3367
3368static struct pernet_operations smc_net_stat_ops = {
3369	.init = smc_net_stat_init,
3370	.exit = smc_net_stat_exit,
3371};
3372
3373static int __init smc_init(void)
3374{
3375	int rc;
3376
3377	rc = register_pernet_subsys(&smc_net_ops);
3378	if (rc)
3379		return rc;
3380
3381	rc = register_pernet_subsys(&smc_net_stat_ops);
3382	if (rc)
3383		goto out_pernet_subsys;
3384
3385	smc_ism_init();
3386	smc_clc_init();
3387
3388	rc = smc_nl_init();
3389	if (rc)
3390		goto out_pernet_subsys_stat;
3391
3392	rc = smc_pnet_init();
3393	if (rc)
3394		goto out_nl;
3395
3396	rc = -ENOMEM;
3397
3398	smc_tcp_ls_wq = alloc_workqueue("smc_tcp_ls_wq", 0, 0);
3399	if (!smc_tcp_ls_wq)
3400		goto out_pnet;
3401
3402	smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
3403	if (!smc_hs_wq)
3404		goto out_alloc_tcp_ls_wq;
3405
3406	smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
3407	if (!smc_close_wq)
3408		goto out_alloc_hs_wq;
3409
3410	rc = smc_core_init();
3411	if (rc) {
3412		pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
3413		goto out_alloc_wqs;
3414	}
3415
3416	rc = smc_llc_init();
3417	if (rc) {
3418		pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
3419		goto out_core;
3420	}
3421
3422	rc = smc_cdc_init();
3423	if (rc) {
3424		pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
3425		goto out_core;
3426	}
3427
3428	rc = proto_register(&smc_proto, 1);
3429	if (rc) {
3430		pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
3431		goto out_core;
3432	}
3433
3434	rc = proto_register(&smc_proto6, 1);
3435	if (rc) {
3436		pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
3437		goto out_proto;
3438	}
3439
3440	rc = sock_register(&smc_sock_family_ops);
3441	if (rc) {
3442		pr_err("%s: sock_register fails with %d\n", __func__, rc);
3443		goto out_proto6;
3444	}
3445	INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
3446	INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
3447
3448	rc = smc_ib_register_client();
3449	if (rc) {
3450		pr_err("%s: ib_register fails with %d\n", __func__, rc);
3451		goto out_sock;
3452	}
3453
3454	rc = tcp_register_ulp(&smc_ulp_ops);
3455	if (rc) {
3456		pr_err("%s: tcp_ulp_register fails with %d\n", __func__, rc);
3457		goto out_ib;
3458	}
3459
3460	static_branch_enable(&tcp_have_smc);
3461	return 0;
3462
3463out_ib:
3464	smc_ib_unregister_client();
3465out_sock:
3466	sock_unregister(PF_SMC);
3467out_proto6:
3468	proto_unregister(&smc_proto6);
3469out_proto:
3470	proto_unregister(&smc_proto);
3471out_core:
3472	smc_core_exit();
3473out_alloc_wqs:
3474	destroy_workqueue(smc_close_wq);
3475out_alloc_hs_wq:
3476	destroy_workqueue(smc_hs_wq);
3477out_alloc_tcp_ls_wq:
3478	destroy_workqueue(smc_tcp_ls_wq);
3479out_pnet:
3480	smc_pnet_exit();
3481out_nl:
3482	smc_nl_exit();
3483out_pernet_subsys_stat:
3484	unregister_pernet_subsys(&smc_net_stat_ops);
3485out_pernet_subsys:
3486	unregister_pernet_subsys(&smc_net_ops);
3487
3488	return rc;
3489}
3490
3491static void __exit smc_exit(void)
3492{
3493	static_branch_disable(&tcp_have_smc);
3494	tcp_unregister_ulp(&smc_ulp_ops);
3495	sock_unregister(PF_SMC);
3496	smc_core_exit();
3497	smc_ib_unregister_client();
3498	destroy_workqueue(smc_close_wq);
3499	destroy_workqueue(smc_tcp_ls_wq);
3500	destroy_workqueue(smc_hs_wq);
3501	proto_unregister(&smc_proto6);
3502	proto_unregister(&smc_proto);
3503	smc_pnet_exit();
3504	smc_nl_exit();
3505	smc_clc_exit();
3506	unregister_pernet_subsys(&smc_net_stat_ops);
3507	unregister_pernet_subsys(&smc_net_ops);
3508	rcu_barrier();
3509}
3510
3511module_init(smc_init);
3512module_exit(smc_exit);
3513
3514MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
3515MODULE_DESCRIPTION("smc socket address family");
3516MODULE_LICENSE("GPL");
3517MODULE_ALIAS_NETPROTO(PF_SMC);
3518MODULE_ALIAS_TCP_ULP("smc");
3519MODULE_ALIAS_GENL_FAMILY(SMC_GENL_FAMILY_NAME);