1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Kernel Connection Multiplexor
   4 *
   5 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
   6 */
   7
   8#include <linux/bpf.h>
   9#include <linux/errno.h>
  10#include <linux/errqueue.h>
  11#include <linux/file.h>
  12#include <linux/filter.h>
  13#include <linux/in.h>
  14#include <linux/kernel.h>
  15#include <linux/module.h>
  16#include <linux/net.h>
  17#include <linux/netdevice.h>
  18#include <linux/poll.h>
  19#include <linux/rculist.h>
  20#include <linux/skbuff.h>
  21#include <linux/socket.h>
  22#include <linux/uaccess.h>
  23#include <linux/workqueue.h>
  24#include <linux/syscalls.h>
  25#include <linux/sched/signal.h>
  26
  27#include <net/kcm.h>
  28#include <net/netns/generic.h>
  29#include <net/sock.h>
  30#include <uapi/linux/kcm.h>
  31
  32unsigned int kcm_net_id;
  33
  34static struct kmem_cache *kcm_psockp __read_mostly;
  35static struct kmem_cache *kcm_muxp __read_mostly;
  36static struct workqueue_struct *kcm_wq;
  37
  38static inline struct kcm_sock *kcm_sk(const struct sock *sk)
  39{
  40	return (struct kcm_sock *)sk;
  41}
  42
  43static inline struct kcm_tx_msg *kcm_tx_msg(struct sk_buff *skb)
  44{
  45	return (struct kcm_tx_msg *)skb->cb;
  46}
  47
  48static void report_csk_error(struct sock *csk, int err)
  49{
  50	csk->sk_err = EPIPE;
  51	sk_error_report(csk);
  52}
  53
  54static void kcm_abort_tx_psock(struct kcm_psock *psock, int err,
  55			       bool wakeup_kcm)
  56{
  57	struct sock *csk = psock->sk;
  58	struct kcm_mux *mux = psock->mux;
  59
  60	/* Unrecoverable error in transmit */
  61
  62	spin_lock_bh(&mux->lock);
  63
  64	if (psock->tx_stopped) {
  65		spin_unlock_bh(&mux->lock);
  66		return;
  67	}
  68
  69	psock->tx_stopped = 1;
  70	KCM_STATS_INCR(psock->stats.tx_aborts);
  71
  72	if (!psock->tx_kcm) {
  73		/* Take off psocks_avail list */
  74		list_del(&psock->psock_avail_list);
  75	} else if (wakeup_kcm) {
  76		/* In this case psock is being aborted while outside of
  77		 * write_msgs and psock is reserved. Schedule tx_work
  78		 * to handle the failure there. Need to commit tx_stopped
  79		 * before queuing work.
  80		 */
  81		smp_mb();
  82
  83		queue_work(kcm_wq, &psock->tx_kcm->tx_work);
  84	}
  85
  86	spin_unlock_bh(&mux->lock);
  87
  88	/* Report error on lower socket */
  89	report_csk_error(csk, err);
  90}
  91
  92/* RX mux lock held. */
  93static void kcm_update_rx_mux_stats(struct kcm_mux *mux,
  94				    struct kcm_psock *psock)
  95{
  96	STRP_STATS_ADD(mux->stats.rx_bytes,
  97		       psock->strp.stats.bytes -
  98		       psock->saved_rx_bytes);
  99	mux->stats.rx_msgs +=
 100		psock->strp.stats.msgs - psock->saved_rx_msgs;
 101	psock->saved_rx_msgs = psock->strp.stats.msgs;
 102	psock->saved_rx_bytes = psock->strp.stats.bytes;
 103}
 104
 105static void kcm_update_tx_mux_stats(struct kcm_mux *mux,
 106				    struct kcm_psock *psock)
 107{
 108	KCM_STATS_ADD(mux->stats.tx_bytes,
 109		      psock->stats.tx_bytes - psock->saved_tx_bytes);
 110	mux->stats.tx_msgs +=
 111		psock->stats.tx_msgs - psock->saved_tx_msgs;
 112	psock->saved_tx_msgs = psock->stats.tx_msgs;
 113	psock->saved_tx_bytes = psock->stats.tx_bytes;
 114}
 115
 116static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
 117
 118/* KCM is ready to receive messages on its queue-- either the KCM is new or
 119 * has become unblocked after being blocked on full socket buffer. Queue any
 120 * pending ready messages on a psock. RX mux lock held.
 121 */
 122static void kcm_rcv_ready(struct kcm_sock *kcm)
 123{
 124	struct kcm_mux *mux = kcm->mux;
 125	struct kcm_psock *psock;
 126	struct sk_buff *skb;
 127
 128	if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled))
 129		return;
 130
 131	while (unlikely((skb = __skb_dequeue(&mux->rx_hold_queue)))) {
 132		if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
 133			/* Assuming buffer limit has been reached */
 134			skb_queue_head(&mux->rx_hold_queue, skb);
 135			WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
 136			return;
 137		}
 138	}
 139
 140	while (!list_empty(&mux->psocks_ready)) {
 141		psock = list_first_entry(&mux->psocks_ready, struct kcm_psock,
 142					 psock_ready_list);
 143
 144		if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {
 145			/* Assuming buffer limit has been reached */
 146			WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
 147			return;
 148		}
 149
 150		/* Consumed the ready message on the psock. Schedule rx_work to
 151		 * get more messages.
 152		 */
 153		list_del(&psock->psock_ready_list);
 154		psock->ready_rx_msg = NULL;
 155		/* Commit clearing of ready_rx_msg for queuing work */
 156		smp_mb();
 157
 158		strp_unpause(&psock->strp);
 159		strp_check_rcv(&psock->strp);
 160	}
 161
 162	/* Buffer limit is okay now, add to ready list */
 163	list_add_tail(&kcm->wait_rx_list,
 164		      &kcm->mux->kcm_rx_waiters);
 165	/* paired with lockless reads in kcm_rfree() */
 166	WRITE_ONCE(kcm->rx_wait, true);
 167}
 168
 169static void kcm_rfree(struct sk_buff *skb)
 170{
 171	struct sock *sk = skb->sk;
 172	struct kcm_sock *kcm = kcm_sk(sk);
 173	struct kcm_mux *mux = kcm->mux;
 174	unsigned int len = skb->truesize;
 175
 176	sk_mem_uncharge(sk, len);
 177	atomic_sub(len, &sk->sk_rmem_alloc);
 178
 179	/* For reading rx_wait and rx_psock without holding lock */
 180	smp_mb__after_atomic();
 181
 182	if (!READ_ONCE(kcm->rx_wait) && !READ_ONCE(kcm->rx_psock) &&
 183	    sk_rmem_alloc_get(sk) < sk->sk_rcvlowat) {
 184		spin_lock_bh(&mux->rx_lock);
 185		kcm_rcv_ready(kcm);
 186		spin_unlock_bh(&mux->rx_lock);
 187	}
 188}
 189
 190static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
 191{
 192	struct sk_buff_head *list = &sk->sk_receive_queue;
 193
 194	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
 195		return -ENOMEM;
 196
 197	if (!sk_rmem_schedule(sk, skb, skb->truesize))
 198		return -ENOBUFS;
 199
 200	skb->dev = NULL;
 201
 202	skb_orphan(skb);
 203	skb->sk = sk;
 204	skb->destructor = kcm_rfree;
 205	atomic_add(skb->truesize, &sk->sk_rmem_alloc);
 206	sk_mem_charge(sk, skb->truesize);
 207
 208	skb_queue_tail(list, skb);
 209
 210	if (!sock_flag(sk, SOCK_DEAD))
 211		sk->sk_data_ready(sk);
 212
 213	return 0;
 214}
 215
 216/* Requeue received messages for a kcm socket to other kcm sockets. This is
 217 * called with a kcm socket is receive disabled.
 218 * RX mux lock held.
 219 */
 220static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head)
 221{
 222	struct sk_buff *skb;
 223	struct kcm_sock *kcm;
 224
 225	while ((skb = skb_dequeue(head))) {
 226		/* Reset destructor to avoid calling kcm_rcv_ready */
 227		skb->destructor = sock_rfree;
 228		skb_orphan(skb);
 229try_again:
 230		if (list_empty(&mux->kcm_rx_waiters)) {
 231			skb_queue_tail(&mux->rx_hold_queue, skb);
 232			continue;
 233		}
 234
 235		kcm = list_first_entry(&mux->kcm_rx_waiters,
 236				       struct kcm_sock, wait_rx_list);
 237
 238		if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
 239			/* Should mean socket buffer full */
 240			list_del(&kcm->wait_rx_list);
 241			/* paired with lockless reads in kcm_rfree() */
 242			WRITE_ONCE(kcm->rx_wait, false);
 243
 244			/* Commit rx_wait to read in kcm_free */
 245			smp_wmb();
 246
 247			goto try_again;
 248		}
 249	}
 250}
 251
 252/* Lower sock lock held */
 253static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock,
 254				       struct sk_buff *head)
 255{
 256	struct kcm_mux *mux = psock->mux;
 257	struct kcm_sock *kcm;
 258
 259	WARN_ON(psock->ready_rx_msg);
 260
 261	if (psock->rx_kcm)
 262		return psock->rx_kcm;
 263
 264	spin_lock_bh(&mux->rx_lock);
 265
 266	if (psock->rx_kcm) {
 267		spin_unlock_bh(&mux->rx_lock);
 268		return psock->rx_kcm;
 269	}
 270
 271	kcm_update_rx_mux_stats(mux, psock);
 272
 273	if (list_empty(&mux->kcm_rx_waiters)) {
 274		psock->ready_rx_msg = head;
 275		strp_pause(&psock->strp);
 276		list_add_tail(&psock->psock_ready_list,
 277			      &mux->psocks_ready);
 278		spin_unlock_bh(&mux->rx_lock);
 279		return NULL;
 280	}
 281
 282	kcm = list_first_entry(&mux->kcm_rx_waiters,
 283			       struct kcm_sock, wait_rx_list);
 284	list_del(&kcm->wait_rx_list);
 285	/* paired with lockless reads in kcm_rfree() */
 286	WRITE_ONCE(kcm->rx_wait, false);
 287
 288	psock->rx_kcm = kcm;
 289	/* paired with lockless reads in kcm_rfree() */
 290	WRITE_ONCE(kcm->rx_psock, psock);
 291
 292	spin_unlock_bh(&mux->rx_lock);
 293
 294	return kcm;
 295}
 296
 297static void kcm_done(struct kcm_sock *kcm);
 298
 299static void kcm_done_work(struct work_struct *w)
 300{
 301	kcm_done(container_of(w, struct kcm_sock, done_work));
 302}
 303
 304/* Lower sock held */
 305static void unreserve_rx_kcm(struct kcm_psock *psock,
 306			     bool rcv_ready)
 307{
 308	struct kcm_sock *kcm = psock->rx_kcm;
 309	struct kcm_mux *mux = psock->mux;
 310
 311	if (!kcm)
 312		return;
 313
 314	spin_lock_bh(&mux->rx_lock);
 315
 316	psock->rx_kcm = NULL;
 317	/* paired with lockless reads in kcm_rfree() */
 318	WRITE_ONCE(kcm->rx_psock, NULL);
 319
 320	/* Commit kcm->rx_psock before sk_rmem_alloc_get to sync with
 321	 * kcm_rfree
 322	 */
 323	smp_mb();
 324
 325	if (unlikely(kcm->done)) {
 326		spin_unlock_bh(&mux->rx_lock);
 327
 328		/* Need to run kcm_done in a task since we need to qcquire
 329		 * callback locks which may already be held here.
 330		 */
 331		INIT_WORK(&kcm->done_work, kcm_done_work);
 332		schedule_work(&kcm->done_work);
 333		return;
 334	}
 335
 336	if (unlikely(kcm->rx_disabled)) {
 337		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
 338	} else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) {
 339		/* Check for degenerative race with rx_wait that all
 340		 * data was dequeued (accounted for in kcm_rfree).
 341		 */
 342		kcm_rcv_ready(kcm);
 343	}
 344	spin_unlock_bh(&mux->rx_lock);
 345}
 346
 347/* Lower sock lock held */
 348static void psock_data_ready(struct sock *sk)
 349{
 350	struct kcm_psock *psock;
 351
 352	read_lock_bh(&sk->sk_callback_lock);
 353
 354	psock = (struct kcm_psock *)sk->sk_user_data;
 355	if (likely(psock))
 356		strp_data_ready(&psock->strp);
 357
 358	read_unlock_bh(&sk->sk_callback_lock);
 359}
 360
 361/* Called with lower sock held */
 362static void kcm_rcv_strparser(struct strparser *strp, struct sk_buff *skb)
 363{
 364	struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
 365	struct kcm_sock *kcm;
 366
 367try_queue:
 368	kcm = reserve_rx_kcm(psock, skb);
 369	if (!kcm) {
 370		 /* Unable to reserve a KCM, message is held in psock and strp
 371		  * is paused.
 372		  */
 373		return;
 374	}
 375
 376	if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
 377		/* Should mean socket buffer full */
 378		unreserve_rx_kcm(psock, false);
 379		goto try_queue;
 380	}
 381}
 382
 383static int kcm_parse_func_strparser(struct strparser *strp, struct sk_buff *skb)
 384{
 385	struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
 386	struct bpf_prog *prog = psock->bpf_prog;
 387	int res;
 388
 389	res = bpf_prog_run_pin_on_cpu(prog, skb);
 
 
 390	return res;
 391}
 392
 393static int kcm_read_sock_done(struct strparser *strp, int err)
 394{
 395	struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
 396
 397	unreserve_rx_kcm(psock, true);
 398
 399	return err;
 400}
 401
 402static void psock_state_change(struct sock *sk)
 403{
 404	/* TCP only does a EPOLLIN for a half close. Do a EPOLLHUP here
 405	 * since application will normally not poll with EPOLLIN
 406	 * on the TCP sockets.
 407	 */
 408
 409	report_csk_error(sk, EPIPE);
 410}
 411
 412static void psock_write_space(struct sock *sk)
 413{
 414	struct kcm_psock *psock;
 415	struct kcm_mux *mux;
 416	struct kcm_sock *kcm;
 417
 418	read_lock_bh(&sk->sk_callback_lock);
 419
 420	psock = (struct kcm_psock *)sk->sk_user_data;
 421	if (unlikely(!psock))
 422		goto out;
 423	mux = psock->mux;
 424
 425	spin_lock_bh(&mux->lock);
 426
 427	/* Check if the socket is reserved so someone is waiting for sending. */
 428	kcm = psock->tx_kcm;
 429	if (kcm && !unlikely(kcm->tx_stopped))
 430		queue_work(kcm_wq, &kcm->tx_work);
 431
 432	spin_unlock_bh(&mux->lock);
 433out:
 434	read_unlock_bh(&sk->sk_callback_lock);
 435}
 436
 437static void unreserve_psock(struct kcm_sock *kcm);
 438
 439/* kcm sock is locked. */
 440static struct kcm_psock *reserve_psock(struct kcm_sock *kcm)
 441{
 442	struct kcm_mux *mux = kcm->mux;
 443	struct kcm_psock *psock;
 444
 445	psock = kcm->tx_psock;
 446
 447	smp_rmb(); /* Must read tx_psock before tx_wait */
 448
 449	if (psock) {
 450		WARN_ON(kcm->tx_wait);
 451		if (unlikely(psock->tx_stopped))
 452			unreserve_psock(kcm);
 453		else
 454			return kcm->tx_psock;
 455	}
 456
 457	spin_lock_bh(&mux->lock);
 458
 459	/* Check again under lock to see if psock was reserved for this
 460	 * psock via psock_unreserve.
 461	 */
 462	psock = kcm->tx_psock;
 463	if (unlikely(psock)) {
 464		WARN_ON(kcm->tx_wait);
 465		spin_unlock_bh(&mux->lock);
 466		return kcm->tx_psock;
 467	}
 468
 469	if (!list_empty(&mux->psocks_avail)) {
 470		psock = list_first_entry(&mux->psocks_avail,
 471					 struct kcm_psock,
 472					 psock_avail_list);
 473		list_del(&psock->psock_avail_list);
 474		if (kcm->tx_wait) {
 475			list_del(&kcm->wait_psock_list);
 476			kcm->tx_wait = false;
 477		}
 478		kcm->tx_psock = psock;
 479		psock->tx_kcm = kcm;
 480		KCM_STATS_INCR(psock->stats.reserved);
 481	} else if (!kcm->tx_wait) {
 482		list_add_tail(&kcm->wait_psock_list,
 483			      &mux->kcm_tx_waiters);
 484		kcm->tx_wait = true;
 485	}
 486
 487	spin_unlock_bh(&mux->lock);
 488
 489	return psock;
 490}
 491
 492/* mux lock held */
 493static void psock_now_avail(struct kcm_psock *psock)
 494{
 495	struct kcm_mux *mux = psock->mux;
 496	struct kcm_sock *kcm;
 497
 498	if (list_empty(&mux->kcm_tx_waiters)) {
 499		list_add_tail(&psock->psock_avail_list,
 500			      &mux->psocks_avail);
 501	} else {
 502		kcm = list_first_entry(&mux->kcm_tx_waiters,
 503				       struct kcm_sock,
 504				       wait_psock_list);
 505		list_del(&kcm->wait_psock_list);
 506		kcm->tx_wait = false;
 507		psock->tx_kcm = kcm;
 508
 509		/* Commit before changing tx_psock since that is read in
 510		 * reserve_psock before queuing work.
 511		 */
 512		smp_mb();
 513
 514		kcm->tx_psock = psock;
 515		KCM_STATS_INCR(psock->stats.reserved);
 516		queue_work(kcm_wq, &kcm->tx_work);
 517	}
 518}
 519
 520/* kcm sock is locked. */
 521static void unreserve_psock(struct kcm_sock *kcm)
 522{
 523	struct kcm_psock *psock;
 524	struct kcm_mux *mux = kcm->mux;
 525
 526	spin_lock_bh(&mux->lock);
 527
 528	psock = kcm->tx_psock;
 529
 530	if (WARN_ON(!psock)) {
 531		spin_unlock_bh(&mux->lock);
 532		return;
 533	}
 534
 535	smp_rmb(); /* Read tx_psock before tx_wait */
 536
 537	kcm_update_tx_mux_stats(mux, psock);
 538
 539	WARN_ON(kcm->tx_wait);
 540
 541	kcm->tx_psock = NULL;
 542	psock->tx_kcm = NULL;
 543	KCM_STATS_INCR(psock->stats.unreserved);
 544
 545	if (unlikely(psock->tx_stopped)) {
 546		if (psock->done) {
 547			/* Deferred free */
 548			list_del(&psock->psock_list);
 549			mux->psocks_cnt--;
 550			sock_put(psock->sk);
 551			fput(psock->sk->sk_socket->file);
 552			kmem_cache_free(kcm_psockp, psock);
 553		}
 554
 555		/* Don't put back on available list */
 556
 557		spin_unlock_bh(&mux->lock);
 558
 559		return;
 560	}
 561
 562	psock_now_avail(psock);
 563
 564	spin_unlock_bh(&mux->lock);
 565}
 566
 567static void kcm_report_tx_retry(struct kcm_sock *kcm)
 568{
 569	struct kcm_mux *mux = kcm->mux;
 570
 571	spin_lock_bh(&mux->lock);
 572	KCM_STATS_INCR(mux->stats.tx_retries);
 573	spin_unlock_bh(&mux->lock);
 574}
 575
 576/* Write any messages ready on the kcm socket.  Called with kcm sock lock
 577 * held.  Return bytes actually sent or error.
 578 */
 579static int kcm_write_msgs(struct kcm_sock *kcm)
 580{
 581	struct sock *sk = &kcm->sk;
 582	struct kcm_psock *psock;
 583	struct sk_buff *skb, *head;
 584	struct kcm_tx_msg *txm;
 585	unsigned short fragidx, frag_offset;
 586	unsigned int sent, total_sent = 0;
 587	int ret = 0;
 588
 589	kcm->tx_wait_more = false;
 590	psock = kcm->tx_psock;
 591	if (unlikely(psock && psock->tx_stopped)) {
 592		/* A reserved psock was aborted asynchronously. Unreserve
 593		 * it and we'll retry the message.
 594		 */
 595		unreserve_psock(kcm);
 596		kcm_report_tx_retry(kcm);
 597		if (skb_queue_empty(&sk->sk_write_queue))
 598			return 0;
 599
 600		kcm_tx_msg(skb_peek(&sk->sk_write_queue))->sent = 0;
 601
 602	} else if (skb_queue_empty(&sk->sk_write_queue)) {
 603		return 0;
 604	}
 605
 606	head = skb_peek(&sk->sk_write_queue);
 607	txm = kcm_tx_msg(head);
 608
 609	if (txm->sent) {
 610		/* Send of first skbuff in queue already in progress */
 611		if (WARN_ON(!psock)) {
 612			ret = -EINVAL;
 613			goto out;
 614		}
 615		sent = txm->sent;
 616		frag_offset = txm->frag_offset;
 617		fragidx = txm->fragidx;
 618		skb = txm->frag_skb;
 619
 620		goto do_frag;
 621	}
 622
 623try_again:
 624	psock = reserve_psock(kcm);
 625	if (!psock)
 626		goto out;
 627
 628	do {
 629		skb = head;
 630		txm = kcm_tx_msg(head);
 631		sent = 0;
 632
 633do_frag_list:
 634		if (WARN_ON(!skb_shinfo(skb)->nr_frags)) {
 635			ret = -EINVAL;
 636			goto out;
 637		}
 638
 639		for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags;
 640		     fragidx++) {
 641			skb_frag_t *frag;
 642
 643			frag_offset = 0;
 644do_frag:
 645			frag = &skb_shinfo(skb)->frags[fragidx];
 646			if (WARN_ON(!skb_frag_size(frag))) {
 647				ret = -EINVAL;
 648				goto out;
 649			}
 650
 651			ret = kernel_sendpage(psock->sk->sk_socket,
 652					      skb_frag_page(frag),
 653					      skb_frag_off(frag) + frag_offset,
 654					      skb_frag_size(frag) - frag_offset,
 655					      MSG_DONTWAIT);
 656			if (ret <= 0) {
 657				if (ret == -EAGAIN) {
 658					/* Save state to try again when there's
 659					 * write space on the socket
 660					 */
 661					txm->sent = sent;
 662					txm->frag_offset = frag_offset;
 663					txm->fragidx = fragidx;
 664					txm->frag_skb = skb;
 665
 666					ret = 0;
 667					goto out;
 668				}
 669
 670				/* Hard failure in sending message, abort this
 671				 * psock since it has lost framing
 672				 * synchronization and retry sending the
 673				 * message from the beginning.
 674				 */
 675				kcm_abort_tx_psock(psock, ret ? -ret : EPIPE,
 676						   true);
 677				unreserve_psock(kcm);
 678
 679				txm->sent = 0;
 680				kcm_report_tx_retry(kcm);
 681				ret = 0;
 682
 683				goto try_again;
 684			}
 685
 686			sent += ret;
 687			frag_offset += ret;
 688			KCM_STATS_ADD(psock->stats.tx_bytes, ret);
 689			if (frag_offset < skb_frag_size(frag)) {
 690				/* Not finished with this frag */
 691				goto do_frag;
 692			}
 693		}
 694
 695		if (skb == head) {
 696			if (skb_has_frag_list(skb)) {
 697				skb = skb_shinfo(skb)->frag_list;
 698				goto do_frag_list;
 699			}
 700		} else if (skb->next) {
 701			skb = skb->next;
 702			goto do_frag_list;
 703		}
 704
 705		/* Successfully sent the whole packet, account for it. */
 706		skb_dequeue(&sk->sk_write_queue);
 707		kfree_skb(head);
 708		sk->sk_wmem_queued -= sent;
 709		total_sent += sent;
 710		KCM_STATS_INCR(psock->stats.tx_msgs);
 711	} while ((head = skb_peek(&sk->sk_write_queue)));
 712out:
 713	if (!head) {
 714		/* Done with all queued messages. */
 715		WARN_ON(!skb_queue_empty(&sk->sk_write_queue));
 716		unreserve_psock(kcm);
 717	}
 718
 719	/* Check if write space is available */
 720	sk->sk_write_space(sk);
 721
 722	return total_sent ? : ret;
 723}
 724
 725static void kcm_tx_work(struct work_struct *w)
 726{
 727	struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work);
 728	struct sock *sk = &kcm->sk;
 729	int err;
 730
 731	lock_sock(sk);
 732
 733	/* Primarily for SOCK_DGRAM sockets, also handle asynchronous tx
 734	 * aborts
 735	 */
 736	err = kcm_write_msgs(kcm);
 737	if (err < 0) {
 738		/* Hard failure in write, report error on KCM socket */
 739		pr_warn("KCM: Hard failure on kcm_write_msgs %d\n", err);
 740		report_csk_error(&kcm->sk, -err);
 741		goto out;
 742	}
 743
 744	/* Primarily for SOCK_SEQPACKET sockets */
 745	if (likely(sk->sk_socket) &&
 746	    test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
 747		clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 748		sk->sk_write_space(sk);
 749	}
 750
 751out:
 752	release_sock(sk);
 753}
 754
 755static void kcm_push(struct kcm_sock *kcm)
 756{
 757	if (kcm->tx_wait_more)
 758		kcm_write_msgs(kcm);
 759}
 760
 761static ssize_t kcm_sendpage(struct socket *sock, struct page *page,
 762			    int offset, size_t size, int flags)
 763
 764{
 765	struct sock *sk = sock->sk;
 766	struct kcm_sock *kcm = kcm_sk(sk);
 767	struct sk_buff *skb = NULL, *head = NULL;
 768	long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 769	bool eor;
 770	int err = 0;
 771	int i;
 772
 773	if (flags & MSG_SENDPAGE_NOTLAST)
 774		flags |= MSG_MORE;
 775
 776	/* No MSG_EOR from splice, only look at MSG_MORE */
 777	eor = !(flags & MSG_MORE);
 778
 779	lock_sock(sk);
 780
 781	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 782
 783	err = -EPIPE;
 784	if (sk->sk_err)
 785		goto out_error;
 786
 787	if (kcm->seq_skb) {
 788		/* Previously opened message */
 789		head = kcm->seq_skb;
 790		skb = kcm_tx_msg(head)->last_skb;
 791		i = skb_shinfo(skb)->nr_frags;
 792
 793		if (skb_can_coalesce(skb, i, page, offset)) {
 794			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
 795			skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
 796			goto coalesced;
 797		}
 798
 799		if (i >= MAX_SKB_FRAGS) {
 800			struct sk_buff *tskb;
 801
 802			tskb = alloc_skb(0, sk->sk_allocation);
 803			while (!tskb) {
 804				kcm_push(kcm);
 805				err = sk_stream_wait_memory(sk, &timeo);
 806				if (err)
 807					goto out_error;
 808			}
 809
 810			if (head == skb)
 811				skb_shinfo(head)->frag_list = tskb;
 812			else
 813				skb->next = tskb;
 814
 815			skb = tskb;
 816			skb->ip_summed = CHECKSUM_UNNECESSARY;
 817			i = 0;
 818		}
 819	} else {
 820		/* Call the sk_stream functions to manage the sndbuf mem. */
 821		if (!sk_stream_memory_free(sk)) {
 822			kcm_push(kcm);
 823			set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 824			err = sk_stream_wait_memory(sk, &timeo);
 825			if (err)
 826				goto out_error;
 827		}
 828
 829		head = alloc_skb(0, sk->sk_allocation);
 830		while (!head) {
 831			kcm_push(kcm);
 832			err = sk_stream_wait_memory(sk, &timeo);
 833			if (err)
 834				goto out_error;
 835		}
 836
 837		skb = head;
 838		i = 0;
 839	}
 840
 841	get_page(page);
 842	skb_fill_page_desc_noacc(skb, i, page, offset, size);
 843	skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
 844
 845coalesced:
 846	skb->len += size;
 847	skb->data_len += size;
 848	skb->truesize += size;
 849	sk->sk_wmem_queued += size;
 850	sk_mem_charge(sk, size);
 851
 852	if (head != skb) {
 853		head->len += size;
 854		head->data_len += size;
 855		head->truesize += size;
 856	}
 857
 858	if (eor) {
 859		bool not_busy = skb_queue_empty(&sk->sk_write_queue);
 860
 861		/* Message complete, queue it on send buffer */
 862		__skb_queue_tail(&sk->sk_write_queue, head);
 863		kcm->seq_skb = NULL;
 864		KCM_STATS_INCR(kcm->stats.tx_msgs);
 865
 866		if (flags & MSG_BATCH) {
 867			kcm->tx_wait_more = true;
 868		} else if (kcm->tx_wait_more || not_busy) {
 869			err = kcm_write_msgs(kcm);
 870			if (err < 0) {
 871				/* We got a hard error in write_msgs but have
 872				 * already queued this message. Report an error
 873				 * in the socket, but don't affect return value
 874				 * from sendmsg
 875				 */
 876				pr_warn("KCM: Hard failure on kcm_write_msgs\n");
 877				report_csk_error(&kcm->sk, -err);
 878			}
 879		}
 880	} else {
 881		/* Message not complete, save state */
 882		kcm->seq_skb = head;
 883		kcm_tx_msg(head)->last_skb = skb;
 884	}
 885
 886	KCM_STATS_ADD(kcm->stats.tx_bytes, size);
 887
 888	release_sock(sk);
 889	return size;
 890
 891out_error:
 892	kcm_push(kcm);
 893
 894	err = sk_stream_error(sk, flags, err);
 895
 896	/* make sure we wake any epoll edge trigger waiter */
 897	if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
 898		sk->sk_write_space(sk);
 899
 900	release_sock(sk);
 901	return err;
 902}
 903
 904static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
 905{
 906	struct sock *sk = sock->sk;
 907	struct kcm_sock *kcm = kcm_sk(sk);
 908	struct sk_buff *skb = NULL, *head = NULL;
 909	size_t copy, copied = 0;
 910	long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
 911	int eor = (sock->type == SOCK_DGRAM) ?
 912		  !(msg->msg_flags & MSG_MORE) : !!(msg->msg_flags & MSG_EOR);
 913	int err = -EPIPE;
 914
 915	lock_sock(sk);
 916
 917	/* Per tcp_sendmsg this should be in poll */
 918	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 919
 920	if (sk->sk_err)
 921		goto out_error;
 922
 923	if (kcm->seq_skb) {
 924		/* Previously opened message */
 925		head = kcm->seq_skb;
 926		skb = kcm_tx_msg(head)->last_skb;
 927		goto start;
 928	}
 929
 930	/* Call the sk_stream functions to manage the sndbuf mem. */
 931	if (!sk_stream_memory_free(sk)) {
 932		kcm_push(kcm);
 933		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 934		err = sk_stream_wait_memory(sk, &timeo);
 935		if (err)
 936			goto out_error;
 937	}
 938
 939	if (msg_data_left(msg)) {
 940		/* New message, alloc head skb */
 941		head = alloc_skb(0, sk->sk_allocation);
 942		while (!head) {
 943			kcm_push(kcm);
 944			err = sk_stream_wait_memory(sk, &timeo);
 945			if (err)
 946				goto out_error;
 947
 948			head = alloc_skb(0, sk->sk_allocation);
 949		}
 950
 951		skb = head;
 952
 953		/* Set ip_summed to CHECKSUM_UNNECESSARY to avoid calling
 954		 * csum_and_copy_from_iter from skb_do_copy_data_nocache.
 955		 */
 956		skb->ip_summed = CHECKSUM_UNNECESSARY;
 957	}
 958
 959start:
 960	while (msg_data_left(msg)) {
 961		bool merge = true;
 962		int i = skb_shinfo(skb)->nr_frags;
 963		struct page_frag *pfrag = sk_page_frag(sk);
 964
 965		if (!sk_page_frag_refill(sk, pfrag))
 966			goto wait_for_memory;
 967
 968		if (!skb_can_coalesce(skb, i, pfrag->page,
 969				      pfrag->offset)) {
 970			if (i == MAX_SKB_FRAGS) {
 971				struct sk_buff *tskb;
 972
 973				tskb = alloc_skb(0, sk->sk_allocation);
 974				if (!tskb)
 975					goto wait_for_memory;
 976
 977				if (head == skb)
 978					skb_shinfo(head)->frag_list = tskb;
 979				else
 980					skb->next = tskb;
 981
 982				skb = tskb;
 983				skb->ip_summed = CHECKSUM_UNNECESSARY;
 984				continue;
 985			}
 986			merge = false;
 987		}
 988
 989		copy = min_t(int, msg_data_left(msg),
 990			     pfrag->size - pfrag->offset);
 991
 992		if (!sk_wmem_schedule(sk, copy))
 993			goto wait_for_memory;
 994
 995		err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
 996					       pfrag->page,
 997					       pfrag->offset,
 998					       copy);
 999		if (err)
1000			goto out_error;
1001
1002		/* Update the skb. */
1003		if (merge) {
1004			skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1005		} else {
1006			skb_fill_page_desc(skb, i, pfrag->page,
1007					   pfrag->offset, copy);
1008			get_page(pfrag->page);
1009		}
1010
1011		pfrag->offset += copy;
1012		copied += copy;
1013		if (head != skb) {
1014			head->len += copy;
1015			head->data_len += copy;
1016		}
1017
1018		continue;
1019
1020wait_for_memory:
1021		kcm_push(kcm);
1022		err = sk_stream_wait_memory(sk, &timeo);
1023		if (err)
1024			goto out_error;
1025	}
1026
1027	if (eor) {
1028		bool not_busy = skb_queue_empty(&sk->sk_write_queue);
1029
1030		if (head) {
1031			/* Message complete, queue it on send buffer */
1032			__skb_queue_tail(&sk->sk_write_queue, head);
1033			kcm->seq_skb = NULL;
1034			KCM_STATS_INCR(kcm->stats.tx_msgs);
1035		}
1036
1037		if (msg->msg_flags & MSG_BATCH) {
1038			kcm->tx_wait_more = true;
1039		} else if (kcm->tx_wait_more || not_busy) {
1040			err = kcm_write_msgs(kcm);
1041			if (err < 0) {
1042				/* We got a hard error in write_msgs but have
1043				 * already queued this message. Report an error
1044				 * in the socket, but don't affect return value
1045				 * from sendmsg
1046				 */
1047				pr_warn("KCM: Hard failure on kcm_write_msgs\n");
1048				report_csk_error(&kcm->sk, -err);
1049			}
1050		}
1051	} else {
1052		/* Message not complete, save state */
1053partial_message:
1054		if (head) {
1055			kcm->seq_skb = head;
1056			kcm_tx_msg(head)->last_skb = skb;
1057		}
1058	}
1059
1060	KCM_STATS_ADD(kcm->stats.tx_bytes, copied);
1061
1062	release_sock(sk);
1063	return copied;
1064
1065out_error:
1066	kcm_push(kcm);
1067
1068	if (copied && sock->type == SOCK_SEQPACKET) {
1069		/* Wrote some bytes before encountering an
1070		 * error, return partial success.
1071		 */
1072		goto partial_message;
1073	}
1074
1075	if (head != kcm->seq_skb)
1076		kfree_skb(head);
1077
1078	err = sk_stream_error(sk, msg->msg_flags, err);
1079
1080	/* make sure we wake any epoll edge trigger waiter */
1081	if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
1082		sk->sk_write_space(sk);
1083
1084	release_sock(sk);
1085	return err;
1086}
1087
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1088static int kcm_recvmsg(struct socket *sock, struct msghdr *msg,
1089		       size_t len, int flags)
1090{
1091	struct sock *sk = sock->sk;
1092	struct kcm_sock *kcm = kcm_sk(sk);
1093	int err = 0;
 
1094	struct strp_msg *stm;
1095	int copied = 0;
1096	struct sk_buff *skb;
1097
1098	skb = skb_recv_datagram(sk, flags, &err);
 
 
 
 
1099	if (!skb)
1100		goto out;
1101
1102	/* Okay, have a message on the receive queue */
1103
1104	stm = strp_msg(skb);
1105
1106	if (len > stm->full_len)
1107		len = stm->full_len;
1108
1109	err = skb_copy_datagram_msg(skb, stm->offset, msg, len);
1110	if (err < 0)
1111		goto out;
1112
1113	copied = len;
1114	if (likely(!(flags & MSG_PEEK))) {
1115		KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
1116		if (copied < stm->full_len) {
1117			if (sock->type == SOCK_DGRAM) {
1118				/* Truncated message */
1119				msg->msg_flags |= MSG_TRUNC;
1120				goto msg_finished;
1121			}
1122			stm->offset += copied;
1123			stm->full_len -= copied;
1124		} else {
1125msg_finished:
1126			/* Finished with message */
1127			msg->msg_flags |= MSG_EOR;
1128			KCM_STATS_INCR(kcm->stats.rx_msgs);
 
 
1129		}
1130	}
1131
1132out:
1133	skb_free_datagram(sk, skb);
 
1134	return copied ? : err;
1135}
1136
1137static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
1138			       struct pipe_inode_info *pipe, size_t len,
1139			       unsigned int flags)
1140{
1141	struct sock *sk = sock->sk;
1142	struct kcm_sock *kcm = kcm_sk(sk);
 
1143	struct strp_msg *stm;
1144	int err = 0;
1145	ssize_t copied;
1146	struct sk_buff *skb;
1147
1148	/* Only support splice for SOCKSEQPACKET */
1149
1150	skb = skb_recv_datagram(sk, flags, &err);
 
 
 
 
1151	if (!skb)
1152		goto err_out;
1153
1154	/* Okay, have a message on the receive queue */
1155
1156	stm = strp_msg(skb);
1157
1158	if (len > stm->full_len)
1159		len = stm->full_len;
1160
1161	copied = skb_splice_bits(skb, sk, stm->offset, pipe, len, flags);
1162	if (copied < 0) {
1163		err = copied;
1164		goto err_out;
1165	}
1166
1167	KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
1168
1169	stm->offset += copied;
1170	stm->full_len -= copied;
1171
1172	/* We have no way to return MSG_EOR. If all the bytes have been
1173	 * read we still leave the message in the receive socket buffer.
1174	 * A subsequent recvmsg needs to be done to return MSG_EOR and
1175	 * finish reading the message.
1176	 */
1177
1178	skb_free_datagram(sk, skb);
 
1179	return copied;
1180
1181err_out:
1182	skb_free_datagram(sk, skb);
 
1183	return err;
1184}
1185
1186/* kcm sock lock held */
1187static void kcm_recv_disable(struct kcm_sock *kcm)
1188{
1189	struct kcm_mux *mux = kcm->mux;
1190
1191	if (kcm->rx_disabled)
1192		return;
1193
1194	spin_lock_bh(&mux->rx_lock);
1195
1196	kcm->rx_disabled = 1;
1197
1198	/* If a psock is reserved we'll do cleanup in unreserve */
1199	if (!kcm->rx_psock) {
1200		if (kcm->rx_wait) {
1201			list_del(&kcm->wait_rx_list);
1202			/* paired with lockless reads in kcm_rfree() */
1203			WRITE_ONCE(kcm->rx_wait, false);
1204		}
1205
1206		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
1207	}
1208
1209	spin_unlock_bh(&mux->rx_lock);
1210}
1211
1212/* kcm sock lock held */
1213static void kcm_recv_enable(struct kcm_sock *kcm)
1214{
1215	struct kcm_mux *mux = kcm->mux;
1216
1217	if (!kcm->rx_disabled)
1218		return;
1219
1220	spin_lock_bh(&mux->rx_lock);
1221
1222	kcm->rx_disabled = 0;
1223	kcm_rcv_ready(kcm);
1224
1225	spin_unlock_bh(&mux->rx_lock);
1226}
1227
1228static int kcm_setsockopt(struct socket *sock, int level, int optname,
1229			  sockptr_t optval, unsigned int optlen)
1230{
1231	struct kcm_sock *kcm = kcm_sk(sock->sk);
1232	int val, valbool;
1233	int err = 0;
1234
1235	if (level != SOL_KCM)
1236		return -ENOPROTOOPT;
1237
1238	if (optlen < sizeof(int))
1239		return -EINVAL;
1240
1241	if (copy_from_sockptr(&val, optval, sizeof(int)))
1242		return -EFAULT;
1243
1244	valbool = val ? 1 : 0;
1245
1246	switch (optname) {
1247	case KCM_RECV_DISABLE:
1248		lock_sock(&kcm->sk);
1249		if (valbool)
1250			kcm_recv_disable(kcm);
1251		else
1252			kcm_recv_enable(kcm);
1253		release_sock(&kcm->sk);
1254		break;
1255	default:
1256		err = -ENOPROTOOPT;
1257	}
1258
1259	return err;
1260}
1261
1262static int kcm_getsockopt(struct socket *sock, int level, int optname,
1263			  char __user *optval, int __user *optlen)
1264{
1265	struct kcm_sock *kcm = kcm_sk(sock->sk);
1266	int val, len;
1267
1268	if (level != SOL_KCM)
1269		return -ENOPROTOOPT;
1270
1271	if (get_user(len, optlen))
1272		return -EFAULT;
1273
1274	len = min_t(unsigned int, len, sizeof(int));
1275	if (len < 0)
1276		return -EINVAL;
1277
1278	switch (optname) {
1279	case KCM_RECV_DISABLE:
1280		val = kcm->rx_disabled;
1281		break;
1282	default:
1283		return -ENOPROTOOPT;
1284	}
1285
1286	if (put_user(len, optlen))
1287		return -EFAULT;
1288	if (copy_to_user(optval, &val, len))
1289		return -EFAULT;
1290	return 0;
1291}
1292
1293static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)
1294{
1295	struct kcm_sock *tkcm;
1296	struct list_head *head;
1297	int index = 0;
1298
1299	/* For SOCK_SEQPACKET sock type, datagram_poll checks the sk_state, so
1300	 * we set sk_state, otherwise epoll_wait always returns right away with
1301	 * EPOLLHUP
1302	 */
1303	kcm->sk.sk_state = TCP_ESTABLISHED;
1304
1305	/* Add to mux's kcm sockets list */
1306	kcm->mux = mux;
1307	spin_lock_bh(&mux->lock);
1308
1309	head = &mux->kcm_socks;
1310	list_for_each_entry(tkcm, &mux->kcm_socks, kcm_sock_list) {
1311		if (tkcm->index != index)
1312			break;
1313		head = &tkcm->kcm_sock_list;
1314		index++;
1315	}
1316
1317	list_add(&kcm->kcm_sock_list, head);
1318	kcm->index = index;
1319
1320	mux->kcm_socks_cnt++;
1321	spin_unlock_bh(&mux->lock);
1322
1323	INIT_WORK(&kcm->tx_work, kcm_tx_work);
1324
1325	spin_lock_bh(&mux->rx_lock);
1326	kcm_rcv_ready(kcm);
1327	spin_unlock_bh(&mux->rx_lock);
1328}
1329
1330static int kcm_attach(struct socket *sock, struct socket *csock,
1331		      struct bpf_prog *prog)
1332{
1333	struct kcm_sock *kcm = kcm_sk(sock->sk);
1334	struct kcm_mux *mux = kcm->mux;
1335	struct sock *csk;
1336	struct kcm_psock *psock = NULL, *tpsock;
1337	struct list_head *head;
1338	int index = 0;
1339	static const struct strp_callbacks cb = {
1340		.rcv_msg = kcm_rcv_strparser,
1341		.parse_msg = kcm_parse_func_strparser,
1342		.read_sock_done = kcm_read_sock_done,
1343	};
1344	int err = 0;
1345
1346	csk = csock->sk;
1347	if (!csk)
1348		return -EINVAL;
1349
1350	lock_sock(csk);
1351
1352	/* Only allow TCP sockets to be attached for now */
1353	if ((csk->sk_family != AF_INET && csk->sk_family != AF_INET6) ||
1354	    csk->sk_protocol != IPPROTO_TCP) {
1355		err = -EOPNOTSUPP;
1356		goto out;
1357	}
1358
1359	/* Don't allow listeners or closed sockets */
1360	if (csk->sk_state == TCP_LISTEN || csk->sk_state == TCP_CLOSE) {
1361		err = -EOPNOTSUPP;
1362		goto out;
1363	}
1364
1365	psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL);
1366	if (!psock) {
1367		err = -ENOMEM;
1368		goto out;
1369	}
1370
1371	psock->mux = mux;
1372	psock->sk = csk;
1373	psock->bpf_prog = prog;
1374
 
 
 
 
 
 
1375	write_lock_bh(&csk->sk_callback_lock);
1376
1377	/* Check if sk_user_data is already by KCM or someone else.
1378	 * Must be done under lock to prevent race conditions.
1379	 */
1380	if (csk->sk_user_data) {
1381		write_unlock_bh(&csk->sk_callback_lock);
 
 
1382		kmem_cache_free(kcm_psockp, psock);
1383		err = -EALREADY;
1384		goto out;
1385	}
1386
1387	err = strp_init(&psock->strp, csk, &cb);
1388	if (err) {
1389		write_unlock_bh(&csk->sk_callback_lock);
1390		kmem_cache_free(kcm_psockp, psock);
1391		goto out;
1392	}
1393
1394	psock->save_data_ready = csk->sk_data_ready;
1395	psock->save_write_space = csk->sk_write_space;
1396	psock->save_state_change = csk->sk_state_change;
1397	csk->sk_user_data = psock;
1398	csk->sk_data_ready = psock_data_ready;
1399	csk->sk_write_space = psock_write_space;
1400	csk->sk_state_change = psock_state_change;
1401
1402	write_unlock_bh(&csk->sk_callback_lock);
1403
1404	sock_hold(csk);
1405
1406	/* Finished initialization, now add the psock to the MUX. */
1407	spin_lock_bh(&mux->lock);
1408	head = &mux->psocks;
1409	list_for_each_entry(tpsock, &mux->psocks, psock_list) {
1410		if (tpsock->index != index)
1411			break;
1412		head = &tpsock->psock_list;
1413		index++;
1414	}
1415
1416	list_add(&psock->psock_list, head);
1417	psock->index = index;
1418
1419	KCM_STATS_INCR(mux->stats.psock_attach);
1420	mux->psocks_cnt++;
1421	psock_now_avail(psock);
1422	spin_unlock_bh(&mux->lock);
1423
1424	/* Schedule RX work in case there are already bytes queued */
1425	strp_check_rcv(&psock->strp);
1426
1427out:
1428	release_sock(csk);
1429
1430	return err;
1431}
1432
1433static int kcm_attach_ioctl(struct socket *sock, struct kcm_attach *info)
1434{
1435	struct socket *csock;
1436	struct bpf_prog *prog;
1437	int err;
1438
1439	csock = sockfd_lookup(info->fd, &err);
1440	if (!csock)
1441		return -ENOENT;
1442
1443	prog = bpf_prog_get_type(info->bpf_fd, BPF_PROG_TYPE_SOCKET_FILTER);
1444	if (IS_ERR(prog)) {
1445		err = PTR_ERR(prog);
1446		goto out;
1447	}
1448
1449	err = kcm_attach(sock, csock, prog);
1450	if (err) {
1451		bpf_prog_put(prog);
1452		goto out;
1453	}
1454
1455	/* Keep reference on file also */
1456
1457	return 0;
1458out:
1459	sockfd_put(csock);
1460	return err;
1461}
1462
1463static void kcm_unattach(struct kcm_psock *psock)
1464{
1465	struct sock *csk = psock->sk;
1466	struct kcm_mux *mux = psock->mux;
1467
1468	lock_sock(csk);
1469
1470	/* Stop getting callbacks from TCP socket. After this there should
1471	 * be no way to reserve a kcm for this psock.
1472	 */
1473	write_lock_bh(&csk->sk_callback_lock);
1474	csk->sk_user_data = NULL;
1475	csk->sk_data_ready = psock->save_data_ready;
1476	csk->sk_write_space = psock->save_write_space;
1477	csk->sk_state_change = psock->save_state_change;
1478	strp_stop(&psock->strp);
1479
1480	if (WARN_ON(psock->rx_kcm)) {
1481		write_unlock_bh(&csk->sk_callback_lock);
1482		release_sock(csk);
1483		return;
1484	}
1485
1486	spin_lock_bh(&mux->rx_lock);
1487
1488	/* Stop receiver activities. After this point psock should not be
1489	 * able to get onto ready list either through callbacks or work.
1490	 */
1491	if (psock->ready_rx_msg) {
1492		list_del(&psock->psock_ready_list);
1493		kfree_skb(psock->ready_rx_msg);
1494		psock->ready_rx_msg = NULL;
1495		KCM_STATS_INCR(mux->stats.rx_ready_drops);
1496	}
1497
1498	spin_unlock_bh(&mux->rx_lock);
1499
1500	write_unlock_bh(&csk->sk_callback_lock);
1501
1502	/* Call strp_done without sock lock */
1503	release_sock(csk);
1504	strp_done(&psock->strp);
1505	lock_sock(csk);
1506
1507	bpf_prog_put(psock->bpf_prog);
1508
1509	spin_lock_bh(&mux->lock);
1510
1511	aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats);
1512	save_strp_stats(&psock->strp, &mux->aggregate_strp_stats);
1513
1514	KCM_STATS_INCR(mux->stats.psock_unattach);
1515
1516	if (psock->tx_kcm) {
1517		/* psock was reserved.  Just mark it finished and we will clean
1518		 * up in the kcm paths, we need kcm lock which can not be
1519		 * acquired here.
1520		 */
1521		KCM_STATS_INCR(mux->stats.psock_unattach_rsvd);
1522		spin_unlock_bh(&mux->lock);
1523
1524		/* We are unattaching a socket that is reserved. Abort the
1525		 * socket since we may be out of sync in sending on it. We need
1526		 * to do this without the mux lock.
1527		 */
1528		kcm_abort_tx_psock(psock, EPIPE, false);
1529
1530		spin_lock_bh(&mux->lock);
1531		if (!psock->tx_kcm) {
1532			/* psock now unreserved in window mux was unlocked */
1533			goto no_reserved;
1534		}
1535		psock->done = 1;
1536
1537		/* Commit done before queuing work to process it */
1538		smp_mb();
1539
1540		/* Queue tx work to make sure psock->done is handled */
1541		queue_work(kcm_wq, &psock->tx_kcm->tx_work);
1542		spin_unlock_bh(&mux->lock);
1543	} else {
1544no_reserved:
1545		if (!psock->tx_stopped)
1546			list_del(&psock->psock_avail_list);
1547		list_del(&psock->psock_list);
1548		mux->psocks_cnt--;
1549		spin_unlock_bh(&mux->lock);
1550
1551		sock_put(csk);
1552		fput(csk->sk_socket->file);
1553		kmem_cache_free(kcm_psockp, psock);
1554	}
1555
1556	release_sock(csk);
1557}
1558
1559static int kcm_unattach_ioctl(struct socket *sock, struct kcm_unattach *info)
1560{
1561	struct kcm_sock *kcm = kcm_sk(sock->sk);
1562	struct kcm_mux *mux = kcm->mux;
1563	struct kcm_psock *psock;
1564	struct socket *csock;
1565	struct sock *csk;
1566	int err;
1567
1568	csock = sockfd_lookup(info->fd, &err);
1569	if (!csock)
1570		return -ENOENT;
1571
1572	csk = csock->sk;
1573	if (!csk) {
1574		err = -EINVAL;
1575		goto out;
1576	}
1577
1578	err = -ENOENT;
1579
1580	spin_lock_bh(&mux->lock);
1581
1582	list_for_each_entry(psock, &mux->psocks, psock_list) {
1583		if (psock->sk != csk)
1584			continue;
1585
1586		/* Found the matching psock */
1587
1588		if (psock->unattaching || WARN_ON(psock->done)) {
1589			err = -EALREADY;
1590			break;
1591		}
1592
1593		psock->unattaching = 1;
1594
1595		spin_unlock_bh(&mux->lock);
1596
1597		/* Lower socket lock should already be held */
1598		kcm_unattach(psock);
1599
1600		err = 0;
1601		goto out;
1602	}
1603
1604	spin_unlock_bh(&mux->lock);
1605
1606out:
1607	sockfd_put(csock);
1608	return err;
1609}
1610
1611static struct proto kcm_proto = {
1612	.name	= "KCM",
1613	.owner	= THIS_MODULE,
1614	.obj_size = sizeof(struct kcm_sock),
1615};
1616
1617/* Clone a kcm socket. */
1618static struct file *kcm_clone(struct socket *osock)
1619{
1620	struct socket *newsock;
1621	struct sock *newsk;
1622
1623	newsock = sock_alloc();
1624	if (!newsock)
1625		return ERR_PTR(-ENFILE);
1626
1627	newsock->type = osock->type;
1628	newsock->ops = osock->ops;
1629
1630	__module_get(newsock->ops->owner);
1631
1632	newsk = sk_alloc(sock_net(osock->sk), PF_KCM, GFP_KERNEL,
1633			 &kcm_proto, false);
1634	if (!newsk) {
1635		sock_release(newsock);
1636		return ERR_PTR(-ENOMEM);
1637	}
1638	sock_init_data(newsock, newsk);
1639	init_kcm_sock(kcm_sk(newsk), kcm_sk(osock->sk)->mux);
1640
1641	return sock_alloc_file(newsock, 0, osock->sk->sk_prot_creator->name);
1642}
1643
1644static int kcm_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
1645{
1646	int err;
1647
1648	switch (cmd) {
1649	case SIOCKCMATTACH: {
1650		struct kcm_attach info;
1651
1652		if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
1653			return -EFAULT;
1654
1655		err = kcm_attach_ioctl(sock, &info);
1656
1657		break;
1658	}
1659	case SIOCKCMUNATTACH: {
1660		struct kcm_unattach info;
1661
1662		if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
1663			return -EFAULT;
1664
1665		err = kcm_unattach_ioctl(sock, &info);
1666
1667		break;
1668	}
1669	case SIOCKCMCLONE: {
1670		struct kcm_clone info;
1671		struct file *file;
1672
1673		info.fd = get_unused_fd_flags(0);
1674		if (unlikely(info.fd < 0))
1675			return info.fd;
1676
1677		file = kcm_clone(sock);
1678		if (IS_ERR(file)) {
1679			put_unused_fd(info.fd);
1680			return PTR_ERR(file);
1681		}
1682		if (copy_to_user((void __user *)arg, &info,
1683				 sizeof(info))) {
1684			put_unused_fd(info.fd);
1685			fput(file);
1686			return -EFAULT;
1687		}
1688		fd_install(info.fd, file);
1689		err = 0;
1690		break;
1691	}
1692	default:
1693		err = -ENOIOCTLCMD;
1694		break;
1695	}
1696
1697	return err;
1698}
1699
1700static void free_mux(struct rcu_head *rcu)
1701{
1702	struct kcm_mux *mux = container_of(rcu,
1703	    struct kcm_mux, rcu);
1704
1705	kmem_cache_free(kcm_muxp, mux);
1706}
1707
1708static void release_mux(struct kcm_mux *mux)
1709{
1710	struct kcm_net *knet = mux->knet;
1711	struct kcm_psock *psock, *tmp_psock;
1712
1713	/* Release psocks */
1714	list_for_each_entry_safe(psock, tmp_psock,
1715				 &mux->psocks, psock_list) {
1716		if (!WARN_ON(psock->unattaching))
1717			kcm_unattach(psock);
1718	}
1719
1720	if (WARN_ON(mux->psocks_cnt))
1721		return;
1722
1723	__skb_queue_purge(&mux->rx_hold_queue);
1724
1725	mutex_lock(&knet->mutex);
1726	aggregate_mux_stats(&mux->stats, &knet->aggregate_mux_stats);
1727	aggregate_psock_stats(&mux->aggregate_psock_stats,
1728			      &knet->aggregate_psock_stats);
1729	aggregate_strp_stats(&mux->aggregate_strp_stats,
1730			     &knet->aggregate_strp_stats);
1731	list_del_rcu(&mux->kcm_mux_list);
1732	knet->count--;
1733	mutex_unlock(&knet->mutex);
1734
1735	call_rcu(&mux->rcu, free_mux);
1736}
1737
1738static void kcm_done(struct kcm_sock *kcm)
1739{
1740	struct kcm_mux *mux = kcm->mux;
1741	struct sock *sk = &kcm->sk;
1742	int socks_cnt;
1743
1744	spin_lock_bh(&mux->rx_lock);
1745	if (kcm->rx_psock) {
1746		/* Cleanup in unreserve_rx_kcm */
1747		WARN_ON(kcm->done);
1748		kcm->rx_disabled = 1;
1749		kcm->done = 1;
1750		spin_unlock_bh(&mux->rx_lock);
1751		return;
1752	}
1753
1754	if (kcm->rx_wait) {
1755		list_del(&kcm->wait_rx_list);
1756		/* paired with lockless reads in kcm_rfree() */
1757		WRITE_ONCE(kcm->rx_wait, false);
1758	}
1759	/* Move any pending receive messages to other kcm sockets */
1760	requeue_rx_msgs(mux, &sk->sk_receive_queue);
1761
1762	spin_unlock_bh(&mux->rx_lock);
1763
1764	if (WARN_ON(sk_rmem_alloc_get(sk)))
1765		return;
1766
1767	/* Detach from MUX */
1768	spin_lock_bh(&mux->lock);
1769
1770	list_del(&kcm->kcm_sock_list);
1771	mux->kcm_socks_cnt--;
1772	socks_cnt = mux->kcm_socks_cnt;
1773
1774	spin_unlock_bh(&mux->lock);
1775
1776	if (!socks_cnt) {
1777		/* We are done with the mux now. */
1778		release_mux(mux);
1779	}
1780
1781	WARN_ON(kcm->rx_wait);
1782
1783	sock_put(&kcm->sk);
1784}
1785
1786/* Called by kcm_release to close a KCM socket.
1787 * If this is the last KCM socket on the MUX, destroy the MUX.
1788 */
1789static int kcm_release(struct socket *sock)
1790{
1791	struct sock *sk = sock->sk;
1792	struct kcm_sock *kcm;
1793	struct kcm_mux *mux;
1794	struct kcm_psock *psock;
1795
1796	if (!sk)
1797		return 0;
1798
1799	kcm = kcm_sk(sk);
1800	mux = kcm->mux;
1801
1802	lock_sock(sk);
1803	sock_orphan(sk);
1804	kfree_skb(kcm->seq_skb);
1805
 
1806	/* Purge queue under lock to avoid race condition with tx_work trying
1807	 * to act when queue is nonempty. If tx_work runs after this point
1808	 * it will just return.
1809	 */
1810	__skb_queue_purge(&sk->sk_write_queue);
1811
1812	/* Set tx_stopped. This is checked when psock is bound to a kcm and we
1813	 * get a writespace callback. This prevents further work being queued
1814	 * from the callback (unbinding the psock occurs after canceling work.
1815	 */
1816	kcm->tx_stopped = 1;
1817
1818	release_sock(sk);
1819
1820	spin_lock_bh(&mux->lock);
1821	if (kcm->tx_wait) {
1822		/* Take of tx_wait list, after this point there should be no way
1823		 * that a psock will be assigned to this kcm.
1824		 */
1825		list_del(&kcm->wait_psock_list);
1826		kcm->tx_wait = false;
1827	}
1828	spin_unlock_bh(&mux->lock);
1829
1830	/* Cancel work. After this point there should be no outside references
1831	 * to the kcm socket.
1832	 */
1833	cancel_work_sync(&kcm->tx_work);
1834
1835	lock_sock(sk);
1836	psock = kcm->tx_psock;
1837	if (psock) {
1838		/* A psock was reserved, so we need to kill it since it
1839		 * may already have some bytes queued from a message. We
1840		 * need to do this after removing kcm from tx_wait list.
1841		 */
1842		kcm_abort_tx_psock(psock, EPIPE, false);
1843		unreserve_psock(kcm);
1844	}
1845	release_sock(sk);
1846
1847	WARN_ON(kcm->tx_wait);
1848	WARN_ON(kcm->tx_psock);
1849
1850	sock->sk = NULL;
1851
1852	kcm_done(kcm);
1853
1854	return 0;
1855}
1856
1857static const struct proto_ops kcm_dgram_ops = {
1858	.family =	PF_KCM,
1859	.owner =	THIS_MODULE,
1860	.release =	kcm_release,
1861	.bind =		sock_no_bind,
1862	.connect =	sock_no_connect,
1863	.socketpair =	sock_no_socketpair,
1864	.accept =	sock_no_accept,
1865	.getname =	sock_no_getname,
1866	.poll =		datagram_poll,
1867	.ioctl =	kcm_ioctl,
1868	.listen =	sock_no_listen,
1869	.shutdown =	sock_no_shutdown,
1870	.setsockopt =	kcm_setsockopt,
1871	.getsockopt =	kcm_getsockopt,
1872	.sendmsg =	kcm_sendmsg,
1873	.recvmsg =	kcm_recvmsg,
1874	.mmap =		sock_no_mmap,
1875	.sendpage =	kcm_sendpage,
1876};
1877
1878static const struct proto_ops kcm_seqpacket_ops = {
1879	.family =	PF_KCM,
1880	.owner =	THIS_MODULE,
1881	.release =	kcm_release,
1882	.bind =		sock_no_bind,
1883	.connect =	sock_no_connect,
1884	.socketpair =	sock_no_socketpair,
1885	.accept =	sock_no_accept,
1886	.getname =	sock_no_getname,
1887	.poll =		datagram_poll,
1888	.ioctl =	kcm_ioctl,
1889	.listen =	sock_no_listen,
1890	.shutdown =	sock_no_shutdown,
1891	.setsockopt =	kcm_setsockopt,
1892	.getsockopt =	kcm_getsockopt,
1893	.sendmsg =	kcm_sendmsg,
1894	.recvmsg =	kcm_recvmsg,
1895	.mmap =		sock_no_mmap,
1896	.sendpage =	kcm_sendpage,
1897	.splice_read =	kcm_splice_read,
1898};
1899
1900/* Create proto operation for kcm sockets */
1901static int kcm_create(struct net *net, struct socket *sock,
1902		      int protocol, int kern)
1903{
1904	struct kcm_net *knet = net_generic(net, kcm_net_id);
1905	struct sock *sk;
1906	struct kcm_mux *mux;
1907
1908	switch (sock->type) {
1909	case SOCK_DGRAM:
1910		sock->ops = &kcm_dgram_ops;
1911		break;
1912	case SOCK_SEQPACKET:
1913		sock->ops = &kcm_seqpacket_ops;
1914		break;
1915	default:
1916		return -ESOCKTNOSUPPORT;
1917	}
1918
1919	if (protocol != KCMPROTO_CONNECTED)
1920		return -EPROTONOSUPPORT;
1921
1922	sk = sk_alloc(net, PF_KCM, GFP_KERNEL, &kcm_proto, kern);
1923	if (!sk)
1924		return -ENOMEM;
1925
1926	/* Allocate a kcm mux, shared between KCM sockets */
1927	mux = kmem_cache_zalloc(kcm_muxp, GFP_KERNEL);
1928	if (!mux) {
1929		sk_free(sk);
1930		return -ENOMEM;
1931	}
1932
1933	spin_lock_init(&mux->lock);
1934	spin_lock_init(&mux->rx_lock);
1935	INIT_LIST_HEAD(&mux->kcm_socks);
1936	INIT_LIST_HEAD(&mux->kcm_rx_waiters);
1937	INIT_LIST_HEAD(&mux->kcm_tx_waiters);
1938
1939	INIT_LIST_HEAD(&mux->psocks);
1940	INIT_LIST_HEAD(&mux->psocks_ready);
1941	INIT_LIST_HEAD(&mux->psocks_avail);
1942
1943	mux->knet = knet;
1944
1945	/* Add new MUX to list */
1946	mutex_lock(&knet->mutex);
1947	list_add_rcu(&mux->kcm_mux_list, &knet->mux_list);
1948	knet->count++;
1949	mutex_unlock(&knet->mutex);
1950
1951	skb_queue_head_init(&mux->rx_hold_queue);
1952
1953	/* Init KCM socket */
1954	sock_init_data(sock, sk);
1955	init_kcm_sock(kcm_sk(sk), mux);
1956
1957	return 0;
1958}
1959
1960static const struct net_proto_family kcm_family_ops = {
1961	.family = PF_KCM,
1962	.create = kcm_create,
1963	.owner  = THIS_MODULE,
1964};
1965
1966static __net_init int kcm_init_net(struct net *net)
1967{
1968	struct kcm_net *knet = net_generic(net, kcm_net_id);
1969
1970	INIT_LIST_HEAD_RCU(&knet->mux_list);
1971	mutex_init(&knet->mutex);
1972
1973	return 0;
1974}
1975
1976static __net_exit void kcm_exit_net(struct net *net)
1977{
1978	struct kcm_net *knet = net_generic(net, kcm_net_id);
1979
1980	/* All KCM sockets should be closed at this point, which should mean
1981	 * that all multiplexors and psocks have been destroyed.
1982	 */
1983	WARN_ON(!list_empty(&knet->mux_list));
1984}
1985
1986static struct pernet_operations kcm_net_ops = {
1987	.init = kcm_init_net,
1988	.exit = kcm_exit_net,
1989	.id   = &kcm_net_id,
1990	.size = sizeof(struct kcm_net),
1991};
1992
1993static int __init kcm_init(void)
1994{
1995	int err = -ENOMEM;
1996
1997	kcm_muxp = kmem_cache_create("kcm_mux_cache",
1998				     sizeof(struct kcm_mux), 0,
1999				     SLAB_HWCACHE_ALIGN, NULL);
2000	if (!kcm_muxp)
2001		goto fail;
2002
2003	kcm_psockp = kmem_cache_create("kcm_psock_cache",
2004				       sizeof(struct kcm_psock), 0,
2005					SLAB_HWCACHE_ALIGN, NULL);
2006	if (!kcm_psockp)
2007		goto fail;
2008
2009	kcm_wq = create_singlethread_workqueue("kkcmd");
2010	if (!kcm_wq)
2011		goto fail;
2012
2013	err = proto_register(&kcm_proto, 1);
2014	if (err)
2015		goto fail;
2016
2017	err = register_pernet_device(&kcm_net_ops);
2018	if (err)
2019		goto net_ops_fail;
2020
2021	err = sock_register(&kcm_family_ops);
2022	if (err)
2023		goto sock_register_fail;
2024
2025	err = kcm_proc_init();
2026	if (err)
2027		goto proc_init_fail;
2028
2029	return 0;
2030
2031proc_init_fail:
2032	sock_unregister(PF_KCM);
2033
2034sock_register_fail:
2035	unregister_pernet_device(&kcm_net_ops);
2036
2037net_ops_fail:
2038	proto_unregister(&kcm_proto);
2039
2040fail:
2041	kmem_cache_destroy(kcm_muxp);
2042	kmem_cache_destroy(kcm_psockp);
2043
2044	if (kcm_wq)
2045		destroy_workqueue(kcm_wq);
2046
2047	return err;
2048}
2049
2050static void __exit kcm_exit(void)
2051{
2052	kcm_proc_exit();
2053	sock_unregister(PF_KCM);
2054	unregister_pernet_device(&kcm_net_ops);
2055	proto_unregister(&kcm_proto);
2056	destroy_workqueue(kcm_wq);
2057
2058	kmem_cache_destroy(kcm_muxp);
2059	kmem_cache_destroy(kcm_psockp);
2060}
2061
2062module_init(kcm_init);
2063module_exit(kcm_exit);
2064
2065MODULE_LICENSE("GPL");
2066MODULE_ALIAS_NETPROTO(PF_KCM);