1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * rio_cm - RapidIO Channelized Messaging Driver
   4 *
   5 * Copyright 2013-2016 Integrated Device Technology, Inc.
   6 * Copyright (c) 2015, Prodrive Technologies
   7 * Copyright (c) 2015, RapidIO Trade Association
   8 */
   9
  10#include <linux/module.h>
  11#include <linux/kernel.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/delay.h>
  14#include <linux/sched.h>
  15#include <linux/rio.h>
  16#include <linux/rio_drv.h>
  17#include <linux/slab.h>
  18#include <linux/idr.h>
  19#include <linux/interrupt.h>
  20#include <linux/cdev.h>
  21#include <linux/fs.h>
  22#include <linux/poll.h>
  23#include <linux/reboot.h>
  24#include <linux/bitops.h>
  25#include <linux/printk.h>
  26#include <linux/rio_cm_cdev.h>
  27
  28#define DRV_NAME        "rio_cm"
  29#define DRV_VERSION     "1.0.0"
  30#define DRV_AUTHOR      "Alexandre Bounine <alexandre.bounine@idt.com>"
  31#define DRV_DESC        "RapidIO Channelized Messaging Driver"
  32#define DEV_NAME	"rio_cm"
  33
  34/* Debug output filtering masks */
  35enum {
  36	DBG_NONE	= 0,
  37	DBG_INIT	= BIT(0), /* driver init */
  38	DBG_EXIT	= BIT(1), /* driver exit */
  39	DBG_MPORT	= BIT(2), /* mport add/remove */
  40	DBG_RDEV	= BIT(3), /* RapidIO device add/remove */
  41	DBG_CHOP	= BIT(4), /* channel operations */
  42	DBG_WAIT	= BIT(5), /* waiting for events */
  43	DBG_TX		= BIT(6), /* message TX */
  44	DBG_TX_EVENT	= BIT(7), /* message TX event */
  45	DBG_RX_DATA	= BIT(8), /* inbound data messages */
  46	DBG_RX_CMD	= BIT(9), /* inbound REQ/ACK/NACK messages */
  47	DBG_ALL		= ~0,
  48};
  49
  50#ifdef DEBUG
  51#define riocm_debug(level, fmt, arg...) \
  52	do { \
  53		if (DBG_##level & dbg_level) \
  54			pr_debug(DRV_NAME ": %s " fmt "\n", \
  55				__func__, ##arg); \
  56	} while (0)
  57#else
  58#define riocm_debug(level, fmt, arg...) \
  59		no_printk(KERN_DEBUG pr_fmt(DRV_NAME fmt "\n"), ##arg)
  60#endif
  61
  62#define riocm_warn(fmt, arg...) \
  63	pr_warn(DRV_NAME ": %s WARNING " fmt "\n", __func__, ##arg)
  64
  65#define riocm_error(fmt, arg...) \
  66	pr_err(DRV_NAME ": %s ERROR " fmt "\n", __func__, ##arg)
  67
  68
  69static int cmbox = 1;
  70module_param(cmbox, int, S_IRUGO);
  71MODULE_PARM_DESC(cmbox, "RapidIO Mailbox number (default 1)");
  72
  73static int chstart = 256;
  74module_param(chstart, int, S_IRUGO);
  75MODULE_PARM_DESC(chstart,
  76		 "Start channel number for dynamic allocation (default 256)");
  77
  78#ifdef DEBUG
  79static u32 dbg_level = DBG_NONE;
  80module_param(dbg_level, uint, S_IWUSR | S_IRUGO);
  81MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
  82#endif
  83
  84MODULE_AUTHOR(DRV_AUTHOR);
  85MODULE_DESCRIPTION(DRV_DESC);
  86MODULE_LICENSE("GPL");
  87MODULE_VERSION(DRV_VERSION);
  88
  89#define RIOCM_TX_RING_SIZE	128
  90#define RIOCM_RX_RING_SIZE	128
  91#define RIOCM_CONNECT_TO	3 /* connect response TO (in sec) */
  92
  93#define RIOCM_MAX_CHNUM		0xffff /* Use full range of u16 field */
  94#define RIOCM_CHNUM_AUTO	0
  95#define RIOCM_MAX_EP_COUNT	0x10000 /* Max number of endpoints */
  96
  97enum rio_cm_state {
  98	RIO_CM_IDLE,
  99	RIO_CM_CONNECT,
 100	RIO_CM_CONNECTED,
 101	RIO_CM_DISCONNECT,
 102	RIO_CM_CHAN_BOUND,
 103	RIO_CM_LISTEN,
 104	RIO_CM_DESTROYING,
 105};
 106
 107enum rio_cm_pkt_type {
 108	RIO_CM_SYS	= 0xaa,
 109	RIO_CM_CHAN	= 0x55,
 110};
 111
 112enum rio_cm_chop {
 113	CM_CONN_REQ,
 114	CM_CONN_ACK,
 115	CM_CONN_CLOSE,
 116	CM_DATA_MSG,
 117};
 118
 119struct rio_ch_base_bhdr {
 120	u32 src_id;
 121	u32 dst_id;
 122#define RIO_HDR_LETTER_MASK 0xffff0000
 123#define RIO_HDR_MBOX_MASK   0x0000ffff
 124	u8  src_mbox;
 125	u8  dst_mbox;
 126	u8  type;
 127} __attribute__((__packed__));
 128
 129struct rio_ch_chan_hdr {
 130	struct rio_ch_base_bhdr bhdr;
 131	u8 ch_op;
 132	u16 dst_ch;
 133	u16 src_ch;
 134	u16 msg_len;
 135	u16 rsrvd;
 136} __attribute__((__packed__));
 137
 138struct tx_req {
 139	struct list_head node;
 140	struct rio_dev   *rdev;
 141	void		 *buffer;
 142	size_t		 len;
 143};
 144
 145struct cm_dev {
 146	struct list_head	list;
 147	struct rio_mport	*mport;
 148	void			*rx_buf[RIOCM_RX_RING_SIZE];
 149	int			rx_slots;
 150	struct mutex		rx_lock;
 151
 152	void			*tx_buf[RIOCM_TX_RING_SIZE];
 153	int			tx_slot;
 154	int			tx_cnt;
 155	int			tx_ack_slot;
 156	struct list_head	tx_reqs;
 157	spinlock_t		tx_lock;
 158
 159	struct list_head	peers;
 160	u32			npeers;
 161	struct workqueue_struct *rx_wq;
 162	struct work_struct	rx_work;
 163};
 164
 165struct chan_rx_ring {
 166	void	*buf[RIOCM_RX_RING_SIZE];
 167	int	head;
 168	int	tail;
 169	int	count;
 170
 171	/* Tracking RX buffers reported to upper level */
 172	void	*inuse[RIOCM_RX_RING_SIZE];
 173	int	inuse_cnt;
 174};
 175
 176struct rio_channel {
 177	u16			id;	/* local channel ID */
 178	struct kref		ref;	/* channel refcount */
 179	struct file		*filp;
 180	struct cm_dev		*cmdev;	/* associated CM device object */
 181	struct rio_dev		*rdev;	/* remote RapidIO device */
 182	enum rio_cm_state	state;
 183	int			error;
 184	spinlock_t		lock;
 185	void			*context;
 186	u32			loc_destid;	/* local destID */
 187	u32			rem_destid;	/* remote destID */
 188	u16			rem_channel;	/* remote channel ID */
 189	struct list_head	accept_queue;
 190	struct list_head	ch_node;
 191	struct completion	comp;
 192	struct completion	comp_close;
 193	struct chan_rx_ring	rx_ring;
 194};
 195
 196struct cm_peer {
 197	struct list_head node;
 198	struct rio_dev *rdev;
 199};
 200
 201struct rio_cm_work {
 202	struct work_struct work;
 203	struct cm_dev *cm;
 204	void *data;
 205};
 206
 207struct conn_req {
 208	struct list_head node;
 209	u32 destid;	/* requester destID */
 210	u16 chan;	/* requester channel ID */
 211	struct cm_dev *cmdev;
 212};
 213
 214/*
 215 * A channel_dev structure represents a CM_CDEV
 216 * @cdev	Character device
 217 * @dev		Associated device object
 218 */
 219struct channel_dev {
 220	struct cdev	cdev;
 221	struct device	*dev;
 222};
 223
 224static struct rio_channel *riocm_ch_alloc(u16 ch_num);
 225static void riocm_ch_free(struct kref *ref);
 226static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
 227			   void *buffer, size_t len);
 228static int riocm_ch_close(struct rio_channel *ch);
 229
 230static DEFINE_SPINLOCK(idr_lock);
 231static DEFINE_IDR(ch_idr);
 232
 233static LIST_HEAD(cm_dev_list);
 234static DECLARE_RWSEM(rdev_sem);
 235
 236static const struct class dev_class = {
 237	.name = DRV_NAME,
 238};
 239static unsigned int dev_major;
 240static unsigned int dev_minor_base;
 241static dev_t dev_number;
 242static struct channel_dev riocm_cdev;
 243
 244#define is_msg_capable(src_ops, dst_ops)			\
 245			((src_ops & RIO_SRC_OPS_DATA_MSG) &&	\
 246			 (dst_ops & RIO_DST_OPS_DATA_MSG))
 247#define dev_cm_capable(dev) \
 248	is_msg_capable(dev->src_ops, dev->dst_ops)
 249
 250static int riocm_cmp(struct rio_channel *ch, enum rio_cm_state cmp)
 251{
 252	int ret;
 253
 254	spin_lock_bh(&ch->lock);
 255	ret = (ch->state == cmp);
 256	spin_unlock_bh(&ch->lock);
 257	return ret;
 258}
 259
 260static int riocm_cmp_exch(struct rio_channel *ch,
 261			   enum rio_cm_state cmp, enum rio_cm_state exch)
 262{
 263	int ret;
 264
 265	spin_lock_bh(&ch->lock);
 266	ret = (ch->state == cmp);
 267	if (ret)
 268		ch->state = exch;
 269	spin_unlock_bh(&ch->lock);
 270	return ret;
 271}
 272
 273static enum rio_cm_state riocm_exch(struct rio_channel *ch,
 274				    enum rio_cm_state exch)
 275{
 276	enum rio_cm_state old;
 277
 278	spin_lock_bh(&ch->lock);
 279	old = ch->state;
 280	ch->state = exch;
 281	spin_unlock_bh(&ch->lock);
 282	return old;
 283}
 284
 285static struct rio_channel *riocm_get_channel(u16 nr)
 286{
 287	struct rio_channel *ch;
 288
 289	spin_lock_bh(&idr_lock);
 290	ch = idr_find(&ch_idr, nr);
 291	if (ch)
 292		kref_get(&ch->ref);
 293	spin_unlock_bh(&idr_lock);
 294	return ch;
 295}
 296
 297static void riocm_put_channel(struct rio_channel *ch)
 298{
 299	kref_put(&ch->ref, riocm_ch_free);
 300}
 301
 302static void *riocm_rx_get_msg(struct cm_dev *cm)
 303{
 304	void *msg;
 305	int i;
 306
 307	msg = rio_get_inb_message(cm->mport, cmbox);
 308	if (msg) {
 309		for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
 310			if (cm->rx_buf[i] == msg) {
 311				cm->rx_buf[i] = NULL;
 312				cm->rx_slots++;
 313				break;
 314			}
 315		}
 316
 317		if (i == RIOCM_RX_RING_SIZE)
 318			riocm_warn("no record for buffer 0x%p", msg);
 319	}
 320
 321	return msg;
 322}
 323
 324/*
 325 * riocm_rx_fill - fills a ring of receive buffers for given cm device
 326 * @cm: cm_dev object
 327 * @nent: max number of entries to fill
 328 *
 329 * Returns: none
 330 */
 331static void riocm_rx_fill(struct cm_dev *cm, int nent)
 332{
 333	int i;
 334
 335	if (cm->rx_slots == 0)
 336		return;
 337
 338	for (i = 0; i < RIOCM_RX_RING_SIZE && cm->rx_slots && nent; i++) {
 339		if (cm->rx_buf[i] == NULL) {
 340			cm->rx_buf[i] = kmalloc(RIO_MAX_MSG_SIZE, GFP_KERNEL);
 341			if (cm->rx_buf[i] == NULL)
 342				break;
 343			rio_add_inb_buffer(cm->mport, cmbox, cm->rx_buf[i]);
 344			cm->rx_slots--;
 345			nent--;
 346		}
 347	}
 348}
 349
 350/*
 351 * riocm_rx_free - frees all receive buffers associated with given cm device
 352 * @cm: cm_dev object
 353 *
 354 * Returns: none
 355 */
 356static void riocm_rx_free(struct cm_dev *cm)
 357{
 358	int i;
 359
 360	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
 361		if (cm->rx_buf[i] != NULL) {
 362			kfree(cm->rx_buf[i]);
 363			cm->rx_buf[i] = NULL;
 364		}
 365	}
 366}
 367
 368/*
 369 * riocm_req_handler - connection request handler
 370 * @cm: cm_dev object
 371 * @req_data: pointer to the request packet
 372 *
 373 * Returns: 0 if success, or
 374 *          -EINVAL if channel is not in correct state,
 375 *          -ENODEV if cannot find a channel with specified ID,
 376 *          -ENOMEM if unable to allocate memory to store the request
 377 */
 378static int riocm_req_handler(struct cm_dev *cm, void *req_data)
 379{
 380	struct rio_channel *ch;
 381	struct conn_req *req;
 382	struct rio_ch_chan_hdr *hh = req_data;
 383	u16 chnum;
 384
 385	chnum = ntohs(hh->dst_ch);
 386
 387	ch = riocm_get_channel(chnum);
 388
 389	if (!ch)
 390		return -ENODEV;
 391
 392	if (ch->state != RIO_CM_LISTEN) {
 393		riocm_debug(RX_CMD, "channel %d is not in listen state", chnum);
 394		riocm_put_channel(ch);
 395		return -EINVAL;
 396	}
 397
 398	req = kzalloc(sizeof(*req), GFP_KERNEL);
 399	if (!req) {
 400		riocm_put_channel(ch);
 401		return -ENOMEM;
 402	}
 403
 404	req->destid = ntohl(hh->bhdr.src_id);
 405	req->chan = ntohs(hh->src_ch);
 406	req->cmdev = cm;
 407
 408	spin_lock_bh(&ch->lock);
 409	list_add_tail(&req->node, &ch->accept_queue);
 410	spin_unlock_bh(&ch->lock);
 411	complete(&ch->comp);
 412	riocm_put_channel(ch);
 413
 414	return 0;
 415}
 416
 417/*
 418 * riocm_resp_handler - response to connection request handler
 419 * @resp_data: pointer to the response packet
 420 *
 421 * Returns: 0 if success, or
 422 *          -EINVAL if channel is not in correct state,
 423 *          -ENODEV if cannot find a channel with specified ID,
 424 */
 425static int riocm_resp_handler(void *resp_data)
 426{
 427	struct rio_channel *ch;
 428	struct rio_ch_chan_hdr *hh = resp_data;
 429	u16 chnum;
 430
 431	chnum = ntohs(hh->dst_ch);
 432	ch = riocm_get_channel(chnum);
 433	if (!ch)
 434		return -ENODEV;
 435
 436	if (ch->state != RIO_CM_CONNECT) {
 437		riocm_put_channel(ch);
 438		return -EINVAL;
 439	}
 440
 441	riocm_exch(ch, RIO_CM_CONNECTED);
 442	ch->rem_channel = ntohs(hh->src_ch);
 443	complete(&ch->comp);
 444	riocm_put_channel(ch);
 445
 446	return 0;
 447}
 448
 449/*
 450 * riocm_close_handler - channel close request handler
 451 * @req_data: pointer to the request packet
 452 *
 453 * Returns: 0 if success, or
 454 *          -ENODEV if cannot find a channel with specified ID,
 455 *            + error codes returned by riocm_ch_close.
 456 */
 457static int riocm_close_handler(void *data)
 458{
 459	struct rio_channel *ch;
 460	struct rio_ch_chan_hdr *hh = data;
 461	int ret;
 462
 463	riocm_debug(RX_CMD, "for ch=%d", ntohs(hh->dst_ch));
 464
 465	spin_lock_bh(&idr_lock);
 466	ch = idr_find(&ch_idr, ntohs(hh->dst_ch));
 467	if (!ch) {
 468		spin_unlock_bh(&idr_lock);
 469		return -ENODEV;
 470	}
 471	idr_remove(&ch_idr, ch->id);
 472	spin_unlock_bh(&idr_lock);
 473
 474	riocm_exch(ch, RIO_CM_DISCONNECT);
 475
 476	ret = riocm_ch_close(ch);
 477	if (ret)
 478		riocm_debug(RX_CMD, "riocm_ch_close() returned %d", ret);
 479
 480	return 0;
 481}
 482
 483/*
 484 * rio_cm_handler - function that services request (non-data) packets
 485 * @cm: cm_dev object
 486 * @data: pointer to the packet
 487 */
 488static void rio_cm_handler(struct cm_dev *cm, void *data)
 489{
 490	struct rio_ch_chan_hdr *hdr;
 491
 492	if (!rio_mport_is_running(cm->mport))
 493		goto out;
 494
 495	hdr = data;
 496
 497	riocm_debug(RX_CMD, "OP=%x for ch=%d from %d",
 498		    hdr->ch_op, ntohs(hdr->dst_ch), ntohs(hdr->src_ch));
 499
 500	switch (hdr->ch_op) {
 501	case CM_CONN_REQ:
 502		riocm_req_handler(cm, data);
 503		break;
 504	case CM_CONN_ACK:
 505		riocm_resp_handler(data);
 506		break;
 507	case CM_CONN_CLOSE:
 508		riocm_close_handler(data);
 509		break;
 510	default:
 511		riocm_error("Invalid packet header");
 512		break;
 513	}
 514out:
 515	kfree(data);
 516}
 517
 518/*
 519 * rio_rx_data_handler - received data packet handler
 520 * @cm: cm_dev object
 521 * @buf: data packet
 522 *
 523 * Returns: 0 if success, or
 524 *          -ENODEV if cannot find a channel with specified ID,
 525 *          -EIO if channel is not in CONNECTED state,
 526 *          -ENOMEM if channel RX queue is full (packet discarded)
 527 */
 528static int rio_rx_data_handler(struct cm_dev *cm, void *buf)
 529{
 530	struct rio_ch_chan_hdr *hdr;
 531	struct rio_channel *ch;
 532
 533	hdr = buf;
 534
 535	riocm_debug(RX_DATA, "for ch=%d", ntohs(hdr->dst_ch));
 536
 537	ch = riocm_get_channel(ntohs(hdr->dst_ch));
 538	if (!ch) {
 539		/* Discard data message for non-existing channel */
 540		kfree(buf);
 541		return -ENODEV;
 542	}
 543
 544	/* Place pointer to the buffer into channel's RX queue */
 545	spin_lock(&ch->lock);
 546
 547	if (ch->state != RIO_CM_CONNECTED) {
 548		/* Channel is not ready to receive data, discard a packet */
 549		riocm_debug(RX_DATA, "ch=%d is in wrong state=%d",
 550			    ch->id, ch->state);
 551		spin_unlock(&ch->lock);
 552		kfree(buf);
 553		riocm_put_channel(ch);
 554		return -EIO;
 555	}
 556
 557	if (ch->rx_ring.count == RIOCM_RX_RING_SIZE) {
 558		/* If RX ring is full, discard a packet */
 559		riocm_debug(RX_DATA, "ch=%d is full", ch->id);
 560		spin_unlock(&ch->lock);
 561		kfree(buf);
 562		riocm_put_channel(ch);
 563		return -ENOMEM;
 564	}
 565
 566	ch->rx_ring.buf[ch->rx_ring.head] = buf;
 567	ch->rx_ring.head++;
 568	ch->rx_ring.count++;
 569	ch->rx_ring.head %= RIOCM_RX_RING_SIZE;
 570
 571	complete(&ch->comp);
 572
 573	spin_unlock(&ch->lock);
 574	riocm_put_channel(ch);
 575
 576	return 0;
 577}
 578
 579/*
 580 * rio_ibmsg_handler - inbound message packet handler
 581 */
 582static void rio_ibmsg_handler(struct work_struct *work)
 583{
 584	struct cm_dev *cm = container_of(work, struct cm_dev, rx_work);
 585	void *data;
 586	struct rio_ch_chan_hdr *hdr;
 587
 588	if (!rio_mport_is_running(cm->mport))
 589		return;
 590
 591	while (1) {
 592		mutex_lock(&cm->rx_lock);
 593		data = riocm_rx_get_msg(cm);
 594		if (data)
 595			riocm_rx_fill(cm, 1);
 596		mutex_unlock(&cm->rx_lock);
 597
 598		if (data == NULL)
 599			break;
 600
 601		hdr = data;
 602
 603		if (hdr->bhdr.type != RIO_CM_CHAN) {
 604			/* For now simply discard packets other than channel */
 605			riocm_error("Unsupported TYPE code (0x%x). Msg dropped",
 606				    hdr->bhdr.type);
 607			kfree(data);
 608			continue;
 609		}
 610
 611		/* Process a channel message */
 612		if (hdr->ch_op == CM_DATA_MSG)
 613			rio_rx_data_handler(cm, data);
 614		else
 615			rio_cm_handler(cm, data);
 616	}
 617}
 618
 619static void riocm_inb_msg_event(struct rio_mport *mport, void *dev_id,
 620				int mbox, int slot)
 621{
 622	struct cm_dev *cm = dev_id;
 623
 624	if (rio_mport_is_running(cm->mport) && !work_pending(&cm->rx_work))
 625		queue_work(cm->rx_wq, &cm->rx_work);
 626}
 627
 628/*
 629 * rio_txcq_handler - TX completion handler
 630 * @cm: cm_dev object
 631 * @slot: TX queue slot
 632 *
 633 * TX completion handler also ensures that pending request packets are placed
 634 * into transmit queue as soon as a free slot becomes available. This is done
 635 * to give higher priority to request packets during high intensity data flow.
 636 */
 637static void rio_txcq_handler(struct cm_dev *cm, int slot)
 638{
 639	int ack_slot;
 640
 641	/* ATTN: Add TX completion notification if/when direct buffer
 642	 * transfer is implemented. At this moment only correct tracking
 643	 * of tx_count is important.
 644	 */
 645	riocm_debug(TX_EVENT, "for mport_%d slot %d tx_cnt %d",
 646		    cm->mport->id, slot, cm->tx_cnt);
 647
 648	spin_lock(&cm->tx_lock);
 649	ack_slot = cm->tx_ack_slot;
 650
 651	if (ack_slot == slot)
 652		riocm_debug(TX_EVENT, "slot == ack_slot");
 653
 654	while (cm->tx_cnt && ((ack_slot != slot) ||
 655	       (cm->tx_cnt == RIOCM_TX_RING_SIZE))) {
 656
 657		cm->tx_buf[ack_slot] = NULL;
 658		++ack_slot;
 659		ack_slot &= (RIOCM_TX_RING_SIZE - 1);
 660		cm->tx_cnt--;
 661	}
 662
 663	if (cm->tx_cnt < 0 || cm->tx_cnt > RIOCM_TX_RING_SIZE)
 664		riocm_error("tx_cnt %d out of sync", cm->tx_cnt);
 665
 666	WARN_ON((cm->tx_cnt < 0) || (cm->tx_cnt > RIOCM_TX_RING_SIZE));
 667
 668	cm->tx_ack_slot = ack_slot;
 669
 670	/*
 671	 * If there are pending requests, insert them into transmit queue
 672	 */
 673	if (!list_empty(&cm->tx_reqs) && (cm->tx_cnt < RIOCM_TX_RING_SIZE)) {
 674		struct tx_req *req, *_req;
 675		int rc;
 676
 677		list_for_each_entry_safe(req, _req, &cm->tx_reqs, node) {
 678			list_del(&req->node);
 679			cm->tx_buf[cm->tx_slot] = req->buffer;
 680			rc = rio_add_outb_message(cm->mport, req->rdev, cmbox,
 681						  req->buffer, req->len);
 682			kfree(req->buffer);
 683			kfree(req);
 684
 685			++cm->tx_cnt;
 686			++cm->tx_slot;
 687			cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
 688			if (cm->tx_cnt == RIOCM_TX_RING_SIZE)
 689				break;
 690		}
 691	}
 692
 693	spin_unlock(&cm->tx_lock);
 694}
 695
 696static void riocm_outb_msg_event(struct rio_mport *mport, void *dev_id,
 697				 int mbox, int slot)
 698{
 699	struct cm_dev *cm = dev_id;
 700
 701	if (cm && rio_mport_is_running(cm->mport))
 702		rio_txcq_handler(cm, slot);
 703}
 704
 705static int riocm_queue_req(struct cm_dev *cm, struct rio_dev *rdev,
 706			   void *buffer, size_t len)
 707{
 708	unsigned long flags;
 709	struct tx_req *treq;
 710
 711	treq = kzalloc(sizeof(*treq), GFP_KERNEL);
 712	if (treq == NULL)
 713		return -ENOMEM;
 714
 715	treq->rdev = rdev;
 716	treq->buffer = buffer;
 717	treq->len = len;
 718
 719	spin_lock_irqsave(&cm->tx_lock, flags);
 720	list_add_tail(&treq->node, &cm->tx_reqs);
 721	spin_unlock_irqrestore(&cm->tx_lock, flags);
 722	return 0;
 723}
 724
 725/*
 726 * riocm_post_send - helper function that places packet into msg TX queue
 727 * @cm: cm_dev object
 728 * @rdev: target RapidIO device object (required by outbound msg interface)
 729 * @buffer: pointer to a packet buffer to send
 730 * @len: length of data to transfer
 731 * @req: request priority flag
 732 *
 733 * Returns: 0 if success, or error code otherwise.
 734 */
 735static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
 736			   void *buffer, size_t len)
 737{
 738	int rc;
 739	unsigned long flags;
 740
 741	spin_lock_irqsave(&cm->tx_lock, flags);
 742
 743	if (cm->mport == NULL) {
 744		rc = -ENODEV;
 745		goto err_out;
 746	}
 747
 748	if (cm->tx_cnt == RIOCM_TX_RING_SIZE) {
 749		riocm_debug(TX, "Tx Queue is full");
 750		rc = -EBUSY;
 751		goto err_out;
 752	}
 753
 754	cm->tx_buf[cm->tx_slot] = buffer;
 755	rc = rio_add_outb_message(cm->mport, rdev, cmbox, buffer, len);
 756
 757	riocm_debug(TX, "Add buf@%p destid=%x tx_slot=%d tx_cnt=%d",
 758		 buffer, rdev->destid, cm->tx_slot, cm->tx_cnt);
 759
 760	++cm->tx_cnt;
 761	++cm->tx_slot;
 762	cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
 763
 764err_out:
 765	spin_unlock_irqrestore(&cm->tx_lock, flags);
 766	return rc;
 767}
 768
 769/*
 770 * riocm_ch_send - sends a data packet to a remote device
 771 * @ch_id: local channel ID
 772 * @buf: pointer to a data buffer to send (including CM header)
 773 * @len: length of data to transfer (including CM header)
 774 *
 775 * ATTN: ASSUMES THAT THE HEADER SPACE IS RESERVED PART OF THE DATA PACKET
 776 *
 777 * Returns: 0 if success, or
 778 *          -EINVAL if one or more input parameters is/are not valid,
 779 *          -ENODEV if cannot find a channel with specified ID,
 780 *          -EAGAIN if a channel is not in CONNECTED state,
 781 *	    + error codes returned by HW send routine.
 782 */
 783static int riocm_ch_send(u16 ch_id, void *buf, int len)
 784{
 785	struct rio_channel *ch;
 786	struct rio_ch_chan_hdr *hdr;
 787	int ret;
 788
 789	if (buf == NULL || ch_id == 0 || len == 0 || len > RIO_MAX_MSG_SIZE)
 790		return -EINVAL;
 791
 792	ch = riocm_get_channel(ch_id);
 793	if (!ch) {
 794		riocm_error("%s(%d) ch_%d not found", current->comm,
 795			    task_pid_nr(current), ch_id);
 796		return -ENODEV;
 797	}
 798
 799	if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
 800		ret = -EAGAIN;
 801		goto err_out;
 802	}
 803
 804	/*
 805	 * Fill buffer header section with corresponding channel data
 806	 */
 807	hdr = buf;
 808
 809	hdr->bhdr.src_id = htonl(ch->loc_destid);
 810	hdr->bhdr.dst_id = htonl(ch->rem_destid);
 811	hdr->bhdr.src_mbox = cmbox;
 812	hdr->bhdr.dst_mbox = cmbox;
 813	hdr->bhdr.type = RIO_CM_CHAN;
 814	hdr->ch_op = CM_DATA_MSG;
 815	hdr->dst_ch = htons(ch->rem_channel);
 816	hdr->src_ch = htons(ch->id);
 817	hdr->msg_len = htons((u16)len);
 818
 819	/* ATTN: the function call below relies on the fact that underlying
 820	 * HW-specific add_outb_message() routine copies TX data into its own
 821	 * internal transfer buffer (true for all RIONET compatible mport
 822	 * drivers). Must be reviewed if mport driver uses the buffer directly.
 823	 */
 824
 825	ret = riocm_post_send(ch->cmdev, ch->rdev, buf, len);
 826	if (ret)
 827		riocm_debug(TX, "ch %d send_err=%d", ch->id, ret);
 828err_out:
 829	riocm_put_channel(ch);
 830	return ret;
 831}
 832
 833static int riocm_ch_free_rxbuf(struct rio_channel *ch, void *buf)
 834{
 835	int i, ret = -EINVAL;
 836
 837	spin_lock_bh(&ch->lock);
 838
 839	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
 840		if (ch->rx_ring.inuse[i] == buf) {
 841			ch->rx_ring.inuse[i] = NULL;
 842			ch->rx_ring.inuse_cnt--;
 843			ret = 0;
 844			break;
 845		}
 846	}
 847
 848	spin_unlock_bh(&ch->lock);
 849
 850	if (!ret)
 851		kfree(buf);
 852
 853	return ret;
 854}
 855
 856/*
 857 * riocm_ch_receive - fetch a data packet received for the specified channel
 858 * @ch: local channel ID
 859 * @buf: pointer to a packet buffer
 860 * @timeout: timeout to wait for incoming packet (in jiffies)
 861 *
 862 * Returns: 0 and valid buffer pointer if success, or NULL pointer and one of:
 863 *          -EAGAIN if a channel is not in CONNECTED state,
 864 *          -ENOMEM if in-use tracking queue is full,
 865 *          -ETIME if wait timeout expired,
 866 *	    -EINTR if wait was interrupted.
 867 */
 868static int riocm_ch_receive(struct rio_channel *ch, void **buf, long timeout)
 869{
 870	void *rxmsg = NULL;
 871	int i, ret = 0;
 872	long wret;
 873
 874	if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
 875		ret = -EAGAIN;
 876		goto out;
 877	}
 878
 879	if (ch->rx_ring.inuse_cnt == RIOCM_RX_RING_SIZE) {
 880		/* If we do not have entries to track buffers given to upper
 881		 * layer, reject request.
 882		 */
 883		ret = -ENOMEM;
 884		goto out;
 885	}
 886
 887	wret = wait_for_completion_interruptible_timeout(&ch->comp, timeout);
 888
 889	riocm_debug(WAIT, "wait on %d returned %ld", ch->id, wret);
 890
 891	if (!wret)
 892		ret = -ETIME;
 893	else if (wret == -ERESTARTSYS)
 894		ret = -EINTR;
 895	else
 896		ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -ECONNRESET;
 897
 898	if (ret)
 899		goto out;
 900
 901	spin_lock_bh(&ch->lock);
 902
 903	rxmsg = ch->rx_ring.buf[ch->rx_ring.tail];
 904	ch->rx_ring.buf[ch->rx_ring.tail] = NULL;
 905	ch->rx_ring.count--;
 906	ch->rx_ring.tail++;
 907	ch->rx_ring.tail %= RIOCM_RX_RING_SIZE;
 908	ret = -ENOMEM;
 909
 910	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
 911		if (ch->rx_ring.inuse[i] == NULL) {
 912			ch->rx_ring.inuse[i] = rxmsg;
 913			ch->rx_ring.inuse_cnt++;
 914			ret = 0;
 915			break;
 916		}
 917	}
 918
 919	if (ret) {
 920		/* We have no entry to store pending message: drop it */
 921		kfree(rxmsg);
 922		rxmsg = NULL;
 923	}
 924
 925	spin_unlock_bh(&ch->lock);
 926out:
 927	*buf = rxmsg;
 928	return ret;
 929}
 930
 931/*
 932 * riocm_ch_connect - sends a connect request to a remote device
 933 * @loc_ch: local channel ID
 934 * @cm: CM device to send connect request
 935 * @peer: target RapidIO device
 936 * @rem_ch: remote channel ID
 937 *
 938 * Returns: 0 if success, or
 939 *          -EINVAL if the channel is not in IDLE state,
 940 *          -EAGAIN if no connection request available immediately,
 941 *          -ETIME if ACK response timeout expired,
 942 *          -EINTR if wait for response was interrupted.
 943 */
 944static int riocm_ch_connect(u16 loc_ch, struct cm_dev *cm,
 945			    struct cm_peer *peer, u16 rem_ch)
 946{
 947	struct rio_channel *ch = NULL;
 948	struct rio_ch_chan_hdr *hdr;
 949	int ret;
 950	long wret;
 951
 952	ch = riocm_get_channel(loc_ch);
 953	if (!ch)
 954		return -ENODEV;
 955
 956	if (!riocm_cmp_exch(ch, RIO_CM_IDLE, RIO_CM_CONNECT)) {
 957		ret = -EINVAL;
 958		goto conn_done;
 959	}
 960
 961	ch->cmdev = cm;
 962	ch->rdev = peer->rdev;
 963	ch->context = NULL;
 964	ch->loc_destid = cm->mport->host_deviceid;
 965	ch->rem_channel = rem_ch;
 966
 967	/*
 968	 * Send connect request to the remote RapidIO device
 969	 */
 970
 971	hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
 972	if (hdr == NULL) {
 973		ret = -ENOMEM;
 974		goto conn_done;
 975	}
 976
 977	hdr->bhdr.src_id = htonl(ch->loc_destid);
 978	hdr->bhdr.dst_id = htonl(peer->rdev->destid);
 979	hdr->bhdr.src_mbox = cmbox;
 980	hdr->bhdr.dst_mbox = cmbox;
 981	hdr->bhdr.type = RIO_CM_CHAN;
 982	hdr->ch_op = CM_CONN_REQ;
 983	hdr->dst_ch = htons(rem_ch);
 984	hdr->src_ch = htons(loc_ch);
 985
 986	/* ATTN: the function call below relies on the fact that underlying
 987	 * HW-specific add_outb_message() routine copies TX data into its
 988	 * internal transfer buffer. Must be reviewed if mport driver uses
 989	 * this buffer directly.
 990	 */
 991	ret = riocm_post_send(cm, peer->rdev, hdr, sizeof(*hdr));
 992
 993	if (ret != -EBUSY) {
 994		kfree(hdr);
 995	} else {
 996		ret = riocm_queue_req(cm, peer->rdev, hdr, sizeof(*hdr));
 997		if (ret)
 998			kfree(hdr);
 999	}
1000
1001	if (ret) {
1002		riocm_cmp_exch(ch, RIO_CM_CONNECT, RIO_CM_IDLE);
1003		goto conn_done;
1004	}
1005
1006	/* Wait for connect response from the remote device */
1007	wret = wait_for_completion_interruptible_timeout(&ch->comp,
1008							 RIOCM_CONNECT_TO * HZ);
1009	riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1010
1011	if (!wret)
1012		ret = -ETIME;
1013	else if (wret == -ERESTARTSYS)
1014		ret = -EINTR;
1015	else
1016		ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -1;
1017
1018conn_done:
1019	riocm_put_channel(ch);
1020	return ret;
1021}
1022
1023static int riocm_send_ack(struct rio_channel *ch)
1024{
1025	struct rio_ch_chan_hdr *hdr;
1026	int ret;
1027
1028	hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1029	if (hdr == NULL)
1030		return -ENOMEM;
1031
1032	hdr->bhdr.src_id = htonl(ch->loc_destid);
1033	hdr->bhdr.dst_id = htonl(ch->rem_destid);
1034	hdr->dst_ch = htons(ch->rem_channel);
1035	hdr->src_ch = htons(ch->id);
1036	hdr->bhdr.src_mbox = cmbox;
1037	hdr->bhdr.dst_mbox = cmbox;
1038	hdr->bhdr.type = RIO_CM_CHAN;
1039	hdr->ch_op = CM_CONN_ACK;
1040
1041	/* ATTN: the function call below relies on the fact that underlying
1042	 * add_outb_message() routine copies TX data into its internal transfer
1043	 * buffer. Review if switching to direct buffer version.
1044	 */
1045	ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1046
1047	if (ret == -EBUSY && !riocm_queue_req(ch->cmdev,
1048					      ch->rdev, hdr, sizeof(*hdr)))
1049		return 0;
1050	kfree(hdr);
1051
1052	if (ret)
1053		riocm_error("send ACK to ch_%d on %s failed (ret=%d)",
1054			    ch->id, rio_name(ch->rdev), ret);
1055	return ret;
1056}
1057
1058/*
1059 * riocm_ch_accept - accept incoming connection request
1060 * @ch_id: channel ID
1061 * @new_ch_id: local mport device
1062 * @timeout: wait timeout (if 0 non-blocking call, do not wait if connection
1063 *           request is not available).
1064 *
1065 * Returns: pointer to new channel struct if success, or error-valued pointer:
1066 *          -ENODEV - cannot find specified channel or mport,
1067 *          -EINVAL - the channel is not in IDLE state,
1068 *          -EAGAIN - no connection request available immediately (timeout=0),
1069 *          -ENOMEM - unable to allocate new channel,
1070 *          -ETIME - wait timeout expired,
1071 *          -EINTR - wait was interrupted.
1072 */
1073static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
1074					   long timeout)
1075{
1076	struct rio_channel *ch;
1077	struct rio_channel *new_ch;
1078	struct conn_req *req;
1079	struct cm_peer *peer;
1080	int found = 0;
1081	int err = 0;
1082	long wret;
1083
1084	ch = riocm_get_channel(ch_id);
1085	if (!ch)
1086		return ERR_PTR(-EINVAL);
1087
1088	if (!riocm_cmp(ch, RIO_CM_LISTEN)) {
1089		err = -EINVAL;
1090		goto err_put;
1091	}
1092
1093	/* Don't sleep if this is a non blocking call */
1094	if (!timeout) {
1095		if (!try_wait_for_completion(&ch->comp)) {
1096			err = -EAGAIN;
1097			goto err_put;
1098		}
1099	} else {
1100		riocm_debug(WAIT, "on %d", ch->id);
1101
1102		wret = wait_for_completion_interruptible_timeout(&ch->comp,
1103								 timeout);
1104		if (!wret) {
1105			err = -ETIME;
1106			goto err_put;
1107		} else if (wret == -ERESTARTSYS) {
1108			err = -EINTR;
1109			goto err_put;
1110		}
1111	}
1112
1113	spin_lock_bh(&ch->lock);
1114
1115	if (ch->state != RIO_CM_LISTEN) {
1116		err = -ECANCELED;
1117	} else if (list_empty(&ch->accept_queue)) {
1118		riocm_debug(WAIT, "on %d accept_queue is empty on completion",
1119			    ch->id);
1120		err = -EIO;
1121	}
1122
1123	spin_unlock_bh(&ch->lock);
1124
1125	if (err) {
1126		riocm_debug(WAIT, "on %d returns %d", ch->id, err);
1127		goto err_put;
1128	}
1129
1130	/* Create new channel for this connection */
1131	new_ch = riocm_ch_alloc(RIOCM_CHNUM_AUTO);
1132
1133	if (IS_ERR(new_ch)) {
1134		riocm_error("failed to get channel for new req (%ld)",
1135			PTR_ERR(new_ch));
1136		err = -ENOMEM;
1137		goto err_put;
1138	}
1139
1140	spin_lock_bh(&ch->lock);
1141
1142	req = list_first_entry(&ch->accept_queue, struct conn_req, node);
1143	list_del(&req->node);
1144	new_ch->cmdev = ch->cmdev;
1145	new_ch->loc_destid = ch->loc_destid;
1146	new_ch->rem_destid = req->destid;
1147	new_ch->rem_channel = req->chan;
1148
1149	spin_unlock_bh(&ch->lock);
1150	riocm_put_channel(ch);
1151	ch = NULL;
1152	kfree(req);
1153
1154	down_read(&rdev_sem);
1155	/* Find requester's device object */
1156	list_for_each_entry(peer, &new_ch->cmdev->peers, node) {
1157		if (peer->rdev->destid == new_ch->rem_destid) {
1158			riocm_debug(RX_CMD, "found matching device(%s)",
1159				    rio_name(peer->rdev));
1160			found = 1;
1161			break;
1162		}
1163	}
1164	up_read(&rdev_sem);
1165
1166	if (!found) {
1167		/* If peer device object not found, simply ignore the request */
1168		err = -ENODEV;
1169		goto err_put_new_ch;
1170	}
1171
1172	new_ch->rdev = peer->rdev;
1173	new_ch->state = RIO_CM_CONNECTED;
1174	spin_lock_init(&new_ch->lock);
1175
1176	/* Acknowledge the connection request. */
1177	riocm_send_ack(new_ch);
1178
1179	*new_ch_id = new_ch->id;
1180	return new_ch;
1181
1182err_put_new_ch:
1183	spin_lock_bh(&idr_lock);
1184	idr_remove(&ch_idr, new_ch->id);
1185	spin_unlock_bh(&idr_lock);
1186	riocm_put_channel(new_ch);
1187
1188err_put:
1189	if (ch)
1190		riocm_put_channel(ch);
1191	*new_ch_id = 0;
1192	return ERR_PTR(err);
1193}
1194
1195/*
1196 * riocm_ch_listen - puts a channel into LISTEN state
1197 * @ch_id: channel ID
1198 *
1199 * Returns: 0 if success, or
1200 *          -EINVAL if the specified channel does not exists or
1201 *                  is not in CHAN_BOUND state.
1202 */
1203static int riocm_ch_listen(u16 ch_id)
1204{
1205	struct rio_channel *ch = NULL;
1206	int ret = 0;
1207
1208	riocm_debug(CHOP, "(ch_%d)", ch_id);
1209
1210	ch = riocm_get_channel(ch_id);
1211	if (!ch)
1212		return -EINVAL;
1213	if (!riocm_cmp_exch(ch, RIO_CM_CHAN_BOUND, RIO_CM_LISTEN))
1214		ret = -EINVAL;
1215	riocm_put_channel(ch);
1216	return ret;
1217}
1218
1219/*
1220 * riocm_ch_bind - associate a channel object and an mport device
1221 * @ch_id: channel ID
1222 * @mport_id: local mport device ID
1223 * @context: pointer to the additional caller's context
1224 *
1225 * Returns: 0 if success, or
1226 *          -ENODEV if cannot find specified mport,
1227 *          -EINVAL if the specified channel does not exist or
1228 *                  is not in IDLE state.
1229 */
1230static int riocm_ch_bind(u16 ch_id, u8 mport_id, void *context)
1231{
1232	struct rio_channel *ch = NULL;
1233	struct cm_dev *cm;
1234	int rc = -ENODEV;
1235
1236	riocm_debug(CHOP, "ch_%d to mport_%d", ch_id, mport_id);
1237
1238	/* Find matching cm_dev object */
1239	down_read(&rdev_sem);
1240	list_for_each_entry(cm, &cm_dev_list, list) {
1241		if ((cm->mport->id == mport_id) &&
1242		     rio_mport_is_running(cm->mport)) {
1243			rc = 0;
1244			break;
1245		}
1246	}
1247
1248	if (rc)
1249		goto exit;
1250
1251	ch = riocm_get_channel(ch_id);
1252	if (!ch) {
1253		rc = -EINVAL;
1254		goto exit;
1255	}
1256
1257	spin_lock_bh(&ch->lock);
1258	if (ch->state != RIO_CM_IDLE) {
1259		spin_unlock_bh(&ch->lock);
1260		rc = -EINVAL;
1261		goto err_put;
1262	}
1263
1264	ch->cmdev = cm;
1265	ch->loc_destid = cm->mport->host_deviceid;
1266	ch->context = context;
1267	ch->state = RIO_CM_CHAN_BOUND;
1268	spin_unlock_bh(&ch->lock);
1269err_put:
1270	riocm_put_channel(ch);
1271exit:
1272	up_read(&rdev_sem);
1273	return rc;
1274}
1275
1276/*
1277 * riocm_ch_alloc - channel object allocation helper routine
1278 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1279 *
1280 * Return value: pointer to newly created channel object,
1281 *               or error-valued pointer
1282 */
1283static struct rio_channel *riocm_ch_alloc(u16 ch_num)
1284{
1285	int id;
1286	int start, end;
1287	struct rio_channel *ch;
1288
1289	ch = kzalloc(sizeof(*ch), GFP_KERNEL);
1290	if (!ch)
1291		return ERR_PTR(-ENOMEM);
1292
1293	if (ch_num) {
1294		/* If requested, try to obtain the specified channel ID */
1295		start = ch_num;
1296		end = ch_num + 1;
1297	} else {
1298		/* Obtain channel ID from the dynamic allocation range */
1299		start = chstart;
1300		end = RIOCM_MAX_CHNUM + 1;
1301	}
1302
1303	idr_preload(GFP_KERNEL);
1304	spin_lock_bh(&idr_lock);
1305	id = idr_alloc_cyclic(&ch_idr, ch, start, end, GFP_NOWAIT);
1306	spin_unlock_bh(&idr_lock);
1307	idr_preload_end();
1308
1309	if (id < 0) {
1310		kfree(ch);
1311		return ERR_PTR(id == -ENOSPC ? -EBUSY : id);
1312	}
1313
1314	ch->id = (u16)id;
1315	ch->state = RIO_CM_IDLE;
1316	spin_lock_init(&ch->lock);
1317	INIT_LIST_HEAD(&ch->accept_queue);
1318	INIT_LIST_HEAD(&ch->ch_node);
1319	init_completion(&ch->comp);
1320	init_completion(&ch->comp_close);
1321	kref_init(&ch->ref);
1322	ch->rx_ring.head = 0;
1323	ch->rx_ring.tail = 0;
1324	ch->rx_ring.count = 0;
1325	ch->rx_ring.inuse_cnt = 0;
1326
1327	return ch;
1328}
1329
1330/*
1331 * riocm_ch_create - creates a new channel object and allocates ID for it
1332 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1333 *
1334 * Allocates and initializes a new channel object. If the parameter ch_num > 0
1335 * and is within the valid range, riocm_ch_create tries to allocate the
1336 * specified ID for the new channel. If ch_num = 0, channel ID will be assigned
1337 * automatically from the range (chstart ... RIOCM_MAX_CHNUM).
1338 * Module parameter 'chstart' defines start of an ID range available for dynamic
1339 * allocation. Range below 'chstart' is reserved for pre-defined ID numbers.
1340 * Available channel numbers are limited by 16-bit size of channel numbers used
1341 * in the packet header.
1342 *
1343 * Return value: PTR to rio_channel structure if successful (with channel number
1344 *               updated via pointer) or error-valued pointer if error.
1345 */
1346static struct rio_channel *riocm_ch_create(u16 *ch_num)
1347{
1348	struct rio_channel *ch = NULL;
1349
1350	ch = riocm_ch_alloc(*ch_num);
1351
1352	if (IS_ERR(ch))
1353		riocm_debug(CHOP, "Failed to allocate channel %d (err=%ld)",
1354			    *ch_num, PTR_ERR(ch));
1355	else
1356		*ch_num = ch->id;
1357
1358	return ch;
1359}
1360
1361/*
1362 * riocm_ch_free - channel object release routine
1363 * @ref: pointer to a channel's kref structure
1364 */
1365static void riocm_ch_free(struct kref *ref)
1366{
1367	struct rio_channel *ch = container_of(ref, struct rio_channel, ref);
1368	int i;
1369
1370	riocm_debug(CHOP, "(ch_%d)", ch->id);
1371
1372	if (ch->rx_ring.inuse_cnt) {
1373		for (i = 0;
1374		     i < RIOCM_RX_RING_SIZE && ch->rx_ring.inuse_cnt; i++) {
1375			if (ch->rx_ring.inuse[i] != NULL) {
1376				kfree(ch->rx_ring.inuse[i]);
1377				ch->rx_ring.inuse_cnt--;
1378			}
1379		}
1380	}
1381
1382	if (ch->rx_ring.count)
1383		for (i = 0; i < RIOCM_RX_RING_SIZE && ch->rx_ring.count; i++) {
1384			if (ch->rx_ring.buf[i] != NULL) {
1385				kfree(ch->rx_ring.buf[i]);
1386				ch->rx_ring.count--;
1387			}
1388		}
1389
1390	complete(&ch->comp_close);
1391}
1392
1393static int riocm_send_close(struct rio_channel *ch)
1394{
1395	struct rio_ch_chan_hdr *hdr;
1396	int ret;
1397
1398	/*
1399	 * Send CH_CLOSE notification to the remote RapidIO device
1400	 */
1401
1402	hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1403	if (hdr == NULL)
1404		return -ENOMEM;
1405
1406	hdr->bhdr.src_id = htonl(ch->loc_destid);
1407	hdr->bhdr.dst_id = htonl(ch->rem_destid);
1408	hdr->bhdr.src_mbox = cmbox;
1409	hdr->bhdr.dst_mbox = cmbox;
1410	hdr->bhdr.type = RIO_CM_CHAN;
1411	hdr->ch_op = CM_CONN_CLOSE;
1412	hdr->dst_ch = htons(ch->rem_channel);
1413	hdr->src_ch = htons(ch->id);
1414
1415	/* ATTN: the function call below relies on the fact that underlying
1416	 * add_outb_message() routine copies TX data into its internal transfer
1417	 * buffer. Needs to be reviewed if switched to direct buffer mode.
1418	 */
1419	ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1420
1421	if (ret == -EBUSY && !riocm_queue_req(ch->cmdev, ch->rdev,
1422					      hdr, sizeof(*hdr)))
1423		return 0;
1424	kfree(hdr);
1425
1426	if (ret)
1427		riocm_error("ch(%d) send CLOSE failed (ret=%d)", ch->id, ret);
1428
1429	return ret;
1430}
1431
1432/*
1433 * riocm_ch_close - closes a channel object with specified ID (by local request)
1434 * @ch: channel to be closed
1435 */
1436static int riocm_ch_close(struct rio_channel *ch)
1437{
1438	unsigned long tmo = msecs_to_jiffies(3000);
1439	enum rio_cm_state state;
1440	long wret;
1441	int ret = 0;
1442
1443	riocm_debug(CHOP, "ch_%d by %s(%d)",
1444		    ch->id, current->comm, task_pid_nr(current));
1445
1446	state = riocm_exch(ch, RIO_CM_DESTROYING);
1447	if (state == RIO_CM_CONNECTED)
1448		riocm_send_close(ch);
1449
1450	complete_all(&ch->comp);
1451
1452	riocm_put_channel(ch);
1453	wret = wait_for_completion_interruptible_timeout(&ch->comp_close, tmo);
1454
1455	riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1456
1457	if (wret == 0) {
1458		/* Timeout on wait occurred */
1459		riocm_debug(CHOP, "%s(%d) timed out waiting for ch %d",
1460		       current->comm, task_pid_nr(current), ch->id);
1461		ret = -ETIMEDOUT;
1462	} else if (wret == -ERESTARTSYS) {
1463		/* Wait_for_completion was interrupted by a signal */
1464		riocm_debug(CHOP, "%s(%d) wait for ch %d was interrupted",
1465			current->comm, task_pid_nr(current), ch->id);
1466		ret = -EINTR;
1467	}
1468
1469	if (!ret) {
1470		riocm_debug(CHOP, "ch_%d resources released", ch->id);
1471		kfree(ch);
1472	} else {
1473		riocm_debug(CHOP, "failed to release ch_%d resources", ch->id);
1474	}
1475
1476	return ret;
1477}
1478
1479/*
1480 * riocm_cdev_open() - Open character device
1481 */
1482static int riocm_cdev_open(struct inode *inode, struct file *filp)
1483{
1484	riocm_debug(INIT, "by %s(%d) filp=%p ",
1485		    current->comm, task_pid_nr(current), filp);
1486
1487	if (list_empty(&cm_dev_list))
1488		return -ENODEV;
1489
1490	return 0;
1491}
1492
1493/*
1494 * riocm_cdev_release() - Release character device
1495 */
1496static int riocm_cdev_release(struct inode *inode, struct file *filp)
1497{
1498	struct rio_channel *ch, *_c;
1499	unsigned int i;
1500	LIST_HEAD(list);
1501
1502	riocm_debug(EXIT, "by %s(%d) filp=%p",
1503		    current->comm, task_pid_nr(current), filp);
1504
1505	/* Check if there are channels associated with this file descriptor */
1506	spin_lock_bh(&idr_lock);
1507	idr_for_each_entry(&ch_idr, ch, i) {
1508		if (ch && ch->filp == filp) {
1509			riocm_debug(EXIT, "ch_%d not released by %s(%d)",
1510				    ch->id, current->comm,
1511				    task_pid_nr(current));
1512			idr_remove(&ch_idr, ch->id);
1513			list_add(&ch->ch_node, &list);
1514		}
1515	}
1516	spin_unlock_bh(&idr_lock);
1517
1518	if (!list_empty(&list)) {
1519		list_for_each_entry_safe(ch, _c, &list, ch_node) {
1520			list_del(&ch->ch_node);
1521			riocm_ch_close(ch);
1522		}
1523	}
1524
1525	return 0;
1526}
1527
1528/*
1529 * cm_ep_get_list_size() - Reports number of endpoints in the network
1530 */
1531static int cm_ep_get_list_size(void __user *arg)
1532{
1533	u32 __user *p = arg;
1534	u32 mport_id;
1535	u32 count = 0;
1536	struct cm_dev *cm;
1537
1538	if (get_user(mport_id, p))
1539		return -EFAULT;
1540	if (mport_id >= RIO_MAX_MPORTS)
1541		return -EINVAL;
1542
1543	/* Find a matching cm_dev object */
1544	down_read(&rdev_sem);
1545	list_for_each_entry(cm, &cm_dev_list, list) {
1546		if (cm->mport->id == mport_id) {
1547			count = cm->npeers;
1548			up_read(&rdev_sem);
1549			if (copy_to_user(arg, &count, sizeof(u32)))
1550				return -EFAULT;
1551			return 0;
1552		}
1553	}
1554	up_read(&rdev_sem);
1555
1556	return -ENODEV;
1557}
1558
1559/*
1560 * cm_ep_get_list() - Returns list of attached endpoints
1561 */
1562static int cm_ep_get_list(void __user *arg)
1563{
1564	struct cm_dev *cm;
1565	struct cm_peer *peer;
1566	u32 info[2];
1567	void *buf;
1568	u32 nent;
1569	u32 *entry_ptr;
1570	u32 i = 0;
1571	int ret = 0;
1572
1573	if (copy_from_user(&info, arg, sizeof(info)))
1574		return -EFAULT;
1575
1576	if (info[1] >= RIO_MAX_MPORTS || info[0] > RIOCM_MAX_EP_COUNT)
1577		return -EINVAL;
1578
1579	/* Find a matching cm_dev object */
1580	down_read(&rdev_sem);
1581	list_for_each_entry(cm, &cm_dev_list, list)
1582		if (cm->mport->id == (u8)info[1])
1583			goto found;
1584
1585	up_read(&rdev_sem);
1586	return -ENODEV;
1587
1588found:
1589	nent = min(info[0], cm->npeers);
1590	buf = kcalloc(nent + 2, sizeof(u32), GFP_KERNEL);
1591	if (!buf) {
1592		up_read(&rdev_sem);
1593		return -ENOMEM;
1594	}
1595
1596	entry_ptr = (u32 *)((uintptr_t)buf + 2*sizeof(u32));
1597
1598	list_for_each_entry(peer, &cm->peers, node) {
1599		*entry_ptr = (u32)peer->rdev->destid;
1600		entry_ptr++;
1601		if (++i == nent)
1602			break;
1603	}
1604	up_read(&rdev_sem);
1605
1606	((u32 *)buf)[0] = i; /* report an updated number of entries */
1607	((u32 *)buf)[1] = info[1]; /* put back an mport ID */
1608	if (copy_to_user(arg, buf, sizeof(u32) * (info[0] + 2)))
1609		ret = -EFAULT;
1610
1611	kfree(buf);
1612	return ret;
1613}
1614
1615/*
1616 * cm_mport_get_list() - Returns list of available local mport devices
1617 */
1618static int cm_mport_get_list(void __user *arg)
1619{
1620	int ret = 0;
1621	u32 entries;
1622	void *buf;
1623	struct cm_dev *cm;
1624	u32 *entry_ptr;
1625	int count = 0;
1626
1627	if (copy_from_user(&entries, arg, sizeof(entries)))
1628		return -EFAULT;
1629	if (entries == 0 || entries > RIO_MAX_MPORTS)
1630		return -EINVAL;
1631	buf = kcalloc(entries + 1, sizeof(u32), GFP_KERNEL);
1632	if (!buf)
1633		return -ENOMEM;
1634
1635	/* Scan all registered cm_dev objects */
1636	entry_ptr = (u32 *)((uintptr_t)buf + sizeof(u32));
1637	down_read(&rdev_sem);
1638	list_for_each_entry(cm, &cm_dev_list, list) {
1639		if (count++ < entries) {
1640			*entry_ptr = (cm->mport->id << 16) |
1641				      cm->mport->host_deviceid;
1642			entry_ptr++;
1643		}
1644	}
1645	up_read(&rdev_sem);
1646
1647	*((u32 *)buf) = count; /* report a real number of entries */
1648	if (copy_to_user(arg, buf, sizeof(u32) * (count + 1)))
1649		ret = -EFAULT;
1650
1651	kfree(buf);
1652	return ret;
1653}
1654
1655/*
1656 * cm_chan_create() - Create a message exchange channel
1657 */
1658static int cm_chan_create(struct file *filp, void __user *arg)
1659{
1660	u16 __user *p = arg;
1661	u16 ch_num;
1662	struct rio_channel *ch;
1663
1664	if (get_user(ch_num, p))
1665		return -EFAULT;
1666
1667	riocm_debug(CHOP, "ch_%d requested by %s(%d)",
1668		    ch_num, current->comm, task_pid_nr(current));
1669	ch = riocm_ch_create(&ch_num);
1670	if (IS_ERR(ch))
1671		return PTR_ERR(ch);
1672
1673	ch->filp = filp;
1674	riocm_debug(CHOP, "ch_%d created by %s(%d)",
1675		    ch_num, current->comm, task_pid_nr(current));
1676	return put_user(ch_num, p);
1677}
1678
1679/*
1680 * cm_chan_close() - Close channel
1681 * @filp:	Pointer to file object
1682 * @arg:	Channel to close
1683 */
1684static int cm_chan_close(struct file *filp, void __user *arg)
1685{
1686	u16 __user *p = arg;
1687	u16 ch_num;
1688	struct rio_channel *ch;
1689
1690	if (get_user(ch_num, p))
1691		return -EFAULT;
1692
1693	riocm_debug(CHOP, "ch_%d by %s(%d)",
1694		    ch_num, current->comm, task_pid_nr(current));
1695
1696	spin_lock_bh(&idr_lock);
1697	ch = idr_find(&ch_idr, ch_num);
1698	if (!ch) {
1699		spin_unlock_bh(&idr_lock);
1700		return 0;
1701	}
1702	if (ch->filp != filp) {
1703		spin_unlock_bh(&idr_lock);
1704		return -EINVAL;
1705	}
1706	idr_remove(&ch_idr, ch->id);
1707	spin_unlock_bh(&idr_lock);
1708
1709	return riocm_ch_close(ch);
1710}
1711
1712/*
1713 * cm_chan_bind() - Bind channel
1714 * @arg:	Channel number
1715 */
1716static int cm_chan_bind(void __user *arg)
1717{
1718	struct rio_cm_channel chan;
1719
1720	if (copy_from_user(&chan, arg, sizeof(chan)))
1721		return -EFAULT;
1722	if (chan.mport_id >= RIO_MAX_MPORTS)
1723		return -EINVAL;
1724
1725	return riocm_ch_bind(chan.id, chan.mport_id, NULL);
1726}
1727
1728/*
1729 * cm_chan_listen() - Listen on channel
1730 * @arg:	Channel number
1731 */
1732static int cm_chan_listen(void __user *arg)
1733{
1734	u16 __user *p = arg;
1735	u16 ch_num;
1736
1737	if (get_user(ch_num, p))
1738		return -EFAULT;
1739
1740	return riocm_ch_listen(ch_num);
1741}
1742
1743/*
1744 * cm_chan_accept() - Accept incoming connection
1745 * @filp:	Pointer to file object
1746 * @arg:	Channel number
1747 */
1748static int cm_chan_accept(struct file *filp, void __user *arg)
1749{
1750	struct rio_cm_accept param;
1751	long accept_to;
1752	struct rio_channel *ch;
1753
1754	if (copy_from_user(&param, arg, sizeof(param)))
1755		return -EFAULT;
1756
1757	riocm_debug(CHOP, "on ch_%d by %s(%d)",
1758		    param.ch_num, current->comm, task_pid_nr(current));
1759
1760	accept_to = param.wait_to ?
1761			msecs_to_jiffies(param.wait_to) : 0;
1762
1763	ch = riocm_ch_accept(param.ch_num, &param.ch_num, accept_to);
1764	if (IS_ERR(ch))
1765		return PTR_ERR(ch);
1766	ch->filp = filp;
1767
1768	riocm_debug(CHOP, "new ch_%d for %s(%d)",
1769		    ch->id, current->comm, task_pid_nr(current));
1770
1771	if (copy_to_user(arg, &param, sizeof(param)))
1772		return -EFAULT;
1773	return 0;
1774}
1775
1776/*
1777 * cm_chan_connect() - Connect on channel
1778 * @arg:	Channel information
1779 */
1780static int cm_chan_connect(void __user *arg)
1781{
1782	struct rio_cm_channel chan;
1783	struct cm_dev *cm;
1784	struct cm_peer *peer;
1785	int ret = -ENODEV;
1786
1787	if (copy_from_user(&chan, arg, sizeof(chan)))
1788		return -EFAULT;
1789	if (chan.mport_id >= RIO_MAX_MPORTS)
1790		return -EINVAL;
1791
1792	down_read(&rdev_sem);
1793
1794	/* Find matching cm_dev object */
1795	list_for_each_entry(cm, &cm_dev_list, list) {
1796		if (cm->mport->id == chan.mport_id) {
1797			ret = 0;
1798			break;
1799		}
1800	}
1801
1802	if (ret)
1803		goto err_out;
1804
1805	if (chan.remote_destid >= RIO_ANY_DESTID(cm->mport->sys_size)) {
1806		ret = -EINVAL;
1807		goto err_out;
1808	}
1809
1810	/* Find corresponding RapidIO endpoint device object */
1811	ret = -ENODEV;
1812
1813	list_for_each_entry(peer, &cm->peers, node) {
1814		if (peer->rdev->destid == chan.remote_destid) {
1815			ret = 0;
1816			break;
1817		}
1818	}
1819
1820	if (ret)
1821		goto err_out;
1822
1823	up_read(&rdev_sem);
1824
1825	return riocm_ch_connect(chan.id, cm, peer, chan.remote_channel);
1826err_out:
1827	up_read(&rdev_sem);
1828	return ret;
1829}
1830
1831/*
1832 * cm_chan_msg_send() - Send a message through channel
1833 * @arg:	Outbound message information
1834 */
1835static int cm_chan_msg_send(void __user *arg)
1836{
1837	struct rio_cm_msg msg;
1838	void *buf;
1839	int ret;
1840
1841	if (copy_from_user(&msg, arg, sizeof(msg)))
1842		return -EFAULT;
1843	if (msg.size > RIO_MAX_MSG_SIZE)
1844		return -EINVAL;
1845
1846	buf = memdup_user((void __user *)(uintptr_t)msg.msg, msg.size);
1847	if (IS_ERR(buf))
1848		return PTR_ERR(buf);
1849
1850	ret = riocm_ch_send(msg.ch_num, buf, msg.size);
1851
1852	kfree(buf);
1853	return ret;
1854}
1855
1856/*
1857 * cm_chan_msg_rcv() - Receive a message through channel
1858 * @arg:	Inbound message information
1859 */
1860static int cm_chan_msg_rcv(void __user *arg)
1861{
1862	struct rio_cm_msg msg;
1863	struct rio_channel *ch;
1864	void *buf;
1865	long rxto;
1866	int ret = 0, msg_size;
1867
1868	if (copy_from_user(&msg, arg, sizeof(msg)))
1869		return -EFAULT;
1870
1871	if (msg.ch_num == 0 || msg.size == 0)
1872		return -EINVAL;
1873
1874	ch = riocm_get_channel(msg.ch_num);
1875	if (!ch)
1876		return -ENODEV;
1877
1878	rxto = msg.rxto ? msecs_to_jiffies(msg.rxto) : MAX_SCHEDULE_TIMEOUT;
1879
1880	ret = riocm_ch_receive(ch, &buf, rxto);
1881	if (ret)
1882		goto out;
1883
1884	msg_size = min(msg.size, (u16)(RIO_MAX_MSG_SIZE));
1885
1886	if (copy_to_user((void __user *)(uintptr_t)msg.msg, buf, msg_size))
1887		ret = -EFAULT;
1888
1889	riocm_ch_free_rxbuf(ch, buf);
1890out:
1891	riocm_put_channel(ch);
1892	return ret;
1893}
1894
1895/*
1896 * riocm_cdev_ioctl() - IOCTL requests handler
1897 */
1898static long
1899riocm_cdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1900{
1901	switch (cmd) {
1902	case RIO_CM_EP_GET_LIST_SIZE:
1903		return cm_ep_get_list_size((void __user *)arg);
1904	case RIO_CM_EP_GET_LIST:
1905		return cm_ep_get_list((void __user *)arg);
1906	case RIO_CM_CHAN_CREATE:
1907		return cm_chan_create(filp, (void __user *)arg);
1908	case RIO_CM_CHAN_CLOSE:
1909		return cm_chan_close(filp, (void __user *)arg);
1910	case RIO_CM_CHAN_BIND:
1911		return cm_chan_bind((void __user *)arg);
1912	case RIO_CM_CHAN_LISTEN:
1913		return cm_chan_listen((void __user *)arg);
1914	case RIO_CM_CHAN_ACCEPT:
1915		return cm_chan_accept(filp, (void __user *)arg);
1916	case RIO_CM_CHAN_CONNECT:
1917		return cm_chan_connect((void __user *)arg);
1918	case RIO_CM_CHAN_SEND:
1919		return cm_chan_msg_send((void __user *)arg);
1920	case RIO_CM_CHAN_RECEIVE:
1921		return cm_chan_msg_rcv((void __user *)arg);
1922	case RIO_CM_MPORT_GET_LIST:
1923		return cm_mport_get_list((void __user *)arg);
1924	default:
1925		break;
1926	}
1927
1928	return -EINVAL;
1929}
1930
1931static const struct file_operations riocm_cdev_fops = {
1932	.owner		= THIS_MODULE,
1933	.open		= riocm_cdev_open,
1934	.release	= riocm_cdev_release,
1935	.unlocked_ioctl = riocm_cdev_ioctl,
1936};
1937
1938/*
1939 * riocm_add_dev - add new remote RapidIO device into channel management core
1940 * @dev: device object associated with RapidIO device
1941 * @sif: subsystem interface
1942 *
1943 * Adds the specified RapidIO device (if applicable) into peers list of
1944 * the corresponding channel management device (cm_dev).
1945 */
1946static int riocm_add_dev(struct device *dev, struct subsys_interface *sif)
1947{
1948	struct cm_peer *peer;
1949	struct rio_dev *rdev = to_rio_dev(dev);
1950	struct cm_dev *cm;
1951
1952	/* Check if the remote device has capabilities required to support CM */
1953	if (!dev_cm_capable(rdev))
1954		return 0;
1955
1956	riocm_debug(RDEV, "(%s)", rio_name(rdev));
1957
1958	peer = kmalloc(sizeof(*peer), GFP_KERNEL);
1959	if (!peer)
1960		return -ENOMEM;
1961
1962	/* Find a corresponding cm_dev object */
1963	down_write(&rdev_sem);
1964	list_for_each_entry(cm, &cm_dev_list, list) {
1965		if (cm->mport == rdev->net->hport)
1966			goto found;
1967	}
1968
1969	up_write(&rdev_sem);
1970	kfree(peer);
1971	return -ENODEV;
1972
1973found:
1974	peer->rdev = rdev;
1975	list_add_tail(&peer->node, &cm->peers);
1976	cm->npeers++;
1977
1978	up_write(&rdev_sem);
1979	return 0;
1980}
1981
1982/*
1983 * riocm_remove_dev - remove remote RapidIO device from channel management core
1984 * @dev: device object associated with RapidIO device
1985 * @sif: subsystem interface
1986 *
1987 * Removes the specified RapidIO device (if applicable) from peers list of
1988 * the corresponding channel management device (cm_dev).
1989 */
1990static void riocm_remove_dev(struct device *dev, struct subsys_interface *sif)
1991{
1992	struct rio_dev *rdev = to_rio_dev(dev);
1993	struct cm_dev *cm;
1994	struct cm_peer *peer;
1995	struct rio_channel *ch, *_c;
1996	unsigned int i;
1997	bool found = false;
1998	LIST_HEAD(list);
1999
2000	/* Check if the remote device has capabilities required to support CM */
2001	if (!dev_cm_capable(rdev))
2002		return;
2003
2004	riocm_debug(RDEV, "(%s)", rio_name(rdev));
2005
2006	/* Find matching cm_dev object */
2007	down_write(&rdev_sem);
2008	list_for_each_entry(cm, &cm_dev_list, list) {
2009		if (cm->mport == rdev->net->hport) {
2010			found = true;
2011			break;
2012		}
2013	}
2014
2015	if (!found) {
2016		up_write(&rdev_sem);
2017		return;
2018	}
2019
2020	/* Remove remote device from the list of peers */
2021	found = false;
2022	list_for_each_entry(peer, &cm->peers, node) {
2023		if (peer->rdev == rdev) {
2024			riocm_debug(RDEV, "removing peer %s", rio_name(rdev));
2025			found = true;
2026			list_del(&peer->node);
2027			cm->npeers--;
2028			kfree(peer);
2029			break;
2030		}
2031	}
2032
2033	up_write(&rdev_sem);
2034
2035	if (!found)
2036		return;
2037
2038	/*
2039	 * Release channels associated with this peer
2040	 */
2041
2042	spin_lock_bh(&idr_lock);
2043	idr_for_each_entry(&ch_idr, ch, i) {
2044		if (ch && ch->rdev == rdev) {
2045			if (atomic_read(&rdev->state) != RIO_DEVICE_SHUTDOWN)
2046				riocm_exch(ch, RIO_CM_DISCONNECT);
2047			idr_remove(&ch_idr, ch->id);
2048			list_add(&ch->ch_node, &list);
2049		}
2050	}
2051	spin_unlock_bh(&idr_lock);
2052
2053	if (!list_empty(&list)) {
2054		list_for_each_entry_safe(ch, _c, &list, ch_node) {
2055			list_del(&ch->ch_node);
2056			riocm_ch_close(ch);
2057		}
2058	}
2059}
2060
2061/*
2062 * riocm_cdev_add() - Create rio_cm char device
2063 * @devno: device number assigned to device (MAJ + MIN)
2064 */
2065static int riocm_cdev_add(dev_t devno)
2066{
2067	int ret;
2068
2069	cdev_init(&riocm_cdev.cdev, &riocm_cdev_fops);
2070	riocm_cdev.cdev.owner = THIS_MODULE;
2071	ret = cdev_add(&riocm_cdev.cdev, devno, 1);
2072	if (ret < 0) {
2073		riocm_error("Cannot register a device with error %d", ret);
2074		return ret;
2075	}
2076
2077	riocm_cdev.dev = device_create(&dev_class, NULL, devno, NULL, DEV_NAME);
2078	if (IS_ERR(riocm_cdev.dev)) {
2079		cdev_del(&riocm_cdev.cdev);
2080		return PTR_ERR(riocm_cdev.dev);
2081	}
2082
2083	riocm_debug(MPORT, "Added %s cdev(%d:%d)",
2084		    DEV_NAME, MAJOR(devno), MINOR(devno));
2085
2086	return 0;
2087}
2088
2089/*
2090 * riocm_add_mport - add new local mport device into channel management core
2091 * @dev: device object associated with mport
 
2092 *
2093 * When a new mport device is added, CM immediately reserves inbound and
2094 * outbound RapidIO mailboxes that will be used.
2095 */
2096static int riocm_add_mport(struct device *dev)
 
2097{
2098	int rc;
2099	int i;
2100	struct cm_dev *cm;
2101	struct rio_mport *mport = to_rio_mport(dev);
2102
2103	riocm_debug(MPORT, "add mport %s", mport->name);
2104
2105	cm = kzalloc(sizeof(*cm), GFP_KERNEL);
2106	if (!cm)
2107		return -ENOMEM;
2108
2109	cm->mport = mport;
2110
2111	rc = rio_request_outb_mbox(mport, cm, cmbox,
2112				   RIOCM_TX_RING_SIZE, riocm_outb_msg_event);
2113	if (rc) {
2114		riocm_error("failed to allocate OBMBOX_%d on %s",
2115			    cmbox, mport->name);
2116		kfree(cm);
2117		return -ENODEV;
2118	}
2119
2120	rc = rio_request_inb_mbox(mport, cm, cmbox,
2121				  RIOCM_RX_RING_SIZE, riocm_inb_msg_event);
2122	if (rc) {
2123		riocm_error("failed to allocate IBMBOX_%d on %s",
2124			    cmbox, mport->name);
2125		rio_release_outb_mbox(mport, cmbox);
2126		kfree(cm);
2127		return -ENODEV;
2128	}
2129
2130	cm->rx_wq = create_workqueue(DRV_NAME "/rxq");
2131	if (!cm->rx_wq) {
2132		rio_release_inb_mbox(mport, cmbox);
2133		rio_release_outb_mbox(mport, cmbox);
2134		kfree(cm);
2135		return -ENOMEM;
2136	}
2137
2138	/*
2139	 * Allocate and register inbound messaging buffers to be ready
2140	 * to receive channel and system management requests
2141	 */
2142	for (i = 0; i < RIOCM_RX_RING_SIZE; i++)
2143		cm->rx_buf[i] = NULL;
2144
2145	cm->rx_slots = RIOCM_RX_RING_SIZE;
2146	mutex_init(&cm->rx_lock);
2147	riocm_rx_fill(cm, RIOCM_RX_RING_SIZE);
2148	INIT_WORK(&cm->rx_work, rio_ibmsg_handler);
2149
2150	cm->tx_slot = 0;
2151	cm->tx_cnt = 0;
2152	cm->tx_ack_slot = 0;
2153	spin_lock_init(&cm->tx_lock);
2154
2155	INIT_LIST_HEAD(&cm->peers);
2156	cm->npeers = 0;
2157	INIT_LIST_HEAD(&cm->tx_reqs);
2158
2159	down_write(&rdev_sem);
2160	list_add_tail(&cm->list, &cm_dev_list);
2161	up_write(&rdev_sem);
2162
2163	return 0;
2164}
2165
2166/*
2167 * riocm_remove_mport - remove local mport device from channel management core
2168 * @dev: device object associated with mport
 
2169 *
2170 * Removes a local mport device from the list of registered devices that provide
2171 * channel management services. Returns an error if the specified mport is not
2172 * registered with the CM core.
2173 */
2174static void riocm_remove_mport(struct device *dev)
 
2175{
2176	struct rio_mport *mport = to_rio_mport(dev);
2177	struct cm_dev *cm;
2178	struct cm_peer *peer, *temp;
2179	struct rio_channel *ch, *_c;
2180	unsigned int i;
2181	bool found = false;
2182	LIST_HEAD(list);
2183
2184	riocm_debug(MPORT, "%s", mport->name);
2185
2186	/* Find a matching cm_dev object */
2187	down_write(&rdev_sem);
2188	list_for_each_entry(cm, &cm_dev_list, list) {
2189		if (cm->mport == mport) {
2190			list_del(&cm->list);
2191			found = true;
2192			break;
2193		}
2194	}
2195	up_write(&rdev_sem);
2196	if (!found)
2197		return;
2198
2199	flush_workqueue(cm->rx_wq);
2200	destroy_workqueue(cm->rx_wq);
2201
2202	/* Release channels bound to this mport */
2203	spin_lock_bh(&idr_lock);
2204	idr_for_each_entry(&ch_idr, ch, i) {
2205		if (ch->cmdev == cm) {
2206			riocm_debug(RDEV, "%s drop ch_%d",
2207				    mport->name, ch->id);
2208			idr_remove(&ch_idr, ch->id);
2209			list_add(&ch->ch_node, &list);
2210		}
2211	}
2212	spin_unlock_bh(&idr_lock);
2213
2214	if (!list_empty(&list)) {
2215		list_for_each_entry_safe(ch, _c, &list, ch_node) {
2216			list_del(&ch->ch_node);
2217			riocm_ch_close(ch);
2218		}
2219	}
2220
2221	rio_release_inb_mbox(mport, cmbox);
2222	rio_release_outb_mbox(mport, cmbox);
2223
2224	/* Remove and free peer entries */
2225	if (!list_empty(&cm->peers))
2226		riocm_debug(RDEV, "ATTN: peer list not empty");
2227	list_for_each_entry_safe(peer, temp, &cm->peers, node) {
2228		riocm_debug(RDEV, "removing peer %s", rio_name(peer->rdev));
2229		list_del(&peer->node);
2230		kfree(peer);
2231	}
2232
2233	riocm_rx_free(cm);
2234	kfree(cm);
2235	riocm_debug(MPORT, "%s done", mport->name);
2236}
2237
2238static int rio_cm_shutdown(struct notifier_block *nb, unsigned long code,
2239	void *unused)
2240{
2241	struct rio_channel *ch;
2242	unsigned int i;
2243	LIST_HEAD(list);
2244
2245	riocm_debug(EXIT, ".");
2246
2247	/*
2248	 * If there are any channels left in connected state send
2249	 * close notification to the connection partner.
2250	 * First build a list of channels that require a closing
2251	 * notification because function riocm_send_close() should
2252	 * be called outside of spinlock protected code.
2253	 */
2254	spin_lock_bh(&idr_lock);
2255	idr_for_each_entry(&ch_idr, ch, i) {
2256		if (ch->state == RIO_CM_CONNECTED) {
2257			riocm_debug(EXIT, "close ch %d", ch->id);
2258			idr_remove(&ch_idr, ch->id);
2259			list_add(&ch->ch_node, &list);
2260		}
2261	}
2262	spin_unlock_bh(&idr_lock);
2263
2264	list_for_each_entry(ch, &list, ch_node)
2265		riocm_send_close(ch);
2266
2267	return NOTIFY_DONE;
2268}
2269
2270/*
2271 * riocm_interface handles addition/removal of remote RapidIO devices
2272 */
2273static struct subsys_interface riocm_interface = {
2274	.name		= "rio_cm",
2275	.subsys		= &rio_bus_type,
2276	.add_dev	= riocm_add_dev,
2277	.remove_dev	= riocm_remove_dev,
2278};
2279
2280/*
2281 * rio_mport_interface handles addition/removal local mport devices
2282 */
2283static struct class_interface rio_mport_interface __refdata = {
2284	.class = &rio_mport_class,
2285	.add_dev = riocm_add_mport,
2286	.remove_dev = riocm_remove_mport,
2287};
2288
2289static struct notifier_block rio_cm_notifier = {
2290	.notifier_call = rio_cm_shutdown,
2291};
2292
2293static int __init riocm_init(void)
2294{
2295	int ret;
2296
2297	/* Create device class needed by udev */
2298	ret = class_register(&dev_class);
2299	if (ret) {
2300		riocm_error("Cannot create " DRV_NAME " class");
2301		return ret;
2302	}
2303
2304	ret = alloc_chrdev_region(&dev_number, 0, 1, DRV_NAME);
2305	if (ret) {
2306		class_unregister(&dev_class);
2307		return ret;
2308	}
2309
2310	dev_major = MAJOR(dev_number);
2311	dev_minor_base = MINOR(dev_number);
2312	riocm_debug(INIT, "Registered class with %d major", dev_major);
2313
2314	/*
2315	 * Register as rapidio_port class interface to get notifications about
2316	 * mport additions and removals.
2317	 */
2318	ret = class_interface_register(&rio_mport_interface);
2319	if (ret) {
2320		riocm_error("class_interface_register error: %d", ret);
2321		goto err_reg;
2322	}
2323
2324	/*
2325	 * Register as RapidIO bus interface to get notifications about
2326	 * addition/removal of remote RapidIO devices.
2327	 */
2328	ret = subsys_interface_register(&riocm_interface);
2329	if (ret) {
2330		riocm_error("subsys_interface_register error: %d", ret);
2331		goto err_cl;
2332	}
2333
2334	ret = register_reboot_notifier(&rio_cm_notifier);
2335	if (ret) {
2336		riocm_error("failed to register reboot notifier (err=%d)", ret);
2337		goto err_sif;
2338	}
2339
2340	ret = riocm_cdev_add(dev_number);
2341	if (ret) {
2342		unregister_reboot_notifier(&rio_cm_notifier);
2343		ret = -ENODEV;
2344		goto err_sif;
2345	}
2346
2347	return 0;
2348err_sif:
2349	subsys_interface_unregister(&riocm_interface);
2350err_cl:
2351	class_interface_unregister(&rio_mport_interface);
2352err_reg:
2353	unregister_chrdev_region(dev_number, 1);
2354	class_unregister(&dev_class);
2355	return ret;
2356}
2357
2358static void __exit riocm_exit(void)
2359{
2360	riocm_debug(EXIT, "enter");
2361	unregister_reboot_notifier(&rio_cm_notifier);
2362	subsys_interface_unregister(&riocm_interface);
2363	class_interface_unregister(&rio_mport_interface);
2364	idr_destroy(&ch_idr);
2365
2366	device_unregister(riocm_cdev.dev);
2367	cdev_del(&(riocm_cdev.cdev));
2368
2369	class_unregister(&dev_class);
2370	unregister_chrdev_region(dev_number, 1);
2371}
2372
2373late_initcall(riocm_init);
2374module_exit(riocm_exit);