1/*
   2 * This file is provided under a dual BSD/GPLv2 license.  When using or
   3 *   redistributing this file, you may do so under either license.
   4 *
   5 *   GPL LICENSE SUMMARY
   6 *
   7 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
   8 *   Copyright (C) 2017 T-Platforms All Rights Reserved.
   9 *
  10 *   This program is free software; you can redistribute it and/or modify
  11 *   it under the terms of version 2 of the GNU General Public License as
  12 *   published by the Free Software Foundation.
  13 *
  14 *   This program is distributed in the hope that it will be useful, but
  15 *   WITHOUT ANY WARRANTY; without even the implied warranty of
  16 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 *   General Public License for more details.
  18 *
  19 *   BSD LICENSE
  20 *
  21 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  22 *   Copyright (C) 2017 T-Platforms All Rights Reserved.
  23 *
  24 *   Redistribution and use in source and binary forms, with or without
  25 *   modification, are permitted provided that the following conditions
  26 *   are met:
  27 *
  28 *     * Redistributions of source code must retain the above copyright
  29 *       notice, this list of conditions and the following disclaimer.
  30 *     * Redistributions in binary form must reproduce the above copy
  31 *       notice, this list of conditions and the following disclaimer in
  32 *       the documentation and/or other materials provided with the
  33 *       distribution.
  34 *     * Neither the name of Intel Corporation nor the names of its
  35 *       contributors may be used to endorse or promote products derived
  36 *       from this software without specific prior written permission.
  37 *
  38 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  39 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  40 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  41 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  42 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  43 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  44 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  45 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  46 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  47 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  48 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  49 *
  50 * PCIe NTB Debugging Tool Linux driver
  51 */
  52
  53/*
  54 * How to use this tool, by example.
  55 *
  56 * Assuming $DBG_DIR is something like:
  57 * '/sys/kernel/debug/ntb_tool/0000:00:03.0'
  58 * Suppose aside from local device there is at least one remote device
  59 * connected to NTB with index 0.
  60 *-----------------------------------------------------------------------------
  61 * Eg: check local/peer device information.
  62 *
  63 * # Get local device port number
  64 * root@self# cat $DBG_DIR/port
  65 *
  66 * # Check local device functionality
  67 * root@self# ls $DBG_DIR
  68 * db            msg1          msg_sts     peer4/        port
  69 * db_event      msg2          peer0/      peer5/        spad0
  70 * db_mask       msg3          peer1/      peer_db       spad1
  71 * link          msg_event     peer2/      peer_db_mask  spad2
  72 * msg0          msg_mask      peer3/      peer_spad     spad3
  73 * # As one can see it supports:
  74 * # 1) four inbound message registers
  75 * # 2) four inbound scratchpads
  76 * # 3) up to six peer devices
  77 *
  78 * # Check peer device port number
  79 * root@self# cat $DBG_DIR/peer0/port
  80 *
  81 * # Check peer device(s) functionality to be used
  82 * root@self# ls $DBG_DIR/peer0
  83 * link             mw_trans0       mw_trans6        port
  84 * link_event       mw_trans1       mw_trans7        spad0
  85 * msg0             mw_trans2       peer_mw_trans0   spad1
  86 * msg1             mw_trans3       peer_mw_trans1   spad2
  87 * msg2             mw_trans4       peer_mw_trans2   spad3
  88 * msg3             mw_trans5       peer_mw_trans3
  89 * # As one can see we got:
  90 * # 1) four outbound message registers
  91 * # 2) four outbound scratchpads
  92 * # 3) eight inbound memory windows
  93 * # 4) four outbound memory windows
  94 *-----------------------------------------------------------------------------
  95 * Eg: NTB link tests
  96 *
  97 * # Set local link up/down
  98 * root@self# echo Y > $DBG_DIR/link
  99 * root@self# echo N > $DBG_DIR/link
 100 *
 101 * # Check if link with peer device is up/down:
 102 * root@self# cat $DBG_DIR/peer0/link
 103 *
 104 * # Block until the link is up/down
 105 * root@self# echo Y > $DBG_DIR/peer0/link_event
 106 * root@self# echo N > $DBG_DIR/peer0/link_event
 107 *-----------------------------------------------------------------------------
 108 * Eg: Doorbell registers tests (some functionality might be absent)
 109 *
 110 * # Set/clear/get local doorbell
 111 * root@self# echo 's 1' > $DBG_DIR/db
 112 * root@self# echo 'c 1' > $DBG_DIR/db
 113 * root@self# cat  $DBG_DIR/db
 114 *
 115 * # Set/clear/get local doorbell mask
 116 * root@self# echo 's 1' > $DBG_DIR/db_mask
 117 * root@self# echo 'c 1' > $DBG_DIR/db_mask
 118 * root@self# cat $DBG_DIR/db_mask
 119 *
 120 * # Ring/clear/get peer doorbell
 121 * root@peer# echo 's 1' > $DBG_DIR/peer_db
 122 * root@peer# echo 'c 1' > $DBG_DIR/peer_db
 123 * root@peer# cat $DBG_DIR/peer_db
 124 *
 125 * # Set/clear/get peer doorbell mask
 126 * root@self# echo 's 1' > $DBG_DIR/peer_db_mask
 127 * root@self# echo 'c 1' > $DBG_DIR/peer_db_mask
 128 * root@self# cat $DBG_DIR/peer_db_mask
 129 *
 130 * # Block until local doorbell is set with specified value
 131 * root@self# echo 1 > $DBG_DIR/db_event
 132 *-----------------------------------------------------------------------------
 133 * Eg: Message registers tests (functionality might be absent)
 134 *
 135 * # Set/clear/get in/out message registers status
 136 * root@self# echo 's 1' > $DBG_DIR/msg_sts
 137 * root@self# echo 'c 1' > $DBG_DIR/msg_sts
 138 * root@self# cat $DBG_DIR/msg_sts
 139 *
 140 * # Set/clear in/out message registers mask
 141 * root@self# echo 's 1' > $DBG_DIR/msg_mask
 142 * root@self# echo 'c 1' > $DBG_DIR/msg_mask
 143 *
 144 * # Get inbound message register #0 value and source of port index
 145 * root@self# cat  $DBG_DIR/msg0
 146 *
 147 * # Send some data to peer over outbound message register #0
 148 * root@self# echo 0x01020304 > $DBG_DIR/peer0/msg0
 149 *-----------------------------------------------------------------------------
 150 * Eg: Scratchpad registers tests (functionality might be absent)
 151 *
 152 * # Write/read to/from local scratchpad register #0
 153 * root@peer# echo 0x01020304 > $DBG_DIR/spad0
 154 * root@peer# cat $DBG_DIR/spad0
 155 *
 156 * # Write/read to/from peer scratchpad register #0
 157 * root@peer# echo 0x01020304 > $DBG_DIR/peer0/spad0
 158 * root@peer# cat $DBG_DIR/peer0/spad0
 159 *-----------------------------------------------------------------------------
 160 * Eg: Memory windows tests
 161 *
 162 * # Create inbound memory window buffer of specified size/get its base address
 163 * root@peer# echo 16384 > $DBG_DIR/peer0/mw_trans0
 164 * root@peer# cat $DBG_DIR/peer0/mw_trans0
 165 *
 166 * # Write/read data to/from inbound memory window
 167 * root@peer# echo Hello > $DBG_DIR/peer0/mw0
 168 * root@peer# head -c 7 $DBG_DIR/peer0/mw0
 169 *
 170 * # Map outbound memory window/check it settings (on peer device)
 171 * root@peer# echo 0xADD0BA5E:16384 > $DBG_DIR/peer0/peer_mw_trans0
 172 * root@peer# cat $DBG_DIR/peer0/peer_mw_trans0
 173 *
 174 * # Write/read data to/from outbound memory window (on peer device)
 175 * root@peer# echo olleH > $DBG_DIR/peer0/peer_mw0
 176 * root@peer# head -c 7 $DBG_DIR/peer0/peer_mw0
 177 */
 178
 179#include <linux/init.h>
 180#include <linux/kernel.h>
 181#include <linux/module.h>
 182
 183#include <linux/debugfs.h>
 184#include <linux/dma-mapping.h>
 185#include <linux/pci.h>
 186#include <linux/slab.h>
 187#include <linux/uaccess.h>
 188
 189#include <linux/ntb.h>
 190
 191#define DRIVER_NAME		"ntb_tool"
 192#define DRIVER_VERSION		"2.0"
 193
 194MODULE_LICENSE("Dual BSD/GPL");
 195MODULE_VERSION(DRIVER_VERSION);
 196MODULE_AUTHOR("Allen Hubbe <Allen.Hubbe@emc.com>");
 197MODULE_DESCRIPTION("PCIe NTB Debugging Tool");
 198
 199/*
 200 * Inbound and outbound memory windows descriptor. Union members selection
 201 * depends on the MW type the structure describes. mm_base/dma_base are the
 202 * virtual and DMA address of an inbound MW. io_base/tr_base are the MMIO
 203 * mapped virtual and xlat addresses of an outbound MW respectively.
 204 */
 205struct tool_mw {
 206	int widx;
 207	int pidx;
 208	struct tool_ctx *tc;
 209	union {
 210		u8 *mm_base;
 211		u8 __iomem *io_base;
 212	};
 213	union {
 214		dma_addr_t dma_base;
 215		u64 tr_base;
 216	};
 217	resource_size_t size;
 218	struct dentry *dbgfs_file;
 219};
 220
 221/*
 222 * Wrapper structure is used to distinguish the outbound MW peers reference
 223 * within the corresponding DebugFS directory IO operation.
 224 */
 225struct tool_mw_wrap {
 226	int pidx;
 227	struct tool_mw *mw;
 228};
 229
 230struct tool_msg {
 231	int midx;
 232	int pidx;
 233	struct tool_ctx *tc;
 234};
 235
 236struct tool_spad {
 237	int sidx;
 238	int pidx;
 239	struct tool_ctx *tc;
 240};
 241
 242struct tool_peer {
 243	int pidx;
 244	struct tool_ctx *tc;
 245	int inmw_cnt;
 246	struct tool_mw *inmws;
 247	int outmw_cnt;
 248	struct tool_mw_wrap *outmws;
 249	int outmsg_cnt;
 250	struct tool_msg *outmsgs;
 251	int outspad_cnt;
 252	struct tool_spad *outspads;
 253	struct dentry *dbgfs_dir;
 254};
 255
 256struct tool_ctx {
 257	struct ntb_dev *ntb;
 258	wait_queue_head_t link_wq;
 259	wait_queue_head_t db_wq;
 260	wait_queue_head_t msg_wq;
 261	int outmw_cnt;
 262	struct tool_mw *outmws;
 263	int peer_cnt;
 264	struct tool_peer *peers;
 265	int inmsg_cnt;
 266	struct tool_msg *inmsgs;
 267	int inspad_cnt;
 268	struct tool_spad *inspads;
 269	struct dentry *dbgfs_dir;
 270};
 271
 272#define TOOL_FOPS_RDWR(__name, __read, __write) \
 273	const struct file_operations __name = {	\
 274		.owner = THIS_MODULE,		\
 275		.open = simple_open,		\
 276		.read = __read,			\
 277		.write = __write,		\
 278	}
 279
 280#define TOOL_BUF_LEN 32
 281
 282static struct dentry *tool_dbgfs_topdir;
 283
 284/*==============================================================================
 285 *                               NTB events handlers
 286 *==============================================================================
 287 */
 288
 289static void tool_link_event(void *ctx)
 290{
 291	struct tool_ctx *tc = ctx;
 292	enum ntb_speed speed;
 293	enum ntb_width width;
 294	int up;
 295
 296	up = ntb_link_is_up(tc->ntb, &speed, &width);
 297
 298	dev_dbg(&tc->ntb->dev, "link is %s speed %d width %d\n",
 299		up ? "up" : "down", speed, width);
 300
 301	wake_up(&tc->link_wq);
 302}
 303
 304static void tool_db_event(void *ctx, int vec)
 305{
 306	struct tool_ctx *tc = ctx;
 307	u64 db_bits, db_mask;
 308
 309	db_mask = ntb_db_vector_mask(tc->ntb, vec);
 310	db_bits = ntb_db_read(tc->ntb);
 311
 312	dev_dbg(&tc->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
 313		vec, db_mask, db_bits);
 314
 315	wake_up(&tc->db_wq);
 316}
 317
 318static void tool_msg_event(void *ctx)
 319{
 320	struct tool_ctx *tc = ctx;
 321	u64 msg_sts;
 322
 323	msg_sts = ntb_msg_read_sts(tc->ntb);
 324
 325	dev_dbg(&tc->ntb->dev, "message bits %#llx\n", msg_sts);
 326
 327	wake_up(&tc->msg_wq);
 328}
 329
 330static const struct ntb_ctx_ops tool_ops = {
 331	.link_event = tool_link_event,
 332	.db_event = tool_db_event,
 333	.msg_event = tool_msg_event
 334};
 335
 336/*==============================================================================
 337 *                        Common read/write methods
 338 *==============================================================================
 339 */
 340
 341static ssize_t tool_fn_read(struct tool_ctx *tc, char __user *ubuf,
 342			    size_t size, loff_t *offp,
 343			    u64 (*fn_read)(struct ntb_dev *))
 344{
 345	size_t buf_size;
 346	char buf[TOOL_BUF_LEN];
 347	ssize_t pos;
 348
 349	if (!fn_read)
 350		return -EINVAL;
 351
 352	buf_size = min(size, sizeof(buf));
 353
 354	pos = scnprintf(buf, buf_size, "%#llx\n", fn_read(tc->ntb));
 355
 356	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 357}
 358
 359static ssize_t tool_fn_write(struct tool_ctx *tc,
 360			     const char __user *ubuf,
 361			     size_t size, loff_t *offp,
 362			     int (*fn_set)(struct ntb_dev *, u64),
 363			     int (*fn_clear)(struct ntb_dev *, u64))
 364{
 365	char *buf, cmd;
 366	ssize_t ret;
 367	u64 bits;
 368	int n;
 369
 370	if (*offp)
 371		return 0;
 372
 373	buf = memdup_user_nul(ubuf, size);
 374	if (IS_ERR(buf))
 375		return PTR_ERR(buf);
 376
 377	n = sscanf(buf, "%c %lli", &cmd, &bits);
 378
 379	kfree(buf);
 380
 381	if (n != 2) {
 382		ret = -EINVAL;
 383	} else if (cmd == 's') {
 384		if (!fn_set)
 385			ret = -EINVAL;
 386		else
 387			ret = fn_set(tc->ntb, bits);
 388	} else if (cmd == 'c') {
 389		if (!fn_clear)
 390			ret = -EINVAL;
 391		else
 392			ret = fn_clear(tc->ntb, bits);
 393	} else {
 394		ret = -EINVAL;
 395	}
 396
 397	return ret ? : size;
 398}
 399
 400/*==============================================================================
 401 *                            Port read/write methods
 402 *==============================================================================
 403 */
 404
 405static ssize_t tool_port_read(struct file *filep, char __user *ubuf,
 406			      size_t size, loff_t *offp)
 407{
 408	struct tool_ctx *tc = filep->private_data;
 409	char buf[TOOL_BUF_LEN];
 410	int pos;
 411
 412	pos = scnprintf(buf, sizeof(buf), "%d\n", ntb_port_number(tc->ntb));
 413
 414	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 415}
 416
 417static TOOL_FOPS_RDWR(tool_port_fops,
 418		      tool_port_read,
 419		      NULL);
 420
 421static ssize_t tool_peer_port_read(struct file *filep, char __user *ubuf,
 422				   size_t size, loff_t *offp)
 423{
 424	struct tool_peer *peer = filep->private_data;
 425	struct tool_ctx *tc = peer->tc;
 426	char buf[TOOL_BUF_LEN];
 427	int pos;
 428
 429	pos = scnprintf(buf, sizeof(buf), "%d\n",
 430		ntb_peer_port_number(tc->ntb, peer->pidx));
 431
 432	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 433}
 434
 435static TOOL_FOPS_RDWR(tool_peer_port_fops,
 436		      tool_peer_port_read,
 437		      NULL);
 438
 439static int tool_init_peers(struct tool_ctx *tc)
 440{
 441	int pidx;
 442
 443	tc->peer_cnt = ntb_peer_port_count(tc->ntb);
 444	tc->peers = devm_kcalloc(&tc->ntb->dev, tc->peer_cnt,
 445				 sizeof(*tc->peers), GFP_KERNEL);
 446	if (tc->peers == NULL)
 447		return -ENOMEM;
 448
 449	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
 450		tc->peers[pidx].pidx = pidx;
 451		tc->peers[pidx].tc = tc;
 452	}
 453
 454	return 0;
 455}
 456
 457/*==============================================================================
 458 *                       Link state read/write methods
 459 *==============================================================================
 460 */
 461
 462static ssize_t tool_link_write(struct file *filep, const char __user *ubuf,
 463			       size_t size, loff_t *offp)
 464{
 465	struct tool_ctx *tc = filep->private_data;
 466	bool val;
 467	int ret;
 468
 469	ret = kstrtobool_from_user(ubuf, size, &val);
 470	if (ret)
 471		return ret;
 472
 473	if (val)
 474		ret = ntb_link_enable(tc->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
 475	else
 476		ret = ntb_link_disable(tc->ntb);
 477
 478	if (ret)
 479		return ret;
 480
 481	return size;
 482}
 483
 484static TOOL_FOPS_RDWR(tool_link_fops,
 485		      NULL,
 486		      tool_link_write);
 487
 488static ssize_t tool_peer_link_read(struct file *filep, char __user *ubuf,
 489				   size_t size, loff_t *offp)
 490{
 491	struct tool_peer *peer = filep->private_data;
 492	struct tool_ctx *tc = peer->tc;
 493	char buf[3];
 494
 495	if (ntb_link_is_up(tc->ntb, NULL, NULL) & BIT(peer->pidx))
 496		buf[0] = 'Y';
 497	else
 498		buf[0] = 'N';
 499	buf[1] = '\n';
 500	buf[2] = '\0';
 501
 502	return simple_read_from_buffer(ubuf, size, offp, buf, 2);
 503}
 504
 505static TOOL_FOPS_RDWR(tool_peer_link_fops,
 506		      tool_peer_link_read,
 507		      NULL);
 508
 509static ssize_t tool_peer_link_event_write(struct file *filep,
 510					  const char __user *ubuf,
 511					  size_t size, loff_t *offp)
 512{
 513	struct tool_peer *peer = filep->private_data;
 514	struct tool_ctx *tc = peer->tc;
 515	u64 link_msk;
 516	bool val;
 517	int ret;
 518
 519	ret = kstrtobool_from_user(ubuf, size, &val);
 520	if (ret)
 521		return ret;
 522
 523	link_msk = BIT_ULL_MASK(peer->pidx);
 524
 525	if (wait_event_interruptible(tc->link_wq,
 526		!!(ntb_link_is_up(tc->ntb, NULL, NULL) & link_msk) == val))
 527		return -ERESTART;
 528
 529	return size;
 530}
 531
 532static TOOL_FOPS_RDWR(tool_peer_link_event_fops,
 533		      NULL,
 534		      tool_peer_link_event_write);
 535
 536/*==============================================================================
 537 *                  Memory windows read/write/setting methods
 538 *==============================================================================
 539 */
 540
 541static ssize_t tool_mw_read(struct file *filep, char __user *ubuf,
 542			    size_t size, loff_t *offp)
 543{
 544	struct tool_mw *inmw = filep->private_data;
 545
 546	if (inmw->mm_base == NULL)
 547		return -ENXIO;
 548
 549	return simple_read_from_buffer(ubuf, size, offp,
 550				       inmw->mm_base, inmw->size);
 551}
 552
 553static ssize_t tool_mw_write(struct file *filep, const char __user *ubuf,
 554			     size_t size, loff_t *offp)
 555{
 556	struct tool_mw *inmw = filep->private_data;
 557
 558	if (inmw->mm_base == NULL)
 559		return -ENXIO;
 560
 561	return simple_write_to_buffer(inmw->mm_base, inmw->size, offp,
 562				      ubuf, size);
 563}
 564
 565static TOOL_FOPS_RDWR(tool_mw_fops,
 566		      tool_mw_read,
 567		      tool_mw_write);
 568
 569static int tool_setup_mw(struct tool_ctx *tc, int pidx, int widx,
 570			 size_t req_size)
 571{
 572	resource_size_t size, addr_align, size_align;
 573	struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
 574	char buf[TOOL_BUF_LEN];
 575	int ret;
 576
 577	if (inmw->mm_base != NULL)
 578		return 0;
 579
 580	ret = ntb_mw_get_align(tc->ntb, pidx, widx, &addr_align,
 581				&size_align, &size);
 582	if (ret)
 583		return ret;
 584
 585	inmw->size = min_t(resource_size_t, req_size, size);
 586	inmw->size = round_up(inmw->size, addr_align);
 587	inmw->size = round_up(inmw->size, size_align);
 588	inmw->mm_base = dma_alloc_coherent(&tc->ntb->pdev->dev, inmw->size,
 589					   &inmw->dma_base, GFP_KERNEL);
 590	if (!inmw->mm_base)
 591		return -ENOMEM;
 592
 593	if (!IS_ALIGNED(inmw->dma_base, addr_align)) {
 594		ret = -ENOMEM;
 595		goto err_free_dma;
 596	}
 597
 598	ret = ntb_mw_set_trans(tc->ntb, pidx, widx, inmw->dma_base, inmw->size);
 599	if (ret)
 600		goto err_free_dma;
 601
 602	snprintf(buf, sizeof(buf), "mw%d", widx);
 603	inmw->dbgfs_file = debugfs_create_file(buf, 0600,
 604					       tc->peers[pidx].dbgfs_dir, inmw,
 605					       &tool_mw_fops);
 606
 607	return 0;
 608
 609err_free_dma:
 610	dma_free_coherent(&tc->ntb->pdev->dev, inmw->size, inmw->mm_base,
 611			  inmw->dma_base);
 612	inmw->mm_base = NULL;
 613	inmw->dma_base = 0;
 614	inmw->size = 0;
 615
 616	return ret;
 617}
 618
 619static void tool_free_mw(struct tool_ctx *tc, int pidx, int widx)
 620{
 621	struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
 622
 623	debugfs_remove(inmw->dbgfs_file);
 624
 625	if (inmw->mm_base != NULL) {
 626		ntb_mw_clear_trans(tc->ntb, pidx, widx);
 627		dma_free_coherent(&tc->ntb->pdev->dev, inmw->size,
 628				  inmw->mm_base, inmw->dma_base);
 629	}
 630
 631	inmw->mm_base = NULL;
 632	inmw->dma_base = 0;
 633	inmw->size = 0;
 634	inmw->dbgfs_file = NULL;
 635}
 636
 637static ssize_t tool_mw_trans_read(struct file *filep, char __user *ubuf,
 638				  size_t size, loff_t *offp)
 639{
 640	struct tool_mw *inmw = filep->private_data;
 641	resource_size_t addr_align;
 642	resource_size_t size_align;
 643	resource_size_t size_max;
 644	ssize_t ret, off = 0;
 645	size_t buf_size;
 646	char *buf;
 647
 648	buf_size = min_t(size_t, size, 512);
 649
 650	buf = kmalloc(buf_size, GFP_KERNEL);
 651	if (!buf)
 652		return -ENOMEM;
 653
 654	ret = ntb_mw_get_align(inmw->tc->ntb, inmw->pidx, inmw->widx,
 655			       &addr_align, &size_align, &size_max);
 656	if (ret)
 657		goto err;
 658
 659	off += scnprintf(buf + off, buf_size - off,
 660			 "Inbound MW     \t%d\n",
 661			 inmw->widx);
 662
 663	off += scnprintf(buf + off, buf_size - off,
 664			 "Port           \t%d (%d)\n",
 665			 ntb_peer_port_number(inmw->tc->ntb, inmw->pidx),
 666			 inmw->pidx);
 667
 668	off += scnprintf(buf + off, buf_size - off,
 669			 "Window Address \t0x%pK\n", inmw->mm_base);
 670
 671	off += scnprintf(buf + off, buf_size - off,
 672			 "DMA Address    \t%pad\n",
 673			 &inmw->dma_base);
 674
 675	off += scnprintf(buf + off, buf_size - off,
 676			 "Window Size    \t%pap\n",
 677			 &inmw->size);
 678
 679	off += scnprintf(buf + off, buf_size - off,
 680			 "Alignment      \t%pap\n",
 681			 &addr_align);
 682
 683	off += scnprintf(buf + off, buf_size - off,
 684			 "Size Alignment \t%pap\n",
 685			 &size_align);
 686
 687	off += scnprintf(buf + off, buf_size - off,
 688			 "Size Max       \t%pap\n",
 689			 &size_max);
 690
 691	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
 692
 693err:
 694	kfree(buf);
 695
 696	return ret;
 697}
 698
 699static ssize_t tool_mw_trans_write(struct file *filep, const char __user *ubuf,
 700				   size_t size, loff_t *offp)
 701{
 702	struct tool_mw *inmw = filep->private_data;
 703	unsigned int val;
 704	int ret;
 705
 706	ret = kstrtouint_from_user(ubuf, size, 0, &val);
 707	if (ret)
 708		return ret;
 709
 710	tool_free_mw(inmw->tc, inmw->pidx, inmw->widx);
 711	if (val) {
 712		ret = tool_setup_mw(inmw->tc, inmw->pidx, inmw->widx, val);
 713		if (ret)
 714			return ret;
 715	}
 716
 717	return size;
 718}
 719
 720static TOOL_FOPS_RDWR(tool_mw_trans_fops,
 721		      tool_mw_trans_read,
 722		      tool_mw_trans_write);
 723
 724static ssize_t tool_peer_mw_read(struct file *filep, char __user *ubuf,
 725				 size_t size, loff_t *offp)
 726{
 727	struct tool_mw *outmw = filep->private_data;
 728	loff_t pos = *offp;
 729	ssize_t ret;
 730	void *buf;
 731
 732	if (outmw->io_base == NULL)
 733		return -EIO;
 734
 735	if (pos >= outmw->size || !size)
 736		return 0;
 737
 738	if (size > outmw->size - pos)
 739		size = outmw->size - pos;
 740
 741	buf = kmalloc(size, GFP_KERNEL);
 742	if (!buf)
 743		return -ENOMEM;
 744
 745	memcpy_fromio(buf, outmw->io_base + pos, size);
 746	ret = copy_to_user(ubuf, buf, size);
 747	if (ret == size) {
 748		ret = -EFAULT;
 749		goto err_free;
 750	}
 751
 752	size -= ret;
 753	*offp = pos + size;
 754	ret = size;
 755
 756err_free:
 757	kfree(buf);
 758
 759	return ret;
 760}
 761
 762static ssize_t tool_peer_mw_write(struct file *filep, const char __user *ubuf,
 763				  size_t size, loff_t *offp)
 764{
 765	struct tool_mw *outmw = filep->private_data;
 766	ssize_t ret;
 767	loff_t pos = *offp;
 768	void *buf;
 769
 770	if (outmw->io_base == NULL)
 771		return -EIO;
 772
 773	if (pos >= outmw->size || !size)
 774		return 0;
 775	if (size > outmw->size - pos)
 776		size = outmw->size - pos;
 777
 778	buf = kmalloc(size, GFP_KERNEL);
 779	if (!buf)
 780		return -ENOMEM;
 781
 782	ret = copy_from_user(buf, ubuf, size);
 783	if (ret == size) {
 784		ret = -EFAULT;
 785		goto err_free;
 786	}
 787
 788	size -= ret;
 789	*offp = pos + size;
 790	ret = size;
 791
 792	memcpy_toio(outmw->io_base + pos, buf, size);
 793
 794err_free:
 795	kfree(buf);
 796
 797	return ret;
 798}
 799
 800static TOOL_FOPS_RDWR(tool_peer_mw_fops,
 801		      tool_peer_mw_read,
 802		      tool_peer_mw_write);
 803
 804static int tool_setup_peer_mw(struct tool_ctx *tc, int pidx, int widx,
 805			      u64 req_addr, size_t req_size)
 806{
 807	struct tool_mw *outmw = &tc->outmws[widx];
 808	resource_size_t map_size;
 809	phys_addr_t map_base;
 810	char buf[TOOL_BUF_LEN];
 811	int ret;
 812
 813	if (outmw->io_base != NULL)
 814		return 0;
 815
 816	ret = ntb_peer_mw_get_addr(tc->ntb, widx, &map_base, &map_size);
 817	if (ret)
 818		return ret;
 819
 820	ret = ntb_peer_mw_set_trans(tc->ntb, pidx, widx, req_addr, req_size);
 821	if (ret)
 822		return ret;
 823
 824	outmw->io_base = ioremap_wc(map_base, map_size);
 825	if (outmw->io_base == NULL) {
 826		ret = -EFAULT;
 827		goto err_clear_trans;
 828	}
 829
 830	outmw->tr_base = req_addr;
 831	outmw->size = req_size;
 832	outmw->pidx = pidx;
 833
 834	snprintf(buf, sizeof(buf), "peer_mw%d", widx);
 835	outmw->dbgfs_file = debugfs_create_file(buf, 0600,
 836					       tc->peers[pidx].dbgfs_dir, outmw,
 837					       &tool_peer_mw_fops);
 838
 839	return 0;
 840
 841err_clear_trans:
 842	ntb_peer_mw_clear_trans(tc->ntb, pidx, widx);
 843
 844	return ret;
 845}
 846
 847static void tool_free_peer_mw(struct tool_ctx *tc, int widx)
 848{
 849	struct tool_mw *outmw = &tc->outmws[widx];
 850
 851	debugfs_remove(outmw->dbgfs_file);
 852
 853	if (outmw->io_base != NULL) {
 854		iounmap(tc->outmws[widx].io_base);
 855		ntb_peer_mw_clear_trans(tc->ntb, outmw->pidx, widx);
 856	}
 857
 858	outmw->io_base = NULL;
 859	outmw->tr_base = 0;
 860	outmw->size = 0;
 861	outmw->pidx = -1;
 862	outmw->dbgfs_file = NULL;
 863}
 864
 865static ssize_t tool_peer_mw_trans_read(struct file *filep, char __user *ubuf,
 866					size_t size, loff_t *offp)
 867{
 868	struct tool_mw_wrap *outmw_wrap = filep->private_data;
 869	struct tool_mw *outmw = outmw_wrap->mw;
 870	resource_size_t map_size;
 871	phys_addr_t map_base;
 872	ssize_t off = 0;
 873	size_t buf_size;
 874	char *buf;
 875	int ret;
 876
 877	ret = ntb_peer_mw_get_addr(outmw->tc->ntb, outmw->widx,
 878				  &map_base, &map_size);
 879	if (ret)
 880		return ret;
 881
 882	buf_size = min_t(size_t, size, 512);
 883
 884	buf = kmalloc(buf_size, GFP_KERNEL);
 885	if (!buf)
 886		return -ENOMEM;
 887
 888	off += scnprintf(buf + off, buf_size - off,
 889			 "Outbound MW:        \t%d\n", outmw->widx);
 890
 891	if (outmw->io_base != NULL) {
 892		off += scnprintf(buf + off, buf_size - off,
 893			"Port attached       \t%d (%d)\n",
 894			ntb_peer_port_number(outmw->tc->ntb, outmw->pidx),
 895			outmw->pidx);
 896	} else {
 897		off += scnprintf(buf + off, buf_size - off,
 898				 "Port attached       \t-1 (-1)\n");
 899	}
 900
 901	off += scnprintf(buf + off, buf_size - off,
 902			 "Virtual address     \t0x%pK\n", outmw->io_base);
 903
 904	off += scnprintf(buf + off, buf_size - off,
 905			 "Phys Address        \t%pap\n", &map_base);
 906
 907	off += scnprintf(buf + off, buf_size - off,
 908			 "Mapping Size        \t%pap\n", &map_size);
 909
 910	off += scnprintf(buf + off, buf_size - off,
 911			 "Translation Address \t0x%016llx\n", outmw->tr_base);
 912
 913	off += scnprintf(buf + off, buf_size - off,
 914			 "Window Size         \t%pap\n", &outmw->size);
 915
 916	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
 917	kfree(buf);
 918
 919	return ret;
 920}
 921
 922static ssize_t tool_peer_mw_trans_write(struct file *filep,
 923					const char __user *ubuf,
 924					size_t size, loff_t *offp)
 925{
 926	struct tool_mw_wrap *outmw_wrap = filep->private_data;
 927	struct tool_mw *outmw = outmw_wrap->mw;
 928	size_t buf_size, wsize;
 929	char buf[TOOL_BUF_LEN];
 930	int ret, n;
 931	u64 addr;
 932
 933	buf_size = min(size, (sizeof(buf) - 1));
 934	if (copy_from_user(buf, ubuf, buf_size))
 935		return -EFAULT;
 936
 937	buf[buf_size] = '\0';
 938
 939	n = sscanf(buf, "%lli:%zi", &addr, &wsize);
 940	if (n != 2)
 941		return -EINVAL;
 942
 943	tool_free_peer_mw(outmw->tc, outmw->widx);
 944	if (wsize) {
 945		ret = tool_setup_peer_mw(outmw->tc, outmw_wrap->pidx,
 946					 outmw->widx, addr, wsize);
 947		if (ret)
 948			return ret;
 949	}
 950
 951	return size;
 952}
 953
 954static TOOL_FOPS_RDWR(tool_peer_mw_trans_fops,
 955		      tool_peer_mw_trans_read,
 956		      tool_peer_mw_trans_write);
 957
 958static int tool_init_mws(struct tool_ctx *tc)
 959{
 960	int widx, pidx;
 961
 962	/* Initialize outbound memory windows */
 963	tc->outmw_cnt = ntb_peer_mw_count(tc->ntb);
 964	tc->outmws = devm_kcalloc(&tc->ntb->dev, tc->outmw_cnt,
 965				  sizeof(*tc->outmws), GFP_KERNEL);
 966	if (tc->outmws == NULL)
 967		return -ENOMEM;
 968
 969	for (widx = 0; widx < tc->outmw_cnt; widx++) {
 970		tc->outmws[widx].widx = widx;
 971		tc->outmws[widx].pidx = -1;
 972		tc->outmws[widx].tc = tc;
 973	}
 974
 975	/* Initialize inbound memory windows and outbound MWs wrapper */
 976	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
 977		tc->peers[pidx].inmw_cnt = ntb_mw_count(tc->ntb, pidx);
 978		tc->peers[pidx].inmws =
 979			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].inmw_cnt,
 980				    sizeof(*tc->peers[pidx].inmws), GFP_KERNEL);
 981		if (tc->peers[pidx].inmws == NULL)
 982			return -ENOMEM;
 983
 984		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
 985			tc->peers[pidx].inmws[widx].widx = widx;
 986			tc->peers[pidx].inmws[widx].pidx = pidx;
 987			tc->peers[pidx].inmws[widx].tc = tc;
 988		}
 989
 990		tc->peers[pidx].outmw_cnt = ntb_peer_mw_count(tc->ntb);
 991		tc->peers[pidx].outmws =
 992			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmw_cnt,
 993				   sizeof(*tc->peers[pidx].outmws), GFP_KERNEL);
 994		if (tc->peers[pidx].outmws == NULL)
 995			return -ENOMEM;
 996
 997		for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
 998			tc->peers[pidx].outmws[widx].pidx = pidx;
 999			tc->peers[pidx].outmws[widx].mw = &tc->outmws[widx];
1000		}
1001	}
1002
1003	return 0;
1004}
1005
1006static void tool_clear_mws(struct tool_ctx *tc)
1007{
1008	int widx, pidx;
1009
1010	/* Free outbound memory windows */
1011	for (widx = 0; widx < tc->outmw_cnt; widx++)
1012		tool_free_peer_mw(tc, widx);
1013
1014	/* Free outbound memory windows */
1015	for (pidx = 0; pidx < tc->peer_cnt; pidx++)
1016		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++)
1017			tool_free_mw(tc, pidx, widx);
1018}
1019
1020/*==============================================================================
1021 *                       Doorbell read/write methods
1022 *==============================================================================
1023 */
1024
1025static ssize_t tool_db_read(struct file *filep, char __user *ubuf,
1026			    size_t size, loff_t *offp)
1027{
1028	struct tool_ctx *tc = filep->private_data;
1029
1030	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read);
1031}
1032
1033static ssize_t tool_db_write(struct file *filep, const char __user *ubuf,
1034			     size_t size, loff_t *offp)
1035{
1036	struct tool_ctx *tc = filep->private_data;
1037
1038	return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set,
1039			     tc->ntb->ops->db_clear);
1040}
1041
1042static TOOL_FOPS_RDWR(tool_db_fops,
1043		      tool_db_read,
1044		      tool_db_write);
1045
1046static ssize_t tool_db_valid_mask_read(struct file *filep, char __user *ubuf,
1047				       size_t size, loff_t *offp)
1048{
1049	struct tool_ctx *tc = filep->private_data;
1050
1051	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_valid_mask);
1052}
1053
1054static TOOL_FOPS_RDWR(tool_db_valid_mask_fops,
1055		      tool_db_valid_mask_read,
1056		      NULL);
1057
1058static ssize_t tool_db_mask_read(struct file *filep, char __user *ubuf,
1059				 size_t size, loff_t *offp)
1060{
1061	struct tool_ctx *tc = filep->private_data;
1062
1063	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read_mask);
1064}
1065
1066static ssize_t tool_db_mask_write(struct file *filep, const char __user *ubuf,
1067			       size_t size, loff_t *offp)
1068{
1069	struct tool_ctx *tc = filep->private_data;
1070
1071	return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set_mask,
1072			     tc->ntb->ops->db_clear_mask);
1073}
1074
1075static TOOL_FOPS_RDWR(tool_db_mask_fops,
1076		      tool_db_mask_read,
1077		      tool_db_mask_write);
1078
1079static ssize_t tool_peer_db_read(struct file *filep, char __user *ubuf,
1080				 size_t size, loff_t *offp)
1081{
1082	struct tool_ctx *tc = filep->private_data;
1083
1084	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->peer_db_read);
1085}
1086
1087static ssize_t tool_peer_db_write(struct file *filep, const char __user *ubuf,
1088				  size_t size, loff_t *offp)
1089{
1090	struct tool_ctx *tc = filep->private_data;
1091
1092	return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->peer_db_set,
1093			     tc->ntb->ops->peer_db_clear);
1094}
1095
1096static TOOL_FOPS_RDWR(tool_peer_db_fops,
1097		      tool_peer_db_read,
1098		      tool_peer_db_write);
1099
1100static ssize_t tool_peer_db_mask_read(struct file *filep, char __user *ubuf,
1101				   size_t size, loff_t *offp)
1102{
1103	struct tool_ctx *tc = filep->private_data;
1104
1105	return tool_fn_read(tc, ubuf, size, offp,
1106			    tc->ntb->ops->peer_db_read_mask);
1107}
1108
1109static ssize_t tool_peer_db_mask_write(struct file *filep,
1110				       const char __user *ubuf,
1111				       size_t size, loff_t *offp)
1112{
1113	struct tool_ctx *tc = filep->private_data;
1114
1115	return tool_fn_write(tc, ubuf, size, offp,
1116			     tc->ntb->ops->peer_db_set_mask,
1117			     tc->ntb->ops->peer_db_clear_mask);
1118}
1119
1120static TOOL_FOPS_RDWR(tool_peer_db_mask_fops,
1121		      tool_peer_db_mask_read,
1122		      tool_peer_db_mask_write);
1123
1124static ssize_t tool_db_event_write(struct file *filep,
1125				   const char __user *ubuf,
1126				   size_t size, loff_t *offp)
1127{
1128	struct tool_ctx *tc = filep->private_data;
1129	u64 val;
1130	int ret;
1131
1132	ret = kstrtou64_from_user(ubuf, size, 0, &val);
1133	if (ret)
1134		return ret;
1135
1136	if (wait_event_interruptible(tc->db_wq, ntb_db_read(tc->ntb) == val))
1137		return -ERESTART;
1138
1139	return size;
1140}
1141
1142static TOOL_FOPS_RDWR(tool_db_event_fops,
1143		      NULL,
1144		      tool_db_event_write);
1145
1146/*==============================================================================
1147 *                       Scratchpads read/write methods
1148 *==============================================================================
1149 */
1150
1151static ssize_t tool_spad_read(struct file *filep, char __user *ubuf,
1152			      size_t size, loff_t *offp)
1153{
1154	struct tool_spad *spad = filep->private_data;
1155	char buf[TOOL_BUF_LEN];
1156	ssize_t pos;
1157
1158	if (!spad->tc->ntb->ops->spad_read)
1159		return -EINVAL;
1160
1161	pos = scnprintf(buf, sizeof(buf), "%#x\n",
1162		ntb_spad_read(spad->tc->ntb, spad->sidx));
1163
1164	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1165}
1166
1167static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf,
1168			       size_t size, loff_t *offp)
1169{
1170	struct tool_spad *spad = filep->private_data;
1171	u32 val;
1172	int ret;
1173
1174	if (!spad->tc->ntb->ops->spad_write) {
1175		dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
1176		return -EINVAL;
1177	}
1178
1179	ret = kstrtou32_from_user(ubuf, size, 0, &val);
1180	if (ret)
1181		return ret;
1182
1183	ret = ntb_spad_write(spad->tc->ntb, spad->sidx, val);
1184
1185	return ret ?: size;
1186}
1187
1188static TOOL_FOPS_RDWR(tool_spad_fops,
1189		      tool_spad_read,
1190		      tool_spad_write);
1191
1192static ssize_t tool_peer_spad_read(struct file *filep, char __user *ubuf,
1193				   size_t size, loff_t *offp)
1194{
1195	struct tool_spad *spad = filep->private_data;
1196	char buf[TOOL_BUF_LEN];
1197	ssize_t pos;
1198
1199	if (!spad->tc->ntb->ops->peer_spad_read)
1200		return -EINVAL;
1201
1202	pos = scnprintf(buf, sizeof(buf), "%#x\n",
1203		ntb_peer_spad_read(spad->tc->ntb, spad->pidx, spad->sidx));
1204
1205	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1206}
1207
1208static ssize_t tool_peer_spad_write(struct file *filep, const char __user *ubuf,
1209				    size_t size, loff_t *offp)
1210{
1211	struct tool_spad *spad = filep->private_data;
1212	u32 val;
1213	int ret;
1214
1215	if (!spad->tc->ntb->ops->peer_spad_write) {
1216		dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
1217		return -EINVAL;
1218	}
1219
1220	ret = kstrtou32_from_user(ubuf, size, 0, &val);
1221	if (ret)
1222		return ret;
1223
1224	ret = ntb_peer_spad_write(spad->tc->ntb, spad->pidx, spad->sidx, val);
1225
1226	return ret ?: size;
1227}
1228
1229static TOOL_FOPS_RDWR(tool_peer_spad_fops,
1230		      tool_peer_spad_read,
1231		      tool_peer_spad_write);
1232
1233static int tool_init_spads(struct tool_ctx *tc)
1234{
1235	int sidx, pidx;
1236
1237	/* Initialize inbound scratchpad structures */
1238	tc->inspad_cnt = ntb_spad_count(tc->ntb);
1239	tc->inspads = devm_kcalloc(&tc->ntb->dev, tc->inspad_cnt,
1240				   sizeof(*tc->inspads), GFP_KERNEL);
1241	if (tc->inspads == NULL)
1242		return -ENOMEM;
1243
1244	for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
1245		tc->inspads[sidx].sidx = sidx;
1246		tc->inspads[sidx].pidx = -1;
1247		tc->inspads[sidx].tc = tc;
1248	}
1249
1250	/* Initialize outbound scratchpad structures */
1251	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1252		tc->peers[pidx].outspad_cnt = ntb_spad_count(tc->ntb);
1253		tc->peers[pidx].outspads =
1254			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outspad_cnt,
1255				sizeof(*tc->peers[pidx].outspads), GFP_KERNEL);
1256		if (tc->peers[pidx].outspads == NULL)
1257			return -ENOMEM;
1258
1259		for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
1260			tc->peers[pidx].outspads[sidx].sidx = sidx;
1261			tc->peers[pidx].outspads[sidx].pidx = pidx;
1262			tc->peers[pidx].outspads[sidx].tc = tc;
1263		}
1264	}
1265
1266	return 0;
1267}
1268
1269/*==============================================================================
1270 *                       Messages read/write methods
1271 *==============================================================================
1272 */
1273
1274static ssize_t tool_inmsg_read(struct file *filep, char __user *ubuf,
1275			       size_t size, loff_t *offp)
1276{
1277	struct tool_msg *msg = filep->private_data;
1278	char buf[TOOL_BUF_LEN];
1279	ssize_t pos;
1280	u32 data;
1281	int pidx;
1282
1283	data = ntb_msg_read(msg->tc->ntb, &pidx, msg->midx);
1284
1285	pos = scnprintf(buf, sizeof(buf), "0x%08x<-%d\n", data, pidx);
1286
1287	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1288}
1289
1290static TOOL_FOPS_RDWR(tool_inmsg_fops,
1291		      tool_inmsg_read,
1292		      NULL);
1293
1294static ssize_t tool_outmsg_write(struct file *filep,
1295				 const char __user *ubuf,
1296				 size_t size, loff_t *offp)
1297{
1298	struct tool_msg *msg = filep->private_data;
1299	u32 val;
1300	int ret;
1301
1302	ret = kstrtou32_from_user(ubuf, size, 0, &val);
1303	if (ret)
1304		return ret;
1305
1306	ret = ntb_peer_msg_write(msg->tc->ntb, msg->pidx, msg->midx, val);
1307
1308	return ret ? : size;
1309}
1310
1311static TOOL_FOPS_RDWR(tool_outmsg_fops,
1312		      NULL,
1313		      tool_outmsg_write);
1314
1315static ssize_t tool_msg_sts_read(struct file *filep, char __user *ubuf,
1316				 size_t size, loff_t *offp)
1317{
1318	struct tool_ctx *tc = filep->private_data;
1319
1320	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_read_sts);
1321}
1322
1323static ssize_t tool_msg_sts_write(struct file *filep, const char __user *ubuf,
1324				  size_t size, loff_t *offp)
1325{
1326	struct tool_ctx *tc = filep->private_data;
1327
1328	return tool_fn_write(tc, ubuf, size, offp, NULL,
1329			     tc->ntb->ops->msg_clear_sts);
1330}
1331
1332static TOOL_FOPS_RDWR(tool_msg_sts_fops,
1333		      tool_msg_sts_read,
1334		      tool_msg_sts_write);
1335
1336static ssize_t tool_msg_inbits_read(struct file *filep, char __user *ubuf,
1337				    size_t size, loff_t *offp)
1338{
1339	struct tool_ctx *tc = filep->private_data;
1340
1341	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_inbits);
1342}
1343
1344static TOOL_FOPS_RDWR(tool_msg_inbits_fops,
1345		      tool_msg_inbits_read,
1346		      NULL);
1347
1348static ssize_t tool_msg_outbits_read(struct file *filep, char __user *ubuf,
1349				     size_t size, loff_t *offp)
1350{
1351	struct tool_ctx *tc = filep->private_data;
1352
1353	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_outbits);
1354}
1355
1356static TOOL_FOPS_RDWR(tool_msg_outbits_fops,
1357		      tool_msg_outbits_read,
1358		      NULL);
1359
1360static ssize_t tool_msg_mask_write(struct file *filep, const char __user *ubuf,
1361				   size_t size, loff_t *offp)
1362{
1363	struct tool_ctx *tc = filep->private_data;
1364
1365	return tool_fn_write(tc, ubuf, size, offp,
1366			     tc->ntb->ops->msg_set_mask,
1367			     tc->ntb->ops->msg_clear_mask);
1368}
1369
1370static TOOL_FOPS_RDWR(tool_msg_mask_fops,
1371		      NULL,
1372		      tool_msg_mask_write);
1373
1374static ssize_t tool_msg_event_write(struct file *filep,
1375				    const char __user *ubuf,
1376				    size_t size, loff_t *offp)
1377{
1378	struct tool_ctx *tc = filep->private_data;
1379	u64 val;
1380	int ret;
1381
1382	ret = kstrtou64_from_user(ubuf, size, 0, &val);
1383	if (ret)
1384		return ret;
1385
1386	if (wait_event_interruptible(tc->msg_wq,
1387		ntb_msg_read_sts(tc->ntb) == val))
1388		return -ERESTART;
1389
1390	return size;
1391}
1392
1393static TOOL_FOPS_RDWR(tool_msg_event_fops,
1394		      NULL,
1395		      tool_msg_event_write);
1396
1397static int tool_init_msgs(struct tool_ctx *tc)
1398{
1399	int midx, pidx;
1400
1401	/* Initialize inbound message structures */
1402	tc->inmsg_cnt = ntb_msg_count(tc->ntb);
1403	tc->inmsgs = devm_kcalloc(&tc->ntb->dev, tc->inmsg_cnt,
1404				   sizeof(*tc->inmsgs), GFP_KERNEL);
1405	if (tc->inmsgs == NULL)
1406		return -ENOMEM;
1407
1408	for (midx = 0; midx < tc->inmsg_cnt; midx++) {
1409		tc->inmsgs[midx].midx = midx;
1410		tc->inmsgs[midx].pidx = -1;
1411		tc->inmsgs[midx].tc = tc;
1412	}
1413
1414	/* Initialize outbound message structures */
1415	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1416		tc->peers[pidx].outmsg_cnt = ntb_msg_count(tc->ntb);
1417		tc->peers[pidx].outmsgs =
1418			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmsg_cnt,
1419				sizeof(*tc->peers[pidx].outmsgs), GFP_KERNEL);
1420		if (tc->peers[pidx].outmsgs == NULL)
1421			return -ENOMEM;
1422
1423		for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
1424			tc->peers[pidx].outmsgs[midx].midx = midx;
1425			tc->peers[pidx].outmsgs[midx].pidx = pidx;
1426			tc->peers[pidx].outmsgs[midx].tc = tc;
1427		}
1428	}
1429
1430	return 0;
1431}
1432
1433/*==============================================================================
1434 *                          Initialization methods
1435 *==============================================================================
1436 */
1437
1438static struct tool_ctx *tool_create_data(struct ntb_dev *ntb)
1439{
1440	struct tool_ctx *tc;
1441
1442	tc = devm_kzalloc(&ntb->dev, sizeof(*tc), GFP_KERNEL);
1443	if (tc == NULL)
1444		return ERR_PTR(-ENOMEM);
1445
1446	tc->ntb = ntb;
1447	init_waitqueue_head(&tc->link_wq);
1448	init_waitqueue_head(&tc->db_wq);
1449	init_waitqueue_head(&tc->msg_wq);
1450
1451	if (ntb_db_is_unsafe(ntb))
1452		dev_dbg(&ntb->dev, "doorbell is unsafe\n");
1453
1454	if (ntb_spad_is_unsafe(ntb))
1455		dev_dbg(&ntb->dev, "scratchpad is unsafe\n");
1456
1457	return tc;
1458}
1459
1460static void tool_clear_data(struct tool_ctx *tc)
1461{
1462	wake_up(&tc->link_wq);
1463	wake_up(&tc->db_wq);
1464	wake_up(&tc->msg_wq);
1465}
1466
1467static int tool_init_ntb(struct tool_ctx *tc)
1468{
1469	return ntb_set_ctx(tc->ntb, tc, &tool_ops);
1470}
1471
1472static void tool_clear_ntb(struct tool_ctx *tc)
1473{
1474	ntb_clear_ctx(tc->ntb);
1475	ntb_link_disable(tc->ntb);
1476}
1477
1478static void tool_setup_dbgfs(struct tool_ctx *tc)
1479{
1480	int pidx, widx, sidx, midx;
1481	char buf[TOOL_BUF_LEN];
1482
1483	/* This modules is useless without dbgfs... */
1484	if (!tool_dbgfs_topdir) {
1485		tc->dbgfs_dir = NULL;
1486		return;
1487	}
1488
1489	tc->dbgfs_dir = debugfs_create_dir(dev_name(&tc->ntb->dev),
1490					   tool_dbgfs_topdir);
1491
1492	debugfs_create_file("port", 0600, tc->dbgfs_dir,
1493			    tc, &tool_port_fops);
1494
1495	debugfs_create_file("link", 0600, tc->dbgfs_dir,
1496			    tc, &tool_link_fops);
1497
1498	debugfs_create_file("db", 0600, tc->dbgfs_dir,
1499			    tc, &tool_db_fops);
1500
1501	debugfs_create_file("db_valid_mask", 0600, tc->dbgfs_dir,
1502			    tc, &tool_db_valid_mask_fops);
1503
1504	debugfs_create_file("db_mask", 0600, tc->dbgfs_dir,
1505			    tc, &tool_db_mask_fops);
1506
1507	debugfs_create_file("db_event", 0600, tc->dbgfs_dir,
1508			    tc, &tool_db_event_fops);
1509
1510	debugfs_create_file("peer_db", 0600, tc->dbgfs_dir,
1511			    tc, &tool_peer_db_fops);
1512
1513	debugfs_create_file("peer_db_mask", 0600, tc->dbgfs_dir,
1514			    tc, &tool_peer_db_mask_fops);
1515
1516	if (tc->inspad_cnt != 0) {
1517		for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
1518			snprintf(buf, sizeof(buf), "spad%d", sidx);
1519
1520			debugfs_create_file(buf, 0600, tc->dbgfs_dir,
1521					   &tc->inspads[sidx], &tool_spad_fops);
1522		}
1523	}
1524
1525	if (tc->inmsg_cnt != 0) {
1526		for (midx = 0; midx < tc->inmsg_cnt; midx++) {
1527			snprintf(buf, sizeof(buf), "msg%d", midx);
1528			debugfs_create_file(buf, 0600, tc->dbgfs_dir,
1529					   &tc->inmsgs[midx], &tool_inmsg_fops);
1530		}
1531
1532		debugfs_create_file("msg_sts", 0600, tc->dbgfs_dir,
1533				    tc, &tool_msg_sts_fops);
1534
1535		debugfs_create_file("msg_inbits", 0600, tc->dbgfs_dir,
1536				    tc, &tool_msg_inbits_fops);
1537
1538		debugfs_create_file("msg_outbits", 0600, tc->dbgfs_dir,
1539				    tc, &tool_msg_outbits_fops);
1540
1541		debugfs_create_file("msg_mask", 0600, tc->dbgfs_dir,
1542				    tc, &tool_msg_mask_fops);
1543
1544		debugfs_create_file("msg_event", 0600, tc->dbgfs_dir,
1545				    tc, &tool_msg_event_fops);
1546	}
1547
1548	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1549		snprintf(buf, sizeof(buf), "peer%d", pidx);
1550		tc->peers[pidx].dbgfs_dir =
1551			debugfs_create_dir(buf, tc->dbgfs_dir);
1552
1553		debugfs_create_file("port", 0600,
1554				    tc->peers[pidx].dbgfs_dir,
1555				    &tc->peers[pidx], &tool_peer_port_fops);
1556
1557		debugfs_create_file("link", 0200,
1558				    tc->peers[pidx].dbgfs_dir,
1559				    &tc->peers[pidx], &tool_peer_link_fops);
1560
1561		debugfs_create_file("link_event", 0200,
1562				  tc->peers[pidx].dbgfs_dir,
1563				  &tc->peers[pidx], &tool_peer_link_event_fops);
1564
1565		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
1566			snprintf(buf, sizeof(buf), "mw_trans%d", widx);
1567			debugfs_create_file(buf, 0600,
1568					    tc->peers[pidx].dbgfs_dir,
1569					    &tc->peers[pidx].inmws[widx],
1570					    &tool_mw_trans_fops);
1571		}
1572
1573		for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
1574			snprintf(buf, sizeof(buf), "peer_mw_trans%d", widx);
1575			debugfs_create_file(buf, 0600,
1576					    tc->peers[pidx].dbgfs_dir,
1577					    &tc->peers[pidx].outmws[widx],
1578					    &tool_peer_mw_trans_fops);
1579		}
1580
1581		for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
1582			snprintf(buf, sizeof(buf), "spad%d", sidx);
1583
1584			debugfs_create_file(buf, 0600,
1585					    tc->peers[pidx].dbgfs_dir,
1586					    &tc->peers[pidx].outspads[sidx],
1587					    &tool_peer_spad_fops);
1588		}
1589
1590		for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
1591			snprintf(buf, sizeof(buf), "msg%d", midx);
1592			debugfs_create_file(buf, 0600,
1593					    tc->peers[pidx].dbgfs_dir,
1594					    &tc->peers[pidx].outmsgs[midx],
1595					    &tool_outmsg_fops);
1596		}
1597	}
1598}
1599
1600static void tool_clear_dbgfs(struct tool_ctx *tc)
1601{
1602	debugfs_remove_recursive(tc->dbgfs_dir);
1603}
1604
1605static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
1606{
1607	struct tool_ctx *tc;
1608	int ret;
1609
1610	tc = tool_create_data(ntb);
1611	if (IS_ERR(tc))
1612		return PTR_ERR(tc);
1613
1614	ret = tool_init_peers(tc);
1615	if (ret != 0)
1616		goto err_clear_data;
1617
1618	ret = tool_init_mws(tc);
1619	if (ret != 0)
1620		goto err_clear_data;
1621
1622	ret = tool_init_spads(tc);
1623	if (ret != 0)
1624		goto err_clear_mws;
1625
1626	ret = tool_init_msgs(tc);
1627	if (ret != 0)
1628		goto err_clear_mws;
1629
1630	ret = tool_init_ntb(tc);
1631	if (ret != 0)
1632		goto err_clear_mws;
1633
1634	tool_setup_dbgfs(tc);
1635
1636	return 0;
1637
1638err_clear_mws:
1639	tool_clear_mws(tc);
1640
1641err_clear_data:
1642	tool_clear_data(tc);
1643
1644	return ret;
1645}
1646
1647static void tool_remove(struct ntb_client *self, struct ntb_dev *ntb)
1648{
1649	struct tool_ctx *tc = ntb->ctx;
1650
1651	tool_clear_dbgfs(tc);
1652
1653	tool_clear_ntb(tc);
1654
1655	tool_clear_mws(tc);
1656
1657	tool_clear_data(tc);
1658}
1659
1660static struct ntb_client tool_client = {
1661	.ops = {
1662		.probe = tool_probe,
1663		.remove = tool_remove,
1664	}
1665};
1666
1667static int __init tool_init(void)
1668{
1669	int ret;
1670
1671	if (debugfs_initialized())
1672		tool_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1673
1674	ret = ntb_register_client(&tool_client);
1675	if (ret)
1676		debugfs_remove_recursive(tool_dbgfs_topdir);
1677
1678	return ret;
1679}
1680module_init(tool_init);
1681
1682static void __exit tool_exit(void)
1683{
1684	ntb_unregister_client(&tool_client);
1685	debugfs_remove_recursive(tool_dbgfs_topdir);
1686}
1687module_exit(tool_exit);