1/*
   2 * This file is provided under a dual BSD/GPLv2 license.  When using or
   3 *   redistributing this file, you may do so under either license.
   4 *
   5 *   GPL LICENSE SUMMARY
   6 *
   7 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
   8 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
   9 *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  10 *
  11 *   This program is free software; you can redistribute it and/or modify
  12 *   it under the terms of version 2 of the GNU General Public License as
  13 *   published by the Free Software Foundation.
  14 *
  15 *   BSD LICENSE
  16 *
  17 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  18 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  19 *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  20 *
  21 *   Redistribution and use in source and binary forms, with or without
  22 *   modification, are permitted provided that the following conditions
  23 *   are met:
  24 *
  25 *     * Redistributions of source code must retain the above copyright
  26 *       notice, this list of conditions and the following disclaimer.
  27 *     * Redistributions in binary form must reproduce the above copy
  28 *       notice, this list of conditions and the following disclaimer in
  29 *       the documentation and/or other materials provided with the
  30 *       distribution.
  31 *     * Neither the name of Intel Corporation nor the names of its
  32 *       contributors may be used to endorse or promote products derived
  33 *       from this software without specific prior written permission.
  34 *
  35 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 *
  47 * Intel PCIe NTB Linux driver
  48 */
  49
  50#include <linux/debugfs.h>
  51#include <linux/delay.h>
  52#include <linux/init.h>
  53#include <linux/interrupt.h>
  54#include <linux/module.h>
  55#include <linux/pci.h>
  56#include <linux/random.h>
  57#include <linux/slab.h>
  58#include <linux/ntb.h>
  59
  60#include "ntb_hw_intel.h"
  61#include "ntb_hw_gen1.h"
  62#include "ntb_hw_gen3.h"
  63#include "ntb_hw_gen4.h"
  64
  65#define NTB_NAME	"ntb_hw_intel"
  66#define NTB_DESC	"Intel(R) PCI-E Non-Transparent Bridge Driver"
  67#define NTB_VER		"2.0"
  68
  69MODULE_DESCRIPTION(NTB_DESC);
  70MODULE_VERSION(NTB_VER);
  71MODULE_LICENSE("Dual BSD/GPL");
  72MODULE_AUTHOR("Intel Corporation");
  73
  74#define bar0_off(base, bar) ((base) + ((bar) << 2))
  75#define bar2_off(base, bar) bar0_off(base, (bar) - 2)
  76
  77static const struct intel_ntb_reg xeon_reg;
  78static const struct intel_ntb_alt_reg xeon_pri_reg;
  79static const struct intel_ntb_alt_reg xeon_sec_reg;
  80static const struct intel_ntb_alt_reg xeon_b2b_reg;
  81static const struct intel_ntb_xlat_reg xeon_pri_xlat;
  82static const struct intel_ntb_xlat_reg xeon_sec_xlat;
  83static const struct ntb_dev_ops intel_ntb_ops;
  84
  85static const struct file_operations intel_ntb_debugfs_info;
  86static struct dentry *debugfs_dir;
  87
  88static int b2b_mw_idx = -1;
  89module_param(b2b_mw_idx, int, 0644);
  90MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb.  A "
  91		 "value of zero or positive starts from first mw idx, and a "
  92		 "negative value starts from last mw idx.  Both sides MUST "
  93		 "set the same value here!");
  94
  95static unsigned int b2b_mw_share;
  96module_param(b2b_mw_share, uint, 0644);
  97MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
  98		 "ntb so that the peer ntb only occupies the first half of "
  99		 "the mw, so the second half can still be used as a mw.  Both "
 100		 "sides MUST set the same value here!");
 101
 102module_param_named(xeon_b2b_usd_bar2_addr64,
 103		   xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
 104MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
 105		 "XEON B2B USD BAR 2 64-bit address");
 106
 107module_param_named(xeon_b2b_usd_bar4_addr64,
 108		   xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
 109MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
 110		 "XEON B2B USD BAR 4 64-bit address");
 111
 112module_param_named(xeon_b2b_usd_bar4_addr32,
 113		   xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
 114MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
 115		 "XEON B2B USD split-BAR 4 32-bit address");
 116
 117module_param_named(xeon_b2b_usd_bar5_addr32,
 118		   xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
 119MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
 120		 "XEON B2B USD split-BAR 5 32-bit address");
 121
 122module_param_named(xeon_b2b_dsd_bar2_addr64,
 123		   xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
 124MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
 125		 "XEON B2B DSD BAR 2 64-bit address");
 126
 127module_param_named(xeon_b2b_dsd_bar4_addr64,
 128		   xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
 129MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
 130		 "XEON B2B DSD BAR 4 64-bit address");
 131
 132module_param_named(xeon_b2b_dsd_bar4_addr32,
 133		   xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
 134MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
 135		 "XEON B2B DSD split-BAR 4 32-bit address");
 136
 137module_param_named(xeon_b2b_dsd_bar5_addr32,
 138		   xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
 139MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
 140		 "XEON B2B DSD split-BAR 5 32-bit address");
 141
 142
 143static int xeon_init_isr(struct intel_ntb_dev *ndev);
 144
 145static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
 146{
 147	ndev->unsafe_flags = 0;
 148	ndev->unsafe_flags_ignore = 0;
 149
 150	/* Only B2B has a workaround to avoid SDOORBELL */
 151	if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
 152		if (!ntb_topo_is_b2b(ndev->ntb.topo))
 153			ndev->unsafe_flags |= NTB_UNSAFE_DB;
 154
 155	/* No low level workaround to avoid SB01BASE */
 156	if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
 157		ndev->unsafe_flags |= NTB_UNSAFE_DB;
 158		ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
 159	}
 160}
 161
 162static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
 163				 unsigned long flag)
 164{
 165	return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
 166}
 167
 168static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
 169				     unsigned long flag)
 170{
 171	flag &= ndev->unsafe_flags;
 172	ndev->unsafe_flags_ignore |= flag;
 173
 174	return !!flag;
 175}
 176
 177int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
 178{
 179	if (idx < 0 || idx >= ndev->mw_count)
 180		return -EINVAL;
 181	return ndev->reg->mw_bar[idx];
 182}
 183
 184void ndev_db_addr(struct intel_ntb_dev *ndev,
 185			       phys_addr_t *db_addr, resource_size_t *db_size,
 186			       phys_addr_t reg_addr, unsigned long reg)
 187{
 188	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 189		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 190
 191	if (db_addr) {
 192		*db_addr = reg_addr + reg;
 193		dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
 194	}
 195
 196	if (db_size) {
 197		*db_size = ndev->reg->db_size;
 198		dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
 199	}
 200}
 201
 202u64 ndev_db_read(struct intel_ntb_dev *ndev,
 203			       void __iomem *mmio)
 204{
 205	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 206		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 207
 208	return ndev->reg->db_ioread(mmio);
 209}
 210
 211int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
 212				void __iomem *mmio)
 213{
 214	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 215		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 216
 217	if (db_bits & ~ndev->db_valid_mask)
 218		return -EINVAL;
 219
 220	ndev->reg->db_iowrite(db_bits, mmio);
 221
 222	return 0;
 223}
 224
 225static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 226				   void __iomem *mmio)
 227{
 228	unsigned long irqflags;
 229
 230	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 231		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 232
 233	if (db_bits & ~ndev->db_valid_mask)
 234		return -EINVAL;
 235
 236	spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 237	{
 238		ndev->db_mask |= db_bits;
 239		ndev->reg->db_iowrite(ndev->db_mask, mmio);
 240	}
 241	spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 242
 243	return 0;
 244}
 245
 246static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 247				     void __iomem *mmio)
 248{
 249	unsigned long irqflags;
 250
 251	if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 252		pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 253
 254	if (db_bits & ~ndev->db_valid_mask)
 255		return -EINVAL;
 256
 257	spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 258	{
 259		ndev->db_mask &= ~db_bits;
 260		ndev->reg->db_iowrite(ndev->db_mask, mmio);
 261	}
 262	spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 263
 264	return 0;
 265}
 266
 267static inline u64 ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
 268{
 269	u64 shift, mask;
 270
 271	shift = ndev->db_vec_shift;
 272	mask = BIT_ULL(shift) - 1;
 273
 274	return mask << (shift * db_vector);
 275}
 276
 277static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
 278				 phys_addr_t *spad_addr, phys_addr_t reg_addr,
 279				 unsigned long reg)
 280{
 281	if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 282		pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 283
 284	if (idx < 0 || idx >= ndev->spad_count)
 285		return -EINVAL;
 286
 287	if (spad_addr) {
 288		*spad_addr = reg_addr + reg + (idx << 2);
 289		dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
 290			*spad_addr);
 291	}
 292
 293	return 0;
 294}
 295
 296static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
 297				 void __iomem *mmio)
 298{
 299	if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 300		pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 301
 302	if (idx < 0 || idx >= ndev->spad_count)
 303		return 0;
 304
 305	return ioread32(mmio + (idx << 2));
 306}
 307
 308static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
 309				  void __iomem *mmio)
 310{
 311	if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 312		pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 313
 314	if (idx < 0 || idx >= ndev->spad_count)
 315		return -EINVAL;
 316
 317	iowrite32(val, mmio + (idx << 2));
 318
 319	return 0;
 320}
 321
 322static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
 323{
 324	u64 vec_mask;
 325
 326	vec_mask = ndev_vec_mask(ndev, vec);
 327
 328	if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
 329		vec_mask |= ndev->db_link_mask;
 330
 331	dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
 332
 333	ndev->last_ts = jiffies;
 334
 335	if (vec_mask & ndev->db_link_mask) {
 336		if (ndev->reg->poll_link(ndev))
 337			ntb_link_event(&ndev->ntb);
 338	}
 339
 340	if (vec_mask & ndev->db_valid_mask)
 341		ntb_db_event(&ndev->ntb, vec);
 342
 343	return IRQ_HANDLED;
 344}
 345
 346static irqreturn_t ndev_vec_isr(int irq, void *dev)
 347{
 348	struct intel_ntb_vec *nvec = dev;
 349
 350	dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d  nvec->num: %d\n",
 351		irq, nvec->num);
 352
 353	return ndev_interrupt(nvec->ndev, nvec->num);
 354}
 355
 356static irqreturn_t ndev_irq_isr(int irq, void *dev)
 357{
 358	struct intel_ntb_dev *ndev = dev;
 359
 360	return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
 361}
 362
 363int ndev_init_isr(struct intel_ntb_dev *ndev,
 364			 int msix_min, int msix_max,
 365			 int msix_shift, int total_shift)
 366{
 367	struct pci_dev *pdev;
 368	int rc, i, msix_count, node;
 369
 370	pdev = ndev->ntb.pdev;
 371
 372	node = dev_to_node(&pdev->dev);
 373
 374	/* Mask all doorbell interrupts */
 375	ndev->db_mask = ndev->db_valid_mask;
 376	ndev->reg->db_iowrite(ndev->db_mask,
 377			      ndev->self_mmio +
 378			      ndev->self_reg->db_mask);
 379
 380	/* Try to set up msix irq */
 381
 382	ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
 383				 GFP_KERNEL, node);
 384	if (!ndev->vec)
 385		goto err_msix_vec_alloc;
 386
 387	ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
 388				  GFP_KERNEL, node);
 389	if (!ndev->msix)
 390		goto err_msix_alloc;
 391
 392	for (i = 0; i < msix_max; ++i)
 393		ndev->msix[i].entry = i;
 394
 395	msix_count = pci_enable_msix_range(pdev, ndev->msix,
 396					   msix_min, msix_max);
 397	if (msix_count < 0)
 398		goto err_msix_enable;
 399
 400	for (i = 0; i < msix_count; ++i) {
 401		ndev->vec[i].ndev = ndev;
 402		ndev->vec[i].num = i;
 403		rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
 404				 "ndev_vec_isr", &ndev->vec[i]);
 405		if (rc)
 406			goto err_msix_request;
 407	}
 408
 409	dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
 410	ndev->db_vec_count = msix_count;
 411	ndev->db_vec_shift = msix_shift;
 412	return 0;
 413
 414err_msix_request:
 415	while (i-- > 0)
 416		free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 417	pci_disable_msix(pdev);
 418err_msix_enable:
 419	kfree(ndev->msix);
 420err_msix_alloc:
 421	kfree(ndev->vec);
 422err_msix_vec_alloc:
 423	ndev->msix = NULL;
 424	ndev->vec = NULL;
 425
 426	/* Try to set up msi irq */
 427
 428	rc = pci_enable_msi(pdev);
 429	if (rc)
 430		goto err_msi_enable;
 431
 432	rc = request_irq(pdev->irq, ndev_irq_isr, 0,
 433			 "ndev_irq_isr", ndev);
 434	if (rc)
 435		goto err_msi_request;
 436
 437	dev_dbg(&pdev->dev, "Using msi interrupts\n");
 438	ndev->db_vec_count = 1;
 439	ndev->db_vec_shift = total_shift;
 440	return 0;
 441
 442err_msi_request:
 443	pci_disable_msi(pdev);
 444err_msi_enable:
 445
 446	/* Try to set up intx irq */
 447
 448	pci_intx(pdev, 1);
 449
 450	rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
 451			 "ndev_irq_isr", ndev);
 452	if (rc)
 453		goto err_intx_request;
 454
 455	dev_dbg(&pdev->dev, "Using intx interrupts\n");
 456	ndev->db_vec_count = 1;
 457	ndev->db_vec_shift = total_shift;
 458	return 0;
 459
 460err_intx_request:
 461	return rc;
 462}
 463
 464static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
 465{
 466	struct pci_dev *pdev;
 467	int i;
 468
 469	pdev = ndev->ntb.pdev;
 470
 471	/* Mask all doorbell interrupts */
 472	ndev->db_mask = ndev->db_valid_mask;
 473	ndev->reg->db_iowrite(ndev->db_mask,
 474			      ndev->self_mmio +
 475			      ndev->self_reg->db_mask);
 476
 477	if (ndev->msix) {
 478		i = ndev->db_vec_count;
 479		while (i--)
 480			free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 481		pci_disable_msix(pdev);
 482		kfree(ndev->msix);
 483		kfree(ndev->vec);
 484	} else {
 485		free_irq(pdev->irq, ndev);
 486		if (pci_dev_msi_enabled(pdev))
 487			pci_disable_msi(pdev);
 488	}
 489}
 490
 491static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
 492				     size_t count, loff_t *offp)
 493{
 494	struct intel_ntb_dev *ndev;
 495	struct pci_dev *pdev;
 496	void __iomem *mmio;
 497	char *buf;
 498	size_t buf_size;
 499	ssize_t ret, off;
 500	union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
 501
 502	ndev = filp->private_data;
 503	pdev = ndev->ntb.pdev;
 504	mmio = ndev->self_mmio;
 505
 506	buf_size = min(count, 0x800ul);
 507
 508	buf = kmalloc(buf_size, GFP_KERNEL);
 509	if (!buf)
 510		return -ENOMEM;
 511
 512	off = 0;
 513
 514	off += scnprintf(buf + off, buf_size - off,
 515			 "NTB Device Information:\n");
 516
 517	off += scnprintf(buf + off, buf_size - off,
 518			 "Connection Topology -\t%s\n",
 519			 ntb_topo_string(ndev->ntb.topo));
 520
 521	if (ndev->b2b_idx != UINT_MAX) {
 522		off += scnprintf(buf + off, buf_size - off,
 523				 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
 524		off += scnprintf(buf + off, buf_size - off,
 525				 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
 526	}
 527
 528	off += scnprintf(buf + off, buf_size - off,
 529			 "BAR4 Split -\t\t%s\n",
 530			 ndev->bar4_split ? "yes" : "no");
 531
 532	off += scnprintf(buf + off, buf_size - off,
 533			 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
 534	off += scnprintf(buf + off, buf_size - off,
 535			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
 536
 537	if (!ndev->reg->link_is_up(ndev)) {
 538		off += scnprintf(buf + off, buf_size - off,
 539				 "Link Status -\t\tDown\n");
 540	} else {
 541		off += scnprintf(buf + off, buf_size - off,
 542				 "Link Status -\t\tUp\n");
 543		off += scnprintf(buf + off, buf_size - off,
 544				 "Link Speed -\t\tPCI-E Gen %u\n",
 545				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
 546		off += scnprintf(buf + off, buf_size - off,
 547				 "Link Width -\t\tx%u\n",
 548				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
 549	}
 550
 551	off += scnprintf(buf + off, buf_size - off,
 552			 "Memory Window Count -\t%u\n", ndev->mw_count);
 553	off += scnprintf(buf + off, buf_size - off,
 554			 "Scratchpad Count -\t%u\n", ndev->spad_count);
 555	off += scnprintf(buf + off, buf_size - off,
 556			 "Doorbell Count -\t%u\n", ndev->db_count);
 557	off += scnprintf(buf + off, buf_size - off,
 558			 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
 559	off += scnprintf(buf + off, buf_size - off,
 560			 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
 561
 562	off += scnprintf(buf + off, buf_size - off,
 563			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
 564	off += scnprintf(buf + off, buf_size - off,
 565			 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
 566	off += scnprintf(buf + off, buf_size - off,
 567			 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
 568
 569	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
 570	off += scnprintf(buf + off, buf_size - off,
 571			 "Doorbell Mask -\t\t%#llx\n", u.v64);
 572
 573	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
 574	off += scnprintf(buf + off, buf_size - off,
 575			 "Doorbell Bell -\t\t%#llx\n", u.v64);
 576
 577	off += scnprintf(buf + off, buf_size - off,
 578			 "\nNTB Window Size:\n");
 579
 580	pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
 581	off += scnprintf(buf + off, buf_size - off,
 582			 "PBAR23SZ %hhu\n", u.v8);
 583	if (!ndev->bar4_split) {
 584		pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
 585		off += scnprintf(buf + off, buf_size - off,
 586				 "PBAR45SZ %hhu\n", u.v8);
 587	} else {
 588		pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
 589		off += scnprintf(buf + off, buf_size - off,
 590				 "PBAR4SZ %hhu\n", u.v8);
 591		pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
 592		off += scnprintf(buf + off, buf_size - off,
 593				 "PBAR5SZ %hhu\n", u.v8);
 594	}
 595
 596	pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
 597	off += scnprintf(buf + off, buf_size - off,
 598			 "SBAR23SZ %hhu\n", u.v8);
 599	if (!ndev->bar4_split) {
 600		pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
 601		off += scnprintf(buf + off, buf_size - off,
 602				 "SBAR45SZ %hhu\n", u.v8);
 603	} else {
 604		pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
 605		off += scnprintf(buf + off, buf_size - off,
 606				 "SBAR4SZ %hhu\n", u.v8);
 607		pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
 608		off += scnprintf(buf + off, buf_size - off,
 609				 "SBAR5SZ %hhu\n", u.v8);
 610	}
 611
 612	off += scnprintf(buf + off, buf_size - off,
 613			 "\nNTB Incoming XLAT:\n");
 614
 615	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
 616	off += scnprintf(buf + off, buf_size - off,
 617			 "XLAT23 -\t\t%#018llx\n", u.v64);
 618
 619	if (ndev->bar4_split) {
 620		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
 621		off += scnprintf(buf + off, buf_size - off,
 622				 "XLAT4 -\t\t\t%#06x\n", u.v32);
 623
 624		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
 625		off += scnprintf(buf + off, buf_size - off,
 626				 "XLAT5 -\t\t\t%#06x\n", u.v32);
 627	} else {
 628		u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
 629		off += scnprintf(buf + off, buf_size - off,
 630				 "XLAT45 -\t\t%#018llx\n", u.v64);
 631	}
 632
 633	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
 634	off += scnprintf(buf + off, buf_size - off,
 635			 "LMT23 -\t\t\t%#018llx\n", u.v64);
 636
 637	if (ndev->bar4_split) {
 638		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
 639		off += scnprintf(buf + off, buf_size - off,
 640				 "LMT4 -\t\t\t%#06x\n", u.v32);
 641		u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
 642		off += scnprintf(buf + off, buf_size - off,
 643				 "LMT5 -\t\t\t%#06x\n", u.v32);
 644	} else {
 645		u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
 646		off += scnprintf(buf + off, buf_size - off,
 647				 "LMT45 -\t\t\t%#018llx\n", u.v64);
 648	}
 649
 650	if (pdev_is_gen1(pdev)) {
 651		if (ntb_topo_is_b2b(ndev->ntb.topo)) {
 652			off += scnprintf(buf + off, buf_size - off,
 653					 "\nNTB Outgoing B2B XLAT:\n");
 654
 655			u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
 656			off += scnprintf(buf + off, buf_size - off,
 657					 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
 658
 659			if (ndev->bar4_split) {
 660				u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
 661				off += scnprintf(buf + off, buf_size - off,
 662						 "B2B XLAT4 -\t\t%#06x\n",
 663						 u.v32);
 664				u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
 665				off += scnprintf(buf + off, buf_size - off,
 666						 "B2B XLAT5 -\t\t%#06x\n",
 667						 u.v32);
 668			} else {
 669				u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
 670				off += scnprintf(buf + off, buf_size - off,
 671						 "B2B XLAT45 -\t\t%#018llx\n",
 672						 u.v64);
 673			}
 674
 675			u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
 676			off += scnprintf(buf + off, buf_size - off,
 677					 "B2B LMT23 -\t\t%#018llx\n", u.v64);
 678
 679			if (ndev->bar4_split) {
 680				u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
 681				off += scnprintf(buf + off, buf_size - off,
 682						 "B2B LMT4 -\t\t%#06x\n",
 683						 u.v32);
 684				u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
 685				off += scnprintf(buf + off, buf_size - off,
 686						 "B2B LMT5 -\t\t%#06x\n",
 687						 u.v32);
 688			} else {
 689				u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
 690				off += scnprintf(buf + off, buf_size - off,
 691						 "B2B LMT45 -\t\t%#018llx\n",
 692						 u.v64);
 693			}
 694
 695			off += scnprintf(buf + off, buf_size - off,
 696					 "\nNTB Secondary BAR:\n");
 697
 698			u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
 699			off += scnprintf(buf + off, buf_size - off,
 700					 "SBAR01 -\t\t%#018llx\n", u.v64);
 701
 702			u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
 703			off += scnprintf(buf + off, buf_size - off,
 704					 "SBAR23 -\t\t%#018llx\n", u.v64);
 705
 706			if (ndev->bar4_split) {
 707				u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
 708				off += scnprintf(buf + off, buf_size - off,
 709						 "SBAR4 -\t\t\t%#06x\n", u.v32);
 710				u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
 711				off += scnprintf(buf + off, buf_size - off,
 712						 "SBAR5 -\t\t\t%#06x\n", u.v32);
 713			} else {
 714				u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
 715				off += scnprintf(buf + off, buf_size - off,
 716						 "SBAR45 -\t\t%#018llx\n",
 717						 u.v64);
 718			}
 719		}
 720
 721		off += scnprintf(buf + off, buf_size - off,
 722				 "\nXEON NTB Statistics:\n");
 723
 724		u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
 725		off += scnprintf(buf + off, buf_size - off,
 726				 "Upstream Memory Miss -\t%u\n", u.v16);
 727
 728		off += scnprintf(buf + off, buf_size - off,
 729				 "\nXEON NTB Hardware Errors:\n");
 730
 731		if (!pci_read_config_word(pdev,
 732					  XEON_DEVSTS_OFFSET, &u.v16))
 733			off += scnprintf(buf + off, buf_size - off,
 734					 "DEVSTS -\t\t%#06x\n", u.v16);
 735
 736		if (!pci_read_config_word(pdev,
 737					  XEON_LINK_STATUS_OFFSET, &u.v16))
 738			off += scnprintf(buf + off, buf_size - off,
 739					 "LNKSTS -\t\t%#06x\n", u.v16);
 740
 741		if (!pci_read_config_dword(pdev,
 742					   XEON_UNCERRSTS_OFFSET, &u.v32))
 743			off += scnprintf(buf + off, buf_size - off,
 744					 "UNCERRSTS -\t\t%#06x\n", u.v32);
 745
 746		if (!pci_read_config_dword(pdev,
 747					   XEON_CORERRSTS_OFFSET, &u.v32))
 748			off += scnprintf(buf + off, buf_size - off,
 749					 "CORERRSTS -\t\t%#06x\n", u.v32);
 750	}
 751
 752	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
 753	kfree(buf);
 754	return ret;
 755}
 756
 757static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
 758				 size_t count, loff_t *offp)
 759{
 760	struct intel_ntb_dev *ndev = filp->private_data;
 761
 762	if (pdev_is_gen1(ndev->ntb.pdev))
 763		return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
 764	else if (pdev_is_gen3(ndev->ntb.pdev))
 765		return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
 766	else if (pdev_is_gen4(ndev->ntb.pdev) || pdev_is_gen5(ndev->ntb.pdev))
 767		return ndev_ntb4_debugfs_read(filp, ubuf, count, offp);
 768
 769	return -ENXIO;
 770}
 771
 772static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
 773{
 774	if (!debugfs_dir) {
 775		ndev->debugfs_dir = NULL;
 776		ndev->debugfs_info = NULL;
 777	} else {
 778		ndev->debugfs_dir =
 779			debugfs_create_dir(pci_name(ndev->ntb.pdev),
 780					   debugfs_dir);
 781		if (!ndev->debugfs_dir)
 782			ndev->debugfs_info = NULL;
 783		else
 784			ndev->debugfs_info =
 785				debugfs_create_file("info", S_IRUSR,
 786						    ndev->debugfs_dir, ndev,
 787						    &intel_ntb_debugfs_info);
 788	}
 789}
 790
 791static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
 792{
 793	debugfs_remove_recursive(ndev->debugfs_dir);
 794}
 795
 796int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 797{
 798	if (pidx != NTB_DEF_PEER_IDX)
 799		return -EINVAL;
 800
 801	return ntb_ndev(ntb)->mw_count;
 802}
 803
 804int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
 805			   resource_size_t *addr_align,
 806			   resource_size_t *size_align,
 807			   resource_size_t *size_max)
 808{
 809	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 810	resource_size_t bar_size, mw_size;
 811	int bar;
 812
 813	if (pidx != NTB_DEF_PEER_IDX)
 814		return -EINVAL;
 815
 816	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
 817		idx += 1;
 818
 819	bar = ndev_mw_to_bar(ndev, idx);
 820	if (bar < 0)
 821		return bar;
 822
 823	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
 824
 825	if (idx == ndev->b2b_idx)
 826		mw_size = bar_size - ndev->b2b_off;
 827	else
 828		mw_size = bar_size;
 829
 830	if (addr_align)
 831		*addr_align = pci_resource_len(ndev->ntb.pdev, bar);
 832
 833	if (size_align)
 834		*size_align = 1;
 835
 836	if (size_max)
 837		*size_max = mw_size;
 838
 839	return 0;
 840}
 841
 842static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
 843				  dma_addr_t addr, resource_size_t size)
 844{
 845	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 846	unsigned long base_reg, xlat_reg, limit_reg;
 847	resource_size_t bar_size, mw_size;
 848	void __iomem *mmio;
 849	u64 base, limit, reg_val;
 850	int bar;
 851
 852	if (pidx != NTB_DEF_PEER_IDX)
 853		return -EINVAL;
 854
 855	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
 856		idx += 1;
 857
 858	bar = ndev_mw_to_bar(ndev, idx);
 859	if (bar < 0)
 860		return bar;
 861
 862	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
 863
 864	if (idx == ndev->b2b_idx)
 865		mw_size = bar_size - ndev->b2b_off;
 866	else
 867		mw_size = bar_size;
 868
 869	/* hardware requires that addr is aligned to bar size */
 870	if (addr & (bar_size - 1))
 871		return -EINVAL;
 872
 873	/* make sure the range fits in the usable mw size */
 874	if (size > mw_size)
 875		return -EINVAL;
 876
 877	mmio = ndev->self_mmio;
 878	base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
 879	xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
 880	limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
 881
 882	if (bar < 4 || !ndev->bar4_split) {
 883		base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
 884
 885		/* Set the limit if supported, if size is not mw_size */
 886		if (limit_reg && size != mw_size)
 887			limit = base + size;
 888		else
 889			limit = 0;
 890
 891		/* set and verify setting the translation address */
 892		iowrite64(addr, mmio + xlat_reg);
 893		reg_val = ioread64(mmio + xlat_reg);
 894		if (reg_val != addr) {
 895			iowrite64(0, mmio + xlat_reg);
 896			return -EIO;
 897		}
 898
 899		/* set and verify setting the limit */
 900		iowrite64(limit, mmio + limit_reg);
 901		reg_val = ioread64(mmio + limit_reg);
 902		if (reg_val != limit) {
 903			iowrite64(base, mmio + limit_reg);
 904			iowrite64(0, mmio + xlat_reg);
 905			return -EIO;
 906		}
 907	} else {
 908		/* split bar addr range must all be 32 bit */
 909		if (addr & (~0ull << 32))
 910			return -EINVAL;
 911		if ((addr + size) & (~0ull << 32))
 912			return -EINVAL;
 913
 914		base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
 915
 916		/* Set the limit if supported, if size is not mw_size */
 917		if (limit_reg && size != mw_size)
 918			limit = base + size;
 919		else
 920			limit = 0;
 921
 922		/* set and verify setting the translation address */
 923		iowrite32(addr, mmio + xlat_reg);
 924		reg_val = ioread32(mmio + xlat_reg);
 925		if (reg_val != addr) {
 926			iowrite32(0, mmio + xlat_reg);
 927			return -EIO;
 928		}
 929
 930		/* set and verify setting the limit */
 931		iowrite32(limit, mmio + limit_reg);
 932		reg_val = ioread32(mmio + limit_reg);
 933		if (reg_val != limit) {
 934			iowrite32(base, mmio + limit_reg);
 935			iowrite32(0, mmio + xlat_reg);
 936			return -EIO;
 937		}
 938	}
 939
 940	return 0;
 941}
 942
 943u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed,
 944			 enum ntb_width *width)
 945{
 946	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 947
 948	if (ndev->reg->link_is_up(ndev)) {
 949		if (speed)
 950			*speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
 951		if (width)
 952			*width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
 953		return 1;
 954	} else {
 955		/* TODO MAYBE: is it possible to observe the link speed and
 956		 * width while link is training? */
 957		if (speed)
 958			*speed = NTB_SPEED_NONE;
 959		if (width)
 960			*width = NTB_WIDTH_NONE;
 961		return 0;
 962	}
 963}
 964
 965static int intel_ntb_link_enable(struct ntb_dev *ntb,
 966				 enum ntb_speed max_speed,
 967				 enum ntb_width max_width)
 968{
 969	struct intel_ntb_dev *ndev;
 970	u32 ntb_ctl;
 971
 972	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
 973
 974	if (ndev->ntb.topo == NTB_TOPO_SEC)
 975		return -EINVAL;
 976
 977	dev_dbg(&ntb->pdev->dev,
 978		"Enabling link with max_speed %d max_width %d\n",
 979		max_speed, max_width);
 980	if (max_speed != NTB_SPEED_AUTO)
 981		dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
 982	if (max_width != NTB_WIDTH_AUTO)
 983		dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
 984
 985	ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
 986	ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
 987	ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
 988	ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
 989	if (ndev->bar4_split)
 990		ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
 991	iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
 992
 993	return 0;
 994}
 995
 996int intel_ntb_link_disable(struct ntb_dev *ntb)
 997{
 998	struct intel_ntb_dev *ndev;
 999	u32 ntb_cntl;
1000
1001	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1002
1003	if (ndev->ntb.topo == NTB_TOPO_SEC)
1004		return -EINVAL;
1005
1006	dev_dbg(&ntb->pdev->dev, "Disabling link\n");
1007
1008	/* Bring NTB link down */
1009	ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1010	ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
1011	ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
1012	if (ndev->bar4_split)
1013		ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
1014	ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
1015	iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1016
1017	return 0;
1018}
1019
1020int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
1021{
1022	/* Numbers of inbound and outbound memory windows match */
1023	return ntb_ndev(ntb)->mw_count;
1024}
1025
1026int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
1027			       phys_addr_t *base, resource_size_t *size)
1028{
1029	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1030	int bar;
1031
1032	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1033		idx += 1;
1034
1035	bar = ndev_mw_to_bar(ndev, idx);
1036	if (bar < 0)
1037		return bar;
1038
1039	if (base)
1040		*base = pci_resource_start(ndev->ntb.pdev, bar) +
1041			(idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1042
1043	if (size)
1044		*size = pci_resource_len(ndev->ntb.pdev, bar) -
1045			(idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1046
1047	return 0;
1048}
1049
1050static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1051{
1052	return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1053}
1054
1055u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1056{
1057	return ntb_ndev(ntb)->db_valid_mask;
1058}
1059
1060int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1061{
1062	struct intel_ntb_dev *ndev;
1063
1064	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1065
1066	return ndev->db_vec_count;
1067}
1068
1069u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1070{
1071	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1072
1073	if (db_vector < 0 || db_vector > ndev->db_vec_count)
1074		return 0;
1075
1076	return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1077}
1078
1079static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1080{
1081	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1082
1083	return ndev_db_read(ndev,
1084			    ndev->self_mmio +
1085			    ndev->self_reg->db_bell);
1086}
1087
1088static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1089{
1090	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1091
1092	return ndev_db_write(ndev, db_bits,
1093			     ndev->self_mmio +
1094			     ndev->self_reg->db_bell);
1095}
1096
1097int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1098{
1099	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1100
1101	return ndev_db_set_mask(ndev, db_bits,
1102				ndev->self_mmio +
1103				ndev->self_reg->db_mask);
1104}
1105
1106int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1107{
1108	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1109
1110	return ndev_db_clear_mask(ndev, db_bits,
1111				  ndev->self_mmio +
1112				  ndev->self_reg->db_mask);
1113}
1114
1115static int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
1116			   resource_size_t *db_size, u64 *db_data, int db_bit)
1117{
1118	u64 db_bits;
1119	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1120
1121	if (unlikely(db_bit >= BITS_PER_LONG_LONG))
1122		return -EINVAL;
1123
1124	db_bits = BIT_ULL(db_bit);
1125
1126	if (unlikely(db_bits & ~ntb_ndev(ntb)->db_valid_mask))
1127		return -EINVAL;
1128
1129	ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1130			    ndev->peer_reg->db_bell);
1131
1132	if (db_data)
1133		*db_data = db_bits;
1134
1135
1136	return 0;
1137}
1138
1139static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1140{
1141	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1142
1143	return ndev_db_write(ndev, db_bits,
1144			     ndev->peer_mmio +
1145			     ndev->peer_reg->db_bell);
1146}
1147
1148int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1149{
1150	return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1151}
1152
1153int intel_ntb_spad_count(struct ntb_dev *ntb)
1154{
1155	struct intel_ntb_dev *ndev;
1156
1157	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1158
1159	return ndev->spad_count;
1160}
1161
1162u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1163{
1164	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1165
1166	return ndev_spad_read(ndev, idx,
1167			      ndev->self_mmio +
1168			      ndev->self_reg->spad);
1169}
1170
1171int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
1172{
1173	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1174
1175	return ndev_spad_write(ndev, idx, val,
1176			       ndev->self_mmio +
1177			       ndev->self_reg->spad);
1178}
1179
1180int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
1181			     phys_addr_t *spad_addr)
1182{
1183	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1184
1185	return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
1186			      ndev->peer_reg->spad);
1187}
1188
1189u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
1190{
1191	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1192
1193	return ndev_spad_read(ndev, sidx,
1194			      ndev->peer_mmio +
1195			      ndev->peer_reg->spad);
1196}
1197
1198int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx,
1199			      u32 val)
1200{
1201	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1202
1203	return ndev_spad_write(ndev, sidx, val,
1204			       ndev->peer_mmio +
1205			       ndev->peer_reg->spad);
1206}
1207
1208static u64 xeon_db_ioread(const void __iomem *mmio)
1209{
1210	return (u64)ioread16(mmio);
1211}
1212
1213static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
1214{
1215	iowrite16((u16)bits, mmio);
1216}
1217
1218static int xeon_poll_link(struct intel_ntb_dev *ndev)
1219{
1220	u16 reg_val;
1221	int rc;
1222
1223	ndev->reg->db_iowrite(ndev->db_link_mask,
1224			      ndev->self_mmio +
1225			      ndev->self_reg->db_bell);
1226
1227	rc = pci_read_config_word(ndev->ntb.pdev,
1228				  XEON_LINK_STATUS_OFFSET, &reg_val);
1229	if (rc)
1230		return 0;
1231
1232	if (reg_val == ndev->lnk_sta)
1233		return 0;
1234
1235	ndev->lnk_sta = reg_val;
1236
1237	return 1;
1238}
1239
1240int xeon_link_is_up(struct intel_ntb_dev *ndev)
1241{
1242	if (ndev->ntb.topo == NTB_TOPO_SEC)
1243		return 1;
1244
1245	return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
1246}
1247
1248enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1249{
1250	switch (ppd & XEON_PPD_TOPO_MASK) {
1251	case XEON_PPD_TOPO_B2B_USD:
1252		return NTB_TOPO_B2B_USD;
1253
1254	case XEON_PPD_TOPO_B2B_DSD:
1255		return NTB_TOPO_B2B_DSD;
1256
1257	case XEON_PPD_TOPO_PRI_USD:
1258	case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1259		return NTB_TOPO_PRI;
1260
1261	case XEON_PPD_TOPO_SEC_USD:
1262	case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1263		return NTB_TOPO_SEC;
1264	}
1265
1266	return NTB_TOPO_NONE;
1267}
1268
1269static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1270{
1271	if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
1272		dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
1273		return 1;
1274	}
1275	return 0;
1276}
1277
1278static int xeon_init_isr(struct intel_ntb_dev *ndev)
1279{
1280	return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
1281			     XEON_DB_MSIX_VECTOR_COUNT,
1282			     XEON_DB_MSIX_VECTOR_SHIFT,
1283			     XEON_DB_TOTAL_SHIFT);
1284}
1285
1286static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
1287{
1288	ndev_deinit_isr(ndev);
1289}
1290
1291static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
1292			     const struct intel_b2b_addr *addr,
1293			     const struct intel_b2b_addr *peer_addr)
1294{
1295	struct pci_dev *pdev;
1296	void __iomem *mmio;
1297	resource_size_t bar_size;
1298	phys_addr_t bar_addr;
1299	int b2b_bar;
1300	u8 bar_sz;
1301
1302	pdev = ndev->ntb.pdev;
1303	mmio = ndev->self_mmio;
1304
1305	if (ndev->b2b_idx == UINT_MAX) {
1306		dev_dbg(&pdev->dev, "not using b2b mw\n");
1307		b2b_bar = 0;
1308		ndev->b2b_off = 0;
1309	} else {
1310		b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1311		if (b2b_bar < 0)
1312			return -EIO;
1313
1314		dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
1315
1316		bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1317
1318		dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
1319
1320		if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
1321			dev_dbg(&pdev->dev, "b2b using first half of bar\n");
1322			ndev->b2b_off = bar_size >> 1;
1323		} else if (XEON_B2B_MIN_SIZE <= bar_size) {
1324			dev_dbg(&pdev->dev, "b2b using whole bar\n");
1325			ndev->b2b_off = 0;
1326			--ndev->mw_count;
1327		} else {
1328			dev_dbg(&pdev->dev, "b2b bar size is too small\n");
1329			return -EIO;
1330		}
1331	}
1332
1333	/* Reset the secondary bar sizes to match the primary bar sizes,
1334	 * except disable or halve the size of the b2b secondary bar.
1335	 *
1336	 * Note: code for each specific bar size register, because the register
1337	 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
1338	 */
1339	pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
1340	dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
1341	if (b2b_bar == 2) {
1342		if (ndev->b2b_off)
1343			bar_sz -= 1;
1344		else
1345			bar_sz = 0;
1346	}
1347	pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
1348	pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
1349	dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
1350
1351	if (!ndev->bar4_split) {
1352		pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
1353		dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
1354		if (b2b_bar == 4) {
1355			if (ndev->b2b_off)
1356				bar_sz -= 1;
1357			else
1358				bar_sz = 0;
1359		}
1360		pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
1361		pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
1362		dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
1363	} else {
1364		pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
1365		dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
1366		if (b2b_bar == 4) {
1367			if (ndev->b2b_off)
1368				bar_sz -= 1;
1369			else
1370				bar_sz = 0;
1371		}
1372		pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
1373		pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
1374		dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
1375
1376		pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
1377		dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
1378		if (b2b_bar == 5) {
1379			if (ndev->b2b_off)
1380				bar_sz -= 1;
1381			else
1382				bar_sz = 0;
1383		}
1384		pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
1385		pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
1386		dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
1387	}
1388
1389	/* SBAR01 hit by first part of the b2b bar */
1390	if (b2b_bar == 0)
1391		bar_addr = addr->bar0_addr;
1392	else if (b2b_bar == 2)
1393		bar_addr = addr->bar2_addr64;
1394	else if (b2b_bar == 4 && !ndev->bar4_split)
1395		bar_addr = addr->bar4_addr64;
1396	else if (b2b_bar == 4)
1397		bar_addr = addr->bar4_addr32;
1398	else if (b2b_bar == 5)
1399		bar_addr = addr->bar5_addr32;
1400	else
1401		return -EIO;
1402
1403	dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
1404	iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
1405
1406	/* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
1407	 * The b2b bar is either disabled above, or configured half-size, and
1408	 * it starts at the PBAR xlat + offset.
1409	 */
1410
1411	bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1412	iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
1413	bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
1414	dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
1415
1416	if (!ndev->bar4_split) {
1417		bar_addr = addr->bar4_addr64 +
1418			(b2b_bar == 4 ? ndev->b2b_off : 0);
1419		iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
1420		bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
1421		dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
1422	} else {
1423		bar_addr = addr->bar4_addr32 +
1424			(b2b_bar == 4 ? ndev->b2b_off : 0);
1425		iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
1426		bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
1427		dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
1428
1429		bar_addr = addr->bar5_addr32 +
1430			(b2b_bar == 5 ? ndev->b2b_off : 0);
1431		iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
1432		bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
1433		dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
1434	}
1435
1436	/* setup incoming bar limits == base addrs (zero length windows) */
1437
1438	bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1439	iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
1440	bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
1441	dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
1442
1443	if (!ndev->bar4_split) {
1444		bar_addr = addr->bar4_addr64 +
1445			(b2b_bar == 4 ? ndev->b2b_off : 0);
1446		iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
1447		bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
1448		dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
1449	} else {
1450		bar_addr = addr->bar4_addr32 +
1451			(b2b_bar == 4 ? ndev->b2b_off : 0);
1452		iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
1453		bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
1454		dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
1455
1456		bar_addr = addr->bar5_addr32 +
1457			(b2b_bar == 5 ? ndev->b2b_off : 0);
1458		iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
1459		bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
1460		dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
1461	}
1462
1463	/* zero incoming translation addrs */
1464	iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
1465
1466	if (!ndev->bar4_split) {
1467		iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
1468	} else {
1469		iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
1470		iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
1471	}
1472
1473	/* zero outgoing translation limits (whole bar size windows) */
1474	iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
1475	if (!ndev->bar4_split) {
1476		iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
1477	} else {
1478		iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
1479		iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
1480	}
1481
1482	/* set outgoing translation offsets */
1483	bar_addr = peer_addr->bar2_addr64;
1484	iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
1485	bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
1486	dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
1487
1488	if (!ndev->bar4_split) {
1489		bar_addr = peer_addr->bar4_addr64;
1490		iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
1491		bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
1492		dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
1493	} else {
1494		bar_addr = peer_addr->bar4_addr32;
1495		iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
1496		bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
1497		dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
1498
1499		bar_addr = peer_addr->bar5_addr32;
1500		iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
1501		bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
1502		dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
1503	}
1504
1505	/* set the translation offset for b2b registers */
1506	if (b2b_bar == 0)
1507		bar_addr = peer_addr->bar0_addr;
1508	else if (b2b_bar == 2)
1509		bar_addr = peer_addr->bar2_addr64;
1510	else if (b2b_bar == 4 && !ndev->bar4_split)
1511		bar_addr = peer_addr->bar4_addr64;
1512	else if (b2b_bar == 4)
1513		bar_addr = peer_addr->bar4_addr32;
1514	else if (b2b_bar == 5)
1515		bar_addr = peer_addr->bar5_addr32;
1516	else
1517		return -EIO;
1518
1519	/* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
1520	dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
1521	iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
1522	iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
1523
1524	if (b2b_bar) {
1525		/* map peer ntb mmio config space registers */
1526		ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1527					    XEON_B2B_MIN_SIZE);
1528		if (!ndev->peer_mmio)
1529			return -EIO;
1530
1531		ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
1532	}
1533
1534	return 0;
1535}
1536
1537static int xeon_init_ntb(struct intel_ntb_dev *ndev)
1538{
1539	struct device *dev = &ndev->ntb.pdev->dev;
1540	int rc;
1541	u32 ntb_ctl;
1542
1543	if (ndev->bar4_split)
1544		ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1545	else
1546		ndev->mw_count = XEON_MW_COUNT;
1547
1548	ndev->spad_count = XEON_SPAD_COUNT;
1549	ndev->db_count = XEON_DB_COUNT;
1550	ndev->db_link_mask = XEON_DB_LINK_BIT;
1551
1552	switch (ndev->ntb.topo) {
1553	case NTB_TOPO_PRI:
1554		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1555			dev_err(dev, "NTB Primary config disabled\n");
1556			return -EINVAL;
1557		}
1558
1559		/* enable link to allow secondary side device to appear */
1560		ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1561		ntb_ctl &= ~NTB_CTL_DISABLE;
1562		iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1563
1564		/* use half the spads for the peer */
1565		ndev->spad_count >>= 1;
1566		ndev->self_reg = &xeon_pri_reg;
1567		ndev->peer_reg = &xeon_sec_reg;
1568		ndev->xlat_reg = &xeon_sec_xlat;
1569		break;
1570
1571	case NTB_TOPO_SEC:
1572		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1573			dev_err(dev, "NTB Secondary config disabled\n");
1574			return -EINVAL;
1575		}
1576		/* use half the spads for the peer */
1577		ndev->spad_count >>= 1;
1578		ndev->self_reg = &xeon_sec_reg;
1579		ndev->peer_reg = &xeon_pri_reg;
1580		ndev->xlat_reg = &xeon_pri_xlat;
1581		break;
1582
1583	case NTB_TOPO_B2B_USD:
1584	case NTB_TOPO_B2B_DSD:
1585		ndev->self_reg = &xeon_pri_reg;
1586		ndev->peer_reg = &xeon_b2b_reg;
1587		ndev->xlat_reg = &xeon_sec_xlat;
1588
1589		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1590			ndev->peer_reg = &xeon_pri_reg;
1591
1592			if (b2b_mw_idx < 0)
1593				ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
1594			else
1595				ndev->b2b_idx = b2b_mw_idx;
1596
1597			if (ndev->b2b_idx >= ndev->mw_count) {
1598				dev_dbg(dev,
1599					"b2b_mw_idx %d invalid for mw_count %u\n",
1600					b2b_mw_idx, ndev->mw_count);
1601				return -EINVAL;
1602			}
1603
1604			dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
1605				b2b_mw_idx, ndev->b2b_idx);
1606
1607		} else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
1608			dev_warn(dev, "Reduce doorbell count by 1\n");
1609			ndev->db_count -= 1;
1610		}
1611
1612		if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1613			rc = xeon_setup_b2b_mw(ndev,
1614					       &xeon_b2b_dsd_addr,
1615					       &xeon_b2b_usd_addr);
1616		} else {
1617			rc = xeon_setup_b2b_mw(ndev,
1618					       &xeon_b2b_usd_addr,
1619					       &xeon_b2b_dsd_addr);
1620		}
1621		if (rc)
1622			return rc;
1623
1624		/* Enable Bus Master and Memory Space on the secondary side */
1625		iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1626			  ndev->self_mmio + XEON_SPCICMD_OFFSET);
1627
1628		break;
1629
1630	default:
1631		return -EINVAL;
1632	}
1633
1634	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1635
1636	ndev->reg->db_iowrite(ndev->db_valid_mask,
1637			      ndev->self_mmio +
1638			      ndev->self_reg->db_mask);
1639
1640	return 0;
1641}
1642
1643static int xeon_init_dev(struct intel_ntb_dev *ndev)
1644{
1645	struct pci_dev *pdev;
1646	u8 ppd;
1647	int rc, mem;
1648
1649	pdev = ndev->ntb.pdev;
1650
1651	switch (pdev->device) {
1652	/* There is a Xeon hardware errata related to writes to SDOORBELL or
1653	 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
1654	 * which may hang the system.  To workaround this use the second memory
1655	 * window to access the interrupt and scratch pad registers on the
1656	 * remote system.
1657	 */
1658	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1659	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1660	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1661	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1662	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1663	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1664	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1665	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1666	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1667	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1668	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1669	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1670	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1671	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1672	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1673		ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
1674		break;
1675	}
1676
1677	switch (pdev->device) {
1678	/* There is a hardware errata related to accessing any register in
1679	 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
1680	 */
1681	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1682	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1683	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1684	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1685	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1686	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1687	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1688	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1689	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1690		ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
1691		break;
1692	}
1693
1694	switch (pdev->device) {
1695	/* HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
1696	 * mirrored to the remote system.  Shrink the number of bits by one,
1697	 * since bit 14 is the last bit.
1698	 */
1699	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1700	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1701	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1702	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1703	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1704	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1705	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1706	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1707	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1708	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1709	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1710	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1711	case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1712	case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1713	case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1714		ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
1715		break;
1716	}
1717
1718	ndev->reg = &xeon_reg;
1719
1720	rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1721	if (rc)
1722		return -EIO;
1723
1724	ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1725	dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
1726		ntb_topo_string(ndev->ntb.topo));
1727	if (ndev->ntb.topo == NTB_TOPO_NONE)
1728		return -EINVAL;
1729
1730	if (ndev->ntb.topo != NTB_TOPO_SEC) {
1731		ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
1732		dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
1733			ppd, ndev->bar4_split);
1734	} else {
1735		/* This is a way for transparent BAR to figure out if we are
1736		 * doing split BAR or not. There is no way for the hw on the
1737		 * transparent side to know and set the PPD.
1738		 */
1739		mem = pci_select_bars(pdev, IORESOURCE_MEM);
1740		ndev->bar4_split = hweight32(mem) ==
1741			HSX_SPLIT_BAR_MW_COUNT + 1;
1742		dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
1743			mem, ndev->bar4_split);
1744	}
1745
1746	rc = xeon_init_ntb(ndev);
1747	if (rc)
1748		return rc;
1749
1750	return xeon_init_isr(ndev);
1751}
1752
1753static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
1754{
1755	xeon_deinit_isr(ndev);
1756}
1757
1758static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1759{
1760	int rc;
1761
1762	pci_set_drvdata(pdev, ndev);
1763
1764	rc = pci_enable_device(pdev);
1765	if (rc)
1766		goto err_pci_enable;
1767
1768	rc = pci_request_regions(pdev, NTB_NAME);
1769	if (rc)
1770		goto err_pci_regions;
1771
1772	pci_set_master(pdev);
1773
1774	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1775	if (rc) {
1776		rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1777		if (rc)
1778			goto err_dma_mask;
1779		dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1780	}
1781
1782	ndev->self_mmio = pci_iomap(pdev, 0, 0);
1783	if (!ndev->self_mmio) {
1784		rc = -EIO;
1785		goto err_mmio;
1786	}
1787	ndev->peer_mmio = ndev->self_mmio;
1788	ndev->peer_addr = pci_resource_start(pdev, 0);
1789
1790	return 0;
1791
1792err_mmio:
1793err_dma_mask:
1794	pci_clear_master(pdev);
1795	pci_release_regions(pdev);
1796err_pci_regions:
1797	pci_disable_device(pdev);
1798err_pci_enable:
1799	pci_set_drvdata(pdev, NULL);
1800	return rc;
1801}
1802
1803static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
1804{
1805	struct pci_dev *pdev = ndev->ntb.pdev;
1806
1807	if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
1808		pci_iounmap(pdev, ndev->peer_mmio);
1809	pci_iounmap(pdev, ndev->self_mmio);
1810
1811	pci_clear_master(pdev);
1812	pci_release_regions(pdev);
1813	pci_disable_device(pdev);
1814	pci_set_drvdata(pdev, NULL);
1815}
1816
1817static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
1818				    struct pci_dev *pdev)
1819{
1820	ndev->ntb.pdev = pdev;
1821	ndev->ntb.topo = NTB_TOPO_NONE;
1822	ndev->ntb.ops = &intel_ntb_ops;
1823
1824	ndev->b2b_off = 0;
1825	ndev->b2b_idx = UINT_MAX;
1826
1827	ndev->bar4_split = 0;
1828
1829	ndev->mw_count = 0;
1830	ndev->spad_count = 0;
1831	ndev->db_count = 0;
1832	ndev->db_vec_count = 0;
1833	ndev->db_vec_shift = 0;
1834
1835	ndev->ntb_ctl = 0;
1836	ndev->lnk_sta = 0;
1837
1838	ndev->db_valid_mask = 0;
1839	ndev->db_link_mask = 0;
1840	ndev->db_mask = 0;
1841
1842	spin_lock_init(&ndev->db_mask_lock);
1843}
1844
1845static int intel_ntb_pci_probe(struct pci_dev *pdev,
1846			       const struct pci_device_id *id)
1847{
1848	struct intel_ntb_dev *ndev;
1849	int rc, node;
1850
1851	node = dev_to_node(&pdev->dev);
1852	ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1853	if (!ndev) {
1854		rc = -ENOMEM;
1855		goto err_ndev;
1856	}
1857
1858	ndev_init_struct(ndev, pdev);
1859
1860	if (pdev_is_gen1(pdev)) {
1861		rc = intel_ntb_init_pci(ndev, pdev);
1862		if (rc)
1863			goto err_init_pci;
1864
1865		rc = xeon_init_dev(ndev);
1866		if (rc)
1867			goto err_init_dev;
1868	} else if (pdev_is_gen3(pdev)) {
1869		ndev->ntb.ops = &intel_ntb3_ops;
1870		rc = intel_ntb_init_pci(ndev, pdev);
1871		if (rc)
1872			goto err_init_pci;
1873
1874		rc = gen3_init_dev(ndev);
1875		if (rc)
1876			goto err_init_dev;
1877	} else if (pdev_is_gen4(pdev) || pdev_is_gen5(pdev)) {
1878		ndev->ntb.ops = &intel_ntb4_ops;
1879		rc = intel_ntb_init_pci(ndev, pdev);
1880		if (rc)
1881			goto err_init_pci;
1882
1883		rc = gen4_init_dev(ndev);
1884		if (rc)
1885			goto err_init_dev;
1886	} else {
1887		rc = -EINVAL;
1888		goto err_init_pci;
1889	}
1890
1891	ndev_reset_unsafe_flags(ndev);
1892
1893	ndev->reg->poll_link(ndev);
1894
1895	ndev_init_debugfs(ndev);
1896
1897	rc = ntb_register_device(&ndev->ntb);
1898	if (rc)
1899		goto err_register;
1900
1901	dev_info(&pdev->dev, "NTB device registered.\n");
1902
1903	return 0;
1904
1905err_register:
1906	ndev_deinit_debugfs(ndev);
1907	if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) ||
1908	    pdev_is_gen4(pdev) || pdev_is_gen5(pdev))
1909		xeon_deinit_dev(ndev);
1910err_init_dev:
1911	intel_ntb_deinit_pci(ndev);
1912err_init_pci:
1913	kfree(ndev);
1914err_ndev:
1915	return rc;
1916}
1917
1918static void intel_ntb_pci_remove(struct pci_dev *pdev)
1919{
1920	struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
1921
1922	ntb_unregister_device(&ndev->ntb);
1923	ndev_deinit_debugfs(ndev);
1924	if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) ||
1925	    pdev_is_gen4(pdev) || pdev_is_gen5(pdev))
1926		xeon_deinit_dev(ndev);
1927	intel_ntb_deinit_pci(ndev);
1928	kfree(ndev);
1929}
1930
1931static const struct intel_ntb_reg xeon_reg = {
1932	.poll_link		= xeon_poll_link,
1933	.link_is_up		= xeon_link_is_up,
1934	.db_ioread		= xeon_db_ioread,
1935	.db_iowrite		= xeon_db_iowrite,
1936	.db_size		= sizeof(u32),
1937	.ntb_ctl		= XEON_NTBCNTL_OFFSET,
1938	.mw_bar			= {2, 4, 5},
1939};
1940
1941static const struct intel_ntb_alt_reg xeon_pri_reg = {
1942	.db_bell		= XEON_PDOORBELL_OFFSET,
1943	.db_mask		= XEON_PDBMSK_OFFSET,
1944	.spad			= XEON_SPAD_OFFSET,
1945};
1946
1947static const struct intel_ntb_alt_reg xeon_sec_reg = {
1948	.db_bell		= XEON_SDOORBELL_OFFSET,
1949	.db_mask		= XEON_SDBMSK_OFFSET,
1950	/* second half of the scratchpads */
1951	.spad			= XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
1952};
1953
1954static const struct intel_ntb_alt_reg xeon_b2b_reg = {
1955	.db_bell		= XEON_B2B_DOORBELL_OFFSET,
1956	.spad			= XEON_B2B_SPAD_OFFSET,
1957};
1958
1959static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
1960	/* Note: no primary .bar0_base visible to the secondary side.
1961	 *
1962	 * The secondary side cannot get the base address stored in primary
1963	 * bars.  The base address is necessary to set the limit register to
1964	 * any value other than zero, or unlimited.
1965	 *
1966	 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
1967	 * window by setting the limit equal to base, nor can it limit the size
1968	 * of the memory window by setting the limit to base + size.
1969	 */
1970	.bar2_limit		= XEON_PBAR23LMT_OFFSET,
1971	.bar2_xlat		= XEON_PBAR23XLAT_OFFSET,
1972};
1973
1974static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
1975	.bar0_base		= XEON_SBAR0BASE_OFFSET,
1976	.bar2_limit		= XEON_SBAR23LMT_OFFSET,
1977	.bar2_xlat		= XEON_SBAR23XLAT_OFFSET,
1978};
1979
1980struct intel_b2b_addr xeon_b2b_usd_addr = {
1981	.bar2_addr64		= XEON_B2B_BAR2_ADDR64,
1982	.bar4_addr64		= XEON_B2B_BAR4_ADDR64,
1983	.bar4_addr32		= XEON_B2B_BAR4_ADDR32,
1984	.bar5_addr32		= XEON_B2B_BAR5_ADDR32,
1985};
1986
1987struct intel_b2b_addr xeon_b2b_dsd_addr = {
1988	.bar2_addr64		= XEON_B2B_BAR2_ADDR64,
1989	.bar4_addr64		= XEON_B2B_BAR4_ADDR64,
1990	.bar4_addr32		= XEON_B2B_BAR4_ADDR32,
1991	.bar5_addr32		= XEON_B2B_BAR5_ADDR32,
1992};
1993
1994/* operations for primary side of local ntb */
1995static const struct ntb_dev_ops intel_ntb_ops = {
1996	.mw_count		= intel_ntb_mw_count,
1997	.mw_get_align		= intel_ntb_mw_get_align,
1998	.mw_set_trans		= intel_ntb_mw_set_trans,
1999	.peer_mw_count		= intel_ntb_peer_mw_count,
2000	.peer_mw_get_addr	= intel_ntb_peer_mw_get_addr,
2001	.link_is_up		= intel_ntb_link_is_up,
2002	.link_enable		= intel_ntb_link_enable,
2003	.link_disable		= intel_ntb_link_disable,
2004	.db_is_unsafe		= intel_ntb_db_is_unsafe,
2005	.db_valid_mask		= intel_ntb_db_valid_mask,
2006	.db_vector_count	= intel_ntb_db_vector_count,
2007	.db_vector_mask		= intel_ntb_db_vector_mask,
2008	.db_read		= intel_ntb_db_read,
2009	.db_clear		= intel_ntb_db_clear,
2010	.db_set_mask		= intel_ntb_db_set_mask,
2011	.db_clear_mask		= intel_ntb_db_clear_mask,
2012	.peer_db_addr		= intel_ntb_peer_db_addr,
2013	.peer_db_set		= intel_ntb_peer_db_set,
2014	.spad_is_unsafe		= intel_ntb_spad_is_unsafe,
2015	.spad_count		= intel_ntb_spad_count,
2016	.spad_read		= intel_ntb_spad_read,
2017	.spad_write		= intel_ntb_spad_write,
2018	.peer_spad_addr		= intel_ntb_peer_spad_addr,
2019	.peer_spad_read		= intel_ntb_peer_spad_read,
2020	.peer_spad_write	= intel_ntb_peer_spad_write,
2021};
2022
2023static const struct file_operations intel_ntb_debugfs_info = {
2024	.owner = THIS_MODULE,
2025	.open = simple_open,
2026	.read = ndev_debugfs_read,
2027};
2028
2029static const struct pci_device_id intel_ntb_pci_tbl[] = {
2030	/* GEN1 */
2031	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2032	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2033	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2034	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2035	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
2036	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2037	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2038	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2039	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2040	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
2041	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2042	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2043	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2044	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2045	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
2046
2047	/* GEN3 */
2048	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
2049
2050	/* GEN4 */
2051	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_ICX)},
2052	/* GEN5 PCIe */
2053	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_GNR)},
2054	{0}
2055};
2056MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2057
2058static struct pci_driver intel_ntb_pci_driver = {
2059	.name = KBUILD_MODNAME,
2060	.id_table = intel_ntb_pci_tbl,
2061	.probe = intel_ntb_pci_probe,
2062	.remove = intel_ntb_pci_remove,
2063};
2064
2065static int __init intel_ntb_pci_driver_init(void)
2066{
 
2067	pr_info("%s %s\n", NTB_DESC, NTB_VER);
2068
2069	if (debugfs_initialized())
2070		debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2071
2072	return pci_register_driver(&intel_ntb_pci_driver);
 
 
 
 
2073}
2074module_init(intel_ntb_pci_driver_init);
2075
2076static void __exit intel_ntb_pci_driver_exit(void)
2077{
2078	pci_unregister_driver(&intel_ntb_pci_driver);
2079
2080	debugfs_remove_recursive(debugfs_dir);
2081}
2082module_exit(intel_ntb_pci_driver_exit);