1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright IBM Corp. 2012
   4 *
   5 * Author(s):
   6 *   Jan Glauber <jang@linux.vnet.ibm.com>
   7 *
   8 * The System z PCI code is a rewrite from a prototype by
   9 * the following people (Kudoz!):
  10 *   Alexander Schmidt
  11 *   Christoph Raisch
  12 *   Hannes Hering
  13 *   Hoang-Nam Nguyen
  14 *   Jan-Bernd Themann
  15 *   Stefan Roscher
  16 *   Thomas Klein
  17 */
  18
  19#define KMSG_COMPONENT "zpci"
  20#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22#include <linux/kernel.h>
  23#include <linux/slab.h>
  24#include <linux/err.h>
  25#include <linux/export.h>
  26#include <linux/delay.h>
  27#include <linux/seq_file.h>
  28#include <linux/jump_label.h>
  29#include <linux/pci.h>
  30#include <linux/printk.h>
  31#include <linux/lockdep.h>
  32#include <linux/list_sort.h>
  33
  34#include <asm/isc.h>
  35#include <asm/airq.h>
  36#include <asm/facility.h>
  37#include <asm/pci_insn.h>
  38#include <asm/pci_clp.h>
  39#include <asm/pci_dma.h>
  40
  41#include "pci_bus.h"
  42#include "pci_iov.h"
  43
  44/* list of all detected zpci devices */
  45static LIST_HEAD(zpci_list);
  46static DEFINE_SPINLOCK(zpci_list_lock);
  47
  48static DECLARE_BITMAP(zpci_domain, ZPCI_DOMAIN_BITMAP_SIZE);
  49static DEFINE_SPINLOCK(zpci_domain_lock);
  50
  51#define ZPCI_IOMAP_ENTRIES						\
  52	min(((unsigned long) ZPCI_NR_DEVICES * PCI_STD_NUM_BARS / 2),	\
  53	    ZPCI_IOMAP_MAX_ENTRIES)
  54
  55unsigned int s390_pci_no_rid;
  56
  57static DEFINE_SPINLOCK(zpci_iomap_lock);
  58static unsigned long *zpci_iomap_bitmap;
  59struct zpci_iomap_entry *zpci_iomap_start;
  60EXPORT_SYMBOL_GPL(zpci_iomap_start);
  61
  62DEFINE_STATIC_KEY_FALSE(have_mio);
  63
  64static struct kmem_cache *zdev_fmb_cache;
  65
  66/* AEN structures that must be preserved over KVM module re-insertion */
  67union zpci_sic_iib *zpci_aipb;
  68EXPORT_SYMBOL_GPL(zpci_aipb);
  69struct airq_iv *zpci_aif_sbv;
  70EXPORT_SYMBOL_GPL(zpci_aif_sbv);
  71
  72struct zpci_dev *get_zdev_by_fid(u32 fid)
  73{
  74	struct zpci_dev *tmp, *zdev = NULL;
  75
  76	spin_lock(&zpci_list_lock);
  77	list_for_each_entry(tmp, &zpci_list, entry) {
  78		if (tmp->fid == fid) {
  79			zdev = tmp;
  80			zpci_zdev_get(zdev);
  81			break;
  82		}
  83	}
  84	spin_unlock(&zpci_list_lock);
  85	return zdev;
  86}
  87
  88void zpci_remove_reserved_devices(void)
  89{
  90	struct zpci_dev *tmp, *zdev;
  91	enum zpci_state state;
  92	LIST_HEAD(remove);
  93
  94	spin_lock(&zpci_list_lock);
  95	list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) {
  96		if (zdev->state == ZPCI_FN_STATE_STANDBY &&
  97		    !clp_get_state(zdev->fid, &state) &&
  98		    state == ZPCI_FN_STATE_RESERVED)
  99			list_move_tail(&zdev->entry, &remove);
 100	}
 101	spin_unlock(&zpci_list_lock);
 102
 103	list_for_each_entry_safe(zdev, tmp, &remove, entry)
 104		zpci_device_reserved(zdev);
 105}
 106
 107int pci_domain_nr(struct pci_bus *bus)
 108{
 109	return ((struct zpci_bus *) bus->sysdata)->domain_nr;
 110}
 111EXPORT_SYMBOL_GPL(pci_domain_nr);
 112
 113int pci_proc_domain(struct pci_bus *bus)
 114{
 115	return pci_domain_nr(bus);
 116}
 117EXPORT_SYMBOL_GPL(pci_proc_domain);
 118
 119/* Modify PCI: Register I/O address translation parameters */
 120int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
 121		       u64 base, u64 limit, u64 iota, u8 *status)
 122{
 123	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);
 124	struct zpci_fib fib = {0};
 125	u8 cc;
 126
 127	WARN_ON_ONCE(iota & 0x3fff);
 128	fib.pba = base;
 129	/* Work around off by one in ISM virt device */
 130	if (zdev->pft == PCI_FUNC_TYPE_ISM && limit > base)
 131		fib.pal = limit + (1 << 12);
 132	else
 133		fib.pal = limit;
 134	fib.iota = iota | ZPCI_IOTA_RTTO_FLAG;
 135	fib.gd = zdev->gisa;
 136	cc = zpci_mod_fc(req, &fib, status);
 137	if (cc)
 138		zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, *status);
 139	return cc;
 140}
 141EXPORT_SYMBOL_GPL(zpci_register_ioat);
 142
 143/* Modify PCI: Unregister I/O address translation parameters */
 144int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
 145{
 146	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT);
 147	struct zpci_fib fib = {0};
 148	u8 cc, status;
 149
 150	fib.gd = zdev->gisa;
 151
 152	cc = zpci_mod_fc(req, &fib, &status);
 153	if (cc)
 154		zpci_dbg(3, "unreg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status);
 155	return cc;
 156}
 157
 158/* Modify PCI: Set PCI function measurement parameters */
 159int zpci_fmb_enable_device(struct zpci_dev *zdev)
 160{
 161	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
 162	struct zpci_iommu_ctrs *ctrs;
 163	struct zpci_fib fib = {0};
 164	unsigned long flags;
 165	u8 cc, status;
 166
 167	if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
 168		return -EINVAL;
 169
 170	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
 171	if (!zdev->fmb)
 172		return -ENOMEM;
 173	WARN_ON((u64) zdev->fmb & 0xf);
 174
 175	/* reset software counters */
 176	spin_lock_irqsave(&zdev->dom_lock, flags);
 177	ctrs = zpci_get_iommu_ctrs(zdev);
 178	if (ctrs) {
 179		atomic64_set(&ctrs->mapped_pages, 0);
 180		atomic64_set(&ctrs->unmapped_pages, 0);
 181		atomic64_set(&ctrs->global_rpcits, 0);
 182		atomic64_set(&ctrs->sync_map_rpcits, 0);
 183		atomic64_set(&ctrs->sync_rpcits, 0);
 184	}
 185	spin_unlock_irqrestore(&zdev->dom_lock, flags);
 186
 187
 188	fib.fmb_addr = virt_to_phys(zdev->fmb);
 189	fib.gd = zdev->gisa;
 190	cc = zpci_mod_fc(req, &fib, &status);
 191	if (cc) {
 192		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
 193		zdev->fmb = NULL;
 194	}
 195	return cc ? -EIO : 0;
 196}
 197
 198/* Modify PCI: Disable PCI function measurement */
 199int zpci_fmb_disable_device(struct zpci_dev *zdev)
 200{
 201	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
 202	struct zpci_fib fib = {0};
 203	u8 cc, status;
 204
 205	if (!zdev->fmb)
 206		return -EINVAL;
 207
 208	fib.gd = zdev->gisa;
 209
 210	/* Function measurement is disabled if fmb address is zero */
 211	cc = zpci_mod_fc(req, &fib, &status);
 212	if (cc == 3) /* Function already gone. */
 213		cc = 0;
 214
 215	if (!cc) {
 216		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
 217		zdev->fmb = NULL;
 218	}
 219	return cc ? -EIO : 0;
 220}
 221
 222static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
 223{
 224	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
 225	u64 data;
 226	int rc;
 227
 228	rc = __zpci_load(&data, req, offset);
 229	if (!rc) {
 230		data = le64_to_cpu((__force __le64) data);
 231		data >>= (8 - len) * 8;
 232		*val = (u32) data;
 233	} else
 234		*val = 0xffffffff;
 235	return rc;
 236}
 237
 238static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
 239{
 240	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
 241	u64 data = val;
 242	int rc;
 243
 244	data <<= (8 - len) * 8;
 245	data = (__force u64) cpu_to_le64(data);
 246	rc = __zpci_store(data, req, offset);
 247	return rc;
 248}
 249
 250resource_size_t pcibios_align_resource(void *data, const struct resource *res,
 251				       resource_size_t size,
 252				       resource_size_t align)
 253{
 254	return 0;
 255}
 256
 257void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size,
 258			   unsigned long prot)
 259{
 260	/*
 261	 * When PCI MIO instructions are unavailable the "physical" address
 262	 * encodes a hint for accessing the PCI memory space it represents.
 263	 * Just pass it unchanged such that ioread/iowrite can decode it.
 264	 */
 
 
 
 
 
 
 
 
 265	if (!static_branch_unlikely(&have_mio))
 266		return (void __iomem *)phys_addr;
 267
 268	return generic_ioremap_prot(phys_addr, size, __pgprot(prot));
 
 
 
 
 
 
 
 
 
 
 
 
 269}
 270EXPORT_SYMBOL(ioremap_prot);
 271
 272void iounmap(volatile void __iomem *addr)
 273{
 274	if (static_branch_likely(&have_mio))
 275		generic_iounmap(addr);
 276}
 277EXPORT_SYMBOL(iounmap);
 278
 279/* Create a virtual mapping cookie for a PCI BAR */
 280static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar,
 281					unsigned long offset, unsigned long max)
 282{
 283	struct zpci_dev *zdev =	to_zpci(pdev);
 284	int idx;
 285
 286	idx = zdev->bars[bar].map_idx;
 287	spin_lock(&zpci_iomap_lock);
 288	/* Detect overrun */
 289	WARN_ON(!++zpci_iomap_start[idx].count);
 290	zpci_iomap_start[idx].fh = zdev->fh;
 291	zpci_iomap_start[idx].bar = bar;
 292	spin_unlock(&zpci_iomap_lock);
 293
 294	return (void __iomem *) ZPCI_ADDR(idx) + offset;
 295}
 296
 297static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar,
 298					 unsigned long offset,
 299					 unsigned long max)
 300{
 301	unsigned long barsize = pci_resource_len(pdev, bar);
 302	struct zpci_dev *zdev = to_zpci(pdev);
 303	void __iomem *iova;
 304
 305	iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize);
 306	return iova ? iova + offset : iova;
 307}
 308
 309void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar,
 310			      unsigned long offset, unsigned long max)
 311{
 312	if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar))
 313		return NULL;
 314
 315	if (static_branch_likely(&have_mio))
 316		return pci_iomap_range_mio(pdev, bar, offset, max);
 317	else
 318		return pci_iomap_range_fh(pdev, bar, offset, max);
 319}
 320EXPORT_SYMBOL(pci_iomap_range);
 321
 322void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
 323{
 324	return pci_iomap_range(dev, bar, 0, maxlen);
 325}
 326EXPORT_SYMBOL(pci_iomap);
 327
 328static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar,
 329					    unsigned long offset, unsigned long max)
 330{
 331	unsigned long barsize = pci_resource_len(pdev, bar);
 332	struct zpci_dev *zdev = to_zpci(pdev);
 333	void __iomem *iova;
 334
 335	iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize);
 336	return iova ? iova + offset : iova;
 337}
 338
 339void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar,
 340				 unsigned long offset, unsigned long max)
 341{
 342	if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar))
 343		return NULL;
 344
 345	if (static_branch_likely(&have_mio))
 346		return pci_iomap_wc_range_mio(pdev, bar, offset, max);
 347	else
 348		return pci_iomap_range_fh(pdev, bar, offset, max);
 349}
 350EXPORT_SYMBOL(pci_iomap_wc_range);
 351
 352void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen)
 353{
 354	return pci_iomap_wc_range(dev, bar, 0, maxlen);
 355}
 356EXPORT_SYMBOL(pci_iomap_wc);
 357
 358static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr)
 359{
 360	unsigned int idx = ZPCI_IDX(addr);
 361
 362	spin_lock(&zpci_iomap_lock);
 363	/* Detect underrun */
 364	WARN_ON(!zpci_iomap_start[idx].count);
 365	if (!--zpci_iomap_start[idx].count) {
 366		zpci_iomap_start[idx].fh = 0;
 367		zpci_iomap_start[idx].bar = 0;
 368	}
 369	spin_unlock(&zpci_iomap_lock);
 370}
 371
 372static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr)
 373{
 374	iounmap(addr);
 375}
 376
 377void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
 378{
 379	if (static_branch_likely(&have_mio))
 380		pci_iounmap_mio(pdev, addr);
 381	else
 382		pci_iounmap_fh(pdev, addr);
 383}
 384EXPORT_SYMBOL(pci_iounmap);
 385
 386static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
 387		    int size, u32 *val)
 388{
 389	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);
 390
 391	return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV;
 392}
 393
 394static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
 395		     int size, u32 val)
 396{
 397	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);
 398
 399	return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV;
 400}
 401
 402static struct pci_ops pci_root_ops = {
 403	.read = pci_read,
 404	.write = pci_write,
 405};
 406
 
 
 
 
 
 
 
 
 
 407static void zpci_map_resources(struct pci_dev *pdev)
 408{
 409	struct zpci_dev *zdev = to_zpci(pdev);
 410	resource_size_t len;
 411	int i;
 412
 413	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 414		len = pci_resource_len(pdev, i);
 415		if (!len)
 416			continue;
 417
 418		if (zpci_use_mio(zdev))
 419			pdev->resource[i].start =
 420				(resource_size_t __force) zdev->bars[i].mio_wt;
 421		else
 422			pdev->resource[i].start = (resource_size_t __force)
 423				pci_iomap_range_fh(pdev, i, 0, 0);
 424		pdev->resource[i].end = pdev->resource[i].start + len - 1;
 425	}
 426
 427	zpci_iov_map_resources(pdev);
 
 
 
 
 
 
 
 
 
 428}
 429
 430static void zpci_unmap_resources(struct pci_dev *pdev)
 431{
 432	struct zpci_dev *zdev = to_zpci(pdev);
 433	resource_size_t len;
 434	int i;
 435
 436	if (zpci_use_mio(zdev))
 437		return;
 438
 439	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 440		len = pci_resource_len(pdev, i);
 441		if (!len)
 442			continue;
 443		pci_iounmap_fh(pdev, (void __iomem __force *)
 444			       pdev->resource[i].start);
 445	}
 446}
 447
 448static int zpci_alloc_iomap(struct zpci_dev *zdev)
 449{
 450	unsigned long entry;
 451
 452	spin_lock(&zpci_iomap_lock);
 453	entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES);
 454	if (entry == ZPCI_IOMAP_ENTRIES) {
 455		spin_unlock(&zpci_iomap_lock);
 456		return -ENOSPC;
 457	}
 458	set_bit(entry, zpci_iomap_bitmap);
 459	spin_unlock(&zpci_iomap_lock);
 460	return entry;
 461}
 462
 463static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
 464{
 465	spin_lock(&zpci_iomap_lock);
 466	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
 467	clear_bit(entry, zpci_iomap_bitmap);
 468	spin_unlock(&zpci_iomap_lock);
 469}
 470
 471static void zpci_do_update_iomap_fh(struct zpci_dev *zdev, u32 fh)
 472{
 473	int bar, idx;
 474
 475	spin_lock(&zpci_iomap_lock);
 476	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
 477		if (!zdev->bars[bar].size)
 478			continue;
 479		idx = zdev->bars[bar].map_idx;
 480		if (!zpci_iomap_start[idx].count)
 481			continue;
 482		WRITE_ONCE(zpci_iomap_start[idx].fh, zdev->fh);
 483	}
 484	spin_unlock(&zpci_iomap_lock);
 485}
 486
 487void zpci_update_fh(struct zpci_dev *zdev, u32 fh)
 488{
 489	if (!fh || zdev->fh == fh)
 490		return;
 491
 492	zdev->fh = fh;
 493	if (zpci_use_mio(zdev))
 494		return;
 495	if (zdev->has_resources && zdev_enabled(zdev))
 496		zpci_do_update_iomap_fh(zdev, fh);
 497}
 498
 499static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
 500				    unsigned long size, unsigned long flags)
 501{
 502	struct resource *r;
 503
 504	r = kzalloc(sizeof(*r), GFP_KERNEL);
 505	if (!r)
 506		return NULL;
 507
 508	r->start = start;
 509	r->end = r->start + size - 1;
 510	r->flags = flags;
 511	r->name = zdev->res_name;
 512
 513	if (request_resource(&iomem_resource, r)) {
 514		kfree(r);
 515		return NULL;
 516	}
 517	return r;
 518}
 519
 520int zpci_setup_bus_resources(struct zpci_dev *zdev)
 
 521{
 522	unsigned long addr, size, flags;
 523	struct resource *res;
 524	int i, entry;
 525
 526	snprintf(zdev->res_name, sizeof(zdev->res_name),
 527		 "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR);
 528
 529	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 530		if (!zdev->bars[i].size)
 531			continue;
 532		entry = zpci_alloc_iomap(zdev);
 533		if (entry < 0)
 534			return entry;
 535		zdev->bars[i].map_idx = entry;
 536
 537		/* only MMIO is supported */
 538		flags = IORESOURCE_MEM;
 539		if (zdev->bars[i].val & 8)
 540			flags |= IORESOURCE_PREFETCH;
 541		if (zdev->bars[i].val & 4)
 542			flags |= IORESOURCE_MEM_64;
 543
 544		if (zpci_use_mio(zdev))
 545			addr = (unsigned long) zdev->bars[i].mio_wt;
 546		else
 547			addr = ZPCI_ADDR(entry);
 548		size = 1UL << zdev->bars[i].size;
 549
 550		res = __alloc_res(zdev, addr, size, flags);
 551		if (!res) {
 552			zpci_free_iomap(zdev, entry);
 553			return -ENOMEM;
 554		}
 555		zdev->bars[i].res = res;
 
 556	}
 557	zdev->has_resources = 1;
 558
 559	return 0;
 560}
 561
 562static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
 563{
 564	struct resource *res;
 565	int i;
 566
 567	pci_lock_rescan_remove();
 568	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 569		res = zdev->bars[i].res;
 570		if (!res)
 571			continue;
 572
 573		release_resource(res);
 574		pci_bus_remove_resource(zdev->zbus->bus, res);
 575		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
 576		zdev->bars[i].res = NULL;
 577		kfree(res);
 578	}
 579	zdev->has_resources = 0;
 580	pci_unlock_rescan_remove();
 581}
 582
 583int pcibios_device_add(struct pci_dev *pdev)
 584{
 585	struct zpci_dev *zdev = to_zpci(pdev);
 586	struct resource *res;
 587	int i;
 588
 589	/* The pdev has a reference to the zdev via its bus */
 590	zpci_zdev_get(zdev);
 591	if (pdev->is_physfn)
 592		pdev->no_vf_scan = 1;
 593
 
 
 594	zpci_map_resources(pdev);
 595
 596	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 597		res = &pdev->resource[i];
 598		if (res->parent || !res->flags)
 599			continue;
 600		pci_claim_resource(pdev, i);
 601	}
 602
 603	return 0;
 604}
 605
 606void pcibios_release_device(struct pci_dev *pdev)
 607{
 608	struct zpci_dev *zdev = to_zpci(pdev);
 609
 610	zpci_unmap_resources(pdev);
 611	zpci_zdev_put(zdev);
 612}
 613
 614int pcibios_enable_device(struct pci_dev *pdev, int mask)
 615{
 616	struct zpci_dev *zdev = to_zpci(pdev);
 617
 618	zpci_debug_init_device(zdev, dev_name(&pdev->dev));
 619	zpci_fmb_enable_device(zdev);
 620
 621	return pci_enable_resources(pdev, mask);
 622}
 623
 624void pcibios_disable_device(struct pci_dev *pdev)
 625{
 626	struct zpci_dev *zdev = to_zpci(pdev);
 627
 628	zpci_fmb_disable_device(zdev);
 629	zpci_debug_exit_device(zdev);
 630}
 631
 632static int __zpci_register_domain(int domain)
 633{
 634	spin_lock(&zpci_domain_lock);
 635	if (test_bit(domain, zpci_domain)) {
 636		spin_unlock(&zpci_domain_lock);
 637		pr_err("Domain %04x is already assigned\n", domain);
 638		return -EEXIST;
 639	}
 640	set_bit(domain, zpci_domain);
 641	spin_unlock(&zpci_domain_lock);
 642	return domain;
 643}
 644
 645static int __zpci_alloc_domain(void)
 646{
 647	int domain;
 648
 649	spin_lock(&zpci_domain_lock);
 650	/*
 651	 * We can always auto allocate domains below ZPCI_NR_DEVICES.
 652	 * There is either a free domain or we have reached the maximum in
 653	 * which case we would have bailed earlier.
 654	 */
 655	domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
 656	set_bit(domain, zpci_domain);
 657	spin_unlock(&zpci_domain_lock);
 658	return domain;
 659}
 660
 661int zpci_alloc_domain(int domain)
 662{
 663	if (zpci_unique_uid) {
 664		if (domain)
 665			return __zpci_register_domain(domain);
 666		pr_warn("UID checking was active but no UID is provided: switching to automatic domain allocation\n");
 667		update_uid_checking(false);
 668	}
 669	return __zpci_alloc_domain();
 670}
 671
 672void zpci_free_domain(int domain)
 673{
 674	spin_lock(&zpci_domain_lock);
 675	clear_bit(domain, zpci_domain);
 676	spin_unlock(&zpci_domain_lock);
 677}
 678
 679
 680int zpci_enable_device(struct zpci_dev *zdev)
 681{
 682	u32 fh = zdev->fh;
 683	int rc = 0;
 684
 685	if (clp_enable_fh(zdev, &fh, ZPCI_NR_DMA_SPACES))
 686		rc = -EIO;
 687	else
 688		zpci_update_fh(zdev, fh);
 
 
 
 
 
 
 
 
 
 
 689	return rc;
 690}
 691EXPORT_SYMBOL_GPL(zpci_enable_device);
 692
 693int zpci_disable_device(struct zpci_dev *zdev)
 694{
 695	u32 fh = zdev->fh;
 696	int cc, rc = 0;
 697
 698	cc = clp_disable_fh(zdev, &fh);
 699	if (!cc) {
 700		zpci_update_fh(zdev, fh);
 701	} else if (cc == CLP_RC_SETPCIFN_ALRDY) {
 702		pr_info("Disabling PCI function %08x had no effect as it was already disabled\n",
 703			zdev->fid);
 704		/* Function is already disabled - update handle */
 705		rc = clp_refresh_fh(zdev->fid, &fh);
 706		if (!rc) {
 707			zpci_update_fh(zdev, fh);
 708			rc = -EINVAL;
 709		}
 710	} else {
 711		rc = -EIO;
 712	}
 713	return rc;
 714}
 715EXPORT_SYMBOL_GPL(zpci_disable_device);
 716
 717/**
 718 * zpci_hot_reset_device - perform a reset of the given zPCI function
 719 * @zdev: the slot which should be reset
 720 *
 721 * Performs a low level reset of the zPCI function. The reset is low level in
 722 * the sense that the zPCI function can be reset without detaching it from the
 723 * common PCI subsystem. The reset may be performed while under control of
 724 * either DMA or IOMMU APIs in which case the existing DMA/IOMMU translation
 725 * table is reinstated at the end of the reset.
 726 *
 727 * After the reset the functions internal state is reset to an initial state
 728 * equivalent to its state during boot when first probing a driver.
 729 * Consequently after reset the PCI function requires re-initialization via the
 730 * common PCI code including re-enabling IRQs via pci_alloc_irq_vectors()
 731 * and enabling the function via e.g. pci_enable_device_flags(). The caller
 732 * must guard against concurrent reset attempts.
 733 *
 734 * In most cases this function should not be called directly but through
 735 * pci_reset_function() or pci_reset_bus() which handle the save/restore and
 736 * locking - asserted by lockdep.
 737 *
 738 * Return: 0 on success and an error value otherwise
 739 */
 740int zpci_hot_reset_device(struct zpci_dev *zdev)
 741{
 742	u8 status;
 743	int rc;
 744
 745	lockdep_assert_held(&zdev->state_lock);
 746	zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh);
 747	if (zdev_enabled(zdev)) {
 748		/* Disables device access, DMAs and IRQs (reset state) */
 749		rc = zpci_disable_device(zdev);
 750		/*
 751		 * Due to a z/VM vs LPAR inconsistency in the error state the
 752		 * FH may indicate an enabled device but disable says the
 753		 * device is already disabled don't treat it as an error here.
 754		 */
 755		if (rc == -EINVAL)
 756			rc = 0;
 757		if (rc)
 758			return rc;
 759	}
 760
 761	rc = zpci_enable_device(zdev);
 762	if (rc)
 763		return rc;
 764
 765	if (zdev->dma_table)
 766		rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
 767					virt_to_phys(zdev->dma_table), &status);
 768	if (rc) {
 769		zpci_disable_device(zdev);
 770		return rc;
 771	}
 772
 773	return 0;
 774}
 775
 776/**
 777 * zpci_create_device() - Create a new zpci_dev and add it to the zbus
 778 * @fid: Function ID of the device to be created
 779 * @fh: Current Function Handle of the device to be created
 780 * @state: Initial state after creation either Standby or Configured
 781 *
 782 * Allocates a new struct zpci_dev and queries the platform for its details.
 783 * If successful the device can subsequently be added to the zPCI subsystem
 784 * using zpci_add_device().
 785 *
 786 * Returns: the zdev on success or an error pointer otherwise
 787 */
 788struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state)
 789{
 790	struct zpci_dev *zdev;
 791	int rc;
 792
 793	zdev = kzalloc(sizeof(*zdev), GFP_KERNEL);
 794	if (!zdev)
 795		return ERR_PTR(-ENOMEM);
 796
 797	/* FID and Function Handle are the static/dynamic identifiers */
 798	zdev->fid = fid;
 799	zdev->fh = fh;
 800
 801	/* Query function properties and update zdev */
 802	rc = clp_query_pci_fn(zdev);
 803	if (rc)
 804		goto error;
 805	zdev->state =  state;
 806
 807	mutex_init(&zdev->state_lock);
 808	mutex_init(&zdev->fmb_lock);
 809	mutex_init(&zdev->kzdev_lock);
 810
 811	return zdev;
 812
 813error:
 814	zpci_dbg(0, "crt fid:%x, rc:%d\n", fid, rc);
 815	kfree(zdev);
 816	return ERR_PTR(rc);
 817}
 818
 819/**
 820 * zpci_add_device() - Add a previously created zPCI device to the zPCI subsystem
 821 * @zdev: The zPCI device to be added
 822 *
 823 * A struct zpci_dev is added to the zPCI subsystem and to a virtual PCI bus creating
 824 * a new one as necessary. A hotplug slot is created and events start to be handled.
 825 * If successful from this point on zpci_zdev_get() and zpci_zdev_put() must be used.
 826 * If adding the struct zpci_dev fails the device was not added and should be freed.
 827 *
 828 * Return: 0 on success, or an error code otherwise
 829 */
 830int zpci_add_device(struct zpci_dev *zdev)
 831{
 832	int rc;
 833
 834	zpci_dbg(1, "add fid:%x, fh:%x, c:%d\n", zdev->fid, zdev->fh, zdev->state);
 835	rc = zpci_init_iommu(zdev);
 836	if (rc)
 837		goto error;
 838
 839	rc = zpci_bus_device_register(zdev, &pci_root_ops);
 840	if (rc)
 841		goto error_destroy_iommu;
 842
 843	kref_init(&zdev->kref);
 
 844	spin_lock(&zpci_list_lock);
 845	list_add_tail(&zdev->entry, &zpci_list);
 846	spin_unlock(&zpci_list_lock);
 847	return 0;
 848
 849error_destroy_iommu:
 850	zpci_destroy_iommu(zdev);
 851error:
 852	zpci_dbg(0, "add fid:%x, rc:%d\n", zdev->fid, rc);
 853	return rc;
 854}
 855
 856bool zpci_is_device_configured(struct zpci_dev *zdev)
 857{
 858	enum zpci_state state = zdev->state;
 859
 860	return state != ZPCI_FN_STATE_RESERVED &&
 861		state != ZPCI_FN_STATE_STANDBY;
 862}
 863
 864/**
 865 * zpci_scan_configured_device() - Scan a freshly configured zpci_dev
 866 * @zdev: The zpci_dev to be configured
 867 * @fh: The general function handle supplied by the platform
 868 *
 869 * Given a device in the configuration state Configured, enables, scans and
 870 * adds it to the common code PCI subsystem if possible. If any failure occurs,
 871 * the zpci_dev is left disabled.
 872 *
 873 * Return: 0 on success, or an error code otherwise
 874 */
 875int zpci_scan_configured_device(struct zpci_dev *zdev, u32 fh)
 876{
 877	zpci_update_fh(zdev, fh);
 878	return zpci_bus_scan_device(zdev);
 879}
 880
 881/**
 882 * zpci_deconfigure_device() - Deconfigure a zpci_dev
 883 * @zdev: The zpci_dev to configure
 884 *
 885 * Deconfigure a zPCI function that is currently configured and possibly known
 886 * to the common code PCI subsystem.
 887 * If any failure occurs the device is left as is.
 888 *
 889 * Return: 0 on success, or an error code otherwise
 890 */
 891int zpci_deconfigure_device(struct zpci_dev *zdev)
 892{
 893	int rc;
 894
 895	lockdep_assert_held(&zdev->state_lock);
 896	if (zdev->state != ZPCI_FN_STATE_CONFIGURED)
 897		return 0;
 898
 899	if (zdev->zbus->bus)
 900		zpci_bus_remove_device(zdev, false);
 901
 902	if (zdev_enabled(zdev)) {
 903		rc = zpci_disable_device(zdev);
 
 904		if (rc)
 905			return rc;
 906	}
 907
 908	rc = sclp_pci_deconfigure(zdev->fid);
 909	zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, rc);
 910	if (rc)
 911		return rc;
 912	zdev->state = ZPCI_FN_STATE_STANDBY;
 913
 914	return 0;
 915}
 916
 917/**
 918 * zpci_device_reserved() - Mark device as reserved
 919 * @zdev: the zpci_dev that was reserved
 920 *
 921 * Handle the case that a given zPCI function was reserved by another system.
 922 * After a call to this function the zpci_dev can not be found via
 923 * get_zdev_by_fid() anymore but may still be accessible via existing
 924 * references though it will not be functional anymore.
 925 */
 926void zpci_device_reserved(struct zpci_dev *zdev)
 927{
 928	/*
 929	 * Remove device from zpci_list as it is going away. This also
 930	 * makes sure we ignore subsequent zPCI events for this device.
 931	 */
 932	spin_lock(&zpci_list_lock);
 933	list_del(&zdev->entry);
 934	spin_unlock(&zpci_list_lock);
 935	zdev->state = ZPCI_FN_STATE_RESERVED;
 936	zpci_dbg(3, "rsv fid:%x\n", zdev->fid);
 937	zpci_zdev_put(zdev);
 938}
 939
 940void zpci_release_device(struct kref *kref)
 941{
 942	struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref);
 943
 944	WARN_ON(zdev->state != ZPCI_FN_STATE_RESERVED);
 945
 946	if (zdev->zbus->bus)
 947		zpci_bus_remove_device(zdev, false);
 948
 949	if (zdev_enabled(zdev))
 
 
 950		zpci_disable_device(zdev);
 951
 952	if (zdev->has_hp_slot)
 953		zpci_exit_slot(zdev);
 954
 955	if (zdev->has_resources)
 956		zpci_cleanup_bus_resources(zdev);
 
 
 
 
 
 
 957
 958	zpci_bus_device_unregister(zdev);
 959	zpci_destroy_iommu(zdev);
 
 960	zpci_dbg(3, "rem fid:%x\n", zdev->fid);
 961	kfree_rcu(zdev, rcu);
 962}
 963
 964int zpci_report_error(struct pci_dev *pdev,
 965		      struct zpci_report_error_header *report)
 966{
 967	struct zpci_dev *zdev = to_zpci(pdev);
 968
 969	return sclp_pci_report(report, zdev->fh, zdev->fid);
 970}
 971EXPORT_SYMBOL(zpci_report_error);
 972
 973/**
 974 * zpci_clear_error_state() - Clears the zPCI error state of the device
 975 * @zdev: The zdev for which the zPCI error state should be reset
 976 *
 977 * Clear the zPCI error state of the device. If clearing the zPCI error state
 978 * fails the device is left in the error state. In this case it may make sense
 979 * to call zpci_io_perm_failure() on the associated pdev if it exists.
 980 *
 981 * Returns: 0 on success, -EIO otherwise
 982 */
 983int zpci_clear_error_state(struct zpci_dev *zdev)
 984{
 985	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_ERROR);
 986	struct zpci_fib fib = {0};
 987	u8 status;
 988	int cc;
 989
 990	cc = zpci_mod_fc(req, &fib, &status);
 991	if (cc) {
 992		zpci_dbg(3, "ces fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);
 993		return -EIO;
 994	}
 995
 996	return 0;
 997}
 998
 999/**
1000 * zpci_reset_load_store_blocked() - Re-enables L/S from error state
1001 * @zdev: The zdev for which to unblock load/store access
1002 *
1003 * Re-enables load/store access for a PCI function in the error state while
1004 * keeping DMA blocked. In this state drivers can poke MMIO space to determine
1005 * if error recovery is possible while catching any rogue DMA access from the
1006 * device.
1007 *
1008 * Returns: 0 on success, -EIO otherwise
1009 */
1010int zpci_reset_load_store_blocked(struct zpci_dev *zdev)
1011{
1012	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_BLOCK);
1013	struct zpci_fib fib = {0};
1014	u8 status;
1015	int cc;
1016
1017	cc = zpci_mod_fc(req, &fib, &status);
1018	if (cc) {
1019		zpci_dbg(3, "rls fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);
1020		return -EIO;
1021	}
1022
1023	return 0;
1024}
1025
1026static int zpci_mem_init(void)
1027{
1028	BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) ||
1029		     __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb));
1030
1031	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
1032					   __alignof__(struct zpci_fmb), 0, NULL);
1033	if (!zdev_fmb_cache)
1034		goto error_fmb;
1035
1036	zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES,
1037				   sizeof(*zpci_iomap_start), GFP_KERNEL);
1038	if (!zpci_iomap_start)
1039		goto error_iomap;
1040
1041	zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES),
1042				    sizeof(*zpci_iomap_bitmap), GFP_KERNEL);
1043	if (!zpci_iomap_bitmap)
1044		goto error_iomap_bitmap;
1045
1046	if (static_branch_likely(&have_mio))
1047		clp_setup_writeback_mio();
1048
1049	return 0;
1050error_iomap_bitmap:
1051	kfree(zpci_iomap_start);
1052error_iomap:
1053	kmem_cache_destroy(zdev_fmb_cache);
1054error_fmb:
1055	return -ENOMEM;
1056}
1057
1058static void zpci_mem_exit(void)
1059{
1060	kfree(zpci_iomap_bitmap);
1061	kfree(zpci_iomap_start);
1062	kmem_cache_destroy(zdev_fmb_cache);
1063}
1064
1065static unsigned int s390_pci_probe __initdata = 1;
 
1066unsigned int s390_pci_force_floating __initdata;
1067static unsigned int s390_pci_initialized;
1068
1069char * __init pcibios_setup(char *str)
1070{
1071	if (!strcmp(str, "off")) {
1072		s390_pci_probe = 0;
1073		return NULL;
1074	}
1075	if (!strcmp(str, "nomio")) {
1076		get_lowcore()->machine_flags &= ~MACHINE_FLAG_PCI_MIO;
1077		return NULL;
1078	}
1079	if (!strcmp(str, "force_floating")) {
1080		s390_pci_force_floating = 1;
1081		return NULL;
1082	}
1083	if (!strcmp(str, "norid")) {
1084		s390_pci_no_rid = 1;
1085		return NULL;
1086	}
1087	return str;
1088}
1089
1090bool zpci_is_enabled(void)
1091{
1092	return s390_pci_initialized;
1093}
1094
1095static int zpci_cmp_rid(void *priv, const struct list_head *a,
1096			const struct list_head *b)
1097{
1098	struct zpci_dev *za = container_of(a, struct zpci_dev, entry);
1099	struct zpci_dev *zb = container_of(b, struct zpci_dev, entry);
1100
1101	/*
1102	 * PCI functions without RID available maintain original order
1103	 * between themselves but sort before those with RID.
1104	 */
1105	if (za->rid == zb->rid)
1106		return za->rid_available > zb->rid_available;
1107	/*
1108	 * PCI functions with RID sort by RID ascending.
1109	 */
1110	return za->rid > zb->rid;
1111}
1112
1113static void zpci_add_devices(struct list_head *scan_list)
1114{
1115	struct zpci_dev *zdev, *tmp;
1116
1117	list_sort(NULL, scan_list, &zpci_cmp_rid);
1118	list_for_each_entry_safe(zdev, tmp, scan_list, entry) {
1119		list_del_init(&zdev->entry);
1120		if (zpci_add_device(zdev))
1121			kfree(zdev);
1122	}
1123}
1124
1125int zpci_scan_devices(void)
1126{
1127	LIST_HEAD(scan_list);
1128	int rc;
1129
1130	rc = clp_scan_pci_devices(&scan_list);
1131	if (rc)
1132		return rc;
1133
1134	zpci_add_devices(&scan_list);
1135	zpci_bus_scan_busses();
1136	return 0;
1137}
1138
1139static int __init pci_base_init(void)
1140{
1141	int rc;
1142
1143	if (!s390_pci_probe)
1144		return 0;
1145
1146	if (!test_facility(69) || !test_facility(71)) {
1147		pr_info("PCI is not supported because CPU facilities 69 or 71 are not available\n");
1148		return 0;
1149	}
1150
1151	if (MACHINE_HAS_PCI_MIO) {
1152		static_branch_enable(&have_mio);
1153		system_ctl_set_bit(2, CR2_MIO_ADDRESSING_BIT);
1154	}
1155
1156	rc = zpci_debug_init();
1157	if (rc)
1158		goto out;
1159
1160	rc = zpci_mem_init();
1161	if (rc)
1162		goto out_mem;
1163
1164	rc = zpci_irq_init();
1165	if (rc)
1166		goto out_irq;
1167
1168	rc = zpci_scan_devices();
 
 
 
 
1169	if (rc)
1170		goto out_find;
1171
1172	s390_pci_initialized = 1;
1173	return 0;
1174
1175out_find:
 
 
1176	zpci_irq_exit();
1177out_irq:
1178	zpci_mem_exit();
1179out_mem:
1180	zpci_debug_exit();
1181out:
1182	return rc;
1183}
1184subsys_initcall_sync(pci_base_init);