Linux Audio

Check our new training course

Loading...
v5.4
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright IBM Corp. 2012
  4 *
  5 * Author(s):
  6 *   Jan Glauber <jang@linux.vnet.ibm.com>
  7 *
  8 * The System z PCI code is a rewrite from a prototype by
  9 * the following people (Kudoz!):
 10 *   Alexander Schmidt
 11 *   Christoph Raisch
 12 *   Hannes Hering
 13 *   Hoang-Nam Nguyen
 14 *   Jan-Bernd Themann
 15 *   Stefan Roscher
 16 *   Thomas Klein
 17 */
 18
 19#define KMSG_COMPONENT "zpci"
 20#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 21
 22#include <linux/kernel.h>
 23#include <linux/slab.h>
 24#include <linux/err.h>
 25#include <linux/export.h>
 26#include <linux/delay.h>
 
 
 27#include <linux/seq_file.h>
 28#include <linux/jump_label.h>
 29#include <linux/pci.h>
 
 30
 31#include <asm/isc.h>
 32#include <asm/airq.h>
 33#include <asm/facility.h>
 34#include <asm/pci_insn.h>
 35#include <asm/pci_clp.h>
 36#include <asm/pci_dma.h>
 37
 
 
 
 
 
 
 
 
 38/* list of all detected zpci devices */
 39static LIST_HEAD(zpci_list);
 40static DEFINE_SPINLOCK(zpci_list_lock);
 41
 
 
 
 
 
 
 42static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES);
 43static DEFINE_SPINLOCK(zpci_domain_lock);
 44
 
 
 
 
 
 
 
 
 
 
 
 45#define ZPCI_IOMAP_ENTRIES						\
 46	min(((unsigned long) ZPCI_NR_DEVICES * PCI_BAR_COUNT / 2),	\
 47	    ZPCI_IOMAP_MAX_ENTRIES)
 48
 49static DEFINE_SPINLOCK(zpci_iomap_lock);
 50static unsigned long *zpci_iomap_bitmap;
 51struct zpci_iomap_entry *zpci_iomap_start;
 52EXPORT_SYMBOL_GPL(zpci_iomap_start);
 53
 54DEFINE_STATIC_KEY_FALSE(have_mio);
 55
 56static struct kmem_cache *zdev_fmb_cache;
 57
 58struct zpci_dev *get_zdev_by_fid(u32 fid)
 59{
 60	struct zpci_dev *tmp, *zdev = NULL;
 61
 62	spin_lock(&zpci_list_lock);
 63	list_for_each_entry(tmp, &zpci_list, entry) {
 64		if (tmp->fid == fid) {
 65			zdev = tmp;
 66			break;
 67		}
 68	}
 69	spin_unlock(&zpci_list_lock);
 70	return zdev;
 71}
 72
 73void zpci_remove_reserved_devices(void)
 74{
 75	struct zpci_dev *tmp, *zdev;
 76	enum zpci_state state;
 77	LIST_HEAD(remove);
 78
 79	spin_lock(&zpci_list_lock);
 80	list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) {
 81		if (zdev->state == ZPCI_FN_STATE_STANDBY &&
 82		    !clp_get_state(zdev->fid, &state) &&
 83		    state == ZPCI_FN_STATE_RESERVED)
 84			list_move_tail(&zdev->entry, &remove);
 85	}
 86	spin_unlock(&zpci_list_lock);
 87
 88	list_for_each_entry_safe(zdev, tmp, &remove, entry)
 89		zpci_remove_device(zdev);
 90}
 91
 92static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus)
 93{
 94	return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL;
 95}
 96
 97int pci_domain_nr(struct pci_bus *bus)
 98{
 99	return ((struct zpci_dev *) bus->sysdata)->domain;
100}
101EXPORT_SYMBOL_GPL(pci_domain_nr);
102
103int pci_proc_domain(struct pci_bus *bus)
104{
105	return pci_domain_nr(bus);
106}
107EXPORT_SYMBOL_GPL(pci_proc_domain);
108
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
109/* Modify PCI: Register I/O address translation parameters */
110int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
111		       u64 base, u64 limit, u64 iota)
112{
113	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);
114	struct zpci_fib fib = {0};
115	u8 status;
116
117	WARN_ON_ONCE(iota & 0x3fff);
118	fib.pba = base;
119	fib.pal = limit;
120	fib.iota = iota | ZPCI_IOTA_RTTO_FLAG;
121	return zpci_mod_fc(req, &fib, &status) ? -EIO : 0;
122}
123
124/* Modify PCI: Unregister I/O address translation parameters */
125int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
126{
127	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT);
128	struct zpci_fib fib = {0};
129	u8 cc, status;
 
 
 
 
 
 
130
131	cc = zpci_mod_fc(req, &fib, &status);
132	if (cc == 3) /* Function already gone. */
133		cc = 0;
134	return cc ? -EIO : 0;
135}
136
137/* Modify PCI: Set PCI function measurement parameters */
138int zpci_fmb_enable_device(struct zpci_dev *zdev)
139{
140	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
141	struct zpci_fib fib = {0};
142	u8 cc, status;
143
144	if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
145		return -EINVAL;
146
147	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
148	if (!zdev->fmb)
149		return -ENOMEM;
150	WARN_ON((u64) zdev->fmb & 0xf);
151
152	/* reset software counters */
153	atomic64_set(&zdev->allocated_pages, 0);
154	atomic64_set(&zdev->mapped_pages, 0);
155	atomic64_set(&zdev->unmapped_pages, 0);
156
157	fib.fmb_addr = virt_to_phys(zdev->fmb);
158	cc = zpci_mod_fc(req, &fib, &status);
159	if (cc) {
160		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
161		zdev->fmb = NULL;
162	}
163	return cc ? -EIO : 0;
164}
165
166/* Modify PCI: Disable PCI function measurement */
167int zpci_fmb_disable_device(struct zpci_dev *zdev)
168{
169	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
170	struct zpci_fib fib = {0};
171	u8 cc, status;
172
173	if (!zdev->fmb)
174		return -EINVAL;
175
176	/* Function measurement is disabled if fmb address is zero */
177	cc = zpci_mod_fc(req, &fib, &status);
178	if (cc == 3) /* Function already gone. */
179		cc = 0;
180
181	if (!cc) {
182		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
183		zdev->fmb = NULL;
184	}
185	return cc ? -EIO : 0;
186}
187
 
 
188static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
189{
190	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
191	u64 data;
192	int rc;
193
194	rc = __zpci_load(&data, req, offset);
195	if (!rc) {
196		data = le64_to_cpu((__force __le64) data);
197		data >>= (8 - len) * 8;
198		*val = (u32) data;
199	} else
200		*val = 0xffffffff;
201	return rc;
202}
203
204static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
205{
206	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
207	u64 data = val;
208	int rc;
209
210	data <<= (8 - len) * 8;
211	data = (__force u64) cpu_to_le64(data);
212	rc = __zpci_store(data, req, offset);
213	return rc;
214}
215
 
 
 
 
216resource_size_t pcibios_align_resource(void *data, const struct resource *res,
217				       resource_size_t size,
218				       resource_size_t align)
219{
220	return 0;
221}
222
223/* combine single writes by using store-block insn */
224void __iowrite64_copy(void __iomem *to, const void *from, size_t count)
225{
226       zpci_memcpy_toio(to, from, count);
227}
228
229void __iomem *ioremap(unsigned long ioaddr, unsigned long size)
230{
231	struct vm_struct *area;
232	unsigned long offset;
233
234	if (!size)
235		return NULL;
236
237	if (!static_branch_unlikely(&have_mio))
238		return (void __iomem *) ioaddr;
239
240	offset = ioaddr & ~PAGE_MASK;
241	ioaddr &= PAGE_MASK;
242	size = PAGE_ALIGN(size + offset);
243	area = get_vm_area(size, VM_IOREMAP);
244	if (!area)
245		return NULL;
246
247	if (ioremap_page_range((unsigned long) area->addr,
248			       (unsigned long) area->addr + size,
249			       ioaddr, PAGE_KERNEL)) {
250		vunmap(area->addr);
251		return NULL;
252	}
253	return (void __iomem *) ((unsigned long) area->addr + offset);
254}
255EXPORT_SYMBOL(ioremap);
256
257void iounmap(volatile void __iomem *addr)
258{
259	if (static_branch_likely(&have_mio))
260		vunmap((__force void *) ((unsigned long) addr & PAGE_MASK));
261}
262EXPORT_SYMBOL(iounmap);
263
264/* Create a virtual mapping cookie for a PCI BAR */
265static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar,
266					unsigned long offset, unsigned long max)
 
 
267{
268	struct zpci_dev *zdev =	to_zpci(pdev);
269	int idx;
270
 
 
 
271	idx = zdev->bars[bar].map_idx;
272	spin_lock(&zpci_iomap_lock);
273	/* Detect overrun */
274	WARN_ON(!++zpci_iomap_start[idx].count);
275	zpci_iomap_start[idx].fh = zdev->fh;
276	zpci_iomap_start[idx].bar = bar;
277	spin_unlock(&zpci_iomap_lock);
278
279	return (void __iomem *) ZPCI_ADDR(idx) + offset;
280}
281
282static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar,
283					 unsigned long offset,
284					 unsigned long max)
285{
286	unsigned long barsize = pci_resource_len(pdev, bar);
287	struct zpci_dev *zdev = to_zpci(pdev);
288	void __iomem *iova;
289
290	iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize);
291	return iova ? iova + offset : iova;
292}
293
294void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar,
295			      unsigned long offset, unsigned long max)
296{
297	if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT)
298		return NULL;
299
300	if (static_branch_likely(&have_mio))
301		return pci_iomap_range_mio(pdev, bar, offset, max);
302	else
303		return pci_iomap_range_fh(pdev, bar, offset, max);
304}
305EXPORT_SYMBOL(pci_iomap_range);
306
307void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
308{
309	return pci_iomap_range(dev, bar, 0, maxlen);
310}
311EXPORT_SYMBOL(pci_iomap);
312
313static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar,
314					    unsigned long offset, unsigned long max)
315{
316	unsigned long barsize = pci_resource_len(pdev, bar);
317	struct zpci_dev *zdev = to_zpci(pdev);
318	void __iomem *iova;
319
320	iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize);
321	return iova ? iova + offset : iova;
322}
323
324void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar,
325				 unsigned long offset, unsigned long max)
326{
327	if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT)
328		return NULL;
329
330	if (static_branch_likely(&have_mio))
331		return pci_iomap_wc_range_mio(pdev, bar, offset, max);
332	else
333		return pci_iomap_range_fh(pdev, bar, offset, max);
334}
335EXPORT_SYMBOL(pci_iomap_wc_range);
336
337void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen)
338{
339	return pci_iomap_wc_range(dev, bar, 0, maxlen);
340}
341EXPORT_SYMBOL(pci_iomap_wc);
342
343static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr)
344{
345	unsigned int idx = ZPCI_IDX(addr);
346
347	spin_lock(&zpci_iomap_lock);
348	/* Detect underrun */
349	WARN_ON(!zpci_iomap_start[idx].count);
350	if (!--zpci_iomap_start[idx].count) {
351		zpci_iomap_start[idx].fh = 0;
352		zpci_iomap_start[idx].bar = 0;
353	}
354	spin_unlock(&zpci_iomap_lock);
355}
356
357static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr)
358{
359	iounmap(addr);
360}
361
362void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
363{
364	if (static_branch_likely(&have_mio))
365		pci_iounmap_mio(pdev, addr);
366	else
367		pci_iounmap_fh(pdev, addr);
368}
369EXPORT_SYMBOL(pci_iounmap);
370
371static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
372		    int size, u32 *val)
373{
374	struct zpci_dev *zdev = get_zdev_by_bus(bus);
375	int ret;
376
377	if (!zdev || devfn != ZPCI_DEVFN)
378		ret = -ENODEV;
379	else
380		ret = zpci_cfg_load(zdev, where, val, size);
381
382	return ret;
383}
384
385static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
386		     int size, u32 val)
387{
388	struct zpci_dev *zdev = get_zdev_by_bus(bus);
389	int ret;
390
391	if (!zdev || devfn != ZPCI_DEVFN)
392		ret = -ENODEV;
393	else
394		ret = zpci_cfg_store(zdev, where, val, size);
395
396	return ret;
397}
398
399static struct pci_ops pci_root_ops = {
400	.read = pci_read,
401	.write = pci_write,
402};
403
404#ifdef CONFIG_PCI_IOV
405static struct resource iov_res = {
406	.name	= "PCI IOV res",
407	.start	= 0,
408	.end	= -1,
409	.flags	= IORESOURCE_MEM,
410};
411#endif
 
 
 
 
 
 
 
 
 
 
 
412
413static void zpci_map_resources(struct pci_dev *pdev)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
414{
415	struct zpci_dev *zdev = to_zpci(pdev);
416	resource_size_t len;
417	int i;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
418
419	for (i = 0; i < PCI_BAR_COUNT; i++) {
420		len = pci_resource_len(pdev, i);
421		if (!len)
422			continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
423
424		if (zpci_use_mio(zdev))
425			pdev->resource[i].start =
426				(resource_size_t __force) zdev->bars[i].mio_wb;
 
427		else
428			pdev->resource[i].start = (resource_size_t __force)
429				pci_iomap_range_fh(pdev, i, 0, 0);
430		pdev->resource[i].end = pdev->resource[i].start + len - 1;
 
 
 
 
431	}
432
433#ifdef CONFIG_PCI_IOV
434	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
435		int bar = i + PCI_IOV_RESOURCES;
 
 
 
 
 
 
436
437		len = pci_resource_len(pdev, bar);
 
438		if (!len)
439			continue;
440		pdev->resource[bar].parent = &iov_res;
 
 
441	}
442#endif
443}
444
445static void zpci_unmap_resources(struct pci_dev *pdev)
446{
447	struct zpci_dev *zdev = to_zpci(pdev);
448	resource_size_t len;
449	int i;
450
451	if (zpci_use_mio(zdev))
452		return;
453
454	for (i = 0; i < PCI_BAR_COUNT; i++) {
455		len = pci_resource_len(pdev, i);
456		if (!len)
457			continue;
458		pci_iounmap_fh(pdev, (void __iomem __force *)
459			       pdev->resource[i].start);
460	}
461}
462
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
463static int zpci_alloc_iomap(struct zpci_dev *zdev)
464{
465	unsigned long entry;
466
467	spin_lock(&zpci_iomap_lock);
468	entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES);
469	if (entry == ZPCI_IOMAP_ENTRIES) {
470		spin_unlock(&zpci_iomap_lock);
471		return -ENOSPC;
472	}
473	set_bit(entry, zpci_iomap_bitmap);
474	spin_unlock(&zpci_iomap_lock);
475	return entry;
476}
477
478static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
479{
480	spin_lock(&zpci_iomap_lock);
481	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
482	clear_bit(entry, zpci_iomap_bitmap);
483	spin_unlock(&zpci_iomap_lock);
484}
485
486static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
487				    unsigned long size, unsigned long flags)
488{
489	struct resource *r;
490
491	r = kzalloc(sizeof(*r), GFP_KERNEL);
492	if (!r)
493		return NULL;
494
495	r->start = start;
496	r->end = r->start + size - 1;
497	r->flags = flags;
498	r->name = zdev->res_name;
499
500	if (request_resource(&iomem_resource, r)) {
501		kfree(r);
502		return NULL;
503	}
504	return r;
505}
506
507static int zpci_setup_bus_resources(struct zpci_dev *zdev,
508				    struct list_head *resources)
509{
510	unsigned long addr, size, flags;
511	struct resource *res;
512	int i, entry;
513
514	snprintf(zdev->res_name, sizeof(zdev->res_name),
515		 "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR);
516
517	for (i = 0; i < PCI_BAR_COUNT; i++) {
518		if (!zdev->bars[i].size)
519			continue;
520		entry = zpci_alloc_iomap(zdev);
521		if (entry < 0)
522			return entry;
523		zdev->bars[i].map_idx = entry;
524
525		/* only MMIO is supported */
526		flags = IORESOURCE_MEM;
527		if (zdev->bars[i].val & 8)
528			flags |= IORESOURCE_PREFETCH;
529		if (zdev->bars[i].val & 4)
530			flags |= IORESOURCE_MEM_64;
531
532		if (zpci_use_mio(zdev))
533			addr = (unsigned long) zdev->bars[i].mio_wb;
534		else
535			addr = ZPCI_ADDR(entry);
536		size = 1UL << zdev->bars[i].size;
537
538		res = __alloc_res(zdev, addr, size, flags);
539		if (!res) {
540			zpci_free_iomap(zdev, entry);
541			return -ENOMEM;
542		}
543		zdev->bars[i].res = res;
544		pci_add_resource(resources, res);
545	}
546
547	return 0;
548}
549
550static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
551{
552	int i;
553
554	for (i = 0; i < PCI_BAR_COUNT; i++) {
555		if (!zdev->bars[i].size || !zdev->bars[i].res)
556			continue;
557
558		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
559		release_resource(zdev->bars[i].res);
560		kfree(zdev->bars[i].res);
561	}
562}
563
564int pcibios_add_device(struct pci_dev *pdev)
565{
566	struct resource *res;
567	int i;
568
569	if (pdev->is_physfn)
570		pdev->no_vf_scan = 1;
571
572	pdev->dev.groups = zpci_attr_groups;
573	pdev->dev.dma_ops = &s390_pci_dma_ops;
574	zpci_map_resources(pdev);
575
576	for (i = 0; i < PCI_BAR_COUNT; i++) {
577		res = &pdev->resource[i];
578		if (res->parent || !res->flags)
579			continue;
580		pci_claim_resource(pdev, i);
581	}
582
583	return 0;
584}
585
586void pcibios_release_device(struct pci_dev *pdev)
587{
588	zpci_unmap_resources(pdev);
589}
590
591int pcibios_enable_device(struct pci_dev *pdev, int mask)
592{
593	struct zpci_dev *zdev = to_zpci(pdev);
594
595	zpci_debug_init_device(zdev, dev_name(&pdev->dev));
596	zpci_fmb_enable_device(zdev);
597
598	return pci_enable_resources(pdev, mask);
599}
600
601void pcibios_disable_device(struct pci_dev *pdev)
602{
603	struct zpci_dev *zdev = to_zpci(pdev);
604
605	zpci_fmb_disable_device(zdev);
606	zpci_debug_exit_device(zdev);
607}
608
609#ifdef CONFIG_HIBERNATE_CALLBACKS
610static int zpci_restore(struct device *dev)
611{
612	struct pci_dev *pdev = to_pci_dev(dev);
613	struct zpci_dev *zdev = to_zpci(pdev);
614	int ret = 0;
615
616	if (zdev->state != ZPCI_FN_STATE_ONLINE)
617		goto out;
618
619	ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
620	if (ret)
621		goto out;
622
623	zpci_map_resources(pdev);
624	zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
625			   (u64) zdev->dma_table);
626
627out:
628	return ret;
629}
630
631static int zpci_freeze(struct device *dev)
632{
633	struct pci_dev *pdev = to_pci_dev(dev);
634	struct zpci_dev *zdev = to_zpci(pdev);
635
636	if (zdev->state != ZPCI_FN_STATE_ONLINE)
637		return 0;
638
639	zpci_unregister_ioat(zdev, 0);
640	zpci_unmap_resources(pdev);
641	return clp_disable_fh(zdev);
642}
643
644struct dev_pm_ops pcibios_pm_ops = {
645	.thaw_noirq = zpci_restore,
646	.freeze_noirq = zpci_freeze,
647	.restore_noirq = zpci_restore,
648	.poweroff_noirq = zpci_freeze,
649};
650#endif /* CONFIG_HIBERNATE_CALLBACKS */
651
652static int zpci_alloc_domain(struct zpci_dev *zdev)
653{
654	if (zpci_unique_uid) {
655		zdev->domain = (u16) zdev->uid;
656		if (zdev->domain >= ZPCI_NR_DEVICES)
657			return 0;
658
659		spin_lock(&zpci_domain_lock);
660		if (test_bit(zdev->domain, zpci_domain)) {
661			spin_unlock(&zpci_domain_lock);
662			return -EEXIST;
663		}
664		set_bit(zdev->domain, zpci_domain);
665		spin_unlock(&zpci_domain_lock);
666		return 0;
667	}
668
669	spin_lock(&zpci_domain_lock);
670	zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
671	if (zdev->domain == ZPCI_NR_DEVICES) {
672		spin_unlock(&zpci_domain_lock);
673		return -ENOSPC;
674	}
675	set_bit(zdev->domain, zpci_domain);
676	spin_unlock(&zpci_domain_lock);
677	return 0;
678}
679
680static void zpci_free_domain(struct zpci_dev *zdev)
681{
682	if (zdev->domain >= ZPCI_NR_DEVICES)
683		return;
684
685	spin_lock(&zpci_domain_lock);
686	clear_bit(zdev->domain, zpci_domain);
687	spin_unlock(&zpci_domain_lock);
688}
689
690void pcibios_remove_bus(struct pci_bus *bus)
691{
692	struct zpci_dev *zdev = get_zdev_by_bus(bus);
693
694	zpci_exit_slot(zdev);
695	zpci_cleanup_bus_resources(zdev);
696	zpci_destroy_iommu(zdev);
697	zpci_free_domain(zdev);
698
699	spin_lock(&zpci_list_lock);
700	list_del(&zdev->entry);
701	spin_unlock(&zpci_list_lock);
702
703	zpci_dbg(3, "rem fid:%x\n", zdev->fid);
704	kfree(zdev);
705}
706
707static int zpci_scan_bus(struct zpci_dev *zdev)
708{
709	LIST_HEAD(resources);
710	int ret;
711
712	ret = zpci_setup_bus_resources(zdev, &resources);
713	if (ret)
714		goto error;
715
716	zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops,
717				      zdev, &resources);
718	if (!zdev->bus) {
719		ret = -EIO;
720		goto error;
721	}
722	zdev->bus->max_bus_speed = zdev->max_bus_speed;
723	pci_bus_add_devices(zdev->bus);
724	return 0;
725
726error:
727	zpci_cleanup_bus_resources(zdev);
728	pci_free_resource_list(&resources);
729	return ret;
730}
731
732int zpci_enable_device(struct zpci_dev *zdev)
733{
734	int rc;
735
736	rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
737	if (rc)
738		goto out;
739
740	rc = zpci_dma_init_device(zdev);
741	if (rc)
742		goto out_dma;
743
744	zdev->state = ZPCI_FN_STATE_ONLINE;
745	return 0;
746
747out_dma:
748	clp_disable_fh(zdev);
749out:
750	return rc;
751}
752EXPORT_SYMBOL_GPL(zpci_enable_device);
753
754int zpci_disable_device(struct zpci_dev *zdev)
755{
756	zpci_dma_exit_device(zdev);
757	return clp_disable_fh(zdev);
758}
759EXPORT_SYMBOL_GPL(zpci_disable_device);
760
761int zpci_create_device(struct zpci_dev *zdev)
762{
763	int rc;
764
765	rc = zpci_alloc_domain(zdev);
766	if (rc)
767		goto out;
768
769	rc = zpci_init_iommu(zdev);
770	if (rc)
771		goto out_free;
772
773	mutex_init(&zdev->lock);
774	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
775		rc = zpci_enable_device(zdev);
776		if (rc)
777			goto out_destroy_iommu;
778	}
779	rc = zpci_scan_bus(zdev);
780	if (rc)
781		goto out_disable;
782
783	spin_lock(&zpci_list_lock);
784	list_add_tail(&zdev->entry, &zpci_list);
785	spin_unlock(&zpci_list_lock);
786
787	zpci_init_slot(zdev);
788
789	return 0;
790
791out_disable:
792	if (zdev->state == ZPCI_FN_STATE_ONLINE)
793		zpci_disable_device(zdev);
794out_destroy_iommu:
795	zpci_destroy_iommu(zdev);
796out_free:
797	zpci_free_domain(zdev);
798out:
799	return rc;
800}
801
802void zpci_remove_device(struct zpci_dev *zdev)
803{
804	if (!zdev->bus)
805		return;
806
807	pci_stop_root_bus(zdev->bus);
808	pci_remove_root_bus(zdev->bus);
809}
 
810
811int zpci_report_error(struct pci_dev *pdev,
812		      struct zpci_report_error_header *report)
813{
814	struct zpci_dev *zdev = to_zpci(pdev);
815
816	return sclp_pci_report(report, zdev->fh, zdev->fid);
817}
818EXPORT_SYMBOL(zpci_report_error);
819
820static int zpci_mem_init(void)
821{
822	BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) ||
823		     __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb));
824
825	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
826					   __alignof__(struct zpci_fmb), 0, NULL);
827	if (!zdev_fmb_cache)
828		goto error_fmb;
829
830	zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES,
831				   sizeof(*zpci_iomap_start), GFP_KERNEL);
832	if (!zpci_iomap_start)
833		goto error_iomap;
834
835	zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES),
836				    sizeof(*zpci_iomap_bitmap), GFP_KERNEL);
837	if (!zpci_iomap_bitmap)
838		goto error_iomap_bitmap;
839
840	return 0;
841error_iomap_bitmap:
842	kfree(zpci_iomap_start);
843error_iomap:
844	kmem_cache_destroy(zdev_fmb_cache);
845error_fmb:
846	return -ENOMEM;
847}
848
849static void zpci_mem_exit(void)
850{
851	kfree(zpci_iomap_bitmap);
852	kfree(zpci_iomap_start);
853	kmem_cache_destroy(zdev_fmb_cache);
854}
855
856static unsigned int s390_pci_probe __initdata = 1;
857static unsigned int s390_pci_no_mio __initdata;
858unsigned int s390_pci_force_floating __initdata;
859static unsigned int s390_pci_initialized;
860
861char * __init pcibios_setup(char *str)
862{
863	if (!strcmp(str, "off")) {
864		s390_pci_probe = 0;
865		return NULL;
866	}
867	if (!strcmp(str, "nomio")) {
868		s390_pci_no_mio = 1;
869		return NULL;
870	}
871	if (!strcmp(str, "force_floating")) {
872		s390_pci_force_floating = 1;
873		return NULL;
874	}
875	return str;
876}
877
878bool zpci_is_enabled(void)
879{
880	return s390_pci_initialized;
881}
882
883static int __init pci_base_init(void)
884{
885	int rc;
886
887	if (!s390_pci_probe)
888		return 0;
889
890	if (!test_facility(69) || !test_facility(71))
891		return 0;
892
893	if (test_facility(153) && !s390_pci_no_mio) {
894		static_branch_enable(&have_mio);
895		ctl_set_bit(2, 5);
896	}
897
898	rc = zpci_debug_init();
899	if (rc)
900		goto out;
901
902	rc = zpci_mem_init();
903	if (rc)
904		goto out_mem;
905
906	rc = zpci_irq_init();
907	if (rc)
908		goto out_irq;
909
910	rc = zpci_dma_init();
911	if (rc)
912		goto out_dma;
913
914	rc = clp_scan_pci_devices();
915	if (rc)
916		goto out_find;
917
918	s390_pci_initialized = 1;
919	return 0;
920
921out_find:
922	zpci_dma_exit();
923out_dma:
924	zpci_irq_exit();
925out_irq:
926	zpci_mem_exit();
927out_mem:
928	zpci_debug_exit();
929out:
930	return rc;
931}
932subsys_initcall_sync(pci_base_init);
933
934void zpci_rescan(void)
935{
936	if (zpci_is_enabled())
937		clp_rescan_pci_devices_simple();
938}
v4.6
 
  1/*
  2 * Copyright IBM Corp. 2012
  3 *
  4 * Author(s):
  5 *   Jan Glauber <jang@linux.vnet.ibm.com>
  6 *
  7 * The System z PCI code is a rewrite from a prototype by
  8 * the following people (Kudoz!):
  9 *   Alexander Schmidt
 10 *   Christoph Raisch
 11 *   Hannes Hering
 12 *   Hoang-Nam Nguyen
 13 *   Jan-Bernd Themann
 14 *   Stefan Roscher
 15 *   Thomas Klein
 16 */
 17
 18#define KMSG_COMPONENT "zpci"
 19#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 20
 21#include <linux/kernel.h>
 22#include <linux/slab.h>
 23#include <linux/err.h>
 24#include <linux/export.h>
 25#include <linux/delay.h>
 26#include <linux/irq.h>
 27#include <linux/kernel_stat.h>
 28#include <linux/seq_file.h>
 
 29#include <linux/pci.h>
 30#include <linux/msi.h>
 31
 32#include <asm/isc.h>
 33#include <asm/airq.h>
 34#include <asm/facility.h>
 35#include <asm/pci_insn.h>
 36#include <asm/pci_clp.h>
 37#include <asm/pci_dma.h>
 38
 39#define DEBUG				/* enable pr_debug */
 40
 41#define	SIC_IRQ_MODE_ALL		0
 42#define	SIC_IRQ_MODE_SINGLE		1
 43
 44#define ZPCI_NR_DMA_SPACES		1
 45#define ZPCI_NR_DEVICES			CONFIG_PCI_NR_FUNCTIONS
 46
 47/* list of all detected zpci devices */
 48static LIST_HEAD(zpci_list);
 49static DEFINE_SPINLOCK(zpci_list_lock);
 50
 51static struct irq_chip zpci_irq_chip = {
 52	.name = "zPCI",
 53	.irq_unmask = pci_msi_unmask_irq,
 54	.irq_mask = pci_msi_mask_irq,
 55};
 56
 57static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES);
 58static DEFINE_SPINLOCK(zpci_domain_lock);
 59
 60static struct airq_iv *zpci_aisb_iv;
 61static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES];
 62
 63/* Adapter interrupt definitions */
 64static void zpci_irq_handler(struct airq_struct *airq);
 65
 66static struct airq_struct zpci_airq = {
 67	.handler = zpci_irq_handler,
 68	.isc = PCI_ISC,
 69};
 70
 71#define ZPCI_IOMAP_ENTRIES						\
 72	min(((unsigned long) CONFIG_PCI_NR_FUNCTIONS * PCI_BAR_COUNT),	\
 73	    ZPCI_IOMAP_MAX_ENTRIES)
 74
 75static DEFINE_SPINLOCK(zpci_iomap_lock);
 76static unsigned long *zpci_iomap_bitmap;
 77struct zpci_iomap_entry *zpci_iomap_start;
 78EXPORT_SYMBOL_GPL(zpci_iomap_start);
 79
 
 
 80static struct kmem_cache *zdev_fmb_cache;
 81
 82struct zpci_dev *get_zdev_by_fid(u32 fid)
 83{
 84	struct zpci_dev *tmp, *zdev = NULL;
 85
 86	spin_lock(&zpci_list_lock);
 87	list_for_each_entry(tmp, &zpci_list, entry) {
 88		if (tmp->fid == fid) {
 89			zdev = tmp;
 90			break;
 91		}
 92	}
 93	spin_unlock(&zpci_list_lock);
 94	return zdev;
 95}
 96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 97static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus)
 98{
 99	return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL;
100}
101
102int pci_domain_nr(struct pci_bus *bus)
103{
104	return ((struct zpci_dev *) bus->sysdata)->domain;
105}
106EXPORT_SYMBOL_GPL(pci_domain_nr);
107
108int pci_proc_domain(struct pci_bus *bus)
109{
110	return pci_domain_nr(bus);
111}
112EXPORT_SYMBOL_GPL(pci_proc_domain);
113
114/* Modify PCI: Register adapter interruptions */
115static int zpci_set_airq(struct zpci_dev *zdev)
116{
117	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT);
118	struct zpci_fib fib = {0};
119
120	fib.isc = PCI_ISC;
121	fib.sum = 1;		/* enable summary notifications */
122	fib.noi = airq_iv_end(zdev->aibv);
123	fib.aibv = (unsigned long) zdev->aibv->vector;
124	fib.aibvo = 0;		/* each zdev has its own interrupt vector */
125	fib.aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8;
126	fib.aisbo = zdev->aisb & 63;
127
128	return zpci_mod_fc(req, &fib);
129}
130
131struct mod_pci_args {
132	u64 base;
133	u64 limit;
134	u64 iota;
135	u64 fmb_addr;
136};
137
138static int mod_pci(struct zpci_dev *zdev, int fn, u8 dmaas, struct mod_pci_args *args)
139{
140	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, fn);
141	struct zpci_fib fib = {0};
142
143	fib.pba = args->base;
144	fib.pal = args->limit;
145	fib.iota = args->iota;
146	fib.fmb_addr = args->fmb_addr;
147
148	return zpci_mod_fc(req, &fib);
149}
150
151/* Modify PCI: Register I/O address translation parameters */
152int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
153		       u64 base, u64 limit, u64 iota)
154{
155	struct mod_pci_args args = { base, limit, iota, 0 };
 
 
156
157	WARN_ON_ONCE(iota & 0x3fff);
158	args.iota |= ZPCI_IOTA_RTTO_FLAG;
159	return mod_pci(zdev, ZPCI_MOD_FC_REG_IOAT, dmaas, &args);
 
 
160}
161
162/* Modify PCI: Unregister I/O address translation parameters */
163int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
164{
165	struct mod_pci_args args = { 0, 0, 0, 0 };
166
167	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_IOAT, dmaas, &args);
168}
169
170/* Modify PCI: Unregister adapter interruptions */
171static int zpci_clear_airq(struct zpci_dev *zdev)
172{
173	struct mod_pci_args args = { 0, 0, 0, 0 };
174
175	return mod_pci(zdev, ZPCI_MOD_FC_DEREG_INT, 0, &args);
 
 
 
176}
177
178/* Modify PCI: Set PCI function measurement parameters */
179int zpci_fmb_enable_device(struct zpci_dev *zdev)
180{
181	struct mod_pci_args args = { 0, 0, 0, 0 };
 
 
182
183	if (zdev->fmb)
184		return -EINVAL;
185
186	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
187	if (!zdev->fmb)
188		return -ENOMEM;
189	WARN_ON((u64) zdev->fmb & 0xf);
190
191	/* reset software counters */
192	atomic64_set(&zdev->allocated_pages, 0);
193	atomic64_set(&zdev->mapped_pages, 0);
194	atomic64_set(&zdev->unmapped_pages, 0);
195
196	args.fmb_addr = virt_to_phys(zdev->fmb);
197	return mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);
 
 
 
 
 
198}
199
200/* Modify PCI: Disable PCI function measurement */
201int zpci_fmb_disable_device(struct zpci_dev *zdev)
202{
203	struct mod_pci_args args = { 0, 0, 0, 0 };
204	int rc;
 
205
206	if (!zdev->fmb)
207		return -EINVAL;
208
209	/* Function measurement is disabled if fmb address is zero */
210	rc = mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);
211
212	kmem_cache_free(zdev_fmb_cache, zdev->fmb);
213	zdev->fmb = NULL;
214	return rc;
 
 
 
 
215}
216
217#define ZPCI_PCIAS_CFGSPC	15
218
219static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
220{
221	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
222	u64 data;
223	int rc;
224
225	rc = zpci_load(&data, req, offset);
226	if (!rc) {
227		data = data << ((8 - len) * 8);
228		data = le64_to_cpu(data);
229		*val = (u32) data;
230	} else
231		*val = 0xffffffff;
232	return rc;
233}
234
235static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
236{
237	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
238	u64 data = val;
239	int rc;
240
241	data = cpu_to_le64(data);
242	data = data >> ((8 - len) * 8);
243	rc = zpci_store(data, req, offset);
244	return rc;
245}
246
247void pcibios_fixup_bus(struct pci_bus *bus)
248{
249}
250
251resource_size_t pcibios_align_resource(void *data, const struct resource *res,
252				       resource_size_t size,
253				       resource_size_t align)
254{
255	return 0;
256}
257
258/* combine single writes by using store-block insn */
259void __iowrite64_copy(void __iomem *to, const void *from, size_t count)
260{
261       zpci_memcpy_toio(to, from, count);
262}
263
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
264/* Create a virtual mapping cookie for a PCI BAR */
265void __iomem *pci_iomap_range(struct pci_dev *pdev,
266			      int bar,
267			      unsigned long offset,
268			      unsigned long max)
269{
270	struct zpci_dev *zdev =	to_zpci(pdev);
271	int idx;
272
273	if (!pci_resource_len(pdev, bar))
274		return NULL;
275
276	idx = zdev->bars[bar].map_idx;
277	spin_lock(&zpci_iomap_lock);
278	/* Detect overrun */
279	WARN_ON(!++zpci_iomap_start[idx].count);
280	zpci_iomap_start[idx].fh = zdev->fh;
281	zpci_iomap_start[idx].bar = bar;
282	spin_unlock(&zpci_iomap_lock);
283
284	return (void __iomem *) ZPCI_ADDR(idx) + offset;
285}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
286EXPORT_SYMBOL(pci_iomap_range);
287
288void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
289{
290	return pci_iomap_range(dev, bar, 0, maxlen);
291}
292EXPORT_SYMBOL(pci_iomap);
293
294void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
295{
296	unsigned int idx = ZPCI_IDX(addr);
297
298	spin_lock(&zpci_iomap_lock);
299	/* Detect underrun */
300	WARN_ON(!zpci_iomap_start[idx].count);
301	if (!--zpci_iomap_start[idx].count) {
302		zpci_iomap_start[idx].fh = 0;
303		zpci_iomap_start[idx].bar = 0;
304	}
305	spin_unlock(&zpci_iomap_lock);
306}
 
 
 
 
 
 
 
 
 
 
 
 
 
307EXPORT_SYMBOL(pci_iounmap);
308
309static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
310		    int size, u32 *val)
311{
312	struct zpci_dev *zdev = get_zdev_by_bus(bus);
313	int ret;
314
315	if (!zdev || devfn != ZPCI_DEVFN)
316		ret = -ENODEV;
317	else
318		ret = zpci_cfg_load(zdev, where, val, size);
319
320	return ret;
321}
322
323static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
324		     int size, u32 val)
325{
326	struct zpci_dev *zdev = get_zdev_by_bus(bus);
327	int ret;
328
329	if (!zdev || devfn != ZPCI_DEVFN)
330		ret = -ENODEV;
331	else
332		ret = zpci_cfg_store(zdev, where, val, size);
333
334	return ret;
335}
336
337static struct pci_ops pci_root_ops = {
338	.read = pci_read,
339	.write = pci_write,
340};
341
342static void zpci_irq_handler(struct airq_struct *airq)
343{
344	unsigned long si, ai;
345	struct airq_iv *aibv;
346	int irqs_on = 0;
347
348	inc_irq_stat(IRQIO_PCI);
349	for (si = 0;;) {
350		/* Scan adapter summary indicator bit vector */
351		si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv));
352		if (si == -1UL) {
353			if (irqs_on++)
354				/* End of second scan with interrupts on. */
355				break;
356			/* First scan complete, reenable interrupts. */
357			zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
358			si = 0;
359			continue;
360		}
361
362		/* Scan the adapter interrupt vector for this device. */
363		aibv = zpci_aibv[si];
364		for (ai = 0;;) {
365			ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv));
366			if (ai == -1UL)
367				break;
368			inc_irq_stat(IRQIO_MSI);
369			airq_iv_lock(aibv, ai);
370			generic_handle_irq(airq_iv_get_data(aibv, ai));
371			airq_iv_unlock(aibv, ai);
372		}
373	}
374}
375
376int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
377{
378	struct zpci_dev *zdev = to_zpci(pdev);
379	unsigned int hwirq, msi_vecs;
380	unsigned long aisb;
381	struct msi_desc *msi;
382	struct msi_msg msg;
383	int rc, irq;
384
385	if (type == PCI_CAP_ID_MSI && nvec > 1)
386		return 1;
387	msi_vecs = min_t(unsigned int, nvec, zdev->max_msi);
388
389	/* Allocate adapter summary indicator bit */
390	rc = -EIO;
391	aisb = airq_iv_alloc_bit(zpci_aisb_iv);
392	if (aisb == -1UL)
393		goto out;
394	zdev->aisb = aisb;
395
396	/* Create adapter interrupt vector */
397	rc = -ENOMEM;
398	zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK);
399	if (!zdev->aibv)
400		goto out_si;
401
402	/* Wire up shortcut pointer */
403	zpci_aibv[aisb] = zdev->aibv;
404
405	/* Request MSI interrupts */
406	hwirq = 0;
407	for_each_pci_msi_entry(msi, pdev) {
408		rc = -EIO;
409		irq = irq_alloc_desc(0);	/* Alloc irq on node 0 */
410		if (irq < 0)
411			goto out_msi;
412		rc = irq_set_msi_desc(irq, msi);
413		if (rc)
414			goto out_msi;
415		irq_set_chip_and_handler(irq, &zpci_irq_chip,
416					 handle_simple_irq);
417		msg.data = hwirq;
418		msg.address_lo = zdev->msi_addr & 0xffffffff;
419		msg.address_hi = zdev->msi_addr >> 32;
420		pci_write_msi_msg(irq, &msg);
421		airq_iv_set_data(zdev->aibv, hwirq, irq);
422		hwirq++;
423	}
424
425	/* Enable adapter interrupts */
426	rc = zpci_set_airq(zdev);
427	if (rc)
428		goto out_msi;
429
430	return (msi_vecs == nvec) ? 0 : msi_vecs;
431
432out_msi:
433	for_each_pci_msi_entry(msi, pdev) {
434		if (hwirq-- == 0)
435			break;
436		irq_set_msi_desc(msi->irq, NULL);
437		irq_free_desc(msi->irq);
438		msi->msg.address_lo = 0;
439		msi->msg.address_hi = 0;
440		msi->msg.data = 0;
441		msi->irq = 0;
442	}
443	zpci_aibv[aisb] = NULL;
444	airq_iv_release(zdev->aibv);
445out_si:
446	airq_iv_free_bit(zpci_aisb_iv, aisb);
447out:
448	return rc;
449}
450
451void arch_teardown_msi_irqs(struct pci_dev *pdev)
452{
453	struct zpci_dev *zdev = to_zpci(pdev);
454	struct msi_desc *msi;
455	int rc;
456
457	/* Disable adapter interrupts */
458	rc = zpci_clear_airq(zdev);
459	if (rc)
460		return;
461
462	/* Release MSI interrupts */
463	for_each_pci_msi_entry(msi, pdev) {
464		if (msi->msi_attrib.is_msix)
465			__pci_msix_desc_mask_irq(msi, 1);
466		else
467			__pci_msi_desc_mask_irq(msi, 1, 1);
468		irq_set_msi_desc(msi->irq, NULL);
469		irq_free_desc(msi->irq);
470		msi->msg.address_lo = 0;
471		msi->msg.address_hi = 0;
472		msi->msg.data = 0;
473		msi->irq = 0;
474	}
475
476	zpci_aibv[zdev->aisb] = NULL;
477	airq_iv_release(zdev->aibv);
478	airq_iv_free_bit(zpci_aisb_iv, zdev->aisb);
479}
480
481static void zpci_map_resources(struct pci_dev *pdev)
482{
483	resource_size_t len;
484	int i;
485
486	for (i = 0; i < PCI_BAR_COUNT; i++) {
487		len = pci_resource_len(pdev, i);
488		if (!len)
489			continue;
490		pdev->resource[i].start =
491			(resource_size_t __force) pci_iomap(pdev, i, 0);
492		pdev->resource[i].end = pdev->resource[i].start + len - 1;
493	}
 
494}
495
496static void zpci_unmap_resources(struct pci_dev *pdev)
497{
 
498	resource_size_t len;
499	int i;
500
 
 
 
501	for (i = 0; i < PCI_BAR_COUNT; i++) {
502		len = pci_resource_len(pdev, i);
503		if (!len)
504			continue;
505		pci_iounmap(pdev, (void __iomem __force *)
506			    pdev->resource[i].start);
507	}
508}
509
510static int __init zpci_irq_init(void)
511{
512	int rc;
513
514	rc = register_adapter_interrupt(&zpci_airq);
515	if (rc)
516		goto out;
517	/* Set summary to 1 to be called every time for the ISC. */
518	*zpci_airq.lsi_ptr = 1;
519
520	rc = -ENOMEM;
521	zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC);
522	if (!zpci_aisb_iv)
523		goto out_airq;
524
525	zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
526	return 0;
527
528out_airq:
529	unregister_adapter_interrupt(&zpci_airq);
530out:
531	return rc;
532}
533
534static void zpci_irq_exit(void)
535{
536	airq_iv_release(zpci_aisb_iv);
537	unregister_adapter_interrupt(&zpci_airq);
538}
539
540static int zpci_alloc_iomap(struct zpci_dev *zdev)
541{
542	unsigned long entry;
543
544	spin_lock(&zpci_iomap_lock);
545	entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES);
546	if (entry == ZPCI_IOMAP_ENTRIES) {
547		spin_unlock(&zpci_iomap_lock);
548		return -ENOSPC;
549	}
550	set_bit(entry, zpci_iomap_bitmap);
551	spin_unlock(&zpci_iomap_lock);
552	return entry;
553}
554
555static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
556{
557	spin_lock(&zpci_iomap_lock);
558	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
559	clear_bit(entry, zpci_iomap_bitmap);
560	spin_unlock(&zpci_iomap_lock);
561}
562
563static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
564				    unsigned long size, unsigned long flags)
565{
566	struct resource *r;
567
568	r = kzalloc(sizeof(*r), GFP_KERNEL);
569	if (!r)
570		return NULL;
571
572	r->start = start;
573	r->end = r->start + size - 1;
574	r->flags = flags;
575	r->name = zdev->res_name;
576
577	if (request_resource(&iomem_resource, r)) {
578		kfree(r);
579		return NULL;
580	}
581	return r;
582}
583
584static int zpci_setup_bus_resources(struct zpci_dev *zdev,
585				    struct list_head *resources)
586{
587	unsigned long addr, size, flags;
588	struct resource *res;
589	int i, entry;
590
591	snprintf(zdev->res_name, sizeof(zdev->res_name),
592		 "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR);
593
594	for (i = 0; i < PCI_BAR_COUNT; i++) {
595		if (!zdev->bars[i].size)
596			continue;
597		entry = zpci_alloc_iomap(zdev);
598		if (entry < 0)
599			return entry;
600		zdev->bars[i].map_idx = entry;
601
602		/* only MMIO is supported */
603		flags = IORESOURCE_MEM;
604		if (zdev->bars[i].val & 8)
605			flags |= IORESOURCE_PREFETCH;
606		if (zdev->bars[i].val & 4)
607			flags |= IORESOURCE_MEM_64;
608
609		addr = ZPCI_ADDR(entry);
 
 
 
610		size = 1UL << zdev->bars[i].size;
611
612		res = __alloc_res(zdev, addr, size, flags);
613		if (!res) {
614			zpci_free_iomap(zdev, entry);
615			return -ENOMEM;
616		}
617		zdev->bars[i].res = res;
618		pci_add_resource(resources, res);
619	}
620
621	return 0;
622}
623
624static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
625{
626	int i;
627
628	for (i = 0; i < PCI_BAR_COUNT; i++) {
629		if (!zdev->bars[i].size || !zdev->bars[i].res)
630			continue;
631
632		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
633		release_resource(zdev->bars[i].res);
634		kfree(zdev->bars[i].res);
635	}
636}
637
638int pcibios_add_device(struct pci_dev *pdev)
639{
640	struct resource *res;
641	int i;
642
 
 
 
643	pdev->dev.groups = zpci_attr_groups;
644	pdev->dev.archdata.dma_ops = &s390_pci_dma_ops;
645	zpci_map_resources(pdev);
646
647	for (i = 0; i < PCI_BAR_COUNT; i++) {
648		res = &pdev->resource[i];
649		if (res->parent || !res->flags)
650			continue;
651		pci_claim_resource(pdev, i);
652	}
653
654	return 0;
655}
656
657void pcibios_release_device(struct pci_dev *pdev)
658{
659	zpci_unmap_resources(pdev);
660}
661
662int pcibios_enable_device(struct pci_dev *pdev, int mask)
663{
664	struct zpci_dev *zdev = to_zpci(pdev);
665
666	zpci_debug_init_device(zdev, dev_name(&pdev->dev));
667	zpci_fmb_enable_device(zdev);
668
669	return pci_enable_resources(pdev, mask);
670}
671
672void pcibios_disable_device(struct pci_dev *pdev)
673{
674	struct zpci_dev *zdev = to_zpci(pdev);
675
676	zpci_fmb_disable_device(zdev);
677	zpci_debug_exit_device(zdev);
678}
679
680#ifdef CONFIG_HIBERNATE_CALLBACKS
681static int zpci_restore(struct device *dev)
682{
683	struct pci_dev *pdev = to_pci_dev(dev);
684	struct zpci_dev *zdev = to_zpci(pdev);
685	int ret = 0;
686
687	if (zdev->state != ZPCI_FN_STATE_ONLINE)
688		goto out;
689
690	ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
691	if (ret)
692		goto out;
693
694	zpci_map_resources(pdev);
695	zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
696			   (u64) zdev->dma_table);
697
698out:
699	return ret;
700}
701
702static int zpci_freeze(struct device *dev)
703{
704	struct pci_dev *pdev = to_pci_dev(dev);
705	struct zpci_dev *zdev = to_zpci(pdev);
706
707	if (zdev->state != ZPCI_FN_STATE_ONLINE)
708		return 0;
709
710	zpci_unregister_ioat(zdev, 0);
711	zpci_unmap_resources(pdev);
712	return clp_disable_fh(zdev);
713}
714
715struct dev_pm_ops pcibios_pm_ops = {
716	.thaw_noirq = zpci_restore,
717	.freeze_noirq = zpci_freeze,
718	.restore_noirq = zpci_restore,
719	.poweroff_noirq = zpci_freeze,
720};
721#endif /* CONFIG_HIBERNATE_CALLBACKS */
722
723static int zpci_alloc_domain(struct zpci_dev *zdev)
724{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
725	spin_lock(&zpci_domain_lock);
726	zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
727	if (zdev->domain == ZPCI_NR_DEVICES) {
728		spin_unlock(&zpci_domain_lock);
729		return -ENOSPC;
730	}
731	set_bit(zdev->domain, zpci_domain);
732	spin_unlock(&zpci_domain_lock);
733	return 0;
734}
735
736static void zpci_free_domain(struct zpci_dev *zdev)
737{
 
 
 
738	spin_lock(&zpci_domain_lock);
739	clear_bit(zdev->domain, zpci_domain);
740	spin_unlock(&zpci_domain_lock);
741}
742
743void pcibios_remove_bus(struct pci_bus *bus)
744{
745	struct zpci_dev *zdev = get_zdev_by_bus(bus);
746
747	zpci_exit_slot(zdev);
748	zpci_cleanup_bus_resources(zdev);
 
749	zpci_free_domain(zdev);
750
751	spin_lock(&zpci_list_lock);
752	list_del(&zdev->entry);
753	spin_unlock(&zpci_list_lock);
754
 
755	kfree(zdev);
756}
757
758static int zpci_scan_bus(struct zpci_dev *zdev)
759{
760	LIST_HEAD(resources);
761	int ret;
762
763	ret = zpci_setup_bus_resources(zdev, &resources);
764	if (ret)
765		goto error;
766
767	zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops,
768				      zdev, &resources);
769	if (!zdev->bus) {
770		ret = -EIO;
771		goto error;
772	}
773	zdev->bus->max_bus_speed = zdev->max_bus_speed;
774	pci_bus_add_devices(zdev->bus);
775	return 0;
776
777error:
778	zpci_cleanup_bus_resources(zdev);
779	pci_free_resource_list(&resources);
780	return ret;
781}
782
783int zpci_enable_device(struct zpci_dev *zdev)
784{
785	int rc;
786
787	rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
788	if (rc)
789		goto out;
790
791	rc = zpci_dma_init_device(zdev);
792	if (rc)
793		goto out_dma;
794
795	zdev->state = ZPCI_FN_STATE_ONLINE;
796	return 0;
797
798out_dma:
799	clp_disable_fh(zdev);
800out:
801	return rc;
802}
803EXPORT_SYMBOL_GPL(zpci_enable_device);
804
805int zpci_disable_device(struct zpci_dev *zdev)
806{
807	zpci_dma_exit_device(zdev);
808	return clp_disable_fh(zdev);
809}
810EXPORT_SYMBOL_GPL(zpci_disable_device);
811
812int zpci_create_device(struct zpci_dev *zdev)
813{
814	int rc;
815
816	rc = zpci_alloc_domain(zdev);
817	if (rc)
818		goto out;
819
 
 
 
 
820	mutex_init(&zdev->lock);
821	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
822		rc = zpci_enable_device(zdev);
823		if (rc)
824			goto out_free;
825	}
826	rc = zpci_scan_bus(zdev);
827	if (rc)
828		goto out_disable;
829
830	spin_lock(&zpci_list_lock);
831	list_add_tail(&zdev->entry, &zpci_list);
832	spin_unlock(&zpci_list_lock);
833
834	zpci_init_slot(zdev);
835
836	return 0;
837
838out_disable:
839	if (zdev->state == ZPCI_FN_STATE_ONLINE)
840		zpci_disable_device(zdev);
 
 
841out_free:
842	zpci_free_domain(zdev);
843out:
844	return rc;
845}
846
847void zpci_stop_device(struct zpci_dev *zdev)
848{
849	zpci_dma_exit_device(zdev);
850	/*
851	 * Note: SCLP disables fh via set-pci-fn so don't
852	 * do that here.
853	 */
854}
855EXPORT_SYMBOL_GPL(zpci_stop_device);
856
857static inline int barsize(u8 size)
 
858{
859	return (size) ? (1 << size) >> 10 : 0;
 
 
860}
 
861
862static int zpci_mem_init(void)
863{
864	BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) ||
865		     __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb));
866
867	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
868					   __alignof__(struct zpci_fmb), 0, NULL);
869	if (!zdev_fmb_cache)
870		goto error_fmb;
871
872	zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES,
873				   sizeof(*zpci_iomap_start), GFP_KERNEL);
874	if (!zpci_iomap_start)
875		goto error_iomap;
876
877	zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES),
878				    sizeof(*zpci_iomap_bitmap), GFP_KERNEL);
879	if (!zpci_iomap_bitmap)
880		goto error_iomap_bitmap;
881
882	return 0;
883error_iomap_bitmap:
884	kfree(zpci_iomap_start);
885error_iomap:
886	kmem_cache_destroy(zdev_fmb_cache);
887error_fmb:
888	return -ENOMEM;
889}
890
891static void zpci_mem_exit(void)
892{
893	kfree(zpci_iomap_bitmap);
894	kfree(zpci_iomap_start);
895	kmem_cache_destroy(zdev_fmb_cache);
896}
897
898static unsigned int s390_pci_probe = 1;
 
 
899static unsigned int s390_pci_initialized;
900
901char * __init pcibios_setup(char *str)
902{
903	if (!strcmp(str, "off")) {
904		s390_pci_probe = 0;
905		return NULL;
906	}
 
 
 
 
 
 
 
 
907	return str;
908}
909
910bool zpci_is_enabled(void)
911{
912	return s390_pci_initialized;
913}
914
915static int __init pci_base_init(void)
916{
917	int rc;
918
919	if (!s390_pci_probe)
920		return 0;
921
922	if (!test_facility(69) || !test_facility(71) || !test_facility(72))
923		return 0;
 
 
 
 
 
924
925	rc = zpci_debug_init();
926	if (rc)
927		goto out;
928
929	rc = zpci_mem_init();
930	if (rc)
931		goto out_mem;
932
933	rc = zpci_irq_init();
934	if (rc)
935		goto out_irq;
936
937	rc = zpci_dma_init();
938	if (rc)
939		goto out_dma;
940
941	rc = clp_scan_pci_devices();
942	if (rc)
943		goto out_find;
944
945	s390_pci_initialized = 1;
946	return 0;
947
948out_find:
949	zpci_dma_exit();
950out_dma:
951	zpci_irq_exit();
952out_irq:
953	zpci_mem_exit();
954out_mem:
955	zpci_debug_exit();
956out:
957	return rc;
958}
959subsys_initcall_sync(pci_base_init);
960
961void zpci_rescan(void)
962{
963	if (zpci_is_enabled())
964		clp_rescan_pci_devices_simple();
965}